From e94f2580cd782b1e18f163a92010ad87c2f293ea Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Thu, 19 Sep 2019 16:43:10 -0700 Subject: [PATCH 001/153] Add pipxrc in each venv dir to hold pipxrc info This commit adds only the key 'package_or_url' to the pipxrc JSON file dict. It is written in 'install' and read in 'upgrade-all' and 'reinstall-all'. --- src/pipx/commands.py | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/src/pipx/commands.py b/src/pipx/commands.py index faab901ce8..4bd75d661f 100644 --- a/src/pipx/commands.py +++ b/src/pipx/commands.py @@ -2,6 +2,7 @@ import datetime import hashlib +import json import logging import multiprocessing import shlex @@ -255,12 +256,13 @@ def upgrade_all( for venv_dir in venv_container.iter_venv_dirs(): num_packages += 1 package = venv_dir.name + pipxrc_info = _read_pipxrc(venv_dir) if package in skip: continue if package == "pipx": package_or_url = PIPX_PACKAGE_NAME else: - package_or_url = package + package_or_url = pipxrc_info.get('package_or_url', package) try: packages_upgraded += upgrade( venv_dir, @@ -319,6 +321,7 @@ def install( venv.remove_venv() raise PipxError(f"Could not find package {package}. Is the name correct?") + _create_pipxrc(venv_dir, package_or_url) _run_post_install_actions( venv, package, local_bin_dir, venv_dir, include_dependencies, force=force ) @@ -328,6 +331,23 @@ def install( raise +def _create_pipxrc(venv_dir: Path, package_or_url: str): + # TODO 20190919: raise exception on failure? + with open(venv_dir / 'pipxrc', 'w') as pipxrc_fh: + json.dump({'package_or_url': package_or_url}, pipxrc_fh) + + +def _read_pipxrc(venv_dir: Path): + try: + with open(venv_dir / 'pipxrc', 'r') as pipxrc_fh: + pipxrc_info = json.load(pipxrc_fh) + except IOError: + # return empty dict if no pipxrc file or unreadable + return {} + + return pipxrc_info + + def _run_post_install_actions( venv: Venv, package: str, @@ -492,11 +512,12 @@ def reinstall_all( ): for venv_dir in venv_container.iter_venv_dirs(): package = venv_dir.name + pipxrc_info = _read_pipxrc(venv_dir) if package in skip: continue uninstall(venv_dir, package, local_bin_dir, verbose) - package_or_url = package + package_or_url = pipxrc_info.get('package_or_url', package) install( venv_dir, package, From 0a549502398b5e160ff36025a5b673589f5a43be Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Thu, 19 Sep 2019 17:04:37 -0700 Subject: [PATCH 002/153] Add return value type to _read_pipxrc. --- src/pipx/commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pipx/commands.py b/src/pipx/commands.py index 4bd75d661f..9bc2989146 100644 --- a/src/pipx/commands.py +++ b/src/pipx/commands.py @@ -337,7 +337,7 @@ def _create_pipxrc(venv_dir: Path, package_or_url: str): json.dump({'package_or_url': package_or_url}, pipxrc_fh) -def _read_pipxrc(venv_dir: Path): +def _read_pipxrc(venv_dir: Path) -> dict: try: with open(venv_dir / 'pipxrc', 'r') as pipxrc_fh: pipxrc_info = json.load(pipxrc_fh) From 88aed02d4b0714a12a8c4e8aa23999c73ada55ca Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Thu, 19 Sep 2019 17:10:33 -0700 Subject: [PATCH 003/153] Rename _create_pipxrc -> _write_pipxrc, change arg. Change argument to be more generic pipxrc_info dict instead of specific 'package_or_url' key. --- src/pipx/commands.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/pipx/commands.py b/src/pipx/commands.py index 9bc2989146..f4d96b0e1c 100644 --- a/src/pipx/commands.py +++ b/src/pipx/commands.py @@ -321,7 +321,8 @@ def install( venv.remove_venv() raise PipxError(f"Could not find package {package}. Is the name correct?") - _create_pipxrc(venv_dir, package_or_url) + pipxrc_info = {'package_or_url': package_or_url} + _write_pipxrc(venv_dir, pipxrc_info) _run_post_install_actions( venv, package, local_bin_dir, venv_dir, include_dependencies, force=force ) @@ -331,7 +332,7 @@ def install( raise -def _create_pipxrc(venv_dir: Path, package_or_url: str): +def _write_pipxrc(venv_dir: Path, pipxrc_info: dict): # TODO 20190919: raise exception on failure? with open(venv_dir / 'pipxrc', 'w') as pipxrc_fh: json.dump({'package_or_url': package_or_url}, pipxrc_fh) From 7876b86d638fb83c54db3deaaba653b5b79acac8 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Fri, 20 Sep 2019 16:12:41 -0700 Subject: [PATCH 004/153] New home for all pipxrc-related functions. --- pipx/pipxrc.py | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 pipx/pipxrc.py diff --git a/pipx/pipxrc.py b/pipx/pipxrc.py new file mode 100644 index 0000000000..336951589d --- /dev/null +++ b/pipx/pipxrc.py @@ -0,0 +1,42 @@ +import json +from pathlib import Path + +from pipx.Venv import PipxVenvMetadata + + +class JsonPathEncoder(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, Path): + return {'__type__':'Path', '__Path__':str(obj)} + return super().default(self, obj) + + +def _json_decoder_object_hook(json_dict): + if json_dict.get('__type__', None) == 'Path' and '__Path__' in json_dict: + return Path(json_dict['__Path__']) + return json_dict + + +def write_pipxrc(venv_dir: Path, pipxrc_info: dict): + if 'venv_metadata' in pipxrc_info: + # serialize PipxVenvMetadata + pipxrc_info['venv_metadata'] = dict(pipxrc_info['venv_metadata']._asdict()) + # TODO 20190919: raise exception on failure? + with open(venv_dir / 'pipxrc', 'w') as pipxrc_fh: + json.dump(pipxrc_info, pipxrc_fh, cls=JsonPathEncoder) + + +def read_pipxrc(venv_dir: Path) -> dict: + try: + with open(venv_dir / 'pipxrc', 'r') as pipxrc_fh: + pipxrc_info = json.load(pipxrc_fh, object_hook=_json_decoder_object_hook) + except IOError: + # return empty dict if no pipxrc file or unreadable + return {} + # convert venv_data back to NamedTuple + if 'venv_metadata' in pipxrc_info: + pipxrc_info['venv_metadata'] = PipxVenvMetadata(**pipxrc_info['venv_metadata']) + + return pipxrc_info + + From eabb88f7f9e18525b05b9458e33c0b55cfbc33d7 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Fri, 20 Sep 2019 16:15:30 -0700 Subject: [PATCH 005/153] Use pipxrc functions from pipxrc.py. --- src/pipx/commands.py | 34 +++++++++++----------------------- 1 file changed, 11 insertions(+), 23 deletions(-) diff --git a/src/pipx/commands.py b/src/pipx/commands.py index f4d96b0e1c..07b3ea163a 100644 --- a/src/pipx/commands.py +++ b/src/pipx/commands.py @@ -2,7 +2,6 @@ import datetime import hashlib -import json import logging import multiprocessing import shlex @@ -31,6 +30,7 @@ rmdir, run_pypackage_bin, ) +from pipx.pipxrc import read_pipxrc, write_pipxrc from pipx.venv import Venv, VenvContainer @@ -256,7 +256,7 @@ def upgrade_all( for venv_dir in venv_container.iter_venv_dirs(): num_packages += 1 package = venv_dir.name - pipxrc_info = _read_pipxrc(venv_dir) + pipxrc_info = read_pipxrc(venv_dir) if package in skip: continue if package == "pipx": @@ -316,13 +316,18 @@ def install( try: venv.create_venv(venv_args, pip_args) venv.install_package(package_or_url, pip_args) + venv_metadata = venv.get_venv_metadata_for_package(package) - if venv.get_venv_metadata_for_package(package).package_version is None: + if venv_metadata.package_version is None: venv.remove_venv() raise PipxError(f"Could not find package {package}. Is the name correct?") - pipxrc_info = {'package_or_url': package_or_url} - _write_pipxrc(venv_dir, pipxrc_info) + pipxrc_info = { + 'package_or_url': package_or_url, + 'venv_metadata': venv_metadata, + 'injected_packages': {} + } + write_pipxrc(venv_dir, pipxrc_info) _run_post_install_actions( venv, package, local_bin_dir, venv_dir, include_dependencies, force=force ) @@ -332,23 +337,6 @@ def install( raise -def _write_pipxrc(venv_dir: Path, pipxrc_info: dict): - # TODO 20190919: raise exception on failure? - with open(venv_dir / 'pipxrc', 'w') as pipxrc_fh: - json.dump({'package_or_url': package_or_url}, pipxrc_fh) - - -def _read_pipxrc(venv_dir: Path) -> dict: - try: - with open(venv_dir / 'pipxrc', 'r') as pipxrc_fh: - pipxrc_info = json.load(pipxrc_fh) - except IOError: - # return empty dict if no pipxrc file or unreadable - return {} - - return pipxrc_info - - def _run_post_install_actions( venv: Venv, package: str, @@ -513,9 +501,9 @@ def reinstall_all( ): for venv_dir in venv_container.iter_venv_dirs(): package = venv_dir.name - pipxrc_info = _read_pipxrc(venv_dir) if package in skip: continue + pipxrc_info = read_pipxrc(venv_dir) uninstall(venv_dir, package, local_bin_dir, verbose) package_or_url = pipxrc_info.get('package_or_url', package) From 7c283efe2b6a9eab77a4a2ca27e93df54b843f11 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Fri, 20 Sep 2019 16:16:30 -0700 Subject: [PATCH 006/153] Get cached venv metadata from pipxrc. Fixes #146 by allowing reinstall-all() to call uninstall() without breaking. --- src/pipx/commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pipx/commands.py b/src/pipx/commands.py index 07b3ea163a..145bb70c22 100644 --- a/src/pipx/commands.py +++ b/src/pipx/commands.py @@ -462,7 +462,7 @@ def uninstall(venv_dir: Path, package: str, local_bin_dir: Path, verbose: bool): venv = Venv(venv_dir, verbose=verbose) - metadata = venv.get_venv_metadata_for_package(package) + metadata = read_pipxrc(venv_dir)['venv_metadata'] app_paths = metadata.app_paths for dep_paths in metadata.app_paths_of_dependencies.values(): app_paths += dep_paths From 7625e6b6c1da3e93970097ee78d479cfa453b320 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Fri, 20 Sep 2019 16:18:41 -0700 Subject: [PATCH 007/153] Store new injected package in pipxrc. This is to enable reinstall-all to re-inject packages at a later time. --- src/pipx/commands.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/pipx/commands.py b/src/pipx/commands.py index 145bb70c22..edcd9c937e 100644 --- a/src/pipx/commands.py +++ b/src/pipx/commands.py @@ -445,6 +445,15 @@ def inject( include_dependencies, force=force, ) + pipxrc_info = read_pipxrc(venv_dir) + pipxrc_info['injected_packages'][package] = { + 'pip_args': pip_args, + 'verbose': verbose, + 'include_apps': include_apps, + 'include_dependencies': include_dependencies, + 'force': force, + } + write_pipxrc(venv_dir, pipxrc_info) print(f" injected package {bold(package)} into venv {bold(venv_dir.name)}") print(f"done! {stars}", file=sys.stderr) From 48b3e13255788f1b763ca4c05ff57c4e7b5d213e Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Fri, 20 Sep 2019 16:30:36 -0700 Subject: [PATCH 008/153] Fall back on venv.get_venv_metadata if no cached. If pipxrc does not exist or it does not contain the key 'venv_metadata' fall back on using venv.get_venv_metadata. --- src/pipx/commands.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/pipx/commands.py b/src/pipx/commands.py index edcd9c937e..fa5a2dcb1e 100644 --- a/src/pipx/commands.py +++ b/src/pipx/commands.py @@ -471,7 +471,9 @@ def uninstall(venv_dir: Path, package: str, local_bin_dir: Path, verbose: bool): venv = Venv(venv_dir, verbose=verbose) - metadata = read_pipxrc(venv_dir)['venv_metadata'] + metadata = read_pipxrc(venv_dir).get( + 'venv_metadata', venv.get_venv_metadata_for_package(package) + ) app_paths = metadata.app_paths for dep_paths in metadata.app_paths_of_dependencies.values(): app_paths += dep_paths From 8045dde125fd1a8efbe67d688c48ac03c54d7c3f Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Fri, 20 Sep 2019 18:36:50 -0700 Subject: [PATCH 009/153] Rename JsonPathEncoder -> JsonEncoderPipx, ser changes. Update serialization/deserialization code to standardize special objects to be all converted to {'__type__':, '____':} format. --- pipx/pipxrc.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/pipx/pipxrc.py b/pipx/pipxrc.py index 336951589d..bab41ae6b6 100644 --- a/pipx/pipxrc.py +++ b/pipx/pipxrc.py @@ -4,8 +4,9 @@ from pipx.Venv import PipxVenvMetadata -class JsonPathEncoder(json.JSONEncoder): +class JsonEncoderPipx(json.JSONEncoder): def default(self, obj): + # only handles what json.JSONEncoder doesn't understand by default if isinstance(obj, Path): return {'__type__':'Path', '__Path__':str(obj)} return super().default(self, obj) @@ -14,16 +15,23 @@ def default(self, obj): def _json_decoder_object_hook(json_dict): if json_dict.get('__type__', None) == 'Path' and '__Path__' in json_dict: return Path(json_dict['__Path__']) + if json_dict.get('__type__', None) == 'PipxVenvMetadata' and '__PipxVenvMetadata__' in json_dict: + return PipxVenvMetadata(**json_dict['__PipxVenvMetadata__']) return json_dict def write_pipxrc(venv_dir: Path, pipxrc_info: dict): - if 'venv_metadata' in pipxrc_info: - # serialize PipxVenvMetadata - pipxrc_info['venv_metadata'] = dict(pipxrc_info['venv_metadata']._asdict()) + # json thinks PipxVenvMetadata is just another tuple, so we override here + # (JSONEncoder override is harder and messier.) + for key in pipxrc_info: + if isinstance(pipxrc_info[key], PipxVenvMetadata): + pipxrc_info[key] = { + '__type__':'PipxVenvMetadata', + '__PipxVenvMetadata__': dict(pipxrc_info[key]._asdict()) + } # TODO 20190919: raise exception on failure? with open(venv_dir / 'pipxrc', 'w') as pipxrc_fh: - json.dump(pipxrc_info, pipxrc_fh, cls=JsonPathEncoder) + json.dump(pipxrc_info, pipxrc_fh, cls=JsonEncoderPipx) def read_pipxrc(venv_dir: Path) -> dict: @@ -33,10 +41,5 @@ def read_pipxrc(venv_dir: Path) -> dict: except IOError: # return empty dict if no pipxrc file or unreadable return {} - # convert venv_data back to NamedTuple - if 'venv_metadata' in pipxrc_info: - pipxrc_info['venv_metadata'] = PipxVenvMetadata(**pipxrc_info['venv_metadata']) return pipxrc_info - - From dbba307c4a600e7a9ee572d0ec3aa71986703869 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Fri, 20 Sep 2019 18:39:54 -0700 Subject: [PATCH 010/153] Make pipxrc pretty-printed. --- pipx/pipxrc.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pipx/pipxrc.py b/pipx/pipxrc.py index bab41ae6b6..38e2db14ce 100644 --- a/pipx/pipxrc.py +++ b/pipx/pipxrc.py @@ -31,7 +31,11 @@ def write_pipxrc(venv_dir: Path, pipxrc_info: dict): } # TODO 20190919: raise exception on failure? with open(venv_dir / 'pipxrc', 'w') as pipxrc_fh: - json.dump(pipxrc_info, pipxrc_fh, cls=JsonEncoderPipx) + json.dump( + pipxrc_info, pipxrc_fh, + indent=4, sort_keys=True, + cls=JsonEncoderPipx + ) def read_pipxrc(venv_dir: Path) -> dict: From 7606834b16fbd2a733093a9d25d1cd059c9cf57c Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Fri, 20 Sep 2019 19:25:50 -0700 Subject: [PATCH 011/153] Make reinstall-all re-inject injected packages. --- src/pipx/commands.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/pipx/commands.py b/src/pipx/commands.py index fa5a2dcb1e..0cbfb61cd8 100644 --- a/src/pipx/commands.py +++ b/src/pipx/commands.py @@ -530,6 +530,17 @@ def reinstall_all( force=True, include_dependencies=include_dependencies, ) + for injected_package in pipxrc_info.get('injected_packages', {}): + pkg_info = pipxrc_info['injected_packages'][injected_package] + inject( + venv_dir, + injected_package, + pkg_info['pip_args'], + verbose = pkg_info['verbose'], + include_apps = pkg_info['include_apps'], + include_dependencies = pkg_info['include_dependencies'], + force = pkg_info['force'], + ) def _expose_apps_globally( From 5f0854ca9e0221b3eac7b355cefc15aa08ab9f47 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Fri, 20 Sep 2019 19:30:57 -0700 Subject: [PATCH 012/153] Add TODO comment. --- src/pipx/commands.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/pipx/commands.py b/src/pipx/commands.py index 0cbfb61cd8..cc4d24267b 100644 --- a/src/pipx/commands.py +++ b/src/pipx/commands.py @@ -328,6 +328,8 @@ def install( 'injected_packages': {} } write_pipxrc(venv_dir, pipxrc_info) + # TODO 20190920: Does _run_post_install_actions affect venv metadata? + # if so, we need to get metadata for pipxrc after this _run_post_install_actions( venv, package, local_bin_dir, venv_dir, include_dependencies, force=force ) From bc36f45c2d182686acbc90addf0c7f6b6c913d91 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Fri, 20 Sep 2019 21:34:02 -0700 Subject: [PATCH 013/153] Fixed lint according to black. --- pipx/pipxrc.py | 29 ++++++++++++------------ src/pipx/commands.py | 54 ++++++++++++++++++++++++-------------------- 2 files changed, 43 insertions(+), 40 deletions(-) diff --git a/pipx/pipxrc.py b/pipx/pipxrc.py index 38e2db14ce..b94fb9e1ed 100644 --- a/pipx/pipxrc.py +++ b/pipx/pipxrc.py @@ -8,15 +8,18 @@ class JsonEncoderPipx(json.JSONEncoder): def default(self, obj): # only handles what json.JSONEncoder doesn't understand by default if isinstance(obj, Path): - return {'__type__':'Path', '__Path__':str(obj)} + return {"__type__": "Path", "__Path__": str(obj)} return super().default(self, obj) def _json_decoder_object_hook(json_dict): - if json_dict.get('__type__', None) == 'Path' and '__Path__' in json_dict: - return Path(json_dict['__Path__']) - if json_dict.get('__type__', None) == 'PipxVenvMetadata' and '__PipxVenvMetadata__' in json_dict: - return PipxVenvMetadata(**json_dict['__PipxVenvMetadata__']) + if json_dict.get("__type__", None) == "Path" and "__Path__" in json_dict: + return Path(json_dict["__Path__"]) + if ( + json_dict.get("__type__", None) == "PipxVenvMetadata" + and "__PipxVenvMetadata__" in json_dict + ): + return PipxVenvMetadata(**json_dict["__PipxVenvMetadata__"]) return json_dict @@ -26,21 +29,17 @@ def write_pipxrc(venv_dir: Path, pipxrc_info: dict): for key in pipxrc_info: if isinstance(pipxrc_info[key], PipxVenvMetadata): pipxrc_info[key] = { - '__type__':'PipxVenvMetadata', - '__PipxVenvMetadata__': dict(pipxrc_info[key]._asdict()) - } + "__type__": "PipxVenvMetadata", + "__PipxVenvMetadata__": dict(pipxrc_info[key]._asdict()), + } # TODO 20190919: raise exception on failure? - with open(venv_dir / 'pipxrc', 'w') as pipxrc_fh: - json.dump( - pipxrc_info, pipxrc_fh, - indent=4, sort_keys=True, - cls=JsonEncoderPipx - ) + with open(venv_dir / "pipxrc", "w") as pipxrc_fh: + json.dump(pipxrc_info, pipxrc_fh, indent=4, sort_keys=True, cls=JsonEncoderPipx) def read_pipxrc(venv_dir: Path) -> dict: try: - with open(venv_dir / 'pipxrc', 'r') as pipxrc_fh: + with open(venv_dir / "pipxrc", "r") as pipxrc_fh: pipxrc_info = json.load(pipxrc_fh, object_hook=_json_decoder_object_hook) except IOError: # return empty dict if no pipxrc file or unreadable diff --git a/src/pipx/commands.py b/src/pipx/commands.py index cc4d24267b..8862a0fcea 100644 --- a/src/pipx/commands.py +++ b/src/pipx/commands.py @@ -210,6 +210,10 @@ def upgrade( old_version = venv.get_venv_metadata_for_package(package).package_version + # if default package_or_url, check pipxrc for better url + if package_or_url == package: + package_or_url = read_pipxrc(venv_dir).get("package_or_url", package) + # Upgrade shared libraries (pip, setuptools and wheel) venv.upgrade_packaging_libraries(pip_args) @@ -262,7 +266,7 @@ def upgrade_all( if package == "pipx": package_or_url = PIPX_PACKAGE_NAME else: - package_or_url = pipxrc_info.get('package_or_url', package) + package_or_url = pipxrc_info.get("package_or_url", package) try: packages_upgraded += upgrade( venv_dir, @@ -323,10 +327,10 @@ def install( raise PipxError(f"Could not find package {package}. Is the name correct?") pipxrc_info = { - 'package_or_url': package_or_url, - 'venv_metadata': venv_metadata, - 'injected_packages': {} - } + "package_or_url": package_or_url, + "venv_metadata": venv_metadata, + "injected_packages": {}, + } write_pipxrc(venv_dir, pipxrc_info) # TODO 20190920: Does _run_post_install_actions affect venv metadata? # if so, we need to get metadata for pipxrc after this @@ -448,13 +452,13 @@ def inject( force=force, ) pipxrc_info = read_pipxrc(venv_dir) - pipxrc_info['injected_packages'][package] = { - 'pip_args': pip_args, - 'verbose': verbose, - 'include_apps': include_apps, - 'include_dependencies': include_dependencies, - 'force': force, - } + pipxrc_info["injected_packages"][package] = { + "pip_args": pip_args, + "verbose": verbose, + "include_apps": include_apps, + "include_dependencies": include_dependencies, + "force": force, + } write_pipxrc(venv_dir, pipxrc_info) print(f" injected package {bold(package)} into venv {bold(venv_dir.name)}") @@ -474,8 +478,8 @@ def uninstall(venv_dir: Path, package: str, local_bin_dir: Path, verbose: bool): venv = Venv(venv_dir, verbose=verbose) metadata = read_pipxrc(venv_dir).get( - 'venv_metadata', venv.get_venv_metadata_for_package(package) - ) + "venv_metadata", venv.get_venv_metadata_for_package(package) + ) app_paths = metadata.app_paths for dep_paths in metadata.app_paths_of_dependencies.values(): app_paths += dep_paths @@ -519,7 +523,7 @@ def reinstall_all( pipxrc_info = read_pipxrc(venv_dir) uninstall(venv_dir, package, local_bin_dir, verbose) - package_or_url = pipxrc_info.get('package_or_url', package) + package_or_url = pipxrc_info.get("package_or_url", package) install( venv_dir, package, @@ -532,17 +536,17 @@ def reinstall_all( force=True, include_dependencies=include_dependencies, ) - for injected_package in pipxrc_info.get('injected_packages', {}): - pkg_info = pipxrc_info['injected_packages'][injected_package] + for injected_package in pipxrc_info.get("injected_packages", {}): + pkg_info = pipxrc_info["injected_packages"][injected_package] inject( - venv_dir, - injected_package, - pkg_info['pip_args'], - verbose = pkg_info['verbose'], - include_apps = pkg_info['include_apps'], - include_dependencies = pkg_info['include_dependencies'], - force = pkg_info['force'], - ) + venv_dir, + injected_package, + pkg_info["pip_args"], + verbose=pkg_info["verbose"], + include_apps=pkg_info["include_apps"], + include_dependencies=pkg_info["include_dependencies"], + force=pkg_info["force"], + ) def _expose_apps_globally( From f23483c3b77e597d3e93b6468cc267503fa1ec30 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Mon, 23 Sep 2019 21:21:11 -0700 Subject: [PATCH 014/153] Move write of pipxrc to end of install. Especially after _run_post_install_actions(). --- src/pipx/commands.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/pipx/commands.py b/src/pipx/commands.py index 8862a0fcea..0a8b8423e0 100644 --- a/src/pipx/commands.py +++ b/src/pipx/commands.py @@ -320,18 +320,12 @@ def install( try: venv.create_venv(venv_args, pip_args) venv.install_package(package_or_url, pip_args) - venv_metadata = venv.get_venv_metadata_for_package(package) + venv.get_venv_metadata_for_package(package) - if venv_metadata.package_version is None: + if venv.get_venv_metadata_for_package(package).package_version is None: venv.remove_venv() raise PipxError(f"Could not find package {package}. Is the name correct?") - pipxrc_info = { - "package_or_url": package_or_url, - "venv_metadata": venv_metadata, - "injected_packages": {}, - } - write_pipxrc(venv_dir, pipxrc_info) # TODO 20190920: Does _run_post_install_actions affect venv metadata? # if so, we need to get metadata for pipxrc after this _run_post_install_actions( @@ -342,6 +336,14 @@ def install( venv.remove_venv() raise + # if all is well, write out pipxrc file + pipxrc_info = { + "package_or_url": package_or_url, + "venv_metadata": venv.get_venv_metadata_for_package(package), + "injected_packages": {}, + } + write_pipxrc(venv_dir, pipxrc_info) + def _run_post_install_actions( venv: Venv, From 00dcae9ed4d01439bee2d6007f3e10a3f6eee814 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Mon, 23 Sep 2019 21:47:23 -0700 Subject: [PATCH 015/153] Updates to install pipxrc behavior. Add pip_args, venv_args, and include_dependencies options to pipxrc so reinstall-all and upgrade-all can know package-specific options. Add notes that reinstall-all should be using pipxrc info, not global for pip_args, venv_args, include_dependencies. Add TODO that we need to get absolute path for a package url that is a local path. --- src/pipx/commands.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/pipx/commands.py b/src/pipx/commands.py index 0a8b8423e0..bcdb0e57d0 100644 --- a/src/pipx/commands.py +++ b/src/pipx/commands.py @@ -337,8 +337,15 @@ def install( raise # if all is well, write out pipxrc file + # TODO 20190923: if package_or_url is a local path, we need to make it + # an absolute path pipxrc_info = { "package_or_url": package_or_url, + "install": { + "pip_args": pip_args, + "venv_args": venv_args, + "include_dependencies": include_dependencies, + }, "venv_metadata": venv.get_venv_metadata_for_package(package), "injected_packages": {}, } @@ -532,11 +539,11 @@ def reinstall_all( package_or_url, local_bin_dir, python, - pip_args, - venv_args, + pip_args, # TODO 20190923: use pipxrc_info + venv_args, # TODO 20190923: use pipxrc_info verbose, force=True, - include_dependencies=include_dependencies, + include_dependencies=include_dependencies, # TODO 20190923: use pipxrc_info ) for injected_package in pipxrc_info.get("injected_packages", {}): pkg_info = pipxrc_info["injected_packages"][injected_package] From 8816177a5838517939b0dd968c257c86efe69ec9 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Mon, 23 Sep 2019 21:53:15 -0700 Subject: [PATCH 016/153] Add notes to upgrade_all() to use pipxrc for args. Instead of using global pip_args, include_dependencies, we should use the pipxrc-stored options for these on a per-package basis. --- src/pipx/commands.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pipx/commands.py b/src/pipx/commands.py index bcdb0e57d0..75313d042a 100644 --- a/src/pipx/commands.py +++ b/src/pipx/commands.py @@ -272,10 +272,10 @@ def upgrade_all( venv_dir, package, package_or_url, - pip_args, + pip_args, # TODO 20190923: use pipxrc_info verbose, upgrading_all=True, - include_dependencies=include_dependencies, + include_dependencies=include_dependencies, # TODO 20190923: use pipxrc_info force=force, ) except Exception: From ac192dd9a8ebdfe21dc2bc60ec4bc4b336b37b16 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Mon, 23 Sep 2019 21:55:24 -0700 Subject: [PATCH 017/153] Lint fixes for black. --- src/pipx/commands.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/pipx/commands.py b/src/pipx/commands.py index 75313d042a..b25db182bf 100644 --- a/src/pipx/commands.py +++ b/src/pipx/commands.py @@ -272,10 +272,10 @@ def upgrade_all( venv_dir, package, package_or_url, - pip_args, # TODO 20190923: use pipxrc_info + pip_args, # TODO 20190923: use pipxrc_info verbose, upgrading_all=True, - include_dependencies=include_dependencies, # TODO 20190923: use pipxrc_info + include_dependencies=include_dependencies, # TODO 20190923: use pipxrc_info force=force, ) except Exception: @@ -539,11 +539,11 @@ def reinstall_all( package_or_url, local_bin_dir, python, - pip_args, # TODO 20190923: use pipxrc_info - venv_args, # TODO 20190923: use pipxrc_info + pip_args, # TODO 20190923: use pipxrc_info + venv_args, # TODO 20190923: use pipxrc_info verbose, force=True, - include_dependencies=include_dependencies, # TODO 20190923: use pipxrc_info + include_dependencies=include_dependencies, # TODO 20190923: use pipxrc_info ) for injected_package in pipxrc_info.get("injected_packages", {}): pkg_info = pipxrc_info["injected_packages"][injected_package] From d66f56d0eadc3dcf32a9b23acb46dbc41a217cf4 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Mon, 23 Sep 2019 22:58:09 -0700 Subject: [PATCH 018/153] Remove obsolete TODO. --- src/pipx/commands.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/pipx/commands.py b/src/pipx/commands.py index b25db182bf..4a512e4d9d 100644 --- a/src/pipx/commands.py +++ b/src/pipx/commands.py @@ -326,8 +326,6 @@ def install( venv.remove_venv() raise PipxError(f"Could not find package {package}. Is the name correct?") - # TODO 20190920: Does _run_post_install_actions affect venv metadata? - # if so, we need to get metadata for pipxrc after this _run_post_install_actions( venv, package, local_bin_dir, venv_dir, include_dependencies, force=force ) From fbe6dec4e4d200163061f2a9ff3152a339487f7e Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Mon, 23 Sep 2019 23:26:15 -0700 Subject: [PATCH 019/153] Make reinstall-all use each PKG original options. Remove many reinstall-all options, and instead use remembered install options via pipxrc from the first install of each PACKAGE. Change the switches and help text of reinstall-all to reflect this change. --- src/pipx/commands.py | 9 +++------ src/pipx/main.py | 17 +++-------------- 2 files changed, 6 insertions(+), 20 deletions(-) diff --git a/src/pipx/commands.py b/src/pipx/commands.py index 4a512e4d9d..0f82b12b61 100644 --- a/src/pipx/commands.py +++ b/src/pipx/commands.py @@ -516,10 +516,7 @@ def reinstall_all( venv_container: VenvContainer, local_bin_dir: Path, python: str, - pip_args: List[str], - venv_args: List[str], verbose: bool, - include_dependencies: bool, *, skip: List[str], ): @@ -537,11 +534,11 @@ def reinstall_all( package_or_url, local_bin_dir, python, - pip_args, # TODO 20190923: use pipxrc_info - venv_args, # TODO 20190923: use pipxrc_info + pipxrc_info["install"]["pip_args"], + pipxrc_info["install"]["venv_args"], verbose, force=True, - include_dependencies=include_dependencies, # TODO 20190923: use pipxrc_info + include_dependencies=pipxrc_info["install"]["include_dependencies"], ) for injected_package in pipxrc_info.get("injected_packages", {}): pkg_info = pipxrc_info["injected_packages"][injected_package] diff --git a/src/pipx/main.py b/src/pipx/main.py index 21898a45ef..bd1992f4c2 100644 --- a/src/pipx/main.py +++ b/src/pipx/main.py @@ -215,14 +215,7 @@ def run_pipx_command(args): # noqa: C901 ) elif args.command == "reinstall-all": return commands.reinstall_all( - venv_container, - constants.LOCAL_BIN_DIR, - args.python, - pip_args, - venv_args, - verbose, - args.include_deps, - skip=args.skip, + venv_container, constants.LOCAL_BIN_DIR, args.python, verbose, skip=args.skip ) elif args.command == "runpip": if not venv_dir: @@ -397,19 +390,15 @@ def _add_reinstall_all(subparsers): """ Reinstalls all packages using a different version of Python. - Packages are uninstalled, then installed with pipx install PACKAGE. + Packages are uninstalled, then installed with pipx install PACKAGE + with the same options used in the original install of PACKAGE. This is useful if you upgraded to a new version of Python and want all your packages to use the latest as well. - If you originally installed a package from a source other than PyPI, - this command may behave in unexpected ways since it will reinstall from PyPI. - """ ), ) p.add_argument("python") - add_include_dependencies(p) - add_pip_venv_args(p) p.add_argument("--skip", nargs="+", default=[], help="skip these packages") p.add_argument("--verbose", action="store_true") From ba55fc522b302253f0322df6a761a80af8cf1fff Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Mon, 23 Sep 2019 23:35:31 -0700 Subject: [PATCH 020/153] Add update to pipxrc after upgrade. Important to update cached venv metadata after a package upgrade. --- src/pipx/commands.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/pipx/commands.py b/src/pipx/commands.py index 0f82b12b61..9e55104b12 100644 --- a/src/pipx/commands.py +++ b/src/pipx/commands.py @@ -207,12 +207,13 @@ def upgrade( ) venv = Venv(venv_dir, verbose=verbose) + pipxrc_info = read_pipxrc(venv_dir) old_version = venv.get_venv_metadata_for_package(package).package_version # if default package_or_url, check pipxrc for better url if package_or_url == package: - package_or_url = read_pipxrc(venv_dir).get("package_or_url", package) + package_or_url = pipxrc_info.get("package_or_url", package) # Upgrade shared libraries (pip, setuptools and wheel) venv.upgrade_packaging_libraries(pip_args) @@ -243,6 +244,8 @@ def upgrade( print( f"upgraded package {package} from {old_version} to {new_version} (location: {str(venv_dir)})" ) + pipxrc_info["venv_metadata"] = venv.get_venv_metadata_for_package(package) + write_pipxrc(venv_dir, pipxrc_info) return 1 From 409d1c8773d2c941a1b0c5adebb1031622afdac2 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Mon, 23 Sep 2019 23:52:40 -0700 Subject: [PATCH 021/153] Make upgrade-all use each PKG's orig. options. Remove many upgrade-all options, and instead use remembered install options via pipxrc from the first install of each PACKAGE. Change the switches and help text of upgrade-all to reflect this change. --- src/pipx/commands.py | 12 +++--------- src/pipx/main.py | 11 +---------- 2 files changed, 4 insertions(+), 19 deletions(-) diff --git a/src/pipx/commands.py b/src/pipx/commands.py index 9e55104b12..2620dec86d 100644 --- a/src/pipx/commands.py +++ b/src/pipx/commands.py @@ -250,13 +250,7 @@ def upgrade( def upgrade_all( - venv_container: VenvContainer, - pip_args: List[str], - verbose: bool, - *, - include_dependencies: bool, - skip: List[str], - force: bool, + venv_container: VenvContainer, verbose: bool, *, skip: List[str], force: bool ): packages_upgraded = 0 num_packages = 0 @@ -275,10 +269,10 @@ def upgrade_all( venv_dir, package, package_or_url, - pip_args, # TODO 20190923: use pipxrc_info + pipxrc_info["install"]["pip_args"], verbose, upgrading_all=True, - include_dependencies=include_dependencies, # TODO 20190923: use pipxrc_info + include_dependencies=pipxrc_info["install"]["include_dependencies"], force=force, ) except Exception: diff --git a/src/pipx/main.py b/src/pipx/main.py index bd1992f4c2..560b0f4588 100644 --- a/src/pipx/main.py +++ b/src/pipx/main.py @@ -205,14 +205,7 @@ def run_pipx_command(args): # noqa: C901 elif args.command == "uninstall-all": return commands.uninstall_all(venv_container, constants.LOCAL_BIN_DIR, verbose) elif args.command == "upgrade-all": - return commands.upgrade_all( - venv_container, - pip_args, - verbose, - include_dependencies=args.include_deps, - skip=args.skip, - force=args.force, - ) + return commands.upgrade_all(venv_container, verbose, skip=args.skip, force=args.force) elif args.command == "reinstall-all": return commands.reinstall_all( venv_container, constants.LOCAL_BIN_DIR, args.python, verbose, skip=args.skip @@ -350,8 +343,6 @@ def _add_upgrade_all(subparsers): description="Upgrades all packages within their virtual environments by running 'pip install --upgrade PACKAGE'", ) - add_include_dependencies(p) - add_pip_venv_args(p) p.add_argument("--skip", nargs="+", default=[], help="skip these packages") p.add_argument( "--force", From 06af0f8b57cd3f930b8ae2c6864b401bf3720803 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Tue, 24 Sep 2019 11:59:50 -0700 Subject: [PATCH 022/153] Remove accidentally-left useless line. --- src/pipx/commands.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/pipx/commands.py b/src/pipx/commands.py index 2620dec86d..bd366ed151 100644 --- a/src/pipx/commands.py +++ b/src/pipx/commands.py @@ -317,7 +317,6 @@ def install( try: venv.create_venv(venv_args, pip_args) venv.install_package(package_or_url, pip_args) - venv.get_venv_metadata_for_package(package) if venv.get_venv_metadata_for_package(package).package_version is None: venv.remove_venv() From e3ce60e8fa219902e5cf463eecd484bfd705636e Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Wed, 25 Sep 2019 17:07:25 -0700 Subject: [PATCH 023/153] Add pipxrc.pipxrc_info_template. pipxrc_info_template is now part of pipxrc.py so that the general format of the written dict can be prescribed in pipxrc.py. Additionally, a pipxrc_version key/item pair is now included that will help in the future with any possible pipxrc format changes. --- pipx/pipxrc.py | 10 ++++++++++ src/pipx/commands.py | 18 +++++++----------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/pipx/pipxrc.py b/pipx/pipxrc.py index b94fb9e1ed..a3e39e1e1b 100644 --- a/pipx/pipxrc.py +++ b/pipx/pipxrc.py @@ -4,6 +4,16 @@ from pipx.Venv import PipxVenvMetadata +# handy, helps enforce some consistency, and adds pipxrc_version +pipxrc_info_template = { + "package_or_url": None, + "install": {"pip_args": [], "venv_args": [], "include_dependencies": None}, + "venv_metadata": None, + "injected_packages": {}, + "pipxrc_version": 0.1, +} + + class JsonEncoderPipx(json.JSONEncoder): def default(self, obj): # only handles what json.JSONEncoder doesn't understand by default diff --git a/src/pipx/commands.py b/src/pipx/commands.py index bd366ed151..2afa1636e1 100644 --- a/src/pipx/commands.py +++ b/src/pipx/commands.py @@ -30,7 +30,7 @@ rmdir, run_pypackage_bin, ) -from pipx.pipxrc import read_pipxrc, write_pipxrc +from pipx.pipxrc import read_pipxrc, write_pipxrc, pipxrc_info_template from pipx.venv import Venv, VenvContainer @@ -333,16 +333,12 @@ def install( # if all is well, write out pipxrc file # TODO 20190923: if package_or_url is a local path, we need to make it # an absolute path - pipxrc_info = { - "package_or_url": package_or_url, - "install": { - "pip_args": pip_args, - "venv_args": venv_args, - "include_dependencies": include_dependencies, - }, - "venv_metadata": venv.get_venv_metadata_for_package(package), - "injected_packages": {}, - } + pipxrc_info = pipxrc_info_template + pipxrc_info["package_or_url"] = package_or_url + pipxrc_info["install"]["pip_args"] = pip_args + pipxrc_info["install"]["venv_args"] = venv_args + pipxrc_info["install"]["include_dependencies"] = include_dependencies + pipxrc_info["venv_metadata"] = venv.get_venv_metadata_for_package(package) write_pipxrc(venv_dir, pipxrc_info) From 3f9da7a22fa00d3845e908fb44ff2868ee1039e9 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Wed, 25 Sep 2019 22:52:16 -0700 Subject: [PATCH 024/153] Add typing info for mypy. --- pipx/pipxrc.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pipx/pipxrc.py b/pipx/pipxrc.py index a3e39e1e1b..e5529a57c4 100644 --- a/pipx/pipxrc.py +++ b/pipx/pipxrc.py @@ -1,11 +1,12 @@ import json from pathlib import Path +from typing import Dict, Any from pipx.Venv import PipxVenvMetadata # handy, helps enforce some consistency, and adds pipxrc_version -pipxrc_info_template = { +pipxrc_info_template: Dict[str, Any] = { "package_or_url": None, "install": {"pip_args": [], "venv_args": [], "include_dependencies": None}, "venv_metadata": None, From 2b157d68431b04bcb1784d35f53cd41eb754f93b Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Thu, 26 Sep 2019 12:52:55 -0700 Subject: [PATCH 025/153] Change pipxrc_version from float to str. --- pipx/pipxrc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipx/pipxrc.py b/pipx/pipxrc.py index e5529a57c4..b86859a939 100644 --- a/pipx/pipxrc.py +++ b/pipx/pipxrc.py @@ -11,7 +11,7 @@ "install": {"pip_args": [], "venv_args": [], "include_dependencies": None}, "venv_metadata": None, "injected_packages": {}, - "pipxrc_version": 0.1, + "pipxrc_version": "0.1", } From 52c0a433ba23b5716a9150a50591ca5825c61be6 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Thu, 26 Sep 2019 16:27:46 -0700 Subject: [PATCH 026/153] Convert pipxrc interface to class-based. Roll all pipxrc operations into class Pipxrc. This has the advantage of confining all code that might affect the format of the pipxrc files to pipxrc.py. It's also a cleaner interface. This commit has a mypy error I'm unable to figure out right now: pipx/pipxrc.py:119: error: Unsupported target for indexed assignment --- pipx/pipxrc.py | 177 ++++++++++++++++++++++++++++++++++--------- src/pipx/commands.py | 80 ++++++++++--------- 2 files changed, 180 insertions(+), 77 deletions(-) diff --git a/pipx/pipxrc.py b/pipx/pipxrc.py index b86859a939..f87e50e557 100644 --- a/pipx/pipxrc.py +++ b/pipx/pipxrc.py @@ -1,26 +1,17 @@ +import copy import json from pathlib import Path -from typing import Dict, Any +from typing import List, Dict, Any from pipx.Venv import PipxVenvMetadata -# handy, helps enforce some consistency, and adds pipxrc_version -pipxrc_info_template: Dict[str, Any] = { - "package_or_url": None, - "install": {"pip_args": [], "venv_args": [], "include_dependencies": None}, - "venv_metadata": None, - "injected_packages": {}, - "pipxrc_version": "0.1", -} - - class JsonEncoderPipx(json.JSONEncoder): def default(self, obj): # only handles what json.JSONEncoder doesn't understand by default if isinstance(obj, Path): return {"__type__": "Path", "__Path__": str(obj)} - return super().default(self, obj) + return super().default(obj) def _json_decoder_object_hook(json_dict): @@ -34,26 +25,142 @@ def _json_decoder_object_hook(json_dict): return json_dict -def write_pipxrc(venv_dir: Path, pipxrc_info: dict): - # json thinks PipxVenvMetadata is just another tuple, so we override here - # (JSONEncoder override is harder and messier.) - for key in pipxrc_info: - if isinstance(pipxrc_info[key], PipxVenvMetadata): - pipxrc_info[key] = { - "__type__": "PipxVenvMetadata", - "__PipxVenvMetadata__": dict(pipxrc_info[key]._asdict()), - } - # TODO 20190919: raise exception on failure? - with open(venv_dir / "pipxrc", "w") as pipxrc_fh: - json.dump(pipxrc_info, pipxrc_fh, indent=4, sort_keys=True, cls=JsonEncoderPipx) - - -def read_pipxrc(venv_dir: Path) -> dict: - try: - with open(venv_dir / "pipxrc", "r") as pipxrc_fh: - pipxrc_info = json.load(pipxrc_fh, object_hook=_json_decoder_object_hook) - except IOError: - # return empty dict if no pipxrc file or unreadable - return {} - - return pipxrc_info +class Pipxrc: + def __init__(self, venv_dir: Path, read: bool = True): + self.venv_dir = venv_dir + # Reference for Pipx.pipxrc_info, never modify this in runtime + self.pipxrc_info_template: Dict[str, Any] = { + "package_or_url": None, + "install": { + "pip_args": None, + "venv_args": None, + "include_dependencies": None, + }, + "venv_metadata": None, + "injected_packages": None, + "pipxrc_version": "0.1", + } + self.pipxrc_info: Dict[str, Any] = {} + self.reset() + if read: + self.read() + + def reset(self): + self.pipxrc_info = copy.deepcopy(self.pipxrc_info_template) + + def get_package_or_url(self, default: str) -> str: + if self.pipxrc_info["package_or_url"] is not None: + return self.pipxrc_info["package_or_url"] + else: + return default + + def get_install_pip_args(self, default: List) -> List: + if self.pipxrc_info["install"]["pip_args"] is not None: + return self.pipxrc_info["install"]["pip_args"] + else: + return default + + def get_install_venv_args(self, default: List) -> List: + if self.pipxrc_info["install"]["venv_args"] is not None: + return self.pipxrc_info["install"]["venv_args"] + else: + return default + + def get_install_include_dependencies(self, default: bool) -> bool: + if self.pipxrc_info["install"]["include_dependencies"] is not None: + return self.pipxrc_info["install"]["include_dependencies"] + else: + return default + + def get_venv_metadata(self, default: PipxVenvMetadata) -> PipxVenvMetadata: + if self.pipxrc_info["venv_metadata"] is not None: + return self.pipxrc_info["venv_metadata"] + else: + return default + + def get_injected_packages(self, default: List) -> List: + if self.pipxrc_info["injected_packages"] is not None: + injected_packages = [] + for package in self.pipxrc_info["injected_packages"]: + package_info = {"package": package} + package_info.update(self.pipxrc_info["injected_packages"][package]) + injected_packages.append(package_info) + return injected_packages + else: + return default + + def set_package_or_url(self, package_or_url: str): + # TODO 20190923: if package_or_url is a local path, we need to make it + # an absolute path + self.pipxrc_info["package_or_url"] = package_or_url + + def set_venv_metadata(self, venv_metadata: PipxVenvMetadata): + self.pipxrc_info["venv_metadata"] = venv_metadata + + def set_install_options( + self, pip_args: List, venv_args: List, include_dependencies: bool + ): + self.pipxrc_info["install"]["pip_args"] = pip_args + self.pipxrc_info["install"]["venv_args"] = venv_args + self.pipxrc_info["install"]["include_dependencies"] = include_dependencies + + def add_injected_package( + self, + package: str, + pip_args: List, + verbose: bool, + include_apps: bool, + include_dependencies: bool, + force: bool, + ): + if self.pipxrc_info["injected_packages"] is None: + self.pipxrc_info["injected_packages"] = {} + + self.pipxrc_info["injected_packages"][package] = { + "pip_args": pip_args, + "verbose": verbose, + "include_apps": include_apps, + "include_dependencies": include_dependencies, + "force": force, + } + + def _get_serializable(self): + pipxrc_info_ser = copy.deepcopy(self.pipxrc_info) + + # json thinks PipxVenvMetadata is just another tuple, so we override here + # (JSONEncoder override is harder and messier.) + for key in pipxrc_info_ser: + if isinstance(pipxrc_info_ser[key], PipxVenvMetadata): + pipxrc_info_ser[key] = { + "__type__": "PipxVenvMetadata", + "__PipxVenvMetadata__": dict(pipxrc_info_ser[key]._asdict()), + } + return pipxrc_info_ser + + def write(self): + # If writing out, make sure injected_packages is not None, so next + # successful read of pipxrc does not use default in + # get_injected_packages() + if self.pipxrc_info["injected_packages"] is None: + self.pipxrc_info["injected_packages"] = {} + + pipxrc_info_ser = self._get_serializable() + # TODO 20190919: raise exception on failure? + with open(self.venv_dir / "pipxrc", "w") as pipxrc_fh: + json.dump( + pipxrc_info_ser, + pipxrc_fh, + indent=4, + sort_keys=True, + cls=JsonEncoderPipx, + ) + + def read(self): + try: + with open(self.venv_dir / "pipxrc", "r") as pipxrc_fh: + self.pipxrc_info = json.load( + pipxrc_fh, object_hook=_json_decoder_object_hook + ) + except IOError: # Reset self.pipxrc_info if problem reading + self.reset() + return diff --git a/src/pipx/commands.py b/src/pipx/commands.py index 2afa1636e1..115808d9a8 100644 --- a/src/pipx/commands.py +++ b/src/pipx/commands.py @@ -30,7 +30,7 @@ rmdir, run_pypackage_bin, ) -from pipx.pipxrc import read_pipxrc, write_pipxrc, pipxrc_info_template +from pipx.pipxrc import Pipxrc from pipx.venv import Venv, VenvContainer @@ -207,13 +207,15 @@ def upgrade( ) venv = Venv(venv_dir, verbose=verbose) - pipxrc_info = read_pipxrc(venv_dir) + pipxrc = Pipxrc(venv_dir) old_version = venv.get_venv_metadata_for_package(package).package_version # if default package_or_url, check pipxrc for better url + # TODO 20190926: main.py should communicate if this is spec or copied from + # package if package_or_url == package: - package_or_url = pipxrc_info.get("package_or_url", package) + package_or_url = pipxrc.get_package_or_url(default=package) # Upgrade shared libraries (pip, setuptools and wheel) venv.upgrade_packaging_libraries(pip_args) @@ -244,8 +246,8 @@ def upgrade( print( f"upgraded package {package} from {old_version} to {new_version} (location: {str(venv_dir)})" ) - pipxrc_info["venv_metadata"] = venv.get_venv_metadata_for_package(package) - write_pipxrc(venv_dir, pipxrc_info) + pipxrc.set_venv_metadata(venv.get_venv_metadata_for_package(package)) + pipxrc.write() return 1 @@ -257,22 +259,24 @@ def upgrade_all( for venv_dir in venv_container.iter_venv_dirs(): num_packages += 1 package = venv_dir.name - pipxrc_info = read_pipxrc(venv_dir) + pipxrc = Pipxrc(venv_dir) if package in skip: continue if package == "pipx": package_or_url = PIPX_PACKAGE_NAME else: - package_or_url = pipxrc_info.get("package_or_url", package) + package_or_url = pipxrc.get_package_or_url(default=package) try: packages_upgraded += upgrade( venv_dir, package, package_or_url, - pipxrc_info["install"]["pip_args"], + pipxrc.get_install_pip_args(default=[]), verbose, upgrading_all=True, - include_dependencies=pipxrc_info["install"]["include_dependencies"], + include_dependencies=pipxrc.get_install_include_dependencies( + default=False + ), force=force, ) except Exception: @@ -331,15 +335,11 @@ def install( raise # if all is well, write out pipxrc file - # TODO 20190923: if package_or_url is a local path, we need to make it - # an absolute path - pipxrc_info = pipxrc_info_template - pipxrc_info["package_or_url"] = package_or_url - pipxrc_info["install"]["pip_args"] = pip_args - pipxrc_info["install"]["venv_args"] = venv_args - pipxrc_info["install"]["include_dependencies"] = include_dependencies - pipxrc_info["venv_metadata"] = venv.get_venv_metadata_for_package(package) - write_pipxrc(venv_dir, pipxrc_info) + pipxrc = Pipxrc(venv_dir, read=False) + pipxrc.set_package_or_url(package_or_url) + pipxrc.set_install_options(pip_args, venv_args, include_dependencies) + pipxrc.set_venv_metadata(venv.get_venv_metadata_for_package(package)) + pipxrc.write() def _run_post_install_actions( @@ -450,15 +450,11 @@ def inject( include_dependencies, force=force, ) - pipxrc_info = read_pipxrc(venv_dir) - pipxrc_info["injected_packages"][package] = { - "pip_args": pip_args, - "verbose": verbose, - "include_apps": include_apps, - "include_dependencies": include_dependencies, - "force": force, - } - write_pipxrc(venv_dir, pipxrc_info) + pipxrc = Pipxrc(venv_dir) + pipxrc.add_injected_package( + package, pip_args, verbose, include_apps, include_dependencies, force + ) + pipxrc.write() print(f" injected package {bold(package)} into venv {bold(venv_dir.name)}") print(f"done! {stars}", file=sys.stderr) @@ -475,9 +471,10 @@ def uninstall(venv_dir: Path, package: str, local_bin_dir: Path, verbose: bool): return venv = Venv(venv_dir, verbose=verbose) + pipxrc = Pipxrc(venv_dir) - metadata = read_pipxrc(venv_dir).get( - "venv_metadata", venv.get_venv_metadata_for_package(package) + metadata = pipxrc.get_venv_metadata( + default=venv.get_venv_metadata_for_package(package) ) app_paths = metadata.app_paths for dep_paths in metadata.app_paths_of_dependencies.values(): @@ -516,32 +513,31 @@ def reinstall_all( package = venv_dir.name if package in skip: continue - pipxrc_info = read_pipxrc(venv_dir) + pipxrc = Pipxrc(venv_dir) uninstall(venv_dir, package, local_bin_dir, verbose) - package_or_url = pipxrc_info.get("package_or_url", package) + package_or_url = pipxrc.get_package_or_url(default=package) install( venv_dir, package, package_or_url, local_bin_dir, python, - pipxrc_info["install"]["pip_args"], - pipxrc_info["install"]["venv_args"], + pipxrc.get_install_pip_args(default=[]), + pipxrc.get_install_venv_args(default=[]), verbose, force=True, - include_dependencies=pipxrc_info["install"]["include_dependencies"], + include_dependencies=pipxrc.get_install_include_dependencies(default=False), ) - for injected_package in pipxrc_info.get("injected_packages", {}): - pkg_info = pipxrc_info["injected_packages"][injected_package] + for injected in pipxrc.get_injected_packages(default=[]): inject( venv_dir, - injected_package, - pkg_info["pip_args"], - verbose=pkg_info["verbose"], - include_apps=pkg_info["include_apps"], - include_dependencies=pkg_info["include_dependencies"], - force=pkg_info["force"], + injected["package"], + injected["pip_args"], + verbose=injected["verbose"], + include_apps=injected["include_apps"], + include_dependencies=injected["include_dependencies"], + force=injected["force"], ) From c2a5ffbd46eea7928402f2d6957e787b5d5ec3e0 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Mon, 30 Sep 2019 17:56:10 -0700 Subject: [PATCH 027/153] New PipxrcInfo data class. New mutable data structure to satisfy mypy. --- pipx/pipxrc.py | 114 +++++++++++++++++++++++++------------------------ 1 file changed, 59 insertions(+), 55 deletions(-) diff --git a/pipx/pipxrc.py b/pipx/pipxrc.py index f87e50e557..eb5dcdd879 100644 --- a/pipx/pipxrc.py +++ b/pipx/pipxrc.py @@ -25,65 +25,83 @@ def _json_decoder_object_hook(json_dict): return json_dict +class PipxrcInfo: + def __init__(self): + self.package_or_url: Union[str, None] = None + self.install: Dict[str, Union[List[str], bool, None]] = { + "pip_args": None, + "venv_args": None, + "include_dependencies": None, + } + self.venv_metadata: Union[PipxVenvMetadata, None] = None + self.injected_packages: Union[Dict[str, Any], None] = None + self._pipxrc_version: str = "0.1" + + def _serialize(self): + return { + "package_or_url": self.package_or_url, + "install": self.install, + "venv_metadata": { + "__type__": "PipxVenvMetadata", + "__PipxVenvMetadata__": dict(self.venv_metadata._asdict()), + }, + "injected_packages": self.injected_packages, + "pipxrc_version": self._pipxrc_version, + } + + def _unserialize(self, pipxrc_info_dict): + self.package_or_url = pipxrc_info_dict["package_or_url"] + self.install = pipxrc_info_dict["install"] + self.venv_metadata = pipxrc_info_dict["venv_metadata"] + self.injected_packages = pipxrc_info_dict["injected_packages"] + + class Pipxrc: def __init__(self, venv_dir: Path, read: bool = True): self.venv_dir = venv_dir - # Reference for Pipx.pipxrc_info, never modify this in runtime - self.pipxrc_info_template: Dict[str, Any] = { - "package_or_url": None, - "install": { - "pip_args": None, - "venv_args": None, - "include_dependencies": None, - }, - "venv_metadata": None, - "injected_packages": None, - "pipxrc_version": "0.1", - } - self.pipxrc_info: Dict[str, Any] = {} - self.reset() + self.pipxrc_info = PipxrcInfo() if read: self.read() def reset(self): - self.pipxrc_info = copy.deepcopy(self.pipxrc_info_template) + self.pipxrc_info = PipxrcInfo() def get_package_or_url(self, default: str) -> str: - if self.pipxrc_info["package_or_url"] is not None: - return self.pipxrc_info["package_or_url"] + if self.pipxrc_info.package_or_url is not None: + return self.pipxrc_info.package_or_url else: return default def get_install_pip_args(self, default: List) -> List: - if self.pipxrc_info["install"]["pip_args"] is not None: - return self.pipxrc_info["install"]["pip_args"] + if self.pipxrc_info.install["pip_args"] is not None: + return self.pipxrc_info.install["pip_args"] else: return default def get_install_venv_args(self, default: List) -> List: - if self.pipxrc_info["install"]["venv_args"] is not None: - return self.pipxrc_info["install"]["venv_args"] + if self.pipxrc_info.install["venv_args"] is not None: + return self.pipxrc_info.install["venv_args"] else: return default def get_install_include_dependencies(self, default: bool) -> bool: - if self.pipxrc_info["install"]["include_dependencies"] is not None: - return self.pipxrc_info["install"]["include_dependencies"] + if self.pipxrc_info.install["include_dependencies"] is not None: + return self.pipxrc_info.install["include_dependencies"] else: return default def get_venv_metadata(self, default: PipxVenvMetadata) -> PipxVenvMetadata: - if self.pipxrc_info["venv_metadata"] is not None: - return self.pipxrc_info["venv_metadata"] + if self.pipxrc_info.venv_metadata is not None: + return self.pipxrc_info.venv_metadata else: return default def get_injected_packages(self, default: List) -> List: - if self.pipxrc_info["injected_packages"] is not None: + if self.pipxrc_info.injected_packages is not None: injected_packages = [] - for package in self.pipxrc_info["injected_packages"]: + for package in self.pipxrc_info.injected_packages: package_info = {"package": package} - package_info.update(self.pipxrc_info["injected_packages"][package]) + package_info.update(self.pipxrc_info.injected_packages[package]) injected_packages.append(package_info) return injected_packages else: @@ -92,17 +110,17 @@ def get_injected_packages(self, default: List) -> List: def set_package_or_url(self, package_or_url: str): # TODO 20190923: if package_or_url is a local path, we need to make it # an absolute path - self.pipxrc_info["package_or_url"] = package_or_url + self.pipxrc_info.package_or_url = package_or_url def set_venv_metadata(self, venv_metadata: PipxVenvMetadata): - self.pipxrc_info["venv_metadata"] = venv_metadata + self.pipxrc_info.venv_metadata = venv_metadata def set_install_options( self, pip_args: List, venv_args: List, include_dependencies: bool ): - self.pipxrc_info["install"]["pip_args"] = pip_args - self.pipxrc_info["install"]["venv_args"] = venv_args - self.pipxrc_info["install"]["include_dependencies"] = include_dependencies + self.pipxrc_info.install["pip_args"] = pip_args + self.pipxrc_info.install["venv_args"] = venv_args + self.pipxrc_info.install["include_dependencies"] = include_dependencies def add_injected_package( self, @@ -113,10 +131,10 @@ def add_injected_package( include_dependencies: bool, force: bool, ): - if self.pipxrc_info["injected_packages"] is None: - self.pipxrc_info["injected_packages"] = {} + if self.pipxrc_info.injected_packages is None: + self.pipxrc_info.injected_packages = {} - self.pipxrc_info["injected_packages"][package] = { + self.pipxrc_info.injected_packages[package] = { "pip_args": pip_args, "verbose": verbose, "include_apps": include_apps, @@ -124,31 +142,17 @@ def add_injected_package( "force": force, } - def _get_serializable(self): - pipxrc_info_ser = copy.deepcopy(self.pipxrc_info) - - # json thinks PipxVenvMetadata is just another tuple, so we override here - # (JSONEncoder override is harder and messier.) - for key in pipxrc_info_ser: - if isinstance(pipxrc_info_ser[key], PipxVenvMetadata): - pipxrc_info_ser[key] = { - "__type__": "PipxVenvMetadata", - "__PipxVenvMetadata__": dict(pipxrc_info_ser[key]._asdict()), - } - return pipxrc_info_ser - def write(self): # If writing out, make sure injected_packages is not None, so next # successful read of pipxrc does not use default in # get_injected_packages() - if self.pipxrc_info["injected_packages"] is None: - self.pipxrc_info["injected_packages"] = {} + if self.pipxrc_info.injected_packages is None: + self.pipxrc_info.injected_packages = {} - pipxrc_info_ser = self._get_serializable() # TODO 20190919: raise exception on failure? with open(self.venv_dir / "pipxrc", "w") as pipxrc_fh: json.dump( - pipxrc_info_ser, + self.pipxrc_info._serialize(), pipxrc_fh, indent=4, sort_keys=True, @@ -158,8 +162,8 @@ def write(self): def read(self): try: with open(self.venv_dir / "pipxrc", "r") as pipxrc_fh: - self.pipxrc_info = json.load( - pipxrc_fh, object_hook=_json_decoder_object_hook + self.pipxrc_info._unserialize( + json.load(pipxrc_fh, object_hook=_json_decoder_object_hook) ) except IOError: # Reset self.pipxrc_info if problem reading self.reset() From bcf4ca79cf3ee54949092831e050cdc9a17fa596 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Mon, 30 Sep 2019 20:09:00 -0700 Subject: [PATCH 028/153] Rename PipxrcInfo helpers non-private and better. --- pipx/pipxrc.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pipx/pipxrc.py b/pipx/pipxrc.py index eb5dcdd879..94f31db440 100644 --- a/pipx/pipxrc.py +++ b/pipx/pipxrc.py @@ -37,7 +37,7 @@ def __init__(self): self.injected_packages: Union[Dict[str, Any], None] = None self._pipxrc_version: str = "0.1" - def _serialize(self): + def to_dict(self): return { "package_or_url": self.package_or_url, "install": self.install, @@ -49,7 +49,7 @@ def _serialize(self): "pipxrc_version": self._pipxrc_version, } - def _unserialize(self, pipxrc_info_dict): + def from_dict(self, pipxrc_info_dict): self.package_or_url = pipxrc_info_dict["package_or_url"] self.install = pipxrc_info_dict["install"] self.venv_metadata = pipxrc_info_dict["venv_metadata"] @@ -152,7 +152,7 @@ def write(self): # TODO 20190919: raise exception on failure? with open(self.venv_dir / "pipxrc", "w") as pipxrc_fh: json.dump( - self.pipxrc_info._serialize(), + self.pipxrc_info.to_dict(), pipxrc_fh, indent=4, sort_keys=True, @@ -162,7 +162,7 @@ def write(self): def read(self): try: with open(self.venv_dir / "pipxrc", "r") as pipxrc_fh: - self.pipxrc_info._unserialize( + self.pipxrc_info.from_dict( json.load(pipxrc_fh, object_hook=_json_decoder_object_hook) ) except IOError: # Reset self.pipxrc_info if problem reading From 3425fb2a952846274e5b98223f147a752f5a5661 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Mon, 30 Sep 2019 20:21:09 -0700 Subject: [PATCH 029/153] Fix typing, remove obsolete import copy. --- pipx/pipxrc.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/pipx/pipxrc.py b/pipx/pipxrc.py index 94f31db440..4a7eb35e1a 100644 --- a/pipx/pipxrc.py +++ b/pipx/pipxrc.py @@ -1,7 +1,6 @@ -import copy import json from pathlib import Path -from typing import List, Dict, Any +from typing import List, Dict, Any, Union from pipx.Venv import PipxVenvMetadata @@ -28,7 +27,7 @@ def _json_decoder_object_hook(json_dict): class PipxrcInfo: def __init__(self): self.package_or_url: Union[str, None] = None - self.install: Dict[str, Union[List[str], bool, None]] = { + self.install: Dict[str, Any] = { "pip_args": None, "venv_args": None, "include_dependencies": None, @@ -72,13 +71,13 @@ def get_package_or_url(self, default: str) -> str: else: return default - def get_install_pip_args(self, default: List) -> List: + def get_install_pip_args(self, default: List[str]) -> List[str]: if self.pipxrc_info.install["pip_args"] is not None: return self.pipxrc_info.install["pip_args"] else: return default - def get_install_venv_args(self, default: List) -> List: + def get_install_venv_args(self, default: List[str]) -> List[str]: if self.pipxrc_info.install["venv_args"] is not None: return self.pipxrc_info.install["venv_args"] else: From 9b705a141a9a2ac4b118174ef4e614e5de64f1da Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Mon, 30 Sep 2019 22:31:18 -0700 Subject: [PATCH 030/153] Use InjPkg NamedTuple for injected_packages. --- pipx/pipxrc.py | 48 +++++++++++++++++++++++++++--------------------- 1 file changed, 27 insertions(+), 21 deletions(-) diff --git a/pipx/pipxrc.py b/pipx/pipxrc.py index 4a7eb35e1a..a7cec13824 100644 --- a/pipx/pipxrc.py +++ b/pipx/pipxrc.py @@ -1,6 +1,6 @@ import json from pathlib import Path -from typing import List, Dict, Any, Union +from typing import List, Dict, Any, Union, NamedTuple from pipx.Venv import PipxVenvMetadata @@ -16,14 +16,17 @@ def default(self, obj): def _json_decoder_object_hook(json_dict): if json_dict.get("__type__", None) == "Path" and "__Path__" in json_dict: return Path(json_dict["__Path__"]) - if ( - json_dict.get("__type__", None) == "PipxVenvMetadata" - and "__PipxVenvMetadata__" in json_dict - ): - return PipxVenvMetadata(**json_dict["__PipxVenvMetadata__"]) return json_dict +class InjPkg(NamedTuple): + pip_args: List[str] + verbose: bool + include_apps: bool + include_dependencies: bool + force: bool + + class PipxrcInfo: def __init__(self): self.package_or_url: Union[str, None] = None @@ -33,26 +36,27 @@ def __init__(self): "include_dependencies": None, } self.venv_metadata: Union[PipxVenvMetadata, None] = None - self.injected_packages: Union[Dict[str, Any], None] = None + self.injected_packages: Union[Dict[str, InjPkg], None] = None self._pipxrc_version: str = "0.1" def to_dict(self): return { "package_or_url": self.package_or_url, "install": self.install, - "venv_metadata": { - "__type__": "PipxVenvMetadata", - "__PipxVenvMetadata__": dict(self.venv_metadata._asdict()), + "venv_metadata": self.venv_metadata._asdict(), + "injected_packages": { + k: v._asdict() for (k, v) in self.injected_packages.items() }, - "injected_packages": self.injected_packages, "pipxrc_version": self._pipxrc_version, } def from_dict(self, pipxrc_info_dict): self.package_or_url = pipxrc_info_dict["package_or_url"] self.install = pipxrc_info_dict["install"] - self.venv_metadata = pipxrc_info_dict["venv_metadata"] - self.injected_packages = pipxrc_info_dict["injected_packages"] + self.venv_metadata = PipxVenvMetadata(**pipxrc_info_dict["venv_metadata"]) + self.injected_packages = { + k: InjPkg(**v) for (k, v) in pipxrc_info_dict["injected_packages"].items() + } class Pipxrc: @@ -100,7 +104,9 @@ def get_injected_packages(self, default: List) -> List: injected_packages = [] for package in self.pipxrc_info.injected_packages: package_info = {"package": package} - package_info.update(self.pipxrc_info.injected_packages[package]) + package_info.update( + self.pipxrc_info.injected_packages[package]._asdict() + ) injected_packages.append(package_info) return injected_packages else: @@ -133,13 +139,13 @@ def add_injected_package( if self.pipxrc_info.injected_packages is None: self.pipxrc_info.injected_packages = {} - self.pipxrc_info.injected_packages[package] = { - "pip_args": pip_args, - "verbose": verbose, - "include_apps": include_apps, - "include_dependencies": include_dependencies, - "force": force, - } + self.pipxrc_info.injected_packages[package] = InjPkg( + pip_args=pip_args, + verbose=verbose, + include_apps=include_apps, + include_dependencies=include_dependencies, + force=force, + ) def write(self): # If writing out, make sure injected_packages is not None, so next From 4f9714101bbad7f7a6223c85668c3b435eb6048c Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Mon, 30 Sep 2019 22:53:26 -0700 Subject: [PATCH 031/153] Use InstallOpts NamedTuple for install options. --- pipx/pipxrc.py | 40 +++++++++++++++++++++++----------------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/pipx/pipxrc.py b/pipx/pipxrc.py index a7cec13824..a851cbd696 100644 --- a/pipx/pipxrc.py +++ b/pipx/pipxrc.py @@ -1,6 +1,6 @@ import json from pathlib import Path -from typing import List, Dict, Any, Union, NamedTuple +from typing import List, Dict, Union, NamedTuple from pipx.Venv import PipxVenvMetadata @@ -27,14 +27,18 @@ class InjPkg(NamedTuple): force: bool +class InstallOpts(NamedTuple): + pip_args: Union[List[str], None] + venv_args: Union[List[str], None] + include_dependencies: Union[bool, None] + + class PipxrcInfo: def __init__(self): self.package_or_url: Union[str, None] = None - self.install: Dict[str, Any] = { - "pip_args": None, - "venv_args": None, - "include_dependencies": None, - } + self.install: InstallOpts = InstallOpts( + pip_args=None, venv_args=None, include_dependencies=None + ) self.venv_metadata: Union[PipxVenvMetadata, None] = None self.injected_packages: Union[Dict[str, InjPkg], None] = None self._pipxrc_version: str = "0.1" @@ -42,7 +46,7 @@ def __init__(self): def to_dict(self): return { "package_or_url": self.package_or_url, - "install": self.install, + "install": self.install._asdict(), "venv_metadata": self.venv_metadata._asdict(), "injected_packages": { k: v._asdict() for (k, v) in self.injected_packages.items() @@ -52,7 +56,7 @@ def to_dict(self): def from_dict(self, pipxrc_info_dict): self.package_or_url = pipxrc_info_dict["package_or_url"] - self.install = pipxrc_info_dict["install"] + self.install = InstallOpts(**pipxrc_info_dict["install"]) self.venv_metadata = PipxVenvMetadata(**pipxrc_info_dict["venv_metadata"]) self.injected_packages = { k: InjPkg(**v) for (k, v) in pipxrc_info_dict["injected_packages"].items() @@ -76,20 +80,20 @@ def get_package_or_url(self, default: str) -> str: return default def get_install_pip_args(self, default: List[str]) -> List[str]: - if self.pipxrc_info.install["pip_args"] is not None: - return self.pipxrc_info.install["pip_args"] + if self.pipxrc_info.install.pip_args is not None: + return self.pipxrc_info.install.pip_args else: return default def get_install_venv_args(self, default: List[str]) -> List[str]: - if self.pipxrc_info.install["venv_args"] is not None: - return self.pipxrc_info.install["venv_args"] + if self.pipxrc_info.install.venv_args is not None: + return self.pipxrc_info.install.venv_args else: return default def get_install_include_dependencies(self, default: bool) -> bool: - if self.pipxrc_info.install["include_dependencies"] is not None: - return self.pipxrc_info.install["include_dependencies"] + if self.pipxrc_info.install.include_dependencies is not None: + return self.pipxrc_info.install.include_dependencies else: return default @@ -123,9 +127,11 @@ def set_venv_metadata(self, venv_metadata: PipxVenvMetadata): def set_install_options( self, pip_args: List, venv_args: List, include_dependencies: bool ): - self.pipxrc_info.install["pip_args"] = pip_args - self.pipxrc_info.install["venv_args"] = venv_args - self.pipxrc_info.install["include_dependencies"] = include_dependencies + self.pipxrc_info.install = InstallOpts( + pip_args=pip_args, + venv_args=venv_args, + include_dependencies=include_dependencies, + ) def add_injected_package( self, From 6b8a01012b1c4b950d8bc1beba5dc30bc816f26c Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Mon, 30 Sep 2019 23:06:52 -0700 Subject: [PATCH 032/153] New _val_or_default() for get_ member fxns. --- pipx/pipxrc.py | 31 ++++++++++++------------------- 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/pipx/pipxrc.py b/pipx/pipxrc.py index a851cbd696..2bc26af05f 100644 --- a/pipx/pipxrc.py +++ b/pipx/pipxrc.py @@ -73,35 +73,28 @@ def __init__(self, venv_dir: Path, read: bool = True): def reset(self): self.pipxrc_info = PipxrcInfo() - def get_package_or_url(self, default: str) -> str: - if self.pipxrc_info.package_or_url is not None: - return self.pipxrc_info.package_or_url + def _val_or_default(self, value, default): + if value is not None: + return value else: return default + def get_package_or_url(self, default: str) -> str: + return self._val_or_default(self.pipxrc_info.package_or_url, default) + def get_install_pip_args(self, default: List[str]) -> List[str]: - if self.pipxrc_info.install.pip_args is not None: - return self.pipxrc_info.install.pip_args - else: - return default + return self._val_or_default(self.pipxrc_info.install.pip_args, default) def get_install_venv_args(self, default: List[str]) -> List[str]: - if self.pipxrc_info.install.venv_args is not None: - return self.pipxrc_info.install.venv_args - else: - return default + return self._val_or_default(self.pipxrc_info.install.venv_args, default) def get_install_include_dependencies(self, default: bool) -> bool: - if self.pipxrc_info.install.include_dependencies is not None: - return self.pipxrc_info.install.include_dependencies - else: - return default + return self._val_or_default( + self.pipxrc_info.install.include_dependencies, default + ) def get_venv_metadata(self, default: PipxVenvMetadata) -> PipxVenvMetadata: - if self.pipxrc_info.venv_metadata is not None: - return self.pipxrc_info.venv_metadata - else: - return default + return self._val_or_default(self.pipxrc_info.venv_metadata, default) def get_injected_packages(self, default: List) -> List: if self.pipxrc_info.injected_packages is not None: From d937266f912017d42e82efe64887e8ce7cdc6c59 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Tue, 1 Oct 2019 11:37:15 -0700 Subject: [PATCH 033/153] Add typing, fix PipxrcInfo.to_dict error. PipxrcInfo.to_dict() used to have an error condition if some members were None. This is now handled. Union[X, None] were replaced in the code with the equivalent Optional[X] --- pipx/pipxrc.py | 58 ++++++++++++++++++++++++++++++-------------------- 1 file changed, 35 insertions(+), 23 deletions(-) diff --git a/pipx/pipxrc.py b/pipx/pipxrc.py index 2bc26af05f..54f9163e74 100644 --- a/pipx/pipxrc.py +++ b/pipx/pipxrc.py @@ -1,6 +1,6 @@ import json from pathlib import Path -from typing import List, Dict, Union, NamedTuple +from typing import List, Dict, NamedTuple, Any, Optional from pipx.Venv import PipxVenvMetadata @@ -28,33 +28,45 @@ class InjPkg(NamedTuple): class InstallOpts(NamedTuple): - pip_args: Union[List[str], None] - venv_args: Union[List[str], None] - include_dependencies: Union[bool, None] + pip_args: Optional[List[str]] + venv_args: Optional[List[str]] + include_dependencies: Optional[bool] class PipxrcInfo: def __init__(self): - self.package_or_url: Union[str, None] = None + self.package_or_url: Optional[str] = None self.install: InstallOpts = InstallOpts( pip_args=None, venv_args=None, include_dependencies=None ) - self.venv_metadata: Union[PipxVenvMetadata, None] = None - self.injected_packages: Union[Dict[str, InjPkg], None] = None + self.venv_metadata: Optional[PipxVenvMetadata] = None + self.injected_packages: Optional[Dict[str, InjPkg]] = None self._pipxrc_version: str = "0.1" - def to_dict(self): + def to_dict(self) -> Dict[str, Any]: + venv_metadata: Optional[Dict[str, Any]] + injected_packages: Optional[Dict[str, Dict[str, Any]]] + + if self.venv_metadata is not None: + venv_metadata = self.venv_metadata._asdict() + else: + venv_metadata = None + if self.injected_packages is not None: + injected_packages = { + k: v._asdict() for (k, v) in self.injected_packages.items() + } + else: + injected_packages = None + return { "package_or_url": self.package_or_url, "install": self.install._asdict(), - "venv_metadata": self.venv_metadata._asdict(), - "injected_packages": { - k: v._asdict() for (k, v) in self.injected_packages.items() - }, + "venv_metadata": venv_metadata, + "injected_packages": injected_packages, "pipxrc_version": self._pipxrc_version, } - def from_dict(self, pipxrc_info_dict): + def from_dict(self, pipxrc_info_dict) -> None: self.package_or_url = pipxrc_info_dict["package_or_url"] self.install = InstallOpts(**pipxrc_info_dict["install"]) self.venv_metadata = PipxVenvMetadata(**pipxrc_info_dict["venv_metadata"]) @@ -70,7 +82,7 @@ def __init__(self, venv_dir: Path, read: bool = True): if read: self.read() - def reset(self): + def reset(self) -> None: self.pipxrc_info = PipxrcInfo() def _val_or_default(self, value, default): @@ -96,7 +108,7 @@ def get_install_include_dependencies(self, default: bool) -> bool: def get_venv_metadata(self, default: PipxVenvMetadata) -> PipxVenvMetadata: return self._val_or_default(self.pipxrc_info.venv_metadata, default) - def get_injected_packages(self, default: List) -> List: + def get_injected_packages(self, default: List[Dict[str, Any]]) -> List[Dict[str, Any]]: if self.pipxrc_info.injected_packages is not None: injected_packages = [] for package in self.pipxrc_info.injected_packages: @@ -109,17 +121,17 @@ def get_injected_packages(self, default: List) -> List: else: return default - def set_package_or_url(self, package_or_url: str): + def set_package_or_url(self, package_or_url: str) -> None: # TODO 20190923: if package_or_url is a local path, we need to make it # an absolute path self.pipxrc_info.package_or_url = package_or_url - def set_venv_metadata(self, venv_metadata: PipxVenvMetadata): + def set_venv_metadata(self, venv_metadata: PipxVenvMetadata) -> None: self.pipxrc_info.venv_metadata = venv_metadata def set_install_options( - self, pip_args: List, venv_args: List, include_dependencies: bool - ): + self, pip_args: List[str], venv_args: List[str], include_dependencies: bool + ) -> None: self.pipxrc_info.install = InstallOpts( pip_args=pip_args, venv_args=venv_args, @@ -129,12 +141,12 @@ def set_install_options( def add_injected_package( self, package: str, - pip_args: List, + pip_args: List[str], verbose: bool, include_apps: bool, include_dependencies: bool, force: bool, - ): + ) -> None: if self.pipxrc_info.injected_packages is None: self.pipxrc_info.injected_packages = {} @@ -146,7 +158,7 @@ def add_injected_package( force=force, ) - def write(self): + def write(self) -> None: # If writing out, make sure injected_packages is not None, so next # successful read of pipxrc does not use default in # get_injected_packages() @@ -163,7 +175,7 @@ def write(self): cls=JsonEncoderPipx, ) - def read(self): + def read(self) -> None: try: with open(self.venv_dir / "pipxrc", "r") as pipxrc_fh: self.pipxrc_info.from_dict( From d0a59e49f940299e3877c1e199ac72e996b6969a Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Tue, 1 Oct 2019 11:39:04 -0700 Subject: [PATCH 034/153] Fix formatting for black. --- pipx/pipxrc.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pipx/pipxrc.py b/pipx/pipxrc.py index 54f9163e74..0abb7db541 100644 --- a/pipx/pipxrc.py +++ b/pipx/pipxrc.py @@ -108,7 +108,9 @@ def get_install_include_dependencies(self, default: bool) -> bool: def get_venv_metadata(self, default: PipxVenvMetadata) -> PipxVenvMetadata: return self._val_or_default(self.pipxrc_info.venv_metadata, default) - def get_injected_packages(self, default: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + def get_injected_packages( + self, default: List[Dict[str, Any]] + ) -> List[Dict[str, Any]]: if self.pipxrc_info.injected_packages is not None: injected_packages = [] for package in self.pipxrc_info.injected_packages: From 692d4146192618ab7f744eb732aa2bfa392559ad Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Tue, 1 Oct 2019 16:39:39 -0700 Subject: [PATCH 035/153] Add _abs_path_if_local() for local pacakge paths. Allows pipxrc to get a absolute pathname from a relative package_or_url path specification. --- src/pipx/commands.py | 53 +++++++++++++++++++++++++++++++++++++++++++- src/pipx/venv.py | 6 +++++ 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/src/pipx/commands.py b/src/pipx/commands.py index 115808d9a8..7888f788af 100644 --- a/src/pipx/commands.py +++ b/src/pipx/commands.py @@ -4,6 +4,7 @@ import hashlib import logging import multiprocessing +import re import shlex import shutil import subprocess @@ -335,13 +336,63 @@ def install( raise # if all is well, write out pipxrc file + package_or_url_pipxrc = _abs_path_if_local(package_or_url, venv, pip_args) pipxrc = Pipxrc(venv_dir, read=False) - pipxrc.set_package_or_url(package_or_url) + pipxrc.set_package_or_url(package_or_url_pipxrc) pipxrc.set_install_options(pip_args, venv_args, include_dependencies) pipxrc.set_venv_metadata(venv.get_venv_metadata_for_package(package)) pipxrc.write() +def _abs_path_if_local(package_or_url: str, venv: Venv, pip_args: List[str]) -> str: + # if not pip --editable, then pip assumes any one-word word spec MUST + # be a pypi package + pkg_path = Path(package_or_url) + if not pkg_path.exists(): + # no existing path, must be pypi package or non-existent + return package_or_url + + valid_pkg_name = bool( + re.search(r"^([A-Z0-9]|[A-Z0-9][A-Z0-9._-]*[A-Z0-9])$", package_or_url, re.I) + ) + + if not valid_pkg_name: + return str(pkg_path.resolve()) + + pip_search_args: List[str] + + try: + arg_i = pip_args.index("--index-url") + except ValueError: + pip_search_args = [] + else: + pip_search_args = pip_args[arg_i : arg_i + 2] + pip_search_result_str = venv.pip_search(package_or_url, pip_search_args) + + pip_search_results = pip_search_result_str.split("\n") + + pkg_found = False + pip_search_found = [] + for pip_search_line in pip_search_results: + if pkg_found: + if re.search(r"^\s", pip_search_line): + pip_search_found.append(pip_search_line) + else: + break + elif pip_search_line.startswith(package_or_url): + pip_search_found.append(pip_search_line) + pkg_found = True + + if ( + len(pip_search_found) > 1 + and pip_search_found[0].startswith(package_or_url) + and "INSTALLED" in pip_search_found[-1] + ): + return package_or_url + else: + return str(pkg_path.resolve()) + + def _run_post_install_actions( venv: Venv, package: str, diff --git a/src/pipx/venv.py b/src/pipx/venv.py index b602c783bd..1fd0be8db9 100644 --- a/src/pipx/venv.py +++ b/src/pipx/venv.py @@ -185,6 +185,12 @@ def get_python_version(self) -> str: .strip() ) + def pip_search(self, search_term, pip_search_args): + cmd = [str(self.python_path), "-m", "pip", "search"] + cmd += pip_search_args + [search_term] + cmd_run = subprocess.run(cmd, stdout=subprocess.PIPE) + return cmd_run.stdout.decode().strip() + def run_app(self, app: str, app_args: List[str]): cmd = [str(self.bin_path / app)] + app_args try: From 5f632b7cd346412bb2bc5f99d8b1dd8b6e0faa52 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Tue, 1 Oct 2019 19:29:33 -0700 Subject: [PATCH 036/153] Clean up of _abs_path_if_local(). Add comments. Also add check for --editable in pip_args, which indicates not a pypi package for sure. --- src/pipx/commands.py | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/src/pipx/commands.py b/src/pipx/commands.py index 7888f788af..986dbd4ca9 100644 --- a/src/pipx/commands.py +++ b/src/pipx/commands.py @@ -345,32 +345,43 @@ def install( def _abs_path_if_local(package_or_url: str, venv: Venv, pip_args: List[str]) -> str: - # if not pip --editable, then pip assumes any one-word word spec MUST - # be a pypi package + # TODO 20191001 if not pip --editable, then pip assumes any one-word word + # spec MUST be a pypi package? + pkg_path = Path(package_or_url) if not pkg_path.exists(): # no existing path, must be pypi package or non-existent return package_or_url + # Editable packages are either local or url, non-url must be local. + if "--editable" in pip_args and pkg_path.exists(): + return str(pkg_path.resolve()) + + # https://www.python.org/dev/peps/pep-0508/#names valid_pkg_name = bool( re.search(r"^([A-Z0-9]|[A-Z0-9][A-Z0-9._-]*[A-Z0-9])$", package_or_url, re.I) ) - if not valid_pkg_name: return str(pkg_path.resolve()) + # If all of the above conditions do not return, we may have used a pypi + # package. + # If we find a pypi package with this name installed, assume we just + # installed it. pip_search_args: List[str] + # If user-defined pypi index url, then use it for search try: arg_i = pip_args.index("--index-url") except ValueError: pip_search_args = [] else: pip_search_args = pip_args[arg_i : arg_i + 2] - pip_search_result_str = venv.pip_search(package_or_url, pip_search_args) + pip_search_result_str = venv.pip_search(package_or_url, pip_search_args) pip_search_results = pip_search_result_str.split("\n") + # Get package_or_url and following related lines from pip search stdout pkg_found = False pip_search_found = [] for pip_search_line in pip_search_results: @@ -382,12 +393,9 @@ def _abs_path_if_local(package_or_url: str, venv: Venv, pip_args: List[str]) -> elif pip_search_line.startswith(package_or_url): pip_search_found.append(pip_search_line) pkg_found = True + pip_found_str = " ".join(pip_search_found) - if ( - len(pip_search_found) > 1 - and pip_search_found[0].startswith(package_or_url) - and "INSTALLED" in pip_search_found[-1] - ): + if pip_found_str.startswith(package_or_url) and "INSTALLED" in pip_found_str: return package_or_url else: return str(pkg_path.resolve()) From 59fcb19976bdf303be4f3069fcbebaaa6b412387 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Tue, 1 Oct 2019 20:46:16 -0700 Subject: [PATCH 037/153] Add more typing to _val_or_default(). --- pipx/pipxrc.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pipx/pipxrc.py b/pipx/pipxrc.py index 0abb7db541..43ee856e9a 100644 --- a/pipx/pipxrc.py +++ b/pipx/pipxrc.py @@ -1,6 +1,6 @@ import json from pathlib import Path -from typing import List, Dict, NamedTuple, Any, Optional +from typing import List, Dict, NamedTuple, Any, Optional, TypeVar from pipx.Venv import PipxVenvMetadata @@ -19,6 +19,10 @@ def _json_decoder_object_hook(json_dict): return json_dict +# Used for consistent types of multiple kinds +Multi = TypeVar("Multi") + + class InjPkg(NamedTuple): pip_args: List[str] verbose: bool @@ -85,7 +89,7 @@ def __init__(self, venv_dir: Path, read: bool = True): def reset(self) -> None: self.pipxrc_info = PipxrcInfo() - def _val_or_default(self, value, default): + def _val_or_default(self, value: Optional[Multi], default: Multi) -> Multi: if value is not None: return value else: From 17776f9fe02d38a642b140a18726a4971ffa241c Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Wed, 16 Oct 2019 15:02:21 -0700 Subject: [PATCH 038/153] Change pipxrc -> pipxrc.json, make pipx info filename a global. --- pipx/pipxrc.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pipx/pipxrc.py b/pipx/pipxrc.py index 43ee856e9a..821adf20a0 100644 --- a/pipx/pipxrc.py +++ b/pipx/pipxrc.py @@ -5,6 +5,9 @@ from pipx.Venv import PipxVenvMetadata +PIPX_INFO_FILENAME = "pipxrc.json" + + class JsonEncoderPipx(json.JSONEncoder): def default(self, obj): # only handles what json.JSONEncoder doesn't understand by default @@ -172,7 +175,7 @@ def write(self) -> None: self.pipxrc_info.injected_packages = {} # TODO 20190919: raise exception on failure? - with open(self.venv_dir / "pipxrc", "w") as pipxrc_fh: + with open(self.venv_dir / PIPX_INFO_FILENAME, "w") as pipxrc_fh: json.dump( self.pipxrc_info.to_dict(), pipxrc_fh, @@ -183,7 +186,7 @@ def write(self) -> None: def read(self) -> None: try: - with open(self.venv_dir / "pipxrc", "r") as pipxrc_fh: + with open(self.venv_dir / PIPX_INFO_FILENAME, "r") as pipxrc_fh: self.pipxrc_info.from_dict( json.load(pipxrc_fh, object_hook=_json_decoder_object_hook) ) From be0c2a903cc4799f18a64af0a8fd2a65ade79b60 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Thu, 17 Oct 2019 22:16:18 -0700 Subject: [PATCH 039/153] Change abbreviated names to full names. --- pipx/pipxrc.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/pipx/pipxrc.py b/pipx/pipxrc.py index 821adf20a0..3178906caf 100644 --- a/pipx/pipxrc.py +++ b/pipx/pipxrc.py @@ -26,7 +26,7 @@ def _json_decoder_object_hook(json_dict): Multi = TypeVar("Multi") -class InjPkg(NamedTuple): +class InjectedPackage(NamedTuple): pip_args: List[str] verbose: bool include_apps: bool @@ -34,7 +34,7 @@ class InjPkg(NamedTuple): force: bool -class InstallOpts(NamedTuple): +class InstallOptions(NamedTuple): pip_args: Optional[List[str]] venv_args: Optional[List[str]] include_dependencies: Optional[bool] @@ -43,11 +43,11 @@ class InstallOpts(NamedTuple): class PipxrcInfo: def __init__(self): self.package_or_url: Optional[str] = None - self.install: InstallOpts = InstallOpts( + self.install: InstallOptions = InstallOptions( pip_args=None, venv_args=None, include_dependencies=None ) self.venv_metadata: Optional[PipxVenvMetadata] = None - self.injected_packages: Optional[Dict[str, InjPkg]] = None + self.injected_packages: Optional[Dict[str, InjectedPackage]] = None self._pipxrc_version: str = "0.1" def to_dict(self) -> Dict[str, Any]: @@ -75,10 +75,11 @@ def to_dict(self) -> Dict[str, Any]: def from_dict(self, pipxrc_info_dict) -> None: self.package_or_url = pipxrc_info_dict["package_or_url"] - self.install = InstallOpts(**pipxrc_info_dict["install"]) + self.install = InstallOptions(**pipxrc_info_dict["install"]) self.venv_metadata = PipxVenvMetadata(**pipxrc_info_dict["venv_metadata"]) self.injected_packages = { - k: InjPkg(**v) for (k, v) in pipxrc_info_dict["injected_packages"].items() + k: InjectedPackage(**v) + for (k, v) in pipxrc_info_dict["injected_packages"].items() } @@ -141,7 +142,7 @@ def set_venv_metadata(self, venv_metadata: PipxVenvMetadata) -> None: def set_install_options( self, pip_args: List[str], venv_args: List[str], include_dependencies: bool ) -> None: - self.pipxrc_info.install = InstallOpts( + self.pipxrc_info.install = InstallOptions( pip_args=pip_args, venv_args=venv_args, include_dependencies=include_dependencies, @@ -159,7 +160,7 @@ def add_injected_package( if self.pipxrc_info.injected_packages is None: self.pipxrc_info.injected_packages = {} - self.pipxrc_info.injected_packages[package] = InjPkg( + self.pipxrc_info.injected_packages[package] = InjectedPackage( pip_args=pip_args, verbose=verbose, include_apps=include_apps, From ad4494cf6933e33fdde246363853ad21bfefc1b2 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Thu, 17 Oct 2019 22:33:01 -0700 Subject: [PATCH 040/153] Remove args from merge that were removed in upgrade_all and reinstall_all. --- src/pipx/main.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/pipx/main.py b/src/pipx/main.py index 560b0f4588..4332147a99 100644 --- a/src/pipx/main.py +++ b/src/pipx/main.py @@ -205,10 +205,19 @@ def run_pipx_command(args): # noqa: C901 elif args.command == "uninstall-all": return commands.uninstall_all(venv_container, constants.LOCAL_BIN_DIR, verbose) elif args.command == "upgrade-all": - return commands.upgrade_all(venv_container, verbose, skip=args.skip, force=args.force) + return commands.upgrade_all( + venv_container, + verbose, + skip=args.skip, + force=args.force, + ) elif args.command == "reinstall-all": return commands.reinstall_all( - venv_container, constants.LOCAL_BIN_DIR, args.python, verbose, skip=args.skip + venv_container, + constants.LOCAL_BIN_DIR, + args.python, + verbose, + skip=args.skip, ) elif args.command == "runpip": if not venv_dir: From f31926368b60fd738c7f6f20d9545dbe5c13a739 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Thu, 17 Oct 2019 22:33:31 -0700 Subject: [PATCH 041/153] Black reformat. --- src/pipx/main.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/pipx/main.py b/src/pipx/main.py index 4332147a99..e8a13b1ad3 100644 --- a/src/pipx/main.py +++ b/src/pipx/main.py @@ -206,10 +206,7 @@ def run_pipx_command(args): # noqa: C901 return commands.uninstall_all(venv_container, constants.LOCAL_BIN_DIR, verbose) elif args.command == "upgrade-all": return commands.upgrade_all( - venv_container, - verbose, - skip=args.skip, - force=args.force, + venv_container, verbose, skip=args.skip, force=args.force ) elif args.command == "reinstall-all": return commands.reinstall_all( From 49be1f462da6c2f144b6f4c9d6e5b3e310272f2d Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Sat, 19 Oct 2019 23:43:51 -0700 Subject: [PATCH 042/153] Updated docs from new help. --- docs/docs.md | 35 ++++------------------------------- 1 file changed, 4 insertions(+), 31 deletions(-) diff --git a/docs/docs.md b/docs/docs.md index 5e078bdff7..daad583b68 100644 --- a/docs/docs.md +++ b/docs/docs.md @@ -206,25 +206,13 @@ optional arguments: ``` pipx upgrade-all --help -usage: pipx upgrade-all [-h] [--include-deps] [--system-site-packages] - [--index-url INDEX_URL] [--editable] - [--pip-args PIP_ARGS] [--skip SKIP [SKIP ...]] - [--force] [--verbose] +usage: pipx upgrade-all [-h] [--skip SKIP [SKIP ...]] [--force] [--verbose] Upgrades all packages within their virtual environments by running 'pip install --upgrade PACKAGE' optional arguments: -h, --help show this help message and exit - --include-deps Include apps of dependent packages - --system-site-packages - Give the virtual environment access to the system - site-packages dir. - --index-url INDEX_URL, -i INDEX_URL - Base URL of Python Package Index - --editable, -e Install a project in editable mode - --pip-args PIP_ARGS Arbitrary pip arguments to pass directly to pip - install/upgrade commands --skip SKIP [SKIP ...] skip these packages --force, -f Modify existing virtual environment and files in @@ -307,35 +295,20 @@ optional arguments: ``` pipx reinstall-all --help -usage: pipx reinstall-all [-h] [--include-deps] [--system-site-packages] - [--index-url INDEX_URL] [--editable] - [--pip-args PIP_ARGS] [--skip SKIP [SKIP ...]] - [--verbose] - python +usage: pipx reinstall-all [-h] [--skip SKIP [SKIP ...]] [--verbose] python Reinstalls all packages using a different version of Python. -Packages are uninstalled, then installed with pipx install PACKAGE. +Packages are uninstalled, then installed with pipx install PACKAGE +with the same options used in the original install of PACKAGE. This is useful if you upgraded to a new version of Python and want all your packages to use the latest as well. -If you originally installed a package from a source other than PyPI, -this command may behave in unexpected ways since it will reinstall from PyPI. - positional arguments: python optional arguments: -h, --help show this help message and exit - --include-deps Include apps of dependent packages - --system-site-packages - Give the virtual environment access to the system - site-packages dir. - --index-url INDEX_URL, -i INDEX_URL - Base URL of Python Package Index - --editable, -e Install a project in editable mode - --pip-args PIP_ARGS Arbitrary pip arguments to pass directly to pip - install/upgrade commands --skip SKIP [SKIP ...] skip these packages --verbose From 86b6ecbfae8ebd6884e4d62af9a6da302e627c05 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Sun, 20 Oct 2019 17:09:00 -0700 Subject: [PATCH 043/153] Rename JsonEncoderPipx -> JsonEncoderHandlesPath. --- pipx/pipxrc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pipx/pipxrc.py b/pipx/pipxrc.py index 3178906caf..9a59454e1f 100644 --- a/pipx/pipxrc.py +++ b/pipx/pipxrc.py @@ -8,7 +8,7 @@ PIPX_INFO_FILENAME = "pipxrc.json" -class JsonEncoderPipx(json.JSONEncoder): +class JsonEncoderHandlesPath(json.JSONEncoder): def default(self, obj): # only handles what json.JSONEncoder doesn't understand by default if isinstance(obj, Path): @@ -182,7 +182,7 @@ def write(self) -> None: pipxrc_fh, indent=4, sort_keys=True, - cls=JsonEncoderPipx, + cls=JsonEncoderHandlesPath, ) def read(self) -> None: From fb0a1d35b176d1f348ce2d4997cec0d92c8d008a Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Sun, 20 Oct 2019 20:07:40 -0700 Subject: [PATCH 044/153] Log warning if failure to read or write pipxrc.json. --- pipx/pipxrc.py | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/pipx/pipxrc.py b/pipx/pipxrc.py index 9a59454e1f..2c45da5745 100644 --- a/pipx/pipxrc.py +++ b/pipx/pipxrc.py @@ -1,5 +1,7 @@ import json +import logging from pathlib import Path +import textwrap from typing import List, Dict, NamedTuple, Any, Optional, TypeVar from pipx.Venv import PipxVenvMetadata @@ -175,15 +177,25 @@ def write(self) -> None: if self.pipxrc_info.injected_packages is None: self.pipxrc_info.injected_packages = {} - # TODO 20190919: raise exception on failure? - with open(self.venv_dir / PIPX_INFO_FILENAME, "w") as pipxrc_fh: - json.dump( - self.pipxrc_info.to_dict(), - pipxrc_fh, - indent=4, - sort_keys=True, - cls=JsonEncoderHandlesPath, + try: + with open(self.venv_dir / PIPX_INFO_FILENAME, "w") as pipxrc_fh: + json.dump( + self.pipxrc_info.to_dict(), + pipxrc_fh, + indent=4, + sort_keys=True, + cls=JsonEncoderHandlesPath, + ) + except IOError: + logging.warning( + textwrap.fill( + f"Unable to write {PIPX_INFO_FILENAME} to {self.venv_dir}. " + f"This may cause future pipx operations involving " + f"{self.venv_dir.name} to fail or behave incorrectly.", + width=79, + ) ) + pass def read(self) -> None: try: @@ -192,5 +204,13 @@ def read(self) -> None: json.load(pipxrc_fh, object_hook=_json_decoder_object_hook) ) except IOError: # Reset self.pipxrc_info if problem reading + logging.warning( + textwrap.fill( + f"Unable to read {PIPX_INFO_FILENAME} in {self.venv_dir}. " + f"This may cause this or future pipx operations involving " + f"{self.venv_dir.name} to fail or behave incorrectly.", + width=79, + ) + ) self.reset() return From ed53006af0dbefe31b197ef1e4bbd53c0141231c Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Sun, 20 Oct 2019 20:23:00 -0700 Subject: [PATCH 045/153] New comments, remove old TODO. --- pipx/pipxrc.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pipx/pipxrc.py b/pipx/pipxrc.py index 2c45da5745..bbb0dbeaf2 100644 --- a/pipx/pipxrc.py +++ b/pipx/pipxrc.py @@ -42,6 +42,10 @@ class InstallOptions(NamedTuple): include_dependencies: Optional[bool] +# PipxrcInfo members start with the value None to indicate the information is +# missing. This means either a pipxrc.json file has never been read, or a new +# PipxrcInfo object was created and the information was not filled in +# properly. class PipxrcInfo: def __init__(self): self.package_or_url: Optional[str] = None @@ -134,8 +138,7 @@ def get_injected_packages( return default def set_package_or_url(self, package_or_url: str) -> None: - # TODO 20190923: if package_or_url is a local path, we need to make it - # an absolute path + # if package_or_url is a local path, it MUST be an absolute path self.pipxrc_info.package_or_url = package_or_url def set_venv_metadata(self, venv_metadata: PipxVenvMetadata) -> None: From 2e0350b28b705490edb407a09d5aafe4a19d5628 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Sun, 20 Oct 2019 21:02:59 -0700 Subject: [PATCH 046/153] Change get_injected_packages to return Dict[str,InjectedPackage] --- pipx/pipxrc.py | 18 +++++------------- src/pipx/commands.py | 16 +++++++++------- 2 files changed, 14 insertions(+), 20 deletions(-) diff --git a/pipx/pipxrc.py b/pipx/pipxrc.py index bbb0dbeaf2..89993af151 100644 --- a/pipx/pipxrc.py +++ b/pipx/pipxrc.py @@ -123,19 +123,9 @@ def get_venv_metadata(self, default: PipxVenvMetadata) -> PipxVenvMetadata: return self._val_or_default(self.pipxrc_info.venv_metadata, default) def get_injected_packages( - self, default: List[Dict[str, Any]] - ) -> List[Dict[str, Any]]: - if self.pipxrc_info.injected_packages is not None: - injected_packages = [] - for package in self.pipxrc_info.injected_packages: - package_info = {"package": package} - package_info.update( - self.pipxrc_info.injected_packages[package]._asdict() - ) - injected_packages.append(package_info) - return injected_packages - else: - return default + self, default: Dict[str, InjectedPackage] + ) -> Dict[str, InjectedPackage]: + return self._val_or_default(self.pipxrc_info.injected_packages, default) def set_package_or_url(self, package_or_url: str) -> None: # if package_or_url is a local path, it MUST be an absolute path @@ -217,3 +207,5 @@ def read(self) -> None: ) self.reset() return + + # TODO 20191020: Move _abs_path_if_local here? diff --git a/src/pipx/commands.py b/src/pipx/commands.py index 986dbd4ca9..1c7f865ead 100644 --- a/src/pipx/commands.py +++ b/src/pipx/commands.py @@ -588,15 +588,17 @@ def reinstall_all( force=True, include_dependencies=pipxrc.get_install_include_dependencies(default=False), ) - for injected in pipxrc.get_injected_packages(default=[]): + for (injected_package, package_specs) in pipxrc.get_injected_packages( + default={} + ).items(): inject( venv_dir, - injected["package"], - injected["pip_args"], - verbose=injected["verbose"], - include_apps=injected["include_apps"], - include_dependencies=injected["include_dependencies"], - force=injected["force"], + injected_package, + package_specs.pip_args, + verbose=package_specs.verbose, + include_apps=package_specs.include_apps, + include_dependencies=package_specs.include_dependencies, + force=package_specs.force, ) From 428b2f8899e88e3db7e89a992a79c7f7a61131eb Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Mon, 21 Oct 2019 17:31:13 -0700 Subject: [PATCH 047/153] Add colon after TODO. --- src/pipx/commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pipx/commands.py b/src/pipx/commands.py index 1c7f865ead..a92aa6dc3d 100644 --- a/src/pipx/commands.py +++ b/src/pipx/commands.py @@ -345,7 +345,7 @@ def install( def _abs_path_if_local(package_or_url: str, venv: Venv, pip_args: List[str]) -> str: - # TODO 20191001 if not pip --editable, then pip assumes any one-word word + # TODO 20191001: if not pip --editable, then pip assumes any one-word word # spec MUST be a pypi package? pkg_path = Path(package_or_url) From cd026662a3a0b7fc22651f16d82e571fd4d26e40 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Mon, 21 Oct 2019 19:51:35 -0700 Subject: [PATCH 048/153] Resolved pip --editable TODO. --- src/pipx/commands.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/pipx/commands.py b/src/pipx/commands.py index a92aa6dc3d..a88c9ca4cd 100644 --- a/src/pipx/commands.py +++ b/src/pipx/commands.py @@ -345,15 +345,13 @@ def install( def _abs_path_if_local(package_or_url: str, venv: Venv, pip_args: List[str]) -> str: - # TODO 20191001: if not pip --editable, then pip assumes any one-word word - # spec MUST be a pypi package? - pkg_path = Path(package_or_url) if not pkg_path.exists(): # no existing path, must be pypi package or non-existent return package_or_url # Editable packages are either local or url, non-url must be local. + # https://pip.pypa.io/en/stable/reference/pip_install/#editable-installs if "--editable" in pip_args and pkg_path.exists(): return str(pkg_path.resolve()) From d5dea0e6e45a7a50acc9135bbaa7456b5019b249 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Mon, 21 Oct 2019 19:53:29 -0700 Subject: [PATCH 049/153] Remove TODO, keep _abs_path_if_local in commands.py. --- pipx/pipxrc.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pipx/pipxrc.py b/pipx/pipxrc.py index 89993af151..1064630d22 100644 --- a/pipx/pipxrc.py +++ b/pipx/pipxrc.py @@ -207,5 +207,3 @@ def read(self) -> None: ) self.reset() return - - # TODO 20191020: Move _abs_path_if_local here? From 9a1dc0ae3dd0fd2294cda4091131ec5c07571ecc Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Thu, 24 Oct 2019 17:46:45 -0700 Subject: [PATCH 050/153] Move abs_path_if_local to pipxrc. --- pipx/pipxrc.py | 61 +++++++++++++++++++++++++++++++++++++++++++- src/pipx/commands.py | 60 ++----------------------------------------- 2 files changed, 62 insertions(+), 59 deletions(-) diff --git a/pipx/pipxrc.py b/pipx/pipxrc.py index 1064630d22..6a6fd3cf5a 100644 --- a/pipx/pipxrc.py +++ b/pipx/pipxrc.py @@ -1,10 +1,11 @@ import json import logging from pathlib import Path +import re import textwrap from typing import List, Dict, NamedTuple, Any, Optional, TypeVar -from pipx.Venv import PipxVenvMetadata +from pipx.Venv import PipxVenvMetadata, Venv PIPX_INFO_FILENAME = "pipxrc.json" @@ -207,3 +208,61 @@ def read(self) -> None: ) self.reset() return + + +def abs_path_if_local(package_or_url: str, venv: Venv, pip_args: List[str]) -> str: + """Return the absolute path if package_or_url represents a filepath + and not a pypi package + """ + pkg_path = Path(package_or_url) + if not pkg_path.exists(): + # no existing path, must be pypi package or non-existent + return package_or_url + + # Editable packages are either local or url, non-url must be local. + # https://pip.pypa.io/en/stable/reference/pip_install/#editable-installs + if "--editable" in pip_args and pkg_path.exists(): + return str(pkg_path.resolve()) + + # https://www.python.org/dev/peps/pep-0508/#names + valid_pkg_name = bool( + re.search(r"^([A-Z0-9]|[A-Z0-9][A-Z0-9._-]*[A-Z0-9])$", package_or_url, re.I) + ) + if not valid_pkg_name: + return str(pkg_path.resolve()) + + # If all of the above conditions do not return, we may have used a pypi + # package. + # If we find a pypi package with this name installed, assume we just + # installed it. + pip_search_args: List[str] + + # If user-defined pypi index url, then use it for search + try: + arg_i = pip_args.index("--index-url") + except ValueError: + pip_search_args = [] + else: + pip_search_args = pip_args[arg_i : arg_i + 2] + + pip_search_result_str = venv.pip_search(package_or_url, pip_search_args) + pip_search_results = pip_search_result_str.split("\n") + + # Get package_or_url and following related lines from pip search stdout + pkg_found = False + pip_search_found = [] + for pip_search_line in pip_search_results: + if pkg_found: + if re.search(r"^\s", pip_search_line): + pip_search_found.append(pip_search_line) + else: + break + elif pip_search_line.startswith(package_or_url): + pip_search_found.append(pip_search_line) + pkg_found = True + pip_found_str = " ".join(pip_search_found) + + if pip_found_str.startswith(package_or_url) and "INSTALLED" in pip_found_str: + return package_or_url + else: + return str(pkg_path.resolve()) diff --git a/src/pipx/commands.py b/src/pipx/commands.py index a88c9ca4cd..f1b92abf20 100644 --- a/src/pipx/commands.py +++ b/src/pipx/commands.py @@ -4,7 +4,6 @@ import hashlib import logging import multiprocessing -import re import shlex import shutil import subprocess @@ -31,7 +30,7 @@ rmdir, run_pypackage_bin, ) -from pipx.pipxrc import Pipxrc +from pipx.pipxrc import Pipxrc, abs_path_if_local from pipx.venv import Venv, VenvContainer @@ -336,7 +335,7 @@ def install( raise # if all is well, write out pipxrc file - package_or_url_pipxrc = _abs_path_if_local(package_or_url, venv, pip_args) + package_or_url_pipxrc = abs_path_if_local(package_or_url, venv, pip_args) pipxrc = Pipxrc(venv_dir, read=False) pipxrc.set_package_or_url(package_or_url_pipxrc) pipxrc.set_install_options(pip_args, venv_args, include_dependencies) @@ -344,61 +343,6 @@ def install( pipxrc.write() -def _abs_path_if_local(package_or_url: str, venv: Venv, pip_args: List[str]) -> str: - pkg_path = Path(package_or_url) - if not pkg_path.exists(): - # no existing path, must be pypi package or non-existent - return package_or_url - - # Editable packages are either local or url, non-url must be local. - # https://pip.pypa.io/en/stable/reference/pip_install/#editable-installs - if "--editable" in pip_args and pkg_path.exists(): - return str(pkg_path.resolve()) - - # https://www.python.org/dev/peps/pep-0508/#names - valid_pkg_name = bool( - re.search(r"^([A-Z0-9]|[A-Z0-9][A-Z0-9._-]*[A-Z0-9])$", package_or_url, re.I) - ) - if not valid_pkg_name: - return str(pkg_path.resolve()) - - # If all of the above conditions do not return, we may have used a pypi - # package. - # If we find a pypi package with this name installed, assume we just - # installed it. - pip_search_args: List[str] - - # If user-defined pypi index url, then use it for search - try: - arg_i = pip_args.index("--index-url") - except ValueError: - pip_search_args = [] - else: - pip_search_args = pip_args[arg_i : arg_i + 2] - - pip_search_result_str = venv.pip_search(package_or_url, pip_search_args) - pip_search_results = pip_search_result_str.split("\n") - - # Get package_or_url and following related lines from pip search stdout - pkg_found = False - pip_search_found = [] - for pip_search_line in pip_search_results: - if pkg_found: - if re.search(r"^\s", pip_search_line): - pip_search_found.append(pip_search_line) - else: - break - elif pip_search_line.startswith(package_or_url): - pip_search_found.append(pip_search_line) - pkg_found = True - pip_found_str = " ".join(pip_search_found) - - if pip_found_str.startswith(package_or_url) and "INSTALLED" in pip_found_str: - return package_or_url - else: - return str(pkg_path.resolve()) - - def _run_post_install_actions( venv: Venv, package: str, From 2ed291145fb8ae6ebe43be0a09fbc9f6fe525926 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Thu, 24 Oct 2019 23:29:04 -0700 Subject: [PATCH 051/153] Remove getter/setter, remove PipxrcInfo, refactor. Some huge changes here. Much smaller implementation. Separate classes / data structures combined. Defaults are now initialized in Pipxrc where they can be known. --- pipx/pipxrc.py | 173 ++++++++++++------------------------------- src/pipx/commands.py | 85 +++++++++++++-------- 2 files changed, 104 insertions(+), 154 deletions(-) diff --git a/pipx/pipxrc.py b/pipx/pipxrc.py index 6a6fd3cf5a..588426b2bb 100644 --- a/pipx/pipxrc.py +++ b/pipx/pipxrc.py @@ -3,9 +3,10 @@ from pathlib import Path import re import textwrap -from typing import List, Dict, NamedTuple, Any, Optional, TypeVar +from typing import List, Dict, NamedTuple, Any, Optional from pipx.Venv import PipxVenvMetadata, Venv +from pipx.util import PipxError PIPX_INFO_FILENAME = "pipxrc.json" @@ -25,156 +26,80 @@ def _json_decoder_object_hook(json_dict): return json_dict -# Used for consistent types of multiple kinds -Multi = TypeVar("Multi") - - -class InjectedPackage(NamedTuple): +class PackageInfo(NamedTuple): + package_or_url: Optional[str] pip_args: List[str] - verbose: bool - include_apps: bool include_dependencies: bool - force: bool - + include_apps: bool -class InstallOptions(NamedTuple): - pip_args: Optional[List[str]] - venv_args: Optional[List[str]] - include_dependencies: Optional[bool] +class Pipxrc: + def __init__(self, venv_dir: Path, read: bool = True): + self.venv_dir = venv_dir + self.reset() + if read: + self.read() -# PipxrcInfo members start with the value None to indicate the information is -# missing. This means either a pipxrc.json file has never been read, or a new -# PipxrcInfo object was created and the information was not filled in -# properly. -class PipxrcInfo: - def __init__(self): - self.package_or_url: Optional[str] = None - self.install: InstallOptions = InstallOptions( - pip_args=None, venv_args=None, include_dependencies=None + def reset(self) -> None: + # We init this instance with reasonable fallback defaults for all + # members, EXCEPT for those we cannot know: + # self.main_package.package_or_url=None + # self.venv_metadata.package_or_url=None + self.main_package = PackageInfo( + package_or_url=None, + pip_args=[], + include_dependencies=False, + include_apps=True, # always True for main_package ) self.venv_metadata: Optional[PipxVenvMetadata] = None - self.injected_packages: Optional[Dict[str, InjectedPackage]] = None + self.venv_args: List[str] = [] + self.injected_packages: List[PackageInfo] = [] + # Only change this if file format changes self._pipxrc_version: str = "0.1" def to_dict(self) -> Dict[str, Any]: venv_metadata: Optional[Dict[str, Any]] - injected_packages: Optional[Dict[str, Dict[str, Any]]] - if self.venv_metadata is not None: venv_metadata = self.venv_metadata._asdict() else: venv_metadata = None - if self.injected_packages is not None: - injected_packages = { - k: v._asdict() for (k, v) in self.injected_packages.items() - } - else: - injected_packages = None return { - "package_or_url": self.package_or_url, - "install": self.install._asdict(), + "main_package": self.main_package._asdict(), "venv_metadata": venv_metadata, - "injected_packages": injected_packages, + "venv_args": self.venv_args, + "injected_packages": [x._asdict() for x in self.injected_packages], "pipxrc_version": self._pipxrc_version, } - def from_dict(self, pipxrc_info_dict) -> None: - self.package_or_url = pipxrc_info_dict["package_or_url"] - self.install = InstallOptions(**pipxrc_info_dict["install"]) - self.venv_metadata = PipxVenvMetadata(**pipxrc_info_dict["venv_metadata"]) - self.injected_packages = { - k: InjectedPackage(**v) - for (k, v) in pipxrc_info_dict["injected_packages"].items() - } - - -class Pipxrc: - def __init__(self, venv_dir: Path, read: bool = True): - self.venv_dir = venv_dir - self.pipxrc_info = PipxrcInfo() - if read: - self.read() - - def reset(self) -> None: - self.pipxrc_info = PipxrcInfo() - - def _val_or_default(self, value: Optional[Multi], default: Multi) -> Multi: - if value is not None: - return value + def from_dict(self, input_dict: Dict[str, Any]) -> None: + venv_metadata: Optional[PipxVenvMetadata] + if input_dict["venv_metadata"] is not None: + venv_metadata = PipxVenvMetadata(**input_dict["venv_metadata"]) else: - return default - - def get_package_or_url(self, default: str) -> str: - return self._val_or_default(self.pipxrc_info.package_or_url, default) - - def get_install_pip_args(self, default: List[str]) -> List[str]: - return self._val_or_default(self.pipxrc_info.install.pip_args, default) - - def get_install_venv_args(self, default: List[str]) -> List[str]: - return self._val_or_default(self.pipxrc_info.install.venv_args, default) - - def get_install_include_dependencies(self, default: bool) -> bool: - return self._val_or_default( - self.pipxrc_info.install.include_dependencies, default - ) - - def get_venv_metadata(self, default: PipxVenvMetadata) -> PipxVenvMetadata: - return self._val_or_default(self.pipxrc_info.venv_metadata, default) - - def get_injected_packages( - self, default: Dict[str, InjectedPackage] - ) -> Dict[str, InjectedPackage]: - return self._val_or_default(self.pipxrc_info.injected_packages, default) - - def set_package_or_url(self, package_or_url: str) -> None: - # if package_or_url is a local path, it MUST be an absolute path - self.pipxrc_info.package_or_url = package_or_url - - def set_venv_metadata(self, venv_metadata: PipxVenvMetadata) -> None: - self.pipxrc_info.venv_metadata = venv_metadata - - def set_install_options( - self, pip_args: List[str], venv_args: List[str], include_dependencies: bool - ) -> None: - self.pipxrc_info.install = InstallOptions( - pip_args=pip_args, - venv_args=venv_args, - include_dependencies=include_dependencies, - ) - - def add_injected_package( - self, - package: str, - pip_args: List[str], - verbose: bool, - include_apps: bool, - include_dependencies: bool, - force: bool, - ) -> None: - if self.pipxrc_info.injected_packages is None: - self.pipxrc_info.injected_packages = {} + venv_metadata = None - self.pipxrc_info.injected_packages[package] = InjectedPackage( - pip_args=pip_args, - verbose=verbose, - include_apps=include_apps, - include_dependencies=include_dependencies, - force=force, - ) + self.main_package = PackageInfo(**input_dict["main_package"]) + self.venv_metadata = venv_metadata + self.venv_args = input_dict["venv_args"] + self.injected_packages = [ + PackageInfo(**x) for x in input_dict["injected_packages"] + ] + + def validate_before_write(self): + if ( + self.venv_metadata is None + or self.main_package.package_or_url is None + or not self.main_package.include_apps + ): + raise PipxError("Internal Error: Pipxrc data is corrupt, cannot write.") def write(self) -> None: - # If writing out, make sure injected_packages is not None, so next - # successful read of pipxrc does not use default in - # get_injected_packages() - if self.pipxrc_info.injected_packages is None: - self.pipxrc_info.injected_packages = {} - + self.validate_before_write() try: with open(self.venv_dir / PIPX_INFO_FILENAME, "w") as pipxrc_fh: json.dump( - self.pipxrc_info.to_dict(), + self.to_dict(), pipxrc_fh, indent=4, sort_keys=True, @@ -194,7 +119,7 @@ def write(self) -> None: def read(self) -> None: try: with open(self.venv_dir / PIPX_INFO_FILENAME, "r") as pipxrc_fh: - self.pipxrc_info.from_dict( + self.from_dict( json.load(pipxrc_fh, object_hook=_json_decoder_object_hook) ) except IOError: # Reset self.pipxrc_info if problem reading diff --git a/src/pipx/commands.py b/src/pipx/commands.py index f1b92abf20..12093a7a7f 100644 --- a/src/pipx/commands.py +++ b/src/pipx/commands.py @@ -30,7 +30,7 @@ rmdir, run_pypackage_bin, ) -from pipx.pipxrc import Pipxrc, abs_path_if_local +from pipx.pipxrc import Pipxrc, abs_path_if_local, PackageInfo from pipx.venv import Venv, VenvContainer @@ -215,7 +215,10 @@ def upgrade( # TODO 20190926: main.py should communicate if this is spec or copied from # package if package_or_url == package: - package_or_url = pipxrc.get_package_or_url(default=package) + if pipxrc.main_package.package_or_url is not None: + package_or_url = pipxrc.main_package.package_or_url + else: + package_or_url = package # Upgrade shared libraries (pip, setuptools and wheel) venv.upgrade_packaging_libraries(pip_args) @@ -246,7 +249,7 @@ def upgrade( print( f"upgraded package {package} from {old_version} to {new_version} (location: {str(venv_dir)})" ) - pipxrc.set_venv_metadata(venv.get_venv_metadata_for_package(package)) + pipxrc.venv_metadata = venv.get_venv_metadata_for_package(package) pipxrc.write() return 1 @@ -265,20 +268,23 @@ def upgrade_all( if package == "pipx": package_or_url = PIPX_PACKAGE_NAME else: - package_or_url = pipxrc.get_package_or_url(default=package) + if pipxrc.main_package.package_or_url is not None: + package_or_url = pipxrc.main_package.package_or_url + else: + package_or_url = package try: packages_upgraded += upgrade( venv_dir, package, package_or_url, - pipxrc.get_install_pip_args(default=[]), + pipxrc.main_package.pip_args, verbose, upgrading_all=True, - include_dependencies=pipxrc.get_install_include_dependencies( - default=False - ), + include_dependencies=pipxrc.main_package.include_dependencies, force=force, ) + # TODO 20191024: Upgrade injected packages + except Exception: logging.error(f"Error encountered when upgrading {package}") @@ -335,11 +341,15 @@ def install( raise # if all is well, write out pipxrc file - package_or_url_pipxrc = abs_path_if_local(package_or_url, venv, pip_args) pipxrc = Pipxrc(venv_dir, read=False) - pipxrc.set_package_or_url(package_or_url_pipxrc) - pipxrc.set_install_options(pip_args, venv_args, include_dependencies) - pipxrc.set_venv_metadata(venv.get_venv_metadata_for_package(package)) + pipxrc.main_package = PackageInfo( + package_or_url=abs_path_if_local(package_or_url, venv, pip_args), + pip_args=pip_args, + include_dependencies=include_dependencies, + include_apps=True, + ) + pipxrc.venv_args = venv_args + pipxrc.venv_metadata = venv.get_venv_metadata_for_package(package) pipxrc.write() @@ -452,8 +462,13 @@ def inject( force=force, ) pipxrc = Pipxrc(venv_dir) - pipxrc.add_injected_package( - package, pip_args, verbose, include_apps, include_dependencies, force + pipxrc.injected_packages.append( + PackageInfo( + package_or_url=package, + pip_args=pip_args, + include_apps=include_apps, + include_dependencies=include_dependencies, + ) ) pipxrc.write() @@ -471,12 +486,16 @@ def uninstall(venv_dir: Path, package: str, local_bin_dir: Path, verbose: bool): ) return + # TODO 20191024: Uninstall injected packages + venv = Venv(venv_dir, verbose=verbose) pipxrc = Pipxrc(venv_dir) - metadata = pipxrc.get_venv_metadata( - default=venv.get_venv_metadata_for_package(package) - ) + if pipxrc.venv_metadata is not None: + metadata = pipxrc.venv_metadata + else: + metadata = venv.get_venv_metadata_for_package(package) + app_paths = metadata.app_paths for dep_paths in metadata.app_paths_of_dependencies.values(): app_paths += dep_paths @@ -517,30 +536,36 @@ def reinstall_all( pipxrc = Pipxrc(venv_dir) uninstall(venv_dir, package, local_bin_dir, verbose) - package_or_url = pipxrc.get_package_or_url(default=package) + if pipxrc.main_package.package_or_url is not None: + package_or_url = pipxrc.main_package.package_or_url + else: + package_or_url = package + install( venv_dir, package, package_or_url, local_bin_dir, python, - pipxrc.get_install_pip_args(default=[]), - pipxrc.get_install_venv_args(default=[]), + pipxrc.main_package.pip_args, + pipxrc.venv_args, verbose, force=True, - include_dependencies=pipxrc.get_install_include_dependencies(default=False), + include_dependencies=pipxrc.main_package.include_dependencies, ) - for (injected_package, package_specs) in pipxrc.get_injected_packages( - default={} - ).items(): + for injected_package in pipxrc.injected_packages: + if injected_package.package_or_url is None: + # This should never happen, but package_or_url is type + # Optional[str] so mypy thinks it could be None + raise PipxError("Internal Error injecting package") inject( venv_dir, - injected_package, - package_specs.pip_args, - verbose=package_specs.verbose, - include_apps=package_specs.include_apps, - include_dependencies=package_specs.include_dependencies, - force=package_specs.force, + injected_package.package_or_url, + injected_package.pip_args, + verbose=verbose, + include_apps=injected_package.include_apps, + include_dependencies=injected_package.include_dependencies, + force=True, ) From 2e6af1b5cc5e265efb55af7683d8d1f18e1a01c2 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Fri, 25 Oct 2019 15:27:06 -0700 Subject: [PATCH 052/153] Init all member variables in Pipxrc.__init__() --- pipx/pipxrc.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/pipx/pipxrc.py b/pipx/pipxrc.py index 588426b2bb..67029884a6 100644 --- a/pipx/pipxrc.py +++ b/pipx/pipxrc.py @@ -36,7 +36,19 @@ class PackageInfo(NamedTuple): class Pipxrc: def __init__(self, venv_dir: Path, read: bool = True): self.venv_dir = venv_dir - self.reset() + self.main_package = PackageInfo( + package_or_url=None, + pip_args=[], + include_dependencies=False, + include_apps=True, # always True for main_package + ) + self.venv_metadata: Optional[PipxVenvMetadata] = None + self.venv_args: List[str] = [] + self.injected_packages: List[PackageInfo] = [] + + # Only change this if file format changes + self._pipxrc_version: str = "0.1" + if read: self.read() @@ -54,8 +66,6 @@ def reset(self) -> None: self.venv_metadata: Optional[PipxVenvMetadata] = None self.venv_args: List[str] = [] self.injected_packages: List[PackageInfo] = [] - # Only change this if file format changes - self._pipxrc_version: str = "0.1" def to_dict(self) -> Dict[str, Any]: venv_metadata: Optional[Dict[str, Any]] From 5a489fef88d74468b08941ecf9ec66d5c885f8c7 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Fri, 25 Oct 2019 15:34:16 -0700 Subject: [PATCH 053/153] Fix last commit, dont redeclare types in reset. --- pipx/pipxrc.py | 6 +++--- tests/conftest.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pipx/pipxrc.py b/pipx/pipxrc.py index 67029884a6..3e2fbe2e12 100644 --- a/pipx/pipxrc.py +++ b/pipx/pipxrc.py @@ -63,9 +63,9 @@ def reset(self) -> None: include_dependencies=False, include_apps=True, # always True for main_package ) - self.venv_metadata: Optional[PipxVenvMetadata] = None - self.venv_args: List[str] = [] - self.injected_packages: List[PackageInfo] = [] + self.venv_metadata = None + self.venv_args = [] + self.injected_packages = [] def to_dict(self) -> Dict[str, Any]: venv_metadata: Optional[Dict[str, Any]] diff --git a/tests/conftest.py b/tests/conftest.py index f3d1c64b06..eb9ef8add8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -21,4 +21,4 @@ def pipx_temp_env(tmp_path, monkeypatch): monkeypatch.setattr(constants, "LOCAL_BIN_DIR", bin_dir) monkeypatch.setattr(constants, "PIPX_LOCAL_VENVS", home_dir / "venvs") - monkeypatch.setenv("PATH", str(bin_dir)) + monkeypatch.setenv("PATH", str(bin_dir) + ":/bin:/usr/bin") From e594e8919fca67bc4b2f516edbc4a9589f69fc33 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Sat, 26 Oct 2019 19:12:38 -0700 Subject: [PATCH 054/153] Rename Pipxrc, PipxVenvMetadata objects. Rename: Pipxrc -> PipxMetadata, pipxrc -> pipx_metadata, pipxrc.json -> pipx_metadata.json, PipxVenvMetadata -> VenvMetadata, --- pipx/pipxrc.py | 28 ++++++++++---------- src/pipx/commands.py | 62 ++++++++++++++++++++++---------------------- src/pipx/venv.py | 6 ++--- 3 files changed, 48 insertions(+), 48 deletions(-) diff --git a/pipx/pipxrc.py b/pipx/pipxrc.py index 3e2fbe2e12..86d6179812 100644 --- a/pipx/pipxrc.py +++ b/pipx/pipxrc.py @@ -5,11 +5,11 @@ import textwrap from typing import List, Dict, NamedTuple, Any, Optional -from pipx.Venv import PipxVenvMetadata, Venv +from pipx.Venv import VenvMetadata, Venv from pipx.util import PipxError -PIPX_INFO_FILENAME = "pipxrc.json" +PIPX_INFO_FILENAME = "pipx_metadata.json" class JsonEncoderHandlesPath(json.JSONEncoder): @@ -33,7 +33,7 @@ class PackageInfo(NamedTuple): include_apps: bool -class Pipxrc: +class PipxMetadata: def __init__(self, venv_dir: Path, read: bool = True): self.venv_dir = venv_dir self.main_package = PackageInfo( @@ -42,12 +42,12 @@ def __init__(self, venv_dir: Path, read: bool = True): include_dependencies=False, include_apps=True, # always True for main_package ) - self.venv_metadata: Optional[PipxVenvMetadata] = None + self.venv_metadata: Optional[VenvMetadata] = None self.venv_args: List[str] = [] self.injected_packages: List[PackageInfo] = [] # Only change this if file format changes - self._pipxrc_version: str = "0.1" + self._pipx_metadata_version: str = "0.1" if read: self.read() @@ -79,13 +79,13 @@ def to_dict(self) -> Dict[str, Any]: "venv_metadata": venv_metadata, "venv_args": self.venv_args, "injected_packages": [x._asdict() for x in self.injected_packages], - "pipxrc_version": self._pipxrc_version, + "pipx_metadata_version": self._pipx_metadata_version, } def from_dict(self, input_dict: Dict[str, Any]) -> None: - venv_metadata: Optional[PipxVenvMetadata] + venv_metadata: Optional[VenvMetadata] if input_dict["venv_metadata"] is not None: - venv_metadata = PipxVenvMetadata(**input_dict["venv_metadata"]) + venv_metadata = VenvMetadata(**input_dict["venv_metadata"]) else: venv_metadata = None @@ -102,15 +102,15 @@ def validate_before_write(self): or self.main_package.package_or_url is None or not self.main_package.include_apps ): - raise PipxError("Internal Error: Pipxrc data is corrupt, cannot write.") + raise PipxError("Internal Error: PipxMetadata is corrupt, cannot write.") def write(self) -> None: self.validate_before_write() try: - with open(self.venv_dir / PIPX_INFO_FILENAME, "w") as pipxrc_fh: + with open(self.venv_dir / PIPX_INFO_FILENAME, "w") as pipx_metadata_fh: json.dump( self.to_dict(), - pipxrc_fh, + pipx_metadata_fh, indent=4, sort_keys=True, cls=JsonEncoderHandlesPath, @@ -128,11 +128,11 @@ def write(self) -> None: def read(self) -> None: try: - with open(self.venv_dir / PIPX_INFO_FILENAME, "r") as pipxrc_fh: + with open(self.venv_dir / PIPX_INFO_FILENAME, "r") as pipx_metadata_fh: self.from_dict( - json.load(pipxrc_fh, object_hook=_json_decoder_object_hook) + json.load(pipx_metadata_fh, object_hook=_json_decoder_object_hook) ) - except IOError: # Reset self.pipxrc_info if problem reading + except IOError: # Reset self if problem reading logging.warning( textwrap.fill( f"Unable to read {PIPX_INFO_FILENAME} in {self.venv_dir}. " diff --git a/src/pipx/commands.py b/src/pipx/commands.py index 12093a7a7f..f2b16664be 100644 --- a/src/pipx/commands.py +++ b/src/pipx/commands.py @@ -30,7 +30,7 @@ rmdir, run_pypackage_bin, ) -from pipx.pipxrc import Pipxrc, abs_path_if_local, PackageInfo +from pipx.pipxrc import PipxMetadata, abs_path_if_local, PackageInfo from pipx.venv import Venv, VenvContainer @@ -207,16 +207,16 @@ def upgrade( ) venv = Venv(venv_dir, verbose=verbose) - pipxrc = Pipxrc(venv_dir) + pipx_metadata = PipxMetadata(venv_dir) old_version = venv.get_venv_metadata_for_package(package).package_version - # if default package_or_url, check pipxrc for better url + # if default package_or_url, check pipx_metadata for better url # TODO 20190926: main.py should communicate if this is spec or copied from # package if package_or_url == package: - if pipxrc.main_package.package_or_url is not None: - package_or_url = pipxrc.main_package.package_or_url + if pipx_metadata.main_package.package_or_url is not None: + package_or_url = pipx_metadata.main_package.package_or_url else: package_or_url = package @@ -249,8 +249,8 @@ def upgrade( print( f"upgraded package {package} from {old_version} to {new_version} (location: {str(venv_dir)})" ) - pipxrc.venv_metadata = venv.get_venv_metadata_for_package(package) - pipxrc.write() + pipx_metadata.venv_metadata = venv.get_venv_metadata_for_package(package) + pipx_metadata.write() return 1 @@ -262,14 +262,14 @@ def upgrade_all( for venv_dir in venv_container.iter_venv_dirs(): num_packages += 1 package = venv_dir.name - pipxrc = Pipxrc(venv_dir) + pipx_metadata = PipxMetadata(venv_dir) if package in skip: continue if package == "pipx": package_or_url = PIPX_PACKAGE_NAME else: - if pipxrc.main_package.package_or_url is not None: - package_or_url = pipxrc.main_package.package_or_url + if pipx_metadata.main_package.package_or_url is not None: + package_or_url = pipx_metadata.main_package.package_or_url else: package_or_url = package try: @@ -277,10 +277,10 @@ def upgrade_all( venv_dir, package, package_or_url, - pipxrc.main_package.pip_args, + pipx_metadata.main_package.pip_args, verbose, upgrading_all=True, - include_dependencies=pipxrc.main_package.include_dependencies, + include_dependencies=pipx_metadata.main_package.include_dependencies, force=force, ) # TODO 20191024: Upgrade injected packages @@ -340,17 +340,17 @@ def install( venv.remove_venv() raise - # if all is well, write out pipxrc file - pipxrc = Pipxrc(venv_dir, read=False) - pipxrc.main_package = PackageInfo( + # if all is well, write out pipx_metadata file + pipx_metadata = PipxMetadata(venv_dir, read=False) + pipx_metadata.main_package = PackageInfo( package_or_url=abs_path_if_local(package_or_url, venv, pip_args), pip_args=pip_args, include_dependencies=include_dependencies, include_apps=True, ) - pipxrc.venv_args = venv_args - pipxrc.venv_metadata = venv.get_venv_metadata_for_package(package) - pipxrc.write() + pipx_metadata.venv_args = venv_args + pipx_metadata.venv_metadata = venv.get_venv_metadata_for_package(package) + pipx_metadata.write() def _run_post_install_actions( @@ -461,8 +461,8 @@ def inject( include_dependencies, force=force, ) - pipxrc = Pipxrc(venv_dir) - pipxrc.injected_packages.append( + pipx_metadata = PipxMetadata(venv_dir) + pipx_metadata.injected_packages.append( PackageInfo( package_or_url=package, pip_args=pip_args, @@ -470,7 +470,7 @@ def inject( include_dependencies=include_dependencies, ) ) - pipxrc.write() + pipx_metadata.write() print(f" injected package {bold(package)} into venv {bold(venv_dir.name)}") print(f"done! {stars}", file=sys.stderr) @@ -489,10 +489,10 @@ def uninstall(venv_dir: Path, package: str, local_bin_dir: Path, verbose: bool): # TODO 20191024: Uninstall injected packages venv = Venv(venv_dir, verbose=verbose) - pipxrc = Pipxrc(venv_dir) + pipx_metadata = PipxMetadata(venv_dir) - if pipxrc.venv_metadata is not None: - metadata = pipxrc.venv_metadata + if pipx_metadata.venv_metadata is not None: + metadata = pipx_metadata.venv_metadata else: metadata = venv.get_venv_metadata_for_package(package) @@ -533,11 +533,11 @@ def reinstall_all( package = venv_dir.name if package in skip: continue - pipxrc = Pipxrc(venv_dir) + pipx_metadata = PipxMetadata(venv_dir) uninstall(venv_dir, package, local_bin_dir, verbose) - if pipxrc.main_package.package_or_url is not None: - package_or_url = pipxrc.main_package.package_or_url + if pipx_metadata.main_package.package_or_url is not None: + package_or_url = pipx_metadata.main_package.package_or_url else: package_or_url = package @@ -547,13 +547,13 @@ def reinstall_all( package_or_url, local_bin_dir, python, - pipxrc.main_package.pip_args, - pipxrc.venv_args, + pipx_metadata.main_package.pip_args, + pipx_metadata.venv_args, verbose, force=True, - include_dependencies=pipxrc.main_package.include_dependencies, + include_dependencies=pipx_metadata.main_package.include_dependencies, ) - for injected_package in pipxrc.injected_packages: + for injected_package in pipx_metadata.injected_packages: if injected_package.package_or_url is None: # This should never happen, but package_or_url is type # Optional[str] so mypy thinks it could be None diff --git a/src/pipx/venv.py b/src/pipx/venv.py index 1fd0be8db9..70d09dbf18 100644 --- a/src/pipx/venv.py +++ b/src/pipx/venv.py @@ -49,7 +49,7 @@ def verify_shared_libs(self): Venv(p) -class PipxVenvMetadata(NamedTuple): +class VenvMetadata(NamedTuple): apps: List[str] app_paths: List[Path] apps_of_dependencies: List[str] @@ -151,7 +151,7 @@ def install_package(self, package_or_url: str, pip_args: List[str]) -> None: cmd = ["install"] + pip_args + [package_or_url] self._run_pip(cmd) - def get_venv_metadata_for_package(self, package: str) -> PipxVenvMetadata: + def get_venv_metadata_for_package(self, package: str) -> VenvMetadata: data = json.loads( get_script_output( @@ -176,7 +176,7 @@ def get_venv_metadata_for_package(self, package: str) -> PipxVenvMetadata: data["app_paths_of_dependencies"][dep] = paths data["apps_of_dependencies"] += [path.name for path in paths] - return PipxVenvMetadata(**data) + return VenvMetadata(**data) def get_python_version(self) -> str: return ( From d21842c0d67268311225ef99fca49eff22548bd5 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Sat, 26 Oct 2019 19:32:29 -0700 Subject: [PATCH 055/153] Start moving pipxrc actions into Venv.py --- pipx/pipxrc.py | 2 ++ src/pipx/venv.py | 1 + 2 files changed, 3 insertions(+) diff --git a/pipx/pipxrc.py b/pipx/pipxrc.py index 86d6179812..8acd083b87 100644 --- a/pipx/pipxrc.py +++ b/pipx/pipxrc.py @@ -127,6 +127,8 @@ def write(self) -> None: pass def read(self) -> None: + # TODO 20191026: make verbose argument, only show warning if verbose + # and make it less alarming. try: with open(self.venv_dir / PIPX_INFO_FILENAME, "r") as pipx_metadata_fh: self.from_dict( diff --git a/src/pipx/venv.py b/src/pipx/venv.py index 70d09dbf18..037388383a 100644 --- a/src/pipx/venv.py +++ b/src/pipx/venv.py @@ -75,6 +75,7 @@ def __init__( self.root = path self._python = python self.bin_path, self.python_path = get_venv_paths(self.root) + self.pipx_metadata = pipxrc.PipxMetadata(venv_dir=path, read=False) self.verbose = verbose self.do_animation = not verbose try: From 47ec9608e0f77433812869662bc43ca92957d65a Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Sat, 26 Oct 2019 19:54:49 -0700 Subject: [PATCH 056/153] Rearrange object locations to avoid circular imports. abs_val_if_local moved to Venv.py VenvMetadata moved to util.py --- pipx/pipxrc.py | 63 ++----------------------------------- src/pipx/commands.py | 4 +-- src/pipx/util.py | 11 ++++++- src/pipx/venv.py | 74 +++++++++++++++++++++++++++++++++++++------- 4 files changed, 77 insertions(+), 75 deletions(-) diff --git a/pipx/pipxrc.py b/pipx/pipxrc.py index 8acd083b87..0146132bea 100644 --- a/pipx/pipxrc.py +++ b/pipx/pipxrc.py @@ -1,12 +1,11 @@ import json import logging from pathlib import Path -import re import textwrap from typing import List, Dict, NamedTuple, Any, Optional -from pipx.Venv import VenvMetadata, Venv -from pipx.util import PipxError +# from pipx.Venv import VenvMetadata, Venv +from pipx.util import PipxError, VenvMetadata PIPX_INFO_FILENAME = "pipx_metadata.json" @@ -145,61 +144,3 @@ def read(self) -> None: ) self.reset() return - - -def abs_path_if_local(package_or_url: str, venv: Venv, pip_args: List[str]) -> str: - """Return the absolute path if package_or_url represents a filepath - and not a pypi package - """ - pkg_path = Path(package_or_url) - if not pkg_path.exists(): - # no existing path, must be pypi package or non-existent - return package_or_url - - # Editable packages are either local or url, non-url must be local. - # https://pip.pypa.io/en/stable/reference/pip_install/#editable-installs - if "--editable" in pip_args and pkg_path.exists(): - return str(pkg_path.resolve()) - - # https://www.python.org/dev/peps/pep-0508/#names - valid_pkg_name = bool( - re.search(r"^([A-Z0-9]|[A-Z0-9][A-Z0-9._-]*[A-Z0-9])$", package_or_url, re.I) - ) - if not valid_pkg_name: - return str(pkg_path.resolve()) - - # If all of the above conditions do not return, we may have used a pypi - # package. - # If we find a pypi package with this name installed, assume we just - # installed it. - pip_search_args: List[str] - - # If user-defined pypi index url, then use it for search - try: - arg_i = pip_args.index("--index-url") - except ValueError: - pip_search_args = [] - else: - pip_search_args = pip_args[arg_i : arg_i + 2] - - pip_search_result_str = venv.pip_search(package_or_url, pip_search_args) - pip_search_results = pip_search_result_str.split("\n") - - # Get package_or_url and following related lines from pip search stdout - pkg_found = False - pip_search_found = [] - for pip_search_line in pip_search_results: - if pkg_found: - if re.search(r"^\s", pip_search_line): - pip_search_found.append(pip_search_line) - else: - break - elif pip_search_line.startswith(package_or_url): - pip_search_found.append(pip_search_line) - pkg_found = True - pip_found_str = " ".join(pip_search_found) - - if pip_found_str.startswith(package_or_url) and "INSTALLED" in pip_found_str: - return package_or_url - else: - return str(pkg_path.resolve()) diff --git a/src/pipx/commands.py b/src/pipx/commands.py index f2b16664be..a4b6d4aa49 100644 --- a/src/pipx/commands.py +++ b/src/pipx/commands.py @@ -30,8 +30,8 @@ rmdir, run_pypackage_bin, ) -from pipx.pipxrc import PipxMetadata, abs_path_if_local, PackageInfo -from pipx.venv import Venv, VenvContainer +from pipx.pipxrc import PipxMetadata, PackageInfo +from pipx.venv import Venv, VenvContainer, abs_path_if_local def run( diff --git a/src/pipx/util.py b/src/pipx/util.py index 05bafc1eaa..56c8e22cec 100644 --- a/src/pipx/util.py +++ b/src/pipx/util.py @@ -4,7 +4,7 @@ import subprocess import sys from pathlib import Path -from typing import List, Sequence, Tuple, Union +from typing import List, Sequence, Tuple, Union, Dict, NamedTuple from pipx.constants import WINDOWS @@ -13,6 +13,15 @@ class PipxError(Exception): pass +class VenvMetadata(NamedTuple): + apps: List[str] + app_paths: List[Path] + apps_of_dependencies: List[str] + app_paths_of_dependencies: Dict[str, List[Path]] + package_version: str + python_version: str + + def rmdir(path: Path): logging.info(f"removing directory {path}") try: diff --git a/src/pipx/venv.py b/src/pipx/venv.py index 037388383a..c8d8baeeba 100644 --- a/src/pipx/venv.py +++ b/src/pipx/venv.py @@ -1,15 +1,18 @@ import json import logging import pkgutil +import re import subprocess from pathlib import Path -from typing import Dict, Generator, List, NamedTuple +from typing import Generator, List from pipx.animate import animate from pipx.constants import DEFAULT_PYTHON, PIPX_SHARED_PTH, WINDOWS +from pipx.pipxrc import PipxMetadata from pipx.shared_libs import shared_libs from pipx.util import ( PipxError, + VenvMetadata, get_script_output, get_site_packages, get_venv_paths, @@ -49,15 +52,6 @@ def verify_shared_libs(self): Venv(p) -class VenvMetadata(NamedTuple): - apps: List[str] - app_paths: List[Path] - apps_of_dependencies: List[str] - app_paths_of_dependencies: Dict[str, List[Path]] - package_version: str - python_version: str - - venv_metadata_inspector_raw = pkgutil.get_data("pipx", "venv_metadata_inspector.py") assert venv_metadata_inspector_raw is not None, ( "pipx could not find required file venv_metadata_inspector.py. " @@ -75,7 +69,7 @@ def __init__( self.root = path self._python = python self.bin_path, self.python_path = get_venv_paths(self.root) - self.pipx_metadata = pipxrc.PipxMetadata(venv_dir=path, read=False) + self.pipx_metadata = PipxMetadata(venv_dir=path, read=False) self.verbose = verbose self.do_animation = not verbose try: @@ -208,3 +202,61 @@ def _run_pip(self, cmd): if not self.verbose: cmd.append("-q") return run(cmd) + + +def abs_path_if_local(package_or_url: str, venv: Venv, pip_args: List[str]) -> str: + """Return the absolute path if package_or_url represents a filepath + and not a pypi package + """ + pkg_path = Path(package_or_url) + if not pkg_path.exists(): + # no existing path, must be pypi package or non-existent + return package_or_url + + # Editable packages are either local or url, non-url must be local. + # https://pip.pypa.io/en/stable/reference/pip_install/#editable-installs + if "--editable" in pip_args and pkg_path.exists(): + return str(pkg_path.resolve()) + + # https://www.python.org/dev/peps/pep-0508/#names + valid_pkg_name = bool( + re.search(r"^([A-Z0-9]|[A-Z0-9][A-Z0-9._-]*[A-Z0-9])$", package_or_url, re.I) + ) + if not valid_pkg_name: + return str(pkg_path.resolve()) + + # If all of the above conditions do not return, we may have used a pypi + # package. + # If we find a pypi package with this name installed, assume we just + # installed it. + pip_search_args: List[str] + + # If user-defined pypi index url, then use it for search + try: + arg_i = pip_args.index("--index-url") + except ValueError: + pip_search_args = [] + else: + pip_search_args = pip_args[arg_i : arg_i + 2] + + pip_search_result_str = venv.pip_search(package_or_url, pip_search_args) + pip_search_results = pip_search_result_str.split("\n") + + # Get package_or_url and following related lines from pip search stdout + pkg_found = False + pip_search_found = [] + for pip_search_line in pip_search_results: + if pkg_found: + if re.search(r"^\s", pip_search_line): + pip_search_found.append(pip_search_line) + else: + break + elif pip_search_line.startswith(package_or_url): + pip_search_found.append(pip_search_line) + pkg_found = True + pip_found_str = " ".join(pip_search_found) + + if pip_found_str.startswith(package_or_url) and "INSTALLED" in pip_found_str: + return package_or_url + else: + return str(pkg_path.resolve()) From 8b62ae144f4ce7063f74519464b1e081acf1182c Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Sat, 26 Oct 2019 22:15:08 -0700 Subject: [PATCH 057/153] Put package data from VenvMetadata in PackageInfo. Move the following per-package items into PackageInfo: apps: List[str] app_paths: List[Path] apps_of_dependencies: List[str] app_paths_of_dependencies: Dict[str, List[Path]] package_version: str --- pipx/pipxrc.py | 50 +++++++++++++--------------------- src/pipx/commands.py | 65 ++++++++++++++++++++++++++++++++++++-------- src/pipx/venv.py | 2 ++ 3 files changed, 74 insertions(+), 43 deletions(-) diff --git a/pipx/pipxrc.py b/pipx/pipxrc.py index 0146132bea..09e46fa5e7 100644 --- a/pipx/pipxrc.py +++ b/pipx/pipxrc.py @@ -4,7 +4,6 @@ import textwrap from typing import List, Dict, NamedTuple, Any, Optional -# from pipx.Venv import VenvMetadata, Venv from pipx.util import PipxError, VenvMetadata @@ -30,18 +29,32 @@ class PackageInfo(NamedTuple): pip_args: List[str] include_dependencies: bool include_apps: bool + apps: List[str] + app_paths: List[Path] + apps_of_dependencies: List[str] + app_paths_of_dependencies: Dict[str, List[Path]] + package_version: str class PipxMetadata: def __init__(self, venv_dir: Path, read: bool = True): self.venv_dir = venv_dir + # We init this instance with reasonable fallback defaults for all + # members, EXCEPT for those we cannot know: + # self.main_package.package_or_url=None + # self.venv_metadata.package_or_url=None self.main_package = PackageInfo( package_or_url=None, pip_args=[], include_dependencies=False, include_apps=True, # always True for main_package + # + apps=[], + app_paths=[], + app_paths_of_dependencies={}, + package_version="", ) - self.venv_metadata: Optional[VenvMetadata] = None + self.python_version = None self.venv_args: List[str] = [] self.injected_packages: List[PackageInfo] = [] @@ -52,44 +65,20 @@ def __init__(self, venv_dir: Path, read: bool = True): self.read() def reset(self) -> None: - # We init this instance with reasonable fallback defaults for all - # members, EXCEPT for those we cannot know: - # self.main_package.package_or_url=None - # self.venv_metadata.package_or_url=None - self.main_package = PackageInfo( - package_or_url=None, - pip_args=[], - include_dependencies=False, - include_apps=True, # always True for main_package - ) - self.venv_metadata = None - self.venv_args = [] - self.injected_packages = [] + self.__init__(self.venv_dir, read=False) def to_dict(self) -> Dict[str, Any]: - venv_metadata: Optional[Dict[str, Any]] - if self.venv_metadata is not None: - venv_metadata = self.venv_metadata._asdict() - else: - venv_metadata = None - return { "main_package": self.main_package._asdict(), - "venv_metadata": venv_metadata, + "python_version": self.python_version, "venv_args": self.venv_args, "injected_packages": [x._asdict() for x in self.injected_packages], "pipx_metadata_version": self._pipx_metadata_version, } def from_dict(self, input_dict: Dict[str, Any]) -> None: - venv_metadata: Optional[VenvMetadata] - if input_dict["venv_metadata"] is not None: - venv_metadata = VenvMetadata(**input_dict["venv_metadata"]) - else: - venv_metadata = None - self.main_package = PackageInfo(**input_dict["main_package"]) - self.venv_metadata = venv_metadata + self.python_version = input_dict["python_version"] self.venv_args = input_dict["venv_args"] self.injected_packages = [ PackageInfo(**x) for x in input_dict["injected_packages"] @@ -97,8 +86,7 @@ def from_dict(self, input_dict: Dict[str, Any]) -> None: def validate_before_write(self): if ( - self.venv_metadata is None - or self.main_package.package_or_url is None + self.main_package.package_or_url is None or not self.main_package.include_apps ): raise PipxError("Internal Error: PipxMetadata is corrupt, cannot write.") diff --git a/src/pipx/commands.py b/src/pipx/commands.py index a4b6d4aa49..664eff26ca 100644 --- a/src/pipx/commands.py +++ b/src/pipx/commands.py @@ -30,8 +30,8 @@ rmdir, run_pypackage_bin, ) -from pipx.pipxrc import PipxMetadata, PackageInfo -from pipx.venv import Venv, VenvContainer, abs_path_if_local +from pipx.pipxrc import PipxMetadata, abs_path_if_local, PackageInfo +from pipx.venv import Venv, VenvContainer def run( @@ -224,6 +224,8 @@ def upgrade( venv.upgrade_packaging_libraries(pip_args) venv.upgrade_package(package_or_url, pip_args) + # TODO 20191026: Should we upgrade injected packages also? + new_version = venv.get_venv_metadata_for_package(package).package_version metadata = venv.get_venv_metadata_for_package(package) @@ -249,7 +251,22 @@ def upgrade( print( f"upgraded package {package} from {old_version} to {new_version} (location: {str(venv_dir)})" ) - pipx_metadata.venv_metadata = venv.get_venv_metadata_for_package(package) + venv_package_metadata = venv.get_venv_metadata_for_package(package) + orig_package_info = pipx_metadata.main_package + # TODO 20191026: we need to know if user is making a conscious decision + # to override existing package_or_url, pip_args, include_dependencies + # or just wishing to replicate original args + pipx_metadata.main_package = PackageInfo( + package_or_url=orig_package_info.package_or_url, + pip_args=orig_package_info.pip_args, + include_dependencies=orig_package_info.include_dependencies, + include_apps=orig_package_info.include_apps, + apps=venv_package_metadata.apps, + app_paths=venv_package_metadata.app_paths, + apps_of_dependencies=venv_package_metadata.apps_of_dependencies, + app_paths_of_dependencies=venv_package_metadata.app_paths_of_dependencies, + package_version=venv_package_metadata.package_version, + ) pipx_metadata.write() return 1 @@ -340,6 +357,8 @@ def install( venv.remove_venv() raise + venv_package_metadata = venv.get_venv_metadata_for_package(package) + # if all is well, write out pipx_metadata file pipx_metadata = PipxMetadata(venv_dir, read=False) pipx_metadata.main_package = PackageInfo( @@ -347,9 +366,14 @@ def install( pip_args=pip_args, include_dependencies=include_dependencies, include_apps=True, + apps=venv_package_metadata.apps, + app_paths=venv_package_metadata.app_paths, + apps_of_dependencies=venv_package_metadata.apps_of_dependencies, + app_paths_of_dependencies=venv_package_metadata.app_paths_of_dependencies, + package_version=venv_package_metadata.package_version, ) + pipx_metadata.python_version = venv_package_metadata.python_version pipx_metadata.venv_args = venv_args - pipx_metadata.venv_metadata = venv.get_venv_metadata_for_package(package) pipx_metadata.write() @@ -461,6 +485,9 @@ def inject( include_dependencies, force=force, ) + + venv_package_metadata = venv.get_venv_metadata_for_package(package) + pipx_metadata = PipxMetadata(venv_dir) pipx_metadata.injected_packages.append( PackageInfo( @@ -468,6 +495,11 @@ def inject( pip_args=pip_args, include_apps=include_apps, include_dependencies=include_dependencies, + apps=venv_package_metadata.apps, + app_paths=venv_package_metadata.app_paths, + apps_of_dependencies=venv_package_metadata.apps_of_dependencies, + app_paths_of_dependencies=venv_package_metadata.app_paths_of_dependencies, + package_version=venv_package_metadata.package_version, ) ) pipx_metadata.write() @@ -486,19 +518,24 @@ def uninstall(venv_dir: Path, package: str, local_bin_dir: Path, verbose: bool): ) return - # TODO 20191024: Uninstall injected packages - + # TODO 20191024: Uninstall injected packages apps in bin + # TODO 20191026: Handle error if package is not installed? venv = Venv(venv_dir, verbose=verbose) pipx_metadata = PipxMetadata(venv_dir) - if pipx_metadata.venv_metadata is not None: - metadata = pipx_metadata.venv_metadata + if pipx_metadata.main_package is not None: + all_packages = [pipx_metadata.main_package] + pipx_metadata.injected_packages + app_paths = [] + for viewed_package in all_packages: + app_paths += viewed_package.app_paths + for dep_paths in viewed_package.app_paths_of_dependencies.values(): + app_paths += dep_paths else: - metadata = venv.get_venv_metadata_for_package(package) + venv_package_metadata = venv.get_venv_metadata_for_package(package) + app_paths = venv_package_metadata.app_paths + for dep_paths in venv_package_metadata.app_paths_of_dependencies.values(): + app_paths += dep_paths - app_paths = metadata.app_paths - for dep_paths in metadata.app_paths_of_dependencies.values(): - app_paths += dep_paths for file in local_bin_dir.iterdir(): if WINDOWS: for b in app_paths: @@ -534,6 +571,7 @@ def reinstall_all( if package in skip: continue pipx_metadata = PipxMetadata(venv_dir) + uninstall(venv_dir, package, local_bin_dir, verbose) if pipx_metadata.main_package.package_or_url is not None: @@ -541,6 +579,7 @@ def reinstall_all( else: package_or_url = package + # install main package first install( venv_dir, package, @@ -553,6 +592,8 @@ def reinstall_all( force=True, include_dependencies=pipx_metadata.main_package.include_dependencies, ) + + # now install injected packages for injected_package in pipx_metadata.injected_packages: if injected_package.package_or_url is None: # This should never happen, but package_or_url is type diff --git a/src/pipx/venv.py b/src/pipx/venv.py index c8d8baeeba..57ee9032e2 100644 --- a/src/pipx/venv.py +++ b/src/pipx/venv.py @@ -69,6 +69,8 @@ def __init__( self.root = path self._python = python self.bin_path, self.python_path = get_venv_paths(self.root) + # TODO 20191026: probably always need to try and read, silently fail + # if this is a new Venv yet to be created self.pipx_metadata = PipxMetadata(venv_dir=path, read=False) self.verbose = verbose self.do_animation = not verbose From 2329fa820c898af8326d9104a87207f3be704ca1 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Sat, 26 Oct 2019 22:23:07 -0700 Subject: [PATCH 058/153] Fix lint errors. --- pipx/pipxrc.py | 25 ++++++++++++++++++++++--- src/pipx/commands.py | 6 +++--- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/pipx/pipxrc.py b/pipx/pipxrc.py index 09e46fa5e7..3189678752 100644 --- a/pipx/pipxrc.py +++ b/pipx/pipxrc.py @@ -4,7 +4,7 @@ import textwrap from typing import List, Dict, NamedTuple, Any, Optional -from pipx.util import PipxError, VenvMetadata +from pipx.util import PipxError PIPX_INFO_FILENAME = "pipx_metadata.json" @@ -51,10 +51,11 @@ def __init__(self, venv_dir: Path, read: bool = True): # apps=[], app_paths=[], + apps_of_dependencies=[], app_paths_of_dependencies={}, package_version="", ) - self.python_version = None + self.python_version: Optional[str] = None self.venv_args: List[str] = [] self.injected_packages: List[PackageInfo] = [] @@ -65,7 +66,25 @@ def __init__(self, venv_dir: Path, read: bool = True): self.read() def reset(self) -> None: - self.__init__(self.venv_dir, read=False) + # We init this instance with reasonable fallback defaults for all + # members, EXCEPT for those we cannot know: + # self.main_package.package_or_url=None + # self.venv_metadata.package_or_url=None + self.main_package = PackageInfo( + package_or_url=None, + pip_args=[], + include_dependencies=False, + include_apps=True, # always True for main_package + # + apps=[], + app_paths=[], + apps_of_dependencies=[], + app_paths_of_dependencies={}, + package_version="", + ) + self.python_version = None + self.venv_args = [] + self.injected_packages = [] def to_dict(self) -> Dict[str, Any]: return { diff --git a/src/pipx/commands.py b/src/pipx/commands.py index 664eff26ca..2c28b8ebab 100644 --- a/src/pipx/commands.py +++ b/src/pipx/commands.py @@ -30,8 +30,8 @@ rmdir, run_pypackage_bin, ) -from pipx.pipxrc import PipxMetadata, abs_path_if_local, PackageInfo -from pipx.venv import Venv, VenvContainer +from pipx.pipxrc import PipxMetadata, PackageInfo +from pipx.venv import Venv, VenvContainer, abs_path_if_local def run( @@ -525,7 +525,7 @@ def uninstall(venv_dir: Path, package: str, local_bin_dir: Path, verbose: bool): if pipx_metadata.main_package is not None: all_packages = [pipx_metadata.main_package] + pipx_metadata.injected_packages - app_paths = [] + app_paths: List[Path] = [] for viewed_package in all_packages: app_paths += viewed_package.app_paths for dep_paths in viewed_package.app_paths_of_dependencies.values(): From 3d43a78a3c693ee096255bfbd042d5fb21f182a9 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Sat, 26 Oct 2019 22:27:24 -0700 Subject: [PATCH 059/153] Undo accidental commit. --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index eb9ef8add8..f3d1c64b06 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -21,4 +21,4 @@ def pipx_temp_env(tmp_path, monkeypatch): monkeypatch.setattr(constants, "LOCAL_BIN_DIR", bin_dir) monkeypatch.setattr(constants, "PIPX_LOCAL_VENVS", home_dir / "venvs") - monkeypatch.setenv("PATH", str(bin_dir) + ":/bin:/usr/bin") + monkeypatch.setenv("PATH", str(bin_dir)) From 556293346a25f7f64d32821d3275354355e09370 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Sat, 26 Oct 2019 22:46:53 -0700 Subject: [PATCH 060/153] Move VenvMetadata back into Venv.py --- pipx/pipxrc.py | 19 +++++++++---------- src/pipx/util.py | 11 +---------- src/pipx/venv.py | 14 +++++++++++--- 3 files changed, 21 insertions(+), 23 deletions(-) diff --git a/pipx/pipxrc.py b/pipx/pipxrc.py index 3189678752..1d4c0b160f 100644 --- a/pipx/pipxrc.py +++ b/pipx/pipxrc.py @@ -132,22 +132,21 @@ def write(self) -> None: ) pass - def read(self) -> None: - # TODO 20191026: make verbose argument, only show warning if verbose - # and make it less alarming. + def read(self, verbose: bool = False) -> None: try: with open(self.venv_dir / PIPX_INFO_FILENAME, "r") as pipx_metadata_fh: self.from_dict( json.load(pipx_metadata_fh, object_hook=_json_decoder_object_hook) ) except IOError: # Reset self if problem reading - logging.warning( - textwrap.fill( - f"Unable to read {PIPX_INFO_FILENAME} in {self.venv_dir}. " - f"This may cause this or future pipx operations involving " - f"{self.venv_dir.name} to fail or behave incorrectly.", - width=79, + if verbose: + logging.warning( + textwrap.fill( + f"Unable to read {PIPX_INFO_FILENAME} in {self.venv_dir}. " + f"This may cause this or future pipx operations involving " + f"{self.venv_dir.name} to fail or behave incorrectly.", + width=79, + ) ) - ) self.reset() return diff --git a/src/pipx/util.py b/src/pipx/util.py index 56c8e22cec..05bafc1eaa 100644 --- a/src/pipx/util.py +++ b/src/pipx/util.py @@ -4,7 +4,7 @@ import subprocess import sys from pathlib import Path -from typing import List, Sequence, Tuple, Union, Dict, NamedTuple +from typing import List, Sequence, Tuple, Union from pipx.constants import WINDOWS @@ -13,15 +13,6 @@ class PipxError(Exception): pass -class VenvMetadata(NamedTuple): - apps: List[str] - app_paths: List[Path] - apps_of_dependencies: List[str] - app_paths_of_dependencies: Dict[str, List[Path]] - package_version: str - python_version: str - - def rmdir(path: Path): logging.info(f"removing directory {path}") try: diff --git a/src/pipx/venv.py b/src/pipx/venv.py index 57ee9032e2..512edd1294 100644 --- a/src/pipx/venv.py +++ b/src/pipx/venv.py @@ -4,7 +4,7 @@ import re import subprocess from pathlib import Path -from typing import Generator, List +from typing import Generator, List, NamedTuple, Dict from pipx.animate import animate from pipx.constants import DEFAULT_PYTHON, PIPX_SHARED_PTH, WINDOWS @@ -12,7 +12,6 @@ from pipx.shared_libs import shared_libs from pipx.util import ( PipxError, - VenvMetadata, get_script_output, get_site_packages, get_venv_paths, @@ -52,6 +51,15 @@ def verify_shared_libs(self): Venv(p) +class VenvMetadata(NamedTuple): + apps: List[str] + app_paths: List[Path] + apps_of_dependencies: List[str] + app_paths_of_dependencies: Dict[str, List[Path]] + package_version: str + python_version: str + + venv_metadata_inspector_raw = pkgutil.get_data("pipx", "venv_metadata_inspector.py") assert venv_metadata_inspector_raw is not None, ( "pipx could not find required file venv_metadata_inspector.py. " @@ -71,7 +79,7 @@ def __init__( self.bin_path, self.python_path = get_venv_paths(self.root) # TODO 20191026: probably always need to try and read, silently fail # if this is a new Venv yet to be created - self.pipx_metadata = PipxMetadata(venv_dir=path, read=False) + self.pipx_metadata = PipxMetadata(venv_dir=path) self.verbose = verbose self.do_animation = not verbose try: From f7b32dd8274b77c0ab2a4f8971f7407c1f70b381 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Sat, 26 Oct 2019 23:49:04 -0700 Subject: [PATCH 061/153] Now only use pipx_metadata through venv. --- src/pipx/commands.py | 109 ++++++++++++++++++------------------------- src/pipx/venv.py | 59 ++++++++++++++++++++++- 2 files changed, 104 insertions(+), 64 deletions(-) diff --git a/src/pipx/commands.py b/src/pipx/commands.py index 2c28b8ebab..6a3d7de78c 100644 --- a/src/pipx/commands.py +++ b/src/pipx/commands.py @@ -30,8 +30,7 @@ rmdir, run_pypackage_bin, ) -from pipx.pipxrc import PipxMetadata, PackageInfo -from pipx.venv import Venv, VenvContainer, abs_path_if_local +from pipx.venv import Venv, VenvContainer def run( @@ -207,7 +206,6 @@ def upgrade( ) venv = Venv(venv_dir, verbose=verbose) - pipx_metadata = PipxMetadata(venv_dir) old_version = venv.get_venv_metadata_for_package(package).package_version @@ -215,8 +213,8 @@ def upgrade( # TODO 20190926: main.py should communicate if this is spec or copied from # package if package_or_url == package: - if pipx_metadata.main_package.package_or_url is not None: - package_or_url = pipx_metadata.main_package.package_or_url + if venv.pipx_metadata.main_package.package_or_url is not None: + package_or_url = venv.pipx_metadata.main_package.package_or_url else: package_or_url = package @@ -251,23 +249,15 @@ def upgrade( print( f"upgraded package {package} from {old_version} to {new_version} (location: {str(venv_dir)})" ) - venv_package_metadata = venv.get_venv_metadata_for_package(package) - orig_package_info = pipx_metadata.main_package - # TODO 20191026: we need to know if user is making a conscious decision - # to override existing package_or_url, pip_args, include_dependencies - # or just wishing to replicate original args - pipx_metadata.main_package = PackageInfo( - package_or_url=orig_package_info.package_or_url, + orig_package_info = venv.pipx_metadata.main_package + venv.update_package_metadata( + package=package, + package_or_url=package_or_url, pip_args=orig_package_info.pip_args, include_dependencies=orig_package_info.include_dependencies, include_apps=orig_package_info.include_apps, - apps=venv_package_metadata.apps, - app_paths=venv_package_metadata.app_paths, - apps_of_dependencies=venv_package_metadata.apps_of_dependencies, - app_paths_of_dependencies=venv_package_metadata.app_paths_of_dependencies, - package_version=venv_package_metadata.package_version, + is_main=True, ) - pipx_metadata.write() return 1 @@ -279,14 +269,14 @@ def upgrade_all( for venv_dir in venv_container.iter_venv_dirs(): num_packages += 1 package = venv_dir.name - pipx_metadata = PipxMetadata(venv_dir) + venv = Venv(venv_dir, verbose=verbose) if package in skip: continue if package == "pipx": package_or_url = PIPX_PACKAGE_NAME else: - if pipx_metadata.main_package.package_or_url is not None: - package_or_url = pipx_metadata.main_package.package_or_url + if venv.pipx_metadata.main_package.package_or_url is not None: + package_or_url = venv.pipx_metadata.main_package.package_or_url else: package_or_url = package try: @@ -294,10 +284,10 @@ def upgrade_all( venv_dir, package, package_or_url, - pipx_metadata.main_package.pip_args, + venv.pipx_metadata.main_package.pip_args, verbose, upgrading_all=True, - include_dependencies=pipx_metadata.main_package.include_dependencies, + include_dependencies=venv.pipx_metadata.main_package.include_dependencies, force=force, ) # TODO 20191024: Upgrade injected packages @@ -357,24 +347,15 @@ def install( venv.remove_venv() raise - venv_package_metadata = venv.get_venv_metadata_for_package(package) - - # if all is well, write out pipx_metadata file - pipx_metadata = PipxMetadata(venv_dir, read=False) - pipx_metadata.main_package = PackageInfo( - package_or_url=abs_path_if_local(package_or_url, venv, pip_args), + # if all is well, write pipx_metadata + venv.update_package_metadata( + package=package, + package_or_url=package_or_url, pip_args=pip_args, include_dependencies=include_dependencies, include_apps=True, - apps=venv_package_metadata.apps, - app_paths=venv_package_metadata.app_paths, - apps_of_dependencies=venv_package_metadata.apps_of_dependencies, - app_paths_of_dependencies=venv_package_metadata.app_paths_of_dependencies, - package_version=venv_package_metadata.package_version, + is_main=True, ) - pipx_metadata.python_version = venv_package_metadata.python_version - pipx_metadata.venv_args = venv_args - pipx_metadata.write() def _run_post_install_actions( @@ -456,6 +437,7 @@ def _warn_if_not_on_path(local_bin_dir: Path): def inject( venv_dir: Path, package: str, + # TODO 20191026: Need package_or_url separate from package pip_args: List[str], *, verbose: bool, @@ -486,29 +468,22 @@ def inject( force=force, ) - venv_package_metadata = venv.get_venv_metadata_for_package(package) - - pipx_metadata = PipxMetadata(venv_dir) - pipx_metadata.injected_packages.append( - PackageInfo( - package_or_url=package, - pip_args=pip_args, - include_apps=include_apps, - include_dependencies=include_dependencies, - apps=venv_package_metadata.apps, - app_paths=venv_package_metadata.app_paths, - apps_of_dependencies=venv_package_metadata.apps_of_dependencies, - app_paths_of_dependencies=venv_package_metadata.app_paths_of_dependencies, - package_version=venv_package_metadata.package_version, - ) + venv.append_injected_package_metadata( + package=package, + package_or_url=package, + pip_args=pip_args, + include_apps=include_apps, + include_dependencies=include_dependencies, ) - pipx_metadata.write() print(f" injected package {bold(package)} into venv {bold(venv_dir.name)}") print(f"done! {stars}", file=sys.stderr) def uninstall(venv_dir: Path, package: str, local_bin_dir: Path, verbose: bool): + """Uninstall entire venv_dir, including main package and all injected + packages. + """ if not venv_dir.exists(): print(f"Nothing to uninstall for {package} 😴") app = which(package) @@ -521,10 +496,11 @@ def uninstall(venv_dir: Path, package: str, local_bin_dir: Path, verbose: bool): # TODO 20191024: Uninstall injected packages apps in bin # TODO 20191026: Handle error if package is not installed? venv = Venv(venv_dir, verbose=verbose) - pipx_metadata = PipxMetadata(venv_dir) - if pipx_metadata.main_package is not None: - all_packages = [pipx_metadata.main_package] + pipx_metadata.injected_packages + if venv.pipx_metadata.main_package is not None: + all_packages = [ + venv.pipx_metadata.main_package + ] + venv.pipx_metadata.injected_packages app_paths: List[Path] = [] for viewed_package in all_packages: app_paths += viewed_package.app_paths @@ -570,12 +546,19 @@ def reinstall_all( package = venv_dir.name if package in skip: continue - pipx_metadata = PipxMetadata(venv_dir) + + venv = Venv(venv_dir, verbose=verbose) + + # TODO 20191026: store this away in case uninstalling removes metadata + # orig_pipx_metadata = venv.pipx_metadata uninstall(venv_dir, package, local_bin_dir, verbose) + # TODO 20191026: also uninstall all injected packages + # injected packages will be cluttering up metadata if their metadata + # is not removed also - if pipx_metadata.main_package.package_or_url is not None: - package_or_url = pipx_metadata.main_package.package_or_url + if venv.pipx_metadata.main_package.package_or_url is not None: + package_or_url = venv.pipx_metadata.main_package.package_or_url else: package_or_url = package @@ -586,15 +569,15 @@ def reinstall_all( package_or_url, local_bin_dir, python, - pipx_metadata.main_package.pip_args, - pipx_metadata.venv_args, + venv.pipx_metadata.main_package.pip_args, + venv.pipx_metadata.venv_args, verbose, force=True, - include_dependencies=pipx_metadata.main_package.include_dependencies, + include_dependencies=venv.pipx_metadata.main_package.include_dependencies, ) # now install injected packages - for injected_package in pipx_metadata.injected_packages: + for injected_package in venv.pipx_metadata.injected_packages: if injected_package.package_or_url is None: # This should never happen, but package_or_url is type # Optional[str] so mypy thinks it could be None diff --git a/src/pipx/venv.py b/src/pipx/venv.py index 512edd1294..5e481294cc 100644 --- a/src/pipx/venv.py +++ b/src/pipx/venv.py @@ -8,7 +8,7 @@ from pipx.animate import animate from pipx.constants import DEFAULT_PYTHON, PIPX_SHARED_PTH, WINDOWS -from pipx.pipxrc import PipxMetadata +from pipx.pipxrc import PipxMetadata, PackageInfo from pipx.shared_libs import shared_libs from pipx.util import ( PipxError, @@ -129,6 +129,10 @@ def create_venv(self, venv_args: List[str], pip_args: List[str]) -> None: # its contents are additional items (one per line) to be added to sys.path pipx_pth.write_text(str(shared_libs.site_packages) + "\n", encoding="utf-8") + # write venv-specific metadata + self.pipx_metadata.venv_args = venv_args + self.pipx_metadata.python_version = self.get_python_version() + def safe_to_remove(self) -> bool: return not self._existing @@ -183,6 +187,59 @@ def get_venv_metadata_for_package(self, package: str) -> VenvMetadata: return VenvMetadata(**data) + def update_package_metadata( + self, + package: str, + package_or_url: str, + pip_args: List[str], + include_dependencies: bool, + include_apps: bool, + is_main: bool, + ): + venv_package_metadata = self.get_venv_metadata_for_package(package) + if is_main: + self.pipx_metadata.main_package = PackageInfo( + package_or_url=abs_path_if_local(package_or_url, self, pip_args), + pip_args=pip_args, + include_dependencies=include_dependencies, + include_apps=True, + apps=venv_package_metadata.apps, + app_paths=venv_package_metadata.app_paths, + apps_of_dependencies=venv_package_metadata.apps_of_dependencies, + app_paths_of_dependencies=venv_package_metadata.app_paths_of_dependencies, + package_version=venv_package_metadata.package_version, + ) + else: + # TODO 20191026: see if we can use this for injected packages also + raise Exception("Internal Error: is_main=False is unimplemented.") + + self.pipx_metadata.write() + + def append_injected_package_metadata( + self, + package: str, + package_or_url: str, + pip_args: List[str], + include_apps: bool, + include_dependencies: bool, + ): + venv_package_metadata = self.get_venv_metadata_for_package(package) + + self.pipx_metadata.injected_packages.append( + PackageInfo( + package_or_url=package_or_url, + pip_args=pip_args, + include_apps=include_apps, + include_dependencies=include_dependencies, + apps=venv_package_metadata.apps, + app_paths=venv_package_metadata.app_paths, + apps_of_dependencies=venv_package_metadata.apps_of_dependencies, + app_paths_of_dependencies=venv_package_metadata.app_paths_of_dependencies, + package_version=venv_package_metadata.package_version, + ) + ) + self.pipx_metadata.write() + def get_python_version(self) -> str: return ( subprocess.run([str(self.python_path), "--version"], stdout=subprocess.PIPE) From 17a00a317c5312853c7f01c32e54d2209ad83b00 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Sun, 27 Oct 2019 00:28:13 -0700 Subject: [PATCH 062/153] Add package_or_url to inject. Change PipxMetadata. PipxMetadata.injected_packages is now a dict. --- pipx/pipxrc.py | 15 +++++++----- src/pipx/commands.py | 55 ++++++++++++++++++++++++-------------------- src/pipx/main.py | 1 + src/pipx/venv.py | 22 ++++++++---------- 4 files changed, 50 insertions(+), 43 deletions(-) diff --git a/pipx/pipxrc.py b/pipx/pipxrc.py index 1d4c0b160f..49e40403f3 100644 --- a/pipx/pipxrc.py +++ b/pipx/pipxrc.py @@ -57,7 +57,7 @@ def __init__(self, venv_dir: Path, read: bool = True): ) self.python_version: Optional[str] = None self.venv_args: List[str] = [] - self.injected_packages: List[PackageInfo] = [] + self.injected_packages: Dict[str, PackageInfo] = {} # Only change this if file format changes self._pipx_metadata_version: str = "0.1" @@ -84,14 +84,16 @@ def reset(self) -> None: ) self.python_version = None self.venv_args = [] - self.injected_packages = [] + self.injected_packages = {} def to_dict(self) -> Dict[str, Any]: return { "main_package": self.main_package._asdict(), "python_version": self.python_version, "venv_args": self.venv_args, - "injected_packages": [x._asdict() for x in self.injected_packages], + "injected_packages": { + name: data._asdict() for (name, data) in self.injected_packages.items() + }, "pipx_metadata_version": self._pipx_metadata_version, } @@ -99,9 +101,10 @@ def from_dict(self, input_dict: Dict[str, Any]) -> None: self.main_package = PackageInfo(**input_dict["main_package"]) self.python_version = input_dict["python_version"] self.venv_args = input_dict["venv_args"] - self.injected_packages = [ - PackageInfo(**x) for x in input_dict["injected_packages"] - ] + self.injected_packages = { + name: PackageInfo(**data) + for (name, data) in input_dict["injected_packages"].items() + } def validate_before_write(self): if ( diff --git a/src/pipx/commands.py b/src/pipx/commands.py index 6a3d7de78c..b91bf7e83d 100644 --- a/src/pipx/commands.py +++ b/src/pipx/commands.py @@ -339,6 +339,16 @@ def install( venv.remove_venv() raise PipxError(f"Could not find package {package}. Is the name correct?") + # if installed ok, write pipx_metadata + venv.update_package_metadata( + package=package, + package_or_url=package_or_url, + pip_args=pip_args, + include_dependencies=include_dependencies, + include_apps=True, + is_main=True, + ) + _run_post_install_actions( venv, package, local_bin_dir, venv_dir, include_dependencies, force=force ) @@ -347,16 +357,6 @@ def install( venv.remove_venv() raise - # if all is well, write pipx_metadata - venv.update_package_metadata( - package=package, - package_or_url=package_or_url, - pip_args=pip_args, - include_dependencies=include_dependencies, - include_apps=True, - is_main=True, - ) - def _run_post_install_actions( venv: Venv, @@ -367,7 +367,8 @@ def _run_post_install_actions( *, force: bool, ): - metadata = venv.get_venv_metadata_for_package(package) + if package == venv_dir.name: + metadata = venv.pipx_metadata.main_package if not metadata.app_paths and not include_dependencies: # No apps associated with this package and we aren't including dependencies. @@ -437,7 +438,7 @@ def _warn_if_not_on_path(local_bin_dir: Path): def inject( venv_dir: Path, package: str, - # TODO 20191026: Need package_or_url separate from package + package_or_url: str, pip_args: List[str], *, verbose: bool, @@ -456,7 +457,15 @@ def inject( ) venv = Venv(venv_dir, verbose=verbose) - venv.install_package(package, pip_args) + venv.install_package(package_or_url, pip_args) + # TODO 20191026: verify installed ok like install()? + venv.append_injected_package_metadata( + package=package, + package_or_url=package_or_url, + pip_args=pip_args, + include_apps=include_apps, + include_dependencies=include_dependencies, + ) if include_apps: _run_post_install_actions( @@ -468,14 +477,6 @@ def inject( force=force, ) - venv.append_injected_package_metadata( - package=package, - package_or_url=package, - pip_args=pip_args, - include_apps=include_apps, - include_dependencies=include_dependencies, - ) - print(f" injected package {bold(package)} into venv {bold(venv_dir.name)}") print(f"done! {stars}", file=sys.stderr) @@ -498,9 +499,9 @@ def uninstall(venv_dir: Path, package: str, local_bin_dir: Path, verbose: bool): venv = Venv(venv_dir, verbose=verbose) if venv.pipx_metadata.main_package is not None: - all_packages = [ - venv.pipx_metadata.main_package - ] + venv.pipx_metadata.injected_packages + all_packages = [venv.pipx_metadata.main_package] + list( + venv.pipx_metadata.injected_packages.values() + ) app_paths: List[Path] = [] for viewed_package in all_packages: app_paths += viewed_package.app_paths @@ -577,13 +578,17 @@ def reinstall_all( ) # now install injected packages - for injected_package in venv.pipx_metadata.injected_packages: + for ( + injected_name, + injected_package, + ) in venv.pipx_metadata.injected_packages.items(): if injected_package.package_or_url is None: # This should never happen, but package_or_url is type # Optional[str] so mypy thinks it could be None raise PipxError("Internal Error injecting package") inject( venv_dir, + injected_name, injected_package.package_or_url, injected_package.pip_args, verbose=verbose, diff --git a/src/pipx/main.py b/src/pipx/main.py index e8a13b1ad3..7b503b7056 100644 --- a/src/pipx/main.py +++ b/src/pipx/main.py @@ -178,6 +178,7 @@ def run_pipx_command(args): # noqa: C901 commands.inject( venv_dir, dep, + dep, # NOP now, but in future can be package_or_url --spec pip_args, verbose=verbose, include_apps=args.include_apps, diff --git a/src/pipx/venv.py b/src/pipx/venv.py index 5e481294cc..745eceddcd 100644 --- a/src/pipx/venv.py +++ b/src/pipx/venv.py @@ -225,18 +225,16 @@ def append_injected_package_metadata( ): venv_package_metadata = self.get_venv_metadata_for_package(package) - self.pipx_metadata.injected_packages.append( - PackageInfo( - package_or_url=package_or_url, - pip_args=pip_args, - include_apps=include_apps, - include_dependencies=include_dependencies, - apps=venv_package_metadata.apps, - app_paths=venv_package_metadata.app_paths, - apps_of_dependencies=venv_package_metadata.apps_of_dependencies, - app_paths_of_dependencies=venv_package_metadata.app_paths_of_dependencies, - package_version=venv_package_metadata.package_version, - ) + self.pipx_metadata.injected_packages[package] = PackageInfo( + package_or_url=package_or_url, + pip_args=pip_args, + include_apps=include_apps, + include_dependencies=include_dependencies, + apps=venv_package_metadata.apps, + app_paths=venv_package_metadata.app_paths, + apps_of_dependencies=venv_package_metadata.apps_of_dependencies, + app_paths_of_dependencies=venv_package_metadata.app_paths_of_dependencies, + package_version=venv_package_metadata.package_version, ) self.pipx_metadata.write() From fd515a39a84a4c18111c57779f0f5c71fc2d930b Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Sun, 27 Oct 2019 02:09:23 -0700 Subject: [PATCH 063/153] Remove obsolete comment. --- pipx/pipxrc.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pipx/pipxrc.py b/pipx/pipxrc.py index 49e40403f3..d409c9b593 100644 --- a/pipx/pipxrc.py +++ b/pipx/pipxrc.py @@ -42,7 +42,6 @@ def __init__(self, venv_dir: Path, read: bool = True): # We init this instance with reasonable fallback defaults for all # members, EXCEPT for those we cannot know: # self.main_package.package_or_url=None - # self.venv_metadata.package_or_url=None self.main_package = PackageInfo( package_or_url=None, pip_args=[], From a824147bd2dc925379a13350ed655b8ef938692b Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Sun, 27 Oct 2019 02:10:01 -0700 Subject: [PATCH 064/153] Remove most venv.get_venv_metadata_for_package. Add property package_metadata to venv that returns a dict of package data to streamline access. Move append_injected_package_metadata functionality into update_package_metadata member function in Venv. --- src/pipx/commands.py | 73 ++++++++++++++++++++++++++------------------ src/pipx/venv.py | 47 +++++++++++----------------- 2 files changed, 61 insertions(+), 59 deletions(-) diff --git a/src/pipx/commands.py b/src/pipx/commands.py index b91bf7e83d..1a71fe30d9 100644 --- a/src/pipx/commands.py +++ b/src/pipx/commands.py @@ -207,7 +207,8 @@ def upgrade( venv = Venv(venv_dir, verbose=verbose) - old_version = venv.get_venv_metadata_for_package(package).package_version + old_package_metadata = venv.package_metadata[package] + old_version = old_package_metadata.package_version # if default package_or_url, check pipx_metadata for better url # TODO 20190926: main.py should communicate if this is spec or copied from @@ -224,15 +225,25 @@ def upgrade( venv.upgrade_package(package_or_url, pip_args) # TODO 20191026: Should we upgrade injected packages also? - new_version = venv.get_venv_metadata_for_package(package).package_version + venv.update_package_metadata( + package=package, + package_or_url=package_or_url, + pip_args=old_package_metadata.pip_args, + # TODO 20191026: should this be old_package_metadata.include_dependencies? + include_dependencies=include_dependencies, + include_apps=old_package_metadata.include_apps, + is_main=True, + ) + + package_metadata = venv.package_metadata[package] + new_version = package_metadata.package_version - metadata = venv.get_venv_metadata_for_package(package) _expose_apps_globally( - constants.LOCAL_BIN_DIR, metadata.app_paths, package, force=force + constants.LOCAL_BIN_DIR, package_metadata.app_paths, package, force=force ) if include_dependencies: - for _, app_paths in metadata.app_paths_of_dependencies.items(): + for _, app_paths in package_metadata.app_paths_of_dependencies.items(): _expose_apps_globally( constants.LOCAL_BIN_DIR, app_paths, package, force=force ) @@ -249,15 +260,6 @@ def upgrade( print( f"upgraded package {package} from {old_version} to {new_version} (location: {str(venv_dir)})" ) - orig_package_info = venv.pipx_metadata.main_package - venv.update_package_metadata( - package=package, - package_or_url=package_or_url, - pip_args=orig_package_info.pip_args, - include_dependencies=orig_package_info.include_dependencies, - include_apps=orig_package_info.include_apps, - is_main=True, - ) return 1 @@ -367,13 +369,12 @@ def _run_post_install_actions( *, force: bool, ): - if package == venv_dir.name: - metadata = venv.pipx_metadata.main_package + package_metadata = venv.package_metadata[package] - if not metadata.app_paths and not include_dependencies: + if not package_metadata.app_paths and not include_dependencies: # No apps associated with this package and we aren't including dependencies. # This package has nothing for pipx to use, so this is an error. - for dep, dependent_apps in metadata.app_paths_of_dependencies.items(): + for dep, dependent_apps in package_metadata.app_paths_of_dependencies.items(): print( f"Note: Dependent package '{dep}' contains {len(dependent_apps)} apps" ) @@ -383,7 +384,7 @@ def _run_post_install_actions( if venv.safe_to_remove(): venv.remove_venv() - if len(metadata.app_paths_of_dependencies.keys()): + if len(package_metadata.app_paths_of_dependencies.keys()): raise PipxError( f"No apps associated with package {package}. " "Try again with '--include-deps' to include apps of dependent packages, " @@ -398,9 +399,9 @@ def _run_post_install_actions( "Consider using pip or a similar tool instead." ) - if metadata.apps: + if package_metadata.apps: pass - elif metadata.apps_of_dependencies and include_dependencies: + elif package_metadata.apps_of_dependencies and include_dependencies: pass else: # No apps associated with this package and we aren't including dependencies. @@ -413,10 +414,12 @@ def _run_post_install_actions( "Consider using pip or a similar tool instead." ) - _expose_apps_globally(local_bin_dir, metadata.app_paths, package, force=force) + _expose_apps_globally( + local_bin_dir, package_metadata.app_paths, package, force=force + ) if include_dependencies: - for _, app_paths in metadata.app_paths_of_dependencies.items(): + for _, app_paths in package_metadata.app_paths_of_dependencies.items(): _expose_apps_globally(local_bin_dir, app_paths, package, force=force) print(_get_package_summary(venv_dir, package=package, new_install=True)) @@ -459,12 +462,13 @@ def inject( venv = Venv(venv_dir, verbose=verbose) venv.install_package(package_or_url, pip_args) # TODO 20191026: verify installed ok like install()? - venv.append_injected_package_metadata( + venv.update_package_metadata( package=package, package_or_url=package_or_url, pip_args=pip_args, include_apps=include_apps, include_dependencies=include_dependencies, + is_main=False, ) if include_apps: @@ -674,22 +678,31 @@ def _get_package_summary( python_path = venv.python_path.resolve() if package is None: package = path.name - metadata = venv.get_venv_metadata_for_package(package) + package_metadata = venv.package_metadata[package] - if metadata.package_version is None: + if package_metadata.package_version is None: not_installed = red("is not installed") return f" package {bold(package)} {not_installed} in the venv {str(path)}" - apps = metadata.apps + metadata.apps_of_dependencies + apps = package_metadata.apps + package_metadata.apps_of_dependencies exposed_app_paths = _get_exposed_app_paths_for_package( venv.bin_path, apps, constants.LOCAL_BIN_DIR ) exposed_binary_names = sorted(p.name for p in exposed_app_paths) - unavailable_binary_names = sorted(set(metadata.apps) - set(exposed_binary_names)) + unavailable_binary_names = sorted( + set(package_metadata.apps) - set(exposed_binary_names) + ) + # The following is to satisfy mypy that python_version is str and not + # Optional[str] + python_version = ( + venv.pipx_metadata.python_version + if venv.pipx_metadata.python_version is not None + else "" + ) return _get_list_output( - metadata.python_version, + python_version, python_path, - metadata.package_version, + package_metadata.package_version, package, new_install, exposed_binary_names, diff --git a/src/pipx/venv.py b/src/pipx/venv.py index 745eceddcd..b1d53790db 100644 --- a/src/pipx/venv.py +++ b/src/pipx/venv.py @@ -77,9 +77,8 @@ def __init__( self.root = path self._python = python self.bin_path, self.python_path = get_venv_paths(self.root) - # TODO 20191026: probably always need to try and read, silently fail - # if this is a new Venv yet to be created self.pipx_metadata = PipxMetadata(venv_dir=path) + self.pipx_metadata.python_version = self.get_python_version() self.verbose = verbose self.do_animation = not verbose try: @@ -112,6 +111,12 @@ def uses_shared_libs(self): # always use shared libs when creating a new venv return True + @property + def package_metadata(self) -> Dict[str, PackageInfo]: + return_dict = self.pipx_metadata.injected_packages.copy() + return_dict[self.root.name] = self.pipx_metadata.main_package + return return_dict + def create_venv(self, venv_args: List[str], pip_args: List[str]) -> None: with animate("creating virtual environment", self.do_animation): cmd = [self._python, "-m", "venv", "--without-pip"] @@ -131,7 +136,6 @@ def create_venv(self, venv_args: List[str], pip_args: List[str]) -> None: # write venv-specific metadata self.pipx_metadata.venv_args = venv_args - self.pipx_metadata.python_version = self.get_python_version() def safe_to_remove(self) -> bool: return not self._existing @@ -161,7 +165,6 @@ def install_package(self, package_or_url: str, pip_args: List[str]) -> None: self._run_pip(cmd) def get_venv_metadata_for_package(self, package: str) -> VenvMetadata: - data = json.loads( get_script_output( self.python_path, VENV_METADATA_INSPECTOR, package, str(self.bin_path) @@ -210,32 +213,18 @@ def update_package_metadata( package_version=venv_package_metadata.package_version, ) else: - # TODO 20191026: see if we can use this for injected packages also - raise Exception("Internal Error: is_main=False is unimplemented.") - - self.pipx_metadata.write() - - def append_injected_package_metadata( - self, - package: str, - package_or_url: str, - pip_args: List[str], - include_apps: bool, - include_dependencies: bool, - ): - venv_package_metadata = self.get_venv_metadata_for_package(package) + self.pipx_metadata.injected_packages[package] = PackageInfo( + package_or_url=package_or_url, + pip_args=pip_args, + include_apps=include_apps, + include_dependencies=include_dependencies, + apps=venv_package_metadata.apps, + app_paths=venv_package_metadata.app_paths, + apps_of_dependencies=venv_package_metadata.apps_of_dependencies, + app_paths_of_dependencies=venv_package_metadata.app_paths_of_dependencies, + package_version=venv_package_metadata.package_version, + ) - self.pipx_metadata.injected_packages[package] = PackageInfo( - package_or_url=package_or_url, - pip_args=pip_args, - include_apps=include_apps, - include_dependencies=include_dependencies, - apps=venv_package_metadata.apps, - app_paths=venv_package_metadata.app_paths, - apps_of_dependencies=venv_package_metadata.apps_of_dependencies, - app_paths_of_dependencies=venv_package_metadata.app_paths_of_dependencies, - package_version=venv_package_metadata.package_version, - ) self.pipx_metadata.write() def get_python_version(self) -> str: From 25be4b0b9ca42079f531db41726b889e82825125 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Sun, 27 Oct 2019 02:24:53 -0700 Subject: [PATCH 065/153] Moving python_version check back to create_venv(). --- src/pipx/venv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pipx/venv.py b/src/pipx/venv.py index b1d53790db..08fc013292 100644 --- a/src/pipx/venv.py +++ b/src/pipx/venv.py @@ -78,7 +78,6 @@ def __init__( self._python = python self.bin_path, self.python_path = get_venv_paths(self.root) self.pipx_metadata = PipxMetadata(venv_dir=path) - self.pipx_metadata.python_version = self.get_python_version() self.verbose = verbose self.do_animation = not verbose try: @@ -136,6 +135,7 @@ def create_venv(self, venv_args: List[str], pip_args: List[str]) -> None: # write venv-specific metadata self.pipx_metadata.venv_args = venv_args + self.pipx_metadata.python_version = self.get_python_version() def safe_to_remove(self) -> bool: return not self._existing From 9619dae1dcb4a37685668909589d6f9dbf8e5756 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Sun, 27 Oct 2019 02:25:13 -0700 Subject: [PATCH 066/153] Update comment. --- pipx/pipxrc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pipx/pipxrc.py b/pipx/pipxrc.py index d409c9b593..7136557234 100644 --- a/pipx/pipxrc.py +++ b/pipx/pipxrc.py @@ -42,6 +42,7 @@ def __init__(self, venv_dir: Path, read: bool = True): # We init this instance with reasonable fallback defaults for all # members, EXCEPT for those we cannot know: # self.main_package.package_or_url=None + # self.python_version=None self.main_package = PackageInfo( package_or_url=None, pip_args=[], From 7066ffc422df113961dfd219f9535542bff71fa3 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Sun, 27 Oct 2019 02:25:38 -0700 Subject: [PATCH 067/153] Revert back to using old_package_metadata for include_dependencies. --- src/pipx/commands.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pipx/commands.py b/src/pipx/commands.py index 1a71fe30d9..7f12928195 100644 --- a/src/pipx/commands.py +++ b/src/pipx/commands.py @@ -229,8 +229,8 @@ def upgrade( package=package, package_or_url=package_or_url, pip_args=old_package_metadata.pip_args, - # TODO 20191026: should this be old_package_metadata.include_dependencies? - include_dependencies=include_dependencies, + # TODO 20191026: should this be `include_dependencies`? + include_dependencies=old_package_metadata.include_dependencies, include_apps=old_package_metadata.include_apps, is_main=True, ) From 80ce2f0c144b699a1d63c3d09a646732267c6e52 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Sun, 27 Oct 2019 12:47:39 -0700 Subject: [PATCH 068/153] Remove another get_venv_metadata_for_package(). --- src/pipx/commands.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/pipx/commands.py b/src/pipx/commands.py index 7f12928195..ed5e9bc171 100644 --- a/src/pipx/commands.py +++ b/src/pipx/commands.py @@ -336,11 +336,6 @@ def install( try: venv.create_venv(venv_args, pip_args) venv.install_package(package_or_url, pip_args) - - if venv.get_venv_metadata_for_package(package).package_version is None: - venv.remove_venv() - raise PipxError(f"Could not find package {package}. Is the name correct?") - # if installed ok, write pipx_metadata venv.update_package_metadata( package=package, @@ -351,6 +346,10 @@ def install( is_main=True, ) + if venv.package_metadata[package].package_version is None: + venv.remove_venv() + raise PipxError(f"Could not find package {package}. Is the name correct?") + _run_post_install_actions( venv, package, local_bin_dir, venv_dir, include_dependencies, force=force ) From c1768c776d7b344a0663bdc5b8bb99843dc7bae7 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Sun, 27 Oct 2019 15:18:06 -0700 Subject: [PATCH 069/153] update_package_metadata(): Rename is_main to is_main_package. --- src/pipx/commands.py | 6 +++--- src/pipx/venv.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/pipx/commands.py b/src/pipx/commands.py index ed5e9bc171..deee684017 100644 --- a/src/pipx/commands.py +++ b/src/pipx/commands.py @@ -232,7 +232,7 @@ def upgrade( # TODO 20191026: should this be `include_dependencies`? include_dependencies=old_package_metadata.include_dependencies, include_apps=old_package_metadata.include_apps, - is_main=True, + is_main_package=True, ) package_metadata = venv.package_metadata[package] @@ -343,7 +343,7 @@ def install( pip_args=pip_args, include_dependencies=include_dependencies, include_apps=True, - is_main=True, + is_main_package=True, ) if venv.package_metadata[package].package_version is None: @@ -467,7 +467,7 @@ def inject( pip_args=pip_args, include_apps=include_apps, include_dependencies=include_dependencies, - is_main=False, + is_main_package=False, ) if include_apps: diff --git a/src/pipx/venv.py b/src/pipx/venv.py index 08fc013292..551529d937 100644 --- a/src/pipx/venv.py +++ b/src/pipx/venv.py @@ -197,10 +197,10 @@ def update_package_metadata( pip_args: List[str], include_dependencies: bool, include_apps: bool, - is_main: bool, + is_main_package: bool, ): venv_package_metadata = self.get_venv_metadata_for_package(package) - if is_main: + if is_main_package: self.pipx_metadata.main_package = PackageInfo( package_or_url=abs_path_if_local(package_or_url, self, pip_args), pip_args=pip_args, From 34d04558ebe3f93b5647d65ba79c8395199d3cac Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Sun, 27 Oct 2019 15:23:34 -0700 Subject: [PATCH 070/153] Rename pipxrc.py -> pipx_metadata_file.py. --- pipx/{pipxrc.py => pipx_metadata_file.py} | 0 src/pipx/venv.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename pipx/{pipxrc.py => pipx_metadata_file.py} (100%) diff --git a/pipx/pipxrc.py b/pipx/pipx_metadata_file.py similarity index 100% rename from pipx/pipxrc.py rename to pipx/pipx_metadata_file.py diff --git a/src/pipx/venv.py b/src/pipx/venv.py index 551529d937..7e8049d7b2 100644 --- a/src/pipx/venv.py +++ b/src/pipx/venv.py @@ -8,7 +8,7 @@ from pipx.animate import animate from pipx.constants import DEFAULT_PYTHON, PIPX_SHARED_PTH, WINDOWS -from pipx.pipxrc import PipxMetadata, PackageInfo +from pipx.pipx_metadata_file import PipxMetadata, PackageInfo from pipx.shared_libs import shared_libs from pipx.util import ( PipxError, From fa5ccc65958e539800b52d9bb9b3c464bf651a26 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Sun, 27 Oct 2019 23:57:58 -0700 Subject: [PATCH 071/153] Integrate Venv.update_package_metadata() in install(). Also make Venv.install() check for proper package install and raise Error if install failed. --- src/pipx/commands.py | 69 +++++++++++++++++++++++++++----------------- src/pipx/venv.py | 25 +++++++++++++++- 2 files changed, 66 insertions(+), 28 deletions(-) diff --git a/src/pipx/commands.py b/src/pipx/commands.py index deee684017..f2bce28244 100644 --- a/src/pipx/commands.py +++ b/src/pipx/commands.py @@ -30,7 +30,7 @@ rmdir, run_pypackage_bin, ) -from pipx.venv import Venv, VenvContainer +from pipx.venv import Venv, VenvContainer, PackageInstallFailureError def run( @@ -115,7 +115,7 @@ def run( def _download_and_run( venv_dir: Path, - package: str, + package_or_url: str, app: str, binary_args: List[str], python: str, @@ -123,9 +123,22 @@ def _download_and_run( venv_args: List[str], verbose: bool, ): + # placeholder package name to refer to metadata in venv + package = "run_package" + venv = Venv(venv_dir, python=python, verbose=verbose) venv.create_venv(venv_args, pip_args) - venv.install_package(package, pip_args) + try: + venv.install_package( + package=package, + package_or_url=package_or_url, + pip_args=pip_args, + include_dependencies=False, + include_apps=True, + is_main_package=True, + ) + except PackageInstallFailureError: + raise PipxError(f"Unable to install {package_or_url}") if not (venv.bin_path / app).exists(): apps = venv.get_venv_metadata_for_package(package).apps @@ -335,20 +348,20 @@ def install( venv = Venv(venv_dir, python=python, verbose=verbose) try: venv.create_venv(venv_args, pip_args) - venv.install_package(package_or_url, pip_args) - # if installed ok, write pipx_metadata - venv.update_package_metadata( - package=package, - package_or_url=package_or_url, - pip_args=pip_args, - include_dependencies=include_dependencies, - include_apps=True, - is_main_package=True, - ) - - if venv.package_metadata[package].package_version is None: + try: + venv.install_package( + package=package, + package_or_url=package_or_url, + pip_args=pip_args, + include_dependencies=include_dependencies, + include_apps=True, + is_main_package=True, + ) + except PackageInstallFailureError: venv.remove_venv() - raise PipxError(f"Could not find package {package}. Is the name correct?") + raise PipxError( + f"Could not install package {package}. Is the name or spec correct?" + ) _run_post_install_actions( venv, package, local_bin_dir, venv_dir, include_dependencies, force=force @@ -459,17 +472,19 @@ def inject( ) venv = Venv(venv_dir, verbose=verbose) - venv.install_package(package_or_url, pip_args) - # TODO 20191026: verify installed ok like install()? - venv.update_package_metadata( - package=package, - package_or_url=package_or_url, - pip_args=pip_args, - include_apps=include_apps, - include_dependencies=include_dependencies, - is_main_package=False, - ) - + try: + venv.install_package( + package=package, + package_or_url=package_or_url, + pip_args=pip_args, + include_dependencies=include_dependencies, + include_apps=include_apps, + is_main_package=False, + ) + except PackageInstallFailureError: + raise PipxError( + f"Could not inject package {package}. Is the name or spec correct?" + ) if include_apps: _run_post_install_actions( venv, diff --git a/src/pipx/venv.py b/src/pipx/venv.py index 7e8049d7b2..395749bb71 100644 --- a/src/pipx/venv.py +++ b/src/pipx/venv.py @@ -20,6 +20,10 @@ ) +class PackageInstallFailureError(Exception): + pass + + class VenvContainer: """A collection of venvs managed by pipx. """ @@ -157,12 +161,31 @@ def upgrade_packaging_libraries(self, pip_args: List[str]) -> None: # but shared libs code does. self.upgrade_package("pip", pip_args) - def install_package(self, package_or_url: str, pip_args: List[str]) -> None: + def install_package( + self, + package: str, + package_or_url: str, + pip_args: List[str], + include_dependencies: bool, + include_apps: bool, + is_main_package: bool, + ) -> None: with animate(f"installing package {package_or_url!r}", self.do_animation): if pip_args is None: pip_args = [] cmd = ["install"] + pip_args + [package_or_url] self._run_pip(cmd) + self.update_package_metadata( + package=package, + package_or_url=package_or_url, + pip_args=pip_args, + include_dependencies=include_dependencies, + include_apps=include_apps, + is_main_package=is_main_package, + ) + # Verify package installed ok + if self.package_metadata[package].package_version is None: + raise PackageInstallFailureError def get_venv_metadata_for_package(self, package: str) -> VenvMetadata: data = json.loads( From 8afdfba1cff76b9c8dacc28c740245133a683f4d Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Mon, 28 Oct 2019 01:50:12 -0700 Subject: [PATCH 072/153] Enable install_package to determine package name. --- src/pipx/venv.py | 42 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/src/pipx/venv.py b/src/pipx/venv.py index 395749bb71..e74b63bb6f 100644 --- a/src/pipx/venv.py +++ b/src/pipx/venv.py @@ -4,7 +4,7 @@ import re import subprocess from pathlib import Path -from typing import Generator, List, NamedTuple, Dict +from typing import Generator, List, NamedTuple, Dict, Set from pipx.animate import animate from pipx.constants import DEFAULT_PYTHON, PIPX_SHARED_PTH, WINDOWS @@ -163,18 +163,28 @@ def upgrade_packaging_libraries(self, pip_args: List[str]) -> None: def install_package( self, - package: str, + package: str, # if "", will be determined package_or_url: str, pip_args: List[str], include_dependencies: bool, include_apps: bool, is_main_package: bool, ) -> None: + if not package: + # If no package name is supplied, find out package name installed + # by comparing old_package_set with new one after install + old_package_set = self.list_installed_packages() + with animate(f"installing package {package_or_url!r}", self.do_animation): if pip_args is None: pip_args = [] cmd = ["install"] + pip_args + [package_or_url] self._run_pip(cmd) + + if not package: + installed_packages = self.list_installed_packages() - old_package_set + package = self.top_of_deptree(installed_packages) + self.update_package_metadata( package=package, package_or_url=package_or_url, @@ -183,6 +193,7 @@ def install_package( include_apps=include_apps, is_main_package=is_main_package, ) + # Verify package installed ok if self.package_metadata[package].package_version is None: raise PackageInstallFailureError @@ -263,6 +274,33 @@ def pip_search(self, search_term, pip_search_args): cmd_run = subprocess.run(cmd, stdout=subprocess.PIPE) return cmd_run.stdout.decode().strip() + def list_installed_packages(self) -> Set[str]: + cmd = [str(self.python_path), "-m", "pip", "list", "--format=json"] + cmd_run = subprocess.run(cmd, stdout=subprocess.PIPE) + pip_list = json.loads(cmd_run.stdout.decode().strip()) + return set([x["name"] for x in pip_list]) + + def top_of_deptree(self, packages): + top_package_name = "" + + cmd = [str(self.python_path), "-m", "pip", "show"] + list(packages) + cmd_run = subprocess.run(cmd, stdout=subprocess.PIPE) + pip_show_stdout = cmd_run.stdout.decode().strip() + + package_data: Dict[str, Dict[str, str]] = {} + for line in pip_show_stdout.split("\n"): + key_value_re = re.search(r"^([^:]+):\s(.*)", line) + if key_value_re: + key = key_value_re.group(1) + value = key_value_re.group(2) + if key == "Name": + package_name = value + if key == "Required-by" and value == "": + top_package_name = package_name + break + + return top_package_name + def run_app(self, app: str, app_args: List[str]): cmd = [str(self.bin_path / app)] + app_args try: From 42786563b89a1c0cec957e8b3fe58fe955bbf828 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Mon, 28 Oct 2019 02:07:25 -0700 Subject: [PATCH 073/153] Add package name to PackageInfo. --- pipx/pipx_metadata_file.py | 4 ++++ src/pipx/venv.py | 6 +++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/pipx/pipx_metadata_file.py b/pipx/pipx_metadata_file.py index 7136557234..2b6049591c 100644 --- a/pipx/pipx_metadata_file.py +++ b/pipx/pipx_metadata_file.py @@ -25,6 +25,7 @@ def _json_decoder_object_hook(json_dict): class PackageInfo(NamedTuple): + package: Optional[str] package_or_url: Optional[str] pip_args: List[str] include_dependencies: bool @@ -41,9 +42,11 @@ def __init__(self, venv_dir: Path, read: bool = True): self.venv_dir = venv_dir # We init this instance with reasonable fallback defaults for all # members, EXCEPT for those we cannot know: + # self.main_package.package=None # self.main_package.package_or_url=None # self.python_version=None self.main_package = PackageInfo( + package=None, package_or_url=None, pip_args=[], include_dependencies=False, @@ -71,6 +74,7 @@ def reset(self) -> None: # self.main_package.package_or_url=None # self.venv_metadata.package_or_url=None self.main_package = PackageInfo( + package=None, package_or_url=None, pip_args=[], include_dependencies=False, diff --git a/src/pipx/venv.py b/src/pipx/venv.py index e74b63bb6f..9fd0d244ea 100644 --- a/src/pipx/venv.py +++ b/src/pipx/venv.py @@ -117,7 +117,9 @@ def uses_shared_libs(self): @property def package_metadata(self) -> Dict[str, PackageInfo]: return_dict = self.pipx_metadata.injected_packages.copy() - return_dict[self.root.name] = self.pipx_metadata.main_package + return_dict[ + self.pipx_metadata.main_package.package + ] = self.pipx_metadata.main_package return return_dict def create_venv(self, venv_args: List[str], pip_args: List[str]) -> None: @@ -236,6 +238,7 @@ def update_package_metadata( venv_package_metadata = self.get_venv_metadata_for_package(package) if is_main_package: self.pipx_metadata.main_package = PackageInfo( + package=package, package_or_url=abs_path_if_local(package_or_url, self, pip_args), pip_args=pip_args, include_dependencies=include_dependencies, @@ -248,6 +251,7 @@ def update_package_metadata( ) else: self.pipx_metadata.injected_packages[package] = PackageInfo( + package=package, package_or_url=package_or_url, pip_args=pip_args, include_apps=include_apps, From e9f64e651d37706974c66a7f3ba41340c4709eef Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Mon, 28 Oct 2019 02:16:20 -0700 Subject: [PATCH 074/153] Make run find package name or use empty string for install_package. --- src/pipx/commands.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/pipx/commands.py b/src/pipx/commands.py index f2bce28244..e888ea15c1 100644 --- a/src/pipx/commands.py +++ b/src/pipx/commands.py @@ -123,11 +123,12 @@ def _download_and_run( venv_args: List[str], verbose: bool, ): - # placeholder package name to refer to metadata in venv - package = "run_package" - venv = Venv(venv_dir, python=python, verbose=verbose) venv.create_venv(venv_args, pip_args) + # if venv is pre-existing and already installed with package, we + # can get package name from it, otherwise use empty string + package = venv.pipx_metadata.main_package.package or "" + try: venv.install_package( package=package, @@ -141,7 +142,7 @@ def _download_and_run( raise PipxError(f"Unable to install {package_or_url}") if not (venv.bin_path / app).exists(): - apps = venv.get_venv_metadata_for_package(package).apps + apps = venv.pipx_metadata.main_package.apps raise PipxError( f"'{app}' executable script not found in package '{package}'. " "Available executable scripts: " From d58887d2a40b22c7eb94f4975c15f5e5f9c4556a Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Mon, 28 Oct 2019 02:17:17 -0700 Subject: [PATCH 075/153] Change package_metadata to get name of main_package from PackageInfo. --- src/pipx/venv.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/pipx/venv.py b/src/pipx/venv.py index 9fd0d244ea..5657359fd4 100644 --- a/src/pipx/venv.py +++ b/src/pipx/venv.py @@ -117,9 +117,10 @@ def uses_shared_libs(self): @property def package_metadata(self) -> Dict[str, PackageInfo]: return_dict = self.pipx_metadata.injected_packages.copy() - return_dict[ - self.pipx_metadata.main_package.package - ] = self.pipx_metadata.main_package + if self.pipx_metadata.main_package.package is not None: + return_dict[ + self.pipx_metadata.main_package.package + ] = self.pipx_metadata.main_package return return_dict def create_venv(self, venv_args: List[str], pip_args: List[str]) -> None: @@ -291,7 +292,6 @@ def top_of_deptree(self, packages): cmd_run = subprocess.run(cmd, stdout=subprocess.PIPE) pip_show_stdout = cmd_run.stdout.decode().strip() - package_data: Dict[str, Dict[str, str]] = {} for line in pip_show_stdout.split("\n"): key_value_re = re.search(r"^([^:]+):\s(.*)", line) if key_value_re: From 198141e410a2c91dc01949586355114baf7f1d7d Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Mon, 28 Oct 2019 02:33:03 -0700 Subject: [PATCH 076/153] In Venv move update_package_metadata into upgrade_package. --- src/pipx/commands.py | 12 +++++------- src/pipx/venv.py | 25 +++++++++++++++++++++++-- 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/src/pipx/commands.py b/src/pipx/commands.py index e888ea15c1..a0f9a62408 100644 --- a/src/pipx/commands.py +++ b/src/pipx/commands.py @@ -236,18 +236,16 @@ def upgrade( # Upgrade shared libraries (pip, setuptools and wheel) venv.upgrade_packaging_libraries(pip_args) - venv.upgrade_package(package_or_url, pip_args) - # TODO 20191026: Should we upgrade injected packages also? - - venv.update_package_metadata( - package=package, - package_or_url=package_or_url, - pip_args=old_package_metadata.pip_args, + venv.upgrade_package( + package, + package_or_url, + pip_args, # TODO 20191026: should this be `include_dependencies`? include_dependencies=old_package_metadata.include_dependencies, include_apps=old_package_metadata.include_apps, is_main_package=True, ) + # TODO 20191026: Should we upgrade injected packages also? package_metadata = venv.package_metadata[package] new_version = package_metadata.package_version diff --git a/src/pipx/venv.py b/src/pipx/venv.py index 5657359fd4..8c5513903a 100644 --- a/src/pipx/venv.py +++ b/src/pipx/venv.py @@ -162,7 +162,7 @@ def upgrade_packaging_libraries(self, pip_args: List[str]) -> None: else: # TODO: setuptools and wheel? Original code didn't bother # but shared libs code does. - self.upgrade_package("pip", pip_args) + self._upgrade_package_no_metadata("pip", pip_args) def install_package( self, @@ -312,10 +312,31 @@ def run_app(self, app: str, app_args: List[str]): except KeyboardInterrupt: pass - def upgrade_package(self, package_or_url: str, pip_args: List[str]): + def _upgrade_package_no_metadata(self, package_or_url: str, pip_args: List[str]): with animate(f"upgrading package {package_or_url!r}", self.do_animation): self._run_pip(["install"] + pip_args + ["--upgrade", package_or_url]) + def upgrade_package( + self, + package: str, + package_or_url: str, + pip_args: List[str], + include_dependencies: bool, + include_apps: bool, + is_main_package: bool, + ): + with animate(f"upgrading package {package_or_url!r}", self.do_animation): + self._run_pip(["install"] + pip_args + ["--upgrade", package_or_url]) + + self.update_package_metadata( + package=package, + package_or_url=package_or_url, + pip_args=pip_args, + include_dependencies=include_dependencies, + include_apps=include_apps, + is_main_package=is_main_package, + ) + def _run_pip(self, cmd): cmd = [self.python_path, "-m", "pip"] + cmd if not self.verbose: From 17d241fb9325aafd3df78e66701ea8fdb60ea5f5 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Mon, 28 Oct 2019 22:44:54 -0700 Subject: [PATCH 077/153] Simplify code of update_package_metadata(). --- src/pipx/venv.py | 38 ++++++++++++++------------------------ 1 file changed, 14 insertions(+), 24 deletions(-) diff --git a/src/pipx/venv.py b/src/pipx/venv.py index 8c5513903a..bfd97783f2 100644 --- a/src/pipx/venv.py +++ b/src/pipx/venv.py @@ -237,32 +237,22 @@ def update_package_metadata( is_main_package: bool, ): venv_package_metadata = self.get_venv_metadata_for_package(package) + package_info = PackageInfo( + package=package, + package_or_url=abs_path_if_local(package_or_url, self, pip_args), + pip_args=pip_args, + include_apps=include_apps, + include_dependencies=include_dependencies, + apps=venv_package_metadata.apps, + app_paths=venv_package_metadata.app_paths, + apps_of_dependencies=venv_package_metadata.apps_of_dependencies, + app_paths_of_dependencies=venv_package_metadata.app_paths_of_dependencies, + package_version=venv_package_metadata.package_version, + ) if is_main_package: - self.pipx_metadata.main_package = PackageInfo( - package=package, - package_or_url=abs_path_if_local(package_or_url, self, pip_args), - pip_args=pip_args, - include_dependencies=include_dependencies, - include_apps=True, - apps=venv_package_metadata.apps, - app_paths=venv_package_metadata.app_paths, - apps_of_dependencies=venv_package_metadata.apps_of_dependencies, - app_paths_of_dependencies=venv_package_metadata.app_paths_of_dependencies, - package_version=venv_package_metadata.package_version, - ) + self.pipx_metadata.main_package = package_info else: - self.pipx_metadata.injected_packages[package] = PackageInfo( - package=package, - package_or_url=package_or_url, - pip_args=pip_args, - include_apps=include_apps, - include_dependencies=include_dependencies, - apps=venv_package_metadata.apps, - app_paths=venv_package_metadata.app_paths, - apps_of_dependencies=venv_package_metadata.apps_of_dependencies, - app_paths_of_dependencies=venv_package_metadata.app_paths_of_dependencies, - package_version=venv_package_metadata.package_version, - ) + self.pipx_metadata.injected_packages[package] = package_info self.pipx_metadata.write() From d61fa620933281066c7671f8f72ff7e7f82f8e50 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Tue, 29 Oct 2019 19:06:07 -0700 Subject: [PATCH 078/153] Start of test_pipx_metadata_file.py. --- tests/test_pipx_metadata_file.py | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 tests/test_pipx_metadata_file.py diff --git a/tests/test_pipx_metadata_file.py b/tests/test_pipx_metadata_file.py new file mode 100644 index 0000000000..e272e04a39 --- /dev/null +++ b/tests/test_pipx_metadata_file.py @@ -0,0 +1,10 @@ +from pipx.pipx_metadata_file import PipxMetadata, PackageInfo + + +# test to make sure we never have duplicate injected packages +# this might happen during reinstall_all if we don't properly uninstall +# injected packages or don't remove their metadata + + +def test_pipx_metadata_file_create(): + assert False From 5f2040ef0a9d7e74051bd966676acaf0ad127127 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Tue, 29 Oct 2019 19:06:26 -0700 Subject: [PATCH 079/153] Clarify comment. --- src/pipx/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pipx/main.py b/src/pipx/main.py index 7b503b7056..c92af308b8 100644 --- a/src/pipx/main.py +++ b/src/pipx/main.py @@ -178,7 +178,7 @@ def run_pipx_command(args): # noqa: C901 commands.inject( venv_dir, dep, - dep, # NOP now, but in future can be package_or_url --spec + dep, # NOP now, but in future can be package_or_url from --spec pip_args, verbose=verbose, include_apps=args.include_apps, From c4f44aafcaacebbc9a0dbf01435f8b26a841b090 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Wed, 30 Oct 2019 14:33:42 -0700 Subject: [PATCH 080/153] Comment false assert. --- tests/test_pipx_metadata_file.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_pipx_metadata_file.py b/tests/test_pipx_metadata_file.py index e272e04a39..7548f7d7ed 100644 --- a/tests/test_pipx_metadata_file.py +++ b/tests/test_pipx_metadata_file.py @@ -7,4 +7,5 @@ def test_pipx_metadata_file_create(): - assert False + pass + #assert False From 12c82a7bcd190334261353e03b89038c4a0e3600 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Wed, 30 Oct 2019 14:40:45 -0700 Subject: [PATCH 081/153] Add temp_path fixture. --- tests/test_pipx_metadata_file.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_pipx_metadata_file.py b/tests/test_pipx_metadata_file.py index 7548f7d7ed..4059e0781a 100644 --- a/tests/test_pipx_metadata_file.py +++ b/tests/test_pipx_metadata_file.py @@ -6,6 +6,7 @@ # injected packages or don't remove their metadata -def test_pipx_metadata_file_create(): - pass - #assert False +def test_pipx_metadata_file_create(tmp_path): + venv_temp = tmp_path / "venv1" + pipx_metadata = PipxMetadata(venv_temp) + # assert False From a86d0dd97ed3f5f2106cc919833297b7e40eca3d Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Wed, 30 Oct 2019 16:19:03 -0700 Subject: [PATCH 082/153] Add mock PackageInfo to test. --- tests/test_pipx_metadata_file.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/test_pipx_metadata_file.py b/tests/test_pipx_metadata_file.py index 4059e0781a..82a9dc83ee 100644 --- a/tests/test_pipx_metadata_file.py +++ b/tests/test_pipx_metadata_file.py @@ -9,4 +9,15 @@ def test_pipx_metadata_file_create(tmp_path): venv_temp = tmp_path / "venv1" pipx_metadata = PipxMetadata(venv_temp) - # assert False + package_info = PackageInfo( + package="test_package", + package_or_url="test_package_url", + pip_args=[], + include_apps=True, + include_dependencies=True, + apps=[], + app_paths=[], + apps_of_dependencies=[], + app_paths_of_dependencies=[], + package_version="0.0.0", + ) From 314e73312aab5bc8b5908e2dd7e93e1d0c1db96a Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Wed, 30 Oct 2019 23:10:40 -0700 Subject: [PATCH 083/153] More tests. --- tests/test_pipx_metadata_file.py | 79 ++++++++++++++++++++++++++------ 1 file changed, 65 insertions(+), 14 deletions(-) diff --git a/tests/test_pipx_metadata_file.py b/tests/test_pipx_metadata_file.py index 82a9dc83ee..223e73221f 100644 --- a/tests/test_pipx_metadata_file.py +++ b/tests/test_pipx_metadata_file.py @@ -1,4 +1,34 @@ +from pathlib import Path +import pytest # type: ignore + from pipx.pipx_metadata_file import PipxMetadata, PackageInfo +from pipx.util import PipxError + + +TEST_PACKAGE1 = PackageInfo( + package="test_package", + package_or_url="test_package_url", + pip_args=[], + include_apps=True, + include_dependencies=False, + apps=["testapp"], + app_paths=[Path("/usr/bin")], + apps_of_dependencies=["dep1"], + app_paths_of_dependencies={"dep1": [Path("bin")]}, + package_version="0.1.2", +) +TEST_PACKAGE2 = PackageInfo( + package="inj_package", + package_or_url="inj_package_url", + pip_args=["-e"], + include_apps=True, + include_dependencies=False, + apps=["injapp"], + app_paths=[Path("/usr/bin")], + apps_of_dependencies=["dep2"], + app_paths_of_dependencies={"dep2": [Path("bin")]}, + package_version="6.7.8", +) # test to make sure we never have duplicate injected packages @@ -7,17 +37,38 @@ def test_pipx_metadata_file_create(tmp_path): - venv_temp = tmp_path / "venv1" - pipx_metadata = PipxMetadata(venv_temp) - package_info = PackageInfo( - package="test_package", - package_or_url="test_package_url", - pip_args=[], - include_apps=True, - include_dependencies=True, - apps=[], - app_paths=[], - apps_of_dependencies=[], - app_paths_of_dependencies=[], - package_version="0.0.0", - ) + + pipx_metadata = PipxMetadata(tmp_path) + pipx_metadata.main_package = TEST_PACKAGE1 + pipx_metadata.python_version = "3.4.5" + pipx_metadata.venv_args = ["--system-site-packages"] + pipx_metadata.injected_packages = {"injected": TEST_PACKAGE2} + + pipx_metadata.write() + del pipx_metadata + + pipx_metadata2 = PipxMetadata(tmp_path) + assert pipx_metadata2.main_package == TEST_PACKAGE1 + assert pipx_metadata2.python_version == "3.4.5" + assert pipx_metadata2.venv_args == ["--system-site-packages"] + assert pipx_metadata2.injected_packages == {"injected": TEST_PACKAGE2} + + +def test_pipx_metadata_file_validation(tmp_path): + venv_dir1 = tmp_path / "venv1" + venv_dir1.mkdir() + venv_dir2 = tmp_path / "venv2" + venv_dir2.mkdir() + + test_package1_data = TEST_PACKAGE1._asdict() + test_package1_data["include_apps"] = False + test_package1 = PackageInfo(**test_package1_data) + + pipx_metadata = PipxMetadata(venv_dir1) + pipx_metadata.main_package = test_package1 + pipx_metadata.python_version = "3.4.5" + pipx_metadata.venv_args = ["--system-site-packages"] + pipx_metadata.injected_packages = {} + + with pytest.raises(PipxError): + pipx_metadata.write() From c727870a19aabd59a7f7cbc8954479580b38a70a Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Wed, 30 Oct 2019 23:18:49 -0700 Subject: [PATCH 084/153] Add another test to validate_before_write(). --- pipx/pipx_metadata_file.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pipx/pipx_metadata_file.py b/pipx/pipx_metadata_file.py index 2b6049591c..56f82a9d21 100644 --- a/pipx/pipx_metadata_file.py +++ b/pipx/pipx_metadata_file.py @@ -112,7 +112,8 @@ def from_dict(self, input_dict: Dict[str, Any]) -> None: def validate_before_write(self): if ( - self.main_package.package_or_url is None + self.main_package.package is None + or self.main_package.package_or_url is None or not self.main_package.include_apps ): raise PipxError("Internal Error: PipxMetadata is corrupt, cannot write.") From d73819de37fb4881778756690f6f6d1c513d997e Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Wed, 30 Oct 2019 23:20:44 -0700 Subject: [PATCH 085/153] Add more file validation testing. --- tests/test_pipx_metadata_file.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/tests/test_pipx_metadata_file.py b/tests/test_pipx_metadata_file.py index 223e73221f..2671c74609 100644 --- a/tests/test_pipx_metadata_file.py +++ b/tests/test_pipx_metadata_file.py @@ -64,11 +64,20 @@ def test_pipx_metadata_file_validation(tmp_path): test_package1_data["include_apps"] = False test_package1 = PackageInfo(**test_package1_data) - pipx_metadata = PipxMetadata(venv_dir1) - pipx_metadata.main_package = test_package1 - pipx_metadata.python_version = "3.4.5" - pipx_metadata.venv_args = ["--system-site-packages"] - pipx_metadata.injected_packages = {} + test_package2_data = TEST_PACKAGE1._asdict() + test_package2_data["package"] = None + test_package2 = PackageInfo(**test_package2_data) + + test_package3_data = TEST_PACKAGE1._asdict() + test_package3_data["package_or_url"] = None + test_package3 = PackageInfo(**test_package3_data) + + for test_package in [test_package1, test_package2, test_package3]: + pipx_metadata = PipxMetadata(venv_dir1) + pipx_metadata.main_package = test_package + pipx_metadata.python_version = "3.4.5" + pipx_metadata.venv_args = ["--system-site-packages"] + pipx_metadata.injected_packages = {} - with pytest.raises(PipxError): - pipx_metadata.write() + with pytest.raises(PipxError): + pipx_metadata.write() From 4fb7da3010871d2be7c807b871b9460db8301c76 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Thu, 31 Oct 2019 12:20:28 -0700 Subject: [PATCH 086/153] Simplify test package creation. --- tests/test_pipx_metadata_file.py | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/tests/test_pipx_metadata_file.py b/tests/test_pipx_metadata_file.py index 2671c74609..94bbc8be4c 100644 --- a/tests/test_pipx_metadata_file.py +++ b/tests/test_pipx_metadata_file.py @@ -60,17 +60,9 @@ def test_pipx_metadata_file_validation(tmp_path): venv_dir2 = tmp_path / "venv2" venv_dir2.mkdir() - test_package1_data = TEST_PACKAGE1._asdict() - test_package1_data["include_apps"] = False - test_package1 = PackageInfo(**test_package1_data) - - test_package2_data = TEST_PACKAGE1._asdict() - test_package2_data["package"] = None - test_package2 = PackageInfo(**test_package2_data) - - test_package3_data = TEST_PACKAGE1._asdict() - test_package3_data["package_or_url"] = None - test_package3 = PackageInfo(**test_package3_data) + test_package1 = TEST_PACKAGE1._replace(include_apps=False) + test_package2 = TEST_PACKAGE1._replace(package=None) + test_package3 = TEST_PACKAGE1._replace(package_or_url=None) for test_package in [test_package1, test_package2, test_package3]: pipx_metadata = PipxMetadata(venv_dir1) From 7c4daa6e0346956db992b45fe97b5aff93364dbe Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Fri, 1 Nov 2019 22:43:34 -0700 Subject: [PATCH 087/153] Add test of package install. --- tests/test_pipx_metadata_file.py | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/tests/test_pipx_metadata_file.py b/tests/test_pipx_metadata_file.py index 94bbc8be4c..3e9cd1f6eb 100644 --- a/tests/test_pipx_metadata_file.py +++ b/tests/test_pipx_metadata_file.py @@ -1,9 +1,13 @@ from pathlib import Path import pytest # type: ignore +from helpers import assert_not_in_virtualenv, run_pipx_cli +import pipx.constants from pipx.pipx_metadata_file import PipxMetadata, PackageInfo from pipx.util import PipxError +assert_not_in_virtualenv() + TEST_PACKAGE1 = PackageInfo( package="test_package", @@ -31,11 +35,6 @@ ) -# test to make sure we never have duplicate injected packages -# this might happen during reinstall_all if we don't properly uninstall -# injected packages or don't remove their metadata - - def test_pipx_metadata_file_create(tmp_path): pipx_metadata = PipxMetadata(tmp_path) @@ -73,3 +72,25 @@ def test_pipx_metadata_file_validation(tmp_path): with pytest.raises(PipxError): pipx_metadata.write() + + +def test_package_install(monkeypatch, tmp_path): + pipx_home = tmp_path / ".local" / "pipx" + pipx_home.mkdir(parents=True) + local_bin_dir = tmp_path / ".local" / "bin" + local_bin_dir.mkdir(parents=True) + pipx_local_venvs = pipx_home / "venvs" + pipx_local_venvs.mkdir(parents=True) + pipx_shared_libs = pipx_home / "shared" + pipx_shared_libs.mkdir(parents=True) + monkeypatch.setattr(pipx.constants, "PIPX_HOME", pipx_home) + monkeypatch.setattr(pipx.constants, "PIPX_LOCAL_VENVS", pipx_local_venvs) + monkeypatch.setattr(pipx.constants, "PIPX_SHARED_LIBS", pipx_shared_libs) + monkeypatch.setattr(pipx.constants, "LOCAL_BIN_DIR", local_bin_dir) + + run_pipx_cli(["install", "pycowsay"]) + assert (pipx_home / "venvs" / "pycowsay" / "pipx_metadata.json").is_file() + + +# confirm that package install creates pipx_metadata.json +# confirm that package inject adds injected package to pipx metadata From 226389f86b81cda4db55b35320ccf5cd98908e5a Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Sun, 3 Nov 2019 17:20:34 -0800 Subject: [PATCH 088/153] Fix rebase miss: pipx_metadata_file.py in wrong place. --- {pipx => src/pipx}/pipx_metadata_file.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {pipx => src/pipx}/pipx_metadata_file.py (100%) diff --git a/pipx/pipx_metadata_file.py b/src/pipx/pipx_metadata_file.py similarity index 100% rename from pipx/pipx_metadata_file.py rename to src/pipx/pipx_metadata_file.py From b16d30fad2b6426eccf29ddae845173dd01d2159 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Sun, 3 Nov 2019 20:15:57 -0800 Subject: [PATCH 089/153] Rename update_package_metadata -> _update_package_metadata. --- src/pipx/venv.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pipx/venv.py b/src/pipx/venv.py index bfd97783f2..fee939e0f2 100644 --- a/src/pipx/venv.py +++ b/src/pipx/venv.py @@ -188,7 +188,7 @@ def install_package( installed_packages = self.list_installed_packages() - old_package_set package = self.top_of_deptree(installed_packages) - self.update_package_metadata( + self._update_package_metadata( package=package, package_or_url=package_or_url, pip_args=pip_args, @@ -227,7 +227,7 @@ def get_venv_metadata_for_package(self, package: str) -> VenvMetadata: return VenvMetadata(**data) - def update_package_metadata( + def _update_package_metadata( self, package: str, package_or_url: str, @@ -318,7 +318,7 @@ def upgrade_package( with animate(f"upgrading package {package_or_url!r}", self.do_animation): self._run_pip(["install"] + pip_args + ["--upgrade", package_or_url]) - self.update_package_metadata( + self._update_package_metadata( package=package, package_or_url=package_or_url, pip_args=pip_args, From 01abd314268d3879b1ff2af9d359d18740b9b864 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Sun, 3 Nov 2019 20:17:01 -0800 Subject: [PATCH 090/153] Rename validate_before_write -> _validate_before_write. --- src/pipx/pipx_metadata_file.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pipx/pipx_metadata_file.py b/src/pipx/pipx_metadata_file.py index 56f82a9d21..66675d798c 100644 --- a/src/pipx/pipx_metadata_file.py +++ b/src/pipx/pipx_metadata_file.py @@ -110,7 +110,7 @@ def from_dict(self, input_dict: Dict[str, Any]) -> None: for (name, data) in input_dict["injected_packages"].items() } - def validate_before_write(self): + def _validate_before_write(self): if ( self.main_package.package is None or self.main_package.package_or_url is None @@ -119,7 +119,7 @@ def validate_before_write(self): raise PipxError("Internal Error: PipxMetadata is corrupt, cannot write.") def write(self) -> None: - self.validate_before_write() + self._validate_before_write() try: with open(self.venv_dir / PIPX_INFO_FILENAME, "w") as pipx_metadata_fh: json.dump( From 84846bbe08bba0b2fb23153dc265c1325b0a9000 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Sun, 3 Nov 2019 20:22:39 -0800 Subject: [PATCH 091/153] Remove spurious comments. --- src/pipx/pipx_metadata_file.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/pipx/pipx_metadata_file.py b/src/pipx/pipx_metadata_file.py index 66675d798c..fa870258a2 100644 --- a/src/pipx/pipx_metadata_file.py +++ b/src/pipx/pipx_metadata_file.py @@ -51,7 +51,6 @@ def __init__(self, venv_dir: Path, read: bool = True): pip_args=[], include_dependencies=False, include_apps=True, # always True for main_package - # apps=[], app_paths=[], apps_of_dependencies=[], @@ -79,7 +78,6 @@ def reset(self) -> None: pip_args=[], include_dependencies=False, include_apps=True, # always True for main_package - # apps=[], app_paths=[], apps_of_dependencies=[], From 205cc9468ccb35fafd9a32342ec3991c75f0f818 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Sun, 3 Nov 2019 21:07:52 -0800 Subject: [PATCH 092/153] Use fixture pipx_temp_env for installing. --- tests/test_pipx_metadata_file.py | 22 ++++------------------ 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/tests/test_pipx_metadata_file.py b/tests/test_pipx_metadata_file.py index 3e9cd1f6eb..a3c753b504 100644 --- a/tests/test_pipx_metadata_file.py +++ b/tests/test_pipx_metadata_file.py @@ -74,23 +74,9 @@ def test_pipx_metadata_file_validation(tmp_path): pipx_metadata.write() -def test_package_install(monkeypatch, tmp_path): - pipx_home = tmp_path / ".local" / "pipx" - pipx_home.mkdir(parents=True) - local_bin_dir = tmp_path / ".local" / "bin" - local_bin_dir.mkdir(parents=True) - pipx_local_venvs = pipx_home / "venvs" - pipx_local_venvs.mkdir(parents=True) - pipx_shared_libs = pipx_home / "shared" - pipx_shared_libs.mkdir(parents=True) - monkeypatch.setattr(pipx.constants, "PIPX_HOME", pipx_home) - monkeypatch.setattr(pipx.constants, "PIPX_LOCAL_VENVS", pipx_local_venvs) - monkeypatch.setattr(pipx.constants, "PIPX_SHARED_LIBS", pipx_shared_libs) - monkeypatch.setattr(pipx.constants, "LOCAL_BIN_DIR", local_bin_dir) - +def test_package_install(monkeypatch, tmp_path, pipx_temp_env): run_pipx_cli(["install", "pycowsay"]) assert (pipx_home / "venvs" / "pycowsay" / "pipx_metadata.json").is_file() - - -# confirm that package install creates pipx_metadata.json -# confirm that package inject adds injected package to pipx metadata + # confirm pipx_metadata.json attributes + # confirm that package inject adds injected package to pipx metadata + # confirm pipx_metadata.json injected package attributes From ff3cd5ee64ab2d56456ec3f35fd8be691da768e0 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Sun, 3 Nov 2019 21:23:13 -0800 Subject: [PATCH 093/153] Fix bug specifying pipx_home. --- tests/test_pipx_metadata_file.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_pipx_metadata_file.py b/tests/test_pipx_metadata_file.py index a3c753b504..6483cb5732 100644 --- a/tests/test_pipx_metadata_file.py +++ b/tests/test_pipx_metadata_file.py @@ -76,7 +76,9 @@ def test_pipx_metadata_file_validation(tmp_path): def test_package_install(monkeypatch, tmp_path, pipx_temp_env): run_pipx_cli(["install", "pycowsay"]) - assert (pipx_home / "venvs" / "pycowsay" / "pipx_metadata.json").is_file() + assert ( + pipx.constants.PIPX_HOME / "venvs" / "pycowsay" / "pipx_metadata.json" + ).is_file() # confirm pipx_metadata.json attributes # confirm that package inject adds injected package to pipx metadata # confirm pipx_metadata.json injected package attributes From 3d51c4a1fa4418c3993ab83ddcd3bde34c02bb1e Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Sun, 3 Nov 2019 21:51:30 -0800 Subject: [PATCH 094/153] Add install and inject metadata tests. --- tests/test_pipx_metadata_file.py | 44 +++++++++++++++++++++++++++----- 1 file changed, 37 insertions(+), 7 deletions(-) diff --git a/tests/test_pipx_metadata_file.py b/tests/test_pipx_metadata_file.py index 6483cb5732..9286d3a4c1 100644 --- a/tests/test_pipx_metadata_file.py +++ b/tests/test_pipx_metadata_file.py @@ -36,7 +36,6 @@ def test_pipx_metadata_file_create(tmp_path): - pipx_metadata = PipxMetadata(tmp_path) pipx_metadata.main_package = TEST_PACKAGE1 pipx_metadata.python_version = "3.4.5" @@ -75,10 +74,41 @@ def test_pipx_metadata_file_validation(tmp_path): def test_package_install(monkeypatch, tmp_path, pipx_temp_env): + pipx_venvs_dir = pipx.constants.PIPX_HOME / "venvs" + run_pipx_cli(["install", "pycowsay"]) - assert ( - pipx.constants.PIPX_HOME / "venvs" / "pycowsay" / "pipx_metadata.json" - ).is_file() - # confirm pipx_metadata.json attributes - # confirm that package inject adds injected package to pipx metadata - # confirm pipx_metadata.json injected package attributes + assert (pipx_venvs_dir / "pycowsay" / "pipx_metadata.json").is_file() + + pipx_metadata = PipxMetadata(pipx_venvs_dir / "pycowsay") + assert pipx_metadata.main_package.package == "pycowsay" + assert pipx_metadata.main_package.package_or_url == "pycowsay" + assert pipx_metadata.main_package.pip_args == [] + assert pipx_metadata.main_package.include_dependencies == False + assert pipx_metadata.main_package.include_apps == True + assert pipx_metadata.main_package.apps == ["pycowsay"] + assert pipx_metadata.main_package.app_paths == [ + pipx_venvs_dir / "pycowsay" / "bin" / "pycowsay" + ] + assert pipx_metadata.main_package.apps_of_dependencies == [] + assert pipx_metadata.main_package.app_paths_of_dependencies == {} + assert pipx_metadata.main_package.package_version != "" + + del pipx_metadata + + # TODO 20191103: need simpler non-gcc-compiling package besides black! + run_pipx_cli(["inject", "pycowsay", "black"]) + + pipx_metadata = PipxMetadata(pipx_venvs_dir / "pycowsay") + assert pipx_metadata.injected_packages["black"].package == "black" + assert pipx_metadata.injected_packages["black"].package_or_url == "black" + assert pipx_metadata.injected_packages["black"].pip_args == [] + assert pipx_metadata.injected_packages["black"].include_dependencies == False + assert pipx_metadata.injected_packages["black"].include_apps == False + assert pipx_metadata.injected_packages["black"].apps == ["black", "blackd"] + assert pipx_metadata.injected_packages["black"].app_paths == [ + pipx_venvs_dir / "pycowsay" / "bin" / "black", + pipx_venvs_dir / "pycowsay" / "bin" / "blackd", + ] + assert pipx_metadata.injected_packages["black"].apps_of_dependencies == [] + assert pipx_metadata.injected_packages["black"].app_paths_of_dependencies == {} + assert pipx_metadata.injected_packages["black"].package_version != "" From c7dbdb5af60b042e5182bf3bd25c610843ae76bc Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Sun, 3 Nov 2019 21:57:38 -0800 Subject: [PATCH 095/153] Add noqa to asserts for flake8. --- tests/test_pipx_metadata_file.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/test_pipx_metadata_file.py b/tests/test_pipx_metadata_file.py index 9286d3a4c1..7758de9c50 100644 --- a/tests/test_pipx_metadata_file.py +++ b/tests/test_pipx_metadata_file.py @@ -83,8 +83,8 @@ def test_package_install(monkeypatch, tmp_path, pipx_temp_env): assert pipx_metadata.main_package.package == "pycowsay" assert pipx_metadata.main_package.package_or_url == "pycowsay" assert pipx_metadata.main_package.pip_args == [] - assert pipx_metadata.main_package.include_dependencies == False - assert pipx_metadata.main_package.include_apps == True + assert pipx_metadata.main_package.include_dependencies == False # noqa: E712 + assert pipx_metadata.main_package.include_apps == True # noqa: E712 assert pipx_metadata.main_package.apps == ["pycowsay"] assert pipx_metadata.main_package.app_paths == [ pipx_venvs_dir / "pycowsay" / "bin" / "pycowsay" @@ -102,8 +102,10 @@ def test_package_install(monkeypatch, tmp_path, pipx_temp_env): assert pipx_metadata.injected_packages["black"].package == "black" assert pipx_metadata.injected_packages["black"].package_or_url == "black" assert pipx_metadata.injected_packages["black"].pip_args == [] - assert pipx_metadata.injected_packages["black"].include_dependencies == False - assert pipx_metadata.injected_packages["black"].include_apps == False + assert ( + pipx_metadata.injected_packages["black"].include_dependencies == False + ) # noqa: E712 + assert pipx_metadata.injected_packages["black"].include_apps == False # noqa: E712 assert pipx_metadata.injected_packages["black"].apps == ["black", "blackd"] assert pipx_metadata.injected_packages["black"].app_paths == [ pipx_venvs_dir / "pycowsay" / "bin" / "black", From 2c2189f65545a51eb2afa6a4048b18b168162271 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Sun, 3 Nov 2019 22:08:57 -0800 Subject: [PATCH 096/153] Fix expected pacakge metadata for Windows. --- tests/test_pipx_metadata_file.py | 39 ++++++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/tests/test_pipx_metadata_file.py b/tests/test_pipx_metadata_file.py index 7758de9c50..f24658ec51 100644 --- a/tests/test_pipx_metadata_file.py +++ b/tests/test_pipx_metadata_file.py @@ -85,10 +85,17 @@ def test_package_install(monkeypatch, tmp_path, pipx_temp_env): assert pipx_metadata.main_package.pip_args == [] assert pipx_metadata.main_package.include_dependencies == False # noqa: E712 assert pipx_metadata.main_package.include_apps == True # noqa: E712 - assert pipx_metadata.main_package.apps == ["pycowsay"] - assert pipx_metadata.main_package.app_paths == [ - pipx_venvs_dir / "pycowsay" / "bin" / "pycowsay" - ] + if pipx.constants.WINDOWS: + assert pipx_metadata.main_package.apps == ["pycowsay", "pycowsay.exe"] + assert pipx_metadata.main_package.app_paths == [ + pipx_venvs_dir / "pycowsay" / "bin" / "pycowsay", + pipx_venvs_dir / "pycowsay" / "bin" / "pycowsay.exe", + ] + else: + assert pipx_metadata.main_package.apps == ["pycowsay"] + assert pipx_metadata.main_package.app_paths == [ + pipx_venvs_dir / "pycowsay" / "bin" / "pycowsay" + ] assert pipx_metadata.main_package.apps_of_dependencies == [] assert pipx_metadata.main_package.app_paths_of_dependencies == {} assert pipx_metadata.main_package.package_version != "" @@ -106,11 +113,25 @@ def test_package_install(monkeypatch, tmp_path, pipx_temp_env): pipx_metadata.injected_packages["black"].include_dependencies == False ) # noqa: E712 assert pipx_metadata.injected_packages["black"].include_apps == False # noqa: E712 - assert pipx_metadata.injected_packages["black"].apps == ["black", "blackd"] - assert pipx_metadata.injected_packages["black"].app_paths == [ - pipx_venvs_dir / "pycowsay" / "bin" / "black", - pipx_venvs_dir / "pycowsay" / "bin" / "blackd", - ] + if pipx.constants.WINDOWS: + assert pipx_metadata.injected_packages["black"].apps == [ + "black", + "black.exe", + "blackd", + "blackd.exe", + ] + assert pipx_metadata.injected_packages["black"].app_paths == [ + pipx_venvs_dir / "pycowsay" / "bin" / "black", + pipx_venvs_dir / "pycowsay" / "bin" / "black.exe", + pipx_venvs_dir / "pycowsay" / "bin" / "blackd", + pipx_venvs_dir / "pycowsay" / "bin" / "blackd.exe", + ] + else: + assert pipx_metadata.injected_packages["black"].apps == ["black", "blackd"] + assert pipx_metadata.injected_packages["black"].app_paths == [ + pipx_venvs_dir / "pycowsay" / "bin" / "black", + pipx_venvs_dir / "pycowsay" / "bin" / "blackd", + ] assert pipx_metadata.injected_packages["black"].apps_of_dependencies == [] assert pipx_metadata.injected_packages["black"].app_paths_of_dependencies == {} assert pipx_metadata.injected_packages["black"].package_version != "" From 6313d0d2579ab2d82b8943eb72229b1f07528567 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Sun, 3 Nov 2019 22:09:18 -0800 Subject: [PATCH 097/153] Remove obsolete TODO. --- src/pipx/commands.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/pipx/commands.py b/src/pipx/commands.py index a0f9a62408..31a6279d5d 100644 --- a/src/pipx/commands.py +++ b/src/pipx/commands.py @@ -512,7 +512,6 @@ def uninstall(venv_dir: Path, package: str, local_bin_dir: Path, verbose: bool): return # TODO 20191024: Uninstall injected packages apps in bin - # TODO 20191026: Handle error if package is not installed? venv = Venv(venv_dir, verbose=verbose) if venv.pipx_metadata.main_package is not None: From d6dcf42c8fc630e7bcffb20bf2d3da7bafe91c01 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Sun, 3 Nov 2019 22:25:34 -0800 Subject: [PATCH 098/153] Add Windows fixes. --- tests/test_pipx_metadata_file.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/tests/test_pipx_metadata_file.py b/tests/test_pipx_metadata_file.py index f24658ec51..f7f85aeddd 100644 --- a/tests/test_pipx_metadata_file.py +++ b/tests/test_pipx_metadata_file.py @@ -88,8 +88,7 @@ def test_package_install(monkeypatch, tmp_path, pipx_temp_env): if pipx.constants.WINDOWS: assert pipx_metadata.main_package.apps == ["pycowsay", "pycowsay.exe"] assert pipx_metadata.main_package.app_paths == [ - pipx_venvs_dir / "pycowsay" / "bin" / "pycowsay", - pipx_venvs_dir / "pycowsay" / "bin" / "pycowsay.exe", + pipx_venvs_dir / "pycowsay" / "Scripts" / "pycowsay.exe" ] else: assert pipx_metadata.main_package.apps == ["pycowsay"] @@ -121,10 +120,8 @@ def test_package_install(monkeypatch, tmp_path, pipx_temp_env): "blackd.exe", ] assert pipx_metadata.injected_packages["black"].app_paths == [ - pipx_venvs_dir / "pycowsay" / "bin" / "black", - pipx_venvs_dir / "pycowsay" / "bin" / "black.exe", - pipx_venvs_dir / "pycowsay" / "bin" / "blackd", - pipx_venvs_dir / "pycowsay" / "bin" / "blackd.exe", + pipx_venvs_dir / "pycowsay" / "Scripts" / "black.exe", + pipx_venvs_dir / "pycowsay" / "Scripts" / "blackd.exe", ] else: assert pipx_metadata.injected_packages["black"].apps == ["black", "blackd"] From ba67fcb43d9304201af35cd5e1ffec84183e3107 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Sun, 3 Nov 2019 23:13:13 -0800 Subject: [PATCH 099/153] Dont depend on order of list items when it is not important. --- tests/test_pipx_metadata_file.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/tests/test_pipx_metadata_file.py b/tests/test_pipx_metadata_file.py index f7f85aeddd..4dfa5e60ad 100644 --- a/tests/test_pipx_metadata_file.py +++ b/tests/test_pipx_metadata_file.py @@ -113,22 +113,28 @@ def test_package_install(monkeypatch, tmp_path, pipx_temp_env): ) # noqa: E712 assert pipx_metadata.injected_packages["black"].include_apps == False # noqa: E712 if pipx.constants.WINDOWS: - assert pipx_metadata.injected_packages["black"].apps == [ + # order is not important, so we compare sets + assert isinstance(pipx_metadata.injected_packages["black"].apps, list) + assert set(pipx_metadata.injected_packages["black"].apps) == { "black", "black.exe", "blackd", "blackd.exe", - ] - assert pipx_metadata.injected_packages["black"].app_paths == [ + } + assert isinstance(pipx_metadata.injected_packages["black"].app_paths, list) + assert set(pipx_metadata.injected_packages["black"].app_paths) == { pipx_venvs_dir / "pycowsay" / "Scripts" / "black.exe", pipx_venvs_dir / "pycowsay" / "Scripts" / "blackd.exe", - ] + } else: - assert pipx_metadata.injected_packages["black"].apps == ["black", "blackd"] - assert pipx_metadata.injected_packages["black"].app_paths == [ + # order is not important, so we compare sets + assert isinstance(pipx_metadata.injected_packages["black"].apps, list) + assert set(pipx_metadata.injected_packages["black"].apps) == {"black", "blackd"} + assert isinstance(pipx_metadata.injected_packages["black"].app_paths, list) + assert set(pipx_metadata.injected_packages["black"].app_paths) == { pipx_venvs_dir / "pycowsay" / "bin" / "black", pipx_venvs_dir / "pycowsay" / "bin" / "blackd", - ] + } assert pipx_metadata.injected_packages["black"].apps_of_dependencies == [] assert pipx_metadata.injected_packages["black"].app_paths_of_dependencies == {} assert pipx_metadata.injected_packages["black"].package_version != "" From b79b6b40c432bb9b3e34c5998a500f94f5135b42 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Mon, 4 Nov 2019 21:12:22 -0800 Subject: [PATCH 100/153] Merge part 2, fix other conflict. --- src/pipx/commands/commands.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/pipx/commands/commands.py b/src/pipx/commands/commands.py index cddfd8c30e..5df4111c8b 100644 --- a/src/pipx/commands/commands.py +++ b/src/pipx/commands/commands.py @@ -592,14 +592,7 @@ def _get_package_summary( python_path = venv.python_path.resolve() if package is None: package = path.name -<<<<<<< HEAD package_metadata = venv.package_metadata[package] -======= - if not python_path.is_file(): - return f" package {red(bold(package))} has invalid interpreter {str(python_path)}" - - metadata = venv.get_venv_metadata_for_package(package) ->>>>>>> 5417f064581cdfc289f7db8aabf6255db613bd68 if package_metadata.package_version is None: not_installed = red("is not installed") From 25a9fd599630c70cc0b6087aa6bc6fd1eb671e20 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Mon, 4 Nov 2019 21:27:21 -0800 Subject: [PATCH 101/153] Merge part3, put missing python error back in _get_package_summary. --- src/pipx/commands/commands.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/pipx/commands/commands.py b/src/pipx/commands/commands.py index 5df4111c8b..e48f3f0180 100644 --- a/src/pipx/commands/commands.py +++ b/src/pipx/commands/commands.py @@ -592,6 +592,9 @@ def _get_package_summary( python_path = venv.python_path.resolve() if package is None: package = path.name + if not python_path.is_file(): + return f" package {red(bold(package))} has invalid interpreter {str(python_path)}" + package_metadata = venv.package_metadata[package] if package_metadata.package_version is None: From f8eb8163e6eec80e53f4faed698338d4df31f182 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Tue, 5 Nov 2019 15:02:22 -0800 Subject: [PATCH 102/153] Remove --spec from pipx upgrade. --- src/pipx/commands/upgrade.py | 14 +++++--------- src/pipx/main.py | 5 ----- 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/src/pipx/commands/upgrade.py b/src/pipx/commands/upgrade.py index 40292ccec5..68745f1835 100644 --- a/src/pipx/commands/upgrade.py +++ b/src/pipx/commands/upgrade.py @@ -16,7 +16,6 @@ def upgrade( venv_dir: Path, package: str, - package_or_url: str, pip_args: List[str], verbose: bool, *, @@ -37,14 +36,11 @@ def upgrade( old_package_metadata = venv.package_metadata[package] old_version = old_package_metadata.package_version - # if default package_or_url, check pipx_metadata for better url - # TODO 20190926: main.py should communicate if this is spec or copied from - # package - if package_or_url == package: - if venv.pipx_metadata.main_package.package_or_url is not None: - package_or_url = venv.pipx_metadata.main_package.package_or_url - else: - package_or_url = package + if venv.pipx_metadata.main_package.package_or_url is not None: + package_or_url = venv.pipx_metadata.main_package.package_or_url + else: + # fallback if no metadata + package_or_url = package # Upgrade shared libraries (pip, setuptools and wheel) venv.upgrade_packaging_libraries(pip_args) diff --git a/src/pipx/main.py b/src/pipx/main.py index a8851c90a2..b284ed9b76 100644 --- a/src/pipx/main.py +++ b/src/pipx/main.py @@ -186,13 +186,9 @@ def run_pipx_command(args): # noqa: C901 force=args.force, ) elif args.command == "upgrade": - package_or_url = ( - args.spec if ("spec" in args and args.spec is not None) else package - ) return commands.upgrade( venv_dir, package, - package_or_url, pip_args, verbose, upgrading_all=False, @@ -330,7 +326,6 @@ def _add_upgrade(subparsers, autocomplete_list_of_installed_packages): description="Upgrade a package in a pipx-managed Virtual Environment by running 'pip install --upgrade PACKAGE'", ) p.add_argument("package").completer = autocomplete_list_of_installed_packages - p.add_argument("--spec", help=SPEC_HELP) p.add_argument( "--force", "-f", From 6379abdf9477f2364cca45b2d519c9bd983b63b8 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Tue, 5 Nov 2019 15:13:14 -0800 Subject: [PATCH 103/153] Remove --include-deps as option for upgrade. --- src/pipx/commands/upgrade.py | 4 +--- src/pipx/main.py | 9 +-------- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/src/pipx/commands/upgrade.py b/src/pipx/commands/upgrade.py index 68745f1835..c6e142851e 100644 --- a/src/pipx/commands/upgrade.py +++ b/src/pipx/commands/upgrade.py @@ -20,7 +20,6 @@ def upgrade( verbose: bool, *, upgrading_all: bool, - include_dependencies: bool, force: bool, ) -> int: """Returns nonzero if package was upgraded, 0 if version did not change""" @@ -49,7 +48,6 @@ def upgrade( package, package_or_url, pip_args, - # TODO 20191026: should this be `include_dependencies`? include_dependencies=old_package_metadata.include_dependencies, include_apps=old_package_metadata.include_apps, is_main_package=True, @@ -63,7 +61,7 @@ def upgrade( constants.LOCAL_BIN_DIR, package_metadata.app_paths, package, force=force ) - if include_dependencies: + if old_package_metadata.include_dependencies: for _, app_paths in package_metadata.app_paths_of_dependencies.items(): _expose_apps_globally( constants.LOCAL_BIN_DIR, app_paths, package, force=force diff --git a/src/pipx/main.py b/src/pipx/main.py index b284ed9b76..61c50edd39 100644 --- a/src/pipx/main.py +++ b/src/pipx/main.py @@ -187,13 +187,7 @@ def run_pipx_command(args): # noqa: C901 ) elif args.command == "upgrade": return commands.upgrade( - venv_dir, - package, - pip_args, - verbose, - upgrading_all=False, - include_dependencies=args.include_deps, - force=args.force, + venv_dir, package, pip_args, verbose, upgrading_all=False, force=args.force ) elif args.command == "list": return commands.list_packages(venv_container) @@ -332,7 +326,6 @@ def _add_upgrade(subparsers, autocomplete_list_of_installed_packages): action="store_true", help="Modify existing virtual environment and files in PIPX_BIN_DIR", ) - add_include_dependencies(p) add_pip_venv_args(p) p.add_argument("--verbose", action="store_true") From a91e3ab68ce262ad0241e6a523db6e44646ad967 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Tue, 5 Nov 2019 15:49:28 -0800 Subject: [PATCH 104/153] Fix upgrade_all instantiation of upgrade, clean up upgrade. --- src/pipx/commands/upgrade.py | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/src/pipx/commands/upgrade.py b/src/pipx/commands/upgrade.py index c6e142851e..a56eb818fa 100644 --- a/src/pipx/commands/upgrade.py +++ b/src/pipx/commands/upgrade.py @@ -32,14 +32,21 @@ def upgrade( venv = Venv(venv_dir, verbose=verbose) - old_package_metadata = venv.package_metadata[package] - old_version = old_package_metadata.package_version - - if venv.pipx_metadata.main_package.package_or_url is not None: - package_or_url = venv.pipx_metadata.main_package.package_or_url + package_metadata = venv.package_metadata[package] + if package_metadata.package_or_url is not None: + package_or_url = package_metadata.package_or_url + old_version = package_metadata.package_version + include_apps = package_metadata.include_apps + include_dependencies = package_metadata.include_dependencies else: # fallback if no metadata package_or_url = package + old_version = "" + include_apps = True + include_dependencies = False + + if package == "pipx": + package_or_url = PIPX_PACKAGE_NAME # Upgrade shared libraries (pip, setuptools and wheel) venv.upgrade_packaging_libraries(pip_args) @@ -48,8 +55,8 @@ def upgrade( package, package_or_url, pip_args, - include_dependencies=old_package_metadata.include_dependencies, - include_apps=old_package_metadata.include_apps, + include_dependencies=include_dependencies, + include_apps=include_apps, is_main_package=True, ) # TODO 20191026: Should we upgrade injected packages also? @@ -61,7 +68,7 @@ def upgrade( constants.LOCAL_BIN_DIR, package_metadata.app_paths, package, force=force ) - if old_package_metadata.include_dependencies: + if include_dependencies: for _, app_paths in package_metadata.app_paths_of_dependencies.items(): _expose_apps_globally( constants.LOCAL_BIN_DIR, app_paths, package, force=force @@ -93,22 +100,13 @@ def upgrade_all( venv = Venv(venv_dir, verbose=verbose) if package in skip: continue - if package == "pipx": - package_or_url = PIPX_PACKAGE_NAME - else: - if venv.pipx_metadata.main_package.package_or_url is not None: - package_or_url = venv.pipx_metadata.main_package.package_or_url - else: - package_or_url = package try: packages_upgraded += upgrade( venv_dir, package, - package_or_url, venv.pipx_metadata.main_package.pip_args, verbose, upgrading_all=True, - include_dependencies=venv.pipx_metadata.main_package.include_dependencies, force=force, ) # TODO 20191024: Upgrade injected packages From 42b6b35c20558821b796039193bdbf55a70426b5 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Tue, 5 Nov 2019 21:37:06 -0800 Subject: [PATCH 105/153] Remove TODO, add clarifying comment. --- src/pipx/commands/commands.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/pipx/commands/commands.py b/src/pipx/commands/commands.py index e48f3f0180..e5ec750756 100644 --- a/src/pipx/commands/commands.py +++ b/src/pipx/commands/commands.py @@ -468,14 +468,12 @@ def reinstall_all( venv = Venv(venv_dir, verbose=verbose) - # TODO 20191026: store this away in case uninstalling removes metadata - # orig_pipx_metadata = venv.pipx_metadata - uninstall(venv_dir, package, local_bin_dir, verbose) # TODO 20191026: also uninstall all injected packages # injected packages will be cluttering up metadata if their metadata # is not removed also + # venv instance still holds old loaded metadata even after uninstall if venv.pipx_metadata.main_package.package_or_url is not None: package_or_url = venv.pipx_metadata.main_package.package_or_url else: From a75f814e490726bfc5ec311a282fbce83f3c6fe9 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Tue, 5 Nov 2019 21:44:28 -0800 Subject: [PATCH 106/153] Remove obsolete TODOs. --- src/pipx/commands/commands.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/pipx/commands/commands.py b/src/pipx/commands/commands.py index e5ec750756..876e4397e1 100644 --- a/src/pipx/commands/commands.py +++ b/src/pipx/commands/commands.py @@ -395,7 +395,6 @@ def uninstall(venv_dir: Path, package: str, local_bin_dir: Path, verbose: bool): ) return - # TODO 20191024: Uninstall injected packages apps in bin venv = Venv(venv_dir, verbose=verbose) if venv.pipx_metadata.main_package is not None: @@ -469,9 +468,6 @@ def reinstall_all( venv = Venv(venv_dir, verbose=verbose) uninstall(venv_dir, package, local_bin_dir, verbose) - # TODO 20191026: also uninstall all injected packages - # injected packages will be cluttering up metadata if their metadata - # is not removed also # venv instance still holds old loaded metadata even after uninstall if venv.pipx_metadata.main_package.package_or_url is not None: From 6601b281cefb66a78484d00383e0fef8f67d2112 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Wed, 6 Nov 2019 11:25:36 -0800 Subject: [PATCH 107/153] Simplify code using property from venv. --- src/pipx/commands/commands.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/pipx/commands/commands.py b/src/pipx/commands/commands.py index 876e4397e1..710171fa7f 100644 --- a/src/pipx/commands/commands.py +++ b/src/pipx/commands/commands.py @@ -398,11 +398,8 @@ def uninstall(venv_dir: Path, package: str, local_bin_dir: Path, verbose: bool): venv = Venv(venv_dir, verbose=verbose) if venv.pipx_metadata.main_package is not None: - all_packages = [venv.pipx_metadata.main_package] + list( - venv.pipx_metadata.injected_packages.values() - ) app_paths: List[Path] = [] - for viewed_package in all_packages: + for viewed_package in venv.package_metadata.values(): app_paths += viewed_package.app_paths for dep_paths in viewed_package.app_paths_of_dependencies.values(): app_paths += dep_paths From a80c74702ccfcdfdc4817e867116735e3116d386 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Tue, 12 Nov 2019 14:41:52 -0800 Subject: [PATCH 108/153] Pass None to Venv.install_package in package if it is unknown. --- src/pipx/commands/commands.py | 11 ++++++----- src/pipx/venv.py | 8 ++++---- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/pipx/commands/commands.py b/src/pipx/commands/commands.py index 710171fa7f..28206a8fd5 100644 --- a/src/pipx/commands/commands.py +++ b/src/pipx/commands/commands.py @@ -124,13 +124,14 @@ def _download_and_run( ): venv = Venv(venv_dir, python=python, verbose=verbose) venv.create_venv(venv_args, pip_args) - # if venv is pre-existing and already installed with package, we - # can get package name from it, otherwise use empty string - package = venv.pipx_metadata.main_package.package or "" + + # venv.pipx_metadata.main_package.package contains package name if it is + # pre-existing, otherwise is None to instruct venv.install_package to + # determine package name. try: venv.install_package( - package=package, + package=venv.pipx_metadata.main_package.package, package_or_url=package_or_url, pip_args=pip_args, include_dependencies=False, @@ -143,7 +144,7 @@ def _download_and_run( if not (venv.bin_path / app).exists(): apps = venv.pipx_metadata.main_package.apps raise PipxError( - f"'{app}' executable script not found in package '{package}'. " + f"'{app}' executable script not found in package '{package_or_url}'. " "Available executable scripts: " f"{', '.join(b for b in apps)}" ) diff --git a/src/pipx/venv.py b/src/pipx/venv.py index fee939e0f2..9747a7562e 100644 --- a/src/pipx/venv.py +++ b/src/pipx/venv.py @@ -4,7 +4,7 @@ import re import subprocess from pathlib import Path -from typing import Generator, List, NamedTuple, Dict, Set +from typing import Generator, List, NamedTuple, Dict, Set, Optional from pipx.animate import animate from pipx.constants import DEFAULT_PYTHON, PIPX_SHARED_PTH, WINDOWS @@ -166,14 +166,14 @@ def upgrade_packaging_libraries(self, pip_args: List[str]) -> None: def install_package( self, - package: str, # if "", will be determined + package: Optional[str], # if None, will be determined in this function package_or_url: str, pip_args: List[str], include_dependencies: bool, include_apps: bool, is_main_package: bool, ) -> None: - if not package: + if package is None: # If no package name is supplied, find out package name installed # by comparing old_package_set with new one after install old_package_set = self.list_installed_packages() @@ -184,7 +184,7 @@ def install_package( cmd = ["install"] + pip_args + [package_or_url] self._run_pip(cmd) - if not package: + if package is None: installed_packages = self.list_installed_packages() - old_package_set package = self.top_of_deptree(installed_packages) From 9d5ea3f27e760b009f71c175f14f82f64d2321a9 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Tue, 12 Nov 2019 14:45:34 -0800 Subject: [PATCH 109/153] Move uninstall closer to install. --- src/pipx/commands/commands.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/pipx/commands/commands.py b/src/pipx/commands/commands.py index 28206a8fd5..5685b950fa 100644 --- a/src/pipx/commands/commands.py +++ b/src/pipx/commands/commands.py @@ -465,14 +465,13 @@ def reinstall_all( venv = Venv(venv_dir, verbose=verbose) - uninstall(venv_dir, package, local_bin_dir, verbose) - - # venv instance still holds old loaded metadata even after uninstall if venv.pipx_metadata.main_package.package_or_url is not None: package_or_url = venv.pipx_metadata.main_package.package_or_url else: package_or_url = package + uninstall(venv_dir, package, local_bin_dir, verbose) + # install main package first install( venv_dir, From 7e2138c31de03fddfbd6ef39ea3e9a0080f89cb9 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Tue, 12 Nov 2019 14:48:40 -0800 Subject: [PATCH 110/153] Remove constant PIPX_PACKAGE_NAME, replace in code with "pipx". --- src/pipx/commands/upgrade.py | 3 +-- src/pipx/constants.py | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/pipx/commands/upgrade.py b/src/pipx/commands/upgrade.py index a56eb818fa..b0048249cd 100644 --- a/src/pipx/commands/upgrade.py +++ b/src/pipx/commands/upgrade.py @@ -5,7 +5,6 @@ from pipx import constants -from pipx.constants import PIPX_PACKAGE_NAME from pipx.emojies import sleep from pipx.util import PipxError @@ -46,7 +45,7 @@ def upgrade( include_dependencies = False if package == "pipx": - package_or_url = PIPX_PACKAGE_NAME + package_or_url = "pipx" # Upgrade shared libraries (pip, setuptools and wheel) venv.upgrade_packaging_libraries(pip_args) diff --git a/src/pipx/constants.py b/src/pipx/constants.py index f2cd5acec7..861090da85 100644 --- a/src/pipx/constants.py +++ b/src/pipx/constants.py @@ -15,7 +15,6 @@ PIPX_SHARED_PTH = "pipx_shared.pth" LOCAL_BIN_DIR = Path(os.environ.get("PIPX_BIN_DIR", DEFAULT_PIPX_BIN_DIR)).resolve() PIPX_VENV_CACHEDIR = PIPX_HOME / ".cache" -PIPX_PACKAGE_NAME = "pipx" TEMP_VENV_EXPIRATION_THRESHOLD_DAYS = 14 try: WindowsError From 8d741baf6185b3ad94c79a78cb570be9ded457af Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Tue, 12 Nov 2019 15:02:10 -0800 Subject: [PATCH 111/153] Streamline pip_search cmd list creation. --- src/pipx/venv.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/pipx/venv.py b/src/pipx/venv.py index 9747a7562e..d53a09e739 100644 --- a/src/pipx/venv.py +++ b/src/pipx/venv.py @@ -263,9 +263,12 @@ def get_python_version(self) -> str: .strip() ) - def pip_search(self, search_term, pip_search_args): - cmd = [str(self.python_path), "-m", "pip", "search"] - cmd += pip_search_args + [search_term] + def pip_search(self, search_term: str, pip_search_args: List[str]) -> str: + cmd = ( + [str(self.python_path), "-m", "pip", "search"] + + pip_search_args + + [search_term] + ) cmd_run = subprocess.run(cmd, stdout=subprocess.PIPE) return cmd_run.stdout.decode().strip() From f744c4330d381abaeb98c139b7f08362ef255dd0 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Tue, 12 Nov 2019 15:19:55 -0800 Subject: [PATCH 112/153] Add typing information to return values. --- src/pipx/venv.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/pipx/venv.py b/src/pipx/venv.py index d53a09e739..71dc2055b6 100644 --- a/src/pipx/venv.py +++ b/src/pipx/venv.py @@ -4,7 +4,7 @@ import re import subprocess from pathlib import Path -from typing import Generator, List, NamedTuple, Dict, Set, Optional +from typing import Generator, List, NamedTuple, Dict, Set, Optional, Iterable from pipx.animate import animate from pipx.constants import DEFAULT_PYTHON, PIPX_SHARED_PTH, WINDOWS @@ -106,7 +106,7 @@ def __init__( ) @property - def uses_shared_libs(self): + def uses_shared_libs(self) -> bool: if self._existing: pth_files = self.root.glob("**/" + PIPX_SHARED_PTH) return next(pth_files, None) is not None @@ -235,7 +235,7 @@ def _update_package_metadata( include_dependencies: bool, include_apps: bool, is_main_package: bool, - ): + ) -> None: venv_package_metadata = self.get_venv_metadata_for_package(package) package_info = PackageInfo( package=package, @@ -278,7 +278,7 @@ def list_installed_packages(self) -> Set[str]: pip_list = json.loads(cmd_run.stdout.decode().strip()) return set([x["name"] for x in pip_list]) - def top_of_deptree(self, packages): + def top_of_deptree(self, packages: Iterable[str]) -> str: top_package_name = "" cmd = [str(self.python_path), "-m", "pip", "show"] + list(packages) @@ -298,14 +298,16 @@ def top_of_deptree(self, packages): return top_package_name - def run_app(self, app: str, app_args: List[str]): + def run_app(self, app: str, app_args: List[str]) -> int: cmd = [str(self.bin_path / app)] + app_args try: return run(cmd, check=False) except KeyboardInterrupt: - pass + return 130 # shell code for Ctrl-C - def _upgrade_package_no_metadata(self, package_or_url: str, pip_args: List[str]): + def _upgrade_package_no_metadata( + self, package_or_url: str, pip_args: List[str] + ) -> None: with animate(f"upgrading package {package_or_url!r}", self.do_animation): self._run_pip(["install"] + pip_args + ["--upgrade", package_or_url]) @@ -317,7 +319,7 @@ def upgrade_package( include_dependencies: bool, include_apps: bool, is_main_package: bool, - ): + ) -> None: with animate(f"upgrading package {package_or_url!r}", self.do_animation): self._run_pip(["install"] + pip_args + ["--upgrade", package_or_url]) @@ -330,8 +332,8 @@ def upgrade_package( is_main_package=is_main_package, ) - def _run_pip(self, cmd): - cmd = [self.python_path, "-m", "pip"] + cmd + def _run_pip(self, cmd: List[str]) -> int: + cmd = [str(self.python_path), "-m", "pip"] + cmd if not self.verbose: cmd.append("-q") return run(cmd) From 096dddbcc9f86f5f7152621f4209387e47562d77 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Tue, 12 Nov 2019 15:22:38 -0800 Subject: [PATCH 113/153] Remove unnecessary comment. --- src/pipx/venv.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/pipx/venv.py b/src/pipx/venv.py index 71dc2055b6..87ebf9a80e 100644 --- a/src/pipx/venv.py +++ b/src/pipx/venv.py @@ -140,7 +140,6 @@ def create_venv(self, venv_args: List[str], pip_args: List[str]) -> None: # its contents are additional items (one per line) to be added to sys.path pipx_pth.write_text(str(shared_libs.site_packages) + "\n", encoding="utf-8") - # write venv-specific metadata self.pipx_metadata.venv_args = venv_args self.pipx_metadata.python_version = self.get_python_version() From be80ce04f08f632e950fcf78e3a7cd97f441c5d6 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Tue, 12 Nov 2019 16:44:48 -0800 Subject: [PATCH 114/153] Remove top_package_name, simplify code. --- src/pipx/venv.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/pipx/venv.py b/src/pipx/venv.py index 87ebf9a80e..f964139e0f 100644 --- a/src/pipx/venv.py +++ b/src/pipx/venv.py @@ -278,8 +278,6 @@ def list_installed_packages(self) -> Set[str]: return set([x["name"] for x in pip_list]) def top_of_deptree(self, packages: Iterable[str]) -> str: - top_package_name = "" - cmd = [str(self.python_path), "-m", "pip", "show"] + list(packages) cmd_run = subprocess.run(cmd, stdout=subprocess.PIPE) pip_show_stdout = cmd_run.stdout.decode().strip() @@ -292,10 +290,9 @@ def top_of_deptree(self, packages: Iterable[str]) -> str: if key == "Name": package_name = value if key == "Required-by" and value == "": - top_package_name = package_name - break + return package_name - return top_package_name + return "" def run_app(self, app: str, app_args: List[str]) -> int: cmd = [str(self.bin_path / app)] + app_args From da3e58b40200913f326688dbf058cc55816f9c75 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Tue, 12 Nov 2019 17:00:09 -0800 Subject: [PATCH 115/153] Use is False/True to get rid of noqa. --- tests/test_pipx_metadata_file.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tests/test_pipx_metadata_file.py b/tests/test_pipx_metadata_file.py index 4dfa5e60ad..566f5c2a35 100644 --- a/tests/test_pipx_metadata_file.py +++ b/tests/test_pipx_metadata_file.py @@ -83,8 +83,8 @@ def test_package_install(monkeypatch, tmp_path, pipx_temp_env): assert pipx_metadata.main_package.package == "pycowsay" assert pipx_metadata.main_package.package_or_url == "pycowsay" assert pipx_metadata.main_package.pip_args == [] - assert pipx_metadata.main_package.include_dependencies == False # noqa: E712 - assert pipx_metadata.main_package.include_apps == True # noqa: E712 + assert pipx_metadata.main_package.include_dependencies is False + assert pipx_metadata.main_package.include_apps is True if pipx.constants.WINDOWS: assert pipx_metadata.main_package.apps == ["pycowsay", "pycowsay.exe"] assert pipx_metadata.main_package.app_paths == [ @@ -108,10 +108,8 @@ def test_package_install(monkeypatch, tmp_path, pipx_temp_env): assert pipx_metadata.injected_packages["black"].package == "black" assert pipx_metadata.injected_packages["black"].package_or_url == "black" assert pipx_metadata.injected_packages["black"].pip_args == [] - assert ( - pipx_metadata.injected_packages["black"].include_dependencies == False - ) # noqa: E712 - assert pipx_metadata.injected_packages["black"].include_apps == False # noqa: E712 + assert pipx_metadata.injected_packages["black"].include_dependencies is False + assert pipx_metadata.injected_packages["black"].include_apps is False if pipx.constants.WINDOWS: # order is not important, so we compare sets assert isinstance(pipx_metadata.injected_packages["black"].apps, list) From e9a992de2a49c7f9979ee2093d062ab795a26173 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Tue, 12 Nov 2019 19:08:26 -0800 Subject: [PATCH 116/153] Simplify test_pipx_metadata_file_create. --- tests/test_pipx_metadata_file.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tests/test_pipx_metadata_file.py b/tests/test_pipx_metadata_file.py index 566f5c2a35..7d64863359 100644 --- a/tests/test_pipx_metadata_file.py +++ b/tests/test_pipx_metadata_file.py @@ -41,15 +41,10 @@ def test_pipx_metadata_file_create(tmp_path): pipx_metadata.python_version = "3.4.5" pipx_metadata.venv_args = ["--system-site-packages"] pipx_metadata.injected_packages = {"injected": TEST_PACKAGE2} - pipx_metadata.write() - del pipx_metadata pipx_metadata2 = PipxMetadata(tmp_path) - assert pipx_metadata2.main_package == TEST_PACKAGE1 - assert pipx_metadata2.python_version == "3.4.5" - assert pipx_metadata2.venv_args == ["--system-site-packages"] - assert pipx_metadata2.injected_packages == {"injected": TEST_PACKAGE2} + assert pipx_metadata2 = pipx_metadata def test_pipx_metadata_file_validation(tmp_path): From 279220571557f016be41ff0ea97ed0b85f4d4715 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Tue, 12 Nov 2019 20:26:58 -0800 Subject: [PATCH 117/153] Remove self.reset if read fails. --- src/pipx/pipx_metadata_file.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/pipx/pipx_metadata_file.py b/src/pipx/pipx_metadata_file.py index fa870258a2..64526c3c47 100644 --- a/src/pipx/pipx_metadata_file.py +++ b/src/pipx/pipx_metadata_file.py @@ -154,5 +154,4 @@ def read(self, verbose: bool = False) -> None: width=79, ) ) - self.reset() return From fd2aaec766a0c8213f187088e6ee2ca84c7a7992 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Tue, 12 Nov 2019 20:59:48 -0800 Subject: [PATCH 118/153] Parametrize test_pipx_metadata_file_validation. Also fix bug in test_pipx_metadata_file_create. --- tests/test_pipx_metadata_file.py | 47 +++++++++++++++++++------------- 1 file changed, 28 insertions(+), 19 deletions(-) diff --git a/tests/test_pipx_metadata_file.py b/tests/test_pipx_metadata_file.py index 7d64863359..86a44db39f 100644 --- a/tests/test_pipx_metadata_file.py +++ b/tests/test_pipx_metadata_file.py @@ -44,28 +44,37 @@ def test_pipx_metadata_file_create(tmp_path): pipx_metadata.write() pipx_metadata2 = PipxMetadata(tmp_path) - assert pipx_metadata2 = pipx_metadata + for attribute in [ + "venv_dir", + "main_package", + "python_version", + "venv_args", + "injected_packages", + ]: + assert getattr(pipx_metadata, attribute) == getattr(pipx_metadata2, attribute) + + +@pytest.mark.parametrize( + "test_package", + [ + TEST_PACKAGE1._replace(include_apps=False), + TEST_PACKAGE1._replace(package=None), + TEST_PACKAGE1._replace(package_or_url=None), + ], +) +def test_pipx_metadata_file_validation(tmp_path, test_package): + venv_dir = tmp_path / "venv" + venv_dir.mkdir() -def test_pipx_metadata_file_validation(tmp_path): - venv_dir1 = tmp_path / "venv1" - venv_dir1.mkdir() - venv_dir2 = tmp_path / "venv2" - venv_dir2.mkdir() - - test_package1 = TEST_PACKAGE1._replace(include_apps=False) - test_package2 = TEST_PACKAGE1._replace(package=None) - test_package3 = TEST_PACKAGE1._replace(package_or_url=None) - - for test_package in [test_package1, test_package2, test_package3]: - pipx_metadata = PipxMetadata(venv_dir1) - pipx_metadata.main_package = test_package - pipx_metadata.python_version = "3.4.5" - pipx_metadata.venv_args = ["--system-site-packages"] - pipx_metadata.injected_packages = {} + pipx_metadata = PipxMetadata(venv_dir) + pipx_metadata.main_package = test_package + pipx_metadata.python_version = "3.4.5" + pipx_metadata.venv_args = ["--system-site-packages"] + pipx_metadata.injected_packages = {} - with pytest.raises(PipxError): - pipx_metadata.write() + with pytest.raises(PipxError): + pipx_metadata.write() def test_package_install(monkeypatch, tmp_path, pipx_temp_env): From faa6a8be1f718e300ba5aa1b9a890f94583c8d9d Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Tue, 12 Nov 2019 21:29:10 -0800 Subject: [PATCH 119/153] Print stderr for test_run_ensure_null_pythonpath to diagnose. --- tests/test_run.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/test_run.py b/tests/test_run.py index 995bf09897..04020c6f7b 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -49,6 +49,23 @@ def test_run_script_from_internet(pipx_temp_env, capsys): def test_run_ensure_null_pythonpath(): env = os.environ.copy() env["PYTHONPATH"] = "test" + print( + subprocess.run( + [ + sys.executable, + "-m", + "pipx", + "run", + "ipython", + "-c", + "import os; print(os.environ.get('PYTHONPATH'))", + ], + universal_newlines=True, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ).stderr + ) assert ( "None" in subprocess.run( From 01199f3e5193c3b412b37d0855e67216a40e12d8 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Tue, 12 Nov 2019 21:41:27 -0800 Subject: [PATCH 120/153] Add TODO about Issue #217 in test_package_install. --- tests/test_pipx_metadata_file.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_pipx_metadata_file.py b/tests/test_pipx_metadata_file.py index 86a44db39f..2cc57a8e13 100644 --- a/tests/test_pipx_metadata_file.py +++ b/tests/test_pipx_metadata_file.py @@ -117,6 +117,7 @@ def test_package_install(monkeypatch, tmp_path, pipx_temp_env): if pipx.constants.WINDOWS: # order is not important, so we compare sets assert isinstance(pipx_metadata.injected_packages["black"].apps, list) + # TODO: Issue #217 - Windows should not have non-exe black, blackd assert set(pipx_metadata.injected_packages["black"].apps) == { "black", "black.exe", From 6b9e13849339d848eb2eeacdafde5758d48ba6e8 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Tue, 12 Nov 2019 21:56:39 -0800 Subject: [PATCH 121/153] Revert "Print stderr for test_run_ensure_null_pythonpath to diagnose." This reverts commit faa6a8be1f718e300ba5aa1b9a890f94583c8d9d. --- tests/test_run.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/tests/test_run.py b/tests/test_run.py index 04020c6f7b..995bf09897 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -49,23 +49,6 @@ def test_run_script_from_internet(pipx_temp_env, capsys): def test_run_ensure_null_pythonpath(): env = os.environ.copy() env["PYTHONPATH"] = "test" - print( - subprocess.run( - [ - sys.executable, - "-m", - "pipx", - "run", - "ipython", - "-c", - "import os; print(os.environ.get('PYTHONPATH'))", - ], - universal_newlines=True, - env=env, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ).stderr - ) assert ( "None" in subprocess.run( From bbb6f66eac33931b2502931b211a03973de2aff5 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Tue, 12 Nov 2019 22:09:57 -0800 Subject: [PATCH 122/153] Another try to diagnose Windows test_run problem. --- tests/test_run.py | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/tests/test_run.py b/tests/test_run.py index 995bf09897..95c0d67710 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -49,21 +49,23 @@ def test_run_script_from_internet(pipx_temp_env, capsys): def test_run_ensure_null_pythonpath(): env = os.environ.copy() env["PYTHONPATH"] = "test" - assert ( - "None" - in subprocess.run( - [ - sys.executable, - "-m", - "pipx", - "run", - "ipython", - "-c", - "import os; print(os.environ.get('PYTHONPATH'))", - ], - universal_newlines=True, - env=env, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ).stdout + test_command = subprocess.run( + [ + sys.executable, + "-m", + "pipx", + "run", + "ipython", + "-c", + "import os; print(os.environ.get('PYTHONPATH'))", + ], + universal_newlines=True, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, ) + print("test_command.stdout") + print(test_command.stdout) + print("test_command.stderr") + print(test_command.stderr) + assert "None" in test_command.stdout From f4fb258b1e2a1b73dd1aec0644a681c419edee04 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Tue, 12 Nov 2019 22:19:18 -0800 Subject: [PATCH 123/153] Revert "Another try to diagnose Windows test_run problem." This reverts commit bbb6f66eac33931b2502931b211a03973de2aff5. --- tests/test_run.py | 36 +++++++++++++++++------------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/tests/test_run.py b/tests/test_run.py index 95c0d67710..995bf09897 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -49,23 +49,21 @@ def test_run_script_from_internet(pipx_temp_env, capsys): def test_run_ensure_null_pythonpath(): env = os.environ.copy() env["PYTHONPATH"] = "test" - test_command = subprocess.run( - [ - sys.executable, - "-m", - "pipx", - "run", - "ipython", - "-c", - "import os; print(os.environ.get('PYTHONPATH'))", - ], - universal_newlines=True, - env=env, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + assert ( + "None" + in subprocess.run( + [ + sys.executable, + "-m", + "pipx", + "run", + "ipython", + "-c", + "import os; print(os.environ.get('PYTHONPATH'))", + ], + universal_newlines=True, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ).stdout ) - print("test_command.stdout") - print(test_command.stdout) - print("test_command.stderr") - print(test_command.stderr) - assert "None" in test_command.stdout From a9f4d1490923ef17eda8964a90e85fede19cfa81 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Tue, 12 Nov 2019 22:28:00 -0800 Subject: [PATCH 124/153] REVERT ME: to test Windows CI failures. --- src/pipx/venv.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/pipx/venv.py b/src/pipx/venv.py index f964139e0f..386c054dd6 100644 --- a/src/pipx/venv.py +++ b/src/pipx/venv.py @@ -197,6 +197,10 @@ def install_package( ) # Verify package installed ok + print("package") + print(package) + print("self.package_metadata[package].package_version") + print(self.package_metadata[package].package_version) if self.package_metadata[package].package_version is None: raise PackageInstallFailureError From a7089bea819a3b641846fdb32c25e5054e5bdd88 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Tue, 12 Nov 2019 22:33:44 -0800 Subject: [PATCH 125/153] REVERT ME: to test Windows CI failures2. --- src/pipx/venv.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/pipx/venv.py b/src/pipx/venv.py index 386c054dd6..bed358f515 100644 --- a/src/pipx/venv.py +++ b/src/pipx/venv.py @@ -199,8 +199,6 @@ def install_package( # Verify package installed ok print("package") print(package) - print("self.package_metadata[package].package_version") - print(self.package_metadata[package].package_version) if self.package_metadata[package].package_version is None: raise PackageInstallFailureError From bc78e21d4890f37a91aa8af0cf50ee326dd5d738 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Tue, 12 Nov 2019 22:35:23 -0800 Subject: [PATCH 126/153] REVERT ME: to test Windows CI failures3. --- src/pipx/venv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pipx/venv.py b/src/pipx/venv.py index bed358f515..5d745352ce 100644 --- a/src/pipx/venv.py +++ b/src/pipx/venv.py @@ -197,7 +197,7 @@ def install_package( ) # Verify package installed ok - print("package") + print("package") # print(package) if self.package_metadata[package].package_version is None: raise PackageInstallFailureError From 02678c6768d18d8185e2e979fabccd7c456acca6 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Tue, 12 Nov 2019 22:41:42 -0800 Subject: [PATCH 127/153] REVERT ME: to test Windows CI failures4. --- src/pipx/venv.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/pipx/venv.py b/src/pipx/venv.py index 5d745352ce..34b0707b3b 100644 --- a/src/pipx/venv.py +++ b/src/pipx/venv.py @@ -185,6 +185,8 @@ def install_package( if package is None: installed_packages = self.list_installed_packages() - old_package_set + print("installed_packages") + print(installed_packages) package = self.top_of_deptree(installed_packages) self._update_package_metadata( @@ -197,7 +199,7 @@ def install_package( ) # Verify package installed ok - print("package") # + print("package") print(package) if self.package_metadata[package].package_version is None: raise PackageInstallFailureError From b4f6a03da3b59f88873f2b2a467ec2ec60a6fc77 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Wed, 13 Nov 2019 15:37:27 -0800 Subject: [PATCH 128/153] Revert "REVERT ME: to test Windows CI failures4." This reverts commit 02678c6768d18d8185e2e979fabccd7c456acca6. --- src/pipx/venv.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/pipx/venv.py b/src/pipx/venv.py index 34b0707b3b..5d745352ce 100644 --- a/src/pipx/venv.py +++ b/src/pipx/venv.py @@ -185,8 +185,6 @@ def install_package( if package is None: installed_packages = self.list_installed_packages() - old_package_set - print("installed_packages") - print(installed_packages) package = self.top_of_deptree(installed_packages) self._update_package_metadata( @@ -199,7 +197,7 @@ def install_package( ) # Verify package installed ok - print("package") + print("package") # print(package) if self.package_metadata[package].package_version is None: raise PackageInstallFailureError From e75e09981d99cac55f85fa01205cb90d8e104c7b Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Wed, 13 Nov 2019 15:37:44 -0800 Subject: [PATCH 129/153] Revert "REVERT ME: to test Windows CI failures3." This reverts commit bc78e21d4890f37a91aa8af0cf50ee326dd5d738. --- src/pipx/venv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pipx/venv.py b/src/pipx/venv.py index 5d745352ce..bed358f515 100644 --- a/src/pipx/venv.py +++ b/src/pipx/venv.py @@ -197,7 +197,7 @@ def install_package( ) # Verify package installed ok - print("package") # + print("package") print(package) if self.package_metadata[package].package_version is None: raise PackageInstallFailureError From d53096187c34696a14590331c90ea61c295a9690 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Wed, 13 Nov 2019 15:38:07 -0800 Subject: [PATCH 130/153] Revert "REVERT ME: to test Windows CI failures2." This reverts commit a7089bea819a3b641846fdb32c25e5054e5bdd88. --- src/pipx/venv.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/pipx/venv.py b/src/pipx/venv.py index bed358f515..386c054dd6 100644 --- a/src/pipx/venv.py +++ b/src/pipx/venv.py @@ -199,6 +199,8 @@ def install_package( # Verify package installed ok print("package") print(package) + print("self.package_metadata[package].package_version") + print(self.package_metadata[package].package_version) if self.package_metadata[package].package_version is None: raise PackageInstallFailureError From ab7281cc150c63a689020d6272d29f525b2c5cfd Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Wed, 13 Nov 2019 15:38:38 -0800 Subject: [PATCH 131/153] Revert "REVERT ME: to test Windows CI failures." This reverts commit a9f4d1490923ef17eda8964a90e85fede19cfa81. --- src/pipx/venv.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/pipx/venv.py b/src/pipx/venv.py index 386c054dd6..f964139e0f 100644 --- a/src/pipx/venv.py +++ b/src/pipx/venv.py @@ -197,10 +197,6 @@ def install_package( ) # Verify package installed ok - print("package") - print(package) - print("self.package_metadata[package].package_version") - print(self.package_metadata[package].package_version) if self.package_metadata[package].package_version is None: raise PackageInstallFailureError From 80b6792ac93a58b9aa278056ad443074c886b4f0 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Wed, 13 Nov 2019 15:40:20 -0800 Subject: [PATCH 132/153] Add universal_newlines=True to subprocess.run. --- src/pipx/venv.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pipx/venv.py b/src/pipx/venv.py index f964139e0f..2f8df65011 100644 --- a/src/pipx/venv.py +++ b/src/pipx/venv.py @@ -279,8 +279,8 @@ def list_installed_packages(self) -> Set[str]: def top_of_deptree(self, packages: Iterable[str]) -> str: cmd = [str(self.python_path), "-m", "pip", "show"] + list(packages) - cmd_run = subprocess.run(cmd, stdout=subprocess.PIPE) - pip_show_stdout = cmd_run.stdout.decode().strip() + cmd_run = subprocess.run(cmd, stdout=subprocess.PIPE, universal_newlines=True) + pip_show_stdout = cmd_run.stdout for line in pip_show_stdout.split("\n"): key_value_re = re.search(r"^([^:]+):\s(.*)", line) From e367e1b6381faf58d0f6eda84445195dc9f9ceee Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Wed, 13 Nov 2019 20:28:54 -0800 Subject: [PATCH 133/153] Streamline test_package_install. --- tests/test_pipx_metadata_file.py | 123 +++++++++++++++++++------------ 1 file changed, 76 insertions(+), 47 deletions(-) diff --git a/tests/test_pipx_metadata_file.py b/tests/test_pipx_metadata_file.py index 2cc57a8e13..1abb76912d 100644 --- a/tests/test_pipx_metadata_file.py +++ b/tests/test_pipx_metadata_file.py @@ -34,6 +34,32 @@ package_version="6.7.8", ) +# Reference metadata for various packages +PYCOWSAY_PACKAGE_REF = PackageInfo( + package="pycowsay", + package_or_url="pycowsay", + pip_args=[], + include_dependencies=False, + include_apps=True, + apps=["pycowsay"], + app_paths=["pycowsay/bin/pycowsay"], # Placeholder, not real path + apps_of_dependencies=[], + app_paths_of_dependencies={}, + package_version="0.0.0.1", +) +BLACK_PACKAGE_REF = PackageInfo( + package="black", + package_or_url="black", + pip_args=[], + include_dependencies=False, + include_apps=True, + apps=["pycowsay"], + app_paths=["black/bin/black"], # Placeholder, not real path + apps_of_dependencies=[], + app_paths_of_dependencies={}, + package_version="19.10b3", +) + def test_pipx_metadata_file_create(tmp_path): pipx_metadata = PipxMetadata(tmp_path) @@ -77,6 +103,13 @@ def test_pipx_metadata_file_validation(tmp_path, test_package): pipx_metadata.write() +def assert_package_metadata(test_metadata, ref_metadata): + assert test_metadata.package_version != "" + assert test_metadata == ref_metadata._replace( + package_version=test_metadata.package_version + ) + + def test_package_install(monkeypatch, tmp_path, pipx_temp_env): pipx_venvs_dir = pipx.constants.PIPX_HOME / "venvs" @@ -84,60 +117,56 @@ def test_package_install(monkeypatch, tmp_path, pipx_temp_env): assert (pipx_venvs_dir / "pycowsay" / "pipx_metadata.json").is_file() pipx_metadata = PipxMetadata(pipx_venvs_dir / "pycowsay") - assert pipx_metadata.main_package.package == "pycowsay" - assert pipx_metadata.main_package.package_or_url == "pycowsay" - assert pipx_metadata.main_package.pip_args == [] - assert pipx_metadata.main_package.include_dependencies is False - assert pipx_metadata.main_package.include_apps is True + if pipx.constants.WINDOWS: - assert pipx_metadata.main_package.apps == ["pycowsay", "pycowsay.exe"] - assert pipx_metadata.main_package.app_paths == [ - pipx_venvs_dir / "pycowsay" / "Scripts" / "pycowsay.exe" - ] + assert_package_metadata( + pipx_metadata.main_package, + PYCOWSAY_PACKAGE_REF._replace( + app_paths=[pipx_venvs_dir / "pycowsay" / "Scripts" / "pycowsay.exe"], + apps=["pycowsay", "pycowsay.exe"], + include_apps=True, + ), + ) else: - assert pipx_metadata.main_package.apps == ["pycowsay"] - assert pipx_metadata.main_package.app_paths == [ - pipx_venvs_dir / "pycowsay" / "bin" / "pycowsay" - ] - assert pipx_metadata.main_package.apps_of_dependencies == [] - assert pipx_metadata.main_package.app_paths_of_dependencies == {} - assert pipx_metadata.main_package.package_version != "" + assert_package_metadata( + pipx_metadata.main_package, + PYCOWSAY_PACKAGE_REF._replace( + app_paths=[pipx_venvs_dir / "pycowsay" / "bin" / "pycowsay"], + include_apps=True, + ), + ) del pipx_metadata - # TODO 20191103: need simpler non-gcc-compiling package besides black! + # test package injection + run_pipx_cli(["inject", "pycowsay", "black"]) pipx_metadata = PipxMetadata(pipx_venvs_dir / "pycowsay") - assert pipx_metadata.injected_packages["black"].package == "black" - assert pipx_metadata.injected_packages["black"].package_or_url == "black" - assert pipx_metadata.injected_packages["black"].pip_args == [] - assert pipx_metadata.injected_packages["black"].include_dependencies is False - assert pipx_metadata.injected_packages["black"].include_apps is False + if pipx.constants.WINDOWS: - # order is not important, so we compare sets - assert isinstance(pipx_metadata.injected_packages["black"].apps, list) - # TODO: Issue #217 - Windows should not have non-exe black, blackd - assert set(pipx_metadata.injected_packages["black"].apps) == { - "black", - "black.exe", - "blackd", - "blackd.exe", - } - assert isinstance(pipx_metadata.injected_packages["black"].app_paths, list) - assert set(pipx_metadata.injected_packages["black"].app_paths) == { - pipx_venvs_dir / "pycowsay" / "Scripts" / "black.exe", - pipx_venvs_dir / "pycowsay" / "Scripts" / "blackd.exe", - } + assert_package_metadata( + pipx_metadata.injected_packages["black"], + BLACK_PACKAGE_REF._replace( + apps=["black", "black.exe", "blackd", "blackd.exe"], + app_paths=[ + pipx_venvs_dir / "pycowsay" / "bin" / "black", + pipx_venvs_dir / "pycowsay" / "bin" / "black.exe", + pipx_venvs_dir / "pycowsay" / "bin" / "blackd", + pipx_venvs_dir / "pycowsay" / "bin" / "blackd.exe", + ], + include_apps=False + ), + ) else: - # order is not important, so we compare sets - assert isinstance(pipx_metadata.injected_packages["black"].apps, list) - assert set(pipx_metadata.injected_packages["black"].apps) == {"black", "blackd"} - assert isinstance(pipx_metadata.injected_packages["black"].app_paths, list) - assert set(pipx_metadata.injected_packages["black"].app_paths) == { - pipx_venvs_dir / "pycowsay" / "bin" / "black", - pipx_venvs_dir / "pycowsay" / "bin" / "blackd", - } - assert pipx_metadata.injected_packages["black"].apps_of_dependencies == [] - assert pipx_metadata.injected_packages["black"].app_paths_of_dependencies == {} - assert pipx_metadata.injected_packages["black"].package_version != "" + assert_package_metadata( + pipx_metadata.injected_packages["black"], + BLACK_PACKAGE_REF._replace( + apps=["black", "blackd"], + app_paths=[ + pipx_venvs_dir / "pycowsay" / "bin" / "black", + pipx_venvs_dir / "pycowsay" / "bin" / "blackd", + ], + include_apps=False, + ), + ) From 685e9a27206bf151d57754f2fd6395f31349004f Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Wed, 13 Nov 2019 20:31:54 -0800 Subject: [PATCH 134/153] Black reformat. --- tests/test_pipx_metadata_file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_pipx_metadata_file.py b/tests/test_pipx_metadata_file.py index 1abb76912d..a08f08e250 100644 --- a/tests/test_pipx_metadata_file.py +++ b/tests/test_pipx_metadata_file.py @@ -155,7 +155,7 @@ def test_package_install(monkeypatch, tmp_path, pipx_temp_env): pipx_venvs_dir / "pycowsay" / "bin" / "blackd", pipx_venvs_dir / "pycowsay" / "bin" / "blackd.exe", ], - include_apps=False + include_apps=False, ), ) else: From 72d9f3d52da78035a68f710338b838b351b171a9 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Wed, 13 Nov 2019 20:35:46 -0800 Subject: [PATCH 135/153] Make app_paths placeholder a valid Path. --- tests/test_pipx_metadata_file.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_pipx_metadata_file.py b/tests/test_pipx_metadata_file.py index a08f08e250..278b16411c 100644 --- a/tests/test_pipx_metadata_file.py +++ b/tests/test_pipx_metadata_file.py @@ -42,7 +42,7 @@ include_dependencies=False, include_apps=True, apps=["pycowsay"], - app_paths=["pycowsay/bin/pycowsay"], # Placeholder, not real path + app_paths=[Path("pycowsay/bin/pycowsay")], # Placeholder, not real path apps_of_dependencies=[], app_paths_of_dependencies={}, package_version="0.0.0.1", @@ -54,7 +54,7 @@ include_dependencies=False, include_apps=True, apps=["pycowsay"], - app_paths=["black/bin/black"], # Placeholder, not real path + app_paths=[Path("black/bin/black")], # Placeholder, not real path apps_of_dependencies=[], app_paths_of_dependencies={}, package_version="19.10b3", From 5bfb9854edbe0ba75864b80e17211ba2e4e1c984 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Wed, 13 Nov 2019 20:45:03 -0800 Subject: [PATCH 136/153] Fix test_package_install for windows. --- tests/test_pipx_metadata_file.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/test_pipx_metadata_file.py b/tests/test_pipx_metadata_file.py index a08f08e250..6eb4b367f0 100644 --- a/tests/test_pipx_metadata_file.py +++ b/tests/test_pipx_metadata_file.py @@ -150,10 +150,8 @@ def test_package_install(monkeypatch, tmp_path, pipx_temp_env): BLACK_PACKAGE_REF._replace( apps=["black", "black.exe", "blackd", "blackd.exe"], app_paths=[ - pipx_venvs_dir / "pycowsay" / "bin" / "black", - pipx_venvs_dir / "pycowsay" / "bin" / "black.exe", - pipx_venvs_dir / "pycowsay" / "bin" / "blackd", - pipx_venvs_dir / "pycowsay" / "bin" / "blackd.exe", + pipx_venvs_dir / "pycowsay" / "Scripts" / "black.exe", + pipx_venvs_dir / "pycowsay" / "Scripts" / "blackd.exe", ], include_apps=False, ), From 5aa1b5759fe6ef65bdd63f863e29b747cc9b2fb7 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Wed, 13 Nov 2019 21:01:04 -0800 Subject: [PATCH 137/153] Compare sets not lists in assert_package_metadata. --- tests/test_pipx_metadata_file.py | 86 +++++++++++++++++--------------- 1 file changed, 45 insertions(+), 41 deletions(-) diff --git a/tests/test_pipx_metadata_file.py b/tests/test_pipx_metadata_file.py index 57c04d2110..64f96b9da4 100644 --- a/tests/test_pipx_metadata_file.py +++ b/tests/test_pipx_metadata_file.py @@ -104,67 +104,71 @@ def test_pipx_metadata_file_validation(tmp_path, test_package): def assert_package_metadata(test_metadata, ref_metadata): + # update package version of ref with recent package version + # only compare sets for apps, app_paths so order is not important + assert test_metadata.package_version != "" - assert test_metadata == ref_metadata._replace( - package_version=test_metadata.package_version + assert isinstance(test_metadata.apps, list) + assert isinstance(test_metadata.app_paths, list) + + test_metadata_replaced = test_metadata._replace( + apps=set(test_metadata.apps), app_paths=set(test_metadata.apps) + ) + ref_metadata_replaced = ref_metadata._replace( + apps=set(ref_metadata.apps), + app_paths=set(ref_metadata.apps), + package_version=test_metadata.package_version, ) + assert test_metadata_replaced == ref_metadata_replaced def test_package_install(monkeypatch, tmp_path, pipx_temp_env): pipx_venvs_dir = pipx.constants.PIPX_HOME / "venvs" + # test metadata after package install run_pipx_cli(["install", "pycowsay"]) assert (pipx_venvs_dir / "pycowsay" / "pipx_metadata.json").is_file() pipx_metadata = PipxMetadata(pipx_venvs_dir / "pycowsay") if pipx.constants.WINDOWS: - assert_package_metadata( - pipx_metadata.main_package, - PYCOWSAY_PACKAGE_REF._replace( - app_paths=[pipx_venvs_dir / "pycowsay" / "Scripts" / "pycowsay.exe"], - apps=["pycowsay", "pycowsay.exe"], - include_apps=True, - ), - ) + ref_replacement_fields = { + "app_paths": [pipx_venvs_dir / "pycowsay" / "Scripts" / "pycowsay.exe"], + "apps": ["pycowsay", "pycowsay.exe"], + } else: - assert_package_metadata( - pipx_metadata.main_package, - PYCOWSAY_PACKAGE_REF._replace( - app_paths=[pipx_venvs_dir / "pycowsay" / "bin" / "pycowsay"], - include_apps=True, - ), - ) + ref_replacement_fields = { + "app_paths": [pipx_venvs_dir / "pycowsay" / "bin" / "pycowsay"] + } + assert_package_metadata( + pipx_metadata.main_package, + PYCOWSAY_PACKAGE_REF._replace(include_apps=True, **ref_replacement_fields), + ) del pipx_metadata - # test package injection - + # test metadata after package inject run_pipx_cli(["inject", "pycowsay", "black"]) pipx_metadata = PipxMetadata(pipx_venvs_dir / "pycowsay") if pipx.constants.WINDOWS: - assert_package_metadata( - pipx_metadata.injected_packages["black"], - BLACK_PACKAGE_REF._replace( - apps=["black", "black.exe", "blackd", "blackd.exe"], - app_paths=[ - pipx_venvs_dir / "pycowsay" / "Scripts" / "black.exe", - pipx_venvs_dir / "pycowsay" / "Scripts" / "blackd.exe", - ], - include_apps=False, - ), - ) + ref_replacement_fields = { + "apps": ["black", "black.exe", "blackd", "blackd.exe"], + "app_paths": [ + pipx_venvs_dir / "pycowsay" / "Scripts" / "black.exe", + pipx_venvs_dir / "pycowsay" / "Scripts" / "blackd.exe", + ], + } else: - assert_package_metadata( - pipx_metadata.injected_packages["black"], - BLACK_PACKAGE_REF._replace( - apps=["black", "blackd"], - app_paths=[ - pipx_venvs_dir / "pycowsay" / "bin" / "black", - pipx_venvs_dir / "pycowsay" / "bin" / "blackd", - ], - include_apps=False, - ), - ) + ref_replacement_fields = { + "apps": ["black", "blackd"], + "app_paths": [ + pipx_venvs_dir / "pycowsay" / "bin" / "black", + pipx_venvs_dir / "pycowsay" / "bin" / "blackd", + ], + } + assert_package_metadata( + pipx_metadata.injected_packages["black"], + BLACK_PACKAGE_REF._replace(include_apps=False, **ref_replacement_fields), + ) From 52d85672c5197b306e48831e47c29055617a0808 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Wed, 13 Nov 2019 21:12:17 -0800 Subject: [PATCH 138/153] Split test_package_inject from test_package_install. --- tests/test_pipx_metadata_file.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/test_pipx_metadata_file.py b/tests/test_pipx_metadata_file.py index 64f96b9da4..49b4917ad4 100644 --- a/tests/test_pipx_metadata_file.py +++ b/tests/test_pipx_metadata_file.py @@ -125,7 +125,6 @@ def assert_package_metadata(test_metadata, ref_metadata): def test_package_install(monkeypatch, tmp_path, pipx_temp_env): pipx_venvs_dir = pipx.constants.PIPX_HOME / "venvs" - # test metadata after package install run_pipx_cli(["install", "pycowsay"]) assert (pipx_venvs_dir / "pycowsay" / "pipx_metadata.json").is_file() @@ -145,13 +144,18 @@ def test_package_install(monkeypatch, tmp_path, pipx_temp_env): PYCOWSAY_PACKAGE_REF._replace(include_apps=True, **ref_replacement_fields), ) - del pipx_metadata - # test metadata after package inject +def test_package_inject(monkeypatch, tmp_path, pipx_temp_env): + pipx_venvs_dir = pipx.constants.PIPX_HOME / "venvs" + + run_pipx_cli(["install", "pycowsay"]) run_pipx_cli(["inject", "pycowsay", "black"]) + assert (pipx_venvs_dir / "pycowsay" / "pipx_metadata.json").is_file() pipx_metadata = PipxMetadata(pipx_venvs_dir / "pycowsay") + assert pipx_metadata.injected_packages.keys() == ["black"] + if pipx.constants.WINDOWS: ref_replacement_fields = { "apps": ["black", "black.exe", "blackd", "blackd.exe"], From 7b67b7849795038d937350fce99ccd0cdeae5758 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Wed, 13 Nov 2019 21:17:42 -0800 Subject: [PATCH 139/153] Change keys() assertion to list to be compatible with dict_keys. --- tests/test_pipx_metadata_file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_pipx_metadata_file.py b/tests/test_pipx_metadata_file.py index 49b4917ad4..ee1591eb62 100644 --- a/tests/test_pipx_metadata_file.py +++ b/tests/test_pipx_metadata_file.py @@ -154,7 +154,7 @@ def test_package_inject(monkeypatch, tmp_path, pipx_temp_env): pipx_metadata = PipxMetadata(pipx_venvs_dir / "pycowsay") - assert pipx_metadata.injected_packages.keys() == ["black"] + assert pipx_metadata.injected_packages.keys() == {"black"} if pipx.constants.WINDOWS: ref_replacement_fields = { From 689a90f5da773997f482a156da232f47edafb8f8 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Wed, 13 Nov 2019 21:25:52 -0800 Subject: [PATCH 140/153] Make extra time on check_animate_output 0.5 seconds. --- tests/test_animate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_animate.py b/tests/test_animate.py index 0389fafeb9..acba0966bb 100644 --- a/tests/test_animate.py +++ b/tests/test_animate.py @@ -19,7 +19,7 @@ def check_animate_output( chars_to_test = 1 + len("".join(frame_strings[:frames_to_test])) with pipx.animate.animate(test_string, do_animation=True): - time.sleep(frame_period * (frames_to_test - 1) + 0.2) + time.sleep(frame_period * (frames_to_test - 1) + 0.5) captured = capsys.readouterr() assert captured.err[:chars_to_test] == expected_string[:chars_to_test] From 2edb95eaf1ce997f3ebb65ba6561dae8fd024019 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Thu, 14 Nov 2019 16:02:20 -0800 Subject: [PATCH 141/153] New improved method for package name determination. --- src/pipx/venv.py | 41 ++++++++++++++++++----------------------- 1 file changed, 18 insertions(+), 23 deletions(-) diff --git a/src/pipx/venv.py b/src/pipx/venv.py index 2f8df65011..182bff4f86 100644 --- a/src/pipx/venv.py +++ b/src/pipx/venv.py @@ -172,20 +172,32 @@ def install_package( include_apps: bool, is_main_package: bool, ) -> None: - if package is None: - # If no package name is supplied, find out package name installed - # by comparing old_package_set with new one after install - old_package_set = self.list_installed_packages() with animate(f"installing package {package_or_url!r}", self.do_animation): if pip_args is None: pip_args = [] + if package is None: + # If no package name is supplied, install only main package + # first in order to see what its name is + old_package_set = self.list_installed_packages() + cmd = ["install"] + pip_args + ["--no-dependencies"] + [package_or_url] + self._run_pip(cmd) + installed_packages = self.list_installed_packages() - old_package_set + if len(installed_packages) == 1: + package = installed_packages.pop() + logging.info(f"Determined package name: '{package}'") + else: + package = None cmd = ["install"] + pip_args + [package_or_url] self._run_pip(cmd) if package is None: - installed_packages = self.list_installed_packages() - old_package_set - package = self.top_of_deptree(installed_packages) + logging.warning( + f"Cannot determine package name for package_or_url='{package_or_url}'. " + f"Unable to retrieve package metadata. " + f"Unable to verify if package was installed properly." + ) + return self._update_package_metadata( package=package, @@ -277,23 +289,6 @@ def list_installed_packages(self) -> Set[str]: pip_list = json.loads(cmd_run.stdout.decode().strip()) return set([x["name"] for x in pip_list]) - def top_of_deptree(self, packages: Iterable[str]) -> str: - cmd = [str(self.python_path), "-m", "pip", "show"] + list(packages) - cmd_run = subprocess.run(cmd, stdout=subprocess.PIPE, universal_newlines=True) - pip_show_stdout = cmd_run.stdout - - for line in pip_show_stdout.split("\n"): - key_value_re = re.search(r"^([^:]+):\s(.*)", line) - if key_value_re: - key = key_value_re.group(1) - value = key_value_re.group(2) - if key == "Name": - package_name = value - if key == "Required-by" and value == "": - return package_name - - return "" - def run_app(self, app: str, app_args: List[str]) -> int: cmd = [str(self.bin_path / app)] + app_args try: From 6341b9c52128aab3e8c1921bd1e8fc8ec37a2feb Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Thu, 14 Nov 2019 16:21:18 -0800 Subject: [PATCH 142/153] Remove Iterable from typing import. --- src/pipx/venv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pipx/venv.py b/src/pipx/venv.py index 182bff4f86..aff814cd1e 100644 --- a/src/pipx/venv.py +++ b/src/pipx/venv.py @@ -4,7 +4,7 @@ import re import subprocess from pathlib import Path -from typing import Generator, List, NamedTuple, Dict, Set, Optional, Iterable +from typing import Generator, List, NamedTuple, Dict, Set, Optional from pipx.animate import animate from pipx.constants import DEFAULT_PYTHON, PIPX_SHARED_PTH, WINDOWS From 6308136ee4137d19a4dabf0730f98961247c8149 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Thu, 14 Nov 2019 16:21:38 -0800 Subject: [PATCH 143/153] Add test for package name determination. --- tests/test_run.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/tests/test_run.py b/tests/test_run.py index 995bf09897..0ccd07de2e 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -67,3 +67,33 @@ def test_run_ensure_null_pythonpath(): stderr=subprocess.PIPE, ).stdout ) + + +@pytest.mark.parametrize( + "package, package_or_url, app_args", + [ + ("pycowsay", "pycowsay", ["pycowsay", "hello"]), + ("black", "black", ["black", "--help"]), + ("cloudtoken", "cloudtoken", ["cloudtoken", "--help"]), + ("awscli", "awscli", ["aws", "--help"]), + ("shell-functools", "shell-functools", ["filter", "--help"]), + ("pylint", "pylint", ["pylint", "--help"]), + ("kaggle", "kaggle", ["kaggle", "--help"]), + ("ipython", "ipython", ["ipython", "--version"]), + # ("ansible", "ansible", ["ansible", "--help"]), # takes too long + ], +) +def test_package_determination( + caplog, pipx_temp_env, package, package_or_url, app_args +): + caplog.set_level(logging.INFO) # "root.venv") + + run_pipx_cli(["run", "--verbose", "--spec", package_or_url, "--"] + app_args) + + print(f"package={package}") + print(f"package_or_url={package_or_url}") + print(f"app_args={app_args}") + print("caplog.text") + print(caplog.text) + assert "Cannot determine package name" not in caplog.text + assert f"Determined package name: '{package}'" in caplog.text From 346931dc882c6f104b73bf6f673cb9ee24a0d8f2 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Thu, 14 Nov 2019 16:38:04 -0800 Subject: [PATCH 144/153] Clean up test_package_determination code. --- tests/test_run.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/test_run.py b/tests/test_run.py index 0ccd07de2e..17dd02af00 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -69,24 +69,25 @@ def test_run_ensure_null_pythonpath(): ) +# packages listed roughly in order of increasing test duration @pytest.mark.parametrize( "package, package_or_url, app_args", [ ("pycowsay", "pycowsay", ["pycowsay", "hello"]), - ("black", "black", ["black", "--help"]), - ("cloudtoken", "cloudtoken", ["cloudtoken", "--help"]), - ("awscli", "awscli", ["aws", "--help"]), ("shell-functools", "shell-functools", ["filter", "--help"]), + ("black", "black", ["black", "--help"]), ("pylint", "pylint", ["pylint", "--help"]), ("kaggle", "kaggle", ["kaggle", "--help"]), ("ipython", "ipython", ["ipython", "--version"]), + ("cloudtoken", "cloudtoken", ["cloudtoken", "--help"]), + ("awscli", "awscli", ["aws", "--help"]), # ("ansible", "ansible", ["ansible", "--help"]), # takes too long ], ) def test_package_determination( caplog, pipx_temp_env, package, package_or_url, app_args ): - caplog.set_level(logging.INFO) # "root.venv") + caplog.set_level(logging.INFO) run_pipx_cli(["run", "--verbose", "--spec", package_or_url, "--"] + app_args) From aa1544e54240f02096484ce3ffb003b1fc4bcd1f Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Thu, 14 Nov 2019 16:39:43 -0800 Subject: [PATCH 145/153] Clean up test_package_determination code 2. --- tests/test_run.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/test_run.py b/tests/test_run.py index 17dd02af00..9b279d67e7 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -91,10 +91,5 @@ def test_package_determination( run_pipx_cli(["run", "--verbose", "--spec", package_or_url, "--"] + app_args) - print(f"package={package}") - print(f"package_or_url={package_or_url}") - print(f"app_args={app_args}") - print("caplog.text") - print(caplog.text) assert "Cannot determine package name" not in caplog.text assert f"Determined package name: '{package}'" in caplog.text From e829ea02658e38ee3b99a14c99e8bf32eebec136 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Thu, 14 Nov 2019 17:21:42 -0800 Subject: [PATCH 146/153] Add and use util.run_stdout_stderr. --- src/pipx/util.py | 23 +++++++++++++++++++++++ src/pipx/venv.py | 8 +++++--- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/src/pipx/util.py b/src/pipx/util.py index 21a25e2cc6..a7a5436f87 100644 --- a/src/pipx/util.py +++ b/src/pipx/util.py @@ -103,3 +103,26 @@ def run(cmd: Sequence[Union[str, Path]], check=True) -> int: if check and returncode: raise PipxError(f"{cmd_str!r} failed") return returncode + + +def run_stdout_stderr( + cmd: Sequence[Union[str, Path]], check=True +) -> subprocess.CompletedProcess: + """Run arbitrary command as subprocess, capturing stderr and stout""" + + env = {k: v for k, v in os.environ.items() if k.upper() != "PYTHONPATH"} + cmd_str = " ".join(str(c) for c in cmd) + logging.info(f"running {cmd_str}") + # windows cannot take Path objects, only strings + cmd_str_list = [str(c) for c in cmd] + command_obj = subprocess.run( + cmd_str_list, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, # implies encoded strings in stdout, stderr + ) + returncode = command_obj.returncode + if check and returncode: + raise PipxError(f"{cmd_str!r} failed") + return command_obj diff --git a/src/pipx/venv.py b/src/pipx/venv.py index aff814cd1e..fa59f3b3bd 100644 --- a/src/pipx/venv.py +++ b/src/pipx/venv.py @@ -17,6 +17,7 @@ get_venv_paths, rmdir, run, + run_stdout_stderr, ) @@ -284,9 +285,10 @@ def pip_search(self, search_term: str, pip_search_args: List[str]) -> str: return cmd_run.stdout.decode().strip() def list_installed_packages(self) -> Set[str]: - cmd = [str(self.python_path), "-m", "pip", "list", "--format=json"] - cmd_run = subprocess.run(cmd, stdout=subprocess.PIPE) - pip_list = json.loads(cmd_run.stdout.decode().strip()) + cmd_run = run_stdout_stderr( + [str(self.python_path), "-m", "pip", "list", "--format=json"] + ) + pip_list = json.loads(cmd_run.stdout.strip()) return set([x["name"] for x in pip_list]) def run_app(self, app: str, app_args: List[str]) -> int: From be2f62b6c06e52553708f8a3b01755a2e47553ba Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Thu, 14 Nov 2019 17:28:22 -0800 Subject: [PATCH 147/153] Convert more subprocess.run calls to run_stdout_stderr. --- src/pipx/venv.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/src/pipx/venv.py b/src/pipx/venv.py index fa59f3b3bd..2fdfe205d5 100644 --- a/src/pipx/venv.py +++ b/src/pipx/venv.py @@ -2,7 +2,6 @@ import logging import pkgutil import re -import subprocess from pathlib import Path from typing import Generator, List, NamedTuple, Dict, Set, Optional @@ -269,20 +268,15 @@ def _update_package_metadata( self.pipx_metadata.write() def get_python_version(self) -> str: - return ( - subprocess.run([str(self.python_path), "--version"], stdout=subprocess.PIPE) - .stdout.decode() - .strip() - ) + return run_stdout_stderr([str(self.python_path), "--version"]).stdout.strip() def pip_search(self, search_term: str, pip_search_args: List[str]) -> str: - cmd = ( + cmd_run = run_stdout_stderr( [str(self.python_path), "-m", "pip", "search"] + pip_search_args + [search_term] ) - cmd_run = subprocess.run(cmd, stdout=subprocess.PIPE) - return cmd_run.stdout.decode().strip() + return cmd_run.stdout.strip() def list_installed_packages(self) -> Set[str]: cmd_run = run_stdout_stderr( From f4a896804672c7b67fce158b8db18b4001714bab Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Thu, 14 Nov 2019 17:42:32 -0800 Subject: [PATCH 148/153] Change run_stdout_stderr to run_subprocess, use inside of run. --- src/pipx/util.py | 38 +++++++++++++++++--------------------- src/pipx/venv.py | 8 ++++---- 2 files changed, 21 insertions(+), 25 deletions(-) diff --git a/src/pipx/util.py b/src/pipx/util.py index a7a5436f87..e9283db1cd 100644 --- a/src/pipx/util.py +++ b/src/pipx/util.py @@ -91,22 +91,10 @@ def get_site_packages(python: Path) -> Path: return Path(output.strip()) -def run(cmd: Sequence[Union[str, Path]], check=True) -> int: - """Run arbitrary command as subprocess""" - - env = {k: v for k, v in os.environ.items() if k.upper() != "PYTHONPATH"} - cmd_str = " ".join(str(c) for c in cmd) - logging.info(f"running {cmd_str}") - # windows cannot take Path objects, only strings - cmd_str_list = [str(c) for c in cmd] - returncode = subprocess.run(cmd_str_list, env=env).returncode - if check and returncode: - raise PipxError(f"{cmd_str!r} failed") - return returncode - - -def run_stdout_stderr( - cmd: Sequence[Union[str, Path]], check=True +def run_subprocess( + cmd: Sequence[Union[str, Path]], + capture_stdout: bool = True, + capture_stderr: bool = True, ) -> subprocess.CompletedProcess: """Run arbitrary command as subprocess, capturing stderr and stout""" @@ -115,14 +103,22 @@ def run_stdout_stderr( logging.info(f"running {cmd_str}") # windows cannot take Path objects, only strings cmd_str_list = [str(c) for c in cmd] - command_obj = subprocess.run( + return subprocess.run( cmd_str_list, env=env, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + stdout=subprocess.PIPE if capture_stdout else None, + stderr=subprocess.PIPE if capture_stderr else None, universal_newlines=True, # implies encoded strings in stdout, stderr ) - returncode = command_obj.returncode + + +def run(cmd: Sequence[Union[str, Path]], check=True) -> int: + """Run arbitrary command as subprocess""" + + returncode = run_subprocess( + cmd_str_list, capture_stdout=False, capture_stderr=False + ).returncode + if check and returncode: raise PipxError(f"{cmd_str!r} failed") - return command_obj + return returncode diff --git a/src/pipx/venv.py b/src/pipx/venv.py index 2fdfe205d5..4548db4beb 100644 --- a/src/pipx/venv.py +++ b/src/pipx/venv.py @@ -16,7 +16,7 @@ get_venv_paths, rmdir, run, - run_stdout_stderr, + run_subprocess, ) @@ -268,10 +268,10 @@ def _update_package_metadata( self.pipx_metadata.write() def get_python_version(self) -> str: - return run_stdout_stderr([str(self.python_path), "--version"]).stdout.strip() + return run_subprocess([str(self.python_path), "--version"]).stdout.strip() def pip_search(self, search_term: str, pip_search_args: List[str]) -> str: - cmd_run = run_stdout_stderr( + cmd_run = run_subprocess( [str(self.python_path), "-m", "pip", "search"] + pip_search_args + [search_term] @@ -279,7 +279,7 @@ def pip_search(self, search_term: str, pip_search_args: List[str]) -> str: return cmd_run.stdout.strip() def list_installed_packages(self) -> Set[str]: - cmd_run = run_stdout_stderr( + cmd_run = run_subprocess( [str(self.python_path), "-m", "pip", "list", "--format=json"] ) pip_list = json.loads(cmd_run.stdout.strip()) From 1ec79594014ca1355ec591fa8f0de1e208dad98a Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Thu, 14 Nov 2019 17:48:01 -0800 Subject: [PATCH 149/153] Fix typo in comment. --- src/pipx/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pipx/util.py b/src/pipx/util.py index e9283db1cd..89e994e9c4 100644 --- a/src/pipx/util.py +++ b/src/pipx/util.py @@ -108,7 +108,7 @@ def run_subprocess( env=env, stdout=subprocess.PIPE if capture_stdout else None, stderr=subprocess.PIPE if capture_stderr else None, - universal_newlines=True, # implies encoded strings in stdout, stderr + universal_newlines=True, # implies decoded strings in stdout, stderr ) From 0011da23c8bafda310d5a8d50fb118c50deb3651 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Thu, 14 Nov 2019 17:50:05 -0800 Subject: [PATCH 150/153] Fix bug in run, misnamed variable. --- src/pipx/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pipx/util.py b/src/pipx/util.py index 89e994e9c4..a3de23c656 100644 --- a/src/pipx/util.py +++ b/src/pipx/util.py @@ -116,7 +116,7 @@ def run(cmd: Sequence[Union[str, Path]], check=True) -> int: """Run arbitrary command as subprocess""" returncode = run_subprocess( - cmd_str_list, capture_stdout=False, capture_stderr=False + cmd, capture_stdout=False, capture_stderr=False ).returncode if check and returncode: From d11a7d53dd162831e352d02e49afec9dd8260251 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Thu, 14 Nov 2019 21:47:10 -0800 Subject: [PATCH 151/153] Refine TODOs, add Issue for tracking. --- src/pipx/commands/upgrade.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/pipx/commands/upgrade.py b/src/pipx/commands/upgrade.py index b0048249cd..00804fca46 100644 --- a/src/pipx/commands/upgrade.py +++ b/src/pipx/commands/upgrade.py @@ -58,7 +58,7 @@ def upgrade( include_apps=include_apps, is_main_package=True, ) - # TODO 20191026: Should we upgrade injected packages also? + # TODO 20191026: upgrade injected packages also (Issue #79) package_metadata = venv.package_metadata[package] new_version = package_metadata.package_version @@ -108,7 +108,6 @@ def upgrade_all( upgrading_all=True, force=force, ) - # TODO 20191024: Upgrade injected packages except Exception: logging.error(f"Error encountered when upgrading {package}") From 6f621ac01c8b8f61b7b60f48b98b5201bb00e8f2 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Thu, 14 Nov 2019 22:02:07 -0800 Subject: [PATCH 152/153] Add cmd_str for error message in run(). --- src/pipx/util.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/pipx/util.py b/src/pipx/util.py index a3de23c656..f976b2febd 100644 --- a/src/pipx/util.py +++ b/src/pipx/util.py @@ -119,6 +119,8 @@ def run(cmd: Sequence[Union[str, Path]], check=True) -> int: cmd, capture_stdout=False, capture_stderr=False ).returncode + cmd_str = " ".join(str(c) for c in cmd) + if check and returncode: raise PipxError(f"{cmd_str!r} failed") return returncode From 1a0035fc40fd16c46b2f134e19583aa2e3776866 Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Fri, 15 Nov 2019 13:07:08 -0800 Subject: [PATCH 153/153] Add comment explaining why we delete PYTHONPATH. --- src/pipx/util.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/pipx/util.py b/src/pipx/util.py index f976b2febd..8cd7a95aee 100644 --- a/src/pipx/util.py +++ b/src/pipx/util.py @@ -98,6 +98,9 @@ def run_subprocess( ) -> subprocess.CompletedProcess: """Run arbitrary command as subprocess, capturing stderr and stout""" + # Null out PYTHONPATH because some platforms (macOS with Homebrew) add + # pipx directories to it, and can make it appear to venvs as though + # pipx dependencies are in the venv path (#233) env = {k: v for k, v in os.environ.items() if k.upper() != "PYTHONPATH"} cmd_str = " ".join(str(c) for c in cmd) logging.info(f"running {cmd_str}")