From 5f83bdce27532699509a6874014adc53ac32d662 Mon Sep 17 00:00:00 2001 From: Mike Hendricks Date: Fri, 22 Oct 2021 10:09:15 -0700 Subject: [PATCH] Simplify and add testing of update-egg-info Make use of existing setuptools_scm git code Add GitWorkdir.get_git_dir to find the .git directory Correct the environment variable name Improve documentation of limitations Add update-egg-info testing --- MANIFEST.in | 1 + README.rst | 37 ++++-- src/setuptools_scm/git.py | 5 + src/setuptools_scm/pre_commit_hook.py | 168 +++++++++++------------ testing/test_git.py | 185 ++++++++++++++++++++++++++ 5 files changed, 290 insertions(+), 106 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 92e3a537..13f4ce91 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,6 @@ exclude *.nix exclude .pre-commit-config.yaml +include .pre-commit-hooks.yaml include *.py include testing/*.py include tox.ini diff --git a/README.rst b/README.rst index bc7c4492..df5e8ee8 100644 --- a/README.rst +++ b/README.rst @@ -457,6 +457,13 @@ Environment variables when defined, a ``os.pathsep`` separated list of directory names to ignore for root finding + +:SETUPTOOLS_SCM_SKIP_UPDATE_EGG_INFO: + when set to 1, skip running the build command if the ``update-egg-info`` pre-commit + hooks are installed. Use this if you are doing a lot of git operations and don't + want to wait for the somewhat expensive build command to be run after each of them + without uninstalling those pre-commit hooks. + Extending setuptools_scm ------------------------ @@ -586,19 +593,25 @@ its still possible to install a more recent version of setuptools in order to ha and/or install the package by using wheels or eggs. -Automatic Update of Editable Install Version --------------------------------------------- -When developing a pip package you often will use an editable install of your pip package +Automatic Update of Editable Installs Version +--------------------------------------------- +While developing a pip package you often will use an editable install of your package to make development easier. If you have complex dependencies you may run into -``VersionConflict`` errors when another package updates its minimum version of a package you -have editable installed. +``VersionConflict`` errors when another package updates its requirements when using +``pkg_resources`` to process entry points including console_scripts. + +.. code-block: python + + pkg_resources.VersionConflict: (My_Package 2.75.0.dev0+g54e73c49.d20211024 (c:\dev\my_package), Requirement.parse('My_Package>=0.0.14')) + -You can use `pre-commit `_ to install several post hooks that -automatically call ``python setup.py egg_info``. If you are using setuptools_scm to +You can use `pre-commit `_ to install several post git hooks +that automatically call ``python setup.py egg_info``. If you are using setuptools_scm to generate your version this will update the installed version for all editable installs using this repo. -To configure your repo to make use of this, add this info to it's ``.pre-commit-config.yaml`` file. +To configure your repo to make use of this, add this info to it's +``.pre-commit-config.yaml`` file. .. code-block:: yaml @@ -609,22 +622,22 @@ To configure your repo to make use of this, add this info to it's ``.pre-commit- - id: update-egg-info - id: update-egg-info-rewrite -You will need to be using pre-commit >= 2.15.0. You will need to specify multiple hook-type -arguments to install hooks. +You will need to be using pre-commit >= 2.15.0. You will need to specify multiple +hook-type arguments to install the required hooks. .. code-block:: shell $ pre-commit install -t post-commit -t post-merge -t post-checkout -t post-rewrite While this is installed any time you commit, merge, rebase, checkout changes on your git -repo, it the version of your editable pip package will be up to date. +repo, the version of your editable pip package will be kept up to date. Customizing the Build Command ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ By default update-egg-info will use the command ``python setup.py egg_info``, this takes -the least amount of time to update the version metadata for a standard setuptools +the least amount of time to update the version metadata for a standard setuptools_scm editable install to update the version information. However if you need to customize the build command you can pass a custom build command to each hook. diff --git a/src/setuptools_scm/git.py b/src/setuptools_scm/git.py index 22e870c4..91eeb8c0 100644 --- a/src/setuptools_scm/git.py +++ b/src/setuptools_scm/git.py @@ -57,6 +57,11 @@ def get_branch(self): branch = None return branch + def get_git_dir(self): + """Returns the absolute path to the .git directory""" + out, err, ret = self.do_ex("git rev-parse --absolute-git-dir") + return out.strip() + def get_head_date(self): timestamp, err, ret = self.do_ex("git log -n 1 HEAD --format=%cI") if ret: diff --git a/src/setuptools_scm/pre_commit_hook.py b/src/setuptools_scm/pre_commit_hook.py index c4100bf3..65016031 100644 --- a/src/setuptools_scm/pre_commit_hook.py +++ b/src/setuptools_scm/pre_commit_hook.py @@ -1,107 +1,81 @@ -import glob import os -import subprocess import sys +from setuptools_scm.git import GitWorkdir + class PreCommitHook: def __init__(self, repo_dir, build_command): - self._git_repo_dir = None - self._is_git_repo = None self.build_command = build_command - self.repo_dir = repo_dir - - def call_subprocess(self, cmd, env=None, cwd=None, communicate=True): - kwargs = { - "bufsize": 1, - "stdout": subprocess.PIPE, - "stderr": subprocess.STDOUT, - "universal_newlines": True, - } - if env: - kwargs["env"] = env - if not cwd: - kwargs["cwd"] = self.repo_dir - - if sys.platform == "win32" and "pythonw" in sys.executable.lower(): - # Windows 7 has an issue with making subprocesses - # if the stdin handle is None, so pass the PIPE - kwargs["stdin"] = subprocess.PIPE - # Then, because there is no current window for stdout, any - # subprocesses will try to create a new window - # This constant comes from the WindowsAPI, but is not - # defined in subprocess until python 3.7 - create_no_window = 0x08000000 - kwargs["creationflags"] = create_no_window - - proc = subprocess.Popen(cmd, **kwargs) - - out = None - err = None - if communicate: - out, err = proc.communicate() - # Report any errors that happened while running the hook - if proc.returncode: - print(" stdout ".center(50, "-")) - print(out) - print(" stderr ".center(50, "-")) - print(err) - raise Exception( - "Error {} returned on on subprocess call: {}".format( - proc.returncode, cmd - ) - ) - return proc, out, err - - @property - def git_repo_dir(self): - """Returns the root of the git repo if this is a git repo or None.""" - if self._git_repo_dir is None and self.is_git_repo: - proc, output, _ = self.call_subprocess( - ["git", "rev-parse", "--absolute-git-dir"] - ) - self._git_repo_dir = output.strip() - return self._git_repo_dir + self.work_dir = GitWorkdir.from_potential_worktree(repo_dir) def is_git_rebasing(self): - """Checks if git is currently in rebase mode.""" - rebase_merge = os.path.join(self.git_repo_dir, "rebase-merge") - rebase_apply = os.path.join(self.git_repo_dir, "rebase-apply") + """Checks if git is currently in rebase mode and returns the output of build_command. + + This detection code could be improved, Here is a summary of how well this + handles various git operations. + Rebase: A series of post-commit hooks where `is_git_rebasing` returns true, + and finally the post-rewrite hook is called. This is why we have a separate + `update-egg-info-rewrite` hook that passes `--post-rewrite` to force the + running of build_command. + Squash: For each commit being squashed, `is_git_rebasing` returns true for + the post-commit hook, but the post-rewrite hook is called several times, + calling build_command each of those times. + Amend: This ends up calling build_command twice. + """ + git_dir = self.work_dir.get_git_dir() + rebase_merge = os.path.join(git_dir, "rebase-merge") + rebase_apply = os.path.join(git_dir, "rebase-apply") return os.path.exists(rebase_merge) or os.path.exists(rebase_apply) - def is_git_repo(self): - if self._is_git_repo is None: - proc, output, _ = self.call_subprocess( - ["git", "rev-parse", "--is-inside-work-tree"] + def update_egg_info(self, force_on_rebase=False): + """Run the `python setup.py egg_info` if this is a editable install.""" + + # Allow users to temporarily disable the hook. `is_git_rebasing` doesn't work + # in all cases and there may be cases where you want to disable this hook to + # speed up git operations. Running egg_info adds some slowness to the git + # commands, especially for large packages with a lot of files. + if os.getenv("SETUPTOOLS_SCM_SKIP_UPDATE_EGG_INFO", "0") != "0": + output = ( + 'update-egg-info: Skipping, "SETUPTOOLS_SCM_SKIP_UPDATE_EGG_INFO" ' + "env var set." ) - self._is_git_repo = proc.output.strip() == "true" - if None is self._is_git_repo: - raise OSError("git rev-parse error") - return self._is_git_repo + return output, 0 - def run_egg_info(self): - """Run the `python setup.py egg_info` if this is a editable install.""" + # If this is not an editable install there is no need to run it + contents = os.listdir(self.work_dir.path) - if not os.path.exists(os.path.join(self.repo_dir, "setup.py")): - # This is not a setup.py pip package so there is nothing to do. - return False + # If there is not a setup.cfg or setup.py file then this can't be an editable + # install, no need to try to run build_command. + if "setup.cfg" not in contents and "setup.py" not in contents: + output = "update-egg-info: Skipping, no setup script found." + return output, 0 - egg_info = os.path.join(self.repo_dir, "*.egg-info") + # If a egg_info directory doesn't exist its not currently an editable install + # don't turn it into one. + if not any(filter(lambda i: os.path.splitext(i)[-1] == ".egg-info", contents)): + output = "update-egg-info: Skipping, no .egg-info directory found." + return output, 0 - if not glob.glob(egg_info): - # This package is not currently a editable install, so there is no need - # to update the .egg-info folder. - return False + # Check if git is currently rebasing, if so and force_on_rebase is False, this + # is likely a post-commit call, and the post-rewrite hook will be called later, + # skip running build_command for now. + if not force_on_rebase and self.is_git_rebasing(): + output = "update-egg-info: Skipping, rebase in progress." + return output, 0 - # This environment variable prevents setup.py from running this recursively - if "PILLAR_VERSION_NO_EGG_INFO" not in os.environ: - env = dict(os.environ) - env["PILLAR_VERSION_NO_EGG_INFO"] = "1" + # Run the build command + output = [f"update-egg-info: Running command: {self.build_command}"] + cmd_out, error, returncode = self.work_dir.do_ex(self.build_command) - proc, output, error = self.call_subprocess(self.build_command, env=env) + if cmd_out: + output += cmd_out + if returncode and error: + output.append("update-egg-info: Error running build_command:") + output.append(error) - return proc.returncode - return 0 + output = "\n".join(output) + return output, returncode def main(): @@ -113,7 +87,15 @@ def main(): "editable installs." ) ) - parser.add_argument("--post-rewrite", action="store_true") + parser.add_argument( + "--post-rewrite", + action="store_true", + help=( + "Force running `command` after a rebase. update-egg-info skips calling " + "`command` if it detects that git is performing a rebase as generating the" + "egg_info takes extra time, especially for larger packages." + ), + ) parser.add_argument( "command", default=["python", "setup.py", "egg_info"], @@ -126,14 +108,12 @@ def main(): args = parser.parse_args() hook = PreCommitHook(os.getcwd(), build_command=args.command) - # TODO: look at the contents of .git to see if we can auto-detect a post-rewrite - # print(os.listdir(r'C:\blur\dev\chernobyl\.git')) - if not args.post_rewrite and hook.is_git_rebasing(): - print("It's a rebase, skipping.") - return - - print("Running run_egg_info.") - sys.exit(hook.run_egg_info()) + output, returncode = hook.update_egg_info(force_on_rebase=args.post_rewrite) + + print(output) + + # Return the error code so pre-commit can report the failure. + sys.exit(returncode) if __name__ == "__main__": diff --git a/testing/test_git.py b/testing/test_git.py index c9115510..c5034516 100644 --- a/testing/test_git.py +++ b/testing/test_git.py @@ -1,4 +1,5 @@ import os +import subprocess import sys from datetime import date from datetime import datetime @@ -12,8 +13,11 @@ from setuptools_scm import integration from setuptools_scm import NonNormalizedVersion from setuptools_scm.file_finder_git import git_find_files +from setuptools_scm.pre_commit_hook import PreCommitHook +from setuptools_scm.utils import _always_strings from setuptools_scm.utils import do from setuptools_scm.utils import has_command +from setuptools_scm.utils import no_git_env pytestmark = pytest.mark.skipif( @@ -401,3 +405,184 @@ def test_git_getdate_badgit( git_wd = git.GitWorkdir(os.fspath(wd.cwd)) with patch.object(git_wd, "do_ex", Mock(return_value=("%cI", "", 0))): assert git_wd.get_head_date() is None + + +def create_pip_package(wd): + """Creates a simple pip package in this git repo supporting setuptools_scm""" + wd.cwd.joinpath("setup.py").write_text( + "\n".join(("from setuptools import setup", "setup(use_scm_version=True)", "")) + ) + wd.cwd.joinpath("pyproject.toml").write_text( + "\n".join( + ( + "[build-system]", + 'requires = ["setuptools>=45", "wheel", "setuptools_scm>=6.2"]', + "[tool.setuptools_scm]", + 'write_to = "version.py"', + ) + ) + ) + wd.cwd.joinpath("setup.cfg").write_text( + "\n".join(("[metadata]", "name = test_package")) + ) + wd("git add setup.py setup.cfg pyproject.toml") + wd.commit() + + +def do_interactive(cwd, branch="master"): + """Force git into an interactive rebase state.""" + # Remove all the git variables from the environment so we can add a required one. + env = _always_strings( + dict( + no_git_env(os.environ), + # os.environ, + # try to disable i18n + LC_ALL="C", + LANGUAGE="", + HGPLAIN="1", + ) + ) + + # Force git into an interactive rebase state + # Adapted from https://stackoverflow.com/a/15394837 + env["GIT_SEQUENCE_EDITOR"] = "sed -i -re 's/^noop/e HEAD/'" + + proc = subprocess.Popen( + f"git rebase -i {branch}", + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + cwd=str(cwd), + env=env, + ) + proc.wait() + + +def test_is_git_rebasing(wd): + hook = PreCommitHook(wd.cwd, []) + wd.commit_testfile() + + # git_dir = hook.work_dir.get_git_dir() + assert hook.is_git_rebasing() is False + do_interactive(wd.cwd) + assert hook.is_git_rebasing() is True + + +def test_pre_commit_hook(wd, monkeypatch): + build_command = ["python", "setup.py", "egg_info"] + hook = PreCommitHook(wd.cwd, build_command) + success_message = f"update-egg-info: Running command: {build_command}" + git_wd = git.GitWorkdir(os.fspath(wd.cwd)) + + # 1. Test the various skip methods + out, ret = hook.update_egg_info() + # All skipped hooks should not return a error level + assert ret == 0 + assert out == "update-egg-info: Skipping, no setup script found." + + create_pip_package(wd) + + # At this point there is a valid config but it has not been "editable installed" + out, ret = hook.update_egg_info() + assert ret == 0 + assert out == "update-egg-info: Skipping, no .egg-info directory found." + + # Simulate an editable install by creating the egg_info folder + git_wd.do_ex("python setup.py egg_info") + version_path = wd.cwd.joinpath("version.py") + + # Remove the version.py file so we can verify that build_command was run + version_path.unlink() + + # Check that "SETUPTOOLS_SCM_SKIP_UPDATE_EGG_INFO" env var is respected + monkeypatch.setenv("SETUPTOOLS_SCM_SKIP_UPDATE_EGG_INFO", "1") + out, ret = hook.update_egg_info() + assert ret == 0 + assert out == ( + 'update-egg-info: Skipping, "SETUPTOOLS_SCM_SKIP_UPDATE_EGG_INFO" ' + "env var set." + ) + assert not version_path.exists() + + monkeypatch.setenv("SETUPTOOLS_SCM_SKIP_UPDATE_EGG_INFO", "0") + out, ret = hook.update_egg_info() + assert ret == 0 + assert success_message in out + assert version_path.exists() + monkeypatch.delenv("SETUPTOOLS_SCM_SKIP_UPDATE_EGG_INFO") + + # 2. Test a correctly configured setuptools_scm working copy + version_path.unlink() + + # Simulate post-commit, post-checkout or post-merge hook call + out, ret = hook.update_egg_info() + assert ret == 0 + assert success_message in out + assert version_path.exists() + version_path.unlink() + + # Use an interactive rebase to check that the hook skips processing if + # `--post-rewrite` is not passed + do_interactive(wd.cwd) + + out, ret = hook.update_egg_info() + assert ret == 0 + assert out == "update-egg-info: Skipping, rebase in progress." + assert not version_path.exists() + + # If `--post-rewrite` is passed by the post-rewrite hook, build_command is run + # even if git is currently rebasing + out, ret = hook.update_egg_info(force_on_rebase=True) + assert ret == 0 + assert success_message in out + assert version_path.exists() + + # After the rebase is finished, the hook runs again + version_path.unlink() + git_wd.do_ex("git rebase --continue") + out, ret = hook.update_egg_info() + assert ret == 0 + assert success_message in out + assert version_path.exists() + + # 3. Check that if there is a error calling build_command it is reported to git + with wd.cwd.joinpath("setup.py").open("a") as f: + f.write("\nsyntax error") + out, ret = hook.update_egg_info() + assert ret == 1 + assert "update-egg-info: Error running build_command:" in out + + +def test_pre_commit_hook_cli(wd): + # Create a working pip package with the egg_info folder + create_pip_package(wd) + git_wd = git.GitWorkdir(os.fspath(wd.cwd)) + git_wd.do_ex("python setup.py egg_info") + version_path = wd.cwd.joinpath("version.py") + # Remove version.py created by egg_info command + version_path.unlink() + + build_command = ["python", "setup.py", "egg_info"] + success_message = "update-egg-info: Running command: {}" + + # Test post-commit, post-checkout or post-merge hook call + out, _, ret = git_wd.do_ex("python -m setuptools_scm.pre_commit_hook") + assert ret == 0 + assert success_message.format(build_command) in out + assert version_path.exists() + + # Test that the post-rewrite flag is respected + version_path.unlink() + out, _, ret = git_wd.do_ex( + "python -m setuptools_scm.pre_commit_hook --post-rewrite" + ) + assert ret == 0 + assert success_message.format(build_command) in out + assert version_path.exists() + + # Test that passing a custom command evaluates correctly + out, err, ret = git_wd.do_ex( + "python -m setuptools_scm.pre_commit_hook touch test.txt" + ) + assert ret == 0 + assert success_message.format(["touch", "test.txt"]) in out + assert wd.cwd.joinpath("test.txt").exists()