Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

MAINT: Fixup and remove bundled dependencies #1234

Merged
merged 16 commits into from
Feb 27, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .appveyor.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,9 @@ install:

# Install the build and runtime dependencies of the project.
- conda update conda -y
- conda install -q --yes python=%PYTHON_VERSION% conda pip pytest pytest-xdist pytest-timeout filelock selenium conda-build bzip2
- conda install -q --yes python=%PYTHON_VERSION% conda pip pytest pytest-xdist pytest-timeout filelock selenium conda-build bzip2 pympler
- python -m pip install -U pip
- python -m pip install pytest-rerunfailures
- python -m pip install pytest-rerunfailures json5 pympler

# Check that we have the expected version of Python
- python --version
Expand Down
65 changes: 58 additions & 7 deletions asv/benchmark.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@
import cProfile as profile
import ctypes
import importlib.machinery
import importlib.util
import inspect
import itertools
import json
Expand All @@ -58,13 +59,15 @@
import traceback
import contextlib
import math
from pathlib import Path
from hashlib import sha256
from importlib import import_module
from collections import Counter
from time import process_time

wall_timer = timeit.default_timer

ON_PYPY = hasattr(sys, 'pypy_version_info')

if sys.platform.startswith('win'):
import ctypes.wintypes
Expand Down Expand Up @@ -704,16 +707,64 @@ def __init__(self, name, func, attr_sources):
self.unit = "bytes"

def run(self, *param):
# We can't import asizeof directly, because we haven't loaded
# the asv package in the benchmarking process.
path = os.path.join(
os.path.dirname(__file__), 'extern', 'asizeof.py')
asizeof = importlib.machinery.SourceFileLoader('asizeof', path).load_module()
if ON_PYPY:
raise NotImplementedError("asizeof doesn't work on pypy")
return

def import_asizeof():
"""Import asizeof, searching system Pythons in PATH."""
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why do we need all this? If the import fails, can't we just fail the run?

Copy link
Member Author

@HaoZeke HaoZeke Feb 26, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The problem here is that the import fails where it shouldn't, e.g. on the CI. This seems to be because of the process isolation (each ASV run is in a different subprocess, which doesn't include asv and on CI cannot use pip either). Perhaps, now that I'm thinking about it, it would be cleaner to instead import asv itself, which would automatically bring in pympler and is discussed in #908

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If I went with that approach then the Import Mechanism would instead load ASV rather than asizeof, but I'm not sure how to simplify this further without breaking the subprocess approach.

path_dirs = os.environ.get("PATH", "").split(os.pathsep)

# On Windows, append the directories containing the Python executables
if os.name == "nt":
path_dirs += [sys.base_exec_prefix, sys.base_exec_prefix + "/Scripts"]

asizeof_paths = set()
for path in path_dirs:
python_path = os.path.join(path, "python")
if os.path.isfile(python_path) and os.access(python_path, os.X_OK):
cand_path = Path(python_path).parent.parent / "lib"
if cand_path not in asizeof_paths:
asizeof_paths.add(cand_path)
for asizeof_path in cand_path.rglob("asizeof.py"):
try: # Still returns the first importable asizeof
loader = importlib.machinery.SourceFileLoader(
"asizeof", str(asizeof_path)
)
return loader.load_module()
except ImportError:
pass

# Try conda, mamba explicitly, needed on Windows
try:
env_path = os.environ["CONDA_PREFIX"]
except KeyError:
pass
else:
cand_path = Path(env_path) / "lib"
if cand_path not in asizeof_paths:
asizeof_paths.add(cand_path)
for asizeof_path in cand_path.rglob("asizeof.py"):
try: # Still returns the first importable asizeof
loader = importlib.machinery.SourceFileLoader(
"asizeof", str(asizeof_path)
)
return loader.load_module()
except ImportError:
pass

return NotImplementedError("asizeof not found anywhere")

try:
from pympler.asizeof import asizeof
except ImportError:
asizeof = import_asizeof()
from asizeof import asizeof

obj = self.func(*param)

sizeof2 = asizeof.asizeof([obj, obj])
sizeofcopy = asizeof.asizeof([obj, copy.copy(obj)])
sizeof2 = asizeof([obj, obj])
sizeofcopy = asizeof([obj, copy.copy(obj)])

return sizeofcopy - sizeof2

Expand Down
Empty file removed asv/extern/__init__.py
Empty file.
Loading