Skip to content

Commit

Permalink
perf: lazily import our modules and external libraries (#624)
Browse files Browse the repository at this point in the history
### Summary of Changes

We depend on several large libraries that take a while to load.
Previously, importing `Table` would already import almost all of them,
leading to horrendous startup times:

```
(Measure-Command { python -c "from safeds.data.tabular.containers import Table" }).TotalSeconds
```

➡️ 3.5068337 (seconds to import `Table` on main)

Now, we lazily import our own modules in the `__init__.py` files, and we
lazily import external libraries. The latter part is quite ugly, since
each function must now contain their external imports at the start.
There is no better solution, however, and the improvements are huge:

```
(Measure-Command { python -c "from safeds.data.tabular.containers import Table" }).TotalSeconds
```

➡️ 0.1683219 (seconds to import `Table` in this branch)

We still have to pay the cost for an import once we first import a
module, but at least this no longer has to happen fully upfront.

---------

Co-authored-by: megalinter-bot <[email protected]>
  • Loading branch information
lars-reimann and megalinter-bot authored Apr 17, 2024
1 parent e856cd5 commit 20fc313
Show file tree
Hide file tree
Showing 56 changed files with 800 additions and 291 deletions.
4 changes: 4 additions & 0 deletions .github/linters/.ruff.toml
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,10 @@ ignore = [
"D106",
"D107",
]
"__init__.py" = [
# runtime-import-in-type-checking-block: Does not work with apipkg.
"TCH004",
]

[pydocstyle]
convention = "numpy"
16 changes: 13 additions & 3 deletions poetry.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 2 additions & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ packages = [

[tool.poetry.dependencies]
python = "^3.11,<3.13"
apipkg = "^3.0.2"
ipython = "^8.8.0"
levenshtein = ">=0.21.1,<0.26.0"
matplotlib = "^3.6.3"
Expand All @@ -23,10 +24,10 @@ pillow = ">=9.5,<11.0"
scikit-image = ">=0.21,<0.23"
scikit-learn = "^1.2.0"
seaborn = "^0.13.0"
statsmodels = "^0.14.1"
torch = {version = "^2.2.0", source = "torch_cuda121"}
torchvision = {version = "^0.17.0", source = "torch_cuda121"}
xxhash = "^3.4.1"
statsmodels = "^0.14.1"

[tool.poetry.group.dev.dependencies]
pytest = ">=7.2.1,<9.0.0"
Expand Down
14 changes: 13 additions & 1 deletion src/safeds/_config/__init__.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,18 @@
"""Configuration for Safe-DS."""

from ._device import _get_device
from typing import TYPE_CHECKING

import apipkg

if TYPE_CHECKING:
from ._device import _get_device

apipkg.initpkg(
__name__,
{
"_get_device": "._device:_get_device",
},
)

__all__ = [
"_get_device",
Expand Down
10 changes: 8 additions & 2 deletions src/safeds/_config/_device.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,12 @@
import torch
from torch.types import Device
from __future__ import annotations

from typing import TYPE_CHECKING

if TYPE_CHECKING:
from torch.types import Device


def _get_device() -> Device:
import torch

return torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")
14 changes: 13 additions & 1 deletion src/safeds/_utils/__init__.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,18 @@
"""Utilities for Safe-DS."""

from ._hashing import _structural_hash
from typing import TYPE_CHECKING

import apipkg

if TYPE_CHECKING:
from ._hashing import _structural_hash

apipkg.initpkg(
__name__,
{
"_structural_hash": "._hashing:_structural_hash",
},
)

__all__ = [
"_structural_hash",
Expand Down
4 changes: 2 additions & 2 deletions src/safeds/_utils/_hashing.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,6 @@
import struct
from typing import Any

import xxhash


def _structural_hash(*value: Any) -> int:
"""
Expand All @@ -20,6 +18,8 @@ def _structural_hash(*value: Any) -> int:
hash
Deterministic hash value
"""
import xxhash

return xxhash.xxh3_64(_value_to_bytes(value)).intdigest()


Expand Down
17 changes: 15 additions & 2 deletions src/safeds/data/image/containers/__init__.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,20 @@
"""Classes that can store image data."""

from ._image import Image
from ._image_list import ImageList
from typing import TYPE_CHECKING

import apipkg

if TYPE_CHECKING:
from ._image import Image
from ._image_list import ImageList

apipkg.initpkg(
__name__,
{
"Image": "._image:Image",
"ImageList": "._image_list:ImageList",
},
)

__all__ = [
"Image",
Expand Down
7 changes: 3 additions & 4 deletions src/safeds/data/image/containers/_empty_image_list.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@
from typing import TYPE_CHECKING, Self

from safeds._utils import _structural_hash
from safeds.data.image.containers._image_list import ImageList
from safeds.data.image.containers._single_size_image_list import _SingleSizeImageList
from safeds.data.image.utils._image_transformation_error_and_warning_checks import (
_check_add_noise_errors,
_check_adjust_brightness_errors_and_warnings,
Expand All @@ -15,6 +17,7 @@
_check_resize_errors,
_check_sharpen_errors_and_warnings,
)
from safeds.exceptions import IndexOutOfBoundsError

if TYPE_CHECKING:
from pathlib import Path
Expand All @@ -23,10 +26,6 @@

from safeds.data.image.containers import Image

from safeds.data.image.containers._image_list import ImageList
from safeds.data.image.containers._single_size_image_list import _SingleSizeImageList
from safeds.exceptions import IndexOutOfBoundsError


class _EmptyImageList(ImageList):
"""
Expand Down
Loading

0 comments on commit 20fc313

Please sign in to comment.