From 139161bbcdd8b88b3b49a6519ea1afe0906837df Mon Sep 17 00:00:00 2001 From: KafCoppelia Date: Mon, 20 May 2024 10:08:56 +0800 Subject: [PATCH 1/7] =?UTF-8?q?=F0=9F=A4=96=20updated=20workflow?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/release-drafter.yml | 2 +- .github/workflows/pytest-ci.yml | 14 +++++++++++--- .github/workflows/release-drafter.yml | 6 ++---- .pre-commit-config.yaml | 2 +- 4 files changed, 15 insertions(+), 9 deletions(-) diff --git a/.github/release-drafter.yml b/.github/release-drafter.yml index b14be329..93a18c1a 100644 --- a/.github/release-drafter.yml +++ b/.github/release-drafter.yml @@ -34,4 +34,4 @@ template: | $CHANGES - Changelog: https://github.com/PAICookers/PAIBox/compare/$PREVIOUS_TAG...v$RESOLVED_VERSION + **Full Changelog**: https://github.com/PAICookers/PAIBox/compare/$PREVIOUS_TAG...v$RESOLVED_VERSION diff --git a/.github/workflows/pytest-ci.yml b/.github/workflows/pytest-ci.yml index e232a4af..1253e6bf 100644 --- a/.github/workflows/pytest-ci.yml +++ b/.github/workflows/pytest-ci.yml @@ -2,7 +2,13 @@ name: Python CI with pytest on: pull_request: - branches: [master, dev] + branches: + - master + - dev + types: [opened, synchronize, reopened] + +permissions: + contents: read jobs: pytest-ci: @@ -14,17 +20,19 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + - name: Install poetry uses: abatilo/actions-poetry@v2 - with: - poetry-version: "1.8.2" + - name: Install test dependencies run: | poetry install --with test + - name: Run pytest uses: pavelzw/pytest-action@v2 with: diff --git a/.github/workflows/release-drafter.yml b/.github/workflows/release-drafter.yml index e981aa37..35271f32 100644 --- a/.github/workflows/release-drafter.yml +++ b/.github/workflows/release-drafter.yml @@ -7,8 +7,7 @@ on: pull_request_target: branches: - master - types: - - closed + types: [opened, reopened, synchronize] permissions: contents: read @@ -16,10 +15,9 @@ permissions: jobs: update_release_draft: permissions: - # write permission is required to create a github release contents: write + pull-requests: write runs-on: ubuntu-latest - steps: - uses: actions/checkout@v4 with: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 80cdaed4..e3d27505 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ default_install_hook_types: [pre-commit, prepare-commit-msg] ci: autofix_commit_msg: ":rotating_light: auto fix by pre-commit hooks" autofix_prs: true - autoupdate_branch: master + autoupdate_branch: dev autoupdate_schedule: monthly autoupdate_commit_msg: ":arrow_up: auto update by pre-commit hooks" repos: From b87e28b14c756cd4949f6445ee766c203adfef9b Mon Sep 17 00:00:00 2001 From: KafCoppelia Date: Mon, 20 May 2024 10:09:20 +0800 Subject: [PATCH 2/7] =?UTF-8?q?=F0=9F=90=9B=20bugifx:=20shape?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- paibox/components/neuron/base.py | 2 +- paibox/components/synapses/synapses.py | 4 ++-- paibox/types.py | 2 +- paibox/utils.py | 9 ++++----- 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/paibox/components/neuron/base.py b/paibox/components/neuron/base.py index 02f9ca12..5f16315e 100644 --- a/paibox/components/neuron/base.py +++ b/paibox/components/neuron/base.py @@ -47,8 +47,8 @@ def __init__( """Stateless attributes. Scalar.""" # Basic attributes. self.keep_shape = keep_shape - self._n_neuron = shape2num(shape) self._shape = as_shape(shape) + self._n_neuron = shape2num(self._shape) # DO NOT modify the names of the following variables. # They will be exported to the parameter verification model. diff --git a/paibox/components/synapses/synapses.py b/paibox/components/synapses/synapses.py index c4568839..b715da58 100644 --- a/paibox/components/synapses/synapses.py +++ b/paibox/components/synapses/synapses.py @@ -191,8 +191,8 @@ def __init__( kernel, _single(stride), _single(padding), - _single(output_padding), _single(1), + _single(output_padding), kernel_order, name, ) @@ -241,8 +241,8 @@ def __init__( kernel, _pair(stride), _pair(padding), - _pair(output_padding), _pair(1), + _pair(output_padding), kernel_order, name, ) diff --git a/paibox/types.py b/paibox/types.py index f0be6fe5..dcb44871 100644 --- a/paibox/types.py +++ b/paibox/types.py @@ -16,7 +16,7 @@ from numpy.typing import NDArray from typing_extensions import TypeAlias -Shape = TypeVar("Shape", int, Tuple[int, ...], List[int]) +Shape = TypeVar("Shape", int, Tuple[int, ...], List[int], np.ndarray) ArrayType = TypeVar("ArrayType", List[int], Tuple[int, ...], np.ndarray) Scalar = TypeVar("Scalar", int, float, np.generic) IntScalarType = TypeVar("IntScalarType", int, np.bool_, np.integer) diff --git a/paibox/utils.py b/paibox/utils.py index b8f233cb..53936f22 100644 --- a/paibox/utils.py +++ b/paibox/utils.py @@ -72,6 +72,8 @@ def shape2num(shape: Shape) -> int: """Convert a shape to a number""" if isinstance(shape, int): return shape + elif isinstance(shape, np.ndarray): + return int(np.prod(shape)) else: a = 1 for b in shape: @@ -83,12 +85,9 @@ def shape2num(shape: Shape) -> int: def as_shape(x, min_dim: int = 0) -> Tuple[int, ...]: """Return a tuple if `x` is iterable or `(x,)` if `x` is integer.""" if is_integer(x): - _shape = (x,) + _shape = (int(x),) elif is_iterable(x): - if isinstance(x, np.ndarray): - _shape = tuple(x.astype(int)) - else: - _shape = tuple(x) + _shape = tuple(int(e) for e in x) else: raise ValueError(f"{x} cannot be safely converted to a shape.") From 65cdb6cb16b13a7d276f041b70f98f9006e6200d Mon Sep 17 00:00:00 2001 From: KafCoppelia Date: Mon, 20 May 2024 14:28:34 +0800 Subject: [PATCH 3/7] =?UTF-8?q?=E2=AC=86=EF=B8=8F=20bump=20paicorelib=20^1?= =?UTF-8?q?.1.2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 4 ++-- poetry.lock | 6 +++--- pyproject.toml | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 8081396f..73b483e2 100644 --- a/README.md +++ b/README.md @@ -8,8 +8,8 @@ - - + + diff --git a/poetry.lock b/poetry.lock index dbe17341..dc018182 100644 --- a/poetry.lock +++ b/poetry.lock @@ -434,7 +434,7 @@ reference = "tsinghua" [[package]] name = "paicorelib" -version = "1.1.1" +version = "1.1.2" description = "Library of PAICORE 2.0" optional = false python-versions = "^3.8" @@ -449,7 +449,7 @@ pydantic = "^2.0" type = "git" url = "https://github.com/PAICookers/PAIlib.git" reference = "dev" -resolved_reference = "27f8ffba9ce717fbf43d945805ae2c0199724880" +resolved_reference = "6caecbc017654a9aba495749cd1efa6f791a69c2" [[package]] name = "pluggy" @@ -746,4 +746,4 @@ reference = "tsinghua" [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "1f509a9a743d277a9625cd2255835b12ddb47e17842ebf52e1d81b4805192b3e" +content-hash = "a7752c7014de56276fed6b5b2911582453facdb20604aef44e64bfcf31a8a4e5" diff --git a/pyproject.toml b/pyproject.toml index 8782b1ed..e1193124 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "paibox" -version = "1.1.0a1" +version = "1.1.0a2" description = "Toolchain of PAICORE 2.0" authors = ["Ziru Pan "] maintainers = [ @@ -38,7 +38,7 @@ include = ["docs/Guide-of-PAIBox.md", "CHANGELOG.md"] python = "^3.8" pydantic = "^2.0" numpy = "^1.24.0" -paicorelib = "^1.1.1" +paicorelib = "^1.1.2" [tool.poetry.group.test] optional = true From 22797927f66cfc173eee9f6ea133a890a9da9f2e Mon Sep 17 00:00:00 2001 From: KafCoppelia Date: Mon, 20 May 2024 16:54:41 +0800 Subject: [PATCH 4/7] =?UTF-8?q?=F0=9F=A7=91=E2=80=8D=F0=9F=92=BB=20add=20a?= =?UTF-8?q?rgument=20use=5Fhw=5Fsim=20for=20mapper.export?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- paibox/backend/conf_template.py | 25 +++++++++++++------------ paibox/backend/mapper.py | 14 +++++++++++++- 2 files changed, 26 insertions(+), 13 deletions(-) diff --git a/paibox/backend/conf_template.py b/paibox/backend/conf_template.py index a35ea042..feac54af 100644 --- a/paibox/backend/conf_template.py +++ b/paibox/backend/conf_template.py @@ -1,6 +1,6 @@ from enum import Enum from pathlib import Path -from typing import Any, ClassVar, Dict, List, Literal, NamedTuple, TypedDict +from typing import Any, ClassVar, Dict, List, NamedTuple, TypedDict import numpy as np from numpy.typing import NDArray @@ -335,7 +335,7 @@ def gen_config_frames_by_coreconf( write_to_file: bool, fp: Path, split_by_coord: bool, - format: Literal["txt", "bin", "npy"], + formats: List[str], ) -> Dict[Coord, FrameArrayType]: """Generate configuration frames by given the `CorePlmConfig`. @@ -344,17 +344,18 @@ def gen_config_frames_by_coreconf( - write_to_file: whether to write frames to file. - fp: If `write_to_file` is `True`, specify the path. - split_by_coord: whether to split the generated frames file by the core coordinates. - - format: `txt`, `bin`, or `npy`. + - format: a list of formats to export. """ def _write_to_f(name: str, array: FrameArrayType) -> None: - _fp = fp / (name + f".{format}") - if format == "npy": - np2npy(_fp, array) - elif format == "bin": - np2bin(_fp, array) - else: - np2txt(_fp, array) + for format in formats: + _fp = fp / (name + f".{format}") + if format == "npy": + np2npy(_fp, array) + elif format == "bin": + np2bin(_fp, array) + else: + np2txt(_fp, array) _default_rid = RId(0, 0) _debug_dict: Dict[Coord, Dict[str, Any]] = dict() @@ -443,10 +444,10 @@ def _write_to_f(name: str, array: FrameArrayType) -> None: addr = core_coord.address _write_to_f(f"config_core{addr}", f) else: - _f = np.concatenate( + f = np.concatenate( list(frame_arrays_on_core.values()), dtype=FRAME_DTYPE, casting="no" ) - _write_to_f(f"config_cores_all", _f) + _write_to_f(f"config_cores_all", f) return frame_arrays_on_core diff --git a/paibox/backend/mapper.py b/paibox/backend/mapper.py index e404aa0b..56683e77 100644 --- a/paibox/backend/mapper.py +++ b/paibox/backend/mapper.py @@ -502,6 +502,7 @@ def export( format: Literal["txt", "bin", "npy"] = "bin", split_by_coord: bool = False, export_core_params: bool = False, + use_hw_sim: bool = True, ) -> Dict[Coord, Any]: """Generate configuration frames & export to file. @@ -511,15 +512,26 @@ def export( - format: `txt`, `bin`, or `npy`. `bin` is recommended. - split_by_coord: whether to split the generated frames file by the core coordinates. - export_core_params: whether to export the parameters of occupied cores. + - use_hw_sim: whether to use hardware simulator. If use, '.txt' will be exported. Return: a dictionary of configurations. """ if format not in ("bin", "npy", "txt"): raise ValueError(f"format {format} is not supported.") + formats = [format] + if use_hw_sim: + formats.append("txt") + + formats = list(set(formats)) + _fp = _fp_check(fp) config_dict = gen_config_frames_by_coreconf( - self.graph_info["members"], write_to_file, _fp, split_by_coord, format + self.graph_info["members"], + write_to_file, + _fp, + split_by_coord, + formats, ) if export_core_params: From 6d317a69f6cf83ca51f2bb7f361661c1dc2003eb Mon Sep 17 00:00:00 2001 From: KafCoppelia Date: Mon, 20 May 2024 17:20:58 +0800 Subject: [PATCH 5/7] =?UTF-8?q?=E2=9C=85=20added=20utils=20&=20removed=20u?= =?UTF-8?q?seless=20func?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- paibox/utils.py | 83 +---------------------------------- tests/conftest.py | 13 ++++++ tests/simulator/test_probe.py | 3 +- tests/test_base.py | 10 ++--- tests/utils.py | 25 +++++++++++ 5 files changed, 46 insertions(+), 88 deletions(-) create mode 100644 tests/utils.py diff --git a/paibox/utils.py b/paibox/utils.py index 53936f22..1a492d06 100644 --- a/paibox/utils.py +++ b/paibox/utils.py @@ -1,4 +1,4 @@ -from typing import Any, Iterable, List, Optional, Sequence, Tuple, Union +from typing import Any, Iterable, List, Sequence, Tuple import numpy as np @@ -130,84 +130,3 @@ def is_iterable(obj: Any) -> bool: def fn_sgn(a, b) -> int: """Signal function.""" return 1 if a > b else -1 if a < b else 0 - - -def bin_split(x: int, pos: int, high_mask: Optional[int] = None) -> Tuple[int, int]: - """Split an integer, return the high and low part. - - Argument: - - x: the integer - - pos: the position (LSB) to split the binary. - - high_mask: mask for the high part. Optional. - - Example:: - - >>> bin_split(0b1100001001, 3) - 97(0b1100001), 1 - """ - low = x & ((1 << pos) - 1) - - if isinstance(high_mask, int): - high = (x >> pos) & high_mask - else: - high = x >> pos - - return high, low - - -def bin_combine(high: int, low: int, pos: int) -> int: - """Combine two integers, return the result. - - Argument: - - high: the integer on the high bit. - - low: the integer on the low bit. - - pos: the combination bit if provided. Must be equal or greater than `low.bit_length()`. - - Example:: - - >>> bin_combine(0b11000, 0b101, 5) - 773(0b11000_00101) - """ - if pos < 0: - raise ValueError("position must be greater than 0") - - if low > 0 and pos < low.bit_length(): - raise ValueError( - f"Postion of combination must be greater than the bit length of low({low.bit_length()})" - ) - - return (high << pos) + low - - -def bin_combine_x(*components: int, pos: Union[int, List[int], Tuple[int, ...]]) -> int: - """Combine more than two integers, return the result. - - Argument: - - components: the list of integers to be combined. - - pos: the combination bit(s) if provided. Every bit must be equal or greater than `low.bit_length()`. - - Example:: - - >>> bin_combine_x(0b11000, 0b101, 0b1011, pos=[10, 5]) - 24747(0b11000_00101_01011) - """ - if isinstance(pos, (list, tuple)): - if len(components) != len(pos) + 1: - raise ValueError( - f"Length of components and positions illegal: {len(components)}, {len(pos)}" - ) - else: - if len(components) != 2: - raise ValueError( - f"Length of components must be 2: {len(components)} when position is an integer." - ) - - return bin_combine(*components, pos=pos) - - result = components[-1] - - # Traverse every position from the end to the start - for i in range(len(pos) - 1, -1, -1): - result = bin_combine(components[i], result, pos[i]) - - return result diff --git a/tests/conftest.py b/tests/conftest.py index ca0f896b..9f510fef 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -12,6 +12,7 @@ from paibox.naming import clear_name_cache from .shared_networks import * +from .utils import * if sys.version_info >= (3, 11): from typing import NotRequired @@ -65,6 +66,18 @@ def clean_name_dict(): clear_name_cache(ignore_warn=True) +@pytest.fixture +def perf_fixture(request): + with measure_time(f"{request.node.name}"): + yield + + +@pytest.fixture +def random_fixture(): + with fixed_random_seed(42): + yield + + class ParametrizedTestData(TypedDict): """Parametrized test data in dictionary format.""" diff --git a/tests/simulator/test_probe.py b/tests/simulator/test_probe.py index 7b830206..2db2571f 100644 --- a/tests/simulator/test_probe.py +++ b/tests/simulator/test_probe.py @@ -1,11 +1,12 @@ import pytest import paibox as pb +from paibox.base import PAIBoxObject from paibox.exceptions import PAIBoxWarning def test_probe_instanece(): - pbobj = pb.base.PAIBoxObject() + pbobj = PAIBoxObject() pb1 = pb.Probe(pbobj, "name", name="pb1") pb2 = pb.Probe(pbobj, "name", name="pb2") diff --git a/tests/test_base.py b/tests/test_base.py index c5cd9c72..b9e7ba4b 100644 --- a/tests/test_base.py +++ b/tests/test_base.py @@ -1,6 +1,6 @@ import pytest - import paibox as pb +from paibox.base import PAIBoxObject from paibox.exceptions import RegisterError @@ -9,9 +9,9 @@ def test_paibox_version(): def test_paiboxobject_eq(): - obj1 = pb.base.PAIBoxObject(name="obj1") - obj2 = pb.base.PAIBoxObject(name="obj2") - obj3 = pb.base.PAIBoxObject() + obj1 = PAIBoxObject(name="obj1") + obj2 = PAIBoxObject(name="obj2") + obj3 = PAIBoxObject() # eq assert obj1 != obj2 @@ -26,7 +26,7 @@ def test_paiboxobject_eq(): def test_paiboxobject_nodes(): - obj1 = pb.base.PAIBoxObject(name="obj111") + obj1 = PAIBoxObject(name="obj111") nodes1 = obj1.nodes(method="absolute", level=1, include_self=True) assert nodes1["obj111"] == obj1 diff --git a/tests/utils.py b/tests/utils.py new file mode 100644 index 00000000..8a8cd5b1 --- /dev/null +++ b/tests/utils.py @@ -0,0 +1,25 @@ +from contextlib import contextmanager +import time +import numpy as np +from typing import Any, Generator, Optional + + +@contextmanager +def measure_time(desc: str) -> Generator[None, Any, None]: + start_time = time.time() + try: + yield + finally: + end_time = time.time() + elapsed = end_time - start_time + print(f"{desc} executed in: {elapsed:.2f} secs") + + +@contextmanager +def fixed_random_seed(seed: Optional[int] = None) -> Generator[None, Any, None]: + state = np.random.get_state() + np.random.seed(seed) + try: + yield + finally: + np.random.set_state(state) From 341bfea02eb40d7baf9e3364e4ac44c4a945409b Mon Sep 17 00:00:00 2001 From: KafCoppelia Date: Mon, 20 May 2024 19:19:08 +0800 Subject: [PATCH 6/7] =?UTF-8?q?=E2=9C=85=20always=20set=20default=20for=20?= =?UTF-8?q?backend=20context=20after=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- paibox/backend/context.py | 8 ++++++-- tests/conftest.py | 9 +++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/paibox/backend/context.py b/paibox/backend/context.py index 68d3bf51..91c256ba 100644 --- a/paibox/backend/context.py +++ b/paibox/backend/context.py @@ -18,7 +18,7 @@ class _BackendContext(_Context): - _DefaultContext = { + _default = { "output_chip_addr": DEFAULT_OUTPUT_CHIP_ADDR, # RO mostly "target_chip_addr": DEFAULT_LOCAL_CHIP_ADDR, # RO mostly "build_directory": Path.cwd(), # R/W @@ -31,7 +31,7 @@ class _BackendContext(_Context): def __init__(self) -> None: super().__init__() - self.update(self._DefaultContext) + self.update(self._default) @property def target_chip_addr(self) -> List[ChipCoord]: @@ -86,6 +86,10 @@ def output_dir(self, p: Union[str, Path]) -> None: def cflags(self) -> Dict[str, Any]: """Compilation options.""" return self["cflags"] + + def set_default(self) -> None: + self.clear_all() + self.update(self._default) _BACKEND_CONTEXT = _BackendContext() diff --git a/tests/conftest.py b/tests/conftest.py index 9f510fef..5791a2c9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,6 +8,7 @@ import pytest import paibox as pb +from paibox.base import SynSys from paibox.components import Neuron from paibox.naming import clear_name_cache @@ -66,6 +67,14 @@ def clean_name_dict(): clear_name_cache(ignore_warn=True) +@pytest.fixture(autouse=True) +def backend_context_setdefault(): + """Set the default backend context after each test automatically.""" + yield + SynSys.CFLAG_ENABLE_WP_OPTIMIZATION = True + pb.BACKEND_CONFIG.set_default() + + @pytest.fixture def perf_fixture(request): with measure_time(f"{request.node.name}"): From ab5fab383e2ecf648f28d9de28c5323647f9f7fa Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 21 May 2024 10:02:30 +0000 Subject: [PATCH 7/7] :rotating_light: auto fix by pre-commit hooks --- paibox/backend/context.py | 2 +- tests/test_base.py | 1 + tests/utils.py | 5 +++-- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/paibox/backend/context.py b/paibox/backend/context.py index 91c256ba..ba4a1b3f 100644 --- a/paibox/backend/context.py +++ b/paibox/backend/context.py @@ -86,7 +86,7 @@ def output_dir(self, p: Union[str, Path]) -> None: def cflags(self) -> Dict[str, Any]: """Compilation options.""" return self["cflags"] - + def set_default(self) -> None: self.clear_all() self.update(self._default) diff --git a/tests/test_base.py b/tests/test_base.py index b9e7ba4b..5adb2c7f 100644 --- a/tests/test_base.py +++ b/tests/test_base.py @@ -1,4 +1,5 @@ import pytest + import paibox as pb from paibox.base import PAIBoxObject from paibox.exceptions import RegisterError diff --git a/tests/utils.py b/tests/utils.py index 8a8cd5b1..333c0cdc 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,8 +1,9 @@ -from contextlib import contextmanager import time -import numpy as np +from contextlib import contextmanager from typing import Any, Generator, Optional +import numpy as np + @contextmanager def measure_time(desc: str) -> Generator[None, Any, None]: