Skip to content

Commit

Permalink
feat: add optional cython extension (#44)
Browse files Browse the repository at this point in the history
  • Loading branch information
bdraco authored Oct 1, 2022
1 parent 1ca2f12 commit b737574
Show file tree
Hide file tree
Showing 11 changed files with 396 additions and 205 deletions.
10 changes: 9 additions & 1 deletion .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,9 @@ jobs:
- "3.10"
os:
- ubuntu-latest
extension:
- "skip_cython"
- "use_cython"
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v3
Expand All @@ -55,7 +58,12 @@ jobs:
python-version: ${{ matrix.python-version }}
cache: "poetry"
- name: Install Dependencies
run: poetry install
run: |
if [ "${{ matrix.extension }}" = "skip_cython" ]; then
SKIP_CYTHON=1 poetry install
else
poetry install
fi
- name: Test with Pytest
run: export $(dbus-launch); poetry run pytest --cov-report=xml
- name: Upload coverage to Codecov
Expand Down
3 changes: 2 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,8 @@ Desktop users can use this library to create their own scripts and utilities to

dbus-fast plans to improve over other DBus libraries for Python in the following ways:

- Zero dependencies and pure Python 3.
- Zero dependencies and pure Python 3
- An optional cython extension is available to speed up (un)marshalling
- Focus on performance
- Support for multiple IO backends including asyncio and the GLib main loop.
- Nonblocking IO suitable for GUI development.
Expand Down
3 changes: 3 additions & 0 deletions bench/unmarshall.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,9 @@

from dbus_fast._private.unmarshaller import Unmarshaller

# cythonize -X language_level=3 -a -i src/dbus_fast/_private/unmarshaller.py


bluez_rssi_message = (
"6c04010134000000e25389019500000001016f00250000002f6f72672f626c75657a2f686369302f6465"
"765f30385f33415f46325f31455f32425f3631000000020173001f0000006f72672e667265656465736b"
Expand Down
32 changes: 32 additions & 0 deletions build.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
"""Build optional cython modules."""

import contextlib
import os
from distutils.command.build_ext import build_ext


class BuildExt(build_ext):
def build_extensions(self):
try:
super().build_extensions()
except Exception:
pass


def build(setup_kwargs):
if os.environ.get("SKIP_CYTHON", False):
return
with contextlib.suppress(Exception):
from Cython.Build import cythonize

setup_kwargs.update(
dict(
ext_modules=cythonize(
[
"src/dbus_fast/_private/marshaller.py",
"src/dbus_fast/_private/unmarshaller.py",
]
),
cmdclass=dict(build_ext=BuildExt),
)
)
289 changes: 168 additions & 121 deletions poetry.lock

Large diffs are not rendered by default.

32 changes: 18 additions & 14 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ classifiers = [
"Operating System :: OS Independent",
"Topic :: Software Development :: Libraries",
]
build = "build.py"
packages = [
{ include = "dbus_fast", from = "src" },
]
Expand All @@ -24,23 +25,24 @@ packages = [

[tool.poetry.dependencies]
python = "^3.7"
async-timeout = ">=3.0.0"

# Documentation Dependencies
sphinxcontrib-asyncio = {version = "^0.3.0", extras = ["docs"]}
sphinxcontrib-fulltoc = {version = "^1.2.0", extras = ["docs"]}
Sphinx = {version = "^5.1.1", extras = ["docs"]}
myst-parser = {version = "^0.18.0", extras = ["docs"]}
sphinx-rtd-theme = {version = "^1.0.0", extras = ["docs"]}
async-timeout = ">=3.0.0"
sphinxcontrib-asyncio = {version = "^0.3.0", optional = true}
sphinxcontrib-fulltoc = {version = "^1.2.0", optional = true}
Sphinx = {version = "^5.1.1", optional = true}
myst-parser = {version = "^0.18.0", optional = true}
sphinx-rtd-theme = {version = "^1.0.0", optional = true}

[tool.poetry.extras]
docs = [
"myst-parser",
"sphinx",
"sphinx-rtd-theme",
"sphinxcontrib-asyncio",
"sphinxcontrib-fulltoc"
]
docs = ["myst-parser", "sphinx", "sphinx-rtd-theme", "sphinxcontrib-asyncio", "sphinxcontrib-fulltoc"]

[tool.poetry.group.docs.dependencies]
myst-parser = "^0.18.0"
sphinx = "^5.1.1"
sphinx-rtd-theme = "^1.0.0"
sphinxcontrib-asyncio = "^0.3.0"
sphinxcontrib-fulltoc = "^1.2.0"

[tool.poetry.dev-dependencies]
pytest = "^7.0"
Expand All @@ -50,6 +52,8 @@ pytest-asyncio = "^0.19.0"
[tool.poetry.group.dev.dependencies]
pycairo = "^1.21.0"
PyGObject = "^3.42.2"
Cython = "^0.29.32"
setuptools = "^65.4.1"

[tool.semantic_release]
branch = "main"
Expand Down Expand Up @@ -100,5 +104,5 @@ module = "docs.*"
ignore_errors = true

[build-system]
requires = ["poetry-core>=1.0.0"]
requires = ['setuptools>=65.4.1', 'wheel', 'Cython', "poetry-core>=1.0.0"]
build-backend = "poetry.core.masonry.api"
9 changes: 0 additions & 9 deletions setup.py

This file was deleted.

34 changes: 34 additions & 0 deletions src/dbus_fast/_private/marshaller.pxd
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
"""cdefs for marshaller.py"""

import cython


cdef class Marshaller:

cdef object signature_tree
cdef object _buf
cdef object body


@cython.locals(
offset=cython.ulong,
)
cpdef int align(self, unsigned long n)


@cython.locals(
signature_len=cython.uint,
written=cython.uint,
)
cpdef write_string(self, object value, _ = *)

@cython.locals(
array_len=cython.uint,
written=cython.uint,
)
cpdef write_array(self, object array, object type)

@cython.locals(
written=cython.uint,
)
cpdef write_single(self, object type_, object body)
61 changes: 35 additions & 26 deletions src/dbus_fast/_private/marshaller.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from struct import Struct, error, pack
from struct import Struct, error
from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple

from ..signature import SignatureTree, SignatureType, Variant
Expand All @@ -7,39 +7,48 @@


class Marshaller:
def __init__(self, signature: str, body: Any) -> None:
"""Marshall data for Dbus."""

__slots__ = ("signature_tree", "_buf", "body")

def __init__(self, signature: str, body: List[Any]) -> None:
"""Marshaller constructor."""
self.signature_tree = SignatureTree._get(signature)
self.buffer = bytearray()
self._buf = bytearray()
self.body = body

@property
def buffer(self):
return self._buf

def align(self, n) -> int:
offset = n - len(self.buffer) % n
offset = n - len(self._buf) % n
if offset == 0 or offset == n:
return 0
self.buffer.extend(bytes(offset))
self._buf.extend(bytes(offset))
return offset

def write_boolean(self, boolean: bool, _=None) -> int:
written = self.align(4)
self.buffer.extend(PACK_UINT32(int(boolean)))
self._buf.extend(PACK_UINT32(int(boolean)))
return written + 4

def write_signature(self, signature: str, _=None) -> int:
signature = signature.encode()
signature_bytes = signature.encode()
signature_len = len(signature)
self.buffer.append(signature_len)
self.buffer.extend(signature)
self.buffer.append(0)
self._buf.append(signature_len)
self._buf.extend(signature_bytes)
self._buf.append(0)
return signature_len + 2

def write_string(self, value: str, _=None) -> int:
value = value.encode()
def write_string(self, value, _=None) -> int:
value_bytes = value.encode()
value_len = len(value)
written = self.align(4) + 4
self.buffer.extend(PACK_UINT32(value_len))
self.buffer.extend(value)
self._buf.extend(PACK_UINT32(value_len))
self._buf.extend(value_bytes)
written += value_len
self.buffer.append(0)
self._buf.append(0)
written += 1
return written

Expand All @@ -52,9 +61,9 @@ def write_array(self, array: Iterable[Any], type_: SignatureType) -> int:
# TODO max array size is 64MiB (67108864 bytes)
written = self.align(4)
# length placeholder
offset = len(self.buffer)
offset = len(self._buf)
written += self.align(4) + 4
self.buffer.extend(PACK_UINT32(0))
self._buf.extend(PACK_UINT32(0))
child_type = type_.children[0]

if child_type.token in "xtd{(":
Expand All @@ -67,13 +76,13 @@ def write_array(self, array: Iterable[Any], type_: SignatureType) -> int:
array_len += self.write_dict_entry([key, value], child_type)
elif child_type.token == "y":
array_len = len(array)
self.buffer.extend(array)
self._buf.extend(array)
elif child_type.token in self._writers:
writer, packer, size = self._writers[child_type.token]
if not writer:
for value in array:
array_len += self.align(size) + size
self.buffer.extend(packer(value))
self._buf.extend(packer(value))
else:
for value in array:
array_len += writer(self, value, child_type)
Expand All @@ -84,7 +93,7 @@ def write_array(self, array: Iterable[Any], type_: SignatureType) -> int:

array_len_packed = PACK_UINT32(array_len)
for i in range(offset, offset + 4):
self.buffer[i] = array_len_packed[i - offset]
self._buf[i] = array_len_packed[i - offset]

return written + array_len

Expand All @@ -109,7 +118,7 @@ def write_single(self, type_: SignatureType, body: Any) -> int:
writer, packer, size = self._writers[t]
if not writer:
written = self.align(size)
self.buffer.extend(packer(body))
self._buf.extend(packer(body))
return written + size
return writer(self, body, type_)

Expand All @@ -119,10 +128,10 @@ def marshall(self):
self._construct_buffer()
except error:
self.signature_tree.verify(self.body)
return self.buffer
return self._buf

def _construct_buffer(self):
self.buffer.clear()
self._buf.clear()
for i, type_ in enumerate(self.signature_tree.types):
t = type_.token
if t not in self._writers:
Expand All @@ -132,11 +141,11 @@ def _construct_buffer(self):
if not writer:

# In-line align
offset = size - len(self.buffer) % size
offset = size - len(self._buf) % size
if offset != 0 and offset != size:
self.buffer.extend(bytes(offset))
self._buf.extend(bytes(offset))

self.buffer.extend(packer(self.body[i]))
self._buf.extend(packer(self.body[i]))
else:
writer(self, self.body[i], type_)

Expand Down
62 changes: 62 additions & 0 deletions src/dbus_fast/_private/unmarshaller.pxd
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
"""cdefs for unmarshaller.py"""

import cython

from ..signature import SignatureType


cdef unsigned int UINT32_SIZE
cdef unsigned int HEADER_ARRAY_OF_STRUCT_SIGNATURE_POSITION
cdef unsigned int HEADER_SIGNATURE_SIZE

cdef class Unmarshaller:

cdef object unix_fds
cdef object buf
cdef object view
cdef unsigned int pos
cdef object stream
cdef object sock
cdef object _message
cdef object readers
cdef unsigned int body_len
cdef unsigned int serial
cdef unsigned int header_len
cdef object message_type
cdef object flag
cdef unsigned int msg_len
cdef object _uint32_unpack


@cython.locals(
start_len=cython.ulong,
missing_bytes=cython.ulong,
)
cpdef read_to_pos(self, unsigned long pos)

cpdef read_string_cast(self, type_ = *)

@cython.locals(
beginning_pos=cython.ulong,
array_length=cython.uint,
)
cpdef read_array(self, object type_)

@cython.locals(
o=cython.ulong,
signature_len=cython.uint,
)
cpdef read_signature(self, type_ = *)

@cython.locals(
endian=cython.uint,
protocol_version=cython.uint,
)
cpdef _read_header(self)

@cython.locals(
beginning_pos=cython.ulong,
o=cython.ulong,
signature_len=cython.uint,
)
cpdef header_fields(self, unsigned int header_length)
Loading

0 comments on commit b737574

Please sign in to comment.