From 0b2e0b0c8b2692ba8348d16e066c9ea30dd7e9dd Mon Sep 17 00:00:00 2001 From: Andrew Hawker Date: Sun, 30 Aug 2020 14:14:33 -0700 Subject: [PATCH] Add microsecond provider The microsecond provider uses more precise clock resolution to create randomness values that should monotonically increase on timestamp (millisecond) collision. When creating a new ULID, we extract the number of microseconds from our timestamp and use them as the first two bytes of the randomness value. Caveats: When using `ulid.from_timestamp`, we fallback to the default implementation of 10 bytes of pure randomness. The reason for this, is that the timestamp that is being passed in by the caller is in milliseconds. Since we don't have a more accurate timestamp to "steal" bytes from, there's not much we can do. We cannot make a new epoch timestamp as the microsecond remainder is only accurate when working within the same millisecond. --- tests/test_api_api.py | 17 ++-- tests/test_api_default.py | 7 ++ tests/test_api_microsecond.py | 30 +++++++ tests/test_api_monotonic.py | 7 ++ tests/test_providers_default.py | 12 +++ tests/test_providers_microsecond.py | 115 ++++++++++++++++++++++++ tests/test_providers_monotonic.py | 12 +++ tests/test_providers_package.py | 2 +- tests/test_providers_time_default.py | 4 + tests/test_providers_time_nanosecond.py | 4 + ulid/__init__.py | 2 +- ulid/api/api.py | 3 +- ulid/api/default.py | 2 +- ulid/api/microsecond.py | 33 +++++++ ulid/api/monotonic.py | 2 +- ulid/providers/__init__.py | 5 +- ulid/providers/base.py | 14 +++ ulid/providers/microsecond.py | 54 +++++++++++ 18 files changed, 305 insertions(+), 20 deletions(-) create mode 100644 tests/test_api_microsecond.py create mode 100644 tests/test_providers_microsecond.py create mode 100644 ulid/api/microsecond.py create mode 100644 ulid/providers/microsecond.py diff --git a/tests/test_api_api.py b/tests/test_api_api.py index f841d13..a9c3103 100644 --- a/tests/test_api_api.py +++ b/tests/test_api_api.py @@ -16,6 +16,7 @@ def mock_provider(mocker): Fixture that yields a mock provider. """ provider = mocker.Mock(spec=providers.Provider) + provider.new = mocker.Mock(side_effect=providers.DEFAULT.new) provider.timestamp = mocker.Mock(side_effect=providers.DEFAULT.timestamp) provider.randomness = mocker.Mock(side_effect=providers.DEFAULT.randomness) return provider @@ -55,22 +56,14 @@ def test_all_defined_expected_methods(): ] -def test_api_new_calls_provider_timestamp(mock_api): +def test_api_new_calls_provider_new(mock_api): """ - Assert :meth:`~ulid.api.api.Api.new` calls :meth:`~ulid.providers.base.Provider.timestamp` for a value. + Assert :meth:`~ulid.api.api.Api.new` calls :meth:`~ulid.providers.base.Provider.new` for timestamp + and randomness values. """ mock_api.new() - mock_api.provider.timestamp.assert_called_once_with() - - -def test_api_new_calls_provider_randomness(mocker, mock_api): - """ - Assert :meth:`~ulid.api.api.Api.new` calls :meth:`~ulid.providers.base.Provider.randomness` for a value. - """ - mock_api.new() - - mock_api.provider.randomness.assert_called_once_with(mocker.ANY) + mock_api.provider.new.assert_called_once_with() def test_api_from_timestamp_calls_provider_randomness(mocker, mock_api, valid_bytes_48): diff --git a/tests/test_api_default.py b/tests/test_api_default.py index 6e0edbc..aec6c10 100644 --- a/tests/test_api_default.py +++ b/tests/test_api_default.py @@ -21,3 +21,10 @@ def test_module_exposes_expected_interface(): Assert that :attr:`~ulid.api.default.__all__` exposes expected interface. """ assert default.__all__ == ALL + + +def test_module_api_uses_correct_provider(): + """ + Assert that the API instance uses the correct provider type. + """ + assert isinstance(default.API.provider, type(default.providers.DEFAULT)) diff --git a/tests/test_api_microsecond.py b/tests/test_api_microsecond.py new file mode 100644 index 0000000..009db6e --- /dev/null +++ b/tests/test_api_microsecond.py @@ -0,0 +1,30 @@ +""" + test_api_microsecond + ~~~~~~~~~~~~~~~~~~~~ + + Tests for the :mod:`~ulid.api.microsecond` module. +""" +from ulid.api import microsecond +from ulid.api.api import ALL + + +def test_module_has_dunder_all(): + """ + Assert that :mod:`~ulid.api.microsecond` exposes the :attr:`~ulid.api.__all__` attribute as a list. + """ + assert hasattr(microsecond, '__all__') + assert isinstance(microsecond.__all__, list) + + +def test_module_exposes_expected_interface(): + """ + Assert that :attr:`~ulid.api.microsecond.__all__` exposes expected interface. + """ + assert microsecond.__all__ == ALL + + +def test_module_api_uses_correct_provider(): + """ + Assert that the API instance uses the correct provider type. + """ + assert isinstance(microsecond.API.provider, type(microsecond.providers.MICROSECOND)) diff --git a/tests/test_api_monotonic.py b/tests/test_api_monotonic.py index 68d3695..d9b90c2 100644 --- a/tests/test_api_monotonic.py +++ b/tests/test_api_monotonic.py @@ -21,3 +21,10 @@ def test_module_exposes_expected_interface(): Assert that :attr:`~ulid.api.monotonic.__all__` exposes expected interface. """ assert monotonic.__all__ == ALL + + +def test_module_api_uses_correct_provider(): + """ + Assert that the API instance uses the correct provider type. + """ + assert isinstance(monotonic.API.provider, type(monotonic.providers.MONOTONIC)) diff --git a/tests/test_providers_default.py b/tests/test_providers_default.py index 2546a21..1491800 100644 --- a/tests/test_providers_default.py +++ b/tests/test_providers_default.py @@ -26,6 +26,18 @@ def test_provider_derives_from_base(): assert issubclass(default.Provider, base.Provider) +def test_provider_new_returns_bytes_pair(provider): + """ + Assert that :meth:`~ulid.providers.default.Provider.new` returns timestamp and randomness + bytes of expected length as a two item tuple. + """ + value = provider.new() + assert isinstance(value, tuple) + assert len(value) == 2 + assert len(value[0]) == 6 + assert len(value[1]) == 10 + + def test_provider_timestamp_returns_bytes(provider): """ Assert that :meth:`~ulid.providers.default.Provider.timestamp` returns bytes of expected length. diff --git a/tests/test_providers_microsecond.py b/tests/test_providers_microsecond.py new file mode 100644 index 0000000..d082854 --- /dev/null +++ b/tests/test_providers_microsecond.py @@ -0,0 +1,115 @@ +""" + test_providers_microsecond + ~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Tests for the :mod:`~ulid.providers.microsecond` module. +""" +import pytest + +from ulid.providers import base, default, microsecond, time + + +@pytest.fixture(scope='function') +def provider(): + """ + Fixture that yields a microsecond provider instance. + """ + return microsecond.Provider(default.Provider()) + + +@pytest.fixture(scope='function') +def valid_epoch_milliseconds(): + """ + Fixture that yields a epoch value in milliseconds. + """ + return time.PROVIDER.milliseconds() + + +@pytest.fixture(scope='function') +def valid_epoch_microseconds(): + """ + Fixture that yields a epoch value in microseconds. + """ + return time.PROVIDER.microseconds() + + +@pytest.fixture(scope='function') +def mock_time_provider(mocker, valid_epoch_milliseconds, valid_epoch_microseconds): + """ + Fixture that yields a mock time provider. + """ + provider = mocker.Mock(spec=time.Provider) + provider.milliseconds = mocker.Mock(return_value=valid_epoch_milliseconds) + provider.microseconds = mocker.Mock(return_value=valid_epoch_microseconds) + return provider + + +@pytest.fixture(scope='function') +def provider_time_mock(mocker, mock_time_provider): + """ + Fixture that yields a provider with time mock. + """ + mocker.patch.object(microsecond.time, 'milliseconds', side_effect=mock_time_provider.milliseconds) + mocker.patch.object(microsecond.time, 'microseconds', side_effect=mock_time_provider.microseconds) + return microsecond.Provider(default.Provider()) + + +def test_provider_derives_from_base(): + """ + Assert that :class:`~ulid.providers.microsecond.Provider` derives from :class:`~ulid.providers.base.Provider`. + """ + assert issubclass(microsecond.Provider, base.Provider) + + +def test_provider_new_returns_bytes_pair(provider): + """ + Assert that :meth:`~ulid.providers.microsecond.Provider.new` returns timestamp and randomness + bytes of expected length as a two item tuple. + """ + value = provider.new() + assert isinstance(value, tuple) + assert len(value) == 2 + assert len(value[0]) == 6 + assert len(value[1]) == 10 + + +def test_provider_new_returns_randomness_with_microseconds(provider_time_mock): + """ + Assert that :meth:`~ulid.providers.default.Provider.new` returns timestamp and randomness + bytes that use microseconds as the first two bytes of randomness. + """ + epoch_us = time.microseconds() + epoch_ms = epoch_us // 1000 + microseconds = epoch_us % epoch_ms + + _, randomness = provider_time_mock.new() + + assert randomness[:2] == microseconds.to_bytes(2, byteorder='big') + + +def test_provider_timestamp_returns_bytes(provider): + """ + Assert that :meth:`~ulid.providers.microsecond.Provider.timestamp` returns bytes of expected length. + """ + value = provider.timestamp() + assert isinstance(value, bytes) + assert len(value) == 6 + + +def test_provider_timestamp_uses_time_epoch(provider): + """ + Assert that :meth:`~ulid.providers.microsecond.Provider.timestamp` returns the current time milliseconds + since epoch in bytes. + """ + timestamp_bytes = provider.timestamp() + timestamp_int = int.from_bytes(timestamp_bytes, byteorder='big') + assert timestamp_int <= time.milliseconds() + + +def test_provider_randomness_returns_bytes(provider): + """ + Assert that :meth:`~ulid.providers.microsecond.Provider.randomness` returns bytes of expected length. + """ + value = provider.randomness(provider.timestamp()) + assert isinstance(value, bytes) + assert len(value) == 10 diff --git a/tests/test_providers_monotonic.py b/tests/test_providers_monotonic.py index 474e5e9..a5845b0 100644 --- a/tests/test_providers_monotonic.py +++ b/tests/test_providers_monotonic.py @@ -28,6 +28,18 @@ def test_provider_derives_from_base(): assert issubclass(monotonic.Provider, base.Provider) +def test_provider_new_returns_bytes_pair(provider): + """ + Assert that :meth:`~ulid.providers.monotonic.Provider.new` returns timestamp and randomness + bytes of expected length as a two item tuple. + """ + value = provider.new() + assert isinstance(value, tuple) + assert len(value) == 2 + assert len(value[0]) == 6 + assert len(value[1]) == 10 + + def test_provider_timestamp_returns_bytes(provider): """ Assert that :meth:`~ulid.providers.monotonic.Provider.timestamp` returns bytes of expected length. diff --git a/tests/test_providers_package.py b/tests/test_providers_package.py index 0b79f55..2836b33 100644 --- a/tests/test_providers_package.py +++ b/tests/test_providers_package.py @@ -20,7 +20,7 @@ def test_package_exposes_expected_interface(): """ Assert that :attr:`~ulid.providers.__all__` exposes expected interface. """ - assert providers.__all__ == ['Provider', 'DEFAULT', 'MONOTONIC'] + assert providers.__all__ == ['Provider', 'DEFAULT', 'MICROSECOND', 'MONOTONIC'] def test_package_has_default_provider(): diff --git a/tests/test_providers_time_default.py b/tests/test_providers_time_default.py index 7c64073..d692cbf 100644 --- a/tests/test_providers_time_default.py +++ b/tests/test_providers_time_default.py @@ -48,7 +48,9 @@ def test_provider_milliseconds_is_unix_epoch(provider): since epoch. """ x = int(time.time() * 1000) + time.sleep(1) y = provider.milliseconds() + time.sleep(1) z = int(time.time() * 1000) assert x <= y <= z @@ -60,7 +62,9 @@ def test_provider_microseconds_is_unix_epoch(provider): since epoch. """ x = int(time.time() * 1000 * 1000) + time.sleep(1) y = provider.microseconds() + time.sleep(1) z = int(time.time() * 1000 * 1000) assert x <= y <= z diff --git a/tests/test_providers_time_nanosecond.py b/tests/test_providers_time_nanosecond.py index 844fc8b..5a51d1a 100644 --- a/tests/test_providers_time_nanosecond.py +++ b/tests/test_providers_time_nanosecond.py @@ -52,7 +52,9 @@ def test_provider_milliseconds_is_unix_epoch(provider): since epoch. """ x = int(time.time() * 1000) + time.sleep(1) y = provider.milliseconds() + time.sleep(1) z = int(time.time() * 1000) assert x <= y <= z @@ -64,7 +66,9 @@ def test_provider_microseconds_is_unix_epoch(provider): since epoch. """ x = int(time.time() * 1000 * 1000) + time.sleep(1) y = provider.microseconds() + time.sleep(1) z = int(time.time() * 1000 * 1000) assert x <= y <= z diff --git a/ulid/__init__.py b/ulid/__init__.py index e6edc9f..252b4c4 100644 --- a/ulid/__init__.py +++ b/ulid/__init__.py @@ -7,7 +7,7 @@ :copyright: (c) 2017 Andrew Hawker. :license: Apache 2.0, see LICENSE for more details. """ -from .api import default, monotonic +from .api import default, microsecond, monotonic create = default.create from_bytes = default.from_bytes diff --git a/ulid/api/api.py b/ulid/api/api.py index 43e7b44..8bfc8c0 100644 --- a/ulid/api/api.py +++ b/ulid/api/api.py @@ -58,8 +58,7 @@ def new(self) -> ulid.ULID: :return: ULID from current timestamp :rtype: :class:`~ulid.ulid.ULID` """ - timestamp = self.provider.timestamp() - randomness = self.provider.randomness(timestamp) + timestamp, randomness = self.provider.new() return ulid.ULID(timestamp + randomness) def parse(self, value: ULIDPrimitive) -> ulid.ULID: diff --git a/ulid/api/default.py b/ulid/api/default.py index 8dbabe8..698585f 100644 --- a/ulid/api/default.py +++ b/ulid/api/default.py @@ -2,7 +2,7 @@ ulid/api/default ~~~~~~~~~~~~~~~~ - Defaults the public API of the `ulid` package using the default provider. + Contains the public API of the `ulid` package using the default provider. """ from .. import consts, providers, ulid from . import api diff --git a/ulid/api/microsecond.py b/ulid/api/microsecond.py new file mode 100644 index 0000000..a8afe02 --- /dev/null +++ b/ulid/api/microsecond.py @@ -0,0 +1,33 @@ +""" + ulid/api/microsecond + ~~~~~~~~~~~~~~~~~~~~ + + Contains the public API of the `ulid` package using the microsecond provider. +""" +from .. import consts, providers, ulid +from . import api + +API = api.Api(providers.MICROSECOND) + +create = API.create +from_bytes = API.from_bytes +from_int = API.from_int +from_randomness = API.from_randomness +from_str = API.from_str +from_timestamp = API.from_timestamp +from_uuid = API.from_uuid +new = API.new +parse = API.parse + +MIN_TIMESTAMP = consts.MIN_TIMESTAMP +MAX_TIMESTAMP = consts.MAX_TIMESTAMP +MIN_RANDOMNESS = consts.MIN_RANDOMNESS +MAX_RANDOMNESS = consts.MAX_RANDOMNESS +MIN_ULID = consts.MIN_ULID +MAX_ULID = consts.MAX_ULID + +Timestamp = ulid.Timestamp +Randomness = ulid.Randomness +ULID = ulid.ULID + +__all__ = api.ALL diff --git a/ulid/api/monotonic.py b/ulid/api/monotonic.py index d4a17cc..2734933 100644 --- a/ulid/api/monotonic.py +++ b/ulid/api/monotonic.py @@ -2,7 +2,7 @@ ulid/api/monotonic ~~~~~~~~~~~~~~~~~~ - Defaults the public API of the `ulid` package using a monotonic randomness provider. + Contains the public API of the `ulid` package using the monotonic provider. """ from .. import consts, providers, ulid from . import api diff --git a/ulid/providers/__init__.py b/ulid/providers/__init__.py index 33819ea..49f1d2c 100644 --- a/ulid/providers/__init__.py +++ b/ulid/providers/__init__.py @@ -5,10 +5,11 @@ Contains functionality for timestamp/randomness data providers. """ -from . import base, default, monotonic +from . import base, default, microsecond, monotonic Provider = base.Provider DEFAULT = default.Provider() +MICROSECOND = microsecond.Provider(DEFAULT) MONOTONIC = monotonic.Provider(DEFAULT) -__all__ = ['Provider', 'DEFAULT', 'MONOTONIC'] +__all__ = ['Provider', 'DEFAULT', 'MICROSECOND', 'MONOTONIC'] diff --git a/ulid/providers/base.py b/ulid/providers/base.py index 9e16618..7fe6758 100644 --- a/ulid/providers/base.py +++ b/ulid/providers/base.py @@ -5,14 +5,28 @@ Contains provider abstract classes. """ import abc +import typing from .. import hints +#: Type hint that defines a two item tuple of bytes returned by the provider. +TimestampRandomnessBytes = typing.Tuple[hints.Bytes, hints.Bytes] # pylint: disable=invalid-name + class Provider(metaclass=abc.ABCMeta): """ Abstract class that defines providers that yield timestamp and randomness values. """ + def new(self) -> TimestampRandomnessBytes: + """ + Create a new timestamp and randomness value. + + :return: Two item tuple containing timestamp and randomness values as :class:`~bytes`. + :rtype: :class:`~tuple` + """ + timestamp = self.timestamp() + randomness = self.randomness(timestamp) + return timestamp, randomness @abc.abstractmethod def timestamp(self) -> hints.Bytes: diff --git a/ulid/providers/microsecond.py b/ulid/providers/microsecond.py new file mode 100644 index 0000000..e05423a --- /dev/null +++ b/ulid/providers/microsecond.py @@ -0,0 +1,54 @@ +""" + ulid/providers/microsecond + ~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Contains data provider that uses precise timestamp and most significant randomness bits. +""" +import os + +from .. import hints +from . import base, time + + +class Provider(base.Provider): + """ + Provider that uses microsecond values for timestamp and most significant randomness bits. + """ + def __init__(self, default: base.Provider): + self.default = default + + def new(self) -> base.TimestampRandomnessBytes: + """ + Create a new timestamp and randomness value. + + :return: Two item tuple containing timestamp and randomness values as :class:`~bytes`. + :rtype: :class:`~tuple` + """ + epoch_us = time.microseconds() + epoch_ms = epoch_us // 1000 + microseconds = epoch_us % epoch_ms + + timestamp = epoch_ms.to_bytes(6, byteorder='big') + randomness = microseconds.to_bytes(2, byteorder='big') + os.urandom(8) + + return timestamp, randomness + + def timestamp(self) -> hints.Bytes: + """ + Create a new timestamp value. + + :return: Timestamp value in bytes. + :rtype: :class:`~bytes` + """ + return self.default.timestamp() + + def randomness(self, timestamp: hints.Bytes) -> hints.Bytes: + """ + Create a new randomness value. + + :param timestamp: Timestamp in milliseconds + :type timestamp: :class:`~bytes` + :return: Randomness value in bytes. + :rtype: :class:`~bytes` + """ + return self.default.randomness(timestamp)