From 1a95f97bfb7966ac0412ebee01a42475db121fd7 Mon Sep 17 00:00:00 2001 From: LaunchDarklyReleaseBot <86431345+LaunchDarklyReleaseBot@users.noreply.github.com> Date: Fri, 3 Dec 2021 19:46:15 -0500 Subject: [PATCH] prepare 7.2.1 release (#160) --- .circleci/config.yml | 42 +++-- .gitignore | 1 + .ldrelease/config.yml | 14 +- CHANGELOG.md | 6 +- CONTRIBUTING.md | 2 +- README.md | 8 +- docs/index.rst | 2 +- ldclient/client.py | 2 +- ldclient/flags_state.py | 2 +- ldclient/impl/sse.py | 191 ++++++++++++++++++++++ ldclient/integrations.py | 10 +- ldclient/sse_client.py | 22 ++- ldclient/streaming.py | 7 +- ldclient/util.py | 2 +- setup.py | 3 + sse-contract-tests/Makefile | 27 +++ sse-contract-tests/README.md | 5 + sse-contract-tests/requirements.txt | 2 + sse-contract-tests/service.py | 91 +++++++++++ sse-contract-tests/stream_entity.py | 99 +++++++++++ test-filesource-optional-requirements.txt | 2 +- testing/impl/__init__.py | 0 testing/impl/test_sse.py | 89 ++++++++++ 23 files changed, 584 insertions(+), 45 deletions(-) create mode 100644 ldclient/impl/sse.py create mode 100644 sse-contract-tests/Makefile create mode 100644 sse-contract-tests/README.md create mode 100644 sse-contract-tests/requirements.txt create mode 100644 sse-contract-tests/service.py create mode 100644 sse-contract-tests/stream_entity.py create mode 100644 testing/impl/__init__.py create mode 100644 testing/impl/test_sse.py diff --git a/.circleci/config.yml b/.circleci/config.yml index 4153459f..8aea6976 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,19 +8,23 @@ workflows: jobs: - test-linux: name: Python 3.5 - docker-image: circleci/python:3.5-jessie + docker-image: cimg/python:3.5 + skip-sse-contract-tests: true # the test service app has dependencies that aren't available in 3.5, which is EOL anyway - test-linux: name: Python 3.6 - docker-image: circleci/python:3.6-jessie + docker-image: cimg/python:3.6 - test-linux: name: Python 3.7 - docker-image: circleci/python:3.7-stretch + docker-image: cimg/python:3.7 - test-linux: name: Python 3.8 - docker-image: circleci/python:3.8-buster + docker-image: cimg/python:3.8 - test-linux: name: Python 3.9 - docker-image: circleci/python:3.9-rc-buster + docker-image: cimg/python:3.9 + - test-linux: + name: Python 3.10 + docker-image: cimg/python:3.10 - test-windows: name: Windows Python 3 py3: true @@ -39,6 +43,9 @@ jobs: test-with-mypy: type: boolean default: true + skip-sse-contract-tests: + type: boolean + default: false docker: - image: <> - image: redis @@ -49,12 +56,10 @@ jobs: - run: name: install requirements command: | - sudo pip install --upgrade pip; - sudo pip install 'virtualenv~=16.0'; - sudo pip install -r test-requirements.txt; - sudo pip install -r test-filesource-optional-requirements.txt; - sudo pip install -r consul-requirements.txt; - sudo python setup.py install; + pip install -r test-requirements.txt; + pip install -r test-filesource-optional-requirements.txt; + pip install -r consul-requirements.txt; + python setup.py install; pip freeze - when: condition: <> @@ -89,6 +94,21 @@ jobs: command: | export PATH="/home/circleci/.local/bin:$PATH" mypy --config-file mypy.ini ldclient testing + + - unless: + condition: <> + steps: + - run: + name: build SSE contract test service + command: cd sse-contract-tests && make build-test-service + - run: + name: start SSE contract test service + command: cd sse-contract-tests && make start-test-service + background: true + - run: + name: run SSE contract tests + command: cd sse-contract-tests && make run-contract-tests + - store_test_results: path: test-reports - store_artifacts: diff --git a/.gitignore b/.gitignore index f0def2a6..291d3e29 100644 --- a/.gitignore +++ b/.gitignore @@ -69,3 +69,4 @@ p2venv test-packaging-venv .vscode/ +.python-version diff --git a/.ldrelease/config.yml b/.ldrelease/config.yml index 5615e7d2..b7db59ad 100644 --- a/.ldrelease/config.yml +++ b/.ldrelease/config.yml @@ -1,3 +1,5 @@ +version: 2 + repo: public: python-server-sdk private: python-server-sdk-private @@ -8,15 +10,17 @@ publications: - url: https://launchdarkly-python-sdk.readthedocs.io/en/latest/ description: documentation (readthedocs.io) -releasableBranches: +branches: - name: master description: 7.x - name: 6.x -template: - name: python - env: - LD_SKIP_DATABASE_TESTS: 1 +jobs: + - docker: {} + template: + name: python + env: + LD_SKIP_DATABASE_TESTS: 1 sdk: displayName: "Python" diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b43cb59..f24c6b9c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -168,11 +168,11 @@ Note that starting with this release, generated API documentation is available o ## [6.8.0] - 2019-01-31 ### Added: -- It is now possible to use Consul as a persistent feature store, similar to the existing Redis and DynamoDB integrations. See `Consul` in `ldclient.integrations`, and the reference guide for ["Using a persistent feature store"](https://docs.launchdarkly.com/v2.0/docs/using-a-persistent-feature-store). +- It is now possible to use Consul as a persistent feature store, similar to the existing Redis and DynamoDB integrations. See `Consul` in `ldclient.integrations`, and the reference guide for ["Storing data"](https://docs.launchdarkly.com/sdk/features/storing-data#python). ## [6.7.0] - 2019-01-15 ### Added: -- It is now possible to use DynamoDB as a persistent feature store, similar to the existing Redis integration. See `DynamoDB` in `ldclient.integrations`, and the reference guide to ["Using a persistent feature store"](https://docs.launchdarkly.com/v2.0/docs/using-a-persistent-feature-store). +- It is now possible to use DynamoDB as a persistent feature store, similar to the existing Redis integration. See `DynamoDB` in `ldclient.integrations`, and the reference guide to ["Storing data"](https://docs.launchdarkly.com/sdk/features/storing-data#python). - The new class `CacheConfig` (in `ldclient.feature_store`) encapsulates all the parameters that control local caching in database feature stores. This takes the place of the `expiration` and `capacity` parameters that are in the deprecated `RedisFeatureStore` constructor; it can be used with DynamoDB and any other database integrations in the future, and if more caching options are added to `CacheConfig` they will be automatically supported in all of the feature stores. ### Deprecated: @@ -261,7 +261,7 @@ _This release was broken and has been removed._ ## [6.0.0] - 2018-05-10 ### Changed: -- To reduce the network bandwidth used for analytics events, feature request events are now sent as counters rather than individual events, and user details are now sent only at intervals rather than in each event. These behaviors can be modified through the LaunchDarkly UI and with the new configuration option `inline_users_in_events`. For more details, see [Analytics Data Stream Reference](https://docs.launchdarkly.com/v2.0/docs/analytics-data-stream-reference). +- To reduce the network bandwidth used for analytics events, feature request events are now sent as counters rather than individual events, and user details are now sent only at intervals rather than in each event. These behaviors can be modified through the LaunchDarkly UI and with the new configuration option `inline_users_in_events`. - The analytics event processor now flushes events at a configurable interval defaulting to 5 seconds, like the other SDKs (previously it flushed if no events had been posted for 5 seconds, or if events exceeded a configurable number). This interval is set by the new `Config` property `flush_interval`. ### Removed: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7d2a9b8a..32425905 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,6 +1,6 @@ # Contributing to the LaunchDarkly Server-side SDK for Python -LaunchDarkly has published an [SDK contributor's guide](https://docs.launchdarkly.com/docs/sdk-contributors-guide) that provides a detailed explanation of how our SDKs work. See below for additional information on how to contribute to this SDK. +LaunchDarkly has published an [SDK contributor's guide](https://docs.launchdarkly.com/sdk/concepts/contributors-guide) that provides a detailed explanation of how our SDKs work. See below for additional information on how to contribute to this SDK. ## Submitting bug reports and feature requests diff --git a/README.md b/README.md index 5782eff1..8ea3a283 100644 --- a/README.md +++ b/README.md @@ -7,17 +7,17 @@ ## LaunchDarkly overview -[LaunchDarkly](https://www.launchdarkly.com) is a feature management platform that serves over 100 billion feature flags daily to help teams build better software, faster. [Get started](https://docs.launchdarkly.com/docs/getting-started) using LaunchDarkly today! +[LaunchDarkly](https://www.launchdarkly.com) is a feature management platform that serves over 100 billion feature flags daily to help teams build better software, faster. [Get started](https://docs.launchdarkly.com/home/getting-started) using LaunchDarkly today! [![Twitter Follow](https://img.shields.io/twitter/follow/launchdarkly.svg?style=social&label=Follow&maxAge=2592000)](https://twitter.com/intent/follow?screen_name=launchdarkly) ## Supported Python versions -This version of the LaunchDarkly SDK is compatible with Python 3.5 through 3.9. It is tested with the most recent patch releases of those versions. Python versions 2.7 to 3.4 are no longer supported. +This version of the LaunchDarkly SDK is compatible with Python 3.5 through 3.10. It is tested with the most recent patch releases of those versions. Python versions 2.7 to 3.4 are no longer supported. ## Getting started -Refer to the [SDK reference guide](https://docs.launchdarkly.com/docs/python-sdk-reference) for instructions on getting started with using the SDK. +Refer to the [SDK reference guide](https://docs.launchdarkly.com/sdk/server-side/python) for instructions on getting started with using the SDK. ## Learn more @@ -40,7 +40,7 @@ We encourage pull requests and other contributions from the community. Check out * Gradually roll out a feature to an increasing percentage of users, and track the effect that the feature has on key metrics (for instance, how likely is a user to complete a purchase if they have feature A versus feature B?). * Turn off a feature that you realize is causing performance problems in production, without needing to re-deploy, or even restart the application with a changed configuration file. * Grant access to certain features based on user attributes, like payment plan (eg: users on the ‘gold’ plan get access to more features than users in the ‘silver’ plan). Disable parts of your application to facilitate maintenance, without taking everything offline. -* LaunchDarkly provides feature flag SDKs for a wide variety of languages and technologies. Check out [our documentation](https://docs.launchdarkly.com/docs) for a complete list. +* LaunchDarkly provides feature flag SDKs for a wide variety of languages and technologies. Read [our documentation](https://docs.launchdarkly.com/sdk) for a complete list. * Explore LaunchDarkly * [launchdarkly.com](https://www.launchdarkly.com/ "LaunchDarkly Main Website") for more information * [docs.launchdarkly.com](https://docs.launchdarkly.com/ "LaunchDarkly Documentation") for our documentation and SDK reference guides diff --git a/docs/index.rst b/docs/index.rst index 1be4daca..12e66506 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -10,7 +10,7 @@ This is the API reference for the `LaunchDarkly `_ SD The latest version of the SDK can be found on `PyPI `_, and the source code is on `GitHub `_. -For more information, see LaunchDarkly's `Quickstart `_ and `SDK Reference Guide `_. +For more information, see LaunchDarkly's `Quickstart `_ and `SDK Reference Guide `_. .. toctree:: :maxdepth: 2 diff --git a/ldclient/client.py b/ldclient/client.py index d401df39..330e0f29 100644 --- a/ldclient/client.py +++ b/ldclient/client.py @@ -331,7 +331,7 @@ def all_flags_state(self, user: dict, **kwargs) -> FeatureFlagsState: """Returns an object that encapsulates the state of all feature flags for a given user, including the flag values and also metadata that can be used on the front end. See the JavaScript SDK Reference Guide on - `Bootstrapping `_. + `Bootstrapping `_. This method does not send analytics events back to LaunchDarkly. diff --git a/ldclient/flags_state.py b/ldclient/flags_state.py index 547a5d16..0bb0dbd0 100644 --- a/ldclient/flags_state.py +++ b/ldclient/flags_state.py @@ -12,7 +12,7 @@ class FeatureFlagsState: calling the :func:`ldclient.client.LDClient.all_flags_state()` method. Serializing this object to JSON, using the :func:`to_json_dict` method or ``jsonpickle``, will produce the appropriate data structure for bootstrapping the LaunchDarkly JavaScript client. See the - JavaScript SDK Reference Guide on `Bootstrapping `_. + JavaScript SDK Reference Guide on `Bootstrapping `_. """ def __init__(self, valid: bool): self.__flag_values = {} # type: Dict[str, Any] diff --git a/ldclient/impl/sse.py b/ldclient/impl/sse.py new file mode 100644 index 00000000..5a867096 --- /dev/null +++ b/ldclient/impl/sse.py @@ -0,0 +1,191 @@ +import urllib3 + +from ldclient.config import HTTPConfig +from ldclient.impl.http import HTTPFactory +from ldclient.util import throw_if_unsuccessful_response + + +class _BufferedLineReader: + """ + Helper class that encapsulates the logic for reading UTF-8 stream data as a series of text lines, + each of which can be terminated by \n, \r, or \r\n. + """ + def lines_from(chunks): + """ + Takes an iterable series of encoded chunks (each of "bytes" type) and parses it into an iterable + series of strings, each of which is one line of text. The line does not include the terminator. + """ + last_char_was_cr = False + partial_line = None + + for chunk in chunks: + if len(chunk) == 0: + continue + + # bytes.splitlines() will correctly break lines at \n, \r, or \r\n, and is faster than + # iterating through the characters in Python code. However, we have to adjust the results + # in several ways as described below. + lines = chunk.splitlines() + if last_char_was_cr: + last_char_was_cr = False + if chunk[0] == 10: + # If the last character we saw was \r, and then the first character in buf is \n, then + # that's just a single \r\n terminator, so we should remove the extra blank line that + # splitlines added for that first \n. + lines.pop(0) + if len(lines) == 0: + continue # ran out of data, continue to get next chunk + if partial_line is not None: + # On our last time through the loop, we ended up with an unterminated line, so we should + # treat our first parsed line here as a continuation of that. + lines[0] = partial_line + lines[0] + partial_line = None + # Check whether the buffer really ended in a terminator. If it did not, then the last line in + # lines is a partial line and should not be emitted yet. + last_char = chunk[len(chunk)-1] + if last_char == 13: + last_char_was_cr = True # remember this in case the next chunk starts with \n + elif last_char != 10: + partial_line = lines.pop() # remove last element which is the partial line + for line in lines: + yield line.decode() + + +class Event: + """ + An event received by SSEClient. + """ + def __init__(self, event='message', data='', last_event_id=None): + self._event = event + self._data = data + self._id = last_event_id + + @property + def event(self): + """ + The event type, or "message" if not specified. + """ + return self._event + + @property + def data(self): + """ + The event data. + """ + return self._data + + @property + def last_event_id(self): + """ + The last non-empty "id" value received from this stream so far. + """ + return self._id + + def dump(self): + lines = [] + if self.id: + lines.append('id: %s' % self.id) + + # Only include an event line if it's not the default already. + if self.event != 'message': + lines.append('event: %s' % self.event) + + lines.extend('data: %s' % d for d in self.data.split('\n')) + return '\n'.join(lines) + '\n\n' + + +class SSEClient: + """ + A simple Server-Sent Events client. + + This implementation does not include automatic retrying of a dropped connection; the caller will do that. + If a connection ends, the events iterator will simply end. + """ + def __init__(self, url, last_id=None, http_factory=None, **kwargs): + self.url = url + self.last_id = last_id + self._chunk_size = 10000 + + if http_factory is None: + http_factory = HTTPFactory({}, HTTPConfig()) + self._timeout = http_factory.timeout + base_headers = http_factory.base_headers + + self.http = http_factory.create_pool_manager(1, url) + + # Any extra kwargs will be fed into the request call later. + self.requests_kwargs = kwargs + + # The SSE spec requires making requests with Cache-Control: nocache + if 'headers' not in self.requests_kwargs: + self.requests_kwargs['headers'] = {} + + self.requests_kwargs['headers'].update(base_headers) + + self.requests_kwargs['headers']['Cache-Control'] = 'no-cache' + + # The 'Accept' header is not required, but explicit > implicit + self.requests_kwargs['headers']['Accept'] = 'text/event-stream' + + self._connect() + + def _connect(self): + if self.last_id: + self.requests_kwargs['headers']['Last-Event-ID'] = self.last_id + + # Use session if set. Otherwise fall back to requests module. + self.resp = self.http.request( + 'GET', + self.url, + timeout=self._timeout, + preload_content=False, + retries=0, # caller is responsible for implementing appropriate retry semantics, e.g. backoff + **self.requests_kwargs) + + # Raw readlines doesn't work because we may be missing newline characters until the next chunk + # For some reason, we also need to specify a chunk size because stream=True doesn't seem to guarantee + # that we get the newlines in a timeline manner + self.resp_file = self.resp.stream(amt=self._chunk_size) + + # TODO: Ensure we're handling redirects. Might also stick the 'origin' + # attribute on Events like the Javascript spec requires. + throw_if_unsuccessful_response(self.resp) + + @property + def events(self): + """ + An iterable series of Event objects received from the stream. + """ + event_type = "" + event_data = None + for line in _BufferedLineReader.lines_from(self.resp_file): + if line == "": + if event_data is not None: + yield Event("message" if event_type == "" else event_type, event_data, self.last_id) + event_type = "" + event_data = None + continue + colon_pos = line.find(':') + if colon_pos < 0: + continue # malformed line - ignore + if colon_pos == 0: + continue # comment - currently we're not surfacing these + name = line[0:colon_pos] + if colon_pos < (len(line) - 1) and line[colon_pos + 1] == ' ': + colon_pos += 1 + value = line[colon_pos+1:] + if name == 'event': + event_type = value + elif name == 'data': + event_data = value if event_data is None else (event_data + "\n" + value) + elif name == 'id': + self.last_id = value + elif name == 'retry': + pass # auto-reconnect is not implemented in this simplified client + # unknown field names are ignored in SSE + + def __enter__(self): + return self + + def __exit__(self, type, value, traceback): + self.close() diff --git a/ldclient/integrations.py b/ldclient/integrations.py index e0f0050c..550f0177 100644 --- a/ldclient/integrations.py +++ b/ldclient/integrations.py @@ -27,7 +27,7 @@ def new_feature_store(host: str=None, caching: CacheConfig=CacheConfig.default()) -> CachingStoreWrapper: """Creates a Consul-backed implementation of :class:`ldclient.interfaces.FeatureStore`. For more details about how and why you can use a persistent feature store, see the - `SDK reference guide `_. + `SDK reference guide `_. To use this method, you must first install the ``python-consul`` package. Then, put the object returned by this method into the ``feature_store`` property of your client configuration @@ -65,7 +65,7 @@ def new_feature_store(table_name: str, caching: CacheConfig=CacheConfig.default()) -> CachingStoreWrapper: """Creates a DynamoDB-backed implementation of :class:`ldclient.interfaces.FeatureStore`. For more details about how and why you can use a persistent feature store, see the - `SDK reference guide `_. + `SDK reference guide `_. To use this method, you must first install the ``boto3`` package containing the AWS SDK gems. Then, put the object returned by this method into the ``feature_store`` property of your @@ -110,7 +110,7 @@ def new_feature_store(url: str='redis://localhost:6379/0', caching: CacheConfig=CacheConfig.default()) -> CachingStoreWrapper: """Creates a Redis-backed implementation of :class:`ldclient.interfaces.FeatureStore`. For more details about how and why you can use a persistent feature store, see the - `SDK reference guide `_. + `SDK reference guide `_. To use this method, you must first install the ``redis`` package. Then, put the object returned by this method into the ``feature_store`` property of your client configuration @@ -161,8 +161,8 @@ def new_data_source(paths: List[str], client may still make network connections to send analytics events, unless you have disabled this in your configuration with ``send_events`` or ``offline``. - The format of the data files is described in the SDK Reference Guide on - `Reading flags from a file `_. + The format of the data files is described in the SDK Reference Guide on + `Reading flags from a file `_. Note that in order to use YAML, you will need to install the ``pyyaml`` package. If the data source encounters any error in any file-- malformed content, a missing file, or a diff --git a/ldclient/sse_client.py b/ldclient/sse_client.py index e1531f8c..80dea242 100644 --- a/ldclient/sse_client.py +++ b/ldclient/sse_client.py @@ -1,10 +1,14 @@ -""" -Server-Sent Events implementation for streaming. - -Based on: https://bitbucket.org/btubbs/sseclient/src/a47a380a3d7182a205c0f1d5eb470013ce796b4d/sseclient.py?at=default&fileviewer=file-view-default -""" -# currently excluded from documentation - see docs/README.md - +# +# This deprecated implementation was based on: +# https://bitbucket.org/btubbs/sseclient/src/a47a380a3d7182a205c0f1d5eb470013ce796b4d/sseclient.py?at=default&fileviewer=file-view-default +# +# It has the following known issues: +# - It does not properly handle line terminators other than \n. +# - It does not properly handle multi-line data that starts with a blank line. +# - It fails if a multi-byte character is split across chunks of the stream. +# +# It is replaced by the ldclient.impl.sse module. +# import re import time @@ -21,6 +25,10 @@ class SSEClient: + """ + This class is deprecated and no longer used in the SDK. It is retained here for backward compatibility in case + any external code was referencing it, but it will be removed in a future major version. + """ def __init__(self, url, last_id=None, retry=3000, connect_timeout=10, read_timeout=300, chunk_size=10000, verify_ssl=False, http=None, http_proxy=None, http_factory=None, **kwargs): self.url = url diff --git a/ldclient/streaming.py b/ldclient/streaming.py index 061bca65..2255b419 100644 --- a/ldclient/streaming.py +++ b/ldclient/streaming.py @@ -9,13 +9,12 @@ from threading import Thread import logging -import math import time from ldclient.impl.http import HTTPFactory, _http_factory from ldclient.impl.retry_delay import RetryDelayStrategy, DefaultBackoffStrategy, DefaultJitterStrategy +from ldclient.impl.sse import SSEClient from ldclient.interfaces import UpdateProcessor -from ldclient.sse_client import SSEClient from ldclient.util import log, UnsuccessfulResponseException, http_error_message, is_http_error_recoverable from ldclient.versioned_data_kind import FEATURES, SEGMENTS @@ -106,11 +105,11 @@ def _connect(self): # We don't want the stream to use the same read timeout as the rest of the SDK. http_factory = _http_factory(self._config) stream_http_factory = HTTPFactory(http_factory.base_headers, http_factory.http_config, override_read_timeout=stream_read_timeout) - return SSEClient( + client = SSEClient( self._uri, - retry = None, # we're implementing our own retry http_factory = stream_http_factory ) + return client.events def stop(self): log.info("Stopping StreamingUpdateProcessor") diff --git a/ldclient/util.py b/ldclient/util.py index 2479fe67..66c0c70b 100644 --- a/ldclient/util.py +++ b/ldclient/util.py @@ -43,7 +43,7 @@ def check_uwsgi(): if uwsgi.opt.get('threads') is not None and int(uwsgi.opt.get('threads')) > 1: return log.error("The LaunchDarkly client requires the 'enable-threads' or 'threads' option be passed to uWSGI. " - 'To learn more, see https://docs.launchdarkly.com/sdk/server-side/python#configuring-uwsgi') + 'To learn more, read https://docs.launchdarkly.com/sdk/server-side/python#configuring-uwsgi') class Event: diff --git a/setup.py b/setup.py index 18ccade9..cf3312f8 100644 --- a/setup.py +++ b/setup.py @@ -66,6 +66,9 @@ def run(self): 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', 'Topic :: Software Development', 'Topic :: Software Development :: Libraries', ], diff --git a/sse-contract-tests/Makefile b/sse-contract-tests/Makefile new file mode 100644 index 00000000..37f69644 --- /dev/null +++ b/sse-contract-tests/Makefile @@ -0,0 +1,27 @@ + +TEMP_TEST_OUTPUT=/tmp/sse-contract-test-service.log + +# port 8000 is already used in the CI environment because we're running a DynamoDB container +PORT=9000 + +# we're skipping the "reconnection" test group because the simplified SSE client we're currently using +# does not do automatic retrying of connections - that is done at a higher level in the SDK +EXTRA_TEST_PARAMS=-skip reconnection + +build-test-service: + @pip install -r requirements.txt + +start-test-service: + @python service.py $(PORT) + +start-test-service-bg: + @echo "Test service output will be captured in $(TEMP_TEST_OUTPUT)" + @make start-test-service >$(TEMP_TEST_OUTPUT) 2>&1 & + +run-contract-tests: + @curl -s https://raw.githubusercontent.com/launchdarkly/sse-contract-tests/master/downloader/run.sh \ + | VERSION=v1 PARAMS="-url http://localhost:$(PORT) -debug -stop-service-at-end $(EXTRA_TEST_PARAMS)" sh + +contract-tests: build-test-service start-test-service-bg run-contract-tests + +.PHONY: build-test-service start-test-service start-test-service-bg run-contract-tests contract-tests diff --git a/sse-contract-tests/README.md b/sse-contract-tests/README.md new file mode 100644 index 00000000..f5892c91 --- /dev/null +++ b/sse-contract-tests/README.md @@ -0,0 +1,5 @@ +# SSE client contract test service + +This directory contains an implementation of the cross-platform SSE testing protocol defined by https://github.com/launchdarkly/sse-contract-tests. See that project's `README` for details of this protocol, and the kinds of SSE client capabilities that are relevant to the contract tests. This code should not need to be updated unless the SSE client has added or removed such capabilities. + +To run these tests locally, run `make contract-tests`. This downloads the correct version of the test harness tool automatically. diff --git a/sse-contract-tests/requirements.txt b/sse-contract-tests/requirements.txt new file mode 100644 index 00000000..2d1d2a7b --- /dev/null +++ b/sse-contract-tests/requirements.txt @@ -0,0 +1,2 @@ +Flask==2.0.2 +urllib3>=1.22.0 diff --git a/sse-contract-tests/service.py b/sse-contract-tests/service.py new file mode 100644 index 00000000..6d07fc59 --- /dev/null +++ b/sse-contract-tests/service.py @@ -0,0 +1,91 @@ +from stream_entity import StreamEntity + +import json +import logging +import os +import sys +import urllib3 +from flask import Flask, request +from flask.logging import default_handler +from logging.config import dictConfig + +default_port = 8000 + +# logging configuration +dictConfig({ + 'version': 1, + 'formatters': { + 'default': { + 'format': '[%(asctime)s] [%(name)s] %(levelname)s: %(message)s', + } + }, + 'handlers': { + 'console': { + 'class': 'logging.StreamHandler', + 'formatter': 'default' + } + }, + 'root': { + 'level': 'INFO', + 'handlers': ['console'] + }, + 'loggers': { + 'werkzeug': { 'level': 'ERROR' } # disable irrelevant Flask app logging + } +}) + +app = Flask(__name__) +app.logger.removeHandler(default_handler) + +stream_counter = 0 +streams = {} +global_log = logging.getLogger('testservice') + +http_client = urllib3.PoolManager() + +@app.route('/', methods=['GET']) +def status(): + body = { + 'capabilities': [ + 'headers', + 'last-event-id' + ] + } + return (json.dumps(body), 200, {'Content-type': 'application/json'}) + +@app.route('/', methods=['DELETE']) +def delete_stop_service(): + print("Test service has told us to exit") + quit() + +@app.route('/', methods=['POST']) +def post_create_stream(): + global stream_counter, streams + + options = json.loads(request.data) + + stream_counter += 1 + stream_id = str(stream_counter) + resource_url = '/streams/%s' % stream_id + + stream = StreamEntity(options) + streams[stream_id] = stream + + return ('', 201, {'Location': resource_url}) + +@app.route('/streams/', methods=['DELETE']) +def delete_stream(id): + global streams + + stream = streams[id] + if stream is None: + return ('', 404) + stream.close() + return ('', 204) + +if __name__ == "__main__": + port = default_port + if sys.argv[len(sys.argv) - 1] != 'service.py': + port = int(sys.argv[len(sys.argv) - 1]) + global_log.info('Listening on port %d', port) + app.run(host='0.0.0.0', port=port) diff --git a/sse-contract-tests/stream_entity.py b/sse-contract-tests/stream_entity.py new file mode 100644 index 00000000..ac5c7d00 --- /dev/null +++ b/sse-contract-tests/stream_entity.py @@ -0,0 +1,99 @@ +import json +import logging +import os +import sys +import threading +import traceback +import urllib3 + +# Import ldclient from parent directory +sys.path.insert(1, os.path.join(sys.path[0], '..')) +from ldclient.config import HTTPConfig +from ldclient.impl.http import HTTPFactory +from ldclient.impl.sse import SSEClient + +port = 8000 + +stream_counter = 0 +streams = {} + +http_client = urllib3.PoolManager() + +class StreamEntity: + def __init__(self, options): + self.options = options + self.callback_url = options["callbackUrl"] + self.log = logging.getLogger(options["tag"]) + self.closed = False + self.callback_counter = 0 + + thread = threading.Thread(target=self.run) + thread.start() + + def run(self): + stream_url = self.options["streamUrl"] + http_factory = HTTPFactory( + self.options.get("headers", {}), + HTTPConfig(read_timeout = + None if self.options.get("readTimeoutMs") is None else + self.options["readTimeoutMs"] / 1000) + ) + try: + self.log.info('Opening stream from %s', stream_url) + sse = SSEClient( + stream_url, + # Currently this client implementation does not support automatic retry + # retry = + # None if self.options.get("initialDelayMs") is None else + # self.options.get("initialDelayMs") / 1000, + last_id = self.options.get("lastEventId"), + http_factory = http_factory + ) + self.sse = sse + for message in sse.events: + self.log.info('Received event from stream (%s)', message.event) + self.send_message({ + 'kind': 'event', + 'event': { + 'type': message.event, + 'data': message.data, + 'id': message.last_event_id + } + }) + self.send_message({ + 'kind': 'error', + 'error': 'Stream closed' + }) + except Exception as e: + self.log.info('Received error from stream: %s', e) + self.log.info(traceback.format_exc()) + self.send_message({ + 'kind': 'error', + 'error': str(e) + }) + + def send_message(self, message): + global http_client + + if self.closed: + return + self.callback_counter += 1 + callback_url = "%s/%d" % (self.options["callbackUrl"], self.callback_counter) + + try: + resp = http_client.request( + 'POST', + callback_url, + headers = {'Content-Type': 'application/json'}, + body = json.dumps(message) + ) + if resp.status >= 300 and not self.closed: + self.log.error('Callback request returned HTTP error %d', resp.status) + except Exception as e: + if not self.closed: + self.log.error('Callback request failed: %s', e) + + def close(self): + # how to close the stream?? + self.closed = True + self.log.info('Test ended') diff --git a/test-filesource-optional-requirements.txt b/test-filesource-optional-requirements.txt index 3cfa747b..38bdc65b 100644 --- a/test-filesource-optional-requirements.txt +++ b/test-filesource-optional-requirements.txt @@ -1,2 +1,2 @@ pyyaml>=3.0,<5.2 -watchdog>=0.9,<1.0 +watchdog>=0.9,<1.0,!=0.10.5 diff --git a/testing/impl/__init__.py b/testing/impl/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/testing/impl/test_sse.py b/testing/impl/test_sse.py new file mode 100644 index 00000000..9e006531 --- /dev/null +++ b/testing/impl/test_sse.py @@ -0,0 +1,89 @@ +from ldclient.impl.sse import _BufferedLineReader, SSEClient + +from testing.http_util import ChunkedResponse, start_server + +import pytest + + +class TestBufferedLineReader: + @pytest.fixture(params = ["\r", "\n", "\r\n"]) + def terminator(self, request): + return request.param + + @pytest.fixture(params = [ + [ + [ "first line*", "second line*", "3rd line*" ], + [ "first line", "second line", "3rd line"] + ], + [ + [ "*", "second line*", "3rd line*" ], + [ "", "second line", "3rd line"] + ], + [ + [ "first line*", "*", "3rd line*" ], + [ "first line", "", "3rd line"] + ], + [ + [ "first line*", "*", "*", "*", "3rd line*" ], + [ "first line", "", "", "", "3rd line" ] + ], + [ + [ "first line*second line*third", " line*fourth line*"], + [ "first line", "second line", "third line", "fourth line" ] + ], + ]) + def inputs_outputs(self, terminator, request): + inputs = list(s.replace("*", terminator).encode() for s in request.param[0]) + return [inputs, request.param[1]] + + def test_parsing(self, inputs_outputs): + assert list(_BufferedLineReader.lines_from(inputs_outputs[0])) == inputs_outputs[1] + + def test_mixed_terminators(self): + chunks = [ + b"first line\nsecond line\r\nthird line\r", + b"\nfourth line\r", + b"\r\nlast\r\n" + ] + expected = [ + "first line", + "second line", + "third line", + "fourth line", + "", + "last" + ] + assert list(_BufferedLineReader.lines_from(chunks)) == expected + + +# The tests for SSEClient are fairly basic, just ensuring that it is really making HTTP requests and that the +# API works as expected. The contract test suite is much more thorough - see sse-contract-tests. + +class TestSSEClient: + def test_sends_expected_headers(self): + with start_server() as server: + with ChunkedResponse({ 'Content-Type': 'text/event-stream' }) as stream: + server.for_path('/', stream) + client = SSEClient(server.uri) + + r = server.await_request() + assert r.headers['Accept'] == 'text/event-stream' + assert r.headers['Cache-Control'] == 'no-cache' + + def test_receives_messages(self): + with start_server() as server: + with ChunkedResponse({ 'Content-Type': 'text/event-stream' }) as stream: + server.for_path('/', stream) + client = SSEClient(server.uri) + + stream.push("event: event1\ndata: data1\n\nevent: event2\ndata: data2\n\n") + + events = client.events + + event1 = next(events) + assert event1.event == 'event1' + assert event1.data == 'data1' + + event2 = next(events) + assert event2.event == 'event2' + assert event2.data == 'data2'