From 413a79d28b536904a493ec72a1c4efab1743d692 Mon Sep 17 00:00:00 2001 From: Mark Drobnak Date: Tue, 11 Aug 2020 15:43:00 -0400 Subject: [PATCH 01/13] Move tests from Travis to CircleCI --- .circleci/config.yml | 44 ++++++++++++++++ .travis.yml | 38 -------------- tests/test_integration.py | 82 ++++++++++++++++++++---------- tests/test_integration_all_rust.py | 16 +++--- 4 files changed, 109 insertions(+), 71 deletions(-) delete mode 100644 .travis.yml diff --git a/.circleci/config.yml b/.circleci/config.yml index 5054e01f5..c09ec6343 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -25,6 +25,43 @@ jobs: command: | # cargo audit echo audit temporarily disabled + + test: + docker: + - image: circleci/python:2.7 + - image: circleci/dynamodb + environment: + AWS_LOCAL_DYNAMODB: http://localhost:8000 + steps: + - checkout + - run: + name: Set up Python + command: | + curl https://raw.githubusercontent.com/mozilla-services/autopush/master/requirements.txt > requirements.txt + curl https://raw.githubusercontent.com/mozilla-services/autopush/master/test-requirements.txt > test-requirements.txt + pip install --upgrade pip + pip install bottle + pip install -r requirements.txt + pip install -r test-requirements.txt + pip install -e git+https://github.com/mozilla-services/autopush.git#egg=autopush + - run: + name: Set up Rust + command: | + curl https://sh.rustup.rs -sSf | sh -s -- -y + export PATH=$PATH:$HOME/.cargo/bin + echo 'export PATH=$PATH:$HOME/.cargo/bin' >> $BASH_ENV + rustc --version + cargo build + - run: + name: Check formatting + command: cargo fmt -- --check + - run: + name: Integration tests + command: py.test -v + - run: + name: Rust tests + command: cargo test + build: docker: - image: docker:18.03.0-ce @@ -98,6 +135,12 @@ workflows: filters: tags: only: /.*/ + + - test: + filters: + tags: + only: /.*/ + - build: filters: tags: @@ -106,6 +149,7 @@ workflows: - deploy: requires: - build + - test filters: tags: only: /.*/ diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 117cd2257..000000000 --- a/.travis.yml +++ /dev/null @@ -1,38 +0,0 @@ -language: python -python: - - "2.7" -cache: - pip: true - directories: - - $HOME/.cargo - - $TRAVIS_BUILD_DIR/target -dist: xenial -os: linux - -install: -- make ddb -- curl https://raw.githubusercontent.com/mozilla-services/autopush/master/requirements.txt > requirements.txt -- curl https://raw.githubusercontent.com/mozilla-services/autopush/master/test-requirements.txt > test-requirements.txt -- pip install --upgrade pip -- pip install bottle -- pip install -r requirements.txt -- pip install -r test-requirements.txt -- pip install -e git+https://github.com/mozilla-services/autopush.git#egg=autopush -- | - curl https://sh.rustup.rs | sh -s -- -y || travis_terminate 1; - export PATH=$PATH:$HOME/.cargo/bin - if [ "${WITH_RUST}" == "release" ]; then - cargo build --release || travis_terminate 1 - else - cargo build || travis_terminate 1 - fi - rustup component add rustfmt-preview - -script: -- cargo fmt -- --check -- py.test -v -- cargo test - -notifications: - slack: - secure: vT9sWtUuxk28g6xYKAsQmiPZllErOYVfx5lcL+/jo1eRFrmbpYnyndT6s+FxGI1547oizZ0IqZbHVvB7BUoSJixXJyQJYXW2MchwN1UeHrey8mYpF1GNEaJT7FMfqSkxUU9gvAZ3IU7zstNeTLbfG1GkLuzybp0WAiHl/ocUTz8= diff --git a/tests/test_integration.py b/tests/test_integration.py index 162e702fb..fbeea6654 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -19,8 +19,6 @@ from threading import Event, Thread from unittest import SkipTest -import autopush.tests -import autopush.tests as ap_tests import bottle import ecdsa import psutil @@ -28,7 +26,7 @@ import twisted.internet.base from autopush.config import AutopushConfig from autopush.db import (Message, get_month, has_connected_this_month, - make_rotating_tablename) + make_rotating_tablename, DynamoDBResource, create_rotating_message_table) from autopush.logging import begin_or_register from autopush.main import ( ConnectionApplication, @@ -42,10 +40,13 @@ from autopush.utils import base64url_encode from cryptography.fernet import Fernet from Queue import Empty, Queue + +from twisted.internet import reactor from twisted.internet.defer import inlineCallbacks, returnValue from twisted.internet.threads import deferToThread from twisted.logger import globalLogPublisher from twisted.trial import unittest +from typing import Optional app = bottle.Bottle() log = logging.getLogger(__name__) @@ -53,6 +54,11 @@ here_dir = os.path.abspath(os.path.dirname(__file__)) root_dir = os.path.dirname(here_dir) +DDB_JAR = os.path.join(root_dir, "ddb", "DynamoDBLocal.jar") +DDB_LIB_DIR = os.path.join(root_dir, "ddb", "DynamoDBLocal_lib") +DDB_PROCESS = None # type: Optional[subprocess.Popen] +BOTO_RESOURCE = None # type: DynamoDBResource + twisted.internet.base.DelayedCall.debug = True ROUTER_TABLE = os.environ.get("ROUTER_TABLE", "router_int_test") @@ -170,12 +176,44 @@ def send_bad_data(self): self.ws.send("bad-data") +def kill_process(process): + # This kinda sucks, but its the only way to nuke the child procs + proc = psutil.Process(pid=process.pid) + child_procs = proc.children(recursive=True) + for p in [proc] + child_procs: + os.kill(p.pid, signal.SIGTERM) + process.wait() + + +def setup_dynamodb(): + global DDB_PROCESS, BOTO_RESOURCE + + for name in ('boto3', 'botocore'): + logging.getLogger(name).setLevel(logging.CRITICAL) + + if os.getenv("AWS_LOCAL_DYNAMODB") is None: + cmd = " ".join([ + "java", "-Djava.library.path=%s" % DDB_LIB_DIR, + "-jar", DDB_JAR, "-sharedDb", "-inMemory" + ]) + DDB_PROCESS = subprocess.Popen(cmd, shell=True, env=os.environ) + os.environ["AWS_LOCAL_DYNAMODB"] = "http://127.0.0.1:8000" + + # Setup the necessary tables + BOTO_RESOURCE = DynamoDBResource() + message_table = os.environ.get("MESSAGE_TABLE", "message_int_test") + create_rotating_message_table(prefix=message_table, delta=-1, + boto_resource=BOTO_RESOURCE) + create_rotating_message_table(prefix=message_table, + boto_resource=BOTO_RESOURCE) + pool = reactor.getThreadPool() + pool.adjustPoolsize(minthreads=pool.max) + + def setup_module(): global CN_SERVER, CN_QUEUES, CN_MP_SERVER, MOCK_SERVER_THREAD, \ STRICT_LOG_COUNTS - ap_tests.ddb_jar = os.path.join(root_dir, "ddb", "DynamoDBLocal.jar") - ap_tests.ddb_lib_dir = os.path.join(root_dir, "ddb", "DynamoDBLocal_lib") - ap_tests.setUp() + setup_dynamodb() logging.getLogger('boto').setLevel(logging.CRITICAL) if "SKIP_INTEGRATION" in os.environ: # pragma: nocover raise SkipTest("Skipping integration tests") @@ -269,20 +307,12 @@ def setup_module(): def teardown_module(): - global CN_SERVER, CN_MP_SERVER - ap_tests.tearDown() - # This kinda sucks, but its the only way to nuke the child procs - proc = psutil.Process(pid=CN_SERVER.pid) - child_procs = proc.children(recursive=True) - for p in [proc] + child_procs: - os.kill(p.pid, signal.SIGTERM) - CN_SERVER.wait() + global CN_SERVER, CN_MP_SERVER, DDB_PROCESS - proc = psutil.Process(pid=CN_MP_SERVER.pid) - child_procs = proc.children(recursive=True) - for p in [proc] + child_procs: - os.kill(p.pid, signal.SIGTERM) - CN_MP_SERVER.wait() + if DDB_PROCESS: + kill_process(DDB_PROCESS) + kill_process(CN_SERVER) + kill_process(CN_MP_SERVER) class TestRustWebPush(unittest.TestCase): @@ -306,7 +336,7 @@ def start_ep(self, ep_conf): # Endpoint HTTP router self.ep = ep = EndpointApplication( ep_conf, - resource=autopush.tests.boto_resource + resource=BOTO_RESOURCE ) ep.setup(rotate_tables=False) ep.startService() @@ -1033,7 +1063,7 @@ def start_ep(self, ep_conf): # Endpoint HTTP router self.ep = ep = EndpointApplication( ep_conf, - resource=autopush.tests.boto_resource + resource=BOTO_RESOURCE ) ep.setup(rotate_tables=False) ep.startService() @@ -1058,7 +1088,7 @@ def setUp(self): ) self.conn = conn = ConnectionApplication( self._conn_conf, - resource=autopush.tests.boto_resource, + resource=BOTO_RESOURCE, ) conn.setup(rotate_tables=True) @@ -1327,7 +1357,7 @@ def test_webpush_monthly_rotation_prior_record_exists(self): last_month = make_rotating_tablename( prefix=self.conn.conf.message_table.tablename, delta=-1) lm_message = Message(last_month, - boto_resource=autopush.tests.boto_resource) + boto_resource=BOTO_RESOURCE) yield deferToThread( self.conn.db.router.update_message_month, client.uaid, @@ -1478,7 +1508,7 @@ def test_webpush_monthly_rotation_old_user_dropped(self): client.uaid, old_month ) - _ = Message(old_month, boto_resource=autopush.tests.boto_resource) + _ = Message(old_month, boto_resource=BOTO_RESOURCE) # Verify the move c = yield deferToThread(self.conn.db.router.get_uaid, client.uaid) @@ -1523,7 +1553,7 @@ def start_ep(self, ep_conf): # Endpoint HTTP router self.ep = ep = EndpointApplication( ep_conf, - resource=autopush.tests.boto_resource + resource=BOTO_RESOURCE ) ep.setup(rotate_tables=False) ep.startService() @@ -1534,7 +1564,7 @@ def start_conn(self, conn_conf): # the module global Rust one as well self.conn = conn = ConnectionApplication( conn_conf, - resource=autopush.tests.boto_resource, + resource=BOTO_RESOURCE, ) conn.setup(rotate_tables=False) conn.startService() diff --git a/tests/test_integration_all_rust.py b/tests/test_integration_all_rust.py index d10d8c3ae..7f99e7fb4 100644 --- a/tests/test_integration_all_rust.py +++ b/tests/test_integration_all_rust.py @@ -31,6 +31,7 @@ from twisted.internet import reactor from twisted.internet.defer import inlineCallbacks, returnValue from twisted.trial import unittest +from typing import Optional app = bottle.Bottle() logging.basicConfig(level=logging.DEBUG) @@ -41,7 +42,7 @@ DDB_JAR = os.path.join(root_dir, "ddb", "DynamoDBLocal.jar") DDB_LIB_DIR = os.path.join(root_dir, "ddb", "DynamoDBLocal_lib") -DDB_PROCESS = None # type: subprocess.Popen +DDB_PROCESS = None # type: Optional[subprocess.Popen] twisted.internet.base.DelayedCall.debug = True @@ -252,12 +253,12 @@ def capture_output_to_queue(output_stream): def setup_dynamodb(): global DDB_PROCESS - cmd = " ".join([ - "java", "-Djava.library.path=%s" % DDB_LIB_DIR, - "-jar", DDB_JAR, "-sharedDb", "-inMemory" - ]) - DDB_PROCESS = subprocess.Popen(cmd, shell=True, env=os.environ) if os.getenv("AWS_LOCAL_DYNAMODB") is None: + cmd = " ".join([ + "java", "-Djava.library.path=%s" % DDB_LIB_DIR, + "-jar", DDB_JAR, "-sharedDb", "-inMemory" + ]) + DDB_PROCESS = subprocess.Popen(cmd, shell=True, env=os.environ) os.environ["AWS_LOCAL_DYNAMODB"] = "http://127.0.0.1:8000" # Setup the necessary tables @@ -351,7 +352,8 @@ def setup_module(): def teardown_module(): - kill_process(DDB_PROCESS) + if DDB_PROCESS: + kill_process(DDB_PROCESS) kill_process(CN_SERVER) kill_process(CN_MP_SERVER) kill_process(EP_SERVER) From c1a917124e7f411c22628275032bc46fbed824a9 Mon Sep 17 00:00:00 2001 From: Mark Drobnak Date: Wed, 12 Aug 2020 09:42:08 -0400 Subject: [PATCH 02/13] Use a shared, in-memory DynamoDB instance when testing --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index c09ec6343..f4eca14da 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -30,6 +30,7 @@ jobs: docker: - image: circleci/python:2.7 - image: circleci/dynamodb + command: -sharedDb -inMemory environment: AWS_LOCAL_DYNAMODB: http://localhost:8000 steps: From 9d3a1002e1acc1e8c0de04d79e2293165ca9e9ea Mon Sep 17 00:00:00 2001 From: Mark Drobnak Date: Wed, 12 Aug 2020 10:12:50 -0400 Subject: [PATCH 03/13] Add more logging around DynamoDB setup in integration tests --- tests/test_integration.py | 3 +++ tests/test_integration_all_rust.py | 3 +++ 2 files changed, 6 insertions(+) diff --git a/tests/test_integration.py b/tests/test_integration.py index fbeea6654..fe370c034 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -192,12 +192,15 @@ def setup_dynamodb(): logging.getLogger(name).setLevel(logging.CRITICAL) if os.getenv("AWS_LOCAL_DYNAMODB") is None: + print "Starting new DynamoDB instance" cmd = " ".join([ "java", "-Djava.library.path=%s" % DDB_LIB_DIR, "-jar", DDB_JAR, "-sharedDb", "-inMemory" ]) DDB_PROCESS = subprocess.Popen(cmd, shell=True, env=os.environ) os.environ["AWS_LOCAL_DYNAMODB"] = "http://127.0.0.1:8000" + else: + print "Using existing DynamoDB instance" # Setup the necessary tables BOTO_RESOURCE = DynamoDBResource() diff --git a/tests/test_integration_all_rust.py b/tests/test_integration_all_rust.py index 7f99e7fb4..e51f8ebce 100644 --- a/tests/test_integration_all_rust.py +++ b/tests/test_integration_all_rust.py @@ -254,12 +254,15 @@ def setup_dynamodb(): global DDB_PROCESS if os.getenv("AWS_LOCAL_DYNAMODB") is None: + print "Starting new DynamoDB instance" cmd = " ".join([ "java", "-Djava.library.path=%s" % DDB_LIB_DIR, "-jar", DDB_JAR, "-sharedDb", "-inMemory" ]) DDB_PROCESS = subprocess.Popen(cmd, shell=True, env=os.environ) os.environ["AWS_LOCAL_DYNAMODB"] = "http://127.0.0.1:8000" + else: + print "Using existing DynamoDB instance" # Setup the necessary tables boto_resource = DynamoDBResource() From 4d6a1a6bced1b22d8ddc5ba9455dcf06041c1d3b Mon Sep 17 00:00:00 2001 From: Mark Drobnak Date: Wed, 12 Aug 2020 10:17:01 -0400 Subject: [PATCH 04/13] Temporarily use a different branch for integration test dependencies --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f4eca14da..fe7f39e04 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -38,8 +38,8 @@ jobs: - run: name: Set up Python command: | - curl https://raw.githubusercontent.com/mozilla-services/autopush/master/requirements.txt > requirements.txt - curl https://raw.githubusercontent.com/mozilla-services/autopush/master/test-requirements.txt > test-requirements.txt + curl https://raw.githubusercontent.com/mozilla-services/autopush/fix/editable-dependencies/requirements.txt > requirements.txt + curl https://raw.githubusercontent.com/mozilla-services/autopush/fix/editable-dependencies/test-requirements.txt > test-requirements.txt pip install --upgrade pip pip install bottle pip install -r requirements.txt From bbb383652b315260ac3b48536841bd07efa3a948 Mon Sep 17 00:00:00 2001 From: Mark Drobnak Date: Wed, 12 Aug 2020 10:22:32 -0400 Subject: [PATCH 05/13] Don't install autopush (Python) as editable We don't want to run its tests --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index fe7f39e04..0e8ba6b3e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -44,7 +44,7 @@ jobs: pip install bottle pip install -r requirements.txt pip install -r test-requirements.txt - pip install -e git+https://github.com/mozilla-services/autopush.git#egg=autopush + pip install git+https://github.com/mozilla-services/autopush.git#egg=autopush - run: name: Set up Rust command: | From cf2167c3bcb9e32d207154108d86f4f528e330c1 Mon Sep 17 00:00:00 2001 From: Mark Drobnak Date: Wed, 12 Aug 2020 14:00:33 -0400 Subject: [PATCH 06/13] Fix integration test errors by avoiding rotating tables By adding table rotation, the Rust autopush and autoendpoint code select different message tables. Autopush (specifically autopush-common) selects the last alphanumerically sorted message table, whereas autoendpoint uses the exact table name it was given. When running the Python autopush integration tests before the Rust autopush integration tests, the rotated tables would be created and trigger this issue. Since table rotation is no longer used, the tests have been removed/modified to not create the month-specific rotation tables. Also resets the AWS_LOCAL_DYNAMODB env variable if we started our own DynamoDB instance. Also fixes an issue where the create_router_table function did not check if the table already exists. This is fixed by using the get_router_table function. --- tests/test_integration.py | 327 +---------------------------- tests/test_integration_all_rust.py | 5 +- 2 files changed, 11 insertions(+), 321 deletions(-) diff --git a/tests/test_integration.py b/tests/test_integration.py index fe370c034..a4d563e3f 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -25,8 +25,7 @@ import requests import twisted.internet.base from autopush.config import AutopushConfig -from autopush.db import (Message, get_month, has_connected_this_month, - make_rotating_tablename, DynamoDBResource, create_rotating_message_table) +from autopush.db import (DynamoDBResource, create_message_table) from autopush.logging import begin_or_register from autopush.main import ( ConnectionApplication, @@ -43,7 +42,6 @@ from twisted.internet import reactor from twisted.internet.defer import inlineCallbacks, returnValue -from twisted.internet.threads import deferToThread from twisted.logger import globalLogPublisher from twisted.trial import unittest from typing import Optional @@ -202,13 +200,9 @@ def setup_dynamodb(): else: print "Using existing DynamoDB instance" - # Setup the necessary tables + # Setup the necessary tables (router table is created automatically) BOTO_RESOURCE = DynamoDBResource() - message_table = os.environ.get("MESSAGE_TABLE", "message_int_test") - create_rotating_message_table(prefix=message_table, delta=-1, - boto_resource=BOTO_RESOURCE) - create_rotating_message_table(prefix=message_table, - boto_resource=BOTO_RESOURCE) + create_message_table(MESSAGE_TABLE, boto_resource=BOTO_RESOURCE) pool = reactor.getThreadPool() pool.adjustPoolsize(minthreads=pool.max) @@ -313,6 +307,7 @@ def teardown_module(): global CN_SERVER, CN_MP_SERVER, DDB_PROCESS if DDB_PROCESS: + os.unsetenv("AWS_LOCAL_DYNAMODB") kill_process(DDB_PROCESS) kill_process(CN_SERVER) kill_process(CN_MP_SERVER) @@ -329,6 +324,7 @@ class TestRustWebPush(unittest.TestCase): router_table=dict(tablename=ROUTER_TABLE), message_table=dict(tablename=MESSAGE_TABLE), use_cryptography=True, + allow_table_rotation=False, ) # Max log lines allowed to be emitted by each node type @@ -1044,19 +1040,7 @@ class TestRustWebPushBroadcast(unittest.TestCase): router_table=dict(tablename=ROUTER_TABLE), message_table=dict(tablename=MESSAGE_TABLE), use_cryptography=True, - ) - - _conn_defaults = dict( - hostname='localhost', - port=RP_CONNECTION_PORT, - endpoint_port=ENDPOINT_PORT, - router_port=RP_ROUTER_PORT, - endpoint_scheme='http', - statsd_host=None, - router_table=dict(tablename=ROUTER_TABLE), - message_table=dict(tablename=MESSAGE_TABLE), - use_cryptography=True, - human_logs=False, + allow_table_rotation=False, ) max_endpoint_logs = 4 @@ -1084,26 +1068,12 @@ def setUp(self): self.start_ep(self._ep_conf) - # Create a Python connection application for accessing the db - self._conn_conf = AutopushConfig( - crypto_key=CRYPTO_KEY, - **self.conn_kwargs() - ) - self.conn = conn = ConnectionApplication( - self._conn_conf, - resource=BOTO_RESOURCE, - ) - conn.setup(rotate_tables=True) - def tearDown(self): process_logs(self) def endpoint_kwargs(self): return self._endpoint_defaults - def conn_kwargs(self): - return self._conn_defaults - @inlineCallbacks def quick_register(self, sslcontext=None, connection_port=None): conn_port = connection_port or MP_CONNECTION_PORT @@ -1239,289 +1209,6 @@ def test_broadcast_no_changes(self): yield self.shut_down(client) - @inlineCallbacks - def test_webpush_monthly_rotation(self): - client = yield self.quick_register() - yield client.disconnect() - - # Move the client back one month to the past - last_month = make_rotating_tablename( - prefix=self.conn.conf.message_table.tablename, delta=-1) - lm_message = Message(last_month, boto_resource=self.conn.db.resource) - yield deferToThread( - self.conn.db.router.update_message_month, - client.uaid, - last_month, - ) - - # Verify the move - c = yield deferToThread(self.conn.db.router.get_uaid, - client.uaid) - assert c["current_month"] == last_month - - # Verify last_connect is current, then move that back - assert has_connected_this_month(c) - today = get_month(delta=-1) - last_connect = int("%s%s020001" % (today.year, - str(today.month).zfill(2))) - - yield deferToThread( - self.conn.db.router._update_last_connect, - client.uaid, - last_connect) - c = yield deferToThread(self.conn.db.router.get_uaid, - client.uaid) - assert has_connected_this_month(c) is False - - # Move the clients channels back one month - exists, chans = yield deferToThread( - self.conn.db.message.all_channels, - client.uaid - ) - assert exists is True - assert len(chans) == 1 - yield deferToThread( - lm_message.save_channels, - client.uaid, - chans, - ) - - # Remove the channels entry entirely from this month - yield deferToThread( - self.conn.db.message.table.delete_item, - Key={'uaid': client.uaid, 'chidmessageid': ' '} - ) - - # Verify the channel is gone - exists, chans = yield deferToThread( - self.conn.db.message.all_channels, - client.uaid, - ) - assert exists is False - assert len(chans) == 0 - - # Send in a notification, verify it landed in last months notification - # table - data = uuid.uuid4().hex - yield client.send_notification(data=data) - ts, notifs = yield deferToThread(lm_message.fetch_timestamp_messages, - uuid.UUID(client.uaid), - " ") - assert len(notifs) == 1 - - # Connect the client, verify the migration - yield client.connect() - yield client.hello() - - # Pull down the notification - result = yield client.get_notification() - chan = client.channels.keys()[0] - assert result is not None - assert chan == result["channelID"] - - # Acknowledge the notification, which triggers the migration - yield client.ack(chan, result["version"]) - - # Wait up to 4 seconds for the table rotation to occur - start = time.time() - while time.time()-start < 4: - c = yield deferToThread( - self.conn.db.router.get_uaid, - client.uaid) - if c["current_month"] == self.conn.db.current_msg_month: - break - else: - yield deferToThread(time.sleep, 0.2) - - # Verify the month update in the router table - c = yield deferToThread( - self.conn.db.router.get_uaid, - client.uaid) - assert c["current_month"] == self.conn.db.current_msg_month - - # Verify the client moved last_connect - assert has_connected_this_month(c) is True - - # Verify the channels were moved - exists, chans = yield deferToThread( - self.conn.db.message.all_channels, - client.uaid - ) - assert exists is True - assert len(chans) == 1 - yield self.shut_down(client) - - @inlineCallbacks - def test_webpush_monthly_rotation_prior_record_exists(self): - client = yield self.quick_register() - yield client.disconnect() - - # Move the client back one month to the past - last_month = make_rotating_tablename( - prefix=self.conn.conf.message_table.tablename, delta=-1) - lm_message = Message(last_month, - boto_resource=BOTO_RESOURCE) - yield deferToThread( - self.conn.db.router.update_message_month, - client.uaid, - last_month, - ) - - # Verify the move - c = yield deferToThread(self.conn.db.router.get_uaid, - client.uaid) - assert c["current_month"] == last_month - - # Verify last_connect is current, then move that back - assert has_connected_this_month(c) - today = get_month(delta=-1) - yield deferToThread( - self.conn.db.router._update_last_connect, - client.uaid, - int("%s%s020001" % (today.year, str(today.month).zfill(2))), - ) - c = yield deferToThread(self.conn.db.router.get_uaid, client.uaid) - assert has_connected_this_month(c) is False - - # Move the clients channels back one month - exists, chans = yield deferToThread( - self.conn.db.message.all_channels, - client.uaid, - ) - assert exists is True - assert len(chans) == 1 - yield deferToThread( - lm_message.save_channels, - client.uaid, - chans, - ) - - # Send in a notification, verify it landed in last months notification - # table - data = uuid.uuid4().hex - yield client.send_notification(data=data) - _, notifs = yield deferToThread(lm_message.fetch_timestamp_messages, - uuid.UUID(client.uaid), - " ") - assert len(notifs) == 1 - - # Connect the client, verify the migration - yield client.connect() - yield client.hello() - - # Pull down the notification - result = yield client.get_notification() - chan = client.channels.keys()[0] - assert result is not None - assert chan == result["channelID"] - - # Acknowledge the notification, which triggers the migration - yield client.ack(chan, result["version"]) - - # Wait up to 4 seconds for the table rotation to occur - start = time.time() - while time.time()-start < 4: - c = yield deferToThread( - self.conn.db.router.get_uaid, - client.uaid) - if c["current_month"] == self.conn.db.current_msg_month: - break - else: - yield deferToThread(time.sleep, 0.2) - - # Verify the month update in the router table - c = yield deferToThread(self.conn.db.router.get_uaid, client.uaid) - assert c["current_month"] == self.conn.db.current_msg_month - - # Verify the client moved last_connect - assert has_connected_this_month(c) is True - - # Verify the channels were moved - exists, chans = yield deferToThread( - self.conn.db.message.all_channels, - client.uaid - ) - assert exists is True - assert len(chans) == 1 - yield self.shut_down(client) - - @inlineCallbacks - def test_webpush_monthly_rotation_no_channels(self): - client = Client("ws://localhost:{}/".format(MP_CONNECTION_PORT)) - yield client.connect() - yield client.hello() - yield client.disconnect() - - # Move the client back one month to the past - last_month = make_rotating_tablename( - prefix=self.conn.conf.message_table.tablename, delta=-1) - yield deferToThread( - self.conn.db.router.update_message_month, - client.uaid, - last_month - ) - - # Verify the move - c = yield deferToThread(self.conn.db.router.get_uaid, - client.uaid - ) - assert c["current_month"] == last_month - - # Verify there's no channels - exists, chans = yield deferToThread( - self.conn.db.message.all_channels, - client.uaid, - ) - assert exists is False - assert len(chans) == 0 - - # Connect the client, verify the migration - yield client.connect() - yield client.hello() - - # Wait up to 2 seconds for the table rotation to occur - start = time.time() - while time.time()-start < 2: - c = yield deferToThread( - self.conn.db.router.get_uaid, - client.uaid, - ) - if c["current_month"] == self.conn.db.current_msg_month: - break - else: - yield deferToThread(time.sleep, 0.2) - - # Verify the month update in the router table - c = yield deferToThread(self.conn.db.router.get_uaid, - client.uaid) - assert c["current_month"] == self.conn.db.current_msg_month - yield self.shut_down(client) - - @inlineCallbacks - def test_webpush_monthly_rotation_old_user_dropped(self): - client = yield self.quick_register() - uaid = client.uaid - yield client.disconnect() - - # Move the client back 2 months to the past - old_month = make_rotating_tablename( - prefix=self.conn.conf.message_table.tablename, delta=-3) - yield deferToThread( - self.conn.db.router.update_message_month, - client.uaid, - old_month - ) - _ = Message(old_month, boto_resource=BOTO_RESOURCE) - - # Verify the move - c = yield deferToThread(self.conn.db.router.get_uaid, client.uaid) - assert c["current_month"] == old_month - - # Connect the client and verify its uaid was dropped - yield client.connect() - yield client.hello() - assert client.uaid != uaid - class TestRustAndPythonWebPush(unittest.TestCase): _endpoint_defaults = dict( @@ -1534,6 +1221,7 @@ class TestRustAndPythonWebPush(unittest.TestCase): router_table=dict(tablename=ROUTER_TABLE), message_table=dict(tablename=MESSAGE_TABLE), use_cryptography=True, + allow_table_rotation=False, ) _conn_defaults = dict( @@ -1547,6 +1235,7 @@ class TestRustAndPythonWebPush(unittest.TestCase): message_table=dict(tablename=MESSAGE_TABLE), use_cryptography=True, human_logs=False, + allow_table_rotation=False, ) max_endpoint_logs = 1 diff --git a/tests/test_integration_all_rust.py b/tests/test_integration_all_rust.py index e51f8ebce..7ce78717a 100644 --- a/tests/test_integration_all_rust.py +++ b/tests/test_integration_all_rust.py @@ -22,7 +22,7 @@ import requests import twisted.internet.base from autopush.db import ( - DynamoDBResource, create_message_table, create_router_table + DynamoDBResource, create_message_table, get_router_table ) from autopush.tests.test_integration import Client, _get_vapid from autopush.utils import base64url_encode @@ -267,7 +267,7 @@ def setup_dynamodb(): # Setup the necessary tables boto_resource = DynamoDBResource() create_message_table(MESSAGE_TABLE, boto_resource=boto_resource) - create_router_table(ROUTER_TABLE, boto_resource=boto_resource) + get_router_table(ROUTER_TABLE, boto_resource=boto_resource) def setup_mock_server(): @@ -356,6 +356,7 @@ def setup_module(): def teardown_module(): if DDB_PROCESS: + os.unsetenv("AWS_LOCAL_DYNAMODB") kill_process(DDB_PROCESS) kill_process(CN_SERVER) kill_process(CN_MP_SERVER) From 903c65bc33c042cd89c945264776391c36bb867e Mon Sep 17 00:00:00 2001 From: Mark Drobnak Date: Wed, 12 Aug 2020 14:32:10 -0400 Subject: [PATCH 07/13] Switch back to master branch requirements.txt --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0e8ba6b3e..ca68c8633 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -38,8 +38,8 @@ jobs: - run: name: Set up Python command: | - curl https://raw.githubusercontent.com/mozilla-services/autopush/fix/editable-dependencies/requirements.txt > requirements.txt - curl https://raw.githubusercontent.com/mozilla-services/autopush/fix/editable-dependencies/test-requirements.txt > test-requirements.txt + curl https://raw.githubusercontent.com/mozilla-services/autopush/master/requirements.txt > requirements.txt + curl https://raw.githubusercontent.com/mozilla-services/autopush/master/test-requirements.txt > test-requirements.txt pip install --upgrade pip pip install bottle pip install -r requirements.txt From f644728a69fe04c784b2c8c64f3e6f5a2d2e0b3c Mon Sep 17 00:00:00 2001 From: Mark Drobnak Date: Wed, 12 Aug 2020 15:08:36 -0400 Subject: [PATCH 08/13] Add caching to the test job --- .circleci/config.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index ca68c8633..7cc9da84b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -35,6 +35,12 @@ jobs: AWS_LOCAL_DYNAMODB: http://localhost:8000 steps: - checkout + - restore_cache: + name: Restoring Python cache + key: python-v1-{{ checksum "requirements.txt" }}-{{ checksum "test-requirements.txt" }} + - restore_cache: + name: Restoring Rust cache + key: rust-v1-{{ .Branch }}-{{ checksum "Cargo.lock" }} - run: name: Set up Python command: | @@ -62,6 +68,19 @@ jobs: - run: name: Rust tests command: cargo test + - save_cache: + name: Save Python cache + key: python-v1-{{ checksum "requirements.txt" }}-{{ checksum "test-requirements.txt" }} + paths: + - /home/circleci/.local/bin/ + - /home/circleci/.local/lib/ + - save_cache: + name: Save Rust cache + key: rust-v1-{{ .Branch }}-{{ checksum "Cargo.lock" }} + paths: + - target + - ~/.cargo/registry + - ~/.cargo/git build: docker: From 4232950264c929a438f96d42403018bfc1deb78d Mon Sep 17 00:00:00 2001 From: Mark Drobnak Date: Wed, 12 Aug 2020 15:16:36 -0400 Subject: [PATCH 09/13] Fix python cache key missing required files --- .circleci/config.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7cc9da84b..94072a0da 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -35,6 +35,13 @@ jobs: AWS_LOCAL_DYNAMODB: http://localhost:8000 steps: - checkout + # Need to download the requirements.txt files so we can use their + # checksums in restore_cache. + - run: + name: Download Python dependency lists + command: | + curl https://raw.githubusercontent.com/mozilla-services/autopush/master/requirements.txt > requirements.txt + curl https://raw.githubusercontent.com/mozilla-services/autopush/master/test-requirements.txt > test-requirements.txt - restore_cache: name: Restoring Python cache key: python-v1-{{ checksum "requirements.txt" }}-{{ checksum "test-requirements.txt" }} @@ -44,8 +51,6 @@ jobs: - run: name: Set up Python command: | - curl https://raw.githubusercontent.com/mozilla-services/autopush/master/requirements.txt > requirements.txt - curl https://raw.githubusercontent.com/mozilla-services/autopush/master/test-requirements.txt > test-requirements.txt pip install --upgrade pip pip install bottle pip install -r requirements.txt From a82c905b2708ff4c1e7376812ae8d7a7ec557bba Mon Sep 17 00:00:00 2001 From: Mark Drobnak Date: Fri, 14 Aug 2020 09:26:18 -0400 Subject: [PATCH 10/13] Only explicitly install test-requirements.txt It contains includes requirements.txt by way of a `-r` command. --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8aa79595e..f0c2c90a6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -47,7 +47,7 @@ jobs: command: | pip install --upgrade pip pip install bottle - pip install -r requirements.txt + # test-requirements.txt includes requirements.txt pip install -r test-requirements.txt pip install git+https://github.com/mozilla-services/autopush.git#egg=autopush - run: From 6c79124c25e2b5cd6f242178d3b88b63d88f81ad Mon Sep 17 00:00:00 2001 From: jrconlin Date: Mon, 17 Aug 2020 20:49:07 +0000 Subject: [PATCH 11/13] f kick circleci --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index f0c2c90a6..f69052472 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -5,6 +5,7 @@ # DOCKER_USER # DOCKER_PASS # +# version: 2 jobs: audit: From 112ba3b0f866fd68e910b602b1d8b83aad69aa5b Mon Sep 17 00:00:00 2001 From: jrconlin Date: Mon, 17 Aug 2020 23:49:07 +0000 Subject: [PATCH 12/13] f post merge fix to circleci config --- .circleci/config.yml | 88 ++++++++++++++++++++++++++++++++------------ 1 file changed, 64 insertions(+), 24 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f69052472..e776cff67 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,12 +1,15 @@ # These environment variables must be set in CircleCI UI # # DOCKERHUB_REPO - docker hub repo, format: / +# DOCKERHUB_ENDPOINT_REPO - same as DOCKERHUB_REPO, but for autoendpoint # DOCKER_EMAIL - login info for docker hub # DOCKER_USER # DOCKER_PASS # # -version: 2 + +version: 2.1 + jobs: audit: docker: @@ -86,14 +89,20 @@ jobs: docker: - image: docker:18.03.0-ce working_directory: /dockerflow + parameters: + image: + type: string + crate: + type: string + binary: + type: string steps: + # Install these packages before checkout because git may not exist or work - run: name: Install Docker build dependencies command: apk add --no-cache openssh-client git - - checkout - setup_remote_docker - - run: name: Create a version.json command: | @@ -104,47 +113,52 @@ jobs: "$CIRCLE_PROJECT_USERNAME" \ "$CIRCLE_PROJECT_REPONAME" \ "$CIRCLE_BUILD_URL" > version.json - - run: name: Build Docker image - command: docker build -t app:build . - - # save the built docker container into CircleCI's cache. This is + command: | + docker build -t <> \ + --build-arg CRATE=<> \ + --build-arg BINARY=<> . + # save the built docker container into CircleCI's workspace cache. This is # required since Workflows do not have the same remote docker instance. - run: - name: docker save app:build - command: mkdir -p /cache; docker save -o /cache/docker.tar "app:build" - - save_cache: - key: v1-{{ .Branch }}-{{ .Environment.CIRCLE_TAG }}-{{ epoch }} + name: docker save <> + command: mkdir -p /cache; docker save -o /cache/docker.tar "<>" + - persist_to_workspace: + root: /cache paths: - - /cache/docker.tar + - docker.tar deploy: docker: - image: docker:18.03.0-ce + parameters: + image: + type: string + repo: + type: string steps: - setup_remote_docker - - restore_cache: - key: v1-{{ .Branch }}-{{ .Environment.CIRCLE_TAG }} + - attach_workspace: + at: /cache - run: name: Restore Docker image cache command: docker load -i /cache/docker.tar - - run: name: Deploy to Dockerhub command: | - # deploy master if [ "${CIRCLE_BRANCH}" == "master" ]; then + # deploy master docker login -u $DOCKER_USER -p $DOCKER_PASS - docker tag app:build ${DOCKERHUB_REPO}:latest - docker push ${DOCKERHUB_REPO}:latest + docker tag <> <>:latest + docker push <>:latest elif [ ! -z "${CIRCLE_TAG}" ]; then - # deploy a release tag... + # deploy a release tag docker login -u $DOCKER_USER -p $DOCKER_PASS - echo "${DOCKERHUB_REPO}:${CIRCLE_TAG}" - docker tag app:build "${DOCKERHUB_REPO}:${CIRCLE_TAG}" + echo "<>:${CIRCLE_TAG}" + docker tag <> "<>:${CIRCLE_TAG}" docker images - docker push "${DOCKERHUB_REPO}:${CIRCLE_TAG}" + docker push "<>:${CIRCLE_TAG}" fi workflows: @@ -155,20 +169,46 @@ workflows: filters: tags: only: /.*/ - - test: filters: tags: only: /.*/ + - build: + name: build-autopush + image: autopush:build + crate: autopush + binary: autopush_rs + filters: + tags: + only: /.*/ - build: + name: build-autoendpoint + image: autoendpoint:build + crate: autoendpoint + binary: autoendpoint filters: tags: only: /.*/ - deploy: + name: deploy-autopush + image: autopush:build + repo: ${DOCKERHUB_REPO} + requires: + - build-autopush + filters: + tags: + only: /.*/ + branches: + only: master + + - deploy: + name: deploy-autoendpoint + image: autoendpoint:build + repo: ${DOCKERHUB_ENDPOINT_REPO} requires: - - build + - build-autoendpoint - test filters: tags: From a732d0e19d7ab847d78e1b438b6518df5d69a21c Mon Sep 17 00:00:00 2001 From: jrconlin Date: Tue, 18 Aug 2020 16:17:55 +0000 Subject: [PATCH 13/13] f add test to deploy-autopush --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index e776cff67..25baedc11 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -197,6 +197,7 @@ workflows: repo: ${DOCKERHUB_REPO} requires: - build-autopush + - test filters: tags: only: /.*/