From 0ab86ac6150eda700273250fca04a9fc8522ec04 Mon Sep 17 00:00:00 2001 From: jrconlin Date: Tue, 9 Jan 2024 09:48:24 -0800 Subject: [PATCH 01/17] bug: Copy modified purge script from old tokenserver repo --- tools/tokenserver/purge_old_records.py | 173 ++++++++++++++++--------- 1 file changed, 115 insertions(+), 58 deletions(-) diff --git a/tools/tokenserver/purge_old_records.py b/tools/tokenserver/purge_old_records.py index b3d4df6a97..fbacddefa0 100644 --- a/tools/tokenserver/purge_old_records.py +++ b/tools/tokenserver/purge_old_records.py @@ -5,7 +5,8 @@ Script to purge user records that have been replaced. -This script purges any obsolete user records from the database. +This script takes a tokenserver config file, uses it to load the assignment +backend, and then purges any obsolete user records from that backend. Obsolete records are those that have been replaced by a newer record for the same user. @@ -15,33 +16,31 @@ """ -import binascii -import hawkauthlib +import os +import time +import random import logging import optparse -import random + import requests -import time import tokenlib +import hawkauthlib -import util -from database import Database -from util import format_key_id +import tokenserver.scripts +from tokenserver.assignment import INodeAssignment +from tokenserver.util import format_key_id logger = logging.getLogger("tokenserver.scripts.purge_old_records") -PATTERN = "{node}/1.5/{uid}" +def purge_old_records(config_file, grace_period=-1, max_per_loop=10, + max_offset=0, request_timeout=60, settings=None): + """Purge old records from the assignment backend in the given config file. -def purge_old_records(secret, grace_period=-1, max_per_loop=10, max_offset=0, - request_timeout=60): - """Purge old records from the database. - - This function queries all of the old user records in the database, deletes - the Tokenserver database record for each of the users, and issues a delete - request to each user's storage node. The result is a gradual pruning of - expired items from each database. + This function iterates through each storage backend in the given config + file and calls its purge_expired_items() method. The result is a + gradual pruning of expired items from each database. `max_offset` is used to select a random offset into the list of purgeable records. With multiple tasks running concurrently, this will provide each @@ -49,59 +48,107 @@ def purge_old_records(secret, grace_period=-1, max_per_loop=10, max_offset=0, randomization. """ logger.info("Purging old user records") + logger.debug("Using config file %r", config_file) + config = tokenserver.scripts.load_configurator(config_file) + config.begin() try: - database = Database() - # Process batches of items, until we run out. - while True: - offset = random.randint(0, max_offset) - kwds = { - "grace_period": grace_period, - "limit": max_per_loop, - "offset": offset, - } - rows = list(database.get_old_user_records(**kwds)) - logger.info("Fetched %d rows at offset %d", len(rows), offset) - for row in rows: - # Don't attempt to purge data from downed nodes. - # Instead wait for them to either come back up or to be - # completely removed from service. - if row.node is None: - logger.info("Deleting user record for uid %s on %s", - row.uid, row.node) - database.delete_user_record(row.uid) - elif not row.downed: - logger.info("Purging uid %s on %s", row.uid, row.node) - delete_service_data(row, secret, timeout=request_timeout) - database.delete_user_record(row.uid) - if len(rows) < max_per_loop: - break - except Exception: - logger.exception("Error while purging old user records") + backend = config.registry.getUtility(INodeAssignment) + patterns = config.registry['endpoints_patterns'] + for service in patterns: + previous_list = [] + logger.debug("Purging old user records for service: %s", service) + # Process batches of items, until we run out. + while True: + offset = random.randint(0, max_offset) + kwds = { + "grace_period": grace_period, + "limit": max_per_loop, + "offset": offset, + } + rows = list(backend.get_old_user_records(service, **kwds)) + if not rows: + logger.info("No more data for %s", service) + break + if rows == previous_list: + raise Exception("Loop detected") + previous_list = rows + logger.info("Fetched %d rows at offset %d", len(rows), offset) + counter = 0 + for row in rows: + # Don't attempt to purge data from downed nodes. + # Instead wait for them to either come back up or to be + # completely removed from service. + if row.node is None: + logger.info("Deleting user record for uid %s on %s", + row.uid, row.node) + if settings and not settings.dryrun: + backend.delete_user_record(service, row.uid) + elif not row.downed: + logger.info("Purging uid %s on %s", row.uid, row.node) + if settings and not settings.dryrun: + delete_service_data(config, service, row, + timeout=request_timeout, + settings=settings) + backend.delete_user_record(service, row.uid) + counter += 1 + elif settings and settings.force: + logger.info( + "Forcing tokenserver record delete: {}".format( + row.uid + ) + ) + logger.info("Forcing Purge uid %s on %s", + row.uid, + row.node) + if not settings.dryrun: + delete_service_data(config, service, row, + timeout=request_timeout, + settings=settings) + backend.delete_user_record(service, row.uid) + counter += 1 + if settings and settings.max_records: + if counter >= settings.max_records: + logger.info("Reached max_records, exiting") + return True + if len(rows) < max_per_loop: + break + except Exception as e: + logger.exception("Error while purging old user records: {}".format(e)) return False else: logger.info("Finished purging old user records") return True + finally: + config.end() -def delete_service_data(user, secret, timeout=60): +def delete_service_data(config, service, user, timeout=60, settings=None): """Send a data-deletion request to the user's service node. This is a little bit of hackery to cause the user's service node to remove any data it still has stored for the user. We simulate a DELETE request from the user's own account. """ + secrets = config.registry.settings['tokenserver.secrets'] + pattern = config.registry['endpoints_patterns'][service] + node_secrets = secrets.get(user.node) + if not node_secrets: + msg = "The node %r does not have any shared secret" % (user.node,) + raise ValueError(msg) token = tokenlib.make_token({ "uid": user.uid, "node": user.node, "fxa_uid": user.email.split("@", 1)[0], "fxa_kid": format_key_id( user.keys_changed_at or user.generation, - binascii.unhexlify(user.client_state) + user.client_state.decode('hex') ), - }, secret=secret) - secret = tokenlib.get_derived_secret(token, secret=secret) - endpoint = PATTERN.format(uid=user.uid, node=user.node) + }, secret=node_secrets[-1]) + secret = tokenlib.get_derived_secret(token, secret=node_secrets[-1]) + endpoint = pattern.format(uid=user.uid, service=service, node=user.node) auth = HawkAuth(token, secret) + if settings and settings.dryrun: + return resp = requests.delete(endpoint, auth=auth, timeout=timeout) if resp.status_code >= 400 and resp.status_code != 404: resp.raise_for_status() @@ -125,7 +172,7 @@ def main(args=None): This function parses command-line arguments and passes them on to the purge_old_records() function. """ - usage = "usage: %prog [options] secret" + usage = "usage: %prog [options] config_file" parser = optparse.OptionParser(usage=usage) parser.add_option("", "--purge-interval", type="int", default=3600, help="Interval to sleep between purging runs") @@ -141,23 +188,31 @@ def main(args=None): help="Timeout for service deletion requests") parser.add_option("", "--oneshot", action="store_true", help="Do a single purge run and then exit") + parser.add_option("", "--max-records", type="int", default=0, + help="Max records to delete") + parser.add_option("", "--dryrun", action="store_true", + help="Don't do destructive things") + parser.add_option("", "--force", action="store_true", + help="force record to be deleted from TS db," + " even if node is down") parser.add_option("-v", "--verbose", action="count", dest="verbosity", help="Control verbosity of log messages") opts, args = parser.parse_args(args) - if len(args) != 2: + if len(args) != 1: parser.print_usage() return 1 - secret = args[1] + tokenserver.scripts.configure_script_logging(opts) - util.configure_script_logging(opts) + config_file = os.path.abspath(args[0]) - purge_old_records(secret, + purge_old_records(config_file, grace_period=opts.grace_period, max_per_loop=opts.max_per_loop, max_offset=opts.max_offset, - request_timeout=opts.request_timeout) + request_timeout=opts.request_timeout, + settings=opts) if not opts.oneshot: while True: # Randomize sleep interval +/- thirty percent to desynchronize @@ -166,12 +221,14 @@ def main(args=None): sleep_time += random.randint(-0.3 * sleep_time, 0.3 * sleep_time) logger.debug("Sleeping for %d seconds", sleep_time) time.sleep(sleep_time) - purge_old_records(grace_period=opts.grace_period, + purge_old_records(config_file, + grace_period=opts.grace_period, max_per_loop=opts.max_per_loop, max_offset=opts.max_offset, - request_timeout=opts.request_timeout) + request_timeout=opts.request_timeout, + settings=opts) return 0 if __name__ == "__main__": - util.run_script(main) + tokenserver.scripts.run_script(main) From c4be49c47cb67a3f351bbaf37c82b3e5230fa50c Mon Sep 17 00:00:00 2001 From: jrconlin Date: Tue, 9 Jan 2024 10:22:59 -0800 Subject: [PATCH 02/17] f modify old purge script --- tools/tokenserver/purge_old_records.py | 188 +++++++++++-------------- 1 file changed, 80 insertions(+), 108 deletions(-) diff --git a/tools/tokenserver/purge_old_records.py b/tools/tokenserver/purge_old_records.py index fbacddefa0..815f1218c3 100644 --- a/tools/tokenserver/purge_old_records.py +++ b/tools/tokenserver/purge_old_records.py @@ -5,8 +5,7 @@ Script to purge user records that have been replaced. -This script takes a tokenserver config file, uses it to load the assignment -backend, and then purges any obsolete user records from that backend. +This script purges any obsolete user records from the database. Obsolete records are those that have been replaced by a newer record for the same user. @@ -16,31 +15,33 @@ """ -import os -import time -import random +import binascii +import hawkauthlib import logging import optparse - +import random import requests +import time import tokenlib -import hawkauthlib -import tokenserver.scripts -from tokenserver.assignment import INodeAssignment -from tokenserver.util import format_key_id +import util +from database import Database +from util import format_key_id logger = logging.getLogger("tokenserver.scripts.purge_old_records") +PATTERN = "{node}/1.5/{uid}" -def purge_old_records(config_file, grace_period=-1, max_per_loop=10, - max_offset=0, request_timeout=60, settings=None): - """Purge old records from the assignment backend in the given config file. - This function iterates through each storage backend in the given config - file and calls its purge_expired_items() method. The result is a - gradual pruning of expired items from each database. +def purge_old_records(secret, grace_period=-1, max_per_loop=10, max_offset=0, + request_timeout=60, settings=None): + """Purge old records from the database. + + This function queries all of the old user records in the database, deletes + the Tokenserver database record for each of the users, and issues a delete + request to each user's storage node. The result is a gradual pruning of + expired items from each database. `max_offset` is used to select a random offset into the list of purgeable records. With multiple tasks running concurrently, this will provide each @@ -48,107 +49,81 @@ def purge_old_records(config_file, grace_period=-1, max_per_loop=10, randomization. """ logger.info("Purging old user records") - logger.debug("Using config file %r", config_file) - config = tokenserver.scripts.load_configurator(config_file) - config.begin() try: - backend = config.registry.getUtility(INodeAssignment) - patterns = config.registry['endpoints_patterns'] - for service in patterns: - previous_list = [] - logger.debug("Purging old user records for service: %s", service) - # Process batches of items, until we run out. - while True: - offset = random.randint(0, max_offset) - kwds = { - "grace_period": grace_period, - "limit": max_per_loop, - "offset": offset, - } - rows = list(backend.get_old_user_records(service, **kwds)) - if not rows: - logger.info("No more data for %s", service) - break - if rows == previous_list: - raise Exception("Loop detected") - previous_list = rows - logger.info("Fetched %d rows at offset %d", len(rows), offset) - counter = 0 - for row in rows: - # Don't attempt to purge data from downed nodes. - # Instead wait for them to either come back up or to be - # completely removed from service. - if row.node is None: - logger.info("Deleting user record for uid %s on %s", - row.uid, row.node) - if settings and not settings.dryrun: - backend.delete_user_record(service, row.uid) - elif not row.downed: - logger.info("Purging uid %s on %s", row.uid, row.node) - if settings and not settings.dryrun: - delete_service_data(config, service, row, - timeout=request_timeout, - settings=settings) - backend.delete_user_record(service, row.uid) - counter += 1 - elif settings and settings.force: - logger.info( - "Forcing tokenserver record delete: {}".format( - row.uid - ) - ) - logger.info("Forcing Purge uid %s on %s", - row.uid, - row.node) - if not settings.dryrun: - delete_service_data(config, service, row, - timeout=request_timeout, - settings=settings) - backend.delete_user_record(service, row.uid) - counter += 1 - if settings and settings.max_records: - if counter >= settings.max_records: - logger.info("Reached max_records, exiting") - return True - if len(rows) < max_per_loop: - break - except Exception as e: - logger.exception("Error while purging old user records: {}".format(e)) + database = Database() + previous_list = [] + # Process batches of items, until we run out. + while True: + offset = random.randint(0, max_offset) + kwds = { + "grace_period": grace_period, + "limit": max_per_loop, + "offset": offset, + } + rows = list(database.get_old_user_records(**kwds)) + if not rows: + logger.info("No more data") + break + if rows == previous_list: + raise Exception("Loop detected") + logger.info("Fetched %d rows at offset %d", len(rows), offset) + counter = 0 + for row in rows: + # Don't attempt to purge data from downed nodes. + # Instead wait for them to either come back up or to be + # completely removed from service. + if row.node is None: + logger.info("Deleting user record for uid %s on %s", + row.uid, row.node) + if settings and not settings.dryrun: + database.delete_user_record(row.uid) + elif not row.downed: + logger.info("Purging uid %s on %s", row.uid, row.node) + if settings and not settings.dryrun: + delete_service_data(row, secret, timeout=request_timeout, settings=settings) + database.delete_user_record(row.uid) + counter += 1 + elif settings and settings.force: + logger.info( + f"Forcing tokenserver record delete: {row.uid} on {row.node}" + ) + if not settings.dryrun: + delete_service_data(row, secret, timeout=request_timeout, settings=settings) + database.delete_user_record(row.uid) + counter += 1 + if settings and settings.max_records: + if counter >= settings.max_records: + logger.info("Reached max_records, exiting") + return True + if len(rows) < max_per_loop: + break + except Exception: + logger.exception("Error while purging old user records") return False else: logger.info("Finished purging old user records") return True - finally: - config.end() -def delete_service_data(config, service, user, timeout=60, settings=None): +def delete_service_data(user, secret, timeout=60, settings=None): """Send a data-deletion request to the user's service node. This is a little bit of hackery to cause the user's service node to remove any data it still has stored for the user. We simulate a DELETE request from the user's own account. """ - secrets = config.registry.settings['tokenserver.secrets'] - pattern = config.registry['endpoints_patterns'][service] - node_secrets = secrets.get(user.node) - if not node_secrets: - msg = "The node %r does not have any shared secret" % (user.node,) - raise ValueError(msg) token = tokenlib.make_token({ "uid": user.uid, "node": user.node, "fxa_uid": user.email.split("@", 1)[0], "fxa_kid": format_key_id( user.keys_changed_at or user.generation, - user.client_state.decode('hex') + binascii.unhexlify(user.client_state) ), - }, secret=node_secrets[-1]) - secret = tokenlib.get_derived_secret(token, secret=node_secrets[-1]) - endpoint = pattern.format(uid=user.uid, service=service, node=user.node) + }, secret=secret) + secret = tokenlib.get_derived_secret(token, secret=secret) + endpoint = PATTERN.format(uid=user.uid, node=user.node) auth = HawkAuth(token, secret) - if settings and settings.dryrun: - return resp = requests.delete(endpoint, auth=auth, timeout=timeout) if resp.status_code >= 400 and resp.status_code != 404: resp.raise_for_status() @@ -172,7 +147,7 @@ def main(args=None): This function parses command-line arguments and passes them on to the purge_old_records() function. """ - usage = "usage: %prog [options] config_file" + usage = "usage: %prog [options] secret" parser = optparse.OptionParser(usage=usage) parser.add_option("", "--purge-interval", type="int", default=3600, help="Interval to sleep between purging runs") @@ -188,26 +163,24 @@ def main(args=None): help="Timeout for service deletion requests") parser.add_option("", "--oneshot", action="store_true", help="Do a single purge run and then exit") - parser.add_option("", "--max-records", type="int", default=0, - help="Max records to delete") + parser.add_option("-v", "--verbose", action="count", dest="verbosity", + help="Control verbosity of log messages") parser.add_option("", "--dryrun", action="store_true", help="Don't do destructive things") parser.add_option("", "--force", action="store_true", help="force record to be deleted from TS db," " even if node is down") - parser.add_option("-v", "--verbose", action="count", dest="verbosity", - help="Control verbosity of log messages") opts, args = parser.parse_args(args) - if len(args) != 1: + if len(args) != 2: parser.print_usage() return 1 - tokenserver.scripts.configure_script_logging(opts) + secret = args[1] - config_file = os.path.abspath(args[0]) + util.configure_script_logging(opts) - purge_old_records(config_file, + purge_old_records(secret, grace_period=opts.grace_period, max_per_loop=opts.max_per_loop, max_offset=opts.max_offset, @@ -221,8 +194,7 @@ def main(args=None): sleep_time += random.randint(-0.3 * sleep_time, 0.3 * sleep_time) logger.debug("Sleeping for %d seconds", sleep_time) time.sleep(sleep_time) - purge_old_records(config_file, - grace_period=opts.grace_period, + purge_old_records(grace_period=opts.grace_period, max_per_loop=opts.max_per_loop, max_offset=opts.max_offset, request_timeout=opts.request_timeout, @@ -231,4 +203,4 @@ def main(args=None): if __name__ == "__main__": - tokenserver.scripts.run_script(main) + util.run_script(main) From 2e664fe6883581487c0d18910f2e8f77cf1b7caa Mon Sep 17 00:00:00 2001 From: jrconlin Date: Tue, 9 Jan 2024 10:25:15 -0800 Subject: [PATCH 03/17] f missed dryrun for `delete_service_data` --- tools/tokenserver/purge_old_records.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/tokenserver/purge_old_records.py b/tools/tokenserver/purge_old_records.py index 815f1218c3..555438d673 100644 --- a/tools/tokenserver/purge_old_records.py +++ b/tools/tokenserver/purge_old_records.py @@ -124,6 +124,8 @@ def delete_service_data(user, secret, timeout=60, settings=None): secret = tokenlib.get_derived_secret(token, secret=secret) endpoint = PATTERN.format(uid=user.uid, node=user.node) auth = HawkAuth(token, secret) + if settings and settings.dryrun: + return resp = requests.delete(endpoint, auth=auth, timeout=timeout) if resp.status_code >= 400 and resp.status_code != 404: resp.raise_for_status() From 9dcc14e7e6ae4cdbc3c2c7507146898c38bf74c7 Mon Sep 17 00:00:00 2001 From: jrconlin Date: Tue, 9 Jan 2024 12:27:04 -0800 Subject: [PATCH 04/17] f fix logic --- tools/tokenserver/purge_old_records.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/tokenserver/purge_old_records.py b/tools/tokenserver/purge_old_records.py index 555438d673..4147941bfb 100644 --- a/tools/tokenserver/purge_old_records.py +++ b/tools/tokenserver/purge_old_records.py @@ -19,6 +19,7 @@ import hawkauthlib import logging import optparse +import os import random import requests import time @@ -28,8 +29,8 @@ from database import Database from util import format_key_id - logger = logging.getLogger("tokenserver.scripts.purge_old_records") +logger.setLevel(os.environ.get("PYTHON_LOG", "ERROR").upper()) PATTERN = "{node}/1.5/{uid}" @@ -80,6 +81,8 @@ def purge_old_records(secret, grace_period=-1, max_per_loop=10, max_offset=0, elif not row.downed: logger.info("Purging uid %s on %s", row.uid, row.node) if settings and not settings.dryrun: + pass + else: delete_service_data(row, secret, timeout=request_timeout, settings=settings) database.delete_user_record(row.uid) counter += 1 From 88489ef09f6905e313f3fa77797dd10f0c6a0f68 Mon Sep 17 00:00:00 2001 From: jrconlin Date: Tue, 9 Jan 2024 12:36:47 -0800 Subject: [PATCH 05/17] f flake8 < black --- tools/tokenserver/purge_old_records.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/tools/tokenserver/purge_old_records.py b/tools/tokenserver/purge_old_records.py index 4147941bfb..18a9f4ed02 100644 --- a/tools/tokenserver/purge_old_records.py +++ b/tools/tokenserver/purge_old_records.py @@ -83,15 +83,24 @@ def purge_old_records(secret, grace_period=-1, max_per_loop=10, max_offset=0, if settings and not settings.dryrun: pass else: - delete_service_data(row, secret, timeout=request_timeout, settings=settings) + delete_service_data( + row, + secret, + timeout=request_timeout, + settings=settings) database.delete_user_record(row.uid) counter += 1 elif settings and settings.force: logger.info( - f"Forcing tokenserver record delete: {row.uid} on {row.node}" + "Forcing tokenserver record delete: " + f"{row.uid} on {row.node}" ) if not settings.dryrun: - delete_service_data(row, secret, timeout=request_timeout, settings=settings) + delete_service_data( + row, + secret, + timeout=request_timeout, + settings=settings) database.delete_user_record(row.uid) counter += 1 if settings and settings.max_records: From 42a785fb7af3fe7bf9897c7eb0232494f1b8e66b Mon Sep 17 00:00:00 2001 From: jrconlin Date: Tue, 9 Jan 2024 13:00:52 -0800 Subject: [PATCH 06/17] f apt-key is dead. --- Dockerfile | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index b53191f128..82c900989e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -32,12 +32,12 @@ COPY --from=cacher /app/target /app/target COPY --from=cacher $CARGO_HOME /app/$CARGO_HOME RUN \ - echo "deb https://repo.mysql.com/apt/debian/ bullseye mysql-8.0" >> /etc/apt/sources.list && \ - # mysql_pubkey.asc from: # https://dev.mysql.com/doc/refman/8.0/en/checking-gpg-signature.html # related: # https://dev.mysql.com/doc/mysql-apt-repo-quick-guide/en/#repo-qg-apt-repo-manual-setup - apt-key adv --import mysql_pubkey.asc && \ + # Fetch and load the MySQL public key. We need to install libmysqlclient-dev to build syncstorage-rs + # which wants the mariadb + wget -qO- https://repo.mysql.com/RPM-GPG-KEY-mysql-2023 > /etc/apt/trusted.gpg.d/mysql.asc && \ apt-get -q update && \ apt-get -q install -y --no-install-recommends libmysqlclient-dev cmake golang-go python3-dev python3-pip python3-setuptools python3-wheel && \ pip3 install -r requirements.txt && \ @@ -68,7 +68,12 @@ RUN \ # and ca-certificates needed for https://repo.mysql.com apt-get install -y gnupg ca-certificates && \ echo "deb https://repo.mysql.com/apt/debian/ bullseye mysql-8.0" >> /etc/apt/sources.list && \ - apt-key adv --import mysql_pubkey.asc && \ + # https://dev.mysql.com/doc/refman/8.0/en/checking-gpg-signature.html + # related: + # https://dev.mysql.com/doc/mysql-apt-repo-quick-guide/en/#repo-qg-apt-repo-manual-setup + # Fetch and load the MySQL public key. We need to install libmysqlclient-dev to build syncstorage-rs + # which wants the mariadb + wget -qO- https://repo.mysql.com/RPM-GPG-KEY-mysql-2023 > /etc/apt/trusted.gpg.d/mysql.asc && \ # update again now that we trust repo.mysql.com apt-get -q update && \ apt-get -q install -y build-essential libmysqlclient-dev libssl-dev libffi-dev libcurl4 python3-dev python3-pip python3-setuptools python3-wheel cargo curl jq && \ From 96c7e8ddc9f1cd7a93c093026155ccb8d120c9a0 Mon Sep 17 00:00:00 2001 From: jrconlin Date: Tue, 9 Jan 2024 13:33:18 -0800 Subject: [PATCH 07/17] f remove other apt-key --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 82c900989e..043c35a605 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,7 +17,6 @@ RUN \ # https://dev.mysql.com/doc/refman/8.0/en/checking-gpg-signature.html # related: # https://dev.mysql.com/doc/mysql-apt-repo-quick-guide/en/#repo-qg-apt-repo-manual-setup - apt-key adv --import mysql_pubkey.asc && \ apt-get -q update && \ apt-get -q install -y --no-install-recommends libmysqlclient-dev cmake From 71d95e9055aa91f5825a35815d70e2068d88a0dc Mon Sep 17 00:00:00 2001 From: jrconlin Date: Tue, 9 Jan 2024 15:30:01 -0800 Subject: [PATCH 08/17] f update Dockerfile --- .dockerignore | 1 + Dockerfile | 11 ++++++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.dockerignore b/.dockerignore index 5ef407a275..b9eefd9fcb 100644 --- a/.dockerignore +++ b/.dockerignore @@ -10,3 +10,4 @@ PULL_REQUEST_TEMPLATE.md README.md target tools/examples +db/ diff --git a/Dockerfile b/Dockerfile index 043c35a605..aef590bfc3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,13 +12,14 @@ COPY --from=planner /app/mysql_pubkey.asc mysql_pubkey.asc # cmake is required to build grpcio-sys for Spanner builds RUN \ + ( wget -qO- https://repo.mysql.com/RPM-GPG-KEY-mysql-2023 > /etc/apt/trusted.gpg.d/mysql.asc ) && \ echo "deb https://repo.mysql.com/apt/debian/ bullseye mysql-8.0" >> /etc/apt/sources.list && \ # mysql_pubkey.asc from: # https://dev.mysql.com/doc/refman/8.0/en/checking-gpg-signature.html # related: # https://dev.mysql.com/doc/mysql-apt-repo-quick-guide/en/#repo-qg-apt-repo-manual-setup apt-get -q update && \ - apt-get -q install -y --no-install-recommends libmysqlclient-dev cmake + apt-get -q install -y --no-install-recommends libmariadb-dev cmake COPY --from=planner /app/recipe.json recipe.json RUN cargo chef cook --release --no-default-features --features=syncstorage-db/$DATABASE_BACKEND --recipe-path recipe.json @@ -36,9 +37,9 @@ RUN \ # https://dev.mysql.com/doc/mysql-apt-repo-quick-guide/en/#repo-qg-apt-repo-manual-setup # Fetch and load the MySQL public key. We need to install libmysqlclient-dev to build syncstorage-rs # which wants the mariadb - wget -qO- https://repo.mysql.com/RPM-GPG-KEY-mysql-2023 > /etc/apt/trusted.gpg.d/mysql.asc && \ + ( wget -qO- https://repo.mysql.com/RPM-GPG-KEY-mysql-2023 > /etc/apt/trusted.gpg.d/mysql.asc ) && \ apt-get -q update && \ - apt-get -q install -y --no-install-recommends libmysqlclient-dev cmake golang-go python3-dev python3-pip python3-setuptools python3-wheel && \ + apt-get -q install -y --no-install-recommends libmariadb-dev cmake golang-go python3-dev python3-pip python3-setuptools python3-wheel && \ pip3 install -r requirements.txt && \ rm -rf /var/lib/apt/lists/* @@ -72,10 +73,10 @@ RUN \ # https://dev.mysql.com/doc/mysql-apt-repo-quick-guide/en/#repo-qg-apt-repo-manual-setup # Fetch and load the MySQL public key. We need to install libmysqlclient-dev to build syncstorage-rs # which wants the mariadb - wget -qO- https://repo.mysql.com/RPM-GPG-KEY-mysql-2023 > /etc/apt/trusted.gpg.d/mysql.asc && \ + ( wget -qO- https://repo.mysql.com/RPM-GPG-KEY-mysql-2023 > /etc/apt/trusted.gpg.d/mysql.asc ) && \ # update again now that we trust repo.mysql.com apt-get -q update && \ - apt-get -q install -y build-essential libmysqlclient-dev libssl-dev libffi-dev libcurl4 python3-dev python3-pip python3-setuptools python3-wheel cargo curl jq && \ + apt-get -q install -y build-essential libmariadb-dev libssl-dev libffi-dev libcurl4 python3-dev python3-pip python3-setuptools python3-wheel cargo curl jq && \ # The python3-cryptography debian package installs version 2.6.1, but we # we want to use the version specified in requirements.txt. To do this, # we have to remove the python3-cryptography package here. From fd2187315396df70b96a4f670a49b6d47e2889b3 Mon Sep 17 00:00:00 2001 From: jrconlin Date: Tue, 9 Jan 2024 15:55:16 -0800 Subject: [PATCH 09/17] f add wget --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index aef590bfc3..0b1efbf7a1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -66,7 +66,7 @@ RUN \ # first, an apt-get update is required for gnupg, which is required for apt-key adv apt-get -q update && \ # and ca-certificates needed for https://repo.mysql.com - apt-get install -y gnupg ca-certificates && \ + apt-get install -y gnupg ca-certificates wget && \ echo "deb https://repo.mysql.com/apt/debian/ bullseye mysql-8.0" >> /etc/apt/sources.list && \ # https://dev.mysql.com/doc/refman/8.0/en/checking-gpg-signature.html # related: From c7f6323a3c42c78eef8c430c9a00016115c0514f Mon Sep 17 00:00:00 2001 From: jrconlin Date: Fri, 12 Jan 2024 16:48:48 -0800 Subject: [PATCH 10/17] f revert to libmysqlclient --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5347f15f23..c66201d926 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,7 +16,7 @@ RUN \ wget -qO- https://repo.mysql.com/RPM-GPG-KEY-mysql-2023 > /etc/apt/trusted.gpg.d/mysql.asc && \ echo "deb https://repo.mysql.com/apt/debian/ bullseye mysql-8.0" >> /etc/apt/sources.list && \ apt-get -q update && \ - apt-get -q install -y --no-install-recommends libmariadb-dev cmake + apt-get -q install -y --no-install-recommends libmysqlclient-dev cmake COPY --from=planner /app/recipe.json recipe.json RUN cargo chef cook --release --no-default-features --features=syncstorage-db/$DATABASE_BACKEND --recipe-path recipe.json @@ -37,7 +37,7 @@ RUN \ # related: # https://dev.mysql.com/doc/mysql-apt-repo-quick-guide/en/#repo-qg-apt-repo-manual-setup apt-get -q update && \ - apt-get -q install -y --no-install-recommends libmariadb-dev cmake golang-go python3-dev python3-pip python3-setuptools python3-wheel && \ + apt-get -q install -y --no-install-recommends libmysqlclient-dev cmake golang-go python3-dev python3-pip python3-setuptools python3-wheel && \ pip3 install -r requirements.txt && \ rm -rf /var/lib/apt/lists/* From a9a6c2200f9c58f26f3fd60f70c21a62e393630d Mon Sep 17 00:00:00 2001 From: jrconlin Date: Tue, 16 Jan 2024 16:47:40 -0800 Subject: [PATCH 11/17] f fix logic bug --- tools/tokenserver/purge_old_records.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/tokenserver/purge_old_records.py b/tools/tokenserver/purge_old_records.py index 18a9f4ed02..b3b95fa687 100644 --- a/tools/tokenserver/purge_old_records.py +++ b/tools/tokenserver/purge_old_records.py @@ -80,7 +80,7 @@ def purge_old_records(secret, grace_period=-1, max_per_loop=10, max_offset=0, database.delete_user_record(row.uid) elif not row.downed: logger.info("Purging uid %s on %s", row.uid, row.node) - if settings and not settings.dryrun: + if settings and settings.dryrun: pass else: delete_service_data( From 689efc3d7b0a4e9f2e225159710ff514854aa4cb Mon Sep 17 00:00:00 2001 From: jrconlin Date: Wed, 17 Jan 2024 15:22:01 -0800 Subject: [PATCH 12/17] f add tests for `force` and `dryrun` --- tools/tokenserver/test_purge_old_records.py | 50 +++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/tools/tokenserver/test_purge_old_records.py b/tools/tokenserver/test_purge_old_records.py index 370e5dfff8..f9183298bb 100644 --- a/tools/tokenserver/test_purge_old_records.py +++ b/tools/tokenserver/test_purge_old_records.py @@ -56,6 +56,18 @@ def tearDown(self): del self.service_requests[:] + def test_settings(self, args:dict[str, any]=dict()): + class Settings(object): + pass + + settings = Settings() + setattr(settings, "force", args.get("force", False)) + setattr(settings, "dryrun", args.get("dryrun", False)) + setattr(settings, "max_records", args.get("max_records", 20)) + + return settings + + @classmethod def tearDownClass(cls): cls.service.shutdown() @@ -138,3 +150,41 @@ def test_purging_is_not_done_on_downed_nodes(self): user_records = list(self.database.get_user_records(email)) self.assertEqual(len(user_records), 1) self.assertEqual(len(self.service_requests), 1) + + def test_force(self): + # Make some old user records. + node_secret = "SECRET" + email = "test@mozilla.com" + user = self.database.allocate_user(email, client_state="aa") + self.database.update_user(user, client_state="bb") + user_records = list(self.database.get_user_records(email)) + self.assertEqual(len(user_records), 2) + + # With the node down, we should be able to purge any records. + self.database.update_node(self.service_node, downed=1) + + settings = self.test_settings({"force":True}) + self.assertTrue(purge_old_records(node_secret, grace_period=0, settings=settings)) + + user_records = list(self.database.get_user_records(email)) + self.assertEqual(len(user_records), 1) + self.assertEqual(len(self.service_requests), 1) + + def test_dry_run(self): + # Make some old user records. + node_secret = "SECRET" + email = "test@mozilla.com" + user = self.database.allocate_user(email, client_state="aa") + self.database.update_user(user, client_state="bb") + user_records = list(self.database.get_user_records(email)) + self.assertEqual(len(user_records), 2) + + self.database.update_node(self.service_node, downed=1) + + # Don't actually perform anything destructive. + settings = self.test_settings({"dryrun": True}) + self.assertTrue(purge_old_records(node_secret, grace_period=0, settings=settings)) + + user_records = list(self.database.get_user_records(email)) + self.assertEqual(len(user_records), 2) + self.assertEqual(len(self.service_requests), 0) From 36a7cdd7b7a3719b168824d814fc3d81fafae15c Mon Sep 17 00:00:00 2001 From: jrconlin Date: Wed, 17 Jan 2024 15:24:36 -0800 Subject: [PATCH 13/17] f fmt --- tools/tokenserver/test_purge_old_records.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/tools/tokenserver/test_purge_old_records.py b/tools/tokenserver/test_purge_old_records.py index f9183298bb..6cdfa179b8 100644 --- a/tools/tokenserver/test_purge_old_records.py +++ b/tools/tokenserver/test_purge_old_records.py @@ -56,7 +56,7 @@ def tearDown(self): del self.service_requests[:] - def test_settings(self, args:dict[str, any]=dict()): + def test_settings(self, args: dict[str, any] = dict()): class Settings(object): pass @@ -67,7 +67,6 @@ class Settings(object): return settings - @classmethod def tearDownClass(cls): cls.service.shutdown() @@ -163,8 +162,13 @@ def test_force(self): # With the node down, we should be able to purge any records. self.database.update_node(self.service_node, downed=1) - settings = self.test_settings({"force":True}) - self.assertTrue(purge_old_records(node_secret, grace_period=0, settings=settings)) + settings = self.test_settings({"force": True}) + self.assertTrue( + purge_old_records( + node_secret, + grace_period=0, + settings=settings) + ) user_records = list(self.database.get_user_records(email)) self.assertEqual(len(user_records), 1) @@ -183,7 +187,12 @@ def test_dry_run(self): # Don't actually perform anything destructive. settings = self.test_settings({"dryrun": True}) - self.assertTrue(purge_old_records(node_secret, grace_period=0, settings=settings)) + self.assertTrue( + purge_old_records( + node_secret, + grace_period=0, + settings=settings) + ) user_records = list(self.database.get_user_records(email)) self.assertEqual(len(user_records), 2) From de4a77b0bac5968389f73384ab07454be69fe711 Mon Sep 17 00:00:00 2001 From: jrconlin Date: Wed, 17 Jan 2024 15:35:09 -0800 Subject: [PATCH 14/17] f audit --- .cargo/audit.toml | 1 + Cargo.lock | 262 +++++++++++++++++++++++----------------------- 2 files changed, 130 insertions(+), 133 deletions(-) diff --git a/.cargo/audit.toml b/.cargo/audit.toml index 10ee95c3fc..cdfc329fe4 100644 --- a/.cargo/audit.toml +++ b/.cargo/audit.toml @@ -9,4 +9,5 @@ ignore = [ "RUSTSEC-2023-0034", # Bound by actix-http 2.2, Reqwest 0.10 "RUSTSEC-2023-0044", # Bound to native-tls 0.2.11, request 0.10.10, hyper-tls 0.4.3 "RUSTSEC-2023-0052", # Bound by reqwest, various tls libs + "RUSTSEC-2024-0003", # Bound by reqwest, actix-http ] diff --git a/Cargo.lock b/Cargo.lock index ea3c7e6f8b..e0ced0ca17 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -90,7 +90,7 @@ dependencies = [ "pin-project 1.1.3", "rand 0.7.3", "regex", - "serde 1.0.194", + "serde 1.0.195", "serde_json", "serde_urlencoded", "sha-1", @@ -118,7 +118,7 @@ dependencies = [ "http", "log", "regex", - "serde 1.0.194", + "serde 1.0.195", ] [[package]] @@ -257,7 +257,7 @@ dependencies = [ "mime", "pin-project 1.1.3", "regex", - "serde 1.0.194", + "serde 1.0.195", "serde_json", "serde_urlencoded", "socket2 0.3.19", @@ -364,7 +364,7 @@ version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47e4f2b81832e72834d7518d8487a0396a28cc408186a2e8854c0f98011faf12" dependencies = [ - "serde 1.0.194", + "serde 1.0.195", "serde_json", ] @@ -376,7 +376,7 @@ checksum = "c980ee35e870bd1a4d2c8294d4c04d0499e67bca1e4b5cefcc693c2fa00caea9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.47", + "syn 2.0.48", ] [[package]] @@ -415,7 +415,7 @@ dependencies = [ "mime", "percent-encoding 2.3.1", "rand 0.7.3", - "serde 1.0.194", + "serde 1.0.195", "serde_json", "serde_urlencoded", ] @@ -455,9 +455,9 @@ checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" [[package]] name = "base64" -version = "0.21.5" +version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" [[package]] name = "bindgen" @@ -490,9 +490,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.4.1" +version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" +checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf" [[package]] name = "block-buffer" @@ -681,7 +681,7 @@ checksum = "19b076e143e1d9538dde65da30f8481c2a6c44040edb8e02b9bf1351edb92ce3" dependencies = [ "lazy_static", "nom 5.1.3", - "serde 1.0.194", + "serde 1.0.195", ] [[package]] @@ -693,7 +693,7 @@ dependencies = [ "lazy_static", "nom 5.1.3", "rust-ini", - "serde 1.0.194", + "serde 1.0.195", "serde-hjson", "serde_json", "toml", @@ -747,9 +747,9 @@ checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" [[package]] name = "cpufeatures" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce420fe07aecd3e67c5f910618fe65e94158f6dcc0adf44e00d69ce2bdfe0fd0" +checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" dependencies = [ "libc", ] @@ -765,12 +765,11 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.10" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82a9b73a36529d9c47029b9fb3a6f0ea3cc916a261195352ba19e770fc1748b2" +checksum = "176dc175b78f56c0f321911d9c8eb2b77a78a4860b9c19db83835fea1a46649b" dependencies = [ - "cfg-if 1.0.0", - "crossbeam-utils 0.8.18", + "crossbeam-utils 0.8.19", ] [[package]] @@ -797,12 +796,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.18" +version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3a430a770ebd84726f584a90ee7f020d28db52c6d02138900f22341f866d39c" -dependencies = [ - "cfg-if 1.0.0", -] +checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" [[package]] name = "crypto-common" @@ -853,7 +849,7 @@ dependencies = [ "config 0.10.1", "crossbeam-queue", "num_cpus", - "serde 1.0.194", + "serde 1.0.195", "tokio 0.2.25", ] @@ -863,7 +859,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef552e6f588e446098f6ba40d89ac146c8c7b64aade83c051ee00bb5d2bc18d" dependencies = [ - "serde 1.0.194", + "serde 1.0.195", "uuid", ] @@ -994,7 +990,7 @@ checksum = "7f3f119846c823f9eafcf953a8f6ffb6ed69bf6240883261a7f13b634579a51f" dependencies = [ "lazy_static", "regex", - "serde 1.0.194", + "serde 1.0.195", "strsim 0.10.0", ] @@ -1069,7 +1065,7 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c138974f9d5e7fe373eb04df7cae98833802ae4b11c24ac7039a21d5af4b26c" dependencies = [ - "serde 1.0.194", + "serde 1.0.195", ] [[package]] @@ -1218,7 +1214,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.47", + "syn 2.0.48", ] [[package]] @@ -1284,9 +1280,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f" +checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5" dependencies = [ "cfg-if 1.0.0", "libc", @@ -1369,9 +1365,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.22" +version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d6250322ef6e60f93f9a2162799302cd6f68f79f6e5d85c8c16f14d1d958178" +checksum = "bb2c4422095b67ee78da96fbb51a4cc413b3b25883c7717ff7ca1ab31022c9c9" dependencies = [ "bytes 1.5.0", "fnv", @@ -1430,9 +1426,9 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" +checksum = "5d3d0e0f38255e7fa3cf31335b3a56f05febd18025f4db5ef7a0cfb4f8da651f" [[package]] name = "hex" @@ -1568,7 +1564,7 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "h2 0.3.22", + "h2 0.3.24", "http", "http-body 0.4.6", "httparse", @@ -1763,7 +1759,7 @@ version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0bad00257d07be169d870ab665980b06cdb366d792ad690bf2e76876dc503455" dependencies = [ - "hermit-abi 0.3.3", + "hermit-abi 0.3.4", "rustix", "windows-sys 0.52.0", ] @@ -1782,9 +1778,9 @@ checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" [[package]] name = "js-sys" -version = "0.3.66" +version = "0.3.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cee9c64da59eae3b50095c18d3e74f8b73c0b86d2792824ff01bbce68ba229ca" +checksum = "9a1d36f1235bc969acba30b7f5990b864423a6068a10f7c90ae8f0112e3a59d1" dependencies = [ "wasm-bindgen", ] @@ -1832,9 +1828,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.151" +version = "0.2.152" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "302d7ab3130588088d277783b1e2d2e10c9e9e4a16dd9050e6ec93fb3e7048f4" +checksum = "13e3bf6590cbc649f4d1a3eefc9d5d6eb746f5200ffb04e5e142700b8faa56e7" [[package]] name = "libloading" @@ -1852,16 +1848,16 @@ version = "0.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85c833ca1e66078851dba29046874e38f08b2c883700aa29a03ddd3b23814ee8" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", "libc", "redox_syscall 0.4.1", ] [[package]] name = "libz-sys" -version = "1.1.12" +version = "1.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d97137b25e321a73eef1418d1d5d2eda4d77e12813f8e6dead84bc52c5870a7b" +checksum = "295c17e837573c8c821dbaeb3cceb3d745ad082f7572191409e69cbc1b3fd050" dependencies = [ "cc", "libc", @@ -1877,9 +1873,9 @@ checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" [[package]] name = "linux-raw-sys" -version = "0.4.12" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4cd1a83af159aa67994778be9070f0ae1bd732942279cabb14f86f986a21456" +checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" [[package]] name = "lock_api" @@ -2146,7 +2142,7 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "hermit-abi 0.3.3", + "hermit-abi 0.3.4", "libc", ] @@ -2186,7 +2182,7 @@ version = "0.10.62" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8cde4d2d9200ad5909f8dac647e29482e07c3a35de8a13fce7c9c7747ad9f671" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", "cfg-if 1.0.0", "foreign-types", "libc", @@ -2203,7 +2199,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.47", + "syn 2.0.48", ] [[package]] @@ -2231,7 +2227,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "006e42d5b888366f1880eda20371fedde764ed2213dc8496f49622fa0c99cd5e" dependencies = [ "log", - "serde 1.0.194", + "serde 1.0.195", "winapi 0.3.9", ] @@ -2338,7 +2334,7 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ "proc-macro2", "quote", - "syn 2.0.47", + "syn 2.0.48", ] [[package]] @@ -2361,9 +2357,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkg-config" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69d3587f8a9e599cc7ec2c00e331f71c4e69a5f9a4b8a6efd5b07466b9736f9a" +checksum = "2900ede94e305130c13ddd391e0ab7cbaeb783945ae07a279c268cb05109c6cb" [[package]] name = "powerfmt" @@ -2409,9 +2405,9 @@ checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" [[package]] name = "proc-macro2" -version = "1.0.75" +version = "1.0.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "907a61bd0f64c2f29cd1cf1dc34d05176426a3f504a78010f08416ddb7b13708" +checksum = "95fc56cda0b5c3325f5fbbd7ff9fda9e02bb00bb3dac51252d2f1bfa1cb8cc8c" dependencies = [ "unicode-ident", ] @@ -2424,9 +2420,9 @@ checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94" [[package]] name = "pyo3" -version = "0.20.1" +version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e82ad98ce1991c9c70c3464ba4187337b9c45fcbbb060d46dca15f0c075e14e2" +checksum = "9a89dc7a5850d0e983be1ec2a463a171d20990487c3cfcd68b5363f1ee3d6fe0" dependencies = [ "cfg-if 1.0.0", "indoc", @@ -2441,9 +2437,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.20.1" +version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5503d0b3aee2c7a8dbb389cd87cd9649f675d4c7f60ca33699a3e3859d81a891" +checksum = "07426f0d8fe5a601f26293f300afd1a7b1ed5e78b2a705870c5f30893c5163be" dependencies = [ "once_cell", "target-lexicon", @@ -2451,9 +2447,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.20.1" +version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18a79e8d80486a00d11c0dcb27cd2aa17c022cc95c677b461f01797226ba8f41" +checksum = "dbb7dec17e17766b46bca4f1a4215a85006b4c2ecde122076c562dd058da6cf1" dependencies = [ "libc", "pyo3-build-config", @@ -2461,26 +2457,26 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.20.1" +version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f4b0dc7eaa578604fab11c8c7ff8934c71249c61d4def8e272c76ed879f03d4" +checksum = "05f738b4e40d50b5711957f142878cfa0f28e054aa0ebdfc3fd137a843f74ed3" dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn 2.0.47", + "syn 2.0.48", ] [[package]] name = "pyo3-macros-backend" -version = "0.20.1" +version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "816a4f709e29ddab2e3cdfe94600d554c5556cad0ddfeea95c47b580c3247fa4" +checksum = "0fc910d4851847827daf9d6cdd4a823fbdaab5b8818325c5e97a86da79e8881f" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.47", + "syn 2.0.48", ] [[package]] @@ -2568,7 +2564,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom 0.2.11", + "getrandom 0.2.12", ] [[package]] @@ -2604,7 +2600,7 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a18479200779601e498ada4e8c1e1f50e3ee19deb0259c25825a98b5603b2cb4" dependencies = [ - "getrandom 0.2.11", + "getrandom 0.2.12", "libredox", "thiserror", ] @@ -2664,7 +2660,7 @@ dependencies = [ "percent-encoding 2.3.1", "pin-project-lite 0.2.13", "rustls", - "serde 1.0.194", + "serde 1.0.195", "serde_json", "serde_urlencoded", "tokio 0.2.25", @@ -2684,12 +2680,12 @@ version = "0.11.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37b1ae8d9ac08420c66222fb9096fc5de435c3c48542bc5336c51892cffafb41" dependencies = [ - "base64 0.21.5", + "base64 0.21.7", "bytes 1.5.0", "encoding_rs", "futures-core", "futures-util", - "h2 0.3.22", + "h2 0.3.24", "http", "http-body 0.4.6", "hyper 0.14.28", @@ -2702,7 +2698,7 @@ dependencies = [ "once_cell", "percent-encoding 2.3.1", "pin-project-lite 0.2.13", - "serde 1.0.194", + "serde 1.0.195", "serde_json", "serde_urlencoded", "system-configuration", @@ -2779,11 +2775,11 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.28" +version = "0.38.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72e572a5e8ca657d7366229cdde4bd14c4eb5499a9573d4d366fe1b599daa316" +checksum = "322394588aaf33c24007e8bb3238ee3e4c5c09c084ab32bc73890b99ff326bca" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", "errno", "libc", "linux-raw-sys", @@ -2957,7 +2953,7 @@ dependencies = [ "once_cell", "rand 0.8.5", "sentry-types", - "serde 1.0.194", + "serde 1.0.195", "serde_json", ] @@ -3003,7 +2999,7 @@ dependencies = [ "debugid", "hex", "rand 0.8.5", - "serde 1.0.194", + "serde 1.0.195", "serde_json", "thiserror", "time 0.3.31", @@ -3019,9 +3015,9 @@ checksum = "9dad3f759919b92c3068c696c15c3d17238234498bbdcc80f2c469606f948ac8" [[package]] name = "serde" -version = "1.0.194" +version = "1.0.195" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b114498256798c94a0689e1a15fec6005dee8ac1f41de56404b67afc2a4b773" +checksum = "63261df402c67811e9ac6def069e4786148c4563f4b50fd4bf30aa370d626b02" dependencies = [ "serde_derive", ] @@ -3040,24 +3036,24 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.194" +version = "1.0.195" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3385e45322e8f9931410f01b3031ec534c3947d0e94c18049af4d9f9907d4e0" +checksum = "46fe8f8603d81ba86327b23a2e9cdf49e1255fb94a4c5f297f6ee0547178ea2c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.47", + "syn 2.0.48", ] [[package]] name = "serde_json" -version = "1.0.110" +version = "1.0.111" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fbd975230bada99c8bb618e0c365c2eefa219158d5c6c29610fd09ff1833257" +checksum = "176e46fa42316f18edd598015a5166857fc835ec732f5215eac6b7bdbf0a84f4" dependencies = [ "itoa 1.0.10", "ryu", - "serde 1.0.194", + "serde 1.0.195", ] [[package]] @@ -3069,7 +3065,7 @@ dependencies = [ "form_urlencoded", "itoa 1.0.10", "ryu", - "serde 1.0.194", + "serde 1.0.195", ] [[package]] @@ -3178,7 +3174,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f400f1c5db96f1f52065e8931ca0c524cceb029f7537c9e6d5424488ca137ca0" dependencies = [ "chrono", - "serde 1.0.194", + "serde 1.0.195", "serde_json", "slog", ] @@ -3220,9 +3216,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.11.2" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970" +checksum = "2593d31f82ead8df961d8bd23a64c2ccf2eb5dd34b0a34bfb4dd54011c72009e" [[package]] name = "socket2" @@ -3298,7 +3294,7 @@ checksum = "c87a60a40fccc84bef0652345bbbbbe20a605bf5d0ce81719fc476f5c03b50ef" dependencies = [ "proc-macro2", "quote", - "serde 1.0.194", + "serde 1.0.195", "serde_derive", "syn 1.0.109", ] @@ -3312,7 +3308,7 @@ dependencies = [ "base-x", "proc-macro2", "quote", - "serde 1.0.194", + "serde 1.0.195", "serde_derive", "serde_json", "sha1", @@ -3356,9 +3352,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.47" +version = "2.0.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1726efe18f42ae774cc644f330953a5e7b3c3003d3edcecf18850fe9d4dd9afb" +checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f" dependencies = [ "proc-macro2", "quote", @@ -3375,7 +3371,7 @@ dependencies = [ "actix-web", "async-trait", "backtrace", - "base64 0.21.5", + "base64 0.21.7", "cadence", "chrono", "docopt", @@ -3393,7 +3389,7 @@ dependencies = [ "reqwest 0.10.10", "sentry", "sentry-backtrace", - "serde 1.0.194", + "serde 1.0.195", "serde_derive", "serde_json", "sha2", @@ -3430,7 +3426,7 @@ dependencies = [ "cadence", "futures 0.3.30", "hkdf", - "serde 1.0.194", + "serde 1.0.195", "serde_json", "sha2", "slog", @@ -3457,7 +3453,7 @@ version = "0.14.4" dependencies = [ "config 0.11.0", "num_cpus", - "serde 1.0.194", + "serde 1.0.195", "slog-scope", "syncserver-common", "syncstorage-settings", @@ -3500,7 +3496,7 @@ dependencies = [ "futures 0.3.30", "http", "lazy_static", - "serde 1.0.194", + "serde 1.0.195", "serde_json", "syncserver-common", "syncserver-db-common", @@ -3513,7 +3509,7 @@ version = "0.14.4" dependencies = [ "async-trait", "backtrace", - "base64 0.21.5", + "base64 0.21.7", "diesel", "diesel_logger", "diesel_migrations", @@ -3535,7 +3531,7 @@ name = "syncstorage-settings" version = "0.14.4" dependencies = [ "rand 0.8.5", - "serde 1.0.194", + "serde 1.0.195", "syncserver-common", "time 0.3.31", ] @@ -3626,9 +3622,9 @@ dependencies = [ [[package]] name = "termcolor" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff1bc3d3f05aff0403e8ac0d92ced918ec05b666a43f83297ccef5bea8a3d449" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" dependencies = [ "winapi-util", ] @@ -3659,7 +3655,7 @@ checksum = "fa0faa943b50f3db30a20aa7e265dbc66076993efed8463e8de414e5d06d3471" dependencies = [ "proc-macro2", "quote", - "syn 2.0.47", + "syn 2.0.48", ] [[package]] @@ -3707,7 +3703,7 @@ dependencies = [ "libc", "num_threads", "powerfmt", - "serde 1.0.194", + "serde 1.0.195", "time-core", "time-macros 0.2.16", ] @@ -3775,7 +3771,7 @@ dependencies = [ "mockito", "pyo3", "reqwest 0.10.10", - "serde 1.0.194", + "serde 1.0.195", "serde_json", "syncserver-common", "tokenserver-common", @@ -3789,7 +3785,7 @@ version = "0.14.4" dependencies = [ "actix-web", "backtrace", - "serde 1.0.194", + "serde 1.0.195", "serde_json", "syncserver-common", "thiserror", @@ -3807,7 +3803,7 @@ dependencies = [ "env_logger 0.10.1", "futures 0.3.30", "http", - "serde 1.0.194", + "serde 1.0.195", "serde_derive", "serde_json", "slog-scope", @@ -3824,7 +3820,7 @@ dependencies = [ name = "tokenserver-settings" version = "0.14.4" dependencies = [ - "serde 1.0.194", + "serde 1.0.195", "tokenserver-common", ] @@ -3943,7 +3939,7 @@ version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" dependencies = [ - "serde 1.0.194", + "serde 1.0.195", ] [[package]] @@ -4063,9 +4059,9 @@ dependencies = [ [[package]] name = "unicode-bidi" -version = "0.3.14" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f2528f27a9eb2b21e69c95319b30bd0efd85d09c379741b0f78ea1d86be2416" +checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" [[package]] name = "unicode-ident" @@ -4106,7 +4102,7 @@ version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8cdd25c339e200129fe4de81451814e5228c9b771d57378817d6117cc2b3f97" dependencies = [ - "base64 0.21.5", + "base64 0.21.7", "log", "native-tls", "once_cell", @@ -4133,7 +4129,7 @@ dependencies = [ "form_urlencoded", "idna 0.5.0", "percent-encoding 2.3.1", - "serde 1.0.194", + "serde 1.0.195", ] [[package]] @@ -4148,8 +4144,8 @@ version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e395fcf16a7a3d8127ec99782007af141946b4795001f876d54fb0d55978560" dependencies = [ - "getrandom 0.2.11", - "serde 1.0.194", + "getrandom 0.2.12", + "serde 1.0.195", ] [[package]] @@ -4161,7 +4157,7 @@ dependencies = [ "idna 0.4.0", "lazy_static", "regex", - "serde 1.0.194", + "serde 1.0.195", "serde_derive", "serde_json", "url 2.5.0", @@ -4250,36 +4246,36 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.89" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ed0d4f68a3015cc185aff4db9506a015f4b96f95303897bfa23f846db54064e" +checksum = "b1223296a201415c7fad14792dbefaace9bd52b62d33453ade1c5b5f07555406" dependencies = [ "cfg-if 1.0.0", - "serde 1.0.194", + "serde 1.0.195", "serde_json", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.89" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b56f625e64f3a1084ded111c4d5f477df9f8c92df113852fa5a374dbda78826" +checksum = "fcdc935b63408d58a32f8cc9738a0bffd8f05cc7c002086c6ef20b7312ad9dcd" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 2.0.47", + "syn 2.0.48", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.39" +version = "0.4.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac36a15a220124ac510204aec1c3e5db8a22ab06fd6706d881dc6149f8ed9a12" +checksum = "bde2032aeb86bdfaecc8b261eef3cba735cc426c1f3a3416d1e0791be95fc461" dependencies = [ "cfg-if 1.0.0", "js-sys", @@ -4289,9 +4285,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.89" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0162dbf37223cd2afce98f3d0785506dcb8d266223983e4b5b525859e6e182b2" +checksum = "3e4c238561b2d428924c49815533a8b9121c664599558a5d9ec51f8a1740a999" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -4299,28 +4295,28 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.89" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0eb82fcb7930ae6219a7ecfd55b217f5f0893484b7a13022ebb2b2bf20b5283" +checksum = "bae1abb6806dc1ad9e560ed242107c0f6c84335f1749dd4e8ddb012ebd5e25a7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.47", + "syn 2.0.48", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.89" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ab9b36309365056cd639da3134bf87fa8f3d86008abf99e612384a6eecd459f" +checksum = "4d91413b1c31d7539ba5ef2451af3f0b833a005eb27a631cec32bc0635a8602b" [[package]] name = "web-sys" -version = "0.3.66" +version = "0.3.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50c24a44ec86bb68fbecd1b3efed7e85ea5621b39b35ef2766b66cd984f8010f" +checksum = "58cd2333b6e0be7a39605f0e255892fd7418a682d8da8fe042fe25128794d2ed" dependencies = [ "js-sys", "wasm-bindgen", From e944f2ea5d011ece508d9a87fcb56635a1b9cb77 Mon Sep 17 00:00:00 2001 From: jrconlin Date: Wed, 14 Feb 2024 15:18:54 -0800 Subject: [PATCH 15/17] f trap error on `tearDownClass` --- tools/integration_tests/tokenserver/test_e2e.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/integration_tests/tokenserver/test_e2e.py b/tools/integration_tests/tokenserver/test_e2e.py index 85eab417cc..932e9d6c06 100644 --- a/tools/integration_tests/tokenserver/test_e2e.py +++ b/tools/integration_tests/tokenserver/test_e2e.py @@ -69,7 +69,10 @@ def setUpClass(cls): @classmethod def tearDownClass(cls): cls.acct.clear() - cls.client.destroy_account(cls.acct.email, cls.fxa_password) + try: + cls.client.destroy_account(cls.acct.email, cls.fxa_password) + except Exception as ex: + print(f"warning: Encountered error when cleaning up: {ex}") @staticmethod def _generate_password(): From 7bced33917755c38d05a929067eb0cf7a8303cee Mon Sep 17 00:00:00 2001 From: jrconlin Date: Wed, 14 Feb 2024 15:54:07 -0800 Subject: [PATCH 16/17] f add comment --- tools/integration_tests/tokenserver/test_e2e.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tools/integration_tests/tokenserver/test_e2e.py b/tools/integration_tests/tokenserver/test_e2e.py index 932e9d6c06..c0aa619d40 100644 --- a/tools/integration_tests/tokenserver/test_e2e.py +++ b/tools/integration_tests/tokenserver/test_e2e.py @@ -16,6 +16,7 @@ from cryptography.hazmat.backends import default_backend from fxa.core import Client from fxa.oauth import Client as OAuthClient +from fxa.errors import ServerError from fxa.tests.utils import TestEmailAccount from hashlib import sha256 @@ -69,9 +70,14 @@ def setUpClass(cls): @classmethod def tearDownClass(cls): cls.acct.clear() + # A teardown of some of the tests can produce a 401 error because + # of a race condition, where the record had already been removed. + # This causes `destroy_account` to return an error if it attempts + # to parse the invalid JSON response. + # This traps for that event. try: cls.client.destroy_account(cls.acct.email, cls.fxa_password) - except Exception as ex: + except ServerError as ex: print(f"warning: Encountered error when cleaning up: {ex}") @staticmethod From 7f67ba0f31eed1ae231765c2f1d8d9a49ee46844 Mon Sep 17 00:00:00 2001 From: jrconlin Date: Mon, 26 Feb 2024 13:07:13 -0800 Subject: [PATCH 17/17] f r's --- tools/tokenserver/purge_old_records.py | 49 +++++++++++++-------- tools/tokenserver/test_purge_old_records.py | 17 +------ 2 files changed, 32 insertions(+), 34 deletions(-) diff --git a/tools/tokenserver/purge_old_records.py b/tools/tokenserver/purge_old_records.py index b3b95fa687..81279c92d2 100644 --- a/tools/tokenserver/purge_old_records.py +++ b/tools/tokenserver/purge_old_records.py @@ -36,7 +36,8 @@ def purge_old_records(secret, grace_period=-1, max_per_loop=10, max_offset=0, - request_timeout=60, settings=None): + max_records=0, request_timeout=60, dryrun=False, + force=False): """Purge old records from the database. This function queries all of the old user records in the database, deletes @@ -67,6 +68,7 @@ def purge_old_records(secret, grace_period=-1, max_per_loop=10, max_offset=0, break if rows == previous_list: raise Exception("Loop detected") + previous_list = rows logger.info("Fetched %d rows at offset %d", len(rows), offset) counter = 0 for row in rows: @@ -76,37 +78,36 @@ def purge_old_records(secret, grace_period=-1, max_per_loop=10, max_offset=0, if row.node is None: logger.info("Deleting user record for uid %s on %s", row.uid, row.node) - if settings and not settings.dryrun: + if not dryrun: database.delete_user_record(row.uid) + # NOTE: only delete_user+service_data calls count + # against the counter elif not row.downed: logger.info("Purging uid %s on %s", row.uid, row.node) - if settings and settings.dryrun: - pass - else: + if not dryrun: delete_service_data( row, secret, timeout=request_timeout, - settings=settings) + dryrun=dryrun) database.delete_user_record(row.uid) counter += 1 - elif settings and settings.force: + elif force: logger.info( "Forcing tokenserver record delete: " f"{row.uid} on {row.node}" ) - if not settings.dryrun: + if not dryrun: delete_service_data( row, secret, timeout=request_timeout, - settings=settings) + dryrun=dryrun) database.delete_user_record(row.uid) counter += 1 - if settings and settings.max_records: - if counter >= settings.max_records: - logger.info("Reached max_records, exiting") - return True + if max_records and counter >= max_records: + logger.info("Reached max_records, exiting") + return True if len(rows) < max_per_loop: break except Exception: @@ -117,7 +118,7 @@ def purge_old_records(secret, grace_period=-1, max_per_loop=10, max_offset=0, return True -def delete_service_data(user, secret, timeout=60, settings=None): +def delete_service_data(user, secret, timeout=60, dryrun=False): """Send a data-deletion request to the user's service node. This is a little bit of hackery to cause the user's service node to @@ -136,7 +137,10 @@ def delete_service_data(user, secret, timeout=60, settings=None): secret = tokenlib.get_derived_secret(token, secret=secret) endpoint = PATTERN.format(uid=user.uid, node=user.node) auth = HawkAuth(token, secret) - if settings and settings.dryrun: + if dryrun: + # NOTE: this function currently isn't called during dryrun + # (but we may want to add logging here and change that in the + # future) return resp = requests.delete(endpoint, auth=auth, timeout=timeout) if resp.status_code >= 400 and resp.status_code != 404: @@ -173,6 +177,9 @@ def main(args=None): # selects will return zero rows. Choose this value accordingly. parser.add_option("", "--max-offset", type="int", default=0, help="Use random offset from 0 to max_offset") + parser.add_option("", "--max-records", type="int", default=0, + help="Max number of syncstorage data purges to " + "make") parser.add_option("", "--request-timeout", type="int", default=60, help="Timeout for service deletion requests") parser.add_option("", "--oneshot", action="store_true", @@ -182,8 +189,8 @@ def main(args=None): parser.add_option("", "--dryrun", action="store_true", help="Don't do destructive things") parser.add_option("", "--force", action="store_true", - help="force record to be deleted from TS db," - " even if node is down") + help="Force syncstorage data to be purged, even " + "if the user's node is marked as down") opts, args = parser.parse_args(args) if len(args) != 2: @@ -198,8 +205,10 @@ def main(args=None): grace_period=opts.grace_period, max_per_loop=opts.max_per_loop, max_offset=opts.max_offset, + max_records=opts.max_records, request_timeout=opts.request_timeout, - settings=opts) + dryrun=opts.dryrun, + force=opts.force) if not opts.oneshot: while True: # Randomize sleep interval +/- thirty percent to desynchronize @@ -211,8 +220,10 @@ def main(args=None): purge_old_records(grace_period=opts.grace_period, max_per_loop=opts.max_per_loop, max_offset=opts.max_offset, + max_records=opts.max_records, request_timeout=opts.request_timeout, - settings=opts) + dryrun=opts.dryrun, + force=opts.force) return 0 diff --git a/tools/tokenserver/test_purge_old_records.py b/tools/tokenserver/test_purge_old_records.py index 6cdfa179b8..80d506935d 100644 --- a/tools/tokenserver/test_purge_old_records.py +++ b/tools/tokenserver/test_purge_old_records.py @@ -56,17 +56,6 @@ def tearDown(self): del self.service_requests[:] - def test_settings(self, args: dict[str, any] = dict()): - class Settings(object): - pass - - settings = Settings() - setattr(settings, "force", args.get("force", False)) - setattr(settings, "dryrun", args.get("dryrun", False)) - setattr(settings, "max_records", args.get("max_records", 20)) - - return settings - @classmethod def tearDownClass(cls): cls.service.shutdown() @@ -162,12 +151,11 @@ def test_force(self): # With the node down, we should be able to purge any records. self.database.update_node(self.service_node, downed=1) - settings = self.test_settings({"force": True}) self.assertTrue( purge_old_records( node_secret, grace_period=0, - settings=settings) + force=True) ) user_records = list(self.database.get_user_records(email)) @@ -186,12 +174,11 @@ def test_dry_run(self): self.database.update_node(self.service_node, downed=1) # Don't actually perform anything destructive. - settings = self.test_settings({"dryrun": True}) self.assertTrue( purge_old_records( node_secret, grace_period=0, - settings=settings) + dryrun=True) ) user_records = list(self.database.get_user_records(email))