From b538f33d59b756d052089cec717f3a263bbd917d Mon Sep 17 00:00:00 2001 From: David Robertson Date: Thu, 6 Jan 2022 16:27:32 +0000 Subject: [PATCH 1/7] Optionally use an on-disk sqlite db in tests When debugging a test it is sometimes useful to inspect the state of the DB. This is not easy when the db is in-memory: one cannot attach the sqlite CLI to another process's DB. With this change, if SYNAPSE_TEST_SQLITE_DB_LOCATION is set, we try to write the DB to disk at that location (deleting any existing file as necessary). Quick and dirty. --- tests/server.py | 11 ++++++++++- tests/utils.py | 4 ++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/tests/server.py b/tests/server.py index ca2b7a5b9771..e94796b5625b 100644 --- a/tests/server.py +++ b/tests/server.py @@ -14,6 +14,7 @@ import hashlib import json import logging +import os import time import uuid import warnings @@ -71,6 +72,7 @@ POSTGRES_HOST, POSTGRES_PASSWORD, POSTGRES_USER, + SQLITE_TEST_DB_LOCATION, USE_POSTGRES_FOR_TESTS, MockClock, default_config, @@ -739,9 +741,16 @@ def setup_test_homeserver( }, } else: + if SQLITE_TEST_DB_LOCATION != ":memory:": + # nuke the DB on disk + try: + os.remove(SQLITE_TEST_DB_LOCATION) + except FileNotFoundError: + pass + database_config = { "name": "sqlite3", - "args": {"database": ":memory:", "cp_min": 1, "cp_max": 1}, + "args": {"database": SQLITE_TEST_DB_LOCATION, "cp_min": 1, "cp_max": 1}, } if "db_txn_limit" in kwargs: diff --git a/tests/utils.py b/tests/utils.py index 6d013e851845..4389264c83b9 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -42,6 +42,10 @@ POSTGRES_PASSWORD = os.environ.get("SYNAPSE_POSTGRES_PASSWORD", None) POSTGRES_BASE_DB = "_synapse_unit_tests_base_%s" % (os.getpid(),) +# When debugging a specific test, it's occasionally useful to write the +# DB to /tmp and query it with the sqlite CLI. +SQLITE_TEST_DB_LOCATION = os.environ.get("SYNAPSE_TEST_SQLITE_DB_LOCATION", ":memory:") + # the dbname we will connect to in order to create the base database. POSTGRES_DBNAME_FOR_INITIAL_CREATE = "postgres" From 9114f2baeeeab8ba2542c36039fc174cd9b5f616 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Thu, 6 Jan 2022 18:08:52 +0000 Subject: [PATCH 2/7] Changelog --- changelog.d/11702.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/11702.misc diff --git a/changelog.d/11702.misc b/changelog.d/11702.misc new file mode 100644 index 000000000000..fc1069cae034 --- /dev/null +++ b/changelog.d/11702.misc @@ -0,0 +1 @@ +Add the option to write sqlite test dbs to disk when running tests. \ No newline at end of file From 6c1a8dd9a17885b9499c4dce79e44b0594c74dae Mon Sep 17 00:00:00 2001 From: David Robertson Date: Fri, 7 Jan 2022 16:51:07 +0000 Subject: [PATCH 3/7] Persist to $TRIAL_TEMP/test.db; a few renames --- tests/server.py | 19 ++++++++++++++----- tests/utils.py | 4 ++-- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/tests/server.py b/tests/server.py index e94796b5625b..287fe0716fe1 100644 --- a/tests/server.py +++ b/tests/server.py @@ -15,6 +15,7 @@ import json import logging import os +import os.path import time import uuid import warnings @@ -72,7 +73,7 @@ POSTGRES_HOST, POSTGRES_PASSWORD, POSTGRES_USER, - SQLITE_TEST_DB_LOCATION, + SQLITE_PERSIST_DB, USE_POSTGRES_FOR_TESTS, MockClock, default_config, @@ -741,16 +742,24 @@ def setup_test_homeserver( }, } else: - if SQLITE_TEST_DB_LOCATION != ":memory:": - # nuke the DB on disk + if SQLITE_PERSIST_DB: + test_db_location = os.path.abspath("test.db") + logger.debug( + "Will persist db to %s", + ) + # nuke the DB on disk if it already exists try: - os.remove(SQLITE_TEST_DB_LOCATION) + os.remove(test_db_location) except FileNotFoundError: pass + else: + logger.debug("Removed existing DB at %s", test_db_location) + else: + test_db_location = ":memory:" database_config = { "name": "sqlite3", - "args": {"database": SQLITE_TEST_DB_LOCATION, "cp_min": 1, "cp_max": 1}, + "args": {"database": test_db_location, "cp_min": 1, "cp_max": 1}, } if "db_txn_limit" in kwargs: diff --git a/tests/utils.py b/tests/utils.py index 4389264c83b9..c06fc320f3df 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -43,8 +43,8 @@ POSTGRES_BASE_DB = "_synapse_unit_tests_base_%s" % (os.getpid(),) # When debugging a specific test, it's occasionally useful to write the -# DB to /tmp and query it with the sqlite CLI. -SQLITE_TEST_DB_LOCATION = os.environ.get("SYNAPSE_TEST_SQLITE_DB_LOCATION", ":memory:") +# DB to disk and query it with the sqlite CLI. +SQLITE_PERSIST_DB = os.environ.get("SYNAPSE_TEST_PERSIST_SQLITE_DB") is not None # the dbname we will connect to in order to create the base database. POSTGRES_DBNAME_FOR_INITIAL_CREATE = "postgres" From 47e07c2be18dd13a9db5db3e540c2ddab7e6d9c9 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Fri, 7 Jan 2022 18:27:38 +0000 Subject: [PATCH 4/7] Actually log the location, durr Could have sworn I did this? Co-authored-by: Patrick Cloke --- tests/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/server.py b/tests/server.py index 287fe0716fe1..57071d31bd0e 100644 --- a/tests/server.py +++ b/tests/server.py @@ -745,7 +745,7 @@ def setup_test_homeserver( if SQLITE_PERSIST_DB: test_db_location = os.path.abspath("test.db") logger.debug( - "Will persist db to %s", + "Will persist db to %s", test_db_location ) # nuke the DB on disk if it already exists try: From 24c50d596e703effbfea986639ac4870e8d9d8d3 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Fri, 7 Jan 2022 18:27:47 +0000 Subject: [PATCH 5/7] Clarify comment Co-authored-by: Patrick Cloke --- tests/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/server.py b/tests/server.py index 57071d31bd0e..ff08f68eb26f 100644 --- a/tests/server.py +++ b/tests/server.py @@ -747,7 +747,7 @@ def setup_test_homeserver( logger.debug( "Will persist db to %s", test_db_location ) - # nuke the DB on disk if it already exists + # Ensure each test gets a clean database. try: os.remove(test_db_location) except FileNotFoundError: From 1753a6b45c2913d97759c0b2ee71586b6969ba83 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Fri, 7 Jan 2022 18:32:32 +0000 Subject: [PATCH 6/7] Note that cwd is _trial_temp Co-authored-by: Patrick Cloke --- tests/server.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/server.py b/tests/server.py index ff08f68eb26f..e7ac3ec8e06b 100644 --- a/tests/server.py +++ b/tests/server.py @@ -743,6 +743,7 @@ def setup_test_homeserver( } else: if SQLITE_PERSIST_DB: + # The current working directory is in _trial_temp, so this gets created within that directory. test_db_location = os.path.abspath("test.db") logger.debug( "Will persist db to %s", test_db_location From 547874a8c419386d59b172030b9016eb328fe8e2 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Fri, 7 Jan 2022 18:42:57 +0000 Subject: [PATCH 7/7] lint --- tests/server.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/server.py b/tests/server.py index e7ac3ec8e06b..a0cd14ea45b4 100644 --- a/tests/server.py +++ b/tests/server.py @@ -745,9 +745,7 @@ def setup_test_homeserver( if SQLITE_PERSIST_DB: # The current working directory is in _trial_temp, so this gets created within that directory. test_db_location = os.path.abspath("test.db") - logger.debug( - "Will persist db to %s", test_db_location - ) + logger.debug("Will persist db to %s", test_db_location) # Ensure each test gets a clean database. try: os.remove(test_db_location)