From 9aabe2aee4bbfe9af1cb9424a93cf6a59f13b9a6 Mon Sep 17 00:00:00 2001
From: Sean Klein <sean@oxide.computer>
Date: Thu, 5 Oct 2023 09:46:26 -0700
Subject: [PATCH 1/2] Add transaction retry to schema upgrade integration tests
 (#4209)

Fixes https://github.com/oxidecomputer/omicron/issues/4207
---
 nexus/tests/integration_tests/schema.rs | 51 ++++++++++++++++++++-----
 1 file changed, 42 insertions(+), 9 deletions(-)

diff --git a/nexus/tests/integration_tests/schema.rs b/nexus/tests/integration_tests/schema.rs
index 2c62f156e1..1d4556e8ed 100644
--- a/nexus/tests/integration_tests/schema.rs
+++ b/nexus/tests/integration_tests/schema.rs
@@ -62,6 +62,47 @@ async fn test_setup<'a>(
     builder
 }
 
+// Attempts to apply an update as a transaction.
+//
+// Only returns an error if the transaction failed to commit.
+async fn apply_update_as_transaction_inner(
+    client: &omicron_test_utils::dev::db::Client,
+    sql: &str,
+) -> Result<(), tokio_postgres::Error> {
+    client.batch_execute("BEGIN;").await.expect("Failed to BEGIN transaction");
+    client.batch_execute(&sql).await.expect("Failed to execute update");
+    client.batch_execute("COMMIT;").await?;
+    Ok(())
+}
+
+// Applies an update as a transaction.
+//
+// Automatically retries transactions that can be retried client-side.
+async fn apply_update_as_transaction(
+    log: &Logger,
+    client: &omicron_test_utils::dev::db::Client,
+    sql: &str,
+) {
+    loop {
+        match apply_update_as_transaction_inner(client, sql).await {
+            Ok(()) => break,
+            Err(err) => {
+                client
+                    .batch_execute("ROLLBACK;")
+                    .await
+                    .expect("Failed to ROLLBACK failed transaction");
+                if let Some(code) = err.code() {
+                    if code == &tokio_postgres::error::SqlState::T_R_SERIALIZATION_FAILURE {
+                        warn!(log, "Transaction retrying");
+                        continue;
+                    }
+                }
+                panic!("Failed to apply update: {err}");
+            }
+        }
+    }
+}
+
 async fn apply_update(
     log: &Logger,
     crdb: &CockroachInstance,
@@ -87,15 +128,7 @@ async fn apply_update(
 
     for _ in 0..times_to_apply {
         for sql in sqls.iter() {
-            client
-                .batch_execute("BEGIN;")
-                .await
-                .expect("Failed to BEGIN update");
-            client.batch_execute(&sql).await.expect("Failed to execute update");
-            client
-                .batch_execute("COMMIT;")
-                .await
-                .expect("Failed to COMMIT update");
+            apply_update_as_transaction(log, &client, sql).await;
         }
     }
 

From 6cf8181ba678855e3f131ad2914e90d06de02ac3 Mon Sep 17 00:00:00 2001
From: David Pacheco <dap@oxidecomputer.com>
Date: Thu, 5 Oct 2023 10:28:21 -0700
Subject: [PATCH 2/2] top-level cleanup: move `thing-flinger` into `dev-tools`
 (#4213)

---
 Cargo.toml                                                    | 4 ++--
 {deploy => dev-tools/thing-flinger}/.gitignore                | 0
 {deploy => dev-tools/thing-flinger}/Cargo.toml                | 0
 {deploy => dev-tools/thing-flinger}/README.adoc               | 0
 .../thing-flinger}/src/bin/deployment-example.toml            | 0
 {deploy => dev-tools/thing-flinger}/src/bin/thing-flinger.rs  | 0
 6 files changed, 2 insertions(+), 2 deletions(-)
 rename {deploy => dev-tools/thing-flinger}/.gitignore (100%)
 rename {deploy => dev-tools/thing-flinger}/Cargo.toml (100%)
 rename {deploy => dev-tools/thing-flinger}/README.adoc (100%)
 rename {deploy => dev-tools/thing-flinger}/src/bin/deployment-example.toml (100%)
 rename {deploy => dev-tools/thing-flinger}/src/bin/thing-flinger.rs (100%)

diff --git a/Cargo.toml b/Cargo.toml
index 2af44b5559..29291e8a19 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -8,9 +8,9 @@ members = [
     "common",
     "crdb-seed",
     "ddm-admin-client",
-    "deploy",
     "dev-tools/omdb",
     "dev-tools/omicron-dev",
+    "dev-tools/thing-flinger",
     "dev-tools/xtask",
     "dns-server",
     "dns-service-client",
@@ -75,9 +75,9 @@ default-members = [
     "common",
     "ddm-admin-client",
     "dpd-client",
-    "deploy",
     "dev-tools/omdb",
     "dev-tools/omicron-dev",
+    "dev-tools/thing-flinger",
     "dev-tools/xtask",
     "dns-server",
     "dns-service-client",
diff --git a/deploy/.gitignore b/dev-tools/thing-flinger/.gitignore
similarity index 100%
rename from deploy/.gitignore
rename to dev-tools/thing-flinger/.gitignore
diff --git a/deploy/Cargo.toml b/dev-tools/thing-flinger/Cargo.toml
similarity index 100%
rename from deploy/Cargo.toml
rename to dev-tools/thing-flinger/Cargo.toml
diff --git a/deploy/README.adoc b/dev-tools/thing-flinger/README.adoc
similarity index 100%
rename from deploy/README.adoc
rename to dev-tools/thing-flinger/README.adoc
diff --git a/deploy/src/bin/deployment-example.toml b/dev-tools/thing-flinger/src/bin/deployment-example.toml
similarity index 100%
rename from deploy/src/bin/deployment-example.toml
rename to dev-tools/thing-flinger/src/bin/deployment-example.toml
diff --git a/deploy/src/bin/thing-flinger.rs b/dev-tools/thing-flinger/src/bin/thing-flinger.rs
similarity index 100%
rename from deploy/src/bin/thing-flinger.rs
rename to dev-tools/thing-flinger/src/bin/thing-flinger.rs