feat: add deadman switch for exhausted connection pools.

*Ops:*

Adds `SYNC_DATABASE_POOL_CONNECTION_DEADMAN_SWITCH` which is the
number of milliseconds the pool can report being at 0 available
connections before the application triggers a `panic!`.

The current default is `0` meaning the deadman switch is inactive.

Issue #945

spanner_config.ini

@@ -1,2 +1,6 @@
 # Limit the number of incoming records (See issue #298/#333/#869)
 limits.max_total_records=1664
+
+## number of miliseconds the database connection pool can be at 0
+## available connections before the application panics.
+# database_pool_connection_deadman_switch = 0

src/db/error.rs

@@ -55,6 +55,9 @@ pub enum DbErrorKind {
 
     #[fail(display = "User over quota")]
     Quota,
+
+    #[fail(display = "Database Connection Pool Exhausted")]
+    ExhaustedPanic,
 }
 
 impl DbError {

src/db/mod.rs

@@ -11,10 +11,13 @@ mod tests;
 pub mod transaction;
 pub mod util;
 
-use std::{fmt::Debug, time::Duration};
+use std::{
+    fmt::Debug,
+    time::{Duration, Instant},
+};
 
 use async_trait::async_trait;
-use cadence::{Gauged, StatsdClient};
+use cadence::{Counted, Gauged, StatsdClient};
 use futures::future::{self, LocalBoxFuture, TryFutureExt};
 use lazy_static::lazy_static;
 use serde::Deserialize;
@@ -282,17 +285,37 @@ pub fn spawn_pool_periodic_reporter(
     interval: Duration,
     metrics: StatsdClient,
     pool: Box<dyn DbPool>,
+    deadman_switch: Option<u32>,
 ) -> Result<(), DbError> {
     let hostname = hostname::get()
         .expect("Couldn't get hostname")
         .into_string()
         .expect("Couldn't get hostname");
+    let deadman = deadman_switch.unwrap_or_default();
     actix_rt::spawn(async move {
+        let mut start_time = Instant::now();
+        let mut prior_state = 0;
         loop {
             let results::PoolState {
                 connections,
                 idle_connections,
             } = pool.state();
+            if deadman > 0 {
+                if idle_connections == 0 {
+                    if prior_state != 0 {
+                        // We just hit zero connections, start the clock
+                        start_time = Instant::now();
+                    }
+                    if Instant::now() - start_time > Duration::from_millis(deadman as u64) {
+                        metrics
+                            .incr_with_tags("storage.pool.exhausted")
+                            .with_tag("hostname", &hostname)
+                            .send();
+                        panic!("Database connection pool exhausted")
+                    }
+                }
+                prior_state = idle_connections;
+            }
             metrics
                 .gauge_with_tags(
                     "storage.pool.connections.active",

src/server/mod.rs

@@ -183,7 +183,12 @@ impl Server {
         let quota_enabled = settings.enable_quota;
         let actix_keep_alive = settings.actix_keep_alive;
 
-        spawn_pool_periodic_reporter(Duration::from_secs(10), metrics.clone(), db_pool.clone())?;
+        spawn_pool_periodic_reporter(
+            Duration::from_secs(10),
+            metrics.clone(),
+            db_pool.clone(),
+            settings.database_pool_connection_deadman_switch,
+        )?;
 
         let mut server = HttpServer::new(move || {
             // Setup the server state

src/settings.rs

@@ -45,6 +45,7 @@ pub struct Settings {
     pub database_pool_min_idle: Option<u32>,
     /// Pool timeout when waiting for a slot to become available, in seconds
     pub database_pool_connection_timeout: Option<u32>,
+    pub database_pool_connection_deadman_switch: Option<u32>,
     #[cfg(test)]
     pub database_use_test_transactions: bool,
 
@@ -82,6 +83,7 @@ impl Default for Settings {
             database_pool_max_size: None,
             database_pool_min_idle: None,
             database_pool_connection_timeout: Some(30),
+            database_pool_connection_deadman_switch: Some(0),
             #[cfg(test)]
             database_use_test_transactions: false,
             actix_keep_alive: None,
@@ -112,6 +114,7 @@ impl Settings {
         s.set_default("human_logs", false)?;
         #[cfg(test)]
         s.set_default("database_pool_connection_timeout", Some(30))?;
+        s.set_default("database_pool_connection_deadman_switch", Some(0))?;
         s.set_default("database_use_test_transactions", false)?;
         s.set_default("master_secret", "")?;
         s.set_default::<Option<String>>("tokenserver_database_url", None)?;