Bug 1656589 - Track the database size on initialization

CHANGELOG.md

@@ -2,6 +2,8 @@
 
 [Full changelog](https://github.com/mozilla/glean/compare/v32.2.0...main)
 
+* General
+  * Track the size of the database file at startup ([#1141](https://github.com/mozilla/glean/pull/1141)).
 * Android
   * Handle ping registration off the main thread. This removes a potential blocking call ([#1132](https://github.com/mozilla/glean/pull/1132)).
 * iOS

docs/user/collected-metrics/metrics.md

@@ -128,6 +128,7 @@ The following metrics are added to the ping:
 
 | Name | Type | Description | Data reviews | Extras | Expiration | [Data Sensitivity](https://wiki.mozilla.org/Firefox/Data_Collection) |
 | --- | --- | --- | --- | --- | --- | --- |
+| glean.database.size |[memory_distribution](https://mozilla.github.io/glean/book/user/metrics/memory_distribution.html) |The size of the database file at startup.  |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1656589#c7)||never |1 |
 | glean.error.preinit_tasks_overflow |[counter](https://mozilla.github.io/glean/book/user/metrics/counter.html) |The number of tasks queued in the pre-initialization buffer. Only sent if the buffer overflows.  |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1609482#c3)||never |1 |
 | glean.upload.deleted_pings_after_quota_hit |[counter](https://mozilla.github.io/glean/book/user/metrics/counter.html) |The number of pings deleted after the quota for the size of the pending pings directory is hit. Since quota is only calculated for the pending pings directory, and deletion request ping live in a different directory, deletion request pings are never deleted.  |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1601550#c3)||never |1 |
 | glean.upload.discarded_exceeding_pings_size |[memory_distribution](https://mozilla.github.io/glean/book/user/metrics/memory_distribution.html) |The size of pings that exceeded the maximum ping size allowed for upload.  |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1597761#c10)||never |1 |

glean-core/android/src/test/java/mozilla/telemetry/glean/GleanTest.kt

@@ -69,6 +69,7 @@ class GleanTest {
     // New from glean-core.
     @Test
     fun `send a ping`() {
+        delayMetricsPing(context)
         val server = getMockWebServer()
         resetGlean(context, Glean.configuration.copy(
             serverEndpoint = "http://" + server.hostName + ":" + server.port
@@ -89,6 +90,7 @@ class GleanTest {
 
     @Test
     fun `X-Debug-ID header is correctly added when debug view tag is set`() {
+        delayMetricsPing(context)
         val server = getMockWebServer()
         resetGlean(context, Glean.configuration.copy(
             serverEndpoint = "http://" + server.hostName + ":" + server.port
@@ -181,6 +183,7 @@ class GleanTest {
         )
 
         val context = getContextWithMockedInfo()
+        delayMetricsPing(context)
         resetGlean(context, Glean.configuration.copy(
             serverEndpoint = "http://" + server.hostName + ":" + server.port
         ))
@@ -258,6 +261,7 @@ class GleanTest {
         // Restart glean and don't clear the stores.
         val server = getMockWebServer()
         val context = getContextWithMockedInfo()
+        delayMetricsPing(context)
         resetGlean(context, Glean.configuration.copy(
             serverEndpoint = "http://" + server.hostName + ":" + server.port
         ), false)
@@ -432,6 +436,7 @@ class GleanTest {
         val server = getMockWebServer()
 
         val context = getContextWithMockedInfo()
+        delayMetricsPing(context)
         resetGlean(context, Glean.configuration.copy(
             serverEndpoint = "http://" + server.hostName + ":" + server.port
         ))
@@ -574,6 +579,7 @@ class GleanTest {
 
     @Test
     fun `overflowing the task queue records telemetry`() {
+        delayMetricsPing(context)
         val server = getMockWebServer()
         Dispatchers.API.setTestingMode(true)
         Dispatchers.API.setTaskQueueing(true)
@@ -683,6 +689,7 @@ class GleanTest {
         // Restart glean and don't clear the stores.
         val server = getMockWebServer()
         val context = getContextWithMockedInfo()
+        delayMetricsPing(context)
         resetGlean(context, Glean.configuration.copy(
             serverEndpoint = "http://" + server.hostName + ":" + server.port
         ), false)
@@ -759,6 +766,9 @@ class GleanTest {
         // This test relies on Glean not being initialized, we do that ourselves.
         Glean.testDestroyGleanHandle()
 
+        val context = getContextWithMockedInfo()
+        delayMetricsPing(context)
+
         // This test relies on testing mode to be disabled, since we need to prove the
         // real-world async behaviour of this.
         // We don't need to care about clearing it,
@@ -783,7 +793,6 @@ class GleanTest {
         )
 
         val server = getMockWebServer()
-        val context = getContextWithMockedInfo()
         val config = Glean.configuration.copy(
             serverEndpoint = "http://" + server.hostName + ":" + server.port
         )

glean-core/android/src/test/java/mozilla/telemetry/glean/TestUtil.kt

@@ -7,6 +7,7 @@
 package mozilla.telemetry.glean
 
 import android.content.Context
+import android.os.SystemClock
 import android.content.pm.PackageInfo
 import android.content.pm.PackageManager
 import androidx.test.core.app.ApplicationProvider
@@ -22,15 +23,20 @@ import org.mockito.Mockito
 import mozilla.telemetry.glean.config.Configuration
 import mozilla.telemetry.glean.scheduler.PingUploadWorker
 import mozilla.telemetry.glean.private.PingTypeBase
+import mozilla.telemetry.glean.private.TimeUnit
 import mozilla.telemetry.glean.utils.decompressGZIP
+import mozilla.telemetry.glean.utils.getISOTimeString
+import mozilla.telemetry.glean.scheduler.MetricsPingScheduler
 import okhttp3.mockwebserver.Dispatcher
 import okhttp3.mockwebserver.MockResponse
 import okhttp3.mockwebserver.MockWebServer
 import okhttp3.mockwebserver.RecordedRequest
 import org.junit.Assert
 import org.mockito.ArgumentCaptor
+import org.mockito.Mockito.spy
 import org.robolectric.shadows.ShadowLog
 import java.io.ByteArrayInputStream
+import java.util.Calendar
 import java.util.UUID
 import java.util.concurrent.ExecutionException
 
@@ -257,6 +263,27 @@ fun RecordedRequest.getPlainBody(): String {
     }
 }
 
+/**
+ * Ensure no overdue metrics ping is triggered on `Glean.initialize`.
+ *
+ * This sets a fake date and time and changes the metrics ping scheduler
+ * to assume that now was the last time a metrics ping was sent.
+ * This can be used when tests should receive other pings,
+ * but don't want to deal with a potential overdue metrics ping first
+ */
+internal fun delayMetricsPing(context: Context) {
+    // Set the current system time to a known datetime.
+    val fakeNow = Calendar.getInstance()
+    fakeNow.clear()
+    fakeNow.set(2015, 6, 11, 2, 0, 0)
+    SystemClock.setCurrentTimeMillis(fakeNow.timeInMillis)
+
+    // Set the last sent date to yesterday.
+    val mps = MetricsPingScheduler(context)
+
+    mps.updateSentDate(getISOTimeString(fakeNow, truncateTo = TimeUnit.Day))
+}
+
 // The following Mockito fixups are copied over from support-test (Matchers.kt) from
 // Android-Components. We copied them over since A-C uses the Glean SDK, and we don't
 // want a dependency on A-C.

glean-core/android/src/test/java/mozilla/telemetry/glean/pings/CustomPingTest.kt

@@ -15,6 +15,7 @@ import mozilla.telemetry.glean.resetGlean
 import mozilla.telemetry.glean.testing.GleanTestRule
 import mozilla.telemetry.glean.triggerWorkManager
 import mozilla.telemetry.glean.utils.tryGetLong
+import mozilla.telemetry.glean.delayMetricsPing
 import org.junit.Assert.assertEquals
 import org.junit.Rule
 import org.junit.Test
@@ -43,6 +44,7 @@ class CustomPingTest {
 
         val server = getMockWebServer()
 
+        delayMetricsPing(context)
         resetGlean(context, Glean.configuration.copy(
             serverEndpoint = "http://" + server.hostName + ":" + server.port
         ), clearStores = true, uploadEnabled = true)
@@ -67,6 +69,7 @@ class CustomPingTest {
     fun `multiple pings in one go`() {
         val server = getMockWebServer()
 
+        delayMetricsPing(context)
         resetGlean(context, Glean.configuration.copy(
             serverEndpoint = "http://" + server.hostName + ":" + server.port
         ), clearStores = true, uploadEnabled = true)

glean-core/android/src/test/java/mozilla/telemetry/glean/private/EventMetricTypeTest.kt

@@ -20,6 +20,7 @@ import mozilla.telemetry.glean.Dispatchers
 import mozilla.telemetry.glean.getContextWithMockedInfo
 import mozilla.telemetry.glean.getMockWebServer
 import mozilla.telemetry.glean.resetGlean
+import mozilla.telemetry.glean.delayMetricsPing
 import java.lang.NullPointerException
 import java.util.concurrent.TimeUnit
 import mozilla.telemetry.glean.testing.ErrorType
@@ -292,6 +293,7 @@ class EventMetricTypeTest {
         val server = getMockWebServer()
 
         val context = getContextWithMockedInfo()
+        delayMetricsPing(context)
         resetGlean(
             context,
             Glean.configuration.copy(
@@ -401,6 +403,7 @@ class EventMetricTypeTest {
     fun `overdue events are submitted in registered custom pings`() {
         val server = getMockWebServer()
         val context = getContextWithMockedInfo()
+        delayMetricsPing(context)
 
         resetGlean(
             context,
@@ -475,6 +478,7 @@ class EventMetricTypeTest {
 
         val server = getMockWebServer()
         val context = getContextWithMockedInfo()
+        delayMetricsPing(context)
 
         resetGlean(
             context,

glean-core/android/src/test/java/mozilla/telemetry/glean/private/PingTypeTest.kt

@@ -16,6 +16,7 @@ import mozilla.telemetry.glean.resetGlean
 import mozilla.telemetry.glean.scheduler.PingUploadWorker
 import mozilla.telemetry.glean.testing.GleanTestRule
 import mozilla.telemetry.glean.triggerWorkManager
+import mozilla.telemetry.glean.delayMetricsPing
 import org.json.JSONObject
 import org.junit.Assert.assertEquals
 import org.junit.Assert.assertFalse
@@ -38,6 +39,7 @@ class PingTypeTest {
         val server = getMockWebServer()
 
         val context = getContextWithMockedInfo()
+        delayMetricsPing(context)
         resetGlean(context, Glean.configuration.copy(
             serverEndpoint = "http://" + server.hostName + ":" + server.port
         ))
@@ -78,6 +80,7 @@ class PingTypeTest {
         val server = getMockWebServer()
 
         val context = getContextWithMockedInfo()
+        delayMetricsPing(context)
         resetGlean(context, Glean.configuration.copy(
             serverEndpoint = "http://" + server.hostName + ":" + server.port
         ))
@@ -118,6 +121,7 @@ class PingTypeTest {
         val server = getMockWebServer()
 
         val context = getContextWithMockedInfo()
+        delayMetricsPing(context)
         resetGlean(context, Glean.configuration.copy(
             serverEndpoint = "http://" + server.hostName + ":" + server.port
         ))
@@ -158,6 +162,7 @@ class PingTypeTest {
         val server = getMockWebServer()
 
         val context = getContextWithMockedInfo()
+        delayMetricsPing(context)
         resetGlean(context, Glean.configuration.copy(
             serverEndpoint = "http://" + server.hostName + ":" + server.port
         ))
@@ -206,6 +211,7 @@ class PingTypeTest {
         )
 
         val context = getContextWithMockedInfo()
+        delayMetricsPing(context)
         resetGlean(context, Glean.configuration.copy(
             serverEndpoint = "http://" + server.hostName + ":" + server.port
         ))

glean-core/metrics.yaml

@@ -429,3 +429,19 @@ glean.upload:
     expires: never
     no_lint:
       - COMMON_PREFIX
+
+glean.database:
+  size:
+    type: memory_distribution
+    description: >
+      The size of the database file at startup.
+    memory_unit: byte
+    bugs:
+      - https://bugzilla.mozilla.org/show_bug.cgi?id=1656589
+    data_reviews:
+      - https://bugzilla.mozilla.org/show_bug.cgi?id=1656589#c7
+    data_sensitivity:
+      - technical
+    notification_emails:
+      - [email protected]
+    expires: never

glean-core/src/database/mod.rs

@@ -5,6 +5,8 @@
 use std::collections::btree_map::Entry;
 use std::collections::BTreeMap;
 use std::fs;
+use std::num::NonZeroU64;
+use std::path::Path;
 use std::str;
 use std::sync::RwLock;
 
@@ -32,6 +34,9 @@ pub struct Database {
     /// we will save metrics with 'ping' lifetime data in a map temporarily
     /// so as to persist them to disk using rkv in bulk on demand.
     ping_lifetime_data: Option<RwLock<BTreeMap<String, Metric>>>,
+
+    // Initial file size when opening the database.
+    file_size: Option<NonZeroU64>,
 }
 
 impl std::fmt::Debug for Database {
@@ -46,6 +51,24 @@ impl std::fmt::Debug for Database {
     }
 }
 
+/// Get the file size of a file in the given path and file.
+///
+/// # Arguments
+///
+/// - `path` - The path
+///
+/// # Returns
+///
+/// Returns the non-zero file size in bytes,
+/// or `None` on error or if the size is `0`.
+fn file_size(path: &Path) -> Option<NonZeroU64> {
+    log::trace!("Getting file size for path: {}", path.display());
+    fs::metadata(path)
+        .ok()
+        .map(|stat| stat.len())
+        .and_then(NonZeroU64::new)
+}
+
 impl Database {
     /// Initializes the data store.
     ///
@@ -55,7 +78,12 @@ impl Database {
     /// It also loads any Lifetime::Ping data that might be
     /// persisted, in case `delay_ping_lifetime_io` is set.
     pub fn new(data_path: &str, delay_ping_lifetime_io: bool) -> Result<Self> {
-        let rkv = Self::open_rkv(data_path)?;
+        let path = Path::new(data_path).join("db");
+        log::debug!("Database path: {:?}", path.display());
+
+        let file_size = file_size(&path.join("data.mdb"));
+
+        let rkv = Self::open_rkv(&path)?;
         let user_store = rkv.open_single(Lifetime::User.as_str(), StoreOptions::create())?;
         let ping_store = rkv.open_single(Lifetime::Ping.as_str(), StoreOptions::create())?;
         let application_store =
@@ -72,13 +100,19 @@ impl Database {
             ping_store,
             application_store,
             ping_lifetime_data,
+            file_size,
         };
 
         db.load_ping_lifetime_data();
 
         Ok(db)
     }
 
+    /// Get the initial database file size.
+    pub fn file_size(&self) -> Option<NonZeroU64> {
+        self.file_size
+    }
+
     fn get_store(&self, lifetime: Lifetime) -> &SingleStore {
         match lifetime {
             Lifetime::User => &self.user_store,
@@ -88,9 +122,7 @@ impl Database {
     }
 
     /// Creates the storage directories and inits rkv.
-    fn open_rkv(path: &str) -> Result<Rkv> {
-        let path = std::path::Path::new(path).join("db");
-        log::debug!("Database path: {:?}", path.display());
+    fn open_rkv(path: &Path) -> Result<Rkv> {
         fs::create_dir_all(&path)?;
 
         let rkv = Rkv::new(&path)?;

glean-core/src/internal_metrics.rs

@@ -111,3 +111,26 @@ impl UploadMetrics {
         }
     }
 }
+
+#[derive(Debug)]
+pub struct DatabaseMetrics {
+    pub size: MemoryDistributionMetric,
+}
+
+impl DatabaseMetrics {
+    pub fn new() -> DatabaseMetrics {
+        DatabaseMetrics {
+            size: MemoryDistributionMetric::new(
+                CommonMetricData {
+                    name: "size".into(),
+                    category: "glean.database".into(),
+                    send_in_pings: vec!["metrics".into()],
+                    lifetime: Lifetime::Ping,
+                    disabled: false,
+                    dynamic_label: None,
+                },
+                MemoryUnit::Byte,
+            ),
+        }
+    }
+}