Merge branch 'master' into faster_init

.circleci/config.yml

@@ -151,7 +151,7 @@ commands:
             GHR=ghr_v0.13.0_linux_amd64
             GHR_SHA256=c428627270ae26e206cb526cb8c7bdfba475dd278f6691ddaf863355adadfa13
             curl -sfSL --retry 5 --retry-delay 10 -O "https://github.com/tcnksm/ghr/releases/download/v0.13.0/${GHR}.tar.gz"
-            echo "${GHR_SHA256} *${GHR}.tar.gz" | shasum -a 256 -c -
+            echo "${GHR_SHA256} *${GHR}.tar.gz" | sha256sum -c -
             tar -xf "${GHR}.tar.gz"
             cp ./${GHR}/ghr ghr
 

CHANGELOG.md

@@ -6,6 +6,14 @@
   * `ping_type` is not included in the `ping_info` any more ([#653](https://github.com/mozilla/glean/pull/653)), the pipeline takes the value from the submission URL.
 * Android:
   * The `Glean.initialize` method runs mostly off the main thread ([#672](https://github.com/mozilla/glean/pull/672)).
+* iOS:
+  * The baseline ping will now include `reason` codes that indicate why it was
+    submitted. If an unclean shutdown is detected (e.g. due to force-close), this
+    ping will be sent at startup with `reason: dirty_startup`.
+  * Per [Bug 1614785](https://bugzilla.mozilla.org/show_bug.cgi?id=1614785), the
+    clearing of application lifetime metrics now occurs after the metrics ping is
+    sent in order to preserve values meant to be included in the startup metrics
+    ping.
 
 # v24.2.0 (2020-02-11)
 

docs/dev/cut-a-new-release.md

@@ -215,13 +215,17 @@ After following one of the above instructions to make a Glean SDK release:
 
 1. Ensure that CI has completed and the artifacts have published to [the Glean GitHub releases page](https://github.com/mozilla/glean/releases/).
 
-2. Publish Glean to [Mozilla's Maven repository](https://maven.mozilla.org/). 
+2. Publish Glean to [Mozilla's Maven repository](https://maven.mozilla.org/).
    This is currently a manual step that can only performed by individuals with the necessary credentials.
    Ask in `#releaseduty-mobile` on Mozilla's internal Slack for assistance.
-   
+
 3. Create a pull request against `android-components` to update the Glean version with the following changes:
-   
+
    - The Glean version is updated in the `mozilla_glean` variable in the [`buildSrc/src/main/java/Dependencies.kt`](https://github.com/mozilla-mobile/android-components/blob/69546999739ab19d21425e9a98e107e438a3f905/buildSrc/src/main/java/Dependencies.kt#L32) file.
-   
+
    - The relevant parts of the Glean changelog copied into the top part of the [`android-components` changelog](https://github.com/mozilla-mobile/android-components/blob/master/docs/changelog.md).
      This involves copying the Android-specific changes and the general changes to Glean, but can omit other platform-specific changes.
+
+**IMPORTANT:** Until the [Glean Gradle plugin work is complete](https://bugzilla.mozilla.org/show_bug.cgi?id=1592947), all downstream consumers of android-components will also need to update their version of Glean to match the version used in android-components so that their unit tests can run correctly.
+
+In Fenix, for example, the [Glean version is specified here](https://github.com/mozilla-mobile/fenix/blob/5e4ef202b85e273cf46ec5c7ec1b80f30ca4e77c/buildSrc/src/main/java/Dependencies.kt#L44).

docs/user/metrics/index.md

@@ -44,8 +44,15 @@ Labeled metrics come in two forms:
 
 - **Static labels**: The labels are specified at build time in the `metrics.yaml` file.
   If a label that isn't part of this set is used at run time, it is converted to the special label `__other__`.
-  
+
 - **Dynamic labels**: The labels aren't known at build time, so are set at run time.
   Only the first 16 labels seen by the Glean SDK will be tracked. After that, any additional labels are converted to the special label `__other__`.
 
 > **Note**: Be careful with using arbitrary strings as labels and make sure they can't accidentally contain identifying data (like directory paths or user input).
+
+## Adding or changing metric types
+Glean has a [well-defined process](https://wiki.mozilla.org/Glean/Adding_or_changing_Glean_metric_types) for requesting changes to existing metric types or suggesting the implementation of new metric types:
+
+1.  Glean consumers need to file a bug in the [Data platforms & tools::Glean Metric Types](https://bugzilla.mozilla.org/enter_bug.cgi?product=Data%20Platform%20and%20Tools&component=Glean%20Metric%20Types) component, filling in the provided form;
+2.  The triage owner of the Bugzilla component prioritizes this within 6 business days and kicks off the [decision making process](https://wiki.mozilla.org/Glean/Adding_or_changing_Glean_metric_types#The_decision_making_process).
+3.  Once the decision process is completed, the bug is closed with a comment outlining the decision that was made.

glean-core/ios/Glean/Glean.swift

@@ -65,6 +65,7 @@ public class Glean {
     ///       If disabled, all persisted metrics, events and queued pings (except
     ///       first_run_date) are cleared.
     ///     * configuration: A Glean `Configuration` object with global settings.
+    // swiftlint:disable function_body_length
     public func initialize(uploadEnabled: Bool,
                            configuration: Configuration = Configuration()) {
         if self.isInitialized() {
@@ -102,7 +103,13 @@ public class Glean {
             self.pingTypeQueue.removeAll()
         }
 
-        initializeCoreMetrics()
+        // If this is the first time ever the Glean SDK runs, make sure to set
+        // some initial core metrics in case we need to generate early pings.
+        // The next times we start, we would have them around already.
+        let isFirstRun = glean_is_first_run().toBool()
+        if isFirstRun {
+            initializeCoreMetrics()
+        }
 
         // Deal with any pending events so we can start recording new ones
         Dispatchers.shared.serialOperationQueue.addOperation {
@@ -116,9 +123,36 @@ public class Glean {
         // Check for overdue metrics pings
         metricsPingScheduler.schedule()
 
+        // From the second time we run, after all startup pings are generated,
+        // make sure to clear `lifetime: application` metrics and set them again.
+        // Any new value will be sent in newly generted pings after startup.
+        // NOTE: we are adding this directly to the serialOperationQueue which
+        // bypasses the queue for initial tasks, otherwise this could get lost
+        // if the initial tasks queue overflows.
+        if !isFirstRun {
+            Dispatchers.shared.serialOperationQueue.addOperation {
+                glean_clear_application_lifetime_metrics()
+                self.initializeCoreMetrics()
+            }
+        }
+
         // Signal Dispatcher that init is complete
         Dispatchers.shared.flushQueuedInitialTasks()
 
+        // Check if the "dirty flag" is set. That means the product was probably
+        // force-closed. If that's the case, submit a 'baseline' ping with the
+        // reason "dirty_startup". We only do that from the second run.
+        if !isFirstRun {
+            Dispatchers.shared.launchAPI {
+                if glean_is_dirty_flag_set().toBool() {
+                    self.submitPingByNameSync(
+                        pingName: "baseline",
+                        reason: "dirty_startup"
+                    )
+                }
+            }
+        }
+
         self.observer = GleanLifecycleObserver()
 
         if !uploadEnabled {

glean-core/ios/Glean/Scheduler/GleanLifecycleObserver.swift

@@ -29,12 +29,25 @@ class GleanLifecycleObserver {
         // because it belongs to the baseline ping and that ping is sent every
         // time the app goes to background.
         GleanBaseline.duration.start()
+
+        // Set the "dirty flag" to `true`.
+        Dispatchers.shared.launchAPI {
+            glean_set_dirty_flag(true.toByte())
+        }
     }
 
     @objc func appDidEnterBackground(notification _: NSNotification) {
         // We're going to background, so store how much time we spent
         // on foreground.
         GleanBaseline.duration.stop()
         Glean.shared.handleBackgroundEvent()
+
+        // Clear the "dirty flag" as the last thing when going to background.
+        // If the application is not being force-closed, we should still be
+        // alive and allowed to change this. If we're being force-closed and
+        // don't get to this point, next time Glean runs it will be detected.
+        Dispatchers.shared.launchAPI {
+            glean_set_dirty_flag(false.toByte())
+        }
     }
 }

glean-core/ios/GleanTests/GleanTests.swift

@@ -7,13 +7,16 @@ import OHHTTPStubs
 import XCTest
 
 class GleanTests: XCTestCase {
+    var expectation: XCTestExpectation?
+
     override func setUp() {
         Glean.shared.resetGlean(clearStores: true)
         Glean.shared.enableTestingMode()
     }
 
     override func tearDown() {
         Glean.shared.setUploadEnabled(true)
+        expectation = nil
     }
 
     func testInitializeGlean() {
@@ -97,4 +100,51 @@ class GleanTests: XCTestCase {
             "Experiment must not be active"
         )
     }
+
+    func testSendingOfStartupBaselinePing() {
+        // Set the dirty flag
+        glean_set_dirty_flag(true.toByte())
+
+        // Set up the test stub based on the default telemetry endpoint
+        let host = URL(string: Configuration.Constants.defaultTelemetryEndpoint)!.host!
+        stub(condition: isHost(host)) { data in
+            let body = (data as NSURLRequest).ohhttpStubs_HTTPBody()
+            let json = try! JSONSerialization.jsonObject(with: body!, options: []) as? [String: Any]
+            XCTAssert(json != nil)
+
+            // Check for the "dirty_startup" flag
+            let pingInfo = json?["ping_info"] as? [String: Any]
+            XCTAssertEqual("dirty_startup", pingInfo?["reason"] as? String)
+
+            // Ensure there is only the expected locale string metric
+            let metrics = json?["metrics"] as? [String: Any]
+            let strings = metrics?["string"] as? [String: Any]
+            XCTAssertEqual(1, strings?.count, "Must contain only the expected metric")
+            let locale = strings?["glean.baseline.locale"] as? String
+            XCTAssertNotNil(locale, "Locale is not nil")
+
+            // We should not have a duration for a ping with the "dirty_startup" flag
+            XCTAssertNil(metrics?["timespan"])
+
+            DispatchQueue.main.async {
+                // let the response get processed before we mark the expectation fulfilled
+                self.expectation?.fulfill()
+            }
+
+            return OHHTTPStubsResponse(
+                jsonObject: [],
+                statusCode: 200,
+                headers: ["Content-Type": "application/json"]
+            )
+        }
+
+        // Set up the expectation that will be fulfilled by the stub above
+        expectation = expectation(description: "Baseline Ping Received")
+
+        // Restart Glean and don't clear the stores and then await the expectation
+        Glean.shared.resetGlean(clearStores: false)
+        waitForExpectations(timeout: 5.0) { error in
+            XCTAssertNil(error, "Test timed out waiting for upload: \(error!)")
+        }
+    }
 }