Add option to defer ping lifetime metric persistence

glean-core/android/src/main/java/mozilla/telemetry/glean/Glean.kt

@@ -142,7 +142,8 @@ open class GleanInternalAPI internal constructor () {
             dataDir = this.gleanDataDir.path,
             packageName = applicationContext.packageName,
             uploadEnabled = uploadEnabled,
-            maxEvents = this.configuration.maxEvents
+            maxEvents = this.configuration.maxEvents,
+            delayPingLifetimeIO = false
         )
 
         // Start the migration from glean-ac, if needed.

glean-core/android/src/main/java/mozilla/telemetry/glean/config/Configuration.kt

@@ -17,12 +17,13 @@ import mozilla.telemetry.glean.net.PingUploader
  * **CAUTION**: This must match _exactly_ the definition on the Rust side.
  *  If this side is changed, the Rust side need to be changed, too.
  */
-@Structure.FieldOrder("dataDir", "packageName", "uploadEnabled", "maxEvents")
+@Structure.FieldOrder("dataDir", "packageName", "uploadEnabled", "maxEvents", "delayPingLifetimeIO")
 internal class FfiConfiguration(
     dataDir: String,
     packageName: String,
     uploadEnabled: Boolean,
-    maxEvents: Int? = null
+    maxEvents: Int? = null,
+    delayPingLifetimeIO: Boolean
 ) : Structure() {
     /**
      * Expose all structure fields as actual fields,
@@ -37,6 +38,8 @@ internal class FfiConfiguration(
     public var uploadEnabled: Byte = uploadEnabled.toByte()
     @JvmField
     public var maxEvents: IntByReference = if (maxEvents == null) IntByReference() else IntByReference(maxEvents)
+    @JvmField
+    public var delayPingLifetimeIO: Byte = delayPingLifetimeIO.toByte()
 
     init {
         // Force UTF-8 string encoding when passing strings over the FFI
@@ -64,6 +67,7 @@ data class Configuration internal constructor(
     val channel: String? = null,
     val userAgent: String = DEFAULT_USER_AGENT,
     val maxEvents: Int? = null,
+    val delayPingLifetimeIO: Boolean = false,
     val logPings: Boolean = DEFAULT_LOG_PINGS,
     // NOTE: since only simple object or strings can be made `const val`s, if the
     // default values for the lines below are ever changed, they are required
@@ -96,6 +100,7 @@ data class Configuration internal constructor(
         serverEndpoint = serverEndpoint,
         userAgent = DEFAULT_USER_AGENT,
         maxEvents = maxEvents,
+        delayPingLifetimeIO = false,
         logPings = DEFAULT_LOG_PINGS,
         httpClient = httpClient,
         pingTag = null,

glean-core/examples/sample.rs

@@ -23,6 +23,7 @@ fn main() {
         application_id: "org.mozilla.glean_core.example".into(),
         upload_enabled: true,
         max_events: None,
+        delay_ping_lifetime_io: false,
     };
     let mut glean = Glean::new(cfg).unwrap();
     glean.register_ping_type(&PingType::new("baseline", true, false));

glean-core/ffi/glean.h

@@ -80,6 +80,7 @@ typedef struct {
   FfiStr package_name;
   uint8_t upload_enabled;
   const int32_t *max_events;
+  uint8_t delay_ping_lifetime_io;
 } FfiConfiguration;
 
 /**

glean-core/ffi/src/lib.rs

@@ -101,6 +101,7 @@ pub struct FfiConfiguration<'a> {
     package_name: FfiStr<'a>,
     upload_enabled: u8,
     max_events: Option<&'a i32>,
+    delay_ping_lifetime_io: u8,
 }
 
 /// Convert the FFI-compatible configuration object into the proper Rust configuration object.
@@ -112,12 +113,14 @@ impl TryFrom<&FfiConfiguration<'_>> for glean_core::Configuration {
         let application_id = cfg.package_name.to_string_fallible()?;
         let upload_enabled = cfg.upload_enabled != 0;
         let max_events = cfg.max_events.filter(|&&i| i >= 0).map(|m| *m as usize);
+        let delay_ping_lifetime_io = cfg.delay_ping_lifetime_io != 0;
 
         Ok(Self {
             upload_enabled,
             data_path,
             application_id,
             max_events,
+            delay_ping_lifetime_io,
         })
     }
 }

glean-core/ios/Glean/GleanFfi.h

@@ -80,6 +80,7 @@ typedef struct {
   FfiStr package_name;
   uint8_t upload_enabled;
   const int32_t *max_events;
+  uint8_t delay_ping_lifetime_io;
 } FfiConfiguration;
 
 /**

glean-core/ios/Glean/Utils/Utils.swift

@@ -119,7 +119,8 @@ func withFfiConfiguration<R>(
         data_dir: dataDir,
         package_name: packageName,
         upload_enabled: uploadEnabled.toByte(),
-        max_events: maxEventsPtr
+        max_events: maxEventsPtr,
+        delay_ping_lifetime_io: false.toByte()
     )
     return body(cfg)
 }

glean-core/python/glean/_ffi.py

@@ -46,7 +46,11 @@ lib = ffi.dlopen(str(Path(__file__).parent / get_shared_object_filename()))
 
 
 def make_config(
-    data_dir: Path, package_name: str, upload_enabled: bool, max_events: int
+    data_dir: Path,
+    package_name: str,
+    upload_enabled: bool,
+    max_events: int,
+    delay_ping_lifetime_io: bool = False,
 ) -> Any:
     """
     Make an `FfiConfiguration` object.
@@ -65,6 +69,7 @@ def make_config(
     cfg.package_name = package_name
     cfg.upload_enabled = upload_enabled
     cfg.max_events = max_events
+    cfg.delay_ping_lifetime_io = delay_ping_lifetime_io
 
     _global_weakkeydict[cfg] = (data_dir, package_name, max_events)
 

glean-core/src/database/mod.rs

@@ -22,6 +22,12 @@ pub struct Database {
     // as the application lives: they don't need to be persisted
     // to disk using rkv. Store them in a map.
     app_lifetime_data: RwLock<BTreeMap<String, Metric>>,
+    // If the `delay_ping_lifetime_io` Glean config option is `true`,
+    // we will save metrics with 'ping' lifetime data in a map temporarily
+    // so as to persist them to disk using rkv in bulk on shutdown,
+    // or after a given interval, instead of everytime a new metric
+    // is created / updated.
+    ping_lifetime_data: Option<RwLock<BTreeMap<String, Metric>>>,
 }
 
 impl Database {
@@ -29,10 +35,15 @@ impl Database {
     ///
     /// This opens the underlying rkv store and creates
     /// the underlying directory structure.
-    pub fn new(data_path: &str) -> Result<Self> {
+    pub fn new(data_path: &str, delay_ping_lifetime_io: bool) -> Result<Self> {
         Ok(Self {
             rkv: Self::open_rkv(data_path)?,
             app_lifetime_data: RwLock::new(BTreeMap::new()),
+            ping_lifetime_data: if delay_ping_lifetime_io {
+                Some(RwLock::new(BTreeMap::new()))
+            } else {
+                None
+            },
         })
     }
 
@@ -117,6 +128,23 @@ impl Database {
             return;
         }
 
+        // Lifetime::Ping data is not persisted to disk if
+        // Glean has `delay_ping_lifetime_io` set to true
+        if lifetime == Lifetime::Ping {
+            if let Some(ping_lifetime_data) = &self.ping_lifetime_data {
+                let data = ping_lifetime_data
+                    .read()
+                    .expect("Can't read ping lifetime data");
+                for (key, value) in data.iter() {
+                    if key.starts_with(&iter_start) {
+                        let key = &key[len..];
+                        transaction_fn(key.as_bytes(), value);
+                    }
+                }
+                return;
+            }
+        }
+
         let store: SingleStore = unwrap_or!(
             self.rkv
                 .open_single(lifetime.as_str(), StoreOptions::create()),
@@ -170,6 +198,17 @@ impl Database {
                 .unwrap_or(false);
         }
 
+        // Lifetime::Ping data is not persisted to disk if
+        // Glean has `delay_ping_lifetime_io` set to true
+        if lifetime == Lifetime::Ping {
+            if let Some(ping_lifetime_data) = &self.ping_lifetime_data {
+                return ping_lifetime_data
+                    .read()
+                    .map(|data| data.contains_key(&key))
+                    .unwrap_or(false);
+            }
+        }
+
         let store: SingleStore = unwrap_or!(
             self.rkv
                 .open_single(lifetime.as_str(), StoreOptions::create()),
@@ -243,6 +282,18 @@ impl Database {
             return Ok(());
         }
 
+        // Lifetime::Ping data is not persisted to disk if
+        // Glean has `delay_ping_lifetime_io` set to true
+        if lifetime == Lifetime::Ping {
+            if let Some(ping_lifetime_data) = &self.ping_lifetime_data {
+                let mut data = ping_lifetime_data
+                    .write()
+                    .expect("Can't read ping lifetime data");
+                data.insert(final_key, metric.clone());
+                return Ok(());
+            }
+        }
+
         let encoded = bincode::serialize(&metric).expect("IMPOSSIBLE: Serializing metric failed");
         let value = rkv::Value::Blob(&encoded);
 
@@ -313,6 +364,27 @@ impl Database {
             return Ok(());
         }
 
+        // Lifetime::Ping data is not persisted to disk if
+        // Glean has `delay_ping_lifetime_io` set to true
+        if lifetime == Lifetime::Ping {
+            if let Some(ping_lifetime_data) = &self.ping_lifetime_data {
+                let mut data = ping_lifetime_data
+                    .write()
+                    .expect("Can't access ping lifetime data as writable");
+                let entry = data.entry(final_key);
+                match entry {
+                    Entry::Vacant(entry) => {
+                        entry.insert(transform(None));
+                    }
+                    Entry::Occupied(mut entry) => {
+                        let old_value = entry.get().clone();
+                        entry.insert(transform(Some(old_value)));
+                    }
+                }
+                return Ok(());
+            }
+        }
+
         let store_name = lifetime.as_str();
         let store = self.rkv.open_single(store_name, StoreOptions::create())?;
 
@@ -351,6 +423,16 @@ impl Database {
     ///
     /// * This function will **not** panic on database errors.
     pub fn clear_ping_lifetime_storage(&self, storage_name: &str) -> Result<()> {
+        // Lifetime::Ping might have data saved to `ping_lifetime_data`
+        // in case `delay_ping_lifetime_io` is set to true
+        if let Some(ping_lifetime_data) = &self.ping_lifetime_data {
+            ping_lifetime_data
+                .write()
+                .expect("Can't access ping lifetime data as writable")
+                .clear();
+            return Ok(());
+        }
+
         self.write_with_store(Lifetime::Ping, |mut writer, store| {
             let mut metrics = Vec::new();
             {
@@ -413,6 +495,18 @@ impl Database {
             return Ok(());
         }
 
+        // Lifetime::Ping data is not persisted to disk if
+        // Glean has `delay_ping_lifetime_io` set to true
+        if lifetime == Lifetime::Ping {
+            if let Some(ping_lifetime_data) = &self.ping_lifetime_data {
+                let mut data = ping_lifetime_data
+                    .write()
+                    .expect("Can't access app lifetime data as writable");
+                data.remove(&final_key);
+                return Ok(());
+            }
+        }
+
         self.write_with_store(lifetime, |mut writer, store| {
             store.delete(&mut writer, final_key.clone())?;
             writer.commit()?;
@@ -443,6 +537,13 @@ impl Database {
             .write()
             .expect("Can't access app lifetime data as writable")
             .clear();
+
+        if let Some(ping_lifetime_data) = &self.ping_lifetime_data {
+            ping_lifetime_data
+                .write()
+                .expect("Can't access ping lifetime data as writable")
+                .clear();
+        }
     }
 }
 
@@ -454,7 +555,7 @@ mod test {
 
     #[test]
     fn test_panicks_if_fails_dir_creation() {
-        assert!(Database::new("/!#\"'@#°ç").is_err());
+        assert!(Database::new("/!#\"'@#°ç", false).is_err());
     }
 
     #[test]
@@ -462,7 +563,7 @@ mod test {
         let dir = tempdir().unwrap();
         let str_dir = dir.path().display().to_string();
 
-        Database::new(&str_dir).unwrap();
+        Database::new(&str_dir, false).unwrap();
 
         assert!(dir.path().exists());
     }
@@ -472,7 +573,9 @@ mod test {
         // Init the database in a temporary directory.
         let dir = tempdir().unwrap();
         let str_dir = dir.path().display().to_string();
-        let db = Database::new(&str_dir).unwrap();
+        let db = Database::new(&str_dir, false).unwrap();
+
+        assert!(db.ping_lifetime_data.is_none());
 
         // Attempt to record a known value.
         let test_value = "test-value";
@@ -507,7 +610,7 @@ mod test {
         // Init the database in a temporary directory.
         let dir = tempdir().unwrap();
         let str_dir = dir.path().display().to_string();
-        let db = Database::new(&str_dir).unwrap();
+        let db = Database::new(&str_dir, false).unwrap();
 
         // Attempt to record a known value.
         let test_value = "test-value";
@@ -545,7 +648,7 @@ mod test {
         // Init the database in a temporary directory.
         let dir = tempdir().unwrap();
         let str_dir = dir.path().display().to_string();
-        let db = Database::new(&str_dir).unwrap();
+        let db = Database::new(&str_dir, false).unwrap();
 
         // Attempt to record a known value.
         let test_value = "test-value";
@@ -580,7 +683,7 @@ mod test {
         // Init the database in a temporary directory.
         let dir = tempdir().unwrap();
         let str_dir = dir.path().display().to_string();
-        let db = Database::new(&str_dir).unwrap();
+        let db = Database::new(&str_dir, false).unwrap();
 
         // Attempt to record a known value for every single lifetime.
         let test_storage = "test-storage";
@@ -656,7 +759,7 @@ mod test {
         // Init the database in a temporary directory.
         let dir = tempdir().unwrap();
         let str_dir = dir.path().display().to_string();
-        let db = Database::new(&str_dir).unwrap();
+        let db = Database::new(&str_dir, false).unwrap();
 
         let test_storage = "test-storage-single-lifetime";
         let metric_id_pattern = "telemetry_test.single_metric";
@@ -707,4 +810,53 @@ mod test {
             );
         }
     }
+
+    #[test]
+    fn test_deferred_ping_lifetime_collection() {
+        // Init the database in a temporary directory.
+        let dir = tempdir().unwrap();
+        let str_dir = dir.path().display().to_string();
+        let db = Database::new(&str_dir, true).unwrap();
+
+        assert!(db.ping_lifetime_data.is_some());
+
+        // Attempt to record a known value.
+        let test_value = "test-value";
+        let test_storage = "test-storage1";
+        let test_metric_id = "telemetry_test.test_name";
+        db.record_per_lifetime(
+            Lifetime::Ping,
+            test_storage,
+            test_metric_id,
+            &Metric::String(test_value.to_string()),
+        )
+        .unwrap();
+
+        // Verify that the data is correctly recorded.
+        let mut found_metrics = 0;
+        let mut snapshotter = |metric_name: &[u8], metric: &Metric| {
+            found_metrics += 1;
+            let metric_id = String::from_utf8_lossy(metric_name).into_owned();
+            assert_eq!(test_metric_id, metric_id);
+            match metric {
+                Metric::String(s) => assert_eq!(test_value, s),
+                _ => panic!("Unexpected data found"),
+            }
+        };
+
+        db.iter_store_from(Lifetime::Ping, test_storage, None, &mut snapshotter);
+        assert_eq!(1, found_metrics, "We only expect 1 Lifetime.Ping metric.");
+
+        // Make sure data was **not** persisted with rkv.
+        let store: SingleStore = unwrap_or!(
+            db.rkv
+                .open_single(Lifetime::Ping.as_str(), StoreOptions::create()),
+            panic!()
+        );
+        let reader = unwrap_or!(db.rkv.read(), panic!());
+        assert!(store
+            .get(&reader, &test_metric_id)
+            .unwrap_or(None)
+            .is_none());
+    }
 }

glean-core/src/lib.rs

@@ -67,6 +67,8 @@ pub struct Configuration {
     pub application_id: String,
     /// The maximum number of events to store before sending a ping containing events.
     pub max_events: Option<usize>,
+    /// Whether Glean should delay persistence of data from metrics with ping lifetime.
+    pub delay_ping_lifetime_io: bool,
 }
 
 /// The object holding meta information about a Glean instance.
@@ -83,6 +85,7 @@ pub struct Configuration {
 ///     application_id: "glean.sample.app".into(),
 ///     upload_enabled: true,
 ///     max_events: None,
+///     delay_ping_lifetime_io: false,
 /// };
 /// let mut glean = Glean::new(cfg).unwrap();
 /// let ping = PingType::new("sample", true, false);
@@ -130,7 +133,7 @@ impl Glean {
 
         // Creating the data store creates the necessary path as well.
         // If that fails we bail out and don't initialize further.
-        let data_store = Database::new(&cfg.data_path)?;
+        let data_store = Database::new(&cfg.data_path, cfg.delay_ping_lifetime_io)?;
         let event_data_store = EventDatabase::new(&cfg.data_path)?;
 
         let mut glean = Self {
@@ -194,6 +197,7 @@ impl Glean {
             application_id: application_id.into(),
             upload_enabled,
             max_events: None,
+            delay_ping_lifetime_io: false,
         };
 
         Self::new(cfg)

glean-core/src/lib_unit_tests.rs

@@ -344,6 +344,7 @@ fn glean_inits_with_migration_when_no_db_dir_exists() {
         application_id: GLOBAL_APPLICATION_ID.to_string(),
         upload_enabled: false,
         max_events: None,
+        delay_ping_lifetime_io: false,
     };
 
     let mut ac_seq_numbers = HashMap::new();

glean-core/tests/boolean.rs

@@ -22,6 +22,7 @@ fn boolean_serializer_should_correctly_serialize_boolean() {
         application_id: GLOBAL_APPLICATION_ID.into(),
         upload_enabled: true,
         max_events: None,
+        delay_ping_lifetime_io: false,
     };
 
     {

glean-core/tests/common/mod.rs

@@ -54,6 +54,7 @@ pub fn new_glean() -> (Glean, tempfile::TempDir) {
         application_id: GLOBAL_APPLICATION_ID.into(),
         upload_enabled: true,
         max_events: None,
+        delay_ping_lifetime_io: false,
     };
     let glean = Glean::new(cfg).unwrap();
 

glean-core/tests/counter.rs

@@ -25,6 +25,7 @@ fn counter_serializer_should_correctly_serialize_counters() {
         application_id: GLOBAL_APPLICATION_ID.into(),
         upload_enabled: true,
         max_events: None,
+        delay_ping_lifetime_io: false,
     };
 
     {

glean-core/tests/custom_distribution.rs

@@ -26,6 +26,7 @@ mod linear {
             application_id: GLOBAL_APPLICATION_ID.into(),
             upload_enabled: true,
             max_events: None,
+            delay_ping_lifetime_io: false,
         };
 
         {
@@ -245,6 +246,7 @@ mod exponential {
             application_id: GLOBAL_APPLICATION_ID.into(),
             upload_enabled: true,
             max_events: None,
+            delay_ping_lifetime_io: false,
         };
 
         {

glean-core/tests/datetime.rs

@@ -24,6 +24,7 @@ fn datetime_serializer_should_correctly_serialize_datetime() {
         application_id: GLOBAL_APPLICATION_ID.into(),
         upload_enabled: true,
         max_events: None,
+        delay_ping_lifetime_io: false,
     };
 
     {

glean-core/tests/labeled.rs

@@ -333,6 +333,7 @@ fn seen_labels_get_reloaded_from_disk() {
         application_id: GLOBAL_APPLICATION_ID.into(),
         upload_enabled: true,
         max_events: None,
+        delay_ping_lifetime_io: false,
     };
 
     let glean = Glean::new(cfg.clone()).unwrap();

glean-core/tests/memory_distribution.rs

@@ -26,6 +26,7 @@ fn serializer_should_correctly_serialize_memory_distribution() {
         application_id: GLOBAL_APPLICATION_ID.into(),
         upload_enabled: true,
         max_events: None,
+        delay_ping_lifetime_io: false,
     };
 
     {

glean-core/tests/ping_maker.rs

@@ -18,6 +18,7 @@ fn set_up_basic_ping() -> (Glean, PingMaker, PingType, tempfile::TempDir) {
         application_id: GLOBAL_APPLICATION_ID.into(),
         upload_enabled: true,
         max_events: None,
+        delay_ping_lifetime_io: false,
     };
     let mut glean = Glean::new(cfg).unwrap();
     let ping_maker = PingMaker::new();

glean-core/tests/quantity.rs

@@ -25,6 +25,7 @@ fn quantity_serializer_should_correctly_serialize_quantities() {
         application_id: GLOBAL_APPLICATION_ID.into(),
         upload_enabled: true,
         max_events: None,
+        delay_ping_lifetime_io: false,
     };
 
     {

glean-core/tests/string.rs

@@ -23,6 +23,7 @@ fn string_serializer_should_correctly_serialize_strings() {
         application_id: GLOBAL_APPLICATION_ID.into(),
         upload_enabled: true,
         max_events: None,
+        delay_ping_lifetime_io: false,
     };
 
     {

glean-core/tests/string_list.rs

@@ -49,6 +49,7 @@ fn stringlist_serializer_should_correctly_serialize_stringlists() {
         application_id: GLOBAL_APPLICATION_ID.into(),
         upload_enabled: true,
         max_events: None,
+        delay_ping_lifetime_io: false,
     };
 
     {

glean-core/tests/timespan.rs

@@ -24,6 +24,7 @@ fn serializer_should_correctly_serialize_timespans() {
         application_id: GLOBAL_APPLICATION_ID.into(),
         upload_enabled: true,
         max_events: None,
+        delay_ping_lifetime_io: false,
     };
 
     let duration = 60;

glean-core/tests/timing_distribution.rs

@@ -28,6 +28,7 @@ fn serializer_should_correctly_serialize_timing_distribution() {
         application_id: GLOBAL_APPLICATION_ID.into(),
         upload_enabled: true,
         max_events: None,
+        delay_ping_lifetime_io: false,
     };
 
     {

glean-core/tests/uuid.rs

@@ -47,6 +47,7 @@ fn uuid_serializer_should_correctly_serialize_uuids() {
         application_id: GLOBAL_APPLICATION_ID.into(),
         upload_enabled: true,
         max_events: None,
+        delay_ping_lifetime_io: false,
     };
 
     {