diff --git a/Cargo.lock b/Cargo.lock index f99bb0852f7d..af7e0bb146e0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2079,9 +2079,9 @@ dependencies = [ [[package]] name = "glean" -version = "33.5.0" +version = "33.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1d5556ec294a763121f57384cf09be9b7f8eebbfc075040f9120b84f6a1160b" +checksum = "8f52254ae2baf857eec45b424a0d2dfe6ac63f353b594cfa4bee033f8386b25c" dependencies = [ "crossbeam-channel", "glean-core", @@ -2091,14 +2091,13 @@ dependencies = [ "serde", "serde_json", "thiserror", - "uuid", ] [[package]] name = "glean-core" -version = "33.5.0" +version = "33.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "657786648aaf8df52b059cb5cedff3be250df149bd8f6364919166e9d02a398d" +checksum = "82e3c82289c1ce270c1accc086f9cac083c15ddc01d167e9c6e0b8ed32a4b47c" dependencies = [ "bincode", "chrono", diff --git a/third_party/rust/glean-core/.cargo-checksum.json b/third_party/rust/glean-core/.cargo-checksum.json index 84ec1051b2c4..5666f29551ab 100644 --- a/third_party/rust/glean-core/.cargo-checksum.json +++ b/third_party/rust/glean-core/.cargo-checksum.json @@ -1 +1 @@ -{"files":{"Cargo.lock":"dd3e09018353851aba9cb7f530a130d3e12c6b4955c8d8429edb2333f5607c82","Cargo.toml":"cd47b3cbe2659ff84a910beae026e2c4560e709020216e0170d29c83c7f0c805","LICENSE":"1f256ecad192880510e84ad60474eab7589218784b9a50bc7ceee34c2b91f1d5","README.md":"026495898699b54608eb4ec16074ffafc57920d80ccb59961c501a1ea28c9985","examples/sample.rs":"0454ca14d7f7196e1e3582d06e066f9f8b22c16221c1677470551fe6c96a7e31","src/common_metric_data.rs":"f129b428de6a7351a5fdb735fa7a087d1ae6e98834680317fe3dc639e5bd5183","src/database/mod.rs":"b0ec91ad79ff984582b0c575b855be241186990990a4ebb6e47949ec8146a776","src/debug.rs":"3b572057525a11034097823bedc8bf91ed3cbe79ee62ca484c3f216c54bd991c","src/error.rs":"2901776054943e8a6216e71d8fac44d4ef7485baee0fc45e424adadd22da30b6","src/error_recording.rs":"30049d508ac23005d93ab307b6782e7ce1298aa0416d6fe583ebe4e56d53cb8c","src/event_database/mod.rs":"61a20d29a506431e575d437e2ca94d860d5db19eadd07bbf6e8dd4f64b52b2df","src/histogram/exponential.rs":"9b2d288df4ce7b942b9606072dcf1fd0796edd26e3cc4d1879d27356e7db69fb","src/histogram/functional.rs":"48db95ef426578c784f08b7c065bbd0625184321ee4c6636d9ee8ff9fb45b5b8","src/histogram/linear.rs":"08ebb9e5cc0caaf3c69bcfdf018457aa28e4f9ceee04c95e62d2f21133fff1d1","src/histogram/mod.rs":"490c5ae9e359e8733a219853605a92fb3fc9fefe8f98adfe04537301698f95d1","src/internal_metrics.rs":"c33fdf080730b0c20260ed55704305d74c34caa5497d2966bb001e0338c29a03","src/internal_pings.rs":"f9f589f30510b207845a3b80e219b976e719db1266f7cffd8a589bfdc75b096e","src/lib.rs":"fc83c863d09e89570386ffe076a848a690a7e55844c056cb7ab55ef5015ad93b","src/lib_unit_tests.rs":"dc735281aea8c1458fdb6b8f33581bdfa61a74be280f32964bb44a8b921c0a4a","src/macros.rs":"3f3fc8c36dd05b7528b66b27e95b6494505a7a1f48abd013218de23ada6b5a43","src/metrics/boolean.rs":"c7e1e1beaf7d19c16da9c412834e76675b938db73c0b54d052ff457990c79a31","src/metrics/counter.rs":"625518bb2e306b968d1498e3639fcf69ca93651657624d5ffd9aee3b23738fb0","src/metrics/custom_distribution.rs":"25616facb184baf1f3a9e2629bcbbbb937f19f2f39f86b93e3413cc3fabac5e5","src/metrics/datetime.rs":"7dd9f1913fb45c4bc1dd7152ac2b42486e36ef6adba49c5bf34452c4cf91e582","src/metrics/event.rs":"4df5c8e3e34886b2d5b715e08bffe4520d4aa0c08e4333f0320d6c3168260cb8","src/metrics/experiment.rs":"0e1051778100bdf05767474a44f136f790f466a361735f104bd6be769d05a711","src/metrics/jwe.rs":"843f8810622af83e923e1cf99285e63e8186104ee3cac225cc59a38fde81eb87","src/metrics/labeled.rs":"d091c3b24fb61b480d09728b7f932e94cc79a282b55e211f65e680f82f182c0b","src/metrics/memory_distribution.rs":"14872ca5e0e530b1933cb86134fec9cb6a416cab01bba0bb6b4bc0801c359b75","src/metrics/memory_unit.rs":"973dde8ecd51adb47afbb0e841aac7dca5bd7e0c1be2386a203930b70c2d5964","src/metrics/mod.rs":"01a78030cf340337cc9a16c74199886ed8cfa422249be6a9f2e947ad4c66e16a","src/metrics/ping.rs":"7e20d184ff14810f834df61dba42dc3cf7d21aaf35dcb519b20354f70dae99c2","src/metrics/quantity.rs":"74aa7560a06e7ba6129ad9e2731b4b6e67ba4aa2725f5988519ad66c60184e55","src/metrics/string.rs":"592594c0978894542498ce809e484bfd1ab5013e364c5000298d97f7a7de4eee","src/metrics/string_list.rs":"4ec427020ff70f49383da4d2adef38e75d72f6af464c256701caef3eecbb7a58","src/metrics/time_unit.rs":"71bc38497ad9a21cbcb2461672d4269db75569c6a8a52cb68ebe773688ca02b7","src/metrics/timespan.rs":"57c657fa0460450b2a81b192aa8bdf0d71596ac717116cd847a0bf5d574badea","src/metrics/timing_distribution.rs":"fe0d2079e7bfef57a9574038bbcca9321449272c0abf811af6e584c0137d3cbe","src/metrics/uuid.rs":"69d780c96736d0fb5dd1e0c2777ec37db712d639c8533adf5e8cefe97bc0d9de","src/ping/mod.rs":"36bd57c88ab28deedceec945a6783669f760d1bb77e2e02c09b38460b5392ec2","src/storage/mod.rs":"ae1ea3d7a36b9c02fa408c600c609364c18419c0e1b207a61063f8ea0c2ab90a","src/system.rs":"5856288c10c7791f42e61054ede5b8cf66dc0f77372a7095a76b1c0ed1a4bf44","src/traits/boolean.rs":"09590f06bcaf49e6328b97832c6200b4264b3ac0504b7df03077fdbe1fda6583","src/traits/counter.rs":"2d6c2af955f921a3227a3bf45c3efe89e26b12a8da431363bb043b4e419c3f38","src/traits/custom_distribution.rs":"5865480096be648f701612afa10fcd54676951944a8dee8af3d86892804a19fe","src/traits/datetime.rs":"22e1867370534f3f165e5d4c54d7ecb0d961f7b67e76f5444385b7b44db80a15","src/traits/event.rs":"ec4fea9f797cd78222ad3cbd57bdf8bbcc9fa8c9d13a7d44d375bf83f8e564d1","src/traits/jwe.rs":"5d1ccea9ebdbef1360c30f1de538d9bdaf87cb34877f0ccfbea550168b93c343","src/traits/labeled.rs":"0ed06cfca61c95d956ca8447176dd621a84664fd283f66cd3540d6cb4183d6ab","src/traits/memory_distribution.rs":"ad1b51c4340654022a77208a370f7eb14979f01afa48e9bb8e8a3063d3a252bf","src/traits/mod.rs":"4982b12db6b7b37cee9b7f6b5a132bb163915a21beaa439dc772e182728ede97","src/traits/ping.rs":"4d009fbf6e61c6f1e48bb3401832bca6168611f847825ce363be2747f661ee12","src/traits/quantity.rs":"b00fc2fd0fabdd905092a93610aa30bb41ea4339ce3d3ed1b9fa2b34d744c4a0","src/traits/string.rs":"b09a6eff1b9abd9fd930ffb31cb9e5ae203e1bdb4d6de44d860fa94993dc1ac8","src/traits/string_list.rs":"902aeaa3decce6e065185eee563e33a28e8398676b89c9f5f1b9d3b6ce357e67","src/traits/timespan.rs":"7c2157ccf6399a49fe642e8a79b613b8f08ddec0861aa2f4bca0da9b6f34def0","src/traits/timing_distribution.rs":"399f8cfbeea8f4f56e50adda2aa21ebfe642d260ceaaf798b3e64ee6f30d5485","src/traits/uuid.rs":"701257639e2fcd1c425ec3a4d010ce88fcbe3d359c3371022ebe5fe33a6b2009","src/upload/directory.rs":"e67233a13cee73987627eeb34bba23689c7d74d4e1f1b40f951aac8c26cb6581","src/upload/mod.rs":"6aad3e4f2cdd672ef7ee13ec98ebbe57d24f0f98a252a9589e96c9ff7b4506a3","src/upload/policy.rs":"7df2aa21088a46bcde365207527557d7b24c5ccc28ee463cb391dc64898a10df","src/upload/request.rs":"67098f1ae8998408cd944c55548b910827e870f13d4b82402177619116d7affe","src/upload/result.rs":"ba92b12109e4f41c96dc8cdc5736f19b409fd862bfd65264888746484ac56a6a","src/util.rs":"d33920ae03582440e7fb503188984b7a25ce5ac5fc040ca5ad74380700fa7257","tests/boolean.rs":"9f4830f3b5f5ab6768060917deb1f546324a4744b8ed893197e31a0aed092947","tests/common/mod.rs":"88d7d2c551b8c8f9ebc9de79f6c35368f4914521f98d4287925b390888491681","tests/counter.rs":"39a2cd40335c164b3e7407f5505bbf571d7e4a71f032d476a7939f7dd1af90d0","tests/custom_distribution.rs":"09b5f5a18802c904ebfb0263daced188466c4e10e222fe8f126c23a99e528dcb","tests/datetime.rs":"230698c7101176730483289e90c320ef559197111a814cf2b8db185ab68f82fb","tests/event.rs":"a7dab9050d2c4bd32d5ebdee896928836897fe413338486ad20364bf64272b83","tests/jwe.rs":"fe3034e96e6a9468eabeb36f32465bf8fbf4a261b7b27498c112ad8b279af55f","tests/labeled.rs":"eadaf032d02539a4edb274779aec8922fbb813694f171821c8d9e624c8123089","tests/memory_distribution.rs":"d3be31e2de39085114155fb27916470665185caaf239ebee873d149e531f69ff","tests/metrics.rs":"0560e0645321628015610bbf1a90c0adbb12ad42e5a068399073572120377cbb","tests/ping.rs":"9f300fb017e188fee126e11923d82a3e012e534a925ff34e913738cadce1f0e7","tests/ping_maker.rs":"6b4295c9477f1ce5cecd6130b036fbe505b5af10920cda3a56d68f82601c71ab","tests/quantity.rs":"2543f0f57085d27b0bc0c25c24c17a7e376ac673cf9265a3a9dec202920679c5","tests/storage.rs":"d8d50aba92e17a3053fd80788f5fbbcc8f82293c11c19b58e2de6c4ee45f71ff","tests/string.rs":"959cbb11a229eea3d70ca88ea6ee38588473ea75bd06f3e7b1ab32536a9e3797","tests/string_list.rs":"4a0d1ac5667e9fe48d62fda1b2aef297152b94522bd031f9e65b696839c55235","tests/timespan.rs":"5d5d582483c779196c7d622dab208645df45a6856a5ae2c263274fba52181b80","tests/timing_distribution.rs":"83fa32334f342f56042a3ca0730b954f0eb25dbcdafb6b959f5f2911f9e55c63","tests/uuid.rs":"21f79bc0d456caefc1e7e7931f803da6144eb6af6e1d41aa687a6c9c32175f80"},"package":"657786648aaf8df52b059cb5cedff3be250df149bd8f6364919166e9d02a398d"} \ No newline at end of file +{"files":{"Cargo.lock":"b38b817b3098783ee8b9fe9d2c41cfae3c3e120dd056f2e280fe9bbf00579dba","Cargo.toml":"e735478488f802ebda924dee0d9a529dc263abca81d38c17f5b62f29fb92d5ac","LICENSE":"1f256ecad192880510e84ad60474eab7589218784b9a50bc7ceee34c2b91f1d5","README.md":"026495898699b54608eb4ec16074ffafc57920d80ccb59961c501a1ea28c9985","examples/sample.rs":"ea5b72b702cb6bfb4ad38041a006c40960f40664865a73a904a2d033c7f9f7f0","src/common_metric_data.rs":"3a8072d5939d3abade3fb4ab49156cde38902c40a96b215f01442f2bcfd1733b","src/database/mod.rs":"016c5505ac945a826f95519e985e7a2080865423d7525371475edff3746b4a83","src/debug.rs":"2a88e4236124935c9b284ae9090a0f064d299d7b986a5d487a64b220044eb7d8","src/error.rs":"e8f72051be943bc2e8e49b597499c841f2b895101378520cf690f31cfeb5a9da","src/error_recording.rs":"76e0abe16e375716c50c047ebf419a5bc26127470812eb7d984a9c1885407059","src/event_database/mod.rs":"34390ce4b3ee3c6961bf58630d4e059a021ba3f12ac05fab9954056c7b9a1f12","src/histogram/exponential.rs":"389d48bab03f7229445ee4d4c8a4c49f6b43303f658dc954da75142f3c040dc6","src/histogram/functional.rs":"1a63a305b48bcef7bc38136b40d916df4bb8f098dc602514ada54a9b091f6951","src/histogram/linear.rs":"5642c8983c3c1ce9b69c6ac99133c8ed5e67abe2398c5dbcf67e69d66880f4ca","src/histogram/mod.rs":"11e5d3b6440a33a3e5544d0e03ffd69fa2fdcabe603ad75e0327f3ed548e8ce3","src/internal_metrics.rs":"0da1bb710346f7ba995d773f9d18fb2ddc8fddb45da84dce3e320d00bbee3e43","src/internal_pings.rs":"f9f589f30510b207845a3b80e219b976e719db1266f7cffd8a589bfdc75b096e","src/lib.rs":"37c2acb80c3cba51aed986fe9bfc2892eab3424c92387da9bd55d20c41ac09bc","src/lib_unit_tests.rs":"ee5d8b332afea9169a14dd8584cef717f1901a5a15883d144b08c54b514c1168","src/macros.rs":"3f3fc8c36dd05b7528b66b27e95b6494505a7a1f48abd013218de23ada6b5a43","src/metrics/boolean.rs":"0b1b96a65da89aaa511a3c14fabe1ca41f5bff5479bbaa1394bd809a91ddc40f","src/metrics/counter.rs":"0fbd8bd3c3a57540d5b1d00701d841a7d03d99a0b1c9c99bf48320af0efc4ca8","src/metrics/custom_distribution.rs":"1a3cab15651d84bb8da30c9ce3e75acd11b67d3c57fa3fa07c8a6152c13c3c2d","src/metrics/datetime.rs":"043804f2b3f3f1eef40cd0cd7f68d6261dff07be5db2cf8351c1434f8b6c72c1","src/metrics/event.rs":"22f6bf3b8b544f9a3121d8c727ef83bf7103dc83adef8aedbddf89e7061757c1","src/metrics/experiment.rs":"6a1277186f4b2a4bdb20c19e7880232f009f2c72e7997ce8ac195d75f7b20c4d","src/metrics/jwe.rs":"562103bb34591f5d3da5932c6e5315df3a9d96ce3db50cc15ac905fb4dd52818","src/metrics/labeled.rs":"240955b03655ccb21df50b456b437bc888a26e773da8b370d45c182661e54e60","src/metrics/memory_distribution.rs":"b3e70b68938cef56d4fd7103a3583e6e679335928bb70c007904d880e774878d","src/metrics/memory_unit.rs":"c022ee797c3aae99e8c5f358ac9bbad3eaa5c924702e58d39711301ae75fb3a1","src/metrics/mod.rs":"8ae2ae5420f97f043a85ae890eadf222b2f301c05392e4fdec4e582b524e41e0","src/metrics/ping.rs":"314431e58e5bb57dca0fab48d2ffbe3cc8a232c834187ad86d279bb1a8ef84c4","src/metrics/quantity.rs":"96fa43f0739f831c9953cb418d51f480152ebe0fc7e42afe1af9120ed1afcf60","src/metrics/string.rs":"93f33e37bb883c7f553b7592d7984f61f07ec87e0c950e3298602861ca403caa","src/metrics/string_list.rs":"0903d5be42210d0f30371a3014cd36daa4636ff0793c9322ea84da94268b1184","src/metrics/time_unit.rs":"9bb3f6c03b1d385008b254f36ed35daf22f9a6fa2d18e007c9b70aac767cde50","src/metrics/timespan.rs":"03a2aeaedb8b4db7c2d5d5ebe2549961b153501449b98122395f9f198c004f05","src/metrics/timing_distribution.rs":"c5482b5bc1e999e938893e56dbd3bb2c10ebc1937209cccaeb68521b76c4fa95","src/metrics/uuid.rs":"89c9be40b2b87eb902922f4b7042ad344964721c24432b3f9e98f45e9cf9c448","src/ping/mod.rs":"fe44f78948b795b1cb8024c54adac1d41ef14e7249daf757bb5d2ba02d6addb5","src/storage/mod.rs":"511e863f430771bf488e0dc42b1f042575464cb03561f9e5333262a4e6f6c10d","src/system.rs":"cb33444d393bf4960ebe716a01de3465f2d5a28969ca1034d9cd1a5b31ed2a7f","src/traits/boolean.rs":"b2dd5aab4f7ccf082f8b6544f7c30da48f44e6be266a5f8e35cac4e01b583251","src/traits/counter.rs":"8187a36cf1f113c4d74db6d6d1b098ce12772bdaf035eeb8993d38ed5092fa24","src/traits/custom_distribution.rs":"5345ad30677a99c84c23a65a88c6583e204a494a4c22e40b5d344f44b7c9a365","src/traits/datetime.rs":"f8e710b391c6794dcb06d0da72384672704b166dfc49eacb469ebec6e851b8d8","src/traits/event.rs":"4fbeb2db3056254cd104c7686eb78b30578f164f2536431b54f4cb9d122c41be","src/traits/jwe.rs":"46afc39dfed61dec1c5ef5a46bb580d9813f8f5e7d841bff988e2c8af64c52e8","src/traits/labeled.rs":"ea2a8fdada07de456ec3fbd0945f694147e33051d79b287b6a62989e9972c3fa","src/traits/memory_distribution.rs":"f3fa94072c1a98da4ca355ce0dc191fa527881fe0a5e822b0a52a87894c071c5","src/traits/mod.rs":"80acb7ebe787219649c33cb759add781bbcdfffedebbd4ff38690ec25ced73e8","src/traits/ping.rs":"f920224255ed147bf1dd8904341c63b01721f44f6d45076778f9a2671a06bd5c","src/traits/quantity.rs":"efadd4bf188a3d6a70fbc68ac4e9662ca10e8de83f13c8ed2f4c4551d7a4a282","src/traits/string.rs":"32e9730039bdd86d2c609caf3643b87d9fff02ac31fba64e62f38d426b8b0dd9","src/traits/string_list.rs":"9f6013665f86b0061cccf0d69e6e223530002c4205ea1ebffda9bafb8747b059","src/traits/timespan.rs":"9ddfcb7bc82761349da31edf0374bbcd02db5fb46d9c4097eef6616cfb1c4e6b","src/traits/timing_distribution.rs":"be9bfc69467291fa9ce26758a4d5b058f453556b481152650fecfa4d97df1e46","src/traits/uuid.rs":"8f118bd888c10aeb7581faff998d036f846488bd0b75d381f0aa99959e20d915","src/upload/directory.rs":"f544576e72f1bd0747b5279e52bca78208044efb855018d3de0e2b287367297a","src/upload/mod.rs":"a7ea8744435d1f4801cf28b0400fc7d5fba3a07f38a5b035e713a1fc28ecec78","src/upload/policy.rs":"c250957a37783e74af8002cd80ba06ef9780a389fb0f61b8b665b79688f0a360","src/upload/request.rs":"37539bd7d5c4feaa26dd3667f949d395ae7837cf4a2d8173d2a7ddecc08c0632","src/upload/result.rs":"d73454e81f185e4e28c887f7b496bffad7baac74c8f1b719885f940f909a530d","src/util.rs":"c92d0d7b23e14fac46d8a003e3e2d81a58c6ff1b598bdbf5f116da9eb99d4d2f","tests/boolean.rs":"9f4830f3b5f5ab6768060917deb1f546324a4744b8ed893197e31a0aed092947","tests/common/mod.rs":"f4856ee4fdf0b29cfea7b251ad7048ec34e5d3d24b75befc649ac13a91cea2df","tests/counter.rs":"39a2cd40335c164b3e7407f5505bbf571d7e4a71f032d476a7939f7dd1af90d0","tests/custom_distribution.rs":"596575504ad4eab2765b5b88245dcc92dc68e83d04c93deb4b27692c142c0b94","tests/datetime.rs":"230698c7101176730483289e90c320ef559197111a814cf2b8db185ab68f82fb","tests/event.rs":"cb78dfc0a2315253da8ef6663b81979dd5b0638bb6c31456f16ce4e9bb50c67a","tests/jwe.rs":"dc8a73d1e1925ac49061e8bb5c8de01230aa364942a22470e37aa88965b0eaba","tests/labeled.rs":"4ac1a36223b5218232c6d83c5a2bfebbf547abe78db4b0fa3609abacc70c39bc","tests/memory_distribution.rs":"0c03d0a491150f508dc12ecac7e5db70043f3a7de2f90cc6b3509aba90549821","tests/metrics.rs":"0560e0645321628015610bbf1a90c0adbb12ad42e5a068399073572120377cbb","tests/ping.rs":"8b80e7d1cdffd6afc513780e9704e83de338697db8a459f0d34830f87aaa5be8","tests/ping_maker.rs":"f65ba2797c0d3acacd8a8b8cc6326d2d60a86284c48b15e707457644ecc680a6","tests/quantity.rs":"2543f0f57085d27b0bc0c25c24c17a7e376ac673cf9265a3a9dec202920679c5","tests/storage.rs":"d8d50aba92e17a3053fd80788f5fbbcc8f82293c11c19b58e2de6c4ee45f71ff","tests/string.rs":"9c3e21bfa3ad0b75373f8f5addb8ac023ac5336ac9f969e92b89ea6fda675fdd","tests/string_list.rs":"2b83710b949bea4bd0f6b2029069887f3ea87312b00091e5aa3c7bda0fb9073c","tests/timespan.rs":"5d5d582483c779196c7d622dab208645df45a6856a5ae2c263274fba52181b80","tests/timing_distribution.rs":"577f5ff0059509f6005014604097f31c7d01536d7f05571c76564af4b88edcbd","tests/uuid.rs":"21f79bc0d456caefc1e7e7931f803da6144eb6af6e1d41aa687a6c9c32175f80"},"package":"82e3c82289c1ce270c1accc086f9cac083c15ddc01d167e9c6e0b8ed32a4b47c"} \ No newline at end of file diff --git a/third_party/rust/glean-core/Cargo.lock b/third_party/rust/glean-core/Cargo.lock index 407736958d01..5fb503b86a45 100644 --- a/third_party/rust/glean-core/Cargo.lock +++ b/third_party/rust/glean-core/Cargo.lock @@ -53,9 +53,9 @@ checksum = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de" [[package]] name = "cc" -version = "1.0.65" +version = "1.0.62" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95752358c8f7552394baf48cd82695b345628ad3f170d607de3ca03b8dacca15" +checksum = "f1770ced377336a88a67c473594ccc14eca6f4559217c34f64aac8f83d641b40" [[package]] name = "cfg-if" @@ -180,7 +180,7 @@ dependencies = [ [[package]] name = "glean-core" -version = "33.5.0" +version = "33.4.0" dependencies = [ "bincode", "chrono", @@ -543,9 +543,9 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.53" +version = "1.0.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8833e20724c24de12bbaba5ad230ea61c3eafb05b881c7c9d3cfe8638b187e68" +checksum = "cc371affeffc477f42a221a1e4297aedcea33d47d19b61455588bd9d8f6b19ac" dependencies = [ "proc-macro2", "quote", @@ -580,9 +580,9 @@ dependencies = [ [[package]] name = "termcolor" -version = "1.1.2" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dfed899f0eb03f32ee8c6a0aabdb8a7949659e3466561fc0adf54e26d88c5f4" +checksum = "bb6bfa289a4d7c5766392812c0a1f4c1ba45afa1ad47803c11e1f407d846d75f" dependencies = [ "winapi-util", ] @@ -600,9 +600,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.1.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccf8dbc19eb42fba10e8feaaec282fb50e2c14b2726d6301dbfeed0f73306a6f" +checksum = "b78a366903f506d2ad52ca8dc552102ffdd3e937ba8a227f024dc1d1eae28575" dependencies = [ "tinyvec_macros", ] @@ -624,9 +624,9 @@ dependencies = [ [[package]] name = "unicode-normalization" -version = "0.1.16" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a13e63ab62dbe32aeee58d1c5408d35c36c392bba5d9d3142287219721afe606" +checksum = "b7f98e67a4d84f730d343392f9bfff7d21e3fca562b9cb7a43b768350beeddc6" dependencies = [ "tinyvec", ] diff --git a/third_party/rust/glean-core/Cargo.toml b/third_party/rust/glean-core/Cargo.toml index 79364c3d8ac0..e37485f387c1 100644 --- a/third_party/rust/glean-core/Cargo.toml +++ b/third_party/rust/glean-core/Cargo.toml @@ -13,7 +13,7 @@ [package] edition = "2018" name = "glean-core" -version = "33.5.0" +version = "33.4.0" authors = ["Jan-Erik Rediger ", "The Glean Team "] include = ["/README.md", "/LICENSE", "/src", "/examples", "/tests", "/Cargo.toml"] description = "A modern Telemetry library" diff --git a/third_party/rust/glean-core/examples/sample.rs b/third_party/rust/glean-core/examples/sample.rs index 87227e7cb7c1..7f292dde14c5 100644 --- a/third_party/rust/glean-core/examples/sample.rs +++ b/third_party/rust/glean-core/examples/sample.rs @@ -1,77 +1,77 @@ -use std::env; - -use glean_core::metrics::*; -use glean_core::ping::PingMaker; -use glean_core::{CommonMetricData, Glean}; -use tempfile::Builder; - -fn main() { - env_logger::init(); - - let mut args = env::args().skip(1); - - let data_path = if let Some(path) = args.next() { - path - } else { - let root = Builder::new().prefix("simple-db").tempdir().unwrap(); - root.path().display().to_string() - }; - - let cfg = glean_core::Configuration { - data_path, - application_id: "org.mozilla.glean_core.example".into(), - language_binding_name: "Rust".into(), - upload_enabled: true, - max_events: None, - delay_ping_lifetime_io: false, - }; - let mut glean = Glean::new(cfg).unwrap(); - glean.register_ping_type(&PingType::new("baseline", true, false, vec![])); - glean.register_ping_type(&PingType::new("metrics", true, false, vec![])); - - let local_metric: StringMetric = StringMetric::new(CommonMetricData { - name: "local_metric".into(), - category: "local".into(), - send_in_pings: vec!["baseline".into()], - ..Default::default() - }); - - let call_counter: CounterMetric = CounterMetric::new(CommonMetricData { - name: "calls".into(), - category: "local".into(), - send_in_pings: vec!["baseline".into(), "metrics".into()], - ..Default::default() - }); - - local_metric.set(&glean, "I can set this"); - call_counter.add(&glean, 1); - - println!("Baseline Data:\n{}", glean.snapshot("baseline", true)); - - call_counter.add(&glean, 2); - println!("Metrics Data:\n{}", glean.snapshot("metrics", true)); - - call_counter.add(&glean, 3); - - println!(); - println!("Baseline Data 2:\n{}", glean.snapshot("baseline", false)); - println!("Metrics Data 2:\n{}", glean.snapshot("metrics", true)); - - let list: StringListMetric = StringListMetric::new(CommonMetricData { - name: "list".into(), - category: "local".into(), - send_in_pings: vec!["baseline".into()], - ..Default::default() - }); - list.add(&glean, "once"); - list.add(&glean, "upon"); - - let ping_maker = PingMaker::new(); - let ping = ping_maker - .collect_string(&glean, glean.get_ping_by_name("baseline").unwrap(), None) - .unwrap(); - println!("Baseline Ping:\n{}", ping); - - let ping = ping_maker.collect_string(&glean, glean.get_ping_by_name("metrics").unwrap(), None); - println!("Metrics Ping: {:?}", ping); -} +use std::env; + +use glean_core::metrics::*; +use glean_core::ping::PingMaker; +use glean_core::{CommonMetricData, Glean}; +use tempfile::Builder; + +fn main() { + env_logger::init(); + + let mut args = env::args().skip(1); + + let data_path = if let Some(path) = args.next() { + path + } else { + let root = Builder::new().prefix("simple-db").tempdir().unwrap(); + root.path().display().to_string() + }; + + let cfg = glean_core::Configuration { + data_path, + application_id: "org.mozilla.glean_core.example".into(), + language_binding_name: "Rust".into(), + upload_enabled: true, + max_events: None, + delay_ping_lifetime_io: false, + }; + let mut glean = Glean::new(cfg).unwrap(); + glean.register_ping_type(&PingType::new("baseline", true, false, vec![])); + glean.register_ping_type(&PingType::new("metrics", true, false, vec![])); + + let local_metric: StringMetric = StringMetric::new(CommonMetricData { + name: "local_metric".into(), + category: "local".into(), + send_in_pings: vec!["baseline".into()], + ..Default::default() + }); + + let call_counter: CounterMetric = CounterMetric::new(CommonMetricData { + name: "calls".into(), + category: "local".into(), + send_in_pings: vec!["baseline".into(), "metrics".into()], + ..Default::default() + }); + + local_metric.set(&glean, "I can set this"); + call_counter.add(&glean, 1); + + println!("Baseline Data:\n{}", glean.snapshot("baseline", true)); + + call_counter.add(&glean, 2); + println!("Metrics Data:\n{}", glean.snapshot("metrics", true)); + + call_counter.add(&glean, 3); + + println!(); + println!("Baseline Data 2:\n{}", glean.snapshot("baseline", false)); + println!("Metrics Data 2:\n{}", glean.snapshot("metrics", true)); + + let list: StringListMetric = StringListMetric::new(CommonMetricData { + name: "list".into(), + category: "local".into(), + send_in_pings: vec!["baseline".into()], + ..Default::default() + }); + list.add(&glean, "once"); + list.add(&glean, "upon"); + + let ping_maker = PingMaker::new(); + let ping = ping_maker + .collect_string(&glean, glean.get_ping_by_name("baseline").unwrap(), None) + .unwrap(); + println!("Baseline Ping:\n{}", ping); + + let ping = ping_maker.collect_string(&glean, glean.get_ping_by_name("metrics").unwrap(), None); + println!("Metrics Ping: {:?}", ping); +} diff --git a/third_party/rust/glean-core/src/common_metric_data.rs b/third_party/rust/glean-core/src/common_metric_data.rs index 8369d37c03ab..e5b162ca3ab5 100644 --- a/third_party/rust/glean-core/src/common_metric_data.rs +++ b/third_party/rust/glean-core/src/common_metric_data.rs @@ -1,127 +1,127 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -use std::convert::TryFrom; - -use crate::error::{Error, ErrorKind}; -use crate::metrics::dynamic_label; -use crate::Glean; - -/// The supported metrics' lifetimes. -/// -/// A metric's lifetime determines when its stored data gets reset. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -#[repr(i32)] // Use i32 to be compatible with our JNA definition -pub enum Lifetime { - /// The metric is reset with each sent ping - Ping, - /// The metric is reset on application restart - Application, - /// The metric is reset with each user profile - User, -} - -impl Default for Lifetime { - fn default() -> Self { - Lifetime::Ping - } -} - -impl Lifetime { - /// String representation of the lifetime. - pub fn as_str(self) -> &'static str { - match self { - Lifetime::Ping => "ping", - Lifetime::Application => "app", - Lifetime::User => "user", - } - } -} - -impl TryFrom for Lifetime { - type Error = Error; - - fn try_from(value: i32) -> Result { - match value { - 0 => Ok(Lifetime::Ping), - 1 => Ok(Lifetime::Application), - 2 => Ok(Lifetime::User), - e => Err(ErrorKind::Lifetime(e).into()), - } - } -} - -/// The common set of data shared across all different metric types. -#[derive(Default, Debug, Clone)] -pub struct CommonMetricData { - /// The metric's name. - pub name: String, - /// The metric's category. - pub category: String, - /// List of ping names to include this metric in. - pub send_in_pings: Vec, - /// The metric's lifetime. - pub lifetime: Lifetime, - /// Whether or not the metric is disabled. - /// - /// Disabled metrics are never recorded. - pub disabled: bool, - /// Dynamic label. - /// When a LabeledMetric factory creates the specific metric to be - /// recorded to, dynamic labels are stored in the specific label so that we - /// can validate them when the Glean singleton is available. - pub dynamic_label: Option, -} - -impl CommonMetricData { - /// Creates a new metadata object. - pub fn new, B: Into, C: Into>( - category: A, - name: B, - ping_name: C, - ) -> CommonMetricData { - CommonMetricData { - name: name.into(), - category: category.into(), - send_in_pings: vec![ping_name.into()], - ..Default::default() - } - } - - /// The metric's base identifier, including the category and name, but not the label. - /// - /// If `category` is empty, it's ommitted. - /// Otherwise, it's the combination of the metric's `category` and `name`. - pub(crate) fn base_identifier(&self) -> String { - if self.category.is_empty() { - self.name.clone() - } else { - format!("{}.{}", self.category, self.name) - } - } - - /// The metric's unique identifier, including the category, name and label. - /// - /// If `category` is empty, it's ommitted. - /// Otherwise, it's the combination of the metric's `category`, `name` and `label`. - pub(crate) fn identifier(&self, glean: &Glean) -> String { - let base_identifier = self.base_identifier(); - - if let Some(label) = &self.dynamic_label { - dynamic_label(glean, self, &base_identifier, label) - } else { - base_identifier - } - } - - /// Whether this metric should be recorded. - pub fn should_record(&self) -> bool { - !self.disabled - } - - /// The list of storages this metric should be recorded into. - pub fn storage_names(&self) -> &[String] { - &self.send_in_pings - } -} +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::convert::TryFrom; + +use crate::error::{Error, ErrorKind}; +use crate::metrics::dynamic_label; +use crate::Glean; + +/// The supported metrics' lifetimes. +/// +/// A metric's lifetime determines when its stored data gets reset. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +#[repr(i32)] // Use i32 to be compatible with our JNA definition +pub enum Lifetime { + /// The metric is reset with each sent ping + Ping, + /// The metric is reset on application restart + Application, + /// The metric is reset with each user profile + User, +} + +impl Default for Lifetime { + fn default() -> Self { + Lifetime::Ping + } +} + +impl Lifetime { + /// String representation of the lifetime. + pub fn as_str(self) -> &'static str { + match self { + Lifetime::Ping => "ping", + Lifetime::Application => "app", + Lifetime::User => "user", + } + } +} + +impl TryFrom for Lifetime { + type Error = Error; + + fn try_from(value: i32) -> Result { + match value { + 0 => Ok(Lifetime::Ping), + 1 => Ok(Lifetime::Application), + 2 => Ok(Lifetime::User), + e => Err(ErrorKind::Lifetime(e).into()), + } + } +} + +/// The common set of data shared across all different metric types. +#[derive(Default, Debug, Clone)] +pub struct CommonMetricData { + /// The metric's name. + pub name: String, + /// The metric's category. + pub category: String, + /// List of ping names to include this metric in. + pub send_in_pings: Vec, + /// The metric's lifetime. + pub lifetime: Lifetime, + /// Whether or not the metric is disabled. + /// + /// Disabled metrics are never recorded. + pub disabled: bool, + /// Dynamic label. + /// When a LabeledMetric factory creates the specific metric to be + /// recorded to, dynamic labels are stored in the specific label so that we + /// can validate them when the Glean singleton is available. + pub dynamic_label: Option, +} + +impl CommonMetricData { + /// Creates a new metadata object. + pub fn new, B: Into, C: Into>( + category: A, + name: B, + ping_name: C, + ) -> CommonMetricData { + CommonMetricData { + name: name.into(), + category: category.into(), + send_in_pings: vec![ping_name.into()], + ..Default::default() + } + } + + /// The metric's base identifier, including the category and name, but not the label. + /// + /// If `category` is empty, it's ommitted. + /// Otherwise, it's the combination of the metric's `category` and `name`. + pub(crate) fn base_identifier(&self) -> String { + if self.category.is_empty() { + self.name.clone() + } else { + format!("{}.{}", self.category, self.name) + } + } + + /// The metric's unique identifier, including the category, name and label. + /// + /// If `category` is empty, it's ommitted. + /// Otherwise, it's the combination of the metric's `category`, `name` and `label`. + pub(crate) fn identifier(&self, glean: &Glean) -> String { + let base_identifier = self.base_identifier(); + + if let Some(label) = &self.dynamic_label { + dynamic_label(glean, self, &base_identifier, label) + } else { + base_identifier + } + } + + /// Whether this metric should be recorded. + pub fn should_record(&self) -> bool { + !self.disabled + } + + /// The list of storages this metric should be recorded into. + pub fn storage_names(&self) -> &[String] { + &self.send_in_pings + } +} diff --git a/third_party/rust/glean-core/src/database/mod.rs b/third_party/rust/glean-core/src/database/mod.rs index c95567b33e20..6abe1ada95a6 100644 --- a/third_party/rust/glean-core/src/database/mod.rs +++ b/third_party/rust/glean-core/src/database/mod.rs @@ -1,1572 +1,1572 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -use std::collections::btree_map::Entry; -use std::collections::BTreeMap; -use std::fs; -use std::num::NonZeroU64; -use std::path::Path; -use std::str; -use std::sync::RwLock; - -use rkv::StoreOptions; - -// Select the LMDB-powered storage backend when the feature is not activated. -#[cfg(not(feature = "rkv-safe-mode"))] -mod backend { - use std::path::Path; - - /// cbindgen:ignore - pub type Rkv = rkv::Rkv; - /// cbindgen:ignore - pub type SingleStore = rkv::SingleStore; - /// cbindgen:ignore - pub type Writer<'t> = rkv::Writer>; - - pub fn rkv_new(path: &Path) -> Result { - Rkv::new::(path) - } - - /// No migration necessary when staying with LMDB. - pub fn migrate(_path: &Path, _dst_env: &Rkv) { - // Intentionally left empty. - } -} - -// Select the "safe mode" storage backend when the feature is activated. -#[cfg(feature = "rkv-safe-mode")] -mod backend { - use rkv::migrator::Migrator; - use std::{fs, path::Path}; - - /// cbindgen:ignore - pub type Rkv = rkv::Rkv; - /// cbindgen:ignore - pub type SingleStore = rkv::SingleStore; - /// cbindgen:ignore - pub type Writer<'t> = rkv::Writer>; - - pub fn rkv_new(path: &Path) -> Result { - Rkv::new::(path) - } - - fn delete_and_log(path: &Path, msg: &str) { - if let Err(err) = fs::remove_file(path) { - match err.kind() { - std::io::ErrorKind::NotFound => { - // Silently drop this error, the file was already non-existing. - } - _ => log::warn!("{}", msg), - } - } - } - - fn delete_lmdb_database(path: &Path) { - let datamdb = path.join("data.mdb"); - delete_and_log(&datamdb, "Failed to delete old data."); - - let lockmdb = path.join("lock.mdb"); - delete_and_log(&lockmdb, "Failed to delete old lock."); - } - - /// Migrate from LMDB storage to safe-mode storage. - /// - /// This migrates the data once, then deletes the LMDB storage. - /// The safe-mode storage must be empty for it to work. - /// Existing data will not be overwritten. - /// If the destination database is not empty the LMDB database is deleted - /// without migrating data. - /// This is a no-op if no LMDB database file exists. - pub fn migrate(path: &Path, dst_env: &Rkv) { - use rkv::{MigrateError, StoreError}; - - log::debug!("Migrating files in {}", path.display()); - - // Shortcut if no data to migrate is around. - let datamdb = path.join("data.mdb"); - if !datamdb.exists() { - log::debug!("No data to migrate."); - return; - } - - // We're handling the same error cases as `easy_migrate_lmdb_to_safe_mode`, - // but annotate each why they don't cause problems for Glean. - // Additionally for known cases we delete the LMDB database regardless. - let should_delete = - match Migrator::open_and_migrate_lmdb_to_safe_mode(path, |builder| builder, dst_env) { - // Source environment is corrupted. - // We start fresh with the new database. - Err(MigrateError::StoreError(StoreError::FileInvalid)) => true, - // Path not accessible. - // Somehow our directory vanished between us creating it and reading from it. - // Nothing we can do really. - Err(MigrateError::StoreError(StoreError::IoError(_))) => true, - // Path accessible but incompatible for configuration. - // This should not happen, we never used storages that safe-mode doesn't understand. - // If it does happen, let's start fresh and use the safe-mode from now on. - Err(MigrateError::StoreError(StoreError::UnsuitableEnvironmentPath(_))) => true, - // Nothing to migrate. - // Source database was empty. We just start fresh anyway. - Err(MigrateError::SourceEmpty) => true, - // Migrating would overwrite. - // Either a previous migration failed and we still started writing data, - // or someone placed back an old data file. - // In any case we better stay on the new data and delete the old one. - Err(MigrateError::DestinationNotEmpty) => { - log::warn!("Failed to migrate old data. Destination was not empty"); - true - } - // An internal lock was poisoned. - // This would only happen if multiple things run concurrently and one crashes. - Err(MigrateError::ManagerPoisonError) => false, - // Other store errors are never returned from the migrator. - // We need to handle them to please rustc. - Err(MigrateError::StoreError(_)) => false, - // Other errors can't happen, so this leaves us with the Ok case. - // This already deleted the LMDB files. - Ok(()) => false, - }; - - if should_delete { - log::debug!("Need to delete remaining LMDB files."); - delete_lmdb_database(&path); - } - - log::debug!("Migration ended. Safe-mode database in {}", path.display()); - } -} - -use crate::metrics::Metric; -use crate::CommonMetricData; -use crate::Glean; -use crate::Lifetime; -use crate::Result; -use backend::*; - -pub struct Database { - /// Handle to the database environment. - rkv: Rkv, - - /// Handles to the "lifetime" stores. - /// - /// A "store" is a handle to the underlying database. - /// We keep them open for fast and frequent access. - user_store: SingleStore, - ping_store: SingleStore, - application_store: SingleStore, - - /// If the `delay_ping_lifetime_io` Glean config option is `true`, - /// we will save metrics with 'ping' lifetime data in a map temporarily - /// so as to persist them to disk using rkv in bulk on demand. - ping_lifetime_data: Option>>, - - // Initial file size when opening the database. - file_size: Option, -} - -impl std::fmt::Debug for Database { - fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result { - fmt.debug_struct("Database") - .field("rkv", &self.rkv) - .field("user_store", &"SingleStore") - .field("ping_store", &"SingleStore") - .field("application_store", &"SingleStore") - .field("ping_lifetime_data", &self.ping_lifetime_data) - .finish() - } -} - -/// Calculate the database size from all the files in the directory. -/// -/// # Arguments -/// -/// *`path` - The path to the directory -/// -/// # Returns -/// -/// Returns the non-zero combined size of all files in a directory, -/// or `None` on error or if the size is `0`. -fn database_size(dir: &Path) -> Option { - let mut total_size = 0; - if let Ok(entries) = fs::read_dir(dir) { - for entry in entries { - if let Ok(entry) = entry { - if let Ok(file_type) = entry.file_type() { - if file_type.is_file() { - let path = entry.path(); - if let Ok(metadata) = fs::metadata(path) { - total_size += metadata.len(); - } else { - continue; - } - } - } - } - } - } - - NonZeroU64::new(total_size) -} - -impl Database { - /// Initializes the data store. - /// - /// This opens the underlying rkv store and creates - /// the underlying directory structure. - /// - /// It also loads any Lifetime::Ping data that might be - /// persisted, in case `delay_ping_lifetime_io` is set. - pub fn new(data_path: &str, delay_ping_lifetime_io: bool) -> Result { - let path = Path::new(data_path).join("db"); - log::debug!("Database path: {:?}", path.display()); - let file_size = database_size(&path); - - let rkv = Self::open_rkv(&path)?; - let user_store = rkv.open_single(Lifetime::User.as_str(), StoreOptions::create())?; - let ping_store = rkv.open_single(Lifetime::Ping.as_str(), StoreOptions::create())?; - let application_store = - rkv.open_single(Lifetime::Application.as_str(), StoreOptions::create())?; - let ping_lifetime_data = if delay_ping_lifetime_io { - Some(RwLock::new(BTreeMap::new())) - } else { - None - }; - - let db = Self { - rkv, - user_store, - ping_store, - application_store, - ping_lifetime_data, - file_size, - }; - - db.load_ping_lifetime_data(); - - Ok(db) - } - - /// Get the initial database file size. - pub fn file_size(&self) -> Option { - self.file_size - } - - fn get_store(&self, lifetime: Lifetime) -> &SingleStore { - match lifetime { - Lifetime::User => &self.user_store, - Lifetime::Ping => &self.ping_store, - Lifetime::Application => &self.application_store, - } - } - - /// Creates the storage directories and inits rkv. - fn open_rkv(path: &Path) -> Result { - fs::create_dir_all(&path)?; - - let rkv = rkv_new(&path)?; - migrate(path, &rkv); - - log::info!("Database initialized"); - Ok(rkv) - } - - /// Build the key of the final location of the data in the database. - /// Such location is built using the storage name and the metric - /// key/name (if available). - /// - /// # Arguments - /// - /// * `storage_name` - the name of the storage to store/fetch data from. - /// * `metric_key` - the optional metric key/name. - /// - /// # Returns - /// - /// A string representing the location in the database. - fn get_storage_key(storage_name: &str, metric_key: Option<&str>) -> String { - match metric_key { - Some(k) => format!("{}#{}", storage_name, k), - None => format!("{}#", storage_name), - } - } - - /// Loads Lifetime::Ping data from rkv to memory, - /// if `delay_ping_lifetime_io` is set to true. - /// - /// Does nothing if it isn't or if there is not data to load. - fn load_ping_lifetime_data(&self) { - if let Some(ping_lifetime_data) = &self.ping_lifetime_data { - let mut data = ping_lifetime_data - .write() - .expect("Can't read ping lifetime data"); - - let reader = unwrap_or!(self.rkv.read(), return); - let store = self.get_store(Lifetime::Ping); - let mut iter = unwrap_or!(store.iter_start(&reader), return); - - while let Some(Ok((metric_id, value))) = iter.next() { - let metric_id = match str::from_utf8(metric_id) { - Ok(metric_id) => metric_id.to_string(), - _ => continue, - }; - let metric: Metric = match value { - rkv::Value::Blob(blob) => unwrap_or!(bincode::deserialize(blob), continue), - _ => continue, - }; - - data.insert(metric_id, metric); - } - } - } - - /// Iterates with the provided transaction function - /// over the requested data from the given storage. - /// - /// * If the storage is unavailable, the transaction function is never invoked. - /// * If the read data cannot be deserialized it will be silently skipped. - /// - /// # Arguments - /// - /// * `lifetime` - The metric lifetime to iterate over. - /// * `storage_name` - The storage name to iterate over. - /// * `metric_key` - The metric key to iterate over. All metrics iterated over - /// will have this prefix. For example, if `metric_key` is of the form `{category}.`, - /// it will iterate over all metrics in the given category. If the `metric_key` is of the - /// form `{category}.{name}/`, the iterator will iterate over all specific metrics for - /// a given labeled metric. If not provided, the entire storage for the given lifetime - /// will be iterated over. - /// * `transaction_fn` - Called for each entry being iterated over. It is - /// passed two arguments: `(metric_id: &[u8], metric: &Metric)`. - /// - /// # Panics - /// - /// This function will **not** panic on database errors. - pub fn iter_store_from( - &self, - lifetime: Lifetime, - storage_name: &str, - metric_key: Option<&str>, - mut transaction_fn: F, - ) where - F: FnMut(&[u8], &Metric), - { - let iter_start = Self::get_storage_key(storage_name, metric_key); - let len = iter_start.len(); - - // Lifetime::Ping data is not immediately persisted to disk if - // Glean has `delay_ping_lifetime_io` set to true - if lifetime == Lifetime::Ping { - if let Some(ping_lifetime_data) = &self.ping_lifetime_data { - let data = ping_lifetime_data - .read() - .expect("Can't read ping lifetime data"); - for (key, value) in data.iter() { - if key.starts_with(&iter_start) { - let key = &key[len..]; - transaction_fn(key.as_bytes(), value); - } - } - return; - } - } - - let reader = unwrap_or!(self.rkv.read(), return); - let mut iter = unwrap_or!( - self.get_store(lifetime).iter_from(&reader, &iter_start), - return - ); - - while let Some(Ok((metric_id, value))) = iter.next() { - if !metric_id.starts_with(iter_start.as_bytes()) { - break; - } - - let metric_id = &metric_id[len..]; - let metric: Metric = match value { - rkv::Value::Blob(blob) => unwrap_or!(bincode::deserialize(blob), continue), - _ => continue, - }; - transaction_fn(metric_id, &metric); - } - } - - /// Determines if the storage has the given metric. - /// - /// If data cannot be read it is assumed that the storage does not have the metric. - /// - /// # Arguments - /// - /// * `lifetime` - The lifetime of the metric. - /// * `storage_name` - The storage name to look in. - /// * `metric_identifier` - The metric identifier. - /// - /// # Panics - /// - /// This function will **not** panic on database errors. - pub fn has_metric( - &self, - lifetime: Lifetime, - storage_name: &str, - metric_identifier: &str, - ) -> bool { - let key = Self::get_storage_key(storage_name, Some(metric_identifier)); - - // Lifetime::Ping data is not persisted to disk if - // Glean has `delay_ping_lifetime_io` set to true - if lifetime == Lifetime::Ping { - if let Some(ping_lifetime_data) = &self.ping_lifetime_data { - return ping_lifetime_data - .read() - .map(|data| data.contains_key(&key)) - .unwrap_or(false); - } - } - - let reader = unwrap_or!(self.rkv.read(), return false); - self.get_store(lifetime) - .get(&reader, &key) - .unwrap_or(None) - .is_some() - } - - /// Writes to the specified storage with the provided transaction function. - /// - /// If the storage is unavailable, it will return an error. - /// - /// # Panics - /// - /// * This function will **not** panic on database errors. - fn write_with_store(&self, store_name: Lifetime, mut transaction_fn: F) -> Result<()> - where - F: FnMut(Writer, &SingleStore) -> Result<()>, - { - let writer = self.rkv.write().unwrap(); - let store = self.get_store(store_name); - transaction_fn(writer, store) - } - - /// Records a metric in the underlying storage system. - pub fn record(&self, glean: &Glean, data: &CommonMetricData, value: &Metric) { - // If upload is disabled we don't want to record. - if !glean.is_upload_enabled() { - return; - } - - let name = data.identifier(glean); - - for ping_name in data.storage_names() { - if let Err(e) = self.record_per_lifetime(data.lifetime, ping_name, &name, value) { - log::error!("Failed to record metric into {}: {:?}", ping_name, e); - } - } - } - - /// Records a metric in the underlying storage system, for a single lifetime. - /// - /// # Returns - /// - /// If the storage is unavailable or the write fails, no data will be stored and an error will be returned. - /// - /// Otherwise `Ok(())` is returned. - /// - /// # Panics - /// - /// This function will **not** panic on database errors. - fn record_per_lifetime( - &self, - lifetime: Lifetime, - storage_name: &str, - key: &str, - metric: &Metric, - ) -> Result<()> { - let final_key = Self::get_storage_key(storage_name, Some(key)); - - // Lifetime::Ping data is not immediately persisted to disk if - // Glean has `delay_ping_lifetime_io` set to true - if lifetime == Lifetime::Ping { - if let Some(ping_lifetime_data) = &self.ping_lifetime_data { - let mut data = ping_lifetime_data - .write() - .expect("Can't read ping lifetime data"); - data.insert(final_key, metric.clone()); - return Ok(()); - } - } - - let encoded = bincode::serialize(&metric).expect("IMPOSSIBLE: Serializing metric failed"); - let value = rkv::Value::Blob(&encoded); - - let mut writer = self.rkv.write()?; - self.get_store(lifetime) - .put(&mut writer, final_key, &value)?; - writer.commit()?; - Ok(()) - } - - /// Records the provided value, with the given lifetime, - /// after applying a transformation function. - pub fn record_with(&self, glean: &Glean, data: &CommonMetricData, mut transform: F) - where - F: FnMut(Option) -> Metric, - { - // If upload is disabled we don't want to record. - if !glean.is_upload_enabled() { - return; - } - - let name = data.identifier(glean); - for ping_name in data.storage_names() { - if let Err(e) = - self.record_per_lifetime_with(data.lifetime, ping_name, &name, &mut transform) - { - log::error!("Failed to record metric into {}: {:?}", ping_name, e); - } - } - } - - /// Records a metric in the underlying storage system, - /// after applying the given transformation function, for a single lifetime. - /// - /// # Returns - /// - /// If the storage is unavailable or the write fails, no data will be stored and an error will be returned. - /// - /// Otherwise `Ok(())` is returned. - /// - /// # Panics - /// - /// This function will **not** panic on database errors. - fn record_per_lifetime_with( - &self, - lifetime: Lifetime, - storage_name: &str, - key: &str, - mut transform: F, - ) -> Result<()> - where - F: FnMut(Option) -> Metric, - { - let final_key = Self::get_storage_key(storage_name, Some(key)); - - // Lifetime::Ping data is not persisted to disk if - // Glean has `delay_ping_lifetime_io` set to true - if lifetime == Lifetime::Ping { - if let Some(ping_lifetime_data) = &self.ping_lifetime_data { - let mut data = ping_lifetime_data - .write() - .expect("Can't access ping lifetime data as writable"); - let entry = data.entry(final_key); - match entry { - Entry::Vacant(entry) => { - entry.insert(transform(None)); - } - Entry::Occupied(mut entry) => { - let old_value = entry.get().clone(); - entry.insert(transform(Some(old_value))); - } - } - return Ok(()); - } - } - - let mut writer = self.rkv.write()?; - let store = self.get_store(lifetime); - let new_value: Metric = { - let old_value = store.get(&writer, &final_key)?; - - match old_value { - Some(rkv::Value::Blob(blob)) => { - let old_value = bincode::deserialize(blob).ok(); - transform(old_value) - } - _ => transform(None), - } - }; - - let encoded = - bincode::serialize(&new_value).expect("IMPOSSIBLE: Serializing metric failed"); - let value = rkv::Value::Blob(&encoded); - store.put(&mut writer, final_key, &value)?; - writer.commit()?; - Ok(()) - } - - /// Clears a storage (only Ping Lifetime). - /// - /// # Returns - /// - /// * If the storage is unavailable an error is returned. - /// * If any individual delete fails, an error is returned, but other deletions might have - /// happened. - /// - /// Otherwise `Ok(())` is returned. - /// - /// # Panics - /// - /// This function will **not** panic on database errors. - pub fn clear_ping_lifetime_storage(&self, storage_name: &str) -> Result<()> { - // Lifetime::Ping data will be saved to `ping_lifetime_data` - // in case `delay_ping_lifetime_io` is set to true - if let Some(ping_lifetime_data) = &self.ping_lifetime_data { - ping_lifetime_data - .write() - .expect("Can't access ping lifetime data as writable") - .clear(); - } - - self.write_with_store(Lifetime::Ping, |mut writer, store| { - let mut metrics = Vec::new(); - { - let mut iter = store.iter_from(&writer, &storage_name)?; - while let Some(Ok((metric_id, _))) = iter.next() { - if let Ok(metric_id) = std::str::from_utf8(metric_id) { - if !metric_id.starts_with(&storage_name) { - break; - } - metrics.push(metric_id.to_owned()); - } - } - } - - let mut res = Ok(()); - for to_delete in metrics { - if let Err(e) = store.delete(&mut writer, to_delete) { - log::error!("Can't delete from store: {:?}", e); - res = Err(e); - } - } - - writer.commit()?; - Ok(res?) - }) - } - - /// Removes a single metric from the storage. - /// - /// # Arguments - /// - /// * `lifetime` - the lifetime of the storage in which to look for the metric. - /// * `storage_name` - the name of the storage to store/fetch data from. - /// * `metric_id` - the metric category + name. - /// - /// # Returns - /// - /// * If the storage is unavailable an error is returned. - /// * If the metric could not be deleted, an error is returned. - /// - /// Otherwise `Ok(())` is returned. - /// - /// # Panics - /// - /// This function will **not** panic on database errors. - pub fn remove_single_metric( - &self, - lifetime: Lifetime, - storage_name: &str, - metric_id: &str, - ) -> Result<()> { - let final_key = Self::get_storage_key(storage_name, Some(metric_id)); - - // Lifetime::Ping data is not persisted to disk if - // Glean has `delay_ping_lifetime_io` set to true - if lifetime == Lifetime::Ping { - if let Some(ping_lifetime_data) = &self.ping_lifetime_data { - let mut data = ping_lifetime_data - .write() - .expect("Can't access app lifetime data as writable"); - data.remove(&final_key); - } - } - - self.write_with_store(lifetime, |mut writer, store| { - if let Err(e) = store.delete(&mut writer, final_key.clone()) { - if self.ping_lifetime_data.is_some() { - // If ping_lifetime_data exists, it might be - // that data is in memory, but not yet in rkv. - return Ok(()); - } - return Err(e.into()); - } - writer.commit()?; - Ok(()) - }) - } - - /// Clears all the metrics in the database, for the provided lifetime. - /// - /// Errors are logged. - /// - /// # Panics - /// - /// * This function will **not** panic on database errors. - pub fn clear_lifetime(&self, lifetime: Lifetime) { - let res = self.write_with_store(lifetime, |mut writer, store| { - store.clear(&mut writer)?; - writer.commit()?; - Ok(()) - }); - if let Err(e) = res { - log::error!("Could not clear store for lifetime {:?}: {:?}", lifetime, e); - } - } - - /// Clears all metrics in the database. - /// - /// Errors are logged. - /// - /// # Panics - /// - /// * This function will **not** panic on database errors. - pub fn clear_all(&self) { - if let Some(ping_lifetime_data) = &self.ping_lifetime_data { - ping_lifetime_data - .write() - .expect("Can't access ping lifetime data as writable") - .clear(); - } - - for lifetime in [Lifetime::User, Lifetime::Ping, Lifetime::Application].iter() { - self.clear_lifetime(*lifetime); - } - } - - /// Persists ping_lifetime_data to disk. - /// - /// Does nothing in case there is nothing to persist. - /// - /// # Panics - /// - /// * This function will **not** panic on database errors. - pub fn persist_ping_lifetime_data(&self) -> Result<()> { - if let Some(ping_lifetime_data) = &self.ping_lifetime_data { - let data = ping_lifetime_data - .read() - .expect("Can't read ping lifetime data"); - - self.write_with_store(Lifetime::Ping, |mut writer, store| { - for (key, value) in data.iter() { - let encoded = - bincode::serialize(&value).expect("IMPOSSIBLE: Serializing metric failed"); - // There is no need for `get_storage_key` here because - // the key is already formatted from when it was saved - // to ping_lifetime_data. - store.put(&mut writer, &key, &rkv::Value::Blob(&encoded))?; - } - writer.commit()?; - Ok(()) - })?; - } - Ok(()) - } -} - -#[cfg(test)] -mod test { - use super::*; - use crate::tests::new_glean; - use crate::CommonMetricData; - use std::collections::HashMap; - use tempfile::tempdir; - - #[test] - fn test_panicks_if_fails_dir_creation() { - assert!(Database::new("/!#\"'@#°ç", false).is_err()); - } - - #[test] - fn test_data_dir_rkv_inits() { - let dir = tempdir().unwrap(); - let str_dir = dir.path().display().to_string(); - - Database::new(&str_dir, false).unwrap(); - - assert!(dir.path().exists()); - } - - #[test] - fn test_ping_lifetime_metric_recorded() { - // Init the database in a temporary directory. - let dir = tempdir().unwrap(); - let str_dir = dir.path().display().to_string(); - let db = Database::new(&str_dir, false).unwrap(); - - assert!(db.ping_lifetime_data.is_none()); - - // Attempt to record a known value. - let test_value = "test-value"; - let test_storage = "test-storage"; - let test_metric_id = "telemetry_test.test_name"; - db.record_per_lifetime( - Lifetime::Ping, - test_storage, - test_metric_id, - &Metric::String(test_value.to_string()), - ) - .unwrap(); - - // Verify that the data is correctly recorded. - let mut found_metrics = 0; - let mut snapshotter = |metric_id: &[u8], metric: &Metric| { - found_metrics += 1; - let metric_id = String::from_utf8_lossy(metric_id).into_owned(); - assert_eq!(test_metric_id, metric_id); - match metric { - Metric::String(s) => assert_eq!(test_value, s), - _ => panic!("Unexpected data found"), - } - }; - - db.iter_store_from(Lifetime::Ping, test_storage, None, &mut snapshotter); - assert_eq!(1, found_metrics, "We only expect 1 Lifetime.Ping metric."); - } - - #[test] - fn test_application_lifetime_metric_recorded() { - // Init the database in a temporary directory. - let dir = tempdir().unwrap(); - let str_dir = dir.path().display().to_string(); - let db = Database::new(&str_dir, false).unwrap(); - - // Attempt to record a known value. - let test_value = "test-value"; - let test_storage = "test-storage1"; - let test_metric_id = "telemetry_test.test_name"; - db.record_per_lifetime( - Lifetime::Application, - test_storage, - test_metric_id, - &Metric::String(test_value.to_string()), - ) - .unwrap(); - - // Verify that the data is correctly recorded. - let mut found_metrics = 0; - let mut snapshotter = |metric_id: &[u8], metric: &Metric| { - found_metrics += 1; - let metric_id = String::from_utf8_lossy(metric_id).into_owned(); - assert_eq!(test_metric_id, metric_id); - match metric { - Metric::String(s) => assert_eq!(test_value, s), - _ => panic!("Unexpected data found"), - } - }; - - db.iter_store_from(Lifetime::Application, test_storage, None, &mut snapshotter); - assert_eq!( - 1, found_metrics, - "We only expect 1 Lifetime.Application metric." - ); - } - - #[test] - fn test_user_lifetime_metric_recorded() { - // Init the database in a temporary directory. - let dir = tempdir().unwrap(); - let str_dir = dir.path().display().to_string(); - let db = Database::new(&str_dir, false).unwrap(); - - // Attempt to record a known value. - let test_value = "test-value"; - let test_storage = "test-storage2"; - let test_metric_id = "telemetry_test.test_name"; - db.record_per_lifetime( - Lifetime::User, - test_storage, - test_metric_id, - &Metric::String(test_value.to_string()), - ) - .unwrap(); - - // Verify that the data is correctly recorded. - let mut found_metrics = 0; - let mut snapshotter = |metric_id: &[u8], metric: &Metric| { - found_metrics += 1; - let metric_id = String::from_utf8_lossy(metric_id).into_owned(); - assert_eq!(test_metric_id, metric_id); - match metric { - Metric::String(s) => assert_eq!(test_value, s), - _ => panic!("Unexpected data found"), - } - }; - - db.iter_store_from(Lifetime::User, test_storage, None, &mut snapshotter); - assert_eq!(1, found_metrics, "We only expect 1 Lifetime.User metric."); - } - - #[test] - fn test_clear_ping_storage() { - // Init the database in a temporary directory. - let dir = tempdir().unwrap(); - let str_dir = dir.path().display().to_string(); - let db = Database::new(&str_dir, false).unwrap(); - - // Attempt to record a known value for every single lifetime. - let test_storage = "test-storage"; - db.record_per_lifetime( - Lifetime::User, - test_storage, - "telemetry_test.test_name_user", - &Metric::String("test-value-user".to_string()), - ) - .unwrap(); - db.record_per_lifetime( - Lifetime::Ping, - test_storage, - "telemetry_test.test_name_ping", - &Metric::String("test-value-ping".to_string()), - ) - .unwrap(); - db.record_per_lifetime( - Lifetime::Application, - test_storage, - "telemetry_test.test_name_application", - &Metric::String("test-value-application".to_string()), - ) - .unwrap(); - - // Take a snapshot for the data, all the lifetimes. - { - let mut snapshot: HashMap = HashMap::new(); - let mut snapshotter = |metric_id: &[u8], metric: &Metric| { - let metric_id = String::from_utf8_lossy(metric_id).into_owned(); - match metric { - Metric::String(s) => snapshot.insert(metric_id, s.to_string()), - _ => panic!("Unexpected data found"), - }; - }; - - db.iter_store_from(Lifetime::User, test_storage, None, &mut snapshotter); - db.iter_store_from(Lifetime::Ping, test_storage, None, &mut snapshotter); - db.iter_store_from(Lifetime::Application, test_storage, None, &mut snapshotter); - - assert_eq!(3, snapshot.len(), "We expect all lifetimes to be present."); - assert!(snapshot.contains_key("telemetry_test.test_name_user")); - assert!(snapshot.contains_key("telemetry_test.test_name_ping")); - assert!(snapshot.contains_key("telemetry_test.test_name_application")); - } - - // Clear the Ping lifetime. - db.clear_ping_lifetime_storage(test_storage).unwrap(); - - // Take a snapshot again and check that we're only clearing the Ping lifetime. - { - let mut snapshot: HashMap = HashMap::new(); - let mut snapshotter = |metric_id: &[u8], metric: &Metric| { - let metric_id = String::from_utf8_lossy(metric_id).into_owned(); - match metric { - Metric::String(s) => snapshot.insert(metric_id, s.to_string()), - _ => panic!("Unexpected data found"), - }; - }; - - db.iter_store_from(Lifetime::User, test_storage, None, &mut snapshotter); - db.iter_store_from(Lifetime::Ping, test_storage, None, &mut snapshotter); - db.iter_store_from(Lifetime::Application, test_storage, None, &mut snapshotter); - - assert_eq!(2, snapshot.len(), "We only expect 2 metrics to be left."); - assert!(snapshot.contains_key("telemetry_test.test_name_user")); - assert!(snapshot.contains_key("telemetry_test.test_name_application")); - } - } - - #[test] - fn test_remove_single_metric() { - // Init the database in a temporary directory. - let dir = tempdir().unwrap(); - let str_dir = dir.path().display().to_string(); - let db = Database::new(&str_dir, false).unwrap(); - - let test_storage = "test-storage-single-lifetime"; - let metric_id_pattern = "telemetry_test.single_metric"; - - // Write sample metrics to the database. - let lifetimes = vec![Lifetime::User, Lifetime::Ping, Lifetime::Application]; - - for lifetime in lifetimes.iter() { - for value in &["retain", "delete"] { - db.record_per_lifetime( - *lifetime, - test_storage, - &format!("{}_{}", metric_id_pattern, value), - &Metric::String((*value).to_string()), - ) - .unwrap(); - } - } - - // Remove "telemetry_test.single_metric_delete" from each lifetime. - for lifetime in lifetimes.iter() { - db.remove_single_metric( - *lifetime, - test_storage, - &format!("{}_delete", metric_id_pattern), - ) - .unwrap(); - } - - // Verify that "telemetry_test.single_metric_retain" is still around for all lifetimes. - for lifetime in lifetimes.iter() { - let mut found_metrics = 0; - let mut snapshotter = |metric_id: &[u8], metric: &Metric| { - found_metrics += 1; - let metric_id = String::from_utf8_lossy(metric_id).into_owned(); - assert_eq!(format!("{}_retain", metric_id_pattern), metric_id); - match metric { - Metric::String(s) => assert_eq!("retain", s), - _ => panic!("Unexpected data found"), - } - }; - - // Check the User lifetime. - db.iter_store_from(*lifetime, test_storage, None, &mut snapshotter); - assert_eq!( - 1, found_metrics, - "We only expect 1 metric for this lifetime." - ); - } - } - - #[test] - fn test_delayed_ping_lifetime_persistence() { - // Init the database in a temporary directory. - let dir = tempdir().unwrap(); - let str_dir = dir.path().display().to_string(); - let db = Database::new(&str_dir, true).unwrap(); - let test_storage = "test-storage"; - - assert!(db.ping_lifetime_data.is_some()); - - // Attempt to record a known value. - let test_value1 = "test-value1"; - let test_metric_id1 = "telemetry_test.test_name1"; - db.record_per_lifetime( - Lifetime::Ping, - test_storage, - test_metric_id1, - &Metric::String(test_value1.to_string()), - ) - .unwrap(); - - // Attempt to persist data. - db.persist_ping_lifetime_data().unwrap(); - - // Attempt to record another known value. - let test_value2 = "test-value2"; - let test_metric_id2 = "telemetry_test.test_name2"; - db.record_per_lifetime( - Lifetime::Ping, - test_storage, - test_metric_id2, - &Metric::String(test_value2.to_string()), - ) - .unwrap(); - - { - // At this stage we expect `test_value1` to be persisted and in memory, - // since it was recorded before calling `persist_ping_lifetime_data`, - // and `test_value2` to be only in memory, since it was recorded after. - let store: SingleStore = db - .rkv - .open_single(Lifetime::Ping.as_str(), StoreOptions::create()) - .unwrap(); - let reader = db.rkv.read().unwrap(); - - // Verify that test_value1 is in rkv. - assert!(store - .get(&reader, format!("{}#{}", test_storage, test_metric_id1)) - .unwrap_or(None) - .is_some()); - // Verifiy that test_value2 is **not** in rkv. - assert!(store - .get(&reader, format!("{}#{}", test_storage, test_metric_id2)) - .unwrap_or(None) - .is_none()); - - let data = match &db.ping_lifetime_data { - Some(ping_lifetime_data) => ping_lifetime_data, - None => panic!("Expected `ping_lifetime_data` to exist here!"), - }; - let data = data.read().unwrap(); - // Verify that test_value1 is also in memory. - assert!(data - .get(&format!("{}#{}", test_storage, test_metric_id1)) - .is_some()); - // Verify that test_value2 is in memory. - assert!(data - .get(&format!("{}#{}", test_storage, test_metric_id2)) - .is_some()); - } - - // Attempt to persist data again. - db.persist_ping_lifetime_data().unwrap(); - - { - // At this stage we expect `test_value1` and `test_value2` to - // be persisted, since both were created before a call to - // `persist_ping_lifetime_data`. - let store: SingleStore = db - .rkv - .open_single(Lifetime::Ping.as_str(), StoreOptions::create()) - .unwrap(); - let reader = db.rkv.read().unwrap(); - - // Verify that test_value1 is in rkv. - assert!(store - .get(&reader, format!("{}#{}", test_storage, test_metric_id1)) - .unwrap_or(None) - .is_some()); - // Verifiy that test_value2 is also in rkv. - assert!(store - .get(&reader, format!("{}#{}", test_storage, test_metric_id2)) - .unwrap_or(None) - .is_some()); - - let data = match &db.ping_lifetime_data { - Some(ping_lifetime_data) => ping_lifetime_data, - None => panic!("Expected `ping_lifetime_data` to exist here!"), - }; - let data = data.read().unwrap(); - // Verify that test_value1 is also in memory. - assert!(data - .get(&format!("{}#{}", test_storage, test_metric_id1)) - .is_some()); - // Verify that test_value2 is also in memory. - assert!(data - .get(&format!("{}#{}", test_storage, test_metric_id2)) - .is_some()); - } - } - - #[test] - fn test_load_ping_lifetime_data_from_memory() { - // Init the database in a temporary directory. - let dir = tempdir().unwrap(); - let str_dir = dir.path().display().to_string(); - - let test_storage = "test-storage"; - let test_value = "test-value"; - let test_metric_id = "telemetry_test.test_name"; - - { - let db = Database::new(&str_dir, true).unwrap(); - - // Attempt to record a known value. - db.record_per_lifetime( - Lifetime::Ping, - test_storage, - test_metric_id, - &Metric::String(test_value.to_string()), - ) - .unwrap(); - - // Verify that test_value is in memory. - let data = match &db.ping_lifetime_data { - Some(ping_lifetime_data) => ping_lifetime_data, - None => panic!("Expected `ping_lifetime_data` to exist here!"), - }; - let data = data.read().unwrap(); - assert!(data - .get(&format!("{}#{}", test_storage, test_metric_id)) - .is_some()); - - // Attempt to persist data. - db.persist_ping_lifetime_data().unwrap(); - - // Verify that test_value is now in rkv. - let store: SingleStore = db - .rkv - .open_single(Lifetime::Ping.as_str(), StoreOptions::create()) - .unwrap(); - let reader = db.rkv.read().unwrap(); - assert!(store - .get(&reader, format!("{}#{}", test_storage, test_metric_id)) - .unwrap_or(None) - .is_some()); - } - - // Now create a new instace of the db and check if data was - // correctly loaded from rkv to memory. - { - let db = Database::new(&str_dir, true).unwrap(); - - // Verify that test_value is in memory. - let data = match &db.ping_lifetime_data { - Some(ping_lifetime_data) => ping_lifetime_data, - None => panic!("Expected `ping_lifetime_data` to exist here!"), - }; - let data = data.read().unwrap(); - assert!(data - .get(&format!("{}#{}", test_storage, test_metric_id)) - .is_some()); - - // Verify that test_value is also in rkv. - let store: SingleStore = db - .rkv - .open_single(Lifetime::Ping.as_str(), StoreOptions::create()) - .unwrap(); - let reader = db.rkv.read().unwrap(); - assert!(store - .get(&reader, format!("{}#{}", test_storage, test_metric_id)) - .unwrap_or(None) - .is_some()); - } - } - - #[test] - fn doesnt_record_when_upload_is_disabled() { - let (mut glean, dir) = new_glean(None); - - // Init the database in a temporary directory. - let str_dir = dir.path().display().to_string(); - - let test_storage = "test-storage"; - let test_data = CommonMetricData::new("category", "name", test_storage); - let test_metric_id = test_data.identifier(&glean); - - // Attempt to record metric with the record and record_with functions, - // this should work since upload is enabled. - let db = Database::new(&str_dir, true).unwrap(); - db.record(&glean, &test_data, &Metric::String("record".to_owned())); - db.iter_store_from( - Lifetime::Ping, - test_storage, - None, - &mut |metric_id: &[u8], metric: &Metric| { - assert_eq!( - String::from_utf8_lossy(metric_id).into_owned(), - test_metric_id - ); - match metric { - Metric::String(v) => assert_eq!("record", *v), - _ => panic!("Unexpected data found"), - } - }, - ); - - db.record_with(&glean, &test_data, |_| { - Metric::String("record_with".to_owned()) - }); - db.iter_store_from( - Lifetime::Ping, - test_storage, - None, - &mut |metric_id: &[u8], metric: &Metric| { - assert_eq!( - String::from_utf8_lossy(metric_id).into_owned(), - test_metric_id - ); - match metric { - Metric::String(v) => assert_eq!("record_with", *v), - _ => panic!("Unexpected data found"), - } - }, - ); - - // Disable upload - glean.set_upload_enabled(false); - - // Attempt to record metric with the record and record_with functions, - // this should work since upload is now **disabled**. - db.record(&glean, &test_data, &Metric::String("record_nop".to_owned())); - db.iter_store_from( - Lifetime::Ping, - test_storage, - None, - &mut |metric_id: &[u8], metric: &Metric| { - assert_eq!( - String::from_utf8_lossy(metric_id).into_owned(), - test_metric_id - ); - match metric { - Metric::String(v) => assert_eq!("record_with", *v), - _ => panic!("Unexpected data found"), - } - }, - ); - db.record_with(&glean, &test_data, |_| { - Metric::String("record_with_nop".to_owned()) - }); - db.iter_store_from( - Lifetime::Ping, - test_storage, - None, - &mut |metric_id: &[u8], metric: &Metric| { - assert_eq!( - String::from_utf8_lossy(metric_id).into_owned(), - test_metric_id - ); - match metric { - Metric::String(v) => assert_eq!("record_with", *v), - _ => panic!("Unexpected data found"), - } - }, - ); - } - - #[cfg(feature = "rkv-safe-mode")] - mod safe_mode_migration { - use super::*; - use rkv::Value; - - #[test] - fn migration_works_on_startup() { - let dir = tempdir().unwrap(); - let str_dir = dir.path().display().to_string(); - - let database_dir = dir.path().join("db"); - let datamdb = database_dir.join("data.mdb"); - let lockmdb = database_dir.join("lock.mdb"); - let safebin = database_dir.join("data.safe.bin"); - - assert!(!safebin.exists()); - assert!(!datamdb.exists()); - assert!(!lockmdb.exists()); - - let store_name = "store1"; - let metric_name = "bool"; - let key = Database::get_storage_key(store_name, Some(metric_name)); - - // Ensure some old data in the LMDB format exists. - { - fs::create_dir_all(&database_dir).expect("create dir"); - let rkv_db = rkv::Rkv::new::(&database_dir).expect("rkv env"); - - let store = rkv_db - .open_single("ping", StoreOptions::create()) - .expect("opened"); - let mut writer = rkv_db.write().expect("writer"); - let metric = Metric::Boolean(true); - let value = bincode::serialize(&metric).expect("serialized"); - store - .put(&mut writer, &key, &Value::Blob(&value)) - .expect("wrote"); - writer.commit().expect("committed"); - - assert!(datamdb.exists()); - assert!(lockmdb.exists()); - assert!(!safebin.exists()); - } - - // First open should migrate the data. - { - let db = Database::new(&str_dir, false).unwrap(); - let safebin = database_dir.join("data.safe.bin"); - assert!(safebin.exists(), "safe-mode file should exist"); - assert!(!datamdb.exists(), "LMDB data should be deleted"); - assert!(!lockmdb.exists(), "LMDB lock should be deleted"); - - let mut stored_metrics = vec![]; - let mut snapshotter = |name: &[u8], metric: &Metric| { - let name = str::from_utf8(name).unwrap().to_string(); - stored_metrics.push((name, metric.clone())) - }; - db.iter_store_from(Lifetime::Ping, "store1", None, &mut snapshotter); - - assert_eq!(1, stored_metrics.len()); - assert_eq!(metric_name, stored_metrics[0].0); - assert_eq!(&Metric::Boolean(true), &stored_metrics[0].1); - } - - // Next open should not re-create the LMDB files. - { - let db = Database::new(&str_dir, false).unwrap(); - let safebin = database_dir.join("data.safe.bin"); - assert!(safebin.exists(), "safe-mode file exists"); - assert!(!datamdb.exists(), "LMDB data should not be recreated"); - assert!(!lockmdb.exists(), "LMDB lock should not be recreated"); - - let mut stored_metrics = vec![]; - let mut snapshotter = |name: &[u8], metric: &Metric| { - let name = str::from_utf8(name).unwrap().to_string(); - stored_metrics.push((name, metric.clone())) - }; - db.iter_store_from(Lifetime::Ping, "store1", None, &mut snapshotter); - - assert_eq!(1, stored_metrics.len()); - assert_eq!(metric_name, stored_metrics[0].0); - assert_eq!(&Metric::Boolean(true), &stored_metrics[0].1); - } - } - - #[test] - fn migration_doesnt_overwrite() { - let dir = tempdir().unwrap(); - let str_dir = dir.path().display().to_string(); - - let database_dir = dir.path().join("db"); - let datamdb = database_dir.join("data.mdb"); - let lockmdb = database_dir.join("lock.mdb"); - let safebin = database_dir.join("data.safe.bin"); - - assert!(!safebin.exists()); - assert!(!datamdb.exists()); - assert!(!lockmdb.exists()); - - let store_name = "store1"; - let metric_name = "counter"; - let key = Database::get_storage_key(store_name, Some(metric_name)); - - // Ensure some old data in the LMDB format exists. - { - fs::create_dir_all(&database_dir).expect("create dir"); - let rkv_db = rkv::Rkv::new::(&database_dir).expect("rkv env"); - - let store = rkv_db - .open_single("ping", StoreOptions::create()) - .expect("opened"); - let mut writer = rkv_db.write().expect("writer"); - let metric = Metric::Counter(734); // this value will be ignored - let value = bincode::serialize(&metric).expect("serialized"); - store - .put(&mut writer, &key, &Value::Blob(&value)) - .expect("wrote"); - writer.commit().expect("committed"); - - assert!(datamdb.exists()); - assert!(lockmdb.exists()); - } - - // Ensure some data exists in the new database. - { - fs::create_dir_all(&database_dir).expect("create dir"); - let rkv_db = - rkv::Rkv::new::(&database_dir).expect("rkv env"); - - let store = rkv_db - .open_single("ping", StoreOptions::create()) - .expect("opened"); - let mut writer = rkv_db.write().expect("writer"); - let metric = Metric::Counter(2); - let value = bincode::serialize(&metric).expect("serialized"); - store - .put(&mut writer, &key, &Value::Blob(&value)) - .expect("wrote"); - writer.commit().expect("committed"); - - assert!(safebin.exists()); - } - - // First open should try migration and ignore it, because destination is not empty. - // It also deletes the leftover LMDB database. - { - let db = Database::new(&str_dir, false).unwrap(); - let safebin = database_dir.join("data.safe.bin"); - assert!(safebin.exists(), "safe-mode file should exist"); - assert!(!datamdb.exists(), "LMDB data should be deleted"); - assert!(!lockmdb.exists(), "LMDB lock should be deleted"); - - let mut stored_metrics = vec![]; - let mut snapshotter = |name: &[u8], metric: &Metric| { - let name = str::from_utf8(name).unwrap().to_string(); - stored_metrics.push((name, metric.clone())) - }; - db.iter_store_from(Lifetime::Ping, "store1", None, &mut snapshotter); - - assert_eq!(1, stored_metrics.len()); - assert_eq!(metric_name, stored_metrics[0].0); - assert_eq!(&Metric::Counter(2), &stored_metrics[0].1); - } - } - - #[test] - fn migration_ignores_broken_database() { - let dir = tempdir().unwrap(); - let str_dir = dir.path().display().to_string(); - - let database_dir = dir.path().join("db"); - let datamdb = database_dir.join("data.mdb"); - let lockmdb = database_dir.join("lock.mdb"); - let safebin = database_dir.join("data.safe.bin"); - - assert!(!safebin.exists()); - assert!(!datamdb.exists()); - assert!(!lockmdb.exists()); - - let store_name = "store1"; - let metric_name = "counter"; - let key = Database::get_storage_key(store_name, Some(metric_name)); - - // Ensure some old data in the LMDB format exists. - { - fs::create_dir_all(&database_dir).expect("create dir"); - fs::write(&datamdb, "bogus").expect("dbfile created"); - - assert!(datamdb.exists()); - } - - // Ensure some data exists in the new database. - { - fs::create_dir_all(&database_dir).expect("create dir"); - let rkv_db = - rkv::Rkv::new::(&database_dir).expect("rkv env"); - - let store = rkv_db - .open_single("ping", StoreOptions::create()) - .expect("opened"); - let mut writer = rkv_db.write().expect("writer"); - let metric = Metric::Counter(2); - let value = bincode::serialize(&metric).expect("serialized"); - store - .put(&mut writer, &key, &Value::Blob(&value)) - .expect("wrote"); - writer.commit().expect("committed"); - } - - // First open should try migration and ignore it, because destination is not empty. - // It also deletes the leftover LMDB database. - { - let db = Database::new(&str_dir, false).unwrap(); - let safebin = database_dir.join("data.safe.bin"); - assert!(safebin.exists(), "safe-mode file should exist"); - assert!(!datamdb.exists(), "LMDB data should be deleted"); - assert!(!lockmdb.exists(), "LMDB lock should be deleted"); - - let mut stored_metrics = vec![]; - let mut snapshotter = |name: &[u8], metric: &Metric| { - let name = str::from_utf8(name).unwrap().to_string(); - stored_metrics.push((name, metric.clone())) - }; - db.iter_store_from(Lifetime::Ping, "store1", None, &mut snapshotter); - - assert_eq!(1, stored_metrics.len()); - assert_eq!(metric_name, stored_metrics[0].0); - assert_eq!(&Metric::Counter(2), &stored_metrics[0].1); - } - } - - #[test] - fn migration_ignores_empty_database() { - let dir = tempdir().unwrap(); - let str_dir = dir.path().display().to_string(); - - let database_dir = dir.path().join("db"); - let datamdb = database_dir.join("data.mdb"); - let lockmdb = database_dir.join("lock.mdb"); - let safebin = database_dir.join("data.safe.bin"); - - assert!(!safebin.exists()); - assert!(!datamdb.exists()); - assert!(!lockmdb.exists()); - - // Ensure old LMDB database exists, but is empty. - { - fs::create_dir_all(&database_dir).expect("create dir"); - let rkv_db = rkv::Rkv::new::(&database_dir).expect("rkv env"); - drop(rkv_db); - assert!(datamdb.exists()); - assert!(lockmdb.exists()); - } - - // First open should try migration, but find no data. - // safe-mode does not write an empty database to disk. - // It also deletes the leftover LMDB database. - { - let _db = Database::new(&str_dir, false).unwrap(); - let safebin = database_dir.join("data.safe.bin"); - assert!(!safebin.exists(), "safe-mode file should exist"); - assert!(!datamdb.exists(), "LMDB data should be deleted"); - assert!(!lockmdb.exists(), "LMDB lock should be deleted"); - } - } - } -} +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::collections::btree_map::Entry; +use std::collections::BTreeMap; +use std::fs; +use std::num::NonZeroU64; +use std::path::Path; +use std::str; +use std::sync::RwLock; + +use rkv::StoreOptions; + +// Select the LMDB-powered storage backend when the feature is not activated. +#[cfg(not(feature = "rkv-safe-mode"))] +mod backend { + use std::path::Path; + + /// cbindgen:ignore + pub type Rkv = rkv::Rkv; + /// cbindgen:ignore + pub type SingleStore = rkv::SingleStore; + /// cbindgen:ignore + pub type Writer<'t> = rkv::Writer>; + + pub fn rkv_new(path: &Path) -> Result { + Rkv::new::(path) + } + + /// No migration necessary when staying with LMDB. + pub fn migrate(_path: &Path, _dst_env: &Rkv) { + // Intentionally left empty. + } +} + +// Select the "safe mode" storage backend when the feature is activated. +#[cfg(feature = "rkv-safe-mode")] +mod backend { + use rkv::migrator::Migrator; + use std::{fs, path::Path}; + + /// cbindgen:ignore + pub type Rkv = rkv::Rkv; + /// cbindgen:ignore + pub type SingleStore = rkv::SingleStore; + /// cbindgen:ignore + pub type Writer<'t> = rkv::Writer>; + + pub fn rkv_new(path: &Path) -> Result { + Rkv::new::(path) + } + + fn delete_and_log(path: &Path, msg: &str) { + if let Err(err) = fs::remove_file(path) { + match err.kind() { + std::io::ErrorKind::NotFound => { + // Silently drop this error, the file was already non-existing. + } + _ => log::warn!("{}", msg), + } + } + } + + fn delete_lmdb_database(path: &Path) { + let datamdb = path.join("data.mdb"); + delete_and_log(&datamdb, "Failed to delete old data."); + + let lockmdb = path.join("lock.mdb"); + delete_and_log(&lockmdb, "Failed to delete old lock."); + } + + /// Migrate from LMDB storage to safe-mode storage. + /// + /// This migrates the data once, then deletes the LMDB storage. + /// The safe-mode storage must be empty for it to work. + /// Existing data will not be overwritten. + /// If the destination database is not empty the LMDB database is deleted + /// without migrating data. + /// This is a no-op if no LMDB database file exists. + pub fn migrate(path: &Path, dst_env: &Rkv) { + use rkv::{MigrateError, StoreError}; + + log::debug!("Migrating files in {}", path.display()); + + // Shortcut if no data to migrate is around. + let datamdb = path.join("data.mdb"); + if !datamdb.exists() { + log::debug!("No data to migrate."); + return; + } + + // We're handling the same error cases as `easy_migrate_lmdb_to_safe_mode`, + // but annotate each why they don't cause problems for Glean. + // Additionally for known cases we delete the LMDB database regardless. + let should_delete = + match Migrator::open_and_migrate_lmdb_to_safe_mode(path, |builder| builder, dst_env) { + // Source environment is corrupted. + // We start fresh with the new database. + Err(MigrateError::StoreError(StoreError::FileInvalid)) => true, + // Path not accessible. + // Somehow our directory vanished between us creating it and reading from it. + // Nothing we can do really. + Err(MigrateError::StoreError(StoreError::IoError(_))) => true, + // Path accessible but incompatible for configuration. + // This should not happen, we never used storages that safe-mode doesn't understand. + // If it does happen, let's start fresh and use the safe-mode from now on. + Err(MigrateError::StoreError(StoreError::UnsuitableEnvironmentPath(_))) => true, + // Nothing to migrate. + // Source database was empty. We just start fresh anyway. + Err(MigrateError::SourceEmpty) => true, + // Migrating would overwrite. + // Either a previous migration failed and we still started writing data, + // or someone placed back an old data file. + // In any case we better stay on the new data and delete the old one. + Err(MigrateError::DestinationNotEmpty) => { + log::warn!("Failed to migrate old data. Destination was not empty"); + true + } + // An internal lock was poisoned. + // This would only happen if multiple things run concurrently and one crashes. + Err(MigrateError::ManagerPoisonError) => false, + // Other store errors are never returned from the migrator. + // We need to handle them to please rustc. + Err(MigrateError::StoreError(_)) => false, + // Other errors can't happen, so this leaves us with the Ok case. + // This already deleted the LMDB files. + Ok(()) => false, + }; + + if should_delete { + log::debug!("Need to delete remaining LMDB files."); + delete_lmdb_database(&path); + } + + log::debug!("Migration ended. Safe-mode database in {}", path.display()); + } +} + +use crate::metrics::Metric; +use crate::CommonMetricData; +use crate::Glean; +use crate::Lifetime; +use crate::Result; +use backend::*; + +pub struct Database { + /// Handle to the database environment. + rkv: Rkv, + + /// Handles to the "lifetime" stores. + /// + /// A "store" is a handle to the underlying database. + /// We keep them open for fast and frequent access. + user_store: SingleStore, + ping_store: SingleStore, + application_store: SingleStore, + + /// If the `delay_ping_lifetime_io` Glean config option is `true`, + /// we will save metrics with 'ping' lifetime data in a map temporarily + /// so as to persist them to disk using rkv in bulk on demand. + ping_lifetime_data: Option>>, + + // Initial file size when opening the database. + file_size: Option, +} + +impl std::fmt::Debug for Database { + fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result { + fmt.debug_struct("Database") + .field("rkv", &self.rkv) + .field("user_store", &"SingleStore") + .field("ping_store", &"SingleStore") + .field("application_store", &"SingleStore") + .field("ping_lifetime_data", &self.ping_lifetime_data) + .finish() + } +} + +/// Calculate the database size from all the files in the directory. +/// +/// # Arguments +/// +/// *`path` - The path to the directory +/// +/// # Returns +/// +/// Returns the non-zero combined size of all files in a directory, +/// or `None` on error or if the size is `0`. +fn database_size(dir: &Path) -> Option { + let mut total_size = 0; + if let Ok(entries) = fs::read_dir(dir) { + for entry in entries { + if let Ok(entry) = entry { + if let Ok(file_type) = entry.file_type() { + if file_type.is_file() { + let path = entry.path(); + if let Ok(metadata) = fs::metadata(path) { + total_size += metadata.len(); + } else { + continue; + } + } + } + } + } + } + + NonZeroU64::new(total_size) +} + +impl Database { + /// Initializes the data store. + /// + /// This opens the underlying rkv store and creates + /// the underlying directory structure. + /// + /// It also loads any Lifetime::Ping data that might be + /// persisted, in case `delay_ping_lifetime_io` is set. + pub fn new(data_path: &str, delay_ping_lifetime_io: bool) -> Result { + let path = Path::new(data_path).join("db"); + log::debug!("Database path: {:?}", path.display()); + let file_size = database_size(&path); + + let rkv = Self::open_rkv(&path)?; + let user_store = rkv.open_single(Lifetime::User.as_str(), StoreOptions::create())?; + let ping_store = rkv.open_single(Lifetime::Ping.as_str(), StoreOptions::create())?; + let application_store = + rkv.open_single(Lifetime::Application.as_str(), StoreOptions::create())?; + let ping_lifetime_data = if delay_ping_lifetime_io { + Some(RwLock::new(BTreeMap::new())) + } else { + None + }; + + let db = Self { + rkv, + user_store, + ping_store, + application_store, + ping_lifetime_data, + file_size, + }; + + db.load_ping_lifetime_data(); + + Ok(db) + } + + /// Get the initial database file size. + pub fn file_size(&self) -> Option { + self.file_size + } + + fn get_store(&self, lifetime: Lifetime) -> &SingleStore { + match lifetime { + Lifetime::User => &self.user_store, + Lifetime::Ping => &self.ping_store, + Lifetime::Application => &self.application_store, + } + } + + /// Creates the storage directories and inits rkv. + fn open_rkv(path: &Path) -> Result { + fs::create_dir_all(&path)?; + + let rkv = rkv_new(&path)?; + migrate(path, &rkv); + + log::info!("Database initialized"); + Ok(rkv) + } + + /// Build the key of the final location of the data in the database. + /// Such location is built using the storage name and the metric + /// key/name (if available). + /// + /// # Arguments + /// + /// * `storage_name` - the name of the storage to store/fetch data from. + /// * `metric_key` - the optional metric key/name. + /// + /// # Returns + /// + /// A string representing the location in the database. + fn get_storage_key(storage_name: &str, metric_key: Option<&str>) -> String { + match metric_key { + Some(k) => format!("{}#{}", storage_name, k), + None => format!("{}#", storage_name), + } + } + + /// Loads Lifetime::Ping data from rkv to memory, + /// if `delay_ping_lifetime_io` is set to true. + /// + /// Does nothing if it isn't or if there is not data to load. + fn load_ping_lifetime_data(&self) { + if let Some(ping_lifetime_data) = &self.ping_lifetime_data { + let mut data = ping_lifetime_data + .write() + .expect("Can't read ping lifetime data"); + + let reader = unwrap_or!(self.rkv.read(), return); + let store = self.get_store(Lifetime::Ping); + let mut iter = unwrap_or!(store.iter_start(&reader), return); + + while let Some(Ok((metric_id, value))) = iter.next() { + let metric_id = match str::from_utf8(metric_id) { + Ok(metric_id) => metric_id.to_string(), + _ => continue, + }; + let metric: Metric = match value { + rkv::Value::Blob(blob) => unwrap_or!(bincode::deserialize(blob), continue), + _ => continue, + }; + + data.insert(metric_id, metric); + } + } + } + + /// Iterates with the provided transaction function + /// over the requested data from the given storage. + /// + /// * If the storage is unavailable, the transaction function is never invoked. + /// * If the read data cannot be deserialized it will be silently skipped. + /// + /// # Arguments + /// + /// * `lifetime` - The metric lifetime to iterate over. + /// * `storage_name` - The storage name to iterate over. + /// * `metric_key` - The metric key to iterate over. All metrics iterated over + /// will have this prefix. For example, if `metric_key` is of the form `{category}.`, + /// it will iterate over all metrics in the given category. If the `metric_key` is of the + /// form `{category}.{name}/`, the iterator will iterate over all specific metrics for + /// a given labeled metric. If not provided, the entire storage for the given lifetime + /// will be iterated over. + /// * `transaction_fn` - Called for each entry being iterated over. It is + /// passed two arguments: `(metric_id: &[u8], metric: &Metric)`. + /// + /// # Panics + /// + /// This function will **not** panic on database errors. + pub fn iter_store_from( + &self, + lifetime: Lifetime, + storage_name: &str, + metric_key: Option<&str>, + mut transaction_fn: F, + ) where + F: FnMut(&[u8], &Metric), + { + let iter_start = Self::get_storage_key(storage_name, metric_key); + let len = iter_start.len(); + + // Lifetime::Ping data is not immediately persisted to disk if + // Glean has `delay_ping_lifetime_io` set to true + if lifetime == Lifetime::Ping { + if let Some(ping_lifetime_data) = &self.ping_lifetime_data { + let data = ping_lifetime_data + .read() + .expect("Can't read ping lifetime data"); + for (key, value) in data.iter() { + if key.starts_with(&iter_start) { + let key = &key[len..]; + transaction_fn(key.as_bytes(), value); + } + } + return; + } + } + + let reader = unwrap_or!(self.rkv.read(), return); + let mut iter = unwrap_or!( + self.get_store(lifetime).iter_from(&reader, &iter_start), + return + ); + + while let Some(Ok((metric_id, value))) = iter.next() { + if !metric_id.starts_with(iter_start.as_bytes()) { + break; + } + + let metric_id = &metric_id[len..]; + let metric: Metric = match value { + rkv::Value::Blob(blob) => unwrap_or!(bincode::deserialize(blob), continue), + _ => continue, + }; + transaction_fn(metric_id, &metric); + } + } + + /// Determines if the storage has the given metric. + /// + /// If data cannot be read it is assumed that the storage does not have the metric. + /// + /// # Arguments + /// + /// * `lifetime` - The lifetime of the metric. + /// * `storage_name` - The storage name to look in. + /// * `metric_identifier` - The metric identifier. + /// + /// # Panics + /// + /// This function will **not** panic on database errors. + pub fn has_metric( + &self, + lifetime: Lifetime, + storage_name: &str, + metric_identifier: &str, + ) -> bool { + let key = Self::get_storage_key(storage_name, Some(metric_identifier)); + + // Lifetime::Ping data is not persisted to disk if + // Glean has `delay_ping_lifetime_io` set to true + if lifetime == Lifetime::Ping { + if let Some(ping_lifetime_data) = &self.ping_lifetime_data { + return ping_lifetime_data + .read() + .map(|data| data.contains_key(&key)) + .unwrap_or(false); + } + } + + let reader = unwrap_or!(self.rkv.read(), return false); + self.get_store(lifetime) + .get(&reader, &key) + .unwrap_or(None) + .is_some() + } + + /// Writes to the specified storage with the provided transaction function. + /// + /// If the storage is unavailable, it will return an error. + /// + /// # Panics + /// + /// * This function will **not** panic on database errors. + fn write_with_store(&self, store_name: Lifetime, mut transaction_fn: F) -> Result<()> + where + F: FnMut(Writer, &SingleStore) -> Result<()>, + { + let writer = self.rkv.write().unwrap(); + let store = self.get_store(store_name); + transaction_fn(writer, store) + } + + /// Records a metric in the underlying storage system. + pub fn record(&self, glean: &Glean, data: &CommonMetricData, value: &Metric) { + // If upload is disabled we don't want to record. + if !glean.is_upload_enabled() { + return; + } + + let name = data.identifier(glean); + + for ping_name in data.storage_names() { + if let Err(e) = self.record_per_lifetime(data.lifetime, ping_name, &name, value) { + log::error!("Failed to record metric into {}: {:?}", ping_name, e); + } + } + } + + /// Records a metric in the underlying storage system, for a single lifetime. + /// + /// # Returns + /// + /// If the storage is unavailable or the write fails, no data will be stored and an error will be returned. + /// + /// Otherwise `Ok(())` is returned. + /// + /// # Panics + /// + /// This function will **not** panic on database errors. + fn record_per_lifetime( + &self, + lifetime: Lifetime, + storage_name: &str, + key: &str, + metric: &Metric, + ) -> Result<()> { + let final_key = Self::get_storage_key(storage_name, Some(key)); + + // Lifetime::Ping data is not immediately persisted to disk if + // Glean has `delay_ping_lifetime_io` set to true + if lifetime == Lifetime::Ping { + if let Some(ping_lifetime_data) = &self.ping_lifetime_data { + let mut data = ping_lifetime_data + .write() + .expect("Can't read ping lifetime data"); + data.insert(final_key, metric.clone()); + return Ok(()); + } + } + + let encoded = bincode::serialize(&metric).expect("IMPOSSIBLE: Serializing metric failed"); + let value = rkv::Value::Blob(&encoded); + + let mut writer = self.rkv.write()?; + self.get_store(lifetime) + .put(&mut writer, final_key, &value)?; + writer.commit()?; + Ok(()) + } + + /// Records the provided value, with the given lifetime, + /// after applying a transformation function. + pub fn record_with(&self, glean: &Glean, data: &CommonMetricData, mut transform: F) + where + F: FnMut(Option) -> Metric, + { + // If upload is disabled we don't want to record. + if !glean.is_upload_enabled() { + return; + } + + let name = data.identifier(glean); + for ping_name in data.storage_names() { + if let Err(e) = + self.record_per_lifetime_with(data.lifetime, ping_name, &name, &mut transform) + { + log::error!("Failed to record metric into {}: {:?}", ping_name, e); + } + } + } + + /// Records a metric in the underlying storage system, + /// after applying the given transformation function, for a single lifetime. + /// + /// # Returns + /// + /// If the storage is unavailable or the write fails, no data will be stored and an error will be returned. + /// + /// Otherwise `Ok(())` is returned. + /// + /// # Panics + /// + /// This function will **not** panic on database errors. + fn record_per_lifetime_with( + &self, + lifetime: Lifetime, + storage_name: &str, + key: &str, + mut transform: F, + ) -> Result<()> + where + F: FnMut(Option) -> Metric, + { + let final_key = Self::get_storage_key(storage_name, Some(key)); + + // Lifetime::Ping data is not persisted to disk if + // Glean has `delay_ping_lifetime_io` set to true + if lifetime == Lifetime::Ping { + if let Some(ping_lifetime_data) = &self.ping_lifetime_data { + let mut data = ping_lifetime_data + .write() + .expect("Can't access ping lifetime data as writable"); + let entry = data.entry(final_key); + match entry { + Entry::Vacant(entry) => { + entry.insert(transform(None)); + } + Entry::Occupied(mut entry) => { + let old_value = entry.get().clone(); + entry.insert(transform(Some(old_value))); + } + } + return Ok(()); + } + } + + let mut writer = self.rkv.write()?; + let store = self.get_store(lifetime); + let new_value: Metric = { + let old_value = store.get(&writer, &final_key)?; + + match old_value { + Some(rkv::Value::Blob(blob)) => { + let old_value = bincode::deserialize(blob).ok(); + transform(old_value) + } + _ => transform(None), + } + }; + + let encoded = + bincode::serialize(&new_value).expect("IMPOSSIBLE: Serializing metric failed"); + let value = rkv::Value::Blob(&encoded); + store.put(&mut writer, final_key, &value)?; + writer.commit()?; + Ok(()) + } + + /// Clears a storage (only Ping Lifetime). + /// + /// # Returns + /// + /// * If the storage is unavailable an error is returned. + /// * If any individual delete fails, an error is returned, but other deletions might have + /// happened. + /// + /// Otherwise `Ok(())` is returned. + /// + /// # Panics + /// + /// This function will **not** panic on database errors. + pub fn clear_ping_lifetime_storage(&self, storage_name: &str) -> Result<()> { + // Lifetime::Ping data will be saved to `ping_lifetime_data` + // in case `delay_ping_lifetime_io` is set to true + if let Some(ping_lifetime_data) = &self.ping_lifetime_data { + ping_lifetime_data + .write() + .expect("Can't access ping lifetime data as writable") + .clear(); + } + + self.write_with_store(Lifetime::Ping, |mut writer, store| { + let mut metrics = Vec::new(); + { + let mut iter = store.iter_from(&writer, &storage_name)?; + while let Some(Ok((metric_id, _))) = iter.next() { + if let Ok(metric_id) = std::str::from_utf8(metric_id) { + if !metric_id.starts_with(&storage_name) { + break; + } + metrics.push(metric_id.to_owned()); + } + } + } + + let mut res = Ok(()); + for to_delete in metrics { + if let Err(e) = store.delete(&mut writer, to_delete) { + log::error!("Can't delete from store: {:?}", e); + res = Err(e); + } + } + + writer.commit()?; + Ok(res?) + }) + } + + /// Removes a single metric from the storage. + /// + /// # Arguments + /// + /// * `lifetime` - the lifetime of the storage in which to look for the metric. + /// * `storage_name` - the name of the storage to store/fetch data from. + /// * `metric_id` - the metric category + name. + /// + /// # Returns + /// + /// * If the storage is unavailable an error is returned. + /// * If the metric could not be deleted, an error is returned. + /// + /// Otherwise `Ok(())` is returned. + /// + /// # Panics + /// + /// This function will **not** panic on database errors. + pub fn remove_single_metric( + &self, + lifetime: Lifetime, + storage_name: &str, + metric_id: &str, + ) -> Result<()> { + let final_key = Self::get_storage_key(storage_name, Some(metric_id)); + + // Lifetime::Ping data is not persisted to disk if + // Glean has `delay_ping_lifetime_io` set to true + if lifetime == Lifetime::Ping { + if let Some(ping_lifetime_data) = &self.ping_lifetime_data { + let mut data = ping_lifetime_data + .write() + .expect("Can't access app lifetime data as writable"); + data.remove(&final_key); + } + } + + self.write_with_store(lifetime, |mut writer, store| { + if let Err(e) = store.delete(&mut writer, final_key.clone()) { + if self.ping_lifetime_data.is_some() { + // If ping_lifetime_data exists, it might be + // that data is in memory, but not yet in rkv. + return Ok(()); + } + return Err(e.into()); + } + writer.commit()?; + Ok(()) + }) + } + + /// Clears all the metrics in the database, for the provided lifetime. + /// + /// Errors are logged. + /// + /// # Panics + /// + /// * This function will **not** panic on database errors. + pub fn clear_lifetime(&self, lifetime: Lifetime) { + let res = self.write_with_store(lifetime, |mut writer, store| { + store.clear(&mut writer)?; + writer.commit()?; + Ok(()) + }); + if let Err(e) = res { + log::error!("Could not clear store for lifetime {:?}: {:?}", lifetime, e); + } + } + + /// Clears all metrics in the database. + /// + /// Errors are logged. + /// + /// # Panics + /// + /// * This function will **not** panic on database errors. + pub fn clear_all(&self) { + if let Some(ping_lifetime_data) = &self.ping_lifetime_data { + ping_lifetime_data + .write() + .expect("Can't access ping lifetime data as writable") + .clear(); + } + + for lifetime in [Lifetime::User, Lifetime::Ping, Lifetime::Application].iter() { + self.clear_lifetime(*lifetime); + } + } + + /// Persists ping_lifetime_data to disk. + /// + /// Does nothing in case there is nothing to persist. + /// + /// # Panics + /// + /// * This function will **not** panic on database errors. + pub fn persist_ping_lifetime_data(&self) -> Result<()> { + if let Some(ping_lifetime_data) = &self.ping_lifetime_data { + let data = ping_lifetime_data + .read() + .expect("Can't read ping lifetime data"); + + self.write_with_store(Lifetime::Ping, |mut writer, store| { + for (key, value) in data.iter() { + let encoded = + bincode::serialize(&value).expect("IMPOSSIBLE: Serializing metric failed"); + // There is no need for `get_storage_key` here because + // the key is already formatted from when it was saved + // to ping_lifetime_data. + store.put(&mut writer, &key, &rkv::Value::Blob(&encoded))?; + } + writer.commit()?; + Ok(()) + })?; + } + Ok(()) + } +} + +#[cfg(test)] +mod test { + use super::*; + use crate::tests::new_glean; + use crate::CommonMetricData; + use std::collections::HashMap; + use tempfile::tempdir; + + #[test] + fn test_panicks_if_fails_dir_creation() { + assert!(Database::new("/!#\"'@#°ç", false).is_err()); + } + + #[test] + fn test_data_dir_rkv_inits() { + let dir = tempdir().unwrap(); + let str_dir = dir.path().display().to_string(); + + Database::new(&str_dir, false).unwrap(); + + assert!(dir.path().exists()); + } + + #[test] + fn test_ping_lifetime_metric_recorded() { + // Init the database in a temporary directory. + let dir = tempdir().unwrap(); + let str_dir = dir.path().display().to_string(); + let db = Database::new(&str_dir, false).unwrap(); + + assert!(db.ping_lifetime_data.is_none()); + + // Attempt to record a known value. + let test_value = "test-value"; + let test_storage = "test-storage"; + let test_metric_id = "telemetry_test.test_name"; + db.record_per_lifetime( + Lifetime::Ping, + test_storage, + test_metric_id, + &Metric::String(test_value.to_string()), + ) + .unwrap(); + + // Verify that the data is correctly recorded. + let mut found_metrics = 0; + let mut snapshotter = |metric_id: &[u8], metric: &Metric| { + found_metrics += 1; + let metric_id = String::from_utf8_lossy(metric_id).into_owned(); + assert_eq!(test_metric_id, metric_id); + match metric { + Metric::String(s) => assert_eq!(test_value, s), + _ => panic!("Unexpected data found"), + } + }; + + db.iter_store_from(Lifetime::Ping, test_storage, None, &mut snapshotter); + assert_eq!(1, found_metrics, "We only expect 1 Lifetime.Ping metric."); + } + + #[test] + fn test_application_lifetime_metric_recorded() { + // Init the database in a temporary directory. + let dir = tempdir().unwrap(); + let str_dir = dir.path().display().to_string(); + let db = Database::new(&str_dir, false).unwrap(); + + // Attempt to record a known value. + let test_value = "test-value"; + let test_storage = "test-storage1"; + let test_metric_id = "telemetry_test.test_name"; + db.record_per_lifetime( + Lifetime::Application, + test_storage, + test_metric_id, + &Metric::String(test_value.to_string()), + ) + .unwrap(); + + // Verify that the data is correctly recorded. + let mut found_metrics = 0; + let mut snapshotter = |metric_id: &[u8], metric: &Metric| { + found_metrics += 1; + let metric_id = String::from_utf8_lossy(metric_id).into_owned(); + assert_eq!(test_metric_id, metric_id); + match metric { + Metric::String(s) => assert_eq!(test_value, s), + _ => panic!("Unexpected data found"), + } + }; + + db.iter_store_from(Lifetime::Application, test_storage, None, &mut snapshotter); + assert_eq!( + 1, found_metrics, + "We only expect 1 Lifetime.Application metric." + ); + } + + #[test] + fn test_user_lifetime_metric_recorded() { + // Init the database in a temporary directory. + let dir = tempdir().unwrap(); + let str_dir = dir.path().display().to_string(); + let db = Database::new(&str_dir, false).unwrap(); + + // Attempt to record a known value. + let test_value = "test-value"; + let test_storage = "test-storage2"; + let test_metric_id = "telemetry_test.test_name"; + db.record_per_lifetime( + Lifetime::User, + test_storage, + test_metric_id, + &Metric::String(test_value.to_string()), + ) + .unwrap(); + + // Verify that the data is correctly recorded. + let mut found_metrics = 0; + let mut snapshotter = |metric_id: &[u8], metric: &Metric| { + found_metrics += 1; + let metric_id = String::from_utf8_lossy(metric_id).into_owned(); + assert_eq!(test_metric_id, metric_id); + match metric { + Metric::String(s) => assert_eq!(test_value, s), + _ => panic!("Unexpected data found"), + } + }; + + db.iter_store_from(Lifetime::User, test_storage, None, &mut snapshotter); + assert_eq!(1, found_metrics, "We only expect 1 Lifetime.User metric."); + } + + #[test] + fn test_clear_ping_storage() { + // Init the database in a temporary directory. + let dir = tempdir().unwrap(); + let str_dir = dir.path().display().to_string(); + let db = Database::new(&str_dir, false).unwrap(); + + // Attempt to record a known value for every single lifetime. + let test_storage = "test-storage"; + db.record_per_lifetime( + Lifetime::User, + test_storage, + "telemetry_test.test_name_user", + &Metric::String("test-value-user".to_string()), + ) + .unwrap(); + db.record_per_lifetime( + Lifetime::Ping, + test_storage, + "telemetry_test.test_name_ping", + &Metric::String("test-value-ping".to_string()), + ) + .unwrap(); + db.record_per_lifetime( + Lifetime::Application, + test_storage, + "telemetry_test.test_name_application", + &Metric::String("test-value-application".to_string()), + ) + .unwrap(); + + // Take a snapshot for the data, all the lifetimes. + { + let mut snapshot: HashMap = HashMap::new(); + let mut snapshotter = |metric_id: &[u8], metric: &Metric| { + let metric_id = String::from_utf8_lossy(metric_id).into_owned(); + match metric { + Metric::String(s) => snapshot.insert(metric_id, s.to_string()), + _ => panic!("Unexpected data found"), + }; + }; + + db.iter_store_from(Lifetime::User, test_storage, None, &mut snapshotter); + db.iter_store_from(Lifetime::Ping, test_storage, None, &mut snapshotter); + db.iter_store_from(Lifetime::Application, test_storage, None, &mut snapshotter); + + assert_eq!(3, snapshot.len(), "We expect all lifetimes to be present."); + assert!(snapshot.contains_key("telemetry_test.test_name_user")); + assert!(snapshot.contains_key("telemetry_test.test_name_ping")); + assert!(snapshot.contains_key("telemetry_test.test_name_application")); + } + + // Clear the Ping lifetime. + db.clear_ping_lifetime_storage(test_storage).unwrap(); + + // Take a snapshot again and check that we're only clearing the Ping lifetime. + { + let mut snapshot: HashMap = HashMap::new(); + let mut snapshotter = |metric_id: &[u8], metric: &Metric| { + let metric_id = String::from_utf8_lossy(metric_id).into_owned(); + match metric { + Metric::String(s) => snapshot.insert(metric_id, s.to_string()), + _ => panic!("Unexpected data found"), + }; + }; + + db.iter_store_from(Lifetime::User, test_storage, None, &mut snapshotter); + db.iter_store_from(Lifetime::Ping, test_storage, None, &mut snapshotter); + db.iter_store_from(Lifetime::Application, test_storage, None, &mut snapshotter); + + assert_eq!(2, snapshot.len(), "We only expect 2 metrics to be left."); + assert!(snapshot.contains_key("telemetry_test.test_name_user")); + assert!(snapshot.contains_key("telemetry_test.test_name_application")); + } + } + + #[test] + fn test_remove_single_metric() { + // Init the database in a temporary directory. + let dir = tempdir().unwrap(); + let str_dir = dir.path().display().to_string(); + let db = Database::new(&str_dir, false).unwrap(); + + let test_storage = "test-storage-single-lifetime"; + let metric_id_pattern = "telemetry_test.single_metric"; + + // Write sample metrics to the database. + let lifetimes = vec![Lifetime::User, Lifetime::Ping, Lifetime::Application]; + + for lifetime in lifetimes.iter() { + for value in &["retain", "delete"] { + db.record_per_lifetime( + *lifetime, + test_storage, + &format!("{}_{}", metric_id_pattern, value), + &Metric::String((*value).to_string()), + ) + .unwrap(); + } + } + + // Remove "telemetry_test.single_metric_delete" from each lifetime. + for lifetime in lifetimes.iter() { + db.remove_single_metric( + *lifetime, + test_storage, + &format!("{}_delete", metric_id_pattern), + ) + .unwrap(); + } + + // Verify that "telemetry_test.single_metric_retain" is still around for all lifetimes. + for lifetime in lifetimes.iter() { + let mut found_metrics = 0; + let mut snapshotter = |metric_id: &[u8], metric: &Metric| { + found_metrics += 1; + let metric_id = String::from_utf8_lossy(metric_id).into_owned(); + assert_eq!(format!("{}_retain", metric_id_pattern), metric_id); + match metric { + Metric::String(s) => assert_eq!("retain", s), + _ => panic!("Unexpected data found"), + } + }; + + // Check the User lifetime. + db.iter_store_from(*lifetime, test_storage, None, &mut snapshotter); + assert_eq!( + 1, found_metrics, + "We only expect 1 metric for this lifetime." + ); + } + } + + #[test] + fn test_delayed_ping_lifetime_persistence() { + // Init the database in a temporary directory. + let dir = tempdir().unwrap(); + let str_dir = dir.path().display().to_string(); + let db = Database::new(&str_dir, true).unwrap(); + let test_storage = "test-storage"; + + assert!(db.ping_lifetime_data.is_some()); + + // Attempt to record a known value. + let test_value1 = "test-value1"; + let test_metric_id1 = "telemetry_test.test_name1"; + db.record_per_lifetime( + Lifetime::Ping, + test_storage, + test_metric_id1, + &Metric::String(test_value1.to_string()), + ) + .unwrap(); + + // Attempt to persist data. + db.persist_ping_lifetime_data().unwrap(); + + // Attempt to record another known value. + let test_value2 = "test-value2"; + let test_metric_id2 = "telemetry_test.test_name2"; + db.record_per_lifetime( + Lifetime::Ping, + test_storage, + test_metric_id2, + &Metric::String(test_value2.to_string()), + ) + .unwrap(); + + { + // At this stage we expect `test_value1` to be persisted and in memory, + // since it was recorded before calling `persist_ping_lifetime_data`, + // and `test_value2` to be only in memory, since it was recorded after. + let store: SingleStore = db + .rkv + .open_single(Lifetime::Ping.as_str(), StoreOptions::create()) + .unwrap(); + let reader = db.rkv.read().unwrap(); + + // Verify that test_value1 is in rkv. + assert!(store + .get(&reader, format!("{}#{}", test_storage, test_metric_id1)) + .unwrap_or(None) + .is_some()); + // Verifiy that test_value2 is **not** in rkv. + assert!(store + .get(&reader, format!("{}#{}", test_storage, test_metric_id2)) + .unwrap_or(None) + .is_none()); + + let data = match &db.ping_lifetime_data { + Some(ping_lifetime_data) => ping_lifetime_data, + None => panic!("Expected `ping_lifetime_data` to exist here!"), + }; + let data = data.read().unwrap(); + // Verify that test_value1 is also in memory. + assert!(data + .get(&format!("{}#{}", test_storage, test_metric_id1)) + .is_some()); + // Verify that test_value2 is in memory. + assert!(data + .get(&format!("{}#{}", test_storage, test_metric_id2)) + .is_some()); + } + + // Attempt to persist data again. + db.persist_ping_lifetime_data().unwrap(); + + { + // At this stage we expect `test_value1` and `test_value2` to + // be persisted, since both were created before a call to + // `persist_ping_lifetime_data`. + let store: SingleStore = db + .rkv + .open_single(Lifetime::Ping.as_str(), StoreOptions::create()) + .unwrap(); + let reader = db.rkv.read().unwrap(); + + // Verify that test_value1 is in rkv. + assert!(store + .get(&reader, format!("{}#{}", test_storage, test_metric_id1)) + .unwrap_or(None) + .is_some()); + // Verifiy that test_value2 is also in rkv. + assert!(store + .get(&reader, format!("{}#{}", test_storage, test_metric_id2)) + .unwrap_or(None) + .is_some()); + + let data = match &db.ping_lifetime_data { + Some(ping_lifetime_data) => ping_lifetime_data, + None => panic!("Expected `ping_lifetime_data` to exist here!"), + }; + let data = data.read().unwrap(); + // Verify that test_value1 is also in memory. + assert!(data + .get(&format!("{}#{}", test_storage, test_metric_id1)) + .is_some()); + // Verify that test_value2 is also in memory. + assert!(data + .get(&format!("{}#{}", test_storage, test_metric_id2)) + .is_some()); + } + } + + #[test] + fn test_load_ping_lifetime_data_from_memory() { + // Init the database in a temporary directory. + let dir = tempdir().unwrap(); + let str_dir = dir.path().display().to_string(); + + let test_storage = "test-storage"; + let test_value = "test-value"; + let test_metric_id = "telemetry_test.test_name"; + + { + let db = Database::new(&str_dir, true).unwrap(); + + // Attempt to record a known value. + db.record_per_lifetime( + Lifetime::Ping, + test_storage, + test_metric_id, + &Metric::String(test_value.to_string()), + ) + .unwrap(); + + // Verify that test_value is in memory. + let data = match &db.ping_lifetime_data { + Some(ping_lifetime_data) => ping_lifetime_data, + None => panic!("Expected `ping_lifetime_data` to exist here!"), + }; + let data = data.read().unwrap(); + assert!(data + .get(&format!("{}#{}", test_storage, test_metric_id)) + .is_some()); + + // Attempt to persist data. + db.persist_ping_lifetime_data().unwrap(); + + // Verify that test_value is now in rkv. + let store: SingleStore = db + .rkv + .open_single(Lifetime::Ping.as_str(), StoreOptions::create()) + .unwrap(); + let reader = db.rkv.read().unwrap(); + assert!(store + .get(&reader, format!("{}#{}", test_storage, test_metric_id)) + .unwrap_or(None) + .is_some()); + } + + // Now create a new instace of the db and check if data was + // correctly loaded from rkv to memory. + { + let db = Database::new(&str_dir, true).unwrap(); + + // Verify that test_value is in memory. + let data = match &db.ping_lifetime_data { + Some(ping_lifetime_data) => ping_lifetime_data, + None => panic!("Expected `ping_lifetime_data` to exist here!"), + }; + let data = data.read().unwrap(); + assert!(data + .get(&format!("{}#{}", test_storage, test_metric_id)) + .is_some()); + + // Verify that test_value is also in rkv. + let store: SingleStore = db + .rkv + .open_single(Lifetime::Ping.as_str(), StoreOptions::create()) + .unwrap(); + let reader = db.rkv.read().unwrap(); + assert!(store + .get(&reader, format!("{}#{}", test_storage, test_metric_id)) + .unwrap_or(None) + .is_some()); + } + } + + #[test] + fn doesnt_record_when_upload_is_disabled() { + let (mut glean, dir) = new_glean(None); + + // Init the database in a temporary directory. + let str_dir = dir.path().display().to_string(); + + let test_storage = "test-storage"; + let test_data = CommonMetricData::new("category", "name", test_storage); + let test_metric_id = test_data.identifier(&glean); + + // Attempt to record metric with the record and record_with functions, + // this should work since upload is enabled. + let db = Database::new(&str_dir, true).unwrap(); + db.record(&glean, &test_data, &Metric::String("record".to_owned())); + db.iter_store_from( + Lifetime::Ping, + test_storage, + None, + &mut |metric_id: &[u8], metric: &Metric| { + assert_eq!( + String::from_utf8_lossy(metric_id).into_owned(), + test_metric_id + ); + match metric { + Metric::String(v) => assert_eq!("record", *v), + _ => panic!("Unexpected data found"), + } + }, + ); + + db.record_with(&glean, &test_data, |_| { + Metric::String("record_with".to_owned()) + }); + db.iter_store_from( + Lifetime::Ping, + test_storage, + None, + &mut |metric_id: &[u8], metric: &Metric| { + assert_eq!( + String::from_utf8_lossy(metric_id).into_owned(), + test_metric_id + ); + match metric { + Metric::String(v) => assert_eq!("record_with", *v), + _ => panic!("Unexpected data found"), + } + }, + ); + + // Disable upload + glean.set_upload_enabled(false); + + // Attempt to record metric with the record and record_with functions, + // this should work since upload is now **disabled**. + db.record(&glean, &test_data, &Metric::String("record_nop".to_owned())); + db.iter_store_from( + Lifetime::Ping, + test_storage, + None, + &mut |metric_id: &[u8], metric: &Metric| { + assert_eq!( + String::from_utf8_lossy(metric_id).into_owned(), + test_metric_id + ); + match metric { + Metric::String(v) => assert_eq!("record_with", *v), + _ => panic!("Unexpected data found"), + } + }, + ); + db.record_with(&glean, &test_data, |_| { + Metric::String("record_with_nop".to_owned()) + }); + db.iter_store_from( + Lifetime::Ping, + test_storage, + None, + &mut |metric_id: &[u8], metric: &Metric| { + assert_eq!( + String::from_utf8_lossy(metric_id).into_owned(), + test_metric_id + ); + match metric { + Metric::String(v) => assert_eq!("record_with", *v), + _ => panic!("Unexpected data found"), + } + }, + ); + } + + #[cfg(feature = "rkv-safe-mode")] + mod safe_mode_migration { + use super::*; + use rkv::Value; + + #[test] + fn migration_works_on_startup() { + let dir = tempdir().unwrap(); + let str_dir = dir.path().display().to_string(); + + let database_dir = dir.path().join("db"); + let datamdb = database_dir.join("data.mdb"); + let lockmdb = database_dir.join("lock.mdb"); + let safebin = database_dir.join("data.safe.bin"); + + assert!(!safebin.exists()); + assert!(!datamdb.exists()); + assert!(!lockmdb.exists()); + + let store_name = "store1"; + let metric_name = "bool"; + let key = Database::get_storage_key(store_name, Some(metric_name)); + + // Ensure some old data in the LMDB format exists. + { + fs::create_dir_all(&database_dir).expect("create dir"); + let rkv_db = rkv::Rkv::new::(&database_dir).expect("rkv env"); + + let store = rkv_db + .open_single("ping", StoreOptions::create()) + .expect("opened"); + let mut writer = rkv_db.write().expect("writer"); + let metric = Metric::Boolean(true); + let value = bincode::serialize(&metric).expect("serialized"); + store + .put(&mut writer, &key, &Value::Blob(&value)) + .expect("wrote"); + writer.commit().expect("committed"); + + assert!(datamdb.exists()); + assert!(lockmdb.exists()); + assert!(!safebin.exists()); + } + + // First open should migrate the data. + { + let db = Database::new(&str_dir, false).unwrap(); + let safebin = database_dir.join("data.safe.bin"); + assert!(safebin.exists(), "safe-mode file should exist"); + assert!(!datamdb.exists(), "LMDB data should be deleted"); + assert!(!lockmdb.exists(), "LMDB lock should be deleted"); + + let mut stored_metrics = vec![]; + let mut snapshotter = |name: &[u8], metric: &Metric| { + let name = str::from_utf8(name).unwrap().to_string(); + stored_metrics.push((name, metric.clone())) + }; + db.iter_store_from(Lifetime::Ping, "store1", None, &mut snapshotter); + + assert_eq!(1, stored_metrics.len()); + assert_eq!(metric_name, stored_metrics[0].0); + assert_eq!(&Metric::Boolean(true), &stored_metrics[0].1); + } + + // Next open should not re-create the LMDB files. + { + let db = Database::new(&str_dir, false).unwrap(); + let safebin = database_dir.join("data.safe.bin"); + assert!(safebin.exists(), "safe-mode file exists"); + assert!(!datamdb.exists(), "LMDB data should not be recreated"); + assert!(!lockmdb.exists(), "LMDB lock should not be recreated"); + + let mut stored_metrics = vec![]; + let mut snapshotter = |name: &[u8], metric: &Metric| { + let name = str::from_utf8(name).unwrap().to_string(); + stored_metrics.push((name, metric.clone())) + }; + db.iter_store_from(Lifetime::Ping, "store1", None, &mut snapshotter); + + assert_eq!(1, stored_metrics.len()); + assert_eq!(metric_name, stored_metrics[0].0); + assert_eq!(&Metric::Boolean(true), &stored_metrics[0].1); + } + } + + #[test] + fn migration_doesnt_overwrite() { + let dir = tempdir().unwrap(); + let str_dir = dir.path().display().to_string(); + + let database_dir = dir.path().join("db"); + let datamdb = database_dir.join("data.mdb"); + let lockmdb = database_dir.join("lock.mdb"); + let safebin = database_dir.join("data.safe.bin"); + + assert!(!safebin.exists()); + assert!(!datamdb.exists()); + assert!(!lockmdb.exists()); + + let store_name = "store1"; + let metric_name = "counter"; + let key = Database::get_storage_key(store_name, Some(metric_name)); + + // Ensure some old data in the LMDB format exists. + { + fs::create_dir_all(&database_dir).expect("create dir"); + let rkv_db = rkv::Rkv::new::(&database_dir).expect("rkv env"); + + let store = rkv_db + .open_single("ping", StoreOptions::create()) + .expect("opened"); + let mut writer = rkv_db.write().expect("writer"); + let metric = Metric::Counter(734); // this value will be ignored + let value = bincode::serialize(&metric).expect("serialized"); + store + .put(&mut writer, &key, &Value::Blob(&value)) + .expect("wrote"); + writer.commit().expect("committed"); + + assert!(datamdb.exists()); + assert!(lockmdb.exists()); + } + + // Ensure some data exists in the new database. + { + fs::create_dir_all(&database_dir).expect("create dir"); + let rkv_db = + rkv::Rkv::new::(&database_dir).expect("rkv env"); + + let store = rkv_db + .open_single("ping", StoreOptions::create()) + .expect("opened"); + let mut writer = rkv_db.write().expect("writer"); + let metric = Metric::Counter(2); + let value = bincode::serialize(&metric).expect("serialized"); + store + .put(&mut writer, &key, &Value::Blob(&value)) + .expect("wrote"); + writer.commit().expect("committed"); + + assert!(safebin.exists()); + } + + // First open should try migration and ignore it, because destination is not empty. + // It also deletes the leftover LMDB database. + { + let db = Database::new(&str_dir, false).unwrap(); + let safebin = database_dir.join("data.safe.bin"); + assert!(safebin.exists(), "safe-mode file should exist"); + assert!(!datamdb.exists(), "LMDB data should be deleted"); + assert!(!lockmdb.exists(), "LMDB lock should be deleted"); + + let mut stored_metrics = vec![]; + let mut snapshotter = |name: &[u8], metric: &Metric| { + let name = str::from_utf8(name).unwrap().to_string(); + stored_metrics.push((name, metric.clone())) + }; + db.iter_store_from(Lifetime::Ping, "store1", None, &mut snapshotter); + + assert_eq!(1, stored_metrics.len()); + assert_eq!(metric_name, stored_metrics[0].0); + assert_eq!(&Metric::Counter(2), &stored_metrics[0].1); + } + } + + #[test] + fn migration_ignores_broken_database() { + let dir = tempdir().unwrap(); + let str_dir = dir.path().display().to_string(); + + let database_dir = dir.path().join("db"); + let datamdb = database_dir.join("data.mdb"); + let lockmdb = database_dir.join("lock.mdb"); + let safebin = database_dir.join("data.safe.bin"); + + assert!(!safebin.exists()); + assert!(!datamdb.exists()); + assert!(!lockmdb.exists()); + + let store_name = "store1"; + let metric_name = "counter"; + let key = Database::get_storage_key(store_name, Some(metric_name)); + + // Ensure some old data in the LMDB format exists. + { + fs::create_dir_all(&database_dir).expect("create dir"); + fs::write(&datamdb, "bogus").expect("dbfile created"); + + assert!(datamdb.exists()); + } + + // Ensure some data exists in the new database. + { + fs::create_dir_all(&database_dir).expect("create dir"); + let rkv_db = + rkv::Rkv::new::(&database_dir).expect("rkv env"); + + let store = rkv_db + .open_single("ping", StoreOptions::create()) + .expect("opened"); + let mut writer = rkv_db.write().expect("writer"); + let metric = Metric::Counter(2); + let value = bincode::serialize(&metric).expect("serialized"); + store + .put(&mut writer, &key, &Value::Blob(&value)) + .expect("wrote"); + writer.commit().expect("committed"); + } + + // First open should try migration and ignore it, because destination is not empty. + // It also deletes the leftover LMDB database. + { + let db = Database::new(&str_dir, false).unwrap(); + let safebin = database_dir.join("data.safe.bin"); + assert!(safebin.exists(), "safe-mode file should exist"); + assert!(!datamdb.exists(), "LMDB data should be deleted"); + assert!(!lockmdb.exists(), "LMDB lock should be deleted"); + + let mut stored_metrics = vec![]; + let mut snapshotter = |name: &[u8], metric: &Metric| { + let name = str::from_utf8(name).unwrap().to_string(); + stored_metrics.push((name, metric.clone())) + }; + db.iter_store_from(Lifetime::Ping, "store1", None, &mut snapshotter); + + assert_eq!(1, stored_metrics.len()); + assert_eq!(metric_name, stored_metrics[0].0); + assert_eq!(&Metric::Counter(2), &stored_metrics[0].1); + } + } + + #[test] + fn migration_ignores_empty_database() { + let dir = tempdir().unwrap(); + let str_dir = dir.path().display().to_string(); + + let database_dir = dir.path().join("db"); + let datamdb = database_dir.join("data.mdb"); + let lockmdb = database_dir.join("lock.mdb"); + let safebin = database_dir.join("data.safe.bin"); + + assert!(!safebin.exists()); + assert!(!datamdb.exists()); + assert!(!lockmdb.exists()); + + // Ensure old LMDB database exists, but is empty. + { + fs::create_dir_all(&database_dir).expect("create dir"); + let rkv_db = rkv::Rkv::new::(&database_dir).expect("rkv env"); + drop(rkv_db); + assert!(datamdb.exists()); + assert!(lockmdb.exists()); + } + + // First open should try migration, but find no data. + // safe-mode does not write an empty database to disk. + // It also deletes the leftover LMDB database. + { + let _db = Database::new(&str_dir, false).unwrap(); + let safebin = database_dir.join("data.safe.bin"); + assert!(!safebin.exists(), "safe-mode file should exist"); + assert!(!datamdb.exists(), "LMDB data should be deleted"); + assert!(!lockmdb.exists(), "LMDB lock should be deleted"); + } + } + } +} diff --git a/third_party/rust/glean-core/src/debug.rs b/third_party/rust/glean-core/src/debug.rs index 441023f56b8a..f7336042b418 100644 --- a/third_party/rust/glean-core/src/debug.rs +++ b/third_party/rust/glean-core/src/debug.rs @@ -1,321 +1,321 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -//! # Debug options -//! -//! The debug options for Glean may be set by calling one of the `set_*` functions -//! or by setting specific environment variables. -//! -//! The environment variables will be read only once when the options are initialized. -//! -//! The possible debugging features available out of the box are: -//! -//! * **Ping logging** - logging the contents of ping requests that are correctly assembled; -//! This may be set by calling glean.set_log_pings(value: bool) -//! or by setting the environment variable GLEAN_LOG_PINGS="true"; -//! * **Debug tagging** - Adding the X-Debug-ID header to every ping request, -//! allowing these tagged pings to be sent to the ["Ping Debug Viewer"](https://mozilla.github.io/glean/book/dev/core/internal/debug-pings.html). -//! This may be set by calling glean.set_debug_view_tag(value: &str) -//! or by setting the environment variable GLEAN_DEBUG_VIEW_TAG=; -//! * **Source tagging** - Adding the X-Source-Tags header to every ping request, -//! allowing pings to be tagged with custom labels. -//! This may be set by calling glean.set_source_tags(value: Vec) -//! or by setting the environment variable GLEAN_SOURCE_TAGS=; -//! -//! Bindings may implement other debugging features, e.g. sending pings on demand. - -use std::env; - -const GLEAN_LOG_PINGS: &str = "GLEAN_LOG_PINGS"; -const GLEAN_DEBUG_VIEW_TAG: &str = "GLEAN_DEBUG_VIEW_TAG"; -const GLEAN_SOURCE_TAGS: &str = "GLEAN_SOURCE_TAGS"; -const GLEAN_MAX_SOURCE_TAGS: usize = 5; - -/// A representation of all of Glean's debug options. -pub struct DebugOptions { - /// Option to log the payload of pings that are successfully assembled into a ping request. - pub log_pings: DebugOption, - /// Option to add the X-Debug-ID header to every ping request. - pub debug_view_tag: DebugOption, - /// Option to add the X-Source-Tags header to ping requests. This will allow the data - /// consumers to classify data depending on the applied tags. - pub source_tags: DebugOption>, -} - -impl std::fmt::Debug for DebugOptions { - fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result { - fmt.debug_struct("DebugOptions") - .field("log_pings", &self.log_pings.get()) - .field("debug_view_tag", &self.debug_view_tag.get()) - .field("source_tags", &self.source_tags.get()) - .finish() - } -} - -impl DebugOptions { - pub fn new() -> Self { - Self { - log_pings: DebugOption::new(GLEAN_LOG_PINGS, get_bool_from_str, None), - debug_view_tag: DebugOption::new(GLEAN_DEBUG_VIEW_TAG, Some, Some(validate_tag)), - source_tags: DebugOption::new( - GLEAN_SOURCE_TAGS, - tokenize_string, - Some(validate_source_tags), - ), - } - } -} - -/// A representation of a debug option, -/// where the value can be set programmatically or come from an environment variable. -#[derive(Debug)] -pub struct DebugOption Option, V = fn(&T) -> bool> { - /// The name of the environment variable related to this debug option. - env: String, - /// The actual value of this option. - value: Option, - /// Function to extract the data of type `T` from a `String`, used when - /// extracting data from the environment. - extraction: E, - /// Optional function to validate the value parsed from the environment - /// or passed to the `set` function. - validation: Option, -} - -impl DebugOption -where - T: Clone, - E: Fn(String) -> Option, - V: Fn(&T) -> bool, -{ - /// Creates a new debug option. - /// - /// Tries to get the initial value of the option from the environment. - pub fn new(env: &str, extraction: E, validation: Option) -> Self { - let mut option = Self { - env: env.into(), - value: None, - extraction, - validation, - }; - - option.set_from_env(); - option - } - - fn validate(&self, value: &T) -> bool { - if let Some(f) = self.validation.as_ref() { - f(value) - } else { - true - } - } - - fn set_from_env(&mut self) { - let extract = &self.extraction; - match env::var(&self.env) { - Ok(env_value) => match extract(env_value.clone()) { - Some(v) => { - self.set(v); - } - None => { - log::error!( - "Unable to parse debug option {}={} into {}. Ignoring.", - self.env, - env_value, - std::any::type_name::() - ); - } - }, - Err(env::VarError::NotUnicode(_)) => { - log::error!("The value of {} is not valid unicode. Ignoring.", self.env) - } - // The other possible error is that the env var is not set, - // which is not an error for us and can safely be ignored. - Err(_) => {} - } - } - - /// Tries to set a value for this debug option. - /// - /// Validates the value in case a validation function is available. - /// - /// # Returns - /// - /// Whether the option passed validation and was succesfully set. - pub fn set(&mut self, value: T) -> bool { - let validated = self.validate(&value); - if validated { - log::info!("Setting the debug option {}.", self.env); - self.value = Some(value); - return true; - } - log::info!("Invalid value for debug option {}.", self.env); - false - } - - /// Gets the value of this debug option. - pub fn get(&self) -> Option<&T> { - self.value.as_ref() - } -} - -fn get_bool_from_str(value: String) -> Option { - std::str::FromStr::from_str(&value).ok() -} - -fn tokenize_string(value: String) -> Option> { - let trimmed = value.trim(); - if trimmed.is_empty() { - return None; - } - - Some(trimmed.split(',').map(|s| s.trim().to_string()).collect()) -} - -/// A tag is the value used in both the `X-Debug-ID` and `X-Source-Tags` headers -/// of tagged ping requests, thus is it must be a valid header value. -/// -/// In other words, it must match the regex: "[a-zA-Z0-9-]{1,20}" -/// -/// The regex crate isn't used here because it adds to the binary size, -/// and the Glean SDK doesn't use regular expressions anywhere else. -#[allow(clippy::ptr_arg)] -fn validate_tag(value: &String) -> bool { - if value.is_empty() { - log::error!("A tag must have at least one character."); - return false; - } - - let mut iter = value.chars(); - let mut count = 0; - - loop { - match iter.next() { - // We are done, so the whole expression is valid. - None => return true, - // Valid characters. - Some('-') | Some('a'..='z') | Some('A'..='Z') | Some('0'..='9') => (), - // An invalid character - Some(c) => { - log::error!("Invalid character '{}' in the tag.", c); - return false; - } - } - count += 1; - if count == 20 { - log::error!("A tag cannot exceed 20 characters."); - return false; - } - } -} - -/// Validate the list of source tags. -/// -/// This builds upon the existing `validate_tag` function, since all the -/// tags should respect the same rules to make the pipeline happy. -#[allow(clippy::ptr_arg)] -fn validate_source_tags(tags: &Vec) -> bool { - if tags.is_empty() { - return false; - } - - if tags.len() > GLEAN_MAX_SOURCE_TAGS { - log::error!( - "A list of tags cannot contain more than {} elements.", - GLEAN_MAX_SOURCE_TAGS - ); - return false; - } - - // Filter out tags starting with "glean". They are reserved. - if tags.iter().any(|s| s.starts_with("glean")) { - log::error!("Tags starting with `glean` are reserved and must not be used."); - return false; - } - - tags.iter().all(|x| validate_tag(&x)) -} - -#[cfg(test)] -mod test { - use super::*; - use std::env; - - #[test] - fn debug_option_is_correctly_loaded_from_env() { - env::set_var("GLEAN_TEST_1", "test"); - let option: DebugOption = DebugOption::new("GLEAN_TEST_1", Some, None); - assert_eq!(option.get().unwrap(), "test"); - } - - #[test] - fn debug_option_is_correctly_validated_when_necessary() { - #[allow(clippy::ptr_arg)] - fn validate(value: &String) -> bool { - value == "test" - } - - // Invalid values from the env are not set - env::set_var("GLEAN_TEST_2", "invalid"); - let mut option: DebugOption = - DebugOption::new("GLEAN_TEST_2", Some, Some(validate)); - assert!(option.get().is_none()); - - // Valid values are set using the `set` function - assert!(option.set("test".into())); - assert_eq!(option.get().unwrap(), "test"); - - // Invalid values are not set using the `set` function - assert!(!option.set("invalid".into())); - assert_eq!(option.get().unwrap(), "test"); - } - - #[test] - fn tokenize_string_splits_correctly() { - // Valid list is properly tokenized and spaces are trimmed. - assert_eq!( - Some(vec!["test1".to_string(), "test2".to_string()]), - tokenize_string(" test1, test2 ".to_string()) - ); - - // Empty strings return no item. - assert_eq!(None, tokenize_string("".to_string())); - } - - #[test] - fn validates_tag_correctly() { - assert!(validate_tag(&"valid-value".to_string())); - assert!(validate_tag(&"-also-valid-value".to_string())); - assert!(!validate_tag(&"invalid_value".to_string())); - assert!(!validate_tag(&"invalid value".to_string())); - assert!(!validate_tag(&"!nv@lid-val*e".to_string())); - assert!(!validate_tag( - &"invalid-value-because-way-too-long".to_string() - )); - assert!(!validate_tag(&"".to_string())); - } - - #[test] - fn validates_source_tags_correctly() { - // Empty tags. - assert!(!validate_source_tags(&vec!["".to_string()])); - // Too many tags. - assert!(!validate_source_tags(&vec![ - "1".to_string(), - "2".to_string(), - "3".to_string(), - "4".to_string(), - "5".to_string(), - "6".to_string() - ])); - // Invalid tags. - assert!(!validate_source_tags(&vec!["!nv@lid-val*e".to_string()])); - // Entries starting with 'glean' are filtered out. - assert!(!validate_source_tags(&vec![ - "glean-test1".to_string(), - "test2".to_string() - ])); - } -} +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! # Debug options +//! +//! The debug options for Glean may be set by calling one of the `set_*` functions +//! or by setting specific environment variables. +//! +//! The environment variables will be read only once when the options are initialized. +//! +//! The possible debugging features available out of the box are: +//! +//! * **Ping logging** - logging the contents of ping requests that are correctly assembled; +//! This may be set by calling glean.set_log_pings(value: bool) +//! or by setting the environment variable GLEAN_LOG_PINGS="true"; +//! * **Debug tagging** - Adding the X-Debug-ID header to every ping request, +//! allowing these tagged pings to be sent to the ["Ping Debug Viewer"](https://mozilla.github.io/glean/book/dev/core/internal/debug-pings.html). +//! This may be set by calling glean.set_debug_view_tag(value: &str) +//! or by setting the environment variable GLEAN_DEBUG_VIEW_TAG=; +//! * **Source tagging** - Adding the X-Source-Tags header to every ping request, +//! allowing pings to be tagged with custom labels. +//! This may be set by calling glean.set_source_tags(value: Vec) +//! or by setting the environment variable GLEAN_SOURCE_TAGS=; +//! +//! Bindings may implement other debugging features, e.g. sending pings on demand. + +use std::env; + +const GLEAN_LOG_PINGS: &str = "GLEAN_LOG_PINGS"; +const GLEAN_DEBUG_VIEW_TAG: &str = "GLEAN_DEBUG_VIEW_TAG"; +const GLEAN_SOURCE_TAGS: &str = "GLEAN_SOURCE_TAGS"; +const GLEAN_MAX_SOURCE_TAGS: usize = 5; + +/// A representation of all of Glean's debug options. +pub struct DebugOptions { + /// Option to log the payload of pings that are successfully assembled into a ping request. + pub log_pings: DebugOption, + /// Option to add the X-Debug-ID header to every ping request. + pub debug_view_tag: DebugOption, + /// Option to add the X-Source-Tags header to ping requests. This will allow the data + /// consumers to classify data depending on the applied tags. + pub source_tags: DebugOption>, +} + +impl std::fmt::Debug for DebugOptions { + fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result { + fmt.debug_struct("DebugOptions") + .field("log_pings", &self.log_pings.get()) + .field("debug_view_tag", &self.debug_view_tag.get()) + .field("source_tags", &self.source_tags.get()) + .finish() + } +} + +impl DebugOptions { + pub fn new() -> Self { + Self { + log_pings: DebugOption::new(GLEAN_LOG_PINGS, get_bool_from_str, None), + debug_view_tag: DebugOption::new(GLEAN_DEBUG_VIEW_TAG, Some, Some(validate_tag)), + source_tags: DebugOption::new( + GLEAN_SOURCE_TAGS, + tokenize_string, + Some(validate_source_tags), + ), + } + } +} + +/// A representation of a debug option, +/// where the value can be set programmatically or come from an environment variable. +#[derive(Debug)] +pub struct DebugOption Option, V = fn(&T) -> bool> { + /// The name of the environment variable related to this debug option. + env: String, + /// The actual value of this option. + value: Option, + /// Function to extract the data of type `T` from a `String`, used when + /// extracting data from the environment. + extraction: E, + /// Optional function to validate the value parsed from the environment + /// or passed to the `set` function. + validation: Option, +} + +impl DebugOption +where + T: Clone, + E: Fn(String) -> Option, + V: Fn(&T) -> bool, +{ + /// Creates a new debug option. + /// + /// Tries to get the initial value of the option from the environment. + pub fn new(env: &str, extraction: E, validation: Option) -> Self { + let mut option = Self { + env: env.into(), + value: None, + extraction, + validation, + }; + + option.set_from_env(); + option + } + + fn validate(&self, value: &T) -> bool { + if let Some(f) = self.validation.as_ref() { + f(value) + } else { + true + } + } + + fn set_from_env(&mut self) { + let extract = &self.extraction; + match env::var(&self.env) { + Ok(env_value) => match extract(env_value.clone()) { + Some(v) => { + self.set(v); + } + None => { + log::error!( + "Unable to parse debug option {}={} into {}. Ignoring.", + self.env, + env_value, + std::any::type_name::() + ); + } + }, + Err(env::VarError::NotUnicode(_)) => { + log::error!("The value of {} is not valid unicode. Ignoring.", self.env) + } + // The other possible error is that the env var is not set, + // which is not an error for us and can safely be ignored. + Err(_) => {} + } + } + + /// Tries to set a value for this debug option. + /// + /// Validates the value in case a validation function is available. + /// + /// # Returns + /// + /// Whether the option passed validation and was succesfully set. + pub fn set(&mut self, value: T) -> bool { + let validated = self.validate(&value); + if validated { + log::info!("Setting the debug option {}.", self.env); + self.value = Some(value); + return true; + } + log::info!("Invalid value for debug option {}.", self.env); + false + } + + /// Gets the value of this debug option. + pub fn get(&self) -> Option<&T> { + self.value.as_ref() + } +} + +fn get_bool_from_str(value: String) -> Option { + std::str::FromStr::from_str(&value).ok() +} + +fn tokenize_string(value: String) -> Option> { + let trimmed = value.trim(); + if trimmed.is_empty() { + return None; + } + + Some(trimmed.split(',').map(|s| s.trim().to_string()).collect()) +} + +/// A tag is the value used in both the `X-Debug-ID` and `X-Source-Tags` headers +/// of tagged ping requests, thus is it must be a valid header value. +/// +/// In other words, it must match the regex: "[a-zA-Z0-9-]{1,20}" +/// +/// The regex crate isn't used here because it adds to the binary size, +/// and the Glean SDK doesn't use regular expressions anywhere else. +#[allow(clippy::ptr_arg)] +fn validate_tag(value: &String) -> bool { + if value.is_empty() { + log::error!("A tag must have at least one character."); + return false; + } + + let mut iter = value.chars(); + let mut count = 0; + + loop { + match iter.next() { + // We are done, so the whole expression is valid. + None => return true, + // Valid characters. + Some('-') | Some('a'..='z') | Some('A'..='Z') | Some('0'..='9') => (), + // An invalid character + Some(c) => { + log::error!("Invalid character '{}' in the tag.", c); + return false; + } + } + count += 1; + if count == 20 { + log::error!("A tag cannot exceed 20 characters."); + return false; + } + } +} + +/// Validate the list of source tags. +/// +/// This builds upon the existing `validate_tag` function, since all the +/// tags should respect the same rules to make the pipeline happy. +#[allow(clippy::ptr_arg)] +fn validate_source_tags(tags: &Vec) -> bool { + if tags.is_empty() { + return false; + } + + if tags.len() > GLEAN_MAX_SOURCE_TAGS { + log::error!( + "A list of tags cannot contain more than {} elements.", + GLEAN_MAX_SOURCE_TAGS + ); + return false; + } + + // Filter out tags starting with "glean". They are reserved. + if tags.iter().any(|s| s.starts_with("glean")) { + log::error!("Tags starting with `glean` are reserved and must not be used."); + return false; + } + + tags.iter().all(|x| validate_tag(&x)) +} + +#[cfg(test)] +mod test { + use super::*; + use std::env; + + #[test] + fn debug_option_is_correctly_loaded_from_env() { + env::set_var("GLEAN_TEST_1", "test"); + let option: DebugOption = DebugOption::new("GLEAN_TEST_1", Some, None); + assert_eq!(option.get().unwrap(), "test"); + } + + #[test] + fn debug_option_is_correctly_validated_when_necessary() { + #[allow(clippy::ptr_arg)] + fn validate(value: &String) -> bool { + value == "test" + } + + // Invalid values from the env are not set + env::set_var("GLEAN_TEST_2", "invalid"); + let mut option: DebugOption = + DebugOption::new("GLEAN_TEST_2", Some, Some(validate)); + assert!(option.get().is_none()); + + // Valid values are set using the `set` function + assert!(option.set("test".into())); + assert_eq!(option.get().unwrap(), "test"); + + // Invalid values are not set using the `set` function + assert!(!option.set("invalid".into())); + assert_eq!(option.get().unwrap(), "test"); + } + + #[test] + fn tokenize_string_splits_correctly() { + // Valid list is properly tokenized and spaces are trimmed. + assert_eq!( + Some(vec!["test1".to_string(), "test2".to_string()]), + tokenize_string(" test1, test2 ".to_string()) + ); + + // Empty strings return no item. + assert_eq!(None, tokenize_string("".to_string())); + } + + #[test] + fn validates_tag_correctly() { + assert!(validate_tag(&"valid-value".to_string())); + assert!(validate_tag(&"-also-valid-value".to_string())); + assert!(!validate_tag(&"invalid_value".to_string())); + assert!(!validate_tag(&"invalid value".to_string())); + assert!(!validate_tag(&"!nv@lid-val*e".to_string())); + assert!(!validate_tag( + &"invalid-value-because-way-too-long".to_string() + )); + assert!(!validate_tag(&"".to_string())); + } + + #[test] + fn validates_source_tags_correctly() { + // Empty tags. + assert!(!validate_source_tags(&vec!["".to_string()])); + // Too many tags. + assert!(!validate_source_tags(&vec![ + "1".to_string(), + "2".to_string(), + "3".to_string(), + "4".to_string(), + "5".to_string(), + "6".to_string() + ])); + // Invalid tags. + assert!(!validate_source_tags(&vec!["!nv@lid-val*e".to_string()])); + // Entries starting with 'glean' are filtered out. + assert!(!validate_source_tags(&vec![ + "glean-test1".to_string(), + "test2".to_string() + ])); + } +} diff --git a/third_party/rust/glean-core/src/error.rs b/third_party/rust/glean-core/src/error.rs index c62c73f0a258..e7657b386f07 100644 --- a/third_party/rust/glean-core/src/error.rs +++ b/third_party/rust/glean-core/src/error.rs @@ -1,189 +1,189 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -use std::ffi::OsString; -use std::fmt::{self, Display}; -use std::io; -use std::result; - -use ffi_support::{handle_map::HandleError, ExternError}; - -use rkv::StoreError; - -/// A specialized [`Result`] type for this crate's operations. -/// -/// This is generally used to avoid writing out [Error] directly and -/// is otherwise a direct mapping to [`Result`]. -/// -/// [`Result`]: https://doc.rust-lang.org/stable/std/result/enum.Result.html -/// [`Error`]: std.struct.Error.html -pub type Result = result::Result; - -/// A list enumerating the categories of errors in this crate. -/// -/// [`Error`]: https://doc.rust-lang.org/stable/std/error/trait.Error.html -/// -/// This list is intended to grow over time and it is not recommended to -/// exhaustively match against it. -#[derive(Debug)] -#[non_exhaustive] -pub enum ErrorKind { - /// Lifetime conversion failed - Lifetime(i32), - - /// FFI-Support error - Handle(HandleError), - - /// IO error - IoError(io::Error), - - /// IO error - Rkv(StoreError), - - /// JSON error - Json(serde_json::error::Error), - - /// TimeUnit conversion failed - TimeUnit(i32), - - /// MemoryUnit conversion failed - MemoryUnit(i32), - - /// HistogramType conversion failed - HistogramType(i32), - - /// OsString conversion failed - OsString(OsString), - - /// Unknown error - Utf8Error, - - /// Glean initialization was attempted with an invalid configuration - InvalidConfig, - - /// Glean not initialized - NotInitialized, - - /// Ping request body size overflowed - PingBodyOverflow(usize), -} - -/// A specialized [`Error`] type for this crate's operations. -/// -/// [`Error`]: https://doc.rust-lang.org/stable/std/error/trait.Error.html -#[derive(Debug)] -pub struct Error { - kind: ErrorKind, -} - -impl Error { - /// Returns a new UTF-8 error - /// - /// This is exposed in order to expose conversion errors on the FFI layer. - pub fn utf8_error() -> Error { - Error { - kind: ErrorKind::Utf8Error, - } - } - - /// Indicates an error that no requested global object is initialized - pub fn not_initialized() -> Error { - Error { - kind: ErrorKind::NotInitialized, - } - } - - /// Returns the kind of the current error instance. - pub fn kind(&self) -> &ErrorKind { - &self.kind - } -} - -impl std::error::Error for Error {} - -impl Display for Error { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - use ErrorKind::*; - match self.kind() { - Lifetime(l) => write!(f, "Lifetime conversion from {} failed", l), - Handle(e) => write!(f, "Invalid handle: {}", e), - IoError(e) => write!(f, "An I/O error occurred: {}", e), - Rkv(e) => write!(f, "An Rkv error occurred: {}", e), - Json(e) => write!(f, "A JSON error occurred: {}", e), - TimeUnit(t) => write!(f, "TimeUnit conversion from {} failed", t), - MemoryUnit(m) => write!(f, "MemoryUnit conversion from {} failed", m), - HistogramType(h) => write!(f, "HistogramType conversion from {} failed", h), - OsString(s) => write!(f, "OsString conversion from {:?} failed", s), - Utf8Error => write!(f, "Invalid UTF-8 byte sequence in string"), - InvalidConfig => write!(f, "Invalid Glean configuration provided"), - NotInitialized => write!(f, "Global Glean object missing"), - PingBodyOverflow(s) => write!( - f, - "Ping request body size exceeded maximum size allowed: {}kB.", - s / 1024 - ), - } - } -} - -impl From for Error { - fn from(kind: ErrorKind) -> Error { - Error { kind } - } -} - -impl From for Error { - fn from(error: HandleError) -> Error { - Error { - kind: ErrorKind::Handle(error), - } - } -} - -impl From for Error { - fn from(error: io::Error) -> Error { - Error { - kind: ErrorKind::IoError(error), - } - } -} - -impl From for Error { - fn from(error: StoreError) -> Error { - Error { - kind: ErrorKind::Rkv(error), - } - } -} - -impl From for ExternError { - fn from(error: Error) -> ExternError { - ffi_support::ExternError::new_error(ffi_support::ErrorCode::new(42), format!("{}", error)) - } -} - -impl From for Error { - fn from(error: serde_json::error::Error) -> Error { - Error { - kind: ErrorKind::Json(error), - } - } -} - -impl From for Error { - fn from(error: OsString) -> Error { - Error { - kind: ErrorKind::OsString(error), - } - } -} - -/// To satisfy integer conversion done by the macros on the FFI side, we need to be able to turn -/// something infallible into an error. -/// This will never actually be reached, as an integer-to-integer conversion is infallible. -impl From for Error { - fn from(_: std::convert::Infallible) -> Error { - unreachable!() - } -} +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::ffi::OsString; +use std::fmt::{self, Display}; +use std::io; +use std::result; + +use ffi_support::{handle_map::HandleError, ExternError}; + +use rkv::StoreError; + +/// A specialized [`Result`] type for this crate's operations. +/// +/// This is generally used to avoid writing out [Error] directly and +/// is otherwise a direct mapping to [`Result`]. +/// +/// [`Result`]: https://doc.rust-lang.org/stable/std/result/enum.Result.html +/// [`Error`]: std.struct.Error.html +pub type Result = result::Result; + +/// A list enumerating the categories of errors in this crate. +/// +/// [`Error`]: https://doc.rust-lang.org/stable/std/error/trait.Error.html +/// +/// This list is intended to grow over time and it is not recommended to +/// exhaustively match against it. +#[derive(Debug)] +#[non_exhaustive] +pub enum ErrorKind { + /// Lifetime conversion failed + Lifetime(i32), + + /// FFI-Support error + Handle(HandleError), + + /// IO error + IoError(io::Error), + + /// IO error + Rkv(StoreError), + + /// JSON error + Json(serde_json::error::Error), + + /// TimeUnit conversion failed + TimeUnit(i32), + + /// MemoryUnit conversion failed + MemoryUnit(i32), + + /// HistogramType conversion failed + HistogramType(i32), + + /// OsString conversion failed + OsString(OsString), + + /// Unknown error + Utf8Error, + + /// Glean initialization was attempted with an invalid configuration + InvalidConfig, + + /// Glean not initialized + NotInitialized, + + /// Ping request body size overflowed + PingBodyOverflow(usize), +} + +/// A specialized [`Error`] type for this crate's operations. +/// +/// [`Error`]: https://doc.rust-lang.org/stable/std/error/trait.Error.html +#[derive(Debug)] +pub struct Error { + kind: ErrorKind, +} + +impl Error { + /// Returns a new UTF-8 error + /// + /// This is exposed in order to expose conversion errors on the FFI layer. + pub fn utf8_error() -> Error { + Error { + kind: ErrorKind::Utf8Error, + } + } + + /// Indicates an error that no requested global object is initialized + pub fn not_initialized() -> Error { + Error { + kind: ErrorKind::NotInitialized, + } + } + + /// Returns the kind of the current error instance. + pub fn kind(&self) -> &ErrorKind { + &self.kind + } +} + +impl std::error::Error for Error {} + +impl Display for Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + use ErrorKind::*; + match self.kind() { + Lifetime(l) => write!(f, "Lifetime conversion from {} failed", l), + Handle(e) => write!(f, "Invalid handle: {}", e), + IoError(e) => write!(f, "An I/O error occurred: {}", e), + Rkv(e) => write!(f, "An Rkv error occurred: {}", e), + Json(e) => write!(f, "A JSON error occurred: {}", e), + TimeUnit(t) => write!(f, "TimeUnit conversion from {} failed", t), + MemoryUnit(m) => write!(f, "MemoryUnit conversion from {} failed", m), + HistogramType(h) => write!(f, "HistogramType conversion from {} failed", h), + OsString(s) => write!(f, "OsString conversion from {:?} failed", s), + Utf8Error => write!(f, "Invalid UTF-8 byte sequence in string"), + InvalidConfig => write!(f, "Invalid Glean configuration provided"), + NotInitialized => write!(f, "Global Glean object missing"), + PingBodyOverflow(s) => write!( + f, + "Ping request body size exceeded maximum size allowed: {}kB.", + s / 1024 + ), + } + } +} + +impl From for Error { + fn from(kind: ErrorKind) -> Error { + Error { kind } + } +} + +impl From for Error { + fn from(error: HandleError) -> Error { + Error { + kind: ErrorKind::Handle(error), + } + } +} + +impl From for Error { + fn from(error: io::Error) -> Error { + Error { + kind: ErrorKind::IoError(error), + } + } +} + +impl From for Error { + fn from(error: StoreError) -> Error { + Error { + kind: ErrorKind::Rkv(error), + } + } +} + +impl From for ExternError { + fn from(error: Error) -> ExternError { + ffi_support::ExternError::new_error(ffi_support::ErrorCode::new(42), format!("{}", error)) + } +} + +impl From for Error { + fn from(error: serde_json::error::Error) -> Error { + Error { + kind: ErrorKind::Json(error), + } + } +} + +impl From for Error { + fn from(error: OsString) -> Error { + Error { + kind: ErrorKind::OsString(error), + } + } +} + +/// To satisfy integer conversion done by the macros on the FFI side, we need to be able to turn +/// something infallible into an error. +/// This will never actually be reached, as an integer-to-integer conversion is infallible. +impl From for Error { + fn from(_: std::convert::Infallible) -> Error { + unreachable!() + } +} diff --git a/third_party/rust/glean-core/src/error_recording.rs b/third_party/rust/glean-core/src/error_recording.rs index 747149a3d721..e2b4ab778778 100644 --- a/third_party/rust/glean-core/src/error_recording.rs +++ b/third_party/rust/glean-core/src/error_recording.rs @@ -1,223 +1,223 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -//! # Error Recording -//! -//! Glean keeps track of errors that occured due to invalid labels or invalid values when recording -//! other metrics. -//! -//! Error counts are stored in labeled counters in the `glean.error` category. -//! The labeled counter metrics that store the errors are defined in the `metrics.yaml` for documentation purposes, -//! but are not actually used directly, since the `send_in_pings` value needs to match the pings of the metric that is erroring (plus the "metrics" ping), -//! not some constant value that we could define in `metrics.yaml`. - -use std::convert::TryFrom; -use std::fmt::Display; - -use crate::error::{Error, ErrorKind}; -use crate::metrics::CounterMetric; -use crate::metrics::{combine_base_identifier_and_label, strip_label}; -use crate::CommonMetricData; -use crate::Glean; -use crate::Lifetime; - -/// The possible error types for metric recording. -/// Note: the cases in this enum must be kept in sync with the ones -/// in the platform-specific code (e.g. ErrorType.kt) and with the -/// metrics in the registry files. -#[derive(Debug, PartialEq)] -pub enum ErrorType { - /// For when the value to be recorded does not match the metric-specific restrictions - InvalidValue, - /// For when the label of a labeled metric does not match the restrictions - InvalidLabel, - /// For when the metric caught an invalid state while recording - InvalidState, - /// For when the value to be recorded overflows the metric-specific upper range - InvalidOverflow, -} - -impl ErrorType { - /// The error type's metric id - pub fn as_str(&self) -> &'static str { - match self { - ErrorType::InvalidValue => "invalid_value", - ErrorType::InvalidLabel => "invalid_label", - ErrorType::InvalidState => "invalid_state", - ErrorType::InvalidOverflow => "invalid_overflow", - } - } -} - -impl TryFrom for ErrorType { - type Error = Error; - - fn try_from(value: i32) -> Result { - match value { - 0 => Ok(ErrorType::InvalidValue), - 1 => Ok(ErrorType::InvalidLabel), - 2 => Ok(ErrorType::InvalidState), - 3 => Ok(ErrorType::InvalidOverflow), - e => Err(ErrorKind::Lifetime(e).into()), - } - } -} - -/// For a given metric, get the metric in which to record errors -fn get_error_metric_for_metric(meta: &CommonMetricData, error: ErrorType) -> CounterMetric { - // Can't use meta.identifier here, since that might cause infinite recursion - // if the label on this metric needs to report an error. - let identifier = meta.base_identifier(); - let name = strip_label(&identifier); - - // Record errors in the pings the metric is in, as well as the metrics ping. - let mut send_in_pings = meta.send_in_pings.clone(); - let ping_name = "metrics".to_string(); - if !send_in_pings.contains(&ping_name) { - send_in_pings.push(ping_name); - } - - CounterMetric::new(CommonMetricData { - name: combine_base_identifier_and_label(error.as_str(), name), - category: "glean.error".into(), - lifetime: Lifetime::Ping, - send_in_pings, - ..Default::default() - }) -} - -/// Records an error into Glean. -/// -/// Errors are recorded as labeled counters in the `glean.error` category. -/// -/// *Note*: We do make assumptions here how labeled metrics are encoded, namely by having the name -/// `/