diff --git a/.cargo/config.in b/.cargo/config.in index d873dfa133a1..fb51a8685e96 100644 --- a/.cargo/config.in +++ b/.cargo/config.in @@ -17,6 +17,11 @@ git = "https://github.com/mozilla/neqo" replace-with = "vendored-sources" tag = "v0.2.2" +[source."https://github.com/mozilla/application-services"] +git = "https://github.com/mozilla/application-services" +replace-with = "vendored-sources" +rev = "120e51dd5f2aab4194cf0f7e93b2a8923f4504bb" + [source."https://github.com/mozilla-spidermonkey/jsparagus"] git = "https://github.com/mozilla-spidermonkey/jsparagus" replace-with = "vendored-sources" diff --git a/Cargo.lock b/Cargo.lock index e68682321f5e..5b052c4e56b8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1700,6 +1700,7 @@ dependencies = [ "shift_or_euc_c", "static_prefs", "storage", + "sync15-traits", "unic-langid", "unic-langid-ffi", "webrender_bindings", @@ -4201,6 +4202,28 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "sync-guid" +version = "0.1.0" +source = "git+https://github.com/mozilla/application-services?rev=120e51dd5f2aab4194cf0f7e93b2a8923f4504bb#120e51dd5f2aab4194cf0f7e93b2a8923f4504bb" +dependencies = [ + "serde", +] + +[[package]] +name = "sync15-traits" +version = "0.1.0" +source = "git+https://github.com/mozilla/application-services?rev=120e51dd5f2aab4194cf0f7e93b2a8923f4504bb#120e51dd5f2aab4194cf0f7e93b2a8923f4504bb" +dependencies = [ + "failure", + "ffi-support", + "log", + "serde", + "serde_json", + "sync-guid", + "url", +] + [[package]] name = "synstructure" version = "0.12.1" diff --git a/third_party/rust/nom/.cargo-checksum.json b/third_party/rust/nom/.cargo-checksum.json index 262350f0269c..b236aa3bb55c 100644 --- a/third_party/rust/nom/.cargo-checksum.json +++ b/third_party/rust/nom/.cargo-checksum.json @@ -1 +1 @@ -{"files":{"CHANGELOG.md":"f4015ae48bb8f7c672c4fdf1ca4bdde5910f1b423ddc12e9fe80395911759d3e","Cargo.lock":"d7985e783bf7275009ab524f13aac37adc7291838a1956e1f092413ff3b8ea04","Cargo.toml":"72afa6607971579c7009dff34b6b334d4893b1a9f1983cd80364c5e318e4ec2b","LICENSE":"4dbda04344456f09a7a588140455413a9ac59b6b26a1ef7cdf9c800c012d87f0","build.rs":"fd66799ca3bd6a83b10f18a62e6ffc3b1ac94074fe65de4e4c1c447bf71d6ebb","src/bits/complete.rs":"8a60ae4cd6aaf32cb232b598b1a3dda7858c619944ba12ebdb01a31c75243293","src/bits/macros.rs":"8d9ba23f237b4fc01e3b2106002d0f6d59930a42f34d80662c61e0057dfb4a5b","src/bits/mod.rs":"4ca0148b4ef2de4e88796da7831eaa5c4fcbc5515a101eae1b0bc4853c80b5e7","src/bits/streaming.rs":"7c587808476edee57caeccca7dccd744bdfdbb13ae1156400fb4961980fa325d","src/branch/macros.rs":"8b885185725c16369d90954fed8450070bcd4bf8ae7de1df1bb46bb378450d71","src/branch/mod.rs":"d0c871ad7b74428ddccef445a10254249c3907528005a00275e8eb6516255f2f","src/bytes/complete.rs":"107f776885161e48a596962925d2b1628f20fd4bbe5b3777bb34ec175b97e280","src/bytes/macros.rs":"914a821761bbf49f04425068b7426bdd60394e8cc30e7c2193160b16e06c266f","src/bytes/mod.rs":"577231e6e6bd51726a877a73292d8f1c626f6b32ebe4a57943acaed8a8a2975d","src/bytes/streaming.rs":"4b2e577e6057fda932d1edc2ebe53c5df71a21417a304b35d5230dd7221411f3","src/character/complete.rs":"bb80656f3405eca79ba93c674cae7cd296733d784928e5c45754ee27963b7325","src/character/macros.rs":"f330ab60d469802b1664dcbbccd4bfe3e1ca87b348e92609ef34e25b4d475983","src/character/mod.rs":"9f520d535a88849726eac8648aa5c8b86193ab2f3a984d63c8371b846cc0b72c","src/character/streaming.rs":"9be7051951e8d0a54e294542baaf115aeb6efb818521f045386cd3a1777ca6a0","src/combinator/macros.rs":"df9ba1157bda21313a9c23826baeefd99f54e92c47d60c8dbb25697fe4d52686","src/combinator/mod.rs":"d1b2073683be1c9c4a06d1d3764ac789027ad7401ec726805d1505c1ad8ab1fd","src/error.rs":"ef7feb06b9689aa2f4b11a367b6f7058df8fd151b673c7482edd6600e55e38da","src/internal.rs":"c4029b0e32d41eb6407e517f3102a41a3a80a6a69df4767008ac5656b28d7ab0","src/lib.rs":"f01cdc23cc17201f978796d2d80fb6bba5a9b81ffb4653286e1e53f904079911","src/methods.rs":"56099c30123e92f9f6bacb16017f29fcdbc6546afbf0f80cf4951d2d8093ba83","src/multi/macros.rs":"01d15ae913921bd1ed0ff579a868ea39a314a866e6b5a365ef6b448a75f9b3a8","src/multi/mod.rs":"342c30e0c601558c867df215f7094bc9e43534a7793f2e8b19c114fe07cfea41","src/number/complete.rs":"560dfb2ffbbfe7fe276389b60eec2d717fec20eab68cc72d10d87ff6e2195352","src/number/macros.rs":"e614ee142658c126902a466f29ef250a77016fa126b8dfd59db0c6657a0ef205","src/number/mod.rs":"e432c317ee839a2f928cd0e8e583acadb165ed84fb649e681688a0fcd84b0a75","src/number/streaming.rs":"4c6fbce64e0d535f69d2c928314e1d0a019480db5a09e4f47a654a2e8fd56e8c","src/regexp.rs":"ac6fc61c2e7b629e6786638b44d63d15139c50f9d6a38acd044e4c2b3a699e08","src/sequence/macros.rs":"0e72871cdb2f1bf804f7f637def117c8e531f78cc7da5a6a0e343f0dbfb04271","src/sequence/mod.rs":"ec34b969462703252c4f00c27a03928c9580da612c71b7100e0902190a633ab9","src/str.rs":"fcae4d6f2f7bc921cafe3d0ce682d0566618cbe5f3b3e4b51ca34d11cb0e3e93","src/traits.rs":"2a84c3aa40f1cf78e0149e344da8b85486f7b6c0ddff8f23689bccce0def2260","src/util.rs":"bcedca3c88ac24f11f73e836efd8fe00014562163cc3d43d0cec9d726a4687c3","src/whitespace.rs":"53bddd9d559dc7793effcb828f9c196a7f098338edb0877940d1246834761308","tests/arithmetic.rs":"c57bc547110e498e7bddc973a94202f22356bc525fed55dac4f03daf16eb54f7","tests/arithmetic_ast.rs":"8fbc4c5d8850fa1cf0a16f97e8718d5427b2657f12ca4a0b3b6c1b47fd9e67d4","tests/blockbuf-arithmetic.rs":"099fdf75da97ae032006d8c82ea2207265c5161a08d1370e1ddddb01e73afaf4","tests/css.rs":"b13466eb6a0831f98ede83ffdd752ba821f7d03db724fd92b5bfbc0b9f804a65","tests/custom_errors.rs":"3d2511f8a8d0eb20d9efc19f29ae4ab34389bdd33214a421989d0c47b540b7fd","tests/escaped.rs":"03ecb10828472e4de2ace05a12cb49997b47a113b6a3b0eea3d56bc2bafd8446","tests/float.rs":"92947cc112a6b865f5e19d80edbf300ddc0d0ca4b4e4543eda15337b5c60eedf","tests/inference.rs":"fe476d1367fce9f0baf82295dc037c79321ededf12b9bcc0c5acdc7cefff4720","tests/ini.rs":"04ebf3ead0008974b3bedc387e889bab5942efd0c9c8563fe47e056a9b90bbab","tests/ini_str.rs":"2831a4ee26b37734dba8862cc2972a3a1433abf4fcab253b6cf4fb43f120301d","tests/issues.rs":"142c8d206089b04cf2fd0cbd90f87421ded435ce300516c029294b056354e00f","tests/json.rs":"25476ec2daca19295f5f99b621eecc859a5db5789ac35be381eaf8703a70bce8","tests/mp4.rs":"d0e61bfc93ff40676ca7e9d7813a5ad7c73b1db874599d8f3ea784115bfcab87","tests/multiline.rs":"6a5321cb53c7f88778fa100499533abfa602bada7a6b1d0fbba7ef77b9c110f5","tests/named_args.rs":"bd8095c3abc6fb806c9181c6025c0111d1e7f3b7269ea89ae122bf3bb8ed7e7d","tests/overflow.rs":"d1d6d8ce9b34ed47b42a5f7250ce711805a397691dc6cad3cc8945ec230da161","tests/reborrow_fold.rs":"9328deafc2143c2a2d1a0be86e2448b644cffcb5f0935c8b24eb469f1f9477c0","tests/test1.rs":"06fc9e52638f16bfc3ef69cd26b927e0cf55706d6f132ab7c0f1072208475853"},"package":"0b471253da97532da4b61552249c521e01e736071f71c1a4f7ebbfbf0a06aad6"} \ No newline at end of file +{"files":{".travis.yml":"124c5613de02fe2088a7410a7057b0e0ed49dcbc8509704b690de98e83b289e0","CHANGELOG.md":"f4015ae48bb8f7c672c4fdf1ca4bdde5910f1b423ddc12e9fe80395911759d3e","Cargo.lock":"d7985e783bf7275009ab524f13aac37adc7291838a1956e1f092413ff3b8ea04","Cargo.toml":"72afa6607971579c7009dff34b6b334d4893b1a9f1983cd80364c5e318e4ec2b","LICENSE":"4dbda04344456f09a7a588140455413a9ac59b6b26a1ef7cdf9c800c012d87f0","build.rs":"fd66799ca3bd6a83b10f18a62e6ffc3b1ac94074fe65de4e4c1c447bf71d6ebb","src/bits/complete.rs":"8a60ae4cd6aaf32cb232b598b1a3dda7858c619944ba12ebdb01a31c75243293","src/bits/macros.rs":"8d9ba23f237b4fc01e3b2106002d0f6d59930a42f34d80662c61e0057dfb4a5b","src/bits/mod.rs":"4ca0148b4ef2de4e88796da7831eaa5c4fcbc5515a101eae1b0bc4853c80b5e7","src/bits/streaming.rs":"7c587808476edee57caeccca7dccd744bdfdbb13ae1156400fb4961980fa325d","src/branch/macros.rs":"8b885185725c16369d90954fed8450070bcd4bf8ae7de1df1bb46bb378450d71","src/branch/mod.rs":"d0c871ad7b74428ddccef445a10254249c3907528005a00275e8eb6516255f2f","src/bytes/complete.rs":"107f776885161e48a596962925d2b1628f20fd4bbe5b3777bb34ec175b97e280","src/bytes/macros.rs":"914a821761bbf49f04425068b7426bdd60394e8cc30e7c2193160b16e06c266f","src/bytes/mod.rs":"577231e6e6bd51726a877a73292d8f1c626f6b32ebe4a57943acaed8a8a2975d","src/bytes/streaming.rs":"4b2e577e6057fda932d1edc2ebe53c5df71a21417a304b35d5230dd7221411f3","src/character/complete.rs":"bb80656f3405eca79ba93c674cae7cd296733d784928e5c45754ee27963b7325","src/character/macros.rs":"f330ab60d469802b1664dcbbccd4bfe3e1ca87b348e92609ef34e25b4d475983","src/character/mod.rs":"9f520d535a88849726eac8648aa5c8b86193ab2f3a984d63c8371b846cc0b72c","src/character/streaming.rs":"9be7051951e8d0a54e294542baaf115aeb6efb818521f045386cd3a1777ca6a0","src/combinator/macros.rs":"df9ba1157bda21313a9c23826baeefd99f54e92c47d60c8dbb25697fe4d52686","src/combinator/mod.rs":"d1b2073683be1c9c4a06d1d3764ac789027ad7401ec726805d1505c1ad8ab1fd","src/error.rs":"ef7feb06b9689aa2f4b11a367b6f7058df8fd151b673c7482edd6600e55e38da","src/internal.rs":"c4029b0e32d41eb6407e517f3102a41a3a80a6a69df4767008ac5656b28d7ab0","src/lib.rs":"f01cdc23cc17201f978796d2d80fb6bba5a9b81ffb4653286e1e53f904079911","src/methods.rs":"56099c30123e92f9f6bacb16017f29fcdbc6546afbf0f80cf4951d2d8093ba83","src/multi/macros.rs":"01d15ae913921bd1ed0ff579a868ea39a314a866e6b5a365ef6b448a75f9b3a8","src/multi/mod.rs":"342c30e0c601558c867df215f7094bc9e43534a7793f2e8b19c114fe07cfea41","src/number/complete.rs":"560dfb2ffbbfe7fe276389b60eec2d717fec20eab68cc72d10d87ff6e2195352","src/number/macros.rs":"e614ee142658c126902a466f29ef250a77016fa126b8dfd59db0c6657a0ef205","src/number/mod.rs":"e432c317ee839a2f928cd0e8e583acadb165ed84fb649e681688a0fcd84b0a75","src/number/streaming.rs":"4c6fbce64e0d535f69d2c928314e1d0a019480db5a09e4f47a654a2e8fd56e8c","src/regexp.rs":"ac6fc61c2e7b629e6786638b44d63d15139c50f9d6a38acd044e4c2b3a699e08","src/sequence/macros.rs":"0e72871cdb2f1bf804f7f637def117c8e531f78cc7da5a6a0e343f0dbfb04271","src/sequence/mod.rs":"ec34b969462703252c4f00c27a03928c9580da612c71b7100e0902190a633ab9","src/str.rs":"fcae4d6f2f7bc921cafe3d0ce682d0566618cbe5f3b3e4b51ca34d11cb0e3e93","src/traits.rs":"2a84c3aa40f1cf78e0149e344da8b85486f7b6c0ddff8f23689bccce0def2260","src/util.rs":"bcedca3c88ac24f11f73e836efd8fe00014562163cc3d43d0cec9d726a4687c3","src/whitespace.rs":"53bddd9d559dc7793effcb828f9c196a7f098338edb0877940d1246834761308","tests/arithmetic.rs":"c57bc547110e498e7bddc973a94202f22356bc525fed55dac4f03daf16eb54f7","tests/arithmetic_ast.rs":"8fbc4c5d8850fa1cf0a16f97e8718d5427b2657f12ca4a0b3b6c1b47fd9e67d4","tests/blockbuf-arithmetic.rs":"099fdf75da97ae032006d8c82ea2207265c5161a08d1370e1ddddb01e73afaf4","tests/css.rs":"b13466eb6a0831f98ede83ffdd752ba821f7d03db724fd92b5bfbc0b9f804a65","tests/custom_errors.rs":"3d2511f8a8d0eb20d9efc19f29ae4ab34389bdd33214a421989d0c47b540b7fd","tests/escaped.rs":"03ecb10828472e4de2ace05a12cb49997b47a113b6a3b0eea3d56bc2bafd8446","tests/float.rs":"92947cc112a6b865f5e19d80edbf300ddc0d0ca4b4e4543eda15337b5c60eedf","tests/inference.rs":"fe476d1367fce9f0baf82295dc037c79321ededf12b9bcc0c5acdc7cefff4720","tests/ini.rs":"04ebf3ead0008974b3bedc387e889bab5942efd0c9c8563fe47e056a9b90bbab","tests/ini_str.rs":"2831a4ee26b37734dba8862cc2972a3a1433abf4fcab253b6cf4fb43f120301d","tests/issues.rs":"142c8d206089b04cf2fd0cbd90f87421ded435ce300516c029294b056354e00f","tests/json.rs":"25476ec2daca19295f5f99b621eecc859a5db5789ac35be381eaf8703a70bce8","tests/mp4.rs":"d0e61bfc93ff40676ca7e9d7813a5ad7c73b1db874599d8f3ea784115bfcab87","tests/multiline.rs":"6a5321cb53c7f88778fa100499533abfa602bada7a6b1d0fbba7ef77b9c110f5","tests/named_args.rs":"bd8095c3abc6fb806c9181c6025c0111d1e7f3b7269ea89ae122bf3bb8ed7e7d","tests/overflow.rs":"d1d6d8ce9b34ed47b42a5f7250ce711805a397691dc6cad3cc8945ec230da161","tests/reborrow_fold.rs":"9328deafc2143c2a2d1a0be86e2448b644cffcb5f0935c8b24eb469f1f9477c0","tests/test1.rs":"06fc9e52638f16bfc3ef69cd26b927e0cf55706d6f132ab7c0f1072208475853"},"package":"0b471253da97532da4b61552249c521e01e736071f71c1a4f7ebbfbf0a06aad6"} \ No newline at end of file diff --git a/third_party/rust/nom/.travis.yml b/third_party/rust/nom/.travis.yml new file mode 100644 index 000000000000..e18b14079ddd --- /dev/null +++ b/third_party/rust/nom/.travis.yml @@ -0,0 +1,101 @@ +language: rust +# sudo is required to enable kcov to use the personality syscall +sudo: required +dist: trusty +cache: cargo + +rust: + - nightly + - beta + - stable + - 1.31.0 + +env: + matrix: + - FEATURES='--features "regexp regexp_macros"' + +before_script: + - eval git pull --rebase https://github.com/Geal/nom master + - eval git log --pretty=oneline HEAD~5..HEAD + +matrix: + include: + - rust: nightly + env: FEATURES='--no-default-features' + - rust: nightly + env: FEATURES='--no-default-features --features "alloc"' + - rust: stable + env: FEATURES='' + - rust: nightly + env: DOC_FEATURES='--features "std lexical regexp regexp_macros" --no-default-features' + before_script: + - export PATH=$HOME/.cargo/bin:$PATH + script: + - eval cargo doc --verbose $DOC_FEATURES + - rust: nightly + env: FEATURES='' + before_script: + - export PATH=$HOME/.cargo/bin:$PATH + - cargo install cargo-update || echo "cargo-update already installed" + - cargo install cargo-travis || echo "cargo-travis already installed" + - cargo install-update -a + - mkdir -p target/kcov-master + script: + cargo coveralls --verbose --all-features + allow_failures: + - rust: stable + env: FEATURES='' + before_script: + - export PATH=$HOME/.cargo/bin:$PATH + - rustup component add rustfmt-preview + script: + - eval cargo fmt -- --write-mode=diff + +notifications: + webhooks: + urls: + - https://webhooks.gitter.im/e/9c035a194ac4fd4cc061 + on_success: change + on_failure: always + on_start: false + + +addons: + apt: + packages: + - libcurl4-openssl-dev + - libelf-dev + - libdw-dev + - binutils-dev + - cmake + sources: + - kalakris-cmake + +cache: + directories: + - /home/travis/.cargo + +before_cache: + - rm -rf /home/travis/.cargo/registry + +script: + - eval cargo build --verbose $FEATURES + - eval cargo test --verbose $FEATURES + +after_success: | + case "$TRAVIS_RUST_VERSION" in + nightly) + if [ "${TRAVIS_PULL_REQUEST_BRANCH:-$TRAVIS_BRANCH}" != "master" ]; then + git fetch && + git checkout master && + cargo bench --verbose + fi + + if [ "$FEATURES" == '--features "regexp regexp_macros"' ]; then + cargo bench --verbose + fi + ;; + + *) + ;; + esac diff --git a/third_party/rust/sync-guid/.cargo-checksum.json b/third_party/rust/sync-guid/.cargo-checksum.json new file mode 100644 index 000000000000..6123bab2b13d --- /dev/null +++ b/third_party/rust/sync-guid/.cargo-checksum.json @@ -0,0 +1 @@ +{"files":{"Cargo.toml":"b5cc525d2aa129f84cb3f729a579217591c7705e2be78dbd348a95fc354831be","src/lib.rs":"729e562be4e63ec7db2adc00753a019ae77c11ce82637a893ea18122580c3c98","src/rusqlite_support.rs":"827d314605d8c741efdf238a0780a891c88bc56026a3e6dcfa534772a4852fb3","src/serde_support.rs":"519b5eb59ca7be555d522f2186909db969069dc9586a5fe4047d4ec176b2368a"},"package":null} \ No newline at end of file diff --git a/third_party/rust/sync-guid/Cargo.toml b/third_party/rust/sync-guid/Cargo.toml new file mode 100644 index 000000000000..ca885a95c4c9 --- /dev/null +++ b/third_party/rust/sync-guid/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "sync-guid" +version = "0.1.0" +authors = ["Thom Chiovoloni "] +license = "MPL-2.0" +edition = "2018" + +[dependencies] +rusqlite = { version = "0.21.0", optional = true } +serde = { version = "1.0.104", optional = true } +rand = { version = "0.7", optional = true } +base64 = { version = "0.12.0", optional = true } + +[features] +random = ["rand", "base64"] +rusqlite_support = ["rusqlite"] +serde_support = ["serde"] +# By default we support serde, but not rusqlite. +default = ["serde_support"] + +[dev-dependencies] +serde_test = "1.0.104" diff --git a/third_party/rust/sync-guid/src/lib.rs b/third_party/rust/sync-guid/src/lib.rs new file mode 100644 index 000000000000..87c62758eb57 --- /dev/null +++ b/third_party/rust/sync-guid/src/lib.rs @@ -0,0 +1,466 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#![allow(unknown_lints)] +#![warn(rust_2018_idioms)] +// (It's tempting to avoid the utf8 checks, but they're easy to get wrong, so) +#![deny(unsafe_code)] +#[cfg(feature = "serde_support")] +mod serde_support; + +#[cfg(feature = "rusqlite_support")] +mod rusqlite_support; + +use std::{ + cmp::Ordering, + fmt, + hash::{Hash, Hasher}, + ops, str, +}; + +/// This is a type intended to be used to represent the guids used by sync. It +/// has several benefits over using a `String`: +/// +/// 1. It's more explicit about what is being stored, and could prevent bugs +/// where a Guid is passed to a function expecting text. +/// +/// 2. Guids are guaranteed to be immutable. +/// +/// 3. It's optimized for the guids commonly used by sync. In particular, short guids +/// (including the guids which would meet `PlacesUtils.isValidGuid`) do not incur +/// any heap allocation, and are stored inline. +#[derive(Clone)] +pub struct Guid(Repr); + +// The internal representation of a GUID. Most Sync GUIDs are 12 bytes, +// and contain only base64url characters; we can store them on the stack +// without a heap allocation. However, arbitrary ascii guids of up to length 64 +// are possible, in which case we fall back to a heap-allocated string. +// +// This is separate only because making `Guid` an enum would expose the +// internals. +#[derive(Clone)] +enum Repr { + // see FastGuid for invariants + Fast(FastGuid), + + // invariants: + // - _0.len() > MAX_FAST_GUID_LEN + Slow(String), +} + +/// Invariants: +/// +/// - `len <= MAX_FAST_GUID_LEN`. +/// - `data[0..len]` encodes valid utf8. +/// - `data[len..].iter().all(|&b| b == b'\0')` +/// +/// Note: None of these are required for memory safety, just correctness. +#[derive(Clone)] +struct FastGuid { + len: u8, + data: [u8; MAX_FAST_GUID_LEN], +} + +// This is the maximum length (experimentally determined) we can make it before +// `Repr::Fast` is larger than `Guid::Slow` on 32 bit systems. The important +// thing is really that it's not too big, and is above 12 bytes. +const MAX_FAST_GUID_LEN: usize = 14; + +impl FastGuid { + #[inline] + fn from_slice(bytes: &[u8]) -> Self { + // Checked by the caller, so debug_assert is fine. + debug_assert!( + can_use_fast(bytes), + "Bug: Caller failed to check can_use_fast: {:?}", + bytes + ); + let mut data = [0u8; MAX_FAST_GUID_LEN]; + data[0..bytes.len()].copy_from_slice(bytes); + FastGuid { + len: bytes.len() as u8, + data, + } + } + + #[inline] + fn as_str(&self) -> &str { + // Note: we only use debug_assert! to enusre valid utf8-ness, so this need + str::from_utf8(self.bytes()).expect("Invalid fast guid bytes!") + } + + #[inline] + fn len(&self) -> usize { + self.len as usize + } + + #[inline] + fn bytes(&self) -> &[u8] { + &self.data[0..self.len()] + } +} + +// Returns: +// - true to use Repr::Fast +// - false to use Repr::Slow +#[inline] +fn can_use_fast>(bytes: &T) -> bool { + let bytes = bytes.as_ref(); + // This is fine as a debug_assert since we'll still panic if it's ever used + // in such a way where it would matter. + debug_assert!(str::from_utf8(bytes).is_ok()); + bytes.len() <= MAX_FAST_GUID_LEN +} + +impl Guid { + /// Create a guid from a `str`. + #[inline] + pub fn new(s: &str) -> Self { + Guid::from_slice(s.as_ref()) + } + + /// Create an empty guid. Usable as a constant. + #[inline] + pub const fn empty() -> Self { + Guid(Repr::Fast(FastGuid { + len: 0, + data: [0u8; MAX_FAST_GUID_LEN], + })) + } + + /// Create a random guid (of 12 base64url characters). Requires the `random` + /// feature. + #[cfg(feature = "random")] + pub fn random() -> Self { + let bytes: [u8; 9] = rand::random(); + + // Note: only first 12 bytes are used, but remaining are required to + // build the FastGuid + let mut output = [0u8; MAX_FAST_GUID_LEN]; + + let bytes_written = + base64::encode_config_slice(&bytes, base64::URL_SAFE_NO_PAD, &mut output[..12]); + + debug_assert!(bytes_written == 12); + + Guid(Repr::Fast(FastGuid { + len: 12, + data: output, + })) + } + + /// Convert `b` into a `Guid`. + #[inline] + pub fn from_string(s: String) -> Self { + Guid::from_vec(s.into_bytes()) + } + + /// Convert `b` into a `Guid`. + #[inline] + pub fn from_slice(b: &[u8]) -> Self { + if can_use_fast(b) { + Guid(Repr::Fast(FastGuid::from_slice(b))) + } else { + Guid::new_slow(b.into()) + } + } + + /// Convert `v` to a `Guid`, consuming it. + #[inline] + pub fn from_vec(v: Vec) -> Self { + if can_use_fast(&v) { + Guid(Repr::Fast(FastGuid::from_slice(&v))) + } else { + Guid::new_slow(v) + } + } + + /// Get the data backing this `Guid` as a `&[u8]`. + #[inline] + pub fn as_bytes(&self) -> &[u8] { + match &self.0 { + Repr::Fast(rep) => rep.bytes(), + Repr::Slow(rep) => rep.as_ref(), + } + } + + /// Get the data backing this `Guid` as a `&str`. + #[inline] + pub fn as_str(&self) -> &str { + match &self.0 { + Repr::Fast(rep) => rep.as_str(), + Repr::Slow(rep) => rep.as_ref(), + } + } + + /// Convert this `Guid` into a `String`, consuming it in the process. + #[inline] + pub fn into_string(self) -> String { + match self.0 { + Repr::Fast(rep) => rep.as_str().into(), + Repr::Slow(rep) => rep, + } + } + + /// Returns true for Guids that are deemed valid by the sync server. + /// See https://github.com/mozilla-services/server-syncstorage/blob/d92ef07877aebd05b92f87f6ade341d6a55bffc8/syncstorage/bso.py#L24 + pub fn is_valid_for_sync_server(&self) -> bool { + !self.is_empty() + && self.len() <= 64 + && self.bytes().all(|b| b >= b' ' && b <= b'~' && b != b',') + } + + /// Returns true for Guids that are valid places guids, and false for all others. + pub fn is_valid_for_places(&self) -> bool { + self.len() == 12 && self.bytes().all(Guid::is_valid_places_byte) + } + + /// Returns true if the byte `b` is a valid base64url byte. + #[inline] + pub fn is_valid_places_byte(b: u8) -> bool { + BASE64URL_BYTES[b as usize] == 1 + } + + #[cold] + fn new_slow(v: Vec) -> Self { + assert!( + !can_use_fast(&v), + "Could use fast for guid (len = {})", + v.len() + ); + Guid(Repr::Slow( + String::from_utf8(v).expect("Invalid slow guid bytes!"), + )) + } +} + +// This is used to implement the places tests. +const BASE64URL_BYTES: [u8; 256] = [ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +]; + +impl Ord for Guid { + fn cmp(&self, other: &Self) -> Ordering { + self.as_bytes().cmp(&other.as_bytes()) + } +} + +impl PartialOrd for Guid { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl PartialEq for Guid { + fn eq(&self, other: &Self) -> bool { + self.as_bytes() == other.as_bytes() + } +} + +impl Eq for Guid {} + +impl Hash for Guid { + fn hash(&self, state: &mut H) { + self.as_bytes().hash(state); + } +} + +impl<'a> From<&'a str> for Guid { + #[inline] + fn from(s: &'a str) -> Guid { + Guid::from_slice(s.as_ref()) + } +} +impl<'a> From<&'a &str> for Guid { + #[inline] + fn from(s: &'a &str) -> Guid { + Guid::from_slice(s.as_ref()) + } +} + +impl<'a> From<&'a [u8]> for Guid { + #[inline] + fn from(s: &'a [u8]) -> Guid { + Guid::from_slice(s) + } +} + +impl From for Guid { + #[inline] + fn from(s: String) -> Guid { + Guid::from_string(s) + } +} + +impl From> for Guid { + #[inline] + fn from(v: Vec) -> Guid { + Guid::from_vec(v) + } +} + +impl From for String { + #[inline] + fn from(guid: Guid) -> String { + guid.into_string() + } +} + +impl From for Vec { + #[inline] + fn from(guid: Guid) -> Vec { + guid.into_string().into_bytes() + } +} + +impl AsRef for Guid { + #[inline] + fn as_ref(&self) -> &str { + self.as_str() + } +} + +impl AsRef<[u8]> for Guid { + #[inline] + fn as_ref(&self) -> &[u8] { + self.as_bytes() + } +} + +impl ops::Deref for Guid { + type Target = str; + #[inline] + fn deref(&self) -> &str { + self.as_str() + } +} + +// The default Debug impl is pretty unhelpful here. +impl fmt::Debug for Guid { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "Guid({:?})", self.as_str()) + } +} + +impl fmt::Display for Guid { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self.as_str(), f) + } +} + +impl std::default::Default for Guid { + /// Create a default guid by calling `Guid::empty()` + #[inline] + fn default() -> Self { + Guid::empty() + } +} + +macro_rules! impl_guid_eq { + ($($other: ty),+) => {$( + impl<'a> PartialEq<$other> for Guid { + #[inline] + fn eq(&self, other: &$other) -> bool { + PartialEq::eq(AsRef::<[u8]>::as_ref(self), AsRef::<[u8]>::as_ref(other)) + } + } + + impl<'a> PartialEq for $other { + #[inline] + fn eq(&self, other: &Guid) -> bool { + PartialEq::eq(AsRef::<[u8]>::as_ref(self), AsRef::<[u8]>::as_ref(other)) + } + } + )+} +} + +// Implement direct comparison with some common types from the stdlib. +impl_guid_eq![str, &'a str, String, [u8], &'a [u8], Vec]; + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_base64url_bytes() { + let mut expect = [0u8; 256]; + for b in b'0'..=b'9' { + expect[b as usize] = 1; + } + for b in b'a'..=b'z' { + expect[b as usize] = 1; + } + for b in b'A'..=b'Z' { + expect[b as usize] = 1; + } + expect[b'_' as usize] = 1; + expect[b'-' as usize] = 1; + assert_eq!(&BASE64URL_BYTES[..], &expect[..]); + } + + #[test] + fn test_valid_for_places() { + assert!(Guid::from("aaaabbbbcccc").is_valid_for_places()); + assert!(Guid::from_slice(b"09_az-AZ_09-").is_valid_for_places()); + assert!(!Guid::from("aaaabbbbccccd").is_valid_for_places()); // too long + assert!(!Guid::from("aaaabbbbccc").is_valid_for_places()); // too short + assert!(!Guid::from("aaaabbbbccc=").is_valid_for_places()); // right length, bad character + } + + #[test] + fn test_comparison() { + assert_eq!(Guid::from("abcdabcdabcd"), "abcdabcdabcd"); + assert_ne!(Guid::from("abcdabcdabcd".to_string()), "ABCDabcdabcd"); + + assert_eq!(Guid::from("abcdabcdabcd"), &b"abcdabcdabcd"[..]); // b"abcdabcdabcd" has type &[u8; 12]... + assert_ne!(Guid::from(&b"abcdabcdabcd"[..]), &b"ABCDabcdabcd"[..]); + + assert_eq!( + Guid::from(b"abcdabcdabcd"[..].to_owned()), + "abcdabcdabcd".to_string() + ); + assert_ne!(Guid::from("abcdabcdabcd"), "ABCDabcdabcd".to_string()); + + assert_eq!( + Guid::from("abcdabcdabcd1234"), + Vec::from(b"abcdabcdabcd1234".as_ref()) + ); + assert_ne!( + Guid::from("abcdabcdabcd4321"), + Vec::from(b"ABCDabcdabcd4321".as_ref()) + ); + + // order by data instead of length + assert!(Guid::from("zzz") > Guid::from("aaaaaa")); + assert!(Guid::from("ThisIsASolowGuid") < Guid::from("zzz")); + assert!(Guid::from("ThisIsASolowGuid") > Guid::from("AnotherSlowGuid")); + } + + #[cfg(feature = "random")] + #[test] + fn test_random() { + use std::collections::HashSet; + // Used to verify uniqueness within our sample of 1000. Could cause + // random failures, but desktop has the same test, and it's never caused + // a problem AFAIK. + let mut seen: HashSet = HashSet::new(); + for _ in 0..1000 { + let g = Guid::random(); + assert_eq!(g.len(), 12); + assert!(g.is_valid_for_places()); + let decoded = base64::decode_config(&g, base64::URL_SAFE_NO_PAD).unwrap(); + assert_eq!(decoded.len(), 9); + let no_collision = seen.insert(g.clone().into_string()); + assert!(no_collision, "{}", g); + } + } +} diff --git a/third_party/rust/sync-guid/src/rusqlite_support.rs b/third_party/rust/sync-guid/src/rusqlite_support.rs new file mode 100644 index 000000000000..c4c0f2f0428c --- /dev/null +++ b/third_party/rust/sync-guid/src/rusqlite_support.rs @@ -0,0 +1,23 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#![cfg(feature = "rusqlite_support")] + +use crate::Guid; +use rusqlite::{ + self, + types::{FromSql, FromSqlResult, ToSql, ToSqlOutput, ValueRef}, +}; + +impl ToSql for Guid { + fn to_sql(&self) -> rusqlite::Result> { + Ok(ToSqlOutput::from(self.as_str())) + } +} + +impl FromSql for Guid { + fn column_result(value: ValueRef<'_>) -> FromSqlResult { + value.as_str().map(Guid::from) + } +} diff --git a/third_party/rust/sync-guid/src/serde_support.rs b/third_party/rust/sync-guid/src/serde_support.rs new file mode 100644 index 000000000000..50220ffe1285 --- /dev/null +++ b/third_party/rust/sync-guid/src/serde_support.rs @@ -0,0 +1,61 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#![cfg(feature = "serde_support")] + +use std::fmt; + +use serde::{ + de::{self, Deserialize, Deserializer, Visitor}, + ser::{Serialize, Serializer}, +}; + +use crate::Guid; + +struct GuidVisitor; +impl<'de> Visitor<'de> for GuidVisitor { + type Value = Guid; + #[inline] + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("a sync guid") + } + #[inline] + fn visit_str(self, s: &str) -> Result { + Ok(Guid::from_slice(s.as_ref())) + } +} + +impl<'de> Deserialize<'de> for Guid { + #[inline] + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + deserializer.deserialize_str(GuidVisitor) + } +} + +impl Serialize for Guid { + #[inline] + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_str(self.as_str()) + } +} + +#[cfg(test)] +mod test { + use super::*; + use serde_test::{assert_tokens, Token}; + #[test] + fn test_ser_de() { + let guid = Guid::from("asdffdsa12344321"); + assert_tokens(&guid, &[Token::Str("asdffdsa12344321")]); + + let guid = Guid::from(""); + assert_tokens(&guid, &[Token::Str("")]); + + let guid = Guid::from(&b"abcd43211234"[..]); + assert_tokens(&guid, &[Token::Str("abcd43211234")]); + } +} diff --git a/third_party/rust/sync15-traits/.cargo-checksum.json b/third_party/rust/sync15-traits/.cargo-checksum.json new file mode 100644 index 000000000000..e90493eb8b84 --- /dev/null +++ b/third_party/rust/sync15-traits/.cargo-checksum.json @@ -0,0 +1 @@ +{"files":{"Cargo.toml":"326b1c017a76b1987e34c6dde0fa57f2c85d5de23a9f0cf1dfb029cc99d34471","README.md":"396105211d8ce7f40b05d8062d7ab55d99674555f3ac81c061874ae26656ed7e","src/changeset.rs":"442aa92b5130ec0f8f2b0054acb399c547380e0060015cbf4ca7a72027440d54","src/client.rs":"6be4f550ade823fafc350c5490e031f90a4af833a9bba9739b05568464255a74","src/lib.rs":"9abce82e0248c8aa7e3d55b7db701b95e8f337f6e5d1319381f995a0b708400d","src/payload.rs":"09db1a444e7893990a4f03cb16263b9c15abc9e48ec4f1343227be1b490865a5","src/request.rs":"9e656ec487e53c7485643687e605d73bb25e138056e920d6f4b7d63fc6a8c460","src/server_timestamp.rs":"43d1b98a90e55e49380a0b66c209c9eb393e2aeaa27d843a4726d93cdd4cea02","src/store.rs":"10e215dd24270b6bec10903ac1d5274ce997eb437134f43be7de44e36fb9d1e4","src/telemetry.rs":"027befb099a6fcded3457f7e566296548a0898ff613267190621856b9ef288f6"},"package":null} \ No newline at end of file diff --git a/third_party/rust/sync15-traits/Cargo.toml b/third_party/rust/sync15-traits/Cargo.toml new file mode 100644 index 000000000000..7aa6503f8b0d --- /dev/null +++ b/third_party/rust/sync15-traits/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "sync15-traits" +version = "0.1.0" +authors = ["Thom Chiovoloni "] +license = "MPL-2.0" +edition = "2018" + +[features] +random-guid = ["sync-guid/random"] + +[dependencies] +sync-guid = { path = "../guid" } +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +log = "0.4" +ffi-support = "0.4" +url = "2.1" +failure = "0.1.6" diff --git a/third_party/rust/sync15-traits/README.md b/third_party/rust/sync15-traits/README.md new file mode 100644 index 000000000000..893abcf58707 --- /dev/null +++ b/third_party/rust/sync15-traits/README.md @@ -0,0 +1,4 @@ +# sync15-traits + +Extracted types and traits from sync15. Usable for cases where depending on the +sync15 crate is impossible (like in remerge). diff --git a/third_party/rust/sync15-traits/src/changeset.rs b/third_party/rust/sync15-traits/src/changeset.rs new file mode 100644 index 000000000000..36d8f5b833d1 --- /dev/null +++ b/third_party/rust/sync15-traits/src/changeset.rs @@ -0,0 +1,33 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use crate::{Payload, ServerTimestamp}; + +#[derive(Debug, Clone)] +pub struct RecordChangeset

{ + pub changes: Vec

, + /// For GETs, the last sync timestamp that should be persisted after + /// applying the records. + /// For POSTs, this is the XIUS timestamp. + pub timestamp: ServerTimestamp, + pub collection: std::borrow::Cow<'static, str>, +} + +pub type IncomingChangeset = RecordChangeset<(Payload, ServerTimestamp)>; +pub type OutgoingChangeset = RecordChangeset; + +// TODO: use a trait to unify this with the non-json versions +impl RecordChangeset { + #[inline] + pub fn new( + collection: impl Into>, + timestamp: ServerTimestamp, + ) -> RecordChangeset { + RecordChangeset { + changes: vec![], + timestamp, + collection: collection.into(), + } + } +} diff --git a/third_party/rust/sync15-traits/src/client.rs b/third_party/rust/sync15-traits/src/client.rs new file mode 100644 index 000000000000..f08f07931925 --- /dev/null +++ b/third_party/rust/sync15-traits/src/client.rs @@ -0,0 +1,58 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +//! This module has to be here because of some hard-to-avoid hacks done for the +//! tabs engine... See issue #2590 + +use std::collections::HashMap; + +/// Argument to Store::prepare_for_sync. See comment there for more info. Only +/// really intended to be used by tabs engine. +#[derive(Clone, Debug)] +pub struct ClientData { + pub local_client_id: String, + pub recent_clients: HashMap, +} + +/// Information about a remote client in the clients collection. +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +pub struct RemoteClient { + pub fxa_device_id: Option, + pub device_name: String, + pub device_type: Option, +} + +/// The type of a client. Please keep these variants in sync with the device +/// types in the FxA client and sync manager. +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +pub enum DeviceType { + Desktop, + Mobile, + Tablet, + VR, + TV, +} + +impl DeviceType { + pub fn try_from_str(d: impl AsRef) -> Option { + match d.as_ref() { + "desktop" => Some(DeviceType::Desktop), + "mobile" => Some(DeviceType::Mobile), + "tablet" => Some(DeviceType::Tablet), + "vr" => Some(DeviceType::VR), + "tv" => Some(DeviceType::TV), + _ => None, + } + } + + pub fn as_str(self) -> &'static str { + match self { + DeviceType::Desktop => "desktop", + DeviceType::Mobile => "mobile", + DeviceType::Tablet => "tablet", + DeviceType::VR => "vr", + DeviceType::TV => "tv", + } + } +} diff --git a/third_party/rust/sync15-traits/src/lib.rs b/third_party/rust/sync15-traits/src/lib.rs new file mode 100644 index 000000000000..81858afdd2d2 --- /dev/null +++ b/third_party/rust/sync15-traits/src/lib.rs @@ -0,0 +1,24 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#![warn(rust_2018_idioms)] +mod changeset; +pub mod client; +mod payload; +pub mod request; +mod server_timestamp; +mod store; +pub mod telemetry; + +pub use changeset::{IncomingChangeset, OutgoingChangeset, RecordChangeset}; +pub use payload::Payload; +pub use request::{CollectionRequest, RequestOrder}; +pub use server_timestamp::ServerTimestamp; +pub use store::{CollSyncIds, Store, StoreSyncAssociation}; +pub use sync_guid::Guid; + +// For skip_serializing_if +pub(crate) fn skip_if_default(v: &T) -> bool { + *v == T::default() +} diff --git a/third_party/rust/sync15-traits/src/payload.rs b/third_party/rust/sync15-traits/src/payload.rs new file mode 100644 index 000000000000..d84364477d64 --- /dev/null +++ b/third_party/rust/sync15-traits/src/payload.rs @@ -0,0 +1,91 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +use super::Guid; +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value as JsonValue}; + +/// Represents the decrypted payload in a Bso. Provides a minimal layer of type +/// safety to avoid double-encrypting. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct Payload { + pub id: Guid, + + #[serde(default)] + #[serde(skip_serializing_if = "crate::skip_if_default")] + pub deleted: bool, + + #[serde(flatten)] + pub data: Map, +} + +impl Payload { + pub fn new_tombstone(id: impl Into) -> Payload { + Payload { + id: id.into(), + deleted: true, + data: Map::new(), + } + } + + pub fn new_tombstone_with_ttl(id: impl Into, ttl: u32) -> Payload { + let mut result = Payload::new_tombstone(id); + result.data.insert("ttl".into(), ttl.into()); + result + } + + #[inline] + pub fn with_sortindex(mut self, index: i32) -> Payload { + self.data.insert("sortindex".into(), index.into()); + self + } + + #[inline] + pub fn id(&self) -> &str { + &self.id[..] + } + + #[inline] + pub fn is_tombstone(&self) -> bool { + self.deleted + } + + pub fn from_json(value: JsonValue) -> Result { + serde_json::from_value(value) + } + + pub fn into_record(self) -> Result + where + for<'a> T: Deserialize<'a>, + { + serde_json::from_value(JsonValue::from(self)) + } + + pub fn from_record(v: T) -> Result { + // TODO(issue #2588): This is kind of dumb, we do to_value and then + // from_value. In general a more strongly typed API would help us avoid + // this sort of thing... But also concretely this could probably be + // avoided? At least in some cases. + Ok(Payload::from_json(serde_json::to_value(v)?)?) + } + + pub fn into_json_string(self) -> String { + serde_json::to_string(&JsonValue::from(self)) + .expect("JSON.stringify failed, which shouldn't be possible") + } +} + +impl From for JsonValue { + fn from(cleartext: Payload) -> Self { + let Payload { + mut data, + id, + deleted, + } = cleartext; + data.insert("id".to_string(), JsonValue::String(id.into_string())); + if deleted { + data.insert("deleted".to_string(), JsonValue::Bool(true)); + } + JsonValue::Object(data) + } +} diff --git a/third_party/rust/sync15-traits/src/request.rs b/third_party/rust/sync15-traits/src/request.rs new file mode 100644 index 000000000000..1e51cf1e62ac --- /dev/null +++ b/third_party/rust/sync15-traits/src/request.rs @@ -0,0 +1,175 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +use crate::{Guid, ServerTimestamp}; +use std::borrow::Cow; +use url::{form_urlencoded as form, Url, UrlQuery}; +#[derive(Debug, Clone, PartialEq)] +pub struct CollectionRequest { + pub collection: Cow<'static, str>, + pub full: bool, + pub ids: Option>, + pub limit: usize, + pub older: Option, + pub newer: Option, + pub order: Option, + pub commit: bool, + pub batch: Option, +} + +impl CollectionRequest { + #[inline] + pub fn new(collection: S) -> CollectionRequest + where + S: Into>, + { + CollectionRequest { + collection: collection.into(), + full: false, + ids: None, + limit: 0, + older: None, + newer: None, + order: None, + commit: false, + batch: None, + } + } + + #[inline] + pub fn ids(mut self, v: V) -> CollectionRequest + where + V: IntoIterator, + V::Item: Into, + { + self.ids = Some(v.into_iter().map(|id| id.into()).collect()); + self + } + + #[inline] + pub fn full(mut self) -> CollectionRequest { + self.full = true; + self + } + + #[inline] + pub fn older_than(mut self, ts: ServerTimestamp) -> CollectionRequest { + self.older = Some(ts); + self + } + + #[inline] + pub fn newer_than(mut self, ts: ServerTimestamp) -> CollectionRequest { + self.newer = Some(ts); + self + } + + #[inline] + pub fn sort_by(mut self, order: RequestOrder) -> CollectionRequest { + self.order = Some(order); + self + } + + #[inline] + pub fn limit(mut self, num: usize) -> CollectionRequest { + self.limit = num; + self + } + + #[inline] + pub fn batch(mut self, batch: Option) -> CollectionRequest { + self.batch = batch; + self + } + + #[inline] + pub fn commit(mut self, v: bool) -> CollectionRequest { + self.commit = v; + self + } + + fn build_query(&self, pairs: &mut form::Serializer<'_, UrlQuery<'_>>) { + if self.full { + pairs.append_pair("full", "1"); + } + if self.limit > 0 { + pairs.append_pair("limit", &self.limit.to_string()); + } + if let Some(ids) = &self.ids { + // Most ids are 12 characters, and we comma separate them, so 13. + let mut buf = String::with_capacity(ids.len() * 13); + for (i, id) in ids.iter().enumerate() { + if i > 0 { + buf.push(','); + } + buf.push_str(id.as_str()); + } + pairs.append_pair("ids", &buf); + } + if let Some(batch) = &self.batch { + pairs.append_pair("batch", &batch); + } + if self.commit { + pairs.append_pair("commit", "true"); + } + if let Some(ts) = self.older { + pairs.append_pair("older", &ts.to_string()); + } + if let Some(ts) = self.newer { + pairs.append_pair("newer", &ts.to_string()); + } + if let Some(o) = self.order { + pairs.append_pair("sort", o.as_str()); + } + pairs.finish(); + } + + pub fn build_url(&self, mut base_url: Url) -> Result { + base_url + .path_segments_mut() + .map_err(|_| UnacceptableBaseUrl(()))? + .extend(&["storage", &self.collection]); + self.build_query(&mut base_url.query_pairs_mut()); + // This is strange but just accessing query_pairs_mut makes you have + // a trailing question mark on your url. I don't think anything bad + // would happen here, but I don't know, and also, it looks dumb so + // I'd rather not have it. + if base_url.query() == Some("") { + base_url.set_query(None); + } + Ok(base_url) + } +} +#[derive(Debug)] +pub struct UnacceptableBaseUrl(()); + +impl std::fmt::Display for UnacceptableBaseUrl { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("Storage server URL is not a base") + } +} +impl std::error::Error for UnacceptableBaseUrl {} + +#[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub enum RequestOrder { + Oldest, + Newest, + Index, +} + +impl RequestOrder { + #[inline] + pub fn as_str(self) -> &'static str { + match self { + RequestOrder::Oldest => "oldest", + RequestOrder::Newest => "newest", + RequestOrder::Index => "index", + } + } +} + +impl std::fmt::Display for RequestOrder { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} diff --git a/third_party/rust/sync15-traits/src/server_timestamp.rs b/third_party/rust/sync15-traits/src/server_timestamp.rs new file mode 100644 index 000000000000..109c9575cbda --- /dev/null +++ b/third_party/rust/sync15-traits/src/server_timestamp.rs @@ -0,0 +1,125 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +use std::marker::PhantomData; +use std::time::Duration; + +/// Typesafe way to manage server timestamps without accidentally mixing them up with +/// local ones. +#[derive(Debug, Copy, Clone, PartialEq, PartialOrd, Default)] +pub struct ServerTimestamp(pub i64); + +impl ServerTimestamp { + pub fn from_float_seconds(ts: f64) -> Self { + let rf = (ts * 1000.0).round(); + if !rf.is_finite() || rf < 0.0 || rf >= i64::max_value() as f64 { + log::error!("Illegal timestamp: {}", ts); + ServerTimestamp(0) + } else { + ServerTimestamp(rf as i64) + } + } + + pub fn from_millis(ts: i64) -> Self { + // Catch it in tests, but just complain and replace with 0 otherwise. + debug_assert!(ts >= 0, "Bad timestamp: {}", ts); + if ts >= 0 { + Self(ts) + } else { + log::error!("Illegal timestamp, substituting 0: {}", ts); + Self(0) + } + } +} + +// This lets us use these in hyper header! blocks. +impl std::str::FromStr for ServerTimestamp { + type Err = std::num::ParseFloatError; + fn from_str(s: &str) -> Result { + let val = f64::from_str(s)?; + Ok(Self::from_float_seconds(val)) + } +} + +impl std::fmt::Display for ServerTimestamp { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0 as f64 / 1000.0) + } +} + +impl ServerTimestamp { + pub const EPOCH: ServerTimestamp = ServerTimestamp(0); + + /// Returns None if `other` is later than `self` (Duration may not represent + /// negative timespans in rust). + #[inline] + pub fn duration_since(self, other: ServerTimestamp) -> Option { + let delta = self.0 - other.0; + if delta < 0 { + None + } else { + Some(Duration::from_millis(delta as u64)) + } + } + + /// Get the milliseconds for the timestamp. + #[inline] + pub fn as_millis(self) -> i64 { + self.0 + } +} + +impl serde::ser::Serialize for ServerTimestamp { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_f64(self.0 as f64 / 1000.0) + } +} + +struct TimestampVisitor(PhantomData); + +impl<'de> serde::de::Visitor<'de> for TimestampVisitor { + type Value = ServerTimestamp; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("a floating point number") + } + + fn visit_f64(self, value: f64) -> Result { + Ok(ServerTimestamp::from_float_seconds(value)) + } +} + +impl<'de> serde::de::Deserialize<'de> for ServerTimestamp { + fn deserialize>(deserializer: D) -> Result { + deserializer.deserialize_f64(TimestampVisitor(PhantomData)) + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_server_timestamp() { + let t0 = ServerTimestamp(10_300_150); + let t1 = ServerTimestamp(10_100_050); + assert!(t1.duration_since(t0).is_none()); + assert!(t0.duration_since(t1).is_some()); + let dur = t0.duration_since(t1).unwrap(); + assert_eq!(dur.as_secs(), 200); + assert_eq!(dur.subsec_nanos(), 100_000_000); + } + + #[test] + fn test_serde() { + let ts = ServerTimestamp(123_456); + + // test serialize + let ser = serde_json::to_string(&ts).unwrap(); + assert_eq!("123.456".to_string(), ser); + + // test deserialize + let ts: ServerTimestamp = serde_json::from_str(&ser).unwrap(); + assert_eq!(ServerTimestamp(123_456), ts); + } +} diff --git a/third_party/rust/sync15-traits/src/store.rs b/third_party/rust/sync15-traits/src/store.rs new file mode 100644 index 000000000000..08780904357b --- /dev/null +++ b/third_party/rust/sync15-traits/src/store.rs @@ -0,0 +1,97 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use crate::{ + client::ClientData, telemetry, CollectionRequest, Guid, IncomingChangeset, OutgoingChangeset, + ServerTimestamp, +}; +use failure::Error; + +#[derive(Debug, Clone, PartialEq)] +pub struct CollSyncIds { + pub global: Guid, + pub coll: Guid, +} + +/// Defines how a store is associated with Sync. +#[derive(Debug, Clone, PartialEq)] +pub enum StoreSyncAssociation { + /// This store is disconnected (although it may be connected in the future). + Disconnected, + /// Sync is connected, and has the following sync IDs. + Connected(CollSyncIds), +} + +/// Low-level store functionality. Stores that need custom reconciliation logic +/// should use this. +/// +/// Different stores will produce errors of different types. To accommodate +/// this, we force them all to return failure::Error. +pub trait Store { + fn collection_name(&self) -> std::borrow::Cow<'static, str>; + + /// Prepares the store for syncing. The tabs store currently uses this to + /// store the current list of clients, which it uses to look up device names + /// and types. + /// + /// Note that this method is only called by `sync_multiple`, and only if a + /// command processor is registered. In particular, `prepare_for_sync` will + /// not be called if the store is synced using `sync::synchronize` or + /// `sync_multiple::sync_multiple`. It _will_ be called if the store is + /// synced via the Sync Manager. + /// + /// TODO(issue #2590): This is pretty cludgey and will be hard to extend for + /// any case other than the tabs case. We should find another way to support + /// tabs... + fn prepare_for_sync(&self, _get_client_data: &dyn Fn() -> ClientData) -> Result<(), Error> { + Ok(()) + } + + /// `inbound` is a vector to support the case where + /// `get_collection_requests` returned multiple requests. The changesets are + /// in the same order as the requests were -- e.g. if `vec![req_a, req_b]` + /// was returned from `get_collection_requests`, `inbound` will have the + /// results from `req_a` as its first index, and those from `req_b` as it's + /// second. + fn apply_incoming( + &self, + inbound: Vec, + telem: &mut telemetry::Engine, + ) -> Result; + + fn sync_finished( + &self, + new_timestamp: ServerTimestamp, + records_synced: Vec, + ) -> Result<(), Error>; + + /// The store is responsible for building the collection request. Engines + /// typically will store a lastModified timestamp and use that to build a + /// request saying "give me full records since that date" - however, other + /// engines might do something fancier. This could even later be extended to + /// handle "backfills" etc + /// + /// To support more advanced use cases (e.g. remerge), multiple requests can + /// be returned here. The vast majority of engines will just want to return + /// zero or one item in their vector (zero is a valid optimization when the + /// server timestamp is the same as the engine last saw, one when it is not) + /// + /// Important: In the case when more than one collection is requested, it's + /// assumed the last one is the "canonical" one. (That is, it must be for + /// "this" collection, its timestamp is used to represent the sync, etc). + fn get_collection_requests( + &self, + server_timestamp: ServerTimestamp, + ) -> Result, Error>; + + /// Get persisted sync IDs. If they don't match the global state we'll be + /// `reset()` with the new IDs. + fn get_sync_assoc(&self) -> Result; + + /// Reset the store without wiping local data, ready for a "first sync". + /// `assoc` defines how this store is to be associated with sync. + fn reset(&self, assoc: &StoreSyncAssociation) -> Result<(), Error>; + + fn wipe(&self) -> Result<(), Error>; +} diff --git a/third_party/rust/sync15-traits/src/telemetry.rs b/third_party/rust/sync15-traits/src/telemetry.rs new file mode 100644 index 000000000000..a3609e3d6d24 --- /dev/null +++ b/third_party/rust/sync15-traits/src/telemetry.rs @@ -0,0 +1,777 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +//! Manage recording sync telemetry. Assumes some external telemetry +//! library/code which manages submitting. + +use std::collections::HashMap; +use std::time; + +use serde::{ser, Serialize, Serializer}; + +// A test helper, used by the many test modules below. +#[cfg(test)] +fn assert_json(v: &T, expected: serde_json::Value) +where + T: serde::Serialize, +{ + assert_eq!( + serde_json::to_value(&v).expect("should get a value"), + expected + ); +} + +/// What we record for 'when' and 'took' in a telemetry record. +#[derive(Debug, Serialize)] +struct WhenTook { + when: f64, + #[serde(skip_serializing_if = "crate::skip_if_default")] + took: u64, +} + +/// What we track while recording 'when' and 'took. It serializes as a WhenTook, +/// except when .finished() hasn't been called, in which case it panics. +#[derive(Debug)] +enum Stopwatch { + Started(time::SystemTime, time::Instant), + Finished(WhenTook), +} + +impl Default for Stopwatch { + fn default() -> Self { + Stopwatch::new() + } +} + +impl Stopwatch { + fn new() -> Self { + Stopwatch::Started(time::SystemTime::now(), time::Instant::now()) + } + + // For tests we don't want real timestamps because we test against literals. + #[cfg(test)] + fn finished(&self) -> Self { + Stopwatch::Finished(WhenTook { when: 0.0, took: 0 }) + } + + #[cfg(not(test))] + fn finished(&self) -> Self { + match self { + Stopwatch::Started(st, si) => { + let std = st.duration_since(time::UNIX_EPOCH).unwrap_or_default(); + let when = std.as_secs() as f64; // we don't want sub-sec accuracy. Do we need to write a float? + + let sid = si.elapsed(); + let took = sid.as_secs() * 1000 + (u64::from(sid.subsec_nanos()) / 1_000_000); + Stopwatch::Finished(WhenTook { when, took }) + } + _ => { + unreachable!("can't finish twice"); + } + } + } +} + +impl Serialize for Stopwatch { + fn serialize(&self, serializer: S) -> std::result::Result + where + S: Serializer, + { + match self { + Stopwatch::Started(_, _) => Err(ser::Error::custom("StopWatch has not been finished")), + Stopwatch::Finished(c) => c.serialize(serializer), + } + } +} + +#[cfg(test)] +mod stopwatch_tests { + use super::*; + + // A wrapper struct because we flatten - this struct should serialize with + // 'when' and 'took' keys (but with no 'sw'.) + #[derive(Debug, Serialize)] + struct WT { + #[serde(flatten)] + sw: Stopwatch, + } + + #[test] + fn test_not_finished() { + let wt = WT { + sw: Stopwatch::new(), + }; + serde_json::to_string(&wt).expect_err("unfinished stopwatch should fail"); + } + + #[test] + fn test() { + assert_json( + &WT { + sw: Stopwatch::Finished(WhenTook { when: 1.0, took: 1 }), + }, + serde_json::json!({"when": 1.0, "took": 1}), + ); + assert_json( + &WT { + sw: Stopwatch::Finished(WhenTook { when: 1.0, took: 0 }), + }, + serde_json::json!({"when": 1.0}), + ); + } +} + +/// A generic "Event" - suitable for all kinds of pings (although this module +/// only cares about the sync ping) +#[derive(Debug, Serialize)] +pub struct Event { + // We use static str references as we expect values to be literals. + object: &'static str, + + method: &'static str, + + // Maybe "value" should be a string? + #[serde(skip_serializing_if = "Option::is_none")] + value: Option<&'static str>, + + // we expect the keys to be literals but values are real strings. + #[serde(skip_serializing_if = "Option::is_none")] + extra: Option>, +} + +impl Event { + pub fn new(object: &'static str, method: &'static str) -> Self { + assert!(object.len() <= 20); + assert!(method.len() <= 20); + Self { + object, + method, + value: None, + extra: None, + } + } + + pub fn value(mut self, v: &'static str) -> Self { + assert!(v.len() <= 80); + self.value = Some(v); + self + } + + pub fn extra(mut self, key: &'static str, val: String) -> Self { + assert!(key.len() <= 15); + assert!(val.len() <= 85); + match self.extra { + None => self.extra = Some(HashMap::new()), + Some(ref e) => assert!(e.len() < 10), + } + self.extra.as_mut().unwrap().insert(key, val); + self + } +} + +#[cfg(test)] +mod test_events { + use super::*; + + #[test] + #[should_panic] + fn test_invalid_length_ctor() { + Event::new("A very long object value", "Method"); + } + + #[test] + #[should_panic] + fn test_invalid_length_extra_key() { + Event::new("O", "M").extra("A very long key value", "v".to_string()); + } + + #[test] + #[should_panic] + fn test_invalid_length_extra_val() { + let l = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ + abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"; + Event::new("O", "M").extra("k", l.to_string()); + } + + #[test] + #[should_panic] + fn test_too_many_extras() { + let l = "abcdefghijk"; + let mut e = Event::new("Object", "Method"); + for i in 0..l.len() { + e = e.extra(&l[i..=i], "v".to_string()); + } + } + + #[test] + fn test_json() { + assert_json( + &Event::new("Object", "Method").value("Value"), + serde_json::json!({"object": "Object", "method": "Method", "value": "Value"}), + ); + + assert_json( + &Event::new("Object", "Method").extra("one", "one".to_string()), + serde_json::json!({"object": "Object", + "method": "Method", + "extra": {"one": "one"} + }), + ) + } +} + +/// A Sync failure. +#[derive(Debug, Serialize)] +#[serde(tag = "name")] +pub enum SyncFailure { + #[serde(rename = "shutdownerror")] + Shutdown, + + #[serde(rename = "othererror")] + Other { error: String }, + + #[serde(rename = "unexpectederror")] + Unexpected { error: String }, + + #[serde(rename = "autherror")] + Auth { from: &'static str }, + + #[serde(rename = "httperror")] + Http { code: u16 }, +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn reprs() { + assert_json( + &SyncFailure::Shutdown, + serde_json::json!({"name": "shutdownerror"}), + ); + + assert_json( + &SyncFailure::Other { + error: "dunno".to_string(), + }, + serde_json::json!({"name": "othererror", "error": "dunno"}), + ); + + assert_json( + &SyncFailure::Unexpected { + error: "dunno".to_string(), + }, + serde_json::json!({"name": "unexpectederror", "error": "dunno"}), + ); + + assert_json( + &SyncFailure::Auth { from: "FxA" }, + serde_json::json!({"name": "autherror", "from": "FxA"}), + ); + + assert_json( + &SyncFailure::Http { code: 500 }, + serde_json::json!({"name": "httperror", "code": 500}), + ); + } +} + +/// Incoming record for an engine's sync +#[derive(Debug, Default, Serialize)] +pub struct EngineIncoming { + #[serde(skip_serializing_if = "crate::skip_if_default")] + applied: u32, + + #[serde(skip_serializing_if = "crate::skip_if_default")] + failed: u32, + + #[serde(rename = "newFailed")] + #[serde(skip_serializing_if = "crate::skip_if_default")] + new_failed: u32, + + #[serde(skip_serializing_if = "crate::skip_if_default")] + reconciled: u32, +} + +impl EngineIncoming { + pub fn new() -> Self { + Self { + ..Default::default() + } + } + + // A helper used via skip_serializing_if + fn is_empty(inc: &Option) -> bool { + match inc { + Some(a) => a.applied == 0 && a.failed == 0 && a.new_failed == 0 && a.reconciled == 0, + None => true, + } + } + + /// Increment the value of `applied` by `n`. + #[inline] + pub fn applied(&mut self, n: u32) { + self.applied += n; + } + + /// Increment the value of `failed` by `n`. + #[inline] + pub fn failed(&mut self, n: u32) { + self.failed += n; + } + + /// Increment the value of `new_failed` by `n`. + #[inline] + pub fn new_failed(&mut self, n: u32) { + self.new_failed += n; + } + + /// Increment the value of `reconciled` by `n`. + #[inline] + pub fn reconciled(&mut self, n: u32) { + self.reconciled += n; + } + + /// Get the value of `applied`. Mostly useful for testing. + #[inline] + pub fn get_applied(&self) -> u32 { + self.applied + } + + /// Get the value of `failed`. Mostly useful for testing. + #[inline] + pub fn get_failed(&self) -> u32 { + self.failed + } + + /// Get the value of `new_failed`. Mostly useful for testing. + #[inline] + pub fn get_new_failed(&self) -> u32 { + self.new_failed + } + + /// Get the value of `reconciled`. Mostly useful for testing. + #[inline] + pub fn get_reconciled(&self) -> u32 { + self.reconciled + } +} + +/// Outgoing record for an engine's sync +#[derive(Debug, Default, Serialize)] +pub struct EngineOutgoing { + #[serde(skip_serializing_if = "crate::skip_if_default")] + sent: usize, + + #[serde(skip_serializing_if = "crate::skip_if_default")] + failed: usize, +} + +impl EngineOutgoing { + pub fn new() -> Self { + EngineOutgoing { + ..Default::default() + } + } + + #[inline] + pub fn sent(&mut self, n: usize) { + self.sent += n; + } + + #[inline] + pub fn failed(&mut self, n: usize) { + self.failed += n; + } +} + +/// One engine's sync. +#[derive(Debug, Serialize)] +pub struct Engine { + name: String, + + #[serde(flatten)] + when_took: Stopwatch, + + #[serde(skip_serializing_if = "EngineIncoming::is_empty")] + incoming: Option, + + #[serde(skip_serializing_if = "Vec::is_empty")] + outgoing: Vec, // one for each batch posted. + + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(rename = "failureReason")] + failure: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + validation: Option, +} + +impl Engine { + pub fn new(name: impl Into) -> Self { + Self { + name: name.into(), + when_took: Stopwatch::new(), + incoming: None, + outgoing: Vec::new(), + failure: None, + validation: None, + } + } + + pub fn incoming(&mut self, inc: EngineIncoming) { + assert!(self.incoming.is_none()); + self.incoming = Some(inc); + } + + pub fn outgoing(&mut self, out: EngineOutgoing) { + self.outgoing.push(out); + } + + pub fn failure(&mut self, err: impl Into) { + // Currently we take the first error, under the assumption that the + // first is the most important and all others stem from that. + let failure = err.into(); + if self.failure.is_none() { + self.failure = Some(failure); + } else { + log::warn!( + "engine already has recorded a failure of {:?} - ignoring {:?}", + &self.failure, + &failure + ); + } + } + + pub fn validation(&mut self, v: Validation) { + assert!(self.validation.is_none()); + self.validation = Some(v); + } + + fn finished(&mut self) { + self.when_took = self.when_took.finished(); + } +} + +#[derive(Debug, Default, Serialize)] +pub struct Validation { + version: u32, + + #[serde(skip_serializing_if = "Vec::is_empty")] + problems: Vec, + + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(rename = "failureReason")] + failure: Option, +} + +impl Validation { + pub fn with_version(version: u32) -> Validation { + Validation { + version, + ..Validation::default() + } + } + + pub fn problem(&mut self, name: &'static str, count: usize) -> &mut Self { + if count > 0 { + self.problems.push(Problem { name, count }); + } + self + } +} + +#[derive(Debug, Default, Serialize)] +pub struct Problem { + name: &'static str, + #[serde(skip_serializing_if = "crate::skip_if_default")] + count: usize, +} + +#[cfg(test)] +mod engine_tests { + use super::*; + + #[test] + fn test_engine() { + let mut e = Engine::new("test_engine"); + e.finished(); + assert_json(&e, serde_json::json!({"name": "test_engine", "when": 0.0})); + } + + #[test] + fn test_engine_not_finished() { + let e = Engine::new("test_engine"); + serde_json::to_value(&e).expect_err("unfinished stopwatch should fail"); + } + + #[test] + fn test_incoming() { + let mut i = EngineIncoming::new(); + i.applied(1); + i.failed(2); + let mut e = Engine::new("TestEngine"); + e.incoming(i); + e.finished(); + assert_json( + &e, + serde_json::json!({"name": "TestEngine", "when": 0.0, "incoming": {"applied": 1, "failed": 2}}), + ); + } + + #[test] + fn test_outgoing() { + let mut o = EngineOutgoing::new(); + o.sent(2); + o.failed(1); + let mut e = Engine::new("TestEngine"); + e.outgoing(o); + e.finished(); + assert_json( + &e, + serde_json::json!({"name": "TestEngine", "when": 0.0, "outgoing": [{"sent": 2, "failed": 1}]}), + ); + } + + #[test] + fn test_failure() { + let mut e = Engine::new("TestEngine"); + e.failure(SyncFailure::Http { code: 500 }); + e.finished(); + assert_json( + &e, + serde_json::json!({"name": "TestEngine", + "when": 0.0, + "failureReason": {"name": "httperror", "code": 500} + }), + ); + } + + #[test] + fn test_raw() { + let mut e = Engine::new("TestEngine"); + let mut inc = EngineIncoming::new(); + inc.applied(10); + e.incoming(inc); + let mut out = EngineOutgoing::new(); + out.sent(1); + e.outgoing(out); + e.failure(SyncFailure::Http { code: 500 }); + e.finished(); + + assert_eq!(e.outgoing.len(), 1); + assert_eq!(e.incoming.as_ref().unwrap().applied, 10); + assert_eq!(e.outgoing[0].sent, 1); + assert!(e.failure.is_some()); + serde_json::to_string(&e).expect("should get json"); + } +} + +/// A single sync. May have many engines, may have its own failure. +#[derive(Debug, Serialize, Default)] +pub struct SyncTelemetry { + #[serde(flatten)] + when_took: Stopwatch, + + #[serde(skip_serializing_if = "Vec::is_empty")] + engines: Vec, + + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(rename = "failureReason")] + failure: Option, +} + +impl SyncTelemetry { + pub fn new() -> Self { + Default::default() + } + + pub fn engine(&mut self, mut e: Engine) { + e.finished(); + self.engines.push(e); + } + + pub fn failure(&mut self, failure: SyncFailure) { + assert!(self.failure.is_none()); + self.failure = Some(failure); + } + + // Note that unlike other 'finished' methods, this isn't private - someone + // needs to explicitly call this before handling the json payload to + // whatever ends up submitting it. + pub fn finished(&mut self) { + self.when_took = self.when_took.finished(); + } +} + +#[cfg(test)] +mod sync_tests { + use super::*; + + #[test] + fn test_accum() { + let mut s = SyncTelemetry::new(); + let mut inc = EngineIncoming::new(); + inc.applied(10); + let mut e = Engine::new("test_engine"); + e.incoming(inc); + e.failure(SyncFailure::Http { code: 500 }); + e.finished(); + s.engine(e); + s.finished(); + + assert_json( + &s, + serde_json::json!({ + "when": 0.0, + "engines": [{ + "name":"test_engine", + "when":0.0, + "incoming": { + "applied": 10 + }, + "failureReason": { + "name": "httperror", + "code": 500 + } + }] + }), + ); + } + + #[test] + fn test_multi_engine() { + let mut inc_e1 = EngineIncoming::new(); + inc_e1.applied(1); + let mut e1 = Engine::new("test_engine"); + e1.incoming(inc_e1); + + let mut inc_e2 = EngineIncoming::new(); + inc_e2.failed(1); + let mut e2 = Engine::new("test_engine_2"); + e2.incoming(inc_e2); + let mut out_e2 = EngineOutgoing::new(); + out_e2.sent(1); + e2.outgoing(out_e2); + + let mut s = SyncTelemetry::new(); + s.engine(e1); + s.engine(e2); + s.failure(SyncFailure::Http { code: 500 }); + s.finished(); + assert_json( + &s, + serde_json::json!({ + "when": 0.0, + "engines": [{ + "name": "test_engine", + "when": 0.0, + "incoming": { + "applied": 1 + } + },{ + "name": "test_engine_2", + "when": 0.0, + "incoming": { + "failed": 1 + }, + "outgoing": [{ + "sent": 1 + }] + }], + "failureReason": { + "name": "httperror", + "code": 500 + } + }), + ); + } +} + +/// The Sync ping payload, as documented at +/// https://firefox-source-docs.mozilla.org/toolkit/components/telemetry/telemetry/data/sync-ping.html. +/// May have many syncs, may have many events. However, due to the architecture +/// of apps which use these components, this payload is almost certainly not +/// suitable for submitting directly. For example, we will always return a +/// payload with exactly 1 sync, and it will not know certain other fields +/// in the payload, such as the *hashed* FxA device ID (see +/// https://searchfox.org/mozilla-central/rev/c3ebaf6de2d481c262c04bb9657eaf76bf47e2ac/services/sync/modules/browserid_identity.js#185 +/// for an example of how the device ID is constructed). The intention is that +/// consumers of this will use this to create a "real" payload - eg, accumulating +/// until some threshold number of syncs is reached, and contributing +/// additional data which only the consumer knows. +#[derive(Debug, Serialize, Default)] +pub struct SyncTelemetryPing { + version: u32, + + uid: Option, + + #[serde(skip_serializing_if = "Vec::is_empty")] + events: Vec, + + #[serde(skip_serializing_if = "Vec::is_empty")] + syncs: Vec, +} + +impl SyncTelemetryPing { + pub fn new() -> Self { + Self { + version: 1, + ..Default::default() + } + } + + pub fn uid(&mut self, uid: String) { + if let Some(ref existing) = self.uid { + if *existing != uid { + log::warn!("existing uid ${} being replaced by {}", existing, uid); + } + } + self.uid = Some(uid); + } + + pub fn sync(&mut self, mut s: SyncTelemetry) { + s.finished(); + self.syncs.push(s); + } + + pub fn event(&mut self, e: Event) { + self.events.push(e); + } +} + +ffi_support::implement_into_ffi_by_json!(SyncTelemetryPing); + +#[cfg(test)] +mod ping_tests { + use super::*; + #[test] + fn test_ping() { + let engine = Engine::new("test"); + let mut s = SyncTelemetry::new(); + s.engine(engine); + let mut p = SyncTelemetryPing::new(); + p.uid("user-id".into()); + p.sync(s); + let event = Event::new("foo", "bar"); + p.event(event); + assert_json( + &p, + serde_json::json!({ + "events": [{ + "method": "bar", "object": "foo" + }], + "syncs": [{ + "engines": [{ + "name": "test", "when": 0.0 + }], + "when": 0.0 + }], + "uid": "user-id", + "version": 1 + }), + ); + } +} diff --git a/toolkit/library/rust/shared/Cargo.toml b/toolkit/library/rust/shared/Cargo.toml index f2f781d794b5..6f50ba377842 100644 --- a/toolkit/library/rust/shared/Cargo.toml +++ b/toolkit/library/rust/shared/Cargo.toml @@ -57,6 +57,8 @@ fluent-langneg-ffi = { path = "../../../../intl/locale/rust/fluent-langneg-ffi" fluent = { version = "0.11" , features = ["fluent-pseudo"] } fluent-ffi = { path = "../../../../intl/l10n/rust/fluent-ffi" } +sync15-traits = { git = "https://github.com/mozilla/application-services", rev = "120e51dd5f2aab4194cf0f7e93b2a8923f4504bb" } + [build-dependencies] rustc_version = "0.2" diff --git a/toolkit/library/rust/shared/lib.rs b/toolkit/library/rust/shared/lib.rs index 24dfa15d1907..10c96aa1e72c 100644 --- a/toolkit/library/rust/shared/lib.rs +++ b/toolkit/library/rust/shared/lib.rs @@ -70,6 +70,8 @@ extern crate fluent_langneg_ffi; extern crate fluent; extern crate fluent_ffi; +extern crate sync15_traits; + #[cfg(feature = "remote")] extern crate remote;