diff --git a/.cargo/config.in b/.cargo/config.in index 5581e66e83de..a036233f6acc 100644 --- a/.cargo/config.in +++ b/.cargo/config.in @@ -20,7 +20,7 @@ tag = "v0.2.4" [source."https://github.com/mozilla/application-services"] git = "https://github.com/mozilla/application-services" replace-with = "vendored-sources" -rev = "120e51dd5f2aab4194cf0f7e93b2a8923f4504bb" +rev = "c17198fa5a88295f2cca722586c539280e10201c" [source."https://github.com/mozilla-spidermonkey/jsparagus"] git = "https://github.com/mozilla-spidermonkey/jsparagus" diff --git a/Cargo.lock b/Cargo.lock index 4079b5e3faa4..6b7225587968 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1238,6 +1238,14 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff511d5dc435d703f4971bc399647c9bc38e20cb41452e3b9feb4765419ed3f3" +[[package]] +name = "error-support" +version = "0.1.0" +source = "git+https://github.com/mozilla/application-services?rev=c17198fa5a88295f2cca722586c539280e10201c#c17198fa5a88295f2cca722586c539280e10201c" +dependencies = [ + "failure", +] + [[package]] name = "euclid" version = "0.20.8" @@ -1843,6 +1851,7 @@ dependencies = [ "sync15-traits", "unic-langid", "unic-langid-ffi", + "webext-storage", "webrender_bindings", "wgpu_bindings", "xpcom", @@ -2148,6 +2157,11 @@ dependencies = [ "adler32", ] +[[package]] +name = "interrupt-support" +version = "0.1.0" +source = "git+https://github.com/mozilla/application-services?rev=c17198fa5a88295f2cca722586c539280e10201c#c17198fa5a88295f2cca722586c539280e10201c" + [[package]] name = "intl-memoizer" version = "0.4.0" @@ -2395,6 +2409,7 @@ version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e704a02bcaecd4a08b93a23f6be59d0bd79cd161e0963e9499165a0a35df7bd" dependencies = [ + "cc", "pkg-config", "vcpkg", ] @@ -3077,6 +3092,11 @@ dependencies = [ "nsstring", ] +[[package]] +name = "nss_build_common" +version = "0.1.0" +source = "git+https://github.com/mozilla/application-services?rev=c17198fa5a88295f2cca722586c539280e10201c#c17198fa5a88295f2cca722586c539280e10201c" + [[package]] name = "nsstring" version = "0.1.0" @@ -4202,6 +4222,18 @@ dependencies = [ "spirv-cross-internal", ] +[[package]] +name = "sql-support" +version = "0.1.0" +source = "git+https://github.com/mozilla/application-services?rev=c17198fa5a88295f2cca722586c539280e10201c#c17198fa5a88295f2cca722586c539280e10201c" +dependencies = [ + "ffi-support", + "interrupt-support", + "lazy_static", + "log", + "rusqlite", +] + [[package]] name = "stable_deref_trait" version = "1.0.0" @@ -4390,18 +4422,22 @@ dependencies = [ [[package]] name = "sync-guid" version = "0.1.0" -source = "git+https://github.com/mozilla/application-services?rev=120e51dd5f2aab4194cf0f7e93b2a8923f4504bb#120e51dd5f2aab4194cf0f7e93b2a8923f4504bb" +source = "git+https://github.com/mozilla/application-services?rev=c17198fa5a88295f2cca722586c539280e10201c#c17198fa5a88295f2cca722586c539280e10201c" dependencies = [ + "base64 0.12.0", + "rand", + "rusqlite", "serde", ] [[package]] name = "sync15-traits" version = "0.1.0" -source = "git+https://github.com/mozilla/application-services?rev=120e51dd5f2aab4194cf0f7e93b2a8923f4504bb#120e51dd5f2aab4194cf0f7e93b2a8923f4504bb" +source = "git+https://github.com/mozilla/application-services?rev=c17198fa5a88295f2cca722586c539280e10201c#c17198fa5a88295f2cca722586c539280e10201c" dependencies = [ "failure", "ffi-support", + "interrupt-support", "log", "serde", "serde_json", @@ -4974,6 +5010,7 @@ dependencies = [ "idna", "matches", "percent-encoding", + "serde", ] [[package]] @@ -5108,6 +5145,26 @@ dependencies = [ "warp", ] +[[package]] +name = "webext-storage" +version = "0.1.0" +source = "git+https://github.com/mozilla/application-services?rev=c17198fa5a88295f2cca722586c539280e10201c#c17198fa5a88295f2cca722586c539280e10201c" +dependencies = [ + "error-support", + "failure", + "interrupt-support", + "lazy_static", + "log", + "nss_build_common", + "rusqlite", + "serde", + "serde_derive", + "serde_json", + "sql-support", + "sync-guid", + "url", +] + [[package]] name = "webrender" version = "0.61.0" diff --git a/third_party/rust/error-support/.cargo-checksum.json b/third_party/rust/error-support/.cargo-checksum.json new file mode 100644 index 000000000000..0ca037f9d2fd --- /dev/null +++ b/third_party/rust/error-support/.cargo-checksum.json @@ -0,0 +1 @@ +{"files":{"Cargo.toml":"9ba6f30454cfbe5cc844824a89f31b65d607df6aec569d093eb6307d902c5159","src/lib.rs":"4581b12eb58f9fb5275c7af74fbc4521b82ef224b6ba81f0e785c372ba95f8c6"},"package":null} \ No newline at end of file diff --git a/third_party/rust/error-support/Cargo.toml b/third_party/rust/error-support/Cargo.toml new file mode 100644 index 000000000000..302255445b2c --- /dev/null +++ b/third_party/rust/error-support/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "error-support" +version = "0.1.0" +authors = ["Thom Chiovoloni "] +edition = "2018" +license = "MPL-2.0" + +[dependencies] +failure = "0.1.6" + diff --git a/third_party/rust/error-support/src/lib.rs b/third_party/rust/error-support/src/lib.rs new file mode 100644 index 000000000000..1ecc968dc095 --- /dev/null +++ b/third_party/rust/error-support/src/lib.rs @@ -0,0 +1,99 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/// Define a wrapper around the the provided ErrorKind type. +/// See also `define_error` which is more likely to be what you want. +#[macro_export] +macro_rules! define_error_wrapper { + ($Kind:ty) => { + /// Re-exported, so that using crate::error::* gives you the .context() + /// method, which we don't use much but should *really* use more. + pub use failure::ResultExt; + + pub type Result = std::result::Result; + + #[derive(Debug)] + pub struct Error(Box>); + + impl failure::Fail for Error { + fn cause(&self) -> Option<&dyn failure::Fail> { + self.0.cause() + } + + fn backtrace(&self) -> Option<&failure::Backtrace> { + self.0.backtrace() + } + + fn name(&self) -> Option<&str> { + self.0.name() + } + } + + impl std::fmt::Display for Error { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + std::fmt::Display::fmt(&*self.0, f) + } + } + + impl Error { + pub fn kind(&self) -> &$Kind { + &*self.0.get_context() + } + } + + impl From> for Error { + // Cold to optimize in favor of non-error cases. + #[cold] + fn from(ctx: failure::Context<$Kind>) -> Error { + Error(Box::new(ctx)) + } + } + + impl From<$Kind> for Error { + // Cold to optimize in favor of non-error cases. + #[cold] + fn from(kind: $Kind) -> Self { + Error(Box::new(failure::Context::new(kind))) + } + } + }; +} + +/// Define a set of conversions from external error types into the provided +/// error kind. Use `define_error` to do this at the same time as +/// `define_error_wrapper`. +#[macro_export] +macro_rules! define_error_conversions { + ($Kind:ident { $(($variant:ident, $type:ty)),* $(,)? }) => ($( + impl From<$type> for $Kind { + // Cold to optimize in favor of non-error cases. + #[cold] + fn from(e: $type) -> $Kind { + $Kind::$variant(e) + } + } + + impl From<$type> for Error { + // Cold to optimize in favor of non-error cases. + #[cold] + fn from(e: $type) -> Self { + Error::from($Kind::$variant(e)) + } + } + )*); +} + +/// All the error boilerplate (okay, with a couple exceptions in some cases) in +/// one place. +#[macro_export] +macro_rules! define_error { + ($Kind:ident { $(($variant:ident, $type:ty)),* $(,)? }) => { + $crate::define_error_wrapper!($Kind); + $crate::define_error_conversions! { + $Kind { + $(($variant, $type)),* + } + } + }; +} diff --git a/third_party/rust/interrupt-support/.cargo-checksum.json b/third_party/rust/interrupt-support/.cargo-checksum.json new file mode 100644 index 000000000000..2530becc56fe --- /dev/null +++ b/third_party/rust/interrupt-support/.cargo-checksum.json @@ -0,0 +1 @@ +{"files":{"Cargo.toml":"e4b1f4f6a20cfcbfdfe9e47a875a09d7c37e815953441000c62c191570bfa5de","README.md":"7f1418b4a7c138ba20bcaea077fe6cf0d6ffbaf6df6b90c80efc52aa0d0e2e9f","src/lib.rs":"d7311f1fe25c25e651fae85fcd734cd313331c580a050c31b8bf64d957aede0f"},"package":null} \ No newline at end of file diff --git a/third_party/rust/interrupt-support/Cargo.toml b/third_party/rust/interrupt-support/Cargo.toml new file mode 100644 index 000000000000..3b8647063728 --- /dev/null +++ b/third_party/rust/interrupt-support/Cargo.toml @@ -0,0 +1,6 @@ +[package] +name = "interrupt-support" +version = "0.1.0" +authors = ["application-services@mozilla.com"] +license = "MPL-2.0" +edition = "2018" diff --git a/third_party/rust/interrupt-support/README.md b/third_party/rust/interrupt-support/README.md new file mode 100644 index 000000000000..5a9fbe5f395b --- /dev/null +++ b/third_party/rust/interrupt-support/README.md @@ -0,0 +1,4 @@ +## Interrupt crate + +This create exposes traits and errors to allow for interrupt support across +the various crates in this repository. diff --git a/third_party/rust/interrupt-support/src/lib.rs b/third_party/rust/interrupt-support/src/lib.rs new file mode 100644 index 000000000000..8a365e4adcf2 --- /dev/null +++ b/third_party/rust/interrupt-support/src/lib.rs @@ -0,0 +1,46 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#![allow(unknown_lints)] +#![warn(rust_2018_idioms)] + +// Note that in the future it might make sense to also add a trait for +// an Interruptable, but we don't need this abstraction now and it's unclear +// if we ever will. + +/// Represents the state of something that may be interrupted. Decoupled from +/// the interrupt mechanics so that things which want to check if they have been +/// interrupted are simpler. +pub trait Interruptee { + fn was_interrupted(&self) -> bool; + + fn err_if_interrupted(&self) -> Result<(), Interrupted> { + if self.was_interrupted() { + return Err(Interrupted); + } + Ok(()) + } +} + +/// A convenience implementation, should only be used in tests. +pub struct NeverInterrupts; + +impl Interruptee for NeverInterrupts { + #[inline] + fn was_interrupted(&self) -> bool { + false + } +} + +/// The error returned by err_if_interrupted. +#[derive(Debug, Clone, PartialEq)] +pub struct Interrupted; + +impl std::fmt::Display for Interrupted { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("The operation was interrupted") + } +} + +impl std::error::Error for Interrupted {} diff --git a/third_party/rust/nss_build_common/.cargo-checksum.json b/third_party/rust/nss_build_common/.cargo-checksum.json new file mode 100644 index 000000000000..958b2772f114 --- /dev/null +++ b/third_party/rust/nss_build_common/.cargo-checksum.json @@ -0,0 +1 @@ +{"files":{"Cargo.toml":"4f1d37d926e853eb9f3d8074b45c00a317e2b4aafbc339a471430d28526716e9","src/lib.rs":"bf8f68b313cf179725ecf84960fc0e18dc00cee428ab0d51a038252152427681"},"package":null} \ No newline at end of file diff --git a/third_party/rust/nss_build_common/Cargo.toml b/third_party/rust/nss_build_common/Cargo.toml new file mode 100644 index 000000000000..00288b2f1096 --- /dev/null +++ b/third_party/rust/nss_build_common/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "nss_build_common" +version = "0.1.0" +authors = ["Thom Chiovoloni "] +edition = "2018" +license = "MPL-2.0" + +[dependencies] diff --git a/third_party/rust/nss_build_common/src/lib.rs b/third_party/rust/nss_build_common/src/lib.rs new file mode 100644 index 000000000000..3df67668cc73 --- /dev/null +++ b/third_party/rust/nss_build_common/src/lib.rs @@ -0,0 +1,154 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +//! This shouldn't exist, but does because if something isn't going to link +//! against `nss` but has an `nss`-enabled `sqlcipher` turned on (for example, +//! by a `cargo` feature activated by something else in the workspace). +//! it might need to issue link commands for NSS. + +use std::{ + env, + ffi::OsString, + path::{Path, PathBuf}, +}; + +#[derive(Clone, Copy, PartialEq, Debug)] +pub enum LinkingKind { + Dynamic { folded_libs: bool }, + Static, +} + +#[derive(Debug, PartialEq, Clone)] +pub struct NoNssDir; + +pub fn link_nss() -> Result<(), NoNssDir> { + let is_gecko = env::var_os("MOZ_TOPOBJDIR").is_some(); + if !is_gecko { + let (lib_dir, include_dir) = get_nss()?; + println!( + "cargo:rustc-link-search=native={}", + lib_dir.to_string_lossy() + ); + println!("cargo:include={}", include_dir.to_string_lossy()); + let kind = determine_kind(); + link_nss_libs(kind); + } else { + let libs = match env::var("CARGO_CFG_TARGET_OS") + .as_ref() + .map(std::string::String::as_str) + { + Ok("android") | Ok("macos") => vec!["nss3"], + _ => vec!["nssutil3", "nss3", "plds4", "plc4", "nspr4"], + }; + for lib in &libs { + println!("cargo:rustc-link-lib=dylib={}", lib); + } + } + Ok(()) +} + +fn get_nss() -> Result<(PathBuf, PathBuf), NoNssDir> { + let nss_dir = env("NSS_DIR").ok_or(NoNssDir)?; + let nss_dir = Path::new(&nss_dir); + let lib_dir = nss_dir.join("lib"); + let include_dir = nss_dir.join("include"); + Ok((lib_dir, include_dir)) +} + +fn determine_kind() -> LinkingKind { + if env_flag("NSS_STATIC") { + LinkingKind::Static + } else { + let folded_libs = env_flag("NSS_USE_FOLDED_LIBS"); + LinkingKind::Dynamic { folded_libs } + } +} + +fn link_nss_libs(kind: LinkingKind) { + let libs = get_nss_libs(kind); + // Emit -L flags + let kind_str = match kind { + LinkingKind::Dynamic { .. } => "dylib", + LinkingKind::Static => "static", + }; + for lib in libs { + println!("cargo:rustc-link-lib={}={}", kind_str, lib); + } +} + +fn get_nss_libs(kind: LinkingKind) -> Vec<&'static str> { + match kind { + LinkingKind::Static => { + let mut static_libs = vec![ + "certdb", + "certhi", + "cryptohi", + "freebl_static", + "hw-acc-crypto", + "nspr4", + "nss_static", + "nssb", + "nssdev", + "nsspki", + "nssutil", + "pk11wrap_static", + "plc4", + "plds4", + "softokn_static", + ]; + // Hardware specific libs. + let target_arch = env::var("CARGO_CFG_TARGET_ARCH").unwrap(); + let target_os = env::var("CARGO_CFG_TARGET_OS").unwrap(); + // https://searchfox.org/mozilla-central/rev/1eb05019f47069172ba81a6c108a584a409a24ea/security/nss/lib/freebl/freebl.gyp#159-168 + if target_arch == "x86_64" || target_arch == "x86" { + static_libs.push("gcm-aes-x86_c_lib"); + } else if target_arch == "aarch64" { + static_libs.push("gcm-aes-aarch64_c_lib"); + } + // https://searchfox.org/mozilla-central/rev/1eb05019f47069172ba81a6c108a584a409a24ea/security/nss/lib/freebl/freebl.gyp#224-233 + if ((target_os == "android" || target_os == "linux") && target_arch == "x86_64") + || target_os == "windows" + { + static_libs.push("intel-gcm-wrap_c_lib"); + // https://searchfox.org/mozilla-central/rev/1eb05019f47069172ba81a6c108a584a409a24ea/security/nss/lib/freebl/freebl.gyp#43-47 + if (target_os == "android" || target_os == "linux") && target_arch == "x86_64" { + static_libs.push("intel-gcm-s_lib"); + } + } + static_libs + } + LinkingKind::Dynamic { folded_libs } => { + let mut dylibs = vec!["freebl3", "nss3", "nssckbi", "softokn3"]; + if !folded_libs { + dylibs.append(&mut vec!["nspr4", "nssutil3", "plc4", "plds4"]); + } + dylibs + } + } +} + +pub fn env(name: &str) -> Option { + println!("cargo:rerun-if-env-changed={}", name); + env::var_os(name) +} + +pub fn env_str(name: &str) -> Option { + println!("cargo:rerun-if-env-changed={}", name); + env::var(name).ok() +} + +pub fn env_flag(name: &str) -> bool { + match env_str(name).as_ref().map(String::as_ref) { + Some("1") => true, + Some("0") => false, + Some(s) => { + println!( + "cargo:warning=unknown value for environment var {:?}: {:?}. Ignoring", + name, s + ); + false + } + None => false, + } +} diff --git a/third_party/rust/sql-support/.cargo-checksum.json b/third_party/rust/sql-support/.cargo-checksum.json new file mode 100644 index 000000000000..872951fc7ebc --- /dev/null +++ b/third_party/rust/sql-support/.cargo-checksum.json @@ -0,0 +1 @@ +{"files":{"Cargo.toml":"6ac08b70091eff4fc18499837eef7b330aeeda34da64c707a322a2cdaac0ae31","doc/query-plan.md":"fc877e6cbf1b0e089ec99ee4f34673cd9b3fe1a23c8fcfec20cf286cdc0cd0d0","src/conn_ext.rs":"1126009dd562a333d336c6230814b03de970e2eceaef51b3a3ecd23484a3e23b","src/each_chunk.rs":"8aaba842e43b002fbc0fee95d14ce08faa7187b1979c765b2e270cd4802607a5","src/interrupt.rs":"76c829dce08673e06cf1273030a134cd38f713f9b8a9c80982e753a1fe1437a2","src/lib.rs":"cceb1d597dfc01e1141b89351bc875d7b2a680c272642eee53221c3aab9a70e0","src/maybe_cached.rs":"0b18425595055883a98807fbd62ff27a79c18af34e7cb3439f8c3438463ef2dd","src/query_plan.rs":"c0cc296ddf528a949f683317cea2da67ff5caee8042cf20ff00d9f8f54272ad8","src/repeat.rs":"1885f4dd36cc21fabad1ba28ad2ff213ed17707c57564e1c0d7b0349112118bb"},"package":null} \ No newline at end of file diff --git a/third_party/rust/sql-support/Cargo.toml b/third_party/rust/sql-support/Cargo.toml new file mode 100644 index 000000000000..321f4b3ac222 --- /dev/null +++ b/third_party/rust/sql-support/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "sql-support" +edition = "2018" +version = "0.1.0" +authors = ["Thom Chiovoloni "] +license = "MPL-2.0" + +[features] +default = [] +log_query_plans = [] + +[dependencies] +log = "0.4" +lazy_static = "1.4.0" +interrupt-support = { path = "../interrupt" } +ffi-support = "0.4" + +[dependencies.rusqlite] +version = "0.23.1" +features = ["functions", "limits", "bundled"] diff --git a/third_party/rust/sql-support/doc/query-plan.md b/third_party/rust/sql-support/doc/query-plan.md new file mode 100644 index 000000000000..137058a16cfb --- /dev/null +++ b/third_party/rust/sql-support/doc/query-plan.md @@ -0,0 +1,79 @@ +# Getting query plans out of places/logins/other consumers. + +If these crates are built with the `log_query_plans` feature enabled (or cargo decides to use a version of `sql-support` that has beeen built with that feature), then queries that go through sql-support will have their [query plans](https://www.sqlite.org/eqp.html) logged. The default place they get logged is stdout, however you can also specify a file by setting the `QUERY_PLAN_LOG` variable in the environment to a file where the plans will be appended. + +Worth noting that new logs will be appended to `QUERY_PLAN_LOG`, we don't clear the file. This is so that you can more easily see how the query plan changed during testing. + +The queries that go through this are any that are + +1. Executed entirely within sql-support (we need both the query and it's parameters) +2. Take named (and not positional) parameters. + +At the time of writing this, that includes: + +- `try_query_row` +- `query_rows_and_then_named_cached` +- `query_rows_and_then_named` +- `query_row_and_then_named` +- `query_one` +- `execute_named_cached` +- Possibly more, check [ConnExt](https://github.com/mozilla/application-services/blob/master/components/support/sql/src/conn_ext.rs). + +In particular, this excludes queries where the statement is prepared separately from execution. + +## Usage + +As mentioned, this is turned on with the log_query_plans feature. I don't know why, but I've had mediocre luck enabling it explicitly, but 100% success enabling it via `--all-features`. So that's what I recommend. + +Note that for tests, if you're logging to stdout, you'll need to end the test command with `-- --no-capture`, or else it will hide stdout output from you. You also may want to pass `--test-threads 1` (also after the `--`) so that the plans are logged near the tests that are executing, but it doesn't matter that much, since we log the SQL before the plan. + + +Executing tests, having the output logged to stdout: + +``` +$ cargo test -p logins --all-features -- --no-capture +... +test engine::test::test_general ... +### QUERY PLAN +#### SQL: + SELECT + FROM loginsL + WHERE is_deleted = 0 + AND guid = :guid + UNION ALL + SELECT + FROM loginsM + WHERE is_overridden IS NOT 1 + AND guid = :guid + ORDER BY hostname ASC + LIMIT 1 + +#### PLAN: +QUERY PLAN +`--MERGE (UNION ALL) + |--LEFT + | `--SEARCH TABLE loginsL USING INDEX sqlite_autoindex_loginsL_1 (guid=?) + `--RIGHT + `--SEARCH TABLE loginsM USING INDEX sqlite_autoindex_loginsM_1 (guid=?) +### END QUERY PLAN +... +``` + +Executing an example, with the output logged to a file. + +``` +$ env QUERY_PLAN_LOG=/path/to/my/logfile.txt cargo run -p places --all-features --example autocomplete -- +# (many shells can also do this as follows) +$ QUERY_PLAN_LOG=/path/to/my/logfile.txt cargo run -p places --all-features --example autocomplete -- +``` + +## Using from code + +This is also available as types on `sql_support`. + +```rust +println!("This prints the same output as is normally logged, and works \ + even when the logging feature is off: {}", + sql_support:QueryPlan::new(conn, sql, params)); +``` + diff --git a/third_party/rust/sql-support/src/conn_ext.rs b/third_party/rust/sql-support/src/conn_ext.rs new file mode 100644 index 000000000000..b7324bd5848e --- /dev/null +++ b/third_party/rust/sql-support/src/conn_ext.rs @@ -0,0 +1,380 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use rusqlite::{ + self, + types::{FromSql, ToSql}, + Connection, Result as SqlResult, Row, Savepoint, Transaction, TransactionBehavior, NO_PARAMS, +}; +use std::iter::FromIterator; +use std::ops::Deref; +use std::time::Instant; + +use crate::maybe_cached::MaybeCached; + +pub struct Conn(rusqlite::Connection); + +/// This trait exists so that we can use these helpers on `rusqlite::{Transaction, Connection}`. +/// Note that you must import ConnExt in order to call these methods on anything. +pub trait ConnExt { + /// The method you need to implement to opt in to all of this. + fn conn(&self) -> &Connection; + + /// Set the value of the pragma on the main database. Returns the same object, for chaining. + fn set_pragma(&self, pragma_name: &str, pragma_value: T) -> SqlResult<&Self> + where + T: ToSql, + Self: Sized, + { + // None == Schema name, e.g. `PRAGMA some_attached_db.something = blah` + self.conn() + .pragma_update(None, pragma_name, &pragma_value)?; + Ok(self) + } + + /// Get a cached or uncached statement based on a flag. + fn prepare_maybe_cached<'conn>( + &'conn self, + sql: &str, + cache: bool, + ) -> SqlResult> { + MaybeCached::prepare(self.conn(), sql, cache) + } + + /// Execute all the provided statements. + fn execute_all(&self, stmts: &[&str]) -> SqlResult<()> { + let conn = self.conn(); + for sql in stmts { + let r = conn.execute(sql, NO_PARAMS); + match r { + Ok(_) => {} + // Ignore ExecuteReturnedResults error because they're pointless + // and annoying. + Err(rusqlite::Error::ExecuteReturnedResults) => {} + Err(e) => return Err(e), + } + } + Ok(()) + } + + /// Equivalent to `Connection::execute_named` but caches the statement so that subsequent + /// calls to `execute_cached` will have improved performance. + fn execute_cached

(&self, sql: &str, params: P) -> SqlResult + where + P: IntoIterator, + P::Item: ToSql, + { + let mut stmt = self.conn().prepare_cached(sql)?; + stmt.execute(params) + } + + /// Equivalent to `Connection::execute_named` but caches the statement so that subsequent + /// calls to `execute_named_cached` will have imprroved performance. + fn execute_named_cached(&self, sql: &str, params: &[(&str, &dyn ToSql)]) -> SqlResult { + crate::maybe_log_plan(self.conn(), sql, params); + let mut stmt = self.conn().prepare_cached(sql)?; + stmt.execute_named(params) + } + + /// Execute a query that returns a single result column, and return that result. + fn query_one(&self, sql: &str) -> SqlResult { + crate::maybe_log_plan(self.conn(), sql, &[]); + let res: T = self + .conn() + .query_row_and_then(sql, NO_PARAMS, |row| row.get(0))?; + Ok(res) + } + + /// Execute a query that returns 0 or 1 result columns, returning None + /// if there were no rows, or if the only result was NULL. + fn try_query_one( + &self, + sql: &str, + params: &[(&str, &dyn ToSql)], + cache: bool, + ) -> SqlResult> + where + Self: Sized, + { + crate::maybe_log_plan(self.conn(), sql, params); + use rusqlite::OptionalExtension; + // The outer option is if we got rows, the inner option is + // if the first row was null. + let res: Option> = self + .conn() + .query_row_and_then_named(sql, params, |row| row.get(0), cache) + .optional()?; + // go from Option> to Option + Ok(res.unwrap_or_default()) + } + + /// Equivalent to `rusqlite::Connection::query_row_and_then` but allows use + /// of named parameters, and allows passing a flag to indicate that it's cached. + fn query_row_and_then_named( + &self, + sql: &str, + params: &[(&str, &dyn ToSql)], + mapper: F, + cache: bool, + ) -> Result + where + Self: Sized, + E: From, + F: FnOnce(&Row<'_>) -> Result, + { + crate::maybe_log_plan(self.conn(), sql, params); + Ok(self + .try_query_row(sql, params, mapper, cache)? + .ok_or(rusqlite::Error::QueryReturnedNoRows)?) + } + + /// Helper for when you'd like to get a Vec of all the rows returned by a + /// query that takes named arguments. See also + /// `query_rows_and_then_named_cached`. + fn query_rows_and_then_named( + &self, + sql: &str, + params: &[(&str, &dyn ToSql)], + mapper: F, + ) -> Result, E> + where + Self: Sized, + E: From, + F: FnMut(&Row<'_>) -> Result, + { + crate::maybe_log_plan(self.conn(), sql, params); + query_rows_and_then_named(self.conn(), sql, params, mapper, false) + } + + /// Helper for when you'd like to get a Vec of all the rows returned by a + /// query that takes named arguments. + fn query_rows_and_then_named_cached( + &self, + sql: &str, + params: &[(&str, &dyn ToSql)], + mapper: F, + ) -> Result, E> + where + Self: Sized, + E: From, + F: FnMut(&Row<'_>) -> Result, + { + crate::maybe_log_plan(self.conn(), sql, params); + query_rows_and_then_named(self.conn(), sql, params, mapper, true) + } + + /// Like `query_rows_and_then_named`, but works if you want a non-Vec as a result. + /// # Example: + /// ```rust,no_run + /// # use std::collections::HashSet; + /// # use sql_support::ConnExt; + /// # use rusqlite::Connection; + /// fn get_visit_tombstones(conn: &Connection, id: i64) -> rusqlite::Result> { + /// Ok(conn.query_rows_into( + /// "SELECT visit_date FROM moz_historyvisit_tombstones + /// WHERE place_id = :place_id", + /// &[(":place_id", &id)], + /// |row| row.get::<_, i64>(0))?) + /// } + /// ``` + /// Note if the type isn't inferred, you'll have to do something gross like + /// `conn.query_rows_into::, _, _, _>(...)`. + fn query_rows_into( + &self, + sql: &str, + params: &[(&str, &dyn ToSql)], + mapper: F, + ) -> Result + where + Self: Sized, + E: From, + F: FnMut(&Row<'_>) -> Result, + Coll: FromIterator, + { + crate::maybe_log_plan(self.conn(), sql, params); + query_rows_and_then_named(self.conn(), sql, params, mapper, false) + } + + /// Same as `query_rows_into`, but caches the stmt if possible. + fn query_rows_into_cached( + &self, + sql: &str, + params: &[(&str, &dyn ToSql)], + mapper: F, + ) -> Result + where + Self: Sized, + E: From, + F: FnMut(&Row<'_>) -> Result, + Coll: FromIterator, + { + crate::maybe_log_plan(self.conn(), sql, params); + query_rows_and_then_named(self.conn(), sql, params, mapper, true) + } + + // This should probably have a longer name... + /// Like `query_row_and_then_named` but returns None instead of erroring if no such row exists. + fn try_query_row( + &self, + sql: &str, + params: &[(&str, &dyn ToSql)], + mapper: F, + cache: bool, + ) -> Result, E> + where + Self: Sized, + E: From, + F: FnOnce(&Row<'_>) -> Result, + { + crate::maybe_log_plan(self.conn(), sql, params); + let conn = self.conn(); + let mut stmt = MaybeCached::prepare(conn, sql, cache)?; + let mut rows = stmt.query_named(params)?; + rows.next()?.map(mapper).transpose() + } + + fn unchecked_transaction(&self) -> SqlResult> { + UncheckedTransaction::new(self.conn(), TransactionBehavior::Deferred) + } + + /// Begin `unchecked_transaction` with `TransactionBehavior::Immediate`. Use + /// when the first operation will be a read operation, that further writes + /// depend on for correctness. + fn unchecked_transaction_imm(&self) -> SqlResult> { + UncheckedTransaction::new(self.conn(), TransactionBehavior::Immediate) + } +} + +impl ConnExt for Connection { + #[inline] + fn conn(&self) -> &Connection { + self + } +} + +impl<'conn> ConnExt for Transaction<'conn> { + #[inline] + fn conn(&self) -> &Connection { + &*self + } +} + +impl<'conn> ConnExt for Savepoint<'conn> { + #[inline] + fn conn(&self) -> &Connection { + &*self + } +} + +/// rusqlite, in an attempt to save us from ourselves, needs a mutable ref to +/// a connection to start a transaction. That is a bit of a PITA in some cases, +/// so we offer this as an alternative - but the responsibility of ensuring +/// there are no concurrent transactions is on our head. +/// +/// This is very similar to the rusqlite `Transaction` - it doesn't prevent +/// against nested transactions but does allow you to use an immutable +/// `Connection`. +pub struct UncheckedTransaction<'conn> { + pub conn: &'conn Connection, + pub started_at: Instant, + pub finished: bool, + // we could add drop_behavior etc too, but we don't need it yet - we + // always rollback. +} + +impl<'conn> UncheckedTransaction<'conn> { + /// Begin a new unchecked transaction. Cannot be nested, but this is not + /// enforced by Rust (hence 'unchecked') - however, it is enforced by + /// SQLite; use a rusqlite `savepoint` for nested transactions. + pub fn new(conn: &'conn Connection, behavior: TransactionBehavior) -> SqlResult { + let query = match behavior { + TransactionBehavior::Deferred => "BEGIN DEFERRED", + TransactionBehavior::Immediate => "BEGIN IMMEDIATE", + TransactionBehavior::Exclusive => "BEGIN EXCLUSIVE", + _ => unreachable!(), + }; + conn.execute_batch(query) + .map(move |_| UncheckedTransaction { + conn, + started_at: Instant::now(), + finished: false, + }) + } + + /// Consumes and commits an unchecked transaction. + pub fn commit(mut self) -> SqlResult<()> { + if self.finished { + log::warn!("ignoring request to commit an already finished transaction"); + return Ok(()); + } + self.finished = true; + self.conn.execute_batch("COMMIT")?; + log::debug!("Transaction commited after {:?}", self.started_at.elapsed()); + Ok(()) + } + + /// Consumes and rolls back an unchecked transaction. + pub fn rollback(mut self) -> SqlResult<()> { + if self.finished { + log::warn!("ignoring request to rollback an already finished transaction"); + return Ok(()); + } + self.rollback_() + } + + fn rollback_(&mut self) -> SqlResult<()> { + self.finished = true; + self.conn.execute_batch("ROLLBACK")?; + Ok(()) + } + + fn finish_(&mut self) -> SqlResult<()> { + if self.finished || self.conn.is_autocommit() { + return Ok(()); + } + self.rollback_()?; + Ok(()) + } +} + +impl<'conn> Deref for UncheckedTransaction<'conn> { + type Target = Connection; + + #[inline] + fn deref(&self) -> &Connection { + self.conn + } +} + +impl<'conn> Drop for UncheckedTransaction<'conn> { + fn drop(&mut self) { + if let Err(e) = self.finish_() { + log::warn!("Error dropping an unchecked transaction: {}", e); + } + } +} + +impl<'conn> ConnExt for UncheckedTransaction<'conn> { + #[inline] + fn conn(&self) -> &Connection { + &*self + } +} + +fn query_rows_and_then_named( + conn: &Connection, + sql: &str, + params: &[(&str, &dyn ToSql)], + mapper: F, + cache: bool, +) -> Result +where + E: From, + F: FnMut(&Row<'_>) -> Result, + Coll: FromIterator, +{ + let mut stmt = conn.prepare_maybe_cached(sql, cache)?; + let iter = stmt.query_and_then_named(params, mapper)?; + Ok(iter.collect::>()?) +} diff --git a/third_party/rust/sql-support/src/each_chunk.rs b/third_party/rust/sql-support/src/each_chunk.rs new file mode 100644 index 000000000000..2d738bcb37ab --- /dev/null +++ b/third_party/rust/sql-support/src/each_chunk.rs @@ -0,0 +1,311 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use lazy_static::lazy_static; +use rusqlite::{self, limits::Limit, types::ToSql}; +use std::iter::Map; +use std::slice::Iter; + +/// Returns SQLITE_LIMIT_VARIABLE_NUMBER as read from an in-memory connection and cached. +/// connection and cached. That means this will return the wrong value if it's set to a lower +/// value for a connection using this will return the wrong thing, but doing so is rare enough +/// that we explicitly don't support it (why would you want to lower this at runtime?). +/// +/// If you call this and the actual value was set to a negative number or zero (nothing prevents +/// this beyond a warning in the SQLite documentation), we panic. However, it's unlikely you can +/// run useful queries if this happened anyway. +pub fn default_max_variable_number() -> usize { + lazy_static! { + static ref MAX_VARIABLE_NUMBER: usize = { + let conn = rusqlite::Connection::open_in_memory() + .expect("Failed to initialize in-memory connection (out of memory?)"); + + let limit = conn.limit(Limit::SQLITE_LIMIT_VARIABLE_NUMBER); + assert!( + limit > 0, + "Illegal value for SQLITE_LIMIT_VARIABLE_NUMBER (must be > 0) {}", + limit + ); + limit as usize + }; + } + *MAX_VARIABLE_NUMBER +} + +/// Helper for the case where you have a `&[impl ToSql]` of arbitrary length, but need one +/// of no more than the connection's `MAX_VARIABLE_NUMBER` (rather, +/// `default_max_variable_number()`). This is useful when performing batched updates. +/// +/// The `do_chunk` callback is called with a slice of no more than `default_max_variable_number()` +/// items as it's first argument, and the offset from the start as it's second. +/// +/// See `each_chunk_mapped` for the case where `T` doesn't implement `ToSql`, but can be +/// converted to something that does. +pub fn each_chunk<'a, T, E, F>(items: &'a [T], do_chunk: F) -> Result<(), E> +where + T: 'a, + F: FnMut(&'a [T], usize) -> Result<(), E>, +{ + each_sized_chunk(items, default_max_variable_number(), do_chunk) +} + +/// A version of `each_chunk` for the case when the conversion to `to_sql` requires an custom +/// intermediate step. For example, you might want to grab a property off of an arrray of records +pub fn each_chunk_mapped<'a, T, U, E, Mapper, DoChunk>( + items: &'a [T], + to_sql: Mapper, + do_chunk: DoChunk, +) -> Result<(), E> +where + T: 'a, + U: ToSql + 'a, + Mapper: Fn(&'a T) -> U, + DoChunk: FnMut(Map, &'_ Mapper>, usize) -> Result<(), E>, +{ + each_sized_chunk_mapped(items, default_max_variable_number(), to_sql, do_chunk) +} + +// Split out for testing. Separate so that we can pass an actual slice +// to the callback if they don't need mapping. We could probably unify +// this with each_sized_chunk_mapped with a lot of type system trickery, +// but one of the benefits to each_chunk over the mapped versions is +// that the declaration is simpler. +pub fn each_sized_chunk<'a, T, E, F>( + items: &'a [T], + chunk_size: usize, + mut do_chunk: F, +) -> Result<(), E> +where + T: 'a, + F: FnMut(&'a [T], usize) -> Result<(), E>, +{ + if items.is_empty() { + return Ok(()); + } + let mut offset = 0; + for chunk in items.chunks(chunk_size) { + do_chunk(chunk, offset)?; + offset += chunk.len(); + } + Ok(()) +} + +/// Utility to help perform batched updates, inserts, queries, etc. This is the low-level version +/// of this utility which is wrapped by `each_chunk` and `each_chunk_mapped`, and it allows you to +/// provide both the mapping function, and the chunk size. +/// +/// Note: `mapped` basically just refers to the translating of `T` to some `U` where `U: ToSql` +/// using the `to_sql` function. This is useful for e.g. inserting the IDs of a large list +/// of records. +pub fn each_sized_chunk_mapped<'a, T, U, E, Mapper, DoChunk>( + items: &'a [T], + chunk_size: usize, + to_sql: Mapper, + mut do_chunk: DoChunk, +) -> Result<(), E> +where + T: 'a, + U: ToSql + 'a, + Mapper: Fn(&'a T) -> U, + DoChunk: FnMut(Map, &'_ Mapper>, usize) -> Result<(), E>, +{ + if items.is_empty() { + return Ok(()); + } + let mut offset = 0; + for chunk in items.chunks(chunk_size) { + let mapped = chunk.iter().map(&to_sql); + do_chunk(mapped, offset)?; + offset += chunk.len(); + } + Ok(()) +} + +#[cfg(test)] +fn check_chunk(items: C, expect: &[T], desc: &str) +where + C: IntoIterator, + ::Item: ToSql, + T: ToSql, +{ + let items = items.into_iter().collect::>(); + assert_eq!(items.len(), expect.len()); + // Can't quite make the borrowing work out here w/o a loop, oh well. + for (idx, (got, want)) in items.iter().zip(expect.iter()).enumerate() { + assert_eq!( + got.to_sql().unwrap(), + want.to_sql().unwrap(), + // ToSqlOutput::Owned(Value::Integer(*num)), + "{}: Bad value at index {}", + desc, + idx + ); + } +} + +#[cfg(test)] +mod test_mapped { + use super::*; + + #[test] + fn test_separate() { + let mut iteration = 0; + each_sized_chunk_mapped( + &[1, 2, 3, 4, 5], + 3, + |item| item as &dyn ToSql, + |chunk, offset| { + match offset { + 0 => { + assert_eq!(iteration, 0); + check_chunk(chunk, &[1, 2, 3], "first chunk"); + } + 3 => { + assert_eq!(iteration, 1); + check_chunk(chunk, &[4, 5], "second chunk"); + } + n => { + panic!("Unexpected offset {}", n); + } + } + iteration += 1; + Ok::<(), ()>(()) + }, + ) + .unwrap(); + } + + #[test] + fn test_leq_chunk_size() { + for &check_size in &[5, 6] { + let mut iteration = 0; + each_sized_chunk_mapped( + &[1, 2, 3, 4, 5], + check_size, + |item| item as &dyn ToSql, + |chunk, offset| { + assert_eq!(iteration, 0); + iteration += 1; + assert_eq!(offset, 0); + check_chunk(chunk, &[1, 2, 3, 4, 5], "only iteration"); + Ok::<(), ()>(()) + }, + ) + .unwrap(); + } + } + + #[test] + fn test_empty_chunk() { + let items: &[i64] = &[]; + each_sized_chunk_mapped::<_, _, (), _, _>( + items, + 100, + |item| item as &dyn ToSql, + |_, _| { + panic!("Should never be called"); + }, + ) + .unwrap(); + } + + #[test] + fn test_error() { + let mut iteration = 0; + let e = each_sized_chunk_mapped( + &[1, 2, 3, 4, 5, 6, 7], + 3, + |item| item as &dyn ToSql, + |_, offset| { + if offset == 0 { + assert_eq!(iteration, 0); + iteration += 1; + Ok(()) + } else if offset == 3 { + assert_eq!(iteration, 1); + iteration += 1; + Err("testing".to_string()) + } else { + // Make sure we stopped after the error. + panic!("Shouldn't get called with offset of {}", offset); + } + }, + ) + .expect_err("Should be an error"); + assert_eq!(e, "testing"); + } +} + +#[cfg(test)] +mod test_unmapped { + use super::*; + + #[test] + fn test_separate() { + let mut iteration = 0; + each_sized_chunk(&[1, 2, 3, 4, 5], 3, |chunk, offset| { + match offset { + 0 => { + assert_eq!(iteration, 0); + check_chunk(chunk, &[1, 2, 3], "first chunk"); + } + 3 => { + assert_eq!(iteration, 1); + check_chunk(chunk, &[4, 5], "second chunk"); + } + n => { + panic!("Unexpected offset {}", n); + } + } + iteration += 1; + Ok::<(), ()>(()) + }) + .unwrap(); + } + + #[test] + fn test_leq_chunk_size() { + for &check_size in &[5, 6] { + let mut iteration = 0; + each_sized_chunk(&[1, 2, 3, 4, 5], check_size, |chunk, offset| { + assert_eq!(iteration, 0); + iteration += 1; + assert_eq!(offset, 0); + check_chunk(chunk, &[1, 2, 3, 4, 5], "only iteration"); + Ok::<(), ()>(()) + }) + .unwrap(); + } + } + + #[test] + fn test_empty_chunk() { + let items: &[i64] = &[]; + each_sized_chunk::<_, (), _>(items, 100, |_, _| { + panic!("Should never be called"); + }) + .unwrap(); + } + + #[test] + fn test_error() { + let mut iteration = 0; + let e = each_sized_chunk(&[1, 2, 3, 4, 5, 6, 7], 3, |_, offset| { + if offset == 0 { + assert_eq!(iteration, 0); + iteration += 1; + Ok(()) + } else if offset == 3 { + assert_eq!(iteration, 1); + iteration += 1; + Err("testing".to_string()) + } else { + // Make sure we stopped after the error. + panic!("Shouldn't get called with offset of {}", offset); + } + }) + .expect_err("Should be an error"); + assert_eq!(e, "testing"); + } +} diff --git a/third_party/rust/sql-support/src/interrupt.rs b/third_party/rust/sql-support/src/interrupt.rs new file mode 100644 index 000000000000..1e6a8e2ce1a7 --- /dev/null +++ b/third_party/rust/sql-support/src/interrupt.rs @@ -0,0 +1,89 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use ffi_support::implement_into_ffi_by_pointer; +use interrupt_support::Interruptee; +use rusqlite::InterruptHandle; +use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, +}; + +// SeqCst is overkill for much of this, but whatever. + +/// A Sync+Send type which can be used allow someone to interrupt an +/// operation, even if it happens while rust code (and not SQL) is +/// executing. +pub struct SqlInterruptHandle { + db_handle: InterruptHandle, + interrupt_counter: Arc, +} + +impl SqlInterruptHandle { + pub fn new( + db_handle: InterruptHandle, + interrupt_counter: Arc, + ) -> SqlInterruptHandle { + SqlInterruptHandle { + db_handle, + interrupt_counter, + } + } + + pub fn interrupt(&self) { + self.interrupt_counter.fetch_add(1, Ordering::SeqCst); + self.db_handle.interrupt(); + } +} + +implement_into_ffi_by_pointer!(SqlInterruptHandle); + +/// A helper that can be used to determine if an interrupt request has come in while +/// the object lives. This is used to avoid a case where we aren't running any +/// queries when the request to stop comes in, but we're still not done (for example, +/// maybe we've run some of the autocomplete matchers, and are about to start +/// running the others. If we rely solely on sqlite3_interrupt(), we'd miss +/// the message that we should stop). +#[derive(Debug)] +pub struct SqlInterruptScope { + // The value of the interrupt counter when the scope began + start_value: usize, + // This could be &'conn AtomicUsize, but it would prevent the connection + // from being mutably borrowed for no real reason... + ptr: Arc, +} + +impl SqlInterruptScope { + #[inline] + pub fn new(ptr: Arc) -> Self { + let start_value = ptr.load(Ordering::SeqCst); + Self { start_value, ptr } + } + /// Add this as an inherent method to reduce the amount of things users have to bring in. + #[inline] + pub fn err_if_interrupted(&self) -> Result<(), interrupt_support::Interrupted> { + ::err_if_interrupted(self) + } +} + +impl Interruptee for SqlInterruptScope { + #[inline] + fn was_interrupted(&self) -> bool { + self.ptr.load(Ordering::SeqCst) != self.start_value + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_sync_send() { + fn is_sync() {} + fn is_send() {} + // Make sure this compiles + is_sync::(); + is_send::(); + } +} diff --git a/third_party/rust/sql-support/src/lib.rs b/third_party/rust/sql-support/src/lib.rs new file mode 100644 index 000000000000..d3faa3eb2e7f --- /dev/null +++ b/third_party/rust/sql-support/src/lib.rs @@ -0,0 +1,39 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#![allow(unknown_lints)] +#![warn(rust_2018_idioms)] + +mod conn_ext; +mod each_chunk; +mod interrupt; +mod maybe_cached; +mod query_plan; +mod repeat; + +pub use crate::conn_ext::*; +pub use crate::each_chunk::*; +pub use crate::interrupt::*; +pub use crate::maybe_cached::*; +pub use crate::query_plan::*; +pub use crate::repeat::*; + +/// In PRAGMA foo='bar', `'bar'` must be a constant string (it cannot be a +/// bound parameter), so we need to escape manually. According to +/// https://www.sqlite.org/faq.html, the only character that must be escaped is +/// the single quote, which is escaped by placing two single quotes in a row. +pub fn escape_string_for_pragma(s: &str) -> String { + s.replace("'", "''") +} + +#[cfg(test)] +mod test { + use super::*; + #[test] + fn test_escape_string_for_pragma() { + assert_eq!(escape_string_for_pragma("foobar"), "foobar"); + assert_eq!(escape_string_for_pragma("'foo'bar'"), "''foo''bar''"); + assert_eq!(escape_string_for_pragma("''"), "''''"); + } +} diff --git a/third_party/rust/sql-support/src/maybe_cached.rs b/third_party/rust/sql-support/src/maybe_cached.rs new file mode 100644 index 000000000000..96f99f490cac --- /dev/null +++ b/third_party/rust/sql-support/src/maybe_cached.rs @@ -0,0 +1,64 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use rusqlite::{self, CachedStatement, Connection, Statement}; + +use std::ops::{Deref, DerefMut}; + +/// MaybeCached is a type that can be used to help abstract +/// over cached and uncached rusqlite statements in a transparent manner. +pub enum MaybeCached<'conn> { + Uncached(Statement<'conn>), + Cached(CachedStatement<'conn>), +} + +impl<'conn> Deref for MaybeCached<'conn> { + type Target = Statement<'conn>; + #[inline] + fn deref(&self) -> &Statement<'conn> { + match self { + MaybeCached::Cached(cached) => Deref::deref(cached), + MaybeCached::Uncached(uncached) => uncached, + } + } +} + +impl<'conn> DerefMut for MaybeCached<'conn> { + #[inline] + fn deref_mut(&mut self) -> &mut Statement<'conn> { + match self { + MaybeCached::Cached(cached) => DerefMut::deref_mut(cached), + MaybeCached::Uncached(uncached) => uncached, + } + } +} + +impl<'conn> From> for MaybeCached<'conn> { + #[inline] + fn from(stmt: Statement<'conn>) -> Self { + MaybeCached::Uncached(stmt) + } +} + +impl<'conn> From> for MaybeCached<'conn> { + #[inline] + fn from(stmt: CachedStatement<'conn>) -> Self { + MaybeCached::Cached(stmt) + } +} + +impl<'conn> MaybeCached<'conn> { + #[inline] + pub fn prepare( + conn: &'conn Connection, + sql: &str, + cached: bool, + ) -> rusqlite::Result> { + if cached { + Ok(MaybeCached::Cached(conn.prepare_cached(sql)?)) + } else { + Ok(MaybeCached::Uncached(conn.prepare(sql)?)) + } + } +} diff --git a/third_party/rust/sql-support/src/query_plan.rs b/third_party/rust/sql-support/src/query_plan.rs new file mode 100644 index 000000000000..59328119c25c --- /dev/null +++ b/third_party/rust/sql-support/src/query_plan.rs @@ -0,0 +1,182 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use rusqlite::{types::ToSql, Connection, Result as SqlResult}; + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct QueryPlanStep { + pub node_id: i32, + pub parent_id: i32, + pub aux: i32, + pub detail: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct QueryPlan { + pub query: String, + pub plan: Vec, +} + +impl QueryPlan { + // TODO: support positional params (it's a pain...) + pub fn new(conn: &Connection, sql: &str, params: &[(&str, &dyn ToSql)]) -> SqlResult { + let plan_sql = format!("EXPLAIN QUERY PLAN {}", sql); + let mut stmt = conn.prepare(&plan_sql)?; + let plan = stmt + .query_and_then_named(params, |row| -> SqlResult<_> { + Ok(QueryPlanStep { + node_id: row.get(0)?, + parent_id: row.get(1)?, + aux: row.get(2)?, + detail: row.get(3)?, + }) + })? + .collect::, _>>()?; + Ok(QueryPlan { + query: sql.into(), + plan, + }) + } + + pub fn print_pretty_tree(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + if self.plan.is_empty() { + return writeln!(f, ""); + } + writeln!(f, "QUERY PLAN")?; + let children = self + .plan + .iter() + .filter(|e| e.parent_id == 0) + .collect::>(); + for (i, child) in children.iter().enumerate() { + let last = i == children.len() - 1; + self.print_tree(f, child, "", last)?; + } + Ok(()) + } + + fn print_tree( + &self, + f: &mut std::fmt::Formatter<'_>, + entry: &QueryPlanStep, + prefix: &str, + last_child: bool, + ) -> std::fmt::Result { + let children = self + .plan + .iter() + .filter(|e| e.parent_id == entry.node_id) + .collect::>(); + let next_prefix = if last_child { + writeln!(f, "{}`--{}", prefix, entry.detail)?; + format!("{} ", prefix) + } else { + writeln!(f, "{}|--{}", prefix, entry.detail)?; + format!("{}| ", prefix) + }; + for (i, child) in children.iter().enumerate() { + let last = i == children.len() - 1; + self.print_tree(f, child, &next_prefix, last)?; + } + Ok(()) + } +} + +impl std::fmt::Display for QueryPlan { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + writeln!(f, "### QUERY PLAN")?; + writeln!(f, "#### SQL:\n{}\n#### PLAN:", self.query)?; + self.print_pretty_tree(f)?; + writeln!(f, "### END QUERY PLAN") + } +} + +/// Log a query plan if the `log_query_plans` feature is enabled and it hasn't been logged yet. +#[inline] +pub fn maybe_log_plan(_conn: &Connection, _sql: &str, _params: &[(&str, &dyn ToSql)]) { + // Note: underscores ar needed becasue those go unused if the feature is not turned on. + #[cfg(feature = "log_query_plans")] + { + plan_log::log_plan(_conn, _sql, _params) + } +} + +#[cfg(feature = "log_query_plans")] +mod plan_log { + use super::*; + use std::collections::HashMap; + use std::io::Write; + use std::sync::Mutex; + + struct PlanLogger { + seen: HashMap, + out: Box, + } + + impl PlanLogger { + fn new() -> Self { + let out_file = std::env::var("QUERY_PLAN_LOG").unwrap_or_default(); + let output: Box = if out_file != "" { + let mut file = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(out_file) + .expect("QUERY_PLAN_LOG file does not exist!"); + writeln!( + file, + "\n\n# Query Plan Log starting at time: {:?}\n", + std::time::SystemTime::now() + ) + .expect("Failed to write to plan log file"); + Box::new(file) + } else { + println!("QUERY_PLAN_LOG was not set, logging to stdout"); + Box::new(std::io::stdout()) + }; + Self { + seen: Default::default(), + out: output, + } + } + + fn maybe_log(&mut self, plan: QueryPlan) { + use std::collections::hash_map::Entry; + match self.seen.entry(plan.query.clone()) { + Entry::Occupied(mut o) => { + if o.get() == &plan { + return; + } + // Ignore IO failures. + let _ = writeln!(self.out, "### QUERY PLAN CHANGED!\n{}", plan); + o.insert(plan); + } + Entry::Vacant(v) => { + let _ = writeln!(self.out, "{}", plan); + v.insert(plan); + } + } + let _ = self.out.flush(); + } + } + + lazy_static::lazy_static! { + static ref PLAN_LOGGER: Mutex = Mutex::new(PlanLogger::new()); + } + + pub fn log_plan(conn: &Connection, sql: &str, params: &[(&str, &dyn ToSql)]) { + if sql.starts_with("EXPLAIN") { + return; + } + let plan = match QueryPlan::new(conn, sql, params) { + Ok(plan) => plan, + Err(e) => { + // We're usually doing this during tests where logs often arent available + eprintln!("Failed to get query plan for {}: {}", sql, e); + return; + } + }; + let mut logger = PLAN_LOGGER.lock().unwrap(); + logger.maybe_log(plan); + } +} diff --git a/third_party/rust/sql-support/src/repeat.rs b/third_party/rust/sql-support/src/repeat.rs new file mode 100644 index 000000000000..40b582ec1411 --- /dev/null +++ b/third_party/rust/sql-support/src/repeat.rs @@ -0,0 +1,113 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use std::fmt; + +/// Helper type for printing repeated strings more efficiently. You should use +/// [`repeat_display`](sql_support::repeat_display), or one of the `repeat_sql_*` helpers to +/// construct it. +#[derive(Debug, Clone)] +pub struct RepeatDisplay<'a, F> { + count: usize, + sep: &'a str, + fmt_one: F, +} + +impl<'a, F> fmt::Display for RepeatDisplay<'a, F> +where + F: Fn(usize, &mut fmt::Formatter<'_>) -> fmt::Result, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + for i in 0..self.count { + if i != 0 { + f.write_str(self.sep)?; + } + (self.fmt_one)(i, f)?; + } + Ok(()) + } +} + +/// Construct a RepeatDisplay that will repeatedly call `fmt_one` with a formatter `count` times, +/// separated by `sep`. +/// +/// # Example +/// +/// ```rust +/// # use sql_support::repeat_display; +/// assert_eq!(format!("{}", repeat_display(1, ",", |i, f| write!(f, "({},?)", i))), +/// "(0,?)"); +/// assert_eq!(format!("{}", repeat_display(2, ",", |i, f| write!(f, "({},?)", i))), +/// "(0,?),(1,?)"); +/// assert_eq!(format!("{}", repeat_display(3, ",", |i, f| write!(f, "({},?)", i))), +/// "(0,?),(1,?),(2,?)"); +/// ``` +#[inline] +pub fn repeat_display(count: usize, sep: &str, fmt_one: F) -> RepeatDisplay<'_, F> +where + F: Fn(usize, &mut fmt::Formatter<'_>) -> fmt::Result, +{ + RepeatDisplay { + count, + sep, + fmt_one, + } +} + +/// Returns a value that formats as `count` instances of `?` separated by commas. +/// +/// # Example +/// +/// ```rust +/// # use sql_support::repeat_sql_vars; +/// assert_eq!(format!("{}", repeat_sql_vars(0)), ""); +/// assert_eq!(format!("{}", repeat_sql_vars(1)), "?"); +/// assert_eq!(format!("{}", repeat_sql_vars(2)), "?,?"); +/// assert_eq!(format!("{}", repeat_sql_vars(3)), "?,?,?"); +/// ``` +pub fn repeat_sql_vars(count: usize) -> impl fmt::Display { + repeat_display(count, ",", |_, f| write!(f, "?")) +} + +/// Returns a value that formats as `count` instances of `(?)` separated by commas. +/// +/// # Example +/// +/// ```rust +/// # use sql_support::repeat_sql_values; +/// assert_eq!(format!("{}", repeat_sql_values(0)), ""); +/// assert_eq!(format!("{}", repeat_sql_values(1)), "(?)"); +/// assert_eq!(format!("{}", repeat_sql_values(2)), "(?),(?)"); +/// assert_eq!(format!("{}", repeat_sql_values(3)), "(?),(?),(?)"); +/// ``` +/// +pub fn repeat_sql_values(count: usize) -> impl fmt::Display { + // We could also implement this as `repeat_sql_multi_values(count, 1)`, + // but this is faster and no less clear IMO. + repeat_display(count, ",", |_, f| write!(f, "(?)")) +} + +/// Returns a value that formats as `num_values` instances of `(?,?,?,...)` (where there are +/// `vars_per_value` question marks separated by commas in between the `?`s). +/// +/// Panics if `vars_per_value` is zero (however, `num_values` is allowed to be zero). +/// +/// # Example +/// +/// ```rust +/// # use sql_support::repeat_multi_values; +/// assert_eq!(format!("{}", repeat_multi_values(0, 2)), ""); +/// assert_eq!(format!("{}", repeat_multi_values(1, 5)), "(?,?,?,?,?)"); +/// assert_eq!(format!("{}", repeat_multi_values(2, 3)), "(?,?,?),(?,?,?)"); +/// assert_eq!(format!("{}", repeat_multi_values(3, 1)), "(?),(?),(?)"); +/// ``` +pub fn repeat_multi_values(num_values: usize, vars_per_value: usize) -> impl fmt::Display { + assert_ne!( + vars_per_value, 0, + "Illegal value for `vars_per_value`, must not be zero" + ); + repeat_display(num_values, ",", move |_, f| { + write!(f, "({})", repeat_sql_vars(vars_per_value)) + }) +} diff --git a/third_party/rust/sync-guid/.cargo-checksum.json b/third_party/rust/sync-guid/.cargo-checksum.json index 6123bab2b13d..f95c584de772 100644 --- a/third_party/rust/sync-guid/.cargo-checksum.json +++ b/third_party/rust/sync-guid/.cargo-checksum.json @@ -1 +1 @@ -{"files":{"Cargo.toml":"b5cc525d2aa129f84cb3f729a579217591c7705e2be78dbd348a95fc354831be","src/lib.rs":"729e562be4e63ec7db2adc00753a019ae77c11ce82637a893ea18122580c3c98","src/rusqlite_support.rs":"827d314605d8c741efdf238a0780a891c88bc56026a3e6dcfa534772a4852fb3","src/serde_support.rs":"519b5eb59ca7be555d522f2186909db969069dc9586a5fe4047d4ec176b2368a"},"package":null} \ No newline at end of file +{"files":{"Cargo.toml":"fec1d023581c5e34b5669c1b42efd11819eba4c3c29eca1f6095f6044a1fa5ae","src/lib.rs":"729e562be4e63ec7db2adc00753a019ae77c11ce82637a893ea18122580c3c98","src/rusqlite_support.rs":"827d314605d8c741efdf238a0780a891c88bc56026a3e6dcfa534772a4852fb3","src/serde_support.rs":"519b5eb59ca7be555d522f2186909db969069dc9586a5fe4047d4ec176b2368a"},"package":null} \ No newline at end of file diff --git a/third_party/rust/sync-guid/Cargo.toml b/third_party/rust/sync-guid/Cargo.toml index ca885a95c4c9..a82aedea169c 100644 --- a/third_party/rust/sync-guid/Cargo.toml +++ b/third_party/rust/sync-guid/Cargo.toml @@ -6,7 +6,7 @@ license = "MPL-2.0" edition = "2018" [dependencies] -rusqlite = { version = "0.21.0", optional = true } +rusqlite = { version = "0.23.1", optional = true } serde = { version = "1.0.104", optional = true } rand = { version = "0.7", optional = true } base64 = { version = "0.12.0", optional = true } diff --git a/third_party/rust/sync15-traits/.cargo-checksum.json b/third_party/rust/sync15-traits/.cargo-checksum.json index e90493eb8b84..42e30d9d8068 100644 --- a/third_party/rust/sync15-traits/.cargo-checksum.json +++ b/third_party/rust/sync15-traits/.cargo-checksum.json @@ -1 +1 @@ -{"files":{"Cargo.toml":"326b1c017a76b1987e34c6dde0fa57f2c85d5de23a9f0cf1dfb029cc99d34471","README.md":"396105211d8ce7f40b05d8062d7ab55d99674555f3ac81c061874ae26656ed7e","src/changeset.rs":"442aa92b5130ec0f8f2b0054acb399c547380e0060015cbf4ca7a72027440d54","src/client.rs":"6be4f550ade823fafc350c5490e031f90a4af833a9bba9739b05568464255a74","src/lib.rs":"9abce82e0248c8aa7e3d55b7db701b95e8f337f6e5d1319381f995a0b708400d","src/payload.rs":"09db1a444e7893990a4f03cb16263b9c15abc9e48ec4f1343227be1b490865a5","src/request.rs":"9e656ec487e53c7485643687e605d73bb25e138056e920d6f4b7d63fc6a8c460","src/server_timestamp.rs":"43d1b98a90e55e49380a0b66c209c9eb393e2aeaa27d843a4726d93cdd4cea02","src/store.rs":"10e215dd24270b6bec10903ac1d5274ce997eb437134f43be7de44e36fb9d1e4","src/telemetry.rs":"027befb099a6fcded3457f7e566296548a0898ff613267190621856b9ef288f6"},"package":null} \ No newline at end of file +{"files":{"Cargo.toml":"656c4c4af39bcf924098be33996360250f9610ee3a4090b8152b68bdad03c46e","README.md":"396105211d8ce7f40b05d8062d7ab55d99674555f3ac81c061874ae26656ed7e","src/bridged_engine.rs":"b4d45cd43db3e5926df614ae9706b8d1a5bb96860577463d05b56a4213532ec1","src/changeset.rs":"442aa92b5130ec0f8f2b0054acb399c547380e0060015cbf4ca7a72027440d54","src/client.rs":"6be4f550ade823fafc350c5490e031f90a4af833a9bba9739b05568464255a74","src/lib.rs":"c1ca44e7bb6477b8018bd554479021dbf52754e64577185b3f7e208ae45bf754","src/payload.rs":"09db1a444e7893990a4f03cb16263b9c15abc9e48ec4f1343227be1b490865a5","src/request.rs":"9e656ec487e53c7485643687e605d73bb25e138056e920d6f4b7d63fc6a8c460","src/server_timestamp.rs":"43d1b98a90e55e49380a0b66c209c9eb393e2aeaa27d843a4726d93cdd4cea02","src/store.rs":"10e215dd24270b6bec10903ac1d5274ce997eb437134f43be7de44e36fb9d1e4","src/telemetry.rs":"027befb099a6fcded3457f7e566296548a0898ff613267190621856b9ef288f6"},"package":null} \ No newline at end of file diff --git a/third_party/rust/sync15-traits/Cargo.toml b/third_party/rust/sync15-traits/Cargo.toml index 7aa6503f8b0d..bb8268bd337e 100644 --- a/third_party/rust/sync15-traits/Cargo.toml +++ b/third_party/rust/sync15-traits/Cargo.toml @@ -16,3 +16,5 @@ log = "0.4" ffi-support = "0.4" url = "2.1" failure = "0.1.6" + +interrupt-support = { path = "../interrupt" } diff --git a/third_party/rust/sync15-traits/src/bridged_engine.rs b/third_party/rust/sync15-traits/src/bridged_engine.rs new file mode 100644 index 000000000000..6aba5e2798f7 --- /dev/null +++ b/third_party/rust/sync15-traits/src/bridged_engine.rs @@ -0,0 +1,197 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use std::{sync::Mutex, sync::MutexGuard, sync::PoisonError}; + +use interrupt_support::Interruptee; + +/// A bridged Sync engine implements all the methods needed to support +/// Desktop Sync. +pub trait BridgedEngine { + /// The type returned for errors. + type Error; + + /// Initializes the engine. This is called once, when the engine is first + /// created, and guaranteed to be called before any of the other methods. + /// The default implementation does nothing. + fn initialize(&self) -> Result<(), Self::Error> { + Ok(()) + } + + /// Returns the last sync time, in milliseconds, for this engine's + /// collection. This is called before each sync, to determine the lower + /// bound for new records to fetch from the server. + fn last_sync(&self) -> Result; + + /// Sets the last sync time, in milliseconds. This is called throughout + /// the sync, to fast-forward the stored last sync time to match the + /// timestamp on the uploaded records. + fn set_last_sync(&self, last_sync_millis: i64) -> Result<(), Self::Error>; + + /// Returns the sync ID for this engine's collection. This is only used in + /// tests. + fn sync_id(&self) -> Result, Self::Error>; + + /// Resets the sync ID for this engine's collection, returning the new ID. + /// As a side effect, implementations should reset all local Sync state, + /// as in `reset`. + fn reset_sync_id(&self) -> Result; + + /// Ensures that the locally stored sync ID for this engine's collection + /// matches the `new_sync_id` from the server. If the two don't match, + /// implementations should reset all local Sync state, as in `reset`. + /// This method returns the assigned sync ID, which can be either the + /// `new_sync_id`, or a different one if the engine wants to force other + /// devices to reset their Sync state for this collection the next time they + /// sync. + fn ensure_current_sync_id(&self, new_sync_id: &str) -> Result; + + /// Stages a batch of incoming Sync records. This is called multiple + /// times per sync, once for each batch. Implementations can use the + /// signal to check if the operation was aborted, and cancel any + /// pending work. + fn store_incoming( + &self, + incoming_cleartexts: &[String], + signal: &dyn Interruptee, + ) -> Result<(), Self::Error>; + + /// Applies all staged records, reconciling changes on both sides and + /// resolving conflicts. Returns a list of records to upload. + fn apply(&self, signal: &dyn Interruptee) -> Result; + + /// Indicates that the given record IDs were uploaded successfully to the + /// server. This is called multiple times per sync, once for each batch + /// upload. + fn set_uploaded( + &self, + server_modified_millis: i64, + ids: &[String], + signal: &dyn Interruptee, + ) -> Result<(), Self::Error>; + + /// Indicates that all records have been uploaded. At this point, any record + /// IDs marked for upload that haven't been passed to `set_uploaded`, can be + /// assumed to have failed: for example, because the server rejected a record + /// with an invalid TTL or sort index. + fn sync_finished(&self, signal: &dyn Interruptee) -> Result<(), Self::Error>; + + /// Resets all local Sync state, including any change flags, mirrors, and + /// the last sync time, such that the next sync is treated as a first sync + /// with all new local data. Does not erase any local user data. + fn reset(&self) -> Result<(), Self::Error>; + + /// Erases all local user data for this collection, and any Sync metadata. + /// This method is destructive, and unused for most collections. + fn wipe(&self) -> Result<(), Self::Error>; + + /// Tears down the engine. The opposite of `initialize`, `finalize` is + /// called when an engine is disabled, or otherwise no longer needed. The + /// default implementation does nothing. + fn finalize(&self) -> Result<(), Self::Error> { + Ok(()) + } +} + +#[derive(Clone, Debug, Default)] +pub struct ApplyResults { + /// List of records + pub records: Vec, + /// The number of incoming records whose contents were merged because they + /// changed on both sides. None indicates we aren't reporting this + /// information. + pub num_reconciled: Option, +} + +impl ApplyResults { + pub fn new(records: Vec, num_reconciled: impl Into>) -> Self { + Self { + records, + num_reconciled: num_reconciled.into(), + } + } +} + +// Shorthand for engines that don't care. +impl From> for ApplyResults { + fn from(records: Vec) -> Self { + Self { + records, + num_reconciled: None, + } + } +} + +/// A blanket implementation of `BridgedEngine` for any `Mutex`. +/// This is provided for convenience, since we expect most bridges to hold +/// their engines in an `Arc>`. +impl BridgedEngine for Mutex +where + E: BridgedEngine, + E::Error: for<'a> From>>, +{ + type Error = E::Error; + + fn initialize(&self) -> Result<(), Self::Error> { + self.lock()?.initialize() + } + + fn last_sync(&self) -> Result { + self.lock()?.last_sync() + } + + fn set_last_sync(&self, millis: i64) -> Result<(), Self::Error> { + self.lock()?.set_last_sync(millis) + } + + fn store_incoming( + &self, + incoming_cleartexts: &[String], + signal: &dyn Interruptee, + ) -> Result<(), Self::Error> { + self.lock()?.store_incoming(incoming_cleartexts, signal) + } + + fn apply(&self, signal: &dyn Interruptee) -> Result { + self.lock()?.apply(signal) + } + + fn set_uploaded( + &self, + server_modified_millis: i64, + ids: &[String], + signal: &dyn Interruptee, + ) -> Result<(), Self::Error> { + self.lock()? + .set_uploaded(server_modified_millis, ids, signal) + } + + fn sync_finished(&self, signal: &dyn Interruptee) -> Result<(), Self::Error> { + self.lock()?.sync_finished(signal) + } + + fn reset(&self) -> Result<(), Self::Error> { + self.lock()?.reset() + } + + fn wipe(&self) -> Result<(), Self::Error> { + self.lock()?.wipe() + } + + fn finalize(&self) -> Result<(), Self::Error> { + self.lock()?.finalize() + } + + fn sync_id(&self) -> Result, Self::Error> { + self.lock()?.sync_id() + } + + fn reset_sync_id(&self) -> Result { + self.lock()?.reset_sync_id() + } + + fn ensure_current_sync_id(&self, new_sync_id: &str) -> Result { + self.lock()?.ensure_current_sync_id(new_sync_id) + } +} diff --git a/third_party/rust/sync15-traits/src/lib.rs b/third_party/rust/sync15-traits/src/lib.rs index 81858afdd2d2..d60747e6556e 100644 --- a/third_party/rust/sync15-traits/src/lib.rs +++ b/third_party/rust/sync15-traits/src/lib.rs @@ -3,6 +3,7 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ #![warn(rust_2018_idioms)] +mod bridged_engine; mod changeset; pub mod client; mod payload; @@ -11,6 +12,7 @@ mod server_timestamp; mod store; pub mod telemetry; +pub use bridged_engine::{ApplyResults, BridgedEngine}; pub use changeset::{IncomingChangeset, OutgoingChangeset, RecordChangeset}; pub use payload::Payload; pub use request::{CollectionRequest, RequestOrder}; diff --git a/third_party/rust/webext-storage/.cargo-checksum.json b/third_party/rust/webext-storage/.cargo-checksum.json new file mode 100644 index 000000000000..62f95ace233b --- /dev/null +++ b/third_party/rust/webext-storage/.cargo-checksum.json @@ -0,0 +1 @@ +{"files":{"Cargo.toml":"23ed53b7db21b1015cbb1deafe950f80068691dfe7f2256e9f9237a94910a4d8","README.md":"1fd617294339930ee1ad5172377648b268cce0216fc3971facbfe7c6839e9ab1","build.rs":"2b827a62155a3d724cdb4c198270ea467439e537403f82fa873321ac55a69a63","sql/create_schema.sql":"d50b22cb17fc5d4e2aa4d001e853bd2f67eb3ffdbb1ac29013067dceacaec80e","src/api.rs":"e045fd8f39a8774f5bd05054dcc50381bbd112ffc6638c42540792cdd001811d","src/db.rs":"7f74bcbd1f5bef3bc64f6eccbc89bccda51e130537692fed9cc9a417ff100c29","src/error.rs":"86ba215ec5a889d1ccca9dcd141e42f75914744a4803598ccf3894da4a7f7475","src/lib.rs":"1d40f86404bfd1bb70abe778fa306d8ad937fb47e281a6844975f1b2f37a6468","src/schema.rs":"0d5291ba9a553706e81d27a0d65618100b1bcb16edafe34139c715f84c84f1b4","src/store.rs":"dc1836bfa88b164783d218595358ba531de8eb87165ba3c1ea4075f71c1c3e21"},"package":null} \ No newline at end of file diff --git a/third_party/rust/webext-storage/Cargo.toml b/third_party/rust/webext-storage/Cargo.toml new file mode 100644 index 000000000000..d3bc04b155ac --- /dev/null +++ b/third_party/rust/webext-storage/Cargo.toml @@ -0,0 +1,38 @@ +[package] +name = "webext-storage" +edition = "2018" +version = "0.1.0" +authors = ["sync-team@mozilla.com"] +license = "MPL-2.0" + +[features] +log_query_plans = ["sql-support/log_query_plans"] +default = [] + +[dependencies] +error-support = { path = "../support/error" } +failure = "0.1.6" +interrupt-support = { path = "../support/interrupt" } +lazy_static = "1.4.0" +log = "0.4" +serde = "1" +serde_json = "1" +serde_derive = "1" +sql-support = { path = "../support/sql" } +sync-guid = { path = "../support/guid", features = ["rusqlite_support", "random"] } +url = { version = "2.1", features = ["serde"] } + +[dependencies.rusqlite] +version = "0.23.1" +features = ["functions", "bundled"] + +[dev-dependencies] +env_logger = "0.7.0" + +# A *direct* dep on the -sys crate is required for our build.rs +# to see the DEP_SQLITE3_LINK_TARGET env var that cargo sets +# on its behalf. +libsqlite3-sys = "0.18.0" + +[build-dependencies] +nss_build_common = { path = "../support/rc_crypto/nss/nss_build_common" } diff --git a/third_party/rust/webext-storage/README.md b/third_party/rust/webext-storage/README.md new file mode 100644 index 000000000000..6c1088af466f --- /dev/null +++ b/third_party/rust/webext-storage/README.md @@ -0,0 +1,91 @@ +# WebExtension Storage Component + +The WebExtension Storage component can be used to power an implementation of the +[`chrome.storage.sync`](https://developer.chrome.com/extensions/storage) WebExtension API, +which gives each WebExtensions its own private key-value store that will sync between a user's +devices. This particular implementation sits atop [Firefox Sync](../sync_manager/README.md). + +With a small amount of work, this component would also be capable of powering an implementation +of `chrome.storage.local`, but this is not an explicit goal at this stage. + +* [Features](#features) +* [Using the component](#using-the-component) +* [Working on the component](#working-on-the-component) + +## Features + +The WebExtension Storage component offers: + +1. Local storage of key-value data indexed by WebExtension ID. +1. Basic Create, Read, Update and Delete (CRUD) operations for items in the database. +1. Syncing of stored data between applications, via Firefox Sync. + +The component ***does not*** offer, but may offer in the future: + +1. Separate storage for key-value data that does not sync, per the + `chrome.storage.local` WebExtension API. +1. Import functionality from previous WebExtension storage implementations backed by + [Kinto](https://kinto-storage.org). + +The component ***does not*** offer, and we have no concrete plans to offer: + +1. Any facilities for loading or running WebExtensions, or exposing this data to them. +1. Any helpers to secure data access between different WebExtensions. + +As a consuming application, you will need to implement code that plumbs this component in to your +WebExtensions infrastructure, so that each WebExtension gets access to its own data (and only its +own data) stored in this component. + +## Using the component + +### Prerequisites + +To use this component for local storage of WebExtension data, you will need to know how to integrate appservices components +into an application on your target platform: +* **Firefox Desktop**: There's some custom bridging code in mozilla-central. +* **Android**: Bindings not yet available; please reach out on slack to discuss! +* **iOS**: Bindings not yet available; please reach out on slack to discuss! +* **Other Platforms**: We don't know yet; please reach out on slack to discuss! + +### Core Concepts + +* We assume each WebExtension is uniquely identified by an immutable **extension id**. +* A **WebExtenstion Store** is a database that maps extension ids to key-value JSON maps, one per extension. + It exposes methods that mirror those of the [`chrome.storage` spec](https://developer.chrome.com/extensions/storage) + (e.g. `get`, `set`, and `delete`) and which take an extension id as their first argument. + +## Working on the component + +### Prerequisites + +To effectively work on the WebExtension Storage component, you will need to be familiar with: + +* Our general [guidelines for contributors](../../docs/contributing.md). +* The [core concepts](#core-concepts) for users of the component, outlined above. +* The way we [generate ffi bindings](../../docs/howtos/building-a-rust-component.md) and expose them to + [Kotlin](../../docs/howtos/exposing-rust-components-to-kotlin.md) and + [Swift](../../docs/howtos/exposing-rust-components-to-swift.md). +* The key ideas behind [how Firefox Sync works](../../docs/synconomicon/) and the [sync15 crate](../sync15/README.md). + +### Storage Overview + +This component stores WebExtension data in a SQLite database, one row per extension id. +The key-value map data for each extension is stored as serialized JSON in a `TEXT` field; +this is nice and simple and helps ensure that the stored data has the semantics we want, +which are pretty much just the semantics of JSON. + +For syncing, we maintain a "mirror" table which contains one item per record known to +exist on the server. These items are identified by a randomly-generated GUID, in order +to hide the raw extension ids from the sync server. + +When uploading records to the server, we write one +[encrypted BSO](https://mozilla-services.readthedocs.io/en/latest/sync/storageformat5.html#collection-records) +per extension. Its server-visible id is the randomly-generated GUID, and its encrypted payload +contains the plaintext extension id and corresponding key-value map data. + +The end result is something like this (highly simplified!) diagram: + +[![storage overview diagram](https://docs.google.com/drawings/d/e/2PACX-1vSvCk0uJlXYTtWHmjxhL-mNLGL_q7F50LavltedREH8Ijuqjl875jKYd9PdJ5SrD3mhVOFqANs6A_NB/pub?w=727&h=546)](https://docs.google.com/drawings/d/1MlkFQJ7SUnW4WSEAF9e-2O34EnsAwUFi3Xcf0Lj3Hc8/) + +The details of the encryption are handled by the [sync15 crate](../sync15/README.md), following +the formats defied in [sync storage format v5](https://mozilla-services.readthedocs.io/en/latest/sync/storageformat5.html#collection-records). diff --git a/third_party/rust/webext-storage/build.rs b/third_party/rust/webext-storage/build.rs new file mode 100644 index 000000000000..b53386be4b86 --- /dev/null +++ b/third_party/rust/webext-storage/build.rs @@ -0,0 +1,16 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +//! Work around the fact that `sqlcipher` might get enabled by a cargo feature +//! another crate in the workspace needs, without setting up nss. (This is a +//! gross hack). + +fn main() { + println!("cargo:rerun-if-changed=build.rs"); + // Ugh. This is really really dumb. We don't care about sqlcipher at all. really + if nss_build_common::env_str("DEP_SQLITE3_LINK_TARGET") == Some("sqlcipher".into()) { + // If NSS_DIR isn't set, we don't really care, ignore the Err case. + let _ = nss_build_common::link_nss(); + } +} diff --git a/third_party/rust/webext-storage/sql/create_schema.sql b/third_party/rust/webext-storage/sql/create_schema.sql new file mode 100644 index 000000000000..7a02e84d5070 --- /dev/null +++ b/third_party/rust/webext-storage/sql/create_schema.sql @@ -0,0 +1,49 @@ +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +-- This is a very simple schema for a chrome.storage.* implementation. At time +-- of writing, only chrome.storage.sync is supported, but this can be trivially +-- enhanced to support chrome.storage.local (the api is identical, it's just a +-- different "bucket" and doesn't sync). +-- +-- Even though the spec allows for a single extension to have any number of +-- "keys", we've made the decision to store all keys for a given extension in a +-- single row as a JSON representation of all keys and values. +-- We've done this primarily due to: +-- * The shape of the API is very JSON, and it almost encourages multiple keys +-- to be fetched at one time. +-- * The defined max sizes that extensions are allowed to store using this API +-- is sufficiently small that we don't have many concerns around record sizes. +-- * We'd strongly prefer to keep one record per extension when syncing this +-- data, so having the local store in this shape makes syncing easier. + +CREATE TABLE IF NOT EXISTS storage_sync_data ( + ext_id TEXT NOT NULL PRIMARY KEY, + + /* The JSON payload. NULL means it's a tombstone */ + data TEXT, + + /* Same "sync change counter" strategy used by other components. */ + sync_change_counter INTEGER NOT NULL DEFAULT 1 +); + +CREATE TABLE IF NOT EXISTS storage_sync_mirror ( + guid TEXT NOT NULL PRIMARY KEY, + /* The extension_id is explicitly not the GUID used on the server. + We may end up making this a regular foreign-key relationship back to + storage_sync_data, although maybe not - the ext_id may not exist in + storage_sync_data at the time we populate this table. + We can iterate here as we ramp up sync support. + */ + ext_id TEXT NOT NULL UNIQUE, + + /* The JSON payload. We *do* allow NULL here - it means "deleted" */ + data TEXT +); + +-- This table holds key-value metadata - primarily for sync. +CREATE TABLE IF NOT EXISTS meta ( + key TEXT PRIMARY KEY, + value NOT NULL +) WITHOUT ROWID; diff --git a/third_party/rust/webext-storage/src/api.rs b/third_party/rust/webext-storage/src/api.rs new file mode 100644 index 000000000000..f7e8dc714b3d --- /dev/null +++ b/third_party/rust/webext-storage/src/api.rs @@ -0,0 +1,480 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use crate::error::*; +use rusqlite::{Connection, Transaction}; +use serde::{ser::SerializeMap, Serialize, Serializer}; + +use serde_json::{Map, Value as JsonValue}; +use sql_support::{self, ConnExt}; + +// These constants are defined by the chrome.storage.sync spec. +const QUOTA_BYTES: usize = 102_400; +const QUOTA_BYTES_PER_ITEM: usize = 8_192; +const MAX_ITEMS: usize = 512; +// Note there are also constants for "operations per minute" etc, which aren't +// enforced here. + +type JsonMap = Map; + +fn get_from_db(conn: &Connection, ext_id: &str) -> Result> { + Ok( + match conn.try_query_one::( + "SELECT data FROM storage_sync_data + WHERE ext_id = :ext_id", + &[(":ext_id", &ext_id)], + true, + )? { + Some(s) => match serde_json::from_str(&s)? { + JsonValue::Object(m) => Some(m), + // we could panic here as it's theoretically impossible, but we + // might as well treat it as not existing... + _ => None, + }, + None => None, + }, + ) +} + +fn save_to_db(tx: &Transaction<'_>, ext_id: &str, val: &JsonValue) -> Result<()> { + // The quota is enforced on the byte count, which is what .len() returns. + let sval = val.to_string(); + if sval.len() > QUOTA_BYTES { + return Err(ErrorKind::QuotaError(QuotaReason::TotalBytes).into()); + } + // XXX - sync support will need to do the change_counter thing here. + tx.execute_named( + "INSERT OR REPLACE INTO storage_sync_data(ext_id, data) + VALUES (:ext_id, :data)", + &[(":ext_id", &ext_id), (":data", &sval)], + )?; + Ok(()) +} + +fn remove_from_db(tx: &Transaction<'_>, ext_id: &str) -> Result<()> { + // XXX - sync support will need to do the tombstone thing here. + tx.execute_named( + "DELETE FROM storage_sync_data + WHERE ext_id = :ext_id", + &[(":ext_id", &ext_id)], + )?; + Ok(()) +} + +// This is a "helper struct" for the callback part of the chrome.storage spec, +// but shaped in a way to make it more convenient from the rust side of the +// world. The strings are all json, we keeping them as strings here makes +// various things easier and avoid a round-trip to/from json/string. +#[derive(Debug, Clone, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct StorageValueChange { + #[serde(skip_serializing)] + key: String, + #[serde(skip_serializing_if = "Option::is_none")] + old_value: Option, + #[serde(skip_serializing_if = "Option::is_none")] + new_value: Option, +} + +// This is, largely, a helper so that this serializes correctly as per the +// chrome.storage.sync spec. If not for custom serialization it should just +// be a plain vec +#[derive(Debug, Clone, PartialEq)] +pub struct StorageChanges { + changes: Vec, +} + +impl StorageChanges { + fn new() -> Self { + Self { + changes: Vec::new(), + } + } + + fn with_capacity(n: usize) -> Self { + Self { + changes: Vec::with_capacity(n), + } + } + + fn is_empty(&self) -> bool { + self.changes.is_empty() + } + + fn push(&mut self, change: StorageValueChange) { + self.changes.push(change) + } +} + +// and it serializes as a map. +impl Serialize for StorageChanges { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let mut map = serializer.serialize_map(Some(self.changes.len()))?; + for change in &self.changes { + map.serialize_entry(&change.key, change)?; + } + map.end() + } +} + +/// The implementation of `storage[.sync].set()`. On success this returns the +/// StorageChanges defined by the chrome API - it's assumed the caller will +/// arrange to deliver this to observers as defined in that API. +pub fn set(tx: &Transaction<'_>, ext_id: &str, val: JsonValue) -> Result { + let val_map = match val { + JsonValue::Object(m) => m, + // Not clear what the error semantics should be yet. For now, pretend an empty map. + _ => Map::new(), + }; + + let mut current = get_from_db(tx, ext_id)?.unwrap_or_default(); + + let mut changes = StorageChanges::with_capacity(val_map.len()); + + // iterate over the value we are adding/updating. + for (k, v) in val_map.into_iter() { + let old_value = current.remove(&k); + if current.len() >= MAX_ITEMS { + return Err(ErrorKind::QuotaError(QuotaReason::MaxItems).into()); + } + // Setup the change entry for this key, and we can leverage it to check + // for the quota. + let new_value_s = v.to_string(); + // Reading the chrome docs literally re the quota, the length of the key + // is just the string len, but the value is the json val, as bytes + if k.len() + new_value_s.len() >= QUOTA_BYTES_PER_ITEM { + return Err(ErrorKind::QuotaError(QuotaReason::ItemBytes).into()); + } + let change = StorageValueChange { + key: k.clone(), + old_value: old_value.map(|ov| ov.to_string()), + new_value: Some(new_value_s), + }; + changes.push(change); + current.insert(k, v); + } + + save_to_db(tx, ext_id, &JsonValue::Object(current))?; + Ok(changes) +} + +// A helper which takes a param indicating what keys should be returned and +// converts that to a vec of real strings. Also returns "default" values to +// be used if no item exists for that key. +fn get_keys(keys: JsonValue) -> Vec<(String, Option)> { + match keys { + JsonValue::String(s) => vec![(s, None)], + JsonValue::Array(keys) => { + // because nothing with json is ever simple, each key may not be + // a string. We ignore any which aren't. + keys.iter() + .filter_map(|v| v.as_str().map(|s| (s.to_string(), None))) + .collect() + } + JsonValue::Object(m) => m.into_iter().map(|(k, d)| (k, Some(d))).collect(), + _ => vec![], + } +} + +/// The implementation of `storage[.sync].get()` - on success this always +/// returns a Json object. +pub fn get(conn: &Connection, ext_id: &str, keys: JsonValue) -> Result { + // key is optional, or string or array of string or object keys + let maybe_existing = get_from_db(conn, ext_id)?; + let mut existing = match maybe_existing { + None => return Ok(JsonValue::Object(Map::new())), + Some(v) => v, + }; + // take the quick path for null, where we just return the entire object. + if keys.is_null() { + return Ok(JsonValue::Object(existing)); + } + // OK, so we need to build a list of keys to get. + let keys_and_defaults = get_keys(keys); + let mut result = Map::with_capacity(keys_and_defaults.len()); + for (key, maybe_default) in keys_and_defaults { + // XXX - If a key is requested that doesn't exist, we have 2 options: + // (1) have the key in the result with the value null, or (2) the key + // simply doesn't exist in the result. We assume (2), but should verify + // that's what chrome does. + if let Some(v) = existing.remove(&key) { + result.insert(key, v); + } else if let Some(def) = maybe_default { + result.insert(key, def); + } + } + Ok(JsonValue::Object(result)) +} + +/// The implementation of `storage[.sync].remove()`. On success this returns the +/// StorageChanges defined by the chrome API - it's assumed the caller will +/// arrange to deliver this to observers as defined in that API. +pub fn remove(tx: &Transaction<'_>, ext_id: &str, keys: JsonValue) -> Result { + let mut existing = match get_from_db(tx, ext_id)? { + None => return Ok(StorageChanges::new()), + Some(v) => v, + }; + + let keys_and_defs = get_keys(keys); + + let mut result = StorageChanges::with_capacity(keys_and_defs.len()); + for (key, _) in keys_and_defs { + if let Some(v) = existing.remove(&key) { + result.push(StorageValueChange { + key, + old_value: Some(v.to_string()), + new_value: None, + }); + } + } + if !result.is_empty() { + save_to_db(tx, ext_id, &JsonValue::Object(existing))?; + } + Ok(result) +} + +/// The implementation of `storage[.sync].clear()`. On success this returns the +/// StorageChanges defined by the chrome API - it's assumed the caller will +/// arrange to deliver this to observers as defined in that API. +pub fn clear(tx: &Transaction<'_>, ext_id: &str) -> Result { + let existing = match get_from_db(tx, ext_id)? { + None => return Ok(StorageChanges::new()), + Some(v) => v, + }; + let mut result = StorageChanges::with_capacity(existing.len()); + for (key, val) in existing.into_iter() { + result.push(StorageValueChange { + key: key.to_string(), + new_value: None, + old_value: Some(val.to_string()), + }); + } + remove_from_db(tx, ext_id)?; + Ok(result) +} + +// TODO - get_bytes_in_use() + +#[cfg(test)] +mod tests { + use super::*; + use crate::db::test::new_mem_db; + use serde_json::json; + + #[test] + fn test_serialize_storage_changes() -> Result<()> { + let c = StorageChanges { + changes: vec![StorageValueChange { + key: "key".to_string(), + old_value: Some("old".to_string()), + new_value: None, + }], + }; + assert_eq!(serde_json::to_string(&c)?, r#"{"key":{"oldValue":"old"}}"#); + Ok(()) + } + + fn make_changes(changes: &[(&str, Option, Option)]) -> StorageChanges { + let mut r = StorageChanges::with_capacity(changes.len()); + for (name, old_value, new_value) in changes { + r.push(StorageValueChange { + key: (*name).to_string(), + old_value: old_value.as_ref().map(|v| v.to_string()), + new_value: new_value.as_ref().map(|v| v.to_string()), + }); + } + r + } + + #[test] + fn test_simple() -> Result<()> { + let ext_id = "x"; + let db = new_mem_db(); + let mut conn = db.writer.lock().unwrap(); + let tx = conn.transaction()?; + + // an empty store. + for q in vec![ + JsonValue::Null, + json!("foo"), + json!(["foo"]), + json!({ "foo": null }), + json!({"foo": "default"}), + ] + .into_iter() + { + assert_eq!(get(&tx, &ext_id, q)?, json!({})); + } + + // Single item in the store. + set(&tx, &ext_id, json!({"foo": "bar" }))?; + for q in vec![ + JsonValue::Null, + json!("foo"), + json!(["foo"]), + json!({ "foo": null }), + json!({"foo": "default"}), + ] + .into_iter() + { + assert_eq!(get(&tx, &ext_id, q)?, json!({"foo": "bar" })); + } + + // more complex stuff, including changes checking. + assert_eq!( + set(&tx, &ext_id, json!({"foo": "new", "other": "also new" }))?, + make_changes(&[ + ("foo", Some(json!("bar")), Some(json!("new"))), + ("other", None, Some(json!("also new"))) + ]) + ); + assert_eq!( + get(&tx, &ext_id, JsonValue::Null)?, + json!({"foo": "new", "other": "also new"}) + ); + assert_eq!(get(&tx, &ext_id, json!("foo"))?, json!({"foo": "new"})); + assert_eq!( + get(&tx, &ext_id, json!(["foo", "other"]))?, + json!({"foo": "new", "other": "also new"}) + ); + assert_eq!( + get(&tx, &ext_id, json!({"foo": null, "default": "yo"}))?, + json!({"foo": "new", "default": "yo"}) + ); + + assert_eq!( + remove(&tx, &ext_id, json!("foo"))?, + make_changes(&[("foo", Some(json!("new")), None)]), + ); + // XXX - other variants. + + assert_eq!( + clear(&tx, &ext_id)?, + make_changes(&[("other", Some(json!("also new")), None)]), + ); + assert_eq!(get(&tx, &ext_id, JsonValue::Null)?, json!({})); + + Ok(()) + } + + #[test] + fn test_check_get_impl() -> Result<()> { + // This is a port of checkGetImpl in test_ext_storage.js in Desktop. + let ext_id = "x"; + let db = new_mem_db(); + let mut conn = db.writer.lock().unwrap(); + let tx = conn.transaction()?; + + let prop = "test-prop"; + let value = "test-value"; + + set(&tx, ext_id, json!({ prop: value }))?; + + // this is the checkGetImpl part! + let mut data = get(&tx, &ext_id, json!(null))?; + assert_eq!(value, json!(data[prop]), "null getter worked for {}", prop); + + data = get(&tx, &ext_id, json!(prop))?; + assert_eq!( + value, + json!(data[prop]), + "string getter worked for {}", + prop + ); + assert_eq!( + data.as_object().unwrap().len(), + 1, + "string getter should return an object with a single property" + ); + + data = get(&tx, &ext_id, json!([prop]))?; + assert_eq!(value, json!(data[prop]), "array getter worked for {}", prop); + assert_eq!( + data.as_object().unwrap().len(), + 1, + "array getter with a single key should return an object with a single property" + ); + + // checkGetImpl() uses `{ [prop]: undefined }` - but json!() can't do that :( + // Hopefully it's just testing a simple object, so we use `{ prop: null }` + data = get(&tx, &ext_id, json!({ prop: null }))?; + assert_eq!( + value, + json!(data[prop]), + "object getter worked for {}", + prop + ); + assert_eq!( + data.as_object().unwrap().len(), + 1, + "object getter with a single key should return an object with a single property" + ); + + Ok(()) + } + + #[test] + fn test_bug_1621162() -> Result<()> { + // apparently Firefox, unlike Chrome, will not optimize the changes. + // See bug 1621162 for more! + let db = new_mem_db(); + let mut conn = db.writer.lock().unwrap(); + let tx = conn.transaction()?; + let ext_id = "xyz"; + + set(&tx, &ext_id, json!({"foo": "bar" }))?; + + assert_eq!( + set(&tx, &ext_id, json!({"foo": "bar" }))?, + make_changes(&[("foo", Some(json!("bar")), Some(json!("bar")))]), + ); + Ok(()) + } + + #[test] + fn test_quota_maxitems() -> Result<()> { + let db = new_mem_db(); + let mut conn = db.writer.lock().unwrap(); + let tx = conn.transaction()?; + let ext_id = "xyz"; + for i in 1..MAX_ITEMS + 1 { + set( + &tx, + &ext_id, + json!({ format!("key-{}", i): format!("value-{}", i) }), + )?; + } + let e = set(&tx, &ext_id, json!({"another": "another"})).unwrap_err(); + match e.kind() { + ErrorKind::QuotaError(QuotaReason::MaxItems) => {} + _ => panic!("unexpected error type"), + }; + Ok(()) + } + + #[test] + fn test_quota_bytesperitem() -> Result<()> { + let db = new_mem_db(); + let mut conn = db.writer.lock().unwrap(); + let tx = conn.transaction()?; + let ext_id = "xyz"; + // A string 5 bytes less than the max. This should be counted as being + // 3 bytes less than the max as the quotes are counted. + let val = "x".repeat(QUOTA_BYTES_PER_ITEM - 5); + + // Key length doesn't push it over. + set(&tx, &ext_id, json!({ "x": val }))?; + + // Key length does push it over. + let e = set(&tx, &ext_id, json!({ "xxxx": val })).unwrap_err(); + match e.kind() { + ErrorKind::QuotaError(QuotaReason::ItemBytes) => {} + _ => panic!("unexpected error type"), + }; + Ok(()) + } +} diff --git a/third_party/rust/webext-storage/src/db.rs b/third_party/rust/webext-storage/src/db.rs new file mode 100644 index 000000000000..4fe2dc5b781b --- /dev/null +++ b/third_party/rust/webext-storage/src/db.rs @@ -0,0 +1,230 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use crate::error::*; +use crate::schema; +use rusqlite::types::{FromSql, ToSql}; +use rusqlite::Connection; +use rusqlite::OpenFlags; +use sql_support::ConnExt; +use std::fs; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; +use url::Url; + +/// The entry-point to getting a database connection. No enforcement of this +/// as a singleton is made - that's up to the caller. If you make multiple +/// StorageDbs pointing at the same physical database, you are going to have a +/// bad time. We only support a single writer connection - so that's the only +/// thing we store. It's still a bit overkill, but there's only so many yaks +/// in a day. +pub struct StorageDb { + pub writer: Arc>, +} +impl StorageDb { + /// Create a new, or fetch an already open, StorageDb backed by a file on disk. + pub fn new(db_path: impl AsRef) -> Result { + let db_path = normalize_path(db_path)?; + Self::new_named(db_path) + } + + /// Create a new, or fetch an already open, memory-based StorageDb. You must + /// provide a name, but you are still able to have a single writer and many + /// reader connections to the same memory DB open. + #[cfg(test)] + pub fn new_memory(db_path: &str) -> Result { + let name = PathBuf::from(format!("file:{}?mode=memory&cache=shared", db_path)); + Self::new_named(name) + } + + fn new_named(db_path: PathBuf) -> Result { + // We always create the read-write connection for an initial open so + // we can create the schema and/or do version upgrades. + let flags = OpenFlags::SQLITE_OPEN_NO_MUTEX + | OpenFlags::SQLITE_OPEN_URI + | OpenFlags::SQLITE_OPEN_CREATE + | OpenFlags::SQLITE_OPEN_READ_WRITE; + + let conn = Connection::open_with_flags(db_path.clone(), flags)?; + match init_sql_connection(&conn, true) { + Ok(()) => Ok(Self { + writer: Arc::new(Mutex::new(conn)), + }), + Err(e) => { + // like with places, failure to upgrade means "you lose your data" + if let ErrorKind::DatabaseUpgradeError = e.kind() { + fs::remove_file(&db_path)?; + Self::new_named(db_path) + } else { + Err(e) + } + } + } + } +} + +fn init_sql_connection(conn: &Connection, is_writable: bool) -> Result<()> { + let initial_pragmas = " + -- We don't care about temp tables being persisted to disk. + PRAGMA temp_store = 2; + -- we unconditionally want write-ahead-logging mode + PRAGMA journal_mode=WAL; + -- foreign keys seem worth enforcing! + PRAGMA foreign_keys = ON; + "; + + conn.execute_batch(initial_pragmas)?; + define_functions(&conn)?; + conn.set_prepared_statement_cache_capacity(128); + if is_writable { + let tx = conn.unchecked_transaction()?; + schema::init(&conn)?; + tx.commit()?; + }; + Ok(()) +} + +fn define_functions(_c: &Connection) -> Result<()> { + Ok(()) +} + +// These should be somewhere else... +pub fn put_meta(db: &Connection, key: &str, value: &dyn ToSql) -> Result<()> { + db.conn().execute_named_cached( + "REPLACE INTO meta (key, value) VALUES (:key, :value)", + &[(":key", &key), (":value", value)], + )?; + Ok(()) +} + +pub fn get_meta(db: &Connection, key: &str) -> Result> { + let res = db.conn().try_query_one( + "SELECT value FROM meta WHERE key = :key", + &[(":key", &key)], + true, + )?; + Ok(res) +} + +pub fn delete_meta(db: &Connection, key: &str) -> Result<()> { + db.conn() + .execute_named_cached("DELETE FROM meta WHERE key = :key", &[(":key", &key)])?; + Ok(()) +} + +// Utilities for working with paths. +// (From places_utils - ideally these would be shared, but the use of +// ErrorKind values makes that non-trivial. + +/// `Path` is basically just a `str` with no validation, and so in practice it +/// could contain a file URL. Rusqlite takes advantage of this a bit, and says +/// `AsRef` but really means "anything sqlite can take as an argument". +/// +/// Swift loves using file urls (the only support it has for file manipulation +/// is through file urls), so it's handy to support them if possible. +fn unurl_path(p: impl AsRef) -> PathBuf { + p.as_ref() + .to_str() + .and_then(|s| Url::parse(s).ok()) + .and_then(|u| { + if u.scheme() == "file" { + u.to_file_path().ok() + } else { + None + } + }) + .unwrap_or_else(|| p.as_ref().to_owned()) +} + +/// If `p` is a file URL, return it, otherwise try and make it one. +/// +/// Errors if `p` is a relative non-url path, or if it's a URL path +/// that's isn't a `file:` URL. +pub fn ensure_url_path(p: impl AsRef) -> Result { + if let Some(u) = p.as_ref().to_str().and_then(|s| Url::parse(s).ok()) { + if u.scheme() == "file" { + Ok(u) + } else { + Err(ErrorKind::IllegalDatabasePath(p.as_ref().to_owned()).into()) + } + } else { + let p = p.as_ref(); + let u = Url::from_file_path(p).map_err(|_| ErrorKind::IllegalDatabasePath(p.to_owned()))?; + Ok(u) + } +} + +/// As best as possible, convert `p` into an absolute path, resolving +/// all symlinks along the way. +/// +/// If `p` is a file url, it's converted to a path before this. +fn normalize_path(p: impl AsRef) -> Result { + let path = unurl_path(p); + if let Ok(canonical) = path.canonicalize() { + return Ok(canonical); + } + // It probably doesn't exist yet. This is an error, although it seems to + // work on some systems. + // + // We resolve this by trying to canonicalize the parent directory, and + // appending the requested file name onto that. If we can't canonicalize + // the parent, we return an error. + // + // Also, we return errors if the path ends in "..", if there is no + // parent directory, etc. + let file_name = path + .file_name() + .ok_or_else(|| ErrorKind::IllegalDatabasePath(path.clone()))?; + + let parent = path + .parent() + .ok_or_else(|| ErrorKind::IllegalDatabasePath(path.clone()))?; + + let mut canonical = parent.canonicalize()?; + canonical.push(file_name); + Ok(canonical) +} + +// Helpers for tests +#[cfg(test)] +pub mod test { + use super::*; + use std::sync::atomic::{AtomicUsize, Ordering}; + + // A helper for our tests to get their own memory Api. + static ATOMIC_COUNTER: AtomicUsize = AtomicUsize::new(0); + + pub fn new_mem_db() -> StorageDb { + let _ = env_logger::try_init(); + let counter = ATOMIC_COUNTER.fetch_add(1, Ordering::Relaxed); + StorageDb::new_memory(&format!("test-api-{}", counter)).expect("should get an API") + } +} + +#[cfg(test)] +mod tests { + use super::test::*; + use super::*; + + // Sanity check that we can create a database. + #[test] + fn test_open() { + new_mem_db(); + // XXX - should we check anything else? Seems a bit pointless, but if + // we move the meta functions away from here then it's better than + // nothing. + } + + #[test] + fn test_meta() -> Result<()> { + let db = new_mem_db(); + let writer = db.writer.lock().unwrap(); + assert_eq!(get_meta::(&writer, "foo")?, None); + put_meta(&writer, "foo", &"bar".to_string())?; + assert_eq!(get_meta(&writer, "foo")?, Some("bar".to_string())); + delete_meta(&writer, "foo")?; + assert_eq!(get_meta::(&writer, "foo")?, None); + Ok(()) + } +} diff --git a/third_party/rust/webext-storage/src/error.rs b/third_party/rust/webext-storage/src/error.rs new file mode 100644 index 000000000000..348c87616a47 --- /dev/null +++ b/third_party/rust/webext-storage/src/error.rs @@ -0,0 +1,65 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use failure::Fail; +use interrupt_support::Interrupted; + +#[derive(Debug)] +pub enum QuotaReason { + TotalBytes, + ItemBytes, + MaxItems, +} + +#[derive(Debug, Fail)] +pub enum ErrorKind { + #[fail(display = "Quota exceeded: {:?}", _0)] + QuotaError(QuotaReason), + + #[fail(display = "Error parsing JSON data: {}", _0)] + JsonError(#[fail(cause)] serde_json::Error), + + #[fail(display = "Error executing SQL: {}", _0)] + SqlError(#[fail(cause)] rusqlite::Error), + + #[fail(display = "A connection of this type is already open")] + ConnectionAlreadyOpen, + + #[fail(display = "An invalid connection type was specified")] + InvalidConnectionType, + + #[fail(display = "IO error: {}", _0)] + IoError(#[fail(cause)] std::io::Error), + + #[fail(display = "Operation interrupted")] + InterruptedError(#[fail(cause)] Interrupted), + + #[fail(display = "Tried to close connection on wrong StorageApi instance")] + WrongApiForClose, + + // This will happen if you provide something absurd like + // "/" or "" as your database path. For more subtley broken paths, + // we'll likely return an IoError. + #[fail(display = "Illegal database path: {:?}", _0)] + IllegalDatabasePath(std::path::PathBuf), + + #[fail(display = "UTF8 Error: {}", _0)] + Utf8Error(#[fail(cause)] std::str::Utf8Error), + + #[fail(display = "Database cannot be upgraded")] + DatabaseUpgradeError, + + #[fail(display = "Database version {} is not supported", _0)] + UnsupportedDatabaseVersion(i64), +} + +error_support::define_error! { + ErrorKind { + (JsonError, serde_json::Error), + (SqlError, rusqlite::Error), + (IoError, std::io::Error), + (InterruptedError, Interrupted), + (Utf8Error, std::str::Utf8Error), + } +} diff --git a/third_party/rust/webext-storage/src/lib.rs b/third_party/rust/webext-storage/src/lib.rs new file mode 100644 index 000000000000..187f3f5895cd --- /dev/null +++ b/third_party/rust/webext-storage/src/lib.rs @@ -0,0 +1,26 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#![allow(unknown_lints)] +#![warn(rust_2018_idioms)] + +mod api; +pub mod db; +pub mod error; +mod schema; +pub mod store; + +// This is what we roughly expect the "bridge" used by desktop to do. +// It's primarily here to avoid dead-code warnings (but I don't want to disable +// those warning, as stuff that remains after this is suspect!) +pub fn delme_demo_usage() -> error::Result<()> { + use serde_json::json; + + let store = store::Store::new("webext-storage.db")?; + store.set("ext-id", json!({}))?; + store.get("ext-id", json!({}))?; + store.remove("ext-id", json!({}))?; + store.clear("ext-id")?; + Ok(()) +} diff --git a/third_party/rust/webext-storage/src/schema.rs b/third_party/rust/webext-storage/src/schema.rs new file mode 100644 index 000000000000..02df8c8b8539 --- /dev/null +++ b/third_party/rust/webext-storage/src/schema.rs @@ -0,0 +1,72 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// XXXXXX - This has been cloned from places/src/schema.rs, which has the +// comment: +// // This has been cloned from logins/src/schema.rs, on Thom's +// // wip-sync-sql-store branch. +// // We should work out how to turn this into something that can use a shared +// // db.rs. +// +// And we really should :) But not now. + +use crate::error::Result; +use rusqlite::{Connection, NO_PARAMS}; +use sql_support::ConnExt; + +const VERSION: i64 = 1; // let's avoid bumping this and migrating for now! + +const CREATE_SCHEMA_SQL: &str = include_str!("../sql/create_schema.sql"); + +fn get_current_schema_version(db: &Connection) -> Result { + Ok(db.query_one::("PRAGMA user_version")?) +} + +pub fn init(db: &Connection) -> Result<()> { + let user_version = get_current_schema_version(db)?; + if user_version == 0 { + create(db)?; + } else if user_version != VERSION { + if user_version < VERSION { + panic!("no migrations yet!"); + } else { + log::warn!( + "Loaded future schema version {} (we only understand version {}). \ + Optimistically ", + user_version, + VERSION + ); + // Downgrade the schema version, so that anything added with our + // schema is migrated forward when the newer library reads our + // database. + db.execute_batch(&format!("PRAGMA user_version = {};", VERSION))?; + } + } + Ok(()) +} + +fn create(db: &Connection) -> Result<()> { + log::debug!("Creating schema"); + db.execute_batch(CREATE_SCHEMA_SQL)?; + db.execute( + &format!("PRAGMA user_version = {version}", version = VERSION), + NO_PARAMS, + )?; + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::db::test::new_mem_db; + + #[test] + fn test_create_schema_twice() { + let db = new_mem_db(); + let conn = db.writer.lock().unwrap(); + conn.execute_batch(CREATE_SCHEMA_SQL) + .expect("should allow running twice"); + } +} diff --git a/third_party/rust/webext-storage/src/store.rs b/third_party/rust/webext-storage/src/store.rs new file mode 100644 index 000000000000..0dc80c66e409 --- /dev/null +++ b/third_party/rust/webext-storage/src/store.rs @@ -0,0 +1,72 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use crate::api::{self, StorageChanges}; +use crate::db::StorageDb; +use crate::error::*; +use std::path::Path; + +use serde_json::Value as JsonValue; + +pub struct Store { + db: StorageDb, +} + +impl Store { + // functions to create instances. + pub fn new(db_path: impl AsRef) -> Result { + Ok(Self { + db: StorageDb::new(db_path)?, + }) + } + + #[cfg(test)] + pub fn new_memory(db_path: &str) -> Result { + Ok(Self { + db: StorageDb::new_memory(db_path)?, + }) + } + + // The "public API". + pub fn set(&self, ext_id: &str, val: JsonValue) -> Result { + let mut conn = self.db.writer.lock().unwrap(); + let tx = conn.transaction()?; + let result = api::set(&tx, ext_id, val)?; + tx.commit()?; + Ok(result) + } + + pub fn get(&self, ext_id: &str, keys: JsonValue) -> Result { + // Don't care about transactions here. + let conn = self.db.writer.lock().unwrap(); + api::get(&conn, ext_id, keys) + } + + pub fn remove(&self, ext_id: &str, keys: JsonValue) -> Result { + let mut conn = self.db.writer.lock().unwrap(); + let tx = conn.transaction()?; + let result = api::remove(&tx, ext_id, keys)?; + tx.commit()?; + Ok(result) + } + + pub fn clear(&self, ext_id: &str) -> Result { + let mut conn = self.db.writer.lock().unwrap(); + let tx = conn.transaction()?; + let result = api::clear(&tx, ext_id)?; + tx.commit()?; + Ok(result) + } +} + +#[cfg(test)] +mod test { + use super::*; + #[test] + fn test_send() { + fn ensure_send() {} + // Compile will fail if not send. + ensure_send::(); + } +} diff --git a/toolkit/library/gtest/rust/Cargo.toml b/toolkit/library/gtest/rust/Cargo.toml index 6acbc25b1c57..148bc46bc9fd 100644 --- a/toolkit/library/gtest/rust/Cargo.toml +++ b/toolkit/library/gtest/rust/Cargo.toml @@ -32,6 +32,7 @@ wasm_library_sandboxing = ["gkrust-shared/wasm_library_sandboxing"] webgpu = ["gkrust-shared/webgpu"] remote_agent = ["gkrust-shared/remote"] glean = ["gkrust-shared/glean"] +new_webext_storage = ["gkrust-shared/new_webext_storage"] [dependencies] bench-collections-gtest = { path = "../../../../xpcom/rust/gtest/bench-collections" } diff --git a/toolkit/library/rust/Cargo.toml b/toolkit/library/rust/Cargo.toml index 7b3df3e239a5..bf044e5a6bdc 100644 --- a/toolkit/library/rust/Cargo.toml +++ b/toolkit/library/rust/Cargo.toml @@ -33,6 +33,7 @@ wasm_library_sandboxing = ["gkrust-shared/wasm_library_sandboxing"] webgpu = ["gkrust-shared/webgpu"] remote_agent = ["gkrust-shared/remote"] glean = ["gkrust-shared/glean"] +new_webext_storage = ["gkrust-shared/new_webext_storage"] [dependencies] gkrust-shared = { path = "shared" } diff --git a/toolkit/library/rust/gkrust-features.mozbuild b/toolkit/library/rust/gkrust-features.mozbuild index 96509772a4a8..b96c9dd632a4 100644 --- a/toolkit/library/rust/gkrust-features.mozbuild +++ b/toolkit/library/rust/gkrust-features.mozbuild @@ -79,3 +79,6 @@ if CONFIG['MOZ_GLEAN']: if CONFIG['MOZ_USING_WASM_SANDBOXING']: gkrust_features += ['wasm_library_sandboxing'] + +if CONFIG['MOZ_NEW_WEBEXT_STORAGE']: + gkrust_features += ['new_webext_storage'] diff --git a/toolkit/library/rust/shared/Cargo.toml b/toolkit/library/rust/shared/Cargo.toml index cb5ef4cdcdca..d3870e88601d 100644 --- a/toolkit/library/rust/shared/Cargo.toml +++ b/toolkit/library/rust/shared/Cargo.toml @@ -53,6 +53,7 @@ unic-langid = { version = "0.8", features = ["likelysubtags"] } unic-langid-ffi = { path = "../../../../intl/locale/rust/unic-langid-ffi" } fluent-langneg = { version = "0.12.1", features = ["cldr"] } fluent-langneg-ffi = { path = "../../../../intl/locale/rust/fluent-langneg-ffi" } +webext-storage = { git = "https://github.com/mozilla/application-services", rev = "c17198fa5a88295f2cca722586c539280e10201c", optional = true } # Note: `modern_sqlite` means rusqlite's bindings file be for a sqlite with # version less than or equal to what we link to. This isn't a problem because we @@ -63,7 +64,7 @@ rusqlite = { version = "0.23.1", features = ["modern_sqlite", "in_gecko"] } fluent = { version = "0.11" , features = ["fluent-pseudo"] } fluent-ffi = { path = "../../../../intl/l10n/rust/fluent-ffi" } -sync15-traits = { git = "https://github.com/mozilla/application-services", rev = "120e51dd5f2aab4194cf0f7e93b2a8923f4504bb" } +sync15-traits = { git = "https://github.com/mozilla/application-services", rev = "c17198fa5a88295f2cca722586c539280e10201c" } [build-dependencies] rustc_version = "0.2" @@ -96,6 +97,7 @@ wasm_library_sandboxing = ["rlbox_lucet_sandbox"] webgpu = ["wgpu_bindings"] remote_agent = ["remote"] glean = ["fog"] +new_webext_storage = ["webext-storage"] [lib] path = "lib.rs" diff --git a/toolkit/library/rust/shared/lib.rs b/toolkit/library/rust/shared/lib.rs index d9556b7d3cdd..e21647ef89c4 100644 --- a/toolkit/library/rust/shared/lib.rs +++ b/toolkit/library/rust/shared/lib.rs @@ -52,6 +52,9 @@ extern crate xulstore; extern crate audio_thread_priority; +#[cfg(feature = "new_webext_storage")] +extern crate webext_storage_bridge; + #[cfg(feature = "webrtc")] extern crate mdns_service; extern crate neqo_glue; diff --git a/toolkit/moz.configure b/toolkit/moz.configure index 77c7c78bfbec..6670d42429d0 100644 --- a/toolkit/moz.configure +++ b/toolkit/moz.configure @@ -1989,6 +1989,19 @@ def glean(milestone): set_config('MOZ_GLEAN', True, when=glean) set_define('MOZ_GLEAN', True, when=glean) + +# New WebExtension `storage.sync` implementation in Rust +# ============================================================== + +@depends(milestone) +def new_webext_storage(milestone): + if milestone.is_nightly: + return True + +set_config('MOZ_NEW_WEBEXT_STORAGE', True, when=new_webext_storage) +set_define('MOZ_NEW_WEBEXT_STORAGE', True, when=new_webext_storage) + + # dump_syms # ==============================================================