Implement a logins store backed by SQLCipher.

.gitignore

@@ -4,3 +4,4 @@ Cargo.lock
 credentials.json
 *-engine.json
 .cargo
+*.db

Cargo.toml

@@ -3,7 +3,9 @@ members = [
     "fxa-client",
     "fxa-client/ffi",
     "sandvich/desktop",
-    "sync15-adapter"
+    "sync15-adapter",
+    "logins-sql",
+    "logins-sql/ffi"
 ]
 
 # For RSA keys cloning. Remove once openssl 0.10.8+ is released.

logins-api/android/library/build.gradle

@@ -33,7 +33,7 @@ android {
 
 cargo {
     // The directory of the Cargo.toml to build.
-    module = '../../../sync15/passwords/ffi'
+    module = '../../../logins-sql/ffi'
 
     // The Android NDK API level to target.
     apiLevel = 21

logins-api/android/library/src/main/java/org/mozilla/sync15/logins/DatabaseLoginsStorage.kt

@@ -149,7 +149,7 @@ class DatabaseLoginsStorage(private val dbPath: String) : Closeable, LoginsStora
             try {
                 return p.getString(0, "utf8");
             } finally {
-                PasswordSyncAdapter.INSTANCE.destroy_c_char(p);
+                PasswordSyncAdapter.INSTANCE.sync15_passwords_destroy_string(p);
             }
         }
 

logins-api/android/library/src/main/java/org/mozilla/sync15/logins/rust/PasswordSyncAdapter.kt

@@ -54,8 +54,11 @@ internal interface PasswordSyncAdapter : Library {
 
     fun sync15_passwords_touch(state: RawLoginSyncState, id: String, error: RustError.ByReference)
     fun sync15_passwords_delete(state: RawLoginSyncState, id: String, error: RustError.ByReference): Boolean
+    // Note: returns guid of new login entry (unless one was specifically requested)
+    fun sync15_passwords_add(state: RawLoginSyncState, new_login_json: String, error: RustError.ByReference): Pointer
+    fun sync15_passwords_update(state: RawLoginSyncState, existing_login_json: String, error: RustError.ByReference)
 
-    fun destroy_c_char(p: Pointer)
+    fun sync15_passwords_destroy_string(p: Pointer)
 }
 
 class RawLoginSyncState : PointerType()

logins-api/android/library/src/main/java/org/mozilla/sync15/logins/rust/RustError.kt

@@ -60,7 +60,7 @@ open class RustError : Structure() {
     fun consumeErrorMessage(): String {
         val result = this.getMessage()
         if (this.message != null) {
-            PasswordSyncAdapter.INSTANCE.destroy_c_char(this.message!!);
+            PasswordSyncAdapter.INSTANCE.sync15_passwords_destroy_string(this.message!!);
             this.message = null
         }
         if (result == null) {

logins-api/android/sample/src/main/java/org/mozilla/sync15_logins_example/MainActivity.kt

@@ -162,7 +162,7 @@ class MainActivity : AppCompatActivity() {
 
     fun initStore(): SyncResult<DatabaseLoginsStorage> {
         val appFiles = this.applicationContext.getExternalFilesDir(null)
-        val storage = DatabaseLoginsStorage(appFiles.absolutePath + "/logins.db");
+        val storage = DatabaseLoginsStorage(appFiles.absolutePath + "/logins.sqlite");
         return storage.unlock("my_secret_key").then {
             SyncResult.fromValue(storage)
         }

logins-sql/Cargo.toml

@@ -0,0 +1,28 @@
+[package]
+name = "logins-sql"
+version = "0.1.0"
+authors = ["Thom Chiovoloni <tchiovoloni@mozilla.com>"]
+
+[dependencies]
+sync15-adapter = { path = "../sync15-adapter" }
+serde = "1.0.75"
+serde_derive = "1.0.75"
+serde_json = "1.0.26"
+log = "0.4.4"
+lazy_static = "1.1.0"
+url = "1.7.1"
+failure = "0.1"
+failure_derive = "0.1"
+
+[dependencies.rusqlite]
+version = "0.14.0"
+features = ["sqlcipher", "limits"]
+
+[dev-dependencies]
+more-asserts = "0.2.1"
+env_logger = "0.5.13"
+prettytable-rs = "0.7.0"
+fxa-client = { path = "../fxa-client" }
+webbrowser = "0.3.1"
+chrono = "0.4.6"
+clap = "2.32.0"

logins-sql/examples/sync_pass_sql.rs

@@ -0,0 +1,507 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#![recursion_limit = "4096"]
+
+extern crate logins_sql;
+extern crate sync15_adapter as sync;
+extern crate fxa_client;
+extern crate url;
+#[macro_use]
+extern crate prettytable;
+
+extern crate serde;
+#[macro_use]
+extern crate serde_derive;
+extern crate serde_json;
+
+extern crate rusqlite;
+
+extern crate webbrowser;
+extern crate clap;
+
+#[macro_use]
+extern crate log;
+extern crate env_logger;
+extern crate chrono;
+extern crate failure;
+
+use failure::Fail;
+
+use std::{fs, io::{self, Read, Write}};
+use std::collections::HashMap;
+use fxa_client::{FirefoxAccount, Config, OAuthInfo};
+use sync::{Sync15StorageClientInit, KeyBundle};
+use logins_sql::{PasswordEngine, Login};
+
+const CLIENT_ID: &str = "98adfa37698f255b";
+const REDIRECT_URI: &str = "https://lockbox.firefox.com/fxa/ios-redirect.html";
+
+const CONTENT_BASE: &str = "https://accounts.firefox.com";
+const SYNC_SCOPE: &str = "https://identity.mozilla.com/apps/oldsync";
+
+const SCOPES: &[&str] = &[
+    SYNC_SCOPE,
+    "https://identity.mozilla.com/apps/lockbox",
+];
+
+// I'm completely punting on good error handling here.
+type Result<T> = std::result::Result<T, failure::Error>;
+
+#[derive(Debug, Deserialize)]
+struct ScopedKeyData {
+    k: String,
+    kty: String,
+    kid: String,
+    scope: String,
+}
+
+fn load_fxa_creds(path: &str) -> Result<FirefoxAccount> {
+    let mut file = fs::File::open(path)?;
+    let mut s = String::new();
+    file.read_to_string(&mut s)?;
+    Ok(FirefoxAccount::from_json(&s)?)
+}
+
+fn load_or_create_fxa_creds(path: &str, cfg: Config) -> Result<FirefoxAccount> {
+    load_fxa_creds(path)
+    .or_else(|e| {
+        info!("Failed to load existing FxA credentials from {:?} (error: {}), launching OAuth flow", path, e);
+        create_fxa_creds(path, cfg)
+    })
+}
+
+fn create_fxa_creds(path: &str, cfg: Config) -> Result<FirefoxAccount> {
+    let mut acct = FirefoxAccount::new(cfg, CLIENT_ID, REDIRECT_URI);
+    let oauth_uri = acct.begin_oauth_flow(SCOPES, true)?;
+
+    if let Err(_) = webbrowser::open(&oauth_uri.as_ref()) {
+        warn!("Failed to open a web browser D:");
+        println!("Please visit this URL, sign in, and then copy-paste the final URL below.");
+        println!("\n    {}\n", oauth_uri);
+    } else {
+        println!("Please paste the final URL below:\n");
+    }
+
+    let final_url = url::Url::parse(&prompt_string("Final URL").unwrap_or(String::new()))?;
+    let query_params = final_url.query_pairs().into_owned().collect::<HashMap<String, String>>();
+
+    acct.complete_oauth_flow(&query_params["code"], &query_params["state"])?;
+    let mut file = fs::File::create(path)?;
+    write!(file, "{}", acct.to_json()?)?;
+    file.flush()?;
+    Ok(acct)
+}
+
+fn prompt_string<S: AsRef<str>>(prompt: S) -> Option<String> {
+    print!("{}: ", prompt.as_ref());
+    let _ = io::stdout().flush(); // Don't care if flush fails really.
+    let mut s = String::new();
+    io::stdin().read_line(&mut s).expect("Failed to read line...");
+    if let Some('\n') = s.chars().next_back() { s.pop(); }
+    if let Some('\r') = s.chars().next_back() { s.pop(); }
+    if s.len() == 0 {
+        None
+    } else {
+        Some(s)
+    }
+}
+
+fn prompt_usize<S: AsRef<str>>(prompt: S) -> Option<usize> {
+    if let Some(s) = prompt_string(prompt) {
+        match s.parse::<usize>() {
+            Ok(n) => Some(n),
+            Err(_) => {
+                println!("Couldn't parse!");
+                None
+            }
+        }
+    } else {
+        None
+    }
+}
+
+fn read_login() -> Login {
+    let username = prompt_string("username").unwrap_or_default();
+    let password = prompt_string("password").unwrap_or_default();
+    let form_submit_url = prompt_string("form_submit_url");
+    let hostname = prompt_string("hostname").unwrap_or_default();
+    let http_realm = prompt_string("http_realm");
+    let username_field = prompt_string("username_field").unwrap_or_default();
+    let password_field = prompt_string("password_field").unwrap_or_default();
+    let record = Login {
+        id: sync::util::random_guid().unwrap().into(),
+        username,
+        password,
+        username_field,
+        password_field,
+        form_submit_url,
+        http_realm,
+        hostname,
+        .. Login::default()
+    };
+
+    if let Err(e) = record.check_valid() {
+        warn!("Warning: produced invalid record: {}", e);
+    }
+    record
+}
+
+fn update_string(field_name: &str, field: &mut String, extra: &str) -> bool {
+    let opt_s = prompt_string(format!("new {} [now {}{}]", field_name, field, extra));
+    if let Some(s) = opt_s {
+        *field = s;
+        true
+    } else {
+        false
+    }
+}
+
+fn string_opt(o: &Option<String>) -> Option<&str> {
+    o.as_ref().map(|s| s.as_ref())
+}
+
+fn string_opt_or<'a>(o: &'a Option<String>, or: &'a str) -> &'a str {
+    string_opt(o).unwrap_or(or)
+}
+
+fn update_login(record: &mut Login) {
+    update_string("username", &mut record.username, ", leave blank to keep");
+    update_string("password", &mut record.password, ", leave blank to keep");
+    update_string("hostname", &mut record.hostname, ", leave blank to keep");
+
+    update_string("username_field", &mut record.username_field, ", leave blank to keep");
+    update_string("password_field", &mut record.password_field, ", leave blank to keep");
+
+    if prompt_bool(&format!("edit form_submit_url? (now {}) [yN]", string_opt_or(&record.form_submit_url, "(none)"))).unwrap_or(false) {
+        record.form_submit_url = prompt_string("form_submit_url");
+    }
+
+    if prompt_bool(&format!("edit http_realm? (now {}) [yN]", string_opt_or(&record.http_realm, "(none)"))).unwrap_or(false) {
+        record.http_realm = prompt_string("http_realm");
+    }
+
+    if let Err(e) = record.check_valid() {
+        warn!("Warning: produced invalid record: {}", e);
+    }
+}
+
+fn prompt_bool(msg: &str) -> Option<bool> {
+    let result = prompt_string(msg);
+    result.and_then(|r| match r.chars().next().unwrap() {
+        'y' | 'Y' | 't' | 'T' => Some(true),
+        'n' | 'N' | 'f' | 'F' => Some(false),
+        _ => None
+    })
+}
+
+fn prompt_chars(msg: &str) -> Option<char> {
+    prompt_string(msg).and_then(|r| r.chars().next())
+}
+
+fn timestamp_to_string(milliseconds: i64) -> String {
+    use chrono::{Local, DateTime};
+    use std::time::{UNIX_EPOCH, Duration};
+    let time = UNIX_EPOCH + Duration::from_millis(milliseconds as u64);
+    let dtl: DateTime<Local> = time.into();
+    dtl.format("%l:%M:%S %p%n%h %e, %Y").to_string()
+}
+
+fn show_sql(e: &PasswordEngine, sql: &str) -> Result<()> {
+    use prettytable::{row::Row, cell::Cell, Table};
+    use rusqlite::types::Value;
+    let conn = e.conn();
+    let mut stmt = conn.prepare(sql)?;
+    let cols: Vec<String> = stmt.column_names().into_iter().map(|x| x.to_owned()).collect();
+    let len = cols.len();
+    let mut table = Table::new();
+    table.add_row(Row::new(
+        cols.iter().map(|name| Cell::new(&name).style_spec("bc")).collect()
+    ));
+
+    let rows = stmt.query_map(&[], |row| {
+        (0..len).into_iter().map(|idx| {
+            match row.get::<_, Value>(idx) {
+                Value::Null => Cell::new("null").style_spec("Fd"),
+                Value::Integer(i) => Cell::new(&i.to_string()).style_spec("Fb"),
+                Value::Real(r) => Cell::new(&r.to_string()).style_spec("Fb"),
+                Value::Text(s) => Cell::new(&s.to_string()).style_spec("Fr"),
+                Value::Blob(b) => Cell::new(&format!("{}b blob", b.len()))
+            }
+        }).collect::<Vec<_>>()
+    })?;
+
+    for row in rows {
+        table.add_row(Row::new(row?));
+    }
+    table.printstd();
+    Ok(())
+}
+
+fn show_all(engine: &PasswordEngine) -> Result<Vec<String>> {
+    let records = engine.list()?;
+
+    let mut table = prettytable::Table::new();
+
+    table.add_row(row![bc =>
+        "(idx)",
+        "Guid",
+        "Username",
+        "Password",
+        "Host",
+
+        "Submit URL",
+        "HTTP Realm",
+
+        "User Field",
+        "Pass Field",
+
+        "Uses",
+        "Created At",
+        "Changed At",
+        "Last Used"
+    ]);
+
+    let mut v = Vec::with_capacity(records.len());
+    let mut record_copy = records.clone();
+    record_copy.sort_by(|a, b| a.id.cmp(&b.id));
+    for rec in records.iter() {
+        table.add_row(row![
+            r->v.len(),
+            Fr->&rec.id,
+            &rec.username,
+            Fd->&rec.password,
+
+            &rec.hostname,
+            string_opt_or(&rec.form_submit_url, ""),
+            string_opt_or(&rec.http_realm, ""),
+
+            &rec.username_field,
+            &rec.password_field,
+
+            rec.times_used,
+            timestamp_to_string(rec.time_created),
+            timestamp_to_string(rec.time_password_changed),
+            if rec.time_last_used == 0 {
+                "Never".to_owned()
+            } else {
+                timestamp_to_string(rec.time_last_used)
+            }
+        ]);
+        v.push(rec.id.clone());
+    }
+    table.printstd();
+    Ok(v)
+}
+
+fn prompt_record_id(e: &PasswordEngine, action: &str) -> Result<Option<String>> {
+    let index_to_id = show_all(e)?;
+    let input = if let Some(input) = prompt_usize(&format!("Enter (idx) of record to {}", action)) {
+        input
+    } else {
+        return Ok(None);
+    };
+    if input >= index_to_id.len() {
+        info!("No such index");
+        return Ok(None);
+    }
+    Ok(Some(index_to_id[input].clone()))
+}
+
+fn init_logging() {
+    // Explicitly ignore some rather noisy crates. Turn on trace for everyone else.
+    let spec = "trace,tokio_threadpool=warn,tokio_reactor=warn,tokio_core=warn,tokio=warn,hyper=warn,want=warn,mio=warn,reqwest=warn";
+    env_logger::init_from_env(
+        env_logger::Env::default().filter_or("RUST_LOG", spec)
+    );
+}
+
+fn main() -> Result<()> {
+    init_logging();
+    std::env::set_var("RUST_BACKTRACE", "1");
+
+    let matches = clap::App::new("sync_pass_sql")
+        .about("CLI login syncing tool (backed by sqlcipher)")
+
+        .arg(clap::Arg::with_name("database_path")
+            .short("d")
+            .long("database")
+            .value_name("LOGINS_DATABASE")
+            .takes_value(true)
+            .help("Path to the logins database (default: \"./logins.db\")"))
+
+        .arg(clap::Arg::with_name("encryption_key")
+            .short("k")
+            .long("key")
+            .value_name("ENCRYPTION_KEY")
+            .takes_value(true)
+            .help("Database encryption key.")
+            .required(true))
+
+        .arg(clap::Arg::with_name("credential_file")
+            .short("c")
+            .long("credentials")
+            .value_name("CREDENTIAL_JSON")
+            .takes_value(true)
+            .help("Path to store our cached fxa credentials (defaults to \"./credentials.json\""))
+
+        .get_matches();
+
+    let cred_file = matches.value_of("credential_file").unwrap_or("./credentials.json");
+    let db_path = matches.value_of("database_path").unwrap_or("./logins.db");
+    // This should already be checked by `clap`, IIUC
+    let encryption_key = matches.value_of("encryption_key").expect("Encryption key is not optional");
+
+    // Lets not log the encryption key, it's just not a good habit to be in.
+    debug!("Using credential file = {:?}, db = {:?}", cred_file, db_path);
+
+    // TODO: allow users to use stage/etc.
+    let cfg = Config::import_from(CONTENT_BASE)?;
+    let tokenserver_url = cfg.token_server_endpoint_url()?;
+
+    // TODO: we should probably set a persist callback on acct?
+    let mut acct = load_or_create_fxa_creds(cred_file, cfg.clone())?;
+    let token: OAuthInfo;
+    match acct.get_oauth_token(SCOPES)? {
+        Some(t) => token = t,
+        None => {
+            // The cached credentials did not have appropriate scope, sign in again.
+            warn!("Credentials do not have appropriate scope, launching OAuth flow.");
+            acct = create_fxa_creds(cred_file, cfg.clone())?;
+            token = acct.get_oauth_token(SCOPES)?.unwrap();
+        }
+    }
+
+    let keys: HashMap<String, ScopedKeyData> = serde_json::from_str(&token.keys.unwrap())?;
+
+    let key = keys.get(SYNC_SCOPE).unwrap();
+
+    let client_init = Sync15StorageClientInit {
+        key_id: key.kid.clone(),
+        access_token: token.access_token.clone(),
+        tokenserver_url,
+    };
+    let root_sync_key = KeyBundle::from_ksync_base64(&key.k)?;
+
+    let mut engine = PasswordEngine::new(db_path, Some(encryption_key))?;
+
+    info!("Engine has {} passwords", engine.list()?.len());
+
+    if let Err(e) = show_all(&engine) {
+        warn!("Failed to show initial login data! {}", e);
+    }
+
+    loop {
+        match prompt_chars("[A]dd, [D]elete, [U]pdate, [S]ync, [V]iew, [R]eset, [W]ipe, [T]ouch, E[x]ecute SQL Query, or [Q]uit").unwrap_or('?') {
+            'A' | 'a' => {
+                info!("Adding new record");
+                let record = read_login();
+                if let Err(e) = engine.add(record) {
+                    warn!("Failed to create record! {}", e);
+                }
+            }
+            'D' | 'd' => {
+                info!("Deleting record");
+                match prompt_record_id(&engine, "delete") {
+                    Ok(Some(id)) => {
+                        if let Err(e) = engine.delete(&id) {
+                            warn!("Failed to delete record! {}", e);
+                        }
+                    }
+                    Err(e) => {
+                        warn!("Failed to get record ID! {}", e);
+                    }
+                    _ => {}
+                }
+            }
+            'U' | 'u' => {
+                info!("Updating record fields");
+                match prompt_record_id(&engine, "update") {
+                    Err(e) => {
+                        warn!("Failed to get record ID! {}", e);
+                    }
+                    Ok(Some(id)) => {
+                        let mut login = match engine.get(&id) {
+                            Ok(Some(login)) => login,
+                            Ok(None) => {
+                                warn!("No such login!");
+                                continue
+                            }
+                            Err(e) => {
+                                warn!("Failed to update record (get failed) {}", e);
+                                continue;
+                            }
+                        };
+                        update_login(&mut login);
+                        if let Err(e) = engine.update(login) {
+                            warn!("Failed to update record! {}", e);
+                        }
+                    }
+                    _ => {}
+                }
+            }
+            'R' | 'r' => {
+                info!("Resetting client.");
+                if let Err(e) = engine.reset() {
+                    warn!("Failed to reset! {}", e);
+                }
+            }
+            'W' | 'w' => {
+                info!("Wiping all data from client!");
+                if let Err(e) = engine.wipe() {
+                    warn!("Failed to wipe! {}", e);
+                }
+            }
+            'S' | 's' => {
+                info!("Syncing!");
+                if let Err(e) = engine.sync(&client_init, &root_sync_key) {
+                    warn!("Sync failed! {}", e);
+                    warn!("BT: {:?}", e.backtrace());
+                } else {
+                    info!("Sync was successful!");
+                }
+            }
+            'V' | 'v' => {
+                if let Err(e) = show_all(&engine) {
+                    warn!("Failed to dump passwords? This is probably bad! {}", e);
+                }
+            }
+            'T' | 't' => {
+                info!("Touching (bumping use count) for a record");
+                match prompt_record_id(&engine, "update") {
+                    Err(e) => {
+                        warn!("Failed to get record ID! {}", e);
+                    }
+                    Ok(Some(id)) => {
+                        if let Err(e) = engine.touch(&id) {
+                            warn!("Failed to touch record! {}", e);
+                        }
+                    }
+                    _ => {}
+                }
+            }
+            'x' | 'X' => {
+                info!("Running arbitrary SQL, there's no way this could go wrong!");
+                if let Some(sql) = prompt_string("SQL (one line only, press enter when done):\n") {
+                    if let Err(e) = show_sql(&engine, &sql) {
+                        warn!("Failed to run sql query: {}", e);
+                    }
+                }
+            }
+            'Q' | 'q' => {
+                break;
+            }
+            '?' => {
+                continue;
+            }
+            c => {
+                println!("Unknown action '{}', exiting.", c);
+                break;
+            }
+        }
+    }
+
+    println!("Exiting (bye!)");
+    Ok(())
+}

logins-sql/ffi/Cargo.toml

@@ -0,0 +1,26 @@
+[package]
+name = "loginsql_ffi"
+version = "0.1.0"
+authors = ["Thom Chiovoloni <tchiovoloni@mozilla.com>"]
+
+[lib]
+name = "loginsapi_ffi"
+crate-type = ["lib", "staticlib", "cdylib"]
+
+[dependencies]
+serde_json = "1.0.26"
+log = "0.4.4"
+url = "1.7.1"
+
+[dependencies.rusqlite]
+version = "0.14.0"
+features = ["sqlcipher"]
+
+[dependencies.logins-sql]
+path = ".."
+
+[dependencies.sync15-adapter]
+path = "../../sync15-adapter"
+
+[target.'cfg(target_os = "android")'.dependencies]
+android_logger = "0.6.0"

logins-sql/ffi/src/error.rs

@@ -0,0 +1,221 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+use std::{self, panic, thread, ptr, process};
+use std::os::raw::c_char;
+use std::ffi::CString;
+
+use logins_sql::{
+    Result,
+    Error,
+    ErrorKind,
+};
+
+use sync15_adapter::{
+    ErrorKind as Sync15ErrorKind
+};
+
+#[inline]
+fn string_to_c_char(r_string: String) -> *mut c_char {
+    CString::new(r_string).unwrap().into_raw()
+}
+
+// "Translate" in the next few functions refers to translating a rust Result
+// type into a `(error, value)` tuple (well, sort of -- the `error` is taken as
+// an out parameter and the value is all that's returned, but it's a conceptual
+// tuple).
+
+pub unsafe fn with_translated_result<F, T>(error: *mut ExternError, callback: F) -> *mut T
+where F: FnOnce() -> Result<T> {
+    match try_call_with_result(error, callback) {
+        Some(v) => Box::into_raw(Box::new(v)),
+        None => ptr::null_mut(),
+    }
+}
+
+pub unsafe fn with_translated_void_result<F>(error: *mut ExternError, callback: F)
+where F: FnOnce() -> Result<()> {
+    let _: Option<()> = try_call_with_result(error, callback);
+}
+
+pub unsafe fn with_translated_value_result<F, T>(error: *mut ExternError, callback: F) -> T
+where
+    F: FnOnce() -> Result<T>,
+    T: Default,
+{
+    try_call_with_result(error, callback).unwrap_or_default()
+}
+
+pub unsafe fn with_translated_string_result<F>(error: *mut ExternError, callback: F) -> *mut c_char
+where F: FnOnce() -> Result<String> {
+    if let Some(s) = try_call_with_result(error, callback) {
+        string_to_c_char(s)
+    } else {
+        ptr::null_mut()
+    }
+}
+
+pub unsafe fn with_translated_opt_string_result<F>(error: *mut ExternError, callback: F) -> *mut c_char
+where F: FnOnce() -> Result<Option<String>> {
+    if let Some(Some(s)) = try_call_with_result(error, callback) {
+        string_to_c_char(s)
+    } else {
+        // This is either an error case, or callback returned None.
+        ptr::null_mut()
+    }
+}
+
+unsafe fn try_call_with_result<R, F>(out_error: *mut ExternError, callback: F) -> Option<R>
+where F: FnOnce() -> Result<R> {
+    // Ugh, using AssertUnwindSafe here is safe (in terms of memory safety),
+    // but a lie -- this code may behave improperly in the case that we unwind.
+    // That said, it's UB to unwind across the FFI boundary, and in practice
+    // weird things happen if we do (we aren't caught on the other side).
+    //
+    // We should eventually figure out a better story here, possibly the
+    // PasswordsEngine should get re-initialized if we hit this.
+    let res: thread::Result<(ExternError, Option<R>)> =
+        panic::catch_unwind(panic::AssertUnwindSafe(|| match callback() {
+            Ok(v) => (ExternError::default(), Some(v)),
+            Err(e) => (e.into(), None),
+        }));
+    match res {
+        Ok((err, o)) => {
+            if !out_error.is_null() {
+                let eref = &mut *out_error;
+                *eref = err;
+            } else {
+                error!("Fatal error: an error occurred but no error parameter was given {:?}", err);
+                process::abort();
+            }
+            o
+        }
+        Err(e) => {
+            if !out_error.is_null() {
+                let eref = &mut *out_error;
+                *eref = e.into();
+            } else {
+                let err: ExternError = e.into();
+                error!("Fatal error: a panic occurred but no error parameter was given {:?}", err);
+                process::abort();
+            }
+            None
+        }
+    }
+}
+
+/// C-compatible Error code. Negative codes are not expected to be handled by
+/// the application, a code of zero indicates that no error occurred, and a
+/// positive error code indicates an error that will likely need to be handled
+/// by the application
+#[repr(i32)]
+#[derive(Clone, Copy, Debug)]
+pub enum ExternErrorCode {
+
+    /// An unexpected error occurred which likely cannot be meaningfully handled
+    /// by the application.
+    OtherError = -2,
+
+    /// The rust code hit a `panic!` (or something equivalent, like `assert!`).
+    UnexpectedPanic = -1,
+
+    /// No error occcurred.
+    NoError = 0,
+
+    /// Indicates the FxA credentials are invalid, and should be refreshed.
+    AuthInvalidError = 1,
+
+    // TODO: lockbox indicated that they would want to know when we fail to open
+    // the DB due to invalid key.
+    // https://github.com/mozilla/application-services/issues/231
+}
+
+/// Represents an error that occurred on the rust side. Many rust FFI functions take a
+/// `*mut ExternError` as the last argument. This is an out parameter that indicates an
+/// error that occurred during that function's execution (if any).
+///
+/// For functions that use this pattern, if the ExternError's message property is null, then no
+/// error occurred. If the message is non-null then it contains a string description of the
+/// error that occurred.
+///
+/// Important: This message is allocated on the heap and it is the consumer's responsibility to
+/// free it!
+///
+/// While this pattern is not ergonomic in Rust, it offers two main benefits:
+///
+/// 1. It avoids defining a large number of `Result`-shaped types in the FFI consumer, as would
+///    be required with something like an `struct ExternResult<T> { ok: *mut T, err:... }`
+/// 2. It offers additional type safety over `struct ExternResult { ok: *mut c_void, err:... }`,
+///    which helps avoid memory safety errors.
+#[repr(C)]
+#[derive(Debug)]
+pub struct ExternError {
+
+    /// A string message, primarially intended for debugging. This will be null
+    /// in the case that no error occurred.
+    pub message: *mut c_char,
+
+    /// Error code.
+    /// - A code of 0 indicates no error
+    /// - A negative error code indicates an error which is not expected to be
+    ///   handled by the application.
+    pub code: ExternErrorCode,
+}
+
+impl Default for ExternError {
+    fn default() -> ExternError {
+        ExternError {
+            message: ptr::null_mut(),
+            code: ExternErrorCode::NoError,
+        }
+    }
+}
+
+fn get_code(err: &Error) -> ExternErrorCode {
+    match err.kind() {
+        ErrorKind::SyncAdapterError(e) => {
+            error!("Sync error {:?}", e);
+            match e.kind() {
+                Sync15ErrorKind::TokenserverHttpError(401) => {
+                    ExternErrorCode::AuthInvalidError
+                }
+                _ => ExternErrorCode::OtherError,
+            }
+        }
+        err => {
+            error!("Unexpected error: {:?}", err);
+            ExternErrorCode::OtherError
+        }
+    }
+}
+
+impl From<Error> for ExternError {
+    fn from(e: Error) -> ExternError {
+        let code = get_code(&e);
+        let message = string_to_c_char(e.to_string());
+        ExternError { message, code }
+    }
+}
+
+// This is the `Err` of std::thread::Result, which is what
+// `panic::catch_unwind` returns.
+impl From<Box<std::any::Any + Send + 'static>> for ExternError {
+    fn from(e: Box<std::any::Any + Send + 'static>) -> ExternError {
+        // The documentation suggests that it will usually be a str or String.
+        let message = if let Some(s) = e.downcast_ref::<&'static str>() {
+            string_to_c_char(s.to_string())
+        } else if let Some(s) = e.downcast_ref::<String>() {
+            string_to_c_char(s.clone())
+        } else {
+            // Note that it's important that this be allocated on the heap,
+            // since we'll free it later!
+            string_to_c_char("Unknown panic!".into())
+        };
+
+        ExternError {
+            code: ExternErrorCode::UnexpectedPanic,
+            message,
+        }
+    }
+}

logins-sql/ffi/src/lib.rs

@@ -0,0 +1,230 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+extern crate serde_json;
+extern crate rusqlite;
+extern crate logins_sql;
+extern crate sync15_adapter;
+extern crate url;
+#[macro_use] extern crate log;
+
+#[cfg(target_os = "android")]
+extern crate android_logger;
+
+pub mod error;
+
+use std::os::raw::c_char;
+use std::ffi::{CString, CStr};
+
+use error::{
+    ExternError,
+    with_translated_result,
+    with_translated_value_result,
+    with_translated_void_result,
+    with_translated_string_result,
+    with_translated_opt_string_result,
+};
+
+use logins_sql::{
+    Login,
+    PasswordEngine,
+};
+
+#[inline]
+unsafe fn c_str_to_str<'a>(cstr: *const c_char) -> &'a str {
+    CStr::from_ptr(cstr).to_str().unwrap_or_default()
+}
+
+fn logging_init() {
+    #[cfg(target_os = "android")]
+    {
+        android_logger::init_once(
+            android_logger::Filter::default().with_min_level(log::Level::Trace),
+            Some("libloginsapi_ffi"));
+        debug!("Android logging should be hooked up!")
+    }
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn sync15_passwords_state_new(
+    db_path: *const c_char,
+    encryption_key: *const c_char,
+    error: *mut ExternError
+) -> *mut PasswordEngine {
+    logging_init();
+    trace!("sync15_passwords_state_new");
+    with_translated_result(error, || {
+        let path = c_str_to_str(db_path);
+        let key = c_str_to_str(encryption_key);
+        let state = PasswordEngine::new(path, Some(key))?;
+        Ok(state)
+    })
+}
+
+// indirection to help `?` figure out the target error type
+fn parse_url(url: &str) -> sync15_adapter::Result<url::Url> {
+    Ok(url::Url::parse(url)?)
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn sync15_passwords_sync(
+    state: *mut PasswordEngine,
+    key_id: *const c_char,
+    access_token: *const c_char,
+    sync_key: *const c_char,
+    tokenserver_url: *const c_char,
+    error: *mut ExternError
+) {
+    trace!("sync15_passwords_sync");
+    with_translated_void_result(error, || {
+        assert!(!state.is_null(), "Null state passed to sync15_passwords_sync");
+        let state = &mut *state;
+        state.sync(
+            &sync15_adapter::Sync15StorageClientInit {
+                key_id: c_str_to_str(key_id).into(),
+                access_token: c_str_to_str(access_token).into(),
+                tokenserver_url: parse_url(c_str_to_str(tokenserver_url))?,
+            },
+            &sync15_adapter::KeyBundle::from_ksync_base64(
+                c_str_to_str(sync_key).into()
+            )?
+        )
+    })
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn sync15_passwords_touch(
+    state: *const PasswordEngine,
+    id: *const c_char,
+    error: *mut ExternError
+) {
+    trace!("sync15_passwords_touch");
+    with_translated_void_result(error, || {
+        assert!(!state.is_null(), "Null state passed to sync15_passwords_touch");
+        let state = &*state;
+        state.touch(c_str_to_str(id))
+    })
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn sync15_passwords_delete(
+    state: *const PasswordEngine,
+    id: *const c_char,
+    error: *mut ExternError
+) -> bool {
+    trace!("sync15_passwords_delete");
+    with_translated_value_result(error, || {
+        assert!(!state.is_null(), "Null state passed to sync15_passwords_delete");
+        let state = &*state;
+        state.delete(c_str_to_str(id))
+    })
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn sync15_passwords_wipe(
+    state: *const PasswordEngine,
+    error: *mut ExternError
+) {
+    trace!("sync15_passwords_wipe");
+    with_translated_void_result(error, || {
+        assert!(!state.is_null(), "Null state passed to sync15_passwords_wipe");
+        let state = &*state;
+        state.wipe()
+    })
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn sync15_passwords_reset(
+    state: *const PasswordEngine,
+    error: *mut ExternError
+) {
+    trace!("sync15_passwords_reset");
+    with_translated_void_result(error, || {
+        assert!(!state.is_null(), "Null state passed to sync15_passwords_reset");
+        let state = &*state;
+        state.reset()
+    })
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn sync15_passwords_get_all(
+    state: *const PasswordEngine,
+    error: *mut ExternError
+) -> *mut c_char {
+    trace!("sync15_passwords_get_all");
+    with_translated_string_result(error, || {
+        assert!(!state.is_null(), "Null state passed to sync15_passwords_get_all");
+        let state = &*state;
+        let all_passwords = state.list()?;
+        let result = serde_json::to_string(&all_passwords)?;
+        Ok(result)
+    })
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn sync15_passwords_get_by_id(
+    state: *const PasswordEngine,
+    id: *const c_char,
+    error: *mut ExternError
+) -> *mut c_char {
+    trace!("sync15_passwords_get_by_id");
+    with_translated_opt_string_result(error, || {
+        assert!(!state.is_null(), "Null state passed to sync15_passwords_get_by_id");
+        let state = &*state;
+        if let Some(password) = state.get(c_str_to_str(id))? {
+            Ok(Some(serde_json::to_string(&password)?))
+        } else {
+            Ok(None)
+        }
+    })
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn sync15_passwords_add(
+    state: *const PasswordEngine,
+    record_json: *const c_char,
+    error: *mut ExternError
+) -> *mut c_char {
+    trace!("sync15_passwords_add");
+    with_translated_string_result(error, || {
+        assert!(!state.is_null(), "Null state passed to sync15_passwords_add");
+        let state = &*state;
+        let mut parsed: serde_json::Value = serde_json::from_str(c_str_to_str(record_json))?;
+        if parsed.get("id").is_none() {
+            // Note: we replace this with a real guid in `db.rs`.
+            parsed["id"] = serde_json::Value::String(String::default());
+        }
+        let login: Login = serde_json::from_value(parsed)?;
+        state.add(login)
+    })
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn sync15_passwords_update(
+    state: *const PasswordEngine,
+    record_json: *const c_char,
+    error: *mut ExternError
+) {
+    trace!("sync15_passwords_update");
+    with_translated_void_result(error, || {
+        assert!(!state.is_null(), "Null state passed to sync15_passwords_update");
+        let state = &*state;
+        let parsed: Login = serde_json::from_str(c_str_to_str(record_json))?;
+        state.update(parsed)
+    });
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn sync15_passwords_destroy_string(s: *mut c_char) {
+    if !s.is_null() {
+        drop(CString::from_raw(s));
+    }
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn sync15_passwords_state_destroy(obj: *mut PasswordEngine) {
+    if !obj.is_null() {
+        drop(Box::from_raw(obj));
+    }
+}

logins-sql/src/db.rs

@@ -0,0 +1,764 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+use rusqlite::{Connection, types::{ToSql, FromSql}, Row, limits};
+use std::time::SystemTime;
+use std::path::Path;
+use std::collections::HashSet;
+use error::*;
+use schema;
+use login::{LocalLogin, MirrorLogin, Login, SyncStatus, SyncLoginData};
+use sync::{self, ServerTimestamp, IncomingChangeset, Store, OutgoingChangeset, Payload};
+use update_plan::UpdatePlan;
+use util;
+
+pub struct LoginDb {
+    pub db: Connection,
+    pub max_var_count: usize,
+}
+
+// In PRAGMA foo='bar', `'bar'` must be a constant string (it cannot be a
+// bound parameter), so we need to escape manually. According to
+// https://www.sqlite.org/faq.html, the only character that must be escaped is
+// the single quote, which is escaped by placing two single quotes in a row.
+fn escape_string_for_pragma(s: &str) -> String {
+    s.replace("'", "''")
+}
+
+impl LoginDb {
+    pub fn with_connection(db: Connection, encryption_key: Option<&str>) -> Result<Self> {
+        #[cfg(test)] {
+            util::init_test_logging();
+        }
+
+        let encryption_pragmas = if let Some(key) = encryption_key {
+            // TODO: We probably should support providing a key that doesn't go
+            // through PBKDF2 (e.g. pass it in as hex, or use sqlite3_key
+            // directly. See https://www.zetetic.net/sqlcipher/sqlcipher-api/#key
+            // "Raw Key Data" example. Note that this would be required to open
+            // existing iOS sqlcipher databases).
+            format!("PRAGMA key = '{}';", escape_string_for_pragma(key))
+        } else {
+            "".to_owned()
+        };
+
+        // `temp_store = 2` is required on Android to force the DB to keep temp
+        // files in memory, since on Android there's no tmp partition. See
+        // https://github.com/mozilla/mentat/issues/505. Ideally we'd only
+        // do this on Android, or allow caller to configure it.
+        let initial_pragmas = format!("
+            {}
+            PRAGMA temp_store = 2;
+        ", encryption_pragmas);
+
+        db.execute_batch(&initial_pragmas)?;
+
+        let max_var_limit = db.limit(limits::Limit::SQLITE_LIMIT_VARIABLE_NUMBER);
+        // This is an i32, so check before casting it. We also disallow 0
+        // because it's not clear how we could ever possibly handle it.
+        // (Realistically, we'll also fail to handle low values, but oh well).
+        assert!(max_var_limit > 0,
+                "SQLITE_LIMIT_VARIABLE_NUMBER must not be 0 or negative!");
+
+        let mut logins = Self {
+            db,
+            max_var_count: max_var_limit as usize
+        };
+        schema::init(&mut logins)?;
+        Ok(logins)
+    }
+
+    pub fn open(path: impl AsRef<Path>, encryption_key: Option<&str>) -> Result<Self> {
+        Ok(Self::with_connection(Connection::open(path)?, encryption_key)?)
+    }
+
+    pub fn open_in_memory(encryption_key: Option<&str>) -> Result<Self> {
+        Ok(Self::with_connection(Connection::open_in_memory()?, encryption_key)?)
+    }
+
+    pub fn execute_all(&self, stmts: &[&str]) -> Result<()> {
+        for sql in stmts {
+            self.execute(sql, &[])?;
+        }
+        Ok(())
+    }
+
+    #[inline]
+    pub fn execute(&self, stmt: &str, params: &[(&str, &ToSql)]) -> Result<usize> {
+        Ok(self.do_exec(stmt, params, false)?)
+    }
+
+    #[inline]
+    pub fn execute_cached(&self, stmt: &str, params: &[(&str, &ToSql)]) -> Result<usize> {
+        Ok(self.do_exec(stmt, params, true)?)
+    }
+
+    fn do_exec(&self, sql: &str, params: &[(&str, &ToSql)], cache: bool) -> Result<usize> {
+        let res = if cache {
+            self.db.prepare_cached(sql)
+                   .and_then(|mut s| s.execute_named(params))
+        } else {
+            self.db.execute_named(sql, params)
+        };
+        if let Err(e) = &res {
+            warn!("Error running SQL {}. Statement: {:?}", e, sql);
+        }
+        Ok(res?)
+    }
+
+    pub fn query_one<T: FromSql>(&self, sql: &str) -> Result<T> {
+        let res: T = self.db.query_row(sql, &[], |row| row.get(0))?;
+        Ok(res)
+    }
+
+    // The type returned by prepare/prepare_cached are different, but must live
+    // to the end of the function, so (AFAICT) it's difficult/impossible to
+    // remove the duplication between query_row/query_row_cached.
+
+    pub fn query_row<T>(&self, sql: &str, args: &[(&str, &ToSql)], f: impl FnOnce(&Row) -> Result<T>) -> Result<Option<T>> {
+        let mut stmt = self.db.prepare(sql)?;
+        let res = stmt.query_named(args);
+        if let Err(e) = &res {
+            warn!("Error executing query: {}. Query: {}", e, sql);
+        }
+        let mut rows = res?;
+        match rows.next() {
+            Some(result) => Ok(Some(f(&result?)?)),
+            None => Ok(None),
+        }
+    }
+
+    pub fn query_row_cached<T>(&self, sql: &str, args: &[(&str, &ToSql)], f: impl FnOnce(&Row) -> Result<T>) -> Result<Option<T>> {
+        let mut stmt = self.db.prepare_cached(sql)?;
+        let res = stmt.query_named(args);
+        if let Err(e) = &res {
+            warn!("Error executing query: {}. Query: {}", e, sql);
+        }
+        let mut rows = res?;
+        match rows.next() {
+            Some(result) => Ok(Some(f(&result?)?)),
+            None => Ok(None),
+        }
+    }
+}
+
+// login specific stuff.
+
+impl LoginDb {
+
+    fn mark_as_synchronized(&mut self, guids: &[&str], ts: ServerTimestamp) -> Result<()> {
+        util::each_chunk(guids, self.max_var_count, |chunk, _| {
+            self.db.execute(
+                &format!("DELETE FROM loginsM WHERE guid IN ({vars})",
+                         vars = util::sql_vars(chunk.len())),
+                chunk
+            )?;
+
+            self.db.execute(
+                &format!("
+                    INSERT OR IGNORE INTO loginsM (
+                        {common_cols}, is_overridden, server_modified
+                    )
+                    SELECT {common_cols}, 0, {modified_ms_i64}
+                    FROM loginsL
+                    WHERE is_deleted = 0 AND guid IN ({vars})",
+                    common_cols = schema::COMMON_COLS,
+                    modified_ms_i64 = ts.as_millis() as i64,
+                    vars = util::sql_vars(chunk.len())),
+                chunk
+            )?;
+
+            self.db.execute(
+                &format!("DELETE FROM loginsL WHERE guid IN ({vars})",
+                         vars = util::sql_vars(chunk.len())),
+                chunk
+            )?;
+            Ok(())
+        })?;
+        self.set_last_sync(ts)?;
+        Ok(())
+    }
+
+    // Fetch all the data for the provided IDs.
+    // TODO: Might be better taking a fn instead of returning all of it... But that func will likely
+    // want to insert stuff while we're doing this so ugh.
+    fn fetch_login_data(&self, records: &[(sync::Payload, ServerTimestamp)]) -> Result<Vec<SyncLoginData>> {
+        let mut sync_data = Vec::with_capacity(records.len());
+        {
+            let mut seen_ids: HashSet<String> = HashSet::with_capacity(records.len());
+            for incoming in records.iter() {
+                if seen_ids.contains(&incoming.0.id) {
+                    throw!(ErrorKind::DuplicateGuid(incoming.0.id.to_string()))
+                }
+                seen_ids.insert(incoming.0.id.clone());
+                sync_data.push(SyncLoginData::from_payload(incoming.0.clone(), incoming.1)?);
+            }
+        }
+
+        util::each_chunk_mapped(&records, self.max_var_count, |r| &r.0.id as &ToSql, |chunk, offset| {
+            // pairs the bound parameter for the guid with an integer index.
+            let values_with_idx = util::repeat_display(chunk.len(), ",", |i, f| write!(f, "({},?)", i + offset));
+            let query = format!("
+                WITH to_fetch(guid_idx, fetch_guid) AS (VALUES {vals})
+                SELECT
+                    {common_cols},
+                    is_overridden,
+                    server_modified,
+                    NULL as local_modified,
+                    NULL as is_deleted,
+                    NULL as sync_status,
+                    1 as is_mirror,
+                    to_fetch.guid_idx as guid_idx
+                FROM loginsM
+                JOIN to_fetch
+                  ON loginsM.guid = to_fetch.fetch_guid
+
+                UNION ALL
+
+                SELECT
+                    {common_cols},
+                    NULL as is_overridden,
+                    NULL as server_modified,
+                    local_modified,
+                    is_deleted,
+                    sync_status,
+                    0 as is_mirror,
+                    to_fetch.guid_idx as guid_idx
+                FROM loginsL
+                JOIN to_fetch
+                  ON loginsL.guid = to_fetch.fetch_guid",
+                // give each VALUES item 2 entries, an index and the parameter.
+                vals = values_with_idx,
+                common_cols = schema::COMMON_COLS,
+            );
+
+            let mut stmt = self.db.prepare(&query)?;
+
+            let rows = stmt.query_and_then(chunk, |row| {
+                let guid_idx_i = row.get::<_, i64>("guid_idx");
+                // Hitting this means our math is wrong...
+                assert!(guid_idx_i >= 0);
+
+                let guid_idx = guid_idx_i as usize;
+                let is_mirror: bool = row.get("is_mirror");
+                if is_mirror {
+                    sync_data[guid_idx].set_mirror(MirrorLogin::from_row(row)?)?;
+                } else {
+                    sync_data[guid_idx].set_local(LocalLogin::from_row(row)?)?;
+                }
+                Ok(())
+            })?;
+            // `rows` is an Iterator<Item = Result<()>>, so we need to collect to handle the errors.
+            rows.collect::<Result<_>>()?;
+            Ok(())
+        })?;
+        Ok(sync_data)
+    }
+
+    // It would be nice if this were a batch-ish api (e.g. takes a slice of records and finds dupes
+    // for each one if they exist)... I can't think of how to write that query, though.
+    fn find_dupe(&self, l: &Login) -> Result<Option<Login>> {
+        let form_submit_host_port = l.form_submit_url.as_ref().and_then(|s| util::url_host_port(&s));
+        let args = &[
+            (":hostname", &l.hostname as &ToSql),
+            (":http_realm", &l.http_realm as &ToSql),
+            (":username", &l.username as &ToSql),
+            (":form_submit", &form_submit_host_port as &ToSql),
+        ];
+        let mut query = format!("
+            SELECT {common}
+            FROM loginsL
+            WHERE hostname IS :hostname
+              AND httpRealm IS :http_realm
+              AND username IS :username",
+            common = schema::COMMON_COLS,
+        );
+        if form_submit_host_port.is_some() {
+            // Stolen from iOS
+            query += " AND (formSubmitURL = '' OR (instr(formSubmitURL, :form_submit) > 0))";
+        } else {
+            query += " AND formSubmitURL IS :form_submit"
+        }
+        Ok(self.query_row(&query, args, |row| Login::from_row(row))?)
+    }
+
+    pub fn get_all(&self) -> Result<Vec<Login>> {
+        let mut stmt = self.db.prepare_cached(&GET_ALL_SQL)?;
+        let rows = stmt.query_and_then(&[], Login::from_row)?;
+        rows.collect::<Result<_>>()
+    }
+
+    pub fn get_by_id(&self, id: &str) -> Result<Option<Login>> {
+        // Probably should be cached...
+        self.query_row(&GET_BY_GUID_SQL,
+                       &[(":guid", &id as &ToSql)],
+                       Login::from_row)
+    }
+
+    pub fn touch(&self, id: &str) -> Result<()> {
+        self.ensure_local_overlay_exists(id)?;
+        self.mark_mirror_overridden(id)?;
+        let now_ms = util::system_time_ms_i64(SystemTime::now());
+        // As on iOS, just using a record doesn't flip it's status to changed.
+        // TODO: this might be wrong for lockbox!
+        self.execute_cached("
+            UPDATE loginsL
+               SET timeLastUsed = :now_millis,
+                   timesUsed = timesUsed + 1,
+                   local_modified = :now_millis
+               WHERE guid = :guid
+                 AND is_deleted = 0",
+            &[(":now_millis", &now_ms as &ToSql),
+              (":guid", &id as &ToSql)]
+        )?;
+        Ok(())
+    }
+
+    pub fn add(&self, mut login: Login) -> Result<Login> {
+        login.check_valid()?;
+
+        let now_ms = util::system_time_ms_i64(SystemTime::now());
+
+        // Allow an empty GUID to be passed to indicate that we should generate
+        // one. (Note that the FFI, does not require that the `id` field be
+        // present in the JSON, and replaces it with an empty string if missing).
+        if login.id.is_empty() {
+            // Our FFI handles panics so this is fine. In practice there's not
+            // much we can do here. Using a CSPRNG for this is probably
+            // unnecessary, so we likely could fall back to something less
+            // fallible eventually, but it's unlikely very much else will work
+            // if this fails, so it doesn't matter much.
+            login.id = sync::util::random_guid()
+                .expect("Failed to generate failed to generate random bytes for GUID");
+        }
+
+        // Fill in default metadata.
+        // TODO: allow this to be provided for testing?
+        login.time_created = now_ms;
+        login.time_password_changed = now_ms;
+        login.time_last_used = now_ms;
+        login.times_used = 1;
+
+        let sql = format!("
+            INSERT OR IGNORE INTO loginsL (
+                hostname,
+                httpRealm,
+                formSubmitURL,
+                usernameField,
+                passwordField,
+                timesUsed,
+                username,
+                password,
+                guid,
+                timeCreated,
+                timeLastUsed,
+                timePasswordChanged,
+                local_modified,
+                is_deleted,
+                sync_status
+            ) VALUES (
+                :hostname,
+                :http_realm,
+                :form_submit_url,
+                :username_field,
+                :password_field,
+                :times_used,
+                :username,
+                :password,
+                :guid,
+                :time_created,
+                :time_last_used,
+                :time_password_changed,
+                :local_modified,
+                0, -- is_deleted
+                {new} -- sync_status
+            )", new = SyncStatus::New as u8);
+
+        let rows_changed = self.execute(&sql, &[
+            (":hostname", &login.hostname as &ToSql),
+            (":http_realm", &login.http_realm as &ToSql),
+            (":form_submit_url", &login.form_submit_url as &ToSql),
+            (":username_field", &login.username_field as &ToSql),
+            (":password_field", &login.password_field as &ToSql),
+            (":username", &login.username as &ToSql),
+            (":password", &login.password as &ToSql),
+            (":guid", &login.id as &ToSql),
+            (":time_created", &login.time_created as &ToSql),
+            (":times_used", &login.times_used as &ToSql),
+            (":time_last_used", &login.time_last_used as &ToSql),
+            (":time_password_changed", &login.time_password_changed as &ToSql),
+            (":local_modified", &now_ms as &ToSql)
+        ])?;
+        if rows_changed == 0 {
+            error!("Record {:?} already exists (use `update` to update records, not add)",
+                   login.id);
+            throw!(ErrorKind::DuplicateGuid(login.id));
+        }
+        Ok(login)
+    }
+
+    pub fn update(&self, login: Login) -> Result<()> {
+        login.check_valid()?;
+        // Note: These fail with DuplicateGuid if the record doesn't exist.
+        self.ensure_local_overlay_exists(login.guid_str())?;
+        self.mark_mirror_overridden(login.guid_str())?;
+
+        let now_ms = util::system_time_ms_i64(SystemTime::now());
+
+        let sql = format!("
+            UPDATE loginsL
+            SET local_modified      = :now_millis,
+                timeLastUsed        = :now_millis,
+                -- Only update timePasswordChanged if, well, the password changed.
+                timePasswordChanged = (CASE
+                    WHEN password = :password
+                    THEN timePasswordChanged
+                    ELSE :now_millis
+                END),
+                httpRealm           = :http_realm,
+                formSubmitURL       = :form_submit_url,
+                usernameField       = :username_field,
+                passwordField       = :password_field,
+                timesUsed           = timesUsed + 1,
+                username            = :username,
+                password            = :password,
+                hostname            = :hostname,
+                -- leave New records as they are, otherwise update them to `changed`
+                sync_status         = max(sync_status, {changed})
+            WHERE guid = :guid",
+            changed = SyncStatus::Changed as u8
+        );
+
+        self.db.execute_named(&sql, &[
+            (":hostname", &login.hostname as &ToSql),
+            (":username", &login.username as &ToSql),
+            (":password", &login.password as &ToSql),
+            (":http_realm", &login.http_realm as &ToSql),
+            (":form_submit_url", &login.form_submit_url as &ToSql),
+            (":username_field", &login.username_field as &ToSql),
+            (":password_field", &login.password_field as &ToSql),
+            (":guid", &login.id as &ToSql),
+            (":now_millis", &now_ms as &ToSql),
+        ])?;
+        Ok(())
+    }
+
+    pub fn exists(&self, id: &str) -> Result<bool> {
+        Ok(self.query_row("
+            SELECT EXISTS(
+                SELECT 1 FROM loginsL
+                WHERE guid = :guid AND is_deleted = 0
+                UNION ALL
+                SELECT 1 FROM loginsM
+                WHERE guid = :guid AND is_overridden IS NOT 1
+            )",
+            &[(":guid", &id as &ToSql)],
+            |row| Ok(row.get(0))
+        )?.unwrap_or(false))
+    }
+
+    /// Delete the record with the provided id. Returns true if the record
+    /// existed already.
+    pub fn delete(&self, id: &str) -> Result<bool> {
+        let exists = self.exists(id)?;
+        let now_ms = util::system_time_ms_i64(SystemTime::now());
+
+        // Directly delete IDs that have not yet been synced to the server
+        self.execute(&format!("
+            DELETE FROM loginsL
+            WHERE guid = :guid
+              AND sync_status = {status_new}",
+            status_new = SyncStatus::New as u8),
+            &[(":guid", &id as &ToSql)]
+        )?;
+
+        // For IDs that have, mark is_deleted and clear sensitive fields
+        self.execute(&format!("
+            UPDATE loginsL
+            SET local_modified = :now_ms,
+                sync_status = {status_changed},
+                is_deleted = 1,
+                password = '',
+                hostname = '',
+                username = ''
+            WHERE guid = :guid",
+            status_changed = SyncStatus::Changed as u8),
+            &[(":now_ms", &now_ms as &ToSql), (":guid", &id as &ToSql)])?;
+
+        // Mark the mirror as overridden
+        self.execute("UPDATE loginsM SET is_overridden = 1 WHERE guid = :guid",
+                     &[(":guid", &id as &ToSql)])?;
+
+        // If we don't have a local record for this ID, but do have it in the mirror
+        // insert a tombstone.
+        self.execute(&format!("
+            INSERT OR IGNORE INTO loginsL
+                    (guid, local_modified, is_deleted, sync_status, hostname, timeCreated, timePasswordChanged, password, username)
+            SELECT   guid, :now_ms,        1,          {changed},   '',       timeCreated, :now_ms,                   '',       ''
+            FROM loginsM
+            WHERE guid = :guid",
+            changed = SyncStatus::Changed as u8),
+            &[(":now_ms", &now_ms as &ToSql),
+              (":guid", &id as &ToSql)])?;
+
+        Ok(exists)
+    }
+
+    fn mark_mirror_overridden(&self, guid: &str) -> Result<()> {
+        self.execute_cached("
+            UPDATE loginsM SET
+            is_overridden = 1
+            WHERE guid = :guid
+        ", &[(":guid", &guid as &ToSql)])?;
+        Ok(())
+    }
+
+    fn ensure_local_overlay_exists(&self, guid: &str) -> Result<()> {
+        let already_have_local: bool = self.query_row_cached(
+            "SELECT EXISTS(SELECT 1 FROM loginsL WHERE guid = :guid)",
+            &[(":guid", &guid as &ToSql)],
+            |row| Ok(row.get(0))
+        )?.unwrap_or_default();
+
+        if already_have_local {
+            return Ok(())
+        }
+
+        debug!("No overlay; cloning one for {:?}.", guid);
+        let changed = self.clone_mirror_to_overlay(guid)?;
+        if changed == 0 {
+            error!("Failed to create local overlay for GUID {:?}.", guid);
+            throw!(ErrorKind::NoSuchRecord(guid.to_owned()));
+        }
+        Ok(())
+    }
+
+    fn clone_mirror_to_overlay(&self, guid: &str) -> Result<usize> {
+        self.execute_cached(
+            &*CLONE_SINGLE_MIRROR_SQL,
+            &[(":guid", &guid as &ToSql)]
+        )
+    }
+
+    pub fn reset(&self) -> Result<()> {
+        info!("Executing reset on password store!");
+        self.execute_all(&[
+            &*CLONE_ENTIRE_MIRROR_SQL,
+            "DELETE FROM loginsM",
+            &format!("UPDATE loginsL SET sync_status = {}", SyncStatus::New as u8),
+        ])?;
+        self.set_last_sync(ServerTimestamp(0.0))?;
+        // TODO: Should we clear global_state?
+        Ok(())
+    }
+
+    pub fn wipe(&self) -> Result<()> {
+        info!("Executing reset on password store!");
+        let now_ms = util::system_time_ms_i64(SystemTime::now());
+
+        self.execute(&format!("DELETE FROM loginsL WHERE sync_status = {new}", new = SyncStatus::New as u8), &[])?;
+        self.execute(
+            &format!("
+                UPDATE loginsL
+                SET local_modified = :now_ms,
+                    sync_status = {changed},
+                    is_deleted = 1,
+                    password = '',
+                    hostname = '',
+                    username = ''
+                WHERE is_deleted = 0",
+                changed = SyncStatus::Changed as u8),
+            &[(":now_ms", &now_ms as &ToSql)])?;
+
+        self.execute("UPDATE loginsM SET is_overridden = 1", &[])?;
+
+        self.execute(
+            &format!("
+                INSERT OR IGNORE INTO loginsL
+                      (guid, local_modified, is_deleted, sync_status, hostname, timeCreated, timePasswordChanged, password, username)
+                SELECT guid, :now_ms,        1,          {changed},   '',       timeCreated, :now_ms,             '',       ''
+                FROM loginsM",
+                changed = SyncStatus::Changed as u8),
+            &[(":now_ms", &now_ms as &ToSql)])?;
+
+        Ok(())
+    }
+
+    fn reconcile(&self, records: Vec<SyncLoginData>, server_now: ServerTimestamp) -> Result<UpdatePlan> {
+        let mut plan = UpdatePlan::default();
+
+        for mut record in records {
+            debug!("Processing remote change {}", record.guid());
+            let upstream = if let Some(inbound) = record.inbound.0.take() {
+                inbound
+            } else {
+                debug!("Processing inbound deletion (always prefer)");
+                plan.plan_delete(record.guid.clone());
+                continue;
+            };
+            let upstream_time = record.inbound.1;
+            match (record.mirror.take(), record.local.take()) {
+                (Some(mirror), Some(local)) => {
+                    debug!("  Conflict between remote and local, Resolving with 3WM");
+                    plan.plan_three_way_merge(
+                        local, mirror, upstream, upstream_time, server_now);
+                }
+                (Some(_mirror), None) => {
+                    debug!("  Forwarding mirror to remote");
+                    plan.plan_mirror_update(upstream, upstream_time);
+                }
+                (None, Some(local)) => {
+                    debug!("  Conflicting record without shared parent, using newer");
+                    plan.plan_two_way_merge(&local.login, (upstream, upstream_time));
+                }
+                (None, None) => {
+                    if let Some(dupe) = self.find_dupe(&upstream)? {
+                        debug!("  Incoming record {} was is a dupe of local record {}", upstream.id, dupe.id);
+                        plan.plan_two_way_merge(&dupe, (upstream, upstream_time));
+                    } else {
+                        debug!("  No dupe found, inserting into mirror");
+                        plan.plan_mirror_insert(upstream, upstream_time, false);
+                    }
+                }
+            }
+        }
+        Ok(plan)
+    }
+
+    fn execute_plan(&mut self, plan: UpdatePlan) -> Result<()> {
+        let mut tx = self.db.transaction()?;
+        plan.execute(&mut tx, self.max_var_count)?;
+        tx.commit()?;
+        Ok(())
+    }
+
+    pub fn fetch_outgoing(&self, st: ServerTimestamp) -> Result<OutgoingChangeset> {
+        let mut outgoing = OutgoingChangeset::new("passwords".into(), st);
+        let mut stmt = self.db.prepare_cached(&format!("
+            SELECT * FROM loginsL
+            WHERE sync_status IS NOT {synced}",
+            synced = SyncStatus::Synced as u8
+        ))?;
+        let rows = stmt.query_and_then(&[], |row| {
+            Ok(if row.get::<_, bool>("is_deleted") {
+                Payload::new_tombstone(row.get_checked::<_, String>("guid")?)
+            } else {
+                let login = Login::from_row(row)?;
+                Payload::from_record(login)?
+            })
+        })?;
+        outgoing.changes = rows.collect::<Result<_>>()?;
+
+        Ok(outgoing)
+    }
+
+    fn do_apply_incoming(
+        &mut self,
+        inbound: IncomingChangeset
+    ) -> Result<OutgoingChangeset> {
+        let data = self.fetch_login_data(&inbound.changes)?;
+        let plan = self.reconcile(data, inbound.timestamp)?;
+        self.execute_plan(plan)?;
+        Ok(self.fetch_outgoing(inbound.timestamp)?)
+    }
+
+    fn put_meta(&self, key: &str, value: &ToSql) -> Result<()> {
+        self.execute_cached(
+            "REPLACE INTO loginsSyncMeta (key, value) VALUES (:key, :value)",
+            &[(":key", &key as &ToSql), (":value", value)]
+        )?;
+        Ok(())
+    }
+
+    fn get_meta<T: FromSql>(&self, key: &str) -> Result<Option<T>> {
+        self.query_row_cached(
+            "SELECT value FROM loginsSyncMeta WHERE key = :key",
+            &[(":key", &key as &ToSql)],
+            |row| Ok(row.get_checked(0)?)
+        )
+    }
+
+    pub fn set_last_sync(&self, last_sync: ServerTimestamp) -> Result<()> {
+        debug!("Updating last sync to {}", last_sync);
+        let last_sync_millis = last_sync.as_millis() as i64;
+        self.put_meta(schema::LAST_SYNC_META_KEY, &last_sync_millis)
+    }
+
+    pub fn set_global_state(&self, global_state: &str) -> Result<()> {
+        self.put_meta(schema::GLOBAL_STATE_META_KEY, &global_state)
+    }
+
+    pub fn get_last_sync(&self) -> Result<Option<ServerTimestamp>> {
+        Ok(self.get_meta::<i64>(schema::LAST_SYNC_META_KEY)?
+            .map(|millis| ServerTimestamp(millis as f64 / 1000.0)))
+    }
+
+    pub fn get_global_state(&self) -> Result<Option<String>> {
+        self.get_meta::<String>(schema::GLOBAL_STATE_META_KEY)
+    }
+}
+
+impl Store for LoginDb {
+    type Error = Error;
+
+    fn apply_incoming(
+        &mut self,
+        inbound: IncomingChangeset
+    ) -> Result<OutgoingChangeset> {
+        self.do_apply_incoming(inbound)
+    }
+
+    fn sync_finished(
+        &mut self,
+        new_timestamp: ServerTimestamp,
+        records_synced: &[String],
+    ) -> Result<()> {
+        self.mark_as_synchronized(
+            &records_synced.iter().map(|r| r.as_str()).collect::<Vec<_>>(),
+            new_timestamp
+        )
+    }
+}
+
+lazy_static! {
+
+    static ref GET_ALL_SQL: String = format!("
+        SELECT {common_cols} FROM loginsL WHERE is_deleted = 0
+        UNION ALL
+        SELECT {common_cols} FROM loginsM WHERE is_overridden = 0
+    ",
+        common_cols = schema::COMMON_COLS,
+    );
+
+    static ref GET_BY_GUID_SQL: String = format!("
+        SELECT {common_cols}
+        FROM loginsL
+        WHERE is_deleted = 0
+          AND guid = :guid
+
+        UNION ALL
+
+        SELECT {common_cols}
+        FROM loginsM
+        WHERE is_overridden IS NOT 1
+          AND guid = :guid
+        ORDER BY hostname ASC
+
+        LIMIT 1
+    ",
+        common_cols = schema::COMMON_COLS,
+    );
+
+    static ref CLONE_ENTIRE_MIRROR_SQL: String = format!("
+        INSERT OR IGNORE INTO loginsL ({common_cols}, local_modified, is_deleted, sync_status)
+        SELECT {common_cols}, NULL AS local_modified, 0 AS is_deleted, 0 AS sync_status
+        FROM loginsM",
+        common_cols = schema::COMMON_COLS,
+    );
+
+    static ref CLONE_SINGLE_MIRROR_SQL: String = format!(
+        "{} WHERE guid = :guid",
+        &*CLONE_ENTIRE_MIRROR_SQL,
+    );
+}

logins-sql/src/engine.rs

@@ -0,0 +1,296 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+use login::Login;
+use error::*;
+use sync::{self, Sync15StorageClient, Sync15StorageClientInit, GlobalState, KeyBundle};
+use db::LoginDb;
+use std::path::Path;
+use serde_json;
+use rusqlite;
+
+#[derive(Debug)]
+pub(crate) struct SyncInfo {
+    pub state: GlobalState,
+    pub client: Sync15StorageClient,
+    // Used so that we know whether or not we need to re-initialize `client`
+    pub last_client_init: Sync15StorageClientInit,
+}
+
+// This isn't really an engine in the firefox sync15 desktop sense -- it's
+// really a bundle of state that contains the sync storage client, the sync
+// state, and the login DB.
+pub struct PasswordEngine {
+    sync: Option<SyncInfo>,
+    db: LoginDb,
+}
+
+impl PasswordEngine {
+
+    pub fn new(path: impl AsRef<Path>, encryption_key: Option<&str>) -> Result<Self> {
+        let db = LoginDb::open(path, encryption_key)?;
+        Ok(Self { db, sync: None })
+    }
+
+    pub fn new_in_memory(encryption_key: Option<&str>) -> Result<Self> {
+        let db = LoginDb::open_in_memory(encryption_key)?;
+        Ok(Self { db, sync: None })
+    }
+
+    pub fn list(&self) -> Result<Vec<Login>> {
+        self.db.get_all()
+    }
+
+    pub fn get(&self, id: &str) -> Result<Option<Login>> {
+        self.db.get_by_id(id)
+    }
+
+    pub fn touch(&self, id: &str) -> Result<()> {
+        self.db.touch(id)
+    }
+
+    pub fn delete(&self, id: &str) -> Result<bool> {
+        self.db.delete(id)
+    }
+
+    pub fn wipe(&self) -> Result<()> {
+        self.db.wipe()
+    }
+
+    pub fn reset(&self) -> Result<()> {
+        self.db.reset()
+    }
+
+    pub fn update(&self, login: Login) -> Result<()> {
+        self.db.update(login)
+    }
+
+    pub fn add(&self, login: Login) -> Result<String> {
+        // Just return the record's ID (which we may have generated).
+        self.db.add(login).map(|record| record.id)
+    }
+
+    // This is basiclaly exposed just for sync_pass_sql, but it doesn't seem
+    // unreasonable.
+    pub fn conn(&self) -> &rusqlite::Connection {
+        &self.db.db
+    }
+
+    pub fn sync(
+        &mut self,
+        storage_init: &Sync15StorageClientInit,
+        root_sync_key: &KeyBundle
+    ) -> Result<()> {
+
+        // Note: If `to_ready` (or anything else with a ?) fails below, this
+        // `take()` means we end up with `state.sync.is_none()`, which means the
+        // next sync will redownload meta/global, crypto/keys, etc. without
+        // needing to. Apparently this is both okay and by design.
+        let maybe_sync_info = self.sync.take().map(Ok);
+
+        // `maybe_sync_info` is None if we haven't called `sync` since
+        // restarting the browser.
+        //
+        // If this is the case we may or may not have a persisted version of
+        // GlobalState stored in the DB (we will iff we've synced before, unless
+        // we've `reset()`, which clears it out).
+        let mut sync_info = maybe_sync_info.unwrap_or_else(|| -> Result<SyncInfo> {
+            info!("First time through since unlock. Trying to load persisted global state.");
+            let state = if let Some(persisted_global_state) = self.db.get_global_state()? {
+                serde_json::from_str::<GlobalState>(&persisted_global_state)
+                .unwrap_or_else(|_| {
+                    // Don't log the error since it might contain sensitive
+                    // info like keys (the JSON does, after all).
+                    error!("Failed to parse GlobalState from JSON! Falling back to default");
+                    // Unstick ourselves by using the default state.
+                    GlobalState::default()
+                })
+            } else {
+                info!("No previously persisted global state, using default");
+                GlobalState::default()
+            };
+            let client = Sync15StorageClient::new(storage_init.clone())?;
+            Ok(SyncInfo {
+                state,
+                client,
+                last_client_init: storage_init.clone(),
+            })
+        })?;
+
+        // If the options passed for initialization of the storage client aren't
+        // the same as the ones we used last time, reinitialize it. (Note that
+        // we could avoid the comparison in the case where we had `None` in
+        // `state.sync` before, but this probably doesn't matter).
+        //
+        // It's a little confusing that we do things this way (transparently
+        // re-initialize the client), but it reduces the size of the API surface
+        // exposed over the FFI, and simplifies the states that the client code
+        // has to consider (as far as it's concerned it just has to pass
+        // `current` values for these things, and not worry about having to
+        // re-initialize the sync state).
+        if storage_init != &sync_info.last_client_init {
+            info!("Detected change in storage client init, updating");
+            sync_info.client = Sync15StorageClient::new(storage_init.clone())?;
+            sync_info.last_client_init = storage_init.clone();
+        }
+
+        // Advance the state machine to the point where it can perform a full
+        // sync. This may involve uploading meta/global, crypto/keys etc.
+        {
+            // Scope borrow of `sync_info.client`
+            let mut state_machine =
+                sync::SetupStateMachine::for_full_sync(&sync_info.client, &root_sync_key);
+            info!("Advancing state machine to ready (full)");
+            let next_sync_state = state_machine.to_ready(sync_info.state)?;
+            sync_info.state = next_sync_state;
+        }
+
+        // Reset our local state if necessary.
+        if sync_info.state.engines_that_need_local_reset().contains("passwords") {
+            info!("Passwords sync ID changed; engine needs local reset");
+            self.db.reset()?;
+        }
+
+        // Persist the current sync state in the DB.
+        info!("Updating persisted global state");
+        let s = sync_info.state.to_persistable_string();
+        self.db.set_global_state(&s)?;
+
+        info!("Syncing passwords engine!");
+
+        let ts = self.db.get_last_sync()?.unwrap_or_default();
+
+        // We don't use `?` here so that we can restore the value of of
+        // `self.sync` even if sync fails.
+        let result = sync::synchronize(
+            &sync_info.client,
+            &sync_info.state,
+            &mut self.db,
+            "passwords".into(),
+            ts,
+            true
+        );
+
+        match &result {
+            Ok(()) => info!("Sync was successful!"),
+            Err(e) => warn!("Sync failed! {:?}", e),
+        }
+
+        // Restore our value of `sync_info` even if the sync failed.
+        self.sync = Some(sync_info);
+
+        Ok(result?)
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use std::time::SystemTime;
+    use util;
+    // Doesn't check metadata fields
+    fn assert_logins_equiv(a: &Login, b: &Login) {
+        assert_eq!(b.id, a.id);
+        assert_eq!(b.hostname, a.hostname);
+        assert_eq!(b.form_submit_url, a.form_submit_url);
+        assert_eq!(b.http_realm, a.http_realm);
+        assert_eq!(b.username, a.username);
+        assert_eq!(b.password, a.password);
+        assert_eq!(b.username_field, a.username_field);
+        assert_eq!(b.password_field, a.password_field);
+    }
+
+    #[test]
+    fn test_general() {
+        let engine = PasswordEngine::new_in_memory(Some("secret")).unwrap();
+        let list = engine.list().expect("Grabbing Empty list to work");
+        assert_eq!(list.len(), 0);
+        let start_us = util::system_time_ms_i64(SystemTime::now());
+
+        let a = Login {
+            id: "aaaaaaaaaaaa".into(),
+            hostname: "https://www.example.com".into(),
+            form_submit_url: Some("https://www.example.com/login".into()),
+            username: "coolperson21".into(),
+            password: "p4ssw0rd".into(),
+            username_field: "user_input".into(),
+            password_field: "pass_input".into(),
+            .. Login::default()
+        };
+
+        let b = Login {
+            // Note: no ID, should be autogenerated for us
+            hostname: "https://www.example2.com".into(),
+            http_realm: Some("Some String Here".into()),
+            username: "asdf".into(),
+            password: "fdsa".into(),
+            username_field: "input_user".into(),
+            password_field: "input_pass".into(),
+            .. Login::default()
+        };
+
+        let a_id = engine.add(a.clone()).expect("added a");
+        let b_id = engine.add(b.clone()).expect("added b");
+
+        assert_eq!(a_id, a.id);
+
+        assert_ne!(b_id, b.id, "Should generate guid when none provided");
+
+        let a_from_db = engine.get(&a_id)
+            .expect("Not to error getting a")
+            .expect("a to exist");
+
+        assert_logins_equiv(&a, &a_from_db);
+        assert_ge!(a_from_db.time_created, start_us);
+        assert_ge!(a_from_db.time_password_changed, start_us);
+        assert_ge!(a_from_db.time_last_used, start_us);
+        assert_eq!(a_from_db.times_used, 1);
+
+        let b_from_db = engine.get(&b_id)
+            .expect("Not to error getting b")
+            .expect("b to exist");
+
+        assert_logins_equiv(&b_from_db, &Login {
+            id: b_id.clone(),
+            .. b.clone()
+        });
+        assert_ge!(b_from_db.time_created, start_us);
+        assert_ge!(b_from_db.time_password_changed, start_us);
+        assert_ge!(b_from_db.time_last_used, start_us);
+        assert_eq!(b_from_db.times_used, 1);
+
+        let mut list = engine.list().expect("Grabbing list to work");
+        assert_eq!(list.len(), 2);
+        let mut expect = vec![a_from_db.clone(), b_from_db.clone()];
+
+        list.sort_by(|a, b| b.id.cmp(&a.id));
+        expect.sort_by(|a, b| b.id.cmp(&a.id));
+        assert_eq!(list, expect);
+
+        engine.delete(&a_id).expect("Successful delete");
+        assert!(engine.get(&a_id)
+            .expect("get after delete should still work")
+            .is_none());
+
+        let list = engine.list().expect("Grabbing list to work");
+        assert_eq!(list.len(), 1);
+        assert_eq!(list[0], b_from_db);
+
+        let now_us = util::system_time_ms_i64(SystemTime::now());
+        let b2 = Login { password: "newpass".into(), id: b_id.clone(), .. b.clone() };
+
+        engine.update(b2.clone()).expect("update b should work");
+
+        let b_after_update = engine.get(&b_id)
+            .expect("Not to error getting b")
+            .expect("b to exist");
+
+        assert_logins_equiv(&b_after_update, &b2);
+        assert_ge!(b_after_update.time_created, start_us);
+        assert_le!(b_after_update.time_created, now_us);
+        assert_ge!(b_after_update.time_password_changed, now_us);
+        assert_ge!(b_after_update.time_last_used, now_us);
+        // Should be two even though we updated twice
+        assert_eq!(b_after_update.times_used, 2);
+    }
+}

logins-sql/src/error.rs

@@ -0,0 +1,132 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+use failure::{Fail, Context, Backtrace};
+use std::{self, fmt};
+use std::boxed::Box;
+use rusqlite;
+use serde_json;
+use sync;
+use url;
+
+pub type Result<T> = std::result::Result<T, Error>;
+
+// Backported part of the (someday real) failure 1.x API, basically equivalent
+// to error_chain's `bail!` (We don't call it that because `failure` has a
+// `bail` macro with different semantics)
+macro_rules! throw {
+    ($e:expr) => {
+        return Err(::std::convert::Into::into($e));
+    }
+}
+
+#[derive(Debug)]
+pub struct Error(Box<Context<ErrorKind>>);
+
+impl Fail for Error {
+    #[inline]
+    fn cause(&self) -> Option<&Fail> {
+        self.0.cause()
+    }
+
+    #[inline]
+    fn backtrace(&self) -> Option<&Backtrace> {
+        self.0.backtrace()
+    }
+}
+
+impl fmt::Display for Error {
+    #[inline]
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        fmt::Display::fmt(&*self.0, f)
+    }
+}
+
+impl Error {
+    #[inline]
+    pub fn kind(&self) -> &ErrorKind {
+        &*self.0.get_context()
+    }
+}
+
+impl From<ErrorKind> for Error {
+    #[inline]
+    fn from(kind: ErrorKind) -> Error {
+        Error(Box::new(Context::new(kind)))
+    }
+}
+
+impl From<Context<ErrorKind>> for Error {
+    #[inline]
+    fn from(inner: Context<ErrorKind>) -> Error {
+        Error(Box::new(inner))
+    }
+}
+
+
+#[derive(Debug, Fail)]
+pub enum ErrorKind {
+    #[fail(display = "Invalid login: {}", _0)]
+    InvalidLogin(InvalidLogin),
+
+    #[fail(display = "The `sync_status` column in DB has an illegal value: {}", _0)]
+    BadSyncStatus(u8),
+
+    #[fail(display = "A duplicate GUID is present: {:?}", _0)]
+    DuplicateGuid(String),
+
+    #[fail(display = "No record with guid exists (when one was required): {:?}", _0)]
+    NoSuchRecord(String),
+
+    #[fail(display = "Error synchronizing: {}", _0)]
+    SyncAdapterError(#[fail(cause)] sync::Error),
+
+    #[fail(display = "Error parsing JSON data: {}", _0)]
+    JsonError(#[fail(cause)] serde_json::Error),
+
+    #[fail(display = "Error executing SQL: {}", _0)]
+    SqlError(#[fail(cause)] rusqlite::Error),
+
+    #[fail(display = "Error parsing URL: {}", _0)]
+    UrlParseError(#[fail(cause)] url::ParseError),
+}
+
+macro_rules! impl_from_error {
+    ($(($variant:ident, $type:ty)),+) => ($(
+        impl From<$type> for ErrorKind {
+            #[inline]
+            fn from(e: $type) -> ErrorKind {
+                ErrorKind::$variant(e)
+            }
+        }
+
+        impl From<$type> for Error {
+            #[inline]
+            fn from(e: $type) -> Error {
+                ErrorKind::from(e).into()
+            }
+        }
+    )*);
+}
+
+impl_from_error! {
+    (SyncAdapterError, sync::Error),
+    (JsonError, serde_json::Error),
+    (UrlParseError, url::ParseError),
+    (SqlError, rusqlite::Error),
+    (InvalidLogin, InvalidLogin)
+}
+
+#[derive(Debug, Fail)]
+pub enum InvalidLogin {
+    #[fail(display = "Hostname is empty")]
+    EmptyHostname,
+    #[fail(display = "Password is empty")]
+    EmptyPassword,
+    #[fail(display = "Both `formSubmitUrl` and `httpRealm` are present")]
+    BothTargets,
+    #[fail(display = "Neither `formSubmitUrl` and `httpRealm` are present")]
+    NoTarget,
+}
+

logins-sql/src/lib.rs

@@ -0,0 +1,50 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+extern crate sync15_adapter as sync;
+
+#[macro_use]
+extern crate log;
+
+#[cfg(test)]
+extern crate env_logger;
+
+#[macro_use]
+extern crate lazy_static;
+
+extern crate failure;
+
+#[macro_use]
+extern crate failure_derive;
+
+#[cfg(test)]
+#[macro_use]
+extern crate more_asserts;
+
+extern crate url;
+
+extern crate rusqlite;
+
+extern crate serde;
+extern crate serde_json;
+
+#[macro_use]
+extern crate serde_derive;
+
+#[macro_use]
+mod error;
+mod login;
+
+pub mod schema;
+mod util;
+mod db;
+mod engine;
+mod update_plan;
+
+pub use error::*;
+pub use login::*;
+pub use engine::*;
+
+
+

logins-sql/src/login.rs

@@ -0,0 +1,431 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+use sync::{self, ServerTimestamp};
+use rusqlite::Row;
+use util;
+use std::time::{self, SystemTime};
+use error::*;
+
+#[derive(Debug, Clone, Hash, PartialEq, Serialize, Deserialize, Default)]
+#[serde(rename_all = "camelCase")]
+pub struct Login {
+    // TODO: consider `#[serde(rename = "id")] pub guid: String` to avoid confusion
+    pub id: String,
+
+    pub hostname: String,
+
+    // rename_all = "camelCase" by default will do formSubmitUrl, but we can just
+    // override this one field.
+    #[serde(rename = "formSubmitURL")]
+    pub form_submit_url: Option<String>,
+
+    pub http_realm: Option<String>,
+
+    #[serde(default)]
+    pub username: String,
+
+    pub password: String,
+
+    #[serde(default)]
+    pub username_field: String,
+
+    #[serde(default)]
+    pub password_field: String,
+
+    #[serde(default)]
+    pub time_created: i64,
+
+    #[serde(default)]
+    pub time_password_changed: i64,
+
+    #[serde(default)]
+    pub time_last_used: i64,
+
+    #[serde(default)]
+    pub times_used: i64,
+}
+
+fn string_or_default(row: &Row, col: &str) -> Result<String> {
+    Ok(row.get_checked::<_, Option<String>>(col)?.unwrap_or_default())
+}
+
+impl Login {
+    #[inline]
+    pub fn guid(&self) -> &String {
+        &self.id
+    }
+
+    #[inline]
+    pub fn guid_str(&self) -> &str {
+        self.id.as_str()
+    }
+
+    pub fn check_valid(&self) -> Result<()> {
+        if self.hostname.is_empty() {
+            throw!(InvalidLogin::EmptyHostname);
+        }
+
+        if self.password.is_empty() {
+            throw!(InvalidLogin::EmptyPassword);
+        }
+
+        if self.form_submit_url.is_some() && self.http_realm.is_some() {
+            throw!(InvalidLogin::BothTargets);
+        }
+
+        if self.form_submit_url.is_none() && self.http_realm.is_none() {
+            throw!(InvalidLogin::NoTarget);
+        }
+        Ok(())
+    }
+
+    pub(crate) fn from_row(row: &Row) -> Result<Login> {
+        Ok(Login {
+            id: row.get_checked("guid")?,
+            password: row.get_checked("password")?,
+            username: string_or_default(row, "username")?,
+
+            hostname:   row.get_checked("hostname")?,
+            http_realm: row.get_checked("httpRealm")?,
+
+            form_submit_url: row.get_checked("formSubmitURL")?,
+
+            username_field: string_or_default(row, "usernameField")?,
+            password_field: string_or_default(row, "passwordField")?,
+
+            time_created:   row.get_checked("timeCreated")?,
+            // Might be null
+            time_last_used: row.get_checked::<_, Option<i64>>("timeLastUsed")?.unwrap_or_default(),
+
+            time_password_changed: row.get_checked("timePasswordChanged")?,
+            times_used:            row.get_checked("timesUsed")?,
+        })
+    }
+}
+
+#[derive(Clone, Debug)]
+pub(crate) struct MirrorLogin {
+    pub login: Login,
+    pub is_overridden: bool,
+    pub server_modified: ServerTimestamp,
+}
+
+impl MirrorLogin {
+    #[inline]
+    pub fn guid_str(&self) -> &str {
+        self.login.guid_str()
+    }
+
+    pub(crate) fn from_row(row: &Row) -> Result<MirrorLogin> {
+        Ok(MirrorLogin {
+            login: Login::from_row(row)?,
+            is_overridden: row.get_checked("is_overridden")?,
+            server_modified: ServerTimestamp(
+                row.get_checked::<_, i64>("server_modified")? as f64 / 1000.0)
+        })
+    }
+}
+
+// This doesn't really belong here.
+#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash)]
+#[repr(u8)]
+pub(crate) enum SyncStatus {
+    Synced = 0,
+    Changed = 1,
+    New = 2,
+}
+
+impl SyncStatus {
+    #[inline]
+    pub fn from_u8(v: u8) -> Result<Self> {
+        match v {
+            0 => Ok(SyncStatus::Synced),
+            1 => Ok(SyncStatus::Changed),
+            2 => Ok(SyncStatus::New),
+            v => throw!(ErrorKind::BadSyncStatus(v)),
+        }
+    }
+}
+
+#[derive(Clone, Debug)]
+pub(crate) struct LocalLogin {
+    pub login: Login,
+    pub sync_status: SyncStatus,
+    pub is_deleted: bool,
+    pub local_modified: SystemTime,
+}
+
+impl LocalLogin {
+    #[inline]
+    pub fn guid_str(&self) -> &str {
+        self.login.guid_str()
+    }
+
+    pub(crate) fn from_row(row: &Row) -> Result<LocalLogin> {
+        Ok(LocalLogin {
+            login: Login::from_row(row)?,
+            sync_status: SyncStatus::from_u8(row.get_checked("sync_status")?)?,
+            is_deleted: row.get_checked("is_deleted")?,
+            local_modified: util::system_time_millis_from_row(row, "local_modified")?
+        })
+    }
+}
+
+macro_rules! impl_login {
+    ($ty:ty { $($fields:tt)* }) => {
+        impl AsRef<Login> for $ty {
+            #[inline]
+            fn as_ref(&self) -> &Login {
+                &self.login
+            }
+        }
+
+        impl AsMut<Login> for $ty {
+            #[inline]
+            fn as_mut(&mut self) -> &mut Login {
+                &mut self.login
+            }
+        }
+
+        impl From<$ty> for Login {
+            #[inline]
+            fn from(l: $ty) -> Self {
+                l.login
+            }
+        }
+
+        impl From<Login> for $ty {
+            #[inline]
+            fn from(login: Login) -> Self {
+                Self { login, $($fields)* }
+            }
+        }
+    };
+}
+
+impl_login!(LocalLogin {
+    sync_status: SyncStatus::New,
+    is_deleted: false,
+    local_modified: time::UNIX_EPOCH
+});
+
+impl_login!(MirrorLogin {
+    is_overridden: false,
+    server_modified: ServerTimestamp(0.0)
+});
+
+// Stores data needed to do a 3-way merge
+pub(crate) struct SyncLoginData {
+    pub guid: String,
+    pub local: Option<LocalLogin>,
+    pub mirror: Option<MirrorLogin>,
+    // None means it's a deletion
+    pub inbound: (Option<Login>, ServerTimestamp),
+}
+
+impl SyncLoginData {
+    #[inline]
+    pub fn guid_str(&self) -> &str {
+        &self.guid[..]
+    }
+
+    #[inline]
+    pub fn guid(&self) -> &String {
+        &self.guid
+    }
+
+    #[inline]
+    pub fn from_payload(payload: sync::Payload, ts: ServerTimestamp) -> Result<Self> {
+        let guid = payload.id.clone();
+        let login: Option<Login> =
+            if payload.is_tombstone() {
+                None
+            } else {
+                let record: Login = payload.into_record()?;
+                Some(record)
+            };
+        Ok(Self { guid, local: None, mirror: None, inbound: (login, ts) })
+    }
+}
+
+macro_rules! impl_login_setter {
+    ($setter_name:ident, $field:ident, $Login:ty) => {
+        impl SyncLoginData {
+            pub(crate) fn $setter_name (&mut self, record: $Login) -> Result<()> {
+                // TODO: We probably shouldn't panic in this function!
+                if self.$field.is_some() {
+                    // Shouldn't be possible (only could happen if UNIQUE fails in sqlite, or if we
+                    // get duplicate guids somewhere,but we check).
+                    panic!("SyncLoginData::{} called on object that already has {} data",
+                           stringify!($setter_name),
+                           stringify!($field));
+                }
+
+                if self.guid_str() != record.guid_str() {
+                    // This is almost certainly a bug in our code.
+                    panic!("Wrong guid on login in {}: {:?} != {:?}",
+                           stringify!($setter_name),
+                           self.guid_str(), record.guid_str());
+                }
+
+                self.$field = Some(record);
+                Ok(())
+            }
+        }
+    };
+}
+
+impl_login_setter!(set_local, local, LocalLogin);
+impl_login_setter!(set_mirror, mirror, MirrorLogin);
+
+#[derive(Debug, Default, Clone)]
+pub(crate) struct LoginDelta {
+
+    // "non-commutative" fields
+    pub hostname: Option<String>,
+    pub password: Option<String>,
+    pub username: Option<String>,
+    pub http_realm: Option<String>,
+    pub form_submit_url: Option<String>,
+
+    pub time_created: Option<i64>,
+    pub time_last_used: Option<i64>,
+    pub time_password_changed: Option<i64>,
+
+    // "non-conflicting" fields (which are the same)
+    pub password_field: Option<String>,
+    pub username_field: Option<String>,
+
+    // Commutative field
+    pub times_used: i64,
+}
+
+macro_rules! merge_field {
+    ($merged:ident, $b:ident, $prefer_b:expr, $field:ident) => {
+        if let Some($field) = $b.$field.take() {
+            if $merged.$field.is_some() {
+                warn!("Collision merging login field {}", stringify!($field));
+                if $prefer_b {
+                    $merged.$field = Some($field);
+                }
+            } else {
+                $merged.$field = Some($field);
+            }
+        }
+    };
+}
+
+impl LoginDelta {
+    pub fn merge(self, mut b: LoginDelta, b_is_newer: bool) -> LoginDelta {
+        let mut merged = self;
+        merge_field!(merged, b, b_is_newer, hostname);
+        merge_field!(merged, b, b_is_newer, password);
+        merge_field!(merged, b, b_is_newer, username);
+        merge_field!(merged, b, b_is_newer, http_realm);
+        merge_field!(merged, b, b_is_newer, form_submit_url);
+
+        merge_field!(merged, b, b_is_newer, time_created);
+        merge_field!(merged, b, b_is_newer, time_last_used);
+        merge_field!(merged, b, b_is_newer, time_password_changed);
+
+        merge_field!(merged, b, b_is_newer, password_field);
+        merge_field!(merged, b, b_is_newer, username_field);
+
+        // commutative fields
+        merged.times_used += b.times_used;
+
+        merged
+    }
+}
+
+macro_rules! apply_field {
+    ($login:ident, $delta:ident, $field:ident) => {
+        if let Some($field) = $delta.$field.take() {
+            $login.$field = $field.into();
+        }
+    };
+}
+
+impl Login {
+
+    pub(crate) fn apply_delta(&mut self, mut delta: LoginDelta) {
+
+        apply_field!(self, delta, hostname);
+
+        apply_field!(self, delta, password);
+        apply_field!(self, delta, username);
+
+        apply_field!(self, delta, time_created);
+        apply_field!(self, delta, time_last_used);
+        apply_field!(self, delta, time_password_changed);
+
+        apply_field!(self, delta, password_field);
+        apply_field!(self, delta, username_field);
+
+        // Use Some("") to indicate that it should be changed to be None (hacky...)
+        if let Some(realm) = delta.http_realm.take() {
+            self.http_realm = if realm.is_empty() { None } else { Some(realm) };
+        }
+
+        if let Some(url) = delta.form_submit_url.take() {
+            self.form_submit_url = if url.is_empty() { None } else { Some(url) };
+        }
+
+        self.times_used += delta.times_used;
+    }
+
+    pub(crate) fn delta(&self, older: &Login) -> LoginDelta {
+        let mut delta = LoginDelta::default();
+
+        if self.form_submit_url != older.form_submit_url {
+            delta.form_submit_url = Some(self.form_submit_url.clone().unwrap_or_default());
+        }
+
+        if self.http_realm != older.http_realm {
+            delta.http_realm = Some(self.http_realm.clone().unwrap_or_default());
+        }
+
+        if self.hostname != older.hostname {
+            delta.hostname = Some(self.hostname.clone());
+        }
+        if self.username != older.username {
+            delta.username = Some(self.username.clone());
+        }
+        if self.password != older.password {
+            delta.password = Some(self.password.clone());
+        }
+        if self.password_field != older.password_field {
+            delta.password_field = Some(self.password_field.clone());
+        }
+        if self.username_field != older.username_field {
+            delta.username_field = Some(self.username_field.clone());
+        }
+
+        // We discard zero (and negative numbers) for timestamps so that a
+        // record that doesn't contain this information (these are
+        // `#[serde(default)]`) doesn't skew our records.
+        //
+        // Arguably, we should also also ignore values later than our
+        // `time_created`, or earlier than our `time_last_used` or
+        // `time_password_changed`. Doing this properly would probably require
+        // a scheme analogous to Desktop's weak-reupload system, so I'm punting
+        // on it for now.
+        if self.time_created > 0 && self.time_created != older.time_created {
+            delta.time_created = Some(self.time_created);
+        }
+        if self.time_last_used > 0 && self.time_last_used != older.time_last_used {
+            delta.time_last_used = Some(self.time_last_used);
+        }
+        if self.time_password_changed > 0 && self.time_password_changed != older.time_password_changed {
+            delta.time_password_changed = Some(self.time_password_changed);
+        }
+
+        if self.times_used > 0 && self.times_used != older.times_used {
+            delta.times_used = self.times_used - older.times_used;
+        }
+
+        delta
+    }
+}

logins-sql/src/schema.rs

@@ -0,0 +1,304 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+//! Logins Schema v4
+//! ================
+//!
+//! The schema we use is a evolution of the firefox-ios logins database format.
+//! There are three tables:
+//!
+//! - `loginsL`: The local table.
+//! - `loginsM`: The mirror table.
+//! - `loginsSyncMeta`: The table used to to store various sync metadata.
+//!
+//! ## `loginsL`
+//!
+//! This stores local login information, also known as the "overlay".
+//!
+//! `loginsL` is essentially unchanged from firefox-ios, however note the
+//! semantic change v4 makes to timestamp fields (which is explained in more
+//! detail in the [COMMON_COLS] documentation).
+//!
+//! It is important to note that `loginsL` is not guaranteed to be present for
+//! all records. Synced records may only exist in `loginsM` (although this is
+//! not guaranteed). In either case, queries should read from both `loginsL` and
+//! `loginsM`.
+//!
+//! ### `loginsL` Columns
+//!
+//! Contains all fields in [COMMON_COLS], as well as the following additional
+//! columns:
+//!
+//! - `local_modified`: A millisecond local timestamp indicating when the record
+//!   was changed locally, or NULL if the record has never been changed locally.
+//!
+//! - `is_deleted`: A boolean indicating whether or not this record is a
+//!   tombstone.
+//!
+//! - `sync_status`: A `SyncStatus` enum value, one of
+//!
+//!     - `0` (`SyncStatus::Synced`): Indicating that the record has been synced
+//!
+//!     - `1` (`SyncStatus::Changed`): Indicating that the record should be
+//!       has changed locally and is known to exist on the server.
+//!
+//!     - `2` (`SyncStatus::New`): Indicating that the record has never been
+//!       synced, or we have been reset since the last time it synced.
+//!
+//! ## `loginsM`
+//!
+//! This stores server-side login information, also known as the "mirror".
+//!
+//! Like `loginsL`, `loginM` has not changed from firefox-ios, beyond the
+//! change to store timestamps as milliseconds explained in [COMMON_COLS].
+//!
+//! Also like `loginsL`, `loginsM` is not guaranteed to have rows for all
+//! records. It should not have rows for records which were not synced!
+//!
+//! It is important to note that `loginsL` is not guaranteed to be present for
+//! all records. Synced records may only exist in `loginsM`! Queries should
+//! test against both!
+//!
+//! ### `loginsM` Columns
+//!
+//! Contains all fields in [COMMON_COLS], as well as the following additional
+//! columns:
+//!
+//! - `server_modified`: the most recent server-modification timestamp
+//!   ([sync15_adapter::ServerTimestamp]) we've seen for this record. Stored as
+//!   a millisecond value.
+//!
+//! - `is_overridden`: A boolean indicating whether or not the mirror contents
+//!   are invalid, and that we should defer to the data stored in `loginsL`.
+//!
+//! ## `loginsSyncMeta`
+//!
+//! This is a simple key-value table based on the `moz_meta` table in places.
+//! This table was added (by this rust crate) in version 4, and so is not
+//! present in firefox-ios.
+//!
+//! Currently it is used to store two items:
+//!
+//! 1. The last sync timestamp is stored under [LAST_SYNC_META_KEY], a
+//!    `sync15_adapter::ServerTimestamp` stored in integer milliseconds.
+//!
+//! 2. The persisted sync state machine information is stored under
+//!    [GLOBAL_STATE_META_KEY]. This is a `sync15_adapter::GlobalState` stored as
+//!    JSON.
+//!
+
+use error::*;
+use db;
+
+/// Note that firefox-ios is currently on version 3. Version 4 is this version,
+/// which adds a metadata table and changes timestamps to be in milliseconds
+pub const VERSION: i64 = 4;
+
+/// Every column shared by both tables except for `id`
+///
+/// Note: `timeCreated`, `timeLastUsed`, and `timePasswordChanged` are in
+/// milliseconds. This is in line with how the server and Desktop handle it, but
+/// counter to how firefox-ios handles it (hence needing to fix them up
+/// firefox-ios on schema upgrade from 3, the last firefox-ios password schema
+/// version).
+///
+/// The reason for breaking from how firefox-ios does things is just because it
+/// complicates the code to have multiple kinds of timestamps, for very little
+/// benefit. It also makes it unclear what's stored on the server, leading to
+/// further confusion.
+///
+/// However, note that the `local_modified` (of `loginsL`) and `server_modified`
+/// (of `loginsM`) are stored as milliseconds as well both on firefox-ios and
+/// here (and so they do not need to be updated with the `timeLastUsed`/
+/// `timePasswordChanged`/`timeCreated` timestamps.
+pub const COMMON_COLS: &'static str = "
+    guid,
+    username,
+    password,
+    hostname,
+    httpRealm,
+    formSubmitURL,
+    usernameField,
+    passwordField,
+    timeCreated,
+    timeLastUsed,
+    timePasswordChanged,
+    timesUsed
+";
+
+
+const COMMON_SQL: &'static str = "
+    id                  INTEGER PRIMARY KEY AUTOINCREMENT,
+    hostname            TEXT NOT NULL,
+    -- Exactly one of httpRealm or formSubmitURL should be set
+    httpRealm           TEXT,
+    formSubmitURL       TEXT,
+    usernameField       TEXT,
+    passwordField       TEXT,
+    timesUsed           INTEGER NOT NULL DEFAULT 0,
+    timeCreated         INTEGER NOT NULL,
+    timeLastUsed        INTEGER,
+    timePasswordChanged INTEGER NOT NULL,
+    username            TEXT,
+    password            TEXT NOT NULL,
+    guid                TEXT NOT NULL UNIQUE
+";
+
+lazy_static! {
+    static ref CREATE_LOCAL_TABLE_SQL: String = format!(
+        "CREATE TABLE IF NOT EXISTS loginsL (
+            {common_sql},
+            -- Milliseconds, or NULL if never modified locally.
+            local_modified INTEGER,
+
+            is_deleted     TINYINT NOT NULL DEFAULT 0,
+            sync_status    TINYINT NOT NULL DEFAULT 0
+        )",
+        common_sql = COMMON_SQL
+    );
+
+    static ref CREATE_MIRROR_TABLE_SQL: String = format!(
+        "CREATE TABLE IF NOT EXISTS loginsM (
+            {common_sql},
+            -- Milliseconds (a sync15_adapter::ServerTimestamp multiplied by
+            -- 1000 and truncated)
+            server_modified INTEGER NOT NULL,
+            is_overridden   TINYINT NOT NULL DEFAULT 0
+        )",
+        common_sql = COMMON_SQL
+    );
+
+    static ref SET_VERSION_SQL: String = format!(
+        "PRAGMA user_version = {version}",
+        version = VERSION
+    );
+}
+
+const CREATE_META_TABLE_SQL: &'static str = "
+    CREATE TABLE IF NOT EXISTS loginsSyncMeta (
+        key TEXT PRIMARY KEY,
+        value NOT NULL
+    )
+";
+
+const CREATE_OVERRIDE_HOSTNAME_INDEX_SQL: &'static str = "
+    CREATE INDEX IF NOT EXISTS idx_loginsM_is_overridden_hostname
+    ON loginsM (is_overridden, hostname)
+";
+
+const CREATE_DELETED_HOSTNAME_INDEX_SQL: &'static str = "
+    CREATE INDEX IF NOT EXISTS idx_loginsL_is_deleted_hostname
+    ON loginsL (is_deleted, hostname)
+";
+
+// As noted above, we use these when updating from schema v3 (firefox-ios's
+// last schema) to convert from microsecond timestamps to milliseconds.
+const UPDATE_LOCAL_TIMESTAMPS_TO_MILLIS_SQL: &'static str = "
+    UPDATE loginsL
+    SET timeCreated = timeCreated / 1000,
+        timeLastUsed = timeLastUsed / 1000,
+        timePasswordChanged = timePasswordChanged / 1000
+";
+
+const UPDATE_MIRROR_TIMESTAMPS_TO_MILLIS_SQL: &'static str = "
+    UPDATE loginsM
+    SET timeCreated = timeCreated / 1000,
+        timeLastUsed = timeLastUsed / 1000,
+        timePasswordChanged = timePasswordChanged / 1000
+";
+
+pub(crate) static LAST_SYNC_META_KEY:    &'static str = "last_sync_time";
+pub(crate) static GLOBAL_STATE_META_KEY: &'static str = "global_state";
+
+pub(crate) fn init(db: &db::LoginDb) -> Result<()> {
+    let user_version = db.query_one::<i64>("PRAGMA user_version")?;
+    if user_version == 0 {
+        // This logic is largely taken from firefox-ios. AFAICT at some point
+        // they went from having schema versions tracked using a table named
+        // `tableList` to using `PRAGMA user_version`. This leads to the
+        // following logic:
+        //
+        // - If `tableList` exists, we're hopelessly far in the past, drop any
+        //   tables we have (to ensure we avoid name collisions/stale data) and
+        //   recreate. (This is captured by the `upgrade` case where from == 0)
+        //
+        // - If `tableList` doesn't exist and `PRAGMA user_version` is 0, it's
+        //   the first time through, just create the new tables.
+        //
+        // - Otherwise, it's a normal schema upgrade from an earlier
+        //   `PRAGMA user_version`.
+        let table_list_exists = db.query_one::<i64>(
+            "SELECT count(*) FROM sqlite_master WHERE type = 'table' AND name = 'tableList'"
+        )? != 0;
+
+        if table_list_exists {
+            drop(db)?;
+        }
+        return create(db);
+    }
+    if user_version != VERSION {
+        if user_version < VERSION {
+            upgrade(db, user_version)?;
+        } else {
+            warn!("Loaded future schema version {} (we only understand version {}). \
+                   Optimisitically ",
+                  user_version, VERSION)
+        }
+    }
+    Ok(())
+}
+
+// https://github.com/mozilla-mobile/firefox-ios/blob/master/Storage/SQL/LoginsSchema.swift#L100
+fn upgrade(db: &db::LoginDb, from: i64) -> Result<()> {
+    debug!("Upgrading schema from {} to {}", from, VERSION);
+    if from == VERSION {
+        return Ok(());
+    }
+    assert_ne!(from, 0,
+        "Upgrading from user_version = 0 should already be handled (in `init`)");
+    if from < 3 {
+        // These indices were added in v3 (apparently)
+        db.execute_all(&[
+            CREATE_OVERRIDE_HOSTNAME_INDEX_SQL,
+            CREATE_DELETED_HOSTNAME_INDEX_SQL,
+        ])?;
+    }
+    if from < 4 {
+        // This is the update from the firefox-ios schema to our schema.
+        // The `loginsSyncMeta` table was added in v4, and we moved
+        // from using microseconds to milliseconds for `timeCreated`,
+        // `timeLastUsed`, and `timePasswordChanged`.
+        db.execute_all(&[
+            CREATE_META_TABLE_SQL,
+            UPDATE_LOCAL_TIMESTAMPS_TO_MILLIS_SQL,
+            UPDATE_MIRROR_TIMESTAMPS_TO_MILLIS_SQL,
+            &*SET_VERSION_SQL,
+        ])?;
+    }
+    Ok(())
+}
+
+pub(crate) fn create(db: &db::LoginDb) -> Result<()> {
+    debug!("Creating schema");
+    db.execute_all(&[
+        &*CREATE_LOCAL_TABLE_SQL,
+        &*CREATE_MIRROR_TABLE_SQL,
+        CREATE_OVERRIDE_HOSTNAME_INDEX_SQL,
+        CREATE_DELETED_HOSTNAME_INDEX_SQL,
+        CREATE_META_TABLE_SQL,
+        &*SET_VERSION_SQL,
+    ])?;
+    Ok(())
+}
+
+pub(crate) fn drop(db: &db::LoginDb) -> Result<()> {
+    debug!("Dropping schema");
+    db.execute_all(&[
+        "DROP TABLE IF EXISTS loginsM",
+        "DROP TABLE IF EXISTS loginsL",
+        "DROP TABLE IF EXISTS loginsSyncMeta",
+        "PRAGMA user_version = 0",
+    ])?;
+    Ok(())
+}

logins-sql/src/update_plan.rs

@@ -0,0 +1,249 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+use rusqlite::{types::ToSql, Transaction};
+use std::time::SystemTime;
+use error::*;
+use login::{LocalLogin, MirrorLogin, Login, SyncStatus};
+use sync::ServerTimestamp;
+use util;
+
+#[derive(Default, Debug, Clone)]
+pub(crate) struct UpdatePlan {
+    pub delete_mirror: Vec<String>,
+    pub delete_local: Vec<String>,
+    pub local_updates: Vec<MirrorLogin>,
+    // the bool is the `is_overridden` flag, the i64 is ServerTimestamp in millis
+    pub mirror_inserts: Vec<(Login, i64, bool)>,
+    pub mirror_updates: Vec<(Login, i64)>,
+}
+
+impl UpdatePlan {
+    pub fn plan_two_way_merge(&mut self, local: &Login, upstream: (Login, ServerTimestamp)) {
+        let is_override = local.time_password_changed > upstream.0.time_password_changed;
+        self.mirror_inserts.push((upstream.0, upstream.1.as_millis() as i64, is_override));
+        if !is_override {
+            self.delete_local.push(local.id.to_string());
+        }
+    }
+
+    pub fn plan_three_way_merge(
+        &mut self,
+        local: LocalLogin,
+        shared: MirrorLogin,
+        upstream: Login,
+        upstream_time: ServerTimestamp,
+        server_now: ServerTimestamp
+    ) {
+        let local_age = SystemTime::now().duration_since(local.local_modified).unwrap_or_default();
+        let remote_age = server_now.duration_since(upstream_time).unwrap_or_default();
+
+        let local_delta = local.login.delta(&shared.login);
+        let upstream_delta = upstream.delta(&shared.login);
+
+        let merged_delta = local_delta.merge(upstream_delta, remote_age < local_age);
+
+        // Update mirror to upstream
+        self.mirror_updates.push((upstream, upstream_time.as_millis() as i64));
+        let mut new = shared;
+
+        new.login.apply_delta(merged_delta);
+        new.server_modified = upstream_time;
+        self.local_updates.push(new);
+    }
+
+    pub fn plan_delete(&mut self, id: String) {
+        self.delete_local.push(id.to_string());
+        self.delete_mirror.push(id.to_string());
+    }
+
+    pub fn plan_mirror_update(&mut self, login: Login, time: ServerTimestamp) {
+        self.mirror_updates.push((login, time.as_millis() as i64));
+    }
+
+    pub fn plan_mirror_insert(&mut self, login: Login, time: ServerTimestamp, is_override: bool) {
+        self.mirror_inserts.push((login, time.as_millis() as i64, is_override));
+    }
+
+    fn perform_deletes(&self, tx: &mut Transaction, max_var_count: usize) -> Result<()> {
+        util::each_chunk(&self.delete_local, max_var_count, |chunk, _| {
+            tx.execute(&format!("DELETE FROM loginsL WHERE guid IN ({vars})",
+                                vars = util::sql_vars(chunk.len())),
+                       chunk)?;
+            Ok(())
+        })?;
+
+        util::each_chunk(&self.delete_mirror, max_var_count, |chunk, _| {
+            tx.execute(&format!("DELETE FROM loginsM WHERE guid IN ({vars})",
+                                vars = util::sql_vars(chunk.len())),
+                       chunk)?;
+            Ok(())
+        })?;
+        Ok(())
+    }
+
+    // These aren't batched but probably should be.
+    fn perform_mirror_updates(&self, tx: &mut Transaction) -> Result<()> {
+        let sql = "
+            UPDATE loginsM
+            SET server_modified = :server_modified,
+                httpRealm       = :http_realm,
+                formSubmitURL   = :form_submit_url,
+                usernameField   = :username_field,
+                passwordField   = :password_field,
+                password        = :password,
+                hostname        = :hostname,
+                username        = :username,
+                -- Avoid zeroes if the remote has been overwritten by an older client.
+                timesUsed           = coalesce(nullif(:times_used,            0), timesUsed),
+                timeLastUsed        = coalesce(nullif(:time_last_used,        0), timeLastUsed),
+                timePasswordChanged = coalesce(nullif(:time_password_changed, 0), timePasswordChanged),
+                timeCreated         = coalesce(nullif(:time_created,          0), timeCreated)
+            WHERE guid = :guid
+        ";
+        let mut stmt = tx.prepare_cached(sql)?;
+        for (login, timestamp) in &self.mirror_updates {
+            trace!("Updating mirror {:?}", login.guid_str());
+            stmt.execute_named(&[
+               (":server_modified", timestamp as &ToSql),
+               (":http_realm",      &login.http_realm as &ToSql),
+               (":form_submit_url", &login.form_submit_url as &ToSql),
+               (":username_field",  &login.username_field as &ToSql),
+               (":password_field",  &login.password_field as &ToSql),
+               (":password",        &login.password as &ToSql),
+               (":hostname",        &login.hostname as &ToSql),
+               (":username",        &login.username as &ToSql),
+
+               (":times_used",            &login.times_used as &ToSql),
+               (":time_last_used",        &login.time_last_used as &ToSql),
+               (":time_password_changed", &login.time_password_changed as &ToSql),
+               (":time_created",          &login.time_created as &ToSql),
+
+               (":guid", &login.guid_str() as &ToSql),
+            ])?;
+        }
+        Ok(())
+    }
+
+    fn perform_mirror_inserts(&self, tx: &mut Transaction) -> Result<()> {
+        let sql = "
+            INSERT OR IGNORE INTO loginsM (
+                is_overridden,
+                server_modified,
+
+                httpRealm,
+                formSubmitURL,
+                usernameField,
+                passwordField,
+                password,
+                hostname,
+                username,
+
+                timesUsed,
+                timeLastUsed,
+                timePasswordChanged,
+                timeCreated,
+
+                guid
+            ) VALUES (
+                :is_overridden,
+                :server_modified,
+
+                :http_realm,
+                :form_submit_url,
+                :username_field,
+                :password_field,
+                :password,
+                :hostname,
+                :username,
+
+                :times_used,
+                :time_last_used,
+                :time_password_changed,
+                :time_created,
+
+                :guid
+            )";
+        let mut stmt = tx.prepare_cached(&sql)?;
+
+        for (login, timestamp, is_overridden) in &self.mirror_inserts {
+            trace!("Inserting mirror {:?}", login.guid_str());
+            stmt.execute_named(&[
+                (":is_overridden", is_overridden as &ToSql),
+                (":server_modified", timestamp as &ToSql),
+
+                (":http_realm",      &login.http_realm as &ToSql),
+                (":form_submit_url", &login.form_submit_url as &ToSql),
+                (":username_field",  &login.username_field as &ToSql),
+                (":password_field",  &login.password_field as &ToSql),
+                (":password",        &login.password as &ToSql),
+                (":hostname",        &login.hostname as &ToSql),
+                (":username",        &login.username as &ToSql),
+
+                (":times_used",            &login.times_used as &ToSql),
+                (":time_last_used",        &login.time_last_used as &ToSql),
+                (":time_password_changed", &login.time_password_changed as &ToSql),
+                (":time_created",          &login.time_created as &ToSql),
+
+                (":guid", &login.guid_str() as &ToSql),
+            ])?;
+        }
+        Ok(())
+    }
+
+    fn perform_local_updates(&self, tx: &mut Transaction) -> Result<()> {
+        let sql = format!("
+            UPDATE loginsL
+            SET local_modified      = :local_modified,
+                httpRealm           = :http_realm,
+                formSubmitURL       = :form_submit_url,
+                usernameField       = :username_field,
+                passwordField       = :password_field,
+                timeLastUsed        = :time_last_used,
+                timePasswordChanged = :time_password_changed,
+                timesUsed           = :times_used,
+                password            = :password,
+                hostname            = :hostname,
+                username            = :username,
+                sync_status         = {changed}
+            WHERE guid = :guid",
+            changed = SyncStatus::Changed as u8);
+        let mut stmt = tx.prepare_cached(&sql)?;
+        // XXX OutgoingChangeset should no longer have timestamp.
+        let local_ms: i64 = util::system_time_ms_i64(SystemTime::now());
+        for l in &self.local_updates {
+            trace!("Updating local {:?}", l.guid_str());
+            stmt.execute_named(&[
+                (":local_modified", &local_ms as &ToSql),
+
+                (":http_realm",      &l.login.http_realm as &ToSql),
+                (":form_submit_url", &l.login.form_submit_url as &ToSql),
+                (":username_field",  &l.login.username_field as &ToSql),
+                (":password_field",  &l.login.password_field as &ToSql),
+                (":password",        &l.login.password as &ToSql),
+                (":hostname",        &l.login.hostname as &ToSql),
+                (":username",        &l.login.username as &ToSql),
+
+                (":time_last_used",        &l.login.time_last_used as &ToSql),
+                (":time_password_changed", &l.login.time_password_changed as &ToSql),
+                (":times_used",            &l.login.times_used as &ToSql),
+
+                (":guid", &l.guid_str() as &ToSql),
+            ])?;
+        }
+        Ok(())
+    }
+
+    pub fn execute(&self, tx: &mut Transaction, max_var_count: usize) -> Result<()> {
+        debug!("UpdatePlan: deleting records...");
+        self.perform_deletes(tx, max_var_count)?;
+        debug!("UpdatePlan: Updating existing mirror records...");
+        self.perform_mirror_updates(tx)?;
+        debug!("UpdatePlan: Inserting new mirror records...");
+        self.perform_mirror_inserts(tx)?;
+        debug!("UpdatePlan: Updating reconciled local records...");
+        self.perform_local_updates(tx)?;
+        Ok(())
+    }
+}

logins-sql/src/util.rs

@@ -0,0 +1,126 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+use error::*;
+use rusqlite::{types::ToSql, Row};
+use std::{fmt, time};
+use url::Url;
+
+// `mapped` basically just refers to the translating of `T` to `&dyn ToSql`
+// using the `to_sql` function. It's annoying that this is needed.
+pub fn each_chunk_mapped<'a, T: 'a>(
+    items: &'a [T],
+    chunk_size: usize,
+    to_sql: impl Fn(&'a T) -> &'a ToSql,
+    mut do_chunk: impl FnMut(&[&ToSql], usize) -> Result<()>
+) -> Result<()> {
+    if items.is_empty() {
+        return Ok(());
+    }
+    let mut vec = Vec::with_capacity(chunk_size.min(items.len()));
+    let mut offset = 0;
+    for chunk in items.chunks(chunk_size) {
+        vec.clear();
+        vec.extend(chunk.iter().map(|v| to_sql(v)));
+        do_chunk(&vec, offset)?;
+        offset += chunk.len();
+    }
+    Ok(())
+}
+
+pub fn each_chunk<'a, T: ToSql + 'a>(
+    items: &[T],
+    chunk_size: usize,
+    do_chunk: impl FnMut(&[&ToSql], usize) -> Result<()>
+) -> Result<()> {
+    each_chunk_mapped(items, chunk_size, |t| t as &ToSql, do_chunk)
+}
+
+#[derive(Debug, Clone)]
+pub struct RepeatDisplay<'a, F> {
+    count: usize,
+    sep: &'a str,
+    fmt_one: F
+}
+
+impl<'a, F> fmt::Display for RepeatDisplay<'a, F>
+where F: Fn(usize, &mut fmt::Formatter) -> fmt::Result {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        for i in 0..self.count {
+            if i != 0 {
+                f.write_str(self.sep)?;
+            }
+            (self.fmt_one)(i, f)?;
+        }
+        Ok(())
+    }
+}
+
+pub fn repeat_display<'a, F>(count: usize, sep: &'a str, fmt_one: F) -> RepeatDisplay<'a, F>
+where F: Fn(usize, &mut fmt::Formatter) -> fmt::Result {
+    RepeatDisplay { count, sep, fmt_one }
+}
+
+pub fn sql_vars(count: usize) -> impl fmt::Display {
+    repeat_display(count, ",", |_, f| write!(f, "?"))
+}
+
+pub fn url_host_port(url_str: &str) -> Option<String> {
+    let url = Url::parse(url_str).ok()?;
+    let host = url.host_str()?;
+    Some(if let Some(p) = url.port() {
+        format!("{}:{}", host, p)
+    } else {
+        host.to_string()
+    })
+}
+
+pub fn system_time_millis_from_row(row: &Row, col_name: &str) -> Result<time::SystemTime> {
+    let time_ms = row.get_checked::<_, Option<i64>>(col_name)?.unwrap_or_default() as u64;
+    Ok(time::UNIX_EPOCH + time::Duration::from_millis(time_ms))
+}
+
+pub fn duration_ms_i64(d: time::Duration) -> i64 {
+    (d.as_secs() as i64) * 1000 + ((d.subsec_nanos() as i64) / 1_000_000)
+}
+
+pub fn system_time_ms_i64(t: time::SystemTime) -> i64 {
+    duration_ms_i64(t.duration_since(time::UNIX_EPOCH).unwrap_or_default())
+}
+
+// Unfortunately, there's not a better way to turn on logging in tests AFAICT
+#[cfg(test)]
+pub(crate) fn init_test_logging() {
+    use env_logger;
+    use std::sync::{Once, ONCE_INIT};
+    static INIT_LOGGING: Once = ONCE_INIT;
+    INIT_LOGGING.call_once(|| {
+        env_logger::init_from_env(
+            env_logger::Env::default().filter_or("RUST_LOG", "trace")
+        );
+    });
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_vars() {
+        assert_eq!(format!("{}", sql_vars(1)), "?");
+        assert_eq!(format!("{}", sql_vars(2)), "?,?");
+        assert_eq!(format!("{}", sql_vars(3)), "?,?,?");
+    }
+
+    #[test]
+    fn test_repeat_disp() {
+        assert_eq!(format!("{}", repeat_display(1, ",", |i, f| write!(f, "({},?)", i))),
+                   "(0,?)");
+        assert_eq!(format!("{}", repeat_display(2, ",", |i, f| write!(f, "({},?)", i))),
+                   "(0,?),(1,?)");
+        assert_eq!(format!("{}", repeat_display(3, ",", |i, f| write!(f, "({},?)", i))),
+                   "(0,?),(1,?),(2,?)");
+    }
+}
+

sync15-adapter/src/client.rs

@@ -176,7 +176,7 @@ impl Sync15StorageClient {
                 resp.url().path()
             );
             return Err(ErrorKind::StorageHttpError {
-                code: resp.status(),
+                code: resp.status().as_u16(),
                 route: resp.url().path().into(),
             }.into());
         }

sync15-adapter/src/collection_keys.rs

@@ -9,7 +9,7 @@ use error::Result;
 use record_types::CryptoKeysRecord;
 use util::ServerTimestamp;
 
-#[derive(Clone, Debug, PartialEq)]
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
 pub struct CollectionKeys {
     pub timestamp: ServerTimestamp,
     pub default: KeyBundle,

sync15-adapter/src/error.rs

@@ -3,7 +3,7 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 use std::time::SystemTime;
-use reqwest::{self, StatusCode as HttpStatusCode};
+use reqwest;
 use failure::{self, Fail, Context, Backtrace, SyncFailure};
 use std::{fmt, result, string};
 use std::boxed::Box;
@@ -44,7 +44,7 @@ impl Error {
 
     pub fn is_not_found(&self) -> bool {
         match self.kind() {
-            ErrorKind::StorageHttpError { code: HttpStatusCode::NotFound, .. } => true,
+            ErrorKind::StorageHttpError { code: 404, .. } => true,
             _ => false
         }
     }
@@ -72,12 +72,11 @@ pub enum ErrorKind {
     #[fail(display = "SHA256 HMAC Mismatch error")]
     HmacMismatch,
 
-    // TODO: it would be nice if this were _0.to_u16(), but we cant have an expression there...
     #[fail(display = "HTTP status {} when requesting a token from the tokenserver", _0)]
-    TokenserverHttpError(HttpStatusCode),
+    TokenserverHttpError(u16),
 
     #[fail(display = "HTTP status {} during a storage request to \"{}\"", code, route)]
-    StorageHttpError { code: HttpStatusCode, route: String },
+    StorageHttpError { code: u16, route: String },
 
     #[fail(display = "Server requested backoff. Retry after {:?}", _0)]
     BackoffError(SystemTime),

sync15-adapter/src/key_bundle.rs

@@ -10,7 +10,7 @@ use openssl::hash::MessageDigest;
 use openssl::pkey::PKey;
 use openssl::sign::Signer;
 
-#[derive(Clone, PartialEq, Eq, Hash, Debug)]
+#[derive(Clone, PartialEq, Eq, Hash, Debug, Serialize, Deserialize)]
 pub struct KeyBundle {
     enc_key: Vec<u8>,
     mac_key: Vec<u8>,

sync15-adapter/src/request.rs

@@ -198,7 +198,7 @@ impl LimitTracker {
     }
 }
 
-#[derive(Deserialize, Debug, Clone)]
+#[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct InfoConfiguration {
     /// The maximum size in bytes of the overall HTTP request body that will be accepted by the
     /// server.
@@ -247,7 +247,7 @@ impl Default for InfoConfiguration {
     }
 }
 
-#[derive(Clone, Debug, Default, Deserialize)]
+#[derive(Clone, Debug, Default, Deserialize, Serialize)]
 pub struct InfoCollections(HashMap<String, ServerTimestamp>);
 
 impl InfoCollections {
@@ -363,7 +363,7 @@ impl PostResponseHandler for NormalResponseHandler {
                 return Err(ErrorKind::BatchInterrupted.into());
             } else {
                 return Err(ErrorKind::StorageHttpError {
-                    code: r.status,
+                    code: r.status.as_u16(),
                     route: "collection storage (TODO: record route somewhere)".into()
                 }.into());
             }
@@ -519,7 +519,7 @@ where
         let resp = resp_or_error?;
 
         if !resp.status.is_success() {
-            let code = resp.status;
+            let code = resp.status.as_u16();
             self.on_response.handle_response(resp, !want_commit)?;
             error!("Bug: expected OnResponse to have bailed out!");
             // Should we assert here instead?

sync15-adapter/src/state.rs

@@ -12,6 +12,7 @@ use key_bundle::KeyBundle;
 use record_types::{MetaGlobalEngine, MetaGlobalRecord};
 use request::{InfoCollections, InfoConfiguration};
 use util::{random_guid, ServerTimestamp, SERVER_EPOCH};
+use serde_json;
 
 use self::SetupState::*;
 
@@ -39,10 +40,16 @@ lazy_static! {
     static ref DEFAULT_DECLINED: Vec<&'static str> = vec![];
 }
 
+#[derive(Debug, Serialize, Deserialize)]
+#[serde(tag = "schema_version")]
+enum PersistedState {
+    V1(GlobalState),
+}
+
 /// Holds global Sync state, including server upload limits, and the
 /// last-fetched collection modified times, `meta/global` record, and
 /// collection encryption keys.
-#[derive(Debug, Default)]
+#[derive(Debug, Clone, Default, Serialize, Deserialize)]
 pub struct GlobalState {
     pub config: InfoConfiguration,
     pub collections: InfoCollections,
@@ -52,6 +59,18 @@ pub struct GlobalState {
 }
 
 impl GlobalState {
+    pub fn to_persistable_string(&self) -> String {
+        let state = PersistedState::V1(self.clone());
+        serde_json::to_string(&state)
+            .expect("Should only fail for recursive types (this is not recursive)")
+    }
+
+    pub fn from_persisted_string(data: &str) -> error::Result<Self> {
+        match serde_json::from_str(data)? {
+            PersistedState::V1(global_state) => Ok(global_state)
+        }
+    }
+
     pub fn key_for_collection(&self, collection: &str) -> error::Result<&KeyBundle> {
         Ok(self.keys
             .as_ref()
@@ -590,7 +609,7 @@ impl FetchAction {
 }
 
 /// Flags an engine for enablement or disablement.
-#[derive(Debug)]
+#[derive(Debug, Serialize, Deserialize, Clone)]
 pub enum EngineStateChange {
     ResetAll,
     ResetAllExcept(HashSet<String>),
@@ -602,7 +621,6 @@ pub enum EngineStateChange {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use reqwest;
 
     use bso_record::{BsoRecord, EncryptedBso, EncryptedPayload};
 
@@ -618,7 +636,7 @@ mod tests {
             match &self.info_configuration {
                 Ok(config) => Ok(config.clone()),
                 Err(_) => Err(ErrorKind::StorageHttpError {
-                    code: reqwest::StatusCode::InternalServerError,
+                    code: 500,
                     route: "info/configuration".to_string(),
                 }.into()),
             }
@@ -628,7 +646,7 @@ mod tests {
             match &self.info_collections {
                 Ok(collections) => Ok(collections.clone()),
                 Err(_) => Err(ErrorKind::StorageHttpError {
-                    code: reqwest::StatusCode::InternalServerError,
+                    code: 500,
                     route: "info/collections".to_string(),
                 }.into()),
             }
@@ -640,15 +658,15 @@ mod tests {
                 // TODO(lina): Special handling for 404s, we want to ensure we
                 // handle missing keys and other server errors correctly.
                 Err(_) => Err(ErrorKind::StorageHttpError {
-                    code: reqwest::StatusCode::InternalServerError,
+                    code: 500,
                     route: "meta/global".to_string(),
                 }.into()),
             }
         }
 
-        fn put_meta_global(&self, global: &BsoRecord<MetaGlobalRecord>) -> error::Result<()> {
+        fn put_meta_global(&self, _global: &BsoRecord<MetaGlobalRecord>) -> error::Result<()> {
             Err(ErrorKind::StorageHttpError {
-                code: reqwest::StatusCode::InternalServerError,
+                code: 500,
                 route: "meta/global".to_string(),
             }.into())
         }
@@ -658,15 +676,15 @@ mod tests {
                 Ok(keys) => Ok(keys.clone()),
                 // TODO(lina): Same as above, for 404s.
                 Err(_) => Err(ErrorKind::StorageHttpError {
-                    code: reqwest::StatusCode::InternalServerError,
+                    code: 500,
                     route: "crypto/keys".to_string(),
                 }.into()),
             }
         }
 
-        fn put_crypto_keys(&self, keys: &EncryptedBso) -> error::Result<()> {
+        fn put_crypto_keys(&self, _keys: &EncryptedBso) -> error::Result<()> {
             Err(ErrorKind::StorageHttpError {
-                code: reqwest::StatusCode::InternalServerError,
+                code: 500,
                 route: "crypto/keys".to_string(),
             }.into())
         }

sync15-adapter/src/sync.rs

@@ -46,9 +46,6 @@ where E: From<error::Error>
     info!("Downloaded {} remote changes", incoming_changes.changes.len());
     let mut outgoing = store.apply_incoming(incoming_changes)?;
 
-    assert_eq!(outgoing.timestamp, timestamp,
-        "last sync timestamp should never change unless we change it");
-
     outgoing.timestamp = last_changed_remote;
 
     info!("Uploading {} outgoing changes", outgoing.changes.len());

sync15-adapter/src/token.rs

@@ -82,7 +82,8 @@ impl TokenFetcher for TokenServerFetcher {
                 let when = self.now() + Duration::from_millis(ms);
                 return Err(ErrorKind::BackoffError(when).into());
             }
-            return Err(ErrorKind::TokenserverHttpError(resp.status()).into());
+            let status = resp.status().as_u16();
+            return Err(ErrorKind::TokenserverHttpError(status).into());
         }
 
         let token: TokenserverToken = resp.json()?;