From 03f1c2c37e71d6eef48687c792de80721e0a5aa0 Mon Sep 17 00:00:00 2001 From: Ben Dean-Kawamura Date: Thu, 17 Oct 2024 16:25:38 -0400 Subject: [PATCH] Starting new Remote Settings API Implemented the high-level API and the client functionality. Storage is a big TODO. Added a CLI to test it, you can run it using `cargo remote-settings`. --- .cargo/config.toml | 1 + Cargo.lock | 14 + components/remote_settings/Cargo.toml | 2 + components/remote_settings/src/client.rs | 295 ++++++++++++++++++++++ components/remote_settings/src/config.rs | 15 ++ components/remote_settings/src/lib.rs | 155 +++++++++++- components/remote_settings/src/service.rs | 126 +++++++++ components/remote_settings/src/storage.rs | 98 +++++++ examples/remote-settings-cli/Cargo.toml | 14 + examples/remote-settings-cli/src/main.rs | 113 +++++++++ 10 files changed, 830 insertions(+), 3 deletions(-) create mode 100644 components/remote_settings/src/service.rs create mode 100644 components/remote_settings/src/storage.rs create mode 100644 examples/remote-settings-cli/Cargo.toml create mode 100644 examples/remote-settings-cli/src/main.rs diff --git a/.cargo/config.toml b/.cargo/config.toml index 56b2f2cdf..7f6c29abc 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -9,4 +9,5 @@ suggest-bench = ["bench", "-p", "suggest", "--features", "benchmark_api"] suggest-debug-ingestion-sizes = ["run", "-p", "suggest", "--bin", "debug_ingestion_sizes", "--features", "benchmark_api"] relevancy = ["run", "-p", "examples-relevancy-cli", "--"] suggest = ["run", "-p", "examples-suggest-cli", "--"] +remote-settings = ["run", "-p", "examples-remote-settings-cli", "--"] start-bindings = ["run", "-p", "start-bindings", "--"] diff --git a/Cargo.lock b/Cargo.lock index 37f6171d4..26d52e9c2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1416,6 +1416,18 @@ dependencies = [ "viaduct-reqwest", ] +[[package]] +name = "examples-remote-settings-cli" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap 4.2.2", + "env_logger", + "log", + "remote_settings", + "viaduct-reqwest", +] + [[package]] name = "examples-suggest-cli" version = "0.1.0" @@ -3608,9 +3620,11 @@ dependencies = [ name = "remote_settings" version = "0.1.0" dependencies = [ + "camino", "error-support", "expect-test", "log", + "mockall", "mockito", "parking_lot", "serde", diff --git a/components/remote_settings/Cargo.toml b/components/remote_settings/Cargo.toml index 1979e6cc6..0980b5941 100644 --- a/components/remote_settings/Cargo.toml +++ b/components/remote_settings/Cargo.toml @@ -20,6 +20,7 @@ parking_lot = "0.12" error-support = { path = "../support/error" } viaduct = { path = "../viaduct" } url = "2.1" # mozilla-central can't yet take 2.2 (see bug 1734538) +camino = "1.0" [build-dependencies] uniffi = { workspace = true, features = ["build"] } @@ -27,6 +28,7 @@ uniffi = { workspace = true, features = ["build"] } [dev-dependencies] expect-test = "1.4" viaduct-reqwest = { path = "../support/viaduct-reqwest" } +mockall = "0.11" mockito = "0.31" # We add the perserve_order feature to guarantee ordering of the keys in our # JSON objects as they get serialized/deserialized. diff --git a/components/remote_settings/src/client.rs b/components/remote_settings/src/client.rs index 8475b6689..1067bc63b 100644 --- a/components/remote_settings/src/client.rs +++ b/components/remote_settings/src/client.rs @@ -4,6 +4,7 @@ use crate::config::RemoteSettingsConfig; use crate::error::{Error, Result}; +use crate::storage::Storage; use crate::{RemoteSettingsServer, UniffiCustomTypeConverter}; use parking_lot::Mutex; use serde::{Deserialize, Serialize}; @@ -18,6 +19,242 @@ const HEADER_BACKOFF: &str = "Backoff"; const HEADER_ETAG: &str = "ETag"; const HEADER_RETRY_AFTER: &str = "Retry-After"; +/// Internal Remote settings client API +/// +/// This stores an ApiClient implementation. In the real-world, this is always ViaductApiClient, +/// but the tests use a mock client. +pub struct RemoteSettingsClient { + // This is immutable, so it can be outside the mutex + collection_name: String, + inner: Mutex>, +} + +struct RemoteSettingsClientInner { + storage: Storage, + api_client: C, +} + +impl RemoteSettingsClient { + pub fn new_from_parts(collection_name: String, storage: Storage, api_client: C) -> Self { + Self { + collection_name, + inner: Mutex::new(RemoteSettingsClientInner { + storage, + api_client, + }), + } + } + pub fn collection_name(&self) -> &str { + &self.collection_name + } + + /// Get the current set of records. + /// + /// If records are not present in storage this will normally return None. Use `sync_if_empty = + /// true` to change this behavior and perform a network request in this case. + pub fn get_records(&self, sync_if_empty: bool) -> Result>> { + let mut inner = self.inner.lock(); + let collection_url = inner.api_client.collection_url(); + + let cached_records = inner.storage.get_records(&collection_url)?; + if cached_records.is_some() || !sync_if_empty { + return Ok(cached_records); + } + + let records = inner.api_client.get_records(None)?; + inner.storage.set_records(&collection_url, &records)?; + Ok(Some(records)) + } + + pub fn sync(&self) -> Result<()> { + let mut inner = self.inner.lock(); + let collection_url = inner.api_client.collection_url(); + let mtime = inner.storage.get_last_modified_timestamp(&collection_url)?; + let records = inner.api_client.get_records(mtime)?; + inner.storage.set_records(&collection_url, &records) + } + + /// Downloads an attachment from [attachment_location]. NOTE: there are no guarantees about a + /// maximum size, so use care when fetching potentially large attachments. + pub fn get_attachment(&self, attachment_location: &str) -> Result> { + self.inner + .lock() + .api_client + .get_attachment(attachment_location) + } +} + +impl RemoteSettingsClient { + pub fn new( + server_url: Url, + bucket_name: String, + collection_name: String, + storage: Storage, + ) -> Result { + let api_client = ViaductApiClient::new(server_url, &bucket_name, &collection_name)?; + Ok(Self::new_from_parts(collection_name, storage, api_client)) + } + + pub fn update_config(&self, server_url: Url, bucket_name: String) -> Result<()> { + let mut inner = self.inner.lock(); + inner.api_client = ViaductApiClient::new(server_url, &bucket_name, &self.collection_name)?; + inner.storage.empty() + } +} + +#[cfg_attr(test, mockall::automock)] +pub trait ApiClient { + /// Get the Bucket URL for this client. + /// + /// This is a URL that includes the server URL, bucket name, and collection name. This is used + /// to check if the application has switched the remote settings config and therefore we should + /// throw away any cached data + /// + /// Returns it as a String, since that's what the storage expects + fn collection_url(&self) -> String; + + /// Fetch records from the server + fn get_records(&mut self, timestamp: Option) -> Result>; + + /// Fetch an attachment from the server + fn get_attachment(&mut self, attachment_location: &str) -> Result>; +} + +/// Client for Remote settings API requests +pub struct ViaductApiClient { + /// Base URL for requests to a collections endpoint + /// + /// This is something like + /// `https://[server-url]/v1/buckets/[bucket-name]/collections/[collection-name]/" + /// + /// Note: this is different than the `base_url` used for other client implementations ( + /// (`https://[server-url]/v1). The main reason to use the collection_url is that we can use + /// it to check if we need to invalidate the cached data stored in the [Storage] layer. + collection_url: Url, + remote_state: RemoteState, +} + +impl ViaductApiClient { + fn new(server_url: Url, bucket_name: &str, collection_name: &str) -> Result { + let collection_url = server_url.join(&format!( + "v1/buckets/{bucket_name}/collections/{collection_name}/" + ))?; + Ok(Self { + collection_url, + remote_state: RemoteState::default(), + }) + } + + fn make_request(&mut self, url: Url) -> Result { + log::trace!("make_request: {url}"); + self.ensure_no_backoff()?; + + let req = Request::get(url); + let resp = req.send()?; + + self.handle_backoff_hint(&resp)?; + + if resp.is_success() { + Ok(resp) + } else { + Err(Error::ResponseError(format!( + "status code: {}", + resp.status + ))) + } + } + + fn ensure_no_backoff(&mut self) -> Result<()> { + if let BackoffState::Backoff { + observed_at, + duration, + } = self.remote_state.backoff + { + let elapsed_time = observed_at.elapsed(); + if elapsed_time >= duration { + self.remote_state.backoff = BackoffState::Ok; + } else { + let remaining = duration - elapsed_time; + return Err(Error::BackoffError(remaining.as_secs())); + } + } + Ok(()) + } + + fn handle_backoff_hint(&mut self, response: &Response) -> Result<()> { + let extract_backoff_header = |header| -> Result { + Ok(response + .headers + .get_as::(header) + .transpose() + .unwrap_or_default() // Ignore number parsing errors. + .unwrap_or(0)) + }; + // In practice these two headers are mutually exclusive. + let backoff = extract_backoff_header(HEADER_BACKOFF)?; + let retry_after = extract_backoff_header(HEADER_RETRY_AFTER)?; + let max_backoff = backoff.max(retry_after); + + if max_backoff > 0 { + self.remote_state.backoff = BackoffState::Backoff { + observed_at: Instant::now(), + duration: Duration::from_secs(max_backoff), + }; + } + Ok(()) + } +} + +impl ApiClient for ViaductApiClient { + fn collection_url(&self) -> String { + self.collection_url.to_string() + } + + fn get_records(&mut self, timestamp: Option) -> Result> { + let mut url = self.collection_url.join("changeset")?; + // 0 is used as an arbitrary value for `_expected` because the current implementation does + // not leverage push timestamps or polling from the monitor/changes endpoint. More + // details: + // + // https://remote-settings.readthedocs.io/en/latest/client-specifications.html#cache-busting + url.query_pairs_mut().append_pair("_expected", "0"); + if let Some(timestamp) = timestamp { + url.query_pairs_mut() + .append_pair("_since", ×tamp.to_string()); + } + + let resp = self.make_request(url)?; + + if resp.is_success() { + Ok(resp.json::()?.changes) + } else { + Err(Error::ResponseError(format!( + "status code: {}", + resp.status + ))) + } + } + + fn get_attachment(&mut self, attachment_location: &str) -> Result> { + let attachments_base_url = match &self.remote_state.attachments_base_url { + Some(attachments_base_url) => attachments_base_url.to_owned(), + None => { + let collection_url = self.collection_url.clone(); + let server_info = self.make_request(collection_url)?.json::()?; + let attachments_base_url = match server_info.capabilities.attachments { + Some(capability) => Url::parse(&capability.base_url)?, + None => Err(Error::AttachmentsUnsupportedError)?, + }; + self.remote_state.attachments_base_url = Some(attachments_base_url.clone()); + attachments_base_url + } + }; + + let resp = self.make_request(attachments_base_url.join(attachment_location)?)?; + Ok(resp.body) + } +} + /// A simple HTTP client that can retrieve Remote Settings data using the properties by [ClientConfig]. /// Methods defined on this will fetch data from /// /v1/buckets//collections// @@ -226,6 +463,11 @@ struct RecordsResponse { data: Vec, } +#[derive(Deserialize, Serialize)] +struct ChangesetResponse { + changes: Vec, +} + /// A parsed Remote Settings record. Records can contain arbitrary fields, so clients /// are required to further extract expected values from the [fields] member. #[derive(Clone, Debug, Deserialize, Serialize, Eq, PartialEq, uniffi::Record)] @@ -1155,3 +1397,56 @@ mod test { } "#; } + +#[cfg(test)] +mod test_new_client { + use super::*; + + use serde_json::json; + + #[test] + fn test_get_records_none_cached() { + let mut api_client = MockApiClient::new(); + api_client.expect_collection_url().returning(|| { + "http://rs.example.com/v1/main-workspace/collections/test-collection".into() + }); + // Note, don't make any api_client.expect_*() calls, the RemoteSettingsClient should not + // attempt to make any requests for this scenario + let storage = Storage::new(":memory:".into()).expect("Error creating storage"); + let rs_client = + RemoteSettingsClient::new_from_parts("test-collection".into(), storage, api_client); + assert_eq!( + rs_client.get_records(false).expect("Error getting records"), + None + ); + } + + #[test] + fn test_get_records_none_cached_sync_with_empty() { + let mut api_client = MockApiClient::new(); + let records = vec![RemoteSettingsRecord { + id: "record-0001".into(), + last_modified: 100, + deleted: false, + attachment: None, + fields: json!({"foo": "bar"}).as_object().unwrap().clone(), + }]; + api_client.expect_collection_url().returning(|| { + "http://rs.example.com/v1/main-workspace/collections/test-collection".into() + }); + api_client.expect_get_records().returning({ + let records = records.clone(); + move |timestamp| { + assert_eq!(timestamp, None); + Ok(records.clone()) + } + }); + let storage = Storage::new(":memory:".into()).expect("Error creating storage"); + let rs_client = + RemoteSettingsClient::new_from_parts("test-collection".into(), storage, api_client); + assert_eq!( + rs_client.get_records(true).expect("Error getting records"), + Some(records) + ); + } +} diff --git a/components/remote_settings/src/config.rs b/components/remote_settings/src/config.rs index f4029e5bc..8cb9baf7b 100644 --- a/components/remote_settings/src/config.rs +++ b/components/remote_settings/src/config.rs @@ -12,6 +12,21 @@ use url::Url; use crate::{ApiResult, Error, Result}; +/// Remote settings configuration +/// +/// This is the version used in the new API, hence the `2` at the end. The plan is to move +/// consumers to the new API, remove the RemoteSettingsConfig struct, then remove the `2` from this +/// name. +#[derive(Debug, Clone, uniffi::Record)] +pub struct RemoteSettingsConfig2 { + /// The Remote Settings server to use. Defaults to [RemoteSettingsServer::Prod], + #[uniffi(default = None)] + pub server: Option, + /// Bucket name to use, defaults to "main". Use "main-preview" for a preview bucket + #[uniffi(default = None)] + pub bucket_name: Option, +} + /// Custom configuration for the client. /// Currently includes the following: /// - `server`: The Remote Settings server to use. If not specified, defaults to the production server (`RemoteSettingsServer::Prod`). diff --git a/components/remote_settings/src/lib.rs b/components/remote_settings/src/lib.rs index e40d2d5f8..00480e95d 100644 --- a/components/remote_settings/src/lib.rs +++ b/components/remote_settings/src/lib.rs @@ -2,24 +2,173 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -use std::{fs::File, io::prelude::Write}; +use std::{collections::HashMap, fs::File, io::prelude::Write, sync::Arc}; -use error_support::handle_error; +use error_support::{convert_log_report_error, handle_error}; +use url::Url; pub mod cache; pub mod client; pub mod config; pub mod error; +pub mod service; +pub mod storage; pub use client::{Attachment, RemoteSettingsRecord, RemoteSettingsResponse, RsJsonObject}; -pub use config::{RemoteSettingsConfig, RemoteSettingsServer}; +pub use config::{RemoteSettingsConfig, RemoteSettingsConfig2, RemoteSettingsServer}; pub use error::{ApiResult, RemoteSettingsError, Result}; use client::Client; use error::Error; +use storage::Storage; uniffi::setup_scaffolding!("remote_settings"); +/// Application-level Remote Settings manager. +/// +/// This handles application-level operations, like syncing all the collections, and acts as a +/// factory for creating clients. +#[derive(uniffi::Object)] +pub struct RemoteSettingsService { + // This struct adapts server::RemoteSettingsService into the public API + internal: service::RemoteSettingsService, +} + +#[uniffi::export] +impl RemoteSettingsService { + /// Construct a [RemoteSettingsService] + /// + /// This is typically done early in the application-startup process + #[uniffi::constructor] + #[handle_error(Error)] + pub fn new(storage_dir: String, config: RemoteSettingsConfig2) -> ApiResult { + Ok(Self { + internal: service::RemoteSettingsService::new(storage_dir, config)?, + }) + } + + /// Create a new Remote Settings client + #[handle_error(Error)] + pub fn make_client(&self, collection_name: String) -> ApiResult> { + self.internal.make_client(collection_name) + } + + /// Sync collections for all active clients + #[handle_error(Error)] + pub fn sync(&self) -> ApiResult> { + self.internal.sync() + } + + /// Update the remote settings config + /// + /// This will cause all current and future clients to use new config and will delete any stored + /// records causing the clients to return new results from the new config. + /// + /// Only intended for QA/debugging. Swapping the remote settings server in the middle of + /// execution can cause weird effects. + #[handle_error(Error)] + pub fn update_config(&self, config: RemoteSettingsConfig2) -> ApiResult<()> { + self.internal.update_config(config) + } +} + +/// Client for a single Remote Settings collection +/// +/// Use [RemoteSettingsService::make_client] to create these. +#[derive(uniffi::Object)] +pub struct RemoteSettingsClient { + // This struct adapts client::RemoteSettingsClient into the public API + internal: client::RemoteSettingsClient, +} + +#[uniffi::export] +impl RemoteSettingsClient { + /// Collection this client is for + pub fn collection_name(&self) -> String { + self.internal.collection_name().to_owned() + } + + /// Get the current set of records. + /// + /// This method normally fetches records from the last sync. This means that it returns fast + /// and does not make any network requests. + /// + /// If records have not yet been synced it will return None. Use `sync_if_empty = true` to + /// change this behavior and perform a network request in this case. That this is probably a + /// bad idea if you want to fetch the setting in application startup or when building the UI. + /// + /// None will also be returned on disk IO errors or other unexpected errors. The reason for + /// this is that there is not much an application can do in this situation other than fall back + /// to the same default handling as if records have not been synced. + /// + /// TODO(Bug 1919141): + /// + /// Application-services schedules regular dumps of the server data for specific collections. + /// For these collections, `get_records` will never return None. If you would like to add your + /// collection to this list, please reach out to the DISCO team. + #[uniffi::method(default(sync_if_empty = false))] + pub fn get_records(&self, sync_if_empty: bool) -> Option> { + match self.internal.get_records(sync_if_empty) { + Ok(records) => records, + Err(e) => { + // Log/report the error + log::trace!("get_records error: {e}"); + convert_log_report_error(e); + // Throw away the converted result and return None, there's nothing a client can + // really do with an error except treat it as the None case + None + } + } + } + + /// Get the current set of records as a map of record_id -> record. + /// + /// See [Self::get_records] for an explanation of when this makes network requests, error + /// handling, and how the `sync_if_empty` param works. + #[uniffi::method(default(sync_if_empty = false))] + pub fn get_records_map( + &self, + sync_if_empty: bool, + ) -> Option> { + self.get_records(sync_if_empty) + .map(|records| records.into_iter().map(|r| (r.id.clone(), r)).collect()) + } + + /// Get attachment data for a remote settings record + /// + /// Attachments are large binary blobs used for data that doesn't fit in a normal record. They + /// are handled differently than other record data: + /// + /// - Attachments are not downloaded in [RemoteSettingsService::sync] + /// - This method will make network requests if the attachment is not cached + /// - This method will throw if there is a network or other error when fetching the + /// attachment data. + #[handle_error(Error)] + pub fn get_attachment(&self, attachment_id: String) -> ApiResult> { + self.internal.get_attachment(&attachment_id) + } +} + +impl RemoteSettingsClient { + /// Create a new client. This is not exposed to foreign code, consumers need to call + /// [RemoteSettingsService::make_client] + fn new( + base_url: Url, + bucket_name: String, + collection_name: String, + storage: Storage, + ) -> Result { + Ok(Self { + internal: client::RemoteSettingsClient::new( + base_url, + bucket_name, + collection_name, + storage, + )?, + }) + } +} + #[derive(uniffi::Object)] pub struct RemoteSettings { pub config: RemoteSettingsConfig, diff --git a/components/remote_settings/src/service.rs b/components/remote_settings/src/service.rs new file mode 100644 index 000000000..75d2a3370 --- /dev/null +++ b/components/remote_settings/src/service.rs @@ -0,0 +1,126 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use std::{ + collections::HashSet, + sync::{Arc, Weak}, +}; + +use camino::Utf8PathBuf; +use parking_lot::Mutex; +use url::Url; + +use crate::{ + storage::Storage, RemoteSettingsClient, RemoteSettingsConfig2, RemoteSettingsServer, Result, +}; + +/// Internal Remote settings service API +pub struct RemoteSettingsService { + inner: Mutex, +} + +struct RemoteSettingsServiceInner { + storage_dir: Utf8PathBuf, + base_url: Url, + bucket_name: String, + /// Weakrefs for all clients that we've created. Note: this stores the + /// top-level/public `RemoteSettingsClient` structs rather than `client::RemoteSettingsClient`. + /// The reason for this is that we return Arcs to the public struct to the foreign code, so we + /// need to use the same type for our weakrefs. The alternative would be to create 2 Arcs for + /// each client, which is wasteful. + clients: Vec>, +} + +impl RemoteSettingsService { + /// Construct a [RemoteSettingsService] + /// + /// This is typically done early in the application-startup process + pub fn new(storage_dir: String, config: RemoteSettingsConfig2) -> Result { + let storage_dir = storage_dir.into(); + let base_url = config + .server + .unwrap_or(RemoteSettingsServer::Prod) + .get_url()?; + let bucket_name = config.bucket_name.unwrap_or_else(|| String::from("main")); + + Ok(Self { + inner: Mutex::new(RemoteSettingsServiceInner { + storage_dir, + base_url, + bucket_name, + clients: vec![], + }), + }) + } + + /// Create a new Remote Settings client + pub fn make_client(&self, collection_name: String) -> Result> { + let mut inner = self.inner.lock(); + let storage = Storage::new(inner.storage_dir.join(format!("{collection_name}.sql")))?; + let client = Arc::new(RemoteSettingsClient::new( + inner.base_url.clone(), + inner.bucket_name.clone(), + collection_name.clone(), + storage, + )?); + inner.clients.push(Arc::downgrade(&client)); + Ok(client) + } + + /// Sync collections for all active clients + pub fn sync(&self) -> Result> { + // Make sure we only sync each collection once, even if there are multiple clients + let mut synced_collections = HashSet::new(); + + // TODO: poll the server using `/buckets/monitor/collections/changes/changeset` to fetch + // the current timestamp for all collections. That way we can avoid fetching collections + // we know haven't changed and also pass the `?_expected{ts}` param to the server. + + for client in self.inner.lock().active_clients() { + if synced_collections.insert(client.collection_name()) { + client.internal.sync()?; + } + } + Ok(synced_collections.into_iter().collect()) + } + + /// Update the remote settings config + /// + /// This will cause all current and future clients to use new config and will delete any stored + /// records causing the clients to return new results from the new config. + pub fn update_config(&self, config: RemoteSettingsConfig2) -> Result<()> { + let base_url = config + .server + .unwrap_or(RemoteSettingsServer::Prod) + .get_url()?; + let bucket_name = config.bucket_name.unwrap_or_else(|| String::from("main")); + let mut inner = self.inner.lock(); + for client in inner.active_clients() { + client + .internal + .update_config(base_url.clone(), bucket_name.clone())?; + } + inner.base_url = base_url; + inner.bucket_name = bucket_name; + Ok(()) + } +} + +impl RemoteSettingsServiceInner { + // Find live clients in self.clients + // + // Also, drop dead weakrefs from the vec + fn active_clients(&mut self) -> Vec> { + let mut active_clients = vec![]; + self.clients.retain(|weak| { + if let Some(client) = weak.upgrade() { + active_clients.push(client); + true + } else { + false + } + }); + active_clients + } +} diff --git a/components/remote_settings/src/storage.rs b/components/remote_settings/src/storage.rs new file mode 100644 index 000000000..a4bf71779 --- /dev/null +++ b/components/remote_settings/src/storage.rs @@ -0,0 +1,98 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use camino::Utf8PathBuf; + +use crate::{Attachment, RemoteSettingsRecord, Result}; + +/// Internal storage type +/// +/// This will store downloaded records/attachments in a SQLite database. Nothing is implemented +/// yet other than the initial API. +/// +/// Most methods input a `collection_url` parameter, is a URL that includes the remote settings +/// server, bucket, and collection. If the `collection_url` for a get method does not match the one +/// for a set method, then this means the application has switched their remote settings config and +/// [Storage] should pretend like nothing is stored in the database. +/// +/// The reason for this is the [crate::RemoteSettingsService::update_config] method. If a consumer +/// passes a new server or bucket to `update_config`, we don't want to be using cached data from +/// the previous config. +/// +/// Notes: +/// - I'm thinking we'll create a separate SQLite database per collection. That reduces +/// contention when multiple clients try to get records at once. +/// - Still, there might be contention if there are multiple clients for the same collection, or +/// if RemoteSettingsService::sync() and RemoteSettingsClient::get_records(true) are called at +/// the same time. Maybe we should create a single write connection and put it behind a mutex +/// to avoid the possibility of SQLITE_BUSY. Or maybe not, the writes seem like they should be +/// very fast. +/// - Maybe we should refactor this to use the DAO pattern like suggest does. +pub struct Storage {} + +impl Storage { + pub fn new(_path: Utf8PathBuf) -> Result { + Ok(Self {}) + } + + /// Get the last modified timestamp for the stored records + /// + /// Returns None if no records are stored or if `collection_url` does not match the + /// `collection_url` passed to `set_records`. + pub fn get_last_modified_timestamp(&self, _collection_url: &str) -> Result> { + Ok(None) + } + + /// Get cached records for this collection + /// + /// Returns None if no records are stored or if `collection_url` does not match the + /// `collection_url` passed to `set_records`. + pub fn get_records(&self, _collection_url: &str) -> Result>> { + Ok(None) + } + + /// Get cached attachment data + /// + /// This returns the last attachment data sent to [Self::set_attachment]. + /// + /// Returns None if no attachment data is stored or if `collection_url` does not match the `collection_url` + /// passed to `set_attachment`. + pub fn get_attachment( + &self, + _collection_url: &str, + _attachment_id: &str, + ) -> Result> { + Ok(None) + } + + /// Set the list of records stored in the database, clearing out any previously stored records + pub fn set_records( + &self, + _collection_url: &str, + records: &[RemoteSettingsRecord], + ) -> Result<()> { + for record in records { + println!("Should store record: {record:?}"); + } + Ok(()) + } + + /// Set the attachment data stored in the database, clearing out any previously stored data + pub fn set_attachment( + &self, + _collection_url: &str, + attachment_id: &str, + _attachment: Attachment, + ) -> Result<()> { + println!("Should store attachment: {attachment_id}"); + Ok(()) + } + + /// Empty out all cached values and start from scratch. This is called when + /// RemoteSettingsService::update_config() is called, since that could change the remote + /// settings server which would invalidate all cached data. + pub fn empty(&self) -> Result<()> { + Ok(()) + } +} diff --git a/examples/remote-settings-cli/Cargo.toml b/examples/remote-settings-cli/Cargo.toml new file mode 100644 index 000000000..3d6bffe25 --- /dev/null +++ b/examples/remote-settings-cli/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "examples-remote-settings-cli" +version = "0.1.0" +license = "MPL-2.0" +edition = "2021" +publish = false + +[dependencies] +remote_settings = { path = "../../components/remote_settings" } +viaduct-reqwest = { path = "../../components/support/viaduct-reqwest" } +log = "0.4" +clap = {version = "4.2", features = ["derive"]} +anyhow = "1.0" +env_logger = { version = "0.10", default-features = false, features = ["humantime"] } diff --git a/examples/remote-settings-cli/src/main.rs b/examples/remote-settings-cli/src/main.rs new file mode 100644 index 000000000..b0c5cf1a7 --- /dev/null +++ b/examples/remote-settings-cli/src/main.rs @@ -0,0 +1,113 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use anyhow::Result; +use clap::{Parser, Subcommand, ValueEnum}; + +use remote_settings::{RemoteSettingsConfig2, RemoteSettingsServer, RemoteSettingsService}; + +const DEFAULT_LOG_FILTER: &str = "remote_settings=info"; +const DEFAULT_LOG_FILTER_VERBOSE: &str = "remote_settings=trace"; + +#[derive(Debug, Parser)] +#[command(about, long_about = None)] +struct Cli { + #[arg(short = 's')] + server: Option, + #[arg(short = 'b')] + bucket: Option, + #[arg(short = 'd')] + storage_dir: Option, + #[arg(long, short, action)] + verbose: bool, + #[command(subcommand)] + command: Commands, +} + +#[derive(Clone, Debug, ValueEnum)] +enum RemoteSettingsServerArg { + Prod, + Stage, + Dev, +} + +#[derive(Debug, Subcommand)] +enum Commands { + /// Sync collections + Sync { + #[clap(required = true)] + collections: Vec, + }, + /// Query against ingested data + Get { + collection: String, + #[arg(long)] + sync_if_empty: bool, + }, +} + +fn main() -> Result<()> { + let cli = Cli::parse(); + env_logger::init_from_env(env_logger::Env::default().filter_or( + "RUST_LOG", + if cli.verbose { + DEFAULT_LOG_FILTER_VERBOSE + } else { + DEFAULT_LOG_FILTER + }, + )); + viaduct_reqwest::use_reqwest_backend(); + let service = build_service(&cli)?; + match cli.command { + Commands::Sync { collections } => sync(service, collections), + Commands::Get { + collection, + sync_if_empty, + } => get_records(service, collection, sync_if_empty), + } +} + +fn build_service(cli: &Cli) -> Result { + let config = RemoteSettingsConfig2 { + server: cli.server.as_ref().map(|s| match s { + RemoteSettingsServerArg::Dev => RemoteSettingsServer::Dev, + RemoteSettingsServerArg::Stage => RemoteSettingsServer::Stage, + RemoteSettingsServerArg::Prod => RemoteSettingsServer::Prod, + }), + bucket_name: cli.bucket.clone(), + }; + Ok(RemoteSettingsService::new( + cli.storage_dir + .clone() + .unwrap_or_else(|| "remote-settings-data".into()), + config, + )?) +} + +fn sync(service: RemoteSettingsService, collections: Vec) -> Result<()> { + // Create a bunch of clients so that sync() syncs their collections + let _clients = collections + .into_iter() + .map(|collection| Ok(service.make_client(collection)?)) + .collect::>>()?; + service.sync()?; + Ok(()) +} + +fn get_records( + service: RemoteSettingsService, + collection: String, + sync_if_empty: bool, +) -> Result<()> { + let client = service.make_client(collection)?; + match client.get_records(sync_if_empty) { + Some(records) => { + for record in records { + println!("{record:?}"); + } + } + None => println!("No cached records"), + } + Ok(()) +}