Starting new Remote Settings API
Implemented the high-level API and the client functionality. Storage is a big TODO. Added a CLI to test it, you can run it using `cargo remote-settings`.
This commit is contained in:
Ben Dean-Kawamura
bendk
Родитель
7e71b6a672
Коммит
03f1c2c37e
|
|
@ -9,4 +9,5 @@ suggest-bench = ["bench", "-p", "suggest", "--features", "benchmark_api"]
|
|||
suggest-debug-ingestion-sizes = ["run", "-p", "suggest", "--bin", "debug_ingestion_sizes", "--features", "benchmark_api"]
|
||||
relevancy = ["run", "-p", "examples-relevancy-cli", "--"]
|
||||
suggest = ["run", "-p", "examples-suggest-cli", "--"]
|
||||
remote-settings = ["run", "-p", "examples-remote-settings-cli", "--"]
|
||||
start-bindings = ["run", "-p", "start-bindings", "--"]
|
||||
|
|
|
|||
|
|
@ -1416,6 +1416,18 @@ dependencies = [
|
|||
"viaduct-reqwest",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "examples-remote-settings-cli"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"clap 4.2.2",
|
||||
"env_logger",
|
||||
"log",
|
||||
"remote_settings",
|
||||
"viaduct-reqwest",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "examples-suggest-cli"
|
||||
version = "0.1.0"
|
||||
|
|
@ -3608,9 +3620,11 @@ dependencies = [
|
|||
name = "remote_settings"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"camino",
|
||||
"error-support",
|
||||
"expect-test",
|
||||
"log",
|
||||
"mockall",
|
||||
"mockito",
|
||||
"parking_lot",
|
||||
"serde",
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ parking_lot = "0.12"
|
|||
error-support = { path = "../support/error" }
|
||||
viaduct = { path = "../viaduct" }
|
||||
url = "2.1" # mozilla-central can't yet take 2.2 (see bug 1734538)
|
||||
camino = "1.0"
|
||||
|
||||
[build-dependencies]
|
||||
uniffi = { workspace = true, features = ["build"] }
|
||||
|
|
@ -27,6 +28,7 @@ uniffi = { workspace = true, features = ["build"] }
|
|||
[dev-dependencies]
|
||||
expect-test = "1.4"
|
||||
viaduct-reqwest = { path = "../support/viaduct-reqwest" }
|
||||
mockall = "0.11"
|
||||
mockito = "0.31"
|
||||
# We add the perserve_order feature to guarantee ordering of the keys in our
|
||||
# JSON objects as they get serialized/deserialized.
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@
|
|||
|
||||
use crate::config::RemoteSettingsConfig;
|
||||
use crate::error::{Error, Result};
|
||||
use crate::storage::Storage;
|
||||
use crate::{RemoteSettingsServer, UniffiCustomTypeConverter};
|
||||
use parking_lot::Mutex;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
|
@ -18,6 +19,242 @@ const HEADER_BACKOFF: &str = "Backoff";
|
|||
const HEADER_ETAG: &str = "ETag";
|
||||
const HEADER_RETRY_AFTER: &str = "Retry-After";
|
||||
|
||||
/// Internal Remote settings client API
|
||||
///
|
||||
/// This stores an ApiClient implementation. In the real-world, this is always ViaductApiClient,
|
||||
/// but the tests use a mock client.
|
||||
pub struct RemoteSettingsClient<C = ViaductApiClient> {
|
||||
// This is immutable, so it can be outside the mutex
|
||||
collection_name: String,
|
||||
inner: Mutex<RemoteSettingsClientInner<C>>,
|
||||
}
|
||||
|
||||
struct RemoteSettingsClientInner<C> {
|
||||
storage: Storage,
|
||||
api_client: C,
|
||||
}
|
||||
|
||||
impl<C: ApiClient> RemoteSettingsClient<C> {
|
||||
pub fn new_from_parts(collection_name: String, storage: Storage, api_client: C) -> Self {
|
||||
Self {
|
||||
collection_name,
|
||||
inner: Mutex::new(RemoteSettingsClientInner {
|
||||
storage,
|
||||
api_client,
|
||||
}),
|
||||
}
|
||||
}
|
||||
pub fn collection_name(&self) -> &str {
|
||||
&self.collection_name
|
||||
}
|
||||
|
||||
/// Get the current set of records.
|
||||
///
|
||||
/// If records are not present in storage this will normally return None. Use `sync_if_empty =
|
||||
/// true` to change this behavior and perform a network request in this case.
|
||||
pub fn get_records(&self, sync_if_empty: bool) -> Result<Option<Vec<RemoteSettingsRecord>>> {
|
||||
let mut inner = self.inner.lock();
|
||||
let collection_url = inner.api_client.collection_url();
|
||||
|
||||
let cached_records = inner.storage.get_records(&collection_url)?;
|
||||
if cached_records.is_some() || !sync_if_empty {
|
||||
return Ok(cached_records);
|
||||
}
|
||||
|
||||
let records = inner.api_client.get_records(None)?;
|
||||
inner.storage.set_records(&collection_url, &records)?;
|
||||
Ok(Some(records))
|
||||
}
|
||||
|
||||
pub fn sync(&self) -> Result<()> {
|
||||
let mut inner = self.inner.lock();
|
||||
let collection_url = inner.api_client.collection_url();
|
||||
let mtime = inner.storage.get_last_modified_timestamp(&collection_url)?;
|
||||
let records = inner.api_client.get_records(mtime)?;
|
||||
inner.storage.set_records(&collection_url, &records)
|
||||
}
|
||||
|
||||
/// Downloads an attachment from [attachment_location]. NOTE: there are no guarantees about a
|
||||
/// maximum size, so use care when fetching potentially large attachments.
|
||||
pub fn get_attachment(&self, attachment_location: &str) -> Result<Vec<u8>> {
|
||||
self.inner
|
||||
.lock()
|
||||
.api_client
|
||||
.get_attachment(attachment_location)
|
||||
}
|
||||
}
|
||||
|
||||
impl RemoteSettingsClient<ViaductApiClient> {
|
||||
pub fn new(
|
||||
server_url: Url,
|
||||
bucket_name: String,
|
||||
collection_name: String,
|
||||
storage: Storage,
|
||||
) -> Result<Self> {
|
||||
let api_client = ViaductApiClient::new(server_url, &bucket_name, &collection_name)?;
|
||||
Ok(Self::new_from_parts(collection_name, storage, api_client))
|
||||
}
|
||||
|
||||
pub fn update_config(&self, server_url: Url, bucket_name: String) -> Result<()> {
|
||||
let mut inner = self.inner.lock();
|
||||
inner.api_client = ViaductApiClient::new(server_url, &bucket_name, &self.collection_name)?;
|
||||
inner.storage.empty()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg_attr(test, mockall::automock)]
|
||||
pub trait ApiClient {
|
||||
/// Get the Bucket URL for this client.
|
||||
///
|
||||
/// This is a URL that includes the server URL, bucket name, and collection name. This is used
|
||||
/// to check if the application has switched the remote settings config and therefore we should
|
||||
/// throw away any cached data
|
||||
///
|
||||
/// Returns it as a String, since that's what the storage expects
|
||||
fn collection_url(&self) -> String;
|
||||
|
||||
/// Fetch records from the server
|
||||
fn get_records(&mut self, timestamp: Option<u64>) -> Result<Vec<RemoteSettingsRecord>>;
|
||||
|
||||
/// Fetch an attachment from the server
|
||||
fn get_attachment(&mut self, attachment_location: &str) -> Result<Vec<u8>>;
|
||||
}
|
||||
|
||||
/// Client for Remote settings API requests
|
||||
pub struct ViaductApiClient {
|
||||
/// Base URL for requests to a collections endpoint
|
||||
///
|
||||
/// This is something like
|
||||
/// `https://[server-url]/v1/buckets/[bucket-name]/collections/[collection-name]/"
|
||||
///
|
||||
/// Note: this is different than the `base_url` used for other client implementations (
|
||||
/// (`https://[server-url]/v1). The main reason to use the collection_url is that we can use
|
||||
/// it to check if we need to invalidate the cached data stored in the [Storage] layer.
|
||||
collection_url: Url,
|
||||
remote_state: RemoteState,
|
||||
}
|
||||
|
||||
impl ViaductApiClient {
|
||||
fn new(server_url: Url, bucket_name: &str, collection_name: &str) -> Result<Self> {
|
||||
let collection_url = server_url.join(&format!(
|
||||
"v1/buckets/{bucket_name}/collections/{collection_name}/"
|
||||
))?;
|
||||
Ok(Self {
|
||||
collection_url,
|
||||
remote_state: RemoteState::default(),
|
||||
})
|
||||
}
|
||||
|
||||
fn make_request(&mut self, url: Url) -> Result<Response> {
|
||||
log::trace!("make_request: {url}");
|
||||
self.ensure_no_backoff()?;
|
||||
|
||||
let req = Request::get(url);
|
||||
let resp = req.send()?;
|
||||
|
||||
self.handle_backoff_hint(&resp)?;
|
||||
|
||||
if resp.is_success() {
|
||||
Ok(resp)
|
||||
} else {
|
||||
Err(Error::ResponseError(format!(
|
||||
"status code: {}",
|
||||
resp.status
|
||||
)))
|
||||
}
|
||||
}
|
||||
|
||||
fn ensure_no_backoff(&mut self) -> Result<()> {
|
||||
if let BackoffState::Backoff {
|
||||
observed_at,
|
||||
duration,
|
||||
} = self.remote_state.backoff
|
||||
{
|
||||
let elapsed_time = observed_at.elapsed();
|
||||
if elapsed_time >= duration {
|
||||
self.remote_state.backoff = BackoffState::Ok;
|
||||
} else {
|
||||
let remaining = duration - elapsed_time;
|
||||
return Err(Error::BackoffError(remaining.as_secs()));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn handle_backoff_hint(&mut self, response: &Response) -> Result<()> {
|
||||
let extract_backoff_header = |header| -> Result<u64> {
|
||||
Ok(response
|
||||
.headers
|
||||
.get_as::<u64, _>(header)
|
||||
.transpose()
|
||||
.unwrap_or_default() // Ignore number parsing errors.
|
||||
.unwrap_or(0))
|
||||
};
|
||||
// In practice these two headers are mutually exclusive.
|
||||
let backoff = extract_backoff_header(HEADER_BACKOFF)?;
|
||||
let retry_after = extract_backoff_header(HEADER_RETRY_AFTER)?;
|
||||
let max_backoff = backoff.max(retry_after);
|
||||
|
||||
if max_backoff > 0 {
|
||||
self.remote_state.backoff = BackoffState::Backoff {
|
||||
observed_at: Instant::now(),
|
||||
duration: Duration::from_secs(max_backoff),
|
||||
};
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl ApiClient for ViaductApiClient {
|
||||
fn collection_url(&self) -> String {
|
||||
self.collection_url.to_string()
|
||||
}
|
||||
|
||||
fn get_records(&mut self, timestamp: Option<u64>) -> Result<Vec<RemoteSettingsRecord>> {
|
||||
let mut url = self.collection_url.join("changeset")?;
|
||||
// 0 is used as an arbitrary value for `_expected` because the current implementation does
|
||||
// not leverage push timestamps or polling from the monitor/changes endpoint. More
|
||||
// details:
|
||||
//
|
||||
// https://remote-settings.readthedocs.io/en/latest/client-specifications.html#cache-busting
|
||||
url.query_pairs_mut().append_pair("_expected", "0");
|
||||
if let Some(timestamp) = timestamp {
|
||||
url.query_pairs_mut()
|
||||
.append_pair("_since", ×tamp.to_string());
|
||||
}
|
||||
|
||||
let resp = self.make_request(url)?;
|
||||
|
||||
if resp.is_success() {
|
||||
Ok(resp.json::<ChangesetResponse>()?.changes)
|
||||
} else {
|
||||
Err(Error::ResponseError(format!(
|
||||
"status code: {}",
|
||||
resp.status
|
||||
)))
|
||||
}
|
||||
}
|
||||
|
||||
fn get_attachment(&mut self, attachment_location: &str) -> Result<Vec<u8>> {
|
||||
let attachments_base_url = match &self.remote_state.attachments_base_url {
|
||||
Some(attachments_base_url) => attachments_base_url.to_owned(),
|
||||
None => {
|
||||
let collection_url = self.collection_url.clone();
|
||||
let server_info = self.make_request(collection_url)?.json::<ServerInfo>()?;
|
||||
let attachments_base_url = match server_info.capabilities.attachments {
|
||||
Some(capability) => Url::parse(&capability.base_url)?,
|
||||
None => Err(Error::AttachmentsUnsupportedError)?,
|
||||
};
|
||||
self.remote_state.attachments_base_url = Some(attachments_base_url.clone());
|
||||
attachments_base_url
|
||||
}
|
||||
};
|
||||
|
||||
let resp = self.make_request(attachments_base_url.join(attachment_location)?)?;
|
||||
Ok(resp.body)
|
||||
}
|
||||
}
|
||||
|
||||
/// A simple HTTP client that can retrieve Remote Settings data using the properties by [ClientConfig].
|
||||
/// Methods defined on this will fetch data from
|
||||
/// <base_url>/v1/buckets/<bucket_name>/collections/<collection_name>/
|
||||
|
|
@ -226,6 +463,11 @@ struct RecordsResponse {
|
|||
data: Vec<RemoteSettingsRecord>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Serialize)]
|
||||
struct ChangesetResponse {
|
||||
changes: Vec<RemoteSettingsRecord>,
|
||||
}
|
||||
|
||||
/// A parsed Remote Settings record. Records can contain arbitrary fields, so clients
|
||||
/// are required to further extract expected values from the [fields] member.
|
||||
#[derive(Clone, Debug, Deserialize, Serialize, Eq, PartialEq, uniffi::Record)]
|
||||
|
|
@ -1155,3 +1397,56 @@ mod test {
|
|||
}
|
||||
"#;
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod test_new_client {
|
||||
use super::*;
|
||||
|
||||
use serde_json::json;
|
||||
|
||||
#[test]
|
||||
fn test_get_records_none_cached() {
|
||||
let mut api_client = MockApiClient::new();
|
||||
api_client.expect_collection_url().returning(|| {
|
||||
"http://rs.example.com/v1/main-workspace/collections/test-collection".into()
|
||||
});
|
||||
// Note, don't make any api_client.expect_*() calls, the RemoteSettingsClient should not
|
||||
// attempt to make any requests for this scenario
|
||||
let storage = Storage::new(":memory:".into()).expect("Error creating storage");
|
||||
let rs_client =
|
||||
RemoteSettingsClient::new_from_parts("test-collection".into(), storage, api_client);
|
||||
assert_eq!(
|
||||
rs_client.get_records(false).expect("Error getting records"),
|
||||
None
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_get_records_none_cached_sync_with_empty() {
|
||||
let mut api_client = MockApiClient::new();
|
||||
let records = vec![RemoteSettingsRecord {
|
||||
id: "record-0001".into(),
|
||||
last_modified: 100,
|
||||
deleted: false,
|
||||
attachment: None,
|
||||
fields: json!({"foo": "bar"}).as_object().unwrap().clone(),
|
||||
}];
|
||||
api_client.expect_collection_url().returning(|| {
|
||||
"http://rs.example.com/v1/main-workspace/collections/test-collection".into()
|
||||
});
|
||||
api_client.expect_get_records().returning({
|
||||
let records = records.clone();
|
||||
move |timestamp| {
|
||||
assert_eq!(timestamp, None);
|
||||
Ok(records.clone())
|
||||
}
|
||||
});
|
||||
let storage = Storage::new(":memory:".into()).expect("Error creating storage");
|
||||
let rs_client =
|
||||
RemoteSettingsClient::new_from_parts("test-collection".into(), storage, api_client);
|
||||
assert_eq!(
|
||||
rs_client.get_records(true).expect("Error getting records"),
|
||||
Some(records)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -12,6 +12,21 @@ use url::Url;
|
|||
|
||||
use crate::{ApiResult, Error, Result};
|
||||
|
||||
/// Remote settings configuration
|
||||
///
|
||||
/// This is the version used in the new API, hence the `2` at the end. The plan is to move
|
||||
/// consumers to the new API, remove the RemoteSettingsConfig struct, then remove the `2` from this
|
||||
/// name.
|
||||
#[derive(Debug, Clone, uniffi::Record)]
|
||||
pub struct RemoteSettingsConfig2 {
|
||||
/// The Remote Settings server to use. Defaults to [RemoteSettingsServer::Prod],
|
||||
#[uniffi(default = None)]
|
||||
pub server: Option<RemoteSettingsServer>,
|
||||
/// Bucket name to use, defaults to "main". Use "main-preview" for a preview bucket
|
||||
#[uniffi(default = None)]
|
||||
pub bucket_name: Option<String>,
|
||||
}
|
||||
|
||||
/// Custom configuration for the client.
|
||||
/// Currently includes the following:
|
||||
/// - `server`: The Remote Settings server to use. If not specified, defaults to the production server (`RemoteSettingsServer::Prod`).
|
||||
|
|
|
|||
|
|
@ -2,24 +2,173 @@
|
|||
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
|
||||
|
||||
use std::{fs::File, io::prelude::Write};
|
||||
use std::{collections::HashMap, fs::File, io::prelude::Write, sync::Arc};
|
||||
|
||||
use error_support::handle_error;
|
||||
use error_support::{convert_log_report_error, handle_error};
|
||||
use url::Url;
|
||||
|
||||
pub mod cache;
|
||||
pub mod client;
|
||||
pub mod config;
|
||||
pub mod error;
|
||||
pub mod service;
|
||||
pub mod storage;
|
||||
|
||||
pub use client::{Attachment, RemoteSettingsRecord, RemoteSettingsResponse, RsJsonObject};
|
||||
pub use config::{RemoteSettingsConfig, RemoteSettingsServer};
|
||||
pub use config::{RemoteSettingsConfig, RemoteSettingsConfig2, RemoteSettingsServer};
|
||||
pub use error::{ApiResult, RemoteSettingsError, Result};
|
||||
|
||||
use client::Client;
|
||||
use error::Error;
|
||||
use storage::Storage;
|
||||
|
||||
uniffi::setup_scaffolding!("remote_settings");
|
||||
|
||||
/// Application-level Remote Settings manager.
|
||||
///
|
||||
/// This handles application-level operations, like syncing all the collections, and acts as a
|
||||
/// factory for creating clients.
|
||||
#[derive(uniffi::Object)]
|
||||
pub struct RemoteSettingsService {
|
||||
// This struct adapts server::RemoteSettingsService into the public API
|
||||
internal: service::RemoteSettingsService,
|
||||
}
|
||||
|
||||
#[uniffi::export]
|
||||
impl RemoteSettingsService {
|
||||
/// Construct a [RemoteSettingsService]
|
||||
///
|
||||
/// This is typically done early in the application-startup process
|
||||
#[uniffi::constructor]
|
||||
#[handle_error(Error)]
|
||||
pub fn new(storage_dir: String, config: RemoteSettingsConfig2) -> ApiResult<Self> {
|
||||
Ok(Self {
|
||||
internal: service::RemoteSettingsService::new(storage_dir, config)?,
|
||||
})
|
||||
}
|
||||
|
||||
/// Create a new Remote Settings client
|
||||
#[handle_error(Error)]
|
||||
pub fn make_client(&self, collection_name: String) -> ApiResult<Arc<RemoteSettingsClient>> {
|
||||
self.internal.make_client(collection_name)
|
||||
}
|
||||
|
||||
/// Sync collections for all active clients
|
||||
#[handle_error(Error)]
|
||||
pub fn sync(&self) -> ApiResult<Vec<String>> {
|
||||
self.internal.sync()
|
||||
}
|
||||
|
||||
/// Update the remote settings config
|
||||
///
|
||||
/// This will cause all current and future clients to use new config and will delete any stored
|
||||
/// records causing the clients to return new results from the new config.
|
||||
///
|
||||
/// Only intended for QA/debugging. Swapping the remote settings server in the middle of
|
||||
/// execution can cause weird effects.
|
||||
#[handle_error(Error)]
|
||||
pub fn update_config(&self, config: RemoteSettingsConfig2) -> ApiResult<()> {
|
||||
self.internal.update_config(config)
|
||||
}
|
||||
}
|
||||
|
||||
/// Client for a single Remote Settings collection
|
||||
///
|
||||
/// Use [RemoteSettingsService::make_client] to create these.
|
||||
#[derive(uniffi::Object)]
|
||||
pub struct RemoteSettingsClient {
|
||||
// This struct adapts client::RemoteSettingsClient into the public API
|
||||
internal: client::RemoteSettingsClient,
|
||||
}
|
||||
|
||||
#[uniffi::export]
|
||||
impl RemoteSettingsClient {
|
||||
/// Collection this client is for
|
||||
pub fn collection_name(&self) -> String {
|
||||
self.internal.collection_name().to_owned()
|
||||
}
|
||||
|
||||
/// Get the current set of records.
|
||||
///
|
||||
/// This method normally fetches records from the last sync. This means that it returns fast
|
||||
/// and does not make any network requests.
|
||||
///
|
||||
/// If records have not yet been synced it will return None. Use `sync_if_empty = true` to
|
||||
/// change this behavior and perform a network request in this case. That this is probably a
|
||||
/// bad idea if you want to fetch the setting in application startup or when building the UI.
|
||||
///
|
||||
/// None will also be returned on disk IO errors or other unexpected errors. The reason for
|
||||
/// this is that there is not much an application can do in this situation other than fall back
|
||||
/// to the same default handling as if records have not been synced.
|
||||
///
|
||||
/// TODO(Bug 1919141):
|
||||
///
|
||||
/// Application-services schedules regular dumps of the server data for specific collections.
|
||||
/// For these collections, `get_records` will never return None. If you would like to add your
|
||||
/// collection to this list, please reach out to the DISCO team.
|
||||
#[uniffi::method(default(sync_if_empty = false))]
|
||||
pub fn get_records(&self, sync_if_empty: bool) -> Option<Vec<RemoteSettingsRecord>> {
|
||||
match self.internal.get_records(sync_if_empty) {
|
||||
Ok(records) => records,
|
||||
Err(e) => {
|
||||
// Log/report the error
|
||||
log::trace!("get_records error: {e}");
|
||||
convert_log_report_error(e);
|
||||
// Throw away the converted result and return None, there's nothing a client can
|
||||
// really do with an error except treat it as the None case
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Get the current set of records as a map of record_id -> record.
|
||||
///
|
||||
/// See [Self::get_records] for an explanation of when this makes network requests, error
|
||||
/// handling, and how the `sync_if_empty` param works.
|
||||
#[uniffi::method(default(sync_if_empty = false))]
|
||||
pub fn get_records_map(
|
||||
&self,
|
||||
sync_if_empty: bool,
|
||||
) -> Option<HashMap<String, RemoteSettingsRecord>> {
|
||||
self.get_records(sync_if_empty)
|
||||
.map(|records| records.into_iter().map(|r| (r.id.clone(), r)).collect())
|
||||
}
|
||||
|
||||
/// Get attachment data for a remote settings record
|
||||
///
|
||||
/// Attachments are large binary blobs used for data that doesn't fit in a normal record. They
|
||||
/// are handled differently than other record data:
|
||||
///
|
||||
/// - Attachments are not downloaded in [RemoteSettingsService::sync]
|
||||
/// - This method will make network requests if the attachment is not cached
|
||||
/// - This method will throw if there is a network or other error when fetching the
|
||||
/// attachment data.
|
||||
#[handle_error(Error)]
|
||||
pub fn get_attachment(&self, attachment_id: String) -> ApiResult<Vec<u8>> {
|
||||
self.internal.get_attachment(&attachment_id)
|
||||
}
|
||||
}
|
||||
|
||||
impl RemoteSettingsClient {
|
||||
/// Create a new client. This is not exposed to foreign code, consumers need to call
|
||||
/// [RemoteSettingsService::make_client]
|
||||
fn new(
|
||||
base_url: Url,
|
||||
bucket_name: String,
|
||||
collection_name: String,
|
||||
storage: Storage,
|
||||
) -> Result<Self> {
|
||||
Ok(Self {
|
||||
internal: client::RemoteSettingsClient::new(
|
||||
base_url,
|
||||
bucket_name,
|
||||
collection_name,
|
||||
storage,
|
||||
)?,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(uniffi::Object)]
|
||||
pub struct RemoteSettings {
|
||||
pub config: RemoteSettingsConfig,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,126 @@
|
|||
/* This Source Code Form is subject to the terms of the Mozilla Public
|
||||
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
|
||||
|
||||
use std::{
|
||||
collections::HashSet,
|
||||
sync::{Arc, Weak},
|
||||
};
|
||||
|
||||
use camino::Utf8PathBuf;
|
||||
use parking_lot::Mutex;
|
||||
use url::Url;
|
||||
|
||||
use crate::{
|
||||
storage::Storage, RemoteSettingsClient, RemoteSettingsConfig2, RemoteSettingsServer, Result,
|
||||
};
|
||||
|
||||
/// Internal Remote settings service API
|
||||
pub struct RemoteSettingsService {
|
||||
inner: Mutex<RemoteSettingsServiceInner>,
|
||||
}
|
||||
|
||||
struct RemoteSettingsServiceInner {
|
||||
storage_dir: Utf8PathBuf,
|
||||
base_url: Url,
|
||||
bucket_name: String,
|
||||
/// Weakrefs for all clients that we've created. Note: this stores the
|
||||
/// top-level/public `RemoteSettingsClient` structs rather than `client::RemoteSettingsClient`.
|
||||
/// The reason for this is that we return Arcs to the public struct to the foreign code, so we
|
||||
/// need to use the same type for our weakrefs. The alternative would be to create 2 Arcs for
|
||||
/// each client, which is wasteful.
|
||||
clients: Vec<Weak<RemoteSettingsClient>>,
|
||||
}
|
||||
|
||||
impl RemoteSettingsService {
|
||||
/// Construct a [RemoteSettingsService]
|
||||
///
|
||||
/// This is typically done early in the application-startup process
|
||||
pub fn new(storage_dir: String, config: RemoteSettingsConfig2) -> Result<Self> {
|
||||
let storage_dir = storage_dir.into();
|
||||
let base_url = config
|
||||
.server
|
||||
.unwrap_or(RemoteSettingsServer::Prod)
|
||||
.get_url()?;
|
||||
let bucket_name = config.bucket_name.unwrap_or_else(|| String::from("main"));
|
||||
|
||||
Ok(Self {
|
||||
inner: Mutex::new(RemoteSettingsServiceInner {
|
||||
storage_dir,
|
||||
base_url,
|
||||
bucket_name,
|
||||
clients: vec![],
|
||||
}),
|
||||
})
|
||||
}
|
||||
|
||||
/// Create a new Remote Settings client
|
||||
pub fn make_client(&self, collection_name: String) -> Result<Arc<RemoteSettingsClient>> {
|
||||
let mut inner = self.inner.lock();
|
||||
let storage = Storage::new(inner.storage_dir.join(format!("{collection_name}.sql")))?;
|
||||
let client = Arc::new(RemoteSettingsClient::new(
|
||||
inner.base_url.clone(),
|
||||
inner.bucket_name.clone(),
|
||||
collection_name.clone(),
|
||||
storage,
|
||||
)?);
|
||||
inner.clients.push(Arc::downgrade(&client));
|
||||
Ok(client)
|
||||
}
|
||||
|
||||
/// Sync collections for all active clients
|
||||
pub fn sync(&self) -> Result<Vec<String>> {
|
||||
// Make sure we only sync each collection once, even if there are multiple clients
|
||||
let mut synced_collections = HashSet::new();
|
||||
|
||||
// TODO: poll the server using `/buckets/monitor/collections/changes/changeset` to fetch
|
||||
// the current timestamp for all collections. That way we can avoid fetching collections
|
||||
// we know haven't changed and also pass the `?_expected{ts}` param to the server.
|
||||
|
||||
for client in self.inner.lock().active_clients() {
|
||||
if synced_collections.insert(client.collection_name()) {
|
||||
client.internal.sync()?;
|
||||
}
|
||||
}
|
||||
Ok(synced_collections.into_iter().collect())
|
||||
}
|
||||
|
||||
/// Update the remote settings config
|
||||
///
|
||||
/// This will cause all current and future clients to use new config and will delete any stored
|
||||
/// records causing the clients to return new results from the new config.
|
||||
pub fn update_config(&self, config: RemoteSettingsConfig2) -> Result<()> {
|
||||
let base_url = config
|
||||
.server
|
||||
.unwrap_or(RemoteSettingsServer::Prod)
|
||||
.get_url()?;
|
||||
let bucket_name = config.bucket_name.unwrap_or_else(|| String::from("main"));
|
||||
let mut inner = self.inner.lock();
|
||||
for client in inner.active_clients() {
|
||||
client
|
||||
.internal
|
||||
.update_config(base_url.clone(), bucket_name.clone())?;
|
||||
}
|
||||
inner.base_url = base_url;
|
||||
inner.bucket_name = bucket_name;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl RemoteSettingsServiceInner {
|
||||
// Find live clients in self.clients
|
||||
//
|
||||
// Also, drop dead weakrefs from the vec
|
||||
fn active_clients(&mut self) -> Vec<Arc<RemoteSettingsClient>> {
|
||||
let mut active_clients = vec![];
|
||||
self.clients.retain(|weak| {
|
||||
if let Some(client) = weak.upgrade() {
|
||||
active_clients.push(client);
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
});
|
||||
active_clients
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,98 @@
|
|||
/* This Source Code Form is subject to the terms of the Mozilla Public
|
||||
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
|
||||
|
||||
use camino::Utf8PathBuf;
|
||||
|
||||
use crate::{Attachment, RemoteSettingsRecord, Result};
|
||||
|
||||
/// Internal storage type
|
||||
///
|
||||
/// This will store downloaded records/attachments in a SQLite database. Nothing is implemented
|
||||
/// yet other than the initial API.
|
||||
///
|
||||
/// Most methods input a `collection_url` parameter, is a URL that includes the remote settings
|
||||
/// server, bucket, and collection. If the `collection_url` for a get method does not match the one
|
||||
/// for a set method, then this means the application has switched their remote settings config and
|
||||
/// [Storage] should pretend like nothing is stored in the database.
|
||||
///
|
||||
/// The reason for this is the [crate::RemoteSettingsService::update_config] method. If a consumer
|
||||
/// passes a new server or bucket to `update_config`, we don't want to be using cached data from
|
||||
/// the previous config.
|
||||
///
|
||||
/// Notes:
|
||||
/// - I'm thinking we'll create a separate SQLite database per collection. That reduces
|
||||
/// contention when multiple clients try to get records at once.
|
||||
/// - Still, there might be contention if there are multiple clients for the same collection, or
|
||||
/// if RemoteSettingsService::sync() and RemoteSettingsClient::get_records(true) are called at
|
||||
/// the same time. Maybe we should create a single write connection and put it behind a mutex
|
||||
/// to avoid the possibility of SQLITE_BUSY. Or maybe not, the writes seem like they should be
|
||||
/// very fast.
|
||||
/// - Maybe we should refactor this to use the DAO pattern like suggest does.
|
||||
pub struct Storage {}
|
||||
|
||||
impl Storage {
|
||||
pub fn new(_path: Utf8PathBuf) -> Result<Self> {
|
||||
Ok(Self {})
|
||||
}
|
||||
|
||||
/// Get the last modified timestamp for the stored records
|
||||
///
|
||||
/// Returns None if no records are stored or if `collection_url` does not match the
|
||||
/// `collection_url` passed to `set_records`.
|
||||
pub fn get_last_modified_timestamp(&self, _collection_url: &str) -> Result<Option<u64>> {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
/// Get cached records for this collection
|
||||
///
|
||||
/// Returns None if no records are stored or if `collection_url` does not match the
|
||||
/// `collection_url` passed to `set_records`.
|
||||
pub fn get_records(&self, _collection_url: &str) -> Result<Option<Vec<RemoteSettingsRecord>>> {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
/// Get cached attachment data
|
||||
///
|
||||
/// This returns the last attachment data sent to [Self::set_attachment].
|
||||
///
|
||||
/// Returns None if no attachment data is stored or if `collection_url` does not match the `collection_url`
|
||||
/// passed to `set_attachment`.
|
||||
pub fn get_attachment(
|
||||
&self,
|
||||
_collection_url: &str,
|
||||
_attachment_id: &str,
|
||||
) -> Result<Option<Attachment>> {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
/// Set the list of records stored in the database, clearing out any previously stored records
|
||||
pub fn set_records(
|
||||
&self,
|
||||
_collection_url: &str,
|
||||
records: &[RemoteSettingsRecord],
|
||||
) -> Result<()> {
|
||||
for record in records {
|
||||
println!("Should store record: {record:?}");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Set the attachment data stored in the database, clearing out any previously stored data
|
||||
pub fn set_attachment(
|
||||
&self,
|
||||
_collection_url: &str,
|
||||
attachment_id: &str,
|
||||
_attachment: Attachment,
|
||||
) -> Result<()> {
|
||||
println!("Should store attachment: {attachment_id}");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Empty out all cached values and start from scratch. This is called when
|
||||
/// RemoteSettingsService::update_config() is called, since that could change the remote
|
||||
/// settings server which would invalidate all cached data.
|
||||
pub fn empty(&self) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
[package]
|
||||
name = "examples-remote-settings-cli"
|
||||
version = "0.1.0"
|
||||
license = "MPL-2.0"
|
||||
edition = "2021"
|
||||
publish = false
|
||||
|
||||
[dependencies]
|
||||
remote_settings = { path = "../../components/remote_settings" }
|
||||
viaduct-reqwest = { path = "../../components/support/viaduct-reqwest" }
|
||||
log = "0.4"
|
||||
clap = {version = "4.2", features = ["derive"]}
|
||||
anyhow = "1.0"
|
||||
env_logger = { version = "0.10", default-features = false, features = ["humantime"] }
|
||||
|
|
@ -0,0 +1,113 @@
|
|||
/* This Source Code Form is subject to the terms of the Mozilla Public
|
||||
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
|
||||
|
||||
use anyhow::Result;
|
||||
use clap::{Parser, Subcommand, ValueEnum};
|
||||
|
||||
use remote_settings::{RemoteSettingsConfig2, RemoteSettingsServer, RemoteSettingsService};
|
||||
|
||||
const DEFAULT_LOG_FILTER: &str = "remote_settings=info";
|
||||
const DEFAULT_LOG_FILTER_VERBOSE: &str = "remote_settings=trace";
|
||||
|
||||
#[derive(Debug, Parser)]
|
||||
#[command(about, long_about = None)]
|
||||
struct Cli {
|
||||
#[arg(short = 's')]
|
||||
server: Option<RemoteSettingsServerArg>,
|
||||
#[arg(short = 'b')]
|
||||
bucket: Option<String>,
|
||||
#[arg(short = 'd')]
|
||||
storage_dir: Option<String>,
|
||||
#[arg(long, short, action)]
|
||||
verbose: bool,
|
||||
#[command(subcommand)]
|
||||
command: Commands,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, ValueEnum)]
|
||||
enum RemoteSettingsServerArg {
|
||||
Prod,
|
||||
Stage,
|
||||
Dev,
|
||||
}
|
||||
|
||||
#[derive(Debug, Subcommand)]
|
||||
enum Commands {
|
||||
/// Sync collections
|
||||
Sync {
|
||||
#[clap(required = true)]
|
||||
collections: Vec<String>,
|
||||
},
|
||||
/// Query against ingested data
|
||||
Get {
|
||||
collection: String,
|
||||
#[arg(long)]
|
||||
sync_if_empty: bool,
|
||||
},
|
||||
}
|
||||
|
||||
fn main() -> Result<()> {
|
||||
let cli = Cli::parse();
|
||||
env_logger::init_from_env(env_logger::Env::default().filter_or(
|
||||
"RUST_LOG",
|
||||
if cli.verbose {
|
||||
DEFAULT_LOG_FILTER_VERBOSE
|
||||
} else {
|
||||
DEFAULT_LOG_FILTER
|
||||
},
|
||||
));
|
||||
viaduct_reqwest::use_reqwest_backend();
|
||||
let service = build_service(&cli)?;
|
||||
match cli.command {
|
||||
Commands::Sync { collections } => sync(service, collections),
|
||||
Commands::Get {
|
||||
collection,
|
||||
sync_if_empty,
|
||||
} => get_records(service, collection, sync_if_empty),
|
||||
}
|
||||
}
|
||||
|
||||
fn build_service(cli: &Cli) -> Result<RemoteSettingsService> {
|
||||
let config = RemoteSettingsConfig2 {
|
||||
server: cli.server.as_ref().map(|s| match s {
|
||||
RemoteSettingsServerArg::Dev => RemoteSettingsServer::Dev,
|
||||
RemoteSettingsServerArg::Stage => RemoteSettingsServer::Stage,
|
||||
RemoteSettingsServerArg::Prod => RemoteSettingsServer::Prod,
|
||||
}),
|
||||
bucket_name: cli.bucket.clone(),
|
||||
};
|
||||
Ok(RemoteSettingsService::new(
|
||||
cli.storage_dir
|
||||
.clone()
|
||||
.unwrap_or_else(|| "remote-settings-data".into()),
|
||||
config,
|
||||
)?)
|
||||
}
|
||||
|
||||
fn sync(service: RemoteSettingsService, collections: Vec<String>) -> Result<()> {
|
||||
// Create a bunch of clients so that sync() syncs their collections
|
||||
let _clients = collections
|
||||
.into_iter()
|
||||
.map(|collection| Ok(service.make_client(collection)?))
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
service.sync()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn get_records(
|
||||
service: RemoteSettingsService,
|
||||
collection: String,
|
||||
sync_if_empty: bool,
|
||||
) -> Result<()> {
|
||||
let client = service.make_client(collection)?;
|
||||
match client.get_records(sync_if_empty) {
|
||||
Some(records) => {
|
||||
for record in records {
|
||||
println!("{record:?}");
|
||||
}
|
||||
}
|
||||
None => println!("No cached records"),
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
Загрузка…
Ссылка в новой задаче