feat: add jexl to remote-settings component

Cargo.lock

@@ -2121,9 +2121,9 @@ checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b"
 
 [[package]]
 name = "jexl-eval"
-version = "0.2.2"
+version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "41bb0cb58f20e39d58869c2cf1a69b88aaa7854557904ed404e829d3b64e1046"
+checksum = "fdd8dfc8744f1f59d47f7f3bc1378047ecc15fef5709942fbcc8d0d9f846e506"
 dependencies = [
  "anyhow",
  "jexl-parser",
@@ -2134,9 +2134,9 @@ dependencies = [
 
 [[package]]
 name = "jexl-parser"
-version = "0.2.2"
+version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0ab43a7eb505d3871a8debbaebc35b3402e26968dd4bcac258e6f40881238673"
+checksum = "07cc5fb813f07eceed953a76345a8af76038ee4101c32dc3740e040595013a84"
 dependencies = [
  "lalrpop-util",
  "regex",
@@ -3654,13 +3654,17 @@ dependencies = [
 name = "remote_settings"
 version = "0.1.0"
 dependencies = [
+ "anyhow",
  "camino",
  "error-support",
  "expect-test",
+ "firefox-versioning",
+ "jexl-eval",
  "log",
  "mockall",
  "mockito",
  "parking_lot",
+ "regex",
  "rusqlite",
  "serde",
  "serde_json",

components/nimbus/Cargo.toml

@@ -28,7 +28,7 @@ log = "0.4"
 thiserror = "1"
 url = "2.5"
 rkv = { version = "0.19", optional = true }
-jexl-eval = "0.2.2"
+jexl-eval = "0.3.0"
 uuid = { version = "1.7", features = ["serde", "v4"]}
 sha2 = "^0.10"
 hex = "0.4"

components/remote_settings/Cargo.toml

@@ -22,6 +22,10 @@ viaduct = { path = "../viaduct" }
 url = "2.1" # mozilla-central can't yet take 2.2 (see bug 1734538)
 camino = "1.0"
 rusqlite = { workspace = true, features = ["bundled"] }
+jexl-eval = "0.3.0"
+regex = "1.10.5"
+anyhow = "1.0"
+firefox-versioning = { path = "../support/firefox-versioning" }
 
 [build-dependencies]
 uniffi = { workspace = true, features = ["build"] }

components/remote_settings/src/cache.rs

@@ -143,7 +143,7 @@ mod test {
                         last_modified: 1300,
                         deleted: false,
                         attachment: None,
-                        fields: fields("d")
+                        fields: fields("d"),
                     },
                 ],
             }

components/remote_settings/src/client.rs

@@ -4,8 +4,9 @@
 
 use crate::config::RemoteSettingsConfig;
 use crate::error::{Error, Result};
+use crate::jexl_filter::JexlFilter;
 use crate::storage::Storage;
-use crate::{RemoteSettingsServer, UniffiCustomTypeConverter};
+use crate::{RemoteSettingsContext, RemoteSettingsServer, UniffiCustomTypeConverter};
 use parking_lot::Mutex;
 use serde::{Deserialize, Serialize};
 use std::{
@@ -26,6 +27,7 @@ const HEADER_RETRY_AFTER: &str = "Retry-After";
 pub struct RemoteSettingsClient<C = ViaductApiClient> {
     // This is immutable, so it can be outside the mutex
     collection_name: String,
+    jexl_filter: JexlFilter,
     inner: Mutex<RemoteSettingsClientInner<C>>,
 }
 
@@ -35,9 +37,15 @@ struct RemoteSettingsClientInner<C> {
 }
 
 impl<C: ApiClient> RemoteSettingsClient<C> {
-    pub fn new_from_parts(collection_name: String, storage: Storage, api_client: C) -> Self {
+    pub fn new_from_parts(
+        collection_name: String,
+        storage: Storage,
+        jexl_filter: JexlFilter,
+        api_client: C,
+    ) -> Self {
         Self {
             collection_name,
+            jexl_filter,
             inner: Mutex::new(RemoteSettingsClientInner {
                 storage,
                 api_client,
@@ -48,6 +56,19 @@ impl<C: ApiClient> RemoteSettingsClient<C> {
         &self.collection_name
     }
 
+    /// Filters records based on the presence and evaluation of `filter_expression`.
+    fn filter_records(&self, records: Vec<RemoteSettingsRecord>) -> Vec<RemoteSettingsRecord> {
+        records
+            .into_iter()
+            .filter(|record| match record.fields.get("filter_expression") {
+                Some(serde_json::Value::String(filter_expr)) => {
+                    self.jexl_filter.evaluate(filter_expr).unwrap_or(false)
+                }
+                _ => true, // Include records without a valid filter expression by default
+            })
+            .collect()
+    }
+
     /// Get the current set of records.
     ///
     /// If records are not present in storage this will normally return None.  Use `sync_if_empty =
@@ -56,14 +77,29 @@ impl<C: ApiClient> RemoteSettingsClient<C> {
         let mut inner = self.inner.lock();
         let collection_url = inner.api_client.collection_url();
 
+        // Try to retrieve and filter cached records first
         let cached_records = inner.storage.get_records(&collection_url)?;
-        if cached_records.is_some() || !sync_if_empty {
-            return Ok(cached_records);
-        }
 
-        let records = inner.api_client.get_records(None)?;
-        inner.storage.set_records(&collection_url, &records)?;
-        Ok(Some(records))
+        match cached_records {
+            Some(records) if !records.is_empty() || !sync_if_empty => {
+                // Filter and return cached records if they're present or if we don't need to sync
+                let filtered_records = self.filter_records(records);
+                Ok(Some(filtered_records))
+            }
+            None if !sync_if_empty => {
+                // No cached records and sync_if_empty is false, so we return None
+                Ok(None)
+            }
+            _ => {
+                // Fetch new records if no cached records or if sync is required
+                let records = inner.api_client.get_records(None)?;
+                inner.storage.set_records(&collection_url, &records)?;
+
+                // Apply filtering to the newly fetched records
+                let filtered_records = self.filter_records(records);
+                Ok(Some(filtered_records))
+            }
+        }
     }
 
     pub fn sync(&self) -> Result<()> {
@@ -89,10 +125,18 @@ impl RemoteSettingsClient<ViaductApiClient> {
         server_url: Url,
         bucket_name: String,
         collection_name: String,
+        context: Option<RemoteSettingsContext>,
         storage: Storage,
     ) -> Result<Self> {
         let api_client = ViaductApiClient::new(server_url, &bucket_name, &collection_name)?;
-        Ok(Self::new_from_parts(collection_name, storage, api_client))
+        let jexl_filter = JexlFilter::new(context);
+
+        Ok(Self::new_from_parts(
+            collection_name,
+            storage,
+            jexl_filter,
+            api_client,
+        ))
     }
 
     pub fn update_config(&self, server_url: Url, bucket_name: String) -> Result<()> {
@@ -1414,11 +1458,11 @@ mod test {
       "title": "jpg-attachment",
       "content": "content",
       "attachment": {
-      "filename": "jgp-attachment.jpg",
-      "location": "the-bucket/the-collection/d3a5eccc-f0ca-42c3-b0bb-c0d4408c21c9.jpg",
-      "hash": "2cbd593f3fd5f1585f92265433a6696a863bc98726f03e7222135ff0d8e83543",
-      "mimetype": "image/jpeg",
-      "size": 1374325
+          "filename": "jgp-attachment.jpg",
+          "location": "the-bucket/the-collection/d3a5eccc-f0ca-42c3-b0bb-c0d4408c21c9.jpg",
+          "hash": "2cbd593f3fd5f1585f92265433a6696a863bc98726f03e7222135ff0d8e83543",
+          "mimetype": "image/jpeg",
+          "size": 1374325
       },
       "id": "c5dcd1da-7126-4abb-846b-ec85b0d4d0d7",
       "schema": 1677694447771,
@@ -1500,8 +1544,12 @@ mod test_new_client {
         // Note, don't make any api_client.expect_*() calls, the RemoteSettingsClient should not
         // attempt to make any requests for this scenario
         let storage = Storage::new(":memory:".into()).expect("Error creating storage");
-        let rs_client =
-            RemoteSettingsClient::new_from_parts("test-collection".into(), storage, api_client);
+        let rs_client = RemoteSettingsClient::new_from_parts(
+            "test-collection".into(),
+            storage,
+            JexlFilter::new(None),
+            api_client,
+        );
         assert_eq!(
             rs_client.get_records(false).expect("Error getting records"),
             None
@@ -1529,11 +1577,115 @@ mod test_new_client {
             }
         });
         let storage = Storage::new(":memory:".into()).expect("Error creating storage");
-        let rs_client =
-            RemoteSettingsClient::new_from_parts("test-collection".into(), storage, api_client);
+        let rs_client = RemoteSettingsClient::new_from_parts(
+            "test-collection".into(),
+            storage,
+            JexlFilter::new(None),
+            api_client,
+        );
         assert_eq!(
             rs_client.get_records(true).expect("Error getting records"),
             Some(records)
         );
     }
 }
+
+#[cfg(test)]
+mod test_filtering_records {
+    use super::*;
+    use serde_json::json;
+
+    #[test]
+    fn test_get_records_filtered_app_version_pass() {
+        let mut api_client = MockApiClient::new();
+        let records = vec![RemoteSettingsRecord {
+            id: "record-0001".into(),
+            last_modified: 100,
+            deleted: false,
+            attachment: None,
+            fields: json!({"filter_expression": "app_version|versionCompare('4.0') >= 0"})
+                .as_object()
+                .unwrap()
+                .clone(),
+        }];
+        api_client.expect_collection_url().returning(|| {
+            "http://rs.example.com/v1/buckets/main/collections/test-collection".into()
+        });
+        api_client.expect_get_records().returning({
+            let records = records.clone();
+            move |timestamp| {
+                assert_eq!(timestamp, None);
+                Ok(records.clone())
+            }
+        });
+
+        let context = RemoteSettingsContext {
+            app_version: Some("4.4".to_string()),
+            ..Default::default()
+        };
+
+        let mut storage = Storage::new(":memory:".into()).expect("Error creating storage");
+        let _ = storage.set_records(
+            "http://rs.example.com/v1/buckets/main/collections/test-collection",
+            &records,
+        );
+
+        let rs_client = RemoteSettingsClient::new_from_parts(
+            "test-collection".into(),
+            storage,
+            JexlFilter::new(Some(context)),
+            api_client,
+        );
+        assert_eq!(
+            rs_client.get_records(false).expect("Error getting records"),
+            Some(records)
+        );
+    }
+
+    #[test]
+    fn test_get_records_filtered_app_version_too_low() {
+        let mut api_client = MockApiClient::new();
+        let records = vec![RemoteSettingsRecord {
+            id: "record-0001".into(),
+            last_modified: 100,
+            deleted: false,
+            attachment: None,
+            fields: json!({"filter_expression": "app_version|versionCompare('4.0') >= 0"})
+                .as_object()
+                .unwrap()
+                .clone(),
+        }];
+        api_client.expect_collection_url().returning(|| {
+            "http://rs.example.com/v1/buckets/main/collections/test-collection".into()
+        });
+        api_client.expect_get_records().returning({
+            let records = records.clone();
+            move |timestamp| {
+                assert_eq!(timestamp, None);
+                Ok(records.clone())
+            }
+        });
+
+        let context = RemoteSettingsContext {
+            app_version: Some("3.9".to_string()),
+            ..Default::default()
+        };
+
+        let mut storage = Storage::new(":memory:".into()).expect("Error creating storage");
+        let _ = storage.set_records(
+            "http://rs.example.com/v1/buckets/main/collections/test-collection",
+            &records,
+        );
+
+        let rs_client = RemoteSettingsClient::new_from_parts(
+            "test-collection".into(),
+            storage,
+            JexlFilter::new(Some(context)),
+            api_client,
+        );
+        assert_eq!(
+            rs_client.get_records(false).expect("Error getting records"),
+            Some(vec![])
+        );
+    }
+}

components/remote_settings/src/jexl_filter.rs

@@ -0,0 +1,60 @@
+use crate::RemoteSettingsContext;
+use firefox_versioning::compare::version_compare;
+use jexl_eval::Evaluator;
+use serde_json::{json, Value};
+use thiserror::Error;
+
+#[derive(Error, Debug)]
+pub(crate) enum ParseError {
+    #[error("Evaluation error: {0}")]
+    EvaluationError(String),
+    #[error("Invalid result type")]
+    InvalidResultType,
+}
+
+/// The JEXL filter is getting instatiated when a new `RemoteSettingsClient` is being created.
+pub struct JexlFilter {
+    /// a JEXL `Evaluator` to run transforms and evaluations on.
+    evaluator: Evaluator<'static>,
+    /// The transformed `RemoteSettingsContext` as a `serde_json::Value`
+    context: Value,
+}
+
+impl JexlFilter {
+    /// Creating a new `JEXL` filter. If no `context` is set, all future `records` are being
+    /// evaluated as `true` by default.
+    pub(crate) fn new(context: Option<RemoteSettingsContext>) -> Self {
+        let env_context = match context {
+            Some(ctx) => {
+                serde_json::to_value(ctx).expect("Failed to serialize RemoteSettingsContext")
+            }
+            None => json!({}),
+        };
+
+        Self {
+            evaluator: Evaluator::new()
+                // We want to add more transforms later on. We started with `versionCompare`.
+                // https://remote-settings.readthedocs.io/en/latest/target-filters.html#transforms
+                // The goal is to get on pare with the desktop.
+                .with_transform("versionCompare", |args| Ok(version_compare(args)?)),
+            context: env_context,
+        }
+    }
+
+    /// Evaluates the given filter expression in the provided context.
+    /// Returns `Ok(true)` if the expression evaluates to true, `Ok(false)` otherwise.
+    pub(crate) fn evaluate(&self, filter_expr: &str) -> Result<bool, ParseError> {
+        if filter_expr.trim().is_empty() {
+            return Ok(true);
+        }
+
+        let result = self
+            .evaluator
+            .eval_in_context(filter_expr, &self.context)
+            .map_err(|e| {
+                ParseError::EvaluationError(format!("Failed to evaluate '{}': {}", filter_expr, e))
+            })?;
+
+        result.as_bool().ok_or(ParseError::InvalidResultType)
+    }
+}

components/remote_settings/src/lib.rs

@@ -2,6 +2,8 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
+use serde::{Deserialize, Serialize};
+use serde_json::{Map, Value};
 use std::{collections::HashMap, fs::File, io::prelude::Write, sync::Arc};
 
 use error_support::{convert_log_report_error, handle_error};
@@ -14,6 +16,8 @@ pub mod error;
 pub mod service;
 pub mod storage;
 
+pub(crate) mod jexl_filter;
+
 pub use client::{Attachment, RemoteSettingsRecord, RemoteSettingsResponse, RsJsonObject};
 pub use config::{RemoteSettingsConfig, RemoteSettingsConfig2, RemoteSettingsServer};
 pub use error::{ApiResult, RemoteSettingsError, Result};
@@ -24,6 +28,50 @@ use storage::Storage;
 
 uniffi::setup_scaffolding!("remote_settings");
 
+/// The `RemoteSettingsContext` object represents the parameters and characteristics of the
+/// consuming application. For `remote-settings`, it is used to filter locally stored `records`.
+///
+/// We always fetch all `records` from the remote-settings storage. Some records could have a `filter_expression`
+/// attached to them, which will be matched against the `RemoteSettingsContext`.
+///
+/// When set, only records where the expression is true will be returned.
+#[derive(Deserialize, Serialize, Debug, Clone, Default, uniffi::Record)]
+pub struct RemoteSettingsContext {
+    /// Name of the application (e.g. "Fenix" or "Firefox iOS")
+    pub app_name: String,
+    /// Application identifier, especially for mobile (e.g. "org.mozilla.fenix")
+    pub app_id: String,
+    /// The delivery channel of the application (e.g "nightly")
+    pub channel: String,
+    /// User visible version string (e.g. "1.0.3")
+    pub app_version: Option<String>,
+    /// Build identifier generated by the CI system (e.g. "1234/A")
+    pub app_build: Option<String>,
+    /// The architecture of the device, (e.g. "arm", "x86")
+    pub architecture: Option<String>,
+    /// The manufacturer of the device the application is running on
+    pub device_manufacturer: Option<String>,
+    /// The model of the device the application is running on
+    pub device_model: Option<String>,
+    /// The locale of the application during initialization (e.g. "es-ES")
+    pub locale: Option<String>,
+    /// The name of the operating system (e.g. "Android", "iOS", "Darwin", "Windows")
+    pub os: Option<String>,
+    /// The user-visible version of the operating system (e.g. "1.2.3")
+    pub os_version: Option<String>,
+    /// Android specific for targeting specific sdk versions
+    pub android_sdk_version: Option<String>,
+    /// Used for debug purposes as a way to match only developer builds, etc.
+    pub debug_tag: Option<String>,
+    /// The date the application installed the app
+    pub installation_date: Option<i64>,
+    /// The application's home directory
+    pub home_directory: Option<String>,
+    /// Contains attributes specific to the application, derived by the application
+    #[serde(flatten)]
+    pub custom_targeting_attributes: Option<Map<String, Value>>,
+}
+
 /// Application-level Remote Settings manager.
 ///
 /// This handles application-level operations, like syncing all the collections, and acts as a
@@ -49,8 +97,12 @@ impl RemoteSettingsService {
 
     /// Create a new Remote Settings client
     #[handle_error(Error)]
-    pub fn make_client(&self, collection_name: String) -> ApiResult<Arc<RemoteSettingsClient>> {
-        self.internal.make_client(collection_name)
+    pub fn make_client(
+        &self,
+        collection_name: String,
+        app_context: Option<RemoteSettingsContext>,
+    ) -> ApiResult<Arc<RemoteSettingsClient>> {
+        self.internal.make_client(collection_name, app_context)
     }
 
     /// Sync collections for all active clients
@@ -156,6 +208,7 @@ impl RemoteSettingsClient {
         base_url: Url,
         bucket_name: String,
         collection_name: String,
+        context: Option<RemoteSettingsContext>,
         storage: Storage,
     ) -> Result<Self> {
         Ok(Self {
@@ -163,6 +216,7 @@ impl RemoteSettingsClient {
                 base_url,
                 bucket_name,
                 collection_name,
+                context,
                 storage,
             )?,
         })

components/remote_settings/src/service.rs

@@ -12,7 +12,8 @@ use parking_lot::Mutex;
 use url::Url;
 
 use crate::{
-    storage::Storage, RemoteSettingsClient, RemoteSettingsConfig2, RemoteSettingsServer, Result,
+    storage::Storage, RemoteSettingsClient, RemoteSettingsConfig2, RemoteSettingsContext,
+    RemoteSettingsServer, Result,
 };
 
 /// Internal Remote settings service API
@@ -55,13 +56,18 @@ impl RemoteSettingsService {
     }
 
     /// Create a new Remote Settings client
-    pub fn make_client(&self, collection_name: String) -> Result<Arc<RemoteSettingsClient>> {
+    pub fn make_client(
+        &self,
+        collection_name: String,
+        context: Option<RemoteSettingsContext>,
+    ) -> Result<Arc<RemoteSettingsClient>> {
         let mut inner = self.inner.lock();
         let storage = Storage::new(inner.storage_dir.join(format!("{collection_name}.sql")))?;
         let client = Arc::new(RemoteSettingsClient::new(
             inner.base_url.clone(),
             inner.bucket_name.clone(),
             collection_name.clone(),
+            context,
             storage,
         )?);
         inner.clients.push(Arc::downgrade(&client));

components/remote_settings/src/storage.rs

@@ -2,12 +2,11 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
+use crate::{Attachment, RemoteSettingsRecord, Result};
 use camino::Utf8PathBuf;
 use rusqlite::{params, Connection, OptionalExtension};
 use serde_json;
 
-use crate::{Attachment, RemoteSettingsRecord, Result};
-
 /// Internal storage type
 ///
 /// This will store downloaded records/attachments in a SQLite database.  Nothing is implemented
@@ -82,7 +81,6 @@ impl Storage {
     ) -> Result<Option<Vec<RemoteSettingsRecord>>> {
         let tx = self.conn.transaction()?;
 
-        // Check if the collection has been fetched before
         let fetched: Option<bool> = tx
             .prepare("SELECT fetched FROM collection_metadata WHERE collection_url = ?")?
             .query_row(params![collection_url], |row| row.get(0))
@@ -151,6 +149,7 @@ impl Storage {
             for record in records {
                 max_last_modified = max_last_modified.max(record.last_modified);
                 let data = serde_json::to_vec(record)?;
+
                 stmt.execute(params![record.id, collection_url, data])?;
             }
         }
@@ -163,6 +162,7 @@ impl Storage {
         )?;
 
         tx.commit()?;
+
         Ok(())
     }
 
@@ -497,21 +497,29 @@ mod tests {
 
         // Set records for collection_url1
         storage.set_records(collection_url1, &records_collection_url1)?;
-        // Set records for collection_url2
+
+        // Verify records for collection_url1
+        let fetched_records = storage.get_records(collection_url1)?;
+        assert!(fetched_records.is_some());
+        let fetched_records = fetched_records.unwrap();
+        assert_eq!(fetched_records.len(), 1);
+        assert_eq!(fetched_records, records_collection_url1);
+
+        // Set records for collection_url2, which will clear records for all collections
         storage.set_records(collection_url2, &records_collection_url2)?;
 
-        // We should delete all records for url1
+        // Verify that records for collection_url1 have been cleared
         let fetched_records = storage.get_records(collection_url1)?;
         assert!(fetched_records.is_none());
 
-        // Get records for collection_url2
+        // Verify records for collection_url2 are correctly stored
         let fetched_records = storage.get_records(collection_url2)?;
         assert!(fetched_records.is_some());
         let fetched_records = fetched_records.unwrap();
         assert_eq!(fetched_records.len(), 1);
         assert_eq!(fetched_records, records_collection_url2);
 
-        // Get last modified timestamps
+        // Verify last modified timestamps only for collection_url2
         let last_modified1 = storage.get_last_modified_timestamp(collection_url1)?;
         assert_eq!(last_modified1, None);
         let last_modified2 = storage.get_last_modified_timestamp(collection_url2)?;
@@ -550,7 +558,7 @@ mod tests {
             last_modified: 200,
             deleted: false,
             attachment: None,
-            fields: serde_json::json!({"key": "value2"})
+            fields: serde_json::json!({"key": "value2_updated"})
                 .as_object()
                 .unwrap()
                 .clone(),

examples/remote-settings-cli/src/main.rs

@@ -89,7 +89,7 @@ fn sync(service: RemoteSettingsService, collections: Vec<String>) -> Result<()>
     // Create a bunch of clients so that sync() syncs their collections
     let _clients = collections
         .into_iter()
-        .map(|collection| Ok(service.make_client(collection)?))
+        .map(|collection| Ok(service.make_client(collection, None)?))
         .collect::<Result<Vec<_>>>()?;
     service.sync()?;
     Ok(())
@@ -100,7 +100,7 @@ fn get_records(
     collection: String,
     sync_if_empty: bool,
 ) -> Result<()> {
-    let client = service.make_client(collection)?;
+    let client = service.make_client(collection, None)?;
     match client.get_records(sync_if_empty) {
         Some(records) => {
             for record in records {