From 2c2bf6197774ff0bd592e48e3335a7c3c5eabab3 Mon Sep 17 00:00:00 2001 From: Chris H-C Date: Mon, 11 Mar 2024 18:46:52 +0000 Subject: [PATCH] Bug 1862002 - Prototype OHTTP in FOG r=wstuckey,TravisLong Differential Revision: https://phabricator.services.mozilla.com/D203529 --- Cargo.lock | 4 + toolkit/components/glean/Cargo.toml | 4 + toolkit/components/glean/api/src/pings.rs | 2 +- .../glean_parser_ext/run_glean_parser.py | 27 +++ .../glean_parser_ext/templates/ohttp.jinja2 | 13 ++ toolkit/components/glean/moz.build | 11 + .../glean/src/init/viaduct_uploader.rs | 193 ++++++++++++++---- toolkit/components/glean/src/lib.rs | 1 + toolkit/components/glean/src/ohttp_pings.rs | 13 ++ .../glean/tests/pytest/jogfile_output | 8 + .../glean/tests/pytest/pings_test.yaml | 18 +- .../glean/tests/pytest/pings_test_output | 15 ++ .../glean/tests/pytest/pings_test_output_cpp | 7 + .../tests/pytest/pings_test_output_js_cpp | 10 +- .../glean/tests/pytest/pings_test_output_js_h | 2 +- .../components/glean/tests/test_pings.yaml | 19 ++ .../components/glean/tests/xpcshell/head.js | 152 ++++++++++++++ .../glean/tests/xpcshell/test_OHTTP.js | 32 +++ .../glean/tests/xpcshell/xpcshell.toml | 3 + 19 files changed, 492 insertions(+), 42 deletions(-) create mode 100644 toolkit/components/glean/build_scripts/glean_parser_ext/templates/ohttp.jinja2 create mode 100644 toolkit/components/glean/src/ohttp_pings.rs create mode 100644 toolkit/components/glean/tests/xpcshell/test_OHTTP.js diff --git a/Cargo.lock b/Cargo.lock index c03da8107e1d..4133a93b0e48 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1852,15 +1852,19 @@ dependencies = [ name = "fog_control" version = "0.1.0" dependencies = [ + "bhttp", "cstr", "firefox-on-glean", "glean", "log", + "mozbuild", "nserror", "nsstring", + "ohttp", "once_cell", "static_prefs", "thin-vec", + "thiserror", "url", "viaduct", "xpcom", diff --git a/toolkit/components/glean/Cargo.toml b/toolkit/components/glean/Cargo.toml index 73e82c01e187..640ae17ea7ff 100644 --- a/toolkit/components/glean/Cargo.toml +++ b/toolkit/components/glean/Cargo.toml @@ -18,6 +18,10 @@ cstr = "0.2" viaduct = "0.1" url = "2.1" thin-vec = { version = "0.2.1", features = ["gecko-ffi"] } +ohttp = { version = "0.3", default-features = false, features = ["gecko", "nss", "client"] } +bhttp = "0.3" +thiserror = "1.0" +mozbuild = "0.1" [features] # Leave data collection enabled, but disable upload. diff --git a/toolkit/components/glean/api/src/pings.rs b/toolkit/components/glean/api/src/pings.rs index f1d033269532..21eb3855ee3e 100644 --- a/toolkit/components/glean/api/src/pings.rs +++ b/toolkit/components/glean/api/src/pings.rs @@ -6,7 +6,7 @@ //! //! The contents of this module are generated by //! `toolkit/components/glean/build_scripts/glean_parser_ext/run_glean_parser.py`, from -//! 'toolkit/components/glean/pings.yaml`. +//! ping definitions files identified by `toolkit/components/glean/metrics_index.py`. include!(mozbuild::objdir_path!( "toolkit/components/glean/api/src/pings.rs" diff --git a/toolkit/components/glean/build_scripts/glean_parser_ext/run_glean_parser.py b/toolkit/components/glean/build_scripts/glean_parser_ext/run_glean_parser.py index 1d7d97cf7373..bc9f09f0d3d5 100644 --- a/toolkit/components/glean/build_scripts/glean_parser_ext/run_glean_parser.py +++ b/toolkit/components/glean/build_scripts/glean_parser_ext/run_glean_parser.py @@ -230,5 +230,32 @@ def jog_file(output_fd, *args): return get_deps() +def ohttp_pings(output_fd, *args): + all_objs, options = parse(args) + ohttp_pings = [] + for ping in all_objs["pings"].values(): + if ping.metadata.get("use_ohttp", False): + if ping.include_info_sections: + raise ParserError( + "Cannot send pings with OHTTP that contain {client|ping}_info sections. Specify `metadata: include_info_sections: false`" + ) + ohttp_pings.append(ping.name) + + env = jinja2.Environment( + loader=jinja2.PackageLoader("run_glean_parser", "templates"), + trim_blocks=True, + lstrip_blocks=True, + ) + env.filters["quote_and_join"] = lambda l: "\n| ".join(f'"{x}"' for x in l) + template = env.get_template("ohttp.jinja2") + output_fd.write( + template.render( + ohttp_pings=ohttp_pings, + ) + ) + output_fd.write("\n") + return get_deps() + + if __name__ == "__main__": main(sys.stdout, *sys.argv[1:]) diff --git a/toolkit/components/glean/build_scripts/glean_parser_ext/templates/ohttp.jinja2 b/toolkit/components/glean/build_scripts/glean_parser_ext/templates/ohttp.jinja2 new file mode 100644 index 000000000000..d6e124802170 --- /dev/null +++ b/toolkit/components/glean/build_scripts/glean_parser_ext/templates/ohttp.jinja2 @@ -0,0 +1,13 @@ +// -*- mode: Rust -*- + +// AUTOGENERATED BY glean_parser. DO NOT EDIT. +{# The rendered source is autogenerated, but this +Jinja2 template is not. Please file bugs! #} + +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +pub fn uses_ohttp(ping_name: &str) -> bool { + matches!(ping_name, {{ ohttp_pings|quote_and_join }}) +} diff --git a/toolkit/components/glean/moz.build b/toolkit/components/glean/moz.build index 690a0b8f5258..df76dbcbc4a8 100644 --- a/toolkit/components/glean/moz.build +++ b/toolkit/components/glean/moz.build @@ -199,6 +199,17 @@ if CONFIG["MOZ_ARTIFACT_BUILDS"]: # Once generated, it needs to be placed in GreD so it can be found. FINAL_TARGET_FILES += ["!jogfile.json"] +# OHTTP support requires the fog_control crate know which pings wish to be sent +# using OHTTP. fog_control has no access to the firefox_on_glean crate, so it +# needs its own codegen. +GeneratedFile( + "src/ohttp_pings.rs", + script="build_scripts/glean_parser_ext/run_glean_parser.py", + entry_point="ohttp_pings", + flags=[CONFIG["MOZ_APP_VERSION"]], + inputs=pings_yamls + tags_yamls, +) + DIRS += [ "tests", # Must be in DIRS, not TEST_DIRS or python-test won't find it. "xpcom", diff --git a/toolkit/components/glean/src/init/viaduct_uploader.rs b/toolkit/components/glean/src/init/viaduct_uploader.rs index 6f24e13612d2..648ce2c353d0 100644 --- a/toolkit/components/glean/src/init/viaduct_uploader.rs +++ b/toolkit/components/glean/src/init/viaduct_uploader.rs @@ -3,6 +3,8 @@ // file, You can obtain one at https://mozilla.org/MPL/2.0/. use glean::net::{PingUploadRequest, PingUploader, UploadResult}; +use once_cell::sync::OnceCell; +use std::sync::Once; use url::Url; use viaduct::{Error::*, Request}; @@ -21,53 +23,174 @@ impl PingUploader for ViaductUploader { /// /// * `upload_request` - the ping and its metadata to upload. fn upload(&self, upload_request: PingUploadRequest) -> UploadResult { - let PingUploadRequest { - url, body, headers, .. - } = upload_request; - log::trace!("FOG Ping Uploader uploading to {}", url); - let url_clone = url.clone(); - let result: std::result::Result = (move || { - // SAFETY NOTE: Safe because it returns a primitive by value. - if unsafe { FOG_TooLateToSend() } { - log::trace!("Attempted to send ping too late into shutdown."); - return Ok(UploadResult::done()); - } - let debug_tagged = headers.iter().any(|(name, _)| name == "X-Debug-ID"); - let localhost_port = static_prefs::pref!("telemetry.fog.test.localhost_port"); - if localhost_port < 0 - || (localhost_port == 0 && !debug_tagged && cfg!(feature = "disable_upload")) - { - log::info!("FOG Ping uploader faking success"); - return Ok(UploadResult::http_status(200)); - } - let parsed_url = Url::parse(&url_clone)?; + log::trace!("FOG Ping Uploader uploading to {}", upload_request.url); - log::info!("FOG Ping uploader uploading to {:?}", parsed_url); + // SAFETY NOTE: Safe because it returns a primitive by value. + if unsafe { FOG_TooLateToSend() } { + log::trace!("Attempted to send ping too late into shutdown."); + return UploadResult::done(); + } - let mut req = Request::post(parsed_url.clone()).body(body.clone()); - for (header_key, header_value) in &headers { - req = req.header(header_key.to_owned(), header_value)?; - } + let debug_tagged = upload_request + .headers + .iter() + .any(|(name, _)| name == "X-Debug-ID"); + let localhost_port = static_prefs::pref!("telemetry.fog.test.localhost_port"); + if localhost_port < 0 + || (localhost_port == 0 && !debug_tagged && cfg!(feature = "disable_upload")) + { + log::info!("FOG Ping uploader faking success"); + return UploadResult::http_status(200); + } + + // Localhost-destined pings are sent without OHTTP, + // even if configured to use OHTTP. + let result = if localhost_port == 0 && should_ohttp_upload(&upload_request) { + ohttp_upload(upload_request) + } else { + viaduct_upload(upload_request) + }; - log::trace!("FOG Ping Uploader sending ping to {}", parsed_url); - let res = req.send()?; - Ok(UploadResult::http_status(res.status as i32)) - })(); log::trace!( - "FOG Ping Uploader completed uploading to {} (Result {:?})", - url, + "FOG Ping Uploader completed uploading (Result {:?})", result ); + match result { Ok(result) => result, - Err(NonTlsUrl | UrlError(_)) => UploadResult::unrecoverable_failure(), - Err( + Err(ViaductUploaderError::Viaduct(ve)) => match ve { + NonTlsUrl | UrlError(_) => UploadResult::unrecoverable_failure(), RequestHeaderError(_) | BackendError(_) | NetworkError(_) | BackendNotInitialized - | SetBackendError, - ) => UploadResult::recoverable_failure(), + | SetBackendError => UploadResult::recoverable_failure(), + }, + Err( + ViaductUploaderError::Bhttp(_) + | ViaductUploaderError::Ohttp(_) + | ViaductUploaderError::Fatal, + ) => UploadResult::unrecoverable_failure(), } } } + +fn viaduct_upload(upload_request: PingUploadRequest) -> Result { + let parsed_url = Url::parse(&upload_request.url)?; + + log::info!("FOG viaduct uploader uploading to {:?}", parsed_url); + + let mut req = Request::post(parsed_url.clone()).body(upload_request.body); + for (header_key, header_value) in &upload_request.headers { + req = req.header(header_key.to_owned(), header_value)?; + } + + log::trace!("FOG viaduct uploader sending ping to {:?}", parsed_url); + let res = req.send()?; + Ok(UploadResult::http_status(res.status as i32)) +} + +fn should_ohttp_upload(upload_request: &PingUploadRequest) -> bool { + crate::ohttp_pings::uses_ohttp(&upload_request.ping_name) + && !upload_request.body_has_info_sections +} + +fn ohttp_upload(upload_request: PingUploadRequest) -> Result { + static CELL: OnceCell> = once_cell::sync::OnceCell::new(); + let config = CELL.get_or_try_init(|| get_config())?; + + let binary_request = bhttp_encode(upload_request)?; + + static OHTTP_INIT: Once = Once::new(); + OHTTP_INIT.call_once(|| { + ohttp::init(); + }); + + let ohttp_request = ohttp::ClientRequest::new(config)?; + let (capsule, ohttp_response) = ohttp_request.encapsulate(&binary_request)?; + + const OHTTP_RELAY_URL: &str = "https://mozilla-ohttp-dev.fastly-edge.com/"; + let parsed_relay_url = Url::parse(OHTTP_RELAY_URL)?; + + log::trace!("FOG ohttp uploader uploading to {}", parsed_relay_url); + + const OHTTP_MESSAGE_CONTENT_TYPE: &str = "message/ohttp-req"; + let req = Request::post(parsed_relay_url) + .header( + viaduct::header_names::CONTENT_TYPE, + OHTTP_MESSAGE_CONTENT_TYPE, + )? + .body(capsule); + let res = req.send()?; + + if res.status == 200 { + // This just tells us the HTTP went well. Check OHTTP's status. + let binary_response = ohttp_response.decapsulate(&res.body)?; + let mut cursor = std::io::Cursor::new(binary_response); + let bhttp_message = bhttp::Message::read_bhttp(&mut cursor)?; + let res = bhttp_message + .control() + .status() + .ok_or(ViaductUploaderError::Fatal)?; + Ok(UploadResult::http_status(res as i32)) + } else { + Ok(UploadResult::http_status(res.status as i32)) + } +} + +fn get_config() -> Result, ViaductUploaderError> { + const OHTTP_CONFIG_URL: &str = + "https://stage.ohttp-gateway.nonprod.webservices.mozgcp.net/ohttp-configs"; + log::trace!("Getting OHTTP config from {}", OHTTP_CONFIG_URL); + let parsed_config_url = Url::parse(OHTTP_CONFIG_URL)?; + Ok(Request::get(parsed_config_url).send()?.body) +} + +/// Encode the ping upload request in binary HTTP. +/// (draft-ietf-httpbis-binary-message) +fn bhttp_encode(upload_request: PingUploadRequest) -> Result, ViaductUploaderError> { + let parsed_url = Url::parse(&upload_request.url)?; + let mut message = bhttp::Message::request( + "POST".into(), + parsed_url.scheme().into(), + parsed_url + .host_str() + .ok_or(ViaductUploaderError::Fatal)? + .into(), + parsed_url.path().into(), + ); + + upload_request + .headers + .into_iter() + .for_each(|(k, v)| message.put_header(k, v)); + + message.write_content(upload_request.body); + + let mut encoded = vec![]; + message.write_bhttp(bhttp::Mode::KnownLength, &mut encoded)?; + + Ok(encoded) +} + +/// Unioned error across upload backends. +#[derive(Debug, thiserror::Error)] +enum ViaductUploaderError { + #[error("bhttp::Error {0}")] + Bhttp(#[from] bhttp::Error), + + #[error("ohttp::Error {0}")] + Ohttp(#[from] ohttp::Error), + + #[error("viaduct::Error {0}")] + Viaduct(#[from] viaduct::Error), + + #[error("Fatal upload error")] + Fatal, +} + +impl From for ViaductUploaderError { + fn from(e: url::ParseError) -> Self { + ViaductUploaderError::Viaduct(viaduct::Error::from(e)) + } +} diff --git a/toolkit/components/glean/src/lib.rs b/toolkit/components/glean/src/lib.rs index e1e67202c796..b682f4306615 100644 --- a/toolkit/components/glean/src/lib.rs +++ b/toolkit/components/glean/src/lib.rs @@ -28,6 +28,7 @@ extern crate cstr; extern crate xpcom; mod init; +mod ohttp_pings; pub use init::fog_init; diff --git a/toolkit/components/glean/src/ohttp_pings.rs b/toolkit/components/glean/src/ohttp_pings.rs new file mode 100644 index 000000000000..71b9512a8bfa --- /dev/null +++ b/toolkit/components/glean/src/ohttp_pings.rs @@ -0,0 +1,13 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! This file contains the generated logic for ohttp pings. +//! +//! The contents of this module are generated by +//! `toolkit/components/glean/build_scripts/glean_parser_ext/run_glean_parser.py`, from +//! ping definitions files identified by `toolkit/components/glean/metrics_index.py`. + +include!(mozbuild::objdir_path!( + "toolkit/components/glean/src/ohttp_pings.rs" +)); diff --git a/toolkit/components/glean/tests/pytest/jogfile_output b/toolkit/components/glean/tests/pytest/jogfile_output index 1b642b6189c5..510b4995c57b 100644 --- a/toolkit/components/glean/tests/pytest/jogfile_output +++ b/toolkit/components/glean/tests/pytest/jogfile_output @@ -352,6 +352,14 @@ "tomorrow", "upgrade" ] + ], + [ + "not-ohttp", + false, + true, + true, + false, + [] ] ] } \ No newline at end of file diff --git a/toolkit/components/glean/tests/pytest/pings_test.yaml b/toolkit/components/glean/tests/pytest/pings_test.yaml index efa6ba9e0ab5..558cbb10224d 100644 --- a/toolkit/components/glean/tests/pytest/pings_test.yaml +++ b/toolkit/components/glean/tests/pytest/pings_test.yaml @@ -7,7 +7,7 @@ # `glean_parser` PyPI package. --- -$schema: moz://mozilla.org/schemas/glean/pings/1-0-0 +$schema: moz://mozilla.org/schemas/glean/pings/2-0-0 not-baseline: description: > @@ -114,3 +114,19 @@ not-deletion-request: - https://bugzilla.mozilla.org/show_bug.cgi?id=1587095#c6 notification_emails: - glean-team@mozilla.com + +not-ohttp: + description: > + A fake OHTTP-using ping + include_client_id: false + metadata: + include_info_sections: false + use_ohttp: true + send_if_empty: true + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1862002 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1862002 + notification_emails: + - chutten@mozilla.com + - glean-team@mozilla.com diff --git a/toolkit/components/glean/tests/pytest/pings_test_output b/toolkit/components/glean/tests/pytest/pings_test_output index aa12bcff5533..97c0793b1ff6 100644 --- a/toolkit/components/glean/tests/pytest/pings_test_output +++ b/toolkit/components/glean/tests/pytest/pings_test_output @@ -76,6 +76,19 @@ pub static not_metrics: Lazy = Lazy::new(|| { ) }); +#[allow(non_upper_case_globals)] +/// A fake OHTTP-using ping +pub static not_ohttp: Lazy = Lazy::new(|| { + Ping::new( + "not-ohttp", + false, + true, + true, + false, + vec![], + ) +}); + /// Instantiate custom pings once to trigger registration. /// @@ -91,6 +104,7 @@ pub fn register_pings(application_id: Option<&str>) { let _ = &*not_deletion_request; let _ = &*not_events; let _ = &*not_metrics; + let _ = &*not_ohttp; } } } @@ -114,6 +128,7 @@ pub(crate) fn submit_ping_by_id(id: u32, reason: Option<&str>) { 2 => not_deletion_request.submit(reason), 3 => not_events.submit(reason), 4 => not_metrics.submit(reason), + 5 => not_ohttp.submit(reason), _ => { // TODO: instrument this error. log::error!("Cannot submit unknown ping {} by id.", id); diff --git a/toolkit/components/glean/tests/pytest/pings_test_output_cpp b/toolkit/components/glean/tests/pytest/pings_test_output_cpp index 289529b11855..899420463413 100644 --- a/toolkit/components/glean/tests/pytest/pings_test_output_cpp +++ b/toolkit/components/glean/tests/pytest/pings_test_output_cpp @@ -56,6 +56,13 @@ constexpr glean::impl::Ping NotEvents(3); */ constexpr glean::impl::Ping NotMetrics(4); +/* + * Generated from not-ohttp. + * + * A fake OHTTP-using ping + */ +constexpr glean::impl::Ping NotOhttp(5); + } // namespace mozilla::glean_pings diff --git a/toolkit/components/glean/tests/pytest/pings_test_output_js_cpp b/toolkit/components/glean/tests/pytest/pings_test_output_js_cpp index 139eb2914828..bc4f235d1e40 100644 --- a/toolkit/components/glean/tests/pytest/pings_test_output_js_cpp +++ b/toolkit/components/glean/tests/pytest/pings_test_output_js_cpp @@ -33,15 +33,17 @@ constexpr char gPingStringTable[] = { /* 12 - "notDeletionRequest" */ 'n', 'o', 't', 'D', 'e', 'l', 'e', 't', 'i', 'o', 'n', 'R', 'e', 'q', 'u', 'e', 's', 't', '\0', /* 31 - "notEvents" */ 'n', 'o', 't', 'E', 'v', 'e', 'n', 't', 's', '\0', /* 41 - "notMetrics" */ 'n', 'o', 't', 'M', 'e', 't', 'r', 'i', 'c', 's', '\0', + /* 52 - "notOhttp" */ 'n', 'o', 't', 'O', 'h', 't', 't', 'p', '\0', }; const ping_entry_t sPingByNameLookupEntries[] = { 65536, + 327732, + 262185, 131084, - 196639, - 262185 + 196639 }; @@ -88,7 +90,7 @@ PingByNameLookup(const nsACString& aKey) 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -108,7 +110,7 @@ PingByNameLookup(const nsACString& aKey) 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, diff --git a/toolkit/components/glean/tests/pytest/pings_test_output_js_h b/toolkit/components/glean/tests/pytest/pings_test_output_js_h index 0c89de93aefa..af21515ccca9 100644 --- a/toolkit/components/glean/tests/pytest/pings_test_output_js_h +++ b/toolkit/components/glean/tests/pytest/pings_test_output_js_h @@ -28,6 +28,6 @@ const char* GetPingName(ping_entry_t aEntry); */ Maybe PingByNameLookup(const nsACString&); -extern const ping_entry_t sPingByNameLookupEntries[4]; +extern const ping_entry_t sPingByNameLookupEntries[5]; } // namespace mozilla::glean #endif // mozilla_GleanJSPingsLookup_h diff --git a/toolkit/components/glean/tests/test_pings.yaml b/toolkit/components/glean/tests/test_pings.yaml index d62f682109d7..8bae13215ebd 100644 --- a/toolkit/components/glean/tests/test_pings.yaml +++ b/toolkit/components/glean/tests/test_pings.yaml @@ -40,3 +40,22 @@ test-ping: - glean-team@mozilla.com no_lint: - REDUNDANT_PING + +test-ohttp-ping: + description: | + This ping is for tests only. + Resembles how OHTTP pings are defined. + include_client_id: false + metadata: + include_info_sections: false + use_ohttp: true + send_if_empty: true + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1862002 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1862002 + notification_emails: + - chutten@mozilla.com + - glean-team@mozilla.com + no_lint: + - REDUNDANT_PING diff --git a/toolkit/components/glean/tests/xpcshell/head.js b/toolkit/components/glean/tests/xpcshell/head.js index f42bd02822e5..eaf8fa9c61b9 100644 --- a/toolkit/components/glean/tests/xpcshell/head.js +++ b/toolkit/components/glean/tests/xpcshell/head.js @@ -4,3 +4,155 @@ const { XPCOMUtils } = ChromeUtils.importESModule( "resource://gre/modules/XPCOMUtils.sys.mjs" ); + +ChromeUtils.defineESModuleGetters(this, { + HttpServer: "resource://testing-common/httpd.sys.mjs", + NetUtil: "resource://gre/modules/NetUtil.sys.mjs", +}); + +const PingServer = { + _httpServer: null, + _started: false, + _defers: [Promise.withResolvers()], + _currentDeferred: 0, + + get port() { + return this._httpServer.identity.primaryPort; + }, + + get host() { + return this._httpServer.identity.primaryHost; + }, + + get started() { + return this._started; + }, + + registerPingHandler(handler) { + this._httpServer.registerPrefixHandler("/submit/", handler); + }, + + resetPingHandler() { + this.registerPingHandler(request => { + let r = request; + console.trace( + `defaultPingHandler() - ${r.method} ${r.scheme}://${r.host}:${r.port}${r.path}` + ); + let deferred = this._defers[this._defers.length - 1]; + this._defers.push(Promise.withResolvers()); + deferred.resolve(request); + }); + }, + + start() { + this._httpServer = new HttpServer(); + this._httpServer.start(-1); + this._started = true; + this.clearRequests(); + this.resetPingHandler(); + }, + + stop() { + return new Promise(resolve => { + this._httpServer.stop(resolve); + this._started = false; + }); + }, + + clearRequests() { + this._defers = [Promise.withResolvers()]; + this._currentDeferred = 0; + }, + + promiseNextRequest() { + const deferred = this._defers[this._currentDeferred++]; + // Send the ping to the consumer on the next tick, so that the completion gets + // signaled to Telemetry. + return new Promise(r => + Services.tm.dispatchToMainThread(() => r(deferred.promise)) + ); + }, + + promiseNextPing() { + return this.promiseNextRequest().then(request => + decodeRequestPayload(request) + ); + }, + + async promiseNextRequests(count) { + let results = []; + for (let i = 0; i < count; ++i) { + results.push(await this.promiseNextRequest()); + } + + return results; + }, + + promiseNextPings(count) { + return this.promiseNextRequests(count).then(requests => { + return Array.from(requests, decodeRequestPayload); + }); + }, +}; + +/** + * Decode the payload of an HTTP request into a ping. + * + * @param {object} request The data representing an HTTP request (nsIHttpRequest). + * @returns {object} The decoded ping payload. + */ +function decodeRequestPayload(request) { + let s = request.bodyInputStream; + let payload = null; + + if ( + request.hasHeader("content-encoding") && + request.getHeader("content-encoding") == "gzip" + ) { + let observer = { + buffer: "", + onStreamComplete(loader, context, status, length, result) { + // String.fromCharCode can only deal with 500,000 characters + // at a time, so chunk the result into parts of that size. + const chunkSize = 500000; + for (let offset = 0; offset < result.length; offset += chunkSize) { + this.buffer += String.fromCharCode.apply( + String, + result.slice(offset, offset + chunkSize) + ); + } + }, + }; + + let scs = Cc["@mozilla.org/streamConverters;1"].getService( + Ci.nsIStreamConverterService + ); + let listener = Cc["@mozilla.org/network/stream-loader;1"].createInstance( + Ci.nsIStreamLoader + ); + listener.init(observer); + let converter = scs.asyncConvertData( + "gzip", + "uncompressed", + listener, + null + ); + converter.onStartRequest(null, null); + converter.onDataAvailable(null, s, 0, s.available()); + converter.onStopRequest(null, null, null); + // TODO: nsIScriptableUnicodeConverter is deprecated + // But I can't figure out how else to ungzip bodyInputStream. + let unicodeConverter = Cc[ + "@mozilla.org/intl/scriptableunicodeconverter" + ].createInstance(Ci.nsIScriptableUnicodeConverter); + unicodeConverter.charset = "UTF-8"; + let utf8string = unicodeConverter.ConvertToUnicode(observer.buffer); + utf8string += unicodeConverter.Finish(); + payload = JSON.parse(utf8string); + } else { + let bytes = NetUtil.readInputStream(s, s.available()); + payload = JSON.parse(new TextDecoder().decode(bytes)); + } + + return payload; +} diff --git a/toolkit/components/glean/tests/xpcshell/test_OHTTP.js b/toolkit/components/glean/tests/xpcshell/test_OHTTP.js new file mode 100644 index 000000000000..76d1d2a67b75 --- /dev/null +++ b/toolkit/components/glean/tests/xpcshell/test_OHTTP.js @@ -0,0 +1,32 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +add_setup(async () => { + // FOG needs a profile dir to put its data in. + do_get_profile(); + + PingServer.start(); + + registerCleanupFunction(async () => { + await PingServer.stop(); + }); + + Services.prefs.setIntPref( + "telemetry.fog.test.localhost_port", + PingServer.port + ); + // Port pref needs to be set before init, so let's reset to reinit. + Services.fog.testResetFOG(); +}); + +add_task(async () => { + PingServer.clearRequests(); + GleanPings.testOhttpPing.submit(); + + let ping = await PingServer.promiseNextPing(); + + ok(!("client_info" in ping), "No client_info allowed."); + ok(!("ping_info" in ping), "No ping_info allowed."); +}); diff --git a/toolkit/components/glean/tests/xpcshell/xpcshell.toml b/toolkit/components/glean/tests/xpcshell/xpcshell.toml index 40b1a22bf496..22806a32e603 100644 --- a/toolkit/components/glean/tests/xpcshell/xpcshell.toml +++ b/toolkit/components/glean/tests/xpcshell/xpcshell.toml @@ -30,3 +30,6 @@ skip-if = ["os == 'android'"] # Server Knobs on mobile will be handled by the sp ["test_MillionQ.js"] skip-if = ["os == 'android'"] # Android inits its own FOG, so the test won't work. + +["test_OHTTP.js"] +skip-if = ["os == 'android'"] # FOG isn't responsible for monitoring prefs and controlling upload on Android