Bug 1628068 p2 - Implement viaduct gecko backend. r=tcsc,dragana,froydnj

Differential Revision: https://phabricator.services.mozilla.com/D70257

toolkit/components/moz.build

@@ -78,6 +78,7 @@ DIRS += [
     'utils',
     'url-classifier',
     'urlformatter',
+    'viaduct',
     'viewconfig',
     'viewsource',
     'windowcreator',

toolkit/components/protobuf/regenerate_cpp_files.sh

@@ -26,3 +26,5 @@ regenerate gfx/layers/protobuf/ LayerScopePacket.proto
 regenerate devtools/shared/heapsnapshot/ CoreDump.proto
 regenerate toolkit/components/reputationservice/chromium/chrome/common/safe_browsing/ csd.proto
 regenerate toolkit/components/url-classifier/chromium/ safebrowsing.proto
+command cp third_party/rust/viaduct/src/fetch_msg_types.proto toolkit/components/viaduct/fetch_msg_types.proto
+regenerate toolkit/components/viaduct/ fetch_msg_types.proto

toolkit/components/viaduct/Viaduct.cpp

@@ -0,0 +1,52 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
+
+#include "mozilla/Viaduct.h"
+
+#include "mozilla/ClearOnShutdown.h"
+#include "mozilla/ScopeExit.h"
+#include "mozilla/ViaductRequest.h"
+
+namespace mozilla {
+
+namespace {
+
+extern "C" {
+uint8_t viaduct_initialize(
+    ViaductByteBuffer (*fetch_callback)(ViaductByteBuffer));
+}
+
+}  // namespace
+
+static StaticRefPtr<Viaduct> gViaduct;
+
+// static
+already_AddRefed<Viaduct> Viaduct::GetSingleton() {
+  if (gViaduct) {
+    return do_AddRef(gViaduct);
+  }
+
+  gViaduct = new Viaduct();
+  ClearOnShutdown(&gViaduct);
+  return do_AddRef(gViaduct);
+}
+
+NS_IMETHODIMP
+Viaduct::EnsureInitialized() {
+  if (mInitialized.compareExchange(false, true)) {
+    viaduct_initialize(ViaductCallback);
+  }
+  return NS_OK;
+}
+
+// static
+ViaductByteBuffer Viaduct::ViaductCallback(ViaductByteBuffer buffer) {
+  MOZ_ASSERT(!NS_IsMainThread(), "Background thread only!");
+  RefPtr<ViaductRequest> request = new ViaductRequest();
+  return request->MakeRequest(buffer);
+}
+
+NS_IMPL_ISUPPORTS(Viaduct, mozIViaduct)
+
+};  // namespace mozilla

toolkit/components/viaduct/Viaduct.h

@@ -0,0 +1,72 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
+
+#ifndef mozilla_Viaduct_h
+#define mozilla_Viaduct_h
+
+#include "mozIViaduct.h"
+#include "mozilla/Atomics.h"
+
+/**
+ * Viaduct is a way for Application Services Rust components
+ * (https://github.com/mozilla/application-services) to make network requests
+ * using a trusted stack (gecko).
+ *
+ * The way it works is roughly as follows:
+ * - First we register a callback using `viaduct_initialize`
+ * (Viaduct::Initialize). This callback is stored on the Rust side
+ * in a static variable, therefore Initialize() must be called only once.
+ *
+ * - When the Rust code needs to make a network request, our callback
+ * (Viaduct::ViaductCallback) will be called with a protocol buffer describing
+ * the request to make on their behalf. Note 1: The callback MUST be called from
+ * a background thread as it is blocking. Note 2: It is our side responsibility
+ * to call `viaduct_destroy_bytebuffer` on the buffer.
+ *
+ * - We set a semaphore to make the background thread wait while we make the
+ * request on the main thread using nsIChannel. (ViaductRequest::MakeRequest)
+ *
+ * - Once a response is received, we allocate a bytebuffer to store the
+ * response using `viaduct_alloc_bytebuffer` and unlock the semaphore.
+ * (ViaductRequest::OnStopRequest)
+ *
+ * - The background thread is unlocked, and the callback returns the response to
+ * the Rust caller. (Viaduct::ViaductCallback)
+ *
+ * - The Rust caller will free the response buffer we allocated earlier.
+ *
+ * Reference:
+ * https://github.com/mozilla/application-services/blob/master/components/viaduct/README.md
+ */
+
+namespace mozilla {
+
+namespace {
+
+// A mapping of the ByteBuffer repr(C) Rust struct.
+typedef struct ViaductByteBuffer {
+  int64_t len;
+  uint8_t* data;
+} ViaductByteBuffer;
+
+}  // namespace
+
+class Viaduct final : public mozIViaduct {
+ public:
+  NS_DECL_THREADSAFE_ISUPPORTS
+  NS_DECL_MOZIVIADUCT
+
+  Viaduct() : mInitialized(false) {}
+  static already_AddRefed<Viaduct> GetSingleton();
+
+ private:
+  static ViaductByteBuffer ViaductCallback(ViaductByteBuffer buffer);
+  Atomic<bool> mInitialized;
+
+  ~Viaduct() = default;
+};
+
+};  // namespace mozilla
+
+#endif  // mozilla_Viaduct_h

toolkit/components/viaduct/ViaductRequest.cpp

@@ -0,0 +1,314 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
+
+#include "mozilla/ViaductRequest.h"
+
+#include "mozilla/ErrorNames.h"
+#include "mozilla/ResultExtensions.h"
+#include "mozilla/ScopeExit.h"
+#include "mozilla/Services.h"
+
+#include "nsContentUtils.h"
+#include "nsIAsyncVerifyRedirectCallback.h"
+#include "nsIHttpChannel.h"
+#include "nsIHttpHeaderVisitor.h"
+#include "nsIInputStream.h"
+#include "nsIUploadChannel2.h"
+#include "nsIURI.h"
+#include "nsNetUtil.h"
+#include "nsString.h"
+#include "nsStringStream.h"
+
+namespace mozilla {
+
+namespace {
+
+extern "C" {
+ViaductByteBuffer viaduct_alloc_bytebuffer(int32_t);
+void viaduct_destroy_bytebuffer(ViaductByteBuffer);
+}
+
+}  // namespace
+
+class HeaderVisitor final : public nsIHttpHeaderVisitor {
+ public:
+  NS_DECL_ISUPPORTS
+  NS_DECL_NSIHTTPHEADERVISITOR
+
+  explicit HeaderVisitor(
+      google::protobuf::Map<std::string, std::string>* aHeaders)
+      : mHeaders(aHeaders) {}
+
+ private:
+  google::protobuf::Map<std::string, std::string>* mHeaders;
+  ~HeaderVisitor() = default;
+};
+
+NS_IMETHODIMP
+HeaderVisitor::VisitHeader(const nsACString& aHeader,
+                           const nsACString& aValue) {
+  (*mHeaders)[aHeader.BeginReading()] = aValue.BeginReading();
+  return NS_OK;
+}
+NS_IMPL_ISUPPORTS(HeaderVisitor, nsIHttpHeaderVisitor)
+
+nsCString ConvertMethod(
+    appservices::httpconfig::protobuf::Request_Method method);
+
+///////////////////////////////////////////////////////////////////////////////
+// ViaductRequest implementation
+
+ViaductByteBuffer ViaductRequest::MakeRequest(ViaductByteBuffer reqBuf) {
+  MOZ_ASSERT(!NS_IsMainThread(), "Background thread only!");
+  auto clearBuf = MakeScopeExit([&] { viaduct_destroy_bytebuffer(reqBuf); });
+  // We keep the protobuf parsing/serializing in the background thread.
+  appservices::httpconfig::protobuf::Request request;
+  if (!request.ParseFromArray(static_cast<const void*>(reqBuf.data),
+                              reqBuf.len)) {
+    // We still need to return something!
+    return ViaductByteBuffer{.len = 0, .data = nullptr};
+  }
+  MonitorAutoLock lock(mMonitor);
+  NS_DispatchToMainThread(NS_NewRunnableFunction(
+      "ViaductRequest::LaunchRequest", [this, &request]() {
+        nsresult rv = LaunchRequest(request);
+        if (NS_WARN_IF(NS_FAILED(rv))) {
+          // Something went very very wrong, but we still have to unblock
+          // the calling thread.
+          NotifyMonitor();
+        }
+      }));
+  while (!mDone) {
+    mMonitor.Wait();
+  }
+  ViaductByteBuffer respBuf =
+      viaduct_alloc_bytebuffer(mResponse.ByteSizeLong());
+  if (!mResponse.SerializeToArray(respBuf.data, respBuf.len)) {
+    viaduct_destroy_bytebuffer(respBuf);
+    return ViaductByteBuffer{.len = 0, .data = nullptr};
+  }
+  return respBuf;
+}
+
+nsresult ViaductRequest::LaunchRequest(
+    appservices::httpconfig::protobuf::Request& request) {
+  nsCOMPtr<nsIURI> uri;
+  nsresult rv = NS_NewURI(getter_AddRefs(uri), request.url().c_str());
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  nsSecurityFlags secFlags = nsILoadInfo::SEC_ALLOW_CROSS_ORIGIN_DATA_IS_NULL |
+                             nsILoadInfo::SEC_COOKIES_OMIT;
+  uint32_t loadFlags = 0;
+
+  if (!request.use_caches()) {
+    loadFlags |= nsIRequest::LOAD_BYPASS_CACHE;
+  }
+
+  if (!request.follow_redirects()) {
+    secFlags |= nsILoadInfo::SEC_DONT_FOLLOW_REDIRECTS;
+  }
+
+  rv = NS_NewChannel(getter_AddRefs(mChannel), uri,
+                     nsContentUtils::GetSystemPrincipal(), secFlags,
+                     nsIContentPolicy::TYPE_OTHER,
+                     nullptr,  // nsICookieJarSettings
+                     nullptr,  // aPerformanceStorage
+                     nullptr,  // aLoadGroup
+                     nullptr, loadFlags);
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  nsCOMPtr<nsIHttpChannel> httpChannel = do_QueryInterface(mChannel);
+  nsCString method = ConvertMethod(request.method());
+  rv = httpChannel->SetRequestMethod(method);
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  for (auto& header : request.headers()) {
+    rv = httpChannel->SetRequestHeader(
+        nsDependentCString(header.first.c_str(), header.first.size()),
+        nsDependentCString(header.second.c_str(), header.second.size()),
+        false /* merge */);
+    NS_ENSURE_SUCCESS(rv, rv);
+  }
+
+  // Body
+  if (request.has_body()) {
+    const std::string& body = request.body();
+    nsCOMPtr<nsIStringInputStream> stream(
+        do_CreateInstance(NS_STRINGINPUTSTREAM_CONTRACTID));
+    rv = stream->SetData(body.data(), body.size());
+    NS_ENSURE_SUCCESS(rv, rv);
+    nsCOMPtr<nsIUploadChannel2> uploadChannel = do_QueryInterface(mChannel);
+    uploadChannel->ExplicitSetUploadStream(stream, VoidCString(), -1, method,
+                                           false /* aStreamHasHeaders */);
+  }
+
+  MOZ_TRY_VAR(
+      mConnectTimeoutTimer,
+      NS_NewTimerWithCallback(this, request.connect_timeout_secs() * 1000,
+                              nsITimer::TYPE_ONE_SHOT));
+  MOZ_TRY_VAR(mReadTimeoutTimer,
+              NS_NewTimerWithCallback(this, request.read_timeout_secs() * 1000,
+                                      nsITimer::TYPE_ONE_SHOT));
+
+  rv = httpChannel->AsyncOpen(this);
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  return rv;
+}
+
+nsCString ConvertMethod(
+    appservices::httpconfig::protobuf::Request_Method method) {
+  using appservices::httpconfig::protobuf::Request_Method;
+  switch (method) {
+    case Request_Method::Request_Method_GET:
+      return NS_LITERAL_CSTRING("GET");
+    case Request_Method::Request_Method_HEAD:
+      return NS_LITERAL_CSTRING("HEAD");
+    case Request_Method::Request_Method_POST:
+      return NS_LITERAL_CSTRING("POST");
+    case Request_Method::Request_Method_PUT:
+      return NS_LITERAL_CSTRING("PUT");
+    case Request_Method::Request_Method_DELETE:
+      return NS_LITERAL_CSTRING("DELETE");
+    case Request_Method::Request_Method_CONNECT:
+      return NS_LITERAL_CSTRING("CONNECT");
+    case Request_Method::Request_Method_OPTIONS:
+      return NS_LITERAL_CSTRING("OPTIONS");
+    case Request_Method::Request_Method_TRACE:
+      return NS_LITERAL_CSTRING("TRACE");
+  }
+  return NS_LITERAL_CSTRING("UNKNOWN");
+}
+
+void ViaductRequest::ClearTimers() {
+  if (mConnectTimeoutTimer) {
+    mConnectTimeoutTimer->Cancel();
+    mConnectTimeoutTimer = nullptr;
+  }
+  if (mReadTimeoutTimer) {
+    mReadTimeoutTimer->Cancel();
+    mReadTimeoutTimer = nullptr;
+  }
+}
+
+void ViaductRequest::NotifyMonitor() {
+  mDone = true;
+  mMonitor.Notify();
+}
+
+ViaductRequest::~ViaductRequest() {
+  ClearTimers();
+  if (mChannel) {
+    mChannel->Cancel(NS_ERROR_ABORT);
+    mChannel = nullptr;
+  }
+  NotifyMonitor();
+}
+
+NS_IMPL_ISUPPORTS(ViaductRequest, nsIStreamListener, nsITimerCallback,
+                  nsIChannelEventSink)
+
+///////////////////////////////////////////////////////////////////////////////
+// nsIStreamListener implementation
+
+NS_IMETHODIMP
+ViaductRequest::OnStartRequest(nsIRequest* aRequest) {
+  if (mConnectTimeoutTimer) {
+    mConnectTimeoutTimer->Cancel();
+    mConnectTimeoutTimer = nullptr;
+  }
+  return NS_OK;
+}
+
+static nsresult AssignResponseToBuffer(nsIInputStream* aIn, void* aClosure,
+                                       const char* aFromRawSegment,
+                                       uint32_t aToOffset, uint32_t aCount,
+                                       uint32_t* aWriteCount) {
+  nsCString* buf = static_cast<nsCString*>(aClosure);
+  buf->Append(aFromRawSegment, aCount);
+  *aWriteCount = aCount;
+  return NS_OK;
+}
+
+NS_IMETHODIMP
+ViaductRequest::OnDataAvailable(nsIRequest* aRequest,
+                                nsIInputStream* aInputStream, uint64_t aOffset,
+                                uint32_t aCount) {
+  nsresult rv;
+  uint32_t readCount;
+  rv = aInputStream->ReadSegments(AssignResponseToBuffer, &mBodyBuffer, aCount,
+                                  &readCount);
+  NS_ENSURE_SUCCESS(rv, rv);
+  return NS_OK;
+}
+
+NS_IMETHODIMP
+ViaductRequest::OnStopRequest(nsIRequest* aRequest, nsresult aStatusCode) {
+  ClearTimers();
+  auto defer = MakeScopeExit([&] {
+    mChannel = nullptr;
+    NotifyMonitor();
+  });
+
+  if (NS_FAILED(aStatusCode)) {
+    nsCString errorName;
+    GetErrorName(aStatusCode, errorName);
+    nsPrintfCString msg("Request error: %s", errorName.get());
+    mResponse.set_exception_message(msg.BeginReading());
+  } else {
+    nsresult rv;
+    nsCOMPtr<nsIHttpChannel> httpChannel = do_QueryInterface(aRequest, &rv);
+    // Early return is OK because MakeScopeExit will call Notify()
+    // and unblock the original calling thread.
+    NS_ENSURE_SUCCESS(rv, rv);
+
+    uint32_t httpStatus;
+    rv = httpChannel->GetResponseStatus(&httpStatus);
+    NS_ENSURE_SUCCESS(rv, rv);
+    mResponse.set_status(httpStatus);
+
+    nsCOMPtr<nsIURI> uri;
+    httpChannel->GetURI(getter_AddRefs(uri));
+    nsAutoCString uriStr;
+    uri->GetSpec(uriStr);
+    mResponse.set_url(uriStr.BeginReading());
+
+    auto* headers = mResponse.mutable_headers();
+    nsCOMPtr<nsIHttpHeaderVisitor> visitor = new HeaderVisitor(headers);
+    rv = httpChannel->VisitResponseHeaders(visitor);
+    NS_ENSURE_SUCCESS(rv, rv);
+
+    mResponse.set_body(mBodyBuffer.BeginReading());
+  }
+
+  return NS_OK;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// nsIChannelEventSink implementation
+
+NS_IMETHODIMP
+ViaductRequest::AsyncOnChannelRedirect(
+    nsIChannel* aOldChannel, nsIChannel* aNewChannel, uint32_t flags,
+    nsIAsyncVerifyRedirectCallback* callback) {
+  mChannel = aNewChannel;
+  callback->OnRedirectVerifyCallback(NS_OK);
+  return NS_OK;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// nsITimerCallback implementation
+
+NS_IMETHODIMP
+ViaductRequest::Notify(nsITimer* timer) {
+  ClearTimers();
+  // Cancelling the channel will trigger OnStopRequest.
+  if (mChannel) {
+    mChannel->Cancel(NS_ERROR_ABORT);
+    mChannel = nullptr;
+  }
+  return NS_OK;
+}
+
+};  // namespace mozilla

toolkit/components/viaduct/ViaductRequest.h

@@ -0,0 +1,50 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
+
+#ifndef mozilla_ViaductRequest_h
+#define mozilla_ViaductRequest_h
+
+#include "mozilla/Monitor.h"
+#include "mozilla/fetch_msg_types.pb.h"
+#include "mozilla/Viaduct.h"
+
+#include "nsCOMPtr.h"
+#include "nsIChannel.h"
+#include "nsIChannelEventSink.h"
+#include "nsIStreamListener.h"
+#include "nsITimer.h"
+
+namespace mozilla {
+
+class ViaductRequest final : public nsIStreamListener,
+                             public nsITimerCallback,
+                             public nsIChannelEventSink {
+ public:
+  NS_DECL_THREADSAFE_ISUPPORTS
+  NS_DECL_NSIREQUESTOBSERVER
+  NS_DECL_NSISTREAMLISTENER
+  NS_DECL_NSITIMERCALLBACK
+  NS_DECL_NSICHANNELEVENTSINK
+
+  ViaductRequest()
+      : mDone(false), mChannel(nullptr), mMonitor("ViaductRequest") {}
+  ViaductByteBuffer MakeRequest(ViaductByteBuffer reqBuf);
+
+ private:
+  nsresult LaunchRequest(appservices::httpconfig::protobuf::Request&);
+  void ClearTimers();
+  void NotifyMonitor();
+  bool mDone;
+  nsCOMPtr<nsIChannel> mChannel;
+  nsCString mBodyBuffer;
+  nsCOMPtr<nsITimer> mConnectTimeoutTimer;
+  nsCOMPtr<nsITimer> mReadTimeoutTimer;
+  appservices::httpconfig::protobuf::Response mResponse;
+  Monitor mMonitor;
+  ~ViaductRequest();
+};
+
+};  // namespace mozilla
+
+#endif  // mozilla_ViaductRequest_h

toolkit/components/viaduct/components.conf

@@ -0,0 +1,14 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+Classes = [
+    {
+        'cid': '{1e668408-40aa-4185-991c-12671e2471dc}',
+        'contract_ids': ['@mozilla.org/toolkit/viaduct;1'],
+        'singleton': True,
+        'type': 'mozilla::Viaduct',
+        'headers': ['mozilla/Viaduct.h'],
+        'constructor': 'mozilla::Viaduct::GetSingleton',
+    }
+]

toolkit/components/viaduct/fetch_msg_types.proto

@@ -0,0 +1,41 @@
+syntax = "proto2";
+
+// Note: this file name must be unique due to how the iOS megazord works :(
+
+package mozilla.appservices.httpconfig.protobuf;
+
+option java_package = "mozilla.appservices.httpconfig";
+option java_outer_classname = "MsgTypes";
+option swift_prefix = "MsgTypes_";
+option optimize_for = LITE_RUNTIME;
+
+message Request {
+    enum Method {
+        GET = 0;
+        HEAD = 1;
+        POST = 2;
+        PUT = 3;
+        DELETE = 4;
+        CONNECT = 5;
+        OPTIONS = 6;
+        TRACE = 7;
+    }
+    required Method method = 1;
+    required string url = 2;
+    optional bytes body = 3;
+    map<string, string> headers = 4;
+    required bool follow_redirects = 5;
+    required bool use_caches = 6;
+    required int32 connect_timeout_secs = 7;
+    required int32 read_timeout_secs = 8;
+}
+
+message Response {
+    // If this is present, nothing else is.
+    optional string exception_message = 1;
+    optional string url = 2;
+    optional int32 status = 3;
+    optional bytes body = 4;
+    map<string, string> headers = 5;
+}
+

toolkit/components/viaduct/moz.build

@@ -0,0 +1,32 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+XPIDL_MODULE = 'viaduct'
+
+XPIDL_SOURCES += [
+    'mozIViaduct.idl',
+]
+
+XPCOM_MANIFESTS += [
+    'components.conf',
+]
+
+UNIFIED_SOURCES += [
+    'fetch_msg_types.pb.cc',
+    'Viaduct.cpp',
+    'ViaductRequest.cpp',
+]
+
+EXPORTS.mozilla += [
+    'fetch_msg_types.pb.h',
+    'Viaduct.h',
+    'ViaductRequest.h',
+]
+
+include('/ipc/chromium/chromium-config.mozbuild')
+
+DEFINES['GOOGLE_PROTOBUF_NO_RTTI'] = True
+DEFINES['GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER'] = True
+
+FINAL_LIBRARY = 'xul'

toolkit/components/viaduct/mozIViaduct.idl

@@ -0,0 +1,11 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "nsISupports.idl"
+
+[uuid(f8f11a45-ff04-4d8b-80e0-9ce14824538e), scriptable, builtinclass]
+interface mozIViaduct: nsISupports
+{
+  void EnsureInitialized();
+};

tools/rewriting/ThirdPartyPaths.txt

@@ -168,6 +168,8 @@ toolkit/components/normandy/vendor/
 toolkit/components/protobuf/
 toolkit/components/url-classifier/chromium/
 toolkit/components/utils/mozjexl.js
+toolkit/components/viaduct/fetch_msg_types.pb.cc
+toolkit/components/viaduct/fetch_msg_types.pb.h
 toolkit/crashreporter/breakpad-client/
 toolkit/crashreporter/google-breakpad/
 tools/fuzzing/libfuzzer/