Bug 1628068 p2 - Implement viaduct gecko backend. r=tcsc,dragana,froydnj

Differential Revision: https://phabricator.services.mozilla.com/D70257
This commit is contained in:
Edouard Oger 2020-05-11 21:01:19 +00:00
Родитель 4e005845ab
Коммит 1493d71e42
13 изменённых файлов: 2968 добавлений и 0 удалений

Просмотреть файл

@ -78,6 +78,7 @@ DIRS += [
'utils',
'url-classifier',
'urlformatter',
'viaduct',
'viewconfig',
'viewsource',
'windowcreator',

Просмотреть файл

@ -26,3 +26,5 @@ regenerate gfx/layers/protobuf/ LayerScopePacket.proto
regenerate devtools/shared/heapsnapshot/ CoreDump.proto
regenerate toolkit/components/reputationservice/chromium/chrome/common/safe_browsing/ csd.proto
regenerate toolkit/components/url-classifier/chromium/ safebrowsing.proto
command cp third_party/rust/viaduct/src/fetch_msg_types.proto toolkit/components/viaduct/fetch_msg_types.proto
regenerate toolkit/components/viaduct/ fetch_msg_types.proto

Просмотреть файл

@ -0,0 +1,52 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
#include "mozilla/Viaduct.h"
#include "mozilla/ClearOnShutdown.h"
#include "mozilla/ScopeExit.h"
#include "mozilla/ViaductRequest.h"
namespace mozilla {
namespace {
extern "C" {
uint8_t viaduct_initialize(
ViaductByteBuffer (*fetch_callback)(ViaductByteBuffer));
}
} // namespace
static StaticRefPtr<Viaduct> gViaduct;
// static
already_AddRefed<Viaduct> Viaduct::GetSingleton() {
if (gViaduct) {
return do_AddRef(gViaduct);
}
gViaduct = new Viaduct();
ClearOnShutdown(&gViaduct);
return do_AddRef(gViaduct);
}
NS_IMETHODIMP
Viaduct::EnsureInitialized() {
if (mInitialized.compareExchange(false, true)) {
viaduct_initialize(ViaductCallback);
}
return NS_OK;
}
// static
ViaductByteBuffer Viaduct::ViaductCallback(ViaductByteBuffer buffer) {
MOZ_ASSERT(!NS_IsMainThread(), "Background thread only!");
RefPtr<ViaductRequest> request = new ViaductRequest();
return request->MakeRequest(buffer);
}
NS_IMPL_ISUPPORTS(Viaduct, mozIViaduct)
}; // namespace mozilla

Просмотреть файл

@ -0,0 +1,72 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
#ifndef mozilla_Viaduct_h
#define mozilla_Viaduct_h
#include "mozIViaduct.h"
#include "mozilla/Atomics.h"
/**
* Viaduct is a way for Application Services Rust components
* (https://github.com/mozilla/application-services) to make network requests
* using a trusted stack (gecko).
*
* The way it works is roughly as follows:
* - First we register a callback using `viaduct_initialize`
* (Viaduct::Initialize). This callback is stored on the Rust side
* in a static variable, therefore Initialize() must be called only once.
*
* - When the Rust code needs to make a network request, our callback
* (Viaduct::ViaductCallback) will be called with a protocol buffer describing
* the request to make on their behalf. Note 1: The callback MUST be called from
* a background thread as it is blocking. Note 2: It is our side responsibility
* to call `viaduct_destroy_bytebuffer` on the buffer.
*
* - We set a semaphore to make the background thread wait while we make the
* request on the main thread using nsIChannel. (ViaductRequest::MakeRequest)
*
* - Once a response is received, we allocate a bytebuffer to store the
* response using `viaduct_alloc_bytebuffer` and unlock the semaphore.
* (ViaductRequest::OnStopRequest)
*
* - The background thread is unlocked, and the callback returns the response to
* the Rust caller. (Viaduct::ViaductCallback)
*
* - The Rust caller will free the response buffer we allocated earlier.
*
* Reference:
* https://github.com/mozilla/application-services/blob/master/components/viaduct/README.md
*/
namespace mozilla {
namespace {
// A mapping of the ByteBuffer repr(C) Rust struct.
typedef struct ViaductByteBuffer {
int64_t len;
uint8_t* data;
} ViaductByteBuffer;
} // namespace
class Viaduct final : public mozIViaduct {
public:
NS_DECL_THREADSAFE_ISUPPORTS
NS_DECL_MOZIVIADUCT
Viaduct() : mInitialized(false) {}
static already_AddRefed<Viaduct> GetSingleton();
private:
static ViaductByteBuffer ViaductCallback(ViaductByteBuffer buffer);
Atomic<bool> mInitialized;
~Viaduct() = default;
};
}; // namespace mozilla
#endif // mozilla_Viaduct_h

Просмотреть файл

@ -0,0 +1,314 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
#include "mozilla/ViaductRequest.h"
#include "mozilla/ErrorNames.h"
#include "mozilla/ResultExtensions.h"
#include "mozilla/ScopeExit.h"
#include "mozilla/Services.h"
#include "nsContentUtils.h"
#include "nsIAsyncVerifyRedirectCallback.h"
#include "nsIHttpChannel.h"
#include "nsIHttpHeaderVisitor.h"
#include "nsIInputStream.h"
#include "nsIUploadChannel2.h"
#include "nsIURI.h"
#include "nsNetUtil.h"
#include "nsString.h"
#include "nsStringStream.h"
namespace mozilla {
namespace {
extern "C" {
ViaductByteBuffer viaduct_alloc_bytebuffer(int32_t);
void viaduct_destroy_bytebuffer(ViaductByteBuffer);
}
} // namespace
class HeaderVisitor final : public nsIHttpHeaderVisitor {
public:
NS_DECL_ISUPPORTS
NS_DECL_NSIHTTPHEADERVISITOR
explicit HeaderVisitor(
google::protobuf::Map<std::string, std::string>* aHeaders)
: mHeaders(aHeaders) {}
private:
google::protobuf::Map<std::string, std::string>* mHeaders;
~HeaderVisitor() = default;
};
NS_IMETHODIMP
HeaderVisitor::VisitHeader(const nsACString& aHeader,
const nsACString& aValue) {
(*mHeaders)[aHeader.BeginReading()] = aValue.BeginReading();
return NS_OK;
}
NS_IMPL_ISUPPORTS(HeaderVisitor, nsIHttpHeaderVisitor)
nsCString ConvertMethod(
appservices::httpconfig::protobuf::Request_Method method);
///////////////////////////////////////////////////////////////////////////////
// ViaductRequest implementation
ViaductByteBuffer ViaductRequest::MakeRequest(ViaductByteBuffer reqBuf) {
MOZ_ASSERT(!NS_IsMainThread(), "Background thread only!");
auto clearBuf = MakeScopeExit([&] { viaduct_destroy_bytebuffer(reqBuf); });
// We keep the protobuf parsing/serializing in the background thread.
appservices::httpconfig::protobuf::Request request;
if (!request.ParseFromArray(static_cast<const void*>(reqBuf.data),
reqBuf.len)) {
// We still need to return something!
return ViaductByteBuffer{.len = 0, .data = nullptr};
}
MonitorAutoLock lock(mMonitor);
NS_DispatchToMainThread(NS_NewRunnableFunction(
"ViaductRequest::LaunchRequest", [this, &request]() {
nsresult rv = LaunchRequest(request);
if (NS_WARN_IF(NS_FAILED(rv))) {
// Something went very very wrong, but we still have to unblock
// the calling thread.
NotifyMonitor();
}
}));
while (!mDone) {
mMonitor.Wait();
}
ViaductByteBuffer respBuf =
viaduct_alloc_bytebuffer(mResponse.ByteSizeLong());
if (!mResponse.SerializeToArray(respBuf.data, respBuf.len)) {
viaduct_destroy_bytebuffer(respBuf);
return ViaductByteBuffer{.len = 0, .data = nullptr};
}
return respBuf;
}
nsresult ViaductRequest::LaunchRequest(
appservices::httpconfig::protobuf::Request& request) {
nsCOMPtr<nsIURI> uri;
nsresult rv = NS_NewURI(getter_AddRefs(uri), request.url().c_str());
NS_ENSURE_SUCCESS(rv, rv);
nsSecurityFlags secFlags = nsILoadInfo::SEC_ALLOW_CROSS_ORIGIN_DATA_IS_NULL |
nsILoadInfo::SEC_COOKIES_OMIT;
uint32_t loadFlags = 0;
if (!request.use_caches()) {
loadFlags |= nsIRequest::LOAD_BYPASS_CACHE;
}
if (!request.follow_redirects()) {
secFlags |= nsILoadInfo::SEC_DONT_FOLLOW_REDIRECTS;
}
rv = NS_NewChannel(getter_AddRefs(mChannel), uri,
nsContentUtils::GetSystemPrincipal(), secFlags,
nsIContentPolicy::TYPE_OTHER,
nullptr, // nsICookieJarSettings
nullptr, // aPerformanceStorage
nullptr, // aLoadGroup
nullptr, loadFlags);
NS_ENSURE_SUCCESS(rv, rv);
nsCOMPtr<nsIHttpChannel> httpChannel = do_QueryInterface(mChannel);
nsCString method = ConvertMethod(request.method());
rv = httpChannel->SetRequestMethod(method);
NS_ENSURE_SUCCESS(rv, rv);
auto headers = request.headers();
for (auto& [k, v] : headers) {
auto name = nsDependentCString(k.c_str(), k.size());
auto value = nsDependentCString(v.c_str(), v.size());
rv = httpChannel->SetRequestHeader(name, value, false /* merge */);
NS_ENSURE_SUCCESS(rv, rv);
}
// Body
if (request.has_body()) {
const std::string& body = request.body();
nsCOMPtr<nsIStringInputStream> stream(
do_CreateInstance(NS_STRINGINPUTSTREAM_CONTRACTID));
rv = stream->SetData(body.data(), body.size());
NS_ENSURE_SUCCESS(rv, rv);
nsCOMPtr<nsIUploadChannel2> uploadChannel = do_QueryInterface(mChannel);
uploadChannel->ExplicitSetUploadStream(stream, VoidCString(), -1, method,
false /* aStreamHasHeaders */);
}
MOZ_TRY_VAR(
mConnectTimeoutTimer,
NS_NewTimerWithCallback(this, request.connect_timeout_secs() * 1000,
nsITimer::TYPE_ONE_SHOT));
MOZ_TRY_VAR(mReadTimeoutTimer,
NS_NewTimerWithCallback(this, request.read_timeout_secs() * 1000,
nsITimer::TYPE_ONE_SHOT));
rv = httpChannel->AsyncOpen(this);
NS_ENSURE_SUCCESS(rv, rv);
return rv;
}
nsCString ConvertMethod(
appservices::httpconfig::protobuf::Request_Method method) {
using appservices::httpconfig::protobuf::Request_Method;
switch (method) {
case Request_Method::Request_Method_GET:
return NS_LITERAL_CSTRING("GET");
case Request_Method::Request_Method_HEAD:
return NS_LITERAL_CSTRING("HEAD");
case Request_Method::Request_Method_POST:
return NS_LITERAL_CSTRING("POST");
case Request_Method::Request_Method_PUT:
return NS_LITERAL_CSTRING("PUT");
case Request_Method::Request_Method_DELETE:
return NS_LITERAL_CSTRING("DELETE");
case Request_Method::Request_Method_CONNECT:
return NS_LITERAL_CSTRING("CONNECT");
case Request_Method::Request_Method_OPTIONS:
return NS_LITERAL_CSTRING("OPTIONS");
case Request_Method::Request_Method_TRACE:
return NS_LITERAL_CSTRING("TRACE");
}
return NS_LITERAL_CSTRING("UNKNOWN");
}
void ViaductRequest::ClearTimers() {
if (mConnectTimeoutTimer) {
mConnectTimeoutTimer->Cancel();
mConnectTimeoutTimer = nullptr;
}
if (mReadTimeoutTimer) {
mReadTimeoutTimer->Cancel();
mReadTimeoutTimer = nullptr;
}
}
void ViaductRequest::NotifyMonitor() {
mDone = true;
mMonitor.Notify();
}
ViaductRequest::~ViaductRequest() {
ClearTimers();
if (mChannel) {
mChannel->Cancel(NS_ERROR_ABORT);
mChannel = nullptr;
}
NotifyMonitor();
}
NS_IMPL_ISUPPORTS(ViaductRequest, nsIStreamListener, nsITimerCallback,
nsIChannelEventSink)
///////////////////////////////////////////////////////////////////////////////
// nsIStreamListener implementation
NS_IMETHODIMP
ViaductRequest::OnStartRequest(nsIRequest* aRequest) {
if (mConnectTimeoutTimer) {
mConnectTimeoutTimer->Cancel();
mConnectTimeoutTimer = nullptr;
}
return NS_OK;
}
static nsresult AssignResponseToBuffer(nsIInputStream* aIn, void* aClosure,
const char* aFromRawSegment,
uint32_t aToOffset, uint32_t aCount,
uint32_t* aWriteCount) {
nsCString* buf = static_cast<nsCString*>(aClosure);
buf->Append(aFromRawSegment, aCount);
*aWriteCount = aCount;
return NS_OK;
}
NS_IMETHODIMP
ViaductRequest::OnDataAvailable(nsIRequest* aRequest,
nsIInputStream* aInputStream, uint64_t aOffset,
uint32_t aCount) {
nsresult rv;
uint32_t readCount;
rv = aInputStream->ReadSegments(AssignResponseToBuffer, &mBodyBuffer, aCount,
&readCount);
NS_ENSURE_SUCCESS(rv, rv);
return NS_OK;
}
NS_IMETHODIMP
ViaductRequest::OnStopRequest(nsIRequest* aRequest, nsresult aStatusCode) {
ClearTimers();
auto defer = MakeScopeExit([&] {
mChannel = nullptr;
NotifyMonitor();
});
if (NS_FAILED(aStatusCode)) {
nsCString errorName;
GetErrorName(aStatusCode, errorName);
nsPrintfCString msg("Request error: %s", errorName.get());
mResponse.set_exception_message(msg.BeginReading());
} else {
nsresult rv;
nsCOMPtr<nsIHttpChannel> httpChannel = do_QueryInterface(aRequest, &rv);
// Early return is OK because MakeScopeExit will call Notify()
// and unblock the original calling thread.
NS_ENSURE_SUCCESS(rv, rv);
uint32_t httpStatus;
rv = httpChannel->GetResponseStatus(&httpStatus);
NS_ENSURE_SUCCESS(rv, rv);
mResponse.set_status(httpStatus);
nsCOMPtr<nsIURI> uri;
httpChannel->GetURI(getter_AddRefs(uri));
nsAutoCString uriStr;
uri->GetSpec(uriStr);
mResponse.set_url(uriStr.BeginReading());
auto* headers = mResponse.mutable_headers();
nsCOMPtr<nsIHttpHeaderVisitor> visitor = new HeaderVisitor(headers);
rv = httpChannel->VisitResponseHeaders(visitor);
NS_ENSURE_SUCCESS(rv, rv);
mResponse.set_body(mBodyBuffer.BeginReading());
}
return NS_OK;
}
///////////////////////////////////////////////////////////////////////////////
// nsIChannelEventSink implementation
NS_IMETHODIMP
ViaductRequest::AsyncOnChannelRedirect(
nsIChannel* aOldChannel, nsIChannel* aNewChannel, uint32_t flags,
nsIAsyncVerifyRedirectCallback* callback) {
mChannel = aNewChannel;
callback->OnRedirectVerifyCallback(NS_OK);
return NS_OK;
}
///////////////////////////////////////////////////////////////////////////////
// nsITimerCallback implementation
NS_IMETHODIMP
ViaductRequest::Notify(nsITimer* timer) {
ClearTimers();
// Cancelling the channel will trigger OnStopRequest.
if (mChannel) {
mChannel->Cancel(NS_ERROR_ABORT);
mChannel = nullptr;
}
return NS_OK;
}
}; // namespace mozilla

Просмотреть файл

@ -0,0 +1,50 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
#ifndef mozilla_ViaductRequest_h
#define mozilla_ViaductRequest_h
#include "mozilla/Monitor.h"
#include "mozilla/fetch_msg_types.pb.h"
#include "mozilla/Viaduct.h"
#include "nsCOMPtr.h"
#include "nsIChannel.h"
#include "nsIChannelEventSink.h"
#include "nsIStreamListener.h"
#include "nsITimer.h"
namespace mozilla {
class ViaductRequest final : public nsIStreamListener,
public nsITimerCallback,
public nsIChannelEventSink {
public:
NS_DECL_THREADSAFE_ISUPPORTS
NS_DECL_NSIREQUESTOBSERVER
NS_DECL_NSISTREAMLISTENER
NS_DECL_NSITIMERCALLBACK
NS_DECL_NSICHANNELEVENTSINK
ViaductRequest()
: mDone(false), mChannel(nullptr), mMonitor("ViaductRequest") {}
ViaductByteBuffer MakeRequest(ViaductByteBuffer reqBuf);
private:
nsresult LaunchRequest(appservices::httpconfig::protobuf::Request&);
void ClearTimers();
void NotifyMonitor();
bool mDone;
nsCOMPtr<nsIChannel> mChannel;
nsCString mBodyBuffer;
nsCOMPtr<nsITimer> mConnectTimeoutTimer;
nsCOMPtr<nsITimer> mReadTimeoutTimer;
appservices::httpconfig::protobuf::Response mResponse;
Monitor mMonitor;
~ViaductRequest();
};
}; // namespace mozilla
#endif // mozilla_ViaductRequest_h

Просмотреть файл

@ -0,0 +1,14 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
Classes = [
{
'cid': '{1e668408-40aa-4185-991c-12671e2471dc}',
'contract_ids': ['@mozilla.org/toolkit/viaduct;1'],
'singleton': True,
'type': 'mozilla::Viaduct',
'headers': ['mozilla/Viaduct.h'],
'constructor': 'mozilla::Viaduct::GetSingleton',
}
]

Разница между файлами не показана из-за своего большого размера Загрузить разницу

Разница между файлами не показана из-за своего большого размера Загрузить разницу

Просмотреть файл

@ -0,0 +1,41 @@
syntax = "proto2";
// Note: this file name must be unique due to how the iOS megazord works :(
package mozilla.appservices.httpconfig.protobuf;
option java_package = "mozilla.appservices.httpconfig";
option java_outer_classname = "MsgTypes";
option swift_prefix = "MsgTypes_";
option optimize_for = LITE_RUNTIME;
message Request {
enum Method {
GET = 0;
HEAD = 1;
POST = 2;
PUT = 3;
DELETE = 4;
CONNECT = 5;
OPTIONS = 6;
TRACE = 7;
}
required Method method = 1;
required string url = 2;
optional bytes body = 3;
map<string, string> headers = 4;
required bool follow_redirects = 5;
required bool use_caches = 6;
required int32 connect_timeout_secs = 7;
required int32 read_timeout_secs = 8;
}
message Response {
// If this is present, nothing else is.
optional string exception_message = 1;
optional string url = 2;
optional int32 status = 3;
optional bytes body = 4;
map<string, string> headers = 5;
}

Просмотреть файл

@ -0,0 +1,32 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
XPIDL_MODULE = 'viaduct'
XPIDL_SOURCES += [
'mozIViaduct.idl',
]
XPCOM_MANIFESTS += [
'components.conf',
]
UNIFIED_SOURCES += [
'fetch_msg_types.pb.cc',
'Viaduct.cpp',
'ViaductRequest.cpp',
]
EXPORTS.mozilla += [
'fetch_msg_types.pb.h',
'Viaduct.h',
'ViaductRequest.h',
]
include('/ipc/chromium/chromium-config.mozbuild')
DEFINES['GOOGLE_PROTOBUF_NO_RTTI'] = True
DEFINES['GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER'] = True
FINAL_LIBRARY = 'xul'

Просмотреть файл

@ -0,0 +1,11 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#include "nsISupports.idl"
[uuid(f8f11a45-ff04-4d8b-80e0-9ce14824538e), scriptable, builtinclass]
interface mozIViaduct: nsISupports
{
void EnsureInitialized();
};

Просмотреть файл

@ -168,6 +168,8 @@ toolkit/components/normandy/vendor/
toolkit/components/protobuf/
toolkit/components/url-classifier/chromium/
toolkit/components/utils/mozjexl.js
toolkit/components/viaduct/fetch_msg_types.pb.cc
toolkit/components/viaduct/fetch_msg_types.pb.h
toolkit/crashreporter/breakpad-client/
toolkit/crashreporter/google-breakpad/
tools/fuzzing/libfuzzer/