Finish up using protobufs in places, and document everything

This commit is contained in:
Thom Chiovoloni 2019-02-13 16:06:30 -08:00 коммит произвёл Thom
Родитель e36ecacc64
Коммит 940a3bb2ed
21 изменённых файлов: 625 добавлений и 108 удалений

4
Cargo.lock сгенерированный
Просмотреть файл

@ -1164,6 +1164,7 @@ name = "places"
version = "0.1.0"
dependencies = [
"bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)",
"bytes 0.4.11 (registry+https://github.com/rust-lang/crates.io-index)",
"caseless 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
"clap 2.32.0 (registry+https://github.com/rust-lang/crates.io-index)",
"cli-support 0.1.0",
@ -1178,6 +1179,9 @@ dependencies = [
"log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
"memchr 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
"more-asserts 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
"prost 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"prost-build 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"prost-derive 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"rand 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)",
"rusqlite 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 1.0.87 (registry+https://github.com/rust-lang/crates.io-index)",

Просмотреть файл

@ -10,7 +10,7 @@ import com.sun.jna.Native
import com.sun.jna.Pointer
import com.sun.jna.PointerType
import java.lang.reflect.Proxy
import mozilla.appservices.support.ByteBuffer
import mozilla.appservices.support.RustBuffer
@Suppress("FunctionNaming", "TooManyFunctions", "TooGenericExceptionThrown")
internal interface FxaClient : Library {
@ -73,7 +73,7 @@ internal interface FxaClient : Library {
e: Error.ByReference
): Pointer?
fun fxa_profile(fxa: FxaHandle, ignoreCache: Boolean, e: Error.ByReference): ByteBuffer.ByValue
fun fxa_profile(fxa: FxaHandle, ignoreCache: Boolean, e: Error.ByReference): RustBuffer.ByValue
fun fxa_get_token_server_endpoint_url(fxa: FxaHandle, e: Error.ByReference): Pointer?
fun fxa_get_connection_success_url(fxa: FxaHandle, e: Error.ByReference): Pointer?
@ -90,7 +90,7 @@ internal interface FxaClient : Library {
// when using Structure.
fun fxa_oauth_info_free(ptr: Pointer)
fun fxa_bytebuffer_free(buffer: ByteBuffer.ByValue)
fun fxa_bytebuffer_free(buffer: RustBuffer.ByValue)
}
internal typealias FxaHandle = Long

Просмотреть файл

@ -4,10 +4,10 @@
package org.mozilla.fxaclient.internal
import mozilla.appservices.support.ByteBuffer
import mozilla.appservices.support.RustBuffer
import org.mozilla.fxaclient.internal.MsgTypes.Profile as RawProfile
class Profile internal constructor(byteBuffer: ByteBuffer.ByValue) {
class Profile internal constructor(byteBuffer: RustBuffer.ByValue) {
val uid: String?
val email: String?

Просмотреть файл

@ -160,5 +160,6 @@ macro_rules! implement_into_ffi_converting {
#[cfg(feature = "browserid")]
implement_into_ffi_converting!(SyncKeys, SyncKeysC);
implement_into_ffi_converting!(AccessTokenInfo, AccessTokenInfoC);
implement_into_ffi_by_protobuf!(msg_types::Profile);
implement_into_ffi_by_delegation!(Profile, msg_types::Profile);

Просмотреть файл

@ -23,10 +23,13 @@ caseless = "0.2.1"
unicode-normalization = "0.1.7"
sql-support = { path = "../support/sql" }
url_serde = "0.2.0"
ffi-support = { path = "../support/ffi", optional = true }
ffi-support = { path = "../support/ffi", optional = true, features = ["prost_support"] }
bitflags = "1.0.4"
idna = "0.1.5"
memchr = "2.1.3"
prost = "0.4.0"
prost-derive = "0.4.0"
bytes = "0.4.11"
[dependencies.rusqlite]
version = "0.16.0"
@ -45,6 +48,8 @@ criterion = "0.2.9"
tempdir = "0.3.7"
cli-support = { path = "../support/cli" }
[build-dependencies]
prost-build = "0.4.0"
# While we don't have a replacement for termion on Windows yet (and thus
# our example doesn't work on Windows), it does get further in the compilation

Просмотреть файл

@ -2,6 +2,7 @@ apply plugin: 'com.android.library'
apply plugin: 'org.mozilla.rust-android-gradle.rust-android'
apply plugin: 'kotlin-android'
apply plugin: 'kotlin-android-extensions'
apply plugin: 'com.google.protobuf'
android {
compileSdkVersion 27
@ -27,6 +28,12 @@ android {
sourceSets {
test.resources.srcDirs += "$buildDir/rustJniLibs/desktop"
main {
proto {
srcDir '../src'
}
}
}
// Help folks debugging by including symbols in our native libraries. Yes, this makes the
@ -95,12 +102,35 @@ configurations {
jnaForTest
}
protobuf {
protoc {
artifact = 'com.google.protobuf:protoc:3.0.0'
}
plugins {
javalite {
artifact = 'com.google.protobuf:protoc-gen-javalite:3.0.0'
}
}
generateProtoTasks {
all().each { task ->
task.builtins {
remove java
}
task.plugins {
javalite { }
}
}
}
}
dependencies {
jnaForTest 'net.java.dev.jna:jna:4.5.2@jar'
implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk7:$kotlin_version"
implementation 'net.java.dev.jna:jna:4.5.2@aar'
implementation 'com.google.protobuf:protobuf-lite:3.0.0'
implementation project(':as-support-library')
// For reasons unknown, resolving the jnaForTest configuration directly
// trips a nasty issue with the Android-Gradle plugin 3.2.1, like `Cannot
// change attributes of configuration ':PROJECT:kapt' after it has been

Просмотреть файл

@ -11,6 +11,7 @@ import com.sun.jna.Pointer
import com.sun.jna.PointerType
import com.sun.jna.StringArray
import java.lang.reflect.Proxy
import mozilla.appservices.support.RustBuffer
internal interface LibPlacesFFI : Library {
companion object {
@ -146,7 +147,7 @@ internal interface LibPlacesFFI : Library {
startDate: Long,
endDate: Long,
error: RustError.ByReference
): Pointer?
): RustBuffer.ByValue
fun sync15_history_sync(
handle: PlacesConnectionHandle,
@ -165,6 +166,8 @@ internal interface LibPlacesFFI : Library {
fun places_connection_destroy(handle: PlacesConnectionHandle, out_err: RustError.ByReference)
/** Destroy handle created using `places_new_interrupt_handle` */
fun places_interrupt_handle_destroy(obj: RawPlacesInterruptHandle)
fun places_destroy_bytebuffer(bb: RustBuffer.ByValue)
}
internal typealias PlacesConnectionHandle = Long;

Просмотреть файл

@ -141,11 +141,16 @@ class PlacesConnection(path: String, encryption_key: String? = null) : PlacesAPI
}
override fun getVisitInfos(start: Long, end: Long): List<VisitInfo> {
val infoJson = rustCallForString { error ->
val infoBuffer = rustCall { error ->
LibPlacesFFI.INSTANCE.places_get_visit_infos(
this.handle.get(), start, end, error)
}
return VisitInfo.fromJSONArray(infoJson)
try {
val infos = MsgTypes.HistoryVisitInfos.parseFrom(infoBuffer.asCodedInputStream()!!)
return VisitInfo.fromMessage(infos)
} finally {
LibPlacesFFI.INSTANCE.places_destroy_bytebuffer(infoBuffer)
}
}
override fun deletePlace(url: String) {
@ -554,23 +559,13 @@ data class VisitInfo(
val visitType: VisitType
) {
companion object {
fun fromJSON(jsonObject: JSONObject): VisitInfo {
return VisitInfo(
url = jsonObject.getString("url"),
title = stringOrNull(jsonObject, "title"),
visitTime = jsonObject.getLong("visit_date"),
visitType = intToVisitType.get(jsonObject.getInt("visit_type"))!!
)
}
fun fromJSONArray(jsonArrayText: String): List<VisitInfo> {
val array = JSONArray(jsonArrayText)
val result: ArrayList<VisitInfo> = ArrayList(array.length())
for (index in 0 until array.length()) {
result.add(fromJSON(array.getJSONObject(index)))
internal fun fromMessage(msg: MsgTypes.HistoryVisitInfos): List<VisitInfo> {
return msg.infosList.map {
VisitInfo(url = it.url,
title = it.title,
visitTime = it.timestamp,
visitType = intToVisitType[it.visitType]!!)
}
return result
}
}
}

Просмотреть файл

@ -166,5 +166,19 @@ class PlacesConnectionTest {
assertEquals("https://news.ycombinator.com/", db.matchUrl("news"))
}
// Basically equivalent to test_get_visited in rust, but exercises the FFI,
// as well as the handling of invalid urls.
@Test
fun testGetVisitInfos() {
db.noteObservation(VisitObservation(url = "https://www.example.com/1", visitType = VisitType.LINK, at = 100000))
db.noteObservation(VisitObservation(url = "https://www.example.com/2", visitType = VisitType.LINK, at = 150000))
db.noteObservation(VisitObservation(url = "https://www.example.com/3", visitType = VisitType.LINK, at = 200000))
db.noteObservation(VisitObservation(url = "https://www.example.com/4", visitType = VisitType.LINK, at = 250000))
val infos = db.getVisitInfos(125000, 225000)
assert(infos.size == 2)
assert(infos[0].url == "https://www.example.com/2")
assert(infos[1].url == "https://www.example.com/3")
}
}

Просмотреть файл

@ -0,0 +1,7 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
fn main() {
prost_build::compile_protos(&["src/msg_types.proto"], &["src/"]).unwrap();
}

Просмотреть файл

@ -3,8 +3,9 @@
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
use ffi_support::{
define_box_destructor, define_handle_map_deleter, define_string_destructor, rust_str_from_c,
rust_string_from_c, ConcurrentHandleMap, ExternError,
define_box_destructor, define_bytebuffer_destructor, define_handle_map_deleter,
define_string_destructor, rust_str_from_c, rust_string_from_c, ByteBuffer, ConcurrentHandleMap,
ExternError,
};
use places::history_sync::store::HistoryStore;
use places::{db::PlacesInterruptHandle, storage, PlacesDb};
@ -251,7 +252,7 @@ pub unsafe extern "C" fn places_get_visit_infos(
start_date: i64,
end_date: i64,
error: &mut ExternError,
) -> *const c_char {
) -> ByteBuffer {
log::debug!("places_get_visit_infos");
CONNECTIONS.call_with_result(error, handle, |conn| -> places::Result<_> {
Ok(storage::history::get_visit_infos(
@ -291,5 +292,6 @@ pub unsafe extern "C" fn sync15_history_sync(
}
define_string_destructor!(places_destroy_string);
define_bytebuffer_destructor!(places_destroy_bytebuffer);
define_handle_map_deleter!(CONNECTIONS, places_connection_destroy);
define_box_destructor!(PlacesInterruptHandle, places_interrupt_handle_destroy);

Просмотреть файл

@ -9,9 +9,9 @@
use crate::api::matcher::SearchResult;
use crate::db::PlacesInterruptHandle;
use crate::error::{Error, ErrorKind};
use crate::storage::HistoryVisitInfo;
use ffi_support::{
implement_into_ffi_by_json, implement_into_ffi_by_pointer, ErrorCode, ExternError,
implement_into_ffi_by_json, implement_into_ffi_by_pointer, implement_into_ffi_by_protobuf,
ErrorCode, ExternError,
};
pub mod error_codes {
@ -80,5 +80,5 @@ impl From<Error> for ExternError {
}
implement_into_ffi_by_json!(SearchResult);
implement_into_ffi_by_json!(HistoryVisitInfo);
implement_into_ffi_by_pointer!(PlacesInterruptHandle);
implement_into_ffi_by_protobuf!(crate::msg_types::HistoryVisitInfos);

Просмотреть файл

@ -19,6 +19,11 @@ pub mod storage;
mod util;
mod valid_guid;
pub mod msg_types {
use prost_derive::Message;
include!(concat!(env!("OUT_DIR"), "/msg_types.rs"));
}
pub use crate::api::apply_observation;
pub use crate::db::{PlacesDb, PlacesInterruptHandle};
pub use crate::error::*;

Просмотреть файл

@ -0,0 +1,17 @@
syntax = "proto2";
package msg_types;
option java_package = "org.mozilla.places";
option java_outer_classname = "MsgTypes";
message HistoryVisitInfo {
required string url = 1;
optional string title = 2;
required int64 timestamp = 3;
required int32 visit_type = 4;
}
message HistoryVisitInfos {
repeated HistoryVisitInfo infos = 1;
}

Просмотреть файл

@ -2,11 +2,12 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
use super::{fetch_page_info, new_page_info, HistoryVisitInfo, PageInfo, RowId};
use super::{fetch_page_info, new_page_info, PageInfo, RowId};
use crate::db::PlacesDb;
use crate::error::Result;
use crate::frecency;
use crate::hash;
use crate::msg_types::{HistoryVisitInfo, HistoryVisitInfos};
use crate::observation::VisitObservation;
use crate::types::{SyncGuid, SyncStatus, Timestamp, VisitTransition};
use rusqlite::types::ToSql;
@ -979,8 +980,8 @@ pub fn get_visit_infos(
db: &PlacesDb,
start: Timestamp,
end: Timestamp,
) -> Result<Vec<HistoryVisitInfo>> {
Ok(db.query_rows_and_then_named_cached(
) -> Result<HistoryVisitInfos> {
let infos = db.query_rows_and_then_named_cached(
"SELECT h.url, h.title, v.visit_date, v.visit_type
FROM moz_places h
JOIN moz_historyvisits v
@ -989,7 +990,8 @@ pub fn get_visit_infos(
ORDER BY v.visit_date",
&[(":start", &start), (":end", &end)],
HistoryVisitInfo::from_row,
)?)
)?;
Ok(HistoryVisitInfos { infos })
}
#[cfg(test)]

Просмотреть файл

@ -9,6 +9,7 @@ pub mod bookmarks;
pub mod history;
use crate::error::Result;
use crate::msg_types::HistoryVisitInfo;
use crate::types::{SyncGuid, SyncStatus, Timestamp, VisitTransition};
use rusqlite::types::{FromSql, FromSqlResult, ToSql, ToSqlOutput, ValueRef};
use rusqlite::Result as RusqliteResult;
@ -164,25 +165,19 @@ fn new_page_info(db: &impl ConnExt, url: &Url, new_guid: Option<SyncGuid>) -> Re
})
}
#[derive(Debug, Clone, PartialEq, serde_derive::Serialize)]
pub struct HistoryVisitInfo {
pub url: String,
pub title: Option<String>,
pub visit_date: Timestamp,
pub visit_type: VisitTransition,
}
impl HistoryVisitInfo {
pub fn from_row(row: &rusqlite::Row) -> Result<Self> {
pub(crate) fn from_row(row: &rusqlite::Row) -> Result<Self> {
let visit_type = VisitTransition::from_primitive(row.get_checked::<_, u8>("visit_type")?)
// Do we have an existing error we use for this? For now they
// probably don't care too much about VisitTransition, so this
// is fine.
.unwrap_or(VisitTransition::Link);
let visit_date: Timestamp = row.get_checked("visit_date")?;
Ok(Self {
url: row.get_checked("url")?,
title: row.get_checked("title")?,
visit_date: row.get_checked("visit_date")?,
visit_type: VisitTransition::from_primitive(row.get_checked::<_, u8>("visit_type")?)
// Do we have an existing error we use for this? For now they
// probably don't care too much about VisitTransition, so this
// is fine.
.unwrap_or(VisitTransition::Link),
timestamp: visit_date.0 as i64,
visit_type: visit_type as i32,
})
}
}

Просмотреть файл

@ -1,54 +0,0 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
package mozilla.appservices.support
import com.google.protobuf.CodedInputStream
import com.sun.jna.Pointer
import com.sun.jna.Structure
import java.util.Arrays
/**
* This is a mapping for the ByteBuffer type from ffi_support.
*
* # Caveats:
*
* 1. It is for passing data *FROM* Rust code *TO* Kotlin/Java code.
* Do *not* use this to pass data in the other direction! Rust code
* assumes that it owns ByteBuffers, and will release their memory
* when it `Drop`s them.
*
* (Instead, just pass the data and length as two arguments).
*
* 2. A ByteBuffer passed into kotlin code must be freed by kotlin
* code. The rust code must expose a destructor for this purpose,
* and it should be called in the finally block after the data
* is read from the CodedInputStream.
*
* 3. You almost always should use `ByteBuffer.ByValue` instead
* of ByteBuffer. E.g.
* `fun mylib_get_stuff(some: X, args: Y): ByteBuffer.ByValue`
* for the function returning the ByteBuffer, and
* `fun mylib_destroy_bytebuffer(bb: ByteBuffer.ByValue)`.
*/
open class ByteBuffer : Structure() {
@JvmField var len: Long = 0
@JvmField var data: Pointer? = null
init {
read()
}
override fun getFieldOrder(): List<String> {
return Arrays.asList("len", "data")
}
fun asCodedInputStream(): CodedInputStream? {
return this.data?.let {
CodedInputStream.newInstance(it.getByteBuffer(0, this.len))
}
}
class ByValue : ByteBuffer(), Structure.ByValue
}

Просмотреть файл

@ -0,0 +1,26 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
package mozilla.appservices.support
import com.google.protobuf.MessageLite
import com.google.protobuf.CodedOutputStream
import java.nio.ByteBuffer
import java.nio.ByteOrder
/**
* A helper for converting a protobuf Message into a direct `java.nio.ByteBuffer`
* and it's length. This avoids a copy when passing data to Rust, when compared
* to using an `Array<Byte>`
*/
fun <T: MessageLite> T.toNioDirectBuffer(): Pair<ByteBuffer, Int> {
val len = this.serializedSize
val nioBuf = ByteBuffer.allocateDirect(len)
nioBuf.order(ByteOrder.nativeOrder())
val output = CodedOutputStream.newInstance(nioBuf)
this.writeTo(output)
output.checkNoSpaceLeft()
return Pair(first = nioBuf, second = len)
}

Просмотреть файл

@ -0,0 +1,62 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
package mozilla.appservices.support
import com.google.protobuf.CodedInputStream
import com.sun.jna.Pointer
import com.sun.jna.Structure
import java.util.Arrays
/**
* This is a mapping for the `ffi_support::ByteBuffer` struct.
*
* The name differs for two reasons.
*
* 1. To that the memory this type manages is allocated from rust code,
* and must subsequently be freed by rust code.
*
* 2. To avoid confusion with java's nio ByteBuffer, which we use for
* passing data *to* Rust without incurring additional copies.
*
* # Caveats:
*
* 1. It is for receiving data *FROM* Rust, and not the other direction.
* Rust code assumes that it owns `RustBuffer`s, and will release
* their memory when it `Drop`s them. (RustBuffer doesn't expose
* a way to inspect its contents from Rust anyway).
*
* 2. A `RustBuffer` passed into kotlin code must be freed by kotlin
* code *after* the protobuf message is completely deserialized.
*
* The rust code must expose a destructor for this purpose,
* and it should be called in the finally block after the data
* is read from the `CodedInputStream` (and not before).
*
* 3. You almost always should use `RustBuffer.ByValue` instead
* of `RustBuffer`. E.g.
* `fun mylib_get_stuff(some: X, args: Y): RustBuffer.ByValue`
* for the function returning the RustBuffer, and
* `fun mylib_destroy_bytebuffer(bb: RustBuffer.ByValue)`.
*/
open class RustBuffer : Structure() {
@JvmField var len: Long = 0
@JvmField var data: Pointer? = null
init {
read()
}
override fun getFieldOrder(): List<String> {
return Arrays.asList("len", "data")
}
fun asCodedInputStream(): CodedInputStream? {
return this.data?.let {
CodedInputStream.newInstance(it.getByteBuffer(0, this.len))
}
}
class ByValue : RustBuffer(), Structure.ByValue
}

Просмотреть файл

@ -0,0 +1,403 @@
# Using protobuf-encoded data over Rust FFI.
This assumes you already have your FFI mostly set up. If you don't that part
should be covered by another document, which may or may not exist yet (at the time of this writing, it does not).
Most of this is concerned with how to do it the first time as well. If your rust
component already is returning protobuf-encoded data, you probably just need to
follow the examples of the other steps it takes.
## Rust Changes
1. To your main rust crate, add dependencies on the `prost`, `prost-derive`, and
`bytes` crates.
2. Add `features = ["prost_support"]` to the `ffi_support` dependency.
3. Add `prost-build` to your build dependencies (e.g. you probably have to add
both of these):
```toml
[build-dependencies]
prost-build = "check what version our other crates are using"
```
4. In the same directory as your main crate's Cargo.toml, add a `build.rs` file.
Paste the following into it:
```rust
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
fn main() {
prost_build::compile_protos(&["src/msg_types.proto"], &["src/"]).unwrap();
}
```
5. Create a new file named `msg_types.proto` in your main crate's src folder.
This is what is referenced in that bit you pasted above, so if you want or
need to change its name, you must do so consistently.
1. This file should start with
```
syntax = "proto2";
package msg_types;
```
The package name is going to determine where the .rs file is output, which
will be relevant shortly.
2. Fill in your definitions in the rest of the file. See
https://developers.google.com/protocol-buffers/docs/proto for examples.
6. Into your main crate's lib.rs file, add something equivalent to the following:
```rust
pub mod msg_types {
use prost_derive::Message;
include!(concat!(env!("OUT_DIR"), "/msg_types.rs"));
}
```
This exposes the file your `build.rs` generates (from the .proto file) as a
rust module.
7. Open your main crates's src/ffi.rs (note: *not* ffi/src/lib.rs! We'll get
there shortly!)
For each type you declare in your .proto file, first decide if you want to
use this as the primary type to represent this data, or if you want to convert
it from a more idiomatic Rust type into the message type when returning.
If it's something that exists solely to return over the FFI, or you may have
a large number of them (or if you need to return them in an array, see the
FAQ question on this) it *may* be best to just use the type from msg_types in your rust code.
We'll what you do in both cases. The parts only relevant if you are converting
between a rust type and the protobuf start with "*(optional unless converting types)*".
Note that if your canonical rust type is defined in another crate, or if it's
something like `Vec<T>`, you will need to use a wrapper. See the FAQ question
on `Vec<T>` about this.
1. *(optional unless converting types)* Define the conversion between the
idiomatic Rust type and the type produced from `msg_types`. This will
likely look something like this:
```rust
impl From<HistoryVisitInfo> for msg_types::HistoryVisitInfo {
fn from(hvi: HistoryVisitInfo) -> Self {
Self {
// convert url::Url to String
url: hvi.url.into_string(),
// Title is already an Option<String>
title: hvi.title,
// Convert Timestamp to i64
timestamp: hvi.title.0 as i64,
// Convert rust enum to i32
visit_type: hvi.visit_type as i32,
}
}
}
```
2. Add a call to:
```rust
ffi_support::implement_into_ffi_by_protobuf!(msg_types::MyType);
```
3. *(optional unless converting types)* Add a call to
```rust
ffi_support::implement_into_ffi_by_delegation!(MyType, msg_types::MyType);
```
If `MyType` is something that you were previously returning via JSON, you need
to remove the call to `implement_into_ffi_by_json!`, and you may also want to
delete `Serialize` from it's `#[derive(...)]` while you're at it, unless you
still need it.
8. In your ffi crate's lib.rs, make the following changes:
1. Any function that conceptually returns a protobuf type must now return
`ffi_support::ByteBuffer` (if it returned via JSON before, this should be
a change of `-> *mut c_char` to `-> ByteBuffer`).
2. You must add a call to
`ffi_support::define_bytebuffer_destructor!(mylib_destroy_bytebuffer)`.
The name you chose for `mylib_destroy_bytebuffer` **must not** collide with the name anybody else uses for this.
## Kotlin Changes
1. Inside your component's build.gradle (e.g.
`components/mything/android/build.gradle`, not the top level one):
1. Add `apply plugin: 'com.google.protobuf'` to the top of the file.
2. Into the `android { ... }` block, add:
```groovy
sourceSets {
main {
proto {
srcDir '../src'
}
}
}
```
3. Add a new top level block:
```groovy
protobuf {
protoc {
artifact = 'com.google.protobuf:protoc:3.0.0'
}
plugins {
javalite {
artifact = 'com.google.protobuf:protoc-gen-javalite:3.0.0'
}
}
generateProtoTasks {
all().each { task ->
task.builtins {
remove java
}
task.plugins {
javalite { }
}
}
}
}
```
4. Add the following to your dependencies:
```groovy
implementation 'com.google.protobuf:protobuf-lite:3.0.0'
implementation project(':as-support-library')
```
2. Add a new file, ByteBuffer.kt:
In the future we will share this class (for once we actually could!) however,
at the moment we cannot, and so you must copy/paste it.
3. In the file where the foreign functions are defined, make sure that the
function returning this type returns a `RustBuffer.ByValue` (`RustBuffer` is
in `mozilla.appservices.support`).
Additionally, add a declaration for `mylib_destroy_bytebuffer` (the name must match what was used in the `ffi/src/lib.rs` above). This should look like:
```kotlin
fun mylib_destroy_bytebuffer(v: RustBuffer.ByValue)
```
4. Usage code then looks as follows:
```kotlin
val rustBuffer = rustCall { error ->
MyLibFFI.INSTANCE.call_thing_returning_rustbuffer(...)
}
try {
val message = MsgTypes.SomeMessageData.parseFrom(
infoBuffer.asCodedInputStream()!!)
// use `message` to produce the higher level type you want to return.
} finally {
LibPlacesFFI.INSTANCE.mylib_destroy_bytebuffer(infoBuffer)
}
```
## Swift
Someone should document me! Until then, taking a look at the changes that were
made for FxA in https://github.com/mozilla/application-services/pull/626 is not
a bad first step! Also, ask in #rust-components on slack.
# Using protobuf to pass data *into* Rust code
## Kotlin/Android
Don't pass `ffi_support::ByteBuffer`/`RustBuffer` into rust.
It is a type for going in the other direction.
Instead, you should pass the data and length separately. There are two ways of
doing this for android. You can use either a `Array<Byte>` or a `Pointer`,
which you can get from a "direct" `java.nio.ByteBuffer`. We recommend the
latter, as it avoids an additional copy, which can be done as follows (using
the `toNioDirectBuffer` our kotlin support library provides):
In Kotlin:
```kotlin
// In the com.sun.jna.Library
fun rust_fun_taking_protobuf(data: Pointer, len: Int, out: RustError.ByReference)
// In some your wrapper (note: `toNioDirectBuffer` is defined by our
// support library)
val (len, nioBuf) = theProtobufType.toNioDirectBuffer()
rustCall { err ->
val ptr = Native.getDirectBufferPointer(nioBuf)
MyLib.INSTANCE.rust_fun_taking_protobuf(ptr, len, err)
}
```
Note that the `toNioDirectBuffer` helper can't return the Pointer directly, as
it is only valid until the NIO buffer is garbage collected, and if the pointer
were returned it would not be reachable.
In Rust:
```rust
#[no_mangle]
pub unsafe extern "C" fn rust_fun_taking_protobuf(
data *const u8,
len: i32,
error: &mut ExternError,
) {
// Or another call_with_blah function as needed
ffi_support::call_with_result(error, || {
// TODO: We should find a way to share some of this boilerplate
assert!(len >= 0, "Bad buffer len: {}", len);
let bytes = if len == 0 {
// This will still fail, but as a bad protobuf format.
&[]
} else {
assert!(!data.is_null(), "Unexpected null data pointer");
std::slice::from_raw_parts(data, len as usize)
};
let my_thing: MyMsgType = prost::Message::decode(bytes)?;
// Do stuff with my_thing...
Ok(())
})
}
```
## Swift
Someone should document me! Until then, taking a look at the changes that were
made for FxA in https://github.com/mozilla/application-services/pull/626 is not
a bad first step! Also, ask in #rust-components on slack.
# FAQ
### What are the downsides of using types from `msg_types.proto` heavily?
1. It doesn't lead to particularly idiomatic Rust code.
2. We loose the ability to enforce many type invariants that we'd like. For
example, we cannot declare that a field holds a `Url`, and must use a
`String` instead.
### I'd like to expose a function returning a `Vec<T>`.
If T is a type from msg_types.proto, then this is fairly easy:
Don't, instead add a new msg_type that contains a repeated T field, and make
that rust function return that.
Then, make so long as the new msg_type has `implement_into_ffi_by_protobuf!` and the ffi function returns a ByteBuffer, things should "Just Work".
---
Unfortunately, if T is merely *convertable* to something from msg_types.proto,
this adds a bunch of boilerplate.
Say we have the following msg_types.proto:
```proto
message HistoryVisitInfo {
required string url = 1;
optional string title = 2;
required int64 timestamp = 3;
required int32 visit_type = 4;
}
message HistoryVisitInfos {
repeated HistoryVisitInfo infos = 1;
}
```
in src/ffi.rs, we then need
```rust
// Convert from idiomatic rust HistoryVisitInfo to msg_type HistoryVisitInfo
impl From<HistoryVisitInfo> for msg_types::HistoryVisitInfo {
fn from(hvi: HistoryVisitInfo) -> Self {
Self {
url: hvi.url,
title: hvi.title,
timestamp: hvi.title.0 as i64,
visit_type: hvi.visit_type as i32,
}
}
}
// Declare a type that exists to wrap the vec (see the next question about
// why this is needed)
pub struct HistoryVisitInfos(pub Vec<HistoryVisitInfo>);
// Define the conversion between said wrapper and the protobuf
// HistoryVisitInfos
impl From<HistoryVisitInfos> for msg_types::HistoryVisitInfos {
fn from(hvis: HistoryVisitInfos) -> Self {
Self {
infos: hvis.0
.into_iter()
.map(msg_types::HistoryVisitInfo::from)
.collect()
}
}
}
// generate the IntoFfi for msg_types::HistoryVisitInfos
implement_into_ffi_by_protobuf!(msg_types::HistoryVisitInfos);
// Use it to implement it for HistoryVisitInfos
implement_into_ffi_by_delegation!(HistoryVisitInfos, msg_types::HistoryVisitInfos);
```
Then, in `ffi/src/lib.rs`, where you currently return the Vec, you need to
change it to return wrap that in main_crate::ffi::HistoryVisitInfos, something like
```rust
CONNECTIONS.call_with_result(error, handle, |conn| -> places::Result<_> {
Ok(HistoryVisitInfos(storage::history::get_visit_infos(
conn,
places::Timestamp(start_date.max(0) as u64),
places::Timestamp(end_date.max(0) as u64),
)?))
})
```
### Why is that so painful?
Yep. There are a few reasons for this.
`ffi_support` is the only one who is in a position to decide how a `Vec<T>` is
returned over the FFI. Rust has a rule that either the trait (in this case
`IntoFfi`) or the type (in this case `Vec`) must be implemented in the crate
where the `impl` block happens. This is known as the orphan rule.
Additionally, until rust gains support for
[specialization](https://github.com/rust-lang/rust/issues/31844), we have very
little flexibility with how this works. We can't implement it one way for some
kinds of T's, and another way for others (however, we can, and do, make it
opt-in, but that's unrelated).
This means ffi_support is in the position of deciding how `Vec<T>` goes over the
FFI for all T. At one point, the reasonable choice seemed to be JSON. This is
still used fairly heavily for returning arrays of things, and so until we move
*everything* to use protobufs, we don't really want to take that out.
Unfortunately even we no longer use JSON for this, the conversion between
`Vec<T>` and the ByteBuffer has to happen through an intermediate type, due to
the way protobuf messages work (you can't have a message that's an array, but
you *can* have one that is a single item type which contains a repeated array),
and it isn't clear how to make this work (it can't be an argument to a macro, as
that would violate the orphan rule).
The only thing that would work is if we use the types generated by prost for more
than just returning things over the FFI. e.g. the rust `get_visit_infos()` call would return `HistoryVisitInfos` struct that is generated from a `.proto` file.
#### Could this be worked around by using length-delimited protobuf messages?
Yes, possibly. Looking into this is something we may do in the future.
### Why is the module produced from .proto `msg_types` and not `ffi_types`?
We use `msg_types` and not e.g. `ffi_types`, since in some cases (see the next
FAQ about returning arrays, for example) it can reduce boilerplate a lot to use
these for returning the data to rust code directly (particularly when the rust
API exists almost exclusively to be called from the FFI).
Using a name like `ffi_types`, while possibly intuitive, gives the impression
that these types should not be used outside the FFI, and that it may even be
unsafe to do so.

Просмотреть файл

@ -23,7 +23,7 @@
EB879D64221231F400753DC9 /* CommonErrors.swift in Sources */ = {isa = PBXBuildFile; fileRef = EB879D4D221231F400753DC9 /* CommonErrors.swift */; };
EB879D7F221234EB00753DC9 /* MozillaAppServices.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = CE9D202020914D0D00F1C8FA /* MozillaAppServices.framework */; };
EB879D8B22123FD900753DC9 /* MozillaAppServicesTest.swift in Sources */ = {isa = PBXBuildFile; fileRef = EB879D8A22123FD900753DC9 /* MozillaAppServicesTest.swift */; };
EBB0D55E2214D10A00C8B2F9 /* ffi_types.proto in Sources */ = {isa = PBXBuildFile; fileRef = EBB0D55D2214D10900C8B2F9 /* ffi_types.proto */; };
EBB0D55E2214D10A00C8B2F9 /* msg_types.proto in Sources */ = {isa = PBXBuildFile; fileRef = EBB0D55D2214D10900C8B2F9 /* msg_types.proto */; };
/* End PBXBuildFile section */
/* Begin PBXBuildRule section */
@ -76,7 +76,7 @@
EBA8770621F5FB9A004F63F0 /* base.xcconfig */ = {isa = PBXFileReference; lastKnownFileType = text.xcconfig; path = base.xcconfig; sourceTree = "<group>"; };
EBA8770721F5FB9A004F63F0 /* debug.xcconfig */ = {isa = PBXFileReference; lastKnownFileType = text.xcconfig; path = debug.xcconfig; sourceTree = "<group>"; };
EBA8770821F5FB9A004F63F0 /* release.xcconfig */ = {isa = PBXFileReference; lastKnownFileType = text.xcconfig; path = release.xcconfig; sourceTree = "<group>"; };
EBB0D55D2214D10900C8B2F9 /* ffi_types.proto */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.protobuf; name = ffi_types.proto; path = ../../src/ffi_types.proto; sourceTree = "<group>"; };
EBB0D55D2214D10900C8B2F9 /* msg_types.proto */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.protobuf; name = msg_types.proto; path = ../../src/msg_types.proto; sourceTree = "<group>"; };
/* End PBXFileReference section */
/* Begin PBXFrameworksBuildPhase section */
@ -145,7 +145,7 @@
C852EEDA220A2A2B00A6E79A /* FxAClient */ = {
isa = PBXGroup;
children = (
EBB0D55D2214D10900C8B2F9 /* ffi_types.proto */,
EBB0D55D2214D10900C8B2F9 /* msg_types.proto */,
C852EEDB220A2A2B00A6E79A /* Rust */,
C852EEDD220A2A2B00A6E79A /* FirefoxAccount.swift */,
C852EEDE220A2A2B00A6E79A /* Extensions */,
@ -369,7 +369,7 @@
C852EEE8220A2A2B00A6E79A /* String+Free_FxAClient.swift in Sources */,
C852EEEB220A2A2B00A6E79A /* FxAError.swift in Sources */,
C852EED7220A29FE00A6E79A /* LoginStoreError.swift in Sources */,
EBB0D55E2214D10A00C8B2F9 /* ffi_types.proto in Sources */,
EBB0D55E2214D10A00C8B2F9 /* msg_types.proto in Sources */,
EB7DE84F2214D39600E7CF17 /* RustProtobuf.swift in Sources */,
EB879D64221231F400753DC9 /* CommonErrors.swift in Sources */,
C852EEE6220A2A2B00A6E79A /* RustPointer.swift in Sources */,