mozilla
/
application-services
зеркало из
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Bug 1876211 - Add MDN for Suggest

This commit is contained in:
Nan Jiang 2024-01-30 16:33:23 -05:00 коммит произвёл Nan Jiang
Родитель dba564f2c5
Коммит f85fcae0bf
7 изменённых файлов: 354 добавлений и 10 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

151
components/suggest/src/db.rs
Просмотреть файл

@ -14,7 +14,7 @@ use rusqlite::{
};
use sql_support::{open_database::open_database_with_flags, ConnExt};
use crate::rs::{DownloadedAmoSuggestion, DownloadedPocketSuggestion};
use crate::rs::{DownloadedAmoSuggestion, DownloadedMdnSuggestion, DownloadedPocketSuggestion};
use crate::{
keyword::full_keyword,
pocket::{split_keyword, KeywordConfidence},
@ -138,6 +138,7 @@ impl<'a> SuggestDao<'a> {
SuggestionProvider::Amo => self.fetch_amo_suggestions(query),
SuggestionProvider::Pocket => self.fetch_pocket_suggestions(query),
SuggestionProvider::Yelp => self.fetch_yelp_suggestions(query),
SuggestionProvider::Mdn => self.fetch_mdn_suggestions(query),
}?;
acc.extend(suggestions);
Ok(acc)
@ -356,6 +357,76 @@ impl<'a> SuggestDao<'a> {
Ok(suggestions)
}
/// Fetches suggestions for MDN
pub fn fetch_mdn_suggestions(&self, query: &SuggestionQuery) -> Result<Vec<Suggestion>> {
let keyword_lowercased = &query.keyword.to_lowercase();
let (keyword_prefix, keyword_suffix) = split_keyword(keyword_lowercased);
let suggestions_limit = &query.limit.unwrap_or(-1);
let suggestions = self
.conn
.query_rows_and_then_cached(
r#"
SELECT
s.id, s.title, s.url, s.provider, s.score, k.keyword_suffix
FROM
suggestions s
JOIN
prefix_keywords k ON k.suggestion_id = s.id
WHERE
k.keyword_prefix = :keyword_prefix
AND
s.provider = :provider
ORDER BY
s.score DESC
LIMIT :suggestions_limit
"#,
named_params! {
":keyword_prefix": keyword_prefix,
":provider": SuggestionProvider::Mdn,
":suggestions_limit": suggestions_limit,
},
|row| -> Result<Option<Suggestion>> {
let suggestion_id: i64 = row.get("id")?;
let title = row.get("title")?;
let raw_url = row.get::<_, String>("url")?;
let score = row.get::<_, f64>("score")?;
let full_suffix = row.get::<_, String>("keyword_suffix")?;
full_suffix
.starts_with(keyword_suffix)
.then(|| {
self.conn.query_row_and_then(
r#"
SELECT
description
FROM
mdn_custom_details
WHERE
suggestion_id = :suggestion_id
"#,
named_params! {
":suggestion_id": suggestion_id
},
|row| {
Ok(Suggestion::Mdn {
title,
url: raw_url,
description: row.get("description")?,
score,
})
},
)
})
.transpose()
},
)?
.into_iter()
.flatten()
.collect();
Ok(suggestions)
}
/// Inserts all suggestions from a downloaded AMO attachment into
/// the database.
pub fn insert_amo_suggestions(
@ -637,6 +708,84 @@ impl<'a> SuggestDao<'a> {
Ok(())
}
/// Inserts all suggestions from a downloaded MDN attachment into
/// the database.
pub fn insert_mdn_suggestions(
&mut self,
record_id: &SuggestRecordId,
suggestions: &[DownloadedMdnSuggestion],
) -> Result<()> {
for suggestion in suggestions {
self.scope.err_if_interrupted()?;
let suggestion_id: i64 = self.conn.query_row_and_then_cachable(
&format!(
"INSERT INTO suggestions(
record_id,
provider,
title,
url,
score
)
VALUES(
:record_id,
{},
:title,
:url,
:score
)
RETURNING id",
SuggestionProvider::Mdn as u8
),
named_params! {
":record_id": record_id.as_str(),
":title": suggestion.title,
":url": suggestion.url,
":score": suggestion.score,
},
|row| row.get(0),
true,
)?;
self.conn.execute_cached(
"INSERT INTO mdn_custom_details(
suggestion_id,
description
)
VALUES(
:suggestion_id,
:description
)",
named_params! {
":suggestion_id": suggestion_id,
":description": suggestion.description,
},
)?;
for (index, keyword) in suggestion.keywords.iter().enumerate() {
let (keyword_prefix, keyword_suffix) = split_keyword(keyword);
self.conn.execute_cached(
"INSERT INTO prefix_keywords(
keyword_prefix,
keyword_suffix,
suggestion_id,
rank
)
VALUES(
:keyword_prefix,
:keyword_suffix,
:suggestion_id,
:rank
)",
named_params! {
":keyword_prefix": keyword_prefix,
":keyword_suffix": keyword_suffix,
":rank": index,
":suggestion_id": suggestion_id,
},
)?;
}
}
Ok(())
}
/// Inserts or replaces an icon for a suggestion into the database.
pub fn put_icon(&mut self, icon_id: &str, data: &[u8]) -> Result<()> {
self.conn.execute(

2
components/suggest/src/provider.rs
Просмотреть файл

@ -17,6 +17,7 @@ pub enum SuggestionProvider {
Amo = 3,
Pocket = 4,
Yelp = 5,
Mdn = 6,
}
impl FromSql for SuggestionProvider {
@ -38,6 +39,7 @@ impl SuggestionProvider {
3 => Some(SuggestionProvider::Amo),
4 => Some(SuggestionProvider::Pocket),
5 => Some(SuggestionProvider::Yelp),
6 => Some(SuggestionProvider::Mdn),
_ => None,
}
}

12
components/suggest/src/rs.rs
Просмотреть файл

@ -92,6 +92,8 @@ pub(crate) enum SuggestRecord {
Pocket,
#[serde(rename = "yelp-suggestions")]
Yelp,
#[serde(rename = "mdn-suggestions")]
Mdn,
}
/// Represents either a single value, or a list of values. This is used to
@ -289,3 +291,13 @@ pub(crate) struct DownloadedYelpSuggestion {
#[serde(rename = "yelpModifiers")]
pub yelp_modifiers: Vec<String>,
}
/// An MDN suggestion to ingest from an attachment
#[derive(Clone, Debug, Deserialize)]
pub(crate) struct DownloadedMdnSuggestion {
pub url: String,
pub title: String,
pub description: String,
pub keywords: Vec<String>,
pub score: f64,
}

10
components/suggest/src/schema.rs
Просмотреть файл

@ -6,7 +6,7 @@
use rusqlite::{Connection, Transaction};
use sql_support::open_database::{self, ConnectionInitializer};
pub const VERSION: u32 = 11;
pub const VERSION: u32 = 12;
pub const SQL: &str = "
CREATE TABLE meta(
@ -91,6 +91,12 @@ pub const SQL: &str = "
need_location INTEGER NOT NULL,
record_id TEXT NOT NULL
) WITHOUT ROWID;
CREATE TABLE mdn_custom_details(
suggestion_id INTEGER PRIMARY KEY,
description TEXT NOT NULL,
FOREIGN KEY(suggestion_id) REFERENCES suggestions(id) ON DELETE CASCADE
);
";
/// Initializes an SQLite connection to the Suggest database, performing
@ -121,7 +127,7 @@ impl ConnectionInitializer for SuggestConnectionInitializer {
fn upgrade_from(&self, _db: &Transaction<'_>, version: u32) -> open_database::Result<()> {
match version {
1..=10 => {
1..=11 => {
// These schema versions were used during development, and never
// shipped in any applications. Treat these databases as
// corrupt, so that they'll be replaced.

174
components/suggest/src/store.rs
Просмотреть файл

@ -342,6 +342,15 @@ where
},
)?;
}
SuggestRecord::Mdn => {
self.ingest_suggestions_from_record(
writer,
record,
|dao, record_id, suggestions| {
dao.insert_mdn_suggestions(record_id, suggestions)
},
)?;
}
}
}
Ok(())
@ -1746,6 +1755,17 @@ mod tests {
"hash": "",
"size": 0,
},
}, {
"id": "data-5",
"type": "mdn-suggestions",
"last_modified": 15,
"attachment": {
"filename": "data-5.json",
"mimetype": "application/json",
"location": "data-5.json",
"hash": "",
"size": 0,
},
}, {
"id": "icon-2",
"type": "icon",
@ -1871,6 +1891,18 @@ mod tests {
"yelpModifiers": ["yelp", "yelp keyword"],
}),
)?
.with_data(
"data-5.json",
json!([
{
"description": "Javascript Array",
"url": "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array",
"keywords": ["array javascript", "javascript array", "wildcard"],
"title": "Array",
"score": 0.24
},
]),
)?
.with_icon("icon-2.png", "i-am-an-icon".as_bytes().into())
.with_icon("icon-3.png", "also-an-icon".as_bytes().into());
@ -3278,10 +3310,10 @@ mod tests {
UnparsableRecords(
{
"clippy-2": UnparsableRecord {
schema_version: 11,
schema_version: 12,
},
"fancy-new-suggestions-1": UnparsableRecord {
schema_version: 11,
schema_version: 12,
},
},
),
@ -3347,10 +3379,10 @@ mod tests {
UnparsableRecords(
{
"clippy-2": UnparsableRecord {
schema_version: 11,
schema_version: 12,
},
"fancy-new-suggestions-1": UnparsableRecord {
schema_version: 11,
schema_version: 12,
},
},
),
@ -3454,10 +3486,10 @@ mod tests {
UnparsableRecords(
{
"clippy-2": UnparsableRecord {
schema_version: 11,
schema_version: 12,
},
"fancy-new-suggestions-1": UnparsableRecord {
schema_version: 11,
schema_version: 12,
},
},
),
@ -3476,4 +3508,134 @@ mod tests {
assert_eq!(serde_json::to_value(unparseable_record)?, json!({ "v": 1 }),);
Ok(())
}
#[test]
fn query_mdn() -> anyhow::Result<()> {
before_each();
let snapshot = Snapshot::with_records(json!([{
"id": "data-1",
"type": "mdn-suggestions",
"last_modified": 15,
"attachment": {
"filename": "data-1.json",
"mimetype": "application/json",
"location": "data-1.json",
"hash": "",
"size": 0,
},
}]))?
.with_data(
"data-1.json",
json!([
{
"description": "Javascript Array",
"url": "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array",
"keywords": ["array javascript", "javascript array", "wildcard"],
"title": "Array",
"score": 0.24
},
]),
)?;
let store = unique_test_store(SnapshotSettingsClient::with_snapshot(snapshot));
store.ingest(SuggestIngestionConstraints::default())?;
let table = [
(
"keyword = prefix; MDN only",
SuggestionQuery {
keyword: "array".into(),
providers: vec![SuggestionProvider::Mdn],
limit: None,
},
expect![[r#"
[
Mdn {
title: "Array",
url: "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array",
description: "Javascript Array",
score: 0.24,
},
]
"#]],
),
(
"keyword = prefix + partial suffix; MDN only",
SuggestionQuery {
keyword: "array java".into(),
providers: vec![SuggestionProvider::Mdn],
limit: None,
},
expect![[r#"
[
Mdn {
title: "Array",
url: "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array",
description: "Javascript Array",
score: 0.24,
},
]
"#]],
),
(
"keyword = prefix + entire suffix; MDN only",
SuggestionQuery {
keyword: "javascript array".into(),
providers: vec![SuggestionProvider::Mdn],
limit: None,
},
expect![[r#"
[
Mdn {
title: "Array",
url: "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array",
description: "Javascript Array",
score: 0.24,
},
]
"#]],
),
(
"keyword = `partial prefix word`; MDN only",
SuggestionQuery {
keyword: "wild".into(),
providers: vec![SuggestionProvider::Mdn],
limit: None,
},
expect![[r#"
[]
"#]],
),
(
"keyword = single word; MDN only",
SuggestionQuery {
keyword: "wildcard".into(),
providers: vec![SuggestionProvider::Mdn],
limit: None,
},
expect![[r#"
[
Mdn {
title: "Array",
url: "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array",
description: "Javascript Array",
score: 0.24,
},
]
"#]],
),
];
for (what, query, expect) in table {
expect.assert_debug_eq(
&store
.query(query)
.with_context(|| format!("Couldn't query store for {}", what))?,
);
}
Ok(())
}
}

9
components/suggest/src/suggest.udl
Просмотреть файл

@ -22,7 +22,8 @@ enum SuggestionProvider {
"Pocket",
"Wikipedia",
"Amo",
"Yelp"
"Yelp",
"Mdn",
};
[Enum]
@ -67,6 +68,12 @@ interface Suggestion {
string url,
string title
);
Mdn(
string title,
string url,
string description,
f64 score
);
};
dictionary SuggestionQuery {

6
components/suggest/src/suggestion.rs
Просмотреть файл

@ -59,6 +59,12 @@ pub enum Suggestion {
url: String,
title: String,
},
Mdn {
title: String,
url: String,
description: String,
score: f64,
},
}
impl PartialOrd for Suggestion {