mozilla
/
gecko-dev
зеркало из https://github.com/mozilla/gecko-dev.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Bug 1621018 - Move v2 personalization into promise worker. r=gvn,jonathankoren,mconley 

Differential Revision: https://phabricator.services.mozilla.com/D67171

--HG--
rename : browser/components/newtab/lib/NaiveBayesTextTagger.jsm => browser/components/newtab/lib/PersonalityProvider/NaiveBayesTextTagger.jsm
rename : browser/components/newtab/lib/NmfTextTagger.jsm => browser/components/newtab/lib/PersonalityProvider/NmfTextTagger.jsm
rename : browser/components/newtab/lib/RecipeExecutor.jsm => browser/components/newtab/lib/PersonalityProvider/RecipeExecutor.jsm
rename : browser/components/newtab/lib/Tokenize.jsm => browser/components/newtab/lib/PersonalityProvider/Tokenize.jsm
rename : browser/components/newtab/test/unit/lib/NaiveBayesTextTagger.test.js => browser/components/newtab/test/unit/lib/PersonalityProvider/NaiveBayesTextTagger.test.js
rename : browser/components/newtab/test/unit/lib/NmfTextTagger.test.js => browser/components/newtab/test/unit/lib/PersonalityProvider/NmfTextTagger.test.js
rename : browser/components/newtab/test/unit/lib/RecipeExecutor.test.js => browser/components/newtab/test/unit/lib/PersonalityProvider/RecipeExecutor.test.js
rename : browser/components/newtab/test/unit/lib/Tokenize.test.js => browser/components/newtab/test/unit/lib/PersonalityProvider/Tokenize.test.js
extra : moz-landing-system : lando
This commit is contained in:
Scott 2020-03-27 19:37:22 +00:00
Родитель 5ff55aadfc
Коммит 033996289a
21 изменённых файлов: 1671 добавлений и 1269 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

112
browser/components/newtab/lib/DiscoveryStreamFeed.jsm
Просмотреть файл

@ -566,16 +566,19 @@ this.DiscoveryStreamFeed = class DiscoveryStreamFeed {
});
}
// I wonder, can this be better as a reducer?
// See Bug https://bugzilla.mozilla.org/show_bug.cgi?id=1606717
placementsForEach(callback) {
getPlacements() {
const { placements } = this.store.getState().DiscoveryStream.spocs;
// Backwards comp for before we had placements, assume just a single spocs placement.
if (!placements || !placements.length) {
[{ name: "spocs" }].forEach(callback);
} else {
placements.forEach(callback);
return [{ name: "spocs" }];
}
return placements;
}
// I wonder, can this be better as a reducer?
// See Bug https://bugzilla.mozilla.org/show_bug.cgi?id=1606717
placementsForEach(callback) {
this.getPlacements().forEach(callback);
}
// Bug 1567271 introduced meta data on a list of spocs.
@ -662,7 +665,7 @@ this.DiscoveryStreamFeed = class DiscoveryStreamFeed {
let blockedItems = [];
let belowMinScore = [];
let flightDupes = [];
this.placementsForEach(placement => {
const spocsResultPromises = this.getPlacements().map(async placement => {
const freshSpocs = spocsState.spocs[placement.name];
if (!freshSpocs) {
@ -710,9 +713,10 @@ this.DiscoveryStreamFeed = class DiscoveryStreamFeed {
);
blockedItems = [...blockedItems, ...blocks];
let { data: transformResult, filtered: transformFilter } = this.transform(
blockedResults
);
let {
data: transformResult,
filtered: transformFilter,
} = await this.transform(blockedResults);
let {
below_min_score: minScoreFilter,
flight_duplicate: dupes,
@ -730,6 +734,7 @@ this.DiscoveryStreamFeed = class DiscoveryStreamFeed {
},
};
});
await Promise.all(spocsResultPromises);
sendUpdate({
type: at.DISCOVERY_STREAM_SPOCS_UPDATE,
@ -853,11 +858,11 @@ this.DiscoveryStreamFeed = class DiscoveryStreamFeed {
}
}
scoreItems(items) {
async scoreItems(items) {
const filtered = [];
const scoreStart = perfService.absNow();
const data = items
.map(item => this.scoreItem(item))
const data = (await Promise.all(items.map(item => this.scoreItem(item))))
// Remove spocs that are scored too low.
.filter(s => {
if (s.score >= s.min_score) {
@ -875,14 +880,14 @@ this.DiscoveryStreamFeed = class DiscoveryStreamFeed {
return { data, filtered };
}
scoreItem(item) {
async scoreItem(item) {
item.score = item.item_score;
item.min_score = item.min_score || 0;
if (item.score !== 0 && !item.score) {
item.score = 1;
}
if (this.personalized) {
this.providerSwitcher.calculateItemRelevanceScore(item);
await this.providerSwitcher.calculateItemRelevanceScore(item);
}
return item;
}
@ -908,7 +913,7 @@ this.DiscoveryStreamFeed = class DiscoveryStreamFeed {
return { data, filtered };
}
transform(spocs) {
async transform(spocs) {
if (spocs && spocs.length) {
const spocsPerDomain =
this.store.getState().DiscoveryStream.spocs.spocs_per_domain || 1;
@ -916,10 +921,10 @@ this.DiscoveryStreamFeed = class DiscoveryStreamFeed {
const flightDuplicates = [];
// This order of operations is intended.
// scoreItems must be first because it creates this.score.
const { data: items, filtered: belowMinScoreItems } = this.scoreItems(
spocs
);
const {
data: items,
filtered: belowMinScoreItems,
} = await this.scoreItems(spocs);
// This removes flight dupes.
// We do this only after scoring and sorting because that way
// we can keep the first item we see, and end up keeping the highest scored.
@ -1075,7 +1080,7 @@ this.DiscoveryStreamFeed = class DiscoveryStreamFeed {
if (this.isExpired({ cachedData, key: "feed", url: feedUrl, isStartup })) {
const feedResponse = await this.fetchFromEndpoint(feedUrl);
if (feedResponse) {
const { data: scoredItems } = this.scoreItems(
const { data: scoredItems } = await this.scoreItems(
feedResponse.recommendations
);
const { recsExpireTime } = feedResponse.settings;
@ -1168,42 +1173,43 @@ this.DiscoveryStreamFeed = class DiscoveryStreamFeed {
async scoreFeeds(feedsState) {
if (feedsState.data) {
const feedsResult = Object.keys(feedsState.data).reduce((feeds, url) => {
const feeds = {};
const feedsPromises = Object.keys(feedsState.data).map(url => {
let feed = feedsState.data[url];
const { data: scoredItems } = this.scoreItems(
feed.data.recommendations
);
const { recsExpireTime } = feed.data.settings;
const recommendations = this.rotate(scoredItems, recsExpireTime);
feed = {
...feed,
data: {
...feed.data,
recommendations,
},
};
feeds[url] = feed;
this.store.dispatch(
ac.AlsoToPreloaded({
type: at.DISCOVERY_STREAM_FEED_UPDATE,
const feedPromise = this.scoreItems(feed.data.recommendations);
feedPromise.then(({ data: scoredItems }) => {
const { recsExpireTime } = feed.data.settings;
const recommendations = this.rotate(scoredItems, recsExpireTime);
feed = {
...feed,
data: {
feed,
url,
...feed.data,
recommendations,
},
})
);
return feeds;
}, {});
await this.cache.set("feeds", feedsResult);
};
feeds[url] = feed;
this.store.dispatch(
ac.AlsoToPreloaded({
type: at.DISCOVERY_STREAM_FEED_UPDATE,
data: {
feed,
url,
},
})
);
});
return feedPromise;
});
await Promise.all(feedsPromises);
await this.cache.set("feeds", feeds);
}
}
async scoreSpocs(spocsState) {
let belowMinScore = [];
this.placementsForEach(placement => {
const spocsResultPromises = this.getPlacements().map(async placement => {
const nextSpocs = spocsState.data[placement.name] || {};
const { items } = nextSpocs;
@ -1211,9 +1217,10 @@ this.DiscoveryStreamFeed = class DiscoveryStreamFeed {
return;
}
const { data: scoreResult, filtered: minScoreFilter } = this.scoreItems(
items
);
const {
data: scoreResult,
filtered: minScoreFilter,
} = await this.scoreItems(items);
belowMinScore = [...belowMinScore, ...minScoreFilter];
@ -1225,6 +1232,7 @@ this.DiscoveryStreamFeed = class DiscoveryStreamFeed {
},
};
});
await Promise.all(spocsResultPromises);
// Update cache here so we don't need to re calculate scores on loads from cache.
// Related Bug 1606276

497
browser/components/newtab/lib/PersonalityProvider.jsm
Просмотреть файл

@ -1,497 +0,0 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
"use strict";
const { RemoteSettings } = ChromeUtils.import(
"resource://services-settings/remote-settings.js"
);
const { actionCreators: ac } = ChromeUtils.import(
"resource://activity-stream/common/Actions.jsm"
);
ChromeUtils.defineModuleGetter(
this,
"perfService",
"resource://activity-stream/common/PerfService.jsm"
);
const { NaiveBayesTextTagger } = ChromeUtils.import(
"resource://activity-stream/lib/NaiveBayesTextTagger.jsm"
);
const { NmfTextTagger } = ChromeUtils.import(
"resource://activity-stream/lib/NmfTextTagger.jsm"
);
const { RecipeExecutor } = ChromeUtils.import(
"resource://activity-stream/lib/RecipeExecutor.jsm"
);
ChromeUtils.defineModuleGetter(
this,
"NewTabUtils",
"resource://gre/modules/NewTabUtils.jsm"
);
const { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm");
const { XPCOMUtils } = ChromeUtils.import(
"resource://gre/modules/XPCOMUtils.jsm"
);
ChromeUtils.defineModuleGetter(this, "OS", "resource://gre/modules/osfile.jsm");
XPCOMUtils.defineLazyGlobalGetters(this, ["fetch"]);
XPCOMUtils.defineLazyGetter(this, "gTextDecoder", () => new TextDecoder());
XPCOMUtils.defineLazyGetter(this, "baseAttachmentsURL", async () => {
const server = Services.prefs.getCharPref("services.settings.server");
const serverInfo = await (
await fetch(`${server}/`, {
credentials: "omit",
})
).json();
const {
capabilities: {
attachments: { base_url },
},
} = serverInfo;
return base_url;
});
const PERSONALITY_PROVIDER_DIR = OS.Path.join(
OS.Constants.Path.localProfileDir,
"personality-provider"
);
const RECIPE_NAME = "personality-provider-recipe";
const MODELS_NAME = "personality-provider-models";
function getHash(aStr) {
// return the two-digit hexadecimal code for a byte
let toHexString = charCode => `0${charCode.toString(16)}`.slice(-2);
let hasher = Cc["@mozilla.org/security/hash;1"].createInstance(
Ci.nsICryptoHash
);
hasher.init(Ci.nsICryptoHash.SHA256);
let stringStream = Cc["@mozilla.org/io/string-input-stream;1"].createInstance(
Ci.nsIStringInputStream
);
stringStream.data = aStr;
hasher.updateFromStream(stringStream, -1);
// convert the binary hash data to a hex string.
let binary = hasher.finish(false);
return Array.from(binary, (c, i) => toHexString(binary.charCodeAt(i)))
.join("")
.toLowerCase();
}
/**
* V2 provider builds and ranks an interest profile (also called an “interest vector”) off the browse history.
* This allows Firefox to classify pages into topics, by examining the text found on the page.
* It does this by looking at the history text content, title, and description.
*/
this.PersonalityProvider = class PersonalityProvider {
constructor(
timeSegments,
parameterSets,
maxHistoryQueryResults,
version,
scores,
v2Params
) {
this.v2Params = v2Params || {};
this.dispatch = this.v2Params.dispatch || (() => {});
this.modelKeys = this.v2Params.modelKeys;
this.timeSegments = timeSegments;
this.parameterSets = parameterSets;
this.maxHistoryQueryResults = maxHistoryQueryResults;
this.version = version;
this.scores = scores || {};
this.interestConfig = this.scores.interestConfig;
this.interestVector = this.scores.interestVector;
this.onSync = this.onSync.bind(this);
this.setup();
}
setup() {
this.setupSyncAttachment(RECIPE_NAME);
this.setupSyncAttachment(MODELS_NAME);
}
teardown() {
this.teardownSyncAttachment(RECIPE_NAME);
this.teardownSyncAttachment(MODELS_NAME);
}
async onSync(event) {
const {
data: { created, updated, deleted },
} = event;
// Remove every removed attachment.
const toRemove = deleted.concat(updated.map(u => u.old));
await Promise.all(toRemove.map(record => this.deleteAttachment(record)));
// Download every new/updated attachment.
const toDownload = created.concat(updated.map(u => u.new));
await Promise.all(
toDownload.map(record => this.maybeDownloadAttachment(record))
);
}
setupSyncAttachment(collection) {
RemoteSettings(collection).on("sync", this.onSync);
}
teardownSyncAttachment(collection) {
RemoteSettings(collection).off("sync", this.onSync);
}
/**
* Downloads the attachment to disk assuming the dir already exists
* and any existing files matching the filename are clobbered.
*/
async _downloadAttachment(record) {
const {
attachment: { location, filename },
} = record;
const remoteFilePath = (await baseAttachmentsURL) + location;
const localFilePath = OS.Path.join(PERSONALITY_PROVIDER_DIR, filename);
const headers = new Headers();
headers.set("Accept-Encoding", "gzip");
const resp = await fetch(remoteFilePath, { headers, credentials: "omit" });
if (!resp.ok) {
Cu.reportError(`Failed to fetch ${remoteFilePath}: ${resp.status}`);
return;
}
const buffer = await resp.arrayBuffer();
const bytes = new Uint8Array(buffer);
await OS.File.writeAtomic(localFilePath, bytes, {
tmpPath: `${localFilePath}.tmp`,
});
}
/**
* Attempts to download the attachment, but only if it doesn't already exist.
*/
async maybeDownloadAttachment(record, retries = 3) {
const {
attachment: { filename, hash, size },
} = record;
await OS.File.makeDir(PERSONALITY_PROVIDER_DIR);
const localFilePath = OS.Path.join(PERSONALITY_PROVIDER_DIR, filename);
let retry = 0;
while (
retry++ < retries &&
(!(await OS.File.exists(localFilePath)) ||
(await OS.File.stat(localFilePath)).size !== size ||
getHash(await this._getFileStr(localFilePath)) !== hash)
) {
await this._downloadAttachment(record);
}
}
async deleteAttachment(record) {
const {
attachment: { filename },
} = record;
await OS.File.makeDir(PERSONALITY_PROVIDER_DIR);
const path = OS.Path.join(PERSONALITY_PROVIDER_DIR, filename);
await OS.File.remove(path, { ignoreAbsent: true });
return OS.File.removeEmptyDir(PERSONALITY_PROVIDER_DIR, {
ignoreAbsent: true,
});
}
/**
* Gets contents of the attachment if it already exists on file,
* and if not attempts to download it.
*/
async getAttachment(record) {
const {
attachment: { filename },
} = record;
const filepath = OS.Path.join(PERSONALITY_PROVIDER_DIR, filename);
try {
await this.maybeDownloadAttachment(record);
return JSON.parse(await this._getFileStr(filepath));
} catch (error) {
Cu.reportError(`Failed to load ${filepath}: ${error.message}`);
}
return {};
}
// A helper function to read and decode a file, it isn't a stand alone function.
// If you use this, ensure you check the file exists and you have a try catch.
async _getFileStr(filepath) {
const binaryData = await OS.File.read(filepath);
return gTextDecoder.decode(binaryData);
}
async init(callback) {
const perfStart = perfService.absNow();
this.interestConfig = this.interestConfig || (await this.getRecipe());
if (!this.interestConfig) {
this.dispatch(
ac.PerfEvent({ event: "PERSONALIZATION_V2_GET_RECIPE_ERROR" })
);
return;
}
this.recipeExecutor = await this.generateRecipeExecutor();
if (!this.recipeExecutor) {
this.dispatch(
ac.PerfEvent({
event: "PERSONALIZATION_V2_GENERATE_RECIPE_EXECUTOR_ERROR",
})
);
return;
}
this.interestVector =
this.interestVector || (await this.createInterestVector());
if (!this.interestVector) {
this.dispatch(
ac.PerfEvent({
event: "PERSONALIZATION_V2_CREATE_INTEREST_VECTOR_ERROR",
})
);
return;
}
this.dispatch(
ac.PerfEvent({
event: "PERSONALIZATION_V2_TOTAL_DURATION",
value: Math.round(perfService.absNow() - perfStart),
})
);
this.initialized = true;
if (callback) {
callback();
}
}
dispatchRelevanceScoreDuration(start) {
// If v2 is not yet initialized we don't bother tracking yet.
// Before it is initialized it doesn't do any ranking.
// Once it's initialized it ensures ranking is done.
// v1 doesn't have any initialized issues around ranking,
// and should be ready right away.
if (this.initialized) {
this.dispatch(
ac.PerfEvent({
event: "PERSONALIZATION_V2_ITEM_RELEVANCE_SCORE_DURATION",
value: Math.round(perfService.absNow() - start),
})
);
}
}
async getFromRemoteSettings(name) {
const result = await RemoteSettings(name).get();
return Promise.all(
result.map(async record => ({
...(await this.getAttachment(record)),
recordKey: record.key,
}))
);
}
/**
* Returns a Recipe from remote settings to be consumed by a RecipeExecutor.
* A Recipe is a set of instructions on how to processes a RecipeExecutor.
*/
async getRecipe() {
if (!this.recipes || !this.recipes.length) {
const start = perfService.absNow();
this.recipes = await this.getFromRemoteSettings(RECIPE_NAME);
this.dispatch(
ac.PerfEvent({
event: "PERSONALIZATION_V2_GET_RECIPE_DURATION",
value: Math.round(perfService.absNow() - start),
})
);
}
return this.recipes[0];
}
/**
* Returns a Recipe Executor.
* A Recipe Executor is a set of actions that can be consumed by a Recipe.
* The Recipe determines the order and specifics of which the actions are called.
*/
async generateRecipeExecutor() {
if (!this.taggers) {
const startTaggers = perfService.absNow();
let nbTaggers = [];
let nmfTaggers = {};
const models = await this.getFromRemoteSettings(MODELS_NAME);
if (models.length === 0) {
return null;
}
for (let model of models) {
if (!this.modelKeys.includes(model.recordKey)) {
continue;
}
if (model.model_type === "nb") {
nbTaggers.push(new NaiveBayesTextTagger(model));
} else if (model.model_type === "nmf") {
nmfTaggers[model.parent_tag] = new NmfTextTagger(model);
}
}
this.dispatch(
ac.PerfEvent({
event: "PERSONALIZATION_V2_TAGGERS_DURATION",
value: Math.round(perfService.absNow() - startTaggers),
})
);
this.taggers = { nbTaggers, nmfTaggers };
}
const startRecipeExecutor = perfService.absNow();
const recipeExecutor = new RecipeExecutor(
this.taggers.nbTaggers,
this.taggers.nmfTaggers
);
this.dispatch(
ac.PerfEvent({
event: "PERSONALIZATION_V2_RECIPE_EXECUTOR_DURATION",
value: Math.round(perfService.absNow() - startRecipeExecutor),
})
);
return recipeExecutor;
}
/**
* Grabs a slice of browse history for building a interest vector
*/
async fetchHistory(columns, beginTimeSecs, endTimeSecs) {
let sql = `SELECT url, title, visit_count, frecency, last_visit_date, description
FROM moz_places
WHERE last_visit_date >= ${beginTimeSecs * 1000000}
AND last_visit_date < ${endTimeSecs * 1000000}`;
columns.forEach(requiredColumn => {
sql += ` AND IFNULL(${requiredColumn}, '') <> ''`;
});
sql += " LIMIT 30000";
const { activityStreamProvider } = NewTabUtils;
const history = await activityStreamProvider.executePlacesQuery(sql, {
columns,
params: {},
});
return history;
}
/**
* Examines the user's browse history and returns an interest vector that
* describes the topics the user frequently browses.
*/
async createInterestVector() {
let interestVector = {};
let endTimeSecs = new Date().getTime() / 1000;
let beginTimeSecs = endTimeSecs - this.interestConfig.history_limit_secs;
let history = await this.fetchHistory(
this.interestConfig.history_required_fields,
beginTimeSecs,
endTimeSecs
);
this.dispatch(
ac.PerfEvent({
event: "PERSONALIZATION_V2_HISTORY_SIZE",
value: history.length,
})
);
const start = perfService.absNow();
for (let historyRec of history) {
let ivItem = this.recipeExecutor.executeRecipe(
historyRec,
this.interestConfig.history_item_builder
);
if (ivItem === null) {
continue;
}
interestVector = this.recipeExecutor.executeCombinerRecipe(
interestVector,
ivItem,
this.interestConfig.interest_combiner
);
if (interestVector === null) {
return null;
}
}
const finalResult = this.recipeExecutor.executeRecipe(
interestVector,
this.interestConfig.interest_finalizer
);
this.dispatch(
ac.PerfEvent({
event: "PERSONALIZATION_V2_CREATE_INTEREST_VECTOR_DURATION",
value: Math.round(perfService.absNow() - start),
})
);
return finalResult;
}
/**
* Calculates a score of a Pocket item when compared to the user's interest
* vector. Returns the score. Higher scores are better. Assumes this.interestVector
* is populated.
*/
calculateItemRelevanceScore(pocketItem) {
if (!this.initialized) {
return pocketItem.item_score || 1;
}
let scorableItem = this.recipeExecutor.executeRecipe(
pocketItem,
this.interestConfig.item_to_rank_builder
);
if (scorableItem === null) {
return -1;
}
let rankingVector = JSON.parse(JSON.stringify(this.interestVector));
Object.keys(scorableItem).forEach(key => {
rankingVector[key] = scorableItem[key];
});
rankingVector = this.recipeExecutor.executeRecipe(
rankingVector,
this.interestConfig.item_ranker
);
if (rankingVector === null) {
return -1;
}
return rankingVector.score;
}
/**
* Returns an object holding the settings and affinity scores of this provider instance.
*/
getAffinities() {
return {
timeSegments: this.timeSegments,
parameterSets: this.parameterSets,
maxHistoryQueryResults: this.maxHistoryQueryResults,
version: this.version,
scores: {
// We cannot return taggers here.
// What we return here goes into persistent cache, and taggers have functions on it.
// If we attempted to save taggers into persistent cache, it would store it to disk,
// and the next time we load it, it would start thowing function is not defined.
interestConfig: this.interestConfig,
interestVector: this.interestVector,
},
};
}
};
const EXPORTED_SYMBOLS = ["PersonalityProvider"];

17
browser/components/newtab/lib/NaiveBayesTextTagger.jsm → browser/components/newtab/lib/PersonalityProvider/NaiveBayesTextTagger.jsm
Просмотреть файл

@ -1,15 +1,18 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
"use strict";
const { toksToTfIdfVector } = ChromeUtils.import(
"resource://activity-stream/lib/Tokenize.jsm"
);
// We load this into a worker using importScripts, and in tests using import.
// We use var to avoid name collision errors.
// eslint-disable-next-line no-var
var EXPORTED_SYMBOLS = ["NaiveBayesTextTagger"];
this.NaiveBayesTextTagger = class NaiveBayesTextTagger {
constructor(model) {
const NaiveBayesTextTagger = class NaiveBayesTextTagger {
constructor(model, toksToTfIdfVector) {
this.model = model;
this.toksToTfIdfVector = toksToTfIdfVector;
}
/**
@ -20,7 +23,7 @@ this.NaiveBayesTextTagger = class NaiveBayesTextTagger {
* label. If the negative class is matched, then "label" is set to null.
*/
tagTokens(tokens) {
let fv = toksToTfIdfVector(tokens, this.model.vocab_idfs);
let fv = this.toksToTfIdfVector(tokens, this.model.vocab_idfs);
let bestLogProb = null;
let bestClassId = -1;
@ -62,5 +65,3 @@ this.NaiveBayesTextTagger = class NaiveBayesTextTagger {
};
}
};
const EXPORTED_SYMBOLS = ["NaiveBayesTextTagger"];

17
browser/components/newtab/lib/NmfTextTagger.jsm → browser/components/newtab/lib/PersonalityProvider/NmfTextTagger.jsm
Просмотреть файл

@ -1,15 +1,18 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
"use strict";
const { toksToTfIdfVector } = ChromeUtils.import(
"resource://activity-stream/lib/Tokenize.jsm"
);
// We load this into a worker using importScripts, and in tests using import.
// We use var to avoid name collision errors.
// eslint-disable-next-line no-var
var EXPORTED_SYMBOLS = ["NmfTextTagger"];
this.NmfTextTagger = class NmfTextTagger {
constructor(model) {
const NmfTextTagger = class NmfTextTagger {
constructor(model, toksToTfIdfVector) {
this.model = model;
this.toksToTfIdfVector = toksToTfIdfVector;
}
/**
@ -20,7 +23,7 @@ this.NmfTextTagger = class NmfTextTagger {
* consumer of this data determine what classes are most valuable.
*/
tagTokens(tokens) {
let fv = toksToTfIdfVector(tokens, this.model.vocab_idfs);
let fv = this.toksToTfIdfVector(tokens, this.model.vocab_idfs);
let fve = Object.values(fv);
// normalize by the sum of the vector
@ -60,5 +63,3 @@ this.NmfTextTagger = class NmfTextTagger {
return predictions;
}
};
const EXPORTED_SYMBOLS = ["NmfTextTagger"];

378
browser/components/newtab/lib/PersonalityProvider/PersonalityProvider.jsm Normal file
Просмотреть файл

@ -0,0 +1,378 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
"use strict";
ChromeUtils.defineModuleGetter(
this,
"RemoteSettings",
"resource://services-settings/remote-settings.js"
);
const { actionCreators: ac } = ChromeUtils.import(
"resource://activity-stream/common/Actions.jsm"
);
ChromeUtils.defineModuleGetter(
this,
"NewTabUtils",
"resource://gre/modules/NewTabUtils.jsm"
);
const { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm");
const { XPCOMUtils } = ChromeUtils.import(
"resource://gre/modules/XPCOMUtils.jsm"
);
ChromeUtils.defineModuleGetter(this, "OS", "resource://gre/modules/osfile.jsm");
XPCOMUtils.defineLazyGlobalGetters(this, ["fetch"]);
const { BasePromiseWorker } = ChromeUtils.import(
"resource://gre/modules/PromiseWorker.jsm"
);
const RECIPE_NAME = "personality-provider-recipe";
const MODELS_NAME = "personality-provider-models";
this.PersonalityProvider = class PersonalityProvider {
constructor(
timeSegments,
parameterSets,
maxHistoryQueryResults,
version,
scores,
v2Params
) {
this.timeSegments = timeSegments;
this.parameterSets = parameterSets;
this.v2Params = v2Params || {};
this.modelKeys = this.v2Params.modelKeys;
this.dispatch = this.v2Params.dispatch;
if (!this.dispatch) {
this.dispatch = () => {};
}
this.maxHistoryQueryResults = maxHistoryQueryResults;
this.version = version;
this.onSync = this.onSync.bind(this);
this.setup();
}
get personalityProviderWorker() {
if (this._personalityProviderWorker) {
return this._personalityProviderWorker;
}
this._personalityProviderWorker = new BasePromiseWorker(
"resource://activity-stream/lib/PersonalityProvider/PersonalityProviderWorker.js"
);
// As the PersonalityProviderWorker performs I/O, we can receive instances of
// OS.File.Error, so we need to install a decoder.
this._personalityProviderWorker.ExceptionHandlers["OS.File.Error"] =
OS.File.Error.fromMsg;
return this._personalityProviderWorker;
}
get baseAttachmentsURL() {
// Returning a promise, so we can have an async getter.
return this._getBaseAttachmentsURL();
}
async _getBaseAttachmentsURL() {
if (this._baseAttachmentsURL) {
return this._baseAttachmentsURL;
}
const server = Services.prefs.getCharPref("services.settings.server");
const serverInfo = await (
await fetch(`${server}/`, {
credentials: "omit",
})
).json();
const {
capabilities: {
attachments: { base_url },
},
} = serverInfo;
this._baseAttachmentsURL = base_url;
return this._baseAttachmentsURL;
}
setup() {
this.setupSyncAttachment(RECIPE_NAME);
this.setupSyncAttachment(MODELS_NAME);
}
teardown() {
this.teardownSyncAttachment(RECIPE_NAME);
this.teardownSyncAttachment(MODELS_NAME);
}
setupSyncAttachment(collection) {
RemoteSettings(collection).on("sync", this.onSync);
}
teardownSyncAttachment(collection) {
RemoteSettings(collection).off("sync", this.onSync);
}
onSync(event) {
this.personalityProviderWorker.post("onSync", [event]);
}
/**
* Gets contents of the attachment if it already exists on file,
* and if not attempts to download it.
*/
getAttachment(record) {
return this.personalityProviderWorker.post("getAttachment", [record]);
}
/**
* Returns a Recipe from remote settings to be consumed by a RecipeExecutor.
* A Recipe is a set of instructions on how to processes a RecipeExecutor.
*/
async getRecipe() {
if (!this.recipes || !this.recipes.length) {
const start = Cu.now();
const result = await RemoteSettings(RECIPE_NAME).get();
this.recipes = await Promise.all(
result.map(async record => ({
...(await this.getAttachment(record)),
recordKey: record.key,
}))
);
this.dispatch(
ac.PerfEvent({
event: "PERSONALIZATION_V2_GET_RECIPE_DURATION",
value: Math.round(Cu.now() - start),
})
);
}
return this.recipes[0];
}
/**
* Grabs a slice of browse history for building a interest vector
*/
async fetchHistory(columns, beginTimeSecs, endTimeSecs) {
let sql = `SELECT url, title, visit_count, frecency, last_visit_date, description
FROM moz_places
WHERE last_visit_date >= ${beginTimeSecs * 1000000}
AND last_visit_date < ${endTimeSecs * 1000000}`;
columns.forEach(requiredColumn => {
sql += ` AND IFNULL(${requiredColumn}, '') <> ''`;
});
sql += " LIMIT 30000";
const { activityStreamProvider } = NewTabUtils;
const history = await activityStreamProvider.executePlacesQuery(sql, {
columns,
params: {},
});
return history;
}
/**
* Handles setup and metrics of history fetch.
*/
async getHistory() {
let endTimeSecs = new Date().getTime() / 1000;
let beginTimeSecs = endTimeSecs - this.interestConfig.history_limit_secs;
if (
!this.interestConfig ||
!this.interestConfig.history_required_fields ||
!this.interestConfig.history_required_fields.length
) {
return [];
}
let history = await this.fetchHistory(
this.interestConfig.history_required_fields,
beginTimeSecs,
endTimeSecs
);
this.dispatch(
ac.PerfEvent({
event: "PERSONALIZATION_V2_HISTORY_SIZE",
value: history.length,
})
);
return history;
}
async setBaseAttachmentsURL() {
await this.personalityProviderWorker.post("setBaseAttachmentsURL", [
await this.baseAttachmentsURL,
]);
}
async setInterestConfig() {
this.interestConfig = this.interestConfig || (await this.getRecipe());
await this.personalityProviderWorker.post("setInterestConfig", [
this.interestConfig,
]);
}
async setInterestVector() {
await this.personalityProviderWorker.post("setInterestVector", [
this.interestVector,
]);
}
async fetchModels() {
const models = await RemoteSettings(MODELS_NAME).get();
return this.personalityProviderWorker.post("fetchModels", [models]);
}
async generateTaggers() {
const start = Cu.now();
await this.personalityProviderWorker.post("generateTaggers", [
this.modelKeys,
]);
this.dispatch(
ac.PerfEvent({
event: "PERSONALIZATION_V2_TAGGERS_DURATION",
value: Math.round(Cu.now() - start),
})
);
}
async generateRecipeExecutor() {
const start = Cu.now();
await this.personalityProviderWorker.post("generateRecipeExecutor");
this.dispatch(
ac.PerfEvent({
event: "PERSONALIZATION_V2_RECIPE_EXECUTOR_DURATION",
value: Math.round(Cu.now() - start),
})
);
}
async createInterestVector() {
const history = await this.getHistory();
const interestVectorPerfStart = Cu.now();
const interestVectorResult = await this.personalityProviderWorker.post(
"createInterestVector",
[history]
);
this.dispatch(
ac.PerfEvent({
event: "PERSONALIZATION_V2_CREATE_INTEREST_VECTOR_DURATION",
value: Math.round(Cu.now() - interestVectorPerfStart),
})
);
return interestVectorResult;
}
async init(callback) {
const perfStart = Cu.now();
await this.setBaseAttachmentsURL();
await this.setInterestConfig();
if (!this.interestConfig) {
this.dispatch(
ac.PerfEvent({ event: "PERSONALIZATION_V2_GET_RECIPE_ERROR" })
);
return;
}
// We always generate a recipe executor, no cache used here.
// This is because the result of this is an object with
// functions (taggers) so storing it in cache is not possible.
// Thus we cannot use it to rehydrate anything.
const fetchModelsResult = await this.fetchModels();
// If this fails, log an error and return.
if (!fetchModelsResult.ok) {
this.dispatch(
ac.PerfEvent({
event: "PERSONALIZATION_V2_FETCH_MODELS_ERROR",
})
);
return;
}
await this.generateTaggers();
await this.generateRecipeExecutor();
// If we don't have a cached vector, create a new one.
if (!this.interestVector) {
const interestVectorResult = await this.createInterestVector();
// If that failed, log an error and return.
if (!interestVectorResult.ok) {
this.dispatch(
ac.PerfEvent({
event: "PERSONALIZATION_V2_CREATE_INTEREST_VECTOR_ERROR",
})
);
return;
}
this.interestVector = interestVectorResult.interestVector;
}
// This happens outside the createInterestVector call above,
// because create can be skipped if rehydrating from cache.
// In that case, the interest vector is provided and not created, so we just set it.
await this.setInterestVector();
this.dispatch(
ac.PerfEvent({
event: "PERSONALIZATION_V2_TOTAL_DURATION",
value: Math.round(Cu.now() - perfStart),
})
);
this.initialized = true;
if (callback) {
callback();
}
}
dispatchRelevanceScoreDuration(start) {
// If v2 is not yet initialized we don't bother tracking yet.
// Before it is initialized it doesn't do any ranking.
// Once it's initialized it ensures ranking is done.
// v1 doesn't have any initialized issues around ranking,
// and should be ready right away.
if (this.initialized) {
this.dispatch(
ac.PerfEvent({
event: "PERSONALIZATION_V2_ITEM_RELEVANCE_SCORE_DURATION",
value: Math.round(Cu.now() - start),
})
);
}
}
calculateItemRelevanceScore(pocketItem) {
if (!this.initialized) {
return pocketItem.item_score || 1;
}
return this.personalityProviderWorker.post("calculateItemRelevanceScore", [
pocketItem,
]);
}
/**
* Returns an object holding the settings and affinity scores of this provider instance.
*/
getAffinities() {
return {
timeSegments: this.timeSegments,
parameterSets: this.parameterSets,
maxHistoryQueryResults: this.maxHistoryQueryResults,
version: this.version,
scores: {
// We cannot return taggers here.
// What we return here goes into persistent cache, and taggers have functions on it.
// If we attempted to save taggers into persistent cache, it would store it to disk,
// and the next time we load it, it would start thowing function is not defined.
interestConfig: this.interestConfig,
interestVector: this.interestVector,
},
};
}
};
const EXPORTED_SYMBOLS = ["PersonalityProvider"];

37
browser/components/newtab/lib/PersonalityProvider/PersonalityProviderWorker.js Normal file
Просмотреть файл

@ -0,0 +1,37 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
/* eslint-env mozilla/chrome-worker */
"use strict";
// Order of these are important.
importScripts(
"resource://gre/modules/workers/require.js",
"resource://gre/modules/osfile.jsm",
"resource://activity-stream/lib/PersonalityProvider/Tokenize.jsm",
"resource://activity-stream/lib/PersonalityProvider/NaiveBayesTextTagger.jsm",
"resource://activity-stream/lib/PersonalityProvider/NmfTextTagger.jsm",
"resource://activity-stream/lib/PersonalityProvider/RecipeExecutor.jsm",
"resource://activity-stream/lib/PersonalityProvider/PersonalityProviderWorkerClass.jsm"
);
const PromiseWorker = require("resource://gre/modules/workers/PromiseWorker.js");
// eslint-disable-next-line no-undef
const personalityProviderWorker = new PersonalityProviderWorker();
// This is boiler plate worker stuff that connects it to the main thread PromiseWorker.
const worker = new PromiseWorker.AbstractWorker();
worker.dispatch = function(method, args = []) {
return personalityProviderWorker[method](...args);
};
worker.postMessage = function(message, ...transfers) {
self.postMessage(message, ...transfers);
};
worker.close = function() {
self.close();
};
self.addEventListener("message", msg => worker.handleMessage(msg));

264
browser/components/newtab/lib/PersonalityProvider/PersonalityProviderWorkerClass.jsm Normal file
Просмотреть файл

@ -0,0 +1,264 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
"use strict";
// We load this into a worker using importScripts, and in tests using import.
// We use var to avoid name collision errors.
// eslint-disable-next-line no-var
var EXPORTED_SYMBOLS = ["PersonalityProviderWorker"];
const PERSONALITY_PROVIDER_DIR = OS.Path.join(
OS.Constants.Path.localProfileDir,
"personality-provider"
);
/**
* V2 provider builds and ranks an interest profile (also called an “interest vector”) off the browse history.
* This allows Firefox to classify pages into topics, by examining the text found on the page.
* It does this by looking at the history text content, title, and description.
*/
const PersonalityProviderWorker = class PersonalityProviderWorker {
// A helper function to read and decode a file, it isn't a stand alone function.
// If you use this, ensure you check the file exists and you have a try catch.
_getFileStr(filepath) {
const binaryData = OS.File.read(filepath);
const gTextDecoder = new TextDecoder();
return gTextDecoder.decode(binaryData);
}
setBaseAttachmentsURL(url) {
this.baseAttachmentsURL = url;
}
setInterestConfig(interestConfig) {
this.interestConfig = interestConfig;
}
setInterestVector(interestVector) {
this.interestVector = interestVector;
}
onSync(event) {
const {
data: { created, updated, deleted },
} = event;
// Remove every removed attachment.
const toRemove = deleted.concat(updated.map(u => u.old));
toRemove.map(record => this.deleteAttachment(record));
// Download every new/updated attachment.
const toDownload = created.concat(updated.map(u => u.new));
toDownload.map(record => this.maybeDownloadAttachment(record));
}
/**
* Attempts to download the attachment, but only if it doesn't already exist.
*/
maybeDownloadAttachment(record, retries = 3) {
const {
attachment: { filename, size },
} = record;
OS.File.makeDir(PERSONALITY_PROVIDER_DIR);
const localFilePath = OS.Path.join(PERSONALITY_PROVIDER_DIR, filename);
let retry = 0;
while (
retry++ < retries &&
// exists is an issue for perf because I might not need to call it.
(!OS.File.exists(localFilePath) ||
OS.File.stat(localFilePath).size !== size)
) {
this._downloadAttachment(record);
}
}
/**
* Downloads the attachment to disk assuming the dir already exists
* and any existing files matching the filename are clobbered.
*/
_downloadAttachment(record) {
const {
attachment: { location, filename },
} = record;
const remoteFilePath = this.baseAttachmentsURL + location;
const localFilePath = OS.Path.join(PERSONALITY_PROVIDER_DIR, filename);
const xhr = new XMLHttpRequest();
// Set false here for a synchronous request, because we're in a worker.
xhr.open("GET", remoteFilePath, false);
xhr.setRequestHeader("Accept-Encoding", "gzip");
xhr.responseType = "arraybuffer";
xhr.withCredentials = false;
xhr.send(null);
if (xhr.status !== 200) {
console.error(`Failed to fetch ${remoteFilePath}: ${xhr.statusText}`);
return;
}
const buffer = xhr.response;
const bytes = new Uint8Array(buffer);
OS.File.writeAtomic(localFilePath, bytes, {
tmpPath: `${localFilePath}.tmp`,
});
}
deleteAttachment(record) {
const {
attachment: { filename },
} = record;
OS.File.makeDir(PERSONALITY_PROVIDER_DIR);
const path = OS.Path.join(PERSONALITY_PROVIDER_DIR, filename);
OS.File.remove(path, { ignoreAbsent: true });
OS.File.removeEmptyDir(PERSONALITY_PROVIDER_DIR, {
ignoreAbsent: true,
});
}
/**
* Gets contents of the attachment if it already exists on file,
* and if not attempts to download it.
*/
getAttachment(record) {
const {
attachment: { filename },
} = record;
const filepath = OS.Path.join(PERSONALITY_PROVIDER_DIR, filename);
try {
this.maybeDownloadAttachment(record);
return JSON.parse(this._getFileStr(filepath));
} catch (error) {
console.error(`Failed to load ${filepath}: ${error.message}`);
}
return {};
}
fetchModels(models) {
this.models = models.map(record => ({
...this.getAttachment(record),
recordKey: record.key,
}));
if (!this.models.length) {
return {
ok: false,
};
}
return {
ok: true,
};
}
generateTaggers(modelKeys) {
if (!this.taggers) {
let nbTaggers = [];
let nmfTaggers = {};
for (let model of this.models) {
if (!modelKeys.includes(model.recordKey)) {
continue;
}
if (model.model_type === "nb") {
// eslint-disable-next-line no-undef
nbTaggers.push(new NaiveBayesTextTagger(model, toksToTfIdfVector));
} else if (model.model_type === "nmf") {
// eslint-disable-next-line no-undef
nmfTaggers[model.parent_tag] = new NmfTextTagger(
model,
// eslint-disable-next-line no-undef
toksToTfIdfVector
);
}
}
this.taggers = { nbTaggers, nmfTaggers };
}
}
/**
* Sets and generates a Recipe Executor.
* A Recipe Executor is a set of actions that can be consumed by a Recipe.
* The Recipe determines the order and specifics of which the actions are called.
*/
generateRecipeExecutor() {
// eslint-disable-next-line no-undef
const recipeExecutor = new RecipeExecutor(
this.taggers.nbTaggers,
this.taggers.nmfTaggers,
// eslint-disable-next-line no-undef
tokenize
);
this.recipeExecutor = recipeExecutor;
}
/**
* Examines the user's browse history and returns an interest vector that
* describes the topics the user frequently browses.
*/
createInterestVector(history) {
let interestVector = {};
for (let historyRec of history) {
let ivItem = this.recipeExecutor.executeRecipe(
historyRec,
this.interestConfig.history_item_builder
);
if (ivItem === null) {
continue;
}
interestVector = this.recipeExecutor.executeCombinerRecipe(
interestVector,
ivItem,
this.interestConfig.interest_combiner
);
if (interestVector === null) {
return null;
}
}
const finalResult = this.recipeExecutor.executeRecipe(
interestVector,
this.interestConfig.interest_finalizer
);
return {
ok: true,
interestVector: finalResult,
};
}
/**
* Calculates a score of a Pocket item when compared to the user's interest
* vector. Returns the score. Higher scores are better. Assumes this.interestVector
* is populated.
*/
calculateItemRelevanceScore(pocketItem) {
let scorableItem = this.recipeExecutor.executeRecipe(
pocketItem,
this.interestConfig.item_to_rank_builder
);
if (scorableItem === null) {
return -1;
}
// We're doing a deep copy on an object.
let rankingVector = JSON.parse(JSON.stringify(this.interestVector));
Object.keys(scorableItem).forEach(key => {
rankingVector[key] = scorableItem[key];
});
rankingVector = this.recipeExecutor.executeRecipe(
rankingVector,
this.interestConfig.item_ranker
);
if (rankingVector === null) {
return -1;
}
return rankingVector.score;
}
};

45
browser/components/newtab/lib/RecipeExecutor.jsm → browser/components/newtab/lib/PersonalityProvider/RecipeExecutor.jsm
Просмотреть файл

@ -1,11 +1,13 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
"use strict";
const { tokenize } = ChromeUtils.import(
"resource://activity-stream/lib/Tokenize.jsm"
);
// We load this into a worker using importScripts, and in tests using import.
// We use var to avoid name collision errors.
// eslint-disable-next-line no-var
var EXPORTED_SYMBOLS = ["RecipeExecutor"];
/**
* RecipeExecutor is the core feature engineering pipeline for the in-browser
@ -29,8 +31,8 @@ const { tokenize } = ChromeUtils.import(
* RecipeExecutor.ITEM_BUILDER_REGISTRY, while combiner functions are whitelisted
* in RecipeExecutor.ITEM_COMBINER_REGISTRY .
*/
this.RecipeExecutor = class RecipeExecutor {
constructor(nbTaggers, nmfTaggers) {
const RecipeExecutor = class RecipeExecutor {
constructor(nbTaggers, nmfTaggers, tokenize) {
this.ITEM_BUILDER_REGISTRY = {
nb_tag: this.naiveBayesTag,
conditionally_nmf_tag: this.conditionallyNmfTag,
@ -63,6 +65,7 @@ this.RecipeExecutor = class RecipeExecutor {
};
this.nbTaggers = nbTaggers;
this.nmfTaggers = nmfTaggers;
this.tokenize = tokenize;
}
/**
@ -138,7 +141,7 @@ this.RecipeExecutor = class RecipeExecutor {
*/
naiveBayesTag(item, config) {
let text = this._assembleText(item, config.fields);
let tokens = tokenize(text);
let tokens = this.tokenize(text);
let tags = {};
let extended_tags = {};
@ -256,20 +259,20 @@ this.RecipeExecutor = class RecipeExecutor {
if (domain.startsWith("www.")) {
domain = domain.substring(4);
}
let toks = tokenize(domain);
let pathToks = tokenize(
let toks = this.tokenize(domain);
let pathToks = this.tokenize(
decodeURIComponent(url.pathname.replace(/\+/g, " "))
);
for (let tok of pathToks) {
toks.push(tok);
}
for (let pair of url.searchParams.entries()) {
let k = tokenize(decodeURIComponent(pair[0].replace(/\+/g, " ")));
let k = this.tokenize(decodeURIComponent(pair[0].replace(/\+/g, " ")));
for (let tok of k) {
toks.push(tok);
}
if (pair[1] !== null && pair[1] !== "") {
let v = tokenize(decodeURIComponent(pair[1].replace(/\+/g, " ")));
let v = this.tokenize(decodeURIComponent(pair[1].replace(/\+/g, " ")));
for (let tok of v) {
toks.push(tok);
}
@ -326,7 +329,7 @@ this.RecipeExecutor = class RecipeExecutor {
return null;
}
item[config.dest] = tokenize(item[config.field]);
item[config.dest] = this.tokenize(item[config.field]);
return item;
}
@ -1087,14 +1090,16 @@ this.RecipeExecutor = class RecipeExecutor {
*/
executeRecipe(item, recipe) {
let newItem = item;
for (let step of recipe) {
let op = this.ITEM_BUILDER_REGISTRY[step.function];
if (op === undefined) {
return null;
}
newItem = op.call(this, newItem, step);
if (newItem === null) {
break;
if (recipe) {
for (let step of recipe) {
let op = this.ITEM_BUILDER_REGISTRY[step.function];
if (op === undefined) {
return null;
}
newItem = op.call(this, newItem, step);
if (newItem === null) {
break;
}
}
}
return newItem;
@ -1119,5 +1124,3 @@ this.RecipeExecutor = class RecipeExecutor {
return newItem1;
}
};
const EXPORTED_SYMBOLS = ["RecipeExecutor"];

7
browser/components/newtab/lib/Tokenize.jsm → browser/components/newtab/lib/PersonalityProvider/Tokenize.jsm
Просмотреть файл

@ -3,6 +3,11 @@
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
"use strict";
// We load this into a worker using importScripts, and in tests using import.
// We use var to avoid name collision errors.
// eslint-disable-next-line no-var
var EXPORTED_SYMBOLS = ["tokenize", "toksToTfIdfVector"];
// Unicode specifies certain mnemonics for code pages and character classes.
// They call them "character properties" https://en.wikipedia.org/wiki/Unicode_character_property .
// These mnemonics are have been adopted by many regular expression libraries,
@ -82,5 +87,3 @@ function toksToTfIdfVector(tokens, vocab_idfs) {
return tfidfs;
}
const EXPORTED_SYMBOLS = ["tokenize", "toksToTfIdfVector"];

6
browser/components/newtab/lib/RecommendationProviderSwitcher.jsm
Просмотреть файл

@ -10,7 +10,7 @@ ChromeUtils.defineModuleGetter(
ChromeUtils.defineModuleGetter(
this,
"PersonalityProvider",
"resource://activity-stream/lib/PersonalityProvider.jsm"
"resource://activity-stream/lib/PersonalityProvider/PersonalityProvider.jsm"
);
const { actionTypes: at, actionCreators: ac } = ChromeUtils.import(
"resource://activity-stream/common/Actions.jsm"
@ -139,9 +139,9 @@ this.RecommendationProviderSwitcher = class RecommendationProviderSwitcher {
}
}
calculateItemRelevanceScore(item) {
async calculateItemRelevanceScore(item) {
if (this.affinityProvider) {
const scoreResult = this.affinityProvider.calculateItemRelevanceScore(
const scoreResult = await this.affinityProvider.calculateItemRelevanceScore(
item
);
if (scoreResult === 0 || scoreResult) {

2
browser/components/newtab/lib/TopStoriesFeed.jsm
Просмотреть файл

@ -28,7 +28,7 @@ const { UserDomainAffinityProvider } = ChromeUtils.import(
"resource://activity-stream/lib/UserDomainAffinityProvider.jsm"
);
const { PersonalityProvider } = ChromeUtils.import(
"resource://activity-stream/lib/PersonalityProvider.jsm"
"resource://activity-stream/lib/PersonalityProvider/PersonalityProvider.jsm"
);
const { PersistentCache } = ChromeUtils.import(
"resource://activity-stream/lib/PersistentCache.jsm"

44
browser/components/newtab/test/unit/lib/DiscoveryStreamFeed.test.js
Просмотреть файл

@ -1105,13 +1105,13 @@ describe("DiscoveryStreamFeed", () => {
});
describe("#transform", () => {
it("should return initial data if spocs are empty", () => {
const { data: result } = feed.transform({ spocs: [] });
it("should return initial data if spocs are empty", async () => {
const { data: result } = await feed.transform({ spocs: [] });
assert.equal(result.spocs.length, 0);
});
it("should sort based on item_score", () => {
const { data: result } = feed.transform([
it("should sort based on item_score", async () => {
const { data: result } = await feed.transform([
{ id: 2, flight_id: 2, item_score: 0.8, min_score: 0.1 },
{ id: 3, flight_id: 3, item_score: 0.7, min_score: 0.1 },
{ id: 1, flight_id: 1, item_score: 0.9, min_score: 0.1 },
@ -1123,8 +1123,8 @@ describe("DiscoveryStreamFeed", () => {
{ id: 3, flight_id: 3, item_score: 0.7, score: 0.7, min_score: 0.1 },
]);
});
it("should remove items with scores lower than min_score", () => {
const { data: result, filtered } = feed.transform([
it("should remove items with scores lower than min_score", async () => {
const { data: result, filtered } = await feed.transform([
{ id: 2, flight_id: 2, item_score: 0.8, min_score: 0.9 },
{ id: 3, flight_id: 3, item_score: 0.7, min_score: 0.7 },
{ id: 1, flight_id: 1, item_score: 0.9, min_score: 0.8 },
@ -1139,15 +1139,15 @@ describe("DiscoveryStreamFeed", () => {
{ id: 2, flight_id: 2, item_score: 0.8, min_score: 0.9, score: 0.8 },
]);
});
it("should add a score prop to spocs", () => {
const { data: result } = feed.transform([
it("should add a score prop to spocs", async () => {
const { data: result } = await feed.transform([
{ flight_id: 1, item_score: 0.9, min_score: 0.1 },
]);
assert.equal(result[0].score, 0.9);
});
it("should filter out duplicate flights", () => {
const { data: result, filtered } = feed.transform([
it("should filter out duplicate flights", async () => {
const { data: result, filtered } = await feed.transform([
{ id: 1, flight_id: 2, item_score: 0.8, min_score: 0.1 },
{ id: 2, flight_id: 3, item_score: 0.6, min_score: 0.1 },
{ id: 3, flight_id: 1, item_score: 0.9, min_score: 0.1 },
@ -1166,14 +1166,14 @@ describe("DiscoveryStreamFeed", () => {
{ id: 2, flight_id: 3, item_score: 0.6, min_score: 0.1, score: 0.6 },
]);
});
it("should filter out duplicate flight while using spocs_per_domain", () => {
it("should filter out duplicate flight while using spocs_per_domain", async () => {
sandbox.stub(feed.store, "getState").returns({
DiscoveryStream: {
spocs: { spocs_per_domain: 2 },
},
});
const { data: result, filtered } = feed.transform([
const { data: result, filtered } = await feed.transform([
{ id: 1, flight_id: 2, item_score: 0.8, min_score: 0.1 },
{ id: 2, flight_id: 3, item_score: 0.6, min_score: 0.1 },
{ id: 3, flight_id: 1, item_score: 0.6, min_score: 0.1 },
@ -2900,8 +2900,8 @@ describe("DiscoveryStreamFeed", () => {
});
});
describe("#scoreItems", () => {
it("should score items using item_score and min_score", () => {
const { data: result, filtered } = feed.scoreItems([
it("should score items using item_score and min_score", async () => {
const { data: result, filtered } = await feed.scoreItems([
{ item_score: 0.8, min_score: 0.1 },
{ item_score: 0.5, min_score: 0.6 },
{ item_score: 0.7, min_score: 0.1 },
@ -2916,20 +2916,20 @@ describe("DiscoveryStreamFeed", () => {
{ item_score: 0.5, min_score: 0.6, score: 0.5 },
]);
});
it("should fire dispatchRelevanceScoreDuration if available", () => {
it("should fire dispatchRelevanceScoreDuration if available", async () => {
feed.providerSwitcher.dispatchRelevanceScoreDuration = sandbox
.stub()
.returns();
feed._prefCache.config = {
personalized: true,
};
feed.scoreItems([]);
await feed.scoreItems([]);
assert.calledOnce(feed.providerSwitcher.dispatchRelevanceScoreDuration);
});
});
describe("#scoreItem", () => {
it("should call calculateItemRelevanceScore with affinity provider", () => {
it("should call calculateItemRelevanceScore with affinity provider", async () => {
const item = {};
feed._prefCache.config = {
personalized: true,
@ -2937,10 +2937,10 @@ describe("DiscoveryStreamFeed", () => {
feed.providerSwitcher.calculateItemRelevanceScore = sandbox
.stub()
.returns();
feed.scoreItem(item);
await feed.scoreItem(item);
assert.calledOnce(feed.providerSwitcher.calculateItemRelevanceScore);
});
it("should use item_score score without affinity provider score", () => {
it("should use item_score score without affinity provider score", async () => {
const item = {
item_score: 0.6,
};
@ -2950,10 +2950,10 @@ describe("DiscoveryStreamFeed", () => {
feed.affinityProvider = {
calculateItemRelevanceScore: () => {},
};
const result = feed.scoreItem(item);
const result = await feed.scoreItem(item);
assert.equal(result.score, 0.6);
});
it("should add min_score of 0 if undefined", () => {
it("should add min_score of 0 if undefined", async () => {
const item = {};
feed._prefCache.config = {
personalized: true,
@ -2961,7 +2961,7 @@ describe("DiscoveryStreamFeed", () => {
feed.affinityProvider = {
calculateItemRelevanceScore: () => 0.5,
};
const result = feed.scoreItem(item);
const result = await feed.scoreItem(item);
assert.equal(result.min_score, 0);
});
});

643
browser/components/newtab/test/unit/lib/PersonalityProvider.test.js
Просмотреть файл

@ -1,643 +0,0 @@
import { GlobalOverrider } from "test/unit/utils";
import injector from "inject!lib/PersonalityProvider.jsm";
const TIME_SEGMENTS = [
{ id: "hour", startTime: 3600, endTime: 0, weightPosition: 1 },
{ id: "day", startTime: 86400, endTime: 3600, weightPosition: 0.75 },
{ id: "week", startTime: 604800, endTime: 86400, weightPosition: 0.5 },
{ id: "weekPlus", startTime: null, endTime: 604800, weightPosition: 0.25 },
];
const PARAMETER_SETS = {
paramSet1: {
recencyFactor: 0.5,
frequencyFactor: 0.5,
combinedDomainFactor: 0.5,
perfectFrequencyVisits: 10,
perfectCombinedDomainScore: 2,
multiDomainBoost: 0.1,
itemScoreFactor: 0,
},
paramSet2: {
recencyFactor: 1,
frequencyFactor: 0.7,
combinedDomainFactor: 0.8,
perfectFrequencyVisits: 10,
perfectCombinedDomainScore: 2,
multiDomainBoost: 0.1,
itemScoreFactor: 0,
},
};
describe("Personality Provider", () => {
let instance;
let PersonalityProvider;
let globals;
let NaiveBayesTextTaggerStub;
let NmfTextTaggerStub;
let RecipeExecutorStub;
let baseURLStub;
beforeEach(() => {
globals = new GlobalOverrider();
const testUrl = "www.somedomain.com";
globals.sandbox
.stub(global.Services.io, "newURI")
.returns({ host: testUrl });
globals.sandbox.stub(global.PlacesUtils.history, "executeQuery").returns({
root: {
childCount: 1,
getChild: index => ({ uri: testUrl, accessCount: 1 }),
},
});
globals.sandbox
.stub(global.PlacesUtils.history, "getNewQuery")
.returns({ TIME_RELATIVE_NOW: 1 });
globals.sandbox
.stub(global.PlacesUtils.history, "getNewQueryOptions")
.returns({});
NaiveBayesTextTaggerStub = globals.sandbox.stub();
NmfTextTaggerStub = globals.sandbox.stub();
RecipeExecutorStub = globals.sandbox.stub();
baseURLStub = "";
global.fetch = async server => ({
ok: true,
json: async () => {
if (server === "services.settings.server/") {
return { capabilities: { attachments: { base_url: baseURLStub } } };
}
return {};
},
});
globals.sandbox
.stub(global.Services.prefs, "getCharPref")
.callsFake(pref => pref);
({ PersonalityProvider } = injector({
"lib/NaiveBayesTextTagger.jsm": {
NaiveBayesTextTagger: NaiveBayesTextTaggerStub,
},
"lib/NmfTextTagger.jsm": { NmfTextTagger: NmfTextTaggerStub },
"lib/RecipeExecutor.jsm": { RecipeExecutor: RecipeExecutorStub },
}));
instance = new PersonalityProvider(TIME_SEGMENTS, PARAMETER_SETS);
instance.interestConfig = {
history_item_builder: "history_item_builder",
history_required_fields: ["a", "b", "c"],
interest_finalizer: "interest_finalizer",
item_to_rank_builder: "item_to_rank_builder",
item_ranker: "item_ranker",
interest_combiner: "interest_combiner",
};
// mock the RecipeExecutor
instance.recipeExecutor = {
executeRecipe: (item, recipe) => {
if (recipe === "history_item_builder") {
if (item.title === "fail") {
return null;
}
return {
title: item.title,
score: item.frecency,
type: "history_item",
};
} else if (recipe === "interest_finalizer") {
return {
title: item.title,
score: item.score * 100,
type: "interest_vector",
};
} else if (recipe === "item_to_rank_builder") {
if (item.title === "fail") {
return null;
}
return {
item_title: item.title,
item_score: item.score,
type: "item_to_rank",
};
} else if (recipe === "item_ranker") {
if (item.title === "fail" || item.item_title === "fail") {
return null;
}
return {
title: item.title,
score: item.item_score * item.score,
type: "ranked_item",
};
}
return null;
},
executeCombinerRecipe: (item1, item2, recipe) => {
if (recipe === "interest_combiner") {
if (
item1.title === "combiner_fail" ||
item2.title === "combiner_fail"
) {
return null;
}
if (item1.type === undefined) {
item1.type = "combined_iv";
}
if (item1.score === undefined) {
item1.score = 0;
}
return { type: item1.type, score: item1.score + item2.score };
}
return null;
},
};
});
afterEach(() => {
globals.restore();
});
describe("#init", () => {
it("should return correct data for getAffinities", () => {
const affinities = instance.getAffinities();
assert.isDefined(affinities.timeSegments);
assert.isDefined(affinities.parameterSets);
});
it("should return early and not initialize if getRecipe fails", async () => {
sinon.stub(instance, "getRecipe").returns(Promise.resolve());
await instance.init();
assert.isUndefined(instance.initialized);
});
it("should return early if get recipe fails", async () => {
sinon.stub(instance, "getRecipe").returns(Promise.resolve());
sinon.stub(instance, "generateRecipeExecutor").returns(Promise.resolve());
instance.interestConfig = undefined;
await instance.init();
assert.calledOnce(instance.getRecipe);
assert.notCalled(instance.generateRecipeExecutor);
assert.isUndefined(instance.initialized);
assert.isUndefined(instance.interestConfig);
});
it("should call callback on successful init", async () => {
sinon.stub(instance, "getRecipe").returns(Promise.resolve(true));
instance.interestConfig = undefined;
const callback = globals.sandbox.stub();
instance.createInterestVector = async () => ({});
sinon
.stub(instance, "generateRecipeExecutor")
.returns(Promise.resolve(true));
await instance.init(callback);
assert.calledOnce(instance.getRecipe);
assert.calledOnce(instance.generateRecipeExecutor);
assert.calledOnce(callback);
assert.isDefined(instance.interestVector);
assert.isTrue(instance.initialized);
});
it("should return early and not initialize if generateRecipeExecutor fails", async () => {
sinon.stub(instance, "getRecipe").returns(Promise.resolve(true));
sinon.stub(instance, "generateRecipeExecutor").returns(Promise.resolve());
instance.interestConfig = undefined;
await instance.init();
assert.calledOnce(instance.getRecipe);
assert.isUndefined(instance.initialized);
});
it("should return early and not initialize if createInterestVector fails", async () => {
sinon.stub(instance, "getRecipe").returns(Promise.resolve(true));
instance.interestConfig = undefined;
sinon
.stub(instance, "generateRecipeExecutor")
.returns(Promise.resolve(true));
instance.createInterestVector = async () => null;
await instance.init();
assert.calledOnce(instance.getRecipe);
assert.calledOnce(instance.generateRecipeExecutor);
assert.isUndefined(instance.initialized);
});
it("should do generic init stuff when calling init with no cache", async () => {
sinon.stub(instance, "getRecipe").returns(Promise.resolve(true));
instance.interestConfig = undefined;
instance.createInterestVector = async () => ({});
sinon
.stub(instance, "generateRecipeExecutor")
.returns(Promise.resolve(true));
await instance.init();
assert.calledOnce(instance.getRecipe);
assert.calledOnce(instance.generateRecipeExecutor);
assert.isDefined(instance.interestVector);
assert.isTrue(instance.initialized);
});
});
describe("#remote-settings", () => {
it("should return a remote setting for getFromRemoteSettings", async () => {
const settings = await instance.getFromRemoteSettings("attachment");
assert.equal(typeof settings, "object");
assert.equal(settings.length, 1);
});
});
describe("#executor", () => {
it("should return null if generateRecipeExecutor has no models", async () => {
assert.isNull(await instance.generateRecipeExecutor());
});
it("should not generate taggers if already available", async () => {
instance.taggers = {
nbTaggers: ["first"],
nmfTaggers: { first: "first" },
};
await instance.generateRecipeExecutor();
assert.calledOnce(RecipeExecutorStub);
const { args } = RecipeExecutorStub.firstCall;
assert.equal(args[0].length, 1);
assert.equal(args[0], "first");
assert.equal(args[1].first, "first");
});
it("should pass recipe models to getRecipeExecutor on generateRecipeExecutor", async () => {
instance.modelKeys = ["nb_model_sports", "nmf_model_sports"];
instance.getFromRemoteSettings = async name => [
{ recordKey: "nb_model_sports", model_type: "nb" },
{
recordKey: "nmf_model_sports",
model_type: "nmf",
parent_tag: "nmf_sports_parent_tag",
},
];
await instance.generateRecipeExecutor();
assert.calledOnce(RecipeExecutorStub);
assert.calledOnce(NaiveBayesTextTaggerStub);
assert.calledOnce(NmfTextTaggerStub);
const { args } = RecipeExecutorStub.firstCall;
assert.equal(args[0].length, 1);
assert.isDefined(args[1].nmf_sports_parent_tag);
});
it("should skip any models not in modelKeys", async () => {
instance.modelKeys = ["nb_model_sports"];
instance.getFromRemoteSettings = async name => [
{ recordKey: "nb_model_sports", model_type: "nb" },
{
recordKey: "nmf_model_sports",
model_type: "nmf",
parent_tag: "nmf_sports_parent_tag",
},
];
await instance.generateRecipeExecutor();
assert.calledOnce(RecipeExecutorStub);
assert.calledOnce(NaiveBayesTextTaggerStub);
assert.notCalled(NmfTextTaggerStub);
const { args } = RecipeExecutorStub.firstCall;
assert.equal(args[0].length, 1);
assert.equal(Object.keys(args[1]).length, 0);
});
it("should skip any models not defined", async () => {
instance.modelKeys = ["nb_model_sports", "nmf_model_sports"];
instance.getFromRemoteSettings = async name => [
{ recordKey: "nb_model_sports", model_type: "nb" },
];
await instance.generateRecipeExecutor();
assert.calledOnce(RecipeExecutorStub);
assert.calledOnce(NaiveBayesTextTaggerStub);
assert.notCalled(NmfTextTaggerStub);
const { args } = RecipeExecutorStub.firstCall;
assert.equal(args[0].length, 1);
assert.equal(Object.keys(args[1]).length, 0);
});
});
describe("#recipe", () => {
it("should get and fetch a new recipe on first getRecipe", async () => {
sinon
.stub(instance, "getFromRemoteSettings")
.returns(Promise.resolve([]));
await instance.getRecipe();
assert.calledOnce(instance.getFromRemoteSettings);
assert.calledWith(
instance.getFromRemoteSettings,
"personality-provider-recipe"
);
});
it("should not fetch a recipe on getRecipe if cached", async () => {
sinon
.stub(instance, "getFromRemoteSettings")
.returns(Promise.resolve([]));
instance.recipes = ["blah"];
await instance.getRecipe();
assert.notCalled(instance.getFromRemoteSettings);
});
});
describe("#createInterestVector", () => {
let mockHistory = [];
beforeEach(() => {
mockHistory = [
{
title: "automotive",
description: "something about automotive",
url: "http://example.com/automotive",
frecency: 10,
},
{
title: "fashion",
description: "something about fashion",
url: "http://example.com/fashion",
frecency: 5,
},
{
title: "tech",
description: "something about tech",
url: "http://example.com/tech",
frecency: 1,
},
];
instance.fetchHistory = async () => mockHistory;
});
afterEach(() => {
globals.restore();
});
it("should gracefully handle history entries that fail", async () => {
mockHistory.push({ title: "fail" });
assert.isNotNull(await instance.createInterestVector());
});
it("should fail if the combiner fails", async () => {
mockHistory.push({ title: "combiner_fail", frecency: 111 });
let actual = await instance.createInterestVector();
assert.isNull(actual);
});
it("should process history, combine, and finalize", async () => {
let actual = await instance.createInterestVector();
assert.equal(actual.score, 1600);
});
});
describe("#calculateItemRelevanceScore", () => {
it("it should return score for uninitialized provider", () => {
instance.initialized = false;
assert.equal(instance.calculateItemRelevanceScore({ item_score: 2 }), 2);
});
it("it should return 1 for uninitialized provider and no score", () => {
instance.initialized = false;
assert.equal(instance.calculateItemRelevanceScore({}), 1);
});
it("it should return -1 for busted item", () => {
instance.initialized = true;
assert.equal(instance.calculateItemRelevanceScore({ title: "fail" }), -1);
});
it("it should return -1 for a busted ranking", () => {
instance.initialized = true;
instance.interestVector = { title: "fail", score: 10 };
assert.equal(
instance.calculateItemRelevanceScore({ title: "some item", score: 6 }),
-1
);
});
it("it should return a score, and not change with interestVector", () => {
instance.interestVector = { score: 10 };
instance.initialized = true;
assert.equal(instance.calculateItemRelevanceScore({ score: 2 }), 20);
assert.deepEqual(instance.interestVector, { score: 10 });
});
});
describe("#dispatchRelevanceScoreDuration", () => {
it("should dispatch PERSONALIZATION_V2_ITEM_RELEVANCE_SCORE_DURATION only if initialized", () => {
let dispatch = globals.sandbox.stub();
instance.dispatch = dispatch;
instance.initialized = false;
instance.dispatchRelevanceScoreDuration(1000);
assert.notCalled(dispatch);
instance.initialized = true;
instance.dispatchRelevanceScoreDuration(1000);
assert.calledOnce(dispatch);
assert.equal(
dispatch.firstCall.args[0].data.event,
"PERSONALIZATION_V2_ITEM_RELEVANCE_SCORE_DURATION"
);
});
});
describe("#setup", () => {
it("should setup two sync attachments", () => {
sinon.spy(instance, "setupSyncAttachment");
instance.setup();
assert.calledTwice(instance.setupSyncAttachment);
});
});
describe("#teardown", () => {
it("should teardown two sync attachments", () => {
sinon.spy(instance, "teardownSyncAttachment");
instance.teardown();
assert.calledTwice(instance.teardownSyncAttachment);
});
});
describe("#fetchHistory", () => {
it("should return a history object for fetchHistory", async () => {
const history = await instance.fetchHistory(["requiredColumn"], 1, 1);
assert.equal(
history.sql,
`SELECT url, title, visit_count, frecency, last_visit_date, description\n FROM moz_places\n WHERE last_visit_date >= 1000000\n AND last_visit_date < 1000000 AND IFNULL(requiredColumn, '') <> '' LIMIT 30000`
);
assert.equal(history.options.columns.length, 1);
assert.equal(Object.keys(history.options.params).length, 0);
});
});
describe("#attachments", () => {
it("should sync remote settings collection from onSync", async () => {
sinon.stub(instance, "deleteAttachment").returns(Promise.resolve({}));
sinon
.stub(instance, "maybeDownloadAttachment")
.returns(Promise.resolve({}));
await instance.onSync({
data: {
created: ["create-1", "create-2"],
updated: [
{ old: "update-old-1", new: "update-new-1" },
{ old: "update-old-2", new: "update-new-2" },
],
deleted: ["delete-2", "delete-1"],
},
});
assert(instance.maybeDownloadAttachment.withArgs("create-1").calledOnce);
assert(instance.maybeDownloadAttachment.withArgs("create-2").calledOnce);
assert(
instance.maybeDownloadAttachment.withArgs("update-new-1").calledOnce
);
assert(
instance.maybeDownloadAttachment.withArgs("update-new-2").calledOnce
);
assert(instance.deleteAttachment.withArgs("delete-1").calledOnce);
assert(instance.deleteAttachment.withArgs("delete-2").calledOnce);
assert(instance.deleteAttachment.withArgs("update-old-1").calledOnce);
assert(instance.deleteAttachment.withArgs("update-old-2").calledOnce);
});
it("should write a file from _downloadAttachment", async () => {
const fetchStub = globals.sandbox.stub(global, "fetch").resolves({
ok: true,
arrayBuffer: async () => {},
});
baseURLStub = "/";
const writeAtomicStub = globals.sandbox
.stub(global.OS.File, "writeAtomic")
.resolves(Promise.resolve());
globals.sandbox
.stub(global.OS.Path, "join")
.callsFake((first, second) => first + second);
globals.set("Uint8Array", class Uint8Array {});
await instance._downloadAttachment({
attachment: { location: "location", filename: "filename" },
});
const fetchArgs = fetchStub.firstCall.args;
assert.equal(fetchArgs[0], "/location");
const writeArgs = writeAtomicStub.firstCall.args;
assert.equal(writeArgs[0], "/filename");
assert.equal(writeArgs[2].tmpPath, "/filename.tmp");
});
it("should call reportError from _downloadAttachment if not valid response", async () => {
globals.sandbox.stub(global, "fetch").resolves({ ok: false });
globals.sandbox.spy(global.Cu, "reportError");
baseURLStub = "/";
await instance._downloadAttachment({
attachment: { location: "location", filename: "filename" },
});
assert.calledWith(Cu.reportError, "Failed to fetch /location: undefined");
});
it("should attempt _downloadAttachment three times for maybeDownloadAttachment", async () => {
let existsStub;
let statStub;
let attachmentStub;
sinon.stub(instance, "_downloadAttachment").returns(Promise.resolve());
sinon.stub(instance, "_getFileStr").returns(Promise.resolve("1"));
const makeDirStub = globals.sandbox
.stub(global.OS.File, "makeDir")
.returns(Promise.resolve());
globals.sandbox
.stub(global.OS.Path, "join")
.callsFake((first, second) => first + second);
existsStub = globals.sandbox
.stub(global.OS.File, "exists")
.returns(Promise.resolve(true));
statStub = globals.sandbox
.stub(global.OS.File, "stat")
.returns(Promise.resolve({ size: "1" }));
attachmentStub = {
attachment: {
filename: "file",
hash: "30",
size: "1",
},
};
await instance.maybeDownloadAttachment(attachmentStub);
assert.calledWith(makeDirStub, "/");
assert.calledOnce(existsStub);
assert.calledOnce(statStub);
assert.calledOnce(instance._getFileStr);
assert.notCalled(instance._downloadAttachment);
instance._getFileStr.resetHistory();
existsStub.resetHistory();
statStub.resetHistory();
attachmentStub = {
attachment: {
filename: "file",
hash: "31",
size: "1",
},
};
await instance.maybeDownloadAttachment(attachmentStub);
assert.calledThrice(existsStub);
assert.calledThrice(statStub);
assert.calledThrice(instance._getFileStr);
assert.calledThrice(instance._downloadAttachment);
});
it("should remove attachments when calling deleteAttachment", async () => {
const makeDirStub = globals.sandbox
.stub(global.OS.File, "makeDir")
.returns(Promise.resolve());
const removeStub = globals.sandbox
.stub(global.OS.File, "remove")
.returns(Promise.resolve());
const removeEmptyDirStub = globals.sandbox
.stub(global.OS.File, "removeEmptyDir")
.returns(Promise.resolve());
globals.sandbox
.stub(global.OS.Path, "join")
.callsFake((first, second) => first + second);
await instance.deleteAttachment({ attachment: { filename: "filename" } });
assert.calledOnce(makeDirStub);
assert.calledOnce(removeStub);
assert.calledOnce(removeEmptyDirStub);
assert.calledWith(removeStub, "/filename", { ignoreAbsent: true });
});
it("should return JSON when calling getAttachment", async () => {
sinon
.stub(instance, "maybeDownloadAttachment")
.returns(Promise.resolve());
sinon.stub(instance, "_getFileStr").returns(Promise.resolve("{}"));
const reportErrorStub = globals.sandbox.stub(global.Cu, "reportError");
globals.sandbox
.stub(global.OS.Path, "join")
.callsFake((first, second) => first + second);
const record = { attachment: { filename: "filename" } };
let returnValue = await instance.getAttachment(record);
assert.notCalled(reportErrorStub);
assert.calledOnce(instance._getFileStr);
assert.calledWith(instance._getFileStr, "/filename");
assert.calledOnce(instance.maybeDownloadAttachment);
assert.calledWith(instance.maybeDownloadAttachment, record);
assert.deepEqual(returnValue, {});
instance._getFileStr.restore();
sinon.stub(instance, "_getFileStr").returns(Promise.resolve({}));
returnValue = await instance.getAttachment(record);
assert.calledOnce(reportErrorStub);
assert.calledWith(
reportErrorStub,
"Failed to load /filename: JSON.parse: unexpected character at line 1 column 2 of the JSON data"
);
assert.deepEqual(returnValue, {});
});
it("should read and decode a file with _getFileStr", async () => {
global.OS.File.read = async path => {
if (path === "/filename") {
return "binaryData";
}
return "";
};
globals.set("gTextDecoder", {
decode: async binaryData => {
if (binaryData === "binaryData") {
return "binaryData";
}
return "";
},
});
const returnValue = await instance._getFileStr("/filename");
assert.equal(returnValue, "binaryData");
});
});
});

9
browser/components/newtab/test/unit/lib/NaiveBayesTextTagger.test.js → browser/components/newtab/test/unit/lib/PersonalityProvider/NaiveBayesTextTagger.test.js
Просмотреть файл

@ -1,5 +1,8 @@
import { NaiveBayesTextTagger } from "lib/NaiveBayesTextTagger.jsm";
import { tokenize } from "lib/Tokenize.jsm";
import { NaiveBayesTextTagger } from "lib/PersonalityProvider/NaiveBayesTextTagger.jsm";
import {
tokenize,
toksToTfIdfVector,
} from "lib/PersonalityProvider/Tokenize.jsm";
const EPSILON = 0.00001;
@ -58,7 +61,7 @@ describe("Naive Bayes Tagger", () => {
words: [10, 5.070533913528382],
},
};
let instance = new NaiveBayesTextTagger(model);
let instance = new NaiveBayesTextTagger(model, toksToTfIdfVector);
let testCases = [
{

9
browser/components/newtab/test/unit/lib/NmfTextTagger.test.js → browser/components/newtab/test/unit/lib/PersonalityProvider/NmfTextTagger.test.js
Просмотреть файл

@ -1,5 +1,8 @@
import { NmfTextTagger } from "lib/NmfTextTagger.jsm";
import { tokenize } from "lib/Tokenize.jsm";
import { NmfTextTagger } from "lib/PersonalityProvider/NmfTextTagger.jsm";
import {
tokenize,
toksToTfIdfVector,
} from "lib/PersonalityProvider/Tokenize.jsm";
const EPSILON = 0.00001;
@ -992,7 +995,7 @@ describe("NMF Tagger", () => {
},
};
let instance = new NmfTextTagger(model);
let instance = new NmfTextTagger(model, toksToTfIdfVector);
let testCases = [
{

398
browser/components/newtab/test/unit/lib/PersonalityProvider/PersonalityProvider.test.js Normal file
Просмотреть файл

@ -0,0 +1,398 @@
import { GlobalOverrider } from "test/unit/utils";
import { PersonalityProvider } from "lib/PersonalityProvider/PersonalityProvider.jsm";
const TIME_SEGMENTS = [
{ id: "hour", startTime: 3600, endTime: 0, weightPosition: 1 },
{ id: "day", startTime: 86400, endTime: 3600, weightPosition: 0.75 },
{ id: "week", startTime: 604800, endTime: 86400, weightPosition: 0.5 },
{ id: "weekPlus", startTime: null, endTime: 604800, weightPosition: 0.25 },
];
const PARAMETER_SETS = {
paramSet1: {
recencyFactor: 0.5,
frequencyFactor: 0.5,
combinedDomainFactor: 0.5,
perfectFrequencyVisits: 10,
perfectCombinedDomainScore: 2,
multiDomainBoost: 0.1,
itemScoreFactor: 0,
},
paramSet2: {
recencyFactor: 1,
frequencyFactor: 0.7,
combinedDomainFactor: 0.8,
perfectFrequencyVisits: 10,
perfectCombinedDomainScore: 2,
multiDomainBoost: 0.1,
itemScoreFactor: 0,
},
};
describe("Personality Provider", () => {
let instance;
let RemoteSettingsStub;
let RemoteSettingsOnStub;
let RemoteSettingsOffStub;
let RemoteSettingsGetStub;
let sandbox;
let globals;
let baseURLStub;
beforeEach(() => {
sandbox = sinon.createSandbox();
globals = new GlobalOverrider();
RemoteSettingsOnStub = sandbox.stub().returns();
RemoteSettingsOffStub = sandbox.stub().returns();
RemoteSettingsGetStub = sandbox.stub().returns([]);
RemoteSettingsStub = name => ({
get: RemoteSettingsGetStub,
on: RemoteSettingsOnStub,
off: RemoteSettingsOffStub,
});
sinon.spy(global, "BasePromiseWorker");
sinon.spy(global.BasePromiseWorker.prototype, "post");
baseURLStub = "https://baseattachmentsurl";
global.fetch = async server => ({
ok: true,
json: async () => {
if (server === "services.settings.server/") {
return { capabilities: { attachments: { base_url: baseURLStub } } };
}
return {};
},
});
globals.sandbox
.stub(global.Services.prefs, "getCharPref")
.callsFake(pref => pref);
globals.set("RemoteSettings", RemoteSettingsStub);
instance = new PersonalityProvider(TIME_SEGMENTS, PARAMETER_SETS);
instance.interestConfig = {
history_item_builder: "history_item_builder",
history_required_fields: ["a", "b", "c"],
interest_finalizer: "interest_finalizer",
item_to_rank_builder: "item_to_rank_builder",
item_ranker: "item_ranker",
interest_combiner: "interest_combiner",
};
});
afterEach(() => {
sinon.restore();
sandbox.restore();
globals.restore();
});
describe("#personalityProviderWorker", () => {
it("should create a new promise worker on first call", async () => {
const { personalityProviderWorker } = instance;
assert.calledOnce(global.BasePromiseWorker);
assert.isDefined(personalityProviderWorker);
});
it("should cache _personalityProviderWorker on first call", async () => {
instance._personalityProviderWorker = null;
const { personalityProviderWorker } = instance;
assert.isDefined(instance._personalityProviderWorker);
assert.isDefined(personalityProviderWorker);
});
it("should use old promise worker on second call", async () => {
let { personalityProviderWorker } = instance;
personalityProviderWorker = instance.personalityProviderWorker;
assert.calledOnce(global.BasePromiseWorker);
assert.isDefined(personalityProviderWorker);
});
});
describe("#_getBaseAttachmentsURL", () => {
it("should return a fresh value", async () => {
await instance._getBaseAttachmentsURL();
assert.equal(instance._baseAttachmentsURL, baseURLStub);
});
it("should return a cached value", async () => {
const cachedURL = "cached";
instance._baseAttachmentsURL = cachedURL;
await instance._getBaseAttachmentsURL();
assert.equal(instance._baseAttachmentsURL, cachedURL);
});
});
describe("#setup", () => {
it("should setup two sync attachments", () => {
sinon.spy(instance, "setupSyncAttachment");
instance.setup();
assert.calledTwice(instance.setupSyncAttachment);
});
});
describe("#teardown", () => {
it("should teardown two sync attachments", () => {
sinon.spy(instance, "teardownSyncAttachment");
instance.teardown();
assert.calledTwice(instance.teardownSyncAttachment);
});
});
describe("#setupSyncAttachment", () => {
it("should call remote settings on twice for setupSyncAttachment", () => {
assert.calledTwice(RemoteSettingsOnStub);
});
});
describe("#teardownSyncAttachment", () => {
it("should call remote settings off for teardownSyncAttachment", () => {
instance.teardownSyncAttachment();
assert.calledOnce(RemoteSettingsOffStub);
});
});
describe("#onSync", () => {
it("should call worker onSync", () => {
instance.onSync();
assert.calledWith(global.BasePromiseWorker.prototype.post, "onSync");
});
});
describe("#getAttachment", () => {
it("should call worker onSync", () => {
instance.getAttachment();
assert.calledWith(
global.BasePromiseWorker.prototype.post,
"getAttachment"
);
});
});
describe("#getRecipe", () => {
it("should call worker getRecipe and remote settings get", async () => {
RemoteSettingsGetStub = sandbox.stub().returns([
{
key: 1,
},
]);
sinon.spy(instance, "getAttachment");
RemoteSettingsStub = name => ({
get: RemoteSettingsGetStub,
on: RemoteSettingsOnStub,
off: RemoteSettingsOffStub,
});
globals.set("RemoteSettings", RemoteSettingsStub);
const result = await instance.getRecipe();
assert.calledOnce(RemoteSettingsGetStub);
assert.calledOnce(instance.getAttachment);
assert.equal(result.recordKey, 1);
});
});
describe("#fetchHistory", () => {
it("should return a history object for fetchHistory", async () => {
const history = await instance.fetchHistory(["requiredColumn"], 1, 1);
assert.equal(
history.sql,
`SELECT url, title, visit_count, frecency, last_visit_date, description\n FROM moz_places\n WHERE last_visit_date >= 1000000\n AND last_visit_date < 1000000 AND IFNULL(requiredColumn, '') <> '' LIMIT 30000`
);
assert.equal(history.options.columns.length, 1);
assert.equal(Object.keys(history.options.params).length, 0);
});
});
describe("#getHistory", () => {
it("should return an empty array", async () => {
instance.interestConfig = {
history_required_fields: [],
};
const result = await instance.getHistory();
assert.equal(result.length, 0);
});
it("should call fetchHistory", async () => {
sinon.spy(instance, "fetchHistory");
await instance.getHistory();
});
});
describe("#setBaseAttachmentsURL", () => {
it("should call worker setBaseAttachmentsURL", async () => {
await instance.setBaseAttachmentsURL();
assert.calledWith(
global.BasePromiseWorker.prototype.post,
"setBaseAttachmentsURL"
);
});
});
describe("#setInterestConfig", () => {
it("should call worker setInterestConfig", async () => {
await instance.setInterestConfig();
assert.calledWith(
global.BasePromiseWorker.prototype.post,
"setInterestConfig"
);
});
});
describe("#setInterestVector", () => {
it("should call worker setInterestVector", async () => {
await instance.setInterestVector();
assert.calledWith(
global.BasePromiseWorker.prototype.post,
"setInterestVector"
);
});
});
describe("#fetchModels", () => {
it("should call worker fetchModels and remote settings get", async () => {
await instance.fetchModels();
assert.calledOnce(RemoteSettingsGetStub);
assert.calledWith(global.BasePromiseWorker.prototype.post, "fetchModels");
});
});
describe("#generateTaggers", () => {
it("should call worker generateTaggers", async () => {
await instance.generateTaggers();
assert.calledWith(
global.BasePromiseWorker.prototype.post,
"generateTaggers"
);
});
});
describe("#generateRecipeExecutor", () => {
it("should call worker generateRecipeExecutor", async () => {
await instance.generateRecipeExecutor();
assert.calledWith(
global.BasePromiseWorker.prototype.post,
"generateRecipeExecutor"
);
});
});
describe("#createInterestVector", () => {
it("should call worker createInterestVector", async () => {
await instance.createInterestVector();
assert.calledWith(
global.BasePromiseWorker.prototype.post,
"createInterestVector"
);
});
});
describe("#init", () => {
beforeEach(() => {
sandbox.stub(instance, "dispatch").returns();
});
it("should return early if setInterestConfig fails", async () => {
sandbox.stub(instance, "setBaseAttachmentsURL").returns();
sandbox.stub(instance, "setInterestConfig").returns();
instance.interestConfig = null;
await instance.init();
assert.calledWithMatch(instance.dispatch, {
data: {
event: "PERSONALIZATION_V2_GET_RECIPE_ERROR",
},
});
});
it("should return early if fetchModels fails", async () => {
sandbox.stub(instance, "setBaseAttachmentsURL").returns();
sandbox.stub(instance, "setInterestConfig").returns();
sandbox.stub(instance, "fetchModels").resolves({
ok: false,
});
await instance.init();
assert.calledWithMatch(instance.dispatch, {
data: {
event: "PERSONALIZATION_V2_FETCH_MODELS_ERROR",
},
});
});
it("should return early if createInterestVector fails", async () => {
sandbox.stub(instance, "setBaseAttachmentsURL").returns();
sandbox.stub(instance, "setInterestConfig").returns();
sandbox.stub(instance, "fetchModels").resolves({
ok: true,
});
sandbox.stub(instance, "generateRecipeExecutor").resolves({
ok: true,
});
sandbox.stub(instance, "createInterestVector").resolves({
ok: false,
});
await instance.init();
assert.calledWithMatch(instance.dispatch, {
data: {
event: "PERSONALIZATION_V2_CREATE_INTEREST_VECTOR_ERROR",
},
});
});
it("should call callback on successful init", async () => {
sandbox.stub(instance, "setBaseAttachmentsURL").returns();
sandbox.stub(instance, "setInterestConfig").returns();
sandbox.stub(instance, "fetchModels").resolves({
ok: true,
});
sandbox.stub(instance, "generateRecipeExecutor").resolves({
ok: true,
});
sandbox.stub(instance, "createInterestVector").resolves({
ok: true,
});
sandbox.stub(instance, "setInterestVector").resolves();
const callback = globals.sandbox.stub();
await instance.init(callback);
assert.calledOnce(callback);
assert.isTrue(instance.initialized);
});
it("should do generic init stuff when calling init with no cache", async () => {
sandbox.stub(instance, "setBaseAttachmentsURL").returns();
sandbox.stub(instance, "setInterestConfig").returns();
sandbox.stub(instance, "fetchModels").resolves({
ok: true,
});
sandbox.stub(instance, "generateRecipeExecutor").resolves({
ok: true,
});
sandbox.stub(instance, "createInterestVector").resolves({
ok: true,
interestVector: "interestVector",
});
sandbox.stub(instance, "setInterestVector").resolves();
await instance.init();
assert.calledOnce(instance.setBaseAttachmentsURL);
assert.calledOnce(instance.setInterestConfig);
assert.calledOnce(instance.fetchModels);
assert.calledOnce(instance.generateRecipeExecutor);
assert.calledOnce(instance.createInterestVector);
assert.calledOnce(instance.setInterestVector);
});
});
describe("#dispatchRelevanceScoreDuration", () => {
beforeEach(() => {
sandbox.stub(instance, "dispatch").returns();
});
it("should dispatch PERSONALIZATION_V2_ITEM_RELEVANCE_SCORE_DURATION only if initialized", () => {
let dispatch = globals.sandbox.stub();
instance.dispatch = dispatch;
instance.initialized = false;
instance.dispatchRelevanceScoreDuration(1000);
assert.notCalled(dispatch);
instance.initialized = true;
instance.dispatchRelevanceScoreDuration(1000);
assert.calledOnce(dispatch);
assert.equal(
dispatch.firstCall.args[0].data.event,
"PERSONALIZATION_V2_ITEM_RELEVANCE_SCORE_DURATION"
);
});
});
describe("#calculateItemRelevanceScore", () => {
it("should return score for uninitialized provider", () => {
instance.initialized = false;
assert.equal(instance.calculateItemRelevanceScore({ item_score: 2 }), 2);
});
it("should post calculateItemRelevanceScore to PersonalityProviderWorker", async () => {
instance.initialized = true;
await instance.calculateItemRelevanceScore({ item_score: 2 });
assert.calledWith(
global.BasePromiseWorker.prototype.post,
"calculateItemRelevanceScore"
);
});
});
describe("#getAffinities", () => {
it("should return correct data for getAffinities", () => {
const affinities = instance.getAffinities();
assert.isDefined(affinities.timeSegments);
assert.isDefined(affinities.parameterSets);
});
});
});

430
browser/components/newtab/test/unit/lib/PersonalityProvider/PersonalityProviderWorkerClass.test.js Normal file
Просмотреть файл

@ -0,0 +1,430 @@
import { GlobalOverrider } from "test/unit/utils";
import { PersonalityProviderWorker } from "lib/PersonalityProvider/PersonalityProviderWorkerClass.jsm";
import {
tokenize,
toksToTfIdfVector,
} from "lib/PersonalityProvider/Tokenize.jsm";
import { RecipeExecutor } from "lib/PersonalityProvider/RecipeExecutor.jsm";
import { NmfTextTagger } from "lib/PersonalityProvider/NmfTextTagger.jsm";
import { NaiveBayesTextTagger } from "lib/PersonalityProvider/NaiveBayesTextTagger.jsm";
describe("Personality Provider Worker Class", () => {
let instance;
let globals;
let sandbox;
beforeEach(() => {
sandbox = sinon.createSandbox();
globals = new GlobalOverrider();
globals.set("tokenize", tokenize);
globals.set("toksToTfIdfVector", toksToTfIdfVector);
globals.set("NaiveBayesTextTagger", NaiveBayesTextTagger);
globals.set("NmfTextTagger", NmfTextTagger);
globals.set("RecipeExecutor", RecipeExecutor);
instance = new PersonalityProviderWorker();
// mock the RecipeExecutor
instance.recipeExecutor = {
executeRecipe: (item, recipe) => {
if (recipe === "history_item_builder") {
if (item.title === "fail") {
return null;
}
return {
title: item.title,
score: item.frecency,
type: "history_item",
};
} else if (recipe === "interest_finalizer") {
return {
title: item.title,
score: item.score * 100,
type: "interest_vector",
};
} else if (recipe === "item_to_rank_builder") {
if (item.title === "fail") {
return null;
}
return {
item_title: item.title,
item_score: item.score,
type: "item_to_rank",
};
} else if (recipe === "item_ranker") {
if (item.title === "fail" || item.item_title === "fail") {
return null;
}
return {
title: item.title,
score: item.item_score * item.score,
type: "ranked_item",
};
}
return null;
},
executeCombinerRecipe: (item1, item2, recipe) => {
if (recipe === "interest_combiner") {
if (
item1.title === "combiner_fail" ||
item2.title === "combiner_fail"
) {
return null;
}
if (item1.type === undefined) {
item1.type = "combined_iv";
}
if (item1.score === undefined) {
item1.score = 0;
}
return { type: item1.type, score: item1.score + item2.score };
}
return null;
},
};
instance.interestConfig = {
history_item_builder: "history_item_builder",
history_required_fields: ["a", "b", "c"],
interest_finalizer: "interest_finalizer",
item_to_rank_builder: "item_to_rank_builder",
item_ranker: "item_ranker",
interest_combiner: "interest_combiner",
};
});
afterEach(() => {
sinon.restore();
sandbox.restore();
globals.restore();
});
describe("#_getFileStr", () => {
it("should decode file from filepath", () => {
globals.set(
"TextDecoder",
class {
decode() {
return "DECODED!";
}
}
);
const result = instance._getFileStr("filepath");
assert.equal(result, "DECODED!");
});
});
describe("#setBaseAttachmentsURL", () => {
it("should set baseAttachmentsURL", () => {
instance.setBaseAttachmentsURL("url");
assert.equal(instance.baseAttachmentsURL, "url");
});
});
describe("#setInterestConfig", () => {
it("should set interestConfig", () => {
instance.setInterestConfig("config");
assert.equal(instance.interestConfig, "config");
});
});
describe("#setInterestVector", () => {
it("should set interestVector", () => {
instance.setInterestVector("vector");
assert.equal(instance.interestVector, "vector");
});
});
describe("#onSync", () => {
it("should sync remote settings collection from onSync", () => {
sinon.stub(instance, "deleteAttachment").returns({});
sinon.stub(instance, "maybeDownloadAttachment").returns({});
instance.onSync({
data: {
created: ["create-1", "create-2"],
updated: [
{ old: "update-old-1", new: "update-new-1" },
{ old: "update-old-2", new: "update-new-2" },
],
deleted: ["delete-2", "delete-1"],
},
});
assert(instance.maybeDownloadAttachment.withArgs("create-1").calledOnce);
assert(instance.maybeDownloadAttachment.withArgs("create-2").calledOnce);
assert(
instance.maybeDownloadAttachment.withArgs("update-new-1").calledOnce
);
assert(
instance.maybeDownloadAttachment.withArgs("update-new-2").calledOnce
);
assert(instance.deleteAttachment.withArgs("delete-1").calledOnce);
assert(instance.deleteAttachment.withArgs("delete-2").calledOnce);
assert(instance.deleteAttachment.withArgs("update-old-1").calledOnce);
assert(instance.deleteAttachment.withArgs("update-old-2").calledOnce);
});
});
describe("#maybeDownloadAttachment", () => {
it("should attempt _downloadAttachment three times for maybeDownloadAttachment", () => {
let existsStub;
let statStub;
let attachmentStub;
sinon.stub(instance, "_downloadAttachment").returns();
const makeDirStub = globals.sandbox
.stub(global.OS.File, "makeDir")
.returns();
globals.sandbox
.stub(global.OS.Path, "join")
.callsFake((first, second) => first + second);
existsStub = globals.sandbox.stub(global.OS.File, "exists").returns(true);
statStub = globals.sandbox
.stub(global.OS.File, "stat")
.returns({ size: "1" });
attachmentStub = {
attachment: {
filename: "file",
size: "1",
},
};
instance.maybeDownloadAttachment(attachmentStub);
assert.calledWith(makeDirStub, "/");
assert.calledOnce(existsStub);
assert.calledOnce(statStub);
assert.notCalled(instance._downloadAttachment);
existsStub.resetHistory();
statStub.resetHistory();
attachmentStub = {
attachment: {
filename: "file",
size: "2",
},
};
instance.maybeDownloadAttachment(attachmentStub);
assert.calledThrice(existsStub);
assert.calledThrice(statStub);
assert.calledThrice(instance._downloadAttachment);
});
});
describe("#_downloadAttachment", () => {
beforeEach(() => {
globals.sandbox
.stub(global.OS.Path, "join")
.callsFake((first, second) => first + second);
globals.set("Uint8Array", class Uint8Array {});
});
it("should write a file from _downloadAttachment", () => {
globals.set(
"XMLHttpRequest",
class {
constructor() {
this.status = 200;
this.response = "response!";
}
open() {}
setRequestHeader() {}
send() {}
}
);
const writeAtomicStub = globals.sandbox
.stub(global.OS.File, "writeAtomic")
.resolves();
instance._downloadAttachment({
attachment: { location: "location", filename: "filename" },
});
const writeArgs = writeAtomicStub.firstCall.args;
assert.equal(writeArgs[0], "/filename");
assert.equal(writeArgs[2].tmpPath, "/filename.tmp");
});
it("should call console.error from _downloadAttachment if not valid response", () => {
globals.set(
"XMLHttpRequest",
class {
constructor() {
this.status = 0;
this.response = "response!";
}
open() {}
setRequestHeader() {}
send() {}
}
);
const writeAtomicStub = globals.sandbox
.stub(global.OS.File, "writeAtomic")
.resolves();
instance._downloadAttachment({
attachment: { location: "location", filename: "filename" },
});
assert.notCalled(writeAtomicStub);
});
});
describe("#deleteAttachment", () => {
it("should remove attachments when calling deleteAttachment", () => {
const makeDirStub = globals.sandbox
.stub(global.OS.File, "makeDir")
.returns();
const removeStub = globals.sandbox
.stub(global.OS.File, "remove")
.returns();
const removeEmptyDirStub = globals.sandbox
.stub(global.OS.File, "removeEmptyDir")
.returns();
globals.sandbox
.stub(global.OS.Path, "join")
.callsFake((first, second) => first + second);
instance.deleteAttachment({ attachment: { filename: "filename" } });
assert.calledOnce(makeDirStub);
assert.calledOnce(removeStub);
assert.calledOnce(removeEmptyDirStub);
assert.calledWith(removeStub, "/filename", { ignoreAbsent: true });
});
});
describe("#getAttachment", () => {
it("should return JSON when calling getAttachment", () => {
sinon.stub(instance, "maybeDownloadAttachment").returns();
sinon.stub(instance, "_getFileStr").returns("{}");
globals.sandbox
.stub(global.OS.Path, "join")
.callsFake((first, second) => first + second);
const record = { attachment: { filename: "filename" } };
let returnValue = instance.getAttachment(record);
assert.calledOnce(instance._getFileStr);
assert.calledWith(instance._getFileStr, "/filename");
assert.calledOnce(instance.maybeDownloadAttachment);
assert.calledWith(instance.maybeDownloadAttachment, record);
assert.deepEqual(returnValue, {});
instance._getFileStr.restore();
sinon.stub(instance, "_getFileStr").returns({});
returnValue = instance.getAttachment(record);
assert.deepEqual(returnValue, {});
});
});
describe("#fetchModels", () => {
it("should return ok true", () => {
sinon.stub(instance, "getAttachment").returns();
const result = instance.fetchModels([{ key: 1234 }]);
assert.isTrue(result.ok);
assert.deepEqual(instance.models, [{ recordKey: 1234 }]);
});
it("should return ok false", () => {
sinon.stub(instance, "getAttachment").returns();
const result = instance.fetchModels([]);
assert.isTrue(!result.ok);
});
});
describe("#generateTaggers", () => {
it("should generate taggers from modelKeys", () => {
const modelKeys = ["nb_model_sports", "nmf_model_sports"];
instance.models = [
{ recordKey: "nb_model_sports", model_type: "nb" },
{
recordKey: "nmf_model_sports",
model_type: "nmf",
parent_tag: "nmf_sports_parent_tag",
},
];
instance.generateTaggers(modelKeys);
assert.equal(instance.taggers.nbTaggers.length, 1);
assert.equal(Object.keys(instance.taggers.nmfTaggers).length, 1);
});
it("should skip any models not in modelKeys", () => {
const modelKeys = ["nb_model_sports"];
instance.models = [
{ recordKey: "nb_model_sports", model_type: "nb" },
{
recordKey: "nmf_model_sports",
model_type: "nmf",
parent_tag: "nmf_sports_parent_tag",
},
];
instance.generateTaggers(modelKeys);
assert.equal(instance.taggers.nbTaggers.length, 1);
assert.equal(Object.keys(instance.taggers.nmfTaggers).length, 0);
});
it("should skip any models not defined", () => {
const modelKeys = ["nb_model_sports", "nmf_model_sports"];
instance.models = [{ recordKey: "nb_model_sports", model_type: "nb" }];
instance.generateTaggers(modelKeys);
assert.equal(instance.taggers.nbTaggers.length, 1);
assert.equal(Object.keys(instance.taggers.nmfTaggers).length, 0);
});
});
describe("#generateRecipeExecutor", () => {
it("should generate a recipeExecutor", () => {
instance.recipeExecutor = null;
instance.taggers = {};
instance.generateRecipeExecutor();
assert.isNotNull(instance.recipeExecutor);
});
});
describe("#createInterestVector", () => {
let mockHistory = [];
beforeEach(() => {
mockHistory = [
{
title: "automotive",
description: "something about automotive",
url: "http://example.com/automotive",
frecency: 10,
},
{
title: "fashion",
description: "something about fashion",
url: "http://example.com/fashion",
frecency: 5,
},
{
title: "tech",
description: "something about tech",
url: "http://example.com/tech",
frecency: 1,
},
];
});
it("should gracefully handle history entries that fail", () => {
mockHistory.push({ title: "fail" });
assert.isNotNull(instance.createInterestVector(mockHistory));
});
it("should fail if the combiner fails", () => {
mockHistory.push({ title: "combiner_fail", frecency: 111 });
let actual = instance.createInterestVector(mockHistory);
assert.isNull(actual);
});
it("should process history, combine, and finalize", () => {
let actual = instance.createInterestVector(mockHistory);
assert.equal(actual.interestVector.score, 1600);
});
});
describe("#calculateItemRelevanceScore", () => {
it("should return -1 for busted item", () => {
assert.equal(instance.calculateItemRelevanceScore({ title: "fail" }), -1);
});
it("should return -1 for a busted ranking", () => {
instance.interestVector = { title: "fail", score: 10 };
assert.equal(
instance.calculateItemRelevanceScore({ title: "some item", score: 6 }),
-1
);
});
it("should return a score, and not change with interestVector", () => {
instance.interestVector = { score: 10 };
assert.equal(instance.calculateItemRelevanceScore({ score: 2 }), 20);
assert.deepEqual(instance.interestVector, { score: 10 });
});
});
});

6
browser/components/newtab/test/unit/lib/RecipeExecutor.test.js → browser/components/newtab/test/unit/lib/PersonalityProvider/RecipeExecutor.test.js
Просмотреть файл

@ -1,4 +1,5 @@
import { RecipeExecutor } from "lib/RecipeExecutor.jsm";
import { RecipeExecutor } from "lib/PersonalityProvider/RecipeExecutor.jsm";
import { tokenize } from "lib/PersonalityProvider/Tokenize.jsm";
class MockTagger {
constructor(mode, tagScoreMap) {
@ -113,7 +114,8 @@ describe("RecipeExecutor", () => {
tag33: 0.5,
}),
tag4: new MockTagger("nmf", { tag41: 0.99 }),
}
},
tokenize
);
let item = null;

5
browser/components/newtab/test/unit/lib/Tokenize.test.js → browser/components/newtab/test/unit/lib/PersonalityProvider/Tokenize.test.js
Просмотреть файл

@ -1,4 +1,7 @@
import { tokenize, toksToTfIdfVector } from "lib/Tokenize.jsm";
import {
tokenize,
toksToTfIdfVector,
} from "lib/PersonalityProvider/Tokenize.jsm";
const EPSILON = 0.00001;

6
browser/components/newtab/test/unit/lib/RecommendationProviderSwitcher.test.js
Просмотреть файл

@ -138,12 +138,12 @@ describe("RecommendationProviderSwitcher", () => {
});
describe("#calculateItemRelevanceScore", () => {
it("should use personalized score with affinity provider", () => {
it("should use personalized score with affinity provider", async () => {
const item = {};
feed.affinityProvider = {
calculateItemRelevanceScore: () => 0.5,
calculateItemRelevanceScore: async () => 0.5,
};
feed.calculateItemRelevanceScore(item);
await feed.calculateItemRelevanceScore(item);
assert.equal(item.score, 0.5);
});
});

8
browser/components/newtab/test/unit/unit-entry.js
Просмотреть файл

@ -61,6 +61,12 @@ const TEST_GLOBAL = {
platform: "win",
},
UpdateUtils: { getUpdateChannel() {} },
BasePromiseWorker: class {
constructor() {
this.ExceptionHandlers = [];
}
post() {}
},
BrowserWindowTracker: { getTopWindow() {} },
ChromeUtils: {
defineModuleGetter() {},
@ -183,6 +189,8 @@ const TEST_GLOBAL = {
writeAtomic() {},
makeDir() {},
stat() {},
Error: {},
read() {},
exists() {},
remove() {},
removeEmptyDir() {},