keyword ranking page
This commit is contained in:
Olivier Yiptong
Родитель
0f5bdde5c1
Коммит
bd26c0c9e0
|
|
@ -0,0 +1,58 @@
|
|||
<!doctype html>
|
||||
<html ng-app="aboutKeywords">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<title>about:keywords</title>
|
||||
</head>
|
||||
<body ng-cloak>
|
||||
<div class="container">
|
||||
<div class="navbar">
|
||||
<h1>Keywords from your History</h1>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="container content" ng-controller="vizCtrl">
|
||||
<div class="dropdowns">
|
||||
<select ng-init="selectedType = getTypes()[0]" ng-model="selectedType" ng-options="val for val in getTypes()">
|
||||
<option value="" disabled>Select Type</option>
|
||||
</select>
|
||||
</div>
|
||||
<div class="btn-group">
|
||||
<button class="btn" ng-disabled='historyComputeInProgress' ng-click="processHistory()">Analyse Full History</button>
|
||||
</div>
|
||||
|
||||
<div class="alerts">
|
||||
<div ng-show="historyComputeInProgress" ng-hide="historyComputeComplete" class="alert alert-info">
|
||||
<p>Ranking data is being computed. This may take few minutes</p>
|
||||
<div class="progress progress-striped active">
|
||||
<div id="progressBar" class="bar" style="width: 0%;"></div>
|
||||
</div>
|
||||
<p><span class="badge">{{daysLeft}}</span> days of your history remaining</p>
|
||||
</div>
|
||||
|
||||
<div ng-show="historyComputeComplete && !countsAvailable" class="alert alert-info">{{emptyMessage}}</div>
|
||||
</div>
|
||||
|
||||
<div class="keywordsSection">
|
||||
<div ng-show="typeKeywords">
|
||||
|
||||
<h3>Keywords for dataset: {{selectedType}}</h3>
|
||||
<table class="table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Rank</th>
|
||||
<th>Keyword</th>
|
||||
<th>Occurrences</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr ng-repeat="keyword in typeKeywords">
|
||||
<td>{{$index + 1}}</td><td>{{keyword.key}}</td><td>{{keyword.value}}</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -0,0 +1,92 @@
|
|||
"use strict";
|
||||
|
||||
///// Chart initialization /////
|
||||
let types = ["url_title", "title"];
|
||||
|
||||
let DataService = function($rootScope) {
|
||||
this.rootScope = $rootScope;
|
||||
|
||||
// relay messages from the addon to the page
|
||||
self.port.on("message", message => {
|
||||
this.rootScope.$apply(_ => {
|
||||
this.rootScope.$broadcast(message.content.topic, message.content.data);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
DataService.prototype = {
|
||||
send: function _send(message, obj) {
|
||||
self.port.emit(message, obj);
|
||||
},
|
||||
}
|
||||
|
||||
let aboutKeywords = angular.module("aboutKeywords", []);
|
||||
aboutKeywords.service("dataService", DataService);
|
||||
|
||||
aboutKeywords.controller("vizCtrl", function($scope, dataService) {
|
||||
/** controller helpers **/
|
||||
$scope.getTypes = function () {
|
||||
return types;
|
||||
}
|
||||
|
||||
$scope._initialize = function () {
|
||||
$scope.historyComputeInProgress = false;
|
||||
$scope.historyComputeComplete = false;
|
||||
$scope.emptyMessage = "Your History was not analysed, please run the Full History Analysis.";
|
||||
$scope.countsAvailable = false;
|
||||
$scope.keywordCounts = [];
|
||||
$scope.daysLeft = null;
|
||||
$scope.daysLeftStart = null;
|
||||
dataService.send("chart_data_request");
|
||||
}
|
||||
$scope._initialize();
|
||||
|
||||
/** UI functionality **/
|
||||
|
||||
$scope.processHistory = function() {
|
||||
$scope._initialize();
|
||||
dataService.send("history_process");
|
||||
$scope.historyComputeInProgress = true;
|
||||
}
|
||||
|
||||
$scope.updateGraphs = function() {
|
||||
dataService.send("chart_data_request");
|
||||
}
|
||||
|
||||
$scope.$on("days_left", function(event, data) {
|
||||
$scope.historyComputeInProgress = true;
|
||||
if (!$scope.daysLeftStart) {
|
||||
$scope.daysLeftStart = data;
|
||||
}
|
||||
$scope.daysLeft = data;
|
||||
$scope.updateProgressBar();
|
||||
});
|
||||
|
||||
$scope.$watch("selectedType", () => {
|
||||
$scope.updateRankingDisplay();
|
||||
});
|
||||
|
||||
$scope.updateRankingDisplay = function() {
|
||||
let data = $scope.keywordCounts[$scope.selectedType];
|
||||
$scope.typeKeywords = data;
|
||||
}
|
||||
|
||||
$scope.$on("chart_init", function(event, data) {
|
||||
//ChartManager.graphKeywordsFromScratch(data, $scope.selectedType);
|
||||
$scope.keywordCounts = data;
|
||||
$scope.updateRankingDisplay();
|
||||
});
|
||||
|
||||
$scope.updateProgressBar = function() {
|
||||
let elem = document.querySelector("#progressBar");
|
||||
elem.style.width = (100 - Math.round($scope.daysLeft/$scope.daysLeftStart*100)) + "%";
|
||||
}
|
||||
});
|
||||
|
||||
self.port.on("style", function(file) {
|
||||
let link = document.createElement("link");
|
||||
link.setAttribute("href", file);
|
||||
link.setAttribute("rel", "stylesheet");
|
||||
link.setAttribute("type", "text/css");
|
||||
document.head.appendChild(link);
|
||||
});
|
||||
|
|
@ -296,6 +296,73 @@ let AboutInterests = {
|
|||
},
|
||||
};
|
||||
|
||||
let AboutKeywords = {
|
||||
_workers: [],
|
||||
factory: {
|
||||
contract: "@mozilla.org/network/protocol/about;1?what=keywords",
|
||||
|
||||
Component: Class({
|
||||
extends: Unknown,
|
||||
interfaces: ["nsIAboutModule"],
|
||||
|
||||
newChannel: function(uri) {
|
||||
let chan = Services.io.newChannel(data.url("about-keywords.html"), null, null);
|
||||
chan.originalURI = uri;
|
||||
return chan;
|
||||
},
|
||||
|
||||
getURIFlags: function(uri) {
|
||||
return Ci.nsIAboutModule.URI_SAFE_FOR_UNTRUSTED_CONTENT;
|
||||
}
|
||||
})
|
||||
},
|
||||
|
||||
processingDaysLeft: function(totalKeywordCountBolt) {
|
||||
for (let worker of AboutKeywords._workers) {
|
||||
worker.port.emit("message", {content: {topic: "days_left", data: totalKeywordCountBolt.numFromToday}});
|
||||
}
|
||||
},
|
||||
|
||||
historySubmitComplete: function() {
|
||||
for (let worker of AboutKeywords._workers) {
|
||||
worker.port.emit("message", {content: {topic: "ranking_data",
|
||||
data: { rankings: StudyApp.controller.getRankedInterests(), submitComplete: true}}});
|
||||
}
|
||||
},
|
||||
|
||||
observe : function(aSubject, aTopic, aData) {
|
||||
let dataObj = JSON.parse(aData);
|
||||
switch (aTopic) {
|
||||
}
|
||||
},
|
||||
|
||||
page: {
|
||||
contentScriptWhen: "start",
|
||||
contentScriptFile: [
|
||||
data.url("js/angular.min.js"),
|
||||
data.url("vendor/d3/d3.v3.min.js"),
|
||||
data.url("about-keywords.js"),
|
||||
],
|
||||
|
||||
include: ["about:keywords"],
|
||||
onAttach: function(worker) {
|
||||
AboutKeywords._workers.push(worker); // Set worker so that callback functions can access it after a page refresh.
|
||||
|
||||
worker.port.emit("style", data.url("css/devmenu/bootstrap.min.css"));
|
||||
worker.port.emit("style", data.url("css/devmenu/bootstrap-responsive.min.css"));
|
||||
worker.port.emit("style", data.url("css/devmenu/styles.css"));
|
||||
|
||||
worker.port.on("chart_data_request", () => {
|
||||
worker.port.emit("message", {content: {topic: "chart_init", data: StudyApp.controller.getTopKeywords(50)}});
|
||||
});
|
||||
worker.port.on("history_process", () => {
|
||||
storage.chartData = {};
|
||||
StudyApp.controller.resubmitHistory({report: AboutKeywords.processingDaysLeft, flush: true}).then(AboutKeywords.historySubmitComplete);
|
||||
});
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
let NYTimesRecommendations = {
|
||||
debug: false,
|
||||
workers: [],
|
||||
|
|
@ -560,6 +627,10 @@ let StudyApp = {
|
|||
Factory(AboutInterests.factory);
|
||||
PageMod(AboutInterests.page);
|
||||
|
||||
// about keywords
|
||||
Factory(AboutKeywords.factory);
|
||||
PageMod(AboutKeywords.page);
|
||||
|
||||
// get addon source URL
|
||||
AddonManager.getAddonByID(id, addon => {
|
||||
StudyApp.setSourceUri(addon.sourceURI);
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ const {WorkerFactory} = require("WorkerFactory");
|
|||
const {Dispatcher} = require("Dispatcher");
|
||||
const {Stream} = require("streams/core");
|
||||
const {DailyInterestsSpout} = require("streams/dailyInterestsSpout");
|
||||
const {DailyKeywordsSpout} = require("streams/dailyKeywordsSpout");
|
||||
const {TotalKeywordCountBolt} = require("streams/totalKeywordCountBolt");
|
||||
const {DayCountRankerBolt} = require("streams/dayCountRankerBolt");
|
||||
const {HostStripBolt} = require("streams/hostStripBolt");
|
||||
const {ChartDataProcessorBolt} = require("streams/chartDataProcessorBolt");
|
||||
|
|
@ -85,7 +85,7 @@ Controller.prototype = {
|
|||
// setup stream workers
|
||||
let streamObjects = {
|
||||
dailyInterestsSpout: DailyInterestsSpout.create(storageBackend),
|
||||
dailyKeywordsSpout: DailyKeywordsSpout.create(storageBackend),
|
||||
totalKeywordCountBolt: TotalKeywordCountBolt.create(storageBackend),
|
||||
rankerBolts: DayCountRankerBolt.batchCreate(this._workerFactory.getRankersDefinitions(), storageBackend),
|
||||
hostStripBolt: HostStripBolt.create(),
|
||||
chartDataProcessorBolt: ChartDataProcessorBolt.create(),
|
||||
|
|
@ -97,7 +97,7 @@ Controller.prototype = {
|
|||
}
|
||||
let stream = streamObjects.stream;
|
||||
stream.addNode(streamObjects.dailyInterestsSpout, true);
|
||||
stream.addNode(streamObjects.dailyKeywordsSpout, true);
|
||||
stream.addNode(streamObjects.totalKeywordCountBolt, true);
|
||||
streamObjects.rankerBolts.forEach(ranker => {
|
||||
stream.addNode(ranker);
|
||||
});
|
||||
|
|
@ -225,6 +225,7 @@ Controller.prototype = {
|
|||
this._streamObjects.rankerBolts.forEach(ranker => {
|
||||
ranker.clearData();
|
||||
});
|
||||
this._streamObjects.totalKeywordCountBolt.clearData();
|
||||
this._streamObjects.dailyInterestsSpout.clear();
|
||||
return this.submitHistory({daysAgo:this._historyDaysToResubmit, flush:options.flush, report:options.report});
|
||||
},
|
||||
|
|
@ -239,6 +240,26 @@ Controller.prototype = {
|
|||
return this._streamObjects.rankerBolts[0].getInterests();
|
||||
},
|
||||
|
||||
/**
|
||||
* Return the top <code>limit</code> keywords for a user.
|
||||
* @returns an object with the text as key and the count as value
|
||||
*/
|
||||
getTopKeywords: function(limit) {
|
||||
limit = limit || 250;
|
||||
let tokenCounts = {};
|
||||
for (let type in storage.keywordCounts) {
|
||||
tokenCounts[type] = [];
|
||||
for (let token in storage.keywordCounts[type]) {
|
||||
tokenCounts[type].push({key: token, value: storage.keywordCounts[type][token]});
|
||||
}
|
||||
tokenCounts[type].sort((a, b) => {
|
||||
return b.value - a.value;
|
||||
});
|
||||
tokenCounts[type] = tokenCounts[type].slice(0, limit);
|
||||
}
|
||||
return tokenCounts;
|
||||
},
|
||||
|
||||
getRankedInterestsForSurvey: function(len=30) {
|
||||
return Surveyor.orderInterestsForSurvey(
|
||||
this._streamObjects.rankerBolts.map(ranker => {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,62 @@
|
|||
"use strict";
|
||||
|
||||
const {createNode} = require("streams/core");
|
||||
const {storage} = require("sdk/simple-storage");
|
||||
const {mergeObjects} = require("Utils");
|
||||
const {DateUtils} = require("DateUtils");
|
||||
|
||||
let TotalKeywordCountBolt = {
|
||||
create: function _TKCB_create(storageBackend) {
|
||||
let totalKeywordCountBolt = createNode({
|
||||
identifier: "totalKeywordCountBolt",
|
||||
listenType: "keyword",
|
||||
emitType: null,
|
||||
|
||||
init: function _TKCB_init() {
|
||||
if (!this.storage.keywords) {
|
||||
this.storage.keywordCounts = {};
|
||||
this.latestProcessedDate = null;
|
||||
this.numFromToday = Number.POSITIVE_INFINITY;
|
||||
}
|
||||
},
|
||||
|
||||
_init_storage_entry: function _TKCB__init_storage_entry(type, keywords) {
|
||||
if (this.storage.keywordCounts[type] == null) {
|
||||
this.storage.keywordCounts[type] = {};
|
||||
}
|
||||
for (let kw of keywords) {
|
||||
if (this.storage.keywordCounts[type][kw] == null) {
|
||||
this.storage.keywordCounts[type][kw] = 0;
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
ingest: function _TKCB_ingest(message) {
|
||||
for(let i=0; i < message.length; i++) {
|
||||
let {details, dateVisits} = message[i];
|
||||
let {host, visitDate, visitCount, namespace, results} = details;
|
||||
for (let result of results) {
|
||||
this._init_storage_entry(result.type, result.keywords);
|
||||
for (let kw of result.keywords) {
|
||||
Object.keys(dateVisits).forEach(date => {
|
||||
this.storage.keywordCounts[result.type][kw] += 1;
|
||||
this.latestProcessedDate = date;
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
this.numFromToday = DateUtils.today() - this.latestProcessedDate;
|
||||
},
|
||||
|
||||
clearData: function _TKCB_clearData() {
|
||||
this.storage.keywordCounts = {};
|
||||
},
|
||||
|
||||
clearStorage: function _TKCB_clearStorage() {
|
||||
delete this.storage.keywordCounts;
|
||||
},
|
||||
}, {storage: storageBackend || storage});
|
||||
return totalKeywordCountBolt;
|
||||
}
|
||||
};
|
||||
exports.TotalKeywordCountBolt = TotalKeywordCountBolt;
|
||||
|
|
@ -0,0 +1,68 @@
|
|||
"use strict";
|
||||
|
||||
const {Cc, Ci, Cu} = require("chrome");
|
||||
Cu.import("resource://gre/modules/Task.jsm");
|
||||
|
||||
const {DateUtils} = require("DateUtils");
|
||||
const {TotalKeywordCountBolt} = require("streams/totalKeywordCountBolt");
|
||||
const test = require("sdk/test");
|
||||
|
||||
let today = DateUtils.today();
|
||||
let keywordWorkerOutput = {
|
||||
results: [
|
||||
{
|
||||
type: "groovy",
|
||||
keywords: ["toejam", "earl", "aliens", "earth"]
|
||||
},
|
||||
{
|
||||
type: "awesome",
|
||||
keywords: ["earthworm", "jim", "psycrow", "flying", "cow", "aliens"]
|
||||
},
|
||||
{
|
||||
type: "fantastic",
|
||||
keywords: ["cool", "spot", "sunglasses"]
|
||||
},
|
||||
]
|
||||
};
|
||||
|
||||
function setExpectedResults(count) {
|
||||
let keywordCounts = {};
|
||||
for (let typeData of keywordWorkerOutput.results) {
|
||||
if (keywordCounts[typeData.type] == null) {
|
||||
keywordCounts[typeData.type] = {};
|
||||
}
|
||||
for (let kw of typeData.keywords) {
|
||||
keywordCounts[typeData.type][kw] = count;
|
||||
}
|
||||
}
|
||||
return keywordCounts;
|
||||
}
|
||||
|
||||
exports["test totalKeywordCountBolt"] = function test_totalKeywordCountBolt(assert, done) {
|
||||
Task.spawn(function() {
|
||||
let dateVisits = {};
|
||||
dateVisits[today-2] = 1;
|
||||
|
||||
let storage = {};
|
||||
let totalKeywordCountBolt = TotalKeywordCountBolt.create(storage);
|
||||
|
||||
yield totalKeywordCountBolt.consume({meta: {}, message: [{details: keywordWorkerOutput, dateVisits: dateVisits}]});
|
||||
|
||||
let keywordCounts;
|
||||
|
||||
keywordCounts = setExpectedResults(1);
|
||||
assert.deepEqual(storage.keywordCounts, keywordCounts, "storage backend contains keyword counts");
|
||||
assert.equal(totalKeywordCountBolt.numFromToday, 2, "numFromToday counter is set correctly");
|
||||
|
||||
dateVisits = {};
|
||||
dateVisits[today-1] = 1;
|
||||
|
||||
yield totalKeywordCountBolt.consume({meta: {}, message: [{details: keywordWorkerOutput, dateVisits: dateVisits}]});
|
||||
keywordCounts = setExpectedResults(2);
|
||||
assert.deepEqual(storage.keywordCounts, keywordCounts, "keyword counts are incremental");
|
||||
assert.equal(totalKeywordCountBolt.numFromToday, 1, "numFromToday updates correctly");
|
||||
|
||||
}).then(done);
|
||||
};
|
||||
|
||||
test.run(exports);
|
||||
Загрузка…
Ссылка в новой задаче