Change k.Graph's data format to datum style.

This means that instead of an array of x/y pairs for each line on the
graph (which duplicates data like the created times), there is an array
of objects which represent a moment in time, and all the values at that
time.

This assumes that each point in time will have a value for all of the
data points, or at least most of them. Rickshaw does not make this
assumption, but k.Graph now does, mainly because it has held true in all
the data we have used so far.

This makes progress towards completing bug 865378.
This commit is contained in:
Mike Cooper 2013-04-29 15:46:57 -07:00
Родитель b79c5d12a4
Коммит d514a0fdb3
8 изменённых файлов: 234 добавлений и 231 удалений

Просмотреть файл

@ -8,8 +8,8 @@
{% block content %}
<h1>{{ _('Question Statistics') }}</h1>
{% if histogram %}
<div id="topic-stats" data-histogram="{{ histogram|json }}">
{% if graph %}
<div id="topic-stats" data-graph="{{ graph|json }}">
<h2 class="grid_12">{{ _('Topics') }}</h2>
<div class="graph-container grid_9 alpha">

Просмотреть файл

@ -1236,12 +1236,10 @@ def stats_topic_data(bucket_days, start, end):
# Massage the data to achieve 2 things:
# - All points between the earliest and the latest values have data,
# at a resolution of 1 day.
# - It is in a format Rickshaw will like.
# - It is in a format usable by k.Graph.
# - ie: [{"created": 1362774285, 'topic-1': 10, 'topic-2': 20}, ...]
# Construct a intermediatery data structure that allows for easy
# manipulation. Also find the min and max data at the same time.
# {'topic-1': [{1362774285: 100}, {1362784285: 200} ...}
for series in histograms_data.values():
for series in histograms_data.itervalues():
if series:
earliest_point = series[0]['key']
break
@ -1254,37 +1252,33 @@ def stats_topic_data(bucket_days, start, end):
for key, data in histograms_data.iteritems():
if not data:
continue
interim_data[key] = {}
for point in data:
x = point['key']
y = point['count']
earliest_point = min(earliest_point, x)
latest_point = max(latest_point, x)
interim_data[key][x] = y
timestamp = point['key']
value = point['count']
earliest_point = min(earliest_point, timestamp)
latest_point = max(latest_point, timestamp)
datum = interim_data.get(timestamp, {'date': timestamp})
datum[key] = value
interim_data[timestamp] = datum
# Interim data is now like
# {
# 1362774285: {'date': 1362774285, 'topic-1': 100, 'topic-2': 200},
# }
# Zero fill the interim data.
timestamp = earliest_point
while timestamp <= latest_point:
for key in interim_data:
if timestamp not in interim_data[key]:
interim_data[key][timestamp] = 0
datum = interim_data.get(timestamp, {'date': timestamp})
for key in histograms_data.iterkeys():
if key not in datum:
datum[key] = 0
timestamp += bucket
# Convert it into a format Rickshaw will be happy with.
# [
# {'name': 'series1', 'data': [{'x': 1362774285, 'y': 100}, ...]},
# ...
# ]
histograms = [
{
'name': name,
'data': sorted(({'x': x, 'y': y} for x, y in data.iteritems()),
key=lambda p: p['x']),
}
for name, data in interim_data.iteritems()
]
return histograms
# The keys are irrelevant, and the values are exactly what we want.
return interim_data.values()
def stats(request):
@ -1300,10 +1294,8 @@ def stats(request):
start = date.today() - timedelta(days=30)
end = date.today()
histogram = stats_topic_data(bucket_days, start, end)
data = {
'histogram': histogram,
'graph': stats_topic_data(bucket_days, start, end),
'form': form,
}

Просмотреть файл

@ -2171,13 +2171,10 @@ class HelpfulVoteTests(TestCaseBase):
args=[r.document.slug])
eq_(200, resp.status_code)
data = json.loads(resp.content)
eq_(3, len(data['series']))
eq_('yes', data['series'][0]['slug'])
eq_(1, len(data['series'][0]['data']))
eq_('no', data['series'][1]['slug'])
eq_(1, len(data['series'][1]['data']))
eq_('percent', data['series'][2]['slug'])
eq_(1, len(data['series'][2]['data']))
eq_(1, len(data['datums']))
assert 'yes' in data['datums'][0]
assert 'no' in data['datums'][0]
def test_helpfulvotes_graph_async_no(self):
r = self.document.current_revision
@ -2190,13 +2187,10 @@ class HelpfulVoteTests(TestCaseBase):
args=[r.document.slug])
eq_(200, resp.status_code)
data = json.loads(resp.content)
eq_(3, len(data['series']))
eq_('yes', data['series'][0]['slug'])
eq_(1, len(data['series'][0]['data']))
eq_('no', data['series'][1]['slug'])
eq_(1, len(data['series'][1]['data']))
eq_('percent', data['series'][2]['slug'])
eq_(1, len(data['series'][2]['data']))
eq_(1, len(data['datums']))
assert 'yes' in data['datums'][0]
assert 'no' in data['datums'][0]
def test_helpfulvotes_graph_async_no_votes(self):
r = self.document.current_revision
@ -2205,7 +2199,7 @@ class HelpfulVoteTests(TestCaseBase):
args=[r.document.slug])
eq_(200, resp.status_code)
data = json.loads(resp.content)
eq_(0, len(data['series']))
eq_(0, len(data['datums']))
class SelectLocaleTests(TestCaseBase):

Просмотреть файл

@ -910,9 +910,7 @@ def get_helpful_votes_async(request, document_slug):
document = get_object_or_404(
Document, locale=request.LANGUAGE_CODE, slug=document_slug)
yes_data = []
no_data = []
perc_data = []
datums = []
flag_data = []
rev_data = []
revisions = set()
@ -932,16 +930,17 @@ def get_helpful_votes_async(request, document_slug):
results = cursor.fetchall()
for res in results:
created = int(time.mktime(res[3].timetuple()) / 86400) * 86400
percent = float(res[1]) / (float(res[1]) + float(res[2]))
yes_data.append({'x': created, 'y': int(res[1])})
no_data.append({'x': created, 'y': int(res[2])})
perc_data.append({'x': created, 'y': percent})
revisions.add(int(res[0]))
created_list.append(res[3])
datums.append({
'yes': int(res[1]),
'no': int(res[2]),
'date': int(time.mktime(res[3].timetuple()) / 86400) * 86400,
})
if not created_list:
send = {'series': [], 'annotations': []}
send = {'datums': [], 'annotations': []}
return HttpResponse(json.dumps(send), mimetype='application/json')
min_created = min(created_list)
@ -965,26 +964,7 @@ def get_helpful_votes_async(request, document_slug):
# Rickshaw wants data like
# [{'name': 'series1', 'data': [{'x': 1362774285, 'y': 100}, ...]},]
send = {'series': [], 'annotations': []}
if yes_data:
send['series'].append({
'name': _('Yes'),
'slug': 'yes',
'data': yes_data,
})
if no_data:
send['series'].append({
'name': _('No'),
'slug': 'no',
'data': no_data,
})
if perc_data:
send['series'].append({
'name': _('Percent Helpful'),
'slug': 'percent',
'data': perc_data,
})
send = {'datums': datums, 'annotations': []}
if flag_data:
send['annotations'].append({

Просмотреть файл

@ -16,7 +16,7 @@
type: "GET",
url: $('#helpful-graph').data('url'),
success: function (data) {
if (data.series.length > 0) {
if (data.datums.length > 0) {
rickshawGraph(data);
$('#show-graph').hide();
} else {
@ -38,6 +38,27 @@
sets[gettext('Votes')] = ['yes', 'no'];
sets[gettext('Percent')] = ['percent'];
data.seriesSpec = [
{
name: gettext('Yes'),
slug: 'yes',
func: k.Graph.identity('yes'),
color: '#21de2b'
},
{
name: gettext('No'),
slug: 'no',
func: k.Graph.identity('no'),
color: '#de2b21'
},
{
name: gettext('Percent'),
slug: 'percent',
func: k.Graph.percentage('yes', 'no'),
color: '#2b21de'
}
];
$container.show();
var graph = new k.Graph($container, {
data: data,
@ -47,15 +68,7 @@
bucket: true
},
metadata: {
sets: sets,
colors: {
'yes': '#21de2b',
'no': '#de2b21',
'percent': '#2b21de'
},
bucketMethods: {
'percent': 'average'
}
sets: sets
}
});

Просмотреть файл

@ -1,17 +1,5 @@
(function() {
function _makePercent(numerator, denominator) {
return function(d) {
return d[numerator] / d[denominator];
};
}
function _makeIdentity(key) {
return function(d) {
return d[key];
};
}
function init() {
window.App = new KpiDashboard({
el: document.getElementById('kpi-dash-app')
@ -21,40 +9,35 @@ function init() {
{
'name': gettext('Article Votes: % Helpful'),
'slug': 'wiki_percent',
'func': _makePercent('kb_helpful', 'kb_votes')
'func': k.Graph.fraction('kb_helpful', 'kb_votes')
},
{
'name': gettext('Article Votes: % Helpful'),
'name': gettext('Answer Votes: % Helpful'),
'slug': 'ans_percent',
'func': _makePercent('ans_helpful', 'ans_votes')
'func': k.Graph.fraction('ans_helpful', 'ans_votes')
}
], {
bucketMethods: {
wiki_percent: 'average',
ans_percent: 'average'
}
});
]);
makeKPIGraph($('#kpi-active-contributors'), [
{
name: gettext('en-US KB'),
slug: 'en_us',
func: _makeIdentity('en_us')
func: k.Graph.identity('en_us')
},
{
name: gettext('non en-US KB'),
slug: 'non_en_us',
func: _makeIdentity('non_en_us')
func: k.Graph.identity('non_en_us')
},
{
name: gettext('Support Forum'),
slug: 'support_forum',
func: _makeIdentity('support_forum')
func: k.Graph.identity('support_forum')
},
{
name: gettext('Army of Awesome'),
slug: 'aoa',
func: _makeIdentity('aoa')
func: k.Graph.identity('aoa')
}
]);
@ -62,19 +45,15 @@ function init() {
{
name: gettext('CTR %'),
slug: 'ctr',
func: _makePercent('clicks', 'searches')
func: k.Graph.fraction('clicks', 'searches')
}
], {
bucketMethods: {
ctr: 'average',
}
});
]);
makeKPIGraph($('#kpi-visitors'), [
{
name: gettext('Visitors'),
slug: 'visitors',
func: _makeIdentity('visitors')
func: k.Graph.identity('visitors')
}
]);
@ -85,11 +64,7 @@ function init() {
// the api returns 0 to 100, we want 0.0 to 1.0.
func: function(d) { return d['coverage'] / 100; }
}
], {
bucketMethods: {
ctr: 'average'
}
});
]);
}
@ -99,49 +74,24 @@ function parseNum(n) {
return parseInt(n, 10);
}
/* Take an array of datums and make a set of named x/y series, suitable
* for Rickshaw. Each series is generated by one of the key functions.
*
* `keys` is an array of objects that define a name, a slug, and a
* function to calculate data. Each data function will be used as a map
* function on the datum objects to generate a series.
*/
function makeSeries(objects, descriptors) {
var i, j;
var datum, series = [];
var split, date;
for (i = 0; i < descriptors.length; i++) {
var key = descriptors[i];
series[i] = {
name: key.name,
slug: key.slug,
data: _.map(objects, function(datum) {
date = datum.date || datum.start;
split = _.map(date.split('-'), parseNum);
// The Data constructor takes months as 0 through 11. Wtf.
date = +new Date(split[0], split[1] - 1, split[2]) / 1000;
return {x: date, y: key.func(datum)};
})
};
}
// Rickshaw gets angry when its data isn't sorted.
for (i = 0; i < descriptors.length; i++) {
series[i].data.sort(function(a, b) { return a.x - b.x; });
}
return series;
}
function makeKPIGraph($container, descriptors, metadata) {
$.getJSON($container.data('url'), function(data) {
var series = makeSeries(data.objects, descriptors);
var date, series, graph;
var graph = new k.Graph($container, {
$.each(data.objects, function(d) {
date = this.date || this.created || this.start;
// Assume something like 2013-12-31
split = _.map(date.split('-'), parseNum);
// The Data constructor takes months as 0 through 11. Wtf.
this.date = +new Date(split[0], split[1] - 1, split[2]) / 1000;
this.start = undefined;
this.created = undefined;
});
new k.Graph($container, {
data: {
series: series
datums: data.objects,
seriesSpec: descriptors
},
options: {
legend: false,
@ -153,8 +103,7 @@ function makeKPIGraph($container, descriptors, metadata) {
height: 300
},
metadata: metadata
});
graph.render();
}).render();
});
}

Просмотреть файл

@ -1,14 +1,36 @@
(function() {
function init() {
var $topic, datums, seriesSpec, key;
$('input[type=date]').datepicker({
dateFormat: 'yy-mm-dd'
});
$topics = $('#topic-stats');
datums = $topics.data('graph');
seriesSpec = [];
window.datums = datums;
var min = 3;
for (key in datums[0]) {
if (key === 'date' || !datums[0].hasOwnProperty(key)) continue;
// TODO: these names should be localized.
seriesSpec.push({
name: key,
slug: key,
func: k.Graph.identity(key)
});
}
new k.Graph($topics, {
data: {
series: $topics.data('histogram')
datums: datums,
seriesSpec: seriesSpec
},
graph: {
renderer: 'bar'
},
options: {
slider: false

Просмотреть файл

@ -17,7 +17,9 @@ k.Graph = function($elem, extra) {
},
data: {
series: [],
datums: [],
seriesSpec: [],
annotations: [],
bucketed: []
},
@ -52,7 +54,6 @@ k.Graph.prototype.init = function() {
this.initBucketUI();
this.initData();
this.initGraph();
this.initMetadata();
this.initSlider();
this.initAxises();
this.initLegend();
@ -60,80 +61,90 @@ k.Graph.prototype.init = function() {
};
k.Graph.prototype.initData = function() {
var buckets;
var buckets = {};
var bucketed = [];
var i, j, d;
var key;
var line;
// Empty the list.
this.data.bucketed.splice(0, this.data.bucketed.length);
if (this.data.bucketSize) {
for (i = 0; i < this.data.datums.length; i++) {
// make a copy.
d = $.extend({}, this.data.datums[i]);
d.date = Math.floor(d.date / this.data.bucketSize) * this.data.bucketSize;
for (i=0; i < this.data.series.length; i++) {
line = this.data.series[i];
buckets = {};
if (buckets[d.date] === undefined) {
buckets[d.date] = [d];
} else {
buckets[d.date].push(d);
}
}
for (j=0; j < line.data.length; j++) {
// make a copy.
d = $.extend({}, line.data[j]);
d.x = Math.floor(d.x / this.data.bucketSize) * this.data.bucketSize;
bucketed = $.map(buckets, function(dList) {
var sum = 0, i, method, out;
var key;
out = $.extend({}, dList[0]);
if (buckets[d.x] === undefined) {
buckets[d.x] = [d];
} else {
buckets[d.x].push(d);
for (key in out) {
if (key === 'date' || !out.hasOwnProperty(key)) continue;
for (i = 1; i < dList.length; i++) {
out[key] += dList[i][key];
}
}
this.data.bucketed.push({
name: line.name,
slug: line.slug,
disabled: line.disabled,
color: line.color,
data: $.map(buckets, function(dList) {
var sum = 0, i, method, out;
out = $.extend({}, dList[0]);
for (i=0; i < dList.length; i++) {
sum += dList[i].y;
}
method = this.metadata.bucketMethods[line.slug];
if (method === 'average') {
out.y = sum / dList.length;
} else {
out.y = sum;
}
return out;
}.bind(this))
});
}
return out;
});
} else {
this.data.bucketed = this.data.series.slice();
bucketed = this.data.datums.slice();
}
this.data.series = this.makeSeries(bucketed, this.data.seriesSpec);
};
k.Graph.prototype.initMetadata = function() {
var series, key, i;
/* Take an array of datums and make a set of named x/y series, suitable
* for Rickshaw. Each series is generated by one of the key functions.
*
* `descriptors` is an array of objects that define a name, a slug, and
* a function to calculate data. Each data function will be used as a
* map function on the datum objects to generate a series.
*
* Each descriptor may also optionally contain:
* color: The color to draw this series in. The default is to use a
* color generated by rickshaw.
* disabled: If true, this graph will not be drawn. The default is false.
*/
k.Graph.prototype.makeSeries = function(objects, descriptors) {
var i;
var datum, series = [];
var split, date;
var desc;
this.data.lines = {};
series = this.data.series;
for (i=0; i < series.length; i++) {
s = series[i];
this.data.lines[s.slug] = s;
for (i = 0; i < descriptors.length; i++) {
desc = descriptors[i];
series[i] = {
name: desc.name,
slug: desc.slug,
color: desc.color,
disabled: desc.disabled || false,
data: _.map(objects, function(datum) {
return {x: datum.date, y: desc.func(datum)};
})
};
}
for (key in this.metadata.colors) {
if (!this.metadata.colors.hasOwnProperty(key)) continue;
this.data.lines[key].color = this.metadata.colors[key];
// Rickshaw gets angry when its data isn't sorted.
for (i = 0; i < descriptors.length; i++) {
series[i].data.sort(function(a, b) { return a.x - b.x; });
}
return series;
};
k.Graph.prototype.getGraphData = function() {
var palette = new Rickshaw.Color.Palette();
var series = new Rickshaw.Series(this.data.bucketed, palette);
var series = new Rickshaw.Series(this.data.series, palette);
series.active = function() {
// filter by active.
@ -173,6 +184,7 @@ k.Graph.prototype.initBucketUI = function() {
var self = this;
$select.on('change', function() {
self.data.bucketSize = parseInt($(this).val(), 10);
self.initData();
self.update();
});
};
@ -184,9 +196,7 @@ k.Graph.prototype._xFormatter = function(seconds) {
sizes[7 * DAY_S] = gettext('Week beginning %(year)s-%(month)s-%(date)s');
sizes[30 * DAY_S] = gettext('Month beginning %(year)s-%(month)s-%(date)s');
console.log(sizes);
var key = this.data.bucketSize;
console.log('key: ' + key + ' -> ' + sizes[key]);
var format = sizes[key];
if (format === undefined) {
format = '%(year)s-%(month)s-%(date)s';
@ -227,7 +237,7 @@ k.Graph.prototype.initGraph = function() {
graph: this.rickshaw.graph
}, this.hover);
if (this.rickshaw.graph.renderer === 'bar') {
if (this.graph.renderer === 'bar') {
hoverClass = Rickshaw.Graph.BarHoverDetail;
} else {
hoverClass = Rickshaw.Graph.HoverDetail;
@ -362,18 +372,27 @@ k.Graph.prototype.initSets = function() {
var self = this;
$sets.on('change', 'input[name=sets]', function() {
var $this = $(this);
var lineName, set, i;
var line, set, i, key;
var should = {};
for (lineName in self.metadata.sets) {
set = self.metadata.sets[lineName];
for (key in self.metadata.sets) {
if (!self.metadata.sets.hasOwnProperty(key)) continue;
set = self.metadata.sets[key];
for (i=0; i < set.length; i++) {
// Check or uncheck the line based on it's presence in a set and
// the radio's value.
disabled = !!(($this.attr('value') === lineName) ^ $this.prop('checked'));
self.data.lines[set[i]].disabled = disabled;
disabled = !!(($this.attr('value') === key) ^ $this.prop('checked'));
should[set[i]] = disabled;
}
}
for (i = 0; i < self.data.series.length; i++) {
line = self.data.series[i];
line.disabled = should[line.slug];
self.data.seriesSpec[i].disabled = should[line.slug];
}
self.update();
});
@ -397,16 +416,50 @@ k.Graph.prototype.render = function() {
k.Graph.prototype.update = function() {
var newSeries, i;
this.initData();
this.rickshaw.graph.series = this.getGraphData();
this.rickshaw.graph.stackedData = false;
this.rickshaw.graph.update();
this.initMetadata();
};
/* end Graph */
/* These are datum transforming methods. They take an object like
* {created: 1367270055, foo: 10, bar: 20, baz: 30} and return a number.
*/
// Returns the value associated with a key.
// identity('foo') -> 10
k.Graph.identity = function(key) {
return function(d) {
return d[key];
};
};
// Divides the first key by the second.
// fraction('foo', 'bar') -> 0.5
k.Graph.fraction = function(topKey, bottomKey) {
return function(d) {
return d[topKey] / d[bottomKey];
};
};
/* Takes two or more arguments. The arguments are the keys that
* represent an entire collection (all pieces in a pie). The first key
* is the current slice of the pie. Returns what percent the first key
* is of the total, as a decimal between 0.0 and 1.0.
*
* percentage('foo', 'bar', 'baz') -> 10 / (10 + 20 + 30) = ~0.166
*/
k.Graph.percentage = function(partKey /* *restKeys */) {
var allKeys = Array.prototype.slice.call(arguments);
return function(d) {
var sum = 0;
_.each(allKeys, function(key) {
sum += d[key];
});
return d[partKey] / sum;
};
};
Rickshaw.namespace('Rickshaw.Graph.BarHoverDetail');