Fix bug 1067405 - Move event stats within process

This commit is contained in:
Jon Buckley 2014-09-15 12:47:24 -04:00
Родитель 463f42bc03
Коммит 86c21a53e1
5 изменённых файлов: 886 добавлений и 77 удалений

70
app.js
Просмотреть файл

@ -2,12 +2,12 @@ var habitat = require('habitat');
var express = require('express');
var nunjucks = require('nunjucks');
var path = require('path');
var fs = require('fs');
var https = require('https');
var heatmap = require('makerparty-heatmap');
var fork = require( 'child_process' ).fork;
var i18n = require('webmaker-i18n');
var nunjucksEnv = new nunjucks.Environment( new nunjucks.FileSystemLoader(path.join(__dirname, 'views')));
var getEventStats = require('./lib/getEventStats');
habitat.load();
@ -104,37 +104,36 @@ app.get('/live-updates', function(req, res){
res.render('live-updates.html');
});
// Run event stats generation every 1 hour
var stats_cache = require("./stats_cache/makerparty2014-2014-09-16");
var eventStatsBuffer = new Buffer(JSON.stringify(stats_cache));
var heatmapBuffer = new Buffer(heatmap.generateHeatmap(stats_cache.byCountry));
var refreshEventStatsBuffer = function() {
getEventStats({
url: env.get('EVENTS_SERVICE'),
start_date: env.get('EVENTS_START_DATE'),
end_date: env.get('EVENTS_END_DATE')
}, function(error, event_stats) {
if (error) {
return console.log(error);
}
eventStatsBuffer = new Buffer(JSON.stringify(event_stats));
heatmapBuffer = new Buffer(heatmap.generateHeatmap(event_stats.byCountry));
});
};
setInterval(refreshEventStatsBuffer, 3600000);
refreshEventStatsBuffer();
// Maker Party Event Stats
app.get( '/event-stats', function( req, res ) {
// res.json( event_stats_cache );
// send back contents of cache file, else empty object.
var stats = '';
try {
stats = fs.readFileSync( './event-stats.json', 'utf-8' );
stats = JSON.parse( stats );
res.json( stats );
}
catch ( e ) {
stats = {};
res.status( 503 ).json( stats );
}
res.type('application/json; charset=utf-8');
res.send(eventStatsBuffer);
});
app.get('/heatmap.svg', function(req, res) {
var stats = {};
try {
stats = fs.readFileSync( './event-stats.json', 'utf-8' );
stats = JSON.parse( stats );
res.send( heatmap.generateHeatmap( stats.byCountry ) );
}
catch ( e ) {
stats = {};
res.status( 500 ).send( heatmap.generateHeatmap() );
}
res.type('image/svg+xml; charset=utf-8');
res.send(heatmapBuffer);
});
app.get('/heatmap.base.svg', function(req, res) {
@ -177,20 +176,3 @@ app.get('/strings/:lang?', i18n.stringsRoute('en-US'));
app.listen(env.get('PORT'), function () {
console.log('Now listening on %d', env.get('PORT'));
});
// Run event stats generation every 5 minutes
var getEventStats;
function getEventStatsFork() {
if( getEventStats ) {
getEventStats.kill();
return;
}
getEventStats = fork( './bin/getEventStats' );
getEventStats.on( 'exit', function( code, signal ) {
getEventStats = null;
});
}
setInterval( getEventStatsFork, 300000 );
getEventStatsFork();

Просмотреть файл

@ -5,12 +5,11 @@ export PORT=5000
export SUPPORTED_LANGS='[ "*" ]'
# Events service to use
export EVENTS_SERVICE='http://events-api.webmaker.org'
export EVENTS_SERVICE='https://events-api.webmaker.org'
# When to start getting events from (must work in Date.parse)
# leave blank for all events
export EVENTS_START_DATE=''
export EVENTS_END_DATE=''
export EVENTS_START_DATE='2014-06-01T00:00:00.000Z'
export EVENTS_END_DATE='2014-10-31T00:00:00.000Z'
# Flickr API Key
export FLICKR_API_KEY='your_flickr_key'

Просмотреть файл

@ -1,19 +1,77 @@
#!/usr/bin/env node
'use strict';
var habitat = require('habitat');
var async = require('async');
var country_data = require('country-data');
var request = require('request');
var fs = require('fs');
var env = new habitat();
module.exports = function(options, callback) {
options = options || {};
console.log( '[%s] Generating event statistics between %s and %s', Date(), env.get( 'EVENTS_START_DATE' ), env.get( 'EVENTS_END_DATE' ) );
if (!options.url) {
return process.nextTick(function() {
callback(new Error("Required parameter 'url' missing"));
});
}
// request full data from the events api (since specified data)
request.get( env.get( 'EVENTS_SERVICE' ) + '/events?after=' + env.get( 'EVENTS_START_DATE' ), function( error, response, events ) {
if( !error && response.statusCode === 200 ) {
events = JSON.parse( events );
if (!options.start_date) {
return process.nextTick(function() {
callback(new Error("Required parameter 'start_date' missing"));
});
}
if (!options.end_date) {
return process.nextTick(function() {
callback(new Error("Required parameter 'end_date' missing"));
});
}
var events = [];
var event_index = 0;
var MAX_EVENTS = 100;
var total_events = 0;
async.doUntil(
function(done) {
var req_opts = {
gzip: true,
headers: {
'Range': event_index + '-' + (event_index + MAX_EVENTS - 1),
'Range-Unit': 'items'
},
json: true,
method: 'GET',
uri: options.url + '/events?after=' + encodeURIComponent(options.start_date) + '&before=' + encodeURIComponent(options.end_date)
};
request(req_opts, function(events_fetch_error, response, body) {
if (events_fetch_error) {
return done(events_fetch_error);
}
if (response.statusCode !== 200) {
return callback(new Error("GET /events returned HTTP " + response.statusCode));
}
event_index = parseInt(response.headers['content-range'].split('/')[0].split('-')[1], 10) + 1;
total_events = parseInt(response.headers['content-range'].split('/')[1], 10);
events = events.concat(body);
done();
});
},
function() {
return event_index > total_events;
},
function(async_error) {
if (async_error) {
return callback(async_error);
}
calculate_stats(events, callback);
}
);
var calculate_stats = function(events, callback) {
// arrays to dedupe things w/
var event_hosts = [];
var countries = [];
@ -35,12 +93,6 @@ request.get( env.get( 'EVENTS_SERVICE' ) + '/events?after=' + env.get( 'EVENTS_S
// make access to other stats easier to figure
events.forEach( function( event, idx ) {
// we have to check ourselves as the events platform
// does not support checking for events before a date.
if( (new Date( event.beginDate )) > (new Date( env.get( 'EVENTS_END_DATE' ) )) ) {
return;
}
// up the number of events by one each time we have an event in out timeframe
event_stats.events += 1;
@ -180,16 +232,6 @@ request.get( env.get( 'EVENTS_SERVICE' ) + '/events?after=' + env.get( 'EVENTS_S
event_stats.mentors = mentors.length;
event_stats.coorganizers = coorganizers.length;
// save the results in a cache file
fs.writeFileSync( './event-stats.json', JSON.stringify( event_stats ) );
console.log( '[%s] Finished generating event statistics', Date() );
// all done, end task
process.exit();
}
else if( error ) {
console.error( error );
process.exit( 1 );
}
});
callback(null, event_stats);
};
};

Просмотреть файл

@ -10,7 +10,8 @@
"keywords": [],
"author": "author",
"dependencies": {
"bower": "1.3.3",
"async": "^0.9.0",
"bower": "1.3.10",
"country-data": "0.0.13",
"express": "3.4.8",
"grunt": "0.4.2",

Просмотреть файл

@ -0,0 +1,785 @@
{
"hosts": 662,
"estimatedAttendees": 127165,
"events": 2511,
"countries": 86,
"cities": 450,
"mentors": 184,
"coorganizers": 188,
"byCountry": {
"AE": {
"country": "United Arab Emirates",
"events": 1,
"hosts": 0,
"mentors": 0,
"coorganizers": 0,
"estimatedAttendees": 5,
"cities": 0
},
"AL": {
"country": "Albania",
"events": 1,
"hosts": 1,
"mentors": 0,
"coorganizers": 0,
"estimatedAttendees": 35,
"cities": 1
},
"AR": {
"country": "Argentina",
"events": 5,
"hosts": 1,
"mentors": 0,
"coorganizers": 2,
"estimatedAttendees": 1130,
"cities": 1
},
"AU": {
"country": "Australia",
"events": 2,
"hosts": 2,
"mentors": 0,
"coorganizers": 1,
"estimatedAttendees": 185,
"cities": 1
},
"Algérie": {
"country": "Algérie",
"events": 2,
"hosts": 2,
"mentors": 0,
"coorganizers": 2,
"estimatedAttendees": 50,
"cities": 2
},
"BD": {
"country": "Bangladesh",
"events": 47,
"hosts": 30,
"mentors": 6,
"coorganizers": 8,
"estimatedAttendees": 1475,
"cities": 5
},
"BH": {
"country": "Bahrain",
"events": 1,
"hosts": 1,
"mentors": 0,
"coorganizers": 0,
"estimatedAttendees": 35,
"cities": 1
},
"BR": {
"country": "Brazil",
"events": 23,
"hosts": 2,
"mentors": 0,
"coorganizers": 2,
"estimatedAttendees": 260,
"cities": 3
},
"België": {
"country": "België",
"events": 1,
"hosts": 1,
"mentors": 0,
"coorganizers": 0,
"estimatedAttendees": 15,
"cities": 1
},
"Bosnia and Herzegovina": {
"country": "Bosnia and Herzegovina",
"events": 1,
"hosts": 1,
"mentors": 0,
"coorganizers": 0,
"estimatedAttendees": 15,
"cities": 0
},
"Brasil": {
"country": "Brasil",
"events": 3,
"hosts": 1,
"mentors": 0,
"coorganizers": 2,
"estimatedAttendees": 125,
"cities": 1
},
"CA": {
"country": "Canada",
"events": 170,
"hosts": 27,
"mentors": 5,
"coorganizers": 6,
"estimatedAttendees": 4385,
"cities": 17
},
"CH": {
"country": "Switzerland",
"events": 3,
"hosts": 1,
"mentors": 0,
"coorganizers": 0,
"estimatedAttendees": 300,
"cities": 2
},
"CO": {
"country": "Colombia",
"events": 1,
"hosts": 1,
"mentors": 0,
"coorganizers": 1,
"estimatedAttendees": 15,
"cities": 1
},
"Cameroun": {
"country": "Cameroun",
"events": 1,
"hosts": 1,
"mentors": 0,
"coorganizers": 0,
"estimatedAttendees": 5,
"cities": 1
},
"Côte d'Ivoire": {
"country": "Côte d'Ivoire",
"events": 3,
"hosts": 1,
"mentors": 0,
"coorganizers": 0,
"estimatedAttendees": 105,
"cities": 1
},
"Côte dIvoire": {
"country": "Côte dIvoire",
"events": 2,
"hosts": 1,
"mentors": 0,
"coorganizers": 1,
"estimatedAttendees": 225,
"cities": 1
},
"DE": {
"country": "Germany",
"events": 6,
"hosts": 1,
"mentors": 4,
"coorganizers": 0,
"estimatedAttendees": 680,
"cities": 2
},
"EG": {
"country": "Egypt",
"events": 1,
"hosts": 1,
"mentors": 0,
"coorganizers": 0,
"estimatedAttendees": 5,
"cities": 0
},
"ET": {
"country": "Ethiopia",
"events": 1,
"hosts": 1,
"mentors": 0,
"coorganizers": 0,
"estimatedAttendees": 150,
"cities": 1
},
"Espanya": {
"country": "Espanya",
"events": 1,
"hosts": 1,
"mentors": 0,
"coorganizers": 0,
"estimatedAttendees": 35,
"cities": 0
},
"España": {
"country": "España",
"events": 2,
"hosts": 1,
"mentors": 0,
"coorganizers": 0,
"estimatedAttendees": 10,
"cities": 1
},
"FI": {
"country": "Finland",
"events": 4,
"hosts": 2,
"mentors": 0,
"coorganizers": 0,
"estimatedAttendees": 170,
"cities": 1
},
"FR": {
"country": "France",
"events": 5,
"hosts": 3,
"mentors": 0,
"coorganizers": 0,
"estimatedAttendees": 610,
"cities": 5
},
"GB": {
"country": "United Kingdom",
"events": 178,
"hosts": 25,
"mentors": 9,
"coorganizers": 5,
"estimatedAttendees": 5510,
"cities": 94
},
"GH": {
"country": "Ghana",
"events": 2,
"hosts": 1,
"mentors": 1,
"coorganizers": 1,
"estimatedAttendees": 70,
"cities": 1
},
"HR": {
"country": "Croatia",
"events": 1,
"hosts": 1,
"mentors": 0,
"coorganizers": 0,
"estimatedAttendees": 35,
"cities": 1
},
"HU": {
"country": "Hungary",
"events": 3,
"hosts": 0,
"mentors": 0,
"coorganizers": 0,
"estimatedAttendees": 0,
"cities": 3
},
"ID": {
"country": "Indonesia",
"events": 20,
"hosts": 14,
"mentors": 10,
"coorganizers": 6,
"estimatedAttendees": 1860,
"cities": 8
},
"IE": {
"country": "Ireland",
"events": 1,
"hosts": 1,
"mentors": 0,
"coorganizers": 0,
"estimatedAttendees": 75,
"cities": 0
},
"IN": {
"country": "India",
"events": 252,
"hosts": 135,
"mentors": 51,
"coorganizers": 63,
"estimatedAttendees": 23745,
"cities": 88
},
"IS": {
"country": "Iceland",
"events": 1,
"hosts": 0,
"mentors": 0,
"coorganizers": 0,
"estimatedAttendees": 0,
"cities": 1
},
"IT": {
"country": "Italy",
"events": 2,
"hosts": 0,
"mentors": 0,
"coorganizers": 0,
"estimatedAttendees": 150,
"cities": 1
},
"Iran": {
"country": "Iran",
"events": 5,
"hosts": 5,
"mentors": 0,
"coorganizers": 0,
"estimatedAttendees": 85,
"cities": 5
},
"Italia": {
"country": "Italia",
"events": 2,
"hosts": 2,
"mentors": 2,
"coorganizers": 1,
"estimatedAttendees": 110,
"cities": 2
},
"JM": {
"country": "Jamaica",
"events": 1,
"hosts": 1,
"mentors": 0,
"coorganizers": 0,
"estimatedAttendees": 35,
"cities": 1
},
"JO": {
"country": "Jordan",
"events": 5,
"hosts": 3,
"mentors": 0,
"coorganizers": 0,
"estimatedAttendees": 145,
"cities": 4
},
"JP": {
"country": "Japan",
"events": 5,
"hosts": 2,
"mentors": 1,
"coorganizers": 1,
"estimatedAttendees": 135,
"cities": 3
},
"KE": {
"country": "Kenya",
"events": 18,
"hosts": 11,
"mentors": 0,
"coorganizers": 1,
"estimatedAttendees": 980,
"cities": 8
},
"LK": {
"country": "Sri Lanka",
"events": 2,
"hosts": 1,
"mentors": 0,
"coorganizers": 0,
"estimatedAttendees": 75,
"cities": 2
},
"LT": {
"country": "Lithuania",
"events": 1,
"hosts": 1,
"mentors": 0,
"coorganizers": 0,
"estimatedAttendees": 0,
"cities": 1
},
"ML": {
"country": "Mali",
"events": 1,
"hosts": 0,
"mentors": 0,
"coorganizers": 0,
"estimatedAttendees": 0,
"cities": 1
},
"MN": {
"country": "Mongolia",
"events": 3,
"hosts": 1,
"mentors": 0,
"coorganizers": 0,
"estimatedAttendees": 0,
"cities": 1
},
"MU": {
"country": "Mauritius",
"events": 9,
"hosts": 2,
"mentors": 2,
"coorganizers": 4,
"estimatedAttendees": 140,
"cities": 4
},
"MX": {
"country": "Mexico",
"events": 2,
"hosts": 1,
"mentors": 0,
"coorganizers": 0,
"estimatedAttendees": 75,
"cities": 1
},
"MY": {
"country": "Malaysia",
"events": 7,
"hosts": 2,
"mentors": 0,
"coorganizers": 0,
"estimatedAttendees": 230,
"cities": 6
},
"México": {
"country": "México",
"events": 3,
"hosts": 3,
"mentors": 0,
"coorganizers": 0,
"estimatedAttendees": 230,
"cities": 3
},
"NG": {
"country": "Nigeria",
"events": 8,
"hosts": 7,
"mentors": 1,
"coorganizers": 0,
"estimatedAttendees": 525,
"cities": 4
},
"NI": {
"country": "Nicaragua",
"events": 2,
"hosts": 1,
"mentors": 1,
"coorganizers": 0,
"estimatedAttendees": 35,
"cities": 1
},
"NL": {
"country": "Netherlands",
"events": 1,
"hosts": 1,
"mentors": 1,
"coorganizers": 0,
"estimatedAttendees": 35,
"cities": 0
},
"NO": {
"country": "Norway",
"events": 2,
"hosts": 2,
"mentors": 0,
"coorganizers": 0,
"estimatedAttendees": 535,
"cities": 2
},
"NP": {
"country": "Nepal",
"events": 1,
"hosts": 0,
"mentors": 0,
"coorganizers": 0,
"estimatedAttendees": 0,
"cities": 1
},
"NZ": {
"country": "New Zealand",
"events": 1,
"hosts": 1,
"mentors": 0,
"coorganizers": 0,
"estimatedAttendees": 75,
"cities": 0
},
"PE": {
"country": "Peru",
"events": 1,
"hosts": 0,
"mentors": 0,
"coorganizers": 0,
"estimatedAttendees": 0,
"cities": 1
},
"PH": {
"country": "Philippines",
"events": 18,
"hosts": 6,
"mentors": 0,
"coorganizers": 0,
"estimatedAttendees": 1210,
"cities": 11
},
"PK": {
"country": "Pakistan",
"events": 1,
"hosts": 0,
"mentors": 0,
"coorganizers": 0,
"estimatedAttendees": 15,
"cities": 1
},
"PL": {
"country": "Poland",
"events": 9,
"hosts": 0,
"mentors": 0,
"coorganizers": 0,
"estimatedAttendees": 445,
"cities": 4
},
"PT": {
"country": "Portugal",
"events": 1,
"hosts": 1,
"mentors": 0,
"coorganizers": 1,
"estimatedAttendees": 15,
"cities": 1
},
"PY": {
"country": "Paraguay",
"events": 13,
"hosts": 3,
"mentors": 6,
"coorganizers": 0,
"estimatedAttendees": 395,
"cities": 4
},
"RW": {
"country": "Rwanda",
"events": 1,
"hosts": 0,
"mentors": 0,
"coorganizers": 0,
"estimatedAttendees": 0,
"cities": 1
},
"Republic of Kosovo": {
"country": "Republic of Kosovo",
"events": 1,
"hosts": 1,
"mentors": 0,
"coorganizers": 0,
"estimatedAttendees": 5,
"cities": 0
},
"República Federativa do Brasil": {
"country": "República Federativa do Brasil",
"events": 1,
"hosts": 0,
"mentors": 2,
"coorganizers": 0,
"estimatedAttendees": 75,
"cities": 1
},
"SD": {
"country": "Sudan",
"events": 2,
"hosts": 1,
"mentors": 0,
"coorganizers": 0,
"estimatedAttendees": 225,
"cities": 1
},
"SE": {
"country": "Sweden",
"events": 3,
"hosts": 2,
"mentors": 0,
"coorganizers": 0,
"estimatedAttendees": 20,
"cities": 0
},
"SI": {
"country": "Slovenia",
"events": 2,
"hosts": 1,
"mentors": 0,
"coorganizers": 0,
"estimatedAttendees": 50,
"cities": 1
},
"Sverige": {
"country": "Sverige",
"events": 7,
"hosts": 4,
"mentors": 5,
"coorganizers": 4,
"estimatedAttendees": 125,
"cities": 2
},
"Sénégal": {
"country": "Sénégal",
"events": 1,
"hosts": 1,
"mentors": 0,
"coorganizers": 1,
"estimatedAttendees": 35,
"cities": 1
},
"TN": {
"country": "Tunisia",
"events": 1,
"hosts": 1,
"mentors": 0,
"coorganizers": 0,
"estimatedAttendees": 15,
"cities": 1
},
"TR": {
"country": "Turkey",
"events": 1,
"hosts": 1,
"mentors": 0,
"coorganizers": 0,
"estimatedAttendees": 5,
"cities": 0
},
"Taiwan": {
"country": "Taiwan",
"events": 1,
"hosts": 0,
"mentors": 0,
"coorganizers": 0,
"estimatedAttendees": 0,
"cities": 1
},
"Tanzania": {
"country": "Tanzania",
"events": 1,
"hosts": 1,
"mentors": 0,
"coorganizers": 0,
"estimatedAttendees": 5,
"cities": 1
},
"The Netherlands": {
"country": "The Netherlands",
"events": 4,
"hosts": 2,
"mentors": 0,
"coorganizers": 0,
"estimatedAttendees": 170,
"cities": 3
},
"Türkiye": {
"country": "Türkiye",
"events": 1,
"hosts": 1,
"mentors": 0,
"coorganizers": 0,
"estimatedAttendees": 500,
"cities": 1
},
"UA": {
"country": "Ukraine",
"events": 1,
"hosts": 1,
"mentors": 0,
"coorganizers": 0,
"estimatedAttendees": 5,
"cities": 1
},
"UG": {
"country": "Uganda",
"events": 7,
"hosts": 4,
"mentors": 1,
"coorganizers": 1,
"estimatedAttendees": 2010,
"cities": 3
},
"UNKNOWN": {
"country": null,
"events": 1111,
"hosts": 205,
"mentors": 56,
"coorganizers": 60,
"estimatedAttendees": 51520,
"cities": 1
},
"US": {
"country": "United States",
"events": 480,
"hosts": 107,
"mentors": 19,
"coorganizers": 14,
"estimatedAttendees": 24175,
"cities": 101
},
"Venezuela": {
"country": "Venezuela",
"events": 3,
"hosts": 2,
"mentors": 0,
"coorganizers": 0,
"estimatedAttendees": 50,
"cities": 2
},
"Vietnam": {
"country": "Vietnam",
"events": 1,
"hosts": 1,
"mentors": 0,
"coorganizers": 0,
"estimatedAttendees": 5,
"cities": 1
},
"Việt Nam": {
"country": "Việt Nam",
"events": 2,
"hosts": 1,
"mentors": 0,
"coorganizers": 0,
"estimatedAttendees": 10,
"cities": 1
},
"ZA": {
"country": "South Africa",
"events": 3,
"hosts": 2,
"mentors": 0,
"coorganizers": 0,
"estimatedAttendees": 50,
"cities": 2
},
"ZM": {
"country": "Zambia",
"events": 1,
"hosts": 1,
"mentors": 0,
"coorganizers": 0,
"estimatedAttendees": 350,
"cities": 1
},
"ZW": {
"country": "Zimbabwe",
"events": 3,
"hosts": 2,
"mentors": 1,
"coorganizers": 0,
"estimatedAttendees": 535,
"cities": 2
},
"Ελλάδα": {
"country": "Ελλάδα",
"events": 1,
"hosts": 1,
"mentors": 0,
"coorganizers": 0,
"estimatedAttendees": 5,
"cities": 1
},
"বাংলাদেশ": {
"country": "বাংলাদেশ",
"events": 1,
"hosts": 1,
"mentors": 0,
"coorganizers": 0,
"estimatedAttendees": 150,
"cities": 1
},
"日本": {
"country": "日本",
"events": 1,
"hosts": 0,
"mentors": 0,
"coorganizers": 0,
"estimatedAttendees": 75,
"cities": 1
}
}
}