mozilla
/
pjs
зеркало из https://github.com/mozilla/pjs.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность
pjs/tools/download-stats/generate-stats.pl

336 строки
10 KiB
Perl
Executable File
Исходник Ответственный История

#!/usr/bin/perl
use DBI;
use LWP::Simple;
use Template;
use strict;
my $VERBOSE = 1;
use Getopt::Long;
my $VERBOSE = 0;
GetOptions("verbose" => \$VERBOSE);
# Establish a database connection.
my $dsn = "DBI:mysql:host=mecha.mozilla.org;database=downloadstats;port=3306";
my $dbh = DBI->connect($dsn,
"logprocessord",
"1ssw?w?",
{ RaiseError => 1,
PrintError => 0,
ShowErrorStatement => 1 }
);
################################################################################
# Stats Configuration
# All these variables can be redefined in the stats definition file.
# These are just the default values.
# List of stats to generate.
our @stats;
# Date range to which we limit the query.
our ($start_date, $end_date);
# Whether or not to try to add up partial downloads from the same client
# to see if they count as a complete download. Doesn't work well without
# DNS lookups, which we aren't doing for performance reasons.
our $do_segment_count = 0;
# Do a platform-breakout report by default; this can get overridden by the defs file.
our $report_type = "platform_build";
# Parse the definition file and make sure it defines some stats.
my $stats_defs = $ARGV[0];
defined($stats_defs) or die "You didn't reference a stats definition file.\n";
do $stats_defs || die "Couldn't parse stats definition file: $!\n";
defined(@stats) or die "The stats definition file didn't define any stats.\n";
################################################################################
# Data Validation
if ($start_date) {
$start_date =~ /^\d\d\d\d-\d\d-\d\d( \d\d:\d\d(:\d\d)?)?$/
or die "Invalid start date $start_date (must be in format yyyy-mm-dd (hh:mm(:ss)?)?).";
}
if ($end_date) {
$end_date =~ /^\d\d\d\d-\d\d-\d\d( \d\d:\d\d(:\d\d)?)?$/
or die "Invalid end date $end_date (must be in format yyyy-mm-dd (hh:mm(:ss)?)?).";
}
################################################################################
# Queries
my @date_criteria = ("1=1");
if ($start_date) { push(@date_criteria, "date_time >= '$start_date'") }
if ($end_date) { push(@date_criteria, "date_time <= '$end_date'") }
my $date_clause = join(" AND ", @date_criteria);
# Completed downloads.
my $done =
$dbh->prepare("SELECT COUNT(*) FROM entries JOIN files ON entries.file_id = files.id " .
"WHERE $date_clause AND files.path = ? AND files.name = ? AND bytes = ?");
# Not completed downloads.
my $not_done =
$dbh->prepare("SELECT COUNT(*) FROM entries JOIN files ON entries.file_id = files.id " .
"WHERE $date_clause AND files.path = ? AND files.name = ? AND bytes != ? " .
"AND status = 200");
# Partial content requests; may or may not be completed.
my $may_be_done =
$dbh->prepare("SELECT COUNT(*) FROM entries JOIN files ON entries.file_id = files.id " .
"WHERE $date_clause AND files.path = ? AND files.name = ? AND status = 206");
# A way to get the count of people who altogether completed a download.
# Only run if $do_segment_count is true. Note that this query is expensive
# and only ever returns a fraction of the total, so it's not that useful.
# Also, it probably doesn't work unless we reverse DNS every address
# in the logs, which we aren't doing at the moment for performance.
my $done_in_segments =
$dbh->prepare("SELECT 1 FROM entries JOIN files ON entries.file_id = files.id " .
"WHERE $date_clause AND files.path = ? AND files.name = ? " .
"GROUP BY client HAVING SUM(bytes) = ? AND COUNT(bytes) > 1");
# Grab a list of files in a given directory, recursing infinitely.
# The first condition matches the directory itself, while the second
# matches subdirectories and should be passed the value '<dir>/%'.
my $files_in_directory =
$dbh->prepare("SELECT path, name FROM files WHERE files.path = ? OR files.path LIKE ?");
################################################################################
# Stats Retrieval
sub retrieve_by_file {
foreach my $stat (@stats) {
next if !$stat->{isactive};
print STDERR "$stat->{name} $stat->{version}...\n" if $VERBOSE;
# Expand directories in the file list.
my @files;
foreach my $file (@{$stat->{files}}) {
if ($file->{name}) {
print STDERR "$file->{name}\n" if $VERBOSE;
push(@files, $file);
}
elsif (!$file->{path}) {
print STDERR "file has no name or path\n" if $VERBOSE;
next;
}
else {
print STDERR "expanding $file->{path}\n" if $VERBOSE;
$files_in_directory->execute($file->{path}, "$file->{path}/%");
while (my $file = $files_in_directory->fetchrow_hashref()) {
print STDERR "expanding to include $file->{path}/$file->{name}\n" if $VERBOSE;
push(@files, $file);
}
}
}
$stat->{files} = \@files;
foreach my $file (@files) {
(undef, $file->{size}) = head("http://ftp.mozilla.org$file->{path}/$file->{name}");
next if !$file->{size};
$done->execute($file->{path}, $file->{name}, $file->{size});
$file->{counts} = {};
($file->{counts}->{done}) = $done->fetchrow_array();
}
}
}
sub retrieve_by_platform_build {
foreach my $stat (@stats) {
next if !$stat->{isactive};
print STDERR "$stat->{name} $stat->{version}...\n" if $VERBOSE;
my $platforms = $stat->{platforms};
foreach my $platform (keys %$platforms) {
print STDERR " $platform\n" if $VERBOSE;
my $files = $platforms->{$platform};
foreach my $type (keys %$files) {
print STDERR " $type: " if $VERBOSE;
my $file = $files->{$type};
my (undef, $file_size) = head("http://ftp.mozilla.org$stat->{path}/$file->{name}");
$file_size ||= $file->{size}
or die "Can't figure out the size of $stat->{path}/$file->{name}.";
$done->execute($stat->{path}, $file->{name}, $file_size);
my ($done_count) = $done->fetchrow_array();
my $done_in_segments_count = "N/A";
my $total_done = $done_count;
if ($do_segment_count) {
$done_in_segments->execute($stat->{path}, $file->{name}, $file_size);
$done_in_segments_count = $done_in_segments->fetchall_arrayref();
$done_in_segments_count = scalar(@$done_in_segments_count);
$total_done += $done_in_segments_count;
}
# commenting these out for speed -bryner
#$not_done->execute($stat->{path}, $file->{name}, $file_size);
#my ($not_done_count) = $not_done->fetchrow_array();
my ($not_done_count) = 0;
#$may_be_done->execute($stat->{path}, $file->{name});
#my ($may_be_done_count) = $may_be_done->fetchrow_array();
my ($may_be_done_count) = 0;
$file->{counts} = {
complete_uni => $done_count,
#complete_multi => $done_in_segments_count,
#incomplete => $not_done_count,
#partial => $may_be_done_count,
};
print STDERR "$done_count / $not_done_count / $may_be_done_count / $done_in_segments_count / $total_done\n" if $VERBOSE;
}
}
}
}
################################################################################
# Output
my $platform_build_template = <<'EOF';
<html>
<head>
<title></title>
<style type="text/css">
th { text-align: left; }
th, td { border: solid 1px black; }
table { border-collapse: collapse;
border: solid 1px black; }
</style>
</head>
<body>
[% FOREACH stat = stats %]
[% NEXT IF !stat.isactive %]
[% app_total = 0 %]
<h2>[% stat.name %] [%+ stat.version %] Download Stats</h2>
<p>[% start_date || "the beginning of time" %] to [% end_date || "the end of time" %]</p>
<table summary="[% stat.name %] [%+ stat.version %] Downloads">
<tr>
<th>Build</th>
<th>Downloads</th>
</tr>
[% FOREACH platform = stat.platforms %]
[% platform_total = 0 %]
<tr>
<td colspan="2"><h3>[% platform.key %]</h3></td>
</tr>
[% files = platform.value %]
[% FOREACH file = files %]
[% file_total = file.value.counts.complete_uni + file.value.counts.complete_multi %]
<tr>
<td>[% file.key %]:[% file.value.name %]</td>
<td>[% file_total %]</td>
</tr>
[% platform_total = platform_total + file_total %]
[% END %]
<tr>
<td>total for [% platform.key %]</td>
<td>[% platform_total %]</td>
</tr>
[% app_total = app_total + platform_total %]
[% END %]
<tr>
<td>grand total</td>
<td>[% app_total %]</td>
</tr>
</table>
[% END %]
</body>
</html>
EOF
my $file_template = <<'EOF';
<html>
<head>
<title></title>
<style type="text/css">
th { text-align: left; }
th, td { border: solid 1px black; }
table { border-collapse: collapse;
border: solid 1px black; }
</style>
</head>
<body>
[% FOREACH stat = stats %]
[% NEXT IF !stat.isactive %]
<h2>[% stat.name %] [%+ stat.version %] Download Stats</h2>
<p>[% start_date || "the beginning of time" %] to [% end_date || "the end of time" %]</p>
<table summary="[% stat.name %] [%+ stat.version %] Downloads">
<tr>
<th>File</th>
<th>Downloads</th>
</tr>
[% FOREACH file = stat.files %]
<tr>
<td>[% file.path %]/[% file.name %]</td>
<td>[% file.counts.done %]</td>
</tr>
[% END %]
</table>
[% END %]
</body>
</html>
EOF
my $total_template = <<'EOF';
[% grand_total = 0 %]
[% FOREACH stat = stats %]
[% NEXT IF !stat.isactive %]
[% app_total = 0 %]
[% FOREACH platform = stat.platforms %]
[% platform_total = 0 %]
[% files = platform.value %]
[% FOREACH file = files %]
[% file_total = file.value.counts.complete_uni + file.value.counts.complete_multi %]
[% platform_total = platform_total + file_total %]
[% END %]
[% app_total = app_total + platform_total %]
[% END %]
[% grand_total = grand_total + app_total %]
[% END %]
[% grand_total %]
EOF
my $template;
if ($report_type eq "file") {
retrieve_by_file();
$template = $file_template;
}
elsif ($report_type eq "platform_build") {
retrieve_by_platform_build();
$template = $platform_build_template;
}
elsif ($report_type eq "total_downloads") {
retrieve_by_platform_build();
$template = $total_template;
}
my $tt = new Template({ PRE_CHOMP => 1, POST_CHOMP => 1});
$tt->process(\$template, {stats => \@stats,
start_date => $start_date,
end_date => $end_date})
|| die "Template process failed: ", $template->error(), "\n";
Ссылка в новой задаче Показать git blame Копировать постоянную ссылку