diff --git a/webtools/tinderbox2/src/bin/processmail b/webtools/tinderbox2/src/bin/processmail deleted file mode 100755 index e9e853334bae..000000000000 --- a/webtools/tinderbox2/src/bin/processmail +++ /dev/null @@ -1,1044 +0,0 @@ -#!#perl# #perlflags# -- -# -*- Mode: perl; indent-tabs-mode: nil -*- -# - -# processmail - the mail processing engine for tinderbox. The build -# machines (clients) send their build logs to the server to report on -# the status of the builds and this program accepts the mail (via an -# MTA like sendmail) and acts as the MDA (mail delivery agent). This -# program process the build logs for the tinderbox server (tinder.cgi) -# and enforms the server of any status updates. It also stores the -# logs in compressed form for future reference. - - -# $Revision: 1.10 $ -# $Date: 2000/11/09 19:43:49 $ -# $Author: kestes%staff.mail.com $ -# $Source: /home/hwine/cvs_conversion/cvsroot/mozilla/webtools/tinderbox2/src/bin/Attic/processmail,v $ -# $Name: $ - - -# The contents of this file are subject to the Mozilla Public -# License Version 1.1 (the "License"); you may not use this file -# except in compliance with the License. You may obtain a copy of -# the License at http://www.mozilla.org/NPL/ -# -# Software distributed under the License is distributed on an "AS -# IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or -# implied. See the License for the specific language governing -# rights and limitations under the License. -# -# The Original Code is the Tinderbox build tool. -# -# The Initial Developer of the Original Code is Netscape Communications -# Corporation. Portions created by Netscape are -# Copyright (C) 1998 Netscape Communications Corporation. All -# Rights Reserved. -# - -# complete rewrite by Ken Estes, Mail.com (kestes@staff.mail.com). -# Contributor(s): - - -# Standard perl libraries -use File::Basename; -use Sys::Hostname; -use File::stat; -use Getopt::Long; - -# Tinderbox libraries -use lib '#tinder_libdir#'; - -use TinderConfig; -use TreeData; -use FileStructure; -use HTMLPopUp; -use VCDisplay; -use Error_Parse; -use BuildStatus; -use Utils; -use Persistence; -use TinderDB::Build; -use MailProcess; - - - -sub usage { - - - my $usage =< <% leaks delta> <% bloat delta> - -The leakage information is removed from this line, formated into valid -HTML and passed to the Tinderbox server for displaying on the Status -page. The rendered HTML is found in the - - tinderbox: bloatdata : - -entry of the update file. - - - -Log Files - -The basename of the log files is passed to the tinderbox server via -the tinderbox variables: - - tinderbox: full-log - tinderbox: brief-log - -The files are found in the directories - - FileStructure::get_filename(\$tree, 'full-log') - FileStructure::get_filename(\$tree, 'brief-log') - - - - -Files: - -File names for the output files are contructed from the values returned by: - - FileStructure::get_filename(\$tree, 'brief-log') - FileStructure::get_filename(\$tree, 'build_bin_dir') - FileStructure::get_filename(\$tree, 'full-log') - FileStructure::get_filename(\$tree, 'brief-log') - FileStructure::get_filename(\$tree, 'TinderDB_Dir') - - - -EOF - - print $usage; - exit 0; - -} # usage - - - -sub set_filenames{ - - # These are temporary files, used by the mail processsor and will be - # erased before the program exits. We use filenames which end in - # html for debugging the mail processor in isolation. We do not - # glob on the prefix in the mailprocessor. - - my ($tree) = $TINDERBOX{'tree'}; - - %TMP_FILE = ( - 'errorpick' => (FileStructure::get_filename($tree, 'brief-log'). - "$tree_dir/Tmp.header.$main::UID.html"), - - 'binaryfile' => (FileStructure::get_filename($tree, 'build_bin_dir'). - "/Tmp.$main::UID.html"), - - 'full-log' => (FileStructure::get_filename($tree, 'full-log'). - "/Tmp.$main::UID.html"), - - 'brief-log' => (FileStructure::get_filename($tree, 'brief-log'). - "/Tmp.$main::UID.html"), - - ); - - - %FILE = ( - - # there is also a binaryfile created in assemble_files() - # which gets created using information from get_filename() - # and $TINDERBOX{} - - 'full-log' => (FileStructure::get_filename($tree, 'full-log'). - "/$main::UID.html"), - - 'brief-log' => (FileStructure::get_filename($tree, 'brief-log'). - "/$main::UID.html"), - - 'update_time_stamp' => (FileStructure::get_filename($tree, 'update_time_stamp'). - ""), - ); - - foreach $file ( (values %FILE), (values %TMP_FILE) ) { - main::mkdir_R(dirname($file)); - } - - # set the URL for how to get to the logfiles we will generate. - - # We can not end the cgibin calls in '.html.gz' or IE will think we - # are returning a gziped file. Gunzip knows to add the extension and - # to add the dirname to find the file. - - - $TINDERBOX{'brieflog'} = - ("$FileStructure::URLS{'gunzip'}?". - "tree=$tree&brief-log=$main::UID"); - - $TINDERBOX{'fulllog'} = - ("$FileStructure::URLS{'gunzip'}?". - "tree=$tree&full-log=$main::UID"); - - # record the basename of the log file (not used but useful for debugging.) - - $TINDERBOX{'full-log'} = File::Basename::basename($FILE{'full-log'}); - $TINDERBOX{'brief-log'} = File::Basename::basename($FILE{'brief-log'}); - - return 1; -} - - - -# If the field is a date prefer the output of fix_time_format to what -# is passed into the program. We may recieve human readable strings -# and these need to be converted to time() format. - - -sub fix_tinderboxvar_time_format { - - foreach $key (@DATE_FIELDS) { - ($TINDERBOX{$key}) || next; - $TINDERBOX{$key} = main::fix_time_format($TINDERBOX{$key}); - } - - return 1; -} - -# this function is used to change the tinderbox variables from the old -# format into the new format. It will help ease transitions to the new -# tinderbox code. I believe the new format will be more maintainable -# going forward. - -sub backward_compatibility { - - if ($TINDERBOX{'status'} eq 'busted') { - $TINDERBOX{'status'} = 'build_failed'; - } - - if ($TINDERBOX{'builddate'}) { - $TINDERBOX{'starttime'} = $TINDERBOX{'builddate'}; - $TINDERBOX{'builddate'} = undef; - } - - return 1; -} - - - - -=pod - -"errorparser" is a bad name, we want a "buildtools" which discribes -the compilers and make programs. The buildtools module is shared with -the buildscript and the logparser. We might want to have the -configure output drive the choice of errorparser or tell configure -what choices to make based on the error parser. Notice that if we -take this to extremes we need to have a set of Object Oriented parsers -(cc, cpp, make, perl) each looking for errors it knows how to handle. - - -May want further checks on who can submit what information to -tinderbox. - -need to also check that the current time on the server agrees with -current time on client within 10 minutes. - -need to check that this update either - - 1) has the same starttime as the last update of this tree/build - 2) has a starttime grater then min time between builds. - -=cut - ; - - -# The next set of functions wrap all implementations from package -# Error_Parse. They change their implementations based on the global -# variable $TINDERBOX{'errorparser'}, so this value must be extracted -# from the message before they are called. - - -sub check_required_vars { - - my ($err_string) = ''; - - $TINDERBOX{'starttime'} = main::extract_digits($TINDERBOX{'starttime'}); - if ($build_start != $TINDERBOX{'starttime'} ) { - overwrite_file( $stamp_file, - "$TINDERBOX{'starttime'}\n"); - } - - if( $TINDERBOX{'tree'} eq ''){ - $err_string .= "Variable: 'tinderbox: tree' not set.\n"; - } elsif ( !(TreeData::tree_exists($TINDERBOX{'tree'})) ) { - - $err_string .= "Variable: 'tinderbox: tree' ". - "is not set to a valid tree.\n"; - } - - if(($MAIL_HEADER{'To'} =~ m/external/i || - $MAIL_HEADER{'CC'} =~ m/external/i) && - $TINDERBOX{'tree'} !~ m/external/i) { - $err_string .= "Data from an external source didn't specify ". - "an 'external' tree.\n"; - } - - ($TINDERBOX{'buildname'}) || - ($err_string .= "Variable: 'tinderbox: buildname:' not set.\n"); - - ($TINDERBOX{'administrator'}) || - ($err_string .= "Variable: 'tinderbox: administrator:' not set.\n"); - - ($TINDERBOX{'administrator'} =~ m/\@\w/) || - ($err_string .= "Variable: 'tinderbox: administrator:' ". - "is not a valid email address.\n"); - - ($TINDERBOX{'errorparser'}) || - ($err_string .= "Variable: 'tinderbox: errorparser:' not set.\n"); - - my ($func) = "Error_Parse::$TINDERBOX{'errorparser'}::parse_errorline"; - my ($ignore) = eval &$func(@_); - - ($@) && - ($err_string .= ( - "Errorparser: ". - "'$TINDERBOX{'errorparser'}::parse_errorline'". - " does not exist.\n" - )); - - my ($func) = "Error_Parse::$TINDERBOX{'errorparser'}::line_type"; - my ($ignore) = eval &$func(@_); - - ($@) && - ($err_string .= "Errorparser: '$TINDERBOX{'errorparser'}::line_type'". - " does not exist.\n"); - - # Note at this point we could have sourced only the implementation - # of the error parser that we need. I think programs which use an - # autoload for libaraies are messy since they cause variable run - # time requirements. - - $TINDERBOX{'timenow'} = main::extract_digits($TINDERBOX{'timenow'}); - if ($TINDERBOX{'timenow'} eq '') { - $err_string .= "Variable: 'timenow' not set.\n"; - } else { - - if( abs($TINDERBOX{'timenow'} - $TIME) < ($main::SECONDS_PER_HOUR*10) ){ - $timenow = $TINDERBOX{'timenow'}; - } else { - $err_string .= ( - "Variable: 'tinderbox: timenow:',". - " is not of the form ". - "MM/DD/YY HH:MM:SS, or unix date, ". - "or your clock is set incorrectly, ". - "or the mail was delayed for a long time.\n". - "variable timenow: $TINDERBOX{'timenow'}, ". - "timenow: $TIME, (". - ( ($TINDERBOX{'timenow'}-$TIME) / - $main::SECONDS_PER_MINUTE ). - " minutes)\n". - ""); - } - - } - - $TINDERBOX{'starttime'} = main::extract_digits($TINDERBOX{'starttime'}); - if( $TINDERBOX{'starttime'} eq ''){ - $err_string .= "Variable: 'tinderbox: starttime' not set.\n"; - } else { - if( abs($TINDERBOX{'starttime'} - $TIME) < ($main::SECONDS_PER_HOUR*10) ){ - $starttime = $TINDERBOX{'starttime'}; - } else { - $err_string .= ( - "Variable: 'tinderbox: starttime:',". - " is not of the form ". - "MM/DD/YY HH:MM:SS, or unix date, ". - "or your clock is set incorrectly, ". - "or the mail was delayed for a long time.\n". - "starttime: $TINDERBOX{'starttime'}, timenow: $TIME, (". - ( ($TINDERBOX{'starttime'}-$TIME) / - $main::SECONDS_PER_MINUTE ). - " minutes)\n". - ""); - } - - } - - # Build Status - - ($TINDERBOX{'status'}) || - ($err_string .= "Variable: 'tinderbox: status:' not set.\n"); - - ( BuildStatus::is_status_valid($TINDERBOX{'status'}) ) || - ($err_string .= "Variable: 'tinderbox: status:' must be one of: ". - join( ', ', BuildStatus::get_all_sorted_status()). - "Variable: 'tinderbox: status is set to $TINDERBOX{status}', \n". - "\n"); - - # Report errors by bouncing mail - - ($err_string) && - die($err_string); - -} - - -sub format_bloat_delta { - my ($value,) = @_; - my $units = ' '; - $value = $value || 0; - - if ($value >= 1000000) { - $value = int($value / 1000000); - $min = int($min / 1000000); - $units = 'M'; - } elsif ($value >= 1000) { - $value = int($value / 1000); - $min = int($min / 1000); - $units = 'K'; - } - - return "$value$units"; -} - - - - -# prepare bloat data for display - -sub process_bloat_data { - my ($line, $func_line_type,) = @_; - - # we have to call &highlight_errors() even thought there are no - # errors, or the line numbers will get messed up - - my ($line_type) = &$func_line_type($line); - @html = &highlight_errors($line, $line_type, $lineno, \$next_err, - $func_parse_errorline); - print FULL $html[1]; - if ( $line =~ m/^TOTAL/) { - # Line format: - # TOTAL <% leaks delta> <% bloat delta> - chomp; - my ($leaks, $bloat) = (split /\s+/, $line)[1,3]; - my $bloat_string = ( - "
Lk:". format_bloat_delta($leaks). - "
Bl:". format_bloat_delta($bloat,) - ); - - $TINDERBOX{'bloatdata'} = $bloat_string; - } - return ; -} - - - -# create the HTML file headers for both the full and brief log -# files. This must run after the message body has been parsed as some -# of the data comes from there. - -sub log_header { - my ($logtype) = @_; - - my ($out) = ''; - my ($tree) = $TINDERBOX{'tree'}; - - $out .= HTMLPopUp::page_header('title'=>"$logtype Build Log ". - "for tree: $tree "); - - my $notlogtype = ( $logtype eq 'full' ? "brief" : "full"); - - $out .= "\n"; - $out .= "\n"; - $out .= "
"; - $out .= HTMLPopUp::Link( - "linktxt"=>"Show $notlogtype Log\n", - "href"=> ("$FileStructure::URLS{'gunzip'}?". - "tree=$tree&". - "$notlogtype-log=$main::UID"), - ); - $out .= "
"; - $out .= HTMLPopUp::Link( - "linktxt"=>("Return to the $TINDERBOX{'tree'} ". - "Status Page\n"), - "href"=>(FileStructure::get_filename($tree, - 'tree_URL'). - "/status\.html"), - ); - $out .= "\n"; - - $out .= "\n"; - $out .= "

Build Data

\n"; - - $out .= "
\n";
-  
-  # notice that since we generate this output after processing the
-  # whole log file the tinderbox variables may contain 'derived' data
-  # which was not in the original mail message.
-
-  $out .= MailProcess::format_tinder_vars(%TINDERBOX);
-  $out .= "
\n"; - - $out .= "

Pick an error message to see Log

\n"; - - return $out; - -} - - -# create the HTML links for errors this is for both -# 1) the $headerline, which will appear in the error picklist -# at the top of the log file and points to the error -# in the same HTML file. -# 2) the $logline which is the actual error displayed as a HTML -# link which points to the CVS file and line which is the problem. - - -sub highlight_errors { - my( $line, $line_type, $lineno, - $next_err_ref, $func_parse_errorline) = @_; - - my( @error ) = (); - - # clean up any embedded HTML in the mail - $line = HTMLPopUp::escapeHTML($line); - - my ($logline) = ''; - my ($headerline) = ''; - - # All log lines get line numbers so users can send mail pointing to - # any line in the log file, example: - # Someone should look at the Makefile, this - # make line - # is the same as this one - - # line numbers start AFTER the tinderbox variables have been read. - - $logline .= HTMLPopUp::Link( - "name"=>$lineno, - "linktxt"=>$lineno, - "href"=>"\#$lineno", - ); - - # add spaces so line numbers are left justified to 6 chars - - $logline .= (' ' x ($Error_Parse::LINENO_COLUMN-length($lineno))); - - if ( ( $line_type eq 'error' ) && ( !($last_was_error) ) ) { - - my ($err_ref) = $$next_err_ref; - $$next_err_ref++; - - # tag this line and link to next error line - - $logline .= HTMLPopUp::Link( - "name"=>"err".($err_ref), - "linktxt"=>"NEXT", - "href"=>"\#err".($$next_err_ref), - ); - - $headerline .= HTMLPopUp::Link( - "linktxt"=>$line, - "href"=>"\#err".($err_ref), - ); - - } else { - - # indent, so first column can be the 'next' link - - $logline .= (" " x length("NEXT")); - $headerline .= ''; - - } - - # separate the logs from "NEXT" just a bit more. - - $logline .= " "; - - # markup the line if it looks interesting - - if ($line_type ne 'info') { - @error= &$func_parse_errorline($line); - if ("@error") { - $line = VCDisplay::guess - ( - 'tree' => $TINDERBOX{'tree'}, - 'file' => $error[0], - 'line' => $error[1], - 'linktxt' => $line, - ); - } - - my $color = $Error_Parser::LINE_TYPE2COLOR{$line_type}; - ($color) && - ($color = "color=$color"); - - $logline .= ("". - $line. - ""); - } else { - $logline .= $line; - } - - - return ($headerline, $logline); -} - - -sub parse_mail_body { - - # create several files with a single pass over the mail message: - - # BINARY: the uuencoded file, if present, in the mailmessage. - # FULL: a complete copy of the logfile with HTML markup. - # BRIEF: contains only the error messages with some lines of - # surrounding context and HTML markup. - # ERROR_PICK: a list of html links to error messages, this will apear - # on the top of both brief and full when our - # processing is complete. - - # it is a bug in processing should this function be called before - # the tinderbox variables are set. We intentionally leave off the \n - # on some of thed die's to get tracebacks of internal errors. - - # We must use HTMLPopUp::escapeHTML to clean up any embedded HTML in the - # mail messege to prevent attacks. Though it may be more friendly - # to let compilers and things embed URLS in their error messages: - - # http://www.ciac.org/ciac/bulletins/k-021.shtml - - # When a victim with scripts enabled in their browser reads this - # message, the malicious code may be executed - # unexpectedly. Scripting tags that can be embedded in this way - # include