pjs/webtools/bugzilla/processmail

#!/usr/bonsaitools/bin/perl -w
# -*- Mode: perl; indent-tabs-mode: nil -*-
#
# The contents of this file are subject to the Mozilla Public
# License Version 1.1 (the "License"); you may not use this file
# except in compliance with the License. You may obtain a copy of
# the License at http://www.mozilla.org/MPL/
#
# Software distributed under the License is distributed on an "AS
# IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
# implied. See the License for the specific language governing
# rights and limitations under the License.
#
# The Original Code is the Bugzilla Bug Tracking System.
#
# The Initial Developer of the Original Code is Netscape Communications
# Corporation. Portions created by Netscape are
# Copyright (C) 1998 Netscape Communications Corporation. All
# Rights Reserved.
#
# Contributor(s): Terry Weissman <terry@mozilla.org>,
#                 Bryce Nesbitt <bryce-mozilla@nextbus.com>
#                 Dan Mosedale <dmose@mozilla.org>
#                 Alan Raetz <al_raetz@yahoo.com>
#

# To recreate the shadow database,  run "processmail regenerate" .

use diagnostics;
use strict;

require "globals.pl";

use RelationSet;

$| = 1;

umask(0);

$::lockcount = 0;
my $regenerate = 0;
my $nametoexclude = "";

my @excludedAddresses = ();

# disable email flag for offline debugging work
my $enableSendMail = 1;

my %force;
@{$force{'QAcontact'}} = ();
@{$force{'Owner'}} = ();
@{$force{'Reporter'}} = ();
@{$force{'CClist'}} = ();

sub Lock {
    if ($::lockcount <= 0) {
        $::lockcount = 0;
        if (!open(LOCKFID, ">>data/maillock")) {
            mkdir "data", 0777;
            chmod 0777, "data";
            open(LOCKFID, ">>data/maillock") || die "Can't open lockfile.";
        }
        my $val = flock(LOCKFID,2);
        if (!$val) { # '2' is magic 'exclusive lock' const.
            print "Lock failed: $val\n";
        }
        chmod 0666, "data/maillock";
    }
    $::lockcount++;
}

sub Unlock {
    $::lockcount--;
    if ($::lockcount <= 0) {
        flock(LOCKFID,8);       # '8' is magic 'unlock' const.
        close LOCKFID;
    }
}

sub FileSize {
    my ($filename) = (@_);
    my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size,
        $atime,$mtime,$ctime,$blksize,$blocks)
        = stat($filename);
    if (defined $size) {
        return $size;
    }
    return -1;
}



sub Different {
    my ($file1, $file2) = (@_);
    my $size1 = FileSize($file1);
    my $size2 = FileSize($file2);
    if ($size1 != $size2) {
        return 1;
    }
    open(FID1, "<$file1") || die "Can't open $file1";
    open(FID2, "<$file2") || die "Can't open $file2";
    my $d1; 
    my $d2; 
    if (read(FID1, $d1, $size1) ne $size1) {
        die "Can't read $size1 bytes from $file1";
    }
    if (read(FID2, $d2, $size2) ne $size2) {
        die "Can't read $size2 bytes from $file2";
    }
    close FID1;
    close FID2;
    return ($d1 ne $d2);
}


sub DescCC {
    my $cclist = shift();

    return "" if ( $cclist->size() == 0 );

    return "Cc: " . $cclist->toString() . "\n";
}


sub DescDependencies {
    my ($id) = (@_);
    if (!Param("usedependencies")) {
        return "";
    }
    my $result = "";
    my $me = "blocked";
    my $target = "dependson";
    my $title = "BugsThisDependsOn";
    for (1..2) {
        SendSQL("select $target from dependencies where $me = $id order by $target");
        my @list;
        while (MoreSQLData()) {
            push(@list, FetchOneColumn());
        }
        if (@list) {
            my @verbose;
            my $count = 0;
            foreach my $i (@list) {
                SendSQL("select bug_status, resolution from bugs where bug_id = $i");
                my ($bug_status, $resolution) = (FetchSQLData());
                my $desc;
                if ($bug_status eq "NEW" || $bug_status eq "ASSIGNED" ||
                    $bug_status eq "REOPENED" || $bug_status eq "UNCONFIRMED") {
                    $desc = "";
                } else {
                    $desc = "[$resolution]";
                }
                push(@verbose, $i . "$desc");
                $count++;
            }
            if ($count > 5) {
                $result .= "$title: Big list (more than 5) has been omitted\n";
            } else {
                $result .= "$title: " . join(', ', @verbose) . "\n";
            }
        }
        my $tmp = $me;
        $me = $target;
        $target = $tmp;
        $title = "OtherBugsDependingOnThis";
    }
    return $result;
}



sub GetBugText {
    my ($id) = (@_);
    undef %::bug;
    
    my @collist = ("bug_id", "product", "version", "rep_platform", "op_sys",
                   "bug_status", "resolution", "priority", "bug_severity",
                   "assigned_to", "reporter", "bug_file_loc",
                   "short_desc", "component", "qa_contact", "target_milestone",
                   "status_whiteboard", "groupset");

    my $query = "select " . join(", ", @collist) .
        " from bugs where bug_id = $id";

    SendSQL($query);

    my @row;
    if (!(@row = FetchSQLData())) {
        return "";
    }
    foreach my $field (@collist) {
        $::bug{$field} = shift @row;
        if (!defined $::bug{$field}) {
            $::bug{$field} = "";
        }
    }

    $::bug{'assigned_to'} = DBID_to_name($::bug{'assigned_to'});
    $::bug{'reporter'} = DBID_to_name($::bug{'reporter'});
    my $qa_contact = "";
    my $target_milestone = "";
    my $status_whiteboard = "";
    if (Param('useqacontact') && $::bug{'qa_contact'} > 0) {
        $::bug{'qa_contact'} = DBID_to_name($::bug{'qa_contact'});
        $qa_contact = "QAContact: $::bug{'qa_contact'}\n";
    } else {
        $::bug{'qa_contact'} = "";
    }
    if (Param('usetargetmilestone') && $::bug{'target_milestone'} ne "") {
        $target_milestone = "TargetMilestone: $::bug{'target_milestone'}\n";
    }
    if (Param('usestatuswhiteboard') && $::bug{'status_whiteboard'} ne "") {
        $status_whiteboard = "StatusWhiteboard: $::bug{'status_whiteboard'}\n";
    }

    $::bug{'long_desc'} = GetLongDescriptionAsText($id);

    my $cclist = new RelationSet();
    $cclist->mergeFromDB("select who from cc where bug_id = $id");
    my @voterlist;
    SendSQL("select profiles.login_name from votes, profiles where votes.bug_id = $id and profiles.userid = votes.who");
    while (MoreSQLData()) {
        my $v = FetchOneColumn();
        push(@voterlist, $v);
    }
    $::bug{'cclist'} = $cclist->toString();
    $::bug{'voterlist'} = join(',', @voterlist);

    if (Param("prettyasciimail")) {
        $^A = "";
        my $temp = formline <<'END',$::bug{'short_desc'},$id,$::bug{'product'},$::bug{'bug_status'},$::bug{'version'},$::bug{'resolution'},$::bug{'rep_platform'},$::bug{'bug_severity'},$::bug{'op_sys'},$::bug{'priority'},$::bug{'component'},$::bug{'assigned_to'},$::bug{'reporter'},$qa_contact,DescCC($cclist),$target_milestone,${status_whiteboard},$::bug{'bug_file_loc'},DescDependencies($id);
+============================================================================+
| @<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< |
+----------------------------------------------------------------------------+
|        Bug #: @<<<<<<<<<<<                Product: @<<<<<<<<<<<<<<<<<<<<<< |
|       Status: @<<<<<<<<<<<<<<<<<<         Version: @<<<<<<<<<<<<<<<<<<<<<< |
|   Resolution: @<<<<<<<<<<<<<<<<<<        Platform: @<<<<<<<<<<<<<<<<<<<<<< |
|     Severity: @<<<<<<<<<<<<<<<<<<      OS/Version: @<<<<<<<<<<<<<<<<<<<<<< |
|     Priority: @<<<<<<<<<<<<<<<<<<       Component: @<<<<<<<<<<<<<<<<<<<<<< |
+----------------------------------------------------------------------------+
|  Assigned To: @<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< |
|  Reported By: @<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< |
|  ~QA Contact: ^<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< |
|  ~   CC list: ^<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< |
+----------------------------------------------------------------------------+
| ~  Milestone: ^<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< |
|~  Whiteboard: ^<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< |
|          URL: @<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< |
|~Dependencies: ^<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< |
+============================================================================+
|                              DESCRIPTION                                   |
END

    my $prettymail = $^A . $::bug{'long_desc'};
        return $prettymail;


    } else {
        return "Bug\#: $id
Product: $::bug{'product'}
Version: $::bug{'version'}
Platform: $::bug{'rep_platform'}
OS/Version: $::bug{'op_sys'}
Status: $::bug{'bug_status'}   
Resolution: $::bug{'resolution'}
Severity: $::bug{'bug_severity'}
Priority: $::bug{'priority'}
Component: $::bug{'component'}
AssignedTo: $::bug{'assigned_to'}                            
ReportedBy: $::bug{'reporter'}               
$qa_contact$target_milestone${status_whiteboard}URL: $::bug{'bug_file_loc'}
" . DescCC($cclist) . "Summary: $::bug{'short_desc'}
" . DescDependencies($id) . "
$::bug{'long_desc'}
";
}

}


my %seen;
my @sentlist;
sub fixaddresses {
    my ($field, $list) = (@_);
    my @result;
    foreach my $i (@$list) {
        if (!defined $i || $i eq "") {
            next;
        }
        SendSQL("select emailnotification, groupset & $::bug{'groupset'} from profiles where login_name = " .
                SqlQuote($i));
        my ($emailnotification, $groupset) = (FetchSQLData());
        if ($groupset ne $::bug{'groupset'}) {
            next;
        }
        if ($emailnotification eq "CConly") {
            if ($field ne "cc") {
                next;
            }
        }
        if ($emailnotification eq "ExcludeSelfChanges" &&
           (lc($i) eq $nametoexclude)) {
            push @excludedAddresses, $nametoexclude;
            next;
        }
        
        if (!defined $::nomail{$i} && !defined $seen{$i}) {
            push(@result, $i . Param('emailsuffix'));
            $seen{$i} = 1;
        }
    }
    return join(", ",  @result);
}


sub Log {
    my ($str) = (@_);
    Lock();
    open(FID, ">>data/maillog") || die "Can't write to data/maillog";
    print FID time2str("%D %H:%M", time()) . ": $str\n";
    close FID;
    Unlock();
}
    

sub FormatTriple {
    my ($a, $b, $c) = (@_);
    $^A = "";
    my $temp = formline << 'END', $a, $b, $c;
^>>>>>>>>>>>>>>>>>>|^<<<<<<<<<<<<<<<<<<<<<<<<<<<|^<<<<<<<<<<<<<<<<<<<<<<<<<<<~~
END
    ; # This semicolon appeases my emacs editor macros. :-)
    return $^A;
}
    
sub FormatDouble {
    my ($a, $b) = (@_);
    $a .= ":";
    $^A = "";
    my $temp = formline << 'END', $a, $b;
^>>>>>>>>>>>>>>>>>> ^<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<~~
END
    ; # This semicolon appeases my emacs editor macros. :-)
    return $^A;
}
    

sub NewProcessOneBug {
    my ($id) = (@_);

    my @headerlist;
    my %values;
    my %defmailhead;
    my %fielddescription;

    my $msg = "";

    SendSQL("SELECT name, description, mailhead FROM fielddefs " .
            "ORDER BY sortkey");
    while (MoreSQLData()) {
        my ($field, $description, $mailhead) = (FetchSQLData());
        push(@headerlist, $field);
        $defmailhead{$field} = $mailhead;
        $fielddescription{$field} = $description;
    }
    SendSQL("SELECT " . join(',', @::log_columns) . ", lastdiffed, now() " .
            "FROM bugs WHERE bug_id = $id");
    my @row = FetchSQLData();
    foreach my $i (@::log_columns) {
        $values{$i} = shift(@row);
    }
    my ($start, $end) = (@row);
    my $ccSet = new RelationSet();
    $ccSet->mergeFromDB("SELECT who FROM cc WHERE bug_id = $id");
    $values{'cc'} = $ccSet->toString();
    
    my @voterlist;
    SendSQL("SELECT profiles.login_name FROM votes, profiles " .
            "WHERE votes.bug_id = $id AND profiles.userid = votes.who");
    while (MoreSQLData()) {
        push(@voterlist, FetchOneColumn());
    }

    $values{'assigned_to'} = DBID_to_name($values{'assigned_to'});
    $values{'reporter'} = DBID_to_name($values{'reporter'});
    if ($values{'qa_contact'}) {
        $values{'qa_contact'} = DBID_to_name($values{'qa_contact'});
    }

    my @diffs;


    SendSQL("SELECT profiles.login_name, fielddefs.description, " .
            "       bug_when, oldvalue, newvalue " .
            "FROM bugs_activity, fielddefs, profiles " .
            "WHERE bug_id = $id " .
            "  AND fielddefs.fieldid = bugs_activity.fieldid " .
            "  AND profiles.userid = who " .
            "  AND bug_when > '$start' " .
            "  AND bug_when <= '$end' " .
            "ORDER BY bug_when"
            );

    while (MoreSQLData()) {
        my @row = FetchSQLData();
        push(@diffs, \@row);
    }

    my $difftext = "";
    my $lastwho = "";
    foreach my $ref (@diffs) {
        my ($who, $what, $when, $old, $new) = (@$ref);
        if ($who ne $lastwho) {
            $lastwho = $who;
            $difftext .= "\n$who" . Param('emailsuffix') . " changed:\n\n";
            $difftext .= FormatTriple("What    ", "Old Value", "New Value");
            $difftext .= ('-' x 76) . "\n";
        }
        $difftext .= FormatTriple($what, $old, $new);
    }

    $difftext = trim($difftext);


    my $deptext = "";

    my $resid = 

    SendSQL("SELECT bugs_activity.bug_id, fielddefs.name, " .
            "       oldvalue, newvalue " .
            "FROM bugs_activity, dependencies, fielddefs ".
            "WHERE bugs_activity.bug_id = dependencies.dependson " .
            "  AND dependencies.blocked = $id " .
            "  AND fielddefs.fieldid = bugs_activity.fieldid" .
            "  AND (fielddefs.name = 'bug_status' " .
            "    OR fielddefs.name = 'resolution') " .
            "  AND bug_when > '$start' " .
            "  AND bug_when <= '$end' " .
            "ORDER BY bug_when, bug_id");
    
    my $thisdiff = "";
    my $lastbug = "";
    my $interestingchange = 0;
    while (MoreSQLData()) {
        my ($bug, $what, $old, $new) = (FetchSQLData());
        if ($bug ne $lastbug) {
            if ($interestingchange) {
                $deptext .= $thisdiff;
            }
            $lastbug = $bug;
            $thisdiff =
                "\nThis bug depends on bug $bug, which changed state:\n\n";
            $thisdiff .= FormatTriple("What    ", "Old Value", "New Value");
            $thisdiff .= ('-' x 76) . "\n";
            $interestingchange = 0;
        }
        $thisdiff .= FormatTriple($fielddescription{$what}, $old, $new);
        if ($what eq 'bug_status' && IsOpenedState($old) ne IsOpenedState($new)) {
            $interestingchange = 1;
        }
    }
    if ($interestingchange) {
        $deptext .= $thisdiff;
    }

    $deptext = trim($deptext);

    if ($deptext) {
        $difftext = trim($difftext . "\n\n" . $deptext);
    }


    my $newcomments = GetLongDescriptionAsText($id, $start, $end);

    if (Param('newemailtech')) {
        
        #
        # Start of email filtering code
        #
        # Even if the user sending the email has not enabled #
        # 'newEmailTech', we still want to filter the email
        # based on other user's email preferences if the global Param 
        # 'newemailtech' is enabled.
        #
        # Note: users who have not enabled newEmailTech will default
        # to no filtering (they will get all email Bugzilla sends).
        
        my $count = 0;
        
        my @currentEmailAttributes = getEmailAttributes($newcomments,
                                                        @diffs);
        my (@assigned_toList,@reporterList,@qa_contactList,@ccList) = 
          ();
        
        #open(LOG, ">>/tmp/maillog");
        #print LOG "\nBug ID: $id   CurrentEmailAttributes:";
        #print LOG join(',', @currentEmailAttributes) . "\n";
        
        @excludedAddresses = (); # zero out global list 

        @assigned_toList = filterEmailGroup('Owner',
                                            \@currentEmailAttributes,
                                            $values{'assigned_to'});
        @reporterList = filterEmailGroup('Reporter', 
                                         \@currentEmailAttributes,
                                         $values{'reporter'});
        if (Param('useqacontact') && $values{'qa_contact'}) {
            @qa_contactList = filterEmailGroup('QAcontact',
                                               \@currentEmailAttributes,
                                               $values{'qa_contact'});
        } else { 
            @qa_contactList = (); 
        }

        @ccList = filterEmailGroup('CClist', \@currentEmailAttributes,
                                   $values{'cc'});

        my @emailList = (@assigned_toList, @reporterList, 
                         @qa_contactList, @ccList);

        # only need one entry per person
        my @allEmail = ();
        my %AlreadySeen = ();
        foreach my $person (@emailList) {
            if ( !($AlreadySeen{$person}) ) {
                push(@allEmail,$person);
                $AlreadySeen{$person}++;
            }
        }

        #print LOG "\nbug $id email sent: " . join(',', @allEmail) . "\n";
        
        @excludedAddresses = filterExcludeList(\@excludedAddresses,
                                               \@allEmail);

        # print LOG "excluded: " . join(',',@excludedAddresses) . "\n\n";

        foreach my $person ( @allEmail ) {
            $count++;
            if ( !defined(NewProcessOnePerson($person, $count, \@headerlist,
                                              \%values, \%defmailhead, 
                                              \%fielddescription, $difftext, 
                                              $newcomments, $start, $id, 1))) {

                # if a value is not returned, this means that the person
                # was not sent mail.  add them to the excludedAddresses list.
                # it will be filtered later for dups.
                #
                push @excludedAddresses, $person;

            }
        }

    } else {
        my $count = 0;
        my @personlist = ($values{'assigned_to'}, $values{'reporter'},
                          split(/,/, $values{'cc'}),
                          @voterlist,
                          $force{'CClist'});
        if ($values{'qa_contact'}) { push @personlist, $values{'qa_contact'} }
        for my $person (@personlist) {
            $count++;

            my $match = "^[^@, ]*@[^@, ]*\.[^@, ]*\$";
            if ($person !~ /$match/) {
                $person = $person . Param('emailsuffix');
            }

            if ( !defined(NewProcessOnePerson($person, $count, \@headerlist, 
                                              \%values, \%defmailhead, 
                                              \%fielddescription, $difftext, 
                                              $newcomments, $start, $id, 1))) {

                # if a value is not returned, this means that the person
                # was not sent mail.  add them to the excludedAddresses list.
                # it will be filtered later for dups.
                #
                push @excludedAddresses, $person;
            }
        }
    }

    SendSQL("UPDATE bugs SET lastdiffed = '$end', delta_ts = delta_ts " .
            "WHERE bug_id = $id");
}

# When one person is in different fields on one bug, they may be
# excluded from email because of one relationship to the bug (eg
# they're the QA contact) but included because of another (eg they
# also reported the bug).  Inclusion takes precedence, so this
# function looks for and removes any users from the exclude list who
# are also on the include list.  Additionally, it removes duplicate
# entries from the exclude list.  
#
# Arguments are the exclude list and the include list; the cleaned up
# exclude list is returned.
#
sub filterExcludeList ($$) {

    if ($#_ != 1) {
        die ("filterExcludeList called with wrong number of args");
    }

    my ($refExcluded, $refAll) = @_;

    my @excludedAddrs = @$refExcluded;
    my @allEmail = @$refAll;
    my @tmpList = @excludedAddrs;
    my (@result,@uniqueResult) = ();
    my %alreadySeen;

    foreach my $excluded (@tmpList) {

        push (@result,$excluded);
        foreach my $included (@allEmail) {

            # match found, so we remove the entry
            if ($included eq $excluded) {
                pop(@result);
            }
        }
    }

    # only need one entry per person
    foreach my $person (@result) {
        if ( !($alreadySeen{$person}) ) {
            push(@uniqueResult,$person);
            $alreadySeen{$person}++;
        }
    }

    return @uniqueResult;
}

# if the Status was changed to Resolved or Verified
#       set the Resolved flag
#
# else if Severity, Status OR Priority fields have any change
#       set the Status flag
#
# else if Keywords has changed
#       set the Keywords flag
#
# else if CC has changed
#       set the CC flag
#
# if the Comments field shows an attachment
#       set the Attachment flag
#
# else if Comments exist
#       set the Comments flag
#
# if no flags are set and there was some other field change
#       set the Status flag
#
sub getEmailAttributes ($@) {
    
    my ($commentField,@fieldDiffs) = @_;
    my (@flags,@uniqueFlags,%alreadySeen) = ();
    my $Status = 0;
    
    foreach my $ref (@fieldDiffs) {
        my ($who, $fieldName, $when, $old, $new) = (@$ref);
        
        #print qq{field: $fieldName $new<br>};
        
        # the STATUS will be flagged for Severity, Status and 
        # Priority changes
        #
        if ( $fieldName eq 'Status') {
            if ($new eq 'RESOLVED' || $new eq 'VERIFIED') {
                push (@flags, 'Resolved');
            }
            $Status = 1;
        }
        elsif ( $fieldName eq 'Severity' || $fieldName eq 'Status' ||
                $fieldName eq 'Priority' ) {
            push (@flags, 'Status');
        } elsif ( $fieldName eq 'Keywords') {
            push (@flags, 'Keywords');
        } elsif ( $fieldName eq 'CC') {
            push (@flags, 'CC');
        }
    }
    
    if ( $commentField =~ /Created an attachment \(/ ) {
        push (@flags, 'Attachments');
    }
    elsif ( $commentField ne '' && !($Status)) {
        push (@flags, 'Comments');
    }
    
    # default fallthrough for any unflagged change is 'Other'
    if ( @flags == 0 && @fieldDiffs != 0 ) {
        push (@flags, 'Other');
    }
    
    # only need one flag per attribute type
    foreach my $flag (@flags) {
        if ( !($alreadySeen{$flag}) ) {
            push(@uniqueFlags,$flag);
            $alreadySeen{$flag}++;
        }
    }
    #print "\nEmail Attributes: ", join(' ,',@uniqueFlags), "<br>\n";
    
    # catch-all default, just in case the above logic is faulty
    if ( @uniqueFlags == 0) {
        push (@uniqueFlags, 'Comments');
    }
    
    return @uniqueFlags;
}

sub filterEmailGroup ($$$) {
    
    my ($emailGroup,$refAttributes,$emailList) = @_;
    my @emailAttributes = @$refAttributes;
    my @emailList = split(/,/,$emailList);
    my @filteredList = ();
    

    # the force list for this email group needs to be checked as well
    #
    push @emailList, @{$force{$emailGroup}};

    foreach my $person (@emailList) {
        
        my $userid;
        my $lastCount = @filteredList;

        if ( $person eq '' ) { next; }

        SendSQL("SELECT userid FROM profiles WHERE login_name = " 
                . SqlQuote($person) );

        if ( !($userid = FetchSQLData()) ) {
            push(@filteredList,$person);
            next;
        }
        
        SendSQL("SELECT emailflags, newemailtech FROM profiles WHERE " .
                "userid = $userid" );
        
        my ($userFlagString, $newemailtech) = FetchSQLData();
        
        # people who are not using newemailtech get skipped; they will 
        # be dealt with later by the old email tech code in 
        # ProcessOneBug().
        #
        if (!defined($newemailtech) || $newemailtech == 0) {
            next;
        }

        # If the sender doesn't want email, exclude them from list
        
        if (lc($person) eq $nametoexclude) {
            
            if ( defined ($userFlagString) && 
                 $userFlagString =~ /ExcludeSelf\~on/ ) {
                
                push (@excludedAddresses,$person);
                next;
            }
        }
        
        # if the users database entry is empty, send them all email
        # by default (they have not accessed userprefs.cgi recently).
        
        if ( !defined($userFlagString) || !($userFlagString =~ /email/) ) {
            push(@filteredList,$person);
        }
        else {

            # the 255 param is here, because without a third param, 
            # split will trim any trailing null fields, which causes perl 
            # to eject lots of warnings.  any suitably large number would 
            # do.
                
            my %userFlags = split(/~/, $userFlagString, 255);

            # The default condition is to send each person email.
            # If we match the email attribute with the user flag, and
            # they do not want email, then remove them from the list.

            push(@filteredList,$person);            

            foreach my $attribute (@emailAttributes) {

                my $matchName = 'email' . $emailGroup . $attribute;
                
                while ((my $flagName, my $flagValue) = each %userFlags) {
                    
                    if ($flagName !~ /$emailGroup/) { 
                        next; 
                    }

                    if ( $flagName eq $matchName  && $flagValue ne 'on') {
                        pop(@filteredList);
                    }

                } # for each userFlag

            } # for each email attribute

            # check to see if the person was removed from this email 
            # group.

            if ( grep ($_ eq $person, @{$force{$emailGroup}} ) ) {

                # if so, see if they want mail about that
                #
                if ( $userFlags{'email' . $emailGroup . 'Removeme'} eq 'on' ) {

                    # we definitely want mail sent to this person, since
                    # inclusion on a mail takes precedence over the previous
                    # exclusion

                    # have they been filtered for some other reason?
                    #
                    if (@filteredList == $lastCount) {
                        
                        # if so, put them back
                        #
                        push (@filteredList, $person);
                    }
                }
            }

        } # if $userFlagString is valid

        # has the person been moved off the filtered list?
        #
        if (@filteredList == $lastCount ) {

            # mark them as excluded
            #
            push (@excludedAddresses,$person);
        } 
            
    } # for each person

    return @filteredList;
}

sub NewProcessOnePerson ($$$$$$$$$$$) {
    my ($person, $count, $hlRef, $valueRef, $dmhRef, $fdRef, $difftext, 
        $newcomments, $start, $id, $checkWatchers) = @_;

    my %values = %$valueRef;
    my @headerlist = @$hlRef;
    my %defmailhead = %$dmhRef;
    my %fielddescription = %$fdRef;

    if ($seen{$person}) {
      return;
    }
        
    SendSQL("SELECT userid, emailnotification, newemailtech," .
            "       groupset & $values{'groupset'} " .
            "FROM profiles WHERE login_name = " . SqlQuote($person));
    my ($userid, $emailnotification, $newemailtech, 
        $groupset) =  (FetchSQLData());
    
    # check for watchers, and recurse if we find any, but tell the
    # recursive call not to check for watchers 
    #
    if (Param("supportwatchers") && $checkWatchers)  {
      my $personId = DBname_to_id($person);
      my $watcherSet = new RelationSet();
      $watcherSet->mergeFromDB("SELECT watcher FROM watch WHERE" .
                               " watched = $personId");

      foreach my $watcher ( $watcherSet->toArray() ) {
        
        &NewProcessOnePerson(DBID_to_name($watcher) . Param('emailsuffix'), 
                            $count, \@headerlist, \%values,
                            \%defmailhead, \%fielddescription, $difftext, 
                            $newcomments, $start, $id, 0);
      }

    }

    if (!$newemailtech || !Param('newemailtech')) {
      return;
    }
    $seen{$person} = 1;


    # if this person doesn't have permission to see info on this bug, 
    # return.
    #
    # XXX - I _think_ this currently means that if a bug is suddenly given
    # more restrictive permissions, people without those permissions won't
    # see the action of restricting the bug itself; the bug will just 
    # quietly disappear from their radar.
    #
    if ($groupset ne $values{'groupset'}) {
      return;
    }
    if ($emailnotification eq "ExcludeSelfChanges" &&
        lc($person) eq $nametoexclude) {
      push @excludedAddresses, $nametoexclude;
      return;
    }
    # "$count < 3" means "this person is either assigned_to or reporter"
    #
    if ($emailnotification eq "CCOnly" && $count < 3) {
      return;
    }
    
    my %mailhead = %defmailhead;
    
    my $head = "";
    
    foreach my $f (@headerlist) {
      if ($mailhead{$f}) {
        my $value = $values{$f};
        if (!defined $value) {
          # Probaby ought to whine or something. ###
          next;
        }
        my $desc = $fielddescription{$f};
        $head .= FormatDouble($desc, $value);
      }
    }
    
    if ($difftext eq "" && $newcomments eq "") {
      # Whoops, no differences!
      return;
    }
    
    my $isnew = ($start !~ m/[1-9]/);
    
    my %substs;

    $person .= Param('emailsuffix');
# 09/13/2000 cyeh@bluemartini.com
# If a bug is changed, don't put the word "Changed" in the subject mail
# since if the bug didn't change, you wouldn't be getting mail
# in the first place! see http://bugzilla.mozilla.org/show_bug.cgi?id=29820 
# for details.
    $substs{"neworchanged"} = $isnew ? "New" : "";
    $substs{"to"} = $person;
    $substs{"cc"} = '';
    $substs{"bugid"} = $id;
    if ($isnew) {
      $substs{"diffs"} = $head . "\n\n" . $newcomments;
    } else {
      $substs{"diffs"} = $difftext . "\n\n" . $newcomments;
    }
    $substs{"summary"} = $values{'short_desc'};
    
    my $template = Param("newchangedmail");
    
    my $msg = PerformSubsts($template, \%substs);

    my $sendmailparam = "-ODeliveryMode=deferred";
    if (Param("sendmailnow")) {
       $sendmailparam = "";
    }

    if ($enableSendMail == 1) {
    open(SENDMAIL, "|/usr/lib/sendmail $sendmailparam -t") ||
      die "Can't open sendmail";
    
    print SENDMAIL trim($msg) . "\n";
    close SENDMAIL;
    }
    push(@sentlist, $person);
    return 1;
}


sub ProcessOneBug {
  my $i = $_[0];
  NewProcessOneBug($i);

  # Make sure that everyone who was excluded because of the advanced
  # filtering options (and thus are using new email tech) has the
  # corresponding element in %seen set.  This is so that they won't
  # also be processed by the old email tech code, which follows.
  #
  # It's necessary because people who are excluded by the advanced
  # filtering code never make it into NewProcessOnePerson(), which is
  # where %seen would have otherwise been touched for them.
  # 
  foreach my $person (@excludedAddresses) {
      $seen{$person} = 1;
  }

  my $old = "shadow/$i";
  my $new = "shadow/$i.tmp.$$";
  my $diffs = "shadow/$i.diffs.$$";
  my $verb = "Changed";
  if (!stat($old)) {
    mkdir "shadow", 0777;
    chmod 0777, "shadow";
        open(OLD, ">$old") || die "Couldn't create null $old";
        close OLD;
        $verb = "New";
    }
    my $text = GetBugText($i);
    if ($text eq "") {
        die "Couldn't find bug $i.";
    }
    open(FID, ">$new") || die "Couldn't create $new";
    print FID $text;
    close FID;
    if (Different($old, $new)) {
        system("diff -c -b $old $new > $diffs");
        my $tolist = fixaddresses("to",
                                  [$::bug{'assigned_to'}, $::bug{'reporter'},
                                   $::bug{'qa_contact'}]);
        my @combinedcc;
        foreach my $v (split(/,/, "$::bug{'cclist'},$::bug{'voterlist'}")) {
            push @combinedcc, $v;
        }
        push (@combinedcc, (@{$force{'CClist'}}, @{$force{'QAcontact'}}, 
                            @{$force{'Reporter'}}, @{$force{'Owner'}}));
        my $cclist = fixaddresses("cc", \@combinedcc);
        my $logstr = "Bug $i $verb";
        if ($tolist ne "" || $cclist ne "") {
            my %substs;

            $substs{"fullbugreport"} = $text; # added ability to include the full bug report
            $substs{"to"} = $tolist;
            $substs{"cc"} = $cclist;
            $substs{"bugid"} = $i;
            $substs{"diffs"} = "";
            open(DIFFS, "<$diffs") || die "Can't open $diffs";
            while (<DIFFS>) {
                $substs{"diffs"} .= $_;
            }
            close DIFFS;
            $substs{"neworchanged"} = $verb;
            $substs{"summary"} = $::bug{'short_desc'};
            my $msg = PerformSubsts(Param("changedmail"), \%substs);

            if (!$regenerate) {
                # Note: fixaddresses may result in a Cc: only.  This seems
                # harmless.
                my $sendmailparam = "-ODeliveryMode=deferred";
                if (Param("sendmailnow")) {
                    $sendmailparam = "";
                }
                if ($enableSendMail==1) {
                open(SENDMAIL, 
                     "|/usr/lib/sendmail $sendmailparam -t") ||
                       die "Can't open sendmail";

                print SENDMAIL $msg;
                close SENDMAIL;
                }
                foreach my $n (split(/[, ]+/, "$tolist,$cclist")) {
                    if ($n ne "") {
                        push(@sentlist, $n);
                    }
                }

                $logstr = "$logstr; mail sent to $tolist, $cclist";
            }
        }
        unlink($diffs);
        Log($logstr);
    }

    # on the off chance that there are duplicate addresses in the exclude list,
    # we filter it for dups one more time.  They could have gotten there in 
    # fixaddresses(), NewProcessOnePerson(), or NewProcessOneBug.
    # 
    @excludedAddresses = filterExcludeList(\@excludedAddresses, 
                                           \@sentlist);

    if (!$regenerate) {
    if (@sentlist) {
            print "<B>Email sent to:</B> " . join(", ", @sentlist) . "<br>\n";
        } else {
            print "<B>Email sent to:</B> no one<br>\n";
        }

        if ( @excludedAddresses ) {
            print "<br><B>Excluding: </B>" . join(", ", @excludedAddresses) . 
              "\n";
        }

        print "<br><br><center>New: <a href=\"userprefs.cgi\?bank=diffs\">" .
          "change your email preferences<\/a> if you wish to tweak the " .
            "kinds of Bugzilla email you get.<\/center>\n";
    }
    rename($new, $old) || die "Can't rename $new to $old";
    chmod 0666, $old;
    if ($regenerate) {
        print "$i ";
    }
    %seen = ();
    @sentlist = ();
}

# Code starts here

ConnectToDatabase();
GetVersionTable();
Lock();

if (open(FID, "<data/nomail")) {
    while (<FID>) {
        $::nomail{trim($_)} = 1;
    }
    close FID;
}

# To recreate the shadow database,  run "processmail regenerate" .
if ($#ARGV >= 0 && $ARGV[0] eq "regenerate") {
    $regenerate = 1;
    shift @ARGV;
    SendSQL("select bug_id from bugs order by bug_id");
    my @regenerate_list;
    while (my @row = FetchSQLData()) {
        push @regenerate_list, $row[0];
    }
    foreach my $i (@regenerate_list) {
        ProcessOneBug($i);
        Unlock();
        Lock();
    }
    print("\n");
    exit;
}

if ($#ARGV >= 0 && $ARGV[0] eq "-forcecc") {
    shift(@ARGV);
    foreach my $i (split(/,/, shift(@ARGV))) {
        push(@{$force{'CClist'}}, trim($i));
    }
}

if ($#ARGV >= 0 && $ARGV[0] eq "-forceowner") {
    shift(@ARGV);
    @{$force{'Owner'}} = (trim(shift(@ARGV)));
}

if ($#ARGV >= 0 && $ARGV[0] eq "-forceqacontact") {
    shift(@ARGV);
    @{$force{'QAcontact'}} = (trim(shift(@ARGV)));
}

if (($#ARGV < 0) || ($#ARGV > 1)) {
    print "Usage:\n processmail {bugid} {nametoexclude} " . 
      "[-forcecc list,of,users]\n             [-forceowner name] " .
      "[-forceqacontact name]\nor\n processmail regenerate\nor\n" .
      " processmail rescanall\n";
    exit;
}

if ($#ARGV == 1) {
    $nametoexclude = lc($ARGV[1]);
}

if ($ARGV[0] eq "rescanall") {
    print "<br> Collecting bug ids...\n";
    SendSQL("select bug_id from bugs where to_days(now()) - to_days(delta_ts) <= 2 order by bug_id");
    my @list;
    while (my @row = FetchSQLData()) {
        push @list, $row[0];
    }
    foreach my $id (@list) {
        $ARGV[0] = $id;
        print "<br> Doing bug $id\n";
        ProcessOneBug($ARGV[0]);
    }
} else {
    ProcessOneBug($ARGV[0]);
}

exit;