Bug 350921: [email_in] Create an email interface that can create a bug in Bugzilla

Patch By Max Kanat-Alexander <mkanat@bugzilla.org> r=colin, r=ghendricks, a=myk

webtools/bugzilla/Bugzilla.pm

@@ -175,6 +175,11 @@ sub user {
     return request_cache()->{user};
 }
 
+sub set_user {
+    my ($class, $user) = @_;
+    $class->request_cache->{user} = $user;
+}
+
 sub sudoer {
     my $class = shift;    
     return request_cache()->{sudoer};
@@ -196,6 +201,8 @@ sub sudo_request {
 sub login {
     my ($class, $type) = @_;
 
+    return Bugzilla->user if Bugzilla->usage_mode == USAGE_MODE_EMAIL;
+
     my $authorizer = new Bugzilla::Auth();
     $type = LOGIN_REQUIRED if Bugzilla->cgi->param('GoAheadAndLogIn');
     if (!defined $type || $type == LOGIN_NORMAL) {
@@ -222,7 +229,7 @@ sub login {
         !($sudo_target->in_group('bz_sudo_protect'))
        )
     {
-        request_cache()->{user}   = $sudo_target;
+        Bugzilla->set_user($sudo_target);
         request_cache()->{sudoer} = $authenticated_user;
         # And make sure that both users have the same Auth object,
         # since we never call Auth::login for the sudo target.
@@ -231,10 +238,10 @@ sub login {
         # NOTE: If you want to do any special logging, do it here.
     }
     else {
-        request_cache()->{user} = $authenticated_user;
+        Bugzilla->set_user($authenticated_user);
     }
     
-    return request_cache()->{user};
+    return Bugzilla->user;
 }
 
 sub logout {
@@ -303,6 +310,9 @@ sub usage_mode {
         elsif ($newval == USAGE_MODE_WEBSERVICE) {
             $class->error_mode(ERROR_MODE_DIE_SOAP_FAULT);
         }
+        elsif ($newval == USAGE_MODE_EMAIL) {
+            $class->error_mode(ERROR_MODE_DIE);
+        }
         else {
             ThrowCodeError('usage_mode_invalid',
                            {'invalid_usage_mode', $newval});
@@ -476,6 +486,12 @@ yet been run.  If an sudo session is in progress, the C<Bugzilla::User>
 corresponding to the person who is being impersonated.  If no session is in
 progress, the current C<Bugzilla::User>.
 
+=item C<set_user>
+
+Allows you to directly set what L</user> will return. You can use this
+if you want to bypass L</login> for some reason and directly "log in"
+a specific L<Bugzilla::User>. Be careful with it, though!
+
 =item C<sudoer>
 
 C<undef> if there is no currently logged in user, the currently logged in user

webtools/bugzilla/Bugzilla/Constants.pm

@@ -113,6 +113,7 @@ use File::Basename;
     USAGE_MODE_BROWSER
     USAGE_MODE_CMDLINE
     USAGE_MODE_WEBSERVICE
+    USAGE_MODE_EMAIL
 
     ERROR_MODE_WEBPAGE
     ERROR_MODE_DIE
@@ -317,6 +318,7 @@ use constant BUG_STATE_OPEN => ('NEW', 'REOPENED', 'ASSIGNED',
 use constant USAGE_MODE_BROWSER    => 0;
 use constant USAGE_MODE_CMDLINE    => 1;
 use constant USAGE_MODE_WEBSERVICE => 2;
+use constant USAGE_MODE_EMAIL      => 3;
 
 # Error modes. Default set by Bugzilla->usage_mode (so ERROR_MODE_WEBPAGE
 # usually). Use with Bugzilla->error_mode.

webtools/bugzilla/Bugzilla/Install/Filesystem.pm

@@ -108,6 +108,7 @@ sub FILESYSTEM {
         'testserver.pl'   => { perms => $ws_executable },
         'whine.pl'        => { perms => $ws_executable },
         'customfield.pl'  => { perms => $owner_executable },
+        'email_in.pl'     => { perms => $owner_executable },
 
         'docs/makedocs.pl'   => { perms => $owner_executable },
         'docs/rel_notes.txt' => { perms => $ws_readable },

webtools/bugzilla/Bugzilla/Install/Requirements.pm

@@ -184,6 +184,39 @@ sub OPTIONAL_MODULES {
         version => 0,
         feature => 'More HTML in Product/Group Descriptions'
     },
+
+    # Inbound Email
+    {
+        # Attachment::Stripper requires this, but doesn't pull it in
+        # when you install it from CPAN.
+        package => 'MIME-Types',
+        module  => 'MIME::Types',
+        version => 0,
+        feature => 'Inbound Email',
+    },
+    {
+        # Email::MIME::Attachment::Stripper can throw an error with
+        # earlier versions.
+        # This also pulls in Email::MIME and Email::Address for us.
+        package => 'Email-MIME-Modifier',
+        module  => 'Email::MIME::Modifier',
+        version => '1.43',
+        feature => 'Inbound Email'
+    },
+    {
+        package => 'Email-MIME-Attachment-Stripper',
+        module  => 'Email::MIME::Attachment::Stripper',
+        version => 0,
+        feature => 'Inbound Email'
+    },
+    {
+        package => 'Email-Reply',
+        module  => 'Email::Reply',
+        version => 0,
+        feature => 'Inbound Email'
+    },
+
+    # mod_perl
     {
         package => 'mod_perl',
         module  => 'mod_perl2',

webtools/bugzilla/email_in.pl

@@ -0,0 +1,423 @@
+#!/usr/bin/perl -w
+# -*- Mode: perl; indent-tabs-mode: nil -*-
+#
+# The contents of this file are subject to the Mozilla Public
+# License Version 1.1 (the "License"); you may not use this file
+# except in compliance with the License. You may obtain a copy of
+# the License at http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS
+# IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
+# implied. See the License for the specific language governing
+# rights and limitations under the License.
+#
+# The Original Code is the Bugzilla Inbound Email System.
+#
+# The Initial Developer of the Original Code is Akamai Technologies, Inc.
+# Portions created by Akamai are Copyright (C) 2006 Akamai Technologies, 
+# Inc. All Rights Reserved.
+#
+# Contributor(s): Max Kanat-Alexander <mkanat@bugzilla.org>
+
+use strict;
+use warnings;
+
+# MTAs may call this script from any directory, but it should always
+# run from this one so that it can find its modules.
+BEGIN {
+    require File::Basename;
+    chdir(File::Basename::dirname($0)); 
+}
+
+use Data::Dumper;
+use Email::Address;
+use Email::Reply qw(reply);
+use Email::MIME;
+use Email::MIME::Attachment::Stripper;
+use Getopt::Long qw(:config bundling);
+use Pod::Usage;
+
+use Bugzilla;
+use Bugzilla::Bug qw(ValidateBugID);
+use Bugzilla::Constants qw(USAGE_MODE_EMAIL);
+use Bugzilla::Error;
+use Bugzilla::Mailer;
+use Bugzilla::User;
+use Bugzilla::Util;
+
+#############
+# Constants #
+#############
+
+# This is the USENET standard line for beginning a signature block
+# in a message. RFC-compliant mailers use this.
+use constant SIGNATURE_DELIMITER => '-- ';
+
+# These fields must all be defined or post_bug complains. They don't have
+# to have values--they just have to be defined. There's not yet any
+# way to require custom fields have values, for enter_bug, so we don't
+# have to worry about those yet.
+use constant REQUIRED_ENTRY_FIELDS => qw(
+    reporter
+    short_desc
+    product
+    component
+    version
+
+    assigned_to
+    platform
+    op_sys
+    priority
+    severity
+    bug_file_loc
+);
+
+# Fields that must be defined during process_bug. They *do* have to
+# have values. The script will grab their values from the current
+# bug object, if they're not specified.
+use constant REQUIRED_PROCESS_FIELDS => qw(
+    dependson
+    blocked
+    version
+    product
+    target_milestone
+    rep_platform
+    op_sys
+    priority
+    bug_severity
+    bug_file_loc
+    component
+    short_desc
+);
+
+# $input_email is a global so that it can be used in die_handler.
+our ($input_email, %switch);
+
+####################
+# Main Subroutines #
+####################
+
+sub parse_mail {
+    my ($mail_text) = @_;
+    debug_print('Parsing Email');
+    $input_email = Email::MIME->new($mail_text);
+    
+    my %fields;
+
+    # Email::Address->parse returns an array
+    my ($reporter) = Email::Address->parse($input_email->header('From'));
+    $fields{'reporter'} = $reporter->address;
+    my $summary = $input_email->header('Subject');
+    if ($summary =~ /\[Bug (\d+)\](.*)/i) {
+        $fields{'bug_id'} = $1;
+        $summary = trim($2);
+    }
+
+    my ($body, $attachments) = get_body_and_attachments($input_email);
+    if (@$attachments) {
+        $fields{'attachments'} = $attachments;
+    }
+
+    debug_print("Body:\n" . $body, 3);
+
+    $body = remove_leading_blank_lines($body);
+    my @body_lines = split("\n", $body);
+
+    # If there are fields specified.
+    if ($body =~ /^\s*@/s) {
+        my $current_field;
+        while (my $line = shift @body_lines) {
+            # If the sig is starting, we want to keep this in the 
+            # @body_lines so that we don't keep the sig as part of the 
+            # comment down below.
+            if ($line eq SIGNATURE_DELIMITER) {
+                unshift(@body_lines, $line);
+                last;
+            }
+            # Otherwise, we stop parsing fields on the first blank line.
+            $line = trim($line);
+            last if !$line;
+            
+            if ($line =~ /^@(\S+)\s*=\s*(.*)\s*/) {
+                $current_field = lc($1);
+                $fields{$current_field} = $2;
+            }
+            else {
+                $fields{$current_field} .= " $line";
+            }
+        }
+    }
+
+
+    # The summary line only affects us if we're doing a post_bug.
+    # We have to check it down here because there might have been
+    # a bug_id specified in the body of the email.
+    if (!$fields{'bug_id'} && !$fields{'short_desc'}) {
+        $fields{'short_desc'} = $summary;
+    }
+
+    my $comment = '';
+    # Get the description, except the signature.
+    foreach my $line (@body_lines) {
+        last if $line eq SIGNATURE_DELIMITER;
+        $comment .= "$line\n";
+    }
+    $fields{'comment'} = $comment;
+
+    debug_print("Parsed Fields:\n" . Dumper(\%fields), 2);
+
+    return \%fields;
+}
+
+sub post_bug {
+    my ($fields_in) = @_;
+    my %fields = %$fields_in;
+
+    debug_print('Posting a new bug...');
+
+    $fields{'platform'} ||= Bugzilla->params->{'defaultplatform'};
+    $fields{'op_sys'}   ||= Bugzilla->params->{'defaultopsys'};
+    $fields{'priority'} ||= Bugzilla->params->{'defaultpriority'};
+    $fields{'severity'} ||= Bugzilla->params->{'defaultseverity'};
+
+    foreach my $field (REQUIRED_ENTRY_FIELDS) {
+        $fields{$field} ||= '';
+    }
+
+    my $cgi = Bugzilla->cgi;
+    foreach my $field (keys %fields) {
+        $cgi->param(-name => $field, -value => $fields{$field});
+    }
+
+    $cgi->param(-name => 'inbound_email', -value => 1);
+
+    require 'post_bug.cgi';
+}
+
+######################
+# Helper Subroutines #
+######################
+
+sub debug_print {
+    my ($str, $level) = @_;
+    $level ||= 1;
+    print STDERR "$str\n" if $level <= $switch{'verbose'};
+}
+
+sub get_body_and_attachments {
+    my ($email) = @_;
+
+    my $ct = $email->content_type;
+    debug_print("Splitting Body and Attachments [Type: $ct]...");
+
+    my $body;
+    my $attachments = [];
+    if ($ct =~ /^multipart\/alternative/i) {
+        $body = get_text_alternative($email);
+    }
+    else {
+        my $stripper = new Email::MIME::Attachment::Stripper(
+            $email, force_filename => 1);
+        my $message = $stripper->message;
+        $body = get_text_alternative($message);
+        $attachments = [$stripper->attachments];
+    }
+
+    return ($body, $attachments);
+}
+
+sub get_text_alternative {
+    my ($email) = @_;
+
+    my @parts = $email->parts;
+    my $body;
+    foreach my $part (@parts) {
+        my $ct = $part->content_type;
+        debug_print("Part Content-Type: $ct", 2);
+        if (!$ct || $ct =~ /^text\/plain/i) {
+            $body = $part->body;
+            last;
+        }
+    }
+
+    if (!defined $body) {
+        # Note that this only happens if the email does not contain any
+        # text/plain parts. If the email has an empty text/plain part,
+        # you're fine, and this message does NOT get thrown.
+        ThrowUserError('email_no_text_plain');
+    }
+
+    return $body;
+}
+
+sub remove_leading_blank_lines {
+    my ($text) = @_;
+    $text =~ s/^(\s*\n)+//s;
+    return $text;
+}
+
+sub html_strip {
+    my ($var) = @_;
+    # Trivial HTML tag remover (this is just for error messages, really.)
+    $var =~ s/<[^>]*>//g;
+    # And this basically reverses the Template-Toolkit html filter.
+    $var =~ s/\&amp;/\&/g;
+    $var =~ s/\&lt;/</g;
+    $var =~ s/\&gt;/>/g;
+    $var =~ s/\&quot;/\"/g;
+    $var =~ s/&#64;/@/g;
+    return $var;
+}
+
+
+sub die_handler {
+    my ($msg) = @_;
+
+    # In Template-Toolkit, [% RETURN %] is implemented as a call to "die".
+    # But of course, we really don't want to actually *die* just because
+    # the user-error or code-error template ended. So we don't really die.
+    return if $msg->isa('Template::Exception') && $msg->type eq 'return';
+
+    # We can't depend on the MTA to send an error message, so we have
+    # to generate one properly.
+    if ($input_email) {
+       $msg =~ s/at .+ line.*$//ms;
+       $msg =~ s/^Compilation failed in require.+$//ms;
+       $msg = html_strip($msg);
+       my $reply = reply(to => $input_email, top_post => 1, body => "$msg\n");
+       MessageToMTA($reply->as_string);
+    }
+    print STDERR $msg;
+    # We exit with a successful value, because we don't want the MTA
+    # to *also* send a failure notice.
+    exit;
+}
+
+###############
+# Main Script #
+###############
+
+$SIG{__DIE__} = \&die_handler;
+
+GetOptions(\%switch, 'help|h', 'verbose|v+');
+$switch{'verbose'} ||= 0;
+
+# Print the help message if that switch was selected.
+pod2usage({-verbose => 0, -exitval => 1}) if $switch{'help'};
+
+Bugzilla->usage_mode(USAGE_MODE_EMAIL);
+
+
+my @mail_lines = <STDIN>;
+my $mail_text = join("", @mail_lines);
+my $mail_fields = parse_mail($mail_text);
+
+my $username = $mail_fields->{'reporter'};
+my $user = Bugzilla::User->new({ name => $username })
+    || ThrowUserError('invalid_username', { name => $username });
+
+Bugzilla->set_user($user);
+
+post_bug($mail_fields);
+
+__END__
+
+=head1 NAME
+
+email_in.pl - The Bugzilla Inbound Email Interface
+
+=head1 SYNOPSIS
+
+ ./email_in.pl [-vvv] < email.txt
+
+ Reads an email on STDIN (the standard input).
+
+  Options:
+    --verbose (-v) - Make the script print more to STDERR.
+                     Specify multiple times to print even more.
+
+=head1 DESCRIPTION
+
+This script processes inbound email and creates a bug, or appends data
+to an existing bug.
+
+=head2 Creating a New Bug
+
+The script expects to read an email with the following format:
+
+ From: account@domain.com
+ Subject: Bug Summary
+
+ @product = ProductName
+ @component = ComponentName
+ @version = 1.0
+
+ This is a bug description. It will be entered into the bug exactly as
+ written here.
+
+ It can be multiple paragraphs.
+
+ -- 
+ This is a signature line, and will be removed automatically, It will not
+ be included in the bug description.
+
+The C<@> labels can be any valid field name in Bugzilla that can be
+set on C<enter_bug.cgi>. For the list of field names, see the
+C<fielddefs> table in the database. The above example shows the
+minimum fields you B<must> specify.
+
+The values for the fields can be split across multiple lines, but
+note that a newline will be parsed as a single space, for the value.
+So, for example:
+
+ @short_desc = This is a very long
+ description
+
+Will be parsed as "This is a very long description".
+
+If you specify C<@short_desc>, it will override the summary you specify
+in the Subject header.
+
+C<account@domain.com> must be a valid Bugzilla account.
+
+Note that signatures must start with '-- ', the standard signature
+border.
+
+=head2 Errors
+
+If your request cannot be completed for any reason, Bugzilla will
+send an email back to you. If your request succeeds, Bugzilla will
+not send you anything.
+
+If any part of your request fails, all of it will fail. No partial
+changes will happen. The only exception is attachments--one attachment
+may succeed, and be inserted into the database, and a later attachment
+may fail.
+
+=head1 CAUTION
+
+The script does not do any validation that the user is who they say
+they are. That is, it accepts I<any> 'From' address, as long as it's
+a valid Bugzilla account. So make sure that your MTA validates that
+the message is actually coming from who it says it's coming from,
+and only allow access to the inbound email system from people you trust.
+
+=head1 LIMITATIONS
+
+Note that the email interface has the same limitations as the
+normal Bugzilla interface. So, for example, you cannot reassign
+a bug and change its status at the same time.
+
+The email interface only accepts emails that are correctly formatted
+perl RFC2822. If you send it an incorrectly formatted message, it
+may behave in an unpredictable fashion.
+
+You cannot send an HTML mail along with attachments. If you do, Bugzilla
+will reject your email, saying that it doesn't contain any text. This
+is a bug in L<Email::MIME::Attachment::Stripper> that we can't work
+around.
+
+If you send multiple attachments in one email, they will all be attached,
+but Bugzilla may not send an email notice out for all of them.
+
+You cannot modify Flags through the email interface.

webtools/bugzilla/post_bug.cgi

@@ -29,6 +29,7 @@ use lib qw(.);
 
 use Bugzilla;
 use Bugzilla::Attachment;
+use Bugzilla::BugMail;
 use Bugzilla::Constants;
 use Bugzilla::Util;
 use Bugzilla::Error;
@@ -243,8 +244,13 @@ if ($token) {
              ("createbug:$id", $token));
 }
 
-print $cgi->header();
-$template->process("bug/create/created.html.tmpl", $vars)
-  || ThrowTemplateError($template->error());
-
+if (Bugzilla->usage_mode == USAGE_MODE_EMAIL) {
+    Bugzilla::BugMail::Send($id, $vars->{'mailrecipients'});
+}
+else {
+    print $cgi->header();
+    $template->process("bug/create/created.html.tmpl", $vars)
+        || ThrowTemplateError($template->error());
+}
 
+1;

webtools/bugzilla/template/en/default/global/user-error.html.tmpl

@@ -380,6 +380,10 @@
     [% title = "Email Address Confirmation Failed" %]
     Email address confirmation failed.
 
+  [% ELSIF error == "email_no_text_plain" %]
+    Your message did not contain any text.[% terms.Bugzilla %] does not
+    accept HTML-only email, or HTML email with attachments.
+
   [% ELSIF error == "empty_group_description" %]
     [% title = "The group description can not be empty" %]
     You must enter a description for the group.