move to new RunShellCommand, split into substeps, various fixes and features. r=preed, b=368579

tools/release/Bootstrap/Step.pm

@@ -4,8 +4,12 @@
 
 package Bootstrap::Step;
 use IO::Handle;
-use MozBuild::Util qw(RunShellCommand);
+use File::Spec::Functions;
+use Bootstrap::Config;
+use MozBuild::Util qw(RunShellCommand Email);
 use POSIX qw(strftime);
+use base 'Exporter';
+our @EXPORT = qw(catfile);
 
 my $DEFAULT_TIMEOUT = 3600;
 
@@ -21,33 +25,39 @@ sub Shell {
     my $this = shift;
     my %args = @_;
     my $cmd = $args{'cmd'};
+    my $cmdArgs = defined($args{'cmdArgs'}) ? $args{'cmdArgs'} : [];
     my $dir = $args{'dir'};
     my $timeout = $args{'timeout'} ? $args{'timeout'} : $DEFAULT_TIMEOUT;
     my $logFile = $args{'logFile'};
     my $rv = '';
 
+    if (ref($cmdArgs) ne 'ARRAY') {
+        die "ASSERT: Bootstrap::Step(): cmdArgs is not an array ref\n" 
+    }
+
     if ($dir) {
         $this->Log('msg' => 'Changing directory to ' . $dir);
         chdir($dir) or die "Cannot chdir to $dir: $!";
-        $this->Log('msg' => 'Running shell command ' . $cmd . ' in dir ' . $dir);
-    } else {
-        $this->Log('msg' => 'Running shell command ' . $cmd);
     }
 
+    $this->Log('msg' => 'Running shell command:');
+    $this->Log('msg' => '  arg0: ' . $cmd); 
+    my $argNum = 1;
+    foreach my $arg (@{$cmdArgs}) {
+        $this->Log('msg' => '  arg' . $argNum . ': ' . $arg); 
+        $argNum += 1;
+    }
     $this->Log('msg' => 'Starting time is ' . $this->CurrentTime());
+    $this->Log('msg' => 'Logging output to ' . $logFile);
 
-    print "Timeout: $timeout\n";
+    $this->Log('msg' => 'Timeout: ' . $timeout);
 
     if ($timeout) {
         $rv = RunShellCommand(
-           'command' => "$cmd",
+           'command' => $cmd,
-           'timeout' => "$timeout",
+           'args' => $cmdArgs,
-           'logfile' => "$logFile",
+           'timeout' => $timeout,
-        );
+           'logfile' => $logFile,
-    } else {
-        $rv = RunShellCommand(
-           'command' => "$cmd",
-           'logfile' => "$logFile",
         );
     }
 
@@ -55,14 +65,12 @@ sub Shell {
     my $timedOut  = $rv->{'timedOut'};
     my $signalName  = $rv->{'signalName'};
     my $dumpedCore = $rv->{'dumpedCore'};
-    my $pid = $rv->{'pid'};
-    print "Pid: $pid\n";
     if ($timedOut) {
         $this->Log('msg' => "output: $rv->{'output'}") if $rv->{'output'};
         die("FAIL shell call timed out after $timeout seconds");
     }
     if ($signalName) {
-        print ("WARNING shell recieved signal $signalName");
+        $this->Log('msg' => 'WARNING shell recieved signal' . $signalName);
     }
     if ($dumpedCore) {
         $this->Log('msg' => "output: $rv->{'output'}") if $rv->{'output'};
@@ -135,4 +143,42 @@ sub CurrentTime() {
     return strftime("%T %D", localtime());
 }
 
+# Overridden by child if needed
+sub Push() {
+    my $this = shift;
+}
+
+# Overridden by child if needed
+sub Announce() {
+    my $this = shift;
+}
+
+sub SendAnnouncement() {
+    my $this = shift;
+    my %args = @_;
+    
+    my $config = new Bootstrap::Config();
+
+    my $from = $args{'from'} ? $args{'from'} : $config->Get(var => 'from');
+    my $to = $args{'to'} ? $args{'to'} : $config->Get(var => 'to');
+    my $cc = $args{'cc'} ? $args{'cc'} : $config->Get(var => 'cc');
+    my $subject = $args{'subject'};
+    my $message = $args{'message'};
+
+    my @ccList = split(', ', $cc);
+
+    eval {
+        Email(
+          from => $from,
+          to => $to,
+          cc => \@ccList,
+          subject => $subject,
+          message => $message,
+        );
+    };
+    if ($@) {
+        die("Could not send announcement email: $@");
+    }
+}
+    
 1;

tools/release/Bootstrap/Step/Build.pm

@@ -18,15 +18,18 @@ sub Execute {
     my $logDir = $config->Get('var' => 'logDir');
     my $rcTag = $productTag . '_RC' . $rc;
 
-    my $lastBuilt = $buildDir . '/' . $buildPlatform . '/last-built';
+    my $lastBuilt = catfile($buildDir, $buildPlatform, 'last-built');
     unlink($lastBuilt) 
       or $this->Log('msg' => "Cannot unlink last-built file $lastBuilt: $!");
     $this->Log('msg' => "Unlinked $lastBuilt");
 
-    my $buildLog = $logDir . '/' . $rcTag . '-build.log';
+    my $buildLog = catfile($logDir, 'build_' . $rcTag . '-build.log');
  
     $this->Shell(
-      'cmd' => './build-seamonkey.pl --once --mozconfig mozconfig --depend --config-cvsup-dir ' . $buildDir . '/tinderbox-configs',
+      'cmd' => './build-seamonkey.pl',
+      'cmdArgs' => ['--once', '--mozconfig', 'mozconfig', '--depend', 
+                    '--config-cvsup-dir', 
+                    catfile($buildDir, 'tinderbox-configs')],
       'dir' => $buildDir,
       'logFile' => $buildLog,
       'timeout' => 36000
@@ -42,7 +45,7 @@ sub Verify {
     my $rcTag = $productTag.'_RC'.$rc;
     my $logDir = $config->Get('var' => 'logDir');
 
-    my $buildLog = $logDir . '/' . $rcTag . '-build.log';
+    my $buildLog = catfile($logDir, 'build_' . $rcTag . '-build.log');
 
     $this->CheckLog(
         'log' => $buildLog,
@@ -55,4 +58,22 @@ sub Verify {
 #    );
 }
 
+sub Announce {
+    my $this = shift;
+
+    my $product = $config->Get('var' => 'product');
+    my $productTag = $config->Get('var' => 'productTag');
+    my $version = $config->Get('var' => 'version');
+    my $rc = $config->Get('var' => 'rc');
+    my $logDir = $config->Get('var' => 'logDir');
+
+    my $rcTag = $productTag . '_RC' . $rc;
+    my $buildLog = catfile($logDir, 'build_' . $rcTag . '-build.log');
+
+    $this->SendAnnouncement(
+      subject => "$product $version build step finished",
+      message => "$product $version en-US build is ready to be copied to the candidates dir.",
+    );
+}
+
 1;

tools/release/Bootstrap/Step/Repack.pm

@@ -5,6 +5,7 @@
 package Bootstrap::Step::Repack;
 use Bootstrap::Step;
 use Bootstrap::Config;
+use MozBuild::Util qw(MkdirWithPath);
 @ISA = ("Bootstrap::Step");
 
 my $config = new Bootstrap::Config;
@@ -19,14 +20,17 @@ sub Execute {
     my $buildPlatform = $config->Get('var' => 'buildPlatform');
     my $rcTag = $productTag . '_RC' . $rc;
 
-    my $buildLog = $logDir . '/' . $rcTag . '-build-l10n.log';
+    my $buildLog = catfile($logDir, 'repack_' . $rcTag . '-build-l10n.log');
-    my $lastBuilt = $buildDir . '/' . $buildPlatform . '/last-built';
+    my $lastBuilt = catfile($buildDir, $buildPlatform, 'last-built');
     unlink($lastBuilt) 
       or $this->Log('msg' => "Cannot unlink last-built file $lastBuilt: $!");
     $this->Log('msg' => "Unlinked $lastBuilt");
 
     $this->Shell(
-      'cmd' => './build-seamonkey.pl --once --mozconfig mozconfig --depend --config-cvsup-dir ' . $buildDir . '/tinderbox-configs',
+      'cmd' => './build-seamonkey.pl',
+      'cmdArgs' => ['--once', '--mozconfig', 'mozconfig', '--depend', 
+                    '--config-cvsup-dir', 
+                    catfile($buildDir, 'tinderbox-configs')],
       'dir' => $buildDir,
       'logFile' => $buildLog,
       'timeout' => 36000
@@ -38,17 +42,136 @@ sub Verify {
 
     my $buildDir = $config->Get('var' => 'buildDir');
     my $productTag = $config->Get('var' => 'productTag');
+    my $product = $config->Get('var' => 'product');
     my $rc = $config->Get('var' => 'rc');
-    my $rcTag = $productTag.'_RC'.$rc;
+    my $oldRc = $config->Get('var' => 'oldRc');
     my $logDir = $config->Get('var' => 'logDir');
+    my $version = $config->Get('var' => 'version');
+    my $oldVersion = $config->Get('var' => 'oldVersion');
+    my $mozillaCvsroot = $config->Get('var' => 'mozillaCvsroot');
+    my $verifyDir = $config->Get('var' => 'verifyDir');
 
-    my $buildLog = $logDir . '/' . $rcTag . '-build.log';
+    my $rcTag = $productTag.'_RC'.$rc;
 
 # XXX temp disabled
 #    $this->CheckLog(
 #        'log' => $buildLog,
 #        'notAllowed' => 'failed',
 #    );
+
+    # l10n metadiff test
+
+    my $verifyDirVersion = catfile($verifyDir, $product . '-' . $version);
+
+    MkdirWithPath('dir' => $verifyDirVersion)
+      or die "Cannot mkdir $verifyDirVersion: $!";
+
+    # check out l10n verification scripts
+    foreach my $dir ('common', 'l10n') {
+        $this->Shell(
+          'cmd' => 'cvs',
+          'cmdArgs' => ['-d', $mozillaCvsroot, 
+                        'co', '-d', $dir, 
+                        catfile('mozilla', 'testing', 'release', $dir)],
+          'dir' => $verifyDirVersion,
+          'logFile' => catfile($logDir, 
+                               'repack_checkout-l10n_verification.log'),
+        );
+    }
+
+    # Download current release
+    $this->Shell(
+      'cmd' => 'echo',
+      'cmdArgs' => ['-av', 
+                    '-e', 'ssh', 
+                    '--include="*.dmg"',
+                    '--include="*.exe"',
+                    '--include="*.tar.gz"',
+                    '--exclude=\'*\'',
+                    'stage.mozilla.org:/home/ftp/pub/' . $product
+                     . '/nightly/' . $version . '-candidates/rc' . $rc . '/',
+                    $product . '-' . $version . '-rc' . $rc . '/',
+                   ],
+      'dir' => $buildDir,
+      'logFile' => 
+        catfile($logDir, 'repack_verify-download_' . $version . '.log'),
+      'timeout' => 3600
+    );
+
+    # Download previous release
+    $this->Shell(
+      'cmd' => 'echo',
+      'cmdArgs' => ['-av', 
+                    '-e', 'ssh', 
+                    '--include="*.dmg"',
+                    '--include="*.exe"',
+                    '--include="*.tar.gz"',
+                    '--exclude=\'*\'',
+                    'stage.mozilla.org:/home/ftp/pub/' . $product
+                     . '/nightly/' . $oldVersion . '-candidates/rc' 
+                     . $oldRc . '/',
+                    $product . '-' . $oldVersion . '-rc' . $oldRc . '/',
+                   ],
+      'dir' => $buildDir,
+      'logFile' => 
+        catfile($logDir, 'repack_verify-download_' . $oldVersion . '.log'),
+      'timeout' => 3600
+    );
+
+    my $newProduct = $product . '-' . $version . '-' . 'rc' . $rc;
+    my $oldProduct = $product . '-' . $oldVersion . '-' . 'rc' . $rc;
+
+    foreach my $product ($newProduct, $oldProduct) {
+        MkdirWithPath('dir' => catfile($verifyDirVersion, 'l10n', $product))
+          or die "Cannot mkdir $verifyDirVersion/$product: $!";
+
+        $this->Shell(
+          'cmd' => './verify_l10n.sh',
+          'cmdArgs' => [$product],
+          'dir' => catfile($verifyDirVersion, 'l10n'),
+          'logFile' => catfile($logDir, 
+                               'repack_' . $product . '-l10n_verification.log'),
+        );
+
+
+        foreach my $rule ('^Only', '^Binary') {
+            $this->CheckLog(
+                'log' => $logDir . 
+                         '/repack_' . $product . '-l10n_verification.log',
+                'notAllowed' => $rule,
+            );
+        }
+    }
+
+    # generate metadiff
+    $this->Shell(
+      'cmd' => 'diff',
+      'cmdArgs' => ['-r', 
+                    catfile($logDir, 
+                            'repack_' . $newProduct . '-l10n_verification.log'),
+                    catfile($logDir, 
+                            'repack_' . $oldProduct . '-l10n_verification.log')],
+      'dir' => catfile($verifyDirVersion, 'l10n'),
+      'logFile' => catfile($logDir, 'repack_metadiff-l10n_verification.log'),
+    );
+}
+
+sub Announce {
+    my $this = shift;
+
+    my $product = $config->Get('var' => 'product');
+    my $version = $config->Get('var' => 'version');
+    my $productTag = $config->Get('var' => 'productTag');
+    my $rc = $config->Get('var' => 'rc');
+    my $logDir = $config->Get('var' => 'logDir');
+
+    my $rcTag = $productTag . '_RC' . $rc;
+    my $buildLog = catfile($logDir, 'repack_' . $rcTag . '-build-l10n.log');
+
+    $this->SendAnnouncement(
+      subject => "$product $version l10n repack step finished",
+      message => "$product $version l10n builds are ready to be copied to the candidates directory.",
+    );
 }
 
 1;

tools/release/Bootstrap/Step/Sign.pm

@@ -13,8 +13,9 @@ sub Execute {
     my $logDir = $config->Get('var' => 'logDir');
 
     $this->Shell(
-      'cmd' => 'echo sign',
+      'cmd' => 'echo',
-      'logFile' => $logDir . '/sign.log',
+      'cmdArgs' => ['sign'],
+      'logFile' => catfile($logDir, 'sign.log'),
     );
 }
 
@@ -23,8 +24,9 @@ sub Verify {
     my $logDir = $config->Get('var' => 'logDir');
 
     $this->Shell(
-      'cmd' => 'echo Verify sign',
+      'cmd' => 'echo',
-      'logFile' => $logDir . '/verify-sign.log',
+      'cmdArgs' => ['Verify sign'],
+      'logFile' => catfile($logDir, 'sign_verify.log'),
     );
 }
 

tools/release/Bootstrap/Step/Source.pm

@@ -22,32 +22,44 @@ sub Execute {
     my $stageHome = $config->Get('var' => 'stageHome');
 
     # create staging area
-    my $stageDir = 
+    my $stageDir = catfile($stageHome, $product . '-' . $version, 
-      $stageHome . '/' . $product . '-' . $version . '/batch-source/rc' . $rc;
+                           'batch-source', 'rc' . $rc);
 
     if (not -d $stageDir) {
         MkdirWithPath('dir' => $stageDir) 
           or die "Cannot create $stageDir: $!";
     }
 
+    my $srcScript = $product . '-src-tarball-nobuild';
     $this->Shell(
-      'cmd' => $stageHome . '/bin/' . $product . '-src-tarball-nobuild -r ' . $productTag . '_RELEASE -m ' . $version,
+      'cmd' => catfile($stageHome, 'bin', $srcScript),
+      'cmdArgs' => ['-r', $productTag . '_RELEASE', '-m', $version],
       'dir' => $stageDir,
-      'logFile' => $logDir . '/source.log',
+      'logFile' => catfile($logDir, 'source.log'),
     );
               
     move("$stageDir/../*.bz2", $stageDir);
     chmod(0644, glob("$stageDir/*.bz2"));
-
-#    $this->Shell(
-#      'cmd' => 'rsync -av *.bz2 /home/ftp/pub/' . $product . '/nightly/' . $version . '-candidates/rc' . $rc,
-#      'dir' => $stageDir,
-#    );
 }
 
 sub Verify {
     my $this = shift;
-    #$this->Shell('cmd' => 'echo Verify source');
+    # TODO verify source archive
+}
+
+sub Announce {
+    my $this = shift;
+
+    my $product = $config->Get('var' => 'product');
+    my $version = $config->Get('var' => 'version');
+    my $logDir = $config->Get('var' => 'logDir');
+
+    my $logFile = catfile($logDir, 'source.log');
+
+    $this->SendAnnouncement(
+      subject => "$product $version source step finished",
+      message => "$product $version source archive is ready to be copied to the candidates dir.",
+    );
 }
 
 1;

tools/release/Bootstrap/Step/Stage.pm

@@ -26,8 +26,8 @@ sub Execute {
     ## Prepare the staging directory for the release.
     # Create the staging directory.
 
-    my $stageDir = $stageHome . '/' . $product . '-' . $version;
+    my $stageDir = catfile($stageHome, $product . '-' . $version);
-    my $mergeDir = $stageDir . '/stage-merged';
+    my $mergeDir = catfile($stageDir, 'stage-merged');
 
     if (not -d $stageDir) {
         MkdirWithPath('dir' => $stageDir) 
@@ -36,7 +36,7 @@ sub Execute {
     }
  
     # Create skeleton batch directory.
-    my $skelDir = "$stageDir/batch-skel/stage";
+    my $skelDir = catfile($stageDir, 'batch-skel', 'stage');
     if (not -d "$skelDir") {
         MkdirWithPath('dir' => $skelDir) 
           or die "Cannot create $skelDir: $!";
@@ -49,94 +49,88 @@ sub Execute {
     # Create the contrib and contrib-localized directories with expected 
     # access rights.
     for my $dir ('contrib', 'contrib-localized') {
-        if (not -d "$skelDir/$dir") {
+        my $fullDir = catfile($skelDir, $dir);
-            MkdirWithPath('dir' => "$skelDir/$dir") 
+        if (not -d $fullDir) {
-              or die "Could not mkdir $skelDir/$dir : $!";
+            MkdirWithPath('dir' => $fullDir) 
-            $this->Log('msg' => "Created directory $skelDir/$dir");
+              or die "Could not mkdir $fullDir : $!";
+            $this->Log('msg' => "Created directory $fullDir");
         }
 
-        chmod(oct(2775), "$skelDir/$dir")
+        chmod(oct(2775), $fullDir)
-          or die "Cannot change mode on $skelDir/$dir to 2775: $!";
+          or die "Cannot change mode on $fullDir to 2775: $!";
-        $this->Log('msg' => "Changed mode of $dir to 2775");
+        $this->Log('msg' => "Changed mode of $fullDir to 2775");
-        chown(-1, $gid, "$skelDir/$dir")
+        chown(-1, $gid, $fullDir)
-          or die "Cannot chgrp $skelDir/$dir to $product: $!";
+          or die "Cannot chgrp $fullDir to $product: $!";
-        $this->Log('msg' => "Changed group of $skelDir/$dir to $product");
+        $this->Log('msg' => "Changed group of $fullDir to $product");
     }
 
-    my (undef, undef, $gid) = getgrnam($product)
-      or die "Could not getgrname for $product: $!";
-
-    my $dir = "$skelDir/contrib";
-    chmod(oct(2775), $dir)
-      or die "Cannot change mode on $skelDir/contrib to 2775: $!";
-    $this->Log('msg' => "Changed mode of $dir to 2775");
-    chown(-1, $gid, $dir)
-      or die "Cannot chgrp $skelDir/contrib to $product: $!";
-    $this->Log('msg' => "Changed group of $dir to $product");
- 
     # NOTE - should have a standard "master" copy somewhere else
     # Copy the KEY file from the previous release directory.
-    my $keyFile = '/home/ftp/pub/' . $product . '/releases/1.5/KEY';
+    my $keyFile = catfile('/home', 'ftp', 'pub', $product, 'releases', '1.5',
-    copy($keyFile, $skelDir . '/') 
+                          'KEY');
-      or die("Could not copy $keyFile to $skelDir: $!");
+    copy($keyFile, $skelDir) or die("Could not copy $keyFile to $skelDir: $!");
 
     ## Prepare the merging directory.
     $this->Shell(
-      'cmd' => 'rsync -av batch-skel/stage/ stage-merged/',
+      'cmd' => 'rsync',
-      'logFile' => $logDir . '/stage-merge_skel.log',
+      'cmdArgs' => ['-av', 'batch-skel/stage/', 'stage-merged/'],
+      'logFile' => catfile($logDir, 'stage_merge_skel.log'),
       'dir' => $stageDir,
     );
     
     # Collect the release files onto stage.mozilla.org.
 
-    if (not -d "$stageDir/batch1/prestage") {
+    my $prestageDir = catfile($stageDir, 'batch1', 'prestage');
-        MkdirWithPath('dir' => "$stageDir/batch1/prestage") 
+    if (not -d $prestageDir) {
-          or die "Cannot create $stageDir/batch1/prestage: $!";
+        MkdirWithPath('dir' => $prestageDir) 
-        $this->Log('msg' => "Created directory $stageDir/batch1/prestage");
+          or die "Cannot create $prestageDir: $!";
+        $this->Log('msg' => "Created directory $prestageDir");
     }
 
     $this->Shell(
-      'cmd' => 'rsync -Lav /home/ftp/pub/' . $product . '/nightly/' . $version . '-candidates/rc' . $rc . '/ ./',
+      'cmd' => 'rsync',
-      'logFile' => $logDir . '/stage-collect.log',
+      'cmdArgs' => ['-Lav', catfile('/home', 'ftp', 'pub', $product, 'nightly',
-      'dir' => $stageDir . '/batch1/prestage',
+                                    $version . '-candidates', 'rc' . $rc . '/'),
+                    './'],
+      'logFile' => catfile($logDir, 'stage_collect.log'),
+      'dir' => catfile($stageDir, 'batch1', 'prestage'),
     );
 
     # Remove unreleased builds
     $this->Shell(
-      'cmd' => 'rsync -av prestage/ prestage-trimmed/',
+      'cmd' => 'rsync',
-      'logFile' => $logDir . '/stage-collect_trimmed.log',
+      'cmdArgs' => ['-av', 'prestage/', 'prestage-trimmed/'],
-      'dir' => $stageDir . '/batch1/',
+      'logFile' => catfile($logDir, 'stage_collect_trimmed.log'),
+      'dir' => catfile($stageDir, 'batch1'),
     );
 
     # Remove unshipped files and set proper mode on dirs
     find(sub { return $this->TrimCallback(); },
-     $stageDir . '/batch1/prestage-trimmed/');
+     $stageDir . catfile('batch1', 'prestage-trimmed'));
     
     $this->Shell(
-      'cmd' => 'rsync -Lav prestage-trimmed/ stage/',
+      'cmd' => 'rsync',
-      'logFile' => $logDir . '/stage-collect_stage.log',
+      'cmdArgs' => ['-Lav', 'prestage-trimmed/', 'stage/'],
-      'dir' => $stageDir . '/batch1',
+      'logFile' => catfile($logDir, 'stage_collect_stage.log'),
+      'dir' => catfile($stageDir, 'batch1'),
     );
 
     # Nightly builds using a different naming scheme than production.
     # Rename the files.
     # TODO should support --long filenames, for e.g. Alpha and Beta
     $this->Shell(
-      'cmd' => $stageHome . '/bin/groom-files --short=' . $version . ' .',
+      'cmd' => catfile($stageHome, 'bin', 'groom-files'),
-      'logFile' => $logDir . '/stage-groom_files.log',
+      'cmdArgs' => ['--short=' . $version, '.'],
-      'dir' => $stageDir . '/batch1/stage',
+      'logFile' => catfile($logDir, 'stage_groom_files.log'),
+      'dir' => catfile($stageDir, 'batch1', 'stage'),
     );
 
     # fix xpi dir names
-    move("$stageDir/batch1/stage/linux-xpi", 
+    my $fromFile = catfile($stageDir, 'batch1', 'stage', 'windows');
-           "$stageDir/batch1/stage/linux-i686/xpi")
+    my $toFile = catfile($stageDir, 'batch1', 'stage', 'win32');
-      or die('msg' => "Cannot rename $stageDir/batch1/stage/linux-xpi $stageDir/batch1/stage/linux-i686/xpi: $!");
+    move($fromFile, $toFile)
-    move("$stageDir/batch1/stage/windows-xpi", 
+      or die('msg' => "Cannot rename $fromFile $toFile: $!");
-           "$stageDir/batch1/stage/win32/xpi")
+    $this->Log('msg' => "Moved $fromFile $toFile");
-      or die('msg' => "Cannot rename $stageDir/batch1/stage/windows-xpi $stageDir/batch1/stage/win32/xpi: $!");
-    move("$stageDir/batch1/stage/mac-xpi", 
-           "$stageDir/batch1/stage/mac/xpi")
-      or die('msg' => "Cannot rename $stageDir/batch1/stage/mac-xpi $stageDir/batch1/stage/mac/xpi: $!");
 }
 
 sub Verify {
@@ -152,13 +146,15 @@ sub Verify {
     ## Prepare the staging directory for the release.
     # Create the staging directory.
 
-    my $stageDir = $stageHome . '/' . $product . '-' . $version;
+    my $stageDir = catfile($stageHome, $product . '-' . $version);
 
     # Verify locales
     $this->Shell(
-      'cmd' => $stageHome . '/bin/verify-locales.pl -m ' . $stageDir . '/batch-source/rc' . $rc . '/mozilla/' . $appName . '/locales/shipped-locales',
+      'cmd' => catfile($stageHome, 'bin', 'verify-locales.pl'),
-      'logFile' => $logDir . '/stage-verify_l10n.log',
+      'cmdArgs' => ['-m', catfile($stageDir, 'batch-source', 'rc' . $rc,
-      'dir' => $stageDir . '/batch1/stage',
+                    'mozilla', $appName, 'locales', 'shipped-locales')],
+      'logFile' => catfile($logDir, 'stage_verify_l10n.log'),
+      'dir' => catfile($stageDir, 'batch1', 'stage'),
     );
 }
 
@@ -203,4 +199,16 @@ sub TrimCallback {
     }
 }
 
+sub Announce {
+    my $this = shift;
+
+    my $product = $config->Get('var' => 'product');
+    my $version = $config->Get('var' => 'version');
+
+    $this->SendAnnouncement(
+      subject => "$product $version stage step finished",
+      message => "$product $version staging area has been created.",
+    );
+}
+
 1;

tools/release/Bootstrap/Step/Tag.pm

@@ -1,36 +1,35 @@
 #
-# Tag step. Applies a CVS tag to the appropriate repositories.
+# Tag step. Sets up the tagging directory, and checks out the mozilla source.
 # 
 package Bootstrap::Step::Tag;
 use Bootstrap::Step;
+use Bootstrap::Step::Tag::Bump;
+use Bootstrap::Step::Tag::Mozilla;
+use Bootstrap::Step::Tag::l10n;
+use Bootstrap::Step::Tag::Talkback;
 use Bootstrap::Config;
 use File::Copy qw(move);
 use MozBuild::Util qw(MkdirWithPath);
-@ISA = ("Bootstrap::Step");
+@ISA = qw(Bootstrap::Step);
 
 my $config = new Bootstrap::Config;
 
+my @subSteps = ('Bump', 'Mozilla', 'l10n', 'Talkback');
+
 sub Execute {
     my $this = shift;
 
-    my $product = $config->Get('var' => 'product');
     my $productTag = $config->Get('var' => 'productTag');
+    my $rc = $config->Get('var' => 'rc');
+    my $tagDir = $config->Get('var' => 'tagDir');
+    my $mozillaCvsroot = $config->Get('var' => 'mozillaCvsroot');
     my $branchTag = $config->Get('var' => 'branchTag');
     my $pullDate = $config->Get('var' => 'pullDate');
-    my $l10n_pullDate = $config->Get('var' => 'l10n_pullDate');
-    my $rc = $config->Get('var' => 'rc');
-    my $version = $config->Get('var' => 'version');
-    my $appName = $config->Get('var' => 'appName');
     my $logDir = $config->Get('var' => 'logDir');
-    my $mozillaCvsroot = $config->Get('var' => 'mozillaCvsroot');
-    my $l10nCvsroot = $config->Get('var' => 'l10nCvsroot');
-    my $mofoCvsroot = $config->Get('var' => 'mofoCvsroot');
-    my $tagDir = $config->Get('var' => 'tagDir');
 
     my $releaseTag = $productTag.'_RELEASE';
     my $rcTag = $productTag.'_RC'.$rc;
-    my $minibranchTag = $productTag.'_MINIBRANCH';
+    my $releaseTagDir = catfile($tagDir, $releaseTag);
-    my $releaseTagDir = $tagDir . '/' . $releaseTag;
 
     # create the main tag directory
     if (not -d $releaseTagDir) {
@@ -39,185 +38,90 @@ sub Execute {
     }
 
     # Symlink to to RC dir
-    my $fromLink = $tagDir . '/' . $releaseTag;
+    my $fromLink = catfile($tagDir, $releaseTag);
-    my $toLink   = $tagDir . '/' .  $rcTag;
+    my $toLink   = catfile($tagDir, $rcTag);
     if (not -e $toLink) {
         symlink($fromLink, $toLink) 
           or die "Cannot symlink $fromLink $toLink: $!";
     }
 
     # Tagging area for Mozilla
-    if (not -d $releaseTagDir . '/cvsroot') {
+    my $cvsrootTagDir = catfile($releaseTagDir, 'cvsroot');
-        MkdirWithPath('dir' => $releaseTagDir . '/cvsroot') 
+    if (not -d $cvsrootTagDir) {
-          or die "Cannot mkdir $releaseTagDir/cvsroot: $!";
+        MkdirWithPath('dir' => $cvsrootTagDir) 
+          or die "Cannot mkdir $cvsrootTagDir: $!";
     }
 
     # Check out Mozilla from the branch you want to tag.
     # TODO this should support running without branch tag or pull date.
+
     $this->Shell(
-      'cmd' => 'cvs -d ' . $mozillaCvsroot . ' co -r ' . $branchTag . ' -D "' . $pullDate . '" mozilla/client.mk ',
+      'cmd' => 'cvs',
-      'dir' => $releaseTagDir . '/cvsroot',
+      'cmdArgs' => ['-d', $mozillaCvsroot, 
-      'logFile' => $logDir . '/client_mk.log',
+                    'co', 
+                    '-r', $branchTag, 
+                    '-D', $pullDate, 
+                     'mozilla/client.mk',
+                   ],
+      'dir' => $cvsrootTagDir,
+      'logFile' => catfile($logDir, 'tag_checkout_client_mk.log'),
     );
+
     $this->CheckLog(
-      'log' => $logDir . '/client_mk.log',
+      'log' => catfile($logDir, 'tag_checkout_client_mk.log'),
       'checkForOnly' => '^U mozilla/client.mk',
     );
-    $this->Shell(
-      'cmd' => 'gmake -f client.mk checkout MOZ_CO_PROJECT=all MOZ_CO_DATE="' . $pullDate . '"',
-      'dir' => $releaseTagDir . '/cvsroot/mozilla',
-      'timeout' => '3600',
-      'logFile' => $logDir . '/mozilla-checkout.log',
-    );
-  
-    # Create the RELEASE tag
-    $this->CvsTag(
-      'tagName' => $releaseTag,
-      'coDir'   => $releaseTagDir . '/cvsroot/mozilla',
-      'timeout' => '3600',
-      'logFile' => $logDir . '/cvsroot_tag-' . $releaseTag . '.log',
-    );
-
-    # Create the RC tag
-    $this->CvsTag(
-      'tagName' => $rcTag,
-      'coDir'   => $releaseTagDir . '/cvsroot/mozilla',
-      'timeout' => '3600',
-      'logFile' => $logDir . '/cvsroot_tag-' . $rcTag . '.log',
-    );
-
-    # Create a minibranch for the pull scripts so we can change them without
-    # changing anything on the original branch.
-    $this->CvsTag(
-      'tagName' => $minibranchTag,
-      'branch'  => '1',
-      'files'   => ['client.mk'],
-      'coDir'   => $releaseTagDir . '/cvsroot/mozilla',
-      'logFile' => $logDir . '/cvsroot_tag-' . $minibranchTag. '.log',
-    );
-
-    # Update client.mk to the minibranch you just created.
-    $this->Shell(
-      'cmd' => 'cvs up -r ' . $minibranchTag . ' client.mk',
-      'dir' => $releaseTagDir . '/cvsroot/mozilla',
-      'logFile' => $logDir . '/client_mk-update.log',
-    );
-
-    # Add the new product tag to the client.mk
-    open(INFILE,  "<$releaseTagDir/cvsroot/mozilla/client.mk");
-    open(OUTFILE, ">$releaseTagDir/cvsroot/mozilla/client.mk.tmp");
-    while(<INFILE>) {
-        $_ =~ s/$branchTag/$releaseTag/g;
-        print OUTFILE $_;
-    }
-    close INFILE;
-    close OUTFILE;
-
-    if (not move("$releaseTagDir/cvsroot/mozilla/client.mk.tmp", 
-                   "$releaseTagDir/cvsroot/mozilla/client.mk")) {
-        die "Cannot rename $releaseTagDir/cvsroot/mozilla/client.mk.tmp to $releaseTagDir/cvsroot/mozilla/client.mk";
-    }
 
     $this->Shell(
-      'cmd' => 'cvs commit -m "For ' . $product . ' ' . $version . ', redirect client.mk onto the ' . $releaseTag . ' tag." client.mk',
+      'cmd' => 'gmake',
-      'dir' => $releaseTagDir . '/cvsroot/mozilla',
+      'cmdArgs' => ['-f', 'client.mk', 'checkout', 'MOZ_CO_PROJECT=all', 
-      'logFile' => $logDir . '/client_mk-release_tag.log',
+                    'MOZ_CO_DATE=' . $pullDate],
-    );
+      'dir' => catfile($cvsrootTagDir, 'mozilla'),
-    $this->CheckLog(
+      'logFile' => catfile($logDir, 'tag_mozilla-checkout.log'),
-      'log' => $logDir . '/client_mk-release_tag.log',
-      'checkFor' => '^Checking in client.mk;',
-    );
-    $this->CheckLog(
-      'log' => $logDir . '/client_mk-release_tag.log',
-      'checkFor' => '^done',
     );
 
-    # Move the release tag onto the modified version of the pull scripts.
+    # Call substeps
-    $this->CvsTag(
+    my $numSteps = scalar(@subSteps);
-      'tagName' => $releaseTag,
+    my $currentStep = 0;
-      'force'   => '1',
+    while ($currentStep < $numSteps) {
-      'files'   => ['client.mk'],
+        my $stepName = $subSteps[$currentStep];
-      'coDir'   => $releaseTagDir . '/cvsroot/mozilla',
+        eval {
-      'logFile' => $logDir . '/cvsroot_clientmk_tag-' . $releaseTag. '.log',
+            $this->Log('Tag running substep' . $stepName);
-    );
+            my $step = "Bootstrap::Step::Tag::$stepName"->new();
-
+            $step->Execute();
-    # Move the RC tag onto the modified version of the pull scripts.
+        };
-    $this->CvsTag(
+        if ($@) {
-      'tagName' => $rcTag,
+            die("Tag substep $stepName Execute died: $@");
-      'force'   => '1',
-      'files'   => ['client.mk'],
-      'coDir'   => $releaseTagDir . '/cvsroot/mozilla',
-      'logFile' => $logDir . '/cvsroot_clientmk_tag-' . $rcTag. '.log',
-    );
-
-    # Create the mofo tag directory.
-    if (not -d "$releaseTagDir/mofo") {
-        MkdirWithPath('dir' => "$releaseTagDir/mofo") 
-          or die "Cannot mkdir $releaseTagDir/mofo: $!";
-    }
-
-    # Check out the talkback files from the branch you want to tag.
-    $this->Shell(
-      'cmd' => 'cvs -d ' . $mofoCvsroot . ' co -r ' . $branchTag . ' -D "' . $pullDate . '" talkback/fullsoft',
-      'dir' => $releaseTagDir . '/mofo',
-      'logFile' => $logDir . '/mofo-checkout.log'
-    );
-
-    # Create the talkback RELEASE tag.
-    $this->CvsTag(
-      'tagName' => $releaseTag,
-      'coDir'   => $releaseTagDir . '/mofo/talkback/fullsoft',
-      'logFile' => $logDir . '/mofo_tag-' . $releaseTag. '.log',
-    );
-
-    # Create the l10n tag directory.
-    if (not -d "$releaseTagDir/l10n") {
-        MkdirWithPath('dir' => "$releaseTagDir/l10n") 
-          or die "Cannot mkdir $releaseTagDir/l10n: $!";
-    }
-
-    # Grab list of shipped locales
-    my $shippedLocales = 
-      $releaseTagDir . '/cvsroot/mozilla/' . $appName . '/locales/shipped-locales';
-    open (FILE, "< $shippedLocales") 
-      or die "Cannot open file $shippedLocales: $!";
-    my @locales = <FILE>;
-    close FILE or die "Cannot close file $shippedLocales: $!";
-
-    # Check out the l10n files from the branch you want to tag.
-    for my $locale (@locales) {
-        # only keep first column
-        $locale =~ s/(\s+).*//;
-        # skip en-US, this is the default locale
-        if ($locale eq 'en-US') {
-            next;
         }
-        $this->Shell(
+        $currentStep += 1;
-            'cmd' => 'cvs -d ' . $l10nCvsroot . ' co -r ' . $branchTag . ' -D "' . $l10n_pullDate . '" l10n/' . $locale,
-            'dir' => $releaseTagDir . '/l10n',
-            'logFile' => $logDir . '/l10n-checkout.log',
-        );
     }
-
-    # Create the l10n RELEASE tag.
-    $this->CvsTag(
-      'tagName' => $releaseTag,
-      'coDir'   => $releaseTagDir . '/l10n/l10n',
-      'logFile' => $logDir . '/l10n_tag-' . $releaseTag. '.log',
-    );
-
-    # Create the RC tag.
-    $this->CvsTag(
-      'tagName' => $rcTag,
-      'coDir'   => $releaseTagDir . '/l10n/l10n',
-      'logFile' => $logDir . '/l10n_tag-' . $rcTag. '.log',
-    );
 }
 
 sub Verify {
     my $this = shift;
-    # TODO - independently verify that tag was applied
+
-    #$this->Shell('cmd' => 'echo Verify tag');
+    my $logDir = $config->Get('var' => 'logDir');
+
+    $this->CheckLog(
+      'log' => catfile($logDir, 'tag_mozilla-checkout.log'),
+      'checkFor' => '^U',
+    );
+
+    # Call substeps
+    my $numSteps = scalar(@subSteps);
+    my $currentStep = 0;
+    while ($currentStep < $numSteps) {
+        my $stepName = $subSteps[$currentStep];
+        eval {
+            $this->Log('Tag running substep' . $stepName);
+            my $step = "Bootstrap::Step::Tag::$stepName"->new();
+            $step->Verify();
+        };
+        if ($@) {
+            die("Tag substep $stepName Verify died: $@");
+        }
+        $currentStep += 1;
+    }
 }
 
 sub CvsTag {
@@ -235,34 +139,31 @@ sub CvsTag {
 
     # only force or branch specific files, not the whole tree
     if ($force and scalar(@{$files}) <= 0 ) {
-        die("ASSERT: Cannot specify force without files");
+        die("ASSERT: Bootstrap::Step::Tag::CvsTag(): Cannot specify force without files");
     } elsif ($branch and scalar(@{$files}) <= 0) {
-        die("ASSERT: Cannot specify branch without files");
+        die("ASSERT: Bootstrap::Step::Tag::CvsTag(): Cannot specify branch without files");
     } elsif ($branch and $force) {
-        die("ASSERT: Cannot specify both branch and force");
+        die("ASSERT: Bootstrap::Step::Tag::CvsTag(): Cannot specify both branch and force");
     } elsif (not $tagName) {
-        die("ASSERT: tagName must be specified");
+        die("ASSERT: Bootstrap::Step::Tag::CvsTag(): tagName must be specified");
     } elsif (not $logFile) {
-        die("ASSERT: logFile must be specified");
+        die("ASSERT: Bootstrap::Step::Tag::CvsTag(): logFile must be specified");
     }
 
-    my $cmd = 'cvs -q tag';
+    my @cmdArgs;
-    $cmd .= ' -F ' if ($force);
+    push(@cmdArgs, '-q');
-    $cmd .= ' -b ' if ($branch);
+    push(@cmdArgs, 'tag');
-    $cmd .= ' ' . $tagName;
+    push(@cmdArgs, '-F') if ($force);
-    $cmd .= ' ' . join(' ', @{$files}) if (defined($files));
+    push(@cmdArgs, '-b') if ($branch);
+    push(@cmdArgs, $tagName);
+    push(@cmdArgs, @$files) if defined($files);
 
     $this->Shell(
-      'cmd' => $cmd,
+      'cmd' => 'cvs',
+      'cmdArgs' => \@cmdArgs,
       'dir' => $coDir,
-      'timeout' => 3600,
       'logFile' => $logFile,
     );
-
-    $this->CheckLog(
-      'log' => $logFile,
-      'checkForOnly' => '^T ',
-    );
 }
 
 1;

tools/release/Bootstrap/Step/Updates.pm

@@ -29,16 +29,19 @@ sub Execute {
     }
 
     $this->Shell(
-      'cmd' => 'cvs -d ' . $mozillaCvsroot . ' co -d patcher mozilla/tools/patcher',
+      'cmd' => 'cvs',
-      'logFile' => $logDir . '/patcher_checkout.log',
+      'cmdArgs' => ['-d', $mozillaCvsroot, 'co', '-d', 'patcher', 
+                    catfile('mozilla', 'tools', 'patcher')],
+      'logFile' => catfile($logDir, 'updates_patcher-checkout.log'),
       'dir' => $updateDir,
-      'timeout' => 3600,
     );
 
     # config lives in private repo
     $this->Shell(
-      'cmd' => 'cvs -d ' . $mofoCvsroot . ' co -d config release/patcher/' . $patcherConfig,
+      'cmd' => 'cvs',
-      'logFile' => $logDir . '/patcher_config-checkout.log',
+      'cmdArgs' => ['-d', $mofoCvsroot, 'co', '-d', 'config',  
+                    catfile('release', 'patcher', $patcherConfig)],
+      'logFile' => catfile($logDir, 'updates_patcher-config-checkout.log'),
       'dir' => $updateDir,
     );
 
@@ -46,10 +49,11 @@ sub Execute {
     my $originalCvsrootEnv = $ENV{'CVSROOT'};
     $ENV{'CVSROOT'} = $mozillaCvsroot;
     $this->Shell(
-      'cmd' => './patcher2.pl --build-tools --app=' . $product . ' --config=../config/moz180-branch-patcher2.cfg',
+      'cmd' => './patcher2.pl',
-      'logFile' => $logDir . '/patcher_build-tools.log',
+      'cmdArgs' => ['--build-tools', '--app=' . $product,
-      'dir' => $updateDir . '/patcher',
+                    '--config=../config/' . $patcherConfig],
-      'timeout' => 3600,
+      'logFile' => catfile($logDir, 'updates_patcher-build-tools.log'),
+      'dir' => catfile($updateDir, 'patcher'),
     );
     if ($originalCvsrootEnv) {
         $ENV{'CVSROOT'} = $originalCvsrootEnv;
@@ -57,34 +61,29 @@ sub Execute {
     
     # download complete MARs
     $this->Shell(
-      'cmd' => './patcher2.pl --download --app=' . $product . ' --config=../config/moz180-branch-patcher2.cfg',
+      'cmd' => './patcher2.pl',
-      'logFile' => $logDir . '/patcher_download.log',
+      'cmdArgs' => ['--download', '--app=' . $product,
-      'dir' => $updateDir . '/patcher',
+                    '--config=../config/' . $patcherConfig],
-      'timeout' => 3600,
+      'logFile' => catfile($logDir, 'updates_patcher-download.log'),
+      'dir' => catfile($updateDir, 'patcher'),
     );
 
     # Create partial patches and snippets
     $this->Shell(
-      'cmd' => './patcher2.pl --create-patches -app=' . $product . ' --config=../config/moz180-branch-patcher2.cfg',
+      'cmd' => './patcher2.pl',
-      'logFile' => $logDir . '/patcher_create-patches.log',
+      'cmdArgs' => ['--create-patches', '--app=' . $product, 
-      'dir' => $updateDir . '/patcher',
+                    '--config=../config/' . $patcherConfig],
+      'logFile' => catfile($logDir, 'updates_patcher-create-patches.log'),
+      'dir' => catfile($updateDir, 'patcher'),
       'timeout' => 18000,
     );
     
-    # prepare aus2-staging
+    ### quick verification
-    # ssh aus2-staging.mozilla.org
+    # ensure that there are only test channels
-    # cd /opt/aus2/incoming/3-staging
+    my $testDir = catfile($updateDir, 'patcher', 'temp', $product,  
-    # tmpdir="`date +%Y%m%d`-${SHORT_PRODUCT}-${VERSION}"
+                          $oldVersion . '-' . $version, 'aus2.test');
-    # sudo mkdir ${tmpdir}-test
+
-    # sudo chown cltbld ${tmpdir}-test
+    File::Find::find(\&TestAusCallback, $testDir);
-    # sudo mkdir ${tmpdir}
-    # sudo chown cltbld ${tmpdir}
-    
-    # # copy updates from prometheus-vm.mozilla.org
-    # ssh prometheus-vm.mozilla.org
-    # cd /builds/${VERSION}-updates/release/patcher/temp/firefox/${PREVIOUS_VERSION}-${VERSION}/
-    # rsync -nav -e "ssh -i $HOME/.ssh/aus" aus2.test/ aus2-staging.mozilla.org:/opt/aus2/incoming/3-staging/${tmpdir}-test/
-    # rsync -nav -e "ssh -i $HOME/.ssh/aus" aus2/ aus2-staging.mozilla.org:/opt/aus2/incoming/3-staging/${tmpdir}/
 }
 
 sub Verify {
@@ -98,11 +97,30 @@ sub Verify {
     my $verifyDir = $config->Get('var' => 'verifyDir');
     my $product = $config->Get('var' => 'product');
 
-    ### quick verification
+    # Create verification area.
-    # ensure that there are only test channels
+    my $verifyDirVersion = catfile($verifyDir, $product . '-' . $version);
-    my $testDir = $updateDir . '/patcher/temp/' . $product . '/' . $oldVersion . '-' . $version . '/aus2.test';
+    MkdirWithPath('dir' => $verifyDirVersion) 
+      or die("Could not mkdir $verifyDirVersion: $!");
 
-    File::Find::find(\&TestAusCallback, $testDir);
+    foreach my $dir ('updates', 'common') {
+        $this->Shell(
+          'cmd' => 'cvs',
+          'cmdArgs' => ['-d', $mozillaCvsroot, 'co', '-d', $dir,
+                        catfile('mozilla', 'testing', 'release', $dir)],
+          'logFile' => catfile($logDir, 
+                               'updates_verify_checkout-' . $dir . '.log'),
+          'dir' => $verifyDirVersion,
+        );
+    }
+    
+    # Customize updates.cfg to contain the channels you are interested in 
+    # testing.
+    $this->Shell(
+      'cmd' => './verify.sh', 
+      'cmdArgs' => ['-c'],
+      'logFile' => catfile($logDir, 'updates_verify.log'),
+      'dir' => catfile($verifyDirVersion, 'updates'),
+    );
 }
 
 sub TestAusCallback { 
@@ -114,4 +132,16 @@ sub TestAusCallback {
     }
 }
 
+sub Announce {
+    my $this = shift;
+
+    my $product = $config->Get('var' => 'product');
+    my $version = $config->Get('var' => 'version');
+
+    $this->SendAnnouncement(
+      subject => "$product $version update step finished",
+      message => "$product $version updates are ready to be deployed to AUS and the candidates dir.",
+    );
+}
+
 1;

tools/release/Makefile

@@ -1,5 +1,6 @@
 test:
 	for f in release t/test.pl `find . -name "*.pm"`; do perl -c $$f; done
+	if [ -f t/test.log ]; then rm t/test.log; fi
 	./t/test.pl
 
 stage:
@@ -15,10 +16,10 @@ stage:
 	# staging environment
 	mkdir -p /data/cltbld/firefox-1.5.0.7/batch1/stage
 
-cvsmirror: cvsmirror_main cvsmirror_mofo
+cvsmirror: cvsmirror_mofo cvsmirror_main
 
 cvsmirror_main:
-	rsync -a --delete-after cvs-mirror.mozilla.org::mozilla/ /builds/cvsmirror/cvsroot/
+	rsync -a --delete-after --exclude=CVSROOT/config --exclude=CVSROOT/loginfo cvs-mirror.mozilla.org::mozilla/ /builds/cvsmirror/cvsroot/
 	rsync -a --delete-after cvs-mirror.mozilla.org::l10n/ /builds/cvsmirror/l10n/
 	chgrp -R cvs /builds/cvsmirror/cvsroot /builds/cvsmirror/l10n /builds/cvsmirror/mofo
 	chmod -R g+rw /builds/cvsmirror/cvsroot /builds/cvsmirror/l10n /builds/cvsmirror/mofo
@@ -40,7 +41,7 @@ clean_stage:
 	rm -rf /builds/release/logs/*
 	rm -rf /builds/updates/*
 	rm -rf /builds/verify/*
-	rm -rf /data/cltbld/firefox-1.5.0.7/
+	rm -rf /data/cltbld/firefox-*/
 	rm -rf /home/ftp/pub/firefox/*
 
 clean_cvsmirror:

tools/release/MozBuild/Util.pm

@@ -11,7 +11,7 @@ use Cwd;
 
 use base qw(Exporter);
 
-our @EXPORT_OK = qw(RunShellCommand MkdirWithPath HashFile DownloadFile);
+our @EXPORT_OK = qw(RunShellCommand MkdirWithPath HashFile DownloadFile Email);
 
 my $DEFAULT_EXEC_TIMEOUT = 600;
 my $EXEC_IO_READINCR = 1000;
@@ -299,6 +299,53 @@ sub HashFile {
    return $hashValue;
 }
 
+sub Email {
+    my %args = @_;
+
+    my $from = $args{'from'};
+    my $to = $args{'to'};
+    my $ccList = $args{'cc'} ? $args{'cc'} : '';
+    my $subject = $args{'subject'};
+    my $message = $args{'message'};
+
+    if (not defined($from)) {
+        die("ASSERT: MozBuild::Utils::Email(): from is required");
+    } elsif (not defined($to)) {
+        die("ASSERT: MozBuild::Utils::Email(): to is required");
+    } elsif (not defined($subject)) {
+        die("ASSERT: MozBuild::Utils::Email(): subject is required");
+    } elsif (not defined($message)) {
+        die("ASSERT: MozBuild::Utils::Email(): subject is required");
+    }    
+
+    if (defined($ccList) and ref($ccList) ne 'ARRAY') {
+        die "ASSERT: MozBuild::Utils::Email(): ccList is not an array ref\n"
+    }
+
+    my $sendmailBinary = '/usr/lib/sendmail';
+    my $blatBinary = 'c:\moztools\blat.exe';
+
+    if (-f $sendmailBinary) {
+        open(SENDMAIL, "|$sendmailBinary -oi -t")
+          or die "Can’t fork for sendmail: $!\n";
+    } elsif(-f $blatBinary) {
+        open(SENDMAIL, "|$blatBinary")
+          or die "Can’t fork for blat: $!\n";
+    } else {
+        die("ASSERT: cannot find $sendmailBinary or $blatBinary");
+    }
+
+    print SENDMAIL "From: $from\n";
+    print SENDMAIL "To: $to\n";
+    foreach my $cc (@{$ccList}) {
+        print SENDMAIL "CC: $cc\n";
+    }
+    print SENDMAIL "Subject: $subject\n\n";
+    print SENDMAIL "$message";
+    close(SENDMAIL) or warn "sendmail didn’t close nicely: $!";
+
+}
+
 sub DownloadFile {
     my %args = @_;
 

tools/release/bootstrap.cfg

@@ -2,6 +2,7 @@ productTag      = THUNDERBIRD_1_5_0_9
 branchTag       = MOZILLA_1_8_0_BRANCH
 pullDate        = 2006-12-06 22:10 PST
 rc              = 1
+oldRc           = 1
 version         = 1.5.0.9
 oldVersion      = 1.5.0.8
 appName         = mail
@@ -18,3 +19,7 @@ verifyDir       = /builds/verify
 patcherConfig   = moz180-branch-patcher2.cfg
 tagDir          = /builds/tags
 buildPlatform   = Linux_2.4.21-37.EL_Depend
+from            = Bootstrap <build@example.com>
+to              = engineer@example.com
+cc              = interested_users@example.com, others@example.com
+patcherConfig   = moz180-branch-patcher2.cfg

tools/release/release

@@ -2,7 +2,7 @@
 use strict;
 
 use Getopt::Long;
-use MozBuild::Util;
+use MozBuild::Util qw(Email);
 use Bootstrap::Step::Tag;
 use Bootstrap::Step::Build;
 use Bootstrap::Step::Source;
@@ -32,6 +32,7 @@ sub ProcessArgs {
     GetOptions(
         \%config,
         "step|s=s", "only|o=s", "list|l", "help|h", "execute|e", "verify|v",
+        "push|p", "announce|a"
     );
 
     if ($config{'list'}) {
@@ -43,19 +44,21 @@ sub ProcessArgs {
     }
 
     if ($config{'help'}) {
-        print "Usage: release [-l] [-s Step] [-o Step] [-e | v] [-h]\n";
+        print "Usage: release [-l] [-s Step] [-o Step] [-e | -v | -p | -a] [-h]\n";
         print "    -l list all Steps\n";
         print "    -s start at Step\n";
         print "    -o only run one Step\n";
         print "    -e only run Execute\n";
         print "    -v only run Verify\n";
+        print "    -p only run Push\n";
+        print "    -a only run Announce\n";
         print "    -h this usage message\n";
         exit(0);
     }
 }
 
 sub DetermineSteps() {
-    my $currentStep = 0;
+    my $currentStep;
     my $desiredStep;
 
     if (defined($config{'step'})) {
@@ -74,12 +77,13 @@ sub DetermineSteps() {
             } else {
                 $stepNumber += 1;
             }
-            if ($stepNumber > scalar(@allSteps)) {
+        }
-                die("Step $desiredStep not found!\n");
+        if (not defined($currentStep)) {
-            }
+            die("Step $desiredStep not found!\n");
         }
     } else {
         print "Bootstrap running default steps.\n";
+        $currentStep = 0;
     }
 
     while ($currentStep < scalar(@allSteps)) {
@@ -109,15 +113,42 @@ sub PerformStep {
         } elsif (defined($config{'verify'})) {
             print "Bootstrap only running Verify\n";
             $step->Verify();
+        } elsif (defined($config{'push'})) {
+            print "Bootstrap only running Push\n";
+            $step->Push();
+        } elsif (defined($config{'announce'})) {
+            print "Bootstrap only running Announce\n";
+            $step->Announce();
         } else {
             $step->Execute();
             $step->Verify();
+            $step->Push();
+            $step->Announce();
         }
     };
-    if ($@) 
+    if ($@) {
-    {
         print "Step $stepName died: $@";
-        exit(1);
+        my $conf = new Bootstrap::Config;
+        my $from = $conf->Get(var => 'from');
+        my $to = $conf->Get(var => 'to');
+        my $cc = $conf->Get(var => 'cc');
+
+        my @ccList = split(', ', $cc);
+ 
+        eval {
+            Email(
+              from => $from,
+              to => $to,
+              cc => \@ccList,
+              subject => "Step $stepName died: $@",
+              message => "Step $stepName died: $@\nSee the release.log for more information.",
+            );
+            exit(1);
+        };
+        if ($@) {
+            print "Unable to email failure message to $to: $@";
+            exit(1);
+        }
     }
 }
 

tools/release/t/Bootstrap/Step/Dummy.pm

@@ -14,6 +14,28 @@ sub Execute {
         print("testfailed, could not get productTag var from Config: $!\n");
     }
 
+    eval {
+        $this->Shell('cmd' => 'echo', 
+                     'cmdArgs' => ['success'],
+                     'logFile' => 't/test.log' 
+        );
+    };
+    if ($@) {
+        print("testfailed, shell call to echo success to test log should not throw exception: $!\n");
+    }
+
+    eval {
+        $this->CheckLog( 
+            'log' => './t/test.log',
+            'checkForOnly' => '^success',
+        );
+    };
+  
+    if ($@) {
+        print("testfailed, should not throw exception, log contains only success: $!\n");
+    }
+
+
     eval {
         $this->Shell( 'cmd' => 'true', logFile => 't/test.log' );
     };
@@ -30,27 +52,6 @@ sub Execute {
         print("testfailed, shell call to false should throw exception: $!\n");
     }
 
-    eval {
-        $this->CheckLog( 
-            'log' => './t/test.log',
-            'checkForOnly' => '^success',
-        );
-    };
-  
-    if (not $@) {
-        print("testfailed, should throw exception, log contains more than success: $!\n");
-    }
-    eval {
-        $this->CheckLog( 
-            'log' => './t/test.log',
-            'checkForOnly' => '^success',
-        );
-    };
-  
-    if (not $@) {
-        print("testfailed, should throw exception, log contains more than success: $!\n");
-    }
-
     eval {
         $this->CheckLog( 
             'log' => './t/test.log',
@@ -66,7 +67,8 @@ sub Execute {
 
 sub Verify {
     my $this = shift;
-    $this->Shell('cmd' => 'echo Verify tag', logFile => 't/test.log');
+    $this->Shell(
+      'cmd' => 'echo', 'cmdArgs' => ['Verify', 'tag'], logFile => 't/test.log');
     $this->Log('msg' => 'finished');
 }
 

tools/release/t/test.pl

@@ -2,8 +2,15 @@
 use strict;
 use Bootstrap::Step;
 use t::Bootstrap::Step::Dummy;
+use t::Bootstrap::Step::Tag;
 
 my $step = t::Bootstrap::Step::Dummy->new();
 
 $step->Execute();
 $step->Verify();
+$step->Push();
+$step->Announce();
+
+#$step = t::Bootstrap::Step::Tag->new();
+
+#$step->Execute();