framework changes for bootstrap b=352230 r=preed

tools/release/Bootstrap/Step.pm

@@ -5,11 +5,14 @@
 package Bootstrap::Step;
 use MozBuild::Util;
 use Config::General;
+use POSIX qw(strftime);
+
+my $DEFAULT_TIMEOUT = 3600;
 
 # shared static config
 my $conf = new Config::General("bootstrap.cfg");
 if (not $conf) {
-    die "Config is null: $!";
+    die "Config is null!";
 }
 
 sub new {
@@ -25,23 +28,32 @@ sub Shell {
     my %args = @_;
     my $cmd = $args{'cmd'};
     my $dir = $args{'dir'};
-    my $timeout = $args{'timeout'};
+    my $timeout = $args{'timeout'} ? $args{'timeout'} : $DEFAULT_TIMEOUT;
+    my $logFile = $args{'logFile'};
     my $rv = '';
 
     if ($dir) {
-        chdir($dir) || die "Cannot chdir to $dir: $!";
+        $this->Log('msg' => 'Changing directory to ' . $dir);
+        chdir($dir) or die "Cannot chdir to $dir: $!";
         $this->Log('msg' => 'Running shell command ' . $cmd . ' in dir ' . $dir);
     } else {
         $this->Log('msg' => 'Running shell command ' . $cmd);
     }
+
+    $this->Log('msg' => 'Starting time is ' . $this->CurrentTime());
+
+    print "Timeout: $timeout\n";
+
     if ($timeout) {
         $rv = MozBuild::Util::RunShellCommand(
            'command' => "$cmd",
            'timeout' => "$timeout",
+           'logfile' => "$logFile",
         );
     } else {
         $rv = MozBuild::Util::RunShellCommand(
            'command' => "$cmd",
+           'logfile' => "$logFile",
         );
     }
 
@@ -49,28 +61,32 @@ sub Shell {
     my $timedOut  = $rv->{'timedOut'};
     my $signalName  = $rv->{'signalName'};
     my $dumpedCore = $rv->{'dumpedCore'};
-    if ($exitValue) {
+    my $pid = $rv->{'pid'};
-        if ($exitValue != 0) {
+    print "Pid: $pid\n";
-            $this->Log('msg' => "output: $rv->{'output'}");
-            die("shell call returned bad exit code: $exitValue");
-        }
-    }
     if ($timedOut) {
-        $this->Log('msg' => "output: $rv->{'output'}");
+        $this->Log('msg' => "output: $rv->{'output'}") if $rv->{'output'};
         die("FAIL shell call timed out after $timeout seconds");
     }
     if ($signalName) {
-        $this->Log('msg' => "output: $rv->{'output'}");
         print ("WARNING shell recieved signal $signalName");
     }
     if ($dumpedCore) {
-        $this->Log('msg' => "output: $rv->{'output'}");
+        $this->Log('msg' => "output: $rv->{'output'}") if $rv->{'output'};
         die("FAIL shell call dumped core");
     }
+    if ($exitValue) {
+        if ($exitValue != 0) {
+            $this->Log('msg' => "output: $rv->{'output'}") if $rv->{'output'};
+            die("shell call returned bad exit code: $exitValue");
+        }
+    }
 
-    if($rc->{'output'}) {
+    if ($rv->{'output'} && not defined($logFile)) {
         $this->Log('msg' => "output: $rv->{'output'}");
     }
+
+    # current time
+    $this->Log('msg' => 'Ending time is ' . $this->CurrentTime());
 }
 
 sub Log {
@@ -89,8 +105,8 @@ sub Config {
 
     my %config = $conf->getall();
 
-    if ($config{'app'}{'firefox'}{'release'}{'1.5.0.7'}{$var}) {
+    if ($config{'app'}{$var}) {
-        return $config{'app'}{'firefox'}{'release'}{'1.5.0.7'}{$var};
+        return $config{'app'}{$var};
     } else {
         die("No such config variable: $var\n");
     }
@@ -105,30 +121,41 @@ sub CheckLog {
     my $checkFor = $args{'checkFor'};
     my $checkForOnly = $args{'checkForOnly'};
 
-    open (FILE, "< $log") || die "Cannot open file $log: $!";
+    if (not defined($log)) {
+        die "No log file specified";
+    }
+
+    open (FILE, "< $log") or die "Cannot open file $log: $!";
     my @contents = <FILE>;
-    close FILE || die "Cannot close file $log: $!";
+    close FILE or die "Cannot close file $log: $!";
   
     if ($notAllowed) {
         my @errors = grep(/$notAllowed/i, @contents);
         if (@errors) {
-            die "Errors in log ($log): \n\n @errors \n $!";
+            die "Errors in log ($log): \n\n @errors \n";
         }
     }
     if ($checkFor) {
         if (not grep(/$checkFor/i, @contents)) {
-            die "$checkFor is not present in file $log: $!";
+            die "$checkFor is not present in file $log \n";
         }
     }
     if ($checkForOnly) {
         if (not grep(/$checkForOnly/i, @contents)) {
-            die "$checkForOnly is not present in file $log: $!";
+            die "$checkForOnly is not present in file $log \n";
         }
         my @errors = grep(!/$checkForOnly/i, @contents);
         if (@errors) {
-            die "Errors in log ($log): \n\n @errors \n $!";
+            die "Errors in log ($log): \n\n @errors \n";
         }
     }
 }
 
+sub CurrentTime() {
+    my $this = shift;
+    my $args = @_;
+
+    return strftime("%T %D", localtime());
+}
+
 1;

tools/release/MozBuild/Util.pm

@@ -1,12 +1,12 @@
 package MozBuild::Util;
-
+use File::Path;
-# Stolen from the code for bug 336463
 
 my $EXEC_TIMEOUT = '600';
 
 sub RunShellCommand {
     my %args = @_;
     my $shellCommand = $args{'command'};
+    my $logfile = $args{'logfile'};
 
     # optional
     my $timeout = exists($args{'timeout'}) ? $args{'timeout'} : $EXEC_TIMEOUT;
@@ -24,20 +24,31 @@ sub RunShellCommand {
     my $dumpedCore;
     my $timedOut;
     my $output = '';
+    my $pid;
 
     eval {
         local $SIG{'ALRM'} = sub { die "alarm\n" };
         alarm $timeout;
  
-        if (! $redirectStderr || $shellCommand =~ "2>&1") {
+        if (! $redirectStderr or $shellCommand =~ "2>&1") {
-            open CMD, "$shellCommand |" or die "Could not run command $shellCommand: $!";
+            $pid = open CMD, "$shellCommand |" or die "Could not run command $shellCommand: $!";
         } else {
-            open CMD, "$shellCommand 2>&1 |" or die "Could not close command $shellCommand: $!";
+            $pid = open CMD, "$shellCommand 2>&1 |" or die "Could not close command $shellCommand: $!";
         }
 
+        if (defined($logfile)) {
+            open(LOGFILE, ">> $logfile") or die "Could not open logfile $logfile: $!";
+            LOGFILE->autoflush(1);
+        }
         while (<CMD>) {
             $output .= $_;
             print $_ if ($printOutputImmediately);
+            if (defined($logfile)) {
+                print LOGFILE $_;
+            }
+        }
+        if (defined($logfile)) {
+            close(LOGFILE) or die "Could not close logfile $logfile: $!";
         }
 
         close CMD;# or die "Could not close command: $!";
@@ -51,13 +62,14 @@ sub RunShellCommand {
     if ($@) {
         if ($@ eq "alarm\n") {
             $timedOut = 1;
+            kill(9, $pid) or die "Could not kill timed-out $pid: $!";
         } else {
             warn "Error running $shellCommand: $@\n";
             $output = $@;
         }
     }
 
-    if ($exitValue || $timedOut || $dumpedCore || $signalNum) {
+    if ($exitValue or $timedOut or $dumpedCore or $signalNum) {
         if ($timedOut) {
             # callers expect exitValue to be non-zero if request timed out
             $exitValue = 1;
@@ -68,7 +80,32 @@ sub RunShellCommand {
              exitValue => $exitValue,
              sigName => $sigName,
              output => $output,
-             dumpedCore => $dumpedCore };
+             dumpedCore => $dumpedCore,
+             pid => $pid,
+    };
+}
+
+## This is a wrapper function to get easy true/false return values from a
+## mkpath()-like function. mkpath() *actually* returns the list of directories
+## it created in the pursuit of your request, and keeps its actual success 
+## status in $@. 
+
+sub MkdirWithPath
+{
+    my %args = @_;
+
+    my $dirToCreate = $args{'dir'};
+    my $printProgress = $args{'printProgress'};
+    my $dirMask = $args{'dirMask'};
+
+    die "ASSERT: MkdirWithPath() needs an arg" if not defined($dirToCreate);
+
+    ## Defaults based on what mkpath does...
+    $printProgress = defined($printProgress) ? $printProgress : 0;
+    $dirMask = defined($dirMask) ? $dirMask : 0777;
+
+    eval { mkpath($dirToCreate, $printProgress, $dirMask) };
+    return defined($@);
 }
 
 1;

tools/release/README

@@ -4,10 +4,33 @@ Bootstrap Release
 Bootstrap is a release automation tool.
 Use "release -h" for help.
 
-Tools run first
+Pre-flight Checklist
 -----------------
-* checklist/config generator
+There are a number of manual steps that must be performed, so a default
-* version bump tool
+end-to-end run will generally not work.
+
+Before any steps:
+
+* verify shipped-locales
+* edit bootstrap.cfg
+* edit tinder-config.pl/mozconfig 
+* version bump
+
+After Build and Repack steps:
+
+* rsync builds to candidates dir
+
+After the Update step:
+
+* edit patcher config
+* edit mozilla/testing/release/updates/updates.cfg
+
+After Sign step:
+
+* create bouncer links
+* rsync builds to mirrors
+* wait 12 hours for mirrors to catch up
+* rsync production AUS config
 
 Steps are in dependency order. The process may be restarted at any step as 
 long as all previous steps are satisfied.
@@ -16,53 +39,46 @@ PASS/FAIL verification is run after every step.
 
 Steps 
 -----------------
-1) tag 
+1) Tag 
-2) build 
+2) Build 
-2.1) push
+2.1) Push
-2.2) announce
+2.2) Announce
-3) source
+3) Source
-4) repack
+4) Repack
-4.1) push
+4.1) Push
-4.2) announce
+4.2) Announce
-5) updates
+5) Updates
-5.1) push
+5.1) Push
-5.2) announce
+5.2) Announce
-6) stage
+6) Stage
-6.1) merge
+6.1) Merge
-6.2) announce
+6.2) Announce
-7) sign
+7) Sign
-8) release
-8.1) announce
    
 Details
 -----------------
-tag 
+Tag 
-   _RELEASE and _RCn for mozilla and talkback
+   _RELEASE and _RCn for mozilla, l10n and talkback
-build 
+Build 
   en-US build from source (based on tag)
   push to stage
   announce
-source
+Source
-  bz2 archive based on tag
+  bz2 archive (based on tag)
   push to stage
-repack
+Repack
-  repack l10n, uses en-US build
+  repack l10n, uses en-US build (based on tag)
   push to stage
   announce
-updates
+Updates
   uses patcher
   generate partials and AUS config ("snippets")
   push to stage
   announce
-stage
+Stage
   uses groom-files
   create release directory/filename structure
   merge updates
   announce
-sign
+Sign
   manual
-release
-  create bouncer links
-  rsync builds to mirrors
-  rsync production AUS config
-  announce

tools/release/bootstrap.cfg

@@ -1,13 +1,23 @@
-<app firefox>
+<app>
-  <release 1.5.0.7>
+  productTag      = FIREFOX_1_6_0_7
-    productTag = FIREFOX_1_5_0_7
+  branchTag       = MOZILLA_1_8_0_BRANCH
-    branchTag  = MOZILLA_1_8_0_7
+  pullDate        = 2006-11-03 12:00 PDT
-    pullDate   = 2006-09-06 18:00 PDT
+  rc              = 1
-    rc         = 1
+  version         = 1.6.0.7
-    version    = 1.5.0.7
+  oldVersion      = 1.5.0.7
-    appName    = browser
+  appName         = browser
-    product    = firefox
+  product         = firefox
-    buildDir   = /builds/tinderbox/
+  buildDir        = /builds/tinderbox/Fx-Mozilla1.8.0-Release
-    pushDate   = 2006090601
+  l10n-buildDir   = /builds/tinderbox/Fx-Mozilla1.8.0-l10n-Release
-  </release>
+  pushDate        = 2006-10-30-22
+  logDir          = /builds/release/logs
+  mozillaCvsroot  = /builds/cvsmirror/cvsroot
+  l10nCvsroot     = /builds/cvsmirror/l10n
+  mofoCvsroot     = /builds/cvsmirror/mofo
+  stageHome       = /data/cltbld
+  updateDir       = /builds/updates
+  verifyDir       = /builds/verify
+  patcherConfig   = moz180-branch-patcher2.cfg
+  tagDir          = /builds/tags
+  buildPlatform   = Linux_2.4.21-37.EL_Depend
 </app>

tools/release/release

@@ -33,7 +33,7 @@ sub main {
 sub ProcessArgs {
     GetOptions(
         \%config,
-        "step|s=s", "only|o=s", "list|l", "help|h",
+        "step|s=s", "only|o=s", "list|l", "help|h", "execute|e", "verify|v",
     );
 
     if ($config{'list'}) {
@@ -45,10 +45,12 @@ sub ProcessArgs {
     }
 
     if ($config{'help'}) {
-        print "Usage: release [-l] [-s Step] [-o Step] [-h]\n";
+        print "Usage: release [-l] [-s Step] [-o Step] [-e | v] [-h]\n";
         print "    -l list all Steps\n";
         print "    -s start at Step\n";
         print "    -o only run one Step\n";
+        print "    -e only run Execute\n";
+        print "    -v only run Verify\n";
         print "    -h this usage message\n";
         exit(0);
     }
@@ -103,8 +105,16 @@ sub PerformStep {
     print "Bootstrap running $stepName\n";
     my $step = "Bootstrap::Step::$stepName"->new();
     eval {
-        $step->Execute();
+        if (defined($config{'execute'})) {
-        $step->Verify();
+            print "Bootstrap only running Execute\n";
+            $step->Execute();
+        } elsif (defined($config{'verify'})) {
+            print "Bootstrap only running Verify\n";
+            $step->Verify();
+        } else {
+            $step->Execute();
+            $step->Verify();
+        }
     };
     if ($@) 
     {

tools/release/t/Bootstrap/Step/Dummy.pm

@@ -12,7 +12,7 @@ sub Execute {
     }
 
     eval {
-        $this->Shell( 'cmd' => 'true' );
+        $this->Shell( 'cmd' => 'true', logFile => 't/test.log' );
     };
 
     if ($@) {
@@ -20,7 +20,7 @@ sub Execute {
     }
 
     eval {
-        $this->Shell( 'cmd' => 'false' );
+        $this->Shell( 'cmd' => 'false', logFile => 't/test.log' );
     };
 
     if (not $@) {
@@ -63,7 +63,7 @@ sub Execute {
 
 sub Verify {
     my $this = shift;
-    $this->Shell('cmd' => 'echo Verify tag');
+    $this->Shell('cmd' => 'echo Verify tag', logFile => 't/test.log');
     $this->Log('msg' => 'finished');
 }
 

tools/release/t/test.log

@@ -2,3 +2,10 @@ success
 failed
 success
 failed
+Verify tag
+Verify tag
+Verify tag
+Verify tag
+Verify tag
+Verify tag
+Verify tag