ci: bump bootstrap version

Signed-off-by: Vicent Marti <vmg@strn.cat>

.github/docker/cluster_test_14/Dockerfile

@@ -1,6 +1,6 @@
 # DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows"
 
-ARG bootstrap_version=3
+ARG bootstrap_version=4
 ARG image="vitess/bootstrap:${bootstrap_version}-mysql57"
 
 FROM "${image}"
@@ -39,4 +39,4 @@ RUN mkdir -p $VTDATAROOT
 RUN go install golang.org/x/tools/cmd/goimports@latest
 
 # sleep for 50 minutes
-CMD sleep 3000
+CMD sleep 3000

.github/docker/cluster_test_vtorc/Dockerfile

@@ -1,6 +1,6 @@
 # DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows"
 
-ARG bootstrap_version=3
+ARG bootstrap_version=4
 ARG image="vitess/bootstrap:${bootstrap_version}-mysql57"
 
 FROM "${image}"
@@ -39,4 +39,4 @@ RUN mkdir -p $VTDATAROOT
 RUN go install golang.org/x/tools/cmd/goimports@latest
 
 # sleep for 50 minutes
-CMD sleep 3000
+CMD sleep 3000

Makefile

@@ -244,7 +244,7 @@ $(PROTO_GO_OUTS): minimaltools install_protoc-gen-go proto/*.proto
 # This rule builds the bootstrap images for all flavors.
 DOCKER_IMAGES_FOR_TEST = mariadb mariadb103 mysql56 mysql57 mysql80 percona percona57 percona80
 DOCKER_IMAGES = common $(DOCKER_IMAGES_FOR_TEST)
-BOOTSTRAP_VERSION=3
+BOOTSTRAP_VERSION=4
 ensure_bootstrap_version:
 	find docker/ -type f -exec sed -i "s/^\(ARG bootstrap_version\)=.*/\1=${BOOTSTRAP_VERSION}/" {} \;
 	sed -i 's/\(^.*flag.String(\"bootstrap-version\",\) *\"[^\"]\+\"/\1 \"${BOOTSTRAP_VERSION}\"/' test.go

bootstrap.sh

@@ -110,7 +110,7 @@ install_protoc() {
 
   # This is how we'd download directly from source:
   # wget https://github.com/protocolbuffers/protobuf/releases/download/v$version/protoc-$version-$platform-${target}.zip
-  wget "${VITESS_RESOURCES_DOWNLOAD_URL}/protoc-$version-$platform-${target}.zip"
+  $VTROOT/tools/wget-retry "${VITESS_RESOURCES_DOWNLOAD_URL}/protoc-$version-$platform-${target}.zip"
   unzip "protoc-$version-$platform-${target}.zip"
   ln -snf "$dist/bin/protoc" "$VTROOT/bin/protoc"
 }
@@ -124,7 +124,7 @@ install_zookeeper() {
   zk="zookeeper-$version"
   # This is how we'd download directly from source:
   # wget "https://archive.apache.org/dist/zookeeper/$zk/$zk.tar.gz"
-  wget "${VITESS_RESOURCES_DOWNLOAD_URL}/${zk}.tar.gz"
+  $VTROOT/tools/wget-retry "${VITESS_RESOURCES_DOWNLOAD_URL}/${zk}.tar.gz"
   tar -xzf "$zk.tar.gz"
   ant -f "$zk/build.xml" package
   ant -f "$zk/zookeeper-contrib/zookeeper-contrib-fatjar/build.xml" jar
@@ -156,7 +156,7 @@ install_etcd() {
   # This is how we'd download directly from source:
   # download_url=https://github.com/etcd-io/etcd/releases/download
   # wget "$download_url/$version/$file"
-  wget "${VITESS_RESOURCES_DOWNLOAD_URL}/${file}"
+  $VTROOT/tools/wget-retry "${VITESS_RESOURCES_DOWNLOAD_URL}/${file}"
   if [ "$ext" = "tar.gz" ]; then
     tar xzf "$file"
   else
@@ -190,7 +190,7 @@ install_k3s() {
   # This is how we'd download directly from source:
   # download_url=https://github.com/rancher/k3s/releases/download
   # wget -O  $dest "$download_url/$version/$file"
-  wget -O  $dest "${VITESS_RESOURCES_DOWNLOAD_URL}/$file-$version"
+  $VTROOT/tools/wget-retry -O $dest "${VITESS_RESOURCES_DOWNLOAD_URL}/$file-$version"
   chmod +x $dest
   ln -snf  $dest "$VTROOT/bin/k3s"
 }
@@ -215,7 +215,7 @@ install_consul() {
   # This is how we'd download directly from source:
   # download_url=https://releases.hashicorp.com/consul
   # wget "${download_url}/${version}/consul_${version}_${platform}_${target}.zip"
-  wget "${VITESS_RESOURCES_DOWNLOAD_URL}/consul_${version}_${platform}_${target}.zip"
+  $VTROOT/tools/wget-retry "${VITESS_RESOURCES_DOWNLOAD_URL}/consul_${version}_${platform}_${target}.zip"
   unzip "consul_${version}_${platform}_${target}.zip"
   ln -snf "$dist/consul" "$VTROOT/bin/consul"
 }
@@ -242,11 +242,11 @@ install_chromedriver() {
         ;;
       esac
       echo "For Arm64, using prebuilt binary from electron (https://github.com/electron/electron/) of version 76.0.3809.126"
-      wget https://github.com/electron/electron/releases/download/v6.0.3/chromedriver-v6.0.3-linux-arm64.zip
+      $VTROOT/tools/wget-retry https://github.com/electron/electron/releases/download/v6.0.3/chromedriver-v6.0.3-linux-arm64.zip
       unzip -o -q chromedriver-v6.0.3-linux-arm64.zip -d "$dist"
       rm chromedriver-v6.0.3-linux-arm64.zip
   else
-      curl -sL "https://chromedriver.storage.googleapis.com/$version/chromedriver_linux64.zip" > chromedriver_linux64.zip
+      $VTROOT/tools/wget-retry "https://chromedriver.storage.googleapis.com/$version/chromedriver_linux64.zip"
       unzip -o -q chromedriver_linux64.zip -d "$dist"
       rm chromedriver_linux64.zip
   fi

docker/base/Dockerfile

@@ -21,7 +21,7 @@
 # TODO(mberlin): Remove the symlink and this note once
 # https://github.com/docker/hub-feedback/issues/292 is fixed.
 
-ARG bootstrap_version=3
+ARG bootstrap_version=4
 ARG image="vitess/bootstrap:${bootstrap_version}-mysql57"
 
 FROM "${image}"

docker/base/Dockerfile.mariadb

@@ -1,4 +1,4 @@
-ARG bootstrap_version=3
+ARG bootstrap_version=4
 ARG image="vitess/bootstrap:${bootstrap_version}-mariadb"
 
 FROM "${image}"

docker/base/Dockerfile.mariadb103

@@ -1,4 +1,4 @@
-ARG bootstrap_version=3
+ARG bootstrap_version=4
 ARG image="vitess/bootstrap:${bootstrap_version}-mariadb103"
 
 FROM "${image}"

docker/base/Dockerfile.mysql56

@@ -1,4 +1,4 @@
-ARG bootstrap_version=3
+ARG bootstrap_version=4
 ARG image="vitess/bootstrap:${bootstrap_version}-mysql56"
 
 FROM "${image}"

docker/base/Dockerfile.mysql80

@@ -1,4 +1,4 @@
-ARG bootstrap_version=3
+ARG bootstrap_version=4
 ARG image="vitess/bootstrap:${bootstrap_version}-mysql80"
 
 FROM "${image}"

docker/base/Dockerfile.percona

@@ -1,4 +1,4 @@
-ARG bootstrap_version=3
+ARG bootstrap_version=4
 ARG image="vitess/bootstrap:${bootstrap_version}-percona"
 
 FROM "${image}"

docker/base/Dockerfile.percona57

@@ -1,4 +1,4 @@
-ARG bootstrap_version=3
+ARG bootstrap_version=4
 ARG image="vitess/bootstrap:${bootstrap_version}-percona57"
 
 FROM "${image}"

docker/base/Dockerfile.percona80

@@ -1,4 +1,4 @@
-ARG bootstrap_version=3
+ARG bootstrap_version=4
 ARG image="vitess/bootstrap:${bootstrap_version}-percona80"
 
 FROM "${image}"

docker/lite/Dockerfile.alpine

@@ -17,7 +17,7 @@
 #       ensure images contain the right binaries.
 
 # Use a temporary layer for the build stage.
-ARG bootstrap_version=3
+ARG bootstrap_version=4
 ARG image="vitess/bootstrap:${bootstrap_version}-mariadb103"
 
 FROM "${image}" AS builder

docker/lite/Dockerfile.mariadb

@@ -17,7 +17,7 @@
 #       ensure images contain the right binaries.
 
 # Use a temporary layer for the build stage.
-ARG bootstrap_version=3
+ARG bootstrap_version=4
 ARG image="vitess/bootstrap:${bootstrap_version}-mariadb"
 
 FROM "${image}" AS builder

docker/lite/Dockerfile.mariadb103

@@ -17,7 +17,7 @@
 #       ensure images contain the right binaries.
 
 # Use a temporary layer for the build stage.
-ARG bootstrap_version=3
+ARG bootstrap_version=4
 ARG image="vitess/bootstrap:${bootstrap_version}-mariadb103"
 
 FROM "${image}" AS builder

docker/lite/Dockerfile.mysql56

@@ -17,7 +17,7 @@
 #       ensure images contain the right binaries.
 
 # Use a temporary layer for the build stage.
-ARG bootstrap_version=3
+ARG bootstrap_version=4
 ARG image="vitess/bootstrap:${bootstrap_version}-mysql56"
 
 FROM "${image}" AS builder

docker/lite/Dockerfile.mysql57

@@ -17,7 +17,7 @@
 #       ensure images contain the right binaries.
 
 # Use a temporary layer for the build stage.
-ARG bootstrap_version=3
+ARG bootstrap_version=4
 ARG image="vitess/bootstrap:${bootstrap_version}-mysql57"
 
 FROM "${image}" AS builder

docker/lite/Dockerfile.mysql80

@@ -17,7 +17,7 @@
 #       ensure images contain the right binaries.
 
 # Use a temporary layer for the build stage.
-ARG bootstrap_version=3
+ARG bootstrap_version=4
 ARG image="vitess/bootstrap:${bootstrap_version}-mysql80"
 
 FROM "${image}" AS builder

docker/lite/Dockerfile.percona

@@ -17,7 +17,7 @@
 #       ensure images contain the right binaries.
 
 # Use a temporary layer for the build stage.
-ARG bootstrap_version=3
+ARG bootstrap_version=4
 ARG image="vitess/bootstrap:${bootstrap_version}-percona"
 
 FROM "${image}" AS builder

docker/lite/Dockerfile.percona57

@@ -17,7 +17,7 @@
 #       ensure images contain the right binaries.
 
 # Use a temporary layer for the build stage.
-ARG bootstrap_version=3
+ARG bootstrap_version=4
 ARG image="vitess/bootstrap:${bootstrap_version}-percona57"
 
 FROM "${image}" AS builder

docker/lite/Dockerfile.percona80

@@ -17,7 +17,7 @@
 #       ensure images contain the right binaries.
 
 # Use a temporary layer for the build stage.
-ARG bootstrap_version=3
+ARG bootstrap_version=4
 ARG image="vitess/bootstrap:${bootstrap_version}-percona80"
 
 FROM "${image}" AS builder

docker/lite/Dockerfile.testing

@@ -17,7 +17,7 @@
 #       ensure images contain the right binaries.
 
 # Use a temporary layer for the build stage.
-ARG bootstrap_version=3
+ARG bootstrap_version=4
 ARG image="vitess/bootstrap:${bootstrap_version}-mysql57"
 
 FROM "${image}" AS builder

docker/lite/Dockerfile.ubi7.mysql57

@@ -17,7 +17,7 @@
 #       ensure images contain the right binaries.
 
 # Use a temporary layer for the build stage.
-ARG bootstrap_version=3
+ARG bootstrap_version=4
 ARG image="vitess/bootstrap:${bootstrap_version}-mysql57"
 
 FROM "${image}" AS builder

docker/lite/Dockerfile.ubi7.mysql80

@@ -17,7 +17,7 @@
 #       ensure images contain the right binaries.
 
 # Use a temporary layer for the build stage.
-ARG bootstrap_version=3
+ARG bootstrap_version=4
 ARG image="vitess/bootstrap:${bootstrap_version}-mysql80"
 
 FROM "${image}" AS builder

docker/lite/Dockerfile.ubi7.percona57

@@ -17,7 +17,7 @@
 #       ensure images contain the right binaries.
 
 # Use a temporary layer for the build stage.
-ARG bootstrap_version=3
+ARG bootstrap_version=4
 ARG image="vitess/bootstrap:${bootstrap_version}-percona57"
 
 FROM "${image}" AS builder

docker/lite/Dockerfile.ubi7.percona80

@@ -17,7 +17,7 @@
 #       ensure images contain the right binaries.
 
 # Use a temporary layer for the build stage.
-ARG bootstrap_version=3
+ARG bootstrap_version=4
 ARG image="vitess/bootstrap:${bootstrap_version}-percona80"
 
 FROM "${image}" AS builder

docker/local/Dockerfile

@@ -1,4 +1,4 @@
-ARG bootstrap_version=3
+ARG bootstrap_version=4
 ARG image="vitess/bootstrap:${bootstrap_version}-common"
 
 FROM "${image}"

docker/vttestserver/Dockerfile.mysql57

@@ -17,7 +17,7 @@
 #       ensure images contain the right binaries.
 
 # Use a temporary layer for the build stage.
-ARG bootstrap_version=3
+ARG bootstrap_version=4
 ARG image="vitess/bootstrap:${bootstrap_version}-mysql57"
 
 FROM "${image}" AS builder

docker/vttestserver/Dockerfile.mysql80

@@ -17,7 +17,7 @@
 #       ensure images contain the right binaries.
 
 # Use a temporary layer for the build stage.
-ARG bootstrap_version=3
+ARG bootstrap_version=4
 ARG image="vitess/bootstrap:${bootstrap_version}-mysql80"
 
 FROM "${image}" AS builder

test.go

@@ -74,7 +74,7 @@ For example:
 // Flags
 var (
 	flavor           = flag.String("flavor", "mysql57", "comma-separated bootstrap flavor(s) to run against (when using Docker mode). Available flavors: all,"+flavors)
-	bootstrapVersion = flag.String("bootstrap-version", "3", "the version identifier to use for the docker images")
+	bootstrapVersion = flag.String("bootstrap-version", "4", "the version identifier to use for the docker images")
 	runCount         = flag.Int("runs", 1, "run each test this many times")
 	retryMax         = flag.Int("retry", 3, "max number of retries, to detect flaky tests")
 	logPass          = flag.Bool("log-pass", false, "log test output even if it passes")

test/templates/dockerfile.tpl

@@ -1,4 +1,4 @@
-ARG bootstrap_version=3
+ARG bootstrap_version=4
 ARG image="vitess/bootstrap:${bootstrap_version}-{{.Platform}}"
 
 FROM "${image}"
@@ -44,4 +44,4 @@ RUN make tools
 {{end}}
 
 # sleep for 50 minutes
-CMD sleep 3000
+CMD sleep 3000

tools/wget-retry

@@ -0,0 +1,126 @@
+#!/usr/bin/env perl
+
+# AUTHORITY
+# DATE
+# DIST
+# VERSION
+
+use strict;
+use warnings;
+
+use Getopt::Long;
+
+my %Opts = (
+    tries => 0,
+    waitretry => 10,
+    exit_statuses => [
+        split /\s*,\s*/,
+        (defined $ENV{WGET_RETRY_EXIT_STATUSES} ?
+             $ENV{WGET_RETRY_EXIT_STATUSES} : "1,3,4,5,6,7,8")],
+);
+
+my @ORIG_ARGV = @ARGV;
+Getopt::Long::Configure(
+    'bundling', 'pass_through', 'no_auto_abbrev', 'permute');
+GetOptions(
+    'help|h|?' => sub {
+        print <<'_';
+Usage: wget-retry [options] <url>...
+
+Options:
+  --help, -h, -?   Show this message and exit.
+  --version        Show program version and exit.
+
+All the other options will be passed to wget.
+
+See manpage for more detailed documentation.
+_
+        exit 0;
+    },
+    'version' => sub {
+        no warnings 'once';
+        print "wget-retry version ", ($main::VERSION || "dev"),
+            ($main::DATE ? " ($main::DATE)" : ""), "\n";
+        exit 0;
+    },
+
+    'tries|t=i' => \$Opts{tries},
+    'waitretry=i' => \$Opts{waitretry},
+);
+
+my $wget_cmd = $ENV{WGET_RETRY_WGET_CMD} || "wget";
+
+my $retries = 0;
+while (1) {
+    system {$wget_cmd} $wget_cmd, @ORIG_ARGV;
+    last unless $?;
+    my $exit_code = $? >> 8;
+    if (grep { $exit_code == $_ } @{ $Opts{exit_statuses} }) {
+        $retries++;
+        if ($Opts{tries} == 0 || $retries <= $Opts{tries}) {
+            warn "wget-retry: $wget_cmd exit-code is $exit_code, retrying ($retries) after $Opts{waitretry} second(s) ...\n";
+            sleep $Opts{waitretry};
+            next;
+        } else {
+            warn "wget-retry: $wget_cmd exit-code is $exit_code, won't retry anymore, exiting\n";
+            exit $exit_code;
+        }
+    } else {
+        exit $exit_code;
+    }
+}
+
+# ABSTRACT: Wget wrapper to retry harder
+# PODNAME:
+
+=head1 SYNOPSIS
+
+Use like you would use B<wget>:
+
+ % wget-retry -c -t0 https://example.com/url1 ...
+
+
+=head1 DESCRIPTION
+
+By default, B<wget> doesn't retry harder; only upon disconnection in the middle
+of downloading (with C<-t>/C<--tries>, e.g. C<-t 0>) and on connection refused
+(with C<--retry-connrefused>) but not on other network failures, e.g. DNS
+resolution failure (which can happen sometimes).
+
+This wrapper runs B<wget> then checks its exit code. If exit code indicates
+network failure (4) it will re-run wget.
+
+The number of tries is unlimited, or from the C<-t> (<--tries>) option. The
+number of seconds to wait before each try is 10 seconds or from the
+C<--waitretry> option.
+
+
+=head1 OPTIONS
+
+=head2 --help
+
+Shortcuts: -h, -?.
+
+=head2 --version
+
+
+=head1 ENVIRONMENT
+
+=head2 WGET_RETRY_EXIT_STATUSES
+
+A comma-separated list of exit statuses to retry. For example, C<1,3,4,5,6,7,8>
+means generic error (1), file I/O error (3), network failure (4), SSL
+verification failure (5), username/password authentication failure (6), protocol
+errors (7), as well as error response from server (8) will make wget-retry rerun
+wget. The default is 1,3,4,5,6,7,8. For more details on wget exit statuses, see
+the wget's manpage.
+
+=head2 WGET_RETRY_WGET_CMD
+
+String. Wget command to use. Defaults to C<wget>. Can be used to chain several
+wrappers together.
+
+
+=head1 SEE ALSO
+
+B<wget>.