Add wget replacement go-downloader (#6630)

toolkit/scripts/srpm_pack.mk

@@ -61,12 +61,12 @@ $(BUILD_SRPMS_DIR): $(STATUS_FLAGS_DIR)/build_srpms.flag
 	@echo Finished updating $@
 
 ifeq ($(DOWNLOAD_SRPMS),y)
-$(STATUS_FLAGS_DIR)/build_srpms.flag: $(local_specs) $(local_spec_dirs) $(local_spec_sources) $(SPECS_DIR)
+$(STATUS_FLAGS_DIR)/build_srpms.flag: $(local_specs) $(local_spec_dirs) $(local_spec_sources) $(SPECS_DIR) $(go-downloader)
 	for spec in $(local_specs); do \
 		spec_file=$${spec} && \
 		srpm_file=$$(rpmspec -q $${spec_file} --srpm --define='with_check 1' --define='dist $(DIST_TAG)' --queryformat %{NAME}-%{VERSION}-%{RELEASE}.src.rpm) && \
 		for url in $(SRPM_URL_LIST); do \
-			wget $${url}/$${srpm_file} \
+			$(go-downloader) $${url}/$${srpm_file} \
 				-O $(BUILD_SRPMS_DIR)/$${srpm_file} \
 				--no-verbose \
 				$(if $(TLS_CERT),--certificate=$(TLS_CERT)) \

toolkit/scripts/toolchain.mk

@@ -184,7 +184,7 @@ $(raw_toolchain): $(toolchain_files)
 # - ALLOW_TOOLCHAIN_DOWNLOAD_FAIL = n: This flag explicitly disables partial toolchain rehydration from repos
 # In these cases, we just create empty files for each possible rehydrated RPM.
 ifeq ($(strip $(INCREMENTAL_TOOLCHAIN))$(strip $(REBUILD_TOOLCHAIN))$(strip $(ALLOW_TOOLCHAIN_DOWNLOAD_FAIL)),yyy)
-$(toolchain_rpms_rehydrated): $(TOOLCHAIN_MANIFEST)
+$(toolchain_rpms_rehydrated): $(TOOLCHAIN_MANIFEST) $(go-downloader)
 	@rpm_filename="$(notdir $@)" && \
 	rpm_dir="$(dir $@)" && \
 	log_file="$(toolchain_downloads_logs_dir)/$$rpm_filename.log" && \
@@ -192,10 +192,10 @@ $(toolchain_rpms_rehydrated): $(TOOLCHAIN_MANIFEST)
 	mkdir -p $$rpm_dir && \
 	cd $$rpm_dir && \
 	for url in $(PACKAGE_URL_LIST); do \
-		wget -nv --no-clobber $$url/$$rpm_filename \
+		$(go-downloader) --no-verbose --no-clobber $$url/$$rpm_filename \
 			$(if $(TLS_CERT),--certificate=$(TLS_CERT)) \
 			$(if $(TLS_KEY),--private-key=$(TLS_KEY)) \
-			-a $$log_file && \
+			--log-file $$log_file && \
 		echo "Downloaded toolchain RPM: $$rpm_filename" >> $$log_file && \
 		echo "$$rpm_filename" >> $(toolchain_downloads_manifest) | tee -a "$$log_file" && \
 		touch $@ && \
@@ -303,7 +303,7 @@ $(toolchain_rpms): $(TOOLCHAIN_MANIFEST) $(STATUS_FLAGS_DIR)/toolchain_local_tem
 
 # No archive was selected, so download from online package server instead. All packages must be available for this step to succeed.
 else
-$(toolchain_rpms): $(TOOLCHAIN_MANIFEST) $(depend_REBUILD_TOOLCHAIN)
+$(toolchain_rpms): $(TOOLCHAIN_MANIFEST) $(depend_REBUILD_TOOLCHAIN) $(go-downloader)
 	@rpm_filename="$(notdir $@)" && \
 	rpm_dir="$(dir $@)" && \
 	log_file="$(toolchain_downloads_logs_dir)/$$rpm_filename.log" && \
@@ -311,10 +311,10 @@ $(toolchain_rpms): $(TOOLCHAIN_MANIFEST) $(depend_REBUILD_TOOLCHAIN)
 	mkdir -p $$rpm_dir && \
 	cd $$rpm_dir && \
 	for url in $(PACKAGE_URL_LIST); do \
-		wget -nv --no-clobber $$url/$$rpm_filename \
+		$(go-downloader) --no-verbose --no-clobber $$url/$$rpm_filename \
 			$(if $(TLS_CERT),--certificate=$(TLS_CERT)) \
 			$(if $(TLS_KEY),--private-key=$(TLS_KEY)) \
-			-a $$log_file && \
+			--log-file $$log_file && \
 		echo "Downloaded toolchain RPM: $$rpm_filename" >> $$log_file && \
 		touch $@ && \
 		break; \

toolkit/scripts/tools.mk

@@ -32,6 +32,7 @@ go_tool_list = \
 	bldtracker \
 	boilerplate \
 	depsearch \
+	downloader \
 	grapher \
 	graphpkgfetcher \
 	graphanalytics \

toolkit/tools/downloader/downloader.go

@@ -0,0 +1,139 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+// A very simple replacement for wget.
+
+package main
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+	"net/url"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/microsoft/CBL-Mariner/toolkit/tools/internal/exe"
+	"github.com/microsoft/CBL-Mariner/toolkit/tools/internal/file"
+	"github.com/microsoft/CBL-Mariner/toolkit/tools/internal/logger"
+	"github.com/microsoft/CBL-Mariner/toolkit/tools/internal/network"
+	"github.com/microsoft/CBL-Mariner/toolkit/tools/internal/retry"
+	"github.com/sirupsen/logrus"
+
+	"gopkg.in/alecthomas/kingpin.v2"
+)
+
+var (
+	app = kingpin.New("downloader", "Download files to a location")
+
+	logFile   = exe.LogFileFlag(app)
+	logLevel  = exe.LogLevelFlag(app)
+	noClobber = app.Flag("no-clobber", "Do not overwrite existing files").Bool()
+	noVerbose = app.Flag("no-verbose", "Do not print verbose output").Bool()
+
+	caCertFile    = app.Flag("ca-certificate", "Root certificate authority to use when downloading files.").String()
+	tlsClientCert = app.Flag("certificate", "TLS client certificate to use when downloading files.").String()
+	tlsClientKey  = app.Flag("private-key", "TLS client key to use when downloading files.").String()
+
+	dstFile   = app.Flag("output-file", "Destination file to download to").Short('O').String()
+	prefixDir = app.Flag("directory-prefix", "Directory to download to").Short('P').String()
+	srcUrl    = app.Arg("url", "URL to download").Required().String()
+)
+
+func main() {
+	app.Version(exe.ToolkitVersion)
+	kingpin.MustParse(app.Parse(os.Args[1:]))
+	logger.InitBestEffort(*logFile, *logLevel)
+	if *noVerbose {
+		logger.Log.SetLevel(logrus.WarnLevel)
+	}
+
+	if *noClobber {
+		exists, err := file.PathExists(*dstFile)
+		if err != nil {
+			logger.Log.Fatalf("Failed to check if file (%s) exists. Error:\n%s", *dstFile, err)
+		}
+		if exists {
+			logger.Log.Infof("File (%s) already exists, skipping download.", *dstFile)
+			return
+		}
+	}
+
+	caCerts, err := x509.SystemCertPool()
+	if err != nil {
+		logger.Log.Fatalf("Failed to load system certificate pool. Error:\n%s", err)
+	}
+	if *caCertFile != "" {
+		newCACert, err := os.ReadFile(*caCertFile)
+		if err != nil {
+			logger.Log.Fatalf("Invalid CA certificate (%s), Error:\n%s", *caCertFile, err)
+		}
+
+		caCerts.AppendCertsFromPEM(newCACert)
+	}
+
+	var tlsCerts []tls.Certificate
+	if *tlsClientCert != "" && *tlsClientKey != "" {
+		cert, err := tls.LoadX509KeyPair(*tlsClientCert, *tlsClientKey)
+		if err != nil {
+			logger.Log.Fatalf("Invalid TLS client key pair (%s) (%s), Error:\n%s", *tlsClientCert, *tlsClientKey, err)
+		}
+
+		tlsCerts = append(tlsCerts, cert)
+	}
+
+	// dst may be empty, in which case the file will be downloaded to the current directory. Generate dst from src's basename.
+	// The url may include query strings which should be stripped.
+	if *dstFile != "" && *prefixDir != "" {
+		logger.Log.Fatalf("Cannot specify both --output-file and --directory-prefix")
+	}
+	if *dstFile == "" {
+		// strip query strings from url
+		u, err := url.Parse(*srcUrl)
+		if err != nil {
+			logger.Log.Fatalf("Invalid URL (%s), Error:\n%s", *srcUrl, err)
+		}
+		*dstFile = filepath.Base(u.Path)
+		if *prefixDir != "" {
+			*dstFile = filepath.Join(*prefixDir, *dstFile)
+		}
+	}
+
+	err = downloadFile(*srcUrl, *dstFile, caCerts, tlsCerts)
+	if err != nil {
+		logger.Log.Fatalf("Failed to download (%s) to (%s). Error:\n%s", *srcUrl, *dstFile, err)
+	}
+}
+
+func downloadFile(srcUrl, dstFile string, caCerts *x509.CertPool, tlsCerts []tls.Certificate) (err error) {
+	const (
+		// With 6 attempts, initial delay of 1 second, and a backoff factor of 3.0 the total time spent retrying will be
+		// 1 + 3 + 9 + 27 + 81 = 121 seconds.
+		downloadRetryAttempts = 6
+		failureBackoffBase    = 3.0
+		downloadRetryDuration = time.Second
+	)
+	var noCancel chan struct{} = nil
+
+	retryNum := 1
+	_, err = retry.RunWithExpBackoff(func() error {
+		netErr := network.DownloadFile(srcUrl, dstFile, caCerts, tlsCerts)
+		if netErr != nil {
+			// Check if the error contains the string "invalid response: 404", we should print a warning in that case so the
+			// sees it even if we are running with --no-verbose.
+			if netErr.Error() == "invalid response: 404" {
+				logger.Log.Warnf("Attempt %d/%d: Failed to download '%s' with error: '%s'", retryNum, downloadRetryAttempts, srcUrl, netErr)
+			} else {
+				logger.Log.Infof("Attempt %d/%d: Failed to download '%s' with error: '%s'", retryNum, downloadRetryAttempts, srcUrl, netErr)
+			}
+		}
+		retryNum++
+		return netErr
+	}, downloadRetryAttempts, downloadRetryDuration, failureBackoffBase, noCancel)
+
+	if err != nil {
+		err = fmt.Errorf("failed to download (%s) to (%s). Error:\n%w", srcUrl, dstFile, err)
+	}
+	return
+}