Add wget replacement go-downloader (#6630)
This commit is contained in:
Daniel McIlvaney
GitHub
Родитель
742489e5dd
Коммит
89675cb7e7
|
|
@ -61,12 +61,12 @@ $(BUILD_SRPMS_DIR): $(STATUS_FLAGS_DIR)/build_srpms.flag
|
|||
@echo Finished updating $@
|
||||
|
||||
ifeq ($(DOWNLOAD_SRPMS),y)
|
||||
$(STATUS_FLAGS_DIR)/build_srpms.flag: $(local_specs) $(local_spec_dirs) $(local_spec_sources) $(SPECS_DIR)
|
||||
$(STATUS_FLAGS_DIR)/build_srpms.flag: $(local_specs) $(local_spec_dirs) $(local_spec_sources) $(SPECS_DIR) $(go-downloader)
|
||||
for spec in $(local_specs); do \
|
||||
spec_file=$${spec} && \
|
||||
srpm_file=$$(rpmspec -q $${spec_file} --srpm --define='with_check 1' --define='dist $(DIST_TAG)' --queryformat %{NAME}-%{VERSION}-%{RELEASE}.src.rpm) && \
|
||||
for url in $(SRPM_URL_LIST); do \
|
||||
wget $${url}/$${srpm_file} \
|
||||
$(go-downloader) $${url}/$${srpm_file} \
|
||||
-O $(BUILD_SRPMS_DIR)/$${srpm_file} \
|
||||
--no-verbose \
|
||||
$(if $(TLS_CERT),--certificate=$(TLS_CERT)) \
|
||||
|
|
|
|||
|
|
@ -184,7 +184,7 @@ $(raw_toolchain): $(toolchain_files)
|
|||
# - ALLOW_TOOLCHAIN_DOWNLOAD_FAIL = n: This flag explicitly disables partial toolchain rehydration from repos
|
||||
# In these cases, we just create empty files for each possible rehydrated RPM.
|
||||
ifeq ($(strip $(INCREMENTAL_TOOLCHAIN))$(strip $(REBUILD_TOOLCHAIN))$(strip $(ALLOW_TOOLCHAIN_DOWNLOAD_FAIL)),yyy)
|
||||
$(toolchain_rpms_rehydrated): $(TOOLCHAIN_MANIFEST)
|
||||
$(toolchain_rpms_rehydrated): $(TOOLCHAIN_MANIFEST) $(go-downloader)
|
||||
@rpm_filename="$(notdir $@)" && \
|
||||
rpm_dir="$(dir $@)" && \
|
||||
log_file="$(toolchain_downloads_logs_dir)/$$rpm_filename.log" && \
|
||||
|
|
@ -192,10 +192,10 @@ $(toolchain_rpms_rehydrated): $(TOOLCHAIN_MANIFEST)
|
|||
mkdir -p $$rpm_dir && \
|
||||
cd $$rpm_dir && \
|
||||
for url in $(PACKAGE_URL_LIST); do \
|
||||
wget -nv --no-clobber $$url/$$rpm_filename \
|
||||
$(go-downloader) --no-verbose --no-clobber $$url/$$rpm_filename \
|
||||
$(if $(TLS_CERT),--certificate=$(TLS_CERT)) \
|
||||
$(if $(TLS_KEY),--private-key=$(TLS_KEY)) \
|
||||
-a $$log_file && \
|
||||
--log-file $$log_file && \
|
||||
echo "Downloaded toolchain RPM: $$rpm_filename" >> $$log_file && \
|
||||
echo "$$rpm_filename" >> $(toolchain_downloads_manifest) | tee -a "$$log_file" && \
|
||||
touch $@ && \
|
||||
|
|
@ -303,7 +303,7 @@ $(toolchain_rpms): $(TOOLCHAIN_MANIFEST) $(STATUS_FLAGS_DIR)/toolchain_local_tem
|
|||
|
||||
# No archive was selected, so download from online package server instead. All packages must be available for this step to succeed.
|
||||
else
|
||||
$(toolchain_rpms): $(TOOLCHAIN_MANIFEST) $(depend_REBUILD_TOOLCHAIN)
|
||||
$(toolchain_rpms): $(TOOLCHAIN_MANIFEST) $(depend_REBUILD_TOOLCHAIN) $(go-downloader)
|
||||
@rpm_filename="$(notdir $@)" && \
|
||||
rpm_dir="$(dir $@)" && \
|
||||
log_file="$(toolchain_downloads_logs_dir)/$$rpm_filename.log" && \
|
||||
|
|
@ -311,10 +311,10 @@ $(toolchain_rpms): $(TOOLCHAIN_MANIFEST) $(depend_REBUILD_TOOLCHAIN)
|
|||
mkdir -p $$rpm_dir && \
|
||||
cd $$rpm_dir && \
|
||||
for url in $(PACKAGE_URL_LIST); do \
|
||||
wget -nv --no-clobber $$url/$$rpm_filename \
|
||||
$(go-downloader) --no-verbose --no-clobber $$url/$$rpm_filename \
|
||||
$(if $(TLS_CERT),--certificate=$(TLS_CERT)) \
|
||||
$(if $(TLS_KEY),--private-key=$(TLS_KEY)) \
|
||||
-a $$log_file && \
|
||||
--log-file $$log_file && \
|
||||
echo "Downloaded toolchain RPM: $$rpm_filename" >> $$log_file && \
|
||||
touch $@ && \
|
||||
break; \
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@ go_tool_list = \
|
|||
bldtracker \
|
||||
boilerplate \
|
||||
depsearch \
|
||||
downloader \
|
||||
grapher \
|
||||
graphpkgfetcher \
|
||||
graphanalytics \
|
||||
|
|
|
|||
|
|
@ -0,0 +1,139 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
// A very simple replacement for wget.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/microsoft/CBL-Mariner/toolkit/tools/internal/exe"
|
||||
"github.com/microsoft/CBL-Mariner/toolkit/tools/internal/file"
|
||||
"github.com/microsoft/CBL-Mariner/toolkit/tools/internal/logger"
|
||||
"github.com/microsoft/CBL-Mariner/toolkit/tools/internal/network"
|
||||
"github.com/microsoft/CBL-Mariner/toolkit/tools/internal/retry"
|
||||
"github.com/sirupsen/logrus"
|
||||
|
||||
"gopkg.in/alecthomas/kingpin.v2"
|
||||
)
|
||||
|
||||
var (
|
||||
app = kingpin.New("downloader", "Download files to a location")
|
||||
|
||||
logFile = exe.LogFileFlag(app)
|
||||
logLevel = exe.LogLevelFlag(app)
|
||||
noClobber = app.Flag("no-clobber", "Do not overwrite existing files").Bool()
|
||||
noVerbose = app.Flag("no-verbose", "Do not print verbose output").Bool()
|
||||
|
||||
caCertFile = app.Flag("ca-certificate", "Root certificate authority to use when downloading files.").String()
|
||||
tlsClientCert = app.Flag("certificate", "TLS client certificate to use when downloading files.").String()
|
||||
tlsClientKey = app.Flag("private-key", "TLS client key to use when downloading files.").String()
|
||||
|
||||
dstFile = app.Flag("output-file", "Destination file to download to").Short('O').String()
|
||||
prefixDir = app.Flag("directory-prefix", "Directory to download to").Short('P').String()
|
||||
srcUrl = app.Arg("url", "URL to download").Required().String()
|
||||
)
|
||||
|
||||
func main() {
|
||||
app.Version(exe.ToolkitVersion)
|
||||
kingpin.MustParse(app.Parse(os.Args[1:]))
|
||||
logger.InitBestEffort(*logFile, *logLevel)
|
||||
if *noVerbose {
|
||||
logger.Log.SetLevel(logrus.WarnLevel)
|
||||
}
|
||||
|
||||
if *noClobber {
|
||||
exists, err := file.PathExists(*dstFile)
|
||||
if err != nil {
|
||||
logger.Log.Fatalf("Failed to check if file (%s) exists. Error:\n%s", *dstFile, err)
|
||||
}
|
||||
if exists {
|
||||
logger.Log.Infof("File (%s) already exists, skipping download.", *dstFile)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
caCerts, err := x509.SystemCertPool()
|
||||
if err != nil {
|
||||
logger.Log.Fatalf("Failed to load system certificate pool. Error:\n%s", err)
|
||||
}
|
||||
if *caCertFile != "" {
|
||||
newCACert, err := os.ReadFile(*caCertFile)
|
||||
if err != nil {
|
||||
logger.Log.Fatalf("Invalid CA certificate (%s), Error:\n%s", *caCertFile, err)
|
||||
}
|
||||
|
||||
caCerts.AppendCertsFromPEM(newCACert)
|
||||
}
|
||||
|
||||
var tlsCerts []tls.Certificate
|
||||
if *tlsClientCert != "" && *tlsClientKey != "" {
|
||||
cert, err := tls.LoadX509KeyPair(*tlsClientCert, *tlsClientKey)
|
||||
if err != nil {
|
||||
logger.Log.Fatalf("Invalid TLS client key pair (%s) (%s), Error:\n%s", *tlsClientCert, *tlsClientKey, err)
|
||||
}
|
||||
|
||||
tlsCerts = append(tlsCerts, cert)
|
||||
}
|
||||
|
||||
// dst may be empty, in which case the file will be downloaded to the current directory. Generate dst from src's basename.
|
||||
// The url may include query strings which should be stripped.
|
||||
if *dstFile != "" && *prefixDir != "" {
|
||||
logger.Log.Fatalf("Cannot specify both --output-file and --directory-prefix")
|
||||
}
|
||||
if *dstFile == "" {
|
||||
// strip query strings from url
|
||||
u, err := url.Parse(*srcUrl)
|
||||
if err != nil {
|
||||
logger.Log.Fatalf("Invalid URL (%s), Error:\n%s", *srcUrl, err)
|
||||
}
|
||||
*dstFile = filepath.Base(u.Path)
|
||||
if *prefixDir != "" {
|
||||
*dstFile = filepath.Join(*prefixDir, *dstFile)
|
||||
}
|
||||
}
|
||||
|
||||
err = downloadFile(*srcUrl, *dstFile, caCerts, tlsCerts)
|
||||
if err != nil {
|
||||
logger.Log.Fatalf("Failed to download (%s) to (%s). Error:\n%s", *srcUrl, *dstFile, err)
|
||||
}
|
||||
}
|
||||
|
||||
func downloadFile(srcUrl, dstFile string, caCerts *x509.CertPool, tlsCerts []tls.Certificate) (err error) {
|
||||
const (
|
||||
// With 6 attempts, initial delay of 1 second, and a backoff factor of 3.0 the total time spent retrying will be
|
||||
// 1 + 3 + 9 + 27 + 81 = 121 seconds.
|
||||
downloadRetryAttempts = 6
|
||||
failureBackoffBase = 3.0
|
||||
downloadRetryDuration = time.Second
|
||||
)
|
||||
var noCancel chan struct{} = nil
|
||||
|
||||
retryNum := 1
|
||||
_, err = retry.RunWithExpBackoff(func() error {
|
||||
netErr := network.DownloadFile(srcUrl, dstFile, caCerts, tlsCerts)
|
||||
if netErr != nil {
|
||||
// Check if the error contains the string "invalid response: 404", we should print a warning in that case so the
|
||||
// sees it even if we are running with --no-verbose.
|
||||
if netErr.Error() == "invalid response: 404" {
|
||||
logger.Log.Warnf("Attempt %d/%d: Failed to download '%s' with error: '%s'", retryNum, downloadRetryAttempts, srcUrl, netErr)
|
||||
} else {
|
||||
logger.Log.Infof("Attempt %d/%d: Failed to download '%s' with error: '%s'", retryNum, downloadRetryAttempts, srcUrl, netErr)
|
||||
}
|
||||
}
|
||||
retryNum++
|
||||
return netErr
|
||||
}, downloadRetryAttempts, downloadRetryDuration, failureBackoffBase, noCancel)
|
||||
|
||||
if err != nil {
|
||||
err = fmt.Errorf("failed to download (%s) to (%s). Error:\n%w", srcUrl, dstFile, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
Загрузка…
Ссылка в новой задаче