github
/
vitess-gh
зеркало из https://github.com/github/vitess-gh.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Incorporating feedback from pull request. 

Giving Env paramaters for secrets, not just CLI.
Rename prefix to be storage_root i.a.w what we do for s3 and gcs

Signed-off-by: Dan Kozlowski <koz@planetscale.com>
This commit is contained in:
Dan Kozlowski 2020-01-10 11:32:12 -08:00
Родитель d9d0ad6c94
Коммит 7c84818955
3 изменённых файлов: 53 добавлений и 15 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

3
go.mod
Просмотреть файл

@ -5,6 +5,7 @@ go 1.12
require (
cloud.google.com/go v0.45.1
github.com/Azure/azure-storage-blob-go v0.8.0
github.com/Azure/go-autorest/autorest/adal v0.8.1 // indirect
github.com/Bowery/prompt v0.0.0-20190419144237-972d0ceb96f5 // indirect
github.com/GeertJohan/go.rice v1.0.0
github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 // indirect
@ -79,7 +80,7 @@ require (
github.com/ugorji/go v1.1.7 // indirect
github.com/uudashr/gocognit v1.0.1 // indirect
github.com/z-division/go-zookeeper v0.0.0-20190128072838-6d7457066b9b
golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd
golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413
golang.org/x/lint v0.0.0-20190409202823-959b441ac422
golang.org/x/net v0.0.0-20190926025831-c00fd9afed17
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45

18
go.sum
Просмотреть файл

@ -16,6 +16,20 @@ github.com/Azure/azure-pipeline-go v0.2.1 h1:OLBdZJ3yvOn2MezlWvbrBMTEUQC72zAftRZ
github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4=
github.com/Azure/azure-storage-blob-go v0.8.0 h1:53qhf0Oxa0nOjgbDeeYPUeyiNmafAFEY95rZLK0Tj6o=
github.com/Azure/azure-storage-blob-go v0.8.0/go.mod h1:lPI3aLPpuLTeUwh1sViKXFxwl2B6teiRqI0deQUvsw0=
github.com/Azure/go-autorest/autorest v0.9.0 h1:MRvx8gncNaXJqOoLmhNjUAKh33JJF8LyxPhomEtOsjs=
github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI=
github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0=
github.com/Azure/go-autorest/autorest/adal v0.8.1 h1:pZdL8o72rK+avFWl+p9nE8RWi1JInZrWJYlnpfXJwHk=
github.com/Azure/go-autorest/autorest/adal v0.8.1/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q=
github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA=
github.com/Azure/go-autorest/autorest/date v0.2.0 h1:yW+Zlqf26583pE43KhfnhFcdmSWlm5Ew6bxipnr/tbM=
github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g=
github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0=
github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0=
github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM=
github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc=
github.com/Azure/go-autorest/tracing v0.5.0 h1:TRn4WjSnkcSy5AEG3pnbtFSwNtwzjr4VYyQflFE619k=
github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk=
github.com/Bowery/prompt v0.0.0-20190419144237-972d0ceb96f5 h1:7tNlRGC3pUEPKS3DwgX5L0s+cBloaq/JBoi9ceN1MCM=
github.com/Bowery/prompt v0.0.0-20190419144237-972d0ceb96f5/go.mod h1:4/6eNcqZ09BZ9wLK3tZOjBA1nDj+B0728nlX5YRlSmQ=
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
@ -697,8 +711,8 @@ golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472/go.mod h1:yigFU9vqHzYiE8U
golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd h1:GGJVjV8waZKRHrgwvtH66z9ZGVurTD1MT0n1Bb+q4aM=
golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413 h1:ULYEB3JvPRE/IfO+9uO7vKV/xzVTO7XPAwm8xbf4w2g=
golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=

47
go/vt/mysqlctl/azblobbackupstorage/azblob.go
Просмотреть файл

@ -24,6 +24,7 @@ import (
"fmt"
"io"
"net/url"
"os"
"strings"
"sync"
"time"
@ -36,18 +37,18 @@ import (
var (
// This is the account name
accountName = flag.String("azblob_backup_account_name", "", "Azure Account Name for Backups")
accountName = flag.String("azblob_backup_account_name", "", "Azure Account Name for Backups, alternative environment paramater is VITESS_AZBLOB_ACCOUNT_NAME")
// This is the private access key
accountKey = flag.String("azblob_backup_account_key", "", "Azure Account Key")
accountKey = flag.String("azblob_backup_account_key", "", "Azure Account Key, alternative environment paramater is VITESS_AZBLOB_ACCOUNT_KEY")
// This is the name of the container that will store the backups
containerName = flag.String("azblob_backup_contain_name", "", "Azure Blob Contain Name")
containerName = flag.String("azblob_backup_container_name", "", "Azure Blob Container Name")
// This is an optional previx to prepend to all files
prefix = flag.String("azblob_backup_prefix", "", "Azure Blob prefix")
storageRoot = flag.String("azblob_backup_storage_root", "", "Azure Blob storage root")
azBlobParallelism = flag.Int("azblob_parallelism", 1, "Azure blob operation parallelism (requires extra memory when increased)")
azBlobParallelism = flag.Int("azblob_backup_parallelism", 1, "Azure blob operation parallelism (requires extra memory when increased)")
)
const (
@ -55,7 +56,25 @@ const (
delimiter = "/"
)
// Return a Shared credential from the available credential sources.
// We will use credentials in the following order
// 1. Direct Command Line Flag (azblob_backup_account_name, azblob_backup_account_key)
// 2. Environment Paramaters
func azCredentials() (*azblob.SharedKeyCredential, error) {
actName := *accountName
if len(actName) == 0 {
// Check the Environmental Value
actName = os.Getenv("VITESS_AZBLOB_ACCOUNT_NAME")
}
actKey := *accountKey
if len(actKey) == 0 {
actKey = os.Getenv("VITESS_AZBLOB_ACCOUNT_NAME")
}
if len(actName) == 0 || len(actKey) == 0 {
return nil, fmt.Errorf("can not get Account Credentials from CLI or Environment paramaters")
}
return azblob.NewSharedKeyCredential(*accountName, *accountKey)
}
@ -152,7 +171,7 @@ func (bh *AZBlobBackupHandle) ReadFile(ctx context.Context, filename string) (io
return nil, fmt.Errorf("ReadFile cannot be called on read-write backup")
}
obj := objName(bh.dir, bh.name, filename)
obj := objName(bh.dir, filename)
containerURL, err := bh.bs.containerURL()
if err != nil {
return nil, err
@ -166,7 +185,10 @@ func (bh *AZBlobBackupHandle) ReadFile(ctx context.Context, filename string) (io
return resp.Body(azblob.RetryReaderOptions{
MaxRetryRequests: defaultRetryCount,
NotifyFailedRead: func(failureCount int, lastError error, offset int64, count int64, willRetry bool) {
bh.errors.RecordError(lastError)
log.Infof("ReadFile: [azblob] container: %s, directory: %s, filename: %s, error: %v", *containerName, objName(bh.dir, ""), filename, lastError)
if !willRetry {
bh.errors.RecordError(lastError)
}
},
TreatEarlyCloseAsError: true,
}), nil
@ -187,7 +209,7 @@ func (bs *AZBlobBackupStorage) containerURL() (*azblob.ContainerURL, error) {
// ListBackups implements BackupStorage.
func (bs *AZBlobBackupStorage) ListBackups(ctx context.Context, dir string) ([]backupstorage.BackupHandle, error) {
log.Infof("ListBackups: [azblob] container: %v, prefix: %v", *containerName, dir)
log.Infof("ListBackups: [azblob] container: %s, directory: %v", *containerName, objName(dir, ""))
containerURL, err := bs.containerURL()
if err != nil {
@ -200,6 +222,7 @@ func (bs *AZBlobBackupStorage) ListBackups(ctx context.Context, dir string) ([]b
var subdirs []string
for marker := (azblob.Marker{}); marker.NotDone(); {
// This returns Blobs in sorted order so we don't need to sort them a second time
resp, err := containerURL.ListBlobsHierarchySegment(ctx, marker, delimiter, azblob.ListBlobsSegmentOptions{
Prefix: searchPrefix,
MaxResults: 0,
@ -242,7 +265,7 @@ func (bs *AZBlobBackupStorage) StartBackup(ctx context.Context, dir, name string
// RemoveBackup implements BackupStorage.
func (bs *AZBlobBackupStorage) RemoveBackup(ctx context.Context, dir, name string) error {
log.Infof("ListBackups: [azblob] container: %v, prefix: %v", *containerName, dir)
log.Infof("ListBackups: [azblob] container: %s, directory: %s", *containerName, objName(dir, ""))
containerURL, err := bs.containerURL()
if err != nil {
@ -284,10 +307,10 @@ func (bs *AZBlobBackupStorage) Close() error {
// objName joins path parts into an object name.
// Unlike path.Join, it doesn't collapse ".." or strip trailing slashes.
// It also adds the value of the -gcs_backup_storage_root flag if set.
// It also adds the value of the -azblob_backup_storage_root flag if set.
func objName(parts ...string) string {
if *prefix != "" {
return *prefix + "/" + strings.Join(parts, "/")
if *storageRoot != "" {
return *storageRoot + "/" + strings.Join(parts, "/")
}
return strings.Join(parts, "/")
}