docker
/
compose-cli
зеркало из https://github.com/docker/compose-cli.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Merge pull request #812 from docker/barbarian 

Introduce use of EFS Access Point to mount filesystems as volumes
This commit is contained in:
Nicolas De loof 2020-10-20 16:23:50 +02:00 коммит произвёл GitHub
Родитель 1c211550fc 6fa30284d9
Коммит 75a5a0f205
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
13 изменённых файлов: 356 добавлений и 131 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

2
ecs/autoscaling.go
Просмотреть файл

@ -62,7 +62,7 @@ func (b *ecsAPIService) createAutoscalingPolicy(project *types.Project, resource
}
// Why isn't this just the service ARN ?????
resourceID := cloudformation.Join("/", []string{"service", resources.cluster, cloudformation.GetAtt(serviceResourceName(service.Name), "Name")})
resourceID := cloudformation.Join("/", []string{"service", resources.cluster.ID(), cloudformation.GetAtt(serviceResourceName(service.Name), "Name")})
target := fmt.Sprintf("%sScalableTarget", normalizeResourceName(service.Name))
template.Resources[target] = &applicationautoscaling.ScalableTarget{

10
ecs/aws.go
Просмотреть файл

@ -35,11 +35,11 @@ const (
// API hides aws-go-sdk into a simpler, focussed API subset
type API interface {
CheckRequirements(ctx context.Context, region string) error
ClusterExists(ctx context.Context, name string) (bool, error)
ResolveCluster(ctx context.Context, nameOrArn string) (awsResource, error)
CreateCluster(ctx context.Context, name string) (string, error)
CheckVPC(ctx context.Context, vpcID string) error
GetDefaultVPC(ctx context.Context) (string, error)
GetSubNets(ctx context.Context, vpcID string) ([]string, error)
GetSubNets(ctx context.Context, vpcID string) ([]awsResource, error)
GetRoleArn(ctx context.Context, name string) (string, error)
StackExists(ctx context.Context, name string) (bool, error)
CreateStack(ctx context.Context, name string, template []byte) error
@ -66,14 +66,14 @@ type API interface {
getURLWithPortMapping(ctx context.Context, targetGroupArns []string) ([]compose.PortPublisher, error)
ListTasks(ctx context.Context, cluster string, family string) ([]string, error)
GetPublicIPs(ctx context.Context, interfaces ...string) (map[string]string, error)
LoadBalancerType(ctx context.Context, arn string) (string, error)
ResolveLoadBalancer(ctx context.Context, nameOrArn string) (awsResource, string, error)
GetLoadBalancerURL(ctx context.Context, arn string) (string, error)
GetParameter(ctx context.Context, name string) (string, error)
SecurityGroupExists(ctx context.Context, sg string) (bool, error)
DeleteCapacityProvider(ctx context.Context, arn string) error
DeleteAutoscalingGroup(ctx context.Context, arn string) error
FileSystemExists(ctx context.Context, id string) (bool, error)
FindFileSystem(ctx context.Context, tags map[string]string) (string, error)
ResolveFileSystem(ctx context.Context, id string) (awsResource, error)
FindFileSystem(ctx context.Context, tags map[string]string) (awsResource, error)
CreateFileSystem(ctx context.Context, tags map[string]string) (string, error)
DeleteFileSystem(ctx context.Context, id string) error
}

133
ecs/awsResources.go
Просмотреть файл

@ -38,13 +38,13 @@ import (
// awsResources hold the AWS component being used or created to support services definition
type awsResources struct {
vpc string
subnets []string
cluster string
loadBalancer string
vpc string // shouldn't this also be an awsResource ?
subnets []awsResource
cluster awsResource
loadBalancer awsResource
loadBalancerType string
securityGroups map[string]string
filesystems map[string]string
filesystems map[string]awsResource
}
func (r *awsResources) serviceSecurityGroups(service types.ServiceConfig) []string {
@ -63,6 +63,63 @@ func (r *awsResources) allSecurityGroups() []string {
return securityGroups
}
func (r *awsResources) subnetsIDs() []string {
var ids []string
for _, r := range r.subnets {
ids = append(ids, r.ID())
}
return ids
}
// awsResource is abstract representation for any (existing or future) AWS resource that we can refer both by ID or full ARN
type awsResource interface {
ARN() string
ID() string
}
// existingAWSResource hold references to an existing AWS component
type existingAWSResource struct {
arn string
id string
}
func (r existingAWSResource) ARN() string {
return r.arn
}
func (r existingAWSResource) ID() string {
return r.id
}
// cloudformationResource hold references to a future AWS resource managed by CloudFormation
// to be used by CloudFormation resources where Ref returns the Amazon Resource ID
type cloudformationResource struct {
logicalName string
}
func (r cloudformationResource) ARN() string {
return cloudformation.GetAtt(r.logicalName, "Arn")
}
func (r cloudformationResource) ID() string {
return cloudformation.Ref(r.logicalName)
}
// cloudformationARNResource hold references to a future AWS resource managed by CloudFormation
// to be used by CloudFormation resources where Ref returns the Amazon Resource Name (ARN)
type cloudformationARNResource struct {
logicalName string
nameProperty string
}
func (r cloudformationARNResource) ARN() string {
return cloudformation.Ref(r.logicalName)
}
func (r cloudformationARNResource) ID() string {
return cloudformation.GetAtt(r.logicalName, r.nameProperty)
}
// parse look into compose project for configured resource to use, and check they are valid
func (b *ecsAPIService) parse(ctx context.Context, project *types.Project, template *cloudformation.Template) (awsResources, error) {
r := awsResources{}
@ -90,28 +147,28 @@ func (b *ecsAPIService) parse(ctx context.Context, project *types.Project, templ
return r, nil
}
func (b *ecsAPIService) parseClusterExtension(ctx context.Context, project *types.Project, template *cloudformation.Template) (string, error) {
func (b *ecsAPIService) parseClusterExtension(ctx context.Context, project *types.Project, template *cloudformation.Template) (awsResource, error) {
if x, ok := project.Extensions[extensionCluster]; ok {
cluster := x.(string)
ok, err := b.aws.ClusterExists(ctx, cluster)
nameOrArn := x.(string) // can be name _or_ ARN.
cluster, err := b.aws.ResolveCluster(ctx, nameOrArn)
if err != nil {
return "", err
return nil, err
}
if !ok {
return "", errors.Wrapf(errdefs.ErrNotFound, "cluster %q does not exist", cluster)
return nil, errors.Wrapf(errdefs.ErrNotFound, "cluster %q does not exist", cluster)
}
template.Metadata["Cluster"] = cluster
template.Metadata["Cluster"] = cluster.ARN()
return cluster, nil
}
return "", nil
return nil, nil
}
func (b *ecsAPIService) parseVPCExtension(ctx context.Context, project *types.Project) (string, []string, error) {
func (b *ecsAPIService) parseVPCExtension(ctx context.Context, project *types.Project) (string, []awsResource, error) {
var vpc string
if x, ok := project.Extensions[extensionVPC]; ok {
vpc = x.(string)
err := b.aws.CheckVPC(ctx, vpc)
vpcID := x.(string)
err := b.aws.CheckVPC(ctx, vpcID)
if err != nil {
return "", nil, err
}
@ -134,22 +191,22 @@ func (b *ecsAPIService) parseVPCExtension(ctx context.Context, project *types.Pr
return vpc, subNets, nil
}
func (b *ecsAPIService) parseLoadBalancerExtension(ctx context.Context, project *types.Project) (string, string, error) {
func (b *ecsAPIService) parseLoadBalancerExtension(ctx context.Context, project *types.Project) (awsResource, string, error) {
if x, ok := project.Extensions[extensionLoadBalancer]; ok {
loadBalancer := x.(string)
loadBalancerType, err := b.aws.LoadBalancerType(ctx, loadBalancer)
nameOrArn := x.(string)
loadBalancer, loadBalancerType, err := b.aws.ResolveLoadBalancer(ctx, nameOrArn)
if err != nil {
return "", "", err
return nil, "", err
}
required := getRequiredLoadBalancerType(project)
if loadBalancerType != required {
return "", "", fmt.Errorf("load balancer %s is of type %s, project require a %s", loadBalancer, loadBalancerType, required)
return nil, "", fmt.Errorf("load balancer %q is of type %s, project require a %s", nameOrArn, loadBalancerType, required)
}
return loadBalancer, loadBalancerType, nil
return loadBalancer, loadBalancerType, err
}
return "", "", nil
return nil, "", nil
}
func (b *ecsAPIService) parseExternalNetworks(ctx context.Context, project *types.Project) (map[string]string, error) {
@ -179,18 +236,15 @@ func (b *ecsAPIService) parseExternalNetworks(ctx context.Context, project *type
return securityGroups, nil
}
func (b *ecsAPIService) parseExternalVolumes(ctx context.Context, project *types.Project) (map[string]string, error) {
filesystems := make(map[string]string, len(project.Volumes))
func (b *ecsAPIService) parseExternalVolumes(ctx context.Context, project *types.Project) (map[string]awsResource, error) {
filesystems := make(map[string]awsResource, len(project.Volumes))
for name, vol := range project.Volumes {
if vol.External.External {
exists, err := b.aws.FileSystemExists(ctx, vol.Name)
arn, err := b.aws.ResolveFileSystem(ctx, vol.Name)
if err != nil {
return nil, err
}
if !exists {
return nil, errors.Wrapf(errdefs.ErrNotFound, "EFS file system %q doesn't exist", vol.Name)
}
filesystems[name] = vol.Name
filesystems[name] = arn
continue
}
@ -199,12 +253,12 @@ func (b *ecsAPIService) parseExternalVolumes(ctx context.Context, project *types
compose.ProjectTag: project.Name,
compose.VolumeTag: name,
}
id, err := b.aws.FindFileSystem(ctx, tags)
fileSystem, err := b.aws.FindFileSystem(ctx, tags)
if err != nil {
return nil, err
}
if id != "" {
filesystems[name] = id
if fileSystem != nil {
filesystems[name] = fileSystem
}
}
return filesystems, nil
@ -223,14 +277,14 @@ func (b *ecsAPIService) ensureResources(resources *awsResources, project *types.
}
func (b *ecsAPIService) ensureCluster(r *awsResources, project *types.Project, template *cloudformation.Template) {
if r.cluster != "" {
if r.cluster != nil {
return
}
template.Resources["Cluster"] = &ecs.Cluster{
ClusterName: project.Name,
Tags: projectTags(project),
}
r.cluster = cloudformation.Ref("Cluster")
r.cluster = cloudformationResource{logicalName: "Cluster"}
}
func (b *ecsAPIService) ensureNetworks(r *awsResources, project *types.Project, template *cloudformation.Template) {
@ -319,13 +373,13 @@ func (b *ecsAPIService) ensureVolumes(r *awsResources, project *types.Project, t
ThroughputMode: throughputMode,
AWSCloudFormationDeletionPolicy: "Retain",
}
r.filesystems[name] = cloudformation.Ref(n)
r.filesystems[name] = cloudformationResource{logicalName: n}
}
return nil
}
func (b *ecsAPIService) ensureLoadBalancer(r *awsResources, project *types.Project, template *cloudformation.Template) {
if r.loadBalancer != "" {
if r.loadBalancer != nil {
return
}
if allServices(project.Services, func(it types.ServiceConfig) bool {
@ -345,11 +399,14 @@ func (b *ecsAPIService) ensureLoadBalancer(r *awsResources, project *types.Proje
template.Resources["LoadBalancer"] = &elasticloadbalancingv2.LoadBalancer{
Scheme: elbv2.LoadBalancerSchemeEnumInternetFacing,
SecurityGroups: securityGroups,
Subnets: r.subnets,
Subnets: r.subnetsIDs(),
Tags: projectTags(project),
Type: balancerType,
}
r.loadBalancer = cloudformation.Ref("LoadBalancer")
r.loadBalancer = cloudformationARNResource{
logicalName: "LoadBalancer",
nameProperty: "LoadBalancerName",
}
r.loadBalancerType = balancerType
}

86
ecs/aws_mock.go
Просмотреть файл

@ -6,12 +6,13 @@ package ecs
import (
context "context"
reflect "reflect"
cloudformation "github.com/aws/aws-sdk-go/service/cloudformation"
ecs "github.com/aws/aws-sdk-go/service/ecs"
compose "github.com/docker/compose-cli/api/compose"
secrets "github.com/docker/compose-cli/api/secrets"
gomock "github.com/golang/mock/gomock"
reflect "reflect"
)
// MockAPI is a mock of API interface
@ -65,21 +66,6 @@ func (mr *MockAPIMockRecorder) CheckVPC(arg0, arg1 interface{}) *gomock.Call {
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CheckVPC", reflect.TypeOf((*MockAPI)(nil).CheckVPC), arg0, arg1)
}
// ClusterExists mocks base method
func (m *MockAPI) ClusterExists(arg0 context.Context, arg1 string) (bool, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "ClusterExists", arg0, arg1)
ret0, _ := ret[0].(bool)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// ClusterExists indicates an expected call of ClusterExists
func (mr *MockAPIMockRecorder) ClusterExists(arg0, arg1 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClusterExists", reflect.TypeOf((*MockAPI)(nil).ClusterExists), arg0, arg1)
}
// CreateChangeSet mocks base method
func (m *MockAPI) CreateChangeSet(arg0 context.Context, arg1 string, arg2 []byte) (string, error) {
m.ctrl.T.Helper()
@ -254,26 +240,11 @@ func (mr *MockAPIMockRecorder) DescribeStackEvents(arg0, arg1 interface{}) *gomo
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DescribeStackEvents", reflect.TypeOf((*MockAPI)(nil).DescribeStackEvents), arg0, arg1)
}
// FileSystemExists mocks base method
func (m *MockAPI) FileSystemExists(arg0 context.Context, arg1 string) (bool, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "FileSystemExists", arg0, arg1)
ret0, _ := ret[0].(bool)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// FileSystemExists indicates an expected call of FileSystemExists
func (mr *MockAPIMockRecorder) FileSystemExists(arg0, arg1 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FileSystemExists", reflect.TypeOf((*MockAPI)(nil).FileSystemExists), arg0, arg1)
}
// FindFileSystem mocks base method
func (m *MockAPI) FindFileSystem(arg0 context.Context, arg1 map[string]string) (string, error) {
func (m *MockAPI) FindFileSystem(arg0 context.Context, arg1 map[string]string) (awsResource, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "FindFileSystem", arg0, arg1)
ret0, _ := ret[0].(string)
ret0, _ := ret[0].(awsResource)
ret1, _ := ret[1].(error)
return ret0, ret1
}
@ -439,10 +410,10 @@ func (mr *MockAPIMockRecorder) GetStackID(arg0, arg1 interface{}) *gomock.Call {
}
// GetSubNets mocks base method
func (m *MockAPI) GetSubNets(arg0 context.Context, arg1 string) ([]string, error) {
func (m *MockAPI) GetSubNets(arg0 context.Context, arg1 string) ([]awsResource, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "GetSubNets", arg0, arg1)
ret0, _ := ret[0].([]string)
ret0, _ := ret[0].([]awsResource)
ret1, _ := ret[1].(error)
return ret0, ret1
}
@ -573,19 +544,50 @@ func (mr *MockAPIMockRecorder) ListTasks(arg0, arg1, arg2 interface{}) *gomock.C
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListTasks", reflect.TypeOf((*MockAPI)(nil).ListTasks), arg0, arg1, arg2)
}
// LoadBalancerType mocks base method
func (m *MockAPI) LoadBalancerType(arg0 context.Context, arg1 string) (string, error) {
// ResolveCluster mocks base method
func (m *MockAPI) ResolveCluster(arg0 context.Context, arg1 string) (awsResource, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "LoadBalancerType", arg0, arg1)
ret0, _ := ret[0].(string)
ret := m.ctrl.Call(m, "ResolveCluster", arg0, arg1)
ret0, _ := ret[0].(awsResource)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// LoadBalancerType indicates an expected call of LoadBalancerType
func (mr *MockAPIMockRecorder) LoadBalancerType(arg0, arg1 interface{}) *gomock.Call {
// ResolveCluster indicates an expected call of ResolveCluster
func (mr *MockAPIMockRecorder) ResolveCluster(arg0, arg1 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LoadBalancerType", reflect.TypeOf((*MockAPI)(nil).LoadBalancerType), arg0, arg1)
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResolveCluster", reflect.TypeOf((*MockAPI)(nil).ResolveCluster), arg0, arg1)
}
// ResolveFileSystem mocks base method
func (m *MockAPI) ResolveFileSystem(arg0 context.Context, arg1 string) (awsResource, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "ResolveFileSystem", arg0, arg1)
ret0, _ := ret[0].(awsResource)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// ResolveFileSystem indicates an expected call of ResolveFileSystem
func (mr *MockAPIMockRecorder) ResolveFileSystem(arg0, arg1 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResolveFileSystem", reflect.TypeOf((*MockAPI)(nil).ResolveFileSystem), arg0, arg1)
}
// ResolveLoadBalancer mocks base method
func (m *MockAPI) ResolveLoadBalancer(arg0 context.Context, arg1 string) (awsResource, string, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "ResolveLoadBalancer", arg0, arg1)
ret0, _ := ret[0].(awsResource)
ret1, _ := ret[1].(string)
ret2, _ := ret[2].(error)
return ret0, ret1, ret2
}
// ResolveLoadBalancer indicates an expected call of ResolveLoadBalancer
func (mr *MockAPIMockRecorder) ResolveLoadBalancer(arg0, arg1 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResolveLoadBalancer", reflect.TypeOf((*MockAPI)(nil).ResolveLoadBalancer), arg0, arg1)
}
// SecurityGroupExists mocks base method

6
ecs/backend.go
Просмотреть файл

@ -19,9 +19,6 @@ package ecs
import (
"context"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/docker/compose-cli/api/compose"
"github.com/docker/compose-cli/api/containers"
"github.com/docker/compose-cli/api/resources"
@ -32,6 +29,9 @@ import (
"github.com/docker/compose-cli/context/cloud"
"github.com/docker/compose-cli/context/store"
"github.com/docker/compose-cli/errdefs"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
)
const backendType = store.EcsContextType

21
ecs/cloudformation.go
Просмотреть файл

@ -77,6 +77,8 @@ func (b *ecsAPIService) convert(ctx context.Context, project *types.Project) (*c
b.createNFSMountTarget(project, resources, template)
b.createAccessPoints(project, resources, template)
for _, service := range project.Services {
err := b.createService(project, service, template, resources)
if err != nil {
@ -96,7 +98,7 @@ func (b *ecsAPIService) convert(ctx context.Context, project *types.Project) (*c
func (b *ecsAPIService) createService(project *types.Project, service types.ServiceConfig, template *cloudformation.Template, resources awsResources) error {
taskExecutionRole := b.createTaskExecutionRole(project, service, template)
taskRole := b.createTaskRole(project, service, template)
taskRole := b.createTaskRole(project, service, template, resources)
definition, err := b.createTaskDefinition(project, service, resources)
if err != nil {
@ -166,7 +168,7 @@ func (b *ecsAPIService) createService(project *types.Project, service types.Serv
template.Resources[serviceResourceName(service.Name)] = &ecs.Service{
AWSCloudFormationDependsOn: dependsOn,
Cluster: resources.cluster,
Cluster: resources.cluster.ARN(),
DesiredCount: desiredCount,
DeploymentController: &ecs.Service_DeploymentController{
Type: ecsapi.DeploymentControllerTypeEcs,
@ -182,7 +184,7 @@ func (b *ecsAPIService) createService(project *types.Project, service types.Serv
AwsvpcConfiguration: &ecs.Service_AwsVpcConfiguration{
AssignPublicIp: assignPublicIP,
SecurityGroups: resources.serviceSecurityGroups(service),
Subnets: resources.subnets,
Subnets: resources.subnetsIDs(),
},
},
PlatformVersion: platformVersion,
@ -287,7 +289,7 @@ func computeRollingUpdateLimits(service types.ServiceConfig) (int, int, error) {
func (b *ecsAPIService) createListener(service types.ServiceConfig, port types.ServicePortConfig,
template *cloudformation.Template,
targetGroupName string, loadBalancerARN string, protocol string) string {
targetGroupName string, loadBalancer awsResource, protocol string) string {
listenerName := fmt.Sprintf(
"%s%s%dListener",
normalizeResourceName(service.Name),
@ -309,7 +311,7 @@ func (b *ecsAPIService) createListener(service types.ServiceConfig, port types.S
Type: elbv2.ActionTypeEnumForward,
},
},
LoadBalancerArn: loadBalancerARN,
LoadBalancerArn: loadBalancer.ARN(),
Protocol: protocol,
Port: int(port.Target),
}
@ -375,14 +377,21 @@ func (b *ecsAPIService) createTaskExecutionRole(project *types.Project, service
return taskExecutionRole
}
func (b *ecsAPIService) createTaskRole(project *types.Project, service types.ServiceConfig, template *cloudformation.Template) string {
func (b *ecsAPIService) createTaskRole(project *types.Project, service types.ServiceConfig, template *cloudformation.Template, resources awsResources) string {
taskRole := fmt.Sprintf("%sTaskRole", normalizeResourceName(service.Name))
rolePolicies := []iam.Role_Policy{}
if roles, ok := service.Extensions[extensionRole]; ok {
rolePolicies = append(rolePolicies, iam.Role_Policy{
PolicyName: fmt.Sprintf("%s%sPolicy", normalizeResourceName(project.Name), normalizeResourceName(service.Name)),
PolicyDocument: roles,
})
}
for _, vol := range service.Volumes {
rolePolicies = append(rolePolicies, iam.Role_Policy{
PolicyName: fmt.Sprintf("%s%sVolumeMountPolicy", normalizeResourceName(project.Name), normalizeResourceName(service.Name)),
PolicyDocument: volumeMountPolicyDocument(vol.Source, resources.filesystems[vol.Source].ARN()),
})
}
managedPolicies := []string{}
if v, ok := service.Extensions[extensionManagedPolicies]; ok {
for _, s := range v.([]interface{}) {

35
ecs/cloudformation_test.go
Просмотреть файл

@ -365,7 +365,7 @@ volumes:
external: true
name: fs-123abc
`, useDefaultVPC, func(m *MockAPIMockRecorder) {
m.FileSystemExists(gomock.Any(), "fs-123abc").Return(true, nil)
m.ResolveFileSystem(gomock.Any(), "fs-123abc").Return(existingAWSResource{id: "fs-123abc"}, nil)
})
s := template.Resources["DbdataNFSMountTargetOnSubnet1"].(*efs.MountTarget)
assert.Check(t, s != nil)
@ -393,7 +393,7 @@ volumes:
m.FindFileSystem(gomock.Any(), map[string]string{
compose.ProjectTag: t.Name(),
compose.VolumeTag: "db-data",
}).Return("", nil)
}).Return(nil, nil)
})
n := volumeResourceName("db-data")
f := template.Resources[n].(*efs.FileSystem)
@ -409,6 +409,25 @@ volumes:
assert.Equal(t, s.FileSystemId, cloudformation.Ref(n)) //nolint:staticcheck
}
func TestCreateAccessPoint(t *testing.T) {
template := convertYaml(t, `
services:
test:
image: nginx
volumes:
db-data:
driver_opts:
uid: 1002
gid: 1002
`, useDefaultVPC, func(m *MockAPIMockRecorder) {
m.FindFileSystem(gomock.Any(), gomock.Any()).Return(nil, nil)
})
a := template.Resources["DbdataAccessPoint"].(*efs.AccessPoint)
assert.Check(t, a != nil)
assert.Equal(t, a.PosixUser.Uid, "1002") //nolint:staticcheck
assert.Equal(t, a.PosixUser.Gid, "1002") //nolint:staticcheck
}
func TestReusePreviousVolume(t *testing.T) {
template := convertYaml(t, `
services:
@ -420,7 +439,7 @@ volumes:
m.FindFileSystem(gomock.Any(), map[string]string{
compose.ProjectTag: t.Name(),
compose.VolumeTag: "db-data",
}).Return("fs-123abc", nil)
}).Return(existingAWSResource{id: "fs-123abc"}, nil)
})
s := template.Resources["DbdataNFSMountTargetOnSubnet1"].(*efs.MountTarget)
assert.Check(t, s != nil)
@ -497,7 +516,10 @@ services:
test:
image: nginx
`, useDefaultVPC, func(m *MockAPIMockRecorder) {
m.ClusterExists(gomock.Any(), "arn:aws:ecs:region:account:cluster/name").Return(true, nil)
m.ResolveCluster(gomock.Any(), "arn:aws:ecs:region:account:cluster/name").Return(existingAWSResource{
arn: "arn:aws:ecs:region:account:cluster/name",
id: "name",
}, nil)
})
assert.Equal(t, template.Metadata["Cluster"], "arn:aws:ecs:region:account:cluster/name")
}
@ -546,7 +568,10 @@ func getMainContainer(def *ecs.TaskDefinition, t *testing.T) ecs.TaskDefinition_
func useDefaultVPC(m *MockAPIMockRecorder) {
m.GetDefaultVPC(gomock.Any()).Return("vpc-123", nil)
m.GetSubNets(gomock.Any(), "vpc-123").Return([]string{"subnet1", "subnet2"}, nil)
m.GetSubNets(gomock.Any(), "vpc-123").Return([]awsResource{
existingAWSResource{id: "subnet1"},
existingAWSResource{id: "subnet2"},
}, nil)
}
func useGPU(m *MockAPIMockRecorder) {

10
ecs/convert.go
Просмотреть файл

@ -81,11 +81,15 @@ func (b *ecsAPIService) createTaskDefinition(project *types.Project, service typ
}
for _, v := range service.Volumes {
volume := project.Volumes[v.Source]
n := fmt.Sprintf("%sAccessPoint", normalizeResourceName(v.Source))
volumes = append(volumes, ecs.TaskDefinition_Volume{
EFSVolumeConfiguration: &ecs.TaskDefinition_EFSVolumeConfiguration{
FilesystemId: resources.filesystems[v.Source],
RootDirectory: volume.DriverOpts["root_directory"],
AuthorizationConfig: &ecs.TaskDefinition_AuthorizationConfig{
AccessPointId: cloudformation.Ref(n),
IAM: "ENABLED",
},
FilesystemId: resources.filesystems[v.Source].ID(),
TransitEncryption: "ENABLED",
},
Name: v.Source,
})

2
ecs/ec2.go
Просмотреть файл

@ -83,7 +83,7 @@ func (b *ecsAPIService) createCapacityProvider(ctx context.Context, project *typ
LaunchConfigurationName: cloudformation.Ref("LaunchConfiguration"),
MaxSize: "10", //TODO
MinSize: "1",
VPCZoneIdentifier: resources.subnets,
VPCZoneIdentifier: resources.subnetsIDs(),
}
userData := base64.StdEncoding.EncodeToString([]byte(

37
ecs/iam.go
Просмотреть файл

@ -16,6 +16,12 @@
package ecs
import (
"fmt"
"github.com/awslabs/goformation/v4/cloudformation"
)
const (
ecsTaskExecutionPolicy = "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy"
ecrReadOnlyPolicy = "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly"
@ -49,7 +55,31 @@ func policyDocument(service string) PolicyDocument {
},
},
}
}
func volumeMountPolicyDocument(volume string, filesystem string) PolicyDocument {
ap := fmt.Sprintf("%sAccessPoint", normalizeResourceName(volume))
return PolicyDocument{
Version: "2012-10-17", // https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements_version.html
Statement: []PolicyStatement{
{
Effect: "Allow",
Resource: []string{
filesystem,
},
Action: []string{
"elasticfilesystem:ClientMount",
"elasticfilesystem:ClientWrite",
"elasticfilesystem:ClientRootAccess",
},
Condition: Condition{
StringEquals: map[string]string{
"elasticfilesystem:AccessPointArn": cloudformation.Ref(ap),
},
},
},
},
}
}
// PolicyDocument describes an IAM policy document
@ -65,9 +95,16 @@ type PolicyStatement struct {
Action []string `json:",omitempty"`
Principal PolicyPrincipal `json:",omitempty"`
Resource []string `json:",omitempty"`
Condition Condition `json:",omitempty"`
}
// PolicyPrincipal describes an IAM policy principal
type PolicyPrincipal struct {
Service string `json:",omitempty"`
}
// Condition is the map of all conditions in the statement entry.
type Condition struct {
StringEquals map[string]string `json:",omitempty"`
Bool map[string]string `json:",omitempty"`
}

78
ecs/sdk.go
Просмотреть файл

@ -29,6 +29,7 @@ import (
"github.com/docker/compose-cli/internal"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/arn"
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/autoscaling"
@ -107,15 +108,22 @@ func (s sdk) CheckRequirements(ctx context.Context, region string) error {
return nil
}
func (s sdk) ClusterExists(ctx context.Context, name string) (bool, error) {
logrus.Debug("CheckRequirements if cluster was already created: ", name)
func (s sdk) ResolveCluster(ctx context.Context, nameOrArn string) (awsResource, error) {
logrus.Debug("CheckRequirements if cluster was already created: ", nameOrArn)
clusters, err := s.ECS.DescribeClustersWithContext(ctx, &ecs.DescribeClustersInput{
Clusters: []*string{aws.String(name)},
Clusters: []*string{aws.String(nameOrArn)},
})
if err != nil {
return false, err
return nil, err
}
return len(clusters.Clusters) > 0, nil
if len(clusters.Clusters) == 0 {
return nil, errors.Wrapf(errdefs.ErrNotFound, "cluster %q does not exist", nameOrArn)
}
it := clusters.Clusters[0]
return existingAWSResource{
arn: aws.StringValue(it.ClusterArn),
id: aws.StringValue(it.ClusterName),
}, nil
}
func (s sdk) CreateCluster(ctx context.Context, name string) (string, error) {
@ -139,7 +147,7 @@ func (s sdk) CheckVPC(ctx context.Context, vpcID string) error {
if !*output.EnableDnsSupport.Value {
return fmt.Errorf("VPC %q doesn't have DNS resolution enabled", vpcID)
}
return err
return nil
}
func (s sdk) GetDefaultVPC(ctx context.Context) (string, error) {
@ -161,7 +169,7 @@ func (s sdk) GetDefaultVPC(ctx context.Context) (string, error) {
return *vpcs.Vpcs[0].VpcId, nil
}
func (s sdk) GetSubNets(ctx context.Context, vpcID string) ([]string, error) {
func (s sdk) GetSubNets(ctx context.Context, vpcID string) ([]awsResource, error) {
logrus.Debug("Retrieve SubNets")
subnets, err := s.EC2.DescribeSubnetsWithContext(ctx, &ec2.DescribeSubnetsInput{
DryRun: nil,
@ -176,9 +184,12 @@ func (s sdk) GetSubNets(ctx context.Context, vpcID string) ([]string, error) {
return nil, err
}
ids := []string{}
ids := []awsResource{}
for _, subnet := range subnets.Subnets {
ids = append(ids, *subnet.SubnetId)
ids = append(ids, existingAWSResource{
arn: aws.StringValue(subnet.SubnetArn),
id: aws.StringValue(subnet.SubnetId),
})
}
return ids, nil
}
@ -784,18 +795,31 @@ func (s sdk) GetPublicIPs(ctx context.Context, interfaces ...string) (map[string
return publicIPs, nil
}
func (s sdk) LoadBalancerType(ctx context.Context, arn string) (string, error) {
logrus.Debug("Check if LoadBalancer exists: ", arn)
func (s sdk) ResolveLoadBalancer(ctx context.Context, nameOrarn string) (awsResource, string, error) {
logrus.Debug("Check if LoadBalancer exists: ", nameOrarn)
var arns []*string
var names []*string
if arn.IsARN(nameOrarn) {
arns = append(arns, aws.String(nameOrarn))
} else {
names = append(names, aws.String(nameOrarn))
}
lbs, err := s.ELB.DescribeLoadBalancersWithContext(ctx, &elbv2.DescribeLoadBalancersInput{
LoadBalancerArns: []*string{aws.String(arn)},
LoadBalancerArns: arns,
Names: names,
})
if err != nil {
return "", err
return nil, "", err
}
if len(lbs.LoadBalancers) == 0 {
return "", fmt.Errorf("load balancer does not exist: %s", arn)
return nil, "", errors.Wrapf(errdefs.ErrNotFound, "load balancer %q does not exist", nameOrarn)
}
return aws.StringValue(lbs.LoadBalancers[0].Type), nil
it := lbs.LoadBalancers[0]
return existingAWSResource{
arn: aws.StringValue(it.LoadBalancerArn),
id: aws.StringValue(it.LoadBalancerName),
}, aws.StringValue(it.Type), nil
}
func (s sdk) GetLoadBalancerURL(ctx context.Context, arn string) (string, error) {
@ -863,32 +887,42 @@ func (s sdk) DeleteAutoscalingGroup(ctx context.Context, arn string) error {
return err
}
func (s sdk) FileSystemExists(ctx context.Context, id string) (bool, error) {
func (s sdk) ResolveFileSystem(ctx context.Context, id string) (awsResource, error) {
desc, err := s.EFS.DescribeFileSystemsWithContext(ctx, &efs.DescribeFileSystemsInput{
FileSystemId: aws.String(id),
})
if err != nil {
return false, err
return nil, err
}
return len(desc.FileSystems) > 0, nil
if len(desc.FileSystems) == 0 {
return nil, errors.Wrapf(errdefs.ErrNotFound, "EFS file system %q doesn't exist", id)
}
it := desc.FileSystems[0]
return existingAWSResource{
arn: aws.StringValue(it.FileSystemArn),
id: aws.StringValue(it.FileSystemId),
}, nil
}
func (s sdk) FindFileSystem(ctx context.Context, tags map[string]string) (string, error) {
func (s sdk) FindFileSystem(ctx context.Context, tags map[string]string) (awsResource, error) {
var token *string
for {
desc, err := s.EFS.DescribeFileSystemsWithContext(ctx, &efs.DescribeFileSystemsInput{
Marker: token,
})
if err != nil {
return "", err
return nil, err
}
for _, filesystem := range desc.FileSystems {
if containsAll(filesystem.Tags, tags) {
return aws.StringValue(filesystem.FileSystemId), nil
return existingAWSResource{
arn: aws.StringValue(filesystem.FileSystemArn),
id: aws.StringValue(filesystem.FileSystemId),
}, nil
}
}
if desc.NextMarker == token {
return "", nil
return nil, nil
}
token = desc.NextMarker
}

6
ecs/testdata/simple/simple-cloudformation-conversion.golden поставляемый
Просмотреть файл

@ -98,7 +98,10 @@
],
"Properties": {
"Cluster": {
"Ref": "Cluster"
"Fn::GetAtt": [
"Cluster",
"Arn"
]
},
"DeploymentConfiguration": {
"MaximumPercent": 200,
@ -299,6 +302,7 @@
"Action": [
"sts:AssumeRole"
],
"Condition": {},
"Effect": "Allow",
"Principal": {
"Service": "ecs-tasks.amazonaws.com"

61
ecs/volumes.go
Просмотреть файл

@ -19,6 +19,8 @@ package ecs
import (
"fmt"
"github.com/docker/compose-cli/api/compose"
"github.com/awslabs/goformation/v4/cloudformation"
"github.com/awslabs/goformation/v4/cloudformation/efs"
"github.com/compose-spec/compose-go/types"
@ -27,11 +29,11 @@ import (
func (b *ecsAPIService) createNFSMountTarget(project *types.Project, resources awsResources, template *cloudformation.Template) {
for volume := range project.Volumes {
for _, subnet := range resources.subnets {
name := fmt.Sprintf("%sNFSMountTargetOn%s", normalizeResourceName(volume), normalizeResourceName(subnet))
name := fmt.Sprintf("%sNFSMountTargetOn%s", normalizeResourceName(volume), normalizeResourceName(subnet.ID()))
template.Resources[name] = &efs.MountTarget{
FileSystemId: resources.filesystems[volume],
FileSystemId: resources.filesystems[volume].ID(),
SecurityGroups: resources.allSecurityGroups(),
SubnetId: subnet,
SubnetId: subnet.ID(),
}
}
}
@ -40,7 +42,58 @@ func (b *ecsAPIService) createNFSMountTarget(project *types.Project, resources a
func (b *ecsAPIService) mountTargets(volume string, resources awsResources) []string {
var refs []string
for _, subnet := range resources.subnets {
refs = append(refs, fmt.Sprintf("%sNFSMountTargetOn%s", normalizeResourceName(volume), normalizeResourceName(subnet)))
refs = append(refs, fmt.Sprintf("%sNFSMountTargetOn%s", normalizeResourceName(volume), normalizeResourceName(subnet.ID())))
}
return refs
}
func (b *ecsAPIService) createAccessPoints(project *types.Project, r awsResources, template *cloudformation.Template) {
for name, volume := range project.Volumes {
n := fmt.Sprintf("%sAccessPoint", normalizeResourceName(name))
uid := volume.DriverOpts["uid"]
gid := volume.DriverOpts["gid"]
permissions := volume.DriverOpts["permissions"]
path := volume.DriverOpts["root_directory"]
ap := efs.AccessPoint{
AccessPointTags: []efs.AccessPoint_AccessPointTag{
{
Key: compose.ProjectTag,
Value: project.Name,
},
{
Key: compose.VolumeTag,
Value: name,
},
{
Key: "Name",
Value: fmt.Sprintf("%s_%s", project.Name, name),
},
},
FileSystemId: r.filesystems[name].ID(),
}
if uid != "" {
ap.PosixUser = &efs.AccessPoint_PosixUser{
Uid: uid,
Gid: gid,
}
}
if path != "" {
root := efs.AccessPoint_RootDirectory{
Path: path,
}
ap.RootDirectory = &root
if uid != "" {
root.CreationInfo = &efs.AccessPoint_CreationInfo{
OwnerUid: uid,
OwnerGid: gid,
Permissions: permissions,
}
}
}
template.Resources[n] = &ap
}
}