Azure
/
ARO-RP
зеркало из https://github.com/Azure/ARO-RP.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

enable cluster ssh access

This commit is contained in:
Jim Minter 2020-09-26 17:07:39 -05:00
Родитель 1c5b774e38
Коммит cc92374a9d
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 0730CBDA10D1A2D3
4 изменённых файлов: 500 добавлений и 1 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

81
pkg/cluster/deployresources_resources.go
Просмотреть файл

@ -174,6 +174,15 @@ func (m *manager) networkInternalLoadBalancer(installConfig *installconfig.Insta
{
Name: to.StringPtr(m.doc.OpenShiftCluster.Properties.InfraID),
},
{
Name: to.StringPtr("ssh-0"),
},
{
Name: to.StringPtr("ssh-1"),
},
{
Name: to.StringPtr("ssh-2"),
},
},
LoadBalancingRules: &[]mgmtnetwork.LoadBalancingRule{
{
@ -215,6 +224,66 @@ func (m *manager) networkInternalLoadBalancer(installConfig *installconfig.Insta
},
Name: to.StringPtr("sint-v4"),
},
{
LoadBalancingRulePropertiesFormat: &mgmtnetwork.LoadBalancingRulePropertiesFormat{
FrontendIPConfiguration: &mgmtnetwork.SubResource{
ID: to.StringPtr(fmt.Sprintf("[resourceId('Microsoft.Network/loadBalancers/frontendIPConfigurations', '%s-internal', 'internal-lb-ip-v4')]", m.doc.OpenShiftCluster.Properties.InfraID)),
},
BackendAddressPool: &mgmtnetwork.SubResource{
ID: to.StringPtr(fmt.Sprintf("[resourceId('Microsoft.Network/loadBalancers/backendAddressPools', '%s-internal', 'ssh-0')]", m.doc.OpenShiftCluster.Properties.InfraID)),
},
Probe: &mgmtnetwork.SubResource{
ID: to.StringPtr(fmt.Sprintf("[resourceId('Microsoft.Network/loadBalancers/probes', '%s-internal', 'ssh')]", m.doc.OpenShiftCluster.Properties.InfraID)),
},
Protocol: mgmtnetwork.TransportProtocolTCP,
LoadDistribution: mgmtnetwork.LoadDistributionDefault,
FrontendPort: to.Int32Ptr(2200),
BackendPort: to.Int32Ptr(22),
IdleTimeoutInMinutes: to.Int32Ptr(30),
DisableOutboundSnat: to.BoolPtr(true),
},
Name: to.StringPtr("ssh-0"),
},
{
LoadBalancingRulePropertiesFormat: &mgmtnetwork.LoadBalancingRulePropertiesFormat{
FrontendIPConfiguration: &mgmtnetwork.SubResource{
ID: to.StringPtr(fmt.Sprintf("[resourceId('Microsoft.Network/loadBalancers/frontendIPConfigurations', '%s-internal', 'internal-lb-ip-v4')]", m.doc.OpenShiftCluster.Properties.InfraID)),
},
BackendAddressPool: &mgmtnetwork.SubResource{
ID: to.StringPtr(fmt.Sprintf("[resourceId('Microsoft.Network/loadBalancers/backendAddressPools', '%s-internal', 'ssh-1')]", m.doc.OpenShiftCluster.Properties.InfraID)),
},
Probe: &mgmtnetwork.SubResource{
ID: to.StringPtr(fmt.Sprintf("[resourceId('Microsoft.Network/loadBalancers/probes', '%s-internal', 'ssh')]", m.doc.OpenShiftCluster.Properties.InfraID)),
},
Protocol: mgmtnetwork.TransportProtocolTCP,
LoadDistribution: mgmtnetwork.LoadDistributionDefault,
FrontendPort: to.Int32Ptr(2201),
BackendPort: to.Int32Ptr(22),
IdleTimeoutInMinutes: to.Int32Ptr(30),
DisableOutboundSnat: to.BoolPtr(true),
},
Name: to.StringPtr("ssh-1"),
},
{
LoadBalancingRulePropertiesFormat: &mgmtnetwork.LoadBalancingRulePropertiesFormat{
FrontendIPConfiguration: &mgmtnetwork.SubResource{
ID: to.StringPtr(fmt.Sprintf("[resourceId('Microsoft.Network/loadBalancers/frontendIPConfigurations', '%s-internal', 'internal-lb-ip-v4')]", m.doc.OpenShiftCluster.Properties.InfraID)),
},
BackendAddressPool: &mgmtnetwork.SubResource{
ID: to.StringPtr(fmt.Sprintf("[resourceId('Microsoft.Network/loadBalancers/backendAddressPools', '%s-internal', 'ssh-2')]", m.doc.OpenShiftCluster.Properties.InfraID)),
},
Probe: &mgmtnetwork.SubResource{
ID: to.StringPtr(fmt.Sprintf("[resourceId('Microsoft.Network/loadBalancers/probes', '%s-internal', 'ssh')]", m.doc.OpenShiftCluster.Properties.InfraID)),
},
Protocol: mgmtnetwork.TransportProtocolTCP,
LoadDistribution: mgmtnetwork.LoadDistributionDefault,
FrontendPort: to.Int32Ptr(2202),
BackendPort: to.Int32Ptr(22),
IdleTimeoutInMinutes: to.Int32Ptr(30),
DisableOutboundSnat: to.BoolPtr(true),
},
Name: to.StringPtr("ssh-2"),
},
},
Probes: &[]mgmtnetwork.Probe{
{
@ -237,6 +306,15 @@ func (m *manager) networkInternalLoadBalancer(installConfig *installconfig.Insta
},
Name: to.StringPtr("sint-probe"),
},
{
ProbePropertiesFormat: &mgmtnetwork.ProbePropertiesFormat{
Protocol: mgmtnetwork.ProbeProtocolTCP,
Port: to.Int32Ptr(22),
IntervalInSeconds: to.Int32Ptr(5),
NumberOfProbes: to.Int32Ptr(2),
},
Name: to.StringPtr("ssh"),
},
},
},
Name: to.StringPtr(m.doc.OpenShiftCluster.Properties.InfraID + "-internal"),
@ -385,6 +463,9 @@ func (m *manager) networkMasterNICs(installConfig *installconfig.InstallConfig)
{
ID: to.StringPtr(fmt.Sprintf("[resourceId('Microsoft.Network/loadBalancers/backendAddressPools', '%s-internal', '%[1]s')]", m.doc.OpenShiftCluster.Properties.InfraID)),
},
{
ID: to.StringPtr(fmt.Sprintf("[resourceId('Microsoft.Network/loadBalancers/backendAddressPools', '%s-internal', concat('ssh-', copyIndex()))]", m.doc.OpenShiftCluster.Properties.InfraID)),
},
},
Subnet: &mgmtnetwork.Subnet{
ID: to.StringPtr(m.doc.OpenShiftCluster.Properties.MasterProfile.SubnetID),

154
pkg/cluster/fixssh.go Normal file
Просмотреть файл

@ -0,0 +1,154 @@
package cluster
// Copyright (c) Microsoft Corporation.
// Licensed under the Apache License 2.0.
import (
"context"
"fmt"
"strings"
mgmtnetwork "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-07-01/network"
"github.com/Azure/go-autorest/autorest/to"
"github.com/Azure/ARO-RP/pkg/api"
"github.com/Azure/ARO-RP/pkg/util/stringutils"
)
func (m *manager) fixSSH(ctx context.Context) error {
infraID := m.doc.OpenShiftCluster.Properties.InfraID
if infraID == "" {
infraID = "aro"
}
resourceGroup := stringutils.LastTokenByte(m.doc.OpenShiftCluster.Properties.ClusterProfile.ResourceGroupID, '/')
var lbName string
switch m.doc.OpenShiftCluster.Properties.ArchitectureVersion {
case api.ArchitectureVersionV1:
lbName = infraID + "-internal-lb"
case api.ArchitectureVersionV2:
lbName = infraID + "-internal"
default:
return fmt.Errorf("unknown architecture version %d", m.doc.OpenShiftCluster.Properties.ArchitectureVersion)
}
lb, err := m.loadBalancers.Get(ctx, resourceGroup, lbName, "")
if err != nil {
return err
}
changed := updateLB(&lb)
if changed {
m.log.Printf("updating %s", lbName)
err = m.loadBalancers.CreateOrUpdateAndWait(ctx, resourceGroup, lbName, lb)
if err != nil {
return err
}
}
for i := 0; i < 3; i++ {
nicName := fmt.Sprintf("%s-master%d-nic", infraID, i)
nic, err := m.interfaces.Get(ctx, resourceGroup, nicName, "")
if err != nil {
return err
}
changed = updateNIC(&nic, &lb, i)
if changed {
m.log.Printf("updating %s", nicName)
err = m.interfaces.CreateOrUpdateAndWait(ctx, resourceGroup, nicName, nic)
if err != nil {
return err
}
}
}
return nil
}
func updateNIC(nic *mgmtnetwork.Interface, lb *mgmtnetwork.LoadBalancer, i int) bool {
id := fmt.Sprintf("%s/backendAddressPools/ssh-%d", *lb.ID, i)
for _, p := range *(*nic.IPConfigurations)[0].LoadBalancerBackendAddressPools {
if strings.EqualFold(*p.ID, id) {
return false
}
}
*(*nic.IPConfigurations)[0].LoadBalancerBackendAddressPools = append(*(*nic.IPConfigurations)[0].LoadBalancerBackendAddressPools, mgmtnetwork.BackendAddressPool{
ID: &id,
})
return true
}
func updateLB(lb *mgmtnetwork.LoadBalancer) (changed bool) {
backendAddressPools:
for i := 0; i < 3; i++ {
name := fmt.Sprintf("ssh-%d", i)
for _, p := range *lb.BackendAddressPools {
if strings.EqualFold(*p.Name, name) {
continue backendAddressPools
}
}
changed = true
*lb.BackendAddressPools = append(*lb.BackendAddressPools, mgmtnetwork.BackendAddressPool{
Name: &name,
})
}
loadBalancingRules:
for i := 0; i < 3; i++ {
name := fmt.Sprintf("ssh-%d", i)
for _, r := range *lb.LoadBalancingRules {
if strings.EqualFold(*r.Name, name) {
continue loadBalancingRules
}
}
changed = true
*lb.LoadBalancingRules = append(*lb.LoadBalancingRules, mgmtnetwork.LoadBalancingRule{
LoadBalancingRulePropertiesFormat: &mgmtnetwork.LoadBalancingRulePropertiesFormat{
FrontendIPConfiguration: &mgmtnetwork.SubResource{
ID: (*lb.FrontendIPConfigurations)[0].ID,
},
BackendAddressPool: &mgmtnetwork.SubResource{
ID: to.StringPtr(fmt.Sprintf("%s/backendAddressPools/ssh-%d", *lb.ID, i)),
},
Probe: &mgmtnetwork.SubResource{
ID: to.StringPtr(*lb.ID + "/probes/ssh"),
},
Protocol: mgmtnetwork.TransportProtocolTCP,
LoadDistribution: mgmtnetwork.LoadDistributionDefault,
FrontendPort: to.Int32Ptr(2200 + int32(i)),
BackendPort: to.Int32Ptr(22),
IdleTimeoutInMinutes: to.Int32Ptr(30),
DisableOutboundSnat: to.BoolPtr(true),
},
Name: &name,
})
}
for _, p := range *lb.Probes {
if strings.EqualFold(*p.Name, "ssh") {
return changed
}
}
changed = true
*lb.Probes = append(*lb.Probes, mgmtnetwork.Probe{
ProbePropertiesFormat: &mgmtnetwork.ProbePropertiesFormat{
Protocol: mgmtnetwork.ProbeProtocolTCP,
Port: to.Int32Ptr(22),
IntervalInSeconds: to.Int32Ptr(5),
NumberOfProbes: to.Int32Ptr(2),
},
Name: to.StringPtr("ssh"),
})
return changed
}

263
pkg/cluster/fixssh_test.go Normal file
Просмотреть файл

@ -0,0 +1,263 @@
package cluster
// Copyright (c) Microsoft Corporation.
// Licensed under the Apache License 2.0.
import (
"context"
"fmt"
"testing"
mgmtnetwork "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-07-01/network"
"github.com/Azure/go-autorest/autorest/to"
"github.com/golang/mock/gomock"
"github.com/sirupsen/logrus"
"github.com/Azure/ARO-RP/pkg/api"
mock_network "github.com/Azure/ARO-RP/pkg/util/mocks/azureclient/mgmt/network"
)
func TestFixSSH(t *testing.T) {
resourceGroup := "rg"
infraID := "infra"
ipc := "internal-lb-ip-v4"
lbBefore := func(lbID string) *mgmtnetwork.LoadBalancer {
return &mgmtnetwork.LoadBalancer{
ID: to.StringPtr(lbID),
LoadBalancerPropertiesFormat: &mgmtnetwork.LoadBalancerPropertiesFormat{
FrontendIPConfigurations: &[]mgmtnetwork.FrontendIPConfiguration{
{
ID: to.StringPtr(lbID + "/frontendIPConfigurations/" + ipc),
},
},
BackendAddressPools: &[]mgmtnetwork.BackendAddressPool{},
LoadBalancingRules: &[]mgmtnetwork.LoadBalancingRule{},
Probes: &[]mgmtnetwork.Probe{},
},
}
}
lbAfter := func(lbID string) *mgmtnetwork.LoadBalancer {
return &mgmtnetwork.LoadBalancer{
ID: to.StringPtr(lbID),
LoadBalancerPropertiesFormat: &mgmtnetwork.LoadBalancerPropertiesFormat{
FrontendIPConfigurations: &[]mgmtnetwork.FrontendIPConfiguration{
{
ID: to.StringPtr(lbID + "/frontendIPConfigurations/" + ipc),
},
},
BackendAddressPools: &[]mgmtnetwork.BackendAddressPool{
{
Name: to.StringPtr("ssh-0"),
},
{
Name: to.StringPtr("ssh-1"),
},
{
Name: to.StringPtr("ssh-2"),
},
},
LoadBalancingRules: &[]mgmtnetwork.LoadBalancingRule{
{
LoadBalancingRulePropertiesFormat: &mgmtnetwork.LoadBalancingRulePropertiesFormat{
FrontendIPConfiguration: &mgmtnetwork.SubResource{
ID: to.StringPtr(lbID + "/frontendIPConfigurations/" + ipc),
},
BackendAddressPool: &mgmtnetwork.SubResource{
ID: to.StringPtr(lbID + "/backendAddressPools/ssh-0"),
},
Probe: &mgmtnetwork.SubResource{
ID: to.StringPtr(lbID + "/probes/ssh"),
},
Protocol: mgmtnetwork.TransportProtocolTCP,
LoadDistribution: mgmtnetwork.LoadDistributionDefault,
FrontendPort: to.Int32Ptr(2200),
BackendPort: to.Int32Ptr(22),
IdleTimeoutInMinutes: to.Int32Ptr(30),
DisableOutboundSnat: to.BoolPtr(true),
},
Name: to.StringPtr("ssh-0"),
},
{
LoadBalancingRulePropertiesFormat: &mgmtnetwork.LoadBalancingRulePropertiesFormat{
FrontendIPConfiguration: &mgmtnetwork.SubResource{
ID: to.StringPtr(lbID + "/frontendIPConfigurations/" + ipc),
},
BackendAddressPool: &mgmtnetwork.SubResource{
ID: to.StringPtr(lbID + "/backendAddressPools/ssh-1"),
},
Probe: &mgmtnetwork.SubResource{
ID: to.StringPtr(lbID + "/probes/ssh"),
},
Protocol: mgmtnetwork.TransportProtocolTCP,
LoadDistribution: mgmtnetwork.LoadDistributionDefault,
FrontendPort: to.Int32Ptr(2201),
BackendPort: to.Int32Ptr(22),
IdleTimeoutInMinutes: to.Int32Ptr(30),
DisableOutboundSnat: to.BoolPtr(true),
},
Name: to.StringPtr("ssh-1"),
},
{
LoadBalancingRulePropertiesFormat: &mgmtnetwork.LoadBalancingRulePropertiesFormat{
FrontendIPConfiguration: &mgmtnetwork.SubResource{
ID: to.StringPtr(lbID + "/frontendIPConfigurations/" + ipc),
},
BackendAddressPool: &mgmtnetwork.SubResource{
ID: to.StringPtr(lbID + "/backendAddressPools/ssh-2"),
},
Probe: &mgmtnetwork.SubResource{
ID: to.StringPtr(lbID + "/probes/ssh"),
},
Protocol: mgmtnetwork.TransportProtocolTCP,
LoadDistribution: mgmtnetwork.LoadDistributionDefault,
FrontendPort: to.Int32Ptr(2202),
BackendPort: to.Int32Ptr(22),
IdleTimeoutInMinutes: to.Int32Ptr(30),
DisableOutboundSnat: to.BoolPtr(true),
},
Name: to.StringPtr("ssh-2"),
},
},
Probes: &[]mgmtnetwork.Probe{
{
ProbePropertiesFormat: &mgmtnetwork.ProbePropertiesFormat{
Protocol: mgmtnetwork.ProbeProtocolTCP,
Port: to.Int32Ptr(22),
IntervalInSeconds: to.Int32Ptr(5),
NumberOfProbes: to.Int32Ptr(2),
},
Name: to.StringPtr("ssh"),
},
},
},
}
}
ifBefore := func(lbID string, i int) *mgmtnetwork.Interface {
return &mgmtnetwork.Interface{
InterfacePropertiesFormat: &mgmtnetwork.InterfacePropertiesFormat{
IPConfigurations: &[]mgmtnetwork.InterfaceIPConfiguration{
{
InterfaceIPConfigurationPropertiesFormat: &mgmtnetwork.InterfaceIPConfigurationPropertiesFormat{
LoadBalancerBackendAddressPools: &[]mgmtnetwork.BackendAddressPool{},
},
},
},
},
}
}
ifAfter := func(lbID string, i int) *mgmtnetwork.Interface {
return &mgmtnetwork.Interface{
InterfacePropertiesFormat: &mgmtnetwork.InterfacePropertiesFormat{
IPConfigurations: &[]mgmtnetwork.InterfaceIPConfiguration{
{
InterfaceIPConfigurationPropertiesFormat: &mgmtnetwork.InterfaceIPConfigurationPropertiesFormat{
LoadBalancerBackendAddressPools: &[]mgmtnetwork.BackendAddressPool{
{
ID: to.StringPtr(fmt.Sprintf(lbID+"/backendAddressPools/ssh-%d", i)),
},
},
},
},
},
},
}
}
for _, tt := range []struct {
name string
architectureVersion api.ArchitectureVersion
lb string
lbID string
loadbalancer func(string) *mgmtnetwork.LoadBalancer
iface func(string, int) *mgmtnetwork.Interface
mocks func(string, string, *mock_network.MockLoadBalancersClient, *mock_network.MockInterfacesClient)
}{
{
name: "updates v1 resources correctly",
lb: infraID + "-internal-lb",
lbID: "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/" + resourceGroup + "/providers/Microsoft.Network/loadBalancers/" + infraID + "-internal-lb",
loadbalancer: lbBefore,
iface: ifBefore,
mocks: func(lb, lbID string, loadBalancers *mock_network.MockLoadBalancersClient, interfaces *mock_network.MockInterfacesClient) {
loadBalancers.EXPECT().CreateOrUpdateAndWait(gomock.Any(), resourceGroup, lb, *lbAfter(lbID))
for i := 0; i < 3; i++ {
interfaces.EXPECT().CreateOrUpdateAndWait(gomock.Any(), resourceGroup, fmt.Sprintf(infraID+"-master%d-nic", i), *ifAfter(lbID, i))
}
},
},
{
name: "v1 noop",
lb: infraID + "-internal-lb",
lbID: "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/" + resourceGroup + "/providers/Microsoft.Network/loadBalancers/" + infraID + "-internal-lb",
loadbalancer: lbAfter,
iface: ifAfter,
},
{
name: "updates v2 resources correctly",
architectureVersion: api.ArchitectureVersionV2,
lb: infraID + "-internal",
lbID: "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/" + resourceGroup + "/providers/Microsoft.Network/loadBalancers/" + infraID + "-internal",
loadbalancer: lbBefore,
iface: ifBefore,
mocks: func(lb, lbID string, loadBalancers *mock_network.MockLoadBalancersClient, interfaces *mock_network.MockInterfacesClient) {
loadBalancers.EXPECT().CreateOrUpdateAndWait(gomock.Any(), resourceGroup, lb, *lbAfter(lbID))
for i := 0; i < 3; i++ {
interfaces.EXPECT().CreateOrUpdateAndWait(gomock.Any(), resourceGroup, fmt.Sprintf(infraID+"-master%d-nic", i), *ifAfter(lbID, i))
}
},
},
{
name: "v2 noop",
architectureVersion: api.ArchitectureVersionV2,
lb: infraID + "-internal",
lbID: "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/" + resourceGroup + "/providers/Microsoft.Network/loadBalancers/" + infraID + "-internal",
loadbalancer: lbAfter,
iface: ifAfter,
},
} {
t.Run(tt.name, func(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()
interfaces := mock_network.NewMockInterfacesClient(ctrl)
loadBalancers := mock_network.NewMockLoadBalancersClient(ctrl)
loadBalancers.EXPECT().Get(gomock.Any(), resourceGroup, tt.lb, "").Return(*tt.loadbalancer(tt.lbID), nil)
for i := 0; i < 3; i++ {
interfaces.EXPECT().Get(gomock.Any(), resourceGroup, fmt.Sprintf(infraID+"-master%d-nic", i), "").Return(*tt.iface(tt.lbID, i), nil)
}
if tt.mocks != nil {
tt.mocks(tt.lb, tt.lbID, loadBalancers, interfaces)
}
m := &manager{
log: logrus.NewEntry(logrus.StandardLogger()),
doc: &api.OpenShiftClusterDocument{
OpenShiftCluster: &api.OpenShiftCluster{
Properties: api.OpenShiftClusterProperties{
ArchitectureVersion: tt.architectureVersion,
ClusterProfile: api.ClusterProfile{
ResourceGroupID: "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/" + resourceGroup,
},
InfraID: infraID,
},
},
},
interfaces: interfaces,
loadBalancers: loadBalancers,
}
err := m.fixSSH(context.Background())
if err != nil {
t.Error(err)
}
})
}
}

1
pkg/cluster/install.go
Просмотреть файл

@ -32,6 +32,7 @@ func (m *manager) AdminUpdate(ctx context.Context) error {
steps.Action(m.startVMs),
steps.Condition(m.apiServersReady, 30*time.Minute),
steps.Action(m.ensureBillingRecord), // belt and braces
steps.Action(m.fixSSH),
steps.Action(m.fixPullSecret), // TODO(mj): Remove when operator deployed
steps.Action(m.ensureAROOperator),
steps.Condition(m.aroDeploymentReady, 20*time.Minute),