зеркало из https://github.com/Azure/ARO-RP.git
Revert 'CI E2E in SimplySecure'
This commit is contained in:
Christoph Blecker
Родитель
a7f976228f
Коммит
385435b82c
|
|
@ -43,9 +43,6 @@ jobs:
|
|||
az account set -s $AZURE_SUBSCRIPTION_ID
|
||||
|
||||
set -x
|
||||
|
||||
export PRIVATE_CLUSTER=true
|
||||
|
||||
. ./hack/e2e/run-rp-and-e2e.sh
|
||||
trap 'set +e; kill_rp; clean_e2e_db; kill_vpn' EXIT
|
||||
|
||||
|
|
@ -58,7 +55,5 @@ jobs:
|
|||
register_sub
|
||||
|
||||
export CI=true
|
||||
|
||||
make test-e2e
|
||||
|
||||
- template: ./templates/template-az-cli-logout.yml
|
||||
|
|
|
|||
|
|
@ -456,8 +456,6 @@ each of the bash functions below.
|
|||
deploy_rp_dev
|
||||
# Deploy the proxy and VPN
|
||||
deploy_env_dev
|
||||
# Deploy AKS resources for Hive
|
||||
deploy_aks_dev
|
||||
```
|
||||
|
||||
If you encounter a "VirtualNetworkGatewayCannotUseStandardPublicIP" error
|
||||
|
|
|
|||
|
|
@ -48,7 +48,6 @@ deploy_env_dev_ci() {
|
|||
"proxyImage=arointsvc.azurecr.io/proxy:latest" \
|
||||
"proxyImageAuth=$(jq -r '.auths["arointsvc.azurecr.io"].auth' <<<$PULL_SECRET)" \
|
||||
"proxyKey=$(base64 -w0 <secrets/proxy.key)" \
|
||||
"vpnCACertificate=$(base64 -w0 <secrets/vpn-ca.crt)" \
|
||||
"sshPublicKey=$(<secrets/proxy_id_rsa.pub)" >/dev/null
|
||||
}
|
||||
|
||||
|
|
@ -78,7 +77,8 @@ deploy_aks_dev() {
|
|||
--parameters \
|
||||
"adminObjectId=$ADMIN_OBJECT_ID" \
|
||||
"dnsZone=$DOMAIN_NAME" \
|
||||
"sshRSAPublicKey=$(<secrets/proxy_id_rsa.pub)" >/dev/null
|
||||
"sshRSAPublicKey=$(<secrets/proxy_id_rsa.pub)" \
|
||||
"vpnCACertificate=$(base64 -w0 <secrets/vpn-ca.crt)" >/dev/null
|
||||
}
|
||||
|
||||
deploy_env_dev_override() {
|
||||
|
|
|
|||
|
|
@ -76,16 +76,6 @@ clean_e2e_db(){
|
|||
--resource-group $RESOURCEGROUP >/dev/null
|
||||
}
|
||||
|
||||
run_vpn() {
|
||||
sudo openvpn --config secrets/$VPN --daemon --writepid vpnpid
|
||||
sleep 10
|
||||
}
|
||||
|
||||
kill_vpn() {
|
||||
while read pid; do sudo kill $pid; done < vpnpid
|
||||
}
|
||||
|
||||
|
||||
# TODO: CLUSTER and is also recalculated in multiple places
|
||||
# in the billing pipelines :-(
|
||||
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ func (m *manager) clusterNSG(infraID, location string) *arm.Resource {
|
|||
SourceAddressPrefix: to.StringPtr("*"),
|
||||
DestinationAddressPrefix: to.StringPtr("*"),
|
||||
Access: mgmtnetwork.SecurityRuleAccessAllow,
|
||||
Priority: to.Int32Ptr(120),
|
||||
Priority: to.Int32Ptr(101),
|
||||
Direction: mgmtnetwork.SecurityRuleDirectionInbound,
|
||||
},
|
||||
Name: to.StringPtr("apiserver_in"),
|
||||
|
|
|
|||
|
|
@ -168,6 +168,20 @@
|
|||
"description": "Specifies the address prefix of the subnet hosting the pods of the AKS cluster."
|
||||
}
|
||||
},
|
||||
"gatewaySubnetName": {
|
||||
"type": "string",
|
||||
"defaultValue": "GatewaySubnet",
|
||||
"metadata": {
|
||||
"description": "Subnet name that will contain the App Service Environment"
|
||||
}
|
||||
},
|
||||
"gatewaySubnetPrefix": {
|
||||
"type": "string",
|
||||
"defaultValue": "10.128.0.0/24",
|
||||
"metadata": {
|
||||
"description": "Subnet address prefix"
|
||||
}
|
||||
},
|
||||
"serviceCidr": {
|
||||
"type": "string",
|
||||
"defaultValue": "10.130.0.0/16",
|
||||
|
|
@ -251,6 +265,10 @@
|
|||
"metadata": {
|
||||
"description": "Specifies the max graceful termination time interval in seconds for the auto-scaler of the AKS cluster."
|
||||
}
|
||||
},
|
||||
"vpnCACertificate": {
|
||||
"type": "string",
|
||||
"defaultValue": ""
|
||||
}
|
||||
},
|
||||
"variables": {
|
||||
|
|
@ -335,6 +353,12 @@
|
|||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "[parameters('gatewaySubnetName')]",
|
||||
"properties": {
|
||||
"addressPrefix": "[parameters('gatewaySubnetPrefix')]"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -412,39 +436,15 @@
|
|||
}
|
||||
},
|
||||
{
|
||||
"type": "Microsoft.Network/virtualNetworks/virtualNetworkPeerings",
|
||||
"name": "[concat('dev-vpn-vnet/peering-', parameters('vnetName'))]",
|
||||
"apiVersion": "2021-12-01",
|
||||
"name": "aks-vpn-pip",
|
||||
"type": "Microsoft.Network/publicIPAddresses",
|
||||
"location": "[resourceGroup().location]",
|
||||
"dependsOn": [
|
||||
"[variables('vnetId')]"
|
||||
],
|
||||
"apiVersion": "2020-08-01",
|
||||
"sku": {
|
||||
"name": "Basic"
|
||||
},
|
||||
"properties": {
|
||||
"allowVirtualNetworkAccess": true,
|
||||
"allowForwardedTraffic": true,
|
||||
"allowGatewayTransit": true,
|
||||
"useRemoteGateways": false,
|
||||
"remoteVirtualNetwork": {
|
||||
"id": "[variables('vnetId')]"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "Microsoft.Network/virtualNetworks/virtualNetworkPeerings",
|
||||
"name": "[concat(parameters('vnetName'), '/peering-dev-vpn-vnet')]",
|
||||
"apiVersion": "2021-12-01",
|
||||
"location": "[resourceGroup().location]",
|
||||
"dependsOn": [
|
||||
"[variables('vnetId')]"
|
||||
],
|
||||
"properties": {
|
||||
"allowVirtualNetworkAccess": true,
|
||||
"allowForwardedTraffic": true,
|
||||
"allowGatewayTransit": false,
|
||||
"useRemoteGateways": true,
|
||||
"remoteVirtualNetwork": {
|
||||
"id": "[resourceId('Microsoft.Network/virtualNetworks', 'dev-vpn-vnet')]"
|
||||
}
|
||||
"publicIPAllocationMethod": "Dynamic"
|
||||
}
|
||||
},
|
||||
{
|
||||
|
|
@ -612,6 +612,55 @@
|
|||
],
|
||||
"enableSoftDelete": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "Microsoft.Network/virtualNetworkGateways",
|
||||
"name": "aks-vpn",
|
||||
"apiVersion": "2020-08-01",
|
||||
"location": "[resourceGroup().location]",
|
||||
"dependsOn": [
|
||||
"[resourceId('Microsoft.Network/publicIPAddresses', 'aks-vpn-pip')]",
|
||||
"[variables('vnetId')]",
|
||||
"[parameters('aksClusterName')]"
|
||||
],
|
||||
"properties": {
|
||||
"ipConfigurations": [
|
||||
{
|
||||
"properties": {
|
||||
"subnet": {
|
||||
"id": "[resourceId('Microsoft.Network/virtualNetworks/subnets', parameters('vnetName'), parameters('gatewaySubnetName'))]"
|
||||
},
|
||||
"publicIPAddress": {
|
||||
"id": "[resourceId('Microsoft.Network/publicIPAddresses', 'aks-vpn-pip')]"
|
||||
}
|
||||
},
|
||||
"name": "default"
|
||||
}
|
||||
],
|
||||
"vpnType": "RouteBased",
|
||||
"sku": {
|
||||
"name": "VpnGw1",
|
||||
"tier": "VpnGw1"
|
||||
},
|
||||
"vpnClientConfiguration": {
|
||||
"vpnClientAddressPool": {
|
||||
"addressPrefixes": [
|
||||
"192.168.254.0/24"
|
||||
]
|
||||
},
|
||||
"vpnClientRootCertificates": [
|
||||
{
|
||||
"properties": {
|
||||
"publicCertData": "[parameters('vpnCACertificate')]"
|
||||
},
|
||||
"name": "aks-vpn-ca"
|
||||
}
|
||||
],
|
||||
"vpnClientProtocols": [
|
||||
"OpenVPN"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,8 +4,6 @@ package network
|
|||
// Licensed under the Apache License 2.0.
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
mgmtnetwork "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2020-08-01/network"
|
||||
"github.com/Azure/go-autorest/autorest"
|
||||
|
||||
|
|
@ -13,8 +11,6 @@ import (
|
|||
)
|
||||
|
||||
type VirtualNetworkPeeringsClient interface {
|
||||
CreateOrUpdate(ctx context.Context, resourceGroupName string, virtualNetworkName string, virtualNetworkPeeringName string, virtualNetworkPeeringParameters mgmtnetwork.VirtualNetworkPeering) (result mgmtnetwork.VirtualNetworkPeeringsCreateOrUpdateFuture, err error)
|
||||
Delete(ctx context.Context, resourceGroupName string, virtualNetworkName string, virtualNetworkPeeringName string) (result mgmtnetwork.VirtualNetworkPeeringsDeleteFuture, err error)
|
||||
VirtualNetworkPeeringsAddons
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -5,24 +5,12 @@ package network
|
|||
|
||||
import (
|
||||
"context"
|
||||
|
||||
mgmtnetwork "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2020-08-01/network"
|
||||
)
|
||||
|
||||
type VirtualNetworkPeeringsAddons interface {
|
||||
CreateOrUpdateAndWait(ctx context.Context, resourceGroupName string, virtualNetworkName string, virtualNetworkPeeringName string, virtualNetworkPeeringParameters mgmtnetwork.VirtualNetworkPeering) (err error)
|
||||
DeleteAndWait(ctx context.Context, resourceGroupName string, virtualNetworkName string, virtualNetworkPeeringName string) (err error)
|
||||
}
|
||||
|
||||
func (c *virtualNetworkPeeringsClient) CreateOrUpdateAndWait(ctx context.Context, resourceGroupName string, virtualNetworkName string, virtualNetworkPeeringName string, virtualNetworkPeeringParameters mgmtnetwork.VirtualNetworkPeering) (err error) {
|
||||
future, err := c.CreateOrUpdate(ctx, resourceGroupName, virtualNetworkName, virtualNetworkPeeringName, virtualNetworkPeeringParameters)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return future.WaitForCompletionRef(ctx, c.Client)
|
||||
}
|
||||
|
||||
func (c *virtualNetworkPeeringsClient) DeleteAndWait(ctx context.Context, resourceGroupName string, virtualNetworkName string, virtualNetworkPeeringName string) (err error) {
|
||||
future, err := c.VirtualNetworkPeeringsClient.Delete(ctx, resourceGroupName, virtualNetworkName, virtualNetworkPeeringName)
|
||||
if err != nil {
|
||||
|
|
|
|||
|
|
@ -20,7 +20,6 @@ import (
|
|||
mgmtauthorization "github.com/Azure/azure-sdk-for-go/services/preview/authorization/mgmt/2018-09-01-preview/authorization"
|
||||
mgmtfeatures "github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2019-07-01/features"
|
||||
"github.com/Azure/go-autorest/autorest"
|
||||
"github.com/Azure/go-autorest/autorest/azure"
|
||||
"github.com/Azure/go-autorest/autorest/azure/auth"
|
||||
"github.com/Azure/go-autorest/autorest/to"
|
||||
"github.com/sirupsen/logrus"
|
||||
|
|
@ -47,10 +46,9 @@ import (
|
|||
)
|
||||
|
||||
type Cluster struct {
|
||||
log *logrus.Entry
|
||||
env env.Core
|
||||
ci bool
|
||||
ciParentVnet string
|
||||
log *logrus.Entry
|
||||
env env.Core
|
||||
ci bool
|
||||
|
||||
deployments features.DeploymentsClient
|
||||
groups features.ResourceGroupsClient
|
||||
|
|
@ -65,7 +63,6 @@ type Cluster struct {
|
|||
routetables network.RouteTablesClient
|
||||
roleassignments authorization.RoleAssignmentsClient
|
||||
peerings network.VirtualNetworkPeeringsClient
|
||||
ciParentVnetPeerings network.VirtualNetworkPeeringsClient
|
||||
vaultsClient keyvaultclient.VaultsClient
|
||||
}
|
||||
|
||||
|
|
@ -124,17 +121,6 @@ func New(log *logrus.Entry, environment env.Core, ci bool) (*Cluster, error) {
|
|||
vaultsClient: keyvaultclient.NewVaultsClient(environment.Environment(), environment.SubscriptionID(), authorizer),
|
||||
}
|
||||
|
||||
// Only peer if CI=true and cluster is PublicCloud
|
||||
if ci {
|
||||
c.ciParentVnet = fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/virtualNetworks/dev-vpn-vnet", c.env.SubscriptionID(), c.env.ResourceGroup())
|
||||
r, err := azure.ParseResourceID(c.ciParentVnet)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
c.ciParentVnetPeerings = network.NewVirtualNetworkPeeringsClient(environment.Environment(), r.SubscriptionID, authorizer)
|
||||
}
|
||||
|
||||
return c, nil
|
||||
}
|
||||
|
||||
|
|
@ -322,12 +308,6 @@ func (c *Cluster) Create(ctx context.Context, vnetResourceGroup, clusterName str
|
|||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
c.log.Info("peering subnets to CI infra")
|
||||
err = c.peerSubnetsToCI(ctx, vnetResourceGroup, clusterName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
c.log.Info("done")
|
||||
|
|
@ -335,15 +315,13 @@ func (c *Cluster) Create(ctx context.Context, vnetResourceGroup, clusterName str
|
|||
}
|
||||
|
||||
func (c *Cluster) generateSubnets() (vnetPrefix string, masterSubnet string, workerSubnet string) {
|
||||
// pick a random 23 in range [10.3.0.0, 10.127.255.0]
|
||||
// 10.0.0.0 is used by dev-vnet to host CI
|
||||
// 10.1.0.0 is used by rp-vnet to host Proxy VM
|
||||
// 10.2.0.0 is used by dev-vpn-vnet to host VirtualNetworkGateway
|
||||
// pick a random /23 in the range [10.0.2.0, 10.128.0.0). 10.0.0.0 is used
|
||||
// by dev-vnet to host CI; 10.128.0.0+ is used for pods.
|
||||
var x, y int
|
||||
rand.Seed(time.Now().UnixNano())
|
||||
for x == 0 && y == 0 {
|
||||
x, y = rand.Intn((124))+3, 2*rand.Intn(128)
|
||||
x, y = rand.Intn(128), 2*rand.Intn(128)
|
||||
}
|
||||
|
||||
vnetPrefix = fmt.Sprintf("10.%d.%d.0/23", x, y)
|
||||
masterSubnet = fmt.Sprintf("10.%d.%d.0/24", x, y)
|
||||
workerSubnet = fmt.Sprintf("10.%d.%d.0/24", x, y+1)
|
||||
|
|
@ -381,14 +359,6 @@ func (c *Cluster) Delete(ctx context.Context, vnetResourceGroup, clusterName str
|
|||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
// Only do this if CI=true and cloud = Public Cloud
|
||||
r, err := azure.ParseResourceID(c.ciParentVnet)
|
||||
if err == nil {
|
||||
err = c.ciParentVnetPeerings.DeleteAndWait(ctx, r.ResourceGroup, r.ResourceName, vnetResourceGroup+"-peer")
|
||||
}
|
||||
if err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
} else {
|
||||
// Deleting the deployment does not clean up the associated resources
|
||||
c.log.Info("deleting deployment")
|
||||
|
|
@ -604,41 +574,3 @@ func (c *Cluster) deleteRoleAssignments(ctx context.Context, vnetResourceGroup,
|
|||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Cluster) peerSubnetsToCI(ctx context.Context, vnetResourceGroup, clusterName string) error {
|
||||
cluster := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/virtualNetworks/dev-vnet", c.env.SubscriptionID(), vnetResourceGroup)
|
||||
|
||||
r, err := azure.ParseResourceID(c.ciParentVnet)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
clusterProp := &mgmtnetwork.VirtualNetworkPeeringPropertiesFormat{
|
||||
RemoteVirtualNetwork: &mgmtnetwork.SubResource{
|
||||
ID: &c.ciParentVnet,
|
||||
},
|
||||
AllowVirtualNetworkAccess: to.BoolPtr(true),
|
||||
AllowForwardedTraffic: to.BoolPtr(true),
|
||||
UseRemoteGateways: to.BoolPtr(true),
|
||||
}
|
||||
rpProp := &mgmtnetwork.VirtualNetworkPeeringPropertiesFormat{
|
||||
RemoteVirtualNetwork: &mgmtnetwork.SubResource{
|
||||
ID: &cluster,
|
||||
},
|
||||
AllowVirtualNetworkAccess: to.BoolPtr(true),
|
||||
AllowForwardedTraffic: to.BoolPtr(true),
|
||||
AllowGatewayTransit: to.BoolPtr(true),
|
||||
}
|
||||
|
||||
err = c.peerings.CreateOrUpdateAndWait(ctx, vnetResourceGroup, "dev-vnet", r.ResourceGroup+"-peer", mgmtnetwork.VirtualNetworkPeering{VirtualNetworkPeeringPropertiesFormat: clusterProp})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = c.ciParentVnetPeerings.CreateOrUpdateAndWait(ctx, r.ResourceGroup, r.ResourceName, vnetResourceGroup+"-peer", mgmtnetwork.VirtualNetworkPeering{VirtualNetworkPeeringPropertiesFormat: rpProp})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -591,50 +591,6 @@ func (m *MockVirtualNetworkPeeringsClient) EXPECT() *MockVirtualNetworkPeeringsC
|
|||
return m.recorder
|
||||
}
|
||||
|
||||
// CreateOrUpdate mocks base method.
|
||||
func (m *MockVirtualNetworkPeeringsClient) CreateOrUpdate(arg0 context.Context, arg1, arg2, arg3 string, arg4 network.VirtualNetworkPeering) (network.VirtualNetworkPeeringsCreateOrUpdateFuture, error) {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "CreateOrUpdate", arg0, arg1, arg2, arg3, arg4)
|
||||
ret0, _ := ret[0].(network.VirtualNetworkPeeringsCreateOrUpdateFuture)
|
||||
ret1, _ := ret[1].(error)
|
||||
return ret0, ret1
|
||||
}
|
||||
|
||||
// CreateOrUpdate indicates an expected call of CreateOrUpdate.
|
||||
func (mr *MockVirtualNetworkPeeringsClientMockRecorder) CreateOrUpdate(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateOrUpdate", reflect.TypeOf((*MockVirtualNetworkPeeringsClient)(nil).CreateOrUpdate), arg0, arg1, arg2, arg3, arg4)
|
||||
}
|
||||
|
||||
// CreateOrUpdateAndWait mocks base method.
|
||||
func (m *MockVirtualNetworkPeeringsClient) CreateOrUpdateAndWait(arg0 context.Context, arg1, arg2, arg3 string, arg4 network.VirtualNetworkPeering) error {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "CreateOrUpdateAndWait", arg0, arg1, arg2, arg3, arg4)
|
||||
ret0, _ := ret[0].(error)
|
||||
return ret0
|
||||
}
|
||||
|
||||
// CreateOrUpdateAndWait indicates an expected call of CreateOrUpdateAndWait.
|
||||
func (mr *MockVirtualNetworkPeeringsClientMockRecorder) CreateOrUpdateAndWait(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateOrUpdateAndWait", reflect.TypeOf((*MockVirtualNetworkPeeringsClient)(nil).CreateOrUpdateAndWait), arg0, arg1, arg2, arg3, arg4)
|
||||
}
|
||||
|
||||
// Delete mocks base method.
|
||||
func (m *MockVirtualNetworkPeeringsClient) Delete(arg0 context.Context, arg1, arg2, arg3 string) (network.VirtualNetworkPeeringsDeleteFuture, error) {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "Delete", arg0, arg1, arg2, arg3)
|
||||
ret0, _ := ret[0].(network.VirtualNetworkPeeringsDeleteFuture)
|
||||
ret1, _ := ret[1].(error)
|
||||
return ret0, ret1
|
||||
}
|
||||
|
||||
// Delete indicates an expected call of Delete.
|
||||
func (mr *MockVirtualNetworkPeeringsClientMockRecorder) Delete(arg0, arg1, arg2, arg3 interface{}) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockVirtualNetworkPeeringsClient)(nil).Delete), arg0, arg1, arg2, arg3)
|
||||
}
|
||||
|
||||
// DeleteAndWait mocks base method.
|
||||
func (m *MockVirtualNetworkPeeringsClient) DeleteAndWait(arg0 context.Context, arg1, arg2, arg3 string) error {
|
||||
m.ctrl.T.Helper()
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче