diff --git a/cmd/aro/operator.go b/cmd/aro/operator.go index e9ef57292..8c118c112 100644 --- a/cmd/aro/operator.go +++ b/cmd/aro/operator.go @@ -23,6 +23,7 @@ import ( aroclient "github.com/Azure/ARO-RP/pkg/operator/clientset/versioned" "github.com/Azure/ARO-RP/pkg/operator/controllers" "github.com/Azure/ARO-RP/pkg/operator/controllers/alertwebhook" + "github.com/Azure/ARO-RP/pkg/operator/controllers/azurensg" "github.com/Azure/ARO-RP/pkg/operator/controllers/checker" "github.com/Azure/ARO-RP/pkg/operator/controllers/clusteroperatoraro" "github.com/Azure/ARO-RP/pkg/operator/controllers/dnsmasq" @@ -156,6 +157,11 @@ func operator(ctx context.Context, log *logrus.Entry) error { kubernetescli)).SetupWithManager(mgr); err != nil { return fmt.Errorf("unable to create controller Node: %v", err) } + if err = (azurensg.NewReconciler( + log.WithField("controller", controllers.AzureNSGControllerName), + arocli, maocli, kubernetescli)).SetupWithManager(mgr); err != nil { + return fmt.Errorf("unable to create controller AzureNSG: %v", err) + } } if err = (checker.NewReconciler( diff --git a/pkg/operator/README.md b/pkg/operator/README.md index 9891a4541..63c40ee24 100644 --- a/pkg/operator/README.md +++ b/pkg/operator/README.md @@ -25,6 +25,9 @@ There will be use cases where we may want to remediate end user decisions automatically. Carrying out remediation locally is advantageous because it is likely to be simpler, more reliable, and with a shorter time to remediate. +Remediations in place: +* periodically reset NSGs in the master and worker subnets to the defaults (controlled by the reconcileNSGs feature flag) + ### End user warnings * [TODO] see https://docs.openshift.com/container-platform/4.4/web_console/customizing-the-web-console.html#creating-custom-notification-banners_customizing-web-console diff --git a/pkg/operator/apis/aro.openshift.io/v1alpha1/cluster_types.go b/pkg/operator/apis/aro.openshift.io/v1alpha1/cluster_types.go index 0076db4bd..2f9ed595e 100644 --- a/pkg/operator/apis/aro.openshift.io/v1alpha1/cluster_types.go +++ b/pkg/operator/apis/aro.openshift.io/v1alpha1/cluster_types.go @@ -41,22 +41,26 @@ type InternetCheckerSpec struct { // ClusterSpec defines the desired state of Cluster type ClusterSpec struct { // ResourceID is the Azure resourceId of the cluster - ResourceID string `json:"resourceId,omitempty"` - Domain string `json:"domain,omitempty"` - ACRDomain string `json:"acrDomain,omitempty"` - AZEnvironment string `json:"azEnvironment,omitempty"` - Location string `json:"location,omitempty"` - GenevaLogging GenevaLoggingSpec `json:"genevaLogging,omitempty"` - InternetChecker InternetCheckerSpec `json:"internetChecker,omitempty"` - VnetID string `json:"vnetId,omitempty"` - APIIntIP string `json:"apiIntIP,omitempty"` - IngressIP string `json:"ingressIP,omitempty"` + ResourceID string `json:"resourceId,omitempty"` + Domain string `json:"domain,omitempty"` + ACRDomain string `json:"acrDomain,omitempty"` + AZEnvironment string `json:"azEnvironment,omitempty"` + Location string `json:"location,omitempty"` + InfraID string `json:"infraId,omitempty"` + ArchitectureVersion int `json:"architectureVersion,omitempty"` + GenevaLogging GenevaLoggingSpec `json:"genevaLogging,omitempty"` + InternetChecker InternetCheckerSpec `json:"internetChecker,omitempty"` + VnetID string `json:"vnetId,omitempty"` + APIIntIP string `json:"apiIntIP,omitempty"` + IngressIP string `json:"ingressIP,omitempty"` Features FeaturesSpec `json:"features,omitempty"` } // FeaturesSpec defines ARO operator feature gates -type FeaturesSpec struct{} +type FeaturesSpec struct { + ReconcileNSGs bool `json:"reconcileNSGs,omitempty"` +} // ClusterStatus defines the observed state of Cluster type ClusterStatus struct { diff --git a/pkg/operator/controllers/azurensg/azurensg_controller.go b/pkg/operator/controllers/azurensg/azurensg_controller.go new file mode 100644 index 000000000..dbf5b7795 --- /dev/null +++ b/pkg/operator/controllers/azurensg/azurensg_controller.go @@ -0,0 +1,103 @@ +package azurensg + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + + "github.com/Azure/go-autorest/autorest/azure" + machinev1beta1 "github.com/openshift/machine-api-operator/pkg/apis/machine/v1beta1" + maoclient "github.com/openshift/machine-api-operator/pkg/generated/clientset/versioned" + "github.com/sirupsen/logrus" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" + + arov1alpha1 "github.com/Azure/ARO-RP/pkg/operator/apis/aro.openshift.io/v1alpha1" + aroclient "github.com/Azure/ARO-RP/pkg/operator/clientset/versioned" + "github.com/Azure/ARO-RP/pkg/operator/controllers" + "github.com/Azure/ARO-RP/pkg/util/aad" + "github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/network" + "github.com/Azure/ARO-RP/pkg/util/clusterauthorizer" +) + +// AzureNSGReconciler is the controller struct +type AzureNSGReconciler struct { + arocli aroclient.Interface + maocli maoclient.Interface + kubernetescli kubernetes.Interface + log *logrus.Entry +} + +// NewReconciler creates a new Reconciler +func NewReconciler(log *logrus.Entry, arocli aroclient.Interface, maocli maoclient.Interface, kubernetescli kubernetes.Interface) *AzureNSGReconciler { + return &AzureNSGReconciler{ + arocli: arocli, + maocli: maocli, + kubernetescli: kubernetescli, + log: log, + } +} + +//Reconcile fixes the Network Security Groups +func (r *AzureNSGReconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.Result, error) { + instance, err := r.arocli.AroV1alpha1().Clusters().Get(ctx, arov1alpha1.SingletonClusterName, metav1.GetOptions{}) + if err != nil { + return reconcile.Result{}, err + } + + if !instance.Spec.Features.ReconcileNSGs { + // reconciling NSGs is disabled + return reconcile.Result{}, nil + } + + // Get endpoints from operator + azEnv, err := azure.EnvironmentFromName(instance.Spec.AZEnvironment) + if err != nil { + return reconcile.Result{}, err + } + // Grab azure-credentials from secret + credentials, err := clusterauthorizer.AzCredentials(ctx, r.kubernetescli) + if err != nil { + return reconcile.Result{}, err + } + resource, err := azure.ParseResourceID(instance.Spec.ResourceID) + if err != nil { + return reconcile.Result{}, err + } + // create service principal token from azure-credentials + token, err := aad.GetToken(ctx, r.log, string(credentials.ClientID), string(credentials.ClientSecret), string(credentials.TenantID), azEnv.ActiveDirectoryEndpoint, azEnv.ResourceManagerEndpoint) + if err != nil { + return reconcile.Result{}, err + } + // create refreshable authorizer from token + authorizer, err := clusterauthorizer.NewAzRefreshableAuthorizer(token) + if err != nil { + return reconcile.Result{}, err + } + subnetsClient := network.NewSubnetsClient(&azEnv, resource.SubscriptionID, authorizer) + + return reconcile.Result{}, r.reconcileSubnetNSG(ctx, instance, resource.SubscriptionID, subnetsClient) +} + +// SetupWithManager creates the controller +func (r *AzureNSGReconciler) SetupWithManager(mgr ctrl.Manager) error { + aroClusterPredicate := predicate.NewPredicateFuncs(func(o client.Object) bool { + return o.GetName() == arov1alpha1.SingletonClusterName + }) + + return ctrl.NewControllerManagedBy(mgr). + For(&arov1alpha1.Cluster{}, builder.WithPredicates(aroClusterPredicate)). + Watches(&source.Kind{Type: &machinev1beta1.Machine{}}, &handler.EnqueueRequestForObject{}). // to reconcile on machine replacement + Watches(&source.Kind{Type: &corev1.Node{}}, &handler.EnqueueRequestForObject{}). // to reconcile on node status change + Named(controllers.AzureNSGControllerName). + Complete(r) +} diff --git a/pkg/operator/controllers/azurensg/const.go b/pkg/operator/controllers/azurensg/const.go new file mode 100644 index 000000000..bce0bca24 --- /dev/null +++ b/pkg/operator/controllers/azurensg/const.go @@ -0,0 +1,8 @@ +package azurensg + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +const ( + machineSetsNamespace = "openshift-machine-api" +) diff --git a/pkg/operator/controllers/azurensg/subnetnsg.go b/pkg/operator/controllers/azurensg/subnetnsg.go new file mode 100644 index 000000000..7ab4c6f1d --- /dev/null +++ b/pkg/operator/controllers/azurensg/subnetnsg.go @@ -0,0 +1,116 @@ +package azurensg + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + mgmtnetwork "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-07-01/network" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + azureproviderv1beta1 "sigs.k8s.io/cluster-api-provider-azure/pkg/apis/azureprovider/v1beta1" + + "github.com/Azure/ARO-RP/pkg/api" + arov1alpha1 "github.com/Azure/ARO-RP/pkg/operator/apis/aro.openshift.io/v1alpha1" + "github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/network" + "github.com/Azure/ARO-RP/pkg/util/subnet" +) + +type subnetDescriptor struct { + resourceGroup string + vnetName string + subnetName string +} + +func (r *AzureNSGReconciler) reconcileSubnetNSG(ctx context.Context, instance *arov1alpha1.Cluster, subscriptionID string, subnetsClient network.SubnetsClient) error { + // the main logic starts here + subnets, masterResourceGroup, err := r.getSubnets(ctx) + if err != nil { + return err + } + for ws := range subnets { + err = r.ensureSubnetNSG(ctx, subnetsClient, subscriptionID, masterResourceGroup, instance.Spec.InfraID, api.ArchitectureVersion(instance.Spec.ArchitectureVersion), ws.resourceGroup, ws.vnetName, ws.subnetName, subnets[ws]) + if err != nil { + return err + } + } + return nil +} + +func (r *AzureNSGReconciler) ensureSubnetNSG(ctx context.Context, subnetsClient network.SubnetsClient, subscriptionID, resourcesResourceGroup, infraID string, architectureVersion api.ArchitectureVersion, vnetResourceGroup, vnetName, subnetName string, isWorkerSubnet bool) error { + subnetObject, err := subnetsClient.Get(ctx, vnetResourceGroup, vnetName, subnetName, "") + if err != nil { + return err + } + if subnetObject.SubnetPropertiesFormat == nil || subnetObject.SubnetPropertiesFormat.NetworkSecurityGroup == nil { + return fmt.Errorf("received nil, expected a value in SubnetProperties when trying to Get subnet %s/%s in resource group %s", vnetName, subnetName, vnetResourceGroup) + } + correctNSGResourceID, err := subnet.NetworkSecurityGroupIDExpanded(architectureVersion, resourcesResourceGroup, infraID, isWorkerSubnet) + if err != nil { + return err + } + correctNSGResourceID = fmt.Sprintf("/subscriptions/%s/resourceGroups/%s", subscriptionID, correctNSGResourceID) + + if !strings.EqualFold(*subnetObject.NetworkSecurityGroup.ID, correctNSGResourceID) { + r.log.Infof("Fixing NSG from %s to %s", *subnetObject.NetworkSecurityGroup.ID, correctNSGResourceID) + // NSG doesn't match - fixing + subnetObject.NetworkSecurityGroup = &mgmtnetwork.SecurityGroup{ID: &correctNSGResourceID} + err = subnetsClient.CreateOrUpdateAndWait(ctx, vnetResourceGroup, vnetName, subnetName, subnetObject) + if err != nil { + return err + } + } + return nil +} + +func (r *AzureNSGReconciler) getSubnets(ctx context.Context) (map[subnetDescriptor]bool, string, error) { + subnetMap := make(map[subnetDescriptor]bool) // bool is true for master subnets + var masterResourceGroup *string + // select all workers by the machine.openshift.io/cluster-api-machine-role: not equal to master Label + machines, err := r.maocli.MachineV1beta1().Machines(machineSetsNamespace).List(ctx, metav1.ListOptions{LabelSelector: "machine.openshift.io/cluster-api-machine-role!=master"}) + if err != nil { + return nil, "", err + } + + for _, machine := range machines.Items { + _, subnetDesc, err := r.getDescriptorFromProviderSpec(machine.Spec.ProviderSpec.Value) + if err != nil { + return nil, "", err + } + + subnetMap[*subnetDesc] = false + } + machines, err = r.maocli.MachineV1beta1().Machines(machineSetsNamespace).List(ctx, metav1.ListOptions{LabelSelector: "machine.openshift.io/cluster-api-machine-role=master"}) + if err != nil { + return nil, "", err + } + for _, machine := range machines.Items { + var subnetDesc *subnetDescriptor // declared here due to := rescoping of the masterResourceGroup variable below + masterResourceGroup, subnetDesc, err = r.getDescriptorFromProviderSpec(machine.Spec.ProviderSpec.Value) + if err != nil { + return nil, "", err + } + subnetMap[*subnetDesc] = true + } + if masterResourceGroup == nil { + return nil, "", fmt.Errorf("master resource group not found") + } + return subnetMap, *masterResourceGroup, nil +} + +func (r *AzureNSGReconciler) getDescriptorFromProviderSpec(providerSpec *runtime.RawExtension) (*string, *subnetDescriptor, error) { + var spec azureproviderv1beta1.AzureMachineProviderSpec + err := json.Unmarshal(providerSpec.Raw, &spec) + if err != nil { + return nil, nil, err + } + return &spec.ResourceGroup, &subnetDescriptor{ + resourceGroup: spec.NetworkResourceGroup, + vnetName: spec.Vnet, + subnetName: spec.Subnet, + }, nil +} diff --git a/pkg/operator/controllers/azurensg/subnetnsg_test.go b/pkg/operator/controllers/azurensg/subnetnsg_test.go new file mode 100644 index 000000000..2490d0b34 --- /dev/null +++ b/pkg/operator/controllers/azurensg/subnetnsg_test.go @@ -0,0 +1,219 @@ +package azurensg + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "fmt" + "strings" + "testing" + + mgmtnetwork "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-07-01/network" + "github.com/Azure/go-autorest/autorest/azure" + "github.com/Azure/go-autorest/autorest/to" + "github.com/golang/mock/gomock" + machinev1beta1 "github.com/openshift/machine-api-operator/pkg/apis/machine/v1beta1" + maofake "github.com/openshift/machine-api-operator/pkg/generated/clientset/versioned/fake" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + + "github.com/Azure/ARO-RP/pkg/api" + utillog "github.com/Azure/ARO-RP/pkg/util/log" + mock_network "github.com/Azure/ARO-RP/pkg/util/mocks/azureclient/mgmt/network" +) + +var ( + vnet = azure.Resource{ + SubscriptionID: "0000000-0000-0000-0000-000000000000", + ResourceGroup: "vnet-rg", + ResourceName: "vnet-name", + } +) + +func TestEnsureSubnetNSG(t *testing.T) { + r := AzureNSGReconciler{log: utillog.GetLogger()} + for _, tt := range []struct { + name string + nsgname string + architectureVersion api.ArchitectureVersion + expectedValue string + modifySubnet func(*mgmtnetwork.Subnet) + expectedErr error + expectUpdate bool + }{ + { + name: "mismatched NSG in Architecture v1", + nsgname: "someotherv1-nsg", + architectureVersion: api.ArchitectureVersionV1, + expectedValue: "/subscriptions/0000000-0000-0000-0000-000000000000/resourceGroups/resourcegroup/providers/Microsoft.Network/networkSecurityGroups/infraid-node-nsg", + expectUpdate: true, + }, + { + name: "correct NSG in Architecture v2", + nsgname: "infraid-nsg", + architectureVersion: api.ArchitectureVersionV2, + expectedValue: "/subscriptions/0000000-0000-0000-0000-000000000000/resourceGroups/resourcegroup/providers/Microsoft.Network/networkSecurityGroups/infraid-nsg", + expectUpdate: false, + }, + { + name: "mismatched NSG in Architecture v2", + nsgname: "someotherv2-nsg", + architectureVersion: api.ArchitectureVersionV2, + expectedValue: "/subscriptions/0000000-0000-0000-0000-000000000000/resourceGroups/resourcegroup/providers/Microsoft.Network/networkSecurityGroups/infraid-nsg", + expectUpdate: true, + }, + { + name: "missing fields", + nsgname: "someotherv2-nsg", + architectureVersion: api.ArchitectureVersionV2, + expectedValue: "/subscriptions/0000000-0000-0000-0000-000000000000/resourceGroups/resourcegroup/providers/Microsoft.Network/networkSecurityGroups/infraid-nsg", + modifySubnet: func(subnet *mgmtnetwork.Subnet) { + subnet.SubnetPropertiesFormat = nil + }, + expectedErr: fmt.Errorf("received nil, expected a value in SubnetProperties when trying to Get subnet vnet-name/subnet in resource group vnet-rg"), + expectUpdate: false, + }, + } { + t.Run(tt.name, func(t *testing.T) { + controller := gomock.NewController(t) + defer controller.Finish() + fakeSubnet := mgmtnetwork.Subnet{SubnetPropertiesFormat: &mgmtnetwork.SubnetPropertiesFormat{ + NetworkSecurityGroup: &mgmtnetwork.SecurityGroup{ + ID: to.StringPtr("/subscriptions/0000000-0000-0000-0000-000000000000/resourceGroups/resourcegroup/providers/Microsoft.Network/networkSecurityGroups/" + tt.nsgname), + }, + }} + if tt.modifySubnet != nil { + tt.modifySubnet(&fakeSubnet) + } + subnetsClient := mock_network.NewMockSubnetsClient(controller) + subnetsClient.EXPECT().Get(context.Background(), "vnet-rg", "vnet-name", "subnet", ""). + Return(fakeSubnet, nil) + if tt.expectUpdate { + subnetsClient.EXPECT().CreateOrUpdateAndWait(context.Background(), "vnet-rg", "vnet-name", "subnet", fakeSubnet). + Return(nil) + } + err := r.ensureSubnetNSG(context.Background(), subnetsClient, vnet.SubscriptionID, "resourcegroup", "infraid", tt.architectureVersion, vnet.ResourceGroup, vnet.ResourceName, "subnet", true) + if err != nil { + if tt.expectedErr == nil { + t.Fatal(err) + } + if !strings.EqualFold(tt.expectedErr.Error(), err.Error()) { + t.Errorf("Expected Error %s, got %s when processing %s testcase", tt.expectedErr.Error(), err.Error(), tt.name) + } + return + } + if !strings.EqualFold(tt.expectedValue, *fakeSubnet.SubnetPropertiesFormat.NetworkSecurityGroup.ID) { + t.Errorf("Expected NSG ID %s, got %s when processing %s testcase", tt.expectedValue, *fakeSubnet.SubnetPropertiesFormat.NetworkSecurityGroup.ID, tt.name) + } + }) + } +} + +func TestGetSubnets(t *testing.T) { + r := AzureNSGReconciler{log: utillog.GetLogger()} + for _, tt := range []struct { + name string + machinelabel string + expectedMap map[subnetDescriptor]bool + expectedMasterRG string + modify func(*machinev1beta1.Machine, *machinev1beta1.Machine) + expectedErr error + }{ + { + name: "main path", + expectedMap: map[subnetDescriptor]bool{ + { + resourceGroup: "netRG", + vnetName: "workerVnet", + subnetName: "workerSubnet", + }: false, + { + resourceGroup: "netRG", + vnetName: "masterVnet", + subnetName: "masterSubnet", + }: true, + }, + expectedMasterRG: "masterRG", + modify: func(worker *machinev1beta1.Machine, master *machinev1beta1.Machine) {}, + }, + { + name: "missing providerSpec", + expectedMap: nil, + expectedMasterRG: "", + modify: func(worker *machinev1beta1.Machine, master *machinev1beta1.Machine) { + master.Spec.ProviderSpec.Value.Raw = []byte("") + }, + expectedErr: fmt.Errorf("unexpected end of JSON input"), + }, + { + name: "missing master nodes", + expectedMap: nil, + expectedMasterRG: "masterRG", + modify: func(worker *machinev1beta1.Machine, master *machinev1beta1.Machine) { + master.Labels = map[string]string{} + }, + expectedErr: fmt.Errorf("master resource group not found"), + }, + } { + t.Run(tt.name, func(t *testing.T) { + masterMachine := machinev1beta1.Machine{ + ObjectMeta: metav1.ObjectMeta{ + Name: "master-0", + Namespace: "openshift-machine-api", + Labels: map[string]string{"machine.openshift.io/cluster-api-machine-role": "master"}, + }, + Spec: machinev1beta1.MachineSpec{ + ProviderSpec: machinev1beta1.ProviderSpec{ + Value: &runtime.RawExtension{ + Raw: []byte("{\"resourceGroup\":\"masterRG\",\"publicIP\":false,\"osDisk\":{\"diskSizeGB\": 1024,\"managedDisk\":{\"storageAccountType\": \"Premium_LRS\"},\"osType\":\"Linux\"},\"image\":{\"offer\": \"aro4\",\"publisher\": \"azureopenshift\", \"resourceID\": \"\", \"sku\": \"aro_43\", \"version\": \"43.81.20200311\"},\"networkResourceGroup\":\"netRG\",\"vnet\":\"masterVnet\",\"subnet\":\"masterSubnet\"}"), + }, + }, + }, + } + workerMachine := machinev1beta1.Machine{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-0", + Namespace: "openshift-machine-api", + Labels: map[string]string{"machine.openshift.io/cluster-api-machine-role": "worker"}, + }, + Spec: machinev1beta1.MachineSpec{ + ProviderSpec: machinev1beta1.ProviderSpec{ + Value: &runtime.RawExtension{ + Raw: []byte("{\"resourceGroup\":\"workerRG\",\"publicIP\":false,\"osDisk\":{\"diskSizeGB\": 1024,\"managedDisk\":{\"storageAccountType\": \"Premium_LRS\"},\"osType\":\"Linux\"},\"image\":{\"offer\": \"aro4\",\"publisher\": \"azureopenshift\", \"resourceID\": \"\", \"sku\": \"aro_43\", \"version\": \"43.81.20200311\"},\"networkResourceGroup\":\"netRG\",\"vnet\":\"workerVnet\",\"subnet\":\"workerSubnet\"}"), + }, + }, + }, + } + tt.modify(&workerMachine, &masterMachine) + r.maocli = maofake.NewSimpleClientset(&workerMachine, &masterMachine) + subnetMap, masterRG, err := r.getSubnets(context.Background()) + if err != nil { + if tt.expectedErr == nil { + t.Fatal(err) + } + if !strings.EqualFold(err.Error(), tt.expectedErr.Error()) { + t.Errorf("Expected Error %s, got %s when processing %s testcase", tt.expectedErr.Error(), err.Error(), tt.name) + } + return + } + if !strings.EqualFold(tt.expectedMasterRG, masterRG) { + t.Errorf("Expected Master Resource Group %s, got %s when processing %s testcase", tt.expectedMasterRG, masterRG, tt.name) + } + if tt.expectedMap != nil { + if len(tt.expectedMap) != len(subnetMap) { + t.Errorf("Expected Map length %d, doesn't match result map length %d when processing %s testcase", len(tt.expectedMap), len(subnetMap), tt.name) + } + for subnet := range tt.expectedMap { + value, present := subnetMap[subnet] + if !present { + t.Errorf("Subnet %s, %s, %s expected but not present in result when processing %s testcase", subnet.resourceGroup, subnet.vnetName, subnet.subnetName, tt.name) + } + if tt.expectedMap[subnet] != value { + t.Errorf("Value of isMaster boolean doesn't match for subnet %s, %s, %s when processing %s testcase", subnet.resourceGroup, subnet.vnetName, subnet.subnetName, tt.name) + } + } + } + }) + } +} diff --git a/pkg/operator/controllers/checker/const.go b/pkg/operator/controllers/checker/const.go index a4f70c304..72dd43a06 100644 --- a/pkg/operator/controllers/checker/const.go +++ b/pkg/operator/controllers/checker/const.go @@ -4,7 +4,5 @@ package checker // Licensed under the Apache License 2.0. const ( - azureCredentialSecretNamespace = "kube-system" - azureCredentialSecretName = "azure-credentials" - machineSetsNamespace = "openshift-machine-api" + machineSetsNamespace = "openshift-machine-api" ) diff --git a/pkg/operator/controllers/checker/serviceprincipalchecker.go b/pkg/operator/controllers/checker/serviceprincipalchecker.go index 3e2b1f680..9612d269d 100644 --- a/pkg/operator/controllers/checker/serviceprincipalchecker.go +++ b/pkg/operator/controllers/checker/serviceprincipalchecker.go @@ -20,6 +20,7 @@ import ( aroclient "github.com/Azure/ARO-RP/pkg/operator/clientset/versioned" "github.com/Azure/ARO-RP/pkg/operator/controllers" "github.com/Azure/ARO-RP/pkg/util/aad" + "github.com/Azure/ARO-RP/pkg/util/clusterauthorizer" ) type ServicePrincipalChecker struct { @@ -67,12 +68,12 @@ func (r *ServicePrincipalChecker) Check(ctx context.Context) error { return err } - azCred, err := azCredentials(ctx, r.kubernetescli) + azCred, err := clusterauthorizer.AzCredentials(ctx, r.kubernetescli) if err != nil { return err } - _, err = aad.GetToken(ctx, r.log, azCred.clientID, azCred.clientSecret, azCred.tenantID, azEnv.ActiveDirectoryEndpoint, azEnv.ResourceManagerEndpoint) + _, err = aad.GetToken(ctx, r.log, string(azCred.ClientID), string(azCred.ClientSecret), string(azCred.TenantID), azEnv.ActiveDirectoryEndpoint, azEnv.ResourceManagerEndpoint) if err != nil { updateFailedCondition(cond, err) } @@ -82,7 +83,7 @@ func (r *ServicePrincipalChecker) Check(ctx context.Context) error { return err } - err = spDynamic.ValidateServicePrincipal(ctx, azCred.clientID, azCred.clientSecret, azCred.tenantID) + err = spDynamic.ValidateServicePrincipal(ctx, string(azCred.ClientID), string(azCred.ClientSecret), string(azCred.TenantID)) if err != nil { updateFailedCondition(cond, err) } @@ -90,27 +91,6 @@ func (r *ServicePrincipalChecker) Check(ctx context.Context) error { return controllers.SetCondition(ctx, r.arocli, cond, r.role) } -type credentials struct { - clientID string - clientSecret string - tenantID string -} - -func azCredentials(ctx context.Context, kubernetescli kubernetes.Interface) (*credentials, error) { - var creds credentials - - mysec, err := kubernetescli.CoreV1().Secrets(azureCredentialSecretNamespace).Get(ctx, azureCredentialSecretName, metav1.GetOptions{}) - if err != nil { - return nil, err - } - - creds.clientID = string(mysec.Data["azure_client_id"]) - creds.clientSecret = string(mysec.Data["azure_client_secret"]) - creds.tenantID = string(mysec.Data["azure_tenant_id"]) - - return &creds, nil -} - func updateFailedCondition(cond *status.Condition, err error) { cond.Status = corev1.ConditionFalse if tErr, ok := err.(*api.CloudError); ok { diff --git a/pkg/operator/controllers/const.go b/pkg/operator/controllers/const.go index 0f4fb148a..5a802aaee 100644 --- a/pkg/operator/controllers/const.go +++ b/pkg/operator/controllers/const.go @@ -5,6 +5,7 @@ package controllers const ( AlertwebhookControllerName = "Alertwebhook" + AzureNSGControllerName = "AzureNSG" ClusterOperatorAROName = "ClusterOperatorARO" GenevaLoggingControllerName = "GenevaLogging" PullSecretControllerName = "PullSecret" diff --git a/pkg/operator/deploy/bindata.go b/pkg/operator/deploy/bindata.go index 53535ff82..ec68300c6 100644 --- a/pkg/operator/deploy/bindata.go +++ b/pkg/operator/deploy/bindata.go @@ -86,7 +86,7 @@ func (fi bindataFileInfo) Sys() interface{} { return nil } -var _aroOpenshiftIo_clustersYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xb4\x58\x4b\x8f\xe3\xc8\x0d\xbe\xfb\x57\x10\x9d\xc3\x1c\xd2\x56\xcf\x20\x08\x90\xf8\x36\xe8\xd9\x4d\x8c\xdd\x64\x1b\x33\x8d\xb9\x6c\xef\x81\x2e\xd1\x32\xd3\xa5\x2a\x85\x45\xb9\xc7\x13\xe4\xbf\x07\x2c\x49\x7e\x4a\xfd\x58\x20\x75\x31\x54\x55\x7c\x93\x1f\x59\x9e\xcd\xe7\xf3\x19\x36\xfc\x95\x24\x71\x0c\x0b\xc0\x86\xe9\x9b\x52\xb0\xaf\x54\x3c\xfe\x25\x15\x1c\x6f\xb6\x1f\x66\x8f\x1c\xca\x05\xdc\xb6\x49\x63\xfd\x99\x52\x6c\xc5\xd1\x27\x5a\x73\x60\xe5\x18\x66\x35\x29\x96\xa8\xb8\x98\x01\x60\x08\x51\xd1\xb6\x93\x7d\x02\xb8\x18\x54\xa2\xf7\x24\xf3\x8a\x42\xf1\xd8\xae\x68\xd5\xb2\x2f\x49\x32\xf3\x41\xf4\xf6\x7d\xf1\xe7\xe2\xfd\x0c\xc0\x09\x65\xf2\x7b\xae\x29\x29\xd6\xcd\x02\x42\xeb\xfd\x0c\x20\x60\x4d\x0b\x70\xbe\x4d\x4a\x92\x0a\x94\x58\xc4\x86\x42\xda\xf0\x5a\x0b\x8e\xb3\xd4\x90\x33\x99\x95\xc4\xb6\x59\xc0\xc5\x79\xc7\xa1\x57\xab\x37\xa9\x63\x96\x77\x3c\x27\xfd\xe9\x78\xf7\x67\x4e\x9a\x4f\x1a\xdf\x0a\xfa\x83\xe8\xbc\x99\x38\x54\xad\x47\xd9\x6f\xcf\x00\x92\x8b\x0d\x1d\x73\xed\xcd\xcb\x32\xe7\xbd\x01\xdb\x0f\xe8\x9b\x0d\x7e\xe8\xb8\xb8\x0d\xd5\xd8\xa9\x04\x60\xea\x7e\xbc\x5b\x7e\xfd\xd3\x97\x93\x6d\x80\x92\x92\x13\x6e\x34\xbb\xaa\x67\x0f\x9c\x40\x37\x04\xdd\x5d\x58\x47\xc9\x9f\x83\x92\xf0\xf1\x6e\xb9\xa7\x6f\x24\x36\x24\xca\x83\xf5\xdd\x3a\x0a\xfd\xd1\xee\x99\xb4\x77\xa6\x50\x77\x0b\x4a\x8b\x39\x75\x62\x7b\xd3\xa8\xec\x6d\x80\xb8\x06\xdd\x70\x02\xa1\x46\x28\x51\xe8\xb2\xe0\x84\x31\xd8\x25\x0c\x10\x57\xff\x22\xa7\x05\x7c\x21\x31\x36\x90\x36\xb1\xf5\xa5\xa5\xca\x96\x44\x41\xc8\xc5\x2a\xf0\xf7\x3d\xef\x04\x1a\xb3\x50\x8f\x4a\x7d\x50\x0e\x8b\x83\x92\x04\xf4\xb0\x45\xdf\xd2\x35\x60\x28\xa1\xc6\x1d\x08\x99\x14\x68\xc3\x11\xbf\x7c\x25\x15\xf0\x8f\x28\x04\x1c\xd6\x71\x01\x1b\xd5\x26\x2d\x6e\x6e\x2a\xd6\x21\xe5\x5d\xac\xeb\x36\xb0\xee\x6e\x72\xf6\xf2\xaa\xd5\x28\xe9\xa6\xa4\x2d\xf9\x9b\xc4\xd5\x1c\xc5\x6d\x58\xc9\x69\x2b\x74\x83\x0d\xcf\xb3\xea\x21\xa7\x7d\x51\x97\x7f\x90\xbe\x48\xd2\xbb\x13\x5d\x75\x67\xe9\x91\x54\x38\x54\x47\x07\x39\x17\x9f\x89\x80\x65\xa5\x45\x1b\x7b\xd2\xce\x8a\x83\xa3\x6d\xcb\xbc\xf3\xf9\x87\x2f\xf7\x30\x88\xce\xc1\x38\xf7\x7e\xf6\xfb\x81\x30\x1d\x42\x60\x0e\xe3\xb0\x26\xe9\x82\xb8\x96\x58\x67\x9e\x14\xca\x26\x72\xd0\x3e\xb7\x98\xc2\xb9\xfb\x53\xbb\xaa\x59\x2d\xee\xff\x6e\x29\xa9\xc5\xaa\x80\xdb\x8c\x03\xb0\x22\x68\x9b\x12\x95\xca\x02\x96\x01\x6e\xb1\x26\x7f\x8b\x89\xfe\xef\x01\x30\x4f\xa7\xb9\x39\xf6\x75\x21\x38\x86\xb0\xf3\xcb\x9d\xd7\x8e\x0e\x06\xa0\x99\x88\x57\x5f\x9f\x5f\x1a\x72\x27\x15\x53\x52\x62\xb1\x9c\x56\x54\xb2\x4a\x38\x46\x9f\x61\x8d\x57\xaa\x2d\x74\xf2\x29\xd6\xc8\xe1\xfc\x60\xd2\x28\xe8\x6a\x7c\x19\x74\x79\xf7\x36\xa2\xef\x3f\x84\x2d\x4b\x0c\x35\x05\x7d\x13\x65\xf9\x76\x0d\xd7\x84\x16\xc5\x0b\x7b\xcf\xbc\xfa\x63\x7f\xed\xc4\xad\x1f\x3f\xff\x62\xa0\x29\xa8\x51\x06\x46\x50\x19\x48\x4c\x68\x70\x11\x4b\x5b\x15\x05\xda\xe2\xcf\xb1\xaa\x38\x54\x97\x6a\x4c\x87\x04\xba\xfe\xb6\xe6\x6a\x14\x47\xf7\x0c\x50\x0d\xa1\x16\xf0\xee\xd7\xf7\xf3\xbf\xfe\xf6\xc7\xa2\xfb\x79\x37\x7a\xf9\x19\x4f\xd9\xaa\x63\x60\x8d\x76\xf8\xb7\xdb\x2f\xcf\x46\xc9\x16\x85\xb6\x1e\x3f\x99\xc3\x27\xc6\x2a\xc4\xa4\xec\xd2\x9d\xc4\x72\xe2\xd6\xfd\x25\xde\xbe\x42\xcf\x67\x9c\xcd\xa1\x12\x4a\xe9\x8d\x19\xd9\x61\x3c\xe9\xed\x86\xdc\x23\xc9\x5b\x83\xd4\x8a\x1f\xdd\x07\x60\xa5\x7a\xe2\xe8\xc5\x58\x0c\x17\x50\x04\x77\x6f\xf1\x81\x8f\x2e\x37\xc7\x37\xb9\x60\x00\xf6\x65\xf9\x42\xa5\x0c\x13\xda\xf2\xd3\x30\x22\x7c\xfc\x6e\x75\x71\x60\xd0\xf5\x6a\x3a\x9a\x5c\x5e\xad\xc5\x36\x90\x8e\x69\x30\x49\x32\x85\xa1\x8a\xda\xa6\x57\xa0\x68\xbe\x77\x82\xa3\x71\x95\xac\x69\xfd\x6e\x20\x75\x31\x94\x7c\x34\xa1\x4e\xab\xb0\xbf\xd8\x77\x5f\xd2\x2c\x6d\xd8\x06\x0e\x49\x31\x38\x4a\xc5\x05\xa3\xc9\xbc\x3a\x91\x70\x75\xe0\x75\x68\xca\xdd\x84\x64\x36\xe6\x24\x39\x99\x99\xde\x5d\x02\xdb\xe0\x4d\x2a\x8e\x15\x46\x21\xa3\xda\x8f\xf3\x50\x93\xdb\x60\xe0\x54\xe7\x5a\x0a\x25\x95\x36\x54\x59\x83\x4e\x34\x5e\xfc\x4f\x1b\x0a\x7d\xe3\x52\x64\x9f\xf6\x8a\x1c\x54\x33\x29\xd6\xe7\x11\x1a\xe1\x28\x0c\x8f\x21\x3e\x05\x88\x02\x4f\x36\xd1\x8d\xb2\xcd\xf7\x9b\xc6\xef\x4c\x3e\x7a\x7f\xf0\x62\x16\x00\x15\x6f\x29\x80\xcd\x3c\x05\x3c\x84\x63\x9b\xba\x31\x71\x94\xe9\x8a\x00\xcb\xde\x26\xfa\xd6\x78\x76\xac\x7e\xd7\x4d\x94\xbb\xa3\x5c\x00\xdd\xa0\x9a\xc9\x92\xf2\x9c\xe8\x62\xdd\xc4\x60\x5e\x1f\x65\xeb\xb2\x1b\x57\xb1\x55\x10\xd4\x4d\x9e\x8e\x30\xe4\x51\x87\xa5\x1b\xbb\x62\xa2\x13\xfe\xd9\xa7\x79\x92\x92\x09\xbf\xe6\xd9\x2a\x66\x6e\x47\xbe\x4c\x05\xfc\x12\x1c\xf5\x99\x5e\x5e\x67\xcf\xd7\x84\xc1\xc4\x64\xc7\xec\x3d\x31\xa1\x6a\x80\x7e\xe4\xb2\x40\x57\x54\x02\xca\x8a\x55\x50\xd8\xef\x60\x0e\x6c\x67\x2e\xd6\x94\xa0\x41\xd1\x01\x03\x3e\xde\x2d\xf3\xc8\x3c\xca\x74\x83\x5d\xc9\x25\xac\x09\x56\xe8\x1e\x9f\x50\xca\x34\xcf\xae\x5b\x47\xe9\xbe\xcc\x87\xa8\xbc\x62\xcf\x9a\x5d\xee\x48\x82\x05\x73\x94\x25\x86\x5d\x6f\xfc\x99\x16\xc5\xd5\xc8\xfd\xe7\x61\x1d\xc0\x63\xd2\x7b\xc1\x90\x78\x78\x33\x4e\x61\xf9\x3a\x4a\x8d\xba\x00\x9b\x46\xe7\xca\x35\xfd\x5e\xcc\xaf\x29\x25\xac\x26\xe5\xbc\x48\x2f\x84\x69\x6a\x52\x98\x02\xa0\xcf\x99\xc6\x50\xe8\xac\x78\x11\x62\xa0\xf9\x53\x94\xf2\xfa\x30\x5d\x4f\xb0\x86\xb3\xa7\xd9\xbe\x0b\xa0\x52\x15\x65\x67\xdf\x0e\xdb\x44\xfb\x83\x56\x84\x82\xf6\x58\x7d\x89\x71\xc3\x5a\xea\x88\x66\x06\x2b\xc0\x21\xe7\x03\x1b\xcf\x56\x9b\x56\xaf\x21\xb5\x6e\x03\x98\xb2\xde\x9e\xc3\xb4\xb2\x8f\xed\x8a\x9c\x7a\xa8\x0c\x75\x7b\x62\xcb\x3b\x0e\x90\xda\xba\x46\xe1\xef\xb9\x34\x5c\xa7\x66\x8f\x1f\xd9\x80\x49\x5d\x5f\x0c\xce\x58\x5b\x7a\x03\x79\xbe\xf0\x9a\xc8\x1e\x80\xff\x7e\xd7\xd0\xd0\xa7\x8d\x7c\xef\xfc\x7d\x67\x98\x2a\x4e\x5b\x46\xb8\x6b\xd8\xa1\xf7\x3b\x83\x88\x21\x05\x4a\xb0\x9c\x30\x20\x4e\x9b\x28\x0a\xcd\x46\xf2\xb3\xeb\x18\x50\x27\x99\xe6\xc7\xd3\xf0\x28\xe7\x50\xb2\x65\x48\xdf\x6d\xb9\x6b\x09\x0f\x57\xb8\x0a\x56\x51\x7e\xae\xd2\xd2\xc3\x15\x34\xd1\xa3\xb0\xee\xa6\xd3\xe4\xc7\x28\x40\xdf\xb0\x6e\x3c\x5d\x03\x9f\x5b\x39\xc8\x49\x5d\xdf\x41\x63\xc8\x6e\xd7\x65\xd6\x16\x3d\x97\xd7\xd3\x0a\x67\x8d\x38\x41\xbe\xf7\x70\x05\x0e\x53\x76\x6a\x23\x71\x85\x2b\x6b\x35\x1b\x6b\x54\x52\x5f\x43\x8a\xa7\x82\x27\x99\xf6\xf6\x1b\x9e\xa2\xf7\x54\xc2\xc3\xd5\x32\xf4\x02\x46\xb1\x0a\x5e\xce\x90\xae\x71\xd0\xc8\xfc\x64\xb3\x76\x97\x7c\xa3\x47\xc6\x77\xe4\xe0\x99\x19\xf3\xb9\xe1\x74\x78\x32\x4d\xbe\x5a\x9e\x1d\x43\xcb\xbf\xa3\xfe\x44\xbb\x74\xd7\x61\xc9\x25\xf5\xe4\xec\xf3\x8a\x37\xc3\xa5\xba\xa3\x36\x5e\x6c\x76\x23\xe1\x02\x2c\x1b\xbb\x0d\x8d\x62\x30\x7d\xb4\xd3\xae\xf6\xff\xcb\x0c\xda\xf5\xf5\x0e\xff\xf9\xef\xec\x50\xfa\xe8\x1c\x35\x4a\xe5\x3f\xcf\xff\x2e\xbc\xba\x3a\xf9\x3f\x30\x7f\x1e\x4d\x93\xf0\xeb\x6f\xb3\x4e\x30\x95\x5f\x87\x7f\xfe\x6c\xf3\x7f\x01\x00\x00\xff\xff\xe8\xfb\xb5\xc1\x69\x15\x00\x00") +var _aroOpenshiftIo_clustersYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xb4\x58\x4f\x6f\xeb\xb8\x11\xbf\xfb\x53\x0c\xd2\xc3\x3b\x34\x56\xf6\xa1\x28\xd0\xfa\x16\xe4\xed\x6e\x8d\xdd\xee\x06\x2f\xc1\xbb\x6c\xf6\x30\xa2\xc6\xd2\x34\x14\xa9\x92\x23\xe7\xf9\x15\xfd\xee\xc5\x50\x92\x2d\xdb\x52\x12\x2f\x50\x5e\x0c\x91\xc3\xf9\xff\xe7\x47\x2f\x96\xcb\xe5\x02\x1b\xfe\x42\x21\xb2\x77\x2b\xc0\x86\xe9\xab\x90\xd3\xaf\x98\x3d\xff\x2d\x66\xec\x6f\xb6\x1f\x17\xcf\xec\x8a\x15\xdc\xb5\x51\x7c\xfd\x99\xa2\x6f\x83\xa1\x4f\xb4\x61\xc7\xc2\xde\x2d\x6a\x12\x2c\x50\x70\xb5\x00\x40\xe7\xbc\xa0\x6e\x47\xfd\x04\x30\xde\x49\xf0\xd6\x52\x58\x96\xe4\xb2\xe7\x36\xa7\xbc\x65\x5b\x50\x48\xcc\x07\xd1\xdb\xef\xb2\xbf\x66\xdf\x2d\x00\x4c\xa0\x74\xfd\x91\x6b\x8a\x82\x75\xb3\x02\xd7\x5a\xbb\x00\x70\x58\xd3\x0a\x8c\x6d\xa3\x50\x88\x19\x06\x9f\xf9\x86\x5c\xac\x78\x23\x19\xfb\x45\x6c\xc8\xa8\xcc\x32\xf8\xb6\x59\xc1\xd9\x79\xc7\xa1\x57\xab\x37\xa9\x63\x96\x76\x2c\x47\xf9\x69\xbc\xfb\x33\x47\x49\x27\x8d\x6d\x03\xda\x83\xe8\xb4\x19\xd9\x95\xad\xc5\xb0\xdf\x5e\x00\x44\xe3\x1b\x1a\x73\xed\xcd\x4b\x32\x97\xbd\x01\xdb\x8f\x68\x9b\x0a\x3f\x76\x5c\x4c\x45\x35\x76\x2a\x01\xa8\xba\xb7\xf7\xeb\x2f\x7f\x79\x38\xda\x06\x28\x28\x9a\xc0\x8d\x24\x57\xf5\xec\x81\x23\x48\x45\xd0\xd1\xc2\xc6\x87\xf4\x39\x28\x09\xb7\xf7\xeb\xfd\xfd\x26\xf8\x86\x82\xf0\x60\x7d\xb7\x46\xa1\x1f\xed\x9e\x48\xfb\xa0\x0a\x75\x54\x50\x68\xcc\xa9\x13\xdb\x9b\x46\x45\x6f\x03\xf8\x0d\x48\xc5\x11\x02\x35\x81\x22\xb9\x2e\x0b\x8e\x18\x83\x12\xa1\x03\x9f\xff\x8b\x8c\x64\xf0\x40\x41\xd9\x40\xac\x7c\x6b\x0b\x4d\x95\x2d\x05\x81\x40\xc6\x97\x8e\xbf\xed\x79\x47\x10\x9f\x84\x5a\x14\xea\x83\x72\x58\xec\x84\x82\x43\x0b\x5b\xb4\x2d\x5d\x03\xba\x02\x6a\xdc\x41\x20\x95\x02\xad\x1b\xf1\x4b\x24\x31\x83\x7f\xfa\x40\xc0\x6e\xe3\x57\x50\x89\x34\x71\x75\x73\x53\xb2\x0c\x29\x6f\x7c\x5d\xb7\x8e\x65\x77\x93\xb2\x97\xf3\x56\x7c\x88\x37\x05\x6d\xc9\xde\x44\x2e\x97\x18\x4c\xc5\x42\x46\xda\x40\x37\xd8\xf0\x32\xa9\xee\x52\xda\x67\x75\xf1\xa7\xd0\x17\x49\xfc\x70\xa4\xab\xec\x34\x3d\xa2\x04\x76\xe5\xe8\x20\xe5\xe2\x2b\x11\xd0\xac\xd4\x68\x63\x7f\xb5\xb3\xe2\xe0\x68\xdd\x52\xef\x7c\xfe\xfe\xe1\x11\x06\xd1\x29\x18\xa7\xde\x4f\x7e\x3f\x5c\x8c\x87\x10\xa8\xc3\xd8\x6d\x28\x74\x41\xdc\x04\x5f\x27\x9e\xe4\x8a\xc6\xb3\x93\x3e\xb7\x98\xdc\xa9\xfb\x63\x9b\xd7\x2c\x1a\xf7\x7f\xb7\x14\x45\x63\x95\xc1\x5d\xea\x03\x90\x13\xb4\x4d\x81\x42\x45\x06\x6b\x07\x77\x58\x93\xbd\xc3\x48\xff\xf7\x00\xa8\xa7\xe3\x52\x1d\xfb\xbe\x10\x8c\x5b\xd8\x29\x71\xe7\xb5\xd1\xc1\xd0\x68\x66\xe2\xd5\xd7\xe7\x43\x43\xe6\xa8\x62\x0a\x8a\x1c\x34\xa7\x05\x85\xb4\x12\xc6\xdd\x67\x58\xd3\x95\xaa\x0b\x4d\xf8\xe4\x6b\x64\x77\x7a\x30\x6b\x14\x74\x35\xbe\x76\xb2\xbe\xbf\xec\xd2\xc8\xbb\x93\x1d\xe2\x70\x5f\x8b\xaf\x3c\xb1\x01\x00\xbf\x7d\xef\xb6\x1c\xbc\xab\xc9\xc9\x45\xa2\x8b\xcb\x4d\xdc\x10\xaa\xa2\x67\x0e\x3b\x09\xcb\x0f\x3d\xd9\x51\x5c\x6e\x3f\xff\xaa\x5d\x37\xa0\xf8\x30\x30\x82\x52\xbb\xcc\x19\xb3\xf9\xc8\xe8\xd2\x1e\xe3\x0c\x5b\xfa\xe5\xe1\xc7\x49\x82\xc1\x84\xdc\x7b\x4b\x78\xda\x17\x67\x73\x4d\x57\x49\x8e\xb6\xf8\xb3\x2f\x4b\x76\xe5\x39\xef\xd7\x15\x33\xde\x6d\xb8\x9c\x8d\x62\x62\x80\xa2\x1d\x74\x05\x1f\x7e\xfb\x6e\xf9\xf7\xdf\xff\x9c\x75\x3f\x1f\x5e\xb1\x62\x32\x10\xba\x6a\xef\x58\xbc\x1e\xfe\x78\xf7\xf0\x6a\x12\xe8\x22\xd7\xd6\xd3\x27\x4b\xf8\xc4\x58\x3a\x1f\x85\x4d\xbc\x0f\xbe\x98\xa1\x7a\x3c\x9f\x07\xef\xd0\xf3\x15\x67\xb3\xdb\x04\x5c\x17\x17\x65\x20\xbb\x32\x50\x8c\x17\x56\x59\x37\xb7\x48\xee\x2a\x32\xcf\x14\x2e\x0d\x6c\x1b\xec\x4c\xa2\xb1\x50\x3d\x73\xf4\x66\xfc\x06\x02\x0c\x01\x77\x97\xf8\xcd\x7a\x93\x06\xfe\x45\x2e\x18\x86\xd5\x94\xbf\x8f\x8a\x77\x40\x9d\xeb\x4f\x03\xec\xb9\xfd\xa6\xa5\x7a\x60\xd0\xe1\x0f\x1a\xa1\xb1\x77\x6b\xb1\x75\x24\x17\x45\x7c\x6e\x2e\x08\x4a\x1b\xdf\x31\x19\x12\xdd\xd1\x6c\xf0\x79\xd4\x41\xfc\x87\x87\x83\xf1\xae\xe0\x11\xea\x9e\x57\x61\x4f\xd8\x23\x0a\x92\x24\x6d\xd8\x06\x76\x51\xd0\x19\x8a\xd9\x19\xa3\xd9\xbc\x3a\x92\x70\x75\xe0\x75\x00\x1a\x1d\xea\x53\x1b\x53\x92\x1c\xe1\xc0\x0f\xe7\xbd\x76\xf0\x26\x65\x63\x85\x31\x90\xde\xda\x3f\x51\xa0\x26\x53\xa1\xe3\x58\xa7\x5a\x72\x05\x15\x0a\x14\x15\x74\x44\x9a\x6e\x18\x2f\x15\xb9\x7e\x18\x0b\xb2\x8d\x7b\x45\x0e\xaa\xa9\x14\xc5\x2e\x08\x4d\x60\x1f\x18\x9e\x9d\x7f\x71\xe0\x03\xbc\x28\x4a\x9d\x64\x9b\xe8\x9b\xc6\xee\x54\x3e\x5a\x7b\xf0\x62\x12\x00\x25\x6f\xc9\x81\xe2\xb8\x0c\x9e\xdc\xd8\xa6\x0e\xfa\x4e\x32\xcd\x09\xb0\xe8\x6d\xa2\xaf\x8d\x65\xc3\x62\x77\x1d\x4a\xde\x8d\x72\x01\xa4\x42\x51\x93\x43\x4c\xd8\xd7\xf8\xba\xf1\x4e\xbd\x3e\xc9\xd6\x24\x37\xe6\xbe\x15\x08\x28\x55\x42\x7c\xe8\x12\x7c\xe3\xd0\x41\x49\x1f\xe9\x88\x7f\xf2\x69\x42\x87\x61\xc6\xaf\x09\x2f\xfa\xc4\x6d\xe4\xcb\x98\xc1\xaf\xce\x50\x9f\xe9\xc5\x75\xf2\x7c\x4d\xe8\x54\x4c\x72\xcc\xde\x13\x33\xaa\x3a\xe8\x61\xa4\x06\xba\xa4\x02\x30\xe4\x2c\x01\x03\xdb\x1d\x2c\x81\xf5\xcc\xf8\x9a\x22\x34\x18\x64\xe8\x01\xb7\xf7\xeb\xf4\x0c\x98\x64\x5a\x61\x57\x72\x11\x6b\x82\x1c\xcd\xf3\x0b\x86\x22\x2e\x93\xeb\x36\x3e\x74\x5f\xea\x43\x14\xce\xd9\xb2\x24\x97\x1b\x0a\x4e\x83\x39\xc9\x12\xdd\xae\x37\xfe\x44\x8b\xec\x6a\x82\xfe\xf5\xb6\x0e\x60\x31\xca\x63\x40\x17\x79\x78\x07\xcf\xf5\xf2\x8d\x0f\x35\xca\x0a\x14\x61\x2f\x85\x6b\xfa\xa3\x3d\xbf\xa6\x18\xb1\x9c\x95\xf3\xe6\xfd\x40\x18\xe7\xd0\xc5\x5c\x03\xfa\x9c\xee\x68\x17\x3a\x29\x5e\x04\xef\x68\xf9\xe2\x43\x71\x7d\x78\x31\xcc\xb0\x86\x93\xe7\xe6\x7e\x0a\xa0\x50\xe9\xc3\x4e\xbf\x0d\xb6\x91\xf6\x07\x6d\x08\xe4\xa4\xef\xd5\xe7\x3d\x6e\x58\x6b\x99\xd0\x4c\xdb\x0a\xb0\x4b\xf9\xc0\xca\xb3\x95\xa6\x95\x6b\x88\xad\xa9\x00\x63\xd2\xdb\xb2\x9b\x57\xf6\xb9\xcd\xc9\x88\x85\x52\xbb\x6e\x7f\x59\xf3\x8e\x1d\xc4\xb6\xae\x31\xf0\xb7\x54\x1a\xa6\x53\xb3\xef\x1f\xc9\x80\x59\x5d\xdf\x0c\xce\xd4\x58\xba\xe0\x7a\x22\x78\x4f\x64\x0f\x8d\xff\x71\xd7\xd0\x30\xa7\xf5\xfa\xde\xf9\xfb\xc9\x30\x57\x9c\xba\xf4\xe2\xae\x61\x83\xd6\xee\xb4\x45\x0c\x29\x50\x80\xe6\x84\x36\xe2\x58\xf9\x20\xd0\x54\x21\x3d\x25\xc7\x0d\x75\x96\x69\x7a\x10\x0e\x7f\x34\xb0\x2b\x58\x33\xa4\x9f\xb6\xdc\x8d\x84\xa7\x2b\xcc\x9d\x56\x94\x5d\x4a\x68\xe9\xe9\x0a\x1a\x6f\x31\xb0\xec\xe6\xd3\xe4\x07\x1f\x80\xbe\x62\xdd\x58\xba\x06\x3e\xb5\x72\x90\x13\xbb\xb9\x83\xca\x90\xcd\xae\xcb\xac\x2d\x5a\x2e\xae\xe7\x15\x4e\x1a\x71\x84\x44\xf7\x74\x05\x06\x63\x72\x6a\x13\x7c\x8e\xb9\x8e\x9a\x4a\x07\x55\xa8\xaf\x21\xfa\x63\xc1\xb3\x4c\x7b\xfb\xb5\x9f\xa2\xb5\x54\xc0\xd3\xd5\xda\xf5\x02\x26\x7b\x15\xbc\x9d\x21\xdd\xe0\xa0\x09\xfc\xa4\xf8\xbc\x4b\xbe\xc9\x23\xe5\x3b\x71\xf0\x0a\xc6\x7c\x0d\x9c\x0e\xaf\xb8\x37\xde\xab\x33\x30\xb4\xf8\x07\xca\x4f\xb4\x8b\xf7\x5d\x2f\x39\xbf\x3d\x8b\x7d\xde\xf1\xce\x38\x57\x77\xd2\xc6\xb3\xcd\x0e\x12\xae\x40\xb3\xb1\xdb\x10\x1f\xb4\x4d\x8f\x76\xda\x7c\xff\x5f\xd3\xa0\x5d\x5f\xef\xf0\x9f\xff\x2e\x0e\xa5\x8f\xc6\x50\x23\x54\xfc\x72\xfa\x17\xe8\xd5\xd5\xd1\x7f\x9c\xe9\x73\x84\x26\xe1\xb7\xdf\x17\x9d\x60\x2a\xbe\x0c\xff\x66\xea\xe6\xff\x02\x00\x00\xff\xff\xa3\xfe\x50\xc7\x3d\x16\x00\x00") func aroOpenshiftIo_clustersYamlBytes() ([]byte, error) { return bindataRead( diff --git a/pkg/operator/deploy/deploy.go b/pkg/operator/deploy/deploy.go index 8f1afb0d2..1b01314c8 100644 --- a/pkg/operator/deploy/deploy.go +++ b/pkg/operator/deploy/deploy.go @@ -168,12 +168,14 @@ func (o *operator) resources() ([]runtime.Object, error) { Name: arov1alpha1.SingletonClusterName, }, Spec: arov1alpha1.ClusterSpec{ - ResourceID: o.oc.ID, - Domain: domain, - ACRDomain: o.env.ACRDomain(), - AZEnvironment: o.env.Environment().Name, - Location: o.env.Location(), - VnetID: vnetID, + ResourceID: o.oc.ID, + Domain: domain, + ACRDomain: o.env.ACRDomain(), + AZEnvironment: o.env.Environment().Name, + Location: o.env.Location(), + InfraID: o.oc.Properties.InfraID, + ArchitectureVersion: int(o.oc.Properties.ArchitectureVersion), + VnetID: vnetID, GenevaLogging: arov1alpha1.GenevaLoggingSpec{ ConfigVersion: o.env.ClusterGenevaLoggingConfigVersion(), MonitoringGCSEnvironment: o.env.ClusterGenevaLoggingEnvironment(), diff --git a/pkg/operator/deploy/staticresources/aro.openshift.io_clusters.yaml b/pkg/operator/deploy/staticresources/aro.openshift.io_clusters.yaml index be7ddbc60..bdee816f0 100644 --- a/pkg/operator/deploy/staticresources/aro.openshift.io_clusters.yaml +++ b/pkg/operator/deploy/staticresources/aro.openshift.io_clusters.yaml @@ -40,12 +40,17 @@ spec: type: string apiIntIP: type: string + architectureVersion: + type: integer azEnvironment: type: string domain: type: string features: description: FeaturesSpec defines ARO operator feature gates + properties: + reconcileNSGs: + type: boolean type: object genevaLogging: properties: @@ -58,6 +63,8 @@ spec: - Test type: string type: object + infraId: + type: string ingressIP: type: string internetChecker: diff --git a/pkg/util/clusterauthorizer/authorizer.go b/pkg/util/clusterauthorizer/authorizer.go new file mode 100644 index 000000000..a51918642 --- /dev/null +++ b/pkg/util/clusterauthorizer/authorizer.go @@ -0,0 +1,55 @@ +package clusterauthorizer + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "fmt" + + "github.com/Azure/go-autorest/autorest/adal" + "github.com/form3tech-oss/jwt-go" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + + "github.com/Azure/ARO-RP/pkg/util/azureclaim" + "github.com/Azure/ARO-RP/pkg/util/refreshable" +) + +type Credentials struct { + ClientID []byte + ClientSecret []byte + TenantID []byte +} + +// NewAzRefreshableAuthorizer returns a new refreshable authorizer based on an auth token (see GetToken in /pkg/util/aad) +func NewAzRefreshableAuthorizer(token *adal.ServicePrincipalToken) (refreshable.Authorizer, error) { + p := &jwt.Parser{} + c := &azureclaim.AzureClaim{} + _, _, err := p.ParseUnverified(token.OAuthToken(), c) + if err != nil { + return nil, err + } + + return refreshable.NewAuthorizer(token), nil +} + +// AzCredentials gets Cluster Service Principal credentials from the Kubernetes secrets +func AzCredentials(ctx context.Context, kubernetescli kubernetes.Interface) (*Credentials, error) { + mysec, err := kubernetescli.CoreV1().Secrets(azureCredentialSecretNameSpace).Get(ctx, azureCredentialSecretName, metav1.GetOptions{}) + if err != nil { + return nil, err + } + + for _, key := range []string{"azure_client_id", "azure_client_secret", "azure_tenant_id"} { + if _, ok := mysec.Data[key]; !ok { + return nil, fmt.Errorf("%s does not exist in the secret", key) + } + } + + return &Credentials{ + ClientID: mysec.Data["azure_client_id"], + ClientSecret: mysec.Data["azure_client_secret"], + TenantID: mysec.Data["azure_tenant_id"], + }, nil +} diff --git a/pkg/util/clusterauthorizer/const.go b/pkg/util/clusterauthorizer/const.go new file mode 100644 index 000000000..dc38e4401 --- /dev/null +++ b/pkg/util/clusterauthorizer/const.go @@ -0,0 +1,9 @@ +package clusterauthorizer + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +const ( + azureCredentialSecretName = "azure-credentials" + azureCredentialSecretNameSpace = "kube-system" +) diff --git a/pkg/util/subnet/subnet.go b/pkg/util/subnet/subnet.go index 643e3cdd6..1b53647a0 100644 --- a/pkg/util/subnet/subnet.go +++ b/pkg/util/subnet/subnet.go @@ -132,28 +132,35 @@ func NetworkSecurityGroupID(oc *api.OpenShiftCluster, subnetID string) (string, if infraID == "" { infraID = "aro" } - - switch oc.Properties.ArchitectureVersion { - case api.ArchitectureVersionV1: - return networkSecurityGroupIDV1(oc, subnetID, infraID), nil - case api.ArchitectureVersionV2: - return networkSecurityGroupIDV2(oc, subnetID, infraID), nil - default: - return "", fmt.Errorf("unknown architecture version %d", oc.Properties.ArchitectureVersion) - } - -} - -func networkSecurityGroupIDV1(oc *api.OpenShiftCluster, subnetID, infraID string) string { + isWorkerSubnet := false for _, s := range oc.Properties.WorkerProfiles { if strings.EqualFold(subnetID, s.SubnetID) { - return oc.Properties.ClusterProfile.ResourceGroupID + "/providers/Microsoft.Network/networkSecurityGroups/" + infraID + NSGNodeSuffixV1 + isWorkerSubnet = true + break } - } - return oc.Properties.ClusterProfile.ResourceGroupID + "/providers/Microsoft.Network/networkSecurityGroups/" + infraID + NSGControlPlaneSuffixV1 + return NetworkSecurityGroupIDExpanded(oc.Properties.ArchitectureVersion, oc.Properties.ClusterProfile.ResourceGroupID, infraID, isWorkerSubnet) } -func networkSecurityGroupIDV2(oc *api.OpenShiftCluster, subnetID, infraID string) string { - return oc.Properties.ClusterProfile.ResourceGroupID + "/providers/Microsoft.Network/networkSecurityGroups/" + infraID + NSGSuffixV2 +// NetworkSecurityGroupIDExpanded returns the NetworkSecurityGroup ID for a given subnetID, without the OpenShift Cluster document +func NetworkSecurityGroupIDExpanded(architectureVersion api.ArchitectureVersion, resourceGroupID, infraID string, isWorkerSubnet bool) (string, error) { + switch architectureVersion { + case api.ArchitectureVersionV1: + return networkSecurityGroupIDV1(resourceGroupID, infraID, isWorkerSubnet), nil + case api.ArchitectureVersionV2: + return networkSecurityGroupIDV2(resourceGroupID, infraID), nil + default: + return "", fmt.Errorf("unknown architecture version %d", architectureVersion) + } +} + +func networkSecurityGroupIDV1(resourceGroupID, infraID string, isWorkerSubnet bool) string { + if isWorkerSubnet { + return resourceGroupID + "/providers/Microsoft.Network/networkSecurityGroups/" + infraID + NSGNodeSuffixV1 + } + return resourceGroupID + "/providers/Microsoft.Network/networkSecurityGroups/" + infraID + NSGControlPlaneSuffixV1 +} + +func networkSecurityGroupIDV2(resourceGroupID, infraID string) string { + return resourceGroupID + "/providers/Microsoft.Network/networkSecurityGroups/" + infraID + NSGSuffixV2 }