implement tls signed certificates

2020-02-17 15:08:45 -06:00 · 2020-02-17 15:08:45 -06:00 · d29d2a51eb
--- a/deploy/rp-development.json
+++ b/deploy/rp-development.json
@ -40,7 +40,31 @@
                    "family": "A",
                    "name": "standard"
                },
-                "accessPolicies": []
+                "accessPolicies": [
+                    {
+                        "tenantId": "[subscription().tenantId]",
+                        "objectId": "[parameters('fpServicePrincipalId')]",
+                        "permissions": {
+                            "secrets": [
+                                "get"
+                            ],
+                            "certificates": [
+                                "create",
+                                "delete"
+                            ]
+                        }
+                    },
+                    {
+                        "tenantId": "[subscription().tenantId]",
+                        "objectId": "[parameters('adminObjectId')]",
+                        "permissions": {
+                            "certificates": [
+                                "get",
+                                "list"
+                            ]
+                        }
+                    }
+                ]
            },
            "name": "[concat(parameters('keyvaultPrefix'), '-cls')]",
            "type": "Microsoft.KeyVault/vaults",
--- a/deploy/rp-production.json
+++ b/deploy/rp-production.json
@ -2,7 +2,21 @@
    "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
    "contentVersion": "1.0.0.0",
    "variables": {
-        "clustersKeyvaultAccessPolicies": [],
+        "clustersKeyvaultAccessPolicies": [
+            {
+                "tenantId": "[subscription().tenantId]",
+                "objectId": "[parameters('fpServicePrincipalId')]",
+                "permissions": {
+                    "secrets": [
+                        "get"
+                    ],
+                    "certificates": [
+                        "create",
+                        "delete"
+                    ]
+                }
+            }
+        ],
        "serviceKeyvaultAccessPolicies": [
            {
                "tenantId": "[subscription().tenantId]",
--- a/docs/using-az-aro.md
+++ b/docs/using-az-aro.md
@ -161,9 +161,6 @@ cluster:
   az aro list-credentials -g "$RESOURCEGROUP" -n "$CLUSTER"
   ```

-   Note: the cluster console certificate is not yet signed by a CA: expect a
-   security warning in your browser.
-
 1. Scale the number of cluster VMs:

   ```
--- a/pkg/backend/openshiftcluster/delete.go
+++ b/pkg/backend/openshiftcluster/delete.go
@ -10,6 +10,7 @@ import (

 	"github.com/Azure/go-autorest/autorest"

+	"github.com/Azure/ARO-RP/pkg/env"
 	"github.com/Azure/ARO-RP/pkg/util/subnet"
 )

@ -62,6 +63,27 @@ func (m *Manager) Delete(ctx context.Context) error {
 		return err
 	}

+	if _, ok := m.env.(env.Dev); !ok {
+		managedDomain, err := m.env.ManagedDomain(m.doc.OpenShiftCluster.Properties.ClusterProfile.Domain)
+		if err != nil {
+			return err
+		}
+
+		if managedDomain != "" {
+			m.log.Print("deleting signed apiserver certificate")
+			err = m.keyvault.DeleteCertificate(ctx, m.doc.ID+"-apiserver")
+			if err != nil {
+				return err
+			}
+
+			m.log.Print("deleting signed ingress certificate")
+			err = m.keyvault.DeleteCertificate(ctx, m.doc.ID+"-ingress")
+			if err != nil {
+				return err
+			}
+		}
+	}
+
 	m.log.Printf("deleting resource group %s", resourceGroup)
 	err = m.groups.DeleteAndWait(ctx, resourceGroup)
 	if detailedErr, ok := err.(autorest.DetailedError); ok &&
--- a/pkg/deploy/rp.go
+++ b/pkg/deploy/rp.go
@ -591,13 +591,19 @@ func (g *generator) zone() *arm.Resource {

 func (g *generator) clustersKeyvaultAccessPolicies() []mgmtkeyvault.AccessPolicyEntry {
 	return []mgmtkeyvault.AccessPolicyEntry{
-		// TODO: uncomment when there are permissions we want to grant
-		/*
-			{
-				TenantID: &tenantUUIDHack,
-				ObjectID: to.StringPtr("[parameters('fpServicePrincipalId')]"),
+		{
+			TenantID: &tenantUUIDHack,
+			ObjectID: to.StringPtr("[parameters('fpServicePrincipalId')]"),
+			Permissions: &mgmtkeyvault.Permissions{
+				Secrets: &[]mgmtkeyvault.SecretPermissions{
+					mgmtkeyvault.SecretPermissionsGet,
+				},
+				Certificates: &[]mgmtkeyvault.CertificatePermissions{
+					mgmtkeyvault.Create,
+					mgmtkeyvault.Delete,
+				},
 			},
-		*/
+		},
 	}
 }

@ -634,15 +640,18 @@ func (g *generator) clustersKeyvault() *arm.Resource {
 	}

 	if !g.production {
-		// TODO: uncomment when there are permissions we want to grant
-		/*
-			*vault.Properties.AccessPolicies = append(g.clustersKeyvaultAccessPolicies(),
-				mgmtkeyvault.AccessPolicyEntry{
-					TenantID: &tenantUUIDHack,
-					ObjectID: to.StringPtr("[parameters('adminObjectId')]"),
+		*vault.Properties.AccessPolicies = append(g.clustersKeyvaultAccessPolicies(),
+			mgmtkeyvault.AccessPolicyEntry{
+				TenantID: &tenantUUIDHack,
+				ObjectID: to.StringPtr("[parameters('adminObjectId')]"),
+				Permissions: &mgmtkeyvault.Permissions{
+					Certificates: &[]mgmtkeyvault.CertificatePermissions{
+						mgmtkeyvault.Get,
+						mgmtkeyvault.List,
+					},
 				},
-			)
-		*/
+			},
+		)
 	}

 	return &arm.Resource{
--- a/pkg/install/0-installstorage.go
+++ b/pkg/install/0-installstorage.go
@ -61,7 +61,7 @@ func (i *Installer) installStorage(ctx context.Context, installConfig *installco
 		}
 	}

-	adminClient := g[reflect.TypeOf(&kubeconfig.AdminClient{})].(*kubeconfig.AdminClient)
+	adminInternalClient := g[reflect.TypeOf(&kubeconfig.AdminInternalClient{})].(*kubeconfig.AdminInternalClient)

 	resourceGroup := i.doc.OpenShiftCluster.Properties.ClusterProfile.ResourceGroupID[strings.LastIndexByte(i.doc.OpenShiftCluster.Properties.ClusterProfile.ResourceGroupID, '/')+1:]

@ -245,7 +245,7 @@ func (i *Installer) installStorage(ctx context.Context, installConfig *installco
 		// used for the SAS token with which the bootstrap node retrieves its
 		// ignition payload
 		doc.OpenShiftCluster.Properties.Install.Now = time.Now().UTC()
-		doc.OpenShiftCluster.Properties.AdminKubeconfig = adminClient.File.Data
+		doc.OpenShiftCluster.Properties.AdminKubeconfig = adminInternalClient.File.Data
 		return nil
 	})
 	return err
--- a/pkg/install/install.go
+++ b/pkg/install/install.go
@ -128,15 +128,18 @@ func (i *Installer) Install(ctx context.Context, installConfig *installconfig.In
 			i.installResources,
 			i.createPrivateEndpoint,
 			i.updateAPIIP,
+			i.createCertificates,
 			i.waitForBootstrapConfigmap,
 			i.incrInstallPhase,
 		},
 		api.InstallPhaseRemoveBootstrap: {
 			i.removeBootstrap,
+			i.configureAPIServerCertificate,
 			i.updateConsoleBranding,
 			i.waitForClusterVersion,
 			i.disableUpdates,
 			i.updateRouterIP,
+			i.configureIngressCertificate,
 			i.endOfInstallPhase,
 		},
 	}
--- a/pkg/install/tls.go
+++ b/pkg/install/tls.go
@ -0,0 +1,267 @@
+package install
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+import (
+	"context"
+	"crypto/x509"
+	"encoding/pem"
+	"time"
+
+	configv1 "github.com/openshift/api/config/v1"
+	operatorv1 "github.com/openshift/api/operator/v1"
+	configclient "github.com/openshift/client-go/config/clientset/versioned"
+	operatorclient "github.com/openshift/client-go/operator/clientset/versioned"
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/kubernetes"
+	coreclient "k8s.io/client-go/kubernetes/typed/core/v1"
+	"k8s.io/client-go/util/retry"
+
+	"github.com/Azure/ARO-RP/pkg/env"
+	"github.com/Azure/ARO-RP/pkg/util/restconfig"
+)
+
+func (i *Installer) createCertificates(ctx context.Context) error {
+	if _, ok := i.env.(env.Dev); ok {
+		return nil
+	}
+
+	managedDomain, err := i.env.ManagedDomain(i.doc.OpenShiftCluster.Properties.ClusterProfile.Domain)
+	if err != nil {
+		return err
+	}
+
+	if managedDomain == "" {
+		return nil
+	}
+
+	certs := []struct {
+		certificateName string
+		commonName      string
+	}{
+		{
+			certificateName: i.doc.ID + "-apiserver",
+			commonName:      "api." + managedDomain,
+		},
+		{
+			certificateName: i.doc.ID + "-ingress",
+			commonName:      "*.apps." + managedDomain,
+		},
+	}
+
+	for _, c := range certs {
+		i.log.Printf("creating certificate %s", c.certificateName)
+		err = i.keyvault.CreateCertificate(ctx, c.certificateName, c.commonName)
+		if err != nil {
+			return err
+		}
+	}
+
+	for _, c := range certs {
+		i.log.Printf("waiting for certificate %s", c.certificateName)
+		err = i.keyvault.WaitForCertificateOperation(ctx, c.certificateName)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (i *Installer) ensureSecret(ctx context.Context, secrets coreclient.SecretInterface, certificateName string) error {
+	key, certs, err := i.keyvault.GetSecret(ctx, certificateName)
+	if err != nil {
+		return err
+	}
+
+	b, err := x509.MarshalPKCS8PrivateKey(key)
+	if err != nil {
+		return err
+	}
+
+	var cb []byte
+	for _, cert := range certs {
+		cb = append(cb, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: cert.Raw})...)
+	}
+
+	_, err = secrets.Create(&v1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: certificateName,
+		},
+		Data: map[string][]byte{
+			v1.TLSCertKey:       cb,
+			v1.TLSPrivateKeyKey: pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: b}),
+		},
+		Type: v1.SecretTypeTLS,
+	})
+	if errors.IsAlreadyExists(err) {
+		err = retry.RetryOnConflict(retry.DefaultRetry, func() error {
+			s, err := secrets.Get(certificateName, metav1.GetOptions{})
+			if err != nil {
+				return err
+			}
+
+			s.Data = map[string][]byte{
+				v1.TLSCertKey:       cb,
+				v1.TLSPrivateKeyKey: pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: b}),
+			}
+			s.Type = v1.SecretTypeTLS
+
+			_, err = secrets.Update(s)
+			return err
+		})
+	}
+	return err
+}
+
+func (i *Installer) configureAPIServerCertificate(ctx context.Context) error {
+	if _, ok := i.env.(env.Dev); ok {
+		return nil
+	}
+
+	managedDomain, err := i.env.ManagedDomain(i.doc.OpenShiftCluster.Properties.ClusterProfile.Domain)
+	if err != nil {
+		return err
+	}
+
+	if managedDomain == "" {
+		return nil
+	}
+
+	restConfig, err := restconfig.RestConfig(ctx, i.env, i.doc.OpenShiftCluster)
+	if err != nil {
+		return err
+	}
+
+	cli, err := kubernetes.NewForConfig(restConfig)
+	if err != nil {
+		return err
+	}
+
+	err = i.ensureSecret(ctx, cli.CoreV1().Secrets("openshift-config"), i.doc.ID+"-apiserver")
+	if err != nil {
+		return err
+	}
+
+	ccli, err := configclient.NewForConfig(restConfig)
+	if err != nil {
+		return err
+	}
+
+	err = retry.RetryOnConflict(retry.DefaultRetry, func() error {
+		apiserver, err := ccli.ConfigV1().APIServers().Get("cluster", metav1.GetOptions{})
+		if err != nil {
+			return err
+		}
+
+		apiserver.Spec.ServingCerts.NamedCertificates = []configv1.APIServerNamedServingCert{
+			{
+				Names: []string{
+					"api." + managedDomain,
+				},
+				ServingCertificate: configv1.SecretNameReference{
+					Name: i.doc.ID + "-apiserver",
+				},
+			},
+		}
+
+		_, err = ccli.ConfigV1().APIServers().Update(apiserver)
+		return err
+	})
+	if err != nil {
+		return err
+	}
+
+	ocli, err := operatorclient.NewForConfig(restConfig)
+	if err != nil {
+		return err
+	}
+
+	i.log.Print("waiting for apiservers")
+	timeoutCtx, cancel := context.WithTimeout(ctx, 30*time.Minute)
+	defer cancel()
+	return wait.PollImmediateUntil(10*time.Second, func() (bool, error) {
+		apiserver, err := ocli.OperatorV1().KubeAPIServers().Get("cluster", metav1.GetOptions{})
+		if err == nil {
+			m := make(map[string]operatorv1.ConditionStatus, len(apiserver.Status.Conditions))
+			for _, cond := range apiserver.Status.Conditions {
+				m[cond.Type] = cond.Status
+			}
+			if m["Available"] == operatorv1.ConditionTrue && m["Progressing"] == operatorv1.ConditionFalse {
+				return true, nil
+			}
+		}
+		return false, nil
+	}, timeoutCtx.Done())
+}
+
+func (i *Installer) configureIngressCertificate(ctx context.Context) error {
+	if _, ok := i.env.(env.Dev); ok {
+		return nil
+	}
+
+	managedDomain, err := i.env.ManagedDomain(i.doc.OpenShiftCluster.Properties.ClusterProfile.Domain)
+	if err != nil {
+		return err
+	}
+
+	if managedDomain == "" {
+		return nil
+	}
+
+	restConfig, err := restconfig.RestConfig(ctx, i.env, i.doc.OpenShiftCluster)
+	if err != nil {
+		return err
+	}
+
+	cli, err := kubernetes.NewForConfig(restConfig)
+	if err != nil {
+		return err
+	}
+
+	err = i.ensureSecret(ctx, cli.CoreV1().Secrets("openshift-ingress"), i.doc.ID+"-ingress")
+	if err != nil {
+		return err
+	}
+
+	ocli, err := operatorclient.NewForConfig(restConfig)
+	if err != nil {
+		return err
+	}
+
+	err = retry.RetryOnConflict(retry.DefaultRetry, func() error {
+		ic, err := ocli.OperatorV1().IngressControllers("openshift-ingress-operator").Get("default", metav1.GetOptions{})
+		if err != nil {
+			return err
+		}
+
+		ic.Spec.DefaultCertificate = &v1.LocalObjectReference{
+			Name: i.doc.ID + "-ingress",
+		}
+
+		_, err = ocli.OperatorV1().IngressControllers("openshift-ingress-operator").Update(ic)
+		return err
+	})
+	if err != nil {
+		return err
+	}
+
+	i.log.Print("waiting for ingress controller")
+	timeoutCtx, cancel := context.WithTimeout(ctx, 30*time.Minute)
+	defer cancel()
+	return wait.PollImmediateUntil(10*time.Second, func() (bool, error) {
+		ic, err := ocli.OperatorV1().IngressControllers("openshift-ingress-operator").Get("default", metav1.GetOptions{})
+		if err == nil && ic.Status.ObservedGeneration == ic.Generation {
+			for _, cond := range ic.Status.Conditions {
+				if cond.Type == operatorv1.OperatorStatusTypeAvailable && cond.Status == operatorv1.ConditionTrue {
+					return true, nil
+				}
+			}
+		}
+		return false, nil
+	}, timeoutCtx.Done())
+}
--- a/pkg/util/azureclient/keyvault/base.go
+++ b/pkg/util/azureclient/keyvault/base.go
@ -15,6 +15,9 @@ import (

 // BaseClient is a minimal interface for azure BaseClient
 type BaseClient interface {
+	CreateCertificate(ctx context.Context, vaultBaseURL string, certificateName string, parameters keyvault.CertificateCreateParameters) (result keyvault.CertificateOperation, err error)
+	DeleteCertificate(ctx context.Context, vaultBaseURL string, certificateName string) (result keyvault.DeletedCertificateBundle, err error)
+	GetCertificateOperation(ctx context.Context, vaultBaseURL string, certificateName string) (result keyvault.CertificateOperation, err error)
 	GetSecret(ctx context.Context, vaultBaseURL string, secretName string, secretVersion string) (result keyvault.SecretBundle, err error)
 }

--- a/pkg/util/keyvault/keyvault.go
+++ b/pkg/util/keyvault/keyvault.go
@ -4,13 +4,30 @@ package keyvault
 // Licensed under the Apache License 2.0.

 import (
+	"context"
+	"crypto/rsa"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/Azure/azure-sdk-for-go/services/keyvault/v7.0/keyvault"
 	"github.com/Azure/go-autorest/autorest"
+	"github.com/Azure/go-autorest/autorest/azure"
+	"github.com/Azure/go-autorest/autorest/to"
+	"k8s.io/apimachinery/pkg/util/wait"

 	"github.com/Azure/ARO-RP/pkg/env"
 	basekeyvault "github.com/Azure/ARO-RP/pkg/util/azureclient/keyvault"
+	"github.com/Azure/ARO-RP/pkg/util/pem"
 )

 type Manager interface {
+	CreateCertificate(context.Context, string, string) error
+	DeleteCertificate(context.Context, string) error
+	GetSecret(context.Context, string) (*rsa.PrivateKey, []*x509.Certificate, error)
+	WaitForCertificateOperation(context.Context, string) error
 }

 type manager struct {
@ -25,3 +42,123 @@ func NewManager(env env.Interface, localFPKVAuthorizer autorest.Authorizer) Mana
 		keyvault: basekeyvault.New(localFPKVAuthorizer),
 	}
 }
+
+func (m *manager) CreateCertificate(ctx context.Context, certificateName, commonName string) error {
+	op, err := m.keyvault.CreateCertificate(ctx, m.env.ClustersKeyvaultURI(), certificateName, keyvault.CertificateCreateParameters{
+		CertificatePolicy: &keyvault.CertificatePolicy{
+			KeyProperties: &keyvault.KeyProperties{
+				Exportable: to.BoolPtr(true),
+				KeyType:    keyvault.RSA,
+				KeySize:    to.Int32Ptr(2048),
+			},
+			SecretProperties: &keyvault.SecretProperties{
+				ContentType: to.StringPtr("application/x-pem-file"),
+			},
+			X509CertificateProperties: &keyvault.X509CertificateProperties{
+				Subject: to.StringPtr(pkix.Name{CommonName: commonName}.String()),
+				Ekus: &[]string{
+					"1.3.6.1.5.5.7.3.1", // serverAuth
+				},
+				KeyUsage: &[]keyvault.KeyUsageType{
+					keyvault.DigitalSignature,
+					keyvault.KeyEncipherment,
+				},
+				ValidityInMonths: to.Int32Ptr(12),
+			},
+			IssuerParameters: &keyvault.IssuerParameters{
+				Name: to.StringPtr("digicert01"),
+			},
+		},
+	})
+	if err != nil {
+		return err
+	}
+
+	_, err = checkOperation(&op)
+	return err
+}
+
+func (m *manager) DeleteCertificate(ctx context.Context, certificateName string) error {
+	_, err := m.keyvault.DeleteCertificate(ctx, m.env.ClustersKeyvaultURI(), certificateName)
+	if detailedError, ok := err.(autorest.DetailedError); ok {
+		if requestError, ok := detailedError.Original.(*azure.RequestError); ok &&
+			requestError.ServiceError != nil &&
+			requestError.ServiceError.Code == "CertificateNotFound" {
+			err = nil
+		}
+	}
+
+	return err
+}
+
+func (m *manager) GetSecret(ctx context.Context, secretName string) (key *rsa.PrivateKey, certs []*x509.Certificate, err error) {
+	bundle, err := m.keyvault.GetSecret(ctx, m.env.ClustersKeyvaultURI(), secretName, "")
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return pem.Parse([]byte(*bundle.Value))
+}
+
+func (m *manager) WaitForCertificateOperation(ctx context.Context, certificateName string) error {
+	ctx, cancel := context.WithTimeout(ctx, 15*time.Minute)
+	defer cancel()
+
+	err := wait.PollImmediateUntil(10*time.Second, func() (bool, error) {
+		op, err := m.keyvault.GetCertificateOperation(ctx, m.env.ClustersKeyvaultURI(), certificateName)
+		if err != nil {
+			return false, err
+		}
+
+		return checkOperation(&op)
+	}, ctx.Done())
+	return err
+}
+
+func keyvaultError(err *keyvault.Error) string {
+	if err == nil {
+		return ""
+	}
+
+	var sb strings.Builder
+
+	if err.Code != nil {
+		sb.WriteString(*err.Code)
+	}
+
+	if err.Message != nil {
+		if sb.Len() > 0 {
+			sb.WriteString(": ")
+		}
+		sb.WriteString(*err.Message)
+	}
+
+	inner := keyvaultError(err.InnerError)
+	if inner != "" {
+		if sb.Len() > 0 {
+			sb.WriteString(": ")
+		}
+		sb.WriteString(inner)
+	}
+
+	return sb.String()
+}
+func checkOperation(op *keyvault.CertificateOperation) (bool, error) {
+	switch *op.Status {
+	case "inProgress":
+		return false, nil
+
+	case "completed":
+		return true, nil
+
+	default:
+		err := keyvaultError(op.Error)
+		if op.StatusDetails != nil {
+			if err != "" {
+				err += ": "
+			}
+			err += *op.StatusDetails
+		}
+		return false, fmt.Errorf("certificateOperation %s: %s", *op.Status, err)
+	}
+}
--- a/pkg/util/mocks/azureclient/keyvault/keyvault.go
+++ b/pkg/util/mocks/azureclient/keyvault/keyvault.go
@ -35,6 +35,51 @@ func (m *MockBaseClient) EXPECT() *MockBaseClientMockRecorder {
 	return m.recorder
 }

+// CreateCertificate mocks base method
+func (m *MockBaseClient) CreateCertificate(arg0 context.Context, arg1, arg2 string, arg3 keyvault.CertificateCreateParameters) (keyvault.CertificateOperation, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "CreateCertificate", arg0, arg1, arg2, arg3)
+	ret0, _ := ret[0].(keyvault.CertificateOperation)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// CreateCertificate indicates an expected call of CreateCertificate
+func (mr *MockBaseClientMockRecorder) CreateCertificate(arg0, arg1, arg2, arg3 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateCertificate", reflect.TypeOf((*MockBaseClient)(nil).CreateCertificate), arg0, arg1, arg2, arg3)
+}
+
+// DeleteCertificate mocks base method
+func (m *MockBaseClient) DeleteCertificate(arg0 context.Context, arg1, arg2 string) (keyvault.DeletedCertificateBundle, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "DeleteCertificate", arg0, arg1, arg2)
+	ret0, _ := ret[0].(keyvault.DeletedCertificateBundle)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// DeleteCertificate indicates an expected call of DeleteCertificate
+func (mr *MockBaseClientMockRecorder) DeleteCertificate(arg0, arg1, arg2 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteCertificate", reflect.TypeOf((*MockBaseClient)(nil).DeleteCertificate), arg0, arg1, arg2)
+}
+
+// GetCertificateOperation mocks base method
+func (m *MockBaseClient) GetCertificateOperation(arg0 context.Context, arg1, arg2 string) (keyvault.CertificateOperation, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetCertificateOperation", arg0, arg1, arg2)
+	ret0, _ := ret[0].(keyvault.CertificateOperation)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// GetCertificateOperation indicates an expected call of GetCertificateOperation
+func (mr *MockBaseClientMockRecorder) GetCertificateOperation(arg0, arg1, arg2 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCertificateOperation", reflect.TypeOf((*MockBaseClient)(nil).GetCertificateOperation), arg0, arg1, arg2)
+}
+
 // GetSecret mocks base method
 func (m *MockBaseClient) GetSecret(arg0 context.Context, arg1, arg2, arg3 string) (keyvault.SecretBundle, error) {
 	m.ctrl.T.Helper()