зеркало из https://github.com/Azure/ARO-RP.git
Merge pull request #1373 from jim-minter/dns-update-fixes
DNS update fixes
This commit is contained in:
Коммит
9788ef352d
|
@ -5,19 +5,17 @@ package main
|
|||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
configv1 "github.com/openshift/api/config/v1"
|
||||
configclient "github.com/openshift/client-go/config/clientset/versioned"
|
||||
"github.com/sirupsen/logrus"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/rest"
|
||||
"k8s.io/client-go/tools/clientcmd"
|
||||
|
||||
utillog "github.com/Azure/ARO-RP/pkg/util/log"
|
||||
"github.com/Azure/ARO-RP/pkg/util/version"
|
||||
)
|
||||
|
||||
const discoveryCacheDir = "pkg/util/dynamichelper/discovery/cache"
|
||||
|
@ -57,30 +55,13 @@ func writeVersion(ctx context.Context, restconfig *rest.Config) error {
|
|||
return err
|
||||
}
|
||||
|
||||
clusterVersion, err := getClusterVersion(ctx, configcli)
|
||||
clusterVersion, err := version.GetClusterVersion(ctx, configcli)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
versionPath := filepath.Join(discoveryCacheDir, "assets_version")
|
||||
return ioutil.WriteFile(versionPath, []byte(clusterVersion+"\n"), 0666)
|
||||
}
|
||||
|
||||
func getClusterVersion(ctx context.Context, configcli configclient.Interface) (string, error) {
|
||||
cv, err := configcli.ConfigV1().ClusterVersions().Get(ctx, "version", metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
for _, history := range cv.Status.History {
|
||||
if history.State == configv1.CompletedUpdate {
|
||||
return history.Version, nil
|
||||
}
|
||||
}
|
||||
|
||||
// Should never happen as a successfully created cluster
|
||||
// should have at least one completed update.
|
||||
return "", errors.New("could find actual cluster version")
|
||||
return ioutil.WriteFile(versionPath, []byte(clusterVersion.String()+"\n"), 0666)
|
||||
}
|
||||
|
||||
func main() {
|
||||
|
|
|
@ -5,12 +5,15 @@ package cluster
|
|||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
|
||||
"github.com/Azure/go-autorest/autorest/azure"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/util/retry"
|
||||
|
||||
"github.com/Azure/ARO-RP/pkg/util/ready"
|
||||
"github.com/Azure/ARO-RP/pkg/util/stringutils"
|
||||
"github.com/Azure/ARO-RP/pkg/util/version"
|
||||
)
|
||||
|
||||
func (m *manager) removePrivateDNSZone(ctx context.Context) error {
|
||||
|
@ -23,6 +26,27 @@ func (m *manager) removePrivateDNSZone(ctx context.Context) error {
|
|||
}
|
||||
|
||||
if len(zones) == 0 {
|
||||
// fix up any clusters that we already upgraded
|
||||
err = retry.RetryOnConflict(retry.DefaultRetry, func() error {
|
||||
dns, err := m.configcli.ConfigV1().DNSes().Get(ctx, "cluster", metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if dns.Spec.PrivateZone == nil ||
|
||||
!strings.HasPrefix(strings.ToLower(dns.Spec.PrivateZone.ID), strings.ToLower(m.doc.OpenShiftCluster.Properties.ClusterProfile.ResourceGroupID)) {
|
||||
return nil
|
||||
}
|
||||
|
||||
dns.Spec.PrivateZone = nil
|
||||
|
||||
_, err = m.configcli.ConfigV1().DNSes().Update(ctx, dns, metav1.UpdateOptions{})
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
m.log.Print(err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
@ -32,6 +56,7 @@ func (m *manager) removePrivateDNSZone(ctx context.Context) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
var machineCount int
|
||||
for _, mcp := range mcps.Items {
|
||||
var found bool
|
||||
for _, source := range mcp.Status.Configuration.Source {
|
||||
|
@ -50,6 +75,52 @@ func (m *manager) removePrivateDNSZone(ctx context.Context) error {
|
|||
m.log.Printf("MCP %s not ready", mcp.Name)
|
||||
return nil
|
||||
}
|
||||
|
||||
machineCount += int(mcp.Status.MachineCount)
|
||||
}
|
||||
|
||||
nodes, err := m.kubernetescli.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
|
||||
if err != nil {
|
||||
m.log.Print(err)
|
||||
return nil
|
||||
}
|
||||
|
||||
if len(nodes.Items) != machineCount {
|
||||
m.log.Printf("cluster has %d nodes but %d under MCPs, not removing private DNS zone", len(nodes.Items), machineCount)
|
||||
return nil
|
||||
}
|
||||
|
||||
v, err := version.GetClusterVersion(ctx, m.configcli)
|
||||
if err != nil {
|
||||
m.log.Print(err)
|
||||
return nil
|
||||
}
|
||||
|
||||
if v.Lt(version.NewVersion(4, 4)) {
|
||||
// 4.3 uses SRV records for etcd
|
||||
m.log.Printf("cluster version < 4.4, not removing private DNS zone")
|
||||
return nil
|
||||
}
|
||||
|
||||
err = retry.RetryOnConflict(retry.DefaultRetry, func() error {
|
||||
dns, err := m.configcli.ConfigV1().DNSes().Get(ctx, "cluster", metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if dns.Spec.PrivateZone == nil ||
|
||||
!strings.HasPrefix(strings.ToLower(dns.Spec.PrivateZone.ID), strings.ToLower(m.doc.OpenShiftCluster.Properties.ClusterProfile.ResourceGroupID)) {
|
||||
return nil
|
||||
}
|
||||
|
||||
dns.Spec.PrivateZone = nil
|
||||
|
||||
_, err = m.configcli.ConfigV1().DNSes().Update(ctx, dns, metav1.UpdateOptions{})
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
m.log.Print(err)
|
||||
return nil
|
||||
}
|
||||
|
||||
for _, zone := range zones {
|
||||
|
|
|
@ -10,12 +10,17 @@ import (
|
|||
mgmtprivatedns "github.com/Azure/azure-sdk-for-go/services/privatedns/mgmt/2018-09-01/privatedns"
|
||||
"github.com/Azure/go-autorest/autorest/to"
|
||||
"github.com/golang/mock/gomock"
|
||||
configv1 "github.com/openshift/api/config/v1"
|
||||
configclient "github.com/openshift/client-go/config/clientset/versioned"
|
||||
configfake "github.com/openshift/client-go/config/clientset/versioned/fake"
|
||||
mcv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1"
|
||||
mcoclient "github.com/openshift/machine-config-operator/pkg/generated/clientset/versioned"
|
||||
mcofake "github.com/openshift/machine-config-operator/pkg/generated/clientset/versioned/fake"
|
||||
"github.com/sirupsen/logrus"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
"k8s.io/client-go/kubernetes/fake"
|
||||
|
||||
"github.com/Azure/ARO-RP/pkg/api"
|
||||
mock_privatedns "github.com/Azure/ARO-RP/pkg/util/mocks/azureclient/mgmt/privatedns"
|
||||
|
@ -26,10 +31,13 @@ func TestRemovePrivateDNSZone(t *testing.T) {
|
|||
const resourceGroupID = "/subscriptions/0000000-0000-0000-0000-000000000000/resourceGroups/testGroup"
|
||||
|
||||
for _, tt := range []struct {
|
||||
name string
|
||||
doc *api.OpenShiftClusterDocument
|
||||
mocks func(*mock_privatedns.MockPrivateZonesClient, *mock_privatedns.MockVirtualNetworkLinksClient)
|
||||
mcocli mcoclient.Interface
|
||||
name string
|
||||
doc *api.OpenShiftClusterDocument
|
||||
mocks func(*mock_privatedns.MockPrivateZonesClient, *mock_privatedns.MockVirtualNetworkLinksClient)
|
||||
kubernetescli kubernetes.Interface
|
||||
mcocli mcoclient.Interface
|
||||
configcli configclient.Interface
|
||||
wantDNSPrivateZoneRemoved bool
|
||||
}{
|
||||
{
|
||||
name: "no private zones",
|
||||
|
@ -47,6 +55,19 @@ func TestRemovePrivateDNSZone(t *testing.T) {
|
|||
ListByResourceGroup(ctx, "testGroup", nil).
|
||||
Return(nil, nil)
|
||||
},
|
||||
configcli: configfake.NewSimpleClientset(
|
||||
&configv1.DNS{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "cluster",
|
||||
},
|
||||
Spec: configv1.DNSSpec{
|
||||
PrivateZone: &configv1.DNSZone{
|
||||
ID: "/subscriptions/0000000-0000-0000-0000-000000000000/resourceGroups/testGroup/providers/Microsoft.Network/privateDnsZones/zone1",
|
||||
},
|
||||
},
|
||||
},
|
||||
),
|
||||
wantDNSPrivateZoneRemoved: true,
|
||||
},
|
||||
{
|
||||
name: "has private zone, dnsmasq config not yet reconciled",
|
||||
|
@ -64,7 +85,7 @@ func TestRemovePrivateDNSZone(t *testing.T) {
|
|||
ListByResourceGroup(ctx, "testGroup", nil).
|
||||
Return([]mgmtprivatedns.PrivateZone{
|
||||
{
|
||||
ID: to.StringPtr("/subscriptions/0000000-0000-0000-0000-000000000000/resourceGroups/testGroup/providers/Microsoft.Network/privateZones/zone1"),
|
||||
ID: to.StringPtr("/subscriptions/0000000-0000-0000-0000-000000000000/resourceGroups/testGroup/providers/Microsoft.Network/privateDnsZones/zone1"),
|
||||
},
|
||||
}, nil)
|
||||
},
|
||||
|
@ -88,7 +109,7 @@ func TestRemovePrivateDNSZone(t *testing.T) {
|
|||
ListByResourceGroup(ctx, "testGroup", nil).
|
||||
Return([]mgmtprivatedns.PrivateZone{
|
||||
{
|
||||
ID: to.StringPtr("/subscriptions/0000000-0000-0000-0000-000000000000/resourceGroups/testGroup/providers/Microsoft.Network/privateZones/zone1"),
|
||||
ID: to.StringPtr("/subscriptions/0000000-0000-0000-0000-000000000000/resourceGroups/testGroup/providers/Microsoft.Network/privateDnsZones/zone1"),
|
||||
},
|
||||
}, nil)
|
||||
},
|
||||
|
@ -111,7 +132,7 @@ func TestRemovePrivateDNSZone(t *testing.T) {
|
|||
),
|
||||
},
|
||||
{
|
||||
name: "has private zone, dnsmasq rolled out",
|
||||
name: "has private zone, node mismatch",
|
||||
doc: &api.OpenShiftClusterDocument{
|
||||
OpenShiftCluster: &api.OpenShiftCluster{
|
||||
Properties: api.OpenShiftClusterProperties{
|
||||
|
@ -126,7 +147,105 @@ func TestRemovePrivateDNSZone(t *testing.T) {
|
|||
ListByResourceGroup(ctx, "testGroup", nil).
|
||||
Return([]mgmtprivatedns.PrivateZone{
|
||||
{
|
||||
ID: to.StringPtr("/subscriptions/0000000-0000-0000-0000-000000000000/resourceGroups/testGroup/providers/Microsoft.Network/privateZones/zone1"),
|
||||
ID: to.StringPtr("/subscriptions/0000000-0000-0000-0000-000000000000/resourceGroups/testGroup/providers/Microsoft.Network/privateDnsZones/zone1"),
|
||||
},
|
||||
}, nil)
|
||||
},
|
||||
kubernetescli: fake.NewSimpleClientset(
|
||||
&corev1.Node{},
|
||||
),
|
||||
mcocli: mcofake.NewSimpleClientset(
|
||||
&mcv1.MachineConfigPool{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "master",
|
||||
},
|
||||
Status: mcv1.MachineConfigPoolStatus{
|
||||
Configuration: mcv1.MachineConfigPoolStatusConfiguration{
|
||||
Source: []corev1.ObjectReference{
|
||||
{
|
||||
Name: "99-master-aro-dns",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
),
|
||||
},
|
||||
{
|
||||
name: "has private zone, nodes match, 4.3, dnsmasq rolled out",
|
||||
doc: &api.OpenShiftClusterDocument{
|
||||
OpenShiftCluster: &api.OpenShiftCluster{
|
||||
Properties: api.OpenShiftClusterProperties{
|
||||
ClusterProfile: api.ClusterProfile{
|
||||
ResourceGroupID: resourceGroupID,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
mocks: func(privateZones *mock_privatedns.MockPrivateZonesClient, virtualNetworkLinks *mock_privatedns.MockVirtualNetworkLinksClient) {
|
||||
privateZones.EXPECT().
|
||||
ListByResourceGroup(ctx, "testGroup", nil).
|
||||
Return([]mgmtprivatedns.PrivateZone{
|
||||
{
|
||||
ID: to.StringPtr("/subscriptions/0000000-0000-0000-0000-000000000000/resourceGroups/testGroup/providers/Microsoft.Network/privateDnsZones/zone1"),
|
||||
},
|
||||
}, nil)
|
||||
},
|
||||
kubernetescli: fake.NewSimpleClientset(
|
||||
&corev1.Node{},
|
||||
),
|
||||
mcocli: mcofake.NewSimpleClientset(
|
||||
&mcv1.MachineConfigPool{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "master",
|
||||
},
|
||||
Status: mcv1.MachineConfigPoolStatus{
|
||||
Configuration: mcv1.MachineConfigPoolStatusConfiguration{
|
||||
Source: []corev1.ObjectReference{
|
||||
{
|
||||
Name: "99-master-aro-dns",
|
||||
},
|
||||
},
|
||||
},
|
||||
MachineCount: 1,
|
||||
UpdatedMachineCount: 1,
|
||||
ReadyMachineCount: 1,
|
||||
},
|
||||
},
|
||||
),
|
||||
configcli: configfake.NewSimpleClientset(
|
||||
&configv1.ClusterVersion{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "version",
|
||||
},
|
||||
Status: configv1.ClusterVersionStatus{
|
||||
History: []configv1.UpdateHistory{
|
||||
{
|
||||
State: configv1.CompletedUpdate,
|
||||
Version: "4.3.999",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
),
|
||||
},
|
||||
{
|
||||
name: "has private zone, nodes match, 4.4, dnsmasq rolled out",
|
||||
doc: &api.OpenShiftClusterDocument{
|
||||
OpenShiftCluster: &api.OpenShiftCluster{
|
||||
Properties: api.OpenShiftClusterProperties{
|
||||
ClusterProfile: api.ClusterProfile{
|
||||
ResourceGroupID: resourceGroupID,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
mocks: func(privateZones *mock_privatedns.MockPrivateZonesClient, virtualNetworkLinks *mock_privatedns.MockVirtualNetworkLinksClient) {
|
||||
privateZones.EXPECT().
|
||||
ListByResourceGroup(ctx, "testGroup", nil).
|
||||
Return([]mgmtprivatedns.PrivateZone{
|
||||
{
|
||||
ID: to.StringPtr("/subscriptions/0000000-0000-0000-0000-000000000000/resourceGroups/testGroup/providers/Microsoft.Network/privateDnsZones/zone1"),
|
||||
},
|
||||
}, nil)
|
||||
|
||||
|
@ -146,6 +265,9 @@ func TestRemovePrivateDNSZone(t *testing.T) {
|
|||
DeleteAndWait(ctx, "testGroup", "zone1", "").
|
||||
Return(nil)
|
||||
},
|
||||
kubernetescli: fake.NewSimpleClientset(
|
||||
&corev1.Node{},
|
||||
),
|
||||
mcocli: mcofake.NewSimpleClientset(
|
||||
&mcv1.MachineConfigPool{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
|
@ -159,9 +281,38 @@ func TestRemovePrivateDNSZone(t *testing.T) {
|
|||
},
|
||||
},
|
||||
},
|
||||
MachineCount: 1,
|
||||
UpdatedMachineCount: 1,
|
||||
ReadyMachineCount: 1,
|
||||
},
|
||||
},
|
||||
),
|
||||
configcli: configfake.NewSimpleClientset(
|
||||
&configv1.ClusterVersion{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "version",
|
||||
},
|
||||
Status: configv1.ClusterVersionStatus{
|
||||
History: []configv1.UpdateHistory{
|
||||
{
|
||||
State: configv1.CompletedUpdate,
|
||||
Version: "4.4.0",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
&configv1.DNS{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "cluster",
|
||||
},
|
||||
Spec: configv1.DNSSpec{
|
||||
PrivateZone: &configv1.DNSZone{
|
||||
ID: "/subscriptions/0000000-0000-0000-0000-000000000000/resourceGroups/testGroup/providers/Microsoft.Network/privateDnsZones/zone1",
|
||||
},
|
||||
},
|
||||
},
|
||||
),
|
||||
wantDNSPrivateZoneRemoved: true,
|
||||
},
|
||||
} {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
|
@ -177,13 +328,25 @@ func TestRemovePrivateDNSZone(t *testing.T) {
|
|||
doc: tt.doc,
|
||||
privateZones: privateZones,
|
||||
virtualNetworkLinks: virtualNetworkLinks,
|
||||
kubernetescli: tt.kubernetescli,
|
||||
mcocli: tt.mcocli,
|
||||
configcli: tt.configcli,
|
||||
}
|
||||
|
||||
err := m.removePrivateDNSZone(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if tt.wantDNSPrivateZoneRemoved {
|
||||
dns, err := m.configcli.ConfigV1().DNSes().Get(ctx, "cluster", metav1.GetOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if dns.Spec.PrivateZone != nil {
|
||||
t.Error(dns.Spec.PrivateZone)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
|
@ -5,14 +5,11 @@ package workaround
|
|||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
configv1 "github.com/openshift/api/config/v1"
|
||||
configclient "github.com/openshift/client-go/config/clientset/versioned"
|
||||
mcoclient "github.com/openshift/machine-config-operator/pkg/generated/clientset/versioned"
|
||||
"github.com/sirupsen/logrus"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
"k8s.io/client-go/rest"
|
||||
ctrl "sigs.k8s.io/controller-runtime"
|
||||
|
@ -52,25 +49,11 @@ func NewReconciler(log *logrus.Entry, kubernetescli kubernetes.Interface, config
|
|||
}
|
||||
}
|
||||
|
||||
func (r *WorkaroundReconciler) actualClusterVersion(ctx context.Context) (*version.Version, error) {
|
||||
cv, err := r.configcli.ConfigV1().ClusterVersions().Get(ctx, "version", metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, history := range cv.Status.History {
|
||||
if history.State == configv1.CompletedUpdate {
|
||||
return version.ParseVersion(history.Version)
|
||||
}
|
||||
}
|
||||
return nil, fmt.Errorf("unknown cluster version")
|
||||
}
|
||||
|
||||
// Reconcile makes sure that the workarounds are applied or removed as per the OpenShift version.
|
||||
func (r *WorkaroundReconciler) Reconcile(request ctrl.Request) (ctrl.Result, error) {
|
||||
// TODO(mj): controller-runtime master fixes the need for this (https://github.com/kubernetes-sigs/controller-runtime/blob/master/pkg/reconcile/reconcile.go#L93) but it's not yet released.
|
||||
ctx := context.Background()
|
||||
clusterVersion, err := r.actualClusterVersion(ctx)
|
||||
clusterVersion, err := version.GetClusterVersion(ctx, r.configcli)
|
||||
if err != nil {
|
||||
r.log.Errorf("error getting the OpenShift version: %v", err)
|
||||
return reconcile.Result{}, err
|
||||
|
|
|
@ -0,0 +1,28 @@
|
|||
package version
|
||||
|
||||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the Apache License 2.0.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
configv1 "github.com/openshift/api/config/v1"
|
||||
configclient "github.com/openshift/client-go/config/clientset/versioned"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
)
|
||||
|
||||
func GetClusterVersion(ctx context.Context, configcli configclient.Interface) (*Version, error) {
|
||||
cv, err := configcli.ConfigV1().ClusterVersions().Get(ctx, "version", metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, history := range cv.Status.History {
|
||||
if history.State == configv1.CompletedUpdate {
|
||||
return ParseVersion(history.Version)
|
||||
}
|
||||
}
|
||||
|
||||
return nil, errors.New("unknown cluster version")
|
||||
}
|
Загрузка…
Ссылка в новой задаче