Merge pull request #1373 from jim-minter/dns-update-fixes

DNS update fixes
This commit is contained in:
Jim Minter 2021-03-09 12:08:31 -06:00 коммит произвёл GitHub
Родитель d57fcdc119 12daf605ad
Коммит 9788ef352d
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
5 изменённых файлов: 274 добавлений и 48 удалений

Просмотреть файл

@ -5,19 +5,17 @@ package main
import (
"context"
"errors"
"io/ioutil"
"os"
"path/filepath"
configv1 "github.com/openshift/api/config/v1"
configclient "github.com/openshift/client-go/config/clientset/versioned"
"github.com/sirupsen/logrus"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
utillog "github.com/Azure/ARO-RP/pkg/util/log"
"github.com/Azure/ARO-RP/pkg/util/version"
)
const discoveryCacheDir = "pkg/util/dynamichelper/discovery/cache"
@ -57,30 +55,13 @@ func writeVersion(ctx context.Context, restconfig *rest.Config) error {
return err
}
clusterVersion, err := getClusterVersion(ctx, configcli)
clusterVersion, err := version.GetClusterVersion(ctx, configcli)
if err != nil {
return err
}
versionPath := filepath.Join(discoveryCacheDir, "assets_version")
return ioutil.WriteFile(versionPath, []byte(clusterVersion+"\n"), 0666)
}
func getClusterVersion(ctx context.Context, configcli configclient.Interface) (string, error) {
cv, err := configcli.ConfigV1().ClusterVersions().Get(ctx, "version", metav1.GetOptions{})
if err != nil {
return "", err
}
for _, history := range cv.Status.History {
if history.State == configv1.CompletedUpdate {
return history.Version, nil
}
}
// Should never happen as a successfully created cluster
// should have at least one completed update.
return "", errors.New("could find actual cluster version")
return ioutil.WriteFile(versionPath, []byte(clusterVersion.String()+"\n"), 0666)
}
func main() {

Просмотреть файл

@ -5,12 +5,15 @@ package cluster
import (
"context"
"strings"
"github.com/Azure/go-autorest/autorest/azure"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/util/retry"
"github.com/Azure/ARO-RP/pkg/util/ready"
"github.com/Azure/ARO-RP/pkg/util/stringutils"
"github.com/Azure/ARO-RP/pkg/util/version"
)
func (m *manager) removePrivateDNSZone(ctx context.Context) error {
@ -23,6 +26,27 @@ func (m *manager) removePrivateDNSZone(ctx context.Context) error {
}
if len(zones) == 0 {
// fix up any clusters that we already upgraded
err = retry.RetryOnConflict(retry.DefaultRetry, func() error {
dns, err := m.configcli.ConfigV1().DNSes().Get(ctx, "cluster", metav1.GetOptions{})
if err != nil {
return err
}
if dns.Spec.PrivateZone == nil ||
!strings.HasPrefix(strings.ToLower(dns.Spec.PrivateZone.ID), strings.ToLower(m.doc.OpenShiftCluster.Properties.ClusterProfile.ResourceGroupID)) {
return nil
}
dns.Spec.PrivateZone = nil
_, err = m.configcli.ConfigV1().DNSes().Update(ctx, dns, metav1.UpdateOptions{})
return err
})
if err != nil {
m.log.Print(err)
}
return nil
}
@ -32,6 +56,7 @@ func (m *manager) removePrivateDNSZone(ctx context.Context) error {
return nil
}
var machineCount int
for _, mcp := range mcps.Items {
var found bool
for _, source := range mcp.Status.Configuration.Source {
@ -50,6 +75,52 @@ func (m *manager) removePrivateDNSZone(ctx context.Context) error {
m.log.Printf("MCP %s not ready", mcp.Name)
return nil
}
machineCount += int(mcp.Status.MachineCount)
}
nodes, err := m.kubernetescli.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
if err != nil {
m.log.Print(err)
return nil
}
if len(nodes.Items) != machineCount {
m.log.Printf("cluster has %d nodes but %d under MCPs, not removing private DNS zone", len(nodes.Items), machineCount)
return nil
}
v, err := version.GetClusterVersion(ctx, m.configcli)
if err != nil {
m.log.Print(err)
return nil
}
if v.Lt(version.NewVersion(4, 4)) {
// 4.3 uses SRV records for etcd
m.log.Printf("cluster version < 4.4, not removing private DNS zone")
return nil
}
err = retry.RetryOnConflict(retry.DefaultRetry, func() error {
dns, err := m.configcli.ConfigV1().DNSes().Get(ctx, "cluster", metav1.GetOptions{})
if err != nil {
return err
}
if dns.Spec.PrivateZone == nil ||
!strings.HasPrefix(strings.ToLower(dns.Spec.PrivateZone.ID), strings.ToLower(m.doc.OpenShiftCluster.Properties.ClusterProfile.ResourceGroupID)) {
return nil
}
dns.Spec.PrivateZone = nil
_, err = m.configcli.ConfigV1().DNSes().Update(ctx, dns, metav1.UpdateOptions{})
return err
})
if err != nil {
m.log.Print(err)
return nil
}
for _, zone := range zones {

Просмотреть файл

@ -10,12 +10,17 @@ import (
mgmtprivatedns "github.com/Azure/azure-sdk-for-go/services/privatedns/mgmt/2018-09-01/privatedns"
"github.com/Azure/go-autorest/autorest/to"
"github.com/golang/mock/gomock"
configv1 "github.com/openshift/api/config/v1"
configclient "github.com/openshift/client-go/config/clientset/versioned"
configfake "github.com/openshift/client-go/config/clientset/versioned/fake"
mcv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1"
mcoclient "github.com/openshift/machine-config-operator/pkg/generated/clientset/versioned"
mcofake "github.com/openshift/machine-config-operator/pkg/generated/clientset/versioned/fake"
"github.com/sirupsen/logrus"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/kubernetes/fake"
"github.com/Azure/ARO-RP/pkg/api"
mock_privatedns "github.com/Azure/ARO-RP/pkg/util/mocks/azureclient/mgmt/privatedns"
@ -26,10 +31,13 @@ func TestRemovePrivateDNSZone(t *testing.T) {
const resourceGroupID = "/subscriptions/0000000-0000-0000-0000-000000000000/resourceGroups/testGroup"
for _, tt := range []struct {
name string
doc *api.OpenShiftClusterDocument
mocks func(*mock_privatedns.MockPrivateZonesClient, *mock_privatedns.MockVirtualNetworkLinksClient)
mcocli mcoclient.Interface
name string
doc *api.OpenShiftClusterDocument
mocks func(*mock_privatedns.MockPrivateZonesClient, *mock_privatedns.MockVirtualNetworkLinksClient)
kubernetescli kubernetes.Interface
mcocli mcoclient.Interface
configcli configclient.Interface
wantDNSPrivateZoneRemoved bool
}{
{
name: "no private zones",
@ -47,6 +55,19 @@ func TestRemovePrivateDNSZone(t *testing.T) {
ListByResourceGroup(ctx, "testGroup", nil).
Return(nil, nil)
},
configcli: configfake.NewSimpleClientset(
&configv1.DNS{
ObjectMeta: metav1.ObjectMeta{
Name: "cluster",
},
Spec: configv1.DNSSpec{
PrivateZone: &configv1.DNSZone{
ID: "/subscriptions/0000000-0000-0000-0000-000000000000/resourceGroups/testGroup/providers/Microsoft.Network/privateDnsZones/zone1",
},
},
},
),
wantDNSPrivateZoneRemoved: true,
},
{
name: "has private zone, dnsmasq config not yet reconciled",
@ -64,7 +85,7 @@ func TestRemovePrivateDNSZone(t *testing.T) {
ListByResourceGroup(ctx, "testGroup", nil).
Return([]mgmtprivatedns.PrivateZone{
{
ID: to.StringPtr("/subscriptions/0000000-0000-0000-0000-000000000000/resourceGroups/testGroup/providers/Microsoft.Network/privateZones/zone1"),
ID: to.StringPtr("/subscriptions/0000000-0000-0000-0000-000000000000/resourceGroups/testGroup/providers/Microsoft.Network/privateDnsZones/zone1"),
},
}, nil)
},
@ -88,7 +109,7 @@ func TestRemovePrivateDNSZone(t *testing.T) {
ListByResourceGroup(ctx, "testGroup", nil).
Return([]mgmtprivatedns.PrivateZone{
{
ID: to.StringPtr("/subscriptions/0000000-0000-0000-0000-000000000000/resourceGroups/testGroup/providers/Microsoft.Network/privateZones/zone1"),
ID: to.StringPtr("/subscriptions/0000000-0000-0000-0000-000000000000/resourceGroups/testGroup/providers/Microsoft.Network/privateDnsZones/zone1"),
},
}, nil)
},
@ -111,7 +132,7 @@ func TestRemovePrivateDNSZone(t *testing.T) {
),
},
{
name: "has private zone, dnsmasq rolled out",
name: "has private zone, node mismatch",
doc: &api.OpenShiftClusterDocument{
OpenShiftCluster: &api.OpenShiftCluster{
Properties: api.OpenShiftClusterProperties{
@ -126,7 +147,105 @@ func TestRemovePrivateDNSZone(t *testing.T) {
ListByResourceGroup(ctx, "testGroup", nil).
Return([]mgmtprivatedns.PrivateZone{
{
ID: to.StringPtr("/subscriptions/0000000-0000-0000-0000-000000000000/resourceGroups/testGroup/providers/Microsoft.Network/privateZones/zone1"),
ID: to.StringPtr("/subscriptions/0000000-0000-0000-0000-000000000000/resourceGroups/testGroup/providers/Microsoft.Network/privateDnsZones/zone1"),
},
}, nil)
},
kubernetescli: fake.NewSimpleClientset(
&corev1.Node{},
),
mcocli: mcofake.NewSimpleClientset(
&mcv1.MachineConfigPool{
ObjectMeta: metav1.ObjectMeta{
Name: "master",
},
Status: mcv1.MachineConfigPoolStatus{
Configuration: mcv1.MachineConfigPoolStatusConfiguration{
Source: []corev1.ObjectReference{
{
Name: "99-master-aro-dns",
},
},
},
},
},
),
},
{
name: "has private zone, nodes match, 4.3, dnsmasq rolled out",
doc: &api.OpenShiftClusterDocument{
OpenShiftCluster: &api.OpenShiftCluster{
Properties: api.OpenShiftClusterProperties{
ClusterProfile: api.ClusterProfile{
ResourceGroupID: resourceGroupID,
},
},
},
},
mocks: func(privateZones *mock_privatedns.MockPrivateZonesClient, virtualNetworkLinks *mock_privatedns.MockVirtualNetworkLinksClient) {
privateZones.EXPECT().
ListByResourceGroup(ctx, "testGroup", nil).
Return([]mgmtprivatedns.PrivateZone{
{
ID: to.StringPtr("/subscriptions/0000000-0000-0000-0000-000000000000/resourceGroups/testGroup/providers/Microsoft.Network/privateDnsZones/zone1"),
},
}, nil)
},
kubernetescli: fake.NewSimpleClientset(
&corev1.Node{},
),
mcocli: mcofake.NewSimpleClientset(
&mcv1.MachineConfigPool{
ObjectMeta: metav1.ObjectMeta{
Name: "master",
},
Status: mcv1.MachineConfigPoolStatus{
Configuration: mcv1.MachineConfigPoolStatusConfiguration{
Source: []corev1.ObjectReference{
{
Name: "99-master-aro-dns",
},
},
},
MachineCount: 1,
UpdatedMachineCount: 1,
ReadyMachineCount: 1,
},
},
),
configcli: configfake.NewSimpleClientset(
&configv1.ClusterVersion{
ObjectMeta: metav1.ObjectMeta{
Name: "version",
},
Status: configv1.ClusterVersionStatus{
History: []configv1.UpdateHistory{
{
State: configv1.CompletedUpdate,
Version: "4.3.999",
},
},
},
},
),
},
{
name: "has private zone, nodes match, 4.4, dnsmasq rolled out",
doc: &api.OpenShiftClusterDocument{
OpenShiftCluster: &api.OpenShiftCluster{
Properties: api.OpenShiftClusterProperties{
ClusterProfile: api.ClusterProfile{
ResourceGroupID: resourceGroupID,
},
},
},
},
mocks: func(privateZones *mock_privatedns.MockPrivateZonesClient, virtualNetworkLinks *mock_privatedns.MockVirtualNetworkLinksClient) {
privateZones.EXPECT().
ListByResourceGroup(ctx, "testGroup", nil).
Return([]mgmtprivatedns.PrivateZone{
{
ID: to.StringPtr("/subscriptions/0000000-0000-0000-0000-000000000000/resourceGroups/testGroup/providers/Microsoft.Network/privateDnsZones/zone1"),
},
}, nil)
@ -146,6 +265,9 @@ func TestRemovePrivateDNSZone(t *testing.T) {
DeleteAndWait(ctx, "testGroup", "zone1", "").
Return(nil)
},
kubernetescli: fake.NewSimpleClientset(
&corev1.Node{},
),
mcocli: mcofake.NewSimpleClientset(
&mcv1.MachineConfigPool{
ObjectMeta: metav1.ObjectMeta{
@ -159,9 +281,38 @@ func TestRemovePrivateDNSZone(t *testing.T) {
},
},
},
MachineCount: 1,
UpdatedMachineCount: 1,
ReadyMachineCount: 1,
},
},
),
configcli: configfake.NewSimpleClientset(
&configv1.ClusterVersion{
ObjectMeta: metav1.ObjectMeta{
Name: "version",
},
Status: configv1.ClusterVersionStatus{
History: []configv1.UpdateHistory{
{
State: configv1.CompletedUpdate,
Version: "4.4.0",
},
},
},
},
&configv1.DNS{
ObjectMeta: metav1.ObjectMeta{
Name: "cluster",
},
Spec: configv1.DNSSpec{
PrivateZone: &configv1.DNSZone{
ID: "/subscriptions/0000000-0000-0000-0000-000000000000/resourceGroups/testGroup/providers/Microsoft.Network/privateDnsZones/zone1",
},
},
},
),
wantDNSPrivateZoneRemoved: true,
},
} {
t.Run(tt.name, func(t *testing.T) {
@ -177,13 +328,25 @@ func TestRemovePrivateDNSZone(t *testing.T) {
doc: tt.doc,
privateZones: privateZones,
virtualNetworkLinks: virtualNetworkLinks,
kubernetescli: tt.kubernetescli,
mcocli: tt.mcocli,
configcli: tt.configcli,
}
err := m.removePrivateDNSZone(ctx)
if err != nil {
t.Fatal(err)
}
if tt.wantDNSPrivateZoneRemoved {
dns, err := m.configcli.ConfigV1().DNSes().Get(ctx, "cluster", metav1.GetOptions{})
if err != nil {
t.Fatal(err)
}
if dns.Spec.PrivateZone != nil {
t.Error(dns.Spec.PrivateZone)
}
}
})
}
}

Просмотреть файл

@ -5,14 +5,11 @@ package workaround
import (
"context"
"fmt"
"time"
configv1 "github.com/openshift/api/config/v1"
configclient "github.com/openshift/client-go/config/clientset/versioned"
mcoclient "github.com/openshift/machine-config-operator/pkg/generated/clientset/versioned"
"github.com/sirupsen/logrus"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
ctrl "sigs.k8s.io/controller-runtime"
@ -52,25 +49,11 @@ func NewReconciler(log *logrus.Entry, kubernetescli kubernetes.Interface, config
}
}
func (r *WorkaroundReconciler) actualClusterVersion(ctx context.Context) (*version.Version, error) {
cv, err := r.configcli.ConfigV1().ClusterVersions().Get(ctx, "version", metav1.GetOptions{})
if err != nil {
return nil, err
}
for _, history := range cv.Status.History {
if history.State == configv1.CompletedUpdate {
return version.ParseVersion(history.Version)
}
}
return nil, fmt.Errorf("unknown cluster version")
}
// Reconcile makes sure that the workarounds are applied or removed as per the OpenShift version.
func (r *WorkaroundReconciler) Reconcile(request ctrl.Request) (ctrl.Result, error) {
// TODO(mj): controller-runtime master fixes the need for this (https://github.com/kubernetes-sigs/controller-runtime/blob/master/pkg/reconcile/reconcile.go#L93) but it's not yet released.
ctx := context.Background()
clusterVersion, err := r.actualClusterVersion(ctx)
clusterVersion, err := version.GetClusterVersion(ctx, r.configcli)
if err != nil {
r.log.Errorf("error getting the OpenShift version: %v", err)
return reconcile.Result{}, err

Просмотреть файл

@ -0,0 +1,28 @@
package version
// Copyright (c) Microsoft Corporation.
// Licensed under the Apache License 2.0.
import (
"context"
"errors"
configv1 "github.com/openshift/api/config/v1"
configclient "github.com/openshift/client-go/config/clientset/versioned"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
func GetClusterVersion(ctx context.Context, configcli configclient.Interface) (*Version, error) {
cv, err := configcli.ConfigV1().ClusterVersions().Get(ctx, "version", metav1.GetOptions{})
if err != nil {
return nil, err
}
for _, history := range cv.Status.History {
if history.State == configv1.CompletedUpdate {
return ParseVersion(history.Version)
}
}
return nil, errors.New("unknown cluster version")
}