Use an enum for cluster maintenance states (#3230)

This commit is contained in:
Nicolas Ontiveros 2023-11-28 13:25:49 -08:00 коммит произвёл GitHub
Родитель 9a9edacf6b
Коммит e4eea7f7a8
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
12 изменённых файлов: 687 добавлений и 131 удалений

Просмотреть файл

@ -57,7 +57,7 @@ type OpenShiftClusterProperties struct {
ImageRegistryStorageAccountName string `json:"imageRegistryStorageAccountName,omitempty"`
InfraID string `json:"infraId,omitempty"`
HiveProfile HiveProfile `json:"hiveProfile,omitempty"`
PucmPending bool `json:"pucmPending,omitempty"`
MaintenanceState MaintenanceState `json:"maintenanceState,omitempty"`
}
// ProvisioningState represents a provisioning state.
@ -82,13 +82,29 @@ const (
FipsValidatedModulesDisabled FipsValidatedModules = "Disabled"
)
// MaintenanceState represents the maintenance state of a cluster.
// This is used by cluster monitornig stack to emit maintenance signals to customers.
type MaintenanceState string
const (
MaintenanceStateNone MaintenanceState = "None"
MaintenanceStatePending MaintenanceState = "Pending"
MaintenanceStatePlanned MaintenanceState = "Planned"
MaintenanceStateUnplanned MaintenanceState = "Unplanned"
)
type MaintenanceTask string
const (
MaintenanceTaskEverything MaintenanceTask = "Everything"
MaintenanceTaskOperator MaintenanceTask = "OperatorUpdate"
MaintenanceTaskRenewCerts MaintenanceTask = "CertificatesRenewal"
MaintenanceTaskPucmPending MaintenanceTask = "PucmPending"
MaintenanceTaskEverything MaintenanceTask = "Everything"
MaintenanceTaskOperator MaintenanceTask = "OperatorUpdate"
MaintenanceTaskRenewCerts MaintenanceTask = "CertificatesRenewal"
// Maintenance tasks for updating customer maintenance signals
// None signal should only be used when (1) admin update fails and (2) SRE fixes the failed admin update without running another admin updates
// Admin update success should automatically set the cluster into None state
MaintenanceTaskPending MaintenanceTask = "Pending"
MaintenanceTaskNone MaintenanceTask = "None"
)
// Operator feature flags

Просмотреть файл

@ -31,7 +31,7 @@ func (c openShiftClusterConverter) ToExternal(oc *api.OpenShiftCluster) interfac
CreatedAt: oc.Properties.CreatedAt,
CreatedBy: oc.Properties.CreatedBy,
ProvisionedBy: oc.Properties.ProvisionedBy,
PucmPending: oc.Properties.PucmPending,
MaintenanceState: MaintenanceState(oc.Properties.MaintenanceState),
ClusterProfile: ClusterProfile{
Domain: oc.Properties.ClusterProfile.Domain,
Version: oc.Properties.ClusterProfile.Version,
@ -234,7 +234,7 @@ func (c openShiftClusterConverter) ToInternal(_oc interface{}, out *api.OpenShif
out.Properties.OperatorVersion = oc.Properties.OperatorVersion
out.Properties.CreatedBy = oc.Properties.CreatedBy
out.Properties.ProvisionedBy = oc.Properties.ProvisionedBy
out.Properties.PucmPending = oc.Properties.PucmPending
out.Properties.MaintenanceState = api.MaintenanceState(oc.Properties.MaintenanceState)
out.Properties.ClusterProfile.Domain = oc.Properties.ClusterProfile.Domain
out.Properties.ClusterProfile.FipsValidatedModules = api.FipsValidatedModules(oc.Properties.ClusterProfile.FipsValidatedModules)
out.Properties.ClusterProfile.Version = oc.Properties.ClusterProfile.Version

Просмотреть файл

@ -37,7 +37,8 @@ func validateMaintenanceTask(task MaintenanceTask) error {
task == MaintenanceTaskEverything ||
task == MaintenanceTaskOperator ||
task == MaintenanceTaskRenewCerts ||
task == MaintenanceTaskPucmPending) {
task == MaintenanceTaskPending ||
task == MaintenanceTaskNone) {
return api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodeInvalidParameter, "properties.maintenanceTask", "Invalid enum parameter.")
}

Просмотреть файл

@ -677,11 +677,24 @@ func TestOpenShiftClusterStaticValidateDelta(t *testing.T) {
},
},
{
name: "maintenanceTask change to PucmPending allowed",
name: "maintenanceTask change to pending allowed",
oc: func() *OpenShiftCluster {
return &OpenShiftCluster{
Properties: OpenShiftClusterProperties{
MaintenanceTask: MaintenanceTaskPucmPending,
MaintenanceTask: MaintenanceTaskPending,
},
}
},
modify: func(oc *OpenShiftCluster) {
oc.Properties.MaintenanceTask = ""
},
},
{
name: "maintenanceTask change to none allowed",
oc: func() *OpenShiftCluster {
return &OpenShiftCluster{
Properties: OpenShiftClusterProperties{
MaintenanceTask: MaintenanceTaskNone,
},
}
},

Просмотреть файл

@ -162,7 +162,7 @@ type OpenShiftClusterProperties struct {
HiveProfile HiveProfile `json:"hiveProfile,omitempty"`
PucmPending bool `json:"pucmPending,omitempty"`
MaintenanceState MaintenanceState `json:"maintenanceState,omitempty"`
}
// ProvisioningState represents a provisioning state
@ -180,15 +180,40 @@ const (
ProvisioningStateFailed ProvisioningState = "Failed"
)
// MaintenanceState represents the maintenance state of a cluster.
// This is used by cluster monitornig stack to emit maintenance signals to customers.
type MaintenanceState string
const (
MaintenanceStateNone MaintenanceState = "None"
MaintenanceStatePending MaintenanceState = "Pending"
MaintenanceStatePlanned MaintenanceState = "Planned"
MaintenanceStateUnplanned MaintenanceState = "Unplanned"
)
type MaintenanceTask string
const (
MaintenanceTaskEverything MaintenanceTask = "Everything"
MaintenanceTaskOperator MaintenanceTask = "OperatorUpdate"
MaintenanceTaskRenewCerts MaintenanceTask = "CertificatesRenewal"
MaintenanceTaskPucmPending MaintenanceTask = "PucmPending"
MaintenanceTaskEverything MaintenanceTask = "Everything"
MaintenanceTaskOperator MaintenanceTask = "OperatorUpdate"
MaintenanceTaskRenewCerts MaintenanceTask = "CertificatesRenewal"
// Maintenance tasks for updating customer maintenance signals
// None signal should only be used when (1) admin update fails and (2) SRE fixes the failed admin update without running another admin updates
// Admin update success should automatically set the cluster into None state
MaintenanceTaskPending MaintenanceTask = "Pending"
MaintenanceTaskNone MaintenanceTask = "None"
)
// IsMaintenanceOngoingTask returns true if the maintenance task should change state to maintenance ongoing (planned/unplanned)
func (t MaintenanceTask) IsMaintenanceOngoingTask() bool {
result := (t == MaintenanceTaskEverything) ||
(t == MaintenanceTaskOperator) ||
(t == MaintenanceTaskRenewCerts) ||
(t == "")
return result
}
// Cluster-scoped flags
type OperatorFlags map[string]string

Просмотреть файл

@ -159,9 +159,11 @@ func (ocb *openShiftClusterBackend) handle(ctx context.Context, log *logrus.Entr
err = m.AdminUpdate(ctx)
if err != nil {
// Customer will continue to see the cluster in an ongoing maintenance state
return ocb.endLease(ctx, log, stop, doc, api.ProvisioningStateFailed, err)
}
doc, err = ocb.setNoPucmPending(ctx, doc)
// Maintenance task is complete, so we can clear the maintenance state
doc, err = ocb.setNoMaintenanceState(ctx, doc)
if err != nil {
return ocb.endLease(ctx, log, stop, doc, api.ProvisioningStateFailed, err)
}
@ -366,9 +368,9 @@ func (ocb *openShiftClusterBackend) emitMetrics(doc *api.OpenShiftClusterDocumen
})
}
func (ocb *openShiftClusterBackend) setNoPucmPending(ctx context.Context, doc *api.OpenShiftClusterDocument) (*api.OpenShiftClusterDocument, error) {
func (ocb *openShiftClusterBackend) setNoMaintenanceState(ctx context.Context, doc *api.OpenShiftClusterDocument) (*api.OpenShiftClusterDocument, error) {
return ocb.dbOpenShiftClusters.Patch(ctx, doc.Key, func(doc *api.OpenShiftClusterDocument) error {
doc.OpenShiftCluster.Properties.PucmPending = false
doc.OpenShiftCluster.Properties.MaintenanceState = api.MaintenanceStateNone
return nil
})
}

Просмотреть файл

@ -172,7 +172,7 @@ func TestBackendTry(t *testing.T) {
},
},
{
name: "StateAdminUpdating success sets the last ProvisioningState and clears LastAdminUpdateError and MaintenanceTask",
name: "StateAdminUpdating success sets the last ProvisioningState, clears LastAdminUpdateError and MaintenanceTask, and has maintenance state none",
fixture: func(f *testdatabase.Fixture) {
f.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{
Key: strings.ToLower(resourceID),
@ -186,6 +186,7 @@ func TestBackendTry(t *testing.T) {
LastProvisioningState: api.ProvisioningStateSucceeded,
LastAdminUpdateError: "oh no",
MaintenanceTask: api.MaintenanceTaskEverything,
MaintenanceState: api.MaintenanceStateUnplanned,
},
},
})
@ -203,6 +204,7 @@ func TestBackendTry(t *testing.T) {
Location: "location",
Properties: api.OpenShiftClusterProperties{
ProvisioningState: api.ProvisioningStateSucceeded,
MaintenanceState: api.MaintenanceStateNone,
},
},
})
@ -212,7 +214,7 @@ func TestBackendTry(t *testing.T) {
},
},
{
name: "StateAdminUpdating run failure populates LastAdminUpdateError and restores previous provisioning state + failed provisioning state",
name: "StateAdminUpdating run failure populates LastAdminUpdateError, restores previous provisioning state + failed provisioning state, and sets maintenance state to ongoing",
fixture: func(f *testdatabase.Fixture) {
f.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{
Key: strings.ToLower(resourceID),
@ -226,6 +228,7 @@ func TestBackendTry(t *testing.T) {
LastProvisioningState: api.ProvisioningStateSucceeded,
FailedProvisioningState: api.ProvisioningStateUpdating,
MaintenanceTask: api.MaintenanceTaskEverything,
MaintenanceState: api.MaintenanceStateUnplanned,
},
},
})
@ -245,6 +248,7 @@ func TestBackendTry(t *testing.T) {
ProvisioningState: api.ProvisioningStateSucceeded,
FailedProvisioningState: api.ProvisioningStateUpdating,
LastAdminUpdateError: "oh no!",
MaintenanceState: api.MaintenanceStateUnplanned,
},
},
})

Просмотреть файл

@ -310,24 +310,43 @@ func (f *frontend) ValidateNewCluster(ctx context.Context, subscription *api.Sub
func setUpdateProvisioningState(doc *api.OpenShiftClusterDocument, apiVersion string) {
switch apiVersion {
case admin.APIVersion:
// For PUCM pending update, we don't want to set ProvisioningStateAdminUpdating
// The cluster monitoring stack uses that value to determine if PUCM is ongoing
if doc.OpenShiftCluster.Properties.MaintenanceTask != api.MaintenanceTaskPucmPending {
doc.OpenShiftCluster.Properties.LastProvisioningState = doc.OpenShiftCluster.Properties.ProvisioningState
doc.OpenShiftCluster.Properties.ProvisioningState = api.ProvisioningStateAdminUpdating
doc.OpenShiftCluster.Properties.LastAdminUpdateError = ""
doc.Dequeues = 0
} else {
// No update to provisioning state needed
doc.OpenShiftCluster.Properties.PucmPending = true
// This enables future admin update actions with body `{}` to succeed
doc.OpenShiftCluster.Properties.MaintenanceTask = ""
}
adminUpdateProvisioningState(doc)
default:
// Non-admin update (ex: customer cluster update)
doc.OpenShiftCluster.Properties.LastProvisioningState = doc.OpenShiftCluster.Properties.ProvisioningState
doc.OpenShiftCluster.Properties.ProvisioningState = api.ProvisioningStateUpdating
doc.Dequeues = 0
updateProvisioningState(doc)
}
}
// Non-admin update (ex: customer cluster update)
func updateProvisioningState(doc *api.OpenShiftClusterDocument) {
doc.OpenShiftCluster.Properties.LastProvisioningState = doc.OpenShiftCluster.Properties.ProvisioningState
doc.OpenShiftCluster.Properties.ProvisioningState = api.ProvisioningStateUpdating
doc.Dequeues = 0
}
// Admin update (ex: cluster maintenance)
func adminUpdateProvisioningState(doc *api.OpenShiftClusterDocument) {
if doc.OpenShiftCluster.Properties.MaintenanceTask.IsMaintenanceOngoingTask() {
doc.OpenShiftCluster.Properties.LastProvisioningState = doc.OpenShiftCluster.Properties.ProvisioningState
doc.OpenShiftCluster.Properties.ProvisioningState = api.ProvisioningStateAdminUpdating
doc.OpenShiftCluster.Properties.LastAdminUpdateError = ""
doc.Dequeues = 0
// Set the maintenance to ongoing so we emit the appropriate signal to customerss
if doc.OpenShiftCluster.Properties.MaintenanceState == api.MaintenanceStatePending {
doc.OpenShiftCluster.Properties.MaintenanceState = api.MaintenanceStatePlanned
} else {
doc.OpenShiftCluster.Properties.MaintenanceState = api.MaintenanceStateUnplanned
}
} else {
// No default needed since we're using an enum
switch doc.OpenShiftCluster.Properties.MaintenanceTask {
case api.MaintenanceTaskPending:
doc.OpenShiftCluster.Properties.MaintenanceState = api.MaintenanceStatePending
case api.MaintenanceTaskNone:
doc.OpenShiftCluster.Properties.MaintenanceState = api.MaintenanceStateNone
}
// This enables future admin update actions with body `{}` to succeed
doc.OpenShiftCluster.Properties.MaintenanceTask = ""
}
}

Просмотреть файл

@ -121,7 +121,8 @@ func TestPutOrPatchOpenShiftClusterAdminAPI(t *testing.T) {
MasterProfile: api.MasterProfile{
EncryptionAtHost: api.EncryptionAtHostDisabled,
},
OperatorFlags: api.OperatorFlags{"testFlag": "true"},
OperatorFlags: api.OperatorFlags{"testFlag": "true"},
MaintenanceState: api.MaintenanceStateUnplanned,
},
},
})
@ -150,7 +151,8 @@ func TestPutOrPatchOpenShiftClusterAdminAPI(t *testing.T) {
MasterProfile: admin.MasterProfile{
EncryptionAtHost: admin.EncryptionAtHostDisabled,
},
OperatorFlags: admin.OperatorFlags{"testFlag": "true"},
OperatorFlags: admin.OperatorFlags{"testFlag": "true"},
MaintenanceState: admin.MaintenanceStateUnplanned,
},
},
},
@ -220,7 +222,8 @@ func TestPutOrPatchOpenShiftClusterAdminAPI(t *testing.T) {
MasterProfile: api.MasterProfile{
EncryptionAtHost: api.EncryptionAtHostDisabled,
},
OperatorFlags: api.OperatorFlags{"exploding-flag": "true", "overwrittenFlag": "true", "testFlag": "true"},
OperatorFlags: api.OperatorFlags{"exploding-flag": "true", "overwrittenFlag": "true", "testFlag": "true"},
MaintenanceState: api.MaintenanceStateUnplanned,
},
},
})
@ -249,7 +252,8 @@ func TestPutOrPatchOpenShiftClusterAdminAPI(t *testing.T) {
MasterProfile: admin.MasterProfile{
EncryptionAtHost: admin.EncryptionAtHostDisabled,
},
OperatorFlags: admin.OperatorFlags{"exploding-flag": "true", "overwrittenFlag": "true", "testFlag": "true"},
OperatorFlags: admin.OperatorFlags{"exploding-flag": "true", "overwrittenFlag": "true", "testFlag": "true"},
MaintenanceState: admin.MaintenanceStateUnplanned,
},
},
},
@ -315,7 +319,8 @@ func TestPutOrPatchOpenShiftClusterAdminAPI(t *testing.T) {
MasterProfile: api.MasterProfile{
EncryptionAtHost: api.EncryptionAtHostDisabled,
},
OperatorFlags: api.DefaultOperatorFlags(),
OperatorFlags: api.DefaultOperatorFlags(),
MaintenanceState: api.MaintenanceStateUnplanned,
},
},
})
@ -344,7 +349,8 @@ func TestPutOrPatchOpenShiftClusterAdminAPI(t *testing.T) {
ClusterProfile: admin.ClusterProfile{
FipsValidatedModules: admin.FipsValidatedModulesDisabled,
},
OperatorFlags: admin.OperatorFlags(api.DefaultOperatorFlags()),
OperatorFlags: admin.OperatorFlags(api.DefaultOperatorFlags()),
MaintenanceState: admin.MaintenanceStateUnplanned,
},
},
},
@ -408,7 +414,8 @@ func TestPutOrPatchOpenShiftClusterAdminAPI(t *testing.T) {
MasterProfile: api.MasterProfile{
EncryptionAtHost: api.EncryptionAtHostDisabled,
},
OperatorFlags: api.DefaultOperatorFlags(),
OperatorFlags: api.DefaultOperatorFlags(),
MaintenanceState: api.MaintenanceStateUnplanned,
},
},
})
@ -437,7 +444,8 @@ func TestPutOrPatchOpenShiftClusterAdminAPI(t *testing.T) {
MasterProfile: admin.MasterProfile{
EncryptionAtHost: admin.EncryptionAtHostDisabled,
},
OperatorFlags: admin.OperatorFlags(api.DefaultOperatorFlags()),
OperatorFlags: admin.OperatorFlags(api.DefaultOperatorFlags()),
MaintenanceState: admin.MaintenanceStateUnplanned,
},
},
},
@ -464,6 +472,7 @@ func TestPutOrPatchOpenShiftClusterAdminAPI(t *testing.T) {
ProvisioningState: api.ProvisioningStateSucceeded,
MaintenanceTask: api.MaintenanceTaskEverything,
OperatorFlags: api.OperatorFlags{"testFlag": "true"},
MaintenanceState: api.MaintenanceStateNone,
},
},
})
@ -503,7 +512,8 @@ func TestPutOrPatchOpenShiftClusterAdminAPI(t *testing.T) {
MasterProfile: api.MasterProfile{
EncryptionAtHost: api.EncryptionAtHostDisabled,
},
OperatorFlags: api.OperatorFlags{"testFlag": "true"},
OperatorFlags: api.OperatorFlags{"testFlag": "true"},
MaintenanceState: api.MaintenanceStateUnplanned,
},
},
})
@ -532,7 +542,8 @@ func TestPutOrPatchOpenShiftClusterAdminAPI(t *testing.T) {
MasterProfile: admin.MasterProfile{
EncryptionAtHost: admin.EncryptionAtHostDisabled,
},
OperatorFlags: admin.OperatorFlags{"testFlag": "true"},
OperatorFlags: admin.OperatorFlags{"testFlag": "true"},
MaintenanceState: admin.MaintenanceStateUnplanned,
},
},
},
@ -618,9 +629,9 @@ func TestPutOrPatchOpenShiftClusterAdminAPI(t *testing.T) {
wantError: `400: PropertyChangeNotAllowed: properties.registryProfiles: Changing property 'properties.registryProfiles' is not allowed.`,
},
{
name: "patch a cluster with pucm pending request",
name: "patch an empty maintenance state cluster with maintenance pending request",
request: func(oc *admin.OpenShiftCluster) {
oc.Properties.MaintenanceTask = admin.MaintenanceTaskPucmPending
oc.Properties.MaintenanceTask = admin.MaintenanceTaskPending
},
isPatch: true,
fixture: func(f *testdatabase.Fixture) {
@ -679,8 +690,8 @@ func TestPutOrPatchOpenShiftClusterAdminAPI(t *testing.T) {
MasterProfile: api.MasterProfile{
EncryptionAtHost: api.EncryptionAtHostDisabled,
},
PucmPending: true,
OperatorFlags: api.DefaultOperatorFlags(),
MaintenanceState: api.MaintenanceStatePending,
OperatorFlags: api.DefaultOperatorFlags(),
},
},
})
@ -706,7 +717,7 @@ func TestPutOrPatchOpenShiftClusterAdminAPI(t *testing.T) {
},
},
},
PucmPending: true,
MaintenanceState: admin.MaintenanceStatePending,
MasterProfile: admin.MasterProfile{
EncryptionAtHost: admin.EncryptionAtHostDisabled,
},
@ -714,6 +725,493 @@ func TestPutOrPatchOpenShiftClusterAdminAPI(t *testing.T) {
},
},
},
{
name: "patch a none maintenance state cluster with maintenance pending request",
request: func(oc *admin.OpenShiftCluster) {
oc.Properties.MaintenanceTask = admin.MaintenanceTaskPending
},
isPatch: true,
fixture: func(f *testdatabase.Fixture) {
f.AddSubscriptionDocuments(&api.SubscriptionDocument{
ID: mockSubID,
Subscription: &api.Subscription{
State: api.SubscriptionStateRegistered,
},
})
f.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{
Key: strings.ToLower(testdatabase.GetResourcePath(mockSubID, "resourceName")),
OpenShiftCluster: &api.OpenShiftCluster{
ID: testdatabase.GetResourcePath(mockSubID, "resourceName"),
Type: "Microsoft.RedHatOpenShift/openShiftClusters",
Tags: map[string]string{"tag": "will-be-kept"},
Properties: api.OpenShiftClusterProperties{
ProvisioningState: api.ProvisioningStateSucceeded,
LastProvisioningState: api.ProvisioningStateSucceeded,
MaintenanceTask: "",
MaintenanceState: api.MaintenanceStateNone,
},
},
})
},
wantSystemDataEnriched: true,
wantEnriched: []string{testdatabase.GetResourcePath(mockSubID, "resourceName")},
wantDocuments: func(c *testdatabase.Checker) {
c.AddAsyncOperationDocuments(&api.AsyncOperationDocument{
OpenShiftClusterKey: strings.ToLower(testdatabase.GetResourcePath(mockSubID, "resourceName")),
AsyncOperation: &api.AsyncOperation{
InitialProvisioningState: api.ProvisioningStateSucceeded,
ProvisioningState: api.ProvisioningStateSucceeded,
},
})
c.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{
Key: strings.ToLower(testdatabase.GetResourcePath(mockSubID, "resourceName")),
OpenShiftCluster: &api.OpenShiftCluster{
ID: testdatabase.GetResourcePath(mockSubID, "resourceName"),
Type: "Microsoft.RedHatOpenShift/openShiftClusters",
Tags: map[string]string{"tag": "will-be-kept"},
Properties: api.OpenShiftClusterProperties{
ProvisioningState: api.ProvisioningStateSucceeded,
LastProvisioningState: api.ProvisioningStateSucceeded,
ClusterProfile: api.ClusterProfile{
FipsValidatedModules: api.FipsValidatedModulesDisabled,
},
MaintenanceTask: "",
NetworkProfile: api.NetworkProfile{
OutboundType: api.OutboundTypeLoadbalancer,
PreconfiguredNSG: api.PreconfiguredNSGDisabled,
LoadBalancerProfile: &api.LoadBalancerProfile{
ManagedOutboundIPs: &api.ManagedOutboundIPs{
Count: 1,
},
},
},
MasterProfile: api.MasterProfile{
EncryptionAtHost: api.EncryptionAtHostDisabled,
},
MaintenanceState: api.MaintenanceStatePending,
OperatorFlags: api.DefaultOperatorFlags(),
},
},
})
},
wantAsync: true,
wantStatusCode: http.StatusOK,
wantResponse: &admin.OpenShiftCluster{
ID: testdatabase.GetResourcePath(mockSubID, "resourceName"),
Type: "Microsoft.RedHatOpenShift/openShiftClusters",
Tags: map[string]string{"tag": "will-be-kept"},
Properties: admin.OpenShiftClusterProperties{
ProvisioningState: admin.ProvisioningStateSucceeded,
LastProvisioningState: admin.ProvisioningStateSucceeded,
ClusterProfile: admin.ClusterProfile{
FipsValidatedModules: admin.FipsValidatedModulesDisabled,
},
MaintenanceTask: "",
NetworkProfile: admin.NetworkProfile{
OutboundType: admin.OutboundTypeLoadbalancer,
LoadBalancerProfile: &admin.LoadBalancerProfile{
ManagedOutboundIPs: &admin.ManagedOutboundIPs{
Count: 1,
},
},
},
MaintenanceState: admin.MaintenanceStatePending,
MasterProfile: admin.MasterProfile{
EncryptionAtHost: admin.EncryptionAtHostDisabled,
},
OperatorFlags: admin.OperatorFlags(api.DefaultOperatorFlags()),
},
},
},
{
name: "patch a maintenance state pending cluster with planned maintenance",
request: func(oc *admin.OpenShiftCluster) {
oc.Properties.MaintenanceTask = admin.MaintenanceTaskEverything
},
isPatch: true,
fixture: func(f *testdatabase.Fixture) {
f.AddSubscriptionDocuments(&api.SubscriptionDocument{
ID: mockSubID,
Subscription: &api.Subscription{
State: api.SubscriptionStateRegistered,
},
})
f.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{
Key: strings.ToLower(testdatabase.GetResourcePath(mockSubID, "resourceName")),
OpenShiftCluster: &api.OpenShiftCluster{
ID: testdatabase.GetResourcePath(mockSubID, "resourceName"),
Type: "Microsoft.RedHatOpenShift/openShiftClusters",
Tags: map[string]string{"tag": "will-be-kept"},
Properties: api.OpenShiftClusterProperties{
ProvisioningState: api.ProvisioningStateSucceeded,
LastProvisioningState: api.ProvisioningStateSucceeded,
MaintenanceState: api.MaintenanceStatePending,
},
},
})
},
wantSystemDataEnriched: true,
wantEnriched: []string{testdatabase.GetResourcePath(mockSubID, "resourceName")},
wantDocuments: func(c *testdatabase.Checker) {
c.AddAsyncOperationDocuments(&api.AsyncOperationDocument{
OpenShiftClusterKey: strings.ToLower(testdatabase.GetResourcePath(mockSubID, "resourceName")),
AsyncOperation: &api.AsyncOperation{
InitialProvisioningState: api.ProvisioningStateAdminUpdating,
ProvisioningState: api.ProvisioningStateAdminUpdating,
},
})
c.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{
Key: strings.ToLower(testdatabase.GetResourcePath(mockSubID, "resourceName")),
OpenShiftCluster: &api.OpenShiftCluster{
ID: testdatabase.GetResourcePath(mockSubID, "resourceName"),
Type: "Microsoft.RedHatOpenShift/openShiftClusters",
Tags: map[string]string{"tag": "will-be-kept"},
Properties: api.OpenShiftClusterProperties{
ProvisioningState: api.ProvisioningStateAdminUpdating,
LastProvisioningState: api.ProvisioningStateSucceeded,
ClusterProfile: api.ClusterProfile{
FipsValidatedModules: api.FipsValidatedModulesDisabled,
},
MaintenanceTask: api.MaintenanceTaskEverything,
NetworkProfile: api.NetworkProfile{
OutboundType: api.OutboundTypeLoadbalancer,
PreconfiguredNSG: api.PreconfiguredNSGDisabled,
LoadBalancerProfile: &api.LoadBalancerProfile{
ManagedOutboundIPs: &api.ManagedOutboundIPs{
Count: 1,
},
},
},
MasterProfile: api.MasterProfile{
EncryptionAtHost: api.EncryptionAtHostDisabled,
},
OperatorFlags: api.DefaultOperatorFlags(),
MaintenanceState: api.MaintenanceStatePlanned,
},
},
})
},
wantAsync: true,
wantStatusCode: http.StatusOK,
wantResponse: &admin.OpenShiftCluster{
ID: testdatabase.GetResourcePath(mockSubID, "resourceName"),
Type: "Microsoft.RedHatOpenShift/openShiftClusters",
Tags: map[string]string{"tag": "will-be-kept"},
Properties: admin.OpenShiftClusterProperties{
ProvisioningState: admin.ProvisioningStateAdminUpdating,
LastProvisioningState: admin.ProvisioningStateSucceeded,
ClusterProfile: admin.ClusterProfile{
FipsValidatedModules: admin.FipsValidatedModulesDisabled,
},
MaintenanceTask: admin.MaintenanceTaskEverything,
NetworkProfile: admin.NetworkProfile{
OutboundType: admin.OutboundTypeLoadbalancer,
LoadBalancerProfile: &admin.LoadBalancerProfile{
ManagedOutboundIPs: &admin.ManagedOutboundIPs{
Count: 1,
},
},
},
MasterProfile: admin.MasterProfile{
EncryptionAtHost: admin.EncryptionAtHostDisabled,
},
OperatorFlags: admin.OperatorFlags(api.DefaultOperatorFlags()),
MaintenanceState: admin.MaintenanceStatePlanned,
},
},
},
{
name: "patch a planned maintenance ongoing cluster with maintenance none request",
request: func(oc *admin.OpenShiftCluster) {
oc.Properties.MaintenanceTask = admin.MaintenanceTaskNone
},
isPatch: true,
fixture: func(f *testdatabase.Fixture) {
f.AddSubscriptionDocuments(&api.SubscriptionDocument{
ID: mockSubID,
Subscription: &api.Subscription{
State: api.SubscriptionStateRegistered,
},
})
f.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{
Key: strings.ToLower(testdatabase.GetResourcePath(mockSubID, "resourceName")),
OpenShiftCluster: &api.OpenShiftCluster{
ID: testdatabase.GetResourcePath(mockSubID, "resourceName"),
Type: "Microsoft.RedHatOpenShift/openShiftClusters",
Tags: map[string]string{"tag": "will-be-kept"},
Properties: api.OpenShiftClusterProperties{
ProvisioningState: api.ProvisioningStateSucceeded,
LastProvisioningState: api.ProvisioningStateSucceeded,
MaintenanceTask: "",
MaintenanceState: api.MaintenanceStatePlanned,
},
},
})
},
wantSystemDataEnriched: true,
wantEnriched: []string{testdatabase.GetResourcePath(mockSubID, "resourceName")},
wantDocuments: func(c *testdatabase.Checker) {
c.AddAsyncOperationDocuments(&api.AsyncOperationDocument{
OpenShiftClusterKey: strings.ToLower(testdatabase.GetResourcePath(mockSubID, "resourceName")),
AsyncOperation: &api.AsyncOperation{
InitialProvisioningState: api.ProvisioningStateSucceeded,
ProvisioningState: api.ProvisioningStateSucceeded,
},
})
c.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{
Key: strings.ToLower(testdatabase.GetResourcePath(mockSubID, "resourceName")),
OpenShiftCluster: &api.OpenShiftCluster{
ID: testdatabase.GetResourcePath(mockSubID, "resourceName"),
Type: "Microsoft.RedHatOpenShift/openShiftClusters",
Tags: map[string]string{"tag": "will-be-kept"},
Properties: api.OpenShiftClusterProperties{
ProvisioningState: api.ProvisioningStateSucceeded,
LastProvisioningState: api.ProvisioningStateSucceeded,
ClusterProfile: api.ClusterProfile{
FipsValidatedModules: api.FipsValidatedModulesDisabled,
},
MaintenanceTask: "",
NetworkProfile: api.NetworkProfile{
OutboundType: api.OutboundTypeLoadbalancer,
PreconfiguredNSG: api.PreconfiguredNSGDisabled,
LoadBalancerProfile: &api.LoadBalancerProfile{
ManagedOutboundIPs: &api.ManagedOutboundIPs{
Count: 1,
},
},
},
MasterProfile: api.MasterProfile{
EncryptionAtHost: api.EncryptionAtHostDisabled,
},
MaintenanceState: api.MaintenanceStateNone,
OperatorFlags: api.DefaultOperatorFlags(),
},
},
})
},
wantAsync: true,
wantStatusCode: http.StatusOK,
wantResponse: &admin.OpenShiftCluster{
ID: testdatabase.GetResourcePath(mockSubID, "resourceName"),
Type: "Microsoft.RedHatOpenShift/openShiftClusters",
Tags: map[string]string{"tag": "will-be-kept"},
Properties: admin.OpenShiftClusterProperties{
ProvisioningState: admin.ProvisioningStateSucceeded,
LastProvisioningState: admin.ProvisioningStateSucceeded,
ClusterProfile: admin.ClusterProfile{
FipsValidatedModules: admin.FipsValidatedModulesDisabled,
},
MaintenanceTask: "",
NetworkProfile: admin.NetworkProfile{
OutboundType: admin.OutboundTypeLoadbalancer,
LoadBalancerProfile: &admin.LoadBalancerProfile{
ManagedOutboundIPs: &admin.ManagedOutboundIPs{
Count: 1,
},
},
},
MaintenanceState: admin.MaintenanceStateNone,
MasterProfile: admin.MasterProfile{
EncryptionAtHost: admin.EncryptionAtHostDisabled,
},
OperatorFlags: admin.OperatorFlags(api.DefaultOperatorFlags()),
},
},
},
{
name: "patch an unplanned maintenance ongoing cluster with maintenance none request",
request: func(oc *admin.OpenShiftCluster) {
oc.Properties.MaintenanceTask = admin.MaintenanceTaskNone
},
isPatch: true,
fixture: func(f *testdatabase.Fixture) {
f.AddSubscriptionDocuments(&api.SubscriptionDocument{
ID: mockSubID,
Subscription: &api.Subscription{
State: api.SubscriptionStateRegistered,
},
})
f.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{
Key: strings.ToLower(testdatabase.GetResourcePath(mockSubID, "resourceName")),
OpenShiftCluster: &api.OpenShiftCluster{
ID: testdatabase.GetResourcePath(mockSubID, "resourceName"),
Type: "Microsoft.RedHatOpenShift/openShiftClusters",
Tags: map[string]string{"tag": "will-be-kept"},
Properties: api.OpenShiftClusterProperties{
ProvisioningState: api.ProvisioningStateSucceeded,
LastProvisioningState: api.ProvisioningStateSucceeded,
MaintenanceTask: "",
MaintenanceState: api.MaintenanceStateUnplanned,
},
},
})
},
wantSystemDataEnriched: true,
wantEnriched: []string{testdatabase.GetResourcePath(mockSubID, "resourceName")},
wantDocuments: func(c *testdatabase.Checker) {
c.AddAsyncOperationDocuments(&api.AsyncOperationDocument{
OpenShiftClusterKey: strings.ToLower(testdatabase.GetResourcePath(mockSubID, "resourceName")),
AsyncOperation: &api.AsyncOperation{
InitialProvisioningState: api.ProvisioningStateSucceeded,
ProvisioningState: api.ProvisioningStateSucceeded,
},
})
c.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{
Key: strings.ToLower(testdatabase.GetResourcePath(mockSubID, "resourceName")),
OpenShiftCluster: &api.OpenShiftCluster{
ID: testdatabase.GetResourcePath(mockSubID, "resourceName"),
Type: "Microsoft.RedHatOpenShift/openShiftClusters",
Tags: map[string]string{"tag": "will-be-kept"},
Properties: api.OpenShiftClusterProperties{
ProvisioningState: api.ProvisioningStateSucceeded,
LastProvisioningState: api.ProvisioningStateSucceeded,
ClusterProfile: api.ClusterProfile{
FipsValidatedModules: api.FipsValidatedModulesDisabled,
},
MaintenanceTask: "",
NetworkProfile: api.NetworkProfile{
OutboundType: api.OutboundTypeLoadbalancer,
PreconfiguredNSG: api.PreconfiguredNSGDisabled,
LoadBalancerProfile: &api.LoadBalancerProfile{
ManagedOutboundIPs: &api.ManagedOutboundIPs{
Count: 1,
},
},
},
MasterProfile: api.MasterProfile{
EncryptionAtHost: api.EncryptionAtHostDisabled,
},
MaintenanceState: api.MaintenanceStateNone,
OperatorFlags: api.DefaultOperatorFlags(),
},
},
})
},
wantAsync: true,
wantStatusCode: http.StatusOK,
wantResponse: &admin.OpenShiftCluster{
ID: testdatabase.GetResourcePath(mockSubID, "resourceName"),
Type: "Microsoft.RedHatOpenShift/openShiftClusters",
Tags: map[string]string{"tag": "will-be-kept"},
Properties: admin.OpenShiftClusterProperties{
ProvisioningState: admin.ProvisioningStateSucceeded,
LastProvisioningState: admin.ProvisioningStateSucceeded,
ClusterProfile: admin.ClusterProfile{
FipsValidatedModules: admin.FipsValidatedModulesDisabled,
},
MaintenanceTask: "",
NetworkProfile: admin.NetworkProfile{
OutboundType: admin.OutboundTypeLoadbalancer,
LoadBalancerProfile: &admin.LoadBalancerProfile{
ManagedOutboundIPs: &admin.ManagedOutboundIPs{
Count: 1,
},
},
},
MaintenanceState: admin.MaintenanceStateNone,
MasterProfile: admin.MasterProfile{
EncryptionAtHost: admin.EncryptionAtHostDisabled,
},
OperatorFlags: admin.OperatorFlags(api.DefaultOperatorFlags()),
},
},
},
{
name: "patch a none maintenance state cluster with maintenance unplanned request",
request: func(oc *admin.OpenShiftCluster) {
},
isPatch: true,
fixture: func(f *testdatabase.Fixture) {
f.AddSubscriptionDocuments(&api.SubscriptionDocument{
ID: mockSubID,
Subscription: &api.Subscription{
State: api.SubscriptionStateRegistered,
},
})
f.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{
Key: strings.ToLower(testdatabase.GetResourcePath(mockSubID, "resourceName")),
OpenShiftCluster: &api.OpenShiftCluster{
ID: testdatabase.GetResourcePath(mockSubID, "resourceName"),
Type: "Microsoft.RedHatOpenShift/openShiftClusters",
Tags: map[string]string{"tag": "will-be-kept"},
Properties: api.OpenShiftClusterProperties{
ProvisioningState: api.ProvisioningStateSucceeded,
OperatorFlags: api.OperatorFlags{"testFlag": "true"},
MaintenanceState: api.MaintenanceStateNone,
},
},
})
},
wantSystemDataEnriched: true,
wantEnriched: []string{testdatabase.GetResourcePath(mockSubID, "resourceName")},
wantDocuments: func(c *testdatabase.Checker) {
c.AddAsyncOperationDocuments(&api.AsyncOperationDocument{
OpenShiftClusterKey: strings.ToLower(testdatabase.GetResourcePath(mockSubID, "resourceName")),
AsyncOperation: &api.AsyncOperation{
InitialProvisioningState: api.ProvisioningStateAdminUpdating,
ProvisioningState: api.ProvisioningStateAdminUpdating,
},
})
c.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{
Key: strings.ToLower(testdatabase.GetResourcePath(mockSubID, "resourceName")),
OpenShiftCluster: &api.OpenShiftCluster{
ID: testdatabase.GetResourcePath(mockSubID, "resourceName"),
Type: "Microsoft.RedHatOpenShift/openShiftClusters",
Tags: map[string]string{"tag": "will-be-kept"},
Properties: api.OpenShiftClusterProperties{
ProvisioningState: api.ProvisioningStateAdminUpdating,
LastProvisioningState: api.ProvisioningStateSucceeded,
ClusterProfile: api.ClusterProfile{
FipsValidatedModules: api.FipsValidatedModulesDisabled,
},
MaintenanceTask: api.MaintenanceTaskEverything,
NetworkProfile: api.NetworkProfile{
OutboundType: api.OutboundTypeLoadbalancer,
PreconfiguredNSG: api.PreconfiguredNSGDisabled,
LoadBalancerProfile: &api.LoadBalancerProfile{
ManagedOutboundIPs: &api.ManagedOutboundIPs{
Count: 1,
},
},
},
MasterProfile: api.MasterProfile{
EncryptionAtHost: api.EncryptionAtHostDisabled,
},
OperatorFlags: api.OperatorFlags{"testFlag": "true"},
MaintenanceState: api.MaintenanceStateUnplanned,
},
},
})
},
wantAsync: true,
wantStatusCode: http.StatusOK,
wantResponse: &admin.OpenShiftCluster{
ID: testdatabase.GetResourcePath(mockSubID, "resourceName"),
Type: "Microsoft.RedHatOpenShift/openShiftClusters",
Tags: map[string]string{"tag": "will-be-kept"},
Properties: admin.OpenShiftClusterProperties{
ProvisioningState: admin.ProvisioningStateAdminUpdating,
LastProvisioningState: admin.ProvisioningStateSucceeded,
ClusterProfile: admin.ClusterProfile{
FipsValidatedModules: admin.FipsValidatedModulesDisabled,
},
MaintenanceTask: admin.MaintenanceTaskEverything,
NetworkProfile: admin.NetworkProfile{
OutboundType: admin.OutboundTypeLoadbalancer,
LoadBalancerProfile: &admin.LoadBalancerProfile{
ManagedOutboundIPs: &admin.ManagedOutboundIPs{
Count: 1,
},
},
},
MasterProfile: admin.MasterProfile{
EncryptionAtHost: admin.EncryptionAtHostDisabled,
},
OperatorFlags: admin.OperatorFlags{"testFlag": "true"},
MaintenanceState: admin.MaintenanceStateUnplanned,
},
},
},
} {
t.Run(tt.name, func(t *testing.T) {
ti := newTestInfra(t).

Просмотреть файл

@ -209,7 +209,7 @@ func (mon *Monitor) Monitor(ctx context.Context) (errs []error) {
mon.emitSummary,
mon.emitHiveRegistrationStatus,
mon.emitOperatorFlagsAndSupportBanner,
mon.emitPucmState,
mon.emitMaintenanceState,
mon.emitCertificateExpirationStatuses,
mon.emitEtcdCertificateExpiry,
mon.emitPrometheusAlerts, // at the end for now because it's the slowest/least reliable

Просмотреть файл

@ -10,52 +10,40 @@ import (
)
/**************************************************************
Possible PUCM states:
Possible maintenance states:
(1) PUCM pending
- We will do PUCM, so emit a maintenance pending signal
- Conditions:
* Field pucmPending is true
* Don't meet below conditions for in progress maintenance
(1) Maintenance pending
- We will do maintenance, so emit a maintenance pending signal
(2) Planned PUCM in progress
(2) Planned maintenance in progress
- Emit a planned maintenance in progress signal.
- If first PUCM attempt fails, leave cluster in this state
because we will need to retry PUCM in at a later time.
- Conditions:
* Field pucmPending is true
* One of: (a) provisoning state AdminUpdate or (2) AdminUpdate err is not nil
- If first attempt fails, leave cluster in this state because
we will need to either retry or have an SRE update the state to none.
(3) Unplanned PUCM in progress
(3) Unplanned maintenance in progress
- Emit an unplanned maintenance in progress signal.
- If first PUCM attempt fails, leave cluster in this state
because we will need to retry PUCM in at a later time.
- Conditions:
* Field pucmPending is false
* One of: (a) provisoning state AdminUpdate or (2) AdminUpdate err is not nil
- If first attempt fails, leave cluster in this state because
we will need to either retry or have an SRE update the state to none.
(4) No ongoinig or scheduled PUCM
- Don't emit a signal
- Conditions:
* Field pucmPending is false
* Provisioning state is not AdminUpdate and AdminUpdate err is not nil
(4) No ongoinig or scheduled maintenance
- Emit the none signal.
**************************************************************/
type pucmState string
type maintenanceState string
func (p pucmState) String() string {
return string(p)
func (m maintenanceState) String() string {
return string(m)
}
const (
pucmNone pucmState = "none"
pucmPending pucmState = "pending"
pucmPlanned pucmState = "planned"
pucmUnplanned pucmState = "unplanned"
none maintenanceState = "none"
pending maintenanceState = "pending"
planned maintenanceState = "planned"
unplanned maintenanceState = "unplanned"
)
func (mon *Monitor) emitPucmState(ctx context.Context) error {
state := getPucmState(mon.oc.Properties)
func (mon *Monitor) emitMaintenanceState(ctx context.Context) error {
state := getMaintenanceState(mon.oc.Properties)
mon.emitGauge("cluster.maintenance.pucm", 1, map[string]string{
"state": state.String(),
})
@ -63,22 +51,18 @@ func (mon *Monitor) emitPucmState(ctx context.Context) error {
return nil
}
func getPucmState(clusterProperties api.OpenShiftClusterProperties) pucmState {
if pucmOngoing(clusterProperties) {
if clusterProperties.PucmPending {
return pucmPlanned
}
return pucmUnplanned
func getMaintenanceState(clusterProperties api.OpenShiftClusterProperties) maintenanceState {
switch clusterProperties.MaintenanceState {
case api.MaintenanceStatePending:
return pending
case api.MaintenanceStatePlanned:
return planned
case api.MaintenanceStateUnplanned:
return unplanned
case api.MaintenanceStateNone:
fallthrough
// For new clusters, no maintenance state has been set yet
default:
return none
}
if clusterProperties.PucmPending {
return pucmPending
}
return pucmNone
}
func pucmOngoing(clusterProperties api.OpenShiftClusterProperties) bool {
return clusterProperties.ProvisioningState == api.ProvisioningStateAdminUpdating ||
clusterProperties.LastAdminUpdateError != ""
}

Просмотреть файл

@ -13,48 +13,42 @@ import (
mock_metrics "github.com/Azure/ARO-RP/pkg/util/mocks/metrics"
)
func TestEmitPucmState(t *testing.T) {
func TestEmitMaintenanceState(t *testing.T) {
for _, tt := range []struct {
name string
provisioningState api.ProvisioningState
pucmPending bool
maintenanceState api.MaintenanceState
adminUpdateErr string
expectedPucmState pucmState
expectedState maintenanceState
}{
{
name: "state none",
name: "state none - empty maintenance state",
provisioningState: api.ProvisioningStateSucceeded,
expectedPucmState: pucmNone,
expectedState: none,
},
{
name: "state none - no maintenance state set",
provisioningState: api.ProvisioningStateSucceeded,
maintenanceState: api.MaintenanceStateNone,
expectedState: none,
},
{
name: "state pending",
provisioningState: api.ProvisioningStateSucceeded,
pucmPending: true,
expectedPucmState: pucmPending,
maintenanceState: api.MaintenanceStatePending,
expectedState: pending,
},
{
name: "state unplanned - admin updating in flight and no admin update error",
name: "state unplanned",
provisioningState: api.ProvisioningStateAdminUpdating,
expectedPucmState: pucmUnplanned,
maintenanceState: api.MaintenanceStateUnplanned,
expectedState: unplanned,
},
{
name: "state planned - admin updating in flight and no admin update error",
name: "state planned",
provisioningState: api.ProvisioningStateAdminUpdating,
pucmPending: true,
expectedPucmState: pucmPlanned,
},
{
name: "state unplanned - not admin updating but admin update error",
provisioningState: api.ProvisioningStateFailed,
adminUpdateErr: "PUCM failed",
expectedPucmState: pucmUnplanned,
},
{
name: "state planned - not admin updating but admin update error",
provisioningState: api.ProvisioningStateFailed,
pucmPending: true,
adminUpdateErr: "PUCM failed",
expectedPucmState: pucmPlanned,
maintenanceState: api.MaintenanceStatePlanned,
expectedState: planned,
},
} {
t.Run(tt.name, func(t *testing.T) {
@ -67,7 +61,7 @@ func TestEmitPucmState(t *testing.T) {
oc := &api.OpenShiftCluster{
Properties: api.OpenShiftClusterProperties{
ProvisioningState: tt.provisioningState,
PucmPending: tt.pucmPending,
MaintenanceState: tt.maintenanceState,
LastAdminUpdateError: tt.adminUpdateErr,
},
}
@ -77,10 +71,10 @@ func TestEmitPucmState(t *testing.T) {
}
m.EXPECT().EmitGauge("cluster.maintenance.pucm", int64(1), map[string]string{
"state": tt.expectedPucmState.String(),
"state": tt.expectedState.String(),
})
err := mon.emitPucmState(ctx)
err := mon.emitMaintenanceState(ctx)
if err != nil {
t.Fatal(err)
}