Pause zero scale Framework instead of completing it (#59)

This commit is contained in:
Yuqi Wang 2020-08-11 19:46:34 +08:00 коммит произвёл GitHub
Родитель d67fc76595
Коммит 29e115373a
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
2 изменённых файлов: 69 добавлений и 60 удалений

Просмотреть файл

@ -45,10 +45,11 @@ type FrameworkList struct {
// 2. Partitioned to different heterogeneous TaskRoles which share the same lifecycle
// 3. Ordered in the same homogeneous TaskRole by TaskIndex
// 4. With consistent identity {FrameworkName}-{TaskRoleName}-{TaskIndex} as PodName
// 5. With fine grained RetryPolicy for each Task and the whole Framework
// 6. With fine grained FrameworkAttemptCompletionPolicy for each TaskRole
// 7. With PodGracefulDeletionTimeoutSec for each Task to tune Consistency vs Availability
// 8. With fine grained Status for each TaskAttempt/Task, each TaskRole and the whole
// 5. With fine grained ExecutionType to Start/Stop the whole Framework
// 6. With fine grained RetryPolicy for each Task and the whole Framework
// 7. With fine grained FrameworkAttemptCompletionPolicy for each TaskRole
// 8. With PodGracefulDeletionTimeoutSec for each Task to tune Consistency vs Availability
// 9. With fine grained Status for each TaskAttempt/Task, each TaskRole and the whole
// FrameworkAttempt/Framework
//
// Notes:
@ -204,14 +205,14 @@ type RetryPolicySpec struct {
// from the Task which triggers the completion.
// 3. If MinSucceededTaskCount >= 1 and MinSucceededTaskCount <= succeeded Task
// count of current TaskRole, immediately complete the FrameworkAttempt, regardless
// of any uncompleted Task, and the CompletionStatus is succeeded which is
// inherited from the Task which triggers the completion.
// of any uncompleted Task, and the CompletionStatus is succeeded which is inherited
// from the Task which triggers the completion.
// 4. If multiple above conditions are satisfied at the same time, the behavior can
// be any one of these satisfied conditions.
// 5. If none of above conditions are satisfied until all Tasks of the Framework are
// completed (including a special case that the Framework does even not have any
// Task), immediately complete the FrameworkAttempt and the CompletionStatus is
// succeeded which is not inherited from any Task.
// completed and the Framework has at least one Task, immediately complete the
// FrameworkAttempt and the CompletionStatus is succeeded which is not inherited
// from any Task.
//
// Notes:
// 1. When the FrameworkAttempt is completed, the FrameworkState is transitioned to

Просмотреть файл

@ -1535,53 +1535,57 @@ func (c *FrameworkController) syncFrameworkAttemptCompletionPolicy(
return true
}
// The Framework must not Completing or Completed, so TaskRoles/Tasks in
// f.Spec must fully contain not DeletionPending (ScaleDown) TaskRoles/Tasks
// in f.Status, thus completedTaskCount must <= totalTaskCount.
totalTaskCount := f.GetTotalTaskCountSpec()
completedTaskCount := f.GetTaskCountStatus(completedTaskSelector)
if completedTaskCount >= totalTaskCount {
var lastCompletedTaskStatus *ci.TaskStatus
var lastCompletedTaskRoleName string
for _, taskRoleSpec := range f.Spec.TaskRoles {
taskRoleName := taskRoleSpec.Name
taskRoleStatus := f.GetTaskRoleStatus(taskRoleName)
if taskRoleStatus == nil {
// Unreachable
continue
// At least one completed Task is needed to trigger its
// FrameworkAttemptCompletionPolicy.
if totalTaskCount >= 1 {
completedTaskCount := f.GetTaskCountStatus(completedTaskSelector)
// The Framework must not Completing or Completed, so TaskRoles/Tasks in
// f.Spec must fully contain not DeletionPending (ScaleDown) TaskRoles/Tasks
// in f.Status, thus completedTaskCount must <= totalTaskCount.
if completedTaskCount >= totalTaskCount {
var lastCompletedTaskStatus *ci.TaskStatus
var lastCompletedTaskRoleName string
for _, taskRoleSpec := range f.Spec.TaskRoles {
taskRoleName := taskRoleSpec.Name
taskRoleStatus := f.GetTaskRoleStatus(taskRoleName)
if taskRoleStatus == nil {
// Unreachable
continue
}
roleTotalTaskCount := taskRoleSpec.TaskNumber
if roleTotalTaskCount == 0 {
continue
}
roleLastCompletedTask := taskRoleStatus.CompletionTimeOrderedTaskStatus(
completedTaskSelector, roleTotalTaskCount-1)
if lastCompletedTaskStatus == nil ||
roleLastCompletedTask.CompletionTime.Time.After(
lastCompletedTaskStatus.CompletionTime.Time) {
lastCompletedTaskStatus = roleLastCompletedTask
lastCompletedTaskRoleName = taskRoleName
}
}
roleTotalTaskCount := taskRoleSpec.TaskNumber
if roleTotalTaskCount == 0 {
continue
}
firstTriggerCompletionStatus = ci.NewCompletedTaskTriggeredCompletionStatus(
lastCompletedTaskStatus, lastCompletedTaskRoleName,
completedTaskCount, totalTaskCount)
roleLastCompletedTask := taskRoleStatus.CompletionTimeOrderedTaskStatus(
completedTaskSelector, roleTotalTaskCount-1)
if lastCompletedTaskStatus == nil ||
roleLastCompletedTask.CompletionTime.Time.After(
lastCompletedTaskStatus.CompletionTime.Time) {
lastCompletedTaskStatus = roleLastCompletedTask
lastCompletedTaskRoleName = taskRoleName
if firstTriggerCompletionStatus.Trigger == nil {
klog.Infof("[%v]: syncFrameworkAttemptCompletionPolicy: %v", f.Key(),
firstTriggerCompletionStatus.Diagnostics)
} else {
klog.Infof("[%v][%v][%v]: syncFrameworkAttemptCompletionPolicy: %v", f.Key(),
firstTriggerCompletionStatus.Trigger.TaskRoleName,
firstTriggerCompletionStatus.Trigger.TaskIndex,
firstTriggerCompletionStatus.Trigger.Message)
}
c.completeFrameworkAttempt(f, false, firstTriggerCompletionStatus)
return true
}
firstTriggerCompletionStatus = ci.NewCompletedTaskTriggeredCompletionStatus(
lastCompletedTaskStatus, lastCompletedTaskRoleName,
completedTaskCount, totalTaskCount)
if firstTriggerCompletionStatus.Trigger == nil {
klog.Infof("[%v]: syncFrameworkAttemptCompletionPolicy: %v", f.Key(),
firstTriggerCompletionStatus.Diagnostics)
} else {
klog.Infof("[%v][%v][%v]: syncFrameworkAttemptCompletionPolicy: %v", f.Key(),
firstTriggerCompletionStatus.Trigger.TaskRoleName,
firstTriggerCompletionStatus.Trigger.TaskIndex,
firstTriggerCompletionStatus.Trigger.Message)
}
c.completeFrameworkAttempt(f, false, firstTriggerCompletionStatus)
return true
}
return false
@ -1998,18 +2002,22 @@ func (c *FrameworkController) syncTaskState(
return nil
}
// The Framework must not Completing or Completed, so TaskRoles/Tasks in
// f.Spec must fully contain not DeletionPending (ScaleDown) TaskRoles/Tasks
// in f.Status, thus completedTaskCount must <= totalTaskCount.
totalTaskCount := f.GetTotalTaskCountSpec()
completedTaskCount := f.GetTaskCountStatus(completedTaskSelector)
if completedTaskCount >= totalTaskCount {
triggerCompletionStatus = ci.NewCompletedTaskTriggeredCompletionStatus(
taskStatus, taskRoleName, completedTaskCount, totalTaskCount)
// At least one completed Task is needed to trigger its
// FrameworkAttemptCompletionPolicy.
if taskStatus.IsCompleted(true) && totalTaskCount >= 1 {
completedTaskCount := f.GetTaskCountStatus(completedTaskSelector)
// The Framework must not Completing or Completed, so TaskRoles/Tasks in
// f.Spec must fully contain not DeletionPending (ScaleDown) TaskRoles/Tasks
// in f.Status, thus completedTaskCount must <= totalTaskCount.
if completedTaskCount >= totalTaskCount {
triggerCompletionStatus = ci.NewCompletedTaskTriggeredCompletionStatus(
taskStatus, taskRoleName, completedTaskCount, totalTaskCount)
klog.Info(logPfx + triggerCompletionStatus.Trigger.Message)
c.completeFrameworkAttempt(f, false, triggerCompletionStatus)
return nil
klog.Info(logPfx + triggerCompletionStatus.Trigger.Message)
c.completeFrameworkAttempt(f, false, triggerCompletionStatus)
return nil
}
}
return nil