Skip build for legacy topo aware scheduler.

Skip build for legacy topology aware scheduler through build tag.
This commit is contained in:
Yifan Xiong 2021-11-09 15:36:42 +08:00
Родитель cdaeea3012
Коммит 5c1e5c8c71
4 изменённых файлов: 276 добавлений и 247 удалений

Просмотреть файл

@ -20,6 +20,8 @@
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE
// +build ignore
package algorithm
import (
@ -440,27 +442,6 @@ func removePickedLeafCells(leafCells CellList, indices []int32) CellList {
return leafCells[:len(leafCells)-len(indices)]
}
// findLCA finds the lowest common ancestor of two cells (nil if they have no LCA).
func findLCA(lower Cell, higher Cell) Cell {
for lower.GetLevel() < higher.GetLevel() {
if lower.GetParent() == nil {
return nil
}
lower = lower.GetParent()
}
if CellEqual(lower, higher) {
return lower
}
for !CellEqual(lower.GetParent(), higher.GetParent()) {
if lower.GetParent() == nil || higher.GetParent() == nil {
return nil
}
lower = lower.GetParent()
higher = higher.GetParent()
}
return lower.GetParent()
}
// getLeafCellsFromNode collects free leaf cells and preemptible leaf cells according to the priority.
func getLeafCellsFromNode(c Cell, p CellPriority, freeLeafCells CellList, preemptibleLeafCells CellList) (CellList, CellList) {
if c.GetLevel() > 1 {

Просмотреть файл

@ -421,6 +421,27 @@ func checkOptimalAffinityForCells(
return bestAffinity, false
}
// findLCA finds the lowest common ancestor of two cells (nil if they have no LCA).
func findLCA(lower Cell, higher Cell) Cell {
for lower.GetLevel() < higher.GetLevel() {
if lower.GetParent() == nil {
return nil
}
lower = lower.GetParent()
}
if CellEqual(lower, higher) {
return lower
}
for !CellEqual(lower.GetParent(), higher.GetParent()) {
if lower.GetParent() == nil || higher.GetParent() == nil {
return nil
}
lower = lower.GetParent()
higher = higher.GetParent()
}
return lower.GetParent()
}
// createSkuClusterView returns list of sku cells within
// the given cell, level and priority in virtual cluster view.
func (s *topologyGuaranteeScheduler) createSkuClusterView(

Просмотреть файл

@ -25,11 +25,6 @@ package algorithm
import (
"fmt"
"strings"
"github.com/microsoft/hivedscheduler/pkg/api"
"github.com/microsoft/hivedscheduler/pkg/common"
core "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/types"
)
type (
@ -40,17 +35,6 @@ type (
AffinityGroupState string
)
type schedulingRequest struct {
vc api.VirtualClusterName
pinnedCellId api.PinnedCellId
chain CellChain
affinityGroupName string
affinityGroupPodNums map[int32]int32 // leaf cell number -> pod number
priority CellPriority
suggestedNodes common.Set
ignoreSuggestedNodes bool
}
// CellList is a list of cells at a certain level of a chain.
type CellList []Cell
@ -129,216 +113,6 @@ func (ccl ChainCellList) shallowCopy() ChainCellList {
return copied
}
// AlgoAffinityGroup is the algorithm-internal representation of an affinity group.
type AlgoAffinityGroup struct {
name string
vc api.VirtualClusterName
lazyPreemptionEnable bool
// Whether we should ignore K8s suggested nodes. If false, we will avoid binding cells to non-suggested nodes.
// Note that we always avoid using bad nodes; avoiding non-suggested nodes is optional and best-effort.
ignoreK8sSuggestedNodes bool
priority int32
totalPodNums map[int32]int32 // LeafCellNum -> PodNum
allocatedPods map[int32][]*core.Pod // LeafCellNum -> a list of allocated pods
preemptingPods map[types.UID]*core.Pod
physicalLeafCellPlacement groupPhysicalPlacement
virtualLeafCellPlacement groupVirtualPlacement
state AffinityGroupState
lazyPreemptionStatus *api.LazyPreemptionStatus
}
func newAlgoAffinityGroup(
g *api.AffinityGroupSpec,
vc api.VirtualClusterName,
lazyPreemptionEnable bool,
priority int32,
state AffinityGroupState) *AlgoAffinityGroup {
podNums := make(map[int32]int32)
for _, m := range g.Members {
podNums[m.LeafCellNumber] += m.PodNumber
}
group := &AlgoAffinityGroup{
name: g.Name,
vc: vc,
lazyPreemptionEnable: lazyPreemptionEnable,
priority: priority,
totalPodNums: podNums,
allocatedPods: map[int32][]*core.Pod{},
physicalLeafCellPlacement: groupPhysicalPlacement{},
virtualLeafCellPlacement: groupVirtualPlacement{},
state: state,
}
if state == AffinityGroupState("Preempting") {
group.preemptingPods = map[types.UID]*core.Pod{}
}
for leafCellNum, podNum := range podNums {
group.physicalLeafCellPlacement[leafCellNum] = make([]CellList, podNum)
group.virtualLeafCellPlacement[leafCellNum] = make([]CellList, podNum)
group.allocatedPods[leafCellNum] = make([]*core.Pod, podNum)
for i := int32(0); i < podNum; i++ {
group.physicalLeafCellPlacement[leafCellNum][i] = make(CellList, leafCellNum)
group.virtualLeafCellPlacement[leafCellNum][i] = make(CellList, leafCellNum)
}
}
return group
}
func (aag *AlgoAffinityGroup) ToAffinityGroup() api.AffinityGroup {
ag := api.AffinityGroup{
ObjectMeta: api.ObjectMeta{Name: aag.name},
Status: api.AffinityGroupStatus{
VC: aag.vc,
Priority: aag.priority,
State: api.AffinityGroupState(aag.state),
LazyPreemptionStatus: aag.lazyPreemptionStatus,
},
}
if aag.physicalLeafCellPlacement != nil {
ag.Status.PhysicalPlacement = aag.physicalLeafCellPlacement.nodeToLeafCellIndices()
}
if aag.virtualLeafCellPlacement != nil {
ag.Status.VirtualPlacement = aag.virtualLeafCellPlacement.preassignedCellToLeafCells()
}
for _, pods := range aag.allocatedPods {
for _, p := range pods {
if p != nil {
ag.Status.AllocatedPods = append(ag.Status.AllocatedPods, p.UID)
}
}
}
for p := range aag.preemptingPods {
ag.Status.PreemptingPods = append(ag.Status.PreemptingPods, p)
}
return ag
}
type groupPhysicalPlacement map[int32][]CellList // LeafCellNum -> a list of pods -> a list of physical leaf cells of each pod
type groupVirtualPlacement map[int32][]CellList // LeafCellNum -> a list of pods -> a list of virtual leaf cells of each pod
func (p groupPhysicalPlacement) String() string {
return common.ToJson(p.nodeToLeafCellIndices())
}
func (p groupPhysicalPlacement) nodeToLeafCellIndices() map[string][]int32 {
nodeToLeafCellIndices := map[string][]int32{}
for _, podPlacements := range p {
for _, podPlacement := range podPlacements {
for _, leafCell := range podPlacement {
pLeafCell := leafCell.(*PhysicalCell)
nodes, leafCellIndices := pLeafCell.GetPhysicalPlacement()
if _, ok := nodeToLeafCellIndices[nodes[0]]; !ok {
nodeToLeafCellIndices[nodes[0]] = []int32{}
}
nodeToLeafCellIndices[nodes[0]] = append(nodeToLeafCellIndices[nodes[0]], leafCellIndices[0])
}
}
}
return nodeToLeafCellIndices
}
func (p groupVirtualPlacement) String() string {
return common.ToJson(p.preassignedCellToLeafCells())
}
func (p groupVirtualPlacement) preassignedCellToLeafCells() map[api.CellAddress][]api.CellAddress {
preassignedCellToLeafCells := map[api.CellAddress][]api.CellAddress{}
for _, podPlacements := range p {
for _, podPlacement := range podPlacements {
for _, leafCell := range podPlacement {
vLeafCell := leafCell.(*VirtualCell)
address := vLeafCell.GetAddress()
preassignedAddress := vLeafCell.GetPreassignedCell().GetAddress()
if _, ok := preassignedCellToLeafCells[preassignedAddress]; !ok {
preassignedCellToLeafCells[preassignedAddress] = []api.CellAddress{}
}
preassignedCellToLeafCells[preassignedAddress] = append(
preassignedCellToLeafCells[preassignedAddress], address)
}
}
}
return preassignedCellToLeafCells
}
func (p groupVirtualPlacement) toPhysicalPlacement(
bindings map[api.CellAddress]*PhysicalCell,
leafCellNums []int32) groupPhysicalPlacement {
physicalPlacement := groupPhysicalPlacement{}
for _, podLeafCellNum := range leafCellNums {
podPlacements := p[podLeafCellNum]
physicalPlacement[podLeafCellNum] = make([]CellList, len(podPlacements))
for i, podPlacement := range podPlacements {
physicalPlacement[podLeafCellNum][i] = make(CellList, len(podPlacement))
for j, leafCell := range podPlacement {
pLeafCell := bindings[leafCell.GetAddress()]
physicalPlacement[podLeafCellNum][i][j] = pLeafCell
}
}
}
return physicalPlacement
}
// A binding path is a tree consisting of all cells that should be bound for binding a set of
// lowest-level cells in a physical placement. It is generated by collecting all the unbound
// ancestors for these cells and group them in a tree.
func (p groupVirtualPlacement) toBindingPaths(
leafCellNums []int32,
bindings map[api.CellAddress]*PhysicalCell) (
preassignedCells []*cellBindingPathVertex,
nonPreassignedCells [][]*cellBindingPathVertex) {
allBindingPathVertices := map[api.CellAddress]*cellBindingPathVertex{}
for _, podLeafCellNum := range leafCellNums {
podPlacements := p[podLeafCellNum]
for _, podPlacement := range podPlacements {
for _, leafCell := range podPlacement {
if pLeafCell := leafCell.(*VirtualCell).GetPhysicalCell(); pLeafCell != nil {
bindings[leafCell.GetAddress()] = pLeafCell
continue
}
var bindingPath []*VirtualCell
for c := leafCell; c != nil; c = c.GetParent() {
vc := c.(*VirtualCell)
if vc.GetPhysicalCell() != nil || allBindingPathVertices[vc.GetAddress()] != nil {
break
}
bindingPath = append(bindingPath, vc)
}
pathRoot := bindingPath[len(bindingPath)-1]
n := &cellBindingPathVertex{cell: pathRoot}
allBindingPathVertices[pathRoot.GetAddress()] = n
if parent := pathRoot.GetParent(); parent == nil {
preassignedCells = append(preassignedCells, n)
} else if parent.(*VirtualCell).GetPhysicalCell() != nil {
buddyExist := false
for i := range nonPreassignedCells {
if CellEqual(parent, nonPreassignedCells[i][0].cell.GetParent()) {
buddyExist = true
nonPreassignedCells[i] = append(nonPreassignedCells[i], n)
break
}
}
if !buddyExist {
nonPreassignedCells = append(nonPreassignedCells, []*cellBindingPathVertex{n})
}
} else {
parentNode := allBindingPathVertices[pathRoot.GetParent().GetAddress()]
parentNode.childrenToBind = append(parentNode.childrenToBind, n)
}
for i := len(bindingPath) - 2; i >= 0; i-- {
c := bindingPath[i]
n := &cellBindingPathVertex{cell: c}
parentNode := allBindingPathVertices[c.GetParent().GetAddress()]
parentNode.childrenToBind = append(parentNode.childrenToBind, n)
allBindingPathVertices[c.GetAddress()] = n
}
}
}
}
return preassignedCells, nonPreassignedCells
}
// cellBindingPathVertex is a single vertex in the tree of a cell binding path,
// containing the vertices of its children to bind.
type cellBindingPathVertex struct {

253
pkg/algorithm/types_v1.go Normal file
Просмотреть файл

@ -0,0 +1,253 @@
// MIT License
//
// Copyright (c) Microsoft Corporation. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE
// +build ignore
package algorithm
import (
"github.com/microsoft/hivedscheduler/pkg/api"
"github.com/microsoft/hivedscheduler/pkg/common"
core "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/types"
)
type schedulingRequest struct {
vc api.VirtualClusterName
pinnedCellId api.PinnedCellId
chain CellChain
affinityGroupName string
affinityGroupPodNums map[int32]int32 // leaf cell number -> pod number
priority CellPriority
suggestedNodes common.Set
ignoreSuggestedNodes bool
}
// AlgoAffinityGroup is the algorithm-internal representation of an affinity group.
type AlgoAffinityGroup struct {
name string
vc api.VirtualClusterName
lazyPreemptionEnable bool
// Whether we should ignore K8s suggested nodes. If false, we will avoid binding cells to non-suggested nodes.
// Note that we always avoid using bad nodes; avoiding non-suggested nodes is optional and best-effort.
ignoreK8sSuggestedNodes bool
priority int32
totalPodNums map[int32]int32 // LeafCellNum -> PodNum
allocatedPods map[int32][]*core.Pod // LeafCellNum -> a list of allocated pods
preemptingPods map[types.UID]*core.Pod
physicalLeafCellPlacement groupPhysicalPlacement
virtualLeafCellPlacement groupVirtualPlacement
state AffinityGroupState
lazyPreemptionStatus *api.LazyPreemptionStatus
}
func newAlgoAffinityGroup(
g *api.AffinityGroupSpec,
vc api.VirtualClusterName,
lazyPreemptionEnable bool,
priority int32,
state AffinityGroupState) *AlgoAffinityGroup {
podNums := make(map[int32]int32)
for _, m := range g.Members {
podNums[m.LeafCellNumber] += m.PodNumber
}
group := &AlgoAffinityGroup{
name: g.Name,
vc: vc,
lazyPreemptionEnable: lazyPreemptionEnable,
priority: priority,
totalPodNums: podNums,
allocatedPods: map[int32][]*core.Pod{},
physicalLeafCellPlacement: groupPhysicalPlacement{},
virtualLeafCellPlacement: groupVirtualPlacement{},
state: state,
}
if state == AffinityGroupState("Preempting") {
group.preemptingPods = map[types.UID]*core.Pod{}
}
for leafCellNum, podNum := range podNums {
group.physicalLeafCellPlacement[leafCellNum] = make([]CellList, podNum)
group.virtualLeafCellPlacement[leafCellNum] = make([]CellList, podNum)
group.allocatedPods[leafCellNum] = make([]*core.Pod, podNum)
for i := int32(0); i < podNum; i++ {
group.physicalLeafCellPlacement[leafCellNum][i] = make(CellList, leafCellNum)
group.virtualLeafCellPlacement[leafCellNum][i] = make(CellList, leafCellNum)
}
}
return group
}
func (aag *AlgoAffinityGroup) ToAffinityGroup() api.AffinityGroup {
ag := api.AffinityGroup{
ObjectMeta: api.ObjectMeta{Name: aag.name},
Status: api.AffinityGroupStatus{
VC: aag.vc,
Priority: aag.priority,
State: api.AffinityGroupState(aag.state),
LazyPreemptionStatus: aag.lazyPreemptionStatus,
},
}
if aag.physicalLeafCellPlacement != nil {
ag.Status.PhysicalPlacement = aag.physicalLeafCellPlacement.nodeToLeafCellIndices()
}
if aag.virtualLeafCellPlacement != nil {
ag.Status.VirtualPlacement = aag.virtualLeafCellPlacement.preassignedCellToLeafCells()
}
for _, pods := range aag.allocatedPods {
for _, p := range pods {
if p != nil {
ag.Status.AllocatedPods = append(ag.Status.AllocatedPods, p.UID)
}
}
}
for p := range aag.preemptingPods {
ag.Status.PreemptingPods = append(ag.Status.PreemptingPods, p)
}
return ag
}
type groupPhysicalPlacement map[int32][]CellList // LeafCellNum -> a list of pods -> a list of physical leaf cells of each pod
type groupVirtualPlacement map[int32][]CellList // LeafCellNum -> a list of pods -> a list of virtual leaf cells of each pod
func (p groupPhysicalPlacement) String() string {
return common.ToJson(p.nodeToLeafCellIndices())
}
func (p groupPhysicalPlacement) nodeToLeafCellIndices() map[string][]int32 {
nodeToLeafCellIndices := map[string][]int32{}
for _, podPlacements := range p {
for _, podPlacement := range podPlacements {
for _, leafCell := range podPlacement {
pLeafCell := leafCell.(*PhysicalCell)
nodes, leafCellIndices := pLeafCell.GetPhysicalPlacement()
if _, ok := nodeToLeafCellIndices[nodes[0]]; !ok {
nodeToLeafCellIndices[nodes[0]] = []int32{}
}
nodeToLeafCellIndices[nodes[0]] = append(nodeToLeafCellIndices[nodes[0]], leafCellIndices[0])
}
}
}
return nodeToLeafCellIndices
}
func (p groupVirtualPlacement) String() string {
return common.ToJson(p.preassignedCellToLeafCells())
}
func (p groupVirtualPlacement) preassignedCellToLeafCells() map[api.CellAddress][]api.CellAddress {
preassignedCellToLeafCells := map[api.CellAddress][]api.CellAddress{}
for _, podPlacements := range p {
for _, podPlacement := range podPlacements {
for _, leafCell := range podPlacement {
vLeafCell := leafCell.(*VirtualCell)
address := vLeafCell.GetAddress()
preassignedAddress := vLeafCell.GetPreassignedCell().GetAddress()
if _, ok := preassignedCellToLeafCells[preassignedAddress]; !ok {
preassignedCellToLeafCells[preassignedAddress] = []api.CellAddress{}
}
preassignedCellToLeafCells[preassignedAddress] = append(
preassignedCellToLeafCells[preassignedAddress], address)
}
}
}
return preassignedCellToLeafCells
}
func (p groupVirtualPlacement) toPhysicalPlacement(
bindings map[api.CellAddress]*PhysicalCell,
leafCellNums []int32) groupPhysicalPlacement {
physicalPlacement := groupPhysicalPlacement{}
for _, podLeafCellNum := range leafCellNums {
podPlacements := p[podLeafCellNum]
physicalPlacement[podLeafCellNum] = make([]CellList, len(podPlacements))
for i, podPlacement := range podPlacements {
physicalPlacement[podLeafCellNum][i] = make(CellList, len(podPlacement))
for j, leafCell := range podPlacement {
pLeafCell := bindings[leafCell.GetAddress()]
physicalPlacement[podLeafCellNum][i][j] = pLeafCell
}
}
}
return physicalPlacement
}
// A binding path is a tree consisting of all cells that should be bound for binding a set of
// lowest-level cells in a physical placement. It is generated by collecting all the unbound
// ancestors for these cells and group them in a tree.
func (p groupVirtualPlacement) toBindingPaths(
leafCellNums []int32,
bindings map[api.CellAddress]*PhysicalCell) (
preassignedCells []*cellBindingPathVertex,
nonPreassignedCells [][]*cellBindingPathVertex) {
allBindingPathVertices := map[api.CellAddress]*cellBindingPathVertex{}
for _, podLeafCellNum := range leafCellNums {
podPlacements := p[podLeafCellNum]
for _, podPlacement := range podPlacements {
for _, leafCell := range podPlacement {
if pLeafCell := leafCell.(*VirtualCell).GetPhysicalCell(); pLeafCell != nil {
bindings[leafCell.GetAddress()] = pLeafCell
continue
}
var bindingPath []*VirtualCell
for c := leafCell; c != nil; c = c.GetParent() {
vc := c.(*VirtualCell)
if vc.GetPhysicalCell() != nil || allBindingPathVertices[vc.GetAddress()] != nil {
break
}
bindingPath = append(bindingPath, vc)
}
pathRoot := bindingPath[len(bindingPath)-1]
n := &cellBindingPathVertex{cell: pathRoot}
allBindingPathVertices[pathRoot.GetAddress()] = n
if parent := pathRoot.GetParent(); parent == nil {
preassignedCells = append(preassignedCells, n)
} else if parent.(*VirtualCell).GetPhysicalCell() != nil {
buddyExist := false
for i := range nonPreassignedCells {
if CellEqual(parent, nonPreassignedCells[i][0].cell.GetParent()) {
buddyExist = true
nonPreassignedCells[i] = append(nonPreassignedCells[i], n)
break
}
}
if !buddyExist {
nonPreassignedCells = append(nonPreassignedCells, []*cellBindingPathVertex{n})
}
} else {
parentNode := allBindingPathVertices[pathRoot.GetParent().GetAddress()]
parentNode.childrenToBind = append(parentNode.childrenToBind, n)
}
for i := len(bindingPath) - 2; i >= 0; i-- {
c := bindingPath[i]
n := &cellBindingPathVertex{cell: c}
parentNode := allBindingPathVertices[c.GetParent().GetAddress()]
parentNode.childrenToBind = append(parentNode.childrenToBind, n)
allBindingPathVertices[c.GetAddress()] = n
}
}
}
}
return preassignedCells, nonPreassignedCells
}