From 630cf383b6550d9dc41012870529e5af641d9e54 Mon Sep 17 00:00:00 2001 From: Yuqi Wang Date: Tue, 1 Sep 2020 17:22:58 +0800 Subject: [PATCH] Tune FC, ControllerManager and ApiServer to serve large concurrent active frameworks (>10k) (#4864) --- contrib/aks-engine/templates/k8s.json.j2 | 3 +++ contrib/kubespray/example/openpai.yml | 3 +++ contrib/kubespray/quick-start/openpai.yml.template | 3 +++ .../k8sPaiLibrary/template/controller-manager.yaml.template | 3 +++ .../deploy/frameworkcontroller-config.yaml.template | 4 +++- 5 files changed, 15 insertions(+), 1 deletion(-) diff --git a/contrib/aks-engine/templates/k8s.json.j2 b/contrib/aks-engine/templates/k8s.json.j2 index 503c0aa03..f640909a5 100644 --- a/contrib/aks-engine/templates/k8s.json.j2 +++ b/contrib/aks-engine/templates/k8s.json.j2 @@ -52,6 +52,9 @@ "--max-mutating-requests-inflight": "500" }, "controllerManagerConfig": { + "--kube-api-qps": "200", + "--kube-api-burst": "300", + "--concurrent-gc-syncs": "500", "--node-monitor-grace-period": "5m", "--pod-eviction-timeout": "1m", "--route-reconciliation-period": "1m", diff --git a/contrib/kubespray/example/openpai.yml b/contrib/kubespray/example/openpai.yml index 8d1156ddb..637e5fd82 100644 --- a/contrib/kubespray/example/openpai.yml +++ b/contrib/kubespray/example/openpai.yml @@ -251,6 +251,9 @@ kube_apiserver_cpu_requests: 1000m kube_kubeadm_controller_extra_args: v: 2 + kube-api-qps: 200 + kube-api-burst: 300 + concurrent-gc-syncs: 500 large-cluster-size-threshold: 0 secondary-node-eviction-rate: 0.05 diff --git a/contrib/kubespray/quick-start/openpai.yml.template b/contrib/kubespray/quick-start/openpai.yml.template index 44ee0fbe9..88eba1231 100644 --- a/contrib/kubespray/quick-start/openpai.yml.template +++ b/contrib/kubespray/quick-start/openpai.yml.template @@ -244,6 +244,9 @@ kube_apiserver_cpu_requests: 1000m kube_kubeadm_controller_extra_args: v: 2 + kube-api-qps: 200 + kube-api-burst: 300 + concurrent-gc-syncs: 500 large-cluster-size-threshold: 0 secondary-node-eviction-rate: 0.05 diff --git a/deployment/k8sPaiLibrary/template/controller-manager.yaml.template b/deployment/k8sPaiLibrary/template/controller-manager.yaml.template index a93e5b8b7..b1d6cbdcb 100644 --- a/deployment/k8sPaiLibrary/template/controller-manager.yaml.template +++ b/deployment/k8sPaiLibrary/template/controller-manager.yaml.template @@ -32,6 +32,9 @@ spec: - --service-cluster-ip-range - {{ cluster_cfg['kubernetes']['service-cluster-ip-range'] }} - --leader-elect=true + - --kube-api-qps=200 + - --kube-api-burst=300 + - --concurrent-gc-syncs=500 - --large-cluster-size-threshold=0 - --secondary-node-eviction-rate=0.05 livenessProbe: diff --git a/src/frameworkcontroller/deploy/frameworkcontroller-config.yaml.template b/src/frameworkcontroller/deploy/frameworkcontroller-config.yaml.template index 3847ee845..f0ba5c232 100644 --- a/src/frameworkcontroller/deploy/frameworkcontroller-config.yaml.template +++ b/src/frameworkcontroller/deploy/frameworkcontroller-config.yaml.template @@ -25,7 +25,9 @@ data: #kubeApiServerAddress: http://10.10.10.10:8080 #kubeConfigFilePath: '' - workerNumber: 20 + kubeClientQps: 200 + kubeClientBurst: 300 + workerNumber: 500 largeFrameworkCompression: true