Install image-gc configurable will sane defaults (#1410)

* Adding GCHIgh and GCLow to json input and apimodel.json * adding variables to ConvertoToapi * adding GCHigh / Low to kublet config on Master / Agent * formatting updates * Adding example json files with GC attributes * Updating cluster definition docs
2017-09-08 15:33:23 -05:00 · 2017-09-08 15:33:23 -05:00 · 7cf8d6d8f2
--- a/docs/clusterdefinition.md
+++ b/docs/clusterdefinition.md
@ -38,6 +38,8 @@ Here are the valid values for the orchestrator types:
 |serviceCidr|no|IP range for Service IPs, Default is "10.0.0.0/16". This range is never routed outside of a node so does not need to lie within clusterSubnet or the VNet.|
 |enableRbac|no|Enable [Kubernetes RBAC](https://kubernetes.io/docs/admin/authorization/rbac/) (boolean - default == false) |
 |maxPods|no|The maximum number of pods per node. The minimum valid value, necessary for running kube-system pods, is 5. Default value is 30 when networkPolicy equals azure, 110 otherwise.|
+|gcHighThreshold|no|Sets the --image-gc-high-threshold value on the kublet configuration. Default is 85. [See kubelet Garbage Collection](https://kubernetes.io/docs/concepts/cluster-administration/kubelet-garbage-collection/) |
+|gcLowThreshold|no|Sets the --image-gc-low-threshold value on the kublet configuration. Default is 80. [See kubelet Garbage Collection](https://kubernetes.io/docs/concepts/cluster-administration/kubelet-garbage-collection/) |

 ### masterProfile
 `masterProfile` describes the settings for master configuration.
--- a/examples/kubernetes-config/README.md
+++ b/examples/kubernetes-config/README.md
@ -7,3 +7,4 @@ These cluster definition examples show how to create customized [Kubernetes](../
 1. [**kubernetes-clustersubnet.json**](kubernetes-clustersubnet.json) - Configuring a custom cluster IP subnet.
 2. [**kubernetes-maxpods.json**](kubernetes-maxpods.json) - Configuring a custom maximum limit on the number of pods per node.
 3. [**kubernetes-dockerbridgesubnet.json**](kubernetes-dockerbridgesubnet.json) - Configuring a custom IP subnet used for allocating IP addresses for the docker bridge network on nodes.
+4. [**kubernetes-gc.json**](kubernetes-gc.json) - Configuring custom image garbage collection values.
--- a/examples/kubernetes-config/kubernetes-gc.json
+++ b/examples/kubernetes-config/kubernetes-gc.json
@ -0,0 +1,39 @@
+{
+  "apiVersion": "vlabs",
+  "properties": {
+    "orchestratorProfile": {
+      "orchestratorType": "Kubernetes",
+      "kubernetesConfig": {
+        "gcHighThreshold":70,
+        "gcLowThreshold": 60
+      }
+    },
+    "masterProfile": {
+      "count": 1,
+      "dnsPrefix": "",
+      "vmSize": "Standard_D2_v2"
+    },
+    "agentPoolProfiles": [
+      {
+        "name": "agentpool1",
+        "count": 3,
+        "vmSize": "Standard_D2_v2",
+        "availabilityProfile": "AvailabilitySet"
+      }
+    ],
+    "linuxProfile": {
+      "adminUsername": "azureUser",
+      "ssh": {
+        "publicKeys": [
+          {
+            "keyData": ""
+          }
+        ]
+      }
+    },
+    "servicePrincipalProfile": {
+      "clientId": "",
+      "secret": ""
+    }
+  }
+}
--- a/parts/kubernetesagentcustomdata.yml
+++ b/parts/kubernetesagentcustomdata.yml
@ -114,6 +114,8 @@ write_files:
    KUBE_CTRL_MGR_NODE_MONITOR_GRACE_PERIOD={{WrapAsVariable "kubernetesCtrlMgrNodeMonitorGracePeriod"}}
    KUBE_CTRL_MGR_POD_EVICTION_TIMEOUT={{WrapAsVariable "kubernetesCtrlMgrPodEvictionTimeout"}}
    KUBE_CTRL_MGR_ROUTE_RECONCILIATION_PERIOD={{WrapAsVariable "kubernetesCtrlMgrRouteReconciliationPeriod"}}
+    KUBELET_IMAGE_GC_HIGH_THRESHOLD={{WrapAsVariable "gchighthreshold"}}
+    KUBELET_IMAGE_GC_LOW_THRESHOLD={{WrapAsVariable "gclowthreshold"}}
 {{if IsKubernetesVersionGe "1.6.0"}}
    KUBELET_FEATURE_GATES=--feature-gates=Accelerators=true
 {{end}}
--- a/parts/kuberneteskubelet.service
+++ b/parts/kuberneteskubelet.service
@ -50,6 +50,8 @@ ExecStart=/usr/bin/docker run \
        --network-plugin=${KUBELET_NETWORK_PLUGIN} \
        --max-pods=${KUBELET_MAX_PODS} \
        --node-status-update-frequency=${KUBELET_NODE_STATUS_UPDATE_FREQUENCY} \
+        --image-gc-high-threshold=${KUBELET_IMAGE_GC_HIGH_THRESHOLD} \
+        --image-gc-low-threshold=${KUBELET_IMAGE_GC_LOW_THRESHOLD} \
        --v=2 ${KUBELET_FEATURE_GATES} \
        ${KUBELET_REGISTER_NODE} ${KUBELET_REGISTER_WITH_TAINTS}

--- a/parts/kuberneteskubelet1.5.service
+++ b/parts/kuberneteskubelet1.5.service
@ -50,6 +50,8 @@ ExecStart=/usr/bin/docker run \
        --hairpin-mode=promiscuous-bridge \
        --network-plugin=${KUBELET_NETWORK_PLUGIN} \
        --node-status-update-frequency=${KUBELET_NODE_STATUS_UPDATE_FREQUENCY} \
+        --image-gc-high-threshold=${KUBELET_IMAGE_GC_HIGH_THRESHOLD} \
+        --image-gc-low-threshold=${KUBELET_IMAGE_GC_LOW_THRESHOLD} \
        --v=2 ${KUBELET_FEATURE_GATES}

 [Install]
--- a/parts/kubernetesmastercustomdata.yml
+++ b/parts/kubernetesmastercustomdata.yml
@ -196,6 +196,8 @@ write_files:
    KUBE_CTRL_MGR_NODE_MONITOR_GRACE_PERIOD={{WrapAsVariable "kubernetesCtrlMgrNodeMonitorGracePeriod"}}
    KUBE_CTRL_MGR_POD_EVICTION_TIMEOUT={{WrapAsVariable "kubernetesCtrlMgrPodEvictionTimeout"}}
    KUBE_CTRL_MGR_ROUTE_RECONCILIATION_PERIOD={{WrapAsVariable "kubernetesCtrlMgrRouteReconciliationPeriod"}}
+    KUBELET_IMAGE_GC_HIGH_THRESHOLD={{WrapAsVariable "gchighthreshold"}}
+    KUBELET_IMAGE_GC_LOW_THRESHOLD={{WrapAsVariable "gclowthreshold"}}
 {{if IsKubernetesVersionGe "1.6.0"}}
  {{if HasLinuxAgents}}
    KUBELET_REGISTER_NODE=--register-node=true
--- a/parts/kubernetesmastervars.t
+++ b/parts/kubernetesmastervars.t
@ -40,6 +40,8 @@
    "maxPods": "[parameters('maxPods')]",
    "vnetCidr": "[parameters('vnetCidr')]",
    "calicoConfigURL":"[parameters('calicoConfigURL')]",
+    "gcHighThreshold":"[parameters('gcHighThreshold')]",
+    "gcLowThreshold":"[parameters('gcLowThreshold')]",
 {{ if UseManagedIdentity }}
    "servicePrincipalClientId": "msi",
    "servicePrincipalClientSecret": "msi",
--- a/parts/kubernetesparams.t
+++ b/parts/kubernetesparams.t
@ -301,6 +301,20 @@
      },
      "type": "string"
    },
+    "gcHighThreshold": {
+      "defaultValue": 85,
+      "metadata": {
+        "description": "High Threshold for Image Garbage collection on each node"
+      },
+      "type": "int"
+    },
+    "gcLowThreshold": {
+      "defaultValue": 80,
+      "metadata": {
+        "description": "Low Threshold for Image Garbage collection on each node."
+      },
+      "type": "int"
+    },
 {{ if not UseManagedIdentity }}
    "servicePrincipalClientId": {
      "metadata": {
--- a/pkg/acsengine/const.go
+++ b/pkg/acsengine/const.go
@ -74,6 +74,10 @@ const (
 	// DefaultKubernetesServiceCIDR specifies the IP subnet that kubernetes will
 	// create Service IPs within.
 	DefaultKubernetesServiceCIDR = "10.0.0.0/16"
+	//DefaultKubernetesGCHighThreshold specifies the value for  for the image-gc-high-threshold kubelet flag
+	DefaultKubernetesGCHighThreshold = 85
+	//DefaultKubernetesGCLowThreshold specifies the value for the image-gc-low-threshold kubelet flag
+	DefaultKubernetesGCLowThreshold = 80
 )

 const (
@ -121,6 +125,8 @@ var KubeConfigs = map[string]map[string]string{
 		"backoffexponent": strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffExponent, 'f', -1, 64),
 		"ratelimitqps":    strconv.FormatFloat(DefaultKubernetesCloudProviderRateLimitQPS, 'f', -1, 64),
 		"ratelimitbucket": strconv.Itoa(DefaultKubernetesCloudProviderRateLimitBucket),
+		"gchighthreshold": strconv.Itoa(DefaultKubernetesGCHighThreshold),
+		"gclowthreshold":  strconv.Itoa(DefaultKubernetesGCLowThreshold),
 	},
 	api.KubernetesRelease1Dot6: {
 		"hyperkube":       "hyperkube-amd64:v1.6.9",
@ -144,6 +150,8 @@ var KubeConfigs = map[string]map[string]string{
 		"backoffexponent": strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffExponent, 'f', -1, 64),
 		"ratelimitqps":    strconv.FormatFloat(DefaultKubernetesCloudProviderRateLimitQPS, 'f', -1, 64),
 		"ratelimitbucket": strconv.Itoa(DefaultKubernetesCloudProviderRateLimitBucket),
+		"gchighthreshold": strconv.Itoa(DefaultKubernetesGCHighThreshold),
+		"gclowthreshold":  strconv.Itoa(DefaultKubernetesGCLowThreshold),
 	},
 	api.KubernetesRelease1Dot5: {
 		"hyperkube":       "hyperkube-amd64:v1.5.7",
@ -161,6 +169,8 @@ var KubeConfigs = map[string]map[string]string{
 		"nodegraceperiod": DefaultKubernetesCtrlMgrNodeMonitorGracePeriod,
 		"podeviction":     DefaultKubernetesCtrlMgrPodEvictionTimeout,
 		"routeperiod":     DefaultKubernetesCtrlMgrRouteReconciliationPeriod,
+		"gchighthreshold": strconv.Itoa(DefaultKubernetesGCHighThreshold),
+		"gclowthreshold":  strconv.Itoa(DefaultKubernetesGCLowThreshold),
 	},
 }

--- a/pkg/acsengine/defaults.go
+++ b/pkg/acsengine/defaults.go
@ -173,6 +173,12 @@ func setOrchestratorDefaults(cs *api.ContainerService) {
 				a.OrchestratorProfile.KubernetesConfig.MaxPods = DefaultKubernetesMaxPods
 			}
 		}
+		if a.OrchestratorProfile.KubernetesConfig.GCHighThreshold == 0 {
+			a.OrchestratorProfile.KubernetesConfig.GCHighThreshold = DefaultKubernetesGCHighThreshold
+		}
+		if a.OrchestratorProfile.KubernetesConfig.GCLowThreshold == 0 {
+			a.OrchestratorProfile.KubernetesConfig.GCLowThreshold = DefaultKubernetesGCLowThreshold
+		}
 		if a.OrchestratorProfile.KubernetesConfig.DNSServiceIP == "" {
 			a.OrchestratorProfile.KubernetesConfig.DNSServiceIP = DefaultKubernetesDNSServiceIP
 		}
--- a/pkg/acsengine/engine.go
+++ b/pkg/acsengine/engine.go
@ -529,6 +529,8 @@ func getParameters(cs *api.ContainerService, isClassicMode bool) (paramsMap, err
 		addValue(parametersMap, "vnetCniWindowsPluginsURL", cloudSpecConfig.KubernetesSpecConfig.VnetCNIWindowsPluginsDownloadURL)
 		addValue(parametersMap, "calicoConfigURL", cloudSpecConfig.KubernetesSpecConfig.CalicoConfigDownloadURL)
 		addValue(parametersMap, "maxPods", properties.OrchestratorProfile.KubernetesConfig.MaxPods)
+		addValue(parametersMap, "gchighthreshold", properties.OrchestratorProfile.KubernetesConfig.GCHighThreshold)
+		addValue(parametersMap, "gclowthreshold", properties.OrchestratorProfile.KubernetesConfig.GCLowThreshold)

 		if properties.OrchestratorProfile.KubernetesConfig == nil ||
 			!properties.OrchestratorProfile.KubernetesConfig.UseManagedIdentity {
@ -1060,6 +1062,10 @@ func (t *TemplateGenerator) getTemplateFuncMap(cs *api.ContainerService) templat
 					val = "Tm90QXZhaWxhYmxlCg=="
 				case "dockerBridgeCidr":
 					val = DefaultDockerBridgeSubnet
+				case "gchighthreshold":
+					val = strconv.Itoa(cs.Properties.OrchestratorProfile.KubernetesConfig.GCHighThreshold)
+				case "gclowthreshold":
+					val = strconv.Itoa(cs.Properties.OrchestratorProfile.KubernetesConfig.GCLowThreshold)
 				default:
 					val = ""
 				}
--- a/pkg/api/converterfromapi.go
+++ b/pkg/api/converterfromapi.go
@ -655,6 +655,8 @@ func convertKubernetesConfigToVLabs(api *KubernetesConfig, vlabs *vlabs.Kubernet
 	vlabs.CustomHyperkubeImage = api.CustomHyperkubeImage
 	vlabs.UseInstanceMetadata = api.UseInstanceMetadata
 	vlabs.EnableRbac = api.EnableRbac
+	vlabs.GCHighThreshold = api.GCHighThreshold
+	vlabs.GCLowThreshold = api.GCLowThreshold
 }

 func convertMasterProfileToV20160930(api *MasterProfile, v20160930 *v20160930.MasterProfile) {
--- a/pkg/api/convertertoapi.go
+++ b/pkg/api/convertertoapi.go
@ -608,6 +608,8 @@ func convertVLabsKubernetesConfig(vlabs *vlabs.KubernetesConfig, api *Kubernetes
 	api.CustomHyperkubeImage = vlabs.CustomHyperkubeImage
 	api.UseInstanceMetadata = vlabs.UseInstanceMetadata
 	api.EnableRbac = vlabs.EnableRbac
+	api.GCHighThreshold = vlabs.GCHighThreshold
+	api.GCLowThreshold = vlabs.GCLowThreshold
 }

 func convertV20160930MasterProfile(v20160930 *v20160930.MasterProfile, api *MasterProfile) {
--- a/pkg/api/types.go
+++ b/pkg/api/types.go
@ -166,6 +166,8 @@ type KubernetesConfig struct {
 	CustomHyperkubeImage             string  `json:"customHyperkubeImage,omitempty"`
 	UseInstanceMetadata              bool    `json:"useInstanceMetadata,omitempty"`
 	EnableRbac                       bool    `json:"enableRbac,omitempty"`
+	GCHighThreshold                  int     `json:"gchighthreshold,omitempty"`
+	GCLowThreshold                   int     `json:"gclowthreshold,omitempty"`
 }

 // MasterProfile represents the definition of the master cluster
--- a/pkg/api/vlabs/types.go
+++ b/pkg/api/vlabs/types.go
@ -197,6 +197,8 @@ type KubernetesConfig struct {
 	CustomHyperkubeImage             string  `json:"customHyperkubeImage,omitempty"`
 	UseInstanceMetadata              bool    `json:"useInstanceMetadata,omitempty"`
 	EnableRbac                       bool    `json:"enableRbac,omitempty"`
+	GCHighThreshold                  int     `json:"gchighthreshold,omitempty"`
+	GCLowThreshold                   int     `json:"gclowthreshold,omitempty"`
 }

 // MasterProfile represents the definition of the master cluster