Azure
/
acs-engine
зеркало из https://github.com/Azure/acs-engine.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

ip-masq-agent as addon (#3916)

This commit is contained in:
Jack Francis 2018-10-03 16:15:22 -06:00 коммит произвёл GitHub
Родитель f2d3946920
Коммит f3f949d8ee
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
17 изменённых файлов: 225 добавлений и 14 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

56
parts/k8s/addons/ip-masq-agent.yaml Normal file
Просмотреть файл

@ -0,0 +1,56 @@
apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
name: azure-ip-masq-agent
namespace: kube-system
labels:
component: azure-ip-masq-agent
kubernetes.io/cluster-service: "true"
addonmanager.kubernetes.io/mode: Reconcile
tier: node
spec:
template:
metadata:
labels:
k8s-app: azure-ip-masq-agent
tier: node
spec:
hostNetwork: true
nodeSelector:
beta.kubernetes.io/os: linux
containers:
- name: azure-ip-masq-agent
image: gcr.io/google-containers/ip-masq-agent-amd64:v2.0.0
securityContext:
privileged: true
volumeMounts:
- name: azure-ip-masq-agent-config-volume
mountPath: /etc/config
resources:
requests:
cpu: <kubernetesIPMasqAgentCPURequests>
memory: <kubernetesIPMasqAgentMemoryRequests>
limits:
cpu: <kubernetesIPMasqAgentCPULimit>
memory: <kubernetesIPMasqAgentMemoryLimit>
volumes:
- name: azure-ip-masq-agent-config-volume
configMap:
name: azure-ip-masq-agent-config
---
apiVersion: v1
kind: ConfigMap
metadata:
name: azure-ip-masq-agent-config
namespace: kube-system
labels:
component: azure-ip-masq-agent
kubernetes.io/cluster-service: "true"
addonmanager.kubernetes.io/mode: EnsureExists
data:
ip-masq-agent: |-
nonMasqueradeCIDRs:
- <kubernetesNonMasqueradeCidr>
- <azureCNINonMasqueradeIP>
masqLinkLocal: <masqLinkLocalValue>
resyncInterval: 60s

4
parts/k8s/kubernetesagentcustomdata.yml
Просмотреть файл

@ -188,10 +188,6 @@ AGENT_ARTIFACTS_CONFIG_PLACEHOLDER
owner: "root"
content: |
#!/bin/bash
{{if IsAzureCNI}}
# SNAT outbound traffic from pods to destinations outside of VNET.
iptables -t nat -A POSTROUTING -m iprange ! --dst-range 168.63.129.16 -m addrtype ! --dst-type local ! -d {{WrapAsParameter "vnetCidr"}} -j MASQUERADE
{{end}}
{{if not EnablePodSecurityPolicy}}
sed -i "s|apparmor_parser|d|g" "/etc/systemd/system/kubelet.service"
{{end}}

14
parts/k8s/kubernetesmastercustomdata.yml
Просмотреть файл

@ -242,7 +242,6 @@ MASTER_ARTIFACTS_CONFIG_PLACEHOLDER
{{if IsAzureCNI}}
# SNAT outbound traffic from pods to destinations outside of VNET.
iptables -t nat -A POSTROUTING -m iprange ! --dst-range 168.63.129.16 -m addrtype ! --dst-type local ! -d {{WrapAsParameter "vnetCidr"}} -j MASQUERADE
sed -i "s|<azureCNINetworkMonitorImage>|{{WrapAsParameter "AzureCNINetworkMonitorImageURL"}}|g" "/etc/kubernetes/addons/azure-cni-networkmonitor.yaml"
{{end}}
sed -i "s|<kubernetesAddonManagerSpec>|{{WrapAsParameter "kubernetesAddonManagerSpec"}}|g" "/etc/kubernetes/manifests/kube-addon-manager.yaml"
@ -393,6 +392,19 @@ MASTER_ARTIFACTS_CONFIG_PLACEHOLDER
sed -i "s|<kubernetesOMSAgentMemoryLimit>|{{WrapAsParameter "kubernetesOMSAgentMemoryLimit"}}|g" "/etc/kubernetes/addons/omsagent-daemonset.yaml"
{{end}}
sed -i "s|<kubernetesNonMasqueradeCidr>|{{WrapAsParameter "kubernetesNonMasqueradeCidr"}}|g" "/etc/kubernetes/addons/ip-masq-agent.yaml"
sed -i "s|<kubernetesIPMasqAgentCPURequests>|{{WrapAsParameter "kubernetesIPMasqAgentCPURequests"}}|g" "/etc/kubernetes/addons/ip-masq-agent.yaml"
sed -i "s|<kubernetesIPMasqAgentMemoryRequests>|{{WrapAsParameter "kubernetesIPMasqAgentMemoryRequests"}}|g" "/etc/kubernetes/addons/ip-masq-agent.yaml"
sed -i "s|<kubernetesIPMasqAgentCPULimit>|{{WrapAsParameter "kubernetesIPMasqAgentCPULimit"}}|g" "/etc/kubernetes/addons/ip-masq-agent.yaml"
sed -i "s|<kubernetesIPMasqAgentMemoryLimit>|{{WrapAsParameter "kubernetesIPMasqAgentMemoryLimit"}}|g" "/etc/kubernetes/addons/ip-masq-agent.yaml"
{{if IsAzureCNI}}
sed -i "s|<azureCNINonMasqueradeIP>|168.63.129.16/32|g" "/etc/kubernetes/addons/ip-masq-agent.yaml"
sed -i "s|<masqLinkLocalValue>|true|g" "/etc/kubernetes/addons/ip-masq-agent.yaml"
{{else}}
sed -i "\|<azureCNINonMasqueradeIP>|d" "/etc/kubernetes/addons/ip-masq-agent.yaml"
sed -i "s|<masqLinkLocalValue>|false|g" "/etc/kubernetes/addons/ip-masq-agent.yaml"
{{end}}
- path: "/opt/azure/containers/provision.sh"
permissions: "0744"
encoding: gzip

14
parts/k8s/kubernetesmastercustomdatavmss.yml
Просмотреть файл

@ -244,7 +244,6 @@ MASTER_ARTIFACTS_CONFIG_PLACEHOLDER
{{if IsAzureCNI}}
# SNAT outbound traffic from pods to destinations outside of VNET.
iptables -t nat -A POSTROUTING -m iprange ! --dst-range 168.63.129.16 -m addrtype ! --dst-type local ! -d {{WrapAsParameter "vnetCidr"}} -j MASQUERADE
sed -i "s|<azureCNINetworkMonitorImage>|{{WrapAsParameter "AzureCNINetworkMonitorImageURL"}}|g" "/etc/kubernetes/addons/azure-cni-networkmonitor.yaml"
{{end}}
sed -i "s|<kubernetesAddonManagerSpec>|{{WrapAsParameter "kubernetesAddonManagerSpec"}}|g" "/etc/kubernetes/manifests/kube-addon-manager.yaml"
@ -395,6 +394,19 @@ MASTER_ARTIFACTS_CONFIG_PLACEHOLDER
sed -i "s|<kubernetesOMSAgentMemoryLimit>|{{WrapAsParameter "kubernetesOMSAgentMemoryLimit"}}|g" "/etc/kubernetes/addons/omsagent-daemonset.yaml"
{{end}}
sed -i "s|<kubernetesNonMasqueradeCidr>|{{WrapAsParameter "kubernetesNonMasqueradeCidr"}}|g" "/etc/kubernetes/addons/ip-masq-agent.yaml"
sed -i "s|<kubernetesIPMasqAgentCPURequests>|{{WrapAsParameter "kubernetesIPMasqAgentCPURequests"}}|g" "/etc/kubernetes/addons/ip-masq-agent.yaml"
sed -i "s|<kubernetesIPMasqAgentMemoryRequests>|{{WrapAsParameter "kubernetesIPMasqAgentMemoryRequests"}}|g" "/etc/kubernetes/addons/ip-masq-agent.yaml"
sed -i "s|<kubernetesIPMasqAgentCPULimit>|{{WrapAsParameter "kubernetesIPMasqAgentCPULimit"}}|g" "/etc/kubernetes/addons/ip-masq-agent.yaml"
sed -i "s|<kubernetesIPMasqAgentMemoryLimit>|{{WrapAsParameter "kubernetesIPMasqAgentMemoryLimit"}}|g" "/etc/kubernetes/addons/ip-masq-agent.yaml"
{{if IsAzureCNI}}
sed -i "s|<azureCNINonMasqueradeIP>|168.63.129.16/32|g" "/etc/kubernetes/addons/ip-masq-agent.yaml"
sed -i "s|<masqLinkLocalValue>|true|g" "/etc/kubernetes/addons/ip-masq-agent.yaml"
{{else}}
sed -i "\|<azureCNINonMasqueradeIP>|d" "/etc/kubernetes/addons/ip-masq-agent.yaml"
sed -i "s|<masqLinkLocalValue>|false|g" "/etc/kubernetes/addons/ip-masq-agent.yaml"
{{end}}
- path: "/opt/azure/containers/provision.sh"
permissions: "0744"
encoding: gzip

29
parts/k8s/kubernetesparams.t
Просмотреть файл

@ -205,6 +205,7 @@
"metadata": {
"description": "kubernetesNonMasqueradeCidr cluster subnet"
},
"defaultValue": "{{GetDefaultVNETCIDR}}",
"type": "string"
},
"kubernetesKubeletClusterDomain": {
@ -576,6 +577,32 @@
},
"type": "string"
},
{{end}}
{{if .OrchestratorProfile.KubernetesConfig.IsIPMasqAgentEnabled}}
"kubernetesIPMasqAgentCPURequests": {
"metadata": {
"description": "IP Masq Agent CPU Requests"
},
"type": "string"
},
"kubernetesIPMasqAgentMemoryRequests": {
"metadata": {
"description": "IP Masq Agent Memory Requests"
},
"type": "string"
},
"kubernetesIPMasqAgentCPULimit": {
"metadata": {
"description": "IP Masq Agent CPU Limit"
},
"type": "string"
},
"kubernetesIPMasqAgentMemoryLimit": {
"metadata": {
"description": "IP Masq Agent Memory Limit"
},
"type": "string"
},
{{end}}
"kubernetesPodInfraContainerSpec": {
"metadata": {
@ -695,7 +722,7 @@
"type": "int"
},
"vnetCidr": {
"defaultValue": "10.0.0.0/8",
"defaultValue": "{{GetDefaultVNETCIDR}}",
"metadata": {
"description": "Cluster vnet cidr"
},

15
pkg/acsengine/addons.go
Просмотреть файл

@ -179,6 +179,20 @@ func setAddonsConfig(cs *api.ContainerService) {
},
}
defaultIPMasqAgentAddonsConfig := api.KubernetesAddon{
Name: IPMASQAgentAddonName,
Enabled: helpers.PointerToBool(api.IPMasqAgentAddonEnabled),
Containers: []api.KubernetesContainerSpec{
{
Name: IPMASQAgentAddonName,
CPURequests: "50m",
MemoryRequests: "50Mi",
CPULimits: "50m",
MemoryLimits: "250Mi",
},
},
}
defaultAzureCNINetworkMonitorAddonsConfig := api.KubernetesAddon{
Name: AzureCNINetworkMonitoringAddonName,
Enabled: azureCNINetworkMonitorAddonEnabled(o),
@ -213,6 +227,7 @@ func setAddonsConfig(cs *api.ContainerService) {
defaultContainerMonitoringAddonsConfig,
defaultAzureCNINetworkMonitorAddonsConfig,
defaultAzureNetworkPolicyAddonsConfig,
defaultIPMasqAgentAddonsConfig,
}
// Add default addons specification, if no user-provided spec exists
if o.KubernetesConfig.Addons == nil {

9
pkg/acsengine/artifacts.go
Просмотреть файл

@ -120,7 +120,6 @@ func kubernetesAddonSettingsInit(profile *api.Properties) []kubernetesAddonSetti
profile.OrchestratorProfile.KubernetesConfig.GetAddonScript(DefaultReschedulerAddonName),
},
{
kubernetesFeatureSetting{
"kubernetesmasteraddons-azure-npm-daemonset.yaml",
"azure-npm-daemonset.yaml",
@ -233,6 +232,14 @@ func kubernetesAddonSettingsInit(profile *api.Properties) []kubernetesAddonSetti
},
profile.OrchestratorProfile.KubernetesConfig.GetAddonScript(DefaultELBSVCAddonName),
},
{
kubernetesFeatureSetting{
"ip-masq-agent.yaml",
"ip-masq-agent.yaml",
true,
},
profile.OrchestratorProfile.KubernetesConfig.GetAddonScript(IPMASQAgentAddonName),
},
}
}

4
pkg/acsengine/const.go
Просмотреть файл

@ -31,6 +31,8 @@ const (
// DefaultKubernetesSubnet specifies the default subnet used for all masters, agents and pods
// when VNET integration is enabled.
DefaultKubernetesSubnet = "10.240.0.0/12"
// DefaultVNETCIDR is the default CIDR block for the VNET
DefaultVNETCIDR = "10.0.0.0/8"
// DefaultKubernetesMaxPods is the maximum number of pods to run on a node.
DefaultKubernetesMaxPods = 110
// DefaultKubernetesMaxPodsVNETIntegrated is the maximum number of pods to run on a node when VNET integration is enabled.
@ -160,6 +162,8 @@ const (
AzureCNINetworkMonitoringAddonName = "azure-cni-networkmonitor"
// AzureNetworkPolicyAddonName is the name of the Azure CNI networkmonitor addon
AzureNetworkPolicyAddonName = "azure-npm-daemonset"
// IPMASQAgentAddonName is the name of the ip masq agent addon
IPMASQAgentAddonName = "ip-masq-agent"
// DefaultKubernetesKubeletMaxPods is the max pods per kubelet
DefaultKubernetesKubeletMaxPods = 110
// DefaultMasterEtcdServerPort is the default etcd server port for Kubernetes master nodes

2
pkg/acsengine/defaults-kubelet.go
Просмотреть файл

@ -55,7 +55,7 @@ func setKubeletConfig(cs *api.ContainerService) {
"--node-status-update-frequency": KubeConfigs[o.OrchestratorVersion]["nodestatusfreq"],
"--image-gc-high-threshold": strconv.Itoa(DefaultKubernetesGCHighThreshold),
"--image-gc-low-threshold": strconv.Itoa(DefaultKubernetesGCLowThreshold),
"--non-masquerade-cidr": o.KubernetesConfig.ClusterSubnet,
"--non-masquerade-cidr": "0.0.0.0",
"--cloud-provider": "azure",
"--cloud-config": "/etc/kubernetes/azure.json",
"--azure-container-registry-config": "/etc/kubernetes/azure.json",

20
pkg/acsengine/params_k8s.go
Просмотреть файл

@ -219,6 +219,16 @@ func assignKubernetesParameters(properties *api.Properties, parametersMap params
}
}
}
if kubernetesConfig.IsIPMasqAgentEnabled() {
ipMasqAgentAddon := kubernetesConfig.GetAddonByName(IPMASQAgentAddonName)
i := ipMasqAgentAddon.GetAddonContainersIndexByName(IPMASQAgentAddonName)
if i > -1 {
addValue(parametersMap, "kubernetesIPMasqAgentCPURequests", ipMasqAgentAddon.Containers[c].CPURequests)
addValue(parametersMap, "kubernetesIPMasqAgentMemoryRequests", ipMasqAgentAddon.Containers[c].MemoryRequests)
addValue(parametersMap, "kubernetesIPMasqAgentCPULimit", ipMasqAgentAddon.Containers[c].CPULimits)
addValue(parametersMap, "kubernetesIPMasqAgentMemoryLimit", ipMasqAgentAddon.Containers[c].MemoryLimits)
}
}
if kubernetesConfig.LoadBalancerSku == "Standard" {
random := rand.New(rand.NewSource(time.Now().UnixNano()))
elbsvcName := random.Int()
@ -249,7 +259,15 @@ func assignKubernetesParameters(properties *api.Properties, parametersMap params
CloudProviderRateLimitBucket: kubernetesConfig.CloudProviderRateLimitBucket,
})
addValue(parametersMap, "kubeClusterCidr", kubernetesConfig.ClusterSubnet)
addValue(parametersMap, "kubernetesNonMasqueradeCidr", kubernetesConfig.KubeletConfig["--non-masquerade-cidr"])
if properties.OrchestratorProfile.IsAzureCNI() {
if properties.MasterProfile != nil && properties.MasterProfile.IsCustomVNET() {
addValue(parametersMap, "kubernetesNonMasqueradeCidr", properties.MasterProfile.VnetCidr)
} else {
addValue(parametersMap, "kubernetesNonMasqueradeCidr", DefaultVNETCIDR)
}
} else {
addValue(parametersMap, "kubernetesNonMasqueradeCidr", properties.OrchestratorProfile.KubernetesConfig.ClusterSubnet)
}
addValue(parametersMap, "kubernetesKubeletClusterDomain", kubernetesConfig.KubeletConfig["--cluster-domain"])
addValue(parametersMap, "dockerBridgeCidr", kubernetesConfig.DockerBridgeSubnet)
addValue(parametersMap, "networkPolicy", kubernetesConfig.NetworkPolicy)

3
pkg/acsengine/template_generator.go
Просмотреть файл

@ -522,6 +522,9 @@ func (t *TemplateGenerator) getTemplateFuncMap(cs *api.ContainerService) templat
}
return GetMasterAgentAllowedSizes()
},
"GetDefaultVNETCIDR": func() string {
return DefaultVNETCIDR
},
"GetAgentAllowedSizes": func() string {
if cs.Properties.OrchestratorProfile.IsKubernetes() || cs.Properties.OrchestratorProfile.IsOpenShift() {
return GetKubernetesAgentAllowedSizes()

4
pkg/api/const.go
Просмотреть файл

@ -144,6 +144,8 @@ const (
DefaultContainerMonitoringAddonEnabled = false
// DefaultAzureCNINetworkMonitoringAddonEnabled Azure CNI networkmonitor addon default
DefaultAzureCNINetworkMonitoringAddonEnabled = false
// IPMasqAgentAddonEnabled enables the ip-masq-agent addon
IPMasqAgentAddonEnabled = true
// DefaultTillerAddonName is the name of the tiller addon deployment
DefaultTillerAddonName = "tiller"
// DefaultAADPodIdentityAddonName is the name of the aad-pod-identity addon deployment
@ -168,6 +170,8 @@ const (
NVIDIADevicePluginAddonName = "nvidia-device-plugin"
// ContainerMonitoringAddonName is the name of the kubernetes Container Monitoring addon deployment
ContainerMonitoringAddonName = "container-monitoring"
// IPMASQAgentAddonName is the name of the ip masq agent addon
IPMASQAgentAddonName = "ip-masq-agent"
// DefaultPrivateClusterEnabled determines the acs-engine provided default for enabling kubernetes Private Cluster
DefaultPrivateClusterEnabled = false
// NetworkPolicyAzure is the string expression for Azure CNI network policy manager

5
pkg/api/types.go
Просмотреть файл

@ -1204,6 +1204,11 @@ func (k *KubernetesConfig) IsDashboardEnabled() bool {
return k.isAddonEnabled(DefaultDashboardAddonName, DefaultDashboardAddonEnabled)
}
// IsIPMasqAgentEnabled checks if the ip-masq-agent addon is enabled
func (k *KubernetesConfig) IsIPMasqAgentEnabled() bool {
return k.isAddonEnabled(IPMASQAgentAddonName, IPMasqAgentAddonEnabled)
}
// IsNSeriesSKU returns whether or not the agent pool has Standard_N SKU VMs
func IsNSeriesSKU(p *Properties) bool {
for _, profile := range p.AgentPoolProfiles {

31
pkg/api/types_test.go
Просмотреть файл

@ -1374,6 +1374,37 @@ func TestIsMetricsServerEnabled(t *testing.T) {
}
}
func TestIsIPMasqAgentEnabled(t *testing.T) {
c := KubernetesConfig{
Addons: []KubernetesAddon{
getMockAddon("addon"),
},
}
enabled := c.IsIPMasqAgentEnabled()
enabledDefault := IPMasqAgentAddonEnabled
if enabled != enabledDefault {
t.Fatalf("KubernetesConfig.IsIPMasqAgentEnabled() should return %t when no ip-masq-agent addon has been specified, instead returned %t", enabledDefault, enabled)
}
c.Addons = append(c.Addons, getMockAddon(IPMASQAgentAddonName))
enabled = c.IsIPMasqAgentEnabled()
if !enabled {
t.Fatalf("KubernetesConfig.IsIPMasqAgentEnabled() should return true when ip-masq-agent adddon has been specified, instead returned %t", enabled)
}
b := false
c = KubernetesConfig{
Addons: []KubernetesAddon{
{
Name: IPMASQAgentAddonName,
Enabled: &b,
},
},
}
enabled = c.IsIPMasqAgentEnabled()
if enabled {
t.Fatalf("KubernetesConfig.IsIPMasqAgentEnabled() should return false when ip-masq-agent addon has been specified as disabled, instead returned %t", enabled)
}
}
func TestCloudProviderDefaults(t *testing.T) {
// Test cloudprovider defaults when no user-provided values
v := "1.8.0"

5
pkg/api/vlabs/validate.go
Просмотреть файл

@ -1074,11 +1074,12 @@ func (k *KubernetesConfig) Validate(k8sVersion string, hasWindows bool) error {
}
}
}
if _, ok := k.KubeletConfig["--non-masquerade-cidr"]; ok {
// Re-enable this unit test if --non-masquerade-cidr is re-introduced
/*if _, ok := k.KubeletConfig["--non-masquerade-cidr"]; ok {
if _, _, err := net.ParseCIDR(k.KubeletConfig["--non-masquerade-cidr"]); err != nil {
return errors.Errorf("--non-masquerade-cidr kubelet config '%s' is an invalid CIDR string", k.KubeletConfig["--non-masquerade-cidr"])
}
}
}*/
}
if _, ok := k.ControllerManagerConfig["--pod-eviction-timeout"]; ok {

5
pkg/api/vlabs/validate_test.go
Просмотреть файл

@ -414,14 +414,15 @@ func Test_KubernetesConfig_Validate(t *testing.T) {
t.Error("should not error on valid --non-masquerade-cidr")
}
c = KubernetesConfig{
// Re-implement these tests if we re-introduce --ip-maquerade-cidr
/*c = KubernetesConfig{
KubeletConfig: map[string]string{
"--non-masquerade-cidr": "10.120.1.0/invalid",
},
}
if err := c.Validate(k8sVersion, false); err == nil {
t.Error("should error on invalid --non-masquerade-cidr")
}
}*/
c = KubernetesConfig{
MaxPods: KubernetesMinMaxPods - 1,

19
test/e2e/kubernetes/kubernetes_test.go
Просмотреть файл

@ -427,6 +427,25 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu
}
})
It("should have ip-masq-agent running", func() {
if hasIPMasqAgent, IPMasqAgentAddon := eng.HasAddon("ip-masq-agent"); hasIPMasqAgent {
running, err := pod.WaitOnReady("azure-ip-masq-agent", "kube-system", 3, 30*time.Second, cfg.Timeout)
Expect(err).NotTo(HaveOccurred())
Expect(running).To(Equal(true))
By("Ensuring that the correct resources have been applied")
pods, err := pod.GetAllByPrefix("azure-ip-masq-agent", "kube-system")
Expect(err).NotTo(HaveOccurred())
for _, p := range pods {
for i, c := range IPMasqAgentAddon.Containers {
err := p.Spec.Containers[i].ValidateResources(c)
Expect(err).NotTo(HaveOccurred())
}
}
} else {
Skip("ip-masq-agent disabled for this cluster, will not test")
}
})
It("should have aci-connector running", func() {
if hasACIConnector, ACIConnectorAddon := eng.HasAddon("aci-connector"); hasACIConnector {
running, err := pod.WaitOnReady("aci-connector", "kube-system", 3, 30*time.Second, cfg.Timeout)