From bb4a637c50a321b9be3caf566453168341d29e99 Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Wed, 24 Oct 2018 13:30:49 -0700 Subject: [PATCH] Moby container runtime (#3896) --- .vsts-ci.yaml | 1 + docs/clusterdefinition.md | 1 - packer/feature-flagged.sh | 15 +++++++++++++ packer/install-dependencies.sh | 10 ++++++--- packer/vhd-image-builder.json | 10 +++++++-- parts/k8s/kubernetesagentcustomdata.yml | 3 ++- parts/k8s/kubernetesconfigs.sh | 1 + parts/k8s/kubernetesinstalls.sh | 21 +++++++++--------- parts/k8s/kubernetesmastercustomdata.yml | 3 ++- parts/k8s/kubernetesmastercustomdatavmss.yml | 3 ++- parts/k8s/kubernetesmastervars.t | 2 +- parts/k8s/kubernetesmastervarsvmss.t | 2 +- parts/k8s/kubernetesparams.t | 23 -------------------- parts/k8s/kubernetesprovisionsource.sh | 3 +++ pkg/acsengine/params_k8s.go | 12 ---------- pkg/api/converterfromapi.go | 1 - pkg/api/convertertoapi.go | 1 - pkg/api/k8s_versions.go | 8 ------- pkg/api/k8s_versions_test.go | 9 +------- pkg/api/types.go | 6 ++--- pkg/api/vlabs/types.go | 4 ++-- pkg/api/vlabs/validate.go | 6 ++++- 22 files changed, 65 insertions(+), 80 deletions(-) create mode 100644 packer/feature-flagged.sh diff --git a/.vsts-ci.yaml b/.vsts-ci.yaml index 2ea6942cd..d2f405b62 100644 --- a/.vsts-ci.yaml +++ b/.vsts-ci.yaml @@ -22,6 +22,7 @@ phases: -e AZURE_VM_SIZE=${AZURE_VM_SIZE} \ -e AZURE_RESOURCE_GROUP_NAME=${AZURE_RESOURCE_GROUP_NAME} \ -e AZURE_LOCATION=${AZURE_LOCATION} \ + -e FEATURE_FLAGS=${FEATURE_FLAGS} \ -e GIT_VERSION=$(Build.SourceVersion) \ -e BUILD_ID=$(Build.BuildId) \ -e BUILD_NUMBER=$(Build.BuildNumber) \ diff --git a/docs/clusterdefinition.md b/docs/clusterdefinition.md index 6fe3f290c..d09d70d5c 100644 --- a/docs/clusterdefinition.md +++ b/docs/clusterdefinition.md @@ -48,7 +48,6 @@ To learn more about supported orchestrators and versions, run the orchestrators | customWindowsPackageURL | no | Configure custom windows Kubernetes release package URL for deployment on Windows | | dnsServiceIP | no | IP address for kube-dns to listen on. If specified must be in the range of `serviceCidr` | | dockerBridgeSubnet | no | The specific IP and subnet used for allocating IP addresses for the docker bridge network created on the kubernetes master and agents. Default value is 172.17.0.1/16. This value is used to configure the docker daemon using the [--bip flag](https://docs.docker.com/engine/userguide/networking/default_network/custom-docker0) | -| dockerEngineVersion | no | Which version of docker-engine to use in your cluster: `"17.05.*"`, `"17.04.*"`, `"17.03.*"`, `"1.13.*"`, `"1.12.*"`, and `"1.11.*"` | enableAggregatedAPIs | no | Enable [Kubernetes Aggregated APIs](https://kubernetes.io/docs/concepts/api-extension/apiserver-aggregation/).This is required by [Service Catalog](https://github.com/kubernetes-incubator/service-catalog/blob/master/README.md). (boolean - default is true for k8s versions greater or equal to 1.9.0, false otherwise) | | enableDataEncryptionAtRest | no | Enable [kubernetes data encryption at rest](https://kubernetes.io/docs/tasks/administer-cluster/encrypt-data/).This is currently an alpha feature. (boolean - default == false) | | enableEncryptionWithExternalKms | no | Enable [kubernetes data encryption at rest with external KMS](https://kubernetes.io/docs/tasks/administer-cluster/encrypt-data/).This is currently an alpha feature. (boolean - default == false) | diff --git a/packer/feature-flagged.sh b/packer/feature-flagged.sh new file mode 100644 index 000000000..4864eefe4 --- /dev/null +++ b/packer/feature-flagged.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +function installDockerEngine() { + DOCKER_REPO="https://apt.dockerproject.org/repo" + DOCKER_ENGINE_VERSION="1.13.*" + dockerd --version + if [ $? -eq 0 ]; then + echo "dockerd is already installed, skipping download" + else + retrycmd_if_failure_no_stats 20 1 5 curl -fsSL https://aptdocker.azureedge.net/gpg > /tmp/aptdocker.gpg || exit $ERR_DOCKER_KEY_DOWNLOAD_TIMEOUT + retrycmd_if_failure 10 5 10 apt-key add /tmp/aptdocker.gpg || exit $ERR_DOCKER_APT_KEY_TIMEOUT + echo "deb ${DOCKER_REPO} ubuntu-xenial main" | sudo tee /etc/apt/sources.list.d/docker.list + printf "Package: docker-engine\nPin: version ${DOCKER_ENGINE_VERSION}\nPin-Priority: 550\n" > /etc/apt/preferences.d/docker.pref + fi +} \ No newline at end of file diff --git a/packer/install-dependencies.sh b/packer/install-dependencies.sh index ae1caf10f..48b389949 100644 --- a/packer/install-dependencies.sh +++ b/packer/install-dependencies.sh @@ -2,6 +2,7 @@ source /home/packer/provision_installs.sh source /home/packer/provision_source.sh +source /home/packer/feature-flagged.sh ETCD_VERSION="3.2.24" ETCD_DOWNLOAD_URL="https://acs-mirror.azureedge.net/github-coreos" @@ -9,9 +10,11 @@ installEtcd installDeps -DOCKER_REPO="https://apt.dockerproject.org/repo" -DOCKER_ENGINE_VERSION="1.13.*" -installDocker +if [[ ${FEATURE_FLAGS} == *"docker-engine"* ]]; then + installDockerEngine +else + installMoby +fi installClearContainersRuntime @@ -131,3 +134,4 @@ echo "Install completed successfully on " `date` > /var/log/azure/golden-image-i echo "VSTS Build NUMBER: ${BUILD_NUMBER}" >> /var/log/azure/golden-image-install.complete echo "VSTS Build ID: ${BUILD_ID}" >> /var/log/azure/golden-image-install.complete echo "Commit: ${COMMIT}" >> /var/log/azure/golden-image-install.complete +echo "Feature flags: ${FEATURE_FLAGS}" >> /var/log/azure/golden-image-install.complete diff --git a/packer/vhd-image-builder.json b/packer/vhd-image-builder.json index add18d7e0..cb74e6db5 100644 --- a/packer/vhd-image-builder.json +++ b/packer/vhd-image-builder.json @@ -8,7 +8,8 @@ "vm_size": "{{env `AZURE_VM_SIZE`}}", "build_number": "{{env `BUILD_NUMBER`}}", "build_id": "{{env `BUILD_ID`}}", - "commit": "{{env `GIT_VERSION`}}" + "commit": "{{env `GIT_VERSION`}}", + "feature_flags": "{{env `FEATURE_FLAGS`}}" }, "builders": [ { @@ -62,10 +63,15 @@ "source": "packer/install-dependencies.sh", "destination": "/home/packer/install-dependencies.sh" }, + { + "type": "file", + "source": "packer/feature-flagged.sh", + "destination": "/home/packer/feature-flagged.sh" + }, { "type": "shell", "inline": [ - "sudo BUILD_NUMBER={{user `build_number`}} BUILD_ID={{user `build_id`}} COMMIT={{user `commit`}} /bin/bash -ux /home/packer/install-dependencies.sh", + "sudo FEATURE_FLAGS={{user `feature_flags`}} BUILD_NUMBER={{user `build_number`}} BUILD_ID={{user `build_id`}} COMMIT={{user `commit`}} /bin/bash -ux /home/packer/install-dependencies.sh", "sudo /bin/bash -eux /home/packer/cleanup-vhd.sh", "sudo rm -rf /home/packer" ] diff --git a/parts/k8s/kubernetesagentcustomdata.yml b/parts/k8s/kubernetesagentcustomdata.yml index 2b6fdb4e0..f2c5d1289 100644 --- a/parts/k8s/kubernetesagentcustomdata.yml +++ b/parts/k8s/kubernetesagentcustomdata.yml @@ -100,13 +100,14 @@ write_files: [Service] MountFlags=shared {{end}} + - path: "/etc/systemd/system/docker.service.d/exec_start.conf" permissions: "0644" owner: "root" content: | [Service] ExecStart= - ExecStart=/usr/bin/docker daemon -H fd:// --storage-driver=overlay2 --bip={{WrapAsParameter "dockerBridgeCidr"}} + ExecStart=/usr/bin/dockerd -H fd:// --storage-driver=overlay2 --bip={{WrapAsParameter "dockerBridgeCidr"}} - path: "/etc/docker/daemon.json" permissions: "0644" diff --git a/parts/k8s/kubernetesconfigs.sh b/parts/k8s/kubernetesconfigs.sh index 4f9c7a8f1..5d41302d4 100755 --- a/parts/k8s/kubernetesconfigs.sh +++ b/parts/k8s/kubernetesconfigs.sh @@ -252,6 +252,7 @@ function ensureDocker() { wait_for_file 1200 1 $DOCKER_MONITOR_SYSTEMD_FILE || exit $ERR_FILE_WATCH_TIMEOUT systemctlEnableAndStart docker-monitor.timer || exit $ERR_SYSTEMCTL_START_FAIL } + function ensureKMS() { systemctlEnableAndStart kms || exit $ERR_SYSTEMCTL_START_FAIL } diff --git a/parts/k8s/kubernetesinstalls.sh b/parts/k8s/kubernetesinstalls.sh index 2a77d9eda..6a5196a36 100755 --- a/parts/k8s/kubernetesinstalls.sh +++ b/parts/k8s/kubernetesinstalls.sh @@ -48,25 +48,26 @@ function installGPUDrivers() { function installContainerRuntime() { if [[ "$CONTAINER_RUNTIME" == "docker" ]]; then - installDocker + installMoby elif [[ "$CONTAINER_RUNTIME" == "clear-containers" ]]; then + # Ensure we can nest virtualization if grep -q vmx /proc/cpuinfo; then installClearContainersRuntime fi fi } -function installDocker() { - CURRENT_VERSION=$(docker --version | cut -d " " -f 3 | cut -d "," -f 1) - if [[ "$CURRENT_VERSION" = ${DOCKER_ENGINE_VERSION} ]]; then - echo "docker version ${DOCKER_ENGINE_VERSION} is already installed, skipping download" +function installMoby() { + dockerd --version + if [ $? -eq 0 ]; then + echo "dockerd is already installed, skipping download" else - retrycmd_if_failure_no_stats 20 1 5 curl -fsSL https://aptdocker.azureedge.net/gpg > /tmp/aptdocker.gpg || exit $ERR_DOCKER_KEY_DOWNLOAD_TIMEOUT - retrycmd_if_failure 10 5 10 apt-key add /tmp/aptdocker.gpg || exit $ERR_DOCKER_APT_KEY_TIMEOUT - echo "deb ${DOCKER_REPO} ubuntu-xenial main" | sudo tee /etc/apt/sources.list.d/docker.list - printf "Package: docker-engine\nPin: version ${DOCKER_ENGINE_VERSION}\nPin-Priority: 550\n" > /etc/apt/preferences.d/docker.pref + retrycmd_if_failure_no_stats 20 1 5 curl https://packages.microsoft.com/config/ubuntu/16.04/prod.list > /tmp/microsoft-prod.list || exit $ERR_MOBY_APT_LIST_TIMEOUT + retrycmd_if_failure 10 5 10 cp /tmp/microsoft-prod.list /etc/apt/sources.list.d/ || exit $ERR_MOBY_APT_LIST_TIMEOUT + retrycmd_if_failure_no_stats 20 1 5 curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > /tmp/microsoft.gpg || exit $ERR_MS_GPG_KEY_DOWNLOAD_TIMEOUT + retrycmd_if_failure 10 5 10 cp /tmp/microsoft.gpg /etc/apt/trusted.gpg.d/ || exit $ERR_MS_GPG_KEY_DOWNLOAD_TIMEOUT apt_get_update || exit $ERR_APT_UPDATE_TIMEOUT - apt_get_install 20 30 120 docker-engine || exit $ERR_DOCKER_INSTALL_TIMEOUT + apt_get_install 20 30 120 moby-engine moby-cli || exit $ERR_MOBY_INSTALL_TIMEOUT fi } diff --git a/parts/k8s/kubernetesmastercustomdata.yml b/parts/k8s/kubernetesmastercustomdata.yml index 9329f59c7..c1eb29f93 100644 --- a/parts/k8s/kubernetesmastercustomdata.yml +++ b/parts/k8s/kubernetesmastercustomdata.yml @@ -116,8 +116,9 @@ write_files: {{if .MasterProfile.IsCoreOS}} ExecStart=/usr/bin/env PATH=${TORCX_BINDIR}:${PATH} ${TORCX_BINDIR}/dockerd --host=fd:// --containerd=/var/run/docker/libcontainerd/docker-containerd.sock --storage-driver=overlay2 --bip={{WrapAsParameter "dockerBridgeCidr"}} $DOCKER_SELINUX $DOCKER_OPTS $DOCKER_CGROUPS $DOCKER_OPT_BIP $DOCKER_OPT_MTU $DOCKER_OPT_IPMASQ {{else}} - ExecStart=/usr/bin/docker daemon -H fd:// --storage-driver=overlay2 --bip={{WrapAsParameter "dockerBridgeCidr"}} + ExecStart=/usr/bin/dockerd -H fd:// --storage-driver=overlay2 --bip={{WrapAsParameter "dockerBridgeCidr"}} {{end}} + - path: "/etc/docker/daemon.json" permissions: "0644" owner: "root" diff --git a/parts/k8s/kubernetesmastercustomdatavmss.yml b/parts/k8s/kubernetesmastercustomdatavmss.yml index b5dc24e47..507e69a38 100644 --- a/parts/k8s/kubernetesmastercustomdatavmss.yml +++ b/parts/k8s/kubernetesmastercustomdatavmss.yml @@ -116,8 +116,9 @@ write_files: {{if .MasterProfile.IsCoreOS}} ExecStart=/usr/bin/env PATH=${TORCX_BINDIR}:${PATH} ${TORCX_BINDIR}/dockerd --host=fd:// --containerd=/var/run/docker/libcontainerd/docker-containerd.sock --storage-driver=overlay2 --bip={{WrapAsParameter "dockerBridgeCidr"}} $DOCKER_SELINUX $DOCKER_OPTS $DOCKER_CGROUPS $DOCKER_OPT_BIP $DOCKER_OPT_MTU $DOCKER_OPT_IPMASQ {{else}} - ExecStart=/usr/bin/docker daemon -H fd:// --storage-driver=overlay2 --bip={{WrapAsParameter "dockerBridgeCidr"}} + ExecStart=/usr/bin/dockerd -H fd:// --storage-driver=overlay2 --bip={{WrapAsParameter "dockerBridgeCidr"}} {{end}} + - path: "/etc/docker/daemon.json" permissions: "0644" owner: "root" diff --git a/parts/k8s/kubernetesmastervars.t b/parts/k8s/kubernetesmastervars.t index c321dc619..7d6bd1215 100644 --- a/parts/k8s/kubernetesmastervars.t +++ b/parts/k8s/kubernetesmastervars.t @@ -136,7 +136,7 @@ "sshdConfig": "{{GetB64sshdConfig}}", "systemConf": "{{GetB64systemConf}}", {{if not IsOpenShift}} - "provisionScriptParametersCommon": "[concat('ADMINUSER=',parameters('linuxAdminUsername'),' ETCD_DOWNLOAD_URL=',parameters('etcdDownloadURLBase'),' ETCD_VERSION=',parameters('etcdVersion'),' DOCKER_ENGINE_VERSION=',parameters('dockerEngineVersion'),' DOCKER_REPO=',parameters('dockerEngineDownloadRepo'),' TENANT_ID=',variables('tenantID'),' KUBERNETES_VERSION={{.OrchestratorProfile.OrchestratorVersion}} HYPERKUBE_URL=',parameters('kubernetesHyperkubeSpec'),' APISERVER_PUBLIC_KEY=',parameters('apiserverCertificate'),' SUBSCRIPTION_ID=',variables('subscriptionId'),' RESOURCE_GROUP=',variables('resourceGroup'),' LOCATION=',variables('location'),' VM_TYPE=',variables('vmType'),' SUBNET=',variables('subnetName'),' NETWORK_SECURITY_GROUP=',variables('nsgName'),' VIRTUAL_NETWORK=',variables('virtualNetworkName'),' VIRTUAL_NETWORK_RESOURCE_GROUP=',variables('virtualNetworkResourceGroupName'),' ROUTE_TABLE=',variables('routeTableName'),' PRIMARY_AVAILABILITY_SET=',variables('primaryAvailabilitySetName'),' PRIMARY_SCALE_SET=',variables('primaryScaleSetName'),' SERVICE_PRINCIPAL_CLIENT_ID=',variables('servicePrincipalClientId'),' SERVICE_PRINCIPAL_CLIENT_SECRET=',variables('singleQuote'),variables('servicePrincipalClientSecret'),variables('singleQuote'),' KUBELET_PRIVATE_KEY=',parameters('clientPrivateKey'),' TARGET_ENVIRONMENT=',parameters('targetEnvironment'),' NETWORK_PLUGIN=',parameters('networkPlugin'),' VNET_CNI_PLUGINS_URL=',parameters('vnetCniLinuxPluginsURL'),' CNI_PLUGINS_URL=',parameters('cniPluginsURL'),' CLOUDPROVIDER_BACKOFF=',toLower(string(parameters('cloudproviderConfig').cloudProviderBackoff)),' CLOUDPROVIDER_BACKOFF_RETRIES=',parameters('cloudproviderConfig').cloudProviderBackoffRetries,' CLOUDPROVIDER_BACKOFF_EXPONENT=',parameters('cloudproviderConfig').cloudProviderBackoffExponent,' CLOUDPROVIDER_BACKOFF_DURATION=',parameters('cloudproviderConfig').cloudProviderBackoffDuration,' CLOUDPROVIDER_BACKOFF_JITTER=',parameters('cloudproviderConfig').cloudProviderBackoffJitter,' CLOUDPROVIDER_RATELIMIT=',toLower(string(parameters('cloudproviderConfig').cloudProviderRatelimit)),' CLOUDPROVIDER_RATELIMIT_QPS=',parameters('cloudproviderConfig').cloudProviderRatelimitQPS,' CLOUDPROVIDER_RATELIMIT_BUCKET=',parameters('cloudproviderConfig').cloudProviderRatelimitBucket,' USE_MANAGED_IDENTITY_EXTENSION=',variables('useManagedIdentityExtension'),' USER_ASSIGNED_IDENTITY_ID=',variables('userAssignedClientID'),' USE_INSTANCE_METADATA=',variables('useInstanceMetadata'),' LOAD_BALANCER_SKU=',variables('loadBalancerSku'),' EXCLUDE_MASTER_FROM_STANDARD_LB=',variables('excludeMasterFromStandardLB'),' CONTAINER_RUNTIME=',parameters('containerRuntime'),' CONTAINERD_DOWNLOAD_URL_BASE=',parameters('containerdDownloadURLBase'),' POD_INFRA_CONTAINER_SPEC=',parameters('kubernetesPodInfraContainerSpec'),' KMS_PROVIDER_VAULT_NAME=',variables('clusterKeyVaultName'))]", + "provisionScriptParametersCommon": "[concat('ADMINUSER=',parameters('linuxAdminUsername'),' ETCD_DOWNLOAD_URL=',parameters('etcdDownloadURLBase'),' ETCD_VERSION=',parameters('etcdVersion'),' TENANT_ID=',variables('tenantID'),' KUBERNETES_VERSION={{.OrchestratorProfile.OrchestratorVersion}} HYPERKUBE_URL=',parameters('kubernetesHyperkubeSpec'),' APISERVER_PUBLIC_KEY=',parameters('apiserverCertificate'),' SUBSCRIPTION_ID=',variables('subscriptionId'),' RESOURCE_GROUP=',variables('resourceGroup'),' LOCATION=',variables('location'),' VM_TYPE=',variables('vmType'),' SUBNET=',variables('subnetName'),' NETWORK_SECURITY_GROUP=',variables('nsgName'),' VIRTUAL_NETWORK=',variables('virtualNetworkName'),' VIRTUAL_NETWORK_RESOURCE_GROUP=',variables('virtualNetworkResourceGroupName'),' ROUTE_TABLE=',variables('routeTableName'),' PRIMARY_AVAILABILITY_SET=',variables('primaryAvailabilitySetName'),' PRIMARY_SCALE_SET=',variables('primaryScaleSetName'),' SERVICE_PRINCIPAL_CLIENT_ID=',variables('servicePrincipalClientId'),' SERVICE_PRINCIPAL_CLIENT_SECRET=',variables('singleQuote'),variables('servicePrincipalClientSecret'),variables('singleQuote'),' KUBELET_PRIVATE_KEY=',parameters('clientPrivateKey'),' TARGET_ENVIRONMENT=',parameters('targetEnvironment'),' NETWORK_PLUGIN=',parameters('networkPlugin'),' VNET_CNI_PLUGINS_URL=',parameters('vnetCniLinuxPluginsURL'),' CNI_PLUGINS_URL=',parameters('cniPluginsURL'),' CLOUDPROVIDER_BACKOFF=',toLower(string(parameters('cloudproviderConfig').cloudProviderBackoff)),' CLOUDPROVIDER_BACKOFF_RETRIES=',parameters('cloudproviderConfig').cloudProviderBackoffRetries,' CLOUDPROVIDER_BACKOFF_EXPONENT=',parameters('cloudproviderConfig').cloudProviderBackoffExponent,' CLOUDPROVIDER_BACKOFF_DURATION=',parameters('cloudproviderConfig').cloudProviderBackoffDuration,' CLOUDPROVIDER_BACKOFF_JITTER=',parameters('cloudproviderConfig').cloudProviderBackoffJitter,' CLOUDPROVIDER_RATELIMIT=',toLower(string(parameters('cloudproviderConfig').cloudProviderRatelimit)),' CLOUDPROVIDER_RATELIMIT_QPS=',parameters('cloudproviderConfig').cloudProviderRatelimitQPS,' CLOUDPROVIDER_RATELIMIT_BUCKET=',parameters('cloudproviderConfig').cloudProviderRatelimitBucket,' USE_MANAGED_IDENTITY_EXTENSION=',variables('useManagedIdentityExtension'),' USER_ASSIGNED_IDENTITY_ID=',variables('userAssignedClientID'),' USE_INSTANCE_METADATA=',variables('useInstanceMetadata'),' LOAD_BALANCER_SKU=',variables('loadBalancerSku'),' EXCLUDE_MASTER_FROM_STANDARD_LB=',variables('excludeMasterFromStandardLB'),' CONTAINER_RUNTIME=',parameters('containerRuntime'),' CONTAINERD_DOWNLOAD_URL_BASE=',parameters('containerdDownloadURLBase'),' POD_INFRA_CONTAINER_SPEC=',parameters('kubernetesPodInfraContainerSpec'),' KMS_PROVIDER_VAULT_NAME=',variables('clusterKeyVaultName'))]", {{if not IsHostedMaster}} "provisionScriptParametersMaster": "[concat('MASTER_VM_NAME=',variables('masterVMNames')[variables('masterOffset')],' ETCD_PEER_URL=',variables('masterEtcdPeerURLs')[variables('masterOffset')],' ETCD_CLIENT_URL=',variables('masterEtcdClientURLs')[variables('masterOffset')],' MASTER_NODE=true CLUSTER_AUTOSCALER_ADDON=',parameters('kubernetesClusterAutoscalerEnabled'),' ACI_CONNECTOR_ADDON=',parameters('kubernetesACIConnectorEnabled'),' APISERVER_PRIVATE_KEY=',parameters('apiServerPrivateKey'),' CA_CERTIFICATE=',parameters('caCertificate'),' CA_PRIVATE_KEY=',parameters('caPrivateKey'),' MASTER_FQDN=',variables('masterFqdnPrefix'),' KUBECONFIG_CERTIFICATE=',parameters('kubeConfigCertificate'),' KUBECONFIG_KEY=',parameters('kubeConfigPrivateKey'),' ETCD_SERVER_CERTIFICATE=',parameters('etcdServerCertificate'),' ETCD_CLIENT_CERTIFICATE=',parameters('etcdClientCertificate'),' ETCD_SERVER_PRIVATE_KEY=',parameters('etcdServerPrivateKey'),' ETCD_CLIENT_PRIVATE_KEY=',parameters('etcdClientPrivateKey'),' ETCD_PEER_CERTIFICATES=',string(variables('etcdPeerCertificates')),' ETCD_PEER_PRIVATE_KEYS=',string(variables('etcdPeerPrivateKeys')),' ENABLE_AGGREGATED_APIS=',string(parameters('enableAggregatedAPIs')),' KUBECONFIG_SERVER=',variables('kubeconfigServer'))]", {{end}} diff --git a/parts/k8s/kubernetesmastervarsvmss.t b/parts/k8s/kubernetesmastervarsvmss.t index 9a1468338..f871203e9 100644 --- a/parts/k8s/kubernetesmastervarsvmss.t +++ b/parts/k8s/kubernetesmastervarsvmss.t @@ -126,7 +126,7 @@ "customSearchDomainsScript": "{{GetKubernetesB64CustomSearchDomainsScript}}", "sshdConfig": "{{GetB64sshdConfig}}", "systemConf": "{{GetB64systemConf}}", - "provisionScriptParametersCommon": "[concat('ADMINUSER=',parameters('linuxAdminUsername'),' ETCD_DOWNLOAD_URL=',parameters('etcdDownloadURLBase'),' ETCD_VERSION=',parameters('etcdVersion'),' DOCKER_ENGINE_VERSION=',parameters('dockerEngineVersion'),' DOCKER_REPO=',parameters('dockerEngineDownloadRepo'),' TENANT_ID=',variables('tenantID'),' KUBERNETES_VERSION={{.OrchestratorProfile.OrchestratorVersion}} HYPERKUBE_URL=',parameters('kubernetesHyperkubeSpec'),' APISERVER_PUBLIC_KEY=',parameters('apiserverCertificate'),' SUBSCRIPTION_ID=',variables('subscriptionId'),' RESOURCE_GROUP=',variables('resourceGroup'),' LOCATION=',variables('location'),' VM_TYPE=',variables('vmType'),' SUBNET=',variables('subnetName'),' NETWORK_SECURITY_GROUP=',variables('nsgName'),' VIRTUAL_NETWORK=',variables('virtualNetworkName'),' VIRTUAL_NETWORK_RESOURCE_GROUP=',variables('virtualNetworkResourceGroupName'),' ROUTE_TABLE=',variables('routeTableName'),' PRIMARY_AVAILABILITY_SET=',variables('primaryAvailabilitySetName'),' PRIMARY_SCALE_SET=',variables('primaryScaleSetName'),' SERVICE_PRINCIPAL_CLIENT_ID=',variables('servicePrincipalClientId'),' SERVICE_PRINCIPAL_CLIENT_SECRET=',variables('singleQuote'),variables('servicePrincipalClientSecret'),variables('singleQuote'),' KUBELET_PRIVATE_KEY=',parameters('clientPrivateKey'),' TARGET_ENVIRONMENT=',parameters('targetEnvironment'),' NETWORK_PLUGIN=',parameters('networkPlugin'),' VNET_CNI_PLUGINS_URL=',parameters('vnetCniLinuxPluginsURL'),' CNI_PLUGINS_URL=',parameters('cniPluginsURL'),' CLOUDPROVIDER_BACKOFF=',toLower(string(parameters('cloudproviderConfig').cloudProviderBackoff)),' CLOUDPROVIDER_BACKOFF_RETRIES=',parameters('cloudproviderConfig').cloudProviderBackoffRetries,' CLOUDPROVIDER_BACKOFF_EXPONENT=',parameters('cloudproviderConfig').cloudProviderBackoffExponent,' CLOUDPROVIDER_BACKOFF_DURATION=',parameters('cloudproviderConfig').cloudProviderBackoffDuration,' CLOUDPROVIDER_BACKOFF_JITTER=',parameters('cloudproviderConfig').cloudProviderBackoffJitter,' CLOUDPROVIDER_RATELIMIT=',toLower(string(parameters('cloudproviderConfig').cloudProviderRatelimit)),' CLOUDPROVIDER_RATELIMIT_QPS=',parameters('cloudproviderConfig').cloudProviderRatelimitQPS,' CLOUDPROVIDER_RATELIMIT_BUCKET=',parameters('cloudproviderConfig').cloudProviderRatelimitBucket,' USE_MANAGED_IDENTITY_EXTENSION=',variables('useManagedIdentityExtension'),' USER_ASSIGNED_IDENTITY_ID=',variables('userAssignedClientID'),' USE_INSTANCE_METADATA=',variables('useInstanceMetadata'),' LOAD_BALANCER_SKU=',variables('loadBalancerSku'),' EXCLUDE_MASTER_FROM_STANDARD_LB=',variables('excludeMasterFromStandardLB'),' CONTAINER_RUNTIME=',parameters('containerRuntime'),' CONTAINERD_DOWNLOAD_URL_BASE=',parameters('containerdDownloadURLBase'),' POD_INFRA_CONTAINER_SPEC=',parameters('kubernetesPodInfraContainerSpec'),' KMS_PROVIDER_VAULT_NAME=',variables('clusterKeyVaultName'))]", + "provisionScriptParametersCommon": "[concat('ADMINUSER=',parameters('linuxAdminUsername'),' ETCD_DOWNLOAD_URL=',parameters('etcdDownloadURLBase'),' ETCD_VERSION=',parameters('etcdVersion'),' TENANT_ID=',variables('tenantID'),' KUBERNETES_VERSION={{.OrchestratorProfile.OrchestratorVersion}} HYPERKUBE_URL=',parameters('kubernetesHyperkubeSpec'),' APISERVER_PUBLIC_KEY=',parameters('apiserverCertificate'),' SUBSCRIPTION_ID=',variables('subscriptionId'),' RESOURCE_GROUP=',variables('resourceGroup'),' LOCATION=',variables('location'),' VM_TYPE=',variables('vmType'),' SUBNET=',variables('subnetName'),' NETWORK_SECURITY_GROUP=',variables('nsgName'),' VIRTUAL_NETWORK=',variables('virtualNetworkName'),' VIRTUAL_NETWORK_RESOURCE_GROUP=',variables('virtualNetworkResourceGroupName'),' ROUTE_TABLE=',variables('routeTableName'),' PRIMARY_AVAILABILITY_SET=',variables('primaryAvailabilitySetName'),' PRIMARY_SCALE_SET=',variables('primaryScaleSetName'),' SERVICE_PRINCIPAL_CLIENT_ID=',variables('servicePrincipalClientId'),' SERVICE_PRINCIPAL_CLIENT_SECRET=',variables('singleQuote'),variables('servicePrincipalClientSecret'),variables('singleQuote'),' KUBELET_PRIVATE_KEY=',parameters('clientPrivateKey'),' TARGET_ENVIRONMENT=',parameters('targetEnvironment'),' NETWORK_PLUGIN=',parameters('networkPlugin'),' VNET_CNI_PLUGINS_URL=',parameters('vnetCniLinuxPluginsURL'),' CNI_PLUGINS_URL=',parameters('cniPluginsURL'),' CLOUDPROVIDER_BACKOFF=',toLower(string(parameters('cloudproviderConfig').cloudProviderBackoff)),' CLOUDPROVIDER_BACKOFF_RETRIES=',parameters('cloudproviderConfig').cloudProviderBackoffRetries,' CLOUDPROVIDER_BACKOFF_EXPONENT=',parameters('cloudproviderConfig').cloudProviderBackoffExponent,' CLOUDPROVIDER_BACKOFF_DURATION=',parameters('cloudproviderConfig').cloudProviderBackoffDuration,' CLOUDPROVIDER_BACKOFF_JITTER=',parameters('cloudproviderConfig').cloudProviderBackoffJitter,' CLOUDPROVIDER_RATELIMIT=',toLower(string(parameters('cloudproviderConfig').cloudProviderRatelimit)),' CLOUDPROVIDER_RATELIMIT_QPS=',parameters('cloudproviderConfig').cloudProviderRatelimitQPS,' CLOUDPROVIDER_RATELIMIT_BUCKET=',parameters('cloudproviderConfig').cloudProviderRatelimitBucket,' USE_MANAGED_IDENTITY_EXTENSION=',variables('useManagedIdentityExtension'),' USER_ASSIGNED_IDENTITY_ID=',variables('userAssignedClientID'),' USE_INSTANCE_METADATA=',variables('useInstanceMetadata'),' LOAD_BALANCER_SKU=',variables('loadBalancerSku'),' EXCLUDE_MASTER_FROM_STANDARD_LB=',variables('excludeMasterFromStandardLB'),' CONTAINER_RUNTIME=',parameters('containerRuntime'),' CONTAINERD_DOWNLOAD_URL_BASE=',parameters('containerdDownloadURLBase'),' POD_INFRA_CONTAINER_SPEC=',parameters('kubernetesPodInfraContainerSpec'),' KMS_PROVIDER_VAULT_NAME=',variables('clusterKeyVaultName'))]", {{if not IsHostedMaster}} "provisionScriptParametersMaster": "[concat('MASTER_NODE=true CLUSTER_AUTOSCALER_ADDON=',parameters('kubernetesClusterAutoscalerEnabled'),' ACI_CONNECTOR_ADDON=',parameters('kubernetesACIConnectorEnabled'),' APISERVER_PRIVATE_KEY=',parameters('apiServerPrivateKey'),' CA_CERTIFICATE=',parameters('caCertificate'),' CA_PRIVATE_KEY=',parameters('caPrivateKey'),' MASTER_FQDN=',variables('masterFqdnPrefix'),' KUBECONFIG_CERTIFICATE=',parameters('kubeConfigCertificate'),' KUBECONFIG_KEY=',parameters('kubeConfigPrivateKey'),' ETCD_SERVER_CERTIFICATE=',parameters('etcdServerCertificate'),' ETCD_CLIENT_CERTIFICATE=',parameters('etcdClientCertificate'),' ETCD_SERVER_PRIVATE_KEY=',parameters('etcdServerPrivateKey'),' ETCD_CLIENT_PRIVATE_KEY=',parameters('etcdClientPrivateKey'),' ETCD_PEER_CERTIFICATES=',string(variables('etcdPeerCertificates')),' ETCD_PEER_PRIVATE_KEYS=',string(variables('etcdPeerPrivateKeys')),' ENABLE_AGGREGATED_APIS=',string(parameters('enableAggregatedAPIs')),' KUBECONFIG_SERVER=',variables('kubeconfigServer'))]", {{end}} diff --git a/parts/k8s/kubernetesparams.t b/parts/k8s/kubernetesparams.t index 0c414040a..a85971671 100644 --- a/parts/k8s/kubernetesparams.t +++ b/parts/k8s/kubernetesparams.t @@ -648,29 +648,6 @@ "type": "string" }, {{end}} - {{if not IsOpenShift}} - "dockerEngineDownloadRepo": { - "defaultValue": "https://aptdocker.azureedge.net/repo", - "metadata": { - "description": "The docker engine download url for kubernetes." - }, - "type": "string" - }, - "dockerEngineVersion": { - "metadata": { - "description": "The docker engine version to install." - }, - "allowedValues": [ - "17.05.*", - "17.04.*", - "17.03.*", - "1.13.*", - "1.12.*", - "1.11.*" - ], - "type": "string" - }, - {{end}} "networkPolicy": { "defaultValue": "{{.OrchestratorProfile.KubernetesConfig.NetworkPolicy}}", "metadata": { diff --git a/parts/k8s/kubernetesprovisionsource.sh b/parts/k8s/kubernetesprovisionsource.sh index 2fd23f8b7..fb7a370f3 100755 --- a/parts/k8s/kubernetesprovisionsource.sh +++ b/parts/k8s/kubernetesprovisionsource.sh @@ -18,6 +18,9 @@ ERR_DOCKER_DOWNLOAD_TIMEOUT=21 # Timout waiting for docker download(s) ERR_DOCKER_KEY_DOWNLOAD_TIMEOUT=22 # Timeout waiting to download docker repo key ERR_DOCKER_APT_KEY_TIMEOUT=23 # Timeout waiting for docker apt-key ERR_DOCKER_START_FAIL=24 # Docker could not be started by systemctl +ERR_MOBY_APT_LIST_TIMEOUT=25 # Timeout waiting for moby apt sources +ERR_MS_GPG_KEY_DOWNLOAD_TIMEOUT=26 # Timeout waiting for MS GPG key download +ERR_MOBY_INSTALL_TIMEOUT=27 # Timeout waiting for moby install ERR_K8S_RUNNING_TIMEOUT=30 # Timeout waiting for k8s cluster to be healthy ERR_K8S_DOWNLOAD_TIMEOUT=31 # Timeout waiting for Kubernetes download(s) ERR_KUBECTL_NOT_FOUND=32 # kubectl client binary not found on local disk diff --git a/pkg/acsengine/params_k8s.go b/pkg/acsengine/params_k8s.go index 3a6888f81..300032544 100644 --- a/pkg/acsengine/params_k8s.go +++ b/pkg/acsengine/params_k8s.go @@ -21,9 +21,6 @@ func assignKubernetesParameters(properties *api.Properties, parametersMap params if orchestratorProfile.IsKubernetes() || orchestratorProfile.IsOpenShift() { k8sComponents := api.K8sComponentsByVersionMap[orchestratorProfile.OrchestratorVersion] - - dockerEngineVersion := k8sComponents["dockerEngineVersion"] - kubernetesConfig := orchestratorProfile.KubernetesConfig if kubernetesConfig != nil { @@ -301,10 +298,6 @@ func assignKubernetesParameters(properties *api.Properties, parametersMap params addValue(parametersMap, "jumpboxStorageProfile", kubernetesConfig.PrivateCluster.JumpboxProfile.StorageProfile) } - if kubernetesConfig.DockerEngineVersion != "" { - dockerEngineVersion = kubernetesConfig.DockerEngineVersion - } - addValue(parametersMap, "enableAggregatedAPIs", kubernetesConfig.EnableAggregatedAPIs) } @@ -394,11 +387,6 @@ func assignKubernetesParameters(properties *api.Properties, parametersMap params addValue(parametersMap, "kubernetesEndpoint", properties.HostedMasterProfile.FQDN) } - if !orchestratorProfile.IsOpenShift() { - addValue(parametersMap, "dockerEngineDownloadRepo", cloudSpecConfig.DockerSpecConfig.DockerEngineRepo) - addValue(parametersMap, "dockerEngineVersion", dockerEngineVersion) - } - if properties.AADProfile != nil { addValue(parametersMap, "aadTenantId", properties.AADProfile.TenantID) if properties.AADProfile.AdminGroupID != "" { diff --git a/pkg/api/converterfromapi.go b/pkg/api/converterfromapi.go index 9b76b242d..f2f79d20d 100644 --- a/pkg/api/converterfromapi.go +++ b/pkg/api/converterfromapi.go @@ -731,7 +731,6 @@ func convertKubernetesConfigToVLabs(api *KubernetesConfig, vlabs *vlabs.Kubernet vlabs.UserAssignedID = api.UserAssignedID vlabs.UserAssignedClientID = api.UserAssignedClientID vlabs.CustomHyperkubeImage = api.CustomHyperkubeImage - vlabs.DockerEngineVersion = api.DockerEngineVersion vlabs.CustomCcmImage = api.CustomCcmImage vlabs.UseCloudControllerManager = api.UseCloudControllerManager vlabs.CustomWindowsPackageURL = api.CustomWindowsPackageURL diff --git a/pkg/api/convertertoapi.go b/pkg/api/convertertoapi.go index ff7cdcb3a..9a37b8d6e 100644 --- a/pkg/api/convertertoapi.go +++ b/pkg/api/convertertoapi.go @@ -691,7 +691,6 @@ func convertVLabsKubernetesConfig(vlabs *vlabs.KubernetesConfig, api *Kubernetes api.UserAssignedID = vlabs.UserAssignedID api.UserAssignedClientID = vlabs.UserAssignedClientID api.CustomHyperkubeImage = vlabs.CustomHyperkubeImage - api.DockerEngineVersion = vlabs.DockerEngineVersion api.CustomCcmImage = vlabs.CustomCcmImage api.UseCloudControllerManager = vlabs.UseCloudControllerManager api.CustomWindowsPackageURL = vlabs.CustomWindowsPackageURL diff --git a/pkg/api/k8s_versions.go b/pkg/api/k8s_versions.go index 5aa19aa8f..ea47b2b17 100644 --- a/pkg/api/k8s_versions.go +++ b/pkg/api/k8s_versions.go @@ -318,7 +318,6 @@ func getK8sVersionComponents(version string, overrides map[string]string) map[st "hyperkube": "hyperkube-amd64:v" + version, "ccm": "cloud-controller-manager-amd64:v" + version, "windowszip": "v" + version + "-1int.zip", - "dockerEngineVersion": k8sComponentVersions["1.13"]["dockerEngine"], DefaultDashboardAddonName: k8sComponentVersions["1.13"]["dashboard"], "exechealthz": k8sComponentVersions["1.13"]["exechealthz"], "addonresizer": k8sComponentVersions["1.13"]["addon-resizer"], @@ -355,7 +354,6 @@ func getK8sVersionComponents(version string, overrides map[string]string) map[st "hyperkube": "hyperkube-amd64:v" + version, "ccm": "cloud-controller-manager-amd64:v" + version, "windowszip": "v" + version + "-1int.zip", - "dockerEngineVersion": k8sComponentVersions["1.12"]["dockerEngine"], DefaultDashboardAddonName: k8sComponentVersions["1.12"]["dashboard"], "exechealthz": k8sComponentVersions["1.12"]["exechealthz"], "addonresizer": k8sComponentVersions["1.12"]["addon-resizer"], @@ -392,7 +390,6 @@ func getK8sVersionComponents(version string, overrides map[string]string) map[st "hyperkube": "hyperkube-amd64:v" + version, "ccm": "cloud-controller-manager-amd64:v" + version, "windowszip": "v" + version + "-1int.zip", - "dockerEngineVersion": k8sComponentVersions["1.11"]["dockerEngine"], DefaultDashboardAddonName: k8sComponentVersions["1.11"]["dashboard"], "exechealthz": k8sComponentVersions["1.11"]["exechealthz"], "addonresizer": k8sComponentVersions["1.11"]["addon-resizer"], @@ -428,7 +425,6 @@ func getK8sVersionComponents(version string, overrides map[string]string) map[st "hyperkube": "hyperkube-amd64:v" + version, "ccm": "cloud-controller-manager-amd64:v" + version, "windowszip": "v" + version + "-1int.zip", - "dockerEngineVersion": k8sComponentVersions["1.10"]["dockerEngine"], DefaultDashboardAddonName: k8sComponentVersions["1.10"]["dashboard"], "exechealthz": k8sComponentVersions["1.10"]["exechealthz"], "addonresizer": k8sComponentVersions["1.10"]["addon-resizer"], @@ -464,7 +460,6 @@ func getK8sVersionComponents(version string, overrides map[string]string) map[st "hyperkube": "hyperkube-amd64:v" + version, "ccm": "cloud-controller-manager-amd64:v" + version, "windowszip": "v" + version + "-1int.zip", - "dockerEngineVersion": k8sComponentVersions["1.9"]["dockerEngine"], DefaultDashboardAddonName: k8sComponentVersions["1.9"]["dashboard"], "exechealthz": k8sComponentVersions["1.9"]["exechealthz"], "addonresizer": k8sComponentVersions["1.9"]["addon-resizer"], @@ -499,7 +494,6 @@ func getK8sVersionComponents(version string, overrides map[string]string) map[st "hyperkube": "hyperkube-amd64:v" + version, "ccm": "cloud-controller-manager-amd64:v" + version, "windowszip": "v" + version + "-1int.zip", - "dockerEngineVersion": k8sComponentVersions["1.8"]["dockerEngine"], DefaultDashboardAddonName: k8sComponentVersions["1.8"]["dashboard"], "exechealthz": k8sComponentVersions["1.8"]["exechealthz"], "addonresizer": k8sComponentVersions["1.8"]["addon-resizer"], @@ -530,7 +524,6 @@ func getK8sVersionComponents(version string, overrides map[string]string) map[st case "1.7": ret = map[string]string{ "hyperkube": "hyperkube-amd64:v" + version, - "dockerEngineVersion": k8sComponentVersions["1.7"]["dockerEngine"], DefaultDashboardAddonName: k8sComponentVersions["1.7"]["dashboard"], "exechealthz": k8sComponentVersions["1.7"]["exechealthz"], "addonresizer": k8sComponentVersions["1.7"]["addon-resizer"], @@ -561,7 +554,6 @@ func getK8sVersionComponents(version string, overrides map[string]string) map[st case "1.6": ret = map[string]string{ "hyperkube": "hyperkube-amd64:v" + version, - "dockerEngineVersion": k8sComponentVersions["1.6"]["dockerEngine"], DefaultDashboardAddonName: k8sComponentVersions["1.6"]["dashboard"], "exechealthz": k8sComponentVersions["1.6"]["exechealthz"], "addonresizer": k8sComponentVersions["1.6"]["addon-resizer"], diff --git a/pkg/api/k8s_versions_test.go b/pkg/api/k8s_versions_test.go index f1ea12e7d..a22dd60c1 100644 --- a/pkg/api/k8s_versions_test.go +++ b/pkg/api/k8s_versions_test.go @@ -14,7 +14,6 @@ func TestGetK8sVersionComponents(t *testing.T) { "hyperkube": "hyperkube-amd64:v1.12.0", "ccm": "cloud-controller-manager-amd64:v1.12.0", "windowszip": "v1.12.0-1int.zip", - "dockerEngineVersion": k8sComponentVersions["1.12"]["dockerEngine"], DefaultDashboardAddonName: k8sComponentVersions["1.12"]["dashboard"], "exechealthz": k8sComponentVersions["1.12"]["exechealthz"], "addonresizer": k8sComponentVersions["1.12"]["addon-resizer"], @@ -61,7 +60,6 @@ func TestGetK8sVersionComponents(t *testing.T) { "hyperkube": "hyperkube-amd64:v1.11.0-alpha.1", "ccm": "cloud-controller-manager-amd64:v1.11.0-alpha.1", "windowszip": "v1.11.0-alpha.1-1int.zip", - "dockerEngineVersion": k8sComponentVersions["1.11"]["dockerEngine"], DefaultDashboardAddonName: k8sComponentVersions["1.11"]["dashboard"], "exechealthz": k8sComponentVersions["1.11"]["exechealthz"], "addonresizer": k8sComponentVersions["1.11"]["addon-resizer"], @@ -107,7 +105,6 @@ func TestGetK8sVersionComponents(t *testing.T) { "hyperkube": "hyperkube-amd64:v1.10.0", "ccm": "cloud-controller-manager-amd64:v1.10.0", "windowszip": "v1.10.0-1int.zip", - "dockerEngineVersion": k8sComponentVersions["1.10"]["dockerEngine"], DefaultDashboardAddonName: k8sComponentVersions["1.10"]["dashboard"], "exechealthz": k8sComponentVersions["1.10"]["exechealthz"], "addonresizer": k8sComponentVersions["1.10"]["addon-resizer"], @@ -153,7 +150,6 @@ func TestGetK8sVersionComponents(t *testing.T) { "hyperkube": "hyperkube-amd64:v1.9.3", "ccm": "cloud-controller-manager-amd64:v1.9.3", "windowszip": "v1.9.3-1int.zip", - "dockerEngineVersion": k8sComponentVersions["1.9"]["dockerEngine"], DefaultDashboardAddonName: k8sComponentVersions["1.9"]["dashboard"], "exechealthz": k8sComponentVersions["1.9"]["exechealthz"], "addonresizer": k8sComponentVersions["1.9"]["addon-resizer"], @@ -198,7 +194,6 @@ func TestGetK8sVersionComponents(t *testing.T) { "hyperkube": "hyperkube-amd64:v1.8.8", "ccm": "cloud-controller-manager-amd64:v1.8.8", "windowszip": "v1.8.8-1int.zip", - "dockerEngineVersion": k8sComponentVersions["1.8"]["dockerEngine"], DefaultDashboardAddonName: k8sComponentVersions["1.8"]["dashboard"], "exechealthz": k8sComponentVersions["1.8"]["exechealthz"], "addonresizer": k8sComponentVersions["1.8"]["addon-resizer"], @@ -238,7 +233,6 @@ func TestGetK8sVersionComponents(t *testing.T) { } expected = map[string]string{ "hyperkube": "hyperkube-amd64:v1.7.13", - "dockerEngineVersion": k8sComponentVersions["1.7"]["dockerEngine"], DefaultDashboardAddonName: k8sComponentVersions["1.7"]["dashboard"], "exechealthz": k8sComponentVersions["1.7"]["exechealthz"], "addonresizer": k8sComponentVersions["1.7"]["addon-resizer"], @@ -272,7 +266,7 @@ func TestGetK8sVersionComponents(t *testing.T) { } } - override := getK8sVersionComponents("1.9.3", map[string]string{"windowszip": "v1.9.3-2int.zip", "dockerEngineVersion": "1.99.*"}) + override := getK8sVersionComponents("1.9.3", map[string]string{"windowszip": "v1.9.3-2int.zip"}) if override == nil { t.Fatalf("getK8sVersionComponents() should not return nil for valid version") } @@ -280,7 +274,6 @@ func TestGetK8sVersionComponents(t *testing.T) { "hyperkube": "hyperkube-amd64:v1.9.3", "ccm": "cloud-controller-manager-amd64:v1.9.3", "windowszip": "v1.9.3-2int.zip", - "dockerEngineVersion": "1.99.*", DefaultDashboardAddonName: k8sComponentVersions["1.9"]["dashboard"], "exechealthz": k8sComponentVersions["1.9"]["exechealthz"], "addonresizer": k8sComponentVersions["1.9"]["addon-resizer"], diff --git a/pkg/api/types.go b/pkg/api/types.go index fe2202066..e4bd323f9 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -320,8 +320,8 @@ type KubernetesConfig struct { UserAssignedID string `json:"userAssignedID,omitempty"` UserAssignedClientID string `json:"userAssignedClientID,omitempty"` //Note: cannot be provided in config. Used *only* for transferring this to azure.json. CustomHyperkubeImage string `json:"customHyperkubeImage,omitempty"` - DockerEngineVersion string `json:"dockerEngineVersion,omitempty"` - CustomCcmImage string `json:"customCcmImage,omitempty"` // Image for cloud-controller-manager + DockerEngineVersion string `json:"dockerEngineVersion,omitempty"` // Deprecated + CustomCcmImage string `json:"customCcmImage,omitempty"` // Image for cloud-controller-manager UseCloudControllerManager *bool `json:"useCloudControllerManager,omitempty"` CustomWindowsPackageURL string `json:"customWindowsPackageURL,omitempty"` UseInstanceMetadata *bool `json:"useInstanceMetadata,omitempty"` @@ -1292,7 +1292,7 @@ func (k *KubernetesConfig) PrivateJumpboxProvision() bool { return false } -// RequiresDocker returns if the kubernetes settings require docker to be installed. +// RequiresDocker returns if the kubernetes settings require docker binary to be installed. func (k *KubernetesConfig) RequiresDocker() bool { runtime := strings.ToLower(k.ContainerRuntime) return runtime == "docker" || runtime == "" diff --git a/pkg/api/vlabs/types.go b/pkg/api/vlabs/types.go index 7a6e24b86..7189aa25f 100644 --- a/pkg/api/vlabs/types.go +++ b/pkg/api/vlabs/types.go @@ -277,7 +277,7 @@ type KubernetesConfig struct { UserAssignedID string `json:"userAssignedID,omitempty"` UserAssignedClientID string `json:"userAssignedClientID,omitempty"` //Note: cannot be provided in config. Used *only* for transferring this to azure.json. CustomHyperkubeImage string `json:"customHyperkubeImage,omitempty"` - DockerEngineVersion string `json:"dockerEngineVersion,omitempty"` + DockerEngineVersion string `json:"dockerEngineVersion,omitempty"` // Deprecated CustomCcmImage string `json:"customCcmImage,omitempty"` UseCloudControllerManager *bool `json:"useCloudControllerManager,omitempty"` CustomWindowsPackageURL string `json:"customWindowsPackageURL,omitempty"` @@ -693,7 +693,7 @@ func (o *OrchestratorProfile) IsSwarmMode() bool { return o.OrchestratorType == SwarmMode } -// RequiresDocker returns if the kubernetes settings require docker to be installed. +// RequiresDocker returns if the kubernetes settings require docker binary to be installed. func (k *KubernetesConfig) RequiresDocker() bool { runtime := strings.ToLower(k.ContainerRuntime) return runtime == "docker" || runtime == "" diff --git a/pkg/api/vlabs/validate.go b/pkg/api/vlabs/validate.go index 462ebdc07..15c60943e 100644 --- a/pkg/api/vlabs/validate.go +++ b/pkg/api/vlabs/validate.go @@ -281,6 +281,10 @@ func (a *Properties) validateOrchestratorProfile(isUpdate bool) error { return errors.Errorf("standard loadBalancerSku should exclude master nodes. Please set KubernetesConfig \"ExcludeMasterFromStandardLB\" to \"true\"") } } + + if o.KubernetesConfig.DockerEngineVersion != "" { + log.Warnf("docker-engine is deprecated in favor of moby, but you passed in a dockerEngineVersion configuration. This will be ignored.") + } } case OpenShift: // TODO: add appropriate additional validation logic @@ -1250,7 +1254,7 @@ func (a *Properties) validateContainerRuntime() error { return errors.Errorf("unknown containerRuntime %q specified", containerRuntime) } - // Make sure we don't use clear containers on windows. + // Make sure we don't use unsupported container runtimes on windows. if (containerRuntime == "clear-containers" || containerRuntime == "kata-containers" || containerRuntime == "containerd") && a.HasWindows() { return errors.Errorf("containerRuntime %q is not supporting windows agents", containerRuntime) }