diff --git a/.gitignore b/.gitignore index 8400a8714..c704eca75 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,7 @@ pkg/operations/junit.xml pkg/operations/kubernetesupgrade/junit.xml pkg/acsengine/templates.go pkg/i18n/translations.go +pkg/openshift/certgen/templates/bindata.go _logs/ test/acs-engine-test/report/TestReport.json diff --git a/cmd/root.go b/cmd/root.go index ab7f90a2c..c93b3d23c 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -13,7 +13,7 @@ import ( const ( rootName = "acs-engine" rootShortDescription = "ACS-Engine deploys and manages container orchestrators in Azure" - rootLongDescription = "ACS-Engine deploys and manages Kubernetes, Swarm Mode, and DC/OS clusters in Azure" + rootLongDescription = "ACS-Engine deploys and manages Kubernetes, OpenShift, Swarm Mode, and DC/OS clusters in Azure" ) var ( diff --git a/examples/openshift.json b/examples/openshift.json new file mode 100644 index 000000000..fd724726a --- /dev/null +++ b/examples/openshift.json @@ -0,0 +1,67 @@ +{ + "apiVersion": "vlabs", + "properties": { + "orchestratorProfile": { + "orchestratorType": "OpenShift", + "openShiftConfig": { + "clusterUsername": "demo", + "clusterPassword": "demo" + } + }, + "azProfile": { + "tenantId": "YOUR TENANT ID", + "subscriptionId": "YOUR SUBSCRIPTION ID", + "resourceGroup": "YOUR RESOURCE GROUP", + "location": "YOUR LOCATION" + }, + "masterProfile": { + "count": 1, + "dnsPrefix": "YOUR DNS PREFIX", + "imageReference": { + "name": "YOUR IMAGE", + "resourceGroup": "YOUR IMAGE RESOURCE GROUP" + }, + "storageProfile": "ManagedDisks", + "vmSize": "Standard_D4s_v3" + }, + "agentPoolProfiles": [ + { + "availabilityProfile": "AvailabilitySet", + "count": 1, + "imageReference": { + "name": "YOUR IMAGE", + "resourceGroup": "YOUR IMAGE RESOURCE GROUP" + }, + "name": "compute", + "storageProfile": "ManagedDisks", + "vmSize": "Standard_D4s_v3" + }, + { + "availabilityProfile": "AvailabilitySet", + "count": 1, + "imageReference": { + "name": "YOUR IMAGE", + "resourceGroup": "YOUR IMAGE RESOURCE GROUP" + }, + "role": "infra", + "name": "infra", + "storageProfile": "ManagedDisks", + "vmSize": "Standard_D4s_v3" + } + ], + "linuxProfile": { + "adminUsername": "cloud-user", + "ssh": { + "publicKeys": [ + { + "keyData": "YOUR KEY" + } + ] + } + }, + "servicePrincipalProfile": { + "clientId": "YOUR CLIENTID", + "secret": "YOUR SECRET" + } + } +} diff --git a/parts/agentoutputs.t b/parts/agentoutputs.t index 9b1d6c593..f809141e2 100644 --- a/parts/agentoutputs.t +++ b/parts/agentoutputs.t @@ -1,5 +1,5 @@ {{if IsPublic .Ports}} - {{ if not IsKubernetes}} + {{ if and (not IsKubernetes) (not IsOpenShift)}} "{{.Name}}FQDN": { "type": "string", "value": "[reference(concat('Microsoft.Network/publicIPAddresses/', variables('{{.Name}}IPAddressName'))).dnsSettings.fqdn]" diff --git a/parts/k8s/kubernetesagentresourcesvmas.t b/parts/k8s/kubernetesagentresourcesvmas.t index bbe5af29f..928990073 100644 --- a/parts/k8s/kubernetesagentresourcesvmas.t +++ b/parts/k8s/kubernetesagentresourcesvmas.t @@ -31,6 +31,14 @@ "subnet": { "id": "[variables('{{$.Name}}VnetSubnetID')]" } +{{if eq $.Role "infra"}} + , + "loadBalancerBackendAddressPools": [ + { + "id": "[concat(resourceId('Microsoft.Network/loadBalancers', 'router-lb'), '/backendAddressPools/backend')]" + } + ] +{{end}} } } {{if lt $seq $.IPAddressCount}},{{end}} @@ -160,7 +168,9 @@ "osProfile": { "adminUsername": "[variables('username')]", "computername": "[concat(variables('{{.Name}}VMNamePrefix'), copyIndex(variables('{{.Name}}Offset')))]", + {{if not IsOpenShift}} {{GetKubernetesAgentCustomData .}} + {{end}} "linuxConfiguration": { "disablePasswordAuthentication": "true", "ssh": { @@ -270,7 +280,11 @@ "autoUpgradeMinorVersion": true, "settings": {}, "protectedSettings": { + {{if IsOpenShift }} + "script": "{{ Base64 (OpenShiftGetNodeSh .) }}" + {{else}} "commandToExecute": "[concat(variables('provisionScriptParametersCommon'),' /usr/bin/nohup /bin/bash -c \"/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1\"')]" + {{end}} } } } diff --git a/parts/k8s/kubernetesmasterresources.t b/parts/k8s/kubernetesmasterresources.t index 3d9dee815..ce9ee648e 100644 --- a/parts/k8s/kubernetesmasterresources.t +++ b/parts/k8s/kubernetesmasterresources.t @@ -1,3 +1,115 @@ +{{if IsOpenShift}} + { + "name": "router-ip", + "type": "Microsoft.Network/publicIPAddresses", + "apiVersion": "2017-08-01", + "location": "[variables('location')]", + "properties": { + "publicIPAllocationMethod": "Static", + "dnsSettings": { + "domainNameLabel": "[concat(variables('masterFqdnPrefix'), '-router')]" + } + }, + "sku": { + "name": "Basic" + } + }, + { + "name": "router-lb", + "type": "Microsoft.Network/loadBalancers", + "apiVersion": "2017-10-01", + "location": "[variables('location')]", + "dependsOn": [ + "['Microsoft.Network/publicIPAddresses/router-ip']" + ], + "properties": { + "frontendIPConfigurations": [ + { + "name": "frontend", + "properties": { + "privateIPAllocationMethod": "Dynamic", + "publicIPAddress": { + "id": "[resourceId('Microsoft.Network/publicIPAddresses', 'router-ip')]" + } + } + } + ], + "backendAddressPools": [ + { + "name": "backend" + } + ], + "loadBalancingRules": [ + { + "name": "port-80", + "properties": { + "frontendIPConfiguration": { + "id": "[concat(resourceId('Microsoft.Network/loadBalancers', 'router-lb'), '/frontendIPConfigurations/frontend')]" + }, + "frontendPort": 80, + "backendPort": 80, + "enableFloatingIP": false, + "idleTimeoutInMinutes": 4, + "protocol": "Tcp", + "loadDistribution": "Default", + "backendAddressPool": { + "id": "[concat(resourceId('Microsoft.Network/loadBalancers', 'router-lb'), '/backendAddressPools/backend')]" + }, + "probe": { + "id": "[concat(resourceId('Microsoft.Network/loadBalancers', 'router-lb'), '/probes/port-80')]" + } + } + }, + { + "name": "port-443", + "properties": { + "frontendIPConfiguration": { + "id": "[concat(resourceId('Microsoft.Network/loadBalancers', 'router-lb'), '/frontendIPConfigurations/frontend')]" + }, + "frontendPort": 443, + "backendPort": 443, + "enableFloatingIP": false, + "idleTimeoutInMinutes": 4, + "protocol": "Tcp", + "loadDistribution": "Default", + "backendAddressPool": { + "id": "[concat(resourceId('Microsoft.Network/loadBalancers', 'router-lb'), '/backendAddressPools/backend')]" + }, + "probe": { + "id": "[concat(resourceId('Microsoft.Network/loadBalancers', 'router-lb'), '/probes/port-443')]" + } + } + } + ], + "probes": [ + { + "name": "port-80", + "properties": { + "protocol": "Tcp", + "port": 80, + "intervalInSeconds": 5, + "numberOfProbes": 2 + } + }, + { + "name": "port-443", + "properties": { + "protocol": "Tcp", + "port": 443, + "intervalInSeconds": 5, + "numberOfProbes": 2 + } + } + ], + "inboundNatRules": [], + "outboundNatRules": [], + "inboundNatPools": [] + }, + "sku": { + "name": "Basic" + } + }, +{{end}} {{if .MasterProfile.IsManagedDisks}} { "apiVersion": "[variables('apiVersionStorageManagedDisks')]", @@ -94,6 +206,36 @@ "sourcePortRange": "*" } }, +{{end}} +{{if IsOpenShift}} + { + "name": "allow_http", + "properties": { + "access": "Allow", + "description": "Allow http traffic to infra nodes", + "destinationAddressPrefix": "*", + "destinationPortRange": "80", + "direction": "Inbound", + "priority": 110, + "protocol": "Tcp", + "sourceAddressPrefix": "*", + "sourcePortRange": "*" + } + }, + { + "name": "allow_https", + "properties": { + "access": "Allow", + "description": "Allow https traffic to infra nodes", + "destinationAddressPrefix": "*", + "destinationPortRange": "443", + "direction": "Inbound", + "priority": 111, + "protocol": "Tcp", + "sourceAddressPrefix": "*", + "sourcePortRange": "*" + } + }, {{end}} { "name": "allow_ssh", @@ -115,7 +257,7 @@ "access": "Allow", "description": "Allow kube-apiserver (tls) traffic to master", "destinationAddressPrefix": "*", - "destinationPortRange": "443-443", + "destinationPortRange": {{if IsOpenShift}}"8443-8443"{{else}}"443-443"{{end}}, "direction": "Inbound", "priority": 100, "protocol": "Tcp", @@ -182,8 +324,8 @@ "id": "[concat(variables('masterLbID'), '/backendAddressPools/', variables('masterLbBackendPoolName'))]" }, "protocol": "tcp", - "frontendPort": 443, - "backendPort": 443, + "frontendPort": {{if IsOpenShift}}8443{{else}}443{{end}}, + "backendPort": {{if IsOpenShift}}8443{{else}}443{{end}}, "enableFloatingIP": false, "idleTimeoutInMinutes": 5, "loadDistribution": "Default", @@ -198,7 +340,7 @@ "name": "tcpHTTPSProbe", "properties": { "protocol": "tcp", - "port": 443, + "port": {{if IsOpenShift}}8443{{else}}443{{end}}, "intervalInSeconds": "5", "numberOfProbes": "2" } @@ -563,12 +705,12 @@ "backendAddressPool": { "id": "[concat(variables('masterInternalLbID'), '/backendAddressPools/', variables('masterLbBackendPoolName'))]" }, - "backendPort": 4443, + "backendPort": {{if IsOpenShift}}8443{{else}}4443{{end}}, "enableFloatingIP": false, "frontendIPConfiguration": { "id": "[variables('masterInternalLbIPConfigID')]" }, - "frontendPort": 443, + "frontendPort": {{if IsOpenShift}}8443{{else}}443{{end}}, "idleTimeoutInMinutes": 5, "protocol": "tcp" } @@ -580,7 +722,7 @@ "properties": { "intervalInSeconds": "5", "numberOfProbes": "2", - "port": 4443, + "port": {{if IsOpenShift}}8443{{else}}4443{{end}}, "protocol": "tcp" } } @@ -637,7 +779,9 @@ "osProfile": { "adminUsername": "[variables('username')]", "computername": "[concat(variables('masterVMNamePrefix'), copyIndex(variables('masterOffset')))]", + {{if IsKubernetes}} {{GetKubernetesMasterCustomData .}} + {{end}} "linuxConfiguration": { "disablePasswordAuthentication": true, "ssh": { @@ -760,7 +904,11 @@ "autoUpgradeMinorVersion": true, "settings": {}, "protectedSettings": { + {{if IsOpenShift}} + "script": "{{ Base64 OpenShiftGetMasterSh }}" + {{else}} "commandToExecute": "[concat(variables('provisionScriptParametersCommon'),' ',variables('provisionScriptParametersMaster'), ' MASTER_INDEX=',copyIndex(variables('masterOffset')),' /usr/bin/nohup /bin/bash -c \"stat /opt/azure/containers/provision.complete > /dev/null 2>&1 || /bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1\"')]" + {{end}} } } }{{WriteLinkedTemplatesForExtensions}} diff --git a/parts/k8s/kubernetesmastervars.t b/parts/k8s/kubernetesmastervars.t index 812eb0af6..0ce292220 100644 --- a/parts/k8s/kubernetesmastervars.t +++ b/parts/k8s/kubernetesmastervars.t @@ -2,8 +2,11 @@ "etcdDownloadURLBase": "[parameters('etcdDownloadURLBase')]", "etcdVersion": "[parameters('etcdVersion')]", "maxVMsPerPool": 100, +{{ if not IsOpenShift }} "apiServerCertificate": "[parameters('apiServerCertificate')]", +{{ end }} {{ if not IsHostedMaster }} +{{ if not IsOpenShift }} "apiServerPrivateKey": "[parameters('apiServerPrivateKey')]", "etcdServerCertificate": "[parameters('etcdServerCertificate')]", "etcdServerPrivateKey": "[parameters('etcdServerPrivateKey')]", @@ -65,12 +68,15 @@ "etcdServerCertFilepath": "/etc/kubernetes/certs/etcdserver.crt", "etcdServerKeyFilepath": "/etc/kubernetes/certs/etcdserver.key", {{end}} +{{end}} +{{ if not IsOpenShift }} "caCertificate": "[parameters('caCertificate')]", "caPrivateKey": "[parameters('caPrivateKey')]", "clientCertificate": "[parameters('clientCertificate')]", "clientPrivateKey": "[parameters('clientPrivateKey')]", "kubeConfigCertificate": "[parameters('kubeConfigCertificate')]", "kubeConfigPrivateKey": "[parameters('kubeConfigPrivateKey')]", +{{end}} "kubernetesHyperkubeSpec": "[parameters('kubernetesHyperkubeSpec')]", "kubernetesCcmImageSpec": "[parameters('kubernetesCcmImageSpec')]", "kubernetesAddonManagerSpec": "[parameters('kubernetesAddonManagerSpec')]", @@ -213,11 +219,13 @@ "provisionScript": "{{GetKubernetesB64Provision}}", "provisionSource": "{{GetKubernetesB64ProvisionSource}}", "mountetcdScript": "{{GetKubernetesB64Mountetcd}}", +{{if not IsOpenShift}} {{if not IsHostedMaster}} "provisionScriptParametersMaster": "[concat('MASTER_NODE=true APISERVER_PRIVATE_KEY=',variables('apiServerPrivateKey'),' CA_CERTIFICATE=',variables('caCertificate'),' CA_PRIVATE_KEY=',variables('caPrivateKey'),' MASTER_FQDN=',variables('masterFqdnPrefix'),' KUBECONFIG_CERTIFICATE=',variables('kubeConfigCertificate'),' KUBECONFIG_KEY=',variables('kubeConfigPrivateKey'),' ETCD_SERVER_CERTIFICATE=',variables('etcdServerCertificate'),' ETCD_CLIENT_CERTIFICATE=',variables('etcdClientCertificate'),' ETCD_SERVER_PRIVATE_KEY=',variables('etcdServerPrivateKey'),' ETCD_CLIENT_PRIVATE_KEY=',variables('etcdClientPrivateKey'),' ETCD_PEER_CERTIFICATES=',string(variables('etcdPeerCertificates')),' ETCD_PEER_PRIVATE_KEYS=',string(variables('etcdPeerPrivateKeys')),' ADMINUSER=',variables('username'))]", "provisionScriptParametersCommon": "[concat('TENANT_ID=',variables('tenantID'),' HYPERKUBE_URL=',variables('kubernetesHyperkubeSpec'),' APISERVER_PUBLIC_KEY=',variables('apiserverCertificate'),' SUBSCRIPTION_ID=',variables('subscriptionId'),' RESOURCE_GROUP=',variables('resourceGroup'),' LOCATION=',variables('location'),' SUBNET=',variables('subnetName'),' NETWORK_SECURITY_GROUP=',variables('nsgName'),' VIRTUAL_NETWORK=',variables('virtualNetworkName'),' VIRTUAL_NETWORK_RESOURCE_GROUP=',variables('virtualNetworkResourceGroupName'),' ROUTE_TABLE=',variables('routeTableName'),' PRIMARY_AVAILABILITY_SET=',variables('primaryAvailabilitySetName'),' SERVICE_PRINCIPAL_CLIENT_ID=',variables('servicePrincipalClientId'),' SERVICE_PRINCIPAL_CLIENT_SECRET=',variables('singleQuote'),variables('servicePrincipalClientSecret'),variables('singleQuote'),' KUBELET_PRIVATE_KEY=',variables('clientPrivateKey'),' TARGET_ENVIRONMENT=',variables('targetEnvironment'),' NETWORK_POLICY=',variables('networkPolicy'),' FQDNSuffix=',variables('fqdnEndpointSuffix'),' VNET_CNI_PLUGINS_URL=',variables('vnetCniLinuxPluginsURL'),' CNI_PLUGINS_URL=',variables('cniPluginsURL'),' CLOUDPROVIDER_BACKOFF=',variables('cloudProviderBackoff'),' CLOUDPROVIDER_BACKOFF_RETRIES=',variables('cloudProviderBackoffRetries'),' CLOUDPROVIDER_BACKOFF_EXPONENT=',variables('cloudProviderBackoffExponent'),' CLOUDPROVIDER_BACKOFF_DURATION=',variables('cloudProviderBackoffDuration'),' CLOUDPROVIDER_BACKOFF_JITTER=',variables('cloudProviderBackoffJitter'),' CLOUDPROVIDER_RATELIMIT=',variables('cloudProviderRatelimit'),' CLOUDPROVIDER_RATELIMIT_QPS=',variables('cloudProviderRatelimitQPS'),' CLOUDPROVIDER_RATELIMIT_BUCKET=',variables('cloudProviderRatelimitBucket'),' USE_MANAGED_IDENTITY_EXTENSION=',variables('useManagedIdentityExtension'),' USE_INSTANCE_METADATA=',variables('useInstanceMetadata'),' CONTAINER_RUNTIME=',variables('containerRuntime'),' KUBECONFIG_SERVER=',variables('kubeconfigServer'))]", {{else}} "provisionScriptParametersCommon": "[concat('TENANT_ID=',variables('tenantID'),' HYPERKUBE_URL=',variables('kubernetesHyperkubeSpec'),' APISERVER_PUBLIC_KEY=',variables('apiserverCertificate'),' SUBSCRIPTION_ID=',variables('subscriptionId'),' RESOURCE_GROUP=',variables('resourceGroup'),' LOCATION=',variables('location'),' SUBNET=',variables('subnetName'),' NETWORK_SECURITY_GROUP=',variables('nsgName'),' VIRTUAL_NETWORK=',variables('virtualNetworkName'),' VIRTUAL_NETWORK_RESOURCE_GROUP=',variables('virtualNetworkResourceGroupName'),' ROUTE_TABLE=',variables('routeTableName'),' PRIMARY_AVAILABILITY_SET=',variables('primaryAvailabilitySetName'),' SERVICE_PRINCIPAL_CLIENT_ID=',variables('servicePrincipalClientId'),' SERVICE_PRINCIPAL_CLIENT_SECRET=',variables('singleQuote'),variables('servicePrincipalClientSecret'),variables('singleQuote'),' KUBELET_PRIVATE_KEY=',variables('clientPrivateKey'),' TARGET_ENVIRONMENT=',variables('targetEnvironment'),' NETWORK_POLICY=',variables('networkPolicy'),' FQDNSuffix=',variables('fqdnEndpointSuffix'),' VNET_CNI_PLUGINS_URL=',variables('vnetCniLinuxPluginsURL'),' CNI_PLUGINS_URL=',variables('cniPluginsURL'),' CLOUDPROVIDER_BACKOFF=',variables('cloudProviderBackoff'),' CLOUDPROVIDER_BACKOFF_RETRIES=',variables('cloudProviderBackoffRetries'),' CLOUDPROVIDER_BACKOFF_EXPONENT=',variables('cloudProviderBackoffExponent'),' CLOUDPROVIDER_BACKOFF_DURATION=',variables('cloudProviderBackoffDuration'),' CLOUDPROVIDER_BACKOFF_JITTER=',variables('cloudProviderBackoffJitter'),' CLOUDPROVIDER_RATELIMIT=',variables('cloudProviderRatelimit'),' CLOUDPROVIDER_RATELIMIT_QPS=',variables('cloudProviderRatelimitQPS'),' CLOUDPROVIDER_RATELIMIT_BUCKET=',variables('cloudProviderRatelimitBucket'),' USE_MANAGED_IDENTITY_EXTENSION=',variables('useManagedIdentityExtension'),' USE_INSTANCE_METADATA=',variables('useInstanceMetadata'),' CONTAINER_RUNTIME=',variables('containerRuntime'))]", +{{end}} {{end}} "generateProxyCertsScript": "{{GetKubernetesB64GenerateProxyCerts}}", "orchestratorNameVersionTag": "{{.OrchestratorProfile.OrchestratorType}}:{{.OrchestratorProfile.OrchestratorVersion}}", @@ -364,9 +372,11 @@ "scope": "[resourceGroup().id]", "tenantId": "[subscription().tenantId]", "singleQuote": "'", - "targetEnvironment": "[parameters('targetEnvironment')]", - "dockerEngineDownloadRepo": "[parameters('dockerEngineDownloadRepo')]", - "dockerEngineVersion": "[parameters('dockerEngineVersion')]" + "targetEnvironment": "[parameters('targetEnvironment')]" +{{if not IsOpenShift}} + , "dockerEngineDownloadRepo": "[parameters('dockerEngineDownloadRepo')]" + , "dockerEngineVersion": "[parameters('dockerEngineVersion')]" +{{end}} {{if .LinuxProfile.HasSecrets}} , "linuxProfileSecrets" : [ diff --git a/parts/k8s/kubernetesparams.t b/parts/k8s/kubernetesparams.t index 1b220e968..a2447fc72 100644 --- a/parts/k8s/kubernetesparams.t +++ b/parts/k8s/kubernetesparams.t @@ -22,6 +22,7 @@ "type": "string" }, {{else}} +{{if not IsOpenShift}} "etcdServerCertificate": { "metadata": { "description": "The base 64 server certificate used on the master" @@ -111,6 +112,8 @@ {{end}} {{end}} {{end}} +{{end}} +{{if not IsOpenShift}} "apiServerCertificate": { "metadata": { "description": "The base 64 server certificate used on the master" @@ -160,6 +163,7 @@ }, "type": "securestring" }, +{{end}} "generatorCode": { {{PopulateClassicModeDefaultValue "generatorCode"}} "metadata": { @@ -552,6 +556,7 @@ }, "type": "string" }, + {{if not IsOpenShift}} "dockerEngineDownloadRepo": { "defaultValue": "https://aptdocker.azureedge.net/repo", "metadata": { @@ -574,6 +579,7 @@ ], "type": "string" }, + {{end}} "networkPolicy": { "defaultValue": "{{.OrchestratorProfile.KubernetesConfig.NetworkPolicy}}", "metadata": { diff --git a/parts/k8s/openshiftmasterscript.sh b/parts/k8s/openshiftmasterscript.sh new file mode 100644 index 000000000..a1eb9e18e --- /dev/null +++ b/parts/k8s/openshiftmasterscript.sh @@ -0,0 +1,142 @@ +#!/bin/bash -x + +# TODO: /etc/dnsmasq.d/origin-upstream-dns.conf is currently hardcoded; it +# probably shouldn't be + +SERVICE_TYPE=origin +IMAGE_BASE=openshift/origin +if [ -f "/etc/sysconfig/atomic-openshift-node" ]; then + SERVICE_TYPE=atomic-openshift + IMAGE_BASE=registry.reg-aws.openshift.com:443/openshift3/ose +fi +VERSION="$(rpm -q $SERVICE_TYPE --queryformat %{VERSION})" + +# TODO: remove this once we generate the registry certificate +cat >>/etc/sysconfig/docker <<'EOF' +INSECURE_REGISTRY='--insecure-registry 172.30.0.0/16' +EOF + +systemctl restart docker.service + +echo "BOOTSTRAP_CONFIG_NAME=node-config-master" >>/etc/sysconfig/${SERVICE_TYPE}-node + +for dst in tcp,2379 tcp,2380 tcp,8443 tcp,8444 tcp,8053 udp,8053 tcp,9090; do + proto=${dst%%,*} + port=${dst##*,} + iptables -A OS_FIREWALL_ALLOW -p $proto -m state --state NEW -m $proto --dport $port -j ACCEPT +done + +iptables-save >/etc/sysconfig/iptables + +sed -i -e "s#--master=.*#--master=https://$(hostname --fqdn):8443#" /etc/sysconfig/${SERVICE_TYPE}-master-api + +rm -rf /etc/etcd/* /etc/origin/master/* /etc/origin/node/* + +oc adm create-bootstrap-policy-file --filename=/etc/origin/master/policy.json + +( cd / && base64 -d <<< {{ .ConfigBundle }} | tar -xz) + +chown -R etcd:etcd /etc/etcd +chmod 0600 /etc/origin/master/htpasswd +chmod 1777 /tmp + +cp /etc/origin/node/ca.crt /etc/pki/ca-trust/source/anchors/openshift-ca.crt +update-ca-trust + +### +# retrieve the public ip via dns for the router public ip and sub it in for the routingConfig.subdomain +### +routerLBHost="{{.RouterLBHostname}}" +routerLBIP=$(dig +short $routerLBHost) + +for i in /etc/origin/master/master-config.yaml /tmp/bootstrapconfigs/* /tmp/ansible/azure-local-master-inventory.yml; do + sed -i "s/TEMPROUTERIP/${routerLBIP}/; s|TEMPIMAGEBASE|$IMAGE_BASE|" $i +done + +# TODO: when enabling secure registry, may need: +# ln -s /etc/origin/node/node-client-ca.crt /etc/docker/certs.d/docker-registry.default.svc:5000 + +# note: ${SERVICE_TYPE}-node crash loops until master is up +for unit in etcd.service ${SERVICE_TYPE}-master-api.service ${SERVICE_TYPE}-master-controllers.service; do + systemctl enable $unit + systemctl start $unit +done + +mkdir -p /root/.kube +cp /etc/origin/master/admin.kubeconfig /root/.kube/config + +export KUBECONFIG=/etc/origin/master/admin.kubeconfig + +while ! curl -o /dev/null -m 2 -kfs https://localhost:8443/healthz; do + sleep 1 +done + +while ! oc get svc kubernetes &>/dev/null; do + sleep 1 +done + +oc create -f - <<'EOF' +kind: StorageClass +apiVersion: storage.k8s.io/v1beta1 +metadata: + name: azure + annotations: + storageclass.kubernetes.io/is-default-class: "true" +provisioner: kubernetes.io/azure-disk +parameters: + skuName: Premium_LRS + location: {{ .Location }} +EOF + +oc create configmap node-config-master --namespace openshift-node --from-file=node-config.yaml=/tmp/bootstrapconfigs/master-config.yaml +oc create configmap node-config-compute --namespace openshift-node --from-file=node-config.yaml=/tmp/bootstrapconfigs/compute-config.yaml +oc create configmap node-config-infra --namespace openshift-node --from-file=node-config.yaml=/tmp/bootstrapconfigs/infra-config.yaml + +# must start ${SERVICE_TYPE}-node after master is fully up and running +# otherwise the implicit dns change may cause master startup to fail +systemctl enable ${SERVICE_TYPE}-node.service +systemctl start ${SERVICE_TYPE}-node.service & + +# TODO: run a CSR auto-approver +# https://github.com/kargakis/acs-engine/issues/46 +csrs=($(oc get csr -o name)) +while [[ ${#csrs[@]} != "3" ]]; do + sleep 2 + csrs=($(oc get csr -o name)) + if [[ ${#csrs[@]} == "3" ]]; then + break + fi +done + +for csr in ${csrs[@]}; do + oc adm certificate approve $csr +done + +csrs=($(oc get csr -o name)) +while [[ ${#csrs[@]} != "6" ]]; do + sleep 2 + csrs=($(oc get csr -o name)) + if [[ ${#csrs[@]} == "6" ]]; then + break + fi +done + +for csr in ${csrs[@]}; do + oc adm certificate approve $csr +done + +chmod +x /tmp/ansible/ansible.sh +docker run \ + --rm \ + -u "$(id -u)" \ + -v /etc/origin:/etc/origin:z \ + -v /tmp/ansible:/opt/app-root/src:z \ + -v /root/.kube:/opt/app-root/src/.kube:z \ + -w /opt/app-root/src \ + -e IMAGE_BASE="$IMAGE_BASE" \ + -e VERSION="$VERSION" \ + -e HOSTNAME="$(hostname)" \ + "$IMAGE_BASE-ansible:v$VERSION" \ + /opt/app-root/src/ansible.sh + +exit 0 diff --git a/parts/k8s/openshiftnodescript.sh b/parts/k8s/openshiftnodescript.sh new file mode 100644 index 000000000..29a3ed2cb --- /dev/null +++ b/parts/k8s/openshiftnodescript.sh @@ -0,0 +1,37 @@ +#!/bin/bash -x + +# TODO: /etc/dnsmasq.d/origin-upstream-dns.conf is currently hardcoded; it +# probably shouldn't be +SERVICE_TYPE=origin +if [ -f "/etc/sysconfig/atomic-openshift-node" ]; then + SERVICE_TYPE=atomic-openshift +fi + +# TODO: remove this once we generate the registry certificate +cat >>/etc/sysconfig/docker <<'EOF' +INSECURE_REGISTRY='--insecure-registry 172.30.0.0/16' +EOF + +systemctl restart docker.service + +{{if eq .Role "infra"}} +echo "BOOTSTRAP_CONFIG_NAME=node-config-infra" >>/etc/sysconfig/${SERVICE_TYPE}-node +{{else}} +echo "BOOTSTRAP_CONFIG_NAME=node-config-compute" >>/etc/sysconfig/${SERVICE_TYPE}-node +{{end}} + +rm -rf /etc/etcd/* /etc/origin/master/* /etc/origin/node/* + +( cd / && base64 -d <<< {{ .ConfigBundle }} | tar -xz) + +cp /etc/origin/node/ca.crt /etc/pki/ca-trust/source/anchors/openshift-ca.crt +update-ca-trust + +# TODO: when enabling secure registry, may need: +# ln -s /etc/origin/node/node-client-ca.crt /etc/docker/certs.d/docker-registry.default.svc:5000 + +# note: ${SERVICE_TYPE}-node crash loops until master is up +systemctl enable ${SERVICE_TYPE}-node.service +systemctl start ${SERVICE_TYPE}-node.service + +exit 0 diff --git a/pkg/acsengine/const.go b/pkg/acsengine/const.go index fc4a5d34d..1c613e52b 100644 --- a/pkg/acsengine/const.go +++ b/pkg/acsengine/const.go @@ -1,6 +1,10 @@ package acsengine const ( + // DefaultOpenShiftMasterSubnet is the default value for master subnet for Openshift. + DefaultOpenShiftMasterSubnet = "10.0.0.0/24" + // DefaultOpenShiftFirstConsecutiveStaticIP is the default static ip address for master 0 for Openshift. + DefaultOpenShiftFirstConsecutiveStaticIP = "10.0.0.11" // DefaultMasterSubnet specifies the default master subnet for DCOS or Swarm DefaultMasterSubnet = "172.16.0.0/24" // DefaultFirstConsecutiveStaticIP specifies the static IP address on master 0 for DCOS or Swarm @@ -100,6 +104,8 @@ const ( DefaultGeneratorCode = "acsengine" // DefaultOrchestratorName specifies the 3 character orchestrator code of the cluster template and affects resource naming. DefaultOrchestratorName = "k8s" + // DefaultOpenshiftOrchestratorName specifies the 3 character orchestrator code of the cluster template and affects resource naming. + DefaultOpenshiftOrchestratorName = "ocp" // DefaultEtcdVersion specifies the default etcd version to install DefaultEtcdVersion = "3.2.16" // DefaultEtcdDiskSize specifies the default size for Kubernetes master etcd disk volumes in GB diff --git a/pkg/acsengine/defaults.go b/pkg/acsengine/defaults.go index 3946a0e6a..2bc0fff5a 100644 --- a/pkg/acsengine/defaults.go +++ b/pkg/acsengine/defaults.go @@ -11,6 +11,8 @@ import ( "github.com/Azure/acs-engine/pkg/api" "github.com/Azure/acs-engine/pkg/api/common" "github.com/Azure/acs-engine/pkg/helpers" + "github.com/Azure/acs-engine/pkg/openshift/certgen" + "github.com/Azure/acs-engine/pkg/openshift/filesystem" "github.com/Masterminds/semver" ) @@ -296,7 +298,9 @@ func setOrchestratorDefaults(cs *api.ContainerService) { o.OrchestratorVersion = common.GetValidPatchVersion( o.OrchestratorType, o.OrchestratorVersion) - if o.OrchestratorType == api.Kubernetes { + + switch o.OrchestratorType { + case api.Kubernetes: k8sVersion := o.OrchestratorVersion if o.KubernetesConfig == nil { @@ -489,13 +493,25 @@ func setOrchestratorDefaults(cs *api.ContainerService) { // Configure scheduler setSchedulerConfig(cs) - } else if o.OrchestratorType == api.DCOS { + case api.DCOS: if o.DcosConfig == nil { o.DcosConfig = &api.DcosConfig{} } if o.DcosConfig.DcosWindowsBootstrapURL == "" { o.DcosConfig.DcosWindowsBootstrapURL = DefaultDCOSSpecConfig.DCOSWindowsBootstrapDownloadURL } + case api.OpenShift: + a.MasterProfile.Distro = api.RHEL + kc := a.OrchestratorProfile.OpenShiftConfig.KubernetesConfig + if kc == nil { + kc = &api.KubernetesConfig{} + } + if kc.ContainerRuntime == "" { + kc.ContainerRuntime = DefaultContainerRuntime + } + if kc.NetworkPolicy == "" { + kc.NetworkPolicy = DefaultNetworkPolicy + } } } @@ -545,6 +561,11 @@ func setMasterNetworkDefaults(a *api.Properties, isUpgrade bool) { a.MasterProfile.FirstConsecutiveStaticIP = DefaultFirstConsecutiveKubernetesStaticIP } } + } else if a.OrchestratorProfile.OrchestratorType == api.OpenShift { + a.MasterProfile.Subnet = DefaultOpenShiftMasterSubnet + if !isUpgrade || len(a.MasterProfile.FirstConsecutiveStaticIP) == 0 { + a.MasterProfile.FirstConsecutiveStaticIP = DefaultOpenShiftFirstConsecutiveStaticIP + } } else if a.HasWindows() { a.MasterProfile.Subnet = DefaultSwarmWindowsMasterSubnet // FirstConsecutiveStaticIP is not reset if it is upgrade and some value already exists @@ -585,7 +606,8 @@ func setAgentNetworkDefaults(a *api.Properties) { if a.MasterProfile != nil && !a.MasterProfile.IsCustomVNET() { subnetCounter := 0 for _, profile := range a.AgentPoolProfiles { - if a.OrchestratorProfile.OrchestratorType == api.Kubernetes { + if a.OrchestratorProfile.OrchestratorType == api.Kubernetes || + a.OrchestratorProfile.OrchestratorType == api.OpenShift { profile.Subnet = a.MasterProfile.Subnet } else { profile.Subnet = fmt.Sprintf(DefaultAgentSubnetTemplate, subnetCounter) @@ -636,7 +658,96 @@ func setStorageDefaults(a *api.Properties) { } } +func openShiftSetDefaultCerts(a *api.Properties) (bool, error) { + externalMasterHostname := fmt.Sprintf("%s.%s.cloudapp.azure.com", a.MasterProfile.DNSPrefix, a.AzProfile.Location) + routerLBHostname := fmt.Sprintf("%s-router.%s.cloudapp.azure.com", a.MasterProfile.DNSPrefix, a.AzProfile.Location) + c := certgen.Config{ + Master: &certgen.Master{ + Hostname: fmt.Sprintf("%s-master-%s-0", DefaultOpenshiftOrchestratorName, GenerateClusterID(a)), + IPs: []net.IP{ + net.ParseIP(a.MasterProfile.FirstConsecutiveStaticIP), + }, + Port: 8443, + }, + ExternalMasterHostname: externalMasterHostname, + ClusterUsername: a.OrchestratorProfile.OpenShiftConfig.ClusterUsername, + ClusterPassword: a.OrchestratorProfile.OpenShiftConfig.ClusterPassword, + AzureConfig: certgen.AzureConfig{ + TenantID: a.AzProfile.TenantID, + SubscriptionID: a.AzProfile.SubscriptionID, + AADClientID: a.ServicePrincipalProfile.ClientID, + AADClientSecret: a.ServicePrincipalProfile.Secret, + ResourceGroup: a.AzProfile.ResourceGroup, + Location: a.AzProfile.Location, + }, + } + a.OrchestratorProfile.OpenShiftConfig.ExternalMasterHostname = externalMasterHostname + a.OrchestratorProfile.OpenShiftConfig.RouterLBHostname = routerLBHostname + + err := c.PrepareMasterCerts() + if err != nil { + return false, err + } + err = c.PrepareMasterKubeConfigs() + if err != nil { + return false, err + } + err = c.PrepareMasterFiles() + if err != nil { + return false, err + } + + err = c.PrepareBootstrapKubeConfig() + if err != nil { + return false, err + } + + if a.OrchestratorProfile.OpenShiftConfig.ConfigBundles == nil { + a.OrchestratorProfile.OpenShiftConfig.ConfigBundles = make(map[string][]byte) + } + + masterBundle, err := getConfigBundle(c.WriteMaster) + if err != nil { + return false, err + } + a.OrchestratorProfile.OpenShiftConfig.ConfigBundles["master"] = masterBundle + + nodeBundle, err := getConfigBundle(c.WriteNode) + if err != nil { + return false, err + } + a.OrchestratorProfile.OpenShiftConfig.ConfigBundles["bootstrap"] = nodeBundle + + return false, nil +} + +type writeFn func(filesystem.Filesystem) error + +func getConfigBundle(write writeFn) ([]byte, error) { + b := &bytes.Buffer{} + + fs, err := filesystem.NewTGZFile(b) + if err != nil { + return nil, err + } + + err = write(fs) + if err != nil { + return nil, err + } + + err = fs.Close() + if err != nil { + return nil, err + } + + return b.Bytes(), nil +} + func setDefaultCerts(a *api.Properties) (bool, error) { + if a.MasterProfile != nil && a.OrchestratorProfile.OrchestratorType == api.OpenShift { + return openShiftSetDefaultCerts(a) + } if a.MasterProfile == nil || a.OrchestratorProfile.OrchestratorType != api.Kubernetes { return false, nil diff --git a/pkg/acsengine/engine.go b/pkg/acsengine/engine.go index 3b2103e05..720b15748 100644 --- a/pkg/acsengine/engine.go +++ b/pkg/acsengine/engine.go @@ -39,6 +39,9 @@ const ( kubernetesJumpboxCustomDataYaml = "k8s/kubernetesjumpboxcustomdata.yml" kubeConfigJSON = "k8s/kubeconfig.json" kubernetesWindowsAgentCustomDataPS1 = "k8s/kuberneteswindowssetup.ps1" + // OpenShift custom scripts + openshiftNodeScript = "k8s/openshiftnodescript.sh" + openshiftMasterScript = "k8s/openshiftmasterscript.sh" ) const ( @@ -106,6 +109,7 @@ var dcosTemplateFiles = []string{dcosBaseFile, dcosAgentResourcesVMAS, dcosAgent var kubernetesTemplateFiles = []string{kubernetesBaseFile, kubernetesAgentResourcesVMAS, kubernetesAgentVars, kubernetesMasterResources, kubernetesMasterVars, kubernetesParams, kubernetesWinAgentVars} var swarmTemplateFiles = []string{swarmBaseFile, swarmParams, swarmAgentResourcesVMAS, swarmAgentVars, swarmAgentResourcesVMSS, swarmAgentResourcesClassic, swarmBaseFile, swarmMasterResources, swarmMasterVars, swarmWinAgentResourcesVMAS, swarmWinAgentResourcesVMSS} var swarmModeTemplateFiles = []string{swarmBaseFile, swarmParams, swarmAgentResourcesVMAS, swarmAgentVars, swarmAgentResourcesVMSS, swarmAgentResourcesClassic, swarmBaseFile, swarmMasterResources, swarmMasterVars, swarmWinAgentResourcesVMAS, swarmWinAgentResourcesVMSS} +var openshiftTemplateFiles = append(kubernetesTemplateFiles, openshiftNodeScript, openshiftMasterScript) /** The following parameters could be either a plain text, or referenced to a secret in a keyvault: @@ -169,6 +173,7 @@ func (t *TemplateGenerator) verifyFiles() error { allFiles := commonTemplateFiles allFiles = append(allFiles, dcosTemplateFiles...) allFiles = append(allFiles, kubernetesTemplateFiles...) + allFiles = append(allFiles, openshiftTemplateFiles...) allFiles = append(allFiles, swarmTemplateFiles...) for _, file := range allFiles { if _, err := Asset(file); err != nil { @@ -354,6 +359,9 @@ func (t *TemplateGenerator) prepareTemplateFiles(properties *api.Properties) ([] case api.SwarmMode: files = append(commonTemplateFiles, swarmModeTemplateFiles...) baseFile = swarmBaseFile + case api.OpenShift: + files = append(commonTemplateFiles, openshiftTemplateFiles...) + baseFile = kubernetesBaseFile default: return nil, "", t.Translator.Errorf("orchestrator '%s' is unsupported", properties.OrchestratorProfile.OrchestratorType) } @@ -405,13 +413,15 @@ func GetCloudSpecConfig(location string) AzureEnvironmentSpecConfig { // ValidateDistro checks if the requested orchestrator type is supported on the requested Linux distro. func ValidateDistro(cs *api.ContainerService) bool { // Check Master distro - if cs.Properties.MasterProfile != nil && cs.Properties.MasterProfile.Distro == api.RHEL && cs.Properties.OrchestratorProfile.OrchestratorType != api.SwarmMode { + if cs.Properties.MasterProfile != nil && cs.Properties.MasterProfile.Distro == api.RHEL && + (cs.Properties.OrchestratorProfile.OrchestratorType != api.SwarmMode && cs.Properties.OrchestratorProfile.OrchestratorType != api.OpenShift) { log.Fatalf("Orchestrator type %s not suported on RHEL Master", cs.Properties.OrchestratorProfile.OrchestratorType) return false } // Check Agent distros for _, agentProfile := range cs.Properties.AgentPoolProfiles { - if agentProfile.Distro == api.RHEL && cs.Properties.OrchestratorProfile.OrchestratorType != api.SwarmMode { + if agentProfile.Distro == api.RHEL && + (cs.Properties.OrchestratorProfile.OrchestratorType != api.SwarmMode && cs.Properties.OrchestratorProfile.OrchestratorType != api.OpenShift) { log.Fatalf("Orchestrator type %s not suported on RHEL Agent", cs.Properties.OrchestratorProfile.OrchestratorType) return false } @@ -470,7 +480,7 @@ func getParameters(cs *api.ContainerService, isClassicMode bool, generatorCode s if properties.MasterProfile != nil { if properties.MasterProfile.IsCustomVNET() { addValue(parametersMap, "masterVnetSubnetID", properties.MasterProfile.VnetSubnetID) - if properties.OrchestratorProfile.IsKubernetes() { + if properties.OrchestratorProfile.IsKubernetes() || properties.OrchestratorProfile.IsOpenShift() { addValue(parametersMap, "vnetCidr", properties.MasterProfile.VnetCidr) } } else { @@ -511,7 +521,8 @@ func getParameters(cs *api.ContainerService, isClassicMode bool, generatorCode s } // Kubernetes Parameters - if properties.OrchestratorProfile.OrchestratorType == api.Kubernetes { + if properties.OrchestratorProfile.IsKubernetes() || + properties.OrchestratorProfile.IsOpenShift() { k8sVersion := properties.OrchestratorProfile.OrchestratorVersion kubernetesHyperkubeSpec := properties.OrchestratorProfile.KubernetesConfig.KubernetesImageBase + KubeConfigs[k8sVersion]["hyperkube"] @@ -560,11 +571,14 @@ func getParameters(cs *api.ContainerService, isClassicMode bool, generatorCode s addValue(parametersMap, "kubernetesCcmImageSpec", kubernetesCcmSpec) } - addValue(parametersMap, "dockerEngineDownloadRepo", cloudSpecConfig.DockerSpecConfig.DockerEngineRepo) + if !properties.OrchestratorProfile.IsOpenShift() { + addValue(parametersMap, "dockerEngineDownloadRepo", cloudSpecConfig.DockerSpecConfig.DockerEngineRepo) + addValue(parametersMap, "dockerEngineVersion", dockerEngineVersion) + } + addValue(parametersMap, "kubeDNSServiceIP", properties.OrchestratorProfile.KubernetesConfig.DNSServiceIP) addValue(parametersMap, "kubeServiceCidr", properties.OrchestratorProfile.KubernetesConfig.ServiceCIDR) addValue(parametersMap, "kubernetesHyperkubeSpec", kubernetesHyperkubeSpec) - addValue(parametersMap, "dockerEngineVersion", dockerEngineVersion) addValue(parametersMap, "kubernetesAddonManagerSpec", cloudSpecConfig.KubernetesSpecConfig.KubernetesImageBase+KubeConfigs[k8sVersion]["addonmanager"]) addValue(parametersMap, "kubernetesAddonResizerSpec", cloudSpecConfig.KubernetesSpecConfig.KubernetesImageBase+KubeConfigs[k8sVersion]["addonresizer"]) addValue(parametersMap, "kubernetesDNSMasqSpec", cloudSpecConfig.KubernetesSpecConfig.KubernetesImageBase+KubeConfigs[k8sVersion]["dnsmasq"]) @@ -657,6 +671,8 @@ func getParameters(cs *api.ContainerService, isClassicMode bool, generatorCode s addValue(parametersMap, "generatorCode", generatorCode) if properties.HostedMasterProfile != nil { addValue(parametersMap, "orchestratorName", "aks") + } else if properties.OrchestratorProfile.IsOpenShift() { + addValue(parametersMap, "orchestratorName", DefaultOpenshiftOrchestratorName) } else { addValue(parametersMap, "orchestratorName", DefaultOrchestratorName) } @@ -804,7 +820,7 @@ func getParameters(cs *api.ContainerService, isClassicMode bool, generatorCode s if properties.WindowsProfile.WindowsImageSourceURL != "" { addValue(parametersMap, "agentWindowsSourceUrl", properties.WindowsProfile.WindowsImageSourceURL) } - if properties.OrchestratorProfile.OrchestratorType == api.Kubernetes { + if properties.OrchestratorProfile.IsKubernetes() || properties.OrchestratorProfile.IsOpenShift() { k8sVersion := properties.OrchestratorProfile.OrchestratorVersion addValue(parametersMap, "kubeBinariesSASURL", cloudSpecConfig.KubernetesSpecConfig.KubeBinariesSASURLBase+KubeConfigs[k8sVersion]["windowszip"]) addValue(parametersMap, "windowsPackageSASURLBase", cloudSpecConfig.KubernetesSpecConfig.WindowsPackageSASURLBase) @@ -904,12 +920,12 @@ func (t *TemplateGenerator) getTemplateFuncMap(cs *api.ContainerService) templat "IsKubernetesVersionGe": func(version string) bool { orchestratorVersion, _ := semver.NewVersion(cs.Properties.OrchestratorProfile.OrchestratorVersion) constraint, _ := semver.NewConstraint(">=" + version) - return cs.Properties.OrchestratorProfile.OrchestratorType == api.Kubernetes && constraint.Check(orchestratorVersion) + return cs.Properties.OrchestratorProfile.IsKubernetes() && constraint.Check(orchestratorVersion) }, "IsKubernetesVersionLt": func(version string) bool { orchestratorVersion, _ := semver.NewVersion(cs.Properties.OrchestratorProfile.OrchestratorVersion) constraint, _ := semver.NewConstraint("<" + version) - return cs.Properties.OrchestratorProfile.OrchestratorType == api.Kubernetes && constraint.Check(orchestratorVersion) + return cs.Properties.OrchestratorProfile.IsKubernetes() && constraint.Check(orchestratorVersion) }, "IsKubernetesVersionTilde": func(version string) bool { // examples include @@ -917,7 +933,7 @@ func (t *TemplateGenerator) getTemplateFuncMap(cs *api.ContainerService) templat // ~1.2.x is equivalent to >= 1.2.0, < 1.3.0 orchestratorVersion, _ := semver.NewVersion(cs.Properties.OrchestratorProfile.OrchestratorVersion) constraint, _ := semver.NewConstraint("~" + version) - return cs.Properties.OrchestratorProfile.OrchestratorType == api.Kubernetes && constraint.Check(orchestratorVersion) + return cs.Properties.OrchestratorProfile.IsKubernetes() && constraint.Check(orchestratorVersion) }, "GetMasterKubernetesLabels": func(rg string) string { var buf bytes.Buffer @@ -939,6 +955,9 @@ func (t *TemplateGenerator) getTemplateFuncMap(cs *api.ContainerService) templat return buf.String() }, "GetKubeletConfigKeyVals": func(kc *api.KubernetesConfig) string { + if kc == nil { + return "" + } kubeletConfig := cs.Properties.OrchestratorProfile.KubernetesConfig.KubeletConfig if kc.KubeletConfig != nil { kubeletConfig = kc.KubeletConfig @@ -975,7 +994,7 @@ func (t *TemplateGenerator) getTemplateFuncMap(cs *api.ContainerService) templat return false }, "RequiresFakeAgentOutput": func() bool { - return cs.Properties.OrchestratorProfile.OrchestratorType == api.Kubernetes + return cs.Properties.OrchestratorProfile.IsKubernetes() || cs.Properties.OrchestratorProfile.IsOpenShift() }, "IsSwarmMode": func() bool { return cs.Properties.OrchestratorProfile.IsSwarmMode() @@ -983,6 +1002,9 @@ func (t *TemplateGenerator) getTemplateFuncMap(cs *api.ContainerService) templat "IsKubernetes": func() bool { return cs.Properties.OrchestratorProfile.IsKubernetes() }, + "IsOpenShift": func() bool { + return cs.Properties.OrchestratorProfile.IsOpenShift() + }, "IsPublic": func(ports []int) bool { return len(ports) > 0 }, @@ -1114,7 +1136,7 @@ func (t *TemplateGenerator) getTemplateFuncMap(cs *api.ContainerService) templat "GetAgentAllowedSizes": func() string { if t.ClassicMode { return GetClassicAllowedSizes() - } else if cs.Properties.OrchestratorProfile.OrchestratorType == api.Kubernetes { + } else if cs.Properties.OrchestratorProfile.IsKubernetes() || cs.Properties.OrchestratorProfile.IsOpenShift() { return GetKubernetesAgentAllowedSizes() } return GetMasterAgentAllowedSizes() @@ -1704,6 +1726,36 @@ func (t *TemplateGenerator) getTemplateFuncMap(cs *api.ContainerService) templat "EnablePodSecurityPolicy": func() bool { return helpers.IsTrueBoolPointer(cs.Properties.OrchestratorProfile.KubernetesConfig.EnablePodSecurityPolicy) }, + "OpenShiftGetMasterSh": func() string { + tb := MustAsset("k8s/openshiftmasterscript.sh") + t := template.Must(template.New("master").Parse(string(tb))) + b := &bytes.Buffer{} + t.Execute(b, struct { + ConfigBundle string + ExternalMasterHostname string + RouterLBHostname string + Location string + }{ + ConfigBundle: base64.StdEncoding.EncodeToString(cs.Properties.OrchestratorProfile.OpenShiftConfig.ConfigBundles["master"]), + ExternalMasterHostname: cs.Properties.OrchestratorProfile.OpenShiftConfig.ExternalMasterHostname, + RouterLBHostname: cs.Properties.OrchestratorProfile.OpenShiftConfig.RouterLBHostname, + Location: cs.Properties.AzProfile.Location, + }) + return b.String() + }, + "OpenShiftGetNodeSh": func(profile *api.AgentPoolProfile) string { + tb := MustAsset("k8s/openshiftnodescript.sh") + t := template.Must(template.New("node").Parse(string(tb))) + b := &bytes.Buffer{} + t.Execute(b, struct { + ConfigBundle string + Role api.AgentPoolProfileRole + }{ + ConfigBundle: base64.StdEncoding.EncodeToString(cs.Properties.OrchestratorProfile.OpenShiftConfig.ConfigBundles["bootstrap"]), + Role: profile.Role, + }) + return b.String() + }, // inspired by http://stackoverflow.com/questions/18276173/calling-a-template-with-several-pipeline-parameters/18276968#18276968 "dict": func(values ...interface{}) (map[string]interface{}, error) { if len(values)%2 != 0 { @@ -1731,7 +1783,7 @@ func (t *TemplateGenerator) getTemplateFuncMap(cs *api.ContainerService) templat func makeMasterExtensionScriptCommands(cs *api.ContainerService) string { copyIndex := "',copyIndex(),'" - if cs.Properties.OrchestratorProfile.IsKubernetes() { + if cs.Properties.OrchestratorProfile.IsKubernetes() || cs.Properties.OrchestratorProfile.IsOpenShift() { copyIndex = "',copyIndex(variables('masterOffset')),'" } return makeExtensionScriptCommands(cs.Properties.MasterProfile.PreprovisionExtension, @@ -2520,7 +2572,7 @@ func getMasterLinkedTemplateText(masterProfile *api.MasterProfile, orchestratorT loopCount := "[variables('masterCount')]" loopOffset := "" - if orchestratorType == api.Kubernetes { + if orchestratorType == api.Kubernetes || orchestratorType == api.OpenShift { // Due to upgrade k8s sometimes needs to install just some of the nodes. loopCount = "[sub(variables('masterCount'), variables('masterOffset'))]" loopOffset = "variables('masterOffset')" diff --git a/pkg/api/common/const.go b/pkg/api/common/const.go index dfe95a826..ce465f784 100644 --- a/pkg/api/common/const.go +++ b/pkg/api/common/const.go @@ -12,6 +12,8 @@ const ( Kubernetes string = "Kubernetes" // SwarmMode is the string constant for the Swarm Mode orchestrator type SwarmMode string = "SwarmMode" + // OpenShift is the string constant for the OpenShift orchestrator type + OpenShift string = "OpenShift" ) // validation values @@ -77,3 +79,15 @@ var AllDCOSSupportedVersions = []string{ DCOSVersion1Dot9Dot8, DCOSVersion1Dot8Dot8, } + +const ( + // OpenShiftVersion3Dot9Dot0 is the major.minor.patch string for the 3.9.0 version of OpenShift + OpenShiftVersion3Dot9Dot0 string = "3.9.0" + // OpenShiftDefaultVersion is the default major.minor.patch version for OpenShift + OpenShiftDefaultVersion string = OpenShiftVersion3Dot9Dot0 +) + +// GetAllSupportedOpenShiftVersions returns a slice of all supported OpenShift versions. +func GetAllSupportedOpenShiftVersions() []string { + return []string{OpenShiftVersion3Dot9Dot0} +} diff --git a/pkg/api/common/versions.go b/pkg/api/common/versions.go index 445d5f2bd..253874773 100644 --- a/pkg/api/common/versions.go +++ b/pkg/api/common/versions.go @@ -223,6 +223,9 @@ func GetSupportedVersions(orchType string, hasWindows bool) (versions []string, } return GetAllSupportedKubernetesVersions(), GetDefaultKubernetesVersion() + case OpenShift: + return GetAllSupportedOpenShiftVersions(), string(OpenShiftDefaultVersion) + case DCOS: return AllDCOSSupportedVersions, DCOSDefaultVersion default: diff --git a/pkg/api/const.go b/pkg/api/const.go index 7275aad95..85814999c 100644 --- a/pkg/api/const.go +++ b/pkg/api/const.go @@ -12,6 +12,8 @@ const ( Kubernetes string = "Kubernetes" // SwarmMode is the string constant for the Swarm Mode orchestrator type SwarmMode string = "SwarmMode" + // OpenShift is the string constant for the OpenShift orchestrator type + OpenShift string = "OpenShift" ) // the OSTypes supported by vlabs @@ -114,3 +116,10 @@ const ( // DefaultPrivateClusterEnabled determines the acs-engine provided default for enabling kubernetes Private Cluster DefaultPrivateClusterEnabled = false ) + +const ( + // AgentPoolProfileRoleEmpty is the empty role + AgentPoolProfileRoleEmpty AgentPoolProfileRole = "" + // AgentPoolProfileRoleInfra is the infra role + AgentPoolProfileRoleInfra AgentPoolProfileRole = "infra" +) diff --git a/pkg/api/converterfromapi.go b/pkg/api/converterfromapi.go index 4cdfc63ae..f3646e1c6 100644 --- a/pkg/api/converterfromapi.go +++ b/pkg/api/converterfromapi.go @@ -161,6 +161,8 @@ func ConvertOrchestratorVersionProfileToVLabs(api *OrchestratorVersionProfile) * vlabsProfile.OrchestratorType = vlabs.Swarm case SwarmMode: vlabsProfile.OrchestratorType = vlabs.SwarmMode + case OpenShift: + vlabsProfile.OrchestratorType = vlabs.OpenShift } vlabsProfile.OrchestratorVersion = api.OrchestratorVersion vlabsProfile.Default = api.Default @@ -472,6 +474,10 @@ func convertPropertiesToVLabs(api *Properties, vlabsProps *vlabs.Properties) { vlabsProps.AADProfile = &vlabs.AADProfile{} convertAADProfileToVLabs(api.AADProfile, vlabsProps.AADProfile) } + if api.AzProfile != nil { + vlabsProps.AzProfile = &vlabs.AzProfile{} + convertAzProfileToVLabs(api.AzProfile, vlabsProps.AzProfile) + } } func convertLinuxProfileToV20160930(api *LinuxProfile, obj *v20160930.LinuxProfile) { @@ -633,12 +639,26 @@ func convertOrchestratorProfileToVLabs(api *OrchestratorProfile, o *vlabs.Orches convertKubernetesConfigToVLabs(api.KubernetesConfig, o.KubernetesConfig) } + if api.OpenShiftConfig != nil { + o.OpenShiftConfig = &vlabs.OpenShiftConfig{} + convertOpenShiftConfigToVLabs(api.OpenShiftConfig, o.OpenShiftConfig) + } + if api.DcosConfig != nil { o.DcosConfig = &vlabs.DcosConfig{} convertDcosConfigToVLabs(api.DcosConfig, o.DcosConfig) } } +func convertOpenShiftConfigToVLabs(api *OpenShiftConfig, vl *vlabs.OpenShiftConfig) { + vl.KubernetesConfig = &vlabs.KubernetesConfig{} + if api.KubernetesConfig != nil { + convertKubernetesConfigToVLabs(api.KubernetesConfig, vl.KubernetesConfig) + } + vl.ClusterUsername = api.ClusterUsername + vl.ClusterPassword = api.ClusterPassword +} + func convertDcosConfigToVLabs(api *DcosConfig, vlabs *vlabs.DcosConfig) { vlabs.DcosBootstrapURL = api.DcosBootstrapURL vlabs.DcosWindowsBootstrapURL = api.DcosWindowsBootstrapURL @@ -1072,3 +1092,10 @@ func convertAADProfileToVLabs(api *AADProfile, vlabs *vlabs.AADProfile) { vlabs.TenantID = api.TenantID vlabs.AdminGroupID = api.AdminGroupID } + +func convertAzProfileToVLabs(api *AzProfile, vlabs *vlabs.AzProfile) { + vlabs.Location = api.Location + vlabs.ResourceGroup = api.ResourceGroup + vlabs.SubscriptionID = api.SubscriptionID + vlabs.TenantID = api.TenantID +} diff --git a/pkg/api/convertertoapi.go b/pkg/api/convertertoapi.go index c52f9a400..c1b4c5464 100644 --- a/pkg/api/convertertoapi.go +++ b/pkg/api/convertertoapi.go @@ -366,8 +366,7 @@ func convertVLabsProperties(vlabs *vlabs.Properties, api *Properties) { apiProfile := &AgentPoolProfile{} convertVLabsAgentPoolProfile(p, apiProfile) // by default vlabs will use managed disks for all orchestrators but kubernetes as it has encryption at rest. - if !api.OrchestratorProfile.IsKubernetes() { - // by default vlabs will use managed disks for all orchestrators but kubernetes as it has encryption at rest. + if !api.OrchestratorProfile.IsKubernetes() && !api.OrchestratorProfile.IsOpenShift() { if len(p.StorageProfile) == 0 { apiProfile.StorageProfile = ManagedDisks } @@ -401,6 +400,17 @@ func convertVLabsProperties(vlabs *vlabs.Properties, api *Properties) { api.AADProfile = &AADProfile{} convertVLabsAADProfile(vlabs.AADProfile, api.AADProfile) } + if vlabs.AzProfile != nil { + api.AzProfile = &AzProfile{} + convertVLabsAZProfile(vlabs.AzProfile, api.AzProfile) + } +} + +func convertVLabsAZProfile(vlabs *vlabs.AzProfile, api *AzProfile) { + api.Location = vlabs.Location + api.ResourceGroup = vlabs.ResourceGroup + api.SubscriptionID = vlabs.SubscriptionID + api.TenantID = vlabs.TenantID } func convertV20160930LinuxProfile(obj *v20160930.LinuxProfile, api *LinuxProfile) { @@ -560,6 +570,22 @@ func convertVLabsOrchestratorProfile(vp *vlabs.Properties, api *OrchestratorProf vlabscs := vp.OrchestratorProfile api.OrchestratorType = vlabscs.OrchestratorType switch api.OrchestratorType { + case OpenShift: + if vlabscs.OpenShiftConfig != nil { + api.OpenShiftConfig = &OpenShiftConfig{} + convertVLabsOpenShiftConfig(vlabscs.OpenShiftConfig, api.OpenShiftConfig) + } + // Set api.KubernetesConfig to api.OpenShiftConfig.KubernetesConfig so + // acs-engine can reuse the same code used for generating parameters from + // KubernetesConfig for OpenShiftConfig. + if api.OpenShiftConfig != nil && api.OpenShiftConfig.KubernetesConfig != nil { + api.KubernetesConfig = api.OpenShiftConfig.KubernetesConfig + } + api.OrchestratorVersion = common.RationalizeReleaseAndVersion( + vlabscs.OrchestratorType, + vlabscs.OrchestratorRelease, + vlabscs.OrchestratorVersion, + false) case Kubernetes: if vlabscs.KubernetesConfig != nil { api.KubernetesConfig = &KubernetesConfig{} @@ -604,6 +630,18 @@ func convertVLabsDcosConfig(vlabs *vlabs.DcosConfig, api *DcosConfig) { api.DcosProviderPackageID = vlabs.DcosProviderPackageID } +func convertVLabsOpenShiftConfig(vlabs *vlabs.OpenShiftConfig, api *OpenShiftConfig) { + // NOTE: This is a hack to avoid breaking the rest of the acs-engine + // code when KubernetesConfig is accessed for various things. We don't + // use anything from it today. Maybe do something cleaner here. + api.KubernetesConfig = &KubernetesConfig{} + if vlabs.KubernetesConfig != nil { + convertVLabsKubernetesConfig(vlabs.KubernetesConfig, api.KubernetesConfig) + } + api.ClusterUsername = vlabs.ClusterUsername + api.ClusterPassword = vlabs.ClusterPassword +} + func convertVLabsKubernetesConfig(vlabs *vlabs.KubernetesConfig, api *KubernetesConfig) { api.KubernetesImageBase = vlabs.KubernetesImageBase api.ClusterSubnet = vlabs.ClusterSubnet @@ -933,6 +971,7 @@ func convertVLabsAgentPoolProfile(vlabs *vlabs.AgentPoolProfile, api *AgentPoolP api.ImageRef.Name = vlabs.ImageRef.Name api.ImageRef.ResourceGroup = vlabs.ImageRef.ResourceGroup } + api.Role = AgentPoolProfileRole(vlabs.Role) } func convertVLabsKeyVaultSecrets(vlabs *vlabs.KeyVaultSecrets, api *KeyVaultSecrets) { diff --git a/pkg/api/convertertoapi_test.go b/pkg/api/convertertoapi_test.go index a7c62d2ed..fd24a824c 100644 --- a/pkg/api/convertertoapi_test.go +++ b/pkg/api/convertertoapi_test.go @@ -3,6 +3,9 @@ package api import ( "testing" + "github.com/davecgh/go-spew/spew" + "k8s.io/apimachinery/pkg/api/equality" + "github.com/Azure/acs-engine/pkg/api/common" "github.com/Azure/acs-engine/pkg/api/v20170701" "github.com/Azure/acs-engine/pkg/api/vlabs" @@ -153,6 +156,61 @@ func TestKubernetesVlabsDefaults(t *testing.T) { } } +func TestConvertVLabsOrchestratorProfile(t *testing.T) { + tests := map[string]struct { + props *vlabs.Properties + expect *OrchestratorProfile + }{ + "nilOpenShiftConfig": { + props: &vlabs.Properties{ + OrchestratorProfile: &vlabs.OrchestratorProfile{ + OrchestratorType: OpenShift, + }, + }, + expect: &OrchestratorProfile{ + OrchestratorType: OpenShift, + OrchestratorVersion: common.OpenShiftDefaultVersion, + }, + }, + "setOpenShiftConfig": { + props: &vlabs.Properties{ + OrchestratorProfile: &vlabs.OrchestratorProfile{ + OrchestratorType: OpenShift, + OpenShiftConfig: &vlabs.OpenShiftConfig{ + KubernetesConfig: &vlabs.KubernetesConfig{ + NetworkPolicy: "azure", + ContainerRuntime: "docker", + }, + }, + }, + }, + expect: &OrchestratorProfile{ + OrchestratorType: OpenShift, + OrchestratorVersion: common.OpenShiftDefaultVersion, + KubernetesConfig: &KubernetesConfig{ + NetworkPolicy: "azure", + ContainerRuntime: "docker", + }, + OpenShiftConfig: &OpenShiftConfig{ + KubernetesConfig: &KubernetesConfig{ + NetworkPolicy: "azure", + ContainerRuntime: "docker", + }, + }, + }, + }, + } + + for name, test := range tests { + t.Logf("running scenario %q", name) + actual := &OrchestratorProfile{} + convertVLabsOrchestratorProfile(test.props, actual) + if !equality.Semantic.DeepEqual(test.expect, actual) { + t.Errorf(spew.Sprintf("Expected:\n%+v\nGot:\n%+v", test.expect, actual)) + } + } +} + func makeKubernetesProperties() *Properties { ap := &Properties{} ap.OrchestratorProfile = &OrchestratorProfile{} diff --git a/pkg/api/orchestrators.go b/pkg/api/orchestrators.go index 894b34477..5147b2edd 100644 --- a/pkg/api/orchestrators.go +++ b/pkg/api/orchestrators.go @@ -20,6 +20,7 @@ func init() { DCOS: dcosInfo, Swarm: swarmInfo, SwarmMode: dockerceInfo, + OpenShift: openShiftInfo, } } @@ -33,6 +34,8 @@ func validate(orchestrator, version string) (string, error) { return Swarm, nil case strings.EqualFold(orchestrator, SwarmMode): return SwarmMode, nil + case strings.EqualFold(orchestrator, OpenShift): + return OpenShift, nil case orchestrator == "": if version != "" { return "", fmt.Errorf("Must specify orchestrator for version '%s'", version) @@ -228,3 +231,44 @@ func dockerceInfo(csOrch *OrchestratorProfile) ([]*OrchestratorVersionProfile, e }, }, nil } + +func openShiftInfo(csOrch *OrchestratorProfile) ([]*OrchestratorVersionProfile, error) { + orchs := []*OrchestratorVersionProfile{} + if csOrch.OrchestratorVersion == "" { + // get info for all supported versions + for _, ver := range common.GetAllSupportedOpenShiftVersions() { + // TODO: populate OrchestratorVersionProfile.Upgrades + orchs = append(orchs, + &OrchestratorVersionProfile{ + OrchestratorProfile: OrchestratorProfile{ + OrchestratorType: OpenShift, + OrchestratorVersion: ver, + }, + Default: ver == common.OpenShiftDefaultVersion, + }) + } + } else { + ver, err := semver.NewVersion(csOrch.OrchestratorVersion) + if err != nil { + return nil, err + } + cons, err := semver.NewConstraint("<3.9.0") + if err != nil { + return nil, err + } + if cons.Check(ver) { + return nil, fmt.Errorf("OpenShift version %s is not supported", csOrch.OrchestratorVersion) + } + + // TODO: populate OrchestratorVersionProfile.Upgrades + orchs = append(orchs, + &OrchestratorVersionProfile{ + OrchestratorProfile: OrchestratorProfile{ + OrchestratorType: OpenShift, + OrchestratorVersion: csOrch.OrchestratorVersion, + }, + Default: csOrch.OrchestratorVersion == common.OpenShiftDefaultVersion, + }) + } + return orchs, nil +} diff --git a/pkg/api/orchestrators_test.go b/pkg/api/orchestrators_test.go index 8f1079d83..6b5428984 100644 --- a/pkg/api/orchestrators_test.go +++ b/pkg/api/orchestrators_test.go @@ -139,7 +139,13 @@ func TestGetOrchestratorVersionProfileListV20170930(t *testing.T) { Expect(e).To(BeNil()) numSwarmVersions := 1 numDockerCEVersions := 1 - totalNumVersions := numSwarmVersions + numDockerCEVersions + len(common.GetAllSupportedKubernetesVersions()) + len(common.AllDCOSSupportedVersions) + + totalNumVersions := numSwarmVersions + + numDockerCEVersions + + len(common.GetAllSupportedKubernetesVersions()) + + len(common.AllDCOSSupportedVersions) + + len(common.GetAllSupportedOpenShiftVersions()) + Expect(len(list.Properties.Orchestrators)).To(Equal(totalNumVersions)) // v20170930 - kubernetes only @@ -171,3 +177,35 @@ func TestKubernetesInfo(t *testing.T) { } } + +func TestOpenshiftInfo(t *testing.T) { + RegisterTestingT(t) + + invalid := []string{ + "invalid number", + "invalid.number", + "a4.b7.c3", + "31.29.", + ".17.02", + "43.156.89.", + "1.2.a"} + + for _, v := range invalid { + csOrch := &OrchestratorProfile{ + OrchestratorType: OpenShift, + OrchestratorVersion: v, + } + + _, e := openShiftInfo(csOrch) + Expect(e).NotTo(BeNil()) + } + + // test good value + csOrch := &OrchestratorProfile{ + OrchestratorType: OpenShift, + OrchestratorVersion: common.OpenShiftDefaultVersion, + } + + _, e := openShiftInfo(csOrch) + Expect(e).To(BeNil()) +} diff --git a/pkg/api/types.go b/pkg/api/types.go index 20759d5f2..40fffab25 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -59,6 +59,7 @@ type Properties struct { CustomProfile *CustomProfile `json:"customProfile,omitempty"` HostedMasterProfile *HostedMasterProfile `json:"hostedMasterProfile,omitempty"` AddonProfiles map[string]AddonProfile `json:"addonProfiles,omitempty"` + AzProfile *AzProfile `json:"azProfile,omitempty"` } // AddonProfile represents an addon for managed cluster @@ -67,6 +68,14 @@ type AddonProfile struct { Config map[string]string `json:"config"` } +// AzProfile holds the azure context for where the cluster resides +type AzProfile struct { + TenantID string `json:"tenantId,omitempty"` + SubscriptionID string `json:"subscriptionId,omitempty"` + ResourceGroup string `json:"resourceGroup,omitempty"` + Location string `json:"location,omitempty"` +} + // ServicePrincipalProfile contains the client and secret used by the cluster for Azure Resource CRUD type ServicePrincipalProfile struct { ClientID string `json:"clientId"` @@ -165,6 +174,7 @@ type OrchestratorProfile struct { OrchestratorType string `json:"orchestratorType"` OrchestratorVersion string `json:"orchestratorVersion"` KubernetesConfig *KubernetesConfig `json:"kubernetesConfig,omitempty"` + OpenShiftConfig *OpenShiftConfig `json:"openshiftConfig,omitempty"` DcosConfig *DcosConfig `json:"dcosConfig,omitempty"` } @@ -305,6 +315,20 @@ type DcosConfig struct { DcosProviderPackageID string `json:"dcosProviderPackageID,omitempty"` // repo url is the location of the build, } +// OpenShiftConfig holds configuration for OpenShift +type OpenShiftConfig struct { + KubernetesConfig *KubernetesConfig `json:"kubernetesConfig,omitempty"` + + // ClusterUsername and ClusterPassword are temporary before AAD + // authentication is enabled, and will be removed subsequently. + ClusterUsername string `json:"clusterUsername,omitempty"` + ClusterPassword string `json:"clusterPassword,omitempty"` + + ConfigBundles map[string][]byte `json:"-"` + ExternalMasterHostname string `json:"-"` + RouterLBHostname string `json:"-"` +} + // MasterProfile represents the definition of the master cluster type MasterProfile struct { Count int `json:"count"` @@ -358,20 +382,21 @@ type Extension struct { // AgentPoolProfile represents an agent pool definition type AgentPoolProfile struct { - Name string `json:"name"` - Count int `json:"count"` - VMSize string `json:"vmSize"` - OSDiskSizeGB int `json:"osDiskSizeGB,omitempty"` - DNSPrefix string `json:"dnsPrefix,omitempty"` - OSType OSType `json:"osType,omitempty"` - Ports []int `json:"ports,omitempty"` - AvailabilityProfile string `json:"availabilityProfile"` - StorageProfile string `json:"storageProfile,omitempty"` - DiskSizesGB []int `json:"diskSizesGB,omitempty"` - VnetSubnetID string `json:"vnetSubnetID,omitempty"` - Subnet string `json:"subnet"` - IPAddressCount int `json:"ipAddressCount,omitempty"` - Distro Distro `json:"distro,omitempty"` + Name string `json:"name"` + Count int `json:"count"` + VMSize string `json:"vmSize"` + OSDiskSizeGB int `json:"osDiskSizeGB,omitempty"` + DNSPrefix string `json:"dnsPrefix,omitempty"` + OSType OSType `json:"osType,omitempty"` + Ports []int `json:"ports,omitempty"` + AvailabilityProfile string `json:"availabilityProfile"` + StorageProfile string `json:"storageProfile,omitempty"` + DiskSizesGB []int `json:"diskSizesGB,omitempty"` + VnetSubnetID string `json:"vnetSubnetID,omitempty"` + Subnet string `json:"subnet"` + IPAddressCount int `json:"ipAddressCount,omitempty"` + Distro Distro `json:"distro,omitempty"` + Role AgentPoolProfileRole `json:"role,omitempty"` FQDN string `json:"fqdn,omitempty"` CustomNodeLabels map[string]string `json:"customNodeLabels,omitempty"` @@ -381,6 +406,9 @@ type AgentPoolProfile struct { ImageRef *ImageReference `json:"imageReference,omitempty"` } +// AgentPoolProfileRole represents an agent role +type AgentPoolProfileRole string + // DiagnosticsProfile setting to enable/disable capturing // diagnostics for VMs hosting container cluster. type DiagnosticsProfile struct { @@ -682,6 +710,11 @@ func (o *OrchestratorProfile) IsKubernetes() bool { return o.OrchestratorType == Kubernetes } +// IsOpenShift returns true if this template is for OpenShift orchestrator +func (o *OrchestratorProfile) IsOpenShift() bool { + return o.OrchestratorType == OpenShift +} + // IsDCOS returns true if this template is for DCOS orchestrator func (o *OrchestratorProfile) IsDCOS() bool { return o.OrchestratorType == DCOS diff --git a/pkg/api/vlabs/const.go b/pkg/api/vlabs/const.go index 8e0860494..d759b5e89 100644 --- a/pkg/api/vlabs/const.go +++ b/pkg/api/vlabs/const.go @@ -15,6 +15,8 @@ const ( Kubernetes string = "Kubernetes" // SwarmMode is the string constant for the Swarm Mode orchestrator type SwarmMode string = "SwarmMode" + // OpenShift is the string constant for the OpenShift orchestrator type + OpenShift string = "OpenShift" ) // the OSTypes supported by vlabs @@ -89,3 +91,10 @@ const ( // DefaultNetworkPolicyWindows defines the network policy to use by default for clusters with Windows agent pools DefaultNetworkPolicyWindows = "azure" ) + +const ( + // AgentPoolProfileRoleEmpty is the empty role + AgentPoolProfileRoleEmpty AgentPoolProfileRole = "" + // AgentPoolProfileRoleInfra is the infra role + AgentPoolProfileRoleInfra AgentPoolProfileRole = "infra" +) diff --git a/pkg/api/vlabs/types.go b/pkg/api/vlabs/types.go index 011402d7f..fb7f447f1 100644 --- a/pkg/api/vlabs/types.go +++ b/pkg/api/vlabs/types.go @@ -40,6 +40,15 @@ type Properties struct { ServicePrincipalProfile *ServicePrincipalProfile `json:"servicePrincipalProfile,omitempty"` CertificateProfile *CertificateProfile `json:"certificateProfile,omitempty"` AADProfile *AADProfile `json:"aadProfile,omitempty"` + AzProfile *AzProfile `json:"azProfile,omitempty"` +} + +// AzProfile holds the azure context for where the cluster resides +type AzProfile struct { + TenantID string `json:"tenantId,omitempty"` + SubscriptionID string `json:"subscriptionId,omitempty"` + ResourceGroup string `json:"resourceGroup,omitempty"` + Location string `json:"location,omitempty"` } // ServicePrincipalProfile contains the client and secret used by the cluster for Azure Resource CRUD @@ -157,6 +166,7 @@ type OrchestratorProfile struct { OrchestratorRelease string `json:"orchestratorRelease,omitempty"` OrchestratorVersion string `json:"orchestratorVersion,omitempty"` KubernetesConfig *KubernetesConfig `json:"kubernetesConfig,omitempty"` + OpenShiftConfig *OpenShiftConfig `json:"openshiftConfig,omitempty"` DcosConfig *DcosConfig `json:"dcosConfig,omitempty"` } @@ -181,6 +191,8 @@ func (o *OrchestratorProfile) UnmarshalJSON(b []byte) error { o.OrchestratorType = Kubernetes case strings.EqualFold(orchestratorType, SwarmMode): o.OrchestratorType = SwarmMode + case strings.EqualFold(orchestratorType, OpenShift): + o.OrchestratorType = OpenShift default: return fmt.Errorf("OrchestratorType has unknown orchestrator: %s", orchestratorType) } @@ -297,6 +309,16 @@ type DcosConfig struct { DcosProviderPackageID string `json:"dcosProviderPackageID,omitempty"` // repo url is the location of the build, } +// OpenShiftConfig holds configuration for OpenShift +type OpenShiftConfig struct { + KubernetesConfig *KubernetesConfig `json:"kubernetesConfig,omitempty"` + + // ClusterUsername and ClusterPassword are temporary before AAD + // authentication is enabled, and will be removed subsequently. + ClusterUsername string `json:"clusterUsername,omitempty"` + ClusterPassword string `json:"clusterPassword,omitempty"` +} + // MasterProfile represents the definition of the master cluster type MasterProfile struct { Count int `json:"count" validate:"required,eq=1|eq=3|eq=5"` @@ -355,21 +377,22 @@ type Extension struct { // AgentPoolProfile represents an agent pool definition type AgentPoolProfile struct { - Name string `json:"name" validate:"required"` - Count int `json:"count" validate:"required,min=1,max=100"` - VMSize string `json:"vmSize" validate:"required"` - OSDiskSizeGB int `json:"osDiskSizeGB,omitempty" validate:"min=0,max=1023"` - DNSPrefix string `json:"dnsPrefix,omitempty"` - OSType OSType `json:"osType,omitempty"` - Ports []int `json:"ports,omitempty" validate:"dive,min=1,max=65535"` - AvailabilityProfile string `json:"availabilityProfile"` - StorageProfile string `json:"storageProfile" validate:"eq=StorageAccount|eq=ManagedDisks|len=0"` - DiskSizesGB []int `json:"diskSizesGB,omitempty" validate:"max=4,dive,min=1,max=1023"` - VnetSubnetID string `json:"vnetSubnetID,omitempty"` - IPAddressCount int `json:"ipAddressCount,omitempty" validate:"min=0,max=256"` - Distro Distro `json:"distro,omitempty"` - KubernetesConfig *KubernetesConfig `json:"kubernetesConfig,omitempty"` - ImageRef *ImageReference `json:"imageReference,omitempty"` + Name string `json:"name" validate:"required"` + Count int `json:"count" validate:"required,min=1,max=100"` + VMSize string `json:"vmSize" validate:"required"` + OSDiskSizeGB int `json:"osDiskSizeGB,omitempty" validate:"min=0,max=1023"` + DNSPrefix string `json:"dnsPrefix,omitempty"` + OSType OSType `json:"osType,omitempty"` + Ports []int `json:"ports,omitempty" validate:"dive,min=1,max=65535"` + AvailabilityProfile string `json:"availabilityProfile"` + StorageProfile string `json:"storageProfile" validate:"eq=StorageAccount|eq=ManagedDisks|len=0"` + DiskSizesGB []int `json:"diskSizesGB,omitempty" validate:"max=4,dive,min=1,max=1023"` + VnetSubnetID string `json:"vnetSubnetID,omitempty"` + IPAddressCount int `json:"ipAddressCount,omitempty" validate:"min=0,max=256"` + Distro Distro `json:"distro,omitempty"` + KubernetesConfig *KubernetesConfig `json:"kubernetesConfig,omitempty"` + ImageRef *ImageReference `json:"imageReference,omitempty"` + Role AgentPoolProfileRole `json:"role,omitempty"` // subnet is internal subnet string @@ -380,6 +403,9 @@ type AgentPoolProfile struct { Extensions []Extension `json:"extensions"` } +// AgentPoolProfileRole represents an agent role +type AgentPoolProfileRole string + // AADProfile specifies attributes for AAD integration type AADProfile struct { // The client AAD application ID. diff --git a/pkg/api/vlabs/validate.go b/pkg/api/vlabs/validate.go index dcfb1cdde..5447928bd 100644 --- a/pkg/api/vlabs/validate.go +++ b/pkg/api/vlabs/validate.go @@ -143,7 +143,19 @@ func (o *OrchestratorProfile) Validate(isUpdate bool) error { } } } - + case OpenShift: + // TODO: add appropriate additional validation logic + version := common.RationalizeReleaseAndVersion( + o.OrchestratorType, + o.OrchestratorRelease, + o.OrchestratorVersion, + false) + if version == "" { + return fmt.Errorf("OrchestratorProfile is not able to be rationalized, check supported Release or Version") + } + if o.OpenShiftConfig == nil || o.OpenShiftConfig.ClusterUsername == "" || o.OpenShiftConfig.ClusterPassword == "" { + return fmt.Errorf("ClusterUsername and ClusterPassword must both be specified") + } default: return fmt.Errorf("OrchestratorProfile has unknown orchestrator: %s", o.OrchestratorType) } @@ -167,8 +179,12 @@ func (o *OrchestratorProfile) Validate(isUpdate bool) error { } } - if o.OrchestratorType != Kubernetes && o.KubernetesConfig != nil { - return fmt.Errorf("KubernetesConfig can be specified only when OrchestratorType is Kubernetes") + if (o.OrchestratorType != Kubernetes && o.OrchestratorType != OpenShift) && o.KubernetesConfig != nil { + return fmt.Errorf("KubernetesConfig can be specified only when OrchestratorType is Kubernetes or OpenShift") + } + + if o.OrchestratorType != OpenShift && o.OpenShiftConfig != nil { + return fmt.Errorf("OpenShiftConfig can be specified only when OrchestratorType is OpenShift") } if o.OrchestratorType != DCOS && o.DcosConfig != nil && (*o.DcosConfig != DcosConfig{}) { @@ -405,12 +421,32 @@ func (a *Properties) Validate(isUpdate bool) error { } } + if a.OrchestratorProfile.OrchestratorType == OpenShift && agentPoolProfile.AvailabilityProfile != AvailabilitySet { + return fmt.Errorf("Only AvailabilityProfile: AvailabilitySet is supported for Orchestrator 'OpenShift'") + } + + validRoles := []AgentPoolProfileRole{AgentPoolProfileRoleEmpty} + if a.OrchestratorProfile.OrchestratorType == OpenShift { + validRoles = append(validRoles, AgentPoolProfileRoleInfra) + } + var found bool + for _, validRole := range validRoles { + if agentPoolProfile.Role == validRole { + found = true + break + } + } + if !found { + return fmt.Errorf("Role %q is not supported for Orchestrator %s", agentPoolProfile.Role, a.OrchestratorProfile.OrchestratorType) + } + /* this switch statement is left to protect newly added orchestrators until they support Managed Disks*/ if agentPoolProfile.StorageProfile == ManagedDisks { switch a.OrchestratorProfile.OrchestratorType { case DCOS: case Swarm: case Kubernetes: + case OpenShift: case SwarmMode: default: return fmt.Errorf("HA volumes are currently unsupported for Orchestrator %s", a.OrchestratorProfile.OrchestratorType) @@ -489,6 +525,19 @@ func (a *Properties) Validate(isUpdate bool) error { } } + switch a.OrchestratorProfile.OrchestratorType { + case OpenShift: + if a.AzProfile == nil || a.AzProfile.Location == "" || + a.AzProfile.ResourceGroup == "" || a.AzProfile.SubscriptionID == "" || + a.AzProfile.TenantID == "" { + return fmt.Errorf("'azProfile' must be supplied in full for orchestrator '%v'", OpenShift) + } + default: + if a.AzProfile != nil { + return fmt.Errorf("'azProfile' is only supported by orchestrator '%v'", OpenShift) + } + } + for _, extension := range a.ExtensionProfiles { if extension.ExtensionParametersKeyVaultRef != nil { if e := validate.Var(extension.ExtensionParametersKeyVaultRef.VaultID, "required"); e != nil { diff --git a/pkg/api/vlabs/validate_test.go b/pkg/api/vlabs/validate_test.go index e722d2fab..32eb9a9ef 100644 --- a/pkg/api/vlabs/validate_test.go +++ b/pkg/api/vlabs/validate_test.go @@ -77,6 +77,26 @@ func Test_OrchestratorProfile_Validate(t *testing.T) { t.Errorf("should not have failed on version with v prefix") } + o = &OrchestratorProfile{ + OrchestratorType: OpenShift, + OrchestratorVersion: "v1.0", + } + + if err := o.Validate(false); err == nil { + t.Errorf("should have failed on old version") + } + if err := o.Validate(true); err != nil { + t.Errorf("should not have failed on old version") + } + + o = &OrchestratorProfile{ + OrchestratorType: Kubernetes, + OrchestratorVersion: "v1.9.0", + OpenShiftConfig: &OpenShiftConfig{}, + } + if err := o.Validate(false); err == nil { + t.Errorf("should have failed on OpenShift config specified with non OpenShift orchestrator type") + } } func Test_KubernetesConfig_Validate(t *testing.T) { diff --git a/pkg/openshift/certgen/config.go b/pkg/openshift/certgen/config.go new file mode 100644 index 000000000..2811589dc --- /dev/null +++ b/pkg/openshift/certgen/config.go @@ -0,0 +1,105 @@ +package certgen + +import ( + "crypto/rsa" + "crypto/x509" + "math/big" + "net" + "sync" + + "github.com/Azure/acs-engine/pkg/openshift/filesystem" +) + +// Config represents an OpenShift configuration +type Config struct { + ExternalMasterHostname string + serial serial + cas map[string]CertAndKey + AuthSecret string + EncSecret string + Master *Master + Bootstrap KubeConfig + ClusterUsername string + ClusterPassword string + AzureConfig AzureConfig +} + +// AzureConfig represents the azure.conf configuration +type AzureConfig struct { + TenantID string + SubscriptionID string + AADClientID string + AADClientSecret string + ResourceGroup string + Location string +} + +// Master represents an OpenShift master configuration +type Master struct { + Hostname string + IPs []net.IP + Port int16 + + certs map[string]CertAndKey + etcdcerts map[string]CertAndKey + kubeconfigs map[string]KubeConfig +} + +// CertAndKey is a certificate and key +type CertAndKey struct { + cert *x509.Certificate + key *rsa.PrivateKey +} + +type serial struct { + m sync.Mutex + i int64 +} + +func (s *serial) Get() *big.Int { + s.m.Lock() + defer s.m.Unlock() + + s.i++ + return big.NewInt(s.i) +} + +// WriteMaster writes the config files for a Master node to a Filesystem. +func (c *Config) WriteMaster(fs filesystem.Filesystem) error { + err := c.WriteMasterCerts(fs) + if err != nil { + return err + } + + err = c.WriteMasterKeypair(fs) + if err != nil { + return err + } + + err = c.WriteMasterKubeConfigs(fs) + if err != nil { + return err + } + + err = c.WriteMasterFiles(fs) + if err != nil { + return err + } + + return c.WriteNode(fs) +} + +// WriteNode writes the config files for bootstrapping a node to a Filesystem. +func (c *Config) WriteNode(fs filesystem.Filesystem) error { + err := c.WriteBootstrapCerts(fs) + if err != nil { + return err + } + + err = c.WriteBootstrapKubeConfig(fs) + if err != nil { + return err + } + + return c.WriteNodeFiles(fs) +} diff --git a/pkg/openshift/certgen/doc.go b/pkg/openshift/certgen/doc.go new file mode 100644 index 000000000..5d8d03db1 --- /dev/null +++ b/pkg/openshift/certgen/doc.go @@ -0,0 +1,4 @@ +// Package certgen provides utilities related +// to generating all the necessary artifacts +// for an OpenShift deployment. +package certgen diff --git a/pkg/openshift/certgen/files.go b/pkg/openshift/certgen/files.go new file mode 100644 index 000000000..f3c6dd888 --- /dev/null +++ b/pkg/openshift/certgen/files.go @@ -0,0 +1,96 @@ +package certgen + +import ( + "bytes" + "crypto/rand" + "encoding/base64" + "regexp" + "strings" + "text/template" + + "github.com/Azure/acs-engine/pkg/openshift/filesystem" + "golang.org/x/crypto/bcrypt" +) + +// PrepareMasterFiles creates the shared authentication and encryption secrets +func (c *Config) PrepareMasterFiles() error { + b := make([]byte, 24) + _, err := rand.Read(b) + if err != nil { + return err + } + c.AuthSecret = base64.StdEncoding.EncodeToString(b) + + _, err = rand.Read(b) + if err != nil { + return err + } + c.EncSecret = base64.StdEncoding.EncodeToString(b) + + return nil +} + +// WriteMasterFiles writes the templated master config +func (c *Config) WriteMasterFiles(fs filesystem.Filesystem) error { + for _, name := range getAssets() { + if !strings.HasPrefix(name, "master/") { + continue + } + tb := assetMustExist(name) + + t, err := template.New("template").Funcs(template.FuncMap{ + "QuoteMeta": regexp.QuoteMeta, + "Bcrypt": func(password string) (string, error) { + h, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost) + return string(h), err + }, + }).Parse(string(tb)) + if err != nil { + return err + } + + b := &bytes.Buffer{} + err = t.Execute(b, c) + if err != nil { + return err + } + + err = fs.WriteFile(strings.TrimPrefix(name, "master/"), b.Bytes(), 0666) + if err != nil { + return err + } + } + + return nil +} + +// WriteNodeFiles writes the templated node config +func (c *Config) WriteNodeFiles(fs filesystem.Filesystem) error { + for _, name := range getAssets() { + if !strings.HasPrefix(name, "node/") { + continue + } + + tb := assetMustExist(name) + + t, err := template.New("template").Funcs(template.FuncMap{ + "QuoteMeta": regexp.QuoteMeta, + }).Parse(string(tb)) + if err != nil { + return err + } + + b := &bytes.Buffer{} + err = t.Execute(b, c) + if err != nil { + return err + } + + err = fs.WriteFile(strings.TrimPrefix(name, "node/"), b.Bytes(), 0666) + if err != nil { + return err + } + } + + return nil +} diff --git a/pkg/openshift/certgen/kubeconfig.go b/pkg/openshift/certgen/kubeconfig.go new file mode 100644 index 000000000..0bfc0879d --- /dev/null +++ b/pkg/openshift/certgen/kubeconfig.go @@ -0,0 +1,284 @@ +package certgen + +import ( + "encoding/base64" + "fmt" + "strings" + + "github.com/Azure/acs-engine/pkg/openshift/filesystem" + "gopkg.in/yaml.v2" +) + +// KubeConfig represents a kubeconfig +type KubeConfig struct { + APIVersion string `yaml:"apiVersion,omitempty"` + Kind string `yaml:"kind,omitempty"` + Clusters []Cluster `yaml:"clusters,omitempty"` + Contexts []Context `yaml:"contexts,omitempty"` + CurrentContext string `yaml:"current-context,omitempty"` + Preferences map[string]interface{} `yaml:"preferences,omitempty"` + Users []User `yaml:"users,omitempty"` +} + +// Cluster represents a kubeconfig cluster +type Cluster struct { + Name string `yaml:"name,omitempty"` + Cluster ClusterInfo `yaml:"cluster,omitempty"` +} + +// ClusterInfo represents a kubeconfig clusterinfo +type ClusterInfo struct { + Server string `yaml:"server,omitempty"` + CertificateAuthorityData string `yaml:"certificate-authority-data,omitempty"` +} + +// Context represents a kubeconfig context +type Context struct { + Name string `yaml:"name,omitempty"` + Context ContextInfo `yaml:"context,omitempty"` +} + +// ContextInfo represents a kubeconfig contextinfo +type ContextInfo struct { + Cluster string `yaml:"cluster,omitempty"` + Namespace string `yaml:"namespace,omitempty"` + User string `yaml:"user,omitempty"` +} + +// User represents a kubeconfig user +type User struct { + Name string `yaml:"name,omitempty"` + User UserInfo `yaml:"user,omitempty"` +} + +// UserInfo represents a kubeconfig userinfo +type UserInfo struct { + ClientCertificateData string `yaml:"client-certificate-data,omitempty"` + ClientKeyData string `yaml:"client-key-data,omitempty"` +} + +// PrepareMasterKubeConfigs creates the master kubeconfigs +func (c *Config) PrepareMasterKubeConfigs() error { + endpoint := fmt.Sprintf("%s:%d", c.Master.Hostname, c.Master.Port) + endpointName := strings.Replace(endpoint, ".", "-", -1) + + externalEndpoint := fmt.Sprintf("%s:%d", c.ExternalMasterHostname, c.Master.Port) + externalEndpointName := strings.Replace(externalEndpoint, ".", "-", -1) + + localhostEndpoint := fmt.Sprintf("localhost:%d", c.Master.Port) + localhostEndpointName := strings.Replace(localhostEndpoint, ".", "-", -1) + + cacert, err := certAsBytes(c.cas["etc/origin/master/ca"].cert) + if err != nil { + return err + } + admincert, err := certAsBytes(c.Master.certs["etc/origin/master/admin"].cert) + if err != nil { + return err + } + adminkey, err := privateKeyAsBytes(c.Master.certs["etc/origin/master/admin"].key) + if err != nil { + return err + } + mastercert, err := certAsBytes(c.Master.certs["etc/origin/master/openshift-master"].cert) + if err != nil { + return err + } + masterkey, err := privateKeyAsBytes(c.Master.certs["etc/origin/master/openshift-master"].key) + if err != nil { + return err + } + aggregatorcert, err := certAsBytes(c.Master.certs["etc/origin/master/aggregator-front-proxy"].cert) + if err != nil { + return err + } + aggregatorkey, err := privateKeyAsBytes(c.Master.certs["etc/origin/master/aggregator-front-proxy"].key) + if err != nil { + return err + } + + c.Master.kubeconfigs = map[string]KubeConfig{ + "etc/origin/master/admin.kubeconfig": { + APIVersion: "v1", + Kind: "Config", + Clusters: []Cluster{ + { + Name: externalEndpointName, + Cluster: ClusterInfo{ + Server: fmt.Sprintf("https://%s", externalEndpoint), + CertificateAuthorityData: base64.StdEncoding.EncodeToString(cacert), + }, + }, + }, + Contexts: []Context{ + { + Name: fmt.Sprintf("default/%s/system:admin", externalEndpointName), + Context: ContextInfo{ + Cluster: externalEndpointName, + Namespace: "default", + User: fmt.Sprintf("system:admin/%s", externalEndpointName), + }, + }, + }, + CurrentContext: fmt.Sprintf("default/%s/system:admin", externalEndpointName), + Users: []User{ + { + Name: fmt.Sprintf("system:admin/%s", externalEndpointName), + User: UserInfo{ + ClientCertificateData: base64.StdEncoding.EncodeToString(admincert), + ClientKeyData: base64.StdEncoding.EncodeToString(adminkey), + }, + }, + }, + }, + "etc/origin/master/aggregator-front-proxy.kubeconfig": { + APIVersion: "v1", + Kind: "Config", + Clusters: []Cluster{ + { + Name: localhostEndpointName, + Cluster: ClusterInfo{ + Server: fmt.Sprintf("https://%s", localhostEndpoint), + CertificateAuthorityData: base64.StdEncoding.EncodeToString(cacert), + }, + }, + }, + Contexts: []Context{ + { + Name: fmt.Sprintf("default/%s/aggregator-front-proxy", localhostEndpointName), + Context: ContextInfo{ + Cluster: localhostEndpointName, + Namespace: "default", + User: fmt.Sprintf("aggregator-front-proxy/%s", localhostEndpointName), + }, + }, + }, + CurrentContext: fmt.Sprintf("default/%s/aggregator-front-proxy", localhostEndpointName), + Users: []User{ + { + Name: fmt.Sprintf("aggregator-front-proxy/%s", localhostEndpointName), + User: UserInfo{ + ClientCertificateData: base64.StdEncoding.EncodeToString(aggregatorcert), + ClientKeyData: base64.StdEncoding.EncodeToString(aggregatorkey), + }, + }, + }, + }, + "etc/origin/master/openshift-master.kubeconfig": { + APIVersion: "v1", + Kind: "Config", + Clusters: []Cluster{ + { + Name: endpointName, + Cluster: ClusterInfo{ + Server: fmt.Sprintf("https://%s", endpoint), + CertificateAuthorityData: base64.StdEncoding.EncodeToString(cacert), + }, + }, + }, + Contexts: []Context{ + { + Name: fmt.Sprintf("default/%s/system:openshift-master", endpointName), + Context: ContextInfo{ + Cluster: endpointName, + Namespace: "default", + User: fmt.Sprintf("system:openshift-master/%s", endpointName), + }, + }, + }, + CurrentContext: fmt.Sprintf("default/%s/system:openshift-master", endpointName), + Users: []User{ + { + Name: fmt.Sprintf("system:openshift-master/%s", endpointName), + User: UserInfo{ + ClientCertificateData: base64.StdEncoding.EncodeToString(mastercert), + ClientKeyData: base64.StdEncoding.EncodeToString(masterkey), + }, + }, + }, + }, + } + + return nil +} + +// PrepareBootstrapKubeConfig creates the node bootstrap kubeconfig +func (c *Config) PrepareBootstrapKubeConfig() error { + ep := fmt.Sprintf("%s:%d", c.ExternalMasterHostname, c.Master.Port) + epName := strings.Replace(ep, ".", "-", -1) + + cacert, err := certAsBytes(c.cas["etc/origin/master/ca"].cert) + if err != nil { + return err + } + + bootstrapCert, err := certAsBytes(c.Master.certs["etc/origin/master/node-bootstrapper"].cert) + if err != nil { + return err + } + bootstrapKey, err := privateKeyAsBytes(c.Master.certs["etc/origin/master/node-bootstrapper"].key) + if err != nil { + return err + } + + c.Bootstrap = KubeConfig{ + APIVersion: "v1", + Kind: "Config", + Clusters: []Cluster{ + { + Name: epName, + Cluster: ClusterInfo{ + Server: fmt.Sprintf("https://%s", ep), + CertificateAuthorityData: base64.StdEncoding.EncodeToString(cacert), + }, + }, + }, + Contexts: []Context{ + { + Name: fmt.Sprintf("default/%s/system:serviceaccount:openshift-infra:node-bootstrapper", epName), + Context: ContextInfo{ + Cluster: epName, + Namespace: "default", + User: fmt.Sprintf("system:serviceaccount:openshift-infra:node-bootstrapper/%s", epName), + }, + }, + }, + CurrentContext: fmt.Sprintf("default/%s/system:serviceaccount:openshift-infra:node-bootstrapper", epName), + Users: []User{ + { + Name: fmt.Sprintf("system:serviceaccount:openshift-infra:node-bootstrapper/%s", epName), + User: UserInfo{ + ClientCertificateData: base64.StdEncoding.EncodeToString(bootstrapCert), + ClientKeyData: base64.StdEncoding.EncodeToString(bootstrapKey), + }, + }, + }, + } + + return nil +} + +// WriteMasterKubeConfigs writes the master kubeconfigs +func (c *Config) WriteMasterKubeConfigs(fs filesystem.Filesystem) error { + for filename, kubeconfig := range c.Master.kubeconfigs { + b, err := yaml.Marshal(&kubeconfig) + if err != nil { + return err + } + err = fs.WriteFile(filename, b, 0600) + if err != nil { + return err + } + } + + return nil +} + +// WriteBootstrapKubeConfig writes the node bootstrap kubeconfig +func (c *Config) WriteBootstrapKubeConfig(fs filesystem.Filesystem) error { + b, err := yaml.Marshal(&c.Bootstrap) + if err != nil { + return err + } + return fs.WriteFile("etc/origin/node/bootstrap.kubeconfig", b, 0600) +} diff --git a/pkg/openshift/certgen/novalidate.go b/pkg/openshift/certgen/novalidate.go new file mode 100644 index 000000000..26d011593 --- /dev/null +++ b/pkg/openshift/certgen/novalidate.go @@ -0,0 +1,19 @@ +package certgen + +// novalidate.go is split out of files.go to avoid static validation. +// `make test-style` is failing non-deterministically (flakying) with +// the following message: +// +// pkg/certgen/files.go:36:23:warning: AssetNames not declared by package templates (unused) + +import ( + "github.com/Azure/acs-engine/pkg/openshift/certgen/templates" +) + +func getAssets() []string { + return templates.AssetNames() +} + +func assetMustExist(name string) []byte { + return templates.MustAsset(name) +} diff --git a/pkg/openshift/certgen/templates/master/etc/etcd/etcd.conf b/pkg/openshift/certgen/templates/master/etc/etcd/etcd.conf new file mode 100644 index 000000000..960318ce6 --- /dev/null +++ b/pkg/openshift/certgen/templates/master/etc/etcd/etcd.conf @@ -0,0 +1,57 @@ +ETCD_NAME={{ .Master.Hostname }} +ETCD_LISTEN_PEER_URLS=https://{{ (index .Master.IPs 0).String }}:2380 +ETCD_DATA_DIR=/var/lib/etcd/ +#ETCD_WAL_DIR="" +#ETCD_SNAPSHOT_COUNT=10000 +ETCD_HEARTBEAT_INTERVAL=500 +ETCD_ELECTION_TIMEOUT=2500 +ETCD_LISTEN_CLIENT_URLS=https://{{ (index .Master.IPs 0).String }}:2379 +#ETCD_MAX_SNAPSHOTS=5 +#ETCD_MAX_WALS=5 +#ETCD_CORS= + + +#[cluster] +ETCD_INITIAL_ADVERTISE_PEER_URLS=https://{{ (index .Master.IPs 0).String }}:2380 +ETCD_INITIAL_CLUSTER={{ .Master.Hostname }}=https://{{ (index .Master.IPs 0).String }}:2380 +ETCD_INITIAL_CLUSTER_STATE=new +ETCD_INITIAL_CLUSTER_TOKEN=etcd-cluster-1 +#ETCD_DISCOVERY= +#ETCD_DISCOVERY_SRV= +#ETCD_DISCOVERY_FALLBACK=proxy +#ETCD_DISCOVERY_PROXY= +ETCD_ADVERTISE_CLIENT_URLS=https://{{ (index .Master.IPs 0).String }}:2379 +#ETCD_STRICT_RECONFIG_CHECK="false" +#ETCD_AUTO_COMPACTION_RETENTION="0" +#ETCD_ENABLE_V2="true" +ETCD_QUOTA_BACKEND_BYTES=4294967296 + +#[proxy] +#ETCD_PROXY=off +#ETCD_PROXY_FAILURE_WAIT="5000" +#ETCD_PROXY_REFRESH_INTERVAL="30000" +#ETCD_PROXY_DIAL_TIMEOUT="1000" +#ETCD_PROXY_WRITE_TIMEOUT="5000" +#ETCD_PROXY_READ_TIMEOUT="0" + +#[security] +ETCD_TRUSTED_CA_FILE=/etc/etcd/ca.crt +ETCD_CLIENT_CERT_AUTH="true" +ETCD_CERT_FILE=/etc/etcd/server.crt +ETCD_KEY_FILE=/etc/etcd/server.key +#ETCD_AUTO_TLS="false" +ETCD_PEER_TRUSTED_CA_FILE=/etc/etcd/ca.crt +ETCD_PEER_CLIENT_CERT_AUTH="true" +ETCD_PEER_CERT_FILE=/etc/etcd/peer.crt +ETCD_PEER_KEY_FILE=/etc/etcd/peer.key +#ETCD_PEER_AUTO_TLS="false" + +#[logging] +ETCD_DEBUG="False" + +#[profiling] +#ETCD_ENABLE_PPROF="false" +#ETCD_METRICS="basic" +# +#[auth] +#ETCD_AUTH_TOKEN="simple" diff --git a/pkg/openshift/certgen/templates/master/etc/origin/master/htpasswd b/pkg/openshift/certgen/templates/master/etc/origin/master/htpasswd new file mode 100644 index 000000000..893fd8848 --- /dev/null +++ b/pkg/openshift/certgen/templates/master/etc/origin/master/htpasswd @@ -0,0 +1 @@ +{{ .ClusterUsername }}:{{ Bcrypt .ClusterPassword }} diff --git a/pkg/openshift/certgen/templates/master/etc/origin/master/master-config.yaml b/pkg/openshift/certgen/templates/master/etc/origin/master/master-config.yaml new file mode 100644 index 000000000..8140b638b --- /dev/null +++ b/pkg/openshift/certgen/templates/master/etc/origin/master/master-config.yaml @@ -0,0 +1,211 @@ +admissionConfig: + pluginConfig: + BuildDefaults: + configuration: + apiVersion: v1 + env: [] + kind: BuildDefaultsConfig + resources: + limits: {} + requests: {} + BuildOverrides: + configuration: + apiVersion: v1 + kind: BuildOverridesConfig + PodPreset: + configuration: + apiVersion: v1 + disable: false + kind: DefaultAdmissionConfig + openshift.io/ImagePolicy: + configuration: + apiVersion: v1 + executionRules: + - matchImageAnnotations: + - key: images.openshift.io/deny-execution + value: 'true' + name: execution-denied + onResources: + - resource: pods + - resource: builds + reject: true + skipOnResolutionFailure: true + kind: ImagePolicyConfig +aggregatorConfig: + proxyClientInfo: + certFile: aggregator-front-proxy.crt + keyFile: aggregator-front-proxy.key +apiLevels: +- v1 +apiVersion: v1 +authConfig: + requestHeader: + clientCA: front-proxy-ca.crt + clientCommonNames: + - aggregator-front-proxy + extraHeaderPrefixes: + - X-Remote-Extra- + groupHeaders: + - X-Remote-Group + usernameHeaders: + - X-Remote-User +controllerConfig: + election: + lockName: openshift-master-controllers + serviceServingCert: + signer: + certFile: service-signer.crt + keyFile: service-signer.key +controllers: '*' +corsAllowedOrigins: +- (?i)//127\.0\.0\.1(:|\z) +- (?i)//localhost(:|\z) +- (?i)//{{ QuoteMeta (index .Master.IPs 0).String }}(:|\z) +- (?i)//kubernetes\.default(:|\z) +- (?i)//kubernetes\.default\.svc\.cluster\.local(:|\z) +- (?i)//kubernetes(:|\z) +- (?i)//{{ QuoteMeta .ExternalMasterHostname }}(:|\z) +- (?i)//openshift\.default(:|\z) +- (?i)//{{ QuoteMeta .Master.Hostname }}(:|\z) +- (?i)//openshift\.default\.svc(:|\z) +- (?i)//kubernetes\.default\.svc(:|\z) +- (?i)//172\.30\.0\.1(:|\z) +- (?i)//openshift\.default\.svc\.cluster\.local(:|\z) +- (?i)//openshift(:|\z) +dnsConfig: + bindAddress: 0.0.0.0:8053 + bindNetwork: tcp4 +etcdClientInfo: + ca: master.etcd-ca.crt + certFile: master.etcd-client.crt + keyFile: master.etcd-client.key + urls: + - https://{{ .Master.Hostname }}:2379 +etcdStorageConfig: + kubernetesStoragePrefix: kubernetes.io + kubernetesStorageVersion: v1 + openShiftStoragePrefix: openshift.io + openShiftStorageVersion: v1 +imageConfig: + format: TEMPIMAGEBASE-${component}:${version} + latest: false +kind: MasterConfig +kubeletClientInfo: + ca: ca-bundle.crt + certFile: master.kubelet-client.crt + keyFile: master.kubelet-client.key + port: 10250 +kubernetesMasterConfig: + apiServerArguments: + runtime-config: + - apis/settings.k8s.io/v1alpha1=true + storage-backend: + - etcd3 + storage-media-type: + - application/vnd.kubernetes.protobuf + cloud-provider: + - "azure" + cloud-config: + - "/etc/azure/azure.conf" + controllerArguments: + cluster-signing-cert-file: + - "/etc/origin/master/ca.crt" + cluster-signing-key-file: + - "/etc/origin/master/ca.key" + cloud-provider: + - "azure" + cloud-config: + - "/etc/azure/azure.conf" + masterCount: 1 + masterIP: {{ (index .Master.IPs 0).String }} + podEvictionTimeout: null + proxyClientInfo: + certFile: master.proxy-client.crt + keyFile: master.proxy-client.key + schedulerArguments: null + schedulerConfigFile: /etc/origin/master/scheduler.json + servicesNodePortRange: '' + servicesSubnet: 172.30.0.0/16 + staticNodeNames: [] +masterClients: + externalKubernetesClientConnectionOverrides: + acceptContentTypes: application/vnd.kubernetes.protobuf,application/json + burst: 400 + contentType: application/vnd.kubernetes.protobuf + qps: 200 + externalKubernetesKubeConfig: '' + openshiftLoopbackClientConnectionOverrides: + acceptContentTypes: application/vnd.kubernetes.protobuf,application/json + burst: 600 + contentType: application/vnd.kubernetes.protobuf + qps: 300 + openshiftLoopbackKubeConfig: openshift-master.kubeconfig +masterPublicURL: https://{{ .ExternalMasterHostname }}:{{ .Master.Port }} +networkConfig: + clusterNetworkCIDR: 10.128.0.0/14 + clusterNetworks: + - cidr: 10.128.0.0/14 + hostSubnetLength: 9 + externalIPNetworkCIDRs: + - 0.0.0.0/0 + hostSubnetLength: 9 + networkPluginName: redhat/openshift-ovs-subnet + serviceNetworkCIDR: 172.30.0.0/16 +oauthConfig: + assetPublicURL: https://{{ .ExternalMasterHostname }}:{{ .Master.Port }}/console/ + grantConfig: + method: auto + identityProviders: + - challenge: true + login: true + mappingMethod: claim + name: htpasswd_auth + provider: + apiVersion: v1 + file: /etc/origin/master/htpasswd + kind: HTPasswdPasswordIdentityProvider + masterCA: ca-bundle.crt + masterPublicURL: https://{{ .ExternalMasterHostname }}:{{ .Master.Port }} + masterURL: https://{{ .ExternalMasterHostname }}:{{ .Master.Port }} + sessionConfig: + sessionMaxAgeSeconds: 3600 + sessionName: ssn + sessionSecretsFile: /etc/origin/master/session-secrets.yaml + tokenConfig: + accessTokenMaxAgeSeconds: 86400 + authorizeTokenMaxAgeSeconds: 500 +pauseControllers: false +policyConfig: + bootstrapPolicyFile: /etc/origin/master/policy.json + openshiftInfrastructureNamespace: openshift-infra + openshiftSharedResourcesNamespace: openshift +projectConfig: + defaultNodeSelector: node-role.kubernetes.io/compute=true + projectRequestMessage: '' + projectRequestTemplate: '' + securityAllocator: + mcsAllocatorRange: s0:/2 + mcsLabelsPerProject: 5 + uidAllocatorRange: 1000000000-1999999999/10000 +routingConfig: + subdomain: TEMPROUTERIP.nip.io +serviceAccountConfig: + limitSecretReferences: false + managedNames: + - default + - builder + - deployer + masterCA: ca-bundle.crt + privateKeyFile: serviceaccounts.private.key + publicKeyFiles: + - serviceaccounts.public.key +servingInfo: + bindAddress: 0.0.0.0:{{ .Master.Port }} + bindNetwork: tcp4 + certFile: master.server.crt + clientCA: ca.crt + keyFile: master.server.key + maxRequestsInFlight: 500 + requestTimeoutSeconds: 3600 +volumeConfig: + dynamicProvisioningEnabled: true diff --git a/pkg/openshift/certgen/templates/master/etc/origin/master/scheduler.json b/pkg/openshift/certgen/templates/master/etc/origin/master/scheduler.json new file mode 100644 index 000000000..c67f1b3b9 --- /dev/null +++ b/pkg/openshift/certgen/templates/master/etc/origin/master/scheduler.json @@ -0,0 +1,88 @@ +{ + "apiVersion": "v1", + "kind": "Policy", + "predicates": [ + { + "name": "NoVolumeZoneConflict" + }, + { + "name": "MaxEBSVolumeCount" + }, + { + "name": "MaxGCEPDVolumeCount" + }, + { + "name": "MaxAzureDiskVolumeCount" + }, + { + "name": "MatchInterPodAffinity" + }, + { + "name": "NoDiskConflict" + }, + { + "name": "GeneralPredicates" + }, + { + "name": "PodToleratesNodeTaints" + }, + { + "name": "CheckNodeMemoryPressure" + }, + { + "name": "CheckNodeDiskPressure" + }, + { + "name": "CheckVolumeBinding" + }, + { + "argument": { + "serviceAffinity": { + "labels": [ + "region" + ] + } + }, + "name": "Region" + } + ], + "priorities": [ + { + "name": "SelectorSpreadPriority", + "weight": 1 + }, + { + "name": "InterPodAffinityPriority", + "weight": 1 + }, + { + "name": "LeastRequestedPriority", + "weight": 1 + }, + { + "name": "BalancedResourceAllocation", + "weight": 1 + }, + { + "name": "NodePreferAvoidPodsPriority", + "weight": 10000 + }, + { + "name": "NodeAffinityPriority", + "weight": 1 + }, + { + "name": "TaintTolerationPriority", + "weight": 1 + }, + { + "argument": { + "serviceAntiAffinity": { + "label": "zone" + } + }, + "name": "Zone", + "weight": 2 + } + ] +} diff --git a/pkg/openshift/certgen/templates/master/etc/origin/master/session-secrets.yaml b/pkg/openshift/certgen/templates/master/etc/origin/master/session-secrets.yaml new file mode 100644 index 000000000..e958ea1d0 --- /dev/null +++ b/pkg/openshift/certgen/templates/master/etc/origin/master/session-secrets.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: SessionSecrets +secrets: +- authentication: "{{ .AuthSecret }}" + encryption: "{{ .EncSecret }}" diff --git a/pkg/openshift/certgen/templates/master/tmp/ansible/ansible.sh b/pkg/openshift/certgen/templates/master/tmp/ansible/ansible.sh new file mode 100644 index 000000000..aac77761e --- /dev/null +++ b/pkg/openshift/certgen/templates/master/tmp/ansible/ansible.sh @@ -0,0 +1,111 @@ +#!/bin/bash -x + +# TODO: do this, and more (registry console, asb), the proper way + +# we get "dial tcp: lookup foo.eastus.cloudapp.azure.com on 10.0.0.11:53: read +# udp 172.17.0.2:56662->10.0.0.11:53: read: no route to host errors" at +# start-up: wait until these subside. +while ! oc version &>/dev/null; do + sleep 1 +done + +oc patch project default -p '{"metadata":{"annotations":{"openshift.io/node-selector": ""}}}' + +oc adm registry --images="$IMAGE_BASE-\${component}:\${version}" --selector='region=infra' + +# Deploy the router reusing relevant parts from openshift-ansible +ANSIBLE_ROLES_PATH=/usr/share/ansible/openshift-ansible/roles/ ansible-playbook -c local deploy-router.yml -i azure-local-master-inventory.yml + +oc create -f - <<'EOF' +kind: Project +apiVersion: v1 +metadata: + name: openshift-web-console + annotations: + openshift.io/node-selector: "" +EOF + +oc process -f /usr/share/ansible/openshift-ansible/roles/openshift_web_console/files/console-template.yaml \ + -p API_SERVER_CONFIG="$(sed -e s/127.0.0.1/{{ .ExternalMasterHostname }}/g