diff --git a/parts/k8s/kubernetesmastercustomdata.yml b/parts/k8s/kubernetesmastercustomdata.yml index 27fc004ba..582d18994 100644 --- a/parts/k8s/kubernetesmastercustomdata.yml +++ b/parts/k8s/kubernetesmastercustomdata.yml @@ -495,10 +495,9 @@ MASTER_ARTIFACTS_CONFIG_PLACEHOLDER {{if IsMasterVirtualMachineScaleSets}} MASTER_VM_NAME=$(hostname) MASTER_VM_NAME_BASE=$(hostname | sed "s/.$//") - MASTER_FIRSTADDR_OCTET4={{WrapAsVariable "masterFirstAddrOctet4"}} + MASTER_FIRSTADDR={{WrapAsVariable "kubernetesAPIServerIP"}} MASTER_INDEX=$(hostname | tail -c 2) PRIVATE_IP=$(hostname -I | cut -d" " -f1) - PRIVATE_IP_BASE=$(hostname -I | cut -d" " -f1 | cut -d. -f1-3) MASTER_COUNT={{WrapAsVariable "masterCount"}} IPADDRESS_COUNT={{WrapAsVariable "masterIpAddressCount"}} echo $IPADDRESS_COUNT @@ -506,12 +505,18 @@ MASTER_ARTIFACTS_CONFIG_PLACEHOLDER ETCD_CLIENT_PORT={{WrapAsVariable "masterEtcdClientPort"}} MASTER_URLS="" index=0 + IFS=. read -r a b c d <<< "$MASTER_FIRSTADDR" + d=$((a * 256 ** 3 + b * 256 ** 2 + c * 256 + d)) + echo $d while [ $index -lt $MASTER_COUNT ] do echo $index - offset=`expr $index \\* $IPADDRESS_COUNT + $MASTER_FIRSTADDR_OCTET4` - echo $offset - MASTER_URLS="$MASTER_URLS$MASTER_VM_NAME_BASE$index=https://$PRIVATE_IP_BASE.$offset:$ETCD_SERVER_PORT," + x=`expr $d + $IPADDRESS_COUNT \\* $index` + echo $x + s="" + for i in 1 2 3 4; do s="."$((x%256))$s && ((x>>=8)); done; + s=$(echo $s | tail -c +2) + MASTER_URLS="$MASTER_URLS$MASTER_VM_NAME_BASE$index=https://$s:$ETCD_SERVER_PORT," index=`expr $index + 1` done MASTER_URLS=$(echo $MASTER_URLS | sed "s/.$//") diff --git a/pkg/api/defaults-openshift-certs.go b/pkg/api/defaults-openshift-certs.go index 8a36effef..bbe835936 100644 --- a/pkg/api/defaults-openshift-certs.go +++ b/pkg/api/defaults-openshift-certs.go @@ -11,10 +11,10 @@ import ( // setOpenShiftSetDefaultCerts sets default certificate and configuration properties in the // openshift orchestrator. -func setOpenShiftSetDefaultCerts(a *Properties, orchestratorName, clusterID string) (bool, error) { +func setOpenShiftSetDefaultCerts(a *Properties, orchestratorName, clusterID string) (bool, []net.IP, error) { if len(a.OrchestratorProfile.OpenShiftConfig.ConfigBundles["master"]) > 0 && len(a.OrchestratorProfile.OpenShiftConfig.ConfigBundles["bootstrap"]) > 0 { - return true, nil + return true, nil, nil } if a.OrchestratorProfile.OpenShiftConfig.ConfigBundles == nil { a.OrchestratorProfile.OpenShiftConfig.ConfigBundles = make(map[string][]byte) @@ -34,13 +34,13 @@ func setOpenShiftSetDefaultCerts(a *Properties, orchestratorName, clusterID stri } if err != nil { - return false, err + return false, nil, err } a.OrchestratorProfile.OpenShiftConfig.ConfigBundles["master"] = masterBundle a.OrchestratorProfile.OpenShiftConfig.ConfigBundles["bootstrap"] = nodeBundle - return true, nil + return true, nil, nil } func createR39Config(a *Properties, orchestratorName, clusterID string) *release39.Config { diff --git a/pkg/api/defaults.go b/pkg/api/defaults.go index 9f79569c4..55a70f173 100644 --- a/pkg/api/defaults.go +++ b/pkg/api/defaults.go @@ -4,6 +4,7 @@ import ( "bytes" "crypto/rand" "encoding/base64" + "encoding/binary" "fmt" "net" "sort" @@ -45,7 +46,7 @@ func (cs *ContainerService) SetPropertiesDefaults(isUpgrade, isScale bool) (bool properties.setHostedMasterProfileDefaults() } - certsGenerated, e := properties.setDefaultCerts() + certsGenerated, _, e := properties.setDefaultCerts() if e != nil { return false, e } @@ -497,19 +498,19 @@ func (p *Properties) setHostedMasterProfileDefaults() { p.HostedMasterProfile.Subnet = DefaultKubernetesMasterSubnet } -func (p *Properties) setDefaultCerts() (bool, error) { +func (p *Properties) setDefaultCerts() (bool, []net.IP, error) { if p.MasterProfile != nil && p.OrchestratorProfile.OrchestratorType == OpenShift { return setOpenShiftSetDefaultCerts(p, DefaultOpenshiftOrchestratorName, p.GetClusterID()) } if p.MasterProfile == nil || p.OrchestratorProfile.OrchestratorType != Kubernetes { - return false, nil + return false, nil, nil } provided := certsAlreadyPresent(p.CertificateProfile, p.MasterProfile.Count) if areAllTrue(provided) { - return false, nil + return false, nil, nil } var azureProdFQDNs []string @@ -521,7 +522,7 @@ func (p *Properties) setDefaultCerts() (bool, error) { firstMasterIP := net.ParseIP(p.MasterProfile.FirstConsecutiveStaticIP).To4() if firstMasterIP == nil { - return false, errors.Errorf("MasterProfile.FirstConsecutiveStaticIP '%s' is an invalid IP address", p.MasterProfile.FirstConsecutiveStaticIP) + return false, nil, errors.Errorf("MasterProfile.FirstConsecutiveStaticIP '%s' is an invalid IP address", p.MasterProfile.FirstConsecutiveStaticIP) } ips := []net.IP{firstMasterIP} @@ -535,9 +536,11 @@ func (p *Properties) setDefaultCerts() (bool, error) { } else { offsetMultiplier = 1 } + addr := binary.BigEndian.Uint32(firstMasterIP) for i := 1; i < p.MasterProfile.Count; i++ { - offset := i * offsetMultiplier - ip := net.IP{firstMasterIP[0], firstMasterIP[1], firstMasterIP[2], firstMasterIP[3] + byte(offset)} + newAddr := getNewAddr(addr, i, offsetMultiplier) + ip := make(net.IP, 4) + binary.BigEndian.PutUint32(ip, newAddr) ips = append(ips, ip) } if p.CertificateProfile == nil { @@ -552,7 +555,7 @@ func (p *Properties) setDefaultCerts() (bool, error) { var err error caPair, err = helpers.CreatePkiKeyCertPair("ca") if err != nil { - return false, err + return false, ips, err } p.CertificateProfile.CaCertificate = caPair.CertificatePem p.CertificateProfile.CaPrivateKey = caPair.PrivateKeyPem @@ -560,13 +563,13 @@ func (p *Properties) setDefaultCerts() (bool, error) { cidrFirstIP, err := common.CidrStringFirstIP(p.OrchestratorProfile.KubernetesConfig.ServiceCIDR) if err != nil { - return false, err + return false, ips, err } ips = append(ips, cidrFirstIP) apiServerPair, clientPair, kubeConfigPair, etcdServerPair, etcdClientPair, etcdPeerPairs, err := helpers.CreatePki(masterExtraFQDNs, ips, DefaultKubernetesClusterDomain, caPair, p.MasterProfile.Count) if err != nil { - return false, err + return false, ips, err } // If no Certificate Authority pair or no cert/key pair was provided, use generated cert/key pairs signed by provided Certificate Authority pair @@ -595,7 +598,7 @@ func (p *Properties) setDefaultCerts() (bool, error) { } } - return true, nil + return true, ips, nil } func areAllTrue(m map[string]bool) bool { @@ -607,6 +610,13 @@ func areAllTrue(m map[string]bool) bool { return true } +// getNewIP returns a new IP derived from an address plus a multiple of an offset +func getNewAddr(addr uint32, count int, offsetMultiplier int) uint32 { + offset := count * offsetMultiplier + newAddr := addr + uint32(offset) + return newAddr +} + // certsAlreadyPresent already present returns a map where each key is a type of cert and each value is true if that cert/key pair is user-provided func certsAlreadyPresent(c *CertificateProfile, m int) map[string]bool { g := map[string]bool{ diff --git a/pkg/api/defaults_test.go b/pkg/api/defaults_test.go index 9557f450e..ab04267d5 100644 --- a/pkg/api/defaults_test.go +++ b/pkg/api/defaults_test.go @@ -2,6 +2,8 @@ package api import ( "encoding/base64" + "encoding/binary" + "net" "reflect" "testing" @@ -963,6 +965,73 @@ func TestDefaultCloudProvider(t *testing.T) { helpers.IsTrueBoolPointer(properties.OrchestratorProfile.KubernetesConfig.CloudProviderBackoff)) } } +func TestSetCertDefaults(t *testing.T) { + cs := &ContainerService{ + Properties: &Properties{ + AzProfile: &AzProfile{ + TenantID: "sampleTenantID", + SubscriptionID: "foobarsubscription", + ResourceGroup: "sampleRG", + Location: "westus2", + }, + ServicePrincipalProfile: &ServicePrincipalProfile{ + ClientID: "barClientID", + Secret: "bazSecret", + }, + MasterProfile: &MasterProfile{ + Count: 3, + DNSPrefix: "myprefix1", + VMSize: "Standard_DS2_v2", + AvailabilityProfile: VirtualMachineScaleSets, + }, + OrchestratorProfile: &OrchestratorProfile{ + OrchestratorType: Kubernetes, + OrchestratorVersion: "1.10.2", + KubernetesConfig: &KubernetesConfig{ + NetworkPlugin: "azure", + }, + }, + }, + } + + cs.setOrchestratorDefaults(false) + cs.Properties.setMasterProfileDefaults(false) + result, ips, err := cs.Properties.setDefaultCerts() + + if !result { + t.Error("expected setDefaultCerts to return true") + } + + if err != nil { + t.Errorf("unexpected error thrown while executing setDefaultCerts %s", err.Error()) + } + + if ips == nil { + t.Error("expected setDefaultCerts to create a list of IPs") + } else { + + if len(ips) != cs.Properties.MasterProfile.Count+2 { + t.Errorf("expected length of IPs from setDefaultCerts %d, actual length %d", cs.Properties.MasterProfile.Count+2, len(ips)) + } + + firstMasterIP := net.ParseIP(cs.Properties.MasterProfile.FirstConsecutiveStaticIP).To4() + var offsetMultiplier int + if cs.Properties.MasterProfile.IsVirtualMachineScaleSets() { + offsetMultiplier = cs.Properties.MasterProfile.IPAddressCount + } else { + offsetMultiplier = 1 + } + addr := binary.BigEndian.Uint32(firstMasterIP) + expectedNewAddr := getNewAddr(addr, cs.Properties.MasterProfile.Count-1, offsetMultiplier) + actualLastIPAddr := binary.BigEndian.Uint32(ips[len(ips)-2]) + if actualLastIPAddr != expectedNewAddr { + expectedLastIP := make(net.IP, 4) + binary.BigEndian.PutUint32(expectedLastIP, expectedNewAddr) + t.Errorf("expected last IP of master vm from setDefaultCerts %d, actual %d", expectedLastIP, ips[len(ips)-2]) + } + } + +} func TestSetOpenShiftCertDefaults(t *testing.T) { cs := &ContainerService{ @@ -992,7 +1061,7 @@ func TestSetOpenShiftCertDefaults(t *testing.T) { cs.Properties.setMasterProfileDefaults(false) - result, err := cs.Properties.setDefaultCerts() + result, _, err := cs.Properties.setDefaultCerts() if !result { t.Error("expected setOpenShiftDefaultCerts to return true") } @@ -1028,7 +1097,7 @@ func TestSetOpenShiftCertDefaults(t *testing.T) { } cs.Properties.setMasterProfileDefaults(false) - result, err = cs.Properties.setDefaultCerts() + result, _, err = cs.Properties.setDefaultCerts() if !result { t.Error("expected setOpenShiftDefaultCerts to return true") @@ -1037,7 +1106,6 @@ func TestSetOpenShiftCertDefaults(t *testing.T) { if err != nil { t.Errorf("unexpected error thrown while executing setOpenShiftDefaultCerts %s", err.Error()) } - } func getMockBaseContainerService(orchestratorVersion string) ContainerService {