зеркало из https://github.com/Azure/acs-engine.git
Update ip assignment and cert gen for vmss masters (#4193)
This commit is contained in:
Родитель
527d968285
Коммит
9d6e3fc1d0
|
@ -495,10 +495,9 @@ MASTER_ARTIFACTS_CONFIG_PLACEHOLDER
|
||||||
{{if IsMasterVirtualMachineScaleSets}}
|
{{if IsMasterVirtualMachineScaleSets}}
|
||||||
MASTER_VM_NAME=$(hostname)
|
MASTER_VM_NAME=$(hostname)
|
||||||
MASTER_VM_NAME_BASE=$(hostname | sed "s/.$//")
|
MASTER_VM_NAME_BASE=$(hostname | sed "s/.$//")
|
||||||
MASTER_FIRSTADDR_OCTET4={{WrapAsVariable "masterFirstAddrOctet4"}}
|
MASTER_FIRSTADDR={{WrapAsVariable "kubernetesAPIServerIP"}}
|
||||||
MASTER_INDEX=$(hostname | tail -c 2)
|
MASTER_INDEX=$(hostname | tail -c 2)
|
||||||
PRIVATE_IP=$(hostname -I | cut -d" " -f1)
|
PRIVATE_IP=$(hostname -I | cut -d" " -f1)
|
||||||
PRIVATE_IP_BASE=$(hostname -I | cut -d" " -f1 | cut -d. -f1-3)
|
|
||||||
MASTER_COUNT={{WrapAsVariable "masterCount"}}
|
MASTER_COUNT={{WrapAsVariable "masterCount"}}
|
||||||
IPADDRESS_COUNT={{WrapAsVariable "masterIpAddressCount"}}
|
IPADDRESS_COUNT={{WrapAsVariable "masterIpAddressCount"}}
|
||||||
echo $IPADDRESS_COUNT
|
echo $IPADDRESS_COUNT
|
||||||
|
@ -506,12 +505,18 @@ MASTER_ARTIFACTS_CONFIG_PLACEHOLDER
|
||||||
ETCD_CLIENT_PORT={{WrapAsVariable "masterEtcdClientPort"}}
|
ETCD_CLIENT_PORT={{WrapAsVariable "masterEtcdClientPort"}}
|
||||||
MASTER_URLS=""
|
MASTER_URLS=""
|
||||||
index=0
|
index=0
|
||||||
|
IFS=. read -r a b c d <<< "$MASTER_FIRSTADDR"
|
||||||
|
d=$((a * 256 ** 3 + b * 256 ** 2 + c * 256 + d))
|
||||||
|
echo $d
|
||||||
while [ $index -lt $MASTER_COUNT ]
|
while [ $index -lt $MASTER_COUNT ]
|
||||||
do
|
do
|
||||||
echo $index
|
echo $index
|
||||||
offset=`expr $index \\* $IPADDRESS_COUNT + $MASTER_FIRSTADDR_OCTET4`
|
x=`expr $d + $IPADDRESS_COUNT \\* $index`
|
||||||
echo $offset
|
echo $x
|
||||||
MASTER_URLS="$MASTER_URLS$MASTER_VM_NAME_BASE$index=https://$PRIVATE_IP_BASE.$offset:$ETCD_SERVER_PORT,"
|
s=""
|
||||||
|
for i in 1 2 3 4; do s="."$((x%256))$s && ((x>>=8)); done;
|
||||||
|
s=$(echo $s | tail -c +2)
|
||||||
|
MASTER_URLS="$MASTER_URLS$MASTER_VM_NAME_BASE$index=https://$s:$ETCD_SERVER_PORT,"
|
||||||
index=`expr $index + 1`
|
index=`expr $index + 1`
|
||||||
done
|
done
|
||||||
MASTER_URLS=$(echo $MASTER_URLS | sed "s/.$//")
|
MASTER_URLS=$(echo $MASTER_URLS | sed "s/.$//")
|
||||||
|
|
|
@ -11,10 +11,10 @@ import (
|
||||||
|
|
||||||
// setOpenShiftSetDefaultCerts sets default certificate and configuration properties in the
|
// setOpenShiftSetDefaultCerts sets default certificate and configuration properties in the
|
||||||
// openshift orchestrator.
|
// openshift orchestrator.
|
||||||
func setOpenShiftSetDefaultCerts(a *Properties, orchestratorName, clusterID string) (bool, error) {
|
func setOpenShiftSetDefaultCerts(a *Properties, orchestratorName, clusterID string) (bool, []net.IP, error) {
|
||||||
if len(a.OrchestratorProfile.OpenShiftConfig.ConfigBundles["master"]) > 0 &&
|
if len(a.OrchestratorProfile.OpenShiftConfig.ConfigBundles["master"]) > 0 &&
|
||||||
len(a.OrchestratorProfile.OpenShiftConfig.ConfigBundles["bootstrap"]) > 0 {
|
len(a.OrchestratorProfile.OpenShiftConfig.ConfigBundles["bootstrap"]) > 0 {
|
||||||
return true, nil
|
return true, nil, nil
|
||||||
}
|
}
|
||||||
if a.OrchestratorProfile.OpenShiftConfig.ConfigBundles == nil {
|
if a.OrchestratorProfile.OpenShiftConfig.ConfigBundles == nil {
|
||||||
a.OrchestratorProfile.OpenShiftConfig.ConfigBundles = make(map[string][]byte)
|
a.OrchestratorProfile.OpenShiftConfig.ConfigBundles = make(map[string][]byte)
|
||||||
|
@ -34,13 +34,13 @@ func setOpenShiftSetDefaultCerts(a *Properties, orchestratorName, clusterID stri
|
||||||
}
|
}
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, err
|
return false, nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
a.OrchestratorProfile.OpenShiftConfig.ConfigBundles["master"] = masterBundle
|
a.OrchestratorProfile.OpenShiftConfig.ConfigBundles["master"] = masterBundle
|
||||||
a.OrchestratorProfile.OpenShiftConfig.ConfigBundles["bootstrap"] = nodeBundle
|
a.OrchestratorProfile.OpenShiftConfig.ConfigBundles["bootstrap"] = nodeBundle
|
||||||
|
|
||||||
return true, nil
|
return true, nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func createR39Config(a *Properties, orchestratorName, clusterID string) *release39.Config {
|
func createR39Config(a *Properties, orchestratorName, clusterID string) *release39.Config {
|
||||||
|
|
|
@ -4,6 +4,7 @@ import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"crypto/rand"
|
"crypto/rand"
|
||||||
"encoding/base64"
|
"encoding/base64"
|
||||||
|
"encoding/binary"
|
||||||
"fmt"
|
"fmt"
|
||||||
"net"
|
"net"
|
||||||
"sort"
|
"sort"
|
||||||
|
@ -45,7 +46,7 @@ func (cs *ContainerService) SetPropertiesDefaults(isUpgrade, isScale bool) (bool
|
||||||
properties.setHostedMasterProfileDefaults()
|
properties.setHostedMasterProfileDefaults()
|
||||||
}
|
}
|
||||||
|
|
||||||
certsGenerated, e := properties.setDefaultCerts()
|
certsGenerated, _, e := properties.setDefaultCerts()
|
||||||
if e != nil {
|
if e != nil {
|
||||||
return false, e
|
return false, e
|
||||||
}
|
}
|
||||||
|
@ -497,19 +498,19 @@ func (p *Properties) setHostedMasterProfileDefaults() {
|
||||||
p.HostedMasterProfile.Subnet = DefaultKubernetesMasterSubnet
|
p.HostedMasterProfile.Subnet = DefaultKubernetesMasterSubnet
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p *Properties) setDefaultCerts() (bool, error) {
|
func (p *Properties) setDefaultCerts() (bool, []net.IP, error) {
|
||||||
if p.MasterProfile != nil && p.OrchestratorProfile.OrchestratorType == OpenShift {
|
if p.MasterProfile != nil && p.OrchestratorProfile.OrchestratorType == OpenShift {
|
||||||
return setOpenShiftSetDefaultCerts(p, DefaultOpenshiftOrchestratorName, p.GetClusterID())
|
return setOpenShiftSetDefaultCerts(p, DefaultOpenshiftOrchestratorName, p.GetClusterID())
|
||||||
}
|
}
|
||||||
|
|
||||||
if p.MasterProfile == nil || p.OrchestratorProfile.OrchestratorType != Kubernetes {
|
if p.MasterProfile == nil || p.OrchestratorProfile.OrchestratorType != Kubernetes {
|
||||||
return false, nil
|
return false, nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
provided := certsAlreadyPresent(p.CertificateProfile, p.MasterProfile.Count)
|
provided := certsAlreadyPresent(p.CertificateProfile, p.MasterProfile.Count)
|
||||||
|
|
||||||
if areAllTrue(provided) {
|
if areAllTrue(provided) {
|
||||||
return false, nil
|
return false, nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
var azureProdFQDNs []string
|
var azureProdFQDNs []string
|
||||||
|
@ -521,7 +522,7 @@ func (p *Properties) setDefaultCerts() (bool, error) {
|
||||||
firstMasterIP := net.ParseIP(p.MasterProfile.FirstConsecutiveStaticIP).To4()
|
firstMasterIP := net.ParseIP(p.MasterProfile.FirstConsecutiveStaticIP).To4()
|
||||||
|
|
||||||
if firstMasterIP == nil {
|
if firstMasterIP == nil {
|
||||||
return false, errors.Errorf("MasterProfile.FirstConsecutiveStaticIP '%s' is an invalid IP address", p.MasterProfile.FirstConsecutiveStaticIP)
|
return false, nil, errors.Errorf("MasterProfile.FirstConsecutiveStaticIP '%s' is an invalid IP address", p.MasterProfile.FirstConsecutiveStaticIP)
|
||||||
}
|
}
|
||||||
|
|
||||||
ips := []net.IP{firstMasterIP}
|
ips := []net.IP{firstMasterIP}
|
||||||
|
@ -535,9 +536,11 @@ func (p *Properties) setDefaultCerts() (bool, error) {
|
||||||
} else {
|
} else {
|
||||||
offsetMultiplier = 1
|
offsetMultiplier = 1
|
||||||
}
|
}
|
||||||
|
addr := binary.BigEndian.Uint32(firstMasterIP)
|
||||||
for i := 1; i < p.MasterProfile.Count; i++ {
|
for i := 1; i < p.MasterProfile.Count; i++ {
|
||||||
offset := i * offsetMultiplier
|
newAddr := getNewAddr(addr, i, offsetMultiplier)
|
||||||
ip := net.IP{firstMasterIP[0], firstMasterIP[1], firstMasterIP[2], firstMasterIP[3] + byte(offset)}
|
ip := make(net.IP, 4)
|
||||||
|
binary.BigEndian.PutUint32(ip, newAddr)
|
||||||
ips = append(ips, ip)
|
ips = append(ips, ip)
|
||||||
}
|
}
|
||||||
if p.CertificateProfile == nil {
|
if p.CertificateProfile == nil {
|
||||||
|
@ -552,7 +555,7 @@ func (p *Properties) setDefaultCerts() (bool, error) {
|
||||||
var err error
|
var err error
|
||||||
caPair, err = helpers.CreatePkiKeyCertPair("ca")
|
caPair, err = helpers.CreatePkiKeyCertPair("ca")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, err
|
return false, ips, err
|
||||||
}
|
}
|
||||||
p.CertificateProfile.CaCertificate = caPair.CertificatePem
|
p.CertificateProfile.CaCertificate = caPair.CertificatePem
|
||||||
p.CertificateProfile.CaPrivateKey = caPair.PrivateKeyPem
|
p.CertificateProfile.CaPrivateKey = caPair.PrivateKeyPem
|
||||||
|
@ -560,13 +563,13 @@ func (p *Properties) setDefaultCerts() (bool, error) {
|
||||||
|
|
||||||
cidrFirstIP, err := common.CidrStringFirstIP(p.OrchestratorProfile.KubernetesConfig.ServiceCIDR)
|
cidrFirstIP, err := common.CidrStringFirstIP(p.OrchestratorProfile.KubernetesConfig.ServiceCIDR)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, err
|
return false, ips, err
|
||||||
}
|
}
|
||||||
ips = append(ips, cidrFirstIP)
|
ips = append(ips, cidrFirstIP)
|
||||||
|
|
||||||
apiServerPair, clientPair, kubeConfigPair, etcdServerPair, etcdClientPair, etcdPeerPairs, err := helpers.CreatePki(masterExtraFQDNs, ips, DefaultKubernetesClusterDomain, caPair, p.MasterProfile.Count)
|
apiServerPair, clientPair, kubeConfigPair, etcdServerPair, etcdClientPair, etcdPeerPairs, err := helpers.CreatePki(masterExtraFQDNs, ips, DefaultKubernetesClusterDomain, caPair, p.MasterProfile.Count)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, err
|
return false, ips, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// If no Certificate Authority pair or no cert/key pair was provided, use generated cert/key pairs signed by provided Certificate Authority pair
|
// If no Certificate Authority pair or no cert/key pair was provided, use generated cert/key pairs signed by provided Certificate Authority pair
|
||||||
|
@ -595,7 +598,7 @@ func (p *Properties) setDefaultCerts() (bool, error) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return true, nil
|
return true, ips, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func areAllTrue(m map[string]bool) bool {
|
func areAllTrue(m map[string]bool) bool {
|
||||||
|
@ -607,6 +610,13 @@ func areAllTrue(m map[string]bool) bool {
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// getNewIP returns a new IP derived from an address plus a multiple of an offset
|
||||||
|
func getNewAddr(addr uint32, count int, offsetMultiplier int) uint32 {
|
||||||
|
offset := count * offsetMultiplier
|
||||||
|
newAddr := addr + uint32(offset)
|
||||||
|
return newAddr
|
||||||
|
}
|
||||||
|
|
||||||
// certsAlreadyPresent already present returns a map where each key is a type of cert and each value is true if that cert/key pair is user-provided
|
// certsAlreadyPresent already present returns a map where each key is a type of cert and each value is true if that cert/key pair is user-provided
|
||||||
func certsAlreadyPresent(c *CertificateProfile, m int) map[string]bool {
|
func certsAlreadyPresent(c *CertificateProfile, m int) map[string]bool {
|
||||||
g := map[string]bool{
|
g := map[string]bool{
|
||||||
|
|
|
@ -2,6 +2,8 @@ package api
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/base64"
|
"encoding/base64"
|
||||||
|
"encoding/binary"
|
||||||
|
"net"
|
||||||
"reflect"
|
"reflect"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
@ -963,6 +965,73 @@ func TestDefaultCloudProvider(t *testing.T) {
|
||||||
helpers.IsTrueBoolPointer(properties.OrchestratorProfile.KubernetesConfig.CloudProviderBackoff))
|
helpers.IsTrueBoolPointer(properties.OrchestratorProfile.KubernetesConfig.CloudProviderBackoff))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
func TestSetCertDefaults(t *testing.T) {
|
||||||
|
cs := &ContainerService{
|
||||||
|
Properties: &Properties{
|
||||||
|
AzProfile: &AzProfile{
|
||||||
|
TenantID: "sampleTenantID",
|
||||||
|
SubscriptionID: "foobarsubscription",
|
||||||
|
ResourceGroup: "sampleRG",
|
||||||
|
Location: "westus2",
|
||||||
|
},
|
||||||
|
ServicePrincipalProfile: &ServicePrincipalProfile{
|
||||||
|
ClientID: "barClientID",
|
||||||
|
Secret: "bazSecret",
|
||||||
|
},
|
||||||
|
MasterProfile: &MasterProfile{
|
||||||
|
Count: 3,
|
||||||
|
DNSPrefix: "myprefix1",
|
||||||
|
VMSize: "Standard_DS2_v2",
|
||||||
|
AvailabilityProfile: VirtualMachineScaleSets,
|
||||||
|
},
|
||||||
|
OrchestratorProfile: &OrchestratorProfile{
|
||||||
|
OrchestratorType: Kubernetes,
|
||||||
|
OrchestratorVersion: "1.10.2",
|
||||||
|
KubernetesConfig: &KubernetesConfig{
|
||||||
|
NetworkPlugin: "azure",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
cs.setOrchestratorDefaults(false)
|
||||||
|
cs.Properties.setMasterProfileDefaults(false)
|
||||||
|
result, ips, err := cs.Properties.setDefaultCerts()
|
||||||
|
|
||||||
|
if !result {
|
||||||
|
t.Error("expected setDefaultCerts to return true")
|
||||||
|
}
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("unexpected error thrown while executing setDefaultCerts %s", err.Error())
|
||||||
|
}
|
||||||
|
|
||||||
|
if ips == nil {
|
||||||
|
t.Error("expected setDefaultCerts to create a list of IPs")
|
||||||
|
} else {
|
||||||
|
|
||||||
|
if len(ips) != cs.Properties.MasterProfile.Count+2 {
|
||||||
|
t.Errorf("expected length of IPs from setDefaultCerts %d, actual length %d", cs.Properties.MasterProfile.Count+2, len(ips))
|
||||||
|
}
|
||||||
|
|
||||||
|
firstMasterIP := net.ParseIP(cs.Properties.MasterProfile.FirstConsecutiveStaticIP).To4()
|
||||||
|
var offsetMultiplier int
|
||||||
|
if cs.Properties.MasterProfile.IsVirtualMachineScaleSets() {
|
||||||
|
offsetMultiplier = cs.Properties.MasterProfile.IPAddressCount
|
||||||
|
} else {
|
||||||
|
offsetMultiplier = 1
|
||||||
|
}
|
||||||
|
addr := binary.BigEndian.Uint32(firstMasterIP)
|
||||||
|
expectedNewAddr := getNewAddr(addr, cs.Properties.MasterProfile.Count-1, offsetMultiplier)
|
||||||
|
actualLastIPAddr := binary.BigEndian.Uint32(ips[len(ips)-2])
|
||||||
|
if actualLastIPAddr != expectedNewAddr {
|
||||||
|
expectedLastIP := make(net.IP, 4)
|
||||||
|
binary.BigEndian.PutUint32(expectedLastIP, expectedNewAddr)
|
||||||
|
t.Errorf("expected last IP of master vm from setDefaultCerts %d, actual %d", expectedLastIP, ips[len(ips)-2])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
func TestSetOpenShiftCertDefaults(t *testing.T) {
|
func TestSetOpenShiftCertDefaults(t *testing.T) {
|
||||||
cs := &ContainerService{
|
cs := &ContainerService{
|
||||||
|
@ -992,7 +1061,7 @@ func TestSetOpenShiftCertDefaults(t *testing.T) {
|
||||||
|
|
||||||
cs.Properties.setMasterProfileDefaults(false)
|
cs.Properties.setMasterProfileDefaults(false)
|
||||||
|
|
||||||
result, err := cs.Properties.setDefaultCerts()
|
result, _, err := cs.Properties.setDefaultCerts()
|
||||||
if !result {
|
if !result {
|
||||||
t.Error("expected setOpenShiftDefaultCerts to return true")
|
t.Error("expected setOpenShiftDefaultCerts to return true")
|
||||||
}
|
}
|
||||||
|
@ -1028,7 +1097,7 @@ func TestSetOpenShiftCertDefaults(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
cs.Properties.setMasterProfileDefaults(false)
|
cs.Properties.setMasterProfileDefaults(false)
|
||||||
result, err = cs.Properties.setDefaultCerts()
|
result, _, err = cs.Properties.setDefaultCerts()
|
||||||
|
|
||||||
if !result {
|
if !result {
|
||||||
t.Error("expected setOpenShiftDefaultCerts to return true")
|
t.Error("expected setOpenShiftDefaultCerts to return true")
|
||||||
|
@ -1037,7 +1106,6 @@ func TestSetOpenShiftCertDefaults(t *testing.T) {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Errorf("unexpected error thrown while executing setOpenShiftDefaultCerts %s", err.Error())
|
t.Errorf("unexpected error thrown while executing setOpenShiftDefaultCerts %s", err.Error())
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func getMockBaseContainerService(orchestratorVersion string) ContainerService {
|
func getMockBaseContainerService(orchestratorVersion string) ContainerService {
|
||||||
|
|
Загрузка…
Ссылка в новой задаче