refactor: Move CNI bridge/transparent routes to common (#694)

* fix: pass host gateway to CNI with Swift to enable Swift+Transparent
This commit is contained in:
Mathew Merrick 2020-11-16 14:24:00 -08:00 коммит произвёл GitHub
Родитель d68c75cd5e
Коммит f0907b4e82
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
7 изменённых файлов: 169 добавлений и 158 удалений

Просмотреть файл

@ -18,7 +18,7 @@ spec:
hostNetwork: true
containers:
- name: azure-cni-installer
image: mcr.microsoft.com/containernetworking/azure-cni-manager:v1.1.9-alphav1
image: mcr.microsoft.com/containernetworking/azure-cni-manager:v1.2.0-2-g0671b63
imagePullPolicy: Always
env:
- name: AZURE_CNI_OS
@ -26,9 +26,9 @@ spec:
- name: AZURE_CNI_TENANCY
value: singletenancy
- name: AZURE_CNI_MODE
value: bridge
value: transparent
- name: AZURE_CNI_IPAM
value: azure-cns
value: azure-vnet-ipam
- name: AZURE_CNI_EXEMPT
value: azure-vnet-telemetry,azure-vnet-telemetry.config
volumeMounts:

Просмотреть файл

@ -9,6 +9,7 @@ import (
"github.com/Azure/azure-container-networking/cni"
"github.com/Azure/azure-container-networking/cns"
"github.com/Azure/azure-container-networking/cns/cnsclient"
"github.com/Azure/azure-container-networking/iptables"
"github.com/Azure/azure-container-networking/log"
"github.com/Azure/azure-container-networking/network"
cniTypes "github.com/containernetworking/cni/pkg/types"
@ -48,7 +49,7 @@ func NewCNSInvoker(podName, namespace string) (*CNSIPAMInvoker, error) {
}
//Add uses the requestipconfig API in cns, and returns ipv4 and a nil ipv6 as CNS doesn't support IPv6 yet
func (invoker *CNSIPAMInvoker) Add(nwCfg *cni.NetworkConfig, subnetPrefix *net.IPNet, options map[string]interface{}) (*cniTypesCurr.Result, *cniTypesCurr.Result, error) {
func (invoker *CNSIPAMInvoker) Add(nwCfg *cni.NetworkConfig, hostSubnetPrefix *net.IPNet, options map[string]interface{}) (*cniTypesCurr.Result, *cniTypesCurr.Result, error) {
// Parse Pod arguments.
podInfo := cns.KubernetesPodInfo{PodName: invoker.podName, PodNamespace: invoker.podNamespace}
@ -61,7 +62,7 @@ func (invoker *CNSIPAMInvoker) Add(nwCfg *cni.NetworkConfig, subnetPrefix *net.I
return nil, nil, err
}
resultIPv4 := IPv4ResultInfo{
info := IPv4ResultInfo{
podIPAddress: response.PodIpInfo.PodIPConfig.IPAddress,
ncSubnetPrefix: response.PodIpInfo.NetworkContainerPrimaryIPConfig.IPSubnet.PrefixLength,
ncPrimaryIP: response.PodIpInfo.NetworkContainerPrimaryIPConfig.IPSubnet.IPAddress,
@ -71,20 +72,46 @@ func (invoker *CNSIPAMInvoker) Add(nwCfg *cni.NetworkConfig, subnetPrefix *net.I
hostGateway: response.PodIpInfo.HostPrimaryIPInfo.Gateway,
}
ncgw := net.ParseIP(resultIPv4.ncGatewayIPAddress)
// set the NC Primary IP in options
options[network.SNATIPKey] = info.ncPrimaryIP
log.Printf("[cni-invoker-cns] Received info %v for pod %v", info, podInfo)
ncgw := net.ParseIP(info.ncGatewayIPAddress)
if ncgw == nil {
return nil, nil, fmt.Errorf("Gateway address %v from response is invalid", resultIPv4.ncGatewayIPAddress)
return nil, nil, fmt.Errorf("Gateway address %v from response is invalid", info.ncGatewayIPAddress)
}
// set the NC Primary IP in options
options[network.SNATIPKey] = resultIPv4.ncPrimaryIP
// set result ipconfig from CNS Response Body
ip, ncipnet, err := net.ParseCIDR(info.podIPAddress + "/" + fmt.Sprint(info.ncSubnetPrefix))
if ip == nil {
return nil, nil, fmt.Errorf("Unable to parse IP from response: %v with err %v", info.podIPAddress, err)
}
// set host gateway in options
options[network.HostGWKey] = resultIPv4.hostGateway
// construct ipnet for result
resultIPnet := net.IPNet{
IP: ip,
Mask: ncipnet.Mask,
}
log.Printf("Received result %+v for pod %v", resultIPv4, podInfo)
result := &cniTypesCurr.Result{
IPs: []*cniTypesCurr.IPConfig{
{
Version: "4",
Address: resultIPnet,
Gateway: ncgw,
},
},
Routes: []*cniTypes.Route{
{
Dst: network.Ipv4DefaultRouteDstPrefix,
GW: ncgw,
},
},
}
result, err := getCNIIPv4Result(resultIPv4, subnetPrefix)
// set subnet prefix for host vm
err = setHostOptions(nwCfg, hostSubnetPrefix, ncipnet, options, info)
if err != nil {
return nil, nil, err
}
@ -93,54 +120,44 @@ func (invoker *CNSIPAMInvoker) Add(nwCfg *cni.NetworkConfig, subnetPrefix *net.I
return result, nil, nil
}
func getCNIIPv4Result(info IPv4ResultInfo, subnetPrefix *net.IPNet) (*cniTypesCurr.Result, error) {
gw := net.ParseIP(info.ncGatewayIPAddress)
if gw == nil {
return nil, fmt.Errorf("Gateway address %v from response is invalid", gw)
}
hostIP := net.ParseIP(info.hostPrimaryIP)
if hostIP == nil {
return nil, fmt.Errorf("Host IP address %v from response is invalid", hostIP)
}
// set result ipconfig from CNS Response Body
ip, ipnet, err := net.ParseCIDR(info.podIPAddress + "/" + fmt.Sprint(info.ncSubnetPrefix))
if ip == nil {
return nil, fmt.Errorf("Unable to parse IP from response: %v", info.podIPAddress)
}
func setHostOptions(nwCfg *cni.NetworkConfig, hostSubnetPrefix *net.IPNet, ncSubnetPrefix *net.IPNet, options map[string]interface{}, info IPv4ResultInfo) error {
// get the name of the primary IP address
_, hostIPNet, err := net.ParseCIDR(info.hostSubnet)
if err != nil {
return nil, err
return err
}
// set subnet prefix for host vm
*subnetPrefix = *hostIPNet
*hostSubnetPrefix = *hostIPNet
// construct ipnet for result
resultIPnet := net.IPNet{
IP: ip,
Mask: ipnet.Mask,
// get the host ip
hostIP := net.ParseIP(info.hostPrimaryIP)
if hostIP == nil {
return fmt.Errorf("Host IP address %v from response is invalid", info.hostPrimaryIP)
}
return &cniTypesCurr.Result{
IPs: []*cniTypesCurr.IPConfig{
{
Version: "4",
Address: resultIPnet,
Gateway: gw,
},
// get host gateway
hostGateway := net.ParseIP(info.hostGateway)
if hostGateway == nil {
return fmt.Errorf("Host Gateway %v from response is invalid", info.hostGateway)
}
// this route is needed when the vm on subnet A needs to send traffic to a pod in subnet B on a different vm
options[network.RoutesKey] = []network.RouteInfo{
{
Dst: *ncSubnetPrefix,
Gw: hostGateway,
},
Routes: []*cniTypes.Route{
{
Dst: network.Ipv4DefaultRouteDstPrefix,
GW: gw,
},
},
}, nil
}
azureDNSMatch := fmt.Sprintf(" -m addrtype ! --dst-type local -s %s -d %s -p %s --dport %d", ncSubnetPrefix.String(), iptables.AzureDNS, iptables.UDP, iptables.DNSPort)
snatPrimaryIPJump := fmt.Sprintf("%s --to %s", iptables.Snat, info.ncPrimaryIP)
options[network.IPTablesKey] = []iptables.IPTableEntry{
iptables.GetCreateChainCmd(iptables.V4, iptables.Nat, iptables.Swift),
iptables.GetAppendIptableRuleCmd(iptables.V4, iptables.Nat, iptables.Postrouting, "", iptables.Swift),
iptables.GetInsertIptableRuleCmd(iptables.V4, iptables.Nat, iptables.Swift, azureDNSMatch, snatPrimaryIPJump),
}
return nil
}
// Delete calls into the releaseipconfiguration API in CNS

Просмотреть файл

@ -85,8 +85,13 @@ var (
DisableIPTableLock bool
)
type IPTableEntry struct {
Version string
Params string
}
// Run iptables command
func runCmd(version, params string) error {
func RunCmd(version, params string) error {
var cmd string
iptCmd := iptables
@ -110,20 +115,26 @@ func runCmd(version, params string) error {
// check if iptable chain alreay exists
func ChainExists(version, tableName, chainName string) bool {
params := fmt.Sprintf("-t %s -L %s", tableName, chainName)
if err := runCmd(version, params); err != nil {
if err := RunCmd(version, params); err != nil {
return false
}
return true
}
func GetCreateChainCmd(version, tableName, chainName string) IPTableEntry {
return IPTableEntry{
Version: version,
Params: fmt.Sprintf("-t %s -N %s", tableName, chainName),
}
}
// create new iptable chain under specified table name
func CreateChain(version, tableName, chainName string) error {
var err error
if !ChainExists(version, tableName, chainName) {
params := fmt.Sprintf("-t %s -N %s", tableName, chainName)
err = runCmd(version, params)
cmd := GetCreateChainCmd(version, tableName, chainName)
err = RunCmd(version, cmd.Params)
} else {
log.Printf("%s Chain exists in table %s", chainName, tableName)
}
@ -134,12 +145,19 @@ func CreateChain(version, tableName, chainName string) error {
// check if iptable rule alreay exists
func RuleExists(version, tableName, chainName, match, target string) bool {
params := fmt.Sprintf("-t %s -C %s %s -j %s", tableName, chainName, match, target)
if err := runCmd(version, params); err != nil {
if err := RunCmd(version, params); err != nil {
return false
}
return true
}
func GetInsertIptableRuleCmd(version, tableName, chainName, match, target string) IPTableEntry {
return IPTableEntry{
Version: version,
Params: fmt.Sprintf("-t %s -I %s 1 %s -j %s", tableName, chainName, match, target),
}
}
// Insert iptable rule at beginning of iptable chain
func InsertIptableRule(version, tableName, chainName, match, target string) error {
if RuleExists(version, tableName, chainName, match, target) {
@ -147,8 +165,15 @@ func InsertIptableRule(version, tableName, chainName, match, target string) erro
return nil
}
params := fmt.Sprintf("-t %s -I %s 1 %s -j %s", tableName, chainName, match, target)
return runCmd(version, params)
cmd := GetInsertIptableRuleCmd(version, tableName, chainName, match, target)
return RunCmd(version, cmd.Params)
}
func GetAppendIptableRuleCmd(version, tableName, chainName, match, target string) IPTableEntry {
return IPTableEntry{
Version: version,
Params: fmt.Sprintf("-t %s -A %s %s -j %s", tableName, chainName, match, target),
}
}
// Append iptable rule at end of iptable chain
@ -158,12 +183,12 @@ func AppendIptableRule(version, tableName, chainName, match, target string) erro
return nil
}
params := fmt.Sprintf("-t %s -A %s %s -j %s", tableName, chainName, match, target)
return runCmd(version, params)
cmd := GetAppendIptableRuleCmd(version, tableName, chainName, match, target)
return RunCmd(version, cmd.Params)
}
// Delete matched iptable rule
func DeleteIptableRule(version, tableName, chainName, match, target string) error {
params := fmt.Sprintf("-t %s -D %s %s -j %s", tableName, chainName, match, target)
return runCmd(version, params)
return RunCmd(version, params)
}

Просмотреть файл

@ -1,9 +1,7 @@
package network
import (
"fmt"
"net"
"strings"
"github.com/Azure/azure-container-networking/ebtables"
"github.com/Azure/azure-container-networking/log"
@ -48,59 +46,6 @@ func (client *LinuxBridgeClient) CreateBridge() error {
return epcommon.DisableRAForInterface(client.bridgeName)
}
func (client *LinuxBridgeClient) AddRoutes(nwInfo *NetworkInfo, interfaceName string) error {
if client.nwInfo.IPAMType == AzureCNS {
// fetch the host gateway IP from options
gwIP := client.nwInfo.Options[HostGWKey]
if gwIP == nil {
return fmt.Errorf("Host gateway IP in Options not set")
}
gatewayIP := net.ParseIP(gwIP.(string))
if gatewayIP == nil {
return fmt.Errorf("Invalid host gateway IP: %+v", gwIP)
}
// add host gateway as the default gateway for pod IP's
devIf, _ := net.InterfaceByName(interfaceName)
ifIndex := devIf.Index
family := netlink.GetIpAddressFamily(gatewayIP)
nlRoute := &netlink.Route{
Family: family,
Dst: &client.nwInfo.PodSubnet.Prefix,
Gw: gatewayIP,
LinkIndex: ifIndex,
}
log.Printf("Adding Swift route %+v", nlRoute)
if err := netlink.AddIpRoute(nlRoute); err != nil {
if !strings.Contains(strings.ToLower(err.Error()), "file exists") {
return fmt.Errorf("Failed to add route to host interface with error: %v", err)
}
log.Printf("[cni-cns-net] route already exists: dst %+v, gw %+v, interfaceName %v", nlRoute.Dst, nlRoute.Gw, interfaceName)
}
// Add snat Rules
snatIP := client.nwInfo.Options[SNATIPKey]
if snatIP == nil {
return fmt.Errorf("snatIP in Options not set %v", snatIP)
}
ncPrimaryIP := net.ParseIP(fmt.Sprintf("%v", snatIP))
if ncPrimaryIP == nil {
return fmt.Errorf("Failed to parse SNAT IP from options %v", client.nwInfo.Options)
}
log.Printf("Adding SNAT rule with snat IP %+v for subnet %s", ncPrimaryIP, client.nwInfo.PodSubnet.Prefix)
return epcommon.SNATfromSubnetToDNSWithNCPrimaryIP(ncPrimaryIP, client.nwInfo.PodSubnet.Prefix)
}
return nil
}
func (client *LinuxBridgeClient) DeleteBridge() error {
// Disconnect external interface from its bridge.
err := netlink.SetLinkMaster(client.hostInterfaceName, "")

Просмотреть файл

@ -240,39 +240,6 @@ func AddSnatRule(match string, ip net.IP) error {
return iptables.InsertIptableRule(version, iptables.Nat, iptables.Postrouting, match, target)
}
// SNATfromSubnetToDNSWithNCPrimaryIP snat's the snattedAddressSpace with the ipForSnat IP
func SNATfromSubnetToDNSWithNCPrimaryIP(ipForSNAT net.IP, snattedAddressSpace net.IPNet) (err error) {
// Create SWIFT chain, this checks if the chain already exists
// Check if theres a primary IP
if ipForSNAT != nil {
// Create SWIFT chain, this checks if the chain already exists
log.Printf("Creating SWIFT chain...")
err := iptables.CreateChain(iptables.V4, iptables.Nat, iptables.Swift)
if err != nil {
return err
}
log.Printf("Creating SWIFT chain jump from POSTROUTING")
// add jump to SWIFT chain from POSTROUTING
err = iptables.AppendIptableRule(iptables.V4, iptables.Nat, iptables.Postrouting, "", iptables.Swift)
if err != nil {
return err
}
log.Printf("Adding rule to SNAT subnet %v DNS requests with ip %v", snattedAddressSpace, ipForSNAT)
// SNAT requests to Azure DNS
azureDNSMatch := fmt.Sprintf(" -m addrtype ! --dst-type local -s %s -d %s -p %s --dport %d", snattedAddressSpace.String(), iptables.AzureDNS, iptables.UDP, iptables.DNSPort)
snatPrimaryIPJump := fmt.Sprintf("%s --to %s", iptables.Snat, ipForSNAT)
err = iptables.InsertIptableRule(iptables.V4, iptables.Nat, iptables.Swift, azureDNSMatch, snatPrimaryIPJump)
if err != nil {
return err
}
}
return nil
}
func DisableRAForInterface(ifName string) error {
raFilePath := fmt.Sprintf(acceptRAV6File, ifName)
exist, err := common.CheckIfFileExists(raFilePath)

Просмотреть файл

@ -21,7 +21,8 @@ const (
VlanIDKey = "VlanID"
AzureCNS = "azure-cns"
SNATIPKey = "NCPrimaryIPKey"
HostGWKey = "HostGatewayIP"
RoutesKey = "RoutesKey"
IPTablesKey = "IPTablesKey"
genericData = "com.docker.network.generic"
)
@ -36,7 +37,6 @@ type NetworkClient interface {
DeleteL2Rules(extIf *externalInterface)
SetBridgeMasterToHostInterface() error
SetHairpinOnHostInterface(bool) error
AddRoutes(nwInfo *NetworkInfo, interfaceName string) error
}
type EndpointClient interface {

Просмотреть файл

@ -54,6 +54,7 @@ func (nm *networkManager) newNetworkImpl(nwInfo *NetworkInfo, extIf *externalInt
switch nwInfo.Mode {
case opModeTunnel:
handleCommonOptions(extIf.Name, nwInfo)
fallthrough
case opModeBridge:
log.Printf("create bridge")
@ -64,8 +65,9 @@ func (nm *networkManager) newNetworkImpl(nwInfo *NetworkInfo, extIf *externalInt
if opt != nil && opt[VlanIDKey] != nil {
vlanid, _ = strconv.Atoi(opt[VlanIDKey].(string))
}
handleCommonOptions(extIf.BridgeName, nwInfo)
case opModeTransparent:
handleCommonOptions(extIf.Name, nwInfo)
break
default:
return nil, errNetworkModeInvalid
@ -85,6 +87,25 @@ func (nm *networkManager) newNetworkImpl(nwInfo *NetworkInfo, extIf *externalInt
return nw, nil
}
func handleCommonOptions(ifname string, nwInfo *NetworkInfo) error {
var err error
if routes, exists := nwInfo.Options[RoutesKey]; exists {
err = AddRoutes(ifname, routes.([]RouteInfo))
if err != nil {
return err
}
}
if iptcmds, exists := nwInfo.Options[IPTablesKey]; exists {
err = AddToIptables(iptcmds.([]iptables.IPTableEntry))
if err != nil {
return err
}
}
return nil
}
// DeleteNetworkImpl deletes an existing container network.
func (nm *networkManager) deleteNetworkImpl(nw *network) error {
var networkClient NetworkClient
@ -324,6 +345,7 @@ func (nm *networkManager) connectExternalInterface(extIf *externalInterface, nwI
// Check if the bridge already exists.
bridge, err := net.InterfaceByName(bridgeName)
if err != nil {
// Create the bridge.
if err = networkClient.CreateBridge(); err != nil {
@ -421,11 +443,6 @@ func (nm *networkManager) connectExternalInterface(extIf *externalInterface, nwI
log.Printf("[net] Applied dns config %v on %v", extIf.DNSInfo, bridgeName)
}
err = networkClient.AddRoutes(nwInfo, bridgeName)
if err != nil {
return err
}
if nwInfo.IPV6Mode == IPV6Nat {
// adds pod cidr gateway ip to bridge
if err = addIpv6NatGateway(nwInfo); err != nil {
@ -480,6 +497,46 @@ func (nm *networkManager) disconnectExternalInterface(extIf *externalInterface,
log.Printf("[net] Disconnected interface %v.", extIf.Name)
}
func AddToIptables(cmds []iptables.IPTableEntry) error {
log.Printf("Adding additional iptable rules...")
for _, cmd := range cmds {
err := iptables.RunCmd(cmd.Version, cmd.Params)
if err != nil {
return err
}
log.Printf("Succesfully run iptables rule %v", cmd)
}
return nil
}
func AddRoutes(bridgeName string, routes []RouteInfo) error {
log.Printf("Adding routes...")
for _, route := range routes {
route.DevName = bridgeName
devIf, _ := net.InterfaceByName(route.DevName)
ifIndex := devIf.Index
gwfamily := netlink.GetIpAddressFamily(route.Gw)
nlRoute := &netlink.Route{
Family: gwfamily,
Dst: &route.Dst,
Gw: route.Gw,
LinkIndex: ifIndex,
}
if err := netlink.AddIpRoute(nlRoute); err != nil {
if !strings.Contains(strings.ToLower(err.Error()), "file exists") {
return fmt.Errorf("Failed to add %+v to host interface with error: %v", nlRoute, err)
}
log.Printf("[cni-net] route already exists: dst %+v, gw %+v, interfaceName %v", nlRoute.Dst, nlRoute.Gw, route.DevName)
}
log.Printf("[cni-net] Added route %+v", route)
}
return nil
}
// Add ipv6 nat gateway IP on bridge
func addIpv6NatGateway(nwInfo *NetworkInfo) error {
log.Printf("[net] Adding ipv6 nat gateway on azure bridge")