feat: Adding MACAddress to CNS endpoint State and refactoring stateless CNI code (#2722)

* Adding MACAddress to CNS endpointState and also refactoring statelss cni code

* Addressing the comments.

* Addressing the comments

cni/windows.Dockerfile

@@ -11,7 +11,7 @@ COPY . .
 RUN GOOS=$OS CGO_ENABLED=0 go build -a -o /go/bin/azure-vnet -trimpath -ldflags "-X main.version="$VERSION"" -gcflags="-dwarflocationlists=true" cni/network/plugin/main.go
 RUN GOOS=$OS CGO_ENABLED=0 go build -a -o /go/bin/azure-vnet-telemetry -trimpath -ldflags "-X main.version="$VERSION"" -gcflags="-dwarflocationlists=true" cni/telemetry/service/telemetrymain.go
 RUN GOOS=$OS CGO_ENABLED=0 go build -a -o /go/bin/azure-vnet-ipam -trimpath -ldflags "-X main.version="$VERSION"" -gcflags="-dwarflocationlists=true" cni/ipam/plugin/main.go
-RUN GOOS=$OS CGO_ENABLED=0 go build -a -o /go/bin/azurecni-stateless -trimpath -ldflags "-X main.version="$VERSION"" -gcflags="-dwarflocationlists=true" cni/network/stateless/main.go
+RUN GOOS=$OS CGO_ENABLED=0 go build -a -o /go/bin/azure-vnet-stateless -trimpath -ldflags "-X main.version="$VERSION"" -gcflags="-dwarflocationlists=true" cni/network/stateless/main.go
 
 FROM --platform=linux/${ARCH} mcr.microsoft.com/cbl-mariner/base/core:2.0 AS compressor
 ARG OS

cns/cnireconciler/podinfoprovider.go

@@ -149,7 +149,7 @@ func cniStateToCnsEndpointState(state *api.AzureCNIState) map[string]*restserver
 
 // extractEndpointInfo extract Interface Name and endpointID for each endpoint based the CNI state
 func extractEndpointInfo(epID, containerID string) (endpointID, interfaceName string) {
-	ifName := restserver.InterfaceName
+	ifName := restserver.InfraInterfaceName
 	if strings.Contains(epID, "-eth") {
 		ifName = epID[len(epID)-4:]
 	}

cns/restserver/ipam.go

@@ -30,8 +30,8 @@ var (
 )
 
 const (
-	ContainerIDLength = 8
-	InterfaceName     = "eth0"
+	ContainerIDLength  = 8
+	InfraInterfaceName = "eth0"
 )
 
 // requestIPConfigHandlerHelper validates the request, assign IPs and return the IPConfigs
@@ -1071,7 +1071,7 @@ func (service *HTTPRestService) GetEndpointHelper(endpointID string) (*EndpointI
 	// This part is a temprory fix if we have endpoint states belong to CNI version 1.4.X on Windows since the states don't have the containerID
 	// In case there was no endpoint founded with ContainerID as the key,
 	// then [First 8 character of containerid]-eth0 will be tried
-	legacyEndpointID := endpointID[:ContainerIDLength] + "-" + InterfaceName
+	legacyEndpointID := endpointID[:ContainerIDLength] + "-" + InfraInterfaceName
 	if endpointInfo, ok := service.EndpointState[legacyEndpointID]; ok {
 		logger.Warnf("[GetEndpointState] Found existing endpoint state for container %s", legacyEndpointID)
 		return endpointInfo, nil
@@ -1129,42 +1129,67 @@ func (service *HTTPRestService) UpdateEndpointHandler(w http.ResponseWriter, r *
 	logger.Response(service.Name, response, response.ReturnCode, err)
 }
 
-// UpdateEndpointHelper updates the state of the given endpointId with HNSId or VethName
+// UpdateEndpointHelper updates the state of the given endpointId with HNSId, VethName or other InterfaceInfo fields
 func (service *HTTPRestService) UpdateEndpointHelper(endpointID string, req map[string]*IPInfo) error {
 	if service.EndpointStateStore == nil {
 		return ErrStoreEmpty
 	}
 	logger.Printf("[updateEndpoint] Updating endpoint state for infra container %s", endpointID)
 	if endpointInfo, ok := service.EndpointState[endpointID]; ok {
-		for ifName, InterfaceInfo := range req {
-			logger.Printf("[updateEndpoint] Found existing endpoint state for infra container %s", endpointID)
-			if InterfaceInfo.HnsEndpointID != "" {
-				service.EndpointState[endpointID].IfnameToIPMap[ifName].HnsEndpointID = InterfaceInfo.HnsEndpointID
-				logger.Printf("[updateEndpoint] update the endpoint %s with HNSID  %s", endpointID, InterfaceInfo.HnsEndpointID)
-			}
-			if InterfaceInfo.HostVethName != "" {
-				service.EndpointState[endpointID].IfnameToIPMap[ifName].HostVethName = InterfaceInfo.HostVethName
-				logger.Printf("[updateEndpoint] update the endpoint %s with vethName  %s", endpointID, InterfaceInfo.HostVethName)
-			}
-			if InterfaceInfo.NICType != "" {
-				service.EndpointState[endpointID].IfnameToIPMap[ifName].NICType = InterfaceInfo.NICType
-				logger.Printf("[updateEndpoint] update the endpoint %s with NICType  %s", endpointID, InterfaceInfo.NICType)
-			}
+		// Updating the InterfaceInfo map of endpoint states with the interfaceInfo map that is given by Stateless Azure CNI
+		for ifName, interfaceInfo := range req {
+			// updating the ipInfoMap
+			updateIPInfoMap(endpointInfo.IfnameToIPMap, interfaceInfo, ifName, endpointID)
 		}
 		err := service.EndpointStateStore.Write(EndpointStoreKey, service.EndpointState)
 		if err != nil {
 			return fmt.Errorf("[updateEndpoint] failed to write endpoint state to store for pod %s :  %w", endpointInfo.PodName, err)
 		}
+		logger.Printf("[updateEndpoint] successfully write the state to the file %s", endpointID)
 		return nil
 	}
 	return errors.New("[updateEndpoint] endpoint could not be found in the statefile")
 }
 
+// updateIPInfoMap updates the IfnameToIPMap of endpoint states with the interfaceInfo map that is given by Stateless Azure CNI
+func updateIPInfoMap(iPInfo map[string]*IPInfo, interfaceInfo *IPInfo, ifName, endpointID string) {
+	// This codition will create a map for SecodaryNIC and also also creates MAP entry for InfraNic in case that the initial goalState is using empty InterfaceName
+	if _, keyExist := iPInfo[ifName]; !keyExist {
+		iPInfo[ifName] = &IPInfo{}
+		if val, emptyKeyExist := iPInfo[""]; emptyKeyExist && ifName == InfraInterfaceName {
+			iPInfo[ifName].IPv4 = val.IPv4
+			iPInfo[ifName].IPv6 = val.IPv6
+			delete(iPInfo, "")
+		}
+	}
+	logger.Printf("[updateEndpoint] Found existing endpoint state for infra container %s with  %s , [%+v]", endpointID, ifName, interfaceInfo)
+	if interfaceInfo.HnsEndpointID != "" {
+		iPInfo[ifName].HnsEndpointID = interfaceInfo.HnsEndpointID
+		logger.Printf("[updateEndpoint] update the endpoint %s with HNSID  %s", endpointID, interfaceInfo.HnsEndpointID)
+	}
+	if interfaceInfo.HnsNetworkID != "" {
+		iPInfo[ifName].HnsNetworkID = interfaceInfo.HnsNetworkID
+		logger.Printf("[updateEndpoint] update the endpoint %s with HnsNetworkID  %s", endpointID, interfaceInfo.HnsEndpointID)
+	}
+	if interfaceInfo.HostVethName != "" {
+		iPInfo[ifName].HostVethName = interfaceInfo.HostVethName
+		logger.Printf("[updateEndpoint] update the endpoint %s with vethName  %s", endpointID, interfaceInfo.HostVethName)
+	}
+	if interfaceInfo.NICType != "" {
+		iPInfo[ifName].NICType = interfaceInfo.NICType
+		logger.Printf("[updateEndpoint] update the endpoint %s with NICType  %s", endpointID, interfaceInfo.NICType)
+	}
+	if interfaceInfo.MacAddress != "" {
+		iPInfo[ifName].MacAddress = interfaceInfo.MacAddress
+		logger.Printf("[updateEndpoint] update the endpoint %s with MacAddress  %s", endpointID, interfaceInfo.MacAddress)
+	}
+}
+
 // verifyUpdateEndpointStateRequest verify the CNI request body for the UpdateENdpointState API
 func verifyUpdateEndpointStateRequest(req map[string]*IPInfo) error {
 	for ifName, InterfaceInfo := range req {
-		if InterfaceInfo.HostVethName == "" && InterfaceInfo.HnsEndpointID == "" && InterfaceInfo.NICType == "" {
-			return errors.New("[updateEndpoint] No NicType, HnsEndpointID or HostVethName has been provided")
+		if InterfaceInfo.HostVethName == "" && InterfaceInfo.HnsEndpointID == "" && InterfaceInfo.NICType == "" && InterfaceInfo.MacAddress == "" {
+			return errors.New("[updateEndpoint] No NicType, MacAddress, HnsEndpointID or HostVethName has been provided")
 		}
 		if ifName == "" {
 			return errors.New("[updateEndpoint] No Interface has been provided")

cns/restserver/restserver.go

@@ -99,7 +99,9 @@ type IPInfo struct {
 	IPv4          []net.IPNet
 	IPv6          []net.IPNet `json:",omitempty"`
 	HnsEndpointID string      `json:",omitempty"`
+	HnsNetworkID  string      `json:",omitempty"`
 	HostVethName  string      `json:",omitempty"`
+	MacAddress    string      `json:",omitempty"`
 	NICType       cns.NICType
 }
 

network/endpoint.go

@@ -30,6 +30,7 @@ type AzureHNSEndpoint struct{}
 type endpoint struct {
 	Id                       string
 	HnsId                    string `json:",omitempty"`
+	HNSNetworkID             string `json:",omitempty"`
 	SandboxKey               string
 	IfName                   string
 	HostIfName               string
@@ -92,6 +93,7 @@ type EndpointInfo struct {
 	NICType                  cns.NICType
 	SkipDefaultRoutes        bool
 	HNSEndpointID            string
+	HNSNetworkID             string
 	HostIfName               string
 }
 
@@ -351,7 +353,7 @@ func GetPodNameWithoutSuffix(podName string) string {
 
 // IsEndpointStateInComplete returns true if both HNSEndpointID and HostVethName are missing.
 func (epInfo *EndpointInfo) IsEndpointStateIncomplete() bool {
-	if epInfo.HNSEndpointID == "" && epInfo.IfName == "" {
+	if epInfo.HNSEndpointID == "" && epInfo.HostIfName == "" {
 		return true
 	}
 	return false

network/endpoint_windows.go

@@ -499,10 +499,10 @@ func (nm *networkManager) updateEndpointImpl(nw *network, existingEpInfo *Endpoi
 
 // GetEndpointInfoByIPImpl returns an endpointInfo with the corrsponding HNS Endpoint ID that matches an specific IP Address.
 func (epInfo *EndpointInfo) GetEndpointInfoByIPImpl(ipAddresses []net.IPNet, networkID string) (*EndpointInfo, error) {
-	// check if network exists, only create the network does not exist
+	logger.Info("Fetching missing HNS endpoint id for endpoints in network with id", zap.String("id", networkID))
 	hnsResponse, err := Hnsv2.GetNetworkByName(networkID)
-	if err != nil {
-		return epInfo, errors.Wrapf(err, "HNS Network not found")
+	if err != nil || hnsResponse == nil {
+		return epInfo, errors.Wrapf(err, "HNS Network or endpoints not found")
 	}
 	hcnEndpoints, err := Hnsv2.ListEndpointsOfNetwork(hnsResponse.Id)
 	if err != nil {
@@ -511,8 +511,8 @@ func (epInfo *EndpointInfo) GetEndpointInfoByIPImpl(ipAddresses []net.IPNet, net
 	for i := range hcnEndpoints {
 		for _, ipConfiguration := range hcnEndpoints[i].IpConfigurations {
 			for _, ipAddress := range ipAddresses {
-				prefixLength, _ := ipAddress.Mask.Size()
-				if ipConfiguration.IpAddress == ipAddress.IP.String() && ipConfiguration.PrefixLength == uint8(prefixLength) {
+				if ipConfiguration.IpAddress == ipAddress.IP.String() {
+					logger.Info("Successfully found hcn endpoint id for endpoint with ip", zap.String("id", hcnEndpoints[i].Id), zap.String("ip", ipAddress.IP.String()))
 					epInfo.HNSEndpointID = hcnEndpoints[i].Id
 					return epInfo, nil
 				}

network/manager.go

@@ -38,6 +38,7 @@ const (
 	InfraInterfaceName   = "eth0"
 	ContainerIDLength    = 8
 	EndpointIfIndex      = 0 // Azure CNI supports only one interface
+	DefaultNetworkID     = "azure"
 )
 
 var Ipv4DefaultRouteDstPrefix = net.IPNet{
@@ -441,8 +442,10 @@ func (nm *networkManager) GetEndpointState(networkID, endpointID string) (*Endpo
 		return nil, errors.Wrapf(err, "Get endpoint API returend with error")
 	}
 	epInfo := cnsEndpointInfotoCNIEpInfo(endpointResponse.EndpointInfo, endpointID)
-
 	if epInfo.IsEndpointStateIncomplete() {
+		if networkID == "" {
+			networkID = DefaultNetworkID
+		}
 		epInfo, err = epInfo.GetEndpointInfoByIPImpl(epInfo.IPAddresses, networkID)
 		if err != nil {
 			return nil, errors.Wrapf(err, "Get endpoint API returend with error")
@@ -689,6 +692,7 @@ func (nm *networkManager) GetEndpointID(containerID, ifName string) string {
 	return containerID + "-" + ifName
 }
 
+// cnsEndpointInfotoCNIEpInfo convert a CNS endpoint state to CNI EndpointInfo
 func cnsEndpointInfotoCNIEpInfo(endpointInfo restserver.EndpointInfo, endpointID string) *EndpointInfo {
 	epInfo := &EndpointInfo{
 		Id:                 endpointID,
@@ -700,31 +704,34 @@ func cnsEndpointInfotoCNIEpInfo(endpointInfo restserver.EndpointInfo, endpointID
 	}
 
 	for ifName, ipInfo := range endpointInfo.IfnameToIPMap {
+		// This is an special case for endpoint state that are being crated by statefull CNI
+		if ifName == "" {
+			ifName = InfraInterfaceName
+		}
+		// TODO: DelegatedNIC state will be added in a future PR
 		if ifName != InfraInterfaceName {
-			// TODO: filling out the SecondaryNICs from the state for Swift 2.0
 			continue
 		}
-		// filling out the InfraNIC from the state
 		epInfo.IPAddresses = ipInfo.IPv4
 		epInfo.IPAddresses = append(epInfo.IPAddresses, ipInfo.IPv6...)
 		epInfo.IfName = ifName
 		epInfo.HostIfName = ipInfo.HostVethName
 		epInfo.HNSEndpointID = ipInfo.HnsEndpointID
+		epInfo.HNSNetworkID = ipInfo.HnsNetworkID
+		epInfo.MacAddress = net.HardwareAddr(ipInfo.MacAddress)
 	}
 	return epInfo
 }
 
+// generateCNSIPInfoMap generates a CNS ifNametoIPInfoMap structure based on CNI endpoint
 func generateCNSIPInfoMap(ep *endpoint) map[string]*restserver.IPInfo {
 	ifNametoIPInfoMap := make(map[string]*restserver.IPInfo) // key : interface name, value : IPInfo
-	if ep.IfName != "" {
-		ifNametoIPInfoMap[ep.IfName].NICType = cns.InfraNIC
-		ifNametoIPInfoMap[ep.IfName].HnsEndpointID = ep.HnsId
-		ifNametoIPInfoMap[ep.IfName].HostVethName = ep.HostIfName
-	}
-	if ep.SecondaryInterfaces != nil {
-		for ifName, InterfaceInfo := range ep.SecondaryInterfaces {
-			ifNametoIPInfoMap[ifName].NICType = InterfaceInfo.NICType
-		}
+	ifNametoIPInfoMap[ep.IfName] = &restserver.IPInfo{
+		NICType:       cns.InfraNIC,
+		HnsEndpointID: ep.HnsId,
+		HnsNetworkID:  ep.HNSNetworkID,
+		HostVethName:  ep.HostIfName,
+		MacAddress:    ep.MacAddress.String(),
 	}
 	return ifNametoIPInfoMap
 }