Add swiftv2 middleware support for standalone service fabric scenario (windows) (#2634)

* feature: add sfswiftv2 middleware support for standalone service fabric swiftv2 windows path

* feat: add switch case for middlewaretype

* review: address comments

* pkg: move swiftv2mode for cnsconfig from configuration to cns package to avoid import-cycle

* address review comment

* linter: context check, wrap error

* linter: fix errors wrap return static err

* review: address comments:

* refactor: keep old variable enableswiftv2, initialization pattern

* address pr comments

* rename the middleware to standaloneswiftv2middleware

* remove HostSecondaryIPInfo from podIpInfo

* address review comments-1

* address review comments-2

* Update cns/service/main.go

Co-authored-by: tamilmani1989 <tamanoha@microsoft.com>
Signed-off-by: Kshitija Murudi <kmurudi@microsoft.com>

* fix an linter issue

* rename the IPConfigRequest func

* fix a comment linter issue

* fix a comment linter issue

* remove checking hnsTransparent network mdoe

* remove orchestrator checker

---------

Signed-off-by: Kshitija Murudi <kmurudi@microsoft.com>
Co-authored-by: tamilmani1989 <tamanoha@microsoft.com>
Co-authored-by: Paul Yu <129891899+paulyufan2@users.noreply.github.com>
Co-authored-by: paulyufan2 <paulyu01@outlook.com>

cns/api.go

@@ -35,6 +35,10 @@ const (
 	V1Prefix                      = "/v0.1"
 	V2Prefix                      = "/v0.2"
 	EndpointPath                  = "/network/endpoints/"
+	// Service Fabric SWIFTV2 mode
+	StandaloneSWIFTV2 SWIFTV2Mode = "StandaloneSWIFTV2"
+	// K8s SWIFTV2 mode
+	K8sSWIFTV2 SWIFTV2Mode = "K8sSWIFTV2"
 )
 
 // HTTPService describes the min API interface that every service should have.
@@ -59,8 +63,12 @@ type IPConfigsHandlerFunc func(context.Context, IPConfigsRequest) (*IPConfigsRes
 // IPConfigsHandlerMiddleware
 type IPConfigsHandlerMiddleware interface {
 	IPConfigsRequestHandlerWrapper(defaultHandler IPConfigsHandlerFunc, failureHandler IPConfigsHandlerFunc) IPConfigsHandlerFunc
+	Type() SWIFTV2Mode
 }
 
+// SWIFTV2Mode describes the orchestrator-related scenario for swiftv2 flow, used in CNSConfig
+type SWIFTV2Mode string
+
 // This is used for KubernetesCRD orchestrator Type where NC has multiple ips.
 // This struct captures the state for SecondaryIPs associated to a given NC
 type IPConfigurationStatus struct {

cns/configuration/configuration.go

@@ -13,16 +13,10 @@ import (
 	"github.com/pkg/errors"
 )
 
-type SWIFTV2Mode string
-
 const (
 	// EnvCNSConfig is the CNS_CONFIGURATION_PATH env var key
 	EnvCNSConfig      = "CNS_CONFIGURATION_PATH"
 	defaultConfigName = "cns_config.json"
-	// Service Fabric SWIFTV2 mode
-	SFSWIFTV2 SWIFTV2Mode = "SFSWIFTV2"
-	// K8s SWIFTV2 mode
-	K8sSWIFTV2 SWIFTV2Mode = "K8sSWIFTV2"
 )
 
 type CNSConfig struct {
@@ -46,7 +40,6 @@ type CNSConfig struct {
 	MellanoxMonitorIntervalSecs int
 	MetricsBindAddress          string
 	ProgramSNATIPTables         bool
-	SWIFTV2Mode                 SWIFTV2Mode
 	SyncHostNCTimeoutMs         int
 	SyncHostNCVersionIntervalMs int
 	TLSCertificatePath          string

cns/middlewares/k8sSwiftV2.go

@@ -245,3 +245,7 @@ func (k *K8sSWIFTv2Middleware) getIPConfig(ctx context.Context, podInfo cns.PodI
 
 	return podIPInfos, nil
 }
+
+func (k *K8sSWIFTv2Middleware) Type() cns.SWIFTV2Mode {
+	return cns.K8sSWIFTV2
+}

cns/middlewares/standaloneSwiftV2.go

@@ -0,0 +1,29 @@
+package middlewares
+
+import (
+	"context"
+
+	"github.com/Azure/azure-container-networking/cns"
+	"github.com/Azure/azure-container-networking/cns/types"
+	"github.com/pkg/errors"
+)
+
+type StandaloneSWIFTv2Middleware struct{}
+
+// IPConfigsRequestHandlerWrapper is the middleware function for handling SWIFT v2 IP config requests for SF standalone scenario. This function wraps the default SWIFT request
+// and release IP configs handlers.
+func (m *StandaloneSWIFTv2Middleware) IPConfigsRequestHandlerWrapper(ipRequestHandler, _ cns.IPConfigsHandlerFunc) cns.IPConfigsHandlerFunc {
+	return func(ctx context.Context, req cns.IPConfigsRequest) (*cns.IPConfigsResponse, error) {
+		ipConfigsResp, err := ipRequestHandler(ctx, req)
+		if err != nil {
+			ipConfigsResp.Response.ReturnCode = types.UnexpectedError
+			return ipConfigsResp, errors.Wrapf(err, "Failed to requestIPConfigs for Standalone-SwiftV2 from IPConfigsRequest %v", req)
+		}
+
+		return ipConfigsResp, nil
+	}
+}
+
+func (m *StandaloneSWIFTv2Middleware) Type() cns.SWIFTV2Mode {
+	return cns.StandaloneSWIFTV2
+}

cns/restserver/ipam.go

@@ -27,6 +27,7 @@ var (
 	ErrNoNCs                  = errors.New("no NCs found in the CNS internal state")
 	ErrOptManageEndpointState = errors.New("CNS is not set to manage the endpoint state")
 	ErrEndpointStateNotFound  = errors.New("endpoint state could not be found in the statefile")
+	ErrGetAllNCResponseEmpty  = errors.New("failed to get NC responses from statefile")
 )
 
 const (
@@ -113,6 +114,81 @@ func (service *HTTPRestService) requestIPConfigHandlerHelper(ctx context.Context
 	}, nil
 }
 
+// requestIPConfigHandlerHelperStandalone validates the request, assign IPs and return the IPConfigs
+func (service *HTTPRestService) requestIPConfigHandlerHelperStandalone(ctx context.Context, ipconfigsRequest cns.IPConfigsRequest) (*cns.IPConfigsResponse, error) {
+	// For SWIFT v2 scenario, the validator function will also modify the ipconfigsRequest.
+	podInfo, returnCode, returnMessage := service.validateIPConfigsRequest(ctx, ipconfigsRequest)
+	if returnCode != types.Success {
+		return &cns.IPConfigsResponse{
+			Response: cns.Response{
+				ReturnCode: returnCode,
+				Message:    returnMessage,
+			},
+		}, errors.New("failed to validate ip config request or unmarshal orchestratorContext")
+	}
+
+	orchestratorContext, err := podInfo.OrchestratorContext()
+	if err != nil {
+		return &cns.IPConfigsResponse{}, fmt.Errorf("error getting orchestrator context from PodInfo %w", err)
+	}
+	cnsRequest := cns.GetNetworkContainerRequest{OrchestratorContext: orchestratorContext}
+	resp := service.getAllNetworkContainerResponses(cnsRequest) //nolint:contextcheck // not passed in any methods, appease linter
+	// return err if returned list has no NCs
+	if len(resp) == 0 {
+		return &cns.IPConfigsResponse{
+			Response: cns.Response{
+				ReturnCode: types.FailedToAllocateIPConfig,
+				Message:    fmt.Sprintf("AllocateIPConfig failed due to not getting NC Response from statefile, IP config request is %v", ipconfigsRequest),
+			},
+		}, ErrGetAllNCResponseEmpty
+	}
+
+	podIPInfoList := make([]cns.PodIpInfo, 0, len(resp))
+	for i := range resp {
+		podIPInfo := cns.PodIpInfo{
+			PodIPConfig:                     resp[i].IPConfiguration.IPSubnet,
+			MacAddress:                      resp[i].NetworkInterfaceInfo.MACAddress,
+			NICType:                         resp[i].NetworkInterfaceInfo.NICType,
+			NetworkContainerPrimaryIPConfig: resp[i].IPConfiguration,
+		}
+		podIPInfoList = append(podIPInfoList, podIPInfo)
+	}
+
+	ipConfigsResp := &cns.IPConfigsResponse{
+		Response: cns.Response{
+			ReturnCode: types.Success,
+		},
+		PodIPInfo: podIPInfoList,
+	}
+
+	err = service.updatePodInfoWithInterfaces(ctx, ipConfigsResp)
+	if err != nil {
+		return &cns.IPConfigsResponse{
+			Response: cns.Response{
+				ReturnCode: types.FailedToAllocateIPConfig,
+				Message:    fmt.Sprintf("AllocateIPConfig failed while updating pod with interfaces: %v, IP config request is %v", err, ipconfigsRequest),
+			},
+		}, err
+	}
+	return ipConfigsResp, nil
+}
+
+func (service *HTTPRestService) updatePodInfoWithInterfaces(ctx context.Context, ipconfigResponse *cns.IPConfigsResponse) error {
+	// fetching primary host interface to use below for updating IPConfigsResponse
+	hostPrimaryInterface, err := service.getPrimaryHostInterface(ctx)
+	if err != nil {
+		return err
+	}
+	for i := range ipconfigResponse.PodIPInfo {
+		ipconfigResponse.PodIPInfo[i].HostPrimaryIPInfo = cns.HostIPInfo{
+			Gateway:   hostPrimaryInterface.Gateway,
+			PrimaryIP: hostPrimaryInterface.PrimaryIP,
+			Subnet:    hostPrimaryInterface.Subnet,
+		}
+	}
+	return nil
+}
+
 // RequestIPConfigHandler requests an IPConfig from the CNS state
 func (service *HTTPRestService) RequestIPConfigHandler(w http.ResponseWriter, r *http.Request) {
 	var ipconfigRequest cns.IPConfigRequest
@@ -201,8 +277,16 @@ func (service *HTTPRestService) RequestIPConfigsHandler(w http.ResponseWriter, r
 
 	// Check if IPConfigsHandlerMiddleware is set
 	if service.IPConfigsHandlerMiddleware != nil {
-		// Wrap the default datapath handlers with the middleware
-		wrappedHandler := service.IPConfigsHandlerMiddleware.IPConfigsRequestHandlerWrapper(service.requestIPConfigHandlerHelper, service.ReleaseIPConfigHandlerHelper)
+		// Wrap the default datapath handlers with the middleware depending on middleware type
+		var wrappedHandler cns.IPConfigsHandlerFunc
+		switch service.IPConfigsHandlerMiddleware.Type() {
+		case cns.K8sSWIFTV2:
+			wrappedHandler = service.IPConfigsHandlerMiddleware.IPConfigsRequestHandlerWrapper(service.requestIPConfigHandlerHelper, service.ReleaseIPConfigHandlerHelper)
+		// this middleware is used for standalone swiftv2 secenario where a different helper is invoked as the PodInfo is read from cns state
+		case cns.StandaloneSWIFTV2:
+			wrappedHandler = service.IPConfigsHandlerMiddleware.IPConfigsRequestHandlerWrapper(service.requestIPConfigHandlerHelperStandalone, nil)
+		}
+
 		ipConfigsResp, err = wrappedHandler(r.Context(), ipconfigsRequest)
 	} else {
 		ipConfigsResp, err = service.requestIPConfigHandlerHelper(r.Context(), ipconfigsRequest) // nolint:contextcheck // appease linter

cns/restserver/util.go

@@ -786,12 +786,8 @@ func (service *HTTPRestService) SendNCSnapShotPeriodically(ctx context.Context,
 }
 
 func (service *HTTPRestService) validateIPConfigsRequest(ctx context.Context, ipConfigsRequest cns.IPConfigsRequest) (cns.PodInfo, types.ResponseCode, string) {
-	if service.state.OrchestratorType != cns.KubernetesCRD && service.state.OrchestratorType != cns.Kubernetes {
-		return nil, types.UnsupportedOrchestratorType, "ReleaseIPConfig API supported only for kubernetes orchestrator"
-	}
-
 	if ipConfigsRequest.OrchestratorContext == nil {
-		return nil, types.EmptyOrchestratorContext, fmt.Sprintf("OrchastratorContext is not set in the req: %+v", ipConfigsRequest)
+		return nil, types.EmptyOrchestratorContext, fmt.Sprintf("OrchestratorContext is not set in the req: %+v", ipConfigsRequest)
 	}
 
 	// retrieve podinfo from orchestrator context

cns/service/main.go

@@ -814,6 +814,12 @@ func main() {
 		if platform.HasMellanoxAdapter() {
 			go platform.MonitorAndSetMellanoxRegKeyPriorityVLANTag(rootCtx, cnsconfig.MellanoxMonitorIntervalSecs)
 		}
+
+		// if swiftv2 scenario is enabled, we need to initialize the ServiceFabric(standalone) swiftv2 middleware to process IPConfigsRequests
+		// RequestIPConfigs() will be invoked only for swiftv2 or swiftv1 k8s scenarios. For swiftv1 direct mode different api will be invoked.
+		// So initializing this middleware always under direct mode should not affect any other scenarios
+		swiftV2Middleware := &middlewares.StandaloneSWIFTv2Middleware{}
+		httpRemoteRestService.AttachIPConfigsHandlerMiddleware(swiftV2Middleware)
 	}
 
 	// Initialze state in if CNS is running in CRD mode
@@ -1486,17 +1492,8 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn
 			return errors.Wrapf(err, "failed to setup mtpnc reconciler with manager")
 		}
 		// if SWIFT v2 is enabled on CNS, attach multitenant middleware to rest service
-		// switch here for different type of swift v2 middleware (k8s or SF)
-		var swiftV2Middleware cns.IPConfigsHandlerMiddleware
-		switch cnsconfig.SWIFTV2Mode {
-		case configuration.K8sSWIFTV2:
-			swiftV2Middleware = &middlewares.K8sSWIFTv2Middleware{Cli: manager.GetClient()}
-		case configuration.SFSWIFTV2:
-		default:
-			// default to K8s middleware for now, in a later changes we where start to pass in
-			// SWIFT v2 mode in CNS config, this should throw an error if the mode is not set.
-			swiftV2Middleware = &middlewares.K8sSWIFTv2Middleware{Cli: manager.GetClient()}
-		}
+		// switch here for AKS(K8s) swiftv2 middleware to process IP configs requests
+		swiftV2Middleware := &middlewares.K8sSWIFTv2Middleware{Cli: manager.GetClient()}
 		httpRestService.AttachIPConfigsHandlerMiddleware(swiftV2Middleware)
 	}
 

cns/wireserver/net.go

@@ -7,7 +7,7 @@ import (
 )
 
 var (
-	// ErrNoPrimaryInterface indicates the wireserver respnose does not have a primary interface indicated.
+	// ErrNoPrimaryInterface indicates the wireserver response does not have a primary interface indicated.
 	ErrNoPrimaryInterface = errors.New("no primary interface found")
 	// ErrInsufficientAddressSpace indicates that the CIDR space is too small to include a gateway IP; it is 1 IP.
 	ErrInsufficientAddressSpace = errors.New("insufficient address space to generate gateway IP")