azure-container-networking/npm/npm.go

// Copyright 2018 Microsoft. All rights reserved.
// MIT License
package npm

import (
	"encoding/json"
	"fmt"
	"time"

	npmconfig "github.com/Azure/azure-container-networking/npm/config"
	"github.com/Azure/azure-container-networking/npm/ipsm"
	"github.com/Azure/azure-container-networking/npm/pkg/controlplane/controllers/common"
	controllersv1 "github.com/Azure/azure-container-networking/npm/pkg/controlplane/controllers/v1"
	controllersv2 "github.com/Azure/azure-container-networking/npm/pkg/controlplane/controllers/v2"
	"github.com/Azure/azure-container-networking/npm/pkg/dataplane"
	"github.com/Azure/azure-container-networking/npm/pkg/models"
	"github.com/Azure/azure-container-networking/npm/util"
	"github.com/pkg/errors"
	"k8s.io/apimachinery/pkg/version"
	"k8s.io/client-go/informers"
	"k8s.io/client-go/tools/cache"
	"k8s.io/klog"
	utilexec "k8s.io/utils/exec"
)

var aiMetadata string //nolint // aiMetadata is set in Makefile

// waitDurationAfterStartingNetPolController is used when configured to apply dataplane in the background
// Worst case, SetPolicy SysCalls take ~30 seconds.
// So with a 3 minute wait, the dataplane can process about 600 (6*maxBatches) NetworkPolicies before starting the Pod controller
var waitDurationAfterStartingNetPolController = 3 * time.Minute

// NetworkPolicyManager contains informers for pod, namespace and networkpolicy.
type NetworkPolicyManager struct {
	config npmconfig.Config

	Dataplane dataplane.GenericDataplane

	NpmLiteToggle bool

	// ipsMgr are shared in all controllers. Thus, only one ipsMgr is created for simple management
	// and uses lock to avoid unintentional race condictions in IpsetManager.
	ipsMgr *ipsm.IpsetManager

	// Informers are the Kubernetes Informer
	// https://pkg.go.dev/k8s.io/client-go/informers
	models.Informers

	// Legacy controllers for handling Kubernetes resource watcher events
	// To be deprecated
	models.K8SControllersV1

	// Controllers for handling Kubernetes resource watcher events
	models.K8SControllersV2

	// Azure-specific variables
	models.AzureConfig
}

// NewNetworkPolicyManager creates a NetworkPolicyManager
func NewNetworkPolicyManager(config npmconfig.Config,
	informerFactory informers.SharedInformerFactory,
	podFactory informers.SharedInformerFactory,
	dp dataplane.GenericDataplane,
	exec utilexec.Interface,
	npmVersion string,
	k8sServerVersion *version.Info) *NetworkPolicyManager {
	klog.Infof("API server version: %+v AI metadata %+v", k8sServerVersion, aiMetadata)

	npMgr := &NetworkPolicyManager{
		config:        config,
		Dataplane:     dp,
		NpmLiteToggle: config.Toggles.EnableNPMLite,
		Informers: models.Informers{
			InformerFactory:    informerFactory,
			PodInformerFactory: podFactory,
			PodInformer:        podFactory.Core().V1().Pods(),
			NsInformer:         informerFactory.Core().V1().Namespaces(),
			NpInformer:         informerFactory.Networking().V1().NetworkPolicies(),
		},
		AzureConfig: models.AzureConfig{
			K8sServerVersion: k8sServerVersion,
			NodeName:         models.GetNodeName(),
			Version:          npmVersion,
			TelemetryEnabled: true,
		},
	}

	// create v2 NPM specific components.
	if npMgr.config.Toggles.EnableV2NPM {
		npMgr.NpmNamespaceCacheV2 = &controllersv2.NpmNamespaceCache{NsMap: make(map[string]*common.Namespace)}
		npMgr.PodControllerV2 = controllersv2.NewPodController(npMgr.PodInformer, dp, npMgr.NpmNamespaceCacheV2)
		npMgr.NamespaceControllerV2 = controllersv2.NewNamespaceController(npMgr.NsInformer, dp, npMgr.NpmNamespaceCacheV2)
		// Question(jungukcho): Is config.Toggles.PlaceAzureChainFirst needed for v2?
		npMgr.NetPolControllerV2 = controllersv2.NewNetworkPolicyController(npMgr.NpInformer, dp, config.Toggles.EnableNPMLite)
		return npMgr
	}

	// create v1 NPM specific components.
	npMgr.ipsMgr = ipsm.NewIpsetManager(exec)

	npMgr.NpmNamespaceCacheV1 = &controllersv1.NpmNamespaceCache{NsMap: make(map[string]*common.Namespace)}
	npMgr.PodControllerV1 = controllersv1.NewPodController(npMgr.PodInformer, npMgr.ipsMgr, npMgr.NpmNamespaceCacheV1)
	npMgr.NamespaceControllerV1 = controllersv1.NewNameSpaceController(npMgr.NsInformer, npMgr.ipsMgr, npMgr.NpmNamespaceCacheV1)
	npMgr.NetPolControllerV1 = controllersv1.NewNetworkPolicyController(npMgr.NpInformer, npMgr.ipsMgr, config.Toggles.PlaceAzureChainFirst)
	return npMgr
}

// Dear Time Traveler:
// This is the server end of the debug dragons den. Several of these properties of the
// npMgr struct have overridden methods which override the MarshalJson, just as this one
// is doing for npMgr. For example, npMgr.NamespaceCacheV2 does not marshal the whole struct,
// but rather the NsMap of type map[string]*Namespace. When unmarshaling, expect this type,
// and pay very close attention. Many hours have been wasted here when unmarshaling mismatched types.
func (npMgr *NetworkPolicyManager) MarshalJSON() ([]byte, error) {
	m := map[models.CacheKey]json.RawMessage{}

	if npMgr.config.Toggles.EnableV2NPM {
		npmNamespaceCacheRaw, err := json.Marshal(npMgr.NpmNamespaceCacheV2)
		if err != nil {
			return nil, errors.Wrapf(err, "failed to marshal v2 ns cache")
		}
		m[models.NsMap] = npmNamespaceCacheRaw

		podControllerRaw, err := json.Marshal(npMgr.PodControllerV2)
		if err != nil {
			return nil, errors.Wrapf(err, "failed to marshal v2 pod controller")
		}
		m[models.PodMap] = podControllerRaw

		setMapRaw, err := json.Marshal(npMgr.Dataplane.GetAllIPSets())
		if err != nil {
			return nil, errors.Wrapf(err, "failed to marshal v2 set map")
		}
		m[models.SetMap] = setMapRaw

	} else {
		npmNamespaceCacheRaw, err := json.Marshal(npMgr.NpmNamespaceCacheV1)
		if err != nil {
			return nil, errors.Wrapf(err, "failed to marshal v1 ns cache")
		}
		m[models.NsMap] = npmNamespaceCacheRaw

		podControllerRaw, err := json.Marshal(npMgr.PodControllerV1)
		if err != nil {
			return nil, errors.Wrapf(err, "failed to marshal v1 pod controller")
		}
		m[models.PodMap] = podControllerRaw

		listMapRaw, err := npMgr.ipsMgr.GetListMapRaw()
		if err != nil {
			return nil, errors.Wrapf(err, "failed to marshal v1 list map")
		}
		m[models.ListMap] = listMapRaw

		setMapRaw, err := npMgr.ipsMgr.GetSetMapRaw()
		if err != nil {
			return nil, errors.Wrapf(err, "failed to marshal v1 set map")
		}
		m[models.SetMap] = setMapRaw

	}

	nodenameRaw, err := json.Marshal(npMgr.NodeName)
	if err != nil {
		return nil, errors.Wrapf(err, "failed to marshal node name")
	}
	m[models.NodeName] = nodenameRaw

	npmCacheRaw, err := json.Marshal(m)
	if err != nil {
		return nil, errors.Wrapf(err, "failed to marshall the cache map")
	}

	return npmCacheRaw, nil
}

// GetAppVersion returns network policy manager app version
func (npMgr *NetworkPolicyManager) GetAppVersion() string {
	return npMgr.Version
}

// Start starts shared informers and waits for the shared informer cache to sync.
func (npMgr *NetworkPolicyManager) Start(config npmconfig.Config, stopCh <-chan struct{}) error {
	if !config.Toggles.EnableV2NPM {
		// Do initialization of data plane before starting syncup of each controller to avoid heavy call to api-server
		if err := npMgr.NetPolControllerV1.BootupDataplane(); err != nil {
			return fmt.Errorf("Failed to initialized data plane with err %w", err)
		}
	}

	// Starts all informers manufactured by npMgr's informerFactory.
	npMgr.InformerFactory.Start(stopCh)

	// npm lite
	if npMgr.NpmLiteToggle {
		npMgr.PodInformerFactory.Start(stopCh)
	}

	// Wait for the initial sync of local cache.
	if !cache.WaitForCacheSync(stopCh, npMgr.PodInformer.Informer().HasSynced) {
		return fmt.Errorf("Pod informer error: %w", models.ErrInformerSyncFailure)
	}

	if !cache.WaitForCacheSync(stopCh, npMgr.NsInformer.Informer().HasSynced) {
		return fmt.Errorf("Namespace informer error: %w", models.ErrInformerSyncFailure)
	}

	if !cache.WaitForCacheSync(stopCh, npMgr.NpInformer.Informer().HasSynced) {
		return fmt.Errorf("NetworkPolicy informer error: %w", models.ErrInformerSyncFailure)
	}

	// start v2 NPM controllers after synced
	if config.Toggles.EnableV2NPM {
		go npMgr.NetPolControllerV2.Run(stopCh)

		if util.IsWindowsDP() && config.Toggles.ApplyInBackground {
			klog.Infof("optimizing NPM bootup by letting NetPol controller process changes first. waiting %v before starting pod and namespace controllers", waitDurationAfterStartingNetPolController)
			time.Sleep(waitDurationAfterStartingNetPolController)
		}

		npMgr.Dataplane.FinishBootupPhase()

		go npMgr.PodControllerV2.Run(stopCh)
		go npMgr.NamespaceControllerV2.Run(stopCh)

		return nil
	}

	// start v1 NPM controllers after synced
	go npMgr.PodControllerV1.Run(stopCh)
	go npMgr.NamespaceControllerV1.Run(stopCh)
	go npMgr.NetPolControllerV1.Run(stopCh)
	go npMgr.NetPolControllerV1.RunPeriodicTasks(stopCh)

	return nil
}

// GetAIMetadata returns ai metadata number
func GetAIMetadata() string {
	return aiMetadata
}