[NPM] Generic Dataplane interface for windows and linux (#984)

* adding initial dataplane interface

* adding some skeleton code for crud functions

* adding dirty cache initial support for ipset manager

* correcting some go lints

* Adding some skeleton around os specific files

* removing interface and adding some network logic

* adding apply ipsets logic

* Addressing some comments and also adding comments to code

* Fixing some golints

* addressing some comments

* adding some golint checks

* Adding a new field setProperty and also adding structure for policies

* applying some comments

* correcting a condition

* Adding some comments

* Adding some test cases

* Addressing some comments

* Addressing more comments

* resolving some comments

* resolving some comments

* fixing some comments

* removing lock on policymap

* fixing some golints

* merging with master

* fixingsome wrap checks

* fixing lints

npm/npm_test.go

@@ -9,7 +9,6 @@ import (
 	"github.com/Azure/azure-container-networking/npm/metrics"
 	"k8s.io/client-go/tools/cache"
 	"k8s.io/utils/exec"
-	utilexec "k8s.io/utils/exec"
 )
 
 // To indicate the object is needed to be DeletedFinalStateUnknown Object
@@ -29,15 +28,6 @@ func getKey(obj interface{}, t *testing.T) string {
 	return key
 }
 
-func newNPMgr(t *testing.T, exec utilexec.Interface) *NetworkPolicyManager {
-	npMgr := &NetworkPolicyManager{
-		ipsMgr:           ipsm.NewIpsetManager(exec),
-		TelemetryEnabled: false,
-	}
-
-	return npMgr
-}
-
 func TestMain(m *testing.M) {
 	metrics.InitializeAll()
 	exec := exec.New()

npm/pkg/dataplane/dataplane.go

@@ -0,0 +1,162 @@
+package dataplane
+
+import (
+	"fmt"
+
+	"github.com/Azure/azure-container-networking/npm"
+	"github.com/Azure/azure-container-networking/npm/pkg/dataplane/ipsets"
+	"github.com/Azure/azure-container-networking/npm/pkg/dataplane/policies"
+)
+
+type DataPlane struct {
+	policyMgr policies.PolicyManager
+	ipsetMgr  ipsets.IPSetManager
+	networkID string
+	// key is PodKey
+	endpointCache map[string]*NPMEndpoint
+}
+
+type NPMEndpoint struct {
+	Name string
+	ID   string
+	// Map with Key as Network Policy name to to emulate set
+	// and value as struct{} for minimal memory consumption
+	NetPolReference map[string]struct{}
+}
+
+func NewDataPlane() *DataPlane {
+	return &DataPlane{
+		policyMgr:     policies.NewPolicyManager(),
+		ipsetMgr:      ipsets.NewIPSetManager(),
+		endpointCache: make(map[string]*NPMEndpoint),
+	}
+}
+
+// InitializeDataPlane helps in setting up dataplane for NPM
+func (dp *DataPlane) InitializeDataPlane() error {
+	return dp.initializeDataPlane()
+}
+
+// ResetDataPlane helps in cleaning up dataplane sets and policies programmed
+// by NPM, retunring a clean slate
+func (dp *DataPlane) ResetDataPlane() error {
+	return dp.resetDataPlane()
+}
+
+// CreateIPSet takes in a set object and updates local cache with this set
+func (dp *DataPlane) CreateIPSet(set *ipsets.IPSet) error {
+	err := dp.ipsetMgr.CreateIPSet(set)
+	if err != nil {
+		return fmt.Errorf("[DataPlane] error while creating set: %w", err)
+	}
+	return nil
+}
+
+// DeleteSet checks for members and references of the given "set" type ipset
+// if not used then will delete it from cache
+func (dp *DataPlane) DeleteSet(name string) error {
+	err := dp.ipsetMgr.DeleteSet(name)
+	if err != nil {
+		return fmt.Errorf("[DataPlane] error while deleting set: %w", err)
+	}
+	return nil
+}
+
+// DeleteList sanity checks and deletes a list ipset
+func (dp *DataPlane) DeleteList(name string) error {
+	err := dp.ipsetMgr.DeleteList(name)
+	if err != nil {
+		return fmt.Errorf("[DataPlane] error while deleting list: %w", err)
+	}
+	return nil
+}
+
+// AddToSet takes in a list of IPset objects along with IP member
+// and then updates it local cache
+func (dp *DataPlane) AddToSet(setNames []*ipsets.IPSet, ip, podKey string) error {
+	err := dp.ipsetMgr.AddToSet(setNames, ip, podKey)
+	if err != nil {
+		return fmt.Errorf("[DataPlane] error while adding to set: %w", err)
+	}
+	return nil
+}
+
+// RemoveFromSet takes in list of setnames from which a given IP member should be
+// removed and will update the local cache
+func (dp *DataPlane) RemoveFromSet(setNames []string, ip, podKey string) error {
+	err := dp.ipsetMgr.RemoveFromSet(setNames, ip, podKey)
+	if err != nil {
+		return fmt.Errorf("[DataPlane] error while removing from set: %w", err)
+	}
+	return nil
+}
+
+// AddToList takes a list name and list of sets which are to be added as members
+// to given list
+func (dp *DataPlane) AddToList(listName string, setNames []string) error {
+	err := dp.ipsetMgr.AddToList(listName, setNames)
+	if err != nil {
+		return fmt.Errorf("[DataPlane] error while adding to list: %w", err)
+	}
+	return nil
+}
+
+// RemoveFromList takes a list name and list of sets which are to be removed as members
+// to given list
+func (dp *DataPlane) RemoveFromList(listName string, setNames []string) error {
+	err := dp.ipsetMgr.RemoveFromList(listName, setNames)
+	if err != nil {
+		return fmt.Errorf("[DataPlane] error while removing from list: %w", err)
+	}
+	return nil
+}
+
+// UpdatePod is to be called by pod_controller ONLY when a new pod is CREATED.
+func (dp *DataPlane) UpdatePod(pod *npm.NpmPod) error {
+	err := dp.updatePod(pod)
+	if err != nil {
+		return fmt.Errorf("[DataPlane] error while updating pod: %w", err)
+	}
+	return nil
+}
+
+// ApplyDataPlane all the IPSet operations just update cache and update a dirty ipset structure,
+// they do not change apply changes into dataplane. This function needs to be called at the
+// end of IPSet operations of a given controller event, it will check for the dirty ipset list
+// and accordingly makes changes in dataplane. This function helps emulate a single call to
+// dataplane instead of multiple ipset operations calls ipset operations calls to dataplane
+func (dp *DataPlane) ApplyDataPlane() error {
+	err := dp.ipsetMgr.ApplyIPSets(dp.networkID)
+	if err != nil {
+		return fmt.Errorf("[DataPlane] error while applying IPSets: %w", err)
+	}
+	return nil
+}
+
+// AddPolicy takes in a translated NPMNetworkPolicy object and applies on dataplane
+func (dp *DataPlane) AddPolicy(policies *policies.NPMNetworkPolicy) error {
+	err := dp.policyMgr.AddPolicy(policies)
+	if err != nil {
+		return fmt.Errorf("[DataPlane] error while adding policy: %w", err)
+	}
+	return nil
+}
+
+// RemovePolicy takes in network policy name and removes it from dataplane and cache
+func (dp *DataPlane) RemovePolicy(policyName string) error {
+	err := dp.policyMgr.RemovePolicy(policyName)
+	if err != nil {
+		return fmt.Errorf("[DataPlane] error while removing policy: %w", err)
+	}
+	return nil
+}
+
+// UpdatePolicy takes in updated policy object, calculates the delta and applies changes
+// onto dataplane accordingly
+func (dp *DataPlane) UpdatePolicy(policies *policies.NPMNetworkPolicy) error {
+	err := dp.policyMgr.UpdatePolicy(policies)
+	if err != nil {
+		return fmt.Errorf("[DataPlane] error while updating policy: %w", err)
+	}
+	return nil
+}

npm/pkg/dataplane/dataplane_linux.go

@@ -0,0 +1,21 @@
+package dataplane
+
+import (
+	"github.com/Azure/azure-container-networking/npm"
+	"k8s.io/klog"
+)
+
+// initializeDataPlane should be adding required chains and rules
+func (dp *DataPlane) initializeDataPlane() error {
+	klog.Infof("Initializing dataplane for linux")
+	return nil
+}
+
+// updatePod is no-op in Linux
+func (dp *DataPlane) updatePod(pod *npm.NpmPod) error {
+	return nil
+}
+
+func (dp *DataPlane) resetDataPlane() error {
+	return nil
+}

npm/pkg/dataplane/dataplane_test.go

@@ -0,0 +1,23 @@
+package dataplane
+
+import (
+	"testing"
+
+	"github.com/Azure/azure-container-networking/npm/metrics"
+	"github.com/Azure/azure-container-networking/npm/pkg/dataplane/ipsets"
+)
+
+func TestNewDataPlane(t *testing.T) {
+	metrics.InitializeAll()
+	dp := NewDataPlane()
+
+	if dp == nil {
+		t.Error("NewDataPlane() returned nil")
+	}
+	set := ipsets.NewIPSet("test", ipsets.NameSpace)
+
+	err := dp.CreateIPSet(set)
+	if err != nil {
+		t.Error("CreateIPSet() returned error")
+	}
+}

npm/pkg/dataplane/dataplane_windows.go

@@ -0,0 +1,54 @@
+package dataplane
+
+import (
+	"github.com/Azure/azure-container-networking/npm"
+	"github.com/Microsoft/hcsshim/hcn"
+	"k8s.io/klog"
+)
+
+const (
+	// Windows specific constants
+	AzureNetworkName = "azure"
+)
+
+// initializeDataPlane will help gather network and endpoint details
+func (dp *DataPlane) initializeDataPlane() error {
+	klog.Infof("Initializing dataplane for windows")
+
+	// Get Network ID
+	network, err := hcn.GetNetworkByName(AzureNetworkName)
+	if err != nil {
+		return err
+	}
+
+	dp.networkID = network.Id
+
+	endpoints, err := hcn.ListEndpointsOfNetwork(dp.networkID)
+	if err != nil {
+		return err
+	}
+
+	for _, endpoint := range endpoints {
+		klog.Infof("Endpoints info %+v", endpoint.Policies)
+		ep := &NPMEndpoint{
+			Name:            endpoint.Name,
+			ID:              endpoint.Id,
+			NetPolReference: make(map[string]struct{}),
+		}
+
+		dp.endpointCache[ep.Name] = ep
+	}
+
+	return nil
+}
+
+// updatePod has two responsibilities in windows
+// 1. Will call into dataplane and updates endpoint references of this pod.
+// 2. Will check for existing applicable network policies and applies it on endpoint
+func (dp *DataPlane) updatePod(pod *npm.NpmPod) error {
+	return nil
+}
+
+func (dp *DataPlane) resetDataPlane() error {
+	return nil
+}

npm/pkg/dataplane/ipsets/ipset.go

@@ -0,0 +1,209 @@
+package ipsets
+
+import (
+	"errors"
+
+	"github.com/Azure/azure-container-networking/npm/util"
+)
+
+type IPSet struct {
+	Name       string
+	HashedName string
+	// SetProperties embedding set properties
+	SetProperties
+	// IpPodKey is used for setMaps to store Ips and ports as keys
+	// and podKey as value
+	IPPodKey map[string]string
+	// This is used for listMaps to store child IP Sets
+	MemberIPSets map[string]*IPSet
+	// Using a map to emulate set and value as struct{} for
+	// minimal memory consumption
+	// SelectorReference holds networkpolicy names where this IPSet
+	// is being used in PodSelector and NameSpace
+	SelectorReference map[string]struct{}
+	// NetPolReference holds networkpolicy names where this IPSet
+	// is being referred as part of rules
+	NetPolReference map[string]struct{}
+	// IpsetReferCount keeps count of 2nd level Nested IPSets
+	// with member as this IPSet
+	IpsetReferCount int
+}
+
+type SetProperties struct {
+	// Stores type of ip grouping
+	Type SetType
+	// Stores kind of ipset in dataplane
+	Kind SetKind
+}
+
+type SetType int8
+
+const (
+	// Unknown SetType
+	Unknown SetType = 0
+	// NameSpace IPSet is created to hold
+	// ips of pods in a given NameSapce
+	NameSpace SetType = 1
+	// KeyLabelOfNameSpace IPSet is a list kind ipset
+	// with members as ipsets of namespace with this Label Key
+	KeyLabelOfNameSpace SetType = 2
+	// KeyValueLabelOfNameSpace IPSet is a list kind ipset
+	// with members as ipsets of namespace with this Label
+	KeyValueLabelOfNameSpace SetType = 3
+	// KeyLabelOfPod IPSet contains IPs of Pods with this Label Key
+	KeyLabelOfPod SetType = 4
+	// KeyValueLabelOfPod IPSet contains IPs of Pods with this Label
+	KeyValueLabelOfPod SetType = 5
+	// NamedPorts IPSets contains a given namedport
+	NamedPorts SetType = 6
+	// NestedLabelOfPod is derived for multivalue matchexpressions
+	NestedLabelOfPod SetType = 7
+	// CIDRBlocks holds CIDR blocks
+	CIDRBlocks SetType = 8
+)
+
+var (
+	setTypeName = map[SetType]string{
+		Unknown:                  "Unknown",
+		NameSpace:                "NameSpace",
+		KeyLabelOfNameSpace:      "KeyLabelOfNameSpace",
+		KeyValueLabelOfNameSpace: "KeyValueLabelOfNameSpace",
+		KeyLabelOfPod:            "KeyLabelOfPod",
+		KeyValueLabelOfPod:       "KeyValueLabelOfPod",
+		NamedPorts:               "NamedPorts",
+		NestedLabelOfPod:         "NestedLabelOfPod",
+		CIDRBlocks:               "CIDRBlocks",
+	}
+	// ErrIPSetInvalidKind is returned when IPSet kind is invalid
+	ErrIPSetInvalidKind = errors.New("Invalid IPSet Kind")
+)
+
+func (x SetType) String() string {
+	return setTypeName[x]
+}
+
+type SetKind string
+
+const (
+	// ListSet is of kind list with members as other IPSets
+	ListSet SetKind = "list"
+	// HashSet is of kind hashset with members as IPs and/or port
+	HashSet SetKind = "set"
+)
+
+func NewIPSet(name string, setType SetType) *IPSet {
+	set := &IPSet{
+		Name:       name,
+		HashedName: util.GetHashedName(name),
+		SetProperties: SetProperties{
+			Type: setType,
+			Kind: getSetKind(setType),
+		},
+		// Map with Key as Network Policy name to to emulate set
+		// and value as struct{} for minimal memory consumption
+		SelectorReference: make(map[string]struct{}),
+		// Map with Key as Network Policy name to to emulate set
+		// and value as struct{} for minimal memory consumption
+		NetPolReference: make(map[string]struct{}),
+		IpsetReferCount: 0,
+	}
+	if set.Kind == HashSet {
+		set.IPPodKey = make(map[string]string)
+	} else {
+		set.MemberIPSets = make(map[string]*IPSet)
+	}
+	return set
+}
+
+func (set *IPSet) GetSetContents() ([]string, error) {
+	switch set.Kind {
+	case HashSet:
+		i := 0
+		contents := make([]string, len(set.IPPodKey))
+		for podIP := range set.IPPodKey {
+			contents[i] = podIP
+			i++
+		}
+		return contents, nil
+	case ListSet:
+		i := 0
+		contents := make([]string, len(set.MemberIPSets))
+		for _, memberSet := range set.MemberIPSets {
+			contents[i] = memberSet.HashedName
+			i++
+		}
+		return contents, nil
+	default:
+		return []string{}, ErrIPSetInvalidKind
+	}
+}
+
+func getSetKind(setType SetType) SetKind {
+	switch setType {
+	case CIDRBlocks:
+		return HashSet
+	case NameSpace:
+		return HashSet
+	case NamedPorts:
+		return HashSet
+	case KeyLabelOfPod:
+		return HashSet
+	case KeyValueLabelOfPod:
+		return HashSet
+	case KeyLabelOfNameSpace:
+		return ListSet
+	case KeyValueLabelOfNameSpace:
+		return ListSet
+	case NestedLabelOfPod:
+		return ListSet
+	case Unknown: // adding this to appease golint
+		return "unknown"
+	default:
+		return "unknown"
+	}
+}
+
+func (set *IPSet) AddMemberIPSet(memberIPSet *IPSet) {
+	set.MemberIPSets[memberIPSet.Name] = memberIPSet
+}
+
+func (set *IPSet) IncIpsetReferCount() {
+	set.IpsetReferCount++
+}
+
+func (set *IPSet) DecIpsetReferCount() {
+	if set.IpsetReferCount == 0 {
+		return
+	}
+	set.IpsetReferCount--
+}
+
+func (set *IPSet) AddSelectorReference(netPolName string) {
+	set.SelectorReference[netPolName] = struct{}{}
+}
+
+func (set *IPSet) DeleteSelectorReference(netPolName string) {
+	delete(set.SelectorReference, netPolName)
+}
+
+func (set *IPSet) AddNetPolReference(netPolName string) {
+	set.NetPolReference[netPolName] = struct{}{}
+}
+
+func (set *IPSet) DeleteNetPolReference(netPolName string) {
+	delete(set.NetPolReference, netPolName)
+}
+
+func (set *IPSet) CanBeDeleted() bool {
+	return len(set.SelectorReference) == 0 &&
+		len(set.NetPolReference) == 0 &&
+		set.IpsetReferCount == 0 &&
+		len(set.MemberIPSets) == 0 &&
+		len(set.IPPodKey) == 0
+}
+
+// UsedByNetPol check if an IPSet is referred in network policies.
+func (set *IPSet) UsedByNetPol() bool {
+	return len(set.SelectorReference) > 0 &&
+		len(set.NetPolReference) > 0
+}

npm/pkg/dataplane/ipsets/ipsetmanager.go

@@ -0,0 +1,295 @@
+package ipsets
+
+import (
+	"fmt"
+	"net"
+	"sync"
+
+	"github.com/Azure/azure-container-networking/log"
+	"github.com/Azure/azure-container-networking/npm/metrics"
+	npmerrors "github.com/Azure/azure-container-networking/npm/util/errors"
+)
+
+type IPSetManager struct {
+	setMap map[string]*IPSet
+	// Map with Key as IPSet name to to emulate set
+	// and value as struct{} for minimal memory consumption
+	dirtyCaches map[string]struct{}
+	sync.Mutex
+}
+
+func (iMgr *IPSetManager) exists(name string) bool {
+	_, ok := iMgr.setMap[name]
+	return ok
+}
+
+func NewIPSetManager() IPSetManager {
+	return IPSetManager{
+		setMap:      make(map[string]*IPSet),
+		dirtyCaches: make(map[string]struct{}),
+	}
+}
+
+func (iMgr *IPSetManager) updateDirtyCache(setName string) {
+	set, exists := iMgr.setMap[setName] // check if the Set exists
+	if !exists {
+		return
+	}
+
+	// If set is not referenced in netpol then ignore the update
+	if len(set.NetPolReference) == 0 && len(set.SelectorReference) == 0 {
+		return
+	}
+
+	iMgr.dirtyCaches[set.Name] = struct{}{}
+	if set.Kind == ListSet {
+		// TODO check if we will need to add all the member ipsets
+		// also to the dirty cache list
+		for _, member := range set.MemberIPSets {
+			iMgr.dirtyCaches[member.Name] = struct{}{}
+		}
+	}
+}
+
+func (iMgr *IPSetManager) clearDirtyCache() {
+	iMgr.dirtyCaches = make(map[string]struct{})
+}
+
+func (iMgr *IPSetManager) CreateIPSet(set *IPSet) error {
+	iMgr.Lock()
+	defer iMgr.Unlock()
+	return iMgr.createIPSet(set)
+}
+
+func (iMgr *IPSetManager) createIPSet(set *IPSet) error {
+	// Check if the Set already exists
+	if iMgr.exists(set.Name) {
+		// ipset already exists
+		// we should calculate a diff if the members are different
+		return nil
+	}
+
+	// append the cache if dataplane specific function
+	// return nil as error
+	iMgr.setMap[set.Name] = set
+	metrics.IncNumIPSets()
+	return nil
+}
+
+func (iMgr *IPSetManager) AddToSet(addToSets []*IPSet, ip, podKey string) error {
+	// check if the IP is IPV4 family
+	if net.ParseIP(ip).To4() == nil {
+		return npmerrors.Errorf(npmerrors.AppendIPSet, false, "IPV6 not supported")
+	}
+	iMgr.Lock()
+	defer iMgr.Unlock()
+
+	for _, updatedSet := range addToSets {
+		set, exists := iMgr.setMap[updatedSet.Name] // check if the Set exists
+		if !exists {
+			err := iMgr.createIPSet(updatedSet)
+			if err != nil {
+				return err
+			}
+			set = iMgr.setMap[updatedSet.Name]
+		}
+
+		if set.Kind != HashSet {
+			return npmerrors.Errorf(npmerrors.AppendIPSet, false, fmt.Sprintf("ipset %s is not a hash set", set.Name))
+		}
+		cachedPodKey, ok := set.IPPodKey[ip]
+		if ok {
+			if cachedPodKey != podKey {
+				log.Logf("AddToSet: PodOwner has changed for Ip: %s, setName:%s, Old podKey: %s, new podKey: %s. Replace context with new PodOwner.",
+					ip, set.Name, cachedPodKey, podKey)
+
+				set.IPPodKey[ip] = podKey
+			}
+			return nil
+		}
+
+		// update the IP ownership with podkey
+		set.IPPodKey[ip] = podKey
+		iMgr.updateDirtyCache(set.Name)
+
+		// Update metrics of the IpSet
+		metrics.AddEntryToIPSet(set.Name)
+	}
+
+	return nil
+}
+
+func (iMgr *IPSetManager) RemoveFromSet(removeFromSets []string, ip, podKey string) error {
+	iMgr.Lock()
+	defer iMgr.Unlock()
+	for _, setName := range removeFromSets {
+		set, exists := iMgr.setMap[setName] // check if the Set exists
+		if !exists {
+			return npmerrors.Errorf(npmerrors.DeleteIPSet, false, fmt.Sprintf("ipset %s does not exist", setName))
+		}
+
+		if set.Kind != HashSet {
+			return npmerrors.Errorf(npmerrors.DeleteIPSet, false, fmt.Sprintf("ipset %s is not a hash set", setName))
+		}
+
+		// in case the IP belongs to a new Pod, then ignore this Delete call as this might be stale
+		cachedPodKey := set.IPPodKey[ip]
+		if cachedPodKey != podKey {
+			log.Logf("DeleteFromSet: PodOwner has changed for Ip: %s, setName:%s, Old podKey: %s, new podKey: %s. Ignore the delete as this is stale update",
+				ip, setName, cachedPodKey, podKey)
+
+			return nil
+		}
+
+		// update the IP ownership with podkey
+		delete(set.IPPodKey, ip)
+		iMgr.updateDirtyCache(set.Name)
+
+		// Update metrics of the IpSet
+		metrics.RemoveEntryFromIPSet(setName)
+	}
+
+	return nil
+}
+
+func (iMgr *IPSetManager) AddToList(listName string, setNames []string) error {
+	iMgr.Lock()
+	defer iMgr.Unlock()
+
+	for _, setName := range setNames {
+		if listName == setName {
+			return npmerrors.Errorf(npmerrors.AppendIPSet, false, fmt.Sprintf("list %s cannot be added to itself", listName))
+		}
+		set, exists := iMgr.setMap[setName] // check if the Set exists
+		if !exists {
+			return npmerrors.Errorf(npmerrors.AppendIPSet, false, fmt.Sprintf("member ipset %s does not exist", setName))
+		}
+
+		// Nested IPSets are only supported for windows
+		// Check if we want to actually use that support
+		if set.Kind != HashSet {
+			return npmerrors.Errorf(npmerrors.DeleteIPSet, false, fmt.Sprintf("member ipset %s is not a Set type and nestetd ipsets are not supported", setName))
+		}
+
+		list, exists := iMgr.setMap[listName] // check if the Set exists
+		if !exists {
+			return npmerrors.Errorf(npmerrors.AppendIPSet, false, fmt.Sprintf("ipset %s does not exist", listName))
+		}
+
+		if list.Kind != ListSet {
+			return npmerrors.Errorf(npmerrors.AppendIPSet, false, fmt.Sprintf("ipset %s is not a list set", listName))
+		}
+
+		// check if Set is a member of List
+		listSet, exists := list.MemberIPSets[setName]
+		if exists {
+			if listSet == set {
+				// Set is already a member of List
+				return nil
+			}
+			// Update the ipset in list
+			list.MemberIPSets[setName] = set
+			return nil
+		}
+
+		// update the Ipset member list of list
+		list.AddMemberIPSet(set)
+		set.IncIpsetReferCount()
+		// Update metrics of the IpSet
+		metrics.AddEntryToIPSet(listName)
+	}
+
+	iMgr.updateDirtyCache(listName)
+
+	return nil
+}
+
+func (iMgr *IPSetManager) RemoveFromList(listName string, setNames []string) error {
+	iMgr.Lock()
+	defer iMgr.Unlock()
+	for _, setName := range setNames {
+		set, exists := iMgr.setMap[setName] // check if the Set exists
+		if !exists {
+			return npmerrors.Errorf(npmerrors.DeleteIPSet, false, fmt.Sprintf("ipset %s does not exist", setName))
+		}
+
+		if set.Kind != HashSet {
+			return npmerrors.Errorf(npmerrors.DeleteIPSet, false, fmt.Sprintf("ipset %s is not a hash set", setName))
+		}
+
+		// Nested IPSets are only supported for windows
+		// Check if we want to actually use that support
+		if set.Kind != HashSet {
+			return npmerrors.Errorf(npmerrors.DeleteIPSet, false, fmt.Sprintf("member ipset %s is not a Set type and nestetd ipsets are not supported", setName))
+		}
+
+		list, exists := iMgr.setMap[listName] // check if the Set exists
+		if !exists {
+			return npmerrors.Errorf(npmerrors.DeleteIPSet, false, fmt.Sprintf("ipset %s does not exist", listName))
+		}
+
+		if list.Kind != ListSet {
+			return npmerrors.Errorf(npmerrors.DeleteIPSet, false, fmt.Sprintf("ipset %s is not a list set", listName))
+		}
+
+		// check if Set is a member of List
+		_, exists = list.MemberIPSets[setName]
+		if !exists {
+			return nil
+		}
+
+		// delete IPSet from the list
+		delete(list.MemberIPSets, setName)
+		set.DecIpsetReferCount()
+		// Update metrics of the IpSet
+		metrics.RemoveEntryFromIPSet(listName)
+	}
+	iMgr.updateDirtyCache(listName)
+
+	return nil
+}
+
+func (iMgr *IPSetManager) DeleteList(name string) error {
+	iMgr.Lock()
+	defer iMgr.Unlock()
+	set, exists := iMgr.setMap[name] // check if the Set exists
+	if !exists {
+		return npmerrors.Errorf(npmerrors.AppendIPSet, false, fmt.Sprintf("member ipset %s does not exist", set.Name))
+	}
+
+	if !set.CanBeDeleted() {
+		return npmerrors.Errorf(npmerrors.DeleteIPSet, false, fmt.Sprintf("ipset %s cannot be deleted", set.Name))
+	}
+
+	delete(iMgr.setMap, name)
+	return nil
+}
+
+func (iMgr *IPSetManager) DeleteSet(name string) error {
+	iMgr.Lock()
+	defer iMgr.Unlock()
+	set, exists := iMgr.setMap[name] // check if the Set exists
+	if !exists {
+		return npmerrors.Errorf(npmerrors.AppendIPSet, false, fmt.Sprintf("member ipset %s does not exist", set.Name))
+	}
+
+	if !set.CanBeDeleted() {
+		return npmerrors.Errorf(npmerrors.DeleteIPSet, false, fmt.Sprintf("ipset %s cannot be deleted", set.Name))
+	}
+	delete(iMgr.setMap, name)
+	return nil
+}
+
+func (iMgr *IPSetManager) ApplyIPSets(networkID string) error {
+	iMgr.Lock()
+	defer iMgr.Unlock()
+
+	// Call the appropriate apply ipsets
+	err := iMgr.applyIPSets(networkID)
+	if err != nil {
+		return err
+	}
+
+	iMgr.clearDirtyCache()
+	return nil
+}

npm/pkg/dataplane/ipsets/ipsetmanager_linux.go

@@ -0,0 +1,21 @@
+package ipsets
+
+import (
+	"fmt"
+
+	"github.com/Azure/azure-container-networking/npm/util/errors"
+	"k8s.io/klog"
+)
+
+func (iMgr *IPSetManager) applyIPSets(networkID string) error {
+	for setName := range iMgr.dirtyCaches {
+		set, exists := iMgr.setMap[setName] // check if the Set exists
+		if !exists {
+			return errors.Errorf(errors.AppendIPSet, false, fmt.Sprintf("member ipset %s does not exist", setName))
+		}
+
+		klog.Infof(set.Name)
+
+	}
+	return nil
+}

npm/pkg/dataplane/ipsets/ipsetmanager_test.go

@@ -0,0 +1,151 @@
+package ipsets
+
+import (
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/Azure/azure-container-networking/npm/metrics"
+)
+
+func TestCreateIPSet(t *testing.T) {
+	iMgr := NewIPSetManager()
+	set := NewIPSet("Test", NameSpace)
+
+	err := iMgr.CreateIPSet(set)
+	if err != nil {
+		t.Errorf("CreateIPSet() returned error %s", err.Error())
+	}
+}
+
+func TestAddToSet(t *testing.T) {
+	iMgr := NewIPSetManager()
+	set := NewIPSet("Test", NameSpace)
+
+	fmt.Println(set.Name)
+	err := iMgr.AddToSet([]*IPSet{set}, "10.0.0.0", "test")
+	if err != nil {
+		t.Errorf("AddToSet() returned error %s", err.Error())
+	}
+}
+
+func TestRemoveFromSet(t *testing.T) {
+	iMgr := NewIPSetManager()
+	set := NewIPSet("Test", NameSpace)
+
+	err := iMgr.AddToSet([]*IPSet{set}, "10.0.0.0", "test")
+	if err != nil {
+		t.Errorf("RemoveFromSet() returned error %s", err.Error())
+	}
+	err = iMgr.RemoveFromSet([]string{"Test"}, "10.0.0.0", "test")
+	if err != nil {
+		t.Errorf("RemoveFromSet() returned error %s", err.Error())
+	}
+}
+
+func TestRemoveFromSetMissing(t *testing.T) {
+	iMgr := NewIPSetManager()
+	err := iMgr.RemoveFromSet([]string{"Test"}, "10.0.0.0", "test")
+	if err == nil {
+		t.Errorf("RemoveFromSet() did not return error")
+	}
+}
+
+func TestAddToListMissing(t *testing.T) {
+	iMgr := NewIPSetManager()
+	err := iMgr.AddToList("test", []string{"newtest"})
+	if err == nil {
+		t.Errorf("AddToList() did not return error")
+	}
+}
+
+func TestAddToList(t *testing.T) {
+	iMgr := NewIPSetManager()
+	set := NewIPSet("newtest", NameSpace)
+	err := iMgr.CreateIPSet(set)
+	if err != nil {
+		t.Errorf("CreateIPSet() returned error %s", err.Error())
+	}
+
+	list := NewIPSet("test", KeyLabelOfNameSpace)
+	err = iMgr.CreateIPSet(list)
+	if err != nil {
+		t.Errorf("CreateIPSet() returned error %s", err.Error())
+	}
+
+	err = iMgr.AddToList("test", []string{"newtest"})
+	if err != nil {
+		t.Errorf("AddToList() returned error %s", err.Error())
+	}
+}
+
+func TestRemoveFromList(t *testing.T) {
+	iMgr := NewIPSetManager()
+	set := NewIPSet("newtest", NameSpace)
+	err := iMgr.CreateIPSet(set)
+	if err != nil {
+		t.Errorf("CreateIPSet() returned error %s", err.Error())
+	}
+
+	list := NewIPSet("test", KeyLabelOfNameSpace)
+	err = iMgr.CreateIPSet(list)
+	if err != nil {
+		t.Errorf("CreateIPSet() returned error %s", err.Error())
+	}
+
+	err = iMgr.AddToList("test", []string{"newtest"})
+	if err != nil {
+		t.Errorf("AddToList() returned error %s", err.Error())
+	}
+
+	err = iMgr.RemoveFromList("test", []string{"newtest"})
+	if err != nil {
+		t.Errorf("RemoveFromList() returned error %s", err.Error())
+	}
+}
+
+func TestRemoveFromListMissing(t *testing.T) {
+	iMgr := NewIPSetManager()
+	err := iMgr.RemoveFromList("test", []string{"newtest"})
+	if err == nil {
+		t.Errorf("RemoveFromList() did not return error")
+	}
+}
+
+func TestDeleteList(t *testing.T) {
+	iMgr := NewIPSetManager()
+	set := NewIPSet("Test", KeyValueLabelOfNameSpace)
+
+	err := iMgr.CreateIPSet(set)
+	if err != nil {
+		t.Errorf("CreateIPSet() returned error %s", err.Error())
+	}
+
+	err = iMgr.DeleteList(set.Name)
+	if err != nil {
+		t.Errorf("DeleteList() returned error %s", err.Error())
+	}
+}
+
+func TestDeleteSet(t *testing.T) {
+	iMgr := NewIPSetManager()
+	set := NewIPSet("Test", NameSpace)
+
+	err := iMgr.CreateIPSet(set)
+	if err != nil {
+		t.Errorf("CreateIPSet() returned error %s", err.Error())
+	}
+
+	err = iMgr.DeleteSet(set.Name)
+	if err != nil {
+		t.Errorf("DeleteSet() returned error %s", err.Error())
+	}
+}
+
+func TestMain(m *testing.M) {
+	metrics.InitializeAll()
+
+	exitCode := m.Run()
+
+	os.Exit(exitCode)
+}

npm/pkg/dataplane/ipsets/ipsetmanager_windows.go

@@ -0,0 +1,181 @@
+package ipsets
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/Azure/azure-container-networking/npm/util"
+	"github.com/Azure/azure-container-networking/npm/util/errors"
+	"github.com/Microsoft/hcsshim/hcn"
+	"k8s.io/klog"
+)
+
+// SetPolicyTypes associated with SetPolicy. Value is IPSET.
+type SetPolicyType string
+
+const (
+	SetPolicyTypeIpSet       SetPolicyType = "IPSET"
+	SetPolicyTypeNestedIpSet SetPolicyType = "NESTEDIPSET"
+)
+
+// SetPolicySetting creates IPSets on network
+type SetPolicySetting struct {
+	Id     string
+	Name   string
+	Type   SetPolicyType
+	Values string
+}
+
+func (iMgr *IPSetManager) applyIPSets(networkID string) error {
+	network, err := hcn.GetNetworkByID(networkID)
+	if err != nil {
+		return err
+	}
+
+	setPolNames, err := getAllSetPolicyNames(network.Policies)
+	if err != nil {
+		return err
+	}
+
+	setPolSettings, err := iMgr.calculateNewSetPolicies(setPolNames)
+	if err != nil {
+		return err
+	}
+
+	policyNetworkRequest := hcn.PolicyNetworkRequest{
+		Policies: []hcn.NetworkPolicy{},
+	}
+
+	for _, policy := range network.Policies {
+		// TODO (vamsi) use NetPolicyType constant setpolicy for below check
+		// after updating HCSShim
+		if policy.Type != "SetPolicy" {
+			policyNetworkRequest.Policies = append(policyNetworkRequest.Policies, policy)
+		}
+	}
+
+	for setPol := range setPolSettings {
+		rawSettings, err := json.Marshal(setPolSettings[setPol])
+		if err != nil {
+			return err
+		}
+		policyNetworkRequest.Policies = append(
+			policyNetworkRequest.Policies,
+			hcn.NetworkPolicy{
+				Type:     "SetPolicy",
+				Settings: rawSettings,
+			},
+		)
+	}
+
+	err = network.AddPolicy(policyNetworkRequest)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (iMgr *IPSetManager) calculateNewSetPolicies(existingSets []string) (map[string]SetPolicySetting, error) {
+	// some of this below logic can be abstracted a step above
+	dirtySets := iMgr.dirtyCaches
+
+	for _, setName := range existingSets {
+		dirtySets[setName] = struct{}{}
+	}
+
+	setsToUpdate := make(map[string]SetPolicySetting)
+	for setName := range dirtySets {
+		set, exists := iMgr.setMap[setName] // check if the Set exists
+		if !exists {
+			return nil, errors.Errorf(errors.AppendIPSet, false, fmt.Sprintf("member ipset %s does not exist", setName))
+		}
+		if !set.UsedByNetPol() {
+			continue
+		}
+
+		setPol, err := convertToSetPolicy(set)
+		if err != nil {
+			return nil, err
+		}
+		setsToUpdate[setName] = setPol
+		if set.Kind == ListSet {
+			for _, memberSet := range set.MemberIPSets {
+				// TODO check whats the name here, hashed or normal
+				if _, ok := setsToUpdate[memberSet.Name]; ok {
+					continue
+				}
+				setPol, err = convertToSetPolicy(memberSet)
+				if err != nil {
+					return nil, err
+				}
+				setsToUpdate[memberSet.Name] = setPol
+			}
+		}
+	}
+
+	return setsToUpdate, nil
+}
+
+func isValidIPSet(set *IPSet) error {
+	if set.Name == "" {
+		return fmt.Errorf("IPSet " + set.Name + " is missing Name")
+	}
+
+	if set.Type == Unknown {
+		return fmt.Errorf("IPSet " + set.Type.String() + " is missing Type")
+	}
+
+	if set.HashedName == "" {
+		return fmt.Errorf("IPSet " + set.HashedName + " is missing HashedName")
+	}
+
+	return nil
+}
+
+func getSetPolicyType(set *IPSet) SetPolicyType {
+	switch set.Kind {
+	case ListSet:
+		return SetPolicyTypeNestedIpSet
+	case HashSet:
+		return SetPolicyTypeIpSet
+	default:
+		return "Unknown"
+	}
+}
+
+func convertToSetPolicy(set *IPSet) (SetPolicySetting, error) {
+	err := isValidIPSet(set)
+	if err != nil {
+		return SetPolicySetting{}, err
+	}
+
+	setContents, err := set.GetSetContents()
+	if err != nil {
+		return SetPolicySetting{}, err
+	}
+
+	setPolicy := SetPolicySetting{
+		Id:     set.HashedName,
+		Name:   set.Name,
+		Type:   getSetPolicyType(set),
+		Values: util.SliceToString(setContents),
+	}
+	return setPolicy, nil
+}
+
+func getAllSetPolicyNames(networkPolicies []hcn.NetworkPolicy) ([]string, error) {
+	setPols := []string{}
+	for _, netpol := range networkPolicies {
+		if netpol.Type == "SetPolicy" {
+			var set SetPolicySetting
+			err := json.Unmarshal(netpol.Settings, &set)
+			if err != nil {
+				klog.Error(err.Error())
+				continue
+			}
+			setPols = append(setPols, set.Name)
+		}
+	}
+	return setPols, nil
+}

npm/pkg/dataplane/policies/policy.go

@@ -0,0 +1,92 @@
+package policies
+
+import (
+	"github.com/Azure/azure-container-networking/npm/pkg/dataplane/ipsets"
+	networkingv1 "k8s.io/api/networking/v1"
+)
+
+type NPMNetworkPolicy struct {
+	Name string
+	// PodSelectorIPSets holds all the IPSets generated from Pod Selector
+	PodSelectorIPSets []*ipsets.IPSet
+	// RuleIPSets holds all IPSets generated from policy's rules
+	// and not from pod selector IPSets
+	RuleIPSets []*ipsets.IPSet
+	ACLs       []*ACLPolicy
+	// Making this a podKey instead should be
+	// use NPMPod obj
+	Pods  []string
+	RawNP *networkingv1.NetworkPolicy
+}
+
+// ACLPolicy equivalent to a single iptable rule in linux
+// or a single HNS rule in windows
+type ACLPolicy struct {
+	// PolicyID is the rules name with a given network policy
+	PolicyID string
+	// Comment is the string attached to rule to identity its representation
+	Comment string
+	// SrcList source IPSets condition setinfos
+	SrcList []SetInfo
+	// DstList destination IPSets condition setinfos
+	DstList []SetInfo
+	// Target defines a target in iptables for linux. i,e, Mark, Accept, Drop
+	// in windows, this is either ALLOW or DENY
+	Target Verdict
+	// Direction defines the flow of traffic
+	Direction Direction
+	// SrcPorts holds the source port information
+	SrcPorts []Ports
+	// DstPorts holds the destination port information
+	DstPorts []Ports
+	// Protocol is the value of traffic protocol
+	Protocol Protocol
+}
+
+// SetInfo helps capture additional details in a matchSet
+// example match set in linux:
+//             ! azure-npm-123 src,src
+// "!" this indicates a negative match of an IPset for src,src
+// Included flag captures the negative or positive match
+// MatchType captures match flags
+type SetInfo struct {
+	IPSet     *ipsets.IPSet
+	Included  bool
+	MatchType string // match type can be “src”, “src,dst” or “dst,dst” etc
+}
+
+type Ports struct {
+	Port    int64
+	EndPort int64
+}
+
+type Verdict string
+
+type Direction string
+
+type Protocol string
+
+const (
+	// Ingress when packet is entering a container
+	Ingress Direction = "IN"
+	// Egress when packet is leaving a container
+	Egress Direction = "OUT"
+	// Both applies to both directions
+	Both Direction = "BOTH"
+
+	// Allowed is accept in linux
+	Allowed Verdict = "ALLOW"
+	// Dropped is denying a flow
+	Dropped Verdict = "DROP"
+
+	// TCP Protocol
+	TCP Protocol = "tcp"
+	// UDP Protocol
+	UDP Protocol = "udp"
+	// SCTP Protocol
+	SCTP Protocol = "sctp"
+	// ICMP Protocol
+	ICMP Protocol = "icmp"
+	// AnyProtocol can be used for all other protocols
+	AnyProtocol Protocol = "any"
+)

npm/pkg/dataplane/policies/policy_windows.go

@@ -0,0 +1,47 @@
+package policies
+
+import (
+	"fmt"
+
+	"github.com/Microsoft/hcsshim/hcn"
+)
+
+var protocolNumMap = map[Protocol]string{
+	TCP:  "6",
+	UDP:  "17",
+	ICMP: "1",
+	SCTP: "132",
+	// HNS thinks 256 as ANY protocol
+	AnyProtocol: "256",
+}
+
+func convertToAclSettings(acl ACLPolicy) (hcn.AclPolicySetting, error) {
+	policySettings := hcn.AclPolicySetting{}
+	for _, setInfo := range acl.SrcList {
+		if !setInfo.Included {
+			return policySettings, fmt.Errorf("Windows Dataplane does not support negative matches. ACL: %+v", acl)
+		}
+	}
+
+	return policySettings, nil
+}
+
+func getHCNDirection(direction Direction) hcn.DirectionType {
+	switch direction {
+	case Ingress:
+		return hcn.DirectionTypeIn
+	case Egress:
+		return hcn.DirectionTypeOut
+	}
+	return ""
+}
+
+func getHCNAction(verdict Verdict) hcn.ActionType {
+	switch verdict {
+	case Allowed:
+		return hcn.ActionTypeAllow
+	case Dropped:
+		return hcn.ActionTypeBlock
+	}
+	return ""
+}

npm/pkg/dataplane/policies/policymanager.go

@@ -0,0 +1,59 @@
+package policies
+
+type PolicyMap struct {
+	cache map[string]*NPMNetworkPolicy
+}
+
+type PolicyManager struct {
+	policyMap *PolicyMap
+}
+
+func NewPolicyManager() PolicyManager {
+	return PolicyManager{
+		policyMap: &PolicyMap{
+			cache: make(map[string]*NPMNetworkPolicy),
+		},
+	}
+}
+
+func (pMgr *PolicyManager) GetPolicy(name string) (*NPMNetworkPolicy, error) {
+	if policy, ok := pMgr.policyMap.cache[name]; ok {
+		return policy, nil
+	}
+
+	return nil, nil
+}
+
+func (pMgr *PolicyManager) AddPolicy(policy *NPMNetworkPolicy) error {
+	// Call actual dataplane function to apply changes
+	err := pMgr.addPolicy(policy)
+	if err != nil {
+		return err
+	}
+
+	pMgr.policyMap.cache[policy.Name] = policy
+	return nil
+}
+
+func (pMgr *PolicyManager) RemovePolicy(name string) error {
+	// Call actual dataplane function to apply changes
+	err := pMgr.removePolicy(name)
+	if err != nil {
+		return err
+	}
+
+	delete(pMgr.policyMap.cache, name)
+
+	return nil
+}
+
+func (pMgr *PolicyManager) UpdatePolicy(policy *NPMNetworkPolicy) error {
+	// check and update
+	// Call actual dataplane function to apply changes
+	err := pMgr.updatePolicy(policy)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

npm/pkg/dataplane/policies/policymanager_linux.go

@@ -0,0 +1,13 @@
+package policies
+
+func (pMgr *PolicyManager) addPolicy(policy *NPMNetworkPolicy) error {
+	return nil
+}
+
+func (pMgr *PolicyManager) removePolicy(name string) error {
+	return nil
+}
+
+func (pMgr *PolicyManager) updatePolicy(policy *NPMNetworkPolicy) error {
+	return nil
+}

npm/pkg/dataplane/policies/policymanager_test.go

@@ -0,0 +1,39 @@
+package policies
+
+import "testing"
+
+func TestAddPolicy(t *testing.T) {
+	pMgr := NewPolicyManager()
+
+	netpol := NPMNetworkPolicy{}
+
+	err := pMgr.AddPolicy(&netpol)
+	if err != nil {
+		t.Errorf("AddPolicy() returned error %s", err.Error())
+	}
+}
+
+func TestRemovePolicy(t *testing.T) {
+	pMgr := NewPolicyManager()
+
+	err := pMgr.RemovePolicy("test")
+	if err != nil {
+		t.Errorf("RemovePolicy() returned error %s", err.Error())
+	}
+}
+
+func TestUpdatePolicy(t *testing.T) {
+	pMgr := NewPolicyManager()
+
+	netpol := NPMNetworkPolicy{}
+
+	err := pMgr.AddPolicy(&netpol)
+	if err != nil {
+		t.Errorf("UpdatePolicy() returned error %s", err.Error())
+	}
+
+	err = pMgr.UpdatePolicy(&netpol)
+	if err != nil {
+		t.Errorf("UpdatePolicy() returned error %s", err.Error())
+	}
+}

npm/pkg/dataplane/policies/policymanager_windows.go

@@ -0,0 +1,13 @@
+package policies
+
+func (pMgr *PolicyManager) addPolicy(policy *NPMNetworkPolicy) error {
+	return nil
+}
+
+func (pMgr *PolicyManager) removePolicy(name string) error {
+	return nil
+}
+
+func (pMgr *PolicyManager) updatePolicy(policy *NPMNetworkPolicy) error {
+	return nil
+}

npm/util/const.go

@@ -145,6 +145,8 @@ const (
 
 	NamespacePrefix string = "ns-"
 	NegationPrefix  string = "not-"
+
+	SetPolicyDelimiter string = ","
 )
 
 // NPM telemetry constants.

npm/util/util.go

@@ -330,3 +330,7 @@ func CompareSlices(list1, list2 []string) bool {
 	}
 	return true
 }
+
+func SliceToString(list []string) string {
+	return strings.Join(list, SetPolicyDelimiter)
+}