feat: add multi-tenant NetworkContainer controller (#876)

Makefile

@@ -51,6 +51,8 @@ CNSFILES = \
 	$(wildcard cns/networkcontainers/*.go) \
 	$(wildcard cns/requestcontroller/*.go) \
 	$(wildcard cns/requestcontroller/kubecontroller/*.go) \
+	$(wildcard cns/multitenantcontroller/*.go) \
+	$(wildcard cns/multitenantcontroller/multitenantoperator/*.go) \
 	$(wildcard cns/fakes/*.go) \
 	$(COREFILES) \
 	$(CNMFILES)
@@ -181,13 +183,13 @@ azure-npm: $(NPM_BUILD_DIR)/azure-npm$(EXE_EXT) npm-archive
 endif
 
 ifeq ($(GOOS),linux)
-all-binaries: azure-cnm-plugin azure-cni-plugin azure-cns azure-cnms azure-npm 
+all-binaries: azure-cnm-plugin azure-cni-plugin azure-cns azure-cnms azure-npm
 else
 all-binaries: azure-cnm-plugin azure-cni-plugin azure-cns
 endif
 
 ifeq ($(GOOS),linux)
-all-images: azure-npm-image azure-cns-image 
+all-images: azure-npm-image azure-cns-image
 else
 all-images:
 	@echo "Nothing to build. Skip."
@@ -223,7 +225,7 @@ $(CNI_BUILD_DIR)/azure-vnet-telemetry$(EXE_EXT): $(CNIFILES)
 # Build the Azure CLI network plugin.
 $(ACNCLI_BUILD_DIR)/acncli$(EXE_EXT): $(CNIFILES)
 	CGO_ENABLED=0 go build -v -o $(ACNCLI_BUILD_DIR)/acn$(EXE_EXT) -ldflags "-X main.version=$(VERSION)" -gcflags="-dwarflocationlists=true" $(ACNCLI_DIR)/*.go
- 
+
 # Build the Azure CNS Service.
 $(CNS_BUILD_DIR)/azure-cns$(EXE_EXT): $(CNSFILES)
 	go build -v -o $(CNS_BUILD_DIR)/azure-cns$(EXE_EXT) -ldflags "-X main.version=$(VERSION) -X $(cnsaipath)=$(CNS_AI_ID)" -gcflags="-dwarflocationlists=true" $(CNS_DIR)/*.go
@@ -259,7 +261,7 @@ all-containerized:
 
 # Make both linux and windows binaries
 .PHONY: all-binaries-platforms
-all-binaries-platforms: 
+all-binaries-platforms:
 	export GOOS=linux; make all-binaries
 	export GOOS=windows; make all-binaries
 
@@ -268,7 +270,7 @@ all-binaries-platforms:
 tools: acncli
 
 .PHONY: tools-images
-tools-images: 
+tools-images:
 	docker build --no-cache -f ./tools/acncli/Dockerfile --build-arg VERSION=$(VERSION) -t $(AZURE_CNI_IMAGE):$(VERSION) .
 
 # Build the Azure CNM plugin image, installable with "docker plugin install".
@@ -409,7 +411,7 @@ ifeq ($(GOOS),linux)
 	cp $(CNI_BUILD_DIR)/azure-vnet$(EXE_EXT) $(CNI_BUILD_DIR)/azure-vnet-ipam$(EXE_EXT) $(CNI_BUILD_DIR)/azure-vnet-telemetry$(EXE_EXT) $(CNI_SWIFT_BUILD_DIR)
 	chmod 0755 $(CNI_SWIFT_BUILD_DIR)/azure-vnet$(EXE_EXT) $(CNI_SWIFT_BUILD_DIR)/azure-vnet-ipam$(EXE_EXT)
 	cd $(CNI_SWIFT_BUILD_DIR) && $(ARCHIVE_CMD) $(CNI_SWIFT_ARCHIVE_NAME) azure-vnet$(EXE_EXT) azure-vnet-ipam$(EXE_EXT) azure-vnet-telemetry$(EXE_EXT) 10-azure.conflist azure-vnet-telemetry.config
-endif	
+endif
 
 # Create a CNM archive for the target platform.
 .PHONY: cnm-archive
@@ -453,7 +455,7 @@ endif
 .PHONY: release
 release:
 	./scripts/semver-release.sh
-	
+
 
 PRETTYGOTEST := $(shell command -v gotest 2> /dev/null)
 

cns/NetworkContainerContract.go

@@ -69,9 +69,10 @@ const (
 
 // ChannelMode :- CNS channel modes
 const (
-	Direct  = "Direct"
-	Managed = "Managed"
-	CRD     = "CRD"
+	Direct         = "Direct"
+	Managed        = "Managed"
+	CRD            = "CRD"
+	MultiTenantCRD = "MultiTenantCRD"
 )
 
 // CreateNetworkContainerRequest specifies request to create a network container or network isolation boundary.

cns/cnsclient/apiclient.go

@@ -8,5 +8,8 @@ import (
 // APIClient interface to update cns state
 type APIClient interface {
 	ReconcileNCState(nc *cns.CreateNetworkContainerRequest, pods map[string]cns.KubernetesPodInfo, scalar nnc.Scaler, spec nnc.NodeNetworkConfigSpec) error
-	CreateOrUpdateNC(nc cns.CreateNetworkContainerRequest, scalar nnc.Scaler, spec nnc.NodeNetworkConfigSpec) error
+	CreateOrUpdateNC(nc cns.CreateNetworkContainerRequest) error
+	UpdateIPAMPoolMonitor(scalar nnc.Scaler, spec nnc.NodeNetworkConfigSpec) error
+	GetNC(nc cns.GetNetworkContainerRequest) (cns.GetNetworkContainerResponse, error)
+	DeleteNC(nc cns.DeleteNetworkContainerRequest) error
 }

cns/cnsclient/cnsclient_test.go

@@ -73,10 +73,15 @@ func addTestStateToRestServer(t *testing.T, secondaryIps []string) {
 		Version: "-1",
 	}
 
-	returnCode := svc.CreateOrUpdateNetworkContainerInternal(req, fakes.NewFakeScalar(releasePercent, requestPercent, batchSize), fakes.NewFakeNodeNetworkConfigSpec(initPoolSize))
+	returnCode := svc.CreateOrUpdateNetworkContainerInternal(req)
 	if returnCode != 0 {
 		t.Fatalf("Failed to createNetworkContainerRequest, req: %+v, err: %d", req, returnCode)
 	}
+
+	returnCode = svc.UpdateIPAMPoolMonitorInternal(fakes.NewFakeScalar(releasePercent, requestPercent, batchSize), fakes.NewFakeNodeNetworkConfigSpec(initPoolSize))
+	if returnCode != 0 {
+		t.Fatalf("Failed to UpdateIPAMPoolMonitorInternal, err: %d", returnCode)
+	}
 }
 
 func getIPNetFromResponse(resp *cns.IPConfigResponse) (net.IPNet, error) {

cns/cnsclient/httpapi/client.go

@@ -14,8 +14,8 @@ type Client struct {
 }
 
 // CreateOrUpdateNC updates cns state
-func (client *Client) CreateOrUpdateNC(ncRequest cns.CreateNetworkContainerRequest, scalar nnc.Scaler, spec nnc.NodeNetworkConfigSpec) error {
-	returnCode := client.RestService.CreateOrUpdateNetworkContainerInternal(ncRequest, scalar, spec)
+func (client *Client) CreateOrUpdateNC(ncRequest cns.CreateNetworkContainerRequest) error {
+	returnCode := client.RestService.CreateOrUpdateNetworkContainerInternal(ncRequest)
 
 	if returnCode != 0 {
 		return fmt.Errorf("Failed to Create NC request: %+v, errorCode: %d", ncRequest, returnCode)
@@ -24,6 +24,17 @@ func (client *Client) CreateOrUpdateNC(ncRequest cns.CreateNetworkContainerReque
 	return nil
 }
 
+// UpdateIPAMPoolMonitor updates IPAM pool monitor.
+func (client *Client) UpdateIPAMPoolMonitor(scalar nnc.Scaler, spec nnc.NodeNetworkConfigSpec) error {
+	returnCode := client.RestService.UpdateIPAMPoolMonitorInternal(scalar, spec)
+
+	if returnCode != 0 {
+		return fmt.Errorf("Failed to update IPAM pool monitor scalar: %+v, spec: %+v, errorCode: %d", scalar, spec, returnCode)
+	}
+
+	return nil
+}
+
 // ReconcileNCState initializes cns state
 func (client *Client) ReconcileNCState(ncRequest *cns.CreateNetworkContainerRequest, podInfoByIP map[string]cns.KubernetesPodInfo, scalar nnc.Scaler, spec nnc.NodeNetworkConfigSpec) error {
 	returnCode := client.RestService.ReconcileNCState(ncRequest, podInfoByIP, scalar, spec)
@@ -34,3 +45,24 @@ func (client *Client) ReconcileNCState(ncRequest *cns.CreateNetworkContainerRequ
 
 	return nil
 }
+
+func (client *Client) GetNC(req cns.GetNetworkContainerRequest) (cns.GetNetworkContainerResponse, error) {
+	response, returnCode := client.RestService.GetNetworkContainerInternal(req)
+	if returnCode != 0 {
+		if returnCode == restserver.UnknownContainerID {
+			return response, fmt.Errorf("NotFound")
+		}
+		return response, fmt.Errorf("Failed to get NC, request: %+v, errorCode: %d", req, returnCode)
+	}
+
+	return response, nil
+}
+
+func (client *Client) DeleteNC(req cns.DeleteNetworkContainerRequest) error {
+	returnCode := client.RestService.DeleteNetworkContainerInternal(req)
+	if returnCode != 0 {
+		return fmt.Errorf("Failed to delete NC, request: %+v, errorCode: %d", req, returnCode)
+	}
+
+	return nil
+}

cns/multitenantcontroller/mockclients/README.md

@@ -0,0 +1,8 @@
+# Mock Clients
+
+Run the following command to generate mock clients:
+
+```sh
+mockgen -source=$GOPATH/src/github.com/Azure/azure-container-networking/cns/cnsclient/apiclient.go -package=mockclients APIClient >cnsclient.go
+mockgen -source=$GOPATH/src/sigs.k8s.io/controller-runtime/pkg/client/interfaces.go -package=mockclients Client >kubeclient.go
+```

cns/multitenantcontroller/mockclients/cnsclient.go

@@ -0,0 +1,106 @@
+// Code generated by MockGen. DO NOT EDIT.
+// Source: /go/src/github.com/Azure/azure-container-networking/cns/cnsclient/apiclient.go
+
+// Package mockclients is a generated GoMock package.
+package mockclients
+
+import (
+	cns "github.com/Azure/azure-container-networking/cns"
+	v1alpha "github.com/Azure/azure-container-networking/nodenetworkconfig/api/v1alpha"
+	gomock "github.com/golang/mock/gomock"
+	reflect "reflect"
+)
+
+// MockAPIClient is a mock of APIClient interface
+type MockAPIClient struct {
+	ctrl     *gomock.Controller
+	recorder *MockAPIClientMockRecorder
+}
+
+// MockAPIClientMockRecorder is the mock recorder for MockAPIClient
+type MockAPIClientMockRecorder struct {
+	mock *MockAPIClient
+}
+
+// NewMockAPIClient creates a new mock instance
+func NewMockAPIClient(ctrl *gomock.Controller) *MockAPIClient {
+	mock := &MockAPIClient{ctrl: ctrl}
+	mock.recorder = &MockAPIClientMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use
+func (m *MockAPIClient) EXPECT() *MockAPIClientMockRecorder {
+	return m.recorder
+}
+
+// ReconcileNCState mocks base method
+func (m *MockAPIClient) ReconcileNCState(nc *cns.CreateNetworkContainerRequest, pods map[string]cns.KubernetesPodInfo, scalar v1alpha.Scaler, spec v1alpha.NodeNetworkConfigSpec) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "ReconcileNCState", nc, pods, scalar, spec)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// ReconcileNCState indicates an expected call of ReconcileNCState
+func (mr *MockAPIClientMockRecorder) ReconcileNCState(nc, pods, scalar, spec interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReconcileNCState", reflect.TypeOf((*MockAPIClient)(nil).ReconcileNCState), nc, pods, scalar, spec)
+}
+
+// CreateOrUpdateNC mocks base method
+func (m *MockAPIClient) CreateOrUpdateNC(nc cns.CreateNetworkContainerRequest) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "CreateOrUpdateNC", nc)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// CreateOrUpdateNC indicates an expected call of CreateOrUpdateNC
+func (mr *MockAPIClientMockRecorder) CreateOrUpdateNC(nc interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateOrUpdateNC", reflect.TypeOf((*MockAPIClient)(nil).CreateOrUpdateNC), nc)
+}
+
+// UpdateIPAMPoolMonitor mocks base method
+func (m *MockAPIClient) UpdateIPAMPoolMonitor(scalar v1alpha.Scaler, spec v1alpha.NodeNetworkConfigSpec) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "UpdateIPAMPoolMonitor", scalar, spec)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// UpdateIPAMPoolMonitor indicates an expected call of UpdateIPAMPoolMonitor
+func (mr *MockAPIClientMockRecorder) UpdateIPAMPoolMonitor(scalar, spec interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateIPAMPoolMonitor", reflect.TypeOf((*MockAPIClient)(nil).UpdateIPAMPoolMonitor), scalar, spec)
+}
+
+// GetNC mocks base method
+func (m *MockAPIClient) GetNC(nc cns.GetNetworkContainerRequest) (cns.GetNetworkContainerResponse, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetNC", nc)
+	ret0, _ := ret[0].(cns.GetNetworkContainerResponse)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// GetNC indicates an expected call of GetNC
+func (mr *MockAPIClientMockRecorder) GetNC(nc interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetNC", reflect.TypeOf((*MockAPIClient)(nil).GetNC), nc)
+}
+
+// DeleteNC mocks base method
+func (m *MockAPIClient) DeleteNC(nc cns.DeleteNetworkContainerRequest) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "DeleteNC", nc)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// DeleteNC indicates an expected call of DeleteNC
+func (mr *MockAPIClientMockRecorder) DeleteNC(nc interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteNC", reflect.TypeOf((*MockAPIClient)(nil).DeleteNC), nc)
+}

cns/multitenantcontroller/mockclients/kubeclient.go

@@ -0,0 +1,540 @@
+// Code generated by MockGen. DO NOT EDIT.
+// Source: /go/src/sigs.k8s.io/controller-runtime/pkg/client/interfaces.go
+
+// Package mockclients is a generated GoMock package.
+package mockclients
+
+import (
+	context "context"
+	gomock "github.com/golang/mock/gomock"
+	runtime "k8s.io/apimachinery/pkg/runtime"
+	types "k8s.io/apimachinery/pkg/types"
+	reflect "reflect"
+	client "sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+// MockPatch is a mock of Patch interface
+type MockPatch struct {
+	ctrl     *gomock.Controller
+	recorder *MockPatchMockRecorder
+}
+
+// MockPatchMockRecorder is the mock recorder for MockPatch
+type MockPatchMockRecorder struct {
+	mock *MockPatch
+}
+
+// NewMockPatch creates a new mock instance
+func NewMockPatch(ctrl *gomock.Controller) *MockPatch {
+	mock := &MockPatch{ctrl: ctrl}
+	mock.recorder = &MockPatchMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use
+func (m *MockPatch) EXPECT() *MockPatchMockRecorder {
+	return m.recorder
+}
+
+// Type mocks base method
+func (m *MockPatch) Type() types.PatchType {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Type")
+	ret0, _ := ret[0].(types.PatchType)
+	return ret0
+}
+
+// Type indicates an expected call of Type
+func (mr *MockPatchMockRecorder) Type() *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Type", reflect.TypeOf((*MockPatch)(nil).Type))
+}
+
+// Data mocks base method
+func (m *MockPatch) Data(obj runtime.Object) ([]byte, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Data", obj)
+	ret0, _ := ret[0].([]byte)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// Data indicates an expected call of Data
+func (mr *MockPatchMockRecorder) Data(obj interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Data", reflect.TypeOf((*MockPatch)(nil).Data), obj)
+}
+
+// MockReader is a mock of Reader interface
+type MockReader struct {
+	ctrl     *gomock.Controller
+	recorder *MockReaderMockRecorder
+}
+
+// MockReaderMockRecorder is the mock recorder for MockReader
+type MockReaderMockRecorder struct {
+	mock *MockReader
+}
+
+// NewMockReader creates a new mock instance
+func NewMockReader(ctrl *gomock.Controller) *MockReader {
+	mock := &MockReader{ctrl: ctrl}
+	mock.recorder = &MockReaderMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use
+func (m *MockReader) EXPECT() *MockReaderMockRecorder {
+	return m.recorder
+}
+
+// Get mocks base method
+func (m *MockReader) Get(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Get", ctx, key, obj)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// Get indicates an expected call of Get
+func (mr *MockReaderMockRecorder) Get(ctx, key, obj interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockReader)(nil).Get), ctx, key, obj)
+}
+
+// List mocks base method
+func (m *MockReader) List(ctx context.Context, list runtime.Object, opts ...client.ListOption) error {
+	m.ctrl.T.Helper()
+	varargs := []interface{}{ctx, list}
+	for _, a := range opts {
+		varargs = append(varargs, a)
+	}
+	ret := m.ctrl.Call(m, "List", varargs...)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// List indicates an expected call of List
+func (mr *MockReaderMockRecorder) List(ctx, list interface{}, opts ...interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	varargs := append([]interface{}{ctx, list}, opts...)
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*MockReader)(nil).List), varargs...)
+}
+
+// MockWriter is a mock of Writer interface
+type MockWriter struct {
+	ctrl     *gomock.Controller
+	recorder *MockWriterMockRecorder
+}
+
+// MockWriterMockRecorder is the mock recorder for MockWriter
+type MockWriterMockRecorder struct {
+	mock *MockWriter
+}
+
+// NewMockWriter creates a new mock instance
+func NewMockWriter(ctrl *gomock.Controller) *MockWriter {
+	mock := &MockWriter{ctrl: ctrl}
+	mock.recorder = &MockWriterMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use
+func (m *MockWriter) EXPECT() *MockWriterMockRecorder {
+	return m.recorder
+}
+
+// Create mocks base method
+func (m *MockWriter) Create(ctx context.Context, obj runtime.Object, opts ...client.CreateOption) error {
+	m.ctrl.T.Helper()
+	varargs := []interface{}{ctx, obj}
+	for _, a := range opts {
+		varargs = append(varargs, a)
+	}
+	ret := m.ctrl.Call(m, "Create", varargs...)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// Create indicates an expected call of Create
+func (mr *MockWriterMockRecorder) Create(ctx, obj interface{}, opts ...interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	varargs := append([]interface{}{ctx, obj}, opts...)
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Create", reflect.TypeOf((*MockWriter)(nil).Create), varargs...)
+}
+
+// Delete mocks base method
+func (m *MockWriter) Delete(ctx context.Context, obj runtime.Object, opts ...client.DeleteOption) error {
+	m.ctrl.T.Helper()
+	varargs := []interface{}{ctx, obj}
+	for _, a := range opts {
+		varargs = append(varargs, a)
+	}
+	ret := m.ctrl.Call(m, "Delete", varargs...)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// Delete indicates an expected call of Delete
+func (mr *MockWriterMockRecorder) Delete(ctx, obj interface{}, opts ...interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	varargs := append([]interface{}{ctx, obj}, opts...)
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockWriter)(nil).Delete), varargs...)
+}
+
+// Update mocks base method
+func (m *MockWriter) Update(ctx context.Context, obj runtime.Object, opts ...client.UpdateOption) error {
+	m.ctrl.T.Helper()
+	varargs := []interface{}{ctx, obj}
+	for _, a := range opts {
+		varargs = append(varargs, a)
+	}
+	ret := m.ctrl.Call(m, "Update", varargs...)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// Update indicates an expected call of Update
+func (mr *MockWriterMockRecorder) Update(ctx, obj interface{}, opts ...interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	varargs := append([]interface{}{ctx, obj}, opts...)
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Update", reflect.TypeOf((*MockWriter)(nil).Update), varargs...)
+}
+
+// Patch mocks base method
+func (m *MockWriter) Patch(ctx context.Context, obj runtime.Object, patch client.Patch, opts ...client.PatchOption) error {
+	m.ctrl.T.Helper()
+	varargs := []interface{}{ctx, obj, patch}
+	for _, a := range opts {
+		varargs = append(varargs, a)
+	}
+	ret := m.ctrl.Call(m, "Patch", varargs...)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// Patch indicates an expected call of Patch
+func (mr *MockWriterMockRecorder) Patch(ctx, obj, patch interface{}, opts ...interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	varargs := append([]interface{}{ctx, obj, patch}, opts...)
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Patch", reflect.TypeOf((*MockWriter)(nil).Patch), varargs...)
+}
+
+// DeleteAllOf mocks base method
+func (m *MockWriter) DeleteAllOf(ctx context.Context, obj runtime.Object, opts ...client.DeleteAllOfOption) error {
+	m.ctrl.T.Helper()
+	varargs := []interface{}{ctx, obj}
+	for _, a := range opts {
+		varargs = append(varargs, a)
+	}
+	ret := m.ctrl.Call(m, "DeleteAllOf", varargs...)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// DeleteAllOf indicates an expected call of DeleteAllOf
+func (mr *MockWriterMockRecorder) DeleteAllOf(ctx, obj interface{}, opts ...interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	varargs := append([]interface{}{ctx, obj}, opts...)
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteAllOf", reflect.TypeOf((*MockWriter)(nil).DeleteAllOf), varargs...)
+}
+
+// MockStatusClient is a mock of StatusClient interface
+type MockStatusClient struct {
+	ctrl     *gomock.Controller
+	recorder *MockStatusClientMockRecorder
+}
+
+// MockStatusClientMockRecorder is the mock recorder for MockStatusClient
+type MockStatusClientMockRecorder struct {
+	mock *MockStatusClient
+}
+
+// NewMockStatusClient creates a new mock instance
+func NewMockStatusClient(ctrl *gomock.Controller) *MockStatusClient {
+	mock := &MockStatusClient{ctrl: ctrl}
+	mock.recorder = &MockStatusClientMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use
+func (m *MockStatusClient) EXPECT() *MockStatusClientMockRecorder {
+	return m.recorder
+}
+
+// Status mocks base method
+func (m *MockStatusClient) Status() client.StatusWriter {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Status")
+	ret0, _ := ret[0].(client.StatusWriter)
+	return ret0
+}
+
+// Status indicates an expected call of Status
+func (mr *MockStatusClientMockRecorder) Status() *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Status", reflect.TypeOf((*MockStatusClient)(nil).Status))
+}
+
+// MockStatusWriter is a mock of StatusWriter interface
+type MockStatusWriter struct {
+	ctrl     *gomock.Controller
+	recorder *MockStatusWriterMockRecorder
+}
+
+// MockStatusWriterMockRecorder is the mock recorder for MockStatusWriter
+type MockStatusWriterMockRecorder struct {
+	mock *MockStatusWriter
+}
+
+// NewMockStatusWriter creates a new mock instance
+func NewMockStatusWriter(ctrl *gomock.Controller) *MockStatusWriter {
+	mock := &MockStatusWriter{ctrl: ctrl}
+	mock.recorder = &MockStatusWriterMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use
+func (m *MockStatusWriter) EXPECT() *MockStatusWriterMockRecorder {
+	return m.recorder
+}
+
+// Update mocks base method
+func (m *MockStatusWriter) Update(ctx context.Context, obj runtime.Object, opts ...client.UpdateOption) error {
+	m.ctrl.T.Helper()
+	varargs := []interface{}{ctx, obj}
+	for _, a := range opts {
+		varargs = append(varargs, a)
+	}
+	ret := m.ctrl.Call(m, "Update", varargs...)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// Update indicates an expected call of Update
+func (mr *MockStatusWriterMockRecorder) Update(ctx, obj interface{}, opts ...interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	varargs := append([]interface{}{ctx, obj}, opts...)
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Update", reflect.TypeOf((*MockStatusWriter)(nil).Update), varargs...)
+}
+
+// Patch mocks base method
+func (m *MockStatusWriter) Patch(ctx context.Context, obj runtime.Object, patch client.Patch, opts ...client.PatchOption) error {
+	m.ctrl.T.Helper()
+	varargs := []interface{}{ctx, obj, patch}
+	for _, a := range opts {
+		varargs = append(varargs, a)
+	}
+	ret := m.ctrl.Call(m, "Patch", varargs...)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// Patch indicates an expected call of Patch
+func (mr *MockStatusWriterMockRecorder) Patch(ctx, obj, patch interface{}, opts ...interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	varargs := append([]interface{}{ctx, obj, patch}, opts...)
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Patch", reflect.TypeOf((*MockStatusWriter)(nil).Patch), varargs...)
+}
+
+// MockClient is a mock of Client interface
+type MockClient struct {
+	ctrl     *gomock.Controller
+	recorder *MockClientMockRecorder
+}
+
+// MockClientMockRecorder is the mock recorder for MockClient
+type MockClientMockRecorder struct {
+	mock *MockClient
+}
+
+// NewMockClient creates a new mock instance
+func NewMockClient(ctrl *gomock.Controller) *MockClient {
+	mock := &MockClient{ctrl: ctrl}
+	mock.recorder = &MockClientMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use
+func (m *MockClient) EXPECT() *MockClientMockRecorder {
+	return m.recorder
+}
+
+// Get mocks base method
+func (m *MockClient) Get(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Get", ctx, key, obj)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// Get indicates an expected call of Get
+func (mr *MockClientMockRecorder) Get(ctx, key, obj interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockClient)(nil).Get), ctx, key, obj)
+}
+
+// List mocks base method
+func (m *MockClient) List(ctx context.Context, list runtime.Object, opts ...client.ListOption) error {
+	m.ctrl.T.Helper()
+	varargs := []interface{}{ctx, list}
+	for _, a := range opts {
+		varargs = append(varargs, a)
+	}
+	ret := m.ctrl.Call(m, "List", varargs...)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// List indicates an expected call of List
+func (mr *MockClientMockRecorder) List(ctx, list interface{}, opts ...interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	varargs := append([]interface{}{ctx, list}, opts...)
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*MockClient)(nil).List), varargs...)
+}
+
+// Create mocks base method
+func (m *MockClient) Create(ctx context.Context, obj runtime.Object, opts ...client.CreateOption) error {
+	m.ctrl.T.Helper()
+	varargs := []interface{}{ctx, obj}
+	for _, a := range opts {
+		varargs = append(varargs, a)
+	}
+	ret := m.ctrl.Call(m, "Create", varargs...)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// Create indicates an expected call of Create
+func (mr *MockClientMockRecorder) Create(ctx, obj interface{}, opts ...interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	varargs := append([]interface{}{ctx, obj}, opts...)
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Create", reflect.TypeOf((*MockClient)(nil).Create), varargs...)
+}
+
+// Delete mocks base method
+func (m *MockClient) Delete(ctx context.Context, obj runtime.Object, opts ...client.DeleteOption) error {
+	m.ctrl.T.Helper()
+	varargs := []interface{}{ctx, obj}
+	for _, a := range opts {
+		varargs = append(varargs, a)
+	}
+	ret := m.ctrl.Call(m, "Delete", varargs...)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// Delete indicates an expected call of Delete
+func (mr *MockClientMockRecorder) Delete(ctx, obj interface{}, opts ...interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	varargs := append([]interface{}{ctx, obj}, opts...)
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockClient)(nil).Delete), varargs...)
+}
+
+// Update mocks base method
+func (m *MockClient) Update(ctx context.Context, obj runtime.Object, opts ...client.UpdateOption) error {
+	m.ctrl.T.Helper()
+	varargs := []interface{}{ctx, obj}
+	for _, a := range opts {
+		varargs = append(varargs, a)
+	}
+	ret := m.ctrl.Call(m, "Update", varargs...)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// Update indicates an expected call of Update
+func (mr *MockClientMockRecorder) Update(ctx, obj interface{}, opts ...interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	varargs := append([]interface{}{ctx, obj}, opts...)
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Update", reflect.TypeOf((*MockClient)(nil).Update), varargs...)
+}
+
+// Patch mocks base method
+func (m *MockClient) Patch(ctx context.Context, obj runtime.Object, patch client.Patch, opts ...client.PatchOption) error {
+	m.ctrl.T.Helper()
+	varargs := []interface{}{ctx, obj, patch}
+	for _, a := range opts {
+		varargs = append(varargs, a)
+	}
+	ret := m.ctrl.Call(m, "Patch", varargs...)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// Patch indicates an expected call of Patch
+func (mr *MockClientMockRecorder) Patch(ctx, obj, patch interface{}, opts ...interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	varargs := append([]interface{}{ctx, obj, patch}, opts...)
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Patch", reflect.TypeOf((*MockClient)(nil).Patch), varargs...)
+}
+
+// DeleteAllOf mocks base method
+func (m *MockClient) DeleteAllOf(ctx context.Context, obj runtime.Object, opts ...client.DeleteAllOfOption) error {
+	m.ctrl.T.Helper()
+	varargs := []interface{}{ctx, obj}
+	for _, a := range opts {
+		varargs = append(varargs, a)
+	}
+	ret := m.ctrl.Call(m, "DeleteAllOf", varargs...)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// DeleteAllOf indicates an expected call of DeleteAllOf
+func (mr *MockClientMockRecorder) DeleteAllOf(ctx, obj interface{}, opts ...interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	varargs := append([]interface{}{ctx, obj}, opts...)
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteAllOf", reflect.TypeOf((*MockClient)(nil).DeleteAllOf), varargs...)
+}
+
+// Status mocks base method
+func (m *MockClient) Status() client.StatusWriter {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Status")
+	ret0, _ := ret[0].(client.StatusWriter)
+	return ret0
+}
+
+// Status indicates an expected call of Status
+func (mr *MockClientMockRecorder) Status() *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Status", reflect.TypeOf((*MockClient)(nil).Status))
+}
+
+// MockFieldIndexer is a mock of FieldIndexer interface
+type MockFieldIndexer struct {
+	ctrl     *gomock.Controller
+	recorder *MockFieldIndexerMockRecorder
+}
+
+// MockFieldIndexerMockRecorder is the mock recorder for MockFieldIndexer
+type MockFieldIndexerMockRecorder struct {
+	mock *MockFieldIndexer
+}
+
+// NewMockFieldIndexer creates a new mock instance
+func NewMockFieldIndexer(ctrl *gomock.Controller) *MockFieldIndexer {
+	mock := &MockFieldIndexer{ctrl: ctrl}
+	mock.recorder = &MockFieldIndexerMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use
+func (m *MockFieldIndexer) EXPECT() *MockFieldIndexerMockRecorder {
+	return m.recorder
+}
+
+// IndexField mocks base method
+func (m *MockFieldIndexer) IndexField(ctx context.Context, obj runtime.Object, field string, extractValue client.IndexerFunc) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "IndexField", ctx, obj, field, extractValue)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// IndexField indicates an expected call of IndexField
+func (mr *MockFieldIndexerMockRecorder) IndexField(ctx, obj, field, extractValue interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IndexField", reflect.TypeOf((*MockFieldIndexer)(nil).IndexField), ctx, obj, field, extractValue)
+}

cns/multitenantcontroller/multitenantcontrollerinterface.go

@@ -0,0 +1,7 @@
+package multitenantcontroller
+
+// MultiTenantController defines the interface for multi-tenant network container operations.
+type MultiTenantController interface {
+	StartMultiTenantController(exitChan <-chan struct{}) error
+	IsStarted() bool
+}

cns/multitenantcontroller/multitenantoperator/multitenantcrdcontroller.go

@@ -0,0 +1,159 @@
+package multitenantoperator
+
+import (
+	"errors"
+	"os"
+	"sync"
+
+	"github.com/Azure/azure-container-networking/cns/cnsclient"
+	"github.com/Azure/azure-container-networking/cns/cnsclient/httpapi"
+	"github.com/Azure/azure-container-networking/cns/logger"
+	"github.com/Azure/azure-container-networking/cns/restserver"
+	ncapi "github.com/Azure/azure-container-networking/networkcontainer/api/v1alpha1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/rest"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/manager"
+)
+
+const (
+	nodeNameEnvVar    = "NODENAME"
+	prometheusAddress = "0" //0 means disabled
+)
+
+// multiTenantController operates multi-tenant CRD.
+type multiTenantController struct {
+	mgr        manager.Manager //Manager starts the reconcile loop which watches for crd status changes
+	KubeClient client.Client   //KubeClient is a cached client which interacts with API server
+	CNSClient  cnsclient.APIClient
+	nodeName   string //name of node running this program
+	Reconciler *multiTenantCrdReconciler
+	Started    bool
+	lock       sync.Mutex
+}
+
+// MultiTenantController create a new multi-tenant CRD operator.
+func NewMultiTenantController(restService *restserver.HTTPRestService, kubeconfig *rest.Config) (*multiTenantController, error) {
+	// Check that logger package has been initialized.
+	if logger.Log == nil {
+		return nil, errors.New("Must initialize logger before calling")
+	}
+
+	// Check that NODENAME environment variable is set. NODENAME is name of node running this program.
+	nodeName := os.Getenv(nodeNameEnvVar)
+	if nodeName == "" {
+		return nil, errors.New("Must declare " + nodeNameEnvVar + " environment variable.")
+	}
+
+	// Add client-go scheme to runtime scheme so manager can recognize it.
+	var scheme = runtime.NewScheme()
+	if err := clientgoscheme.AddToScheme(scheme); err != nil {
+		return nil, errors.New("Error adding client-go scheme to runtime scheme")
+	}
+
+	// Add CRD scheme to runtime sheme so manager can recognize it.
+	if err := ncapi.AddToScheme(scheme); err != nil {
+		return nil, errors.New("Error adding NetworkContainer scheme to runtime scheme")
+	}
+
+	// Create manager for multiTenantController.
+	mgr, err := ctrl.NewManager(kubeconfig, ctrl.Options{
+		Scheme:             scheme,
+		MetricsBindAddress: prometheusAddress,
+	})
+	if err != nil {
+		logger.Errorf("Error creating new multiTenantController: %v", err)
+		return nil, err
+	}
+
+	// Create httpClient
+	httpClient := &httpapi.Client{
+		RestService: restService,
+	}
+
+	// Create multiTenantCrdReconciler
+	reconciler := &multiTenantCrdReconciler{
+		KubeClient: mgr.GetClient(),
+		NodeName:   nodeName,
+		CNSClient:  httpClient,
+	}
+	if err := reconciler.SetupWithManager(mgr); err != nil {
+		logger.Errorf("Error setting up new multiTenantCrdReconciler: %v", err)
+		return nil, err
+	}
+
+	// Create the multiTenantController
+	return &multiTenantController{
+		mgr:        mgr,
+		KubeClient: mgr.GetClient(),
+		CNSClient:  httpClient,
+		nodeName:   nodeName,
+		Reconciler: reconciler,
+	}, nil
+}
+
+// StartMultiTenantController starts the Reconciler loop which watches for CRD status updates.
+// Blocks until SIGINT or SIGTERM is received
+// Notifies exitChan when kill signal received
+func (mtc *multiTenantController) StartMultiTenantController(exitChan <-chan struct{}) error {
+	logger.Printf("Starting MultiTenantController")
+
+	// Setting the started state
+	mtc.lock.Lock()
+	mtc.Started = true
+	mtc.lock.Unlock()
+
+	logger.Printf("Starting reconcile loop")
+	if err := mtc.mgr.Start(exitChan); err != nil {
+		if mtc.isNotDefined(err) {
+			logger.Errorf("multi-tenant CRD is not defined on cluster, starting reconcile loop failed: %v", err)
+			os.Exit(1)
+		}
+
+		return err
+	}
+
+	return nil
+}
+
+// return if RequestController is started
+func (mtc *multiTenantController) IsStarted() bool {
+	defer mtc.lock.Unlock()
+	mtc.lock.Lock()
+	return mtc.Started
+}
+
+// isNotDefined tells whether the given error is a CRD not defined error
+func (mtc *multiTenantController) isNotDefined(err error) bool {
+	var (
+		statusError *apierrors.StatusError
+		ok          bool
+		notDefined  bool
+		cause       metav1.StatusCause
+	)
+
+	if err == nil {
+		return false
+	}
+
+	if statusError, ok = err.(*apierrors.StatusError); !ok {
+		return false
+	}
+
+	if len(statusError.ErrStatus.Details.Causes) > 0 {
+		for _, cause = range statusError.ErrStatus.Details.Causes {
+			if cause.Type == metav1.CauseTypeUnexpectedServerResponse {
+				if apierrors.IsNotFound(err) {
+					notDefined = true
+					break
+				}
+			}
+		}
+	}
+
+	return notDefined
+}

cns/multitenantcontroller/multitenantoperator/multitenantcrdcontroller_test.go

@@ -0,0 +1,39 @@
+package multitenantoperator
+
+import (
+	"os"
+
+	"github.com/Azure/azure-container-networking/cns/logger"
+	"github.com/Azure/azure-container-networking/cns/restserver"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"k8s.io/client-go/rest"
+)
+
+var _ = Describe("multiTenantController", func() {
+	BeforeEach(func() {
+		logger.InitLogger("multiTenantController", 0, 0, "")
+	})
+
+	Context("lifecycle", func() {
+		restService := &restserver.HTTPRestService{}
+		kubeconfig := &rest.Config{}
+
+		It("Should exist with an error when nodeName is not set", func() {
+			ctl, err := NewMultiTenantController(restService, kubeconfig)
+			Expect(ctl).To(BeNil())
+			Expect(err).NotTo(BeNil())
+			Expect(err.Error()).To(Equal("Must declare NODENAME environment variable."))
+		})
+
+		It("Should report an error when apiserver is not available", func() {
+			val := os.Getenv(nodeNameEnvVar)
+			os.Setenv(nodeNameEnvVar, "nodeName")
+			ctl, err := NewMultiTenantController(nil, nil)
+			os.Setenv(nodeNameEnvVar, val)
+			Expect(ctl).To(BeNil())
+			Expect(err).NotTo(BeNil())
+			Expect(err.Error()).To(Equal("must specify Config"))
+		})
+	})
+})

cns/multitenantcontroller/multitenantoperator/multitenantcrdreconciler.go

@@ -0,0 +1,156 @@
+package multitenantoperator
+
+import (
+	"context"
+	"net"
+	"strings"
+
+	"github.com/Azure/azure-container-networking/cns"
+	"github.com/Azure/azure-container-networking/cns/cnsclient"
+	"github.com/Azure/azure-container-networking/cns/logger"
+	ncapi "github.com/Azure/azure-container-networking/networkcontainer/api/v1alpha1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/runtime"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/event"
+	"sigs.k8s.io/controller-runtime/pkg/predicate"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+)
+
+const (
+	NCStateInitialized = "Initialized" // NC has been initialized by DNC.
+	NCStateSucceeded   = "Succeeded"   // NC has been persisted by CNS
+	NCStateTerminated  = "Terminated"  // NC has been cleaned up from CNS
+)
+
+// multiTenantCrdReconciler reconciles multi-tenant network containers.
+type multiTenantCrdReconciler struct {
+	KubeClient client.Client
+	NodeName   string
+	CNSClient  cnsclient.APIClient
+}
+
+// Reconcile is called on multi-tenant CRD status changes.
+func (r *multiTenantCrdReconciler) Reconcile(request reconcile.Request) (reconcile.Result, error) {
+	ctx := context.Background()
+	var nc ncapi.NetworkContainer
+
+	if err := r.KubeClient.Get(ctx, request.NamespacedName, &nc); err != nil {
+		if apierrors.IsNotFound(err) {
+			return ctrl.Result{}, nil
+		}
+
+		logger.Errorf("Failed to fetch network container: %v", err)
+		return ctrl.Result{}, err
+	}
+
+	if !nc.ObjectMeta.DeletionTimestamp.IsZero() {
+		// Do nothing if the NC has already in Terminated state.
+		if nc.Status.State == NCStateTerminated {
+			return ctrl.Result{}, nil
+		}
+
+		// Remove the deleted network container from CNS.
+		err := r.CNSClient.DeleteNC(cns.DeleteNetworkContainerRequest{
+			NetworkContainerid: nc.Spec.UUID,
+		})
+		if err != nil {
+			logger.Errorf("Failed to delete NC %s from CNS: %v", nc.Spec.UUID, err)
+			return ctrl.Result{}, err
+		}
+
+		// Update NC state to Terminated.
+		nc.Status.State = NCStateTerminated
+		if err := r.KubeClient.Status().Update(ctx, &nc); err != nil {
+			logger.Errorf("Failed to update network container state for %s: %v", nc.Spec.UUID, err)
+			return ctrl.Result{}, err
+		}
+
+		logger.Printf("NC has been terminated for %s", nc.Spec.UUID)
+		return ctrl.Result{}, nil
+	}
+
+	// Do nothing if the network container hasn't been initialized yet from control plane.
+	if nc.Status.State != NCStateInitialized {
+		return ctrl.Result{}, nil
+	}
+
+	// Check CNS NC states.
+	_, err := r.CNSClient.GetNC(cns.GetNetworkContainerRequest{
+		NetworkContainerid: nc.Spec.UUID,
+	})
+	if err == nil {
+		logger.Printf("NC %s has already been created in CNS", nc.Spec.UUID)
+		return ctrl.Result{}, nil
+	} else if err.Error() != "NotFound" {
+		logger.Errorf("Failed to fetch NC from CNS: %v", err)
+		return ctrl.Result{}, err
+	}
+
+	// Persist NC states into CNS.
+	_, ipNet, err := net.ParseCIDR(nc.Status.IPSubnet)
+	if err != nil {
+		logger.Errorf("Failed to parse IPSubnet %s for NC %s: %v", nc.Status.IPSubnet, nc.Spec.UUID, err)
+		return ctrl.Result{}, err
+	}
+	prefixLength, _ := ipNet.Mask.Size()
+	networkContainerRequest := cns.CreateNetworkContainerRequest{
+		NetworkContainerid: nc.Spec.UUID,
+		IPConfiguration: cns.IPConfiguration{
+			IPSubnet: cns.IPSubnet{
+				IPAddress:    nc.Status.IP,
+				PrefixLength: uint8(prefixLength),
+			},
+			GatewayIPAddress: nc.Status.Gateway,
+		},
+	}
+	if err = r.CNSClient.CreateOrUpdateNC(networkContainerRequest); err != nil {
+		logger.Errorf("Failed to persist state for NC %s to CNS: %v", nc.Spec.UUID, err)
+		return ctrl.Result{}, err
+	}
+
+	// Update NC state to Succeeded.
+	nc.Status.State = NCStateSucceeded
+	if err := r.KubeClient.Status().Update(ctx, &nc); err != nil {
+		logger.Errorf("Failed to update network container state for %s: %v", nc.Spec.UUID, err)
+		return ctrl.Result{}, err
+	}
+
+	logger.Printf("Reconciled NC %s", nc.Spec.UUID)
+	return reconcile.Result{}, nil
+}
+
+// SetupWithManager Sets up the reconciler with a new manager, filtering using NodeNetworkConfigFilter
+func (r *multiTenantCrdReconciler) SetupWithManager(mgr ctrl.Manager) error {
+	return ctrl.NewControllerManagedBy(mgr).
+		For(&ncapi.NetworkContainer{}).
+		WithEventFilter(r.predicate()).
+		Complete(r)
+}
+
+func (r *multiTenantCrdReconciler) predicate() predicate.Predicate {
+	return predicate.Funcs{
+		CreateFunc: func(e event.CreateEvent) bool {
+			return r.equalNode(e.Object)
+		},
+		GenericFunc: func(e event.GenericEvent) bool {
+			return r.equalNode(e.Object)
+		},
+		UpdateFunc: func(e event.UpdateEvent) bool {
+			return r.equalNode(e.ObjectNew)
+		},
+		DeleteFunc: func(e event.DeleteEvent) bool {
+			return r.equalNode(e.Object)
+		},
+	}
+}
+
+func (r *multiTenantCrdReconciler) equalNode(o runtime.Object) bool {
+	nc, ok := o.(*ncapi.NetworkContainer)
+	if ok {
+		return strings.EqualFold(nc.Spec.Node, r.NodeName)
+	}
+
+	return false
+}

cns/multitenantcontroller/multitenantoperator/multitenantcrdreconciler_test.go

@@ -0,0 +1,161 @@
+package multitenantoperator
+
+import (
+	"fmt"
+
+	"github.com/Azure/azure-container-networking/cns"
+	"github.com/Azure/azure-container-networking/cns/logger"
+	"github.com/Azure/azure-container-networking/cns/multitenantcontroller/mockclients"
+	ncapi "github.com/Azure/azure-container-networking/networkcontainer/api/v1alpha1"
+	"github.com/golang/mock/gomock"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+)
+
+var _ = Describe("multiTenantCrdReconciler", func() {
+	var kubeClient *mockclients.MockClient
+	var cnsClient *mockclients.MockAPIClient
+	var mockCtl *gomock.Controller
+	var reconciler *multiTenantCrdReconciler
+	var mockNodeName = "mockNodeName"
+	var namespacedName = types.NamespacedName{
+		Namespace: "test",
+		Name:      "test",
+	}
+
+	BeforeEach(func() {
+		logger.InitLogger("multiTenantCrdReconciler", 0, 0, "")
+		mockCtl = gomock.NewController(GinkgoT())
+		kubeClient = mockclients.NewMockClient(mockCtl)
+		cnsClient = mockclients.NewMockAPIClient(mockCtl)
+		reconciler = &multiTenantCrdReconciler{
+			KubeClient: kubeClient,
+			NodeName:   mockNodeName,
+			CNSClient:  cnsClient,
+		}
+	})
+
+	Context("lifecycle", func() {
+
+		It("Should succeed when the NC has already been deleted", func() {
+			expectedError := &apierrors.StatusError{
+				ErrStatus: metav1.Status{
+					Reason: metav1.StatusReasonNotFound,
+				},
+			}
+			kubeClient.EXPECT().Get(gomock.Any(), namespacedName, gomock.Any()).Return(expectedError)
+			_, err := reconciler.Reconcile(reconcile.Request{
+				NamespacedName: namespacedName,
+			})
+			Expect(err).To(BeNil())
+		})
+
+		It("Should fail when the kube client reports failure", func() {
+			expectedError := &apierrors.StatusError{
+				ErrStatus: metav1.Status{
+					Reason: metav1.StatusReasonInternalError,
+				},
+			}
+			kubeClient.EXPECT().Get(gomock.Any(), namespacedName, gomock.Any()).Return(expectedError)
+			_, err := reconciler.Reconcile(reconcile.Request{
+				NamespacedName: namespacedName,
+			})
+			Expect(err).NotTo(BeNil())
+			Expect(err).To(Equal(expectedError))
+		})
+
+		It("Should succeed when the NC is in Terminated state", func() {
+			var nc ncapi.NetworkContainer = ncapi.NetworkContainer{
+				ObjectMeta: metav1.ObjectMeta{
+					DeletionTimestamp: &metav1.Time{},
+				},
+				Status: ncapi.NetworkContainerStatus{
+					State: "Terminated",
+				},
+			}
+			kubeClient.EXPECT().Get(gomock.Any(), namespacedName, gomock.Any()).SetArg(2, nc)
+			_, err := reconciler.Reconcile(reconcile.Request{
+				NamespacedName: namespacedName,
+			})
+			Expect(err).To(BeNil())
+		})
+
+		It("Should succeed when the NC is not in Initialized state", func() {
+			var nc ncapi.NetworkContainer = ncapi.NetworkContainer{
+				Status: ncapi.NetworkContainerStatus{
+					State: "Pending",
+				},
+			}
+			kubeClient.EXPECT().Get(gomock.Any(), namespacedName, gomock.Any()).SetArg(2, nc)
+			_, err := reconciler.Reconcile(reconcile.Request{
+				NamespacedName: namespacedName,
+			})
+			Expect(err).To(BeNil())
+		})
+
+		It("Should succeed when the NC is in Initialized state and it has already been persisted in CNS", func() {
+			var uuid = "uuid"
+			var nc ncapi.NetworkContainer = ncapi.NetworkContainer{
+				Spec: ncapi.NetworkContainerSpec{
+					UUID: uuid,
+				},
+				Status: ncapi.NetworkContainerStatus{
+					State: "Initialized",
+				},
+			}
+			kubeClient.EXPECT().Get(gomock.Any(), namespacedName, gomock.Any()).SetArg(2, nc)
+			cnsClient.EXPECT().GetNC(cns.GetNetworkContainerRequest{NetworkContainerid: uuid}).Return(cns.GetNetworkContainerResponse{}, nil)
+			_, err := reconciler.Reconcile(reconcile.Request{
+				NamespacedName: namespacedName,
+			})
+			Expect(err).To(BeNil())
+		})
+
+		It("Should fail when the NC subnet isn't in correct format", func() {
+			var uuid = "uuid"
+			var nc ncapi.NetworkContainer = ncapi.NetworkContainer{
+				Spec: ncapi.NetworkContainerSpec{
+					UUID: uuid,
+				},
+				Status: ncapi.NetworkContainerStatus{
+					State:    "Initialized",
+					IPSubnet: "1.2.3.4.5",
+				},
+			}
+			kubeClient.EXPECT().Get(gomock.Any(), namespacedName, gomock.Any()).SetArg(2, nc)
+			cnsClient.EXPECT().GetNC(cns.GetNetworkContainerRequest{NetworkContainerid: uuid}).Return(cns.GetNetworkContainerResponse{}, fmt.Errorf("NotFound"))
+			_, err := reconciler.Reconcile(reconcile.Request{
+				NamespacedName: namespacedName,
+			})
+			Expect(err).NotTo(BeNil())
+			Expect(err.Error()).To(ContainSubstring("invalid CIDR address"))
+		})
+
+		It("Should succeed when the NC subnet is in correct format", func() {
+			var uuid = "uuid"
+			var nc ncapi.NetworkContainer = ncapi.NetworkContainer{
+				Spec: ncapi.NetworkContainerSpec{
+					UUID: uuid,
+				},
+				Status: ncapi.NetworkContainerStatus{
+					State:    "Initialized",
+					IPSubnet: "1.2.3.0/24",
+				},
+			}
+			kubeClient.EXPECT().Get(gomock.Any(), namespacedName, gomock.Any()).SetArg(2, nc)
+			statusWriter := mockclients.NewMockStatusWriter(mockCtl)
+			statusWriter.EXPECT().Update(gomock.Any(), gomock.Any()).Return(nil)
+			kubeClient.EXPECT().Status().Return(statusWriter)
+			cnsClient.EXPECT().GetNC(cns.GetNetworkContainerRequest{NetworkContainerid: uuid}).Return(cns.GetNetworkContainerResponse{}, fmt.Errorf("NotFound"))
+			cnsClient.EXPECT().CreateOrUpdateNC(gomock.Any()).Return(nil)
+			_, err := reconciler.Reconcile(reconcile.Request{
+				NamespacedName: namespacedName,
+			})
+			Expect(err).To(BeNil())
+		})
+	})
+})

cns/multitenantcontroller/multitenantoperator/multitenantoperator_suite_test.go

@@ -0,0 +1,13 @@
+package multitenantoperator
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestMultitenantoperator(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Multitenantoperator Suite")
+}

cns/requestcontroller/kubecontroller/crdreconciler.go

@@ -67,12 +67,18 @@ func (r *CrdReconciler) Reconcile(request reconcile.Request) (reconcile.Result,
 		return reconcile.Result{}, err
 	}
 
-	if err = r.CNSClient.CreateOrUpdateNC(ncRequest, nodeNetConfig.Status.Scaler, nodeNetConfig.Spec); err != nil {
+	if err = r.CNSClient.CreateOrUpdateNC(ncRequest); err != nil {
 		logger.Errorf("[cns-rc] Error creating or updating NC in reconcile: %v", err)
 		// requeue
 		return reconcile.Result{}, err
 	}
 
+	if err = r.CNSClient.UpdateIPAMPoolMonitor(nodeNetConfig.Status.Scaler, nodeNetConfig.Spec); err != nil {
+		logger.Errorf("[cns-rc] Error update IPAM pool monitor in reconcile: %v", err)
+		// requeue
+		return reconcile.Result{}, err
+	}
+
 	return reconcile.Result{}, err
 }
 

cns/requestcontroller/kubecontroller/crdrequestcontroller_test.go

@@ -99,11 +99,23 @@ type MockCNSClient struct {
 }
 
 // we're just testing that reconciler interacts with CNS on Reconcile().
-func (mi *MockCNSClient) CreateOrUpdateNC(ncRequest cns.CreateNetworkContainerRequest, scalar nnc.Scaler, spec nnc.NodeNetworkConfigSpec) error {
+func (mi *MockCNSClient) CreateOrUpdateNC(ncRequest cns.CreateNetworkContainerRequest) error {
 	mi.MockCNSUpdated = true
 	return nil
 }
 
+func (mi *MockCNSClient) UpdateIPAMPoolMonitor(scalar nnc.Scaler, spec nnc.NodeNetworkConfigSpec) error {
+	return nil
+}
+
+func (mi *MockCNSClient) DeleteNC(nc cns.DeleteNetworkContainerRequest) error {
+	return nil
+}
+
+func (mi *MockCNSClient) GetNC(nc cns.GetNetworkContainerRequest) (cns.GetNetworkContainerResponse, error) {
+	return cns.GetNetworkContainerResponse{NetworkContainerID: nc.NetworkContainerid}, nil
+}
+
 func (mi *MockCNSClient) ReconcileNCState(ncRequest *cns.CreateNetworkContainerRequest, podInfoByIP map[string]cns.KubernetesPodInfo, scalar nnc.Scaler, spec nnc.NodeNetworkConfigSpec) error {
 	mi.MockCNSInitialized = true
 	mi.Pods = podInfoByIP

cns/restserver/internalapi.go

@@ -214,9 +214,12 @@ func (service *HTTPRestService) ReconcileNCState(ncRequest *cns.CreateNetworkCon
 		return Success
 	}
 
-	returnCode := service.CreateOrUpdateNetworkContainerInternal(*ncRequest, scalar, spec)
-
 	// If the NC was created successfully, then reconcile the allocated pod state
+	returnCode := service.CreateOrUpdateNetworkContainerInternal(*ncRequest)
+	if returnCode != Success {
+		return returnCode
+	}
+	returnCode = service.UpdateIPAMPoolMonitorInternal(scalar, spec)
 	if returnCode != Success {
 		return returnCode
 	}
@@ -255,8 +258,42 @@ func (service *HTTPRestService) ReconcileNCState(ncRequest *cns.CreateNetworkCon
 	return 0
 }
 
+// GetNetworkContainerInternal gets network container details.
+func (service *HTTPRestService) GetNetworkContainerInternal(req cns.GetNetworkContainerRequest) (cns.GetNetworkContainerResponse, int) {
+	getNetworkContainerResponse := service.getNetworkContainerResponse(req)
+	returnCode := getNetworkContainerResponse.Response.ReturnCode
+	return getNetworkContainerResponse, returnCode
+}
+
+// DeleteNetworkContainerInternal deletes a network container.
+func (service *HTTPRestService) DeleteNetworkContainerInternal(req cns.DeleteNetworkContainerRequest) int {
+	_, exist := service.getNetworkContainerDetails(req.NetworkContainerid)
+	if !exist {
+		logger.Printf("network container for id %v doesn't exist", req.NetworkContainerid)
+		return Success
+	}
+
+	service.Lock()
+	defer service.Unlock()
+	if service.state.ContainerStatus != nil {
+		delete(service.state.ContainerStatus, req.NetworkContainerid)
+	}
+
+	if service.state.ContainerIDByOrchestratorContext != nil {
+		for orchestratorContext, networkContainerID := range service.state.ContainerIDByOrchestratorContext {
+			if networkContainerID == req.NetworkContainerid {
+				delete(service.state.ContainerIDByOrchestratorContext, orchestratorContext)
+				break
+			}
+		}
+	}
+
+	service.saveState()
+	return Success
+}
+
 // This API will be called by CNS RequestController on CRD update.
-func (service *HTTPRestService) CreateOrUpdateNetworkContainerInternal(req cns.CreateNetworkContainerRequest, scalar nnc.Scaler, spec nnc.NodeNetworkConfigSpec) int {
+func (service *HTTPRestService) CreateOrUpdateNetworkContainerInternal(req cns.CreateNetworkContainerRequest) int {
 	if req.NetworkContainerid == "" {
 		logger.Errorf("[Azure CNS] Error. NetworkContainerid is empty")
 		return NetworkContainerNotSpecified
@@ -287,7 +324,6 @@ func (service *HTTPRestService) CreateOrUpdateNetworkContainerInternal(req cns.C
 
 	// Validate if state exists already
 	existingNCInfo, ok := service.getNetworkContainerDetails(req.NetworkContainerid)
-
 	if ok {
 		existingReq := existingNCInfo.CreateNetworkContainerRequest
 		if reflect.DeepEqual(existingReq.IPConfiguration, req.IPConfiguration) != true {
@@ -306,11 +342,15 @@ func (service *HTTPRestService) CreateOrUpdateNetworkContainerInternal(req cns.C
 		logger.Errorf(returnMessage)
 	}
 
-	if err = service.IPAMPoolMonitor.Update(scalar, spec); err != nil {
+	return returnCode
+}
+
+func (service *HTTPRestService) UpdateIPAMPoolMonitorInternal(scalar nnc.Scaler, spec nnc.NodeNetworkConfigSpec) int {
+	if err := service.IPAMPoolMonitor.Update(scalar, spec); err != nil {
 		logger.Errorf("[cns-rc] Error creating or updating IPAM Pool Monitor: %v", err)
 		// requeue
 		return UnexpectedError
 	}
 
-	return returnCode
+	return 0
 }

cns/restserver/internalapi_test.go

@@ -367,10 +367,14 @@ func validateCreateOrUpdateNCInternal(t *testing.T, secondaryIpCount int, ncVers
 
 func createAndValidateNCRequest(t *testing.T, secondaryIPConfigs map[string]cns.SecondaryIPConfig, ncId, ncVersion string) {
 	req := generateNetworkContainerRequest(secondaryIPConfigs, ncId, ncVersion)
-	returnCode := svc.CreateOrUpdateNetworkContainerInternal(req, fakes.NewFakeScalar(releasePercent, requestPercent, batchSize), fakes.NewFakeNodeNetworkConfigSpec(initPoolSize))
+	returnCode := svc.CreateOrUpdateNetworkContainerInternal(req)
 	if returnCode != 0 {
 		t.Fatalf("Failed to createNetworkContainerRequest, req: %+v, err: %d", req, returnCode)
 	}
+	returnCode = svc.UpdateIPAMPoolMonitorInternal(fakes.NewFakeScalar(releasePercent, requestPercent, batchSize), fakes.NewFakeNodeNetworkConfigSpec(initPoolSize))
+	if returnCode != 0 {
+		t.Fatalf("Failed to UpdateIPAMPoolMonitorInternal, err: %d", returnCode)
+	}
 	validateNetworkRequest(t, req)
 }
 
@@ -535,10 +539,14 @@ func validateNCStateAfterReconcile(t *testing.T, ncRequest *cns.CreateNetworkCon
 
 func createNCReqInternal(t *testing.T, secondaryIPConfigs map[string]cns.SecondaryIPConfig, ncID, ncVersion string) cns.CreateNetworkContainerRequest {
 	req := generateNetworkContainerRequest(secondaryIPConfigs, ncID, ncVersion)
-	returnCode := svc.CreateOrUpdateNetworkContainerInternal(req, fakes.NewFakeScalar(releasePercent, requestPercent, batchSize), fakes.NewFakeNodeNetworkConfigSpec(initPoolSize))
+	returnCode := svc.CreateOrUpdateNetworkContainerInternal(req)
 	if returnCode != 0 {
 		t.Fatalf("Failed to createNetworkContainerRequest, req: %+v, err: %d", req, returnCode)
 	}
+	returnCode = svc.UpdateIPAMPoolMonitorInternal(fakes.NewFakeScalar(releasePercent, requestPercent, batchSize), fakes.NewFakeNodeNetworkConfigSpec(initPoolSize))
+	if returnCode != 0 {
+		t.Fatalf("Failed to UpdateIPAMPoolMonitorInternal, err: %d", returnCode)
+	}
 	return req
 }
 

cns/restserver/ipam_test.go

@@ -600,10 +600,14 @@ func TestIPAMMarkIPAsPendingWithPendingProgrammingIPs(t *testing.T) {
 
 	// createNCRequest with NC version 0
 	req := generateNetworkContainerRequest(secondaryIPConfigs, testNCID, strconv.Itoa(0))
-	returnCode := svc.CreateOrUpdateNetworkContainerInternal(req, fakes.NewFakeScalar(releasePercent, requestPercent, batchSize), fakes.NewFakeNodeNetworkConfigSpec(initPoolSize))
+	returnCode := svc.CreateOrUpdateNetworkContainerInternal(req)
 	if returnCode != 0 {
 		t.Fatalf("Failed to createNetworkContainerRequest, req: %+v, err: %d", req, returnCode)
 	}
+	returnCode = svc.UpdateIPAMPoolMonitorInternal(fakes.NewFakeScalar(releasePercent, requestPercent, batchSize), fakes.NewFakeNodeNetworkConfigSpec(initPoolSize))
+	if returnCode != 0 {
+		t.Fatalf("Failed to UpdateIPAMPoolMonitorInternal, req: %+v, err: %d", req, returnCode)
+	}
 
 	// Release pending programming IPs
 	ips, err := svc.MarkIPAsPendingRelease(2)

cns/service/main.go

@@ -5,12 +5,9 @@ package main
 
 import (
 	"bytes"
+	"context"
 	"encoding/json"
 	"fmt"
-	"github.com/Azure/azure-container-networking/cns/ipampoolmonitor"
-	"github.com/Azure/azure-container-networking/cns/requestcontroller"
-	"github.com/Azure/azure-container-networking/cns/requestcontroller/kubecontroller"
-	localtls "github.com/Azure/azure-container-networking/server/tls"
 	"net/http"
 	"os"
 	"os/signal"
@@ -20,9 +17,6 @@ import (
 	"syscall"
 	"time"
 
-	"github.com/Azure/azure-container-networking/cns/nmagentclient"
-
-	"context"
 	"github.com/Azure/azure-container-networking/aitelemetry"
 	"github.com/Azure/azure-container-networking/cnm/ipam"
 	"github.com/Azure/azure-container-networking/cnm/network"
@@ -32,12 +26,20 @@ import (
 	"github.com/Azure/azure-container-networking/cns/configuration"
 	"github.com/Azure/azure-container-networking/cns/hnsclient"
 	"github.com/Azure/azure-container-networking/cns/imdsclient"
+	"github.com/Azure/azure-container-networking/cns/ipampoolmonitor"
 	"github.com/Azure/azure-container-networking/cns/logger"
+	"github.com/Azure/azure-container-networking/cns/multitenantcontroller"
+	"github.com/Azure/azure-container-networking/cns/multitenantcontroller/multitenantoperator"
+	"github.com/Azure/azure-container-networking/cns/nmagentclient"
+	"github.com/Azure/azure-container-networking/cns/requestcontroller"
+	"github.com/Azure/azure-container-networking/cns/requestcontroller/kubecontroller"
 	"github.com/Azure/azure-container-networking/cns/restserver"
 	acn "github.com/Azure/azure-container-networking/common"
 	"github.com/Azure/azure-container-networking/log"
 	"github.com/Azure/azure-container-networking/platform"
+	localtls "github.com/Azure/azure-container-networking/server/tls"
 	"github.com/Azure/azure-container-networking/store"
+	ctrl "sigs.k8s.io/controller-runtime"
 )
 
 const (
@@ -414,6 +416,8 @@ func main() {
 		nodeID = cnsconfig.ManagedSettings.NodeID
 	} else if cnsconfig.ChannelMode == cns.CRD {
 		config.ChannelMode = cns.CRD
+	} else if cnsconfig.ChannelMode == cns.MultiTenantCRD {
+		config.ChannelMode = cns.MultiTenantCRD
 	} else if acn.GetArg(acn.OptManaged).(bool) {
 		config.ChannelMode = cns.Managed
 	}
@@ -504,13 +508,25 @@ func main() {
 	if config.ChannelMode == cns.CRD {
 		requestControllerStopChannel := make(chan struct{})
 		defer close(requestControllerStopChannel)
-		err = IniitalizeCRDState(httpRestService, cnsconfig, requestControllerStopChannel)
+		err = InitializeCRDState(httpRestService, cnsconfig, requestControllerStopChannel)
 		if err != nil {
 			logger.Errorf("Failed to start CRD Controller, err:%v.\n", err)
 			return
 		}
 	}
 
+	// Initialize multi-tenant controller if the CNS is running in MultiTenantCRD mode.
+	// It must be started before we start HTTPRestService.
+	if config.ChannelMode == cns.MultiTenantCRD {
+		multiTenantControllerStopChannel := make(chan struct{})
+		defer close(multiTenantControllerStopChannel)
+		err = InitializeMultiTenantController(httpRestService, cnsconfig, multiTenantControllerStopChannel)
+		if err != nil {
+			logger.Errorf("Failed to start multiTenantController, err:%v.\n", err)
+			return
+		}
+	}
+
 	logger.Printf("[Azure CNS] Start HTTP listener")
 	if httpRestService != nil {
 		err = httpRestService.Start(&config)
@@ -670,8 +686,74 @@ func main() {
 	logger.Close()
 }
 
+func InitializeMultiTenantController(httpRestService cns.HTTPService, cnsconfig configuration.CNSConfig, exitChan <-chan struct{}) error {
+	var multiTenantController multitenantcontroller.MultiTenantController
+	kubeConfig, err := ctrl.GetConfig()
+	if err != nil {
+		return err
+	}
+
+	//convert interface type to implementation type
+	httpRestServiceImpl, ok := httpRestService.(*restserver.HTTPRestService)
+	if !ok {
+		logger.Errorf("Failed to convert interface httpRestService to implementation: %v", httpRestService)
+		return fmt.Errorf("Failed to convert interface httpRestService to implementation: %v",
+			httpRestService)
+	}
+
+	// Set orchestrator type
+	orchestrator := cns.SetOrchestratorTypeRequest{
+		OrchestratorType: cns.KubernetesCRD,
+	}
+	httpRestServiceImpl.SetNodeOrchestrator(&orchestrator)
+
+	// Create multiTenantController.
+	multiTenantController, err = multitenantoperator.NewMultiTenantController(httpRestServiceImpl, kubeConfig)
+	if err != nil {
+		logger.Errorf("Failed to create multiTenantController:%v", err)
+		return err
+	}
+
+	// Wait for multiTenantController to start.
+	go func() {
+		for {
+			if err := multiTenantController.StartMultiTenantController(exitChan); err != nil {
+				logger.Errorf("Failed to start multiTenantController: %v", err)
+			} else {
+				logger.Printf("Exiting multiTenantController")
+				return
+			}
+
+			// Retry after 1sec
+			time.Sleep(time.Second)
+		}
+	}()
+	for {
+		if multiTenantController.IsStarted() {
+			logger.Printf("MultiTenantController is started")
+			break
+		}
+
+		logger.Printf("Waiting for multiTenantController to start...")
+		time.Sleep(time.Millisecond * 500)
+	}
+
+	// TODO: do we need this to be running?
+	logger.Printf("Starting SyncHostNCVersion")
+	rootCxt := context.Background()
+	go func() {
+		// Periodically poll vfp programmed NC version from NMAgent
+		for {
+			<-time.NewTicker(cnsconfig.SyncHostNCVersionIntervalMs * time.Millisecond).C
+			httpRestServiceImpl.SyncHostNCVersion(rootCxt, cnsconfig.ChannelMode, cnsconfig.SyncHostNCTimeoutMs)
+		}
+	}()
+
+	return nil
+}
+
 // initializeCRD state
-func IniitalizeCRDState(httpRestService cns.HTTPService, cnsconfig configuration.CNSConfig, exitChan <-chan struct{}) error {
+func InitializeCRDState(httpRestService cns.HTTPService, cnsconfig configuration.CNSConfig, exitChan <-chan struct{}) error {
 	var requestController requestcontroller.RequestController
 
 	logger.Printf("[Azure CNS] Starting request controller")

go.mod

@@ -12,6 +12,7 @@ require (
 	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/docker/libnetwork v0.8.0-dev.2.0.20210525090646-64b7a4574d14
 	github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9 // indirect
+	github.com/golang/mock v1.2.0
 	github.com/google/uuid v1.1.1
 	github.com/gorilla/mux v1.8.0
 	github.com/hashicorp/golang-lru v0.5.3 // indirect

go.sum

@@ -163,6 +163,7 @@ github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4er
 github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9 h1:uHTyIjqVhYRhLbJ8nIiOJHkEZZ+5YoOsAbD3sk82NiE=
 github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.2.0 h1:28o5sBqPkBsMGnC6b4MvE2TzSr5/AT4c/1fLqVGIwlk=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -401,6 +402,7 @@ golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTk
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -578,6 +580,7 @@ k8s.io/client-go v0.18.2/go.mod h1:Xcm5wVGXX9HAA2JJ2sSBUn3tCJ+4SVlCbl2MNNv+CIU=
 k8s.io/client-go v0.18.19 h1:ym6jwLYcdWFKrIm0tU4Ct6evujnA8/OQTVdwLKJp5rY=
 k8s.io/client-go v0.18.19/go.mod h1:lB+d4UqdzSjaU41VODLYm/oon3o05LAzsVpm6Me5XkY=
 k8s.io/code-generator v0.18.2/go.mod h1:+UHX5rSbxmR8kzS+FAv7um6dtYrZokQvjHpDSYRVkTc=
+k8s.io/code-generator v0.18.2/go.mod h1:+UHX5rSbxmR8kzS+FAv7um6dtYrZokQvjHpDSYRVkTc=
 k8s.io/component-base v0.18.2/go.mod h1:kqLlMuhJNHQ9lz8Z7V5bxUUtjFZnrypArGl58gmDfUM=
 k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
 k8s.io/gengo v0.0.0-20200114144118-36b2048a9120/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=

vendor/github.com/golang/mock/AUTHORS

@@ -0,0 +1,12 @@
+# This is the official list of GoMock authors for copyright purposes.
+# This file is distinct from the CONTRIBUTORS files.
+# See the latter for an explanation.
+
+# Names should be added to this file as
+#	Name or Organization <email address>
+# The email address is not required for organizations.
+
+# Please keep the list sorted.
+
+Alex Reece <awreece@gmail.com>
+Google Inc.

vendor/github.com/golang/mock/CONTRIBUTORS

@@ -0,0 +1,37 @@
+# This is the official list of people who can contribute (and typically
+# have contributed) code to the gomock repository.
+# The AUTHORS file lists the copyright holders; this file
+# lists people.  For example, Google employees are listed here
+# but not in AUTHORS, because Google holds the copyright.
+#
+# The submission process automatically checks to make sure
+# that people submitting code are listed in this file (by email address).
+#
+# Names should be added to this file only after verifying that
+# the individual or the individual's organization has agreed to
+# the appropriate Contributor License Agreement, found here:
+#
+#     http://code.google.com/legal/individual-cla-v1.0.html
+#     http://code.google.com/legal/corporate-cla-v1.0.html
+#
+# The agreement for individuals can be filled out on the web.
+#
+# When adding J Random Contributor's name to this file,
+# either J's name or J's organization's name should be
+# added to the AUTHORS file, depending on whether the
+# individual or corporate CLA was used.
+
+# Names should be added to this file like so:
+#     Name <email address>
+#
+# An entry with two email addresses specifies that the
+# first address should be used in the submit logs and
+# that the second address should be recognized as the
+# same person when interacting with Rietveld.
+
+# Please keep the list sorted.
+
+Aaron Jacobs <jacobsa@google.com> <aaronjjacobs@gmail.com>
+Alex Reece <awreece@gmail.com>
+David Symonds <dsymonds@golang.org>
+Ryan Barrett <ryanb@google.com>

vendor/github.com/golang/mock/LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

vendor/github.com/golang/mock/gomock/call.go

@@ -0,0 +1,420 @@
+// Copyright 2010 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package gomock
+
+import (
+	"fmt"
+	"reflect"
+	"strconv"
+	"strings"
+)
+
+// Call represents an expected call to a mock.
+type Call struct {
+	t TestHelper // for triggering test failures on invalid call setup
+
+	receiver   interface{}  // the receiver of the method call
+	method     string       // the name of the method
+	methodType reflect.Type // the type of the method
+	args       []Matcher    // the args
+	origin     string       // file and line number of call setup
+
+	preReqs []*Call // prerequisite calls
+
+	// Expectations
+	minCalls, maxCalls int
+
+	numCalls int // actual number made
+
+	// actions are called when this Call is called. Each action gets the args and
+	// can set the return values by returning a non-nil slice. Actions run in the
+	// order they are created.
+	actions []func([]interface{}) []interface{}
+}
+
+// newCall creates a *Call. It requires the method type in order to support
+// unexported methods.
+func newCall(t TestHelper, receiver interface{}, method string, methodType reflect.Type, args ...interface{}) *Call {
+	t.Helper()
+
+	// TODO: check arity, types.
+	margs := make([]Matcher, len(args))
+	for i, arg := range args {
+		if m, ok := arg.(Matcher); ok {
+			margs[i] = m
+		} else if arg == nil {
+			// Handle nil specially so that passing a nil interface value
+			// will match the typed nils of concrete args.
+			margs[i] = Nil()
+		} else {
+			margs[i] = Eq(arg)
+		}
+	}
+
+	origin := callerInfo(3)
+	actions := []func([]interface{}) []interface{}{func([]interface{}) []interface{} {
+		// Synthesize the zero value for each of the return args' types.
+		rets := make([]interface{}, methodType.NumOut())
+		for i := 0; i < methodType.NumOut(); i++ {
+			rets[i] = reflect.Zero(methodType.Out(i)).Interface()
+		}
+		return rets
+	}}
+	return &Call{t: t, receiver: receiver, method: method, methodType: methodType,
+		args: margs, origin: origin, minCalls: 1, maxCalls: 1, actions: actions}
+}
+
+// AnyTimes allows the expectation to be called 0 or more times
+func (c *Call) AnyTimes() *Call {
+	c.minCalls, c.maxCalls = 0, 1e8 // close enough to infinity
+	return c
+}
+
+// MinTimes requires the call to occur at least n times. If AnyTimes or MaxTimes have not been called, MinTimes also
+// sets the maximum number of calls to infinity.
+func (c *Call) MinTimes(n int) *Call {
+	c.minCalls = n
+	if c.maxCalls == 1 {
+		c.maxCalls = 1e8
+	}
+	return c
+}
+
+// MaxTimes limits the number of calls to n times. If AnyTimes or MinTimes have not been called, MaxTimes also
+// sets the minimum number of calls to 0.
+func (c *Call) MaxTimes(n int) *Call {
+	c.maxCalls = n
+	if c.minCalls == 1 {
+		c.minCalls = 0
+	}
+	return c
+}
+
+// DoAndReturn declares the action to run when the call is matched.
+// The return values from this function are returned by the mocked function.
+// It takes an interface{} argument to support n-arity functions.
+func (c *Call) DoAndReturn(f interface{}) *Call {
+	// TODO: Check arity and types here, rather than dying badly elsewhere.
+	v := reflect.ValueOf(f)
+
+	c.addAction(func(args []interface{}) []interface{} {
+		vargs := make([]reflect.Value, len(args))
+		ft := v.Type()
+		for i := 0; i < len(args); i++ {
+			if args[i] != nil {
+				vargs[i] = reflect.ValueOf(args[i])
+			} else {
+				// Use the zero value for the arg.
+				vargs[i] = reflect.Zero(ft.In(i))
+			}
+		}
+		vrets := v.Call(vargs)
+		rets := make([]interface{}, len(vrets))
+		for i, ret := range vrets {
+			rets[i] = ret.Interface()
+		}
+		return rets
+	})
+	return c
+}
+
+// Do declares the action to run when the call is matched. The function's
+// return values are ignored to retain backward compatibility. To use the
+// return values call DoAndReturn.
+// It takes an interface{} argument to support n-arity functions.
+func (c *Call) Do(f interface{}) *Call {
+	// TODO: Check arity and types here, rather than dying badly elsewhere.
+	v := reflect.ValueOf(f)
+
+	c.addAction(func(args []interface{}) []interface{} {
+		vargs := make([]reflect.Value, len(args))
+		ft := v.Type()
+		for i := 0; i < len(args); i++ {
+			if args[i] != nil {
+				vargs[i] = reflect.ValueOf(args[i])
+			} else {
+				// Use the zero value for the arg.
+				vargs[i] = reflect.Zero(ft.In(i))
+			}
+		}
+		v.Call(vargs)
+		return nil
+	})
+	return c
+}
+
+// Return declares the values to be returned by the mocked function call.
+func (c *Call) Return(rets ...interface{}) *Call {
+	c.t.Helper()
+
+	mt := c.methodType
+	if len(rets) != mt.NumOut() {
+		c.t.Fatalf("wrong number of arguments to Return for %T.%v: got %d, want %d [%s]",
+			c.receiver, c.method, len(rets), mt.NumOut(), c.origin)
+	}
+	for i, ret := range rets {
+		if got, want := reflect.TypeOf(ret), mt.Out(i); got == want {
+			// Identical types; nothing to do.
+		} else if got == nil {
+			// Nil needs special handling.
+			switch want.Kind() {
+			case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Ptr, reflect.Slice:
+				// ok
+			default:
+				c.t.Fatalf("argument %d to Return for %T.%v is nil, but %v is not nillable [%s]",
+					i, c.receiver, c.method, want, c.origin)
+			}
+		} else if got.AssignableTo(want) {
+			// Assignable type relation. Make the assignment now so that the generated code
+			// can return the values with a type assertion.
+			v := reflect.New(want).Elem()
+			v.Set(reflect.ValueOf(ret))
+			rets[i] = v.Interface()
+		} else {
+			c.t.Fatalf("wrong type of argument %d to Return for %T.%v: %v is not assignable to %v [%s]",
+				i, c.receiver, c.method, got, want, c.origin)
+		}
+	}
+
+	c.addAction(func([]interface{}) []interface{} {
+		return rets
+	})
+
+	return c
+}
+
+// Times declares the exact number of times a function call is expected to be executed.
+func (c *Call) Times(n int) *Call {
+	c.minCalls, c.maxCalls = n, n
+	return c
+}
+
+// SetArg declares an action that will set the nth argument's value,
+// indirected through a pointer. Or, in the case of a slice, SetArg
+// will copy value's elements into the nth argument.
+func (c *Call) SetArg(n int, value interface{}) *Call {
+	c.t.Helper()
+
+	mt := c.methodType
+	// TODO: This will break on variadic methods.
+	// We will need to check those at invocation time.
+	if n < 0 || n >= mt.NumIn() {
+		c.t.Fatalf("SetArg(%d, ...) called for a method with %d args [%s]",
+			n, mt.NumIn(), c.origin)
+	}
+	// Permit setting argument through an interface.
+	// In the interface case, we don't (nay, can't) check the type here.
+	at := mt.In(n)
+	switch at.Kind() {
+	case reflect.Ptr:
+		dt := at.Elem()
+		if vt := reflect.TypeOf(value); !vt.AssignableTo(dt) {
+			c.t.Fatalf("SetArg(%d, ...) argument is a %v, not assignable to %v [%s]",
+				n, vt, dt, c.origin)
+		}
+	case reflect.Interface:
+		// nothing to do
+	case reflect.Slice:
+		// nothing to do
+	default:
+		c.t.Fatalf("SetArg(%d, ...) referring to argument of non-pointer non-interface non-slice type %v [%s]",
+			n, at, c.origin)
+	}
+
+	c.addAction(func(args []interface{}) []interface{} {
+		v := reflect.ValueOf(value)
+		switch reflect.TypeOf(args[n]).Kind() {
+		case reflect.Slice:
+			setSlice(args[n], v)
+		default:
+			reflect.ValueOf(args[n]).Elem().Set(v)
+		}
+		return nil
+	})
+	return c
+}
+
+// isPreReq returns true if other is a direct or indirect prerequisite to c.
+func (c *Call) isPreReq(other *Call) bool {
+	for _, preReq := range c.preReqs {
+		if other == preReq || preReq.isPreReq(other) {
+			return true
+		}
+	}
+	return false
+}
+
+// After declares that the call may only match after preReq has been exhausted.
+func (c *Call) After(preReq *Call) *Call {
+	c.t.Helper()
+
+	if c == preReq {
+		c.t.Fatalf("A call isn't allowed to be its own prerequisite")
+	}
+	if preReq.isPreReq(c) {
+		c.t.Fatalf("Loop in call order: %v is a prerequisite to %v (possibly indirectly).", c, preReq)
+	}
+
+	c.preReqs = append(c.preReqs, preReq)
+	return c
+}
+
+// Returns true if the minimum number of calls have been made.
+func (c *Call) satisfied() bool {
+	return c.numCalls >= c.minCalls
+}
+
+// Returns true iff the maximum number of calls have been made.
+func (c *Call) exhausted() bool {
+	return c.numCalls >= c.maxCalls
+}
+
+func (c *Call) String() string {
+	args := make([]string, len(c.args))
+	for i, arg := range c.args {
+		args[i] = arg.String()
+	}
+	arguments := strings.Join(args, ", ")
+	return fmt.Sprintf("%T.%v(%s) %s", c.receiver, c.method, arguments, c.origin)
+}
+
+// Tests if the given call matches the expected call.
+// If yes, returns nil. If no, returns error with message explaining why it does not match.
+func (c *Call) matches(args []interface{}) error {
+	if !c.methodType.IsVariadic() {
+		if len(args) != len(c.args) {
+			return fmt.Errorf("Expected call at %s has the wrong number of arguments. Got: %d, want: %d",
+				c.origin, len(args), len(c.args))
+		}
+
+		for i, m := range c.args {
+			if !m.Matches(args[i]) {
+				return fmt.Errorf("Expected call at %s doesn't match the argument at index %s.\nGot: %v\nWant: %v",
+					c.origin, strconv.Itoa(i), args[i], m)
+			}
+		}
+	} else {
+		if len(c.args) < c.methodType.NumIn()-1 {
+			return fmt.Errorf("Expected call at %s has the wrong number of matchers. Got: %d, want: %d",
+				c.origin, len(c.args), c.methodType.NumIn()-1)
+		}
+		if len(c.args) != c.methodType.NumIn() && len(args) != len(c.args) {
+			return fmt.Errorf("Expected call at %s has the wrong number of arguments. Got: %d, want: %d",
+				c.origin, len(args), len(c.args))
+		}
+		if len(args) < len(c.args)-1 {
+			return fmt.Errorf("Expected call at %s has the wrong number of arguments. Got: %d, want: greater than or equal to %d",
+				c.origin, len(args), len(c.args)-1)
+		}
+
+		for i, m := range c.args {
+			if i < c.methodType.NumIn()-1 {
+				// Non-variadic args
+				if !m.Matches(args[i]) {
+					return fmt.Errorf("Expected call at %s doesn't match the argument at index %s.\nGot: %v\nWant: %v",
+						c.origin, strconv.Itoa(i), args[i], m)
+				}
+				continue
+			}
+			// The last arg has a possibility of a variadic argument, so let it branch
+
+			// sample: Foo(a int, b int, c ...int)
+			if i < len(c.args) && i < len(args) {
+				if m.Matches(args[i]) {
+					// Got Foo(a, b, c) want Foo(matcherA, matcherB, gomock.Any())
+					// Got Foo(a, b, c) want Foo(matcherA, matcherB, someSliceMatcher)
+					// Got Foo(a, b, c) want Foo(matcherA, matcherB, matcherC)
+					// Got Foo(a, b) want Foo(matcherA, matcherB)
+					// Got Foo(a, b, c, d) want Foo(matcherA, matcherB, matcherC, matcherD)
+					continue
+				}
+			}
+
+			// The number of actual args don't match the number of matchers,
+			// or the last matcher is a slice and the last arg is not.
+			// If this function still matches it is because the last matcher
+			// matches all the remaining arguments or the lack of any.
+			// Convert the remaining arguments, if any, into a slice of the
+			// expected type.
+			vargsType := c.methodType.In(c.methodType.NumIn() - 1)
+			vargs := reflect.MakeSlice(vargsType, 0, len(args)-i)
+			for _, arg := range args[i:] {
+				vargs = reflect.Append(vargs, reflect.ValueOf(arg))
+			}
+			if m.Matches(vargs.Interface()) {
+				// Got Foo(a, b, c, d, e) want Foo(matcherA, matcherB, gomock.Any())
+				// Got Foo(a, b, c, d, e) want Foo(matcherA, matcherB, someSliceMatcher)
+				// Got Foo(a, b) want Foo(matcherA, matcherB, gomock.Any())
+				// Got Foo(a, b) want Foo(matcherA, matcherB, someEmptySliceMatcher)
+				break
+			}
+			// Wrong number of matchers or not match. Fail.
+			// Got Foo(a, b) want Foo(matcherA, matcherB, matcherC, matcherD)
+			// Got Foo(a, b, c) want Foo(matcherA, matcherB, matcherC, matcherD)
+			// Got Foo(a, b, c, d) want Foo(matcherA, matcherB, matcherC, matcherD, matcherE)
+			// Got Foo(a, b, c, d, e) want Foo(matcherA, matcherB, matcherC, matcherD)
+			// Got Foo(a, b, c) want Foo(matcherA, matcherB)
+			return fmt.Errorf("Expected call at %s doesn't match the argument at index %s.\nGot: %v\nWant: %v",
+				c.origin, strconv.Itoa(i), args[i:], c.args[i])
+
+		}
+	}
+
+	// Check that all prerequisite calls have been satisfied.
+	for _, preReqCall := range c.preReqs {
+		if !preReqCall.satisfied() {
+			return fmt.Errorf("Expected call at %s doesn't have a prerequisite call satisfied:\n%v\nshould be called before:\n%v",
+				c.origin, preReqCall, c)
+		}
+	}
+
+	// Check that the call is not exhausted.
+	if c.exhausted() {
+		return fmt.Errorf("Expected call at %s has already been called the max number of times.", c.origin)
+	}
+
+	return nil
+}
+
+// dropPrereqs tells the expected Call to not re-check prerequisite calls any
+// longer, and to return its current set.
+func (c *Call) dropPrereqs() (preReqs []*Call) {
+	preReqs = c.preReqs
+	c.preReqs = nil
+	return
+}
+
+func (c *Call) call(args []interface{}) []func([]interface{}) []interface{} {
+	c.numCalls++
+	return c.actions
+}
+
+// InOrder declares that the given calls should occur in order.
+func InOrder(calls ...*Call) {
+	for i := 1; i < len(calls); i++ {
+		calls[i].After(calls[i-1])
+	}
+}
+
+func setSlice(arg interface{}, v reflect.Value) {
+	va := reflect.ValueOf(arg)
+	for i := 0; i < v.Len(); i++ {
+		va.Index(i).Set(v.Index(i))
+	}
+}
+
+func (c *Call) addAction(action func([]interface{}) []interface{}) {
+	c.actions = append(c.actions, action)
+}

vendor/github.com/golang/mock/gomock/callset.go

@@ -0,0 +1,108 @@
+// Copyright 2011 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package gomock
+
+import (
+	"bytes"
+	"fmt"
+)
+
+// callSet represents a set of expected calls, indexed by receiver and method
+// name.
+type callSet struct {
+	// Calls that are still expected.
+	expected map[callSetKey][]*Call
+	// Calls that have been exhausted.
+	exhausted map[callSetKey][]*Call
+}
+
+// callSetKey is the key in the maps in callSet
+type callSetKey struct {
+	receiver interface{}
+	fname    string
+}
+
+func newCallSet() *callSet {
+	return &callSet{make(map[callSetKey][]*Call), make(map[callSetKey][]*Call)}
+}
+
+// Add adds a new expected call.
+func (cs callSet) Add(call *Call) {
+	key := callSetKey{call.receiver, call.method}
+	m := cs.expected
+	if call.exhausted() {
+		m = cs.exhausted
+	}
+	m[key] = append(m[key], call)
+}
+
+// Remove removes an expected call.
+func (cs callSet) Remove(call *Call) {
+	key := callSetKey{call.receiver, call.method}
+	calls := cs.expected[key]
+	for i, c := range calls {
+		if c == call {
+			// maintain order for remaining calls
+			cs.expected[key] = append(calls[:i], calls[i+1:]...)
+			cs.exhausted[key] = append(cs.exhausted[key], call)
+			break
+		}
+	}
+}
+
+// FindMatch searches for a matching call. Returns error with explanation message if no call matched.
+func (cs callSet) FindMatch(receiver interface{}, method string, args []interface{}) (*Call, error) {
+	key := callSetKey{receiver, method}
+
+	// Search through the expected calls.
+	expected := cs.expected[key]
+	var callsErrors bytes.Buffer
+	for _, call := range expected {
+		err := call.matches(args)
+		if err != nil {
+			fmt.Fprintf(&callsErrors, "\n%v", err)
+		} else {
+			return call, nil
+		}
+	}
+
+	// If we haven't found a match then search through the exhausted calls so we
+	// get useful error messages.
+	exhausted := cs.exhausted[key]
+	for _, call := range exhausted {
+		if err := call.matches(args); err != nil {
+			fmt.Fprintf(&callsErrors, "\n%v", err)
+		}
+	}
+
+	if len(expected)+len(exhausted) == 0 {
+		fmt.Fprintf(&callsErrors, "there are no expected calls of the method %q for that receiver", method)
+	}
+
+	return nil, fmt.Errorf(callsErrors.String())
+}
+
+// Failures returns the calls that are not satisfied.
+func (cs callSet) Failures() []*Call {
+	failures := make([]*Call, 0, len(cs.expected))
+	for _, calls := range cs.expected {
+		for _, call := range calls {
+			if !call.satisfied() {
+				failures = append(failures, call)
+			}
+		}
+	}
+	return failures
+}

vendor/github.com/golang/mock/gomock/controller.go

@@ -0,0 +1,235 @@
+// Copyright 2010 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// GoMock - a mock framework for Go.
+//
+// Standard usage:
+//   (1) Define an interface that you wish to mock.
+//         type MyInterface interface {
+//           SomeMethod(x int64, y string)
+//         }
+//   (2) Use mockgen to generate a mock from the interface.
+//   (3) Use the mock in a test:
+//         func TestMyThing(t *testing.T) {
+//           mockCtrl := gomock.NewController(t)
+//           defer mockCtrl.Finish()
+//
+//           mockObj := something.NewMockMyInterface(mockCtrl)
+//           mockObj.EXPECT().SomeMethod(4, "blah")
+//           // pass mockObj to a real object and play with it.
+//         }
+//
+// By default, expected calls are not enforced to run in any particular order.
+// Call order dependency can be enforced by use of InOrder and/or Call.After.
+// Call.After can create more varied call order dependencies, but InOrder is
+// often more convenient.
+//
+// The following examples create equivalent call order dependencies.
+//
+// Example of using Call.After to chain expected call order:
+//
+//     firstCall := mockObj.EXPECT().SomeMethod(1, "first")
+//     secondCall := mockObj.EXPECT().SomeMethod(2, "second").After(firstCall)
+//     mockObj.EXPECT().SomeMethod(3, "third").After(secondCall)
+//
+// Example of using InOrder to declare expected call order:
+//
+//     gomock.InOrder(
+//         mockObj.EXPECT().SomeMethod(1, "first"),
+//         mockObj.EXPECT().SomeMethod(2, "second"),
+//         mockObj.EXPECT().SomeMethod(3, "third"),
+//     )
+//
+// TODO:
+//	- Handle different argument/return types (e.g. ..., chan, map, interface).
+package gomock
+
+import (
+	"context"
+	"fmt"
+	"reflect"
+	"runtime"
+	"sync"
+)
+
+// A TestReporter is something that can be used to report test failures.
+// It is satisfied by the standard library's *testing.T.
+type TestReporter interface {
+	Errorf(format string, args ...interface{})
+	Fatalf(format string, args ...interface{})
+}
+
+// TestHelper is a TestReporter that has the Helper method.  It is satisfied
+// by the standard library's *testing.T.
+type TestHelper interface {
+	TestReporter
+	Helper()
+}
+
+// A Controller represents the top-level control of a mock ecosystem.
+// It defines the scope and lifetime of mock objects, as well as their expectations.
+// It is safe to call Controller's methods from multiple goroutines.
+type Controller struct {
+	// T should only be called within a generated mock. It is not intended to
+	// be used in user code and may be changed in future versions. T is the
+	// TestReporter passed in when creating the Controller via NewController.
+	// If the TestReporter does not implment a TestHelper it will be wrapped
+	// with a nopTestHelper.
+	T             TestHelper
+	mu            sync.Mutex
+	expectedCalls *callSet
+	finished      bool
+}
+
+func NewController(t TestReporter) *Controller {
+	h, ok := t.(TestHelper)
+	if !ok {
+		h = nopTestHelper{t}
+	}
+
+	return &Controller{
+		T:             h,
+		expectedCalls: newCallSet(),
+	}
+}
+
+type cancelReporter struct {
+	TestHelper
+	cancel func()
+}
+
+func (r *cancelReporter) Errorf(format string, args ...interface{}) {
+	r.TestHelper.Errorf(format, args...)
+}
+func (r *cancelReporter) Fatalf(format string, args ...interface{}) {
+	defer r.cancel()
+	r.TestHelper.Fatalf(format, args...)
+}
+
+// WithContext returns a new Controller and a Context, which is cancelled on any
+// fatal failure.
+func WithContext(ctx context.Context, t TestReporter) (*Controller, context.Context) {
+	h, ok := t.(TestHelper)
+	if !ok {
+		h = nopTestHelper{t}
+	}
+
+	ctx, cancel := context.WithCancel(ctx)
+	return NewController(&cancelReporter{h, cancel}), ctx
+}
+
+type nopTestHelper struct {
+	TestReporter
+}
+
+func (h nopTestHelper) Helper() {}
+
+func (ctrl *Controller) RecordCall(receiver interface{}, method string, args ...interface{}) *Call {
+	ctrl.T.Helper()
+
+	recv := reflect.ValueOf(receiver)
+	for i := 0; i < recv.Type().NumMethod(); i++ {
+		if recv.Type().Method(i).Name == method {
+			return ctrl.RecordCallWithMethodType(receiver, method, recv.Method(i).Type(), args...)
+		}
+	}
+	ctrl.T.Fatalf("gomock: failed finding method %s on %T", method, receiver)
+	panic("unreachable")
+}
+
+func (ctrl *Controller) RecordCallWithMethodType(receiver interface{}, method string, methodType reflect.Type, args ...interface{}) *Call {
+	ctrl.T.Helper()
+
+	call := newCall(ctrl.T, receiver, method, methodType, args...)
+
+	ctrl.mu.Lock()
+	defer ctrl.mu.Unlock()
+	ctrl.expectedCalls.Add(call)
+
+	return call
+}
+
+func (ctrl *Controller) Call(receiver interface{}, method string, args ...interface{}) []interface{} {
+	ctrl.T.Helper()
+
+	// Nest this code so we can use defer to make sure the lock is released.
+	actions := func() []func([]interface{}) []interface{} {
+		ctrl.T.Helper()
+		ctrl.mu.Lock()
+		defer ctrl.mu.Unlock()
+
+		expected, err := ctrl.expectedCalls.FindMatch(receiver, method, args)
+		if err != nil {
+			origin := callerInfo(2)
+			ctrl.T.Fatalf("Unexpected call to %T.%v(%v) at %s because: %s", receiver, method, args, origin, err)
+		}
+
+		// Two things happen here:
+		// * the matching call no longer needs to check prerequite calls,
+		// * and the prerequite calls are no longer expected, so remove them.
+		preReqCalls := expected.dropPrereqs()
+		for _, preReqCall := range preReqCalls {
+			ctrl.expectedCalls.Remove(preReqCall)
+		}
+
+		actions := expected.call(args)
+		if expected.exhausted() {
+			ctrl.expectedCalls.Remove(expected)
+		}
+		return actions
+	}()
+
+	var rets []interface{}
+	for _, action := range actions {
+		if r := action(args); r != nil {
+			rets = r
+		}
+	}
+
+	return rets
+}
+
+func (ctrl *Controller) Finish() {
+	ctrl.T.Helper()
+
+	ctrl.mu.Lock()
+	defer ctrl.mu.Unlock()
+
+	if ctrl.finished {
+		ctrl.T.Fatalf("Controller.Finish was called more than once. It has to be called exactly once.")
+	}
+	ctrl.finished = true
+
+	// If we're currently panicking, probably because this is a deferred call,
+	// pass through the panic.
+	if err := recover(); err != nil {
+		panic(err)
+	}
+
+	// Check that all remaining expected calls are satisfied.
+	failures := ctrl.expectedCalls.Failures()
+	for _, call := range failures {
+		ctrl.T.Errorf("missing call(s) to %v", call)
+	}
+	if len(failures) != 0 {
+		ctrl.T.Fatalf("aborting test due to missing call(s)")
+	}
+}
+
+func callerInfo(skip int) string {
+	if _, file, line, ok := runtime.Caller(skip + 1); ok {
+		return fmt.Sprintf("%s:%d", file, line)
+	}
+	return "unknown file"
+}

vendor/github.com/golang/mock/gomock/matchers.go

@@ -0,0 +1,122 @@
+// Copyright 2010 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package gomock
+
+import (
+	"fmt"
+	"reflect"
+)
+
+// A Matcher is a representation of a class of values.
+// It is used to represent the valid or expected arguments to a mocked method.
+type Matcher interface {
+	// Matches returns whether x is a match.
+	Matches(x interface{}) bool
+
+	// String describes what the matcher matches.
+	String() string
+}
+
+type anyMatcher struct{}
+
+func (anyMatcher) Matches(x interface{}) bool {
+	return true
+}
+
+func (anyMatcher) String() string {
+	return "is anything"
+}
+
+type eqMatcher struct {
+	x interface{}
+}
+
+func (e eqMatcher) Matches(x interface{}) bool {
+	return reflect.DeepEqual(e.x, x)
+}
+
+func (e eqMatcher) String() string {
+	return fmt.Sprintf("is equal to %v", e.x)
+}
+
+type nilMatcher struct{}
+
+func (nilMatcher) Matches(x interface{}) bool {
+	if x == nil {
+		return true
+	}
+
+	v := reflect.ValueOf(x)
+	switch v.Kind() {
+	case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map,
+		reflect.Ptr, reflect.Slice:
+		return v.IsNil()
+	}
+
+	return false
+}
+
+func (nilMatcher) String() string {
+	return "is nil"
+}
+
+type notMatcher struct {
+	m Matcher
+}
+
+func (n notMatcher) Matches(x interface{}) bool {
+	return !n.m.Matches(x)
+}
+
+func (n notMatcher) String() string {
+	// TODO: Improve this if we add a NotString method to the Matcher interface.
+	return "not(" + n.m.String() + ")"
+}
+
+type assignableToTypeOfMatcher struct {
+	targetType reflect.Type
+}
+
+func (m assignableToTypeOfMatcher) Matches(x interface{}) bool {
+	return reflect.TypeOf(x).AssignableTo(m.targetType)
+}
+
+func (m assignableToTypeOfMatcher) String() string {
+	return "is assignable to " + m.targetType.Name()
+}
+
+// Constructors
+func Any() Matcher             { return anyMatcher{} }
+func Eq(x interface{}) Matcher { return eqMatcher{x} }
+func Nil() Matcher             { return nilMatcher{} }
+func Not(x interface{}) Matcher {
+	if m, ok := x.(Matcher); ok {
+		return notMatcher{m}
+	}
+	return notMatcher{Eq(x)}
+}
+
+// AssignableToTypeOf is a Matcher that matches if the parameter to the mock
+// function is assignable to the type of the parameter to this function.
+//
+// Example usage:
+//
+// 		dbMock.EXPECT().
+// 			Insert(gomock.AssignableToTypeOf(&EmployeeRecord{})).
+// 			Return(errors.New("DB error"))
+//
+func AssignableToTypeOf(x interface{}) Matcher {
+	return assignableToTypeOfMatcher{reflect.TypeOf(x)}
+}

vendor/modules.txt

@@ -84,6 +84,9 @@ github.com/gogo/protobuf/sortkeys
 # github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9
 ## explicit
 github.com/golang/groupcache/lru
+# github.com/golang/mock v1.2.0
+## explicit
+github.com/golang/mock/gomock
 # github.com/golang/protobuf v1.4.1
 github.com/golang/protobuf/proto
 github.com/golang/protobuf/ptypes