Add CNS IPAM check to not overwrite existing state (#614)

* add check to cns to not overwrite state * use CreateOrUpdateNetworkContainer API instead of directly adding ipconfigs * rename test for clarity
2020-07-15 15:22:04 -07:00 · 2020-07-15 15:22:04 -07:00 · 1d31a7f1ef
--- a/cns/NetworkContainerContract.go
+++ b/cns/NetworkContainerContract.go
@ -69,6 +69,7 @@ type CreateNetworkContainerRequest struct {
 	LocalIPConfiguration       IPConfiguration
 	OrchestratorContext        json.RawMessage
 	IPConfiguration            IPConfiguration
+	SecondaryIPConfigs         map[string]ContainerIPConfigState //uuid is key
 	MultiTenancyInfo           MultiTenancyInfo
 	CnetAddressSpace           []IPSubnet // To setup SNAT (should include service endpoint vips).
 	Routes                     []Route
--- a/cns/cnsclient/cnsclient_test.go
+++ b/cns/cnsclient/cnsclient_test.go
@ -33,10 +33,14 @@ var (
 func addTestStateToRestServer(svc *restserver.HTTPRestService) {
 	// set state as already allocated
 	state1, _ := restserver.NewPodStateWithOrchestratorContext(testIP1, 24, testPod1GUID, testNCID, cns.Available, testPod1Info)
-	ipconfigs := []*cns.ContainerIPConfigState{
-		state1,
+	ipconfigs := map[string]cns.ContainerIPConfigState{
+		state1.ID: state1,
 	}
-	svc.AddIPConfigsToState(ipconfigs)
+	nc := cns.CreateNetworkContainerRequest{
+		SecondaryIPConfigs: ipconfigs,
+	}
+
+	svc.CreateOrUpdateNetworkContainerWithSecondaryIPConfigs(nc)
 }

 func getIPConfigFromGetNetworkContainerResponse(resp *cns.GetIPConfigResponse) (net.IPNet, error) {
--- a/cns/restserver/ipam.go
+++ b/cns/restserver/ipam.go
@ -19,10 +19,10 @@ func newIPConfig(ipAddress string, prefixLength uint8) cns.IPSubnet {
 	}
 }

-func NewPodState(ipaddress string, prefixLength uint8, id, ncid, state string) *cns.ContainerIPConfigState {
+func NewPodState(ipaddress string, prefixLength uint8, id, ncid, state string) cns.ContainerIPConfigState {
 	ipconfig := newIPConfig(ipaddress, prefixLength)

-	return &cns.ContainerIPConfigState{
+	return cns.ContainerIPConfigState{
 		IPConfig: ipconfig,
 		ID:       id,
 		NCID:     ncid,
@ -30,10 +30,10 @@ func NewPodState(ipaddress string, prefixLength uint8, id, ncid, state string) *
 	}
 }

-func NewPodStateWithOrchestratorContext(ipaddress string, prefixLength uint8, id, ncid, state string, orchestratorContext cns.KubernetesPodInfo) (*cns.ContainerIPConfigState, error) {
+func NewPodStateWithOrchestratorContext(ipaddress string, prefixLength uint8, id, ncid, state string, orchestratorContext cns.KubernetesPodInfo) (cns.ContainerIPConfigState, error) {
 	ipconfig := newIPConfig(ipaddress, prefixLength)
 	b, err := json.Marshal(orchestratorContext)
-	return &cns.ContainerIPConfigState{
+	return cns.ContainerIPConfigState{
 		IPConfig:            ipconfig,
 		ID:                  id,
 		NCID:                ncid,
@ -47,7 +47,7 @@ func (service *HTTPRestService) requestIPConfigHandler(w http.ResponseWriter, r
 	var (
 		err             error
 		ipconfigRequest cns.GetIPConfigRequest
-		ipState         *cns.ContainerIPConfigState
+		ipState         cns.ContainerIPConfigState
 		returnCode      int
 		returnMessage   string
 	)
@ -122,7 +122,7 @@ func (service *HTTPRestService) releaseIPConfigHandler(w http.ResponseWriter, r
 	return
 }

-func validateIPConfig(ipconfig *cns.ContainerIPConfigState) error {
+func validateIPConfig(ipconfig cns.ContainerIPConfigState) error {
 	if ipconfig.ID == "" {
 		return fmt.Errorf("Failed to add IPConfig to state: %+v, empty ID", ipconfig)
 	}
@ -138,33 +138,52 @@ func validateIPConfig(ipconfig *cns.ContainerIPConfigState) error {
 	return nil
 }

+func (service *HTTPRestService) CreateOrUpdateNetworkContainerWithSecondaryIPConfigs(nc cns.CreateNetworkContainerRequest) error {
+	return service.addIPConfigsToState(nc.SecondaryIPConfigs)
+}
+
 //AddIPConfigsToState takes a lock on the service object, and will add an array of ipconfigs to the CNS Service.
 //Used to add IPConfigs to the CNS pool, specifically in the scenario of rebatching.
-func (service *HTTPRestService) AddIPConfigsToState(ipconfigs []*cns.ContainerIPConfigState) error {
+func (service *HTTPRestService) addIPConfigsToState(ipconfigs map[string]cns.ContainerIPConfigState) error {
 	var (
 		err      error
-		index    int
-		ipconfig *cns.ContainerIPConfigState
+		ipconfig cns.ContainerIPConfigState
 	)

+	addedIPconfigs := make([]cns.ContainerIPConfigState, 0)
+
 	service.Lock()

 	defer func() {
 		service.Unlock()

 		if err != nil {
-			if removeErr := service.RemoveIPConfigsFromState(ipconfigs[0:index]); removeErr != nil {
+			if removeErr := service.removeIPConfigsFromState(addedIPconfigs); removeErr != nil {
 				logger.Printf("Failed remove IPConfig after AddIpConfigs: %v", removeErr)
 			}
 		}
 	}()

-	for index, ipconfig = range ipconfigs {
+	// ensure the ipconfigs we are not attempting to overwrite existing ipconfig state
+	existingIPConfigs := filterIPConfigMap(ipconfigs, func(ipconfig *cns.ContainerIPConfigState) bool {
+		existingIPConfig, exists := service.PodIPConfigState[ipconfig.ID]
+		if exists && existingIPConfig.State != ipconfig.State {
+			return true
+		}
+		return false
+	})
+	if len(existingIPConfigs) > 0 {
+		return fmt.Errorf("Failed to add IPConfigs to state, attempting to overwrite existing ipconfig states: %v", existingIPConfigs)
+	}
+
+	// add ipconfigs to state
+	for _, ipconfig = range ipconfigs {
 		if err = validateIPConfig(ipconfig); err != nil {
 			return err
 		}

 		service.PodIPConfigState[ipconfig.ID] = ipconfig
+		addedIPconfigs = append(addedIPconfigs, ipconfig)

 		if ipconfig.State == cns.Allocated {
 			var podInfo cns.KubernetesPodInfo
@ -179,9 +198,35 @@ func (service *HTTPRestService) AddIPConfigsToState(ipconfigs []*cns.ContainerIP
 	return err
 }

+func filterIPConfigMap(toBeAdded map[string]cns.ContainerIPConfigState, f func(*cns.ContainerIPConfigState) bool) []*cns.ContainerIPConfigState {
+	vsf := make([]*cns.ContainerIPConfigState, 0)
+	for _, v := range toBeAdded {
+		if f(&v) {
+			vsf = append(vsf, &v)
+		}
+	}
+	return vsf
+}
+
+func (service *HTTPRestService) GetAllocatedIPConfigs() []*cns.ContainerIPConfigState {
+	service.RLock()
+	defer service.RUnlock()
+	return filterIPConfigMap(service.PodIPConfigState, func(ipconfig *cns.ContainerIPConfigState) bool {
+		return ipconfig.State == cns.Allocated
+	})
+}
+
+func (service *HTTPRestService) GetAvailableIPConfigs() []*cns.ContainerIPConfigState {
+	service.RLock()
+	defer service.RUnlock()
+	return filterIPConfigMap(service.PodIPConfigState, func(ipconfig *cns.ContainerIPConfigState) bool {
+		return ipconfig.State == cns.Available
+	})
+}
+
 //RemoveIPConfigsFromState takes a lock on the service object, and will remove an array of ipconfigs to the CNS Service.
 //Used to add IPConfigs to the CNS pool, specifically in the scenario of rebatching.
-func (service *HTTPRestService) RemoveIPConfigsFromState(ipconfigs []*cns.ContainerIPConfigState) error {
+func (service *HTTPRestService) removeIPConfigsFromState(ipconfigs []cns.ContainerIPConfigState) error {
 	service.Lock()
 	defer service.Unlock()

@ -201,7 +246,7 @@ func (service *HTTPRestService) RemoveIPConfigsFromState(ipconfigs []*cns.Contai
 }

 //SetIPConfigAsAllocated takes a lock of the service, and sets the ipconfig in the CNS state as allocated, does not take a lock
-func (service *HTTPRestService) setIPConfigAsAllocated(ipconfig *cns.ContainerIPConfigState, podInfo cns.KubernetesPodInfo, marshalledOrchestratorContext json.RawMessage) *cns.ContainerIPConfigState {
+func (service *HTTPRestService) setIPConfigAsAllocated(ipconfig cns.ContainerIPConfigState, podInfo cns.KubernetesPodInfo, marshalledOrchestratorContext json.RawMessage) cns.ContainerIPConfigState {
 	ipconfig.State = cns.Allocated
 	ipconfig.OrchestratorContext = marshalledOrchestratorContext
 	service.PodIPIDByOrchestratorContext[podInfo.GetOrchestratorContextKey()] = ipconfig.ID
@ -210,7 +255,7 @@ func (service *HTTPRestService) setIPConfigAsAllocated(ipconfig *cns.ContainerIP
 }

 //SetIPConfigAsAllocated and sets the ipconfig in the CNS state as allocated, does not take a lock
-func (service *HTTPRestService) setIPConfigAsAvailable(ipconfig *cns.ContainerIPConfigState, podInfo cns.KubernetesPodInfo) *cns.ContainerIPConfigState {
+func (service *HTTPRestService) setIPConfigAsAvailable(ipconfig cns.ContainerIPConfigState, podInfo cns.KubernetesPodInfo) cns.ContainerIPConfigState {
 	ipconfig.State = cns.Available
 	ipconfig.OrchestratorContext = nil
 	service.PodIPConfigState[ipconfig.ID] = ipconfig
@ -238,9 +283,9 @@ func (service *HTTPRestService) ReleaseIPConfig(podInfo cns.KubernetesPodInfo) e
 	return nil
 }

-func (service *HTTPRestService) GetExistingIPConfig(podInfo cns.KubernetesPodInfo) (*cns.ContainerIPConfigState, bool, error) {
+func (service *HTTPRestService) GetExistingIPConfig(podInfo cns.KubernetesPodInfo) (cns.ContainerIPConfigState, bool, error) {
 	var (
-		ipState *cns.ContainerIPConfigState
+		ipState cns.ContainerIPConfigState
 		isExist bool
 	)

@ -259,8 +304,8 @@ func (service *HTTPRestService) GetExistingIPConfig(podInfo cns.KubernetesPodInf
 	return ipState, isExist, nil
 }

-func (service *HTTPRestService) AllocateDesiredIPConfig(podInfo cns.KubernetesPodInfo, desiredIPAddress string, orchestratorContext json.RawMessage) (*cns.ContainerIPConfigState, error) {
-	var ipState *cns.ContainerIPConfigState
+func (service *HTTPRestService) AllocateDesiredIPConfig(podInfo cns.KubernetesPodInfo, desiredIPAddress string, orchestratorContext json.RawMessage) (cns.ContainerIPConfigState, error) {
+	var ipState cns.ContainerIPConfigState

 	service.Lock()
 	defer service.Unlock()
@ -276,8 +321,8 @@ func (service *HTTPRestService) AllocateDesiredIPConfig(podInfo cns.KubernetesPo
 	return ipState, fmt.Errorf("Requested IP not found in pool")
 }

-func (service *HTTPRestService) AllocateAnyAvailableIPConfig(podInfo cns.KubernetesPodInfo, orchestratorContext json.RawMessage) (*cns.ContainerIPConfigState, error) {
-	var ipState *cns.ContainerIPConfigState
+func (service *HTTPRestService) AllocateAnyAvailableIPConfig(podInfo cns.KubernetesPodInfo, orchestratorContext json.RawMessage) (cns.ContainerIPConfigState, error) {
+	var ipState cns.ContainerIPConfigState

 	service.Lock()
 	defer service.Unlock()
@ -291,10 +336,10 @@ func (service *HTTPRestService) AllocateAnyAvailableIPConfig(podInfo cns.Kuberne
 }

 // If IPConfig is already allocated for pod, it returns that else it returns one of the available ipconfigs.
-func requestIPConfigHelper(service *HTTPRestService, req cns.GetIPConfigRequest) (*cns.ContainerIPConfigState, error) {
+func requestIPConfigHelper(service *HTTPRestService, req cns.GetIPConfigRequest) (cns.ContainerIPConfigState, error) {
 	var (
 		podInfo cns.KubernetesPodInfo
-		ipState *cns.ContainerIPConfigState
+		ipState cns.ContainerIPConfigState
 		isExist bool
 		err     error
 	)
--- a/cns/restserver/ipam_test.go
+++ b/cns/restserver/ipam_test.go
@ -50,10 +50,10 @@ func TestIPAMGetAvailableIPConfig(t *testing.T) {
 	svc := getTestService()

 	testState := NewPodState(testIP1, 24, testPod1GUID, testNCID, cns.Available)
-	ipconfigs := []*cns.ContainerIPConfigState{
-		testState,
+	ipconfigs := map[string]cns.ContainerIPConfigState{
+		testState.ID: testState,
 	}
-	svc.AddIPConfigsToState(ipconfigs)
+	svc.addIPConfigsToState(ipconfigs)

 	req := cns.GetIPConfigRequest{}
 	b, _ := json.Marshal(testPod1Info)
@ -81,11 +81,11 @@ func TestIPAMGetNextAvailableIPConfig(t *testing.T) {
 	state1, _ := NewPodStateWithOrchestratorContext(testIP1, 24, testPod1GUID, testNCID, cns.Allocated, testPod1Info)
 	state2 := NewPodState(testIP2, 24, testPod2GUID, testNCID, cns.Available)

-	ipconfigs := []*cns.ContainerIPConfigState{
-		state1,
-		state2,
+	ipconfigs := map[string]cns.ContainerIPConfigState{
+		state1.ID: state1,
+		state2.ID: state2,
 	}
-	err := svc.AddIPConfigsToState(ipconfigs)
+	err := svc.addIPConfigsToState(ipconfigs)
 	if err != nil {
 		t.Fatalf("Expected to not fail adding IP's to state: %+v", err)
 	}
@ -111,10 +111,10 @@ func TestIPAMGetAlreadyAllocatedIPConfigForSamePod(t *testing.T) {

 	// Add Allocated Pod IP to state
 	testState, _ := NewPodStateWithOrchestratorContext(testIP1, 24, testPod1GUID, testNCID, cns.Allocated, testPod1Info)
-	ipconfigs := []*cns.ContainerIPConfigState{
-		testState,
+	ipconfigs := map[string]cns.ContainerIPConfigState{
+		testState.ID: testState,
 	}
-	err := svc.AddIPConfigsToState(ipconfigs)
+	err := svc.addIPConfigsToState(ipconfigs)
 	if err != nil {
 		t.Fatalf("Expected to not fail adding IP's to state: %+v", err)
 	}
@ -140,11 +140,11 @@ func TestIPAMAttemptToRequestIPNotFoundInPool(t *testing.T) {

 	// Add Available Pod IP to state
 	testState := NewPodState(testIP1, 24, testPod1GUID, testNCID, cns.Available)
-	ipconfigs := []*cns.ContainerIPConfigState{
-		testState,
+	ipconfigs := map[string]cns.ContainerIPConfigState{
+		testState.ID: testState,
 	}

-	err := svc.AddIPConfigsToState(ipconfigs)
+	err := svc.addIPConfigsToState(ipconfigs)
 	if err != nil {
 		t.Fatalf("Expected to not fail adding IP's to state: %+v", err)
 	}
@ -165,11 +165,11 @@ func TestIPAMGetDesiredIPConfigWithSpecfiedIP(t *testing.T) {

 	// Add Available Pod IP to state
 	testState := NewPodState(testIP1, 24, testPod1GUID, testNCID, cns.Available)
-	ipconfigs := []*cns.ContainerIPConfigState{
-		testState,
+	ipconfigs := map[string]cns.ContainerIPConfigState{
+		testState.ID: testState,
 	}

-	err := svc.AddIPConfigsToState(ipconfigs)
+	err := svc.addIPConfigsToState(ipconfigs)
 	if err != nil {
 		t.Fatalf("Expected to not fail adding IP's to state: %+v", err)
 	}
@ -197,10 +197,10 @@ func TestIPAMFailToGetDesiredIPConfigWithAlreadyAllocatedSpecfiedIP(t *testing.T

 	// set state as already allocated
 	testState, _ := NewPodStateWithOrchestratorContext(testIP1, 24, testPod1GUID, testNCID, cns.Allocated, testPod1Info)
-	ipconfigs := []*cns.ContainerIPConfigState{
-		testState,
+	ipconfigs := map[string]cns.ContainerIPConfigState{
+		testState.ID: testState,
 	}
-	err := svc.AddIPConfigsToState(ipconfigs)
+	err := svc.addIPConfigsToState(ipconfigs)
 	if err != nil {
 		t.Fatalf("Expected to not fail adding IP's to state: %+v", err)
 	}
@ -224,11 +224,11 @@ func TestIPAMFailToGetIPWhenAllIPsAreAllocated(t *testing.T) {
 	state1, _ := NewPodStateWithOrchestratorContext(testIP1, 24, testPod1GUID, testNCID, cns.Allocated, testPod1Info)
 	state2, _ := NewPodStateWithOrchestratorContext(testIP2, 24, testPod2GUID, testNCID, cns.Allocated, testPod2Info)

-	ipconfigs := []*cns.ContainerIPConfigState{
-		state1,
-		state2,
+	ipconfigs := map[string]cns.ContainerIPConfigState{
+		state1.ID: state1,
+		state2.ID: state2,
 	}
-	err := svc.AddIPConfigsToState(ipconfigs)
+	err := svc.addIPConfigsToState(ipconfigs)
 	if err != nil {
 		t.Fatalf("Expected to not fail adding IP's to state: %+v", err)
 	}
@ -253,11 +253,11 @@ func TestIPAMRequestThenReleaseThenRequestAgain(t *testing.T) {

 	// set state as already allocated
 	state1, _ := NewPodStateWithOrchestratorContext(testIP1, 24, testPod1GUID, testNCID, cns.Allocated, testPod1Info)
-	ipconfigs := []*cns.ContainerIPConfigState{
-		state1,
+	ipconfigs := map[string]cns.ContainerIPConfigState{
+		state1.ID: state1,
 	}

-	err := svc.AddIPConfigsToState(ipconfigs)
+	err := svc.addIPConfigsToState(ipconfigs)
 	if err != nil {
 		t.Fatalf("Expected to not fail adding IP's to state: %+v", err)
 	}
@ -302,32 +302,94 @@ func TestIPAMRequestThenReleaseThenRequestAgain(t *testing.T) {
 	}
 }

-func TestIPAMFailToAddThenCleanThenRequestExpectFail(t *testing.T) {
+func TestIPAMExpectFailWhenAddingBadIPConfig(t *testing.T) {
 	svc := getTestService()

 	var err error

 	// set state as already allocated
 	state1, _ := NewPodStateWithOrchestratorContext(testIP1, 24, testPod1GUID, testNCID, cns.Available, testPod1Info)
-	state2, _ := NewPodStateWithOrchestratorContext("", 24, "", testNCID, cns.Available, testPod1Info)

-	ipconfigs := []*cns.ContainerIPConfigState{
-		state1,
-		state2,
+	ipconfigs := map[string]cns.ContainerIPConfigState{
+		state1.ID: state1,
 	}

-	err = svc.AddIPConfigsToState(ipconfigs)
+	err = svc.addIPConfigsToState(ipconfigs)
+	if err != nil {
+		t.Fatalf("Expected to not fail when good ipconfig is added")
+	}
+
+	// create bad ipconfig
+	state2, _ := NewPodStateWithOrchestratorContext("", 24, "", testNCID, cns.Available, testPod1Info)
+
+	ipconfigs2 := map[string]cns.ContainerIPConfigState{
+		state2.ID: state2,
+	}
+
+	// add a bad ipconfig
+	err = svc.addIPConfigsToState(ipconfigs2)
 	if err == nil {
 		t.Fatalf("Expected add to fail when bad ipconfig is added.")
 	}

-	req := cns.GetIPConfigRequest{}
-	b, _ := json.Marshal(testPod1Info)
-	req.OrchestratorContext = b
+	// ensure state remains untouched
+	if len(svc.PodIPConfigState) != 1 {
+		t.Fatalf("Expected bad ipconfig to not be added added.")
+	}
+}

-	_, err = requestIPConfigHelper(svc, req)
+func TestIPAMStateCleanUpWhenAddingGoodIPConfigWithBadOrchestratorContext(t *testing.T) {
+	svc := getTestService()
+
+	var err error
+
+	// add available state
+	state1, _ := NewPodStateWithOrchestratorContext(testIP1, 24, testPod1GUID, testNCID, cns.Available, testPod1Info)
+
+	ipconfigs := map[string]cns.ContainerIPConfigState{
+		state1.ID: state1,
+	}
+
+	err = svc.addIPConfigsToState(ipconfigs)
+	if err != nil {
+		t.Fatalf("Expected to not fail when good ipconfig is added")
+	}
+
+	// create a good ipconfig
+	state2, _ := NewPodStateWithOrchestratorContext(testIP2, 24, testPod2GUID, testNCID, cns.Allocated, testPod1Info)
+
+	// make it bad with a bad orchestratorcontext and add to good ipconfig
+	b, err := json.Marshal("badstring")
+	state2.OrchestratorContext = b
+
+	ipconfigs2 := map[string]cns.ContainerIPConfigState{
+		state2.ID: state2,
+	}
+
+	err = svc.addIPConfigsToState(ipconfigs2)
 	if err == nil {
-		t.Fatalf("Expected state to be clean when one ipconfig is bad in batch add.")
+		t.Fatalf("Expected add to fail when bad ipconfig is added.")
+	}
+
+	// ensure state remains untouched
+	if len(svc.PodIPConfigState) != 1 {
+		t.Fatalf("Expected bad ipconfig to not be added added.")
+	}
+
+	// ensure we can still get the available ipconfig
+	req := cns.GetIPConfigRequest{}
+	b, _ = json.Marshal(testPod1Info)
+	req.OrchestratorContext = b
+	actualstate, err := requestIPConfigHelper(svc, req)
+	if err != nil {
+		t.Fatalf("Expected IP retrieval to be nil: %v", err)
+	}
+
+	desiredState, _ := NewPodStateWithOrchestratorContext(testIP1, 24, testPod1GUID, testNCID, cns.Allocated, testPod1Info)
+	desiredState.OrchestratorContext = b
+
+	if reflect.DeepEqual(desiredState, actualstate) != true {
+		t.Fatalf("Desired state not matching actual state, expected: %+v, actual: %+v", desiredState, actualstate)
 	}
 }

@ -335,11 +397,11 @@ func TestIPAMReleaseIPIdempotency(t *testing.T) {
 	svc := getTestService()
 	// set state as already allocated
 	state1, _ := NewPodStateWithOrchestratorContext(testIP1, 24, testPod1GUID, testNCID, cns.Allocated, testPod1Info)
-	ipconfigs := []*cns.ContainerIPConfigState{
-		state1,
+	ipconfigs := map[string]cns.ContainerIPConfigState{
+		state1.ID: state1,
 	}

-	err := svc.AddIPConfigsToState(ipconfigs)
+	err := svc.addIPConfigsToState(ipconfigs)
 	if err != nil {
 		t.Fatalf("Expected to not fail adding IP's to state: %+v", err)
 	}
@ -361,17 +423,61 @@ func TestIPAMAllocateIPIdempotency(t *testing.T) {
 	svc := getTestService()
 	// set state as already allocated
 	state1, _ := NewPodStateWithOrchestratorContext(testIP1, 24, testPod1GUID, testNCID, cns.Allocated, testPod1Info)
-	ipconfigs := []*cns.ContainerIPConfigState{
-		state1,
+	ipconfigs := map[string]cns.ContainerIPConfigState{
+		state1.ID: state1,
 	}

-	err := svc.AddIPConfigsToState(ipconfigs)
+	err := svc.addIPConfigsToState(ipconfigs)
 	if err != nil {
 		t.Fatalf("Expected to not fail adding IP's to state: %+v", err)
 	}

-	err = svc.AddIPConfigsToState(ipconfigs)
+	err = svc.addIPConfigsToState(ipconfigs)
 	if err != nil {
 		t.Fatalf("Expected to not fail adding IP's to state: %+v", err)
 	}
 }
+
+func TestIPAMExpectStateToNotChangeWhenChangingAllocatedToAvailable(t *testing.T) {
+	svc := getTestService()
+	// add two ipconfigs, one as available, the other as allocated
+	state1, _ := NewPodStateWithOrchestratorContext(testIP1, 24, testPod1GUID, testNCID, cns.Available, testPod1Info)
+	state2, _ := NewPodStateWithOrchestratorContext(testIP2, 24, testPod2GUID, testNCID, cns.Allocated, testPod2Info)
+
+	ipconfigs := map[string]cns.ContainerIPConfigState{
+		state1.ID: state1,
+		state2.ID: state2,
+	}
+
+	err := svc.addIPConfigsToState(ipconfigs)
+	if err != nil {
+		t.Fatalf("Expected to not fail adding IP's to state: %+v", err)
+	}
+
+	// create state2 again, but as available
+	state2Available, _ := NewPodStateWithOrchestratorContext(testIP2, 24, testPod2GUID, testNCID, cns.Available, testPod2Info)
+
+	// add an available and allocated ipconfig
+	ipconfigsTest := map[string]cns.ContainerIPConfigState{
+		state1.ID: state1,
+		state2.ID: state2Available,
+	}
+
+	// expect to fail overwriting an allocated state with available
+	err = svc.addIPConfigsToState(ipconfigsTest)
+	if err == nil {
+		t.Fatalf("Expected to fail when overwriting an allocated state as available: %+v", err)
+	}
+
+	// get allocated ipconfigs, should only be one from the inital call, and not 2 from the failed call
+	availableIPconfigs := svc.GetAvailableIPConfigs()
+	if len(availableIPconfigs) != 1 {
+		t.Fatalf("More than expected available IP configs in state")
+	}
+
+	// get allocated ipconfigs, should only be one from the inital call, and not 0 from the failed call
+	allocatedIPconfigs := svc.GetAllocatedIPConfigs()
+	if len(allocatedIPconfigs) != 1 {
+		t.Fatalf("More than expected allocated IP configs in state")
+	}
+}
--- a/cns/restserver/restserver.go
+++ b/cns/restserver/restserver.go
@ -52,9 +52,9 @@ type HTTPRestService struct {
 	imdsClient                   *imdsclient.ImdsClient
 	ipamClient                   *ipamclient.IpamClient
 	networkContainer             *networkcontainers.NetworkContainers
-	PodIPIDByOrchestratorContext map[string]string                      // OrchestratorContext is key and value is Pod IP uuid.
-	PodIPConfigState             map[string]*cns.ContainerIPConfigState // seondaryipid(uuid) is key
-	AllocatedIPCount             map[string]allocatedIPCount            // key - ncid
+	PodIPIDByOrchestratorContext map[string]string                     // OrchestratorContext is key and value is Pod IP uuid.
+	PodIPConfigState             map[string]cns.ContainerIPConfigState // seondaryipid(uuid) is key
+	AllocatedIPCount             map[string]allocatedIPCount           // key - ncid
 	routingTable                 *routes.RoutingTable
 	store                        store.KeyValueStore
 	state                        *httpRestServiceState
@ -126,7 +126,7 @@ func NewHTTPRestService(config *common.ServiceConfig) (HTTPService, error) {
 	serviceState.joinedNetworks = make(map[string]struct{})

 	podIPIDByOrchestratorContext := make(map[string]string)
-	podIPConfigState := make(map[string]*cns.ContainerIPConfigState)
+	podIPConfigState := make(map[string]cns.ContainerIPConfigState)
 	allocatedIPCount := make(map[string]allocatedIPCount) // key - ncid

 	return &HTTPRestService{