Dualstack CNS Changes (#1773)
1. Enables CNS to handle multiple NCs in NNC. 2. Adds new APIs that allows multiple IPs to be requested and released. 3. This change is needed for dualstack overlay --------- Co-authored-by: Tim Raymond <traymond@microsoft.com>
This commit is contained in:
rjdenney
GitHub
Родитель
2538e573a8
Коммит
3e06a07ca2
|
|
@ -27,7 +27,9 @@ const (
|
|||
AttachContainerToNetwork = "/network/attachcontainertonetwork"
|
||||
DetachContainerFromNetwork = "/network/detachcontainerfromnetwork"
|
||||
RequestIPConfig = "/network/requestipconfig"
|
||||
RequestIPConfigs = "/network/requestipconfigs"
|
||||
ReleaseIPConfig = "/network/releaseipconfig"
|
||||
ReleaseIPConfigs = "/network/releaseipconfigs"
|
||||
PathDebugIPAddresses = "/debug/ipaddresses"
|
||||
PathDebugPodContext = "/debug/podcontext"
|
||||
PathDebugRestData = "/debug/restdata"
|
||||
|
|
@ -255,9 +257,9 @@ func UnmarshalPodInfo(b []byte) (PodInfo, error) {
|
|||
return p, nil
|
||||
}
|
||||
|
||||
// NewPodInfoFromIPConfigRequest builds and returns an implementation of
|
||||
// PodInfo from the provided IPConfigRequest.
|
||||
func NewPodInfoFromIPConfigRequest(req IPConfigRequest) (PodInfo, error) {
|
||||
// NewPodInfoFromIPConfigsRequest builds and returns an implementation of
|
||||
// PodInfo from the provided IPConfigsRequest.
|
||||
func NewPodInfoFromIPConfigsRequest(req IPConfigsRequest) (PodInfo, error) {
|
||||
p, err := UnmarshalPodInfo(req.OrchestratorContext)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
|
@ -411,9 +413,13 @@ type IPConfigRequest struct {
|
|||
Ifname string // Used by delegated IPAM
|
||||
}
|
||||
|
||||
func (i IPConfigRequest) String() string {
|
||||
return fmt.Sprintf("[IPConfigRequest: DesiredIPAddress %s, PodInterfaceID %s, InfraContainerID %s, OrchestratorContext %s]",
|
||||
i.DesiredIPAddress, i.PodInterfaceID, i.InfraContainerID, string(i.OrchestratorContext))
|
||||
// Same as IPConfigRequest except that DesiredIPAddresses is passed in as a slice
|
||||
type IPConfigsRequest struct {
|
||||
DesiredIPAddresses []string `json:"desiredIPAddresses"`
|
||||
PodInterfaceID string `json:"podInterfaceID"`
|
||||
InfraContainerID string `json:"infraContainerID"`
|
||||
OrchestratorContext json.RawMessage `json:"orchestratorContext"`
|
||||
Ifname string `json:"ifname"` // Used by delegated IPAM
|
||||
}
|
||||
|
||||
// IPConfigResponse is used in CNS IPAM mode as a response to CNI ADD
|
||||
|
|
@ -422,6 +428,12 @@ type IPConfigResponse struct {
|
|||
Response Response
|
||||
}
|
||||
|
||||
// IPConfigsResponse is used in CNS IPAM mode to return a slice of IP configs as a response to CNI ADD
|
||||
type IPConfigsResponse struct {
|
||||
PodIPInfo []PodIpInfo `json:"podIPInfo"`
|
||||
Response Response `json:"response"`
|
||||
}
|
||||
|
||||
// GetIPAddressesRequest is used in CNS IPAM mode to get the states of IPConfigs
|
||||
// The IPConfigStateFilter is a slice of IPs to fetch from CNS that match those states
|
||||
type GetIPAddressesRequest struct {
|
||||
|
|
@ -440,9 +452,9 @@ type GetIPAddressStatusResponse struct {
|
|||
Response Response
|
||||
}
|
||||
|
||||
// GetPodContextResponse is used in CNS Client debug mode to get mapping of Orchestrator Context to Pod IP UUID
|
||||
// GetPodContextResponse is used in CNS Client debug mode to get mapping of Orchestrator Context to Pod IP UUIDs
|
||||
type GetPodContextResponse struct {
|
||||
PodContext map[string]string
|
||||
PodContext map[string][]string // Can have multiple Pod IP UUIDs in the case of dualstack
|
||||
Response Response
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -56,18 +56,18 @@ func TestUnmarshalPodInfo(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestNewPodInfoFromIPConfigRequest(t *testing.T) {
|
||||
func TestNewPodInfoFromIPConfigsRequest(t *testing.T) {
|
||||
GlobalPodInfoScheme = InterfaceIDPodInfoScheme
|
||||
defer func() { GlobalPodInfoScheme = KubernetesPodInfoScheme }()
|
||||
tests := []struct {
|
||||
name string
|
||||
req IPConfigRequest
|
||||
req IPConfigsRequest
|
||||
want PodInfo
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "full req",
|
||||
req: IPConfigRequest{
|
||||
req: IPConfigsRequest{
|
||||
PodInterfaceID: "abcdef-eth0",
|
||||
InfraContainerID: "abcdef",
|
||||
OrchestratorContext: []byte(`{"PodName":"pod","PodNamespace":"namespace"}`),
|
||||
|
|
@ -84,7 +84,7 @@ func TestNewPodInfoFromIPConfigRequest(t *testing.T) {
|
|||
},
|
||||
{
|
||||
name: "empty interface id",
|
||||
req: IPConfigRequest{
|
||||
req: IPConfigsRequest{
|
||||
InfraContainerID: "abcdef",
|
||||
OrchestratorContext: []byte(`{"PodName":"pod","PodNamespace":"namespace"}`),
|
||||
},
|
||||
|
|
@ -94,7 +94,7 @@ func TestNewPodInfoFromIPConfigRequest(t *testing.T) {
|
|||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := NewPodInfoFromIPConfigRequest(tt.req)
|
||||
got, err := NewPodInfoFromIPConfigsRequest(tt.req)
|
||||
if tt.wantErr {
|
||||
assert.Error(t, err)
|
||||
return
|
||||
|
|
|
|||
|
|
@ -30,7 +30,9 @@ var clientPaths = []string{
|
|||
cns.CreateHostNCApipaEndpointPath,
|
||||
cns.DeleteHostNCApipaEndpointPath,
|
||||
cns.RequestIPConfig,
|
||||
cns.RequestIPConfigs,
|
||||
cns.ReleaseIPConfig,
|
||||
cns.ReleaseIPConfigs,
|
||||
cns.PathDebugIPAddresses,
|
||||
cns.PathDebugPodContext,
|
||||
cns.PathDebugRestData,
|
||||
|
|
@ -45,6 +47,8 @@ var clientPaths = []string{
|
|||
cns.GetHomeAz,
|
||||
}
|
||||
|
||||
var errAPINotFound error = errors.New("api not found")
|
||||
|
||||
type do interface {
|
||||
Do(*http.Request) (*http.Response, error)
|
||||
}
|
||||
|
|
@ -374,6 +378,102 @@ func (c *Client) ReleaseIPAddress(ctx context.Context, ipconfig cns.IPConfigRequ
|
|||
return nil
|
||||
}
|
||||
|
||||
// RequestIPs calls the RequestIPConfigs in CNS
|
||||
func (c *Client) RequestIPs(ctx context.Context, ipconfig cns.IPConfigsRequest) (*cns.IPConfigsResponse, error) {
|
||||
var err error
|
||||
defer func() {
|
||||
if err != nil {
|
||||
if e := c.ReleaseIPs(ctx, ipconfig); e != nil {
|
||||
err = errors.Wrap(e, err.Error())
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
var body bytes.Buffer
|
||||
err = json.NewEncoder(&body).Encode(ipconfig)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to encode IPConfigsRequest")
|
||||
}
|
||||
|
||||
u := c.routes[cns.RequestIPConfigs]
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, u.String(), &body)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to build request")
|
||||
}
|
||||
req.Header.Set(headerContentType, contentTypeJSON)
|
||||
res, err := c.client.Do(req)
|
||||
|
||||
// if we get a 404 error
|
||||
if res.StatusCode == http.StatusNotFound {
|
||||
return nil, fmt.Errorf("cannot find API RequestIPs %w: %v", errAPINotFound, err) //nolint:errorlint // multiple %w not supported in 1.19
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "http request failed")
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
if res.StatusCode != http.StatusOK {
|
||||
return nil, errors.Errorf("http response %d", res.StatusCode)
|
||||
}
|
||||
|
||||
var response cns.IPConfigsResponse
|
||||
err = json.NewDecoder(res.Body).Decode(&response)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to decode IPConfigsResponse")
|
||||
}
|
||||
|
||||
if response.Response.ReturnCode != 0 {
|
||||
return nil, errors.New(response.Response.Message)
|
||||
}
|
||||
|
||||
return &response, nil
|
||||
}
|
||||
|
||||
// ReleaseIPs calls releaseIPs on which releases the IPs on the pod
|
||||
func (c *Client) ReleaseIPs(ctx context.Context, ipconfig cns.IPConfigsRequest) error {
|
||||
var body bytes.Buffer
|
||||
err := json.NewEncoder(&body).Encode(ipconfig)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to encode IPConfigsRequest")
|
||||
}
|
||||
|
||||
u := c.routes[cns.ReleaseIPConfigs]
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, u.String(), &body)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to build request")
|
||||
}
|
||||
req.Header.Set(headerContentType, contentTypeJSON)
|
||||
res, err := c.client.Do(req)
|
||||
|
||||
// if we get a 404 error
|
||||
if res.StatusCode == http.StatusNotFound {
|
||||
return fmt.Errorf("cannot find API ReleaseIPs %w: %v", errAPINotFound, err) //nolint:errorlint // multiple %w not supported in 1.19
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "http request failed")
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
if res.StatusCode != http.StatusOK {
|
||||
return errors.Errorf("http response %d", res.StatusCode)
|
||||
}
|
||||
|
||||
var resp cns.Response
|
||||
|
||||
err = json.NewDecoder(res.Body).Decode(&resp)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to decode Response")
|
||||
}
|
||||
|
||||
if resp.ReturnCode != 0 {
|
||||
return errors.New(resp.Message)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetIPAddressesMatchingStates takes a variadic number of string parameters, to get all IP Addresses matching a number of states
|
||||
// usage GetIPAddressesWithStates(ctx, types.Available...)
|
||||
func (c *Client) GetIPAddressesMatchingStates(ctx context.Context, stateFilter ...types.IPState) ([]cns.IPConfigurationStatus, error) {
|
||||
|
|
@ -420,7 +520,7 @@ func (c *Client) GetIPAddressesMatchingStates(ctx context.Context, stateFilter .
|
|||
}
|
||||
|
||||
// GetPodOrchestratorContext calls GetPodIpOrchestratorContext API on CNS
|
||||
func (c *Client) GetPodOrchestratorContext(ctx context.Context) (map[string]string, error) {
|
||||
func (c *Client) GetPodOrchestratorContext(ctx context.Context) (map[string][]string, error) {
|
||||
u := c.routes[cns.PathDebugPodContext]
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
|
||||
if err != nil {
|
||||
|
|
|
|||
|
|
@ -294,6 +294,11 @@ func TestCNSClientRequestAndRelease(t *testing.T) {
|
|||
|
||||
t.Log(ipaddresses)
|
||||
|
||||
addresses := make([]string, len(ipaddresses))
|
||||
for i := range ipaddresses {
|
||||
addresses[i] = ipaddresses[i].IPAddress
|
||||
}
|
||||
|
||||
// release requested IP address, expect success
|
||||
err = cnsClient.ReleaseIPAddress(context.TODO(), cns.IPConfigRequest{DesiredIPAddress: ipaddresses[0].IPAddress, OrchestratorContext: orchestratorContext})
|
||||
assert.NoError(t, err, "Expected to not fail when releasing IP reservation found with context")
|
||||
|
|
@ -378,7 +383,9 @@ func TestCNSClientDebugAPI(t *testing.T) {
|
|||
assert.Len(t, testIpamPoolMonitor.CachedNNC.Status.NetworkContainers, 1, "Expected only one Network Container in the list")
|
||||
|
||||
t.Logf("In-memory Data: ")
|
||||
t.Logf("PodIPIDByOrchestratorContext: %+v", inmemory.HTTPRestServiceData.PodIPIDByPodInterfaceKey)
|
||||
for i := range inmemory.HTTPRestServiceData.PodIPIDByPodInterfaceKey {
|
||||
t.Logf("PodIPIDByOrchestratorContext: %+v", inmemory.HTTPRestServiceData.PodIPIDByPodInterfaceKey[i])
|
||||
}
|
||||
t.Logf("PodIPConfigState: %+v", inmemory.HTTPRestServiceData.PodIPConfigState)
|
||||
t.Logf("IPAMPoolMonitor: %+v", inmemory.HTTPRestServiceData.IPAMPoolMonitor)
|
||||
}
|
||||
|
|
@ -1821,6 +1828,310 @@ func TestReleaseIPAddress(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestRequestIPs(t *testing.T) {
|
||||
emptyRoutes, _ := buildRoutes(defaultBaseURL, clientPaths)
|
||||
tests := []struct {
|
||||
name string
|
||||
ctx context.Context
|
||||
ipconfig cns.IPConfigsRequest
|
||||
mockdo *mockdo
|
||||
routes map[string]url.URL
|
||||
want *cns.IPConfigsResponse
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "happy case 1 IP",
|
||||
ctx: context.TODO(),
|
||||
ipconfig: cns.IPConfigsRequest{
|
||||
DesiredIPAddresses: []string{
|
||||
"testipaddress",
|
||||
},
|
||||
PodInterfaceID: "testpodinterfaceid",
|
||||
InfraContainerID: "testcontainerid",
|
||||
},
|
||||
mockdo: &mockdo{
|
||||
errToReturn: nil,
|
||||
objToReturn: &cns.IPConfigsResponse{},
|
||||
httpStatusCodeToReturn: http.StatusOK,
|
||||
},
|
||||
routes: emptyRoutes,
|
||||
want: &cns.IPConfigsResponse{},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "happy case 2 IPs",
|
||||
ctx: context.TODO(),
|
||||
ipconfig: cns.IPConfigsRequest{
|
||||
DesiredIPAddresses: []string{
|
||||
"testipaddress1",
|
||||
"testipaddress2",
|
||||
},
|
||||
PodInterfaceID: "testpodinterfaceid",
|
||||
InfraContainerID: "testcontainerid",
|
||||
},
|
||||
mockdo: &mockdo{
|
||||
errToReturn: nil,
|
||||
objToReturn: &cns.IPConfigsResponse{},
|
||||
httpStatusCodeToReturn: http.StatusOK,
|
||||
},
|
||||
routes: emptyRoutes,
|
||||
want: &cns.IPConfigsResponse{},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "bad request",
|
||||
ctx: context.TODO(),
|
||||
ipconfig: cns.IPConfigsRequest{
|
||||
DesiredIPAddresses: []string{
|
||||
"testipaddress",
|
||||
},
|
||||
PodInterfaceID: "testpodinterfaceid",
|
||||
InfraContainerID: "testcontainerid",
|
||||
},
|
||||
mockdo: &mockdo{
|
||||
errToReturn: errBadRequest,
|
||||
objToReturn: nil,
|
||||
httpStatusCodeToReturn: http.StatusBadRequest,
|
||||
},
|
||||
routes: emptyRoutes,
|
||||
want: nil,
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "bad decoding",
|
||||
ctx: context.TODO(),
|
||||
ipconfig: cns.IPConfigsRequest{
|
||||
DesiredIPAddresses: []string{
|
||||
"testipaddress",
|
||||
},
|
||||
PodInterfaceID: "testpodinterfaceid",
|
||||
InfraContainerID: "testcontainerid",
|
||||
},
|
||||
mockdo: &mockdo{
|
||||
errToReturn: nil,
|
||||
objToReturn: []cns.IPConfigsResponse{},
|
||||
httpStatusCodeToReturn: http.StatusOK,
|
||||
},
|
||||
routes: emptyRoutes,
|
||||
want: nil,
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "http status not ok",
|
||||
ctx: context.TODO(),
|
||||
ipconfig: cns.IPConfigsRequest{},
|
||||
mockdo: &mockdo{
|
||||
errToReturn: nil,
|
||||
objToReturn: nil,
|
||||
httpStatusCodeToReturn: http.StatusInternalServerError,
|
||||
},
|
||||
routes: emptyRoutes,
|
||||
want: nil,
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "cns return code not zero",
|
||||
ctx: context.TODO(),
|
||||
ipconfig: cns.IPConfigsRequest{
|
||||
DesiredIPAddresses: []string{
|
||||
"testipaddress",
|
||||
},
|
||||
PodInterfaceID: "testpodinterfaceid",
|
||||
InfraContainerID: "testcontainerid",
|
||||
},
|
||||
mockdo: &mockdo{
|
||||
errToReturn: nil,
|
||||
objToReturn: &cns.IPConfigResponse{
|
||||
Response: cns.Response{
|
||||
ReturnCode: types.UnsupportedNetworkType,
|
||||
},
|
||||
},
|
||||
httpStatusCodeToReturn: http.StatusOK,
|
||||
},
|
||||
routes: emptyRoutes,
|
||||
want: nil,
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "nil context",
|
||||
ctx: nil,
|
||||
ipconfig: cns.IPConfigsRequest{
|
||||
DesiredIPAddresses: []string{
|
||||
"testipaddress",
|
||||
},
|
||||
PodInterfaceID: "testpodinterfaceid",
|
||||
InfraContainerID: "testcontainerid",
|
||||
},
|
||||
mockdo: &mockdo{},
|
||||
routes: emptyRoutes,
|
||||
want: nil,
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
tt := tt
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
client := &Client{
|
||||
client: tt.mockdo,
|
||||
routes: tt.routes,
|
||||
}
|
||||
got, err := client.RequestIPs(tt.ctx, tt.ipconfig)
|
||||
if tt.wantErr {
|
||||
assert.Error(t, err)
|
||||
} else {
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
assert.Equal(t, tt.want, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestReleaseIPs(t *testing.T) {
|
||||
emptyRoutes, _ := buildRoutes(defaultBaseURL, clientPaths)
|
||||
tests := []struct {
|
||||
name string
|
||||
ctx context.Context
|
||||
ipconfig cns.IPConfigsRequest
|
||||
mockdo *mockdo
|
||||
routes map[string]url.URL
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "happy case 1 IP",
|
||||
ctx: context.TODO(),
|
||||
ipconfig: cns.IPConfigsRequest{
|
||||
DesiredIPAddresses: []string{
|
||||
"testipaddress",
|
||||
},
|
||||
PodInterfaceID: "testpodinterfaceid",
|
||||
InfraContainerID: "testcontainerid",
|
||||
},
|
||||
mockdo: &mockdo{
|
||||
errToReturn: nil,
|
||||
objToReturn: &cns.Response{},
|
||||
httpStatusCodeToReturn: http.StatusOK,
|
||||
},
|
||||
routes: emptyRoutes,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "happy case 2 IPs",
|
||||
ctx: context.TODO(),
|
||||
ipconfig: cns.IPConfigsRequest{
|
||||
DesiredIPAddresses: []string{
|
||||
"testipaddress",
|
||||
},
|
||||
PodInterfaceID: "testpodinterfaceid",
|
||||
InfraContainerID: "testcontainerid",
|
||||
},
|
||||
mockdo: &mockdo{
|
||||
errToReturn: nil,
|
||||
objToReturn: &cns.Response{},
|
||||
httpStatusCodeToReturn: http.StatusOK,
|
||||
},
|
||||
routes: emptyRoutes,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "bad request",
|
||||
ctx: context.TODO(),
|
||||
ipconfig: cns.IPConfigsRequest{
|
||||
DesiredIPAddresses: []string{
|
||||
"testipaddress",
|
||||
},
|
||||
PodInterfaceID: "testpodinterfaceid",
|
||||
InfraContainerID: "testcontainerid",
|
||||
},
|
||||
mockdo: &mockdo{
|
||||
errToReturn: errBadRequest,
|
||||
objToReturn: nil,
|
||||
httpStatusCodeToReturn: http.StatusBadRequest,
|
||||
},
|
||||
routes: emptyRoutes,
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "bad decoding",
|
||||
ctx: context.TODO(),
|
||||
ipconfig: cns.IPConfigsRequest{
|
||||
DesiredIPAddresses: []string{
|
||||
"testipaddress",
|
||||
},
|
||||
PodInterfaceID: "testpodinterfaceid",
|
||||
InfraContainerID: "testcontainerid",
|
||||
},
|
||||
mockdo: &mockdo{
|
||||
errToReturn: nil,
|
||||
objToReturn: []cns.Response{},
|
||||
httpStatusCodeToReturn: http.StatusOK,
|
||||
},
|
||||
routes: emptyRoutes,
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "http status not ok",
|
||||
ctx: context.TODO(),
|
||||
ipconfig: cns.IPConfigsRequest{},
|
||||
mockdo: &mockdo{
|
||||
errToReturn: nil,
|
||||
objToReturn: nil,
|
||||
httpStatusCodeToReturn: http.StatusInternalServerError,
|
||||
},
|
||||
routes: emptyRoutes,
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "cns return code not zero",
|
||||
ctx: context.TODO(),
|
||||
ipconfig: cns.IPConfigsRequest{
|
||||
DesiredIPAddresses: []string{
|
||||
"testipaddress",
|
||||
},
|
||||
PodInterfaceID: "testpodinterfaceid",
|
||||
InfraContainerID: "testcontainerid",
|
||||
},
|
||||
mockdo: &mockdo{
|
||||
errToReturn: nil,
|
||||
objToReturn: &cns.Response{
|
||||
ReturnCode: types.UnsupportedNetworkType,
|
||||
},
|
||||
httpStatusCodeToReturn: http.StatusOK,
|
||||
},
|
||||
routes: emptyRoutes,
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "nil context",
|
||||
ctx: nil,
|
||||
ipconfig: cns.IPConfigsRequest{
|
||||
DesiredIPAddresses: []string{
|
||||
"testipaddress",
|
||||
},
|
||||
PodInterfaceID: "testpodinterfaceid",
|
||||
InfraContainerID: "testcontainerid",
|
||||
},
|
||||
mockdo: &mockdo{},
|
||||
routes: emptyRoutes,
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
tt := tt
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
client := &Client{
|
||||
client: tt.mockdo,
|
||||
routes: tt.routes,
|
||||
}
|
||||
err := client.ReleaseIPs(tt.ctx, tt.ipconfig)
|
||||
if tt.wantErr {
|
||||
assert.Error(t, err)
|
||||
} else {
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetIPAddressesMatchingStates(t *testing.T) {
|
||||
emptyRoutes, _ := buildRoutes(defaultBaseURL, clientPaths)
|
||||
tests := []struct {
|
||||
|
|
@ -1946,7 +2257,7 @@ func TestGetPodOrchestratorContext(t *testing.T) {
|
|||
ctx context.Context
|
||||
mockdo *mockdo
|
||||
routes map[string]url.URL
|
||||
want map[string]string
|
||||
want map[string][]string
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
|
|
@ -1955,12 +2266,12 @@ func TestGetPodOrchestratorContext(t *testing.T) {
|
|||
mockdo: &mockdo{
|
||||
errToReturn: nil,
|
||||
objToReturn: &cns.GetPodContextResponse{
|
||||
PodContext: map[string]string{},
|
||||
PodContext: map[string][]string{},
|
||||
},
|
||||
httpStatusCodeToReturn: http.StatusOK,
|
||||
},
|
||||
routes: emptyRoutes,
|
||||
want: map[string]string{},
|
||||
want: map[string][]string{},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
|
|
|
|||
|
|
@ -161,10 +161,10 @@ func (ic *IpamClient) ReserveIPAddress(poolID string, reservationID string) (str
|
|||
return "", err
|
||||
}
|
||||
|
||||
// ReleaseIPAddress release an Ip address for the reservation id.
|
||||
// ReleaseIPAddress release an IP address for the reservation id.
|
||||
func (ic *IpamClient) ReleaseIPAddress(poolID string, reservationID string) error {
|
||||
var body bytes.Buffer
|
||||
log.Printf("[Azure CNS] ReleaseIpAddress")
|
||||
log.Printf("[Azure CNS] ReleaseIPAddress")
|
||||
|
||||
client, err := getClient(ic.connectionURL)
|
||||
if err != nil {
|
||||
|
|
|
|||
|
|
@ -273,9 +273,9 @@ func (service *HTTPRestService) ReconcileNCState(ncRequest *cns.CreateNetworkCon
|
|||
}
|
||||
|
||||
// now parse the secondaryIP list, if it exists in PodInfo list, then assign that ip.
|
||||
for _, secIpConfig := range ncRequest.SecondaryIPConfigs {
|
||||
if podInfo, exists := podInfoByIP[secIpConfig.IPAddress]; exists {
|
||||
logger.Printf("SecondaryIP %+v is assigned to Pod. %+v, ncId: %s", secIpConfig, podInfo, ncRequest.NetworkContainerid)
|
||||
for _, secIPConfig := range ncRequest.SecondaryIPConfigs {
|
||||
if podInfo, exists := podInfoByIP[secIPConfig.IPAddress]; exists {
|
||||
logger.Printf("SecondaryIP %+v is assigned to Pod. %+v, ncId: %s", secIPConfig, podInfo, ncRequest.NetworkContainerid)
|
||||
|
||||
jsonContext, err := podInfo.OrchestratorContext()
|
||||
if err != nil {
|
||||
|
|
@ -283,19 +283,19 @@ func (service *HTTPRestService) ReconcileNCState(ncRequest *cns.CreateNetworkCon
|
|||
return types.UnexpectedError
|
||||
}
|
||||
|
||||
ipconfigRequest := cns.IPConfigRequest{
|
||||
DesiredIPAddress: secIpConfig.IPAddress,
|
||||
ipconfigsRequest := cns.IPConfigsRequest{
|
||||
DesiredIPAddresses: []string{secIPConfig.IPAddress},
|
||||
OrchestratorContext: jsonContext,
|
||||
InfraContainerID: podInfo.InfraContainerID(),
|
||||
PodInterfaceID: podInfo.InterfaceID(),
|
||||
}
|
||||
|
||||
if _, err := requestIPConfigHelper(service, ipconfigRequest); err != nil {
|
||||
logger.Errorf("AllocateIPConfig failed for SecondaryIP %+v, podInfo %+v, ncId %s, error: %v", secIpConfig, podInfo, ncRequest.NetworkContainerid, err)
|
||||
if _, err := requestIPConfigsHelper(service, ipconfigsRequest); err != nil {
|
||||
logger.Errorf("AllocateIPConfig failed for SecondaryIP %+v, podInfo %+v, ncId %s, error: %v", secIPConfig, podInfo, ncRequest.NetworkContainerid, err)
|
||||
return types.FailedToAllocateIPConfig
|
||||
}
|
||||
} else {
|
||||
logger.Printf("SecondaryIP %+v is not assigned. ncId: %s", secIpConfig, ncRequest.NetworkContainerid)
|
||||
logger.Printf("SecondaryIP %+v is not assigned. ncId: %s", secIPConfig, ncRequest.NetworkContainerid)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -371,10 +371,10 @@ func (service *HTTPRestService) CreateOrUpdateNetworkContainerInternal(req *cns.
|
|||
}
|
||||
|
||||
// Validate SecondaryIPConfig
|
||||
for _, secIpconfig := range req.SecondaryIPConfigs {
|
||||
for _, secIPConfig := range req.SecondaryIPConfigs {
|
||||
// Validate Ipconfig
|
||||
if secIpconfig.IPAddress == "" {
|
||||
logger.Errorf("Failed to add IPConfig to state: %+v, empty IPSubnet.IPAddress", secIpconfig)
|
||||
if secIPConfig.IPAddress == "" {
|
||||
logger.Errorf("Failed to add IPConfig to state: %+v, empty IPSubnet.IPAddress", secIPConfig)
|
||||
return types.InvalidSecondaryIPConfig
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -122,6 +122,7 @@ func TestSyncHostNCVersion(t *testing.T) {
|
|||
// cns.KubernetesCRD has one more logic compared to other orchestrator type, so test both of them
|
||||
orchestratorTypes := []string{cns.Kubernetes, cns.KubernetesCRD}
|
||||
for _, orchestratorType := range orchestratorTypes {
|
||||
orchestratorType := orchestratorType
|
||||
t.Run(orchestratorType, func(t *testing.T) {
|
||||
req := createNCReqeustForSyncHostNCVersion(t)
|
||||
containerStatus := svc.state.ContainerStatus[req.NetworkContainerid]
|
||||
|
|
@ -417,7 +418,7 @@ func validateCreateOrUpdateNCInternal(t *testing.T, secondaryIpCount int, ncVers
|
|||
|
||||
createAndValidateNCRequest(t, secondaryIPConfigs, ncId, ncVersion)
|
||||
|
||||
// now Validate Update, add more secondaryIpConfig and it should handle the update
|
||||
// now Validate Update, add more secondaryIPConfig and it should handle the update
|
||||
fmt.Println("Validate Scaleup")
|
||||
for i := 0; i < secondaryIpCount; i++ {
|
||||
ipaddress := "10.0.0." + strconv.Itoa(startingIndex)
|
||||
|
|
@ -429,7 +430,7 @@ func validateCreateOrUpdateNCInternal(t *testing.T, secondaryIpCount int, ncVers
|
|||
|
||||
createAndValidateNCRequest(t, secondaryIPConfigs, ncId, ncVersion)
|
||||
|
||||
// now Scale down, delete 3 ipaddresses from secondaryIpConfig req
|
||||
// now Scale down, delete 3 ipaddresses from secondaryIPConfig req
|
||||
fmt.Println("Validate Scale down")
|
||||
count := 0
|
||||
for ipid := range secondaryIPConfigs {
|
||||
|
|
@ -490,11 +491,6 @@ func validateNetworkRequest(t *testing.T, req cns.CreateNetworkContainerRequest)
|
|||
t.Fatalf("Failed as Primary IPAddress doesnt match, expected:%s, actual %s", req.IPConfiguration.IPSubnet.IPAddress, actualReq.IPConfiguration.IPSubnet.IPAddress)
|
||||
}
|
||||
|
||||
// Validate Secondary ips are added in the PodMap
|
||||
if len(svc.PodIPConfigState) != len(req.SecondaryIPConfigs) {
|
||||
t.Fatalf("Failed as Secondary IP count doesnt match in PodIpConfig state, expected:%d, actual %d", len(req.SecondaryIPConfigs), len(svc.PodIPConfigState))
|
||||
}
|
||||
|
||||
var expectedIPStatus types.IPState
|
||||
// 0 is the default NMAgent version return from fake GetNetworkContainerInfoFromHost
|
||||
if containerStatus.CreateNetworkContainerRequest.Version > "0" {
|
||||
|
|
@ -504,35 +500,40 @@ func validateNetworkRequest(t *testing.T, req cns.CreateNetworkContainerRequest)
|
|||
}
|
||||
t.Logf("NC version in container status is %s, HostVersion is %s", containerStatus.CreateNetworkContainerRequest.Version, containerStatus.HostVersion)
|
||||
alreadyValidated := make(map[string]string)
|
||||
ncCount := 0
|
||||
for ipid, ipStatus := range svc.PodIPConfigState {
|
||||
if ipaddress, found := alreadyValidated[ipid]; !found {
|
||||
if secondaryIpConfig, ok := req.SecondaryIPConfigs[ipid]; !ok {
|
||||
t.Fatalf("PodIpConfigState has stale ipId: %s, config: %+v", ipid, ipStatus)
|
||||
} else {
|
||||
if ipStatus.IPAddress != secondaryIpConfig.IPAddress {
|
||||
t.Fatalf("IPId: %s IPSubnet doesnt match: expected %+v, actual: %+v", ipid, secondaryIpConfig.IPAddress, ipStatus.IPAddress)
|
||||
}
|
||||
|
||||
// Validate IP state
|
||||
if ipStatus.PodInfo != nil {
|
||||
if _, exists := svc.PodIPIDByPodInterfaceKey[ipStatus.PodInfo.Key()]; exists {
|
||||
if ipStatus.GetState() != types.Assigned {
|
||||
t.Fatalf("IPId: %s State is not Assigned, ipStatus: %+v", ipid, ipStatus)
|
||||
}
|
||||
} else {
|
||||
t.Fatalf("Failed to find podContext for assigned ip: %+v, podinfo :%+v", ipStatus, ipStatus.PodInfo)
|
||||
ncCount++
|
||||
// ignore any IPs that were added from a previous NC
|
||||
if ncCount > (len(svc.state.ContainerStatus) * len(svc.PodIPConfigState)) {
|
||||
if ipaddress, found := alreadyValidated[ipid]; !found {
|
||||
if secondaryIPConfig, ok := req.SecondaryIPConfigs[ipid]; !ok {
|
||||
t.Fatalf("PodIpConfigState has stale ipId: %s, config: %+v", ipid, ipStatus)
|
||||
} else {
|
||||
if ipStatus.IPAddress != secondaryIPConfig.IPAddress {
|
||||
t.Fatalf("IPId: %s IPSubnet doesnt match: expected %+v, actual: %+v", ipid, secondaryIPConfig.IPAddress, ipStatus.IPAddress)
|
||||
}
|
||||
} else if ipStatus.GetState() != expectedIPStatus {
|
||||
// Todo: Validate for pendingRelease as well
|
||||
t.Fatalf("IPId: %s State is not as expected, ipStatus is : %+v, expected status is %+v", ipid, ipStatus.GetState(), expectedIPStatus)
|
||||
}
|
||||
|
||||
alreadyValidated[ipid] = ipStatus.IPAddress
|
||||
}
|
||||
} else {
|
||||
// if ipaddress is not same, then fail
|
||||
if ipaddress != ipStatus.IPAddress {
|
||||
t.Fatalf("Added the same IP guid :%s with different ipaddress, expected:%s, actual %s", ipid, ipStatus.IPAddress, ipaddress)
|
||||
// Validate IP state
|
||||
if ipStatus.PodInfo != nil {
|
||||
if _, exists := svc.PodIPIDByPodInterfaceKey[ipStatus.PodInfo.Key()]; exists {
|
||||
if ipStatus.GetState() != types.Assigned {
|
||||
t.Fatalf("IPId: %s State is not Assigned, ipStatus: %+v", ipid, ipStatus)
|
||||
}
|
||||
} else {
|
||||
t.Fatalf("Failed to find podContext for assigned ip: %+v, podinfo :%+v", ipStatus, ipStatus.PodInfo)
|
||||
}
|
||||
} else if ipStatus.GetState() != expectedIPStatus {
|
||||
// Todo: Validate for pendingRelease as well
|
||||
t.Fatalf("IPId: %s State is not as expected, ipStatus is : %+v, expected status is %+v", ipid, ipStatus.GetState(), expectedIPStatus)
|
||||
}
|
||||
|
||||
alreadyValidated[ipid] = ipStatus.IPAddress
|
||||
}
|
||||
} else {
|
||||
// if ipaddress is not same, then fail
|
||||
if ipaddress != ipStatus.IPAddress {
|
||||
t.Fatalf("Added the same IP guid :%s with different ipaddress, expected:%s, actual %s", ipid, ipStatus.IPAddress, ipaddress)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -582,27 +583,27 @@ func validateNCStateAfterReconcile(t *testing.T, ncRequest *cns.CreateNetworkCon
|
|||
}
|
||||
|
||||
for ipaddress, podInfo := range expectedAssignedPods {
|
||||
ipId := svc.PodIPIDByPodInterfaceKey[podInfo.Key()]
|
||||
ipConfigstate := svc.PodIPConfigState[ipId]
|
||||
for _, ipID := range svc.PodIPIDByPodInterfaceKey[podInfo.Key()] {
|
||||
ipConfigstate := svc.PodIPConfigState[ipID]
|
||||
if ipConfigstate.GetState() != types.Assigned {
|
||||
t.Fatalf("IpAddress %s is not marked as assigned to Pod: %+v, ipState: %+v", ipaddress, podInfo, ipConfigstate)
|
||||
}
|
||||
|
||||
if ipConfigstate.GetState() != types.Assigned {
|
||||
t.Fatalf("IpAddress %s is not marked as assigned to Pod: %+v, ipState: %+v", ipaddress, podInfo, ipConfigstate)
|
||||
}
|
||||
// Validate if IPAddress matches
|
||||
if ipConfigstate.IPAddress != ipaddress {
|
||||
t.Fatalf("IpAddress %s is not same, for Pod: %+v, actual ipState: %+v", ipaddress, podInfo, ipConfigstate)
|
||||
}
|
||||
|
||||
// Validate if IPAddress matches
|
||||
if ipConfigstate.IPAddress != ipaddress {
|
||||
t.Fatalf("IpAddress %s is not same, for Pod: %+v, actual ipState: %+v", ipaddress, podInfo, ipConfigstate)
|
||||
}
|
||||
// Validate pod context
|
||||
if reflect.DeepEqual(ipConfigstate.PodInfo, podInfo) != true {
|
||||
t.Fatalf("OrchestrationContext: is not same, expected: %+v, actual %+v", ipConfigstate.PodInfo, podInfo)
|
||||
}
|
||||
|
||||
// Valdate pod context
|
||||
if reflect.DeepEqual(ipConfigstate.PodInfo, podInfo) != true {
|
||||
t.Fatalf("OrchestrationContext: is not same, expected: %+v, actual %+v", ipConfigstate.PodInfo, podInfo)
|
||||
}
|
||||
|
||||
// Validate this IP belongs to a valid NCRequest
|
||||
nc := svc.state.ContainerStatus[ipConfigstate.NCID]
|
||||
if _, exists := nc.CreateNetworkContainerRequest.SecondaryIPConfigs[ipConfigstate.ID]; !exists {
|
||||
t.Fatalf("Secondary IP config doest exist in NC, ncid: %s, ipId %s", ipConfigstate.NCID, ipConfigstate.ID)
|
||||
// Validate this IP belongs to a valid NCRequest
|
||||
nc := svc.state.ContainerStatus[ipConfigstate.NCID]
|
||||
if _, exists := nc.CreateNetworkContainerRequest.SecondaryIPConfigs[ipConfigstate.ID]; !exists {
|
||||
t.Fatalf("Secondary IP config doest exist in NC, ncid: %s, ipId %s", ipConfigstate.NCID, ipConfigstate.ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -17,47 +17,35 @@ import (
|
|||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
// used to request an IPConfig from the CNS state
|
||||
func (service *HTTPRestService) requestIPConfigHandler(w http.ResponseWriter, r *http.Request) {
|
||||
var ipconfigRequest cns.IPConfigRequest
|
||||
err := service.Listener.Decode(w, r, &ipconfigRequest)
|
||||
operationName := "requestIPConfigHandler"
|
||||
logger.Request(service.Name+operationName, ipconfigRequest, err)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
var (
|
||||
errStoreEmpty = errors.New("empty endpoint state store")
|
||||
errParsePodIPFailed = errors.New("failed to parse pod's ip")
|
||||
)
|
||||
|
||||
// retrieve ipconfig from nc
|
||||
podInfo, returnCode, returnMessage := service.validateIPConfigRequest(ipconfigRequest)
|
||||
// requestIPConfigHandlerHelper validates the request, assigns IPs, and returns a response
|
||||
func (service *HTTPRestService) requestIPConfigHandlerHelper(ipconfigsRequest cns.IPConfigsRequest) (*cns.IPConfigsResponse, error) {
|
||||
podInfo, returnCode, returnMessage := service.validateIPConfigsRequest(ipconfigsRequest)
|
||||
if returnCode != types.Success {
|
||||
reserveResp := &cns.IPConfigResponse{
|
||||
return &cns.IPConfigsResponse{
|
||||
Response: cns.Response{
|
||||
ReturnCode: returnCode,
|
||||
Message: returnMessage,
|
||||
},
|
||||
}
|
||||
w.Header().Set(cnsReturnCode, reserveResp.Response.ReturnCode.String())
|
||||
err = service.Listener.Encode(w, &reserveResp)
|
||||
logger.ResponseEx(service.Name+operationName, ipconfigRequest, reserveResp, reserveResp.Response.ReturnCode, err)
|
||||
return
|
||||
}, errors.New("failed to validate ip config request")
|
||||
}
|
||||
|
||||
// record a pod requesting an IP
|
||||
service.podsPendingIPAssignment.Push(podInfo.Key())
|
||||
|
||||
podIPInfo, err := requestIPConfigHelper(service, ipconfigRequest)
|
||||
podIPInfo, err := requestIPConfigsHelper(service, ipconfigsRequest)
|
||||
if err != nil {
|
||||
reserveResp := &cns.IPConfigResponse{
|
||||
return &cns.IPConfigsResponse{
|
||||
Response: cns.Response{
|
||||
ReturnCode: types.FailedToAllocateIPConfig,
|
||||
Message: fmt.Sprintf("AllocateIPConfig failed: %v, IP config request is %s", err, ipconfigRequest),
|
||||
Message: fmt.Sprintf("AllocateIPConfig failed: %v, IP config request is %s", err, ipconfigsRequest),
|
||||
},
|
||||
PodIpInfo: podIPInfo,
|
||||
}
|
||||
w.Header().Set(cnsReturnCode, reserveResp.Response.ReturnCode.String())
|
||||
err = service.Listener.Encode(w, &reserveResp)
|
||||
logger.ResponseEx(service.Name+operationName, ipconfigRequest, reserveResp, reserveResp.Response.ReturnCode, err)
|
||||
return
|
||||
PodIPInfo: podIPInfo,
|
||||
}, err
|
||||
}
|
||||
|
||||
// record a pod assigned an IP
|
||||
|
|
@ -70,145 +58,292 @@ func (service *HTTPRestService) requestIPConfigHandler(w http.ResponseWriter, r
|
|||
|
||||
// Check if http rest service managed endpoint state is set
|
||||
if service.Options[common.OptManageEndpointState] == true {
|
||||
err = service.updateEndpointState(ipconfigRequest, podInfo, podIPInfo)
|
||||
err = service.updateEndpointState(ipconfigsRequest, podInfo, podIPInfo)
|
||||
if err != nil {
|
||||
reserveResp := &cns.IPConfigResponse{
|
||||
return &cns.IPConfigsResponse{
|
||||
Response: cns.Response{
|
||||
ReturnCode: types.UnexpectedError,
|
||||
Message: fmt.Sprintf("Update endpoint state failed: %v ", err),
|
||||
},
|
||||
PodIpInfo: podIPInfo,
|
||||
}
|
||||
w.Header().Set(cnsReturnCode, reserveResp.Response.ReturnCode.String())
|
||||
err = service.Listener.Encode(w, &reserveResp)
|
||||
logger.ResponseEx(service.Name+operationName, ipconfigRequest, reserveResp, reserveResp.Response.ReturnCode, err)
|
||||
return
|
||||
PodIPInfo: podIPInfo,
|
||||
}, err
|
||||
}
|
||||
}
|
||||
|
||||
reserveResp := &cns.IPConfigResponse{
|
||||
return &cns.IPConfigsResponse{
|
||||
Response: cns.Response{
|
||||
ReturnCode: types.Success,
|
||||
},
|
||||
PodIpInfo: podIPInfo,
|
||||
PodIPInfo: podIPInfo,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// requestIPConfigHandler requests an IPConfig from the CNS state
|
||||
func (service *HTTPRestService) requestIPConfigHandler(w http.ResponseWriter, r *http.Request) {
|
||||
var ipconfigRequest cns.IPConfigRequest
|
||||
err := service.Listener.Decode(w, r, &ipconfigRequest)
|
||||
operationName := "requestIPConfigHandler"
|
||||
logger.Request(service.Name+operationName, ipconfigRequest, err)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// This method can only return 1 IP. If we have more than one NC then we expect to need to return one IP per NC
|
||||
if len(service.state.ContainerStatus) > 1 {
|
||||
// we send a response back saying that this API won't be able to return the amount of IPs needed to fulfill the request
|
||||
reserveResp := &cns.IPConfigResponse{
|
||||
Response: cns.Response{
|
||||
ReturnCode: types.InvalidRequest,
|
||||
Message: fmt.Sprintf("Called API that can only return 1 IP when expecting %d", len(service.state.ContainerStatus)),
|
||||
},
|
||||
}
|
||||
w.Header().Set(cnsReturnCode, reserveResp.Response.ReturnCode.String())
|
||||
err = service.Listener.Encode(w, &reserveResp)
|
||||
logger.ResponseEx(service.Name+operationName, ipconfigRequest, reserveResp, reserveResp.Response.ReturnCode, err)
|
||||
return
|
||||
}
|
||||
|
||||
var ipconfigsRequest cns.IPConfigsRequest
|
||||
// doesn't fill in DesiredIPAddresses if it is empty in the original request
|
||||
if ipconfigRequest.DesiredIPAddress != "" {
|
||||
ipconfigsRequest = cns.IPConfigsRequest{
|
||||
DesiredIPAddresses: []string{
|
||||
ipconfigRequest.DesiredIPAddress,
|
||||
},
|
||||
PodInterfaceID: ipconfigRequest.PodInterfaceID,
|
||||
InfraContainerID: ipconfigRequest.InfraContainerID,
|
||||
OrchestratorContext: ipconfigRequest.OrchestratorContext,
|
||||
Ifname: ipconfigRequest.Ifname,
|
||||
}
|
||||
} else {
|
||||
ipconfigsRequest = cns.IPConfigsRequest{
|
||||
PodInterfaceID: ipconfigRequest.PodInterfaceID,
|
||||
InfraContainerID: ipconfigRequest.InfraContainerID,
|
||||
OrchestratorContext: ipconfigRequest.OrchestratorContext,
|
||||
Ifname: ipconfigRequest.Ifname,
|
||||
}
|
||||
}
|
||||
|
||||
ipConfigsResp, errResp := service.requestIPConfigHandlerHelper(ipconfigsRequest) //nolint:contextcheck // appease linter
|
||||
if errResp != nil {
|
||||
// As this API is expected to return IPConfigResponse, generate it from the IPConfigsResponse returned above
|
||||
reserveResp := &cns.IPConfigResponse{
|
||||
Response: ipConfigsResp.Response,
|
||||
}
|
||||
w.Header().Set(cnsReturnCode, reserveResp.Response.ReturnCode.String())
|
||||
err = service.Listener.Encode(w, &reserveResp)
|
||||
logger.ResponseEx(service.Name+operationName, ipconfigsRequest, reserveResp, reserveResp.Response.ReturnCode, err)
|
||||
return
|
||||
}
|
||||
|
||||
// As this API is expected to return IPConfigResponse, generate it from the IPConfigsResponse returned above.
|
||||
reserveResp := &cns.IPConfigResponse{
|
||||
Response: ipConfigsResp.Response,
|
||||
PodIpInfo: ipConfigsResp.PodIPInfo[0],
|
||||
}
|
||||
w.Header().Set(cnsReturnCode, reserveResp.Response.ReturnCode.String())
|
||||
err = service.Listener.Encode(w, &reserveResp)
|
||||
logger.ResponseEx(service.Name+operationName, ipconfigRequest, reserveResp, reserveResp.Response.ReturnCode, err)
|
||||
logger.ResponseEx(service.Name+operationName, ipconfigsRequest, reserveResp, reserveResp.Response.ReturnCode, err)
|
||||
}
|
||||
|
||||
var (
|
||||
errStoreEmpty = errors.New("empty endpoint state store")
|
||||
errParsePodIPFailed = errors.New("failed to parse pod's ip")
|
||||
)
|
||||
// requestIPConfigsHandler requests multiple IPConfigs from the CNS state
|
||||
func (service *HTTPRestService) requestIPConfigsHandler(w http.ResponseWriter, r *http.Request) {
|
||||
var ipconfigsRequest cns.IPConfigsRequest
|
||||
err := service.Listener.Decode(w, r, &ipconfigsRequest)
|
||||
operationName := "requestIPConfigsHandler"
|
||||
logger.Request(service.Name+operationName, ipconfigsRequest, err)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
func (service *HTTPRestService) updateEndpointState(ipconfigRequest cns.IPConfigRequest, podInfo cns.PodInfo, podIPInfo cns.PodIpInfo) error {
|
||||
ipConfigsResp, err := service.requestIPConfigHandlerHelper(ipconfigsRequest) // nolint:contextcheck // appease linter
|
||||
if err != nil {
|
||||
w.Header().Set(cnsReturnCode, ipConfigsResp.Response.ReturnCode.String())
|
||||
err = service.Listener.Encode(w, &ipConfigsResp)
|
||||
logger.ResponseEx(service.Name+operationName, ipconfigsRequest, ipConfigsResp, ipConfigsResp.Response.ReturnCode, err)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set(cnsReturnCode, ipConfigsResp.Response.ReturnCode.String())
|
||||
err = service.Listener.Encode(w, &ipConfigsResp)
|
||||
logger.ResponseEx(service.Name+operationName, ipconfigsRequest, ipConfigsResp, ipConfigsResp.Response.ReturnCode, err)
|
||||
}
|
||||
|
||||
func (service *HTTPRestService) updateEndpointState(ipconfigsRequest cns.IPConfigsRequest, podInfo cns.PodInfo, podIPInfo []cns.PodIpInfo) error {
|
||||
if service.EndpointStateStore == nil {
|
||||
return errStoreEmpty
|
||||
}
|
||||
service.Lock()
|
||||
defer service.Unlock()
|
||||
logger.Printf("[updateEndpointState] Updating endpoint state for infra container %s", ipconfigRequest.InfraContainerID)
|
||||
if endpointInfo, ok := service.EndpointState[ipconfigRequest.InfraContainerID]; ok {
|
||||
logger.Warnf("[updateEndpointState] Found existing endpoint state for infra container %s", ipconfigRequest.InfraContainerID)
|
||||
ip := net.ParseIP(podIPInfo.PodIPConfig.IPAddress)
|
||||
if ip == nil {
|
||||
logger.Errorf("failed to parse pod ip address %s", podIPInfo.PodIPConfig.IPAddress)
|
||||
return errParsePodIPFailed
|
||||
}
|
||||
if ip.To4() == nil { // is an ipv6 address
|
||||
ipconfig := net.IPNet{IP: ip, Mask: net.CIDRMask(int(podIPInfo.PodIPConfig.PrefixLength), 128)} // nolint
|
||||
for _, ipconf := range endpointInfo.IfnameToIPMap[ipconfigRequest.Ifname].IPv6 {
|
||||
if ipconf.IP.Equal(ipconfig.IP) {
|
||||
logger.Printf("[updateEndpointState] Found existing ipv6 ipconfig for infra container %s", ipconfigRequest.InfraContainerID)
|
||||
return nil
|
||||
}
|
||||
logger.Printf("[updateEndpointState] Updating endpoint state for infra container %s", ipconfigsRequest.InfraContainerID)
|
||||
for i := range podIPInfo {
|
||||
if endpointInfo, ok := service.EndpointState[ipconfigsRequest.InfraContainerID]; ok {
|
||||
logger.Warnf("[updateEndpointState] Found existing endpoint state for infra container %s", ipconfigsRequest.InfraContainerID)
|
||||
ip := net.ParseIP(podIPInfo[i].PodIPConfig.IPAddress)
|
||||
if ip == nil {
|
||||
logger.Errorf("failed to parse pod ip address %s", podIPInfo[i].PodIPConfig.IPAddress)
|
||||
return errParsePodIPFailed
|
||||
}
|
||||
endpointInfo.IfnameToIPMap[ipconfigRequest.Ifname].IPv6 = append(endpointInfo.IfnameToIPMap[ipconfigRequest.Ifname].IPv6, ipconfig)
|
||||
} else {
|
||||
ipconfig := net.IPNet{IP: ip, Mask: net.CIDRMask(int(podIPInfo.PodIPConfig.PrefixLength), 32)} // nolint
|
||||
for _, ipconf := range endpointInfo.IfnameToIPMap[ipconfigRequest.Ifname].IPv4 {
|
||||
if ipconf.IP.Equal(ipconfig.IP) {
|
||||
logger.Printf("[updateEndpointState] Found existing ipv4 ipconfig for infra container %s", ipconfigRequest.InfraContainerID)
|
||||
return nil
|
||||
if ip.To4() == nil { // is an ipv6 address
|
||||
ipconfig := net.IPNet{IP: ip, Mask: net.CIDRMask(int(podIPInfo[i].PodIPConfig.PrefixLength), 128)} // nolint
|
||||
for _, ipconf := range endpointInfo.IfnameToIPMap[ipconfigsRequest.Ifname].IPv6 {
|
||||
if ipconf.IP.Equal(ipconfig.IP) {
|
||||
logger.Printf("[updateEndpointState] Found existing ipv6 ipconfig for infra container %s", ipconfigsRequest.InfraContainerID)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
endpointInfo.IfnameToIPMap[ipconfigsRequest.Ifname].IPv6 = append(endpointInfo.IfnameToIPMap[ipconfigsRequest.Ifname].IPv6, ipconfig)
|
||||
} else {
|
||||
ipconfig := net.IPNet{IP: ip, Mask: net.CIDRMask(int(podIPInfo[i].PodIPConfig.PrefixLength), 32)} // nolint
|
||||
for _, ipconf := range endpointInfo.IfnameToIPMap[ipconfigsRequest.Ifname].IPv4 {
|
||||
if ipconf.IP.Equal(ipconfig.IP) {
|
||||
logger.Printf("[updateEndpointState] Found existing ipv4 ipconfig for infra container %s", ipconfigsRequest.InfraContainerID)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
endpointInfo.IfnameToIPMap[ipconfigsRequest.Ifname].IPv4 = append(endpointInfo.IfnameToIPMap[ipconfigsRequest.Ifname].IPv4, ipconfig)
|
||||
}
|
||||
endpointInfo.IfnameToIPMap[ipconfigRequest.Ifname].IPv4 = append(endpointInfo.IfnameToIPMap[ipconfigRequest.Ifname].IPv4, ipconfig)
|
||||
}
|
||||
|
||||
service.EndpointState[ipconfigRequest.InfraContainerID] = endpointInfo
|
||||
|
||||
} else {
|
||||
endpointInfo := &EndpointInfo{PodName: podInfo.Name(), PodNamespace: podInfo.Namespace(), IfnameToIPMap: make(map[string]*IPInfo)}
|
||||
ip := net.ParseIP(podIPInfo.PodIPConfig.IPAddress)
|
||||
if ip == nil {
|
||||
logger.Errorf("failed to parse pod ip address %s", podIPInfo.PodIPConfig.IPAddress)
|
||||
return errParsePodIPFailed
|
||||
}
|
||||
ipInfo := &IPInfo{}
|
||||
if ip.To4() == nil { // is an ipv6 address
|
||||
ipconfig := net.IPNet{IP: ip, Mask: net.CIDRMask(int(podIPInfo.PodIPConfig.PrefixLength), 128)} // nolint
|
||||
ipInfo.IPv6 = append(ipInfo.IPv6, ipconfig)
|
||||
service.EndpointState[ipconfigsRequest.InfraContainerID] = endpointInfo
|
||||
} else {
|
||||
ipconfig := net.IPNet{IP: ip, Mask: net.CIDRMask(int(podIPInfo.PodIPConfig.PrefixLength), 32)} // nolint
|
||||
ipInfo.IPv4 = append(ipInfo.IPv4, ipconfig)
|
||||
endpointInfo := &EndpointInfo{PodName: podInfo.Name(), PodNamespace: podInfo.Namespace(), IfnameToIPMap: make(map[string]*IPInfo)}
|
||||
ip := net.ParseIP(podIPInfo[i].PodIPConfig.IPAddress)
|
||||
if ip == nil {
|
||||
logger.Errorf("failed to parse pod ip address %s", podIPInfo[i].PodIPConfig.IPAddress)
|
||||
return errParsePodIPFailed
|
||||
}
|
||||
ipInfo := &IPInfo{}
|
||||
if ip.To4() == nil { // is an ipv6 address
|
||||
ipconfig := net.IPNet{IP: ip, Mask: net.CIDRMask(int(podIPInfo[i].PodIPConfig.PrefixLength), 128)} // nolint
|
||||
ipInfo.IPv6 = append(ipInfo.IPv6, ipconfig)
|
||||
} else {
|
||||
ipconfig := net.IPNet{IP: ip, Mask: net.CIDRMask(int(podIPInfo[i].PodIPConfig.PrefixLength), 32)} // nolint
|
||||
ipInfo.IPv4 = append(ipInfo.IPv4, ipconfig)
|
||||
}
|
||||
endpointInfo.IfnameToIPMap[ipconfigsRequest.Ifname] = ipInfo
|
||||
service.EndpointState[ipconfigsRequest.InfraContainerID] = endpointInfo
|
||||
}
|
||||
endpointInfo.IfnameToIPMap[ipconfigRequest.Ifname] = ipInfo
|
||||
service.EndpointState[ipconfigRequest.InfraContainerID] = endpointInfo
|
||||
}
|
||||
|
||||
err := service.EndpointStateStore.Write(EndpointStoreKey, service.EndpointState)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to write endpoint state to store: %w", err)
|
||||
err := service.EndpointStateStore.Write(EndpointStoreKey, service.EndpointState)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to write endpoint state to store: %w", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// releaseIPConfigHandlerHelper validates the request and removes the endpoint associated with the pod
|
||||
func (service *HTTPRestService) releaseIPConfigHandlerHelper(ipconfigsRequest cns.IPConfigsRequest) (*cns.Response, error) {
|
||||
podInfo, returnCode, returnMessage := service.validateIPConfigsRequest(ipconfigsRequest)
|
||||
if returnCode != types.Success {
|
||||
return &cns.Response{
|
||||
ReturnCode: returnCode,
|
||||
Message: returnMessage,
|
||||
}, fmt.Errorf("failed to validate ip config request") //nolint:goerr113 // return error
|
||||
}
|
||||
// Check if http rest service managed endpoint state is set
|
||||
if service.Options[common.OptManageEndpointState] == true {
|
||||
if err := service.removeEndpointState(podInfo); err != nil {
|
||||
resp := &cns.Response{
|
||||
ReturnCode: types.UnexpectedError,
|
||||
Message: err.Error(),
|
||||
}
|
||||
return resp, fmt.Errorf("releaseIPConfigHandlerHelper remove endpoint state failed because %v, release IP config info %s", resp.Message, ipconfigsRequest) //nolint:goerr113 // return error
|
||||
}
|
||||
}
|
||||
|
||||
if err := service.releaseIPConfigs(podInfo); err != nil {
|
||||
return &cns.Response{
|
||||
ReturnCode: types.UnexpectedError,
|
||||
Message: err.Error(),
|
||||
}, fmt.Errorf("releaseIPConfigHandlerHelper releaseIPConfigs failed because %v, release IP config info %s", returnMessage, ipconfigsRequest) //nolint:goerr113 // return error
|
||||
}
|
||||
|
||||
return &cns.Response{
|
||||
ReturnCode: types.Success,
|
||||
Message: "",
|
||||
}, nil
|
||||
}
|
||||
|
||||
// releaseIPConfigHandler frees the IP assigned to a pod from CNS
|
||||
func (service *HTTPRestService) releaseIPConfigHandler(w http.ResponseWriter, r *http.Request) {
|
||||
var req cns.IPConfigRequest
|
||||
err := service.Listener.Decode(w, r, &req)
|
||||
logger.Request(service.Name+"releaseIPConfigHandler", req, err)
|
||||
var ipconfigRequest cns.IPConfigRequest
|
||||
err := service.Listener.Decode(w, r, &ipconfigRequest)
|
||||
logger.Request(service.Name+"releaseIPConfigHandler", ipconfigRequest, err)
|
||||
if err != nil {
|
||||
resp := cns.Response{
|
||||
ReturnCode: types.UnexpectedError,
|
||||
Message: err.Error(),
|
||||
}
|
||||
logger.Errorf("releaseIPConfigHandler decode failed becase %v, release IP config info %s", resp.Message, req)
|
||||
logger.Errorf("releaseIPConfigHandler decode failed becase %v, release IP config info %s", resp.Message, ipconfigRequest)
|
||||
w.Header().Set(cnsReturnCode, resp.ReturnCode.String())
|
||||
err = service.Listener.Encode(w, &resp)
|
||||
logger.ResponseEx(service.Name, req, resp, resp.ReturnCode, err)
|
||||
logger.ResponseEx(service.Name, ipconfigRequest, resp, resp.ReturnCode, err)
|
||||
return
|
||||
}
|
||||
|
||||
podInfo, returnCode, message := service.validateIPConfigRequest(req)
|
||||
|
||||
// Check if http rest service managed endpoint state is set
|
||||
if service.Options[common.OptManageEndpointState] == true {
|
||||
if err = service.removeEndpointState(podInfo); err != nil {
|
||||
resp := cns.Response{
|
||||
ReturnCode: types.UnexpectedError,
|
||||
Message: err.Error(),
|
||||
}
|
||||
logger.Errorf("releaseIPConfigHandler remove endpoint state failed because %v, release IP config info %s", resp.Message, req)
|
||||
w.Header().Set(cnsReturnCode, resp.ReturnCode.String())
|
||||
err = service.Listener.Encode(w, &resp)
|
||||
logger.ResponseEx(service.Name, req, resp, resp.ReturnCode, err)
|
||||
return
|
||||
// check to make sure there aren't multiple NCs
|
||||
if len(service.state.ContainerStatus) > 1 {
|
||||
reserveResp := &cns.IPConfigResponse{
|
||||
Response: cns.Response{
|
||||
ReturnCode: types.InvalidRequest,
|
||||
Message: fmt.Sprintf("Called API that can only return 1 IP when expecting %d", len(service.state.ContainerStatus)),
|
||||
},
|
||||
}
|
||||
w.Header().Set(cnsReturnCode, reserveResp.Response.ReturnCode.String())
|
||||
err = service.Listener.Encode(w, &reserveResp)
|
||||
logger.ResponseEx(service.Name, ipconfigRequest, reserveResp, reserveResp.Response.ReturnCode, err)
|
||||
return
|
||||
}
|
||||
|
||||
if err = service.releaseIPConfig(podInfo); err != nil {
|
||||
returnCode = types.UnexpectedError
|
||||
message = err.Error()
|
||||
logger.Errorf("releaseIPConfigHandler releaseIPConfig failed because %v, release IP config info %s", message, req)
|
||||
ipconfigsRequest := cns.IPConfigsRequest{
|
||||
DesiredIPAddresses: []string{
|
||||
ipconfigRequest.DesiredIPAddress,
|
||||
},
|
||||
PodInterfaceID: ipconfigRequest.PodInterfaceID,
|
||||
InfraContainerID: ipconfigRequest.InfraContainerID,
|
||||
OrchestratorContext: ipconfigRequest.OrchestratorContext,
|
||||
Ifname: ipconfigRequest.Ifname,
|
||||
}
|
||||
resp := cns.Response{
|
||||
ReturnCode: returnCode,
|
||||
Message: message,
|
||||
|
||||
resp, err := service.releaseIPConfigHandlerHelper(ipconfigsRequest)
|
||||
if err != nil {
|
||||
w.Header().Set(cnsReturnCode, resp.ReturnCode.String())
|
||||
err = service.Listener.Encode(w, &resp)
|
||||
logger.ResponseEx(service.Name, ipconfigRequest, resp, resp.ReturnCode, err)
|
||||
}
|
||||
|
||||
w.Header().Set(cnsReturnCode, resp.ReturnCode.String())
|
||||
err = service.Listener.Encode(w, &resp)
|
||||
logger.ResponseEx(service.Name, req, resp, resp.ReturnCode, err)
|
||||
logger.ResponseEx(service.Name, ipconfigRequest, resp, resp.ReturnCode, err)
|
||||
}
|
||||
|
||||
// releaseIPConfigsHandler frees multiple IPConfigs from the CNS state
|
||||
func (service *HTTPRestService) releaseIPConfigsHandler(w http.ResponseWriter, r *http.Request) {
|
||||
var ipconfigsRequest cns.IPConfigsRequest
|
||||
err := service.Listener.Decode(w, r, &ipconfigsRequest)
|
||||
logger.Request(service.Name+"releaseIPConfigsHandler", ipconfigsRequest, err)
|
||||
if err != nil {
|
||||
resp := cns.Response{
|
||||
ReturnCode: types.UnexpectedError,
|
||||
Message: err.Error(),
|
||||
}
|
||||
logger.Errorf("releaseIPConfigsHandler decode failed because %v, release IP config info %s", resp.Message, ipconfigsRequest)
|
||||
w.Header().Set(cnsReturnCode, resp.ReturnCode.String())
|
||||
err = service.Listener.Encode(w, &resp)
|
||||
logger.ResponseEx(service.Name, ipconfigsRequest, resp, resp.ReturnCode, err)
|
||||
return
|
||||
}
|
||||
|
||||
resp, err := service.releaseIPConfigHandlerHelper(ipconfigsRequest)
|
||||
if err != nil {
|
||||
w.Header().Set(cnsReturnCode, resp.ReturnCode.String())
|
||||
err = service.Listener.Encode(w, &resp)
|
||||
logger.ResponseEx(service.Name, ipconfigsRequest, resp, resp.ReturnCode, err)
|
||||
}
|
||||
|
||||
w.Header().Set(cnsReturnCode, resp.ReturnCode.String())
|
||||
err = service.Listener.Encode(w, &resp)
|
||||
logger.ResponseEx(service.Name, ipconfigsRequest, resp, resp.ReturnCode, err)
|
||||
}
|
||||
|
||||
func (service *HTTPRestService) removeEndpointState(podInfo cns.PodInfo) error {
|
||||
|
|
@ -271,6 +406,7 @@ func (service *HTTPRestService) MarkIPAsPendingRelease(totalIpsToRelease int) (m
|
|||
return pendingReleasedIps, nil
|
||||
}
|
||||
|
||||
// TODO: Add a change so that we should only update the current state if it is different than the new state
|
||||
func (service *HTTPRestService) updateIPConfigState(ipID string, updatedState types.IPState, podInfo cns.PodInfo) (cns.IPConfigurationStatus, error) {
|
||||
if ipConfig, found := service.PodIPConfigState[ipID]; found {
|
||||
logger.Printf("[updateIPConfigState] Changing IpId [%s] state to [%s], podInfo [%+v]. Current config [%+v]", ipID, updatedState, podInfo, ipConfig)
|
||||
|
|
@ -412,7 +548,12 @@ func (service *HTTPRestService) assignIPConfig(ipconfig cns.IPConfigurationStatu
|
|||
return err
|
||||
}
|
||||
|
||||
service.PodIPIDByPodInterfaceKey[podInfo.Key()] = ipconfig.ID
|
||||
if service.PodIPIDByPodInterfaceKey[podInfo.Key()] == nil {
|
||||
logger.Printf("IP config %v initialized", podInfo.Key())
|
||||
service.PodIPIDByPodInterfaceKey[podInfo.Key()] = make([]string, 0)
|
||||
}
|
||||
|
||||
service.PodIPIDByPodInterfaceKey[podInfo.Key()] = append(service.PodIPIDByPodInterfaceKey[podInfo.Key()], ipconfig.ID)
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
@ -431,29 +572,49 @@ func (service *HTTPRestService) unassignIPConfig(ipconfig cns.IPConfigurationSta
|
|||
|
||||
// Todo - CNI should also pass the IPAddress which needs to be released to validate if that is the right IP allcoated
|
||||
// in the first place.
|
||||
func (service *HTTPRestService) releaseIPConfig(podInfo cns.PodInfo) error {
|
||||
func (service *HTTPRestService) releaseIPConfigs(podInfo cns.PodInfo) error {
|
||||
service.Lock()
|
||||
defer service.Unlock()
|
||||
ipsToBeReleased := make([]cns.IPConfigurationStatus, 0)
|
||||
|
||||
ipID := service.PodIPIDByPodInterfaceKey[podInfo.Key()]
|
||||
if ipID != "" {
|
||||
if ipconfig, isExist := service.PodIPConfigState[ipID]; isExist {
|
||||
logger.Printf("[releaseIPConfig] Releasing IP %+v for pod %+v", ipconfig.IPAddress, podInfo)
|
||||
_, err := service.unassignIPConfig(ipconfig, podInfo)
|
||||
if err != nil {
|
||||
return fmt.Errorf("[releaseIPConfig] failed to mark IPConfig [%+v] as Available. err: %v", ipconfig, err)
|
||||
for i, ipID := range service.PodIPIDByPodInterfaceKey[podInfo.Key()] {
|
||||
if ipID != "" {
|
||||
if ipconfig, isExist := service.PodIPConfigState[ipID]; isExist {
|
||||
ipsToBeReleased = append(ipsToBeReleased, ipconfig)
|
||||
} else {
|
||||
//nolint:goerr113 // return error
|
||||
return fmt.Errorf("[releaseIPConfigs] Failed to get ipconfig %+v and pod info is %+v. Pod to IPID exists, but IPID to IPConfig doesn't exist, CNS State potentially corrupt",
|
||||
ipconfig.IPAddress, podInfo)
|
||||
}
|
||||
logger.Printf("[releaseIPConfig] Released IP %+v for pod %+v", ipconfig.IPAddress, podInfo)
|
||||
} else {
|
||||
logger.Errorf("[releaseIPConfig] Failed to get release ipconfig %+v and pod info is %+v. Pod to IPID exists, but IPID to IPConfig doesn't exist, CNS State potentially corrupt",
|
||||
ipconfig.IPAddress, podInfo)
|
||||
return fmt.Errorf("[releaseIPConfig] releaseIPConfig failed. IPconfig %+v and pod info is %+v. Pod to IPID exists, but IPID to IPConfig doesn't exist, CNS State potentially corrupt",
|
||||
ipconfig.IPAddress, podInfo)
|
||||
logger.Errorf("[releaseIPConfigs] releaseIPConfigs could not find ipID at index %d for pod [%+v]", i, podInfo)
|
||||
}
|
||||
} else {
|
||||
logger.Errorf("[releaseIPConfig] SetIPConfigAsAvailable ignoring request to release, no allocation found for pod [%+v]", podInfo)
|
||||
return nil
|
||||
}
|
||||
|
||||
failedToReleaseIP := false
|
||||
for _, ip := range ipsToBeReleased { //nolint:gocritic // ignore copy
|
||||
logger.Printf("[releaseIPConfigs] Releasing IP %s for pod %+v", ip.IPAddress, podInfo)
|
||||
if _, err := service.unassignIPConfig(ip, podInfo); err != nil {
|
||||
logger.Errorf("[releaseIPConfigs] Failed to release IP %s for pod %+v error: %+v", ip.IPAddress, podInfo, err)
|
||||
failedToReleaseIP = true
|
||||
break
|
||||
}
|
||||
|
||||
logger.Printf("[releaseIPConfigs] Released IP %s for pod %+v", ip.IPAddress, podInfo)
|
||||
}
|
||||
|
||||
if failedToReleaseIP {
|
||||
// reassigns all of the released IPs if we aren't able to release all of them
|
||||
for _, ip := range ipsToBeReleased { //nolint:gocritic // ignore copy
|
||||
if err := service.assignIPConfig(ip, podInfo); err != nil {
|
||||
logger.Errorf("[releaseIPConfigs] failed to mark IPConfig [%+v] back to Assigned. err: %v", ip, err)
|
||||
}
|
||||
}
|
||||
//nolint:goerr113 // return error
|
||||
return fmt.Errorf("[releaseIPConfigs] Failed to release one or more IPs. Not releasing any IPs for pod %+v", podInfo)
|
||||
}
|
||||
|
||||
logger.Printf("[releaseIPConfigs] Successfully released all IPs for pod %+v", podInfo)
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
@ -478,101 +639,228 @@ func (service *HTTPRestService) MarkExistingIPsAsPendingRelease(pendingIPIDs []s
|
|||
return nil
|
||||
}
|
||||
|
||||
func (service *HTTPRestService) GetExistingIPConfig(podInfo cns.PodInfo) (cns.PodIpInfo, bool, error) {
|
||||
var (
|
||||
podIpInfo cns.PodIpInfo
|
||||
isExist bool
|
||||
)
|
||||
|
||||
// Returns the current IP configs for a pod if they exist
|
||||
func (service *HTTPRestService) GetExistingIPConfig(podInfo cns.PodInfo) ([]cns.PodIpInfo, bool, error) {
|
||||
service.RLock()
|
||||
defer service.RUnlock()
|
||||
|
||||
ipID := service.PodIPIDByPodInterfaceKey[podInfo.Key()]
|
||||
if ipID != "" {
|
||||
if ipState, isExist := service.PodIPConfigState[ipID]; isExist {
|
||||
err := service.populateIPConfigInfoUntransacted(ipState, &podIpInfo)
|
||||
return podIpInfo, isExist, err
|
||||
}
|
||||
numIPConfigs := len(service.PodIPIDByPodInterfaceKey[podInfo.Key()])
|
||||
podIPInfo := make([]cns.PodIpInfo, numIPConfigs)
|
||||
ipConfigExists := false
|
||||
|
||||
logger.Errorf("Failed to get existing ipconfig. Pod to IPID exists, but IPID to IPConfig doesn't exist, CNS State potentially corrupt")
|
||||
return podIpInfo, isExist, fmt.Errorf("Failed to get existing ipconfig. Pod to IPID exists, but IPID to IPConfig doesn't exist, CNS State potentially corrupt")
|
||||
}
|
||||
|
||||
return podIpInfo, isExist, nil
|
||||
}
|
||||
|
||||
func (service *HTTPRestService) AssignDesiredIPConfig(podInfo cns.PodInfo, desiredIPAddress string) (cns.PodIpInfo, error) {
|
||||
var podIpInfo cns.PodIpInfo
|
||||
service.Lock()
|
||||
defer service.Unlock()
|
||||
|
||||
for _, ipConfig := range service.PodIPConfigState {
|
||||
if ipConfig.IPAddress == desiredIPAddress {
|
||||
switch ipConfig.GetState() { //nolint:exhaustive // ignoring PendingRelease case intentionally
|
||||
case types.Assigned:
|
||||
// This IP has already been assigned, if it is assigned to same pod, then return the same
|
||||
// IPconfiguration
|
||||
if ipConfig.PodInfo.Key() == podInfo.Key() {
|
||||
logger.Printf("[AssignDesiredIPConfig]: IP Config [%+v] is already assigned to this Pod [%+v]", ipConfig, podInfo)
|
||||
} else {
|
||||
return podIpInfo, errors.Errorf("[AssignDesiredIPConfig] Desired IP is already assigned %+v, requested for pod %+v", ipConfig, podInfo)
|
||||
for i, ipID := range service.PodIPIDByPodInterfaceKey[podInfo.Key()] {
|
||||
if ipID != "" {
|
||||
if ipState, isExist := service.PodIPConfigState[ipID]; isExist {
|
||||
if err := service.populateIPConfigInfoUntransacted(ipState, &podIPInfo[i]); err != nil {
|
||||
return podIPInfo, isExist, err
|
||||
}
|
||||
case types.Available, types.PendingProgramming:
|
||||
// This race can happen during restart, where CNS state is lost and thus we have lost the NC programmed version
|
||||
// As part of reconcile, we mark IPs as Assigned which are already assigned to Pods (listed from APIServer)
|
||||
if err := service.assignIPConfig(ipConfig, podInfo); err != nil {
|
||||
return podIpInfo, err
|
||||
}
|
||||
default:
|
||||
return podIpInfo, errors.Errorf("[AllocateDesiredIPConfig] Desired IP is not available %+v", ipConfig)
|
||||
ipConfigExists = true
|
||||
} else {
|
||||
errMsg := fmt.Sprintf("Failed to get existing ipconfig for pod %+v. Pod to IPID exists, but IPID to IPConfig doesn't exist, CNS State potentially corrupt", podInfo)
|
||||
logger.Errorf(errMsg)
|
||||
return podIPInfo, false, errors.New(errMsg)
|
||||
}
|
||||
err := service.populateIPConfigInfoUntransacted(ipConfig, &podIpInfo)
|
||||
return podIpInfo, err
|
||||
}
|
||||
}
|
||||
return podIpInfo, fmt.Errorf("Requested IP not found in pool")
|
||||
|
||||
return podIPInfo, ipConfigExists, nil
|
||||
}
|
||||
|
||||
func (service *HTTPRestService) AssignAnyAvailableIPConfig(podInfo cns.PodInfo) (cns.PodIpInfo, error) {
|
||||
// Assigns a pod with all IPs desired
|
||||
func (service *HTTPRestService) AssignDesiredIPConfigs(podInfo cns.PodInfo, desiredIPAddresses []string) ([]cns.PodIpInfo, error) {
|
||||
numDesiredIPAddresses := len(desiredIPAddresses)
|
||||
podIPInfo := make([]cns.PodIpInfo, numDesiredIPAddresses)
|
||||
// creating a map for the loop to check to see if the IP in the pool is one of the desired IPs
|
||||
desiredIPMap := make(map[string]struct{})
|
||||
// slice to keep track of IP configs to assign
|
||||
ipConfigsToAssign := make([]cns.IPConfigurationStatus, 0)
|
||||
|
||||
service.Lock()
|
||||
defer service.Unlock()
|
||||
|
||||
for _, desiredIP := range desiredIPAddresses {
|
||||
desiredIPMap[desiredIP] = struct{}{}
|
||||
}
|
||||
|
||||
numIPConfigsAssigned := 0
|
||||
for _, ipConfig := range service.PodIPConfigState { //nolint:gocritic // ignore copy
|
||||
_, found := desiredIPMap[ipConfig.IPAddress]
|
||||
// keep searching until the all the desired IPs are found
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
|
||||
switch ipConfig.GetState() { //nolint:exhaustive // ignoring PendingRelease case intentionally
|
||||
case types.Assigned:
|
||||
// This IP has already been assigned, if it is assigned to same pod add the IP to podIPInfo
|
||||
if ipConfig.PodInfo.Key() == podInfo.Key() {
|
||||
logger.Printf("[AssignDesiredIPConfigs]: IP Config [%+v] is already assigned to this Pod [%+v]", ipConfig, podInfo)
|
||||
if err := service.populateIPConfigInfoUntransacted(ipConfig, &podIPInfo[numIPConfigsAssigned]); err != nil {
|
||||
//nolint:goerr113 // return error
|
||||
return []cns.PodIpInfo{}, fmt.Errorf("[AssignDesiredIPConfigs] Failed to assign IP %+v requested for pod %+v since the IP is already assigned to %+v", ipConfig, podInfo, ipConfig.PodInfo)
|
||||
}
|
||||
numIPConfigsAssigned++
|
||||
} else {
|
||||
//nolint:goerr113 // return error
|
||||
return []cns.PodIpInfo{}, fmt.Errorf("[AssignDesiredIPConfigs] Desired IP is already assigned %+v, requested for pod %+v", ipConfig, podInfo)
|
||||
}
|
||||
case types.Available, types.PendingProgramming:
|
||||
// This race can happen during restart, where CNS state is lost and thus we have lost the NC programmed version
|
||||
// As part of reconcile, we mark IPs as Assigned which are already assigned to Pods (listed from APIServer)
|
||||
ipConfigsToAssign = append(ipConfigsToAssign, ipConfig)
|
||||
default:
|
||||
logger.Errorf("[AssignDesiredIPConfigs] Desired IP is not available %+v", ipConfig)
|
||||
//nolint:goerr113 // return error
|
||||
return podIPInfo, fmt.Errorf("IP not available")
|
||||
}
|
||||
|
||||
// checks if found all of the desired IPs either as an available IP or already assigned to the pod
|
||||
if len(ipConfigsToAssign)+numIPConfigsAssigned == numDesiredIPAddresses {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// if we did not find all of the desired IPs return an error
|
||||
if len(ipConfigsToAssign)+numIPConfigsAssigned != numDesiredIPAddresses {
|
||||
//nolint:goerr113 // return error
|
||||
return podIPInfo, fmt.Errorf("not enough desired IPs found in pool")
|
||||
}
|
||||
|
||||
failedToAssignIP := false
|
||||
// assigns all IPs that were found as available to the pod
|
||||
for i := range ipConfigsToAssign {
|
||||
if err := service.assignIPConfig(ipConfigsToAssign[i], podInfo); err != nil {
|
||||
logger.Errorf(err.Error())
|
||||
failedToAssignIP = true
|
||||
break
|
||||
}
|
||||
if err := service.populateIPConfigInfoUntransacted(ipConfigsToAssign[i], &podIPInfo[numIPConfigsAssigned]); err != nil {
|
||||
logger.Errorf(err.Error())
|
||||
failedToAssignIP = true
|
||||
break
|
||||
}
|
||||
// adds to the newly assigned IP to the counter
|
||||
numIPConfigsAssigned++
|
||||
}
|
||||
|
||||
// if we were able to get at least one IP but not all of the desired IPs
|
||||
if failedToAssignIP {
|
||||
logger.Printf("[AssignDesiredIPConfigs] Failed to retrieve all desired IPs. Releasing all IPs that were found")
|
||||
for i := range ipConfigsToAssign {
|
||||
_, err := service.unassignIPConfig(ipConfigsToAssign[i], podInfo)
|
||||
if err != nil {
|
||||
logger.Errorf("[AssignDesiredIPConfigs] failed to mark IPConfig [%+v] back to Available. err: %v", ipConfigsToAssign[i], err)
|
||||
}
|
||||
}
|
||||
//nolint:goerr113 // return error
|
||||
return podIPInfo, fmt.Errorf("not all requested ips %v were found/available in the pool", desiredIPAddresses)
|
||||
}
|
||||
|
||||
logger.Printf("[AssignDesiredIPConfigs] Successfully assigned all desired IPs for pod %+v", podInfo)
|
||||
return podIPInfo, nil
|
||||
}
|
||||
|
||||
// Assigns an available IP from each NC on the NNC. If there is one NC then we expect to only have one IP return
|
||||
// In the case of dualstack we would expect to have one IPv6 from one NC and one IPv4 from a second NC
|
||||
func (service *HTTPRestService) AssignAvailableIPConfigs(podInfo cns.PodInfo) ([]cns.PodIpInfo, error) {
|
||||
service.Lock()
|
||||
defer service.Unlock()
|
||||
// Sets the number of IPs needed equal to the number of NCs so that we can get one IP per NC
|
||||
numIPsNeeded := len(service.state.ContainerStatus)
|
||||
// Creates a slice of PodIpInfo with the size as number of NCs to hold the result for assigned IP configs
|
||||
podIPInfo := make([]cns.PodIpInfo, numIPsNeeded)
|
||||
// This map is used to store whether or not we have found an available IP from an NC when looping through the pool
|
||||
ipsToAssign := make(map[string]cns.IPConfigurationStatus)
|
||||
|
||||
// Searches for available IPs in the pool
|
||||
for _, ipState := range service.PodIPConfigState {
|
||||
if ipState.GetState() == types.Available {
|
||||
if err := service.assignIPConfig(ipState, podInfo); err != nil {
|
||||
return cns.PodIpInfo{}, err
|
||||
}
|
||||
|
||||
podIPInfo := cns.PodIpInfo{}
|
||||
if err := service.populateIPConfigInfoUntransacted(ipState, &podIPInfo); err != nil {
|
||||
return cns.PodIpInfo{}, err
|
||||
}
|
||||
|
||||
return podIPInfo, nil
|
||||
// check if an IP from this NC is already set side for assignment.
|
||||
_, ncAlreadyMarkedForAssignment := ipsToAssign[ipState.NCID]
|
||||
// Checks if we haven't already found an IP from that NC and checks if the current IP is available
|
||||
if ncAlreadyMarkedForAssignment || ipState.GetState() != types.Available {
|
||||
continue
|
||||
}
|
||||
ipsToAssign[ipState.NCID] = ipState
|
||||
// Once one IP per container is found break out of the loop and stop searching
|
||||
if len(ipsToAssign) == numIPsNeeded {
|
||||
break
|
||||
}
|
||||
}
|
||||
//nolint:goerr113
|
||||
return cns.PodIpInfo{}, fmt.Errorf("no IPs available, waiting on Azure CNS to allocate more")
|
||||
|
||||
// Checks to make sure we found one IP for each NC
|
||||
if len(ipsToAssign) != numIPsNeeded {
|
||||
//nolint:goerr113 // return error
|
||||
return podIPInfo, fmt.Errorf("not enough IPs available, waiting on Azure CNS to allocate more")
|
||||
}
|
||||
|
||||
failedToAssignIP := false
|
||||
numIPConfigsAssigned := 0
|
||||
// assigns all IPs in the map to the pod
|
||||
for _, ip := range ipsToAssign { //nolint:gocritic // ignore copy
|
||||
if err := service.assignIPConfig(ip, podInfo); err != nil {
|
||||
logger.Errorf(err.Error())
|
||||
failedToAssignIP = true
|
||||
break
|
||||
}
|
||||
|
||||
if err := service.populateIPConfigInfoUntransacted(ip, &podIPInfo[numIPConfigsAssigned]); err != nil {
|
||||
logger.Errorf(err.Error())
|
||||
failedToAssignIP = true
|
||||
break
|
||||
}
|
||||
numIPConfigsAssigned++
|
||||
}
|
||||
|
||||
// if we were able to find at least one IP but not enough
|
||||
if failedToAssignIP {
|
||||
logger.Printf("[AssignAvailableIPConfigs] failed to assign enough IPs. Releasing all IPs that were found")
|
||||
for _, ipState := range ipsToAssign { //nolint:gocritic // ignore copy
|
||||
_, err := service.unassignIPConfig(ipState, podInfo)
|
||||
if err != nil {
|
||||
logger.Errorf("[AssignAvailableIPConfigs] failed to mark IPConfig [%+v] back to Available. err: %v", ipState, err)
|
||||
}
|
||||
}
|
||||
//nolint:goerr113 // return error
|
||||
return podIPInfo, fmt.Errorf("not enough IPs available, waiting on Azure CNS to allocate more")
|
||||
}
|
||||
|
||||
logger.Printf("[AssignDesiredIPConfigs] Successfully assigned IPs for pod %+v", podInfo)
|
||||
return podIPInfo, nil
|
||||
}
|
||||
|
||||
// If IPConfig is already assigned to pod, it returns that else it returns one of the available ipconfigs.
|
||||
func requestIPConfigHelper(service *HTTPRestService, req cns.IPConfigRequest) (cns.PodIpInfo, error) {
|
||||
// check if ipconfig already assigned tothis pod and return if exists or error
|
||||
// If IPConfigs are already assigned to the pod, it returns that else it returns the available ipconfigs.
|
||||
func requestIPConfigsHelper(service *HTTPRestService, req cns.IPConfigsRequest) ([]cns.PodIpInfo, error) {
|
||||
// check if ipconfigs already assigned to this pod and return if exists or error
|
||||
// if error, ipstate is nil, if exists, ipstate is not nil and error is nil
|
||||
podInfo, err := cns.NewPodInfoFromIPConfigRequest(req)
|
||||
podInfo, err := cns.NewPodInfoFromIPConfigsRequest(req)
|
||||
if err != nil {
|
||||
return cns.PodIpInfo{}, errors.Wrapf(err, "failed to parse IPConfigRequest %v", req)
|
||||
return []cns.PodIpInfo{}, errors.Wrapf(err, "failed to parse IPConfigsRequest %v", req)
|
||||
}
|
||||
|
||||
if podIPInfo, isExist, err := service.GetExistingIPConfig(podInfo); err != nil || isExist {
|
||||
return podIPInfo, err
|
||||
}
|
||||
|
||||
// return desired IPConfig
|
||||
if req.DesiredIPAddress != "" {
|
||||
return service.AssignDesiredIPConfig(podInfo, req.DesiredIPAddress)
|
||||
// if the desired IP configs are not specified, assign any free IPConfigs
|
||||
if len(req.DesiredIPAddresses) == 0 {
|
||||
return service.AssignAvailableIPConfigs(podInfo)
|
||||
}
|
||||
|
||||
// return any free IPConfig
|
||||
return service.AssignAnyAvailableIPConfig(podInfo)
|
||||
if err := validateDesiredIPAddresses(req.DesiredIPAddresses); err != nil {
|
||||
return []cns.PodIpInfo{}, err
|
||||
}
|
||||
|
||||
return service.AssignDesiredIPConfigs(podInfo, req.DesiredIPAddresses)
|
||||
}
|
||||
|
||||
// checks all desired IPs for a request to make sure they are all valid
|
||||
func validateDesiredIPAddresses(desiredIPs []string) error {
|
||||
for _, desiredIP := range desiredIPs {
|
||||
ip := net.ParseIP(desiredIP)
|
||||
if ip.To4() == nil && ip.To16() == nil {
|
||||
//nolint:goerr113 // return error
|
||||
return fmt.Errorf("[validateDesiredIPAddresses] invalid ip %s specified as desired IP", desiredIP)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
Разница между файлами не показана из-за своего большого размера
Загрузить разницу
|
|
@ -60,7 +60,7 @@ type HTTPRestService struct {
|
|||
wsproxy wireserverProxy
|
||||
homeAzMonitor *HomeAzMonitor
|
||||
networkContainer *networkcontainers.NetworkContainers
|
||||
PodIPIDByPodInterfaceKey map[string]string // PodInterfaceId is key and value is Pod IP (SecondaryIP) uuid.
|
||||
PodIPIDByPodInterfaceKey map[string][]string // PodInterfaceId is key and value is slice of Pod IP (SecondaryIP) uuids.
|
||||
PodIPConfigState map[string]cns.IPConfigurationStatus // Secondary IP ID(uuid) is key
|
||||
IPAMPoolMonitor cns.IPAMPoolMonitor
|
||||
routingTable *routes.RoutingTable
|
||||
|
|
@ -108,7 +108,7 @@ type GetHTTPServiceDataResponse struct {
|
|||
|
||||
// HTTPRestServiceData represents in-memory CNS data in the debug API paths.
|
||||
type HTTPRestServiceData struct {
|
||||
PodIPIDByPodInterfaceKey map[string]string // PodInterfaceId is key and value is Pod IP uuid.
|
||||
PodIPIDByPodInterfaceKey map[string][]string // PodInterfaceId is key and value is slice of Pod IP uuids.
|
||||
PodIPConfigState map[string]cns.IPConfigurationStatus // secondaryipid(uuid) is key
|
||||
IPAMPoolMonitor cns.IpamPoolMonitorStateSnapshot
|
||||
}
|
||||
|
|
@ -184,7 +184,7 @@ func NewHTTPRestService(config *common.ServiceConfig, wscli interfaceGetter, wsp
|
|||
primaryInterface: primaryInterface,
|
||||
}
|
||||
|
||||
podIPIDByPodInterfaceKey := make(map[string]string)
|
||||
podIPIDByPodInterfaceKey := make(map[string][]string)
|
||||
podIPConfigState := make(map[string]cns.IPConfigurationStatus)
|
||||
|
||||
if gen == nil {
|
||||
|
|
@ -254,7 +254,9 @@ func (service *HTTPRestService) Init(config *common.ServiceConfig) error {
|
|||
listener.AddHandler(cns.PublishNetworkContainer, service.publishNetworkContainer)
|
||||
listener.AddHandler(cns.UnpublishNetworkContainer, service.unpublishNetworkContainer)
|
||||
listener.AddHandler(cns.RequestIPConfig, newHandlerFuncWithHistogram(service.requestIPConfigHandler, httpRequestLatency))
|
||||
listener.AddHandler(cns.RequestIPConfigs, newHandlerFuncWithHistogram(service.requestIPConfigsHandler, httpRequestLatency))
|
||||
listener.AddHandler(cns.ReleaseIPConfig, newHandlerFuncWithHistogram(service.releaseIPConfigHandler, httpRequestLatency))
|
||||
listener.AddHandler(cns.ReleaseIPConfigs, newHandlerFuncWithHistogram(service.releaseIPConfigsHandler, httpRequestLatency))
|
||||
listener.AddHandler(cns.NmAgentSupportedApisPath, service.nmAgentSupportedApisHandler)
|
||||
listener.AddHandler(cns.PathDebugIPAddresses, service.handleDebugIPAddresses)
|
||||
listener.AddHandler(cns.PathDebugPodContext, service.handleDebugPodContext)
|
||||
|
|
|
|||
|
|
@ -763,21 +763,21 @@ func (service *HTTPRestService) SendNCSnapShotPeriodically(ctx context.Context,
|
|||
}
|
||||
}
|
||||
|
||||
func (service *HTTPRestService) validateIPConfigRequest(
|
||||
ipConfigRequest cns.IPConfigRequest,
|
||||
func (service *HTTPRestService) validateIPConfigsRequest(
|
||||
ipConfigsRequest cns.IPConfigsRequest,
|
||||
) (cns.PodInfo, types.ResponseCode, string) {
|
||||
if service.state.OrchestratorType != cns.KubernetesCRD && service.state.OrchestratorType != cns.Kubernetes {
|
||||
return nil, types.UnsupportedOrchestratorType, "ReleaseIPConfig API supported only for kubernetes orchestrator"
|
||||
}
|
||||
|
||||
if ipConfigRequest.OrchestratorContext == nil {
|
||||
if ipConfigsRequest.OrchestratorContext == nil {
|
||||
return nil,
|
||||
types.EmptyOrchestratorContext,
|
||||
fmt.Sprintf("OrchastratorContext is not set in the req: %+v", ipConfigRequest)
|
||||
fmt.Sprintf("OrchastratorContext is not set in the req: %+v", ipConfigsRequest)
|
||||
}
|
||||
|
||||
// retrieve podinfo from orchestrator context
|
||||
podInfo, err := cns.NewPodInfoFromIPConfigRequest(ipConfigRequest)
|
||||
podInfo, err := cns.NewPodInfoFromIPConfigsRequest(ipConfigsRequest)
|
||||
if err != nil {
|
||||
return podInfo, types.UnsupportedOrchestratorContext, err.Error()
|
||||
}
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче