From 29552c24a9db663fc700c16205d7aeb9d7eb64c2 Mon Sep 17 00:00:00 2001 From: Sundar Date: Thu, 17 Sep 2020 12:22:14 +0530 Subject: [PATCH] Blue green strategy - Refined some details. (#51) * Refined some details. * addressed comments * updates readme * Readme cleanup (#1) * Small updates to readme * added identified service terminology * Adressed PR comments * Added workflow to trigger L2 tests * Renamed workflow file name * Trigger integration tests through script and disable post check-in trigger. Co-authored-by: Anirudh Raghunath <46741940+anraghun@users.noreply.github.com> Co-authored-by: ajinkya599 <11447401+ajinkya599@users.noreply.github.com> --- .github/workflows/TriggerIntegrationTests.sh | 32 ++++++ .github/workflows/integration-tests.yml | 23 ++++ README.md | 78 +++++++++++-- __tests__/blue-green-helper.test.ts | 12 +- lib/actions/promote.js | 8 +- .../strategy-helpers/blue-green-helper.js | 85 +++++--------- .../ingress-blue-green-helper.js | 6 +- .../service-blue-green-helper.js | 45 +++----- .../strategy-helpers/smi-blue-green-helper.js | 98 ++++++++-------- src/actions/promote.ts | 13 ++- .../strategy-helpers/blue-green-helper.ts | 108 ++++++++---------- .../ingress-blue-green-helper.ts | 12 +- .../service-blue-green-helper.ts | 49 +++----- .../strategy-helpers/smi-blue-green-helper.ts | 108 ++++++++---------- 14 files changed, 350 insertions(+), 327 deletions(-) create mode 100644 .github/workflows/TriggerIntegrationTests.sh create mode 100644 .github/workflows/integration-tests.yml diff --git a/.github/workflows/TriggerIntegrationTests.sh b/.github/workflows/TriggerIntegrationTests.sh new file mode 100644 index 00000000..aa9b69c8 --- /dev/null +++ b/.github/workflows/TriggerIntegrationTests.sh @@ -0,0 +1,32 @@ +token=$1 +commit=$2 +repository=$3 +prNumber=$4 +frombranch=$5 +tobranch=$6 + +getPayLoad() { + cat < strategy
Strategy - (Optional) Deployment strategy to be used while applying manifest files on the cluster. Acceptable values: none/canary. none - No deployment strategy is used when deploying. canary - Canary deployment strategy is used when deploying to the cluster + (Optional) Deployment strategy to be used while applying manifest files on the cluster. Acceptable values: none/canary/blue-green. none - No deployment strategy is used when deploying. canary - Canary deployment strategy is used when deploying to the cluster. blue-green - Blue-Green deployment strategy is used when deploying to cluster. traffic-split-method
Traffic split method - (Optional) Acceptable values: pod/smi; Default value: pod
SMI: Percentage traffic split is done at request level using service mesh. Service mesh has to be setup by cluster admin. Orchestration of TrafficSplit objects of SMI is handled by this action.
Pod: Percentage split not possible at request level in the absence of service mesh. So the percentage input is used to calculate the replicas for baseline and canary as a percentage of replicas specified in the input manifests for the stable variant. + (Optional) Acceptable values: pod/smi; Default value: pod
SMI: Percentage traffic split is done at request level using service mesh. Service mesh has to be setup by cluster admin. Orchestration of TrafficSplit objects of SMI is handled by this action.
Pod: Percentage split not possible at request level in the absence of service mesh. So the percentage input is used to calculate the replicas for baseline and canary as a percentage of replicas specified in the input manifests for the stable variant. percentage
Percentage @@ -62,10 +72,21 @@ Following are the key capabilities of this action: baseline-and-canary-replicas
Baseline and canary replicas (Optional; Relevant only if trafficSplitMethod == smi) When trafficSplitMethod == smi, as percentage traffic split is controlled in the service mesh plane, the actual number of replicas for canary and baseline variants could be controlled independently of the traffic split. For example, assume that the input Deployment manifest desired 30 replicas to be used for stable and that the following inputs were specified for the action -
    strategy: canary
    trafficSplitMethod: smi
    percentage: 20
    baselineAndCanaryReplicas: 1
In this case, stable variant will receive 80% traffic while baseline and canary variants will receive 10% each (20% split equally between baseline and canary). However, instead of creating baseline and canary with 3 replicas, the explicit count of baseline and canary replicas is honored. That is, only 1 replica each is created for baseline and canary variants. + + + route-method
Route Method + (Optional; Relevant only if strategy==blue-green) Default value: service. Acceptable values: service/ingress/smi. Traffic is routed based on this input. +
Service: Service selector labels are updated to target '-green' workloads. +
Ingress: Ingress backends are updated to target the new '-green' services which in turn target '-green' deployments. +
SMI: A TrafficSplit object is created for each required service to route traffic to new workloads. + + + version-switch-buffer
Version Switch Buffer + (Optional; Relevant only if strategy==blue-green and action == deploy) Default value: 0. Acceptable values: 1-300. Waits for the given input in minutes before routing traffic to '-green' workloads. action
Action - (Required) Default value: deploy. Acceptable values: deploy/promote/reject. Promote or reject actions are used to promote or reject canary deployments. Sample YAML snippets are provided below for guidance on how to use the same. + (Required) Default value: deploy. Acceptable values: deploy/promote/reject. Promote or reject actions are used to promote or reject canary/blue-green deployments. Sample YAML snippets are provided below for guidance on how to use the same. kubectl-version
Kubectl version @@ -112,7 +133,7 @@ Following are the key capabilities of this action: percentage: 20 ``` -To promote/reject the canary created by the above snippet, the following YAML snippet could be used: +### To promote/reject the canary created by the above snippet, the following YAML snippet could be used: ```yaml - uses: Azure/k8s-deploy@v1 @@ -147,9 +168,7 @@ To promote/reject the canary created by the above snippet, the following YAML sn percentage: 20 baseline-and-canary-replicas: 1 ``` - -To promote/reject the canary created by the above snippet, the following YAML snippet could be used: - +### To promote/reject the canary created by the above snippet, the following YAML snippet could be used: ```yaml - uses: Azure/k8s-deploy@v1 with: @@ -165,6 +184,43 @@ To promote/reject the canary created by the above snippet, the following YAML sn traffic-split-method: smi action: reject # substitute reject if you want to reject ``` +### Deployment Strategies - Blue-Green deployment with different route methods + +```yaml +- uses: Azure/k8s-deploy@v1 + with: + namespace: 'myapp' + images: 'contoso.azurecr.io/myapp:${{ event.run_id }}' + imagepullsecrets: | + image-pull-secret1 + image-pull-secret2 + manifests: | + deployment.yaml + service.yaml + ingress.yml + strategy: blue-green + route-method: ingress # substitute with service/smi as per need + version-switch-buffer: 15 +``` + +### **To promote/reject the green workload created by the above snippet, the following YAML snippet could be used:** + +```yaml +- uses: Azure/k8s-deploy@v1 + with: + namespace: 'myapp' + images: 'contoso.azurecr.io/myapp:${{ event.run_id }}' + imagepullsecrets: | + image-pull-secret1 + image-pull-secret2 + manifests: | + deployment.yaml + service.yaml + ingress-yml + strategy: blue-green + strategy: ingress # should be the same as the value when action was deploy + action: promote # substitute reject if you want to reject +``` ## End to end workflows @@ -271,4 +327,4 @@ provided by the bot. You will only need to do this once across all repos using o This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or -contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. +contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. \ No newline at end of file diff --git a/__tests__/blue-green-helper.test.ts b/__tests__/blue-green-helper.test.ts index 5a74da87..7728b214 100644 --- a/__tests__/blue-green-helper.test.ts +++ b/__tests__/blue-green-helper.test.ts @@ -131,7 +131,6 @@ test("blueGreenReject - routes servcies to old deployment and deletes new deploy expect(kubeCtl.delete).toBeCalledWith(["Deployment", "testapp-green"]); expect(readFileSpy).toBeCalledWith("manifests/bg.yaml"); expect(fileHelperMock.writeObjectsToFile).toBeCalled(); - expect(kubeCtl.getResource).toBeCalledWith("Deployment", "testapp"); }); test("blueGreenReject - deletes services if old deployment does not exist", () => { @@ -149,10 +148,8 @@ test("blueGreenReject - deletes services if old deployment does not exist", () = //Invoke and assert expect(blueGreenHelperService.rejectBlueGreenService(kubeCtl, ['manifests/bg.yaml'])).toMatchObject({}); expect(kubeCtl.delete).toBeCalledWith(["Deployment", "testapp-green"]); - expect(kubeCtl.delete).toBeCalledWith(["Service", "testservice"]); expect(readFileSpy).toBeCalledWith("manifests/bg.yaml"); expect(fileHelperMock.writeObjectsToFile).toBeCalled(); - expect(kubeCtl.getResource).toBeCalledWith("Deployment", "testapp"); }); test("isIngressRoute() - returns true if route-method is ingress", () => { @@ -365,7 +362,6 @@ test("blueGreenRejectSMI - routes servcies to old deployment and deletes new dep expect(kubeCtl.delete).toBeCalledWith(["Service", "testservice-stable"]); expect(kubeCtl.delete).toBeCalledWith(["TrafficSplit", "testservice-trafficsplit"]); expect(readFileSpy).toBeCalledWith("manifests/bg.yaml"); - expect(kubeCtl.getResource).toBeCalledWith("Deployment", "testapp"); }); test("blueGreenRejectSMI - deletes service if stable deployment doesn't exist", () => { @@ -383,12 +379,10 @@ test("blueGreenRejectSMI - deletes service if stable deployment doesn't exist", //Invoke and assert expect(blueGreenHelperSMI.rejectBlueGreenSMI(kubeCtl, ['manifests/bg.yaml'])).toMatchObject({}); expect(kubeCtl.delete).toBeCalledWith(["Deployment", "testapp-green"]); - expect(kubeCtl.delete).toBeCalledWith(["Service", "testservice"]); expect(kubeCtl.delete).toBeCalledWith(["Service", "testservice-green"]); expect(kubeCtl.delete).toBeCalledWith(["Service", "testservice-stable"]); expect(kubeCtl.delete).toBeCalledWith(["TrafficSplit", "testservice-trafficsplit"]); expect(readFileSpy).toBeCalledWith("manifests/bg.yaml"); - expect(kubeCtl.getResource).toBeCalledWith("Deployment", "testapp"); }); // other functions and branches @@ -478,7 +472,7 @@ test("blueGreenRouteIngress - routes to green services in nextlabel is green", ( kubeCtl.apply = jest.fn().mockReturnValue(''); //Invoke and assert - expect(blueGreenHelperIngress.routeBlueGreenIngress(kubeCtl, 'green', serviceEntityMap, serEntList, ingEntList)); + expect(blueGreenHelperIngress.routeBlueGreenIngress(kubeCtl, 'green', serviceEntityMap, ingEntList)); expect(kubeCtl.apply).toBeCalled(); expect(fileHelperMock.writeObjectsToFile).toBeCalled(); }); @@ -599,7 +593,7 @@ test("validateTrafficSplitState - throws if trafficsplit in wrong state", () => kubeCtl.getResource = jest.fn().mockReturnValue(JSON.parse(JSON.stringify(temp))); //Invoke and assert - expect(blueGreenHelperSMI.validateTrafficSplitsState(kubeCtl, depEntList, serEntList)).toBeFalsy(); + expect(blueGreenHelperSMI.validateTrafficSplitsState(kubeCtl, serEntList)).toBeFalsy(); }); test("validateTrafficSplitState - throws if trafficsplit in wrong state", () => { @@ -678,7 +672,7 @@ test("validateTrafficSplitState - throws if trafficsplit in wrong state", () => kubeCtl.getResource = jest.fn().mockReturnValue(JSON.parse(JSON.stringify(temp))); //Invoke and assert - expect(blueGreenHelperSMI.validateTrafficSplitsState(kubeCtl, depEntList, serEntList)).toBeFalsy(); + expect(blueGreenHelperSMI.validateTrafficSplitsState(kubeCtl, serEntList)).toBeFalsy(); }); test("getSuffix() - returns BLUE_GREEN_SUFFIX if BLUE_GREEN_NEW_LABEL_VALUE is given, else emrty string", () => { diff --git a/lib/actions/promote.js b/lib/actions/promote.js index e1ee4678..582afa72 100644 --- a/lib/actions/promote.js +++ b/lib/actions/promote.js @@ -91,16 +91,16 @@ function promoteBlueGreen(kubectl) { yield KubernetesManifestUtility.checkManifestStability(kubectl, resources); core.debug('routing to new deployments'); if (blue_green_helper_2.isIngressRoute()) { - ingress_blue_green_helper_1.routeBlueGreenIngress(kubectl, null, manifestObjects.serviceNameMap, manifestObjects.serviceEntityList, manifestObjects.ingressEntityList); + ingress_blue_green_helper_1.routeBlueGreenIngress(kubectl, null, manifestObjects.serviceNameMap, manifestObjects.ingressEntityList); blue_green_helper_1.deleteWorkloadsAndServicesWithLabel(kubectl, blue_green_helper_2.GREEN_LABEL_VALUE, manifestObjects.deploymentEntityList, manifestObjects.serviceEntityList); } else if (blue_green_helper_2.isSMIRoute()) { - smi_blue_green_helper_1.routeBlueGreenSMI(kubectl, blue_green_helper_2.NONE_LABEL_VALUE, manifestObjects.deploymentEntityList, manifestObjects.serviceEntityList); + smi_blue_green_helper_1.routeBlueGreenSMI(kubectl, blue_green_helper_2.NONE_LABEL_VALUE, manifestObjects.serviceEntityList); blue_green_helper_1.deleteWorkloadsWithLabel(kubectl, blue_green_helper_2.GREEN_LABEL_VALUE, manifestObjects.deploymentEntityList); - smi_blue_green_helper_1.cleanupSMI(kubectl, manifestObjects.deploymentEntityList, manifestObjects.serviceEntityList); + smi_blue_green_helper_1.cleanupSMI(kubectl, manifestObjects.serviceEntityList); } else { - service_blue_green_helper_1.routeBlueGreenService(kubectl, blue_green_helper_2.NONE_LABEL_VALUE, manifestObjects.deploymentEntityList, manifestObjects.serviceEntityList); + service_blue_green_helper_1.routeBlueGreenService(kubectl, blue_green_helper_2.NONE_LABEL_VALUE, manifestObjects.serviceEntityList); blue_green_helper_1.deleteWorkloadsWithLabel(kubectl, blue_green_helper_2.GREEN_LABEL_VALUE, manifestObjects.deploymentEntityList); } }); diff --git a/lib/utilities/strategy-helpers/blue-green-helper.js b/lib/utilities/strategy-helpers/blue-green-helper.js index 2818f59f..9cb409fd 100644 --- a/lib/utilities/strategy-helpers/blue-green-helper.js +++ b/lib/utilities/strategy-helpers/blue-green-helper.js @@ -9,7 +9,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge }); }; Object.defineProperty(exports, "__esModule", { value: true }); -exports.fetchResource = exports.isServiceSelectorSubsetOfMatchLabel = exports.getServiceSelector = exports.getDeploymentMatchLabels = exports.getSpecLabel = exports.getBlueGreenResourceName = exports.addBlueGreenLabelsAndAnnotations = exports.getNewBlueGreenObject = exports.createWorkloadsWithLabel = exports.isServiceRouted = exports.getManifestObjects = exports.getSuffix = exports.deleteObjects = exports.deleteWorkloadsAndServicesWithLabel = exports.cleanUp = exports.deleteWorkloadsWithLabel = exports.routeBlueGreen = exports.isSMIRoute = exports.isIngressRoute = exports.isBlueGreenDeploymentStrategy = exports.STABLE_SUFFIX = exports.GREEN_SUFFIX = exports.BLUE_GREEN_VERSION_LABEL = exports.NONE_LABEL_VALUE = exports.GREEN_LABEL_VALUE = exports.BLUE_GREEN_DEPLOYMENT_STRATEGY = void 0; +exports.fetchResource = exports.isServiceSelectorSubsetOfMatchLabel = exports.getServiceSelector = exports.getDeploymentMatchLabels = exports.getBlueGreenResourceName = exports.addBlueGreenLabelsAndAnnotations = exports.getNewBlueGreenObject = exports.createWorkloadsWithLabel = exports.isServiceRouted = exports.getManifestObjects = exports.getSuffix = exports.deleteObjects = exports.deleteWorkloadsAndServicesWithLabel = exports.deleteWorkloadsWithLabel = exports.routeBlueGreen = exports.isSMIRoute = exports.isIngressRoute = exports.isBlueGreenDeploymentStrategy = exports.STABLE_SUFFIX = exports.GREEN_SUFFIX = exports.BLUE_GREEN_VERSION_LABEL = exports.NONE_LABEL_VALUE = exports.GREEN_LABEL_VALUE = exports.BLUE_GREEN_DEPLOYMENT_STRATEGY = void 0; const core = require("@actions/core"); const fs = require("fs"); const yaml = require("js-yaml"); @@ -50,22 +50,22 @@ function routeBlueGreen(kubectl, inputManifestFiles) { let bufferTime = parseInt(TaskInputParameters.versionSwitchBuffer); //logging start of buffer time let dateNow = new Date(); - console.log('starting buffer time of ' + bufferTime + ' minute/s at ' + dateNow.toISOString() + ' UTC'); + console.log(`Starting buffer time of ${bufferTime} minute(s) at ${dateNow.toISOString()}`); // waiting yield utility_1.sleep(bufferTime * 1000 * 60); // logging end of buffer time dateNow = new Date(); - console.log('stopping buffer time of ' + bufferTime + ' minute/s at ' + dateNow.toISOString() + ' UTC'); + console.log(`Stopping buffer time of ${bufferTime} minute(s) at ${dateNow.toISOString()}`); const manifestObjects = getManifestObjects(inputManifestFiles); // routing to new deployments if (isIngressRoute()) { - ingress_blue_green_helper_1.routeBlueGreenIngress(kubectl, exports.GREEN_LABEL_VALUE, manifestObjects.serviceNameMap, manifestObjects.serviceEntityList, manifestObjects.ingressEntityList); + ingress_blue_green_helper_1.routeBlueGreenIngress(kubectl, exports.GREEN_LABEL_VALUE, manifestObjects.serviceNameMap, manifestObjects.ingressEntityList); } else if (isSMIRoute()) { - smi_blue_green_helper_1.routeBlueGreenSMI(kubectl, exports.GREEN_LABEL_VALUE, manifestObjects.deploymentEntityList, manifestObjects.serviceEntityList); + smi_blue_green_helper_1.routeBlueGreenSMI(kubectl, exports.GREEN_LABEL_VALUE, manifestObjects.serviceEntityList); } else { - service_blue_green_helper_1.routeBlueGreenService(kubectl, exports.GREEN_LABEL_VALUE, manifestObjects.deploymentEntityList, manifestObjects.serviceEntityList); + service_blue_green_helper_1.routeBlueGreenService(kubectl, exports.GREEN_LABEL_VALUE, manifestObjects.serviceEntityList); } }); } @@ -90,26 +90,6 @@ function deleteWorkloadsWithLabel(kubectl, deleteLabel, deploymentEntityList) { deleteObjects(kubectl, resourcesToDelete); } exports.deleteWorkloadsWithLabel = deleteWorkloadsWithLabel; -function cleanUp(kubectl, deploymentEntityList, serviceEntityList) { - // checks if services has some stable deployments to target or deletes them too - let deleteList = []; - deploymentEntityList.forEach((deploymentObject) => { - const existingDeploy = fetchResource(kubectl, deploymentObject.kind, deploymentObject.metadata.name); - if (!existingDeploy) { - serviceEntityList.forEach((serviceObject) => { - const serviceSelector = getServiceSelector(serviceObject); - const matchLabels = getDeploymentMatchLabels(deploymentObject); - if (!!serviceSelector && !!matchLabels && isServiceSelectorSubsetOfMatchLabel(serviceSelector, matchLabels)) { - const resourceToDelete = { name: serviceObject.metadata.name, kind: serviceObject.kind }; - deleteList.push(resourceToDelete); - } - }); - } - }); - // delete service not targeting a deployment - deleteObjects(kubectl, deleteList); -} -exports.cleanUp = cleanUp; function deleteWorkloadsAndServicesWithLabel(kubectl, deleteLabel, deploymentEntityList, serviceEntityList) { // need to delete services and deployments const deletionEntitiesList = deploymentEntityList.concat(serviceEntityList); @@ -156,19 +136,28 @@ exports.getSuffix = getSuffix; // other common functions function getManifestObjects(filePaths) { const deploymentEntityList = []; - const serviceEntityList = []; + const routedServiceEntityList = []; + const unroutedServiceEntityList = []; const ingressEntityList = []; const otherEntitiesList = []; + let serviceNameMap = new Map(); filePaths.forEach((filePath) => { const fileContents = fs.readFileSync(filePath); yaml.safeLoadAll(fileContents, function (inputObject) { if (!!inputObject) { const kind = inputObject.kind; + const name = inputObject.metadata.name; if (helper.isDeploymentEntity(kind)) { deploymentEntityList.push(inputObject); } else if (helper.isServiceEntity(kind)) { - serviceEntityList.push(inputObject); + if (isServiceRouted(inputObject, deploymentEntityList)) { + routedServiceEntityList.push(inputObject); + serviceNameMap.set(name, getBlueGreenResourceName(name, exports.GREEN_SUFFIX)); + } + else { + unroutedServiceEntityList.push(inputObject); + } } else if (helper.isIngressEntity(kind)) { ingressEntityList.push(inputObject); @@ -179,27 +168,20 @@ function getManifestObjects(filePaths) { } }); }); - let serviceNameMap = new Map(); - // find all services and add their names with blue green suffix - serviceEntityList.forEach(inputObject => { - const name = inputObject.metadata.name; - serviceNameMap.set(name, getBlueGreenResourceName(name, exports.GREEN_SUFFIX)); - }); - return { serviceEntityList: serviceEntityList, serviceNameMap: serviceNameMap, deploymentEntityList: deploymentEntityList, ingressEntityList: ingressEntityList, otherObjects: otherEntitiesList }; + return { serviceEntityList: routedServiceEntityList, serviceNameMap: serviceNameMap, unroutedServiceEntityList: unroutedServiceEntityList, deploymentEntityList: deploymentEntityList, ingressEntityList: ingressEntityList, otherObjects: otherEntitiesList }; } exports.getManifestObjects = getManifestObjects; function isServiceRouted(serviceObject, deploymentEntityList) { let shouldBeRouted = false; const serviceSelector = getServiceSelector(serviceObject); if (!!serviceSelector) { - deploymentEntityList.every((depObject) => { + if (deploymentEntityList.some((depObject) => { // finding if there is a deployment in the given manifests the service targets const matchLabels = getDeploymentMatchLabels(depObject); - if (!!matchLabels && isServiceSelectorSubsetOfMatchLabel(serviceSelector, matchLabels)) { - shouldBeRouted = true; - return false; - } - }); + return (!!matchLabels && isServiceSelectorSubsetOfMatchLabel(serviceSelector, matchLabels)); + })) { + shouldBeRouted = true; + } } return shouldBeRouted; } @@ -245,38 +227,31 @@ function getBlueGreenResourceName(name, suffix) { return `${name}${suffix}`; } exports.getBlueGreenResourceName = getBlueGreenResourceName; -function getSpecLabel(inputObject) { - if (!!inputObject && inputObject.spec && inputObject.spec.selector && inputObject.spec.selector.matchLabels && inputObject.spec.selector.matchLabels[exports.BLUE_GREEN_VERSION_LABEL]) { - return inputObject.spec.selector.matchLabels[exports.BLUE_GREEN_VERSION_LABEL]; - } - return ''; -} -exports.getSpecLabel = getSpecLabel; function getDeploymentMatchLabels(deploymentObject) { if (!!deploymentObject && deploymentObject.kind.toUpperCase() == constants_1.KubernetesWorkload.pod.toUpperCase() && !!deploymentObject.metadata && !!deploymentObject.metadata.labels) { - return JSON.stringify(deploymentObject.metadata.labels); + return deploymentObject.metadata.labels; } else if (!!deploymentObject && deploymentObject.spec && deploymentObject.spec.selector && deploymentObject.spec.selector.matchLabels) { - return JSON.stringify(deploymentObject.spec.selector.matchLabels); + return deploymentObject.spec.selector.matchLabels; } - return ''; + return null; } exports.getDeploymentMatchLabels = getDeploymentMatchLabels; function getServiceSelector(serviceObject) { if (!!serviceObject && serviceObject.spec && serviceObject.spec.selector) { - return JSON.stringify(serviceObject.spec.selector); + return serviceObject.spec.selector; } else - return ''; + return null; } exports.getServiceSelector = getServiceSelector; function isServiceSelectorSubsetOfMatchLabel(serviceSelector, matchLabels) { let serviceSelectorMap = new Map(); let matchLabelsMap = new Map(); - JSON.parse(serviceSelector, (key, value) => { + JSON.parse(JSON.stringify(serviceSelector), (key, value) => { serviceSelectorMap.set(key, value); }); - JSON.parse(matchLabels, (key, value) => { + JSON.parse(JSON.stringify(matchLabels), (key, value) => { matchLabelsMap.set(key, value); }); let isMatch = true; diff --git a/lib/utilities/strategy-helpers/ingress-blue-green-helper.js b/lib/utilities/strategy-helpers/ingress-blue-green-helper.js index d8c1da21..345df7f1 100644 --- a/lib/utilities/strategy-helpers/ingress-blue-green-helper.js +++ b/lib/utilities/strategy-helpers/ingress-blue-green-helper.js @@ -28,7 +28,7 @@ function deployBlueGreenIngress(kubectl, filePaths) { core.debug('New blue-green object is: ' + JSON.stringify(newBlueGreenObject)); newObjectsList.push(newBlueGreenObject); }); - newObjectsList = newObjectsList.concat(manifestObjects.otherObjects); + newObjectsList = newObjectsList.concat(manifestObjects.otherObjects).concat(manifestObjects.unroutedServiceEntityList); const manifestFiles = fileHelper.writeObjectsToFile(newObjectsList); kubectl.apply(manifestFiles); // return results to check for manifest stability @@ -62,13 +62,13 @@ function rejectBlueGreenIngress(kubectl, filePaths) { // get all kubernetes objects defined in manifest files const manifestObjects = blue_green_helper_1.getManifestObjects(filePaths); // routing ingress to stables services - routeBlueGreenIngress(kubectl, null, manifestObjects.serviceNameMap, manifestObjects.serviceEntityList, manifestObjects.ingressEntityList); + routeBlueGreenIngress(kubectl, null, manifestObjects.serviceNameMap, manifestObjects.ingressEntityList); // deleting green services and deployments blue_green_helper_1.deleteWorkloadsAndServicesWithLabel(kubectl, blue_green_helper_2.GREEN_LABEL_VALUE, manifestObjects.deploymentEntityList, manifestObjects.serviceEntityList); }); } exports.rejectBlueGreenIngress = rejectBlueGreenIngress; -function routeBlueGreenIngress(kubectl, nextLabel, serviceNameMap, serviceEntityList, ingressEntityList) { +function routeBlueGreenIngress(kubectl, nextLabel, serviceNameMap, ingressEntityList) { let newObjectsList = []; if (!nextLabel) { newObjectsList = newObjectsList.concat(ingressEntityList); diff --git a/lib/utilities/strategy-helpers/service-blue-green-helper.js b/lib/utilities/strategy-helpers/service-blue-green-helper.js index b324e130..f51e67f4 100644 --- a/lib/utilities/strategy-helpers/service-blue-green-helper.js +++ b/lib/utilities/strategy-helpers/service-blue-green-helper.js @@ -19,7 +19,7 @@ function deployBlueGreenService(kubectl, filePaths) { // create deployments with green label value const result = blue_green_helper_1.createWorkloadsWithLabel(kubectl, manifestObjects.deploymentEntityList, blue_green_helper_2.GREEN_LABEL_VALUE); // create other non deployment and non service entities - const newObjectsList = manifestObjects.otherObjects.concat(manifestObjects.ingressEntityList); + const newObjectsList = manifestObjects.otherObjects.concat(manifestObjects.ingressEntityList).concat(manifestObjects.unroutedServiceEntityList); const manifestFiles = fileHelper.writeObjectsToFile(newObjectsList); kubectl.apply(manifestFiles); // returning deployment details to check for rollout stability @@ -29,7 +29,7 @@ exports.deployBlueGreenService = deployBlueGreenService; function promoteBlueGreenService(kubectl, manifestObjects) { return __awaiter(this, void 0, void 0, function* () { // checking if services are in the right state ie. targeting green deployments - if (!validateServicesState(kubectl, manifestObjects.deploymentEntityList, manifestObjects.serviceEntityList)) { + if (!validateServicesState(kubectl, manifestObjects.serviceEntityList)) { throw ('NotInPromoteState'); } // creating stable deployments with new configurations @@ -44,26 +44,17 @@ function rejectBlueGreenService(kubectl, filePaths) { // get all kubernetes objects defined in manifest files const manifestObjects = blue_green_helper_1.getManifestObjects(filePaths); // routing to stable objects - routeBlueGreenService(kubectl, blue_green_helper_2.NONE_LABEL_VALUE, manifestObjects.deploymentEntityList, manifestObjects.serviceEntityList); - // seeing if we should even delete the service - blue_green_helper_1.cleanUp(kubectl, manifestObjects.deploymentEntityList, manifestObjects.serviceEntityList); + routeBlueGreenService(kubectl, blue_green_helper_2.NONE_LABEL_VALUE, manifestObjects.serviceEntityList); // deleting the new deployments with green suffix blue_green_helper_1.deleteWorkloadsWithLabel(kubectl, blue_green_helper_2.GREEN_LABEL_VALUE, manifestObjects.deploymentEntityList); }); } exports.rejectBlueGreenService = rejectBlueGreenService; -function routeBlueGreenService(kubectl, nextLabel, deploymentEntityList, serviceEntityList) { +function routeBlueGreenService(kubectl, nextLabel, serviceEntityList) { const newObjectsList = []; serviceEntityList.forEach((serviceObject) => { - if (blue_green_helper_1.isServiceRouted(serviceObject, deploymentEntityList)) { - // if service is routed, point it to given label - const newBlueGreenServiceObject = getUpdatedBlueGreenService(serviceObject, nextLabel); - newObjectsList.push(newBlueGreenServiceObject); - } - else { - // if service is not routed, just push the original service - newObjectsList.push(serviceObject); - } + const newBlueGreenServiceObject = getUpdatedBlueGreenService(serviceObject, nextLabel); + newObjectsList.push(newBlueGreenServiceObject); }); // configures the services const manifestFiles = fileHelper.writeObjectsToFile(newObjectsList); @@ -77,24 +68,22 @@ function getUpdatedBlueGreenService(inputObject, labelValue) { blue_green_helper_1.addBlueGreenLabelsAndAnnotations(newObject, labelValue); return newObject; } -function validateServicesState(kubectl, deploymentEntityList, serviceEntityList) { +function validateServicesState(kubectl, serviceEntityList) { let areServicesGreen = true; serviceEntityList.forEach((serviceObject) => { - if (blue_green_helper_1.isServiceRouted(serviceObject, deploymentEntityList)) { - // finding the existing routed service - const existingService = blue_green_helper_1.fetchResource(kubectl, serviceObject.kind, serviceObject.metadata.name); - if (!!existingService) { - let currentLabel = getServiceSpecLabel(existingService); - if (currentLabel != blue_green_helper_2.GREEN_LABEL_VALUE) { - // service should be targeting deployments with green label - areServicesGreen = false; - } - } - else { - // service targeting deployment doesn't exist + // finding the existing routed service + const existingService = blue_green_helper_1.fetchResource(kubectl, serviceObject.kind, serviceObject.metadata.name); + if (!!existingService) { + let currentLabel = getServiceSpecLabel(existingService); + if (currentLabel != blue_green_helper_2.GREEN_LABEL_VALUE) { + // service should be targeting deployments with green label areServicesGreen = false; } } + else { + // service targeting deployment doesn't exist + areServicesGreen = false; + } }); return areServicesGreen; } diff --git a/lib/utilities/strategy-helpers/smi-blue-green-helper.js b/lib/utilities/strategy-helpers/smi-blue-green-helper.js index e1db680f..82683621 100644 --- a/lib/utilities/strategy-helpers/smi-blue-green-helper.js +++ b/lib/utilities/strategy-helpers/smi-blue-green-helper.js @@ -23,11 +23,11 @@ function deployBlueGreenSMI(kubectl, filePaths) { // get all kubernetes objects defined in manifest files const manifestObjects = blue_green_helper_1.getManifestObjects(filePaths); // creating services and other objects - const newObjectsList = manifestObjects.otherObjects.concat(manifestObjects.serviceEntityList).concat(manifestObjects.ingressEntityList); + const newObjectsList = manifestObjects.otherObjects.concat(manifestObjects.serviceEntityList).concat(manifestObjects.ingressEntityList).concat(manifestObjects.unroutedServiceEntityList); const manifestFiles = fileHelper.writeObjectsToFile(newObjectsList); kubectl.apply(manifestFiles); // make extraservices and trafficsplit - setupSMI(kubectl, manifestObjects.deploymentEntityList, manifestObjects.serviceEntityList); + setupSMI(kubectl, manifestObjects.serviceEntityList); // create new deloyments const result = blue_green_helper_1.createWorkloadsWithLabel(kubectl, manifestObjects.deploymentEntityList, blue_green_helper_2.GREEN_LABEL_VALUE); // return results to check for manifest stability @@ -37,7 +37,7 @@ exports.deployBlueGreenSMI = deployBlueGreenSMI; function promoteBlueGreenSMI(kubectl, manifestObjects) { return __awaiter(this, void 0, void 0, function* () { // checking if there is something to promote - if (!validateTrafficSplitsState(kubectl, manifestObjects.deploymentEntityList, manifestObjects.serviceEntityList)) { + if (!validateTrafficSplitsState(kubectl, manifestObjects.serviceEntityList)) { throw ('NotInPromoteStateSMI'); } // create stable deployments with new configuration @@ -52,29 +52,25 @@ function rejectBlueGreenSMI(kubectl, filePaths) { // get all kubernetes objects defined in manifest files const manifestObjects = blue_green_helper_1.getManifestObjects(filePaths); // routing trafficsplit to stable deploymetns - routeBlueGreenSMI(kubectl, blue_green_helper_2.NONE_LABEL_VALUE, manifestObjects.deploymentEntityList, manifestObjects.serviceEntityList); - // deciding whether to delete services or not - blue_green_helper_1.cleanUp(kubectl, manifestObjects.deploymentEntityList, manifestObjects.serviceEntityList); + routeBlueGreenSMI(kubectl, blue_green_helper_2.NONE_LABEL_VALUE, manifestObjects.serviceEntityList); // deleting rejected new bluegreen deplyments blue_green_helper_1.deleteWorkloadsWithLabel(kubectl, blue_green_helper_2.GREEN_LABEL_VALUE, manifestObjects.deploymentEntityList); //deleting trafficsplit and extra services - cleanupSMI(kubectl, manifestObjects.deploymentEntityList, manifestObjects.serviceEntityList); + cleanupSMI(kubectl, manifestObjects.serviceEntityList); }); } exports.rejectBlueGreenSMI = rejectBlueGreenSMI; -function setupSMI(kubectl, deploymentEntityList, serviceEntityList) { +function setupSMI(kubectl, serviceEntityList) { const newObjectsList = []; const trafficObjectList = []; serviceEntityList.forEach((serviceObject) => { - if (blue_green_helper_1.isServiceRouted(serviceObject, deploymentEntityList)) { - // create a trafficsplit for service - trafficObjectList.push(serviceObject); - // setting up the services for trafficsplit - const newStableService = getSMIServiceResource(serviceObject, blue_green_helper_2.STABLE_SUFFIX); - const newGreenService = getSMIServiceResource(serviceObject, blue_green_helper_2.GREEN_SUFFIX); - newObjectsList.push(newStableService); - newObjectsList.push(newGreenService); - } + // create a trafficsplit for service + trafficObjectList.push(serviceObject); + // setting up the services for trafficsplit + const newStableService = getSMIServiceResource(serviceObject, blue_green_helper_2.STABLE_SUFFIX); + const newGreenService = getSMIServiceResource(serviceObject, blue_green_helper_2.GREEN_SUFFIX); + newObjectsList.push(newStableService); + newObjectsList.push(newGreenService); }); // creating services const manifestFiles = fileHelper.writeObjectsToFile(newObjectsList); @@ -87,7 +83,9 @@ function setupSMI(kubectl, deploymentEntityList, serviceEntityList) { exports.setupSMI = setupSMI; function createTrafficSplitObject(kubectl, name, nextLabel) { // getting smi spec api version - trafficSplitAPIVersion = kubectlUtils.getTrafficSplitAPIVersion(kubectl); + if (!trafficSplitAPIVersion) { + trafficSplitAPIVersion = kubectlUtils.getTrafficSplitAPIVersion(kubectl); + } // deciding weights based on nextlabel let stableWeight; let greenWeight; @@ -137,54 +135,48 @@ function getSMIServiceResource(inputObject, suffix) { } } exports.getSMIServiceResource = getSMIServiceResource; -function routeBlueGreenSMI(kubectl, nextLabel, deploymentEntityList, serviceEntityList) { +function routeBlueGreenSMI(kubectl, nextLabel, serviceEntityList) { serviceEntityList.forEach((serviceObject) => { - if (blue_green_helper_1.isServiceRouted(serviceObject, deploymentEntityList)) { - // routing trafficsplit to given label - createTrafficSplitObject(kubectl, serviceObject.metadata.name, nextLabel); - } + // routing trafficsplit to given label + createTrafficSplitObject(kubectl, serviceObject.metadata.name, nextLabel); }); } exports.routeBlueGreenSMI = routeBlueGreenSMI; -function validateTrafficSplitsState(kubectl, deploymentEntityList, serviceEntityList) { +function validateTrafficSplitsState(kubectl, serviceEntityList) { let areTrafficSplitsInRightState = true; serviceEntityList.forEach((serviceObject) => { - if (blue_green_helper_1.isServiceRouted(serviceObject, deploymentEntityList)) { - const name = serviceObject.metadata.name; - let trafficSplitObject = blue_green_helper_1.fetchResource(kubectl, TRAFFIC_SPLIT_OBJECT, blue_green_helper_1.getBlueGreenResourceName(name, TRAFFIC_SPLIT_OBJECT_NAME_SUFFIX)); - if (!trafficSplitObject) { - // no trafficplit exits - areTrafficSplitsInRightState = false; - } - trafficSplitObject = JSON.parse(JSON.stringify(trafficSplitObject)); - trafficSplitObject.spec.backends.forEach(element => { - // checking if trafficsplit in right state to deploy - if (element.service === blue_green_helper_1.getBlueGreenResourceName(name, blue_green_helper_2.GREEN_SUFFIX)) { - if (element.weight != MAX_VAL) { - // green service should have max weight - areTrafficSplitsInRightState = false; - } - } - if (element.service === blue_green_helper_1.getBlueGreenResourceName(name, blue_green_helper_2.STABLE_SUFFIX)) { - if (element.weight != MIN_VAL) { - // stable service should have 0 weight - areTrafficSplitsInRightState = false; - } - } - }); + const name = serviceObject.metadata.name; + let trafficSplitObject = blue_green_helper_1.fetchResource(kubectl, TRAFFIC_SPLIT_OBJECT, blue_green_helper_1.getBlueGreenResourceName(name, TRAFFIC_SPLIT_OBJECT_NAME_SUFFIX)); + if (!trafficSplitObject) { + // no trafficplit exits + areTrafficSplitsInRightState = false; } + trafficSplitObject = JSON.parse(JSON.stringify(trafficSplitObject)); + trafficSplitObject.spec.backends.forEach(element => { + // checking if trafficsplit in right state to deploy + if (element.service === blue_green_helper_1.getBlueGreenResourceName(name, blue_green_helper_2.GREEN_SUFFIX)) { + if (element.weight != MAX_VAL) { + // green service should have max weight + areTrafficSplitsInRightState = false; + } + } + if (element.service === blue_green_helper_1.getBlueGreenResourceName(name, blue_green_helper_2.STABLE_SUFFIX)) { + if (element.weight != MIN_VAL) { + // stable service should have 0 weight + areTrafficSplitsInRightState = false; + } + } + }); }); return areTrafficSplitsInRightState; } exports.validateTrafficSplitsState = validateTrafficSplitsState; -function cleanupSMI(kubectl, deploymentEntityList, serviceEntityList) { +function cleanupSMI(kubectl, serviceEntityList) { const deleteList = []; serviceEntityList.forEach((serviceObject) => { - if (blue_green_helper_1.isServiceRouted(serviceObject, deploymentEntityList)) { - deleteList.push({ name: blue_green_helper_1.getBlueGreenResourceName(serviceObject.metadata.name, blue_green_helper_2.GREEN_SUFFIX), kind: serviceObject.kind }); - deleteList.push({ name: blue_green_helper_1.getBlueGreenResourceName(serviceObject.metadata.name, blue_green_helper_2.STABLE_SUFFIX), kind: serviceObject.kind }); - deleteList.push({ name: blue_green_helper_1.getBlueGreenResourceName(serviceObject.metadata.name, TRAFFIC_SPLIT_OBJECT_NAME_SUFFIX), kind: TRAFFIC_SPLIT_OBJECT }); - } + deleteList.push({ name: blue_green_helper_1.getBlueGreenResourceName(serviceObject.metadata.name, blue_green_helper_2.GREEN_SUFFIX), kind: serviceObject.kind }); + deleteList.push({ name: blue_green_helper_1.getBlueGreenResourceName(serviceObject.metadata.name, blue_green_helper_2.STABLE_SUFFIX), kind: serviceObject.kind }); + deleteList.push({ name: blue_green_helper_1.getBlueGreenResourceName(serviceObject.metadata.name, TRAFFIC_SPLIT_OBJECT_NAME_SUFFIX), kind: TRAFFIC_SPLIT_OBJECT }); }); // deleting all objects blue_green_helper_1.deleteObjects(kubectl, deleteList); diff --git a/src/actions/promote.ts b/src/actions/promote.ts index 618bcbe4..c97056e2 100644 --- a/src/actions/promote.ts +++ b/src/actions/promote.ts @@ -1,4 +1,5 @@ 'use strict'; + import * as core from '@actions/core'; import * as deploymentHelper from '../utilities/strategy-helpers/deployment-helper'; import * as canaryDeploymentHelper from '../utilities/strategy-helpers/canary-deployment-helper'; @@ -9,7 +10,7 @@ import { getUpdatedManifestFiles } from '../utilities/manifest-utilities' import * as KubernetesObjectUtility from '../utilities/resource-object-utility'; import * as models from '../constants'; import * as KubernetesManifestUtility from '../utilities/manifest-stability-utility'; -import { getManifestObjects, deleteWorkloadsWithLabel, deleteWorkloadsAndServicesWithLabel } from '../utilities/strategy-helpers/blue-green-helper'; +import { getManifestObjects, deleteWorkloadsWithLabel, deleteWorkloadsAndServicesWithLabel, BlueGreenManifests } from '../utilities/strategy-helpers/blue-green-helper'; import { isBlueGreenDeploymentStrategy, isIngressRoute, isSMIRoute, GREEN_LABEL_VALUE, NONE_LABEL_VALUE } from '../utilities/strategy-helpers/blue-green-helper'; import { routeBlueGreenService, promoteBlueGreenService } from '../utilities/strategy-helpers/service-blue-green-helper'; import { routeBlueGreenIngress, promoteBlueGreenIngress } from '../utilities/strategy-helpers/ingress-blue-green-helper'; @@ -59,7 +60,7 @@ async function promoteCanary(kubectl: Kubectl) { async function promoteBlueGreen(kubectl: Kubectl) { // updated container images and pull secrets let inputManifestFiles: string[] = getUpdatedManifestFiles(TaskInputParameters.manifests); - const manifestObjects = getManifestObjects(inputManifestFiles); + const manifestObjects: BlueGreenManifests = getManifestObjects(inputManifestFiles); core.debug('deleting old deployment and making new ones'); let result; @@ -78,14 +79,14 @@ async function promoteBlueGreen(kubectl: Kubectl) { core.debug('routing to new deployments'); if(isIngressRoute()) { - routeBlueGreenIngress(kubectl, null, manifestObjects.serviceNameMap, manifestObjects.serviceEntityList, manifestObjects.ingressEntityList); + routeBlueGreenIngress(kubectl, null, manifestObjects.serviceNameMap, manifestObjects.ingressEntityList); deleteWorkloadsAndServicesWithLabel(kubectl, GREEN_LABEL_VALUE, manifestObjects.deploymentEntityList, manifestObjects.serviceEntityList); } else if (isSMIRoute()) { - routeBlueGreenSMI(kubectl, NONE_LABEL_VALUE, manifestObjects.deploymentEntityList, manifestObjects.serviceEntityList); + routeBlueGreenSMI(kubectl, NONE_LABEL_VALUE, manifestObjects.serviceEntityList); deleteWorkloadsWithLabel(kubectl, GREEN_LABEL_VALUE, manifestObjects.deploymentEntityList); - cleanupSMI(kubectl, manifestObjects.deploymentEntityList, manifestObjects.serviceEntityList); + cleanupSMI(kubectl, manifestObjects.serviceEntityList); } else { - routeBlueGreenService(kubectl, NONE_LABEL_VALUE, manifestObjects.deploymentEntityList, manifestObjects.serviceEntityList); + routeBlueGreenService(kubectl, NONE_LABEL_VALUE, manifestObjects.serviceEntityList); deleteWorkloadsWithLabel(kubectl, GREEN_LABEL_VALUE, manifestObjects.deploymentEntityList); } } \ No newline at end of file diff --git a/src/utilities/strategy-helpers/blue-green-helper.ts b/src/utilities/strategy-helpers/blue-green-helper.ts index cb20fbd9..2bcfd3e9 100644 --- a/src/utilities/strategy-helpers/blue-green-helper.ts +++ b/src/utilities/strategy-helpers/blue-green-helper.ts @@ -37,27 +37,36 @@ export function isSMIRoute(): boolean { return routeMethod && routeMethod.toUpperCase() === SMI_ROUTE; } +export interface BlueGreenManifests { + serviceEntityList: any[], + serviceNameMap: Map, + unroutedServiceEntityList: any[], + deploymentEntityList: any[], + ingressEntityList: any[], + otherObjects: any[] +} + export async function routeBlueGreen(kubectl: Kubectl, inputManifestFiles: string[]) { // get buffer time let bufferTime: number = parseInt(TaskInputParameters.versionSwitchBuffer); //logging start of buffer time let dateNow = new Date(); - console.log('starting buffer time of '+bufferTime+' minute/s at '+dateNow.toISOString()+' UTC'); + console.log(`Starting buffer time of ${bufferTime} minute(s) at ${dateNow.toISOString()}`); // waiting await sleep(bufferTime*1000*60); // logging end of buffer time dateNow = new Date(); - console.log('stopping buffer time of '+bufferTime+' minute/s at '+dateNow.toISOString()+' UTC'); + console.log(`Stopping buffer time of ${bufferTime} minute(s) at ${dateNow.toISOString()}`); - const manifestObjects = getManifestObjects(inputManifestFiles); + const manifestObjects: BlueGreenManifests = getManifestObjects(inputManifestFiles); // routing to new deployments if (isIngressRoute()) { - routeBlueGreenIngress(kubectl, GREEN_LABEL_VALUE, manifestObjects.serviceNameMap, manifestObjects.serviceEntityList, manifestObjects.ingressEntityList); + routeBlueGreenIngress(kubectl, GREEN_LABEL_VALUE, manifestObjects.serviceNameMap, manifestObjects.ingressEntityList); } else if (isSMIRoute()) { - routeBlueGreenSMI(kubectl, GREEN_LABEL_VALUE, manifestObjects.deploymentEntityList, manifestObjects.serviceEntityList); + routeBlueGreenSMI(kubectl, GREEN_LABEL_VALUE, manifestObjects.serviceEntityList); } else { - routeBlueGreenService(kubectl, GREEN_LABEL_VALUE, manifestObjects.deploymentEntityList, manifestObjects.serviceEntityList); + routeBlueGreenService(kubectl, GREEN_LABEL_VALUE, manifestObjects.serviceEntityList); } } @@ -82,27 +91,6 @@ export function deleteWorkloadsWithLabel(kubectl: Kubectl, deleteLabel: string, deleteObjects(kubectl, resourcesToDelete); } -export function cleanUp(kubectl: Kubectl, deploymentEntityList: any[], serviceEntityList: any[]) { - // checks if services has some stable deployments to target or deletes them too - let deleteList = []; - deploymentEntityList.forEach((deploymentObject) => { - const existingDeploy = fetchResource(kubectl, deploymentObject.kind, deploymentObject.metadata.name); - if (!existingDeploy) { - serviceEntityList.forEach((serviceObject) => { - const serviceSelector: string = getServiceSelector(serviceObject); - const matchLabels: string = getDeploymentMatchLabels(deploymentObject); - if (!!serviceSelector && !!matchLabels && isServiceSelectorSubsetOfMatchLabel(serviceSelector, matchLabels)) { - const resourceToDelete = { name : serviceObject.metadata.name, kind : serviceObject.kind }; - deleteList.push(resourceToDelete); - } - }); - } - }); - - // delete service not targeting a deployment - deleteObjects(kubectl, deleteList); -} - export function deleteWorkloadsAndServicesWithLabel(kubectl: Kubectl, deleteLabel: string, deploymentEntityList: any[], serviceEntityList: any[]) { // need to delete services and deployments const deletionEntitiesList = deploymentEntityList.concat(serviceEntityList); @@ -144,20 +132,28 @@ export function getSuffix(label: string): string { } // other common functions -export function getManifestObjects (filePaths: string[]): any { +export function getManifestObjects (filePaths: string[]): BlueGreenManifests { const deploymentEntityList = []; - const serviceEntityList = []; + const routedServiceEntityList = []; + const unroutedServiceEntityList = []; const ingressEntityList = []; const otherEntitiesList = []; + let serviceNameMap = new Map(); filePaths.forEach((filePath: string) => { const fileContents = fs.readFileSync(filePath); yaml.safeLoadAll(fileContents, function (inputObject) { if(!!inputObject) { const kind = inputObject.kind; + const name = inputObject.metadata.name; if (helper.isDeploymentEntity(kind)) { deploymentEntityList.push(inputObject); } else if (helper.isServiceEntity(kind)) { - serviceEntityList.push(inputObject); + if (isServiceRouted(inputObject, deploymentEntityList)) { + routedServiceEntityList.push(inputObject); + serviceNameMap.set(name, getBlueGreenResourceName(name, GREEN_SUFFIX)); + } else { + unroutedServiceEntityList.push(inputObject); + } } else if (helper.isIngressEntity(kind)) { ingressEntityList.push(inputObject); } else { @@ -166,29 +162,21 @@ export function getManifestObjects (filePaths: string[]): any { } }); }) - - let serviceNameMap = new Map(); - // find all services and add their names with blue green suffix - serviceEntityList.forEach(inputObject => { - const name = inputObject.metadata.name; - serviceNameMap.set(name, getBlueGreenResourceName(name, GREEN_SUFFIX)); - }); - return { serviceEntityList: serviceEntityList, serviceNameMap: serviceNameMap, deploymentEntityList: deploymentEntityList, ingressEntityList: ingressEntityList, otherObjects: otherEntitiesList }; + return { serviceEntityList: routedServiceEntityList, serviceNameMap: serviceNameMap, unroutedServiceEntityList: unroutedServiceEntityList, deploymentEntityList: deploymentEntityList, ingressEntityList: ingressEntityList, otherObjects: otherEntitiesList }; } export function isServiceRouted(serviceObject: any[], deploymentEntityList: any[]): boolean { let shouldBeRouted: boolean = false; - const serviceSelector: string = getServiceSelector(serviceObject); + const serviceSelector: any = getServiceSelector(serviceObject); if (!!serviceSelector) { - deploymentEntityList.every((depObject) => { + if (deploymentEntityList.some((depObject) => { // finding if there is a deployment in the given manifests the service targets - const matchLabels: string = getDeploymentMatchLabels(depObject); - if (!!matchLabels && isServiceSelectorSubsetOfMatchLabel(serviceSelector, matchLabels)) { - shouldBeRouted = true; - return false; - } - }); + const matchLabels: any = getDeploymentMatchLabels(depObject); + return (!!matchLabels && isServiceSelectorSubsetOfMatchLabel(serviceSelector, matchLabels)) + })) { + shouldBeRouted = true; + } } return shouldBeRouted; } @@ -240,36 +228,30 @@ export function getBlueGreenResourceName(name: string, suffix: string) { return `${name}${suffix}`; } -export function getSpecLabel(inputObject: any): string { - if(!!inputObject && inputObject.spec && inputObject.spec.selector && inputObject.spec.selector.matchLabels && inputObject.spec.selector.matchLabels[BLUE_GREEN_VERSION_LABEL]) { - return inputObject.spec.selector.matchLabels[BLUE_GREEN_VERSION_LABEL]; - } - return ''; -} - -export function getDeploymentMatchLabels(deploymentObject: any): string { +export function getDeploymentMatchLabels(deploymentObject: any): any { if (!!deploymentObject && deploymentObject.kind.toUpperCase()==KubernetesWorkload.pod.toUpperCase() && !!deploymentObject.metadata && !!deploymentObject.metadata.labels) { - return JSON.stringify(deploymentObject.metadata.labels); + return deploymentObject.metadata.labels; } else if (!!deploymentObject && deploymentObject.spec && deploymentObject.spec.selector && deploymentObject.spec.selector.matchLabels) { - return JSON.stringify(deploymentObject.spec.selector.matchLabels); + return deploymentObject.spec.selector.matchLabels; } - return ''; + return null; } -export function getServiceSelector(serviceObject: any): string { +export function getServiceSelector(serviceObject: any): any { if (!!serviceObject && serviceObject.spec && serviceObject.spec.selector) { - return JSON.stringify(serviceObject.spec.selector); - } else return ''; + return serviceObject.spec.selector; + } else return null; } -export function isServiceSelectorSubsetOfMatchLabel(serviceSelector: string, matchLabels: string): boolean { +export function isServiceSelectorSubsetOfMatchLabel(serviceSelector: any, matchLabels: any): boolean { let serviceSelectorMap = new Map(); let matchLabelsMap = new Map(); - JSON.parse(serviceSelector, (key, value) => { + JSON.parse(JSON.stringify(serviceSelector), (key, value) => { serviceSelectorMap.set(key, value); }); - JSON.parse(matchLabels, (key, value) => { + + JSON.parse(JSON.stringify(matchLabels), (key, value) => { matchLabelsMap.set(key, value); }); diff --git a/src/utilities/strategy-helpers/ingress-blue-green-helper.ts b/src/utilities/strategy-helpers/ingress-blue-green-helper.ts index b281a5e7..f36f929a 100644 --- a/src/utilities/strategy-helpers/ingress-blue-green-helper.ts +++ b/src/utilities/strategy-helpers/ingress-blue-green-helper.ts @@ -3,13 +3,13 @@ import * as core from '@actions/core'; import { Kubectl } from '../../kubectl-object-model'; import * as fileHelper from '../files-helper'; -import { createWorkloadsWithLabel, getManifestObjects, getNewBlueGreenObject, addBlueGreenLabelsAndAnnotations, deleteWorkloadsAndServicesWithLabel, fetchResource } from './blue-green-helper'; +import { createWorkloadsWithLabel, getManifestObjects, getNewBlueGreenObject, addBlueGreenLabelsAndAnnotations, deleteWorkloadsAndServicesWithLabel, fetchResource, BlueGreenManifests } from './blue-green-helper'; import { GREEN_LABEL_VALUE, NONE_LABEL_VALUE, BLUE_GREEN_VERSION_LABEL } from './blue-green-helper'; const BACKEND = 'BACKEND'; export function deployBlueGreenIngress(kubectl: Kubectl, filePaths: string[]) { // get all kubernetes objects defined in manifest files - const manifestObjects = getManifestObjects(filePaths); + const manifestObjects: BlueGreenManifests = getManifestObjects(filePaths); // create deployments with green label value const result = createWorkloadsWithLabel(kubectl, manifestObjects.deploymentEntityList, GREEN_LABEL_VALUE); @@ -21,7 +21,7 @@ export function deployBlueGreenIngress(kubectl: Kubectl, filePaths: string[]) { core.debug('New blue-green object is: ' + JSON.stringify(newBlueGreenObject)); newObjectsList.push(newBlueGreenObject); }); - newObjectsList = newObjectsList.concat(manifestObjects.otherObjects); + newObjectsList = newObjectsList.concat(manifestObjects.otherObjects).concat(manifestObjects.unroutedServiceEntityList); const manifestFiles = fileHelper.writeObjectsToFile(newObjectsList); kubectl.apply(manifestFiles); @@ -56,16 +56,16 @@ export async function promoteBlueGreenIngress(kubectl: Kubectl, manifestObjects) export async function rejectBlueGreenIngress(kubectl: Kubectl, filePaths: string[]) { // get all kubernetes objects defined in manifest files - const manifestObjects = getManifestObjects(filePaths); + const manifestObjects: BlueGreenManifests = getManifestObjects(filePaths); // routing ingress to stables services - routeBlueGreenIngress(kubectl, null, manifestObjects.serviceNameMap, manifestObjects.serviceEntityList, manifestObjects.ingressEntityList); + routeBlueGreenIngress(kubectl, null, manifestObjects.serviceNameMap, manifestObjects.ingressEntityList); // deleting green services and deployments deleteWorkloadsAndServicesWithLabel(kubectl, GREEN_LABEL_VALUE, manifestObjects.deploymentEntityList, manifestObjects.serviceEntityList); } -export function routeBlueGreenIngress(kubectl: Kubectl, nextLabel: string, serviceNameMap: Map, serviceEntityList: any[], ingressEntityList: any[]) { +export function routeBlueGreenIngress(kubectl: Kubectl, nextLabel: string, serviceNameMap: Map, ingressEntityList: any[]) { let newObjectsList = []; if (!nextLabel) { newObjectsList = newObjectsList.concat(ingressEntityList); diff --git a/src/utilities/strategy-helpers/service-blue-green-helper.ts b/src/utilities/strategy-helpers/service-blue-green-helper.ts index 8adea913..ef9dcf6c 100644 --- a/src/utilities/strategy-helpers/service-blue-green-helper.ts +++ b/src/utilities/strategy-helpers/service-blue-green-helper.ts @@ -2,18 +2,18 @@ import { Kubectl } from '../../kubectl-object-model'; import * as fileHelper from '../files-helper'; -import { createWorkloadsWithLabel, getManifestObjects, addBlueGreenLabelsAndAnnotations, fetchResource, deleteWorkloadsWithLabel, cleanUp, isServiceRouted } from './blue-green-helper'; +import { createWorkloadsWithLabel, getManifestObjects, addBlueGreenLabelsAndAnnotations, fetchResource, deleteWorkloadsWithLabel, BlueGreenManifests } from './blue-green-helper'; import { GREEN_LABEL_VALUE, NONE_LABEL_VALUE, BLUE_GREEN_VERSION_LABEL } from './blue-green-helper'; export function deployBlueGreenService(kubectl: Kubectl, filePaths: string[]) { // get all kubernetes objects defined in manifest files - const manifestObjects = getManifestObjects(filePaths); + const manifestObjects: BlueGreenManifests = getManifestObjects(filePaths); // create deployments with green label value const result = createWorkloadsWithLabel(kubectl, manifestObjects.deploymentEntityList, GREEN_LABEL_VALUE); // create other non deployment and non service entities - const newObjectsList = manifestObjects.otherObjects.concat(manifestObjects.ingressEntityList); + const newObjectsList = manifestObjects.otherObjects.concat(manifestObjects.ingressEntityList).concat(manifestObjects.unroutedServiceEntityList); const manifestFiles = fileHelper.writeObjectsToFile(newObjectsList); kubectl.apply(manifestFiles); @@ -23,7 +23,7 @@ export function deployBlueGreenService(kubectl: Kubectl, filePaths: string[]) { export async function promoteBlueGreenService(kubectl: Kubectl, manifestObjects) { // checking if services are in the right state ie. targeting green deployments - if (!validateServicesState(kubectl, manifestObjects.deploymentEntityList, manifestObjects.serviceEntityList)) { + if (!validateServicesState(kubectl, manifestObjects.serviceEntityList)) { throw('NotInPromoteState'); } @@ -36,29 +36,20 @@ export async function promoteBlueGreenService(kubectl: Kubectl, manifestObjects) export async function rejectBlueGreenService(kubectl: Kubectl, filePaths: string[]) { // get all kubernetes objects defined in manifest files - const manifestObjects = getManifestObjects(filePaths); + const manifestObjects: BlueGreenManifests = getManifestObjects(filePaths); // routing to stable objects - routeBlueGreenService(kubectl, NONE_LABEL_VALUE, manifestObjects.deploymentEntityList, manifestObjects.serviceEntityList); + routeBlueGreenService(kubectl, NONE_LABEL_VALUE, manifestObjects.serviceEntityList); - // seeing if we should even delete the service - cleanUp(kubectl, manifestObjects.deploymentEntityList, manifestObjects.serviceEntityList); - // deleting the new deployments with green suffix deleteWorkloadsWithLabel(kubectl, GREEN_LABEL_VALUE, manifestObjects.deploymentEntityList); } -export function routeBlueGreenService(kubectl: Kubectl, nextLabel: string, deploymentEntityList: any[], serviceEntityList: any[]) { +export function routeBlueGreenService(kubectl: Kubectl, nextLabel: string, serviceEntityList: any[]) { const newObjectsList = []; serviceEntityList.forEach((serviceObject) => { - if (isServiceRouted(serviceObject, deploymentEntityList)) { - // if service is routed, point it to given label - const newBlueGreenServiceObject = getUpdatedBlueGreenService(serviceObject, nextLabel); - newObjectsList.push(newBlueGreenServiceObject); - } else { - // if service is not routed, just push the original service - newObjectsList.push(serviceObject); - } + const newBlueGreenServiceObject = getUpdatedBlueGreenService(serviceObject, nextLabel); + newObjectsList.push(newBlueGreenServiceObject); }); // configures the services const manifestFiles = fileHelper.writeObjectsToFile(newObjectsList); @@ -73,22 +64,20 @@ function getUpdatedBlueGreenService(inputObject: any, labelValue: string): objec return newObject; } -export function validateServicesState(kubectl: Kubectl, deploymentEntityList: any[], serviceEntityList: any[]): boolean { +export function validateServicesState(kubectl: Kubectl, serviceEntityList: any[]): boolean { let areServicesGreen: boolean = true; serviceEntityList.forEach((serviceObject) => { - if (isServiceRouted(serviceObject, deploymentEntityList)) { - // finding the existing routed service - const existingService = fetchResource(kubectl, serviceObject.kind, serviceObject.metadata.name); - if (!!existingService) { - let currentLabel: string = getServiceSpecLabel(existingService); - if(currentLabel != GREEN_LABEL_VALUE) { - // service should be targeting deployments with green label - areServicesGreen = false; - } - } else { - // service targeting deployment doesn't exist + // finding the existing routed service + const existingService = fetchResource(kubectl, serviceObject.kind, serviceObject.metadata.name); + if (!!existingService) { + let currentLabel: string = getServiceSpecLabel(existingService); + if(currentLabel != GREEN_LABEL_VALUE) { + // service should be targeting deployments with green label areServicesGreen = false; } + } else { + // service targeting deployment doesn't exist + areServicesGreen = false; } }); return areServicesGreen; diff --git a/src/utilities/strategy-helpers/smi-blue-green-helper.ts b/src/utilities/strategy-helpers/smi-blue-green-helper.ts index b185fe82..fd77a461 100644 --- a/src/utilities/strategy-helpers/smi-blue-green-helper.ts +++ b/src/utilities/strategy-helpers/smi-blue-green-helper.ts @@ -3,7 +3,7 @@ import { Kubectl } from '../../kubectl-object-model'; import * as kubectlUtils from '../kubectl-util'; import * as fileHelper from '../files-helper'; -import { createWorkloadsWithLabel, getManifestObjects, fetchResource, deleteWorkloadsWithLabel, cleanUp, getNewBlueGreenObject, getBlueGreenResourceName, isServiceRouted, deleteObjects } from './blue-green-helper'; +import { createWorkloadsWithLabel, getManifestObjects, fetchResource, deleteWorkloadsWithLabel, getNewBlueGreenObject, getBlueGreenResourceName, deleteObjects, BlueGreenManifests } from './blue-green-helper'; import { GREEN_LABEL_VALUE, NONE_LABEL_VALUE, GREEN_SUFFIX, STABLE_SUFFIX } from './blue-green-helper'; let trafficSplitAPIVersion = ""; @@ -14,15 +14,15 @@ const MAX_VAL = '100'; export function deployBlueGreenSMI(kubectl: Kubectl, filePaths: string[]) { // get all kubernetes objects defined in manifest files - const manifestObjects = getManifestObjects(filePaths); + const manifestObjects: BlueGreenManifests = getManifestObjects(filePaths); // creating services and other objects - const newObjectsList = manifestObjects.otherObjects.concat(manifestObjects.serviceEntityList).concat(manifestObjects.ingressEntityList); + const newObjectsList = manifestObjects.otherObjects.concat(manifestObjects.serviceEntityList).concat(manifestObjects.ingressEntityList).concat(manifestObjects.unroutedServiceEntityList); const manifestFiles = fileHelper.writeObjectsToFile(newObjectsList); kubectl.apply(manifestFiles); // make extraservices and trafficsplit - setupSMI(kubectl, manifestObjects.deploymentEntityList, manifestObjects.serviceEntityList); + setupSMI(kubectl, manifestObjects.serviceEntityList); // create new deloyments const result = createWorkloadsWithLabel(kubectl, manifestObjects.deploymentEntityList, GREEN_LABEL_VALUE); @@ -33,7 +33,7 @@ export function deployBlueGreenSMI(kubectl: Kubectl, filePaths: string[]) { export async function promoteBlueGreenSMI(kubectl: Kubectl, manifestObjects) { // checking if there is something to promote - if (!validateTrafficSplitsState(kubectl, manifestObjects.deploymentEntityList, manifestObjects.serviceEntityList)) { + if (!validateTrafficSplitsState(kubectl, manifestObjects.serviceEntityList)) { throw('NotInPromoteStateSMI') } @@ -46,34 +46,29 @@ export async function promoteBlueGreenSMI(kubectl: Kubectl, manifestObjects) { export async function rejectBlueGreenSMI(kubectl: Kubectl, filePaths: string[]) { // get all kubernetes objects defined in manifest files - const manifestObjects = getManifestObjects(filePaths); + const manifestObjects: BlueGreenManifests = getManifestObjects(filePaths); // routing trafficsplit to stable deploymetns - routeBlueGreenSMI(kubectl, NONE_LABEL_VALUE, manifestObjects.deploymentEntityList, manifestObjects.serviceEntityList); - - // deciding whether to delete services or not - cleanUp(kubectl, manifestObjects.deploymentEntityList, manifestObjects.serviceEntityList); + routeBlueGreenSMI(kubectl, NONE_LABEL_VALUE, manifestObjects.serviceEntityList); // deleting rejected new bluegreen deplyments deleteWorkloadsWithLabel(kubectl, GREEN_LABEL_VALUE, manifestObjects.deploymentEntityList); //deleting trafficsplit and extra services - cleanupSMI(kubectl, manifestObjects.deploymentEntityList, manifestObjects.serviceEntityList); + cleanupSMI(kubectl, manifestObjects.serviceEntityList); } -export function setupSMI(kubectl: Kubectl, deploymentEntityList: any[], serviceEntityList: any[]) { +export function setupSMI(kubectl: Kubectl, serviceEntityList: any[]) { const newObjectsList = []; const trafficObjectList = [] serviceEntityList.forEach((serviceObject) => { - if (isServiceRouted(serviceObject, deploymentEntityList)) { - // create a trafficsplit for service - trafficObjectList.push(serviceObject); - // setting up the services for trafficsplit - const newStableService = getSMIServiceResource(serviceObject, STABLE_SUFFIX); - const newGreenService = getSMIServiceResource(serviceObject, GREEN_SUFFIX); - newObjectsList.push(newStableService); - newObjectsList.push(newGreenService); - } + // create a trafficsplit for service + trafficObjectList.push(serviceObject); + // setting up the services for trafficsplit + const newStableService = getSMIServiceResource(serviceObject, STABLE_SUFFIX); + const newGreenService = getSMIServiceResource(serviceObject, GREEN_SUFFIX); + newObjectsList.push(newStableService); + newObjectsList.push(newGreenService); }); // creating services @@ -88,7 +83,9 @@ export function setupSMI(kubectl: Kubectl, deploymentEntityList: any[], serviceE function createTrafficSplitObject(kubectl: Kubectl ,name: string, nextLabel: string): any { // getting smi spec api version - trafficSplitAPIVersion = kubectlUtils.getTrafficSplitAPIVersion(kubectl); + if (!trafficSplitAPIVersion) { + trafficSplitAPIVersion = kubectlUtils.getTrafficSplitAPIVersion(kubectl); + } // deciding weights based on nextlabel let stableWeight: number; @@ -140,56 +137,49 @@ export function getSMIServiceResource(inputObject: any, suffix: string): object } } -export function routeBlueGreenSMI(kubectl: Kubectl, nextLabel: string, deploymentEntityList: any[], serviceEntityList: any[]) { +export function routeBlueGreenSMI(kubectl: Kubectl, nextLabel: string, serviceEntityList: any[]) { serviceEntityList.forEach((serviceObject) => { - if (isServiceRouted(serviceObject, deploymentEntityList)) { - // routing trafficsplit to given label - createTrafficSplitObject(kubectl, serviceObject.metadata.name, nextLabel); - } + // routing trafficsplit to given label + createTrafficSplitObject(kubectl, serviceObject.metadata.name, nextLabel); }); } -export function validateTrafficSplitsState(kubectl: Kubectl, deploymentEntityList: any[], serviceEntityList: any[]): boolean { +export function validateTrafficSplitsState(kubectl: Kubectl, serviceEntityList: any[]): boolean { let areTrafficSplitsInRightState: boolean = true; serviceEntityList.forEach((serviceObject) => { - if (isServiceRouted(serviceObject, deploymentEntityList)) { - const name = serviceObject.metadata.name; - let trafficSplitObject = fetchResource(kubectl, TRAFFIC_SPLIT_OBJECT, getBlueGreenResourceName(name, TRAFFIC_SPLIT_OBJECT_NAME_SUFFIX)); - if (!trafficSplitObject) { - // no trafficplit exits - areTrafficSplitsInRightState = false; - } - trafficSplitObject = JSON.parse(JSON.stringify(trafficSplitObject)); - trafficSplitObject.spec.backends.forEach(element => { - // checking if trafficsplit in right state to deploy - if (element.service === getBlueGreenResourceName(name, GREEN_SUFFIX)) { - if (element.weight != MAX_VAL) { - // green service should have max weight - areTrafficSplitsInRightState = false; - } - } - - if (element.service === getBlueGreenResourceName(name, STABLE_SUFFIX)) { - if (element.weight != MIN_VAL) { - // stable service should have 0 weight - areTrafficSplitsInRightState = false; - } - } - }); + const name = serviceObject.metadata.name; + let trafficSplitObject = fetchResource(kubectl, TRAFFIC_SPLIT_OBJECT, getBlueGreenResourceName(name, TRAFFIC_SPLIT_OBJECT_NAME_SUFFIX)); + if (!trafficSplitObject) { + // no trafficplit exits + areTrafficSplitsInRightState = false; } + trafficSplitObject = JSON.parse(JSON.stringify(trafficSplitObject)); + trafficSplitObject.spec.backends.forEach(element => { + // checking if trafficsplit in right state to deploy + if (element.service === getBlueGreenResourceName(name, GREEN_SUFFIX)) { + if (element.weight != MAX_VAL) { + // green service should have max weight + areTrafficSplitsInRightState = false; + } + } + + if (element.service === getBlueGreenResourceName(name, STABLE_SUFFIX)) { + if (element.weight != MIN_VAL) { + // stable service should have 0 weight + areTrafficSplitsInRightState = false; + } + } + }); }); - return areTrafficSplitsInRightState; } -export function cleanupSMI(kubectl: Kubectl, deploymentEntityList: any[], serviceEntityList: any[]) { +export function cleanupSMI(kubectl: Kubectl, serviceEntityList: any[]) { const deleteList = []; serviceEntityList.forEach((serviceObject) => { - if (isServiceRouted(serviceObject, deploymentEntityList)) { - deleteList.push({name: getBlueGreenResourceName(serviceObject.metadata.name, GREEN_SUFFIX), kind: serviceObject.kind}); - deleteList.push({name: getBlueGreenResourceName(serviceObject.metadata.name, STABLE_SUFFIX), kind: serviceObject.kind}); - deleteList.push({name: getBlueGreenResourceName(serviceObject.metadata.name, TRAFFIC_SPLIT_OBJECT_NAME_SUFFIX), kind: TRAFFIC_SPLIT_OBJECT}); - } + deleteList.push({name: getBlueGreenResourceName(serviceObject.metadata.name, GREEN_SUFFIX), kind: serviceObject.kind}); + deleteList.push({name: getBlueGreenResourceName(serviceObject.metadata.name, STABLE_SUFFIX), kind: serviceObject.kind}); + deleteList.push({name: getBlueGreenResourceName(serviceObject.metadata.name, TRAFFIC_SPLIT_OBJECT_NAME_SUFFIX), kind: TRAFFIC_SPLIT_OBJECT}); }); // deleting all objects