Circleci manual holds (#1152)

* ci(circleci): introduces manual holds to prevent resource wastage We were building all commits unnecessarily. The logic was not consistent for versioning and when to build and when to publish. This PR addresses these issues. * Replaces `should-build` with `build-approval` * Replace `should-publish` with `publish-approval` * Clean up image tagging logic - tags as latest when version is semver or semver-alpha.build_number - other commits aren't tagged as latest, and instead tagged with their branch name - move naming logic to common.sh script * Push each tag individually to avoid pollution - docker engine used is shared amongst all runs of speckle-server pipeline, so pushing all tags indiscrimanately pushes images built on other CircleCI runs 😬 * Improve logging * Do not require approval for any tagged commit
2022-11-01 08:52:51 +00:00 · 2022-11-01 08:52:51 +00:00 · a51ba5f4a9
--- a/.circleci/build.sh
+++ b/.circleci/build.sh
@ -1,12 +1,6 @@
 #!/bin/bash
 set -eo pipefail

-SHOULD_BUILD="${SHOULD_BUILD:-false}"
-if [[ "${SHOULD_BUILD}" != "true" ]]; then
-  echo "Not building as the SHOULD_BUILD environment variable is not 'true'."
-  exit 0
-fi
-
 # enables building the test-deployment container with the same script
 # defaults to packages for minimal intervention in the ci config
 FOLDER="${FOLDER:-packages}"
--- a/.circleci/common.sh
+++ b/.circleci/common.sh
@ -5,3 +5,11 @@ DOCKER_IMAGE_TAG="speckle/speckle-${SPECKLE_SERVER_PACKAGE}"
 IMAGE_VERSION_TAG="${IMAGE_VERSION_TAG:-${CIRCLE_SHA1}}"
 # shellcheck disable=SC2034,SC2086
 DOCKER_FILE_NAME="$(echo ${DOCKER_IMAGE_TAG}_${IMAGE_VERSION_TAG} | sed -e 's/[^A-Za-z0-9._-]/_/g')"
+# shellcheck disable=SC2068,SC2046
+LAST_RELEASE="$(git describe --always --tags $(git rev-list --tags) | grep -E '^[0-9]+\.[0-9]+\.[0-9]+$' | head -n 1)"
+# shellcheck disable=SC2034
+NEXT_RELEASE="$(echo "${LAST_RELEASE}" | python -c "parts = input().split('.'); parts[-1] = str(int(parts[-1])+1); print('.'.join(parts))")"
+# shellcheck disable=SC2034
+BRANCH_NAME_TRUNCATED="$(echo "${CIRCLE_BRANCH}" | cut -c -50 | sed 's/[^a-zA-Z0-9_.-]/_/g')" # docker has a 128 character tag limit, so ensuring the branch name will be short enough
+# shellcheck disable=SC2034
+COMMIT_SHA1_TRUNCATED="$(echo "${CIRCLE_SHA1}" | cut -c -7)"
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@ -6,22 +6,24 @@ workflows:
  test-build:
    jobs:
      - test-server:
-          filters: &filters-everything
+          filters: &filters-allow-all
            tags:
              # run tests for any commit on any branch, including any tags
              only: /.*/

      - get-version:
-          filters: *filters-everything
+          filters: *filters-allow-all

-      - should-build:
-          filters: *filters-everything
-
-      - should-publish:
-          filters: *filters-everything
+      - build-approval:
+          type: approval
+          filters: &filters-ignore-main-branch-all-tags
+            branches:
+              ignore: main
+            tags:
+              ignore: /.*/

      - pre-commit:
-          filters: *filters-everything
+          filters: *filters-allow-all

      - docker-build-server:
          filters: &filters-build
@ -30,47 +32,51 @@ workflows:
          requires:
            - test-server
            - get-version
-            - should-build
+            - build-approval

      - docker-build-frontend:
          filters: *filters-build
          requires:
            - get-version
-            - should-build
+            - build-approval

      - docker-build-webhooks:
          filters: *filters-build
          requires:
            - get-version
            - test-server
-            - should-build
+            - build-approval

      - docker-build-file-imports:
          filters: *filters-build
          requires:
            - get-version
            - test-server
-            - should-build
+            - build-approval

      - docker-build-previews:
          filters: *filters-build
          requires:
            - get-version
            - test-server
-            - should-build
+            - build-approval

      - docker-build-test-container:
          filters: *filters-build
          requires:
            - get-version
            - test-server
-            - should-build
+            - build-approval

      - docker-build-monitor-container:
          filters: *filters-build
          requires:
            - get-version
-            - should-build
+            - build-approval
+
+      - publish-approval:
+          type: approval
+          filters: *filters-ignore-main-branch-all-tags

      - docker-publish-server:
          context: &docker-hub-context
@ -82,7 +88,7 @@ workflows:
              only: /.*/
          requires:
            - get-version
-            - should-publish
+            - publish-approval
            - docker-build-server
            - pre-commit

@ -91,7 +97,7 @@ workflows:
          filters: *filters-publish
          requires:
            - get-version
-            - should-publish
+            - publish-approval
            - docker-build-frontend
            - pre-commit

@ -100,7 +106,7 @@ workflows:
          filters: *filters-publish
          requires:
            - get-version
-            - should-publish
+            - publish-approval
            - docker-build-webhooks
            - pre-commit

@ -109,7 +115,7 @@ workflows:
          filters: *filters-publish
          requires:
            - get-version
-            - should-publish
+            - publish-approval
            - docker-build-file-imports
            - pre-commit

@ -118,7 +124,7 @@ workflows:
          filters: *filters-publish
          requires:
            - get-version
-            - should-publish
+            - publish-approval
            - docker-build-previews
            - pre-commit

@ -127,7 +133,7 @@ workflows:
          filters: *filters-publish
          requires:
            - get-version
-            - should-publish
+            - publish-approval
            - docker-build-test-container
            - pre-commit

@ -136,7 +142,7 @@ workflows:
          filters: *filters-publish
          requires:
            - get-version
-            - should-publish
+            - publish-approval
            - docker-build-monitor-container
            - pre-commit

@ -150,7 +156,7 @@ workflows:
              only: &filters-tag /^[0-9]+\.[0-9]+\.[0-9]+$/
          requires:
            - get-version
-            - should-publish
+            - publish-approval
            - docker-publish-server
            - docker-publish-frontend
            - docker-publish-webhooks
@ -189,47 +195,6 @@ jobs:
          paths:
            - env-vars

-  should-publish:
-    docker:
-      - image: cimg/base:2022.08
-    working_directory: *work-dir
-    environment: &publishable-tags-branches
-      PUBLISHABLE_TAGS: '^[0-9]+\.[0-9]+\.[0-9]+$'
-      # £ delimited strings of regex for matches which should be published
-      PUBLISHABLE_BRANCHES: '^main$£^hotfix.*£^alpha.*'
-    steps:
-      - checkout
-      - run: mkdir -p workspace
-      - run:
-          name: determine whether to publish
-          command: |
-            echo "export SHOULD_PUBLISH=$(.circleci/should_publish.sh)" >> workspace/should-publish
-      - run: cat workspace/should-publish >> $BASH_ENV
-      - run: echo "SHOULD_PUBLISH=${SHOULD_PUBLISH}"
-      - persist_to_workspace:
-          root: workspace
-          paths:
-            - should-publish
-
-  should-build:
-    docker:
-      - image: cimg/base:2022.08
-    working_directory: *work-dir
-    environment: *publishable-tags-branches
-    steps:
-      - checkout
-      - run: mkdir -p workspace
-      - run:
-          name: determine whether to build
-          command: |
-            echo "export SHOULD_BUILD=$(.circleci/should_build.sh)" >> workspace/should-build
-      - run: cat workspace/should-build >> $BASH_ENV
-      - run: echo "SHOULD_BUILD=${SHOULD_BUILD}"
-      - persist_to_workspace:
-          root: workspace
-          paths:
-            - should-build
-
  pre-commit:
    parameters:
      config_file:
@ -363,7 +328,7 @@ jobs:

  docker-build: &build-job
    docker: &docker-image
-      - image: cimg/node:16.15
+      - image: cimg/python:3.9.15-node
    resource_class: medium
    working_directory: *work-dir
    steps:
@ -371,19 +336,13 @@ jobs:
      - attach_workspace:
          at: /tmp/ci/workspace
      - run: cat workspace/env-vars >> $BASH_ENV
-      - run: cat workspace/should-build >> $BASH_ENV
-      - run:
-          name: 'Check if should proceed'
-          command: |
-            [[ "${SHOULD_BUILD}" != true ]] && echo "Should not build, stopping" && circleci-agent step halt
-            echo 'Proceeding with build'
      - setup_remote_docker:
          # a weird issue with yarn installing packages throwing EPERM errors
          # this fixes it
          version: 20.10.12
          docker_layer_caching: true
      - run:
-          name: Build and Publish
+          name: Build and Save
          command: ./.circleci/build.sh
      - persist_to_workspace:
          root: workspace
@ -429,7 +388,7 @@ jobs:

  docker-publish: &publish-job
    docker: &base-image
-      - image: cimg/base:2022.08
+      - image: cimg/python:3.9.15-node
    resource_class: medium
    working_directory: *work-dir
    steps:
@ -437,12 +396,6 @@ jobs:
      - attach_workspace:
          at: /tmp/ci/workspace
      - run: cat workspace/env-vars >> $BASH_ENV
-      - run: cat workspace/should-publish >> $BASH_ENV
-      - run:
-          name: 'Check if should proceed'
-          command: |
-            [[ "${SHOULD_PUBLISH}" != true ]] && echo "Should not publish, stopping" && circleci-agent step halt
-            echo 'Proceeding with publish'
      - setup_remote_docker:
          # a weird issue with yarn installing packages throwing EPERM errors
          # this fixes it
--- a/.circleci/get_version.sh
+++ b/.circleci/get_version.sh
@ -1,9 +1,9 @@
 #!/bin/bash
 set -eo pipefail

-# shellcheck disable=SC2068,SC2046
-LAST_RELEASE="$(git describe --always --tags $(git rev-list --tags) | grep -E '^[0-9]+\.[0-9]+\.[0-9]+$' | head -n 1)"
-NEXT_RELEASE="$(echo "${LAST_RELEASE}" | python -c "parts = input().split('.'); parts[-1] = str(int(parts[-1])+1); print('.'.join(parts))")"
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+# shellcheck disable=SC1090,SC1091
+source "${SCRIPT_DIR}/common.sh"

 if [[ "${CIRCLE_TAG}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
    echo "${CIRCLE_TAG}"
@ -15,7 +15,5 @@ if [[ "${CIRCLE_BRANCH}" == "main" ]]; then
    exit 0
 fi

-BRANCH_NAME_TRUNCATED="$(echo "${CIRCLE_BRANCH}" | cut -c -50 | sed 's/[^a-zA-Z0-9_.-]/_/g')" # docker has a 128 character tag limit, so ensuring the branch name will be short enough
-COMMIT_SHA1_TRUNCATED="$(echo "${CIRCLE_SHA1}" | cut -c -7)"
 echo "$NEXT_RELEASE-branch.${BRANCH_NAME_TRUNCATED}.${COMMIT_SHA1_TRUNCATED}.${CIRCLE_BUILD_NUM}"
 exit 0
--- a/.circleci/publish.sh
+++ b/.circleci/publish.sh
@ -1,28 +1,35 @@
 #!/usr/bin/env bash
 set -eo pipefail

-SHOULD_PUBLISH="${SHOULD_PUBLISH:-false}"
-
-if [[ "${SHOULD_PUBLISH}" != "true" ]]; then
-  echo "Not publishing as the SHOULD_PUBLISH environment variable is not 'true'."
-  exit 0
-fi
-
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 # shellcheck disable=SC1090,SC1091
 source "${SCRIPT_DIR}/common.sh"

-echo "Publishing: ${DOCKER_IMAGE_TAG}:${IMAGE_VERSION_TAG}"
+echo "Starting tagging & publishing of image: ${DOCKER_IMAGE_TAG}:${IMAGE_VERSION_TAG}"

 echo "💾 Loading image"
 docker load --input "/tmp/ci/workspace/${DOCKER_FILE_NAME}"

-echo "🐳 Publishing image"
-docker tag "${DOCKER_IMAGE_TAG}:${IMAGE_VERSION_TAG}" "${DOCKER_IMAGE_TAG}:latest"
+echo "🐳 Logging into Docker"
+echo "${DOCKER_REG_PASS}" | docker login -u "${DOCKER_REG_USER}" --password-stdin "${DOCKER_REG_URL}"
+echo "⏫ Pushing loaded image: '${DOCKER_IMAGE_TAG}:${IMAGE_VERSION_TAG}'"
+docker push "${DOCKER_IMAGE_TAG}:${IMAGE_VERSION_TAG}"

-if [[ "${IMAGE_VERSION_TAG}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
-  docker tag "${DOCKER_IMAGE_TAG}:${IMAGE_VERSION_TAG}" "${DOCKER_IMAGE_TAG}:2"
+if [[ "${IMAGE_VERSION_TAG}" =~ ^[0-9]+\.[0-9]+\.[0-9]+(?:-alpha\.[0-9]+)?$ ]]; then
+  echo "🏷 Tagging and pushing image as '${DOCKER_IMAGE_TAG}:latest'"
+  docker tag "${DOCKER_IMAGE_TAG}:${IMAGE_VERSION_TAG}" "${DOCKER_IMAGE_TAG}:latest"
+  docker push "${DOCKER_IMAGE_TAG}:latest"
+
+  if [[ "${IMAGE_VERSION_TAG}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+    echo "🏷 Tagging and pushing image as '${DOCKER_IMAGE_TAG}:2'"
+    docker tag "${DOCKER_IMAGE_TAG}:${IMAGE_VERSION_TAG}" "${DOCKER_IMAGE_TAG}:2"
+    docker push "${DOCKER_IMAGE_TAG}:2"
+  fi
+else
+  BRANCH_TAG="${NEXT_RELEASE}-branch.${BRANCH_NAME_TRUNCATED}"
+  echo "🏷 Tagging and pushing image as '${DOCKER_IMAGE_TAG}:${BRANCH_TAG}'"
+  docker tag "${DOCKER_IMAGE_TAG}:${IMAGE_VERSION_TAG}" "${DOCKER_IMAGE_TAG}:${BRANCH_TAG}"
+  docker push "${DOCKER_IMAGE_TAG}:${BRANCH_TAG}"
 fi

-echo "${DOCKER_REG_PASS}" | docker login -u "${DOCKER_REG_USER}" --password-stdin "${DOCKER_REG_URL}"
-docker push --all-tags "${DOCKER_IMAGE_TAG}"
+echo "✅ Publishing completed."
--- a/.circleci/should_build.sh
+++ b/.circleci/should_build.sh
@ -1,17 +0,0 @@
-#!/bin/bash
-set -eo pipefail
-
-IFS='£' read -r -a PUB_TAGS <<< "${PUBLISHABLE_TAGS}"
-# shellcheck disable=SC2068
-for item in ${PUB_TAGS[@]}; do
-    [[ "${CIRCLE_TAG}" =~ ${item} ]] && echo "true" && exit 0
-done
-
-# it's on the main branch
-[[ "${CIRCLE_BRANCH}" == "main" ]] && echo "true" && exit 0
-
-# or it is on a branch with a Pull Request
-[[ -n "${CIRCLE_PULL_REQUEST}" ]] && echo "true" && exit 0
-
-echo "false"
-exit 0
--- a/.circleci/should_publish.sh
+++ b/.circleci/should_publish.sh
@ -1,17 +0,0 @@
-#!/usr/bin/env bash
-set -eo pipefail
-
-IFS='£' read -r -a PUB_TAGS <<< "${PUBLISHABLE_TAGS}"
-# shellcheck disable=SC2068
-for item in ${PUB_TAGS[@]}; do
-    [[ "${CIRCLE_TAG}" =~ ${item} ]] && echo "true" && exit 0
-done
-
-IFS='£' read -r -a PUB_BRANCHES <<< "${PUBLISHABLE_BRANCHES}"
-# shellcheck disable=SC2068
-for item in ${PUB_BRANCHES[@]}; do
-    [[ "${CIRCLE_BRANCH}" =~ ${item} ]] && echo "true" && exit 0
-done
-
-echo "false"
-exit 0