Merge pull request #11466 from github/redsun82/cache

Swift: optimize bazel caching in CI
2022-11-29 14:31:16 +01:00 · 2022-11-29 14:31:16 +01:00 · d7313082c9
--- a/.github/actions/cache-query-compilation/action.yml
+++ b/.github/actions/cache-query-compilation/action.yml
@ -14,39 +14,14 @@ outputs:
 runs:
  using: composite
  steps:
-    # Cache the query compilation caches.
-    # calculate the merge-base with main, in a way that works both on PRs and pushes to main. 
-    - name: Calculate merge-base
-      shell: bash
-      if: ${{ github.event_name == 'pull_request' }}
-      env:
-        BASE_BRANCH: ${{ github.base_ref }}
-      run: |
-        MERGE_BASE=$(git cat-file commit $GITHUB_SHA | grep '^parent ' | head -1 | cut -f 2 -d " ")
-        echo "merge_base=$MERGE_BASE" >> $GITHUB_ENV
-    - name: Read CodeQL query compilation - PR
-      if: ${{ github.event_name == 'pull_request' }}
-      uses: erik-krogh/actions-cache@a88d0603fe5fb5606db9f002dfcadeb32b5f84c6
+    - name: Cache the query compilation caches
+      uses: ./.github/actions/incremental-cache
      with:
        path: '**/.cache'
-        read-only: true
-        key: codeql-compile-${{ inputs.key }}-pr-${{ github.sha }} # deliberately not using the `compile-compile-main` keys here.
-        restore-keys: |
-          codeql-compile-${{ inputs.key }}-${{ github.base_ref }}-${{ env.merge_base }}
-          codeql-compile-${{ inputs.key }}-${{ github.base_ref }}-
-          codeql-compile-${{ inputs.key }}-main-
-    - name: Fill CodeQL query compilation cache - main
-      if: ${{ github.event_name != 'pull_request' }}
-      uses: erik-krogh/actions-cache@a88d0603fe5fb5606db9f002dfcadeb32b5f84c6
-      with:
-        path: '**/.cache'
-        key: codeql-compile-${{ inputs.key }}-${{ github.ref_name }}-${{ github.sha }} # just fill on main
-        restore-keys: | # restore from another random commit, to speed up compilation.
-          codeql-compile-${{ inputs.key }}-${{ github.ref_name }}- 
-          codeql-compile-${{ inputs.key }}-main-
+        key: codeql-compile-${{ inputs.key }}
    - name: Fill compilation cache directory
      id: fill-compilation-dir
-      shell: bash 
+      shell: bash
      run: |
        # Move all the existing cache into another folder, so we only preserve the cache for the current queries.
        mkdir -p ${COMBINED_CACHE_DIR}
@ -57,5 +32,5 @@ runs:
        rm -rf **/.cache/*

        echo "compdir=${COMBINED_CACHE_DIR}" >> $GITHUB_OUTPUT
-      env: 
-        COMBINED_CACHE_DIR: ${{ github.workspace }}/compilation-dir
+      env:
+        COMBINED_CACHE_DIR: ${{ github.workspace }}/compilation-dir
--- a/.github/actions/incremental-cache/action.yml
+++ b/.github/actions/incremental-cache/action.yml
@ -0,0 +1,44 @@
+name: Setup an incremental cache
+description: Special cache wrapper to be run on pull requests and pushes, that will try to restore
+  a cache as close as possible to the merge base
+
+inputs:
+  path:
+    description: 'The path to cache'
+    required: true
+  key:
+    description: 'The cache key to use - should be unique to the workflow'
+    required: true
+
+runs:
+  using: composite
+  steps:
+    # calculate the merge-base with main, in a way that works both on PRs and pushes to main.
+    - name: Calculate merge-base
+      shell: bash
+      if: ${{ github.event_name == 'pull_request' }}
+      env:
+        BASE_BRANCH: ${{ github.base_ref }}
+      run: |
+        MERGE_BASE=$(git cat-file commit $GITHUB_SHA | grep '^parent ' | head -1 | cut -f 2 -d " ")
+        echo "merge_base=$MERGE_BASE" >> $GITHUB_ENV
+    - name: Restore read-only cache (PR)
+      if: ${{ github.event_name == 'pull_request' }}
+      uses: erik-krogh/actions-cache@a88d0603fe5fb5606db9f002dfcadeb32b5f84c6
+      with:
+        path: ${{ inputs.path }}
+        read-only: true
+        key: ${{ inputs.key }}-pr-${{ github.sha }}
+        restore-keys: |
+          ${{ inputs.key }}-${{ github.base_ref }}-${{ env.merge_base }}
+          ${{ inputs.key }}-${{ github.base_ref }}-
+          ${{ inputs.key }}-main-
+    - name: Fill cache (push)
+      if: ${{ github.event_name != 'pull_request' }}
+      uses: erik-krogh/actions-cache@a88d0603fe5fb5606db9f002dfcadeb32b5f84c6
+      with:
+        path: ${{ inputs.path }}
+        key: ${{ inputs.key }}-${{ github.ref_name }}-${{ github.sha }} # just fill on main
+        restore-keys: | # restore the latest cache if the exact cache is unavailable, to speed up compilation.
+          ${{ inputs.key }}-${{ github.ref_name }}-
+          ${{ inputs.key }}-main-
--- a/.github/workflows/go-tests-other-os.yml
+++ b/.github/workflows/go-tests-other-os.yml
@ -5,8 +5,7 @@ on:
      - "go/**"
      - "!go/ql/**" # don't run other-os if only ql/ files changed
      - .github/workflows/go-tests-other-os.yml
-      - .github/actions/fetch-codeql/action.yml
-      - .github/actions/cache-query-compilation/action.yml
+      - .github/actions/**
      - codeql-workspace.yml
 jobs:
  test-mac:
--- a/.github/workflows/go-tests.yml
+++ b/.github/workflows/go-tests.yml
@ -4,8 +4,7 @@ on:
    paths:
      - "go/**"
      - .github/workflows/go-tests.yml
-      - .github/actions/fetch-codeql/action.yml
-      - .github/actions/cache-query-compilation/action.yml
+      - .github/actions/**
      - codeql-workspace.yml
    branches:
      - main
@ -14,8 +13,7 @@ on:
    paths:
      - "go/**"
      - .github/workflows/go-tests.yml
-      - .github/actions/fetch-codeql/action.yml
-      - .github/actions/cache-query-compilation/action.yml
+      - .github/actions/**
      - codeql-workspace.yml
 jobs:
  test-linux:
@ -64,7 +62,7 @@ jobs:
        uses: ./.github/actions/cache-query-compilation
        with:
          key: go-qltest
-    
+
      - name: Test
        run: |
          cd go
--- a/.github/workflows/swift.yml
+++ b/.github/workflows/swift.yml
@ -7,8 +7,7 @@ on:
      - "misc/bazel/**"
      - "*.bazel*"
      - .github/workflows/swift.yml
-      - .github/actions/fetch-codeql/action.yml
-      - .github/actions/cache-query-compilation/action.yml
+      - .github/actions/**
      - codeql-workspace.yml
      - .pre-commit-config.yaml
      - "!**/*.md"
@ -22,8 +21,7 @@ on:
      - "misc/bazel/**"
      - "*.bazel*"
      - .github/workflows/swift.yml
-      - .github/actions/fetch-codeql/action.yml
-      - .github/actions/cache-query-compilation/action.yml
+      - .github/actions/**
      - codeql-workspace.yml
      - "!**/*.md"
      - "!**/*.qhelp"
@ -35,20 +33,15 @@ jobs:
  # not using a matrix as you cannot depend on a specific job in a matrix, and we want to start linux checks
  # without waiting for the macOS build
  build-and-test-macos:
-    if: ${{ github.event_name == 'pull_request' }}
    runs-on: macos-12-xl
    steps:
      - uses: actions/checkout@v3
-      - uses: ./swift/actions/create-extractor-pack
-      - uses: ./swift/actions/run-quick-tests
-      - uses: ./swift/actions/print-unextracted
+      - uses: ./swift/actions/build-and-test
  build-and-test-linux:
    runs-on: ubuntu-latest-xl
    steps:
      - uses: actions/checkout@v3
-      - uses: ./swift/actions/create-extractor-pack
-      - uses: ./swift/actions/run-quick-tests
-      - uses: ./swift/actions/print-unextracted
+      - uses: ./swift/actions/build-and-test
  qltests-linux:
    needs: build-and-test-linux
    runs-on: ubuntu-latest-xl
@ -80,7 +73,10 @@ jobs:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
-      - uses: ./swift/actions/setup-env
+      - uses: bazelbuild/setup-bazelisk@v2
+      - uses: actions/setup-python@v4
+        with:
+          python-version-file: 'swift/.python-version'
      - uses: pre-commit/action@v3.0.0
        name: Check that python code is properly formatted
        with:
--- a/swift/actions/build-and-test/action.yml
+++ b/swift/actions/build-and-test/action.yml
@ -0,0 +1,54 @@
+name: Build Swift CodeQL pack
+description: Builds the Swift CodeQL pack
+runs:
+  using: composite
+  steps:
+    - uses: bazelbuild/setup-bazelisk@v2
+    - uses: actions/setup-python@v4
+      with:
+        python-version-file: 'swift/.python-version'
+    - name: Mount bazel cache
+      uses: ./.github/actions/incremental-cache
+      with:
+        path: "~/.cache/bazel-repository-cache"
+        key: bazel-cache-${{ runner.os }}-${{ runner.arch }}
+    - name: Mount bazel disk cache
+      uses: ./.github/actions/incremental-cache
+      with:
+        path: "~/.cache/bazel-disk-cache"
+        key: bazel-disk-cache-${{ runner.os }}-${{ runner.arch }}
+    - name: Configure bazel cache
+      shell: bash
+      run: |
+        echo build --repository_cache=~/.cache/bazel-repository-cache --disk_cache=~/.cache/bazel-disk-cache > ~/.bazelrc
+        echo test --test_output=errors >> ~/.bazelrc
+    - name: Print unextracted entities
+      shell: bash
+      run: |
+        bazel run //swift/extractor/print_unextracted
+    - uses: ./swift/actions/share-extractor-pack
+    - name: Build Swift extractor
+      shell: bash
+      run: |
+        bazel run //swift:create-extractor-pack
+    - name: Run xcode-autobuilder tests
+      if : ${{ github.event_name == 'pull_request' && runner.os == 'macOS' }}
+      shell: bash
+      run: |
+        bazel test //swift/xcode-autobuilder/tests
+    - name: Run codegen tests
+      if : ${{ github.event_name == 'pull_request' }}
+      shell: bash
+      run: |
+        bazel test //swift/codegen/test
+    - name: Run qltest tests
+      if : ${{ github.event_name == 'pull_request' }}
+      shell: bash
+      run: |
+        bazel test //swift/tools/test/qltest
+    - name: Evict bazel cache
+      if: ${{ github.event_name != 'pull_request' }}
+      shell: bash
+      run: |
+        find "~/.cache/bazel-repository-cache" "~/.cache/bazel-disk-cache" -atime +0 -type f -delete
+        du -sh "~/.cache/bazel-repository-cache" "~/.cache/bazel-disk-cache"
--- a/swift/actions/create-extractor-pack/action.yml
+++ b/swift/actions/create-extractor-pack/action.yml
@ -1,11 +0,0 @@
-name: Build Swift CodeQL pack
-description: Builds the Swift CodeQL pack
-runs:
-  using: composite
-  steps:
-    - uses: ./swift/actions/setup-env
-    - uses: ./swift/actions/share-extractor-pack
-    - name: Build Swift extractor
-      shell: bash
-      run: |
-        bazel run //swift:create-extractor-pack
--- a/swift/actions/print-unextracted/action.yml
+++ b/swift/actions/print-unextracted/action.yml
@ -1,9 +0,0 @@
-name: Print unextracted entities
-description: Prints all AST and Type entities that we do not extract yet. Must be run after `setup-env`
-runs:
-  using: composite
-  steps:
-    - name: Print unextracted entities
-      shell: bash
-      run: |
-        bazel run //swift/extractor/print_unextracted
--- a/swift/actions/run-quick-tests/action.yml
+++ b/swift/actions/run-quick-tests/action.yml
@ -1,18 +0,0 @@
-name: Run Swift quick tests
-description: Runs Swift tests defined in Bazel. Must be run after `setup-env`
-runs:
-  using: composite
-  steps:
-    - name: Run xcode-autobuilder tests
-      if: runner.os == 'macOS'
-      shell: bash
-      run: |
-        bazel test //swift/xcode-autobuilder/tests
-    - name: Run codegen tests
-      shell: bash
-      run: |
-        bazel test //swift/codegen/test
-    - name: Run qltest tests
-      shell: bash
-      run: |
-        bazel test //swift/tools/test/qltest
--- a/swift/actions/setup-env/action.yml
+++ b/swift/actions/setup-env/action.yml
@ -1,24 +0,0 @@
-name: Setup Swift Package Environment
-description: Sets up the environment in which to build the Swift package and run its tests
-runs:
-  using: composite
-  steps:
-    - uses: bazelbuild/setup-bazelisk@v2
-    - uses: actions/setup-python@v4
-      with:
-        python-version-file: 'swift/.python-version'
-    - name: Mount bazel cache
-      uses: actions/cache@v3
-      with:
-        path: "~/.cache/bazel-repository-cache"
-        key: bazel-cache-${{ runner.os }}-${{ runner.arch }}
-    - name: Mount bazel disk cache
-      uses: actions/cache@v3
-      with:
-        path: "~/.cache/bazel-disk-cache"
-        key: bazel-disk-cache-${{ runner.os }}-${{ runner.arch }}
-    - name: Set up bazel disk cache
-      shell: bash
-      run: |
-        echo build --repository_cache=~/.cache/bazel-repository-cache --disk_cache=~/.cache/bazel-disk-cache > ~/.bazelrc
-        echo test --test_output=errors >> ~/.bazelrc