Make a dedicated, fast, workflow just for docs-internal only (#23289)

* Make a dedicated, fast, workflow just for docs-internal only

Part of #1297

* make staging-build-pr only for github/docs

* prune later

* make it louder and clearer about disabling workflows

* does it merge?

* typo

* rename ref

* rename

* early access should be good to go

* far from perfect

* start with that

* gzip

* rearrange

* html_url

* correction of actions/checkout sha

* correction of actions/setup-node sha

* quote

* ooops

* actually deploy

* move @octokit/rest to dependencies

* await-sleep hack

* reinstall npm

* typo

* CONTEXT_NAME

* deployments:write permission

* pull-requests:read permission

* actions:read and statuses:write permissions

* private repo mention exception

* it's called github.run_id

* Apply suggestions from code review

Co-authored-by: James M. Greene <JamesMGreene@github.com>

* make CONTEXT_NAME optional (if it works)

* comment out CONTEXT_NAME

* simplifying

* going to run on on.pull_request instead

* remove comment

* only the 2-phase staging deploy on github/docs

* better if statement on label check

* refactor of staging-deploy script

* switch to npm install to get the deDependencies back

* using --only=dev

* updating comments

* event_name

* not on pushes to main

* add staging-commit-status-success

* testing testing

* fix linting error

* Remove other docs-internal references from staging-deploy-pr.yml

* Cleaning up new staging-commit-status-success script and usage

* Remove unnecessary environment refs

* Remove unnecessary fallback

Since the only event trigger is pull_request now instead of also push

* Remove unnecessary env vars from workflow

* docs-internal or docs but not both

* Don't provide unnecessary environment refs

* remove now moot exception

* setting it to pull_request_target

Co-authored-by: James M. Greene <JamesMGreene@github.com>
This commit is contained in:
Peter Bengtsson 2021-12-14 08:57:54 -05:00 коммит произвёл GitHub
Родитель a5340e8232
Коммит f68eb74d90
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
8 изменённых файлов: 304 добавлений и 118 удалений

36
.github/actions-scripts/staging-commit-status-success.js поставляемый Executable file
Просмотреть файл

@ -0,0 +1,36 @@
#!/usr/bin/env node
import getOctokit from '../../script/helpers/github.js'
const { GITHUB_TOKEN } = process.env
// Exit if GitHub Actions PAT is not found
if (!GITHUB_TOKEN) {
throw new Error('You must supply a GITHUB_TOKEN environment variable!')
}
// This helper uses the `GITHUB_TOKEN` implicitly!
// We're using our usual version of Octokit vs. the provided `github`
// instance to avoid versioning discrepancies.
const octokit = getOctokit()
const { CONTEXT_NAME, ACTIONS_RUN_LOG, HEAD_SHA } = process.env
if (!CONTEXT_NAME) {
throw new Error('$CONTEXT_NAME not set')
}
if (!ACTIONS_RUN_LOG) {
throw new Error('$ACTIONS_RUN_LOG not set')
}
if (!HEAD_SHA) {
throw new Error('$HEAD_SHA not set')
}
await octokit.repos.createCommitStatus({
owner,
repo,
sha: HEAD_SHA,
context: CONTEXT_NAME,
state: 'success',
description: 'Successfully deployed! See logs.',
target_url: ACTIONS_RUN_LOG,
})

21
.github/actions-scripts/staging-deploy.js поставляемый
Просмотреть файл

@ -21,7 +21,7 @@ if (!HEROKU_API_TOKEN) {
// instance to avoid versioning discrepancies.
const octokit = getOctokit()
const { RUN_ID, PR_URL, SOURCE_BLOB_URL, CONTEXT_NAME, ACTIONS_RUN_LOG, HEAD_SHA } = process.env
const { RUN_ID, PR_URL, SOURCE_BLOB_URL } = process.env
if (!RUN_ID) {
throw new Error('$RUN_ID not set')
}
@ -31,15 +31,6 @@ if (!PR_URL) {
if (!SOURCE_BLOB_URL) {
throw new Error('$SOURCE_BLOB_URL not set')
}
if (!CONTEXT_NAME) {
throw new Error('$CONTEXT_NAME not set')
}
if (!ACTIONS_RUN_LOG) {
throw new Error('$ACTIONS_RUN_LOG not set')
}
if (!HEAD_SHA) {
throw new Error('$HEAD_SHA not set')
}
const { owner, repo, pullNumber } = parsePrUrl(PR_URL)
if (!owner || !repo || !pullNumber) {
@ -62,13 +53,3 @@ await deployToStaging({
sourceBlobUrl: SOURCE_BLOB_URL,
runId: RUN_ID,
})
await octokit.repos.createCommitStatus({
owner,
repo,
sha: HEAD_SHA,
context: CONTEXT_NAME,
state: 'success',
description: 'Successfully deployed! See logs.',
target_url: ACTIONS_RUN_LOG,
})

6
.github/workflows/prod-build-deploy.yml поставляемый
Просмотреть файл

@ -52,6 +52,12 @@ jobs:
DOCUBOT_REPO_PAT: ${{ secrets.DOCUBOT_REPO_PAT }}
GIT_BRANCH: main
- name: Cache nextjs build
uses: actions/cache@c64c572235d810460d0d6876e9c705ad5002b353
with:
path: .next/cache
key: ${{ runner.os }}-nextjs-${{ hashFiles('package*.json') }}
- name: Build
run: npm run build

227
.github/workflows/staging-build-and-deploy-pr.yml поставляемый Normal file
Просмотреть файл

@ -0,0 +1,227 @@
name: Staging - Build and Deploy PR (fast and private-only)
# **What it does**: Builds and deploys PRs to staging but ONLY for docs-internal
# **Why we have it**: Most PRs are made on the private repo. Let's make those extra fast if we can worry less about security.
# **Who does it impact**: All staff.
# This whole workflow is only guaranteed to be secure in the *private
# repo* and because we repo-sync these files over the to the public one,
# IT'S CRUCIALLY IMPORTANT THAT THIS WORKFLOW IS ONLY ENABLED IN docs-internal!
on:
# Ideally, we'd like to use 'pull_request' because we can more easily
# test changes to this workflow without relying on merges to 'main'.
# But this is guaranteed to be safer and won't have the problem of
# necessary secrets not being available.
# Perhaps some day when we're confident this workflow will always
# work in a regular PR, we can switch to that.
pull_request_target:
permissions:
actions: read
contents: read
deployments: write
pull-requests: read
statuses: write
# This allows one Build workflow run to interrupt another
# These are different from the concurrency in that here it checks if the
# whole workflow runs again. The "inner concurrency" is used for
# undeployments to cleaning up resources.
concurrency:
group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label }}'
cancel-in-progress: true
jobs:
build-and-deploy:
# Important. This whole file is only supposed to run in the PRIVATE repo.
if: ${{ github.repository == 'github/docs-internal' }}
# The assumption here is that self-hosted is faster (e.g CPU power)
# that the regular ones. And it matters in this workflow because
# we do heavy CPU stuff with `npm run build` and `tar`
# runs-on: ubuntu-latest
runs-on: self-hosted
timeout-minutes: 5
# This interrupts Build, Deploy, and pre-write Undeploy workflow runs in
# progress for this PR branch.
concurrency:
group: 'PR Staging @ ${{ github.event.pull_request.head.label }}'
cancel-in-progress: true
steps:
- name: Check out repo
uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579
with:
ref: ${{ github.event.pull_request.head.sha }}
lfs: 'true'
# To prevent issues with cloning early access content later
persist-credentials: 'false'
- name: Check out LFS objects
run: git lfs checkout
- name: Setup node
uses: actions/setup-node@04c56d2f954f1e4c69436aa54cfef261a018f458
with:
node-version: 16.13.x
cache: npm
- name: Install dependencies
run: npm ci
- name: Cache nextjs build
uses: actions/cache@c64c572235d810460d0d6876e9c705ad5002b353
with:
path: .next/cache
key: ${{ runner.os }}-nextjs-${{ hashFiles('package*.json') }}
- name: Build
run: npm run build
- name: Clone early access
run: node script/early-access/clone-for-build.js
env:
DOCUBOT_REPO_PAT: ${{ secrets.DOCUBOT_REPO_PAT }}
GIT_BRANCH: ${{ github.event.pull_request.head.sha }}
- name: Check that the PR isn't blocking deploys
# We can't use ${{...}} on this if statement because of this bug
# https://github.com/cschleiden/actions-linter/issues/114
if: github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'automated-block-deploy')
run: |
echo "The PR appears to have the label 'automated-block-deploy'"
echo "Will not proceed to deploy the PR."
exit 2
- name: Create a Heroku build source
id: build-source
uses: actions/github-script@2b34a689ec86a68d8ab9478298f91d5401337b7d
env:
HEROKU_API_TOKEN: ${{ secrets.HEROKU_API_TOKEN }}
with:
script: |
const { owner, repo } = context.repo
if (owner !== 'github') {
throw new Error(`Repository owner must be 'github' but was: ${owner}`)
}
if (repo !== 'docs-internal') {
throw new Error(`Repository name must be 'docs-internal' but was: ${repo}`)
}
const Heroku = require('heroku-client')
const heroku = new Heroku({ token: process.env.HEROKU_API_TOKEN })
try {
const { source_blob: sourceBlob } = await heroku.post('/sources')
const { put_url: uploadUrl, get_url: downloadUrl } = sourceBlob
core.setOutput('upload_url', uploadUrl)
core.setOutput('download_url', downloadUrl)
} catch (error) {
if (error.statusCode === 503) {
console.error('💀 Heroku may be down! Please check its Status page: https://status.heroku.com/')
}
throw error
}
- name: Remove development-only dependencies
run: npm prune --production
- name: Remove all npm scripts
run: npm pkg delete scripts
- name: Set npm script for Heroku build to noop
run: npm set-script heroku-postbuild "echo 'Application was pre-built!'"
- name: Delete heavy things we won't need deployed
run: |
# The dereferenced file is not used in runtime once the
# decorated file has been created from it.
rm -rf lib/rest/static/dereferenced
# Translations are never tested in Staging builds
# but let's keep the empty directory.
rm -rf translations
mkdir translations
# Delete all the big search indexes that are NOT English (`*-en-*`)
pushd lib/search/indexes
ls | grep -v '\-en\-' | xargs rm
popd
# Note! Some day it would be nice to be able to delete
# all the heavy assets because they bloat the tarball.
# But it's not obvious how to test it then. For now, we'll have
# to accept that every staging build has a copy of the images.
# The assumption here is that a staging build will not
# need these legacy redirects. Only the redirects from
# front-matter will be at play.
# These static redirects json files are notoriously large
# and they make the tarball unnecessarily large.
echo '[]' > lib/redirects/static/archived-frontmatter-fallbacks.json
echo '{}' > lib/redirects/static/developer.json
echo '{}' > lib/redirects/static/archived-redirects-from-213-to-217.json
# This will turn every `lib/**/static/*.json` into
# an equivalent `lib/**/static/*.json.br` file.
# Once the server starts, it'll know to fall back to reading
# the `.br` equivalent if the `.json` file isn't present.
node .github/actions-scripts/compress-large-files.js
- name: Make the tarball for Heroku
run: |
# We can't delete the .next/cache directory from the workflow
# because it's needed for caching, but we can at least exclude it
# from the tarball. Then it can be cached but not weigh down the
# tarball we intend to deploy.
tar -zc --exclude=.next/cache --file=app.tar.gz \
node_modules/ \
.next/ \
assets/ \
content/ \
data/ \
includes/ \
lib/ \
middleware/ \
translations/ \
server.mjs \
package*.json \
.npmrc \
feature-flags.json \
next.config.js \
app.json \
Procfile
du -sh app.tar.gz
# See: https://devcenter.heroku.com/articles/build-and-release-using-the-api#sources-endpoint
- name: Upload to the Heroku build source
env:
UPLOAD_URL: ${{ steps.build-source.outputs.upload_url }}
run: |
curl "$UPLOAD_URL" \
-X PUT \
-H 'Content-Type:' \
--data-binary @app.tar.gz
# 'npm install' is faster than 'npm ci' because it only needs to
# *append* what's missing from ./node_modules/
- name: Re-install dependencies so we get devDependencies back
run: npm install --no-audit --no-fund --only=dev
- name: Deploy
id: deploy
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
HEROKU_API_TOKEN: ${{ secrets.HEROKU_API_TOKEN }}
HYDRO_ENDPOINT: ${{ secrets.HYDRO_ENDPOINT }}
HYDRO_SECRET: ${{ secrets.HYDRO_SECRET }}
PR_URL: ${{ github.event.pull_request.html_url }}
SOURCE_BLOB_URL: ${{ steps.build-source.outputs.download_url }}
ALLOWED_POLLING_FAILURES_PER_PHASE: '15'
RUN_ID: ${{ github.run_id }}
run: .github/actions-scripts/staging-deploy.js

46
.github/workflows/staging-build-pr.yml поставляемый
Просмотреть файл

@ -4,6 +4,8 @@ name: Staging - Build PR
# **Why we have it**: Because it's not safe to share our deploy secrets with forked repos: https://securitylab.github.com/research/github-actions-preventing-pwn-requests/
# **Who does it impact**: All contributors.
# IT'S CRUCIALLY IMPORTANT THAT THIS WORKFLOW IS ONLY ENABLED IN docs!
on:
pull_request:
types:
@ -31,7 +33,9 @@ concurrency:
jobs:
build-pr:
if: ${{ github.repository == 'github/docs-internal' || github.repository == 'github/docs' }}
# Important. This whole file is only supposed to run in the PUBLIC repo.
if: ${{ github.repository == 'github/docs' }}
runs-on: ${{ fromJSON('["ubuntu-latest", "self-hosted"]')[github.repository == 'github/docs-internal'] }}
timeout-minutes: 5
# This interrupts Build, Deploy, and pre-write Undeploy workflow runs in
@ -45,7 +49,7 @@ jobs:
# Make sure only approved files are changed if it's in github/docs
- name: Check changed files
if: ${{ github.repository == 'github/docs' && github.event.pull_request.user.login != 'Octomerger' }}
if: ${{ github.event.pull_request.user.login != 'Octomerger' }}
uses: dorny/paths-filter@eb75a1edc117d3756a18ef89958ee59f9500ba58
id: filter
with:
@ -107,44 +111,6 @@ jobs:
- name: Set npm script for Heroku build to noop
run: npm set-script heroku-postbuild "echo 'Application was pre-built!'"
- name: Delete heavy things we won't need deployed
if: ${{ github.repository == 'github/docs-internal' }}
run: |
# The dereferenced file is not used in runtime once the
# decorated file has been created from it.
rm -fr lib/rest/static/dereferenced
# Translations are never tested in Staging builds
# but let's keep the empty directory.
rm -fr translations
mkdir translations
# Delete all the big search indexes that are NOT English (`*-en-*`)
pushd lib/search/indexes
ls | grep -v '\-en\-' | xargs rm
popd
# Note! Some day it would be nice to be able to delete
# all the heavy assets because they bloat the tarball.
# But it's not obvious how to test it then. For now, we'll have
# to accept that every staging build has a copy of the images.
# The assumption here is that a staging build will not
# need these legacy redirects. Only the redirects from
# front-matter will be at play.
# These static redirects json files are notoriously large
# and they make the tarball unnecessarily large.
echo '[]' > lib/redirects/static/archived-frontmatter-fallbacks.json
echo '{}' > lib/redirects/static/developer.json
echo '{}' > lib/redirects/static/archived-redirects-from-213-to-217.json
# This will turn every `lib/**/static/*.json` into
# an equivalent `lib/**/static/*.json.br` file.
# Once the server starts, it'll know to fall back to reading
# the `.br` equivalent if the `.json` file isn't present.
node .github/actions-scripts/compress-large-files.js
- name: Create an archive
# Only bother if this is actually a pull request
if: ${{ github.event.pull_request.number }}

80
.github/workflows/staging-deploy-pr.yml поставляемый
Просмотреть файл

@ -4,6 +4,8 @@ name: Staging - Deploy PR
# **Why we have it**: To deploy with high visibility in case of failures.
# **Who does it impact**: All contributors.
# IT'S CRUCIALLY IMPORTANT THAT THIS WORKFLOW IS ONLY ENABLED IN docs!
on:
workflow_run:
workflows:
@ -46,10 +48,12 @@ jobs:
# This is needed because the workflow we depend on
# (see on.workflow_run.workflows) might be running from pushes on
# main. That's because it needs to do that to popular the cache.
if: >
${{ github.event.workflow_run.event == 'pull_request' &&
github.event.workflow_run.conclusion == 'success' }}
if: >-
${{
github.repository == 'github/docs' &&
github.event.workflow_run.event == 'pull_request' &&
github.event.workflow_run.conclusion == 'success'
}}
runs-on: ubuntu-latest
outputs:
number: ${{ steps.pr.outputs.number }}
@ -164,8 +168,7 @@ jobs:
if: >-
${{
needs.pr-metadata.outputs.number != '0' &&
github.event.workflow_run.conclusion == 'failure' &&
(github.repository == 'github/docs-internal' || github.repository == 'github/docs')
github.event.workflow_run.conclusion == 'failure'
}}
runs-on: ubuntu-latest
timeout-minutes: 1
@ -208,8 +211,7 @@ jobs:
if: >-
${{
needs.pr-metadata.outputs.number != '0' &&
github.event.workflow_run.conclusion == 'success' &&
(github.repository == 'github/docs-internal' || github.repository == 'github/docs')
github.event.workflow_run.conclusion == 'success'
}}
runs-on: ubuntu-latest
# This timeout should match or exceed the value of the timeout for Undeploy
@ -262,7 +264,7 @@ jobs:
prepare-for-deploy:
needs: [pr-metadata, check-pr-before-prepare]
if: ${{ needs.check-pr-before-prepare.outputs.pull_request_state == 'open' }}
runs-on: ${{ fromJSON('["ubuntu-latest", "self-hosted"]')[github.repository == 'github/docs-internal'] }}
runs-on: ubuntu-latest
timeout-minutes: 5
# This interrupts Build, Deploy, and pre-write Undeploy workflow runs in
# progress for this PR branch.
@ -308,22 +310,10 @@ jobs:
node-version: 16.13.x
cache: npm
# Install any dependencies that are needed for the early access script
- if: ${{ github.repository == 'github/docs-internal' }}
name: Install temporary dependencies
run: npm install --no-save dotenv rimraf
# Install any additional dependencies *before* downloading the build artifact
- name: Install Heroku client development-only dependency
run: npm install --no-save heroku-client
- if: ${{ github.repository == 'github/docs-internal' }}
name: Clone early access
run: node script/early-access/clone-for-build.js
env:
DOCUBOT_REPO_PAT: ${{ secrets.DOCUBOT_REPO_PAT }}
GIT_BRANCH: ${{ needs.pr-metadata.outputs.head_ref }}
# Download the previously built "app.tar"
- name: Download build artifact
uses: dawidd6/action-download-artifact@af92a8455a59214b7b932932f2662fdefbd78126
@ -333,35 +323,8 @@ jobs:
name: pr_build
path: ${{ runner.temp }}
# For security reasons, only extract the tar from docs-internal
# This allows us to add search indexes and early access content to the build
- if: ${{ github.repository == 'github/docs-internal' }}
name: Extract user-changes to temp directory
run: |
mkdir $RUNNER_TEMP/app
tar -x --file=$RUNNER_TEMP/app.tar -C "$RUNNER_TEMP/app/"
# Move the LFS content into the temp directory in chunks (destructively)
- if: ${{ github.repository == 'github/docs-internal' }}
name: Move the LFS objects
run: |
git lfs ls-files --name-only | xargs -n 1 -I {} sh -c 'mkdir -p "$RUNNER_TEMP/app/$(dirname {})"; mv {} "$RUNNER_TEMP/app/$(dirname {})/"'
# Move the early access content into the temp directory (destructively)
- if: ${{ github.repository == 'github/docs-internal' }}
name: Move the early access content
run: |
mv assets/images/early-access "$RUNNER_TEMP/app/assets/images/"
mv content/early-access "$RUNNER_TEMP/app/content/"
mv data/early-access "$RUNNER_TEMP/app/data/"
- if: ${{ github.repository == 'github/docs-internal' }}
name: Create a gzipped archive (docs-internal)
run: tar -cz --file app.tar.gz "$RUNNER_TEMP/app/"
# gzip the app.tar from github/docs so we're working with the same format
- if: ${{ github.repository == 'github/docs' }}
name: Create a gzipped archive (docs)
# gzip the app.tar to meet Heroku's expected format
- name: Create a gzipped archive (docs)
run: gzip -9 < "$RUNNER_TEMP/app.tar" > app.tar.gz
- name: Create a Heroku build source
@ -376,8 +339,8 @@ jobs:
if (owner !== 'github') {
throw new Error(`Repository owner must be 'github' but was: ${owner}`)
}
if (repo !== 'docs-internal' && repo !== 'docs') {
throw new Error(`Repository name must be either 'docs-internal' or 'docs' but was: ${repo}`)
if (repo !== 'docs') {
throw new Error(`Repository name must be 'docs' but was: ${repo}`)
}
const Heroku = require('heroku-client')
@ -466,7 +429,7 @@ jobs:
deploy:
needs: [pr-metadata, prepare-for-deploy, check-pr-before-deploy]
if: ${{ needs.check-pr-before-deploy.outputs.pull_request_state == 'open' }}
runs-on: ${{ fromJSON('["ubuntu-latest", "self-hosted"]')[github.repository == 'github/docs-internal'] }}
runs-on: ubuntu-latest
timeout-minutes: 10
# This interrupts Build, Deploy, and pre-write Undeploy workflow runs in
# progress for this PR branch.
@ -495,13 +458,18 @@ jobs:
HYDRO_SECRET: ${{ secrets.HYDRO_SECRET }}
PR_URL: ${{ needs.pr-metadata.outputs.url }}
SOURCE_BLOB_URL: ${{ needs.prepare-for-deploy.outputs.source_blob_url }}
CONTEXT_NAME: ${{ env.CONTEXT_NAME }}
ACTIONS_RUN_LOG: ${{ env.ACTIONS_RUN_LOG }}
HEAD_SHA: ${{ needs.pr-metadata.outputs.head_sha }}
ALLOWED_POLLING_FAILURES_PER_PHASE: '15'
RUN_ID: ${{ github.run_id }}
run: .github/actions-scripts/staging-deploy.js
- name: Create successful commit status
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
CONTEXT_NAME: ${{ env.CONTEXT_NAME }}
ACTIONS_RUN_LOG: ${{ env.ACTIONS_RUN_LOG }}
HEAD_SHA: ${{ needs.pr-metadata.outputs.head_sha }}
run: .github/actions-scripts/staging-commit-status-success.js
- name: Mark the deployment as inactive if timed out
uses: actions/github-script@2b34a689ec86a68d8ab9478298f91d5401337b7d
if: ${{ steps.deploy.outcome == 'cancelled' }}

2
.github/workflows/test.yml поставляемый
Просмотреть файл

@ -86,7 +86,7 @@ jobs:
uses: actions/cache@c64c572235d810460d0d6876e9c705ad5002b353
with:
path: .next/cache
key: ${{ runner.os }}-nextjs-${{ hashFiles('package*.json') }}-${{ hashFiles('.github/workflows/test.yml') }}
key: ${{ runner.os }}-nextjs-${{ hashFiles('package*.json') }}
- name: Run build script
run: npm run build

Просмотреть файл

@ -1,10 +1,12 @@
#!/usr/bin/env node
import sleep from 'await-sleep'
import got from 'got'
import Heroku from 'heroku-client'
import { setOutput } from '@actions/core'
import createStagingAppName from './create-staging-app-name.js'
// Equivalent of the 'await-sleep' module without the install
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms))
const SLEEP_INTERVAL = 5000
const HEROKU_LOG_LINES_TO_SHOW = 25