Deploy to staging manually using a script (#19769)

* Add 'script/deploy' to enable manual deploys to Heroku

* Pass API tokens into 'deploy-to-staging' module usage

* Construct Octokit instance to pass in

* Get PR branch name and verify state

* Reorganize

* Rename option to 'octokit'

* Add missing option

* Actually use the convenience methods for convenience

* Simplify top-level script

* Top-level script revisions

* Add parse-pr-url module

* Add create-staging-app-name module

* Remove misplaced comment

* Pass in owner

* Use owner param

* More variables

* Pass owner along more

* Correct prNumber param reference

* Add WIP deploy-to-staging module

* Prevent 'scripts/' and '.github/actions-scripts/' files from being modified in open source repo

* Extract PR author earlier

* Add note about optionally supplying DOCUBOT_REPO_PAT env var

* Override Heroku env var during AppSetup creation instead of later to avoid triggering a second deploy

* Updates to deploy-to-staging module

* Lots of updates

* Add dyno start-up monitoring and warmup requests

* Ignore 'script/deploy' in the repository-references test

* Correct path to Octokit helper

* Temporarily add a 'gha-' prefix to environment names

* Log whole error if terminal. Good for Octokit errors!

* Correct Octokit preview configuration

* Add more logging around Heroku build and release

* Added more timings to log messages

* Monitor dyno states specifically from the dyno list view to avoid 404 oddities when Free dynos are dropped and non-Free dynos are added

* Don't wait for AppSetup status as it includes the Build time

* Updating logging since we don't see DeploymentStatus update messages in the UI =(

* Remove commented out code

* Refactor to extract more properties from the PR object

* Fix reference to pull request number

* Increase Heroku polling intervals from 2.5 seconds to 5 seconds

* Remove unhelpful createDeploymentStatus API calls

* Workaround Heroku's secondary release upon app creation
This commit is contained in:
James M. Greene 2021-06-14 17:32:07 -05:00 коммит произвёл GitHub
Родитель 70fbdc70c9
Коммит f388a3d550
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
6 изменённых файлов: 654 добавлений и 7 удалений

Просмотреть файл

@ -7,15 +7,17 @@ name: Check unallowed file changes
on:
pull_request_target:
paths:
- '.github/actions-scripts/**'
- '.github/workflows/**'
- '.github/CODEOWNERS'
- 'translations/**'
- 'assets/fonts/**'
- 'data/graphql/**'
- 'lib/graphql/**'
- 'lib/redirects/**'
- 'lib/rest/**'
- 'lib/webhooks/**'
- 'scripts/**'
- 'translations/**'
jobs:
triage:
@ -63,15 +65,17 @@ jobs:
openapi:
- 'lib/rest/static/**'
notAllowed:
- '.github/actions-scripts/**'
- '.github/workflows/**'
- '.github/CODEOWNERS'
- 'translations/**'
- 'assets/fonts/**'
- 'data/graphql/**'
- 'lib/graphql/**'
- 'lib/redirects/**'
- 'lib/rest/**'
- 'lib/webhooks/**'
- 'scripts/**'
- 'translations/**'
# When there are changes to files we can't accept
# and no review exists,leave a REQUEST_CHANGES review
@ -83,17 +87,17 @@ jobs:
github-token: ${{secrets.GITHUB_TOKEN}}
script: |
const badFilesArr = [
'translations/**',
'lib/rest/static/**',
'.github/actions-scripts/**',
'.github/workflows/**',
'.github/CODEOWNERS',
'translations/**',
'assets/fonts/**',
'data/graphql/**',
'lib/graphql/**',
'lib/redirects/**',
'lib/rest/**',
'lib/webhooks/**'
'lib/webhooks/**',
'scripts/**',
'translations/**'
]
const badFiles = badFilesArr.join('\n')

151
script/deploy Executable file
Просмотреть файл

@ -0,0 +1,151 @@
#!/usr/bin/env node
// [start-readme]
//
// This script is run by a GitHub Actions workflow to trigger deployments
// to Heroku for both staging and production apps.
//
// You can also run it locally if you:
// - Supply a GitHub PAT as the GITHUB_TOKEN environment variable
// - Supply a Heroku API token as the HEROKU_API_TOKEN environment variable
// - Optionally, supply a GitHub PAT as the DOCUBOT_REPO_PAT environment
// variable if you want to support content from the `docs-early-access` repo
//
// Examples:
// - Deploy a PR to Staging:
// script/deploy --staging https://github.com/github/docs-internal/pull/12345
//
// - Deploy a PR to Staging and force the Heroku App to be rebuilt from scratch
// script/deploy --staging https://github.com/github/docs/pull/9876 --rebuild
//
// - Deploy the latest from docs-internal `main` to production
// script/deploy --production
//
// [end-readme]
require('dotenv').config()
const { GITHUB_TOKEN, HEROKU_API_TOKEN } = process.env
// Exit if GitHub Actions PAT is not found
if (!GITHUB_TOKEN) {
throw new Error('You must supply a GITHUB_TOKEN environment variable!')
}
// Exit if Heroku API token is not found
if (!HEROKU_API_TOKEN) {
throw new Error('You must supply a HEROKU_API_TOKEN environment variable!')
}
const program = require('commander')
const { has } = require('lodash')
const getOctokit = require('./helpers/github')
const parsePrUrl = require('./deployment/parse-pr-url')
const deployToStaging = require('./deployment/deploy-to-staging')
const STAGING_FLAG = '--staging'
const PRODUCTION_FLAG = '--production'
const ALLOWED_OWNER = 'github'
const ALLOWED_SOURCE_REPOS = ['docs', 'docs-internal']
const EXPECTED_PR_URL_FORMAT = `https://github.com/${ALLOWED_OWNER}/(${ALLOWED_SOURCE_REPOS.join('|')})/pull/123`
program
.description('Trigger a deployment to Heroku for either staging or production apps')
.option(PRODUCTION_FLAG, 'Deploy the latest internal main branch to Production')
.option(`${STAGING_FLAG} <PR_URL>`, 'Deploy a pull request to Staging')
.option('--rebuild', 'Force a Staging deployment to rebuild the Heroku App from scratch')
.parse(process.argv)
const opts = program.opts()
const isProduction = opts.production === true
const isStaging = has(opts, 'staging')
const prUrl = opts.staging
const forceRebuild = opts.rebuild === true
//
// Verify CLI options
//
if (!isProduction && !isStaging) {
return invalidateAndExit(
'commander.missingArgument',
`error: must specify option '${STAGING_FLAG} <PR_URL>' or '${PRODUCTION_FLAG}'`
)
}
if (isProduction && isStaging) {
return invalidateAndExit(
'commander.conflictingArgument',
`error: must specify option '${STAGING_FLAG} <PR_URL>' or '${PRODUCTION_FLAG}' but not both`
)
}
if (isProduction && forceRebuild) {
return invalidateAndExit(
'commander.conflictingArgument',
`error: cannot specify option '--rebuild' combined with option '${PRODUCTION_FLAG}'`
)
}
// Extract the repository name and pull request number from the URL (if any)
const { owner, repo, pullNumber } = parsePrUrl(prUrl)
if (isStaging) {
if (owner !== ALLOWED_OWNER || !ALLOWED_SOURCE_REPOS.includes(repo) || !pullNumber) {
return invalidateAndExit(
'commander.invalidOptionArgument',
`error: option '${STAGING_FLAG}' argument '${prUrl}' is invalid.
Must match URL format '${EXPECTED_PR_URL_FORMAT}'`
)
}
}
deploy()
//
// Function definitions
//
function invalidateAndExit (errorType, message) {
program._displayError(1, errorType, message)
process.exit(1)
}
async function deploy () {
if (isProduction) {
await deployProduction()
} else if (isStaging) {
await deployStaging({ owner, repo, pullNumber, forceRebuild })
}
}
async function deployProduction () {
// TODO: Request confirmation before deploying to production
return invalidateAndExit(
'commander.invalidOptionArgument',
`error: option '${PRODUCTION_FLAG}' is not yet implemented. SOON!`
)
}
async function deployStaging ({ owner, repo, pullNumber, forceRebuild = false }) {
// This helper uses the `GITHUB_TOKEN` implicitly
const octokit = getOctokit()
const { data: pullRequest } = await octokit.pulls.get({
owner,
repo,
pull_number: pullNumber
})
try {
await deployToStaging({
herokuToken: HEROKU_API_TOKEN,
octokit,
pullRequest,
forceRebuild
})
} catch (error) {
console.error(`Failed to deploy to staging: ${error.message}`)
console.error(error)
process.exit(1)
}
}

Просмотреть файл

@ -0,0 +1,18 @@
const slugify = require('github-slugger').slug
const APP_NAME_MAX_LENGTH = 30
module.exports = function ({ repo, pullNumber, branch }) {
//
// TODO: Remove the 'gha-' prefix!!!
//
return `gha-${repo}-${pullNumber}--${slugify(branch)}`
// Shorten the string to the max allowed length
.slice(0, APP_NAME_MAX_LENGTH)
// Convert underscores to dashes
.replace(/_/g, '-')
// Remove trailing dashes
.replace(/-+$/, '')
// Make it all lowercase
.toLowerCase()
}

Просмотреть файл

@ -0,0 +1,453 @@
const sleep = require('await-sleep')
const got = require('got')
const Heroku = require('heroku-client')
const createStagingAppName = require('./create-staging-app-name')
const SLEEP_INTERVAL = 5000
const HEROKU_LOG_LINES_TO_SHOW = 25
module.exports = async function deployToStaging ({ herokuToken, octokit, pullRequest, forceRebuild = false }) {
// Start a timer so we can report how long the deployment takes
const startTime = Date.now()
// Extract some important properties from the PR
const {
number: pullNumber,
base: {
repo: {
name: repo,
owner: { login: owner }
}
},
state,
head: {
ref: branch,
sha
},
user: author
} = pullRequest
// Verify the PR is still open
if (state !== 'open') {
throw new Error(`This pull request is not open. State is: '${state}'`)
}
let deploymentId = null
let logUrl = null
let appIsNewlyCreated = false
const appName = createStagingAppName({ repo, pullNumber, branch })
const homepageUrl = `https://${appName}.herokuapp.com/`
try {
const title = `branch '${branch}' at commit '${sha}' in the 'staging' environment as '${appName}'`
console.log(`About to deploy ${title}...`)
// Kick off a pending GitHub Deployment right away, so the PR author
// will have instant feedback that their work is being deployed.
const { data: deployment } = await octokit.repos.createDeployment({
owner,
repo,
description: `Deploying ${title}`,
// Use a commit SHA instead of a branch name as the ref for more precise
// feedback, and also because the branch may have already been deleted.
ref: sha,
// In the GitHub API, there can only be one active deployment per environment.
// For our many staging apps, we must use the unique appName as the environment.
environment: appName,
// Indicate this environment will no longer exist at some point in the future.
transient_environment: true,
// The status contexts to verify against commit status checks. If you omit
// this parameter, GitHub verifies all unique contexts before creating a
// deployment. To bypass checking entirely, pass an empty array. Defaults
// to all unique contexts.
required_contexts: [],
// Do not try to merge the base branch into the feature branch
auto_merge: false
})
console.log('GitHub Deployment created', deployment)
// Store this ID for later updating
deploymentId = deployment.id
await octokit.repos.createDeploymentStatus({
owner,
repo,
deployment_id: deploymentId,
state: 'in_progress',
description: 'Deploying the app...',
// The 'flash' preview is required for `state` values of 'in_progress' and 'queued'
mediaType: {
previews: ['flash']
}
})
console.log('🚀 Deployment status: in_progress - Preparing to deploy the app...')
// Get a URL for the tarballed source code bundle
const { headers: { location: tarballUrl } } = await octokit.repos.downloadTarballArchive({
owner,
repo,
ref: sha,
// Override the underlying `node-fetch` module's `redirect` option
// configuration to prevent automatically following redirects.
request: {
redirect: 'manual'
}
})
// Time to talk to Heroku...
const heroku = new Heroku({ token: herokuToken })
let appSetup = null
let build = null
// Is there already a Heroku App for this PR?
let appExists = true
try {
await heroku.get(`/apps/${appName}`)
} catch (error) {
appExists = false
}
// If there is an existing app but we want to forcibly rebuild, delete the app first
if (appExists && forceRebuild) {
console.log('🚀 Deployment status: in_progress - Destroying existing Heroku app...')
try {
await heroku.delete(`/apps/${appName}`)
appExists = false
console.log(`Heroku app '${appName}' deleted for forced rebuild`)
} catch (error) {
throw new Error(`Failed to delete Heroku app '${appName}' for forced rebuild. Error: ${error}`)
}
}
// If an app does not exist, create one!
// This action will also trigger a build as a by-product.
if (!appExists) {
appIsNewlyCreated = true
console.log(`Heroku app '${appName}' does not exist. Creating a new AppSetup...`)
console.log('🚀 Deployment status: in_progress - Creating a new Heroku app...')
const appSetupStartTime = Date.now()
try {
// IMPORTANT: These secrets should only be set in the private repo!
const { DOCUBOT_REPO_PAT, HYDRO_ENDPOINT, HYDRO_SECRET } = process.env
const secretEnvVars = {
// This is required for cloning the `docs-early-access` repo
...DOCUBOT_REPO_PAT && { DOCUBOT_REPO_PAT },
// These are required for Hydro event tracking
...(HYDRO_ENDPOINT && HYDRO_SECRET) && { HYDRO_ENDPOINT, HYDRO_SECRET }
}
appSetup = await heroku.post('/app-setups', {
body: {
app: {
name: appName
},
source_blob: {
url: tarballUrl
},
// Pass some secret environment variables to staging apps via Heroku
// config variables.
overrides: {
env: {
...secretEnvVars,
GIT_BRANCH: branch
}
}
}
})
console.log('Heroku AppSetup created', appSetup)
// This probably will not be available yet
build = appSetup.build
} catch (error) {
throw new Error(`Failed to create Heroku app '${appName}'. Error: ${error}`)
}
// Add PR author (if staff) as a collaborator on the new staging app
try {
if (author.site_admin === true) {
await heroku.post(`/apps/${appName}/collaborators`, {
body: {
user: `${author.login}@github.com`,
// We don't want an email invitation for every new staging app
silent: true
}
})
console.log(`Added PR author @${author.login} as a Heroku app collaborator`)
}
} catch (error) {
// It's fine if this fails, it shouldn't block the app from deploying!
console.warn(`Warning: failed to add PR author as a Heroku app collaborator. Error: ${error}`)
}
// A new Build is created as a by-product of creating an AppSetup.
// Poll until there is a Build object attached to the AppSetup.
while (!build || !build.id) {
await sleep(SLEEP_INTERVAL)
appSetup = await heroku.get(`/app-setups/${appSetup.id}`)
build = appSetup.build
console.log(`AppSetup status: ${appSetup.status} (after ${Math.round((Date.now() - appSetupStartTime) / 1000)} seconds)`)
}
console.log('Heroku build detected', build)
} else {
// If the app does exist, just manually trigger a new build
console.log(`Heroku app '${appName}' already exists. Building...`)
try {
build = await heroku.post(`/apps/${appName}/builds`, {
body: {
source_blob: {
url: tarballUrl
}
}
})
} catch (error) {
throw new Error(`Failed to create Heroku build. Error: ${error}`)
}
console.log('Heroku build created', build)
}
const buildStartTime = Date.now() // Close enough...
const buildId = build.id
logUrl = build.output_stream_url
console.log('🚀 Deployment status: in_progress - Building a new Heroku slug...')
// Poll until the Build's status changes from "pending" to "succeeded" or "failed".
while (!build || build.status === 'pending' || !build.release || !build.release.id) {
await sleep(SLEEP_INTERVAL)
try {
build = await heroku.get(`/apps/${appName}/builds/${buildId}`)
} catch (error) {
throw new Error(`Failed to get build status. Error: ${error}`)
}
console.log(`Heroku build status: ${(build || {}).status} (after ${Math.round((Date.now() - buildStartTime) / 1000)} seconds)`)
}
if (build.status !== 'succeeded') {
throw new Error(`Failed to build after ${Math.round((Date.now() - buildStartTime) / 1000)} seconds. See Heroku logs for more information:\n${logUrl}`)
}
console.log(`Finished Heroku build after ${Math.round((Date.now() - buildStartTime) / 1000)} seconds.`, build)
const releaseStartTime = Date.now() // Close enough...
let releaseId = build.release.id
let release = null
// Poll until the associated Release's status changes from "pending" to "succeeded" or "failed".
while (!release || release.status === 'pending') {
await sleep(SLEEP_INTERVAL)
try {
const result = await heroku.get(`/apps/${appName}/releases/${releaseId}`)
// Update the deployment status but only on the first retrieval
if (!release) {
logUrl = result.output_stream_url
console.log('Heroku Release created', result)
console.log('🚀 Deployment status: in_progress - Releasing the built Heroku slug...')
}
release = result
} catch (error) {
throw new Error(`Failed to get release status. Error: ${error}`)
}
console.log(`Release status: ${(release || {}).status} (after ${Math.round((Date.now() - releaseStartTime) / 1000)} seconds)`)
}
if (release.status !== 'succeeded') {
throw new Error(`Failed to release after ${Math.round((Date.now() - releaseStartTime) / 1000)} seconds. See Heroku logs for more information:\n${logUrl}`)
}
console.log(`Finished Heroku release after ${Math.round((Date.now() - releaseStartTime) / 1000)} seconds.`, release)
// Monitor dyno state for this release to ensure it reaches "up" rather than crashing.
// This will help us catch issues with faulty startup code and/or the package manifest.
const dynoBootStartTime = Date.now()
console.log('Checking Heroku dynos...')
logUrl = null
console.log('🚀 Deployment status: in_progress - Monitoring the Heroku dyno start-up...')
// Keep checking while there are still dynos in non-terminal states
let newDynos = []
while (newDynos.length === 0 || newDynos.some(dyno => dyno.state === 'starting')) {
await sleep(SLEEP_INTERVAL)
try {
const dynoList = await heroku.get(`/apps/${appName}/dynos`)
const dynosForThisRelease = dynoList.filter(dyno => dyno.release.id === releaseId)
// If this Heroku app was just newly created, often a secondary release
// is requested to enable automatically managed SSL certificates. The
// release description will read:
// "Enable allow-multiple-sni-endpoints feature"
//
// If that is the case, we need to update to monitor that secondary
// release instead.
if (newDynos.length > 0 && dynosForThisRelease.length === 0) {
// If the app is NOT newly created, fail fast!
if (!appIsNewlyCreated) {
throw new Error('The dynos for this release disappeared unexpectedly')
}
// Check for the secondary release
let nextRelease = null
try {
nextRelease = await heroku.get(`/apps/${appName}/releases/${release.version + 1}`)
} catch (error) {
throw new Error(`Could not find a secondary release to explain the disappearing dynos. Error: ${error}`)
}
if (nextRelease) {
if (nextRelease.description === 'Enable allow-multiple-sni-endpoints feature') {
// Track dynos for the next release instead
release = nextRelease
releaseId = nextRelease.id
console.warn('Switching to monitor secondary release...')
// Allow the loop to repeat to fetch the dynos for the secondary release
} else {
// Otherwise, assume another release replaced this one but it
// PROBABLY would've succeeded...?
newDynos.forEach(dyno => { dyno.state = 'up' })
}
}
// else just keep monitoring and hope for the best
}
newDynos = dynosForThisRelease
console.log(`Dyno states: ${JSON.stringify(newDynos.map(dyno => dyno.state))} (after ${Math.round((Date.now() - dynoBootStartTime) / 1000)} seconds)`)
} catch (error) {
throw new Error(`Failed to find dynos for this release. Error: ${error}`)
}
}
const crashedDynos = newDynos.filter(dyno => ['crashed', 'restarting'].includes(dyno.state))
const runningDynos = newDynos.filter(dyno => dyno.state === 'up')
// If any dynos crashed on start-up, fail the deployment
if (crashedDynos.length > 0) {
const errorMessage = `At least ${crashedDynos.length} Heroku dyno(s) crashed on start-up!`
console.error(errorMessage)
// Attempt to dump some of the Heroku log here for debugging
try {
const logSession = await heroku.post(`/apps/${appName}/log-sessions`, {
body: {
dyno: crashedDynos[0].name,
lines: HEROKU_LOG_LINES_TO_SHOW,
tail: false
}
})
logUrl = logSession.logplex_url
const logText = await got(logUrl).text()
console.error(`Here are the last ${HEROKU_LOG_LINES_TO_SHOW} lines of the Heroku log:\n\n${logText}`)
} catch (error) {
// Don't fail because of this error
console.error(`Failed to retrieve the Heroku logs for the crashed dynos. Error: ${error}`)
}
throw new Error(errorMessage)
}
console.log(`At least ${runningDynos.length} Heroku dyno(s) are ready after ${Math.round((Date.now() - dynoBootStartTime) / 1000)} seconds.`)
// Send a series of requests to trigger the server warmup routines
console.log('🚀 Deployment status: in_progress - Triggering server warmup routines...')
const warmupStartTime = Date.now()
console.log(`Making warmup requests to: ${homepageUrl}`)
try {
await got(homepageUrl, {
timeout: 10000, // Maximum 10 second timeout per request
retry: 7, // About 2 minutes 7 seconds of delay, plus active request time for 8 requests
hooks: {
beforeRetry: [
(options, error = {}, retryCount = '?') => {
const statusCode = error.statusCode || (error.response || {}).statusCode || -1
console.log(`Retrying after warmup request attempt #${retryCount} (${statusCode}) after ${Math.round((Date.now() - warmupStartTime) / 1000)} seconds...`)
}
]
}
})
console.log(`Warmup requests passed after ${Math.round((Date.now() - warmupStartTime) / 1000)} seconds`)
} catch (error) {
throw new Error(`Warmup requests failed after ${Math.round((Date.now() - warmupStartTime) / 1000)} seconds. Error: ${error}`)
}
// Report success!
const successMessage = `Deployment succeeded after ${Math.round((Date.now() - startTime) / 1000)} seconds.`
console.log(successMessage)
await octokit.repos.createDeploymentStatus({
owner,
repo,
deployment_id: deploymentId,
state: 'success',
description: successMessage,
...logUrl && { log_url: logUrl },
environment_url: homepageUrl,
// The 'flash' preview is required for `state` values of 'in_progress' and 'queued'
mediaType: {
previews: ['flash']
}
})
console.log(`🚀 Deployment status: success - ${successMessage}`)
console.log(`Visit the newly deployed app at: ${homepageUrl}`)
} catch (error) {
// Report failure!
const failureMessage = `Deployment failed after ${Math.round((Date.now() - startTime) / 1000)} seconds. See logs for more information.`
console.error(failureMessage)
try {
if (deploymentId) {
await octokit.repos.createDeploymentStatus({
owner,
repo,
deployment_id: deploymentId,
state: 'error',
description: failureMessage,
...logUrl && { log_url: logUrl },
environment_url: homepageUrl,
// The 'flash' preview is required for `state` values of 'in_progress' and 'queued'
mediaType: {
previews: ['flash']
}
})
console.log(
`🚀 Deployment status: error - ${failureMessage}` +
(logUrl ? ` Logs: ${logUrl}` : '')
)
}
} catch (error) {
console.error(`Failed to finalize GitHub DeploymentStatus as a failure. Error: ${error}`)
}
// Re-throw the error to bubble up
throw error
}
}

Просмотреть файл

@ -0,0 +1,20 @@
const USERNAME_FORMAT = '([A-Za-z0-9-]+)'
const REPO_NAME_FORMAT = '([A-Za-z0-9._-]+)'
const PR_NUMBER_FORMAT = '(\\d+)'
const ALLOWED_PR_URL_FORMAT = new RegExp(
'^' +
'[\'"]?' +
`https://github\\.com/${USERNAME_FORMAT}/${REPO_NAME_FORMAT}/pull/${PR_NUMBER_FORMAT}` +
'[\'"]?' +
'$'
)
module.exports = function parsePullRequestUrl (prUrl) {
const [/* fullMatch */, owner, repo, pr] = ((prUrl || '').match(ALLOWED_PR_URL_FORMAT) || [])
return {
owner,
repo,
pullNumber: parseInt(pr, 10) || undefined
}
}

Просмотреть файл

@ -68,7 +68,8 @@ describe('check if a GitHub-owned private repository is referenced', () => {
'**/*.gif', // READMEs or other text-based files from being checked.
'**/*.pdf',
'**/*.ico',
'**/*.woff'
'**/*.woff',
'script/deploy'
]
})