Fix github action workflow syntax issues and update docs (#22455)
* Unify concurrency and inputs for triggerable github action workflows * Remove CI splits input from workflow dispatch trigger * Update github action docs
This commit is contained in:
Родитель
0d7ae9d568
Коммит
8d4b227677
|
@ -1,5 +1,9 @@
|
||||||
name: Build Docker image
|
name: Build Docker image
|
||||||
|
|
||||||
|
run-name: |
|
||||||
|
ref: ${{ github.ref_name }} |
|
||||||
|
push: ${{ inputs.push }} |
|
||||||
|
|
||||||
on:
|
on:
|
||||||
workflow_dispatch:
|
workflow_dispatch:
|
||||||
inputs:
|
inputs:
|
||||||
|
|
|
@ -1,9 +1,9 @@
|
||||||
name: Test Docker Image
|
name: Test Docker Image
|
||||||
|
|
||||||
run-name: |
|
run-name: |
|
||||||
Test ${{ github.workflow }} \
|
ref: ${{ github.ref_name }} |
|
||||||
version: ${{ inputs.version }} \
|
version: ${{ inputs.version }} |
|
||||||
digest: ${{ inputs.digest }}
|
digest: ${{ inputs.digest }} |
|
||||||
|
|
||||||
on:
|
on:
|
||||||
workflow_call:
|
workflow_call:
|
||||||
|
@ -13,9 +13,23 @@ on:
|
||||||
type: string
|
type: string
|
||||||
required: true
|
required: true
|
||||||
digest:
|
digest:
|
||||||
description: The build digest of the image to run
|
description: The build digest of the image to run. Overrides version.
|
||||||
|
type: string
|
||||||
|
required: false
|
||||||
|
workflow_dispatch:
|
||||||
|
inputs:
|
||||||
|
version:
|
||||||
|
description: The version of the image to run
|
||||||
type: string
|
type: string
|
||||||
required: true
|
required: true
|
||||||
|
digest:
|
||||||
|
description: The build digest of the image to run. Overrides version.
|
||||||
|
type: string
|
||||||
|
required: false
|
||||||
|
|
||||||
|
concurrency:
|
||||||
|
group: ${{ github.workflow }}-${{ github.event_name}}-${{ github.ref}}-${{ toJSON(inputs) }}
|
||||||
|
cancel-in-progress: true
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
test:
|
test:
|
||||||
|
|
|
@ -1,10 +1,10 @@
|
||||||
name: Test Docker Image (test_main only)
|
name: Test Docker Image (test_main only)
|
||||||
|
|
||||||
run-name: |
|
run-name: |
|
||||||
Test (main) ${{ github.workflow }} \
|
ref: ${{ github.ref_name }} |
|
||||||
version: ${{ inputs.version }} \
|
version: ${{ inputs.version }} |
|
||||||
digest: ${{ inputs.digest }} \
|
digest: ${{ inputs.digest }} |
|
||||||
splits: ${{ inputs.splits }}
|
splits: ${{ inputs.splits }} |
|
||||||
|
|
||||||
on:
|
on:
|
||||||
workflow_call:
|
workflow_call:
|
||||||
|
@ -12,16 +12,35 @@ on:
|
||||||
version:
|
version:
|
||||||
description: The version of the image to run
|
description: The version of the image to run
|
||||||
type: string
|
type: string
|
||||||
required: true
|
required: false
|
||||||
digest:
|
digest:
|
||||||
description: The build digest of the image to run
|
description: The build digest of the image to run. Overrides version.
|
||||||
type: string
|
type: string
|
||||||
required: true
|
required: false
|
||||||
splits:
|
splits:
|
||||||
description: How many splits for test_main
|
description: How many splits for test_main
|
||||||
type: number
|
type: number
|
||||||
required: false
|
required: false
|
||||||
default: 14
|
default: 14
|
||||||
|
workflow_dispatch:
|
||||||
|
inputs:
|
||||||
|
version:
|
||||||
|
description: The version of the image to run
|
||||||
|
type: string
|
||||||
|
required: true
|
||||||
|
digest:
|
||||||
|
description: The build digest of the image to run. Overrides version.
|
||||||
|
type: string
|
||||||
|
required: false
|
||||||
|
splits:
|
||||||
|
description: How many splits for test_main
|
||||||
|
type: number
|
||||||
|
required: false
|
||||||
|
default: 14
|
||||||
|
|
||||||
|
concurrency:
|
||||||
|
group: ${{ github.event_name}}-${{ github.ref}}-${{ toJSON(inputs) }}
|
||||||
|
cancel-in-progress: true
|
||||||
|
|
||||||
env:
|
env:
|
||||||
log_artifact: test_main_logs
|
log_artifact: test_main_logs
|
||||||
|
|
|
@ -10,15 +10,8 @@ on:
|
||||||
# Runs on pull requests to verify changes and push
|
# Runs on pull requests to verify changes and push
|
||||||
# PR image for local testing
|
# PR image for local testing
|
||||||
pull_request:
|
pull_request:
|
||||||
# Manually dispatch to update cache or to push an image
|
# Manually dispatch run entire CI on a ref
|
||||||
# From any ref
|
|
||||||
workflow_dispatch:
|
workflow_dispatch:
|
||||||
inputs:
|
|
||||||
splits:
|
|
||||||
description: 'The number of splits for test_main'
|
|
||||||
required: true
|
|
||||||
type: number
|
|
||||||
default: 14
|
|
||||||
# Runs when a release is published
|
# Runs when a release is published
|
||||||
# Pushes a tagged image
|
# Pushes a tagged image
|
||||||
# That is deployed to the "staging/production" environments
|
# That is deployed to the "staging/production" environments
|
||||||
|
@ -26,10 +19,7 @@ on:
|
||||||
types: [published]
|
types: [published]
|
||||||
|
|
||||||
concurrency:
|
concurrency:
|
||||||
# different events on the same ref can run in parallel
|
group: ${{ github.workflow }}-${{ github.event_name}}-${{ github.ref}}
|
||||||
# different refs on the same event can run in parallel
|
|
||||||
# different splits on the same ref + event can run in parallel
|
|
||||||
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name}}-${{ inputs.splits}}
|
|
||||||
cancel-in-progress: true
|
cancel-in-progress: true
|
||||||
|
|
||||||
env:
|
env:
|
||||||
|
@ -211,6 +201,3 @@ jobs:
|
||||||
with:
|
with:
|
||||||
version: ${{ needs.build.outputs.version }}
|
version: ${{ needs.build.outputs.version }}
|
||||||
digest: ${{ needs.build.outputs.digest }}
|
digest: ${{ needs.build.outputs.digest }}
|
||||||
# If running from a manual workflow_dispatch event, use the provided input
|
|
||||||
# If no input is given or running on pull_request event, use the default value of 14
|
|
||||||
splits: ${{ fromJson(github.event.inputs.splits) || 14 }}
|
|
||||||
|
|
|
@ -0,0 +1,156 @@
|
||||||
|
# Github Actions
|
||||||
|
|
||||||
|
The **addons-server** project uses GitHub Actions to automate testing and building processes in the CI environment.
|
||||||
|
The CI pipeline broken into workflows, jobs and steps that run on containers in github's cloud environment.
|
||||||
|
All workflows are defined in the `.github/workflows` directory.
|
||||||
|
|
||||||
|
## Overview
|
||||||
|
|
||||||
|
Here’s an overview of the existing CI workflows and their architecture:
|
||||||
|
|
||||||
|
Workflows are run on events. Events determine the reason for running the workflow.
|
||||||
|
A workflow can be triggered by multiple events. {ref}`Learn more <configuration>` about configuring events below.
|
||||||
|
|
||||||
|
Generally, we run addons-server inside it's own docker container. This means every workflow generally follows the same pattern.
|
||||||
|
|
||||||
|
### Build the docker image
|
||||||
|
|
||||||
|
Run {ref}`_build.yml <_build_yml>` to build the docker image from the current ref in github.
|
||||||
|
This job can be configured to your needs to build, push, and or upload the image to be used later.
|
||||||
|
The build passing is itself a good first step to test the code, and you can further test after the image is ready.
|
||||||
|
|
||||||
|
### Run the docker image
|
||||||
|
|
||||||
|
Define a job that uses the {ref}`run-docker <actions_run_docker>` reusable action. This action runs our docker compose
|
||||||
|
project in the CI container allowing you to exeucte the same commands you would run locally.
|
||||||
|
This is the foundation of nearly every test we have in CI.
|
||||||
|
|
||||||
|
(configuration)=
|
||||||
|
## Configuration
|
||||||
|
|
||||||
|
- links to docs for github action configuration and event payloads and syntax
|
||||||
|
- reusable actions have a _ prefix
|
||||||
|
- prefer reusable workflows over actions
|
||||||
|
|
||||||
|
### Workflows
|
||||||
|
|
||||||
|
Workflows are the top level configuration object for github actions. Here is an example:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
name: Set a name
|
||||||
|
|
||||||
|
on:
|
||||||
|
pull_request:
|
||||||
|
branches:
|
||||||
|
- master
|
||||||
|
|
||||||
|
concurrency:
|
||||||
|
group: <some group>
|
||||||
|
cancel-in-progress: true
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
job1:
|
||||||
|
...
|
||||||
|
```
|
||||||
|
|
||||||
|
> Important: Always define the most specific event triggers possible
|
||||||
|
> and always set concurrency groups to prevent too many instances of a workflow running simultaneously
|
||||||
|
|
||||||
|
### Reusable Workflows
|
||||||
|
|
||||||
|
We use reusable workflows to enable calling one workflow from another.
|
||||||
|
This allows better encapsulation of a set of logic that might itself require multiple jobs, matrix jobs
|
||||||
|
or direct access to built in context like secrets. Otherwise they are conceptually similar to {ref}`reusable actions <reusable_actions>`.
|
||||||
|
|
||||||
|
(_build_yml)=
|
||||||
|
#### _build.yml
|
||||||
|
|
||||||
|
[link](../../../.github/workflows/_build.yml)
|
||||||
|
|
||||||
|
This workflow can be triggered by any other workflow or directly via the CLI or github UI.
|
||||||
|
It builds a docker image based on the current state of the code, setting the appropriate metadata
|
||||||
|
based on context.
|
||||||
|
|
||||||
|
(_test_yml)=
|
||||||
|
#### _test.yml
|
||||||
|
|
||||||
|
[link](../../../.github/workflows/_test.yml)
|
||||||
|
|
||||||
|
Our main testing workflow runs a suite of tests verifying the docker image and django code within are running as expected.
|
||||||
|
|
||||||
|
(_test_main_yml_)=
|
||||||
|
#### _test_main.yml
|
||||||
|
|
||||||
|
[link](../../../.github/workflows/_test_main.yml)
|
||||||
|
|
||||||
|
This workflow is a branch of our _test.yml workflow, running specifically the main pytest suite.
|
||||||
|
It is split to its own workflow because it runs via a matrix strategy and spins up a lot of jobs.
|
||||||
|
|
||||||
|
(reusable_actions)=
|
||||||
|
### Reusable Actions
|
||||||
|
|
||||||
|
Reusable actions are like reusable workflows but they do not run on their own runner,
|
||||||
|
but directly as a step in a given workflow runner container.
|
||||||
|
|
||||||
|
(actions_context)=
|
||||||
|
#### context
|
||||||
|
|
||||||
|
[link](../../../.github/actions/context/action.yml)
|
||||||
|
|
||||||
|
This action provides additional context based on the `github` context object. Most importantly it helps us determine
|
||||||
|
if we are running on a fork or if various meta events (releast_tag, release_master) match the current context.
|
||||||
|
These contextual values are relevent globally and should return the same values no matter where context is called,
|
||||||
|
so context runs as an action and accepts no inputs.
|
||||||
|
|
||||||
|
(actions_run_docker)=
|
||||||
|
#### run-docker
|
||||||
|
|
||||||
|
[link](../../../.github/actions/run-docker/action.yml)
|
||||||
|
|
||||||
|
The main action to run our docker compose project. This action is configurable to run a specified command, with specified services,
|
||||||
|
and even configurable compose file. Importantly this action will pull an image via the digest or version, and if it cannot find the image
|
||||||
|
will build it locally to run the current state of the codebase.
|
||||||
|
|
||||||
|
## Gotchas
|
||||||
|
|
||||||
|
- workflow_dispatch and workflow_call inputs should be identical
|
||||||
|
|
||||||
|
Best practice should be to define all _reusable workflows with both a `workflow_dispatch` and `workflow_call` event trigger.
|
||||||
|
The inputs for each should be identical. This allows testing and otherwise triggering reusable workflows directly or via
|
||||||
|
another workflow with the same parameters and expectations.
|
||||||
|
|
||||||
|
- github object contents depend on the event triggering the workflow
|
||||||
|
|
||||||
|
One of the reasons we have the {ref}`context action <actions_context>` is because the information embedded in the github
|
||||||
|
object depends on the event that triggered a workflow, making finding a certain piece of information depend on the 'context'.
|
||||||
|
Be careful using the github object directly as you must consider many edge cases. Consult the context action and potentially
|
||||||
|
introduce an important contextual value their so it can be made consistent across different contexts.
|
||||||
|
|
||||||
|
- github converts job/step/workflow outputs to string regardless of the underlying data type
|
||||||
|
|
||||||
|
Even if you define an input with a specific datatype, outputs for steps, jobs and reusable workflows are all converted to strings.
|
||||||
|
This is important when passing values from outputs to inputs as the data type might not actually be what you want or expect.
|
||||||
|
|
||||||
|
Use:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
uses: <action>
|
||||||
|
with:
|
||||||
|
number: ${{ fromJSON(output.value) }}
|
||||||
|
```
|
||||||
|
|
||||||
|
to convert values back into numbers/booleans.
|
||||||
|
|
||||||
|
- secrets are not available for workflows running on forks.
|
||||||
|
|
||||||
|
Github actions prevents forks from accessing secrets, so workflows that use secrets should be configured to either
|
||||||
|
not rely on secrets or have fallback behaviour in place.
|
||||||
|
|
||||||
|
- use context action to define global context
|
||||||
|
|
||||||
|
Most global context should be defined in the {ref}`context <actions_context>` action instead of directly in workflows.
|
||||||
|
|
||||||
|
- prevent invalid workflow configurations
|
||||||
|
|
||||||
|
When reusable workflows are passed invalid outputs, github will silently fail the workflow, to prevent this you should always
|
||||||
|
check the outcome of reusable workflow calls.
|
|
@ -6,6 +6,7 @@ Development
|
||||||
:maxdepth: 2
|
:maxdepth: 2
|
||||||
|
|
||||||
setup_and_configuration
|
setup_and_configuration
|
||||||
|
github_actions
|
||||||
tests
|
tests
|
||||||
debugging
|
debugging
|
||||||
dependencies
|
dependencies
|
||||||
|
|
|
@ -1,13 +1,11 @@
|
||||||
# Setup and Configuration
|
# Setup and Configuration
|
||||||
|
|
||||||
This section covers how to run `addons-server` both locally and in CI environments. This should be where you start
|
This section covers how to run `addons-server` locally. See [github actions](./github_actions.md) for running in CI.
|
||||||
if you are running `addons-server` for the first time.
|
This should be where you start if you are running `addons-server` for the first time.
|
||||||
|
Setting up the local development environment for **addons-server** involves configuring Docker Compose to run the necessary services.
|
||||||
|
Follow these steps to get started:
|
||||||
|
|
||||||
## Local Development Environment
|
## Prerequisites
|
||||||
|
|
||||||
Setting up the local development environment for **addons-server** involves configuring Docker Compose to run the necessary services. Follow these steps to get started:
|
|
||||||
|
|
||||||
### Prerequisites
|
|
||||||
|
|
||||||
- Ensure Docker and Docker Compose are installed on your system.
|
- Ensure Docker and Docker Compose are installed on your system.
|
||||||
- Clone the **addons-server** repository from GitHub:
|
- Clone the **addons-server** repository from GitHub:
|
||||||
|
@ -18,7 +16,7 @@ Setting up the local development environment for **addons-server** involves conf
|
||||||
```
|
```
|
||||||
|
|
||||||
(running-for-the-first-time)=
|
(running-for-the-first-time)=
|
||||||
### Running for the first time
|
## Running for the first time
|
||||||
|
|
||||||
When running the project for the first time, execute:
|
When running the project for the first time, execute:
|
||||||
|
|
||||||
|
@ -38,7 +36,7 @@ and the app might crash or otherwise be unusable.
|
||||||
Similarly, you can run `make initialize` even after you have an up and running environment, but this will totally reset your database
|
Similarly, you can run `make initialize` even after you have an up and running environment, but this will totally reset your database
|
||||||
as if you were running the application fresh.
|
as if you were running the application fresh.
|
||||||
|
|
||||||
### Updating your environment
|
## Updating your environment
|
||||||
|
|
||||||
> TLDR; Just run `make up`.
|
> TLDR; Just run `make up`.
|
||||||
|
|
||||||
|
@ -60,7 +58,7 @@ What happens if you run `make up` when your environment is already running?
|
||||||
This will result in all services and volumes being recreated as if starting them for the first time,
|
This will result in all services and volumes being recreated as if starting them for the first time,
|
||||||
and will clear any local state from the containers. The `make up` command is {ref}`idempotent <idempotence>` so you can run it over and over.
|
and will clear any local state from the containers. The `make up` command is {ref}`idempotent <idempotence>` so you can run it over and over.
|
||||||
|
|
||||||
### Shutting down your environment
|
## Shutting down your environment
|
||||||
|
|
||||||
> TLDR; just run `make down`
|
> TLDR; just run `make down`
|
||||||
|
|
||||||
|
@ -189,64 +187,12 @@ of running commands via `make` or running the same command directly via the dock
|
||||||
|
|
||||||
Though it is **highly recommended to use the make commands** instead of directly calling docker in your terminal.
|
Though it is **highly recommended to use the make commands** instead of directly calling docker in your terminal.
|
||||||
|
|
||||||
## Continuous Integration Environment
|
### Docker Compose Files
|
||||||
|
|
||||||
The **addons-server** project uses GitHub Actions to automate testing and building processes in the CI environment. Here’s an overview of the existing CI workflows and their architecture:
|
- **[docker-compose.yml][docker-compose]**: The primary Docker Compose file defining services, networks, and volumes for local and CI environments.
|
||||||
|
- **[docker-compose.ci.yml][docker-compose-ci]**: Overrides certain configurations for CI-specific needs, ensuring the environment is optimized for automated testing and builds.
|
||||||
1. **Existing Workflows**:
|
- **[docker-compose.deps.yml][docker-compose-deps]**: Attaches a mount at ./deps to /deps in the container, exposing the contents to the host
|
||||||
- The CI pipeline is defined in the `.github/workflows` directory. The main workflow file, typically named `ci.yml`, orchestrates the build and test processes for the project.
|
- **[docker-compose.private.yml][docker-compose-private]**: Runs addons-server with the `customs` service that is only avaiable to Mozilla employees
|
||||||
|
|
||||||
2. **Reusable Actions/Workflows**:
|
|
||||||
The project leverages reusable actions and workflows:
|
|
||||||
- [build](../../../.github/workflows/_build.yml) to build and or push a docker image
|
|
||||||
- [context](../../../.github/actions/context/action.yml) retrieve context about what should happen
|
|
||||||
- [run-docker](../../../.github/actions/run-docker/action.yml) run the docker compose services and execute a command
|
|
||||||
- [test](../../../.github/workflows/_test.yml) run the CI test suite (minus test_main)
|
|
||||||
- [test_main](../../../.github/workflows/_test_main.yml) run the large pytest main group on a number of matrix split jobs
|
|
||||||
|
|
||||||
These actions simplify the workflow definitions and ensure consistency across different jobs.
|
|
||||||
|
|
||||||
(workflow_example)=
|
|
||||||
3. **Workflow Example**:
|
|
||||||
- A typical workflow file includes steps such as checking out the repository, setting up Docker Buildx, building the Docker image, and running the tests:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
name: CI
|
|
||||||
on: [push, pull_request]
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
build:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
uses: ./.github/workflows/_build.yml
|
|
||||||
permissions:
|
|
||||||
contents: 'read'
|
|
||||||
id-token: 'write'
|
|
||||||
secrets: inherit
|
|
||||||
with:
|
|
||||||
push: true
|
|
||||||
|
|
||||||
run:
|
|
||||||
needs: build
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v4
|
|
||||||
- uses: ./.github/actions/run-docker
|
|
||||||
with:
|
|
||||||
digest: ${{ needs.build.outputs.digest }}
|
|
||||||
version: ${{ needs.build.outputs.version }}
|
|
||||||
run: pytest -vv
|
|
||||||
|
|
||||||
```
|
|
||||||
|
|
||||||
It is important to note, reusable actions cannot checkout code, so code is always checked out on the workflow.
|
|
||||||
|
|
||||||
Also important the running `_build.yml` requires setting premissions for `contents` and `id-token`. These are used by our docker login steps and the main workflow controls permissions for called composite workflows. Finally we pass `secrets: inherit` to give the composite workflow access to repository action secrets without needing to pass manually.
|
|
||||||
|
|
||||||
4. **Docker Compose Files**:
|
|
||||||
- **[docker-compose.yml][docker-compose]**: The primary Docker Compose file defining services, networks, and volumes for local and CI environments.
|
|
||||||
- **[docker-compose.ci.yml][docker-compose-ci]**: Overrides certain configurations for CI-specific needs, ensuring the environment is optimized for automated testing and builds.
|
|
||||||
- **[docker-compose.deps.yml][docker-compose-deps]**: Attaches a mount at ./deps to /deps in the container, exposing the contents to the host
|
|
||||||
- **[docker-compose.private.yml][docker-compose-private]**: Runs addons-server with the `customs` service that is only avaiable to Mozilla employees
|
|
||||||
|
|
||||||
Our docker compose files rely on substituted values, all of which are included in our .env file for direct CLI compatibility.
|
Our docker compose files rely on substituted values, all of which are included in our .env file for direct CLI compatibility.
|
||||||
Any referenced `${VARIABLE}` in the docker-compose files will be replaced with the value from the .env file. We have tests
|
Any referenced `${VARIABLE}` in the docker-compose files will be replaced with the value from the .env file. We have tests
|
||||||
|
@ -390,4 +336,4 @@ If the `_build.yml` workflow is run it requires repository secret and permission
|
||||||
Error: buildx bake failed with: ERROR: failed to solve: failed to push mozilla/addons-server:pr-22446-ci: failed to authorize: failed to fetch oauth token: unexpected status from GET request to https://auth.docker.io/token?scope=repository%3Amozilla%2Faddons-server%3Apull%2Cpush&service=registry.docker.io: 401 Unauthorized
|
Error: buildx bake failed with: ERROR: failed to solve: failed to push mozilla/addons-server:pr-22446-ci: failed to authorize: failed to fetch oauth token: unexpected status from GET request to https://auth.docker.io/token?scope=repository%3Amozilla%2Faddons-server%3Apull%2Cpush&service=registry.docker.io: 401 Unauthorized
|
||||||
```
|
```
|
||||||
|
|
||||||
See the {ref}`workflow example <workflow_example>` for correct usage.
|
See the (workflow example)[./github_actions.md] for correct usage.
|
||||||
|
|
Загрузка…
Ссылка в новой задаче