Because * It would save time if we push the docker build layers from circleci to dockerhub to reuse locally This commit * Uses the new buildkit --cache-from flag to pull in build layers from dockerhub when applicable * Builds all stages and pushes them with their layers to dockerhub on every merge to main * Configures local builds to pull those layers from dockerhub * Removes the build circle stage since each circle task will now be able to pull from dockerhub rather than the local circle docker cache
This commit is contained in:
Jared Lockhart
GitHub
Родитель
71d8081230
Коммит
4c0df6b262
|
|
@ -1,28 +1,9 @@
|
|||
version: 2.1
|
||||
jobs:
|
||||
build:
|
||||
machine:
|
||||
docker_layer_caching: true
|
||||
image: ubuntu-2004:202101-01 # Ubuntu 20.04, Docker v20.10.2, Docker Compose v1.28.2
|
||||
resource_class: large
|
||||
working_directory: ~/experimenter
|
||||
steps:
|
||||
- run:
|
||||
name: Docker info
|
||||
command: docker -v
|
||||
- run:
|
||||
name: Docker compose info
|
||||
command: docker-compose -v
|
||||
- checkout
|
||||
- run:
|
||||
name: Build docker images
|
||||
command: |
|
||||
make build_prod
|
||||
|
||||
check:
|
||||
machine:
|
||||
docker_layer_caching: true
|
||||
image: ubuntu-2004:202101-01 # Ubuntu 20.04, Docker v20.10.2, Docker Compose v1.28.2
|
||||
image: ubuntu-2004:202104-01 # Ubuntu 20.04, Docker v20.10.6, Docker Compose v1.29.1
|
||||
resource_class: large
|
||||
working_directory: ~/experimenter
|
||||
steps:
|
||||
|
|
@ -42,7 +23,7 @@ jobs:
|
|||
publish_storybooks:
|
||||
machine:
|
||||
docker_layer_caching: true
|
||||
image: ubuntu-2004:202101-01 # Ubuntu 20.04, Docker v20.10.2, Docker Compose v1.28.2
|
||||
image: ubuntu-2004:202104-01 # Ubuntu 20.04, Docker v20.10.6, Docker Compose v1.29.1
|
||||
resource_class: medium
|
||||
working_directory: ~/experimenter
|
||||
steps:
|
||||
|
|
@ -62,7 +43,7 @@ jobs:
|
|||
integration_legacy:
|
||||
machine:
|
||||
docker_layer_caching: true
|
||||
image: ubuntu-2004:202101-01 # Ubuntu 20.04, Docker v20.10.2, Docker Compose v1.28.2
|
||||
image: ubuntu-2004:202104-01 # Ubuntu 20.04, Docker v20.10.6, Docker Compose v1.29.1
|
||||
resource_class: xlarge
|
||||
working_directory: ~/experimenter
|
||||
steps:
|
||||
|
|
@ -84,7 +65,7 @@ jobs:
|
|||
integration_nimbus:
|
||||
machine:
|
||||
docker_layer_caching: true
|
||||
image: ubuntu-2004:202101-01 # Ubuntu 20.04, Docker v20.10.2, Docker Compose v1.28.2
|
||||
image: ubuntu-2004:202104-01 # Ubuntu 20.04, Docker v20.10.6, Docker Compose v1.29.1
|
||||
resource_class: xlarge
|
||||
working_directory: ~/experimenter
|
||||
steps:
|
||||
|
|
@ -107,50 +88,43 @@ jobs:
|
|||
working_directory: ~/experimenter
|
||||
machine:
|
||||
docker_layer_caching: true
|
||||
image: ubuntu-2004:202101-01 # Ubuntu 20.04, Docker v20.10.2, Docker Compose v1.28.2
|
||||
image: ubuntu-2004:202104-01 # Ubuntu 20.04, Docker v20.10.6, Docker Compose v1.29.1
|
||||
steps:
|
||||
- checkout
|
||||
- deploy:
|
||||
name: Deploy to latest
|
||||
command: |
|
||||
./scripts/store_git_info.sh
|
||||
make build_prod
|
||||
docker login -u $DOCKER_USER -p $DOCKER_PASS
|
||||
make build_dev
|
||||
make build_test
|
||||
make build_prod
|
||||
docker tag app:dev ${DOCKERHUB_REPO}:build_dev
|
||||
docker tag app:test ${DOCKERHUB_REPO}:build_test
|
||||
docker tag app:deploy ${DOCKERHUB_REPO}:latest
|
||||
docker push ${DOCKERHUB_REPO}:build_dev
|
||||
docker push ${DOCKERHUB_REPO}:build_test
|
||||
docker push ${DOCKERHUB_REPO}:latest
|
||||
|
||||
workflows:
|
||||
version: 2
|
||||
build:
|
||||
jobs:
|
||||
- build:
|
||||
name: build
|
||||
- check:
|
||||
name: check
|
||||
requires:
|
||||
- build
|
||||
- publish_storybooks:
|
||||
name: publish_storybooks
|
||||
requires:
|
||||
- build
|
||||
- integration_legacy:
|
||||
name: integration_legacy
|
||||
requires:
|
||||
- build
|
||||
filters:
|
||||
branches:
|
||||
ignore:
|
||||
- main
|
||||
|
||||
- integration_nimbus:
|
||||
name: integration_nimbus
|
||||
requires:
|
||||
- build
|
||||
filters:
|
||||
branches:
|
||||
ignore:
|
||||
- main
|
||||
|
||||
- deploy:
|
||||
filters:
|
||||
branches:
|
||||
|
|
|
|||
|
|
@ -103,8 +103,6 @@ google-credentials.json
|
|||
app/experimenter/outcomes/jetstream-config*
|
||||
|
||||
# Versioning assets generated on build
|
||||
app/commit-description.txt
|
||||
app/commit-summary.txt
|
||||
app/experimenter/version.json
|
||||
app/version.json
|
||||
|
||||
**/commit-description.txt
|
||||
**/commit-summary.txt
|
||||
**/version.json
|
||||
|
|
|
|||
62
Makefile
62
Makefile
|
|
@ -35,7 +35,7 @@ GENERATE_DOCS = python manage.py generate_docs
|
|||
LOAD_COUNTRIES = python manage.py loaddata ./experimenter/base/fixtures/countries.json
|
||||
LOAD_LOCALES = python manage.py loaddata ./experimenter/base/fixtures/locales.json
|
||||
LOAD_DUMMY_EXPERIMENTS = [[ -z $$SKIP_DUMMY ]] && python manage.py load_dummy_experiments || echo "skipping dummy experiments"
|
||||
PUBLISH_STORYBOOKS = npx github:mozilla-fxa/storybook-gcp-publisher --commit-summary commit-summary.txt --commit-description commit-description.txt --version-json version.json
|
||||
PUBLISH_STORYBOOKS = npx github:mozilla-fxa/storybook-gcp-publisher --commit-summary commit-summary.txt --commit-description commit-description.txt --version-json experimenter/version.json
|
||||
|
||||
ssl: nginx/key.pem nginx/cert.pem
|
||||
|
||||
|
|
@ -54,20 +54,15 @@ jetstream_config:
|
|||
curl -LJ -o app/experimenter/outcomes/jetstream-config.zip https://github.com/mozilla/jetstream-config/archive/main.zip
|
||||
unzip -o -d app/experimenter/outcomes app/experimenter/outcomes/jetstream-config.zip
|
||||
|
||||
build_dev: jetstream_config
|
||||
docker build --target dev -f app/Dockerfile -t app:dev app/
|
||||
build_dev: jetstream_config ssl
|
||||
DOCKER_BUILDKIT=1 docker build --target dev -f app/Dockerfile -t app:dev --build-arg BUILDKIT_INLINE_CACHE=1 --cache-from mozilla/experimenter:build_dev $$([[ -z "$${CIRCLECI}" ]] || echo "--progress=plain") app/
|
||||
|
||||
build_test: jetstream_config
|
||||
docker build --target test -f app/Dockerfile -t app:test app/
|
||||
build_test: jetstream_config ssl
|
||||
DOCKER_BUILDKIT=1 docker build --target test -f app/Dockerfile -t app:test --build-arg BUILDKIT_INLINE_CACHE=1 --cache-from mozilla/experimenter:build_test $$([[ -z "$${CIRCLECI}" ]] || echo "--progress=plain") app/
|
||||
|
||||
build_prod: jetstream_config
|
||||
docker build --target deploy -f app/Dockerfile -t app:deploy app/
|
||||
|
||||
compose_build_test: kill build_test
|
||||
$(COMPOSE_TEST) build
|
||||
|
||||
compose_build: build_dev ssl
|
||||
$(COMPOSE) build
|
||||
build_prod: jetstream_config ssl
|
||||
./scripts/store_git_info.sh
|
||||
DOCKER_BUILDKIT=1 docker build --target deploy -f app/Dockerfile -t app:deploy --build-arg BUILDKIT_INLINE_CACHE=1 --cache-from mozilla/experimenter:latest $$([[ -z "$${CIRCLECI}" ]] || echo "--progress=plain") app/
|
||||
|
||||
compose_stop:
|
||||
$(COMPOSE) kill || true
|
||||
|
|
@ -92,31 +87,31 @@ static_rm:
|
|||
kill: compose_stop compose_rm volumes_rm
|
||||
echo "All containers removed!"
|
||||
|
||||
check: compose_build_test
|
||||
check: build_test
|
||||
$(COMPOSE_TEST) run app sh -c '$(WAIT_FOR_DB) (${PARALLEL} "$(NIMBUS_SCHEMA_CHECK)" "$(PYTHON_CHECK_MIGRATIONS)" "$(CHECK_DOCS)" "${PY_IMPORT_CHECK}" "$(BLACK_CHECK)" "$(FLAKE8)" "$(ESLINT_CORE)" "$(ESLINT_NIMBUS_UI)" "$(TYPECHECK_NIMBUS_UI)" "$(JS_TEST_CORE)" "$(JS_TEST_NIMBUS_UI)" "$(PYTHON_TEST)") ${COLOR_CHECK}'
|
||||
|
||||
pytest: compose_build_test
|
||||
pytest: build_test
|
||||
$(COMPOSE_TEST) run app sh -c '$(WAIT_FOR_DB) $(PYTHON_TEST)'
|
||||
|
||||
up: compose_build
|
||||
up: build_dev
|
||||
$(COMPOSE) up
|
||||
|
||||
up_prod: compose_build build_prod
|
||||
up_prod: build_prod
|
||||
$(COMPOSE_PROD) up
|
||||
|
||||
up_prod_detached: compose_build build_prod
|
||||
up_prod_detached: build_prod
|
||||
$(COMPOSE_PROD) up -d
|
||||
|
||||
up_db: compose_build
|
||||
up_db: build_dev
|
||||
$(COMPOSE) up db redis kinto autograph
|
||||
|
||||
up_django: compose_build
|
||||
up_django: build_dev
|
||||
$(COMPOSE) up nginx app worker beat db redis kinto autograph
|
||||
|
||||
up_detached: compose_build
|
||||
up_detached: build_dev
|
||||
$(COMPOSE) up -d
|
||||
|
||||
generate_docs: compose_build
|
||||
generate_docs: build_dev
|
||||
$(COMPOSE) run app sh -c "$(GENERATE_DOCS)"
|
||||
|
||||
generate_types: build_dev
|
||||
|
|
@ -125,19 +120,19 @@ generate_types: build_dev
|
|||
publish_storybooks: build_test
|
||||
$(COMPOSE_TEST) run app sh -c "$(PUBLISH_STORYBOOKS)"
|
||||
|
||||
code_format: compose_build
|
||||
code_format: build_dev
|
||||
$(COMPOSE) run app sh -c '${PARALLEL} "${PY_IMPORT_SORT};$(BLACK_FIX)" "$(ESLINT_FIX_CORE)" "$(ESLINT_FIX_NIMBUS_UI)"'
|
||||
|
||||
makemigrations: compose_build
|
||||
makemigrations: build_dev
|
||||
$(COMPOSE) run app python manage.py makemigrations
|
||||
|
||||
migrate: compose_build
|
||||
migrate: build_dev
|
||||
$(COMPOSE) run app sh -c "$(WAIT_FOR_DB) $(PYTHON_MIGRATE)"
|
||||
|
||||
bash: compose_build
|
||||
bash: build_dev
|
||||
$(COMPOSE) run app bash
|
||||
|
||||
refresh: kill compose_build
|
||||
refresh: kill build_dev
|
||||
$(COMPOSE) run -e SKIP_DUMMY=$$SKIP_DUMMY app bash -c '$(WAIT_FOR_DB) $(PYTHON_MIGRATE)&&$(LOAD_LOCALES)&&$(LOAD_COUNTRIES)&&$(LOAD_DUMMY_EXPERIMENTS)'
|
||||
|
||||
dependabot_approve:
|
||||
|
|
@ -145,20 +140,17 @@ dependabot_approve:
|
|||
gh pr list --author app/dependabot | awk '{print $$1}' | xargs -n1 gh pr review -a -b "@dependabot squash and merge"
|
||||
|
||||
# integration tests
|
||||
integration_build: build_prod ssl
|
||||
$(COMPOSE_INTEGRATION) build
|
||||
|
||||
integration_shell: integration_build
|
||||
integration_shell:
|
||||
$(COMPOSE_INTEGRATION) run firefox bash
|
||||
|
||||
integration_vnc_up: integration_build
|
||||
integration_vnc_up:
|
||||
$(COMPOSE_INTEGRATION) up
|
||||
|
||||
integration_vnc_up_detached: integration_build
|
||||
integration_vnc_up_detached:
|
||||
$(COMPOSE_INTEGRATION) up -d firefox
|
||||
|
||||
integration_test_legacy: integration_build
|
||||
integration_test_legacy:
|
||||
MOZ_HEADLESS=1 $(COMPOSE_INTEGRATION) run firefox sh -c "sudo chmod a+rwx /code/app/tests/integration/.tox;tox -c app/tests/integration -e integration-test-legacy $(TOX_ARGS) -- -n 4 $(PYTEST_ARGS)"
|
||||
|
||||
integration_test_nimbus: integration_build
|
||||
integration_test_nimbus:
|
||||
MOZ_HEADLESS=1 $(COMPOSE_INTEGRATION) run firefox sh -c "sudo chmod a+rwx /code/app/tests/integration/.tox;tox -c app/tests/integration -e integration-test-nimbus $(TOX_ARGS) -- $(PYTEST_ARGS)"
|
||||
|
|
|
|||
|
|
@ -1,13 +1,13 @@
|
|||
.mypy
|
||||
.pytest_cache
|
||||
.vscode
|
||||
**/__pycache__
|
||||
**/.cache
|
||||
**/.pycache
|
||||
**/__pycache__
|
||||
**/node_modules
|
||||
**/yarn-error.log
|
||||
**/.mypy
|
||||
**/.pytest_cache
|
||||
**/.vscode
|
||||
docs
|
||||
experimenter/legacy-ui/assets
|
||||
experimenter/legacy-ui/core/.cache/
|
||||
experimenter/nimbus-ui/build
|
||||
node_modules
|
||||
experimenter/legacy-ui/core/.cache/
|
||||
experimenter/nimbus-ui/yarn-error.log
|
||||
|
|
|
|||
|
|
@ -1,3 +1,15 @@
|
|||
FROM alpine:3.12.0 as file-loader
|
||||
|
||||
# To preserve layer caching across machines which may have different local file properties
|
||||
# such as permissions, timestamps, etc, all files are copied into a container and their
|
||||
# permissions and timestamps are reset to consistent values
|
||||
# Credit: https://gist.github.com/kekru/8ac61cd87536a4355220b56ae2f4b0a9
|
||||
COPY . /app/
|
||||
RUN chmod -R 555 /app \
|
||||
&& chown -R root:root /app \
|
||||
&& find /app -exec touch -a -m -t 201512180130.09 {} \;
|
||||
|
||||
|
||||
# Dev image
|
||||
FROM python:3.9 AS dev
|
||||
|
||||
|
|
@ -9,7 +21,7 @@ ENV PYTHONDONTWRITEBYTECODE 1
|
|||
|
||||
|
||||
# Scripts for waiting for the db and setting up kinto
|
||||
COPY bin/ /app/bin/
|
||||
COPY --from=file-loader /app/bin/ /app/bin/
|
||||
RUN chmod +x /app/bin/wait-for-it.sh
|
||||
|
||||
|
||||
|
|
@ -34,7 +46,8 @@ RUN apt-get --no-install-recommends install -y apt-utils ca-certificates postgre
|
|||
RUN curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python
|
||||
ENV PATH "/root/.poetry/bin:${PATH}"
|
||||
RUN poetry config virtualenvs.create false
|
||||
COPY poetry.lock pyproject.toml ./
|
||||
COPY --from=file-loader /app/pyproject.toml /app/pyproject.toml
|
||||
COPY --from=file-loader /app/poetry.lock /app/poetry.lock
|
||||
RUN poetry install
|
||||
|
||||
# If any package is installed, that is incompatible by version, this command
|
||||
|
|
@ -43,19 +56,19 @@ RUN poetry check
|
|||
|
||||
|
||||
# Node packages
|
||||
COPY ./package.json /app/package.json
|
||||
COPY ./yarn.lock /app/yarn.lock
|
||||
COPY ./experimenter/legacy-ui/core/package.json /app/experimenter/legacy-ui/core/package.json
|
||||
COPY --from=file-loader /app/package.json /app/package.json
|
||||
COPY --from=file-loader /app/yarn.lock /app/yarn.lock
|
||||
COPY --from=file-loader /app/experimenter/legacy-ui/core/package.json /app/experimenter/legacy-ui/core/package.json
|
||||
RUN yarn install --frozen-lockfile
|
||||
|
||||
COPY ./experimenter/nimbus-ui/package.json /app/experimenter/nimbus-ui/package.json
|
||||
COPY --from=file-loader /app/experimenter/nimbus-ui/package.json /app/experimenter/nimbus-ui/package.json
|
||||
RUN yarn install --frozen-lockfile
|
||||
|
||||
|
||||
FROM dev AS test
|
||||
|
||||
# Copy source
|
||||
COPY . /app
|
||||
COPY --from=file-loader /app/ /app/
|
||||
|
||||
|
||||
# Build image
|
||||
|
|
@ -63,9 +76,9 @@ FROM dev AS build
|
|||
|
||||
|
||||
# Build assets
|
||||
COPY ./experimenter/legacy-ui/ /app/experimenter/legacy-ui/
|
||||
COPY ./experimenter/nimbus-ui/ /app/experimenter/nimbus-ui/
|
||||
COPY --from=file-loader /app/experimenter/legacy-ui/ /app/experimenter/legacy-ui/
|
||||
RUN yarn workspace @experimenter/core build
|
||||
COPY --from=file-loader /app/experimenter/nimbus-ui/ /app/experimenter/nimbus-ui/
|
||||
RUN yarn workspace @experimenter/nimbus-ui build
|
||||
|
||||
|
||||
|
|
@ -77,16 +90,22 @@ EXPOSE 7001
|
|||
|
||||
# Disable python pyc files
|
||||
ENV PYTHONDONTWRITEBYTECODE 1
|
||||
|
||||
|
||||
# Add poetry to path
|
||||
ENV PATH "/root/.poetry/bin:${PATH}"
|
||||
|
||||
|
||||
# System packages
|
||||
RUN apt-get update
|
||||
RUN apt-get --no-install-recommends install -y apt-utils ca-certificates postgresql-client
|
||||
|
||||
|
||||
# Copy source from previously built containers
|
||||
COPY --from=dev /usr/local/bin/ /usr/local/bin/
|
||||
COPY --from=dev /usr/local/lib/python3.9/site-packages/ /usr/local/lib/python3.9/site-packages/
|
||||
COPY --from=dev /app/bin/ /app/bin/
|
||||
COPY ./manage.py /app/manage.py
|
||||
COPY ./experimenter/ /app/experimenter/
|
||||
COPY --from=file-loader /app/manage.py /app/manage.py
|
||||
COPY --from=file-loader /app/experimenter/ /app/experimenter/
|
||||
COPY --from=build /app/experimenter/legacy-ui/assets/ /app/experimenter/legacy-ui/assets/
|
||||
COPY --from=build /app/experimenter/nimbus-ui/build/ /app/experimenter/nimbus-ui/build/
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
./scripts/echo_version_json.sh > ./app/experimenter/version.json
|
||||
cp ./app/experimenter/version.json ./app/version.json
|
||||
git log -n 1 --no-color --pretty='%s' > ./app/commit-summary.txt
|
||||
git log -n 1 --no-color --pretty=medium > ./app/commit-description.txt
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче