fix #9271 chore(project): dockerize schemas (#9616)

Because

* As we add a wider variety of elements to the Nimbus project including
binary packages it becomes increasingly important to have a consistent
environment for each of our modules
* Using Docker allows us to control the environment each module runs in
and use a consistent base layer
* Dockerizing the schemas package will allow us to use identical Python,
Node, and Rust environments as the other modules in Nimbus

This commit

* Wraps the schemas module in a Dockerfile that is consistent with the
other modules in Nimbus
This commit is contained in:
Jared Lockhart 2023-10-24 16:30:55 -04:00 коммит произвёл GitHub
Родитель 7a6274cdef
Коммит bba7fd789b
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
7 изменённых файлов: 175 добавлений и 60 удалений

Просмотреть файл

@ -62,7 +62,7 @@ commands:
jobs:
check_experimenter_x86_64:
machine:
image: ubuntu-2004:2023.04.2 # Ubuntu 20.04, Docker v20.10.24, Docker Compose v2.17.2
image: ubuntu-2004:2023.10.1
docker_layer_caching: true
resource_class: large
working_directory: ~/experimenter
@ -81,7 +81,7 @@ jobs:
check_experimenter_aarch64:
machine:
image: ubuntu-2004:2023.04.2 # Ubuntu 20.04, Docker v20.10.24, Docker Compose v2.17.2
image: ubuntu-2004:2023.10.1
docker_layer_caching: true
resource_class: arm.large
working_directory: ~/experimenter
@ -102,7 +102,7 @@ jobs:
check_cirrus_x86_64:
machine:
image: ubuntu-2004:2023.04.2 # Ubuntu 20.04, Docker v20.10.24, Docker Compose v2.17.2
image: ubuntu-2004:2023.10.1
docker_layer_caching: true
resource_class: large
working_directory: ~/cirrus
@ -120,7 +120,7 @@ jobs:
check_cirrus_aarch64:
machine:
image: ubuntu-2004:2023.04.2 # Ubuntu 20.04, Docker v20.10.24, Docker Compose v2.17.2
image: ubuntu-2004:2023.10.1
docker_layer_caching: true
resource_class: arm.large
working_directory: ~/cirrus
@ -139,10 +139,14 @@ jobs:
make cirrus_check
check_schemas:
docker:
- image: cimg/python:3.10.8-node # poetry 1.2.2
machine:
image: ubuntu-2004:2023.10.1
docker_layer_caching: true
steps:
- checkout
- setup_docker:
username: $DOCKER_USER
password: $DOCKER_PASS
- check_file_paths:
paths: "schemas/"
- run:
@ -152,7 +156,7 @@ jobs:
integration_nimbus_desktop_release_targeting:
machine:
image: ubuntu-2004:2023.04.2 # Ubuntu 20.04, Docker v20.10.24, Docker Compose v2.17.2
image: ubuntu-2004:2023.10.1
docker_layer_caching: true
resource_class: large
working_directory: ~/experimenter
@ -177,7 +181,7 @@ jobs:
integration_nimbus_desktop_beta_targeting:
machine:
image: ubuntu-2004:2023.04.2 # Ubuntu 20.04, Docker v20.10.24, Docker Compose v2.17.2
image: ubuntu-2004:2023.10.1
docker_layer_caching: true
resource_class: large
working_directory: ~/experimenter
@ -202,7 +206,7 @@ jobs:
integration_nimbus_desktop_nightly_targeting:
machine:
image: ubuntu-2004:2023.04.2 # Ubuntu 20.04, Docker v20.10.24, Docker Compose v2.17.2
image: ubuntu-2004:2023.10.1
docker_layer_caching: true
resource_class: large
working_directory: ~/experimenter
@ -227,7 +231,7 @@ jobs:
integration_nimbus_desktop_remote_settings:
machine:
image: ubuntu-2004:2023.04.2 # Ubuntu 20.04, Docker v20.10.24, Docker Compose v2.17.2
image: ubuntu-2004:2023.10.1
docker_layer_caching: true
resource_class: large
working_directory: ~/experimenter
@ -251,7 +255,7 @@ jobs:
integration_nimbus_fenix_remote_settings:
machine:
image: ubuntu-2004:2023.04.2 # Ubuntu 20.04, Docker v20.10.24, Docker Compose v2.17.2
image: ubuntu-2004:2023.10.1
docker_layer_caching: true
resource_class: medium
working_directory: ~/experimenter
@ -275,7 +279,7 @@ jobs:
integration_nimbus_ios_remote_settings:
machine:
image: ubuntu-2004:2023.04.2 # Ubuntu 20.04, Docker v20.10.24, Docker Compose v2.17.2
image: ubuntu-2004:2023.10.1
docker_layer_caching: true
resource_class: medium
working_directory: ~/experimenter
@ -299,7 +303,7 @@ jobs:
integration_nimbus_focus_android_remote_settings:
machine:
image: ubuntu-2004:2023.04.2 # Ubuntu 20.04, Docker v20.10.24, Docker Compose v2.17.2
image: ubuntu-2004:2023.10.1
docker_layer_caching: true
resource_class: medium
working_directory: ~/experimenter
@ -323,7 +327,7 @@ jobs:
integration_nimbus_focus_ios_remote_settings:
machine:
image: ubuntu-2004:2023.04.2 # Ubuntu 20.04, Docker v20.10.24, Docker Compose v2.17.2
image: ubuntu-2004:2023.10.1
docker_layer_caching: true
resource_class: medium
working_directory: ~/experimenter
@ -347,7 +351,7 @@ jobs:
integration_nimbus_desktop_enrollment:
machine:
image: ubuntu-2004:2023.04.2 # Ubuntu 20.04, Docker v20.10.24, Docker Compose v2.17.2
image: ubuntu-2004:2023.10.1
docker_layer_caching: true
resource_class: xlarge
working_directory: ~/experimenter
@ -373,7 +377,7 @@ jobs:
integration_nimbus_desktop_ui:
machine:
image: ubuntu-2004:2023.04.2 # Ubuntu 20.04, Docker v20.10.24, Docker Compose v2.17.2
image: ubuntu-2004:2023.10.1
docker_layer_caching: true
resource_class: large
working_directory: ~/experimenter
@ -397,7 +401,7 @@ jobs:
integration_nimbus_sdk_targeting:
machine:
image: ubuntu-2004:2023.04.2 # Ubuntu 20.04, Docker v20.10.24, Docker Compose v2.17.2
image: ubuntu-2004:2023.10.1
docker_layer_caching: true
resource_class: medium
working_directory: ~/experimenter
@ -418,7 +422,7 @@ jobs:
integration_legacy:
machine:
image: ubuntu-2004:2023.04.2 # Ubuntu 20.04, Docker v20.10.24, Docker Compose v2.17.2
image: ubuntu-2004:2023.10.1
docker_layer_caching: true
resource_class: large
working_directory: ~/experimenter
@ -442,7 +446,7 @@ jobs:
deploy_experimenter:
working_directory: ~/experimenter
machine:
image: ubuntu-2004:2023.04.2 # Ubuntu 20.04, Docker v20.10.24, Docker Compose v2.17.2
image: ubuntu-2004:2023.10.1
docker_layer_caching: true
steps:
- checkout
@ -477,7 +481,7 @@ jobs:
deploy_cirrus:
working_directory: ~/cirrus
machine:
image: ubuntu-2004:2023.04.2 # Ubuntu 20.04, Docker v20.10.24, Docker Compose v2.17.2
image: ubuntu-2004:2023.10.1
docker_layer_caching: true
steps:
- checkout
@ -495,10 +499,14 @@ jobs:
docker push ${DOCKERHUB_CIRRUS_REPO}:latest
deploy_schemas:
docker:
- image: cimg/python:3.10.8-node # poetry 1.2.2
machine:
image: ubuntu-2004:2023.10.1
docker_layer_caching: true
steps:
- checkout
- setup_docker:
username: $DOCKER_USER
password: $DOCKER_PASS
- run:
name: Check for package version change in last commit before proceeding.
command: |
@ -510,9 +518,9 @@ jobs:
circleci-agent step halt
fi
- run:
name: Create the PyPI distribution files
name: Create the distribution files
command: |
make schemas_build_pypi
make schemas_dist
- run:
name: Upload to PyPI
command: |
@ -520,7 +528,7 @@ jobs:
# https://app.circleci.com/settings/project/github/mozilla/experimenter/environment-variables
# For more on twine, see:
# https://twine.readthedocs.io/en/latest/
make schemas_deploy_pypi
make schemas_deploy_pypi SCHEMAS_ENV="-e TWINE_USERNAME=${TWINE_USERNAME} -e TWINE_PASSWORD=${TWINE_PASSWORD}"
- run:
name: NPM Authentication
command: echo "//registry.npmjs.org/:_authToken=${NPM_TOKEN}" > .npmrc
@ -529,12 +537,12 @@ jobs:
command: |
# Relies on the NPM_TOKEN environment variable configured at:
# https://app.circleci.com/settings/project/github/mozilla/experimenter/environment-variables
make schemas_deploy_npm
make schemas_deploy_npm SCHEMAS_ENV="-e NPM_TOKEN=${NPM_TOKEN}"
update_external_configs:
working_directory: ~/experimenter
machine:
image: ubuntu-2004:2023.04.2 # Ubuntu 20.04, Docker v20.10.24, Docker Compose v2.17.2
image: ubuntu-2004:2023.10.1
docker_layer_caching: true
steps:
- add_ssh_keys:
@ -575,7 +583,7 @@ jobs:
build_firefox_versions:
working_directory: ~/experimenter
machine:
image: ubuntu-2004:2023.04.2 # Ubuntu 20.04, Docker v20.10.24, Docker Compose v2.17.2
image: ubuntu-2004:2023.10.1
docker_layer_caching: true
steps:
- checkout
@ -635,7 +643,7 @@ jobs:
build_rust_image:
working_directory: ~/experimenter
machine:
image: ubuntu-2004:2023.04.2 # Ubuntu 20.04, Docker v20.10.24, Docker Compose v2.17.2
image: ubuntu-2004:2023.10.1
docker_layer_caching: true
steps:
- checkout
@ -801,6 +809,7 @@ workflows:
- Check Cirrus x86_64
- Check Cirrus aarch64
- deploy_schemas:
name: Deploy Schemas
filters:
branches:
only: main

Просмотреть файл

@ -246,43 +246,54 @@ build_demo_app:
# nimbus schemas package
SCHEMAS_VERSION_FILE := schemas/VERSION
SCHEMAS_VERSION := $(shell cat ${SCHEMAS_VERSION_FILE})
SCHEMAS_ENV ?= # This is empty by default
SCHEMAS_VERSION = \$$(cat VERSION)
SCHEMAS_RUN = docker run -ti $(SCHEMAS_ENV) -v ./schemas:/schemas -v /schemas/node_modules schemas:dev sh -c
SCHEMAS_BLACK = black --check --diff .
SCHEMAS_RUFF = ruff .
SCHEMAS_DIFF_PYDANTIC = \
pydantic2ts --module mozilla_nimbus_schemas.__init__ --output /tmp/test_index.d.ts --json2ts-cmd 'yarn json2ts' &&\
diff /tmp/test_index.d.ts index.d.ts || (echo nimbus-schemas typescript package is out of sync please run make schemas_build;exit 1) &&\
echo 'Done. No problems found in schemas.'
SCHEMAS_TEST = pytest
SCHEMAS_FORMAT = ruff --fix . && black .
SCHEMAS_DIST_PYPI = poetry build
SCHEMAS_DIST_NPM = pydantic2ts --module mozilla_nimbus_schemas.__init__ --output ./index.d.ts --json2ts-cmd 'yarn json2ts'
SCHEMAS_DEPLOY_PYPI = twine upload --skip-existing dist/*;
SCHEMAS_DEPLOY_NPM = echo "//registry.npmjs.org/:_authToken=${NPM_TOKEN}" > .npmrc;yarn publish --new-version ${SCHEMAS_VERSION} --access public;
SCHEMAS_VERSION_PYPI = poetry version ${SCHEMAS_VERSION};
SCHEMAS_VERSION_NPM = npm version --allow-same-version ${SCHEMAS_VERSION};
schemas_install:
(cd schemas && poetry install && yarn install)
schemas_build:
$(DOCKER_BUILD) --target dev -f schemas/Dockerfile -t schemas:dev schemas/
schemas_black:
(cd schemas && poetry run black --check --diff .)
schemas_bash: schemas_build
$(SCHEMAS_RUN) "bash"
schemas_ruff:
(cd schemas && poetry run ruff .)
schemas_format: schemas_build
$(SCHEMAS_RUN) "$(SCHEMAS_FORMAT)"
schemas_test:
(cd schemas && poetry run pytest)
schemas_check: schemas_build
$(SCHEMAS_RUN) "$(SCHEMAS_BLACK)&&$(SCHEMAS_RUFF)&&$(SCHEMAS_DIFF_PYDANTIC)&&$(SCHEMAS_TEST)"
schemas_check: schemas_install schemas_black schemas_ruff schemas_test
(cd schemas && poetry run pydantic2ts --module mozilla_nimbus_schemas.__init__ --output /tmp/test_index.d.ts --json2ts-cmd "yarn json2ts")
diff /tmp/test_index.d.ts schemas/index.d.ts || (echo nimbus-schemas typescript package is out of sync please run make schemas_build;exit 1)
echo "Done. No problems found in schemas."
schemas_dist_pypi: schemas_build
$(SCHEMAS_RUN) "$(SCHEMAS_DIST_PYPI)"
schemas_code_format:
(cd schemas && poetry run black . && poetry run ruff --fix .)
schemas_dist_npm: schemas_build
$(SCHEMAS_RUN) "$(SCHEMAS_DIST_NPM)"
schemas_build: schemas_install schemas_build_pypi schemas_build_npm
schemas_dist: schemas_build schemas_dist_pypi schemas_dist_npm
schemas_build_pypi:
(cd schemas && poetry build)
schemas_deploy_pypi: schemas_build
$(SCHEMAS_RUN) "$(SCHEMAS_DEPLOY_PYPI)"
schemas_deploy_pypi: schemas_install schemas_build_pypi
cd schemas; poetry run twine upload --skip-existing dist/*;
schemas_deploy_npm: schemas_build
$(SCHEMAS_RUN) "$(SCHEMAS_DEPLOY_NPM)"
schemas_build_npm: schemas_install
(cd schemas && poetry run pydantic2ts --module mozilla_nimbus_schemas.__init__ --output ./index.d.ts --json2ts-cmd "yarn json2ts")
schemas_version_pypi: schemas_build
$(SCHEMAS_RUN) "$(SCHEMAS_VERSION_PYPI)"
schemas_deploy_npm: schemas_build_npm
cd schemas; yarn publish --new-version ${SCHEMAS_VERSION} --access public;
schemas_version_npm: schemas_build
$(SCHEMAS_RUN) "$(SCHEMAS_VERSION_NPM)"
schemas_version:
npm --prefix schemas version --allow-same-version ${SCHEMAS_VERSION};
poetry --directory=schemas version ${SCHEMAS_VERSION};
schemas_version: schemas_version_pypi schemas_version_npm

17
schemas/.dockerignore Normal file
Просмотреть файл

@ -0,0 +1,17 @@
**/__pycache__
**/.cache
**/.coverage
**/.DS_Store
**/.mypy
**/.pycache
**/.pytest_cache
**/.vscode
**/.vscode/
**/coverage/
**/node_modules
**/yarn-error.log
docs
experimenter/legacy/legacy-ui/assets
experimenter/nimbus-ui/build
experimenter/reporting/reporting-ui/assets/
experimenter/served/

78
schemas/Dockerfile Normal file
Просмотреть файл

@ -0,0 +1,78 @@
#-------------------------
FROM alpine:3.12.0 as file-loader
# To preserve layer caching across machines which may have different local file properties
# such as permissions, timestamps, etc, all files are copied into a container and their
# permissions and timestamps are reset to consistent values
# Credit: https://gist.github.com/kekru/8ac61cd87536a4355220b56ae2f4b0a9
COPY . /schemas/
RUN chmod -R 555 /schemas \
&& chown -R root:root /schemas \
&& find /schemas -exec touch -a -m -t 201512180130.09 {} \;
#-------------------------
# System packages
FROM python:3.11.1 AS system-builder
RUN curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add -
RUN echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list
RUN apt-get update
RUN apt-get --no-install-recommends install -y apt-utils ca-certificates yarn parallel
# Install nvm with node and npm
ENV NODE_VERSION=16.19.0
ENV NVM_DIR=/root/.nvm
ENV PATH="/root/.nvm/versions/node/v${NODE_VERSION}/bin/:${PATH}"
RUN curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.35.3/install.sh | bash
RUN . "$NVM_DIR/nvm.sh" && nvm install ${NODE_VERSION}
RUN . "$NVM_DIR/nvm.sh" && nvm use v${NODE_VERSION}
RUN . "$NVM_DIR/nvm.sh" && nvm alias default v${NODE_VERSION}
# Disable python pyc files
ENV PYTHONDONTWRITEBYTECODE 1
# Python packages
RUN curl -sSL https://install.python-poetry.org | python3 -
ENV PATH "/root/.local/bin:$PATH"
RUN poetry config virtualenvs.create false
# Python image
#-------------------------
FROM system-builder AS python-builder
WORKDIR /schemas
COPY --from=file-loader /schemas/pyproject.toml /schemas/pyproject.toml
COPY --from=file-loader /schemas/poetry.lock /schemas/poetry.lock
COPY --from=file-loader /schemas/mozilla_nimbus_schemas /schemas/mozilla_nimbus_schemas
COPY --from=file-loader /schemas/README.md /schemas/README.md
RUN poetry install
# If any package is installed, that is incompatible by version, this command
# will exit non-zero and print what is usually just a warning in `poetry install`
RUN poetry check
# Node image
#-------------------------
FROM system-builder AS node-builder
WORKDIR /schemas
# Node packages for legacy and nimbus ui
COPY --from=file-loader /schemas/package.json /schemas/package.json
COPY --from=file-loader /schemas/yarn.lock /schemas/yarn.lock
RUN yarn install --frozen-lockfile
# Dev image
#-------------------------
FROM system-builder AS dev
WORKDIR /schemas
# Python packages
COPY --from=python-builder /usr/local/bin/ /usr/local/bin/
COPY --from=python-builder /usr/local/lib/python3.11/site-packages/ /usr/local/lib/python3.11/site-packages/
# Node packages
COPY --from=node-builder /schemas/node_modules/ /schemas/node_modules/

Просмотреть файл

@ -1 +1 @@
2023.10.2
2023.10.3

Просмотреть файл

@ -1,6 +1,6 @@
{
"name": "@mozilla/nimbus-schemas",
"version": "2023.10.2",
"version": "2023.10.3",
"description": "Schemas used by Mozilla Nimbus and related projects.",
"main": "index.d.ts",
"repository": {

Просмотреть файл

@ -1,6 +1,6 @@
[tool.poetry]
name = "mozilla-nimbus-schemas"
version = "2023.10.2"
version = "2023.10.3"
description = "Schemas used by Mozilla Nimbus and related projects."
authors = ["mikewilli"]
license = "MPL 2.0"