Because * As we add a wider variety of elements to the Nimbus project including binary packages it becomes increasingly important to have a consistent environment for each of our modules * Using Docker allows us to control the environment each module runs in and use a consistent base layer * Dockerizing the schemas package will allow us to use identical Python, Node, and Rust environments as the other modules in Nimbus This commit * Wraps the schemas module in a Dockerfile that is consistent with the other modules in Nimbus
This commit is contained in:
Родитель
7a6274cdef
Коммит
bba7fd789b
|
@ -62,7 +62,7 @@ commands:
|
|||
jobs:
|
||||
check_experimenter_x86_64:
|
||||
machine:
|
||||
image: ubuntu-2004:2023.04.2 # Ubuntu 20.04, Docker v20.10.24, Docker Compose v2.17.2
|
||||
image: ubuntu-2004:2023.10.1
|
||||
docker_layer_caching: true
|
||||
resource_class: large
|
||||
working_directory: ~/experimenter
|
||||
|
@ -81,7 +81,7 @@ jobs:
|
|||
|
||||
check_experimenter_aarch64:
|
||||
machine:
|
||||
image: ubuntu-2004:2023.04.2 # Ubuntu 20.04, Docker v20.10.24, Docker Compose v2.17.2
|
||||
image: ubuntu-2004:2023.10.1
|
||||
docker_layer_caching: true
|
||||
resource_class: arm.large
|
||||
working_directory: ~/experimenter
|
||||
|
@ -102,7 +102,7 @@ jobs:
|
|||
|
||||
check_cirrus_x86_64:
|
||||
machine:
|
||||
image: ubuntu-2004:2023.04.2 # Ubuntu 20.04, Docker v20.10.24, Docker Compose v2.17.2
|
||||
image: ubuntu-2004:2023.10.1
|
||||
docker_layer_caching: true
|
||||
resource_class: large
|
||||
working_directory: ~/cirrus
|
||||
|
@ -120,7 +120,7 @@ jobs:
|
|||
|
||||
check_cirrus_aarch64:
|
||||
machine:
|
||||
image: ubuntu-2004:2023.04.2 # Ubuntu 20.04, Docker v20.10.24, Docker Compose v2.17.2
|
||||
image: ubuntu-2004:2023.10.1
|
||||
docker_layer_caching: true
|
||||
resource_class: arm.large
|
||||
working_directory: ~/cirrus
|
||||
|
@ -139,10 +139,14 @@ jobs:
|
|||
make cirrus_check
|
||||
|
||||
check_schemas:
|
||||
docker:
|
||||
- image: cimg/python:3.10.8-node # poetry 1.2.2
|
||||
machine:
|
||||
image: ubuntu-2004:2023.10.1
|
||||
docker_layer_caching: true
|
||||
steps:
|
||||
- checkout
|
||||
- setup_docker:
|
||||
username: $DOCKER_USER
|
||||
password: $DOCKER_PASS
|
||||
- check_file_paths:
|
||||
paths: "schemas/"
|
||||
- run:
|
||||
|
@ -152,7 +156,7 @@ jobs:
|
|||
|
||||
integration_nimbus_desktop_release_targeting:
|
||||
machine:
|
||||
image: ubuntu-2004:2023.04.2 # Ubuntu 20.04, Docker v20.10.24, Docker Compose v2.17.2
|
||||
image: ubuntu-2004:2023.10.1
|
||||
docker_layer_caching: true
|
||||
resource_class: large
|
||||
working_directory: ~/experimenter
|
||||
|
@ -177,7 +181,7 @@ jobs:
|
|||
|
||||
integration_nimbus_desktop_beta_targeting:
|
||||
machine:
|
||||
image: ubuntu-2004:2023.04.2 # Ubuntu 20.04, Docker v20.10.24, Docker Compose v2.17.2
|
||||
image: ubuntu-2004:2023.10.1
|
||||
docker_layer_caching: true
|
||||
resource_class: large
|
||||
working_directory: ~/experimenter
|
||||
|
@ -202,7 +206,7 @@ jobs:
|
|||
|
||||
integration_nimbus_desktop_nightly_targeting:
|
||||
machine:
|
||||
image: ubuntu-2004:2023.04.2 # Ubuntu 20.04, Docker v20.10.24, Docker Compose v2.17.2
|
||||
image: ubuntu-2004:2023.10.1
|
||||
docker_layer_caching: true
|
||||
resource_class: large
|
||||
working_directory: ~/experimenter
|
||||
|
@ -227,7 +231,7 @@ jobs:
|
|||
|
||||
integration_nimbus_desktop_remote_settings:
|
||||
machine:
|
||||
image: ubuntu-2004:2023.04.2 # Ubuntu 20.04, Docker v20.10.24, Docker Compose v2.17.2
|
||||
image: ubuntu-2004:2023.10.1
|
||||
docker_layer_caching: true
|
||||
resource_class: large
|
||||
working_directory: ~/experimenter
|
||||
|
@ -251,7 +255,7 @@ jobs:
|
|||
|
||||
integration_nimbus_fenix_remote_settings:
|
||||
machine:
|
||||
image: ubuntu-2004:2023.04.2 # Ubuntu 20.04, Docker v20.10.24, Docker Compose v2.17.2
|
||||
image: ubuntu-2004:2023.10.1
|
||||
docker_layer_caching: true
|
||||
resource_class: medium
|
||||
working_directory: ~/experimenter
|
||||
|
@ -275,7 +279,7 @@ jobs:
|
|||
|
||||
integration_nimbus_ios_remote_settings:
|
||||
machine:
|
||||
image: ubuntu-2004:2023.04.2 # Ubuntu 20.04, Docker v20.10.24, Docker Compose v2.17.2
|
||||
image: ubuntu-2004:2023.10.1
|
||||
docker_layer_caching: true
|
||||
resource_class: medium
|
||||
working_directory: ~/experimenter
|
||||
|
@ -299,7 +303,7 @@ jobs:
|
|||
|
||||
integration_nimbus_focus_android_remote_settings:
|
||||
machine:
|
||||
image: ubuntu-2004:2023.04.2 # Ubuntu 20.04, Docker v20.10.24, Docker Compose v2.17.2
|
||||
image: ubuntu-2004:2023.10.1
|
||||
docker_layer_caching: true
|
||||
resource_class: medium
|
||||
working_directory: ~/experimenter
|
||||
|
@ -323,7 +327,7 @@ jobs:
|
|||
|
||||
integration_nimbus_focus_ios_remote_settings:
|
||||
machine:
|
||||
image: ubuntu-2004:2023.04.2 # Ubuntu 20.04, Docker v20.10.24, Docker Compose v2.17.2
|
||||
image: ubuntu-2004:2023.10.1
|
||||
docker_layer_caching: true
|
||||
resource_class: medium
|
||||
working_directory: ~/experimenter
|
||||
|
@ -347,7 +351,7 @@ jobs:
|
|||
|
||||
integration_nimbus_desktop_enrollment:
|
||||
machine:
|
||||
image: ubuntu-2004:2023.04.2 # Ubuntu 20.04, Docker v20.10.24, Docker Compose v2.17.2
|
||||
image: ubuntu-2004:2023.10.1
|
||||
docker_layer_caching: true
|
||||
resource_class: xlarge
|
||||
working_directory: ~/experimenter
|
||||
|
@ -373,7 +377,7 @@ jobs:
|
|||
|
||||
integration_nimbus_desktop_ui:
|
||||
machine:
|
||||
image: ubuntu-2004:2023.04.2 # Ubuntu 20.04, Docker v20.10.24, Docker Compose v2.17.2
|
||||
image: ubuntu-2004:2023.10.1
|
||||
docker_layer_caching: true
|
||||
resource_class: large
|
||||
working_directory: ~/experimenter
|
||||
|
@ -397,7 +401,7 @@ jobs:
|
|||
|
||||
integration_nimbus_sdk_targeting:
|
||||
machine:
|
||||
image: ubuntu-2004:2023.04.2 # Ubuntu 20.04, Docker v20.10.24, Docker Compose v2.17.2
|
||||
image: ubuntu-2004:2023.10.1
|
||||
docker_layer_caching: true
|
||||
resource_class: medium
|
||||
working_directory: ~/experimenter
|
||||
|
@ -418,7 +422,7 @@ jobs:
|
|||
|
||||
integration_legacy:
|
||||
machine:
|
||||
image: ubuntu-2004:2023.04.2 # Ubuntu 20.04, Docker v20.10.24, Docker Compose v2.17.2
|
||||
image: ubuntu-2004:2023.10.1
|
||||
docker_layer_caching: true
|
||||
resource_class: large
|
||||
working_directory: ~/experimenter
|
||||
|
@ -442,7 +446,7 @@ jobs:
|
|||
deploy_experimenter:
|
||||
working_directory: ~/experimenter
|
||||
machine:
|
||||
image: ubuntu-2004:2023.04.2 # Ubuntu 20.04, Docker v20.10.24, Docker Compose v2.17.2
|
||||
image: ubuntu-2004:2023.10.1
|
||||
docker_layer_caching: true
|
||||
steps:
|
||||
- checkout
|
||||
|
@ -477,7 +481,7 @@ jobs:
|
|||
deploy_cirrus:
|
||||
working_directory: ~/cirrus
|
||||
machine:
|
||||
image: ubuntu-2004:2023.04.2 # Ubuntu 20.04, Docker v20.10.24, Docker Compose v2.17.2
|
||||
image: ubuntu-2004:2023.10.1
|
||||
docker_layer_caching: true
|
||||
steps:
|
||||
- checkout
|
||||
|
@ -495,10 +499,14 @@ jobs:
|
|||
docker push ${DOCKERHUB_CIRRUS_REPO}:latest
|
||||
|
||||
deploy_schemas:
|
||||
docker:
|
||||
- image: cimg/python:3.10.8-node # poetry 1.2.2
|
||||
machine:
|
||||
image: ubuntu-2004:2023.10.1
|
||||
docker_layer_caching: true
|
||||
steps:
|
||||
- checkout
|
||||
- setup_docker:
|
||||
username: $DOCKER_USER
|
||||
password: $DOCKER_PASS
|
||||
- run:
|
||||
name: Check for package version change in last commit before proceeding.
|
||||
command: |
|
||||
|
@ -510,9 +518,9 @@ jobs:
|
|||
circleci-agent step halt
|
||||
fi
|
||||
- run:
|
||||
name: Create the PyPI distribution files
|
||||
name: Create the distribution files
|
||||
command: |
|
||||
make schemas_build_pypi
|
||||
make schemas_dist
|
||||
- run:
|
||||
name: Upload to PyPI
|
||||
command: |
|
||||
|
@ -520,7 +528,7 @@ jobs:
|
|||
# https://app.circleci.com/settings/project/github/mozilla/experimenter/environment-variables
|
||||
# For more on twine, see:
|
||||
# https://twine.readthedocs.io/en/latest/
|
||||
make schemas_deploy_pypi
|
||||
make schemas_deploy_pypi SCHEMAS_ENV="-e TWINE_USERNAME=${TWINE_USERNAME} -e TWINE_PASSWORD=${TWINE_PASSWORD}"
|
||||
- run:
|
||||
name: NPM Authentication
|
||||
command: echo "//registry.npmjs.org/:_authToken=${NPM_TOKEN}" > .npmrc
|
||||
|
@ -529,12 +537,12 @@ jobs:
|
|||
command: |
|
||||
# Relies on the NPM_TOKEN environment variable configured at:
|
||||
# https://app.circleci.com/settings/project/github/mozilla/experimenter/environment-variables
|
||||
make schemas_deploy_npm
|
||||
make schemas_deploy_npm SCHEMAS_ENV="-e NPM_TOKEN=${NPM_TOKEN}"
|
||||
|
||||
update_external_configs:
|
||||
working_directory: ~/experimenter
|
||||
machine:
|
||||
image: ubuntu-2004:2023.04.2 # Ubuntu 20.04, Docker v20.10.24, Docker Compose v2.17.2
|
||||
image: ubuntu-2004:2023.10.1
|
||||
docker_layer_caching: true
|
||||
steps:
|
||||
- add_ssh_keys:
|
||||
|
@ -575,7 +583,7 @@ jobs:
|
|||
build_firefox_versions:
|
||||
working_directory: ~/experimenter
|
||||
machine:
|
||||
image: ubuntu-2004:2023.04.2 # Ubuntu 20.04, Docker v20.10.24, Docker Compose v2.17.2
|
||||
image: ubuntu-2004:2023.10.1
|
||||
docker_layer_caching: true
|
||||
steps:
|
||||
- checkout
|
||||
|
@ -635,7 +643,7 @@ jobs:
|
|||
build_rust_image:
|
||||
working_directory: ~/experimenter
|
||||
machine:
|
||||
image: ubuntu-2004:2023.04.2 # Ubuntu 20.04, Docker v20.10.24, Docker Compose v2.17.2
|
||||
image: ubuntu-2004:2023.10.1
|
||||
docker_layer_caching: true
|
||||
steps:
|
||||
- checkout
|
||||
|
@ -801,6 +809,7 @@ workflows:
|
|||
- Check Cirrus x86_64
|
||||
- Check Cirrus aarch64
|
||||
- deploy_schemas:
|
||||
name: Deploy Schemas
|
||||
filters:
|
||||
branches:
|
||||
only: main
|
||||
|
|
67
Makefile
67
Makefile
|
@ -246,43 +246,54 @@ build_demo_app:
|
|||
|
||||
|
||||
# nimbus schemas package
|
||||
SCHEMAS_VERSION_FILE := schemas/VERSION
|
||||
SCHEMAS_VERSION := $(shell cat ${SCHEMAS_VERSION_FILE})
|
||||
SCHEMAS_ENV ?= # This is empty by default
|
||||
SCHEMAS_VERSION = \$$(cat VERSION)
|
||||
SCHEMAS_RUN = docker run -ti $(SCHEMAS_ENV) -v ./schemas:/schemas -v /schemas/node_modules schemas:dev sh -c
|
||||
SCHEMAS_BLACK = black --check --diff .
|
||||
SCHEMAS_RUFF = ruff .
|
||||
SCHEMAS_DIFF_PYDANTIC = \
|
||||
pydantic2ts --module mozilla_nimbus_schemas.__init__ --output /tmp/test_index.d.ts --json2ts-cmd 'yarn json2ts' &&\
|
||||
diff /tmp/test_index.d.ts index.d.ts || (echo nimbus-schemas typescript package is out of sync please run make schemas_build;exit 1) &&\
|
||||
echo 'Done. No problems found in schemas.'
|
||||
SCHEMAS_TEST = pytest
|
||||
SCHEMAS_FORMAT = ruff --fix . && black .
|
||||
SCHEMAS_DIST_PYPI = poetry build
|
||||
SCHEMAS_DIST_NPM = pydantic2ts --module mozilla_nimbus_schemas.__init__ --output ./index.d.ts --json2ts-cmd 'yarn json2ts'
|
||||
SCHEMAS_DEPLOY_PYPI = twine upload --skip-existing dist/*;
|
||||
SCHEMAS_DEPLOY_NPM = echo "//registry.npmjs.org/:_authToken=${NPM_TOKEN}" > .npmrc;yarn publish --new-version ${SCHEMAS_VERSION} --access public;
|
||||
SCHEMAS_VERSION_PYPI = poetry version ${SCHEMAS_VERSION};
|
||||
SCHEMAS_VERSION_NPM = npm version --allow-same-version ${SCHEMAS_VERSION};
|
||||
|
||||
schemas_install:
|
||||
(cd schemas && poetry install && yarn install)
|
||||
schemas_build:
|
||||
$(DOCKER_BUILD) --target dev -f schemas/Dockerfile -t schemas:dev schemas/
|
||||
|
||||
schemas_black:
|
||||
(cd schemas && poetry run black --check --diff .)
|
||||
schemas_bash: schemas_build
|
||||
$(SCHEMAS_RUN) "bash"
|
||||
|
||||
schemas_ruff:
|
||||
(cd schemas && poetry run ruff .)
|
||||
schemas_format: schemas_build
|
||||
$(SCHEMAS_RUN) "$(SCHEMAS_FORMAT)"
|
||||
|
||||
schemas_test:
|
||||
(cd schemas && poetry run pytest)
|
||||
schemas_check: schemas_build
|
||||
$(SCHEMAS_RUN) "$(SCHEMAS_BLACK)&&$(SCHEMAS_RUFF)&&$(SCHEMAS_DIFF_PYDANTIC)&&$(SCHEMAS_TEST)"
|
||||
|
||||
schemas_check: schemas_install schemas_black schemas_ruff schemas_test
|
||||
(cd schemas && poetry run pydantic2ts --module mozilla_nimbus_schemas.__init__ --output /tmp/test_index.d.ts --json2ts-cmd "yarn json2ts")
|
||||
diff /tmp/test_index.d.ts schemas/index.d.ts || (echo nimbus-schemas typescript package is out of sync please run make schemas_build;exit 1)
|
||||
echo "Done. No problems found in schemas."
|
||||
schemas_dist_pypi: schemas_build
|
||||
$(SCHEMAS_RUN) "$(SCHEMAS_DIST_PYPI)"
|
||||
|
||||
schemas_code_format:
|
||||
(cd schemas && poetry run black . && poetry run ruff --fix .)
|
||||
schemas_dist_npm: schemas_build
|
||||
$(SCHEMAS_RUN) "$(SCHEMAS_DIST_NPM)"
|
||||
|
||||
schemas_build: schemas_install schemas_build_pypi schemas_build_npm
|
||||
schemas_dist: schemas_build schemas_dist_pypi schemas_dist_npm
|
||||
|
||||
schemas_build_pypi:
|
||||
(cd schemas && poetry build)
|
||||
schemas_deploy_pypi: schemas_build
|
||||
$(SCHEMAS_RUN) "$(SCHEMAS_DEPLOY_PYPI)"
|
||||
|
||||
schemas_deploy_pypi: schemas_install schemas_build_pypi
|
||||
cd schemas; poetry run twine upload --skip-existing dist/*;
|
||||
schemas_deploy_npm: schemas_build
|
||||
$(SCHEMAS_RUN) "$(SCHEMAS_DEPLOY_NPM)"
|
||||
|
||||
schemas_build_npm: schemas_install
|
||||
(cd schemas && poetry run pydantic2ts --module mozilla_nimbus_schemas.__init__ --output ./index.d.ts --json2ts-cmd "yarn json2ts")
|
||||
schemas_version_pypi: schemas_build
|
||||
$(SCHEMAS_RUN) "$(SCHEMAS_VERSION_PYPI)"
|
||||
|
||||
schemas_deploy_npm: schemas_build_npm
|
||||
cd schemas; yarn publish --new-version ${SCHEMAS_VERSION} --access public;
|
||||
schemas_version_npm: schemas_build
|
||||
$(SCHEMAS_RUN) "$(SCHEMAS_VERSION_NPM)"
|
||||
|
||||
schemas_version:
|
||||
npm --prefix schemas version --allow-same-version ${SCHEMAS_VERSION};
|
||||
poetry --directory=schemas version ${SCHEMAS_VERSION};
|
||||
schemas_version: schemas_version_pypi schemas_version_npm
|
||||
|
|
|
@ -0,0 +1,17 @@
|
|||
**/__pycache__
|
||||
**/.cache
|
||||
**/.coverage
|
||||
**/.DS_Store
|
||||
**/.mypy
|
||||
**/.pycache
|
||||
**/.pytest_cache
|
||||
**/.vscode
|
||||
**/.vscode/
|
||||
**/coverage/
|
||||
**/node_modules
|
||||
**/yarn-error.log
|
||||
docs
|
||||
experimenter/legacy/legacy-ui/assets
|
||||
experimenter/nimbus-ui/build
|
||||
experimenter/reporting/reporting-ui/assets/
|
||||
experimenter/served/
|
|
@ -0,0 +1,78 @@
|
|||
#-------------------------
|
||||
FROM alpine:3.12.0 as file-loader
|
||||
|
||||
# To preserve layer caching across machines which may have different local file properties
|
||||
# such as permissions, timestamps, etc, all files are copied into a container and their
|
||||
# permissions and timestamps are reset to consistent values
|
||||
# Credit: https://gist.github.com/kekru/8ac61cd87536a4355220b56ae2f4b0a9
|
||||
COPY . /schemas/
|
||||
RUN chmod -R 555 /schemas \
|
||||
&& chown -R root:root /schemas \
|
||||
&& find /schemas -exec touch -a -m -t 201512180130.09 {} \;
|
||||
|
||||
|
||||
#-------------------------
|
||||
# System packages
|
||||
FROM python:3.11.1 AS system-builder
|
||||
|
||||
RUN curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add -
|
||||
RUN echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list
|
||||
RUN apt-get update
|
||||
RUN apt-get --no-install-recommends install -y apt-utils ca-certificates yarn parallel
|
||||
|
||||
# Install nvm with node and npm
|
||||
ENV NODE_VERSION=16.19.0
|
||||
ENV NVM_DIR=/root/.nvm
|
||||
ENV PATH="/root/.nvm/versions/node/v${NODE_VERSION}/bin/:${PATH}"
|
||||
RUN curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.35.3/install.sh | bash
|
||||
RUN . "$NVM_DIR/nvm.sh" && nvm install ${NODE_VERSION}
|
||||
RUN . "$NVM_DIR/nvm.sh" && nvm use v${NODE_VERSION}
|
||||
RUN . "$NVM_DIR/nvm.sh" && nvm alias default v${NODE_VERSION}
|
||||
|
||||
# Disable python pyc files
|
||||
ENV PYTHONDONTWRITEBYTECODE 1
|
||||
|
||||
# Python packages
|
||||
RUN curl -sSL https://install.python-poetry.org | python3 -
|
||||
ENV PATH "/root/.local/bin:$PATH"
|
||||
RUN poetry config virtualenvs.create false
|
||||
|
||||
|
||||
# Python image
|
||||
#-------------------------
|
||||
FROM system-builder AS python-builder
|
||||
WORKDIR /schemas
|
||||
|
||||
COPY --from=file-loader /schemas/pyproject.toml /schemas/pyproject.toml
|
||||
COPY --from=file-loader /schemas/poetry.lock /schemas/poetry.lock
|
||||
COPY --from=file-loader /schemas/mozilla_nimbus_schemas /schemas/mozilla_nimbus_schemas
|
||||
COPY --from=file-loader /schemas/README.md /schemas/README.md
|
||||
RUN poetry install
|
||||
|
||||
# If any package is installed, that is incompatible by version, this command
|
||||
# will exit non-zero and print what is usually just a warning in `poetry install`
|
||||
RUN poetry check
|
||||
|
||||
|
||||
# Node image
|
||||
#-------------------------
|
||||
FROM system-builder AS node-builder
|
||||
WORKDIR /schemas
|
||||
|
||||
# Node packages for legacy and nimbus ui
|
||||
COPY --from=file-loader /schemas/package.json /schemas/package.json
|
||||
COPY --from=file-loader /schemas/yarn.lock /schemas/yarn.lock
|
||||
RUN yarn install --frozen-lockfile
|
||||
|
||||
|
||||
# Dev image
|
||||
#-------------------------
|
||||
FROM system-builder AS dev
|
||||
WORKDIR /schemas
|
||||
|
||||
# Python packages
|
||||
COPY --from=python-builder /usr/local/bin/ /usr/local/bin/
|
||||
COPY --from=python-builder /usr/local/lib/python3.11/site-packages/ /usr/local/lib/python3.11/site-packages/
|
||||
|
||||
# Node packages
|
||||
COPY --from=node-builder /schemas/node_modules/ /schemas/node_modules/
|
|
@ -1 +1 @@
|
|||
2023.10.2
|
||||
2023.10.3
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "@mozilla/nimbus-schemas",
|
||||
"version": "2023.10.2",
|
||||
"version": "2023.10.3",
|
||||
"description": "Schemas used by Mozilla Nimbus and related projects.",
|
||||
"main": "index.d.ts",
|
||||
"repository": {
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
[tool.poetry]
|
||||
name = "mozilla-nimbus-schemas"
|
||||
version = "2023.10.2"
|
||||
version = "2023.10.3"
|
||||
description = "Schemas used by Mozilla Nimbus and related projects."
|
||||
authors = ["mikewilli"]
|
||||
license = "MPL 2.0"
|
||||
|
|
Загрузка…
Ссылка в новой задаче