diff --git a/.circleci/config.yml b/.circleci/config.yml index c84708b6f..430d18cd5 100755 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -62,7 +62,7 @@ commands: jobs: check_experimenter_x86_64: machine: - image: ubuntu-2004:2023.04.2 # Ubuntu 20.04, Docker v20.10.24, Docker Compose v2.17.2 + image: ubuntu-2004:2023.10.1 docker_layer_caching: true resource_class: large working_directory: ~/experimenter @@ -81,7 +81,7 @@ jobs: check_experimenter_aarch64: machine: - image: ubuntu-2004:2023.04.2 # Ubuntu 20.04, Docker v20.10.24, Docker Compose v2.17.2 + image: ubuntu-2004:2023.10.1 docker_layer_caching: true resource_class: arm.large working_directory: ~/experimenter @@ -102,7 +102,7 @@ jobs: check_cirrus_x86_64: machine: - image: ubuntu-2004:2023.04.2 # Ubuntu 20.04, Docker v20.10.24, Docker Compose v2.17.2 + image: ubuntu-2004:2023.10.1 docker_layer_caching: true resource_class: large working_directory: ~/cirrus @@ -120,7 +120,7 @@ jobs: check_cirrus_aarch64: machine: - image: ubuntu-2004:2023.04.2 # Ubuntu 20.04, Docker v20.10.24, Docker Compose v2.17.2 + image: ubuntu-2004:2023.10.1 docker_layer_caching: true resource_class: arm.large working_directory: ~/cirrus @@ -139,10 +139,14 @@ jobs: make cirrus_check check_schemas: - docker: - - image: cimg/python:3.10.8-node # poetry 1.2.2 + machine: + image: ubuntu-2004:2023.10.1 + docker_layer_caching: true steps: - checkout + - setup_docker: + username: $DOCKER_USER + password: $DOCKER_PASS - check_file_paths: paths: "schemas/" - run: @@ -152,7 +156,7 @@ jobs: integration_nimbus_desktop_release_targeting: machine: - image: ubuntu-2004:2023.04.2 # Ubuntu 20.04, Docker v20.10.24, Docker Compose v2.17.2 + image: ubuntu-2004:2023.10.1 docker_layer_caching: true resource_class: large working_directory: ~/experimenter @@ -177,7 +181,7 @@ jobs: integration_nimbus_desktop_beta_targeting: machine: - image: ubuntu-2004:2023.04.2 # Ubuntu 20.04, Docker v20.10.24, Docker Compose v2.17.2 + image: ubuntu-2004:2023.10.1 docker_layer_caching: true resource_class: large working_directory: ~/experimenter @@ -202,7 +206,7 @@ jobs: integration_nimbus_desktop_nightly_targeting: machine: - image: ubuntu-2004:2023.04.2 # Ubuntu 20.04, Docker v20.10.24, Docker Compose v2.17.2 + image: ubuntu-2004:2023.10.1 docker_layer_caching: true resource_class: large working_directory: ~/experimenter @@ -227,7 +231,7 @@ jobs: integration_nimbus_desktop_remote_settings: machine: - image: ubuntu-2004:2023.04.2 # Ubuntu 20.04, Docker v20.10.24, Docker Compose v2.17.2 + image: ubuntu-2004:2023.10.1 docker_layer_caching: true resource_class: large working_directory: ~/experimenter @@ -251,7 +255,7 @@ jobs: integration_nimbus_fenix_remote_settings: machine: - image: ubuntu-2004:2023.04.2 # Ubuntu 20.04, Docker v20.10.24, Docker Compose v2.17.2 + image: ubuntu-2004:2023.10.1 docker_layer_caching: true resource_class: medium working_directory: ~/experimenter @@ -275,7 +279,7 @@ jobs: integration_nimbus_ios_remote_settings: machine: - image: ubuntu-2004:2023.04.2 # Ubuntu 20.04, Docker v20.10.24, Docker Compose v2.17.2 + image: ubuntu-2004:2023.10.1 docker_layer_caching: true resource_class: medium working_directory: ~/experimenter @@ -299,7 +303,7 @@ jobs: integration_nimbus_focus_android_remote_settings: machine: - image: ubuntu-2004:2023.04.2 # Ubuntu 20.04, Docker v20.10.24, Docker Compose v2.17.2 + image: ubuntu-2004:2023.10.1 docker_layer_caching: true resource_class: medium working_directory: ~/experimenter @@ -323,7 +327,7 @@ jobs: integration_nimbus_focus_ios_remote_settings: machine: - image: ubuntu-2004:2023.04.2 # Ubuntu 20.04, Docker v20.10.24, Docker Compose v2.17.2 + image: ubuntu-2004:2023.10.1 docker_layer_caching: true resource_class: medium working_directory: ~/experimenter @@ -347,7 +351,7 @@ jobs: integration_nimbus_desktop_enrollment: machine: - image: ubuntu-2004:2023.04.2 # Ubuntu 20.04, Docker v20.10.24, Docker Compose v2.17.2 + image: ubuntu-2004:2023.10.1 docker_layer_caching: true resource_class: xlarge working_directory: ~/experimenter @@ -373,7 +377,7 @@ jobs: integration_nimbus_desktop_ui: machine: - image: ubuntu-2004:2023.04.2 # Ubuntu 20.04, Docker v20.10.24, Docker Compose v2.17.2 + image: ubuntu-2004:2023.10.1 docker_layer_caching: true resource_class: large working_directory: ~/experimenter @@ -397,7 +401,7 @@ jobs: integration_nimbus_sdk_targeting: machine: - image: ubuntu-2004:2023.04.2 # Ubuntu 20.04, Docker v20.10.24, Docker Compose v2.17.2 + image: ubuntu-2004:2023.10.1 docker_layer_caching: true resource_class: medium working_directory: ~/experimenter @@ -418,7 +422,7 @@ jobs: integration_legacy: machine: - image: ubuntu-2004:2023.04.2 # Ubuntu 20.04, Docker v20.10.24, Docker Compose v2.17.2 + image: ubuntu-2004:2023.10.1 docker_layer_caching: true resource_class: large working_directory: ~/experimenter @@ -442,7 +446,7 @@ jobs: deploy_experimenter: working_directory: ~/experimenter machine: - image: ubuntu-2004:2023.04.2 # Ubuntu 20.04, Docker v20.10.24, Docker Compose v2.17.2 + image: ubuntu-2004:2023.10.1 docker_layer_caching: true steps: - checkout @@ -477,7 +481,7 @@ jobs: deploy_cirrus: working_directory: ~/cirrus machine: - image: ubuntu-2004:2023.04.2 # Ubuntu 20.04, Docker v20.10.24, Docker Compose v2.17.2 + image: ubuntu-2004:2023.10.1 docker_layer_caching: true steps: - checkout @@ -495,10 +499,14 @@ jobs: docker push ${DOCKERHUB_CIRRUS_REPO}:latest deploy_schemas: - docker: - - image: cimg/python:3.10.8-node # poetry 1.2.2 + machine: + image: ubuntu-2004:2023.10.1 + docker_layer_caching: true steps: - checkout + - setup_docker: + username: $DOCKER_USER + password: $DOCKER_PASS - run: name: Check for package version change in last commit before proceeding. command: | @@ -510,9 +518,9 @@ jobs: circleci-agent step halt fi - run: - name: Create the PyPI distribution files + name: Create the distribution files command: | - make schemas_build_pypi + make schemas_dist - run: name: Upload to PyPI command: | @@ -520,7 +528,7 @@ jobs: # https://app.circleci.com/settings/project/github/mozilla/experimenter/environment-variables # For more on twine, see: # https://twine.readthedocs.io/en/latest/ - make schemas_deploy_pypi + make schemas_deploy_pypi SCHEMAS_ENV="-e TWINE_USERNAME=${TWINE_USERNAME} -e TWINE_PASSWORD=${TWINE_PASSWORD}" - run: name: NPM Authentication command: echo "//registry.npmjs.org/:_authToken=${NPM_TOKEN}" > .npmrc @@ -529,12 +537,12 @@ jobs: command: | # Relies on the NPM_TOKEN environment variable configured at: # https://app.circleci.com/settings/project/github/mozilla/experimenter/environment-variables - make schemas_deploy_npm + make schemas_deploy_npm SCHEMAS_ENV="-e NPM_TOKEN=${NPM_TOKEN}" update_external_configs: working_directory: ~/experimenter machine: - image: ubuntu-2004:2023.04.2 # Ubuntu 20.04, Docker v20.10.24, Docker Compose v2.17.2 + image: ubuntu-2004:2023.10.1 docker_layer_caching: true steps: - add_ssh_keys: @@ -575,7 +583,7 @@ jobs: build_firefox_versions: working_directory: ~/experimenter machine: - image: ubuntu-2004:2023.04.2 # Ubuntu 20.04, Docker v20.10.24, Docker Compose v2.17.2 + image: ubuntu-2004:2023.10.1 docker_layer_caching: true steps: - checkout @@ -635,7 +643,7 @@ jobs: build_rust_image: working_directory: ~/experimenter machine: - image: ubuntu-2004:2023.04.2 # Ubuntu 20.04, Docker v20.10.24, Docker Compose v2.17.2 + image: ubuntu-2004:2023.10.1 docker_layer_caching: true steps: - checkout @@ -801,6 +809,7 @@ workflows: - Check Cirrus x86_64 - Check Cirrus aarch64 - deploy_schemas: + name: Deploy Schemas filters: branches: only: main diff --git a/Makefile b/Makefile index 28b4f281e..200819563 100644 --- a/Makefile +++ b/Makefile @@ -246,43 +246,54 @@ build_demo_app: # nimbus schemas package -SCHEMAS_VERSION_FILE := schemas/VERSION -SCHEMAS_VERSION := $(shell cat ${SCHEMAS_VERSION_FILE}) +SCHEMAS_ENV ?= # This is empty by default +SCHEMAS_VERSION = \$$(cat VERSION) +SCHEMAS_RUN = docker run -ti $(SCHEMAS_ENV) -v ./schemas:/schemas -v /schemas/node_modules schemas:dev sh -c +SCHEMAS_BLACK = black --check --diff . +SCHEMAS_RUFF = ruff . +SCHEMAS_DIFF_PYDANTIC = \ + pydantic2ts --module mozilla_nimbus_schemas.__init__ --output /tmp/test_index.d.ts --json2ts-cmd 'yarn json2ts' &&\ + diff /tmp/test_index.d.ts index.d.ts || (echo nimbus-schemas typescript package is out of sync please run make schemas_build;exit 1) &&\ + echo 'Done. No problems found in schemas.' +SCHEMAS_TEST = pytest +SCHEMAS_FORMAT = ruff --fix . && black . +SCHEMAS_DIST_PYPI = poetry build +SCHEMAS_DIST_NPM = pydantic2ts --module mozilla_nimbus_schemas.__init__ --output ./index.d.ts --json2ts-cmd 'yarn json2ts' +SCHEMAS_DEPLOY_PYPI = twine upload --skip-existing dist/*; +SCHEMAS_DEPLOY_NPM = echo "//registry.npmjs.org/:_authToken=${NPM_TOKEN}" > .npmrc;yarn publish --new-version ${SCHEMAS_VERSION} --access public; +SCHEMAS_VERSION_PYPI = poetry version ${SCHEMAS_VERSION}; +SCHEMAS_VERSION_NPM = npm version --allow-same-version ${SCHEMAS_VERSION}; -schemas_install: - (cd schemas && poetry install && yarn install) +schemas_build: + $(DOCKER_BUILD) --target dev -f schemas/Dockerfile -t schemas:dev schemas/ -schemas_black: - (cd schemas && poetry run black --check --diff .) +schemas_bash: schemas_build + $(SCHEMAS_RUN) "bash" -schemas_ruff: - (cd schemas && poetry run ruff .) +schemas_format: schemas_build + $(SCHEMAS_RUN) "$(SCHEMAS_FORMAT)" -schemas_test: - (cd schemas && poetry run pytest) +schemas_check: schemas_build + $(SCHEMAS_RUN) "$(SCHEMAS_BLACK)&&$(SCHEMAS_RUFF)&&$(SCHEMAS_DIFF_PYDANTIC)&&$(SCHEMAS_TEST)" -schemas_check: schemas_install schemas_black schemas_ruff schemas_test - (cd schemas && poetry run pydantic2ts --module mozilla_nimbus_schemas.__init__ --output /tmp/test_index.d.ts --json2ts-cmd "yarn json2ts") - diff /tmp/test_index.d.ts schemas/index.d.ts || (echo nimbus-schemas typescript package is out of sync please run make schemas_build;exit 1) - echo "Done. No problems found in schemas." +schemas_dist_pypi: schemas_build + $(SCHEMAS_RUN) "$(SCHEMAS_DIST_PYPI)" -schemas_code_format: - (cd schemas && poetry run black . && poetry run ruff --fix .) +schemas_dist_npm: schemas_build + $(SCHEMAS_RUN) "$(SCHEMAS_DIST_NPM)" -schemas_build: schemas_install schemas_build_pypi schemas_build_npm +schemas_dist: schemas_build schemas_dist_pypi schemas_dist_npm -schemas_build_pypi: - (cd schemas && poetry build) +schemas_deploy_pypi: schemas_build + $(SCHEMAS_RUN) "$(SCHEMAS_DEPLOY_PYPI)" -schemas_deploy_pypi: schemas_install schemas_build_pypi - cd schemas; poetry run twine upload --skip-existing dist/*; +schemas_deploy_npm: schemas_build + $(SCHEMAS_RUN) "$(SCHEMAS_DEPLOY_NPM)" -schemas_build_npm: schemas_install - (cd schemas && poetry run pydantic2ts --module mozilla_nimbus_schemas.__init__ --output ./index.d.ts --json2ts-cmd "yarn json2ts") +schemas_version_pypi: schemas_build + $(SCHEMAS_RUN) "$(SCHEMAS_VERSION_PYPI)" -schemas_deploy_npm: schemas_build_npm - cd schemas; yarn publish --new-version ${SCHEMAS_VERSION} --access public; +schemas_version_npm: schemas_build + $(SCHEMAS_RUN) "$(SCHEMAS_VERSION_NPM)" -schemas_version: - npm --prefix schemas version --allow-same-version ${SCHEMAS_VERSION}; - poetry --directory=schemas version ${SCHEMAS_VERSION}; +schemas_version: schemas_version_pypi schemas_version_npm diff --git a/schemas/.dockerignore b/schemas/.dockerignore new file mode 100644 index 000000000..e9a7892dd --- /dev/null +++ b/schemas/.dockerignore @@ -0,0 +1,17 @@ +**/__pycache__ +**/.cache +**/.coverage +**/.DS_Store +**/.mypy +**/.pycache +**/.pytest_cache +**/.vscode +**/.vscode/ +**/coverage/ +**/node_modules +**/yarn-error.log +docs +experimenter/legacy/legacy-ui/assets +experimenter/nimbus-ui/build +experimenter/reporting/reporting-ui/assets/ +experimenter/served/ diff --git a/schemas/Dockerfile b/schemas/Dockerfile new file mode 100644 index 000000000..f245e5ca8 --- /dev/null +++ b/schemas/Dockerfile @@ -0,0 +1,78 @@ +#------------------------- +FROM alpine:3.12.0 as file-loader + +# To preserve layer caching across machines which may have different local file properties +# such as permissions, timestamps, etc, all files are copied into a container and their +# permissions and timestamps are reset to consistent values +# Credit: https://gist.github.com/kekru/8ac61cd87536a4355220b56ae2f4b0a9 +COPY . /schemas/ +RUN chmod -R 555 /schemas \ + && chown -R root:root /schemas \ + && find /schemas -exec touch -a -m -t 201512180130.09 {} \; + + +#------------------------- +# System packages +FROM python:3.11.1 AS system-builder + +RUN curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - +RUN echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list +RUN apt-get update +RUN apt-get --no-install-recommends install -y apt-utils ca-certificates yarn parallel + +# Install nvm with node and npm +ENV NODE_VERSION=16.19.0 +ENV NVM_DIR=/root/.nvm +ENV PATH="/root/.nvm/versions/node/v${NODE_VERSION}/bin/:${PATH}" +RUN curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.35.3/install.sh | bash +RUN . "$NVM_DIR/nvm.sh" && nvm install ${NODE_VERSION} +RUN . "$NVM_DIR/nvm.sh" && nvm use v${NODE_VERSION} +RUN . "$NVM_DIR/nvm.sh" && nvm alias default v${NODE_VERSION} + +# Disable python pyc files +ENV PYTHONDONTWRITEBYTECODE 1 + +# Python packages +RUN curl -sSL https://install.python-poetry.org | python3 - +ENV PATH "/root/.local/bin:$PATH" +RUN poetry config virtualenvs.create false + + +# Python image +#------------------------- +FROM system-builder AS python-builder +WORKDIR /schemas + +COPY --from=file-loader /schemas/pyproject.toml /schemas/pyproject.toml +COPY --from=file-loader /schemas/poetry.lock /schemas/poetry.lock +COPY --from=file-loader /schemas/mozilla_nimbus_schemas /schemas/mozilla_nimbus_schemas +COPY --from=file-loader /schemas/README.md /schemas/README.md +RUN poetry install + +# If any package is installed, that is incompatible by version, this command +# will exit non-zero and print what is usually just a warning in `poetry install` +RUN poetry check + + +# Node image +#------------------------- +FROM system-builder AS node-builder +WORKDIR /schemas + +# Node packages for legacy and nimbus ui +COPY --from=file-loader /schemas/package.json /schemas/package.json +COPY --from=file-loader /schemas/yarn.lock /schemas/yarn.lock +RUN yarn install --frozen-lockfile + + +# Dev image +#------------------------- +FROM system-builder AS dev +WORKDIR /schemas + +# Python packages +COPY --from=python-builder /usr/local/bin/ /usr/local/bin/ +COPY --from=python-builder /usr/local/lib/python3.11/site-packages/ /usr/local/lib/python3.11/site-packages/ + +# Node packages +COPY --from=node-builder /schemas/node_modules/ /schemas/node_modules/ diff --git a/schemas/VERSION b/schemas/VERSION index 8bea39fb8..f517d76a8 100644 --- a/schemas/VERSION +++ b/schemas/VERSION @@ -1 +1 @@ -2023.10.2 +2023.10.3 diff --git a/schemas/package.json b/schemas/package.json index affbe4391..b871e7566 100644 --- a/schemas/package.json +++ b/schemas/package.json @@ -1,6 +1,6 @@ { "name": "@mozilla/nimbus-schemas", - "version": "2023.10.2", + "version": "2023.10.3", "description": "Schemas used by Mozilla Nimbus and related projects.", "main": "index.d.ts", "repository": { diff --git a/schemas/pyproject.toml b/schemas/pyproject.toml index 014f23aa1..ea21a091b 100644 --- a/schemas/pyproject.toml +++ b/schemas/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "mozilla-nimbus-schemas" -version = "2023.10.2" +version = "2023.10.3" description = "Schemas used by Mozilla Nimbus and related projects." authors = ["mikewilli"] license = "MPL 2.0"