Remove all reference to update_deps* (#22662)
This commit is contained in:
Kevin Meinhardt
GitHub
Родитель
e799addc7b
Коммит
5d24c7a759
56
Dockerfile
56
Dockerfile
|
|
@ -2,7 +2,7 @@
|
|||
# Read the docs/topics/development/docker.md file for more information about this Dockerfile.
|
||||
####################################################################################################
|
||||
|
||||
FROM python:3.11-slim-bookworm as olympia
|
||||
FROM python:3.11-slim-bookworm AS olympia
|
||||
|
||||
# Set shell to bash with logs and errors for build
|
||||
SHELL ["/bin/bash", "-xue", "-c"]
|
||||
|
|
@ -14,11 +14,11 @@ useradd -u ${OLYMPIA_UID} -g ${OLYMPIA_UID} -s /sbin/nologin -d /data/olympia ol
|
|||
EOF
|
||||
|
||||
# give olympia access to the HOME directory
|
||||
ENV HOME /data/olympia
|
||||
ENV HOME=/data/olympia
|
||||
WORKDIR ${HOME}
|
||||
RUN chown -R olympia:olympia ${HOME}
|
||||
|
||||
FROM olympia as base
|
||||
FROM olympia AS base
|
||||
# Add keys and repos for node and mysql
|
||||
# TODO: replace this with a bind mount on the RUN command
|
||||
COPY docker/*.gpg.asc /etc/apt/trusted.gpg.d/
|
||||
|
|
@ -43,8 +43,8 @@ EOF
|
|||
|
||||
# Compile required locale
|
||||
RUN localedef -i en_US -f UTF-8 en_US.UTF-8
|
||||
ENV LANG en_US.UTF-8
|
||||
ENV LC_ALL en_US.UTF-8
|
||||
ENV LANG=en_US.UTF-8
|
||||
ENV LC_ALL=en_US.UTF-8
|
||||
|
||||
RUN <<EOF
|
||||
# Create directory for dependencies
|
||||
|
|
@ -69,32 +69,34 @@ ENV PIP_BUILD=/deps/build/
|
|||
ENV PIP_CACHE_DIR=/deps/cache/
|
||||
ENV PIP_SRC=/deps/src/
|
||||
ENV PYTHONUSERBASE=/deps
|
||||
ENV PATH $PYTHONUSERBASE/bin:$PATH
|
||||
ENV PATH=$PYTHONUSERBASE/bin:$PATH
|
||||
ENV NPM_CONFIG_PREFIX=/deps/
|
||||
ENV NPM_CACHE_DIR=/deps/cache/npm
|
||||
ENV NPM_DEBUG=true
|
||||
# Set python path to the project root and src to resolve olympia modules correctly
|
||||
ENV PYTHONPATH=${HOME}:${HOME}/src
|
||||
|
||||
ENV PIP_COMMAND="python3 -m pip"
|
||||
ENV NPM_ARGS="--prefix ${NPM_CONFIG_PREFIX} --cache ${NPM_CACHE_DIR} --loglevel verbose"
|
||||
|
||||
# All we need in "base" is pip to be installed
|
||||
#this let's other layers install packages using the correct version.
|
||||
RUN \
|
||||
# Files needed to run the make command
|
||||
--mount=type=bind,source=Makefile-docker,target=${HOME}/Makefile-docker \
|
||||
# Files required to install pip dependencies
|
||||
--mount=type=bind,source=./requirements/pip.txt,target=${HOME}/requirements/pip.txt \
|
||||
--mount=type=cache,target=${PIP_CACHE_DIR},uid=${OLYMPIA_UID},gid=${OLYMPIA_UID} \
|
||||
# Command to install dependencies
|
||||
make -f Makefile-docker update_deps_pip
|
||||
<<EOF
|
||||
# Work arounds "Multiple .dist-info directories" issue.
|
||||
rm -rf /deps/build/*
|
||||
${PIP_COMMAND} install --progress-bar=off --no-deps --exists-action=w -r requirements/pip.txt
|
||||
EOF
|
||||
|
||||
# Define production dependencies as a single layer
|
||||
# let's the rest of the stages inherit prod dependencies
|
||||
# and makes copying the /deps dir to the final layer easy.
|
||||
FROM base as pip_production
|
||||
FROM base AS pip_production
|
||||
|
||||
RUN \
|
||||
# Files needed to run the make command
|
||||
--mount=type=bind,source=Makefile-docker,target=${HOME}/Makefile-docker \
|
||||
# Files required to install pip dependencies
|
||||
--mount=type=bind,source=./requirements/prod.txt,target=${HOME}/requirements/prod.txt \
|
||||
# Files required to install npm dependencies
|
||||
|
|
@ -103,15 +105,16 @@ RUN \
|
|||
# Mounts for caching dependencies
|
||||
--mount=type=cache,target=${PIP_CACHE_DIR},uid=${OLYMPIA_UID},gid=${OLYMPIA_UID} \
|
||||
--mount=type=cache,target=${NPM_CACHE_DIR},uid=${OLYMPIA_UID},gid=${OLYMPIA_UID} \
|
||||
# Command to install dependencies
|
||||
make -f Makefile-docker update_deps_production
|
||||
<<EOF
|
||||
${PIP_COMMAND} install --progress-bar=off --no-deps --exists-action=w -r requirements/prod.txt
|
||||
npm ci ${NPM_ARGS} --include=prod
|
||||
EOF
|
||||
|
||||
FROM pip_production as pip_development
|
||||
FROM base AS pip_development
|
||||
|
||||
RUN \
|
||||
# Files needed to run the make command
|
||||
--mount=type=bind,source=Makefile-docker,target=${HOME}/Makefile-docker \
|
||||
# Files required to install pip dependencies
|
||||
--mount=type=bind,source=./requirements/prod.txt,target=${HOME}/requirements/prod.txt \
|
||||
--mount=type=bind,source=./requirements/dev.txt,target=${HOME}/requirements/dev.txt \
|
||||
# Files required to install npm dependencies
|
||||
--mount=type=bind,source=package.json,target=${HOME}/package.json \
|
||||
|
|
@ -119,10 +122,13 @@ RUN \
|
|||
# Mounts for caching dependencies
|
||||
--mount=type=cache,target=${PIP_CACHE_DIR},uid=${OLYMPIA_UID},gid=${OLYMPIA_UID} \
|
||||
--mount=type=cache,target=${NPM_CACHE_DIR},uid=${OLYMPIA_UID},gid=${OLYMPIA_UID} \
|
||||
# Command to install dependencies
|
||||
make -f Makefile-docker update_deps_development
|
||||
<<EOF
|
||||
${PIP_COMMAND} install --progress-bar=off --no-deps --exists-action=w -r requirements/prod.txt
|
||||
${PIP_COMMAND} install --progress-bar=off --no-deps --exists-action=w -r requirements/dev.txt
|
||||
npm install ${NPM_ARGS} --no-save
|
||||
EOF
|
||||
|
||||
FROM base as locales
|
||||
FROM base AS locales
|
||||
ARG LOCALE_DIR=${HOME}/locale
|
||||
# Compile locales
|
||||
# Copy the locale files from the host so it is writable by the olympia user
|
||||
|
|
@ -136,7 +142,7 @@ RUN \
|
|||
|
||||
# More efficient caching by mounting the exact files we need
|
||||
# and copying only the static/ directory.
|
||||
FROM pip_production as assets
|
||||
FROM pip_production AS assets
|
||||
|
||||
# TODO: only copy the files we need for compiling assets
|
||||
COPY --chown=olympia:olympia static/ ${HOME}/static/
|
||||
|
|
@ -154,7 +160,7 @@ echo "from olympia.lib.settings_base import *" > settings_local.py
|
|||
DJANGO_SETTINGS_MODULE="settings_local" make -f Makefile-docker update_assets
|
||||
EOF
|
||||
|
||||
FROM base as sources
|
||||
FROM base AS sources
|
||||
|
||||
ARG DOCKER_BUILD DOCKER_COMMIT DOCKER_VERSION
|
||||
|
||||
|
|
@ -175,12 +181,12 @@ COPY --from=assets --chown=olympia:olympia ${HOME}/static-build ${HOME}/static-b
|
|||
# Set shell back to sh until we can prove we can use bash at runtime
|
||||
SHELL ["/bin/sh", "-c"]
|
||||
|
||||
FROM sources as development
|
||||
FROM sources AS development
|
||||
|
||||
# Copy dependencies from `pip_development`
|
||||
COPY --from=pip_development --chown=olympia:olympia /deps /deps
|
||||
|
||||
FROM sources as production
|
||||
FROM sources AS production
|
||||
|
||||
# Copy dependencies from `pip_production`
|
||||
COPY --from=pip_production --chown=olympia:olympia /deps /deps
|
||||
|
|
|
|||
|
|
@ -11,20 +11,6 @@ APP=src/olympia/
|
|||
NUM_ADDONS=10
|
||||
NUM_THEMES=$(NUM_ADDONS)
|
||||
|
||||
NPM_ARGS :=
|
||||
|
||||
ifneq ($(NPM_CONFIG_PREFIX),)
|
||||
NPM_ARGS := --prefix $(NPM_CONFIG_PREFIX)
|
||||
endif
|
||||
|
||||
ifneq ($(NPM_CACHE_DIR),)
|
||||
NPM_ARGS := $(NPM_ARGS) --cache $(NPM_CACHE_DIR)
|
||||
endif
|
||||
|
||||
ifneq ($(NPM_DEBUG),)
|
||||
NPM_ARGS := $(NPM_ARGS) --loglevel verbose
|
||||
endif
|
||||
|
||||
NODE_MODULES := $(NPM_CONFIG_PREFIX)node_modules/
|
||||
|
||||
REQUIRED_FILES := \
|
||||
|
|
@ -97,30 +83,6 @@ populate_data: ## populate a new database
|
|||
# changes.
|
||||
$(PYTHON_COMMAND) manage.py generate_default_addons_for_frontend
|
||||
|
||||
.PHONY: update_deps_pip
|
||||
update_deps_pip: ## Install pip
|
||||
# Work arounds "Multiple .dist-info directories" issue.
|
||||
rm -rf /deps/build/*
|
||||
$(PIP_COMMAND) install --progress-bar=off --no-deps --exists-action=w -r requirements/pip.txt
|
||||
|
||||
|
||||
.PHONY: update_deps_development
|
||||
update_deps_development: ## update the python and node dependencies for development
|
||||
$(PIP_COMMAND) install --progress-bar=off --no-deps --exists-action=w -r requirements/dev.txt
|
||||
npm install $(NPM_ARGS) --no-save --include=dev
|
||||
|
||||
.PHONY: update_deps_production
|
||||
update_deps_production: ## update the python and node dependencies for production
|
||||
$(PIP_COMMAND) install --progress-bar=off --no-deps --exists-action=w -r requirements/prod.txt
|
||||
npm ci $(NPM_ARGS) --include=prod
|
||||
|
||||
.PHONY: update_deps
|
||||
update_deps: update_deps_pip update_deps_production update_deps_development ## update the python and node dependencies
|
||||
|
||||
.PHONY: prune_deps
|
||||
prune_deps: ## remove unused dependencies
|
||||
npm prune $(NPM_ARGS) --production
|
||||
|
||||
.PHONY: update_db
|
||||
update_db: ## run the database migrations
|
||||
$(PYTHON_COMMAND) manage.py migrate --noinput
|
||||
|
|
@ -135,7 +97,7 @@ update_assets:
|
|||
$(PYTHON_COMMAND) manage.py collectstatic --noinput
|
||||
|
||||
.PHONY: update
|
||||
update: update_deps update_db update_assets ## update the dependencies, the database, and assets
|
||||
update: update_db update_assets ## update the dependencies, the database, and assets
|
||||
|
||||
.PHONY: reindex
|
||||
reindex: ## reindex everything in elasticsearch, for AMO
|
||||
|
|
@ -195,7 +157,7 @@ dbshell: ## connect to a database shell
|
|||
$(PYTHON_COMMAND) ./manage.py dbshell
|
||||
|
||||
.PHONY: initialize
|
||||
initialize: update_deps initialize_db update_assets populate_data reindex_data ## init the dependencies, the database, and assets
|
||||
initialize: initialize_db update_assets populate_data reindex_data ## init the dependencies, the database, and assets
|
||||
|
||||
reload-uwsgi: reload
|
||||
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ All documentation is in plain text files using primarily
|
|||
If you're unsure, activate your `virtualenv` and run:
|
||||
|
||||
```bash
|
||||
make update_deps
|
||||
make up
|
||||
```
|
||||
|
||||
The documentation is viewable at <http://addons-server.readthedocs.io/>, and
|
||||
|
|
|
|||
|
|
@ -37,21 +37,14 @@ This will add hashes and sort the requirements for you, adding comments to show
|
|||
|
||||
### Managing Python Dependencies
|
||||
|
||||
We have two requirements files for Python dependencies:
|
||||
All Python dependencies are defined in requirements files in the `requirements` directory. Our 3 most important files are:
|
||||
|
||||
- **`pip.txt`**: Specifies the version of pip to guarantee consistency.
|
||||
- **`prod.txt`**: Dependencies required in the production environment.
|
||||
|
||||
```bash
|
||||
make update_deps_prod
|
||||
```
|
||||
|
||||
- **`dev.txt`**: Dependencies used for development, linting, testing, etc.
|
||||
|
||||
```bash
|
||||
make update_deps
|
||||
```
|
||||
|
||||
We use Dependabot to automatically create pull requests for updating dependencies. This is configured in the `.github/dependabot.yml` file targeting files in the `requirements` directory.
|
||||
We use Dependabot to automatically create pull requests for updating dependencies.
|
||||
This is configured in the `.github/dependabot.yml` file targeting files in the `requirements` directory.
|
||||
|
||||
### Managing Transitive Dependencies
|
||||
|
||||
|
|
@ -112,21 +105,8 @@ By caching the `/deps` folder, the project ensures that dependencies are quickly
|
|||
To update/install all dependencies, run the following command:
|
||||
|
||||
```bash
|
||||
make update_deps
|
||||
make up
|
||||
```
|
||||
|
||||
This will install all Python and frontend dependencies. By default, this command runs in a Docker container, but you can run it on the host by targeting the Makefile-docker:
|
||||
|
||||
```bash
|
||||
make -f Makefile-docker update_deps
|
||||
```
|
||||
|
||||
This method is used in GitHub Actions that do not need a full container to run.
|
||||
|
||||
**Note**: If you are adding a new dependency, make sure to update static assets imported from the new versions:
|
||||
|
||||
```bash
|
||||
make update_assets
|
||||
```
|
||||
|
||||
By following these practices, the **addons-server** project ensures efficient and reliable dependency management, both locally and in CI environments. For more detailed instructions, refer to the project's Makefile and Dockerfile configurations in the repository.
|
||||
This will rebuild the Docker image with the current dependencies specified in the `requirements` and `package.json` files.
|
||||
We do not support updating dependencies in a running container as the /deps folder is not writable by the olympia user.
|
||||
|
|
|
|||
|
|
@ -118,14 +118,6 @@ A common benefit of using Makefiles in this manner is the ability to coordinate
|
|||
make build_docker_image
|
||||
```
|
||||
|
||||
4. **`update_deps`**:
|
||||
- **Purpose**: Updates dependencies for production and development environments. This command splits dependencies between production and development to enable efficient caching during the Docker build process.
|
||||
- **Usage**:
|
||||
|
||||
```sh
|
||||
make update_deps
|
||||
```
|
||||
|
||||
## Forcing a Specific Makefile
|
||||
|
||||
You can force Make to run a specific command from a particular Makefile by specifying the file:
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ Effective troubleshooting and debugging practices are essential for maintaining
|
|||
- **Solution**: Ensure that all dependencies are installed by running the following command:
|
||||
|
||||
```sh
|
||||
make update_deps
|
||||
make up
|
||||
```
|
||||
|
||||
4. **Locale Compilation Issues**:
|
||||
|
|
|
|||
|
|
@ -1,121 +0,0 @@
|
|||
#! /bin/bash
|
||||
#
|
||||
# This script will do the following:
|
||||
# - checkout and update git master branch
|
||||
# - create a new branch (name should look like l10n-extract-<date>-<rev>)
|
||||
# - Extract new strings and update the .po/.pot files
|
||||
# - Commit that extraction to the branch
|
||||
#
|
||||
# If you provide a GITHUB_TOKEN variable to the environment then this script
|
||||
# can also automatically push to a remote branch and create a pull request for
|
||||
# you through addons-robot. Ask @diox or @muffinresearch for this token.
|
||||
|
||||
# Exit immediately when a command fails.
|
||||
set -e
|
||||
|
||||
# Make sure exit code are respected in a pipeline.
|
||||
set -o pipefail
|
||||
|
||||
# Treat unset variables as an error an exit immediately.
|
||||
set -u
|
||||
|
||||
INITIAL_GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
|
||||
GIT_CHANGES=$(git status --porcelain)
|
||||
GIT_REMOTE="https://github.com/mozilla/addons-server.git" # Upstream.
|
||||
REV=""
|
||||
MESSAGE=""
|
||||
BRANCH_NAME=""
|
||||
|
||||
DRY_RUN=false
|
||||
|
||||
if [[ "${1:-}" == "--dry-run" ]]; then
|
||||
DRY_RUN=true
|
||||
fi
|
||||
|
||||
info() {
|
||||
local message="$1"
|
||||
|
||||
echo ""
|
||||
echo "INFO: $message"
|
||||
echo ""
|
||||
}
|
||||
|
||||
error() {
|
||||
local message="$1"
|
||||
|
||||
echo "ERROR: $message"
|
||||
exit 1
|
||||
}
|
||||
|
||||
function init_environment {
|
||||
# Detect local (uncommitted) changes.
|
||||
if [[ ! -z "$GIT_CHANGES" ]]; then
|
||||
error "You have local changes, therefore this script cannot continue."
|
||||
fi
|
||||
|
||||
# Switch to the `master` branch if we are not on it already.
|
||||
if [[ "$INITIAL_GIT_BRANCH" != "master" ]]; then
|
||||
git checkout master
|
||||
fi
|
||||
|
||||
# Make sure the 'master' branch is up-to-date.
|
||||
git pull "$GIT_REMOTE" master
|
||||
|
||||
REV=$(git rev-parse --short HEAD)
|
||||
MESSAGE="Extracted l10n messages from $(date -u --iso-8601=date) at $REV"
|
||||
BRANCH_NAME="l10n-extract-$(date -u --iso-8601=date)-$REV"
|
||||
|
||||
# Ensure the branch to extract the locales is clean.
|
||||
if [[ $(git branch --list "$BRANCH_NAME") ]]; then
|
||||
info "Deleting branch '$BRANCH_NAME' because it already exists"
|
||||
git branch -D "$BRANCH_NAME"
|
||||
fi
|
||||
|
||||
info "Creating and switching to branch '$BRANCH_NAME'"
|
||||
git checkout -b "$BRANCH_NAME"
|
||||
|
||||
make -f Makefile-docker update_deps
|
||||
}
|
||||
|
||||
function generate_post_data()
|
||||
{
|
||||
cat <<EOF
|
||||
{
|
||||
"title": "$MESSAGE",
|
||||
"head": "$BRANCH_NAME",
|
||||
"base":"master"
|
||||
}
|
||||
EOF
|
||||
}
|
||||
|
||||
|
||||
function create_pull_request {
|
||||
info "Creating the auto merge pull request for $BRANCH_NAME ..."
|
||||
curl --verbose -H "Authorization: token $GITHUB_TOKEN" --data "$(generate_post_data)" "https://api.github.com/repos/mozilla/addons-server/pulls"
|
||||
info "Pull request created."
|
||||
}
|
||||
|
||||
if [[ $DRY_RUN == true ]]; then
|
||||
info "Dry run only. Not committing."
|
||||
else
|
||||
info "This script will extract new strings and update the .po/.pot files."
|
||||
|
||||
init_environment
|
||||
fi
|
||||
|
||||
./scripts/run_l10n_extraction.sh
|
||||
|
||||
if [[ $DRY_RUN == false ]]; then
|
||||
./scripts/push_l10n_extraction.sh
|
||||
|
||||
# This script is meant to be run inside a virtualenv or inside our docker
|
||||
# container. If it's the latter, it doesn't necessarily have access to the ssh
|
||||
# config, therefore we can't reliably push and create a pull request without a
|
||||
# GitHub API token.
|
||||
if [[ -z "${GITHUB_TOKEN-}" ]]; then
|
||||
info "No github token present. You should now go back to your normal environment to push this branch and create the pull request."
|
||||
else
|
||||
create_pull_request
|
||||
fi
|
||||
fi
|
||||
|
||||
Загрузка…
Ссылка в новой задаче