Merge deployment and development Dockerfiles (#20480)
* Merge deployment and development Dockerfiles * Stop using root to install dependencies * Run make commands from os with --user olympia
This commit is contained in:
Mathieu Pillard
GitHub
Родитель
a5f6eb0b44
Коммит
5c0e7a3c88
|
|
@ -350,9 +350,6 @@ commands:
|
|||
make_release:
|
||||
description: "Builds and pushes a Docker image"
|
||||
parameters:
|
||||
dockerfile:
|
||||
type: string
|
||||
default: "Dockerfile.deploy"
|
||||
image_tag:
|
||||
type: string
|
||||
default: "latest"
|
||||
|
|
@ -370,7 +367,7 @@ commands:
|
|||
command: |
|
||||
docker version
|
||||
docker login -u "${DOCKERHUB_USER}" -p "${DOCKERHUB_PASS}"
|
||||
docker build -t app:build -f << parameters.dockerfile >> --label git.commit="$CIRCLE_SHA1" .
|
||||
docker build -t app:build --label git.commit="$CIRCLE_SHA1" .
|
||||
docker tag app:build "${DOCKERHUB_REPO}":<< parameters.image_tag >>
|
||||
docker push "${DOCKERHUB_REPO}":<< parameters.image_tag >>
|
||||
|
||||
|
|
@ -459,7 +456,7 @@ commands:
|
|||
install_node_dependencies:
|
||||
type: boolean
|
||||
default: false
|
||||
update_deps:
|
||||
install_python_dev_dependencies:
|
||||
type: boolean
|
||||
default: false
|
||||
steps:
|
||||
|
|
@ -498,20 +495,35 @@ commands:
|
|||
command: |
|
||||
echo export PYTHONPATH=src >> $BASH_ENV
|
||||
echo export CPUCOUNT=2 >> $BASH_ENV
|
||||
echo export NPM_CONFIG_PREFIX=/deps/ >> $BASH_ENV
|
||||
echo export CC=\"`python -c 'import sysconfig; print(sysconfig.get_config_var("CC"))'`\" >> $BASH_ENV
|
||||
cat $BASH_ENV
|
||||
- when:
|
||||
condition: << parameters.install_python_test_dependencies >>
|
||||
steps:
|
||||
- run: make install_python_test_dependencies
|
||||
- run:
|
||||
name: Install Python dependencies
|
||||
command: |
|
||||
make install_python_test_dependencies
|
||||
- when:
|
||||
condition: << parameters.install_python_dev_dependencies >>
|
||||
steps:
|
||||
- run:
|
||||
name: Install Python dependencies
|
||||
command: |
|
||||
make install_python_dev_dependencies
|
||||
- when:
|
||||
condition: << parameters.install_node_dependencies >>
|
||||
steps:
|
||||
- run: make install_node_dependencies
|
||||
- when:
|
||||
condition: << parameters.update_deps >>
|
||||
steps:
|
||||
- run: make update_deps
|
||||
- run:
|
||||
name: Install Node dependencies
|
||||
command: |
|
||||
sudo mkdir /deps
|
||||
sudo chown circleci /deps
|
||||
ACTUAL_CIRCLE_WORKING_DIRECTORY="${CIRCLE_WORKING_DIRECTORY/#\~/$HOME}"
|
||||
ln -s ${ACTUAL_CIRCLE_WORKING_DIRECTORY}/package.json /deps/package.json
|
||||
ln -s ${ACTUAL_CIRCLE_WORKING_DIRECTORY}/package-lock.json /deps/package-lock.json
|
||||
make install_node_dependencies
|
||||
# should be executed after all python install commands
|
||||
- run: pyenv rehash
|
||||
|
||||
|
|
@ -546,7 +558,8 @@ jobs:
|
|||
steps:
|
||||
- setup_container:
|
||||
install_python_test_dependencies: false
|
||||
update_deps: true
|
||||
install_python_dev_dependencies: true
|
||||
install_node_dependencies: true
|
||||
- run: make update_assets
|
||||
- run:
|
||||
command: pytest -m "static_assets" -v src/olympia/
|
||||
|
|
@ -558,7 +571,7 @@ jobs:
|
|||
- setup_container:
|
||||
wait_services: false
|
||||
install_node_dependencies: true
|
||||
- run: make setup-codestyle
|
||||
- run: make install_python_codestyle_dependencies
|
||||
- run: pyenv rehash
|
||||
- run: make lint-codestyle
|
||||
|
||||
|
|
@ -632,7 +645,6 @@ jobs:
|
|||
- checkout
|
||||
- make_release:
|
||||
image_tag: latest
|
||||
dockerfile: "Dockerfile.deploy"
|
||||
|
||||
release-tag:
|
||||
<<: *defaults-release
|
||||
|
|
@ -640,7 +652,6 @@ jobs:
|
|||
- checkout
|
||||
- make_release:
|
||||
image_tag: "${CIRCLE_TAG}"
|
||||
dockerfile: "Dockerfile.deploy"
|
||||
|
||||
workflows:
|
||||
version: 2
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
.env
|
||||
.git
|
||||
.cache
|
||||
.tox
|
||||
deps/
|
||||
node_modules/
|
||||
storage/
|
||||
logs/*
|
||||
|
|
|
|||
|
|
@ -48,6 +48,7 @@ storage/shared_storage/*
|
|||
storage/sitemaps/*
|
||||
supervisord.pid
|
||||
tmp/*
|
||||
deps/*
|
||||
|
||||
# private projects
|
||||
private/
|
||||
|
|
@ -55,3 +56,4 @@ private/
|
|||
# do not ignore the following files
|
||||
!docker-compose.private.yml
|
||||
!private/README.md
|
||||
!deps/.keep
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ __pycache__/
|
|||
i18n/
|
||||
img/
|
||||
node_lib/
|
||||
deps/
|
||||
|
||||
# safe-list files we want to process
|
||||
!*.js
|
||||
|
|
|
|||
101
Dockerfile
101
Dockerfile
|
|
@ -1,15 +1,10 @@
|
|||
FROM python:3.10-slim-buster
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1
|
||||
|
||||
ARG GROUP_ID=1000
|
||||
ARG USER_ID=1000
|
||||
|
||||
# Run all initial setup with root user. This is the default but mentioned here
|
||||
# for documentation.
|
||||
# We won't switch to the `olympia` user inside the dockerfile
|
||||
# but rather use the `user` option in docker-compose.yml instead
|
||||
USER root
|
||||
# Should change it to use ARG instead of ENV for OLYMPIA_UID/OLYMPIA_GID
|
||||
# once the jenkins server is upgraded to support docker >= v1.9.0
|
||||
ENV OLYMPIA_UID=9500 \
|
||||
OLYMPIA_GID=9500
|
||||
RUN groupadd -g ${OLYMPIA_GID} olympia && useradd -u ${OLYMPIA_UID} -g ${OLYMPIA_GID} -s /sbin/nologin -d /data/olympia olympia
|
||||
|
||||
# Add support for https apt repos and gpg signed repos
|
||||
RUN apt-get update && apt-get install -y \
|
||||
|
|
@ -24,8 +19,6 @@ RUN APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=DontWarn \
|
|||
apt-key add /etc/pki/gpg/mysql.gpg.key
|
||||
COPY docker/*.list /etc/apt/sources.list.d/
|
||||
|
||||
# IMPORTANT: When editing this list below, make sure to also update
|
||||
# `Dockerfile.deploy`.
|
||||
# Allow scripts to detect we're running in our own container and install
|
||||
# packages.
|
||||
RUN touch /addons-server-docker-container \
|
||||
|
|
@ -34,9 +27,7 @@ RUN touch /addons-server-docker-container \
|
|||
bash-completion \
|
||||
build-essential \
|
||||
curl \
|
||||
libcap-dev \
|
||||
libjpeg-dev \
|
||||
libpcre3-dev \
|
||||
libsasl2-dev \
|
||||
libxml2-dev \
|
||||
libxslt-dev \
|
||||
|
|
@ -44,7 +35,6 @@ RUN touch /addons-server-docker-container \
|
|||
zlib1g-dev \
|
||||
libffi-dev \
|
||||
libssl-dev \
|
||||
libpcre3-dev \
|
||||
nodejs \
|
||||
# Git, because we're using git-checkout dependencies
|
||||
git \
|
||||
|
|
@ -59,66 +49,57 @@ RUN touch /addons-server-docker-container \
|
|||
pngcrush \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# IMPORTANT: When editing one of these lists below, make sure to also update
|
||||
# `Dockerfile.deploy`.
|
||||
# Add our custom mime types (required for for ts/json/md files)
|
||||
ADD docker/etc/mime.types /etc/mime.types
|
||||
|
||||
# Compile required locale
|
||||
RUN localedef -i en_US -f UTF-8 en_US.UTF-8
|
||||
|
||||
# Set the locale. This is mainly so that tests can write non-ascii files to
|
||||
# disk.
|
||||
ENV LANG en_US.UTF-8
|
||||
ENV LC_ALL en_US.UTF-8
|
||||
|
||||
COPY . /code
|
||||
WORKDIR /code
|
||||
ENV HOME /data/olympia
|
||||
|
||||
RUN groupadd -g ${GROUP_ID} olympia
|
||||
RUN useradd -g ${GROUP_ID} -u ${USER_ID} -Md /deps/ olympia
|
||||
# version.json is overwritten by CircleCI (see circle.yml).
|
||||
# The pipeline v2 standard requires the existence of /app/version.json
|
||||
# inside the docker image, thus it's copied there.
|
||||
COPY version.json /app/version.json
|
||||
COPY --chown=olympia:olympia . ${HOME}
|
||||
WORKDIR ${HOME}
|
||||
|
||||
# Create /deps/ and move ownership over to `olympia` user so that
|
||||
# we can install things there
|
||||
# Also run `chown` on `/code/` which technically doesn't change permissions
|
||||
# on the host but ensures that the image knows about correct permissions.
|
||||
RUN mkdir /deps/ && chown -R olympia:olympia /deps/ /code/
|
||||
# Set up directories and links that we'll need later, before switching to the
|
||||
# olympia user.
|
||||
RUN mkdir /deps \
|
||||
&& chown olympia:olympia /deps \
|
||||
&& rm -rf ${HOME}/src/olympia.egg-info \
|
||||
&& mkdir ${HOME}/src/olympia.egg-info \
|
||||
&& chown olympia:olympia ${HOME}/src/olympia.egg-info \
|
||||
# For backwards-compatibility purposes, set up links to uwsgi. Note that
|
||||
# the target doesn't exist yet at this point, but it will later.
|
||||
&& ln -s /deps/bin/uwsgi /usr/bin/uwsgi \
|
||||
&& ln -s /usr/bin/uwsgi /usr/sbin/uwsgi
|
||||
|
||||
USER olympia:olympia
|
||||
|
||||
# Install all dependencies, and add symlink for old uwsgi binary paths
|
||||
ENV PIP_USER=true
|
||||
ENV PIP_BUILD=/deps/build/
|
||||
ENV PIP_CACHE_DIR=/deps/cache/
|
||||
ENV PIP_SRC=/deps/src/
|
||||
|
||||
# Allow us to install all dependencies to the `olympia` users
|
||||
# home directory (which is `/deps/`)
|
||||
ENV PIP_USER=true
|
||||
ENV PYTHONUSERBASE=/deps
|
||||
|
||||
# Make sure that installed binaries are accessible
|
||||
ENV PATH $PYTHONUSERBASE/bin:$PATH
|
||||
|
||||
ENV NPM_CONFIG_PREFIX=/deps/
|
||||
ENV SWIG_FEATURES="-D__x86_64__"
|
||||
RUN ln -s ${HOME}/package.json /deps/package.json \
|
||||
&& ln -s ${HOME}/package-lock.json /deps/package-lock.json \
|
||||
&& make update_deps
|
||||
|
||||
# From now on run everything with the `olympia` user by default.
|
||||
USER olympia
|
||||
WORKDIR ${HOME}
|
||||
|
||||
RUN ln -s /code/package.json /deps/package.json && \
|
||||
ln -s /code/package-lock.json /deps/package-lock.json && \
|
||||
make update_deps && \
|
||||
rm -rf /deps/build/ /deps/cache/
|
||||
|
||||
# Preserve bash history across image updates.
|
||||
# This works best when you link your local source code
|
||||
# as a volume.
|
||||
ENV HISTFILE /code/docker/artifacts/bash_history
|
||||
|
||||
# Configure bash history.
|
||||
ENV HISTSIZE 50000
|
||||
ENV HISTIGNORE ls:exit:"cd .."
|
||||
|
||||
# This prevents dupes but only in memory for the current session.
|
||||
ENV HISTCONTROL erasedups
|
||||
|
||||
ENV CLEANCSS_BIN /deps/node_modules/.bin/cleancss
|
||||
ENV LESS_BIN /deps/node_modules/.bin/lessc
|
||||
ENV JS_MINIFIER_BIN /deps/node_modules/.bin/terser
|
||||
ENV ADDONS_LINTER_BIN /deps/node_modules/.bin/addons-linter
|
||||
# Build locales, assets, build id.
|
||||
RUN echo "from olympia.lib.settings_base import *\n" \
|
||||
> settings_local.py && DJANGO_SETTINGS_MODULE='settings_local' locale/compile-mo.sh locale \
|
||||
&& DJANGO_SETTINGS_MODULE='settings_local' python manage.py compress_assets \
|
||||
&& DJANGO_SETTINGS_MODULE='settings_local' python manage.py generate_jsi18n_files \
|
||||
&& DJANGO_SETTINGS_MODULE='settings_local' python manage.py collectstatic --noinput \
|
||||
&& npm prune --production \
|
||||
&& ./scripts/generate_build.py > build.py \
|
||||
&& rm -f settings_local.py settings_local.pyc
|
||||
|
|
|
|||
|
|
@ -1,96 +0,0 @@
|
|||
FROM python:3.10-slim-buster
|
||||
|
||||
# need to compile swig
|
||||
ENV SWIG_FEATURES="-D__x86_64__"
|
||||
|
||||
# Should change it to use ARG instead of ENV for OLYMPIA_UID
|
||||
# once the jenkins server is upgraded to support docker >= v1.9.0
|
||||
ENV OLYMPIA_UID=9500
|
||||
RUN useradd -u ${OLYMPIA_UID} -s /sbin/nologin olympia
|
||||
|
||||
# Add support for https apt repos and gpg signed repos
|
||||
RUN apt-get update && apt-get install -y \
|
||||
apt-transport-https \
|
||||
gnupg2 \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
# Add keys and repos for node and mysql
|
||||
COPY docker/*.gpg.key /etc/pki/gpg/
|
||||
RUN APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=DontWarn \
|
||||
apt-key add /etc/pki/gpg/nodesource.gpg.key \
|
||||
&& APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=DontWarn \
|
||||
apt-key add /etc/pki/gpg/mysql.gpg.key
|
||||
COPY docker/*.list /etc/apt/sources.list.d/
|
||||
|
||||
# IMPORTANT: When editing this list below, make sure to also update
|
||||
# `Dockerfile.deploy`.
|
||||
# Allow scripts to detect we're running in our own container and install
|
||||
# packages.
|
||||
RUN touch /addons-server-docker-container \
|
||||
&& apt-get update && apt-get -t buster install -y \
|
||||
# General (dev-) dependencies
|
||||
bash-completion \
|
||||
build-essential \
|
||||
curl \
|
||||
libjpeg-dev \
|
||||
libsasl2-dev \
|
||||
libxml2-dev \
|
||||
libxslt-dev \
|
||||
locales \
|
||||
zlib1g-dev \
|
||||
libffi-dev \
|
||||
libssl-dev \
|
||||
nodejs \
|
||||
# Git, because we're using git-checkout dependencies
|
||||
git \
|
||||
# Dependencies for mysql-python (from mysql apt repo, not debian)
|
||||
mysql-client \
|
||||
libmysqlclient-dev \
|
||||
swig \
|
||||
gettext \
|
||||
# Use rsvg-convert to render our static theme previews
|
||||
librsvg2-bin \
|
||||
# Use pngcrush to optimize the PNGs uploaded by developers
|
||||
pngcrush \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# IMPORTANT: When editing one of these lists below, make sure to also update
|
||||
# `Dockerfile`.
|
||||
ADD docker/etc/mime.types /etc/mime.types
|
||||
|
||||
# Compile required locale
|
||||
RUN localedef -i en_US -f UTF-8 en_US.UTF-8
|
||||
ENV LANG en_US.UTF-8
|
||||
ENV LC_ALL en_US.UTF-8
|
||||
|
||||
# version.json is overwritten by CircleCI (see circle.yml).
|
||||
# The pipeline v2 standard requires the existence of /app/version.json
|
||||
# inside the docker image, thus it's copied there.
|
||||
COPY version.json /app/version.json
|
||||
COPY . /data/olympia
|
||||
WORKDIR /data/olympia
|
||||
|
||||
# Install all python requires, and add symlink for old uwsgi binary paths
|
||||
RUN pip3 install --no-cache-dir --exists-action=w --no-deps -r requirements/system.txt \
|
||||
&& pip3 install --no-cache-dir --exists-action=w --no-deps -r requirements/prod.txt \
|
||||
&& pip3 install --no-cache-dir --exists-action=w --no-deps -r requirements/prod_without_hash.txt \
|
||||
&& pip3 install --no-cache-dir --exists-action=w --no-deps -e . \
|
||||
&& ln -s /usr/local/bin/uwsgi /usr/bin/uwsgi \
|
||||
&& ln -s /usr/bin/uwsgi /usr/sbin/uwsgi
|
||||
|
||||
WORKDIR /data/olympia
|
||||
|
||||
# Build locales, assets, build id.
|
||||
RUN echo "from olympia.lib.settings_base import *\n\
|
||||
LESS_BIN = 'node_modules/less/bin/lessc'\n\
|
||||
CLEANCSS_BIN = 'node_modules/clean-css-cli/bin/cleancss'\n\
|
||||
JS_MINIFIER_BIN = 'node_modules/terser/bin/terser'\n\
|
||||
FXA_CONFIG = {'default': {}, 'internal': {}}\n"\
|
||||
> settings_local.py && DJANGO_SETTINGS_MODULE='settings_local' locale/compile-mo.sh locale \
|
||||
&& npm install \
|
||||
&& make -f Makefile-docker copy_node_js \
|
||||
&& DJANGO_SETTINGS_MODULE='settings_local' python manage.py compress_assets \
|
||||
&& DJANGO_SETTINGS_MODULE='settings_local' python manage.py generate_jsi18n_files \
|
||||
&& DJANGO_SETTINGS_MODULE='settings_local' python manage.py collectstatic --noinput \
|
||||
&& npm prune --production \
|
||||
&& ./scripts/generate_build.py > build.py \
|
||||
&& rm -f settings_local.py settings_local.pyc
|
||||
|
|
@ -80,17 +80,20 @@ cleanup_python_build_dir:
|
|||
# Work arounds "Multiple .dist-info directories" issue.
|
||||
rm -rf /deps/build/*
|
||||
|
||||
.PHONY: install_python_codestyle_dependencies
|
||||
install_python_codestyle_dependencies:
|
||||
$(PIP_COMMAND) install --progress-bar=off --no-deps --exists-action=w -r requirements/codestyle.txt
|
||||
|
||||
.PHONY: install_python_test_dependencies
|
||||
install_python_test_dependencies:
|
||||
# Can't use --progress-bar=off for system packages as long as our docker image
|
||||
# doesn't have pip 10 by default.
|
||||
$(PIP_COMMAND) install --no-deps --exists-action=w -r requirements/system.txt
|
||||
$(PIP_COMMAND) install --progress-bar=off --no-deps --exists-action=w -r requirements/prod.txt
|
||||
$(PIP_COMMAND) install --progress-bar=off --no-deps --exists-action=w -r requirements/prod_without_hash.txt
|
||||
$(PIP_COMMAND) install --progress-bar=off --no-deps --exists-action=w -r requirements/tests.txt
|
||||
|
||||
.PHONY: install_python_dev_dependencies
|
||||
install_python_dev_dependencies: install_python_test_dependencies setup-codestyle
|
||||
install_python_dev_dependencies: install_python_test_dependencies install_python_codestyle_dependencies
|
||||
$(PIP_COMMAND) install --progress-bar=off --no-deps --exists-action=w -r requirements/dev.txt
|
||||
$(PIP_COMMAND) install --progress-bar=off --no-deps --exists-action=w -r requirements/docs.txt
|
||||
# pep 517 mode (the default) breaks editable install in our project. https://github.com/mozilla/addons-server/issues/16144
|
||||
|
|
@ -162,10 +165,6 @@ perf-tests: setup-ui-tests
|
|||
$(PIP_COMMAND) install --progress-bar=off --no-deps -r requirements/perftests.txt
|
||||
locust --no-web -c 1 -f tests/performance/locustfile.py --host "http://olympia.test"
|
||||
|
||||
.PHONY: setup-codestyle
|
||||
setup-codestyle:
|
||||
$(PIP_COMMAND) install --progress-bar=off --no-deps --exists-action=w -r requirements/codestyle.txt
|
||||
|
||||
.PHONY: lint
|
||||
lint: ## lint the code
|
||||
black --check src/ tests/
|
||||
|
|
@ -196,7 +195,7 @@ initialize: update_deps initialize_db update_assets populate_data ## init the de
|
|||
|
||||
.PHONY: reload
|
||||
reload: ## force django code reload
|
||||
uwsgi --reload /code/docker/artifacts/addons-server-uwsgi-master.pid
|
||||
uwsgi --reload ${HOME}/docker/artifacts/addons-server-uwsgi-master.pid
|
||||
|
||||
reload-uwsgi: reload
|
||||
|
||||
|
|
|
|||
38
Makefile-os
38
Makefile-os
|
|
@ -1,3 +1,8 @@
|
|||
UID := $(shell id -u)
|
||||
GID := $(shell id -g)
|
||||
export UID
|
||||
export GID
|
||||
|
||||
.PHONY: help_redirect
|
||||
help_redirect:
|
||||
@$(MAKE) help --no-print-directory
|
||||
|
|
@ -11,21 +16,44 @@ help_submake:
|
|||
|
||||
.PHONY: update_docker
|
||||
update_docker: ## update all the docker images
|
||||
docker-compose exec worker make update_deps
|
||||
docker-compose exec web make update
|
||||
docker-compose exec --user olympia worker make update_deps
|
||||
docker-compose exec --user olympia web make update
|
||||
docker-compose restart web
|
||||
docker-compose restart worker
|
||||
|
||||
.PHONY: shell
|
||||
shell: ## connect to a running addons-server docker shell
|
||||
docker-compose exec web bash
|
||||
docker-compose exec --user olympia web bash
|
||||
|
||||
.PHONY: shell
|
||||
.PHONY: rootshell
|
||||
rootshell: ## connect to a running addons-server docker shell with root user
|
||||
docker-compose exec --user root web bash
|
||||
|
||||
.PHONY: create_env_file
|
||||
create_env_file:
|
||||
echo "UID=${UID}\nGID=${GID}" > .env
|
||||
|
||||
.PHONY: initialize_docker
|
||||
initialize_docker: create_env_file
|
||||
# Run a fresh container from the base image to install deps. Since /deps is
|
||||
# shared via a volume in docker-compose.yml, this installs deps for both web
|
||||
# and worker containers, and does so without requiring the containers to be up.
|
||||
# We just create dummy empty package.json and package-lock.json in deps/ so
|
||||
# that docker-compose doesn't create dummy ones itself, as they would be owned
|
||||
# by root. They don't matter: the ones at the root directory are mounted
|
||||
# instead.
|
||||
touch deps/package.json
|
||||
touch deps/package.json
|
||||
# Note that this is running with --user ${UID}:${GID} because the user olympia
|
||||
# would be uid 9500 regardless of host at this point (this is only fixed when
|
||||
# the container is up, through the command defined in docker-compose.yml),
|
||||
# which is wrong for local development.
|
||||
docker-compose run --rm --user ${UID}:${GID} web make update_deps
|
||||
docker-compose up -d
|
||||
docker-compose exec --user olympia web make initialize
|
||||
|
||||
%: ## This directs any other recipe (command) to the web container's make.
|
||||
docker-compose exec web make $(MAKECMDGOALS) ARGS=$(ARGS)
|
||||
docker-compose exec --user olympia web make $(MAKECMDGOALS) ARGS=$(ARGS)
|
||||
|
||||
# You probably want to put new commands in Makefile-docker, unless they operate
|
||||
# on multiple containers or are host-os specific.
|
||||
|
|
|
|||
|
|
@ -17,23 +17,48 @@ x-env-mapping: &env
|
|||
- UITEST_FXA_EMAIL
|
||||
- UITEST_FXA_PASSWORD=uitester
|
||||
- CIRCLECI=${CIRCLECI}
|
||||
- HISTFILE=/data/olympia/docker/artifacts/bash_history
|
||||
- HISTSIZE=50000
|
||||
- HISTIGNORE=ls:exit:"cd .."
|
||||
- HISTCONTROL=erasedups
|
||||
# Note: docker-compose uses the values exported from .env for GID/UID if
|
||||
# they exist. ./docker/fix_olympia_user.sh uses those variables to fix
|
||||
# the uid/gid of the user to match the host if necessary.
|
||||
- UID=${UID:-9500}
|
||||
- GID=${UID:-9500}
|
||||
|
||||
services:
|
||||
worker: &worker
|
||||
<<: *env
|
||||
image: addons/addons-server:latest
|
||||
image: mozilla/addons-server:latest
|
||||
# We drop down to a different user through supervisord, but starting as
|
||||
# root allows us to fix the ownership of files generated at image build
|
||||
# time through the ./docker/fix_olympia_user.sh script.
|
||||
user: root
|
||||
platform: linux/amd64
|
||||
command: supervisord -n -c /code/docker/supervisor-celery.conf
|
||||
user: olympia
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- |
|
||||
./docker/fix_olympia_user.sh
|
||||
supervisord -n -c /data/olympia/docker/supervisor-celery.conf
|
||||
volumes:
|
||||
- .:/code
|
||||
- .:/data/olympia
|
||||
- ./deps:/deps
|
||||
- ./package.json:/deps/package.json
|
||||
- ./package-lock.json:/deps/package-lock.json
|
||||
extra_hosts:
|
||||
- "olympia.test:127.0.0.1"
|
||||
|
||||
web:
|
||||
<<: *worker
|
||||
platform: linux/amd64
|
||||
command: supervisord -n -c /code/docker/supervisor.conf
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- |
|
||||
./docker/fix_olympia_user.sh
|
||||
supervisord -n -c /data/olympia/docker/supervisor.conf
|
||||
|
||||
nginx:
|
||||
image: nginx
|
||||
|
|
@ -97,9 +122,9 @@ services:
|
|||
autograph:
|
||||
image: mozilla/autograph:3.3.2
|
||||
platform: linux/amd64
|
||||
command: /go/bin/autograph -c /code/scripts/autograph_localdev_config.yaml
|
||||
command: /go/bin/autograph -c /data/olympia/scripts/autograph_localdev_config.yaml
|
||||
volumes:
|
||||
- .:/code
|
||||
- .:/data/olympia
|
||||
|
||||
addons-frontend:
|
||||
<<: *env
|
||||
|
|
|
|||
|
|
@ -0,0 +1,3 @@
|
|||
# Alter the uid/gid of the olympia user/group to match the host
|
||||
usermod -u ${UID} olympia
|
||||
groupmod -g ${GID} olympia
|
||||
|
|
@ -5,21 +5,21 @@ merge_slashes off;
|
|||
server {
|
||||
listen 80 default;
|
||||
|
||||
location /code/storage/files/ {
|
||||
location /data/olympia/storage/files/ {
|
||||
internal;
|
||||
# This matches where addons-server `docker-compose.yml` mounts
|
||||
# `./storage/addons/` - as `/srv/user-media/addons/`
|
||||
alias /srv/user-media/addons/;
|
||||
}
|
||||
|
||||
location /code/storage/guarded-addons/ {
|
||||
location /data/olympia/storage/guarded-addons/ {
|
||||
internal;
|
||||
# This matches where addons-server `docker-compose.yml` mounts
|
||||
# `./storage/guarded-addons/` - as `/srv/user-media/guarded-addons/`
|
||||
alias /srv/user-media/guarded-addons/;
|
||||
}
|
||||
|
||||
location /code/storage/sitemaps/ {
|
||||
location /data/olympia/storage/sitemaps/ {
|
||||
internal;
|
||||
# This matches where addons-server `docker-compose.yml` mounts
|
||||
# `./storage/sitemaps/` - as `/srv/user-media/sitemaps/`
|
||||
|
|
|
|||
|
|
@ -1,14 +1,15 @@
|
|||
[supervisord]
|
||||
logfile=/code/logs/supervisord-celery.log
|
||||
logfile=/data/olympia/logs/supervisord-celery.log
|
||||
logfile_maxbytes=1048576
|
||||
logfile_backups=1
|
||||
user=olympia
|
||||
|
||||
[program:olympia-worker]
|
||||
# Local env setup for celery. This is similar to prod, but with only two workers
|
||||
# Prod config for reference:
|
||||
# https://github.com/mozilla-services/cloudops-deployment/blob/master/projects/amo/puppet/modules/olympia/manifests/celery.pp
|
||||
command=watchmedo auto-restart --directory /code/src --pattern '*.py' --recursive -- celery -A olympia.amo.celery:app worker -E -c 2 --loglevel=INFO
|
||||
directory=/code
|
||||
command=watchmedo auto-restart --directory /data/olympia/src --pattern '*.py' --recursive -- celery -A olympia.amo.celery:app worker -E -c 2 --loglevel=INFO
|
||||
directory=/data/olympia
|
||||
stopasgroup=true
|
||||
autostart=true
|
||||
redirect_stderr=true
|
||||
|
|
|
|||
|
|
@ -1,11 +1,12 @@
|
|||
[supervisord]
|
||||
logfile=/code/logs/supervisord-olympia.log
|
||||
logfile=/data/olympia/logs/supervisord-olympia.log
|
||||
logfile_maxbytes=1048576
|
||||
logfile_backups=1
|
||||
user=olympia
|
||||
|
||||
[program:olympia]
|
||||
command=uwsgi --ini /code/docker/uwsgi.ini
|
||||
directory=/code
|
||||
command=uwsgi --ini /data/olympia/docker/uwsgi.ini
|
||||
directory=/data/olympia
|
||||
stopasgroup=true
|
||||
autostart=true
|
||||
redirect_stderr=true
|
||||
|
|
@ -19,7 +20,7 @@ priority=500
|
|||
autostart=true
|
||||
autorestart=false
|
||||
directory=/tmp
|
||||
command=watchmedo shell-command --pattern '*.py' --recursive --command='/usr/bin/touch /code/docker/artifacts/uwsgi-reload-monitor' /code/src
|
||||
command=watchmedo shell-command --pattern '*.py' --recursive --command='/usr/bin/touch /data/olympia/docker/artifacts/uwsgi-reload-monitor' /data/olympia/src
|
||||
priority=991
|
||||
|
||||
# The following sections enable supervisorctl.
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
# addons-server uwsgi config for local envs
|
||||
[uwsgi]
|
||||
base = /code
|
||||
base = /data/olympia
|
||||
chdir = %(base)
|
||||
module = olympia.wsgi:application
|
||||
|
||||
|
|
@ -10,8 +10,8 @@ master = true
|
|||
processes = 4
|
||||
vaccum = true
|
||||
socket = :8001
|
||||
uid = olympia
|
||||
gid = olympia
|
||||
uid = $(UID)
|
||||
gid = $(GID)
|
||||
memory-report = true
|
||||
enable-threads = true
|
||||
|
||||
|
|
@ -27,8 +27,7 @@ max-requests = 5000
|
|||
lazy-apps = true
|
||||
|
||||
# Open log file after we dropped privileges so that the file is being owned
|
||||
# by olympia:olympia and has proper permissions to be readable outside
|
||||
# of docker
|
||||
# by uid:gid and has proper permissions to be readable outside of docker
|
||||
logto2 = %(base)/logs/uwsgi-olympia.log
|
||||
|
||||
# Limit log file size to 10MB
|
||||
|
|
|
|||
|
|
@ -52,7 +52,7 @@ Setting up the containers
|
|||
|
||||
For more information see `switching docker containers`_.
|
||||
|
||||
Failure to do so will result in errors in later steps like ``make initialize``::
|
||||
Failure to do so will result in errors in later steps like::
|
||||
|
||||
ValueError: Unable to configure handler 'statsd': [Errno -2] Name or service not known
|
||||
Makefile-docker:71: recipe for target 'initialize_db' failed
|
||||
|
|
@ -68,17 +68,25 @@ on your host machine::
|
|||
cd addons-server
|
||||
# Download the containers
|
||||
docker-compose pull # Can take a while depending on your internet bandwidth.
|
||||
# Start up the containers
|
||||
make initialize_docker # Answer yes, and create your superuser when asked.
|
||||
# On Windows you can substitute `make initialize_docker` by the following commands:
|
||||
docker-compose run --rm --user olympia web make update_deps
|
||||
docker-compose up -d
|
||||
make initialize # Answer yes, and create your superuser when asked.
|
||||
# On Windows you can substitute `make initialize` for the command:
|
||||
docker-compose exec web make initialize
|
||||
docker-compose exec --user olympia web make initialize
|
||||
|
||||
.. note::
|
||||
|
||||
Docker requires the code checkout to exist within your home directory so
|
||||
that Docker can mount the source-code into the container.
|
||||
|
||||
Because the containers need to match the user/group permissions from your
|
||||
host machine, on Mac and Linux machines make sure to run ``make initialize_docker``
|
||||
once before running ``docker-compose up -d`` for the first time. That will
|
||||
create a ``.env`` file containing the user and group id the container needs
|
||||
to use to match your host permissions, and ensure dependencies are set up
|
||||
properly.
|
||||
|
||||
|
||||
Accessing the web server
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
|
|
@ -112,23 +120,20 @@ Run the tests using ``make``, *outside* of the Docker container::
|
|||
|
||||
make test
|
||||
# or
|
||||
docker-compose exec web pytest src/olympia/
|
||||
docker-compose exec --user olympia web pytest src/olympia/
|
||||
|
||||
You can run commands inside the Docker container by ``ssh``\ing into it using::
|
||||
|
||||
make shell
|
||||
# or
|
||||
docker-compose exec web bash
|
||||
docker-compose exec --user olympia web bash
|
||||
|
||||
Then to run the tests inside the Docker container you can run::
|
||||
|
||||
pytest
|
||||
|
||||
You can also run single commands from your host machine without opening a shell
|
||||
on each container. Here is an example of running the ``pytest`` command on the
|
||||
``web`` container::
|
||||
|
||||
docker-compose run web pytest
|
||||
on each container as described above.
|
||||
|
||||
If you'd like to use a python debugger to interactively
|
||||
debug Django view code, check out the :ref:`debugging` section.
|
||||
|
|
@ -143,7 +148,7 @@ debug Django view code, check out the :ref:`debugging` section.
|
|||
Updating your containers
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Any time you update Olympia (e.g., by running ``git pull``), you should make
|
||||
Any time you update addons-server (e.g., by running ``git pull``), you should make
|
||||
sure to update your Docker image and database with any new requirements or
|
||||
migrations::
|
||||
|
||||
|
|
@ -151,9 +156,11 @@ migrations::
|
|||
docker-compose pull
|
||||
docker-compose up -d
|
||||
make update_docker # Runs database migrations and rebuilds assets.
|
||||
# On Windows you can substitute `make update_docker` for the following two commands:
|
||||
docker-compose exec worker make update_deps
|
||||
docker-compose exec web make update
|
||||
# On Windows you can substitute `make update_docker` for the following commands:
|
||||
docker-compose exec --user olympia worker make update_deps
|
||||
docker-compose exec --user olympia web make update
|
||||
docker-compose restart web
|
||||
docker-compose restart worker
|
||||
|
||||
Gotchas!
|
||||
~~~~~~~~
|
||||
|
|
@ -174,13 +181,13 @@ Another way to find out what's wrong is to run ``docker-compose logs``.
|
|||
Getting "Programming error [table] doesn't exist"?
|
||||
--------------------------------------------------
|
||||
|
||||
Make sure you've run the ``make initialize`` step as detailed in
|
||||
Make sure you've run the ``make initialize_docker`` step as detailed in
|
||||
the initial setup instructions.
|
||||
|
||||
|
||||
ConnectionError during initialize (elasticsearch container fails to start)
|
||||
---------------------------------------------------------------------------------
|
||||
When running ``make initialize`` without a working elasticsearch container,
|
||||
When running ``make initialize_docker`` without a working elasticsearch container,
|
||||
you'll get a ConnectionError. Check the logs with ``docker-compose logs``.
|
||||
If elasticsearch is complaining about ``vm.max_map_count``, run this command on your computer
|
||||
or your docker-machine VM:
|
||||
|
|
@ -194,7 +201,7 @@ Connection to elasticsearch timed out (elasticsearch container exits with code 1
|
|||
------------------------------------------------------------------------------------
|
||||
|
||||
``docker-compose up -d`` brings up all containers, but running
|
||||
``make initialize`` causes the elasticsearch container to go down. Running
|
||||
``make initialize_docker`` causes the elasticsearch container to go down. Running
|
||||
``docker-compose ps`` shows ``Exited (137)`` against it.
|
||||
|
||||
Update default settings in Docker Desktop - we suggest increasing RAM limit to at least 4 GB in the Resources/Advanced section and click on "Apply and Restart".
|
||||
|
|
|
|||
|
|
@ -83,3 +83,38 @@ asttokens==2.2.1 \
|
|||
pure-eval==0.2.2 \
|
||||
--hash=sha256:01eaab343580944bc56080ebe0a674b39ec44a945e6d09ba7db3cb8cec289350 \
|
||||
--hash=sha256:2b45320af6dfaa1750f543d714b6d1c520a1688dec6fd24d339063ce0aaa9ac3
|
||||
# PyYAML is required by watchdog[watchmedo]
|
||||
PyYAML==6.0 \
|
||||
--hash=sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53 \
|
||||
--hash=sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c \
|
||||
--hash=sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc \
|
||||
--hash=sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b \
|
||||
--hash=sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5 \
|
||||
--hash=sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513 \
|
||||
--hash=sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a \
|
||||
--hash=sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86 \
|
||||
--hash=sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f \
|
||||
--hash=sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92 \
|
||||
--hash=sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4 \
|
||||
--hash=sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293 \
|
||||
--hash=sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57 \
|
||||
--hash=sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c \
|
||||
--hash=sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0 \
|
||||
--hash=sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4 \
|
||||
--hash=sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9 \
|
||||
--hash=sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737 \
|
||||
--hash=sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d \
|
||||
--hash=sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b \
|
||||
--hash=sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba \
|
||||
--hash=sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34 \
|
||||
--hash=sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287 \
|
||||
--hash=sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78 \
|
||||
--hash=sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07 \
|
||||
--hash=sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b \
|
||||
--hash=sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174 \
|
||||
--hash=sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803 \
|
||||
--hash=sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3 \
|
||||
--hash=sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0 \
|
||||
--hash=sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb \
|
||||
--hash=sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c \
|
||||
--hash=sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
#!/usr/bin/env python
|
||||
import uuid
|
||||
|
||||
# Generate build id for Dockerfile.deploy
|
||||
# Generate build id for docker image.
|
||||
print('BUILD_ID = "%s"' % uuid.uuid4())
|
||||
|
|
|
|||
|
|
@ -63,7 +63,6 @@ CEF_PRODUCT = STATSD_PREFIX
|
|||
|
||||
NEW_FEATURES = True
|
||||
|
||||
ADDONS_LINTER_BIN = 'node_modules/.bin/addons-linter'
|
||||
# --enable-background-service-worker linter flag value
|
||||
ADDONS_LINTER_ENABLE_SERVICE_WORKER = True
|
||||
|
||||
|
|
|
|||
|
|
@ -50,8 +50,6 @@ CEF_PRODUCT = STATSD_PREFIX
|
|||
|
||||
NEW_FEATURES = True
|
||||
|
||||
ADDONS_LINTER_BIN = 'node_modules/.bin/addons-linter'
|
||||
|
||||
ES_DEFAULT_NUM_SHARDS = 10
|
||||
|
||||
RECOMMENDATION_ENGINE_URL = env(
|
||||
|
|
|
|||
|
|
@ -60,8 +60,6 @@ CEF_PRODUCT = STATSD_PREFIX
|
|||
|
||||
NEW_FEATURES = True
|
||||
|
||||
ADDONS_LINTER_BIN = 'node_modules/.bin/addons-linter'
|
||||
|
||||
ALLOW_SELF_REVIEWS = True
|
||||
|
||||
FXA_CONFIG = {
|
||||
|
|
|
|||
|
|
@ -39,7 +39,7 @@ ALLOWED_HOSTS = [
|
|||
INTERNAL_ROUTES_ALLOWED = env('INTERNAL_ROUTES_ALLOWED', default=False)
|
||||
|
||||
try:
|
||||
# If we have a build id (it should be generated in Dockerfile.deploy),
|
||||
# If we have a build id (it should be generated when building the image),
|
||||
# we'll grab it here and add it to our CACHE_KEY_PREFIX. This will let us
|
||||
# not have to flush memcache during updates and it will let us preload
|
||||
# data into it before a production push.
|
||||
|
|
@ -75,13 +75,15 @@ SILENCED_SYSTEM_CHECKS = (
|
|||
# LESS CSS OPTIONS (Debug only).
|
||||
LESS_PREPROCESS = True # Compile LESS with Node, rather than client-side JS?
|
||||
LESS_LIVE_REFRESH = False # Refresh the CSS on save?
|
||||
LESS_BIN = env('LESS_BIN', default='node_modules/less/bin/lessc')
|
||||
LESS_BIN = env('LESS_BIN', default='/deps/node_modules/less/bin/lessc')
|
||||
|
||||
# Path to cleancss (our CSS minifier).
|
||||
CLEANCSS_BIN = env('CLEANCSS_BIN', default='node_modules/clean-css-cli/bin/cleancss')
|
||||
CLEANCSS_BIN = env(
|
||||
'CLEANCSS_BIN', default='/deps/node_modules/clean-css-cli/bin/cleancss'
|
||||
)
|
||||
|
||||
# Path to our JS minifier.
|
||||
JS_MINIFIER_BIN = env('JS_MINIFIER_BIN', default='node_modules/terser/bin/terser')
|
||||
JS_MINIFIER_BIN = env('JS_MINIFIER_BIN', default='/deps/node_modules/terser/bin/terser')
|
||||
|
||||
# rsvg-convert is used to save our svg static theme previews to png
|
||||
RSVG_CONVERT_BIN = env('RSVG_CONVERT_BIN', default='rsvg-convert')
|
||||
|
|
@ -91,7 +93,7 @@ PNGCRUSH_BIN = env('PNGCRUSH_BIN', default='pngcrush')
|
|||
|
||||
# Path to our addons-linter binary
|
||||
ADDONS_LINTER_BIN = env(
|
||||
'ADDONS_LINTER_BIN', default='node_modules/addons-linter/bin/addons-linter'
|
||||
'ADDONS_LINTER_BIN', default='/deps/node_modules/addons-linter/bin/addons-linter'
|
||||
)
|
||||
# --enable-background-service-worker linter flag value
|
||||
ADDONS_LINTER_ENABLE_SERVICE_WORKER = False
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче