Optimize multi stage docker build (#22195)
* Add checks to verify built docker image TMP: rename dependencies.txt * Remove unused and uninstalled package * More efficient multi stage docker buildTMP: reorganize docker more efficiently * Organize dockerfile - split base and olympia stages - use heredocs for multiline RUN steps - move file copy to source layer - merge run steps where possible * TMP: fix invalid bash syntax in dockerfile * TMP: add comments to dockerfile
This commit is contained in:
Kevin Meinhardt
GitHub
Родитель
96b28d2c86
Коммит
adc37153dd
206
Dockerfile
206
Dockerfile
|
|
@ -2,91 +2,70 @@
|
|||
# Read the docs/topics/development/docker.md file for more information about this Dockerfile.
|
||||
####################################################################################################
|
||||
|
||||
FROM python:3.10-slim-bookworm as base
|
||||
FROM python:3.10-slim-bookworm as olympia
|
||||
|
||||
# Should change it to use ARG instead of ENV for OLYMPIA_UID/OLYMPIA_GID
|
||||
# once the jenkins server is upgraded to support docker >= v1.9.0
|
||||
ENV OLYMPIA_UID=9500 \
|
||||
OLYMPIA_GID=9500
|
||||
RUN groupadd -g ${OLYMPIA_GID} olympia && useradd -u ${OLYMPIA_UID} -g ${OLYMPIA_GID} -s /sbin/nologin -d /data/olympia olympia
|
||||
ENV OLYMPIA_UID=9500 OLYMPIA_GID=9500
|
||||
|
||||
# Add support for https apt repos and gpg signed repos
|
||||
RUN apt-get update && apt-get install -y \
|
||||
apt-transport-https \
|
||||
gnupg2 \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
RUN <<EOF
|
||||
groupadd -g ${OLYMPIA_GID} olympia
|
||||
useradd -u ${OLYMPIA_UID} -g ${OLYMPIA_GID} -s /sbin/nologin -d /data/olympia olympia
|
||||
EOF
|
||||
|
||||
# give olympia access to the HOME directory
|
||||
ENV HOME /data/olympia
|
||||
WORKDIR ${HOME}
|
||||
RUN chown -R olympia:olympia ${HOME}
|
||||
|
||||
FROM olympia as base
|
||||
# Add keys and repos for node and mysql
|
||||
COPY docker/*.gpg.key /etc/pki/gpg/
|
||||
RUN APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=DontWarn \
|
||||
apt-key add /etc/pki/gpg/nodesource.gpg.key \
|
||||
&& APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=DontWarn \
|
||||
apt-key add /etc/pki/gpg/mysql.gpg.key
|
||||
# TODO: replace this with a bind mount on the RUN command
|
||||
COPY docker/*.gpg.asc /etc/apt/trusted.gpg.d/
|
||||
COPY docker/*.list /etc/apt/sources.list.d/
|
||||
|
||||
# Allow scripts to detect we're running in our own container and install
|
||||
# packages.
|
||||
RUN touch /addons-server-docker-container \
|
||||
&& apt-get update && apt-get install -y \
|
||||
# General (dev-) dependencies
|
||||
bash-completion \
|
||||
build-essential \
|
||||
curl \
|
||||
libjpeg-dev \
|
||||
libsasl2-dev \
|
||||
libxml2-dev \
|
||||
libxslt-dev \
|
||||
locales \
|
||||
zlib1g-dev \
|
||||
libffi-dev \
|
||||
libssl-dev \
|
||||
nodejs \
|
||||
# Git, because we're using git-checkout dependencies
|
||||
git \
|
||||
# Dependencies for mysql-python (from mysql apt repo, not debian)
|
||||
pkg-config \
|
||||
mysql-client \
|
||||
libmysqlclient-dev \
|
||||
swig \
|
||||
gettext \
|
||||
# Use rsvg-convert to render our static theme previews
|
||||
librsvg2-bin \
|
||||
# Use pngcrush to optimize the PNGs uploaded by developers
|
||||
pngcrush \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
RUN <<EOF
|
||||
# Add support for https apt repos and gpg signed repos
|
||||
apt-get update
|
||||
apt-get install -y apt-transport-https gnupg2
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
EOF
|
||||
|
||||
# Add our custom mime types (required for for ts/json/md files)
|
||||
ADD docker/etc/mime.types /etc/mime.types
|
||||
RUN --mount=type=bind,source=docker/debian_packages.txt,target=/debian_packages.txt \
|
||||
/bin/bash <<EOF
|
||||
# Allow scripts to detect we're running in our own container
|
||||
touch /addons-server-docker-container
|
||||
# install packages.
|
||||
apt-get update
|
||||
xargs apt-get -y install < <(grep -v '^#' /debian_packages.txt)
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
EOF
|
||||
|
||||
# Compile required locale
|
||||
RUN localedef -i en_US -f UTF-8 en_US.UTF-8
|
||||
ENV LANG en_US.UTF-8
|
||||
ENV LC_ALL en_US.UTF-8
|
||||
|
||||
ENV HOME /data/olympia
|
||||
RUN <<EOF
|
||||
# Create directory for dependencies
|
||||
mkdir /deps
|
||||
chown -R olympia:olympia /deps
|
||||
|
||||
# version.json is overwritten by CircleCI (see circle.yml).
|
||||
# The pipeline v2 standard requires the existence of /app/version.json
|
||||
# inside the docker image, thus it's copied there.
|
||||
COPY version.json /app/version.json
|
||||
WORKDIR ${HOME}
|
||||
# give olympia access to the HOME directory
|
||||
RUN chown -R olympia:olympia ${HOME}
|
||||
# Remove any existing egg info directory and create a new one
|
||||
rm -rf ${HOME}/src/olympia.egg-info
|
||||
mkdir -p ${HOME}/src/olympia.egg-info
|
||||
chown olympia:olympia ${HOME}/src/olympia.egg-info
|
||||
|
||||
# Set up directories and links that we'll need later, before switching to the
|
||||
# olympia user.
|
||||
RUN mkdir /deps \
|
||||
&& chown -R olympia:olympia /deps \
|
||||
&& rm -rf ${HOME}/src/olympia.egg-info \
|
||||
&& mkdir -p ${HOME}/src/olympia.egg-info \
|
||||
&& chown olympia:olympia ${HOME}/src/olympia.egg-info \
|
||||
# For backwards-compatibility purposes, set up links to uwsgi. Note that
|
||||
# the target doesn't exist yet at this point, but it will later.
|
||||
&& ln -s /deps/bin/uwsgi /usr/bin/uwsgi \
|
||||
&& ln -s /usr/bin/uwsgi /usr/sbin/uwsgi
|
||||
# For backwards-compatibility purposes, set up links to uwsgi. Note that
|
||||
# the target doesn't exist yet at this point, but it will later.
|
||||
ln -s /deps/bin/uwsgi /usr/bin/uwsgi
|
||||
ln -s /usr/bin/uwsgi /usr/sbin/uwsgi
|
||||
|
||||
# link to the package*.json at ${HOME} so npm can install in /deps
|
||||
ln -s ${HOME}/package.json /deps/package.json
|
||||
ln -s ${HOME}/package-lock.json /deps/package-lock.json
|
||||
EOF
|
||||
|
||||
USER olympia:olympia
|
||||
|
||||
# Install all dependencies, and add symlink for old uwsgi binary paths
|
||||
ENV PIP_USER=true
|
||||
ENV PIP_BUILD=/deps/build/
|
||||
ENV PIP_CACHE_DIR=/deps/cache/
|
||||
|
|
@ -97,13 +76,27 @@ ENV NPM_CONFIG_PREFIX=/deps/
|
|||
ENV NPM_CACHE_DIR=/deps/cache/npm
|
||||
ENV NPM_DEBUG=true
|
||||
|
||||
# All we need in "base" is pip to be installed
|
||||
#this let's other layers install packages using the correct version.
|
||||
RUN \
|
||||
# Files needed to run the make command
|
||||
--mount=type=bind,source=Makefile,target=${HOME}/Makefile \
|
||||
--mount=type=bind,source=Makefile-docker,target=${HOME}/Makefile-docker \
|
||||
# Files required to install pip dependencies
|
||||
--mount=type=bind,source=setup.py,target=${HOME}/setup.py \
|
||||
--mount=type=bind,source=./requirements,target=${HOME}/requirements \
|
||||
--mount=type=bind,source=./requirements/pip.txt,target=${HOME}/requirements/pip.txt \
|
||||
--mount=type=cache,target=${PIP_CACHE_DIR},uid=${OLYMPIA_UID},gid=${OLYMPIA_UID} \
|
||||
# Command to install dependencies
|
||||
make -f Makefile-docker update_deps_pip
|
||||
|
||||
# Define production dependencies as a single layer
|
||||
# let's the rest of the stages inherit prod dependencies
|
||||
# and makes copying the /deps dir to the final layer easy.
|
||||
FROM base as pip_production
|
||||
|
||||
RUN \
|
||||
# Files needed to run the make command
|
||||
--mount=type=bind,source=Makefile-docker,target=${HOME}/Makefile-docker \
|
||||
# Files required to install pip dependencies
|
||||
--mount=type=bind,source=./requirements/prod.txt,target=${HOME}/requirements/prod.txt \
|
||||
# Files required to install npm dependencies
|
||||
--mount=type=bind,source=package.json,target=${HOME}/package.json \
|
||||
--mount=type=bind,source=package-lock.json,target=${HOME}/package-lock.json \
|
||||
|
|
@ -111,11 +104,9 @@ RUN \
|
|||
--mount=type=cache,target=${PIP_CACHE_DIR},uid=${OLYMPIA_UID},gid=${OLYMPIA_GID} \
|
||||
--mount=type=cache,target=${NPM_CACHE_DIR},uid=${OLYMPIA_UID},gid=${OLYMPIA_GID} \
|
||||
# Command to install dependencies
|
||||
ln -s ${HOME}/package.json /deps/package.json \
|
||||
&& ln -s ${HOME}/package-lock.json /deps/package-lock.json \
|
||||
&& make update_deps_prod
|
||||
make -f Makefile-docker update_deps_production
|
||||
|
||||
FROM base as locales
|
||||
FROM pip_production as locales
|
||||
ARG LOCALE_DIR=${HOME}/locale
|
||||
# Compile locales
|
||||
# Copy the locale files from the host so it is writable by the olympia user
|
||||
|
|
@ -127,21 +118,60 @@ RUN \
|
|||
--mount=type=bind,source=requirements/locale.txt,target=${HOME}/requirements/locale.txt \
|
||||
make -f Makefile-docker compile_locales
|
||||
|
||||
FROM base as final
|
||||
# Only copy our source files after we have installed all dependencies
|
||||
# TODO: split this into a separate stage to make even blazingly faster
|
||||
WORKDIR ${HOME}
|
||||
# Copy compiled locales
|
||||
COPY --from=locales --chown=olympia:olympia ${HOME}/locale ${HOME}/locale
|
||||
# Copy the rest of the source files from the host
|
||||
COPY --chown=olympia:olympia . ${HOME}
|
||||
# More efficient caching by mounting the exact files we need
|
||||
# and copying only the static/ directory.
|
||||
FROM pip_production as assets
|
||||
|
||||
# TODO: This stage depends on `olympia` being installed.
|
||||
# We should decouple the logic from the `olympia` installation
|
||||
# So it can cache more efficiently
|
||||
RUN \
|
||||
# Files needed to run the make command
|
||||
--mount=type=bind,source=Makefile-docker,target=${HOME}/Makefile-docker \
|
||||
# Files required to install pip dependencies
|
||||
--mount=type=bind,source=setup.py,target=${HOME}/setup.py \
|
||||
--mount=type=bind,source=pyproject.toml,target=${HOME}/pyproject.toml \
|
||||
# Command to install dependencies
|
||||
make -f Makefile-docker update_deps_olympia
|
||||
|
||||
# TODO: only copy the files we need for compiling assets
|
||||
COPY --chown=olympia:olympia static/ ${HOME}/static/
|
||||
|
||||
# Finalize the build
|
||||
# TODO: We should move update_assets to the `builder` stage once we can efficiently
|
||||
# Run that command without having to copy the whole source code
|
||||
# This will shave nearly 1 minute off the best case build time
|
||||
RUN echo "from olympia.lib.settings_base import *" > settings_local.py \
|
||||
&& DJANGO_SETTINGS_MODULE="settings_local" make update_assets \
|
||||
&& make prune_deps \
|
||||
&& ./scripts/generate_build.py > build.py \
|
||||
&& rm -f settings_local.py settings_local.pyc
|
||||
RUN \
|
||||
--mount=type=bind,src=src,target=${HOME}/src \
|
||||
--mount=type=bind,src=Makefile-docker,target=${HOME}/Makefile-docker \
|
||||
--mount=type=bind,src=manage.py,target=${HOME}/manage.py \
|
||||
echo "from olympia.lib.settings_base import *" > settings_local.py \
|
||||
&& DJANGO_SETTINGS_MODULE="settings_local" make -f Makefile-docker update_assets
|
||||
|
||||
FROM base as sources
|
||||
|
||||
RUN --mount=type=bind,src=scripts/generate_build.py,target=/generate_build.py \
|
||||
/generate_build.py > build.py
|
||||
|
||||
# Add our custom mime types (required for for ts/json/md files)
|
||||
COPY docker/etc/mime.types /etc/mime.types
|
||||
# Copy the rest of the source files from the host
|
||||
COPY --chown=olympia:olympia . ${HOME}
|
||||
# Copy compiled locales from builder
|
||||
COPY --from=locales --chown=olympia:olympia ${HOME}/locale ${HOME}/locale
|
||||
# Copy assets from assets
|
||||
COPY --from=assets --chown=olympia:olympia ${HOME}/site-static ${HOME}/site-static
|
||||
|
||||
# version.json is overwritten by CircleCI (see circle.yml).
|
||||
# The pipeline v2 standard requires the existence of /app/version.json
|
||||
# inside the docker image, thus it's copied there.
|
||||
COPY version.json /app/version.json
|
||||
|
||||
FROM sources as production
|
||||
|
||||
# Copy dependencies from `pip_production`
|
||||
COPY --from=pip_production --chown=olympia:olympia /deps /deps
|
||||
|
||||
# We have to reinstall olympia after copying source
|
||||
# to ensure the installation syncs files in the src/ directory
|
||||
RUN make -f Makefile-docker update_deps_olympia
|
||||
|
|
|
|||
|
|
@ -53,10 +53,33 @@ jquery-ui/ui/widget.js \
|
|||
jquery-ui/ui/widgets/mouse.js \
|
||||
jquery-ui/ui/widgets/sortable.js
|
||||
|
||||
REQUIRED_FILES := \
|
||||
Makefile \
|
||||
Makefile-os \
|
||||
Makefile-docker \
|
||||
version.json \
|
||||
/deps/package.json \
|
||||
/deps/package-lock.json \
|
||||
/addons-server-docker-container \
|
||||
|
||||
.PHONY: help_redirect
|
||||
help_redirect:
|
||||
@$(MAKE) help --no-print-directory
|
||||
|
||||
.PHONY: check_debian_packages
|
||||
check_debian_packages: ## check the existence of multiple debian packages
|
||||
./scripts/check_debian_packages.sh
|
||||
|
||||
.PHONY: check_files
|
||||
check_files: ## check the existence of multiple files
|
||||
@for file in $(REQUIRED_FILES); do test -f "$$file" || (echo "$$file is missing." && exit 1); done
|
||||
@echo "All required files are present."
|
||||
|
||||
.PHONY: check_olympia_user
|
||||
check_olympia_user: ## check if the olympia user exists and is current user
|
||||
@if [ "$$(id -u olympia)" != "$$(id -u)" ]; then echo "The current user is not the olympia user."; exit 1; fi
|
||||
@echo "The current user is the olympia user."
|
||||
|
||||
.PHONY: check_django
|
||||
check_django: ## check if the django app is configured properly
|
||||
echo 'from olympia.lib.settings_base import *' > settings_local.py
|
||||
|
|
@ -64,7 +87,7 @@ check_django: ## check if the django app is configured properly
|
|||
rm settings_local.py
|
||||
|
||||
.PHONY: check
|
||||
check: check_django
|
||||
check: check_files check_olympia_user check_debian_packages check_django
|
||||
|
||||
.PHONY: initialize_db
|
||||
initialize_db: ## create a new database
|
||||
|
|
@ -92,22 +115,29 @@ populate_data: ## populate a new database
|
|||
# Now that addons have been generated, reindex.
|
||||
$(PYTHON_COMMAND) manage.py reindex --force --noinput
|
||||
|
||||
update_deps_base: ## update the python and node dependencies
|
||||
.PHONY: update_deps_pip
|
||||
update_deps_pip: ## Install pip
|
||||
# Work arounds "Multiple .dist-info directories" issue.
|
||||
rm -rf /deps/build/*
|
||||
$(PIP_COMMAND) install --progress-bar=off --no-deps --exists-action=w -r requirements/pip.txt
|
||||
$(PIP_COMMAND) install --progress-bar=off --no-deps --exists-action=w -r requirements/prod.txt
|
||||
# pep 517 mode (the default) breaks editable install in our project. https://github.com/mozilla/addons-server/issues/16144
|
||||
|
||||
.PHONY: update_deps_olympia
|
||||
update_deps_olympia: ## Install the olympia local package
|
||||
# pep 517 mode (the default) breaks editable install in our project. https://github.com/mozilla/addons-server/issues/16144
|
||||
$(PIP_COMMAND) install --no-use-pep517 -e .
|
||||
|
||||
.PHONY: update_deps
|
||||
update_deps: update_deps_base ## update the python and node dependencies for development
|
||||
.PHONY: update_deps_development
|
||||
update_deps_development: ## update the python and node dependencies for development
|
||||
$(PIP_COMMAND) install --progress-bar=off --no-deps --exists-action=w -r requirements/dev.txt
|
||||
npm install $(NPM_ARGS) --no-save
|
||||
npm install $(NPM_ARGS) --no-save --include=dev
|
||||
|
||||
.PHONY: update_deps_prod
|
||||
update_deps_prod: update_deps_base ## update the python and node dependencies for production
|
||||
npm ci $(NPM_ARGS)
|
||||
.PHONY: update_deps_production
|
||||
update_deps_production: ## update the python and node dependencies for production
|
||||
$(PIP_COMMAND) install --progress-bar=off --no-deps --exists-action=w -r requirements/prod.txt
|
||||
npm ci $(NPM_ARGS) --include=prod
|
||||
|
||||
.PHONY: update_deps
|
||||
update_deps: update_deps_pip update_deps_production update_deps_development update_deps_olympia## update the python and node dependencies
|
||||
|
||||
.PHONY: prune_deps
|
||||
prune_deps: ## remove unused dependencies
|
||||
|
|
|
|||
|
|
@ -0,0 +1,24 @@
|
|||
# General (dev-) dependencies
|
||||
bash-completion
|
||||
build-essential
|
||||
curl
|
||||
libjpeg-dev
|
||||
libsasl2-dev
|
||||
libxml2-dev
|
||||
locales
|
||||
zlib1g-dev
|
||||
libffi-dev
|
||||
libssl-dev
|
||||
nodejs
|
||||
# Git, because we're using git-checkout dependencies
|
||||
git
|
||||
# Dependencies for mysql-python (from mysql apt repo, not debian)
|
||||
pkg-config
|
||||
mysql-client
|
||||
libmysqlclient-dev
|
||||
swig
|
||||
gettext
|
||||
# Use rsvg-convert to render our static theme previews
|
||||
librsvg2-bin
|
||||
# Use pngcrush to optimize the PNGs uploaded by developers
|
||||
pngcrush
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -ue
|
||||
|
||||
while IFS= read -r package; do
|
||||
package=$(echo "$package" | tr -d '[:space:]') # Remove whitespace from package name
|
||||
if ! dpkg -s "$package" >/dev/null 2>&1; then
|
||||
echo "$package is missing."
|
||||
exit 1
|
||||
else
|
||||
echo "$package installed."
|
||||
fi
|
||||
done < <(grep -v '^#' docker/debian_packages.txt)
|
||||
|
||||
echo "All required packages are present."
|
||||
Загрузка…
Ссылка в новой задаче