Efficiently cache docker build (#21914)

* local docker build command

* Move copy . to after installing packages (prevents always re-installing)

* Enable buildkit cache mount to cache pip/npm dependencies
- cache npm/pip dependencies across builds
- add clear logging to npm install for cache hit/miss
- move copy npm files to update_assets

* Split build into stages
- splitting to stages offers better caching of layers and more efficient use of disk/time
- The initial gains will be with better caching of locale compilation, but will expand as we can move more logic from the final stage

TODO: split up apt depedencies to specific stages, move update_assets to pre-final stage to prevent re-running on every build with a * file change.

* Document the docker file and build process

* update docs

.dockerignore

@@ -6,3 +6,7 @@ deps/
 node_modules/
 storage/
 logs/*
+
+# Don't include the docker cache in the build context or you will get memory leaks
+docker-cache/
+docker-cache-new/

.gitignore

@@ -59,3 +59,7 @@ private/
 !docker-compose.private.yml
 !private/README.md
 !deps/.keep
+
+# Local cache directory for docker layer/build cache
+docker-cache/
+docker-cache-new/

Dockerfile

@@ -1,4 +1,8 @@
-FROM python:3.10-slim-buster
+##### Important information for maintaining this Dockerfile ########################################
+# Read the docs/topics/development/docker.md file for more information about this Dockerfile.
+####################################################################################################
+
+FROM python:3.10-slim-buster as base
 
 # Should change it to use ARG instead of ENV for OLYMPIA_UID/OLYMPIA_GID
 # once the jenkins server is upgraded to support docker >= v1.9.0
@@ -63,15 +67,16 @@ ENV HOME /data/olympia
 # The pipeline v2 standard requires the existence of /app/version.json
 # inside the docker image, thus it's copied there.
 COPY version.json /app/version.json
-COPY --chown=olympia:olympia . ${HOME}
 WORKDIR ${HOME}
+# give olympia access to the HOME directory
+RUN chown -R olympia:olympia ${HOME}
 
 # Set up directories and links that we'll need later, before switching to the
 # olympia user.
 RUN mkdir /deps \
-    && chown olympia:olympia /deps \
+    && chown -R olympia:olympia /deps \
     && rm -rf ${HOME}/src/olympia.egg-info \
-    && mkdir ${HOME}/src/olympia.egg-info \
+    && mkdir -p ${HOME}/src/olympia.egg-info \
     && chown olympia:olympia ${HOME}/src/olympia.egg-info \
     # For backwards-compatibility purposes, set up links to uwsgi. Note that
     # the target doesn't exist yet at this point, but it will later.
@@ -88,18 +93,51 @@ ENV PIP_SRC=/deps/src/
 ENV PYTHONUSERBASE=/deps
 ENV PATH $PYTHONUSERBASE/bin:$PATH
 ENV NPM_CONFIG_PREFIX=/deps/
-RUN ln -s ${HOME}/package.json /deps/package.json \
+ENV NPM_CACHE_DIR=/deps/cache/npm
+ENV NPM_DEBUG=true
+
+RUN \
+    # Files needed to run the make command
+    --mount=type=bind,source=Makefile,target=${HOME}/Makefile \
+    --mount=type=bind,source=Makefile-docker,target=${HOME}/Makefile-docker \
+    # Files required to install pip dependencies
+    --mount=type=bind,source=setup.py,target=${HOME}/setup.py \
+    --mount=type=bind,source=./requirements,target=${HOME}/requirements \
+    # Files required to install npm dependencies
+    --mount=type=bind,source=package.json,target=${HOME}/package.json \
+    --mount=type=bind,source=package-lock.json,target=${HOME}/package-lock.json \
+    # Mounts for caching dependencies
+    --mount=type=cache,target=${PIP_CACHE_DIR},uid=${OLYMPIA_UID},gid=${OLYMPIA_GID} \
+    --mount=type=cache,target=${NPM_CACHE_DIR},uid=${OLYMPIA_UID},gid=${OLYMPIA_GID} \
+    # Command to install dependencies
+    ln -s ${HOME}/package.json /deps/package.json \
     && ln -s ${HOME}/package-lock.json /deps/package-lock.json \
     && make update_deps_prod
 
-WORKDIR ${HOME}
+FROM base as builder
+ARG LOCALE_DIR=${HOME}/locale
+# Compile locales
+# Copy the locale files from the host so it is writable by the olympia user
+COPY --chown=olympia:olympia locale ${LOCALE_DIR}
+# Copy the executable individually to improve the cache validity
+RUN --mount=type=bind,source=locale/compile-mo.sh,target=${HOME}/compile-mo.sh \
+    ${HOME}/compile-mo.sh ${LOCALE_DIR}
 
-# Build locales, assets, build id.
-RUN echo "from olympia.lib.settings_base import *\n" \
-> settings_local.py && DJANGO_SETTINGS_MODULE='settings_local' locale/compile-mo.sh locale \
-    && DJANGO_SETTINGS_MODULE='settings_local' python manage.py compress_assets \
-    && DJANGO_SETTINGS_MODULE='settings_local' python manage.py generate_jsi18n_files \
-    && DJANGO_SETTINGS_MODULE='settings_local' python manage.py collectstatic --noinput \
+FROM base as final
+# Only copy our source files after we have installed all dependencies
+# TODO: split this into a separate stage to make even blazingly faster
+WORKDIR ${HOME}
+# Copy compiled locales from builder
+COPY --from=builder --chown=olympia:olympia ${HOME}/locale ${HOME}/locale
+# Copy the rest of the source files from the host
+COPY --chown=olympia:olympia . ${HOME}
+
+# Finalize the build
+# TODO: We should move update_assets to the `builder` stage once we can efficiently
+# Run that command without having to copy the whole source code
+# This will shave nearly 1 minute off the best case build time
+RUN echo "from olympia.lib.settings_base import *" > settings_local.py \
+    && DJANGO_SETTINGS_MODULE="settings_local" make update_assets \
     && npm prune --production \
     && ./scripts/generate_build.py > build.py \
     && rm -f settings_local.py settings_local.pyc

Makefile-docker

@@ -17,6 +17,14 @@ ifneq ($(NPM_CONFIG_PREFIX),)
 	NPM_ARGS := --prefix $(NPM_CONFIG_PREFIX)
 endif
 
+ifneq ($(NPM_CACHE_DIR),)
+	NPM_ARGS := $(NPM_ARGS) --cache $(NPM_CACHE_DIR)
+endif
+
+ifneq ($(NPM_DEBUG),)
+	NPM_ARGS := $(NPM_ARGS) --loglevel verbose
+endif
+
 NODE_MODULES := $(NPM_CONFIG_PREFIX)node_modules/
 STATIC_CSS := static/css/node_lib/
 STATIC_JS := static/js/node_lib/
@@ -75,7 +83,6 @@ populate_data: ## populate a new database
 	# Now that addons have been generated, reindex.
 	$(PYTHON_COMMAND) manage.py reindex --force --noinput
 
-.PHONY: update_deps_base
 update_deps_base: ## update the python and node dependencies
 	# Work arounds "Multiple .dist-info directories" issue.
 	rm -rf /deps/build/*
@@ -84,18 +91,14 @@ update_deps_base: ## update the python and node dependencies
 	# pep 517 mode (the default) breaks editable install in our project. https://github.com/mozilla/addons-server/issues/16144
 	$(PIP_COMMAND) install --no-use-pep517 -e .
 
-	npm install $(NPM_ARGS)
-	for dest in $(NODE_LIBS_CSS) ; do cp $(NODE_MODULES)$$dest $(STATIC_CSS) ; done
-	for dest in $(NODE_LIBS_JS) ; do cp $(NODE_MODULES)$$dest $(STATIC_JS) ; done
-	for dest in $(NODE_LIBS_JQUERY_UI) ; do cp $(NODE_MODULES)$$dest $(STATIC_JQUERY_UI) ; done
-
 .PHONY: update_deps
 update_deps: update_deps_base ## update the python and node dependencies for development
 	$(PIP_COMMAND) install --progress-bar=off --no-deps --exists-action=w -r requirements/dev.txt
+	npm install $(NPM_ARGS)
 
 .PHONY: update_deps_prod
 update_deps_prod: update_deps_base ## update the python and node dependencies for production
-	npm prune --omit=dev
+	npm ci $(NPM_ARGS)
 
 .PHONY: update_db
 update_db: ## run the database migrations
@@ -103,6 +106,11 @@ update_db: ## run the database migrations
 
 .PHONY: update_assets
 update_assets:
+	# Copy files required in compress_assets to the static folder
+	mkdir -p $(STATIC_CSS) $(STATIC_JS) $(STATIC_JQUERY_UI)
+	for dest in $(NODE_LIBS_CSS) ; do cp $(NODE_MODULES)$$dest $(STATIC_CSS) ; done
+	for dest in $(NODE_LIBS_JS) ; do cp $(NODE_MODULES)$$dest $(STATIC_JS) ; done
+	for dest in $(NODE_LIBS_JQUERY_UI) ; do cp $(NODE_MODULES)$$dest $(STATIC_JQUERY_UI) ; done
 	# If changing this here, make sure to adapt tests in amo/test_commands.py
 	$(PYTHON_COMMAND) manage.py compress_assets
 	$(PYTHON_COMMAND) manage.py collectstatic --noinput

Makefile-os

@@ -3,6 +3,13 @@ GID := $(shell id -g)
 export UID
 export GID
 
+export DOCKER_BUILDER=container
+
+TAG := addons-server-test
+PLATFORM := linux/amd64
+PROGRESS := auto
+DOCKER_CACHE_DIR := docker-cache
+
 .PHONY: help_redirect
 help_redirect:
 	@$(MAKE) help --no-print-directory
@@ -33,6 +40,25 @@ rootshell: ## connect to a running addons-server docker shell with root user
 create_env_file:
 	echo "UID=${UID}\nGID=${GID}" > .env
 
+.PHONY: create_docker_builder
+create_docker_builder: ## Create a custom builder for buildkit to efficiently build local images
+	docker buildx use $(DOCKER_BUILDER) 2>/dev/null || docker buildx create \
+		--name $(DOCKER_BUILDER) \
+		--driver=docker-container
+
+.PHONY: build_docker_image
+build_docker_image: create_docker_builder ## Build the docker image
+	DOCKER_BUILDKIT=1 docker build \
+		-t $(TAG) \
+		--load \
+		--platform $(PLATFORM) \
+		--progress=$(PROGRESS) \
+		--cache-to=type=local,dest=$(DOCKER_CACHE_DIR)-new \
+		--cache-from=type=local,src=$(DOCKER_CACHE_DIR),mode=max \
+		--builder=$(DOCKER_BUILDER) .
+	rm -rf $(DOCKER_CACHE_DIR)
+	mv $(DOCKER_CACHE_DIR)-new $(DOCKER_CACHE_DIR)
+
 .PHONY: initialize_docker
 initialize_docker: create_env_file
 # Run a fresh container from the base image to install deps. Since /deps is

docs/topics/development/dependencies.md

@@ -61,3 +61,9 @@ make -f Makefile-docker update_deps
 ```
 
 This is used in github actions for example that do not need a full container to run.
+
+> Note: If you are adding a new dependency, make sure to update static assets imported from the new versions.
+
+```bash
+make update_assets
+```

docs/topics/development/docker.md

@@ -0,0 +1,56 @@
+# Docker
+
+## The Dockerfile
+
+Our Dockerfile is used in both production and development environments, however it is not always and not entirely used in CI (for now at least).
+
+The Dockerfile builds addons-server and runs using docker-compose by specifying the latest image pushed to dockerhub. Keep in mind during local development you are likely not running the current image in your git repository but the latest push to master in github.
+
+### Best Practices for the Dockerfile
+
+- Use as few instructions as possible
+- Split long running tasks into distinct stages to improve caching/concurrency
+- Prefer --mount=type=bind over COPY for files that are needed for a single command
+
+  > bind mounts files as root/docker user, so run the stage from base and chown them to olympia.
+  > bind mounts do not persist data, so if you modify any files, they will **not** be in the final layer.
+
+- If you do use COPY for files that are executed, prefer copying individual files over directories.
+
+  > The larger the directory, the more likely it is to have false cache hits.
+  > Link: <https://github.com/moby/moby/issues/33107>
+
+- Use --mount=type=cache for caching caches npm/pip etc.
+
+  > cache mounts are not persisted in CI due to an existing bug in buildkit. Link: <https://github.com/moby/buildkit/issues/1512>
+
+- Delay copying source files until the end of the Dockerfile to imrove cache validity
+
+## Building locally
+
+To build the Dockerfile locally, run the following command:
+
+```bash
+make build_docker_image
+```
+
+This will build the Dockerfile locally with buildkit and tag it as `addons-server-test` by default. You can control several parameters including the tag and platform. This can be very useful if you are testing a new image or want to test a new platform.
+
+We utilize buildkit layer and mount caching to build extremely efficiently. There are more improvements we can make.
+
+## Clearing cache
+
+Because we use a custom builder to take full advantage of buildkit mount caching clearing your cache means clearing
+the specific builder cache we use, not the docker cache.
+
+Do:
+
+```bash
+docker builder prune
+```
+
+Don't do:
+
+```bash
+docker system prune
+```

docs/topics/development/index.rst

@@ -8,6 +8,7 @@ Development
    tests
    debugging
    dependencies
+   docker
    error_pages
    testing
    style