Make "up" go Brrrrrrrr

data seed command synchronizes storage directory

Remove storage volume (reuse olympia volume)

Remove data_olympia (use anonymous volume)

Remove sources from development DOCKER_TARGET

Invert docker-compose.ci to docker-compose.dev (update defaults in setup script to support prod mode in ci)

.github/actions/build-docker/action.yml

@@ -10,9 +10,6 @@ inputs:
   version:
     required: true
     description: The image version to tag with
-  target:
-    required: true
-    description: The stage to target in the build
   push:
     required: false
     description: Push the image?
@@ -57,7 +54,7 @@ runs:
     - name: Create .env and version.json files
       shell: bash
       run: |
-        echo "DOCKER_TARGET=${{ inputs.target }}" >> $GITHUB_ENV
+        echo "DOCKER_TARGET=production" >> $GITHUB_ENV
         echo "DOCKER_VERSION=${{ steps.meta.outputs.version }}" >> $GITHUB_ENV
         echo "DOCKER_COMMIT=${{ steps.context.outputs.git_sha }}" >> $GITHUB_ENV
         echo "DOCKER_BUILD=${{ steps.context.outputs.git_build_url }}" >> $GITHUB_ENV

.github/actions/run-docker/action.yml

@@ -15,7 +15,7 @@ inputs:
   compose_file:
     description: 'The docker-compose file to use'
     required: false
-    default: 'docker-compose.yml:docker-compose.ci.yml'
+    default: 'docker-compose.yml'
   logs:
     description: 'Show logs'
     required: false

.github/workflows/_test.yml

@@ -41,36 +41,30 @@ jobs:
           -
             name: Needs Locale Compilation
             services: ''
-            compose_file: docker-compose.yml:docker-compose.ci.yml
             run: |
               make compile_locales
               make test_needs_locales_compilation
           -
             name: Static Assets
             services: ''
-            compose_file: docker-compose.yml:docker-compose.ci.yml
             run: make test_static_assets
           -
             name: Internal Routes
             services: ''
-            compose_file: docker-compose.yml:docker-compose.ci.yml
             run: make test_internal_routes_allowed
           -
             name: Elastic Search
             services: ''
-            compose_file: docker-compose.yml:docker-compose.ci.yml
             run: make test_es_tests
           -
             name: Codestyle
             services: web
-            compose_file: docker-compose.yml:docker-compose.ci.yml
             run: make lint-codestyle
           -
             name: Manage Check
             services: web nginx
-            compose_file: docker-compose.yml:docker-compose.ci.yml
             run: make check
-            data_backup_skip: true
+            data_backup_skip: false
     steps:
       - uses: actions/checkout@v4
       - name: Test (${{ matrix.name }})
@@ -79,6 +73,5 @@ jobs:
           version: ${{ inputs.version }}
           digest: ${{ inputs.digest }}
           services: ${{ matrix.services }}
-          compose_file: ${{ matrix.compose_file }}
           run: ${{ matrix.run }}
           data_backup_skip: ${{ matrix.data_backup_skip || 'true' }}

.github/workflows/_test_main.yml

@@ -88,7 +88,7 @@ jobs:
           services: ''
           digest: ${{ inputs.digest }}
           version: ${{ inputs.version }}
-          compose_file: docker-compose.yml
+          compose_file: docker-compose.yml:docker-compose.dev.yml
           run: |
             split="--splits ${{ needs.test_config.outputs.splits }}"
             group="--group ${{ matrix.group }}"

.github/workflows/ci.yml

@@ -72,7 +72,6 @@ jobs:
           registry: ${{ steps.docker_hub.outputs.registry }}
           image: ${{ steps.docker_hub.outputs.image }}
           version: ci-${{ needs.context.outputs.docker_version }}
-          target: development
           push: true
 
   test_make_docker_configuration:
@@ -179,7 +178,7 @@ jobs:
         with:
           digest: ${{ needs.build.outputs.digest }}
           version: ${{ needs.build.outputs.version }}
-          compose_file: docker-compose.yml
+          compose_file: docker-compose.yml:docker-compose.dev.yml
           run: |
             make docs
 
@@ -230,7 +229,7 @@ jobs:
         with:
           digest: ${{ needs.build.outputs.digest }}
           version: ${{ needs.build.outputs.version }}
-          compose_file: docker-compose.yml
+          compose_file: docker-compose.yml:docker-compose.dev.yml
           run: make extract_locales
 
       - name: Push Locales
@@ -297,7 +296,6 @@ jobs:
           registry: ${{ steps.docker_hub.outputs.registry }}
           image: ${{ steps.docker_hub.outputs.image }}
           version: ${{ needs.context.outputs.docker_version }}
-          target: production
           push: true
 
   push_gar:
@@ -329,5 +327,4 @@ jobs:
           registry: ${{ steps.docker_gar.outputs.registry }}
           image: ${{ steps.docker_gar.outputs.image }}
           version: ${{ needs.context.outputs.docker_version }}
-          target: production
           push: true

.gitignore

@@ -53,6 +53,6 @@ tmp/*
 # End of .gitignore. Please keep this in sync with the top section of .dockerignore
 
 # do not ignore the following files
-!docker-compose.development.yml
+!docker-compose.dev.yml
 !docker-compose.private.yml
 !private/README.md

Dockerfile

@@ -100,6 +100,10 @@ EOF
 ARG DOCKER_TARGET
 ENV DOCKER_TARGET=${DOCKER_TARGET}
 
+# Add our custom mime types (required for for ts/json/md files)
+COPY docker/etc/mime.types /etc/mime.types
+
+
 # Define production dependencies as a single layer
 # let's the rest of the stages inherit prod dependencies
 # and makes copying the /deps dir to the final layer easy.
@@ -119,7 +123,10 @@ ${PIP_COMMAND} install --progress-bar=off --no-deps --exists-action=w -r require
 npm ci ${NPM_ARGS} --include=prod
 EOF
 
-FROM base AS pip_development
+# In the local development image, we stop with dependencies
+# Everything else is about compiling production assets which is not needed
+# in the typical development workflow.
+FROM base AS development
 
 RUN \
     # Files required to install pip dependencies
@@ -131,11 +138,7 @@ RUN \
     # Mounts for caching dependencies
     --mount=type=cache,target=${PIP_CACHE_DIR},uid=${OLYMPIA_UID},gid=${OLYMPIA_UID} \
     --mount=type=cache,target=${NPM_CACHE_DIR},uid=${OLYMPIA_UID},gid=${OLYMPIA_UID} \
-<<EOF
-${PIP_COMMAND} install --progress-bar=off --no-deps --exists-action=w -r requirements/prod.txt
-${PIP_COMMAND} install --progress-bar=off --no-deps --exists-action=w -r requirements/dev.txt
-npm install ${NPM_ARGS} --no-save
-EOF
+    make update_deps
 
 FROM base AS locales
 ARG LOCALE_DIR=${HOME}/locale
@@ -172,7 +175,7 @@ echo "from olympia.lib.settings_base import *" > settings_local.py
 DJANGO_SETTINGS_MODULE="settings_local" make -f Makefile-docker update_assets
 EOF
 
-FROM base AS sources
+FROM base AS production
 
 ARG DOCKER_BUILD DOCKER_COMMIT DOCKER_VERSION
 
@@ -180,27 +183,15 @@ ENV DOCKER_BUILD=${DOCKER_BUILD}
 ENV DOCKER_COMMIT=${DOCKER_COMMIT}
 ENV DOCKER_VERSION=${DOCKER_VERSION}
 
-# Add our custom mime types (required for for ts/json/md files)
-COPY docker/etc/mime.types /etc/mime.types
 # Copy the rest of the source files from the host
 COPY --chown=olympia:olympia . ${HOME}
+# Copy compiled locales from builder
+COPY --from=locales --chown=olympia:olympia ${HOME}/locale ${HOME}/locale
 # Copy assets from assets
 COPY --from=assets --chown=olympia:olympia ${HOME}/site-static ${HOME}/site-static
 COPY --from=assets --chown=olympia:olympia ${HOME}/static-build ${HOME}/static-build
-
-# Set shell back to sh until we can prove we can use bash at runtime
-SHELL ["/bin/sh", "-c"]
-
-FROM sources AS development
-
-# Copy dependencies from `pip_development`
-COPY --from=pip_development --chown=olympia:olympia /deps /deps
-
-FROM sources AS production
-
-# Copy compiled locales from builder
-COPY --from=locales --chown=olympia:olympia ${HOME}/locale ${HOME}/locale
 # Copy dependencies from `pip_production`
 COPY --from=pip_production --chown=olympia:olympia /deps /deps
 
-
+# Set shell back to sh until we can prove we can use bash at runtime
+SHELL ["/bin/sh", "-c"]

Makefile-docker

@@ -78,6 +78,11 @@ update_assets:
 	# Collect static files: This MUST be run last or files will be missing
 	$(PYTHON_COMMAND) manage.py collectstatic --noinput
 
+.PHONY: update_deps
+update_deps:
+	$(PIP_COMMAND) install --progress-bar=off --no-deps --exists-action=w -r requirements/prod.txt
+	$(PIP_COMMAND) install --progress-bar=off --no-deps --exists-action=w -r requirements/dev.txt
+	npm install $(NPM_ARGS) --no-save
 
 # TOOD: remove this after we migrate addons-frontned to not depend on it.
 .PHONY: setup-ui-tests

docker-compose.ci.yml

@@ -1,19 +0,0 @@
-services:
-  worker:
-    environment:
-      - HOST_UID=9500
-    volumes:
-      - /data/olympia
-
-  web:
-    extends:
-      service: worker
-    volumes:
-      - storage:/data/olympia/storage
-
-  nginx:
-    volumes:
-      - storage:/srv/storage
-
-volumes:
-  storage:

docker-compose.dev.yml

@@ -0,0 +1,20 @@
+services:
+  worker:
+    environment:
+      - HOST_UID
+    volumes:
+      - .:/data/olympia
+      # Don't mount generated files. They only exist in the container
+      # and would otherwiser be deleted by mounting the cwd volume above
+      - /data/olympia/static-build
+      - /data/olympia/site-static
+      - ./package.json:/deps/package.json
+      - ./package-lock.json:/deps/package-lock.json
+
+  web:
+    extends:
+      service: worker
+
+  nginx:
+    volumes:
+      - .:/srv

docker-compose.yml

@@ -20,7 +20,7 @@ x-env-mapping: &env
     - HISTIGNORE=ls:exit:"cd .."
     - HISTCONTROL=erasedups
     - CIRCLECI
-    - HOST_UID
+    - HOST_UID=9500
     - DEBUG
     - DATA_BACKUP_SKIP
 
@@ -51,14 +51,6 @@ services:
       "--",
       "celery -A olympia.amo.celery:app worker -E -c 2 --loglevel=INFO",
     ]
-    volumes:
-      - .:/data/olympia
-      # Don't mount generated files. They only exist in the container
-      # and would otherwiser be deleted by mounting the cwd volume above
-      - /data/olympia/static-build
-      - /data/olympia/site-static
-      - ./package.json:/deps/package.json
-      - ./package-lock.json:/deps/package-lock.json
     extra_hosts:
      - "olympia.test:127.0.0.1"
     restart: on-failure:5
@@ -91,6 +83,8 @@ services:
       interval: 90s
       start_interval: 1s
       start_period: 120s
+    volumes:
+      - storage:/data/olympia/storage
     command:
       - uwsgi --ini /data/olympia/docker/uwsgi.ini
 
@@ -98,7 +92,7 @@ services:
     image: nginx
     volumes:
       - ./docker/nginx/addons.conf:/etc/nginx/conf.d/addons.conf
-      - .:/srv
+      - storage:/srv/storage
     ports:
       - "80:80"
     networks:
@@ -188,3 +182,5 @@ volumes:
     # External volumes must be manually created/destroyed
     name: addons-server_data_mysqld
     external: true
+  storage:
+

docker/entrypoint.sh

@@ -23,6 +23,17 @@ if [[ -n "${HOST_UID:-}" ]]; then
   echo "${OLYMPIA_USER} UID: ${OLYMPIA_UID} -> ${HOST_UID}"
 fi
 
+# TODO: this doesn't work but we have to check if the image we built is production
+# but we are for some reason trying to run it using development dependencies.
+# this is pretty typical for CI... we could consider just building ci images though as a simpler solution. It's just nice to test the image you already pushed cause muh CI/CD...
+if [[ "${DOCKER_VERSION:-}" != 'local' && "${DOCKER_TARGET:-}" == 'production' ]]; then
+  echo "Running remote image. Re-installing with development dependencies..."
+  rm -rf /deps
+  mkdir -p /deps
+  chown -R ${get_olympia_uid}:${get_olympia_gid} /deps
+  su -s /bin/bash $OLYMPIA_USER -c "make update_deps"
+fi
+
 cat <<EOF | su -s /bin/bash $OLYMPIA_USER
   echo "Running command as ${OLYMPIA_USER} $(get_olympia_uid):$(get_olympia_gid)"
   set -xue

docs/topics/development/building_and_running_services.md

@@ -18,7 +18,7 @@ The Dockerfile for the **addons-server** project uses a multi-stage build to opt
 
 4. **Environment Variables for OLYMPIA_USER**:
    - **Development Setup**: The `HOST_UID` environment variable is set to the host user ID, ensuring that the container runs with the correct permissions.
-   - **CI Setup**: In CI environments, such as defined in `docker-compose.ci.yml`, the user ID is reset to the default 9500, and the Olympia mount is removed. This makes the container a closed system, mimicking production behavior closely.
+   - **CI Setup**: In CI environments, the user ID is reset to the default 9500, and the Olympia mount is removed. This makes the container a closed system, mimicking production behavior closely.
 
 ### Best Practices for the Dockerfile
 
@@ -146,7 +146,7 @@ We use docker compose under the hood to orchestrate container both locally and i
 The `docker-compose.yml` file defines the services, volumes, and networks required for the project.
 
 Our docker compose project is split into a root [docker-compose.yml](../../../docker-compose.yml) file and additional files for specific environments,
-such as [docker-compose.ci.yml](../../../docker-compose.ci.yml) for CI environments.
+such as [docker-compose.dev.yml](../../../docker-compose.dev.yml) for local environments.
 
 ### Healthchecks
 
@@ -155,15 +155,14 @@ The health checks ensure the django wsgi server and celery worker node are runni
 
 ### Environment specific compose files
 
-- **Local Development**: The `docker-compose.yml` file is used for local development. It defines services like `web`, `db`, `redis`, and `elasticsearch`.
-- **CI Environment**: The `docker-compose.ci.yml` file is used for CI environments. It overrides the HOST_UID as well as removing volumes to make the container more production like.
-- **Private**: This file includes the customs service that is not open source and should therefore not be included by default.
+- **Base**: The `docker-compose.yml` file is used for local development. It defines services like `web`, `db`, `redis`, and `elasticsearch`.
+- **Dev Environment**: The `docker-compose.dev.yml` file is used for local development. It overrides the HOST_UID as well as removing volumes to make the container more production like.
 - **Override**: This file allows modifying the default configuration without changing the main `docker-compose.yml` file. This file is larglely obsolete and should not be used.
 
 To mount with a specific  set of docker compose files you can add the COMPOSE_FILE argument to make up. This will persist your setting to .env.
 
 ```sh
-make up COMPOSE_FILE=docker-compose.yml:docker-compose.ci.yml
+make up COMPOSE_FILE=docker-compose.yml:docker-compose.dev.yml
 ```
 
 Files should be separated with a colon.

docs/topics/development/setup_and_configuration.md

@@ -41,6 +41,35 @@ Well that depends on what is changed since the last time you ran it.
 Because `make up` is {ref}`idempotent <idempotence>` it will only run the commands that are necessary to bring your environment up to date.
 If nothing has changed, nothing will happen because your environment is already in the desired state.
 
+### Make up OPTIONS
+
+Make up can be run with several arguments to configure how the proejct is run.
+
+- **COMPOSE_FILE**: The compose file to use. default: `docker-compose.yml:docker-compose.dev.yml`
+  Specifies which compose files to use during docker compose commands.
+- **DOCKER_VERSION**: The version of the docker image to use. default: `local` to build the image local.
+  Otherwise it will pull the image from dockerhub with the specified version.
+- **DOCKER_DIGEST**: (overrides `DOCKER_VERSION`) The digest of the docker image to use.
+  This is useful to run a very specific build of addons-server, e.g. from a specific CI run.
+- **DOCKER_TAG**: The tag of the docker image to use. This is useful if you already know the full tag or
+  want to run an image not on dockerhub.
+- **DOCKER_TARGET**: The target of the docker image to use. default: `development` to run the development target.
+  Set to `production` to run the production target. This is the exact same image we use in production.
+- **DEBUG**: The debug mode to use. default: `True` if `DOCKER_TARGET` is `development` else `False`.
+  This enables certain settings in django to make debugging easier. Works in development or production mode.
+  However, debug_toolbar only works if both `DEBUG` is true and `DOCKER_TARGET` is set to `development`.
+
+Here are some common use cases:
+
+1. True production mode: `make up DOCKER_TARGET=production DEBUG=False COMPOSE_FILE=docker-compose.yml`
+2. development mode: `make up DOCKER_TARGET=development DEBUG=True COMPOSE_FILE=docker-compose.yml:docker-compose.dev.yml`
+
+When you run make up, many of the options are saved to your .env file and will be reused on subsequent calls to make up.
+This allows you to customize your environment to your liking without having to specify the same values over and over.
+
+If you want to reset your .env file to the defaults, you can remove the `.env` file and run `make up` again.
+This will create a new .env file with the default values or with whatever values you override with.
+
 ## Shutting down your environment
 
 > TLDR; just run `make down`
@@ -187,7 +216,7 @@ Though it is **highly recommended to use the make commands** instead of directly
 ### Docker Compose Files
 
 - **[docker-compose.yml][docker-compose]**: The primary Docker Compose file defining services, networks, and volumes for local and CI environments.
-- **[docker-compose.ci.yml][docker-compose-ci]**: Overrides certain configurations for CI-specific needs, ensuring the environment is optimized for automated testing and builds.
+- **[docker-compose.dev.yml][docker-compose-dev]**: Overrides certain configurations for local development needs, ensuring the environment is optimized for local development.
 - **[docker-compose.private.yml][docker-compose-private]**: Runs addons-server with the _customs_ service that is only available to Mozilla employees
 
 Our docker compose files rely on substituted values, all of which are included in our .env file for direct CLI compatibility.
@@ -317,7 +346,7 @@ and docker-comose.yml file locally.
 To fix this error `rm -f .env` to remove your .env and `make up` to restart the containers.
 
 [docker-compose]: ../../../docker-compose.yml
-[docker-compose-ci]: ../../../docker-compose.ci.yml
+[docker-compose-dev]: ../../../docker-compose.dev.yml
 [docker-compose-private]: ../../../docker-compose.private.yml
 [docker-image-digest]: https://github.com/opencontainers/.github/blob/main/docs/docs/introduction/digests.md
 [addons-server-tags]: https://hub.docker.com/r/mozilla/addons-server/tags

src/olympia/urls.py

@@ -2,6 +2,7 @@ from django.conf import settings
 from django.contrib import admin
 from django.shortcuts import redirect
 from django.urls import include, re_path, reverse
+from django.views.i18n import JavaScriptCatalog
 from django.views.static import serve as serve_static
 
 from olympia.amo.utils import urlparams
@@ -132,6 +133,13 @@ if settings.SERVE_STATIC_FILES:
                 serve_static,
                 {'document_root': settings.MEDIA_ROOT},
             ),
+            # Specific fallback for statically generated i18n js files
+            # Not generated in dev mode builds
+            re_path(
+                r'^static/js/i18n/.*\.js$',
+                JavaScriptCatalog.as_view(),
+                name='js-i18n-catalog',
+            ),
             # fallback for static files that are not available directly over nginx.
             # Mostly vendor files from python or npm dependencies that are not available
             # in the static files directory.

tests/make/test_setup.py

@@ -134,14 +134,14 @@ class TestDockerTarget(BaseTestClass):
 class TestComposeFile(BaseTestClass):
     def test_default_compose_file(self):
         main()
-        self.assert_set_env_file_called_with(COMPOSE_FILE='docker-compose.yml')
+        self.assert_set_env_file_called_with(
+            COMPOSE_FILE='docker-compose.yml:docker-compose.dev.yml'
+        )
 
     @override_env(DOCKER_TARGET='production')
     def test_default_target_production(self):
         main()
-        self.assert_set_env_file_called_with(
-            COMPOSE_FILE='docker-compose.yml:docker-compose.ci.yml'
-        )
+        self.assert_set_env_file_called_with(COMPOSE_FILE='docker-compose.yml')
 
     @override_env(COMPOSE_FILE='test')
     def test_compose_file_override(self):