Move assets stage to production only image (#22710)

* Simplify static file mounts in nginx

* Only compile assets in production image

* TMP: ensure volume dirs exist

* TMP: use django to create staticfile dirs

* Update src/olympia/core/apps.py

Co-authored-by: Mathieu Pillard <diox@users.noreply.github.com>

* Update src/olympia/lib/settings_base.py

Co-authored-by: Mathieu Pillard <diox@users.noreply.github.com>

---------

Co-authored-by: Mathieu Pillard <diox@users.noreply.github.com>

.github/workflows/_test.yml

@@ -42,14 +42,21 @@ jobs:
             name: Needs Locale Compilation
             services: ''
             compose_file: docker-compose.yml:docker-compose.ci.yml
+            # We should not do this.. we need to fix running production images
+            # so we can test without running additional commands.
             run: |
               make compile_locales
+              make update_assets
               make test_needs_locales_compilation
           -
             name: Static Assets
             services: ''
             compose_file: docker-compose.yml
-            run: make test_static_assets
+            # We should not do this.. we need to fix running production images
+            # so we can test without running additional commands.
+            run: |
+              make update_assets
+              make test_static_assets
           -
             name: Internal Routes
             services: ''

Dockerfile

@@ -172,9 +172,6 @@ ENV DOCKER_VERSION=${DOCKER_VERSION}
 COPY docker/etc/mime.types /etc/mime.types
 # Copy the rest of the source files from the host
 COPY --chown=olympia:olympia . ${HOME}
-# Copy assets from assets
-COPY --from=assets --chown=olympia:olympia ${HOME}/site-static ${HOME}/site-static
-COPY --from=assets --chown=olympia:olympia ${HOME}/static-build ${HOME}/static-build
 
 # Set shell back to sh until we can prove we can use bash at runtime
 SHELL ["/bin/sh", "-c"]
@@ -190,5 +187,8 @@ FROM sources AS production
 COPY --from=locales --chown=olympia:olympia ${HOME}/locale ${HOME}/locale
 # Copy dependencies from `pip_production`
 COPY --from=pip_production --chown=olympia:olympia /deps /deps
+# Copy assets from assets
+COPY --from=assets --chown=olympia:olympia ${HOME}/site-static ${HOME}/site-static
+COPY --from=assets --chown=olympia:olympia ${HOME}/static-build ${HOME}/static-build
 
 

Makefile-os

@@ -45,6 +45,9 @@ CLEAN_PATHS := \
 	logs \
 	buildx-bake-metadata.json \
 	deps \
+	static-build \
+	site-static \
+
 
 .PHONY: help_redirect
 help_redirect:
@@ -60,6 +63,7 @@ help_submake:
 .PHONY: setup
 setup: ## create configuration files version.json and .env required to run this project
 	for path in $(CLEAN_PATHS); do rm -rf "$(PWD)/$$path" && echo "$$path removed"; done
+	for path in $(DOCKER_VOLUME_DIRS); do mkdir -p "$(PWD)/$$path" && echo "$$path created"; done
 	./scripts/setup.py
 
 .PHONY: push_locales

docker-compose.ci.yml

@@ -9,13 +9,6 @@ services:
   web:
     extends:
       service: worker
-    volumes:
-      - data_site_static:/data/olympia/site-static
-
-  nginx:
-    volumes:
-      - data_site_static:/srv/site-static
 
 volumes:
   data_olympia:
-  data_site_static:

docker-compose.yml

@@ -52,10 +52,6 @@ services:
     ]
     volumes:
       - data_olympia:/data/olympia
-      # Don't mount generated files. They only exist in the container
-      # and would otherwiser be deleted by mounbting data_olympia
-      - /data/olympia/static-build
-      - /data/olympia/site-static
       - storage:/data/olympia/storage
       - ./package.json:/deps/package.json
       - ./package-lock.json:/deps/package-lock.json
@@ -90,7 +86,7 @@ services:
     image: nginx
     volumes:
       - ./docker/nginx/addons.conf:/etc/nginx/conf.d/addons.conf
-      - ./static:/srv/site-static
+      - data_olympia:/data/olympia
       - storage:/srv/user-media
     ports:
       - "80:80"

docker/nginx/addons.conf

@@ -11,12 +11,8 @@ server {
     }
 
     location /static/ {
-         alias /srv/site-static/;
-
-         # Fallback to the uwsgi server if the file is not found in the static files directory.
-         # This will happen for vendor files from pytnon or npm dependencies that won't be available
-         # in the static files directory.
-         error_page 404 = @olympia;
+        alias /data/olympia/static/;
+        try_files $uri $uri/ /data/olympia/site-static$uri @olympia;
     }
 
     location /user-media/ {

src/olympia/core/apps.py

@@ -60,6 +60,12 @@ def version_check(app_configs, **kwargs):
 
 @register(CustomTags.custom_setup)
 def static_check(app_configs, **kwargs):
+    """Check that the compressed static assets exist."""
+    # Only check assets if we are on a production image.
+    # This is the only place where we can guarantee that the assets are built.
+    if not settings.IS_PROD_IMAGE:
+        return []
+
     errors = []
     output = StringIO()
 
@@ -99,8 +105,14 @@ class CoreConfig(AppConfig):
     name = 'olympia.core'
     verbose_name = _('Core')
 
+    def ensure_staticfiles_dirs(self):
+        for dir in settings.STATICFILES_DIRS:
+            if not os.path.exists(dir):
+                os.makedirs(dir)
+
     def ready(self):
         super().ready()
+        self.ensure_staticfiles_dirs()
 
         # Ignore Python warnings unless we're running in debug mode.
         if not settings.DEBUG:

src/olympia/lib/settings_base.py

@@ -1584,3 +1584,9 @@ CINDER_QUEUE_PREFIX = 'amo-dev-'
 SOCKET_LABS_HOST = env('SOCKET_LABS_HOST', default='https://api.socketlabs.com/v2/')
 SOCKET_LABS_TOKEN = env('SOCKET_LABS_TOKEN', default=None)
 SOCKET_LABS_SERVER_ID = env('SOCKET_LABS_SERVER_ID', default=None)
+
+DOCKER_TARGET = env('DOCKER_TARGET', default='development')
+# We can treat any image that is not a production image as a development image.
+# Regardless of which stage was actually targeted.
+# We should only verify precise expectations against production images.
+IS_PROD_IMAGE = DOCKER_TARGET == 'production'