Remove redundant actions in inv catch-up task. Use pip-sync in Dockerfile. Make sure pip-sync and npm tasks in invoke are persistent.

Dockerfile

@@ -85,7 +85,9 @@ RUN pip install -U pip==20.0.2 && pip install pip-tools
 # Normally we won't install dev dependencies in production, but we do it here to optimise 
 # docker build cache for local build
 COPY --chown=mozilla ./requirements.txt ./dev-requirements.txt ./
-RUN pip install -r requirements.txt -r dev-requirements.txt
+# We use pip-tools instead of pip install. This will installing, upgrading, or uninstalling 
+# all dependencies necessary to match the contents of the requirements files. 
+RUN pip-sync requirements.txt dev-requirements.txt
 
 # Copy application code.
 # Any change in this directory is likely to invalidate build cache for all lines below.
@@ -97,13 +99,18 @@ COPY --chown=mozilla . .
 # will need to be recreated by `npm run build`.
 COPY --chown=mozilla --from=frontend /app/network-api/networkapi/frontend ./network-api/networkapi/frontend
 
-# Collect static. This command will move static files from application
-# directories and "network-api/networkapi/frontend" folder to the main static directory that
-# will be served by the WSGI server.
+# Run collectstatic to move static files from application directories and 
+# compiled static directory (network-api/networkapi/frontend) to the site's static 
+# directory in /app/network-api/staticfiles that will be served by the WSGI server.
+# 
+# Note: this is only used where DEBUG=False, and so is not needed on dev builds.
+# The network-api/staticfiles will not be visible after mounting the 
+# network-api directory.
 RUN SECRET_KEY=none python ./network-api/manage.py collectstatic --noinput --clear
 
 # Run the WSGI server. It reads GUNICORN_CMD_ARGS, PORT and WEB_CONCURRENCY
 # environment variable hence we don't specify a lot options below.
+# Note: this will be overridden by other commands below for dev builds.
 CMD gunicorn networkapi.wsgi:application
 
 # Below is used for local dev builds only
@@ -123,6 +130,8 @@ USER mozilla
 
 # Pull in the node modules from the frontend build stage so we don't have to run npm ci again.
 # This is just a copy in the container, and is not visible to the host machine.
+# We can't mount this as the empty directory in the host will obscure our the installed content.
+# See https://docs.docker.com/storage/bind-mounts/#mount-into-a-non-empty-directory-on-the-container
 COPY --chown=mozilla --from=frontend /app/node_modules ./node_modules
 
 # To avoid isort `fatal: detected dubious ownership in repository at '/app'` error

tasks.py

@@ -2,7 +2,7 @@ import os
 import re
 from sys import platform
 
-from invoke import task
+from invoke import exceptions, task
 
 ROOT = os.path.dirname(os.path.realpath(__file__))
 LOCALE_DIR = os.path.realpath(os.path.abspath("network-api/locale"))
@@ -140,11 +140,8 @@ def catch_up(ctx):
     print("* Stopping services first")
     ctx.run("docker-compose down")
     print("* Rebuilding images and install dependencies")
+    # The docker image build will install node and python dependencies.
     ctx.run("docker-compose build")
-    print("* Install Node dependencies")
-    npm_install(ctx)
-    print("* Sync Python dependencies")
-    pip_sync(ctx)
     print("* Applying database migrations.")
     migrate(ctx)
     print("* Updating block information.")
@@ -192,9 +189,10 @@ def npm(ctx, command):
     """Shorthand to npm. inv docker-npm \"[COMMAND] [ARG]\" """
     with ctx.cd(ROOT):
         try:
-            # Using 'exec' instead of 'run' to ensure this runs in the running container.
+            # Using 'exec' instead of 'run --rm' as /node_modules is not mounted.
+            # To make this persistent, use 'exec' to run in the running container.
             ctx.run(f"docker-compose exec backend npm {command}")
-        except Exception:
+        except exceptions.UnexpectedExit:
             print("This command requires a running container.\n")
             print("Please run 'inv start' or 'inv start-lean' in a separate terminal window first.")
 
@@ -203,7 +201,13 @@ def npm(ctx, command):
 def npm_install(ctx):
     """Install Node dependencies"""
     with ctx.cd(ROOT):
-        ctx.run("docker-compose run --rm backend npm ci")
+        # Using 'exec' instead of 'run --rm' as /node_modules is not mounted.
+        # To make this persistent, use 'exec' to run in the running container.
+        try:
+            ctx.run("docker-compose exec backend npm ci")
+        except exceptions.UnexpectedExit:
+            print("This command requires a running container.\n")
+            print("Please run 'inv start' or 'inv start-lean' in a separate terminal window first.")
 
 
 @task(aliases=["copy-stage-db"])
@@ -400,6 +404,9 @@ def format_js(ctx):
 @task
 def format_python(ctx):
     """Run python formatting."""
+    # TODO: isort has problem correcting files which are mount points.
+    # It gets the same 'Device or resource busy' error as pip-compile does.
+    # This will need a workaround.
     isort(ctx)
     black(ctx)
 
@@ -535,10 +542,16 @@ def pip_compile_lock(ctx):
 def pip_sync(ctx):
     """Sync your python virtualenv"""
     with ctx.cd(ROOT):
-        ctx.run(
-            "docker-compose run --rm backend ./dockerpythonvenv/bin/pip-sync requirements.txt dev-requirements.txt",
-            **PLATFORM_ARG,
-        )
+        try:
+            ctx.run(
+                # Using 'exec' instead of 'run --rm' as /dockerpythonvenv is not mounted.
+                # To make this persistent, use 'exec' to run in the running container.
+                "docker-compose exec backend ./dockerpythonvenv/bin/pip-sync requirements.txt dev-requirements.txt",
+                **PLATFORM_ARG,
+            )
+        except exceptions.UnexpectedExit:
+            print("This command requires a running container.\n")
+            print("Please run 'inv start' or 'inv start-lean' in a separate terminal window first.")
 
 
 # Translation