Remove supervisor and enable more efficient container startup (#22592)

* TMP: add http server to expose version endpoint

* Use data volume for mounting repository files

* Use watchmedo directly from docker to run celery

* TMP: fix test

* TMP: remove data_olympia volume on ci yml

* TMP: add settings djanmgo modeul to celery worker.

* TMP: remove references to ssupervisor

* Update docs/topics/development/building_and_running_services.md

Co-authored-by: Mathieu Pillard <diox@users.noreply.github.com>

---------

Co-authored-by: Mathieu Pillard <diox@users.noreply.github.com>

.gitignore

@@ -48,7 +48,6 @@ storage/mlbf/*
 storage/guarded-addons/*
 storage/shared_storage/*
 storage/sitemaps/*
-supervisord.pid
 tmp/*
 deps/*
 

Makefile-docker

@@ -216,10 +216,6 @@ lint-codestyle: lint
 docs: ## build the documentation
 	$(MAKE) -C docs html SPHINXOPTS='-nW'
 
-.PHONY: debug
-debug: ## connect for debugging
-	supervisorctl fg olympia
-
 .PHONY: djshell
 djshell: ## connect to django shell
 	$(PYTHON_COMMAND) ./manage.py shell_plus

docker-compose.ci.yml

@@ -7,3 +7,6 @@ services:
 
   worker:
     <<: *web
+
+volumes:
+  data_olympia:

docker-compose.yml

@@ -27,7 +27,7 @@ x-olympia: &olympia
   image: ${DOCKER_TAG:-}
   # Ignore any linting saying we have an invalid value.
   pull_policy: ${DOCKER_PULL_POLICY:-}
-  # We drop down to a different user through supervisord, but starting as
+  # We drop down to a different user through entrypoint.sh, but starting as
   # root allows us to fix the ownership of files generated at image build
   # time through the ./docker/entrypoint.sh script.
   user: root
@@ -58,10 +58,18 @@ services:
     depends_on:
       olympia:
         condition: service_completed_successfully
-    command:
-      - supervisord -n -c /data/olympia/docker/supervisor-celery.conf
+    command: [
+      "DJANGO_SETTINGS_MODULE=settings",
+      "watchmedo",
+      "auto-restart",
+      "--directory=/data/olympia/src",
+      "--pattern=*.py",
+      "--recursive",
+      "--",
+      "celery -A olympia.amo.celery:app worker -E -c 2 --loglevel=INFO",
+    ]
     volumes:
-      - .:/data/olympia
+      - data_olympia:/data/olympia
       - storage:/data/olympia/storage
       - data_deps:/deps
       - ./package.json:/deps/package.json
@@ -70,20 +78,20 @@ services:
      - "olympia.test:127.0.0.1"
     restart: on-failure:5
     # entrypoint.sh takes some time
-    # we can wait for supervisor to start
-    # indicating the service is ready
+    # we can wait for services to be up and running
     healthcheck:
-      test: [
-        "CMD-SHELL",
-        "if [ -f /data/olympia/supervisord.pid ]; then echo 'File exists'; else echo 'File not found'; exit 1; fi"
-      ]
+      test: ["CMD-SHELL", "DJANGO_SETTINGS_MODULE=olympia celery -A olympia.amo.celery status"]
       interval: 3s
       retries: 100
 
   web:
     <<: *worker
+    healthcheck:
+      test: ["CMD-SHELL", "curl --fail --show-error --include --location http://127.0.0.1:8002/__version__"]
+      interval: 3s
+      retries: 100
     command:
-      - supervisord -n -c /data/olympia/docker/supervisor.conf
+      - uwsgi --ini /data/olympia/docker/uwsgi.ini
   nginx:
     image: nginx
     volumes:
@@ -155,7 +163,7 @@ services:
     platform: linux/amd64
     command: /go/bin/autograph -c /data/olympia/scripts/autograph_localdev_config.yaml
     volumes:
-      - .:/data/olympia
+      - data_olympia:/data/olympia
 
   addons-frontend:
     <<: *env
@@ -192,6 +200,12 @@ volumes:
       type: none
       o: bind
       device: ./deps
+  data_olympia:
+    driver: local
+    driver_opts:
+      type: none
+      o: bind
+      device: ${PWD}
   storage:
     driver: local
     driver_opts:

docker/supervisor-celery.conf

@@ -1,29 +0,0 @@
-[supervisord]
-logfile=/data/olympia/logs/supervisord-celery.log
-logfile_maxbytes=1048576
-logfile_backups=1
-user=olympia
-
-[program:olympia-worker]
-# Local env setup for celery. This is similar to prod, but with only two workers
-# Prod config for reference:
-# https://github.com/mozilla-services/cloudops-deployment/blob/master/projects/amo/puppet/modules/olympia/manifests/celery.pp
-command=watchmedo auto-restart --directory /data/olympia/src --pattern '*.py' --recursive -- celery -A olympia.amo.celery:app worker -E -c 2 --loglevel=INFO
-directory=/data/olympia
-stopasgroup=true
-autostart=true
-redirect_stderr=true
-stdout_logfile=logs/docker-celery.log
-stdout_logfile_maxbytes=10MB
-stdout_logfile_backups=1
-stopsignal=KILL
-environment=DJANGO_SETTINGS_MODULE='settings'
-
-
-# The following sections enable supervisorctl.
-
-[inet_http_server]
-port=9001
-
-[rpcinterface:supervisor]
-supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface

docker/supervisor.conf

@@ -1,32 +0,0 @@
-[supervisord]
-logfile=/data/olympia/logs/supervisord-olympia.log
-logfile_maxbytes=1048576
-logfile_backups=1
-user=olympia
-
-[program:olympia]
-command=uwsgi --ini /data/olympia/docker/uwsgi.ini
-directory=/data/olympia
-stopasgroup=true
-autostart=true
-redirect_stderr=true
-stdout_logfile=logs/docker-olympia.log
-stdout_logfile_maxbytes=10MB
-stdout_logfile_backups=1
-stopsignal=KILL
-priority=500
-
-[program:watcher]
-autostart=true
-autorestart=false
-directory=/tmp
-command=watchmedo shell-command --pattern '*.py' --recursive --command='/usr/bin/touch /data/olympia/docker/artifacts/uwsgi-reload-monitor' /data/olympia/src
-priority=991
-
-# The following sections enable supervisorctl.
-
-[inet_http_server]
-port=9001
-
-[rpcinterface:supervisor]
-supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface

docker/uwsgi.ini

@@ -10,6 +10,7 @@ master          = true
 processes       = 4
 vaccum          = true
 socket          = :8001
+http            = 127.0.0.1:8002
 uid             = $(id -u olympia)
 gid             = $(id -g olympia)
 memory-report   = true
@@ -17,9 +18,7 @@ enable-threads  = true
 
 safe-pidfile = %(base)/docker/artifacts/addons-server-uwsgi-master.pid
 
-# supervisor runs watchmedo (via watchdog) which is configured to touch this
-# file when changes are detected. we set uwsgi to reload when that happens.
-fs-reload = %(base)/docker/artifacts/uwsgi-reload-monitor
+py-autoreload=1
 
 max-requests = 5000
 

docs/topics/development/building_and_running_services.md

@@ -105,6 +105,11 @@ The `docker-compose.yml` file defines the services, volumes, and networks requir
 Our docker compose project is split into a root [docker-compose.yml](../../../docker-compose.yml) file and additional files for specific environments,
 such as [docker-compose.ci.yml](../../../docker-compose.ci.yml) for CI environments.
 
+### Healthchecks
+
+We define healthchecks for the web and worker services to ensure that the containers are healthy and ready to accept traffic.
+The health checks ensure the django wsgi server and celery worker node are running and available to accept requests.
+
 ### Environment specific compose files
 
 - **Local Development**: The `docker-compose.yml` file is used for local development. It defines services like `web`, `db`, `redis`, and `elasticsearch`.

docs/topics/development/troubleshooting_and_debugging.md

@@ -43,7 +43,7 @@ Effective troubleshooting and debugging practices are essential for maintaining
 
 ## Debugging
 
-The Docker setup uses `supervisord` to run the Django runserver. This allows you to access the management server from a shell to run things like `ipdb`.
+You can access the management server from a shell to run things like `ipdb`.
 
 ### Using `ipdb`
 

requirements/dev.txt

@@ -1,33 +1,5 @@
 pyinotify==0.9.6 \
     --hash=sha256:9c998a5d7606ca835065cdabc013ae6c66eb9ea76a00a1e3bc6e0cfe2b4f71f4
-watchdog[watchmedo]==3.0.0 \
-    --hash=sha256:0e06ab8858a76e1219e68c7573dfeba9dd1c0219476c5a44d5333b01d7e1743a \
-    --hash=sha256:13bbbb462ee42ec3c5723e1205be8ced776f05b100e4737518c67c8325cf6100 \
-    --hash=sha256:233b5817932685d39a7896b1090353fc8efc1ef99c9c054e46c8002561252fb8 \
-    --hash=sha256:25f70b4aa53bd743729c7475d7ec41093a580528b100e9a8c5b5efe8899592fc \
-    --hash=sha256:2b57a1e730af3156d13b7fdddfc23dea6487fceca29fc75c5a868beed29177ae \
-    --hash=sha256:336adfc6f5cc4e037d52db31194f7581ff744b67382eb6021c868322e32eef41 \
-    --hash=sha256:3aa7f6a12e831ddfe78cdd4f8996af9cf334fd6346531b16cec61c3b3c0d8da0 \
-    --hash=sha256:3ed7c71a9dccfe838c2f0b6314ed0d9b22e77d268c67e015450a29036a81f60f \
-    --hash=sha256:4c9956d27be0bb08fc5f30d9d0179a855436e655f046d288e2bcc11adfae893c \
-    --hash=sha256:4d98a320595da7a7c5a18fc48cb633c2e73cda78f93cac2ef42d42bf609a33f9 \
-    --hash=sha256:4f94069eb16657d2c6faada4624c39464f65c05606af50bb7902e036e3219be3 \
-    --hash=sha256:5113334cf8cf0ac8cd45e1f8309a603291b614191c9add34d33075727a967709 \
-    --hash=sha256:51f90f73b4697bac9c9a78394c3acbbd331ccd3655c11be1a15ae6fe289a8c83 \
-    --hash=sha256:5d9f3a10e02d7371cd929b5d8f11e87d4bad890212ed3901f9b4d68767bee759 \
-    --hash=sha256:7ade88d0d778b1b222adebcc0927428f883db07017618a5e684fd03b83342bd9 \
-    --hash=sha256:7c5f84b5194c24dd573fa6472685b2a27cc5a17fe5f7b6fd40345378ca6812e3 \
-    --hash=sha256:7e447d172af52ad204d19982739aa2346245cc5ba6f579d16dac4bfec226d2e7 \
-    --hash=sha256:8ae9cda41fa114e28faf86cb137d751a17ffd0316d1c34ccf2235e8a84365c7f \
-    --hash=sha256:8f3ceecd20d71067c7fd4c9e832d4e22584318983cabc013dbf3f70ea95de346 \
-    --hash=sha256:9fac43a7466eb73e64a9940ac9ed6369baa39b3bf221ae23493a9ec4d0022674 \
-    --hash=sha256:a70a8dcde91be523c35b2bf96196edc5730edb347e374c7de7cd20c43ed95397 \
-    --hash=sha256:adfdeab2da79ea2f76f87eb42a3ab1966a5313e5a69a0213a3cc06ef692b0e96 \
-    --hash=sha256:ba07e92756c97e3aca0912b5cbc4e5ad802f4557212788e72a72a47ff376950d \
-    --hash=sha256:c07253088265c363d1ddf4b3cdb808d59a0468ecd017770ed716991620b8f77a \
-    --hash=sha256:c9d8c8ec7efb887333cf71e328e39cffbf771d8f8f95d308ea4125bf5f90ba64 \
-    --hash=sha256:d00e6be486affb5781468457b21a6cbe848c33ef43f9ea4a73b4882e5f188a44 \
-    --hash=sha256:d429c2430c93b7903914e4db9a966c7f2b068dd2ebdd2fa9b9ce094c7d459f33
 backcall==0.2.0 \
     --hash=sha256:fbbce6a29f263178a1f7915c1940bde0ec2b2a967566fe1c65c1dfb7422bd255 \
     --hash=sha256:5cbdbf27be5e7cfadb448baf0aa95508f91f2bbc6c6437cd9cd06e2a4c215e1e
@@ -40,9 +12,6 @@ ptyprocess==0.7.0 \
 pickleshare==0.7.5 \
     --hash=sha256:9649af414d74d4df115d5d718f82acb59c9d418196b7b4290ed47a12ce62df56 \
     --hash=sha256:87683d47965c1da65cdacaf31c8441d12b8044cdec9aca500cd78fc2c683afca
-supervisor==4.2.5 \
-    --hash=sha256:2ecaede32fc25af814696374b79e42644ecaba5c09494c51016ffda9602d0f08 \
-    --hash=sha256:34761bae1a23c58192281a5115fb07fbf22c9b0133c08166beffc70fed3ebc12
 # Sphinx is required by sphinxcontrib-httpdomain
 Sphinx==8.0.2 \
     --hash=sha256:0cce1ddcc4fd3532cf1dd283bc7d886758362c5c1de6598696579ce96d8ffa5b \

requirements/prod.txt

@@ -1098,7 +1098,7 @@ inflection==0.5.1 \
     --hash=sha256:1a29730d366e996aaacffb2f1f1cb9593dc38e2ddd30c91250c6dde09ea9b417 \
     --hash=sha256:f38b2b640938a4f35ade69ac3d053042959b62a0f1076a5bbaa1b9526605a8a2
 # Dependencies for Ipython
-# ipython / ipdb for easier debugging, supervisor to run services
+# ipython / ipdb for easier debugging
 ipdb==0.13.13 \
     --hash=sha256:45529994741c4ab6d2388bfa5d7b725c2cf7fe9deffabdb8a6113aa5ed449ed4 \
     --hash=sha256:e3ac6018ef05126d442af680aad863006ec19d02290561ac88b8b1c0b0cfc726
@@ -1187,3 +1187,31 @@ PyYAML==6.0.1 \
     --hash=sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585 \
     --hash=sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d \
     --hash=sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f
+watchdog[watchmedo]==3.0.0 \
+    --hash=sha256:0e06ab8858a76e1219e68c7573dfeba9dd1c0219476c5a44d5333b01d7e1743a \
+    --hash=sha256:13bbbb462ee42ec3c5723e1205be8ced776f05b100e4737518c67c8325cf6100 \
+    --hash=sha256:233b5817932685d39a7896b1090353fc8efc1ef99c9c054e46c8002561252fb8 \
+    --hash=sha256:25f70b4aa53bd743729c7475d7ec41093a580528b100e9a8c5b5efe8899592fc \
+    --hash=sha256:2b57a1e730af3156d13b7fdddfc23dea6487fceca29fc75c5a868beed29177ae \
+    --hash=sha256:336adfc6f5cc4e037d52db31194f7581ff744b67382eb6021c868322e32eef41 \
+    --hash=sha256:3aa7f6a12e831ddfe78cdd4f8996af9cf334fd6346531b16cec61c3b3c0d8da0 \
+    --hash=sha256:3ed7c71a9dccfe838c2f0b6314ed0d9b22e77d268c67e015450a29036a81f60f \
+    --hash=sha256:4c9956d27be0bb08fc5f30d9d0179a855436e655f046d288e2bcc11adfae893c \
+    --hash=sha256:4d98a320595da7a7c5a18fc48cb633c2e73cda78f93cac2ef42d42bf609a33f9 \
+    --hash=sha256:4f94069eb16657d2c6faada4624c39464f65c05606af50bb7902e036e3219be3 \
+    --hash=sha256:5113334cf8cf0ac8cd45e1f8309a603291b614191c9add34d33075727a967709 \
+    --hash=sha256:51f90f73b4697bac9c9a78394c3acbbd331ccd3655c11be1a15ae6fe289a8c83 \
+    --hash=sha256:5d9f3a10e02d7371cd929b5d8f11e87d4bad890212ed3901f9b4d68767bee759 \
+    --hash=sha256:7ade88d0d778b1b222adebcc0927428f883db07017618a5e684fd03b83342bd9 \
+    --hash=sha256:7c5f84b5194c24dd573fa6472685b2a27cc5a17fe5f7b6fd40345378ca6812e3 \
+    --hash=sha256:7e447d172af52ad204d19982739aa2346245cc5ba6f579d16dac4bfec226d2e7 \
+    --hash=sha256:8ae9cda41fa114e28faf86cb137d751a17ffd0316d1c34ccf2235e8a84365c7f \
+    --hash=sha256:8f3ceecd20d71067c7fd4c9e832d4e22584318983cabc013dbf3f70ea95de346 \
+    --hash=sha256:9fac43a7466eb73e64a9940ac9ed6369baa39b3bf221ae23493a9ec4d0022674 \
+    --hash=sha256:a70a8dcde91be523c35b2bf96196edc5730edb347e374c7de7cd20c43ed95397 \
+    --hash=sha256:adfdeab2da79ea2f76f87eb42a3ab1966a5313e5a69a0213a3cc06ef692b0e96 \
+    --hash=sha256:ba07e92756c97e3aca0912b5cbc4e5ad802f4557212788e72a72a47ff376950d \
+    --hash=sha256:c07253088265c363d1ddf4b3cdb808d59a0468ecd017770ed716991620b8f77a \
+    --hash=sha256:c9d8c8ec7efb887333cf71e328e39cffbf771d8f8f95d308ea4125bf5f90ba64 \
+    --hash=sha256:d00e6be486affb5781468457b21a6cbe848c33ef43f9ea4a73b4882e5f188a44 \
+    --hash=sha256:d429c2430c93b7903914e4db9a966c7f2b068dd2ebdd2fa9b9ce094c7d459f33

settings.py

@@ -79,7 +79,7 @@ MEDIA_URL = '%s/user-media/' % EXTERNAL_SITE_URL
 
 CODE_MANAGER_URL = os.environ.get('CODE_MANAGER_URL') or 'http://olympia.test:5000'
 
-ALLOWED_HOSTS = ALLOWED_HOSTS + [SERVICES_DOMAIN, 'nginx']
+ALLOWED_HOSTS = ALLOWED_HOSTS + [SERVICES_DOMAIN, 'nginx', '127.0.0.1']
 
 # Default AMO user id to use for tasks (from users.json fixture in zadmin).
 TASK_USER_ID = 10968

tests/make/make.spec.js

@@ -181,6 +181,7 @@ const skippedKeys = [
   'DOCKER_COMMIT',
   'DOCKER_VERSION',
   'DOCKER_BUILD',
+  'PWD',
 ];
 
 test('All dynamic properties in any docker compose file are referenced in the test', () => {