task(all): Get new syncstorage-rs service running

Because:
- Locally the sync service wasn't running.
- We couldn't test sync flows against a local sync server
- There's a new version of the sync server that uses rust

This Commit:
- Sets up a dockerfile for syncstorage-rs that is tailored to fxa's local stack
- Updates pm2 sync job to run this docker container and initialize its databases.
- Updates firefox config/profile to point at this service
- Starts firefox up with the FIREFOX_DEBUGGER=true env.
  - Useful for debugging sync
  - Useful for debugging FxA web channel messages

_dev/docker/sync/sync-for-fxa/Dockerfile

@@ -0,0 +1,32 @@
+# Designed to create a development build specifically for fxa's stack
+# Example usage:
+# > docker build . --tag sync-for-fxa
+# > docker run --rm -p 8000:8000 -v  sync-for-fxa
+#
+# Note that adjust settings simply edit
+FROM rust:1.80
+WORKDIR /app
+
+# Add extra libs and setup python
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+    cmake golang-go python3-dev python3-pip python3-setuptools python3-wheel python3.11-venv libmariadb-dev-compat libmariadb-dev
+
+# Checkout the source, and switch to tag 0.17.2 which is what this was last tested with.
+RUN git clone https://github.com/mozilla-services/syncstorage-rs.git . && \
+    git checkout 0.17.2
+
+# Install python dependencie
+RUN rm -rf venv && \
+    python3 -m venv venv && \
+    venv/bin/python -m pip install -r requirements.txt
+
+# Install rust dependencies
+RUN cargo install --path ./syncserver --no-default-features --features=syncstorage-db/mysql --features=py_verifier --locked
+
+# Prebuild for faster startup
+RUN cargo build --no-default-features --features=syncstorage-db/mysql --features=py_verifier
+
+COPY config/local.toml config/local.toml
+
+CMD ["make", "run_mysql"]

_dev/docker/sync/sync-for-fxa/config/local.toml

@@ -0,0 +1,24 @@
+# This config is specific for running Sync in FxA's local development stack.
+
+cors_allowed_origin = "localhost"
+cors_max_age = 86400
+human_logs = 1
+master_secret = "secret0"
+port = 8000
+host = "0.0.0.0"
+
+syncstorage.database_url = "mysql://sync:test@host.docker.internal:3306/syncstorage"
+syncstorage.enable_quota = 0
+syncstorage.enabled = true
+syncstorage.limits.max_total_records = 1666 # See issues #298/#333
+syncstorage.run_migrations = true
+
+tokenserver.database_url = "mysql://sync:test@host.docker.internal:3306/tokenserver"
+tokenserver.enabled = true
+tokenserver.fxa_email_domain = "api-accounts.stage.mozaws.net"
+tokenserver.fxa_metrics_hash_secret = "INSERT_SECRET_KEY_HERE"
+tokenserver.fxa_oauth_server_url = "http://host.docker.internal:9000"
+tokenserver.fxa_browserid_audience = "https://token.stage.mozaws.net"
+tokenserver.fxa_browserid_issuer = "https://api-accounts.stage.mozaws.net"
+tokenserver.fxa_browserid_server_url = "https://verifier.stage.mozaws.net/v2"
+tokenserver.run_migrations = true

_dev/pm2/infrastructure.config.js

@@ -34,8 +34,8 @@ module.exports = {
     },
     {
       name: 'sync',
-      script: '_scripts/syncserver.sh',
-      max_restarts: '1',
+      script: '_scripts/sync.sh',
+      max_restarts: '0',
       min_uptime: '2m',
       autorestart: false,
       kill_timeout: 20000,

_scripts/mysql.sh

@@ -18,6 +18,7 @@ trap on_sigint INT
 
 # Create pushbox db on start (because pushbox doesn't create it)
 docker run --rm --name=mydb \
+  --net fxa \
   -e MYSQL_ALLOW_EMPTY_PASSWORD=true \
   -e MYSQL_ROOT_HOST=% \
   -e MYSQL_DATABASE=pushbox \

_scripts/sync-populate.sh

@@ -0,0 +1,36 @@
+#!/bin/bash -e
+
+# This will populate sync's tokenserver db with minimal state to ensure the token server is
+# functional. Note, that sync will start and autmatically setup the database schemas. Once
+# this happens, we can then run these inserts to setup the initial database state required
+# for run sync to serve requrest from firefox.
+
+_scripts/check-url.sh localhost:8000/__heartbeat__
+
+RETRY=240
+echo "waiting on tokenserver db to be created."
+for i in $(eval echo "{1..$RETRY}"); do
+    db=$(docker exec mydb mysql --silent --skip-column-names -e 'SELECT SCHEMA_NAME FROM information_schema.SCHEMATA WHERE SCHEMA_NAME = "tokenserver";')
+    if [ -z "$db" ]; then
+        echo -e -n "\r   ...no response, retry attempt $i of $RETRY"
+        sleep 1
+    else
+        echo ""
+        echo "sync db exists! populating..."
+
+        docker exec mydb mysql -e 'INSERT INTO tokenserver.services (service, pattern) VALUES ("sync-1.5", "{node}/1.5/{uid}")'
+        docker exec mydb mysql -e 'INSERT INTO tokenserver.nodes (service, node, available, capacity, current_load, backoff, downed, id) VALUES ((select id from tokenserver.services limit 1), "http://localhost:8000", 100, 100, 0, 0, 0, 800)'
+
+        echo "done populating sync database state!"
+
+        echo "token.nodes:"
+        docker exec mydb mysql -e 'select * from tokenserver.nodes;'
+
+        echo "tokenserver.services:"
+        docker exec mydb mysql -e 'select * from tokenserver.services;'
+
+        exit 0
+    fi
+done
+
+echo "tokenserver database was never created.... giving up!"

_scripts/sync.sh

@@ -0,0 +1,47 @@
+#!/bin/bash -e
+
+# We currently don't want don't want to spin this up in the CI, since
+# we don't have any functional Sync tests at the moment.
+if [ "$CI" == "true" ]; then
+  echo Not running sync in CI. Exiting...
+  exit 0
+fi
+
+function on_sigint() {
+  echo "MySQL shutting down."
+  docker stop sync
+  exit 0
+}
+
+trap on_sigint INT
+
+# Make sure the local sql instance is responsive
+_scripts/check-mysql.sh
+
+# Setup sync db user, syncstorage db, and tokenserver db.
+docker exec mydb mysql -e 'DROP USER IF EXISTS sync;'
+docker exec mydb mysql -e 'CREATE USER "sync"@"%" IDENTIFIED BY "test";'
+docker exec mydb mysql -e 'DROP DATABASE IF EXISTS syncstorage;'
+docker exec mydb mysql -e 'CREATE DATABASE syncstorage;'
+docker exec mydb mysql -e 'GRANT ALL PRIVILEGES on syncstorage.* to "sync"@"%";'
+docker exec mydb mysql -e 'DROP DATABASE IF EXISTS tokenserver;'
+docker exec mydb mysql -e 'CREATE DATABASE tokenserver;'
+docker exec mydb mysql -e 'GRANT ALL PRIVILEGES on tokenserver.* to "sync"@"%";'
+
+# Make sure the docker image that runs sync is up to date, and then run it.
+#  - Because image layers are cached by docker, the first build may take a
+#    second, but subsequent builds should have hardly any overhead.
+#  - Note that config modifications can be made by editing the
+#    config/local.toml file.
+cd _dev/docker/sync/sync-for-fxa
+docker build . --tag sync-for-fxa
+cd ../../../..
+
+# Start the sync server and token server on port 8000
+docker run --rm --name sync -p 8000:8000  sync-for-fxa &
+
+# Adds some rows to the sync database that are needed.
+_scripts/sync-populate.sh
+
+# Keep script alive
+while :; do read -r; done

_scripts/syncserver.sh

@@ -1,25 +0,0 @@
-#!/bin/bash -ex
-
-DOCKER_OS="$(docker info --format '{{.OperatingSystem}}')"
-
-if [ "$DOCKER_OS" = 'Docker for Windows' ] || [ "$DOCKER_OS" = 'Docker for Mac' ] || [ "$DOCKER_OS" = 'Docker Desktop' ]; then
-  HOST_ADDR='host.docker.internal'
-else
-  HOST_ADDR='localhost'
-fi
-
-"${0%/*}/check-url.sh" "$HOST_ADDR:3030/.well-known/fxa-client-configuration"
-
-docker run --rm --name syncserver \
-  --net fxa \
-  -p 5000:5000 \
-  -e SYNCSERVER_PUBLIC_URL=http://localhost:5000 \
-  -e SYNCSERVER_IDENTITY_PROVIDER=http://$HOST_ADDR:3030 \
-  -e SYNCSERVER_OAUTH_VERIFIER=http://$HOST_ADDR:9000 \
-  -e SYNCSERVER_BROWSERID_VERIFIER=http://$HOST_ADDR:5050 \
-  -e SYNCSERVER_SECRET=5up3rS3kr1t \
-  -e SYNCSERVER_SQLURI=sqlite:////tmp/syncserver.db \
-  -e SYNCSERVER_BATCH_UPLOAD_ENABLED=true \
-  -e SYNCSERVER_FORCE_WSGI_ENVIRON=true \
-  -e PORT=5000 \
-  mozilla/syncserver:latest

package.json

@@ -18,7 +18,7 @@
     "ports": "pm2 jlist | json -a -c 'this.pm2_env.env.PORT' pm2_env.env.PORT name",
     "heroku-postbuild": "yarn workspaces foreach --verbose --include 123done install",
     "mysql": "docker exec -it $(docker container ls | grep mysql | cut -d' ' -f1) mysql",
-    "firefox": "./packages/fxa-dev-launcher/bin/fxa-dev-launcher.mjs",
+    "firefox": "FIREFOX_DEBUGGER=true ./packages/fxa-dev-launcher/bin/fxa-dev-launcher.mjs",
     "generate-lockfile": "docker build . -f _dev/docker/ci-lockfile-generator/Dockerfile -t generate-lockfile && docker run generate-lockfile > yarn.lock",
     "l10n:clone": "_scripts/l10n/clone.sh",
     "l10n:prime": "_scripts/l10n/prime.sh",

packages/functional-tests/lib/targets/firefoxUserPrefs.ts

@@ -6,7 +6,7 @@ const CONFIGS = {
   local: {
     auth: 'http://localhost:9000/v1',
     content: 'http://localhost:3030/',
-    token: 'http://localhost:5000/token/1.0/sync/1.5',
+    token: 'http://localhost:8000/token/1.0/sync/1.5',
     oauth: 'http://localhost:9000/v1',
     profile: 'http://localhost:1111/v1',
   },

packages/fxa-auth-server/config/index.ts

@@ -1941,7 +1941,7 @@ const convictConf = convict({
     },
   },
   syncTokenserverUrl: {
-    default: 'http://localhost:5000/token',
+    default: 'http://localhost:8000/token',
     doc: 'The url of the Firefox Sync tokenserver',
     env: 'SYNC_TOKENSERVER_URL',
     format: 'url',

packages/fxa-content-server/server/lib/configuration.js

@@ -953,7 +953,7 @@ const conf = (module.exports = convict({
     },
   },
   sync_tokenserver_url: {
-    default: 'http://localhost:5000/token',
+    default: 'http://localhost:8000/token',
     doc: 'The url of the Firefox Sync tokenserver',
     env: 'SYNC_TOKENSERVER_URL',
     format: 'url',

packages/fxa-content-server/tests/intern.js

@@ -14,7 +14,7 @@ const fxaAuthRoot = args.fxaAuthRoot || 'http://localhost:9000/v1';
 const fxaContentRoot = args.fxaContentRoot || 'http://localhost:3030/';
 const fxaOAuthRoot = args.fxaOAuthRoot || 'http://localhost:9000';
 const fxaProfileRoot = args.fxaProfileRoot || 'http://localhost:1111';
-const fxaTokenRoot = args.fxaTokenRoot || 'http://localhost:5000/token';
+const fxaTokenRoot = args.fxaTokenRoot || 'http://localhost:8000/token';
 const fxaEmailRoot = args.fxaEmailRoot || 'http://localhost:9001';
 const fxaOAuthApp = args.fxaOAuthApp || 'http://localhost:8080/';
 const fxaUntrustedOauthApp =

packages/fxa-dev-launcher/profile.mjs

@@ -4,7 +4,7 @@ const CONFIGS = {
   local: {
     auth: 'http://localhost:9000/v1',
     content: 'http://localhost:3030/',
-    token: 'http://localhost:5000/token/1.0/sync/1.5',
+    token: 'http://localhost:8000/1.0/sync/1.5',
     loop: 'http://localhost:10222',
     oauth: 'http://localhost:9000/v1',
     profile: 'http://localhost:1111/v1',