Migrate Travis to CircleCI (#2310)

* add lints to circleci * delete travis config * add circleci orbs * update images * update dockerfile to allow dev build * refactor unit-tests using machine host instead of docker * set postgres user pass to enable docker connection * remove .env-dist quotes in values (broken syntax) * fix broken coveralls test * update readme * use `npm ci` instead of redundant `npm install` * switch integration test network mode to `host` * specify .env path for integration test * update lockfile * remove docker login and obsolete jobs
2021-10-15 11:58:00 -07:00 · 2021-10-15 11:58:00 -07:00 · 3d24595060
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@ -1,163 +1,125 @@
-version: 2.0
-jobs:
-    build:
+version: 2.1
+
+executors:
+    node:
        docker:
-            - image: docker:stable-git
-              auth:
-                username: $DOCKER_USER
-                password: $DOCKER_PASS
-        working_directory: /dockerflow
+            - image: cimg/node:14.17
+    python:
+        docker:
+            - image: cimg/python:3.9.7
+    ubuntu:
+        machine:
+            image: ubuntu-2004:202107-02
+
+orbs:
+    node: circleci/node@4.7.0
+    python: circleci/python@1.4.0
+
+jobs:
+    lint-js:
+        executor: node
        steps:
            - checkout
-            - setup_remote_docker
-            - run:
-                name: Create version.json
-                command: |
-                    # create a version.json per https://github.com/mozilla-services/Dockerflow/blob/master/docs/version_object.md
-                    printf '{"commit":"%s","version":"%s","source":"https://github.com/%s/%s","build":"%s"}\n' \
-                    "$CIRCLE_SHA1" \
-                    "$CIRCLE_TAG" \
-                    "$CIRCLE_PROJECT_USERNAME" \
-                    "$CIRCLE_PROJECT_REPONAME" \
-                    "$CIRCLE_BUILD_URL" > version.json
-
-            - run:
-                name: Login to dockerhub
-                command: |
-                    if [ "${DOCKER_USER}" == "" ] || [ "${DOCKER_PASS}" == "" ]; then
-                      echo "Skipping Login to Dockerhub, credentials not available."
-                    else
-                      echo "${DOCKER_PASS}" | docker login -u="${DOCKER_USER}" --password-stdin
-                    fi
-
-            - run:
-                name: Build docker image
-                command: docker build -t blurts-server .
-
-            # save the built docker container into CircleCI's cache. This is
-            # required since Workflows do not have the same remote docker instance.
-            - run:
-                  name: docker save blurts-server
-                  command: mkdir -p /cache; docker save -o /cache/docker.tar "blurts-server"
-            - save_cache:
-                  key: v1-{{ .Branch }}-{{epoch}}
-                  paths:
-                      - /cache/docker.tar
-
-    test:
-        docker:
-            - image: docker:18.06.3-ce
+            - node/install-packages
+            - run: npm run lint:js
+    lint-css:
+        executor: node
        steps:
-            - setup_remote_docker
-            - restore_cache:
-                key: v1-{{.Branch}}
+            - checkout
+            - node/install-packages
+            - run: npm run lint:css
+    lint-audit:
+        executor: node
+        steps:
+            - checkout
+            - node/install-packages
+            - run: npm run lint:audit
+    lint-l10n:
+        executor: python
+        steps:
+            - checkout
+            - run: pip install compare-locales
            - run:
-                name: Restore Docker image cache
-                command: docker load -i /cache/docker.tar
+                  name: Lint and compare locales
+                  command: |
+                      moz-l10n-lint l10n.toml
+                      compare-locales l10n.toml . `ls locales`
+    unit-tests:
+        executor: ubuntu
+        environment:
+            DATABASE_URL: postgresql://postgres:unsafe-dev-pass@localhost/blurts
+        steps:
+            - checkout
+            - run: 
+                name: Install Postgres
+                # Installing Postgres on a host machine. 
+                # Originally attempted to use a container, but communication between container-within-a-container is not well supported currently
+                # The solutions here were attempted without success: https://stackoverflow.com/questions/48546124/what-is-linux-equivalent-of-host-docker-internal
+                command: |
+                    sudo apt-get update
+                    sudo apt-get install postgresql
+                    pg_isready
            - run:
-                name: Update npm to latest
-                command: docker run blurts-server npm install npm@latest -g
+                name: Create test database
+                command: |
+                    sudo -u postgres createdb test-blurts | cat
+                    # Set password to allow Docker connection
+                    sudo -u postgres psql -c "ALTER USER postgres PASSWORD 'unsafe-dev-pass';" | cat
+            - run:
+                name: Build app container in dev mode
+                command: docker build -t blurts-server . --build-arg dev=true
            - run:
                name: Test Code
-                command: docker run blurts-server npm run lint
+                command: |
+                    docker run --network="host" \
+                    --env-file .env-dist \
+                    -e NODE_ENV=tests \
+                    -e DATABASE_URL=$DATABASE_URL \
+                    -e HIBP_KANON_API_TOKEN=$HIBP_KANON_API_TOKEN \
+                    -e COVERALLS_SERVICE_NAME=circleci \
+                    -e COVERALLS_REPO_TOKEN=$COVERALLS_REPO_TOKEN \
+                    -e COVERALLS_GIT_BRANCH=$CIRCLE_BRANCH \
+                    blurts-server npm test

-    integration-test:
-        machine:
-            image: ubuntu-1604:201903-01
+    integration-tests:
+        executor: ubuntu
        steps:
            - checkout
+            - node/install-npm:
+                # Official node@14 images come packaged with npm@6. For some reason, this ubuntu image comes with node@14 + npm@7
+                # Attempted to roll ubuntu image back, but current version includes docker-compose bug-fix which we use (--env-file)
+                version: "6"
+            - node/install-packages
            - run:
-                name: Build and run tests.
-                command: |
-                    export NVM_DIR="/opt/circleci/.nvm"
-                    [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"
-                    nvm install 14.17 --default
-                    node -v
-                    cp .env-dist .env
-                    npm install
-                    npm install --only=dev
-                    sudo chown -R  $USER:$USER .
-                    mkdir ./tests/integration/errorShots
-                    docker-compose -f tests/integration/docker-compose.yml up --build -d
-                    docker-compose -f tests/integration/docker-compose.yml exec --user root firefox npm run test:integration-headless-ci
+                  name: Run tests
+                  command: |
+                      cp .env-dist .env
+                      sudo chown -R  $USER:$USER .
+                      mkdir ./tests/integration/errorShots
+                      docker-compose --env-file .env -f tests/integration/docker-compose.yml up --build -d
+                      docker-compose --env-file .env -f tests/integration/docker-compose.yml exec --user root firefox npm run test:integration-headless-ci
            - run:
-                name: Capture Logs.
-                when: on_fail
-                command: |
-                    docker-compose -f tests/integration/docker-compose.yml logs server &> tests/integration/server.log
-                    docker-compose -f tests/integration/docker-compose.yml logs postgres &> tests/integration/postgres.log
-                    docker-compose -f tests/integration/docker-compose.yml logs firefox &> tests/integration/firefox.log
+                  name: Capture logs
+                  when: on_fail
+                  command: |
+                      docker-compose -f tests/integration/docker-compose.yml logs server &> tests/integration/server.log
+                      docker-compose -f tests/integration/docker-compose.yml logs postgres &> tests/integration/postgres.log
+                      docker-compose -f tests/integration/docker-compose.yml logs firefox &> tests/integration/firefox.log
            - store_artifacts:
-                path: ./tests/integration/errorShots/
+                  path: ./tests/integration/errorShots/
            - store_artifacts:
-                path: ./tests/integration/server.log
+                  path: ./tests/integration/server.log
            - store_artifacts:
-                path: ./tests/integration/postgres.log
+                  path: ./tests/integration/postgres.log
            - store_artifacts:
-                path: ./tests/integration/firefox.log
-
-    deploy:
-        docker:
-            - image: docker:18.06.3-ce
-        steps:
-            - setup_remote_docker
-            - restore_cache:
-                key: v1-{{.Branch}}
-            - run:
-                name: Restore Docker image cache
-                command: docker load -i /cache/docker.tar
-
-            - run:
-                name: Deploy to Dockerhub
-                command: |
-                    echo $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin
-                    # deploy main
-                    if [ "${CIRCLE_BRANCH}" == "main" ]; then
-                        docker tag blurts-server ${DOCKERHUB_REPO}:latest
-                        docker push ${DOCKERHUB_REPO}:latest
-                    elif  [ ! -z "${CIRCLE_TAG}" ]; then
-                        # deploy a release tag...
-                        echo "${DOCKERHUB_REPO}:${CIRCLE_TAG}"
-                        docker tag blurts-server "${DOCKERHUB_REPO}:${CIRCLE_TAG}"
-                        docker images
-                        docker push "${DOCKERHUB_REPO}:${CIRCLE_TAG}"
-                    fi
-    deploy_static:
-        docker:
-            - image: circleci/python:3.8
-        steps:
-            - checkout
-            - run:
-                name: Install AWS CLI
-                command: |
-                  sudo pip install --upgrade pip
-                  sudo pip install --upgrade awscli
-            - run:
-                name: Sync static content to S3
-                command: |
-                    ./.circleci/scripts/sync-static
+                  path: ./tests/integration/firefox.log

 workflows:
-  version: 2
-  build-test-deploy:
-    jobs:
-      - build:
-          filters:
-            tags:
-              only: /.*/
-      - deploy:
-          requires:
-            - build
-          filters:
-            tags:
-              only: /.*/
-            branches:
-                only: main
-      - deploy_static:
-          filters:
-            branches:
-              only: main
-      - integration-test:
-          requires:
-            - build
-
+    lint-and-test:
+        jobs:
+            - lint-js
+            - lint-css
+            - lint-audit
+            - lint-l10n
+            - unit-tests
+            - integration-tests
--- a/.env-dist
+++ b/.env-dist
@ -8,15 +8,15 @@ SESSION_DURATION_HOURS=48

 # see https://www.npmjs.com/package/mozlog
 # default values are for dev/debug
-MOZLOG_FMT="pretty"
-MOZLOG_LEVEL="debug"
+MOZLOG_FMT=pretty
+MOZLOG_LEVEL=debug

 # 1: disables the dockerflow endpoints
 # see: https://github.com/mozilla-services/Dockerflow#containerized-app-requirements
 DISABLE_DOCKERFLOW=

 # Database server
-DATABASE_URL="postgres://postgres@localhost:5432/blurts"
+DATABASE_URL=postgres://postgres@localhost:5432/blurts
 # How many seconds can unverified subscribers remain in the database
 DELETE_UNVERIFIED_SUBSCRIBERS_TIMER=86400

@ -24,47 +24,47 @@ DELETE_UNVERIFIED_SUBSCRIBERS_TIMER=86400
 PAGE_TOKEN_TIMER=0

 # Email server
-SMTP_URL=""
+SMTP_URL=
 # From: address used in emails
-EMAIL_FROM=""
+EMAIL_FROM=
 # https://docs.aws.amazon.com/ses/latest/DeveloperGuide/using-configuration-sets.html
-SES_CONFIG_SET=""
+SES_CONFIG_SET=
 # 1: only log messages coming back from SES
 SES_NOTIFICATION_LOG_ONLY=

 # Firefox Accounts OAuth
 # leave FXA_ENABLED empty to disable FXA
 FXA_ENABLED=
-FXA_SETTINGS_URL="https://accounts.stage.mozaws.net/settings"
+FXA_SETTINGS_URL=https://accounts.stage.mozaws.net/settings
 OAUTH_CLIENT_ID=edd29a80019d61a1
 OAUTH_CLIENT_SECRET=get-this-from-groovecoder-or-fxmonitor-engineering
-OAUTH_AUTHORIZATION_URI="https://oauth.stage.mozaws.net/v1/authorization"
-OAUTH_PROFILE_URI="https://profile.stage.mozaws.net/v1/profile"
-OAUTH_TOKEN_URI="https://oauth.stage.mozaws.net/v1/token"
+OAUTH_AUTHORIZATION_URI=https://oauth.stage.mozaws.net/v1/authorization
+OAUTH_PROFILE_URI=https://profile.stage.mozaws.net/v1/profile
+OAUTH_TOKEN_URI=https://oauth.stage.mozaws.net/v1/token

 # HIBP API for breach data
 # How many seconds to wait before refreshing upstream breach data from HIBP
 HIBP_RELOAD_BREACHES_TIMER=600
 # HIBP API for range search and subscription
-HIBP_KANON_API_ROOT="https://api.haveibeenpwned.com"
+HIBP_KANON_API_ROOT=https://api.haveibeenpwned.com
 HIBP_KANON_API_TOKEN=
-HIBP_API_ROOT="https://haveibeenpwned.com/api/v2"
+HIBP_API_ROOT=https://haveibeenpwned.com/api/v2
 HIBP_API_TOKEN=
 # How many milliseconds to wait before retrying an HIBP request
 HIBP_THROTTLE_DELAY=2000
 # Max number of times to try an HIBP request before throwing error
 HIBP_THROTTLE_MAX_TRIES=5
 # Authorization token for HIBP to present to /hibp/notify endpoint
-HIBP_NOTIFY_TOKEN="unsafe-default-token-for-dev"
+HIBP_NOTIFY_TOKEN=unsafe-default-token-for-dev

 # Firefox Remote Settings
-FX_REMOTE_SETTINGS_WRITER_SERVER="https://settings-writer.prod.mozaws.net/v1"
-FX_REMOTE_SETTINGS_WRITER_USER=""
-FX_REMOTE_SETTINGS_WRITER_PASS=""
+FX_REMOTE_SETTINGS_WRITER_SERVER=https://settings-writer.prod.mozaws.net/v1
+FX_REMOTE_SETTINGS_WRITER_USER=
+FX_REMOTE_SETTINGS_WRITER_PASS=

 # DSN for Sentry error and event capturing
 # e.g., SENTRY_DSN=https://{key}@sentry.prod.mozaws.net/408
-SENTRY_DSN=""
+SENTRY_DSN=

 BREACH_RESOLUTION_ENABLED=1
 PRODUCT_PROMOS_ENABLED=1
--- a/.travis.yml
+++ b/.travis.yml
@ -1,39 +0,0 @@
-dist: xenial
-services:
-  - postgresql
-addons:
-  postgresql: 9.6
-
-matrix:
-    include:
-        - name: "js lint"
-          language: node_js
-          node_js: 14
-          script: "npm run lint:js"
-        - name: "css lint"
-          language: node_js
-          node_js: 14
-          script: "npm run lint:css"
-        - name: "npm audit"
-          language: node_js
-          node_js: 14
-          script: "npm run lint:audit"
-        - name: "tests"
-          language: node_js
-          node_js: 14
-          env:
-              - NODE_ENV=tests
-          install:
-              - npm install
-          before_script:
-              - cp .env-dist .env
-              - createdb test-blurts
-              - npm install npm@latest -g
-
-        - name: "l10n lint"
-          language: python
-          install:
-              - sudo pip install compare-locales
-          script:
-              - moz-l10n-lint l10n.toml
-              - compare-locales l10n.toml . `ls locales`
--- a/18
+++ b/18
@ -1,8 +1,14 @@
-FROM node:14-alpine
+FROM node:14.17-alpine
+
+ARG dev
+RUN if [[ -n "$dev" ]] ; then \ 
+    echo Adding glibc packages missing from Alpine to satisfy node-canvas, a @wdio dependency ; \
+    apk add --update --no-cache make g++ jpeg-dev cairo-dev pango-dev ; \
+    # https://github.com/node-gfx/node-canvas-prebuilt/issues/77
+fi

 RUN addgroup -g 10001 app && \
    adduser -D -G app -h /app -u 10001 app
-RUN npm update -g
 RUN rm -rf /tmp/*

 WORKDIR /app
@ -12,7 +18,13 @@ USER app
 COPY package.json package.json
 COPY package-lock.json package-lock.json

-RUN npm install --production && rm -rf ~app/.npm /tmp/*
+RUN if [[ -n "$dev" ]] ; then \
+    echo Using npm ci to install production AND development dependencies. ; \
+    npm ci --audit=false && rm -rf ~app/.npm /tmp/* ; \
+else \
+    echo Installing production dependencies only ; \
+    npm install --production && rm -rf ~app/.npm /tmp/* ; \
+fi

 COPY --chown=app:app . /app

--- a/README.md
+++ b/README.md
@ -123,7 +123,11 @@ the `OAUTH_CLIENT_SECRET` value from someone in #fxmonitor-engineering.

 ## Testing

-The full test suite can be run via `npm test`.
+The full test suite can be run via `npm test`.  
+
+At the end of a test suite run, coverage info will be sent to [Coveralls](https://coveralls.io/) to assess coverage changes and provide a neat badge.  For this step to complete locally, you need a root `.coveralls.yml` which contains a token – get this from another member of the Monitor team.  Alternatively, without the token you can simply ignore the `coveralls` error.  
+
+*TODO:* Disable Coveralls step for local testing?

 ### Individual tests

--- a/log.js
+++ b/log.js
@ -4,7 +4,6 @@ const mozlog = require("mozlog");

 const AppConstants = require("./app-constants");

-
 const log = mozlog({
  app: "fx-monitor",
  level: AppConstants.MOZLOG_LEVEL,
--- a/package-lock.json
+++ b/package-lock.json
--- a/tests/integration/Dockerfile.integration-test.yml
+++ b/tests/integration/Dockerfile.integration-test.yml
@ -1,5 +1,5 @@
 # Build app for integration tests
-FROM node:14
+FROM node:14.17

 RUN useradd -d /app -u 1001 -g 1000 app

@ -17,7 +17,7 @@ USER app
 COPY package.json package.json
 COPY package-lock.json package-lock.json

-RUN npm install && rm -rf ~app/.npm /tmp/*
+RUN npm ci --audit=false && rm -rf ~app/.npm /tmp/*

 COPY --chown=app:root . /app

--- a/tests/integration/docker-compose.yml
+++ b/tests/integration/docker-compose.yml
@ -49,7 +49,5 @@ services:
      - ../../:/code
    expose:
      - "4444"
-    ports:
-      - "5900:5900"
    shm_size: 2g
    network_mode: host