Fix #392: Add mock-recipe-server project.

The mock-recipe-server generates static files that simulate the Normandy
API, which are then uploaded to S3 and used for testing the recipe client.

bin/ci/runner.sh

@@ -10,6 +10,7 @@ BASE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && cd ../.. && pwd)"
 PROJECTS=(
     recipe-client-addon
     recipe-server
+    mock-recipe-server
     lints
     compose
     eslint-config-normandy

circle.yml

@@ -1,7 +1,14 @@
 machine:
+  pre:
+    # Install CircleCI's fork for Docker 1.10.0
+    - curl -sSL https://s3.amazonaws.com/circle-downloads/install-circleci-docker.sh | bash -s -- 1.10.0
+    - chmod -R 777 $CIRCLE_TEST_REPORTS $CIRCLE_ARTIFACTS
   services:
     - postgresql
     - docker
+  environment:
+    MOCK_SERVER_DOMAIN: https://normandy-mock.dev.mozaws.net
+    MOCK_SERVER_ARTIFACTS: "${CIRCLE_ARTIFACTS}/mock-recipe-server"
 
 dependencies:
   pre:
@@ -9,9 +16,11 @@ dependencies:
   override:
     - ./bin/ci/runner.sh dependencies
 
+compile:
+  override:
+    - ./bin/ci/runner.sh compile
+
 test:
-  pre:
-    - chmod -R 777 $CIRCLE_TEST_REPORTS
   override:
     - ./bin/ci/runner.sh test
   post:

compose/docker-compose.yml

@@ -5,19 +5,12 @@ services:
     image: postgres:9.5.2
 
   normandy:
-    image: mozilla/normandy:latest
+    extends:
+      file: normandy-base.yml
+      service: normandy-base
     links:
       - database
       - autograph
-    environment:
-      DATABASE_URL: "postgres://postgres@database/postgres"
-      DJANGO_CONFIGURATION: ProductionInsecure
-      DJANGO_AUTOGRAPH_URL: http://autograph:8000/
-      # From etc/autograph.yaml
-      DJANGO_AUTOGRAPH_HAWK_ID: normandev
-      DJANGO_AUTOGRAPH_HAWK_SECRET_KEY: 3dhoaupudifjjvm7xznd9bn73159xn3xwr77b61kzdjwzzsjts
-      DJANGO_CAN_EDIT_ACTIONS_IN_USE: "true"
-    stop_signal: SIGKILL
 
   proxy:
     image: nginx:1.9.14

compose/normandy-base.yml

@@ -0,0 +1,14 @@
+version: '2'
+
+services:
+  normandy-base:
+    image: mozilla/normandy:latest
+    environment:
+      DATABASE_URL: "postgres://postgres@database/postgres"
+      DJANGO_CONFIGURATION: ProductionInsecure
+      DJANGO_AUTOGRAPH_URL: http://autograph:8000/
+      # From etc/autograph.yaml
+      DJANGO_AUTOGRAPH_HAWK_ID: normandev
+      DJANGO_AUTOGRAPH_HAWK_SECRET_KEY: 3dhoaupudifjjvm7xznd9bn73159xn3xwr77b61kzdjwzzsjts
+      DJANGO_CAN_EDIT_ACTIONS_IN_USE: "true"
+    stop_signal: SIGKILL

docs/dev/mock-recipe-server.rst

@@ -0,0 +1,30 @@
+Mock Recipe Server
+==================
+The mock recipe server is a set of static files that mimic the recipe server's
+API to allow for testing of the :ref:`runtime` against test data easily.
+
+Automation is typically set up to run this automatically, but you can manually
+generate the files as well.
+
+Prerequisites
+-------------
+- `Docker <https://docs.docker.com/engine/installation/>`_
+- `docker-compose <https://docs.docker.com/compose/>`_
+
+Generating Files
+----------------
+1. If this is your first time generating the files, you must run the setup
+   script to create the docker containers used for building the files:
+
+   .. code-block:: bash
+
+      # From the root of the repo
+      cd mock-recipe-server
+      ./bin/setup.sh
+
+2. Run the ``generate.sh`` file, passing in the path where you wish to save the
+   generated files, and the domain to use for absolute URLs:
+
+   .. code-block:: bash
+
+      ./bin/generate.sh /path/to/directory https://normandy-mock.dev.mozaws.net

mock-recipe-server/README.md

@@ -0,0 +1,12 @@
+# Mock Recipe Server
+
+This project generates a set of static HTML files that mimic the
+[recipe server][] API for a set of interesting test cases. They're built and
+uploaded to S3 by our CI jobs to help QA test the [recipe client][] against
+test data.
+
+See the [documentation][] for more info.
+
+[recipe server]: https://github.com/mozilla/normandy/tree/master/recipe-server
+[recipe client]: https://github.com/mozilla/normandy/tree/master/recipe-client-addon
+[documentation]: http://normandy.readthedocs.io/en/latest/dev/mock-recipe-server.html

mock-recipe-server/bin/ci/compile

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -eu
+
+# Create artifact directory
+mkdir -p -m 777 $MOCK_SERVER_ARTIFACTS
+
+# Generate mock server files
+./bin/generate.sh $MOCK_SERVER_ARTIFACTS $MOCK_SERVER_DOMAIN

mock-recipe-server/bin/ci/dependencies

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -eu
+
+# Install docker-compose
+sudo pip install docker-compose
+
+# Setup normandy-compose used in build
+./bin/setup.sh

mock-recipe-server/bin/ci/deploy

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+set -eu
+
+./bin/upload.sh $MOCK_SERVER_ARTIFACTS $MOCK_SERVER_S3_BUCKET

mock-recipe-server/bin/generate.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+set -eu
+
+REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && cd ../.. && pwd)"
+
+# MOCK_SERVER_ARTIFACTS is exported so the docker-compose config can
+# use it to mount the artifact volume.
+export MOCK_SERVER_ARTIFACTS=$1
+MOCK_SERVER_DOMAIN=$2
+
+# Generate mock server files
+echo "Generating mock server files"
+docker-compose \
+  -p mockrecipeserver \
+  -f "$REPO_DIR/compose/docker-compose.yml" \
+  -f "$REPO_DIR/mock-recipe-server/docker-compose.yml" \
+  run \
+  testgen /mock-server/generate.py /build $MOCK_SERVER_DOMAIN

mock-recipe-server/bin/setup.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+set -eu
+
+REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && cd ../.. && pwd)"
+
+# Setup recipe-server container
+echo "Initializing mock server containers"
+pushd "$REPO_DIR/compose"
+./bin/genkeys.sh
+docker-compose -p mockrecipeserver run normandy ./manage.py migrate
+docker-compose -p mockrecipeserver run normandy ./manage.py update_actions
+popd

mock-recipe-server/bin/upload.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -eu
+
+BUILD_DIR=$1
+S3_BUCKET=$2
+
+aws s3 rm --recursive "s3://$S3_BUCKET/"
+aws s3 cp --recursive "$BUILD_DIR" "s3://$S3_BUCKET/" --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers

mock-recipe-server/docker-compose.yml

@@ -0,0 +1,14 @@
+version: '2'
+
+services:
+  testgen:
+    extends:
+      file: normandy-base.yml
+      service: normandy-base
+    links:
+      - database
+      - autograph
+      - proxy
+    volumes: # Relative to /compose/docker-compose.yml
+      - ../mock-recipe-server:/mock-server
+      - "${MOCK_SERVER_ARTIFACTS}:/build"

mock-recipe-server/fixtures.py

@@ -0,0 +1,54 @@
+from normandy.recipes.models import Action, Recipe
+from normandy.recipes.tests import ClientFactory, RecipeFactory
+
+
+def console_log_action():
+    return Action.objects.get(name='console-log')
+
+
+def get_fixtures():
+    """Return all defined fixtures."""
+    return [FixtureClass() for FixtureClass in Fixture.__subclasses__()]
+
+
+class Fixture(object):
+    """
+    Collection of data for a specific manual test case. Includes both
+    data to be loaded in the database, and data needed to represent
+    API responses that don't use the database.
+    """
+    @property
+    def name(self):
+        return self.__class__.__name__
+
+    def load(self):
+        """
+        Clear out all existing recipes and load this fixture's data in
+        its place.
+        """
+        Recipe.objects.all().delete()
+        self.load_data()
+
+    def load_data(self):
+        """
+        Create data specific to this fixture. Individual fixtures must
+        override this.
+        """
+        raise NotImplementedError()
+
+    def client(self):
+        """
+        Return a Client object that the client classification endpoint
+        should render for this fixture.
+        """
+        return ClientFactory()
+
+
+class ConsoleLogBasic(Fixture):
+    """A single console-log action."""
+    def load_data(self):
+        RecipeFactory(
+            action=console_log_action(),
+            arguments={'message': 'Test Message'},
+            filter_expression='true',
+        )

mock-recipe-server/generate.py

@@ -0,0 +1,145 @@
+#!/usr/bin/env python
+"""
+Script for generating static HTML files suitable for hosting on a static
+host (like AWS S3) that mock out the Normandy recipe server API for
+particular test cases.
+"""
+import json
+import os
+import sys
+from pathlib import Path
+from urllib.parse import urljoin, urlparse, urlunparse
+
+import configurations
+import requests
+
+
+# Add normandy to the import path and setup Django stuff.
+sys.path.insert(0, '/app')
+sys.path.insert(0, '/mock-server')
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "normandy.settings")
+configurations.setup()
+
+
+# Now that Django is set up we can import Django things.
+from normandy.base.utils import canonical_json_dumps  # noqa
+from normandy.recipes.api.serializers import ClientSerializer  # noqa
+from normandy.recipes.models import Action  # noqa
+
+from fixtures import get_fixtures  # noqa
+
+
+class APIPath(object):
+    """Represents an API URL that is mirrored on the filesystem."""
+    def __init__(self, base_path, base_url, segments=None):
+        self.base_path = base_path
+        self.base_url = base_url
+        self.segments = segments or []
+
+    @property
+    def url(self):
+        """Generate the current URL string."""
+        return urljoin(self.base_url, '/'.join(self.segments) + '/')
+
+    @property
+    def path(self):
+        """Generate a Path object for the current URL."""
+        return Path(self.base_path, *self.segments, 'index.html')
+
+    def add(self, *paths):
+        """Add segments to the current URL."""
+        return APIPath(self.base_path, self.base_url, self.segments + list(paths))
+
+    def fetch(self):
+        """Fetch the response text for the current URL."""
+        response = requests.get(self.url, verify=False)
+        response.raise_for_status()
+        return response.text
+
+    def save(self, data=None):
+        """
+        Save data to the filesystem for the current URL.
+
+        :param data:
+            File contents to save. If not given, the current URL will
+            be remotely fetched and saved.
+        """
+        data = data or self.fetch()
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        with self.path.open(mode='w') as f:
+            f.write(data)
+
+
+def main():
+    """
+    Load each defined fixture from fixtures.py and save the state of the API
+    after each fixture is loaded.
+    """
+    build_path = Path(sys.argv[1])
+    domain = sys.argv[2]
+    for fixture in get_fixtures():
+        fixture.load()
+        fixture_api_path = APIPath(build_path / fixture.name, 'https://proxy:8443')
+        serialize_api(fixture, fixture_api_path, domain)
+
+
+def serialize_api(fixture, api_path, domain):
+    """
+    Fetch API responses from the service and save them to the
+    filesystem.
+
+    :param fixture:
+        Fixture object that was last loaded to provide a client object
+        to serialize to the client classification endpoint.
+    :param api_path:
+        APIPath object for the root URL and path to fetch and save
+        responses from and to.
+    :param domain:
+        Protocol and domain to use for absolute URLs in the serialized
+        API.
+    """
+    root_path = api_path.add('api', 'v1')
+
+    # Recipe endpoints
+    root_path.add('recipe').save()
+    root_path.add('recipe', 'signed').save()
+
+    # Client classification (manually rendered as canonical json)
+    client = fixture.client()
+    client_data = ClientSerializer(client).data
+    client_json = canonical_json_dumps(client_data)
+    root_path.add('classify_client').save(client_json)
+
+    for action in Action.objects.all():
+        # Action
+        action_path = root_path.add('action', action.name)
+
+        action_data = json.loads(action_path.fetch())
+        new_url = update_url(action_data['implementation_url'], fixture, domain)
+        action_data['implementation_url'] = new_url
+        action_json = canonical_json_dumps(action_data)
+        action_path.save(action_json)
+
+        # Action implementation
+        action_path.add('implementation', action.implementation_hash).save()
+
+
+def update_url(url, fixture, domain):
+    """
+    Modify the URL to use the domain and to add the name of the given
+    fixture as the first path segment.
+    """
+    parsed_url = urlparse(url)
+    parsed_domain = urlparse(domain)
+    return urlunparse((
+        parsed_domain.scheme,
+        parsed_domain.netloc,
+        '/' + fixture.name + parsed_url.path,
+        parsed_url.params,
+        parsed_url.query,
+        parsed_url.fragment,
+    ))
+
+
+if __name__ == '__main__':
+    main()