Move from TravisCI to CircleCI 2.0

.circleci/config.yml

@@ -0,0 +1,179 @@
+####################
+# CircleCI configuration reference:
+#   https://circleci.com/docs/2.0/configuration-reference
+####################
+# CircleCI built-in environment variables:
+#   https://circleci.com/docs/2.0/env-vars/#built-in-environment-variables
+####################
+
+
+####################
+# Templates: see "anchors" in https://learnxinyminutes.com/docs/yaml/
+####################
+
+# See available image tags at https://hub.docker.com/r/mozilla/sbt/
+sbt_image: &sbt_image mozilla/sbt:8u171_1.1.6
+
+# The ~/.sbt directory holds any sbt or Scala versions requested by previous builds.
+# The ~/.ivy2 directory holds Java and Scala dependencies declared in the build.
+# Caching these two can significantly reduce sbt startup time.
+save_cache_settings: &save_cache_settings
+  key: telemetry-streaming-{{ .Branch }}-{{ .Environment.CIRCLE_WORKFLOW_ID }}
+  paths:
+    - ~/.ivy2
+    - ~/.sbt
+
+prep_cache_settings: &prep_cache_settings
+  name: Clean directories to restore
+  command: |
+    rm -rf ~/.ivy2 ~/.sbt
+
+restore_cache_settings: &restore_cache_settings
+  keys:
+    - telemetry-streaming-{{ .Branch }}-{{ .Environment.CIRCLE_WORKFLOW_ID }}
+    - telemetry-streaming-{{ .Branch }}
+    - telemetry-streaming-
+
+common_settings: &common_settings
+  docker:
+    - image: *sbt_image
+  working_directory: /telemetry-streaming
+
+# CircleCI generally only triggers builds on commits to the main repository,
+# so PRs coming from branches of the main repo simply reference builds for existing commits
+# (CIRCLE_BRANCH=branchname and CIRCLE_PR_NUMBER is unset);
+# The only time PRs will trigger a build is when the PR is referencing a fork
+# (CIRCLE_BRANCH=pull/XXX and CIRCLE_PR_NUMBER=XXX).
+early_return_for_forked_pull_requests: &early_return_for_forked_pull_requests
+  name: Early return if this build is from a forked PR
+  command: |
+    if [ -n "$CIRCLE_PR_NUMBER" ]; then
+      echo "Nothing to upload for forked PRs, so marking this step successful"
+      circleci step halt
+    fi
+
+early_return_for_skip_tests: &early_return_for_skip_tests
+  name: Early return if the latest non-merge commit message contains "[skip-tests]"
+  command: |
+    COMMIT_MESSAGE=$(git log --format=%B --no-merges -n 1)
+    if [[ "$COMMIT_MESSAGE" =~ "[skip-tests]" ]]; then
+      echo "Skipping tests due to [skip-tests] flag, so marking this step successful"
+      circleci step halt
+    fi
+
+
+####################
+# Jobs: see https://circleci.com/docs/2.0/jobs-steps/
+####################
+
+version: 2
+jobs:
+
+  test:
+    <<: *common_settings
+    steps:
+      - checkout
+      - run: *early_return_for_skip_tests
+      - run: *prep_cache_settings
+      - restore_cache: *restore_cache_settings
+      - run:
+          name: Run tests that don't rely on Kafka
+          command: |
+            TEST_OPTIONS='set testOptions in Test := Seq(Tests.Argument("-l", "DockerComposeTag"))'
+            sbt "$TEST_OPTIONS" coverage test coverageReport
+      - run:
+          name: Submit code coverage data
+          command: |
+            bash <(curl -s https://codecov.io/bash)
+
+  # Instead of using the docker-compose config, we use CircleCI's built-in functionality for
+  # spinning up "secondary" docker images and making them available in the same network. See:
+  # https://circleci.com/docs/2.0/databases/
+  kafka-test:
+    <<: *common_settings
+    docker:
+      - image: *sbt_image
+      - image: wurstmeister/zookeeper
+      - image: wurstmeister/kafka
+        environment:
+          - KAFKA_ZOOKEEPER_CONNECT: localhost:2181
+          - KAFKA_ADVERTISED_HOST_NAME: localhost
+    steps:
+      - checkout
+      - run: *early_return_for_skip_tests
+      - run: *prep_cache_settings
+      - restore_cache: *restore_cache_settings
+      - run:
+          name: Run tests that rely on Kafka
+          command: |
+            TEST_OPTIONS='set testOptions in Test := Seq(Tests.Argument("-n", "DockerComposeTag"))'
+            sbt "$TEST_OPTIONS" coverage test coverageReport
+      - run:
+          name: Submit code coverage data
+          command: |
+            bash <(curl -s https://codecov.io/bash)
+
+  lint:
+    <<: *common_settings
+    steps:
+      - checkout
+      - run: *early_return_for_skip_tests
+      - run: *prep_cache_settings
+      - restore_cache: *restore_cache_settings
+      - run:
+          name: Scalastyle
+          command: |
+            sbt scalastyle test:scalastyle
+
+  # Assemble an uberjar for the deploy step.
+  assembly:
+    <<: *common_settings
+    steps:
+      - run: *early_return_for_forked_pull_requests
+      - checkout
+      - run:
+          name: Assembly
+          command: |
+            sbt assembly
+      - save_cache: *save_cache_settings
+      - persist_to_workspace:
+          root: target
+          paths:
+            - scala-2.11/telemetry-streaming-assembly-0.1-SNAPSHOT.jar
+
+  # Uses a "workspace" to get access to the assembled uberjar from the previous stage.
+  deploy:
+    docker:
+      - image: *sbt_image
+    working_directory: /telemetry-streaming
+    steps:
+      - run: *early_return_for_forked_pull_requests
+      - checkout
+      - attach_workspace:
+          at: /telemetry-streaming/target
+      - run:
+          name: Upload
+          command: |
+            export JAR=target/scala-2.11/telemetry-streaming-assembly-0.1-SNAPSHOT.jar
+            # We pin to a specific commit to avoid unexpected updates to the build code.
+            curl -sL https://raw.githubusercontent.com/mozilla/telemetry-batch-view/912adad51f9938c96694edc7de89fe9b6093e7e7/.circleci/deploy.sh | bash
+
+
+####################
+# Workflows: see https://circleci.com/docs/2.0/workflows/
+####################
+
+workflows:
+  version: 2
+  build:
+    jobs:
+      - lint
+      - test
+      - kafka-test
+      - assembly
+      - deploy:
+          requires:
+            - lint
+            - test
+            - kafka-test
+            - assembly

.jvmopts

@@ -0,0 +1,5 @@
+# See 'sbt -help' for docs on how .jvmopts and .sbtopts are used.
+-Xms512M
+-Xmx2G
+-XX:ReservedCodeCacheSize=128M
+-XX:+CMSClassUnloadingEnabled

.travis.yml

@@ -1,64 +0,0 @@
-sudo: required
-
-language: scala
-
-jdk:
-- oraclejdk8
-
-addons:
-  apt:
-    packages:
-    - net-tools
-
-cache:
-  directories:
-  - "$HOME/.ivy2/cache"
-  - "$HOME/.sbt/boot/"
-
-before_cache:
-  - find $HOME/.ivy2 -name "ivydata-*.properties" -delete
-  - find $HOME/.sbt -name "*.lock" -delete
-
-services:
-  - docker
-
-scala:
-  - 2.11.8
-
-env:
-  - DOCKER_DIR="../docker/"
-
-script:
-  - export TEST_SKIP_REGEX="\[skip-tests\]"
-  - if [[ (! $TRAVIS_COMMIT_MESSAGE =~ $TEST_SKIP_REGEX) || -n $TRAVIS_TAG || $TRAVIS_BRANCH == "master" ]];
-      then sbt ci;
-    else
-      echo "Skipping tests";
-    fi
-
-after_success:
-  - bash <(curl -s https://codecov.io/bash)
-
-before_deploy:
-  - wget https://raw.githubusercontent.com/mozilla/telemetry-batch-view/spark-2.3.0/deploy.sh
-  - export JAR="target/scala-2.11/telemetry-streaming-assembly-0.1-SNAPSHOT.jar"
-  - sbt assembly
-  - git config --local user.name "Auto Deployer"
-  - git config --local user.email "telemetry-alerts@mozilla.com"
-  - git tag "$(date +'%Y%m%d%H%M%S')-$(git log --format=%h -1)"
-
-deploy:
-  - provider: releases
-    skip_cleanup: true
-    api_key:
-      secure: Kac5/sXg9TM8cBHrwNch78UiVTl4aA2KQ9jnv0vipUzi341XfmO3EbdEnSCWN+ew39Pc+tAR+nOB+e/AHbOyFjJ5gggA3Z/3UWPYse/8iIPcRs8GC55KfprDrVkD8HO3pPzOXOyDtn64Y6z3cQXGj110QKfPjG6v+LHzNDSl1oxZB7f8bnoNEom5WxZCMAvZO3odbrnKPP333gPpjtmXfe24WlPqBMg3FN+RJMy1lz+bjlDB0hd1qbVdUO3DYivodcn/q1Sdp98EipwCWnOq3JqrmfDhf9rRL1UnSC+WsbcxJL73ee36EkNORSTYVFzhfoSRmNlhqNgGI8DdDhO654fKecXPiWDHyZw+cF/DQHsvmSk5zr5VobeqyByM9cqFj+5gs3RN3CI2UaUbceRwuerwpuFCAVt2WHSw2heYeOlmoLNX2+HFYPCXFJOkzQ5pZl5fEsc280riU/9wlvNfLhKTIJNyP3n3OlU/YlU0FYfR2jF637KAhLX9z8LxY1aEcdcAI7sWCz5jyMGRwvCS7prRneg1ouPn+bUSab7+GMi8F3Rdf1qWZx8IivpXiOw3MubsLLBeXep+PbE70n6AFB68ThgKEAhJsJpqtZKbqpz3w+w6iOe9Ig85P+WdXTK7zC4itWzZfEihk8TTGzerlSHgpMR9lBgvkzJkFY3MQqI=
-    on:
-      repo: mozilla/telemetry-streaming
-      condition: $TRAVIS_COMMIT_MESSAGE == *"[auto-deploy]"*
-      branch: master
-  - provider: script
-    script: bash deploy.sh
-    skip_cleanup: true
-    on:
-      repo: mozilla/telemetry-streaming
-      all_branches: true

README.md

@@ -32,7 +32,8 @@ the source code and running the tests via sbt. Some common invocations for sbt:
 * `sbt "testOnly *ErrorAgg* -- -z version"  # run the tests only for packages matching ErrorAgg, limited to test cases with "version" in them`
 * `sbt dockerComposeTest  # run the docker compose tests (slow)`
 * `sbt "dockerComposeTest -tags:DockerComposeTag" # run only tests with DockerComposeTag (while using docker)`
-* `sbt ci  # run all tests`
+* `sbt scalastyle test:scalastyle  # run linter`
+* `sbt ci  # run the full set of continuous integration tests`
 
 Some tests need Kafka to run. If one prefers to run them via IDE, it's required to run the test cluster:
 ```bash

build.sbt

@@ -74,9 +74,6 @@ test in assembly := {}
 // Add configs to resources
 unmanagedResourceDirectories in Compile += baseDirectory.value / "configs"
 
-// Default SBT settings suck
-javaOptions ++= Seq("-Xms512M", "-Xmx2048M", "-XX:MaxPermSize=2048M", "-XX:+CMSClassUnloadingEnabled")
-
 parallelExecution in Test := false
 
 scalacOptions ++= Seq(
@@ -99,7 +96,8 @@ assemblyShadeRules in assembly := Seq(
   ShadeRule.rename("com.trueaccord.scalapb.**" -> "shadescalapb.@1").inAll
 )
 
-addCommandAlias("ci", ";clean ;compile ;test:compile ;scalastyle ;test:scalastyle ;coverage ;dockerComposeTest ;coverageReport")
+// Shorthand for locally running the full set of tests that would run in continuous integration.
+addCommandAlias("ci", ";clean ;compile ;test:compile ;scalastyle ;test:scalastyle ;dockerComposeTest")
 
 val scalaStyleConfigUrl = Some(url("https://raw.githubusercontent.com/mozilla/moztelemetry/master/scalastyle-config.xml"))
 (scalastyleConfigUrl in Compile) := scalaStyleConfigUrl