From 4cffdfb29882ff271efbbc1e729a0bb70f56c2fe Mon Sep 17 00:00:00 2001 From: Sunah Suh Date: Wed, 27 Jun 2018 11:15:31 -0500 Subject: [PATCH] Add Event Ping case class --- .../mozilla/telemetry/pings/EventPing.scala | 66 ++++++++++ .../com/mozilla/telemetry/TestUtils.scala | 118 ++++++++++++++++++ .../telemetry/pings/TestEventPing.scala | 32 +++++ 3 files changed, 216 insertions(+) create mode 100644 src/main/scala/com/mozilla/telemetry/pings/EventPing.scala create mode 100644 src/test/scala/com/mozilla/telemetry/pings/TestEventPing.scala diff --git a/src/main/scala/com/mozilla/telemetry/pings/EventPing.scala b/src/main/scala/com/mozilla/telemetry/pings/EventPing.scala new file mode 100644 index 0000000..40604f9 --- /dev/null +++ b/src/main/scala/com/mozilla/telemetry/pings/EventPing.scala @@ -0,0 +1,66 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +package com.mozilla.telemetry.pings + +import com.mozilla.telemetry.heka.Message +import com.mozilla.telemetry.pings.main.Processes +import com.mozilla.telemetry.pings.Ping.messageToPing +import org.json4s.DefaultFormats +import org.json4s.JsonAST.{JNothing, JValue} + +case class EventPingPayload(events: Map[String, JValue], + lostEventsCount: Int, + processStartTimestamp: Long, + reason: String, + sessionId: String, + subsessionId: String) + +case class EventPing(application: Application, + meta: Meta, + payload: EventPingPayload) + extends Ping with HasEnvironment with HasApplication with SendsToAmplitude { + val processEventMap: Map[String, Seq[Event]] = Processes.names.map( + p => p -> Ping.extractEvents(payload.events.getOrElse(p, JNothing), List(Nil)) + ).toMap + + val events: Seq[Event] = processEventMap.flatMap(_._2).toSeq + + def getClientId: Option[String] = meta.clientId + + def sessionStart: Long = payload.processStartTimestamp + + def getCreated: Option[Long] = meta.creationTimestamp.map(t => (t / 1e9).toLong) + + def getLocale: Option[String] = meta.`environment.settings`.map(_.locale).flatten + + def getMSStyleExperiments: Option[Map[String, String]] = { + val experimentsArray = getExperiments + experimentsArray.length match { + case 0 => None + case _ => Some(experimentsArray.flatMap { + case(Some(exp), Some(branch)) => Some(exp -> branch) + case _ => None + }.toMap) + } + } +} + + +object EventPing { + def apply(message: Message): EventPing = { + implicit val formats = DefaultFormats + val jsonFieldNames = List( + "environment.build", + "environment.settings", + "environment.system", + "environment.profile", + "environment.addons", + "environment.experiments" + ) + + val ping = messageToPing(message, jsonFieldNames) + ping.extract[EventPing] + } +} + diff --git a/src/test/scala/com/mozilla/telemetry/TestUtils.scala b/src/test/scala/com/mozilla/telemetry/TestUtils.scala index 7aa529c..a3e19fe 100644 --- a/src/test/scala/com/mozilla/telemetry/TestUtils.scala +++ b/src/test/scala/com/mozilla/telemetry/TestUtils.scala @@ -367,6 +367,124 @@ object TestUtils { } // scalastyle:on methodLength + def generateEventMessages(size: Int, + fieldsOverride: Option[Map[String, Any]] = None, + timestamp: Option[Long] = None): Seq[Message] = { + val defaultMap = Map( + "clientId" -> "client1", + "docType" -> "event", + "documentId" -> "an_id", + "normalizedChannel" -> defaultFirefoxApplication.channel, + "appName" -> defaultFirefoxApplication.name, + "appVersion" -> defaultFirefoxApplication.version.toDouble, + "displayVersion" -> defaultFirefoxApplication.displayVersion.orNull, + "appBuildId" -> defaultFirefoxApplication.buildId, + "geoCountry" -> "IT", + "os" -> "Linux", + "submissionDate" -> "20170101", + "environment.build" -> + s""" + |{ + | "architecture": "${defaultFirefoxApplication.architecture}", + | "buildId": "${defaultFirefoxApplication.buildId}", + | "version": "${defaultFirefoxApplication.version}" + |}""".stripMargin, + "environment.system" -> + """ + |{ + | "os": {"name": "Linux", "version": "42"} + |}""".stripMargin, + "environment.addons" -> + """ + |{ + | "activeExperiment": {"id": "experiment1", "branch": "control"}, + | "activeAddons": {"my-addon": {"isSystem": true}}, + | "theme": {"id": "firefox-compact-dark@mozilla.org"} + |}""".stripMargin, + "environment.profile" -> + s""" + |{ + | "creationDate": ${todayDays-70} + | }""".stripMargin, + "environment.experiments" -> + """ + |{ + | "experiment2": {"branch": "chaos"} + |}""".stripMargin + ) + val outputMap = fieldsOverride match { + case Some(m) => defaultMap ++ m + case _ => defaultMap + } + val applicationJson = compact(render(Extraction.decompose(defaultFirefoxApplication))) + val payload = + s""" + | "reason": "periodic", + | "processStartTimestamp": 1530291900000, + | "sessionId": "dd302e9d-569b-4058-b7e8-02b2ff83522c", + | "subsessionId": "79a2728f-af12-4ed3-b56d-0531a03c2f26", + | "lostEventsCount": 0, + | "events": { + | "parent": [ + | [ + | 4118829, + | "activity_stream", + | "end", + | "session", + | "909", + | { + | "addon_version": "2018.06.22.1337-8d599e17", + | "user_prefs": "63", + | "session_id": "{78fe2428-15fb-4448-b517-cbb85f22def0}", + | "page": "about:newtab" + | } + | ], + | [ + | 4203540, + | "normandy", + | "enroll", + | "preference_study", + | "awesome-experiment", + | { + | "branch": "control", + | "experimentType": "exp" + | } + | ] + | ], + | "dynamic": [ + | [ + | 4203541, + | "test", + | "no", + | "string_value", + | null, + | { + | "hello": "world" + | } + | ], + | [ + | 4203542, + | "test", + | "no", + | "extras" + | ] + | ] + | } + """.stripMargin + 1.to(size) map { index => + RichMessage(s"event-$index", + outputMap, + Some( + s""" + |{ + | "payload": { $payload }, + | "application": $applicationJson + |}""".stripMargin), + timestamp=timestamp.getOrElse(testTimestampNano) + ) + } + } + abstract class AppType case object Firefox extends AppType case object Fennec extends AppType diff --git a/src/test/scala/com/mozilla/telemetry/pings/TestEventPing.scala b/src/test/scala/com/mozilla/telemetry/pings/TestEventPing.scala new file mode 100644 index 0000000..f9aa37b --- /dev/null +++ b/src/test/scala/com/mozilla/telemetry/pings/TestEventPing.scala @@ -0,0 +1,32 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +package com.mozilla.telemetry.pings + +import com.mozilla.telemetry.streaming.TestUtils +import org.scalatest.{FlatSpec, Matchers} + +class TestEventPing extends FlatSpec with Matchers { + val message = TestUtils.generateEventMessages(1).head + val eventPing = EventPing(message) + + "EventPing" should "contain event meta fields" in { + eventPing.payload.lostEventsCount should be (0) + eventPing.payload.processStartTimestamp should be (1530291900000L) + eventPing.payload.reason should be ("periodic") + eventPing.payload.sessionId should be ("dd302e9d-569b-4058-b7e8-02b2ff83522c") + eventPing.payload.subsessionId should be ("79a2728f-af12-4ed3-b56d-0531a03c2f26") + } + + "EventPing" should "parse events correctly" in { + eventPing.events should be(Seq( + Event(4118829, "activity_stream", "end", "session", Some("909"), + Some(Map("addon_version" -> "2018.06.22.1337-8d599e17", "user_prefs" -> "63", + "session_id" -> "{78fe2428-15fb-4448-b517-cbb85f22def0}", "page" -> "about:newtab"))), + Event(4203540, "normandy", "enroll", "preference_study", Some("awesome-experiment"), + Some(Map("branch" -> "control", "experimentType" -> "exp"))), + Event(4203541, "test", "no", "string_value", None, Some(Map("hello" -> "world"))), + Event(4203542, "test", "no", "extras", None, None) + )) + } +}