Bug 1573477 - Geckoview Streaming Telemetry: Scalar part r=janerik,agi

Let's not update the "how to exfiltrate a probe in GeckoView" docs until the
GV and Glean parts are complete.

Differential Revision: https://phabricator.services.mozilla.com/D42208

--HG--
extra : moz-landing-system : lando

toolkit/components/telemetry/Scalars.yaml

@@ -4619,6 +4619,7 @@ telemetry.test:
       - 'firefox'
       - 'fennec'
       - 'geckoview'
+      - 'geckoview_streaming'
     record_in_processes:
       - 'main'
 
@@ -4634,6 +4635,7 @@ telemetry.test:
       - 'firefox'
       - 'fennec'
       - 'geckoview'
+      - 'geckoview_streaming'
     record_in_processes:
       - 'main'
 
@@ -4649,6 +4651,7 @@ telemetry.test:
       - 'firefox'
       - 'fennec'
       - 'geckoview'
+      - 'geckoview_streaming'
     record_in_processes:
       - 'main'
 

toolkit/components/telemetry/core/TelemetryScalar.cpp

@@ -6,6 +6,7 @@
 
 #include "TelemetryScalar.h"
 
+#include "geckoview/streaming/GeckoViewStreamingTelemetry.h"
 #include "ipc/TelemetryComms.h"
 #include "ipc/TelemetryIPCAccumulator.h"
 #include "mozilla/dom/ContentParent.h"
@@ -337,7 +338,8 @@ class ScalarBase {
   explicit ScalarBase(const BaseScalarInfo& aInfo)
       : mStoreCount(aInfo.storeCount()),
         mStoreOffset(aInfo.storeOffset()),
-        mStoreHasValue(mStoreCount) {
+        mStoreHasValue(mStoreCount),
+        mName(aInfo.name()) {
     mStoreHasValue.SetLength(mStoreCount);
     for (auto& val : mStoreHasValue) {
       val = false;
@@ -390,6 +392,9 @@ class ScalarBase {
   const uint32_t mStoreCount;
   const uint32_t mStoreOffset;
   nsTArray<bool> mStoreHasValue;
+
+ protected:
+  const nsCString mName;
 };
 
 ScalarResult ScalarBase::HandleUnsupported() const {
@@ -509,6 +514,10 @@ ScalarResult ScalarUnsigned::SetValue(nsIVariant* aValue) {
 }
 
 void ScalarUnsigned::SetValue(uint32_t aValue) {
+  if (GetCurrentProduct() == SupportedProduct::GeckoviewStreaming) {
+    GeckoViewStreamingTelemetry::UintScalarSet(mName, aValue);
+    return;
+  }
   for (auto& val : mStorage) {
     val = aValue;
   }
@@ -532,6 +541,7 @@ ScalarResult ScalarUnsigned::AddValue(nsIVariant* aValue) {
 }
 
 void ScalarUnsigned::AddValue(uint32_t aValue) {
+  MOZ_ASSERT(GetCurrentProduct() != SupportedProduct::GeckoviewStreaming);
   for (auto& val : mStorage) {
     val += aValue;
   }
@@ -539,6 +549,7 @@ void ScalarUnsigned::AddValue(uint32_t aValue) {
 }
 
 ScalarResult ScalarUnsigned::SetMaximum(nsIVariant* aValue) {
+  MOZ_ASSERT(GetCurrentProduct() != SupportedProduct::GeckoviewStreaming);
   ScalarResult sr = CheckInput(aValue);
   if (sr == ScalarResult::UnsignedNegativeValue) {
     return sr;
@@ -664,6 +675,13 @@ ScalarResult ScalarString::SetValue(nsIVariant* aValue) {
 
 ScalarResult ScalarString::SetValue(const nsAString& aValue) {
   auto str = Substring(aValue, 0, kMaximumStringValueLength);
+  if (GetCurrentProduct() == SupportedProduct::GeckoviewStreaming) {
+    GeckoViewStreamingTelemetry::StringScalarSet(mName,
+                                                 NS_ConvertUTF16toUTF8(str));
+    return aValue.Length() > kMaximumStringValueLength
+               ? ScalarResult::StringTooLong
+               : ScalarResult::Ok;
+  }
   for (auto& val : mStorage) {
     val.Assign(str);
   }
@@ -758,6 +776,10 @@ ScalarResult ScalarBoolean::SetValue(nsIVariant* aValue) {
 };
 
 void ScalarBoolean::SetValue(bool aValue) {
+  if (GetCurrentProduct() == SupportedProduct::GeckoviewStreaming) {
+    GeckoViewStreamingTelemetry::BoolScalarSet(mName, aValue);
+    return;
+  }
   for (auto& val : mStorage) {
     val = aValue;
   }
@@ -3581,6 +3603,10 @@ void TelemetryScalar::RecordDiscardedData(
     return;
   }
 
+  if (GetCurrentProduct() == SupportedProduct::GeckoviewStreaming) {
+    return;
+  }
+
   ScalarBase* scalar = nullptr;
   mozilla::DebugOnly<nsresult> rv;
 

toolkit/components/telemetry/docs/internals/geckoview-streaming.rst

@@ -1,8 +1,8 @@
 GeckoView Streaming API
 =======================
 
-As an alternative to `GeckoView <../internals/geckoview>` mode,
-Telemetry can instead route Histogram accumulations out of Gecko to a Telemetry Delegate.
+As an alternative to :doc:`GeckoView <../internals/geckoview>` mode,
+Telemetry can instead route Histogram samples and Scalar values out of Gecko to a Telemetry Delegate.
 
 To do this, ``toolkit.telemetry.geckoview.streaming`` must be set to true,
 and Gecko must have been built with ``MOZ_WIDGET_ANDROID`` defined.
@@ -10,5 +10,14 @@ and Gecko must have been built with ``MOZ_WIDGET_ANDROID`` defined.
 Details
 =======
 
-Accumulations from Histograms that have a ``products`` list that includes ``geckoview_streaming`` will be redirected to a small batching service in ``toolkit/components/telemetry/geckoview/streaming``.
-The batching service (essentially just a table of histogram names to lists of samples) will hold on to these lists of accumulations paired to the histogram names for a length of time (``toolkit.telemetry.geckoview.batchDurationMS`` (default 5000)) after which the next accumulation will trigger the whole batch (all lists) to be passed over to the ``StreamingTelemetryDelegate``.
+Samples accumulated on Histograms and values set
+(``ScalarAdd`` and ``ScalarSetMaximum`` operations are not supported)
+on Scalars that have ``products`` lists that include ``geckoview_streaming``
+will be redirected to a small batching service in
+``toolkit/components/telemetry/geckoview/streaming``.
+The batching service
+(essentially just tables of histogram/scalar names to lists of samples/values)
+will hold on to these lists of samples/values paired to the histogram/scalar names for a length of time
+(``toolkit.telemetry.geckoview.batchDurationMS`` (default 5000))
+after which the next accumulation or ``ScalarSet`` will trigger the whole batch
+(all lists) to be passed over to the ``StreamingTelemetryDelegate``.

toolkit/components/telemetry/geckoview/gtest/TestGeckoViewStreaming.cpp

@@ -13,6 +13,7 @@ using namespace mozilla;
 using namespace mozilla::Telemetry;
 using namespace TelemetryTestHelpers;
 using GeckoViewStreamingTelemetry::StreamingTelemetryDelegate;
+using mozilla::Telemetry::ScalarID;
 using ::testing::_;
 using ::testing::Eq;
 
@@ -45,6 +46,12 @@ class MockDelegate final : public StreamingTelemetryDelegate {
   MOCK_METHOD2(ReceiveHistogramSamples,
                void(const nsCString& aHistogramName,
                     const nsTArray<uint32_t>& aSamples));
+  MOCK_METHOD2(ReceiveBoolScalarValue,
+               void(const nsCString& aScalarName, bool aValue));
+  MOCK_METHOD2(ReceiveStringScalarValue,
+               void(const nsCString& aScalarName, const nsCString& aValue));
+  MOCK_METHOD2(ReceiveUintScalarValue,
+               void(const nsCString& aScalarName, uint32_t aValue));
 };  // class MockDelegate
 
 TEST_F(TelemetryStreamingFixture, HistogramSamples) {
@@ -137,4 +144,38 @@ TEST_F(TelemetryStreamingFixture, MultipleThreads) {
   Telemetry::Accumulate(kTestHgram2, kSample1);
 }
 
+TEST_F(TelemetryTestFixture, StreamingTelemetryStreamScalarValues) {
+  NS_NAMED_LITERAL_CSTRING(kBoolScalarName, "telemetry.test.boolean_kind");
+  NS_NAMED_LITERAL_CSTRING(kStringScalarName, "telemetry.test.string_kind");
+  NS_NAMED_LITERAL_CSTRING(kUintScalarName, "telemetry.test.unsigned_int_kind");
+
+  const bool kBoolScalarValue = true;
+  NS_NAMED_LITERAL_CSTRING(kStringScalarValue, "a string scalar value");
+  const uint32_t kUintScalarValue = 42;
+
+  auto md = MakeRefPtr<MockDelegate>();
+  EXPECT_CALL(
+      *md, ReceiveBoolScalarValue(Eq(kBoolScalarName), Eq(kBoolScalarValue)));
+  EXPECT_CALL(*md, ReceiveStringScalarValue(Eq(kStringScalarName),
+                                            Eq(kStringScalarValue)));
+  EXPECT_CALL(
+      *md, ReceiveUintScalarValue(Eq(kUintScalarName), Eq(kUintScalarValue)));
+
+  GeckoViewStreamingTelemetry::RegisterDelegate(md);
+
+  Preferences::SetBool(kGeckoViewStreamingPref, true);
+  Preferences::SetInt(kBatchTimeoutPref, 5000);
+
+  Telemetry::ScalarSet(ScalarID::TELEMETRY_TEST_BOOLEAN_KIND, kBoolScalarValue);
+  Telemetry::ScalarSet(ScalarID::TELEMETRY_TEST_STRING_KIND,
+                       NS_ConvertUTF8toUTF16(kStringScalarValue));
+  Preferences::SetInt(kBatchTimeoutPref,
+                      0);  // Trigger batch on next accumulation.
+  Telemetry::ScalarSet(ScalarID::TELEMETRY_TEST_UNSIGNED_INT_KIND,
+                       kUintScalarValue);
+
+  // Clear delegate so it can be deleted when the batch is done.
+  GeckoViewStreamingTelemetry::RegisterDelegate(nullptr);
+}
+
 }  // namespace

toolkit/components/telemetry/geckoview/streaming/GeckoViewStreamingTelemetry.cpp

@@ -20,7 +20,7 @@ using mozilla::StaticMutexNotRecorded;
 using mozilla::StaticRefPtr;
 using mozilla::TimeStamp;
 
-// Batches and streams Histogram accumulations to a JNI delegate which will
+// Batches and streams Telemetry samples to a JNI delegate which will
 // (presumably) do something with the data. Expected to be used to route data
 // up to the Android Components layer to be translated into Glean metrics.
 namespace GeckoViewStreamingTelemetry {
@@ -33,8 +33,15 @@ static StaticMutexNotRecorded gMutex;
 // The time the batch began.
 TimeStamp gBatchBegan;
 // The batch of histograms and samples.
-typedef nsDataHashtable<nsCStringHashKey, nsTArray<uint32_t>> Batch;
-Batch gBatch;
+typedef nsDataHashtable<nsCStringHashKey, nsTArray<uint32_t>> HistogramBatch;
+HistogramBatch gBatch;
+// The batches of Scalars and their values.
+typedef nsDataHashtable<nsCStringHashKey, bool> BoolScalarBatch;
+BoolScalarBatch gBoolScalars;
+typedef nsDataHashtable<nsCStringHashKey, nsCString> StringScalarBatch;
+StringScalarBatch gStringScalars;
+typedef nsDataHashtable<nsCStringHashKey, uint32_t> UintScalarBatch;
+UintScalarBatch gUintScalars;
 // The delegate to receive the Histograms' samples.
 StaticRefPtr<StreamingTelemetryDelegate> gDelegate;
 
@@ -48,10 +55,16 @@ void RegisterDelegate(const RefPtr<StreamingTelemetryDelegate>& aDelegate) {
 class SendBatchRunnable : public Runnable {
  public:
   explicit SendBatchRunnable(RefPtr<StreamingTelemetryDelegate> aDelegate,
-                             Batch&& aBatch)
+                             HistogramBatch&& aBatch,
+                             BoolScalarBatch&& aBoolScalars,
+                             StringScalarBatch&& aStringScalars,
+                             UintScalarBatch&& aUintScalars)
       : Runnable("SendBatchRunnable"),
         mDelegate(std::move(aDelegate)),
-        mBatch(std::move(aBatch)) {}
+        mBatch(std::move(aBatch)),
+        mBoolScalars(std::move(aBoolScalars)),
+        mStringScalars(std::move(aStringScalars)),
+        mUintScalars(std::move(aUintScalars)) {}
 
   NS_IMETHOD Run() override {
     MOZ_ASSERT(NS_IsMainThread());
@@ -63,14 +76,36 @@ class SendBatchRunnable : public Runnable {
 
       mDelegate->ReceiveHistogramSamples(histogramName, samples);
     }
-
     mBatch.Clear();
+
+    for (auto iter = mBoolScalars.Iter(); !iter.Done(); iter.Next()) {
+      const nsCString& scalarName = PromiseFlatCString(iter.Key());
+      mDelegate->ReceiveBoolScalarValue(scalarName, iter.Data());
+    }
+    mBoolScalars.Clear();
+
+    for (auto iter = mStringScalars.Iter(); !iter.Done(); iter.Next()) {
+      const nsCString& scalarName = PromiseFlatCString(iter.Key());
+      const nsCString& scalarValue = PromiseFlatCString(iter.Data());
+      mDelegate->ReceiveStringScalarValue(scalarName, scalarValue);
+    }
+    mStringScalars.Clear();
+
+    for (auto iter = mUintScalars.Iter(); !iter.Done(); iter.Next()) {
+      const nsCString& scalarName = PromiseFlatCString(iter.Key());
+      mDelegate->ReceiveUintScalarValue(scalarName, iter.Data());
+    }
+    mUintScalars.Clear();
+
     return NS_OK;
   }
 
  private:
   RefPtr<StreamingTelemetryDelegate> mDelegate;
-  Batch mBatch;
+  HistogramBatch mBatch;
+  BoolScalarBatch mBoolScalars;
+  StringScalarBatch mStringScalars;
+  UintScalarBatch mUintScalars;
 };  // class SendBatchRunnable
 
 // Can be called on any thread.
@@ -85,31 +120,68 @@ void SendBatch(const StaticMutexAutoLock& aLock) {
   }
 
   // To make it so accumulations within the delegation don't deadlock us,
-  // move the batch's contents into the Runner.
-  Batch copy;
-  gBatch.SwapElements(copy);
-  RefPtr<SendBatchRunnable> runnable =
-      new SendBatchRunnable(gDelegate, std::move(copy));
+  // move the batches' contents into the Runner.
+  HistogramBatch histogramCopy;
+  gBatch.SwapElements(histogramCopy);
+  BoolScalarBatch boolScalarCopy;
+  gBoolScalars.SwapElements(boolScalarCopy);
+  StringScalarBatch stringScalarCopy;
+  gStringScalars.SwapElements(stringScalarCopy);
+  UintScalarBatch uintScalarCopy;
+  gUintScalars.SwapElements(uintScalarCopy);
+  RefPtr<SendBatchRunnable> runnable = new SendBatchRunnable(
+      gDelegate, std::move(histogramCopy), std::move(boolScalarCopy),
+      std::move(stringScalarCopy), std::move(uintScalarCopy));
 
   // To make things easier for the delegate, dispatch to the main thread.
   NS_DispatchToMainThread(runnable);
 }
 
 // Can be called on any thread.
-void HistogramAccumulate(const nsCString& aName, uint32_t aValue) {
-  StaticMutexAutoLock lock(gMutex);
-
-  if (gBatch.Count() == 0) {
+void BatchCheck(const StaticMutexAutoLock& aLock) {
+  if (gBatchBegan.IsNull()) {
     gBatchBegan = TimeStamp::Now();
   }
-  nsTArray<uint32_t>& samples = gBatch.GetOrInsert(aName);
-  samples.AppendElement(aValue);
-
   double batchDurationMs = (TimeStamp::Now() - gBatchBegan).ToMilliseconds();
   if (batchDurationMs >
       mozilla::StaticPrefs::toolkit_telemetry_geckoview_batchDurationMS()) {
-    SendBatch(lock);
+    SendBatch(aLock);
+    gBatchBegan = TimeStamp();
   }
 }
 
+// Can be called on any thread.
+void HistogramAccumulate(const nsCString& aName, uint32_t aValue) {
+  StaticMutexAutoLock lock(gMutex);
+
+  nsTArray<uint32_t>& samples = gBatch.GetOrInsert(aName);
+  samples.AppendElement(aValue);
+
+  BatchCheck(lock);
+}
+
+void BoolScalarSet(const nsCString& aName, bool aValue) {
+  StaticMutexAutoLock lock(gMutex);
+
+  gBoolScalars.Put(aName, aValue);
+
+  BatchCheck(lock);
+}
+
+void StringScalarSet(const nsCString& aName, const nsCString& aValue) {
+  StaticMutexAutoLock lock(gMutex);
+
+  gStringScalars.Put(aName, aValue);
+
+  BatchCheck(lock);
+}
+
+void UintScalarSet(const nsCString& aName, uint32_t aValue) {
+  StaticMutexAutoLock lock(gMutex);
+
+  gUintScalars.Put(aName, aValue);
+
+  BatchCheck(lock);
+}
+
 }  // namespace GeckoViewStreamingTelemetry

toolkit/components/telemetry/geckoview/streaming/GeckoViewStreamingTelemetry.h

@@ -17,16 +17,26 @@ namespace GeckoViewStreamingTelemetry {
 
 void HistogramAccumulate(const nsCString& aName, uint32_t aValue);
 
+void BoolScalarSet(const nsCString& aName, bool aValue);
+void StringScalarSet(const nsCString& aName, const nsCString& aValue);
+void UintScalarSet(const nsCString& aName, uint32_t aValue);
+
 // Classes wishing to receive Streaming Telemetry must implement this interface
 // and register themselves via RegisterDelegate.
 class StreamingTelemetryDelegate
     : public mozilla::RefCounted<StreamingTelemetryDelegate> {
  public:
   MOZ_DECLARE_REFCOUNTED_TYPENAME(StreamingTelemetryDelegate);
-  // Will be called from time to time on the main thread.
+  virtual ~StreamingTelemetryDelegate() = default;
+
+  // Receive* methods will be called from time to time on the main thread.
   virtual void ReceiveHistogramSamples(const nsCString& aName,
                                        const nsTArray<uint32_t>& aSamples) = 0;
-  virtual ~StreamingTelemetryDelegate() = default;
+  virtual void ReceiveBoolScalarValue(const nsCString& aName, bool aValue) = 0;
+  virtual void ReceiveStringScalarValue(const nsCString& aName,
+                                        const nsCString& aValue) = 0;
+  virtual void ReceiveUintScalarValue(const nsCString& aName,
+                                      uint32_t aValue) = 0;
 };
 
 // Registers the provided StreamingTelemetryDelegate to receive Streaming