devops: introduce compressed dashboard

Compressed dashboard is 10 times smaller yet has all the data to
render flakiness.

Drive-by: remove old dashboard implementations since they are no
longer used.

utils/flakiness-dashboard/processing/compress_raw_data.js

@@ -0,0 +1,34 @@
+#!/usr/bin/env node
+/**
+ * Copyright 2017 Google Inc. All rights reserved.
+ * Modifications copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const path = require('path');
+const fs = require('fs');
+const {SimpleBlob} = require('./utils.js');
+const {processDashboardCompressedV1} = require('./dashboard_compressed_v1.js');
+
+(async () => {
+  const sha = process.argv[2];
+  console.log(sha);
+  const dashboardBlob = await SimpleBlob.create('dashboards', `raw/${sha}.json`);
+  const reports = await dashboardBlob.download();
+  if (!reports) {
+    console.error('ERROR: no data found for commit ' + sha);
+    process.exit(1);
+  }
+  await processDashboardCompressedV1({log: console.log}, reports, sha);
+})();

utils/flakiness-dashboard/processing/dashboard_compressed_v1.js

@@ -0,0 +1,119 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the 'License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const {SimpleBlob, flattenSpecs} = require('./utils.js');
+
+async function processDashboardCompressedV1(context, reports, commitSHA) {
+  const timestamp = Date.now();
+  const dashboardBlob = await SimpleBlob.create('dashboards', `compressed_v1/${commitSHA}.json`);
+  await dashboardBlob.uploadGzipped(compressReports(reports));
+
+  context.log(`
+  ===== started dashboard compressed v1 =====
+    SHA: ${commitSHA}
+  ===== complete in ${Date.now() - timestamp}ms =====
+  `);
+}
+
+module.exports = {processDashboardCompressedV1, compressReports};
+
+function compressReports(reports) {
+  const files = {};
+  for (const report of reports) {
+    for (const spec of flattenSpecs(report)) {
+      let specs = files[spec.file];
+      if (!specs) {
+        specs = new Map();
+        files[spec.file] = specs;
+      }
+      const specId = spec.file + '---' + spec.title + ' --- ' + spec.line;
+      let specObject = specs.get(specId);
+      if (!specObject) {
+        specObject = {
+          title: spec.title,
+          line: spec.line,
+          column: spec.column,
+          tests: new Map(),
+        };
+        specs.set(specId, specObject);
+      }
+      for (const test of spec.tests || []) {
+        if (test.runs.length === 1 && !test.runs[0].status)
+          continue;
+        // Overwrite test platform parameter with a more specific information from
+        // build run.
+        const osName = report.metadata.osName.toUpperCase().startsWith('MINGW') ? 'Windows' : report.metadata.osName;
+        const arch = report.metadata.arch && !report.metadata.arch.includes('x86') ? report.metadata.arch : '';
+        const platform = (osName + ' ' + report.metadata.osVersion + ' ' + arch).trim();
+        const browserName = test.parameters.browserName || 'N/A';
+
+        const testName = getTestName(browserName, platform, test.parameters);
+        let testObject = specObject.tests.get(testName);
+        if (!testObject) {
+          testObject = {
+            parameters: {
+              ...test.parameters,
+              browserName,
+              platform,
+            },
+          };
+          // By default, all tests are expected to pass. We can have this as a hidden knowledge.
+          if (test.expectedStatus !== 'passed')
+            testObject.expectedStatus = test.expectedStatus;
+          if (test.annotations.length)
+            testObject.annotations = test.annotations;
+          specObject.tests.set(testName, testObject);
+        }
+
+        for (const run of test.runs) {
+          // Record duration of slow tests only, i.e. > 1s.
+          if (run.status === 'passed' && run.duration > 1000) {
+            testObject.minTime = Math.min((testObject.minTime || Number.MAX_VALUE), run.duration);
+            testObject.maxTime = Math.max((testObject.maxTime || 0), run.duration);
+          }
+          if (run.status === 'failed') {
+            if (!Array.isArray(testObject.failed))
+              testObject.failed = [];
+            testObject.failed.push(run.error);
+          } else {
+            testObject[run.status] = (testObject[run.status] || 0) + 1;
+          }
+        }
+      }
+    }
+  }
+
+  const pojo = Object.entries(files).map(([file, specs]) => ({
+    file,
+    specs: [...specs.values()].map(specObject => ({
+      ...specObject,
+      tests: [...specObject.tests.values()],
+    })),
+  }));
+  return pojo;
+}
+
+function getTestName(browserName, platform, parameters) {
+  return [browserName, platform, ...Object.entries(parameters).filter(([key, value]) => !!value).map(([key, value]) => {
+    if (key === 'browserName' || key === 'platform')
+      return;
+    if (typeof value === 'string')
+      return value;
+    if (typeof value === 'boolean')
+      return key;
+    return `${key}=${value}`;
+  }).filter(Boolean)].join(' / ');
+}

utils/flakiness-dashboard/processing/dashboard_raw.js

@@ -30,6 +30,10 @@ async function processDashboardRaw(context, report) {
     timestamp: ${report.metadata.commitTimestamp}
   ===== complete in ${Date.now() - timestamp}ms =====
   `);
+  return {
+    reports: dashboardData,
+    commitSHA: report.metadata.commitSHA,
+  };
 }
 
 module.exports = {processDashboardRaw};

utils/flakiness-dashboard/processing/dashboard_v1.js

@@ -1,102 +0,0 @@
-/**
- * Copyright (c) Microsoft Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the 'License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-const {SimpleBlob, flattenSpecs} = require('./utils.js');
-
-const DASHBOARD_VERSION = 1;
-
-class Dashboard {
-  constructor() {
-    this._runs = [];
-  }
-
-  initialize(jsonData) {
-    if (jsonData.version !== DASHBOARD_VERSION) {
-      // Run migrations here!
-    }
-    this._runs = jsonData.buildbotRuns;
-  }
-
-  addReport(report) {
-    // We cannot use linenumber to identify specs since line numbers
-    // might be different across commits.
-    const getSpecId = spec => spec.file + ' @@@ ' + spec.title;
-
-    const faultySpecIds = new Set();
-    for (const run of this._runs) {
-      for (const spec of run.specs)
-        faultySpecIds.add(getSpecId(spec));
-    }
-    const specs = [];
-    for (const spec of flattenSpecs(report)) {
-      // Filter out specs that didn't have a single test that was run in the
-      // given shard.
-      if (spec.tests.every(test => test.runs.length === 1 && !test.runs[0].status))
-        continue;
-      const hasFlakyAnnotation = spec.tests.some(test => test.annotations.some(a => a.type === 'flaky'));
-
-      if (!spec.ok || hasFlakyAnnotation || faultySpecIds.has(getSpecId(spec)))
-        specs.push(spec);
-    }
-    if (specs.length) {
-      this._runs.push({
-        metadata: report.metadata,
-        specs,
-      });
-    }
-    return specs.length;
-  }
-
-  serialize(maxCommits = 100) {
-    const shaToTimestamp = new Map();
-    for (const run of this._runs)
-      shaToTimestamp.set(run.metadata.commitSHA, run.metadata.commitTimestamp);
-    const commits = [...shaToTimestamp].sort(([sha1, ts1], [sha2, ts2]) => ts2 - ts1).slice(0, maxCommits);
-    const commitsSet = new Set(commits.map(([sha, ts]) => sha));
-    return {
-      version: DASHBOARD_VERSION,
-      timestamp: Date.now(),
-      buildbotRuns: this._runs.filter(run => commitsSet.has(run.metadata.commitSHA)),
-    };
-  }
-}
-
-async function processDashboardV1(context, report) {
-  const timestamp = Date.now();
-  const dashboardBlob = await SimpleBlob.create('dashboards', 'main.json');
-  const dashboardData = await dashboardBlob.download();
-  const dashboard = new Dashboard();
-  if (dashboardData)
-    dashboard.initialize(dashboardData);
-
-  try {
-    const addedSpecs = dashboard.addReport(report);
-    await dashboardBlob.uploadGzipped(dashboard.serialize());
-    context.log(`
-    ===== started dashboard v1 =====
-      SHA: ${report.metadata.commitSHA}
-      URL: ${report.metadata.runURL}
-      timestamp: ${report.metadata.commitTimestamp}
-      added specs: ${addedSpecs}
-    ===== complete in ${Date.now() - timestamp}ms =====
-    `);
-  } catch (e) {
-    context.log(e);
-    return;
-  }
-}
-
-module.exports = {processDashboardV1};

utils/flakiness-dashboard/processing/dashboard_v2.js

@@ -1,155 +0,0 @@
-/**
- * Copyright (c) Microsoft Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the 'License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-const {SimpleBlob, flattenSpecs} = require('./utils.js');
-
-const DASHBOARD_VERSION = 1;
-
-class Dashboard {
-  constructor() {
-    this._specs = new Map();
-    this._commits = new Map();
-  }
-
-  initialize(jsonData) {
-    if (jsonData.version !== DASHBOARD_VERSION) {
-      // Run migrations here!
-    }
-    for (const spec of jsonData.specs) {
-      const commitCoordinates = new Map();
-      for (const coord of spec.commitCoordinates)
-        commitCoordinates.set(coord.sha, coord);
-      this._specs.set(spec.specId, {
-        specId: spec.specId,
-        file: spec.file,
-        title: spec.title,
-        problematicTests: spec.problematicTests,
-        commitCoordinates,
-      });
-    }
-    for (const commit of jsonData.commits)
-      this._commits.set(commit.sha, commit);
-  }
-
-  addReport(report) {
-    const sha = report.metadata.commitSHA;
-    this._commits.set(sha, {
-      sha,
-      timestamp: report.metadata.commitTimestamp,
-      message: report.metadata.commitTitle,
-      author: report.metadata.commitAuthorName,
-      email: report.metadata.commitAuthorEmail,
-    });
-    let addedSpecs = 0;
-    for (const spec of flattenSpecs(report)) {
-      // We cannot use linenumber to identify specs since line numbers
-      // might be different across commits.
-      const specId = spec.file + ' --- ' + spec.title;
-      const tests = spec.tests.filter(test => !isHealthyTest(test));
-      // If there are no problematic testruns now and before - ignore the spec.
-      if (!tests.length && !this._specs.has(specId))
-        continue;
-      ++addedSpecs;
-      let specInfo = this._specs.get(specId);
-      if (!specInfo) {
-        specInfo = {
-          specId,
-          title: spec.title,
-          file: spec.file,
-          commitCoordinates: new Map(),
-          problematicTests: [],
-        };
-        this._specs.set(specId, specInfo);
-      }
-      specInfo.problematicTests.push(...tests.map(test => ({sha, test})));
-      specInfo.commitCoordinates.set(sha, ({sha, line: spec.line, column: spec.column}));
-    }
-    return addedSpecs;
-  }
-
-  serialize(maxCommits = 100) {
-    const commits = [...this._commits.values()].sort((a, b) => a.timestamp - b.timestamp).slice(-maxCommits);
-    const whitelistedCommits = new Set();
-    for (const c of commits)
-      whitelistedCommits.add(c.sha);
-
-    const specs = [...this._specs.values()].map(spec => ({
-      specId: spec.specId,
-      title: spec.title,
-      file: spec.file,
-      commitCoordinates: [...spec.commitCoordinates.values()].filter(coord => whitelistedCommits.has(coord.sha)),
-      problematicTests: [...spec.problematicTests.values()].filter(test => whitelistedCommits.has(test.sha)),
-    })).filter(spec => spec.commitCoordinates.length && spec.problematicTests.length);
-
-    return {
-      version: DASHBOARD_VERSION,
-      timestamp: Date.now(),
-      commits,
-      specs,
-    };
-  }
-}
-
-async function processDashboardV2(context, report) {
-  const timestamp = Date.now();
-  const dashboardBlob = await SimpleBlob.create('dashboards', 'main_v2.json');
-  const dashboardData = await dashboardBlob.download();
-  const dashboard = new Dashboard();
-  if (dashboardData)
-    dashboard.initialize(dashboardData);
-
-  try {
-    const addedSpecs = dashboard.addReport(report);
-    await dashboardBlob.uploadGzipped(dashboard.serialize());
-    context.log(`
-    ===== started dashboard v2 =====
-      SHA: ${report.metadata.commitSHA}
-      URL: ${report.metadata.runURL}
-      timestamp: ${report.metadata.commitTimestamp}
-      added specs: ${addedSpecs}
-    ===== complete in ${Date.now() - timestamp}ms =====
-    `);
-  } catch (e) {
-    context.log(e);
-    return;
-  }
-}
-
-module.exports = {processDashboardV2};
-
-function isHealthyTest(test) {
-  // If test has any annotations - it's not healthy and requires attention.
-  if (test.annotations.length)
-    return false;
-  // If test does not have annotations and doesn't have runs - it's healthy.
-  if (!test.runs.length)
-    return true;
-  // If test was run more than once - it's been retried and thus unhealthy.
-  if (test.runs.length > 1)
-    return false;
-  const run = test.runs[0];
-  // Test might not have status if it was sharded away - consider it healthy.
-  if (!run.status)
-    return true;
-  // if status is not "passed", then it's a bad test.
-  if (run.status !== 'passed')
-    return false;
-  // if run passed, but that's not what we expected - it's a bad test.
-  if (run.status !== test.expectedStatus)
-    return false;
-  // Otherwise, the test is healthy.
-  return true;
-}

utils/flakiness-dashboard/processing/index.js

@@ -15,9 +15,8 @@
  */
 
 const {blobServiceClient, gunzipAsync, deleteBlob} = require('./utils.js');
-const {processDashboardV1} = require('./dashboard_v1.js');
-const {processDashboardV2} = require('./dashboard_v2.js');
 const {processDashboardRaw} = require('./dashboard_raw.js');
+const {processDashboardCompressedV1} = require('./dashboard_compressed_v1.js');
 
 module.exports = async function(context) {
   // First thing we do - delete the blob.
@@ -28,9 +27,6 @@ module.exports = async function(context) {
   const report = JSON.parse(data.toString('utf8'));
 
   // Process dashboards one-by-one to limit max heap utilization.
-  await processDashboardRaw(context, report);
-  // Disable V1 dashboard since it's crazy expensive to compute.
-  // await processDashboardV1(context, report);
-  // Disable V2 dashboard in favor of raw data.
-  // await processDashboardV2(context, report);
+  const {reports, commitSHA} = await processDashboardRaw(context, report);
+  await processDashboardCompressedV1(context, reports, commitSHA);
 }

utils/flakiness-dashboard/processing/utils.js

@@ -61,7 +61,9 @@ class SimpleBlob {
 
   async uploadGzipped(data) {
     const content = JSON.stringify(data);
-    const zipped = await gzipAsync(content);
+    const zipped = await gzipAsync(content, {
+      level: 9,
+    });
     await this._blockBlobClient.upload(zipped, Buffer.byteLength(zipped), {
       blobHTTPHeaders: {
         blobContentEncoding: 'gzip',