misc(scripts): add trace/devtoolslog minification scripts (#5237)
This commit is contained in:
Patrick Hulce
GitHub
Родитель
804e9b8c97
Коммит
40cf8da6b4
|
|
@ -0,0 +1,107 @@
|
|||
#!/usr/bin/env node
|
||||
/**
|
||||
* @license Copyright 2018 Google Inc. All Rights Reserved.
|
||||
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
|
||||
*/
|
||||
'use strict';
|
||||
|
||||
/* eslint-disable no-console */
|
||||
|
||||
/**
|
||||
* @fileoverview Minifies a devtools log by removing noisy header values, eliminating data URIs, etc.
|
||||
*/
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
if (process.argv.length !== 4) {
|
||||
console.error('Usage $0: <input file> <output file>');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const inputDevtoolsLogPath = path.resolve(process.cwd(), process.argv[2]);
|
||||
const outputDevtoolsLogPath = path.resolve(process.cwd(), process.argv[3]);
|
||||
const inputDevtoolsLogRaw = fs.readFileSync(inputDevtoolsLogPath, 'utf8');
|
||||
/** @type {LH.DevtoolsLog} */
|
||||
const inputDevtoolsLog = JSON.parse(inputDevtoolsLogRaw);
|
||||
|
||||
const headersToKeep = new Set([
|
||||
// Request headers
|
||||
'accept',
|
||||
'accept-encoding',
|
||||
'accept-ranges',
|
||||
// Response headers
|
||||
'status',
|
||||
'content-length',
|
||||
'content-type',
|
||||
'content-encoding',
|
||||
'content-range',
|
||||
'etag',
|
||||
'cache-control',
|
||||
'last-modified',
|
||||
'link',
|
||||
'x-robots-tag',
|
||||
]);
|
||||
|
||||
/** @param {Partial<LH.Crdp.Network.Headers>} [headers] */
|
||||
function cleanHeaders(headers) {
|
||||
if (!headers) return;
|
||||
|
||||
for (const key of Object.keys(headers)) {
|
||||
if (!headersToKeep.has(key.toLowerCase())) headers[key] = undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/** @param {{url: string}} obj */
|
||||
function cleanDataURI(obj) {
|
||||
obj.url = obj.url.replace(/^(data:.*?base64,).*/, '$1FILLER');
|
||||
}
|
||||
|
||||
/** @param {LH.Crdp.Network.Response} [response] */
|
||||
function cleanResponse(response) {
|
||||
if (!response) return;
|
||||
cleanDataURI(response);
|
||||
cleanHeaders(response.requestHeaders);
|
||||
cleanHeaders(response.headers);
|
||||
response.securityDetails = undefined;
|
||||
response.headersText = undefined;
|
||||
response.requestHeadersText = undefined;
|
||||
|
||||
/** @type {any} */
|
||||
const timing = response.timing || {};
|
||||
for (const [k, v] of Object.entries(timing)) {
|
||||
if (v === -1) timing[k] = undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/** @param {LH.DevtoolsLog} log */
|
||||
function filterDevtoolsLogEvents(log) {
|
||||
return log.map(original => {
|
||||
/** @type {LH.Protocol.RawEventMessage} */
|
||||
const entry = JSON.parse(JSON.stringify(original));
|
||||
|
||||
switch (entry.method) {
|
||||
case 'Network.requestWillBeSent':
|
||||
cleanDataURI(entry.params.request);
|
||||
cleanHeaders(entry.params.request.headers);
|
||||
cleanResponse(entry.params.redirectResponse);
|
||||
break;
|
||||
case 'Network.responseReceived':
|
||||
cleanResponse(entry.params.response);
|
||||
break;
|
||||
}
|
||||
|
||||
return entry;
|
||||
});
|
||||
}
|
||||
|
||||
const filteredLog = filterDevtoolsLogEvents(inputDevtoolsLog);
|
||||
const output = `[
|
||||
${filteredLog.map(e => ' ' + JSON.stringify(e)).join(',\n')}
|
||||
]`;
|
||||
|
||||
/** @param {string} s */
|
||||
const size = s => Math.round(s.length / 1024) + 'kb';
|
||||
console.log(`Reduced DevtoolsLog from ${size(inputDevtoolsLogRaw)} to ${size(output)}`);
|
||||
fs.writeFileSync(outputDevtoolsLogPath, output);
|
||||
|
|
@ -0,0 +1,159 @@
|
|||
#!/usr/bin/env node
|
||||
/**
|
||||
* @license Copyright 2018 Google Inc. All Rights Reserved.
|
||||
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
|
||||
*/
|
||||
'use strict';
|
||||
|
||||
/* eslint-disable no-console */
|
||||
|
||||
/**
|
||||
* @fileoverview Minifies a trace by removing unnecessary events, throttling screenshots, etc.
|
||||
* See the following files for necessary events:
|
||||
* - lighthouse-core/gather/computed/trace-of-tab.js
|
||||
* - lighthouse-core/gather/computed/page-dependency-graph.js
|
||||
* - lighthouse-core/lib/traces/tracing-processor.js
|
||||
*/
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
if (process.argv.length !== 4) {
|
||||
console.error('Usage $0: <input file> <output file>');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const inputTracePath = path.resolve(process.cwd(), process.argv[2]);
|
||||
const outputTracePath = path.resolve(process.cwd(), process.argv[3]);
|
||||
const inputTraceRaw = fs.readFileSync(inputTracePath, 'utf8');
|
||||
/** @type {LH.Trace} */
|
||||
const inputTrace = JSON.parse(inputTraceRaw);
|
||||
|
||||
const toplevelTaskNames = new Set([
|
||||
'TaskQueueManager::ProcessTaskFromWorkQueue',
|
||||
'ThreadControllerImpl::DoWork',
|
||||
]);
|
||||
|
||||
const traceEventsToAlwaysKeep = new Set([
|
||||
'Screenshot',
|
||||
'TracingStartedInBrowser',
|
||||
'TracingStartedInPage',
|
||||
'navigationStart',
|
||||
'ParseAuthorStyleSheet',
|
||||
'ParseHTML',
|
||||
'PlatformResourceSendRequest',
|
||||
'ResourceSendRequest',
|
||||
'ResourceReceiveResponse',
|
||||
'ResourceFinish',
|
||||
'ResourceReceivedData',
|
||||
'EventDispatch',
|
||||
]);
|
||||
|
||||
const traceEventsToKeepInToplevelTask = new Set([
|
||||
'TimerInstall',
|
||||
'TimerFire',
|
||||
'InvalidateLayout',
|
||||
'ScheduleStyleRecalculation',
|
||||
'EvaluateScript',
|
||||
'XHRReadyStateChange',
|
||||
'FunctionCall',
|
||||
'v8.compile',
|
||||
]);
|
||||
|
||||
const traceEventsToKeepInProcess = new Set([
|
||||
...toplevelTaskNames,
|
||||
...traceEventsToKeepInToplevelTask,
|
||||
'firstPaint',
|
||||
'firstContentfulPaint',
|
||||
'firstMeaningfulPaint',
|
||||
'firstMeaningfulPaintCandidate',
|
||||
'loadEventEnd',
|
||||
'domContentLoadedEventEnd',
|
||||
]);
|
||||
|
||||
/**
|
||||
* @param {LH.TraceEvent[]} events
|
||||
*/
|
||||
function filterOutUnnecessaryTasksByNameAndDuration(events) {
|
||||
// TODO(phulce): update this once https://github.com/GoogleChrome/lighthouse/pull/5271 lands
|
||||
const startedInPageEvt = events.find(evt => evt.name === 'TracingStartedInPage');
|
||||
if (!startedInPageEvt) throw new Error('Could not find TracingStartedInPage');
|
||||
|
||||
return events.filter(evt => {
|
||||
if (toplevelTaskNames.has(evt.name) && evt.dur < 1000) return false;
|
||||
if (evt.pid === startedInPageEvt.pid && traceEventsToKeepInProcess.has(evt.name)) return true;
|
||||
return traceEventsToAlwaysKeep.has(evt.name);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Filters out tasks that are not within a toplevel task.
|
||||
* @param {LH.TraceEvent[]} events
|
||||
*/
|
||||
function filterOutOrphanedTasks(events) {
|
||||
const toplevelRanges = events
|
||||
.filter(evt => toplevelTaskNames.has(evt.name))
|
||||
.map(evt => [evt.ts, evt.ts + evt.dur]);
|
||||
|
||||
/** @param {LH.TraceEvent} e */
|
||||
const isInToplevelTask = e => toplevelRanges.some(([start, end]) => e.ts >= start && e.ts <= end);
|
||||
|
||||
return events.filter((evt, index) => {
|
||||
if (!traceEventsToKeepInToplevelTask.has(evt.name)) return true;
|
||||
if (!isInToplevelTask(evt)) return false;
|
||||
|
||||
if (evt.ph === 'B') {
|
||||
const endEvent = events.slice(index).find(e => e.name === evt.name && e.ph === 'E');
|
||||
return endEvent && isInToplevelTask(endEvent);
|
||||
} else {
|
||||
return true;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Throttles screenshot events in the trace to 2fps.
|
||||
* @param {LH.TraceEvent[]} events
|
||||
*/
|
||||
function filterOutExcessiveScreenshots(events) {
|
||||
const screenshotTimestamps = events.filter(evt => evt.name === 'Screenshot').map(evt => evt.ts);
|
||||
|
||||
let lastScreenshotTs = -Infinity;
|
||||
return events.filter(evt => {
|
||||
if (evt.name !== 'Screenshot') return true;
|
||||
const timeSinceLastScreenshot = evt.ts - lastScreenshotTs;
|
||||
const nextScreenshotTs = screenshotTimestamps.find(ts => ts > evt.ts);
|
||||
const timeUntilNextScreenshot = nextScreenshotTs ? nextScreenshotTs - evt.ts : Infinity;
|
||||
const threshold = 500 * 1000; // Throttle to ~2fps
|
||||
// Keep the frame if it's been more than 500ms since the last frame we kept or the next frame won't happen for at least 500ms
|
||||
const shouldKeep = timeUntilNextScreenshot > threshold || timeSinceLastScreenshot > threshold;
|
||||
if (shouldKeep) lastScreenshotTs = evt.ts;
|
||||
return shouldKeep;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {LH.TraceEvent[]} events
|
||||
*/
|
||||
function filterTraceEvents(events) {
|
||||
// Filter out event names we don't care about and tasks <1ms
|
||||
let filtered = filterOutUnnecessaryTasksByNameAndDuration(events);
|
||||
// Filter out events not inside a toplevel task
|
||||
filtered = filterOutOrphanedTasks(filtered);
|
||||
// Filter down the screenshots to key moments + 2fps animations
|
||||
return filterOutExcessiveScreenshots(filtered);
|
||||
}
|
||||
|
||||
const filteredEvents = filterTraceEvents(inputTrace.traceEvents);
|
||||
const output = `{
|
||||
"traceEvents": [
|
||||
${filteredEvents.map(e => ' ' + JSON.stringify(e)).join(',\n')}
|
||||
]
|
||||
}`;
|
||||
|
||||
/** @param {string} s */
|
||||
const size = s => Math.round(s.length / 1024) + 'kb';
|
||||
console.log(`Reduced trace from ${size(inputTraceRaw)} to ${size(output)}`);
|
||||
console.log(`Filtered out ${inputTrace.traceEvents.length - filteredEvents.length} trace events`);
|
||||
fs.writeFileSync(outputTracePath, output);
|
||||
|
|
@ -62,7 +62,8 @@
|
|||
"update:sample-json": "node ./lighthouse-cli -A=./lighthouse-core/test/results/artifacts --throttling-method=devtools --output=json --output-path=./lighthouse-core/test/results/sample_v2.json http://localhost/dobetterweb/dbw_tester.html && node lighthouse-core/scripts/cleanup-LHR-for-diff.js ./lighthouse-core/test/results/sample_v2.json --only-remove-timing",
|
||||
"diff:sample-json": "bash lighthouse-core/scripts/assert-golden-lhr-unchanged.sh",
|
||||
"update:crdp-typings": "node lighthouse-core/scripts/extract-crdp-mapping.js",
|
||||
"mixed-content": "./lighthouse-cli/index.js --chrome-flags='--headless' --config-path=./lighthouse-core/config/mixed-content.js"
|
||||
"mixed-content": "./lighthouse-cli/index.js --chrome-flags='--headless' --config-path=./lighthouse-core/config/mixed-content.js",
|
||||
"minify-latest-run": "./lighthouse-core/scripts/lantern/minify-trace.js ./latest-run/defaultPass.trace.json ./latest-run/defaultPass.trace.min.json && ./lighthouse-core/scripts/lantern/minify-devtoolslog.js ./latest-run/defaultPass.devtoolslog.json ./latest-run/defaultPass.devtoolslog.min.json"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/chrome": "^0.0.60",
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче