Implement pino-mozlog in addons-frontend (#11986)

* Implement pino-mozlog in addons-frontend

* Commit fixture file
This commit is contained in:
Bob Silverberg 2023-01-13 11:59:47 -05:00 коммит произвёл GitHub
Родитель fe31ee8e49
Коммит 0c6a32572c
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
9 изменённых файлов: 606 добавлений и 16 удалений

24
bin/pino-mozlog.js Normal file
Просмотреть файл

@ -0,0 +1,24 @@
#!/usr/bin/env node
const pump = require('pump');
const split = require('split2');
const { Transform } = require('readable-stream');
const {
createParseFunction,
createTransformFunction,
parseOptions,
} = require('../src/pino-mozlog/index');
const options = parseOptions(process.argv.slice(2));
const mozlogTransport = new Transform({
objectMode: true,
transform: createTransformFunction({ options }),
});
pump(
process.stdin,
split(createParseFunction({ options })),
mozlogTransport,
process.stdout
);

Просмотреть файл

@ -33,7 +33,7 @@
"prettier": "prettier --write '**'",
"prettier-ci": "prettier --list-different '**' || (echo '\n\nThis failure means you did not run `yarn prettier-dev` before committing\n\n' && exit 1)",
"prettier-dev": "pretty-quick --branch master",
"start": "npm run version-check && better-npm-run --silent node bin/server.js | pino-mozlog",
"start": "npm run version-check && better-npm-run --silent node bin/server.js | node bin/pino-mozlog.js",
"start-func-test-server": "better-npm-run node bin/start-func-test-server.js",
"test-ci": "bin/config-check.js && npm run build-locales && better-npm-run test-ci",
"test-ci-next": "bin/config-check.js && npm run build-locales && better-npm-run test-ci-next",
@ -208,14 +208,16 @@
"jsdom": "21.0.0",
"localforage": "1.10.0",
"lodash.debounce": "4.0.8",
"minimist": "1.2.7",
"moment": "2.29.4",
"nano-time": "1.0.0",
"normalize.css": "8.0.1",
"photon-colors": "3.3.2",
"photoswipe": "4.1.3",
"pino": "8.8.0",
"pino-mozlog": "2.12.0",
"pino-syslog": "2.0.0",
"prop-types": "15.8.1",
"pump": "3.0.0",
"qhistory": "1.1.0",
"qs": "6.11.0",
"rc-tooltip": "5.2.2",
@ -234,6 +236,7 @@
"react-super-responsive-table": "5.2.1",
"react-textarea-autosize": "8.4.0",
"react-transition-group": "4.4.5",
"readable-stream": "4.3.0",
"redux": "4.2.0",
"redux-first-history": "5.1.1",
"redux-logger": "3.0.6",
@ -241,6 +244,7 @@
"response-time": "2.3.2",
"schema-utils": "4.0.0",
"serialize-javascript": "6.0.0",
"split2": "4.1.0",
"touch": "3.1.0",
"ua-parser-js": "1.0.32",
"universal-base64url": "1.1.0",
@ -294,6 +298,7 @@
"jest": "^29.0.0",
"jest-environment-jsdom": "^29.0.0",
"jest-extended": "^3.0.0",
"jest-json-schema": "^6.1.0",
"jest-watch-typeahead": "^2.0.0",
"lint-staged": "^13.0.0",
"mini-css-extract-plugin": "^2.0.0",
@ -345,4 +350,4 @@
"maxSize": "32 kB"
}
]
}
}

124
src/pino-mozlog/index.js Normal file
Просмотреть файл

@ -0,0 +1,124 @@
/*
A transport for transforming pino logs(https://github.com/pinojs) into mozlog
(https://wiki.mozilla.org/Firefox/Services/Logging#MozLog_application_logging_standard).
*/
const minimist = require('minimist');
const pinoSyslog = require('pino-syslog/lib/utils');
const ENV_VERSION = '2.0';
const DEFAULT_OPTIONS = {
silent: false,
type: 'app.log',
};
const STACKDRIVER_LEVEL_MAP = {
[pinoSyslog.severity.emergency]: 800,
[pinoSyslog.severity.alert]: 700,
[pinoSyslog.severity.critical]: 600,
[pinoSyslog.severity.error]: 500,
[pinoSyslog.severity.warning]: 400,
[pinoSyslog.severity.notice]: 300,
[pinoSyslog.severity.info]: 200,
[pinoSyslog.severity.debug]: 100,
};
const getStackdriverSeverity = (severity) => {
return STACKDRIVER_LEVEL_MAP[severity] || 0;
};
const createParseFunction = ({
_console = console,
options = DEFAULT_OPTIONS,
} = {}) => {
return (data) => {
try {
return JSON.parse(data);
} catch (error) {
if (!options.silent) {
_console.error('[pino-mozlog] could not parse:', {
error: error.toString(),
data,
});
}
}
return {};
};
};
const format = (
{
hostname,
level,
name,
pid,
time,
v, // this field is ignored
...fields
},
options = DEFAULT_OPTIONS,
) => {
const syslogSeverity = pinoSyslog.levelToSeverity(level);
return {
EnvVersion: ENV_VERSION,
Fields: fields,
Hostname: hostname,
Logger: name,
Pid: pid,
Severity: syslogSeverity,
Timestamp: time, // should be in nanoseconds
Type: options.type,
// Add a custom key for stackdriver.
severity: getStackdriverSeverity(syslogSeverity),
};
};
const createTransformFunction = ({
_console = console,
_format = format,
options = DEFAULT_OPTIONS,
} = {}) => {
return (record, enc, cb) => {
try {
if (typeof record.time === 'undefined') {
throw new Error('invalid pino record');
}
_console.log(JSON.stringify(_format(record, options)));
} catch (error) {
if (!options.silent) {
_console.error('[pino-mozlog] could not format:', {
error: error.toString(),
record,
});
}
}
cb();
};
};
const parseOptions = (argv) => {
const keys = Object.keys(DEFAULT_OPTIONS);
const { _, ...options } = minimist(argv, {
boolean: keys.filter((k) => typeof DEFAULT_OPTIONS[k] === 'boolean'),
default: DEFAULT_OPTIONS,
string: keys.filter((k) => typeof DEFAULT_OPTIONS[k] === 'string'),
unknown: () => false,
});
return options;
};
module.exports = {
DEFAULT_OPTIONS,
ENV_VERSION,
createParseFunction,
createTransformFunction,
format,
getStackdriverSeverity,
parseOptions,
};

Просмотреть файл

@ -0,0 +1,9 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
exports[`functional tests converts pino logs to mozlog 1`] = `
"{"EnvVersion":"2.0","Fields":{"msg":"Sentry reporting is disabled; Set config.sentryDsn to enable it."},"Hostname":"big","Logger":"amo","Pid":85547,"Severity":4,"Timestamp":1612950421902131000,"Type":"app.log","severity":400}
{"EnvVersion":"2.0","Fields":{"msg":"🔥 Addons-frontend server is running [ENV:development] [isDevelopment:true] [isDeployed:false] [apiHost:http://localhost:3000] [apiPath:/api/] [apiVersion:v5]"},"Hostname":"big","Logger":"amo","Pid":85547,"Severity":6,"Timestamp":1612950421911287000,"Type":"app.log","severity":200}
{"EnvVersion":"2.0","Fields":{"msg":"🚦 Proxy detected, frontend running at http://127.0.0.1:3333."},"Hostname":"big","Logger":"amo","Pid":85547,"Severity":7,"Timestamp":1612950421911469000,"Type":"app.log","severity":100}
{"EnvVersion":"2.0","Fields":{"msg":"👁 Open your browser at http://127.0.0.1:3000 to view it."},"Hostname":"big","Logger":"amo","Pid":85547,"Severity":7,"Timestamp":1612950421911601000,"Type":"app.log","severity":100}
"
`;

Просмотреть файл

@ -0,0 +1,4 @@
{"level":40,"time":1612950421902130927,"pid":85547,"hostname":"big","name":"amo","msg":"Sentry reporting is disabled; Set config.sentryDsn to enable it."}
{"level":30,"time":1612950421911287147,"pid":85547,"hostname":"big","name":"amo","msg":"🔥 Addons-frontend server is running [ENV:development] [isDevelopment:true] [isDeployed:false] [apiHost:http://localhost:3000] [apiPath:/api/] [apiVersion:v5]"}
{"level":20,"time":1612950421911469125,"pid":85547,"hostname":"big","name":"amo","msg":"🚦 Proxy detected, frontend running at http://127.0.0.1:3333."}
{"level":20,"time":1612950421911600770,"pid":85547,"hostname":"big","name":"amo","msg":"👁 Open your browser at http://127.0.0.1:3000 to view it."}

Просмотреть файл

@ -0,0 +1,103 @@
{
"definitions": {
"field_array": {
"minItems": 1,
"oneOf": [
{
"items": {
"type": "string"
}
},
{
"items": {
"type": "number"
}
},
{
"items": {
"type": "boolean"
}
}
],
"type": "array"
},
"field_object": {
"properties": {
"representation": {
"type": "string"
},
"value": {
"oneOf": [
{
"$ref": "#/definitions/field_value"
},
{
"$ref": "#/definitions/field_array"
}
]
}
},
"required": [
"value"
],
"type": "object"
},
"field_value": {
"type": [
"string",
"number",
"boolean"
]
}
},
"properties": {
"EnvVersion": {
"pattern": "^\\d+(?:\\.\\d+){0,2}$",
"type": "string"
},
"Fields": {
"additionalProperties": {
"anyOf": [
{
"$ref": "#/definitions/field_value"
},
{
"$ref": "#/definitions/field_array"
},
{
"$ref": "#/definitions/field_object"
}
]
},
"minProperties": 1,
"type": "object"
},
"Hostname": {
"format": "hostname",
"type": "string"
},
"Logger": {
"type": "string"
},
"Pid": {
"minimum": 0,
"type": "integer"
},
"Severity": {
"maximum": 7,
"minimum": 0,
"type": "integer"
},
"Timestamp": {
"minimum": 0,
"type": "integer"
},
"Type": {
"type": "string"
}
},
"required": [
"Timestamp"
],
"type": "object"
}

Просмотреть файл

@ -0,0 +1,15 @@
const util = require('util');
const exec = util.promisify(require('child_process').exec);
// eslint-disable-next-line amo/describe-with-filename
describe('functional tests', () => {
it('converts pino logs to mozlog', async () => {
const { stdout } = await exec(
'cat tests/pino-mozlog/fixtures/frontend.log | node bin/pino-mozlog.js',
);
// eslint-disable-next-line no-console
console.log(`${stdout}`);
expect(stdout).toMatchSnapshot();
});
});

Просмотреть файл

@ -0,0 +1,268 @@
const { matchers } = require('jest-json-schema');
const pinoSyslog = require('pino-syslog/lib/utils');
const {
DEFAULT_OPTIONS,
ENV_VERSION,
createParseFunction,
createTransformFunction,
format,
getStackdriverSeverity,
parseOptions,
} = require('../../src/pino-mozlog/index');
const mozlogSchema = require('./mozlog-schema');
describe(__filename, () => {
// Setup JSON schema matchers.
expect.extend(matchers);
let _console;
const createPinoRecord = (fields = {}) => {
return {
hostname: 'host.example.org',
level: 10,
msg: 'some message',
name: 'app',
pid: 12345,
time: Date.now(),
v: 1,
...fields,
};
};
beforeEach(() => {
_console = { error: jest.fn() };
});
describe('parse', () => {
let parse;
beforeEach(() => {
parse = createParseFunction({ _console });
});
it('parses a JSON string', () => {
const data = { some: 'object' };
expect(parse(JSON.stringify(data))).toEqual(data);
expect(_console.error).not.toHaveBeenCalled();
});
it('returns an empty object when invalid JSON is supplied', () => {
const data = 'not JSON data';
expect(parse(data)).toEqual({});
expect(_console.error).toHaveBeenCalledWith(
'[pino-mozlog] could not parse:',
{
error: 'SyntaxError: Unexpected token o in JSON at position 1',
data,
},
);
});
it('returns an empty object when an empty string is supplied', () => {
const data = '';
expect(parse(data)).toEqual({});
expect(_console.error).toHaveBeenCalledWith(
'[pino-mozlog] could not parse:',
{
error: 'SyntaxError: Unexpected end of JSON input',
data,
},
);
});
describe('with --silent', () => {
const options = {
...DEFAULT_OPTIONS,
silent: true,
};
beforeEach(() => {
parse = createParseFunction({ _console, options });
});
it('returns an empty object when invalid JSON is supplied', () => {
parse('not JSON data');
expect(_console.error).not.toHaveBeenCalled();
});
it('returns an empty object when an empty string is supplied', () => {
parse('');
expect(_console.error).not.toHaveBeenCalled();
});
});
});
describe('format', () => {
it('formats a record using the mozlog format', () => {
const record = createPinoRecord();
expect(format(record)).toEqual({
EnvVersion: ENV_VERSION,
Fields: {
msg: record.msg,
},
Hostname: record.hostname,
Logger: record.name,
Pid: record.pid,
Severity: 7,
Timestamp: record.time,
Type: DEFAULT_OPTIONS.type,
severity: 100,
});
});
it('adds extra information to Fields', () => {
const fields = { other: 'value', msg: 'important' };
const record = createPinoRecord(fields);
expect(format(record).Fields).toEqual(fields);
});
it('can be configured with a user-defined type', () => {
const record = createPinoRecord();
const type = 'some-type';
const options = {
...DEFAULT_OPTIONS,
type,
};
expect(format(record, options).Type).toEqual(type);
});
it('omits the "v" attribute', () => {
const record = createPinoRecord({ msg: undefined, v: 123 });
expect(format(record).Fields).toEqual({});
});
it('complies with the mozlog JSON schema', () => {
const record = createPinoRecord({ foo: 'foo', bar: true, baz: 123 });
expect(format(record)).toMatchSchema(mozlogSchema);
});
});
describe('createTransformFunction', () => {
it('calls the format function when transforming a record', () => {
const record = createPinoRecord();
const callback = jest.fn();
const _format = jest.fn();
_format.mockImplementation(() => 'a mozlog');
const transform = createTransformFunction({ _format });
transform(record, null, callback);
expect(_format).toHaveBeenCalledWith(record, DEFAULT_OPTIONS);
expect(callback).toHaveBeenCalled();
});
it('does not call the format function when the record is an empty object', () => {
const _format = jest.fn();
const record = {};
const transform = createTransformFunction({ _console, _format });
transform(record, null, jest.fn());
expect(_format).not.toHaveBeenCalled();
expect(_console.error).toHaveBeenCalledWith(
'[pino-mozlog] could not format:',
{
error: 'Error: invalid pino record',
record,
},
);
});
it('calls the callback even in case of an error', () => {
const callback = jest.fn();
const transform = createTransformFunction({ _console });
transform({}, null, callback);
expect(callback).toHaveBeenCalled();
expect(_console.error).toHaveBeenCalled();
});
describe('with --silent', () => {
const options = {
...DEFAULT_OPTIONS,
silent: true,
};
it('does not call the format function when the record is an empty object', () => {
const record = {};
const transform = createTransformFunction({
_console,
options,
});
transform(record, null, jest.fn());
expect(_console.error).not.toHaveBeenCalled();
});
});
});
describe('parseOptions', () => {
it('returns the default options', () => {
const options = parseOptions([]);
expect(options).toEqual(DEFAULT_OPTIONS);
});
it('accepts the --silent boolean option', () => {
const options = parseOptions(['--silent']);
expect(options).toEqual({
...DEFAULT_OPTIONS,
silent: true,
});
});
it('accepts the --type string option', () => {
const type = 'some-type';
const options = parseOptions(['--type', type]);
expect(options).toEqual({
...DEFAULT_OPTIONS,
type,
});
});
it('ignores unknown options', () => {
const options = parseOptions(['--unknown', 'option']);
expect(options).toEqual(DEFAULT_OPTIONS);
});
});
describe('getStackdriverSeverity', () => {
it.each([
[pinoSyslog.severity.emergency, 800],
[pinoSyslog.severity.alert, 700],
[pinoSyslog.severity.critical, 600],
[pinoSyslog.severity.error, 500],
[pinoSyslog.severity.warning, 400],
[pinoSyslog.severity.notice, 300],
[pinoSyslog.severity.info, 200],
[pinoSyslog.severity.debug, 100],
])(
'returns the stackdriver level for (syslog) severity = %d',
(syslogSeverity, stackdriverSeverity) => {
expect(getStackdriverSeverity(syslogSeverity)).toEqual(
stackdriverSeverity,
);
},
);
});
it('returns 0 for unsupported syslog severities', () => {
expect(getStackdriverSeverity(-1)).toEqual(0);
expect(getStackdriverSeverity(123)).toEqual(0);
});
});

Просмотреть файл

@ -2194,6 +2194,16 @@ ajv@^8.0.0, ajv@^8.0.1, ajv@^8.8.0:
require-from-string "^2.0.2"
uri-js "^4.2.2"
ajv@^8.8.2:
version "8.12.0"
resolved "https://registry.yarnpkg.com/ajv/-/ajv-8.12.0.tgz#d1a0527323e22f53562c567c00991577dfbe19d1"
integrity sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA==
dependencies:
fast-deep-equal "^3.1.1"
json-schema-traverse "^1.0.0"
require-from-string "^2.0.2"
uri-js "^4.2.2"
align-text@^0.1.1, align-text@^0.1.3:
version "0.1.4"
resolved "https://registry.yarnpkg.com/align-text/-/align-text-0.1.4.tgz#0cd90a561093f35d0a99256c22b7069433fad117"
@ -3674,6 +3684,11 @@ dezalgo@^1.0.4:
asap "^2.0.0"
wrappy "1"
diff-sequences@^27.5.1:
version "27.5.1"
resolved "https://registry.yarnpkg.com/diff-sequences/-/diff-sequences-27.5.1.tgz#eaecc0d327fd68c8d9672a1e64ab8dccb2ef5327"
integrity sha512-k1gCAXAsNgLwEL+Y8Wvl+M6oEFj5bgazfZULpS5CneoPPXRaCCW7dm+q21Ky2VEE5X+VeRDBVg1Pcvvsr4TtNQ==
diff-sequences@^29.3.1:
version "29.3.1"
resolved "https://registry.yarnpkg.com/diff-sequences/-/diff-sequences-29.3.1.tgz#104b5b95fe725932421a9c6e5b4bef84c3f2249e"
@ -5851,6 +5866,16 @@ jest-config@^29.3.1:
slash "^3.0.0"
strip-json-comments "^3.1.1"
jest-diff@^27.5.1:
version "27.5.1"
resolved "https://registry.yarnpkg.com/jest-diff/-/jest-diff-27.5.1.tgz#a07f5011ac9e6643cf8a95a462b7b1ecf6680def"
integrity sha512-m0NvkX55LDt9T4mctTEgnZk3fmEg3NRYutvMPWM/0iPnkFj2wIeF45O1718cMSOFO1vINkqmxqD8vE37uTEbqw==
dependencies:
chalk "^4.0.0"
diff-sequences "^27.5.1"
jest-get-type "^27.5.1"
pretty-format "^27.5.1"
jest-diff@^29.0.0, jest-diff@^29.3.1:
version "29.3.1"
resolved "https://registry.yarnpkg.com/jest-diff/-/jest-diff-29.3.1.tgz#d8215b72fed8f1e647aed2cae6c752a89e757527"
@ -5913,6 +5938,11 @@ jest-extended@^3.0.0:
jest-diff "^29.0.0"
jest-get-type "^29.0.0"
jest-get-type@^27.5.1:
version "27.5.1"
resolved "https://registry.yarnpkg.com/jest-get-type/-/jest-get-type-27.5.1.tgz#3cd613c507b0f7ace013df407a1c1cd578bcb4f1"
integrity sha512-2KY95ksYSaK7DMBWQn6dQz3kqAf3BB64y2udeG+hv4KfSOb9qwcYQstTJc1KCbsix+wLZWZYN8t7nwX3GOBLRw==
jest-get-type@^29.0.0, jest-get-type@^29.2.0:
version "29.2.0"
resolved "https://registry.yarnpkg.com/jest-get-type/-/jest-get-type-29.2.0.tgz#726646f927ef61d583a3b3adb1ab13f3a5036408"
@ -5937,6 +5967,16 @@ jest-haste-map@^29.3.1:
optionalDependencies:
fsevents "^2.3.2"
jest-json-schema@6.1.0:
version "6.1.0"
resolved "https://registry.yarnpkg.com/jest-json-schema/-/jest-json-schema-6.1.0.tgz#68ccc23424a7b20550d59ab9186530f81113e6af"
integrity sha512-LMHuLmKjr/4X+H8v1xF5TEwfYEkzwGeWJ0epYQVQhlVTDDR5FWCdSO8vmsecb5cLf9NeWAqMKn3qhJvP9um0AA==
dependencies:
ajv "^8.8.2"
ajv-formats "^2.1.1"
chalk "^4.1.2"
jest-matcher-utils "^27.3.1"
jest-leak-detector@^29.3.1:
version "29.3.1"
resolved "https://registry.yarnpkg.com/jest-leak-detector/-/jest-leak-detector-29.3.1.tgz#95336d020170671db0ee166b75cd8ef647265518"
@ -5945,6 +5985,16 @@ jest-leak-detector@^29.3.1:
jest-get-type "^29.2.0"
pretty-format "^29.3.1"
jest-matcher-utils@^27.3.1:
version "27.5.1"
resolved "https://registry.yarnpkg.com/jest-matcher-utils/-/jest-matcher-utils-27.5.1.tgz#9c0cdbda8245bc22d2331729d1091308b40cf8ab"
integrity sha512-z2uTx/T6LBaCoNWNFWwChLBKYxTMcGBRjAt+2SbP929/Fflb9aa5LGma654Rz8z9HLxsrUaYzxE9T/EFIL/PAw==
dependencies:
chalk "^4.0.0"
jest-diff "^27.5.1"
jest-get-type "^27.5.1"
pretty-format "^27.5.1"
jest-matcher-utils@^29.3.1:
version "29.3.1"
resolved "https://registry.yarnpkg.com/jest-matcher-utils/-/jest-matcher-utils-29.3.1.tgz#6e7f53512f80e817dfa148672bd2d5d04914a572"
@ -7603,17 +7653,6 @@ pino-devtools@^2.1.0:
optionalDependencies:
fsevents "2.3.2"
pino-mozlog@2.12.0:
version "2.12.0"
resolved "https://registry.yarnpkg.com/pino-mozlog/-/pino-mozlog-2.12.0.tgz#31676a422de4e57ef3ca8c84cd0c33099be5e1e0"
integrity sha512-IaNEHAhcvAIWdGemcNWnGqSdphIg1v/Otqj5wlMvNAHqnXelTopQInm0qU61b40WRY0h8JiQR1kI1hlJXXr9ew==
dependencies:
minimist "1.2.7"
pino-syslog "2.0.0"
pump "3.0.0"
readable-stream "4.3.0"
split2 "4.1.0"
pino-pretty@^9.0.0:
version "9.1.1"
resolved "https://registry.yarnpkg.com/pino-pretty/-/pino-pretty-9.1.1.tgz#e7d64c1db98266ca428ab56567b844ba780cd0e1"
@ -7697,7 +7736,6 @@ pngjs@^5.0.0:
po2json@mikeedwards/po2json#51e2310485bbe35e9e57f2eee238185459ca0eab:
version "1.0.0-beta-3"
uid "51e2310485bbe35e9e57f2eee238185459ca0eab"
resolved "https://codeload.github.com/mikeedwards/po2json/tar.gz/51e2310485bbe35e9e57f2eee238185459ca0eab"
dependencies:
commander "^6.0.0"
@ -8010,7 +8048,7 @@ pretty-error@^4.0.0:
lodash "^4.17.20"
renderkid "^3.0.0"
pretty-format@^27.0.2:
pretty-format@^27.0.2, pretty-format@^27.5.1:
version "27.5.1"
resolved "https://registry.yarnpkg.com/pretty-format/-/pretty-format-27.5.1.tgz#2181879fdea51a7a5851fb39d920faa63f01d88e"
integrity sha512-Qb1gy5OrP5+zDf2Bvnzdl3jsTf1qXVMazbvCoKhtKqVs4/YK4ozX4gKQJJVyNe+cajNPn0KoC0MC3FUmaHWEmQ==