2020-03-05 03:24:23 +03:00
|
|
|
#!/usr/bin/python
|
|
|
|
# vim:sw=4:ts=4:et:
|
|
|
|
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
|
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
|
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
|
|
|
|
|
|
# This script uses `fix-stacks` to post-process the entries produced by
|
|
|
|
# MozFormatCodeAddress().
|
|
|
|
|
2020-03-13 01:03:19 +03:00
|
|
|
from __future__ import absolute_import, print_function
|
2020-03-05 03:24:23 +03:00
|
|
|
from subprocess import Popen, PIPE
|
2020-05-28 03:34:30 +03:00
|
|
|
import atexit
|
2020-03-05 03:24:23 +03:00
|
|
|
import os
|
|
|
|
import platform
|
2020-03-10 09:19:21 +03:00
|
|
|
import re
|
2020-03-05 03:24:23 +03:00
|
|
|
import sys
|
|
|
|
|
2020-06-02 02:14:05 +03:00
|
|
|
import six
|
|
|
|
|
2020-03-10 09:19:21 +03:00
|
|
|
# Matches lines produced by MozFormatCodeAddress(), e.g.
|
|
|
|
# `#01: ???[tests/example +0x43a0]`.
|
|
|
|
line_re = re.compile("#\d+: .+\[.+ \+0x[0-9A-Fa-f]+\]")
|
2020-03-05 03:24:23 +03:00
|
|
|
|
|
|
|
fix_stacks = None
|
|
|
|
|
2020-04-21 23:12:09 +03:00
|
|
|
|
2020-04-29 02:55:57 +03:00
|
|
|
def fixSymbols(
|
|
|
|
line, jsonMode=False, slowWarning=False, breakpadSymsDir=None, hide_errors=False
|
|
|
|
):
|
2020-03-05 03:24:23 +03:00
|
|
|
global fix_stacks
|
|
|
|
|
2020-06-02 02:14:05 +03:00
|
|
|
line = six.ensure_str(line)
|
2020-04-21 23:12:09 +03:00
|
|
|
result = line_re.search(line)
|
|
|
|
if result is None:
|
|
|
|
return line
|
2020-03-05 03:24:23 +03:00
|
|
|
|
2020-04-21 23:12:09 +03:00
|
|
|
if not fix_stacks:
|
|
|
|
# Look in MOZ_FETCHES_DIR (for automation), then in MOZBUILD_STATE_PATH
|
|
|
|
# (for a local build where the user has that set), then in ~/.mozbuild
|
|
|
|
# (for a local build with default settings).
|
|
|
|
base = os.environ.get(
|
|
|
|
"MOZ_FETCHES_DIR",
|
|
|
|
os.environ.get("MOZBUILD_STATE_PATH", os.path.expanduser("~/.mozbuild")),
|
|
|
|
)
|
|
|
|
fix_stacks_exe = base + "/fix-stacks/fix-stacks"
|
|
|
|
if platform.system() == "Windows":
|
|
|
|
fix_stacks_exe = fix_stacks_exe + ".exe"
|
2020-03-16 04:15:27 +03:00
|
|
|
|
2020-04-21 23:12:09 +03:00
|
|
|
if not (os.path.isfile(fix_stacks_exe) and os.access(fix_stacks_exe, os.X_OK)):
|
|
|
|
raise Exception("cannot find `fix-stacks`; please run `./mach bootstrap`")
|
2020-03-05 03:24:23 +03:00
|
|
|
|
2020-04-21 23:12:09 +03:00
|
|
|
args = [fix_stacks_exe]
|
|
|
|
if jsonMode:
|
|
|
|
args.append("-j")
|
|
|
|
if breakpadSymsDir:
|
|
|
|
args.append("-b")
|
2020-09-01 11:58:14 +03:00
|
|
|
args.append(breakpadSymsDir)
|
Bug 1628527 - Introduce explicit initialization and finalization of `fix-stacks`. r=erahm,perftest-reviewers,sparky,gbrown
Currently AWSY-with-DMD doesn't work on Windows. This is because `fix-stacks`
is initialized lazily, and by the time the initialization happens some file
descriptors for files are open, and that leads to some major Python2-on-Windows
sadness as described in the big comment in the commit.
To fix the problem, this commit adds an `init` function to `fix_stacks.py` so
that `fix-stacks` can be initialized eagerly, hopefully before any file
descriptors for files are open.
For `dmd.py`, other than fixing the AWSY problems, this has little effect,
because `fix-stacks` is always initialized.
For `utils.py`, which is used to process the output of most tests, this has a
more noticeable effect: the `fix-stacks` process is always spawned, rather than
being spawned only when needed. If no stack traces appear in the test output,
this means that `fix-stacks` is spawned unnecessarily. But it's cheap to spawn;
the expensive part only happens when stack traces start getting fixed. So I
think this change in behaviour is acceptable.
Furthermore, the commit adds a `finish` function to `fix_stacks.py`, so that
the `fix-stacks` process can be explicitly shut down. This has never been done
for processes spawned for any of the stack fixing scripts. It's never caused
problems on Linux/Mac, but it seems to be necessary on Windows to avoid
similar "this file is locked" problems with the test_dmd.js test.
The commit also renames some things to more standard Python style, e.g.
`json_mode` instead of `jsonMode`.
Finally, Android tests use `utils.py` from the repository but `fix_stacks.py`
from the Android host utils. Because the two scripts must be updated in tandem,
this commit also updates the Android host utils to a version that contains the
updated `fix_stacks.py`. Thanks to aerickson for packaging up the new Android
host utils and providing the change to the `hostutils.manifest` file.
Differential Revision: https://phabricator.services.mozilla.com/D69478
2020-04-21 01:43:25 +03:00
|
|
|
|
2020-04-29 02:55:57 +03:00
|
|
|
# Sometimes we need to prevent errors from going to stderr.
|
|
|
|
stderr = open(os.devnull) if hide_errors else None
|
|
|
|
|
2020-07-01 01:17:38 +03:00
|
|
|
fix_stacks = Popen(
|
|
|
|
args, stdin=PIPE, stdout=PIPE, stderr=stderr, universal_newlines=True
|
|
|
|
)
|
Bug 1628527 - Introduce explicit initialization and finalization of `fix-stacks`. r=erahm,perftest-reviewers,sparky,gbrown
Currently AWSY-with-DMD doesn't work on Windows. This is because `fix-stacks`
is initialized lazily, and by the time the initialization happens some file
descriptors for files are open, and that leads to some major Python2-on-Windows
sadness as described in the big comment in the commit.
To fix the problem, this commit adds an `init` function to `fix_stacks.py` so
that `fix-stacks` can be initialized eagerly, hopefully before any file
descriptors for files are open.
For `dmd.py`, other than fixing the AWSY problems, this has little effect,
because `fix-stacks` is always initialized.
For `utils.py`, which is used to process the output of most tests, this has a
more noticeable effect: the `fix-stacks` process is always spawned, rather than
being spawned only when needed. If no stack traces appear in the test output,
this means that `fix-stacks` is spawned unnecessarily. But it's cheap to spawn;
the expensive part only happens when stack traces start getting fixed. So I
think this change in behaviour is acceptable.
Furthermore, the commit adds a `finish` function to `fix_stacks.py`, so that
the `fix-stacks` process can be explicitly shut down. This has never been done
for processes spawned for any of the stack fixing scripts. It's never caused
problems on Linux/Mac, but it seems to be necessary on Windows to avoid
similar "this file is locked" problems with the test_dmd.js test.
The commit also renames some things to more standard Python style, e.g.
`json_mode` instead of `jsonMode`.
Finally, Android tests use `utils.py` from the repository but `fix_stacks.py`
from the Android host utils. Because the two scripts must be updated in tandem,
this commit also updates the Android host utils to a version that contains the
updated `fix_stacks.py`. Thanks to aerickson for packaging up the new Android
host utils and providing the change to the `hostutils.manifest` file.
Differential Revision: https://phabricator.services.mozilla.com/D69478
2020-04-21 01:43:25 +03:00
|
|
|
|
2020-05-28 03:34:30 +03:00
|
|
|
# Shut down the fix_stacks process on exit. We use `terminate()`
|
|
|
|
# because it is more forceful than `wait()`, and the Python docs warn
|
|
|
|
# about possible deadlocks with `wait()`.
|
|
|
|
def cleanup(fix_stacks):
|
2020-12-14 17:11:10 +03:00
|
|
|
for fn in [fix_stacks.stdin.close, fix_stacks.terminate]:
|
|
|
|
|
|
|
|
try:
|
|
|
|
fn()
|
|
|
|
except OSError:
|
|
|
|
pass
|
2020-10-26 21:34:53 +03:00
|
|
|
|
2020-05-28 03:34:30 +03:00
|
|
|
atexit.register(cleanup, fix_stacks)
|
|
|
|
|
2020-04-21 23:12:09 +03:00
|
|
|
if slowWarning:
|
|
|
|
print(
|
|
|
|
"Initializing stack-fixing for the first stack frame, this may take a while..."
|
|
|
|
)
|
2020-03-13 01:03:19 +03:00
|
|
|
|
2020-03-10 09:19:21 +03:00
|
|
|
# Sometimes `line` is lacking a trailing newline. If we pass such a `line`
|
|
|
|
# to `fix-stacks` it will wait until it receives a newline, causing this
|
|
|
|
# script to hang. So we add a newline if one is missing and then remove it
|
|
|
|
# from the output.
|
|
|
|
is_missing_newline = not line.endswith("\n")
|
|
|
|
if is_missing_newline:
|
|
|
|
line = line + "\n"
|
2020-03-05 03:24:23 +03:00
|
|
|
fix_stacks.stdin.write(line)
|
|
|
|
fix_stacks.stdin.flush()
|
2020-03-10 09:19:21 +03:00
|
|
|
out = fix_stacks.stdout.readline()
|
|
|
|
if is_missing_newline:
|
|
|
|
out = out[:-1]
|
|
|
|
return out
|
2020-03-05 03:24:23 +03:00
|
|
|
|
|
|
|
|
|
|
|
if __name__ == "__main__":
|
|
|
|
for line in sys.stdin:
|
2020-04-21 23:12:09 +03:00
|
|
|
sys.stdout.write(fixSymbols(line))
|