Bug 937012 - Replace the busted find_vanilla_new_calls script with the much better check_vanilla_allocations.py. r=evilpie,gps.

--HG--
extra : rebase_source : 08f0cf236a1f4269685f38b510e26f76cbf56972
This commit is contained in:
Nicholas Nethercote 2013-11-11 18:37:55 +11:00
Родитель d7f090b746
Коммит bd7840a865
6 изменённых файлов: 371 добавлений и 166 удалений

Просмотреть файл

@ -0,0 +1,159 @@
# vim: set ts=8 sts=4 et sw=4 tw=79:
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
#----------------------------------------------------------------------------
# All heap allocations in SpiderMonkey must go through js_malloc, js_calloc,
# js_realloc, and js_free. This is so that any embedder who uses a custom
# allocator (by defining JS_USE_CUSTOM_ALLOCATOR) will see all heap allocation
# go through that custom allocator.
#
# Therefore, the presence of any calls to "vanilla" allocation/free functions
# (e.g. malloc(), free()) is a bug.
#
# This script checks for the presence of such disallowed vanilla
# allocation/free function in SpiderMonkey when it's built as a library. It
# relies on |nm| from the GNU binutils, and so only works on Linux, but one
# platform is good enough to catch almost all violations.
#
# This checking is only 100% reliable in a JS_USE_CUSTOM_ALLOCATOR build in
# which the default definitions of js_malloc et al (in Utility.h) -- which call
# malloc et al -- are replaced with empty definitions. This is because the
# presence and possible inlining of the default js_malloc et al can cause
# malloc/calloc/realloc/free calls show up in unpredictable places.
#
# Unfortunately, that configuration cannot be tested on Mozilla's standard
# testing infrastructure. Instead, by default this script only tests that none
# of the other vanilla allocation/free functions (operator new, memalign, etc)
# are present. If given the --aggressive flag, it will also check for
# malloc/calloc/realloc/free.
#
# Note: We don't check for |operator delete| and |operator delete[]|. These
# can be present somehow due to virtual destructors, but this is not too
# because vanilla delete/delete[] calls don't make sense without corresponding
# vanilla new/new[] calls, and any explicit calls will be caught by Valgrind's
# mismatched alloc/free checking.
#----------------------------------------------------------------------------
from __future__ import print_function
import argparse
import re
import subprocess
import sys
# The obvious way to implement this script is to search for occurrences of
# malloc et al, succeed if none are found, and fail is some are found.
# However, "none are found" does not necessarily mean "none are present" --
# this script could be buggy. (Or the output format of |nm| might change in
# the future.)
#
# So jsutil.cpp deliberately contains a (never-called) function that contains a
# single use of all the vanilla allocation/free functions. And this script
# fails if it (a) finds uses of those functions in files other than jsutil.cpp,
# *or* (b) fails to find them in jsutil.cpp.
# Tracks overall success of the test.
has_failed = False
def fail(msg):
print('TEST-UNEXPECTED-FAIL | check_vanilla_allocations.py |', msg)
global has_failed
has_failed = True
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--aggressive', action='store_true',
help='also check for malloc, calloc, realloc and free')
parser.add_argument('file', type=str,
help='name of the file to check')
args = parser.parse_args()
# Run |nm|. Options:
# -u: show only undefined symbols
# -C: demangle symbol names
# -l: show a filename and line number for each undefined symbol
cmd = ['nm', '-u', '-C', '-l', args.file]
lines = subprocess.check_output(cmd, universal_newlines=True,
stderr=subprocess.PIPE).split('\n')
# alloc_fns contains all the vanilla allocation/free functions that we look
# for. Regexp chars are escaped appropriately.
alloc_fns = [
# Matches |operator new(unsigned T)|, where |T| is |int| or |long|.
r'operator new\(unsigned',
# Matches |operator new[](unsigned T)|, where |T| is |int| or |long|.
r'operator new\[\]\(unsigned',
r'memalign',
# These two aren't available on all Linux configurations.
#r'posix_memalign',
#r'aligned_alloc',
r'valloc',
r'strdup'
]
if args.aggressive:
alloc_fns += [
r'malloc',
r'calloc',
r'realloc',
r'free'
]
# This is like alloc_fns, but regexp chars are not escaped.
alloc_fns_unescaped = [fn.translate(None, r'\\') for fn in alloc_fns]
# This regexp matches the relevant lines in the output of |nm|, which look
# like the following.
#
# U malloc /path/to/objdir/dist/include/js/Utility.h:142
#
alloc_fns_re = r'U (' + r'|'.join(alloc_fns) + r').*\/([\w\.]+):(\d+)$'
# This tracks which allocation/free functions have been seen in jsutil.cpp.
jsutil_cpp = set([])
for line in lines:
m = re.search(alloc_fns_re, line)
if m is None:
continue
fn = m.group(1)
filename = m.group(2)
linenum = m.group(3)
if filename == 'jsutil.cpp':
jsutil_cpp.add(fn)
else:
# An allocation is present in a non-special file. Fail!
fail("'" + fn + "' present at " + filename + ':' + linenum)
# Check that all functions we expect are used in jsutil.cpp. (This will
# fail if the function-detection code breaks at any point.)
for fn in alloc_fns_unescaped:
if fn not in jsutil_cpp:
fail("'" + fn + "' isn't used as expected in jsutil.cpp")
else:
jsutil_cpp.remove(fn)
# This should never happen, but check just in case.
if jsutil_cpp:
fail('unexpected allocation fns used in jsutil.cpp: ' +
', '.join(jsutil_cpp))
if has_failed:
sys.exit(1)
print('TEST-PASS | check_vanilla_allocations.py | ok')
sys.exit(0)
if __name__ == '__main__':
main()

Просмотреть файл

@ -1,80 +0,0 @@
# /bin/bash
#----------------------------------------------------------------------------
# We must avoid using the vanilla new/new[] operators (and consequently, the
# vanilla delete/delete[] operators) in SpiderMonkey, see bug 624878 for why.
#
# This script:
# - Detects if any of the vanilla new/new[] operators are used in a file.
# Its exit code is 1 if it found some, and 0 if it didn't.
# - Doesn't detect delete/delete[] because it appears they can be present
# somehow due to virtual destructors, but this is ok because vanilla
# delete/delete[] calls don't make sense without corresponding new/new[]
# calls, and any explicit calls will be caught by Valgrind's mismatched
# alloc/free checking.
# - Doesn't detect the 'nothrow' variants, which are ok but probably still
# best avoided.
# - Is designed to only run on Linux (though it may also work on Mac); one
# platform will be enough to catch any violations.
#
# If this script fails:
# - You need to find the uses of vanilla new/delete and replace them with
# {js::OffTheBooks,JSContext,JSRuntime}::{new_,/array_new}.
# - Run this script on each of the .o files, that should narrow it down.
# - After that, one way to find them is to run 'objdump -r -C' on the
# relevant .o files. For example, you might search for 'operator new' and
# find a record like this:
#
# RELOCATION RECORDS FOR [.text._ZN3JSC14ExecutablePool6createEj]:
# OFFSET TYPE VALUE
# 00000009 R_386_PC32 __i686.get_pc_thunk.bx
# 0000000f R_386_GOTPC _GLOBAL_OFFSET_TABLE_
# 0000001b R_386_PLT32 operator new(unsigned int)
# 0000002e R_386_PC32 JSC::ExecutablePool::ExecutablePool(unsigned int)
# 0000004a R_386_PC32 JSC::ExecutablePool::~ExecutablePool()
# 00000052 R_386_PLT32 operator delete(void*)
#
# This says that vanilla 'new' and 'delete' are both used in
# JSC::ExecutablePool::create(unsigned int). This doesn't always work,
# though. (Nb: use 'c++filt' to demangle names like
# _ZN3JSC14ExecutablePool6createEj.)
#
# If that doesn't work, use grep.
#----------------------------------------------------------------------------
if [ -z $1 ] ; then
echo "usage: find_vanilla_new_calls <file>"
exit 1
fi
file=$1
if [ ! -f $file ] ; then
echo "TEST-UNEXPECTED-FAIL | find_vanilla_new_calls | file '$file' not found"
exit 1
fi
tmpfile1=`mktemp`
tmpfile2=`mktemp`
nm -C $file > $tmpfile1
# Need to double-escape '[' and ']' to stop grep from interpreting them
# specially.
grep '^operator new(unsigned int)' $tmpfile1 >> $tmpfile2
grep '^operator new(unsigned long)' $tmpfile1 >> $tmpfile2
grep '^operator new\\[\\](unsigned int)' $tmpfile1 >> $tmpfile2
grep '^operator new\\[\\](unsigned long)' $tmpfile1 >> $tmpfile2
rm -f $tmpfile1
if [ -s $tmpfile2 ] ; then
echo "TEST-UNEXPECTED-FAIL | find_vanilla_new_calls | found calls are listed below"
cat $tmpfile2
echo
rm -f $tmpfile2
exit 1
fi
echo "TEST-PASS | find_vanilla_new_calls | ok"
echo
exit 0

Просмотреть файл

@ -268,14 +268,18 @@ endif
#############################################
# The "find any vanilla new/new[] calls" script is tailored to Linux, so
# only run it there. That should be enough to catch any such calls that
# creep in.
check-vanilla-new:
$(srcdir)/config/find_vanilla_new_calls $(LIBRARY)
# check_vanilla_allocations.py is tailored to Linux, so only run it there.
# That should be enough to catch any problems.
check-vanilla-allocations:
$(PYTHON) $(srcdir)/config/check_vanilla_allocations.py $(REAL_LIBRARY)
# The "aggressive" variant will likely fail on some compiler/platform
# combinations, but is worth running by hand every once in a while.
check-vanilla-allocations-aggressive:
$(PYTHON) $(srcdir)/config/check_vanilla_allocations.py --aggressive $(REAL_LIBRARY)
ifeq ($(OS_ARCH),Linux)
check:: check-vanilla-new
check:: check-vanilla-allocations
endif
# Help ensure that the number of OOM errors in SpiderMonkey doesn't increase.

Просмотреть файл

@ -0,0 +1,159 @@
# vim: set ts=8 sts=4 et sw=4 tw=79:
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
#----------------------------------------------------------------------------
# All heap allocations in SpiderMonkey must go through js_malloc, js_calloc,
# js_realloc, and js_free. This is so that any embedder who uses a custom
# allocator (by defining JS_USE_CUSTOM_ALLOCATOR) will see all heap allocation
# go through that custom allocator.
#
# Therefore, the presence of any calls to "vanilla" allocation/free functions
# (e.g. malloc(), free()) is a bug.
#
# This script checks for the presence of such disallowed vanilla
# allocation/free function in SpiderMonkey when it's built as a library. It
# relies on |nm| from the GNU binutils, and so only works on Linux, but one
# platform is good enough to catch almost all violations.
#
# This checking is only 100% reliable in a JS_USE_CUSTOM_ALLOCATOR build in
# which the default definitions of js_malloc et al (in Utility.h) -- which call
# malloc et al -- are replaced with empty definitions. This is because the
# presence and possible inlining of the default js_malloc et al can cause
# malloc/calloc/realloc/free calls show up in unpredictable places.
#
# Unfortunately, that configuration cannot be tested on Mozilla's standard
# testing infrastructure. Instead, by default this script only tests that none
# of the other vanilla allocation/free functions (operator new, memalign, etc)
# are present. If given the --aggressive flag, it will also check for
# malloc/calloc/realloc/free.
#
# Note: We don't check for |operator delete| and |operator delete[]|. These
# can be present somehow due to virtual destructors, but this is not too
# because vanilla delete/delete[] calls don't make sense without corresponding
# vanilla new/new[] calls, and any explicit calls will be caught by Valgrind's
# mismatched alloc/free checking.
#----------------------------------------------------------------------------
from __future__ import print_function
import argparse
import re
import subprocess
import sys
# The obvious way to implement this script is to search for occurrences of
# malloc et al, succeed if none are found, and fail is some are found.
# However, "none are found" does not necessarily mean "none are present" --
# this script could be buggy. (Or the output format of |nm| might change in
# the future.)
#
# So jsutil.cpp deliberately contains a (never-called) function that contains a
# single use of all the vanilla allocation/free functions. And this script
# fails if it (a) finds uses of those functions in files other than jsutil.cpp,
# *or* (b) fails to find them in jsutil.cpp.
# Tracks overall success of the test.
has_failed = False
def fail(msg):
print('TEST-UNEXPECTED-FAIL | check_vanilla_allocations.py |', msg)
global has_failed
has_failed = True
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--aggressive', action='store_true',
help='also check for malloc, calloc, realloc and free')
parser.add_argument('file', type=str,
help='name of the file to check')
args = parser.parse_args()
# Run |nm|. Options:
# -u: show only undefined symbols
# -C: demangle symbol names
# -l: show a filename and line number for each undefined symbol
cmd = ['nm', '-u', '-C', '-l', args.file]
lines = subprocess.check_output(cmd, universal_newlines=True,
stderr=subprocess.PIPE).split('\n')
# alloc_fns contains all the vanilla allocation/free functions that we look
# for. Regexp chars are escaped appropriately.
alloc_fns = [
# Matches |operator new(unsigned T)|, where |T| is |int| or |long|.
r'operator new\(unsigned',
# Matches |operator new[](unsigned T)|, where |T| is |int| or |long|.
r'operator new\[\]\(unsigned',
r'memalign',
# These two aren't available on all Linux configurations.
#r'posix_memalign',
#r'aligned_alloc',
r'valloc',
r'strdup'
]
if args.aggressive:
alloc_fns += [
r'malloc',
r'calloc',
r'realloc',
r'free'
]
# This is like alloc_fns, but regexp chars are not escaped.
alloc_fns_unescaped = [fn.translate(None, r'\\') for fn in alloc_fns]
# This regexp matches the relevant lines in the output of |nm|, which look
# like the following.
#
# U malloc /path/to/objdir/dist/include/js/Utility.h:142
#
alloc_fns_re = r'U (' + r'|'.join(alloc_fns) + r').*\/([\w\.]+):(\d+)$'
# This tracks which allocation/free functions have been seen in jsutil.cpp.
jsutil_cpp = set([])
for line in lines:
m = re.search(alloc_fns_re, line)
if m is None:
continue
fn = m.group(1)
filename = m.group(2)
linenum = m.group(3)
if filename == 'jsutil.cpp':
jsutil_cpp.add(fn)
else:
# An allocation is present in a non-special file. Fail!
fail("'" + fn + "' present at " + filename + ':' + linenum)
# Check that all functions we expect are used in jsutil.cpp. (This will
# fail if the function-detection code breaks at any point.)
for fn in alloc_fns_unescaped:
if fn not in jsutil_cpp:
fail("'" + fn + "' isn't used as expected in jsutil.cpp")
else:
jsutil_cpp.remove(fn)
# This should never happen, but check just in case.
if jsutil_cpp:
fail('unexpected allocation fns used in jsutil.cpp: ' +
', '.join(jsutil_cpp))
if has_failed:
sys.exit(1)
print('TEST-PASS | check_vanilla_allocations.py | ok')
sys.exit(0)
if __name__ == '__main__':
main()

Просмотреть файл

@ -1,80 +0,0 @@
# /bin/bash
#----------------------------------------------------------------------------
# We must avoid using the vanilla new/new[] operators (and consequently, the
# vanilla delete/delete[] operators) in SpiderMonkey, see bug 624878 for why.
#
# This script:
# - Detects if any of the vanilla new/new[] operators are used in a file.
# Its exit code is 1 if it found some, and 0 if it didn't.
# - Doesn't detect delete/delete[] because it appears they can be present
# somehow due to virtual destructors, but this is ok because vanilla
# delete/delete[] calls don't make sense without corresponding new/new[]
# calls, and any explicit calls will be caught by Valgrind's mismatched
# alloc/free checking.
# - Doesn't detect the 'nothrow' variants, which are ok but probably still
# best avoided.
# - Is designed to only run on Linux (though it may also work on Mac); one
# platform will be enough to catch any violations.
#
# If this script fails:
# - You need to find the uses of vanilla new/delete and replace them with
# {js::OffTheBooks,JSContext,JSRuntime}::{new_,/array_new}.
# - Run this script on each of the .o files, that should narrow it down.
# - After that, one way to find them is to run 'objdump -r -C' on the
# relevant .o files. For example, you might search for 'operator new' and
# find a record like this:
#
# RELOCATION RECORDS FOR [.text._ZN3JSC14ExecutablePool6createEj]:
# OFFSET TYPE VALUE
# 00000009 R_386_PC32 __i686.get_pc_thunk.bx
# 0000000f R_386_GOTPC _GLOBAL_OFFSET_TABLE_
# 0000001b R_386_PLT32 operator new(unsigned int)
# 0000002e R_386_PC32 JSC::ExecutablePool::ExecutablePool(unsigned int)
# 0000004a R_386_PC32 JSC::ExecutablePool::~ExecutablePool()
# 00000052 R_386_PLT32 operator delete(void*)
#
# This says that vanilla 'new' and 'delete' are both used in
# JSC::ExecutablePool::create(unsigned int). This doesn't always work,
# though. (Nb: use 'c++filt' to demangle names like
# _ZN3JSC14ExecutablePool6createEj.)
#
# If that doesn't work, use grep.
#----------------------------------------------------------------------------
if [ -z $1 ] ; then
echo "usage: find_vanilla_new_calls <file>"
exit 1
fi
file=$1
if [ ! -f $file ] ; then
echo "TEST-UNEXPECTED-FAIL | find_vanilla_new_calls | file '$file' not found"
exit 1
fi
tmpfile1=`mktemp`
tmpfile2=`mktemp`
nm -C $file > $tmpfile1
# Need to double-escape '[' and ']' to stop grep from interpreting them
# specially.
grep '^operator new(unsigned int)' $tmpfile1 >> $tmpfile2
grep '^operator new(unsigned long)' $tmpfile1 >> $tmpfile2
grep '^operator new\\[\\](unsigned int)' $tmpfile1 >> $tmpfile2
grep '^operator new\\[\\](unsigned long)' $tmpfile1 >> $tmpfile2
rm -f $tmpfile1
if [ -s $tmpfile2 ] ; then
echo "TEST-UNEXPECTED-FAIL | find_vanilla_new_calls | found calls are listed below"
cat $tmpfile2
echo
rm -f $tmpfile2
exit 1
fi
echo "TEST-PASS | find_vanilla_new_calls | ok"
echo
exit 0

Просмотреть файл

@ -161,6 +161,49 @@ JS_Assert(const char *s, const char *file, int ln)
MOZ_CRASH();
}
#ifdef __linux__
#include <malloc.h>
#include <stdlib.h>
namespace js {
// This function calls all the vanilla heap allocation functions. It is never
// called, and exists purely to help config/check_vanilla_allocations.py. See
// that script for more details.
extern void
AllTheNonBasicVanillaNewAllocations()
{
// posix_memalign and aligned_alloc aren't available on all Linux
// configurations.
//char *q;
//posix_memalign((void**)&q, 16, 16);
intptr_t p =
intptr_t(malloc(16)) +
intptr_t(calloc(1, 16)) +
intptr_t(realloc(nullptr, 16)) +
intptr_t(new char) +
intptr_t(new char) +
intptr_t(new char) +
intptr_t(new char[16]) +
intptr_t(memalign(16, 16)) +
//intptr_t(q) +
//intptr_t(aligned_alloc(16, 16)) +
intptr_t(valloc(4096)) +
intptr_t(strdup("dummy"));
printf("%u\n", uint32_t(p)); // make sure |p| is not optimized away
free((int*)p); // this would crash if ever actually called
MOZ_CRASH();
}
} // namespace js
#endif // __linux__
#ifdef JS_BASIC_STATS
#include <math.h>