completely automatic malloc inclusion in emcc
This commit is contained in:
Alon Zakai
Родитель
ac78c51c1b
Коммит
086526150c
56
emcc
56
emcc
|
|
@ -186,6 +186,8 @@ def unsuffixed(name):
|
|||
def unsuffixed_basename(name):
|
||||
return os.path.basename(unsuffixed(name))
|
||||
|
||||
LLVM_INTERNAL_OPT_LEVEL = 2
|
||||
|
||||
# ---------------- End configs -------------
|
||||
|
||||
if len(sys.argv) == 1 or sys.argv[1] in ['x', 't']:
|
||||
|
|
@ -272,6 +274,7 @@ try:
|
|||
settings_changes = []
|
||||
for i in range(len(newargs)):
|
||||
if newargs[i] == '-s':
|
||||
assert '=' in newargs[i+1], 'Incorrect syntax for -s (use -s OPT=VAL): ' + newargs[i+1]
|
||||
settings_changes.append(newargs[i+1])
|
||||
newargs[i] = newargs[i+1] = ''
|
||||
elif newargs[i].startswith('--typed-arrays'):
|
||||
|
|
@ -324,6 +327,11 @@ try:
|
|||
shared.Settings.DOUBLE_MODE = 0
|
||||
print >> sys.stderr, 'Warning: Applying some potentially unsafe optimizations! (Use -O2 if this fails.)'
|
||||
|
||||
# Apply -s settings in newargs here (after optimization levels, so they can override them)
|
||||
for change in settings_changes:
|
||||
key, value = change.split('=')
|
||||
exec('shared.Settings.' + key + ' = ' + value)
|
||||
|
||||
## Compile source code to bitcode
|
||||
|
||||
if DEBUG: print >> sys.stderr, 'emcc: compiling to bitcode'
|
||||
|
|
@ -343,7 +351,7 @@ try:
|
|||
if llvm_opt_level > 0:
|
||||
if DEBUG: print >> sys.stderr, 'emcc: LLVM opts'
|
||||
for input_file in input_files:
|
||||
shared.Building.llvm_opt(in_temp(unsuffixed_basename(input_file) + '.o'), 2, safe=llvm_opt_level < 2)
|
||||
shared.Building.llvm_opt(in_temp(unsuffixed_basename(input_file) + '.o'), LLVM_INTERNAL_OPT_LEVEL, safe=llvm_opt_level < 2)
|
||||
|
||||
# If we were just asked to generate bitcode, stop there
|
||||
if final_suffix not in ['js', 'html']:
|
||||
|
|
@ -354,7 +362,7 @@ try:
|
|||
if len(input_files) == 1:
|
||||
shutil.move(in_temp(unsuffixed_basename(input_files[0]) + '.o'), specified_target)
|
||||
else:
|
||||
assert not has_dash_c, 'fatal error: cannot specify -o with -c with multiple files'
|
||||
assert not has_dash_c, 'fatal error: cannot specify -o with -c with multiple files' + str(sys.argv)
|
||||
# We have a specified target (-o <target>), which is not JavaScript or HTML, and
|
||||
# we have multiple files: Link them. TODO: Pass complex linker args along
|
||||
shared.Building.link(map(lambda input_file: in_temp(unsuffixed_basename(input_file) + '.o'), input_files), specified_target)
|
||||
|
|
@ -365,18 +373,46 @@ try:
|
|||
|
||||
if DEBUG: print >> sys.stderr, 'emcc: generating JavaScript'
|
||||
|
||||
extra_files_to_link = []
|
||||
|
||||
# Check if we need to include dlmalloc. Note that we assume a single symbol is enough to know if we have/do not have dlmalloc. If you
|
||||
# include just a few symbols but want the rest, this will not work.
|
||||
need_dlmalloc = False
|
||||
has_dlmalloc = False
|
||||
for input_file in input_files:
|
||||
symbols = shared.Building.llvm_nm(in_temp(unsuffixed_basename(input_file) + '.o'))
|
||||
for malloc_def in ['malloc', 'free', 'calloc', 'memalign', 'realloc', 'valloc', 'pvalloc', 'mallinfo', 'mallopt', 'malloc_trim', 'malloc_stats', 'malloc_usable_size', 'malloc_footprint', 'malloc_max_footprint', 'independent_calloc', 'independent_comalloc']:
|
||||
if malloc_def in symbols.undefs:
|
||||
need_dlmalloc = True
|
||||
if malloc_def in symbols.defs:
|
||||
has_dlmalloc = True
|
||||
if need_dlmalloc and not has_dlmalloc:
|
||||
# We need to build and link dlmalloc in
|
||||
if DEBUG: print >> sys.stderr, 'emcc: including dlmalloc'
|
||||
Popen([shared.EMCC, shared.path_from_root('src', 'dlmalloc.c'), '-g', '-o', in_temp('dlmalloc.o')]).communicate()
|
||||
if llvm_opt_level > 0:
|
||||
shared.Building.llvm_opt(in_temp('dlmalloc.o'), LLVM_INTERNAL_OPT_LEVEL, safe=llvm_opt_level < 2)
|
||||
extra_files_to_link.append(in_temp('dlmalloc.o'))
|
||||
|
||||
# dlmalloc needs some sign correction. # If we are in mode 0, switch to 2. We will add our lines
|
||||
try:
|
||||
if shared.Settings.CORRECT_SIGNS == 0: raise Exception('we need to change to 2')
|
||||
except: # we fail if equal to 0 - so we need to switch to 2 - or if CORRECT_SIGNS is not even in Settings
|
||||
shared.Settings.CORRECT_SIGNS = 2
|
||||
if shared.Settings.CORRECT_SIGNS == 2:
|
||||
shared.Settings.CORRECT_SIGNS_LINES = [shared.path_from_root('src', 'dlmalloc.c') + ':' + str(i+4) for i in [4816, 4191, 4246, 4199, 4205, 4235, 4227]]
|
||||
# If we are in mode 1, we are correcting everything anyhow. If we are in mode 3, we will be corrected
|
||||
# so all is well anyhow too.
|
||||
|
||||
# First, combine the bitcode files if there are several
|
||||
if len(input_files) > 1:
|
||||
shared.Building.link(map(lambda input_file: in_temp(unsuffixed_basename(input_file) + '.o'), input_files), in_temp(target_basename + '.bc'))
|
||||
if len(input_files) + len(extra_files_to_link) > 1:
|
||||
shared.Building.link(map(lambda input_file: in_temp(unsuffixed_basename(input_file) + '.o'), input_files) + extra_files_to_link,
|
||||
in_temp(target_basename + '.bc'))
|
||||
# TODO: LLVM link-time opts?
|
||||
else:
|
||||
shutil.move(in_temp(unsuffixed_basename(input_files[0]) + '.o'), in_temp(target_basename + '.bc'))
|
||||
|
||||
# Apply -s settings in newargs here (after -Ox, so they can override it)
|
||||
|
||||
for change in settings_changes:
|
||||
key, value = change.split('=')
|
||||
exec('shared.Settings.' + key + ' = ' + value)
|
||||
|
||||
# Emscripten
|
||||
if opt_level >= 2:
|
||||
print >> sys.stderr, 'Warning: The relooper optimization can be very slow.'
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,18 @@
|
|||
#include<stdio.h>
|
||||
#include<stdlib.h>
|
||||
#include<assert.h>
|
||||
|
||||
int main() {
|
||||
// Check that a real malloc is used by allocating, freeing, then seeing that we did actually free by a new allocation going to the original place
|
||||
int N = 100;
|
||||
void *allocs[N];
|
||||
for (int i = 0; i < N; i++)
|
||||
allocs[i] = malloc((i+1)*1024);
|
||||
for (int i = 0; i < N; i++)
|
||||
free(allocs[i]);
|
||||
void *another = malloc(1024);
|
||||
assert(another == allocs[0]);
|
||||
printf("hello, world!\n");
|
||||
return 1;
|
||||
}
|
||||
|
||||
|
|
@ -10,7 +10,7 @@ extern "C" {
|
|||
|
||||
int main() {
|
||||
char *original = (char*)"h e l l o , w o r l d ! ";
|
||||
char *copy = (char*)malloc(strlen(original));
|
||||
char copy[strlen(original)];
|
||||
for (int i = 0; i < strlen(original); i += 2) {
|
||||
copy[i/2] = original[i];
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,21 @@
|
|||
#include<stdio.h>
|
||||
#include<string.h>
|
||||
#include<stdlib.h>
|
||||
|
||||
extern "C" {
|
||||
void dump(char *s) {
|
||||
printf("%s\n", s);
|
||||
}
|
||||
}
|
||||
|
||||
int main() {
|
||||
char *original = (char*)"h e l l o , w o r l d ! ";
|
||||
char *copy = (char*)malloc(strlen(original));
|
||||
for (int i = 0; i < strlen(original); i += 2) {
|
||||
copy[i/2] = original[i];
|
||||
}
|
||||
copy[strlen(copy)+1] = (int)&original; // force original to be on the stack
|
||||
dump(copy);
|
||||
return 1;
|
||||
}
|
||||
|
||||
|
|
@ -3560,6 +3560,17 @@ at function.:blag
|
|||
self.do_run(src, '*1,0*', ['200', '1'], extra_emscripten_args=['-m'])
|
||||
self.do_run(src, '*400,0*', ['400', '400'], extra_emscripten_args=['-m'], no_build=True)
|
||||
|
||||
if self.use_defaults: # TODO: do this in other passes too, passing their opts into emcc
|
||||
# emcc should build in dlmalloc automatically, and do all the sign correction etc. for it
|
||||
|
||||
try_delete(os.path.join(self.get_dir(), 'src.cpp.o.js'))
|
||||
# XXX find out why we fail without TOTAL_MEMORY here. that should not happen!
|
||||
output = Popen([EMCC, '-g', '-s', 'TOTAL_MEMORY=104857600', path_from_root('tests', 'dlmalloc_test.c'),
|
||||
'-o', os.path.join(self.get_dir(), 'src.cpp.o.js')], stdout=PIPE, stderr=PIPE).communicate()
|
||||
|
||||
self.do_run('x', '*1,0*', ['200', '1'], no_build=True)
|
||||
self.do_run('x', '*400,0*', ['400', '400'], no_build=True)
|
||||
|
||||
def test_libcxx(self):
|
||||
self.do_run(path_from_root('tests', 'libcxx'),
|
||||
'june -> 30\nPrevious (in alphabetical order) is july\nNext (in alphabetical order) is march',
|
||||
|
|
@ -4928,26 +4939,36 @@ Options that are modified or new in %s include:
|
|||
assert os.path.exists(target), 'Expected %s to exist since args are %s : %s' % (target, str(args), '\n'.join(output))
|
||||
self.assertContained('hello, world!', self.run_llvm_interpreter([target]))
|
||||
|
||||
# dlmalloc. dlmalloc is special in that it is the only part of libc that is (1) hard to write well, and
|
||||
# very speed-sensitive. So we do not implement it in JS in library.js, instead we compile it from source
|
||||
for source, has_malloc in [('hello_world' + suffix, False), ('hello_malloc.cpp', True)]:
|
||||
clear()
|
||||
output = Popen([compiler, path_from_root('tests', source)], stdout=PIPE, stderr=PIPE).communicate()
|
||||
assert os.path.exists('a.out.js'), '\n'.join(output)
|
||||
self.assertContained('hello, world!', run_js('a.out.js'))
|
||||
generated = open('a.out.js').read()
|
||||
assert ('function _malloc(bytes) {' in generated) == (not has_malloc), 'If malloc is needed, it should be there, if not not'
|
||||
|
||||
# Optimization: emcc src.cpp -o something.js [-Ox]. -O0 is the same as not specifying any optimization setting
|
||||
for params, opt_level, bc_params, closure in [ # bc params are used after compiling to bitcode
|
||||
(['-o', 'something.js'], 0, None, 0),
|
||||
(['-o', 'something.js', '-O0'], 0, None, 0),
|
||||
(['-o', 'something.js', '-O1'], 1, None, 0),
|
||||
(['-o', 'something.js', '-O1', '--closure', '1'], 1, None, 1),
|
||||
(['-o', 'something.js', '-O2'], 2, None, 1),
|
||||
(['-o', 'something.js', '-O2', '--closure', '0'], 2, None, 0),
|
||||
(['-o', 'something.js', '-O3'], 3, None, 1),
|
||||
(['-o', 'something.js', '-O3', '--closure', '0'], 3, None, 0),
|
||||
for params, opt_level, bc_params, closure, has_malloc in [ # bc params are used after compiling to bitcode
|
||||
(['-o', 'something.js'], 0, None, 0, 1),
|
||||
(['-o', 'something.js', '-O0'], 0, None, 0, 0),
|
||||
(['-o', 'something.js', '-O1'], 1, None, 0, 0),
|
||||
(['-o', 'something.js', '-O1', '--closure', '1'], 1, None, 1, 0),
|
||||
(['-o', 'something.js', '-O2'], 2, None, 1, 1),
|
||||
(['-o', 'something.js', '-O2', '--closure', '0'], 2, None, 0, 0),
|
||||
(['-o', 'something.js', '-O3'], 3, None, 1, 1),
|
||||
(['-o', 'something.js', '-O3', '--closure', '0'], 3, None, 0, 0),
|
||||
# and, test compiling to bitcode first
|
||||
(['-o', 'something.bc'], 0, [], 0),
|
||||
(['-o', 'something.bc'], 0, ['-O0'], 0),
|
||||
(['-o', 'something.bc'], 1, ['-O1'], 0),
|
||||
(['-o', 'something.bc'], 2, ['-O2'], 1),
|
||||
(['-o', 'something.bc'], 3, ['-O3'], 1),
|
||||
(['-o', 'something.bc'], 0, [], 0, 0),
|
||||
(['-o', 'something.bc'], 0, ['-O0'], 0, 0),
|
||||
(['-o', 'something.bc'], 1, ['-O1'], 0, 0),
|
||||
(['-o', 'something.bc'], 2, ['-O2'], 1, 0),
|
||||
(['-o', 'something.bc'], 3, ['-O3'], 1, 0),
|
||||
]:
|
||||
#print params, opt_level, bc_params, closure
|
||||
clear()
|
||||
output = Popen([compiler, path_from_root('tests', 'hello_world_loop.cpp')] + params,
|
||||
output = Popen([compiler, path_from_root('tests', 'hello_world_loop' + ('_malloc' if has_malloc else '') + '.cpp')] + params,
|
||||
stdout=PIPE, stderr=PIPE).communicate()
|
||||
assert len(output[0]) == 0, output[0]
|
||||
if bc_params is not None:
|
||||
|
|
@ -4975,6 +4996,7 @@ Options that are modified or new in %s include:
|
|||
assert 'var $i;' in generated, 'micro opts should always be on'
|
||||
if opt_level >= 1: assert 'HEAP8[HEAP32[' in generated, 'eliminator should create compound expressions, and fewer one-time vars'
|
||||
assert ('_puts(' in generated) == (opt_level >= 1), 'with opt >= 1, llvm opts are run and they should optimize printf to puts'
|
||||
assert ('function _malloc(bytes) {' in generated) == (not has_malloc), 'If malloc is needed, it should be there, if not not'
|
||||
|
||||
# emcc -s RELOOP=1 src.cpp ==> should pass -s to emscripten.py. --typed-arrays is a convenient alias for -s USE_TYPED_ARRAYS
|
||||
for params, test, text in [
|
||||
|
|
@ -5036,7 +5058,6 @@ Options that are modified or new in %s include:
|
|||
self.assertContained('side got: hello from main, over', self.run_llvm_interpreter(['combined.bc']))
|
||||
|
||||
# TODO: compile .ll inputs to emcc into .bc
|
||||
# TODO: dlmalloc in emcc (just pass the arg to emscripten.py)
|
||||
# TODO: test normal project linking, static and dynamic: get_library should not need to be told what to link!
|
||||
# TODO: when ready, switch tools/shared building to use emcc over emmaken
|
||||
# TODO: when this is done, more test runner to test these (i.e., test all -Ox thoroughly)
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import shutil, time, os, json, tempfile
|
||||
import shutil, time, os, sys, json, tempfile
|
||||
from subprocess import Popen, PIPE, STDOUT
|
||||
|
||||
__rootpath__ = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
|
||||
|
|
@ -18,6 +18,7 @@ LLVM_LD=os.path.join(LLVM_ROOT, 'llvm-ld')
|
|||
LLVM_OPT=os.path.expanduser(os.path.join(LLVM_ROOT, 'opt'))
|
||||
LLVM_AS=os.path.expanduser(os.path.join(LLVM_ROOT, 'llvm-as'))
|
||||
LLVM_DIS=os.path.expanduser(os.path.join(LLVM_ROOT, 'llvm-dis'))
|
||||
LLVM_NM=os.path.expanduser(os.path.join(LLVM_ROOT, 'llvm-nm'))
|
||||
LLVM_DIS_OPTS = ['-show-annotations'] # For LLVM 2.8+. For 2.7, you may need to do just []
|
||||
LLVM_INTERPRETER=os.path.expanduser(os.path.join(LLVM_ROOT, 'lli'))
|
||||
LLVM_COMPILER=os.path.expanduser(os.path.join(LLVM_ROOT, 'llc'))
|
||||
|
|
@ -342,6 +343,22 @@ class Building:
|
|||
output = Popen([LLVM_AS, filename + '.o.ll', '-o=' + filename + '.o'], stdout=PIPE).communicate()[0]
|
||||
assert os.path.exists(filename + '.o'), 'Could not create bc file: ' + output
|
||||
|
||||
@staticmethod
|
||||
def llvm_nm(filename):
|
||||
# LLVM binary ==> list of symbols
|
||||
output = Popen([LLVM_NM, filename], stdout=PIPE).communicate()[0]
|
||||
class ret:
|
||||
defs = []
|
||||
undefs = []
|
||||
for line in output.split('\n'):
|
||||
if len(line) == 0: continue
|
||||
status, symbol = filter(lambda seg: len(seg) > 0, line.split(' '))
|
||||
if status == 'U':
|
||||
ret.undefs.append(symbol)
|
||||
else:
|
||||
ret.defs.append(symbol)
|
||||
return ret
|
||||
|
||||
@staticmethod # TODO: make this use emcc instead of emmaken
|
||||
def emmaken(filename, args=[], stdout=None, stderr=None, env=None):
|
||||
try:
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче