From 086526150c8dc53800a63af9571c62410a161cc6 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 16 Dec 2011 12:36:12 -0800 Subject: [PATCH] completely automatic malloc inclusion in emcc --- emcc | 56 +++++++++++++++++++++++++------ tests/hello_malloc.cpp | 18 ++++++++++ tests/hello_world_loop.cpp | 2 +- tests/hello_world_loop_malloc.cpp | 21 ++++++++++++ tests/runner.py | 53 ++++++++++++++++++++--------- tools/shared.py | 19 ++++++++++- 6 files changed, 141 insertions(+), 28 deletions(-) create mode 100644 tests/hello_malloc.cpp create mode 100644 tests/hello_world_loop_malloc.cpp diff --git a/emcc b/emcc index a1db0e361..88730539a 100755 --- a/emcc +++ b/emcc @@ -186,6 +186,8 @@ def unsuffixed(name): def unsuffixed_basename(name): return os.path.basename(unsuffixed(name)) +LLVM_INTERNAL_OPT_LEVEL = 2 + # ---------------- End configs ------------- if len(sys.argv) == 1 or sys.argv[1] in ['x', 't']: @@ -272,6 +274,7 @@ try: settings_changes = [] for i in range(len(newargs)): if newargs[i] == '-s': + assert '=' in newargs[i+1], 'Incorrect syntax for -s (use -s OPT=VAL): ' + newargs[i+1] settings_changes.append(newargs[i+1]) newargs[i] = newargs[i+1] = '' elif newargs[i].startswith('--typed-arrays'): @@ -324,6 +327,11 @@ try: shared.Settings.DOUBLE_MODE = 0 print >> sys.stderr, 'Warning: Applying some potentially unsafe optimizations! (Use -O2 if this fails.)' + # Apply -s settings in newargs here (after optimization levels, so they can override them) + for change in settings_changes: + key, value = change.split('=') + exec('shared.Settings.' + key + ' = ' + value) + ## Compile source code to bitcode if DEBUG: print >> sys.stderr, 'emcc: compiling to bitcode' @@ -343,7 +351,7 @@ try: if llvm_opt_level > 0: if DEBUG: print >> sys.stderr, 'emcc: LLVM opts' for input_file in input_files: - shared.Building.llvm_opt(in_temp(unsuffixed_basename(input_file) + '.o'), 2, safe=llvm_opt_level < 2) + shared.Building.llvm_opt(in_temp(unsuffixed_basename(input_file) + '.o'), LLVM_INTERNAL_OPT_LEVEL, safe=llvm_opt_level < 2) # If we were just asked to generate bitcode, stop there if final_suffix not in ['js', 'html']: @@ -354,7 +362,7 @@ try: if len(input_files) == 1: shutil.move(in_temp(unsuffixed_basename(input_files[0]) + '.o'), specified_target) else: - assert not has_dash_c, 'fatal error: cannot specify -o with -c with multiple files' + assert not has_dash_c, 'fatal error: cannot specify -o with -c with multiple files' + str(sys.argv) # We have a specified target (-o ), which is not JavaScript or HTML, and # we have multiple files: Link them. TODO: Pass complex linker args along shared.Building.link(map(lambda input_file: in_temp(unsuffixed_basename(input_file) + '.o'), input_files), specified_target) @@ -365,18 +373,46 @@ try: if DEBUG: print >> sys.stderr, 'emcc: generating JavaScript' + extra_files_to_link = [] + + # Check if we need to include dlmalloc. Note that we assume a single symbol is enough to know if we have/do not have dlmalloc. If you + # include just a few symbols but want the rest, this will not work. + need_dlmalloc = False + has_dlmalloc = False + for input_file in input_files: + symbols = shared.Building.llvm_nm(in_temp(unsuffixed_basename(input_file) + '.o')) + for malloc_def in ['malloc', 'free', 'calloc', 'memalign', 'realloc', 'valloc', 'pvalloc', 'mallinfo', 'mallopt', 'malloc_trim', 'malloc_stats', 'malloc_usable_size', 'malloc_footprint', 'malloc_max_footprint', 'independent_calloc', 'independent_comalloc']: + if malloc_def in symbols.undefs: + need_dlmalloc = True + if malloc_def in symbols.defs: + has_dlmalloc = True + if need_dlmalloc and not has_dlmalloc: + # We need to build and link dlmalloc in + if DEBUG: print >> sys.stderr, 'emcc: including dlmalloc' + Popen([shared.EMCC, shared.path_from_root('src', 'dlmalloc.c'), '-g', '-o', in_temp('dlmalloc.o')]).communicate() + if llvm_opt_level > 0: + shared.Building.llvm_opt(in_temp('dlmalloc.o'), LLVM_INTERNAL_OPT_LEVEL, safe=llvm_opt_level < 2) + extra_files_to_link.append(in_temp('dlmalloc.o')) + + # dlmalloc needs some sign correction. # If we are in mode 0, switch to 2. We will add our lines + try: + if shared.Settings.CORRECT_SIGNS == 0: raise Exception('we need to change to 2') + except: # we fail if equal to 0 - so we need to switch to 2 - or if CORRECT_SIGNS is not even in Settings + shared.Settings.CORRECT_SIGNS = 2 + if shared.Settings.CORRECT_SIGNS == 2: + shared.Settings.CORRECT_SIGNS_LINES = [shared.path_from_root('src', 'dlmalloc.c') + ':' + str(i+4) for i in [4816, 4191, 4246, 4199, 4205, 4235, 4227]] + # If we are in mode 1, we are correcting everything anyhow. If we are in mode 3, we will be corrected + # so all is well anyhow too. + # First, combine the bitcode files if there are several - if len(input_files) > 1: - shared.Building.link(map(lambda input_file: in_temp(unsuffixed_basename(input_file) + '.o'), input_files), in_temp(target_basename + '.bc')) + if len(input_files) + len(extra_files_to_link) > 1: + shared.Building.link(map(lambda input_file: in_temp(unsuffixed_basename(input_file) + '.o'), input_files) + extra_files_to_link, + in_temp(target_basename + '.bc')) + # TODO: LLVM link-time opts? else: shutil.move(in_temp(unsuffixed_basename(input_files[0]) + '.o'), in_temp(target_basename + '.bc')) - # Apply -s settings in newargs here (after -Ox, so they can override it) - - for change in settings_changes: - key, value = change.split('=') - exec('shared.Settings.' + key + ' = ' + value) - + # Emscripten if opt_level >= 2: print >> sys.stderr, 'Warning: The relooper optimization can be very slow.' diff --git a/tests/hello_malloc.cpp b/tests/hello_malloc.cpp new file mode 100644 index 000000000..d3774f702 --- /dev/null +++ b/tests/hello_malloc.cpp @@ -0,0 +1,18 @@ +#include +#include +#include + +int main() { + // Check that a real malloc is used by allocating, freeing, then seeing that we did actually free by a new allocation going to the original place + int N = 100; + void *allocs[N]; + for (int i = 0; i < N; i++) + allocs[i] = malloc((i+1)*1024); + for (int i = 0; i < N; i++) + free(allocs[i]); + void *another = malloc(1024); + assert(another == allocs[0]); + printf("hello, world!\n"); + return 1; +} + diff --git a/tests/hello_world_loop.cpp b/tests/hello_world_loop.cpp index b93618346..46ec9f23f 100644 --- a/tests/hello_world_loop.cpp +++ b/tests/hello_world_loop.cpp @@ -10,7 +10,7 @@ extern "C" { int main() { char *original = (char*)"h e l l o , w o r l d ! "; - char *copy = (char*)malloc(strlen(original)); + char copy[strlen(original)]; for (int i = 0; i < strlen(original); i += 2) { copy[i/2] = original[i]; } diff --git a/tests/hello_world_loop_malloc.cpp b/tests/hello_world_loop_malloc.cpp new file mode 100644 index 000000000..b93618346 --- /dev/null +++ b/tests/hello_world_loop_malloc.cpp @@ -0,0 +1,21 @@ +#include +#include +#include + +extern "C" { + void dump(char *s) { + printf("%s\n", s); + } +} + +int main() { + char *original = (char*)"h e l l o , w o r l d ! "; + char *copy = (char*)malloc(strlen(original)); + for (int i = 0; i < strlen(original); i += 2) { + copy[i/2] = original[i]; + } + copy[strlen(copy)+1] = (int)&original; // force original to be on the stack + dump(copy); + return 1; +} + diff --git a/tests/runner.py b/tests/runner.py index d5489dd62..9224cb4b0 100644 --- a/tests/runner.py +++ b/tests/runner.py @@ -3560,6 +3560,17 @@ at function.:blag self.do_run(src, '*1,0*', ['200', '1'], extra_emscripten_args=['-m']) self.do_run(src, '*400,0*', ['400', '400'], extra_emscripten_args=['-m'], no_build=True) + if self.use_defaults: # TODO: do this in other passes too, passing their opts into emcc + # emcc should build in dlmalloc automatically, and do all the sign correction etc. for it + + try_delete(os.path.join(self.get_dir(), 'src.cpp.o.js')) + # XXX find out why we fail without TOTAL_MEMORY here. that should not happen! + output = Popen([EMCC, '-g', '-s', 'TOTAL_MEMORY=104857600', path_from_root('tests', 'dlmalloc_test.c'), + '-o', os.path.join(self.get_dir(), 'src.cpp.o.js')], stdout=PIPE, stderr=PIPE).communicate() + + self.do_run('x', '*1,0*', ['200', '1'], no_build=True) + self.do_run('x', '*400,0*', ['400', '400'], no_build=True) + def test_libcxx(self): self.do_run(path_from_root('tests', 'libcxx'), 'june -> 30\nPrevious (in alphabetical order) is july\nNext (in alphabetical order) is march', @@ -4928,26 +4939,36 @@ Options that are modified or new in %s include: assert os.path.exists(target), 'Expected %s to exist since args are %s : %s' % (target, str(args), '\n'.join(output)) self.assertContained('hello, world!', self.run_llvm_interpreter([target])) + # dlmalloc. dlmalloc is special in that it is the only part of libc that is (1) hard to write well, and + # very speed-sensitive. So we do not implement it in JS in library.js, instead we compile it from source + for source, has_malloc in [('hello_world' + suffix, False), ('hello_malloc.cpp', True)]: + clear() + output = Popen([compiler, path_from_root('tests', source)], stdout=PIPE, stderr=PIPE).communicate() + assert os.path.exists('a.out.js'), '\n'.join(output) + self.assertContained('hello, world!', run_js('a.out.js')) + generated = open('a.out.js').read() + assert ('function _malloc(bytes) {' in generated) == (not has_malloc), 'If malloc is needed, it should be there, if not not' + # Optimization: emcc src.cpp -o something.js [-Ox]. -O0 is the same as not specifying any optimization setting - for params, opt_level, bc_params, closure in [ # bc params are used after compiling to bitcode - (['-o', 'something.js'], 0, None, 0), - (['-o', 'something.js', '-O0'], 0, None, 0), - (['-o', 'something.js', '-O1'], 1, None, 0), - (['-o', 'something.js', '-O1', '--closure', '1'], 1, None, 1), - (['-o', 'something.js', '-O2'], 2, None, 1), - (['-o', 'something.js', '-O2', '--closure', '0'], 2, None, 0), - (['-o', 'something.js', '-O3'], 3, None, 1), - (['-o', 'something.js', '-O3', '--closure', '0'], 3, None, 0), + for params, opt_level, bc_params, closure, has_malloc in [ # bc params are used after compiling to bitcode + (['-o', 'something.js'], 0, None, 0, 1), + (['-o', 'something.js', '-O0'], 0, None, 0, 0), + (['-o', 'something.js', '-O1'], 1, None, 0, 0), + (['-o', 'something.js', '-O1', '--closure', '1'], 1, None, 1, 0), + (['-o', 'something.js', '-O2'], 2, None, 1, 1), + (['-o', 'something.js', '-O2', '--closure', '0'], 2, None, 0, 0), + (['-o', 'something.js', '-O3'], 3, None, 1, 1), + (['-o', 'something.js', '-O3', '--closure', '0'], 3, None, 0, 0), # and, test compiling to bitcode first - (['-o', 'something.bc'], 0, [], 0), - (['-o', 'something.bc'], 0, ['-O0'], 0), - (['-o', 'something.bc'], 1, ['-O1'], 0), - (['-o', 'something.bc'], 2, ['-O2'], 1), - (['-o', 'something.bc'], 3, ['-O3'], 1), + (['-o', 'something.bc'], 0, [], 0, 0), + (['-o', 'something.bc'], 0, ['-O0'], 0, 0), + (['-o', 'something.bc'], 1, ['-O1'], 0, 0), + (['-o', 'something.bc'], 2, ['-O2'], 1, 0), + (['-o', 'something.bc'], 3, ['-O3'], 1, 0), ]: #print params, opt_level, bc_params, closure clear() - output = Popen([compiler, path_from_root('tests', 'hello_world_loop.cpp')] + params, + output = Popen([compiler, path_from_root('tests', 'hello_world_loop' + ('_malloc' if has_malloc else '') + '.cpp')] + params, stdout=PIPE, stderr=PIPE).communicate() assert len(output[0]) == 0, output[0] if bc_params is not None: @@ -4975,6 +4996,7 @@ Options that are modified or new in %s include: assert 'var $i;' in generated, 'micro opts should always be on' if opt_level >= 1: assert 'HEAP8[HEAP32[' in generated, 'eliminator should create compound expressions, and fewer one-time vars' assert ('_puts(' in generated) == (opt_level >= 1), 'with opt >= 1, llvm opts are run and they should optimize printf to puts' + assert ('function _malloc(bytes) {' in generated) == (not has_malloc), 'If malloc is needed, it should be there, if not not' # emcc -s RELOOP=1 src.cpp ==> should pass -s to emscripten.py. --typed-arrays is a convenient alias for -s USE_TYPED_ARRAYS for params, test, text in [ @@ -5036,7 +5058,6 @@ Options that are modified or new in %s include: self.assertContained('side got: hello from main, over', self.run_llvm_interpreter(['combined.bc'])) # TODO: compile .ll inputs to emcc into .bc - # TODO: dlmalloc in emcc (just pass the arg to emscripten.py) # TODO: test normal project linking, static and dynamic: get_library should not need to be told what to link! # TODO: when ready, switch tools/shared building to use emcc over emmaken # TODO: when this is done, more test runner to test these (i.e., test all -Ox thoroughly) diff --git a/tools/shared.py b/tools/shared.py index eef1dd683..7bd317295 100644 --- a/tools/shared.py +++ b/tools/shared.py @@ -1,4 +1,4 @@ -import shutil, time, os, json, tempfile +import shutil, time, os, sys, json, tempfile from subprocess import Popen, PIPE, STDOUT __rootpath__ = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) @@ -18,6 +18,7 @@ LLVM_LD=os.path.join(LLVM_ROOT, 'llvm-ld') LLVM_OPT=os.path.expanduser(os.path.join(LLVM_ROOT, 'opt')) LLVM_AS=os.path.expanduser(os.path.join(LLVM_ROOT, 'llvm-as')) LLVM_DIS=os.path.expanduser(os.path.join(LLVM_ROOT, 'llvm-dis')) +LLVM_NM=os.path.expanduser(os.path.join(LLVM_ROOT, 'llvm-nm')) LLVM_DIS_OPTS = ['-show-annotations'] # For LLVM 2.8+. For 2.7, you may need to do just [] LLVM_INTERPRETER=os.path.expanduser(os.path.join(LLVM_ROOT, 'lli')) LLVM_COMPILER=os.path.expanduser(os.path.join(LLVM_ROOT, 'llc')) @@ -342,6 +343,22 @@ class Building: output = Popen([LLVM_AS, filename + '.o.ll', '-o=' + filename + '.o'], stdout=PIPE).communicate()[0] assert os.path.exists(filename + '.o'), 'Could not create bc file: ' + output + @staticmethod + def llvm_nm(filename): + # LLVM binary ==> list of symbols + output = Popen([LLVM_NM, filename], stdout=PIPE).communicate()[0] + class ret: + defs = [] + undefs = [] + for line in output.split('\n'): + if len(line) == 0: continue + status, symbol = filter(lambda seg: len(seg) > 0, line.split(' ')) + if status == 'U': + ret.undefs.append(symbol) + else: + ret.defs.append(symbol) + return ret + @staticmethod # TODO: make this use emcc instead of emmaken def emmaken(filename, args=[], stdout=None, stderr=None, env=None): try: