more carefully split out functions in js optimizer
This commit is contained in:
Alon Zakai
Родитель
bb463d84a9
Коммит
bafab997bd
|
|
@ -10473,41 +10473,43 @@ fi
|
|||
srcs = {}
|
||||
used_jcache = False
|
||||
|
||||
for args, input_file, expect_save, expect_load in [
|
||||
([], 'hello_world_loop.cpp', False, False),
|
||||
(['--jcache'], 'hello_world_loop.cpp', True, False),
|
||||
(['--jcache'], 'hello_world_loop.cpp', False, True),
|
||||
([], 'hello_world_loop.cpp', False, False),
|
||||
for args, input_file, expect_save, expect_load, expected in [
|
||||
([], 'hello_world_loop.cpp', False, False, []),
|
||||
(['--jcache'], 'hello_world_loop.cpp', True, False, []),
|
||||
(['--jcache'], 'hello_world_loop.cpp', False, True, []),
|
||||
([], 'hello_world_loop.cpp', False, False, []),
|
||||
# new
|
||||
([], 'hello_world.cpp', False, False),
|
||||
(['--jcache'], 'hello_world.cpp', True, False),
|
||||
(['--jcache'], 'hello_world.cpp', False, True),
|
||||
([], 'hello_world.cpp', False, False),
|
||||
([], 'hello_world.cpp', False, False, []),
|
||||
(['--jcache'], 'hello_world.cpp', True, False, []),
|
||||
(['--jcache'], 'hello_world.cpp', False, True, []),
|
||||
([], 'hello_world.cpp', False, False, []),
|
||||
# go back to old file, experience caching
|
||||
(['--jcache'], 'hello_world_loop.cpp', False, True),
|
||||
(['--jcache'], 'hello_world_loop.cpp', False, True, []),
|
||||
# new, large file
|
||||
([], 'hello_malloc.cpp', False, False),
|
||||
(['--jcache'], 'hello_malloc.cpp', True, False),
|
||||
(['--jcache'], 'hello_malloc.cpp', False, True),
|
||||
([], 'hello_malloc.cpp', False, False),
|
||||
([], 'hello_malloc.cpp', False, False, []),
|
||||
(['--jcache'], 'hello_malloc.cpp', True, False, []),
|
||||
(['--jcache'], 'hello_malloc.cpp', False, True, []),
|
||||
([], 'hello_malloc.cpp', False, False, []),
|
||||
]:
|
||||
print >> sys.stderr, args, input_file, expect_save, expect_load
|
||||
print >> sys.stderr, args, input_file, expect_save, expect_load, expected
|
||||
self.clear()
|
||||
out, err = Popen(['python', EMCC, '-O2', '--closure', '0', path_from_root('tests', input_file)] + args, stdout=PIPE, stderr=PIPE).communicate()
|
||||
self.assertContained('hello, world!', run_js('a.out.js'))
|
||||
assert (PRE_SAVE_MSG in err) == expect_save, err
|
||||
assert (PRE_LOAD_MSG in err) == expect_load, err
|
||||
assert (FUNC_CHUNKS_SAVE_MSG in err) == expect_save, err
|
||||
assert (FUNC_CHUNKS_LOAD_MSG in err) == expect_load, err
|
||||
assert (JSFUNC_CHUNKS_SAVE_MSG in err) == expect_save, err
|
||||
assert (JSFUNC_CHUNKS_LOAD_MSG in err) == expect_load, err
|
||||
errtail = err.split('emcc invocation')[-1]
|
||||
self.assertContained('hello, world!', run_js('a.out.js'), errtail)
|
||||
assert (PRE_SAVE_MSG in err) == expect_save, errtail
|
||||
assert (PRE_LOAD_MSG in err) == expect_load, errtail
|
||||
assert (FUNC_CHUNKS_SAVE_MSG in err) == expect_save, errtail
|
||||
assert (FUNC_CHUNKS_LOAD_MSG in err) == expect_load, errtail
|
||||
assert (JSFUNC_CHUNKS_SAVE_MSG in err) == expect_save, errtail
|
||||
assert (JSFUNC_CHUNKS_LOAD_MSG in err) == expect_load, errtail
|
||||
for expect in expected: assert expect in err, expect + ' ? ' + errtail
|
||||
curr = open('a.out.js').read()
|
||||
if input_file not in srcs:
|
||||
srcs[input_file] = curr
|
||||
else:
|
||||
open('/home/alon/Dev/emscripten/a', 'w').write(srcs[input_file])
|
||||
open('/home/alon/Dev/emscripten/b', 'w').write(curr)
|
||||
assert abs(len(curr)/float(len(srcs[input_file]))-1)<0.01, 'contents may shift in order, but must remain the same size %d vs %d' % (len(curr), len(srcs[input_file])) + '\n' + err
|
||||
assert abs(len(curr)/float(len(srcs[input_file]))-1)<0.01, 'contents may shift in order, but must remain the same size %d vs %d' % (len(curr), len(srcs[input_file])) + '\n' + errtail
|
||||
used_jcache = used_jcache or ('--jcache' in args)
|
||||
assert used_jcache == os.path.exists(JCache.get_cachename('emscript_files'))
|
||||
|
||||
|
|
|
|||
|
|
@ -1916,6 +1916,5 @@ do {
|
|||
js = js.replace(/\n *\n/g, '\n');
|
||||
} while (js != old);
|
||||
print(js);
|
||||
if (metadata && printMetadata) print(metadata);
|
||||
print('\n');
|
||||
|
||||
|
|
|
|||
|
|
@ -42,29 +42,60 @@ def run(filename, passes, js_engine, jcache):
|
|||
suffix = ''
|
||||
if suffix_start >= 0:
|
||||
suffix = js[suffix_start:js.find('\n', suffix_start)] + '\n'
|
||||
# if there is metadata, we will run only on the generated functions. If there isn't, we will run on everything.
|
||||
generated = set(eval(suffix[len(suffix_marker)+1:]))
|
||||
|
||||
if not suffix and jcache:
|
||||
# JCache cannot be used without metadata, since it might reorder stuff, and that's dangerous since only generated can be reordered
|
||||
# This means jcache does not work after closure compiler runs, for example. But you won't get much benefit from jcache with closure
|
||||
# anyhow (since closure is likely the longest part of the build).
|
||||
if DEBUG: print >>sys.stderr, 'js optimizer: no metadata, so disabling jcache'
|
||||
jcache = False
|
||||
|
||||
# If we process only generated code, find that and save the rest on the side
|
||||
func_sig = re.compile('function (_\w+)\(')
|
||||
if suffix:
|
||||
pos = 0
|
||||
gen_start = 0
|
||||
gen_end = 0
|
||||
while 1:
|
||||
m = func_sig.search(js, pos)
|
||||
if not m: break
|
||||
pos = m.end()
|
||||
ident = m.group(1)
|
||||
if ident in generated:
|
||||
if not gen_start:
|
||||
gen_start = m.start()
|
||||
assert gen_start
|
||||
gen_end = js.find('\n}\n', m.end()) + 3
|
||||
assert gen_end > gen_start
|
||||
pre = js[:gen_start]
|
||||
post = js[gen_end:]
|
||||
js = js[gen_start:gen_end]
|
||||
else:
|
||||
pre = ''
|
||||
post = ''
|
||||
|
||||
# Pick where to split into chunks, so that (1) they do not oom in node/uglify, and (2) we can run them in parallel
|
||||
# If we have metadata, we split only the generated code, and save the pre and post on the side (and do not optimize them)
|
||||
parts = map(lambda part: part, js.split('\n}\n'))
|
||||
funcs = []
|
||||
buffered = []
|
||||
for i in range(len(parts)):
|
||||
func = parts[i]
|
||||
if i < len(parts)-1: func += '\n}\n' # last part needs no } and already has suffix
|
||||
m = re.search('function (_\w+)\(', func)
|
||||
if i < len(parts)-1: func += '\n}\n' # last part needs no }
|
||||
m = func_sig.search(func)
|
||||
if m:
|
||||
ident = m.group(1)
|
||||
if buffered:
|
||||
func = ''.join(buffered) + func
|
||||
buffered = []
|
||||
funcs.append((ident, func))
|
||||
else:
|
||||
buffered.append(func)
|
||||
if buffered:
|
||||
if len(funcs) > 0:
|
||||
funcs[-1] = (funcs[-1][0], funcs[-1][1] + ''.join(buffered))
|
||||
else:
|
||||
funcs.append(('anonymous', ''.join(buffered)))
|
||||
if suffix: continue # ignore whitespace
|
||||
ident = 'anon_%d' % i
|
||||
funcs.append((ident, func))
|
||||
parts = None
|
||||
total_size = len(js)
|
||||
js = None
|
||||
if not suffix: # if there is no metadata, we never see generated, and all funcs are in pre
|
||||
funcs = [('anonymous', code) for code in pre]
|
||||
pre = []
|
||||
|
||||
chunks = shared.JCache.chunkify(funcs, BEST_JS_PROCESS_SIZE, 'jsopt' if jcache else None)
|
||||
|
||||
|
|
@ -85,18 +116,15 @@ def run(filename, passes, js_engine, jcache):
|
|||
else:
|
||||
cached_outputs = []
|
||||
|
||||
if len(chunks) > 1:
|
||||
if len(chunks) > 0:
|
||||
def write_chunk(chunk, i):
|
||||
temp_file = temp_files.get('.jsfunc_%d.ll' % i).name
|
||||
f = open(temp_file, 'w')
|
||||
f.write(chunk)
|
||||
if i < len(chunks)-1:
|
||||
f.write(suffix) # last already has the suffix
|
||||
f.write(suffix)
|
||||
f.close()
|
||||
return temp_file
|
||||
filenames = [write_chunk(chunks[i], i) for i in range(len(chunks))]
|
||||
elif len(chunks) == 1:
|
||||
filenames = [filename] # avoid copying a single file
|
||||
else:
|
||||
filenames = []
|
||||
|
||||
|
|
@ -107,7 +135,7 @@ def run(filename, passes, js_engine, jcache):
|
|||
cores = min(multiprocessing.cpu_count(), filenames)
|
||||
if len(chunks) > 1 and cores >= 2:
|
||||
# We can parallelize
|
||||
if DEBUG: print >> sys.stderr, 'splitting up js optimization into %d chunks, using %d cores (total: %.2f MB)' % (len(chunks), cores, len(js)/(1024*1024.))
|
||||
if DEBUG: print >> sys.stderr, 'splitting up js optimization into %d chunks, using %d cores (total: %.2f MB)' % (len(chunks), cores, total_size/(1024*1024.))
|
||||
pool = multiprocessing.Pool(processes=cores)
|
||||
filenames = pool.map(run_on_chunk, commands, chunksize=1)
|
||||
else:
|
||||
|
|
@ -119,6 +147,7 @@ def run(filename, passes, js_engine, jcache):
|
|||
|
||||
filename += '.jo.js'
|
||||
f = open(filename, 'w')
|
||||
f.write(pre);
|
||||
for out_file in filenames:
|
||||
f.write(open(out_file).read())
|
||||
f.write('\n')
|
||||
|
|
@ -126,7 +155,8 @@ def run(filename, passes, js_engine, jcache):
|
|||
for cached in cached_outputs:
|
||||
f.write(cached); # TODO: preserve order
|
||||
f.write('\n')
|
||||
f.write(suffix)
|
||||
f.write(post);
|
||||
# No need to write suffix: if there was one, it is inside post which exists when suffix is there
|
||||
f.write('\n')
|
||||
f.close()
|
||||
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче