adjust emscrip parallelization heuristics
This commit is contained in:
Родитель
4ab1c8acec
Коммит
dfce473f4e
|
@ -42,7 +42,7 @@ def scan(ll, settings):
|
|||
if len(blockaddrs) > 0:
|
||||
settings['NECESSARY_BLOCKADDRS'] = blockaddrs
|
||||
|
||||
NUM_CHUNKS_PER_CORE = 4
|
||||
NUM_CHUNKS_PER_CORE = 5
|
||||
MIN_CHUNK_SIZE = 1024*1024
|
||||
MAX_CHUNK_SIZE = float(os.environ.get('EMSCRIPT_MAX_CHUNK_SIZE') or 'inf') # configuring this is just for debugging purposes
|
||||
|
||||
|
@ -149,7 +149,7 @@ def emscript(infile, settings, outfile, libraries=[]):
|
|||
if cores > 1:
|
||||
intended_num_chunks = cores * NUM_CHUNKS_PER_CORE
|
||||
chunk_size = max(MIN_CHUNK_SIZE, total_ll_size / intended_num_chunks)
|
||||
chunk_size += len(forwarded_data)/3 + 3*len(meta) # keep ratio of lots of function code to meta (expensive to process) and forwarded (cheap)
|
||||
chunk_size += 3*len(meta) # keep ratio of lots of function code to meta (expensive to process, and done in each parallel task)
|
||||
chunk_size = min(MAX_CHUNK_SIZE, chunk_size)
|
||||
else:
|
||||
chunk_size = MAX_CHUNK_SIZE # if 1 core, just use the max chunk size
|
||||
|
@ -170,7 +170,7 @@ def emscript(infile, settings, outfile, libraries=[]):
|
|||
chunks.append(curr)
|
||||
curr = ''
|
||||
if cores == 1 and total_ll_size < MAX_CHUNK_SIZE: assert len(chunks) == 1, 'no point in splitting up without multiple cores'
|
||||
if DEBUG: print >> sys.stderr, ' emscript: phase 2 working on %d chunks %s (intended chunk size: %.2f MB, meta: %.2f MB, forwarded: %.2f)' % (len(chunks), ('using %d cores' % cores) if len(chunks) > 1 else '', chunk_size/(1024*1024.), len(meta)/(1024*1024.), len(forwarded_data)/(1024*1024.))
|
||||
if DEBUG: print >> sys.stderr, ' emscript: phase 2 working on %d chunks %s (intended chunk size: %.2f MB, meta: %.2f MB, forwarded: %.2f MB, total: %.2f MB)' % (len(chunks), ('using %d cores' % cores) if len(chunks) > 1 else '', chunk_size/(1024*1024.), len(meta)/(1024*1024.), len(forwarded_data)/(1024*1024.), total_ll_size/(1024*1024.))
|
||||
|
||||
commands = [(i, chunks[i] + '\n' + meta, settings_file, compiler, forwarded_file, libraries) for i in range(len(chunks))]
|
||||
|
||||
|
|
Загрузка…
Ссылка в новой задаче