Unity-Technologies
/
emscripten
зеркало из https://github.com/Unity-Technologies/emscripten.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

wip SPLIT_MEMORY option

This commit is contained in:
Alon Zakai 2015-09-14 11:07:17 -07:00
Родитель cc38a8bcce
Коммит 9c3d2efcf7
7 изменённых файлов: 337 добавлений и 49 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

14
emcc
Просмотреть файл

@ -950,6 +950,18 @@ try:
if shared.Settings.EMULATE_FUNCTION_POINTER_CASTS:
shared.Settings.ALIASING_FUNCTION_POINTERS = 0
if shared.Settings.SPLIT_MEMORY:
assert shared.Settings.SPLIT_MEMORY >= 1024*1024, 'SPLIT_MEMORY must be of at least 1048576, chunk size cannot be too small'
assert shared.Settings.SPLIT_MEMORY & (shared.Settings.SPLIT_MEMORY-1) == 0, 'SPLIT_MEMORY must be a power of 2'
if shared.Settings.ASM_JS == 1:
logging.warning('not all asm.js optimizations are possible with SPLIT_MEMORY, disabling those')
shared.Settings.ASM_JS = 2
assert not shared.Settings.ALLOW_MEMORY_GROWTH, 'TODO: memory growth with SPLIT_MEMORY'
assert not shared.Settings.SAFE_HEAP, 'TODO: safe heap with SPLIT_MEMORY'
if not js_opts:
js_opts = True
logging.debug('enabling js opts for SPLIT_MEMORY')
if shared.Settings.STB_IMAGE and final_suffix in JS_CONTAINING_SUFFIXES:
input_files.append((next_arg_index, shared.path_from_root('third_party', 'stb_image.c')))
next_arg_index += 1
@ -1547,6 +1559,8 @@ try:
if js_opts:
if shared.Settings.SAFE_HEAP: js_optimizer_queue += ['safeHeap']
if shared.Settings.SPLIT_MEMORY: js_optimizer_queue += ['splitMemory']
if shared.Settings.OUTLINING_LIMIT > 0:
js_optimizer_queue += ['outline']
js_optimizer_extra_info['sizeToOutline'] = shared.Settings.OUTLINING_LIMIT

73
emscripten.py
Просмотреть файл

@ -864,6 +864,79 @@ function _emscripten_replace_memory(newBuffer) {
buffer = newBuffer;
return true;
}
'''] + ['' if not settings['SPLIT_MEMORY'] else '''
function get8(ptr) {
ptr = ptr | 0;
return HEAP8s[ptr >> SPLIT_MEMORY_BITS][(ptr & SPLIT_MEMORY_MASK) >> 0] | 0;
}
function get16(ptr) {
ptr = ptr | 0;
return HEAP16s[ptr >> SPLIT_MEMORY_BITS][(ptr & SPLIT_MEMORY_MASK) >> 1] | 0;
}
function get32(ptr) {
ptr = ptr | 0;
return HEAP32s[ptr >> SPLIT_MEMORY_BITS][(ptr & SPLIT_MEMORY_MASK) >> 2] | 0;
}
function getU8(ptr) {
ptr = ptr | 0;
return HEAPU8s[ptr >> SPLIT_MEMORY_BITS][(ptr & SPLIT_MEMORY_MASK) >> 0] | 0;
}
function getU16(ptr) {
ptr = ptr | 0;
return HEAPU16s[ptr >> SPLIT_MEMORY_BITS][(ptr & SPLIT_MEMORY_MASK) >> 1] | 0;
}
function getU32(ptr) {
ptr = ptr | 0;
return HEAPU32s[ptr >> SPLIT_MEMORY_BITS][(ptr & SPLIT_MEMORY_MASK) >> 2] | 0;
}
function getF32(ptr) {
ptr = ptr | 0;
return +HEAPF32s[ptr >> SPLIT_MEMORY_BITS][(ptr & SPLIT_MEMORY_MASK) >> 2]; // TODO: fround when present
}
function getF64(ptr) {
ptr = ptr | 0;
return +HEAPF64s[ptr >> SPLIT_MEMORY_BITS][(ptr & SPLIT_MEMORY_MASK) >> 3];
}
function set8(ptr, value) {
ptr = ptr | 0;
value = value | 0;
HEAP8s[ptr >> SPLIT_MEMORY_BITS][(ptr & SPLIT_MEMORY_MASK) >> 0] = value;
}
function set16(ptr, value) {
ptr = ptr | 0;
value = value | 0;
HEAP16s[ptr >> SPLIT_MEMORY_BITS][(ptr & SPLIT_MEMORY_MASK) >> 1] = value;
}
function set32(ptr, value) {
ptr = ptr | 0;
value = value | 0;
HEAP32s[ptr >> SPLIT_MEMORY_BITS][(ptr & SPLIT_MEMORY_MASK) >> 2] = value;
}
function setU8(ptr, value) {
ptr = ptr | 0;
value = value | 0;
HEAPU8s[ptr >> SPLIT_MEMORY_BITS][(ptr & SPLIT_MEMORY_MASK) >> 0] = value;
}
function setU16(ptr, value) {
ptr = ptr | 0;
value = value | 0;
HEAPU16s[ptr >> SPLIT_MEMORY_BITS][(ptr & SPLIT_MEMORY_MASK) >> 1] = value;
}
function setU32(ptr, value) {
ptr = ptr | 0;
value = value | 0;
HEAPU32s[ptr >> SPLIT_MEMORY_BITS][(ptr & SPLIT_MEMORY_MASK) >> 2] = value;
}
function setF32(ptr, value) {
ptr = ptr | 0;
value = +value;
HEAPF32s[ptr >> SPLIT_MEMORY_BITS][(ptr & SPLIT_MEMORY_MASK) >> 2] = value;
}
function setF64(ptr, value) {
ptr = ptr | 0;
value = +value;
HEAPF64s[ptr >> SPLIT_MEMORY_BITS][(ptr & SPLIT_MEMORY_MASK) >> 3] = value;
}
'''] + ['''
// EMSCRIPTEN_START_FUNCS
function stackAlloc(size) {

3
src/jsifier.js
Просмотреть файл

@ -376,6 +376,9 @@ function JSify(data, functionsOnly) {
print('DYNAMIC_BASE = DYNAMICTOP = Runtime.alignMemory(STACK_MAX);\n');
print('assert(DYNAMIC_BASE < TOTAL_MEMORY, "TOTAL_MEMORY not big enough for stack");\n');
}
if (SPLIT_MEMORY) {
print('assert(STACK_MAX < SPLIT_MEMORY, "SPLIT_MEMORY size must be big enough so the entire static memory + stack can fit in one chunk, need " + STACK_MAX);\n');
}
if (asmLibraryFunctions.length > 0) {
print('// ASM_LIBRARY FUNCTIONS');

132
src/preamble.js
Просмотреть файл

@ -1236,6 +1236,8 @@ if (typeof Atomics === 'undefined') {
}
#else // USE_PTHREADS
#if SPLIT_MEMORY == 0
buffer = new ArrayBuffer(TOTAL_MEMORY);
HEAP8 = new Int8Array(buffer);
HEAP16 = new Int16Array(buffer);
@ -1245,6 +1247,136 @@ HEAPU16 = new Uint16Array(buffer);
HEAPU32 = new Uint32Array(buffer);
HEAPF32 = new Float32Array(buffer);
HEAPF64 = new Float64Array(buffer);
#else // SPLIT_MEMORY
// make sure total memory is a multiple of the split memory size
var SPLIT_MEMORY = {{{ SPLIT_MEMORY }}};
var SPLIT_MEMORY_MASK = SPLIT_MEMORY - 1;
var SPLIT_MEMORY_BITS = 0;
if (TOTAL_MEMORY % SPLIT_MEMORY) {
TOTAL_MEMORY += SPLIT_MEMORY - (TOTAL_MEMORY % SPLIT_MEMORY);
Module.printErr('increasing TOTAL_MEMORY to ' + TOTAL_MEMORY + ' to be a multiple of the split memory size ' + SPLIT_MEMORY + ')');
}
var buffers = [], HEAP8s = [], HEAP16s = [], HEAP32s = [], HEAPU8s = [], HEAPU16s = [], HEAPU32s = [], HEAPF32s = [], HEAPF64s = [];
(function() {
var temp = SPLIT_MEMORY;
while (temp) {
temp >>= 1;
SPLIT_MEMORY_BITS++;
}
for (var i = 0; i < TOTAL_MEMORY / SPLIT_MEMORY; i++) {
var curr = new ArrayBuffer(SPLIT_MEMORY);
buffers.push(curr);
HEAP8s.push(new Int8Array(curr));
HEAP16s.push(new Int16Array(curr));
HEAP32s.push(new Int32Array(curr));
HEAPU8s.push(new Uint8Array(curr));
HEAPU16s.push(new Uint16Array(curr));
HEAPU32s.push(new Uint32Array(curr));
HEAPF32s.push(new Float32Array(curr));
HEAPF64s.push(new Float64Array(curr));
}
// support HEAP8.subarray etc.
function fake(real) {
return {
set: function(array, offset) {
if (offset === undefined) offset = 0;
assert((offset & SPLIT_MEMORY_MASK) + (array.length * real[0].BYTES_PER_ELEMENT) < SPLIT_MEMORY, 'TODO: set over split chunks')
var index = offset >> SPLIT_MEMORY_BITS;
return real[index].set(array, offset & SPLIT_MEMORY_MASK);
},
subarray: function(from, to) {
if (to === undefined) to = SPLIT_MEMORY;
var start = from >> SPLIT_MEMORY_BITS;
var end = from >> SPLIT_MEMORY_BITS;
assert(start === end, 'subarray cannot span split chunks');
return real[start].subarray(from & SPLIT_MEMORY_MASK, to & SPLIT_MEMORY_MASK);
}
};
}
HEAP8 = fake(HEAP8s);
HEAP16 = fake(HEAP16s);
HEAP32 = fake(HEAP32s);
HEAPU8 = fake(HEAPU8s);
HEAPU16 = fake(HEAPU16s);
HEAPU32 = fake(HEAPU32s);
HEAPF32 = fake(HEAPF32s);
HEAPF64 = fake(HEAPF64s);
})();
// TODO: add SAFE_HEAP here
function get8(ptr) {
ptr = ptr | 0;
return HEAP8s[ptr >> SPLIT_MEMORY_BITS][(ptr & SPLIT_MEMORY_MASK) >> 0] | 0;
}
function get16(ptr) {
ptr = ptr | 0;
return HEAP16s[ptr >> SPLIT_MEMORY_BITS][(ptr & SPLIT_MEMORY_MASK) >> 1] | 0;
}
function get32(ptr) {
ptr = ptr | 0;
return HEAP32s[ptr >> SPLIT_MEMORY_BITS][(ptr & SPLIT_MEMORY_MASK) >> 2] | 0;
}
function getU8(ptr) {
ptr = ptr | 0;
return HEAPU8s[ptr >> SPLIT_MEMORY_BITS][(ptr & SPLIT_MEMORY_MASK) >> 0] | 0;
}
function getU16(ptr) {
ptr = ptr | 0;
return HEAPU16s[ptr >> SPLIT_MEMORY_BITS][(ptr & SPLIT_MEMORY_MASK) >> 1] | 0;
}
function getU32(ptr) {
ptr = ptr | 0;
return HEAPU32s[ptr >> SPLIT_MEMORY_BITS][(ptr & SPLIT_MEMORY_MASK) >> 2] | 0;
}
function getF32(ptr) {
ptr = ptr | 0;
return +HEAPF32s[ptr >> SPLIT_MEMORY_BITS][(ptr & SPLIT_MEMORY_MASK) >> 2];
}
function getF64(ptr) {
ptr = ptr | 0;
return +HEAPF64s[ptr >> SPLIT_MEMORY_BITS][(ptr & SPLIT_MEMORY_MASK) >> 3];
}
function set8(ptr, value) {
ptr = ptr | 0;
value = value | 0;
HEAP8s[ptr >> SPLIT_MEMORY_BITS][(ptr & SPLIT_MEMORY_MASK) >> 0] = value;
}
function set16(ptr, value) {
ptr = ptr | 0;
value = value | 0;
HEAP16s[ptr >> SPLIT_MEMORY_BITS][(ptr & SPLIT_MEMORY_MASK) >> 1] = value;
}
function set32(ptr, value) {
ptr = ptr | 0;
value = value | 0;
HEAP32s[ptr >> SPLIT_MEMORY_BITS][(ptr & SPLIT_MEMORY_MASK) >> 2] = value;
}
function setU8(ptr, value) {
ptr = ptr | 0;
value = value | 0;
HEAPU8s[ptr >> SPLIT_MEMORY_BITS][(ptr & SPLIT_MEMORY_MASK) >> 0] = value;
}
function setU16(ptr, value) {
ptr = ptr | 0;
value = value | 0;
HEAPU16s[ptr >> SPLIT_MEMORY_BITS][(ptr & SPLIT_MEMORY_MASK) >> 1] = value;
}
function setU32(ptr, value) {
ptr = ptr | 0;
value = value | 0;
HEAPU32s[ptr >> SPLIT_MEMORY_BITS][(ptr & SPLIT_MEMORY_MASK) >> 2] = value;
}
function setF32(ptr, value) {
ptr = ptr | 0;
value = +value;
HEAPF32s[ptr >> SPLIT_MEMORY_BITS][(ptr & SPLIT_MEMORY_MASK) >> 2] = value;
}
function setF64(ptr, value) {
ptr = ptr | 0;
value = +value;
HEAPF64s[ptr >> SPLIT_MEMORY_BITS][(ptr & SPLIT_MEMORY_MASK) >> 3] = value;
}
#endif // SPLIT_MEMORY
#endif // USE_PTHREADS
// Endianness check (note: assumes compiler arch was little-endian)

3
src/settings.js
Просмотреть файл

@ -562,6 +562,9 @@ var EMTERPRETIFY_ADVISE = 0; // Performs a static analysis to suggest which func
// After showing the suggested list, compilation will halt. You can apply the provided list as an
// emcc argument when compiling later.
var SPLIT_MEMORY = 0; // If > 0, we split memory into chunks, of the size given in this parameter.
// TODO: docs
var RUNNING_JS_OPTS = 0; // whether js opts will be run, after the main compiler
var BOOTSTRAPPING_STRUCT_INFO = 0; // whether we are in the generate struct_info bootstrap phase

135
tools/js-optimizer.js
Просмотреть файл

@ -5597,51 +5597,55 @@ function outline(ast) {
});
}
function safeHeap(ast) {
function fixPtr(ptr, heap) {
switch (heap) {
case 'HEAP8': case 'HEAPU8': break;
case 'HEAP16': case 'HEAPU16': {
if (ptr[0] === 'binary') {
assert(ptr[1] === '>>' && ptr[3][0] === 'num' && ptr[3][1] === 1);
ptr = ptr[2]; // skip the shift
} else {
ptr = ['binary', '*', ptr, ['num', 2]]; // was unshifted, convert to absolute address
}
break;
function fixPtr(ptr, heap, shell) {
switch (heap) {
case 'HEAP8': case 'HEAPU8': break;
case 'HEAP16': case 'HEAPU16': {
if (ptr[0] === 'binary') {
assert(ptr[1] === '>>' && ptr[3][0] === 'num' && ptr[3][1] === 1);
ptr = ptr[2]; // skip the shift
} else {
ptr = ['binary', '*', ptr, ['num', 2]]; // was unshifted, convert to absolute address
}
case 'HEAP32': case 'HEAPU32': {
if (ptr[0] === 'binary') {
assert(ptr[1] === '>>' && ptr[3][0] === 'num' && ptr[3][1] === 2);
ptr = ptr[2]; // skip the shift
} else {
ptr = ['binary', '*', ptr, ['num', 4]]; // was unshifted, convert to absolute address
}
break;
}
case 'HEAPF32': {
if (ptr[0] === 'binary') {
assert(ptr[1] === '>>' && ptr[3][0] === 'num' && ptr[3][1] === 2);
ptr = ptr[2]; // skip the shift
} else {
ptr = ['binary', '*', ptr, ['num', 4]]; // was unshifted, convert to absolute address
}
break;
}
case 'HEAPF64': {
if (ptr[0] === 'binary') {
assert(ptr[1] === '>>' && ptr[3][0] === 'num' && ptr[3][1] === 3);
ptr = ptr[2]; // skip the shift
} else {
ptr = ['binary', '*', ptr, ['num', 8]]; // was unshifted, convert to absolute address
}
break;
}
default: throw 'bad heap ' + heap;
break;
}
case 'HEAP32': case 'HEAPU32': {
if (ptr[0] === 'binary') {
assert(ptr[1] === '>>' && ptr[3][0] === 'num' && ptr[3][1] === 2);
ptr = ptr[2]; // skip the shift
} else {
ptr = ['binary', '*', ptr, ['num', 4]]; // was unshifted, convert to absolute address
}
break;
}
case 'HEAPF32': {
if (ptr[0] === 'binary') {
assert(ptr[1] === '>>' && ptr[3][0] === 'num' && ptr[3][1] === 2);
ptr = ptr[2]; // skip the shift
} else {
ptr = ['binary', '*', ptr, ['num', 4]]; // was unshifted, convert to absolute address
}
break;
}
case 'HEAPF64': {
if (ptr[0] === 'binary') {
assert(ptr[1] === '>>' && ptr[3][0] === 'num' && ptr[3][1] === 3);
ptr = ptr[2]; // skip the shift
} else {
ptr = ['binary', '*', ptr, ['num', 8]]; // was unshifted, convert to absolute address
}
break;
}
default: {
if (!shell) throw 'bad heap ' + heap;
return ptr; // unchanged
}
ptr = ['binary', '|', ptr, ['num', 0]];
return ptr;
}
ptr = ['binary', '|', ptr, ['num', 0]];
return ptr;
}
function safeHeap(ast) {
var SAFE_HEAP_FUNCS = set('SAFE_HEAP_LOAD', 'SAFE_HEAP_LOAD_D', 'SAFE_HEAP_STORE', 'SAFE_HEAP_STORE_D', 'SAFE_FT_MASK');
traverseGeneratedFunctions(ast, function(func) {
if (func[1] in SAFE_HEAP_FUNCS) return null;
@ -5716,6 +5720,51 @@ function safeHeap(ast) {
});
}
function splitMemory(ast, shell) {
traverse(ast, function(node, type) {
if (type === 'assign') {
if (node[1] === true && node[2][0] === 'sub') {
var heap = node[2][1][1];
var ptr = fixPtr(node[2][2], heap, shell);
var value = node[3];
switch (heap) {
case 'HEAP8': return ['call', ['name', 'set8'], [ptr, makeAsmCoercion(value, ASM_INT)]];
case 'HEAP16': return ['call', ['name', 'set16'], [ptr, makeAsmCoercion(value, ASM_INT)]];
case 'HEAP32': return ['call', ['name', 'set32'], [ptr, makeAsmCoercion(value, ASM_INT)]];
case 'HEAPU8': return ['call', ['name', 'setU8'], [ptr, makeAsmCoercion(value, ASM_INT)]];
case 'HEAPU16': return ['call', ['name', 'setU16'], [ptr, makeAsmCoercion(value, ASM_INT)]];
case 'HEAPU32': return ['call', ['name', 'setU32'], [ptr, makeAsmCoercion(value, ASM_INT)]];
case 'HEAPF32': return ['call', ['name', 'setF32'], [ptr, makeAsmCoercion(value, ASM_DOUBLE)]];
case 'HEAPF64': return ['call', ['name', 'setF64'], [ptr, makeAsmCoercion(value, ASM_DOUBLE)]];
default: if (!shell) throw 'bad heap ' + heap;
}
}
} else if (type === 'sub') {
var target = node[1][1];
if (target[0] === 'H') {
// heap access
var heap = target;
var ptr = fixPtr(node[2], heap, shell);
switch (heap) {
case 'HEAP8': return ['call', ['name', 'get8'], [ptr]];
case 'HEAP16': return ['call', ['name', 'get16'], [ptr]];
case 'HEAP32': return ['call', ['name', 'get32'], [ptr]];
case 'HEAPU8': return ['call', ['name', 'getU8'], [ptr]];
case 'HEAPU16': return ['call', ['name', 'getU16'], [ptr]];
case 'HEAPU32': return ['call', ['name', 'getU32'], [ptr]];
case 'HEAPF32': return ['call', ['name', 'getF32'], [ptr]];
case 'HEAPF64': return ['call', ['name', 'getF64'], [ptr]];
default: if (!shell) throw 'bad heap ' + heap;
}
}
}
});
}
function splitMemoryShell(ast) {
splitMemory(ast, true);
}
function optimizeFrounds(ast) {
// collapse fround(fround(..)), which can happen due to elimination
// also emit f0 instead of fround(0) (except in returns)
@ -7559,6 +7608,8 @@ var passes = {
relocate: relocate,
outline: outline,
safeHeap: safeHeap,
splitMemory: splitMemory,
splitMemoryShell: splitMemoryShell,
optimizeFrounds: optimizeFrounds,
ensureLabelSet: ensureLabelSet,
emterpretify: emterpretify,

26
tools/js_optimizer.py
Просмотреть файл

@ -305,7 +305,6 @@ def run_on_js(filename, passes, js_engine, source_map=False, extra_info=None, ju
suffix_end = js.find('\n', suffix_start)
suffix = js[suffix_start:suffix_end] + '\n'
# if there is metadata, we will run only on the generated functions. If there isn't, we will run on everything.
generated = set(eval(suffix[len(suffix_marker)+1:]))
# Find markers
start_funcs = js.find(start_funcs_marker)
@ -330,6 +329,10 @@ def run_on_js(filename, passes, js_engine, source_map=False, extra_info=None, ju
if cleanup:
passes = filter(lambda p: p != 'cleanup', passes) # we will do it manually
split_memory = 'splitMemory' in passes
if cleanup:
passes = filter(lambda p: p != 'splitMemory', passes) # we will do it manually
if not minify_globals:
pre = js[:start_funcs + len(start_funcs_marker)]
post = js[end_funcs + len(end_funcs_marker):]
@ -447,7 +450,7 @@ EMSCRIPTEN_FUNCS();
for filename in filenames: temp_files.note(filename)
if closure or cleanup:
if closure or cleanup or split_memory:
# run on the shell code, everything but what we js-optimize
start_asm = '// EMSCRIPTEN_START_ASM\n'
end_asm = '// EMSCRIPTEN_END_ASM\n'
@ -461,14 +464,23 @@ EMSCRIPTEN_FUNCS();
c.write(cl_sep)
c.write(post_2)
c.close()
cld = cle
if split_memory:
if DEBUG: print >> sys.stderr, 'running splitMemory on shell code'
cld = run_on_chunk(js_engine + [JS_OPTIMIZER, cld, 'splitMemoryShell'])
f = open(cld, 'a')
f.write(suffix_marker)
f.close()
if closure:
if DEBUG: print >> sys.stderr, 'running closure on shell code'
cld = shared.Building.closure_compiler(cle, pretty='minifyWhitespace' not in passes)
else:
cld = shared.Building.closure_compiler(cld, pretty='minifyWhitespace' not in passes)
temp_files.note(cld)
elif cleanup:
if DEBUG: print >> sys.stderr, 'running cleanup on shell code'
cld = cle + '.js'
subprocess.Popen(js_engine + [JS_OPTIMIZER, cle, 'noPrintMetadata'] + (['minifyWhitespace'] if 'minifyWhitespace' in passes else []), stdout=open(cld, 'w')).communicate()
temp_files.note(cld)
next = cld + '.cl.js'
temp_files.note(next)
subprocess.Popen(js_engine + [JS_OPTIMIZER, cld, 'noPrintMetadata'] + (['minifyWhitespace'] if 'minifyWhitespace' in passes else []), stdout=open(next, 'w')).communicate()
cld = next
coutput = open(cld).read()
coutput = coutput.replace('wakaUnknownBefore();', start_asm)
after = 'wakaUnknownAfter'