optimize memory initializer if it is large

emcc

@@ -1913,7 +1913,7 @@ try:
       shared.try_delete(memfile)
       def repl(m):
         # handle chunking of the memory initializer
-        s = m.groups(0)[0][1:-1]
+        s = m.groups(0)[0]
         open(memfile, 'wb').write(''.join(map(lambda x: chr(int(x or '0')), s.split(','))))
         if DEBUG:
           # Copy into temp dir as well, so can be run there too
@@ -1924,6 +1924,7 @@ try:
       src = re.sub(shared.JS.memory_initializer_pattern, repl, open(final).read(), count=1)
       open(final + '.mem.js', 'w').write(src)
       final += '.mem.js'
+      src = None
       js_transform_tempfiles[-1] = final # simple text substitution preserves comment line number mappings
       if DEBUG:
         if os.path.exists(memfile):
@@ -1931,6 +1932,15 @@ try:
           logging.debug('wrote memory initialization to %s' % memfile)
         else:
           logging.debug('did not see memory initialization')
+  elif shared.Settings.USE_TYPED_ARRAYS == 2 and not shared.Settings.MAIN_MODULE and not shared.Settings.SIDE_MODULE:
+    # not writing a binary init, but we can at least optimize them by splitting them up
+    src = open(final).read()
+    src = shared.JS.optimize_initializer(src)
+    if src is not None:
+      logging.debug('optimizing memory initialization')
+      open(final + '.mem.js', 'w').write(src)
+      final += '.mem.js'
+      src = None
 
   # It is useful to run several js optimizer passes together, to save on unneeded unparsing/reparsing
   js_optimizer_queue = []

tools/shared.py

@@ -1568,7 +1568,7 @@ JCache = cache.JCache(Cache)
 chunkify = cache.chunkify
 
 class JS:
-  memory_initializer_pattern = '/\* memory initializer \*/ allocate\((\[[\d, ]+\]), "i8", ALLOC_NONE, ([\d+Runtime\.GLOBAL_BASEH]+)\);'
+  memory_initializer_pattern = '/\* memory initializer \*/ allocate\(\[([\d, ]+)\], "i8", ALLOC_NONE, ([\d+Runtime\.GLOBAL_BASEH]+)\);'
   no_memory_initializer_pattern = '/\* no memory initializer \*/'
 
   memory_staticbump_pattern = 'STATICTOP = STATIC_BASE \+ (\d+);'
@@ -1652,6 +1652,75 @@ class JS:
     while x % by != 0: x += 1
     return x
 
+  INITIALIZER_CHUNK_SIZE = 10240
+
+  @staticmethod
+  def collect_initializers(src):
+    ret = []
+    max_offset = -1
+    for init in re.finditer(JS.memory_initializer_pattern, src):
+      contents = init.group(1).split(',')
+      offset = sum([int(x) if x[0] != 'R' else 0 for x in init.group(2).split('+')])
+      ret.append((offset, contents))
+      assert offset > max_offset
+      max_offset = offset
+    return ret
+
+  @staticmethod
+  def split_initializer(contents):
+    # given a memory initializer (see memory_initializer_pattern), split it up into multiple initializers to avoid long runs of zeros or a single overly-large allocator
+    ret = []
+    l = len(contents)
+    maxx = JS.INITIALIZER_CHUNK_SIZE
+    i = 0
+    start = 0
+    while 1:
+      if i - start >= maxx or (i > start and i == l):
+        #print >> sys.stderr, 'new', start, i-start
+        ret.append((start, contents[start:i]))
+        start = i
+      if i == l: break
+      if contents[i] != '0':
+        i += 1
+      else:
+        # look for a sequence of zeros
+        j = i + 1
+        while j < l and contents[j] == '0': j += 1
+        if j-i > maxx/10 or j-start >= maxx:
+          #print >> sys.stderr, 'skip', start, i-start, j-start
+          ret.append((start, contents[start:i])) # skip over the zeros starting at i and ending at j
+          start = j
+        i = j
+    return ret
+
+  @staticmethod
+  def replace_initializers(src, inits):
+    class State:
+      first = True
+    def rep(m):
+      if not State.first: return ''
+      # write out all the new initializers in place of the first old one
+      State.first = False
+      def gen_init(init):
+        offset, contents = init
+        return '/* memory initializer */ allocate([%s], "i8", ALLOC_NONE, Runtime.GLOBAL_BASE%s);' % (
+          ','.join(contents),
+          '' if offset == 0 else ('+%d' % offset)
+        )
+      return '\n'.join(map(gen_init, inits))
+    return re.sub(JS.memory_initializer_pattern, rep, src)
+
+  @staticmethod
+  def optimize_initializer(src):
+    inits = JS.collect_initializers(src)
+    if len(inits) == 0: return None
+    assert len(inits) == 1
+    init = inits[0]
+    offset, contents = init
+    assert offset == 0 # offset 0, singleton
+    if len(contents) <= JS.INITIALIZER_CHUNK_SIZE: return None
+    return JS.replace_initializers(src, JS.split_initializer(contents))
+
 # Compression of code and data for smaller downloads
 class Compression:
   on = False