support for running the js optimizer, eliminator and closure compiler from emcc

emcc

@@ -324,6 +324,25 @@ elif use_compiler:
     shared.Building.emscripten(target_basename + '.bc', append_ext=False)
     shutil.move(target_basename + '.bc.o.js', target_basename + '.js')
 
+    if opt_level >= 1:
+      # js optimizer
+      shared.Building.js_optimizer(target_basename + '.js', 'loopOptimizer')
+      shutil.move(target_basename + '.js.jo.js', target_basename + '.js')
+
+      # eliminator
+      shared.Building.eliminator(target_basename + '.js')
+      shutil.move(target_basename + '.js.el.js', target_basename + '.js')
+
+    if opt_level >= 3:
+      # closure
+      shared.Building.closure_compiler(target_basename + '.js')
+      shutil.move(target_basename + '.js.cc.js', target_basename + '.js')
+
+    if opt_level >= 1:
+      # js optimizer
+      shared.Building.js_optimizer(target_basename + '.js', 'simplifyExpressions')
+      shutil.move(target_basename + '.js.jo.js', target_basename + '.js')
+
     # If we were asked to also generate HTML, do that
     if final_suffix == 'html':
       shell = open(shared.path_from_root('src', 'shell.html')).read()

tests/runner.py

@@ -4952,21 +4952,23 @@ JavaScript in the final linking stage of building.
           assert ('Warning: Applying some potentially unsafe optimizations!' in output[1]) == (opt_level >= 3), 'unsafe warning should appear in opt >= 3'
           self.assertContained('hello, world!', run_js('something.js'))
 
-          # Verify optimization level in the generated code
+          # Verify optimization level etc. in the generated code
           # XXX these are quite sensitive, and will need updating when code generation changes
           generated = open('something.js').read() # TODO: parse out the _main function itself, not support code, if the tests below need that some day
-          assert ('while(1) switch(__label__)' in generated) == (opt_level <= 1), 'relooping should be in opt >= 2'
+          assert ('(__label__)' in generated) == (opt_level <= 1), 'relooping should be in opt >= 2'
           assert ('assert(STACKTOP < STACK_MAX)' in generated) == (opt_level == 0), 'assertions should be in opt == 0'
-          assert ('|0)/2)|0)' in generated) == (opt_level <= 2), 'corrections should be in opt <= 2'
+          assert ('|0)/2)|0)' in generated or '| 0) / 2 | 0)' in generated) == (opt_level <= 2), 'corrections should be in opt <= 2'
-          assert 'var $i;' in generated, 'micro opts should always be on'
+          if opt_level < 3: assert 'var $i;' in generated, 'micro opts should always be on' # TODO: find a way to check it even with closure
-          assert 'HEAP32[' in generated, 'typed arrays 2 should be used by default'
+          assert 'new Uint16Array' in generated and 'new Uint32Array' in generated, 'typed arrays 2 should be used by default'
           assert 'SAFE_HEAP' not in generated, 'safe heap should not be used by default'
+          assert ': while(' not in generated, 'when relooping we also js-optimize, so there should be no labelled whiles'
+          if opt_level >= 1 and opt_level < 3: assert 'HEAP8[HEAP32[' in generated, 'eliminator should create compound expressions, and fewer one-time vars'  # TODO: find a way to check it even with closure
+          if opt_level >= 3: assert 'Module._main = ' in generated, 'closure compiler should have been run'
 
-      # TODO: -O1 plus eliminator, plus js optimizer
-      # emcc --typed-arrays=x .. ==> should use typed arrays. default should be 2
       # emcc --llvm-opts=x .. ==> pick level of LLVM optimizations (default is 0, to be safe?)
       # emcc -s RELOOP=1 src.cpp ==> should pass -s to emscripten.py
       # When doing unsafe opts, can we run -Ox on the source, not just at the very end?
+      #  In fact we can run safe opts at that time too, now we are a gcc replacement. Removes the entire need for llvm opts only at the end.
       # linking - TODO. in particular, test normal project linking, static and dynamic: get_library should not need to be told what to link!
       #   emcc a.cpp b.cpp => one .js
       #   emcc a.cpp b.cpp -c => two .o files
@@ -5302,13 +5304,6 @@ if __name__ == '__main__':
 
   # Sanity checks
 
-  def check_engine(engine):
-    try:
-      return 'hello, world!' in run_js(path_from_root('tests', 'hello_world.js'), engine)
-    except Exception, e:
-      print 'Checking JS engine %s failed. Check ~/.emscripten. Details: %s' % (str(engine), str(e))
-      return False
-
   if not check_engine(COMPILER_ENGINE):
     print 'WARNING: The JavaScript shell used for compiling does not seem to work'
 

tools/shared.py

@@ -114,6 +114,14 @@ class TempFiles:
 
 # Utilities
 
+def check_engine(engine):
+  # TODO: we call this several times, perhaps cache the results?
+  try:
+    return 'hello, world!' in run_js(path_from_root('tests', 'hello_world.js'), engine)
+  except Exception, e:
+    print 'Checking JS engine %s failed. Check ~/.emscripten. Details: %s' % (str(engine), str(e))
+    return False
+
 def timeout_run(proc, timeout, note):
   start = time.time()
   if timeout is not None:
@@ -447,3 +455,46 @@ class Building:
     Building.LLVM_OPT_OPTS = opts
     return opts
 
+  @staticmethod
+  def js_optimizer(filename, passes):
+    if not check_engine(NODE_JS):
+      raise Exception('Node.js appears to be missing or broken, looked at: ' + str(NODE_JS))
+
+    if type(passes) == str:
+      passes = [passes]
+    input = open(filename, 'r').read()
+    output = Popen([NODE_JS, JS_OPTIMIZER] + passes, stdin=PIPE, stdout=PIPE).communicate(input)[0]
+    filename += '.jo.js'
+    f = open(filename, 'w')
+    f.write(output)
+    f.close()
+
+  @staticmethod
+  def eliminator(filename):
+    if not check_engine(NODE_JS):
+      raise Exception('Node.js appears to be missing or broken, looked at: ' + str(NODE_JS))
+
+    coffee = path_from_root('tools', 'eliminator', 'node_modules', 'coffee-script', 'bin', 'coffee')
+    eliminator = path_from_root('tools', 'eliminator', 'eliminator.coffee')
+    input = open(filename, 'r').read()
+    output = Popen([coffee, eliminator], stdin=PIPE, stdout=PIPE, stderr=PIPE).communicate(input)[0]
+    filename += '.el.js'
+    f = open(filename, 'w')
+    f.write(output)
+    f.close()
+
+  @staticmethod
+  def closure_compiler(filename):
+    if not os.path.exists(CLOSURE_COMPILER):
+      raise Exception('Closure compiler appears to be missing, looked at: ' + str(CLOSURE_COMPILER))
+
+    # Something like this (adjust memory as needed):
+    #   java -Xmx1024m -jar CLOSURE_COMPILER --compilation_level ADVANCED_OPTIMIZATIONS --variable_map_output_file src.cpp.o.js.vars --js src.cpp.o.js --js_output_file src.cpp.o.cc.js
+    cc_output = Popen(['java', '-jar', CLOSURE_COMPILER,
+                       '--compilation_level', 'ADVANCED_OPTIMIZATIONS',
+                       #'--formatting', 'PRETTY_PRINT',
+                       #'--variable_map_output_file', filename + '.vars',
+                       '--js', filename, '--js_output_file', filename + '.cc.js'], stdout=PIPE, stderr=STDOUT).communicate()[0]
+    if 'ERROR' in cc_output:
+      raise Exception('Error in cc output: ' + cc_output)
+