source code compression option in emcc

emcc

@@ -160,18 +160,23 @@ Options that are modified or new in %s include:
                            break the generated code! If that happens, try
                            -O2 and then adding dangerous optimizations one
                            by one.
+
   -s OPTION=VALUE          JavaScript code generation option passed
                            into the emscripten compiler. For the
                            available options, see src/settings.js
+
   --typed-arrays <mode>    0: No typed arrays
                            1: Parallel typed arrays
                            2: Shared (C-like) typed arrays (default)
+
   --llvm-opts <level>      0: No LLVM optimizations (default in -O0)
                            1: -O1 LLVM optimizations (default in -O1)
                            2: -O2 LLVM optimizations
                            3: -O3 LLVM optimizations (default in -O2+)
+
   --closure <on>           0: No closure compiler (default in -O0, -O1)
                            1: Run closure compiler (default in -O2, -O3)
+
   --js-transform <cmd>     <cmd> will be called on the generated code
                            before it is optimized. This lets you modify
                            the JavaScript, for example adding some code
@@ -186,18 +191,13 @@ Options that are modified or new in %s include:
                            list of arguments, for example, <cmd> of
                            "python processor.py" will cause a python
                            script to be run.
+
   --pre-js <file>          A file whose contents are added before the
                            generated code
+
   --post-js <file>         A file whose contents are added after the
                            generated code
-  --minify <on>            0: Do not minify the generated JavaScript's
-                              whitespace (default if closure compiler
-                              will not be run)
-                           1: Minify the generated JavaScript's
-                              whitespace (default if closure compiler
-                              will be run). Note that this by itself
-                              will not minify the code (closure does
-                              that)
+
   --embed-file <file>      A file to embed inside the generated
                            JavaScript. The compiled code will be able
                            to access the file in the current directory
@@ -205,6 +205,7 @@ Options that are modified or new in %s include:
                            just the filename, without a path to it).
                            If a directory is passed here, its entire
                            contents will be embedded.
+
   --preload-file <name>    A file to preload before running the
                            compiled code asynchronously. Otherwise
                            similar to --embed-file, except that this
@@ -218,6 +219,31 @@ Options that are modified or new in %s include:
                            the alternative, change the suffix).
                            If a directory is passed here, its entire
                            contents will be preloaded.
+
+  --compression <codec>    Compress both the compiled code and embedded/
+                           preloaded files. <codec> should be a triple,
+
+                              <native_encoder>,<js_decoder>,<js_name>
+
+                           where native_encoder is a native executable
+                           that compresses stdin to stdout (the simplest
+                           possible interface), js_decoder is a
+                           JavaScript file that implements a decoder,
+                           and js_name is the name of the function to
+                           call in the decoder file (which should
+                           receive an array/typed array and return
+                           an array/typed array.
+                           Compression only works when generating HTML.
+
+  --minify <on>            0: Do not minify the generated JavaScript's
+                              whitespace (default if closure compiler
+                              will not be run)
+                           1: Minify the generated JavaScript's
+                              whitespace (default if closure compiler
+                              will be run). Note that this by itself
+                              will not minify the code (closure does
+                              that)
+
   --ignore-dynamic-linking Normally emcc will treat dynamic linking like
                            static linking, by linking in the code from
                            the dynamic library. This fails if the same
@@ -226,6 +252,7 @@ Options that are modified or new in %s include:
                            which allows the build system to proceed without
                            errors. However, you will need to manually
                            link to the shared libraries later on yourself.
+
   --shell-file <path>      The path name to a skeleton HTML file used
                            when generating HTML output. The shell file
                            used needs to have this token inside it:
@@ -349,6 +376,17 @@ else:
 def in_temp(name):
   return os.path.join(temp_dir, name)
 
+class Compression:
+  on = False
+
+  @staticmethod
+  def compressed_name(filename):
+    return filename + '.compress'
+
+  @staticmethod
+  def compress(filename):
+    execute(Compression.encoder, stdin=open(filename, 'rb'), stdout=open(Compression.compressed_name(filename), 'wb'))
+
 try:
   call = CXX if use_cxx else CC
 
@@ -365,6 +403,7 @@ try:
   minify_whitespace = None
   embed_files = []
   preload_files = []
+  compression = None
   ignore_dynamic_linking = False
   shell_path = shared.path_from_root('src', 'shell.html')
 
@@ -419,6 +458,18 @@ try:
       preload_files.append(newargs[i+1])
       newargs[i] = ''
       newargs[i+1] = ''
+    elif newargs[i].startswith('--compression'):
+      check_bad_eq(newargs[i])
+      parts = newargs[i+1].split(',')
+      assert len(parts) == 3, '--compression requires specifying    native_encoder,js_decoder,js_name  - see emcc --help. got: %s' % newargs[i+1]
+      Compression.encoder = parts[0]
+      Compression.decoder = parts[1]
+      Compression.js_name = parts[2]
+      assert os.path.exists(Compression.encoder), 'native encoder %s does not exist' % Compression.encoder
+      assert os.path.exists(Compression.decoder), 'js decoder %s does not exist' % Compression.decoder
+      Compression.on = True
+      newargs[i] = ''
+      newargs[i+1] = ''
     elif newargs[i] == '-MF': # clang cannot handle this, so we fake it
       f = open(newargs[i+1], 'w')
       f.write('\n')
@@ -535,6 +586,8 @@ try:
   else:
     final_suffix = ''
 
+  assert not (Compression.on and final_suffix != 'html'), 'Compression only works when generating HTML'
+
   # Apply optimization level settings
   shared.Settings.apply_opt_level(opt_level, noisy=True)
 
@@ -754,6 +807,9 @@ try:
   # Embed and preload files
   if len(embed_files) + len(preload_files) > 0:
     if DEBUG: print >> sys.stderr, 'emcc: setting up files'
+
+    assert not Compression.on
+
     code = ''
 
     # Sanity checks
@@ -915,7 +971,36 @@ try:
     if DEBUG: print >> sys.stderr, 'emcc: generating HTML'
     shell = open(shell_path).read()
     html = open(target, 'w')
-    html.write(shell.replace('{{{ SCRIPT_CODE }}}', open(final).read()))
+    if not Compression.on:
+      html.write(shell.replace('{{{ SCRIPT_CODE }}}', open(final).read()))
+    else:
+      # Add the decompressor in the html, and code to
+      #   1. download the compressed file
+      #   2. decompress to a typed array
+      #   3. convert to a string of source code
+      #   4. insert a script element with that source code (more effective than eval)
+      js_target = unsuffixed(target) + '.js'
+      shutil.move(final, js_target)
+      Compression.compress(js_target)
+      decoding = open(Compression.decoder).read()
+      decoding += '''
+        var compiledCodeXHR = new XMLHttpRequest();
+        compiledCodeXHR.open('GET', '%s', true);
+        compiledCodeXHR.responseType = 'arraybuffer';
+        compiledCodeXHR.onload = function() {
+          var arrayBuffer = compiledCodeXHR.response;
+          if (!arrayBuffer) throw('Loading compressed code failed.');
+          var byteArray = new Uint8Array(arrayBuffer);
+          var decompressed = %s(byteArray);
+          var source = Array.prototype.slice.apply(decompressed).map(function(x) { return String.fromCharCode(x) }).join('');
+          var scriptTag = document.createElement('script');
+          scriptTag.setAttribute('type', 'text/javascript');
+          scriptTag.innerHTML = source;
+          document.body.appendChild(scriptTag);
+        };
+        compiledCodeXHR.send(null);
+''' % (Compression.compressed_name(js_target), Compression.js_name)
+      html.write(shell.replace('{{{ SCRIPT_CODE }}}', decoding))
     html.close()
   else:
     # copy final JS to output

tests/runner.py

@@ -6220,6 +6220,28 @@ f.close()
           emscripten_run_script(output);
 ''')
 
+    def test_emcc_compression(self):
+      open(os.path.join(self.get_dir(), 'main.cpp'), 'w').write(self.with_report_result(r'''
+        #include <stdio.h>
+        #include <emscripten.h>
+        int main() {
+          printf("hello compressed world\n");
+          int result = 1;
+          REPORT_RESULT();
+          return 0;
+        }
+      '''))
+
+      Popen([EMCC, os.path.join(self.get_dir(), 'main.cpp'), '-o', 'page.html',
+             '--compression', '%s,%s,%s' % (path_from_root('third_party', 'lzma.js', 'lzma-native'),
+                                            path_from_root('third_party', 'lzma.js', 'lzma-decoder.js'),
+                                            'LZMA.decompress')]).communicate()
+      assert os.path.exists(os.path.join(self.get_dir(), 'page.js')), 'must be side js'
+      assert os.path.exists(os.path.join(self.get_dir(), 'page.js.compress')), 'must be side compressed js'
+      assert os.stat(os.path.join(self.get_dir(), 'page.js')).st_size > os.stat(os.path.join(self.get_dir(), 'page.js.compress')).st_size, 'compressed file must be smaller'
+      shutil.move(os.path.join(self.get_dir(), 'page.js'), 'page.js.renamedsoitcannotbefound');
+      self.run_browser('page.html', '', '/report_result?1')
+
     def test_emcc_preload_file(self):
       open(os.path.join(self.get_dir(), 'somefile.txt'), 'w').write('''load me right before running the code please''')
       open(os.path.join(self.get_dir(), 'main.cpp'), 'w').write(self.with_report_result(r'''
@@ -6291,9 +6313,6 @@ f.close()
       Popen([EMCC, os.path.join(self.get_dir(), 'sdl_image.c'), '--preload-file', 'screenshot.jpg', '-o', 'page.html']).communicate()
       self.run_browser('page.html', 'You should see |load me right before|.', '/report_result?600')
 
-    def test_emcc_compression(self):
-      pass # test compression of both the compiled code itself in a side file, and of data files
-
     def test_emcc_worker(self):
       # Test running in a web worker
       output = Popen([EMCC, path_from_root('tests', 'hello_world_worker.cpp'), '-o', 'worker.js'], stdout=PIPE, stderr=PIPE).communicate()