add file preloading in browsers

emcc

@@ -198,11 +198,16 @@ Options that are modified or new in %s include:
                               will be run). Note that this by itself
                               will not minify the code (closure does
                               that)
-  --embed-file <filename>  A file to embed inside the generated
+  --embed-file <file>      A file to embed inside the generated
                            JavaScript. The compiled code will be able
                            to access the file in the current directory
                            with the same basename as given here (that is,
                            just the filename, without a path to it).
+  --preload-file <name>    A file to preload before running the
+                           compiled code asynchronously. Otherwise
+                           similar to --embed-file, except that this
+                           option is only relevant when generating
+                           HTML (it uses asynchronous binary XHRs).
   --ignore-dynamic-linking Normally emcc will treat dynamic linking like
                            static linking, by linking in the code from
                            the dynamic library. This fails if the same
@@ -347,6 +352,7 @@ try:
   post_js = None
   compress_whitespace = None
   embed_files = []
+  preload_files = []
   ignore_dynamic_linking = False
   shell_path = shared.path_from_root('src', 'shell.html')
 
@@ -396,6 +402,11 @@ try:
       embed_files.append(newargs[i+1])
       newargs[i] = ''
       newargs[i+1] = ''
+    elif newargs[i].startswith('--preload-file'):
+      check_bad_eq(newargs[i])
+      preload_files.append(newargs[i+1])
+      newargs[i] = ''
+      newargs[i+1] = ''
     elif newargs[i] == '-MF': # clang cannot handle this, so we fake it
       f = open(newargs[i+1], 'w')
       f.write('\n')
@@ -728,16 +739,35 @@ try:
   final = shared.Building.emscripten(final, append_ext=False)
   if DEBUG: save_intermediate('original')
 
-  # Embed files
+  # Embed and preload files
-  if len(embed_files) > 0:
+  if len(embed_files) + len(preload_files) > 0:
-    if DEBUG: print >> sys.stderr, 'emcc: embedding files'
+    if DEBUG: print >> sys.stderr, 'emcc: setting up files'
-    src = open(final).read().replace(
+    code = ''
-      '// {{PRE_RUN_ADDITIONS}}',
+    for filename in embed_files:
-      '\n'.join(map(lambda embed_file: "FS.createDataFile('/', '%s', %s, true, false);" % (os.path.basename(embed_file), str(map(ord, open(embed_file, 'rb').read()))), embed_files))
+      code += '''FS.createDataFile('/', '%s', %s, true, true);\n''' % (os.path.basename(filename), str(map(ord, open(filename, 'rb').read())))
-    )
+    counter = 0
-    final += '.ef.js'
+    for filename in preload_files:
+      name = 'filePreload%d' % counter
+      counter += 1
+      code += '''
+    var %(name)s = new XMLHttpRequest();
+    %(name)s.open("GET", "%(filename)s", true);
+    %(name)s.responseType = 'arraybuffer';
+    addRunDependency();
+    %(name)s.onload = function (oEvent) {
+      var arrayBuffer = %(name)s.response; // Note: not X.responseText
+      assert(arrayBuffer, 'Loading file %(filename)s failed.');
+      var byteArray = new Uint8Array(arrayBuffer);
+      FS.createDataFile('/', '%(filename)s', byteArray, true, true);
+      removeRunDependency();
+    };
+    %(name)s.send(null);
+''' % { 'name': name, 'filename': filename }
+
+    src = open(final).read().replace('// {{PRE_RUN_ADDITIONS}}', code)
+    final += '.files.js'
     open(final, 'w').write(src)
-    if DEBUG: save_intermediate('embedded_files')
+    if DEBUG: save_intermediate('files')
 
   # Apply pre and postjs files
   if pre_js or post_js:

src/library.js

@@ -255,23 +255,7 @@ LibraryManager.library = {
       var success = true;
       if (typeof XMLHttpRequest !== 'undefined') {
         // Browser.
-        // TODO: Use mozResponseArrayBuffer, responseStream, etc. if available.
+        assert('Cannot do synchronous binary XHRs in modern browsers. Use --embed-file or --preload-file in emcc');
-        var xhr = new XMLHttpRequest();
-        xhr.open('GET', obj.url, false);
-
-        // Some hints to the browser that we want binary data.
-        if (typeof Uint8Array != 'undefined') xhr.responseType = 'arraybuffer';
-        if (xhr.overrideMimeType) {
-          xhr.overrideMimeType('text/plain; charset=x-user-defined');
-        }
-
-        xhr.send(null);
-        if (xhr.status != 200 && xhr.status != 0) success = false;
-        if (xhr.response !== undefined) {
-          obj.contents = new Uint8Array(xhr.response || []);
-        } else {
-          obj.contents = intArrayFromString(xhr.responseText || '', true);
-        }
       } else if (typeof read !== 'undefined') {
         // Command-line.
         try {

src/postamble.js

@@ -53,10 +53,10 @@ if (Module['preRun']) {
 
 #if INVOKE_RUN
 #else
-Module['noInitialRun'] = true;
+addRunDependency();
 #endif
 
-if (!Module['noInitialRun']) {
+if (runDependencies == 0) {
   var ret = run();
 #if CATCH_EXIT_CODE
   print('Exit Status: ' + ret);

src/preamble.js

@@ -873,5 +873,18 @@ var STRING_TABLE = [];
 {{{ unSign }}}
 {{{ reSign }}}
 
+// A counter of dependencies for calling run(). If we need to
+// do asynchronous work before running, increment this and
+// decrement it. Incrementing must happen in Module.preRun
+// or PRE_RUN_ADDITIONS (used by emcc to add file preloading).
+var runDependencies = 0;
+function addRunDependency() {
+  runDependencies++;
+}
+function removeRunDependency() {
+  runDependencies--;
+  if (runDependencies == 0) run();
+}
+
 // === Body ===
 

tests/runner.py

@@ -6175,27 +6175,24 @@ f.close()
     def run_browser(self, html_file, message, expectedResult=None):
       if expectedResult is not None:
         try:
-          try:
+          def server_func(q):
-            def server_func(q):
+            class TestServerHandler(BaseHTTPServer.BaseHTTPRequestHandler):
-              class TestServerHandler(BaseHTTPServer.BaseHTTPRequestHandler):
+              def do_GET(s):
-                def do_GET(s):
+                q.put(s.path)
-                  q.put(s.path)
+            httpd = BaseHTTPServer.HTTPServer(('localhost', 8888), TestServerHandler)
-              httpd = BaseHTTPServer.HTTPServer(('localhost', 8888), TestServerHandler)
+            httpd.serve_forever() # test runner will kill us
-              httpd.serve_forever() # test runner will kill us
+          queue = multiprocessing.Queue()
-            queue = multiprocessing.Queue()
+          server = multiprocessing.Process(target=server_func, args=(queue,))
-            server = multiprocessing.Process(target=server_func, args=(queue,))
+          server.start()
-            server.start()
+          webbrowser.open_new(os.path.abspath(html_file))
-            webbrowser.open_new(os.path.abspath(html_file))
+          output = '[no http server activity]'
-            output = '[no http server activity]'
+          start = time.time()
-            start = time.time()
+          while time.time() - start < 5:
-            while time.time() - start < 5:
+            if not queue.empty():
-              if not queue.empty():
+              output = queue.get()
-                output = queue.get()
+              break
-                break
+            time.sleep(0.1)
-              time.sleep(0.1)
+          self.assertIdentical(expectedResult, output)
-            self.assertIdentical(expectedResult, output)
-          except Exception, e:
-            print e
         finally:
           server.terminate()
       else:
@@ -6213,7 +6210,7 @@ f.close()
       assert os.path.exists('something.html'), output
       self.run_browser('something.html', 'You should see "hello, world!" and a colored cube.')
 
-    def zzztest_emcc_preload_file(self):
+    def test_emcc_preload_file(self):
       open(os.path.join(self.get_dir(), 'somefile.txt'), 'w').write('''load me right before running the code please''')
       open(os.path.join(self.get_dir(), 'main.cpp'), 'w').write(r'''
         #include <stdio.h>

tools/shared.py

@@ -226,7 +226,7 @@ def check_engine(engine):
     print 'Checking JS engine %s failed. Check %s. Details: %s' % (str(engine), EM_CONFIG, str(e))
     return False
 
-def timeout_run(proc, timeout, note):
+def timeout_run(proc, timeout, note='unnamed process'):
   start = time.time()
   if timeout is not None:
     while time.time() - start < timeout and proc.poll() is None: