add tests for chunked binary web worker xhr

src/library.js

@@ -318,14 +318,18 @@ LibraryManager.library = {
         xhr.send(null);
         if (!(xhr.status >= 200 && xhr.status < 300 || xhr.status === 304)) throw new Error("Couldn't load " + url + ". Status: " + xhr.status);
         var datalength = Number(xhr.getResponseHeader("Content-length"));
-	var header;
+        var header;
         var hasByteServing = (header = xhr.getResponseHeader("Accept-Ranges")) && header === "bytes";
+#if SMALL_CHUNKS
+        var chunkSize = 1024; // Chunk size in bytes
+#else
         var chunkSize = 1024*1024; // Chunk size in bytes
-	if (!hasByteServing) chunkSize = datalength;
+#endif
+        if (!hasByteServing) chunkSize = datalength;
   
         // Function to get a range from the remote URL.
         var doXHR = (function(from, to) {
-          if (from >= to) throw new Error("invalid range or no bytes requested!");
+          if (from > to) throw new Error("invalid range (" + from + ", " + to + ") or no bytes requested!");
           if (to > datalength-1) throw new Error("only " + datalength + " bytes available! programmer error!");
   
           // TODO: Use mozResponseArrayBuffer, responseStream, etc. if available.

src/shell.js

@@ -66,7 +66,7 @@ if (ENVIRONMENT_IS_SHELL) {
   }
 }
 
-if (ENVIRONMENT_IS_WEB) {
+if (ENVIRONMENT_IS_WEB && !ENVIRONMENT_IS_WORKER) {
   if (!Module['print']) {
     Module['print'] = function(x) {
       console.log(x);

tests/checksummer.c

@@ -0,0 +1,71 @@
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+const int MOD_ADLER = 65521;
+ 
+uint64_t adler32(unsigned char *data, int32_t len) /* where data is the location of the data in physical memory and 
+                                                       len is the length of the data in bytes */
+{
+    uint64_t a = 1, b = 0;
+    int32_t index;
+ 
+    /* Process each byte of the data in order */
+    for (index = 0; index < len; ++index)
+    {
+        a = (a + data[index]) % MOD_ADLER;
+        b = (b + a) % MOD_ADLER;
+    }
+ 
+    return (b << 16) | a;
+}
+
+int main(int argc, char* argv[]) {
+    long bufsize;
+
+    if (argc != 2) {
+	fputs("Need 1 argument\n", stderr);
+	return (EXIT_FAILURE);
+    }
+
+    unsigned char *source = NULL;
+    FILE *fp = fopen(argv[1], "rb");
+    if (fp != NULL) {
+        /* Go to the end of the file. */
+        if (fseek(fp, 0L, SEEK_END) == 0) {
+            /* Get the size of the file. */
+            bufsize = ftell(fp);
+            if (bufsize == -1) { fputs("Couldn't get size\n", stderr); return (EXIT_FAILURE); }
+    
+            /* Allocate our buffer to that size. */
+            source = malloc(sizeof(char) * (bufsize + 1));
+            if (source == NULL) { fputs("Couldn't allocate\n", stderr); return (EXIT_FAILURE); }
+    
+            /* Go back to the start of the file. */
+            if (fseek(fp, 0L, SEEK_SET) == -1) { fputs("Couldn't seek\n", stderr); return (EXIT_FAILURE); }
+    
+            /* Read the entire file into memory. */
+            size_t newLen = fread(source, sizeof(char), bufsize, fp);
+            if (newLen == 0) {
+                fputs("Error reading file\n", stderr);
+                //return (EXIT_FAILURE);
+            } else {
+                source[++newLen] = '\0'; /* Just to be safe. */
+            }
+        } else {
+          fputs("Couldn't seek to end\n", stderr);
+          return (EXIT_FAILURE);
+        }
+        fclose(fp);
+    } else {
+      fputs("Couldn't open\n", stderr);
+      return (EXIT_FAILURE);
+    }
+
+    printf("%d\n", (uint32_t) adler32(source, bufsize));
+    
+    free(source); /* Don't forget to call free() later! */
+
+    return (EXIT_SUCCESS);
+
+}

tests/runner.py

@@ -8424,6 +8424,114 @@ elif 'browser' in str(sys.argv):
       html_file.close()
       self.run_browser('main.html', 'You should see that the worker was called, and said "hello from worker!"', '/report_result?hello%20from%20worker!')
 
+    def test_chunked_synchronous_xhr(self):
+      main = 'chunked_sync_xhr.html'
+      worker_filename = "download_and_checksum_worker.js"
+
+      html_file = open(main, 'w')
+      html_file.write(r"""
+        <!doctype html>
+        <html>
+        <head><meta charset="utf-8"><title>Chunked XHR</title></head>
+        <html>
+        <body>
+          Chunked XHR Web Worker Test
+          <script>
+            var worker = new Worker(""" + json.dumps(worker_filename) + r""");
+            var buffer = [];
+            worker.onmessage = function(event) {
+              if (event.data.channel === "stdout") {
+                var xhr = new XMLHttpRequest();
+                xhr.open('GET', 'http://localhost:8888/report_result?' + event.data.line);
+                xhr.send();
+                setTimeout(function() { window.close() }, 1000);
+              } else {
+                if (event.data.trace) event.data.trace.split("\n").map(function(v) { console.error(v); });
+                if (event.data.line) {
+                  console.error(event.data.line);
+                } else {
+                  var v = event.data.char;
+                  if (v == 10) {
+                    var line = buffer.splice(0);
+                    console.error(line = line.map(function(charCode){return String.fromCharCode(charCode);}).join(''));
+                  } else {
+                    buffer.push(v);
+                  }
+                }
+              }
+            };
+          </script>
+        </body>
+        </html>
+      """)
+      html_file.close()
+
+      c_source_filename = "checksummer.c"
+
+      prejs_filename = "worker_prejs.js"
+      prejs_file = open(prejs_filename, 'w')
+      prejs_file.write(r"""
+        if (typeof(Module) === "undefined") Module = {};
+        Module["arguments"] = ["/bigfile"];
+        Module["preInit"] = function() {
+            FS.createLazyFile('/', "bigfile", "http://localhost:11111/bogus_file_path", true, false);
+        };
+        var doTrace = true;
+        Module["print"] =    function(s) { self.postMessage({channel: "stdout", line: s}); };
+        Module["stderr"] =   function(s) { self.postMessage({channel: "stderr", char: s, trace: ((doTrace && s === 10) ? new Error().stack : null)}); doTrace = false; };
+      """)
+      prejs_file.close()
+      # vs. os.path.join(self.get_dir(), filename)
+      # vs. path_from_root('tests', 'hello_world_gles.c')
+      Popen(['python', EMCC, path_from_root('tests', c_source_filename), '-g', '-s', 'SMALL_CHUNKS=1', '-o', worker_filename,
+                                           '--pre-js', prejs_filename]).communicate()
+
+      chunkSize = 1024
+      data = os.urandom(10*chunkSize+1) # 10 full chunks and one 1 byte chunk
+      expectedConns = 11
+      import zlib
+      checksum = zlib.adler32(data)
+
+      def chunked_server(support_byte_ranges):
+        class ChunkedServerHandler(BaseHTTPServer.BaseHTTPRequestHandler):
+          @staticmethod
+          def sendheaders(s, extra=[], length=len(data)):
+            s.send_response(200)
+            s.send_header("Content-Length", str(length))
+            s.send_header("Access-Control-Allow-Origin", "http://localhost:8888")
+            s.send_header("Access-Control-Expose-Headers", "Content-Length, Accept-Ranges")
+            s.send_header("Content-type", "application/octet-stream")
+            if support_byte_ranges:
+              s.send_header("Accept-Ranges", "bytes")
+            for i in extra:
+              s.send_header(i[0], i[1])
+            s.end_headers()
+
+          def do_HEAD(s):
+            ChunkedServerHandler.sendheaders(s)
+            
+          def do_GET(s):
+            if not support_byte_ranges:
+              ChunkedServerHandler.sendheaders(s)
+              s.wfile.write(data)
+            else:
+              (start, end) = s.headers.get("range").split("=")[1].split("-")
+              start = int(start)
+              end = int(end)
+              end = min(len(data)-1, end)
+              length = end-start+1
+              ChunkedServerHandler.sendheaders(s,[],length)
+              s.wfile.write(data[start:end+1])
+            s.wfile.close()
+        httpd = BaseHTTPServer.HTTPServer(('localhost', 11111), ChunkedServerHandler)
+        for i in range(expectedConns+1):
+          httpd.handle_request()
+
+      server = multiprocessing.Process(target=chunked_server, args=(True,))
+      server.start()
+      self.run_browser(main, 'Chunked binary synchronous XHR in Web Workers!', '/report_result?' + str(checksum))
+      server.terminate()
+
     def test_glgears(self):
       self.reftest(path_from_root('tests', 'gears.png'))
       Popen(['python', EMCC, path_from_root('tests', 'hello_world_gles.c'), '-o', 'something.html',