handle uncompressible data in lz4

src/library_lz4fs.js

@@ -39,6 +39,7 @@ mergeInto(LibraryManager.library, {
         data = new Uint8Array(data);
         console.log('compressing package of size ' + data.length);
         var compressedChunks = [];
+        var successes = [];
         var offset = 0;
         var total = 0;
         while (offset < data.length) {
@@ -48,10 +49,19 @@ mergeInto(LibraryManager.library, {
           var bound = LZ4FS.LZ4.compressBound(chunk.length);
           var compressed = new Uint8Array(bound);
           var compressedSize = LZ4FS.LZ4.compress(chunk, compressed);
-          assert(compressedSize > 0 && compressedSize <= bound);
-          compressed = compressed.subarray(0, compressedSize);
-          compressedChunks.push(compressed);
-          total += compressedSize;
+          if (compressedSize > 0) {
+            assert(compressedSize <= bound);
+            compressed = compressed.subarray(0, compressedSize);
+            compressedChunks.push(compressed);
+            total += compressedSize;
+            successes.push(1);
+          } else {
+            assert(compressedSize === 0);
+            // failure to compress :(
+            compressedChunks.push(chunk);
+            total += LZ4FS.CHUNK_SIZE;
+            successes.push(0);
+          }
         }
         data = null; // XXX null out pack['data'] too?
         var compressedData = {
@@ -61,6 +71,7 @@ mergeInto(LibraryManager.library, {
           cachedIndex: -1,
           offsets: [], // chunk# => start in compressed data
           sizes: [],
+          successes: successes, // 1 if chunk is compressed
         };
         compressedData.cachedChunk = compressedData.data.subarray(compressedData.cachedOffset);
         assert(compressedData.cachedChunk.length === LZ4FS.CHUNK_SIZE);
@@ -171,19 +182,26 @@ mergeInto(LibraryManager.library, {
           var chunkIndex = Math.floor(start / LZ4FS.CHUNK_SIZE);
           var compressedStart = compressedData.offsets[chunkIndex];
           var compressedSize = compressedData.sizes[chunkIndex];
-          if (chunkIndex !== compressedData.cachedIndex) {
-            // decompress the chunk
-            //console.log('decompressing chunk ' + chunkIndex);
-            var compressed = compressedData.data.subarray(compressedStart, compressedStart + compressedSize);
-            //var t = Date.now();
-            var originalSize = LZ4FS.LZ4.uncompress(compressed, compressedData.cachedChunk);
-            //console.log('decompress time: ' + (Date.now() - t));
-            assert(originalSize === LZ4FS.CHUNK_SIZE);
-            compressedData.cachedIndex = chunkIndex;
+          var currChunk;
+          if (compressedData.successes[chunkIndex]) {
+            if (chunkIndex !== compressedData.cachedIndex) {
+              // decompress the chunk
+              //console.log('decompressing chunk ' + chunkIndex);
+              var compressed = compressedData.data.subarray(compressedStart, compressedStart + compressedSize);
+              //var t = Date.now();
+              var originalSize = LZ4FS.LZ4.uncompress(compressed, compressedData.cachedChunk);
+              //console.log('decompress time: ' + (Date.now() - t));
+              assert(originalSize === LZ4FS.CHUNK_SIZE);
+              compressedData.cachedIndex = chunkIndex;
+            }
+            currChunk = compressedData.cachedChunk;
+          } else {
+            // uncompressed
+            currChunk = compressedData.data.subarray(compressedStart, compressedStart + LZ4FS.CHUNK_SIZE);
           }
           var startInChunk = start % LZ4FS.CHUNK_SIZE;
           var endInChunk = Math.min(startInChunk + desired, LZ4FS.CHUNK_SIZE);
-          buffer.set(compressedData.cachedChunk.subarray(startInChunk, endInChunk), offset + written);
+          buffer.set(currChunk.subarray(startInChunk, endInChunk), offset + written);
           var currWritten = endInChunk - startInChunk;
           written += currWritten;
         }

tests/fs/test_lz4fs.cpp

@@ -38,18 +38,29 @@ void EMSCRIPTEN_KEEPALIVE finish() {
       printf("%d read %d: %d failed num\n", counter, which, i);
       abort();
     }
-    buffer[5] = 0;
-    char correct[] = "01234567890123456789";
-    if (strncmp(buffer, correct + which + off, 5) != 0) {
-      printf("%d read %d: %d (%d) failed data\n", counter, which, i, i % 10);
-      abort();
+    if (which != 2) {
+      buffer[5] = 0;
+      char correct[] = "01234567890123456789";
+      if (strncmp(buffer, correct + which + off, 5) != 0) {
+        printf("%d read %d: %d (%d) failed data\n", counter, which, i, i % 10);
+        abort();
+      }
     }
     counter++;
   }
   double after = emscripten_get_now();
+
+  printf("final test on random data\n");
+  int ret = fseek(f3, 17, SEEK_SET);
+  assert(ret == 0);
+  num = fread(buffer, 1, 1, f3);
+  assert(num == 1);
+  assert(buffer[0] == 'X');
+
   fclose(f1);
   fclose(f2);
   fclose(f3);
+
   printf("success. read IO time: %f (%d reads), total time: %f\n", after - before, counter, after - before_it_all);
 
   // all done

tests/test_browser.py

@@ -1032,9 +1032,12 @@ keydown(100);keyup(100); // trigger the end
     self.btest(os.path.join('fs', 'test_workerfs_package.cpp'), '1', args=['--proxy-to-worker'])
 
   def test_fs_lz4fs_package(self):
+    import random
     open('file1.txt', 'w').write('0123456789' * (1024*128))
     open('file2.txt', 'w').write('1234567890' * (1024*128))
-    open('file3.txt', 'w').write('2345678901' * (1024*128))
+    random_data = [chr(random.randint(0,255)) for x in range(1024*128*10)]
+    random_data[17] = 'X'
+    open('file3.txt', 'w').write(''.join(random_data))
     Popen([PYTHON, FILE_PACKAGER, 'files.data', '--preload', 'file1.txt', 'file2.txt', 'file3.txt', '--separate-metadata', '--js-output=files.js']).communicate()
     self.btest(os.path.join('fs', 'test_lz4fs.cpp'), '1', args=[], timeout=60)