[WASMFS] Enable preloading files (#15368)

Relevant issue: #15041

Enable loading of files when --preload-file is specified.
FS.createDataFile and FS.createPath are emulated in the new file system.
Edited browser.test_preload_file() to check preloading functionality.

emcc.py

@@ -1830,7 +1830,7 @@ def phase_linker_setup(options, state, newargs, settings_map):
     state.forced_stdlibs.append('libwasmfs')
     settings.FILESYSTEM = 0
     settings.SYSCALLS_REQUIRE_FILESYSTEM = 0
-    # settings.JS_LIBRARIES.append((0, 'library_wasmfs.js')) TODO: populate with library_wasmfs.js later
+    settings.JS_LIBRARIES.append((0, 'library_wasmfs.js'))
 
   # Explicitly drop linking in a malloc implementation if program is not using any dynamic allocation calls.
   if not settings.USES_DYNAMIC_ALLOC:

src/library_wasmfs.js

@@ -1,3 +1,63 @@
-var WasmfsLibrary = {}
+var WasmfsLibrary = {
+  $wasmFS$preloadedFiles: [],
+  $wasmFS$preloadedDirs: [],
+  $FS__deps: ['$wasmFS$preloadedFiles', '$wasmFS$preloadedDirs'],
+  $FS : {
+    // TODO: Clean up the following functions - currently copied from library_fs.js directly.
+    createPreloadedFile: function(parent, name, url, canRead, canWrite, onload, onerror, dontCreateFile, canOwn, preFinish) {
+        var fullname = name ? PATH_FS.resolve(PATH.join2(parent, name)) : parent;
+        var dep = getUniqueRunDependency('cp ' + fullname); // might have several active requests for the same fullname
+        function processData(byteArray) {
+          function finish(byteArray) {
+            if (preFinish) preFinish();
+            if (!dontCreateFile) {
+              FS.createDataFile(parent, name, byteArray, canRead, canWrite, canOwn);
+            }
+            if (onload) onload();
+            removeRunDependency(dep);
+          }
+          var handled = false;
+          Module['preloadPlugins'].forEach(function(plugin) {
+            if (handled) return;
+            if (plugin['canHandle'](fullname)) {
+              plugin['handle'](byteArray, fullname, finish, function() {
+                if (onerror) onerror();
+                removeRunDependency(dep);
+              });
+              handled = true;
+            }
+          });
+          if (!handled) finish(byteArray);
+        }
+        addRunDependency(dep);
+        if (typeof url == 'string') {
+          asyncLoad(url, function(byteArray) {
+            processData(byteArray);
+          }, onerror);
+        } else {
+          processData(url);
+        }
+      },
+    getMode: function(canRead, canWrite) {
+      var mode = 0;
+      if (canRead) mode |= {{{ cDefine('S_IRUGO') }}} | {{{ cDefine('S_IXUGO') }}};
+      if (canWrite) mode |= {{{ cDefine('S_IWUGO') }}};
+      return mode;
+    },
+    createDataFile: function(parent, name, data, canRead, canWrite, canOwn) {
+      // Data files must be cached until the file system itself has been initialized.
+      var mode = FS.getMode(canRead, canWrite);
+      wasmFS$preloadedFiles.push({pathName: parent, fileData: data, mode: mode});
+    },
+    createPath: function(parent, path, canRead, canWrite) {
+      // Cache file path directory names.
+      wasmFS$preloadedDirs.push({parentPath: parent, childName: path});
+    }
+  }
+}
 
 mergeInto(LibraryManager.library, WasmfsLibrary);
+
+if (WASMFS) {
+  DEFAULT_LIBRARY_FUNCS_TO_INCLUDE.push('$FS');
+}

src/modules.js

@@ -111,8 +111,9 @@ var LibraryManager = {
         }
         libraries.push('library_noderawfs.js');
       }
+    } else if (WASMFS) {
+      libraries.push('library_wasmfs.js');
     } 
-    // TODO: populate with libraries.push('library_wasmfs.js') later
 
     // Additional JS libraries (without AUTO_JS_LIBRARIES, link to these explicitly via -lxxx.js)
     if (AUTO_JS_LIBRARIES) {

system/lib/wasmfs/file.cpp

@@ -8,6 +8,7 @@
 
 #include "file.h"
 #include "wasmfs.h"
+#include <emscripten/threading.h>
 
 namespace wasmfs {
 //
@@ -39,6 +40,16 @@ __wasi_errno_t MemoryFile::read(uint8_t* buf, size_t len, off_t offset) {
 
   return __WASI_ERRNO_SUCCESS;
 }
+
+void MemoryFile::Handle::preloadFromJS(int index) {
+  getFile()->buffer.resize(EM_ASM_INT({return wasmFS$preloadedFiles[$0].fileData.length}, index));
+  // Ensure that files are preloaded from the main thread.
+  assert(emscripten_is_main_browser_thread());
+  // TODO: Replace every EM_ASM with EM_JS.
+  EM_ASM({ HEAPU8.set(wasmFS$preloadedFiles[$1].fileData, $0); },
+         getFile()->buffer.data(),
+         index);
+}
 //
 // Path Parsing utilities
 //

system/lib/wasmfs/file.h

@@ -160,6 +160,18 @@ class MemoryFile : public DataFile {
 
 public:
   MemoryFile(mode_t mode) : DataFile(mode) {}
+
+  class Handle : public DataFile::Handle {
+
+    std::shared_ptr<MemoryFile> getFile() { return file->cast<MemoryFile>(); }
+
+  public:
+    Handle(std::shared_ptr<File> dataFile) : DataFile::Handle(dataFile) {}
+    // This function copies preloaded files from JS Memory to Wasm Memory.
+    void preloadFromJS(int index);
+  };
+
+  Handle locked() { return Handle(shared_from_this()); }
 };
 
 // Obtains parent directory of a given pathname.

system/lib/wasmfs/syscalls.cpp

@@ -300,7 +300,7 @@ __wasi_fd_t __syscall_open(long pathname, long flags, long mode) {
     return -EINVAL;
   }
 
-  auto base = pathParts[pathParts.size() - 1];
+  auto base = pathParts.back();
 
   // Root directory
   if (pathParts.size() == 1 && pathParts[0] == "/") {
@@ -365,7 +365,7 @@ long __syscall_mkdir(long path, long mode) {
     return -EEXIST;
   }
 
-  auto base = pathParts[pathParts.size() - 1];
+  auto base = pathParts.back();
 
   long err;
   auto parentDir = getDir(pathParts.begin(), pathParts.end() - 1, err);

system/lib/wasmfs/wasmfs.cpp

@@ -7,10 +7,11 @@
 // See https://github.com/emscripten-core/emscripten/issues/15041.
 
 #include "wasmfs.h"
+#include "file.h"
 #include "streams.h"
+#include <emscripten/threading.h>
 
 namespace wasmfs {
-
 // The below lines are included to make the compiler believe that the global
 // constructor is part of a system header, which is necessary to work around a
 // compilation error about using a reserved init priority less than 101. This
@@ -21,9 +22,9 @@ namespace wasmfs {
 // system priority) since wasmFS is a system level component.
 // TODO: consider instead adding this in libc's startup code.
 // WARNING: Maintain # n + 1 "wasmfs.cpp" 3 where n = line number.
-# 25 "wasmfs.cpp" 3
+# 26 "wasmfs.cpp" 3
 __attribute__((init_priority(100))) WasmFS wasmFS;
-# 27 "wasmfs.cpp"
+# 28 "wasmfs.cpp"
 
 std::shared_ptr<Directory> WasmFS::initRootDirectory() {
   auto rootDirectory = std::make_shared<Directory>(S_IRUGO | S_IXUGO);
@@ -38,4 +39,102 @@ std::shared_ptr<Directory> WasmFS::initRootDirectory() {
 
   return rootDirectory;
 }
+
+// Initialize files specified by the --preload-file option.
+// Set up directories and files in wasmFS$preloadedDirs and
+// wasmFS$preloadedFiles from JS. This function will be called before any file
+// operation to ensure any preloaded files are eagerly available for use.
+void WasmFS::preloadFiles() {
+  // Debug builds only: add check to ensure preloadFiles() is called once.
+#ifndef NDEBUG
+  static std::atomic<int> timesCalled;
+  timesCalled++;
+  assert(timesCalled == 1);
+#endif
+
+  // Ensure that files are preloaded from the main thread.
+  assert(emscripten_is_main_browser_thread());
+
+  int numFiles = EM_ASM_INT({return wasmFS$preloadedFiles.length});
+  int numDirs = EM_ASM_INT({return wasmFS$preloadedDirs.length});
+
+  // If there are no preloaded files, exit early.
+  if (numDirs == 0 && numFiles == 0) {
+    return;
+  }
+
+  // Iterate through wasmFS$preloadedDirs to obtain a parent and child pair.
+  // Ex. Module['FS_createPath']("/foo/parent", "child", true, true);
+  for (int i = 0; i < numDirs; i++) {
+    // TODO: Convert every EM_ASM to EM_JS.
+    char parentPath[PATH_MAX] = {};
+    EM_ASM(
+      {
+        var s = wasmFS$preloadedDirs[$0].parentPath;
+        var len = lengthBytesUTF8(s) + 1;
+        stringToUTF8(s, $1, len);
+      },
+      i,
+      parentPath);
+
+    auto pathParts = splitPath(parentPath);
+
+    // TODO: Improvement - cache parent pathnames instead of looking up the
+    // directory every iteration.
+    long err;
+    auto parentDir = getDir(pathParts.begin(), pathParts.end(), err);
+
+    if (!parentDir) {
+      emscripten_console_error(
+        "Fatal error during directory creation in file preloading.");
+      abort();
+    }
+
+    char childName[PATH_MAX] = {};
+    EM_ASM(
+      {
+        var s = wasmFS$preloadedDirs[$0].childName;
+        var len = lengthBytesUTF8(s) + 1;
+        stringToUTF8(s, $1, len);
+      },
+      i,
+      childName);
+
+    auto created = std::make_shared<Directory>(S_IRUGO | S_IXUGO);
+
+    parentDir->locked().setEntry(childName, created);
+  }
+
+  for (int i = 0; i < numFiles; i++) {
+    char fileName[PATH_MAX] = {};
+    EM_ASM(
+      {
+        var s = wasmFS$preloadedFiles[$0].pathName;
+        var len = lengthBytesUTF8(s) + 1;
+        stringToUTF8(s, $1, len);
+      },
+      i,
+      fileName);
+
+    auto mode = EM_ASM_INT({ return wasmFS$preloadedFiles[$0].mode; }, i);
+
+    auto pathParts = splitPath(fileName);
+
+    auto base = pathParts.back();
+
+    auto created = std::make_shared<MemoryFile>((mode_t)mode);
+
+    long err;
+    auto parentDir = getDir(pathParts.begin(), pathParts.end() - 1, err);
+
+    if (!parentDir) {
+      emscripten_console_error("Fatal error during file preloading");
+      abort();
+    }
+
+    parentDir->locked().setEntry(base, created);
+
+    created->locked().preloadFromJS(i);
+  }
+}
 } // namespace wasmfs

system/lib/wasmfs/wasmfs.h

@@ -30,10 +30,13 @@ class WasmFS {
   // dev/stderr. Refers to the same std streams in the open file table.
   std::shared_ptr<Directory> initRootDirectory();
 
+  // Initialize files specified by --preload-file option.
+  void preloadFiles();
+
 public:
   // Files will be preloaded in this constructor.
   // This global constructor has init_priority 100. Please see wasmfs.cpp.
-  WasmFS() : rootDirectory(initRootDirectory()) {}
+  WasmFS() : rootDirectory(initRootDirectory()) { preloadFiles(); };
 
   // This get method returns a locked file table.
   // There is only ever one FileTable in the system.

tests/test_browser.py

@@ -73,6 +73,21 @@ def test_chunked_synchronous_xhr_server(support_byte_ranges, chunkSize, data, ch
     httpd.handle_request()
 
 
+def also_with_wasmfs(f):
+  def metafunc(self, wasmfs):
+    if wasmfs:
+      self.set_setting('WASMFS')
+      self.emcc_args = self.emcc_args.copy() + ['-DWASMFS']
+      f(self, wasmfs)
+    else:
+      f(self)
+
+  metafunc._parameterize = {'': (False,),
+                            'wasmfs': (True,)}
+
+  return metafunc
+
+
 def shell_with_script(shell_file, output_file, replacement):
   shell = read_file(path_from_root('src', shell_file))
   create_file(output_file, shell.replace('{{{ SCRIPT }}}', replacement))
@@ -233,7 +248,8 @@ If manually bisecting:
     self.btest_exit(test_file('emscripten_log/emscripten_log.cpp'),
                     args=['--pre-js', path_from_root('src/emscripten-source-map.min.js'), '-gsource-map'])
 
-  def test_preload_file(self):
+  @also_with_wasmfs
+  def test_preload_file(self, wasmfs=False):
     create_file('somefile.txt', 'load me right before running the code please')
     create_file('.somefile.txt', 'load me right before running the code please')
     create_file('some@file.txt', 'load me right before running the code please')
@@ -243,6 +259,9 @@ If manually bisecting:
     def make_main(path):
       print('make main at', path)
       path = path.replace('\\', '\\\\').replace('"', '\\"') # Escape tricky path name for use inside a C string.
+      # TODO: change this when wasmfs supports relative paths.
+      if wasmfs:
+        path = "/" + path
       create_file('main.cpp', r'''
         #include <assert.h>
         #include <stdio.h>
@@ -296,6 +315,10 @@ If manually bisecting:
     # As an Emscripten-specific feature, the character '@' must be escaped in the form '@@' to not confuse with the 'src@dst' notation.
     self.btest_exit('main.cpp', args=['--preload-file', tricky_filename.replace('@', '@@')])
 
+    # TODO: WASMFS doesn't support the rest of this test yet. Exit early.
+    if wasmfs:
+      return
+
     # By absolute path
 
     make_main('somefile.txt') # absolute becomes relative