[WASMFS] getdents syscall

Relevant Issue: #15041

Implement getdents syscall and add required tests.

system/lib/wasmfs/file.h

@@ -49,6 +49,14 @@ public:
     return std::static_pointer_cast<T>(shared_from_this());
   }
 
+  ino_t getIno() {
+    // Set inode number to the file pointer. This gives a unique inode number.
+    // TODO: For security it would be better to use an indirect mapping.
+    // Ensure that the pointer will not overflow an ino_t.
+    static_assert(sizeof(this) <= sizeof(ino_t));
+    return (ino_t)this;
+  }
+
   class Handle {
     std::unique_lock<std::mutex> lock;
 
@@ -136,6 +144,11 @@ public:
   static constexpr FileKind expectedKind = File::DirectoryKind;
   Directory(mode_t mode) : File(File::DirectoryKind, mode) {}
 
+  struct Entry {
+    std::string name;
+    std::shared_ptr<File> file;
+  };
+
   class Handle : public File::Handle {
     std::shared_ptr<Directory> getDir() { return file->cast<Directory>(); }
 
@@ -169,11 +182,19 @@ public:
       return "";
     }
 
+    // Return a vector of the key-value pairs in entries.
+    std::vector<Directory::Entry> getEntries() {
+      std::vector<Directory::Entry> entries;
+      for (const auto& [key, value] : getDir()->entries) {
+        entries.push_back({key, value});
+      }
+      return entries;
+    }
+
 #ifdef WASMFS_DEBUG
     void printKeys() {
       for (auto keyPair : getDir()->entries) {
-        std::vector<char> temp(keyPair.first.begin(), keyPair.first.end());
-        emscripten_console_log(&temp[0]);
+        emscripten_console_log(keyPair.first.c_str());
       }
     }
 #endif

system/lib/wasmfs/streams.cpp

@@ -28,8 +28,6 @@ static __wasi_errno_t writeStdBuffer(const uint8_t* buf,
 }
 
 std::shared_ptr<StdinFile> StdinFile::getSingleton() {
-  // Give all permissions to user, group and other in stdin, stdout and stderr.
-  // This is a placeholder for now which matches the JS filesystem.
   static const std::shared_ptr<StdinFile> stdinFile =
     std::make_shared<StdinFile>(S_IRUGO);
   return stdinFile;

system/lib/wasmfs/syscalls.cpp

@@ -9,6 +9,7 @@
 #include "file.h"
 #include "file_table.h"
 #include "wasmfs.h"
+#include <dirent.h>
 #include <emscripten/emscripten.h>
 #include <emscripten/html5.h>
 #include <errno.h>
@@ -298,8 +299,9 @@ __wasi_fd_t __syscall_open(long pathname, long flags, ...) {
   }
 
   // TODO: remove assert when all functionality is complete.
-  assert(((flags) & ~(O_CREAT | O_EXCL | O_DIRECTORY | O_TRUNC | O_APPEND |
-                      O_RDWR | O_WRONLY | O_RDONLY | O_LARGEFILE)) == 0);
+  assert(
+    ((flags) & ~(O_CREAT | O_EXCL | O_DIRECTORY | O_TRUNC | O_APPEND | O_RDWR |
+                 O_WRONLY | O_RDONLY | O_LARGEFILE | O_CLOEXEC)) == 0);
 
   auto pathParts = splitPath((char*)pathname);
 
@@ -341,7 +343,6 @@ __wasi_fd_t __syscall_open(long pathname, long flags, ...) {
       mode = va_arg(vl, int);
       va_end(vl);
       // Mask all permissions sent via mode.
-      // This is a placeholder for now which matches the JS filesystem.
       mode &= S_IALLUGO;
 
       // Create an empty in-memory file.
@@ -406,7 +407,6 @@ long __syscall_mkdir(long path, long mode) {
     // Mask rwx permissions for user, group and others, and the sticky bit.
     // This prevents users from entering S_IFREG for example.
     // https://www.gnu.org/software/libc/manual/html_node/Permission-Bits.html
-    // This is a placeholder for now which matches the JS filesystem.
     mode &= S_IRWXUGO | S_ISVTX;
     // Create an empty in-memory directory.
     auto created = std::make_shared<Directory>(mode);
@@ -535,4 +535,74 @@ __wasi_errno_t __wasi_fd_fdstat_get(__wasi_fd_t fd, __wasi_fdstat_t* stat) {
   }
   return __WASI_ERRNO_SUCCESS;
 }
+
+long __syscall_getdents64(long fd, long dirp, long count) {
+  auto openFile = wasmFS.getLockedFileTable()[fd];
+
+  if (!openFile) {
+    return -EBADF;
+  }
+  dirent* result = (dirent*)dirp;
+
+  // Check if the result buffer is too small.
+  if (count / sizeof(dirent) == 0) {
+    return -EINVAL;
+  }
+
+  auto file = openFile.locked().getFile();
+
+  auto directory = file->dynCast<Directory>();
+
+  if (!directory) {
+    return -ENOTDIR;
+  }
+
+  // Hold the locked directory to prevent the state from being changed during
+  // the operation.
+  auto lockedDir = directory->locked();
+
+  off_t bytesRead = 0;
+  // A directory's position corresponds to the index in its entries vector.
+  int index = openFile.locked().position();
+
+  // In the root directory, ".." refers to itself.
+  auto dotdot =
+    file == wasmFS.getRootDirectory() ? file : lockedDir.getParent();
+
+  // If the directory is unlinked then the parent pointer should be null.
+  if (!dotdot) {
+    return -ENOENT;
+  }
+
+  // There are always two hardcoded directories "." and ".."
+  std::vector<Directory::Entry> entries = {{".", file}, {"..", dotdot}};
+  auto dirEntries = lockedDir.getEntries();
+  entries.insert(entries.end(), dirEntries.begin(), dirEntries.end());
+
+#ifdef WASMFS_DEBUG
+  for (auto pair : entries) {
+    emscripten_console_log(pair.name.c_str());
+  }
+#endif
+
+  for (; index < entries.size() && bytesRead + sizeof(dirent) <= count;
+       index++) {
+    auto curr = entries[index];
+    result->d_ino = curr.file->getIno();
+    result->d_off = bytesRead + sizeof(dirent);
+    result->d_reclen = sizeof(dirent);
+    result->d_type =
+      curr.file->is<Directory>() ? DT_DIR : DT_REG; // TODO: add symlinks.
+    // TODO: Enforce that the name can fit in the d_name field.
+    assert(curr.name.size() + 1 <= sizeof(result->d_name));
+    strcpy(result->d_name, curr.name.c_str());
+    ++result;
+    bytesRead += sizeof(dirent);
+  }
+
+  // Set the directory's offset position:
+  openFile.locked().position() = index;
+
+  return bytesRead;
+}
 }

system/lib/wasmfs/wasmfs.cpp

@@ -27,10 +27,6 @@ __attribute__((init_priority(100))) WasmFS wasmFS;
 # 28 "wasmfs.cpp"
 
 std::shared_ptr<Directory> WasmFS::initRootDirectory() {
-  // Give rwx permissions to user, group and other.
-  // Sticky bit is also set:
-  // https://www.gnu.org/software/libc/manual/html_node/Permission-Bits.html
-  // This is a placeholder for now which matches the JS filesystem.
   auto rootDirectory = std::make_shared<Directory>(S_IRUGO | S_IXUGO);
   auto devDirectory = std::make_shared<Directory>(S_IRUGO | S_IXUGO);
   rootDirectory->locked().setEntry("dev", devDirectory);

tests/test_other.py

@@ -11178,6 +11178,12 @@ void foo() {}
   def test_unistd_cwd(self):
     self.do_run_in_out_file_test('wasmfs/wasmfs_chdir.c')
 
+  def test_wasmfs_getdents(self):
+    # TODO: update this test when /dev has been filled out.
+    # Run only in WASMFS for now.
+    self.set_setting('WASMFS')
+    self.do_run_in_out_file_test('wasmfs/wasmfs_getdents.c')
+
   @disabled('Running with initial >2GB heaps is not currently supported on the CI version of Node')
   def test_hello_world_above_2gb(self):
     self.run_process([EMCC, test_file('hello_world.c'), '-sGLOBAL_BASE=2147483648', '-sINITIAL_MEMORY=3GB'])

tests/wasmfs/wasmfs_getdents.c

@@ -0,0 +1,105 @@
+/*
+ * Copyright 2021 The Emscripten Authors.  All rights reserved.
+ * Emscripten is available under two separate licenses, the MIT license and the
+ * University of Illinois/NCSA Open Source License.  Both these licenses can be
+ * found in the LICENSE file.
+ */
+
+#include <assert.h>
+#include <dirent.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/stat.h>
+#include <sys/syscall.h>
+#include <unistd.h>
+
+void print(struct dirent d, int fd) {
+  for (;;) {
+    int nread = getdents(fd, &d, sizeof(d));
+    if (nread == 0) {
+      break;
+    }
+    printf("d.d_name = %s\n", d.d_name);
+    printf("d.d_off = %lld\n", d.d_off);
+    printf("d.d_reclen = %hu\n", d.d_reclen);
+    printf("d.d_type = %s\n\n",
+           (d.d_type == DT_REG)   ? "regular"
+           : (d.d_type == DT_DIR) ? "directory"
+                                  : "???");
+  }
+}
+
+int main() {
+  // Set up test directories.
+  assert(mkdir("working", 0777) != -1);
+  assert(mkdir("/working/test", 0777) != -1);
+
+  struct dirent d;
+
+  // Try opening the directory that was just created.
+  int fd = open("/working", O_RDONLY | O_DIRECTORY);
+  printf("------------- Reading from /working Directory -------------\n");
+  print(d, fd);
+
+  // Try reading an invalid fd.
+  errno = 0;
+  getdents(-1, &d, sizeof(d));
+  printf("Errno: %s\n", strerror(errno));
+  assert(errno == EBADF);
+
+  // Try passing in a size that is too small.
+  // The JS file system doesn't catch this error.
+  // https://man7.org/linux/man-pages/man2/getdents.2.html#ERRORS
+  errno = 0;
+  getdents(fd, &d, sizeof(d) - 1);
+  printf("Errno: %s\n", strerror(errno));
+  assert(errno == EINVAL);
+
+  // Try to read from a file.
+  int fileFd = open("/dev/stdout", O_RDONLY);
+  getdents(fileFd, &d, sizeof(d));
+  printf("Errno: %s\n\n", strerror(errno));
+  assert(errno == ENOTDIR);
+
+  // Try opening the root directory and read its contents.
+  fd = open("/", O_RDONLY | O_DIRECTORY);
+
+  printf("------------- Reading from root Directory -------------\n");
+  print(d, fd);
+
+  // Try opening the dev directory and read its contents.
+  fd = open("/dev", O_RDONLY | O_DIRECTORY);
+  printf("------------- Reading from /dev Directory -------------\n");
+  print(d, fd);
+
+  // Try to advance the offset of the directory.
+  // Expect that '.' will be skipped.
+  fd = open("/working", O_RDONLY | O_DIRECTORY);
+  printf("/working file position is: %lli\n", lseek(fd, 1, SEEK_SET));
+  printf("------------- Reading from /working Directory -------------\n");
+  print(d, fd);
+
+  // Try to add a file to the /working directory.
+  assert(open("/working/foobar", O_CREAT, S_IRGRP) != -1);
+  printf("/working file position is: %lli\n", lseek(fd, 0, SEEK_SET));
+  printf("------------- Reading from /working Directory -------------\n");
+  print(d, fd);
+
+  // The musl implementation of readdir relies on getdents.
+  DIR* pDir;
+  struct dirent* pDirent;
+  pDir = opendir("/dev");
+  assert(pDir != NULL);
+
+  while ((pDirent = readdir(pDir)) != NULL) {
+    printf("pDirent->d_name: %s\n", pDirent->d_name);
+    printf("pDirent->d_off: %lld\n", pDirent->d_off);
+    printf("pDirent->d_reclen: %hu\n", pDirent->d_reclen);
+    printf("pDirent->d_type: %hhu\n\n", pDirent->d_type);
+  }
+  closedir(pDir);
+}

tests/wasmfs/wasmfs_getdents.out

@@ -0,0 +1,125 @@
+------------- Reading from /working Directory -------------
+d.d_name = .
+d.d_off = 280
+d.d_reclen = 280
+d.d_type = directory
+
+d.d_name = ..
+d.d_off = 280
+d.d_reclen = 280
+d.d_type = directory
+
+d.d_name = test
+d.d_off = 280
+d.d_reclen = 280
+d.d_type = directory
+
+Errno: Bad file descriptor
+Errno: Invalid argument
+Errno: Not a directory
+
+------------- Reading from root Directory -------------
+d.d_name = .
+d.d_off = 280
+d.d_reclen = 280
+d.d_type = directory
+
+d.d_name = ..
+d.d_off = 280
+d.d_reclen = 280
+d.d_type = directory
+
+d.d_name = dev
+d.d_off = 280
+d.d_reclen = 280
+d.d_type = directory
+
+d.d_name = working
+d.d_off = 280
+d.d_reclen = 280
+d.d_type = directory
+
+------------- Reading from /dev Directory -------------
+d.d_name = .
+d.d_off = 280
+d.d_reclen = 280
+d.d_type = directory
+
+d.d_name = ..
+d.d_off = 280
+d.d_reclen = 280
+d.d_type = directory
+
+d.d_name = stderr
+d.d_off = 280
+d.d_reclen = 280
+d.d_type = regular
+
+d.d_name = stdin
+d.d_off = 280
+d.d_reclen = 280
+d.d_type = regular
+
+d.d_name = stdout
+d.d_off = 280
+d.d_reclen = 280
+d.d_type = regular
+
+/working file position is: 1
+------------- Reading from /working Directory -------------
+d.d_name = ..
+d.d_off = 280
+d.d_reclen = 280
+d.d_type = directory
+
+d.d_name = test
+d.d_off = 280
+d.d_reclen = 280
+d.d_type = directory
+
+/working file position is: 0
+------------- Reading from /working Directory -------------
+d.d_name = .
+d.d_off = 280
+d.d_reclen = 280
+d.d_type = directory
+
+d.d_name = ..
+d.d_off = 280
+d.d_reclen = 280
+d.d_type = directory
+
+d.d_name = foobar
+d.d_off = 280
+d.d_reclen = 280
+d.d_type = regular
+
+d.d_name = test
+d.d_off = 280
+d.d_reclen = 280
+d.d_type = directory
+
+pDirent->d_name: .
+pDirent->d_off: 280
+pDirent->d_reclen: 280
+pDirent->d_type: 4
+
+pDirent->d_name: ..
+pDirent->d_off: 560
+pDirent->d_reclen: 280
+pDirent->d_type: 4
+
+pDirent->d_name: stderr
+pDirent->d_off: 840
+pDirent->d_reclen: 280
+pDirent->d_type: 8
+
+pDirent->d_name: stdin
+pDirent->d_off: 1120
+pDirent->d_reclen: 280
+pDirent->d_type: 8
+
+pDirent->d_name: stdout
+pDirent->d_off: 1400
+pDirent->d_reclen: 280
+pDirent->d_type: 8