Initial NODERAWFS implementation

emcc.py

@@ -1080,6 +1080,7 @@ def check(input_file):
         shared.Settings.PROXY_TO_WORKER = 1
 
       if options.use_preload_plugins or len(options.preload_files) > 0 or len(options.embed_files) > 0:
+        assert not shared.Settings.NODERAWFS, '--preload-file and --embed-file cannot be used with NODERAWFS which disables virtual filesystem'
         # if we include any files, or intend to use preload plugins, then we definitely need filesystem support
         shared.Settings.FORCE_FILESYSTEM = 1
 

src/library_fs.js

@@ -8,6 +8,9 @@ mergeInto(LibraryManager.library, {
 #endif
 #if __EMSCRIPTEN_HAS_workerfs_js__
     '$WORKERFS',
+#endif
+#if __EMSCRIPTEN_HAS_noderawfs_js__
+    '$NODERAWFS',
 #endif
     'stdin', 'stdout', 'stderr'],
   $FS__postset: 'FS.staticInit();' +

src/library_nodefs.js

@@ -5,6 +5,17 @@ mergeInto(LibraryManager.library, {
     isWindows: false,
     staticInit: function() {
       NODEFS.isWindows = !!process.platform.match(/^win/);
+      var flags = process["binding"]("constants")["fs"];
+      NODEFS.flagsForNodeMap = {
+        "{{{ cDefine('O_APPEND') }}}": flags["O_APPEND"],
+        "{{{ cDefine('O_CREAT') }}}": flags["O_CREAT"],
+        "{{{ cDefine('O_EXCL') }}}": flags["O_EXCL"],
+        "{{{ cDefine('O_RDONLY') }}}": flags["O_RDONLY"],
+        "{{{ cDefine('O_RDWR') }}}": flags["O_RDWR"],
+        "{{{ cDefine('O_DSYNC') }}}": flags["O_SYNC"],
+        "{{{ cDefine('O_TRUNC') }}}": flags["O_TRUNC"],
+        "{{{ cDefine('O_WRONLY') }}}": flags["O_WRONLY"]
+      };
     },
     mount: function (mount) {
       assert(ENVIRONMENT_IS_NODE);
@@ -46,39 +57,21 @@ mergeInto(LibraryManager.library, {
     },
     // This maps the integer permission modes from http://linux.die.net/man/3/open
     // to node.js-specific file open permission strings at http://nodejs.org/api/fs.html#fs_fs_open_path_flags_mode_callback
-    flagsToPermissionStringMap: {
-      0/*O_RDONLY*/: 'r',
-      1/*O_WRONLY*/: 'r+',
-      2/*O_RDWR*/: 'r+',
-      64/*O_CREAT*/: 'r',
-      65/*O_WRONLY|O_CREAT*/: 'r+',
-      66/*O_RDWR|O_CREAT*/: 'r+',
-      129/*O_WRONLY|O_EXCL*/: 'rx+',
-      193/*O_WRONLY|O_CREAT|O_EXCL*/: 'rx+',
-      514/*O_RDWR|O_TRUNC*/: 'w+',
-      577/*O_WRONLY|O_CREAT|O_TRUNC*/: 'w',
-      578/*O_CREAT|O_RDWR|O_TRUNC*/: 'w+',
-      705/*O_WRONLY|O_CREAT|O_EXCL|O_TRUNC*/: 'wx',
-      706/*O_RDWR|O_CREAT|O_EXCL|O_TRUNC*/: 'wx+',
-      1024/*O_APPEND*/: 'a',
-      1025/*O_WRONLY|O_APPEND*/: 'a',
-      1026/*O_RDWR|O_APPEND*/: 'a+',
-      1089/*O_WRONLY|O_CREAT|O_APPEND*/: 'a',
-      1090/*O_RDWR|O_CREAT|O_APPEND*/: 'a+',
-      1153/*O_WRONLY|O_EXCL|O_APPEND*/: 'ax',
-      1154/*O_RDWR|O_EXCL|O_APPEND*/: 'ax+',
-      1217/*O_WRONLY|O_CREAT|O_EXCL|O_APPEND*/: 'ax',
-      1218/*O_RDWR|O_CREAT|O_EXCL|O_APPEND*/: 'ax+',
-      4096/*O_RDONLY|O_DSYNC*/: 'rs',
-      4098/*O_RDWR|O_DSYNC*/: 'rs+'
-    },
-    flagsToPermissionString: function(flags) {
+    flagsForNode: function(flags) {
       flags &= ~0x200000 /*O_PATH*/; // Ignore this flag from musl, otherwise node.js fails to open the file.
       flags &= ~0x800 /*O_NONBLOCK*/; // Ignore this flag from musl, otherwise node.js fails to open the file.
       flags &= ~0x8000 /*O_LARGEFILE*/; // Ignore this flag from musl, otherwise node.js fails to open the file.
       flags &= ~0x80000 /*O_CLOEXEC*/; // Some applications may pass it; it makes no sense for a single process.
-      if (flags in NODEFS.flagsToPermissionStringMap) {
-        return NODEFS.flagsToPermissionStringMap[flags];
+      var newFlags = 0;
+      for (var k in NODEFS.flagsForNodeMap) {
+        if (flags & k) {
+          newFlags |= NODEFS.flagsForNodeMap[k];
+          flags ^= k;
+        }
+      }
+
+      if (!flags) {
+        return newFlags;
       } else {
         throw new FS.ErrnoError(ERRNO_CODES.EINVAL);
       }
@@ -221,7 +214,7 @@ mergeInto(LibraryManager.library, {
         var path = NODEFS.realPath(stream.node);
         try {
           if (FS.isFile(stream.node.mode)) {
-            stream.nfd = fs.openSync(path, NODEFS.flagsToPermissionString(stream.flags));
+            stream.nfd = fs.openSync(path, NODEFS.flagsForNode(stream.flags));
           }
         } catch (e) {
           if (!e.code) throw e;

src/library_noderawfs.js

@@ -0,0 +1,126 @@
+mergeInto(LibraryManager.library, {
+  $NODERAWFS__deps: ['$ERRNO_CODES', '$FS', '$NODEFS'],
+  $NODERAWFS__postset: 'if (ENVIRONMENT_IS_NODE) {' +
+    'var _wrapNodeError = function(func) { return function() { try { return func.apply(this, arguments) } catch (e) { if (!e.code) throw e; throw new FS.ErrnoError(ERRNO_CODES[e.code]); } } };' +
+    'var VFS = Object.assign({}, FS);' +
+    'for (var _key in NODERAWFS) FS[_key] = _wrapNodeError(NODERAWFS[_key]);' +
+    '}' +
+    'else { throw new Error("NODERAWFS is currently only supported on Node.js environment.") }',
+  $NODERAWFS: {
+    lookupPath: function(path) {
+      return { path: path, node: { mode: NODEFS.getMode(path) } };
+    },
+    createStandardStreams: function() {
+      FS.streams[0] = { fd: 0, nfd: 0, position: 0, path: '', flags: 0, tty: true };
+      for (var i = 1; i < 3; i++) {
+        FS.streams[i] = { fd: i, nfd: i, position: 0, path: '', flags: 577, tty: true };
+      }
+    },
+    // generic function for all node creation
+    cwd: function() { return process.cwd(); },
+    chdir: function() { process.chdir.apply(void 0, arguments); },
+    mknod: function(path, mode) {
+      if (FS.isDir(path)) {
+        fs.mkdirSync(path, mode);
+      } else {
+        fs.writeFileSync(path, '', { mode: mode });
+      }
+    },
+    mkdir: function() { fs.mkdirSync.apply(void 0, arguments); },
+    symlink: function() { fs.symlinkSync.apply(void 0, arguments); },
+    rename: function() { fs.renameSync.apply(void 0, arguments); },
+    rmdir: function() { fs.rmdirSync.apply(void 0, arguments); },
+    readdir: function() { fs.readdirSync.apply(void 0, arguments); },
+    unlink: function() { fs.unlinkSync.apply(void 0, arguments); },
+    readlink: function() { return fs.readlinkSync.apply(void 0, arguments); },
+    stat: function() { return fs.statSync.apply(void 0, arguments); },
+    lstat: function() { return fs.lstatSync.apply(void 0, arguments); },
+    chmod: function() { fs.chmodSync.apply(void 0, arguments); },
+    fchmod: function() { fs.fchmodSync.apply(void 0, arguments); },
+    chown: function() { fs.chownSync.apply(void 0, arguments); },
+    fchown: function() { fs.fchownSync.apply(void 0, arguments); },
+    truncate: function() { fs.truncateSync.apply(void 0, arguments); },
+    ftruncate: function() { fs.ftruncateSync.apply(void 0, arguments); },
+    utime: function() { fs.utimesSync.apply(void 0, arguments); },
+    open: function(path, flags, mode, suggestFD) {
+      if (typeof flags === "number") {
+        flags = NODEFS.flagsForNode(flags)
+      }
+      var nfd = fs.openSync(path, flags, mode);
+      var fd = suggestFD != null ? suggestFD : FS.nextfd(nfd);
+      var stream = { fd: fd, nfd: nfd, position: 0, path: path, flags: flags };
+      FS.streams[fd] = stream;
+      return stream;
+    },
+    close: function(stream) {
+      if (!stream.stream_ops) {
+        // this stream is created by in-memory filesystem
+        fs.closeSync(stream.nfd);
+      }
+      FS.closeStream(stream.fd);
+    },
+    llseek: function(stream, offset, whence) {
+      if (stream.stream_ops) {
+        // this stream is created by in-memory filesystem
+        return VFS.llseek(stream, offset, whence);
+      }
+      var position = offset;
+      if (whence === 1) {  // SEEK_CUR.
+        position += stream.position;
+      } else if (whence === 2) {  // SEEK_END.
+        position += fs.fstatSync(stream.nfd).size;
+      }
+
+      if (position < 0) {
+        throw new FS.ErrnoError(ERRNO_CODES.EINVAL);
+      }
+      stream.position = position;
+      return position;
+    },
+    read: function(stream, buffer, offset, length, position) {
+      if (stream.stream_ops) {
+        // this stream is created by in-memory filesystem
+        return VFS.read(stream, buffer, offset, length, position);
+      }
+      var seeking = true;
+      if (typeof position === 'undefined') {
+        seeking = false;
+      }
+      var bytesRead = fs.readSync(stream.nfd, Buffer.from(buffer.buffer), offset, length, position);
+      if (!seeking) stream.position += bytesRead;
+      return bytesRead;
+    },
+    write: function(stream, buffer, offset, length, position) {
+      if (stream.stream_ops) {
+        // this stream is created by in-memory filesystem
+        return VFS.write(stream, buffer, offset, length, position);
+      }
+      if (stream.flags & +"{{{ cDefine('O_APPEND') }}}") {
+        // seek to the end before writing in append mode
+        position = FS.llseek(stream, 0, +"{{{ cDefine('SEEK_END') }}}");
+      }
+      var seeking = true;
+      if (typeof position === 'undefined') {
+        seeking = false;
+      }
+      var bytesWritten = fs.writeSync(stream.nfd, Buffer.from(buffer.buffer), offset, length, position);
+      if (!seeking) stream.position += bytesWritten;
+      return bytesWritten;
+    },
+    allocate: function() {
+      throw new FS.ErrnoError(ERRNO_CODES.EOPNOTSUPP);
+    },
+    mmap: function() {
+      throw new FS.ErrnoError(ERRNO_CODES.ENODEV);
+    },
+    msync: function() {
+      return 0;
+    },
+    munmap: function() {
+      return 0;
+    },
+    ioctl: function() {
+      throw new FS.ErrnoError(ERRNO_CODES.ENOTTY);
+    }
+  }
+});

src/modules.js

@@ -126,6 +126,10 @@ var LibraryManager = {
           'library_workerfs.js',
           'library_lz4.js',
         ]);
+
+        if (NODERAWFS) {
+          libraries.push('library_noderawfs.js')
+        }
       }
     }
 

src/settings.js

@@ -382,6 +382,9 @@ var NO_FILESYSTEM = 0; // If set, does not build in any filesystem support. Usef
 var FORCE_FILESYSTEM = 0; // Makes full filesystem support be included, even if statically it looks like it is not
                           // used. For example, if your C code uses no files, but you include some JS that does,
                           // you might need this.
+var NODERAWFS = 0; // This mode is intended for use with Node.js (and will throw if the build runs in other engines)
+                   // The File System API will directly use Node.js API without requiring `FS.mount()`.
+                   // The initial working directory will be same as process.cwd() instead of VFS root directory.
 
 var EXPORTED_FUNCTIONS = ['_main'];
                                     // Functions that are explicitly exported. These functions are kept alive

src/struct_info.json

@@ -457,6 +457,7 @@
         "defines": [
             "O_CREAT", 
             "O_SYNC", 
+            "O_DSYNC",
             "F_GETFD", 
             "F_SETFL", 
             "O_NOFOLLOW", 

tests/fs/test_nodefs_cloexec.c

@@ -4,14 +4,24 @@
 #include <fcntl.h>
 #include <emscripten.h>
 
+#ifdef NODERAWFS
+#define CWD ""
+#else
+#define CWD "/working/"
+#endif
+
 int main(void)
 {
     EM_ASM(
+#ifdef NODERAWFS
+        FS.close(FS.open('test.txt', 'w'));
+#else
         FS.mkdir('/working');
         FS.mount(NODEFS, {root: '.'}, '/working');
         FS.close(FS.open('/working/test.txt', 'w'));
+#endif
     );
-    assert(open("/working/test.txt", O_RDONLY | O_CLOEXEC) != -1);
+    assert(open(CWD "test.txt", O_RDONLY | O_CLOEXEC) != -1);
     printf("success\n");
     return 0;
 }

tests/fs/test_nodefs_rw.c

@@ -3,6 +3,12 @@
 #include <string.h>
 #include <emscripten.h>
 
+#ifdef NODERAWFS
+#define CWD ""
+#else
+#define CWD "/working/"
+#endif
+
 int main() {
   FILE *file;
   int res;
@@ -14,15 +20,17 @@ int main() {
     fs.writeFileSync('foobar.txt', 'yeehaw');
   );
 
+#ifndef NODERAWFS
   // mount the current folder as a NODEFS instance
   // inside of emscripten
   EM_ASM(
     FS.mkdir('/working');
     FS.mount(NODEFS, { root: '.' }, '/working');
   );
+#endif
 
   // read and validate the contents of the file
-  file = fopen("/working/foobar.txt", "r");
+  file = fopen(CWD "foobar.txt", "r");
   assert(file);
   res = fread(buffer, sizeof(char), 6, file);
   assert(res == 6);
@@ -31,7 +39,7 @@ int main() {
   assert(!strcmp(buffer, "yeehaw"));
 
   // write out something new
-  file = fopen("/working/foobar.txt", "w");
+  file = fopen(CWD "foobar.txt", "w");
   assert(file);
   res = fwrite("cheez", sizeof(char), 5, file);
   assert(res == 5);