Bug 945152 - Part 1: Support mapped array buffer type. r=sfink

js/src/gc/Memory.cpp

@@ -107,6 +107,21 @@ gc::GetPageFaultCount()
     return pmc.PageFaultCount;
 }
 
+void *
+gc::AllocateMappedObject(int fd, int *new_fd, size_t offset, size_t length,
+                         size_t alignment, size_t header)
+{
+    // TODO: to be implemented.
+    return nullptr;
+}
+
+// Deallocate mapped memory for object.
+void
+gc::DeallocateMappedObject(int fd, void *p, size_t length)
+{
+    // TODO: to be implemented.
+}
+
 #elif defined(SOLARIS)
 
 #include <sys/mman.h>
@@ -165,10 +180,28 @@ gc::GetPageFaultCount()
     return 0;
 }
 
+void *
+gc::AllocateMappedObject(int fd, int *new_fd, size_t offset, size_t length,
+                         size_t alignment, size_t header)
+{
+    // TODO: to be implemented.
+    return nullptr;
+}
+
+// Deallocate mapped memory for object.
+void
+gc::DeallocateMappedObject(int fd, void *p, size_t length)
+{
+    // TODO: to be implemented.
+}
+
 #elif defined(XP_UNIX)
 
+#include <algorithm>
 #include <sys/mman.h>
 #include <sys/resource.h>
+#include <sys/stat.h>
+#include <sys/types.h>
 #include <unistd.h>
 
 void
@@ -285,6 +318,90 @@ gc::GetPageFaultCount()
     return usage.ru_majflt;
 }
 
+void *
+gc::AllocateMappedObject(int fd, int *new_fd, size_t offset, size_t length,
+                         size_t alignment, size_t header)
+{
+#define NEED_PAGE_ALIGNED 0
+    size_t pa_start; // Page aligned starting
+    size_t pa_end; // Page aligned ending
+    size_t pa_size; // Total page aligned size
+    size_t page_size = sysconf(_SC_PAGESIZE); // Page size
+    bool page_for_header = false; // Do we need an additional page for header?
+    struct stat st;
+    uint8_t *buf;
+
+    // Make sure file exists and do sanity check for offset and size.
+    if (fstat(fd, &st) < 0 || offset >= (size_t) st.st_size ||
+        length == 0 || length > (size_t) st.st_size - offset)
+        return nullptr;
+
+    // Check for minimal alignment requirement.
+#if NEED_PAGE_ALIGNED
+    alignment = std::max(alignment, page_size);
+#endif
+    if (offset & (alignment - 1))
+        return nullptr;
+
+    // Page aligned starting of the offset.
+    pa_start = offset & ~(page_size - 1);
+    // Calculate page aligned ending by adding one page to the page aligned
+    // starting of data end position(offset + length - 1).
+    pa_end = ((offset + length - 1) & ~(page_size - 1)) + page_size;
+    pa_size = pa_end - pa_start;
+
+    // Do we need one more page for header?
+    if (offset - pa_start < header) {
+        page_for_header = true;
+        pa_size += page_size;
+    }
+
+    // Ask for a continuous memory location.
+    buf = (uint8_t *) MapMemory(pa_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);
+    if (buf == MAP_FAILED)
+        return nullptr;
+
+    // Duplicate a new fd for mapping, so each cloned object uses a different fd.
+    *new_fd = dup(fd);
+
+    // If there's an additional page for header, don't map that page to file.
+    if (page_for_header) {
+        buf = (uint8_t *) mmap(buf + page_size, pa_size - page_size,
+                               PROT_READ | PROT_WRITE,
+                               MAP_PRIVATE | MAP_FIXED, *new_fd, pa_start);
+    } else {
+        buf = (uint8_t *) mmap(buf, pa_size, PROT_READ | PROT_WRITE,
+                               MAP_PRIVATE | MAP_FIXED, *new_fd, pa_start);
+    }
+    if (buf == MAP_FAILED) {
+        close(*new_fd);
+        return nullptr;
+    }
+
+    // Reset the data before target file, which we don't need to see.
+    memset(buf, 0, offset - pa_start);
+
+    // Reset the data after target file, which we don't need to see.
+    memset(buf + (offset - pa_start) + length, 0, pa_end - (offset + length));
+
+    return buf + (offset - pa_start) - header;
+}
+
+void
+gc::DeallocateMappedObject(int fd, void *p, size_t length)
+{
+    void *pa_start; // Page aligned starting
+    size_t page_size = sysconf(_SC_PAGESIZE); // Page size
+    size_t total_size; // Total allocated size
+
+    // The fd is not needed anymore.
+    close(fd);
+
+    pa_start = (void *)(uintptr_t(p) & ~(page_size - 1));
+    total_size = ((uintptr_t(p) + length) & ~(page_size - 1)) + page_size - uintptr_t(pa_start);
+    munmap(pa_start, total_size);
+}
+
 #else
 #error "Memory mapping functions are not defined for your OS."
 #endif

js/src/gc/Memory.h

@@ -41,6 +41,19 @@ MarkPagesInUse(JSRuntime *rt, void *p, size_t size);
 size_t
 GetPageFaultCount();
 
+// Allocate mapped memory for object from file descriptor, offset and length
+// of the file.
+// The new_fd is duplicated from original fd, for the purpose of cloned object.
+// The offset must be aligned according to alignment requirement.
+// An additional page might be allocated depending on offset and header size given.
+void *
+AllocateMappedObject(int fd, int *new_fd, size_t offset, size_t length,
+                     size_t alignment, size_t header);
+
+// Deallocate mapped memory of the object.
+void
+DeallocateMappedObject(int fd, void *p, size_t length);
+
 } // namespace gc
 } // namespace js
 

js/src/jsapi-tests/moz.build

@@ -45,6 +45,7 @@ UNIFIED_SOURCES += [
     'testJSEvaluateScript.cpp',
     'testLookup.cpp',
     'testLooselyEqual.cpp',
+    'testMappedArrayBuffer.cpp',
     'testNewObject.cpp',
     'testNullRoot.cpp',
     'testObjectEmulatingUndefined.cpp',

js/src/jsapi-tests/testMappedArrayBuffer.cpp

@@ -0,0 +1,179 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
+ * vim: set ts=8 sts=4 et sw=4 tw=99:
+ */
+
+#ifdef XP_UNIX
+#include <fcntl.h>
+#include <stdio.h>
+#include <string.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+#include "jsfriendapi.h"
+#include "js/StructuredClone.h"
+#include "jsapi-tests/tests.h"
+#include "vm/ArrayBufferObject.h"
+
+const char test_data[] = "1234567890ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
+const char test_filename[] = "temp-bug945152_MappedArrayBuffer";
+
+BEGIN_TEST(testMappedArrayBuffer_bug945152)
+{
+    TempFile test_file;
+    FILE *test_stream = test_file.open(test_filename);
+    CHECK(fputs(test_data, test_stream) != EOF);
+    test_file.close();
+
+    // Offset 0.
+    CHECK(TestCreateObject(0, 12));
+
+    // Aligned offset.
+    CHECK(TestCreateObject(8, 12));
+
+    // Unaligned offset.
+    CHECK(CreateNewObject(11, 12) == nullptr);
+
+    // Offset + length greater than file size.
+    CHECK(CreateNewObject(8, sizeof(test_data) - 7) == nullptr);
+
+    // Release the mapped content.
+    CHECK(TestReleaseContents());
+
+#ifdef JSGC_USE_EXACT_ROOTING
+    // Ensure that fd is closed after object been GCed.
+    // Check the fd returned from object created in a function,
+    // then do the GC, in order to guarantee the object is freed when
+    // exact rooting is not on.
+    int fd = GetNewObjectFD();
+    GC(cx);
+    CHECK(!fd_is_valid(fd));
+#endif
+
+    // Neuter mapped array buffer.
+    CHECK(TestNeuterObject());
+
+    // Clone mapped array buffer.
+    CHECK(TestCloneObject());
+
+    test_file.remove();
+
+    return true;
+}
+
+JSObject *CreateNewObject(const int offset, const int length)
+{
+    int fd = open(test_filename, O_RDONLY);
+    void *ptr;
+    int new_fd;
+    if (!JS_CreateMappedArrayBufferContents(fd, &new_fd, offset, length, &ptr))
+        return nullptr;
+    JSObject *obj = JS_NewArrayBufferWithContents(cx, ptr);
+    close(fd);
+
+    return obj;
+}
+
+// Return the fd from object created in the stack.
+int GetNewObjectFD()
+{
+    JS::RootedObject obj(cx, CreateNewObject(0, 12));
+    int fd = getFD(obj);
+    CHECK(fd_is_valid(fd));
+
+    return fd;
+}
+
+bool VerifyObject(JS::HandleObject obj, const int offset, const int length)
+{
+    CHECK(obj != nullptr);
+    CHECK(JS_IsArrayBufferObject(obj));
+    CHECK_EQUAL(JS_GetArrayBufferByteLength(obj), length);
+    js::ArrayBufferObject *buf = &obj->as<js::ArrayBufferObject>();
+    CHECK(buf->isMappedArrayBuffer());
+    const char *data = reinterpret_cast<const char *>(JS_GetArrayBufferData(obj));
+    CHECK(data != nullptr);
+    CHECK(memcmp(data, test_data + offset, length) == 0);
+
+    return true;
+}
+
+bool TestCreateObject(const int offset, const int length)
+{
+    JS::RootedObject obj(cx, CreateNewObject(offset, length));
+    CHECK(VerifyObject(obj, offset, length));
+
+    return true;
+}
+
+bool TestReleaseContents()
+{
+    int fd = open(test_filename, O_RDONLY);
+    void *ptr;
+    int new_fd;
+    if (!JS_CreateMappedArrayBufferContents(fd, &new_fd, 0, 12, &ptr))
+        return false;
+    CHECK(fd_is_valid(new_fd));
+    JS_ReleaseMappedArrayBufferContents(new_fd, ptr, 12);
+    CHECK(!fd_is_valid(new_fd));
+    close(fd);
+
+    return true;
+}
+
+bool TestNeuterObject()
+{
+    JS::RootedObject obj(cx, CreateNewObject(8, 12));
+    CHECK(obj != nullptr);
+    int fd = getFD(obj);
+    CHECK(fd_is_valid(fd));
+    JS_NeuterArrayBuffer(cx, obj);
+    CHECK(isNeutered(obj));
+    CHECK(!fd_is_valid(fd));
+
+    return true;
+}
+
+bool TestCloneObject()
+{
+    JS::RootedObject obj1(cx, CreateNewObject(8, 12));
+    CHECK(obj1 != nullptr);
+    JSAutoStructuredCloneBuffer cloned_buffer;
+    JS::RootedValue v1(cx, OBJECT_TO_JSVAL(obj1));
+    const JSStructuredCloneCallbacks *callbacks = js::GetContextStructuredCloneCallbacks(cx);
+    CHECK(cloned_buffer.write(cx, v1, callbacks, nullptr));
+    JS::RootedValue v2(cx);
+    CHECK(cloned_buffer.read(cx, &v2, callbacks, nullptr));
+    JS::RootedObject obj2(cx, JSVAL_TO_OBJECT(v2));
+    CHECK(VerifyObject(obj2, 8, 12));
+
+    return true;
+}
+
+bool isNeutered(JS::HandleObject obj)
+{
+    JS::RootedValue v(cx);
+    return JS_GetProperty(cx, obj, "byteLength", &v) && v.toInt32() == 0;
+}
+
+int getFD(JS::HandleObject obj)
+{
+    CHECK(obj != nullptr);
+    js::ArrayBufferObject *buf = &obj->as<js::ArrayBufferObject>();
+    return buf->getMappingFD();
+}
+
+static bool fd_is_valid(int fd)
+{
+     return fcntl(fd, F_GETFD) != -1 || errno != EBADF;
+}
+
+static void GC(JSContext *cx)
+{
+    JS_GC(JS_GetRuntime(cx));
+    // Trigger another to wait for background finalization to end.
+    JS_GC(JS_GetRuntime(cx));
+}
+
+END_TEST(testMappedArrayBuffer_bug945152)
+#endif

js/src/jsapi.h

@@ -3161,6 +3161,26 @@ JS_AllocateArrayBufferContents(JSContext *maybecx, uint32_t nbytes, void **conte
 extern JS_PUBLIC_API(bool)
 JS_ReallocateArrayBufferContents(JSContext *cx, uint32_t nbytes, void **contents, uint8_t **data);
 
+/*
+ * Create memory mapped array buffer contents.
+ * For cloning, the fd will not be closed after mapping, and the caller must
+ * take care of closing fd after calling this function.
+ * A new duplicated fd used by the mapping is returned in new_fd.
+ */
+extern JS_PUBLIC_API(bool)
+JS_CreateMappedArrayBufferContents(int fd, int *new_fd, size_t offset,
+                                   size_t length, void **contents);
+
+/*
+ * Release the allocated resource of mapped array buffer contents before the
+ * object is created.
+ * If a new object has been created by JS_NewArrayBufferWithContents() with
+ * this content, then JS_NeuterArrayBuffer() should be used instead to release
+ * the resource used by the object.
+ */
+extern JS_PUBLIC_API(void)
+JS_ReleaseMappedArrayBufferContents(int fd, void *contents, size_t length);
+
 extern JS_PUBLIC_API(JSIdArray *)
 JS_Enumerate(JSContext *cx, JSObject *obj);
 

js/src/jsobjinlines.h

@@ -588,6 +588,8 @@ JSObject::finish(js::FreeOp *fop)
         js::ObjectElements *elements = getElementsHeader();
         if (MOZ_UNLIKELY(elements->isAsmJSArrayBuffer()))
             js::ArrayBufferObject::releaseAsmJSArrayBuffer(fop, this);
+        else if (MOZ_UNLIKELY(elements->isMappedArrayBuffer()))
+            js::ArrayBufferObject::releaseMappedArrayBuffer(fop, this);
         else
             fop->free_(elements);
     }

js/src/vm/ArrayBufferObject.cpp

@@ -30,6 +30,7 @@
 
 #include "gc/Barrier.h"
 #include "gc/Marking.h"
+#include "gc/Memory.h"
 #include "jit/AsmJS.h"
 #include "jit/AsmJSModule.h"
 #include "vm/GlobalObject.h"
@@ -478,7 +479,10 @@ ArrayBufferObject::neuter(JSContext *cx)
     JS_ASSERT(!isSharedArrayBuffer());
 
     JS_ASSERT(cx);
-    if (hasDynamicElements() && !isAsmJSArrayBuffer()) {
+    if (isMappedArrayBuffer()) {
+        releaseMappedArrayBuffer(nullptr, this);
+        setFixedElements();
+    } else if (hasDynamicElements() && !isAsmJSArrayBuffer()) {
         ObjectElements *oldHeader = getElementsHeader();
         changeContents(cx, ObjectElements::fromElements(fixedElements()));
 
@@ -630,6 +634,33 @@ ArrayBufferObject::neuterAsmJSArrayBuffer(JSContext *cx, ArrayBufferObject &buff
 #endif
 }
 
+void *
+ArrayBufferObject::createMappedArrayBuffer(int fd, int *new_fd, size_t offset, size_t length)
+{
+    void *ptr = AllocateMappedObject(fd, new_fd, offset, length, 8,
+                                     sizeof(MappingInfoHeader) + sizeof(ObjectElements));
+    if (!ptr)
+        return nullptr;
+
+    ptr = reinterpret_cast<void *>(uintptr_t(ptr) + sizeof(MappingInfoHeader));
+    ObjectElements *header = reinterpret_cast<ObjectElements *>(ptr);
+    initMappedElementsHeader(header, *new_fd, offset, length);
+
+    return ptr;
+}
+
+void
+ArrayBufferObject::releaseMappedArrayBuffer(FreeOp *fop, JSObject *obj)
+{
+    ArrayBufferObject &buffer = obj->as<ArrayBufferObject>();
+    if(!buffer.isMappedArrayBuffer() || buffer.isNeutered())
+        return;
+
+    ObjectElements *header = buffer.getElementsHeader();
+    if (header)
+        DeallocateMappedObject(buffer.getMappingFD(), header, header->initializedLength);
+}
+
 void
 ArrayBufferObject::addView(ArrayBufferViewObject *view)
 {
@@ -1359,6 +1390,21 @@ JS_StealArrayBufferContents(JSContext *cx, HandleObject objArg, void **contents,
     return true;
 }
 
+JS_PUBLIC_API(bool)
+JS_CreateMappedArrayBufferContents(int fd, int *new_fd, size_t offset,
+                                   size_t length, void **contents)
+{
+    *contents = ArrayBufferObject::createMappedArrayBuffer(fd, new_fd, offset, length);
+
+    return *contents;
+}
+
+JS_PUBLIC_API(void)
+JS_ReleaseMappedArrayBufferContents(int fd, void *contents, size_t length)
+{
+    DeallocateMappedObject(fd, contents, length);
+}
+
 JS_FRIEND_API(void *)
 JS_GetArrayBufferViewData(JSObject *obj)
 {

js/src/vm/ArrayBufferObject.h

@@ -18,6 +18,13 @@ namespace js {
 
 class ArrayBufferViewObject;
 
+// Header for mapped array buffer
+struct MappingInfoHeader
+{
+    uint32_t fd;
+    uint32_t offset;
+};
+
 // The inheritance hierarchy for the various classes relating to typed arrays
 // is as follows.
 //
@@ -150,6 +157,33 @@ class ArrayBufferObject : public JSObject
         updateElementsHeader(header, bytes);
     }
 
+    static void initMappedElementsHeader(js::ObjectElements *header, uint32_t fd,
+                                         uint32_t offset, uint32_t bytes) {
+        initElementsHeader(header, bytes);
+        header->setIsMappedArrayBuffer();
+        MappingInfoHeader *mh = getMappingInfoHeader(header);
+        mh->fd = fd;
+        mh->offset = offset;
+    }
+
+    static MappingInfoHeader *getMappingInfoHeader(js::ObjectElements *header) {
+        MOZ_ASSERT(header->isMappedArrayBuffer());
+        return reinterpret_cast<MappingInfoHeader *>(uintptr_t(header) -
+                                                     sizeof(MappingInfoHeader));
+    }
+
+    uint32_t getMappingFD() {
+        MOZ_ASSERT(getElementsHeader()->isMappedArrayBuffer());
+        MappingInfoHeader *mh = getMappingInfoHeader(getElementsHeader());
+        return mh->fd;
+    }
+
+    uint32_t getMappingOffset() const {
+        MOZ_ASSERT(getElementsHeader()->isMappedArrayBuffer());
+        MappingInfoHeader *mh = getMappingInfoHeader(getElementsHeader());
+        return mh->offset;
+    }
+
     static uint32_t headerInitializedLength(const js::ObjectElements *header) {
         return header->initializedLength;
     }
@@ -202,6 +236,15 @@ class ArrayBufferObject : public JSObject
     static bool prepareForAsmJS(JSContext *cx, Handle<ArrayBufferObject*> buffer);
     static bool neuterAsmJSArrayBuffer(JSContext *cx, ArrayBufferObject &buffer);
     static void releaseAsmJSArrayBuffer(FreeOp *fop, JSObject *obj);
+
+    bool isMappedArrayBuffer() const {
+        return getElementsHeader()->isMappedArrayBuffer();
+    }
+    void setIsMappedArrayBuffer() {
+        getElementsHeader()->setIsMappedArrayBuffer();
+    }
+    static void *createMappedArrayBuffer(int fd, int *new_fd, size_t offset, size_t length);
+    static void releaseMappedArrayBuffer(FreeOp *fop, JSObject *obj);
 };
 
 /*

js/src/vm/ObjectImpl.h

@@ -171,10 +171,11 @@ class ObjectElements
         ASMJS_ARRAY_BUFFER          = 0x2,
         NEUTERED_BUFFER             = 0x4,
         SHARED_ARRAY_BUFFER         = 0x8,
+        MAPPED_ARRAY_BUFFER         = 0x10,
 
         // Present only if these elements correspond to an array with
         // non-writable length; never present for non-arrays.
-        NONWRITABLE_ARRAY_LENGTH    = 0x10
+        NONWRITABLE_ARRAY_LENGTH    = 0x20,
     };
 
   private:
@@ -249,6 +250,12 @@ class ObjectElements
     void setIsSharedArrayBuffer() {
         flags |= SHARED_ARRAY_BUFFER;
     }
+    bool isMappedArrayBuffer() const {
+        return flags & MAPPED_ARRAY_BUFFER;
+    }
+    void setIsMappedArrayBuffer() {
+        flags |= MAPPED_ARRAY_BUFFER;
+    }
     bool hasNonwritableArrayLength() const {
         return flags & NONWRITABLE_ARRAY_LENGTH;
     }

js/src/vm/StructuredClone.cpp

@@ -66,6 +66,7 @@ enum StructuredDataType {
     SCTAG_ARRAY_OBJECT,
     SCTAG_OBJECT_OBJECT,
     SCTAG_ARRAY_BUFFER_OBJECT,
+    SCTAG_MAPPED_ARRAY_BUFFER_OBJECT,
     SCTAG_BOOLEAN_OBJECT,
     SCTAG_STRING_OBJECT,
     SCTAG_NUMBER_OBJECT,
@@ -212,6 +213,7 @@ struct JSStructuredCloneReader {
     JSString *readString(uint32_t nchars);
     bool readTypedArray(uint32_t arrayType, uint32_t nelems, js::Value *vp, bool v1Read = false);
     bool readArrayBuffer(uint32_t nbytes, js::Value *vp);
+    bool readMappedArrayBuffer(Value *vp, uint32_t fd, uint32_t offset, uint32_t length);
     bool readV1ArrayBuffer(uint32_t arrayType, uint32_t nelems, js::Value *vp);
     bool readId(jsid *idp);
     bool startRead(js::Value *vp);
@@ -831,6 +833,12 @@ bool
 JSStructuredCloneWriter::writeArrayBuffer(HandleObject obj)
 {
     ArrayBufferObject &buffer = obj->as<ArrayBufferObject>();
+
+    if (buffer.isMappedArrayBuffer()) {
+        return out.writePair(SCTAG_MAPPED_ARRAY_BUFFER_OBJECT, buffer.byteLength()) &&
+               out.writePair(buffer.getMappingFD(), buffer.getMappingOffset());
+    }
+
     return out.writePair(SCTAG_ARRAY_BUFFER_OBJECT, buffer.byteLength()) &&
            out.writeBytes(buffer.dataPointer(), buffer.byteLength());
 }
@@ -1228,6 +1236,26 @@ JSStructuredCloneReader::readArrayBuffer(uint32_t nbytes, Value *vp)
     return in.readArray(buffer.dataPointer(), nbytes);
 }
 
+bool
+JSStructuredCloneReader::readMappedArrayBuffer(Value *vp, uint32_t fd,
+                                               uint32_t offset, uint32_t length)
+{
+    void *ptr;
+    int new_fd;
+    if(!JS_CreateMappedArrayBufferContents(fd, &new_fd, offset, length, &ptr)) {
+        JS_ReportError(context(), "Failed to create mapped array buffer contents");
+        return false;
+    }
+    JSObject *obj = JS_NewArrayBufferWithContents(context(), ptr);
+    if (!obj) {
+        JS_ReleaseMappedArrayBufferContents(new_fd, ptr, length);
+        return false;
+    }
+    vp->setObject(*obj);
+
+    return true;
+}
+
 static size_t
 bytesPerTypedArrayElement(uint32_t arrayType)
 {
@@ -1412,6 +1440,14 @@ JSStructuredCloneReader::startRead(Value *vp)
             return false;
         break;
 
+      case SCTAG_MAPPED_ARRAY_BUFFER_OBJECT:
+        uint32_t fd, offset;
+        if (!in.readPair(&fd, &offset))
+            return false;
+        if (!readMappedArrayBuffer(vp, fd, offset, data))
+            return false;
+        break;
+
       case SCTAG_TYPED_ARRAY_OBJECT:
         // readTypedArray adds the array to allObjs
         uint64_t arrayType;