Bug 1264053 - Transfer DifferentProcess ArrayBuffers by copying, r=jorendorff

--HG--
extra : rebase_source : aed39bb2f92888af7626fd4c37df366cb1761bb8
extra : histedit_source : 4e231e7ef1b0b21d0c4bff2ebaa611e8b321e6d4

js/src/builtin/TestingFunctions.cpp

@@ -2373,12 +2373,31 @@ const JSPropertySpec CloneBufferObject::props_[] = {
     JS_PS_END
 };
 
+static mozilla::Maybe<JS::StructuredCloneScope>
+ParseCloneScope(JSContext* cx, HandleString str)
+{
+    mozilla::Maybe<JS::StructuredCloneScope> scope;
+
+    JSAutoByteString scopeStr(cx, str);
+    if (!scopeStr)
+        return scope;
+
+    if (strcmp(scopeStr.ptr(), "SameProcessSameThread") == 0)
+        scope.emplace(JS::StructuredCloneScope::SameProcessSameThread);
+    else if (strcmp(scopeStr.ptr(), "SameProcessDifferentThread") == 0)
+        scope.emplace(JS::StructuredCloneScope::SameProcessDifferentThread);
+    else if (strcmp(scopeStr.ptr(), "DifferentProcess") == 0)
+        scope.emplace(JS::StructuredCloneScope::DifferentProcess);
+
+    return scope;
+}
+
 static bool
 Serialize(JSContext* cx, unsigned argc, Value* vp)
 {
     CallArgs args = CallArgsFromVp(argc, vp);
 
-    JSAutoStructuredCloneBuffer clonebuf(JS::StructuredCloneScope::SameProcessSameThread, nullptr, nullptr);
+    mozilla::Maybe<JSAutoStructuredCloneBuffer> clonebuf;
     JS::CloneDataPolicy policy;
 
     if (!args.get(2).isUndefined()) {
@@ -2407,12 +2426,30 @@ Serialize(JSContext* cx, unsigned argc, Value* vp)
                 return false;
             }
         }
-    }
 
-    if (!clonebuf.write(cx, args.get(0), args.get(1), policy))
+        if (!JS_GetProperty(cx, opts, "scope", &v))
             return false;
 
-    RootedObject obj(cx, CloneBufferObject::Create(cx, &clonebuf));
+        if (!v.isUndefined()) {
+            RootedString str(cx, JS::ToString(cx, v));
+            if (!str)
+                return false;
+            auto scope = ParseCloneScope(cx, str);
+            if (!scope) {
+                JS_ReportErrorASCII(cx, "Invalid structured clone scope");
+                return false;
+            }
+            clonebuf.emplace(*scope, nullptr, nullptr);
+        }
+    }
+
+    if (!clonebuf)
+        clonebuf.emplace(JS::StructuredCloneScope::SameProcessSameThread, nullptr, nullptr);
+
+    if (!clonebuf->write(cx, args.get(0), args.get(1), policy))
+        return false;
+
+    RootedObject obj(cx, CloneBufferObject::Create(cx, clonebuf.ptr()));
     if (!obj)
         return false;
 
@@ -2425,14 +2462,33 @@ Deserialize(JSContext* cx, unsigned argc, Value* vp)
 {
     CallArgs args = CallArgsFromVp(argc, vp);
 
-    if (args.length() != 1 || !args[0].isObject()) {
+    if (!args.get(0).isObject() || !args[0].toObject().is<CloneBufferObject>()) {
-        JS_ReportErrorASCII(cx, "deserialize requires a single clonebuffer argument");
+        JS_ReportErrorASCII(cx, "deserialize requires a clonebuffer argument");
         return false;
     }
 
-    if (!args[0].toObject().is<CloneBufferObject>()) {
+    JS::StructuredCloneScope scope = JS::StructuredCloneScope::SameProcessSameThread;
-        JS_ReportErrorASCII(cx, "deserialize requires a clonebuffer");
+    if (args.get(1).isObject()) {
+        RootedObject opts(cx, &args[1].toObject());
+        if (!opts)
             return false;
+
+        RootedValue v(cx);
+        if (!JS_GetProperty(cx, opts, "scope", &v))
+            return false;
+
+        if (!v.isUndefined()) {
+            RootedString str(cx, JS::ToString(cx, v));
+            if (!str)
+                return false;
+            auto maybeScope = ParseCloneScope(cx, str);
+            if (!maybeScope) {
+                JS_ReportErrorASCII(cx, "Invalid structured clone scope");
+                return false;
+            }
+
+            scope = *maybeScope;
+        }
     }
 
     Rooted<CloneBufferObject*> obj(cx, &args[0].toObject().as<CloneBufferObject>());
@@ -2451,8 +2507,9 @@ Deserialize(JSContext* cx, unsigned argc, Value* vp)
     RootedValue deserialized(cx);
     if (!JS_ReadStructuredClone(cx, *obj->data(),
                                 JS_STRUCTURED_CLONE_VERSION,
-                                JS::StructuredCloneScope::SameProcessSameThread,
+                                scope,
-                                &deserialized, nullptr, nullptr)) {
+                                &deserialized, nullptr, nullptr))
+    {
         return false;
     }
     args.rval().set(deserialized);
@@ -4495,15 +4552,22 @@ gc::ZealModeHelpText),
     JS_FN_HELP("serialize", Serialize, 1, 0,
 "serialize(data, [transferables, [policy]])",
 "  Serialize 'data' using JS_WriteStructuredClone. Returns a structured\n"
-"  clone buffer object. 'policy' must be an object. The following keys'\n"
+"  clone buffer object. 'policy' may be an options hash. Valid keys:\n"
-"  string values will be used to determine whether the corresponding types\n"
+"    'SharedArrayBuffer' - either 'allow' (the default) or 'deny'\n"
-"  may be serialized (value 'allow', the default) or not (value 'deny').\n"
+"    to specify whether SharedArrayBuffers may be serialized.\n"
-"  If denied types are encountered a TypeError will be thrown during cloning.\n"
+"\n"
-"  Valid keys: 'SharedArrayBuffer'."),
+"    'scope' - SameProcessSameThread, SameProcessDifferentThread, or\n"
+"    DifferentProcess. Determines how some values will be serialized.\n"
+"    Clone buffers may only be deserialized with a compatible scope."),
 
     JS_FN_HELP("deserialize", Deserialize, 1, 0,
-"deserialize(clonebuffer)",
+"deserialize(clonebuffer[, opts])",
-"  Deserialize data generated by serialize."),
+"  Deserialize data generated by serialize. 'opts' is an options hash with one\n"
+"  recognized key 'scope', which limits the clone buffers that are considered\n"
+"  valid. Allowed values: 'SameProcessSameThread', 'SameProcessDifferentThread',\n"
+"  and 'DifferentProcess'. So for example, a DifferentProcess clone buffer\n"
+"  may be deserialized in any scope, but a SameProcessSameThread clone buffer\n"
+"  cannot be deserialized in a DifferentProcess scope."),
 
     JS_FN_HELP("detachArrayBuffer", DetachArrayBuffer, 1, 0,
 "detachArrayBuffer(buffer)",

js/src/tests/js1_8_5/extensions/clone-errors.js

@@ -22,4 +22,20 @@ check(new Proxy({}, {}));
 // A failing getter.
 check({get x() { throw new Error("fail"); }});
 
+// Mismatched scopes.
+for (let [write_scope, read_scope] of [['SameProcessSameThread', 'SameProcessDifferentThread'],
+                                       ['SameProcessSameThread', 'DifferentProcess'],
+                                       ['SameProcessDifferentThread', 'DifferentProcess']])
+{
+  var ab = new ArrayBuffer(12);
+  var buffer = serialize(ab, [ab], { scope: write_scope });
+  var caught = false;
+  try {
+    deserialize(buffer, { scope: read_scope });
+  } catch (exc) {
+    caught = true;
+  }
+  assertEq(caught, true, `${write_scope} clone buffer should not be deserializable as ${read_scope}`);
+}
+
 reportCompare(0, 0, "ok");

js/src/tests/js1_8_5/extensions/clone-transferables.js

@@ -2,13 +2,22 @@
 // Any copyright is dedicated to the Public Domain.
 // http://creativecommons.org/licenses/publicdomain/
 
-function test() {
+function* buffer_options() {
-    for (var size of [0, 8, 16, 200, 1000, 4096, -8, -200, -8192, -65536]) {
+  for (var scope of ["SameProcessSameThread", "SameProcessDifferentThread", "DifferentProcess"]) {
-        size = Math.abs(size);
+    for (var size of [0, 8, 16, 200, 1000, 4096, 8192, 65536]) {
+      yield { scope, size };
+    }
+  }
+}
 
+
+function test() {
+    for (var {scope, size} of buffer_options()) {
         var old = new ArrayBuffer(size);
-        var copy = deserialize(serialize(old, [old]));
+        var copy = deserialize(serialize([old, old], [old], { scope }), { scope });
         assertEq(old.byteLength, 0);
+        assertEq(copy[0] === copy[1], true);
+        copy = copy[0];
         assertEq(copy.byteLength, size);
 
         var constructors = [ Int8Array,
@@ -32,7 +41,7 @@ function test() {
             if (!dataview)
                 assertEq(old_arr.length, size / old_arr.BYTES_PER_ELEMENT);
 
-            var copy_arr = deserialize(serialize(old_arr, [ buf ]));
+            var copy_arr = deserialize(serialize(old_arr, [ buf ], { scope }), { scope });
             assertEq(buf.byteLength, 0,
                      "donor array buffer should be detached");
             if (!dataview) {
@@ -54,7 +63,7 @@ function test() {
             var buf = new ArrayBuffer(size);
             var old_arr = new ctor(buf);
             var dv = new DataView(buf); // Second view
-            var copy_arr = deserialize(serialize(old_arr, [ buf ]));
+            var copy_arr = deserialize(serialize(old_arr, [ buf ], { scope }), { scope });
             assertEq(buf.byteLength, 0,
                      "donor array buffer should be detached");
             assertEq(old_arr.byteLength, 0,
@@ -78,7 +87,7 @@ function test() {
             var view = new Int32Array(old);
             view[0] = 1;
             var mutator = { get foo() { view[0] = 2; } };
-            var copy = deserialize(serialize([ old, mutator ], [old]));
+            var copy = deserialize(serialize([ old, mutator ], [ old ], { scope }), { scope });
             var viewCopy = new Int32Array(copy[0]);
             assertEq(view.length, 0); // Underlying buffer now detached.
             assertEq(viewCopy[0], 2);
@@ -90,7 +99,7 @@ function test() {
             old = new ArrayBuffer(size);
             var mutator = {
                 get foo() {
-                    deserialize(serialize(old, [old]));
+                    deserialize(serialize(old, [old], { scope }), { scope });
                 }
             };
             // The throw is not yet implemented, bug 919259.

js/src/vm/StructuredClone.cpp

@@ -125,6 +125,7 @@ enum StructuredDataType : uint32_t {
     SCTAG_TRANSFER_MAP_HEADER = 0xFFFF0200,
     SCTAG_TRANSFER_MAP_PENDING_ENTRY,
     SCTAG_TRANSFER_MAP_ARRAY_BUFFER,
+    SCTAG_TRANSFER_MAP_STORED_ARRAY_BUFFER,
     SCTAG_TRANSFER_MAP_END_OF_BUILTIN_TYPES,
 
     SCTAG_END_OF_BUILTIN_TYPES
@@ -166,6 +167,19 @@ struct BufferIterator {
         JS_STATIC_ASSERT(8 % sizeof(T) == 0);
     }
 
+    BufferIterator(const BufferIterator& other)
+        : mBuffer(other.mBuffer)
+        , mIter(other.mIter)
+    {
+    }
+
+    BufferIterator& operator=(const BufferIterator& other)
+    {
+        MOZ_ASSERT(&mBuffer == &other.mBuffer);
+        mIter = other.mIter;
+        return *this;
+    }
+
     BufferIterator operator++(int) {
         BufferIterator ret = *this;
         if (!mIter.AdvanceAcrossSegments(mBuffer, sizeof(T))) {
@@ -181,6 +195,11 @@ struct BufferIterator {
         return *this;
     }
 
+    size_t operator-(const BufferIterator& other) {
+        MOZ_ASSERT(&mBuffer == &other.mBuffer);
+        return mBuffer.RangeLength(other.mIter, mIter);
+    }
+
     void next() {
         if (!mIter.AdvanceAcrossSegments(mBuffer, sizeof(T))) {
             MOZ_ASSERT(false, "Failed to read StructuredCloneData. Data incomplete");
@@ -211,6 +230,8 @@ struct BufferIterator {
 
 struct SCOutput {
   public:
+    using Iter = BufferIterator<uint64_t, TempAllocPolicy>;
+
     explicit SCOutput(JSContext* cx);
 
     JSContext* context() const { return cx; }
@@ -229,11 +250,16 @@ struct SCOutput {
     bool extractBuffer(JSStructuredCloneData* data);
     void discardTransferables(const JSStructuredCloneCallbacks* cb, void* cbClosure);
 
+    uint64_t tell() const { return buf.Size(); }
     uint64_t count() const { return buf.Size() / sizeof(uint64_t); }
-    BufferIterator<uint64_t, TempAllocPolicy> iter() {
+    Iter iter() {
         return BufferIterator<uint64_t, TempAllocPolicy>(buf);
     }
 
+    size_t offset(Iter dest) {
+        return dest - iter();
+    }
+
   private:
     JSContext* cx;
     mozilla::BufferList<TempAllocPolicy> buf;
@@ -262,7 +288,9 @@ class SCInput {
     bool get(uint64_t* p);
     bool getPair(uint32_t* tagp, uint32_t* datap);
 
-    BufferIterator tell() const { return point; }
+    const BufferIterator& tell() const { return point; }
+    void seekTo(const BufferIterator& pos) { point = pos; }
+    void seekBy(size_t pos) { point += pos; }
 
     template <class T>
     bool readArray(T* p, size_t nelems);
@@ -290,7 +318,7 @@ struct JSStructuredCloneReader {
     explicit JSStructuredCloneReader(SCInput& in, JS::StructuredCloneScope scope,
                                      const JSStructuredCloneCallbacks* cb,
                                      void* cbClosure)
-        : in(in), scope(scope), objs(in.context()), allObjs(in.context()),
+        : in(in), allowedScope(scope), objs(in.context()), allObjs(in.context()),
           callbacks(cb), closure(cbClosure) { }
 
     SCInput& input() { return in; }
@@ -318,7 +346,18 @@ struct JSStructuredCloneReader {
 
     SCInput& in;
 
-    JS::StructuredCloneScope scope;
+    // The widest scope that the caller will accept, where
+    // SameProcessSameThread is the widest (it can store anything it wants) and
+    // DifferentProcess is the narrowest (it cannot contain pointers and must
+    // be valid cross-process.)
+    JS::StructuredCloneScope allowedScope;
+
+    // The scope the buffer was generated for (what sort of buffer it is.) The
+    // scope is not just a permissions thing; it also affects the storage
+    // format (eg a Transferred ArrayBuffer can be stored as a pointer for
+    // SameProcessSameThread but must have its contents in the clone buffer for
+    // DifferentProcess.)
+    JS::StructuredCloneScope storedScope;
 
     // Stack of objects with properties remaining to be read.
     AutoValueVector objs;
@@ -1438,7 +1477,6 @@ JSStructuredCloneWriter::writeTransferMap()
     RootedObject obj(context());
     for (auto tr = transferableObjects.all(); !tr.empty(); tr.popFront()) {
         obj = tr.front();
-
         if (!memory.put(obj, memory.count())) {
             ReportOutOfMemory(context());
             return false;
@@ -1474,7 +1512,8 @@ JSStructuredCloneWriter::transferOwnership()
     MOZ_ASSERT(NativeEndian::swapFromLittleEndian(point.peek()) == transferableObjects.count());
     point++;
 
-    RootedObject obj(context());
+    JSContext* cx = context();
+    RootedObject obj(cx);
     for (auto tr = transferableObjects.all(); !tr.empty(); tr.popFront()) {
         obj = tr.front();
 
@@ -1490,41 +1529,63 @@ JSStructuredCloneWriter::transferOwnership()
 #endif
 
         ESClass cls;
-        if (!GetBuiltinClass(context(), obj, &cls))
+        if (!GetBuiltinClass(cx, obj, &cls))
             return false;
 
         if (cls == ESClass::ArrayBuffer) {
+            tag = SCTAG_TRANSFER_MAP_ARRAY_BUFFER;
+
             // The current setup of the array buffer inheritance hierarchy doesn't
             // lend itself well to generic manipulation via proxies.
-            Rooted<ArrayBufferObject*> arrayBuffer(context(), &CheckedUnwrap(obj)->as<ArrayBufferObject>());
+            Rooted<ArrayBufferObject*> arrayBuffer(cx, &CheckedUnwrap(obj)->as<ArrayBufferObject>());
-            JSAutoCompartment ac(context(), arrayBuffer);
+            JSAutoCompartment ac(cx, arrayBuffer);
             size_t nbytes = arrayBuffer->byteLength();
 
             if (arrayBuffer->isWasm() || arrayBuffer->isPreparedForAsmJS()) {
-                JS_ReportErrorNumberASCII(context(), GetErrorMessage, nullptr,
+                JS_ReportErrorNumberASCII(cx, GetErrorMessage, nullptr, JSMSG_WASM_NO_TRANSFER);
-                                          JSMSG_WASM_NO_TRANSFER);
                 return false;
             }
 
-            bool hasStealableContents = arrayBuffer->hasStealableContents() &&
+            if (scope == JS::StructuredCloneScope::DifferentProcess) {
-                                        (scope != JS::StructuredCloneScope::DifferentProcess);
+                // Write Transferred ArrayBuffers in DifferentProcess scope at
+                // the end of the clone buffer, and store the offset within the
+                // buffer to where the ArrayBuffer was written. Note that this
+                // will invalidate the current position iterator.
+
+                size_t pointOffset = out.offset(point);
+                tag = SCTAG_TRANSFER_MAP_STORED_ARRAY_BUFFER;
+                ownership = JS::SCTAG_TMO_UNOWNED;
+                content = nullptr;
+                extraData = out.tell() - pointOffset; // Offset from tag to current end of buffer
+                if (!writeArrayBuffer(arrayBuffer))
+                    return false;
+
+                // Must refresh the point iterator after its collection has
+                // been modified.
+                point = out.iter();
+                point += pointOffset;
+
+                if (!JS_DetachArrayBuffer(cx, arrayBuffer))
+                    return false;
+            } else {
+                bool hasStealableContents = arrayBuffer->hasStealableContents();
 
                 ArrayBufferObject::BufferContents bufContents =
-                ArrayBufferObject::stealContents(context(), arrayBuffer, hasStealableContents);
+                    ArrayBufferObject::stealContents(cx, arrayBuffer, hasStealableContents);
                 if (!bufContents)
                     return false; // already transferred data
 
                 content = bufContents.data();
-            tag = SCTAG_TRANSFER_MAP_ARRAY_BUFFER;
                 if (bufContents.kind() == ArrayBufferObject::MAPPED)
                     ownership = JS::SCTAG_TMO_MAPPED_DATA;
                 else
                     ownership = JS::SCTAG_TMO_ALLOC_DATA;
                 extraData = nbytes;
+            }
         } else {
             if (!callbacks || !callbacks->writeTransfer)
                 return reportDataCloneError(JS_SCERR_TRANSFERABLE);
-            if (!callbacks->writeTransfer(context(), obj, closure, &tag, &ownership, &content, &extraData))
+            if (!callbacks->writeTransfer(cx, obj, closure, &tag, &ownership, &content, &extraData))
                 return false;
             MOZ_ASSERT(tag > SCTAG_TRANSFER_MAP_PENDING_ENTRY);
         }
@@ -2100,7 +2161,8 @@ JSStructuredCloneReader::readHeader()
     }
 
     MOZ_ALWAYS_TRUE(in.readPair(&tag, &data));
-    if (data < uint32_t(scope)) {
+    storedScope = JS::StructuredCloneScope(data);
+    if (storedScope < allowedScope) {
         JS_ReportErrorNumberASCII(context(), GetErrorMessage, nullptr, JSMSG_SC_BAD_SERIALIZED_DATA,
                                   "incompatible structured clone scope");
         return false;
@@ -2145,6 +2207,12 @@ JSStructuredCloneReader::readTransferMap()
             return false;
 
         if (tag == SCTAG_TRANSFER_MAP_ARRAY_BUFFER) {
+            if (storedScope == JS::StructuredCloneScope::DifferentProcess) {
+                // Transferred ArrayBuffers in a DifferentProcess clone buffer
+                // are treated as if they weren't Transferred at all.
+                continue;
+            }
+
             size_t nbytes = extraData;
             MOZ_ASSERT(data == JS::SCTAG_TMO_ALLOC_DATA ||
                        data == JS::SCTAG_TMO_MAPPED_DATA);
@@ -2152,6 +2220,22 @@ JSStructuredCloneReader::readTransferMap()
                 obj = JS_NewArrayBufferWithContents(cx, nbytes, content);
             else if (data == JS::SCTAG_TMO_MAPPED_DATA)
                 obj = JS_NewMappedArrayBufferWithContents(cx, nbytes, content);
+        } else if (tag == SCTAG_TRANSFER_MAP_STORED_ARRAY_BUFFER) {
+            auto savedPos = in.tell();
+            auto guard = mozilla::MakeScopeExit([&] {
+                in.seekTo(savedPos);
+            });
+            in.seekTo(pos);
+            in.seekBy(static_cast<size_t>(extraData));
+
+            uint32_t tag, data;
+            if (!in.readPair(&tag, &data))
+                return false;
+            MOZ_ASSERT(tag == SCTAG_ARRAY_BUFFER_OBJECT);
+            RootedValue val(cx);
+            if (!readArrayBuffer(data, &val))
+                return false;
+            obj = &val.toObject();
         } else {
             if (!callbacks || !callbacks->readTransfer) {
                 ReportDataCloneError(cx, callbacks, JS_SCERR_TRANSFERABLE);

mfbt/BufferList.h

@@ -224,6 +224,33 @@ class BufferList : private AllocPolicy
     {
       return mData == mDataEnd;
     }
+
+   private:
+
+    // Count the bytes we would need to advance in order to reach aTarget.
+    size_t BytesUntil(const BufferList& aBuffers, const IterImpl& aTarget) const {
+      size_t offset = 0;
+
+      MOZ_ASSERT(aTarget.IsIn(aBuffers));
+
+      char* data = mData;
+      for (uintptr_t segment = mSegment; segment < aTarget.mSegment; segment++) {
+        offset += aBuffers.mSegments[segment].End() - data;
+        data = aBuffers.mSegments[segment].mData;
+      }
+
+      MOZ_RELEASE_ASSERT(IsIn(aBuffers));
+      MOZ_RELEASE_ASSERT(aTarget.mData >= data);
+
+      offset += aTarget.mData - data;
+      return offset;
+    }
+
+    bool IsIn(const BufferList& aBuffers) const {
+      return mSegment < aBuffers.mSegments.length() &&
+             mData >= aBuffers.mSegments[mSegment].mData &&
+             mData < aBuffers.mSegments[mSegment].End();
+    }
   };
 
   // Special convenience method that returns Iter().Data().
@@ -270,6 +297,13 @@ class BufferList : private AllocPolicy
   // This method requires aIter and aSize to be 8-byte aligned.
   BufferList Extract(IterImpl& aIter, size_t aSize, bool* aSuccess);
 
+  // Return the number of bytes from 'start' to 'end', two iterators within
+  // this BufferList.
+  size_t RangeLength(const IterImpl& start, const IterImpl& end) const {
+    MOZ_ASSERT(start.IsIn(*this) && end.IsIn(*this));
+    return start.BytesUntil(*this, end);
+  }
+
 private:
   explicit BufferList(AllocPolicy aAP)
    : AllocPolicy(aAP),