Bug 1504660 - Implement Xrays for instanceof r=bholley

Ensure that "a instanceof b" has Xray semantics, i.e. that when b is a
XrayWrapper, that the wrapped object's getters, `Symbol.hasInstance`
hook and proxy traps are not triggered.

The toolkit/components/mozintl/test/test_mozintlhelper.js test was
updated to explicitly waive Xrays, instead of relying on the previous
behavior where Xrays were automatically waived.

Depends on D11591

Depends on D11591

Differential Revision: https://phabricator.services.mozilla.com/D11592

--HG--
extra : moz-landing-system : lando

js/public/Wrapper.h

@@ -296,6 +296,8 @@ class JS_FRIEND_API OpaqueCrossCompartmentWrapper
                                ESClass* cls) const override;
   virtual bool isArray(JSContext* cx, HandleObject obj,
                        JS::IsArrayAnswer* answer) const override;
+  virtual bool hasInstance(JSContext* cx, HandleObject wrapper,
+                           MutableHandleValue v, bool* bp) const override;
   virtual const char* className(JSContext* cx,
                                 HandleObject wrapper) const override;
   virtual JSString* fun_toString(JSContext* cx, HandleObject proxy,

js/src/proxy/OpaqueCrossCompartmentWrapper.cpp

@@ -140,6 +140,14 @@ bool OpaqueCrossCompartmentWrapper::isArray(JSContext* cx, HandleObject obj,
   return true;
 }
 
+bool OpaqueCrossCompartmentWrapper::hasInstance(JSContext* cx,
+                                                HandleObject wrapper,
+                                                MutableHandleValue v,
+                                                bool* bp) const {
+  *bp = false;
+  return true;
+}
+
 const char* OpaqueCrossCompartmentWrapper::className(JSContext* cx,
                                                      HandleObject proxy) const {
   return "Opaque";

js/xpconnect/tests/unit/test_xray_instanceof.js

@@ -0,0 +1,210 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+ChromeUtils.import("resource://gre/modules/Services.jsm");
+
+add_task(function instanceof_xrays() {
+  let sandbox = Cu.Sandbox(null);
+  Cu.evalInSandbox(`
+    this.proxy = new Proxy([], {
+      getPrototypeOf() {
+        return Date.prototype;
+      },
+    });
+
+    this.inheritedProxy = Object.create(this.proxy);
+
+    this.FunctionProxy = new Proxy(function() {}, {});
+    this.functionProxyInstance = new this.FunctionProxy();
+
+    this.CustomClass = class {};
+    this.customClassInstance = new this.CustomClass();
+  `, sandbox);
+
+  {
+    // Sanity check that instanceof still works with standard constructors when xrays are present.
+    Assert.ok(Cu.evalInSandbox(`new Date()`, sandbox) instanceof sandbox.Date,
+              "async function result in sandbox instanceof sandbox.Date");
+    Assert.ok(new sandbox.Date() instanceof sandbox.Date,
+              "sandbox.Date() instanceof sandbox.Date");
+
+    Assert.ok(sandbox.CustomClass instanceof sandbox.Function,
+              "Class constructor instanceof sandbox.Function");
+    Assert.ok(sandbox.CustomClass instanceof sandbox.Object,
+              "Class constructor instanceof sandbox.Object");
+
+    // Both operands must have the same kind of Xray vision.
+    Assert.equal(Cu.waiveXrays(sandbox.CustomClass) instanceof sandbox.Function, false,
+                 "Class constructor with waived xrays instanceof sandbox.Function");
+    Assert.equal(Cu.waiveXrays(sandbox.CustomClass) instanceof sandbox.Object, false,
+                 "Class constructor with waived xrays instanceof sandbox.Object");
+  }
+
+  {
+    let {proxy} = sandbox;
+    Assert.equal(proxy instanceof sandbox.Date, false,
+                 "instanceof should ignore the proxy trap");
+    Assert.equal(proxy instanceof sandbox.Array, false,
+                 "instanceof should ignore the proxy target");
+    Assert.equal(Cu.waiveXrays(proxy) instanceof sandbox.Date, false,
+                 "instanceof should ignore the proxy trap despite the waived xrays on the proxy");
+    Assert.equal(Cu.waiveXrays(proxy) instanceof sandbox.Array, false,
+                 "instanceof should ignore the proxy target despite the waived xrays on the proxy");
+
+    Assert.equal(proxy instanceof Cu.waiveXrays(sandbox.Date), false,
+                 "instanceof should ignore the proxy trap despite the waived xrays on the constructor");
+    Assert.equal(proxy instanceof Cu.waiveXrays(sandbox.Array), false,
+                 "instanceof should ignore the proxy target despite the waived xrays on the constructor");
+
+    Assert.ok(Cu.waiveXrays(proxy) instanceof Cu.waiveXrays(sandbox.Date),
+              "instanceof should trigger the proxy trap after waiving both Xrays");
+  }
+
+
+  {
+    let {inheritedProxy} = sandbox;
+    Assert.equal(inheritedProxy instanceof sandbox.Date, false,
+                 "instanceof should ignore the inherited proxy trap");
+    Assert.equal(Cu.waiveXrays(inheritedProxy) instanceof sandbox.Date, false,
+                 "instanceof should ignore the inherited proxy trap despite the waived xrays on the proxy");
+
+    Assert.equal(inheritedProxy instanceof Cu.waiveXrays(sandbox.Date), false,
+                 "instanceof should ignore the inherited proxy trap despite the waived xrays on the constructor");
+
+    Assert.ok(Cu.waiveXrays(inheritedProxy) instanceof Cu.waiveXrays(sandbox.Date),
+              "instanceof should trigger the inherited proxy trap after waiving both Xrays");
+  }
+
+  {
+    let {FunctionProxy, functionProxyInstance} = sandbox;
+
+    // Ideally, the next two test cases should both throw "... not a function".
+    // However, because the opaque XrayWrapper does not override isCallable, an
+    // opaque XrayWrapper is still considered callable if the proxy target is,
+    // and "instanceof" will try to look up the prototype of the wrapper (and
+    // fail because opaque XrayWrappers hide the properties).
+    Assert.throws(
+      () => functionProxyInstance instanceof FunctionProxy,
+      /'prototype' property of FunctionProxy is not an object/,
+      "Opaque constructor proxy should be hidden by Xrays");
+    Assert.throws(
+      () => functionProxyInstance instanceof sandbox.proxy,
+      /sandbox.proxy is not a function/,
+      "Opaque non-constructor proxy should be hidden by Xrays");
+
+    Assert.equal(functionProxyInstance instanceof Cu.waiveXrays(FunctionProxy), false,
+                 "Waiver on opaque constructor proxy should be ignored for XrayWrapped functionProxyInstance");
+    Assert.ok(Cu.waiveXrays(functionProxyInstance) instanceof Cu.waiveXrays(FunctionProxy),
+              "instanceof should get through the proxy after waiving both Xrays");
+  }
+
+  {
+    let {CustomClass, customClassInstance} = sandbox;
+    // Under Xray vision, every JS object is either a plain object or array.
+    // Prototypical inheritance is invisible when the constructor is wrapped.
+    Assert.throws(
+      () => customClassInstance instanceof CustomClass,
+      /TypeError: 'prototype' property of CustomClass is not an object/,
+      "instanceof on a custom JS class with xrays should fail");
+    Assert.equal(customClassInstance instanceof Cu.waiveXrays(CustomClass), false,
+                 "instanceof should return false if the instance and constructor have distinct Xray vision");
+    Assert.ok(Cu.waiveXrays(customClassInstance) instanceof Cu.waiveXrays(CustomClass),
+              "instanceof should see the true prototype of CustomClass after waiving Xrays");
+  }
+});
+
+add_task(function instanceof_dom_xrays_hasInstance() {
+  const principal = Services.scriptSecurityManager.createNullPrincipal({});
+  const webnav = Services.appShell.createWindowlessBrowser(false);
+  webnav.docShell.createAboutBlankContentViewer(principal);
+  let window = webnav.document.defaultView;
+
+  let sandbox = Cu.Sandbox(principal);
+  sandbox.DOMObjectWithHasInstance = window.document;
+  Cu.evalInSandbox(`
+    this.DOMObjectWithHasInstance[Symbol.hasInstance] = function() {
+      return true;
+    };
+    this.ObjectWithHasInstance = {
+      [Symbol.hasInstance](v) {
+        v.throwsIfVCannotBeAccessed;
+        return true;
+      },
+    };
+  `, sandbox);
+
+  // Override the hasInstance handler in the window, so that we can detect when
+  // we end up triggering hasInstance in the window's compartment.
+  window.eval(`
+    document[Symbol.hasInstance] = function() {
+      throw "hasInstance_in_window";
+    };
+  `);
+
+  sandbox.domobj = window.document.body;
+  Assert.ok(sandbox.eval(`domobj.wrappedJSObject`),
+            "DOM object is a XrayWrapper");
+  Assert.ok(sandbox.eval(`DOMObjectWithHasInstance.wrappedJSObject`),
+            "DOM object with Symbol.hasInstance is a XrayWrapper");
+
+  for (let Obj of ["ObjectWithHasInstance", "DOMObjectWithHasInstance"]) {
+    // Tests Xray vision *inside* the sandbox. The Symbol.hasInstance member
+    // is a property / expando object in the sandbox's compartment, so the
+    // "instanceof" operator should always trigger the hasInstance function.
+    Assert.ok(sandbox.eval(`[] instanceof ${Obj}`),
+              `Should call ${Obj}[Symbol.hasInstance] when left operand has no Xrays`);
+    Assert.ok(sandbox.eval(`domobj instanceof ${Obj}`),
+              `Should call ${Obj}[Symbol.hasInstance] when left operand has Xrays`);
+    Assert.ok(sandbox.eval(`domobj.wrappedJSObject instanceof ${Obj}`),
+              `Should call ${Obj}[Symbol.hasInstance] when left operand has waived Xrays`);
+
+    // Tests Xray vision *outside* the sandbox. The Symbol.hasInstance member
+    // should be hidden by Xrays.
+    let sandboxObjWithHasInstance = sandbox[Obj];
+    Assert.ok(Cu.isXrayWrapper(sandboxObjWithHasInstance),
+              `sandbox.${Obj} is a XrayWrapper`);
+    Assert.throws(
+      () => sandbox.Object() instanceof sandboxObjWithHasInstance,
+      /sandboxObjWithHasInstance is not a function/,
+      `sandbox.${Obj}[Symbol.hasInstance] should be hidden by Xrays`);
+
+    Assert.throws(
+      () => Cu.waiveXrays(sandbox.Object()) instanceof sandboxObjWithHasInstance,
+      /sandboxObjWithHasInstance is not a function/,
+      `sandbox.${Obj}[Symbol.hasInstance] should be hidden by Xrays, despite the waived Xrays at the left`);
+
+    // The Xay waiver on the right operand should be ignored if the left
+    // operand still has Xrays.
+    Assert.throws(
+      () => sandbox.Object() instanceof Cu.waiveXrays(sandboxObjWithHasInstance),
+      /Cu.waiveXrays\(\.\.\.\) is not a function/,
+      `Waiver on sandbox.${Obj} should be ignored when the left operand has Xrays`);
+    // (Cases where the left operand has no Xrays are checked below.)
+  }
+
+  // hasInstance is expected to be called, but still trigger an error because
+  // properties of the object from the current context should not be readable
+  // by the hasInstance function in the sandbox with a different principal.
+  Assert.throws(
+    () => [] instanceof Cu.waiveXrays(sandbox.ObjectWithHasInstance),
+    /Permission denied to access property "throwsIfVCannotBeAccessed"/,
+    `Should call (waived) sandbox.ObjectWithHasInstance[Symbol.hasInstance] when the left operand has no Xrays`);
+
+  Assert.ok(Cu.waiveXrays(sandbox.Object()) instanceof Cu.waiveXrays(sandbox.ObjectWithHasInstance),
+            `Should call (waived) sandbox.ObjectWithHasInstance[Symbol.hasInstance] when the left operand has waived Xrays`);
+
+  // When Xrays of the DOM object are waived, we end up in the owner document's
+  // compartment (instead of the sandbox).
+  Assert.throws(
+    () => [] instanceof Cu.waiveXrays(sandbox.DOMObjectWithHasInstance),
+    /hasInstance_in_window/,
+    "Should call (waived) sandbox.DOMObjectWithHasInstance[Symbol.hasInstance] when the left operand has no Xrays");
+
+  Assert.throws(
+    () => Cu.waiveXrays(sandbox.Object()) instanceof Cu.waiveXrays(sandbox.DOMObjectWithHasInstance),
+    /hasInstance_in_window/,
+    "Should call (waived) sandbox.DOMObjectWithHasInstance[Symbol.hasInstance] when the left operand has waived Xrays");
+
+  webnav.close();
+});

js/xpconnect/tests/unit/xpcshell.ini

@@ -137,6 +137,7 @@ head = head_watchdog.js
 [test_xpcwn_tamperproof.js]
 [test_xrayed_arguments.js]
 [test_xrayed_iterator.js]
+[test_xray_instanceof.js]
 [test_xray_named_element_access.js]
 [test_xray_SavedFrame.js]
 [test_xray_SavedFrame-02.js]

js/xpconnect/wrappers/WaiveXrayWrapper.cpp

@@ -89,6 +89,36 @@ bool WaiveXrayWrapper::nativeCall(JSContext* cx, JS::IsAcceptableThis test,
          WrapperFactory::WaiveXrayAndWrap(cx, args.rval());
 }
 
+bool WaiveXrayWrapper::hasInstance(JSContext* cx, HandleObject wrapper,
+                                   MutableHandleValue v, bool* bp) const {
+  if (v.isObject() && WrapperFactory::IsXrayWrapper(&v.toObject())) {
+    // If |v| is a XrayWrapper and in the same compartment as the value
+    // wrapped by |wrapper|, then the Xrays of |v| would be waived upon
+    // calling CrossCompartmentWrapper::hasInstance. This may trigger
+    // getters and proxy traps of unwrapped |v|. To prevent that from
+    // happening, we exit early.
+
+    // |wrapper| is the right operand of "instanceof", and must either be
+    // a function or an object with a @@hasInstance method. We are not going
+    // to call @@hasInstance, so only check whether it is a function.
+    // This check is here for consistency with usual "instanceof" behavior,
+    // which throws if the right operand is not a function. Without this
+    // check, the "instanceof" operator would return false and potentially
+    // hide errors in the code that uses the "instanceof" operator.
+    if (!JS::IsCallable(wrapper)) {
+      RootedValue wrapperv(cx, JS::ObjectValue(*wrapper));
+      js::ReportIsNotFunction(cx, wrapperv);
+      return false;
+    }
+
+    *bp = false;
+    return true;
+  }
+
+  // Both |wrapper| and |v| have no Xrays here.
+  return CrossCompartmentWrapper::hasInstance(cx, wrapper, v, bp);
+}
+
 bool WaiveXrayWrapper::getPrototype(JSContext* cx, HandleObject wrapper,
                                     MutableHandleObject protop) const {
   return CrossCompartmentWrapper::getPrototype(cx, wrapper, protop) &&

js/xpconnect/wrappers/WaiveXrayWrapper.h

@@ -39,6 +39,8 @@ class WaiveXrayWrapper : public js::CrossCompartmentWrapper {
   virtual bool nativeCall(JSContext* cx, JS::IsAcceptableThis test,
                           JS::NativeImpl impl,
                           const JS::CallArgs& args) const override;
+  virtual bool hasInstance(JSContext* cx, JS::HandleObject wrapper,
+                           JS::MutableHandleValue v, bool* bp) const override;
   virtual bool getPropertyDescriptor(
       JSContext* cx, JS::Handle<JSObject*> wrapper, JS::Handle<jsid> id,
       JS::MutableHandle<JS::PropertyDescriptor> desc) const override;

js/xpconnect/wrappers/XrayWrapper.cpp

@@ -2234,6 +2234,19 @@ bool XrayWrapper<Base, Traits>::getBuiltinClass(JSContext* cx,
   return Traits::getBuiltinClass(cx, wrapper, Base::singleton, cls);
 }
 
+template <typename Base, typename Traits>
+bool XrayWrapper<Base, Traits>::hasInstance(JSContext* cx,
+                                            JS::HandleObject wrapper,
+                                            JS::MutableHandleValue v,
+                                            bool* bp) const {
+  assertEnteredPolicy(cx, wrapper, JSID_VOID, BaseProxyHandler::GET);
+
+  // CrossCompartmentWrapper::hasInstance unwraps |wrapper|'s Xrays and enters
+  // its compartment. Any present XrayWrappers should be preserved, so the
+  // standard "instanceof" implementation is called without unwrapping first.
+  return JS::InstanceofOperator(cx, wrapper, v, bp);
+}
+
 template <typename Base, typename Traits>
 const char* XrayWrapper<Base, Traits>::className(JSContext* cx,
                                                  HandleObject wrapper) const {

js/xpconnect/wrappers/XrayWrapper.h

@@ -432,6 +432,8 @@ class XrayWrapper : public Base {
 
   virtual bool getBuiltinClass(JSContext* cx, JS::HandleObject wapper,
                                js::ESClass* cls) const override;
+  virtual bool hasInstance(JSContext* cx, JS::HandleObject wrapper,
+                           JS::MutableHandleValue v, bool* bp) const override;
   virtual const char* className(JSContext* cx,
                                 JS::HandleObject proxy) const override;
 

toolkit/components/mozintl/test/test_mozintlhelper.js

@@ -27,9 +27,9 @@ function test_cross_global(miHelper) {
   miHelper.addGetCalendarInfo(x);
   var waivedX = Cu.waiveXrays(x);
   equal(waivedX.getCalendarInfo instanceof Function, false);
-  equal(waivedX.getCalendarInfo instanceof global.Function, true);
+  equal(waivedX.getCalendarInfo instanceof Cu.waiveXrays(global.Function), true);
   equal(waivedX.getCalendarInfo() instanceof Object, false);
-  equal(waivedX.getCalendarInfo() instanceof global.Object, true);
+  equal(waivedX.getCalendarInfo() instanceof Cu.waiveXrays(global.Object), true);
 }
 
 function test_methods_presence(miHelper) {