Bug 1285057 - Blacklist UBSan detection of the GC's generic interfaces; r=sfink

--HG--
extra : rebase_source : f5367a8685ec462edb7e30e5ed3d274b4a9bcd0f

js/public/RootingAPI.h

@@ -646,7 +646,9 @@ class alignas(8) DispatchWrapper
 
     // Trace the contained storage (of unknown type) using the trace function
     // we set aside when we did know the type.
-    static void TraceWrapped(JSTracer* trc, T* thingp, const char* name) {
+    MOZ_UBSAN_BLACKLIST_FUNCTION static void TraceWrapped(JSTracer* trc, T* thingp,
+                                                          const char* name)
+    {
         auto wrapper = reinterpret_cast<DispatchWrapper*>(
                            uintptr_t(thingp) - offsetof(DispatchWrapper, storage));
         wrapper->tracer(trc, &wrapper->storage, name);

js/public/SweepingAPI.h

@@ -57,7 +57,7 @@ class WeakCache : public js::WeakCacheBase<T>,
     const T& get() const { return cache; }
     T& get() { return cache; }
 
-    void sweep() { sweeper(&cache); }
+    MOZ_UBSAN_BLACKLIST_FUNCTION void sweep() { sweeper(&cache); }
 };
 
 } // namespace JS

mfbt/Attributes.h

@@ -208,6 +208,19 @@
 #  define MOZ_TSAN_BLACKLIST /* nothing */
 #endif
 
+/*
+ * MOZ_UBSAN_BLACKLIST_FUNCTION is a macro to tell UndefinedBehaviorSanitizer (a
+ * run-time instrumentation shipped with Clang) to not check the annotated
+ * function for undefined behaviors. Furthermore it prevents the compiler from
+ * inlining the function because inlining currently breaks the blacklisting
+ * mechanism of UBSan.
+ */
+#if defined(__clang__)
+# define MOZ_UBSAN_BLACKLIST_FUNCTION MOZ_NEVER_INLINE __attribute__((no_sanitize("function")))
+#else
+# define MOZ_UBSAN_BLACKLIST_FUNCTION /* nothing */
+#endif
+
 /**
  * MOZ_ALLOCATOR tells the compiler that the function it marks returns either a
  * "fresh", "pointer-free" block of memory, or nullptr. "Fresh" means that the