Fixing some endian-ness bugs for color management - bug 439704. r=vlad

gfx/thebes/public/gfxPlatform.h

@@ -46,6 +46,7 @@
 
 #include "gfxTypes.h"
 #include "gfxASurface.h"
+#include "gfxColor.h"
 
 #ifdef XP_OS2
 #undef OS2EMX_PLAIN_CHAR
@@ -237,6 +238,13 @@ public:
      */
     static int GetRenderingIntent();
 
+    /**
+     * Convert a pixel using a cms transform in an endian-aware manner.
+     *
+     * Sets 'out' to 'in' if transform is NULL.
+     */
+    static void TransformPixel(const gfxRGBA& in, gfxRGBA& out, cmsHTRANSFORM transform);
+
     /**
      * Return the output device ICC profile.
      */

gfx/thebes/src/gfxContext.cpp

@@ -622,29 +622,16 @@ void
 gfxContext::SetColor(const gfxRGBA& c)
 {
     if (gfxPlatform::GetCMSMode() == eCMSMode_All) {
-        cmsHTRANSFORM transform = gfxPlatform::GetCMSRGBTransform();
-        if (transform) {
-#ifdef IS_LITTLE_ENDIAN
-            PRUint32 packed = c.Packed(gfxRGBA::PACKED_ABGR);
-            cmsDoTransform(transform,
-                           (PRUint8 *)&packed, (PRUint8 *)&packed,
-                           1);
-            gfxRGBA cms(packed, gfxRGBA::PACKED_ABGR);
-#else
-            PRUint32 packed = c.Packed(gfxRGBA::PACKED_ARGB);
-            cmsDoTransform(transform,
-                           (PRUint8 *)&packed + 1, (PRUint8 *)&packed + 1,
-                           1);
-            gfxRGBA cms(packed, gfxRGBA::PACKED_ARGB);
-#endif
-            // Use the original alpha to avoid unnecessary float->byte->float
-            // conversion errors
-            cairo_set_source_rgba(mCairo, cms.r, cms.g, cms.b, c.a);
-            return;
-        }
-    }
 
-    cairo_set_source_rgba(mCairo, c.r, c.g, c.b, c.a);
+        gfxRGBA cms;
+        gfxPlatform::TransformPixel(c, cms, gfxPlatform::GetCMSRGBTransform());
+
+        // Use the original alpha to avoid unnecessary float->byte->float
+        // conversion errors
+        cairo_set_source_rgba(mCairo, cms.r, cms.g, cms.b, c.a);
+    }
+    else
+        cairo_set_source_rgba(mCairo, c.r, c.g, c.b, c.a);
 }
 
 void

gfx/thebes/src/gfxPattern.cpp

@@ -88,29 +88,16 @@ void
 gfxPattern::AddColorStop(gfxFloat offset, const gfxRGBA& c)
 {
     if (gfxPlatform::GetCMSMode() == eCMSMode_All) {
-        cmsHTRANSFORM transform = gfxPlatform::GetCMSRGBTransform();
-        if (transform) {
-#ifdef IS_LITTLE_ENDIAN
-            PRUint32 packed = c.Packed(gfxRGBA::PACKED_ABGR);
-            cmsDoTransform(transform,
-                           (PRUint8 *)&packed, (PRUint8 *)&packed,
-                           1);
-            gfxRGBA cms(packed, gfxRGBA::PACKED_ABGR);
-#else
-            PRUint32 packed = c.Packed(gfxRGBA::PACKED_ARGB);
-            cmsDoTransform(transform,
-                           (PRUint8 *)&packed + 1, (PRUint8 *)&packed + 1,
-                           1);
-            gfxRGBA cms(packed, gfxRGBA::PACKED_ARGB);
-#endif
-            // Use the original alpha to avoid unnecessary float->byte->float
-            // conversion errors
-            cairo_pattern_add_color_stop_rgba(mPattern, offset,
-                                              cms.r, cms.g, cms.b, c.a);
-            return;
-        }
+        gfxRGBA cms;
+        gfxPlatform::TransformPixel(c, cms, gfxPlatform::GetCMSRGBTransform());
+
+        // Use the original alpha to avoid unnecessary float->byte->float
+        // conversion errors
+        cairo_pattern_add_color_stop_rgba(mPattern, offset,
+                                          cms.r, cms.g, cms.b, c.a);
     }
-    cairo_pattern_add_color_stop_rgba(mPattern, offset, c.r, c.g, c.b, c.a);
+    else
+        cairo_pattern_add_color_stop_rgba(mPattern, offset, c.r, c.g, c.b, c.a);
 }
 
 void

gfx/thebes/src/gfxPlatform.cpp

@@ -538,6 +538,31 @@ gfxPlatform::GetRenderingIntent()
     return gCMSIntent;
 }
 
+void 
+gfxPlatform::TransformPixel(const gfxRGBA& in, gfxRGBA& out, cmsHTRANSFORM transform)
+{
+
+    if (transform) {
+#ifdef IS_LITTLE_ENDIAN
+        PRUint32 packed = in.Packed(gfxRGBA::PACKED_ABGR);
+        cmsDoTransform(transform,
+                       (PRUint8 *)&packed, (PRUint8 *)&packed,
+                       1);
+        out.~gfxRGBA();
+        new (&out) gfxRGBA(packed, gfxRGBA::PACKED_ABGR);
+#else
+        PRUint32 packed = in.Packed(gfxRGBA::PACKED_ARGB);
+        cmsDoTransform(transform,
+                       (PRUint8 *)&packed + 1, (PRUint8 *)&packed + 1,
+                       1);
+        out.~gfxRGBA();
+        new (&out) gfxRGBA(packed, gfxRGBA::PACKED_ARGB);
+#endif
+    }
+
+    else if (&out != &in)
+        out = in;
+}
 
 cmsHPROFILE
 gfxPlatform::GetPlatformCMSOutputProfile()

widget/src/cocoa/nsCocoaWindow.mm

@@ -1337,9 +1337,17 @@ NS_IMETHODIMP nsCocoaWindow::SetWindowTitlebarColor(nscolor aColor, PRBool aActi
     // Transform from sRGBA to monitor RGBA. This seems like it would make trying
     // to match the system appearance lame, so probably we just shouldn't color 
     // correct chrome.
-    cmsHTRANSFORM transform = NULL;
-    if ((gfxPlatform::GetCMSMode() == eCMSMode_All) && (transform = gfxPlatform::GetCMSRGBATransform()))
-      cmsDoTransform(transform, &aColor, &aColor, 1);
+    if (gfxPlatform::GetCMSMode() == eCMSMode_All) {
+      cmsHTRANSFORM transform = gfxPlatform::GetCMSRGBATransform();
+      if (transform) {
+        PRUint8 color[3];
+        color[0] = NS_GET_R(aColor);
+        color[1] = NS_GET_G(aColor);
+        color[2] = NS_GET_B(aColor);
+        cmsDoTransform(transform, color, color, 1);
+        aColor = NS_RGB(color[0], color[1], color[2]);
+      }
+    }
 
     [(ToolbarWindow*)mWindow setTitlebarColor:[NSColor colorWithDeviceRed:NS_GET_R(aColor)/255.0
                                                                     green:NS_GET_G(aColor)/255.0