Bug 585175 - Don't automangle ctypes stdcall symbols for WINAPI. r=bsmedberg, a=blocker

js/src/ctypes/CTypes.cpp

@@ -870,7 +870,8 @@ InitTypeClasses(JSContext* cx, JSObject* parent)
 
   // Attach objects representing ABI constants.
   if (!DefineABIConstant(cx, parent, "default_abi", ABI_DEFAULT) ||
-      !DefineABIConstant(cx, parent, "stdcall_abi", ABI_STDCALL))
+      !DefineABIConstant(cx, parent, "stdcall_abi", ABI_STDCALL) ||
+      !DefineABIConstant(cx, parent, "winapi_abi", ABI_WINAPI))
     return false;
 
   // Create objects representing the builtin types, and attach them to the
@@ -2127,8 +2128,11 @@ BuildTypeName(JSContext* cx, JSObject* typeObj)
       FunctionInfo* fninfo = FunctionType::GetFunctionInfo(cx, typeObj);
 
       // Add in the calling convention, if it's not cdecl.
-      if (GetABICode(cx, fninfo->mABI) == ABI_STDCALL)
+      ABICode abi = GetABICode(cx, fninfo->mABI);
+      if (abi == ABI_STDCALL)
         PrependString(result, "__stdcall ");
+      else if (abi == ABI_WINAPI)
+        PrependString(result, "WINAPI ");
 
       // Wrap the entire expression so far with parens.
       PrependString(result, "(");
@@ -2217,6 +2221,9 @@ BuildTypeSource(JSContext* cx,
     case ABI_STDCALL:
       AppendString(result, "ctypes.stdcall_abi, ");
       break;
+    case ABI_WINAPI:
+      AppendString(result, "ctypes.winapi_abi, ");
+      break;
     case INVALID_ABI:
       JS_NOT_REACHED("invalid abi");
       break;
@@ -4510,6 +4517,7 @@ GetABI(JSContext* cx, jsval abiType, ffi_abi* result)
     *result = FFI_DEFAULT_ABI;
     return true;
   case ABI_STDCALL:
+  case ABI_WINAPI:
 #if (defined(_WIN32) && !defined(_WIN64)) || defined(_OS2)
     *result = FFI_STDCALL;
     return true;
@@ -4646,7 +4654,8 @@ FunctionType::BuildSymbolName(JSContext* cx,
 
   switch (GetABICode(cx, fninfo->mABI)) {
   case ABI_DEFAULT:
-    // For cdecl functions, no mangling is necessary.
+  case ABI_WINAPI:
+    // For cdecl or WINAPI functions, no mangling is necessary.
     AppendString(result, name);
     break;
 
@@ -4839,6 +4848,9 @@ FunctionType::CreateInternal(JSContext* cx,
   return typeObj;
 }
 
+// Construct a function pointer to a JS function (see CClosure::Create()).
+// Regular function pointers are constructed directly in
+// PointerType::ConstructData().
 JSBool
 FunctionType::ConstructData(JSContext* cx,
                             JSObject* typeObj,
@@ -4855,6 +4867,11 @@ FunctionType::ConstructData(JSContext* cx,
     JS_ReportError(cx, "Can't declare a variadic callback function");
     return JS_FALSE;
   }
+  if (GetABICode(cx, fninfo->mABI) == ABI_WINAPI) {
+    JS_ReportError(cx, "Can't declare a ctypes.winapi_abi callback function, "
+                   "use ctypes.stdcall_abi instead");
+    return JS_FALSE;
+  }
 
   JSObject* closureObj = CClosure::Create(cx, typeObj, fnObj, thisObj, data);
   if (!closureObj)
@@ -5151,6 +5168,7 @@ CClosure::Create(JSContext* cx,
   // Get the FunctionInfo from the FunctionType.
   FunctionInfo* fninfo = FunctionType::GetFunctionInfo(cx, typeObj);
   JS_ASSERT(!fninfo->mIsVariadic);
+  JS_ASSERT(GetABICode(cx, fninfo->mABI) != ABI_WINAPI);
 
   AutoPtr<ClosureInfo> cinfo(new ClosureInfo());
   if (!cinfo) {

js/src/ctypes/CTypes.h

@@ -236,11 +236,13 @@ JSBool TypeError(JSContext* cx, const char* expected, jsval actual);
  * ABI constants that specify the calling convention to use.
  * ctypes.default_abi corresponds to the cdecl convention, and in almost all
  * cases is the correct choice. ctypes.stdcall_abi is provided for calling
- * functions in the Microsoft Win32 API.
+ * stdcall functions on Win32, and implies stdcall symbol name decoration;
+ * ctypes.winapi_abi is just stdcall but without decoration.
  */
 enum ABICode {
   ABI_DEFAULT,
   ABI_STDCALL,
+  ABI_WINAPI,
   INVALID_ABI
 };
 

toolkit/components/ctypes/tests/unit/test_jsctypes.js.in

@@ -666,6 +666,15 @@ function run_basic_abi_tests(library, t, name, toprimitive,
   }
   run_single_abi_tests(declare_fn_stdcall, ctypes.stdcall_abi, t,
     toprimitive, get_test, set_tests, sum_tests, sum_many_tests);
+
+  // Check that declaring a WINAPI function gets the right symbol name.
+  let libuser32 = ctypes.open("user32.dll");
+  let charupper = libuser32.declare("CharUpperA",
+                                    ctypes.winapi_abi,
+                                    ctypes.char.ptr,
+                                    ctypes.char.ptr);
+  let hello = ctypes.char.array()("hello!");
+  do_check_eq(charupper(hello).readString(), "HELLO!");
 #endif
 #endif
 
@@ -1842,6 +1851,9 @@ function run_FunctionType_tests() {
   f3_t = ctypes.FunctionType(ctypes.stdcall_abi,
     ctypes.char.ptr.array().ptr).ptr.ptr.array(8).array();
   do_check_eq(f3_t.name, "char*(*(__stdcall **[][8])())[]");
+  f3_t = ctypes.FunctionType(ctypes.winapi_abi,
+    ctypes.char.ptr.array().ptr).ptr.ptr.array(8).array();
+  do_check_eq(f3_t.name, "char*(*(WINAPI **[][8])())[]");
 #endif
 #endif
 
@@ -2033,6 +2045,11 @@ function run_void_tests(library) {
 #ifndef HAVE_64BIT_OS
   test_void_t = library.declare("test_void_t_stdcall", ctypes.stdcall_abi, ctypes.void_t);
   do_check_eq(test_void_t(), undefined);
+
+  // Check that WINAPI symbol lookup for a regular stdcall function fails.
+  do_check_throws(function() {
+    let test_winapi_t = library.declare("test_void_t_stdcall", ctypes.winapi_abi, ctypes.void_t);
+  }, Error);
 #endif
 #endif
 }
@@ -2186,6 +2203,14 @@ function run_closure_tests(library)
 #ifdef WIN32
 #ifndef HAVE_64BIT_OS
   run_single_closure_tests(library, ctypes.stdcall_abi, "stdcall");
+
+  // Check that attempting to construct a ctypes.winapi_abi closure throws.
+  function closure_fn()
+  {
+    return 1;
+  }
+  let fn_t = ctypes.FunctionType(ctypes.winapi_abi, ctypes.int32_t, []).ptr;
+  do_check_throws(function() { fn_t(closure_fn) }, Error);
 #endif
 #endif
 }
@@ -2252,6 +2277,10 @@ function run_variadic_tests(library) {
       ctypes.FunctionType(ctypes.stdcall_abi, ctypes.bool,
                           [ctypes.bool, "..."]);
   }, Error);
+  do_check_throws(function() {
+      ctypes.FunctionType(ctypes.winapi_abi, ctypes.bool,
+                          [ctypes.bool, "..."]);
+  }, Error);
 #endif
 #endif