Don't use a varargs convention for calls unprototyped functions where one of the arguments is an AVX vector.

git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@145574 91177308-0d34-0410-b5e6-96231b3b80d8

lib/CodeGen/CGExpr.cpp

@@ -2458,7 +2458,7 @@ RValue CodeGenFunction::EmitCall(QualType CalleeType, llvm::Value *Callee,
   // call.  The way we make this work is to cast to the exact type
   // of the promoted arguments.
   if (isa<FunctionNoProtoType>(FnType) &&
-      !getTargetHooks().isNoProtoCallVariadic(FnType->getCallConv())) {
+      !getTargetHooks().isNoProtoCallVariadic(FnInfo)) {
     assert(cast<llvm::FunctionType>(Callee->getType()->getContainedType(0))
              ->isVarArg());
     llvm::Type *CalleeTy = getTypes().GetFunctionType(FnInfo, false);

lib/CodeGen/TargetInfo.cpp

@@ -98,7 +98,8 @@ unsigned TargetCodeGenInfo::getSizeOfUnwindException() const {
   return 32;
 }
 
-bool TargetCodeGenInfo::isNoProtoCallVariadic(CallingConv CC) const {
+bool TargetCodeGenInfo::isNoProtoCallVariadic(
+                                       const CodeGen::CGFunctionInfo &) const {
   // The following conventions are known to require this to be false:
   //   x86_stdcall
   //   MIPS
@@ -978,13 +979,31 @@ public:
     return X86AdjustInlineAsmType(CGF, Constraint, Ty);
   }
 
-  bool isNoProtoCallVariadic(CallingConv CC) const {
+  bool isNoProtoCallVariadic(const CodeGen::CGFunctionInfo &FI) const {
     // The default CC on x86-64 sets %al to the number of SSA
     // registers used, and GCC sets this when calling an unprototyped
-    // function, so we override the default behavior.
+    // function, so we override the default behavior.  However, don't do
-    if (CC == CC_Default || CC == CC_C) return true;
+    // that when AVX types are involved.
+    if (FI.getCallingConvention() == llvm::CallingConv::C) {
+      bool HasAVXType = false;
+      for (CGFunctionInfo::const_arg_iterator it = FI.arg_begin(),
+                                              ie = FI.arg_end();
+           it != ie; ++it) {
+        if (it->info.isDirect()) {
+          llvm::Type *Ty = it->info.getCoerceToType();
+          if (llvm::VectorType *VTy = dyn_cast_or_null<llvm::VectorType>(Ty)) {
+            if (VTy->getBitWidth() > 128) {
+              HasAVXType = true;
+              break;
+            }
+          }
+        }
+      }
+      if (!HasAVXType)
+        return true;
+    }
 
-    return TargetCodeGenInfo::isNoProtoCallVariadic(CC);
+    return TargetCodeGenInfo::isNoProtoCallVariadic(FI);
   }
 
 };

lib/CodeGen/TargetInfo.h

@@ -32,6 +32,7 @@ namespace clang {
   namespace CodeGen {
     class CodeGenModule;
     class CodeGenFunction;
+    class CGFunctionInfo;
   }
 
   /// TargetCodeGenInfo - This class organizes various target-specific
@@ -160,7 +161,7 @@ namespace clang {
     /// same way and some out-of-band information is passed for the
     /// benefit of variadic callees, as is the case for x86-64.
     /// In this case the ABI should be consulted.
-    virtual bool isNoProtoCallVariadic(CallingConv CC) const;
+    virtual bool isNoProtoCallVariadic(const CodeGen::CGFunctionInfo &) const;
   };
 }
 

test/CodeGen/x86_64-arguments.c

@@ -326,3 +326,11 @@ int foo(long3 X)
 {
   return 0;
 }
+
+// Make sure we don't use a varargs convention for a function without a
+// prototype where AVX types are involved.
+// CHECK: @test45
+// CHECK: call i32 bitcast (i32 (...)* @f45 to i32 (<8 x float>)*)
+int f45();
+__m256 x45;
+void test45() { f45(x45); }