Bug 1690152 - on ppc64 properly skip parameter slots if we overflow GPRs while still having FPRs to burn. r=tcampbell

Differential Revision: https://phabricator.services.mozilla.com/D103724

xpcom/reflect/xptcall/md/unix/xptcinvoke_ppc64_linux.cpp

@@ -91,7 +91,13 @@ extern "C" void invoke_copy_to_stack(uint64_t* gpregs, double* fpregs,
         if (!s->IsIndirect() && s->type == nsXPTType::T_DOUBLE) {
             if (nr_fpr < FPR_COUNT) {
                 fpregs[nr_fpr++] = s->val.d;
-                nr_gpr++;
+                // Even if we have enough FPRs, still skip space in
+                // the parameter area if we ran out of placeholder GPRs.
+                if (nr_gpr < GPR_COUNT) {
+                    nr_gpr++;
+                } else {
+                    d++;
+                }
             } else {
                 *((double *)d) = s->val.d;
                 d++;
@@ -101,7 +107,11 @@ extern "C" void invoke_copy_to_stack(uint64_t* gpregs, double* fpregs,
             if (nr_fpr < FPR_COUNT) {
                 // Single-precision floats are passed in FPRs too.
                 fpregs[nr_fpr++] = s->val.f;
-                nr_gpr++;
+                if (nr_gpr < GPR_COUNT) {
+                    nr_gpr++;
+                } else {
+                    d++;
+                }
             } else {
 #ifdef __LITTLE_ENDIAN__
                 *((float *)d) = s->val.f;

xpcom/reflect/xptcall/md/unix/xptcstubs_ppc64_linux.cpp

@@ -103,7 +103,13 @@ PrepareAndDispatch(nsXPTCStubBase * self, uint32_t methodIndex,
         if (!param.IsOut() && type == nsXPTType::T_DOUBLE) {
             if (nr_fpr < FPR_COUNT) {
                 dp->val.d = fpregs[nr_fpr++];
-                nr_gpr++;
+                // Even if we have enough FPRs, still skip space in
+                // the parameter area if we ran out of placeholder GPRs.
+                if (nr_gpr < GPR_COUNT) {
+                    nr_gpr++;
+                } else {
+                    ap++;
+                }
             } else {
                 dp->val.d = *(double*)ap++;
             }
@@ -113,7 +119,11 @@ PrepareAndDispatch(nsXPTCStubBase * self, uint32_t methodIndex,
             if (nr_fpr < FPR_COUNT) {
                 // Single-precision floats are passed in FPRs too.
                 dp->val.f = (float)fpregs[nr_fpr++];
-                nr_gpr++;
+                if (nr_gpr < GPR_COUNT) {
+                    nr_gpr++;
+                } else {
+                    ap++;
+                }
             } else {
 #ifdef __LITTLE_ENDIAN__
                 dp->val.f = *(float*)ap++;