[runtime] Fix crash in the x86_64 dynamic registrar. (#5989)

In [this][1] innocuous-looking commit I made the x86_64 dynamic registrar
crash when TRACE is defined. The underlying problem is that CallState's rsi
field is re-used as rax upon exit of the trampoline. This means that the
CallState.sel () function that gets the selector, will instead get the return
value upon exit. When TRACE is defined, we try to print the selector upon
exiting the trampoline, and since the rsi field is now the rax field, we end
up trying to print the return value as if it were a selector, and things go
slightly sideways.

[1]: https://github.com/xamarin/xamarin-macios/commit/464882d14a83

runtime/trampolines-x86_64-asm.s

@@ -4,8 +4,9 @@
 #   %rdi, %rsi, %rdx, %rcx, %r8, %r9
 #   %xmm0-%xmm7
 #   an unknown amount of stack space, but we can pass a pointer to the start of this area.
-# in total we need 6*64bits registers + 8*128bits registers + 1*64bit ptr = 184 bytes.
-# additionally we'll use %r11 to specify the type of trampoline was called, so 192 bytes.
+# in total we need 7*64bits registers + 8*128bits registers + 1*64bit ptr = 192 bytes.
+# additionally we'll use %r11 to specify the type of trampoline was called, so 200 bytes.
+# and finally we need to align the long double fields on a 16-byte boundary, so 208 bytes.
 #
 #
 # upon return we may need to write to:
@@ -26,36 +27,37 @@ _xamarin_x86_64_common_trampoline:
 .cfi_offset %rbp, -16
 	movq	%rsp, %rbp
 .cfi_def_cfa_register %rbp
-	subq	$0xC0, %rsp	# allocate 192 bytes from the stack
+	subq	$0xD0, %rsp	# allocate 208 bytes from the stack
 	# todo: verify alignment.
 	movq	%r11,   (%rsp)
 	movq	%rdi,  8(%rsp)
 	movq	%rsi, 16(%rsp)
 	movq	%rdx, 24(%rsp)
+	movq	%rdx, 72(%rsp) # store a copy of rdx where we read the rdx return value from.
 	movq	%rcx, 32(%rsp)
 	movq	%r8,  40(%rsp)
 	movq	%r9,  48(%rsp)
 	movq	%rbp, 56(%rsp)
-	movaps	%xmm0, 64(%rsp)
-	movaps	%xmm1, 80(%rsp)
-	movaps	%xmm2, 96(%rsp)
-	movaps	%xmm3, 112(%rsp)
-	movaps	%xmm4, 128(%rsp)
-	movaps	%xmm5, 144(%rsp)
-	movaps	%xmm6, 160(%rsp)
-	movaps	%xmm7, 176(%rsp)
+	movaps	%xmm0, 80(%rsp)
+	movaps	%xmm1, 96(%rsp)
+	movaps	%xmm2, 112(%rsp)
+	movaps	%xmm3, 128(%rsp)
+	movaps	%xmm4, 144(%rsp)
+	movaps	%xmm5, 160(%rsp)
+	movaps	%xmm6, 176(%rsp)
+	movaps	%xmm7, 192(%rsp)
 
 	movq	%rsp, %rdi
 	callq	_xamarin_arch_trampoline
 
 	# get return value(s)
 
-	movq	16(%rsp), %rax # offset 16 is used for %rsi on entry and %rax on exit.
-	movq	24(%rsp), %rdx
-	movaps	64(%rsp), %xmm0
-	movaps	80(%rsp), %xmm1
+	movq	64(%rsp), %rax
+	movq	72(%rsp), %rdx
+	movaps	80(%rsp), %xmm0
+	movaps	96(%rsp), %xmm1
 	
-	addq	$0xC0, %rsp	# deallocate 192 bytes from the stack
+	addq	$0xD0, %rsp	# deallocate 208 bytes from the stack
 	popq	%rbp
 		
 	ret

runtime/trampolines-x86_64.h

@@ -8,15 +8,14 @@ extern "C" {
 struct XamarinCallState {
 	uint64_t type;
 	uint64_t rdi;                        // 1st argument
-	union {
-		uint64_t rsi; // rsi upon entry. // 2nd argument
-		uint64_t rax; // rax upon exit.
-	};
+	uint64_t rsi; // 2nd argument
 	uint64_t rdx; // 3rd argument
 	uint64_t rcx; // 4th argument
 	uint64_t r8; // 5th argument
 	uint64_t r9; // 6th argument
 	uint64_t rbp;
+	uint64_t rax;
+	uint64_t rdx_out; // use this field as a temporary rdx field to store output. It makes the marshalling code a bit easier to have this field just after the rax field (so we can treat rax+rdx as a continuous block of 32 bytes of memory)
 	long double xmm0;
 	long double xmm1;
 	long double xmm2;

runtime/trampolines-x86_64.m

@@ -79,8 +79,8 @@ get_primitive_size (char type)
 static void
 dump_state (struct XamarinCallState *state)
 {
-	fprintf (stderr, "type: %llu is_stret: %i self: %p SEL: %s rdi: 0x%llx rsi: 0x%llx rdx: 0x%llx rcx: 0x%llx r8: 0x%llx r9: 0x%llx rbp: 0x%llx -- xmm0: %Lf xmm1: %Lf xmm2: %Lf xmm3: %Lf xmm4: %Lf xmm5: %Lf xmm6: %Lf xmm7: %Lf\n",
-		state->type, state->is_stret (), state->self (), sel_getName (state->sel ()), state->rdi, state->rsi, state->rdx, state->rcx, state->r8, state->r9, state->rbp,
+	fprintf (stderr, "type: %llu is_stret: %i self: %p SEL: %s rdi: 0x%llx rsi: 0x%llx rdx: 0x%llx rcx: 0x%llx r8: 0x%llx r9: 0x%llx rbp: 0x%llx rax: 0x%llx rdx out: 0x%llx -- xmm0: %Lf xmm1: %Lf xmm2: %Lf xmm3: %Lf xmm4: %Lf xmm5: %Lf xmm6: %Lf xmm7: %Lf\n",
+		state->type, state->is_stret (), state->self (), sel_getName (state->sel ()), state->rdi, state->rsi, state->rdx, state->rcx, state->r8, state->r9, state->rbp, state->rax, state->rdx_out,
 		state->xmm0, state->xmm1, state->xmm2, state->xmm3, state->xmm4, state->xmm5, state->xmm6, state->xmm7);
 }
 #else