2019-06-04 11:11:33 +03:00
|
|
|
/* SPDX-License-Identifier: GPL-2.0-only */
|
2017-03-31 10:55:33 +03:00
|
|
|
/*
|
|
|
|
* Copyright (C) 2017 Linaro, Ltd. <ard.biesheuvel@linaro.org>
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include <linux/linkage.h>
|
|
|
|
|
|
|
|
ENTRY(absolute_data64)
|
|
|
|
ldr x0, 0f
|
|
|
|
ret
|
|
|
|
0: .quad sym64_abs
|
|
|
|
ENDPROC(absolute_data64)
|
|
|
|
|
|
|
|
ENTRY(absolute_data32)
|
|
|
|
ldr w0, 0f
|
|
|
|
ret
|
|
|
|
0: .long sym32_abs
|
|
|
|
ENDPROC(absolute_data32)
|
|
|
|
|
|
|
|
ENTRY(absolute_data16)
|
|
|
|
adr x0, 0f
|
|
|
|
ldrh w0, [x0]
|
|
|
|
ret
|
|
|
|
0: .short sym16_abs, 0
|
|
|
|
ENDPROC(absolute_data16)
|
|
|
|
|
|
|
|
ENTRY(signed_movw)
|
|
|
|
movz x0, #:abs_g2_s:sym64_abs
|
|
|
|
movk x0, #:abs_g1_nc:sym64_abs
|
|
|
|
movk x0, #:abs_g0_nc:sym64_abs
|
|
|
|
ret
|
|
|
|
ENDPROC(signed_movw)
|
|
|
|
|
|
|
|
ENTRY(unsigned_movw)
|
|
|
|
movz x0, #:abs_g3:sym64_abs
|
|
|
|
movk x0, #:abs_g2_nc:sym64_abs
|
|
|
|
movk x0, #:abs_g1_nc:sym64_abs
|
|
|
|
movk x0, #:abs_g0_nc:sym64_abs
|
|
|
|
ret
|
|
|
|
ENDPROC(unsigned_movw)
|
|
|
|
|
arm64/kernel: don't ban ADRP to work around Cortex-A53 erratum #843419
Working around Cortex-A53 erratum #843419 involves special handling of
ADRP instructions that end up in the last two instruction slots of a
4k page, or whose output register gets overwritten without having been
read. (Note that the latter instruction sequence is never emitted by
a properly functioning compiler, which is why it is disregarded by the
handling of the same erratum in the bfd.ld linker which we rely on for
the core kernel)
Normally, this gets taken care of by the linker, which can spot such
sequences at final link time, and insert a veneer if the ADRP ends up
at a vulnerable offset. However, linux kernel modules are partially
linked ELF objects, and so there is no 'final link time' other than the
runtime loading of the module, at which time all the static relocations
are resolved.
For this reason, we have implemented the #843419 workaround for modules
by avoiding ADRP instructions altogether, by using the large C model,
and by passing -mpc-relative-literal-loads to recent versions of GCC
that may emit adrp/ldr pairs to perform literal loads. However, this
workaround forces us to keep literal data mixed with the instructions
in the executable .text segment, and literal data may inadvertently
turn into an exploitable speculative gadget depending on the relative
offsets of arbitrary symbols.
So let's reimplement this workaround in a way that allows us to switch
back to the small C model, and to drop the -mpc-relative-literal-loads
GCC switch, by patching affected ADRP instructions at runtime:
- ADRP instructions that do not appear at 4k relative offset 0xff8 or
0xffc are ignored
- ADRP instructions that are within 1 MB of their target symbol are
converted into ADR instructions
- remaining ADRP instructions are redirected via a veneer that performs
the load using an unaffected movn/movk sequence.
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
[will: tidied up ADRP -> ADR instruction patching.]
[will: use ULL suffix for 64-bit immediate]
Signed-off-by: Will Deacon <will.deacon@arm.com>
2018-03-06 20:15:33 +03:00
|
|
|
.align 12
|
|
|
|
.space 0xff8
|
2017-03-31 10:55:33 +03:00
|
|
|
ENTRY(relative_adrp)
|
|
|
|
adrp x0, sym64_rel
|
|
|
|
add x0, x0, #:lo12:sym64_rel
|
|
|
|
ret
|
|
|
|
ENDPROC(relative_adrp)
|
|
|
|
|
arm64/kernel: don't ban ADRP to work around Cortex-A53 erratum #843419
Working around Cortex-A53 erratum #843419 involves special handling of
ADRP instructions that end up in the last two instruction slots of a
4k page, or whose output register gets overwritten without having been
read. (Note that the latter instruction sequence is never emitted by
a properly functioning compiler, which is why it is disregarded by the
handling of the same erratum in the bfd.ld linker which we rely on for
the core kernel)
Normally, this gets taken care of by the linker, which can spot such
sequences at final link time, and insert a veneer if the ADRP ends up
at a vulnerable offset. However, linux kernel modules are partially
linked ELF objects, and so there is no 'final link time' other than the
runtime loading of the module, at which time all the static relocations
are resolved.
For this reason, we have implemented the #843419 workaround for modules
by avoiding ADRP instructions altogether, by using the large C model,
and by passing -mpc-relative-literal-loads to recent versions of GCC
that may emit adrp/ldr pairs to perform literal loads. However, this
workaround forces us to keep literal data mixed with the instructions
in the executable .text segment, and literal data may inadvertently
turn into an exploitable speculative gadget depending on the relative
offsets of arbitrary symbols.
So let's reimplement this workaround in a way that allows us to switch
back to the small C model, and to drop the -mpc-relative-literal-loads
GCC switch, by patching affected ADRP instructions at runtime:
- ADRP instructions that do not appear at 4k relative offset 0xff8 or
0xffc are ignored
- ADRP instructions that are within 1 MB of their target symbol are
converted into ADR instructions
- remaining ADRP instructions are redirected via a veneer that performs
the load using an unaffected movn/movk sequence.
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
[will: tidied up ADRP -> ADR instruction patching.]
[will: use ULL suffix for 64-bit immediate]
Signed-off-by: Will Deacon <will.deacon@arm.com>
2018-03-06 20:15:33 +03:00
|
|
|
.align 12
|
|
|
|
.space 0xffc
|
|
|
|
ENTRY(relative_adrp_far)
|
2018-03-13 14:41:41 +03:00
|
|
|
adrp x0, memstart_addr
|
|
|
|
add x0, x0, #:lo12:memstart_addr
|
arm64/kernel: don't ban ADRP to work around Cortex-A53 erratum #843419
Working around Cortex-A53 erratum #843419 involves special handling of
ADRP instructions that end up in the last two instruction slots of a
4k page, or whose output register gets overwritten without having been
read. (Note that the latter instruction sequence is never emitted by
a properly functioning compiler, which is why it is disregarded by the
handling of the same erratum in the bfd.ld linker which we rely on for
the core kernel)
Normally, this gets taken care of by the linker, which can spot such
sequences at final link time, and insert a veneer if the ADRP ends up
at a vulnerable offset. However, linux kernel modules are partially
linked ELF objects, and so there is no 'final link time' other than the
runtime loading of the module, at which time all the static relocations
are resolved.
For this reason, we have implemented the #843419 workaround for modules
by avoiding ADRP instructions altogether, by using the large C model,
and by passing -mpc-relative-literal-loads to recent versions of GCC
that may emit adrp/ldr pairs to perform literal loads. However, this
workaround forces us to keep literal data mixed with the instructions
in the executable .text segment, and literal data may inadvertently
turn into an exploitable speculative gadget depending on the relative
offsets of arbitrary symbols.
So let's reimplement this workaround in a way that allows us to switch
back to the small C model, and to drop the -mpc-relative-literal-loads
GCC switch, by patching affected ADRP instructions at runtime:
- ADRP instructions that do not appear at 4k relative offset 0xff8 or
0xffc are ignored
- ADRP instructions that are within 1 MB of their target symbol are
converted into ADR instructions
- remaining ADRP instructions are redirected via a veneer that performs
the load using an unaffected movn/movk sequence.
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
[will: tidied up ADRP -> ADR instruction patching.]
[will: use ULL suffix for 64-bit immediate]
Signed-off-by: Will Deacon <will.deacon@arm.com>
2018-03-06 20:15:33 +03:00
|
|
|
ret
|
|
|
|
ENDPROC(relative_adrp_far)
|
2017-03-31 10:55:33 +03:00
|
|
|
|
|
|
|
ENTRY(relative_adr)
|
|
|
|
adr x0, sym64_rel
|
|
|
|
ret
|
|
|
|
ENDPROC(relative_adr)
|
|
|
|
|
|
|
|
ENTRY(relative_data64)
|
|
|
|
adr x1, 0f
|
|
|
|
ldr x0, [x1]
|
|
|
|
add x0, x0, x1
|
|
|
|
ret
|
|
|
|
0: .quad sym64_rel - .
|
|
|
|
ENDPROC(relative_data64)
|
|
|
|
|
|
|
|
ENTRY(relative_data32)
|
|
|
|
adr x1, 0f
|
|
|
|
ldr w0, [x1]
|
|
|
|
add x0, x0, x1
|
|
|
|
ret
|
|
|
|
0: .long sym64_rel - .
|
|
|
|
ENDPROC(relative_data32)
|
|
|
|
|
|
|
|
ENTRY(relative_data16)
|
|
|
|
adr x1, 0f
|
|
|
|
ldrsh w0, [x1]
|
|
|
|
add x0, x0, x1
|
|
|
|
ret
|
|
|
|
0: .short sym64_rel - ., 0
|
|
|
|
ENDPROC(relative_data16)
|