зеркало из https://github.com/mozilla/gecko-dev.git
Bug 1653502: Update vendored Cranelift to fix fuzzbug. r=bbouvier
This patch pulls in revision 1b3b2dbfd00492161032760992a8699d19b640ca of Cranelift. This includes PR bytecodealliance/wasmtime#2042, which fixes bug 1653502 by properly masking the shift amount in a shift incorporated into an aarch64 arithmetic instruction. This patch also includes various other miscellaneous Cranelift improvements that have been merged since the last version-bump, including some aarch64 codegen improvements. Differential Revision: https://phabricator.services.mozilla.com/D84101
This commit is contained in:
Chris Fallin
Родитель
264fc4a9fa
Коммит
1438721c3a
|
|
@ -65,7 +65,7 @@ rev = "3224e2dee65c0726c448484d4c3c43956b9330ec"
|
|||
[source."https://github.com/bytecodealliance/wasmtime"]
|
||||
git = "https://github.com/bytecodealliance/wasmtime"
|
||||
replace-with = "vendored-sources"
|
||||
rev = "5e0268a542f612fee36d0256ed1f6a0e18dc02b3"
|
||||
rev = "1b3b2dbfd00492161032760992a8699d19b640ca"
|
||||
|
||||
[source."https://github.com/badboy/failure"]
|
||||
git = "https://github.com/badboy/failure"
|
||||
|
|
|
|||
|
|
@ -734,7 +734,7 @@ dependencies = [
|
|||
[[package]]
|
||||
name = "cranelift-bforest"
|
||||
version = "0.66.0"
|
||||
source = "git+https://github.com/bytecodealliance/wasmtime?rev=5e0268a542f612fee36d0256ed1f6a0e18dc02b3#5e0268a542f612fee36d0256ed1f6a0e18dc02b3"
|
||||
source = "git+https://github.com/bytecodealliance/wasmtime?rev=1b3b2dbfd00492161032760992a8699d19b640ca#1b3b2dbfd00492161032760992a8699d19b640ca"
|
||||
dependencies = [
|
||||
"cranelift-entity 0.66.0",
|
||||
]
|
||||
|
|
@ -742,7 +742,7 @@ dependencies = [
|
|||
[[package]]
|
||||
name = "cranelift-codegen"
|
||||
version = "0.66.0"
|
||||
source = "git+https://github.com/bytecodealliance/wasmtime?rev=5e0268a542f612fee36d0256ed1f6a0e18dc02b3#5e0268a542f612fee36d0256ed1f6a0e18dc02b3"
|
||||
source = "git+https://github.com/bytecodealliance/wasmtime?rev=1b3b2dbfd00492161032760992a8699d19b640ca#1b3b2dbfd00492161032760992a8699d19b640ca"
|
||||
dependencies = [
|
||||
"byteorder",
|
||||
"cranelift-bforest",
|
||||
|
|
@ -759,7 +759,7 @@ dependencies = [
|
|||
[[package]]
|
||||
name = "cranelift-codegen-meta"
|
||||
version = "0.66.0"
|
||||
source = "git+https://github.com/bytecodealliance/wasmtime?rev=5e0268a542f612fee36d0256ed1f6a0e18dc02b3#5e0268a542f612fee36d0256ed1f6a0e18dc02b3"
|
||||
source = "git+https://github.com/bytecodealliance/wasmtime?rev=1b3b2dbfd00492161032760992a8699d19b640ca#1b3b2dbfd00492161032760992a8699d19b640ca"
|
||||
dependencies = [
|
||||
"cranelift-codegen-shared",
|
||||
"cranelift-entity 0.66.0",
|
||||
|
|
@ -768,7 +768,7 @@ dependencies = [
|
|||
[[package]]
|
||||
name = "cranelift-codegen-shared"
|
||||
version = "0.66.0"
|
||||
source = "git+https://github.com/bytecodealliance/wasmtime?rev=5e0268a542f612fee36d0256ed1f6a0e18dc02b3#5e0268a542f612fee36d0256ed1f6a0e18dc02b3"
|
||||
source = "git+https://github.com/bytecodealliance/wasmtime?rev=1b3b2dbfd00492161032760992a8699d19b640ca#1b3b2dbfd00492161032760992a8699d19b640ca"
|
||||
|
||||
[[package]]
|
||||
name = "cranelift-entity"
|
||||
|
|
@ -778,12 +778,12 @@ source = "git+https://github.com/PLSysSec/lucet_sandbox_compiler?rev=5e870faf6f9
|
|||
[[package]]
|
||||
name = "cranelift-entity"
|
||||
version = "0.66.0"
|
||||
source = "git+https://github.com/bytecodealliance/wasmtime?rev=5e0268a542f612fee36d0256ed1f6a0e18dc02b3#5e0268a542f612fee36d0256ed1f6a0e18dc02b3"
|
||||
source = "git+https://github.com/bytecodealliance/wasmtime?rev=1b3b2dbfd00492161032760992a8699d19b640ca#1b3b2dbfd00492161032760992a8699d19b640ca"
|
||||
|
||||
[[package]]
|
||||
name = "cranelift-frontend"
|
||||
version = "0.66.0"
|
||||
source = "git+https://github.com/bytecodealliance/wasmtime?rev=5e0268a542f612fee36d0256ed1f6a0e18dc02b3#5e0268a542f612fee36d0256ed1f6a0e18dc02b3"
|
||||
source = "git+https://github.com/bytecodealliance/wasmtime?rev=1b3b2dbfd00492161032760992a8699d19b640ca#1b3b2dbfd00492161032760992a8699d19b640ca"
|
||||
dependencies = [
|
||||
"cranelift-codegen",
|
||||
"log",
|
||||
|
|
@ -794,7 +794,7 @@ dependencies = [
|
|||
[[package]]
|
||||
name = "cranelift-wasm"
|
||||
version = "0.66.0"
|
||||
source = "git+https://github.com/bytecodealliance/wasmtime?rev=5e0268a542f612fee36d0256ed1f6a0e18dc02b3#5e0268a542f612fee36d0256ed1f6a0e18dc02b3"
|
||||
source = "git+https://github.com/bytecodealliance/wasmtime?rev=1b3b2dbfd00492161032760992a8699d19b640ca#1b3b2dbfd00492161032760992a8699d19b640ca"
|
||||
dependencies = [
|
||||
"cranelift-codegen",
|
||||
"cranelift-entity 0.66.0",
|
||||
|
|
|
|||
|
|
@ -74,8 +74,8 @@ failure_derive = { git = "https://github.com/badboy/failure", rev = "64af847bc5f
|
|||
|
||||
[patch.crates-io.cranelift-codegen]
|
||||
git = "https://github.com/bytecodealliance/wasmtime"
|
||||
rev = "5e0268a542f612fee36d0256ed1f6a0e18dc02b3"
|
||||
rev = "1b3b2dbfd00492161032760992a8699d19b640ca"
|
||||
|
||||
[patch.crates-io.cranelift-wasm]
|
||||
git = "https://github.com/bytecodealliance/wasmtime"
|
||||
rev = "5e0268a542f612fee36d0256ed1f6a0e18dc02b3"
|
||||
rev = "1b3b2dbfd00492161032760992a8699d19b640ca"
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
{"files":{"Cargo.toml":"d01629d478557c181b999c1722b6284435f45f04957d7cb55735b9605136a23e","LICENSE":"268872b9816f90fd8e85db5a28d33f8150ebb8dd016653fb39ef1f94f2686bc5","README.md":"b123f056d0d458396679c5f7f2a16d2762af0258fcda4ac14b6655a95e5a0022","src/cdsl/ast.rs":"84a4b7e3301e3249716958a7aa4ea5ba8c6172e3c02f57ee3880504c4433ff19","src/cdsl/cpu_modes.rs":"996e45b374cfe85ac47c8c86c4459fe4c04b3158102b4c63b6ee434d5eed6a9e","src/cdsl/encodings.rs":"d884a564815a03c23369bcf31d13b122ae5ba84d0c80eda9312f0c0a829bf794","src/cdsl/formats.rs":"63e638305aa3ca6dd409ddf0e5e9605eeac1cc2631103e42fc6cbc87703d9b63","src/cdsl/instructions.rs":"a0f5212fa593caf66371f5ee4b15e501939a9407c4663bff6b3ba356b11ca1b4","src/cdsl/isa.rs":"ccabd6848b69eb069c10db61c7e7f86080777495714bb53d03e663c40541be94","src/cdsl/mod.rs":"0aa827923bf4c45e5ee2359573bd863e00f474acd532739f49dcd74a27553882","src/cdsl/operands.rs":"1c3411504de9c83112ff48e0ff1cfbb2e4ba5a9a15c1716f411ef31a4df59899","src/cdsl/recipes.rs":"80b7cd87332229b569e38086ceee8d557e679b9a32ad2e50bdb15c33337c3418","src/cdsl/regs.rs":"466a42a43355fc7623fe5d8e8d330622207a3af6a80cb9367bc0f06e224c9ee0","src/cdsl/settings.rs":"e6fd9a31925743b93b11f09c9c8271bab6aa2430aa053a2601957b4487df7d77","src/cdsl/type_inference.rs":"1efca8a095ffc899b7527bda6b9d9378c73d7283f8dceaa4819e8af599f8be21","src/cdsl/types.rs":"ff764c9e9c29a05677bff6164e7bc25a0c32655052d77ae580536abba8b1713b","src/cdsl/typevar.rs":"5ae9e5453c3aa8b12a37e5579b602162fa9e153b444e89bb89342614b6a5ed13","src/cdsl/xform.rs":"55da0c3f2403147b535ab6ae5d69c623fbe839edecf2a3af1de84420cd58402d","src/default_map.rs":"101bb0282a124f9c921f6bd095f529e8753621450d783c3273b0b0394c2c5c03","src/error.rs":"e9b11b2feb2d867b94c8810fdc5a6c4e0d9131604a0bfa5340ff2639a55100b4","src/gen_binemit.rs":"515e243420b30d1e01f8ea630282d9b6d78a715e1951f3f20392e19a48164442","src/gen_encodings.rs":"f00cded6b68a9b48c9e3cd39a8b6f0ba136f4062c8f8666109158a72c62c3ed1","src/gen_inst.rs":"88532d2e2c9724dde968d6b046927249c33d2037ab3e3fd1bd7ebfa77fe12bc7","src/gen_legalizer.rs":"a5e507eb46649a28252582cfc1907c77c9266fec7f92e959a03258bed7d124e9","src/gen_registers.rs":"a904119ed803c9de24dedd15149a65337ffc168bb1d63df53d7fdebfb5f4b158","src/gen_settings.rs":"f3cc3d31f6cc898f30606caf084f0de220db2d3b1b5e5e4145fa7c9a9a1597e2","src/gen_types.rs":"f6c090e1646a43bf2fe81ae0a7029cc6f7dc6d43285368f56d86c35a21c469a6","src/isa/arm32/mod.rs":"da18cb40c1a0a6b613ddefcc38a5d01d02c95de6f233ebd4ad84fefb992c008b","src/isa/arm64/mod.rs":"3a815eaa478d82b7f8b536b83f9debb6b79ec860f99fea6485f209a836c6939a","src/isa/mod.rs":"be483f9a406f603e69603f9489a41a53ee02aa0ece07f7ca396956dfe3815f71","src/isa/riscv/encodings.rs":"8abb1968d917588bc5fc5f5be6dd66bdec23ac456ba65f8138237c8e891e843c","src/isa/riscv/mod.rs":"a7b461a30bbfbc1e3b33645422ff40d5b1761c30cb5d4a8aa12e9a3b7f7aee51","src/isa/riscv/recipes.rs":"5be3bf7c9ba3c51ece384b7eee75a8f7fa0cbacc6a5babc9d0e1d92a2e54a4c2","src/isa/x86/encodings.rs":"75fa07c819b467857d1f4231123e16e519312c9a58f5e7b9c633b0768fd88b42","src/isa/x86/instructions.rs":"d4d581448f8f7bd5afb033650af0026468eecc6f4184b3bb7c06232bf08c456b","src/isa/x86/legalize.rs":"186c688dd8ac773f2b2c4c1f1cbdb7a66ca13a8ed90c03f87dfe7fdaa12c15b3","src/isa/x86/mod.rs":"31571c281318e6f9bf17680feb96830983f5c1f9811aa4a89736f99f3d9a1831","src/isa/x86/opcodes.rs":"745ef09f4927b5334d68155fa047910ef96311feef7ec20964bb033c3419cd3c","src/isa/x86/recipes.rs":"c63469f430e457554acf1534f6fe8f37b41984d38d272e023aa0d93b778dc993","src/isa/x86/registers.rs":"4be0a45d8acd465c31746b7976124025b06b453e3f6d587f93efb5af0e12b1a8","src/isa/x86/settings.rs":"47a5e9fb3b7917cfe817d56dcc77c0470545e451e0f38a875af0531fbd9b6a58","src/lib.rs":"23259ba28aa8f0b3586e9c60f4e67ae50660369f146f2a94249e8cff7d07b27b","src/shared/entities.rs":"90f774a70e1c2a2e9a553c07a5e80e0fe54cf127434bd83e67274bba4e1a19ba","src/shared/formats.rs":"2f8cbb008778a49b60efac4647dffef654d225823e03ca6272af2678666dc423","src/shared/immediates.rs":"e4a57657f6af9853794804eb41c01204a2c13a632f44f55d90e156a4b98c5f65","src/shared/instructions.rs":"38e65efb654dd39e9929b8506ba94d6214d996d727815829b659fd1a3b73bdce","src/shared/legalize.rs":"e8fd35104c1907c0e9453fb98372373aea20b54af10457156f6abd86929099dc","src/shared/mod.rs":"c219625990bf15507ac1077b349ce20e5312d4e4707426183676d469e78792b7","src/shared/settings.rs":"7800f51d97a95d572310f6c80ded59c1c84cf3ba06f9425f4205f88ac46b4e98","src/shared/types.rs":"4702df132f4b5d70cc9411ec5221ba0b1bd4479252274e0223ae57b6d0331247","src/srcgen.rs":"dcfc159c8599270f17e6a978c4be255abca51556b5ef0da497faec4a4a1e62ce","src/unique_table.rs":"31aa54330ca4786af772d32e8cb6158b6504b88fa93fe177bf0c6cbe545a8d35"},"package":null}
|
||||
{"files":{"Cargo.toml":"d01629d478557c181b999c1722b6284435f45f04957d7cb55735b9605136a23e","LICENSE":"268872b9816f90fd8e85db5a28d33f8150ebb8dd016653fb39ef1f94f2686bc5","README.md":"b123f056d0d458396679c5f7f2a16d2762af0258fcda4ac14b6655a95e5a0022","src/cdsl/ast.rs":"84a4b7e3301e3249716958a7aa4ea5ba8c6172e3c02f57ee3880504c4433ff19","src/cdsl/cpu_modes.rs":"996e45b374cfe85ac47c8c86c4459fe4c04b3158102b4c63b6ee434d5eed6a9e","src/cdsl/encodings.rs":"d884a564815a03c23369bcf31d13b122ae5ba84d0c80eda9312f0c0a829bf794","src/cdsl/formats.rs":"63e638305aa3ca6dd409ddf0e5e9605eeac1cc2631103e42fc6cbc87703d9b63","src/cdsl/instructions.rs":"a0f5212fa593caf66371f5ee4b15e501939a9407c4663bff6b3ba356b11ca1b4","src/cdsl/isa.rs":"ccabd6848b69eb069c10db61c7e7f86080777495714bb53d03e663c40541be94","src/cdsl/mod.rs":"0aa827923bf4c45e5ee2359573bd863e00f474acd532739f49dcd74a27553882","src/cdsl/operands.rs":"1c3411504de9c83112ff48e0ff1cfbb2e4ba5a9a15c1716f411ef31a4df59899","src/cdsl/recipes.rs":"80b7cd87332229b569e38086ceee8d557e679b9a32ad2e50bdb15c33337c3418","src/cdsl/regs.rs":"466a42a43355fc7623fe5d8e8d330622207a3af6a80cb9367bc0f06e224c9ee0","src/cdsl/settings.rs":"e6fd9a31925743b93b11f09c9c8271bab6aa2430aa053a2601957b4487df7d77","src/cdsl/type_inference.rs":"1efca8a095ffc899b7527bda6b9d9378c73d7283f8dceaa4819e8af599f8be21","src/cdsl/types.rs":"50620fb2a6271a7c9126dc30c433a1bf25646a4d84511f5745650aaaec700f42","src/cdsl/typevar.rs":"5ae9e5453c3aa8b12a37e5579b602162fa9e153b444e89bb89342614b6a5ed13","src/cdsl/xform.rs":"55da0c3f2403147b535ab6ae5d69c623fbe839edecf2a3af1de84420cd58402d","src/default_map.rs":"101bb0282a124f9c921f6bd095f529e8753621450d783c3273b0b0394c2c5c03","src/error.rs":"e9b11b2feb2d867b94c8810fdc5a6c4e0d9131604a0bfa5340ff2639a55100b4","src/gen_binemit.rs":"515e243420b30d1e01f8ea630282d9b6d78a715e1951f3f20392e19a48164442","src/gen_encodings.rs":"f00cded6b68a9b48c9e3cd39a8b6f0ba136f4062c8f8666109158a72c62c3ed1","src/gen_inst.rs":"1ff123ab481b48d82e13363043dfc98eaef837bbf6af485b8259c3863550e29c","src/gen_legalizer.rs":"a5e507eb46649a28252582cfc1907c77c9266fec7f92e959a03258bed7d124e9","src/gen_registers.rs":"a904119ed803c9de24dedd15149a65337ffc168bb1d63df53d7fdebfb5f4b158","src/gen_settings.rs":"f3cc3d31f6cc898f30606caf084f0de220db2d3b1b5e5e4145fa7c9a9a1597e2","src/gen_types.rs":"f6c090e1646a43bf2fe81ae0a7029cc6f7dc6d43285368f56d86c35a21c469a6","src/isa/arm32/mod.rs":"da18cb40c1a0a6b613ddefcc38a5d01d02c95de6f233ebd4ad84fefb992c008b","src/isa/arm64/mod.rs":"3a815eaa478d82b7f8b536b83f9debb6b79ec860f99fea6485f209a836c6939a","src/isa/mod.rs":"be483f9a406f603e69603f9489a41a53ee02aa0ece07f7ca396956dfe3815f71","src/isa/riscv/encodings.rs":"8abb1968d917588bc5fc5f5be6dd66bdec23ac456ba65f8138237c8e891e843c","src/isa/riscv/mod.rs":"a7b461a30bbfbc1e3b33645422ff40d5b1761c30cb5d4a8aa12e9a3b7f7aee51","src/isa/riscv/recipes.rs":"5be3bf7c9ba3c51ece384b7eee75a8f7fa0cbacc6a5babc9d0e1d92a2e54a4c2","src/isa/x86/encodings.rs":"e9f1645fec6e4b5cfba9b08cfff70f9d1a5ad3b392f5ee9f40cb1a8669a7c689","src/isa/x86/instructions.rs":"d4d581448f8f7bd5afb033650af0026468eecc6f4184b3bb7c06232bf08c456b","src/isa/x86/legalize.rs":"186c688dd8ac773f2b2c4c1f1cbdb7a66ca13a8ed90c03f87dfe7fdaa12c15b3","src/isa/x86/mod.rs":"31571c281318e6f9bf17680feb96830983f5c1f9811aa4a89736f99f3d9a1831","src/isa/x86/opcodes.rs":"745ef09f4927b5334d68155fa047910ef96311feef7ec20964bb033c3419cd3c","src/isa/x86/recipes.rs":"3a349dea0231dff14c255837e7717904c2e9b350f6fb4ea66a85f5aea7f29184","src/isa/x86/registers.rs":"4be0a45d8acd465c31746b7976124025b06b453e3f6d587f93efb5af0e12b1a8","src/isa/x86/settings.rs":"47a5e9fb3b7917cfe817d56dcc77c0470545e451e0f38a875af0531fbd9b6a58","src/lib.rs":"23259ba28aa8f0b3586e9c60f4e67ae50660369f146f2a94249e8cff7d07b27b","src/shared/entities.rs":"90f774a70e1c2a2e9a553c07a5e80e0fe54cf127434bd83e67274bba4e1a19ba","src/shared/formats.rs":"2f8cbb008778a49b60efac4647dffef654d225823e03ca6272af2678666dc423","src/shared/immediates.rs":"e4a57657f6af9853794804eb41c01204a2c13a632f44f55d90e156a4b98c5f65","src/shared/instructions.rs":"56d3c9c7b112d16f71625c64e95bfac8baab70c5f806160bcb495218a74143ad","src/shared/legalize.rs":"e8fd35104c1907c0e9453fb98372373aea20b54af10457156f6abd86929099dc","src/shared/mod.rs":"c219625990bf15507ac1077b349ce20e5312d4e4707426183676d469e78792b7","src/shared/settings.rs":"7800f51d97a95d572310f6c80ded59c1c84cf3ba06f9425f4205f88ac46b4e98","src/shared/types.rs":"4702df132f4b5d70cc9411ec5221ba0b1bd4479252274e0223ae57b6d0331247","src/srcgen.rs":"dcfc159c8599270f17e6a978c4be255abca51556b5ef0da497faec4a4a1e62ce","src/unique_table.rs":"31aa54330ca4786af772d32e8cb6158b6504b88fa93fe177bf0c6cbe545a8d35"},"package":null}
|
||||
|
|
@ -407,6 +407,8 @@ impl fmt::Debug for VectorType {
|
|||
#[derive(Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub(crate) enum SpecialType {
|
||||
Flag(shared_types::Flag),
|
||||
// FIXME remove once the old style backends are removed.
|
||||
StructArgument,
|
||||
}
|
||||
|
||||
impl SpecialType {
|
||||
|
|
@ -421,6 +423,9 @@ impl SpecialType {
|
|||
"CPU flags representing the result of a floating point comparison. These
|
||||
flags can be tested with a :type:`floatcc` condition code.",
|
||||
),
|
||||
SpecialType::StructArgument => {
|
||||
String::from("After legalization sarg_t arguments will get this type.")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -428,6 +433,7 @@ impl SpecialType {
|
|||
pub fn lane_bits(self) -> u64 {
|
||||
match self {
|
||||
SpecialType::Flag(_) => 0,
|
||||
SpecialType::StructArgument => 0,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -436,6 +442,7 @@ impl SpecialType {
|
|||
match self {
|
||||
SpecialType::Flag(shared_types::Flag::IFlags) => 1,
|
||||
SpecialType::Flag(shared_types::Flag::FFlags) => 2,
|
||||
SpecialType::StructArgument => 3,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -445,6 +452,7 @@ impl fmt::Display for SpecialType {
|
|||
match *self {
|
||||
SpecialType::Flag(shared_types::Flag::IFlags) => write!(f, "iflags"),
|
||||
SpecialType::Flag(shared_types::Flag::FFlags) => write!(f, "fflags"),
|
||||
SpecialType::StructArgument => write!(f, "sarg_t"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -456,6 +464,7 @@ impl fmt::Debug for SpecialType {
|
|||
"{}",
|
||||
match *self {
|
||||
SpecialType::Flag(_) => format!("FlagsType({})", self),
|
||||
SpecialType::StructArgument => format!("StructArgument"),
|
||||
}
|
||||
)
|
||||
}
|
||||
|
|
@ -469,12 +478,14 @@ impl From<shared_types::Flag> for SpecialType {
|
|||
|
||||
pub(crate) struct SpecialTypeIterator {
|
||||
flag_iter: shared_types::FlagIterator,
|
||||
done: bool,
|
||||
}
|
||||
|
||||
impl SpecialTypeIterator {
|
||||
fn new() -> Self {
|
||||
Self {
|
||||
flag_iter: shared_types::FlagIterator::new(),
|
||||
done: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -484,11 +495,16 @@ impl Iterator for SpecialTypeIterator {
|
|||
fn next(&mut self) -> Option<Self::Item> {
|
||||
if let Some(f) = self.flag_iter.next() {
|
||||
Some(SpecialType::from(f))
|
||||
} else {
|
||||
if !self.done {
|
||||
self.done = true;
|
||||
Some(SpecialType::StructArgument)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Reference type is scalar type, but not lane type.
|
||||
#[derive(Clone, Copy, PartialEq, Eq, Hash)]
|
||||
|
|
|
|||
|
|
@ -407,7 +407,11 @@ fn gen_opcodes(all_inst: &AllInstructions, fmt: &mut Formatter) {
|
|||
All instructions from all supported ISAs are present.
|
||||
"#,
|
||||
);
|
||||
fmt.line("#[repr(u16)]");
|
||||
fmt.line("#[derive(Copy, Clone, PartialEq, Eq, Debug, Hash)]");
|
||||
fmt.line(
|
||||
r#"#[cfg_attr(feature = "enable-peepmatic", derive(serde::Serialize, serde::Deserialize))]"#
|
||||
);
|
||||
|
||||
// We explicitly set the discriminant of the first variant to 1, which allows us to take
|
||||
// advantage of the NonZero optimization, meaning that wrapping enums can use the 0
|
||||
|
|
@ -589,6 +593,24 @@ fn gen_opcodes(all_inst: &AllInstructions, fmt: &mut Formatter) {
|
|||
fmt.empty_line();
|
||||
}
|
||||
|
||||
fn gen_try_from(all_inst: &AllInstructions, fmt: &mut Formatter) {
|
||||
fmt.line("impl core::convert::TryFrom<u16> for Opcode {");
|
||||
fmt.indent(|fmt| {
|
||||
fmt.line("type Error = ();");
|
||||
fmt.line("#[inline]");
|
||||
fmt.line("fn try_from(x: u16) -> Result<Self, ()> {");
|
||||
fmt.indent(|fmt| {
|
||||
fmtln!(fmt, "if 0 < x && x <= {} {{", all_inst.len());
|
||||
fmt.indent(|fmt| fmt.line("Ok(unsafe { core::mem::transmute(x) })"));
|
||||
fmt.line("} else {");
|
||||
fmt.indent(|fmt| fmt.line("Err(())"));
|
||||
fmt.line("}");
|
||||
});
|
||||
fmt.line("}");
|
||||
});
|
||||
fmt.line("}");
|
||||
}
|
||||
|
||||
/// Get the value type constraint for an SSA value operand, where
|
||||
/// `ctrl_typevar` is the controlling type variable.
|
||||
///
|
||||
|
|
@ -1147,7 +1169,10 @@ pub(crate) fn generate(
|
|||
gen_instruction_data_impl(&formats, &mut fmt);
|
||||
fmt.empty_line();
|
||||
gen_opcodes(all_inst, &mut fmt);
|
||||
fmt.empty_line();
|
||||
gen_type_constraints(all_inst, &mut fmt);
|
||||
fmt.empty_line();
|
||||
gen_try_from(all_inst, &mut fmt);
|
||||
fmt.update_file(opcode_filename, out_dir)?;
|
||||
|
||||
// Instruction builder.
|
||||
|
|
|
|||
|
|
@ -465,6 +465,7 @@ fn define_moves(e: &mut PerCpuModeEncodings, shared_defs: &SharedDefinitions, r:
|
|||
let sextend = shared.by_name("sextend");
|
||||
let set_pinned_reg = shared.by_name("set_pinned_reg");
|
||||
let uextend = shared.by_name("uextend");
|
||||
let dummy_sarg_t = shared.by_name("dummy_sarg_t");
|
||||
|
||||
// Shorthands for recipes.
|
||||
let rec_copysp = r.template("copysp");
|
||||
|
|
@ -482,6 +483,7 @@ fn define_moves(e: &mut PerCpuModeEncodings, shared_defs: &SharedDefinitions, r:
|
|||
let rec_umr_reg_to_ssa = r.template("umr_reg_to_ssa");
|
||||
let rec_urm_noflags = r.template("urm_noflags");
|
||||
let rec_urm_noflags_abcd = r.template("urm_noflags_abcd");
|
||||
let rec_dummy_sarg_t = r.recipe("dummy_sarg_t");
|
||||
|
||||
// The pinned reg is fixed to a certain value entirely user-controlled, so it generates nothing!
|
||||
e.enc64_rec(get_pinned_reg.bind(I64), rec_get_pinned_reg, 0);
|
||||
|
|
@ -747,6 +749,8 @@ fn define_moves(e: &mut PerCpuModeEncodings, shared_defs: &SharedDefinitions, r:
|
|||
copy_to_ssa.bind(F32),
|
||||
rec_furm_reg_to_ssa.opcodes(&MOVSS_LOAD),
|
||||
);
|
||||
|
||||
e.enc_32_64_rec(dummy_sarg_t, rec_dummy_sarg_t, 0);
|
||||
}
|
||||
|
||||
#[inline(never)]
|
||||
|
|
|
|||
|
|
@ -1270,6 +1270,12 @@ pub(crate) fn define<'shared>(
|
|||
);
|
||||
}
|
||||
|
||||
recipes.add_recipe(
|
||||
EncodingRecipeBuilder::new("dummy_sarg_t", &formats.nullary, 0)
|
||||
.operands_out(vec![Stack::new(gpr)])
|
||||
.emit(""),
|
||||
);
|
||||
|
||||
// XX+rd id with Abs4 function relocation.
|
||||
recipes.add_template_recipe(
|
||||
EncodingRecipeBuilder::new("fnaddr4", &formats.func_addr, 4)
|
||||
|
|
|
|||
|
|
@ -1863,6 +1863,31 @@ pub(crate) fn define(
|
|||
.can_load(true),
|
||||
);
|
||||
|
||||
let Sarg = &TypeVar::new(
|
||||
"Sarg",
|
||||
"Any scalar or vector type with at most 128 lanes",
|
||||
TypeSetBuilder::new()
|
||||
.specials(vec![crate::cdsl::types::SpecialType::StructArgument])
|
||||
.build(),
|
||||
);
|
||||
let sarg_t = &Operand::new("sarg_t", Sarg);
|
||||
|
||||
// FIXME remove once the old style codegen backends are removed.
|
||||
ig.push(
|
||||
Inst::new(
|
||||
"dummy_sarg_t",
|
||||
r#"
|
||||
This creates a sarg_t
|
||||
|
||||
This instruction is internal and should not be created by
|
||||
Cranelift users.
|
||||
"#,
|
||||
&formats.nullary,
|
||||
)
|
||||
.operands_in(vec![])
|
||||
.operands_out(vec![sarg_t]),
|
||||
);
|
||||
|
||||
let src = &Operand::new("src", &imm.regunit);
|
||||
let dst = &Operand::new("dst", &imm.regunit);
|
||||
|
||||
|
|
|
|||
Различия файлов скрыты, потому что одна или несколько строк слишком длинны
|
|
@ -25,8 +25,11 @@ gimli = { version = "0.21.0", default-features = false, features = ["write"], op
|
|||
smallvec = { version = "1.0.0" }
|
||||
thiserror = "1.0.4"
|
||||
byteorder = { version = "1.3.2", default-features = false }
|
||||
peepmatic-runtime = { path = "../peepmatic/crates/runtime", optional = true, version = "0.2.0" }
|
||||
regalloc = { version = "0.0.28" }
|
||||
peepmatic = { path = "../peepmatic", optional = true, version = "0.66.0" }
|
||||
peepmatic-traits = { path = "../peepmatic/crates/traits", optional = true, version = "0.66.0" }
|
||||
peepmatic-runtime = { path = "../peepmatic/crates/runtime", optional = true, version = "0.66.0" }
|
||||
regalloc = "0.0.28"
|
||||
wast = { version = "15.0.0", optional = true }
|
||||
# It is a goal of the cranelift-codegen crate to have minimal external dependencies.
|
||||
# Please don't add any unless they are essential to the task of creating binary
|
||||
# machine code. Integration tests that need external dependencies can be
|
||||
|
|
@ -34,7 +37,6 @@ regalloc = { version = "0.0.28" }
|
|||
|
||||
[build-dependencies]
|
||||
cranelift-codegen-meta = { path = "meta", version = "0.66.0" }
|
||||
peepmatic = { path = "../peepmatic", optional = true, version = "0.66.0" }
|
||||
|
||||
[features]
|
||||
default = ["std", "unwind"]
|
||||
|
|
@ -80,10 +82,10 @@ regalloc-snapshot = ["bincode", "regalloc/enable-serde"]
|
|||
|
||||
# Recompile our optimizations that are written in the `peepmatic` DSL into a
|
||||
# compact finite-state transducer automaton.
|
||||
rebuild-peephole-optimizers = ["peepmatic"]
|
||||
rebuild-peephole-optimizers = ["peepmatic", "peepmatic-traits", "wast"]
|
||||
|
||||
# Enable the use of `peepmatic`-generated peephole optimizers.
|
||||
enable-peepmatic = ["peepmatic-runtime"]
|
||||
enable-peepmatic = ["peepmatic-runtime", "peepmatic-traits", "serde"]
|
||||
|
||||
[badges]
|
||||
maintenance = { status = "experimental" }
|
||||
|
|
|
|||
|
|
@ -90,20 +90,11 @@ fn main() {
|
|||
}
|
||||
|
||||
#[cfg(feature = "rebuild-peephole-optimizers")]
|
||||
rebuild_peephole_optimizers();
|
||||
{
|
||||
std::fs::write(
|
||||
std::path::Path::new(&out_dir).join("CRANELIFT_CODEGEN_PATH"),
|
||||
cur_dir.to_str().unwrap(),
|
||||
)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[cfg(feature = "rebuild-peephole-optimizers")]
|
||||
fn rebuild_peephole_optimizers() {
|
||||
use std::path::Path;
|
||||
|
||||
let source_path = Path::new("src").join("preopt.peepmatic");
|
||||
println!("cargo:rerun-if-changed={}", source_path.display());
|
||||
|
||||
let preopt =
|
||||
peepmatic::compile_file(&source_path).expect("failed to compile `src/preopt.peepmatic`");
|
||||
|
||||
preopt
|
||||
.serialize_to_file(&Path::new("src").join("preopt.serialized"))
|
||||
.expect("failed to serialize peephole optimizer to `src/preopt.serialized`");
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,6 +18,10 @@ pub enum ArgAction {
|
|||
/// Assign the argument to the given location.
|
||||
Assign(ArgumentLoc),
|
||||
|
||||
/// Assign the argument to the given location and change the type to the specified type.
|
||||
/// This is used by [`ArgumentPurpose::StructArgument`].
|
||||
AssignAndChangeType(ArgumentLoc, Type),
|
||||
|
||||
/// Convert the argument, then call again.
|
||||
///
|
||||
/// This action can split an integer type into two smaller integer arguments, or it can split a
|
||||
|
|
@ -119,6 +123,14 @@ pub fn legalize_args<AA: ArgAssigner>(args: &[AbiParam], aa: &mut AA) -> Option<
|
|||
args.to_mut()[argno].location = loc;
|
||||
argno += 1;
|
||||
}
|
||||
// Assign argument to a location, change type to the requested one and move on to the
|
||||
// next one.
|
||||
ArgAction::AssignAndChangeType(loc, ty) => {
|
||||
let arg = &mut args.to_mut()[argno];
|
||||
arg.location = loc;
|
||||
arg.value_type = ty;
|
||||
argno += 1;
|
||||
}
|
||||
// Split this argument into two smaller ones. Then revisit both.
|
||||
ArgAction::Convert(conv) => {
|
||||
debug_assert!(
|
||||
|
|
|
|||
|
|
@ -282,6 +282,9 @@ pub enum ArgumentPurpose {
|
|||
/// A normal user program value passed to or from a function.
|
||||
Normal,
|
||||
|
||||
/// A C struct passed as argument.
|
||||
StructArgument(u32),
|
||||
|
||||
/// Struct return pointer.
|
||||
///
|
||||
/// When a function needs to return more data than will fit in registers, the caller passes a
|
||||
|
|
@ -334,21 +337,19 @@ pub enum ArgumentPurpose {
|
|||
StackLimit,
|
||||
}
|
||||
|
||||
/// Text format names of the `ArgumentPurpose` variants.
|
||||
static PURPOSE_NAMES: [&str; 8] = [
|
||||
"normal",
|
||||
"sret",
|
||||
"link",
|
||||
"fp",
|
||||
"csr",
|
||||
"vmctx",
|
||||
"sigid",
|
||||
"stack_limit",
|
||||
];
|
||||
|
||||
impl fmt::Display for ArgumentPurpose {
|
||||
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
|
||||
f.write_str(PURPOSE_NAMES[*self as usize])
|
||||
f.write_str(match self {
|
||||
Self::Normal => "normal",
|
||||
Self::StructArgument(size) => return write!(f, "sarg({})", size),
|
||||
Self::StructReturn => "sret",
|
||||
Self::Link => "link",
|
||||
Self::FramePointer => "fp",
|
||||
Self::CalleeSaved => "csr",
|
||||
Self::VMContext => "vmctx",
|
||||
Self::SignatureId => "sigid",
|
||||
Self::StackLimit => "stack_limit",
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -364,6 +365,14 @@ impl FromStr for ArgumentPurpose {
|
|||
"vmctx" => Ok(Self::VMContext),
|
||||
"sigid" => Ok(Self::SignatureId),
|
||||
"stack_limit" => Ok(Self::StackLimit),
|
||||
_ if s.starts_with("sarg(") => {
|
||||
if !s.ends_with(")") {
|
||||
return Err(());
|
||||
}
|
||||
// Parse 'sarg(size)'
|
||||
let size: u32 = s["sarg(".len()..s.len() - 1].parse().map_err(|_| ())?;
|
||||
Ok(Self::StructArgument(size))
|
||||
}
|
||||
_ => Err(()),
|
||||
}
|
||||
}
|
||||
|
|
@ -436,16 +445,17 @@ mod tests {
|
|||
#[test]
|
||||
fn argument_purpose() {
|
||||
let all_purpose = [
|
||||
ArgumentPurpose::Normal,
|
||||
ArgumentPurpose::StructReturn,
|
||||
ArgumentPurpose::Link,
|
||||
ArgumentPurpose::FramePointer,
|
||||
ArgumentPurpose::CalleeSaved,
|
||||
ArgumentPurpose::VMContext,
|
||||
ArgumentPurpose::SignatureId,
|
||||
ArgumentPurpose::StackLimit,
|
||||
(ArgumentPurpose::Normal, "normal"),
|
||||
(ArgumentPurpose::StructReturn, "sret"),
|
||||
(ArgumentPurpose::Link, "link"),
|
||||
(ArgumentPurpose::FramePointer, "fp"),
|
||||
(ArgumentPurpose::CalleeSaved, "csr"),
|
||||
(ArgumentPurpose::VMContext, "vmctx"),
|
||||
(ArgumentPurpose::SignatureId, "sigid"),
|
||||
(ArgumentPurpose::StackLimit, "stack_limit"),
|
||||
(ArgumentPurpose::StructArgument(42), "sarg(42)"),
|
||||
];
|
||||
for (&e, &n) in all_purpose.iter().zip(PURPOSE_NAMES.iter()) {
|
||||
for &(e, n) in &all_purpose {
|
||||
assert_eq!(e.to_string(), n);
|
||||
assert_eq!(Ok(e), n.parse());
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,7 +7,9 @@
|
|||
//! directory.
|
||||
|
||||
use alloc::vec::Vec;
|
||||
use core::convert::{TryFrom, TryInto};
|
||||
use core::fmt::{self, Display, Formatter};
|
||||
use core::num::NonZeroU32;
|
||||
use core::ops::{Deref, DerefMut};
|
||||
use core::str::FromStr;
|
||||
|
||||
|
|
@ -69,6 +71,24 @@ impl Opcode {
|
|||
}
|
||||
}
|
||||
|
||||
impl TryFrom<NonZeroU32> for Opcode {
|
||||
type Error = ();
|
||||
|
||||
#[inline]
|
||||
fn try_from(x: NonZeroU32) -> Result<Self, ()> {
|
||||
let x: u16 = x.get().try_into().map_err(|_| ())?;
|
||||
Self::try_from(x)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<Opcode> for NonZeroU32 {
|
||||
#[inline]
|
||||
fn from(op: Opcode) -> NonZeroU32 {
|
||||
let x = op as u8;
|
||||
NonZeroU32::new(x as u32).unwrap()
|
||||
}
|
||||
}
|
||||
|
||||
// This trait really belongs in cranelift-reader where it is used by the `.clif` file parser, but since
|
||||
// it critically depends on the `opcode_name()` function which is needed here anyway, it lives in
|
||||
// this module. This also saves us from running the build script twice to generate code for the two
|
||||
|
|
|
|||
|
|
@ -286,8 +286,8 @@ impl StackSlots {
|
|||
}
|
||||
|
||||
/// Create a stack slot representing an incoming function argument.
|
||||
pub fn make_incoming_arg(&mut self, ty: Type, offset: StackOffset) -> StackSlot {
|
||||
let mut data = StackSlotData::new(StackSlotKind::IncomingArg, ty.bytes());
|
||||
pub fn make_incoming_arg(&mut self, size: u32, offset: StackOffset) -> StackSlot {
|
||||
let mut data = StackSlotData::new(StackSlotKind::IncomingArg, size);
|
||||
debug_assert!(offset <= StackOffset::max_value() - data.size as StackOffset);
|
||||
data.offset = Some(offset);
|
||||
self.push(data)
|
||||
|
|
@ -300,9 +300,7 @@ impl StackSlots {
|
|||
///
|
||||
/// The requested offset is relative to this function's stack pointer immediately before making
|
||||
/// the call.
|
||||
pub fn get_outgoing_arg(&mut self, ty: Type, offset: StackOffset) -> StackSlot {
|
||||
let size = ty.bytes();
|
||||
|
||||
pub fn get_outgoing_arg(&mut self, size: u32, offset: StackOffset) -> StackSlot {
|
||||
// Look for an existing outgoing stack slot with the same offset and size.
|
||||
let inspos = match self.outgoing.binary_search_by_key(&(offset, size), |&ss| {
|
||||
(self[ss].offset.unwrap(), self[ss].size)
|
||||
|
|
@ -387,9 +385,9 @@ mod tests {
|
|||
fn outgoing() {
|
||||
let mut sss = StackSlots::new();
|
||||
|
||||
let ss0 = sss.get_outgoing_arg(types::I32, 8);
|
||||
let ss1 = sss.get_outgoing_arg(types::I32, 4);
|
||||
let ss2 = sss.get_outgoing_arg(types::I64, 8);
|
||||
let ss0 = sss.get_outgoing_arg(4, 8);
|
||||
let ss1 = sss.get_outgoing_arg(4, 4);
|
||||
let ss2 = sss.get_outgoing_arg(8, 8);
|
||||
|
||||
assert_eq!(sss[ss0].offset, Some(8));
|
||||
assert_eq!(sss[ss0].size, 4);
|
||||
|
|
@ -400,9 +398,9 @@ mod tests {
|
|||
assert_eq!(sss[ss2].offset, Some(8));
|
||||
assert_eq!(sss[ss2].size, 8);
|
||||
|
||||
assert_eq!(sss.get_outgoing_arg(types::I32, 8), ss0);
|
||||
assert_eq!(sss.get_outgoing_arg(types::I32, 4), ss1);
|
||||
assert_eq!(sss.get_outgoing_arg(types::I64, 8), ss2);
|
||||
assert_eq!(sss.get_outgoing_arg(4, 8), ss0);
|
||||
assert_eq!(sss.get_outgoing_arg(4, 4), ss1);
|
||||
assert_eq!(sss.get_outgoing_arg(8, 8), ss2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
|
|||
|
|
@ -343,6 +343,7 @@ impl Display for Type {
|
|||
f.write_str(match *self {
|
||||
IFLAGS => "iflags",
|
||||
FFLAGS => "fflags",
|
||||
SARG_T => "sarg_t",
|
||||
INVALID => panic!("INVALID encountered"),
|
||||
_ => panic!("Unknown Type(0x{:x})", self.0),
|
||||
})
|
||||
|
|
|
|||
|
|
@ -52,6 +52,11 @@ impl ShiftOpShiftImm {
|
|||
pub fn value(self) -> u8 {
|
||||
self.0
|
||||
}
|
||||
|
||||
/// Mask down to a given number of bits.
|
||||
pub fn mask(self, bits: u8) -> ShiftOpShiftImm {
|
||||
ShiftOpShiftImm(self.0 & (bits - 1))
|
||||
}
|
||||
}
|
||||
|
||||
/// A shift operator with an amount, guaranteed to be within range.
|
||||
|
|
|
|||
|
|
@ -2065,7 +2065,12 @@ impl MachInst for Inst {
|
|||
Inst::mov(to_reg, from_reg)
|
||||
}
|
||||
|
||||
fn gen_constant(to_reg: Writable<Reg>, value: u64, ty: Type) -> SmallVec<[Inst; 4]> {
|
||||
fn gen_constant<F: FnMut(RegClass, Type) -> Writable<Reg>>(
|
||||
to_reg: Writable<Reg>,
|
||||
value: u64,
|
||||
ty: Type,
|
||||
_alloc_tmp: F,
|
||||
) -> SmallVec<[Inst; 4]> {
|
||||
if ty == F64 {
|
||||
let mut ret = SmallVec::new();
|
||||
ret.push(Inst::load_fp_constant64(to_reg, f64::from_bits(value)));
|
||||
|
|
|
|||
|
|
@ -219,7 +219,11 @@ pub(crate) fn put_input_in_reg<C: LowerCtx<I = Inst>>(
|
|||
};
|
||||
// Generate constants fresh at each use to minimize long-range register pressure.
|
||||
let to_reg = ctx.alloc_tmp(Inst::rc_for_type(ty).unwrap(), ty);
|
||||
for inst in Inst::gen_constant(to_reg, masked, ty).into_iter() {
|
||||
for inst in Inst::gen_constant(to_reg, masked, ty, |reg_class, ty| {
|
||||
ctx.alloc_tmp(reg_class, ty)
|
||||
})
|
||||
.into_iter()
|
||||
{
|
||||
ctx.emit(inst);
|
||||
}
|
||||
to_reg.to_reg()
|
||||
|
|
@ -317,11 +321,15 @@ fn put_input_in_rs<C: LowerCtx<I = Inst>>(
|
|||
|
||||
// Can we get the shift amount as an immediate?
|
||||
if let Some(shiftimm) = input_to_shiftimm(ctx, shift_amt) {
|
||||
let shiftee_bits = ty_bits(ctx.input_ty(insn, 0));
|
||||
if shiftee_bits <= u8::MAX as usize {
|
||||
let shiftimm = shiftimm.mask(shiftee_bits as u8);
|
||||
let reg = put_input_in_reg(ctx, shiftee, narrow_mode);
|
||||
return ResultRS::RegShift(reg, ShiftOpAndAmt::new(ShiftOp::LSL, shiftimm));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ResultRS::Reg(put_input_in_reg(ctx, input, narrow_mode))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1078,8 +1078,24 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
|
|||
// Nothing.
|
||||
}
|
||||
|
||||
Opcode::Select | Opcode::Selectif | Opcode::SelectifSpectreGuard => {
|
||||
let cond = if op == Opcode::Select {
|
||||
Opcode::Select => {
|
||||
let flag_input = inputs[0];
|
||||
let cond = if let Some(icmp_insn) =
|
||||
maybe_input_insn_via_conv(ctx, flag_input, Opcode::Icmp, Opcode::Bint)
|
||||
{
|
||||
let condcode = inst_condcode(ctx.data(icmp_insn)).unwrap();
|
||||
let cond = lower_condcode(condcode);
|
||||
let is_signed = condcode_is_signed(condcode);
|
||||
lower_icmp_or_ifcmp_to_flags(ctx, icmp_insn, is_signed);
|
||||
cond
|
||||
} else if let Some(fcmp_insn) =
|
||||
maybe_input_insn_via_conv(ctx, flag_input, Opcode::Fcmp, Opcode::Bint)
|
||||
{
|
||||
let condcode = inst_fp_condcode(ctx.data(fcmp_insn)).unwrap();
|
||||
let cond = lower_fp_condcode(condcode);
|
||||
lower_fcmp_or_ffcmp_to_flags(ctx, fcmp_insn);
|
||||
cond
|
||||
} else {
|
||||
let (cmp_op, narrow_mode) = if ty_bits(ctx.input_ty(insn, 0)) > 32 {
|
||||
(ALUOp::SubS64, NarrowValueMode::ZeroExtend64)
|
||||
} else {
|
||||
|
|
@ -1095,7 +1111,24 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
|
|||
rm: zero_reg(),
|
||||
});
|
||||
Cond::Ne
|
||||
};
|
||||
|
||||
// csel.cond rd, rn, rm
|
||||
let rd = get_output_reg(ctx, outputs[0]);
|
||||
let rn = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None);
|
||||
let rm = put_input_in_reg(ctx, inputs[2], NarrowValueMode::None);
|
||||
let ty = ctx.output_ty(insn, 0);
|
||||
let bits = ty_bits(ty);
|
||||
if ty_is_float(ty) && bits == 32 {
|
||||
ctx.emit(Inst::FpuCSel32 { cond, rd, rn, rm });
|
||||
} else if ty_is_float(ty) && bits == 64 {
|
||||
ctx.emit(Inst::FpuCSel64 { cond, rd, rn, rm });
|
||||
} else {
|
||||
ctx.emit(Inst::CSel { cond, rd, rn, rm });
|
||||
}
|
||||
}
|
||||
|
||||
Opcode::Selectif | Opcode::SelectifSpectreGuard => {
|
||||
let condcode = inst_condcode(ctx.data(insn)).unwrap();
|
||||
let cond = lower_condcode(condcode);
|
||||
let is_signed = condcode_is_signed(condcode);
|
||||
|
|
@ -1103,8 +1136,6 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
|
|||
// single-def ifcmp.
|
||||
let ifcmp_insn = maybe_input_insn(ctx, inputs[0], Opcode::Ifcmp).unwrap();
|
||||
lower_icmp_or_ifcmp_to_flags(ctx, ifcmp_insn, is_signed);
|
||||
cond
|
||||
};
|
||||
|
||||
// csel.COND rd, rn, rm
|
||||
let rd = get_output_reg(ctx, outputs[0]);
|
||||
|
|
@ -2152,6 +2183,8 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
|
|||
panic!("x86-specific opcode in supposedly arch-neutral IR!");
|
||||
}
|
||||
|
||||
Opcode::DummySargT => unreachable!(),
|
||||
|
||||
Opcode::AvgRound => unimplemented!(),
|
||||
Opcode::Iabs => unimplemented!(),
|
||||
Opcode::Snarrow
|
||||
|
|
|
|||
|
|
@ -957,17 +957,18 @@ fn adjust_stack<C: LowerCtx<I = Inst>>(ctx: &mut C, amount: u64, is_sub: bool) {
|
|||
}
|
||||
|
||||
fn load_stack(mem: impl Into<SyntheticAmode>, into_reg: Writable<Reg>, ty: Type) -> Inst {
|
||||
let ext_mode = match ty {
|
||||
types::B1 | types::B8 | types::I8 => Some(ExtMode::BQ),
|
||||
types::B16 | types::I16 => Some(ExtMode::WQ),
|
||||
types::B32 | types::I32 => Some(ExtMode::LQ),
|
||||
types::B64 | types::I64 => None,
|
||||
types::F32 => todo!("f32 load_stack"),
|
||||
types::F64 => todo!("f64 load_stack"),
|
||||
_ => unimplemented!("load_stack({})", ty),
|
||||
let (is_int, ext_mode) = match ty {
|
||||
types::B1 | types::B8 | types::I8 => (true, Some(ExtMode::BQ)),
|
||||
types::B16 | types::I16 => (true, Some(ExtMode::WQ)),
|
||||
types::B32 | types::I32 => (true, Some(ExtMode::LQ)),
|
||||
types::B64 | types::I64 => (true, None),
|
||||
types::F32 | types::F64 => (false, None),
|
||||
_ => panic!("load_stack({})", ty),
|
||||
};
|
||||
|
||||
let mem = mem.into();
|
||||
|
||||
if is_int {
|
||||
match ext_mode {
|
||||
Some(ext_mode) => Inst::movsx_rm_r(
|
||||
ext_mode,
|
||||
|
|
@ -977,6 +978,19 @@ fn load_stack(mem: impl Into<SyntheticAmode>, into_reg: Writable<Reg>, ty: Type)
|
|||
),
|
||||
None => Inst::mov64_m_r(mem, into_reg, None /* infallible */),
|
||||
}
|
||||
} else {
|
||||
let sse_op = match ty {
|
||||
types::F32 => SseOpcode::Movss,
|
||||
types::F64 => SseOpcode::Movsd,
|
||||
_ => unreachable!(),
|
||||
};
|
||||
Inst::xmm_mov(
|
||||
sse_op,
|
||||
RegMem::mem(mem),
|
||||
into_reg,
|
||||
None, /* infallible */
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
fn store_stack(mem: impl Into<SyntheticAmode>, from_reg: Reg, ty: Type) -> Inst {
|
||||
|
|
@ -993,7 +1007,12 @@ fn store_stack(mem: impl Into<SyntheticAmode>, from_reg: Reg, ty: Type) -> Inst
|
|||
if is_int {
|
||||
Inst::mov_r_m(size, from_reg, mem, /* infallible store */ None)
|
||||
} else {
|
||||
unimplemented!("f32/f64 store_stack");
|
||||
let sse_op = match size {
|
||||
4 => SseOpcode::Movss,
|
||||
8 => SseOpcode::Movsd,
|
||||
_ => unreachable!(),
|
||||
};
|
||||
Inst::xmm_mov_r_m(sse_op, from_reg, mem, /* infallible store */ None)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ use std::string::{String, ToString};
|
|||
|
||||
use regalloc::{RealRegUniverse, Reg, RegClass, RegUsageCollector, RegUsageMapper};
|
||||
|
||||
use crate::ir::condcodes::IntCC;
|
||||
use crate::ir::condcodes::{FloatCC, IntCC};
|
||||
use crate::machinst::*;
|
||||
|
||||
use super::{
|
||||
|
|
@ -194,6 +194,13 @@ impl RegMemImm {
|
|||
Self::Imm { simm32 }
|
||||
}
|
||||
|
||||
/// Asserts that in register mode, the reg class is the one that's expected.
|
||||
pub(crate) fn assert_regclass_is(&self, expected_reg_class: RegClass) {
|
||||
if let Self::Reg { reg } = self {
|
||||
debug_assert_eq!(reg.get_class(), expected_reg_class);
|
||||
}
|
||||
}
|
||||
|
||||
/// Add the regs mentioned by `self` to `collector`.
|
||||
pub(crate) fn get_regs_as_uses(&self, collector: &mut RegUsageCollector) {
|
||||
match self {
|
||||
|
|
@ -234,6 +241,12 @@ impl RegMem {
|
|||
pub(crate) fn mem(addr: impl Into<SyntheticAmode>) -> Self {
|
||||
Self::Mem { addr: addr.into() }
|
||||
}
|
||||
/// Asserts that in register mode, the reg class is the one that's expected.
|
||||
pub(crate) fn assert_regclass_is(&self, expected_reg_class: RegClass) {
|
||||
if let Self::Reg { reg } = self {
|
||||
debug_assert_eq!(reg.get_class(), expected_reg_class);
|
||||
}
|
||||
}
|
||||
/// Add the regs mentioned by `self` to `collector`.
|
||||
pub(crate) fn get_regs_as_uses(&self, collector: &mut RegUsageCollector) {
|
||||
match self {
|
||||
|
|
@ -346,6 +359,7 @@ pub enum SseOpcode {
|
|||
Minsd,
|
||||
Movaps,
|
||||
Movd,
|
||||
Movq,
|
||||
Movss,
|
||||
Movsd,
|
||||
Mulss,
|
||||
|
|
@ -399,6 +413,7 @@ impl SseOpcode {
|
|||
| SseOpcode::Maxsd
|
||||
| SseOpcode::Minsd
|
||||
| SseOpcode::Movd
|
||||
| SseOpcode::Movq
|
||||
| SseOpcode::Movsd
|
||||
| SseOpcode::Mulsd
|
||||
| SseOpcode::Sqrtsd
|
||||
|
|
@ -411,7 +426,7 @@ impl SseOpcode {
|
|||
}
|
||||
}
|
||||
|
||||
/// Returns the src operand size for an instruction
|
||||
/// Returns the src operand size for an instruction.
|
||||
pub(crate) fn src_size(&self) -> u8 {
|
||||
match self {
|
||||
SseOpcode::Movd => 4,
|
||||
|
|
@ -445,6 +460,7 @@ impl fmt::Debug for SseOpcode {
|
|||
SseOpcode::Minsd => "minsd",
|
||||
SseOpcode::Movaps => "movaps",
|
||||
SseOpcode::Movd => "movd",
|
||||
SseOpcode::Movq => "movq",
|
||||
SseOpcode::Movss => "movss",
|
||||
SseOpcode::Movsd => "movsd",
|
||||
SseOpcode::Mulss => "mulss",
|
||||
|
|
@ -620,6 +636,12 @@ pub enum CC {
|
|||
LE = 14,
|
||||
/// > signed
|
||||
NLE = 15,
|
||||
|
||||
/// parity
|
||||
P = 10,
|
||||
|
||||
/// not parity
|
||||
NP = 11,
|
||||
}
|
||||
|
||||
impl CC {
|
||||
|
|
@ -662,6 +684,33 @@ impl CC {
|
|||
|
||||
CC::LE => CC::NLE,
|
||||
CC::NLE => CC::LE,
|
||||
|
||||
CC::P => CC::NP,
|
||||
CC::NP => CC::P,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn from_floatcc(floatcc: FloatCC) -> Self {
|
||||
match floatcc {
|
||||
FloatCC::Ordered => CC::NP,
|
||||
FloatCC::Unordered => CC::P,
|
||||
// Alias for NE
|
||||
FloatCC::NotEqual | FloatCC::OrderedNotEqual => CC::NZ,
|
||||
// Alias for E
|
||||
FloatCC::UnorderedOrEqual => CC::Z,
|
||||
// Alias for A
|
||||
FloatCC::GreaterThan => CC::NBE,
|
||||
// Alias for AE
|
||||
FloatCC::GreaterThanOrEqual => CC::NB,
|
||||
FloatCC::UnorderedOrLessThan => CC::B,
|
||||
FloatCC::UnorderedOrLessThanOrEqual => CC::BE,
|
||||
FloatCC::Equal
|
||||
| FloatCC::LessThan
|
||||
| FloatCC::LessThanOrEqual
|
||||
| FloatCC::UnorderedOrGreaterThan
|
||||
| FloatCC::UnorderedOrGreaterThanOrEqual => unimplemented!(
|
||||
"No single condition code to guarantee ordered. Treat as special case."
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -687,6 +736,8 @@ impl fmt::Debug for CC {
|
|||
CC::NL => "nl",
|
||||
CC::LE => "le",
|
||||
CC::NLE => "nle",
|
||||
CC::P => "p",
|
||||
CC::NP => "np",
|
||||
};
|
||||
write!(fmt, "{}", name)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1183,6 +1183,28 @@ pub(crate) fn emit(
|
|||
}
|
||||
}
|
||||
|
||||
Inst::XmmCmove {
|
||||
is_64,
|
||||
cc,
|
||||
src,
|
||||
dst,
|
||||
} => {
|
||||
let next = sink.get_label();
|
||||
|
||||
// Jump if cc is *not* set.
|
||||
one_way_jmp(sink, cc.invert(), next);
|
||||
|
||||
let op = if *is_64 {
|
||||
SseOpcode::Movsd
|
||||
} else {
|
||||
SseOpcode::Movss
|
||||
};
|
||||
let inst = Inst::xmm_unary_rm_r(op, src.clone(), *dst);
|
||||
inst.emit(sink, flags, state);
|
||||
|
||||
sink.bind_label(next);
|
||||
}
|
||||
|
||||
Inst::Push64 { src } => {
|
||||
match src {
|
||||
RegMemImm::Reg { reg } => {
|
||||
|
|
@ -1462,18 +1484,22 @@ pub(crate) fn emit(
|
|||
sink.bind_label(else_label);
|
||||
}
|
||||
|
||||
Inst::XMM_Mov_RM_R {
|
||||
Inst::XmmUnaryRmR {
|
||||
op,
|
||||
src: src_e,
|
||||
dst: reg_g,
|
||||
srcloc,
|
||||
} => {
|
||||
let rex = RexFlags::clear_w();
|
||||
|
||||
let (prefix, opcode) = match op {
|
||||
SseOpcode::Movaps => (LegacyPrefix::None, 0x0F28),
|
||||
SseOpcode::Movd => (LegacyPrefix::_66, 0x0F6E),
|
||||
SseOpcode::Movsd => (LegacyPrefix::_F2, 0x0F10),
|
||||
SseOpcode::Movss => (LegacyPrefix::_F3, 0x0F10),
|
||||
SseOpcode::Sqrtss => (LegacyPrefix::_F3, 0x0F51),
|
||||
SseOpcode::Sqrtsd => (LegacyPrefix::_F2, 0x0F51),
|
||||
SseOpcode::Cvtss2sd => (LegacyPrefix::_F3, 0x0F5A),
|
||||
SseOpcode::Cvtsd2ss => (LegacyPrefix::_F2, 0x0F5A),
|
||||
_ => unimplemented!("Opcode {:?} not implemented", op),
|
||||
};
|
||||
|
||||
|
|
@ -1489,7 +1515,7 @@ pub(crate) fn emit(
|
|||
}
|
||||
emit_std_reg_mem(sink, prefix, opcode, 2, reg_g.to_reg(), addr, rex);
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
Inst::XMM_RM_R {
|
||||
|
|
@ -1500,13 +1526,20 @@ pub(crate) fn emit(
|
|||
let rex = RexFlags::clear_w();
|
||||
let (prefix, opcode) = match op {
|
||||
SseOpcode::Addss => (LegacyPrefix::_F3, 0x0F58),
|
||||
SseOpcode::Addsd => (LegacyPrefix::_F2, 0x0F58),
|
||||
SseOpcode::Andps => (LegacyPrefix::None, 0x0F54),
|
||||
SseOpcode::Andnps => (LegacyPrefix::None, 0x0F55),
|
||||
SseOpcode::Divss => (LegacyPrefix::_F3, 0x0F5E),
|
||||
SseOpcode::Mulss => (LegacyPrefix::_F3, 0x0F59),
|
||||
SseOpcode::Mulsd => (LegacyPrefix::_F2, 0x0F59),
|
||||
SseOpcode::Orps => (LegacyPrefix::None, 0x0F56),
|
||||
SseOpcode::Subss => (LegacyPrefix::_F3, 0x0F5C),
|
||||
SseOpcode::Sqrtss => (LegacyPrefix::_F3, 0x0F51),
|
||||
SseOpcode::Subsd => (LegacyPrefix::_F2, 0x0F5C),
|
||||
SseOpcode::Minss => (LegacyPrefix::_F3, 0x0F5D),
|
||||
SseOpcode::Minsd => (LegacyPrefix::_F2, 0x0F5D),
|
||||
SseOpcode::Divss => (LegacyPrefix::_F3, 0x0F5E),
|
||||
SseOpcode::Divsd => (LegacyPrefix::_F2, 0x0F5E),
|
||||
SseOpcode::Maxss => (LegacyPrefix::_F3, 0x0F5F),
|
||||
SseOpcode::Maxsd => (LegacyPrefix::_F2, 0x0F5F),
|
||||
_ => unimplemented!("Opcode {:?} not implemented", op),
|
||||
};
|
||||
|
||||
|
|
@ -1521,25 +1554,72 @@ pub(crate) fn emit(
|
|||
}
|
||||
}
|
||||
|
||||
Inst::XMM_Mov_R_M {
|
||||
Inst::Xmm_Mov_R_M {
|
||||
op,
|
||||
src,
|
||||
dst,
|
||||
srcloc,
|
||||
} => {
|
||||
let rex = RexFlags::clear_w();
|
||||
let (prefix, opcode) = match op {
|
||||
SseOpcode::Movd => (LegacyPrefix::_66, 0x0F7E),
|
||||
SseOpcode::Movss => (LegacyPrefix::_F3, 0x0F11),
|
||||
_ => unimplemented!("Emit xmm mov r m"),
|
||||
SseOpcode::Movsd => (LegacyPrefix::_F2, 0x0F11),
|
||||
_ => unimplemented!("Opcode {:?} not implemented", op),
|
||||
};
|
||||
|
||||
let dst = &dst.finalize(state);
|
||||
if let Some(srcloc) = *srcloc {
|
||||
// Register the offset at which the actual load instruction starts.
|
||||
sink.add_trap(srcloc, TrapCode::HeapOutOfBounds);
|
||||
}
|
||||
emit_std_reg_mem(sink, prefix, opcode, 2, *src, dst, rex);
|
||||
emit_std_reg_mem(sink, prefix, opcode, 2, *src, dst, RexFlags::clear_w());
|
||||
}
|
||||
|
||||
Inst::XmmToGpr { op, src, dst } => {
|
||||
let (rex, prefix, opcode) = match op {
|
||||
SseOpcode::Movd => (RexFlags::clear_w(), LegacyPrefix::_66, 0x0F7E),
|
||||
SseOpcode::Movq => (RexFlags::set_w(), LegacyPrefix::_66, 0x0F7E),
|
||||
_ => panic!("unexpected opcode {:?}", op),
|
||||
};
|
||||
emit_std_reg_reg(sink, prefix, opcode, 2, *src, dst.to_reg(), rex);
|
||||
}
|
||||
|
||||
Inst::GprToXmm {
|
||||
op,
|
||||
src: src_e,
|
||||
dst: reg_g,
|
||||
} => {
|
||||
let (rex, prefix, opcode) = match op {
|
||||
SseOpcode::Movd => (RexFlags::clear_w(), LegacyPrefix::_66, 0x0F6E),
|
||||
SseOpcode::Movq => (RexFlags::set_w(), LegacyPrefix::_66, 0x0F6E),
|
||||
_ => panic!("unexpected opcode {:?}", op),
|
||||
};
|
||||
match src_e {
|
||||
RegMem::Reg { reg: reg_e } => {
|
||||
emit_std_reg_reg(sink, prefix, opcode, 2, reg_g.to_reg(), *reg_e, rex);
|
||||
}
|
||||
RegMem::Mem { addr } => {
|
||||
let addr = &addr.finalize(state);
|
||||
emit_std_reg_mem(sink, prefix, opcode, 2, reg_g.to_reg(), addr, rex);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Inst::XMM_Cmp_RM_R { op, src, dst } => {
|
||||
let rex = RexFlags::clear_w();
|
||||
let (prefix, opcode) = match op {
|
||||
SseOpcode::Ucomisd => (LegacyPrefix::_66, 0x0F2E),
|
||||
SseOpcode::Ucomiss => (LegacyPrefix::None, 0x0F2E),
|
||||
_ => unimplemented!("Emit xmm cmp rm r"),
|
||||
};
|
||||
|
||||
match src {
|
||||
RegMem::Reg { reg } => {
|
||||
emit_std_reg_reg(sink, prefix, opcode, 2, *dst, *reg, rex);
|
||||
}
|
||||
RegMem::Mem { addr } => {
|
||||
let addr = &addr.finalize(state);
|
||||
emit_std_reg_mem(sink, prefix, opcode, 2, *dst, addr, rex);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Inst::LoadExtName {
|
||||
|
|
|
|||
|
|
@ -2711,7 +2711,8 @@ fn test_x64_emit() {
|
|||
insns.push((Inst::setcc(CC::NLE, w_rsi), "400F9FC6", "setnle %sil"));
|
||||
insns.push((Inst::setcc(CC::Z, w_r14), "410F94C6", "setz %r14b"));
|
||||
insns.push((Inst::setcc(CC::LE, w_r14), "410F9EC6", "setle %r14b"));
|
||||
|
||||
insns.push((Inst::setcc(CC::P, w_r9), "410F9AC1", "setp %r9b"));
|
||||
insns.push((Inst::setcc(CC::NP, w_r8), "410F9BC0", "setnp %r8b"));
|
||||
// ========================================================
|
||||
// Cmove
|
||||
insns.push((
|
||||
|
|
@ -2877,28 +2878,45 @@ fn test_x64_emit() {
|
|||
));
|
||||
|
||||
// ========================================================
|
||||
// XMM_RM_R
|
||||
// XMM_CMP_RM_R
|
||||
|
||||
insns.push((
|
||||
Inst::xmm_cmp_rm_r(SseOpcode::Ucomiss, RegMem::reg(xmm1), xmm2),
|
||||
"0F2ED1",
|
||||
"ucomiss %xmm1, %xmm2",
|
||||
));
|
||||
|
||||
insns.push((
|
||||
Inst::xmm_cmp_rm_r(SseOpcode::Ucomiss, RegMem::reg(xmm0), xmm9),
|
||||
"440F2EC8",
|
||||
"ucomiss %xmm0, %xmm9",
|
||||
));
|
||||
|
||||
insns.push((
|
||||
Inst::xmm_cmp_rm_r(SseOpcode::Ucomisd, RegMem::reg(xmm13), xmm4),
|
||||
"66410F2EE5",
|
||||
"ucomisd %xmm13, %xmm4",
|
||||
));
|
||||
|
||||
insns.push((
|
||||
Inst::xmm_cmp_rm_r(SseOpcode::Ucomisd, RegMem::reg(xmm11), xmm12),
|
||||
"66450F2EE3",
|
||||
"ucomisd %xmm11, %xmm12",
|
||||
));
|
||||
|
||||
// ========================================================
|
||||
// XMM_RM_R: float binary ops
|
||||
|
||||
insns.push((
|
||||
Inst::xmm_rm_r(SseOpcode::Addss, RegMem::reg(xmm1), w_xmm0),
|
||||
"F30F58C1",
|
||||
"addss %xmm1, %xmm0",
|
||||
));
|
||||
insns.push((
|
||||
Inst::xmm_rm_r(SseOpcode::Subss, RegMem::reg(xmm0), w_xmm1),
|
||||
"F30F5CC8",
|
||||
"subss %xmm0, %xmm1",
|
||||
));
|
||||
insns.push((
|
||||
Inst::xmm_rm_r(SseOpcode::Addss, RegMem::reg(xmm11), w_xmm13),
|
||||
"F3450F58EB",
|
||||
"addss %xmm11, %xmm13",
|
||||
));
|
||||
insns.push((
|
||||
Inst::xmm_rm_r(SseOpcode::Subss, RegMem::reg(xmm12), w_xmm1),
|
||||
"F3410F5CCC",
|
||||
"subss %xmm12, %xmm1",
|
||||
));
|
||||
insns.push((
|
||||
Inst::xmm_rm_r(
|
||||
SseOpcode::Addss,
|
||||
|
|
@ -2908,6 +2926,22 @@ fn test_x64_emit() {
|
|||
"F3410F5844927B",
|
||||
"addss 123(%r10,%rdx,4), %xmm0",
|
||||
));
|
||||
insns.push((
|
||||
Inst::xmm_rm_r(SseOpcode::Addsd, RegMem::reg(xmm15), w_xmm4),
|
||||
"F2410F58E7",
|
||||
"addsd %xmm15, %xmm4",
|
||||
));
|
||||
|
||||
insns.push((
|
||||
Inst::xmm_rm_r(SseOpcode::Subss, RegMem::reg(xmm0), w_xmm1),
|
||||
"F30F5CC8",
|
||||
"subss %xmm0, %xmm1",
|
||||
));
|
||||
insns.push((
|
||||
Inst::xmm_rm_r(SseOpcode::Subss, RegMem::reg(xmm12), w_xmm1),
|
||||
"F3410F5CCC",
|
||||
"subss %xmm12, %xmm1",
|
||||
));
|
||||
insns.push((
|
||||
Inst::xmm_rm_r(
|
||||
SseOpcode::Subss,
|
||||
|
|
@ -2917,86 +2951,157 @@ fn test_x64_emit() {
|
|||
"F3450F5C94C241010000",
|
||||
"subss 321(%r10,%rax,8), %xmm10",
|
||||
));
|
||||
insns.push((
|
||||
Inst::xmm_rm_r(SseOpcode::Subsd, RegMem::reg(xmm5), w_xmm14),
|
||||
"F2440F5CF5",
|
||||
"subsd %xmm5, %xmm14",
|
||||
));
|
||||
|
||||
insns.push((
|
||||
Inst::xmm_rm_r(SseOpcode::Mulss, RegMem::reg(xmm5), w_xmm4),
|
||||
"F30F59E5",
|
||||
"mulss %xmm5, %xmm4",
|
||||
));
|
||||
insns.push((
|
||||
Inst::xmm_rm_r(SseOpcode::Mulsd, RegMem::reg(xmm5), w_xmm4),
|
||||
"F20F59E5",
|
||||
"mulsd %xmm5, %xmm4",
|
||||
));
|
||||
|
||||
insns.push((
|
||||
Inst::xmm_rm_r(SseOpcode::Divss, RegMem::reg(xmm8), w_xmm7),
|
||||
"F3410F5EF8",
|
||||
"divss %xmm8, %xmm7",
|
||||
));
|
||||
insns.push((
|
||||
Inst::xmm_rm_r(SseOpcode::Sqrtss, RegMem::reg(xmm7), w_xmm8),
|
||||
"F3440F51C7",
|
||||
"sqrtss %xmm7, %xmm8",
|
||||
Inst::xmm_rm_r(SseOpcode::Divsd, RegMem::reg(xmm5), w_xmm4),
|
||||
"F20F5EE5",
|
||||
"divsd %xmm5, %xmm4",
|
||||
));
|
||||
|
||||
insns.push((
|
||||
Inst::xmm_rm_r(SseOpcode::Andps, RegMem::reg(xmm3), w_xmm12),
|
||||
"440F54E3",
|
||||
"andps %xmm3, %xmm12",
|
||||
));
|
||||
|
||||
insns.push((
|
||||
Inst::xmm_rm_r(SseOpcode::Andnps, RegMem::reg(xmm4), w_xmm11),
|
||||
"440F55DC",
|
||||
"andnps %xmm4, %xmm11",
|
||||
));
|
||||
insns.push((
|
||||
Inst::xmm_mov_rm_r(SseOpcode::Movaps, RegMem::reg(xmm5), w_xmm14, None),
|
||||
"440F28F5",
|
||||
"movaps %xmm5, %xmm14",
|
||||
));
|
||||
insns.push((
|
||||
Inst::xmm_mov_rm_r(SseOpcode::Movd, RegMem::reg(rax), w_xmm15, None),
|
||||
"66440F6EF8",
|
||||
"movd %eax, %xmm15",
|
||||
));
|
||||
|
||||
insns.push((
|
||||
Inst::xmm_rm_r(SseOpcode::Orps, RegMem::reg(xmm1), w_xmm15),
|
||||
"440F56F9",
|
||||
"orps %xmm1, %xmm15",
|
||||
));
|
||||
|
||||
insns.push((
|
||||
Inst::xmm_mov_r_m(SseOpcode::Movd, xmm0, Amode::imm_reg(321, rbx), None),
|
||||
"660F7E8341010000",
|
||||
"movd %xmm0, 321(%rbx)",
|
||||
));
|
||||
|
||||
insns.push((
|
||||
Inst::xmm_mov_r_m(SseOpcode::Movss, xmm15, Amode::imm_reg(128, r12), None),
|
||||
"F3450F11BC2480000000",
|
||||
"movss %xmm15, 128(%r12)",
|
||||
));
|
||||
|
||||
insns.push((
|
||||
Inst::xmm_mov_rm_r(
|
||||
SseOpcode::Movd,
|
||||
RegMem::mem(Amode::imm_reg(2, r10)),
|
||||
w_xmm9,
|
||||
None,
|
||||
),
|
||||
"66450F6E4A02",
|
||||
"movd 2(%r10), %xmm9",
|
||||
));
|
||||
|
||||
insns.push((
|
||||
Inst::xmm_rm_r(SseOpcode::Orps, RegMem::reg(xmm5), w_xmm4),
|
||||
"0F56E5",
|
||||
"orps %xmm5, %xmm4",
|
||||
));
|
||||
|
||||
// XMM_Mov_R_M: float stores
|
||||
insns.push((
|
||||
Inst::xmm_mov_rm_r(SseOpcode::Movss, RegMem::reg(xmm13), w_xmm2, None),
|
||||
Inst::xmm_mov_r_m(SseOpcode::Movss, xmm15, Amode::imm_reg(128, r12), None),
|
||||
"F3450F11BC2480000000",
|
||||
"movss %xmm15, 128(%r12)",
|
||||
));
|
||||
insns.push((
|
||||
Inst::xmm_mov_r_m(SseOpcode::Movsd, xmm1, Amode::imm_reg(0, rsi), None),
|
||||
"F20F110E",
|
||||
"movsd %xmm1, 0(%rsi)",
|
||||
));
|
||||
|
||||
// XmmUnary: moves and unary float ops
|
||||
insns.push((
|
||||
Inst::xmm_unary_rm_r(SseOpcode::Movss, RegMem::reg(xmm13), w_xmm2),
|
||||
"F3410F10D5",
|
||||
"movss %xmm13, %xmm2",
|
||||
));
|
||||
|
||||
insns.push((
|
||||
Inst::xmm_mov_rm_r(SseOpcode::Movsd, RegMem::reg(xmm14), w_xmm3, None),
|
||||
Inst::xmm_unary_rm_r(SseOpcode::Movsd, RegMem::reg(xmm0), w_xmm1),
|
||||
"F20F10C8",
|
||||
"movsd %xmm0, %xmm1",
|
||||
));
|
||||
insns.push((
|
||||
Inst::xmm_unary_rm_r(
|
||||
SseOpcode::Movsd,
|
||||
RegMem::mem(Amode::imm_reg(0, rsi)),
|
||||
w_xmm2,
|
||||
),
|
||||
"F20F1016",
|
||||
"movsd 0(%rsi), %xmm2",
|
||||
));
|
||||
insns.push((
|
||||
Inst::xmm_unary_rm_r(SseOpcode::Movsd, RegMem::reg(xmm14), w_xmm3),
|
||||
"F2410F10DE",
|
||||
"movsd %xmm14, %xmm3",
|
||||
));
|
||||
|
||||
insns.push((
|
||||
Inst::xmm_unary_rm_r(SseOpcode::Movaps, RegMem::reg(xmm5), w_xmm14),
|
||||
"440F28F5",
|
||||
"movaps %xmm5, %xmm14",
|
||||
));
|
||||
|
||||
insns.push((
|
||||
Inst::xmm_unary_rm_r(SseOpcode::Sqrtss, RegMem::reg(xmm7), w_xmm8),
|
||||
"F3440F51C7",
|
||||
"sqrtss %xmm7, %xmm8",
|
||||
));
|
||||
insns.push((
|
||||
Inst::xmm_unary_rm_r(SseOpcode::Sqrtsd, RegMem::reg(xmm1), w_xmm2),
|
||||
"F20F51D1",
|
||||
"sqrtsd %xmm1, %xmm2",
|
||||
));
|
||||
|
||||
insns.push((
|
||||
Inst::xmm_unary_rm_r(SseOpcode::Cvtss2sd, RegMem::reg(xmm0), w_xmm1),
|
||||
"F30F5AC8",
|
||||
"cvtss2sd %xmm0, %xmm1",
|
||||
));
|
||||
insns.push((
|
||||
Inst::xmm_unary_rm_r(SseOpcode::Cvtsd2ss, RegMem::reg(xmm1), w_xmm0),
|
||||
"F20F5AC1",
|
||||
"cvtsd2ss %xmm1, %xmm0",
|
||||
));
|
||||
|
||||
// Xmm to int conversions, and conversely.
|
||||
|
||||
insns.push((
|
||||
Inst::xmm_to_gpr(SseOpcode::Movd, xmm0, w_rsi),
|
||||
"660F7EC6",
|
||||
"movd %xmm0, %esi",
|
||||
));
|
||||
insns.push((
|
||||
Inst::xmm_to_gpr(SseOpcode::Movq, xmm2, w_rdi),
|
||||
"66480F7ED7",
|
||||
"movq %xmm2, %rdi",
|
||||
));
|
||||
insns.push((
|
||||
Inst::gpr_to_xmm(SseOpcode::Movd, RegMem::reg(rax), w_xmm15),
|
||||
"66440F6EF8",
|
||||
"movd %eax, %xmm15",
|
||||
));
|
||||
insns.push((
|
||||
Inst::gpr_to_xmm(SseOpcode::Movd, RegMem::mem(Amode::imm_reg(2, r10)), w_xmm9),
|
||||
"66450F6E4A02",
|
||||
"movd 2(%r10), %xmm9",
|
||||
));
|
||||
insns.push((
|
||||
Inst::gpr_to_xmm(SseOpcode::Movd, RegMem::reg(rsi), w_xmm1),
|
||||
"660F6ECE",
|
||||
"movd %esi, %xmm1",
|
||||
));
|
||||
insns.push((
|
||||
Inst::gpr_to_xmm(SseOpcode::Movq, RegMem::reg(rdi), w_xmm15),
|
||||
"664C0F6EFF",
|
||||
"movq %rdi, %xmm15",
|
||||
));
|
||||
|
||||
// ========================================================
|
||||
// Misc instructions.
|
||||
|
||||
|
|
|
|||
|
|
@ -190,18 +190,20 @@ pub enum Inst {
|
|||
|
||||
// =====================================
|
||||
// Floating-point operations.
|
||||
/// Float arithmetic/bit-twiddling: (add sub and or xor mul adc? sbb?) (32 64) (reg addr) reg
|
||||
/// XMM (scalar or vector) binary op: (add sub and or xor mul adc? sbb?) (32 64) (reg addr) reg
|
||||
XMM_RM_R {
|
||||
op: SseOpcode,
|
||||
src: RegMem,
|
||||
dst: Writable<Reg>,
|
||||
},
|
||||
|
||||
/// mov between XMM registers (32 64) (reg addr) reg XMM_Mov_RM_R differs from XMM_RM_R in
|
||||
/// that the dst register of XMM_MOV_RM_R is not used in the computation of the instruction
|
||||
/// dst value and so does not have to be a previously valid value. This is characteristic of
|
||||
/// mov instructions.
|
||||
XMM_Mov_RM_R {
|
||||
/// XMM (scalar or vector) unary op: mov between XMM registers (32 64) (reg addr) reg, sqrt,
|
||||
/// etc.
|
||||
///
|
||||
/// This differs from XMM_RM_R in that the dst register of XmmUnaryRmR is not used in the
|
||||
/// computation of the instruction dst value and so does not have to be a previously valid
|
||||
/// value. This is characteristic of mov instructions.
|
||||
XmmUnaryRmR {
|
||||
op: SseOpcode,
|
||||
src: RegMem,
|
||||
dst: Writable<Reg>,
|
||||
|
|
@ -209,8 +211,8 @@ pub enum Inst {
|
|||
srcloc: Option<SourceLoc>,
|
||||
},
|
||||
|
||||
/// mov reg addr (good for all memory stores from xmm registers)
|
||||
XMM_Mov_R_M {
|
||||
/// XMM (scalar or vector) unary op (from xmm to reg/mem): stores, movd, movq
|
||||
Xmm_Mov_R_M {
|
||||
op: SseOpcode,
|
||||
src: Reg,
|
||||
dst: SyntheticAmode,
|
||||
|
|
@ -218,6 +220,37 @@ pub enum Inst {
|
|||
srcloc: Option<SourceLoc>,
|
||||
},
|
||||
|
||||
/// XMM (scalar) unary op (from xmm to integer reg): movd, movq
|
||||
XmmToGpr {
|
||||
op: SseOpcode,
|
||||
src: Reg,
|
||||
dst: Writable<Reg>,
|
||||
},
|
||||
|
||||
/// XMM (scalar) unary op (from integer to float reg): movd, movq
|
||||
GprToXmm {
|
||||
op: SseOpcode,
|
||||
src: RegMem,
|
||||
dst: Writable<Reg>,
|
||||
},
|
||||
|
||||
/// XMM (scalar) conditional move.
|
||||
/// Overwrites the destination register if cc is set.
|
||||
XmmCmove {
|
||||
/// Whether the cmove is moving either 32 or 64 bits.
|
||||
is_64: bool,
|
||||
cc: CC,
|
||||
src: RegMem,
|
||||
dst: Writable<Reg>,
|
||||
},
|
||||
|
||||
/// Float comparisons/tests: cmp (b w l q) (reg addr imm) reg.
|
||||
XMM_Cmp_RM_R {
|
||||
op: SseOpcode,
|
||||
src: RegMem,
|
||||
dst: Reg,
|
||||
},
|
||||
|
||||
// =====================================
|
||||
// Control flow instructions.
|
||||
/// Direct call: call simm32.
|
||||
|
|
@ -318,6 +351,7 @@ impl Inst {
|
|||
src: RegMemImm,
|
||||
dst: Writable<Reg>,
|
||||
) -> Self {
|
||||
src.assert_regclass_is(RegClass::I64);
|
||||
debug_assert!(dst.to_reg().get_class() == RegClass::I64);
|
||||
Self::Alu_RMI_R {
|
||||
is_64,
|
||||
|
|
@ -333,12 +367,14 @@ impl Inst {
|
|||
src: RegMem,
|
||||
dst: Writable<Reg>,
|
||||
) -> Self {
|
||||
src.assert_regclass_is(RegClass::I64);
|
||||
debug_assert!(dst.to_reg().get_class() == RegClass::I64);
|
||||
debug_assert!(size == 8 || size == 4 || size == 2);
|
||||
Self::UnaryRmR { size, op, src, dst }
|
||||
}
|
||||
|
||||
pub(crate) fn div(size: u8, signed: bool, divisor: RegMem, loc: SourceLoc) -> Inst {
|
||||
divisor.assert_regclass_is(RegClass::I64);
|
||||
debug_assert!(size == 8 || size == 4 || size == 2 || size == 1);
|
||||
Inst::Div {
|
||||
size,
|
||||
|
|
@ -349,6 +385,7 @@ impl Inst {
|
|||
}
|
||||
|
||||
pub(crate) fn mul_hi(size: u8, signed: bool, rhs: RegMem) -> Inst {
|
||||
rhs.assert_regclass_is(RegClass::I64);
|
||||
debug_assert!(size == 8 || size == 4 || size == 2 || size == 1);
|
||||
Inst::MulHi { size, signed, rhs }
|
||||
}
|
||||
|
|
@ -374,20 +411,30 @@ impl Inst {
|
|||
}
|
||||
}
|
||||
|
||||
pub(crate) fn imm32_r_unchecked(simm64: u64, dst: Writable<Reg>) -> Inst {
|
||||
debug_assert!(dst.to_reg().get_class() == RegClass::I64);
|
||||
Inst::Imm_R {
|
||||
dst_is_64: false,
|
||||
simm64,
|
||||
dst,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn mov_r_r(is_64: bool, src: Reg, dst: Writable<Reg>) -> Inst {
|
||||
debug_assert!(src.get_class() == RegClass::I64);
|
||||
debug_assert!(dst.to_reg().get_class() == RegClass::I64);
|
||||
Inst::Mov_R_R { is_64, src, dst }
|
||||
}
|
||||
|
||||
pub(crate) fn xmm_mov_rm_r(
|
||||
pub(crate) fn xmm_mov(
|
||||
op: SseOpcode,
|
||||
src: RegMem,
|
||||
dst: Writable<Reg>,
|
||||
srcloc: Option<SourceLoc>,
|
||||
) -> Inst {
|
||||
src.assert_regclass_is(RegClass::V128);
|
||||
debug_assert!(dst.to_reg().get_class() == RegClass::V128);
|
||||
Inst::XMM_Mov_RM_R {
|
||||
Inst::XmmUnaryRmR {
|
||||
op,
|
||||
src,
|
||||
dst,
|
||||
|
|
@ -395,7 +442,20 @@ impl Inst {
|
|||
}
|
||||
}
|
||||
|
||||
/// Convenient helper for unary float operations.
|
||||
pub(crate) fn xmm_unary_rm_r(op: SseOpcode, src: RegMem, dst: Writable<Reg>) -> Inst {
|
||||
src.assert_regclass_is(RegClass::V128);
|
||||
debug_assert!(dst.to_reg().get_class() == RegClass::V128);
|
||||
Inst::XmmUnaryRmR {
|
||||
op,
|
||||
src,
|
||||
dst,
|
||||
srcloc: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn xmm_rm_r(op: SseOpcode, src: RegMem, dst: Writable<Reg>) -> Self {
|
||||
src.assert_regclass_is(RegClass::V128);
|
||||
debug_assert!(dst.to_reg().get_class() == RegClass::V128);
|
||||
Inst::XMM_RM_R { op, src, dst }
|
||||
}
|
||||
|
|
@ -407,7 +467,7 @@ impl Inst {
|
|||
srcloc: Option<SourceLoc>,
|
||||
) -> Inst {
|
||||
debug_assert!(src.get_class() == RegClass::V128);
|
||||
Inst::XMM_Mov_R_M {
|
||||
Inst::Xmm_Mov_R_M {
|
||||
op,
|
||||
src,
|
||||
dst: dst.into(),
|
||||
|
|
@ -415,12 +475,31 @@ impl Inst {
|
|||
}
|
||||
}
|
||||
|
||||
pub(crate) fn xmm_to_gpr(op: SseOpcode, src: Reg, dst: Writable<Reg>) -> Inst {
|
||||
debug_assert!(src.get_class() == RegClass::V128);
|
||||
debug_assert!(dst.to_reg().get_class() == RegClass::I64);
|
||||
Inst::XmmToGpr { op, src, dst }
|
||||
}
|
||||
|
||||
pub(crate) fn gpr_to_xmm(op: SseOpcode, src: RegMem, dst: Writable<Reg>) -> Inst {
|
||||
src.assert_regclass_is(RegClass::I64);
|
||||
debug_assert!(dst.to_reg().get_class() == RegClass::V128);
|
||||
Inst::GprToXmm { op, src, dst }
|
||||
}
|
||||
|
||||
pub(crate) fn xmm_cmp_rm_r(op: SseOpcode, src: RegMem, dst: Reg) -> Inst {
|
||||
//TODO:: Add assert_reg_type helper
|
||||
debug_assert!(dst.get_class() == RegClass::V128);
|
||||
Inst::XMM_Cmp_RM_R { op, src, dst }
|
||||
}
|
||||
|
||||
pub(crate) fn movzx_rm_r(
|
||||
ext_mode: ExtMode,
|
||||
src: RegMem,
|
||||
dst: Writable<Reg>,
|
||||
srcloc: Option<SourceLoc>,
|
||||
) -> Inst {
|
||||
src.assert_regclass_is(RegClass::I64);
|
||||
debug_assert!(dst.to_reg().get_class() == RegClass::I64);
|
||||
Inst::MovZX_RM_R {
|
||||
ext_mode,
|
||||
|
|
@ -436,6 +515,7 @@ impl Inst {
|
|||
dst: Writable<Reg>,
|
||||
srcloc: Option<SourceLoc>,
|
||||
) -> Inst {
|
||||
src.assert_regclass_is(RegClass::I64);
|
||||
debug_assert!(dst.to_reg().get_class() == RegClass::I64);
|
||||
Inst::MovSX_RM_R {
|
||||
ext_mode,
|
||||
|
|
@ -460,6 +540,7 @@ impl Inst {
|
|||
|
||||
/// A convenience function to be able to use a RegMem as the source of a move.
|
||||
pub(crate) fn mov64_rm_r(src: RegMem, dst: Writable<Reg>, srcloc: Option<SourceLoc>) -> Inst {
|
||||
src.assert_regclass_is(RegClass::I64);
|
||||
match src {
|
||||
RegMem::Reg { reg } => Self::mov_r_r(true, reg, dst),
|
||||
RegMem::Mem { addr } => Self::mov64_m_r(addr, dst, srcloc),
|
||||
|
|
@ -517,6 +598,7 @@ impl Inst {
|
|||
src: RegMemImm,
|
||||
dst: Reg,
|
||||
) -> Inst {
|
||||
src.assert_regclass_is(RegClass::I64);
|
||||
debug_assert!(size == 8 || size == 4 || size == 2 || size == 1);
|
||||
debug_assert!(dst.get_class() == RegClass::I64);
|
||||
Inst::Cmp_RMI_R { size, src, dst }
|
||||
|
|
@ -539,11 +621,24 @@ impl Inst {
|
|||
Inst::Cmove { size, cc, src, dst }
|
||||
}
|
||||
|
||||
pub(crate) fn xmm_cmove(is_64: bool, cc: CC, src: RegMem, dst: Writable<Reg>) -> Inst {
|
||||
src.assert_regclass_is(RegClass::V128);
|
||||
debug_assert!(dst.to_reg().get_class() == RegClass::V128);
|
||||
Inst::XmmCmove {
|
||||
is_64,
|
||||
cc,
|
||||
src,
|
||||
dst,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn push64(src: RegMemImm) -> Inst {
|
||||
src.assert_regclass_is(RegClass::I64);
|
||||
Inst::Push64 { src }
|
||||
}
|
||||
|
||||
pub(crate) fn pop64(dst: Writable<Reg>) -> Inst {
|
||||
debug_assert!(dst.to_reg().get_class() == RegClass::I64);
|
||||
Inst::Pop64 { dst }
|
||||
}
|
||||
|
||||
|
|
@ -570,6 +665,7 @@ impl Inst {
|
|||
loc: SourceLoc,
|
||||
opcode: Opcode,
|
||||
) -> Inst {
|
||||
dest.assert_regclass_is(RegClass::I64);
|
||||
Inst::CallUnknown {
|
||||
dest,
|
||||
uses,
|
||||
|
|
@ -600,6 +696,7 @@ impl Inst {
|
|||
}
|
||||
|
||||
pub(crate) fn jmp_unknown(target: RegMem) -> Inst {
|
||||
target.assert_regclass_is(RegClass::I64);
|
||||
Inst::JmpUnknown { target }
|
||||
}
|
||||
|
||||
|
|
@ -689,6 +786,7 @@ impl ShowWithRRU for Inst {
|
|||
}),
|
||||
divisor.show_rru_sized(mb_rru, *size)
|
||||
),
|
||||
|
||||
Inst::MulHi {
|
||||
size, signed, rhs, ..
|
||||
} => format!(
|
||||
|
|
@ -700,6 +798,7 @@ impl ShowWithRRU for Inst {
|
|||
}),
|
||||
rhs.show_rru_sized(mb_rru, *size)
|
||||
),
|
||||
|
||||
Inst::CheckedDivOrRemSeq {
|
||||
kind,
|
||||
size,
|
||||
|
|
@ -715,6 +814,7 @@ impl ShowWithRRU for Inst {
|
|||
},
|
||||
show_ireg_sized(*divisor, mb_rru, *size),
|
||||
),
|
||||
|
||||
Inst::SignExtendRaxRdx { size } => match size {
|
||||
2 => "cwd",
|
||||
4 => "cdq",
|
||||
|
|
@ -722,24 +822,62 @@ impl ShowWithRRU for Inst {
|
|||
_ => unreachable!(),
|
||||
}
|
||||
.into(),
|
||||
Inst::XMM_Mov_RM_R { op, src, dst, .. } => format!(
|
||||
|
||||
Inst::XmmUnaryRmR { op, src, dst, .. } => format!(
|
||||
"{} {}, {}",
|
||||
ljustify(op.to_string()),
|
||||
src.show_rru_sized(mb_rru, op.src_size()),
|
||||
show_ireg_sized(dst.to_reg(), mb_rru, 8),
|
||||
),
|
||||
Inst::XMM_Mov_R_M { op, src, dst, .. } => format!(
|
||||
|
||||
Inst::Xmm_Mov_R_M { op, src, dst, .. } => format!(
|
||||
"{} {}, {}",
|
||||
ljustify(op.to_string()),
|
||||
show_ireg_sized(*src, mb_rru, 8),
|
||||
dst.show_rru(mb_rru)
|
||||
dst.show_rru(mb_rru),
|
||||
),
|
||||
|
||||
Inst::XMM_RM_R { op, src, dst } => format!(
|
||||
"{} {}, {}",
|
||||
ljustify(op.to_string()),
|
||||
src.show_rru_sized(mb_rru, 8),
|
||||
show_ireg_sized(dst.to_reg(), mb_rru, 8),
|
||||
),
|
||||
|
||||
Inst::XmmToGpr { op, src, dst } => {
|
||||
let dst_size = match op {
|
||||
SseOpcode::Movd => 4,
|
||||
SseOpcode::Movq => 8,
|
||||
_ => panic!("unexpected sse opcode"),
|
||||
};
|
||||
format!(
|
||||
"{} {}, {}",
|
||||
ljustify(op.to_string()),
|
||||
src.show_rru(mb_rru),
|
||||
show_ireg_sized(dst.to_reg(), mb_rru, dst_size),
|
||||
)
|
||||
}
|
||||
|
||||
Inst::GprToXmm { op, src, dst } => {
|
||||
let src_size = match op {
|
||||
SseOpcode::Movd => 4,
|
||||
SseOpcode::Movq => 8,
|
||||
_ => panic!("unexpected sse opcode"),
|
||||
};
|
||||
format!(
|
||||
"{} {}, {}",
|
||||
ljustify(op.to_string()),
|
||||
src.show_rru_sized(mb_rru, src_size),
|
||||
dst.show_rru(mb_rru)
|
||||
)
|
||||
}
|
||||
|
||||
Inst::XMM_Cmp_RM_R { op, src, dst } => format!(
|
||||
"{} {}, {}",
|
||||
ljustify(op.to_string()),
|
||||
src.show_rru_sized(mb_rru, 8),
|
||||
show_ireg_sized(*dst, mb_rru, 8),
|
||||
),
|
||||
Inst::Imm_R {
|
||||
dst_is_64,
|
||||
simm64,
|
||||
|
|
@ -761,12 +899,14 @@ impl ShowWithRRU for Inst {
|
|||
)
|
||||
}
|
||||
}
|
||||
|
||||
Inst::Mov_R_R { is_64, src, dst } => format!(
|
||||
"{} {}, {}",
|
||||
ljustify2("mov".to_string(), suffixLQ(*is_64)),
|
||||
show_ireg_sized(*src, mb_rru, sizeLQ(*is_64)),
|
||||
show_ireg_sized(dst.to_reg(), mb_rru, sizeLQ(*is_64))
|
||||
),
|
||||
|
||||
Inst::MovZX_RM_R {
|
||||
ext_mode, src, dst, ..
|
||||
} => {
|
||||
|
|
@ -786,18 +926,21 @@ impl ShowWithRRU for Inst {
|
|||
)
|
||||
}
|
||||
}
|
||||
|
||||
Inst::Mov64_M_R { src, dst, .. } => format!(
|
||||
"{} {}, {}",
|
||||
ljustify("movq".to_string()),
|
||||
src.show_rru(mb_rru),
|
||||
dst.show_rru(mb_rru)
|
||||
),
|
||||
|
||||
Inst::LoadEffectiveAddress { addr, dst } => format!(
|
||||
"{} {}, {}",
|
||||
ljustify("lea".to_string()),
|
||||
addr.show_rru(mb_rru),
|
||||
dst.show_rru(mb_rru)
|
||||
),
|
||||
|
||||
Inst::MovSX_RM_R {
|
||||
ext_mode, src, dst, ..
|
||||
} => format!(
|
||||
|
|
@ -806,12 +949,14 @@ impl ShowWithRRU for Inst {
|
|||
src.show_rru_sized(mb_rru, ext_mode.src_size()),
|
||||
show_ireg_sized(dst.to_reg(), mb_rru, ext_mode.dst_size())
|
||||
),
|
||||
|
||||
Inst::Mov_R_M { size, src, dst, .. } => format!(
|
||||
"{} {}, {}",
|
||||
ljustify2("mov".to_string(), suffixBWLQ(*size)),
|
||||
show_ireg_sized(*src, mb_rru, *size),
|
||||
dst.show_rru(mb_rru)
|
||||
),
|
||||
|
||||
Inst::Shift_R {
|
||||
is_64,
|
||||
kind,
|
||||
|
|
@ -831,40 +976,67 @@ impl ShowWithRRU for Inst {
|
|||
show_ireg_sized(dst.to_reg(), mb_rru, sizeLQ(*is_64))
|
||||
),
|
||||
},
|
||||
|
||||
Inst::Cmp_RMI_R { size, src, dst } => format!(
|
||||
"{} {}, {}",
|
||||
ljustify2("cmp".to_string(), suffixBWLQ(*size)),
|
||||
src.show_rru_sized(mb_rru, *size),
|
||||
show_ireg_sized(*dst, mb_rru, *size)
|
||||
),
|
||||
|
||||
Inst::Setcc { cc, dst } => format!(
|
||||
"{} {}",
|
||||
ljustify2("set".to_string(), cc.to_string()),
|
||||
show_ireg_sized(dst.to_reg(), mb_rru, 1)
|
||||
),
|
||||
|
||||
Inst::Cmove { size, cc, src, dst } => format!(
|
||||
"{} {}, {}",
|
||||
ljustify(format!("cmov{}{}", cc.to_string(), suffixBWLQ(*size))),
|
||||
src.show_rru_sized(mb_rru, *size),
|
||||
show_ireg_sized(dst.to_reg(), mb_rru, *size)
|
||||
),
|
||||
|
||||
Inst::XmmCmove {
|
||||
is_64,
|
||||
cc,
|
||||
src,
|
||||
dst,
|
||||
} => {
|
||||
let size = if *is_64 { 8 } else { 4 };
|
||||
format!(
|
||||
"j{} $next; mov{} {}, {}; $next: ",
|
||||
cc.invert().to_string(),
|
||||
if *is_64 { "sd" } else { "ss" },
|
||||
src.show_rru_sized(mb_rru, size),
|
||||
show_ireg_sized(dst.to_reg(), mb_rru, size)
|
||||
)
|
||||
}
|
||||
|
||||
Inst::Push64 { src } => {
|
||||
format!("{} {}", ljustify("pushq".to_string()), src.show_rru(mb_rru))
|
||||
}
|
||||
|
||||
Inst::Pop64 { dst } => {
|
||||
format!("{} {}", ljustify("popq".to_string()), dst.show_rru(mb_rru))
|
||||
}
|
||||
|
||||
Inst::CallKnown { dest, .. } => format!("{} {:?}", ljustify("call".to_string()), dest),
|
||||
|
||||
Inst::CallUnknown { dest, .. } => format!(
|
||||
"{} *{}",
|
||||
ljustify("call".to_string()),
|
||||
dest.show_rru(mb_rru)
|
||||
),
|
||||
|
||||
Inst::Ret => "ret".to_string(),
|
||||
|
||||
Inst::EpiloguePlaceholder => "epilogue placeholder".to_string(),
|
||||
|
||||
Inst::JmpKnown { dst } => {
|
||||
format!("{} {}", ljustify("jmp".to_string()), dst.show_rru(mb_rru))
|
||||
}
|
||||
|
||||
Inst::JmpCond {
|
||||
cc,
|
||||
taken,
|
||||
|
|
@ -875,18 +1047,21 @@ impl ShowWithRRU for Inst {
|
|||
taken.show_rru(mb_rru),
|
||||
not_taken.show_rru(mb_rru)
|
||||
),
|
||||
|
||||
Inst::JmpTableSeq { idx, .. } => {
|
||||
format!("{} {}", ljustify("br_table".into()), idx.show_rru(mb_rru))
|
||||
}
|
||||
//
|
||||
|
||||
Inst::JmpUnknown { target } => format!(
|
||||
"{} *{}",
|
||||
ljustify("jmp".to_string()),
|
||||
target.show_rru(mb_rru)
|
||||
),
|
||||
|
||||
Inst::TrapIf { cc, trap_code, .. } => {
|
||||
format!("j{} ; ud2 {} ;", cc.invert().to_string(), trap_code)
|
||||
}
|
||||
|
||||
Inst::LoadExtName {
|
||||
dst, name, offset, ..
|
||||
} => format!(
|
||||
|
|
@ -896,8 +1071,11 @@ impl ShowWithRRU for Inst {
|
|||
offset,
|
||||
show_ireg_sized(dst.to_reg(), mb_rru, 8),
|
||||
),
|
||||
|
||||
Inst::VirtualSPOffsetAdj { offset } => format!("virtual_sp_offset_adjust {}", offset),
|
||||
|
||||
Inst::Hlt => "hlt".into(),
|
||||
|
||||
Inst::Ud2 { trap_info } => format!("ud2 {}", trap_info.1),
|
||||
}
|
||||
}
|
||||
|
|
@ -946,7 +1124,7 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) {
|
|||
collector.add_use(regs::rax());
|
||||
collector.add_mod(Writable::from_reg(regs::rdx()));
|
||||
}
|
||||
Inst::UnaryRmR { src, dst, .. } | Inst::XMM_Mov_RM_R { src, dst, .. } => {
|
||||
Inst::UnaryRmR { src, dst, .. } | Inst::XmmUnaryRmR { src, dst, .. } => {
|
||||
src.get_regs_as_uses(collector);
|
||||
collector.add_def(*dst);
|
||||
}
|
||||
|
|
@ -954,17 +1132,25 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) {
|
|||
src.get_regs_as_uses(collector);
|
||||
collector.add_mod(*dst);
|
||||
}
|
||||
Inst::XMM_Mov_R_M { src, dst, .. } => {
|
||||
Inst::Xmm_Mov_R_M { src, dst, .. } => {
|
||||
collector.add_use(*src);
|
||||
dst.get_regs_as_uses(collector);
|
||||
}
|
||||
Inst::XMM_Cmp_RM_R { src, dst, .. } => {
|
||||
src.get_regs_as_uses(collector);
|
||||
collector.add_use(*dst);
|
||||
}
|
||||
Inst::Imm_R { dst, .. } => {
|
||||
collector.add_def(*dst);
|
||||
}
|
||||
Inst::Mov_R_R { src, dst, .. } => {
|
||||
Inst::Mov_R_R { src, dst, .. } | Inst::XmmToGpr { src, dst, .. } => {
|
||||
collector.add_use(*src);
|
||||
collector.add_def(*dst);
|
||||
}
|
||||
Inst::GprToXmm { src, dst, .. } => {
|
||||
src.get_regs_as_uses(collector);
|
||||
collector.add_def(*dst);
|
||||
}
|
||||
Inst::MovZX_RM_R { src, dst, .. } => {
|
||||
src.get_regs_as_uses(collector);
|
||||
collector.add_def(*dst);
|
||||
|
|
@ -994,7 +1180,7 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) {
|
|||
Inst::Setcc { dst, .. } => {
|
||||
collector.add_def(*dst);
|
||||
}
|
||||
Inst::Cmove { src, dst, .. } => {
|
||||
Inst::Cmove { src, dst, .. } | Inst::XmmCmove { src, dst, .. } => {
|
||||
src.get_regs_as_uses(collector);
|
||||
collector.add_mod(*dst);
|
||||
}
|
||||
|
|
@ -1140,7 +1326,7 @@ fn x64_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
|
|||
}
|
||||
}
|
||||
Inst::SignExtendRaxRdx { .. } => {}
|
||||
Inst::XMM_Mov_RM_R {
|
||||
Inst::XmmUnaryRmR {
|
||||
ref mut src,
|
||||
ref mut dst,
|
||||
..
|
||||
|
|
@ -1161,7 +1347,7 @@ fn x64_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
|
|||
src.map_uses(mapper);
|
||||
map_mod(mapper, dst);
|
||||
}
|
||||
Inst::XMM_Mov_R_M {
|
||||
Inst::Xmm_Mov_R_M {
|
||||
ref mut src,
|
||||
ref mut dst,
|
||||
..
|
||||
|
|
@ -1169,15 +1355,36 @@ fn x64_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
|
|||
map_use(mapper, src);
|
||||
dst.map_uses(mapper);
|
||||
}
|
||||
Inst::XMM_Cmp_RM_R {
|
||||
ref mut src,
|
||||
ref mut dst,
|
||||
..
|
||||
} => {
|
||||
src.map_uses(mapper);
|
||||
map_use(mapper, dst);
|
||||
}
|
||||
Inst::Imm_R { ref mut dst, .. } => map_def(mapper, dst),
|
||||
Inst::Mov_R_R {
|
||||
ref mut src,
|
||||
ref mut dst,
|
||||
..
|
||||
}
|
||||
| Inst::XmmToGpr {
|
||||
ref mut src,
|
||||
ref mut dst,
|
||||
..
|
||||
} => {
|
||||
map_use(mapper, src);
|
||||
map_def(mapper, dst);
|
||||
}
|
||||
Inst::GprToXmm {
|
||||
ref mut src,
|
||||
ref mut dst,
|
||||
..
|
||||
} => {
|
||||
src.map_uses(mapper);
|
||||
map_def(mapper, dst);
|
||||
}
|
||||
Inst::MovZX_RM_R {
|
||||
ref mut src,
|
||||
ref mut dst,
|
||||
|
|
@ -1222,6 +1429,11 @@ fn x64_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
|
|||
ref mut src,
|
||||
ref mut dst,
|
||||
..
|
||||
}
|
||||
| Inst::XmmCmove {
|
||||
ref mut src,
|
||||
ref mut dst,
|
||||
..
|
||||
} => {
|
||||
src.map_uses(mapper);
|
||||
map_mod(mapper, dst)
|
||||
|
|
@ -1309,7 +1521,7 @@ impl MachInst for Inst {
|
|||
Self::Mov_R_R {
|
||||
is_64, src, dst, ..
|
||||
} if *is_64 => Some((*dst, *src)),
|
||||
Self::XMM_Mov_RM_R { op, src, dst, .. }
|
||||
Self::XmmUnaryRmR { op, src, dst, .. }
|
||||
if *op == SseOpcode::Movss
|
||||
|| *op == SseOpcode::Movsd
|
||||
|| *op == SseOpcode::Movaps =>
|
||||
|
|
@ -1357,8 +1569,8 @@ impl MachInst for Inst {
|
|||
match rc_dst {
|
||||
RegClass::I64 => Inst::mov_r_r(true, src_reg, dst_reg),
|
||||
RegClass::V128 => match ty {
|
||||
F32 => Inst::xmm_mov_rm_r(SseOpcode::Movss, RegMem::reg(src_reg), dst_reg, None),
|
||||
F64 => Inst::xmm_mov_rm_r(SseOpcode::Movsd, RegMem::reg(src_reg), dst_reg, None),
|
||||
F32 => Inst::xmm_mov(SseOpcode::Movss, RegMem::reg(src_reg), dst_reg, None),
|
||||
F64 => Inst::xmm_mov(SseOpcode::Movsd, RegMem::reg(src_reg), dst_reg, None),
|
||||
_ => panic!("unexpected V128 type in gen_move"),
|
||||
},
|
||||
_ => panic!("gen_move(x64): unhandled regclass"),
|
||||
|
|
@ -1393,11 +1605,43 @@ impl MachInst for Inst {
|
|||
Inst::jmp_known(BranchTarget::Label(label))
|
||||
}
|
||||
|
||||
fn gen_constant(to_reg: Writable<Reg>, value: u64, ty: Type) -> SmallVec<[Self; 4]> {
|
||||
fn gen_constant<F: FnMut(RegClass, Type) -> Writable<Reg>>(
|
||||
to_reg: Writable<Reg>,
|
||||
value: u64,
|
||||
ty: Type,
|
||||
mut alloc_tmp: F,
|
||||
) -> SmallVec<[Self; 4]> {
|
||||
let mut ret = SmallVec::new();
|
||||
debug_assert!(ty.is_int(), "float constants NYI");
|
||||
if ty.is_int() {
|
||||
let is_64 = ty == I64 && value > 0x7fffffff;
|
||||
ret.push(Inst::imm_r(is_64, value, to_reg));
|
||||
} else {
|
||||
match ty {
|
||||
F32 => {
|
||||
let tmp = alloc_tmp(RegClass::I64, I32);
|
||||
ret.push(Inst::imm32_r_unchecked(value, tmp));
|
||||
|
||||
ret.push(Inst::gpr_to_xmm(
|
||||
SseOpcode::Movd,
|
||||
RegMem::reg(tmp.to_reg()),
|
||||
to_reg,
|
||||
));
|
||||
}
|
||||
|
||||
F64 => {
|
||||
let tmp = alloc_tmp(RegClass::I64, I64);
|
||||
ret.push(Inst::imm_r(true, value, tmp));
|
||||
|
||||
ret.push(Inst::gpr_to_xmm(
|
||||
SseOpcode::Movq,
|
||||
RegMem::reg(tmp.to_reg()),
|
||||
to_reg,
|
||||
));
|
||||
}
|
||||
|
||||
_ => panic!("unexpected type {:?} in gen_constant", ty),
|
||||
}
|
||||
}
|
||||
ret
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -6,14 +6,14 @@ use log::trace;
|
|||
use regalloc::{Reg, RegClass, Writable};
|
||||
use smallvec::SmallVec;
|
||||
|
||||
use alloc::boxed::Box;
|
||||
use alloc::vec::Vec;
|
||||
use std::convert::TryFrom;
|
||||
|
||||
use crate::ir::types;
|
||||
use crate::ir::types::*;
|
||||
use crate::ir::Inst as IRInst;
|
||||
use crate::ir::{condcodes::IntCC, InstructionData, Opcode, TrapCode, Type};
|
||||
use crate::ir::{condcodes::FloatCC, condcodes::IntCC, InstructionData, Opcode, TrapCode, Type};
|
||||
use alloc::boxed::Box;
|
||||
use alloc::vec::Vec;
|
||||
use cranelift_codegen_shared::condcodes::CondCode;
|
||||
use std::convert::TryFrom;
|
||||
|
||||
use crate::machinst::lower::*;
|
||||
use crate::machinst::*;
|
||||
|
|
@ -95,6 +95,16 @@ fn inst_condcode(data: &InstructionData) -> IntCC {
|
|||
}
|
||||
}
|
||||
|
||||
fn inst_fp_condcode(data: &InstructionData) -> Option<FloatCC> {
|
||||
match data {
|
||||
&InstructionData::BranchFloat { cond, .. }
|
||||
| &InstructionData::FloatCompare { cond, .. }
|
||||
| &InstructionData::FloatCond { cond, .. }
|
||||
| &InstructionData::FloatCondTrap { cond, .. } => Some(cond),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn ldst_offset(data: &InstructionData) -> Option<i32> {
|
||||
match data {
|
||||
&InstructionData::Load { offset, .. }
|
||||
|
|
@ -734,6 +744,77 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
|
|||
ctx.emit(Inst::setcc(cc, dst));
|
||||
}
|
||||
|
||||
Opcode::Fcmp => {
|
||||
let condcode = inst_fp_condcode(ctx.data(insn)).unwrap();
|
||||
let input_ty = ctx.input_ty(insn, 0);
|
||||
let op = match input_ty {
|
||||
F32 => SseOpcode::Ucomiss,
|
||||
F64 => SseOpcode::Ucomisd,
|
||||
_ => panic!("Bad input type to Fcmp"),
|
||||
};
|
||||
|
||||
// Unordered is returned by setting ZF, PF, CF <- 111
|
||||
// Greater than by ZF, PF, CF <- 000
|
||||
// Less than by ZF, PF, CF <- 001
|
||||
// Equal by ZF, PF, CF <- 100
|
||||
//
|
||||
// Checking the result of comiss is somewhat annoying because you don't
|
||||
// have setcc instructions that explicitly check simultaneously for the condition
|
||||
// (i.e. eq, le, gt, etc) and orderedness. So that might mean we need more
|
||||
// than one setcc check and then a logical "and" or "or" to determine both.
|
||||
// However knowing that if the parity bit is set, then the result was
|
||||
// considered unordered and knowing that if the parity bit is set, then both
|
||||
// the ZF and CF flag bits must also be set we can getaway with using one setcc
|
||||
// for most condition codes.
|
||||
match condcode {
|
||||
// setb and setbe for ordered LessThan and LessThanOrEqual check if CF = 1 which
|
||||
// doesn't exclude unorderdness. To get around this we can reverse the operands
|
||||
// and the cc test to instead check if CF and ZF are 0 which would also excludes
|
||||
// unorderedness. Using similiar logic we also reverse UnorderedOrGreaterThan and
|
||||
// UnorderedOrGreaterThanOrEqual and assure that ZF or CF is 1 to exclude orderedness.
|
||||
FloatCC::LessThan
|
||||
| FloatCC::LessThanOrEqual
|
||||
| FloatCC::UnorderedOrGreaterThan
|
||||
| FloatCC::UnorderedOrGreaterThanOrEqual => {
|
||||
let lhs = input_to_reg_mem(ctx, inputs[0]);
|
||||
let rhs = input_to_reg(ctx, inputs[1]);
|
||||
let dst = output_to_reg(ctx, outputs[0]);
|
||||
ctx.emit(Inst::xmm_cmp_rm_r(op, lhs, rhs));
|
||||
let condcode = condcode.reverse();
|
||||
let cc = CC::from_floatcc(condcode);
|
||||
ctx.emit(Inst::setcc(cc, dst));
|
||||
}
|
||||
// Outlier case where we cannot get around checking the parity bit to determine
|
||||
// if the result was ordered.
|
||||
FloatCC::Equal => {
|
||||
let lhs = input_to_reg(ctx, inputs[0]);
|
||||
let rhs = input_to_reg_mem(ctx, inputs[1]);
|
||||
let dst = output_to_reg(ctx, outputs[0]);
|
||||
let tmp_gpr1 = ctx.alloc_tmp(RegClass::I64, I32);
|
||||
ctx.emit(Inst::xmm_cmp_rm_r(op, rhs, lhs));
|
||||
ctx.emit(Inst::setcc(CC::NP, tmp_gpr1));
|
||||
ctx.emit(Inst::setcc(CC::Z, dst));
|
||||
ctx.emit(Inst::alu_rmi_r(
|
||||
false,
|
||||
AluRmiROpcode::And,
|
||||
RegMemImm::reg(tmp_gpr1.to_reg()),
|
||||
dst,
|
||||
));
|
||||
}
|
||||
// For all remaining condition codes we can handle things with one check. Condition
|
||||
// ordered NotEqual for example does not need a separate check for the parity bit because
|
||||
// the setnz checks that the zero flag is 0 which is impossible with an unordered result.
|
||||
_ => {
|
||||
let lhs = input_to_reg(ctx, inputs[0]);
|
||||
let rhs = input_to_reg_mem(ctx, inputs[1]);
|
||||
let dst = output_to_reg(ctx, outputs[0]);
|
||||
let cc = CC::from_floatcc(condcode);
|
||||
ctx.emit(Inst::xmm_cmp_rm_r(op, rhs, lhs));
|
||||
ctx.emit(Inst::setcc(cc, dst));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Opcode::FallthroughReturn | Opcode::Return => {
|
||||
for i in 0..ctx.num_inputs(insn) {
|
||||
let src_reg = input_to_reg(ctx, inputs[i]);
|
||||
|
|
@ -792,29 +873,111 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
|
|||
ctx.emit(Inst::Ud2 { trap_info })
|
||||
}
|
||||
|
||||
Opcode::F64const => {
|
||||
// TODO use xorpd for 0
|
||||
let value = ctx.get_constant(insn).unwrap();
|
||||
let dst = output_to_reg(ctx, outputs[0]);
|
||||
for inst in Inst::gen_constant(dst, value, F64, |reg_class, ty| {
|
||||
ctx.alloc_tmp(reg_class, ty)
|
||||
}) {
|
||||
ctx.emit(inst);
|
||||
}
|
||||
}
|
||||
|
||||
Opcode::F32const => {
|
||||
// TODO use xorps for 0.
|
||||
let value = ctx.get_constant(insn).unwrap();
|
||||
let dst = output_to_reg(ctx, outputs[0]);
|
||||
for inst in Inst::gen_constant(dst, value, F32, |reg_class, ty| {
|
||||
ctx.alloc_tmp(reg_class, ty)
|
||||
}) {
|
||||
ctx.emit(inst);
|
||||
}
|
||||
}
|
||||
|
||||
Opcode::Fadd | Opcode::Fsub | Opcode::Fmul | Opcode::Fdiv => {
|
||||
let lhs = input_to_reg(ctx, inputs[0]);
|
||||
let lhs = input_to_reg_mem(ctx, inputs[0]);
|
||||
let rhs = input_to_reg(ctx, inputs[1]);
|
||||
let dst = output_to_reg(ctx, outputs[0]);
|
||||
let is_64 = flt_ty_is_64(ty.unwrap());
|
||||
if !is_64 {
|
||||
let sse_op = match op {
|
||||
Opcode::Fadd => SseOpcode::Addss,
|
||||
Opcode::Fsub => SseOpcode::Subss,
|
||||
Opcode::Fmul => SseOpcode::Mulss,
|
||||
Opcode::Fdiv => SseOpcode::Divss,
|
||||
// TODO Fmax, Fmin.
|
||||
_ => unimplemented!(),
|
||||
|
||||
// Note: min and max can't be handled here, because of the way Cranelift defines them:
|
||||
// if any operand is a NaN, they must return the NaN operand, while the x86 machine
|
||||
// instruction will return the other operand.
|
||||
let (f32_op, f64_op) = match op {
|
||||
Opcode::Fadd => (SseOpcode::Addss, SseOpcode::Addsd),
|
||||
Opcode::Fsub => (SseOpcode::Subss, SseOpcode::Subsd),
|
||||
Opcode::Fmul => (SseOpcode::Mulss, SseOpcode::Mulsd),
|
||||
Opcode::Fdiv => (SseOpcode::Divss, SseOpcode::Divsd),
|
||||
_ => unreachable!(),
|
||||
};
|
||||
ctx.emit(Inst::xmm_mov_rm_r(
|
||||
SseOpcode::Movss,
|
||||
RegMem::reg(lhs),
|
||||
dst,
|
||||
None,
|
||||
));
|
||||
ctx.emit(Inst::xmm_rm_r(sse_op, RegMem::reg(rhs), dst));
|
||||
|
||||
let is_64 = flt_ty_is_64(ty.unwrap());
|
||||
|
||||
let mov_op = if is_64 {
|
||||
SseOpcode::Movsd
|
||||
} else {
|
||||
unimplemented!("unimplemented lowering for opcode {:?}", op);
|
||||
SseOpcode::Movss
|
||||
};
|
||||
ctx.emit(Inst::xmm_mov(mov_op, lhs, dst, None));
|
||||
|
||||
let sse_op = if is_64 { f64_op } else { f32_op };
|
||||
ctx.emit(Inst::xmm_rm_r(sse_op, RegMem::reg(rhs), dst));
|
||||
}
|
||||
|
||||
Opcode::Sqrt => {
|
||||
let src = input_to_reg_mem(ctx, inputs[0]);
|
||||
let dst = output_to_reg(ctx, outputs[0]);
|
||||
|
||||
let (f32_op, f64_op) = match op {
|
||||
Opcode::Sqrt => (SseOpcode::Sqrtss, SseOpcode::Sqrtsd),
|
||||
_ => unreachable!(),
|
||||
};
|
||||
|
||||
let sse_op = if flt_ty_is_64(ty.unwrap()) {
|
||||
f64_op
|
||||
} else {
|
||||
f32_op
|
||||
};
|
||||
ctx.emit(Inst::xmm_unary_rm_r(sse_op, src, dst));
|
||||
}
|
||||
|
||||
Opcode::Fpromote => {
|
||||
let src = input_to_reg_mem(ctx, inputs[0]);
|
||||
let dst = output_to_reg(ctx, outputs[0]);
|
||||
ctx.emit(Inst::xmm_unary_rm_r(SseOpcode::Cvtss2sd, src, dst));
|
||||
}
|
||||
|
||||
Opcode::Fdemote => {
|
||||
let src = input_to_reg_mem(ctx, inputs[0]);
|
||||
let dst = output_to_reg(ctx, outputs[0]);
|
||||
ctx.emit(Inst::xmm_unary_rm_r(SseOpcode::Cvtsd2ss, src, dst));
|
||||
}
|
||||
|
||||
Opcode::Bitcast => {
|
||||
let input_ty = ctx.input_ty(insn, 0);
|
||||
let output_ty = ctx.output_ty(insn, 0);
|
||||
match (input_ty, output_ty) {
|
||||
(F32, I32) => {
|
||||
let src = input_to_reg(ctx, inputs[0]);
|
||||
let dst = output_to_reg(ctx, outputs[0]);
|
||||
ctx.emit(Inst::xmm_to_gpr(SseOpcode::Movd, src, dst));
|
||||
}
|
||||
(I32, F32) => {
|
||||
let src = input_to_reg_mem(ctx, inputs[0]);
|
||||
let dst = output_to_reg(ctx, outputs[0]);
|
||||
ctx.emit(Inst::gpr_to_xmm(SseOpcode::Movd, src, dst));
|
||||
}
|
||||
(F64, I64) => {
|
||||
let src = input_to_reg(ctx, inputs[0]);
|
||||
let dst = output_to_reg(ctx, outputs[0]);
|
||||
ctx.emit(Inst::xmm_to_gpr(SseOpcode::Movq, src, dst));
|
||||
}
|
||||
(I64, F64) => {
|
||||
let src = input_to_reg_mem(ctx, inputs[0]);
|
||||
let dst = output_to_reg(ctx, outputs[0]);
|
||||
ctx.emit(Inst::gpr_to_xmm(SseOpcode::Movq, src, dst));
|
||||
}
|
||||
_ => unreachable!("invalid bitcast from {:?} to {:?}", input_ty, output_ty),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -834,20 +997,19 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
|
|||
let tmp_xmm1 = ctx.alloc_tmp(RegClass::V128, F32);
|
||||
let tmp_xmm2 = ctx.alloc_tmp(RegClass::V128, F32);
|
||||
ctx.emit(Inst::imm_r(true, 0x8000_0000, tmp_gpr1));
|
||||
ctx.emit(Inst::xmm_mov_rm_r(
|
||||
ctx.emit(Inst::gpr_to_xmm(
|
||||
SseOpcode::Movd,
|
||||
RegMem::reg(tmp_gpr1.to_reg()),
|
||||
tmp_xmm1,
|
||||
None,
|
||||
));
|
||||
ctx.emit(Inst::xmm_mov_rm_r(
|
||||
ctx.emit(Inst::xmm_mov(
|
||||
SseOpcode::Movaps,
|
||||
RegMem::reg(tmp_xmm1.to_reg()),
|
||||
dst,
|
||||
None,
|
||||
));
|
||||
ctx.emit(Inst::xmm_rm_r(SseOpcode::Andnps, RegMem::reg(lhs), dst));
|
||||
ctx.emit(Inst::xmm_mov_rm_r(
|
||||
ctx.emit(Inst::xmm_mov(
|
||||
SseOpcode::Movss,
|
||||
RegMem::reg(rhs),
|
||||
tmp_xmm2,
|
||||
|
|
@ -982,8 +1144,9 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
|
|||
}
|
||||
(_, true) => {
|
||||
ctx.emit(match elem_ty {
|
||||
F32 => Inst::xmm_mov_rm_r(SseOpcode::Movss, RegMem::mem(addr), dst, srcloc),
|
||||
_ => unimplemented!("FP load not 32-bit"),
|
||||
F32 => Inst::xmm_mov(SseOpcode::Movss, RegMem::mem(addr), dst, srcloc),
|
||||
F64 => Inst::xmm_mov(SseOpcode::Movsd, RegMem::mem(addr), dst, srcloc),
|
||||
_ => unreachable!("unexpected type for load: {:?}", elem_ty),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
@ -1025,7 +1188,7 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
|
|||
| Opcode::Istore32Complex => {
|
||||
assert!(
|
||||
inputs.len() == 3,
|
||||
"can't handle more than two inputs in complex load"
|
||||
"can't handle more than two inputs in complex store"
|
||||
);
|
||||
let base = input_to_reg(ctx, inputs[1]);
|
||||
let index = input_to_reg(ctx, inputs[2]);
|
||||
|
|
@ -1043,7 +1206,8 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
|
|||
if is_float {
|
||||
ctx.emit(match elem_ty {
|
||||
F32 => Inst::xmm_mov_r_m(SseOpcode::Movss, src, addr, srcloc),
|
||||
_ => unimplemented!("FP store not 32-bit"),
|
||||
F64 => Inst::xmm_mov_r_m(SseOpcode::Movsd, src, addr, srcloc),
|
||||
_ => panic!("unexpected type for store {:?}", elem_ty),
|
||||
});
|
||||
} else {
|
||||
ctx.emit(Inst::mov_r_m(elem_ty.bytes() as u8, src, addr, srcloc));
|
||||
|
|
@ -1119,8 +1283,8 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
|
|||
let dst = output_to_reg(ctx, outputs[0]);
|
||||
|
||||
let ty = ctx.output_ty(insn, 0);
|
||||
assert!(is_int_ty(ty), "float cmov NYI");
|
||||
|
||||
if ty.is_int() {
|
||||
let size = ty.bytes() as u8;
|
||||
if size == 1 {
|
||||
// Sign-extend operands to 32, then do a cmove of size 4.
|
||||
|
|
@ -1132,6 +1296,11 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
|
|||
ctx.emit(Inst::gen_move(dst, rhs, ty));
|
||||
ctx.emit(Inst::cmove(size, cc, lhs, dst));
|
||||
}
|
||||
} else {
|
||||
debug_assert!(ty == F32 || ty == F64);
|
||||
ctx.emit(Inst::gen_move(dst, rhs, ty));
|
||||
ctx.emit(Inst::xmm_cmove(ty == F64, cc, lhs, dst));
|
||||
}
|
||||
}
|
||||
|
||||
Opcode::Udiv | Opcode::Urem | Opcode::Sdiv | Opcode::Srem => {
|
||||
|
|
|
|||
|
|
@ -108,6 +108,19 @@ impl Args {
|
|||
|
||||
impl ArgAssigner for Args {
|
||||
fn assign(&mut self, arg: &AbiParam) -> ArgAction {
|
||||
if let ArgumentPurpose::StructArgument(size) = arg.purpose {
|
||||
if self.call_conv != CallConv::SystemV {
|
||||
panic!(
|
||||
"The sarg argument purpose is not yet implemented for non-systemv call conv {:?}",
|
||||
self.call_conv,
|
||||
);
|
||||
}
|
||||
let loc = ArgumentLoc::Stack(self.offset as i32);
|
||||
self.offset += size;
|
||||
debug_assert!(self.offset <= i32::MAX as u32);
|
||||
return ArgAction::AssignAndChangeType(loc, types::SARG_T);
|
||||
}
|
||||
|
||||
let ty = arg.value_type;
|
||||
|
||||
if ty.bits() > u16::from(self.pointer_bits) {
|
||||
|
|
|
|||
|
|
@ -22,8 +22,9 @@ use crate::cursor::{Cursor, FuncCursor};
|
|||
use crate::flowgraph::ControlFlowGraph;
|
||||
use crate::ir::instructions::CallInfo;
|
||||
use crate::ir::{
|
||||
AbiParam, ArgumentLoc, ArgumentPurpose, Block, DataFlowGraph, Function, Inst, InstBuilder,
|
||||
MemFlags, SigRef, Signature, StackSlotData, StackSlotKind, Type, Value, ValueLoc,
|
||||
AbiParam, ArgumentLoc, ArgumentPurpose, Block, DataFlowGraph, ExtFuncData, ExternalName,
|
||||
Function, Inst, InstBuilder, LibCall, MemFlags, SigRef, Signature, StackSlotData,
|
||||
StackSlotKind, Type, Value, ValueLoc,
|
||||
};
|
||||
use crate::isa::TargetIsa;
|
||||
use crate::legalizer::split::{isplit, vsplit};
|
||||
|
|
@ -113,12 +114,31 @@ fn legalize_entry_params(func: &mut Function, entry: Block) {
|
|||
|
||||
let abi_type = pos.func.signature.params[abi_arg];
|
||||
let arg_type = pos.func.dfg.value_type(arg);
|
||||
if let ArgumentPurpose::StructArgument(size) = abi_type.purpose {
|
||||
let offset = if let ArgumentLoc::Stack(offset) = abi_type.location {
|
||||
offset
|
||||
} else {
|
||||
unreachable!("StructArgument must already have a Stack ArgumentLoc assigned");
|
||||
};
|
||||
let ss = pos.func.stack_slots.make_incoming_arg(size, offset);
|
||||
let struct_arg = pos.ins().stack_addr(arg_type, ss, 0);
|
||||
pos.func.dfg.change_to_alias(arg, struct_arg);
|
||||
let dummy = pos
|
||||
.func
|
||||
.dfg
|
||||
.append_block_param(entry, crate::ir::types::SARG_T);
|
||||
pos.func.locations[dummy] = ValueLoc::Stack(ss);
|
||||
abi_arg += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if arg_type == abi_type.value_type {
|
||||
// No value translation is necessary, this argument matches the ABI type.
|
||||
// Just use the original block argument value. This is the most common case.
|
||||
pos.func.dfg.attach_block_param(entry, arg);
|
||||
match abi_type.purpose {
|
||||
ArgumentPurpose::Normal => {}
|
||||
ArgumentPurpose::StructArgument(_) => unreachable!("Handled above"),
|
||||
ArgumentPurpose::FramePointer => {}
|
||||
ArgumentPurpose::CalleeSaved => {}
|
||||
ArgumentPurpose::StructReturn => {
|
||||
|
|
@ -137,7 +157,7 @@ fn legalize_entry_params(func: &mut Function, entry: Block) {
|
|||
debug_assert!(!has_stack_limit, "Multiple stack_limit arguments found");
|
||||
has_stack_limit = true;
|
||||
}
|
||||
_ => panic!("Unexpected special-purpose arg {}", abi_type),
|
||||
ArgumentPurpose::Link => panic!("Unexpected link arg {}", abi_type),
|
||||
}
|
||||
abi_arg += 1;
|
||||
} else {
|
||||
|
|
@ -168,7 +188,7 @@ fn legalize_entry_params(func: &mut Function, entry: Block) {
|
|||
for &arg in &pos.func.signature.params[abi_arg..] {
|
||||
match arg.purpose {
|
||||
// Any normal parameters should have been processed above.
|
||||
ArgumentPurpose::Normal => {
|
||||
ArgumentPurpose::Normal | ArgumentPurpose::StructArgument(_) => {
|
||||
panic!("Leftover arg: {}", arg);
|
||||
}
|
||||
// The callee-save parameters should not appear until after register allocation is
|
||||
|
|
@ -587,9 +607,14 @@ fn convert_to_abi<PutArg>(
|
|||
|
||||
/// Check if a sequence of arguments match a desired sequence of argument types.
|
||||
fn check_arg_types(dfg: &DataFlowGraph, args: &[Value], types: &[AbiParam]) -> bool {
|
||||
let arg_types = args.iter().map(|&v| dfg.value_type(v));
|
||||
let sig_types = types.iter().map(|&at| at.value_type);
|
||||
arg_types.eq(sig_types)
|
||||
args.len() == types.len()
|
||||
&& args.iter().zip(types.iter()).all(|(v, at)| {
|
||||
if let ArgumentPurpose::StructArgument(_) = at.purpose {
|
||||
true
|
||||
} else {
|
||||
dfg.value_type(*v) == at.value_type
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// Check if the arguments of the call `inst` match the signature.
|
||||
|
|
@ -699,7 +724,12 @@ fn legalize_inst_arguments<ArgType>(
|
|||
.unwrap();
|
||||
let mut put_arg = |func: &mut Function, arg| {
|
||||
let abi_type = get_abi_type(func, abi_arg);
|
||||
if func.dfg.value_type(arg) == abi_type.value_type {
|
||||
let struct_argument = if let ArgumentPurpose::StructArgument(_) = abi_type.purpose {
|
||||
true
|
||||
} else {
|
||||
false
|
||||
};
|
||||
if func.dfg.value_type(arg) == abi_type.value_type || struct_argument {
|
||||
// This is the argument type we need.
|
||||
vlist.as_mut_slice(&mut func.dfg.value_lists)[num_fixed_values + abi_arg] = arg;
|
||||
abi_arg += 1;
|
||||
|
|
@ -762,7 +792,7 @@ pub fn handle_call_abi(
|
|||
|
||||
// Start by checking if the argument types already match the signature.
|
||||
let sig_ref = match check_call_signature(&pos.func.dfg, inst) {
|
||||
Ok(_) => return spill_call_arguments(pos),
|
||||
Ok(_) => return spill_call_arguments(pos, isa),
|
||||
Err(s) => s,
|
||||
};
|
||||
|
||||
|
|
@ -800,7 +830,7 @@ pub fn handle_call_abi(
|
|||
|
||||
// Go back and insert spills for any stack arguments.
|
||||
pos.goto_inst(inst);
|
||||
spill_call_arguments(pos);
|
||||
spill_call_arguments(pos, isa);
|
||||
|
||||
// Yes, we changed stuff.
|
||||
true
|
||||
|
|
@ -990,8 +1020,12 @@ fn spill_entry_params(func: &mut Function, entry: Block) {
|
|||
.iter()
|
||||
.zip(func.dfg.block_params(entry))
|
||||
{
|
||||
if let ArgumentLoc::Stack(offset) = abi.location {
|
||||
let ss = func.stack_slots.make_incoming_arg(abi.value_type, offset);
|
||||
if let ArgumentPurpose::StructArgument(_) = abi.purpose {
|
||||
// Location has already been assigned during legalization.
|
||||
} else if let ArgumentLoc::Stack(offset) = abi.location {
|
||||
let ss = func
|
||||
.stack_slots
|
||||
.make_incoming_arg(abi.value_type.bytes(), offset);
|
||||
func.locations[arg] = ValueLoc::Stack(ss);
|
||||
}
|
||||
}
|
||||
|
|
@ -1005,7 +1039,7 @@ fn spill_entry_params(func: &mut Function, entry: Block) {
|
|||
/// TODO: The outgoing stack slots can be written a bit earlier, as long as there are no branches
|
||||
/// or calls between writing the stack slots and the call instruction. Writing the slots earlier
|
||||
/// could help reduce register pressure before the call.
|
||||
fn spill_call_arguments(pos: &mut FuncCursor) -> bool {
|
||||
fn spill_call_arguments(pos: &mut FuncCursor, isa: &dyn TargetIsa) -> bool {
|
||||
let inst = pos
|
||||
.current_inst()
|
||||
.expect("Cursor must point to a call instruction");
|
||||
|
|
@ -1032,9 +1066,17 @@ fn spill_call_arguments(pos: &mut FuncCursor) -> bool {
|
|||
// Assign `arg` to a new stack slot, unless it's already in the correct
|
||||
// slot. The legalization needs to be idempotent, so we should see a
|
||||
// correct outgoing slot on the second pass.
|
||||
let ss = stack_slots.get_outgoing_arg(abi.value_type, offset);
|
||||
let (ss, size) = match abi.purpose {
|
||||
ArgumentPurpose::StructArgument(size) => {
|
||||
(stack_slots.get_outgoing_arg(size, offset), Some(size))
|
||||
}
|
||||
_ => (
|
||||
stack_slots.get_outgoing_arg(abi.value_type.bytes(), offset),
|
||||
None,
|
||||
),
|
||||
};
|
||||
if locations[arg] != ValueLoc::Stack(ss) {
|
||||
Some((idx, arg, ss))
|
||||
Some((idx, arg, ss, size))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
|
|
@ -1049,9 +1091,48 @@ fn spill_call_arguments(pos: &mut FuncCursor) -> bool {
|
|||
return false;
|
||||
}
|
||||
|
||||
let mut libc_memcpy = None;
|
||||
let mut import_memcpy = |func: &mut Function, pointer_type| {
|
||||
if let Some(libc_memcpy) = libc_memcpy {
|
||||
return libc_memcpy;
|
||||
}
|
||||
|
||||
let signature = {
|
||||
let mut s = Signature::new(isa.default_call_conv());
|
||||
s.params.push(AbiParam::new(pointer_type));
|
||||
s.params.push(AbiParam::new(pointer_type));
|
||||
// The last argument of `memcpy` is a `size_t`. This is the same size as a pointer on
|
||||
// all architectures we are interested in.
|
||||
s.params.push(AbiParam::new(pointer_type));
|
||||
legalize_libcall_signature(&mut s, isa);
|
||||
func.import_signature(s)
|
||||
};
|
||||
|
||||
let func = func.import_function(ExtFuncData {
|
||||
name: ExternalName::LibCall(LibCall::Memcpy),
|
||||
signature,
|
||||
colocated: false,
|
||||
});
|
||||
libc_memcpy = Some(func);
|
||||
func
|
||||
};
|
||||
|
||||
// Insert the spill instructions and rewrite call arguments.
|
||||
for (idx, arg, ss) in arglist {
|
||||
let stack_val = pos.ins().spill(arg);
|
||||
for (idx, arg, ss, size) in arglist {
|
||||
let stack_val = if let Some(size) = size {
|
||||
// Struct argument
|
||||
let pointer_type = pos.func.dfg.value_type(arg);
|
||||
let src = arg;
|
||||
let dest = pos.ins().stack_addr(pointer_type, ss, 0);
|
||||
let size = pos.ins().iconst(pointer_type, i64::from(size));
|
||||
|
||||
let libc_memcpy = import_memcpy(pos.func, pointer_type);
|
||||
pos.ins().call(libc_memcpy, &[dest, src, size]);
|
||||
pos.ins().dummy_sarg_t()
|
||||
} else {
|
||||
// Non struct argument
|
||||
pos.ins().spill(arg)
|
||||
};
|
||||
pos.func.locations[stack_val] = ValueLoc::Stack(ss);
|
||||
pos.func.dfg.inst_variable_args_mut(inst)[idx] = stack_val;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -516,7 +516,11 @@ impl<'func, I: VCodeInst> Lower<'func, I> {
|
|||
|
||||
// Now, finally, deal with the moves whose sources are constants.
|
||||
for (ty, dst_reg, const_u64) in &const_bundles {
|
||||
for inst in I::gen_constant(*dst_reg, *const_u64, *ty).into_iter() {
|
||||
for inst in I::gen_constant(*dst_reg, *const_u64, *ty, |reg_class, ty| {
|
||||
self.alloc_tmp(reg_class, ty)
|
||||
})
|
||||
.into_iter()
|
||||
{
|
||||
self.emit(inst);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -154,7 +154,12 @@ pub trait MachInst: Clone + Debug {
|
|||
fn gen_move(to_reg: Writable<Reg>, from_reg: Reg, ty: Type) -> Self;
|
||||
|
||||
/// Generate a constant into a reg.
|
||||
fn gen_constant(to_reg: Writable<Reg>, value: u64, ty: Type) -> SmallVec<[Self; 4]>;
|
||||
fn gen_constant<F: FnMut(RegClass, Type) -> Writable<Reg>>(
|
||||
to_reg: Writable<Reg>,
|
||||
value: u64,
|
||||
ty: Type,
|
||||
alloc_tmp: F,
|
||||
) -> SmallVec<[Self; 4]>;
|
||||
|
||||
/// Generate a zero-length no-op.
|
||||
fn gen_zero_len_nop() -> Self;
|
||||
|
|
|
|||
|
|
@ -13,27 +13,287 @@ use cranelift_codegen_shared::condcodes::IntCC;
|
|||
use peepmatic_runtime::{
|
||||
cc::ConditionCode,
|
||||
instruction_set::InstructionSet,
|
||||
operator::Operator,
|
||||
part::{Constant, Part},
|
||||
paths::Path,
|
||||
r#type::{BitWidth, Kind, Type},
|
||||
PeepholeOptimizations, PeepholeOptimizer,
|
||||
};
|
||||
use peepmatic_traits::TypingRules;
|
||||
use std::borrow::Cow;
|
||||
use std::boxed::Box;
|
||||
use std::convert::{TryFrom, TryInto};
|
||||
use std::ptr;
|
||||
use std::sync::atomic::{AtomicPtr, Ordering};
|
||||
|
||||
peepmatic_traits::define_parse_and_typing_rules_for_operator! {
|
||||
Opcode {
|
||||
adjust_sp_down => AdjustSpDown {
|
||||
parameters(iNN);
|
||||
result(void);
|
||||
}
|
||||
adjust_sp_down_imm => AdjustSpDownImm {
|
||||
immediates(iNN);
|
||||
result(void);
|
||||
}
|
||||
band => Band {
|
||||
parameters(iNN, iNN);
|
||||
result(iNN);
|
||||
}
|
||||
band_imm => BandImm {
|
||||
immediates(iNN);
|
||||
parameters(iNN);
|
||||
result(iNN);
|
||||
}
|
||||
bconst => Bconst {
|
||||
immediates(b1);
|
||||
result(bNN);
|
||||
}
|
||||
bint => Bint {
|
||||
parameters(bNN);
|
||||
result(iNN);
|
||||
}
|
||||
bor => Bor {
|
||||
parameters(iNN, iNN);
|
||||
result(iNN);
|
||||
}
|
||||
bor_imm => BorImm {
|
||||
immediates(iNN);
|
||||
parameters(iNN);
|
||||
result(iNN);
|
||||
}
|
||||
brnz => Brnz {
|
||||
parameters(bool_or_int);
|
||||
result(void);
|
||||
}
|
||||
brz => Brz {
|
||||
parameters(bool_or_int);
|
||||
result(void);
|
||||
}
|
||||
bxor => Bxor {
|
||||
parameters(iNN, iNN);
|
||||
result(iNN);
|
||||
}
|
||||
bxor_imm => BxorImm {
|
||||
immediates(iNN);
|
||||
parameters(iNN);
|
||||
result(iNN);
|
||||
}
|
||||
iadd => Iadd {
|
||||
parameters(iNN, iNN);
|
||||
result(iNN);
|
||||
}
|
||||
iadd_imm => IaddImm {
|
||||
immediates(iNN);
|
||||
parameters(iNN);
|
||||
result(iNN);
|
||||
}
|
||||
icmp => Icmp {
|
||||
immediates(cc);
|
||||
parameters(iNN, iNN);
|
||||
result(b1);
|
||||
}
|
||||
icmp_imm => IcmpImm {
|
||||
immediates(cc, iNN);
|
||||
parameters(iNN);
|
||||
result(b1);
|
||||
}
|
||||
iconst => Iconst {
|
||||
immediates(iNN);
|
||||
result(iNN);
|
||||
}
|
||||
ifcmp => Ifcmp {
|
||||
parameters(iNN, iNN);
|
||||
result(cpu_flags);
|
||||
}
|
||||
ifcmp_imm => IfcmpImm {
|
||||
immediates(iNN);
|
||||
parameters(iNN);
|
||||
result(cpu_flags);
|
||||
}
|
||||
imul => Imul {
|
||||
parameters(iNN, iNN);
|
||||
result(iNN);
|
||||
}
|
||||
imul_imm => ImulImm {
|
||||
immediates(iNN);
|
||||
parameters(iNN);
|
||||
result(iNN);
|
||||
}
|
||||
ireduce => Ireduce {
|
||||
parameters(iNN);
|
||||
result(iMM);
|
||||
is_reduce(true);
|
||||
}
|
||||
irsub_imm => IrsubImm {
|
||||
immediates(iNN);
|
||||
parameters(iNN);
|
||||
result(iNN);
|
||||
}
|
||||
ishl => Ishl {
|
||||
parameters(iNN, iNN);
|
||||
result(iNN);
|
||||
}
|
||||
ishl_imm => IshlImm {
|
||||
immediates(iNN);
|
||||
parameters(iNN);
|
||||
result(iNN);
|
||||
}
|
||||
isub => Isub {
|
||||
parameters(iNN, iNN);
|
||||
result(iNN);
|
||||
}
|
||||
rotl => Rotl {
|
||||
parameters(iNN, iNN);
|
||||
result(iNN);
|
||||
}
|
||||
rotl_imm => RotlImm {
|
||||
immediates(iNN);
|
||||
parameters(iNN);
|
||||
result(iNN);
|
||||
}
|
||||
rotr => Rotr {
|
||||
parameters(iNN, iNN);
|
||||
result(iNN);
|
||||
}
|
||||
rotr_imm => RotrImm {
|
||||
immediates(iNN);
|
||||
parameters(iNN);
|
||||
result(iNN);
|
||||
}
|
||||
sdiv => Sdiv {
|
||||
parameters(iNN, iNN);
|
||||
result(iNN);
|
||||
}
|
||||
sdiv_imm => SdivImm {
|
||||
immediates(iNN);
|
||||
parameters(iNN);
|
||||
result(iNN);
|
||||
}
|
||||
select => Select {
|
||||
parameters(bool_or_int, any_t, any_t);
|
||||
result(any_t);
|
||||
}
|
||||
sextend => Sextend {
|
||||
parameters(iNN);
|
||||
result(iMM);
|
||||
is_extend(true);
|
||||
}
|
||||
srem => Srem {
|
||||
parameters(iNN, iNN);
|
||||
result(iNN);
|
||||
}
|
||||
srem_imm => SremImm {
|
||||
immediates(iNN);
|
||||
parameters(iNN);
|
||||
result(iNN);
|
||||
}
|
||||
sshr => Sshr {
|
||||
parameters(iNN, iNN);
|
||||
result(iNN);
|
||||
}
|
||||
sshr_imm => SshrImm {
|
||||
immediates(iNN);
|
||||
parameters(iNN);
|
||||
result(iNN);
|
||||
}
|
||||
trapnz => Trapnz {
|
||||
parameters(bool_or_int);
|
||||
result(void);
|
||||
}
|
||||
trapz => Trapz {
|
||||
parameters(bool_or_int);
|
||||
result(void);
|
||||
}
|
||||
udiv => Udiv {
|
||||
parameters(iNN, iNN);
|
||||
result(iNN);
|
||||
}
|
||||
udiv_imm => UdivImm {
|
||||
immediates(iNN);
|
||||
parameters(iNN);
|
||||
result(iNN);
|
||||
}
|
||||
uextend => Uextend {
|
||||
parameters(iNN);
|
||||
result(iMM);
|
||||
is_extend(true);
|
||||
}
|
||||
urem => Urem {
|
||||
parameters(iNN, iNN);
|
||||
result(iNN);
|
||||
}
|
||||
urem_imm => UremImm {
|
||||
immediates(iNN);
|
||||
parameters(iNN);
|
||||
result(iNN);
|
||||
}
|
||||
ushr => Ushr {
|
||||
parameters(iNN, iNN);
|
||||
result(iNN);
|
||||
}
|
||||
ushr_imm => UshrImm {
|
||||
immediates(iNN);
|
||||
parameters(iNN);
|
||||
result(iNN);
|
||||
}
|
||||
}
|
||||
parse_cfg(feature = "rebuild-peephole-optimizers");
|
||||
}
|
||||
|
||||
/// Code required to rebuild Peepmatic-based peephole optimizers.
|
||||
///
|
||||
/// This module is used to scope imports and dependencies that are only required
|
||||
/// for building peephole optimizers (as opposed to just using pre-built
|
||||
/// peephole optimizers). This helps ensure that our regular builds using
|
||||
/// pre-built peephole optimizers stay lean.
|
||||
#[cfg(feature = "rebuild-peephole-optimizers")]
|
||||
mod rebuild {
|
||||
use super::*;
|
||||
use alloc::vec::Vec;
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
|
||||
/// Rebuild the `preopt.peepmatic` peephole optimizer.
|
||||
///
|
||||
/// Saves and overwrites the old `preopt.serialized` build and returns a
|
||||
/// copy of the result.
|
||||
pub fn rebuild_preopt() -> Vec<u8> {
|
||||
let codegen_path = Path::new(include_str!(concat!(
|
||||
env!("OUT_DIR"),
|
||||
"/CRANELIFT_CODEGEN_PATH"
|
||||
)));
|
||||
let source_path = codegen_path.join("src").join("preopt.peepmatic");
|
||||
|
||||
let preopt = peepmatic::compile_file::<Opcode>(&source_path)
|
||||
.expect("failed to compile `src/preopt.peepmatic`");
|
||||
|
||||
let serialized_path = codegen_path.join("src").join("preopt.serialized");
|
||||
preopt
|
||||
.serialize_to_file(&serialized_path)
|
||||
.expect("failed to serialize peephole optimizer to `src/preopt.serialized`");
|
||||
fs::read(&serialized_path).expect("failed to read `src/preopt.serialized`")
|
||||
}
|
||||
}
|
||||
|
||||
/// Get the `preopt.peepmatic` peephole optimizer.
|
||||
pub(crate) fn preopt<'a, 'b>(
|
||||
isa: &'b dyn TargetIsa,
|
||||
) -> PeepholeOptimizer<'static, 'a, &'b dyn TargetIsa> {
|
||||
#[cfg(feature = "rebuild-peephole-optimizers")]
|
||||
fn get_serialized() -> Cow<'static, [u8]> {
|
||||
rebuild::rebuild_preopt().into()
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "rebuild-peephole-optimizers"))]
|
||||
fn get_serialized() -> Cow<'static, [u8]> {
|
||||
static SERIALIZED: &[u8] = include_bytes!("preopt.serialized");
|
||||
SERIALIZED.into()
|
||||
}
|
||||
|
||||
// Once initialized, this must never be re-assigned. The initialized value
|
||||
// is semantically "static data" and is intentionally leaked for the whole
|
||||
// program's lifetime.
|
||||
static DESERIALIZED: AtomicPtr<PeepholeOptimizations> = AtomicPtr::new(ptr::null_mut());
|
||||
static DESERIALIZED: AtomicPtr<PeepholeOptimizations<Opcode>> = AtomicPtr::new(ptr::null_mut());
|
||||
|
||||
// If `DESERIALIZED` has already been initialized, then just use it.
|
||||
let ptr = DESERIALIZED.load(Ordering::SeqCst);
|
||||
|
|
@ -46,7 +306,7 @@ pub(crate) fn preopt<'a, 'b>(
|
|||
// another thread could be doing the same thing concurrently, so there is a
|
||||
// race to see who initializes `DESERIALIZED` first, and we need to be
|
||||
// prepared to both win or lose that race.
|
||||
let peep_opts = PeepholeOptimizations::deserialize(SERIALIZED)
|
||||
let peep_opts = PeepholeOptimizations::deserialize(&get_serialized())
|
||||
.expect("should always be able to deserialize `preopt.serialized`");
|
||||
let peep_opts = Box::into_raw(Box::new(peep_opts));
|
||||
|
||||
|
|
@ -219,69 +479,6 @@ fn part_to_value(pos: &mut FuncCursor, root: Inst, part: Part<ValueOrInst>) -> O
|
|||
}
|
||||
}
|
||||
|
||||
impl Opcode {
|
||||
fn to_peepmatic_operator(&self) -> Option<Operator> {
|
||||
macro_rules! convert {
|
||||
( $( $op:ident $(,)* )* ) => {
|
||||
match self {
|
||||
$( Self::$op => Some(Operator::$op), )*
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
convert!(
|
||||
AdjustSpDown,
|
||||
AdjustSpDownImm,
|
||||
Band,
|
||||
BandImm,
|
||||
Bconst,
|
||||
Bint,
|
||||
Bor,
|
||||
BorImm,
|
||||
Brnz,
|
||||
Brz,
|
||||
Bxor,
|
||||
BxorImm,
|
||||
Iadd,
|
||||
IaddImm,
|
||||
Icmp,
|
||||
IcmpImm,
|
||||
Iconst,
|
||||
Ifcmp,
|
||||
IfcmpImm,
|
||||
Imul,
|
||||
ImulImm,
|
||||
Ireduce,
|
||||
IrsubImm,
|
||||
Ishl,
|
||||
IshlImm,
|
||||
Isub,
|
||||
Rotl,
|
||||
RotlImm,
|
||||
Rotr,
|
||||
RotrImm,
|
||||
Sdiv,
|
||||
SdivImm,
|
||||
Select,
|
||||
Sextend,
|
||||
Srem,
|
||||
SremImm,
|
||||
Sshr,
|
||||
SshrImm,
|
||||
Trapnz,
|
||||
Trapz,
|
||||
Udiv,
|
||||
UdivImm,
|
||||
Uextend,
|
||||
Urem,
|
||||
UremImm,
|
||||
Ushr,
|
||||
UshrImm,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl TryFrom<Constant> for Imm64 {
|
||||
type Error = &'static str;
|
||||
|
||||
|
|
@ -428,6 +625,8 @@ fn peepmatic_ty_to_ir_ty(ty: Type, dfg: &DataFlowGraph, root: Inst) -> types::Ty
|
|||
unsafe impl<'a, 'b> InstructionSet<'b> for &'a dyn TargetIsa {
|
||||
type Context = FuncCursor<'b>;
|
||||
|
||||
type Operator = Opcode;
|
||||
|
||||
type Instruction = ValueOrInst;
|
||||
|
||||
fn replace_instruction(
|
||||
|
|
@ -495,7 +694,7 @@ unsafe impl<'a, 'b> InstructionSet<'b> for &'a dyn TargetIsa {
|
|||
let mut part = Part::Instruction(root);
|
||||
for p in path.0[1..].iter().copied() {
|
||||
let inst = part.as_instruction()?.resolve_inst(&pos.func.dfg)?;
|
||||
let operator = pos.func.dfg[inst].opcode().to_peepmatic_operator()?;
|
||||
let operator = pos.func.dfg[inst].opcode();
|
||||
|
||||
if p < operator.immediates_arity() {
|
||||
part = get_immediate(&pos.func.dfg, inst, p as usize);
|
||||
|
|
@ -512,16 +711,16 @@ unsafe impl<'a, 'b> InstructionSet<'b> for &'a dyn TargetIsa {
|
|||
Some(part)
|
||||
}
|
||||
|
||||
fn operator(&self, pos: &mut FuncCursor<'b>, value_or_inst: ValueOrInst) -> Option<Operator> {
|
||||
fn operator(&self, pos: &mut FuncCursor<'b>, value_or_inst: ValueOrInst) -> Option<Opcode> {
|
||||
let inst = value_or_inst.resolve_inst(&pos.func.dfg)?;
|
||||
pos.func.dfg[inst].opcode().to_peepmatic_operator()
|
||||
Some(pos.func.dfg[inst].opcode())
|
||||
}
|
||||
|
||||
fn make_inst_1(
|
||||
&self,
|
||||
pos: &mut FuncCursor<'b>,
|
||||
root: ValueOrInst,
|
||||
operator: Operator,
|
||||
operator: Opcode,
|
||||
r#type: Type,
|
||||
a: Part<ValueOrInst>,
|
||||
) -> ValueOrInst {
|
||||
|
|
@ -529,27 +728,32 @@ unsafe impl<'a, 'b> InstructionSet<'b> for &'a dyn TargetIsa {
|
|||
|
||||
let root = root.resolve_inst(&pos.func.dfg).unwrap();
|
||||
match operator {
|
||||
Operator::AdjustSpDown => {
|
||||
Opcode::AdjustSpDown => {
|
||||
let a = part_to_value(pos, root, a).unwrap();
|
||||
pos.ins().adjust_sp_down(a).into()
|
||||
}
|
||||
Operator::AdjustSpDownImm => {
|
||||
Opcode::AdjustSpDownImm => {
|
||||
let c = a.unwrap_constant();
|
||||
let imm = Imm64::try_from(c).unwrap();
|
||||
pos.ins().adjust_sp_down_imm(imm).into()
|
||||
}
|
||||
Operator::Bconst => {
|
||||
Opcode::Bconst => {
|
||||
let c = a.unwrap_constant();
|
||||
let val = const_to_value(pos.ins(), c, root);
|
||||
pos.func.dfg.value_def(val).unwrap_inst().into()
|
||||
}
|
||||
Operator::Bint => {
|
||||
Opcode::Bint => {
|
||||
let a = part_to_value(pos, root, a).unwrap();
|
||||
let ty = peepmatic_ty_to_ir_ty(r#type, &pos.func.dfg, root);
|
||||
let val = pos.ins().bint(ty, a);
|
||||
pos.func.dfg.value_def(val).unwrap_inst().into()
|
||||
}
|
||||
Operator::Brnz => {
|
||||
Opcode::Bnot => {
|
||||
let a = part_to_value(pos, root, a).unwrap();
|
||||
let val = pos.ins().bnot(a);
|
||||
pos.func.dfg.value_def(val).unwrap_inst().into()
|
||||
}
|
||||
Opcode::Brnz => {
|
||||
let a = part_to_value(pos, root, a).unwrap();
|
||||
|
||||
// NB: branching instructions must be the root of an
|
||||
|
|
@ -561,37 +765,37 @@ unsafe impl<'a, 'b> InstructionSet<'b> for &'a dyn TargetIsa {
|
|||
|
||||
pos.ins().brnz(a, block, &args).into()
|
||||
}
|
||||
Operator::Brz => {
|
||||
Opcode::Brz => {
|
||||
let a = part_to_value(pos, root, a).unwrap();
|
||||
|
||||
// See the comment in the `Operator::Brnz` match argm.
|
||||
// See the comment in the `Opcode::Brnz` match argm.
|
||||
let block = pos.func.dfg[root].branch_destination().unwrap();
|
||||
let args = pos.func.dfg.inst_args(root)[1..].to_vec();
|
||||
|
||||
pos.ins().brz(a, block, &args).into()
|
||||
}
|
||||
Operator::Iconst => {
|
||||
Opcode::Iconst => {
|
||||
let a = a.unwrap_constant();
|
||||
let val = const_to_value(pos.ins(), a, root);
|
||||
pos.func.dfg.value_def(val).unwrap_inst().into()
|
||||
}
|
||||
Operator::Ireduce => {
|
||||
Opcode::Ireduce => {
|
||||
let a = part_to_value(pos, root, a).unwrap();
|
||||
let ty = peepmatic_ty_to_ir_ty(r#type, &pos.func.dfg, root);
|
||||
let val = pos.ins().ireduce(ty, a);
|
||||
pos.func.dfg.value_def(val).unwrap_inst().into()
|
||||
}
|
||||
Operator::Sextend => {
|
||||
Opcode::Sextend => {
|
||||
let a = part_to_value(pos, root, a).unwrap();
|
||||
let ty = peepmatic_ty_to_ir_ty(r#type, &pos.func.dfg, root);
|
||||
let val = pos.ins().sextend(ty, a);
|
||||
pos.func.dfg.value_def(val).unwrap_inst().into()
|
||||
}
|
||||
Operator::Trapnz => {
|
||||
Opcode::Trapnz => {
|
||||
let a = part_to_value(pos, root, a).unwrap();
|
||||
|
||||
// NB: similar to branching instructions (see comment in the
|
||||
// `Operator::Brnz` match arm) trapping instructions must be the
|
||||
// `Opcode::Brnz` match arm) trapping instructions must be the
|
||||
// root of an optimization's right-hand side, and we get the
|
||||
// trap code from the root of the left-hand side. Peepmatic
|
||||
// doesn't currently represent trap codes.
|
||||
|
|
@ -599,13 +803,13 @@ unsafe impl<'a, 'b> InstructionSet<'b> for &'a dyn TargetIsa {
|
|||
|
||||
pos.ins().trapnz(a, code).into()
|
||||
}
|
||||
Operator::Trapz => {
|
||||
Opcode::Trapz => {
|
||||
let a = part_to_value(pos, root, a).unwrap();
|
||||
// See comment in the `Operator::Trapnz` match arm.
|
||||
// See comment in the `Opcode::Trapnz` match arm.
|
||||
let code = pos.func.dfg[root].trap_code().unwrap();
|
||||
pos.ins().trapz(a, code).into()
|
||||
}
|
||||
Operator::Uextend => {
|
||||
Opcode::Uextend => {
|
||||
let a = part_to_value(pos, root, a).unwrap();
|
||||
let ty = peepmatic_ty_to_ir_ty(r#type, &pos.func.dfg, root);
|
||||
let val = pos.ins().uextend(ty, a);
|
||||
|
|
@ -619,7 +823,7 @@ unsafe impl<'a, 'b> InstructionSet<'b> for &'a dyn TargetIsa {
|
|||
&self,
|
||||
pos: &mut FuncCursor<'b>,
|
||||
root: ValueOrInst,
|
||||
operator: Operator,
|
||||
operator: Opcode,
|
||||
_: Type,
|
||||
a: Part<ValueOrInst>,
|
||||
b: Part<ValueOrInst>,
|
||||
|
|
@ -628,193 +832,193 @@ unsafe impl<'a, 'b> InstructionSet<'b> for &'a dyn TargetIsa {
|
|||
|
||||
let root = root.resolve_inst(&pos.func.dfg).unwrap();
|
||||
match operator {
|
||||
Operator::Band => {
|
||||
Opcode::Band => {
|
||||
let a = part_to_value(pos, root, a).unwrap();
|
||||
let b = part_to_value(pos, root, b).unwrap();
|
||||
let val = pos.ins().band(a, b);
|
||||
pos.func.dfg.value_def(val).unwrap_inst().into()
|
||||
}
|
||||
Operator::BandImm => {
|
||||
Opcode::BandImm => {
|
||||
let a = part_to_imm64(pos, a);
|
||||
let b = part_to_value(pos, root, b).unwrap();
|
||||
let val = pos.ins().band_imm(b, a);
|
||||
pos.func.dfg.value_def(val).unwrap_inst().into()
|
||||
}
|
||||
Operator::Bor => {
|
||||
Opcode::Bor => {
|
||||
let a = part_to_value(pos, root, a).unwrap();
|
||||
let b = part_to_value(pos, root, b).unwrap();
|
||||
let val = pos.ins().bor(a, b);
|
||||
pos.func.dfg.value_def(val).unwrap_inst().into()
|
||||
}
|
||||
Operator::BorImm => {
|
||||
Opcode::BorImm => {
|
||||
let a = part_to_imm64(pos, a);
|
||||
let b = part_to_value(pos, root, b).unwrap();
|
||||
let val = pos.ins().bor_imm(b, a);
|
||||
pos.func.dfg.value_def(val).unwrap_inst().into()
|
||||
}
|
||||
Operator::Bxor => {
|
||||
Opcode::Bxor => {
|
||||
let a = part_to_value(pos, root, a).unwrap();
|
||||
let b = part_to_value(pos, root, b).unwrap();
|
||||
let val = pos.ins().bxor(a, b);
|
||||
pos.func.dfg.value_def(val).unwrap_inst().into()
|
||||
}
|
||||
Operator::BxorImm => {
|
||||
Opcode::BxorImm => {
|
||||
let a = part_to_imm64(pos, a);
|
||||
let b = part_to_value(pos, root, b).unwrap();
|
||||
let val = pos.ins().bxor_imm(b, a);
|
||||
pos.func.dfg.value_def(val).unwrap_inst().into()
|
||||
}
|
||||
Operator::Iadd => {
|
||||
Opcode::Iadd => {
|
||||
let a = part_to_value(pos, root, a).unwrap();
|
||||
let b = part_to_value(pos, root, b).unwrap();
|
||||
let val = pos.ins().iadd(a, b);
|
||||
pos.func.dfg.value_def(val).unwrap_inst().into()
|
||||
}
|
||||
Operator::IaddImm => {
|
||||
Opcode::IaddImm => {
|
||||
let a = part_to_imm64(pos, a);
|
||||
let b = part_to_value(pos, root, b).unwrap();
|
||||
let val = pos.ins().iadd_imm(b, a);
|
||||
pos.func.dfg.value_def(val).unwrap_inst().into()
|
||||
}
|
||||
Operator::Ifcmp => {
|
||||
Opcode::Ifcmp => {
|
||||
let a = part_to_value(pos, root, a).unwrap();
|
||||
let b = part_to_value(pos, root, b).unwrap();
|
||||
let val = pos.ins().ifcmp(a, b);
|
||||
pos.func.dfg.value_def(val).unwrap_inst().into()
|
||||
}
|
||||
Operator::IfcmpImm => {
|
||||
Opcode::IfcmpImm => {
|
||||
let a = part_to_imm64(pos, a);
|
||||
let b = part_to_value(pos, root, b).unwrap();
|
||||
let val = pos.ins().ifcmp_imm(b, a);
|
||||
pos.func.dfg.value_def(val).unwrap_inst().into()
|
||||
}
|
||||
Operator::Imul => {
|
||||
Opcode::Imul => {
|
||||
let a = part_to_value(pos, root, a).unwrap();
|
||||
let b = part_to_value(pos, root, b).unwrap();
|
||||
let val = pos.ins().imul(a, b);
|
||||
pos.func.dfg.value_def(val).unwrap_inst().into()
|
||||
}
|
||||
Operator::ImulImm => {
|
||||
Opcode::ImulImm => {
|
||||
let a = part_to_imm64(pos, a);
|
||||
let b = part_to_value(pos, root, b).unwrap();
|
||||
let val = pos.ins().imul_imm(b, a);
|
||||
pos.func.dfg.value_def(val).unwrap_inst().into()
|
||||
}
|
||||
Operator::IrsubImm => {
|
||||
Opcode::IrsubImm => {
|
||||
let a = part_to_imm64(pos, a);
|
||||
let b = part_to_value(pos, root, b).unwrap();
|
||||
let val = pos.ins().irsub_imm(b, a);
|
||||
pos.func.dfg.value_def(val).unwrap_inst().into()
|
||||
}
|
||||
Operator::Ishl => {
|
||||
Opcode::Ishl => {
|
||||
let a = part_to_value(pos, root, a).unwrap();
|
||||
let b = part_to_value(pos, root, b).unwrap();
|
||||
let val = pos.ins().ishl(a, b);
|
||||
pos.func.dfg.value_def(val).unwrap_inst().into()
|
||||
}
|
||||
Operator::IshlImm => {
|
||||
Opcode::IshlImm => {
|
||||
let a = part_to_imm64(pos, a);
|
||||
let b = part_to_value(pos, root, b).unwrap();
|
||||
let val = pos.ins().ishl_imm(b, a);
|
||||
pos.func.dfg.value_def(val).unwrap_inst().into()
|
||||
}
|
||||
Operator::Isub => {
|
||||
Opcode::Isub => {
|
||||
let a = part_to_value(pos, root, a).unwrap();
|
||||
let b = part_to_value(pos, root, b).unwrap();
|
||||
let val = pos.ins().isub(a, b);
|
||||
pos.func.dfg.value_def(val).unwrap_inst().into()
|
||||
}
|
||||
Operator::Rotl => {
|
||||
Opcode::Rotl => {
|
||||
let a = part_to_value(pos, root, a).unwrap();
|
||||
let b = part_to_value(pos, root, b).unwrap();
|
||||
let val = pos.ins().rotl(a, b);
|
||||
pos.func.dfg.value_def(val).unwrap_inst().into()
|
||||
}
|
||||
Operator::RotlImm => {
|
||||
Opcode::RotlImm => {
|
||||
let a = part_to_imm64(pos, a);
|
||||
let b = part_to_value(pos, root, b).unwrap();
|
||||
let val = pos.ins().rotl_imm(b, a);
|
||||
pos.func.dfg.value_def(val).unwrap_inst().into()
|
||||
}
|
||||
Operator::Rotr => {
|
||||
Opcode::Rotr => {
|
||||
let a = part_to_value(pos, root, a).unwrap();
|
||||
let b = part_to_value(pos, root, b).unwrap();
|
||||
let val = pos.ins().rotr(a, b);
|
||||
pos.func.dfg.value_def(val).unwrap_inst().into()
|
||||
}
|
||||
Operator::RotrImm => {
|
||||
Opcode::RotrImm => {
|
||||
let a = part_to_imm64(pos, a);
|
||||
let b = part_to_value(pos, root, b).unwrap();
|
||||
let val = pos.ins().rotr_imm(b, a);
|
||||
pos.func.dfg.value_def(val).unwrap_inst().into()
|
||||
}
|
||||
Operator::Sdiv => {
|
||||
Opcode::Sdiv => {
|
||||
let a = part_to_value(pos, root, a).unwrap();
|
||||
let b = part_to_value(pos, root, b).unwrap();
|
||||
let val = pos.ins().sdiv(a, b);
|
||||
pos.func.dfg.value_def(val).unwrap_inst().into()
|
||||
}
|
||||
Operator::SdivImm => {
|
||||
Opcode::SdivImm => {
|
||||
let a = part_to_imm64(pos, a);
|
||||
let b = part_to_value(pos, root, b).unwrap();
|
||||
let val = pos.ins().sdiv_imm(b, a);
|
||||
pos.func.dfg.value_def(val).unwrap_inst().into()
|
||||
}
|
||||
Operator::Srem => {
|
||||
Opcode::Srem => {
|
||||
let a = part_to_value(pos, root, a).unwrap();
|
||||
let b = part_to_value(pos, root, b).unwrap();
|
||||
let val = pos.ins().srem(a, b);
|
||||
pos.func.dfg.value_def(val).unwrap_inst().into()
|
||||
}
|
||||
Operator::SremImm => {
|
||||
Opcode::SremImm => {
|
||||
let a = part_to_imm64(pos, a);
|
||||
let b = part_to_value(pos, root, b).unwrap();
|
||||
let val = pos.ins().srem_imm(b, a);
|
||||
pos.func.dfg.value_def(val).unwrap_inst().into()
|
||||
}
|
||||
Operator::Sshr => {
|
||||
Opcode::Sshr => {
|
||||
let a = part_to_value(pos, root, a).unwrap();
|
||||
let b = part_to_value(pos, root, b).unwrap();
|
||||
let val = pos.ins().sshr(a, b);
|
||||
pos.func.dfg.value_def(val).unwrap_inst().into()
|
||||
}
|
||||
Operator::SshrImm => {
|
||||
Opcode::SshrImm => {
|
||||
let a = part_to_imm64(pos, a);
|
||||
let b = part_to_value(pos, root, b).unwrap();
|
||||
let val = pos.ins().sshr_imm(b, a);
|
||||
pos.func.dfg.value_def(val).unwrap_inst().into()
|
||||
}
|
||||
Operator::Udiv => {
|
||||
Opcode::Udiv => {
|
||||
let a = part_to_value(pos, root, a).unwrap();
|
||||
let b = part_to_value(pos, root, b).unwrap();
|
||||
let val = pos.ins().udiv(a, b);
|
||||
pos.func.dfg.value_def(val).unwrap_inst().into()
|
||||
}
|
||||
Operator::UdivImm => {
|
||||
Opcode::UdivImm => {
|
||||
let a = part_to_imm64(pos, a);
|
||||
let b = part_to_value(pos, root, b).unwrap();
|
||||
let val = pos.ins().udiv_imm(b, a);
|
||||
pos.func.dfg.value_def(val).unwrap_inst().into()
|
||||
}
|
||||
Operator::Urem => {
|
||||
Opcode::Urem => {
|
||||
let a = part_to_value(pos, root, a).unwrap();
|
||||
let b = part_to_value(pos, root, b).unwrap();
|
||||
let val = pos.ins().urem(a, b);
|
||||
pos.func.dfg.value_def(val).unwrap_inst().into()
|
||||
}
|
||||
Operator::UremImm => {
|
||||
Opcode::UremImm => {
|
||||
let a = part_to_imm64(pos, a);
|
||||
let b = part_to_value(pos, root, b).unwrap();
|
||||
let val = pos.ins().urem_imm(b, a);
|
||||
pos.func.dfg.value_def(val).unwrap_inst().into()
|
||||
}
|
||||
Operator::Ushr => {
|
||||
Opcode::Ushr => {
|
||||
let a = part_to_value(pos, root, a).unwrap();
|
||||
let b = part_to_value(pos, root, b).unwrap();
|
||||
let val = pos.ins().ushr(a, b);
|
||||
pos.func.dfg.value_def(val).unwrap_inst().into()
|
||||
}
|
||||
Operator::UshrImm => {
|
||||
Opcode::UshrImm => {
|
||||
let a = part_to_imm64(pos, a);
|
||||
let b = part_to_value(pos, root, b).unwrap();
|
||||
let val = pos.ins().ushr_imm(b, a);
|
||||
|
|
@ -828,7 +1032,7 @@ unsafe impl<'a, 'b> InstructionSet<'b> for &'a dyn TargetIsa {
|
|||
&self,
|
||||
pos: &mut FuncCursor<'b>,
|
||||
root: ValueOrInst,
|
||||
operator: Operator,
|
||||
operator: Opcode,
|
||||
_: Type,
|
||||
a: Part<ValueOrInst>,
|
||||
b: Part<ValueOrInst>,
|
||||
|
|
@ -838,7 +1042,7 @@ unsafe impl<'a, 'b> InstructionSet<'b> for &'a dyn TargetIsa {
|
|||
|
||||
let root = root.resolve_inst(&pos.func.dfg).unwrap();
|
||||
match operator {
|
||||
Operator::Icmp => {
|
||||
Opcode::Icmp => {
|
||||
let cond = a.unwrap_condition_code();
|
||||
let cond = peepmatic_to_intcc(cond);
|
||||
let b = part_to_value(pos, root, b).unwrap();
|
||||
|
|
@ -846,7 +1050,7 @@ unsafe impl<'a, 'b> InstructionSet<'b> for &'a dyn TargetIsa {
|
|||
let val = pos.ins().icmp(cond, b, c);
|
||||
pos.func.dfg.value_def(val).unwrap_inst().into()
|
||||
}
|
||||
Operator::IcmpImm => {
|
||||
Opcode::IcmpImm => {
|
||||
let cond = a.unwrap_condition_code();
|
||||
let cond = peepmatic_to_intcc(cond);
|
||||
let imm = part_to_imm64(pos, b);
|
||||
|
|
@ -854,7 +1058,7 @@ unsafe impl<'a, 'b> InstructionSet<'b> for &'a dyn TargetIsa {
|
|||
let val = pos.ins().icmp_imm(cond, c, imm);
|
||||
pos.func.dfg.value_def(val).unwrap_inst().into()
|
||||
}
|
||||
Operator::Select => {
|
||||
Opcode::Select => {
|
||||
let a = part_to_value(pos, root, a).unwrap();
|
||||
let b = part_to_value(pos, root, b).unwrap();
|
||||
let c = part_to_value(pos, root, c).unwrap();
|
||||
|
|
@ -885,3 +1089,21 @@ unsafe impl<'a, 'b> InstructionSet<'b> for &'a dyn TargetIsa {
|
|||
self.pointer_bits()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[cfg(feature = "x86")]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::isa::lookup;
|
||||
use crate::settings::{builder, Flags};
|
||||
use std::str::FromStr;
|
||||
use target_lexicon::triple;
|
||||
|
||||
#[test]
|
||||
fn get_peepmatic_preopt() {
|
||||
let isa = lookup(triple!("x86_64"))
|
||||
.expect("expect x86 ISA")
|
||||
.finish(Flags::new(builder()));
|
||||
let _ = preopt(&*isa);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Двоичный файл не отображается.
|
|
@ -1,7 +1,5 @@
|
|||
//! A Constant-Phi-Node removal pass.
|
||||
|
||||
use log::info;
|
||||
|
||||
use crate::dominator_tree::DominatorTree;
|
||||
use crate::entity::EntityList;
|
||||
use crate::fx::FxHashMap;
|
||||
|
|
@ -384,7 +382,7 @@ pub fn do_remove_constant_phis(func: &mut Function, domtree: &mut DominatorTree)
|
|||
}
|
||||
}
|
||||
|
||||
info!(
|
||||
log::debug!(
|
||||
"do_remove_constant_phis: done, {} iters. {} formals, of which {} const.",
|
||||
iter_no,
|
||||
state.absvals.len(),
|
||||
|
|
|
|||
|
|
@ -152,8 +152,8 @@ mod tests {
|
|||
assert_eq!(layout_stack(sss, is_leaf, 16), Ok(0));
|
||||
|
||||
// Same for incoming arguments with non-negative offsets.
|
||||
let in0 = sss.make_incoming_arg(types::I64, 0);
|
||||
let in1 = sss.make_incoming_arg(types::I64, 8);
|
||||
let in0 = sss.make_incoming_arg(8, 0);
|
||||
let in1 = sss.make_incoming_arg(8, 8);
|
||||
|
||||
assert_eq!(layout_stack(sss, is_leaf, 1), Ok(0));
|
||||
assert_eq!(layout_stack(sss, is_leaf, 16), Ok(0));
|
||||
|
|
@ -178,7 +178,7 @@ mod tests {
|
|||
|
||||
// An incoming argument with negative offset counts towards the total frame size, but it
|
||||
// should still pack nicely with the spill slots.
|
||||
let in2 = sss.make_incoming_arg(types::I32, -4);
|
||||
let in2 = sss.make_incoming_arg(4, -4);
|
||||
|
||||
assert_eq!(layout_stack(sss, is_leaf, 1), Ok(16));
|
||||
assert_eq!(sss[in0].offset, Some(0));
|
||||
|
|
@ -195,7 +195,7 @@ mod tests {
|
|||
assert_eq!(sss[ss1].offset, Some(-8));
|
||||
|
||||
// Finally, make sure there is room for the outgoing args.
|
||||
let out0 = sss.get_outgoing_arg(types::I32, 0);
|
||||
let out0 = sss.get_outgoing_arg(4, 0);
|
||||
|
||||
assert_eq!(layout_stack(sss, is_leaf, 1), Ok(20));
|
||||
assert_eq!(sss[in0].offset, Some(0));
|
||||
|
|
@ -214,7 +214,7 @@ mod tests {
|
|||
assert_eq!(sss[out0].offset, Some(0));
|
||||
|
||||
// Also test that an unsupported offset is rejected.
|
||||
sss.get_outgoing_arg(types::I8, StackOffset::max_value() - 1);
|
||||
sss.get_outgoing_arg(1, StackOffset::max_value() - 1);
|
||||
assert_eq!(
|
||||
layout_stack(sss, is_leaf, 1),
|
||||
Err(CodegenError::ImplLimitExceeded)
|
||||
|
|
|
|||
|
|
@ -65,9 +65,9 @@ use crate::ir;
|
|||
use crate::ir::entities::AnyEntity;
|
||||
use crate::ir::instructions::{BranchInfo, CallInfo, InstructionFormat, ResolvedConstraint};
|
||||
use crate::ir::{
|
||||
types, ArgumentLoc, Block, Constant, FuncRef, Function, GlobalValue, Inst, InstructionData,
|
||||
JumpTable, Opcode, SigRef, StackSlot, StackSlotKind, Type, Value, ValueDef, ValueList,
|
||||
ValueLoc,
|
||||
types, ArgumentLoc, ArgumentPurpose, Block, Constant, FuncRef, Function, GlobalValue, Inst,
|
||||
InstructionData, JumpTable, Opcode, SigRef, StackSlot, StackSlotKind, Type, Value, ValueDef,
|
||||
ValueList, ValueLoc,
|
||||
};
|
||||
use crate::isa::TargetIsa;
|
||||
use crate::iterators::IteratorExtras;
|
||||
|
|
@ -1473,7 +1473,8 @@ impl<'a> Verifier<'a> {
|
|||
),
|
||||
));
|
||||
}
|
||||
if slot.size != abi.value_type.bytes() {
|
||||
if abi.purpose == ArgumentPurpose::StructArgument(slot.size) {
|
||||
} else if slot.size != abi.value_type.bytes() {
|
||||
return errors.fatal((
|
||||
inst,
|
||||
self.context(inst),
|
||||
|
|
@ -1986,6 +1987,20 @@ impl<'a> Verifier<'a> {
|
|||
))
|
||||
});
|
||||
|
||||
self.func
|
||||
.signature
|
||||
.returns
|
||||
.iter()
|
||||
.enumerate()
|
||||
.for_each(|(i, ret)| {
|
||||
if let ArgumentPurpose::StructArgument(_) = ret.purpose {
|
||||
errors.report((
|
||||
AnyEntity::Function,
|
||||
format!("Return value at position {} can't be an struct argument", i),
|
||||
))
|
||||
}
|
||||
});
|
||||
|
||||
if errors.has_error() {
|
||||
Err(())
|
||||
} else {
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче