diff --git a/.cargo/config.in b/.cargo/config.in index faa60b97f44d..1bbe4ac8cedd 100644 --- a/.cargo/config.in +++ b/.cargo/config.in @@ -65,7 +65,7 @@ rev = "3224e2dee65c0726c448484d4c3c43956b9330ec" [source."https://github.com/bytecodealliance/wasmtime"] git = "https://github.com/bytecodealliance/wasmtime" replace-with = "vendored-sources" -rev = "5e0268a542f612fee36d0256ed1f6a0e18dc02b3" +rev = "1b3b2dbfd00492161032760992a8699d19b640ca" [source."https://github.com/badboy/failure"] git = "https://github.com/badboy/failure" diff --git a/Cargo.lock b/Cargo.lock index ba2194863946..5744f5ce4d36 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -734,7 +734,7 @@ dependencies = [ [[package]] name = "cranelift-bforest" version = "0.66.0" -source = "git+https://github.com/bytecodealliance/wasmtime?rev=5e0268a542f612fee36d0256ed1f6a0e18dc02b3#5e0268a542f612fee36d0256ed1f6a0e18dc02b3" +source = "git+https://github.com/bytecodealliance/wasmtime?rev=1b3b2dbfd00492161032760992a8699d19b640ca#1b3b2dbfd00492161032760992a8699d19b640ca" dependencies = [ "cranelift-entity 0.66.0", ] @@ -742,7 +742,7 @@ dependencies = [ [[package]] name = "cranelift-codegen" version = "0.66.0" -source = "git+https://github.com/bytecodealliance/wasmtime?rev=5e0268a542f612fee36d0256ed1f6a0e18dc02b3#5e0268a542f612fee36d0256ed1f6a0e18dc02b3" +source = "git+https://github.com/bytecodealliance/wasmtime?rev=1b3b2dbfd00492161032760992a8699d19b640ca#1b3b2dbfd00492161032760992a8699d19b640ca" dependencies = [ "byteorder", "cranelift-bforest", @@ -759,7 +759,7 @@ dependencies = [ [[package]] name = "cranelift-codegen-meta" version = "0.66.0" -source = "git+https://github.com/bytecodealliance/wasmtime?rev=5e0268a542f612fee36d0256ed1f6a0e18dc02b3#5e0268a542f612fee36d0256ed1f6a0e18dc02b3" +source = "git+https://github.com/bytecodealliance/wasmtime?rev=1b3b2dbfd00492161032760992a8699d19b640ca#1b3b2dbfd00492161032760992a8699d19b640ca" dependencies = [ "cranelift-codegen-shared", "cranelift-entity 0.66.0", @@ -768,7 +768,7 @@ dependencies = [ [[package]] name = "cranelift-codegen-shared" version = "0.66.0" -source = "git+https://github.com/bytecodealliance/wasmtime?rev=5e0268a542f612fee36d0256ed1f6a0e18dc02b3#5e0268a542f612fee36d0256ed1f6a0e18dc02b3" +source = "git+https://github.com/bytecodealliance/wasmtime?rev=1b3b2dbfd00492161032760992a8699d19b640ca#1b3b2dbfd00492161032760992a8699d19b640ca" [[package]] name = "cranelift-entity" @@ -778,12 +778,12 @@ source = "git+https://github.com/PLSysSec/lucet_sandbox_compiler?rev=5e870faf6f9 [[package]] name = "cranelift-entity" version = "0.66.0" -source = "git+https://github.com/bytecodealliance/wasmtime?rev=5e0268a542f612fee36d0256ed1f6a0e18dc02b3#5e0268a542f612fee36d0256ed1f6a0e18dc02b3" +source = "git+https://github.com/bytecodealliance/wasmtime?rev=1b3b2dbfd00492161032760992a8699d19b640ca#1b3b2dbfd00492161032760992a8699d19b640ca" [[package]] name = "cranelift-frontend" version = "0.66.0" -source = "git+https://github.com/bytecodealliance/wasmtime?rev=5e0268a542f612fee36d0256ed1f6a0e18dc02b3#5e0268a542f612fee36d0256ed1f6a0e18dc02b3" +source = "git+https://github.com/bytecodealliance/wasmtime?rev=1b3b2dbfd00492161032760992a8699d19b640ca#1b3b2dbfd00492161032760992a8699d19b640ca" dependencies = [ "cranelift-codegen", "log", @@ -794,7 +794,7 @@ dependencies = [ [[package]] name = "cranelift-wasm" version = "0.66.0" -source = "git+https://github.com/bytecodealliance/wasmtime?rev=5e0268a542f612fee36d0256ed1f6a0e18dc02b3#5e0268a542f612fee36d0256ed1f6a0e18dc02b3" +source = "git+https://github.com/bytecodealliance/wasmtime?rev=1b3b2dbfd00492161032760992a8699d19b640ca#1b3b2dbfd00492161032760992a8699d19b640ca" dependencies = [ "cranelift-codegen", "cranelift-entity 0.66.0", diff --git a/Cargo.toml b/Cargo.toml index 561e3db51948..1e02e148c799 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -74,8 +74,8 @@ failure_derive = { git = "https://github.com/badboy/failure", rev = "64af847bc5f [patch.crates-io.cranelift-codegen] git = "https://github.com/bytecodealliance/wasmtime" -rev = "5e0268a542f612fee36d0256ed1f6a0e18dc02b3" +rev = "1b3b2dbfd00492161032760992a8699d19b640ca" [patch.crates-io.cranelift-wasm] git = "https://github.com/bytecodealliance/wasmtime" -rev = "5e0268a542f612fee36d0256ed1f6a0e18dc02b3" +rev = "1b3b2dbfd00492161032760992a8699d19b640ca" diff --git a/third_party/rust/cranelift-codegen-meta/.cargo-checksum.json b/third_party/rust/cranelift-codegen-meta/.cargo-checksum.json index 8610375e76a7..c3410a307cb5 100644 --- a/third_party/rust/cranelift-codegen-meta/.cargo-checksum.json +++ b/third_party/rust/cranelift-codegen-meta/.cargo-checksum.json @@ -1 +1 @@ -{"files":{"Cargo.toml":"d01629d478557c181b999c1722b6284435f45f04957d7cb55735b9605136a23e","LICENSE":"268872b9816f90fd8e85db5a28d33f8150ebb8dd016653fb39ef1f94f2686bc5","README.md":"b123f056d0d458396679c5f7f2a16d2762af0258fcda4ac14b6655a95e5a0022","src/cdsl/ast.rs":"84a4b7e3301e3249716958a7aa4ea5ba8c6172e3c02f57ee3880504c4433ff19","src/cdsl/cpu_modes.rs":"996e45b374cfe85ac47c8c86c4459fe4c04b3158102b4c63b6ee434d5eed6a9e","src/cdsl/encodings.rs":"d884a564815a03c23369bcf31d13b122ae5ba84d0c80eda9312f0c0a829bf794","src/cdsl/formats.rs":"63e638305aa3ca6dd409ddf0e5e9605eeac1cc2631103e42fc6cbc87703d9b63","src/cdsl/instructions.rs":"a0f5212fa593caf66371f5ee4b15e501939a9407c4663bff6b3ba356b11ca1b4","src/cdsl/isa.rs":"ccabd6848b69eb069c10db61c7e7f86080777495714bb53d03e663c40541be94","src/cdsl/mod.rs":"0aa827923bf4c45e5ee2359573bd863e00f474acd532739f49dcd74a27553882","src/cdsl/operands.rs":"1c3411504de9c83112ff48e0ff1cfbb2e4ba5a9a15c1716f411ef31a4df59899","src/cdsl/recipes.rs":"80b7cd87332229b569e38086ceee8d557e679b9a32ad2e50bdb15c33337c3418","src/cdsl/regs.rs":"466a42a43355fc7623fe5d8e8d330622207a3af6a80cb9367bc0f06e224c9ee0","src/cdsl/settings.rs":"e6fd9a31925743b93b11f09c9c8271bab6aa2430aa053a2601957b4487df7d77","src/cdsl/type_inference.rs":"1efca8a095ffc899b7527bda6b9d9378c73d7283f8dceaa4819e8af599f8be21","src/cdsl/types.rs":"ff764c9e9c29a05677bff6164e7bc25a0c32655052d77ae580536abba8b1713b","src/cdsl/typevar.rs":"5ae9e5453c3aa8b12a37e5579b602162fa9e153b444e89bb89342614b6a5ed13","src/cdsl/xform.rs":"55da0c3f2403147b535ab6ae5d69c623fbe839edecf2a3af1de84420cd58402d","src/default_map.rs":"101bb0282a124f9c921f6bd095f529e8753621450d783c3273b0b0394c2c5c03","src/error.rs":"e9b11b2feb2d867b94c8810fdc5a6c4e0d9131604a0bfa5340ff2639a55100b4","src/gen_binemit.rs":"515e243420b30d1e01f8ea630282d9b6d78a715e1951f3f20392e19a48164442","src/gen_encodings.rs":"f00cded6b68a9b48c9e3cd39a8b6f0ba136f4062c8f8666109158a72c62c3ed1","src/gen_inst.rs":"88532d2e2c9724dde968d6b046927249c33d2037ab3e3fd1bd7ebfa77fe12bc7","src/gen_legalizer.rs":"a5e507eb46649a28252582cfc1907c77c9266fec7f92e959a03258bed7d124e9","src/gen_registers.rs":"a904119ed803c9de24dedd15149a65337ffc168bb1d63df53d7fdebfb5f4b158","src/gen_settings.rs":"f3cc3d31f6cc898f30606caf084f0de220db2d3b1b5e5e4145fa7c9a9a1597e2","src/gen_types.rs":"f6c090e1646a43bf2fe81ae0a7029cc6f7dc6d43285368f56d86c35a21c469a6","src/isa/arm32/mod.rs":"da18cb40c1a0a6b613ddefcc38a5d01d02c95de6f233ebd4ad84fefb992c008b","src/isa/arm64/mod.rs":"3a815eaa478d82b7f8b536b83f9debb6b79ec860f99fea6485f209a836c6939a","src/isa/mod.rs":"be483f9a406f603e69603f9489a41a53ee02aa0ece07f7ca396956dfe3815f71","src/isa/riscv/encodings.rs":"8abb1968d917588bc5fc5f5be6dd66bdec23ac456ba65f8138237c8e891e843c","src/isa/riscv/mod.rs":"a7b461a30bbfbc1e3b33645422ff40d5b1761c30cb5d4a8aa12e9a3b7f7aee51","src/isa/riscv/recipes.rs":"5be3bf7c9ba3c51ece384b7eee75a8f7fa0cbacc6a5babc9d0e1d92a2e54a4c2","src/isa/x86/encodings.rs":"75fa07c819b467857d1f4231123e16e519312c9a58f5e7b9c633b0768fd88b42","src/isa/x86/instructions.rs":"d4d581448f8f7bd5afb033650af0026468eecc6f4184b3bb7c06232bf08c456b","src/isa/x86/legalize.rs":"186c688dd8ac773f2b2c4c1f1cbdb7a66ca13a8ed90c03f87dfe7fdaa12c15b3","src/isa/x86/mod.rs":"31571c281318e6f9bf17680feb96830983f5c1f9811aa4a89736f99f3d9a1831","src/isa/x86/opcodes.rs":"745ef09f4927b5334d68155fa047910ef96311feef7ec20964bb033c3419cd3c","src/isa/x86/recipes.rs":"c63469f430e457554acf1534f6fe8f37b41984d38d272e023aa0d93b778dc993","src/isa/x86/registers.rs":"4be0a45d8acd465c31746b7976124025b06b453e3f6d587f93efb5af0e12b1a8","src/isa/x86/settings.rs":"47a5e9fb3b7917cfe817d56dcc77c0470545e451e0f38a875af0531fbd9b6a58","src/lib.rs":"23259ba28aa8f0b3586e9c60f4e67ae50660369f146f2a94249e8cff7d07b27b","src/shared/entities.rs":"90f774a70e1c2a2e9a553c07a5e80e0fe54cf127434bd83e67274bba4e1a19ba","src/shared/formats.rs":"2f8cbb008778a49b60efac4647dffef654d225823e03ca6272af2678666dc423","src/shared/immediates.rs":"e4a57657f6af9853794804eb41c01204a2c13a632f44f55d90e156a4b98c5f65","src/shared/instructions.rs":"38e65efb654dd39e9929b8506ba94d6214d996d727815829b659fd1a3b73bdce","src/shared/legalize.rs":"e8fd35104c1907c0e9453fb98372373aea20b54af10457156f6abd86929099dc","src/shared/mod.rs":"c219625990bf15507ac1077b349ce20e5312d4e4707426183676d469e78792b7","src/shared/settings.rs":"7800f51d97a95d572310f6c80ded59c1c84cf3ba06f9425f4205f88ac46b4e98","src/shared/types.rs":"4702df132f4b5d70cc9411ec5221ba0b1bd4479252274e0223ae57b6d0331247","src/srcgen.rs":"dcfc159c8599270f17e6a978c4be255abca51556b5ef0da497faec4a4a1e62ce","src/unique_table.rs":"31aa54330ca4786af772d32e8cb6158b6504b88fa93fe177bf0c6cbe545a8d35"},"package":null} \ No newline at end of file +{"files":{"Cargo.toml":"d01629d478557c181b999c1722b6284435f45f04957d7cb55735b9605136a23e","LICENSE":"268872b9816f90fd8e85db5a28d33f8150ebb8dd016653fb39ef1f94f2686bc5","README.md":"b123f056d0d458396679c5f7f2a16d2762af0258fcda4ac14b6655a95e5a0022","src/cdsl/ast.rs":"84a4b7e3301e3249716958a7aa4ea5ba8c6172e3c02f57ee3880504c4433ff19","src/cdsl/cpu_modes.rs":"996e45b374cfe85ac47c8c86c4459fe4c04b3158102b4c63b6ee434d5eed6a9e","src/cdsl/encodings.rs":"d884a564815a03c23369bcf31d13b122ae5ba84d0c80eda9312f0c0a829bf794","src/cdsl/formats.rs":"63e638305aa3ca6dd409ddf0e5e9605eeac1cc2631103e42fc6cbc87703d9b63","src/cdsl/instructions.rs":"a0f5212fa593caf66371f5ee4b15e501939a9407c4663bff6b3ba356b11ca1b4","src/cdsl/isa.rs":"ccabd6848b69eb069c10db61c7e7f86080777495714bb53d03e663c40541be94","src/cdsl/mod.rs":"0aa827923bf4c45e5ee2359573bd863e00f474acd532739f49dcd74a27553882","src/cdsl/operands.rs":"1c3411504de9c83112ff48e0ff1cfbb2e4ba5a9a15c1716f411ef31a4df59899","src/cdsl/recipes.rs":"80b7cd87332229b569e38086ceee8d557e679b9a32ad2e50bdb15c33337c3418","src/cdsl/regs.rs":"466a42a43355fc7623fe5d8e8d330622207a3af6a80cb9367bc0f06e224c9ee0","src/cdsl/settings.rs":"e6fd9a31925743b93b11f09c9c8271bab6aa2430aa053a2601957b4487df7d77","src/cdsl/type_inference.rs":"1efca8a095ffc899b7527bda6b9d9378c73d7283f8dceaa4819e8af599f8be21","src/cdsl/types.rs":"50620fb2a6271a7c9126dc30c433a1bf25646a4d84511f5745650aaaec700f42","src/cdsl/typevar.rs":"5ae9e5453c3aa8b12a37e5579b602162fa9e153b444e89bb89342614b6a5ed13","src/cdsl/xform.rs":"55da0c3f2403147b535ab6ae5d69c623fbe839edecf2a3af1de84420cd58402d","src/default_map.rs":"101bb0282a124f9c921f6bd095f529e8753621450d783c3273b0b0394c2c5c03","src/error.rs":"e9b11b2feb2d867b94c8810fdc5a6c4e0d9131604a0bfa5340ff2639a55100b4","src/gen_binemit.rs":"515e243420b30d1e01f8ea630282d9b6d78a715e1951f3f20392e19a48164442","src/gen_encodings.rs":"f00cded6b68a9b48c9e3cd39a8b6f0ba136f4062c8f8666109158a72c62c3ed1","src/gen_inst.rs":"1ff123ab481b48d82e13363043dfc98eaef837bbf6af485b8259c3863550e29c","src/gen_legalizer.rs":"a5e507eb46649a28252582cfc1907c77c9266fec7f92e959a03258bed7d124e9","src/gen_registers.rs":"a904119ed803c9de24dedd15149a65337ffc168bb1d63df53d7fdebfb5f4b158","src/gen_settings.rs":"f3cc3d31f6cc898f30606caf084f0de220db2d3b1b5e5e4145fa7c9a9a1597e2","src/gen_types.rs":"f6c090e1646a43bf2fe81ae0a7029cc6f7dc6d43285368f56d86c35a21c469a6","src/isa/arm32/mod.rs":"da18cb40c1a0a6b613ddefcc38a5d01d02c95de6f233ebd4ad84fefb992c008b","src/isa/arm64/mod.rs":"3a815eaa478d82b7f8b536b83f9debb6b79ec860f99fea6485f209a836c6939a","src/isa/mod.rs":"be483f9a406f603e69603f9489a41a53ee02aa0ece07f7ca396956dfe3815f71","src/isa/riscv/encodings.rs":"8abb1968d917588bc5fc5f5be6dd66bdec23ac456ba65f8138237c8e891e843c","src/isa/riscv/mod.rs":"a7b461a30bbfbc1e3b33645422ff40d5b1761c30cb5d4a8aa12e9a3b7f7aee51","src/isa/riscv/recipes.rs":"5be3bf7c9ba3c51ece384b7eee75a8f7fa0cbacc6a5babc9d0e1d92a2e54a4c2","src/isa/x86/encodings.rs":"e9f1645fec6e4b5cfba9b08cfff70f9d1a5ad3b392f5ee9f40cb1a8669a7c689","src/isa/x86/instructions.rs":"d4d581448f8f7bd5afb033650af0026468eecc6f4184b3bb7c06232bf08c456b","src/isa/x86/legalize.rs":"186c688dd8ac773f2b2c4c1f1cbdb7a66ca13a8ed90c03f87dfe7fdaa12c15b3","src/isa/x86/mod.rs":"31571c281318e6f9bf17680feb96830983f5c1f9811aa4a89736f99f3d9a1831","src/isa/x86/opcodes.rs":"745ef09f4927b5334d68155fa047910ef96311feef7ec20964bb033c3419cd3c","src/isa/x86/recipes.rs":"3a349dea0231dff14c255837e7717904c2e9b350f6fb4ea66a85f5aea7f29184","src/isa/x86/registers.rs":"4be0a45d8acd465c31746b7976124025b06b453e3f6d587f93efb5af0e12b1a8","src/isa/x86/settings.rs":"47a5e9fb3b7917cfe817d56dcc77c0470545e451e0f38a875af0531fbd9b6a58","src/lib.rs":"23259ba28aa8f0b3586e9c60f4e67ae50660369f146f2a94249e8cff7d07b27b","src/shared/entities.rs":"90f774a70e1c2a2e9a553c07a5e80e0fe54cf127434bd83e67274bba4e1a19ba","src/shared/formats.rs":"2f8cbb008778a49b60efac4647dffef654d225823e03ca6272af2678666dc423","src/shared/immediates.rs":"e4a57657f6af9853794804eb41c01204a2c13a632f44f55d90e156a4b98c5f65","src/shared/instructions.rs":"56d3c9c7b112d16f71625c64e95bfac8baab70c5f806160bcb495218a74143ad","src/shared/legalize.rs":"e8fd35104c1907c0e9453fb98372373aea20b54af10457156f6abd86929099dc","src/shared/mod.rs":"c219625990bf15507ac1077b349ce20e5312d4e4707426183676d469e78792b7","src/shared/settings.rs":"7800f51d97a95d572310f6c80ded59c1c84cf3ba06f9425f4205f88ac46b4e98","src/shared/types.rs":"4702df132f4b5d70cc9411ec5221ba0b1bd4479252274e0223ae57b6d0331247","src/srcgen.rs":"dcfc159c8599270f17e6a978c4be255abca51556b5ef0da497faec4a4a1e62ce","src/unique_table.rs":"31aa54330ca4786af772d32e8cb6158b6504b88fa93fe177bf0c6cbe545a8d35"},"package":null} \ No newline at end of file diff --git a/third_party/rust/cranelift-codegen-meta/src/cdsl/types.rs b/third_party/rust/cranelift-codegen-meta/src/cdsl/types.rs index d971f45c6170..7e03c873dbd4 100644 --- a/third_party/rust/cranelift-codegen-meta/src/cdsl/types.rs +++ b/third_party/rust/cranelift-codegen-meta/src/cdsl/types.rs @@ -407,6 +407,8 @@ impl fmt::Debug for VectorType { #[derive(Clone, Copy, PartialEq, Eq, Hash)] pub(crate) enum SpecialType { Flag(shared_types::Flag), + // FIXME remove once the old style backends are removed. + StructArgument, } impl SpecialType { @@ -421,6 +423,9 @@ impl SpecialType { "CPU flags representing the result of a floating point comparison. These flags can be tested with a :type:`floatcc` condition code.", ), + SpecialType::StructArgument => { + String::from("After legalization sarg_t arguments will get this type.") + } } } @@ -428,6 +433,7 @@ impl SpecialType { pub fn lane_bits(self) -> u64 { match self { SpecialType::Flag(_) => 0, + SpecialType::StructArgument => 0, } } @@ -436,6 +442,7 @@ impl SpecialType { match self { SpecialType::Flag(shared_types::Flag::IFlags) => 1, SpecialType::Flag(shared_types::Flag::FFlags) => 2, + SpecialType::StructArgument => 3, } } } @@ -445,6 +452,7 @@ impl fmt::Display for SpecialType { match *self { SpecialType::Flag(shared_types::Flag::IFlags) => write!(f, "iflags"), SpecialType::Flag(shared_types::Flag::FFlags) => write!(f, "fflags"), + SpecialType::StructArgument => write!(f, "sarg_t"), } } } @@ -456,6 +464,7 @@ impl fmt::Debug for SpecialType { "{}", match *self { SpecialType::Flag(_) => format!("FlagsType({})", self), + SpecialType::StructArgument => format!("StructArgument"), } ) } @@ -469,12 +478,14 @@ impl From for SpecialType { pub(crate) struct SpecialTypeIterator { flag_iter: shared_types::FlagIterator, + done: bool, } impl SpecialTypeIterator { fn new() -> Self { Self { flag_iter: shared_types::FlagIterator::new(), + done: false, } } } @@ -485,7 +496,12 @@ impl Iterator for SpecialTypeIterator { if let Some(f) = self.flag_iter.next() { Some(SpecialType::from(f)) } else { - None + if !self.done { + self.done = true; + Some(SpecialType::StructArgument) + } else { + None + } } } } diff --git a/third_party/rust/cranelift-codegen-meta/src/gen_inst.rs b/third_party/rust/cranelift-codegen-meta/src/gen_inst.rs index bda3e035694b..a2760b34d757 100644 --- a/third_party/rust/cranelift-codegen-meta/src/gen_inst.rs +++ b/third_party/rust/cranelift-codegen-meta/src/gen_inst.rs @@ -407,7 +407,11 @@ fn gen_opcodes(all_inst: &AllInstructions, fmt: &mut Formatter) { All instructions from all supported ISAs are present. "#, ); + fmt.line("#[repr(u16)]"); fmt.line("#[derive(Copy, Clone, PartialEq, Eq, Debug, Hash)]"); + fmt.line( + r#"#[cfg_attr(feature = "enable-peepmatic", derive(serde::Serialize, serde::Deserialize))]"# + ); // We explicitly set the discriminant of the first variant to 1, which allows us to take // advantage of the NonZero optimization, meaning that wrapping enums can use the 0 @@ -589,6 +593,24 @@ fn gen_opcodes(all_inst: &AllInstructions, fmt: &mut Formatter) { fmt.empty_line(); } +fn gen_try_from(all_inst: &AllInstructions, fmt: &mut Formatter) { + fmt.line("impl core::convert::TryFrom for Opcode {"); + fmt.indent(|fmt| { + fmt.line("type Error = ();"); + fmt.line("#[inline]"); + fmt.line("fn try_from(x: u16) -> Result {"); + fmt.indent(|fmt| { + fmtln!(fmt, "if 0 < x && x <= {} {{", all_inst.len()); + fmt.indent(|fmt| fmt.line("Ok(unsafe { core::mem::transmute(x) })")); + fmt.line("} else {"); + fmt.indent(|fmt| fmt.line("Err(())")); + fmt.line("}"); + }); + fmt.line("}"); + }); + fmt.line("}"); +} + /// Get the value type constraint for an SSA value operand, where /// `ctrl_typevar` is the controlling type variable. /// @@ -1147,7 +1169,10 @@ pub(crate) fn generate( gen_instruction_data_impl(&formats, &mut fmt); fmt.empty_line(); gen_opcodes(all_inst, &mut fmt); + fmt.empty_line(); gen_type_constraints(all_inst, &mut fmt); + fmt.empty_line(); + gen_try_from(all_inst, &mut fmt); fmt.update_file(opcode_filename, out_dir)?; // Instruction builder. diff --git a/third_party/rust/cranelift-codegen-meta/src/isa/x86/encodings.rs b/third_party/rust/cranelift-codegen-meta/src/isa/x86/encodings.rs index da04019a1b26..9ee12656c0c5 100644 --- a/third_party/rust/cranelift-codegen-meta/src/isa/x86/encodings.rs +++ b/third_party/rust/cranelift-codegen-meta/src/isa/x86/encodings.rs @@ -465,6 +465,7 @@ fn define_moves(e: &mut PerCpuModeEncodings, shared_defs: &SharedDefinitions, r: let sextend = shared.by_name("sextend"); let set_pinned_reg = shared.by_name("set_pinned_reg"); let uextend = shared.by_name("uextend"); + let dummy_sarg_t = shared.by_name("dummy_sarg_t"); // Shorthands for recipes. let rec_copysp = r.template("copysp"); @@ -482,6 +483,7 @@ fn define_moves(e: &mut PerCpuModeEncodings, shared_defs: &SharedDefinitions, r: let rec_umr_reg_to_ssa = r.template("umr_reg_to_ssa"); let rec_urm_noflags = r.template("urm_noflags"); let rec_urm_noflags_abcd = r.template("urm_noflags_abcd"); + let rec_dummy_sarg_t = r.recipe("dummy_sarg_t"); // The pinned reg is fixed to a certain value entirely user-controlled, so it generates nothing! e.enc64_rec(get_pinned_reg.bind(I64), rec_get_pinned_reg, 0); @@ -747,6 +749,8 @@ fn define_moves(e: &mut PerCpuModeEncodings, shared_defs: &SharedDefinitions, r: copy_to_ssa.bind(F32), rec_furm_reg_to_ssa.opcodes(&MOVSS_LOAD), ); + + e.enc_32_64_rec(dummy_sarg_t, rec_dummy_sarg_t, 0); } #[inline(never)] diff --git a/third_party/rust/cranelift-codegen-meta/src/isa/x86/recipes.rs b/third_party/rust/cranelift-codegen-meta/src/isa/x86/recipes.rs index 74645d0b5922..56c5e41c95e1 100644 --- a/third_party/rust/cranelift-codegen-meta/src/isa/x86/recipes.rs +++ b/third_party/rust/cranelift-codegen-meta/src/isa/x86/recipes.rs @@ -1270,6 +1270,12 @@ pub(crate) fn define<'shared>( ); } + recipes.add_recipe( + EncodingRecipeBuilder::new("dummy_sarg_t", &formats.nullary, 0) + .operands_out(vec![Stack::new(gpr)]) + .emit(""), + ); + // XX+rd id with Abs4 function relocation. recipes.add_template_recipe( EncodingRecipeBuilder::new("fnaddr4", &formats.func_addr, 4) diff --git a/third_party/rust/cranelift-codegen-meta/src/shared/instructions.rs b/third_party/rust/cranelift-codegen-meta/src/shared/instructions.rs index 1c06c4a32576..057ae7a0b2a6 100644 --- a/third_party/rust/cranelift-codegen-meta/src/shared/instructions.rs +++ b/third_party/rust/cranelift-codegen-meta/src/shared/instructions.rs @@ -1863,6 +1863,31 @@ pub(crate) fn define( .can_load(true), ); + let Sarg = &TypeVar::new( + "Sarg", + "Any scalar or vector type with at most 128 lanes", + TypeSetBuilder::new() + .specials(vec![crate::cdsl::types::SpecialType::StructArgument]) + .build(), + ); + let sarg_t = &Operand::new("sarg_t", Sarg); + + // FIXME remove once the old style codegen backends are removed. + ig.push( + Inst::new( + "dummy_sarg_t", + r#" + This creates a sarg_t + + This instruction is internal and should not be created by + Cranelift users. + "#, + &formats.nullary, + ) + .operands_in(vec![]) + .operands_out(vec![sarg_t]), + ); + let src = &Operand::new("src", &imm.regunit); let dst = &Operand::new("dst", &imm.regunit); diff --git a/third_party/rust/cranelift-codegen/.cargo-checksum.json b/third_party/rust/cranelift-codegen/.cargo-checksum.json index 23553f336ac3..c654579ddb30 100644 --- a/third_party/rust/cranelift-codegen/.cargo-checksum.json +++ b/third_party/rust/cranelift-codegen/.cargo-checksum.json @@ -1 +1 @@ -{"files":{"Cargo.toml":"9102d40ab5d223babb54fa16365798a344ff983f56870ad5efd5c1e9f0127286","LICENSE":"268872b9816f90fd8e85db5a28d33f8150ebb8dd016653fb39ef1f94f2686bc5","README.md":"e5127227a7db4a8aa92fa6613ed71801025790e696bb41b0323fb7f3c6f7495a","build.rs":"8914b08140f4fea44138a67363d48e614450bae7bd5dd308062ee4d192d68079","src/abi.rs":"cfc08ffeae7d2333750ba8b0f50a94e79b26bcd81bad997de37609493742d30e","src/binemit/memorysink.rs":"378cf70ad5114a8a056d642b0c2d42b4285fe6902855a43085e30575a67d8a4d","src/binemit/mod.rs":"38ec613f6dd8e4f4ffd05d5ce6c0a65d38f4d5c8ef3f44f69e5e68441e8c729a","src/binemit/relaxation.rs":"0547f1a33ea176b281d6f09d1bd7a253dcba895785008e61c114861f1a86307f","src/binemit/shrink.rs":"552d64dff3b044bca77f39891d0e39ee619f6ec0669bf9917132c97becea79b0","src/binemit/stackmap.rs":"106e022548c7dbd35c2f6bc2f8bc5131fc0550c7063daa9702f1151ef3f16108","src/bitset.rs":"ff667c19a63a6a9bcf06f99a46b6db839323f5d3d5cc5e7b00f1ab77d331fc77","src/cfg_printer.rs":"a4cd85ecb2c403b29dc3fc6c60ee7f97b6ed602e8ba88ec87accb988d009271e","src/constant_hash.rs":"8f53afb38891fbdab8553c7a89e61f0ce30947a22fb24ae84523c9a1ab025c3f","src/context.rs":"f3a82a9fb74706dd4e5f328f7f86e798c56071d1e0884c31ddfbdcef2a28f6dc","src/cursor.rs":"eaa0e4ea65bec30aa79e2fb62e89c713b1adec6eeddd5592237d7fcce47c5fa7","src/dbg.rs":"1898d94cff0975815eb348651702e95c8f2f63886501d3b7043ee75668480472","src/dce.rs":"b98545dbf5c8d0c4a33b2ec6cd905d6371fe843481ad608ff59b4a009fc9be19","src/divconst_magic_numbers.rs":"e7f8f46f3a25ed7155890e9d76c10f5dde9dc5bbcebb623c8166fa415abdd959","src/dominator_tree.rs":"b3a5c7837150390a78ade90807965dfcb8768a7e3ae9ee02f2e4a1ad8f3bafa9","src/flowgraph.rs":"71490e2f7a1ea905758a258b2bebe77f125302f446f9839dd60003fdafaef5fe","src/fx.rs":"8a5d07487906d8316a179e826fcd817a92a4860686256a6fd9d78ba47c63f330","src/inst_predicates.rs":"4c776b22b541d29ff6699dec19b2f7d9b14448b1c0a0a3dbf4aaf64d5fc6f5c2","src/ir/builder.rs":"3425957b118a0c40847ef7f473543baef5a24f0c7d8af198955180c8f297ee5a","src/ir/constant.rs":"9c42014b7182a89bc2387e13d3956bbbd98025f32a3abeb5d253d2ba56ad279c","src/ir/dfg.rs":"e803858c6cef60cec09a82d686a73715d0eb142ac96742023cd6112dcf87073a","src/ir/entities.rs":"f0b49b9c351b94703d6c878b564e53a30010c142c47effaf603ec4ade784686b","src/ir/extfunc.rs":"e53c9a31b0af1976ce14a5b126ddb38913f170f501125d305e50754d49f4612f","src/ir/extname.rs":"977b2e380094e99cfdd080112bda709ca61a6d942e1e4c2015d39eaf3ea80f4c","src/ir/function.rs":"3034e03ac2e6bcf300b91c9df03e144550ff6c91ef76b105c48ccbc7a752401b","src/ir/globalvalue.rs":"2f3a54cc2ade91439536b02c46ce790c3634a386a4cc4d27d7da7ad929b8bb90","src/ir/heap.rs":"a59d3e5901412b53c0b53a8cdf10765ff5921de9c410ae9acea226c89827df3c","src/ir/immediates.rs":"2f5366e9cd9f7731fdf4d9038bdd3c9976ea6dce38626ede289b37580407da16","src/ir/instructions.rs":"4e6dbf44fb63f35d7f3288f34d90c9734d6339634d746669eff2a274efd4e704","src/ir/jumptable.rs":"184fa640957e8b0e451dc594e4b348f9aff5fb5b54ab3d856ab19df25f4f3322","src/ir/layout.rs":"2956643a822e67a0c1196f8d3c81af11d0c0122b2d97427ce3ed0c57bb633cbf","src/ir/libcall.rs":"7c9255bdef9a16937d817a3bfd542be4c2734eea3c8b8a870ff411eac2efaad8","src/ir/memflags.rs":"dbcf3798ab66dc764b73fb7f139a621c54cc6bcc683f1f70a33ed7e8c3486bfd","src/ir/mod.rs":"5329068e89f79fe230f0530b14652c35b07ab8d8f4070742a7dde4a3d5dab531","src/ir/progpoint.rs":"a985a7fccde1e985ca24bf2344f16faa4cda7cffb30cb56bf27fabbb5727bc62","src/ir/sourceloc.rs":"67c66adace734645ddf311ecb22e975f20756e07c91e10f698a830cfc2cd08aa","src/ir/stackslot.rs":"5d489d21f17c9992d0862c38c661fab2a19e025a7f29a4417d0336d9bfc5f7f0","src/ir/table.rs":"dcc3b663a989b2b084402b08dc9a0e928dbd052e194a46a1886cc6f0cf1a5f2c","src/ir/trapcode.rs":"71c1f66c3202c966ad58a03bf0d58a236b1bdcc80e48e690925e7332e9299351","src/ir/types.rs":"71c642c411f37608ce5579eda0d8099e577a9a4a0135902ca4b54313ca5e2395","src/ir/valueloc.rs":"628f292ee6b8375e07f360edaf3609b522fd4d7bea131fb8ad3cab237927e7ab","src/isa/aarch64/abi.rs":"440febb641645d6f7513ec78770c1aaaa3e5aefd0427988ead0c0b85b5cdbe33","src/isa/aarch64/inst/args.rs":"aebe60b10e6dc05cf2e7970a6d8fa8ac62fde77a9c43b0c428ee119c5404da80","src/isa/aarch64/inst/emit.rs":"977509de0ee5c9bae217322d68fc069d7be98d0a05a50e9da739ef7903a7c5f3","src/isa/aarch64/inst/emit_tests.rs":"12ce34d99216838d798a4f62daf0001db093ae69d2b932737026133bcba52ca0","src/isa/aarch64/inst/imms.rs":"ca353eabfdae7c1bdc35919b39ff8a88c8fb598fed3df629e258847c544dd535","src/isa/aarch64/inst/mod.rs":"478fab87afac85c0ff5650e12019656e00de45231c357b7fe4d61103d06d3491","src/isa/aarch64/inst/regs.rs":"2e3f52d48b5963a85a275823ee1fd8bc5debf8ee310c4e8cddc77d04f4da45ea","src/isa/aarch64/lower.rs":"6d69d9cc646df6185951322d3aad055555a17327a5a4ea23a5f1df380ec73e6a","src/isa/aarch64/lower_inst.rs":"2bcc76ff3d5c886cf50ced7f31a24c9cf0a35e8ada9db5c45eb50f00b09eb307","src/isa/aarch64/mod.rs":"1943690939e630dd6fc79631cd4f43368df7fa70e4247198c5eb2a7a67fcf54b","src/isa/arm32/abi.rs":"59abc42d75445f7f3335b035655083772e98e413ee58a72fc4e224620327b5ea","src/isa/arm32/binemit.rs":"eecd7296c27d2520db8e86dd462833ecf06afd33548b03ae676c02454cdd13c2","src/isa/arm32/enc_tables.rs":"e94b12af802de59484cab1124d2ac8a88d08433f6e1a476724ed0403f4b5967f","src/isa/arm32/mod.rs":"63a47a6def4eecc4527a1bd1207b4f08d533a5e17e79eda245ebc62e3c862185","src/isa/arm32/registers.rs":"100033c78e2a891ad90a2765137bd678f1d11ae4ba61974d1083106fa1324037","src/isa/arm32/settings.rs":"2314460f885c24f9571d640f9737a8e0b7d20ca02bcda1127f878fd3891c0529","src/isa/call_conv.rs":"e4782f3a45d292971ad64c7ca6c8f2f3ffe06003bd4bdc47cb3094979b8383bb","src/isa/constraints.rs":"296f473a95146a743ecb73d8d5908675be02e37607efd287f55322549dc25763","src/isa/enc_tables.rs":"382e714f9500afc292c563cb66d4c963d6787e58f197b1db242db7a099c22b9a","src/isa/encoding.rs":"22e21fdb8e15859c69e4f836bb61e1a82cd6838d093ce5fe641a90f16fb65c9e","src/isa/mod.rs":"572d6216f0e188b275d9f31f7930d40fa2b539a939c6ba459b3d8d3e1b10844b","src/isa/registers.rs":"61840d736b1943c3e54ac324db6f7de4f76170800f047dde267dcc9aa2d53e6a","src/isa/riscv/abi.rs":"aa60b701efcef417ee1262a95398343578dc1a30decc8e11044b74d41654ec51","src/isa/riscv/binemit.rs":"264d223da311d4482ebf2f55438b665c67b163058251bc78173c76ba983a31ef","src/isa/riscv/enc_tables.rs":"8491f2082b24c7dedeb7c36cfd913bf9aeaa0a4c8fc754166e9285f4ae002f40","src/isa/riscv/mod.rs":"03ee02848dbc3325a5ef38e66c05be571c380fbe4ca520b4f87c7572db228beb","src/isa/riscv/registers.rs":"6275ec3ef19195d16556c1856cb23095f25a80c31d6b429eaa749d549515a6d3","src/isa/riscv/settings.rs":"e3c063add822ca366e845f5a19d25b56635e828b157a37d29aca5355650e6a66","src/isa/stack.rs":"c391216fb8ee6566e3b14aa4bd83ba85aef2bd23422a9dca2c8c6f47016372e2","src/isa/test_utils.rs":"df889e5a4fe6d1069ca87a34ba6260e077de9b0839db5e5c5780f18983aaeeef","src/isa/unwind.rs":"dab61943d7ca89e099604df2a36dba3b8a661fcf7c10dc13e1257ff8f4890bb3","src/isa/unwind/systemv.rs":"f93c36a366470a54ce2af229eea5ef7c299d28b34093dd656f3c5ad87ea94386","src/isa/unwind/winx64.rs":"2b2cbf0803c7967ecb0159ec2c649b45070f9d79a4f2e425ba51e6017b8167a3","src/isa/x64/abi.rs":"03637a97adca31b54f2a66acfc0ebe6577cd76985267a32fc3a9a2c235f3f1c4","src/isa/x64/inst/args.rs":"cd4db9d1b88c2e3fa49e93336aec1694df2a7721965c22d0b096e926689e7ea5","src/isa/x64/inst/emit.rs":"6706df968958d49f54d656d75fb70af6cad4c53a912920ee7647d07676605a79","src/isa/x64/inst/emit_tests.rs":"6177758896e5626cc0f023904736e86b0ad71663e0bc6adede11b51fe7933afe","src/isa/x64/inst/mod.rs":"da9ded8dff39d775fa1bb3daff100c8446f795133c5bd9e28bc7bac3d0c87aaf","src/isa/x64/inst/regs.rs":"87400ce16de570db0399522a991b10b579e533bb8d97763285a4590115e39428","src/isa/x64/lower.rs":"71c873f87bb71fd0106a41a171169e3f13c6f53147767b2261eb880f6eb29a42","src/isa/x64/mod.rs":"e6d762ec6d1e4d8dd94e43254a79df37419d9bfa64149efabfc8218c0a90c594","src/isa/x64/settings.rs":"8e318036aab3e2c94edfcbf87137325e5423c1c04e14a3fac9e3cc73cebfa6e1","src/isa/x86/abi.rs":"b57605ccd6ddd70a976427f404d3bb72e4f05a17e71d5290b5ec1ec20f67c666","src/isa/x86/binemit.rs":"fb5051471cd9c860455a0c1d74aec7d919d4c7229ada10222631c3694f9a091f","src/isa/x86/enc_tables.rs":"f8e9453eaa7b69b72a373741150db5c951415e7a3ad5c8f5319edadd1aa5cad0","src/isa/x86/mod.rs":"d22ab9590f1c93ac1b60ef8b264c3dc596a0a3d979914b711378b167fece15be","src/isa/x86/registers.rs":"1abbc1aa24c6fc4c7e610b6e840eab29046b821de08cf57fc05e2c2f665479c0","src/isa/x86/settings.rs":"d3e403db3507830f79bcc976c17340b57052cf1b50877fcf1a79549f2a054458","src/isa/x86/unwind.rs":"2fb77f98437c425b3c6438a9e964501568579ca47f74c1a7eba47ab59c7a5046","src/isa/x86/unwind/systemv.rs":"fe3a4e404d2cff5d831a07d82c17bbd3bb539b976a0f9825051208d4520e5403","src/isa/x86/unwind/winx64.rs":"f2e7af1e3d93b0c044adfb0c0e9aa67d13eafeda89e80b399adb528f8707cdc8","src/iterators.rs":"d399a80bc65b97f7d650978e19086dcc1d280ac96d60628eac3d17eb363b0d71","src/legalizer/boundary.rs":"9daba50ac9138b076137a3416f48e20584901630456656221ec090a849f32962","src/legalizer/call.rs":"be6074c64c1a00e5e81159dd94c8401fef62205b22c15e07e0c56cf922554d00","src/legalizer/globalvalue.rs":"a5d09ee41a04ba991d7f5d2d8d8c30a209748d38501a005e0ea568df2663cbb5","src/legalizer/heap.rs":"a6026d44c9ce31e0a21413c50581985dad1584700fde9dbab0b2cefafa5c9d14","src/legalizer/libcall.rs":"859662cfc49a479a0b0ebad401f806eb067bddbc4be01994131302c2a89a8183","src/legalizer/mod.rs":"27f6c0d4d5938e096bbcd1777d6c07da50d0b7e9741559662f71c842bc6a850e","src/legalizer/split.rs":"697f08886dbf35fcc69eccc7b597986a58cc73ca7d3cf5d581fffc658a0dad33","src/legalizer/table.rs":"c36d03525312e3191aba8ee00c26a87c1ea200f9a9a0370f0cc84eeacff71786","src/lib.rs":"baac4bced5536908fd80a14a12b9b89bba9c3ea799d939949abdfaa0a8a46ea2","src/licm.rs":"75e94228c37a7d32cc9b0d51644439f9b1837768df67fd3de08ee43b8cdf8123","src/loop_analysis.rs":"4f23c08df9bc95195d68e9721a3a040c6192276ad33894c1225647034f01b23d","src/machinst/abi.rs":"2f2b743748cdbc8cfc85619bbb52471ae748b4d484e646f7972b6b28be0cc3d6","src/machinst/adapter.rs":"caf5fcde25b0f41b49d108e3da72f6d9e5dfa97e24d5171698cf9eba1d283399","src/machinst/blockorder.rs":"04387238c1855051a44f8faffb76983514251a031af7d1837224551b8f574b60","src/machinst/buffer.rs":"40cbbb124b11d0817c8d498b6db1b7679005e4908351cecf89d0c91edfd2bb49","src/machinst/compile.rs":"e246d37f48d88ead92a101449076c16748a9aaa8e839c4bb19070ba1aa795cbf","src/machinst/lower.rs":"fcee3d960fe63185560730334c99a45e5e3dc82a990f0460c619f86569883d2b","src/machinst/mod.rs":"bcbdce377110923caca0f148a7708e06c6c9282dd706479be66f052be731ef06","src/machinst/pretty_print.rs":"6076d9ae0ec48ada9e674ad46c25f48e502adb6b8049c6e1edbcb3b68bd37f87","src/machinst/vcode.rs":"e98b50f21e009eb39b786ad98b5b63f43614a0720925fe3a933407b9877e2cdf","src/nan_canonicalization.rs":"dd853d0b1c1b5274513e9fb24b8beb8db877aca8fdfea5ceccbd1a80309c8842","src/partition_slice.rs":"861e778f887ea5d0841d730963565690fd8c298edddf08848d0d215ae5125eba","src/peepmatic.rs":"9b9c77486ff912718b501010db459b0a2ecaa3b5a192e32bb95af13c0f109491","src/postopt.rs":"ab74e2811909805d467d470da5e66879328c8f47db263422efedf3f1c449d8b2","src/predicates.rs":"d4fa993d8b3036ac9e19d1c1d8880ab5b33030fff0a38d65e2a24b9f9d3956c9","src/preopt.peepmatic":"8b6a6c0f4bf6dcf06df1cc30c467ac39e285ccd85f13f4bc76496f5fd6268a51","src/preopt.serialized":"a7c08b359a2473c58beb2873a34069bd3257a140d4599f7248132e3620010313","src/print_errors.rs":"075b74520f47070b839b43714f55a46a7cc2697b9b0f24a7162d481b7e46b7d2","src/redundant_reload_remover.rs":"2c72cc013f33e1257425592ef4ee2b9437ab3dc84d9759589c15fd217bde83a2","src/regalloc/affinity.rs":"ec5d688c24043a8aa72efcfbfddc14497cd9bab288c9d339e5c0150cdade2b1d","src/regalloc/branch_splitting.rs":"32e34197f84e30cff758f4db611a9c70dd587dd8d094729c34aa00303538c0d0","src/regalloc/coalescing.rs":"154842e7f380f2626c698283dbc5e0d5e7f0cc5d22b938e90312d17b71a8bb72","src/regalloc/coloring.rs":"ded1d8e531c38412fb19fe746fed65a6b6598819a29cd76c9b4bd5c4d0d6011a","src/regalloc/context.rs":"d85f86a8a79e7c939c0e696d30632ae3856001de75411828fc57c0b5b93e63ef","src/regalloc/diversion.rs":"2e474940b0c38610ca231faba7c7c3cfadf401a2f24247b6f3730ac862fce21f","src/regalloc/live_value_tracker.rs":"845dc3f43cc6b795fea51bf293e7c6ab4961d59ab6ca2670fcab7a2a9bd996be","src/regalloc/liveness.rs":"0b027b8e4444a715af1b93d594a293d2fd430ad06940da05b06a4750535e9652","src/regalloc/liverange.rs":"2e98802e90868051b53ddc8555db0ea98aabc77df7789df2a92627650a43227e","src/regalloc/mod.rs":"50399d6569687a87bf1481304aca42506d946e34465e38ca8093e06485ab5fb6","src/regalloc/pressure.rs":"8408213afe07d4532da699f6604aff111c7061d71b585a84c5ec8db31582314c","src/regalloc/register_set.rs":"c740d10a061c4b8527ce319842b519d743e93e64db53851360f9ca2c099fd652","src/regalloc/reload.rs":"2132bd4cf45ce60b7799277d36bda35c05064ee1c60798388b8f55a0668fca47","src/regalloc/safepoint.rs":"8695ff4bd3ef88f4476a24206163c3a17b2b99ecfccd23c9f1f4bc9dbae9e046","src/regalloc/solver.rs":"5ad745ce9075ae8ca742602411f260036df4598695a4f5f0905bd91efe2c69c9","src/regalloc/spilling.rs":"3b75be8be6568a091dd8a1fd174b099859c6e9969c03bd765b5fb50f52fcccb5","src/regalloc/virtregs.rs":"a01b5d3cb1753e344c6663dd73de00dd452d442990f89477610b22c86c9afdfd","src/remove_constant_phis.rs":"3ee19fa855371c7b113d919b48dbb377ad184646f2ccc2c760f359b4b8d53424","src/result.rs":"7164a1b35f26aeb9a6eda79f773f64ecb97b80b50f5b01ea5d34b64361160cbd","src/scoped_hash_map.rs":"c8d0071ce7e19438e9995b5fca1ea0fca81234811943b06b344331f9742c3718","src/settings.rs":"ec3ae1db27cc2f307f27729bd0bf620053442c10a31ff70a917abf6b0ae76514","src/simple_gvn.rs":"1de1d0c0e028833350eda7186243f255c9db97fe04f0e6fa688b8a710caa78be","src/simple_preopt.rs":"cac21be7637415f54be27af6135c1cc777352146b47bf25ac8e0b30cf5ab4d44","src/stack_layout.rs":"a2a7118ce32053501357a2f56c506ff4c19937c09856e459ee9d361d74a23d9e","src/timing.rs":"eb6d6688f886cfbe287347c59a5970a11244175a6377c7f2106ced308ab13de7","src/topo_order.rs":"c092ee7a44e5f14962501eafd4478dfb855ce66af15d9c94a9b244ea30d6e991","src/unreachable_code.rs":"baea08a55b1e7eb2379fa2c4bb5ed4f5a536a35eafcb377f8ab79dc41d14d3d4","src/value_label.rs":"e464557e5bab579773929fcfca843e553af201174da1a73460d199446abc7fc7","src/verifier/cssa.rs":"2590b0ecbc134bbedac50915ed9c9e054c89f81e455c7bc0f37d4ddf57a38d05","src/verifier/flags.rs":"233a4c6fb42e32d92bcbef4ec094a26aa79bdd25cb478847236b6ce5d88d3d54","src/verifier/liveness.rs":"b6ab6dfb1390cea8091b71a6f2fd629ee356987b6a0714e8773d7b0eb7fa889f","src/verifier/locations.rs":"2b4e62e1bb79551725414b5a77425c00e9ad56ad766d6293db1eb261b64f51f9","src/verifier/mod.rs":"233e8420b95dba0e261d6a85ebc8b6164b25a505883071ec82544f02c8ae0c07","src/write.rs":"d045e8269cd306c76d45925296517908a9cec3a5b3c736511c87f7bcbe6cde4f"},"package":null} \ No newline at end of file +{"files":{"Cargo.toml":"04bab37553829e4392c9d030df9b82a5518cee020c6987ba36d3a6c7830188b4","LICENSE":"268872b9816f90fd8e85db5a28d33f8150ebb8dd016653fb39ef1f94f2686bc5","README.md":"e5127227a7db4a8aa92fa6613ed71801025790e696bb41b0323fb7f3c6f7495a","build.rs":"afe67e345d21f9a137ecad473bb595a8b4983d5d435ca0505c91021954c215a7","src/abi.rs":"8922d75a807798945c64ee135f7f22c58e97d8412c6d5b6bf6c4e75c6823d308","src/binemit/memorysink.rs":"378cf70ad5114a8a056d642b0c2d42b4285fe6902855a43085e30575a67d8a4d","src/binemit/mod.rs":"38ec613f6dd8e4f4ffd05d5ce6c0a65d38f4d5c8ef3f44f69e5e68441e8c729a","src/binemit/relaxation.rs":"0547f1a33ea176b281d6f09d1bd7a253dcba895785008e61c114861f1a86307f","src/binemit/shrink.rs":"552d64dff3b044bca77f39891d0e39ee619f6ec0669bf9917132c97becea79b0","src/binemit/stackmap.rs":"106e022548c7dbd35c2f6bc2f8bc5131fc0550c7063daa9702f1151ef3f16108","src/bitset.rs":"ff667c19a63a6a9bcf06f99a46b6db839323f5d3d5cc5e7b00f1ab77d331fc77","src/cfg_printer.rs":"a4cd85ecb2c403b29dc3fc6c60ee7f97b6ed602e8ba88ec87accb988d009271e","src/constant_hash.rs":"8f53afb38891fbdab8553c7a89e61f0ce30947a22fb24ae84523c9a1ab025c3f","src/context.rs":"f3a82a9fb74706dd4e5f328f7f86e798c56071d1e0884c31ddfbdcef2a28f6dc","src/cursor.rs":"eaa0e4ea65bec30aa79e2fb62e89c713b1adec6eeddd5592237d7fcce47c5fa7","src/dbg.rs":"1898d94cff0975815eb348651702e95c8f2f63886501d3b7043ee75668480472","src/dce.rs":"b98545dbf5c8d0c4a33b2ec6cd905d6371fe843481ad608ff59b4a009fc9be19","src/divconst_magic_numbers.rs":"e7f8f46f3a25ed7155890e9d76c10f5dde9dc5bbcebb623c8166fa415abdd959","src/dominator_tree.rs":"b3a5c7837150390a78ade90807965dfcb8768a7e3ae9ee02f2e4a1ad8f3bafa9","src/flowgraph.rs":"71490e2f7a1ea905758a258b2bebe77f125302f446f9839dd60003fdafaef5fe","src/fx.rs":"8a5d07487906d8316a179e826fcd817a92a4860686256a6fd9d78ba47c63f330","src/inst_predicates.rs":"4c776b22b541d29ff6699dec19b2f7d9b14448b1c0a0a3dbf4aaf64d5fc6f5c2","src/ir/builder.rs":"3425957b118a0c40847ef7f473543baef5a24f0c7d8af198955180c8f297ee5a","src/ir/constant.rs":"9c42014b7182a89bc2387e13d3956bbbd98025f32a3abeb5d253d2ba56ad279c","src/ir/dfg.rs":"e803858c6cef60cec09a82d686a73715d0eb142ac96742023cd6112dcf87073a","src/ir/entities.rs":"f0b49b9c351b94703d6c878b564e53a30010c142c47effaf603ec4ade784686b","src/ir/extfunc.rs":"7fd50239108ac9891ceb8a3104878a143cc86a0904dabba0813c609b90d86f9a","src/ir/extname.rs":"977b2e380094e99cfdd080112bda709ca61a6d942e1e4c2015d39eaf3ea80f4c","src/ir/function.rs":"3034e03ac2e6bcf300b91c9df03e144550ff6c91ef76b105c48ccbc7a752401b","src/ir/globalvalue.rs":"2f3a54cc2ade91439536b02c46ce790c3634a386a4cc4d27d7da7ad929b8bb90","src/ir/heap.rs":"a59d3e5901412b53c0b53a8cdf10765ff5921de9c410ae9acea226c89827df3c","src/ir/immediates.rs":"2f5366e9cd9f7731fdf4d9038bdd3c9976ea6dce38626ede289b37580407da16","src/ir/instructions.rs":"29fc7629ed25421bdfcd9571bf9e227764b3100738c996e4dd2abd24c2c18d90","src/ir/jumptable.rs":"184fa640957e8b0e451dc594e4b348f9aff5fb5b54ab3d856ab19df25f4f3322","src/ir/layout.rs":"2956643a822e67a0c1196f8d3c81af11d0c0122b2d97427ce3ed0c57bb633cbf","src/ir/libcall.rs":"7c9255bdef9a16937d817a3bfd542be4c2734eea3c8b8a870ff411eac2efaad8","src/ir/memflags.rs":"dbcf3798ab66dc764b73fb7f139a621c54cc6bcc683f1f70a33ed7e8c3486bfd","src/ir/mod.rs":"5329068e89f79fe230f0530b14652c35b07ab8d8f4070742a7dde4a3d5dab531","src/ir/progpoint.rs":"a985a7fccde1e985ca24bf2344f16faa4cda7cffb30cb56bf27fabbb5727bc62","src/ir/sourceloc.rs":"67c66adace734645ddf311ecb22e975f20756e07c91e10f698a830cfc2cd08aa","src/ir/stackslot.rs":"f158471dd1f90ffc64465ad406db6ef8d9d2366ce5575ec0451ccb9ac76656ed","src/ir/table.rs":"dcc3b663a989b2b084402b08dc9a0e928dbd052e194a46a1886cc6f0cf1a5f2c","src/ir/trapcode.rs":"71c1f66c3202c966ad58a03bf0d58a236b1bdcc80e48e690925e7332e9299351","src/ir/types.rs":"dfd9470aa4325adc252eb7ebb2951a5b98d22d0dc31be482419f5cbbb74bf86d","src/ir/valueloc.rs":"628f292ee6b8375e07f360edaf3609b522fd4d7bea131fb8ad3cab237927e7ab","src/isa/aarch64/abi.rs":"440febb641645d6f7513ec78770c1aaaa3e5aefd0427988ead0c0b85b5cdbe33","src/isa/aarch64/inst/args.rs":"d8b74e8c8a2d834915ff52e152b5c97dc7c60ed00c9ee1d9a28acb292d3f684c","src/isa/aarch64/inst/emit.rs":"977509de0ee5c9bae217322d68fc069d7be98d0a05a50e9da739ef7903a7c5f3","src/isa/aarch64/inst/emit_tests.rs":"12ce34d99216838d798a4f62daf0001db093ae69d2b932737026133bcba52ca0","src/isa/aarch64/inst/imms.rs":"ca353eabfdae7c1bdc35919b39ff8a88c8fb598fed3df629e258847c544dd535","src/isa/aarch64/inst/mod.rs":"37a1795258def59aa0add9ba2d32c60d59314100ad61a298a9b8ea901de87d80","src/isa/aarch64/inst/regs.rs":"2e3f52d48b5963a85a275823ee1fd8bc5debf8ee310c4e8cddc77d04f4da45ea","src/isa/aarch64/lower.rs":"7815dc09ce658199db51026283a431dcc9eb7a7b7aad67da46f2b34c5a116cf8","src/isa/aarch64/lower_inst.rs":"4f62854c549f4a6939559a9ae9ac33572773111794576b7eff5b294cb234fd54","src/isa/aarch64/mod.rs":"1943690939e630dd6fc79631cd4f43368df7fa70e4247198c5eb2a7a67fcf54b","src/isa/arm32/abi.rs":"59abc42d75445f7f3335b035655083772e98e413ee58a72fc4e224620327b5ea","src/isa/arm32/binemit.rs":"eecd7296c27d2520db8e86dd462833ecf06afd33548b03ae676c02454cdd13c2","src/isa/arm32/enc_tables.rs":"e94b12af802de59484cab1124d2ac8a88d08433f6e1a476724ed0403f4b5967f","src/isa/arm32/mod.rs":"63a47a6def4eecc4527a1bd1207b4f08d533a5e17e79eda245ebc62e3c862185","src/isa/arm32/registers.rs":"100033c78e2a891ad90a2765137bd678f1d11ae4ba61974d1083106fa1324037","src/isa/arm32/settings.rs":"2314460f885c24f9571d640f9737a8e0b7d20ca02bcda1127f878fd3891c0529","src/isa/call_conv.rs":"e4782f3a45d292971ad64c7ca6c8f2f3ffe06003bd4bdc47cb3094979b8383bb","src/isa/constraints.rs":"296f473a95146a743ecb73d8d5908675be02e37607efd287f55322549dc25763","src/isa/enc_tables.rs":"382e714f9500afc292c563cb66d4c963d6787e58f197b1db242db7a099c22b9a","src/isa/encoding.rs":"22e21fdb8e15859c69e4f836bb61e1a82cd6838d093ce5fe641a90f16fb65c9e","src/isa/mod.rs":"572d6216f0e188b275d9f31f7930d40fa2b539a939c6ba459b3d8d3e1b10844b","src/isa/registers.rs":"61840d736b1943c3e54ac324db6f7de4f76170800f047dde267dcc9aa2d53e6a","src/isa/riscv/abi.rs":"aa60b701efcef417ee1262a95398343578dc1a30decc8e11044b74d41654ec51","src/isa/riscv/binemit.rs":"264d223da311d4482ebf2f55438b665c67b163058251bc78173c76ba983a31ef","src/isa/riscv/enc_tables.rs":"8491f2082b24c7dedeb7c36cfd913bf9aeaa0a4c8fc754166e9285f4ae002f40","src/isa/riscv/mod.rs":"03ee02848dbc3325a5ef38e66c05be571c380fbe4ca520b4f87c7572db228beb","src/isa/riscv/registers.rs":"6275ec3ef19195d16556c1856cb23095f25a80c31d6b429eaa749d549515a6d3","src/isa/riscv/settings.rs":"e3c063add822ca366e845f5a19d25b56635e828b157a37d29aca5355650e6a66","src/isa/stack.rs":"c391216fb8ee6566e3b14aa4bd83ba85aef2bd23422a9dca2c8c6f47016372e2","src/isa/test_utils.rs":"df889e5a4fe6d1069ca87a34ba6260e077de9b0839db5e5c5780f18983aaeeef","src/isa/unwind.rs":"dab61943d7ca89e099604df2a36dba3b8a661fcf7c10dc13e1257ff8f4890bb3","src/isa/unwind/systemv.rs":"f93c36a366470a54ce2af229eea5ef7c299d28b34093dd656f3c5ad87ea94386","src/isa/unwind/winx64.rs":"2b2cbf0803c7967ecb0159ec2c649b45070f9d79a4f2e425ba51e6017b8167a3","src/isa/x64/abi.rs":"298a8e3ef43667d77f42fc02dc6dc4f5eb4a4e09d55ba529d07507e71a556ea2","src/isa/x64/inst/args.rs":"9c81c844f7aefa96bab793b8ced4e7c083c665ab679e165d49c216e6b4b00b56","src/isa/x64/inst/emit.rs":"2d4f7e15a812f0a63d4a006fa1be5e3d313446589fb82c9aa691aae90076eeee","src/isa/x64/inst/emit_tests.rs":"8526fc05d7aaeb87ecc22d7ad291018b6a4bf145ed11f83db4db3ddf6781bbee","src/isa/x64/inst/mod.rs":"eaf78c8ff27f85477b7f0885eedb9ba9adf02a9233e8c9911213c9f2ba7d040e","src/isa/x64/inst/regs.rs":"87400ce16de570db0399522a991b10b579e533bb8d97763285a4590115e39428","src/isa/x64/lower.rs":"350949641cb40bf9909b94c760d20efb6a05c7d782b716ad4c38310fac727899","src/isa/x64/mod.rs":"e6d762ec6d1e4d8dd94e43254a79df37419d9bfa64149efabfc8218c0a90c594","src/isa/x64/settings.rs":"8e318036aab3e2c94edfcbf87137325e5423c1c04e14a3fac9e3cc73cebfa6e1","src/isa/x86/abi.rs":"9b6f259585e9e048aa2fc3ecc6b4415ef2c465cd94f93617fdeba85598583759","src/isa/x86/binemit.rs":"fb5051471cd9c860455a0c1d74aec7d919d4c7229ada10222631c3694f9a091f","src/isa/x86/enc_tables.rs":"f8e9453eaa7b69b72a373741150db5c951415e7a3ad5c8f5319edadd1aa5cad0","src/isa/x86/mod.rs":"d22ab9590f1c93ac1b60ef8b264c3dc596a0a3d979914b711378b167fece15be","src/isa/x86/registers.rs":"1abbc1aa24c6fc4c7e610b6e840eab29046b821de08cf57fc05e2c2f665479c0","src/isa/x86/settings.rs":"d3e403db3507830f79bcc976c17340b57052cf1b50877fcf1a79549f2a054458","src/isa/x86/unwind.rs":"2fb77f98437c425b3c6438a9e964501568579ca47f74c1a7eba47ab59c7a5046","src/isa/x86/unwind/systemv.rs":"fe3a4e404d2cff5d831a07d82c17bbd3bb539b976a0f9825051208d4520e5403","src/isa/x86/unwind/winx64.rs":"f2e7af1e3d93b0c044adfb0c0e9aa67d13eafeda89e80b399adb528f8707cdc8","src/iterators.rs":"d399a80bc65b97f7d650978e19086dcc1d280ac96d60628eac3d17eb363b0d71","src/legalizer/boundary.rs":"bd1480dde632dcaa777189f28e82a1736b4adb7173a23313970367f409d441cb","src/legalizer/call.rs":"be6074c64c1a00e5e81159dd94c8401fef62205b22c15e07e0c56cf922554d00","src/legalizer/globalvalue.rs":"a5d09ee41a04ba991d7f5d2d8d8c30a209748d38501a005e0ea568df2663cbb5","src/legalizer/heap.rs":"a6026d44c9ce31e0a21413c50581985dad1584700fde9dbab0b2cefafa5c9d14","src/legalizer/libcall.rs":"859662cfc49a479a0b0ebad401f806eb067bddbc4be01994131302c2a89a8183","src/legalizer/mod.rs":"27f6c0d4d5938e096bbcd1777d6c07da50d0b7e9741559662f71c842bc6a850e","src/legalizer/split.rs":"697f08886dbf35fcc69eccc7b597986a58cc73ca7d3cf5d581fffc658a0dad33","src/legalizer/table.rs":"c36d03525312e3191aba8ee00c26a87c1ea200f9a9a0370f0cc84eeacff71786","src/lib.rs":"baac4bced5536908fd80a14a12b9b89bba9c3ea799d939949abdfaa0a8a46ea2","src/licm.rs":"75e94228c37a7d32cc9b0d51644439f9b1837768df67fd3de08ee43b8cdf8123","src/loop_analysis.rs":"4f23c08df9bc95195d68e9721a3a040c6192276ad33894c1225647034f01b23d","src/machinst/abi.rs":"2f2b743748cdbc8cfc85619bbb52471ae748b4d484e646f7972b6b28be0cc3d6","src/machinst/adapter.rs":"caf5fcde25b0f41b49d108e3da72f6d9e5dfa97e24d5171698cf9eba1d283399","src/machinst/blockorder.rs":"04387238c1855051a44f8faffb76983514251a031af7d1837224551b8f574b60","src/machinst/buffer.rs":"40cbbb124b11d0817c8d498b6db1b7679005e4908351cecf89d0c91edfd2bb49","src/machinst/compile.rs":"e246d37f48d88ead92a101449076c16748a9aaa8e839c4bb19070ba1aa795cbf","src/machinst/lower.rs":"7eb94ce4940a7fb4e77be4c50280f31824e9237f9183e51a8048812d8038fb18","src/machinst/mod.rs":"d6b6d2a87d338741684998fdc6b9848c79c3a94c23629c2031f460a1e267a52e","src/machinst/pretty_print.rs":"6076d9ae0ec48ada9e674ad46c25f48e502adb6b8049c6e1edbcb3b68bd37f87","src/machinst/vcode.rs":"e98b50f21e009eb39b786ad98b5b63f43614a0720925fe3a933407b9877e2cdf","src/nan_canonicalization.rs":"dd853d0b1c1b5274513e9fb24b8beb8db877aca8fdfea5ceccbd1a80309c8842","src/partition_slice.rs":"861e778f887ea5d0841d730963565690fd8c298edddf08848d0d215ae5125eba","src/peepmatic.rs":"78b5fc63ea64356fb4ffc76a3df4663999e36ad1cbfed853609a401072900caf","src/postopt.rs":"ab74e2811909805d467d470da5e66879328c8f47db263422efedf3f1c449d8b2","src/predicates.rs":"d4fa993d8b3036ac9e19d1c1d8880ab5b33030fff0a38d65e2a24b9f9d3956c9","src/preopt.peepmatic":"8b6a6c0f4bf6dcf06df1cc30c467ac39e285ccd85f13f4bc76496f5fd6268a51","src/preopt.serialized":"a7a0968f221417caf91edf9ab9d1a993b8e1aaef009e9f26f477a64b5351de75","src/print_errors.rs":"075b74520f47070b839b43714f55a46a7cc2697b9b0f24a7162d481b7e46b7d2","src/redundant_reload_remover.rs":"2c72cc013f33e1257425592ef4ee2b9437ab3dc84d9759589c15fd217bde83a2","src/regalloc/affinity.rs":"ec5d688c24043a8aa72efcfbfddc14497cd9bab288c9d339e5c0150cdade2b1d","src/regalloc/branch_splitting.rs":"32e34197f84e30cff758f4db611a9c70dd587dd8d094729c34aa00303538c0d0","src/regalloc/coalescing.rs":"154842e7f380f2626c698283dbc5e0d5e7f0cc5d22b938e90312d17b71a8bb72","src/regalloc/coloring.rs":"ded1d8e531c38412fb19fe746fed65a6b6598819a29cd76c9b4bd5c4d0d6011a","src/regalloc/context.rs":"d85f86a8a79e7c939c0e696d30632ae3856001de75411828fc57c0b5b93e63ef","src/regalloc/diversion.rs":"2e474940b0c38610ca231faba7c7c3cfadf401a2f24247b6f3730ac862fce21f","src/regalloc/live_value_tracker.rs":"845dc3f43cc6b795fea51bf293e7c6ab4961d59ab6ca2670fcab7a2a9bd996be","src/regalloc/liveness.rs":"0b027b8e4444a715af1b93d594a293d2fd430ad06940da05b06a4750535e9652","src/regalloc/liverange.rs":"2e98802e90868051b53ddc8555db0ea98aabc77df7789df2a92627650a43227e","src/regalloc/mod.rs":"50399d6569687a87bf1481304aca42506d946e34465e38ca8093e06485ab5fb6","src/regalloc/pressure.rs":"8408213afe07d4532da699f6604aff111c7061d71b585a84c5ec8db31582314c","src/regalloc/register_set.rs":"c740d10a061c4b8527ce319842b519d743e93e64db53851360f9ca2c099fd652","src/regalloc/reload.rs":"2132bd4cf45ce60b7799277d36bda35c05064ee1c60798388b8f55a0668fca47","src/regalloc/safepoint.rs":"8695ff4bd3ef88f4476a24206163c3a17b2b99ecfccd23c9f1f4bc9dbae9e046","src/regalloc/solver.rs":"5ad745ce9075ae8ca742602411f260036df4598695a4f5f0905bd91efe2c69c9","src/regalloc/spilling.rs":"3b75be8be6568a091dd8a1fd174b099859c6e9969c03bd765b5fb50f52fcccb5","src/regalloc/virtregs.rs":"a01b5d3cb1753e344c6663dd73de00dd452d442990f89477610b22c86c9afdfd","src/remove_constant_phis.rs":"0e6c2cdce9229bf9a9275d4895d772b90370ed6b7dfb690e63c245c2a9f6c94d","src/result.rs":"7164a1b35f26aeb9a6eda79f773f64ecb97b80b50f5b01ea5d34b64361160cbd","src/scoped_hash_map.rs":"c8d0071ce7e19438e9995b5fca1ea0fca81234811943b06b344331f9742c3718","src/settings.rs":"ec3ae1db27cc2f307f27729bd0bf620053442c10a31ff70a917abf6b0ae76514","src/simple_gvn.rs":"1de1d0c0e028833350eda7186243f255c9db97fe04f0e6fa688b8a710caa78be","src/simple_preopt.rs":"cac21be7637415f54be27af6135c1cc777352146b47bf25ac8e0b30cf5ab4d44","src/stack_layout.rs":"41d35401faa171d9823e9c6e26c1337f9e16c6b8ba613f0cd98c3c0032930496","src/timing.rs":"eb6d6688f886cfbe287347c59a5970a11244175a6377c7f2106ced308ab13de7","src/topo_order.rs":"c092ee7a44e5f14962501eafd4478dfb855ce66af15d9c94a9b244ea30d6e991","src/unreachable_code.rs":"baea08a55b1e7eb2379fa2c4bb5ed4f5a536a35eafcb377f8ab79dc41d14d3d4","src/value_label.rs":"e464557e5bab579773929fcfca843e553af201174da1a73460d199446abc7fc7","src/verifier/cssa.rs":"2590b0ecbc134bbedac50915ed9c9e054c89f81e455c7bc0f37d4ddf57a38d05","src/verifier/flags.rs":"233a4c6fb42e32d92bcbef4ec094a26aa79bdd25cb478847236b6ce5d88d3d54","src/verifier/liveness.rs":"b6ab6dfb1390cea8091b71a6f2fd629ee356987b6a0714e8773d7b0eb7fa889f","src/verifier/locations.rs":"2b4e62e1bb79551725414b5a77425c00e9ad56ad766d6293db1eb261b64f51f9","src/verifier/mod.rs":"b0054edb93ce7afbca6658ec3f54a83add2b3e4ea5026153c12f4e00d482cba9","src/write.rs":"d045e8269cd306c76d45925296517908a9cec3a5b3c736511c87f7bcbe6cde4f"},"package":null} \ No newline at end of file diff --git a/third_party/rust/cranelift-codegen/Cargo.toml b/third_party/rust/cranelift-codegen/Cargo.toml index d5bd49d0d5aa..26980ed56b74 100644 --- a/third_party/rust/cranelift-codegen/Cargo.toml +++ b/third_party/rust/cranelift-codegen/Cargo.toml @@ -25,8 +25,11 @@ gimli = { version = "0.21.0", default-features = false, features = ["write"], op smallvec = { version = "1.0.0" } thiserror = "1.0.4" byteorder = { version = "1.3.2", default-features = false } -peepmatic-runtime = { path = "../peepmatic/crates/runtime", optional = true, version = "0.2.0" } -regalloc = { version = "0.0.28" } +peepmatic = { path = "../peepmatic", optional = true, version = "0.66.0" } +peepmatic-traits = { path = "../peepmatic/crates/traits", optional = true, version = "0.66.0" } +peepmatic-runtime = { path = "../peepmatic/crates/runtime", optional = true, version = "0.66.0" } +regalloc = "0.0.28" +wast = { version = "15.0.0", optional = true } # It is a goal of the cranelift-codegen crate to have minimal external dependencies. # Please don't add any unless they are essential to the task of creating binary # machine code. Integration tests that need external dependencies can be @@ -34,7 +37,6 @@ regalloc = { version = "0.0.28" } [build-dependencies] cranelift-codegen-meta = { path = "meta", version = "0.66.0" } -peepmatic = { path = "../peepmatic", optional = true, version = "0.66.0" } [features] default = ["std", "unwind"] @@ -80,10 +82,10 @@ regalloc-snapshot = ["bincode", "regalloc/enable-serde"] # Recompile our optimizations that are written in the `peepmatic` DSL into a # compact finite-state transducer automaton. -rebuild-peephole-optimizers = ["peepmatic"] +rebuild-peephole-optimizers = ["peepmatic", "peepmatic-traits", "wast"] # Enable the use of `peepmatic`-generated peephole optimizers. -enable-peepmatic = ["peepmatic-runtime"] +enable-peepmatic = ["peepmatic-runtime", "peepmatic-traits", "serde"] [badges] maintenance = { status = "experimental" } diff --git a/third_party/rust/cranelift-codegen/build.rs b/third_party/rust/cranelift-codegen/build.rs index 2caf32609d3e..c4569088457b 100644 --- a/third_party/rust/cranelift-codegen/build.rs +++ b/third_party/rust/cranelift-codegen/build.rs @@ -90,20 +90,11 @@ fn main() { } #[cfg(feature = "rebuild-peephole-optimizers")] - rebuild_peephole_optimizers(); -} - -#[cfg(feature = "rebuild-peephole-optimizers")] -fn rebuild_peephole_optimizers() { - use std::path::Path; - - let source_path = Path::new("src").join("preopt.peepmatic"); - println!("cargo:rerun-if-changed={}", source_path.display()); - - let preopt = - peepmatic::compile_file(&source_path).expect("failed to compile `src/preopt.peepmatic`"); - - preopt - .serialize_to_file(&Path::new("src").join("preopt.serialized")) - .expect("failed to serialize peephole optimizer to `src/preopt.serialized`"); + { + std::fs::write( + std::path::Path::new(&out_dir).join("CRANELIFT_CODEGEN_PATH"), + cur_dir.to_str().unwrap(), + ) + .unwrap() + } } diff --git a/third_party/rust/cranelift-codegen/src/abi.rs b/third_party/rust/cranelift-codegen/src/abi.rs index cf4780e3536c..883ec546e43d 100644 --- a/third_party/rust/cranelift-codegen/src/abi.rs +++ b/third_party/rust/cranelift-codegen/src/abi.rs @@ -18,6 +18,10 @@ pub enum ArgAction { /// Assign the argument to the given location. Assign(ArgumentLoc), + /// Assign the argument to the given location and change the type to the specified type. + /// This is used by [`ArgumentPurpose::StructArgument`]. + AssignAndChangeType(ArgumentLoc, Type), + /// Convert the argument, then call again. /// /// This action can split an integer type into two smaller integer arguments, or it can split a @@ -119,6 +123,14 @@ pub fn legalize_args(args: &[AbiParam], aa: &mut AA) -> Option< args.to_mut()[argno].location = loc; argno += 1; } + // Assign argument to a location, change type to the requested one and move on to the + // next one. + ArgAction::AssignAndChangeType(loc, ty) => { + let arg = &mut args.to_mut()[argno]; + arg.location = loc; + arg.value_type = ty; + argno += 1; + } // Split this argument into two smaller ones. Then revisit both. ArgAction::Convert(conv) => { debug_assert!( diff --git a/third_party/rust/cranelift-codegen/src/ir/extfunc.rs b/third_party/rust/cranelift-codegen/src/ir/extfunc.rs index 6390f28a4353..16df75805151 100644 --- a/third_party/rust/cranelift-codegen/src/ir/extfunc.rs +++ b/third_party/rust/cranelift-codegen/src/ir/extfunc.rs @@ -282,6 +282,9 @@ pub enum ArgumentPurpose { /// A normal user program value passed to or from a function. Normal, + /// A C struct passed as argument. + StructArgument(u32), + /// Struct return pointer. /// /// When a function needs to return more data than will fit in registers, the caller passes a @@ -334,21 +337,19 @@ pub enum ArgumentPurpose { StackLimit, } -/// Text format names of the `ArgumentPurpose` variants. -static PURPOSE_NAMES: [&str; 8] = [ - "normal", - "sret", - "link", - "fp", - "csr", - "vmctx", - "sigid", - "stack_limit", -]; - impl fmt::Display for ArgumentPurpose { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - f.write_str(PURPOSE_NAMES[*self as usize]) + f.write_str(match self { + Self::Normal => "normal", + Self::StructArgument(size) => return write!(f, "sarg({})", size), + Self::StructReturn => "sret", + Self::Link => "link", + Self::FramePointer => "fp", + Self::CalleeSaved => "csr", + Self::VMContext => "vmctx", + Self::SignatureId => "sigid", + Self::StackLimit => "stack_limit", + }) } } @@ -364,6 +365,14 @@ impl FromStr for ArgumentPurpose { "vmctx" => Ok(Self::VMContext), "sigid" => Ok(Self::SignatureId), "stack_limit" => Ok(Self::StackLimit), + _ if s.starts_with("sarg(") => { + if !s.ends_with(")") { + return Err(()); + } + // Parse 'sarg(size)' + let size: u32 = s["sarg(".len()..s.len() - 1].parse().map_err(|_| ())?; + Ok(Self::StructArgument(size)) + } _ => Err(()), } } @@ -436,16 +445,17 @@ mod tests { #[test] fn argument_purpose() { let all_purpose = [ - ArgumentPurpose::Normal, - ArgumentPurpose::StructReturn, - ArgumentPurpose::Link, - ArgumentPurpose::FramePointer, - ArgumentPurpose::CalleeSaved, - ArgumentPurpose::VMContext, - ArgumentPurpose::SignatureId, - ArgumentPurpose::StackLimit, + (ArgumentPurpose::Normal, "normal"), + (ArgumentPurpose::StructReturn, "sret"), + (ArgumentPurpose::Link, "link"), + (ArgumentPurpose::FramePointer, "fp"), + (ArgumentPurpose::CalleeSaved, "csr"), + (ArgumentPurpose::VMContext, "vmctx"), + (ArgumentPurpose::SignatureId, "sigid"), + (ArgumentPurpose::StackLimit, "stack_limit"), + (ArgumentPurpose::StructArgument(42), "sarg(42)"), ]; - for (&e, &n) in all_purpose.iter().zip(PURPOSE_NAMES.iter()) { + for &(e, n) in &all_purpose { assert_eq!(e.to_string(), n); assert_eq!(Ok(e), n.parse()); } diff --git a/third_party/rust/cranelift-codegen/src/ir/instructions.rs b/third_party/rust/cranelift-codegen/src/ir/instructions.rs index 2ba730b687e7..6c71212020d3 100644 --- a/third_party/rust/cranelift-codegen/src/ir/instructions.rs +++ b/third_party/rust/cranelift-codegen/src/ir/instructions.rs @@ -7,7 +7,9 @@ //! directory. use alloc::vec::Vec; +use core::convert::{TryFrom, TryInto}; use core::fmt::{self, Display, Formatter}; +use core::num::NonZeroU32; use core::ops::{Deref, DerefMut}; use core::str::FromStr; @@ -69,6 +71,24 @@ impl Opcode { } } +impl TryFrom for Opcode { + type Error = (); + + #[inline] + fn try_from(x: NonZeroU32) -> Result { + let x: u16 = x.get().try_into().map_err(|_| ())?; + Self::try_from(x) + } +} + +impl From for NonZeroU32 { + #[inline] + fn from(op: Opcode) -> NonZeroU32 { + let x = op as u8; + NonZeroU32::new(x as u32).unwrap() + } +} + // This trait really belongs in cranelift-reader where it is used by the `.clif` file parser, but since // it critically depends on the `opcode_name()` function which is needed here anyway, it lives in // this module. This also saves us from running the build script twice to generate code for the two diff --git a/third_party/rust/cranelift-codegen/src/ir/stackslot.rs b/third_party/rust/cranelift-codegen/src/ir/stackslot.rs index 5bb70d1f0eea..bad79df5ff03 100644 --- a/third_party/rust/cranelift-codegen/src/ir/stackslot.rs +++ b/third_party/rust/cranelift-codegen/src/ir/stackslot.rs @@ -286,8 +286,8 @@ impl StackSlots { } /// Create a stack slot representing an incoming function argument. - pub fn make_incoming_arg(&mut self, ty: Type, offset: StackOffset) -> StackSlot { - let mut data = StackSlotData::new(StackSlotKind::IncomingArg, ty.bytes()); + pub fn make_incoming_arg(&mut self, size: u32, offset: StackOffset) -> StackSlot { + let mut data = StackSlotData::new(StackSlotKind::IncomingArg, size); debug_assert!(offset <= StackOffset::max_value() - data.size as StackOffset); data.offset = Some(offset); self.push(data) @@ -300,9 +300,7 @@ impl StackSlots { /// /// The requested offset is relative to this function's stack pointer immediately before making /// the call. - pub fn get_outgoing_arg(&mut self, ty: Type, offset: StackOffset) -> StackSlot { - let size = ty.bytes(); - + pub fn get_outgoing_arg(&mut self, size: u32, offset: StackOffset) -> StackSlot { // Look for an existing outgoing stack slot with the same offset and size. let inspos = match self.outgoing.binary_search_by_key(&(offset, size), |&ss| { (self[ss].offset.unwrap(), self[ss].size) @@ -387,9 +385,9 @@ mod tests { fn outgoing() { let mut sss = StackSlots::new(); - let ss0 = sss.get_outgoing_arg(types::I32, 8); - let ss1 = sss.get_outgoing_arg(types::I32, 4); - let ss2 = sss.get_outgoing_arg(types::I64, 8); + let ss0 = sss.get_outgoing_arg(4, 8); + let ss1 = sss.get_outgoing_arg(4, 4); + let ss2 = sss.get_outgoing_arg(8, 8); assert_eq!(sss[ss0].offset, Some(8)); assert_eq!(sss[ss0].size, 4); @@ -400,9 +398,9 @@ mod tests { assert_eq!(sss[ss2].offset, Some(8)); assert_eq!(sss[ss2].size, 8); - assert_eq!(sss.get_outgoing_arg(types::I32, 8), ss0); - assert_eq!(sss.get_outgoing_arg(types::I32, 4), ss1); - assert_eq!(sss.get_outgoing_arg(types::I64, 8), ss2); + assert_eq!(sss.get_outgoing_arg(4, 8), ss0); + assert_eq!(sss.get_outgoing_arg(4, 4), ss1); + assert_eq!(sss.get_outgoing_arg(8, 8), ss2); } #[test] diff --git a/third_party/rust/cranelift-codegen/src/ir/types.rs b/third_party/rust/cranelift-codegen/src/ir/types.rs index c669839da5a3..b9499435b8f8 100644 --- a/third_party/rust/cranelift-codegen/src/ir/types.rs +++ b/third_party/rust/cranelift-codegen/src/ir/types.rs @@ -343,6 +343,7 @@ impl Display for Type { f.write_str(match *self { IFLAGS => "iflags", FFLAGS => "fflags", + SARG_T => "sarg_t", INVALID => panic!("INVALID encountered"), _ => panic!("Unknown Type(0x{:x})", self.0), }) diff --git a/third_party/rust/cranelift-codegen/src/isa/aarch64/inst/args.rs b/third_party/rust/cranelift-codegen/src/isa/aarch64/inst/args.rs index 43e8471ac749..43dcc816e531 100644 --- a/third_party/rust/cranelift-codegen/src/isa/aarch64/inst/args.rs +++ b/third_party/rust/cranelift-codegen/src/isa/aarch64/inst/args.rs @@ -52,6 +52,11 @@ impl ShiftOpShiftImm { pub fn value(self) -> u8 { self.0 } + + /// Mask down to a given number of bits. + pub fn mask(self, bits: u8) -> ShiftOpShiftImm { + ShiftOpShiftImm(self.0 & (bits - 1)) + } } /// A shift operator with an amount, guaranteed to be within range. diff --git a/third_party/rust/cranelift-codegen/src/isa/aarch64/inst/mod.rs b/third_party/rust/cranelift-codegen/src/isa/aarch64/inst/mod.rs index 1c5c6f9a1c02..814f846645e9 100644 --- a/third_party/rust/cranelift-codegen/src/isa/aarch64/inst/mod.rs +++ b/third_party/rust/cranelift-codegen/src/isa/aarch64/inst/mod.rs @@ -2065,7 +2065,12 @@ impl MachInst for Inst { Inst::mov(to_reg, from_reg) } - fn gen_constant(to_reg: Writable, value: u64, ty: Type) -> SmallVec<[Inst; 4]> { + fn gen_constant Writable>( + to_reg: Writable, + value: u64, + ty: Type, + _alloc_tmp: F, + ) -> SmallVec<[Inst; 4]> { if ty == F64 { let mut ret = SmallVec::new(); ret.push(Inst::load_fp_constant64(to_reg, f64::from_bits(value))); diff --git a/third_party/rust/cranelift-codegen/src/isa/aarch64/lower.rs b/third_party/rust/cranelift-codegen/src/isa/aarch64/lower.rs index d60fdfe1440d..39b4b1c0be6a 100644 --- a/third_party/rust/cranelift-codegen/src/isa/aarch64/lower.rs +++ b/third_party/rust/cranelift-codegen/src/isa/aarch64/lower.rs @@ -219,7 +219,11 @@ pub(crate) fn put_input_in_reg>( }; // Generate constants fresh at each use to minimize long-range register pressure. let to_reg = ctx.alloc_tmp(Inst::rc_for_type(ty).unwrap(), ty); - for inst in Inst::gen_constant(to_reg, masked, ty).into_iter() { + for inst in Inst::gen_constant(to_reg, masked, ty, |reg_class, ty| { + ctx.alloc_tmp(reg_class, ty) + }) + .into_iter() + { ctx.emit(inst); } to_reg.to_reg() @@ -317,8 +321,12 @@ fn put_input_in_rs>( // Can we get the shift amount as an immediate? if let Some(shiftimm) = input_to_shiftimm(ctx, shift_amt) { - let reg = put_input_in_reg(ctx, shiftee, narrow_mode); - return ResultRS::RegShift(reg, ShiftOpAndAmt::new(ShiftOp::LSL, shiftimm)); + let shiftee_bits = ty_bits(ctx.input_ty(insn, 0)); + if shiftee_bits <= u8::MAX as usize { + let shiftimm = shiftimm.mask(shiftee_bits as u8); + let reg = put_input_in_reg(ctx, shiftee, narrow_mode); + return ResultRS::RegShift(reg, ShiftOpAndAmt::new(ShiftOp::LSL, shiftimm)); + } } } } diff --git a/third_party/rust/cranelift-codegen/src/isa/aarch64/lower_inst.rs b/third_party/rust/cranelift-codegen/src/isa/aarch64/lower_inst.rs index 88751a1478dc..77fe58aed654 100644 --- a/third_party/rust/cranelift-codegen/src/isa/aarch64/lower_inst.rs +++ b/third_party/rust/cranelift-codegen/src/isa/aarch64/lower_inst.rs @@ -1078,8 +1078,24 @@ pub(crate) fn lower_insn_to_regs>( // Nothing. } - Opcode::Select | Opcode::Selectif | Opcode::SelectifSpectreGuard => { - let cond = if op == Opcode::Select { + Opcode::Select => { + let flag_input = inputs[0]; + let cond = if let Some(icmp_insn) = + maybe_input_insn_via_conv(ctx, flag_input, Opcode::Icmp, Opcode::Bint) + { + let condcode = inst_condcode(ctx.data(icmp_insn)).unwrap(); + let cond = lower_condcode(condcode); + let is_signed = condcode_is_signed(condcode); + lower_icmp_or_ifcmp_to_flags(ctx, icmp_insn, is_signed); + cond + } else if let Some(fcmp_insn) = + maybe_input_insn_via_conv(ctx, flag_input, Opcode::Fcmp, Opcode::Bint) + { + let condcode = inst_fp_condcode(ctx.data(fcmp_insn)).unwrap(); + let cond = lower_fp_condcode(condcode); + lower_fcmp_or_ffcmp_to_flags(ctx, fcmp_insn); + cond + } else { let (cmp_op, narrow_mode) = if ty_bits(ctx.input_ty(insn, 0)) > 32 { (ALUOp::SubS64, NarrowValueMode::ZeroExtend64) } else { @@ -1095,17 +1111,32 @@ pub(crate) fn lower_insn_to_regs>( rm: zero_reg(), }); Cond::Ne - } else { - let condcode = inst_condcode(ctx.data(insn)).unwrap(); - let cond = lower_condcode(condcode); - let is_signed = condcode_is_signed(condcode); - // Verification ensures that the input is always a - // single-def ifcmp. - let ifcmp_insn = maybe_input_insn(ctx, inputs[0], Opcode::Ifcmp).unwrap(); - lower_icmp_or_ifcmp_to_flags(ctx, ifcmp_insn, is_signed); - cond }; + // csel.cond rd, rn, rm + let rd = get_output_reg(ctx, outputs[0]); + let rn = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None); + let rm = put_input_in_reg(ctx, inputs[2], NarrowValueMode::None); + let ty = ctx.output_ty(insn, 0); + let bits = ty_bits(ty); + if ty_is_float(ty) && bits == 32 { + ctx.emit(Inst::FpuCSel32 { cond, rd, rn, rm }); + } else if ty_is_float(ty) && bits == 64 { + ctx.emit(Inst::FpuCSel64 { cond, rd, rn, rm }); + } else { + ctx.emit(Inst::CSel { cond, rd, rn, rm }); + } + } + + Opcode::Selectif | Opcode::SelectifSpectreGuard => { + let condcode = inst_condcode(ctx.data(insn)).unwrap(); + let cond = lower_condcode(condcode); + let is_signed = condcode_is_signed(condcode); + // Verification ensures that the input is always a + // single-def ifcmp. + let ifcmp_insn = maybe_input_insn(ctx, inputs[0], Opcode::Ifcmp).unwrap(); + lower_icmp_or_ifcmp_to_flags(ctx, ifcmp_insn, is_signed); + // csel.COND rd, rn, rm let rd = get_output_reg(ctx, outputs[0]); let rn = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None); @@ -2152,6 +2183,8 @@ pub(crate) fn lower_insn_to_regs>( panic!("x86-specific opcode in supposedly arch-neutral IR!"); } + Opcode::DummySargT => unreachable!(), + Opcode::AvgRound => unimplemented!(), Opcode::Iabs => unimplemented!(), Opcode::Snarrow diff --git a/third_party/rust/cranelift-codegen/src/isa/x64/abi.rs b/third_party/rust/cranelift-codegen/src/isa/x64/abi.rs index dadaa6e35973..25eb6e77d351 100644 --- a/third_party/rust/cranelift-codegen/src/isa/x64/abi.rs +++ b/third_party/rust/cranelift-codegen/src/isa/x64/abi.rs @@ -957,25 +957,39 @@ fn adjust_stack>(ctx: &mut C, amount: u64, is_sub: bool) { } fn load_stack(mem: impl Into, into_reg: Writable, ty: Type) -> Inst { - let ext_mode = match ty { - types::B1 | types::B8 | types::I8 => Some(ExtMode::BQ), - types::B16 | types::I16 => Some(ExtMode::WQ), - types::B32 | types::I32 => Some(ExtMode::LQ), - types::B64 | types::I64 => None, - types::F32 => todo!("f32 load_stack"), - types::F64 => todo!("f64 load_stack"), - _ => unimplemented!("load_stack({})", ty), + let (is_int, ext_mode) = match ty { + types::B1 | types::B8 | types::I8 => (true, Some(ExtMode::BQ)), + types::B16 | types::I16 => (true, Some(ExtMode::WQ)), + types::B32 | types::I32 => (true, Some(ExtMode::LQ)), + types::B64 | types::I64 => (true, None), + types::F32 | types::F64 => (false, None), + _ => panic!("load_stack({})", ty), }; let mem = mem.into(); - match ext_mode { - Some(ext_mode) => Inst::movsx_rm_r( - ext_mode, + + if is_int { + match ext_mode { + Some(ext_mode) => Inst::movsx_rm_r( + ext_mode, + RegMem::mem(mem), + into_reg, + /* infallible load */ None, + ), + None => Inst::mov64_m_r(mem, into_reg, None /* infallible */), + } + } else { + let sse_op = match ty { + types::F32 => SseOpcode::Movss, + types::F64 => SseOpcode::Movsd, + _ => unreachable!(), + }; + Inst::xmm_mov( + sse_op, RegMem::mem(mem), into_reg, - /* infallible load */ None, - ), - None => Inst::mov64_m_r(mem, into_reg, None /* infallible */), + None, /* infallible */ + ) } } @@ -993,7 +1007,12 @@ fn store_stack(mem: impl Into, from_reg: Reg, ty: Type) -> Inst if is_int { Inst::mov_r_m(size, from_reg, mem, /* infallible store */ None) } else { - unimplemented!("f32/f64 store_stack"); + let sse_op = match size { + 4 => SseOpcode::Movss, + 8 => SseOpcode::Movsd, + _ => unreachable!(), + }; + Inst::xmm_mov_r_m(sse_op, from_reg, mem, /* infallible store */ None) } } diff --git a/third_party/rust/cranelift-codegen/src/isa/x64/inst/args.rs b/third_party/rust/cranelift-codegen/src/isa/x64/inst/args.rs index 4090f2d33af7..5697d2dbb5ff 100644 --- a/third_party/rust/cranelift-codegen/src/isa/x64/inst/args.rs +++ b/third_party/rust/cranelift-codegen/src/isa/x64/inst/args.rs @@ -5,7 +5,7 @@ use std::string::{String, ToString}; use regalloc::{RealRegUniverse, Reg, RegClass, RegUsageCollector, RegUsageMapper}; -use crate::ir::condcodes::IntCC; +use crate::ir::condcodes::{FloatCC, IntCC}; use crate::machinst::*; use super::{ @@ -194,6 +194,13 @@ impl RegMemImm { Self::Imm { simm32 } } + /// Asserts that in register mode, the reg class is the one that's expected. + pub(crate) fn assert_regclass_is(&self, expected_reg_class: RegClass) { + if let Self::Reg { reg } = self { + debug_assert_eq!(reg.get_class(), expected_reg_class); + } + } + /// Add the regs mentioned by `self` to `collector`. pub(crate) fn get_regs_as_uses(&self, collector: &mut RegUsageCollector) { match self { @@ -234,6 +241,12 @@ impl RegMem { pub(crate) fn mem(addr: impl Into) -> Self { Self::Mem { addr: addr.into() } } + /// Asserts that in register mode, the reg class is the one that's expected. + pub(crate) fn assert_regclass_is(&self, expected_reg_class: RegClass) { + if let Self::Reg { reg } = self { + debug_assert_eq!(reg.get_class(), expected_reg_class); + } + } /// Add the regs mentioned by `self` to `collector`. pub(crate) fn get_regs_as_uses(&self, collector: &mut RegUsageCollector) { match self { @@ -346,6 +359,7 @@ pub enum SseOpcode { Minsd, Movaps, Movd, + Movq, Movss, Movsd, Mulss, @@ -399,6 +413,7 @@ impl SseOpcode { | SseOpcode::Maxsd | SseOpcode::Minsd | SseOpcode::Movd + | SseOpcode::Movq | SseOpcode::Movsd | SseOpcode::Mulsd | SseOpcode::Sqrtsd @@ -411,7 +426,7 @@ impl SseOpcode { } } - /// Returns the src operand size for an instruction + /// Returns the src operand size for an instruction. pub(crate) fn src_size(&self) -> u8 { match self { SseOpcode::Movd => 4, @@ -445,6 +460,7 @@ impl fmt::Debug for SseOpcode { SseOpcode::Minsd => "minsd", SseOpcode::Movaps => "movaps", SseOpcode::Movd => "movd", + SseOpcode::Movq => "movq", SseOpcode::Movss => "movss", SseOpcode::Movsd => "movsd", SseOpcode::Mulss => "mulss", @@ -620,6 +636,12 @@ pub enum CC { LE = 14, /// > signed NLE = 15, + + /// parity + P = 10, + + /// not parity + NP = 11, } impl CC { @@ -662,6 +684,33 @@ impl CC { CC::LE => CC::NLE, CC::NLE => CC::LE, + + CC::P => CC::NP, + CC::NP => CC::P, + } + } + + pub(crate) fn from_floatcc(floatcc: FloatCC) -> Self { + match floatcc { + FloatCC::Ordered => CC::NP, + FloatCC::Unordered => CC::P, + // Alias for NE + FloatCC::NotEqual | FloatCC::OrderedNotEqual => CC::NZ, + // Alias for E + FloatCC::UnorderedOrEqual => CC::Z, + // Alias for A + FloatCC::GreaterThan => CC::NBE, + // Alias for AE + FloatCC::GreaterThanOrEqual => CC::NB, + FloatCC::UnorderedOrLessThan => CC::B, + FloatCC::UnorderedOrLessThanOrEqual => CC::BE, + FloatCC::Equal + | FloatCC::LessThan + | FloatCC::LessThanOrEqual + | FloatCC::UnorderedOrGreaterThan + | FloatCC::UnorderedOrGreaterThanOrEqual => unimplemented!( + "No single condition code to guarantee ordered. Treat as special case." + ), } } @@ -687,6 +736,8 @@ impl fmt::Debug for CC { CC::NL => "nl", CC::LE => "le", CC::NLE => "nle", + CC::P => "p", + CC::NP => "np", }; write!(fmt, "{}", name) } diff --git a/third_party/rust/cranelift-codegen/src/isa/x64/inst/emit.rs b/third_party/rust/cranelift-codegen/src/isa/x64/inst/emit.rs index 432bbc9a0da5..2ff58a884468 100644 --- a/third_party/rust/cranelift-codegen/src/isa/x64/inst/emit.rs +++ b/third_party/rust/cranelift-codegen/src/isa/x64/inst/emit.rs @@ -1183,6 +1183,28 @@ pub(crate) fn emit( } } + Inst::XmmCmove { + is_64, + cc, + src, + dst, + } => { + let next = sink.get_label(); + + // Jump if cc is *not* set. + one_way_jmp(sink, cc.invert(), next); + + let op = if *is_64 { + SseOpcode::Movsd + } else { + SseOpcode::Movss + }; + let inst = Inst::xmm_unary_rm_r(op, src.clone(), *dst); + inst.emit(sink, flags, state); + + sink.bind_label(next); + } + Inst::Push64 { src } => { match src { RegMemImm::Reg { reg } => { @@ -1462,18 +1484,22 @@ pub(crate) fn emit( sink.bind_label(else_label); } - Inst::XMM_Mov_RM_R { + Inst::XmmUnaryRmR { op, src: src_e, dst: reg_g, srcloc, } => { let rex = RexFlags::clear_w(); + let (prefix, opcode) = match op { SseOpcode::Movaps => (LegacyPrefix::None, 0x0F28), - SseOpcode::Movd => (LegacyPrefix::_66, 0x0F6E), SseOpcode::Movsd => (LegacyPrefix::_F2, 0x0F10), SseOpcode::Movss => (LegacyPrefix::_F3, 0x0F10), + SseOpcode::Sqrtss => (LegacyPrefix::_F3, 0x0F51), + SseOpcode::Sqrtsd => (LegacyPrefix::_F2, 0x0F51), + SseOpcode::Cvtss2sd => (LegacyPrefix::_F3, 0x0F5A), + SseOpcode::Cvtsd2ss => (LegacyPrefix::_F2, 0x0F5A), _ => unimplemented!("Opcode {:?} not implemented", op), }; @@ -1489,7 +1515,7 @@ pub(crate) fn emit( } emit_std_reg_mem(sink, prefix, opcode, 2, reg_g.to_reg(), addr, rex); } - } + }; } Inst::XMM_RM_R { @@ -1500,13 +1526,20 @@ pub(crate) fn emit( let rex = RexFlags::clear_w(); let (prefix, opcode) = match op { SseOpcode::Addss => (LegacyPrefix::_F3, 0x0F58), + SseOpcode::Addsd => (LegacyPrefix::_F2, 0x0F58), SseOpcode::Andps => (LegacyPrefix::None, 0x0F54), SseOpcode::Andnps => (LegacyPrefix::None, 0x0F55), - SseOpcode::Divss => (LegacyPrefix::_F3, 0x0F5E), SseOpcode::Mulss => (LegacyPrefix::_F3, 0x0F59), + SseOpcode::Mulsd => (LegacyPrefix::_F2, 0x0F59), SseOpcode::Orps => (LegacyPrefix::None, 0x0F56), SseOpcode::Subss => (LegacyPrefix::_F3, 0x0F5C), - SseOpcode::Sqrtss => (LegacyPrefix::_F3, 0x0F51), + SseOpcode::Subsd => (LegacyPrefix::_F2, 0x0F5C), + SseOpcode::Minss => (LegacyPrefix::_F3, 0x0F5D), + SseOpcode::Minsd => (LegacyPrefix::_F2, 0x0F5D), + SseOpcode::Divss => (LegacyPrefix::_F3, 0x0F5E), + SseOpcode::Divsd => (LegacyPrefix::_F2, 0x0F5E), + SseOpcode::Maxss => (LegacyPrefix::_F3, 0x0F5F), + SseOpcode::Maxsd => (LegacyPrefix::_F2, 0x0F5F), _ => unimplemented!("Opcode {:?} not implemented", op), }; @@ -1521,25 +1554,72 @@ pub(crate) fn emit( } } - Inst::XMM_Mov_R_M { + Inst::Xmm_Mov_R_M { op, src, dst, srcloc, } => { - let rex = RexFlags::clear_w(); let (prefix, opcode) = match op { - SseOpcode::Movd => (LegacyPrefix::_66, 0x0F7E), SseOpcode::Movss => (LegacyPrefix::_F3, 0x0F11), - _ => unimplemented!("Emit xmm mov r m"), + SseOpcode::Movsd => (LegacyPrefix::_F2, 0x0F11), + _ => unimplemented!("Opcode {:?} not implemented", op), }; - let dst = &dst.finalize(state); if let Some(srcloc) = *srcloc { // Register the offset at which the actual load instruction starts. sink.add_trap(srcloc, TrapCode::HeapOutOfBounds); } - emit_std_reg_mem(sink, prefix, opcode, 2, *src, dst, rex); + emit_std_reg_mem(sink, prefix, opcode, 2, *src, dst, RexFlags::clear_w()); + } + + Inst::XmmToGpr { op, src, dst } => { + let (rex, prefix, opcode) = match op { + SseOpcode::Movd => (RexFlags::clear_w(), LegacyPrefix::_66, 0x0F7E), + SseOpcode::Movq => (RexFlags::set_w(), LegacyPrefix::_66, 0x0F7E), + _ => panic!("unexpected opcode {:?}", op), + }; + emit_std_reg_reg(sink, prefix, opcode, 2, *src, dst.to_reg(), rex); + } + + Inst::GprToXmm { + op, + src: src_e, + dst: reg_g, + } => { + let (rex, prefix, opcode) = match op { + SseOpcode::Movd => (RexFlags::clear_w(), LegacyPrefix::_66, 0x0F6E), + SseOpcode::Movq => (RexFlags::set_w(), LegacyPrefix::_66, 0x0F6E), + _ => panic!("unexpected opcode {:?}", op), + }; + match src_e { + RegMem::Reg { reg: reg_e } => { + emit_std_reg_reg(sink, prefix, opcode, 2, reg_g.to_reg(), *reg_e, rex); + } + RegMem::Mem { addr } => { + let addr = &addr.finalize(state); + emit_std_reg_mem(sink, prefix, opcode, 2, reg_g.to_reg(), addr, rex); + } + } + } + + Inst::XMM_Cmp_RM_R { op, src, dst } => { + let rex = RexFlags::clear_w(); + let (prefix, opcode) = match op { + SseOpcode::Ucomisd => (LegacyPrefix::_66, 0x0F2E), + SseOpcode::Ucomiss => (LegacyPrefix::None, 0x0F2E), + _ => unimplemented!("Emit xmm cmp rm r"), + }; + + match src { + RegMem::Reg { reg } => { + emit_std_reg_reg(sink, prefix, opcode, 2, *dst, *reg, rex); + } + RegMem::Mem { addr } => { + let addr = &addr.finalize(state); + emit_std_reg_mem(sink, prefix, opcode, 2, *dst, addr, rex); + } + } } Inst::LoadExtName { diff --git a/third_party/rust/cranelift-codegen/src/isa/x64/inst/emit_tests.rs b/third_party/rust/cranelift-codegen/src/isa/x64/inst/emit_tests.rs index ee4674729bb8..fc7f4869f5c4 100644 --- a/third_party/rust/cranelift-codegen/src/isa/x64/inst/emit_tests.rs +++ b/third_party/rust/cranelift-codegen/src/isa/x64/inst/emit_tests.rs @@ -2711,7 +2711,8 @@ fn test_x64_emit() { insns.push((Inst::setcc(CC::NLE, w_rsi), "400F9FC6", "setnle %sil")); insns.push((Inst::setcc(CC::Z, w_r14), "410F94C6", "setz %r14b")); insns.push((Inst::setcc(CC::LE, w_r14), "410F9EC6", "setle %r14b")); - + insns.push((Inst::setcc(CC::P, w_r9), "410F9AC1", "setp %r9b")); + insns.push((Inst::setcc(CC::NP, w_r8), "410F9BC0", "setnp %r8b")); // ======================================================== // Cmove insns.push(( @@ -2877,28 +2878,45 @@ fn test_x64_emit() { )); // ======================================================== - // XMM_RM_R + // XMM_CMP_RM_R + + insns.push(( + Inst::xmm_cmp_rm_r(SseOpcode::Ucomiss, RegMem::reg(xmm1), xmm2), + "0F2ED1", + "ucomiss %xmm1, %xmm2", + )); + + insns.push(( + Inst::xmm_cmp_rm_r(SseOpcode::Ucomiss, RegMem::reg(xmm0), xmm9), + "440F2EC8", + "ucomiss %xmm0, %xmm9", + )); + + insns.push(( + Inst::xmm_cmp_rm_r(SseOpcode::Ucomisd, RegMem::reg(xmm13), xmm4), + "66410F2EE5", + "ucomisd %xmm13, %xmm4", + )); + + insns.push(( + Inst::xmm_cmp_rm_r(SseOpcode::Ucomisd, RegMem::reg(xmm11), xmm12), + "66450F2EE3", + "ucomisd %xmm11, %xmm12", + )); + + // ======================================================== + // XMM_RM_R: float binary ops insns.push(( Inst::xmm_rm_r(SseOpcode::Addss, RegMem::reg(xmm1), w_xmm0), "F30F58C1", "addss %xmm1, %xmm0", )); - insns.push(( - Inst::xmm_rm_r(SseOpcode::Subss, RegMem::reg(xmm0), w_xmm1), - "F30F5CC8", - "subss %xmm0, %xmm1", - )); insns.push(( Inst::xmm_rm_r(SseOpcode::Addss, RegMem::reg(xmm11), w_xmm13), "F3450F58EB", "addss %xmm11, %xmm13", )); - insns.push(( - Inst::xmm_rm_r(SseOpcode::Subss, RegMem::reg(xmm12), w_xmm1), - "F3410F5CCC", - "subss %xmm12, %xmm1", - )); insns.push(( Inst::xmm_rm_r( SseOpcode::Addss, @@ -2908,6 +2926,22 @@ fn test_x64_emit() { "F3410F5844927B", "addss 123(%r10,%rdx,4), %xmm0", )); + insns.push(( + Inst::xmm_rm_r(SseOpcode::Addsd, RegMem::reg(xmm15), w_xmm4), + "F2410F58E7", + "addsd %xmm15, %xmm4", + )); + + insns.push(( + Inst::xmm_rm_r(SseOpcode::Subss, RegMem::reg(xmm0), w_xmm1), + "F30F5CC8", + "subss %xmm0, %xmm1", + )); + insns.push(( + Inst::xmm_rm_r(SseOpcode::Subss, RegMem::reg(xmm12), w_xmm1), + "F3410F5CCC", + "subss %xmm12, %xmm1", + )); insns.push(( Inst::xmm_rm_r( SseOpcode::Subss, @@ -2917,86 +2951,157 @@ fn test_x64_emit() { "F3450F5C94C241010000", "subss 321(%r10,%rax,8), %xmm10", )); + insns.push(( + Inst::xmm_rm_r(SseOpcode::Subsd, RegMem::reg(xmm5), w_xmm14), + "F2440F5CF5", + "subsd %xmm5, %xmm14", + )); + insns.push(( Inst::xmm_rm_r(SseOpcode::Mulss, RegMem::reg(xmm5), w_xmm4), "F30F59E5", "mulss %xmm5, %xmm4", )); + insns.push(( + Inst::xmm_rm_r(SseOpcode::Mulsd, RegMem::reg(xmm5), w_xmm4), + "F20F59E5", + "mulsd %xmm5, %xmm4", + )); + insns.push(( Inst::xmm_rm_r(SseOpcode::Divss, RegMem::reg(xmm8), w_xmm7), "F3410F5EF8", "divss %xmm8, %xmm7", )); insns.push(( - Inst::xmm_rm_r(SseOpcode::Sqrtss, RegMem::reg(xmm7), w_xmm8), - "F3440F51C7", - "sqrtss %xmm7, %xmm8", + Inst::xmm_rm_r(SseOpcode::Divsd, RegMem::reg(xmm5), w_xmm4), + "F20F5EE5", + "divsd %xmm5, %xmm4", )); + insns.push(( Inst::xmm_rm_r(SseOpcode::Andps, RegMem::reg(xmm3), w_xmm12), "440F54E3", "andps %xmm3, %xmm12", )); + insns.push(( Inst::xmm_rm_r(SseOpcode::Andnps, RegMem::reg(xmm4), w_xmm11), "440F55DC", "andnps %xmm4, %xmm11", )); - insns.push(( - Inst::xmm_mov_rm_r(SseOpcode::Movaps, RegMem::reg(xmm5), w_xmm14, None), - "440F28F5", - "movaps %xmm5, %xmm14", - )); - insns.push(( - Inst::xmm_mov_rm_r(SseOpcode::Movd, RegMem::reg(rax), w_xmm15, None), - "66440F6EF8", - "movd %eax, %xmm15", - )); + insns.push(( Inst::xmm_rm_r(SseOpcode::Orps, RegMem::reg(xmm1), w_xmm15), "440F56F9", "orps %xmm1, %xmm15", )); - - insns.push(( - Inst::xmm_mov_r_m(SseOpcode::Movd, xmm0, Amode::imm_reg(321, rbx), None), - "660F7E8341010000", - "movd %xmm0, 321(%rbx)", - )); - - insns.push(( - Inst::xmm_mov_r_m(SseOpcode::Movss, xmm15, Amode::imm_reg(128, r12), None), - "F3450F11BC2480000000", - "movss %xmm15, 128(%r12)", - )); - - insns.push(( - Inst::xmm_mov_rm_r( - SseOpcode::Movd, - RegMem::mem(Amode::imm_reg(2, r10)), - w_xmm9, - None, - ), - "66450F6E4A02", - "movd 2(%r10), %xmm9", - )); - insns.push(( Inst::xmm_rm_r(SseOpcode::Orps, RegMem::reg(xmm5), w_xmm4), "0F56E5", "orps %xmm5, %xmm4", )); + + // XMM_Mov_R_M: float stores insns.push(( - Inst::xmm_mov_rm_r(SseOpcode::Movss, RegMem::reg(xmm13), w_xmm2, None), + Inst::xmm_mov_r_m(SseOpcode::Movss, xmm15, Amode::imm_reg(128, r12), None), + "F3450F11BC2480000000", + "movss %xmm15, 128(%r12)", + )); + insns.push(( + Inst::xmm_mov_r_m(SseOpcode::Movsd, xmm1, Amode::imm_reg(0, rsi), None), + "F20F110E", + "movsd %xmm1, 0(%rsi)", + )); + + // XmmUnary: moves and unary float ops + insns.push(( + Inst::xmm_unary_rm_r(SseOpcode::Movss, RegMem::reg(xmm13), w_xmm2), "F3410F10D5", "movss %xmm13, %xmm2", )); + insns.push(( - Inst::xmm_mov_rm_r(SseOpcode::Movsd, RegMem::reg(xmm14), w_xmm3, None), + Inst::xmm_unary_rm_r(SseOpcode::Movsd, RegMem::reg(xmm0), w_xmm1), + "F20F10C8", + "movsd %xmm0, %xmm1", + )); + insns.push(( + Inst::xmm_unary_rm_r( + SseOpcode::Movsd, + RegMem::mem(Amode::imm_reg(0, rsi)), + w_xmm2, + ), + "F20F1016", + "movsd 0(%rsi), %xmm2", + )); + insns.push(( + Inst::xmm_unary_rm_r(SseOpcode::Movsd, RegMem::reg(xmm14), w_xmm3), "F2410F10DE", "movsd %xmm14, %xmm3", )); + insns.push(( + Inst::xmm_unary_rm_r(SseOpcode::Movaps, RegMem::reg(xmm5), w_xmm14), + "440F28F5", + "movaps %xmm5, %xmm14", + )); + + insns.push(( + Inst::xmm_unary_rm_r(SseOpcode::Sqrtss, RegMem::reg(xmm7), w_xmm8), + "F3440F51C7", + "sqrtss %xmm7, %xmm8", + )); + insns.push(( + Inst::xmm_unary_rm_r(SseOpcode::Sqrtsd, RegMem::reg(xmm1), w_xmm2), + "F20F51D1", + "sqrtsd %xmm1, %xmm2", + )); + + insns.push(( + Inst::xmm_unary_rm_r(SseOpcode::Cvtss2sd, RegMem::reg(xmm0), w_xmm1), + "F30F5AC8", + "cvtss2sd %xmm0, %xmm1", + )); + insns.push(( + Inst::xmm_unary_rm_r(SseOpcode::Cvtsd2ss, RegMem::reg(xmm1), w_xmm0), + "F20F5AC1", + "cvtsd2ss %xmm1, %xmm0", + )); + + // Xmm to int conversions, and conversely. + + insns.push(( + Inst::xmm_to_gpr(SseOpcode::Movd, xmm0, w_rsi), + "660F7EC6", + "movd %xmm0, %esi", + )); + insns.push(( + Inst::xmm_to_gpr(SseOpcode::Movq, xmm2, w_rdi), + "66480F7ED7", + "movq %xmm2, %rdi", + )); + insns.push(( + Inst::gpr_to_xmm(SseOpcode::Movd, RegMem::reg(rax), w_xmm15), + "66440F6EF8", + "movd %eax, %xmm15", + )); + insns.push(( + Inst::gpr_to_xmm(SseOpcode::Movd, RegMem::mem(Amode::imm_reg(2, r10)), w_xmm9), + "66450F6E4A02", + "movd 2(%r10), %xmm9", + )); + insns.push(( + Inst::gpr_to_xmm(SseOpcode::Movd, RegMem::reg(rsi), w_xmm1), + "660F6ECE", + "movd %esi, %xmm1", + )); + insns.push(( + Inst::gpr_to_xmm(SseOpcode::Movq, RegMem::reg(rdi), w_xmm15), + "664C0F6EFF", + "movq %rdi, %xmm15", + )); + // ======================================================== // Misc instructions. diff --git a/third_party/rust/cranelift-codegen/src/isa/x64/inst/mod.rs b/third_party/rust/cranelift-codegen/src/isa/x64/inst/mod.rs index d0c9549892b3..7b1e0dedfc28 100644 --- a/third_party/rust/cranelift-codegen/src/isa/x64/inst/mod.rs +++ b/third_party/rust/cranelift-codegen/src/isa/x64/inst/mod.rs @@ -190,18 +190,20 @@ pub enum Inst { // ===================================== // Floating-point operations. - /// Float arithmetic/bit-twiddling: (add sub and or xor mul adc? sbb?) (32 64) (reg addr) reg + /// XMM (scalar or vector) binary op: (add sub and or xor mul adc? sbb?) (32 64) (reg addr) reg XMM_RM_R { op: SseOpcode, src: RegMem, dst: Writable, }, - /// mov between XMM registers (32 64) (reg addr) reg XMM_Mov_RM_R differs from XMM_RM_R in - /// that the dst register of XMM_MOV_RM_R is not used in the computation of the instruction - /// dst value and so does not have to be a previously valid value. This is characteristic of - /// mov instructions. - XMM_Mov_RM_R { + /// XMM (scalar or vector) unary op: mov between XMM registers (32 64) (reg addr) reg, sqrt, + /// etc. + /// + /// This differs from XMM_RM_R in that the dst register of XmmUnaryRmR is not used in the + /// computation of the instruction dst value and so does not have to be a previously valid + /// value. This is characteristic of mov instructions. + XmmUnaryRmR { op: SseOpcode, src: RegMem, dst: Writable, @@ -209,8 +211,8 @@ pub enum Inst { srcloc: Option, }, - /// mov reg addr (good for all memory stores from xmm registers) - XMM_Mov_R_M { + /// XMM (scalar or vector) unary op (from xmm to reg/mem): stores, movd, movq + Xmm_Mov_R_M { op: SseOpcode, src: Reg, dst: SyntheticAmode, @@ -218,6 +220,37 @@ pub enum Inst { srcloc: Option, }, + /// XMM (scalar) unary op (from xmm to integer reg): movd, movq + XmmToGpr { + op: SseOpcode, + src: Reg, + dst: Writable, + }, + + /// XMM (scalar) unary op (from integer to float reg): movd, movq + GprToXmm { + op: SseOpcode, + src: RegMem, + dst: Writable, + }, + + /// XMM (scalar) conditional move. + /// Overwrites the destination register if cc is set. + XmmCmove { + /// Whether the cmove is moving either 32 or 64 bits. + is_64: bool, + cc: CC, + src: RegMem, + dst: Writable, + }, + + /// Float comparisons/tests: cmp (b w l q) (reg addr imm) reg. + XMM_Cmp_RM_R { + op: SseOpcode, + src: RegMem, + dst: Reg, + }, + // ===================================== // Control flow instructions. /// Direct call: call simm32. @@ -318,6 +351,7 @@ impl Inst { src: RegMemImm, dst: Writable, ) -> Self { + src.assert_regclass_is(RegClass::I64); debug_assert!(dst.to_reg().get_class() == RegClass::I64); Self::Alu_RMI_R { is_64, @@ -333,12 +367,14 @@ impl Inst { src: RegMem, dst: Writable, ) -> Self { + src.assert_regclass_is(RegClass::I64); debug_assert!(dst.to_reg().get_class() == RegClass::I64); debug_assert!(size == 8 || size == 4 || size == 2); Self::UnaryRmR { size, op, src, dst } } pub(crate) fn div(size: u8, signed: bool, divisor: RegMem, loc: SourceLoc) -> Inst { + divisor.assert_regclass_is(RegClass::I64); debug_assert!(size == 8 || size == 4 || size == 2 || size == 1); Inst::Div { size, @@ -349,6 +385,7 @@ impl Inst { } pub(crate) fn mul_hi(size: u8, signed: bool, rhs: RegMem) -> Inst { + rhs.assert_regclass_is(RegClass::I64); debug_assert!(size == 8 || size == 4 || size == 2 || size == 1); Inst::MulHi { size, signed, rhs } } @@ -374,20 +411,30 @@ impl Inst { } } + pub(crate) fn imm32_r_unchecked(simm64: u64, dst: Writable) -> Inst { + debug_assert!(dst.to_reg().get_class() == RegClass::I64); + Inst::Imm_R { + dst_is_64: false, + simm64, + dst, + } + } + pub(crate) fn mov_r_r(is_64: bool, src: Reg, dst: Writable) -> Inst { debug_assert!(src.get_class() == RegClass::I64); debug_assert!(dst.to_reg().get_class() == RegClass::I64); Inst::Mov_R_R { is_64, src, dst } } - pub(crate) fn xmm_mov_rm_r( + pub(crate) fn xmm_mov( op: SseOpcode, src: RegMem, dst: Writable, srcloc: Option, ) -> Inst { + src.assert_regclass_is(RegClass::V128); debug_assert!(dst.to_reg().get_class() == RegClass::V128); - Inst::XMM_Mov_RM_R { + Inst::XmmUnaryRmR { op, src, dst, @@ -395,7 +442,20 @@ impl Inst { } } + /// Convenient helper for unary float operations. + pub(crate) fn xmm_unary_rm_r(op: SseOpcode, src: RegMem, dst: Writable) -> Inst { + src.assert_regclass_is(RegClass::V128); + debug_assert!(dst.to_reg().get_class() == RegClass::V128); + Inst::XmmUnaryRmR { + op, + src, + dst, + srcloc: None, + } + } + pub(crate) fn xmm_rm_r(op: SseOpcode, src: RegMem, dst: Writable) -> Self { + src.assert_regclass_is(RegClass::V128); debug_assert!(dst.to_reg().get_class() == RegClass::V128); Inst::XMM_RM_R { op, src, dst } } @@ -407,7 +467,7 @@ impl Inst { srcloc: Option, ) -> Inst { debug_assert!(src.get_class() == RegClass::V128); - Inst::XMM_Mov_R_M { + Inst::Xmm_Mov_R_M { op, src, dst: dst.into(), @@ -415,12 +475,31 @@ impl Inst { } } + pub(crate) fn xmm_to_gpr(op: SseOpcode, src: Reg, dst: Writable) -> Inst { + debug_assert!(src.get_class() == RegClass::V128); + debug_assert!(dst.to_reg().get_class() == RegClass::I64); + Inst::XmmToGpr { op, src, dst } + } + + pub(crate) fn gpr_to_xmm(op: SseOpcode, src: RegMem, dst: Writable) -> Inst { + src.assert_regclass_is(RegClass::I64); + debug_assert!(dst.to_reg().get_class() == RegClass::V128); + Inst::GprToXmm { op, src, dst } + } + + pub(crate) fn xmm_cmp_rm_r(op: SseOpcode, src: RegMem, dst: Reg) -> Inst { + //TODO:: Add assert_reg_type helper + debug_assert!(dst.get_class() == RegClass::V128); + Inst::XMM_Cmp_RM_R { op, src, dst } + } + pub(crate) fn movzx_rm_r( ext_mode: ExtMode, src: RegMem, dst: Writable, srcloc: Option, ) -> Inst { + src.assert_regclass_is(RegClass::I64); debug_assert!(dst.to_reg().get_class() == RegClass::I64); Inst::MovZX_RM_R { ext_mode, @@ -436,6 +515,7 @@ impl Inst { dst: Writable, srcloc: Option, ) -> Inst { + src.assert_regclass_is(RegClass::I64); debug_assert!(dst.to_reg().get_class() == RegClass::I64); Inst::MovSX_RM_R { ext_mode, @@ -460,6 +540,7 @@ impl Inst { /// A convenience function to be able to use a RegMem as the source of a move. pub(crate) fn mov64_rm_r(src: RegMem, dst: Writable, srcloc: Option) -> Inst { + src.assert_regclass_is(RegClass::I64); match src { RegMem::Reg { reg } => Self::mov_r_r(true, reg, dst), RegMem::Mem { addr } => Self::mov64_m_r(addr, dst, srcloc), @@ -517,6 +598,7 @@ impl Inst { src: RegMemImm, dst: Reg, ) -> Inst { + src.assert_regclass_is(RegClass::I64); debug_assert!(size == 8 || size == 4 || size == 2 || size == 1); debug_assert!(dst.get_class() == RegClass::I64); Inst::Cmp_RMI_R { size, src, dst } @@ -539,11 +621,24 @@ impl Inst { Inst::Cmove { size, cc, src, dst } } + pub(crate) fn xmm_cmove(is_64: bool, cc: CC, src: RegMem, dst: Writable) -> Inst { + src.assert_regclass_is(RegClass::V128); + debug_assert!(dst.to_reg().get_class() == RegClass::V128); + Inst::XmmCmove { + is_64, + cc, + src, + dst, + } + } + pub(crate) fn push64(src: RegMemImm) -> Inst { + src.assert_regclass_is(RegClass::I64); Inst::Push64 { src } } pub(crate) fn pop64(dst: Writable) -> Inst { + debug_assert!(dst.to_reg().get_class() == RegClass::I64); Inst::Pop64 { dst } } @@ -570,6 +665,7 @@ impl Inst { loc: SourceLoc, opcode: Opcode, ) -> Inst { + dest.assert_regclass_is(RegClass::I64); Inst::CallUnknown { dest, uses, @@ -600,6 +696,7 @@ impl Inst { } pub(crate) fn jmp_unknown(target: RegMem) -> Inst { + target.assert_regclass_is(RegClass::I64); Inst::JmpUnknown { target } } @@ -689,6 +786,7 @@ impl ShowWithRRU for Inst { }), divisor.show_rru_sized(mb_rru, *size) ), + Inst::MulHi { size, signed, rhs, .. } => format!( @@ -700,6 +798,7 @@ impl ShowWithRRU for Inst { }), rhs.show_rru_sized(mb_rru, *size) ), + Inst::CheckedDivOrRemSeq { kind, size, @@ -715,6 +814,7 @@ impl ShowWithRRU for Inst { }, show_ireg_sized(*divisor, mb_rru, *size), ), + Inst::SignExtendRaxRdx { size } => match size { 2 => "cwd", 4 => "cdq", @@ -722,24 +822,62 @@ impl ShowWithRRU for Inst { _ => unreachable!(), } .into(), - Inst::XMM_Mov_RM_R { op, src, dst, .. } => format!( + + Inst::XmmUnaryRmR { op, src, dst, .. } => format!( "{} {}, {}", ljustify(op.to_string()), src.show_rru_sized(mb_rru, op.src_size()), show_ireg_sized(dst.to_reg(), mb_rru, 8), ), - Inst::XMM_Mov_R_M { op, src, dst, .. } => format!( + + Inst::Xmm_Mov_R_M { op, src, dst, .. } => format!( "{} {}, {}", ljustify(op.to_string()), show_ireg_sized(*src, mb_rru, 8), - dst.show_rru(mb_rru) + dst.show_rru(mb_rru), ), + Inst::XMM_RM_R { op, src, dst } => format!( "{} {}, {}", ljustify(op.to_string()), src.show_rru_sized(mb_rru, 8), show_ireg_sized(dst.to_reg(), mb_rru, 8), ), + + Inst::XmmToGpr { op, src, dst } => { + let dst_size = match op { + SseOpcode::Movd => 4, + SseOpcode::Movq => 8, + _ => panic!("unexpected sse opcode"), + }; + format!( + "{} {}, {}", + ljustify(op.to_string()), + src.show_rru(mb_rru), + show_ireg_sized(dst.to_reg(), mb_rru, dst_size), + ) + } + + Inst::GprToXmm { op, src, dst } => { + let src_size = match op { + SseOpcode::Movd => 4, + SseOpcode::Movq => 8, + _ => panic!("unexpected sse opcode"), + }; + format!( + "{} {}, {}", + ljustify(op.to_string()), + src.show_rru_sized(mb_rru, src_size), + dst.show_rru(mb_rru) + ) + } + + Inst::XMM_Cmp_RM_R { op, src, dst } => format!( + "{} {}, {}", + ljustify(op.to_string()), + src.show_rru_sized(mb_rru, 8), + show_ireg_sized(*dst, mb_rru, 8), + ), Inst::Imm_R { dst_is_64, simm64, @@ -761,12 +899,14 @@ impl ShowWithRRU for Inst { ) } } + Inst::Mov_R_R { is_64, src, dst } => format!( "{} {}, {}", ljustify2("mov".to_string(), suffixLQ(*is_64)), show_ireg_sized(*src, mb_rru, sizeLQ(*is_64)), show_ireg_sized(dst.to_reg(), mb_rru, sizeLQ(*is_64)) ), + Inst::MovZX_RM_R { ext_mode, src, dst, .. } => { @@ -786,18 +926,21 @@ impl ShowWithRRU for Inst { ) } } + Inst::Mov64_M_R { src, dst, .. } => format!( "{} {}, {}", ljustify("movq".to_string()), src.show_rru(mb_rru), dst.show_rru(mb_rru) ), + Inst::LoadEffectiveAddress { addr, dst } => format!( "{} {}, {}", ljustify("lea".to_string()), addr.show_rru(mb_rru), dst.show_rru(mb_rru) ), + Inst::MovSX_RM_R { ext_mode, src, dst, .. } => format!( @@ -806,12 +949,14 @@ impl ShowWithRRU for Inst { src.show_rru_sized(mb_rru, ext_mode.src_size()), show_ireg_sized(dst.to_reg(), mb_rru, ext_mode.dst_size()) ), + Inst::Mov_R_M { size, src, dst, .. } => format!( "{} {}, {}", ljustify2("mov".to_string(), suffixBWLQ(*size)), show_ireg_sized(*src, mb_rru, *size), dst.show_rru(mb_rru) ), + Inst::Shift_R { is_64, kind, @@ -831,40 +976,67 @@ impl ShowWithRRU for Inst { show_ireg_sized(dst.to_reg(), mb_rru, sizeLQ(*is_64)) ), }, + Inst::Cmp_RMI_R { size, src, dst } => format!( "{} {}, {}", ljustify2("cmp".to_string(), suffixBWLQ(*size)), src.show_rru_sized(mb_rru, *size), show_ireg_sized(*dst, mb_rru, *size) ), + Inst::Setcc { cc, dst } => format!( "{} {}", ljustify2("set".to_string(), cc.to_string()), show_ireg_sized(dst.to_reg(), mb_rru, 1) ), + Inst::Cmove { size, cc, src, dst } => format!( "{} {}, {}", ljustify(format!("cmov{}{}", cc.to_string(), suffixBWLQ(*size))), src.show_rru_sized(mb_rru, *size), show_ireg_sized(dst.to_reg(), mb_rru, *size) ), + + Inst::XmmCmove { + is_64, + cc, + src, + dst, + } => { + let size = if *is_64 { 8 } else { 4 }; + format!( + "j{} $next; mov{} {}, {}; $next: ", + cc.invert().to_string(), + if *is_64 { "sd" } else { "ss" }, + src.show_rru_sized(mb_rru, size), + show_ireg_sized(dst.to_reg(), mb_rru, size) + ) + } + Inst::Push64 { src } => { format!("{} {}", ljustify("pushq".to_string()), src.show_rru(mb_rru)) } + Inst::Pop64 { dst } => { format!("{} {}", ljustify("popq".to_string()), dst.show_rru(mb_rru)) } + Inst::CallKnown { dest, .. } => format!("{} {:?}", ljustify("call".to_string()), dest), + Inst::CallUnknown { dest, .. } => format!( "{} *{}", ljustify("call".to_string()), dest.show_rru(mb_rru) ), + Inst::Ret => "ret".to_string(), + Inst::EpiloguePlaceholder => "epilogue placeholder".to_string(), + Inst::JmpKnown { dst } => { format!("{} {}", ljustify("jmp".to_string()), dst.show_rru(mb_rru)) } + Inst::JmpCond { cc, taken, @@ -875,18 +1047,21 @@ impl ShowWithRRU for Inst { taken.show_rru(mb_rru), not_taken.show_rru(mb_rru) ), + Inst::JmpTableSeq { idx, .. } => { format!("{} {}", ljustify("br_table".into()), idx.show_rru(mb_rru)) } - // + Inst::JmpUnknown { target } => format!( "{} *{}", ljustify("jmp".to_string()), target.show_rru(mb_rru) ), + Inst::TrapIf { cc, trap_code, .. } => { format!("j{} ; ud2 {} ;", cc.invert().to_string(), trap_code) } + Inst::LoadExtName { dst, name, offset, .. } => format!( @@ -896,8 +1071,11 @@ impl ShowWithRRU for Inst { offset, show_ireg_sized(dst.to_reg(), mb_rru, 8), ), + Inst::VirtualSPOffsetAdj { offset } => format!("virtual_sp_offset_adjust {}", offset), + Inst::Hlt => "hlt".into(), + Inst::Ud2 { trap_info } => format!("ud2 {}", trap_info.1), } } @@ -946,7 +1124,7 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { collector.add_use(regs::rax()); collector.add_mod(Writable::from_reg(regs::rdx())); } - Inst::UnaryRmR { src, dst, .. } | Inst::XMM_Mov_RM_R { src, dst, .. } => { + Inst::UnaryRmR { src, dst, .. } | Inst::XmmUnaryRmR { src, dst, .. } => { src.get_regs_as_uses(collector); collector.add_def(*dst); } @@ -954,17 +1132,25 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { src.get_regs_as_uses(collector); collector.add_mod(*dst); } - Inst::XMM_Mov_R_M { src, dst, .. } => { + Inst::Xmm_Mov_R_M { src, dst, .. } => { collector.add_use(*src); dst.get_regs_as_uses(collector); } + Inst::XMM_Cmp_RM_R { src, dst, .. } => { + src.get_regs_as_uses(collector); + collector.add_use(*dst); + } Inst::Imm_R { dst, .. } => { collector.add_def(*dst); } - Inst::Mov_R_R { src, dst, .. } => { + Inst::Mov_R_R { src, dst, .. } | Inst::XmmToGpr { src, dst, .. } => { collector.add_use(*src); collector.add_def(*dst); } + Inst::GprToXmm { src, dst, .. } => { + src.get_regs_as_uses(collector); + collector.add_def(*dst); + } Inst::MovZX_RM_R { src, dst, .. } => { src.get_regs_as_uses(collector); collector.add_def(*dst); @@ -994,7 +1180,7 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { Inst::Setcc { dst, .. } => { collector.add_def(*dst); } - Inst::Cmove { src, dst, .. } => { + Inst::Cmove { src, dst, .. } | Inst::XmmCmove { src, dst, .. } => { src.get_regs_as_uses(collector); collector.add_mod(*dst); } @@ -1140,7 +1326,7 @@ fn x64_map_regs(inst: &mut Inst, mapper: &RUM) { } } Inst::SignExtendRaxRdx { .. } => {} - Inst::XMM_Mov_RM_R { + Inst::XmmUnaryRmR { ref mut src, ref mut dst, .. @@ -1161,7 +1347,7 @@ fn x64_map_regs(inst: &mut Inst, mapper: &RUM) { src.map_uses(mapper); map_mod(mapper, dst); } - Inst::XMM_Mov_R_M { + Inst::Xmm_Mov_R_M { ref mut src, ref mut dst, .. @@ -1169,15 +1355,36 @@ fn x64_map_regs(inst: &mut Inst, mapper: &RUM) { map_use(mapper, src); dst.map_uses(mapper); } + Inst::XMM_Cmp_RM_R { + ref mut src, + ref mut dst, + .. + } => { + src.map_uses(mapper); + map_use(mapper, dst); + } Inst::Imm_R { ref mut dst, .. } => map_def(mapper, dst), Inst::Mov_R_R { ref mut src, ref mut dst, .. + } + | Inst::XmmToGpr { + ref mut src, + ref mut dst, + .. } => { map_use(mapper, src); map_def(mapper, dst); } + Inst::GprToXmm { + ref mut src, + ref mut dst, + .. + } => { + src.map_uses(mapper); + map_def(mapper, dst); + } Inst::MovZX_RM_R { ref mut src, ref mut dst, @@ -1222,6 +1429,11 @@ fn x64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut src, ref mut dst, .. + } + | Inst::XmmCmove { + ref mut src, + ref mut dst, + .. } => { src.map_uses(mapper); map_mod(mapper, dst) @@ -1309,7 +1521,7 @@ impl MachInst for Inst { Self::Mov_R_R { is_64, src, dst, .. } if *is_64 => Some((*dst, *src)), - Self::XMM_Mov_RM_R { op, src, dst, .. } + Self::XmmUnaryRmR { op, src, dst, .. } if *op == SseOpcode::Movss || *op == SseOpcode::Movsd || *op == SseOpcode::Movaps => @@ -1357,8 +1569,8 @@ impl MachInst for Inst { match rc_dst { RegClass::I64 => Inst::mov_r_r(true, src_reg, dst_reg), RegClass::V128 => match ty { - F32 => Inst::xmm_mov_rm_r(SseOpcode::Movss, RegMem::reg(src_reg), dst_reg, None), - F64 => Inst::xmm_mov_rm_r(SseOpcode::Movsd, RegMem::reg(src_reg), dst_reg, None), + F32 => Inst::xmm_mov(SseOpcode::Movss, RegMem::reg(src_reg), dst_reg, None), + F64 => Inst::xmm_mov(SseOpcode::Movsd, RegMem::reg(src_reg), dst_reg, None), _ => panic!("unexpected V128 type in gen_move"), }, _ => panic!("gen_move(x64): unhandled regclass"), @@ -1393,11 +1605,43 @@ impl MachInst for Inst { Inst::jmp_known(BranchTarget::Label(label)) } - fn gen_constant(to_reg: Writable, value: u64, ty: Type) -> SmallVec<[Self; 4]> { + fn gen_constant Writable>( + to_reg: Writable, + value: u64, + ty: Type, + mut alloc_tmp: F, + ) -> SmallVec<[Self; 4]> { let mut ret = SmallVec::new(); - debug_assert!(ty.is_int(), "float constants NYI"); - let is_64 = ty == I64 && value > 0x7fffffff; - ret.push(Inst::imm_r(is_64, value, to_reg)); + if ty.is_int() { + let is_64 = ty == I64 && value > 0x7fffffff; + ret.push(Inst::imm_r(is_64, value, to_reg)); + } else { + match ty { + F32 => { + let tmp = alloc_tmp(RegClass::I64, I32); + ret.push(Inst::imm32_r_unchecked(value, tmp)); + + ret.push(Inst::gpr_to_xmm( + SseOpcode::Movd, + RegMem::reg(tmp.to_reg()), + to_reg, + )); + } + + F64 => { + let tmp = alloc_tmp(RegClass::I64, I64); + ret.push(Inst::imm_r(true, value, tmp)); + + ret.push(Inst::gpr_to_xmm( + SseOpcode::Movq, + RegMem::reg(tmp.to_reg()), + to_reg, + )); + } + + _ => panic!("unexpected type {:?} in gen_constant", ty), + } + } ret } diff --git a/third_party/rust/cranelift-codegen/src/isa/x64/lower.rs b/third_party/rust/cranelift-codegen/src/isa/x64/lower.rs index 4e4074e6f573..b7272c9a7837 100644 --- a/third_party/rust/cranelift-codegen/src/isa/x64/lower.rs +++ b/third_party/rust/cranelift-codegen/src/isa/x64/lower.rs @@ -6,14 +6,14 @@ use log::trace; use regalloc::{Reg, RegClass, Writable}; use smallvec::SmallVec; -use alloc::boxed::Box; -use alloc::vec::Vec; -use std::convert::TryFrom; - use crate::ir::types; use crate::ir::types::*; use crate::ir::Inst as IRInst; -use crate::ir::{condcodes::IntCC, InstructionData, Opcode, TrapCode, Type}; +use crate::ir::{condcodes::FloatCC, condcodes::IntCC, InstructionData, Opcode, TrapCode, Type}; +use alloc::boxed::Box; +use alloc::vec::Vec; +use cranelift_codegen_shared::condcodes::CondCode; +use std::convert::TryFrom; use crate::machinst::lower::*; use crate::machinst::*; @@ -95,6 +95,16 @@ fn inst_condcode(data: &InstructionData) -> IntCC { } } +fn inst_fp_condcode(data: &InstructionData) -> Option { + match data { + &InstructionData::BranchFloat { cond, .. } + | &InstructionData::FloatCompare { cond, .. } + | &InstructionData::FloatCond { cond, .. } + | &InstructionData::FloatCondTrap { cond, .. } => Some(cond), + _ => None, + } +} + fn ldst_offset(data: &InstructionData) -> Option { match data { &InstructionData::Load { offset, .. } @@ -734,6 +744,77 @@ fn lower_insn_to_regs>( ctx.emit(Inst::setcc(cc, dst)); } + Opcode::Fcmp => { + let condcode = inst_fp_condcode(ctx.data(insn)).unwrap(); + let input_ty = ctx.input_ty(insn, 0); + let op = match input_ty { + F32 => SseOpcode::Ucomiss, + F64 => SseOpcode::Ucomisd, + _ => panic!("Bad input type to Fcmp"), + }; + + // Unordered is returned by setting ZF, PF, CF <- 111 + // Greater than by ZF, PF, CF <- 000 + // Less than by ZF, PF, CF <- 001 + // Equal by ZF, PF, CF <- 100 + // + // Checking the result of comiss is somewhat annoying because you don't + // have setcc instructions that explicitly check simultaneously for the condition + // (i.e. eq, le, gt, etc) and orderedness. So that might mean we need more + // than one setcc check and then a logical "and" or "or" to determine both. + // However knowing that if the parity bit is set, then the result was + // considered unordered and knowing that if the parity bit is set, then both + // the ZF and CF flag bits must also be set we can getaway with using one setcc + // for most condition codes. + match condcode { + // setb and setbe for ordered LessThan and LessThanOrEqual check if CF = 1 which + // doesn't exclude unorderdness. To get around this we can reverse the operands + // and the cc test to instead check if CF and ZF are 0 which would also excludes + // unorderedness. Using similiar logic we also reverse UnorderedOrGreaterThan and + // UnorderedOrGreaterThanOrEqual and assure that ZF or CF is 1 to exclude orderedness. + FloatCC::LessThan + | FloatCC::LessThanOrEqual + | FloatCC::UnorderedOrGreaterThan + | FloatCC::UnorderedOrGreaterThanOrEqual => { + let lhs = input_to_reg_mem(ctx, inputs[0]); + let rhs = input_to_reg(ctx, inputs[1]); + let dst = output_to_reg(ctx, outputs[0]); + ctx.emit(Inst::xmm_cmp_rm_r(op, lhs, rhs)); + let condcode = condcode.reverse(); + let cc = CC::from_floatcc(condcode); + ctx.emit(Inst::setcc(cc, dst)); + } + // Outlier case where we cannot get around checking the parity bit to determine + // if the result was ordered. + FloatCC::Equal => { + let lhs = input_to_reg(ctx, inputs[0]); + let rhs = input_to_reg_mem(ctx, inputs[1]); + let dst = output_to_reg(ctx, outputs[0]); + let tmp_gpr1 = ctx.alloc_tmp(RegClass::I64, I32); + ctx.emit(Inst::xmm_cmp_rm_r(op, rhs, lhs)); + ctx.emit(Inst::setcc(CC::NP, tmp_gpr1)); + ctx.emit(Inst::setcc(CC::Z, dst)); + ctx.emit(Inst::alu_rmi_r( + false, + AluRmiROpcode::And, + RegMemImm::reg(tmp_gpr1.to_reg()), + dst, + )); + } + // For all remaining condition codes we can handle things with one check. Condition + // ordered NotEqual for example does not need a separate check for the parity bit because + // the setnz checks that the zero flag is 0 which is impossible with an unordered result. + _ => { + let lhs = input_to_reg(ctx, inputs[0]); + let rhs = input_to_reg_mem(ctx, inputs[1]); + let dst = output_to_reg(ctx, outputs[0]); + let cc = CC::from_floatcc(condcode); + ctx.emit(Inst::xmm_cmp_rm_r(op, rhs, lhs)); + ctx.emit(Inst::setcc(cc, dst)); + } + } + } + Opcode::FallthroughReturn | Opcode::Return => { for i in 0..ctx.num_inputs(insn) { let src_reg = input_to_reg(ctx, inputs[i]); @@ -792,29 +873,111 @@ fn lower_insn_to_regs>( ctx.emit(Inst::Ud2 { trap_info }) } + Opcode::F64const => { + // TODO use xorpd for 0 + let value = ctx.get_constant(insn).unwrap(); + let dst = output_to_reg(ctx, outputs[0]); + for inst in Inst::gen_constant(dst, value, F64, |reg_class, ty| { + ctx.alloc_tmp(reg_class, ty) + }) { + ctx.emit(inst); + } + } + + Opcode::F32const => { + // TODO use xorps for 0. + let value = ctx.get_constant(insn).unwrap(); + let dst = output_to_reg(ctx, outputs[0]); + for inst in Inst::gen_constant(dst, value, F32, |reg_class, ty| { + ctx.alloc_tmp(reg_class, ty) + }) { + ctx.emit(inst); + } + } + Opcode::Fadd | Opcode::Fsub | Opcode::Fmul | Opcode::Fdiv => { - let lhs = input_to_reg(ctx, inputs[0]); + let lhs = input_to_reg_mem(ctx, inputs[0]); let rhs = input_to_reg(ctx, inputs[1]); let dst = output_to_reg(ctx, outputs[0]); + + // Note: min and max can't be handled here, because of the way Cranelift defines them: + // if any operand is a NaN, they must return the NaN operand, while the x86 machine + // instruction will return the other operand. + let (f32_op, f64_op) = match op { + Opcode::Fadd => (SseOpcode::Addss, SseOpcode::Addsd), + Opcode::Fsub => (SseOpcode::Subss, SseOpcode::Subsd), + Opcode::Fmul => (SseOpcode::Mulss, SseOpcode::Mulsd), + Opcode::Fdiv => (SseOpcode::Divss, SseOpcode::Divsd), + _ => unreachable!(), + }; + let is_64 = flt_ty_is_64(ty.unwrap()); - if !is_64 { - let sse_op = match op { - Opcode::Fadd => SseOpcode::Addss, - Opcode::Fsub => SseOpcode::Subss, - Opcode::Fmul => SseOpcode::Mulss, - Opcode::Fdiv => SseOpcode::Divss, - // TODO Fmax, Fmin. - _ => unimplemented!(), - }; - ctx.emit(Inst::xmm_mov_rm_r( - SseOpcode::Movss, - RegMem::reg(lhs), - dst, - None, - )); - ctx.emit(Inst::xmm_rm_r(sse_op, RegMem::reg(rhs), dst)); + + let mov_op = if is_64 { + SseOpcode::Movsd } else { - unimplemented!("unimplemented lowering for opcode {:?}", op); + SseOpcode::Movss + }; + ctx.emit(Inst::xmm_mov(mov_op, lhs, dst, None)); + + let sse_op = if is_64 { f64_op } else { f32_op }; + ctx.emit(Inst::xmm_rm_r(sse_op, RegMem::reg(rhs), dst)); + } + + Opcode::Sqrt => { + let src = input_to_reg_mem(ctx, inputs[0]); + let dst = output_to_reg(ctx, outputs[0]); + + let (f32_op, f64_op) = match op { + Opcode::Sqrt => (SseOpcode::Sqrtss, SseOpcode::Sqrtsd), + _ => unreachable!(), + }; + + let sse_op = if flt_ty_is_64(ty.unwrap()) { + f64_op + } else { + f32_op + }; + ctx.emit(Inst::xmm_unary_rm_r(sse_op, src, dst)); + } + + Opcode::Fpromote => { + let src = input_to_reg_mem(ctx, inputs[0]); + let dst = output_to_reg(ctx, outputs[0]); + ctx.emit(Inst::xmm_unary_rm_r(SseOpcode::Cvtss2sd, src, dst)); + } + + Opcode::Fdemote => { + let src = input_to_reg_mem(ctx, inputs[0]); + let dst = output_to_reg(ctx, outputs[0]); + ctx.emit(Inst::xmm_unary_rm_r(SseOpcode::Cvtsd2ss, src, dst)); + } + + Opcode::Bitcast => { + let input_ty = ctx.input_ty(insn, 0); + let output_ty = ctx.output_ty(insn, 0); + match (input_ty, output_ty) { + (F32, I32) => { + let src = input_to_reg(ctx, inputs[0]); + let dst = output_to_reg(ctx, outputs[0]); + ctx.emit(Inst::xmm_to_gpr(SseOpcode::Movd, src, dst)); + } + (I32, F32) => { + let src = input_to_reg_mem(ctx, inputs[0]); + let dst = output_to_reg(ctx, outputs[0]); + ctx.emit(Inst::gpr_to_xmm(SseOpcode::Movd, src, dst)); + } + (F64, I64) => { + let src = input_to_reg(ctx, inputs[0]); + let dst = output_to_reg(ctx, outputs[0]); + ctx.emit(Inst::xmm_to_gpr(SseOpcode::Movq, src, dst)); + } + (I64, F64) => { + let src = input_to_reg_mem(ctx, inputs[0]); + let dst = output_to_reg(ctx, outputs[0]); + ctx.emit(Inst::gpr_to_xmm(SseOpcode::Movq, src, dst)); + } + _ => unreachable!("invalid bitcast from {:?} to {:?}", input_ty, output_ty), } } @@ -834,20 +997,19 @@ fn lower_insn_to_regs>( let tmp_xmm1 = ctx.alloc_tmp(RegClass::V128, F32); let tmp_xmm2 = ctx.alloc_tmp(RegClass::V128, F32); ctx.emit(Inst::imm_r(true, 0x8000_0000, tmp_gpr1)); - ctx.emit(Inst::xmm_mov_rm_r( + ctx.emit(Inst::gpr_to_xmm( SseOpcode::Movd, RegMem::reg(tmp_gpr1.to_reg()), tmp_xmm1, - None, )); - ctx.emit(Inst::xmm_mov_rm_r( + ctx.emit(Inst::xmm_mov( SseOpcode::Movaps, RegMem::reg(tmp_xmm1.to_reg()), dst, None, )); ctx.emit(Inst::xmm_rm_r(SseOpcode::Andnps, RegMem::reg(lhs), dst)); - ctx.emit(Inst::xmm_mov_rm_r( + ctx.emit(Inst::xmm_mov( SseOpcode::Movss, RegMem::reg(rhs), tmp_xmm2, @@ -982,8 +1144,9 @@ fn lower_insn_to_regs>( } (_, true) => { ctx.emit(match elem_ty { - F32 => Inst::xmm_mov_rm_r(SseOpcode::Movss, RegMem::mem(addr), dst, srcloc), - _ => unimplemented!("FP load not 32-bit"), + F32 => Inst::xmm_mov(SseOpcode::Movss, RegMem::mem(addr), dst, srcloc), + F64 => Inst::xmm_mov(SseOpcode::Movsd, RegMem::mem(addr), dst, srcloc), + _ => unreachable!("unexpected type for load: {:?}", elem_ty), }); } } @@ -1025,7 +1188,7 @@ fn lower_insn_to_regs>( | Opcode::Istore32Complex => { assert!( inputs.len() == 3, - "can't handle more than two inputs in complex load" + "can't handle more than two inputs in complex store" ); let base = input_to_reg(ctx, inputs[1]); let index = input_to_reg(ctx, inputs[2]); @@ -1043,7 +1206,8 @@ fn lower_insn_to_regs>( if is_float { ctx.emit(match elem_ty { F32 => Inst::xmm_mov_r_m(SseOpcode::Movss, src, addr, srcloc), - _ => unimplemented!("FP store not 32-bit"), + F64 => Inst::xmm_mov_r_m(SseOpcode::Movsd, src, addr, srcloc), + _ => panic!("unexpected type for store {:?}", elem_ty), }); } else { ctx.emit(Inst::mov_r_m(elem_ty.bytes() as u8, src, addr, srcloc)); @@ -1119,18 +1283,23 @@ fn lower_insn_to_regs>( let dst = output_to_reg(ctx, outputs[0]); let ty = ctx.output_ty(insn, 0); - assert!(is_int_ty(ty), "float cmov NYI"); - let size = ty.bytes() as u8; - if size == 1 { - // Sign-extend operands to 32, then do a cmove of size 4. - let lhs_se = ctx.alloc_tmp(RegClass::I64, I32); - ctx.emit(Inst::movsx_rm_r(ExtMode::BL, lhs, lhs_se, None)); - ctx.emit(Inst::movsx_rm_r(ExtMode::BL, RegMem::reg(rhs), dst, None)); - ctx.emit(Inst::cmove(4, cc, RegMem::reg(lhs_se.to_reg()), dst)); + if ty.is_int() { + let size = ty.bytes() as u8; + if size == 1 { + // Sign-extend operands to 32, then do a cmove of size 4. + let lhs_se = ctx.alloc_tmp(RegClass::I64, I32); + ctx.emit(Inst::movsx_rm_r(ExtMode::BL, lhs, lhs_se, None)); + ctx.emit(Inst::movsx_rm_r(ExtMode::BL, RegMem::reg(rhs), dst, None)); + ctx.emit(Inst::cmove(4, cc, RegMem::reg(lhs_se.to_reg()), dst)); + } else { + ctx.emit(Inst::gen_move(dst, rhs, ty)); + ctx.emit(Inst::cmove(size, cc, lhs, dst)); + } } else { + debug_assert!(ty == F32 || ty == F64); ctx.emit(Inst::gen_move(dst, rhs, ty)); - ctx.emit(Inst::cmove(size, cc, lhs, dst)); + ctx.emit(Inst::xmm_cmove(ty == F64, cc, lhs, dst)); } } diff --git a/third_party/rust/cranelift-codegen/src/isa/x86/abi.rs b/third_party/rust/cranelift-codegen/src/isa/x86/abi.rs index 6cd1175cd637..67d95832ff7c 100644 --- a/third_party/rust/cranelift-codegen/src/isa/x86/abi.rs +++ b/third_party/rust/cranelift-codegen/src/isa/x86/abi.rs @@ -108,6 +108,19 @@ impl Args { impl ArgAssigner for Args { fn assign(&mut self, arg: &AbiParam) -> ArgAction { + if let ArgumentPurpose::StructArgument(size) = arg.purpose { + if self.call_conv != CallConv::SystemV { + panic!( + "The sarg argument purpose is not yet implemented for non-systemv call conv {:?}", + self.call_conv, + ); + } + let loc = ArgumentLoc::Stack(self.offset as i32); + self.offset += size; + debug_assert!(self.offset <= i32::MAX as u32); + return ArgAction::AssignAndChangeType(loc, types::SARG_T); + } + let ty = arg.value_type; if ty.bits() > u16::from(self.pointer_bits) { diff --git a/third_party/rust/cranelift-codegen/src/legalizer/boundary.rs b/third_party/rust/cranelift-codegen/src/legalizer/boundary.rs index aba52a8c2b7f..43d4f9911380 100644 --- a/third_party/rust/cranelift-codegen/src/legalizer/boundary.rs +++ b/third_party/rust/cranelift-codegen/src/legalizer/boundary.rs @@ -22,8 +22,9 @@ use crate::cursor::{Cursor, FuncCursor}; use crate::flowgraph::ControlFlowGraph; use crate::ir::instructions::CallInfo; use crate::ir::{ - AbiParam, ArgumentLoc, ArgumentPurpose, Block, DataFlowGraph, Function, Inst, InstBuilder, - MemFlags, SigRef, Signature, StackSlotData, StackSlotKind, Type, Value, ValueLoc, + AbiParam, ArgumentLoc, ArgumentPurpose, Block, DataFlowGraph, ExtFuncData, ExternalName, + Function, Inst, InstBuilder, LibCall, MemFlags, SigRef, Signature, StackSlotData, + StackSlotKind, Type, Value, ValueLoc, }; use crate::isa::TargetIsa; use crate::legalizer::split::{isplit, vsplit}; @@ -113,12 +114,31 @@ fn legalize_entry_params(func: &mut Function, entry: Block) { let abi_type = pos.func.signature.params[abi_arg]; let arg_type = pos.func.dfg.value_type(arg); + if let ArgumentPurpose::StructArgument(size) = abi_type.purpose { + let offset = if let ArgumentLoc::Stack(offset) = abi_type.location { + offset + } else { + unreachable!("StructArgument must already have a Stack ArgumentLoc assigned"); + }; + let ss = pos.func.stack_slots.make_incoming_arg(size, offset); + let struct_arg = pos.ins().stack_addr(arg_type, ss, 0); + pos.func.dfg.change_to_alias(arg, struct_arg); + let dummy = pos + .func + .dfg + .append_block_param(entry, crate::ir::types::SARG_T); + pos.func.locations[dummy] = ValueLoc::Stack(ss); + abi_arg += 1; + continue; + } + if arg_type == abi_type.value_type { // No value translation is necessary, this argument matches the ABI type. // Just use the original block argument value. This is the most common case. pos.func.dfg.attach_block_param(entry, arg); match abi_type.purpose { ArgumentPurpose::Normal => {} + ArgumentPurpose::StructArgument(_) => unreachable!("Handled above"), ArgumentPurpose::FramePointer => {} ArgumentPurpose::CalleeSaved => {} ArgumentPurpose::StructReturn => { @@ -137,7 +157,7 @@ fn legalize_entry_params(func: &mut Function, entry: Block) { debug_assert!(!has_stack_limit, "Multiple stack_limit arguments found"); has_stack_limit = true; } - _ => panic!("Unexpected special-purpose arg {}", abi_type), + ArgumentPurpose::Link => panic!("Unexpected link arg {}", abi_type), } abi_arg += 1; } else { @@ -168,7 +188,7 @@ fn legalize_entry_params(func: &mut Function, entry: Block) { for &arg in &pos.func.signature.params[abi_arg..] { match arg.purpose { // Any normal parameters should have been processed above. - ArgumentPurpose::Normal => { + ArgumentPurpose::Normal | ArgumentPurpose::StructArgument(_) => { panic!("Leftover arg: {}", arg); } // The callee-save parameters should not appear until after register allocation is @@ -587,9 +607,14 @@ fn convert_to_abi( /// Check if a sequence of arguments match a desired sequence of argument types. fn check_arg_types(dfg: &DataFlowGraph, args: &[Value], types: &[AbiParam]) -> bool { - let arg_types = args.iter().map(|&v| dfg.value_type(v)); - let sig_types = types.iter().map(|&at| at.value_type); - arg_types.eq(sig_types) + args.len() == types.len() + && args.iter().zip(types.iter()).all(|(v, at)| { + if let ArgumentPurpose::StructArgument(_) = at.purpose { + true + } else { + dfg.value_type(*v) == at.value_type + } + }) } /// Check if the arguments of the call `inst` match the signature. @@ -699,7 +724,12 @@ fn legalize_inst_arguments( .unwrap(); let mut put_arg = |func: &mut Function, arg| { let abi_type = get_abi_type(func, abi_arg); - if func.dfg.value_type(arg) == abi_type.value_type { + let struct_argument = if let ArgumentPurpose::StructArgument(_) = abi_type.purpose { + true + } else { + false + }; + if func.dfg.value_type(arg) == abi_type.value_type || struct_argument { // This is the argument type we need. vlist.as_mut_slice(&mut func.dfg.value_lists)[num_fixed_values + abi_arg] = arg; abi_arg += 1; @@ -762,7 +792,7 @@ pub fn handle_call_abi( // Start by checking if the argument types already match the signature. let sig_ref = match check_call_signature(&pos.func.dfg, inst) { - Ok(_) => return spill_call_arguments(pos), + Ok(_) => return spill_call_arguments(pos, isa), Err(s) => s, }; @@ -800,7 +830,7 @@ pub fn handle_call_abi( // Go back and insert spills for any stack arguments. pos.goto_inst(inst); - spill_call_arguments(pos); + spill_call_arguments(pos, isa); // Yes, we changed stuff. true @@ -990,8 +1020,12 @@ fn spill_entry_params(func: &mut Function, entry: Block) { .iter() .zip(func.dfg.block_params(entry)) { - if let ArgumentLoc::Stack(offset) = abi.location { - let ss = func.stack_slots.make_incoming_arg(abi.value_type, offset); + if let ArgumentPurpose::StructArgument(_) = abi.purpose { + // Location has already been assigned during legalization. + } else if let ArgumentLoc::Stack(offset) = abi.location { + let ss = func + .stack_slots + .make_incoming_arg(abi.value_type.bytes(), offset); func.locations[arg] = ValueLoc::Stack(ss); } } @@ -1005,7 +1039,7 @@ fn spill_entry_params(func: &mut Function, entry: Block) { /// TODO: The outgoing stack slots can be written a bit earlier, as long as there are no branches /// or calls between writing the stack slots and the call instruction. Writing the slots earlier /// could help reduce register pressure before the call. -fn spill_call_arguments(pos: &mut FuncCursor) -> bool { +fn spill_call_arguments(pos: &mut FuncCursor, isa: &dyn TargetIsa) -> bool { let inst = pos .current_inst() .expect("Cursor must point to a call instruction"); @@ -1032,9 +1066,17 @@ fn spill_call_arguments(pos: &mut FuncCursor) -> bool { // Assign `arg` to a new stack slot, unless it's already in the correct // slot. The legalization needs to be idempotent, so we should see a // correct outgoing slot on the second pass. - let ss = stack_slots.get_outgoing_arg(abi.value_type, offset); + let (ss, size) = match abi.purpose { + ArgumentPurpose::StructArgument(size) => { + (stack_slots.get_outgoing_arg(size, offset), Some(size)) + } + _ => ( + stack_slots.get_outgoing_arg(abi.value_type.bytes(), offset), + None, + ), + }; if locations[arg] != ValueLoc::Stack(ss) { - Some((idx, arg, ss)) + Some((idx, arg, ss, size)) } else { None } @@ -1049,9 +1091,48 @@ fn spill_call_arguments(pos: &mut FuncCursor) -> bool { return false; } + let mut libc_memcpy = None; + let mut import_memcpy = |func: &mut Function, pointer_type| { + if let Some(libc_memcpy) = libc_memcpy { + return libc_memcpy; + } + + let signature = { + let mut s = Signature::new(isa.default_call_conv()); + s.params.push(AbiParam::new(pointer_type)); + s.params.push(AbiParam::new(pointer_type)); + // The last argument of `memcpy` is a `size_t`. This is the same size as a pointer on + // all architectures we are interested in. + s.params.push(AbiParam::new(pointer_type)); + legalize_libcall_signature(&mut s, isa); + func.import_signature(s) + }; + + let func = func.import_function(ExtFuncData { + name: ExternalName::LibCall(LibCall::Memcpy), + signature, + colocated: false, + }); + libc_memcpy = Some(func); + func + }; + // Insert the spill instructions and rewrite call arguments. - for (idx, arg, ss) in arglist { - let stack_val = pos.ins().spill(arg); + for (idx, arg, ss, size) in arglist { + let stack_val = if let Some(size) = size { + // Struct argument + let pointer_type = pos.func.dfg.value_type(arg); + let src = arg; + let dest = pos.ins().stack_addr(pointer_type, ss, 0); + let size = pos.ins().iconst(pointer_type, i64::from(size)); + + let libc_memcpy = import_memcpy(pos.func, pointer_type); + pos.ins().call(libc_memcpy, &[dest, src, size]); + pos.ins().dummy_sarg_t() + } else { + // Non struct argument + pos.ins().spill(arg) + }; pos.func.locations[stack_val] = ValueLoc::Stack(ss); pos.func.dfg.inst_variable_args_mut(inst)[idx] = stack_val; } diff --git a/third_party/rust/cranelift-codegen/src/machinst/lower.rs b/third_party/rust/cranelift-codegen/src/machinst/lower.rs index 608e8b4896f6..6761372a5031 100644 --- a/third_party/rust/cranelift-codegen/src/machinst/lower.rs +++ b/third_party/rust/cranelift-codegen/src/machinst/lower.rs @@ -516,7 +516,11 @@ impl<'func, I: VCodeInst> Lower<'func, I> { // Now, finally, deal with the moves whose sources are constants. for (ty, dst_reg, const_u64) in &const_bundles { - for inst in I::gen_constant(*dst_reg, *const_u64, *ty).into_iter() { + for inst in I::gen_constant(*dst_reg, *const_u64, *ty, |reg_class, ty| { + self.alloc_tmp(reg_class, ty) + }) + .into_iter() + { self.emit(inst); } } diff --git a/third_party/rust/cranelift-codegen/src/machinst/mod.rs b/third_party/rust/cranelift-codegen/src/machinst/mod.rs index 9a0ae1e820e7..0383ff12f653 100644 --- a/third_party/rust/cranelift-codegen/src/machinst/mod.rs +++ b/third_party/rust/cranelift-codegen/src/machinst/mod.rs @@ -154,7 +154,12 @@ pub trait MachInst: Clone + Debug { fn gen_move(to_reg: Writable, from_reg: Reg, ty: Type) -> Self; /// Generate a constant into a reg. - fn gen_constant(to_reg: Writable, value: u64, ty: Type) -> SmallVec<[Self; 4]>; + fn gen_constant Writable>( + to_reg: Writable, + value: u64, + ty: Type, + alloc_tmp: F, + ) -> SmallVec<[Self; 4]>; /// Generate a zero-length no-op. fn gen_zero_len_nop() -> Self; diff --git a/third_party/rust/cranelift-codegen/src/peepmatic.rs b/third_party/rust/cranelift-codegen/src/peepmatic.rs index 1e2201940769..2fbd58d49cd5 100644 --- a/third_party/rust/cranelift-codegen/src/peepmatic.rs +++ b/third_party/rust/cranelift-codegen/src/peepmatic.rs @@ -13,27 +13,287 @@ use cranelift_codegen_shared::condcodes::IntCC; use peepmatic_runtime::{ cc::ConditionCode, instruction_set::InstructionSet, - operator::Operator, part::{Constant, Part}, paths::Path, r#type::{BitWidth, Kind, Type}, PeepholeOptimizations, PeepholeOptimizer, }; +use peepmatic_traits::TypingRules; +use std::borrow::Cow; use std::boxed::Box; use std::convert::{TryFrom, TryInto}; use std::ptr; use std::sync::atomic::{AtomicPtr, Ordering}; +peepmatic_traits::define_parse_and_typing_rules_for_operator! { + Opcode { + adjust_sp_down => AdjustSpDown { + parameters(iNN); + result(void); + } + adjust_sp_down_imm => AdjustSpDownImm { + immediates(iNN); + result(void); + } + band => Band { + parameters(iNN, iNN); + result(iNN); + } + band_imm => BandImm { + immediates(iNN); + parameters(iNN); + result(iNN); + } + bconst => Bconst { + immediates(b1); + result(bNN); + } + bint => Bint { + parameters(bNN); + result(iNN); + } + bor => Bor { + parameters(iNN, iNN); + result(iNN); + } + bor_imm => BorImm { + immediates(iNN); + parameters(iNN); + result(iNN); + } + brnz => Brnz { + parameters(bool_or_int); + result(void); + } + brz => Brz { + parameters(bool_or_int); + result(void); + } + bxor => Bxor { + parameters(iNN, iNN); + result(iNN); + } + bxor_imm => BxorImm { + immediates(iNN); + parameters(iNN); + result(iNN); + } + iadd => Iadd { + parameters(iNN, iNN); + result(iNN); + } + iadd_imm => IaddImm { + immediates(iNN); + parameters(iNN); + result(iNN); + } + icmp => Icmp { + immediates(cc); + parameters(iNN, iNN); + result(b1); + } + icmp_imm => IcmpImm { + immediates(cc, iNN); + parameters(iNN); + result(b1); + } + iconst => Iconst { + immediates(iNN); + result(iNN); + } + ifcmp => Ifcmp { + parameters(iNN, iNN); + result(cpu_flags); + } + ifcmp_imm => IfcmpImm { + immediates(iNN); + parameters(iNN); + result(cpu_flags); + } + imul => Imul { + parameters(iNN, iNN); + result(iNN); + } + imul_imm => ImulImm { + immediates(iNN); + parameters(iNN); + result(iNN); + } + ireduce => Ireduce { + parameters(iNN); + result(iMM); + is_reduce(true); + } + irsub_imm => IrsubImm { + immediates(iNN); + parameters(iNN); + result(iNN); + } + ishl => Ishl { + parameters(iNN, iNN); + result(iNN); + } + ishl_imm => IshlImm { + immediates(iNN); + parameters(iNN); + result(iNN); + } + isub => Isub { + parameters(iNN, iNN); + result(iNN); + } + rotl => Rotl { + parameters(iNN, iNN); + result(iNN); + } + rotl_imm => RotlImm { + immediates(iNN); + parameters(iNN); + result(iNN); + } + rotr => Rotr { + parameters(iNN, iNN); + result(iNN); + } + rotr_imm => RotrImm { + immediates(iNN); + parameters(iNN); + result(iNN); + } + sdiv => Sdiv { + parameters(iNN, iNN); + result(iNN); + } + sdiv_imm => SdivImm { + immediates(iNN); + parameters(iNN); + result(iNN); + } + select => Select { + parameters(bool_or_int, any_t, any_t); + result(any_t); + } + sextend => Sextend { + parameters(iNN); + result(iMM); + is_extend(true); + } + srem => Srem { + parameters(iNN, iNN); + result(iNN); + } + srem_imm => SremImm { + immediates(iNN); + parameters(iNN); + result(iNN); + } + sshr => Sshr { + parameters(iNN, iNN); + result(iNN); + } + sshr_imm => SshrImm { + immediates(iNN); + parameters(iNN); + result(iNN); + } + trapnz => Trapnz { + parameters(bool_or_int); + result(void); + } + trapz => Trapz { + parameters(bool_or_int); + result(void); + } + udiv => Udiv { + parameters(iNN, iNN); + result(iNN); + } + udiv_imm => UdivImm { + immediates(iNN); + parameters(iNN); + result(iNN); + } + uextend => Uextend { + parameters(iNN); + result(iMM); + is_extend(true); + } + urem => Urem { + parameters(iNN, iNN); + result(iNN); + } + urem_imm => UremImm { + immediates(iNN); + parameters(iNN); + result(iNN); + } + ushr => Ushr { + parameters(iNN, iNN); + result(iNN); + } + ushr_imm => UshrImm { + immediates(iNN); + parameters(iNN); + result(iNN); + } + } + parse_cfg(feature = "rebuild-peephole-optimizers"); +} + +/// Code required to rebuild Peepmatic-based peephole optimizers. +/// +/// This module is used to scope imports and dependencies that are only required +/// for building peephole optimizers (as opposed to just using pre-built +/// peephole optimizers). This helps ensure that our regular builds using +/// pre-built peephole optimizers stay lean. +#[cfg(feature = "rebuild-peephole-optimizers")] +mod rebuild { + use super::*; + use alloc::vec::Vec; + use std::fs; + use std::path::Path; + + /// Rebuild the `preopt.peepmatic` peephole optimizer. + /// + /// Saves and overwrites the old `preopt.serialized` build and returns a + /// copy of the result. + pub fn rebuild_preopt() -> Vec { + let codegen_path = Path::new(include_str!(concat!( + env!("OUT_DIR"), + "/CRANELIFT_CODEGEN_PATH" + ))); + let source_path = codegen_path.join("src").join("preopt.peepmatic"); + + let preopt = peepmatic::compile_file::(&source_path) + .expect("failed to compile `src/preopt.peepmatic`"); + + let serialized_path = codegen_path.join("src").join("preopt.serialized"); + preopt + .serialize_to_file(&serialized_path) + .expect("failed to serialize peephole optimizer to `src/preopt.serialized`"); + fs::read(&serialized_path).expect("failed to read `src/preopt.serialized`") + } +} + /// Get the `preopt.peepmatic` peephole optimizer. pub(crate) fn preopt<'a, 'b>( isa: &'b dyn TargetIsa, ) -> PeepholeOptimizer<'static, 'a, &'b dyn TargetIsa> { - static SERIALIZED: &[u8] = include_bytes!("preopt.serialized"); + #[cfg(feature = "rebuild-peephole-optimizers")] + fn get_serialized() -> Cow<'static, [u8]> { + rebuild::rebuild_preopt().into() + } + + #[cfg(not(feature = "rebuild-peephole-optimizers"))] + fn get_serialized() -> Cow<'static, [u8]> { + static SERIALIZED: &[u8] = include_bytes!("preopt.serialized"); + SERIALIZED.into() + } // Once initialized, this must never be re-assigned. The initialized value // is semantically "static data" and is intentionally leaked for the whole // program's lifetime. - static DESERIALIZED: AtomicPtr = AtomicPtr::new(ptr::null_mut()); + static DESERIALIZED: AtomicPtr> = AtomicPtr::new(ptr::null_mut()); // If `DESERIALIZED` has already been initialized, then just use it. let ptr = DESERIALIZED.load(Ordering::SeqCst); @@ -46,7 +306,7 @@ pub(crate) fn preopt<'a, 'b>( // another thread could be doing the same thing concurrently, so there is a // race to see who initializes `DESERIALIZED` first, and we need to be // prepared to both win or lose that race. - let peep_opts = PeepholeOptimizations::deserialize(SERIALIZED) + let peep_opts = PeepholeOptimizations::deserialize(&get_serialized()) .expect("should always be able to deserialize `preopt.serialized`"); let peep_opts = Box::into_raw(Box::new(peep_opts)); @@ -219,69 +479,6 @@ fn part_to_value(pos: &mut FuncCursor, root: Inst, part: Part) -> O } } -impl Opcode { - fn to_peepmatic_operator(&self) -> Option { - macro_rules! convert { - ( $( $op:ident $(,)* )* ) => { - match self { - $( Self::$op => Some(Operator::$op), )* - _ => None, - } - } - } - - convert!( - AdjustSpDown, - AdjustSpDownImm, - Band, - BandImm, - Bconst, - Bint, - Bor, - BorImm, - Brnz, - Brz, - Bxor, - BxorImm, - Iadd, - IaddImm, - Icmp, - IcmpImm, - Iconst, - Ifcmp, - IfcmpImm, - Imul, - ImulImm, - Ireduce, - IrsubImm, - Ishl, - IshlImm, - Isub, - Rotl, - RotlImm, - Rotr, - RotrImm, - Sdiv, - SdivImm, - Select, - Sextend, - Srem, - SremImm, - Sshr, - SshrImm, - Trapnz, - Trapz, - Udiv, - UdivImm, - Uextend, - Urem, - UremImm, - Ushr, - UshrImm, - ) - } -} - impl TryFrom for Imm64 { type Error = &'static str; @@ -428,6 +625,8 @@ fn peepmatic_ty_to_ir_ty(ty: Type, dfg: &DataFlowGraph, root: Inst) -> types::Ty unsafe impl<'a, 'b> InstructionSet<'b> for &'a dyn TargetIsa { type Context = FuncCursor<'b>; + type Operator = Opcode; + type Instruction = ValueOrInst; fn replace_instruction( @@ -495,7 +694,7 @@ unsafe impl<'a, 'b> InstructionSet<'b> for &'a dyn TargetIsa { let mut part = Part::Instruction(root); for p in path.0[1..].iter().copied() { let inst = part.as_instruction()?.resolve_inst(&pos.func.dfg)?; - let operator = pos.func.dfg[inst].opcode().to_peepmatic_operator()?; + let operator = pos.func.dfg[inst].opcode(); if p < operator.immediates_arity() { part = get_immediate(&pos.func.dfg, inst, p as usize); @@ -512,16 +711,16 @@ unsafe impl<'a, 'b> InstructionSet<'b> for &'a dyn TargetIsa { Some(part) } - fn operator(&self, pos: &mut FuncCursor<'b>, value_or_inst: ValueOrInst) -> Option { + fn operator(&self, pos: &mut FuncCursor<'b>, value_or_inst: ValueOrInst) -> Option { let inst = value_or_inst.resolve_inst(&pos.func.dfg)?; - pos.func.dfg[inst].opcode().to_peepmatic_operator() + Some(pos.func.dfg[inst].opcode()) } fn make_inst_1( &self, pos: &mut FuncCursor<'b>, root: ValueOrInst, - operator: Operator, + operator: Opcode, r#type: Type, a: Part, ) -> ValueOrInst { @@ -529,27 +728,32 @@ unsafe impl<'a, 'b> InstructionSet<'b> for &'a dyn TargetIsa { let root = root.resolve_inst(&pos.func.dfg).unwrap(); match operator { - Operator::AdjustSpDown => { + Opcode::AdjustSpDown => { let a = part_to_value(pos, root, a).unwrap(); pos.ins().adjust_sp_down(a).into() } - Operator::AdjustSpDownImm => { + Opcode::AdjustSpDownImm => { let c = a.unwrap_constant(); let imm = Imm64::try_from(c).unwrap(); pos.ins().adjust_sp_down_imm(imm).into() } - Operator::Bconst => { + Opcode::Bconst => { let c = a.unwrap_constant(); let val = const_to_value(pos.ins(), c, root); pos.func.dfg.value_def(val).unwrap_inst().into() } - Operator::Bint => { + Opcode::Bint => { let a = part_to_value(pos, root, a).unwrap(); let ty = peepmatic_ty_to_ir_ty(r#type, &pos.func.dfg, root); let val = pos.ins().bint(ty, a); pos.func.dfg.value_def(val).unwrap_inst().into() } - Operator::Brnz => { + Opcode::Bnot => { + let a = part_to_value(pos, root, a).unwrap(); + let val = pos.ins().bnot(a); + pos.func.dfg.value_def(val).unwrap_inst().into() + } + Opcode::Brnz => { let a = part_to_value(pos, root, a).unwrap(); // NB: branching instructions must be the root of an @@ -561,37 +765,37 @@ unsafe impl<'a, 'b> InstructionSet<'b> for &'a dyn TargetIsa { pos.ins().brnz(a, block, &args).into() } - Operator::Brz => { + Opcode::Brz => { let a = part_to_value(pos, root, a).unwrap(); - // See the comment in the `Operator::Brnz` match argm. + // See the comment in the `Opcode::Brnz` match argm. let block = pos.func.dfg[root].branch_destination().unwrap(); let args = pos.func.dfg.inst_args(root)[1..].to_vec(); pos.ins().brz(a, block, &args).into() } - Operator::Iconst => { + Opcode::Iconst => { let a = a.unwrap_constant(); let val = const_to_value(pos.ins(), a, root); pos.func.dfg.value_def(val).unwrap_inst().into() } - Operator::Ireduce => { + Opcode::Ireduce => { let a = part_to_value(pos, root, a).unwrap(); let ty = peepmatic_ty_to_ir_ty(r#type, &pos.func.dfg, root); let val = pos.ins().ireduce(ty, a); pos.func.dfg.value_def(val).unwrap_inst().into() } - Operator::Sextend => { + Opcode::Sextend => { let a = part_to_value(pos, root, a).unwrap(); let ty = peepmatic_ty_to_ir_ty(r#type, &pos.func.dfg, root); let val = pos.ins().sextend(ty, a); pos.func.dfg.value_def(val).unwrap_inst().into() } - Operator::Trapnz => { + Opcode::Trapnz => { let a = part_to_value(pos, root, a).unwrap(); // NB: similar to branching instructions (see comment in the - // `Operator::Brnz` match arm) trapping instructions must be the + // `Opcode::Brnz` match arm) trapping instructions must be the // root of an optimization's right-hand side, and we get the // trap code from the root of the left-hand side. Peepmatic // doesn't currently represent trap codes. @@ -599,13 +803,13 @@ unsafe impl<'a, 'b> InstructionSet<'b> for &'a dyn TargetIsa { pos.ins().trapnz(a, code).into() } - Operator::Trapz => { + Opcode::Trapz => { let a = part_to_value(pos, root, a).unwrap(); - // See comment in the `Operator::Trapnz` match arm. + // See comment in the `Opcode::Trapnz` match arm. let code = pos.func.dfg[root].trap_code().unwrap(); pos.ins().trapz(a, code).into() } - Operator::Uextend => { + Opcode::Uextend => { let a = part_to_value(pos, root, a).unwrap(); let ty = peepmatic_ty_to_ir_ty(r#type, &pos.func.dfg, root); let val = pos.ins().uextend(ty, a); @@ -619,7 +823,7 @@ unsafe impl<'a, 'b> InstructionSet<'b> for &'a dyn TargetIsa { &self, pos: &mut FuncCursor<'b>, root: ValueOrInst, - operator: Operator, + operator: Opcode, _: Type, a: Part, b: Part, @@ -628,193 +832,193 @@ unsafe impl<'a, 'b> InstructionSet<'b> for &'a dyn TargetIsa { let root = root.resolve_inst(&pos.func.dfg).unwrap(); match operator { - Operator::Band => { + Opcode::Band => { let a = part_to_value(pos, root, a).unwrap(); let b = part_to_value(pos, root, b).unwrap(); let val = pos.ins().band(a, b); pos.func.dfg.value_def(val).unwrap_inst().into() } - Operator::BandImm => { + Opcode::BandImm => { let a = part_to_imm64(pos, a); let b = part_to_value(pos, root, b).unwrap(); let val = pos.ins().band_imm(b, a); pos.func.dfg.value_def(val).unwrap_inst().into() } - Operator::Bor => { + Opcode::Bor => { let a = part_to_value(pos, root, a).unwrap(); let b = part_to_value(pos, root, b).unwrap(); let val = pos.ins().bor(a, b); pos.func.dfg.value_def(val).unwrap_inst().into() } - Operator::BorImm => { + Opcode::BorImm => { let a = part_to_imm64(pos, a); let b = part_to_value(pos, root, b).unwrap(); let val = pos.ins().bor_imm(b, a); pos.func.dfg.value_def(val).unwrap_inst().into() } - Operator::Bxor => { + Opcode::Bxor => { let a = part_to_value(pos, root, a).unwrap(); let b = part_to_value(pos, root, b).unwrap(); let val = pos.ins().bxor(a, b); pos.func.dfg.value_def(val).unwrap_inst().into() } - Operator::BxorImm => { + Opcode::BxorImm => { let a = part_to_imm64(pos, a); let b = part_to_value(pos, root, b).unwrap(); let val = pos.ins().bxor_imm(b, a); pos.func.dfg.value_def(val).unwrap_inst().into() } - Operator::Iadd => { + Opcode::Iadd => { let a = part_to_value(pos, root, a).unwrap(); let b = part_to_value(pos, root, b).unwrap(); let val = pos.ins().iadd(a, b); pos.func.dfg.value_def(val).unwrap_inst().into() } - Operator::IaddImm => { + Opcode::IaddImm => { let a = part_to_imm64(pos, a); let b = part_to_value(pos, root, b).unwrap(); let val = pos.ins().iadd_imm(b, a); pos.func.dfg.value_def(val).unwrap_inst().into() } - Operator::Ifcmp => { + Opcode::Ifcmp => { let a = part_to_value(pos, root, a).unwrap(); let b = part_to_value(pos, root, b).unwrap(); let val = pos.ins().ifcmp(a, b); pos.func.dfg.value_def(val).unwrap_inst().into() } - Operator::IfcmpImm => { + Opcode::IfcmpImm => { let a = part_to_imm64(pos, a); let b = part_to_value(pos, root, b).unwrap(); let val = pos.ins().ifcmp_imm(b, a); pos.func.dfg.value_def(val).unwrap_inst().into() } - Operator::Imul => { + Opcode::Imul => { let a = part_to_value(pos, root, a).unwrap(); let b = part_to_value(pos, root, b).unwrap(); let val = pos.ins().imul(a, b); pos.func.dfg.value_def(val).unwrap_inst().into() } - Operator::ImulImm => { + Opcode::ImulImm => { let a = part_to_imm64(pos, a); let b = part_to_value(pos, root, b).unwrap(); let val = pos.ins().imul_imm(b, a); pos.func.dfg.value_def(val).unwrap_inst().into() } - Operator::IrsubImm => { + Opcode::IrsubImm => { let a = part_to_imm64(pos, a); let b = part_to_value(pos, root, b).unwrap(); let val = pos.ins().irsub_imm(b, a); pos.func.dfg.value_def(val).unwrap_inst().into() } - Operator::Ishl => { + Opcode::Ishl => { let a = part_to_value(pos, root, a).unwrap(); let b = part_to_value(pos, root, b).unwrap(); let val = pos.ins().ishl(a, b); pos.func.dfg.value_def(val).unwrap_inst().into() } - Operator::IshlImm => { + Opcode::IshlImm => { let a = part_to_imm64(pos, a); let b = part_to_value(pos, root, b).unwrap(); let val = pos.ins().ishl_imm(b, a); pos.func.dfg.value_def(val).unwrap_inst().into() } - Operator::Isub => { + Opcode::Isub => { let a = part_to_value(pos, root, a).unwrap(); let b = part_to_value(pos, root, b).unwrap(); let val = pos.ins().isub(a, b); pos.func.dfg.value_def(val).unwrap_inst().into() } - Operator::Rotl => { + Opcode::Rotl => { let a = part_to_value(pos, root, a).unwrap(); let b = part_to_value(pos, root, b).unwrap(); let val = pos.ins().rotl(a, b); pos.func.dfg.value_def(val).unwrap_inst().into() } - Operator::RotlImm => { + Opcode::RotlImm => { let a = part_to_imm64(pos, a); let b = part_to_value(pos, root, b).unwrap(); let val = pos.ins().rotl_imm(b, a); pos.func.dfg.value_def(val).unwrap_inst().into() } - Operator::Rotr => { + Opcode::Rotr => { let a = part_to_value(pos, root, a).unwrap(); let b = part_to_value(pos, root, b).unwrap(); let val = pos.ins().rotr(a, b); pos.func.dfg.value_def(val).unwrap_inst().into() } - Operator::RotrImm => { + Opcode::RotrImm => { let a = part_to_imm64(pos, a); let b = part_to_value(pos, root, b).unwrap(); let val = pos.ins().rotr_imm(b, a); pos.func.dfg.value_def(val).unwrap_inst().into() } - Operator::Sdiv => { + Opcode::Sdiv => { let a = part_to_value(pos, root, a).unwrap(); let b = part_to_value(pos, root, b).unwrap(); let val = pos.ins().sdiv(a, b); pos.func.dfg.value_def(val).unwrap_inst().into() } - Operator::SdivImm => { + Opcode::SdivImm => { let a = part_to_imm64(pos, a); let b = part_to_value(pos, root, b).unwrap(); let val = pos.ins().sdiv_imm(b, a); pos.func.dfg.value_def(val).unwrap_inst().into() } - Operator::Srem => { + Opcode::Srem => { let a = part_to_value(pos, root, a).unwrap(); let b = part_to_value(pos, root, b).unwrap(); let val = pos.ins().srem(a, b); pos.func.dfg.value_def(val).unwrap_inst().into() } - Operator::SremImm => { + Opcode::SremImm => { let a = part_to_imm64(pos, a); let b = part_to_value(pos, root, b).unwrap(); let val = pos.ins().srem_imm(b, a); pos.func.dfg.value_def(val).unwrap_inst().into() } - Operator::Sshr => { + Opcode::Sshr => { let a = part_to_value(pos, root, a).unwrap(); let b = part_to_value(pos, root, b).unwrap(); let val = pos.ins().sshr(a, b); pos.func.dfg.value_def(val).unwrap_inst().into() } - Operator::SshrImm => { + Opcode::SshrImm => { let a = part_to_imm64(pos, a); let b = part_to_value(pos, root, b).unwrap(); let val = pos.ins().sshr_imm(b, a); pos.func.dfg.value_def(val).unwrap_inst().into() } - Operator::Udiv => { + Opcode::Udiv => { let a = part_to_value(pos, root, a).unwrap(); let b = part_to_value(pos, root, b).unwrap(); let val = pos.ins().udiv(a, b); pos.func.dfg.value_def(val).unwrap_inst().into() } - Operator::UdivImm => { + Opcode::UdivImm => { let a = part_to_imm64(pos, a); let b = part_to_value(pos, root, b).unwrap(); let val = pos.ins().udiv_imm(b, a); pos.func.dfg.value_def(val).unwrap_inst().into() } - Operator::Urem => { + Opcode::Urem => { let a = part_to_value(pos, root, a).unwrap(); let b = part_to_value(pos, root, b).unwrap(); let val = pos.ins().urem(a, b); pos.func.dfg.value_def(val).unwrap_inst().into() } - Operator::UremImm => { + Opcode::UremImm => { let a = part_to_imm64(pos, a); let b = part_to_value(pos, root, b).unwrap(); let val = pos.ins().urem_imm(b, a); pos.func.dfg.value_def(val).unwrap_inst().into() } - Operator::Ushr => { + Opcode::Ushr => { let a = part_to_value(pos, root, a).unwrap(); let b = part_to_value(pos, root, b).unwrap(); let val = pos.ins().ushr(a, b); pos.func.dfg.value_def(val).unwrap_inst().into() } - Operator::UshrImm => { + Opcode::UshrImm => { let a = part_to_imm64(pos, a); let b = part_to_value(pos, root, b).unwrap(); let val = pos.ins().ushr_imm(b, a); @@ -828,7 +1032,7 @@ unsafe impl<'a, 'b> InstructionSet<'b> for &'a dyn TargetIsa { &self, pos: &mut FuncCursor<'b>, root: ValueOrInst, - operator: Operator, + operator: Opcode, _: Type, a: Part, b: Part, @@ -838,7 +1042,7 @@ unsafe impl<'a, 'b> InstructionSet<'b> for &'a dyn TargetIsa { let root = root.resolve_inst(&pos.func.dfg).unwrap(); match operator { - Operator::Icmp => { + Opcode::Icmp => { let cond = a.unwrap_condition_code(); let cond = peepmatic_to_intcc(cond); let b = part_to_value(pos, root, b).unwrap(); @@ -846,7 +1050,7 @@ unsafe impl<'a, 'b> InstructionSet<'b> for &'a dyn TargetIsa { let val = pos.ins().icmp(cond, b, c); pos.func.dfg.value_def(val).unwrap_inst().into() } - Operator::IcmpImm => { + Opcode::IcmpImm => { let cond = a.unwrap_condition_code(); let cond = peepmatic_to_intcc(cond); let imm = part_to_imm64(pos, b); @@ -854,7 +1058,7 @@ unsafe impl<'a, 'b> InstructionSet<'b> for &'a dyn TargetIsa { let val = pos.ins().icmp_imm(cond, c, imm); pos.func.dfg.value_def(val).unwrap_inst().into() } - Operator::Select => { + Opcode::Select => { let a = part_to_value(pos, root, a).unwrap(); let b = part_to_value(pos, root, b).unwrap(); let c = part_to_value(pos, root, c).unwrap(); @@ -885,3 +1089,21 @@ unsafe impl<'a, 'b> InstructionSet<'b> for &'a dyn TargetIsa { self.pointer_bits() } } + +#[cfg(test)] +#[cfg(feature = "x86")] +mod tests { + use super::*; + use crate::isa::lookup; + use crate::settings::{builder, Flags}; + use std::str::FromStr; + use target_lexicon::triple; + + #[test] + fn get_peepmatic_preopt() { + let isa = lookup(triple!("x86_64")) + .expect("expect x86 ISA") + .finish(Flags::new(builder())); + let _ = preopt(&*isa); + } +} diff --git a/third_party/rust/cranelift-codegen/src/preopt.serialized b/third_party/rust/cranelift-codegen/src/preopt.serialized index 1bb11b24ad37..8765e4bb8422 100644 Binary files a/third_party/rust/cranelift-codegen/src/preopt.serialized and b/third_party/rust/cranelift-codegen/src/preopt.serialized differ diff --git a/third_party/rust/cranelift-codegen/src/remove_constant_phis.rs b/third_party/rust/cranelift-codegen/src/remove_constant_phis.rs index 26a9dc8de30f..234d89c26ee6 100644 --- a/third_party/rust/cranelift-codegen/src/remove_constant_phis.rs +++ b/third_party/rust/cranelift-codegen/src/remove_constant_phis.rs @@ -1,7 +1,5 @@ //! A Constant-Phi-Node removal pass. -use log::info; - use crate::dominator_tree::DominatorTree; use crate::entity::EntityList; use crate::fx::FxHashMap; @@ -384,7 +382,7 @@ pub fn do_remove_constant_phis(func: &mut Function, domtree: &mut DominatorTree) } } - info!( + log::debug!( "do_remove_constant_phis: done, {} iters. {} formals, of which {} const.", iter_no, state.absvals.len(), diff --git a/third_party/rust/cranelift-codegen/src/stack_layout.rs b/third_party/rust/cranelift-codegen/src/stack_layout.rs index 55f3eb864f60..2430e8a64375 100644 --- a/third_party/rust/cranelift-codegen/src/stack_layout.rs +++ b/third_party/rust/cranelift-codegen/src/stack_layout.rs @@ -152,8 +152,8 @@ mod tests { assert_eq!(layout_stack(sss, is_leaf, 16), Ok(0)); // Same for incoming arguments with non-negative offsets. - let in0 = sss.make_incoming_arg(types::I64, 0); - let in1 = sss.make_incoming_arg(types::I64, 8); + let in0 = sss.make_incoming_arg(8, 0); + let in1 = sss.make_incoming_arg(8, 8); assert_eq!(layout_stack(sss, is_leaf, 1), Ok(0)); assert_eq!(layout_stack(sss, is_leaf, 16), Ok(0)); @@ -178,7 +178,7 @@ mod tests { // An incoming argument with negative offset counts towards the total frame size, but it // should still pack nicely with the spill slots. - let in2 = sss.make_incoming_arg(types::I32, -4); + let in2 = sss.make_incoming_arg(4, -4); assert_eq!(layout_stack(sss, is_leaf, 1), Ok(16)); assert_eq!(sss[in0].offset, Some(0)); @@ -195,7 +195,7 @@ mod tests { assert_eq!(sss[ss1].offset, Some(-8)); // Finally, make sure there is room for the outgoing args. - let out0 = sss.get_outgoing_arg(types::I32, 0); + let out0 = sss.get_outgoing_arg(4, 0); assert_eq!(layout_stack(sss, is_leaf, 1), Ok(20)); assert_eq!(sss[in0].offset, Some(0)); @@ -214,7 +214,7 @@ mod tests { assert_eq!(sss[out0].offset, Some(0)); // Also test that an unsupported offset is rejected. - sss.get_outgoing_arg(types::I8, StackOffset::max_value() - 1); + sss.get_outgoing_arg(1, StackOffset::max_value() - 1); assert_eq!( layout_stack(sss, is_leaf, 1), Err(CodegenError::ImplLimitExceeded) diff --git a/third_party/rust/cranelift-codegen/src/verifier/mod.rs b/third_party/rust/cranelift-codegen/src/verifier/mod.rs index 00fc31e212a6..aff4bcae2669 100644 --- a/third_party/rust/cranelift-codegen/src/verifier/mod.rs +++ b/third_party/rust/cranelift-codegen/src/verifier/mod.rs @@ -65,9 +65,9 @@ use crate::ir; use crate::ir::entities::AnyEntity; use crate::ir::instructions::{BranchInfo, CallInfo, InstructionFormat, ResolvedConstraint}; use crate::ir::{ - types, ArgumentLoc, Block, Constant, FuncRef, Function, GlobalValue, Inst, InstructionData, - JumpTable, Opcode, SigRef, StackSlot, StackSlotKind, Type, Value, ValueDef, ValueList, - ValueLoc, + types, ArgumentLoc, ArgumentPurpose, Block, Constant, FuncRef, Function, GlobalValue, Inst, + InstructionData, JumpTable, Opcode, SigRef, StackSlot, StackSlotKind, Type, Value, ValueDef, + ValueList, ValueLoc, }; use crate::isa::TargetIsa; use crate::iterators::IteratorExtras; @@ -1473,7 +1473,8 @@ impl<'a> Verifier<'a> { ), )); } - if slot.size != abi.value_type.bytes() { + if abi.purpose == ArgumentPurpose::StructArgument(slot.size) { + } else if slot.size != abi.value_type.bytes() { return errors.fatal(( inst, self.context(inst), @@ -1986,6 +1987,20 @@ impl<'a> Verifier<'a> { )) }); + self.func + .signature + .returns + .iter() + .enumerate() + .for_each(|(i, ret)| { + if let ArgumentPurpose::StructArgument(_) = ret.purpose { + errors.report(( + AnyEntity::Function, + format!("Return value at position {} can't be an struct argument", i), + )) + } + }); + if errors.has_error() { Err(()) } else {