diff --git a/compiler/rustc_codegen_cranelift/src/global_asm.rs b/compiler/rustc_codegen_cranelift/src/global_asm.rs index 9763b0c0fa867..c5b164b8e9cce 100644 --- a/compiler/rustc_codegen_cranelift/src/global_asm.rs +++ b/compiler/rustc_codegen_cranelift/src/global_asm.rs @@ -30,6 +30,7 @@ impl<'tcx> AsmCodegenMethods<'tcx> for GlobalAsmContext<'_, 'tcx> { operands: &[GlobalAsmOperandRef<'tcx>], options: InlineAsmOptions, _line_spans: &[Span], + _extra_rust_target_features: &[String], ) { codegen_global_asm_inner(self.tcx, self.global_asm, template, operands, options); } diff --git a/compiler/rustc_codegen_gcc/src/asm.rs b/compiler/rustc_codegen_gcc/src/asm.rs index ac86fbe7428b0..733dc52465dea 100644 --- a/compiler/rustc_codegen_gcc/src/asm.rs +++ b/compiler/rustc_codegen_gcc/src/asm.rs @@ -928,6 +928,7 @@ impl<'gcc, 'tcx> AsmCodegenMethods<'tcx> for CodegenCx<'gcc, 'tcx> { operands: &[GlobalAsmOperandRef<'tcx>], options: InlineAsmOptions, line_spans: &[Span], + _extra_rust_target_features: &[String], ) { let asm_arch = self.tcx.sess.asm_arch.unwrap(); diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs index 549769547da78..6f9ddc1fe2c88 100644 --- a/compiler/rustc_codegen_llvm/src/asm.rs +++ b/compiler/rustc_codegen_llvm/src/asm.rs @@ -414,6 +414,7 @@ impl<'tcx> AsmCodegenMethods<'tcx> for CodegenCx<'_, 'tcx> { operands: &[GlobalAsmOperandRef<'tcx>], options: InlineAsmOptions, _line_spans: &[Span], + extra_rust_target_features: &[String], ) { let asm_arch = self.tcx.sess.asm_arch.unwrap(); @@ -499,14 +500,26 @@ impl<'tcx> AsmCodegenMethods<'tcx> for CodegenCx<'_, 'tcx> { template_str.push_str("\n.att_syntax\n"); } - let target_features = self.tcx.global_backend_features(()).join(","); - let target_cpu = llvm_util::target_cpu(self.tcx.sess); + // Globally-enabled features that are already in the backend format. + let global_features = self.tcx.global_backend_features(()).iter().map(String::as_str); + + // Features enabled on a particular instance, in the rust format. + // These need to be translated to the LLVM format. + let function_features: Vec<_> = extra_rust_target_features + .iter() + .flat_map(|feat| llvm_util::to_llvm_features(self.tcx.sess, feat)) + .flat_map(|feat| feat.into_iter().map(|f| format!("+{f}"))) + .collect(); + + let function_features = function_features.iter().map(String::as_str); + let target_features = + global_features.chain(function_features).intersperse(",").collect::(); llvm::append_module_inline_asm( self.llmod, template_str.as_bytes(), &target_features, - target_cpu, + llvm_util::target_cpu(self.tcx.sess), ); } diff --git a/compiler/rustc_codegen_ssa/src/base.rs b/compiler/rustc_codegen_ssa/src/base.rs index 9eb4fd510fd7f..c870d1694d068 100644 --- a/compiler/rustc_codegen_ssa/src/base.rs +++ b/compiler/rustc_codegen_ssa/src/base.rs @@ -490,7 +490,7 @@ where }) .collect(); - cx.codegen_global_asm(asm.template, &operands, asm.options, asm.line_spans); + cx.codegen_global_asm(asm.template, &operands, asm.options, asm.line_spans, &[]); } else { span_bug!(item.span, "Mismatch between hir::Item type and MonoItem type") } diff --git a/compiler/rustc_codegen_ssa/src/mir/naked_asm.rs b/compiler/rustc_codegen_ssa/src/mir/naked_asm.rs index 05b87bb6d7159..939e5395e4741 100644 --- a/compiler/rustc_codegen_ssa/src/mir/naked_asm.rs +++ b/compiler/rustc_codegen_ssa/src/mir/naked_asm.rs @@ -54,7 +54,9 @@ pub fn codegen_naked_asm< template_vec.extend(template.iter().cloned()); template_vec.push(rustc_ast::ast::InlineAsmTemplatePiece::String(end.into())); - cx.codegen_global_asm(&template_vec, &operands, options, line_spans); + let target_features: Vec<_> = + cx.tcx().asm_target_features(instance.def_id()).iter().map(|s| s.to_string()).collect(); + cx.codegen_global_asm(&template_vec, &operands, options, line_spans, &target_features); } fn inline_to_global_operand<'a, 'tcx, Cx: LayoutOf<'tcx, LayoutOfResult = TyAndLayout<'tcx>>>( diff --git a/compiler/rustc_codegen_ssa/src/traits/asm.rs b/compiler/rustc_codegen_ssa/src/traits/asm.rs index 85a2fe09ba414..1deed8c4dc016 100644 --- a/compiler/rustc_codegen_ssa/src/traits/asm.rs +++ b/compiler/rustc_codegen_ssa/src/traits/asm.rs @@ -66,12 +66,17 @@ pub trait AsmBuilderMethods<'tcx>: BackendTypes { } pub trait AsmCodegenMethods<'tcx> { + /// Codegen a module-level assembly block. + /// + /// NOTE: the target features must be the rust target feature names, not backend target + /// feature names. This argument is used to forward target features on naked functions. fn codegen_global_asm( &mut self, template: &[InlineAsmTemplatePiece], operands: &[GlobalAsmOperandRef<'tcx>], options: InlineAsmOptions, line_spans: &[Span], + extra_rust_target_features: &[String], ); /// The mangled name of this instance diff --git a/tests/assembly-llvm/naked-functions/target-feature.rs b/tests/assembly-llvm/naked-functions/target-feature.rs new file mode 100644 index 0000000000000..500e2a778e475 --- /dev/null +++ b/tests/assembly-llvm/naked-functions/target-feature.rs @@ -0,0 +1,165 @@ +//@ revisions: aarch64-elf aarch64-macho aarch64-coff x86_64 s390x riscv64 powerpc64 loongarch64 +//@ add-minicore +//@ assembly-output: emit-asm +//@ min-llvm-version: 23 +// +//@ [x86_64] compile-flags: --target x86_64-unknown-linux-gnu +//@ [x86_64] needs-llvm-components: x86 +// +//@ [aarch64-elf] compile-flags: --target aarch64-unknown-linux-gnu +//@ [aarch64-elf] needs-llvm-components: aarch64 +//@ [aarch64-macho] compile-flags: --target aarch64-apple-darwin +//@ [aarch64-macho] needs-llvm-components: aarch64 +//@ [aarch64-coff] compile-flags: --target aarch64-pc-windows-gnullvm +//@ [aarch64-coff] needs-llvm-components: aarch64 +// +//@ [s390x] compile-flags: --target s390x-unknown-linux-gnu +//@ [s390x] needs-llvm-components: systemz +// +//@ [powerpc64] compile-flags: --target powerpc64-unknown-linux-gnu +//@ [powerpc64] needs-llvm-components: powerpc +// +//@ [riscv64] compile-flags: --target riscv64gc-unknown-linux-gnu +//@ [riscv64] needs-llvm-components: riscv +// +// NOTE: loongarch64 does not error when using an instruction without enabling the corresponding +// target feature. +//@ [loongarch64] compile-flags: --target loongarch64-unknown-linux-gnu +//@ [loongarch64] needs-llvm-components: loongarch + +// Test that the #[target_feature(enable = ...)]` works on naked functions. + +#![crate_type = "lib"] +#![feature(no_core, naked_functions_target_feature)] +#![feature(s390x_target_feature, powerpc_target_feature, loongarch_target_feature)] +#![no_core] + +extern crate minicore; +use minicore::*; + +// x86_64-LABEL: vpclmulqdq: +// x86_64: vpclmulqdq +#[no_mangle] +#[unsafe(naked)] +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "vpclmulqdq")] +unsafe extern "C" fn vpclmulqdq() { + naked_asm!("vpclmulqdq zmm1, zmm2, zmm3, 4") +} + +// i8mm is not enabled by default +// +// note that aarch64-apple-darwin enables more features than aarch64-unknown-linux-gnu +// +// aarch64-elf-LABEL: i8mm: +// aarch64-elf: usdot +// aarch64-macho-LABEL: i8mm: +// aarch64-macho: usdot +// aarch64-coff-LABEL: i8mm: +// aarch64-coff: usdot +#[no_mangle] +#[unsafe(naked)] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "i8mm")] +unsafe extern "C" fn i8mm() { + naked_asm!("usdot v0.4s, v1.16b, v2.4b[3]") +} + +// riscv64: sh1add: +// riscv64: sh1add +#[no_mangle] +#[unsafe(naked)] +#[cfg(target_arch = "riscv64")] +#[target_feature(enable = "zba")] +unsafe extern "C" fn sh1add() { + naked_asm!("sh1add a0, a1, a2", "ret"); +} + +#[cfg(target_arch = "s390x")] +mod s390x { + use super::*; + + // s390x: vector: + // s390x: vavglg + #[no_mangle] + #[unsafe(naked)] + #[target_feature(enable = "vector")] + unsafe extern "C" fn vector() { + naked_asm!("vavglg %v0, %v0, %v0") + } + + // s390x: vector_enhancements_1: + // s390x: vfcesbs + #[no_mangle] + #[unsafe(naked)] + #[target_feature(enable = "vector-enhancements-1")] + unsafe extern "C" fn vector_enhancements_1() { + naked_asm!("vfcesbs %v0, %v0, %v0") + } + + // s390x: vector_enhancements_2: + // s390x: vclfp + #[no_mangle] + #[unsafe(naked)] + #[target_feature(enable = "vector-enhancements-2")] + unsafe extern "C" fn vector_enhancements_2() { + naked_asm!("vclfp %v0, %v0, 0, 0, 0") + } + + // s390x: vector_packed_decimal: + // s390x: vlrlr + #[no_mangle] + #[unsafe(naked)] + #[target_feature(enable = "vector-packed-decimal")] + unsafe extern "C" fn vector_packed_decimal() { + naked_asm!("vlrlr %v24, %r3, 0(%r2)", "br %r14") + } + + // s390x: vector_packed_decimal_enhancement: + // s390x: vcvbg + #[no_mangle] + #[unsafe(naked)] + #[target_feature(enable = "vector-packed-decimal-enhancement")] + unsafe extern "C" fn vector_packed_decimal_enhancement() { + naked_asm!("vcvbg %r0, %v0, 0, 1") + } + + // s390x: vector_packed_decimal_enhancement_2: + // s390x: vupkzl + #[no_mangle] + #[unsafe(naked)] + #[target_feature(enable = "vector-packed-decimal-enhancement-2")] + unsafe extern "C" fn vector_packed_decimal_enhancement_2() { + naked_asm!("vupkzl %v0, %v0, 0") + } +} + +// powerpc64: power10_vector: +// powerpc64: xxpermx +#[no_mangle] +#[unsafe(naked)] +#[cfg(target_arch = "powerpc64")] +#[target_feature(enable = "power10-vector")] +unsafe extern "C" fn power10_vector() { + naked_asm!("xxpermx 34, 0, 1, 2, 0", "blr") +} + +// loongarch64: lasx: +// loongarch64: xvadd.b +#[no_mangle] +#[unsafe(naked)] +#[cfg(target_arch = "loongarch64")] +#[target_feature(enable = "lasx")] +unsafe extern "C" fn lasx() { + naked_asm!("xvadd.b $xr0, $xr0, $xr1", "ret") +} + +// wasm32: simd128: +// wasm32: i8x16.shuffle +#[no_mangle] +#[unsafe(naked)] +#[cfg(target_arch = "wasm32")] +#[target_feature(enable = "simd128")] +unsafe extern "C" fn simd128() { + naked_asm!("i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15", "return"); +} diff --git a/tests/ui/asm/naked-functions/target-feature-aarch64.rs b/tests/ui/asm/naked-functions/target-feature-aarch64.rs new file mode 100644 index 0000000000000..f82122f773ca0 --- /dev/null +++ b/tests/ui/asm/naked-functions/target-feature-aarch64.rs @@ -0,0 +1,47 @@ +//@ add-minicore +//@ build-fail +//@ revisions: vanilla sha3 +//@ compile-flags: --target aarch64-unknown-linux-gnu -Z deduplicate-diagnostics=yes +//@[sha3] compile-flags: -Ctarget-feature=+sha3 +//@ needs-llvm-components: aarch64 +//@ min-llvm-version: 23 + +#![crate_type = "lib"] +#![feature(no_core, naked_functions_target_feature)] +#![no_core] + +extern crate minicore; +use minicore::*; + +// check that a naked function using target features does not keep these features enabled +// for subsequent asm blocks. + +#[no_mangle] +#[unsafe(naked)] +#[target_feature(enable = "i8mm")] +unsafe extern "C" fn a() { + naked_asm!("usdot v0.4s, v1.16b, v2.4b[3]") +} + +//~? ERROR instruction requires: i8mm + +#[no_mangle] +#[unsafe(naked)] +unsafe extern "C" fn c() { + naked_asm!("usdot v0.4s, v2.16b, v2.4b[3]") +} + +#[no_mangle] +#[unsafe(naked)] +#[target_feature(enable = "sha3")] +unsafe extern "C" fn d() { + naked_asm!("eor3 v0.16b, v1.16b, v2.16b, v3.16b") +} + +//[vanilla]~? ERROR instruction requires: sha3 + +#[no_mangle] +#[unsafe(naked)] +unsafe extern "C" fn b() { + naked_asm!("eor3 v0.16b, v1.16b, v2.16b, v3.16b") +} diff --git a/tests/ui/asm/naked-functions/target-feature-aarch64.sha3.stderr b/tests/ui/asm/naked-functions/target-feature-aarch64.sha3.stderr new file mode 100644 index 0000000000000..49a65eaadb904 --- /dev/null +++ b/tests/ui/asm/naked-functions/target-feature-aarch64.sha3.stderr @@ -0,0 +1,10 @@ +error: instruction requires: i8mm + | +note: instantiated into assembly here + --> :15:1 + | +LL | usdot v0.4s, v2.16b, v2.4b[3] + | ^ + +error: aborting due to 1 previous error + diff --git a/tests/ui/asm/naked-functions/target-feature-aarch64.vanilla.stderr b/tests/ui/asm/naked-functions/target-feature-aarch64.vanilla.stderr new file mode 100644 index 0000000000000..8ac31d19f5e3e --- /dev/null +++ b/tests/ui/asm/naked-functions/target-feature-aarch64.vanilla.stderr @@ -0,0 +1,18 @@ +error: instruction requires: sha3 + | +note: instantiated into assembly here + --> :6:1 + | +LL | eor3 v0.16b, v1.16b, v2.16b, v3.16b + | ^ + +error: instruction requires: i8mm + | +note: instantiated into assembly here + --> :15:1 + | +LL | usdot v0.4s, v2.16b, v2.4b[3] + | ^ + +error: aborting due to 2 previous errors + diff --git a/tests/ui/asm/naked-functions/target-feature-s390x.rs b/tests/ui/asm/naked-functions/target-feature-s390x.rs new file mode 100644 index 0000000000000..b0f806c4c0a16 --- /dev/null +++ b/tests/ui/asm/naked-functions/target-feature-s390x.rs @@ -0,0 +1,30 @@ +//@ add-minicore +//@ build-fail +//@ compile-flags: --target s390x-unknown-linux-gnu -Z deduplicate-diagnostics=yes +//@ needs-llvm-components: systemz +//@ min-llvm-version: 23 + +#![crate_type = "lib"] +#![feature(no_core, naked_functions_target_feature)] +#![no_core] + +extern crate minicore; +use minicore::*; + +// check that a naked function using target features does not keep these features enabled +// for subsequent asm blocks. + +#[no_mangle] +#[unsafe(naked)] +#[target_feature(enable = "vector-packed-decimal")] +unsafe extern "C" fn a() { + naked_asm!("vlrlr %v24, %r3, 0(%r2)") +} + +//~? ERROR instruction requires: vector-packed-decimal + +#[no_mangle] +#[unsafe(naked)] +unsafe extern "C" fn b() { + naked_asm!("vlrlr %v24, %r3, 0(%r3)") +} diff --git a/tests/ui/asm/naked-functions/target-feature-s390x.stderr b/tests/ui/asm/naked-functions/target-feature-s390x.stderr new file mode 100644 index 0000000000000..84d60c43bc765 --- /dev/null +++ b/tests/ui/asm/naked-functions/target-feature-s390x.stderr @@ -0,0 +1,10 @@ +error: instruction requires: vector-packed-decimal + | +note: instantiated into assembly here + --> :6:1 + | +LL | vlrlr %v24, %r3, 0(%r3) + | ^ + +error: aborting due to 1 previous error +