support #[target_feature(enable = ...)] on #[naked] functions

compiler/rustc_codegen_cranelift/src/global_asm.rs

@@ -30,6 +30,7 @@ impl<'tcx> AsmCodegenMethods<'tcx> for GlobalAsmContext<'_, 'tcx> {
         operands: &[GlobalAsmOperandRef<'tcx>],
         options: InlineAsmOptions,
         _line_spans: &[Span],
+        _extra_rust_target_features: &[String],
     ) {
         codegen_global_asm_inner(self.tcx, self.global_asm, template, operands, options);
     }

compiler/rustc_codegen_gcc/src/asm.rs

@@ -928,6 +928,7 @@ impl<'gcc, 'tcx> AsmCodegenMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
         operands: &[GlobalAsmOperandRef<'tcx>],
         options: InlineAsmOptions,
         line_spans: &[Span],
+        _extra_rust_target_features: &[String],
     ) {
         let asm_arch = self.tcx.sess.asm_arch.unwrap();
 

compiler/rustc_codegen_llvm/src/asm.rs

@@ -414,6 +414,7 @@ impl<'tcx> AsmCodegenMethods<'tcx> for CodegenCx<'_, 'tcx> {
         operands: &[GlobalAsmOperandRef<'tcx>],
         options: InlineAsmOptions,
         _line_spans: &[Span],
+        extra_rust_target_features: &[String],
     ) {
         let asm_arch = self.tcx.sess.asm_arch.unwrap();
 
@@ -499,14 +500,26 @@ impl<'tcx> AsmCodegenMethods<'tcx> for CodegenCx<'_, 'tcx> {
             template_str.push_str("\n.att_syntax\n");
         }
 
-        let target_features = self.tcx.global_backend_features(()).join(",");
-        let target_cpu = llvm_util::target_cpu(self.tcx.sess);
+        // Globally-enabled features that are already in the backend format.
+        let global_features = self.tcx.global_backend_features(()).iter().map(String::as_str);
+
+        // Features enabled on a particular instance, in the rust format.
+        // These need to be translated to the LLVM format.
+        let function_features: Vec<_> = extra_rust_target_features
+            .iter()
+            .flat_map(|feat| llvm_util::to_llvm_features(self.tcx.sess, feat))
+            .flat_map(|feat| feat.into_iter().map(|f| format!("+{f}")))
+            .collect();
+
+        let function_features = function_features.iter().map(String::as_str);
+        let target_features =
+            global_features.chain(function_features).intersperse(",").collect::<String>();
 
         llvm::append_module_inline_asm(
             self.llmod,
             template_str.as_bytes(),
             &target_features,
-            target_cpu,
+            llvm_util::target_cpu(self.tcx.sess),
         );
     }
 

compiler/rustc_codegen_ssa/src/base.rs

@@ -490,7 +490,7 @@ where
             })
             .collect();
 
-        cx.codegen_global_asm(asm.template, &operands, asm.options, asm.line_spans);
+        cx.codegen_global_asm(asm.template, &operands, asm.options, asm.line_spans, &[]);
     } else {
         span_bug!(item.span, "Mismatch between hir::Item type and MonoItem type")
     }

compiler/rustc_codegen_ssa/src/mir/naked_asm.rs

@@ -54,7 +54,9 @@ pub fn codegen_naked_asm<
     template_vec.extend(template.iter().cloned());
     template_vec.push(rustc_ast::ast::InlineAsmTemplatePiece::String(end.into()));
 
-    cx.codegen_global_asm(&template_vec, &operands, options, line_spans);
+    let target_features: Vec<_> =
+        cx.tcx().asm_target_features(instance.def_id()).iter().map(|s| s.to_string()).collect();
+    cx.codegen_global_asm(&template_vec, &operands, options, line_spans, &target_features);
 }
 
 fn inline_to_global_operand<'a, 'tcx, Cx: LayoutOf<'tcx, LayoutOfResult = TyAndLayout<'tcx>>>(

compiler/rustc_codegen_ssa/src/traits/asm.rs

@@ -66,12 +66,17 @@ pub trait AsmBuilderMethods<'tcx>: BackendTypes {
 }
 
 pub trait AsmCodegenMethods<'tcx> {
+    /// Codegen a module-level assembly block.
+    ///
+    /// NOTE: the target features must be the rust target feature names, not backend target
+    /// feature names. This argument is used to forward target features on naked functions.
     fn codegen_global_asm(
         &mut self,
         template: &[InlineAsmTemplatePiece],
         operands: &[GlobalAsmOperandRef<'tcx>],
         options: InlineAsmOptions,
         line_spans: &[Span],
+        extra_rust_target_features: &[String],
     );
 
     /// The mangled name of this instance

tests/assembly-llvm/naked-functions/target-feature.rs

@@ -0,0 +1,165 @@
+//@ revisions: aarch64-elf aarch64-macho aarch64-coff x86_64 s390x riscv64 powerpc64 loongarch64
+//@ add-minicore
+//@ assembly-output: emit-asm
+//@ min-llvm-version: 23
+//
+//@ [x86_64] compile-flags: --target x86_64-unknown-linux-gnu
+//@ [x86_64] needs-llvm-components: x86
+//
+//@ [aarch64-elf] compile-flags: --target aarch64-unknown-linux-gnu
+//@ [aarch64-elf] needs-llvm-components: aarch64
+//@ [aarch64-macho] compile-flags: --target aarch64-apple-darwin
+//@ [aarch64-macho] needs-llvm-components: aarch64
+//@ [aarch64-coff] compile-flags: --target aarch64-pc-windows-gnullvm
+//@ [aarch64-coff] needs-llvm-components: aarch64
+//
+//@ [s390x] compile-flags: --target s390x-unknown-linux-gnu
+//@ [s390x] needs-llvm-components: systemz
+//
+//@ [powerpc64] compile-flags: --target powerpc64-unknown-linux-gnu
+//@ [powerpc64] needs-llvm-components: powerpc
+//
+//@ [riscv64] compile-flags: --target riscv64gc-unknown-linux-gnu
+//@ [riscv64] needs-llvm-components: riscv
+//
+// NOTE: loongarch64 does not error when using an instruction without enabling the corresponding
+// target feature.
+//@ [loongarch64] compile-flags: --target loongarch64-unknown-linux-gnu
+//@ [loongarch64] needs-llvm-components: loongarch
+
+// Test that the #[target_feature(enable = ...)]` works on naked functions.
+
+#![crate_type = "lib"]
+#![feature(no_core, naked_functions_target_feature)]
+#![feature(s390x_target_feature, powerpc_target_feature, loongarch_target_feature)]
+#![no_core]
+
+extern crate minicore;
+use minicore::*;
+
+// x86_64-LABEL: vpclmulqdq:
+// x86_64: vpclmulqdq
+#[no_mangle]
+#[unsafe(naked)]
+#[cfg(target_arch = "x86_64")]
+#[target_feature(enable = "vpclmulqdq")]
+unsafe extern "C" fn vpclmulqdq() {
+    naked_asm!("vpclmulqdq zmm1, zmm2, zmm3, 4")
+}
+
+// i8mm is not enabled by default
+//
+// note that aarch64-apple-darwin enables more features than aarch64-unknown-linux-gnu
+//
+// aarch64-elf-LABEL: i8mm:
+// aarch64-elf: usdot
+// aarch64-macho-LABEL: i8mm:
+// aarch64-macho: usdot
+// aarch64-coff-LABEL: i8mm:
+// aarch64-coff: usdot
+#[no_mangle]
+#[unsafe(naked)]
+#[cfg(target_arch = "aarch64")]
+#[target_feature(enable = "i8mm")]
+unsafe extern "C" fn i8mm() {
+    naked_asm!("usdot   v0.4s, v1.16b, v2.4b[3]")
+}
+
+// riscv64: sh1add:
+// riscv64: sh1add
+#[no_mangle]
+#[unsafe(naked)]
+#[cfg(target_arch = "riscv64")]
+#[target_feature(enable = "zba")]
+unsafe extern "C" fn sh1add() {
+    naked_asm!("sh1add a0, a1, a2", "ret");
+}
+
+#[cfg(target_arch = "s390x")]
+mod s390x {
+    use super::*;
+
+    // s390x: vector:
+    // s390x: vavglg
+    #[no_mangle]
+    #[unsafe(naked)]
+    #[target_feature(enable = "vector")]
+    unsafe extern "C" fn vector() {
+        naked_asm!("vavglg  %v0, %v0, %v0")
+    }
+
+    // s390x: vector_enhancements_1:
+    // s390x: vfcesbs
+    #[no_mangle]
+    #[unsafe(naked)]
+    #[target_feature(enable = "vector-enhancements-1")]
+    unsafe extern "C" fn vector_enhancements_1() {
+        naked_asm!("vfcesbs %v0, %v0, %v0")
+    }
+
+    // s390x: vector_enhancements_2:
+    // s390x: vclfp
+    #[no_mangle]
+    #[unsafe(naked)]
+    #[target_feature(enable = "vector-enhancements-2")]
+    unsafe extern "C" fn vector_enhancements_2() {
+        naked_asm!("vclfp   %v0, %v0, 0, 0, 0")
+    }
+
+    // s390x: vector_packed_decimal:
+    // s390x: vlrlr
+    #[no_mangle]
+    #[unsafe(naked)]
+    #[target_feature(enable = "vector-packed-decimal")]
+    unsafe extern "C" fn vector_packed_decimal() {
+        naked_asm!("vlrlr   %v24, %r3, 0(%r2)", "br      %r14")
+    }
+
+    // s390x: vector_packed_decimal_enhancement:
+    // s390x: vcvbg
+    #[no_mangle]
+    #[unsafe(naked)]
+    #[target_feature(enable = "vector-packed-decimal-enhancement")]
+    unsafe extern "C" fn vector_packed_decimal_enhancement() {
+        naked_asm!("vcvbg   %r0, %v0, 0, 1")
+    }
+
+    // s390x: vector_packed_decimal_enhancement_2:
+    // s390x: vupkzl
+    #[no_mangle]
+    #[unsafe(naked)]
+    #[target_feature(enable = "vector-packed-decimal-enhancement-2")]
+    unsafe extern "C" fn vector_packed_decimal_enhancement_2() {
+        naked_asm!("vupkzl  %v0, %v0, 0")
+    }
+}
+
+// powerpc64: power10_vector:
+// powerpc64: xxpermx
+#[no_mangle]
+#[unsafe(naked)]
+#[cfg(target_arch = "powerpc64")]
+#[target_feature(enable = "power10-vector")]
+unsafe extern "C" fn power10_vector() {
+    naked_asm!("xxpermx 34, 0, 1, 2, 0", "blr")
+}
+
+// loongarch64: lasx:
+// loongarch64: xvadd.b
+#[no_mangle]
+#[unsafe(naked)]
+#[cfg(target_arch = "loongarch64")]
+#[target_feature(enable = "lasx")]
+unsafe extern "C" fn lasx() {
+    naked_asm!("xvadd.b  $xr0, $xr0, $xr1", "ret")
+}
+
+// wasm32: simd128:
+// wasm32: i8x16.shuffle
+#[no_mangle]
+#[unsafe(naked)]
+#[cfg(target_arch = "wasm32")]
+#[target_feature(enable = "simd128")]
+unsafe extern "C" fn simd128() {
+    naked_asm!("i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15", "return");
+}

tests/ui/asm/naked-functions/target-feature-aarch64.rs

@@ -0,0 +1,47 @@
+//@ add-minicore
+//@ build-fail
+//@ revisions: vanilla sha3
+//@ compile-flags: --target aarch64-unknown-linux-gnu -Z deduplicate-diagnostics=yes
+//@[sha3] compile-flags: -Ctarget-feature=+sha3
+//@ needs-llvm-components: aarch64
+//@ min-llvm-version: 23
+
+#![crate_type = "lib"]
+#![feature(no_core, naked_functions_target_feature)]
+#![no_core]
+
+extern crate minicore;
+use minicore::*;
+
+// check that a naked function using target features does not keep these features enabled
+// for subsequent asm blocks.
+
+#[no_mangle]
+#[unsafe(naked)]
+#[target_feature(enable = "i8mm")]
+unsafe extern "C" fn a() {
+    naked_asm!("usdot   v0.4s, v1.16b, v2.4b[3]")
+}
+
+//~? ERROR instruction requires: i8mm
+
+#[no_mangle]
+#[unsafe(naked)]
+unsafe extern "C" fn c() {
+    naked_asm!("usdot   v0.4s, v2.16b, v2.4b[3]")
+}
+
+#[no_mangle]
+#[unsafe(naked)]
+#[target_feature(enable = "sha3")]
+unsafe extern "C" fn d() {
+    naked_asm!("eor3 v0.16b, v1.16b, v2.16b, v3.16b")
+}
+
+//[vanilla]~? ERROR instruction requires: sha3
+
+#[no_mangle]
+#[unsafe(naked)]
+unsafe extern "C" fn b() {
+    naked_asm!("eor3 v0.16b, v1.16b, v2.16b, v3.16b")
+}

tests/ui/asm/naked-functions/target-feature-aarch64.sha3.stderr

@@ -0,0 +1,10 @@
+error: instruction requires: i8mm
+   |
+note: instantiated into assembly here
+  --> <inline asm>:15:1
+   |
+LL | usdot   v0.4s, v2.16b, v2.4b[3]
+   | ^
+
+error: aborting due to 1 previous error
+

tests/ui/asm/naked-functions/target-feature-aarch64.vanilla.stderr

@@ -0,0 +1,18 @@
+error: instruction requires: sha3
+   |
+note: instantiated into assembly here
+  --> <inline asm>:6:1
+   |
+LL | eor3 v0.16b, v1.16b, v2.16b, v3.16b
+   | ^
+
+error: instruction requires: i8mm
+   |
+note: instantiated into assembly here
+  --> <inline asm>:15:1
+   |
+LL | usdot   v0.4s, v2.16b, v2.4b[3]
+   | ^
+
+error: aborting due to 2 previous errors
+

tests/ui/asm/naked-functions/target-feature-s390x.rs

@@ -0,0 +1,30 @@
+//@ add-minicore
+//@ build-fail
+//@ compile-flags: --target s390x-unknown-linux-gnu -Z deduplicate-diagnostics=yes
+//@ needs-llvm-components: systemz
+//@ min-llvm-version: 23
+
+#![crate_type = "lib"]
+#![feature(no_core, naked_functions_target_feature)]
+#![no_core]
+
+extern crate minicore;
+use minicore::*;
+
+// check that a naked function using target features does not keep these features enabled
+// for subsequent asm blocks.
+
+#[no_mangle]
+#[unsafe(naked)]
+#[target_feature(enable = "vector-packed-decimal")]
+unsafe extern "C" fn a() {
+    naked_asm!("vlrlr   %v24, %r3, 0(%r2)")
+}
+
+//~? ERROR instruction requires: vector-packed-decimal
+
+#[no_mangle]
+#[unsafe(naked)]
+unsafe extern "C" fn b() {
+    naked_asm!("vlrlr   %v24, %r3, 0(%r3)")
+}

tests/ui/asm/naked-functions/target-feature-s390x.stderr

@@ -0,0 +1,10 @@
+error: instruction requires: vector-packed-decimal
+   |
+note: instantiated into assembly here
+  --> <inline asm>:6:1
+   |
+LL | vlrlr   %v24, %r3, 0(%r3)
+   | ^
+
+error: aborting due to 1 previous error
+