diff --git a/vm/devices/pci/pci_core/src/spec.rs b/vm/devices/pci/pci_core/src/spec.rs index 73dfbde3d99..3bade498221 100644 --- a/vm/devices/pci/pci_core/src/spec.rs +++ b/vm/devices/pci/pci_core/src/spec.rs @@ -461,6 +461,7 @@ pub mod caps { ARI = 0x0E, SRIOV = 0x10, REBAR = 0x15, + PASID = 0x1B, DVSEC = 0x23, SIOV = 0x38, } diff --git a/vm/devices/pci/vfio_assigned_device/src/iommufd_nesting.rs b/vm/devices/pci/vfio_assigned_device/src/iommufd_nesting.rs index 7d86be0d452..b6908afb6cb 100644 --- a/vm/devices/pci/vfio_assigned_device/src/iommufd_nesting.rs +++ b/vm/devices/pci/vfio_assigned_device/src/iommufd_nesting.rs @@ -65,13 +65,27 @@ use vfio_sys::iommufd::IommufdVeventHeader; use vfio_sys::iommufd::ViommuAlloc; use zerocopy::FromBytes; +/// Capabilities that apply to one assigned endpoint rather than its physical +/// SMMU. Devices behind the same SMMU may report different values. +#[derive(Debug, Clone, Copy)] +pub struct DeviceIommuCaps { + /// Maximum PASID width usable by this endpoint and its host IOMMU. + pub max_pasid_log2: u8, + /// Whether the endpoint supports PASID execute permission. + pub pasid_exec: bool, + /// Whether the endpoint supports PASID privileged mode. + pub pasid_priv: bool, +} + /// Query the physical SMMUv3's capabilities for a device bound to iommufd. /// /// Issues a single `IOMMU_GET_HW_INFO` and hands the host's raw IDR registers /// to [`smmu::HostSmmuCaps::from_idr`], which decodes the fields the vSMMU -/// finalizes against and validates compatibility with (OAS, TTF, TTENDIAN, -/// GRAN4K). -pub fn query_host_caps(ctx: &IommufdCtx, dev_id: u32) -> anyhow::Result { +/// finalizes against and validates compatibility with. +pub fn query_host_caps( + ctx: &IommufdCtx, + dev_id: u32, +) -> anyhow::Result<(smmu::HostSmmuCaps, DeviceIommuCaps)> { let mut info = vfio_sys::iommufd::IommuHwInfoArmSmmuv3 { flags: 0, __reserved: 0, @@ -79,13 +93,20 @@ pub fn query_host_caps(ctx: &IommufdCtx, dev_id: u32) -> anyhow::Result anyhow::Result { + anyhow::ensure!( + capabilities.width <= 20, + "PASID width {} exceeds the PCIe maximum of 20", + capabilities.width + ); + + let capability = (u16::from(capabilities.width) << 8) + | (u16::from(capabilities.exec) * PASID_CAP_EXEC) + | (u16::from(capabilities.privileged) * PASID_CAP_PRIV); + + Ok(Self { + offset: SYNTHETIC_PASID_OFFSET, + capability, + control: 0, + }) + } + + fn register(&self) -> u32 { + u32::from(self.capability) | (u32::from(self.control) << 16) + } + + fn contains_offset(&self, offset: u16) -> bool { + offset == self.offset || offset == self.offset + 4 + } + + fn write(&mut self, offset: u16, value: ByteEnabledDwordWrite) { + if offset == self.offset { + return; + } + debug_assert_eq!(offset, self.offset + 4); + let requested = (value.merge(self.register()) >> 16) as u16; + let mut writable = PASID_CTRL_ENABLE; + if self.capability & PASID_CAP_EXEC != 0 { + writable |= PASID_CAP_EXEC; + } + if self.capability & PASID_CAP_PRIV != 0 { + writable |= PASID_CAP_PRIV; + } + self.control = requested & writable; + } + + fn reset(&mut self) { + self.control = 0; + } +} + /// MSI-X emulation state, discovered from the physical device's capabilities. #[derive(Inspect)] struct MsixEmulationState { @@ -221,6 +291,9 @@ pub(crate) struct VfioAssignedPciDevice { )] config_patches: BTreeMap, + /// Synthetic PASID capability exposed for an accelerated IOMMUFD device. + synthetic_pasid: Option, + /// Accelerated (iommufd-nested) SMMU stream, present only for a device /// behind an accel-capable SMMU. Owns the StreamID derived from the guest /// RequesterID seen on routed config-space writes, and every host object @@ -326,6 +399,7 @@ impl VfioAssignedPciDevice { bar_addresses, // Legacy group/type1 path never does nested S1 (rejected earlier). None, + None, ) .await } @@ -340,6 +414,7 @@ impl VfioAssignedPciDevice { memory_mapper: &dyn MemoryMapper, bar_addresses: [BarAddressConfig; 6], accel_stream: Option, + pasid_capabilities: Option, ) -> anyhow::Result { Self::from_device( device, @@ -350,6 +425,7 @@ impl VfioAssignedPciDevice { memory_mapper, bar_addresses, accel_stream, + pasid_capabilities, ) .await } @@ -363,6 +439,7 @@ impl VfioAssignedPciDevice { memory_mapper: &dyn MemoryMapper, bar_addresses: [BarAddressConfig; 6], accel_stream: Option, + pasid_capabilities: Option, ) -> anyhow::Result { let config_info = vfio_device .region_info(vfio_bindings::bindings::vfio::VFIO_PCI_CONFIG_REGION_INDEX) @@ -445,7 +522,14 @@ impl VfioAssignedPciDevice { let pm_csr_offset = caps.pm_csr_offset; let pcie_flr_control_offset = caps.pcie_flr_control_offset; let af_flr_control_offset = caps.af_flr_control_offset; - let config_patches = caps.config_patches; + let mut config_patches = caps.config_patches; + let synthetic_pasid = synthesize_pasid_capability( + &mut config_patches, + caps.last_ext_cap_offset, + caps.pasid_cap_offset, + vfio_device.config_size, + pasid_capabilities, + )?; // Cache whether the device supports VFIO_DEVICE_RESET so we can skip // the ioctl on every VM reset for devices that don't support it. @@ -600,6 +684,7 @@ impl VfioAssignedPciDevice { supports_reset, bar_direct_maps, config_patches, + synthetic_pasid, accel_stream, binding, }) @@ -1021,6 +1106,10 @@ struct DiscoveredCapabilities { af_flr_control_offset: Option, /// Config space patch table for filtering capabilities from the guest. config_patches: BTreeMap, + /// Last capability in the visible extended-capability chain. + last_ext_cap_offset: Option, + /// Existing PASID capability offset, if VFIO exposes one in the future. + pasid_cap_offset: Option, } /// Walk both the standard (0x34+) and extended (0x100+) PCI capability chains @@ -1042,6 +1131,8 @@ fn discover_capabilities( pcie_flr_control_offset: None, af_flr_control_offset: None, config_patches: BTreeMap::new(), + last_ext_cap_offset: None, + pasid_cap_offset: None, }; // Clear multi-function bit so the device appears as single-function. @@ -1236,6 +1327,7 @@ fn discover_capabilities( let cap_id = caps::ExtendedCapabilityId((header & 0xFFFF) as u16); let cap_next = ((header >> 20) & 0xFFF) as u16; + result.last_ext_cap_offset = Some(offset); tracing::debug!( ?cap_id, @@ -1261,6 +1353,9 @@ fn discover_capabilities( }, ); } + caps::ExtendedCapabilityId::PASID => { + result.pasid_cap_offset = Some(offset); + } _ => {} } @@ -1284,6 +1379,50 @@ fn discover_capabilities( result } +fn synthesize_pasid_capability( + config_patches: &mut BTreeMap, + last_ext_cap_offset: Option, + existing_pasid_offset: Option, + config_size: u64, + capabilities: Option, +) -> anyhow::Result> { + let Some(capabilities) = capabilities.filter(|capabilities| capabilities.width != 0) else { + return Ok(None); + }; + if existing_pasid_offset.is_some() { + return Ok(None); + } + + anyhow::ensure!( + config_size >= 0x1000, + "cannot synthesize PASID capability in {config_size:#x}-byte PCI config space" + ); + let last_ext_cap_offset = last_ext_cap_offset + .context("cannot synthesize PASID capability without an extended-capability chain")?; + anyhow::ensure!( + last_ext_cap_offset < SYNTHETIC_PASID_OFFSET, + "cannot append PASID capability after extended capability at {last_ext_cap_offset:#x}" + ); + + let next_mask = 0xfff0_0000; + let next_value = u32::from(SYNTHETIC_PASID_OFFSET) << 20; + let tail_patch = config_patches + .entry(last_ext_cap_offset) + .or_insert(ConfigPatch { mask: 0, value: 0 }); + tail_patch.value = (tail_patch.value & !next_mask) | next_value; + tail_patch.mask |= next_mask; + + config_patches.insert( + SYNTHETIC_PASID_OFFSET, + ConfigPatch { + mask: u32::MAX, + value: u32::from(caps::ExtendedCapabilityId::PASID.0) | (1 << 16), + }, + ); + + SyntheticPasidCapability::new(capabilities).map(Some) +} + /// Read from the MSI-X emulator at the given offset, handling sub-DWORD /// accesses by aligning to u32 boundaries. fn read_msix_emulator(emulator: &MsixEmulator, offset: u64, data: &mut [u8]) { @@ -1364,7 +1503,8 @@ impl ChangeDeviceState for VfioAssignedPciDevice { ref mut msix, supports_reset, config_patches: _, // immutable — built at init - binding: _, // lifetime handle — no reset needed + ref mut synthetic_pasid, + binding: _, // lifetime handle — no reset needed ref mut accel_stream, } = *self; @@ -1381,6 +1521,9 @@ impl ChangeDeviceState for VfioAssignedPciDevice { msix.enabled = false; msix.capability.reset(); } + if let Some(pasid) = synthetic_pasid { + pasid.reset(); + } // Reset cached BAR addresses to power-on defaults. For passthrough // BARs, this restores the physical addresses so that preserve_bars @@ -1480,6 +1623,14 @@ impl PciConfigSpace for VfioAssignedPciDevice { msix.capability.read(0, v); } } + offset + if self + .synthetic_pasid + .as_ref() + .is_some_and(|pasid| offset.0 == pasid.offset + 4) => + { + value.set(self.synthetic_pasid.as_ref().unwrap().register()); + } // Everything else: read from physical device, applying any // config space patches. _ => { @@ -1661,6 +1812,14 @@ impl PciConfigSpace for VfioAssignedPciDevice { // Skip write_phys_config for MSI-X control register. return IoResult::Ok; } + _ if self + .synthetic_pasid + .as_ref() + .is_some_and(|pasid| pasid.contains_offset(offset)) => + { + self.synthetic_pasid.as_mut().unwrap().write(offset, value); + return IoResult::Ok; + } // All other registers: pass through to physical device. _ => self.write_phys_config(offset, value), } @@ -2188,6 +2347,89 @@ mod tests { ); } + #[test] + fn extended_caps_pasid_is_detected() { + let mut cfg = MockConfigSpace::new(0x200); + cfg.write_u32(0x34, 0x00); + cfg.write_u32( + 0x100, + MockConfigSpace::ext_cap_header(caps::ExtendedCapabilityId::PASID.0, 1, 0), + ); + + let discovered = discover_capabilities(&cfg, &MsiTarget::disconnected()); + assert_eq!(discovered.last_ext_cap_offset, Some(0x100)); + assert_eq!(discovered.pasid_cap_offset, Some(0x100)); + } + + #[test] + fn synthetic_pasid_capability_is_linked_and_writable() { + let mut patches = BTreeMap::from([( + 0x300, + ConfigPatch { + mask: 0x0000_ffff, + value: 0, + }, + )]); + let mut pasid = synthesize_pasid_capability( + &mut patches, + Some(0x300), + None, + 0x1000, + Some(PasidCapabilities { + width: 14, + exec: false, + privileged: true, + }), + ) + .unwrap() + .unwrap(); + + let tail = patches.get(&0x300).unwrap(); + assert_eq!(tail.mask, 0xfff0_ffff); + assert_eq!(tail.value, u32::from(SYNTHETIC_PASID_OFFSET) << 20); + let header = patches.get(&SYNTHETIC_PASID_OFFSET).unwrap(); + assert_eq!(header.mask, u32::MAX); + assert_eq!(header.value, 0x0001_001b); + assert_eq!(pasid.register(), 0x0000_0e04); + + assert!(pasid.contains_offset(SYNTHETIC_PASID_OFFSET)); + pasid.write( + SYNTHETIC_PASID_OFFSET, + ByteEnabledDwordWrite::with_all_bytes_enabled(u32::MAX), + ); + assert_eq!(pasid.control, 0); + + assert!(pasid.contains_offset(SYNTHETIC_PASID_OFFSET + 4)); + pasid.write( + SYNTHETIC_PASID_OFFSET + 4, + ByteEnabledDwordWrite::with_all_bytes_enabled(0x0007_0000), + ); + assert_eq!(pasid.control, PASID_CTRL_ENABLE | PASID_CAP_PRIV); + assert!(!pasid.contains_offset(SYNTHETIC_PASID_OFFSET + 8)); + pasid.reset(); + assert_eq!(pasid.control, 0); + } + + #[test] + fn synthetic_pasid_capability_is_not_duplicated() { + let mut patches = BTreeMap::new(); + let pasid = synthesize_pasid_capability( + &mut patches, + Some(0x300), + Some(0x280), + 0x1000, + Some(PasidCapabilities { + width: 14, + exec: false, + privileged: true, + }), + ) + .unwrap(); + + assert!(pasid.is_none()); + assert!(patches.is_empty()); + } + // --- Malformed capability chains --- #[test] diff --git a/vm/devices/pci/vfio_assigned_device/src/manager.rs b/vm/devices/pci/vfio_assigned_device/src/manager.rs index c7cdf2b0e74..6f5902dfa36 100644 --- a/vm/devices/pci/vfio_assigned_device/src/manager.rs +++ b/vm/devices/pci/vfio_assigned_device/src/manager.rs @@ -847,8 +847,9 @@ impl IoasManager { // IOAS for identity DMA. let (nesting, accel_state_id) = if let Some(vsmmu) = vsmmu { // Query the physical SMMU's capabilities backing this device. - let host_caps = crate::iommufd_nesting::query_host_caps(&self.ctx, devid) - .context("failed to query host SMMU capabilities")?; + let (host_caps, device_caps) = + crate::iommufd_nesting::query_host_caps(&self.ctx, devid) + .context("failed to query host SMMU capabilities")?; // Get or create the shared vIOMMU for this emulated SMMU. The // first device behind the SMMU allocates it; the rest reuse it @@ -889,6 +890,7 @@ impl IoasManager { Some(NestingOutput { accel_state, host_caps, + device_caps, }), Some(accel_state_id), ) @@ -1066,6 +1068,8 @@ pub(crate) struct NestingOutput { pub accel_state: Arc, /// Host SMMU capabilities, to finalize the emulated SMMU's parameters. pub host_caps: smmu::HostSmmuCaps, + /// Per-device PASID capabilities used only for endpoint PCI emulation. + pub device_caps: crate::iommufd_nesting::DeviceIommuCaps, } /// Dispatches cdev device requests to per-iommu [`IoasManager`] tasks. diff --git a/vm/devices/pci/vfio_assigned_device/src/resolver.rs b/vm/devices/pci/vfio_assigned_device/src/resolver.rs index 89c6c959f61..d178a37c1dc 100644 --- a/vm/devices/pci/vfio_assigned_device/src/resolver.rs +++ b/vm/devices/pci/vfio_assigned_device/src/resolver.rs @@ -228,15 +228,25 @@ impl AsyncResolveResource for VfioCde // StreamID here — PCI routing supplies the BDF one is derived from, // so it stays blocked until the guest assigns it. let mut accel_stream = None; + let mut pasid_capabilities = None; if let (Some(ctx), Some(nesting)) = (nesting_ctx, nesting) { // Bind the vSMMU to the physical SMMU and vIOMMU backing this // device, finalizing host-derived parameters (OAS, ...). Runs once // per vSMMU; a later device on a different physical SMMU or vIOMMU // is rejected here. + let host_caps = nesting.host_caps; ctx.shared - .bind_accel_viommu(nesting.host_caps, &nesting.accel_state) + .bind_accel_viommu(host_caps, &nesting.accel_state) .with_context(|| format!("device {pci_id} is incompatible with the host SMMU"))?; + if nesting.device_caps.max_pasid_log2 != 0 { + pasid_capabilities = Some(crate::PasidCapabilities { + width: nesting.device_caps.max_pasid_log2, + exec: nesting.device_caps.pasid_exec, + privileged: nesting.device_caps.pasid_priv, + }); + } + accel_stream = Some( crate::iommufd_nesting::AccelStream::new( &ctx, @@ -263,6 +273,7 @@ impl AsyncResolveResource for VfioCde memory_mapper, bar_addresses, accel_stream, + pasid_capabilities, ) .await?; diff --git a/vm/devices/user_driver/vfio_sys/src/iommufd.rs b/vm/devices/user_driver/vfio_sys/src/iommufd.rs index 42c315a2f1e..9fb6ca9c6da 100644 --- a/vm/devices/user_driver/vfio_sys/src/iommufd.rs +++ b/vm/devices/user_driver/vfio_sys/src/iommufd.rs @@ -225,6 +225,10 @@ pub struct IommuHwptArmSmmuv3 { /// HW info type: ARM SMMUv3. pub const IOMMU_HW_INFO_TYPE_ARM_SMMUV3: u32 = 2; +pub const IOMMU_HW_CAP_PCI_PASID_EXEC: u64 = 1 << 1; +pub const IOMMU_HW_CAP_PCI_PASID_PRIV: u64 = 1 << 2; +pub const IOMMU_HW_CAP_PCI_ATS_NOT_SUPPORTED: u64 = 1 << 3; + #[repr(C)] struct IommuGetHwInfo { size: u32, @@ -567,13 +571,13 @@ impl IommufdCtx { /// Query hardware information for a device's IOMMU. /// - /// Returns `(out_data_type, out_capabilities)`. The type-specific data is - /// written into `out_info`. + /// Returns `(out_data_type, out_max_pasid_log2, out_capabilities)`. The + /// type-specific data is written into `out_info`. pub fn get_hw_info( &self, dev_id: u32, out_info: &mut IommuHwInfoArmSmmuv3, - ) -> anyhow::Result<(u32, u64)> { + ) -> anyhow::Result<(u32, u8, u64)> { let mut cmd = IommuGetHwInfo { size: size_of::() as u32, flags: 0, @@ -595,7 +599,11 @@ impl IommufdCtx { ioctl::iommu_get_hw_info(self.file.as_raw_fd(), &mut cmd) .context("IOMMU_GET_HW_INFO failed")?; } - Ok((cmd.out_data_type, cmd.out_capabilities)) + Ok(( + cmd.out_data_type, + cmd.out_max_pasid_log2, + cmd.out_capabilities, + )) } /// Invalidate IOMMU caches via a nested HWPT or vIOMMU.