aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorJohn Baldwin <jhb@FreeBSD.org>2020-12-14 20:40:21 +0000
committerJohn Baldwin <jhb@FreeBSD.org>2020-12-14 20:40:21 +0000
commitb17a444a2988346c239df4b3c66a550b5b7e2118 (patch)
tree89851a0206c0ef0fa092f9b1fe522a10a16efe11
parent9a3a6b30ae6005b17dbbb0879c9adede5c1cc896 (diff)
downloadsrc-b17a444a2988346c239df4b3c66a550b5b7e2118.tar.gz
src-b17a444a2988346c239df4b3c66a550b5b7e2118.zip
MFC 368003:
Honor the disabled setting for MSI-X interrupts for passthrough devices. Add a new ioctl to disable all MSI-X interrupts for a PCI passthrough device and invoke it if a write to the MSI-X capability registers disables MSI-X. This avoids leaving MSI-X interrupts enabled on the host if a guest device driver has disabled them (e.g. as part of detaching a guest device driver). This was found by Chelsio QA when testing that a Linux guest could switch from MSI-X to MSI interrupts when using the cxgb4vf driver. While here, explicitly fail requests to enable MSI on a passthrough device if MSI-X is enabled and vice versa.
Notes
Notes: svn path=/stable/12/; revision=368641
-rw-r--r--lib/libvmmapi/vmmapi.c16
-rw-r--r--lib/libvmmapi/vmmapi.h1
-rw-r--r--sys/amd64/include/vmm_dev.h3
-rw-r--r--sys/amd64/vmm/io/ppt.c23
-rw-r--r--sys/amd64/vmm/io/ppt.h1
-rw-r--r--sys/amd64/vmm/vmm_dev.c5
-rw-r--r--usr.sbin/bhyve/pci_passthru.c5
7 files changed, 53 insertions, 1 deletions
diff --git a/lib/libvmmapi/vmmapi.c b/lib/libvmmapi/vmmapi.c
index cbb441b84dc0..2bf034ee7f4d 100644
--- a/lib/libvmmapi/vmmapi.c
+++ b/lib/libvmmapi/vmmapi.c
@@ -952,6 +952,19 @@ vm_setup_pptdev_msix(struct vmctx *ctx, int vcpu, int bus, int slot, int func,
return ioctl(ctx->fd, VM_PPTDEV_MSIX, &pptmsix);
}
+int
+vm_disable_pptdev_msix(struct vmctx *ctx, int bus, int slot, int func)
+{
+ struct vm_pptdev ppt;
+
+ bzero(&ppt, sizeof(ppt));
+ ppt.bus = bus;
+ ppt.slot = slot;
+ ppt.func = func;
+
+ return ioctl(ctx->fd, VM_PPTDEV_DISABLE_MSIX, &ppt);
+}
+
uint64_t *
vm_get_stats(struct vmctx *ctx, int vcpu, struct timeval *ret_tv,
int *ret_entries)
@@ -1552,7 +1565,8 @@ vm_get_ioctls(size_t *len)
VM_ISA_DEASSERT_IRQ, VM_ISA_PULSE_IRQ, VM_ISA_SET_IRQ_TRIGGER,
VM_SET_CAPABILITY, VM_GET_CAPABILITY, VM_BIND_PPTDEV,
VM_UNBIND_PPTDEV, VM_MAP_PPTDEV_MMIO, VM_PPTDEV_MSI,
- VM_PPTDEV_MSIX, VM_INJECT_NMI, VM_STATS, VM_STAT_DESC,
+ VM_PPTDEV_MSIX, VM_PPTDEV_DISABLE_MSIX,
+ VM_INJECT_NMI, VM_STATS, VM_STAT_DESC,
VM_SET_X2APIC_STATE, VM_GET_X2APIC_STATE,
VM_GET_HPET_CAPABILITIES, VM_GET_GPA_PMAP, VM_GLA2GPA,
VM_GLA2GPA_NOFAULT,
diff --git a/lib/libvmmapi/vmmapi.h b/lib/libvmmapi/vmmapi.h
index 9819cda16bd9..5971028a0e32 100644
--- a/lib/libvmmapi/vmmapi.h
+++ b/lib/libvmmapi/vmmapi.h
@@ -168,6 +168,7 @@ int vm_setup_pptdev_msi(struct vmctx *ctx, int vcpu, int bus, int slot,
int vm_setup_pptdev_msix(struct vmctx *ctx, int vcpu, int bus, int slot,
int func, int idx, uint64_t addr, uint64_t msg,
uint32_t vector_control);
+int vm_disable_pptdev_msix(struct vmctx *ctx, int bus, int slot, int func);
int vm_get_intinfo(struct vmctx *ctx, int vcpu, uint64_t *i1, uint64_t *i2);
int vm_set_intinfo(struct vmctx *ctx, int vcpu, uint64_t exit_intinfo);
diff --git a/sys/amd64/include/vmm_dev.h b/sys/amd64/include/vmm_dev.h
index d77320dcaef7..5a5834d4a2e5 100644
--- a/sys/amd64/include/vmm_dev.h
+++ b/sys/amd64/include/vmm_dev.h
@@ -281,6 +281,7 @@ enum {
IOCNUM_MAP_PPTDEV_MMIO = 42,
IOCNUM_PPTDEV_MSI = 43,
IOCNUM_PPTDEV_MSIX = 44,
+ IOCNUM_PPTDEV_DISABLE_MSIX = 45,
/* statistics */
IOCNUM_VM_STATS = 50,
@@ -378,6 +379,8 @@ enum {
_IOW('v', IOCNUM_PPTDEV_MSI, struct vm_pptdev_msi)
#define VM_PPTDEV_MSIX \
_IOW('v', IOCNUM_PPTDEV_MSIX, struct vm_pptdev_msix)
+#define VM_PPTDEV_DISABLE_MSIX \
+ _IOW('v', IOCNUM_PPTDEV_DISABLE_MSIX, struct vm_pptdev)
#define VM_INJECT_NMI \
_IOW('v', IOCNUM_INJECT_NMI, struct vm_nmi)
#define VM_STATS \
diff --git a/sys/amd64/vmm/io/ppt.c b/sys/amd64/vmm/io/ppt.c
index 73fd576c0930..782b1c108436 100644
--- a/sys/amd64/vmm/io/ppt.c
+++ b/sys/amd64/vmm/io/ppt.c
@@ -517,6 +517,10 @@ ppt_setup_msi(struct vm *vm, int vcpu, int bus, int slot, int func,
if (ppt->vm != vm) /* Make sure we own this device */
return (EBUSY);
+ /* Reject attempts to enable MSI while MSI-X is active. */
+ if (ppt->msix.num_msgs != 0 && numvec != 0)
+ return (EBUSY);
+
/* Free any allocated resources */
ppt_teardown_msi(ppt);
@@ -606,6 +610,10 @@ ppt_setup_msix(struct vm *vm, int vcpu, int bus, int slot, int func,
if (ppt->vm != vm) /* Make sure we own this device */
return (EBUSY);
+ /* Reject attempts to enable MSI-X while MSI is active. */
+ if (ppt->msi.num_msgs != 0)
+ return (EBUSY);
+
dinfo = device_get_ivars(ppt->dev);
if (!dinfo)
return (ENXIO);
@@ -699,3 +707,18 @@ ppt_setup_msix(struct vm *vm, int vcpu, int bus, int slot, int func,
return (0);
}
+
+int
+ppt_disable_msix(struct vm *vm, int bus, int slot, int func)
+{
+ struct pptdev *ppt;
+
+ ppt = ppt_find(bus, slot, func);
+ if (ppt == NULL)
+ return (ENOENT);
+ if (ppt->vm != vm) /* Make sure we own this device */
+ return (EBUSY);
+
+ ppt_teardown_msix(ppt);
+ return (0);
+}
diff --git a/sys/amd64/vmm/io/ppt.h b/sys/amd64/vmm/io/ppt.h
index df8413031167..223afb343e8c 100644
--- a/sys/amd64/vmm/io/ppt.h
+++ b/sys/amd64/vmm/io/ppt.h
@@ -38,6 +38,7 @@ int ppt_setup_msi(struct vm *vm, int vcpu, int bus, int slot, int func,
uint64_t addr, uint64_t msg, int numvec);
int ppt_setup_msix(struct vm *vm, int vcpu, int bus, int slot, int func,
int idx, uint64_t addr, uint64_t msg, uint32_t vector_control);
+int ppt_disable_msix(struct vm *vm, int bus, int slot, int func);
int ppt_assigned_devices(struct vm *vm);
bool ppt_is_mmio(struct vm *vm, vm_paddr_t gpa);
diff --git a/sys/amd64/vmm/vmm_dev.c b/sys/amd64/vmm/vmm_dev.c
index 0613f3518f3c..c2d0b0a8b81b 100644
--- a/sys/amd64/vmm/vmm_dev.c
+++ b/sys/amd64/vmm/vmm_dev.c
@@ -496,6 +496,11 @@ vmmdev_ioctl(struct cdev *cdev, u_long cmd, caddr_t data, int fflag,
pptmsix->addr, pptmsix->msg,
pptmsix->vector_control);
break;
+ case VM_PPTDEV_DISABLE_MSIX:
+ pptdev = (struct vm_pptdev *)data;
+ error = ppt_disable_msix(sc->vm, pptdev->bus, pptdev->slot,
+ pptdev->func);
+ break;
case VM_MAP_PPTDEV_MMIO:
pptmmio = (struct vm_pptdev_mmio *)data;
error = ppt_map_mmio(sc->vm, pptmmio->bus, pptmmio->slot,
diff --git a/usr.sbin/bhyve/pci_passthru.c b/usr.sbin/bhyve/pci_passthru.c
index 51596b8f39b3..c820eee56baf 100644
--- a/usr.sbin/bhyve/pci_passthru.c
+++ b/usr.sbin/bhyve/pci_passthru.c
@@ -872,6 +872,11 @@ passthru_cfgwrite(struct vmctx *ctx, int vcpu, struct pci_devinst *pi,
if (error)
err(1, "vm_setup_pptdev_msix");
}
+ } else {
+ error = vm_disable_pptdev_msix(ctx, sc->psc_sel.pc_bus,
+ sc->psc_sel.pc_dev, sc->psc_sel.pc_func);
+ if (error)
+ err(1, "vm_disable_pptdev_msix");
}
return (0);
}