From bebe92761b3057aa92f30bf2e6bee49a9a448c93 Mon Sep 17 00:00:00 2001
From: kib <kib@FreeBSD.org>
Date: Fri, 18 Dec 2020 03:46:50 +0000
Subject: [PATCH] pci_iov: When pci_iov_detach(9) is called, destroy VF
 children

instead of bailing out with EBUSY if there are any.

If driver module is unloaded, or just device is forcibly detached from
the driver, there is no way for driver to correctly unload otherwise.
Esp. if there are resources dedicated to the VFs which prevent turning
down other resources.

Reviewed by:	jhb
Sponsored by:	Mellanox Technologies / NVidia Networking
MFC after:	1 week
Differential revision:	https://reviews.freebsd.org/D27615
---
 sys/dev/pci/pci_iov.c | 53 +++++++++++++++++++++++++++++++------------
 1 file changed, 38 insertions(+), 15 deletions(-)

diff --git a/sys/dev/pci/pci_iov.c b/sys/dev/pci/pci_iov.c
index f5063d62ffc..6db2cf44584 100644
--- a/sys/dev/pci/pci_iov.c
+++ b/sys/dev/pci/pci_iov.c
@@ -95,6 +95,7 @@ static void	pci_iov_build_pf_schema(nvlist_t *schema,
 		    nvlist_t **driver_schema);
 static void	pci_iov_build_vf_schema(nvlist_t *schema,
 		    nvlist_t **driver_schema);
+static int	pci_iov_delete_iov_children(struct pci_devinfo *dinfo);
 static nvlist_t	*pci_iov_get_pf_subsystem_schema(void);
 static nvlist_t	*pci_iov_get_vf_subsystem_schema(void);
 
@@ -191,6 +192,7 @@ pci_iov_detach_method(device_t bus, device_t dev)
 {
 	struct pci_devinfo *dinfo;
 	struct pcicfg_iov *iov;
+	int error;
 
 	mtx_lock(&Giant);
 	dinfo = device_get_ivars(dev);
@@ -201,11 +203,17 @@ pci_iov_detach_method(device_t bus, device_t dev)
 		return (0);
 	}
 
-	if (iov->iov_num_vfs != 0 || iov->iov_flags & IOV_BUSY) {
+	if ((iov->iov_flags & IOV_BUSY) != 0) {
 		mtx_unlock(&Giant);
 		return (EBUSY);
 	}
 
+	error = pci_iov_delete_iov_children(dinfo);
+	if (error != 0) {
+		mtx_unlock(&Giant);
+		return (error);
+	}
+
 	dinfo->cfg.iov = NULL;
 
 	if (iov->iov_cdev) {
@@ -823,31 +831,20 @@ pci_iov_is_child_vf(struct pcicfg_iov *pf, device_t child)
 }
 
 static int
-pci_iov_delete(struct cdev *cdev)
+pci_iov_delete_iov_children(struct pci_devinfo *dinfo)
 {
 	device_t bus, dev, vf, *devlist;
-	struct pci_devinfo *dinfo;
 	struct pcicfg_iov *iov;
 	int i, error, devcount;
 	uint32_t iov_ctl;
 
-	mtx_lock(&Giant);
-	dinfo = cdev->si_drv1;
+	mtx_assert(&Giant, MA_OWNED);
+
 	iov = dinfo->cfg.iov;
 	dev = dinfo->cfg.dev;
 	bus = device_get_parent(dev);
 	devlist = NULL;
 
-	if (iov->iov_flags & IOV_BUSY) {
-		mtx_unlock(&Giant);
-		return (EBUSY);
-	}
-
-	if (iov->iov_num_vfs == 0) {
-		mtx_unlock(&Giant);
-		return (ECHILD);
-	}
-
 	iov->iov_flags |= IOV_BUSY;
 
 	error = device_get_children(bus, &devlist, &devcount);
@@ -905,6 +902,32 @@ pci_iov_delete(struct cdev *cdev)
 out:
 	free(devlist, M_TEMP);
 	iov->iov_flags &= ~IOV_BUSY;
+	return (error);
+}
+
+static int
+pci_iov_delete(struct cdev *cdev)
+{
+	struct pci_devinfo *dinfo;
+	struct pcicfg_iov *iov;
+	int error;
+
+	mtx_lock(&Giant);
+	dinfo = cdev->si_drv1;
+	iov = dinfo->cfg.iov;
+
+	if ((iov->iov_flags & IOV_BUSY) != 0) {
+		error = EBUSY;
+		goto out;
+	}
+	if (iov->iov_num_vfs == 0) {
+		error = ECHILD;
+		goto out;
+	}
+
+	error = pci_iov_delete_iov_children(dinfo);
+
+out:
 	mtx_unlock(&Giant);
 	return (error);
 }
-- 
2.45.0