From 5d4e3337970124a94514222b9ec21ace61ad5269 Mon Sep 17 00:00:00 2001 From: kib Date: Mon, 26 Jun 2017 12:30:39 +0000 Subject: [PATCH] MFC r320125: Fix batched unload for DMAR busdma in qi mode. Approved by: re (marius) --- sys/x86/iommu/intel_ctx.c | 29 ++++++++++------------------- sys/x86/iommu/intel_dmar.h | 2 +- sys/x86/iommu/intel_qi.c | 28 ++++++++++++++-------------- 3 files changed, 25 insertions(+), 34 deletions(-) diff --git a/sys/x86/iommu/intel_ctx.c b/sys/x86/iommu/intel_ctx.c index 73ceb0ecdd9..98fb98e94f4 100644 --- a/sys/x86/iommu/intel_ctx.c +++ b/sys/x86/iommu/intel_ctx.c @@ -703,7 +703,7 @@ dmar_domain_unload_entry(struct dmar_map_entry *entry, bool free) if (unit->qi_enabled) { DMAR_LOCK(unit); dmar_qi_invalidate_locked(entry->domain, entry->start, - entry->end - entry->start, &entry->gseq); + entry->end - entry->start, &entry->gseq, true); if (!free) entry->flags |= DMAR_MAP_ENTRY_QI_NF; TAILQ_INSERT_TAIL(&unit->tlb_flush_entries, entry, dmamap_link); @@ -715,16 +715,14 @@ dmar_domain_unload_entry(struct dmar_map_entry *entry, bool free) } } -static struct dmar_qi_genseq * -dmar_domain_unload_gseq(struct dmar_domain *domain, - struct dmar_map_entry *entry, struct dmar_qi_genseq *gseq) +static bool +dmar_domain_unload_emit_wait(struct dmar_domain *domain, + struct dmar_map_entry *entry) { - if (TAILQ_NEXT(entry, dmamap_link) != NULL) - return (NULL); - if (domain->batch_no++ % dmar_batch_coalesce != 0) - return (NULL); - return (gseq); + if (TAILQ_NEXT(entry, dmamap_link) == NULL) + return (true); + return (domain->batch_no++ % dmar_batch_coalesce == 0); } void @@ -733,7 +731,6 @@ dmar_domain_unload(struct dmar_domain *domain, { struct dmar_unit *unit; struct dmar_map_entry *entry, *entry1; - struct dmar_qi_genseq gseq; int error; unit = domain->dmar; @@ -757,17 +754,11 @@ dmar_domain_unload(struct dmar_domain *domain, KASSERT(unit->qi_enabled, ("loaded entry left")); DMAR_LOCK(unit); TAILQ_FOREACH(entry, entries, dmamap_link) { - entry->gseq.gen = 0; - entry->gseq.seq = 0; dmar_qi_invalidate_locked(domain, entry->start, entry->end - - entry->start, dmar_domain_unload_gseq(domain, entry, - &gseq)); - } - TAILQ_FOREACH_SAFE(entry, entries, dmamap_link, entry1) { - entry->gseq = gseq; - TAILQ_REMOVE(entries, entry, dmamap_link); - TAILQ_INSERT_TAIL(&unit->tlb_flush_entries, entry, dmamap_link); + entry->start, &entry->gseq, + dmar_domain_unload_emit_wait(domain, entry)); } + TAILQ_CONCAT(&unit->tlb_flush_entries, entries, dmamap_link); DMAR_UNLOCK(unit); } diff --git a/sys/x86/iommu/intel_dmar.h b/sys/x86/iommu/intel_dmar.h index 698b5362dc5..db7e6d5cc64 100644 --- a/sys/x86/iommu/intel_dmar.h +++ b/sys/x86/iommu/intel_dmar.h @@ -305,7 +305,7 @@ void dmar_disable_qi_intr(struct dmar_unit *unit); int dmar_init_qi(struct dmar_unit *unit); void dmar_fini_qi(struct dmar_unit *unit); void dmar_qi_invalidate_locked(struct dmar_domain *domain, dmar_gaddr_t start, - dmar_gaddr_t size, struct dmar_qi_genseq *pseq); + dmar_gaddr_t size, struct dmar_qi_genseq *psec, bool emit_wait); void dmar_qi_invalidate_ctx_glob_locked(struct dmar_unit *unit); void dmar_qi_invalidate_iotlb_glob_locked(struct dmar_unit *unit); void dmar_qi_invalidate_iec_glob(struct dmar_unit *unit); diff --git a/sys/x86/iommu/intel_qi.c b/sys/x86/iommu/intel_qi.c index 4667ecc707e..087ce5fa71a 100644 --- a/sys/x86/iommu/intel_qi.c +++ b/sys/x86/iommu/intel_qi.c @@ -171,7 +171,8 @@ dmar_qi_emit_wait_descr(struct dmar_unit *unit, uint32_t seq, bool intr, } static void -dmar_qi_emit_wait_seq(struct dmar_unit *unit, struct dmar_qi_genseq *pseq) +dmar_qi_emit_wait_seq(struct dmar_unit *unit, struct dmar_qi_genseq *pseq, + bool emit_wait) { struct dmar_qi_genseq gsec; uint32_t seq; @@ -192,7 +193,10 @@ dmar_qi_emit_wait_seq(struct dmar_unit *unit, struct dmar_qi_genseq *pseq) seq = unit->inv_waitd_seq++; pseq->gen = unit->inv_waitd_gen; pseq->seq = seq; - dmar_qi_emit_wait_descr(unit, seq, true, true, false); + if (emit_wait) { + dmar_qi_ensure(unit, 1); + dmar_qi_emit_wait_descr(unit, seq, true, true, false); + } } static void @@ -215,7 +219,7 @@ dmar_qi_wait_for_seq(struct dmar_unit *unit, const struct dmar_qi_genseq *gseq, void dmar_qi_invalidate_locked(struct dmar_domain *domain, dmar_gaddr_t base, - dmar_gaddr_t size, struct dmar_qi_genseq *pseq) + dmar_gaddr_t size, struct dmar_qi_genseq *pseq, bool emit_wait) { struct dmar_unit *unit; dmar_gaddr_t isize; @@ -232,10 +236,7 @@ dmar_qi_invalidate_locked(struct dmar_domain *domain, dmar_gaddr_t base, DMAR_IQ_DESCR_IOTLB_DID(domain->domain), base | am); } - if (pseq != NULL) { - dmar_qi_ensure(unit, 1); - dmar_qi_emit_wait_seq(unit, pseq); - } + dmar_qi_emit_wait_seq(unit, pseq, emit_wait); dmar_qi_advance_tail(unit); } @@ -247,7 +248,7 @@ dmar_qi_invalidate_ctx_glob_locked(struct dmar_unit *unit) DMAR_ASSERT_LOCKED(unit); dmar_qi_ensure(unit, 2); dmar_qi_emit(unit, DMAR_IQ_DESCR_CTX_INV | DMAR_IQ_DESCR_CTX_GLOB, 0); - dmar_qi_emit_wait_seq(unit, &gseq); + dmar_qi_emit_wait_seq(unit, &gseq, true); dmar_qi_advance_tail(unit); dmar_qi_wait_for_seq(unit, &gseq, false); } @@ -261,7 +262,7 @@ dmar_qi_invalidate_iotlb_glob_locked(struct dmar_unit *unit) dmar_qi_ensure(unit, 2); dmar_qi_emit(unit, DMAR_IQ_DESCR_IOTLB_INV | DMAR_IQ_DESCR_IOTLB_GLOB | DMAR_IQ_DESCR_IOTLB_DW | DMAR_IQ_DESCR_IOTLB_DR, 0); - dmar_qi_emit_wait_seq(unit, &gseq); + dmar_qi_emit_wait_seq(unit, &gseq, true); dmar_qi_advance_tail(unit); dmar_qi_wait_for_seq(unit, &gseq, false); } @@ -274,7 +275,7 @@ dmar_qi_invalidate_iec_glob(struct dmar_unit *unit) DMAR_ASSERT_LOCKED(unit); dmar_qi_ensure(unit, 2); dmar_qi_emit(unit, DMAR_IQ_DESCR_IEC_INV, 0); - dmar_qi_emit_wait_seq(unit, &gseq); + dmar_qi_emit_wait_seq(unit, &gseq, true); dmar_qi_advance_tail(unit); dmar_qi_wait_for_seq(unit, &gseq, false); } @@ -298,7 +299,7 @@ dmar_qi_invalidate_iec(struct dmar_unit *unit, u_int start, u_int cnt) DMAR_IQ_DESCR_IEC_IM(l), 0); } dmar_qi_ensure(unit, 1); - dmar_qi_emit_wait_seq(unit, &gseq); + dmar_qi_emit_wait_seq(unit, &gseq, true); dmar_qi_advance_tail(unit); /* @@ -344,8 +345,7 @@ dmar_qi_task(void *arg, int pending __unused) entry = TAILQ_FIRST(&unit->tlb_flush_entries); if (entry == NULL) break; - if ((entry->gseq.gen == 0 && entry->gseq.seq == 0) || - !dmar_qi_seq_processed(unit, &entry->gseq)) + if (!dmar_qi_seq_processed(unit, &entry->gseq)) break; TAILQ_REMOVE(&unit->tlb_flush_entries, entry, dmamap_link); DMAR_UNLOCK(unit); @@ -432,7 +432,7 @@ dmar_fini_qi(struct dmar_unit *unit) DMAR_LOCK(unit); /* quisce */ dmar_qi_ensure(unit, 1); - dmar_qi_emit_wait_seq(unit, &gseq); + dmar_qi_emit_wait_seq(unit, &gseq, true); dmar_qi_advance_tail(unit); dmar_qi_wait_for_seq(unit, &gseq, false); /* only after the quisce, disable queue */ -- 2.45.0