From c781450a4a6d5faad58948263b4dacbce26cad56 Mon Sep 17 00:00:00 2001 From: marcel Date: Sat, 19 Jun 2010 05:35:47 +0000 Subject: [PATCH] MFC revs 209026 and 209085: o Bump MAX_BPAGES from 256 to 1024. o Synchronize the kernel entry on all CPUs with the use of the ptc.g instruction on a single CPU by implementing a bare-bones readers- writer lock. Approved by: re (kensmith) git-svn-id: svn://svn.freebsd.org/base/releng/8.1@209327 ccf9f872-aa2e-dd11-9fc8-001c23d0bc1f --- sys/ia64/ia64/busdma_machdep.c | 4 ++-- sys/ia64/ia64/exception.S | 38 ++++++++++++++++++++++++++++++++++ sys/ia64/ia64/pmap.c | 35 +++++++++++++++++++++++++------ sys/ia64/ia64/vm_machdep.c | 5 ++--- sys/ia64/include/md_var.h | 1 + 5 files changed, 72 insertions(+), 11 deletions(-) diff --git a/sys/ia64/ia64/busdma_machdep.c b/sys/ia64/ia64/busdma_machdep.c index 7e7366a3..4121742d 100644 --- a/sys/ia64/ia64/busdma_machdep.c +++ b/sys/ia64/ia64/busdma_machdep.c @@ -48,7 +48,7 @@ __FBSDID("$FreeBSD$"); #include #include -#define MAX_BPAGES 256 +#define MAX_BPAGES 1024 struct bus_dma_tag { bus_dma_tag_t parent; @@ -77,7 +77,7 @@ struct bounce_page { STAILQ_ENTRY(bounce_page) links; }; -int busdma_swi_pending; +u_int busdma_swi_pending; static struct mtx bounce_lock; static STAILQ_HEAD(bp_list, bounce_page) bounce_page_list; diff --git a/sys/ia64/ia64/exception.S b/sys/ia64/ia64/exception.S index 4464a880..bbfa7387 100644 --- a/sys/ia64/ia64/exception.S +++ b/sys/ia64/ia64/exception.S @@ -170,6 +170,27 @@ ENTRY_NOPROFILE(exception_save, 0) * r30,r31=trapframe pointers * p14,p15=memory stack switch */ + + /* PTC.G enter non-exclusive */ + mov r24 = ar.ccv + movl r25 = pmap_ptc_g_sem + ;; +.ptc_g_0: + ld8.acq r26 = [r25] + ;; + tbit.nz p12, p0 = r26, 63 +(p12) br.cond.spnt.few .ptc_g_0 + ;; + mov ar.ccv = r26 + adds r27 = 1, r26 + ;; + cmpxchg8.rel r27 = [r25], r27, ar.ccv + ;; + cmp.ne p12, p0 = r26, r27 +(p12) br.cond.spnt.few .ptc_g_0 + ;; + mov ar.ccv = r24 + exception_save_restart: { .mmi st8 [r30]=r19,16 // length @@ -407,6 +428,23 @@ exception_save_restart: movl gp=__gp ;; } + + /* PTC.G leave non-exclusive */ + srlz.d + movl r25 = pmap_ptc_g_sem + ;; +.ptc_g_1: + ld8.acq r26 = [r25] + ;; + mov ar.ccv = r26 + adds r27 = -1, r26 + ;; + cmpxchg8.rel r27 = [r25], r27, ar.ccv + ;; + cmp.ne p12, p0 = r26, r27 +(p12) br.cond.spnt.few .ptc_g_1 + ;; + { .mib srlz.d nop 0 diff --git a/sys/ia64/ia64/pmap.c b/sys/ia64/ia64/pmap.c index 6fe4cdf9..6bb523d7 100644 --- a/sys/ia64/ia64/pmap.c +++ b/sys/ia64/ia64/pmap.c @@ -182,7 +182,8 @@ static uint64_t pmap_ptc_e_count1 = 3; static uint64_t pmap_ptc_e_count2 = 2; static uint64_t pmap_ptc_e_stride1 = 0x2000; static uint64_t pmap_ptc_e_stride2 = 0x100000000; -struct mtx pmap_ptcmutex; + +volatile u_long pmap_ptc_g_sem; /* * Data for the RID allocator @@ -340,7 +341,6 @@ pmap_bootstrap() pmap_ptc_e_count2, pmap_ptc_e_stride1, pmap_ptc_e_stride2); - mtx_init(&pmap_ptcmutex, "Global PTC lock", NULL, MTX_SPIN); /* * Setup RIDs. RIDs 0..7 are reserved for the kernel. @@ -540,7 +540,8 @@ pmap_invalidate_page(vm_offset_t va) { struct ia64_lpte *pte; struct pcpu *pc; - uint64_t tag; + uint64_t tag, sem; + register_t is; u_int vhpt_ofs; critical_enter(); @@ -550,10 +551,32 @@ pmap_invalidate_page(vm_offset_t va) pte = (struct ia64_lpte *)(pc->pc_md.vhpt + vhpt_ofs); atomic_cmpset_64(&pte->tag, tag, 1UL << 63); } - critical_exit(); - mtx_lock_spin(&pmap_ptcmutex); + + /* PTC.G enter exclusive */ + is = intr_disable(); + + /* Atomically assert writer after all writers have gone. */ + do { + /* Wait until there's no more writer. */ + do { + sem = atomic_load_acq_long(&pmap_ptc_g_sem); + tag = sem | (1ul << 63); + } while (sem == tag); + } while (!atomic_cmpset_rel_long(&pmap_ptc_g_sem, sem, tag)); + + /* Wait until all readers are gone. */ + tag = (1ul << 63); + do { + sem = atomic_load_acq_long(&pmap_ptc_g_sem); + } while (sem != tag); + ia64_ptc_ga(va, PAGE_SHIFT << 2); - mtx_unlock_spin(&pmap_ptcmutex); + + /* PTC.G leave exclusive */ + atomic_store_rel_long(&pmap_ptc_g_sem, 0); + + intr_restore(is); + critical_exit(); } static void diff --git a/sys/ia64/ia64/vm_machdep.c b/sys/ia64/ia64/vm_machdep.c index 6222d338..1711f1a6 100644 --- a/sys/ia64/ia64/vm_machdep.c +++ b/sys/ia64/ia64/vm_machdep.c @@ -378,9 +378,8 @@ sf_buf_free(struct sf_buf *sf) */ void swi_vm(void *dummy) -{ -#if 0 +{ + if (busdma_swi_pending != 0) busdma_swi(); -#endif } diff --git a/sys/ia64/include/md_var.h b/sys/ia64/include/md_var.h index 9a2f92a5..f64e4c9b 100644 --- a/sys/ia64/include/md_var.h +++ b/sys/ia64/include/md_var.h @@ -75,6 +75,7 @@ struct ia64_init_return { extern uint64_t ia64_lapic_addr; extern long Maxmem; +extern u_int busdma_swi_pending; void busdma_swi(void); int copyout_regstack(struct thread *, uint64_t *, uint64_t *); -- 2.45.0