aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorMarcel Moolenaar <marcel@FreeBSD.org>2010-06-19 05:35:47 +0000
committerMarcel Moolenaar <marcel@FreeBSD.org>2010-06-19 05:35:47 +0000
commit269758563195bc83aad6c79902f318c9c9ba71dc (patch)
treeb72709e7812a0e186db3127d02eca4d767aeb2fe
parent52e50b42b82593eabe906c8514650e8ff45110f4 (diff)
downloadsrc-269758563195bc83aad6c79902f318c9c9ba71dc.tar.gz
src-269758563195bc83aad6c79902f318c9c9ba71dc.zip
MFC revs 209026 and 209085:
o Bump MAX_BPAGES from 256 to 1024. o Synchronize the kernel entry on all CPUs with the use of the ptc.g instruction on a single CPU by implementing a bare-bones readers- writer lock. Approved by: re (kensmith)
Notes
Notes: svn path=/releng/8.1/; revision=209327
-rw-r--r--sys/ia64/ia64/busdma_machdep.c4
-rw-r--r--sys/ia64/ia64/exception.S38
-rw-r--r--sys/ia64/ia64/pmap.c35
-rw-r--r--sys/ia64/ia64/vm_machdep.c5
-rw-r--r--sys/ia64/include/md_var.h1
5 files changed, 72 insertions, 11 deletions
diff --git a/sys/ia64/ia64/busdma_machdep.c b/sys/ia64/ia64/busdma_machdep.c
index 7e7366a3d513..4121742d459c 100644
--- a/sys/ia64/ia64/busdma_machdep.c
+++ b/sys/ia64/ia64/busdma_machdep.c
@@ -48,7 +48,7 @@ __FBSDID("$FreeBSD$");
#include <machine/bus.h>
#include <machine/md_var.h>
-#define MAX_BPAGES 256
+#define MAX_BPAGES 1024
struct bus_dma_tag {
bus_dma_tag_t parent;
@@ -77,7 +77,7 @@ struct bounce_page {
STAILQ_ENTRY(bounce_page) links;
};
-int busdma_swi_pending;
+u_int busdma_swi_pending;
static struct mtx bounce_lock;
static STAILQ_HEAD(bp_list, bounce_page) bounce_page_list;
diff --git a/sys/ia64/ia64/exception.S b/sys/ia64/ia64/exception.S
index 4464a880b169..bbfa7387bc9b 100644
--- a/sys/ia64/ia64/exception.S
+++ b/sys/ia64/ia64/exception.S
@@ -170,6 +170,27 @@ ENTRY_NOPROFILE(exception_save, 0)
* r30,r31=trapframe pointers
* p14,p15=memory stack switch
*/
+
+ /* PTC.G enter non-exclusive */
+ mov r24 = ar.ccv
+ movl r25 = pmap_ptc_g_sem
+ ;;
+.ptc_g_0:
+ ld8.acq r26 = [r25]
+ ;;
+ tbit.nz p12, p0 = r26, 63
+(p12) br.cond.spnt.few .ptc_g_0
+ ;;
+ mov ar.ccv = r26
+ adds r27 = 1, r26
+ ;;
+ cmpxchg8.rel r27 = [r25], r27, ar.ccv
+ ;;
+ cmp.ne p12, p0 = r26, r27
+(p12) br.cond.spnt.few .ptc_g_0
+ ;;
+ mov ar.ccv = r24
+
exception_save_restart:
{ .mmi
st8 [r30]=r19,16 // length
@@ -407,6 +428,23 @@ exception_save_restart:
movl gp=__gp
;;
}
+
+ /* PTC.G leave non-exclusive */
+ srlz.d
+ movl r25 = pmap_ptc_g_sem
+ ;;
+.ptc_g_1:
+ ld8.acq r26 = [r25]
+ ;;
+ mov ar.ccv = r26
+ adds r27 = -1, r26
+ ;;
+ cmpxchg8.rel r27 = [r25], r27, ar.ccv
+ ;;
+ cmp.ne p12, p0 = r26, r27
+(p12) br.cond.spnt.few .ptc_g_1
+ ;;
+
{ .mib
srlz.d
nop 0
diff --git a/sys/ia64/ia64/pmap.c b/sys/ia64/ia64/pmap.c
index 6fe4cdf987e3..6bb523d7a738 100644
--- a/sys/ia64/ia64/pmap.c
+++ b/sys/ia64/ia64/pmap.c
@@ -182,7 +182,8 @@ static uint64_t pmap_ptc_e_count1 = 3;
static uint64_t pmap_ptc_e_count2 = 2;
static uint64_t pmap_ptc_e_stride1 = 0x2000;
static uint64_t pmap_ptc_e_stride2 = 0x100000000;
-struct mtx pmap_ptcmutex;
+
+volatile u_long pmap_ptc_g_sem;
/*
* Data for the RID allocator
@@ -340,7 +341,6 @@ pmap_bootstrap()
pmap_ptc_e_count2,
pmap_ptc_e_stride1,
pmap_ptc_e_stride2);
- mtx_init(&pmap_ptcmutex, "Global PTC lock", NULL, MTX_SPIN);
/*
* Setup RIDs. RIDs 0..7 are reserved for the kernel.
@@ -540,7 +540,8 @@ pmap_invalidate_page(vm_offset_t va)
{
struct ia64_lpte *pte;
struct pcpu *pc;
- uint64_t tag;
+ uint64_t tag, sem;
+ register_t is;
u_int vhpt_ofs;
critical_enter();
@@ -550,10 +551,32 @@ pmap_invalidate_page(vm_offset_t va)
pte = (struct ia64_lpte *)(pc->pc_md.vhpt + vhpt_ofs);
atomic_cmpset_64(&pte->tag, tag, 1UL << 63);
}
- critical_exit();
- mtx_lock_spin(&pmap_ptcmutex);
+
+ /* PTC.G enter exclusive */
+ is = intr_disable();
+
+ /* Atomically assert writer after all writers have gone. */
+ do {
+ /* Wait until there's no more writer. */
+ do {
+ sem = atomic_load_acq_long(&pmap_ptc_g_sem);
+ tag = sem | (1ul << 63);
+ } while (sem == tag);
+ } while (!atomic_cmpset_rel_long(&pmap_ptc_g_sem, sem, tag));
+
+ /* Wait until all readers are gone. */
+ tag = (1ul << 63);
+ do {
+ sem = atomic_load_acq_long(&pmap_ptc_g_sem);
+ } while (sem != tag);
+
ia64_ptc_ga(va, PAGE_SHIFT << 2);
- mtx_unlock_spin(&pmap_ptcmutex);
+
+ /* PTC.G leave exclusive */
+ atomic_store_rel_long(&pmap_ptc_g_sem, 0);
+
+ intr_restore(is);
+ critical_exit();
}
static void
diff --git a/sys/ia64/ia64/vm_machdep.c b/sys/ia64/ia64/vm_machdep.c
index 6222d3382af4..1711f1a6d425 100644
--- a/sys/ia64/ia64/vm_machdep.c
+++ b/sys/ia64/ia64/vm_machdep.c
@@ -378,9 +378,8 @@ sf_buf_free(struct sf_buf *sf)
*/
void
swi_vm(void *dummy)
-{
-#if 0
+{
+
if (busdma_swi_pending != 0)
busdma_swi();
-#endif
}
diff --git a/sys/ia64/include/md_var.h b/sys/ia64/include/md_var.h
index 9a2f92a53b2c..f64e4c9b8563 100644
--- a/sys/ia64/include/md_var.h
+++ b/sys/ia64/include/md_var.h
@@ -75,6 +75,7 @@ struct ia64_init_return {
extern uint64_t ia64_lapic_addr;
extern long Maxmem;
+extern u_int busdma_swi_pending;
void busdma_swi(void);
int copyout_regstack(struct thread *, uint64_t *, uint64_t *);