From d9753fcc91300f01e119361188066ae736562a35 Mon Sep 17 00:00:00 2001 From: ups Date: Thu, 9 Jun 2005 18:26:31 +0000 Subject: [PATCH] Fix some race conditions for pinned threads that may cause them to run on the wrong CPU. Add IPI support for preempting a thread on another CPU. MFC after:3 weeks --- sys/kern/kern_switch.c | 48 ++++++++----- sys/kern/sched_4bsd.c | 155 ++++++++++++++++++++++++++++------------- 2 files changed, 138 insertions(+), 65 deletions(-) diff --git a/sys/kern/kern_switch.c b/sys/kern/kern_switch.c index fe1c382a8b9..1870d069bf1 100644 --- a/sys/kern/kern_switch.c +++ b/sys/kern/kern_switch.c @@ -341,12 +341,6 @@ maybe_preempt_in_ksegrp(struct thread *td) { struct thread *running_thread; -#ifndef FULL_PREEMPTION - int pri; - pri = td->td_priority; - if (!(pri >= PRI_MIN_ITHD && pri <= PRI_MAX_ITHD)) - return; -#endif mtx_assert(&sched_lock, MA_OWNED); running_thread = curthread; @@ -356,14 +350,21 @@ maybe_preempt_in_ksegrp(struct thread *td) if (td->td_priority >= running_thread->td_priority) return; #ifdef PREEMPTION +#ifndef FULL_PREEMPTION + if (td->td_priority > PRI_MAX_ITHD) { + running_thread->td_flags |= TDF_NEEDRESCHED; + return; + } +#endif /* FULL_PREEMPTION */ + if (running_thread->td_critnest > 1) running_thread->td_owepreempt = 1; else mi_switch(SW_INVOL, NULL); -#else +#else /* PREEMPTION */ running_thread->td_flags |= TDF_NEEDRESCHED; -#endif +#endif /* PREEMPTION */ return; } @@ -377,13 +378,6 @@ maybe_preempt_in_ksegrp(struct thread *td) struct pcpu *best_pcpu; struct thread *cputhread; -#ifndef FULL_PREEMPTION - int pri; - pri = td->td_priority; - if (!(pri >= PRI_MIN_ITHD && pri <= PRI_MAX_ITHD)) - return; -#endif - mtx_assert(&sched_lock, MA_OWNED); running_thread = curthread; @@ -433,6 +427,17 @@ maybe_preempt_in_ksegrp(struct thread *td) if (best_pcpu == NULL) return; +#if defined(IPI_PREEMPTION) && defined(PREEMPTION) + +#if !defined(FULL_PREEMPTION) + if (td->td_priority <= PRI_MAX_ITHD) +#endif /* ! FULL_PREEMPTION */ + { + ipi_selected(best_pcpu->pc_cpumask, IPI_PREEMPT); + return; + } +#endif /* defined(IPI_PREEMPTION) && defined(PREEMPTION) */ + if (PCPU_GET(cpuid) != best_pcpu->pc_cpuid) { best_pcpu->pc_curthread->td_flags |= TDF_NEEDRESCHED; ipi_selected(best_pcpu->pc_cpumask, IPI_AST); @@ -445,14 +450,21 @@ maybe_preempt_in_ksegrp(struct thread *td) if (td->td_priority >= running_thread->td_priority) return; #ifdef PREEMPTION + +#if !defined(FULL_PREEMPTION) + if (td->td_priority > PRI_MAX_ITHD) { + running_thread->td_flags |= TDF_NEEDRESCHED; + } +#endif /* ! FULL_PREEMPTION */ + if (running_thread->td_critnest > 1) running_thread->td_owepreempt = 1; else mi_switch(SW_INVOL, NULL); -#else +#else /* PREEMPTION */ running_thread->td_flags |= TDF_NEEDRESCHED; -#endif +#endif /* PREEMPTION */ return; } #endif /* !SMP */ @@ -664,7 +676,7 @@ maybe_preempt(struct thread *td) TD_IS_INHIBITED(ctd) || td->td_kse->ke_state != KES_THREAD) return (0); #ifndef FULL_PREEMPTION - if (!(pri >= PRI_MIN_ITHD && pri <= PRI_MAX_ITHD) && + if ((pri > PRI_MAX_ITHD) && !(cpri >= PRI_MIN_IDLE)) return (0); #endif diff --git a/sys/kern/sched_4bsd.c b/sys/kern/sched_4bsd.c index e56d156d57f..20312772600 100644 --- a/sys/kern/sched_4bsd.c +++ b/sys/kern/sched_4bsd.c @@ -1083,14 +1083,53 @@ forward_wakeup(int cpunum) } #endif +#ifdef SMP +static void +kick_other_cpu(int pri,int cpuid); + + +static void +kick_other_cpu(int pri,int cpuid) +{ + struct pcpu * pcpu = pcpu_find(cpuid); + int cpri = pcpu->pc_curthread->td_priority; + + if (idle_cpus_mask & pcpu->pc_cpumask) { + forward_wakeups_delivered++; + ipi_selected(pcpu->pc_cpumask, IPI_AST); + return; + } + + if (pri >= cpri) + return; + +#if defined(IPI_PREEMPTION) && defined(PREEMPTION) + +#if !defined(FULL_PREEMPTION) + if (pri <= PRI_MAX_ITHD) +#endif /* ! FULL_PREEMPTION */ + { + ipi_selected(pcpu->pc_cpumask, IPI_PREEMPT); + return; + } +#endif /* defined(IPI_PREEMPTION) && defined(PREEMPTION) */ + + pcpu->pc_curthread->td_flags |= TDF_NEEDRESCHED; + ipi_selected( pcpu->pc_cpumask , IPI_AST); + return; +} + +#endif /* SMP */ + void sched_add(struct thread *td, int flags) +#ifdef SMP { + struct kse *ke; -#ifdef SMP int forwarded = 0; int cpu; -#endif + int single_cpu = 0; ke = td->td_kse; mtx_assert(&sched_lock, MA_OWNED); @@ -1103,25 +1142,76 @@ sched_add(struct thread *td, int flags) td, td->td_proc->p_comm, td->td_priority, curthread, curthread->td_proc->p_comm); -#ifdef SMP - if (KSE_CAN_MIGRATE(ke)) { + + if (td->td_pinned != 0) { + cpu = td->td_lastcpu; + ke->ke_runq = &runq_pcpu[cpu]; + single_cpu = 1; + CTR3(KTR_RUNQ, + "sched_add: Put kse:%p(td:%p) on cpu%d runq", ke, td, cpu); + } else if ((ke)->ke_flags & KEF_BOUND) { + /* Find CPU from bound runq */ + KASSERT(SKE_RUNQ_PCPU(ke),("sched_add: bound kse not on cpu runq")); + cpu = ke->ke_runq - &runq_pcpu[0]; + single_cpu = 1; + CTR3(KTR_RUNQ, + "sched_add: Put kse:%p(td:%p) on cpu%d runq", ke, td, cpu); + } else { CTR2(KTR_RUNQ, "sched_add: adding kse:%p (td:%p) to gbl runq", ke, td); cpu = NOCPU; ke->ke_runq = &runq; + } + + if ((single_cpu) && (cpu != PCPU_GET(cpuid))) { + kick_other_cpu(td->td_priority,cpu); } else { - if (!SKE_RUNQ_PCPU(ke)) - ke->ke_runq = &runq_pcpu[(cpu = PCPU_GET(cpuid))]; - else - cpu = td->td_lastcpu; - CTR3(KTR_RUNQ, - "sched_add: Put kse:%p(td:%p) on cpu%d runq", ke, td, cpu); + + if ( !single_cpu) { + cpumask_t me = PCPU_GET(cpumask); + int idle = idle_cpus_mask & me; + + if ( !idle && ((flags & SRQ_INTR) == 0) && + (idle_cpus_mask & ~(hlt_cpus_mask | me))) + forwarded = forward_wakeup(cpu); + + } + + if (!forwarded) { + if (((flags & SRQ_YIELDING) == 0) && maybe_preempt(td)) + return; + else + maybe_resched(td); + } } -#else + + if ((td->td_proc->p_flag & P_NOLOAD) == 0) + sched_load_add(); + SLOT_USE(td->td_ksegrp); + runq_add(ke->ke_runq, ke, flags); + ke->ke_state = KES_ONRUNQ; +} + + +#else /* SMP */ + +{ + struct kse *ke; + ke = td->td_kse; + mtx_assert(&sched_lock, MA_OWNED); + KASSERT(ke->ke_state != KES_ONRUNQ, + ("sched_add: kse %p (%s) already in run queue", ke, + ke->ke_proc->p_comm)); + KASSERT(ke->ke_proc->p_sflag & PS_INMEM, + ("sched_add: process swapped out")); + CTR5(KTR_SCHED, "sched_add: %p(%s) prio %d by %p(%s)", + td, td->td_proc->p_comm, td->td_priority, curthread, + curthread->td_proc->p_comm); + + CTR2(KTR_RUNQ, "sched_add: adding kse:%p (td:%p) to runq", ke, td); ke->ke_runq = &runq; -#endif /* * If we are yielding (on the way out anyhow) * or the thread being saved is US, @@ -1134,41 +1224,9 @@ sched_add(struct thread *td, int flags) * which also only happens when we are about to yield. */ if((flags & SRQ_YIELDING) == 0) { -#ifdef SMP - cpumask_t me = PCPU_GET(cpumask); - int idle = idle_cpus_mask & me; - /* - * Only try to kick off another CPU if - * the thread is unpinned - * or pinned to another cpu, - * and there are other available and idle CPUs. - * if we are idle, or it's an interrupt, - * then skip straight to preemption. - */ - if ( (! idle) && ((flags & SRQ_INTR) == 0) && - (idle_cpus_mask & ~(hlt_cpus_mask | me)) && - ( KSE_CAN_MIGRATE(ke) || - ke->ke_runq != &runq_pcpu[PCPU_GET(cpuid)])) { - forwarded = forward_wakeup(cpu); - } - /* - * If we failed to kick off another cpu, then look to - * see if we should preempt this CPU. Only allow this - * if it is not pinned or IS pinned to this CPU. - * If we are the idle thread, we also try do preempt. - * as it will be quicker and being idle, we won't - * lose in doing so.. - */ - if ((!forwarded) && - (ke->ke_runq == &runq || - ke->ke_runq == &runq_pcpu[PCPU_GET(cpuid)])) -#endif - - { - if (maybe_preempt(td)) - return; - } - } + if (maybe_preempt(td)) + return; + } if ((td->td_proc->p_flag & P_NOLOAD) == 0) sched_load_add(); SLOT_USE(td->td_ksegrp); @@ -1177,6 +1235,9 @@ sched_add(struct thread *td, int flags) maybe_resched(td); } +#endif /* SMP */ + + void sched_rem(struct thread *td) { -- 2.45.0