NetBSD vmlockingブランチにおける割り込みスレッドの実装

/*
 * softintr()
 *
 * Switch to the LWP assigned to handle interrupts from the given
 * source.  We borrow the VM context from the interrupted LWP.
 *
 * On entry:
 *
 *	%eax		intrsource
 *	%esi		address to return to
 */
IDTVEC(softintr)
	pushl	$_C_LABEL(softintr_ret)	/* set up struct switchframe */
	pushl	%ebx
	pushl	%esi
	pushl	%edi
	movl	$IPL_HIGH,CPUVAR(ILEVEL)
	movl	CPUVAR(CURLWP),%esi
	movl	IS_LWP(%eax),%edi	/* switch to handler LWP */
	movl	%edi,CPUVAR(CURLWP)
	movl	L_ADDR(%edi),%edx
	movl	L_ADDR(%esi),%ecx
	movl	%esp,PCB_ESP(%ecx)
	movl	%ebp,PCB_EBP(%ecx)
	movl	PCB_ESP0(%edx),%esp	/* onto new stack */
	sti
	pushl	IS_MAXLEVEL(%eax)	/* ipl to run at */
	pushl	%esi
	call	_C_LABEL(softint_dispatch)/* run handlers */
	addl	$8,%esp

/*
 * softint_dispatch:
 *
 *	Entry point from machine-dependent code.
 */
void
softint_dispatch(lwp_t *pinned, int s)
{
    struct timeval now;
    softint_t *si;
 	u_int timing;
 	lwp_t *l;
 
 	l = curlwp;
 	si = l->l_private;
 
 	/*
 	 * Note the interrupted LWP, and mark the current LWP as running
 	 * before proceeding.  Although this must as a rule be done with
 	 * the LWP locked, at this point no external agents will want to
 	 * modify the interrupt LWP's state.
 	 */
 	timing = (softint_timing ? LW_TIMEINTR : 0);
 	l->l_switchto = pinned;
 	l->l_stat = LSONPROC;
 	l->l_flag |= (LW_RUNNING | timing);
 
 	/*
 	 * Dispatch the interrupt.  If softints are being timed, charge
	 * for it.
 	 */
 	if (timing)
 		microtime(&l->l_stime);
 	softint_execute(si, l, s);
 	if (timing) {
 		microtime(&now);
 		updatertime(l, &now);
 		l->l_flag &= ~LW_TIMEINTR;
 	}

 /*
  * The machine independent parts of context switch.
  *
  * Returns 1 if another LWP was actually run.
  */
 int
 mi_switch(lwp_t *l)
 {
struct schedstate_percpu *spc;
struct lwp *newl;
int retval, oldspl;
struct timeval tv;
bool returning;

KASSERT(lwp_locked(l, NULL));
LOCKDEBUG_BARRIER(l->l_mutex, 1);

 #ifdef KSTACK_CHECK_MAGIC
kstack_check_magic(l);
 #endif

microtime(&tv);

/*
 * It's safe to read the per CPU schedstate unlocked here, as all we
 * are after is the run time and that's guarenteed to have been last
 * updated by this CPU.
 */
KDASSERT(l->l_cpu == curcpu());

/*
 * Process is about to yield the CPU; clear the appropriate
 * scheduling flags.
 */
spc = &l->l_cpu->ci_schedstate;
returning = false;
newl = NULL;

/*
 * If we have been asked to switch to a specific LWP, then there
 * is no need to inspect the run queues.  If a soft interrupt is
 * blocking, then return to the interrupted thread without adjusting
 * VM context or its start time: neither have been changed in order
 * to take the interrupt.
 */
if (l->l_switchto != NULL) {
	if ((l->l_flag & LW_INTR) != 0) {
		returning = true;
		softint_block.ev_count++;
		if ((l->l_flag & LW_TIMEINTR) != 0)
			updatertime(l, &tv);
	}
	newl = l->l_switchto;
	l->l_switchto = NULL;
}

if (!returning) {
	/* Count time spent in current system call */
	SYSCALL_TIME_SLEEP(l);

	/*
	 * XXXSMP If we are using h/w performance counters,
	 * save context.
	 */
 #if PERFCTRS
	if (PMC_ENABLED(l->l_proc)) {
		pmc_save_context(l->l_proc);
	}
 #endif
	updatertime(l, &tv);
}

/*
 * If on the CPU and we have gotten this far, then we must yield.
 */
mutex_spin_enter(spc->spc_mutex);
KASSERT(l->l_stat != LSRUN);
if (l->l_stat == LSONPROC) {
	KASSERT(lwp_locked(l, &spc->spc_lwplock));
	if ((l->l_flag & LW_IDLE) == 0) {
		l->l_stat = LSRUN;
		lwp_setlock(l, spc->spc_mutex);
		sched_enqueue(l, true);
	} else
		l->l_stat = LSIDL;
}

/*
 * Let sched_nextlwp() select the LWP to run the CPU next. 
 * If no LWP is runnable, switch to the idle LWP.
 */
if (newl == NULL) {
	newl = sched_nextlwp();
	if (newl != NULL) {
		sched_dequeue(newl);
		KASSERT(lwp_locked(newl, spc->spc_mutex));
		newl->l_stat = LSONPROC;
		newl->l_cpu = l->l_cpu;
		newl->l_flag |= LW_RUNNING;
		lwp_setlock(newl, &spc->spc_lwplock);
	} else {
		newl = l->l_cpu->ci_data.cpu_idlelwp;
		newl->l_stat = LSONPROC;
		newl->l_flag |= LW_RUNNING;
	}
	spc->spc_curpriority = newl->l_usrpri;
	newl->l_priority = newl->l_usrpri;
	cpu_did_resched();
	spc->spc_flags &= ~SPCF_SWITCHCLEAR;
}

/* Update the new LWP's start time while it is still locked. */
if (!returning)
	newl->l_stime = tv;

if (l != newl) {
	struct lwp *prevlwp;

	/*
	 * If the old LWP has been moved to a run queue above,
	 * drop the general purpose LWP lock: it's now locked
	 * by the scheduler lock.
	 *
	 * Otherwise, drop the scheduler lock.  We're done with
	 * the run queues for now.
	 */
	if (l->l_mutex == spc->spc_mutex) {
		mutex_spin_exit(&spc->spc_lwplock);
	} else {
		mutex_spin_exit(spc->spc_mutex);
	}

	/* Unlocked, but for statistics only. */
	uvmexp.swtch++;

	/*
	 * Save old VM context, unless a soft interrupt
	 * handler is blocking.
	 */
	if (!returning)
		pmap_deactivate(l);

	/* Switch to the new LWP.. */
	l->l_ncsw++;
	l->l_flag &= ~LW_RUNNING;
	oldspl = MUTEX_SPIN_OLDSPL(l->l_cpu);
	prevlwp = cpu_switchto(l, newl, returning);

 /*
  * struct lwp *cpu_switchto(struct lwp *oldlwp, struct newlwp,
  *			    bool returning)
  *
  *	1. if (oldlwp != NULL), save its context.
  *	2. then, restore context of newlwp.
  *
  * Note that the stack frame layout is known to "struct switchframe" in
  * <machine/frame.h> and to the code in cpu_lwp_fork() which initializes
  * it for a new lwp.
  */
 ENTRY(cpu_switchto)
pushl	%ebx
pushl	%esi
pushl	%edi

movl	16(%esp),%esi		# oldlwp
movl	20(%esp),%edi		# newlwp
movl	24(%esp),%edx		# returning
testl	%esi,%esi
jz	1f

/* Save old context. */
movl	L_ADDR(%esi),%eax
movl	%esp,PCB_ESP(%eax)
movl	%ebp,PCB_EBP(%eax)

/* Switch to newlwp's stack. */
 1:	movl	L_ADDR(%edi),%ebx
movl	PCB_EBP(%ebx),%ebp
movl	PCB_ESP(%ebx),%esp

/* Set curlwp. */
movl	%edi,CPUVAR(CURLWP)

/* Skip the rest if returning to a pinned LWP. */
testl	%edx,%edx
jnz	4f

/* Switch TSS.  Reset "task busy" flag before loading. */
movl	%cr3,%eax
movl	%eax,PCB_CR3(%ebx)	# for TSS gates
movl	CPUVAR(GDT),%eax
movl	L_MD_TSS_SEL(%edi),%edx
andl	$~0x0200,4(%eax,%edx, 1)
ltr	%dx

/* Don't bother with the rest if switching to a system process. */
testl	$LW_SYSTEM,L_FLAG(%edi)
jnz	4f

/* Is this process using RAS (restartable atomic sequences)? */
movl	L_PROC(%edi),%eax
cmpl	$0,P_RASLIST(%eax)
jne	5f

/*
 * Restore cr0 (including FPU state).  Raise the IPL to IPL_IPI.
 * FPU IPIs can alter the LWP's saved cr0.  Dropping the priority
 * is deferred until mi_switch(), when cpu_switchto() returns.
 */
 2:	movl	$IPL_IPI,CPUVAR(ILEVEL)
movl	PCB_CR0(%ebx),%ecx
movl	%cr0,%edx

/*
 * If our floating point registers are on a different CPU,
 * set CR0_TS so we'll trap rather than reuse bogus state.
 */
movl	PCB_FPCPU(%ebx),%eax
cmpl	CPUVAR(SELF),%eax
je	3f
orl	$CR0_TS,%ecx

/* Reloading CR0 is very expensive - avoid if possible. */
 3:	cmpl	%edx,%ecx
je	4f
movl	%ecx,%cr0

/* Return to the new LWP, returning 'oldlwp' in %eax. */
 4:	movl	%esi,%eax
popl	%edi
popl	%esi
popl	%ebx
ret

/* Check for restartable atomic sequences (RAS). */
 5:	movl	L_MD_REGS(%edi),%ecx
pushl	TF_EIP(%ecx)
pushl	%eax
call	_C_LABEL(ras_lookup)
addl	$8,%esp
cmpl	$-1,%eax
je	2b
movl	L_MD_REGS(%edi),%ecx
movl	%eax,TF_EIP(%ecx)
jmp	2b

  /*
  * softintr_ret()
  *
  * Trampoline function that gets returned to by cpu_switchto() when
  * an interrupt handler blocks.  On entry:
  *
  *	%eax		prevlwp from cpu_switchto()
  */
 NENTRY(softintr_ret)
movl	CPUVAR(ILEVEL),%edx
pushl	L_MUTEX(%eax)		/* %eax from cpu_switchto */
movl	%edx,CPUVAR(MTX_OLDSPL)	/* preserve priority until return */
call	_C_LABEL(mutex_spin_exit)/* unlock the old LWP */
addl	$4,%esp
cli
jmp	*%esi			/* back to splx/doreti */

	prevlwp = cpu_switchto(l, newl, returning);

	/*
	 * .. we have switched away and are now back so we must
	 * be the new curlwp.  prevlwp is who we replaced.
	 */
	curlwp = l;
	if (prevlwp != NULL) {
		curcpu()->ci_mtx_oldspl = oldspl;
		lwp_unlock(prevlwp);
	} else {
		splx(oldspl);
	}

	/* Restore VM context. */
	pmap_activate(l);
	retval = 1;
} else {
	/* Nothing to do - just unlock and return. */
	mutex_spin_exit(spc->spc_mutex);
	lwp_unlock(l);
	retval = 0;
}

KASSERT(l == curlwp);
KASSERT(l->l_stat == LSONPROC);
KASSERT(l->l_cpu == curcpu());

/*
 * XXXSMP If we are using h/w performance counters, restore context.
 */
 #if PERFCTRS
if (PMC_ENABLED(l->l_proc)) {
	pmc_restore_context(l->l_proc);
}
 #endif

SYSCALL_TIME_WAKEUP(l);
LOCKDEBUG_BARRIER(NULL, 1);

return retval;
 }

if (timing)
	microtime(&l->l_stime);
softint_execute(si, l, s);
if (timing) {
	microtime(&now);
	updatertime(l, &now);
	l->l_flag &= ~LW_TIMEINTR;
}

	/*
 * If we blocked while handling the interrupt, the pinned LWP is
 * gone so switch to the idle LWP.  It will select a new LWP to
 * run.
 *
 * We must drop the priority level as switching at IPL_HIGH could
 * deadlock the system.  We have already set si->si_active = 0,
 * which means another interrupt at this level can be triggered. 
 * That's not be a problem: we are lowering to level 's' which will
 * prevent softint_dispatch() from being reentered at level 's',
 * until the priority is finally dropped to IPL_NONE on entry to
 * the idle loop.
 */
l->l_stat = LSIDL;
if (l->l_switchto == NULL) {
	splx(s);
	lwp_exit_switchaway(l);
	/* NOTREACHED */
}
l->l_switchto = NULL;
l->l_flag &= ~LW_RUNNING;
 }

pushl	IS_MAXLEVEL(%eax)	/* ipl to run at */
pushl	%esi
call	_C_LABEL(softint_dispatch)/* run handlers */
addl	$8,%esp
movl	L_ADDR(%esi),%ecx
movl	PCB_ESP(%ecx),%esp
movl	%esi,CPUVAR(CURLWP)
popl	%edi			/* unwind switchframe */
popl	%esi
addl	$8,%esp
cli
jmp	*%esi			/* back to splx/doreti */