Re: can't boot with kern.mmxopt=1

[Matthew Dillon <dillon@xxxxxxxxxxxxxxxxxxxx>]

:It works fine when kern.mmxopt=0, but when it's set to 1,
:the kernel hangs after printing the line "SMP: AP CPU #1 Launched!".
:A dmesg when kern.mmxopt=0 can be found below. Please tell me if
:you need additional info, I don't know how to debug this...
:
:	-Richard

    Hmm.  That's very odd.  First try the below patch, which removes
    the 512 byte stack allocation I was using to hold the temporary FP
    state (I moved it into the per-cpu globaldata structure).

    Do a full rebuild of your kernel just to be sure, then report back.  If
    it still crashes try to break into DDB with ctl-alt-esc and get a
    'trace'.

					-Matt
					Matthew Dillon 
					<dillon@xxxxxxxxxxxxx>


Index: i386/i386/bcopy.s
===================================================================
RCS file: /cvs/src/sys/i386/i386/bcopy.s,v
retrieving revision 1.4
diff -u -r1.4 bcopy.s
--- i386/i386/bcopy.s	1 May 2004 03:38:36 -0000	1.4
+++ i386/i386/bcopy.s	2 May 2004 22:51:30 -0000
@@ -208,15 +208,16 @@
 	 *	    the application's saved fp state may already exist
 	 *	    in TD_SAVEFPU.
 	 *
-	 *	(3) We cannot allow the kernel overwrite the application's
-	 *	    FPU state with our own, so we allocate space on the
-	 *	    stack and create a new TD_SAVEFPU, saving the old
-	 *	    pointer.
+	 *	(3) We cannot allow the kernel to overwrite the application's
+	 *	    FPU state with our own, so we make sure the application's
+	 *	    FPU state has been saved and then point TD_SAVEFPU at a
+	 *	    temporary fpu save area in the globaldata structure.
 	 *	    
 	 *	(4) While we are using the FP unit, an interrupt may come
 	 *	    along and preempt us, causing our FP state to be saved.
-	 *	    We will fault/restore upon resumption.  Our FP state
-	 *	    will be saved on the stack.
+	 *	    We will fault/restore upon resumption.  (Our FP state
+	 *	    is saved/restored to a temporary save area in the
+	 *	    globaldata structure).
 	 *
 	 *	(5) To clean up we throw away our FP state and, zero out
 	 *	    npxthread to indicate that the application's FP state
@@ -247,22 +248,20 @@
 #define MMX_SAVE_BLOCK(missfunc)		\
 	cmpl	$2048,%ecx ;			\
 	jb	missfunc ;			\
-	btsl	$1,PCPU(kernel_fpu_lock) ;	\
+	movl	MYCPU,%eax ;			\
+	btsl	$1,GD_FPU_LOCK(%eax) ;		\
 	jc	missfunc ;			\
 	pushl	%ebx ;				\
-	pushl	%ebp ;				\
-	movl	%esp, %ebp ;			\
-	movl	PCPU(curthread),%edx ;		\
+	movl	GD_CURTHREAD(%eax),%edx ;	\
 	movl	TD_SAVEFPU(%edx),%ebx ;		\
-	subl	$512,%esp ;			\
-	andl	$0xfffffff0,%esp ;		\
 	addl	$TDPRI_CRIT,TD_PRI(%edx) ;	\
-	cmpl	%edx,PCPU(npxthread) ;		\
+	cmpl	%edx,GD_NPXTHREAD(%eax) ;	\
 	jne	100f ;				\
 	fxsave	0(%ebx) ;			\
 100: ;						\
-	movl	%esp,TD_SAVEFPU(%edx) ;		\
-	movl	%edx,PCPU(npxthread) ;		\
+	movl	%edx,GD_NPXTHREAD(%eax) ;	\
+	leal	GD_SAVEFPU(%eax),%eax ;		\
+	movl	%eax,TD_SAVEFPU(%edx) ;		\
 	clts ;					\
 	fninit ;				\
 	subl	$TDPRI_CRIT,TD_PRI(%edx) ;	\
@@ -283,21 +282,20 @@
 	MMX_RESTORE_BLOCK2
 
 #define MMX_RESTORE_BLOCK2			\
-	movl	PCPU(curthread),%edx ;		\
-	movl	$0,PCPU(npxthread) ;		\
+	movl	MYCPU,%ecx ;			\
+	movl	GD_CURTHREAD(%ecx),%edx ;	\
+	movl	$0,GD_NPXTHREAD(%ecx) ;		\
 	movl	%ebx,TD_SAVEFPU(%edx) ;		\
 	smsw	%ax ;				\
-	movl	%ebp,%esp ;			\
+	popl	%ebx ;				\
 	orb	$CR0_TS,%al ;			\
-	popl	%ebp ;				\
 	lmsw	%ax ;				\
-	popl	%ebx ;				\
-	movl	$0,PCPU(kernel_fpu_lock)
+	movl	$0,GD_FPU_LOCK(%ecx)
 
 	/*
 	 * xmm/mmx_onfault routine.  Restore the fpu state, skip the normal
 	 * return vector, and return to the caller's on-fault routine
-	 * (which was pushed on the callers stack just before he calle us)
+	 * (which was pushed on the callers stack just before he called us)
 	 */
 mmx_onfault:
 	MMX_RESTORE_BLOCK2
Index: i386/i386/genassym.c
===================================================================
RCS file: /cvs/src/sys/i386/i386/genassym.c,v
retrieving revision 1.38
diff -u -r1.38 genassym.c
--- i386/i386/genassym.c	30 Apr 2004 00:59:52 -0000	1.38
+++ i386/i386/genassym.c	2 May 2004 22:34:25 -0000
@@ -203,7 +203,8 @@
 ASSYM(GD_COMMON_TSSD, offsetof(struct mdglobaldata, gd_common_tssd));
 ASSYM(GD_TSS_GDT, offsetof(struct mdglobaldata, gd_tss_gdt));
 ASSYM(GD_NPXTHREAD, offsetof(struct mdglobaldata, gd_npxthread));
-ASSYM(GD_KERNEL_FPU_LOCK, offsetof(struct mdglobaldata, gd_kernel_fpu_lock));
+ASSYM(GD_FPU_LOCK, offsetof(struct mdglobaldata, gd_fpu_lock));
+ASSYM(GD_SAVEFPU, offsetof(struct mdglobaldata, gd_savefpu));
 ASSYM(GD_OTHER_CPUS, offsetof(struct mdglobaldata, gd_other_cpus));
 ASSYM(GD_SS_EFLAGS, offsetof(struct mdglobaldata, gd_ss_eflags));
 ASSYM(GD_CMAP1, offsetof(struct mdglobaldata, gd_CMAP1));
Index: i386/i386/globals.s
===================================================================
RCS file: /cvs/src/sys/i386/i386/globals.s,v
retrieving revision 1.21
diff -u -r1.21 globals.s
--- i386/i386/globals.s	29 Apr 2004 17:24:58 -0000	1.21
+++ i386/i386/globals.s	2 May 2004 22:34:52 -0000
@@ -69,8 +69,9 @@
 	.globl	gd_currentldt
 	.set	gd_currentldt,globaldata + GD_CURRENTLDT
 
-	.globl	gd_kernel_fpu_lock
-	.set	gd_kernel_fpu_lock, globaldata + GD_KERNEL_FPU_LOCK
+	.globl	gd_fpu_lock, gd_savefpu
+	.set	gd_fpu_lock, globaldata + GD_FPU_LOCK
+	.set	gd_savefpu, globaldata + GD_SAVEFPU
 
 	/*
 	 * The BSP version of these get setup in locore.s and pmap.c, while
Index: i386/i386/pmap.c
===================================================================
RCS file: /cvs/src/sys/i386/i386/pmap.c,v
retrieving revision 1.36
diff -u -r1.36 pmap.c
--- i386/i386/pmap.c	1 May 2004 18:16:41 -0000	1.36
+++ i386/i386/pmap.c	1 May 2004 18:37:34 -0000
@@ -827,16 +827,16 @@
 		pte = (unsigned *)vtopte(va);
 		pteval = VM_PAGE_TO_PHYS(*m) | PG_A | PG_RW | PG_V | pgeflag;
 		if (*pte != pteval) {
-			*mask = cmask;
+			*mask = 0;
 			*pte = pteval;
 			cpu_invlpg((void *)va);
 		} else if ((*mask & cmask) == 0) {
-			*mask |= cmask;
 			cpu_invlpg((void *)va);
 		}
 		va += PAGE_SIZE;
 		m++;
 	}
+	*mask |= cmask;
 }
 
 /*
Index: i386/i386/swtch.s
===================================================================
RCS file: /cvs/src/sys/i386/i386/swtch.s,v
retrieving revision 1.33
diff -u -r1.33 swtch.s
--- i386/i386/swtch.s	30 Apr 2004 00:59:52 -0000	1.33
+++ i386/i386/swtch.s	30 Apr 2004 22:47:06 -0000
@@ -436,12 +436,14 @@
 	testl	%eax,%eax
 	je	1f
 
-	pushl	%ecx
-	movl	TD_SAVEFPU(%eax),%eax
+	pushl	%ecx			/* target pcb */
+	movl	TD_SAVEFPU(%eax),%eax	/* originating savefpu area */
 	pushl	%eax
+
 	pushl	%eax
 	call	npxsave
 	addl	$4,%esp
+
 	popl	%eax
 	popl	%ecx
 
Index: i386/include/asmacros.h
===================================================================
RCS file: /cvs/src/sys/i386/include/asmacros.h,v
retrieving revision 1.6
diff -u -r1.6 asmacros.h
--- i386/include/asmacros.h	4 Dec 2003 20:09:33 -0000	1.6
+++ i386/include/asmacros.h	2 May 2004 22:37:22 -0000
@@ -45,6 +45,7 @@
  * Access to a per-cpu data element
  */
 #define PCPU(x) %fs:gd_ ## x
+#define MYCPU	%fs:0
 
 #endif
 
Index: i386/include/globaldata.h
===================================================================
RCS file: /cvs/src/sys/i386/include/globaldata.h,v
retrieving revision 1.24
diff -u -r1.24 globaldata.h
--- i386/include/globaldata.h	29 Apr 2004 17:25:00 -0000	1.24
+++ i386/include/globaldata.h	2 May 2004 22:47:08 -0000
@@ -48,6 +48,9 @@
 #ifndef _MACHINE_TSS_H_
 #include "tss.h"	/* struct i386tss */
 #endif
+#ifndef _MACHINE_NPX_H_
+#include "npx.h"
+#endif
 
 /*
  * Note on interrupt control.  Pending interrupts not yet dispatched are
@@ -67,7 +70,8 @@
 	struct segment_descriptor *gd_tss_gdt;
 	struct thread   *gd_npxthread;
 	struct i386tss  gd_common_tss;
-	int		gd_kernel_fpu_lock;	/* fast bcopy/zero cpu lock */
+	union savefpu	gd_savefpu;	/* fast bcopy/zero temp fpu save area */
+	int		gd_fpu_lock;	/* fast bcopy/zero cpu lock */
 	int		gd_fpending;	/* fast interrupt pending */
 	int		gd_ipending;	/* normal interrupt pending */
 	int		gd_idelayed;	/* delayed software ints */