[patch] hwpmc [8/13]

[Aggelos Economopoulos <aoiko@xxxxxxxxxxxxxx>]

Sprinkle hooks and support code all over the kernel. Needed for
proper operation.

Obtained-from: FreeBSD

Index: platform/pc32/apic/apic_abi.c
===================================================================
retrieving revision 1.12
diff -u -p -r1.12 apic_abi.c
--- platform/pc32/apic/apic_abi.c
+++ platform/pc32/apic/apic_abi.c
@@ -40,6 +40,8 @@
  * $DragonFly: src/sys/platform/pc32/apic/apic_abi.c,v 1.12 2007/04/30 16:45:55 dillon Exp $
  */
 
+#include "opt_hwpmc_hooks.h"
+
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/kernel.h>
@@ -240,6 +242,11 @@ apic_finalize(void)
     temp = lapic.lvt_lint1;
     temp &= ~APIC_LVT_MASKED;
     lapic.lvt_lint1 = temp;
+#ifdef	HWPMC_HOOKS
+	/* Program the PMC LVT entry if present. */
+    if (((lapic->version & APIC_VER_MAXLVT) >> MAXLVTSHIFT) >= LVT_PMC)
+	lapic->lvt_pcint = lvt_mode(la, LVT_PMC, lapic->lvt_pcint);
+#endif
 
     if (bootverbose)
 	apic_dump("bsp_apic_configure()");
Index: platform/pc32/apic/mpapic.c
===================================================================
retrieving revision 1.21
diff -u -p -r1.21 mpapic.c
--- platform/pc32/apic/mpapic.c
+++ platform/pc32/apic/mpapic.c
@@ -26,6 +26,8 @@
  * $DragonFly: src/sys/platform/pc32/apic/mpapic.c,v 1.21 2007/04/30 16:45:55 dillon Exp $
  */
 
+#include "opt_hwpmc_hooks.h"
+
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <machine/globaldata.h>
@@ -82,6 +84,11 @@ apic_initialize(void)
 		  APIC_LVT_POLARITY_MASK | APIC_LVT_DM_MASK);
 	temp |= APIC_LVT_MASKED | APIC_LVT_DM_NMI;
 	lapic.lvt_lint1 = temp;
+#ifdef	HWPMC_HOOKS
+	/* Program the PMC LVT entry if present. */
+	if (((lapic->version & APIC_VER_MAXLVT) >> MAXLVTSHIFT) >= LVT_PMC)
+		lapic->lvt_pcint = lvt_mode(la, LVT_PMC, lapic->lvt_pcint);
+#endif
 
 	/*
 	 * Mask the apic error interrupt, apic performance counter
@@ -151,7 +158,6 @@ apic_dump(char* str)
 		lapic.lvt_lint0, lapic.lvt_lint1, lapic.tpr, lapic.svr);
 }
 
-
 #if defined(APIC_IO)
 
 /*
Index: platform/pc32/i386/trap.c
===================================================================
retrieving revision 1.107
diff -u -p -r1.107 trap.c
--- platform/pc32/i386/trap.c
+++ platform/pc32/i386/trap.c
@@ -48,6 +48,7 @@
 
 #include "opt_cpu.h"
 #include "opt_ddb.h"
+#include "opt_hwpmc_hooks.h"
 #include "opt_ktrace.h"
 #include "opt_clock.h"
 #include "opt_trap.h"
@@ -69,6 +70,9 @@
 #ifdef KTRACE
 #include <sys/ktrace.h>
 #endif
+#ifdef HWPMC_HOOKS
+#include <sys/pmckern.h>
+#endif
 #include <sys/upcall.h>
 #include <sys/vkernel.h>
 #include <sys/sysproto.h>
@@ -408,6 +412,20 @@ trap(struct trapframe *frame)
 	}
 #endif
 
+#ifdef	HWPMC_HOOKS
+	/*
+	 * CPU PMCs interrupt using an NMI so we check for that first.
+	 * If the HWPMC module is active, 'pmc_hook' will point to
+	 * the function to be called.  A return value of '1' from the
+	 * hook means that the NMI was handled by it and that we can
+	 * return immediately.
+	 */
+	if (type == T_NMI && pmc_intr &&
+	    (*pmc_intr)(mycpu->gd_cpuid, (uintptr_t) frame->tf_eip,
+		TRAPF_USERMODE(frame)))
+	    goto out;
+#endif
+
 	eva = 0;
 	++gd->gd_trap_nesting_level;
 	if (frame->tf_trapno == T_PAGEFLT) {
Index: kern/kern_exec.c
===================================================================
retrieving revision 1.61
diff -u -p -r1.61 kern_exec.c
--- kern/kern_exec.c
+++ kern/kern_exec.c
@@ -27,6 +27,8 @@
  * $DragonFly: src/sys/kern/kern_exec.c,v 1.61 2007/07/30 17:41:23 pavalos Exp $
  */
 
+#include "opt_hwpmc_hooks.h"
+
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/sysproto.h>
@@ -66,6 +68,10 @@
 #include <vm/vm_object.h>
 #include <vm/vm_pager.h>
 
+#ifdef	HWPMC_HOOKS
+#include <sys/pmckern.h>
+#endif
+
 #include <sys/user.h>
 #include <sys/reg.h>
 
@@ -177,6 +183,10 @@ kern_execve(struct nlookupdata *nd, stru
 	struct image_params image_params, *imgp;
 	struct vattr attr;
 	int (*img_first) (struct image_params *);
+	int credential_changing;
+#ifdef HWPMC_HOOKS
+	struct pmckern_procexec pe;
+#endif
 
 	if (debug_execve_args) {
 		kprintf("%s()\n", __func__);
@@ -378,6 +388,7 @@ interpret:
 		wakeup((caddr_t)p->p_pptr);
 	}
 
+	credential_changing = 0;
 	/*
 	 * Implement image setuid/setgid.
 	 *
@@ -393,6 +404,7 @@ interpret:
 		 * root.  Record any set-id flags first to make sure that
 		 * we do not regain any tracing during a possible block.
 		 */
+		credential_changing = !0;
 		setsugid();
 		if (p->p_tracenode && suser(td) != 0) {
 			ktrdestroy(&p->p_tracenode);
@@ -460,6 +472,23 @@ interpret:
 	/* clear "fork but no exec" flag, as we _are_ execing */
 	p->p_acflag &= ~AFORK;
 
+#ifdef	HWPMC_HOOKS
+	/*
+	 * Check if system-wide sampling is in effect or if the
+	 * current process is using PMCs.  If so, do exec() time
+	 * processing.  This processing needs to happen AFTER the
+	 * P_INEXEC flag is cleared.
+	 *
+	 * The proc lock needs to be released before taking the PMC
+	 * SX.
+	 */
+	if (PMC_SYSTEM_SAMPLING_ACTIVE() || PMC_PROC_IS_USING_PMCS(p)) {
+		pe.pm_credentialschanged = credential_changing;
+		pe.pm_entryaddr = imgp->entry_addr;
+
+		PMC_CALL_HOOK_X(td, PMC_FN_PROCESS_EXEC, (void *) &pe);
+	}
+#endif
 	/* Set values passed into the program in registers. */
 	exec_setregs(imgp->entry_addr, (u_long)(uintptr_t)stack_base,
 	    imgp->ps_strings);
Index: kern/kern_linker.c
===================================================================
retrieving revision 1.38
diff -u -p -r1.38 kern_linker.c
--- kern/kern_linker.c
+++ kern/kern_linker.c
@@ -28,6 +28,7 @@
  */
 
 #include "opt_ddb.h"
+#include "opt_hwpmc_hooks.h"
 
 #include <sys/param.h>
 #include <sys/kernel.h>
@@ -51,6 +52,10 @@
 #include <dlfcn.h>
 #endif
 
+#ifdef HWPMC_HOOKS
+#include <sys/pmckern.h>
+#endif
+
 #ifdef KLD_DEBUG
 int kld_debug = 0;
 #endif
@@ -359,6 +364,20 @@ linker_find_file_by_id(int fileid)
     return lf;
 }
 
+static int
+linker_file_foreach(int (*predicate)(linker_file_t, void*), void *context)
+{
+	linker_file_t lf;
+	int retval = 0;
+
+	TAILQ_FOREACH(lf, &linker_files, link) {
+		retval = predicate(lf, context);
+		if (retval != 0)
+			break;
+	}
+	return (retval);
+}
+
 linker_file_t
 linker_make_file(const char* pathname, void* priv, struct linker_file_ops* ops)
 {
@@ -713,6 +732,9 @@ linker_ddb_symbol_values(c_linker_sym_t 
 int
 sys_kldload(struct kldload_args *uap)
 {
+#ifdef HWPMC_HOOKS
+    struct pmckern_map_in pkm;
+#endif
     struct thread *td = curthread;
     char* filename = NULL, *modulename;
     linker_file_t lf;
@@ -743,6 +765,11 @@ sys_kldload(struct kldload_args *uap)
 
     if ((error = linker_load_file(filename, &lf)) != 0)
 	goto out;
+#ifdef HWPMC_HOOKS
+    pkm.pm_file = lf->filename;
+    pkm.pm_address = (uintptr_t) lf->address;
+    PMC_CALL_HOOK(td, PMC_FN_KLD_LOAD, (void *) &pkm);
+#endif
 
     lf->userrefs++;
     uap->sysmsg_result = lf->id;
@@ -756,6 +783,9 @@ out:
 int
 sys_kldunload(struct kldunload_args *uap)
 {
+#ifdef HWPMC_HOOKS
+    struct pmckern_map_out pkm;
+#endif
     struct thread *td = curthread;
     linker_file_t lf;
     int error = 0;
@@ -774,6 +804,11 @@ sys_kldunload(struct kldunload_args *uap
 	    error = EBUSY;
 	    goto out;
 	}
+#ifdef HWPMC_HOOKS
+	/* Save data needed by hwpmc(4) before unloading. */
+	pkm.pm_address = (uintptr_t) lf->address;
+	pkm.pm_size = lf->size;
+#endif
 	lf->userrefs--;
 	error = linker_file_unload(lf);
 	if (error)
@@ -782,6 +817,10 @@ sys_kldunload(struct kldunload_args *uap
 	error = ENOENT;
 
 out:
+#ifdef HWPMC_HOOKS
+    if (error == 0)
+	PMC_CALL_HOOK(td, PMC_FN_KLD_UNLOAD, (void *) &pkm);
+#endif
     return error;
 }
 
@@ -1169,3 +1208,63 @@ linker_search_path(const char *name)
     }
     return(NULL);
 }
+
+#ifdef HWPMC_HOOKS
+
+struct hwpmc_context {
+	int	nobjects;
+	int	nmappings;
+	struct pmckern_map_in *kobase;
+};
+
+static int
+linker_hwpmc_list_object(linker_file_t lf, void *arg)
+{
+	struct hwpmc_context *hc;
+
+	hc = arg;
+
+	/* If we run out of mappings, fail. */
+	if (hc->nobjects >= hc->nmappings)
+		return (1);
+
+	/* Save the info for this linker file. */
+	hc->kobase[hc->nobjects].pm_file = lf->filename;
+	hc->kobase[hc->nobjects].pm_address = (uintptr_t)lf->address;
+	hc->nobjects++;
+	return (0);
+}
+
+/*
+ * Inform hwpmc about the set of kernel modules currently loaded.
+ */
+void *
+linker_hwpmc_list_objects(void)
+{
+	struct hwpmc_context hc;
+
+	hc.nmappings = 15;	/* a reasonable default */
+
+ retry:
+	/* allocate nmappings+1 entries */
+	MALLOC(hc.kobase, struct pmckern_map_in *,
+	    (hc.nmappings + 1) * sizeof(struct pmckern_map_in), M_LINKER,
+	    M_WAITOK | M_ZERO);
+
+	hc.nobjects = 0;
+	if (linker_file_foreach(linker_hwpmc_list_object, &hc) != 0) {
+		hc.nmappings = hc.nobjects;
+		FREE(hc.kobase, M_LINKER);
+		goto retry;
+	}
+
+	KASSERT(hc.nobjects > 0, ("linker_hpwmc_list_objects: no kernel "
+		"objects?"));
+
+	/* The last entry of the malloced area comprises of all zeros. */
+	KASSERT(hc.kobase[hc.nobjects].pm_file == NULL,
+	    ("linker_hwpmc_list_objects: last object not NULL"));
+
+	return ((void *)hc.kobase);
+}
+#endif
Index: kern/kern_timeout.c
===================================================================
retrieving revision 1.26
diff -u -p -r1.26 kern_timeout.c
--- kern/kern_timeout.c
+++ kern/kern_timeout.c
@@ -99,6 +99,7 @@
  * The per-cpu augmentation was done by Matthew Dillon.
  */
 
+#include "opt_hwpmc_hooks.h"
 #include "opt_ddb.h"
 
 #include <sys/param.h>
@@ -110,6 +111,10 @@
 #include <sys/thread2.h>
 #include <ddb/ddb.h>
 
+#ifdef HWPMC_HOOKS
+#include <sys/pmckern.h>
+#endif
+
 #ifndef MAX_SOFTCLOCK_STEPS
 #define MAX_SOFTCLOCK_STEPS 100 /* Maximum allowed value of steps. */
 #endif
@@ -217,6 +222,13 @@ hardclock_softtick(globaldata_t gd)
 	++sc->curticks;
 	if (sc->isrunning)
 		return;
+#ifdef	HWPMC_HOOKS
+	/*
+	 * XXX: does hook sleep? is that ok here? -- agg
+	 */
+	if (PMC_CPU_HAS_SAMPLES(mycpu->gd_cpuid))
+		PMC_CALL_HOOK_UNLOCKED(curthread, PMC_FN_DO_SAMPLES, NULL);
+#endif
 	if (sc->softticks == sc->curticks) {
 		/*
 		 * in sync, only wakeup the thread if there is something to
Index: kern/lwkt_thread.c
===================================================================
retrieving revision 1.109
diff -u -p -r1.109 lwkt_thread.c
--- kern/lwkt_thread.c
+++ kern/lwkt_thread.c
@@ -43,6 +43,8 @@
 
 #ifdef _KERNEL
 
+#include "opt_hwpmc_hooks.h"
+
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/kernel.h>
@@ -93,6 +95,10 @@
 
 #endif
 
+#ifdef HWPMC_HOOKS
+#include <sys/pmckern.h>
+#endif
+
 static int untimely_switch = 0;
 #ifdef	INVARIANTS
 static int panic_on_cscount = 0;
@@ -749,7 +755,15 @@ using_idle_thread:
 #endif
     if (td != ntd) {
 	++switch_count;
+#ifdef	HWPMC_HOOKS
+	if (td->td_proc && PMC_PROC_IS_USING_PMCS(td->td_proc))
+	    PMC_SWITCH_CONTEXT(td, PMC_FN_CSW_OUT);
+#endif
 	td->td_switch(ntd);
+#ifdef	HWPMC_HOOKS
+	if (td->td_proc && PMC_PROC_IS_USING_PMCS(td->td_proc))
+	    PMC_SWITCH_CONTEXT(td, PMC_FN_CSW_IN);
+#endif
     }
     /* NOTE: current cpu may have changed after switch */
     crit_exit_quick(td);
Index: vm/vm_mmap.c
===================================================================
retrieving revision 1.39
diff -u -p -r1.39 vm_mmap.c
--- vm/vm_mmap.c
+++ vm/vm_mmap.c
@@ -46,6 +46,8 @@
  * Mapped file (mmap) interface to VM
  */
 
+#include "opt_hwpmc_hooks.h"
+
 #include <sys/param.h>
 #include <sys/kernel.h>
 #include <sys/systm.h>
@@ -77,6 +79,10 @@
 #include <vm/vm_page.h>
 #include <vm/vm_kern.h>
 
+#ifdef HWPMC_HOOKS
+#include <sys/pmckern.h>
+#endif
+
 #include <sys/file2.h>
 #include <sys/thread2.h>
 
@@ -148,6 +154,10 @@ int
 kern_mmap(struct vmspace *vms, caddr_t uaddr, size_t ulen,
 	  int uprot, int uflags, int fd, off_t upos, void **res)
 {
+#ifdef HWPMC_HOOKS
+	struct pmckern_map_in pkm;
+	objtype_t handle_type;
+#endif
 	struct thread *td = curthread;
  	struct proc *p = td->td_proc;
 	struct file *fp = NULL;
@@ -248,6 +258,7 @@ kern_mmap(struct vmspace *vms, caddr_t u
 		 * Mapping blank space is trivial.
 		 */
 		handle = NULL;
+		handle_type = OBJT_DEFAULT;
 		maxprot = VM_PROT_ALL;
 		pos = 0;
 	} else {
@@ -312,6 +323,7 @@ kern_mmap(struct vmspace *vms, caddr_t u
 		 */
 		if (vp->v_type == VCHR && iszerodev(vp->v_rdev)) {
 			handle = NULL;
+			handle_type = OBJT_DEFAULT;
 			maxprot = VM_PROT_ALL;
 			flags |= MAP_ANON;
 			pos = 0;
@@ -385,6 +397,7 @@ kern_mmap(struct vmspace *vms, caddr_t u
 				maxprot |= VM_PROT_WRITE;
 			}
 			handle = (void *)vp;
+			handle_type = OBJT_VNODE;
 		}
 	}
 
@@ -401,6 +414,15 @@ kern_mmap(struct vmspace *vms, caddr_t u
 
 	error = vm_mmap(&vms->vm_map, &addr, size, prot, maxprot,
 	    flags, handle, pos);
+#ifdef HWPMC_HOOKS
+	/* inform hwpmc(4) if an executable is being mapped */
+	if (error == 0 && handle_type == OBJT_VNODE &&
+	    (prot & PROT_EXEC)) {
+		pkm.pm_file = handle;
+		pkm.pm_address = (uintptr_t) addr;
+		PMC_CALL_HOOK(td, PMC_FN_MMAP, (void *) &pkm);
+	}
+#endif
 	if (error == 0)
 		*res = (void *)(addr + pageoff);
 done:
@@ -495,6 +517,10 @@ sys_msync(struct msync_args *uap)
 int
 sys_munmap(struct munmap_args *uap)
 {
+#ifdef HWPMC_HOOKS
+	struct pmckern_map_out pkm;
+	vm_map_entry_t entry;
+#endif
 	struct proc *p = curproc;
 	vm_offset_t addr;
 	vm_size_t size, pageoff;
@@ -527,6 +553,26 @@ sys_munmap(struct munmap_args *uap)
 	 */
 	if (!vm_map_check_protection(map, addr, addr + size, VM_PROT_NONE))
 		return (EINVAL);
+#ifdef HWPMC_HOOKS
+	/*
+	 * Inform hwpmc if the address range being unmapped contains
+	 * an executable region.
+	 */
+	if (vm_map_lookup_entry(map, addr, &entry)) {
+		for (;
+		     entry != &map->header && entry->start < addr + size;
+		     entry = entry->next) {
+			if (vm_map_check_protection(map, entry->start,
+				entry->end, VM_PROT_EXECUTE) == TRUE) {
+				pkm.pm_address = (uintptr_t) addr;
+				pkm.pm_size = (size_t) size;
+				PMC_CALL_HOOK(curthread, PMC_FN_MUNMAP,
+				    (void *) &pkm);
+				break;
+			}
+		}
+	}
+#endif
 	/* returns nothing but KERN_SUCCESS anyway */
 	vm_map_remove(map, addr, addr + size);
 	return (0);
Index: sys/linker.h
===================================================================
retrieving revision 1.10
diff -u -p -r1.10 linker.h
--- sys/linker.h
+++ sys/linker.h
@@ -225,6 +225,9 @@ int linker_ddb_search_symbol(caddr_t _va
 int linker_ddb_symbol_values(c_linker_sym_t _sym, linker_symval_t *_symval);
 
 
+/* HWPMC helper */
+void *linker_hwpmc_list_objects(void);
+
 #endif	/* _KERNEL */
 
 /*
Index: cpu/i386/include/cpu.h
===================================================================
RCS file: /home/aggelos/imports/vcs/dcvs/src/sys/cpu/i386/include/cpu.h,v
retrieving revision 1.25
diff -u -p -r1.25 cpu.h
--- cpu/i386/include/cpu.h	1 Mar 2007 01:46:52 -0000	1.25
+++ cpu/i386/include/cpu.h	29 Aug 2007 23:14:39 -0000
@@ -67,6 +67,10 @@
 #define CLKF_INTR(framep)	(mycpu->gd_intr_nesting_level > 1 || (curthread->td_flags & TDF_INTTHREAD))
 #define	CLKF_PC(framep)		((framep)->if_eip)
 
+#define	TRAPF_USERMODE(framep) \
+	((ISPL((framep)->tf_cs) == SEL_UPL) || ((framep)->tf_eflags & PSL_VM))
+#define	TRAPF_PC(framep)	((framep)->tf_eip)
+
 /*
  * Preempt the current process if in interrupt from user mode,
  * or after the current trap/syscall if in system mode.