multi-vkd support patch for review

["C.Turner" <c.turner@xxxxxxxxxxxxxxxxxxx>]

Well.. I suppose it's time to stop lurking and come out of the
woodwork..

Attached is a multi-vkd patch for vkernels on the 1.8-RELEASE branch.
(plus a couple of very minor typo fixups in the multi-vke code)

Since I'm pretty out of date on my C, no comments can be too harsh :)

a.k.a

I'm not used to strong typing at the moment, unless you count 'cons' 
or 'my %tab;' a strong type..

so I'll be glad to fix asap - hoping something like this can 
make release, although I know it might be too late..

The patch tries to match the multi-vke organization as much as it can,
maxing out via #define at 16 vkd's, which should be pretty good 
for most purposes. Also, it kludges the '-r' argument to mean
'disk' - let me know if I should fix the manpage / vkernel 
usage statement..

As far as testing goes, I basically made sure I can label, newfs,
mount and fill the disks.. vinum/ccd remains incomplete/unested.. 
(see below)

Simple FFS Check:

Fill up a bunch of single-partition disks..

# uname -sr
DragonFly 1.8.0-RELEASE
# df
Filesystem  1K-blocks   Used  Avail Capacity  Mounted on
/dev/vkd0a    1032142 314428 635144    33%    /
/dev/vkd1a     120926 120898  -9646   109%    /mnt/1
/dev/vkd2a     120926 120898  -9646   109%    /mnt/2
/dev/vkd3a     120926 120898  -9646   109%    /mnt/3
/dev/vkd4a     120926 120898  -9646   109%    /mnt/4
/dev/vkd5a     120926 120898  -9646   109%    /mnt/5
/dev/vkd6a     120926 120898  -9646   109%    /mnt/6
/dev/vkd7a     120926 120898  -9646   109%    /mnt/7
/dev/vkd8a     120926 120898  -9646   109%    /mnt/8
/dev/vkd9a     120926 120898  -9646   109%    /mnt/9
/dev/vkd10a    120926 120898  -9646   109%    /mnt/10
/dev/vkd11a    120926 120898  -9646   109%    /mnt/11
/dev/vkd12a    120926 120898  -9646   109%    /mnt/12
/dev/vkd13a    120926 120898  -9646   109%    /mnt/13
/dev/vkd14a    120926 120898  -9646   109%    /mnt/14
/dev/vkd15a    120926 120898  -9646   109%    /mnt/15
# du -sk /mnt/*/*
120896  /mnt/1/testfile
120896  /mnt/10/testfile
120896  /mnt/11/testfile
120896  /mnt/12/testfile
120896  /mnt/13/testfile
120896  /mnt/14/testfile
120896  /mnt/15/testfile
238224  /mnt/16/testfile
120896  /mnt/2/testfile
120896  /mnt/3/testfile
120896  /mnt/4/testfile
120896  /mnt/5/testfile
120896  /mnt/6/testfile
120896  /mnt/7/testfile
120896  /mnt/8/testfile
120896  /mnt/9/testfile
# uptime
12:09AM  up 42 mins, 1 user, load averages: 6.30, 9.58, 5.45

(/mnt/16/ results from my 'test accessing one too many disks' logic
gone bad)

Vinum Check:

well - perhaps eventually.. current status (after relabeling):

  # cat vinum.conf.disks
  drive d1 device /dev/vkd1a
  drive d2 device /dev/vkd2a
  # vinum create -f vinum.conf.disks
  Bus error (core dumped)

and I think I won't look further as the build box is a PII-450 
that's chewing on a pgksrc bulk at the moment..

Lurker Comments:

I've been using DragonFly since late 1.4.x as my day-to-day desktop,
and I must say the experience is great, I like all the goals, pkgsrc
is a great fit, and I'm looking forward to using the VKERNEL for more 
systems testing (hopefully with a faster box :).. 
thanks to the team for running a great project!

I've got some other vkernel / dfly related ideas floating around 
in my head, that I might work on but would like to throw out there
while I'm on my lurk-box to test the waters in case I don't get 
to them just yet:

  - arbitrary VKERNEL tty's via a fifo for 'detached' operation
  - some kind of control socket protocol 
    (e.g. host-initiated clean shutdown, etc),
    or signal handler hook-scripts, etc.
  - leaving a pidfile somehwere
  - auto host-side pf rulesets on vkernel startup 
    (to constrain the vkernels)
  - expanding vkd's into some kind of spoofed cam / scsi bus 
    to allow for 'hotswap' vkds , etc.
  - importing KAME SCTP, if there's any use for it and I'm capable of
    it.. (that one just got in my head somehow.. *cough* syslink
transport ..?? ;)

Any comments on implementing some of the above would be welcome - 
my kernel skillz are very weak. Need to get the McKusic book methinks :)
Probably the tty-on-a-pipe would be my first choice / next target..

Thanks again for creating a great system!

- Chris

Index: dev/virtual/disk/vdisk.c
===================================================================
RCS file: /nbsd/dcvs/src/sys/dev/virtual/disk/vdisk.c,v
retrieving revision 1.2
diff -u -r1.2 vdisk.c
--- dev/virtual/disk/vdisk.c	9 Jan 2007 18:26:56 -0000	1.2
+++ dev/virtual/disk/vdisk.c	27 Jan 2007 23:00:38 -0000
@@ -63,8 +63,6 @@
 	int fd;
 };
 
-
-
 #define CDEV_MAJOR	97
 
 static d_strategy_t	vkdstrategy;
@@ -82,32 +80,49 @@
 static void
 vkdinit(void *dummy __unused)
 {
+	int vkd_iter = 0;
+	struct vkd_info *myslot;
 	struct vkd_softc *sc;
 	struct stat st;
 
-	if (RootImageFd < 0 || fstat(RootImageFd, &st) < 0)
-		return;
+	KASSERT(VkdNum <= VKD_MAX, ("too many vkd's: %d\n", NetifNum));
+	
+	for (vkd_iter = 0; vkd_iter < VkdNum; vkd_iter++) {
+
+		/* check that the 'bus device' has been initialized */
+		myslot = (struct vkd_info*) NULL;
+		myslot = &VkdInfo[vkd_iter];
+		if (myslot == NULL)
+			continue;
+		if (myslot->fd < 0 || fstat(myslot->fd, &st) < 0)
+			continue;
+
+		/* and create a new device */
+		sc = NULL;
+		sc = kmalloc(sizeof(*sc), M_DEVBUF, M_WAITOK|M_ZERO);
+		sc->unit = myslot->unit;
+		sc->fd = myslot->fd;
+		bioq_init(&sc->bio_queue);
+		devstat_add_entry(&sc->stats, "vkd", sc->unit, DEV_BSIZE,
+				  DEVSTAT_NO_ORDERED_TAGS,
+				  DEVSTAT_TYPE_DIRECT | DEVSTAT_TYPE_IF_OTHER,
+				  DEVSTAT_PRIORITY_DISK);
+		sc->dev = disk_create(sc->unit, &sc->disk, 0, &vkd_ops);
+		sc->dev->si_drv1 = sc;
+		sc->dev->si_iosize_max = 256 * 1024;
 
-	sc = kmalloc(sizeof(*sc), M_DEVBUF, M_WAITOK|M_ZERO);
-	sc->unit = 0;
-	sc->fd = RootImageFd;
-	bioq_init(&sc->bio_queue);
-	devstat_add_entry(&sc->stats, "vkd", sc->unit, DEV_BSIZE,
-			  DEVSTAT_NO_ORDERED_TAGS,
-			  DEVSTAT_TYPE_DIRECT | DEVSTAT_TYPE_IF_OTHER,
-			  DEVSTAT_PRIORITY_DISK);
-	sc->dev = disk_create(sc->unit, &sc->disk, 0, &vkd_ops);
-	sc->dev->si_drv1 = sc;
-	sc->dev->si_iosize_max = 256 * 1024;
+/* not sure why if 0'ed in v1.2... kept here for fun. */
 #if 0
-	dl = &sc->disk.d_label;
-	bzero(dl, sizeof(*dl));
-	dl->d_secsize = DEV_BSIZE;
-	dl->d_nsectors = st.st_size / dl->d_secsize;
-	dl->d_ntracks = 1;
-	dl->d_secpercyl = dl->d_nsectors;
-	dl->d_ncylinders = 1;
+		dl = &sc->disk.d_label;
+		bzero(dl, sizeof(*dl));
+		dl->d_secsize = DEV_BSIZE;
+		dl->d_nsectors = st.st_size / dl->d_secsize;
+		dl->d_ntracks = 1;
+		dl->d_secpercyl = dl->d_nsectors;
+		dl->d_ncylinders = 1;
 #endif
+
+	}
 }
 
 SYSINIT(vkdisk, SI_SUB_DRIVERS, SI_ORDER_FIRST, vkdinit, NULL);
@@ -122,8 +137,6 @@
 
 	dev = ap->a_head.a_dev;
 	sc = dev->si_drv1;
-	if (sc->unit != 0)
-		return(ENXIO);
 	if (fstat(sc->fd, &st) < 0 || st.st_size == 0)
 		return(ENXIO);
 
Index: platform/vkernel/include/md_var.h
===================================================================
RCS file: /nbsd/dcvs/src/sys/platform/vkernel/include/md_var.h,v
retrieving revision 1.17
diff -u -r1.17 md_var.h
--- platform/vkernel/include/md_var.h	22 Jan 2007 19:37:05 -0000	1.17
+++ platform/vkernel/include/md_var.h	27 Jan 2007 20:04:17 -0000
@@ -45,6 +45,7 @@
 #endif
 
 #define VKNETIF_MAX	16
+#define VKD_MAX		16
 
 struct vknetif_info {
 	int		tap_fd;
@@ -53,6 +54,12 @@
 	in_addr_t	netif_mask;
 };
 
+struct vkd_info {
+        int fd;
+        int unit;
+        char fname[MAXPATHLEN];
+};
+
 extern	char	sigcode[];
 extern	int	szsigcode;
 extern	vpte_t	*KernelPTA;	/* NOTE: Offset for direct VA translation */
@@ -63,7 +70,8 @@
 extern  char    cpu_vendor[];	/* XXX belongs in i386 */
 extern  u_int   cpu_id;		/* XXX belongs in i386 */
 
-extern int	RootImageFd;
+extern struct vkd_info VkdInfo[VKD_MAX];
+extern int	VkdNum;
 extern int	MemImageFd;
 extern int	KQueueFd;
 extern struct vknetif_info NetifInfo[VKNETIF_MAX];
Index: platform/vkernel/platform/init.c
===================================================================
RCS file: /nbsd/dcvs/src/sys/platform/vkernel/platform/init.c,v
retrieving revision 1.28
diff -u -r1.28 init.c
--- platform/vkernel/platform/init.c	15 Jan 2007 20:51:15 -0000	1.28
+++ platform/vkernel/platform/init.c	27 Jan 2007 22:44:41 -0000
@@ -76,9 +76,10 @@
 vm_paddr_t Maxmem;
 vm_paddr_t Maxmem_bytes;
 int MemImageFd = -1;
-int RootImageFd = -1;
+struct vkd_info VkdInfo[VKD_MAX];
+int VkdNum = 0;
 struct vknetif_info NetifInfo[VKNETIF_MAX];
-int NetifNum;
+int NetifNum = 0;
 vm_offset_t KvaStart;
 vm_offset_t KvaEnd;
 vm_offset_t KvaSize;
@@ -104,8 +105,8 @@
 static void init_kern_memory(void);
 static void init_globaldata(void);
 static void init_vkernel(void);
-static void init_rootdevice(char *imageFile);
-static void init_netif(char *netifFile[], int netifFileNum);
+static void init_vkd(char *vkdFile[], int vkdExpNum); 
+static void init_netif(char *netifExp[], int netifExpNum);
 static void usage(const char *ctl);
 
 /*
@@ -115,10 +116,11 @@
 main(int ac, char **av)
 {
 	char *memImageFile = NULL;
-	char *rootImageFile = NULL;
 	char *netifFile[VKNETIF_MAX];
+	char *vkdFile[VKD_MAX];
 	char *suffix;
 	int netifFileNum = 0;
+	int vkdFileNum = 0;
 	int c;
 	int i;
 	int n;
@@ -159,7 +161,8 @@
 				netifFile[netifFileNum++] = optarg;
 			break;
 		case 'r':
-			rootImageFile = optarg;
+			if ( vkdFileNum < VKD_MAX )
+				vkdFile[vkdFileNum++] = optarg;
 			break;
 		case 'm':
 			Maxmem_bytes = strtoull(optarg, &suffix, 0);
@@ -197,7 +200,7 @@
 	init_globaldata();
 	init_vkernel();
 	init_kqueue();
-	init_rootdevice(rootImageFile);
+	init_vkd(vkdFile, vkdFileNum);
 	init_netif(netifFile, netifFileNum);
 	init_exceptions();
 	mi_startup();
@@ -538,8 +541,9 @@
 }
 
 /*
- * The root filesystem path for the virtual kernel is optional.  If specified
- * it points to a filesystem image.
+ * Filesystem image paths for the virtual kernel are optional.  
+ * If specified they each should point to a disk image, 
+ * the first of which will become the VKERNEL root disk.
  *
  * The virtual kernel caches data from our 'disk' just like a normal kernel,
  * so we do not really want the real kernel to cache the data too.  Use
@@ -547,18 +551,53 @@
  */
 static
 void
-init_rootdevice(char *imageFile)
+init_vkd ( char *vkdExp[], int vkdExpNum ) 
 {
-	struct stat st;
+	int i;	
+
+        if (vkdExpNum == 0)
+                return;
+
+	for(i=0; i < vkdExpNum; i++){
+		char *fname;
+		fname = vkdExp[i];
+
+        	if (fname == NULL) {
+                        warnx("Invalid argument to '-r'");
+                        continue;
+                }
+
+		if (VkdNum < VKD_MAX ) {
+			struct stat st; 
+			struct vkd_info* info = NULL;
+			int fd;
+       			size_t l = 0;
+
+			fd = open(fname, O_RDWR|O_DIRECT, 0644);
+			if (fd < 0 || fstat(fd, &st) < 0) {
+				err(1, "Unable to open/create %s: %s",
+				    fname, strerror(errno));
+				/* NOT REACHED */
+			}
 
-	if (imageFile) {
-		RootImageFd = open(imageFile, O_RDWR|O_DIRECT, 0644);
-		if (RootImageFd < 0 || fstat(RootImageFd, &st) < 0) {
-			err(1, "Unable to open/create %s: %s",
-			    imageFile, strerror(errno));
-			/* NOT REACHED */
+	       		info = &VkdInfo[VkdNum];
+			l = strlen(fname);
+
+	       		info->unit = i;
+			info->fd = fd;
+	        	memcpy(info->fname, fname, l);
+
+			if(i == 0)
+				rootdevnames[0] = "ufs:vkd0a";
+
+			VkdNum++;
+
+		}
+		else {
+                        warnx("vkd%d (%s) > VKD_MAX",
+				VkdNum, fname);
+                        continue;
 		}
-		rootdevnames[0] = "ufs:vkd0a";
 	}
 }
 
@@ -920,7 +959,7 @@
 
 		netif = strtok(netifExp[i], ":");
 		if (netif == NULL) {
-			warnx("Invalide argument to '-I'");
+			warnx("Invalid argument to '-I'");
 			continue;
 		}