venti: import changes from plan 9

R=rsc
CC=plan9port.codebot
http://codereview.appspot.com/4523057

src/cmd/venti/dump.c

@@ -103,7 +103,6 @@ threadmain(int argc, char *argv[])
 	fmtinstall('F', vtfcallfmt);
 	fmtinstall('V', vtscorefmt);
 
-	type = -1;
 	ARGBEGIN{
 	case 'h':
 		host = EARGF(usage());

src/cmd/venti/root.c

@@ -4,6 +4,12 @@
 #include <libsec.h>
 #include <thread.h>
 
+enum
+{
+	// XXX What to do here?
+	VtMaxLumpSize = 65536,
+};
+
 void
 usage(void)
 {
@@ -38,7 +44,7 @@ threadmain(int argc, char *argv[])
 	if(argc == 0)
 		usage();
 
-	buf = vtmallocz(8192);
+	buf = vtmallocz(VtMaxLumpSize);
 
 	z = vtdial(host);
 	if(z == nil)
@@ -52,7 +58,7 @@ threadmain(int argc, char *argv[])
 			fprint(2, "cannot parse score '%s': %r\n", argv[i]);
 			continue;
 		}
-		n = vtread(z, score, VtRootType, buf, 8192);
+		n = vtread(z, score, VtRootType, buf, VtMaxLumpSize);
 		if(n < 0){
 			fprint(2, "could not read block %V: %r\n", score);
 			continue;

src/cmd/venti/srv/arena.c

@@ -728,9 +728,12 @@ okarena(Arena *arena)
 		logerr(ECorrupt, "arena %s uncompressed size inconsistent with used space %lld %d %lld", arena->name, arena->diskstats.uncsize, arena->diskstats.clumps, arena->diskstats.used);
 	 */
 
+	/*
+	 * this happens; it's harmless.
+	 *
 	if(arena->ctime > arena->wtime)
 		logerr(ECorrupt, "arena %s creation time after last write time", arena->name);
-
+	 */
 	return ok;
 }
 

src/cmd/venti/srv/arenas.c

@@ -148,6 +148,7 @@ initarenapart(Part *part)
 
 	ap->arenas = MKNZ(Arena*, ap->narenas);
 	for(i = 0; i < ap->narenas; i++){
+		debugarena = i;
 		ap->arenas[i] = initarena(part, ap->map[i].start, ap->map[i].stop - ap->map[i].start, ap->blocksize);
 		if(ap->arenas[i] == nil){
 			seterr(ECorrupt, "%s: %r", ap->map[i].name);
@@ -168,8 +169,11 @@ initarenapart(Part *part)
 		}
 	}
 
-	for(i = 0; i < ap->narenas; i++)
+	for(i = 0; i < ap->narenas; i++) {
+		debugarena = i;
 		addarena(ap->arenas[i]);
+	}
+	debugarena = -1;
 
 	return ap;
 }
@@ -359,7 +363,8 @@ parseamap(IFile *f, AMapN *amn)
 	}
 	n = v;
 	if(n > MaxAMap){
-		seterr(ECorrupt, "illegal number of elements in %s", f->name);
+		seterr(ECorrupt, "illegal number of elements %d in %s",
+			n, f->name);
 		return -1;
 	}
 	am = MKNZ(AMap, n);

src/cmd/venti/srv/buildindex.c

@@ -50,18 +50,19 @@ static void	arenapartproc(void*);
 void
 usage(void)
 {
-	fprint(2, "usage: buildindex [-bd] [-i isect]... [-M imem] venti.conf\n");
+	fprint(2, "usage: buildindex [-b] [-i isect]... [-M imem] venti.conf\n");
 	threadexitsall("usage");
 }
 
 void
 threadmain(int argc, char *argv[])
 {
-	int fd, i, napart;
+	int fd, i, napart, nfinish, maxdisks;
 	u32int bcmem, imem;
 	Config conf;
 	Part *p;
 	
+	maxdisks = 100000;
 	ventifmtinstall();
 	imem = 256*1024*1024;
 	ARGBEGIN{
@@ -78,6 +79,9 @@ threadmain(int argc, char *argv[])
 	case 'M':
 		imem = unittoull(EARGF(usage()));
 		break;
+	case 'm':	/* temporary - might go away */
+		maxdisks = atoi(EARGF(usage()));
+		break;
 	default:
 		usage();
 		break;
@@ -146,17 +150,21 @@ threadmain(int argc, char *argv[])
 	/* start arena procs */
 	p = nil;
 	napart = 0;
+	nfinish = 0;
 	arenadonechan = chancreate(sizeof(void*), 0);
 	for(i=0; i<ix->narenas; i++){
 		if(ix->arenas[i]->part != p){
 			p = ix->arenas[i]->part;
 			vtproc(arenapartproc, p);
-			napart++;
+			if(++napart >= maxdisks){
+				recvp(arenadonechan);
+				nfinish++;
+			}
 		}
 	}
 
 	/* wait for arena procs to finish */
-	for(i=0; i<napart; i++)
+	for(nfinish=0; nfinish<napart; nfinish++)
 		recvp(arenadonechan);
 
 	/* tell index procs to finish */
@@ -844,6 +852,11 @@ isectproc(void *v)
 			sysfatal("not enough memory");
 		nminibuf = nbuf;
 	}
+	if (nbuf == 0) {
+		fprint(2, "%s: brand-new index, no work to do\n", argv0);
+		threadexitsall(nil);
+	}
+
 	/* size buffer to use extra memory */
 	bufsize = MinBufSize;
 	while(bufsize*2*nbuf <= isectmem && bufsize < MaxBufSize)

src/cmd/venti/srv/checkarenas.c

@@ -128,8 +128,10 @@ threadmain(int argc, char *argv[])
 	initdcache(8 * MaxDiskBlock);
 
 	for(i = 0; i < ap->narenas; i++)
-		if(should(ap->arenas[i]->name, argc, argv))
+		if(should(ap->arenas[i]->name, argc, argv)) {
+			debugarena = i;
 			checkarena(ap->arenas[i], scan, fix);
+		}
 
 	if(verbose > 1)
 		printstats();

src/cmd/venti/srv/config.c

@@ -75,7 +75,7 @@ runconfig(char *file, Config *config)
 	if(readifile(&f, file) < 0)
 		return -1;
 	memset(config, 0, sizeof *config);
-	config->mem = 0xFFFFFFFFUL;
+	config->mem = Unspecified;
 	ok = -1;
 	line = nil;
 	for(;;){
@@ -142,7 +142,7 @@ runconfig(char *file, Config *config)
 					flds[1], file);
 				break;
 			}
-			if(config->mem != 0xFFFFFFFFUL){
+			if(config->mem != Unspecified){
 				seterr(EAdmin, "duplicate mem lines in config file %s", file);
 				break;
 			}

src/cmd/venti/srv/conv.c

@@ -15,6 +15,8 @@
 #define	U32PUT(p,v)	(p)[0]=((v)>>24)&0xFF;(p)[1]=((v)>>16)&0xFF;(p)[2]=((v)>>8)&0xFF;(p)[3]=(v)&0xFF
 #define	U64PUT(p,v,t32)	t32=(v)>>32;U32PUT(p,t32);t32=(v);U32PUT((p)+4,t32)
 
+int debugarena = -1;		/* hack to improve error reporting */
+
 static struct {
 	u32int m;
 	char *s;
@@ -112,7 +114,9 @@ unpackarena(Arena *arena, u8int *buf)
 
 	m = U32GET(p);
 	if(m != ArenaMagic){
-		seterr(ECorrupt, "arena has wrong magic number: %s expected ArenaMagic (%#lux)", fmtmagic(fbuf, m), ArenaMagic);
+		seterr(ECorrupt, "arena %d has wrong magic number: %s "
+			"expected ArenaMagic (%#lux)", debugarena,
+			fmtmagic(fbuf, m), ArenaMagic);
 		return -1;
 	}
 	p += U32Size;
@@ -308,7 +312,9 @@ unpackarenahead(ArenaHead *head, u8int *buf)
 
 	m = U32GET(p);
 	if(m != ArenaHeadMagic){
-		seterr(ECorrupt, "arena has wrong magic number: %s expected ArenaHeadMagic (%#lux)", fmtmagic(fbuf, m), ArenaHeadMagic);
+		seterr(ECorrupt, "arena %d head has wrong magic number: %s "
+			"expected ArenaHeadMagic (%#lux)", debugarena,
+			fmtmagic(fbuf, m), ArenaHeadMagic);
 		return -1;
 	}
 

src/cmd/venti/srv/dat.h

@@ -54,6 +54,7 @@ enum
 	MaxIo			= 64*1024,	/* max size of a single read or write operation */
 	ICacheBits		= 16,		/* default bits for indexing icache */
 	MaxAMap			= 31*1024,	/* max. allowed arenas in an address mapping; must be < 32*1024 */
+	Unspecified		= TWID32,
 
 	/*
 	 * return codes from syncarena
@@ -750,6 +751,7 @@ extern	int		l1quantum;
 extern	int		ignorebloom;
 extern	int		icacheprefetch;
 extern	int		syncwrites;
+extern	int		debugarena; /* print in arena error msgs; -1==unknown */
 
 extern	Stats	*stathist;
 extern	int	nstathist;

src/cmd/venti/srv/findscore.c

@@ -93,7 +93,7 @@ threadmain(int argc, char *argv[])
 
 	file = argv[0];
 	if(strscore(argv[1], score) < 0)
-		sysfatal("bad score %s\n", argv[1]);
+		sysfatal("bad score %s", argv[1]);
 
 	part = initpart(file, OREAD|ODIRECT);
 	if(part == nil)

src/cmd/venti/srv/fmtarenas.c

@@ -105,7 +105,7 @@ threadmain(int argc, char *argv[])
 		if(limit >= ap->size || ap->size - limit < MinArenaSize){
 			limit = ap->size;
 			if(limit - addr < MinArenaSize)
-				sysfatal("bad arena set math: runt arena at %lld,%lld %lld\n", addr, limit, ap->size);
+				sysfatal("bad arena set math: runt arena at %lld,%lld %lld", addr, limit, ap->size);
 		}
 
 		snprint(aname, ANameSize, "%s%d", name, i);

src/cmd/venti/srv/fmtindex.c

@@ -82,11 +82,11 @@ threadmain(int argc, char *argv[])
 			arenas[n] = ap->arenas[j];
 			if(n < ix->narenas){
 				if(arenas[n] != ix->arenas[n])
-					sysfatal("mismatched arenas %s and %s at slot %d\n",
+					sysfatal("mismatched arenas %s and %s at slot %d",
 						arenas[n]->name, ix->arenas[n]->name, n);
 				amap[n] = ix->amap[n];
 				if(amap[n].start != addr)
-					sysfatal("mis-located arena %s in index %s\n", arenas[n]->name, ix->name);
+					sysfatal("mis-located arena %s in index %s", arenas[n]->name, ix->name);
 				addr = amap[n].stop;
 			}else{
 				amap[n].start = addr;

src/cmd/venti/srv/icachewrite.c

@@ -251,7 +251,6 @@ icachewritecoord(void *v)
 		as = icachestate();
 		if(as.arena==iwrite.as.arena && as.aa==iwrite.as.aa){
 			/* will not be able to do anything more than last flush - kick disk */
-			fprint(2, "icache: nothing to do - kick dcache\n");
 			trace(TraceProc, "icachewritecoord kick dcache");
 			kickdcache();
 			trace(TraceProc, "icachewritecoord kicked dcache");

src/cmd/venti/srv/index.c

@@ -328,16 +328,20 @@ newindex(char *name, ISect **sects, int n)
 	}
 
 	if(nb >= ((u64int)1 << 32)){
-		seterr(EBug, "index too large");
+		fprint(2, "%s: index is 2^32 blocks or more; ignoring some of it\n",
-		return nil;
+			argv0);
+		nb = ((u64int)1 << 32) - 1;
 	}
 
 	div = (((u64int)1 << 32) + nb - 1) / nb;
-	ub = (((u64int)1 << 32) - 1) / div + 1;
 	if(div < 100){
-		seterr(EBug, "index divisor too coarse [%lld buckets]", nb);
+		fprint(2, "%s: index divisor %d too coarse; "
-		return nil;
+			"index larger than needed, ignoring some of it\n",
+			argv0, div);
+		div = 100;
+		nb = (((u64int)1 << 32) - 1) / (100 - 1);
 	}
+	ub = (((u64int)1 << 32) - 1) / div + 1;
 	if(ub > nb){
 		seterr(EBug, "index initialization math wrong");
 		return nil;

src/cmd/venti/srv/printarena.c

@@ -24,7 +24,7 @@ rdarena(Arena *arena, u64int offset)
 	e = arena->base + arena->size;
 	if(offset != ~(u64int)0) {
 		if(offset >= e-a)
-			sysfatal("bad offset %llud >= %llud\n",
+			sysfatal("bad offset %llud >= %llud",
 				offset, e-a);
 		aa = offset;
 	} else
@@ -111,7 +111,7 @@ threadmain(int argc, char *argv[])
 		head.size, head.clumpmagic);
 
 	if(aoffset+head.size > part->size)
-		sysfatal("arena is truncated: want %llud bytes have %llud\n",
+		sysfatal("arena is truncated: want %llud bytes have %llud",
 			head.size, part->size);
 
 	partblocksize(part, head.blocksize);

src/cmd/venti/srv/printarenapart.c

@@ -26,7 +26,7 @@ rdarena(Arena *arena, u64int offset)
 	e = arena->base + arena->size;
 	if(offset != ~(u64int)0) {
 		if(offset >= e-a)
-			sysfatal("bad offset %llud >= %llud\n",
+			sysfatal("bad offset %llud >= %llud",
 				offset, e-a);
 		aa = offset;
 	} else

src/cmd/venti/srv/rdarena.c

@@ -2,7 +2,7 @@
 #include "dat.h"
 #include "fns.h"
 
-static int	verbose;
+static int verbose, quiet;
 
 void
 usage(void)
@@ -18,8 +18,10 @@ rdarena(Arena *arena)
 	u64int a, e;
 	u32int bs;
 
-	fprint(2, "copying %s to standard output\n", arena->name);
+	if (!quiet) {
-	printarena(2, arena);
+		fprint(2, "copying %s to standard output\n", arena->name);
+		printarena(2, arena);
+	}
 
 	bs = MaxIoSize;
 	if(bs < arena->blocksize)
@@ -51,6 +53,9 @@ threadmain(int argc, char *argv[])
 	statsinit();
 
 	ARGBEGIN{
+	case 'q':
+		quiet++;
+		break;
 	case 'v':
 		verbose++;
 		break;
@@ -87,5 +92,5 @@ threadmain(int argc, char *argv[])
 		}
 	}
 
-	sysfatal("couldn't find arena %s\n", aname);
+	sysfatal("couldn't find arena %s", aname);
 }

src/cmd/venti/srv/syncindex.c

@@ -56,7 +56,7 @@ threadmain(int argc, char *argv[])
 	if(verbose)
 		printindex(2, mainindex);
 	if(syncindex(mainindex) < 0)
-		sysfatal("failed to sync index=%s: %r\n", mainindex->name);
+		sysfatal("failed to sync index=%s: %r", mainindex->name);
 	flushicache();
 	flushdcache();
 

src/cmd/venti/srv/venti.c

@@ -109,6 +109,9 @@ threadmain(int argc, char *argv[])
 	fprint(2, "conf...");
 	if(initventi(configfile, &config) < 0)
 		sysfatal("can't init server: %r");
+	/*
+	 * load bloom filter
+	 */
 	if(mainindex->bloom && loadbloom(mainindex->bloom) < 0)
 		sysfatal("can't load bloom filter: %r");
 
@@ -139,15 +142,22 @@ threadmain(int argc, char *argv[])
 
 	if(mem == 0xffffffffUL)
 		mem = 1 * 1024 * 1024;
+
+	/*
+	 * lump cache
+	 */
 	if(0) fprint(2, "initialize %d bytes of lump cache for %d lumps\n",
 		mem, mem / (8 * 1024));
 	initlumpcache(mem, mem / (8 * 1024));
 
+	/*
+	 * index cache
+	 */
 	initicache(icmem);
 	initicachewrite();
 
 	/*
-	 * need a block for every arena and every process
+	 * block cache: need a block for every arena and every process
 	 */
 	minbcmem = maxblocksize * 
 		(mainindex->narenas + mainindex->nsects*4 + 16);
@@ -186,6 +196,8 @@ threadmain(int argc, char *argv[])
 		ventiserver(nil);
 	else
 		vtproc(ventiserver, nil);
+
+	threadexits(nil);
 }
 
 static void

src/cmd/venti/srv/wrarena.c

@@ -75,7 +75,7 @@ rdarena(Arena *arena, u64int offset)
 	e = arena->base + arena->size;
 	if(offset != ~(u64int)0) {
 		if(offset >= e - a)
-			sysfatal("bad offset %#llx >= %#llx\n", offset, e - a);
+			sysfatal("bad offset %#llx >= %#llx", offset, e - a);
 		aa = offset;
 	} else
 		aa = 0;
@@ -87,8 +87,8 @@ rdarena(Arena *arena, u64int offset)
 			break;
 		if(a < aa || ci.type == VtCorruptType){
 			if(ci.type == VtCorruptType)
-				fprint(2, "corrupt at %#llx: +%d\n",
+				fprint(2, "%s: corrupt clump read at %#llx: +%d\n",
-					a, ClumpSize+ci.size);
+					argv0, a, ClumpSize+ci.size);
 			continue;
 		}
 		lump = loadclump(arena, a, 0, &cl, score, 0);
@@ -187,7 +187,7 @@ threadmain(int argc, char *argv[])
 		sysfatal("corrupted arena header: %r");
 
 	if(aoffset+head.size > part->size)
-		sysfatal("arena is truncated: want %llud bytes have %llud\n",
+		sysfatal("arena is truncated: want %llud bytes have %llud",
 			head.size, part->size);
 
 	partblocksize(part, head.blocksize);

src/libavl/avl.c

@@ -96,6 +96,16 @@ balance(Avl **tp, Avl *p)
 	}
 }
 
+static int
+canoncmp(int cmp)
+{
+	if(cmp < 0)
+		return -1;
+	else if(cmp > 0)
+		return 1;
+	return 0;
+}
+
 static int
 _insertavl(Avl **tp, Avl *p, Avl *r, int (*cmp)(Avl*,Avl*), Avl **rfree)
 {
@@ -110,7 +120,7 @@ _insertavl(Avl **tp, Avl *p, Avl *r, int (*cmp)(Avl*,Avl*), Avl **rfree)
 		return 1;
 	}
 	ob = (*tp)->bal;
-	if((i = cmp(r, *tp)) != 0){
+	if((i = canoncmp(cmp(r, *tp))) != 0){
 		(*tp)->bal += i * _insertavl(&(*tp)->n[(i+1)/2], *tp, r, cmp,
 			rfree);
 		balance(tp, p);
@@ -129,23 +139,6 @@ _insertavl(Avl **tp, Avl *p, Avl *r, int (*cmp)(Avl*,Avl*), Avl **rfree)
 	return 0;
 }
 
-static Avl*
-_lookupavl(Avl *t, Avl *r, int (*cmp)(Avl*,Avl*))
-{
-	int i;
-	Avl *p;
-
-	p = nil;
-	while(t != nil){
-		assert(t->p == p);
-		if((i = cmp(r, t)) == 0)
-			return t;
-		p = t;
-		t = t->n[(i+1)/2];
-	}
-	return nil;
-}
-
 static int
 successor(Avl **tp, Avl *p, Avl **r)
 {
@@ -175,7 +168,7 @@ _deleteavl(Avl **tp, Avl *p, Avl *rx, int(*cmp)(Avl*,Avl*), Avl **del,
 		return 0;
 
 	ob = (*tp)->bal;
-	if((i=cmp(rx, *tp)) != 0){
+	if((i=canoncmp(cmp(rx, *tp))) != 0){
 		(*tp)->bal += i * _deleteavl(&(*tp)->n[(i+1)/2], *tp, rx, cmp,
 			del, predel, arg);
 		balance(tp, p);
@@ -259,12 +252,6 @@ insertavl(Avltree *t, Avl *new, Avl **oldp)
 	_insertavl(&t->root, nil, new, t->cmp, oldp);
 }
 
-Avl*
-lookupavl(Avltree *t, Avl *key)
-{
-	return _lookupavl(t->root, key, t->cmp);
-}
-
 static Avl*
 findpredecessor(Avl *a)
 {
@@ -303,6 +290,41 @@ findsuccessor(Avl *a)
 	}
 }
 
+static Avl*
+_lookupavl(Avl *t, Avl *r, int (*cmp)(Avl*,Avl*), int neighbor)
+{
+	int i;
+	Avl *p;
+
+	p = nil;
+	if(t == nil)
+		return nil;
+	do{
+		assert(t->p == p);
+		if((i = canoncmp(cmp(r, t))) == 0)
+			return t;
+		p = t;
+		t = t->n[(i+1)/2];
+	}while(t);
+	if(neighbor == 0)
+		return nil;
+	if(neighbor < 0)
+		return i > 0 ? p : findpredecessor(p);
+	return i < 0 ? p : findsuccessor(p);
+}
+
+Avl*
+searchavl(Avltree *t, Avl *key, int neighbor)
+{
+	return _lookupavl(t->root, key, t->cmp, neighbor);
+}
+
+Avl*
+lookupavl(Avltree *t, Avl *key)
+{
+	return _lookupavl(t->root, key, t->cmp, 0);
+}
+
 static void
 walkdel(Avl *a, void *v)
 {