mirror of
https://github.com/9fans/plan9port.git
synced 2025-01-12 11:10:07 +00:00
venti: fix sync deadlock, add /proc stub
This commit is contained in:
Russ Cox
parent
c5a183de10
commit
45ac814c86
20 changed files with 176 additions and 261 deletions
|
|
@ -293,13 +293,12 @@ ZZZ question: should this distinguish between an arena
|
|||
filling up and real errors writing the clump?
|
||||
*/
|
||||
u64int
|
||||
writeaclump(Arena *arena, Clump *c, u8int *clbuf, u64int start, u64int *pa)
|
||||
writeaclump(Arena *arena, Clump *c, u8int *clbuf)
|
||||
{
|
||||
DBlock *b;
|
||||
u64int a, aa;
|
||||
u32int clump, n, nn, m, off, blocksize;
|
||||
int ok;
|
||||
AState as;
|
||||
|
||||
n = c->info.size + ClumpSize + U32Size;
|
||||
qlock(&arena->lock);
|
||||
|
|
@ -309,10 +308,6 @@ writeaclump(Arena *arena, Clump *c, u8int *clbuf, u64int start, u64int *pa)
|
|||
if(!arena->memstats.sealed){
|
||||
logerr(EOk, "seal memstats %s", arena->name);
|
||||
arena->memstats.sealed = 1;
|
||||
as.arena = arena;
|
||||
as.aa = start+aa;
|
||||
as.stats = arena->memstats;
|
||||
setdcachestate(&as);
|
||||
}
|
||||
qunlock(&arena->lock);
|
||||
return TWID64;
|
||||
|
|
@ -390,14 +385,6 @@ NoCIG:
|
|||
writeclumpinfo(arena, clump, &c->info);
|
||||
wbarena(arena);
|
||||
|
||||
/* set up for call to setdcachestate */
|
||||
as.arena = arena;
|
||||
as.aa = start+arena->memstats.used;
|
||||
as.stats = arena->memstats;
|
||||
|
||||
/* update this before calling setdcachestate so it cannot be behind dcache.diskstate */
|
||||
*pa = start+aa;
|
||||
setdcachestate(&as);
|
||||
qunlock(&arena->lock);
|
||||
|
||||
return aa;
|
||||
|
|
|
|||
|
|
@ -36,7 +36,7 @@ static void arenapartproc(void*);
|
|||
void
|
||||
usage(void)
|
||||
{
|
||||
fprint(2, "usage: buildindex [-b] [-i isect]... [-M imem] venti.conf\n");
|
||||
fprint(2, "usage: buildindex [-bd] [-i isect]... [-M imem] venti.conf\n");
|
||||
threadexitsall("usage");
|
||||
}
|
||||
|
||||
|
|
@ -54,13 +54,13 @@ threadmain(int argc, char *argv[])
|
|||
case 'b':
|
||||
bloom = 1;
|
||||
break;
|
||||
case 'd': /* debugging - make sure to run all 3 passes */
|
||||
dumb = 1;
|
||||
break;
|
||||
case 'i':
|
||||
isect = vtrealloc(isect, (nisect+1)*sizeof(isect[0]));
|
||||
isect[nisect++] = EARGF(usage());
|
||||
break;
|
||||
case 'd': /* debugging - make sure to run all 3 passes */
|
||||
dumb = 1;
|
||||
break;
|
||||
case 'M':
|
||||
imem = unittoull(EARGF(usage()));
|
||||
break;
|
||||
|
|
@ -222,22 +222,28 @@ arenapartproc(void *v)
|
|||
if(a->memstats.clumps)
|
||||
fprint(2, "%T arena %s: %d entries\n",
|
||||
a->name, a->memstats.clumps);
|
||||
addr = ix->amap[i].start;
|
||||
for(clump=0; clump<a->memstats.clumps; clump+=n){
|
||||
/*
|
||||
* Running the loop backwards accesses the
|
||||
* clump info blocks forwards, since they are
|
||||
* stored in reverse order at the end of the arena.
|
||||
* This speeds things slightly.
|
||||
*/
|
||||
addr = ix->amap[i].start + a->memstats.used;
|
||||
for(clump=a->memstats.clumps; clump > 0; clump-=n){
|
||||
n = ClumpChunks;
|
||||
if(n > a->memstats.clumps - clump)
|
||||
n = a->memstats.clumps - clump;
|
||||
if(readclumpinfos(a, clump, cis, n) != n){
|
||||
if(n > clump)
|
||||
n = clump;
|
||||
if(readclumpinfos(a, clump-n, cis, n) != n){
|
||||
fprint(2, "%T arena %s: directory read: %r\n", a->name);
|
||||
errors = 1;
|
||||
break;
|
||||
}
|
||||
for(j=0; j<n; j++){
|
||||
for(j=n-1; j>=0; j--){
|
||||
ci = &cis[j];
|
||||
ie.ia.type = ci->type;
|
||||
ie.ia.size = ci->uncsize;
|
||||
addr -= ci->size + ClumpSize;
|
||||
ie.ia.addr = addr;
|
||||
addr += ci->size + ClumpSize;
|
||||
ie.ia.blocks = (ci->size + ClumpSize + (1<<ABlockLog)-1) >> ABlockLog;
|
||||
scorecp(ie.score, ci->score);
|
||||
if(ci->type == VtCorruptType)
|
||||
|
|
@ -253,6 +259,8 @@ arenapartproc(void *v)
|
|||
}
|
||||
}
|
||||
}
|
||||
if(addr != ix->amap[i].start)
|
||||
fprint(2, "%T arena %s: clump miscalculation %lld != %lld\n", a->name, addr, ix->amap[i].start);
|
||||
}
|
||||
add(&arenaentries, tot);
|
||||
add(&skipentries, nskip);
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ checkarena(Arena *arena, int scan, int fix)
|
|||
|
||||
err = 0;
|
||||
for(;;){
|
||||
e = syncarena(arena, 0, 1000, 0, fix);
|
||||
e = syncarena(arena, 1000, 0, fix);
|
||||
err |= e;
|
||||
if(!(e & SyncHeader))
|
||||
break;
|
||||
|
|
|
|||
|
|
@ -62,19 +62,17 @@ storeclump(Index *ix, ZBlock *zb, u8int *sc, int type, u32int creator, IAddr *ia
|
|||
memset(cb->data+ClumpSize+dsize, 0, 4);
|
||||
cl.info.size = dsize;
|
||||
|
||||
ia->addr = 0;
|
||||
ia->type = type;
|
||||
ia->size = size;
|
||||
ia->blocks = (dsize + ClumpSize + (1 << ABlockLog) - 1) >> ABlockLog;
|
||||
|
||||
a = writeiclump(ix, &cl, cb->data, &ia->addr);
|
||||
|
||||
a = writeiclump(ix, &cl, cb->data);
|
||||
trace(TraceLump, "storeclump exit %lld", a);
|
||||
|
||||
freezblock(cb);
|
||||
if(a == TWID64)
|
||||
return -1;
|
||||
|
||||
ia->addr = a;
|
||||
ia->type = type;
|
||||
ia->size = size;
|
||||
ia->blocks = (dsize + ClumpSize + (1 << ABlockLog) - 1) >> ABlockLog;
|
||||
|
||||
/*
|
||||
qlock(&stats.lock);
|
||||
stats.clumpwrites++;
|
||||
|
|
|
|||
|
|
@ -466,6 +466,8 @@ struct Index
|
|||
AMap *smap; /* mapping of buckets to index sections */
|
||||
int narenas;
|
||||
AMap *amap; /* mapping from index addesses to arenas */
|
||||
|
||||
QLock writing;
|
||||
};
|
||||
|
||||
/*
|
||||
|
|
|
|||
|
|
@ -55,9 +55,6 @@ struct DCache
|
|||
u8int *mem; /* memory for all block descriptors */
|
||||
int ndirty; /* number of dirty blocks */
|
||||
int maxdirty; /* max. number of dirty blocks */
|
||||
|
||||
AState diskstate;
|
||||
AState state;
|
||||
};
|
||||
|
||||
typedef struct Ra Ra;
|
||||
|
|
@ -123,26 +120,6 @@ initdcache(u32int mem)
|
|||
vtproc(delaykickroundproc, &dcache.round);
|
||||
}
|
||||
|
||||
void
|
||||
setdcachestate(AState *a)
|
||||
{
|
||||
trace(TraceBlock, "setdcachestate %s 0x%llux clumps %d", a->arena ? a->arena->name : nil, a->aa, a->stats.clumps);
|
||||
qlock(&dcache.lock);
|
||||
dcache.state = *a;
|
||||
qunlock(&dcache.lock);
|
||||
}
|
||||
|
||||
AState
|
||||
diskstate(void)
|
||||
{
|
||||
AState a;
|
||||
|
||||
qlock(&dcache.lock);
|
||||
a = dcache.diskstate;
|
||||
qunlock(&dcache.lock);
|
||||
return a;
|
||||
}
|
||||
|
||||
static u32int
|
||||
pbhash(u64int addr)
|
||||
{
|
||||
|
|
@ -637,7 +614,6 @@ flushproc(void *v)
|
|||
int i, j, n;
|
||||
ulong t0;
|
||||
DBlock *b, **write;
|
||||
AState as;
|
||||
|
||||
USED(v);
|
||||
threadsetname("flushproc");
|
||||
|
|
@ -648,10 +624,6 @@ flushproc(void *v)
|
|||
t0 = nsec()/1000;
|
||||
trace(TraceProc, "build t=%lud", (ulong)(nsec()/1000)-t0);
|
||||
|
||||
qlock(&dcache.lock);
|
||||
as = dcache.state;
|
||||
qunlock(&dcache.lock);
|
||||
|
||||
write = dcache.write;
|
||||
n = 0;
|
||||
for(i=0; i<dcache.nblocks; i++){
|
||||
|
|
@ -688,7 +660,6 @@ flushproc(void *v)
|
|||
*/
|
||||
trace(TraceProc, "undirty.%d t=%lud", j, (ulong)(nsec()/1000)-t0);
|
||||
qlock(&dcache.lock);
|
||||
dcache.diskstate = as;
|
||||
for(i=0; i<n; i++){
|
||||
b = write[i];
|
||||
--dcache.ndirty;
|
||||
|
|
|
|||
|
|
@ -661,7 +661,7 @@ isonearena(void)
|
|||
return u32(pagein(0, Block)) == ArenaHeadMagic;
|
||||
}
|
||||
|
||||
static int tabsizes[] = { 16*1024, 64*1024, 512*1024, };
|
||||
static int tabsizes[] = { 16*1024, 64*1024, 512*1024, 768*1024, };
|
||||
/*
|
||||
* Poke around on the disk to guess what the ArenaPart numbers are.
|
||||
*/
|
||||
|
|
@ -807,8 +807,9 @@ guessgeometry(void)
|
|||
* Fmtarenas used to use 64k tab, now uses 512k tab.
|
||||
*/
|
||||
if(ap.arenabase == 0){
|
||||
print("trying standard arena bases...\n");
|
||||
for(i=0; i<nelem(tabsizes); i++){
|
||||
ap.arenabase = ROUNDUP(PartBlank+HeadSize, ap.blocksize);
|
||||
ap.arenabase = ROUNDUP(PartBlank+HeadSize+tabsizes[i], ap.blocksize);
|
||||
p = pagein(ap.arenabase, Block);
|
||||
if(u32(p) == ArenaHeadMagic)
|
||||
break;
|
||||
|
|
@ -1554,7 +1555,7 @@ guessarena(vlong offset0, int anum, ArenaHead *head, Arena *arena,
|
|||
bcit = cibuf;
|
||||
ecit = cibuf+ncibuf;
|
||||
|
||||
smart = 1;
|
||||
smart = 0; /* Somehow the smart code doesn't do corrupt clumps right. */
|
||||
Again:
|
||||
nbad = 0;
|
||||
ci = bci;
|
||||
|
|
|
|||
|
|
@ -29,7 +29,6 @@ void delaykickroundproc(void*);
|
|||
void dirtydblock(DBlock*, int);
|
||||
void diskaccess(int);
|
||||
void disksched(void);
|
||||
AState diskstate(void);
|
||||
void *emalloc(ulong);
|
||||
void emptydcache(void);
|
||||
void emptyicache(void);
|
||||
|
|
@ -60,6 +59,7 @@ vlong hargint(HConnect*, char*, vlong);
|
|||
int hdebug(HConnect*);
|
||||
int hdisk(HConnect*);
|
||||
int hnotfound(HConnect*);
|
||||
int hproc(HConnect*);
|
||||
int hsethtml(HConnect*);
|
||||
int hsettext(HConnect*);
|
||||
int httpdinit(char *address, char *webroot);
|
||||
|
|
@ -68,6 +68,7 @@ IEntry* icachedirty(u32int, u32int, u64int);
|
|||
ulong icachedirtyfrac(void);
|
||||
void icacheclean(IEntry*);
|
||||
int icachelookup(u8int *score, int type, IAddr *ia);
|
||||
AState icachestate(void);
|
||||
int ientrycmp(const void *vie1, const void *vie2);
|
||||
char *ifileline(IFile *f);
|
||||
int ifilename(IFile *f, char *dst);
|
||||
|
|
@ -91,7 +92,7 @@ Part* initpart(char *name, int mode);
|
|||
void initround(Round*, char*, int);
|
||||
int initventi(char *config, Config *conf);
|
||||
void insertlump(Lump *lump, Packet *p);
|
||||
int insertscore(u8int *score, IAddr *ia, int state);
|
||||
int insertscore(u8int *score, IAddr *ia, int state, AState *as);
|
||||
void kickdcache(void);
|
||||
void kickicache(void);
|
||||
void kickround(Round*, int wait);
|
||||
|
|
@ -156,7 +157,6 @@ int runconfig(char *config, Config*);
|
|||
int scorecmp(u8int *, u8int *);
|
||||
void scoremem(u8int *score, u8int *buf, int size);
|
||||
void setatailstate(AState*);
|
||||
void setdcachestate(AState*);
|
||||
void seterr(int severity, char *fmt, ...);
|
||||
void setstat(int, long);
|
||||
void settrace(char *type);
|
||||
|
|
@ -170,9 +170,8 @@ int strscore(char *s, u8int *score);
|
|||
int stru32int(char *s, u32int *r);
|
||||
int stru64int(char *s, u64int *r);
|
||||
void sumarena(Arena *arena);
|
||||
int syncarena(Arena *arena, u64int start, u32int n, int zok, int fix);
|
||||
int syncarenaindex(Index *ix, Arena *arena, u32int clump, u64int a, int fix, int *pflush, int check);
|
||||
int syncindex(Index *ix, int fix, int mustflushicache, int check);
|
||||
int syncarena(Arena *arena, u32int n, int zok, int fix);
|
||||
int syncindex(Index *ix);
|
||||
void trace(char *type, char*, ...);
|
||||
void traceinit(void);
|
||||
int u64log2(u64int v);
|
||||
|
|
@ -201,12 +200,12 @@ void wbbloomhead(Bloom*);
|
|||
int wbisect(ISect *is);
|
||||
int wbindex(Index *ix);
|
||||
int whackblock(u8int *dst, u8int *src, int ssize);
|
||||
u64int writeaclump(Arena *a, Clump *c, u8int *clbuf, u64int, u64int*);
|
||||
u64int writeaclump(Arena *a, Clump *c, u8int *clbuf);
|
||||
u32int writearena(Arena *arena, u64int aa, u8int *clbuf, u32int n);
|
||||
int writebloom(Bloom*);
|
||||
int writeclumpinfo(Arena *arean, int clump, ClumpInfo *ci);
|
||||
int writepng(Hio*, Memimage*);
|
||||
u64int writeiclump(Index *ix, Clump *c, u8int *clbuf, u64int*);
|
||||
u64int writeiclump(Index *ix, Clump *c, u8int *clbuf);
|
||||
int writelump(Packet *p, u8int *score, int type, u32int creator, uint ms);
|
||||
int writepart(Part *part, u64int addr, u8int *buf, u32int n);
|
||||
int writeqlump(Lump *u, Packet *p, int creator, uint ms);
|
||||
|
|
|
|||
17
src/cmd/venti/srv/hproc.c
Normal file
17
src/cmd/venti/srv/hproc.c
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
#include "stdinc.h"
|
||||
#include "dat.h"
|
||||
#include "fns.h"
|
||||
#include "xml.h"
|
||||
|
||||
int
|
||||
hproc(HConnect *c)
|
||||
{
|
||||
int r;
|
||||
|
||||
if((r = hsettext(c)) < 0)
|
||||
return r;
|
||||
hprint(&c->hout, "/proc only implemented on Plan 9\n");
|
||||
hflush(&c->hout);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
@ -69,6 +69,7 @@ httpdinit(char *address, char *dir)
|
|||
httpdobj("/emptydcache", hdcacheempty);
|
||||
httpdobj("/disk", hdisk);
|
||||
httpdobj("/debug", hdebug);
|
||||
httpdobj("/proc/", hproc);
|
||||
|
||||
if(vtproc(listenproc, address) < 0)
|
||||
return -1;
|
||||
|
|
@ -565,11 +566,11 @@ darena(Hio *hout, Arena *arena)
|
|||
if(scorecmp(zeroscore, arena->score) != 0)
|
||||
hprint(hout, "\tscore=%V\n", arena->score);
|
||||
|
||||
hprint(hout, "\tmem: clumps=%d compressed clumps=%d data=%,lld compressed data=%,lld storage=%,lld\n",
|
||||
hprint(hout, "\twritten: clumps=%d compressed clumps=%d data=%,lld compressed data=%,lld storage=%,lld\n",
|
||||
arena->memstats.clumps, arena->memstats.cclumps, arena->memstats.uncsize,
|
||||
arena->memstats.used - arena->memstats.clumps * ClumpSize,
|
||||
arena->memstats.used + arena->memstats.clumps * ClumpInfoSize);
|
||||
hprint(hout, "\tdisk: clumps=%d compressed clumps=%d data=%,lld compressed data=%,lld storage=%,lld\n",
|
||||
hprint(hout, "\tindexed: clumps=%d compressed clumps=%d data=%,lld compressed data=%,lld storage=%,lld\n",
|
||||
arena->diskstats.clumps, arena->diskstats.cclumps, arena->diskstats.uncsize,
|
||||
arena->diskstats.used - arena->diskstats.clumps * ClumpSize,
|
||||
arena->diskstats.used + arena->diskstats.clumps * ClumpInfoSize);
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ struct ICache
|
|||
IEntry dirty;
|
||||
u32int maxdirty;
|
||||
u32int ndirty;
|
||||
AState as;
|
||||
|
||||
ISum **sum;
|
||||
int nsum;
|
||||
|
|
@ -398,7 +399,7 @@ icachelookup(u8int score[VtScoreSize], int type, IAddr *ia)
|
|||
}
|
||||
|
||||
int
|
||||
insertscore(u8int score[VtScoreSize], IAddr *ia, int state)
|
||||
insertscore(u8int score[VtScoreSize], IAddr *ia, int state, AState *as)
|
||||
{
|
||||
ISum *toload;
|
||||
|
||||
|
|
@ -409,6 +410,13 @@ insertscore(u8int score[VtScoreSize], IAddr *ia, int state)
|
|||
else{
|
||||
assert(state == IEDirty);
|
||||
toload = nil;
|
||||
if(as == nil)
|
||||
fprint(2, "%T insertscore IEDirty without as; called from %lux\n", getcallerpc(&score));
|
||||
else{
|
||||
if(icache.as.aa > as->aa)
|
||||
fprint(2, "%T insertscore: aa moving backward: %#llux -> %#llux\n", icache.as.aa, as->aa);
|
||||
icache.as = *as;
|
||||
}
|
||||
}
|
||||
qunlock(&icache.lock);
|
||||
if(toload){
|
||||
|
|
@ -443,7 +451,7 @@ lookupscore_untimed(u8int score[VtScoreSize], int type, IAddr *ia)
|
|||
if(loadientry(mainindex, score, type, &d) < 0)
|
||||
return -1;
|
||||
|
||||
insertscore(score, &d.ia, IEClean);
|
||||
insertscore(score, &d.ia, IEClean, nil);
|
||||
*ia = d.ia;
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -507,6 +515,16 @@ icachedirty(u32int lo, u32int hi, u64int limit)
|
|||
return dirty;
|
||||
}
|
||||
|
||||
AState
|
||||
icachestate(void)
|
||||
{
|
||||
AState as;
|
||||
|
||||
qlock(&icache.lock);
|
||||
as = icache.as;
|
||||
qunlock(&icache.lock);
|
||||
return as;
|
||||
}
|
||||
|
||||
/*
|
||||
* The singly-linked non-circular list of index entries ie
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ static void icachewritecoord(void*);
|
|||
static IEntry *iesort(IEntry*);
|
||||
|
||||
int icachesleeptime = 1000; /* milliseconds */
|
||||
int minicachesleeptime = 50;
|
||||
int minicachesleeptime = 0;
|
||||
|
||||
enum
|
||||
{
|
||||
|
|
@ -242,18 +242,20 @@ icachewritecoord(void *v)
|
|||
threadsetname("icachewritecoord");
|
||||
|
||||
ix = mainindex;
|
||||
iwrite.as = diskstate();
|
||||
iwrite.as = icachestate();
|
||||
|
||||
for(;;){
|
||||
trace(TraceProc, "icachewritecoord sleep");
|
||||
waitforkick(&iwrite.round);
|
||||
trace(TraceWork, "start");
|
||||
as = diskstate();
|
||||
as = icachestate();
|
||||
if(as.arena==iwrite.as.arena && as.aa==iwrite.as.aa){
|
||||
/* will not be able to do anything more than last flush - kick disk */
|
||||
fprint(2, "icache: nothing to do - kick dcache\n");
|
||||
trace(TraceProc, "icachewritecoord kick dcache");
|
||||
kickdcache();
|
||||
trace(TraceProc, "icachewritecoord kicked dcache");
|
||||
goto SkipWork; /* won't do anything; don't bother rewriting bloom filter */
|
||||
}
|
||||
iwrite.as = as;
|
||||
|
||||
|
|
@ -271,9 +273,11 @@ icachewritecoord(void *v)
|
|||
err |= recvul(ix->bloom->writedonechan);
|
||||
|
||||
trace(TraceProc, "icachewritecoord donewrite err=%d", err);
|
||||
if(err == 0)
|
||||
if(err == 0){
|
||||
setatailstate(&iwrite.as);
|
||||
}
|
||||
}
|
||||
SkipWork:
|
||||
icacheclean(nil); /* wake up anyone waiting */
|
||||
trace(TraceWork, "finish");
|
||||
addstat(StatIcacheFlush, 1);
|
||||
|
|
|
|||
|
|
@ -541,20 +541,33 @@ ZZZ question: should this distinguish between an arena
|
|||
filling up and real errors writing the clump?
|
||||
*/
|
||||
u64int
|
||||
writeiclump(Index *ix, Clump *c, u8int *clbuf, u64int *pa)
|
||||
writeiclump(Index *ix, Clump *c, u8int *clbuf)
|
||||
{
|
||||
u64int a;
|
||||
int i;
|
||||
IAddr ia;
|
||||
AState as;
|
||||
|
||||
trace(TraceLump, "writeiclump enter");
|
||||
qlock(&ix->writing);
|
||||
for(i = ix->mapalloc; i < ix->narenas; i++){
|
||||
a = writeaclump(ix->arenas[i], c, clbuf, ix->amap[i].start, pa);
|
||||
a = writeaclump(ix->arenas[i], c, clbuf);
|
||||
if(a != TWID64){
|
||||
ix->mapalloc = i; /* assuming write is atomic, race is okay */
|
||||
ix->mapalloc = i;
|
||||
ia.addr = ix->amap[i].start + a;
|
||||
ia.type = c->info.type;
|
||||
ia.size = c->info.uncsize;
|
||||
ia.blocks = (c->info.size + ClumpSize + (1<<ABlockLog) - 1) >> ABlockLog;
|
||||
as.arena = ix->arenas[i];
|
||||
as.aa = ia.addr;
|
||||
as.stats = as.arena->memstats;
|
||||
insertscore(c->info.score, &ia, IEDirty, &as);
|
||||
qunlock(&ix->writing);
|
||||
trace(TraceLump, "writeiclump exit");
|
||||
return a;
|
||||
return ia.addr;
|
||||
}
|
||||
}
|
||||
qunlock(&ix->writing);
|
||||
|
||||
seterr(EAdmin, "no space left in arenas");
|
||||
trace(TraceLump, "writeiclump failed");
|
||||
|
|
|
|||
|
|
@ -173,8 +173,6 @@ writeqlump(Lump *u, Packet *p, int creator, uint ms)
|
|||
flat = packet2zblock(p, packetsize(p));
|
||||
ok = storeclump(mainindex, flat, u->score, u->type, creator, &ia);
|
||||
freezblock(flat);
|
||||
if(ok == 0)
|
||||
ok = insertscore(u->score, &ia, IEDirty);
|
||||
if(ok == 0)
|
||||
insertlump(u, p);
|
||||
else
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ LIBOFILES=\
|
|||
dump.$O\
|
||||
graph.$O\
|
||||
hdisk.$O\
|
||||
hproc.$O\
|
||||
httpd.$O\
|
||||
icache.$O\
|
||||
icachewrite.$O\
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ clumpinfocmp(ClumpInfo *c, ClumpInfo *d)
|
|||
* returns 0 if ok, flags if error occurred
|
||||
*/
|
||||
int
|
||||
syncarena(Arena *arena, u64int start, u32int n, int zok, int fix)
|
||||
syncarena(Arena *arena, u32int n, int zok, int fix)
|
||||
{
|
||||
ZBlock *lump;
|
||||
Clump cl;
|
||||
|
|
@ -53,7 +53,7 @@ syncarena(Arena *arena, u64int start, u32int n, int zok, int fix)
|
|||
fprint(2, "%s: illegal clump magic number=%#8.8ux at clump=%d\n", arena->name, magic, clump);
|
||||
/* err |= SyncDataErr; */
|
||||
if(fix && writeclumpmagic(arena, aa, ClumpFreeMagic) < 0){
|
||||
fprint(2, "can't write corrected clump free magic: %r");
|
||||
fprint(2, "%s: can't write corrected clump free magic: %r", arena->name);
|
||||
err |= SyncFixErr;
|
||||
}
|
||||
break;
|
||||
|
|
@ -136,9 +136,8 @@ syncarena(Arena *arena, u64int start, u32int n, int zok, int fix)
|
|||
|| cclumps != arena->memstats.cclumps
|
||||
|| uncsize != arena->memstats.uncsize){
|
||||
err |= SyncHeader;
|
||||
fprint(2, "arena %s: start=%lld fix=%d flush=%d %lld->%lld %ud->%ud %ud->%ud %lld->%lld\n",
|
||||
fprint(2, "arena %s: fix=%d flush=%d %lld->%lld %ud->%ud %ud->%ud %lld->%lld\n",
|
||||
arena->name,
|
||||
start,
|
||||
fix,
|
||||
flush,
|
||||
used, arena->memstats.used,
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ static int verbose;
|
|||
void
|
||||
usage(void)
|
||||
{
|
||||
fprint(2, "usage: syncindex [-fv] [-B blockcachesize] config\n");
|
||||
fprint(2, "usage: syncindex [-v] [-B blockcachesize] config\n");
|
||||
threadexitsall("usage");
|
||||
}
|
||||
|
||||
|
|
@ -16,9 +16,7 @@ void
|
|||
threadmain(int argc, char *argv[])
|
||||
{
|
||||
u32int bcmem, icmem;
|
||||
int fix;
|
||||
|
||||
fix = 0;
|
||||
bcmem = 0;
|
||||
icmem = 0;
|
||||
ARGBEGIN{
|
||||
|
|
@ -28,9 +26,6 @@ threadmain(int argc, char *argv[])
|
|||
case 'I':
|
||||
icmem = unittoull(EARGF(usage()));
|
||||
break;
|
||||
case 'f':
|
||||
fix++;
|
||||
break;
|
||||
case 'v':
|
||||
verbose++;
|
||||
break;
|
||||
|
|
@ -39,9 +34,6 @@ threadmain(int argc, char *argv[])
|
|||
break;
|
||||
}ARGEND
|
||||
|
||||
if(!fix)
|
||||
readonly = 1;
|
||||
|
||||
if(argc != 1)
|
||||
usage();
|
||||
|
||||
|
|
@ -63,8 +55,10 @@ threadmain(int argc, char *argv[])
|
|||
|
||||
if(verbose)
|
||||
printindex(2, mainindex);
|
||||
if(syncindex(mainindex, fix, 1, 0) < 0)
|
||||
if(syncindex(mainindex) < 0)
|
||||
sysfatal("failed to sync index=%s: %r\n", mainindex->name);
|
||||
flushicache();
|
||||
flushdcache();
|
||||
|
||||
threadexitsall(0);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,184 +2,92 @@
|
|||
#include "dat.h"
|
||||
#include "fns.h"
|
||||
|
||||
enum
|
||||
static int
|
||||
syncarenaindex(Arena *arena, u64int a0)
|
||||
{
|
||||
ClumpChunks = 32*1024
|
||||
};
|
||||
|
||||
static int missing, wrong;
|
||||
|
||||
/*
|
||||
* shell sort is plenty good enough
|
||||
* because we're going to do a bunch of disk i/o's
|
||||
*/
|
||||
static void
|
||||
sortclumpinfo(ClumpInfo *ci, int *s, int n)
|
||||
{
|
||||
int i, j, m, t;
|
||||
|
||||
for(m = (n + 3) / 5; m > 0; m = (m + 1) / 3){
|
||||
for(i = n - m; i-- > 0;){
|
||||
for(j = i + m; j < n; j += m){
|
||||
if(memcmp(ci[s[j - m]].score, ci[s[j]].score, VtScoreSize) <= 0)
|
||||
break;
|
||||
t = s[j];
|
||||
s[j] = s[j - m];
|
||||
s[j - m] = t;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int
|
||||
syncarenaindex(Index *ix, Arena *arena, u32int clump, u64int a, int fix, int *pflush, int check)
|
||||
{
|
||||
Packet *pack;
|
||||
IEntry ie;
|
||||
int ok;
|
||||
u32int clump;
|
||||
u64int a;
|
||||
ClumpInfo ci;
|
||||
IAddr ia;
|
||||
ClumpInfo *ci, *cis;
|
||||
u64int *addrs;
|
||||
int i, n, ok, *s, flush;
|
||||
AState as;
|
||||
|
||||
if(arena->diskstats.clumps == arena->memstats.clumps)
|
||||
return 0;
|
||||
|
||||
memset(&as, 0, sizeof as);
|
||||
as.arena = arena;
|
||||
as.stats = arena->diskstats;
|
||||
|
||||
trace(TraceProc, "syncarenaindex enter");
|
||||
|
||||
flush = 0;
|
||||
cis = MKN(ClumpInfo, ClumpChunks);
|
||||
addrs = MKN(u64int, ClumpChunks);
|
||||
s = MKN(int, ClumpChunks);
|
||||
ok = 0;
|
||||
for(; clump < arena->memstats.clumps; clump += n){
|
||||
n = ClumpChunks;
|
||||
if(n > arena->memstats.clumps - clump)
|
||||
n = arena->memstats.clumps - clump;
|
||||
n = readclumpinfos(arena, clump, cis, n);
|
||||
if(n <= 0){
|
||||
fprint(2, "arena directory read failed\n");
|
||||
a = a0 + arena->diskstats.used;
|
||||
for(clump=arena->diskstats.clumps; clump < arena->memstats.clumps; clump++){
|
||||
if(readclumpinfo(arena, clump, &ci) < 0){
|
||||
fprint(2, "%s: clump %d: cannot read clumpinfo\n",
|
||||
arena->name, clump);
|
||||
ok = -1;
|
||||
break;
|
||||
}
|
||||
|
||||
for(i = 0; i < n; i++){
|
||||
addrs[i] = a;
|
||||
a += cis[i].size + ClumpSize;
|
||||
s[i] = i;
|
||||
}
|
||||
ia.type = ci.type;
|
||||
ia.size = ci.uncsize;
|
||||
ia.addr = a;
|
||||
ia.blocks = (ClumpSize + ci.size + (1 << ABlockLog) - 1) >> ABlockLog;
|
||||
a += ClumpSize + ci.size;
|
||||
|
||||
sortclumpinfo(cis, s, n);
|
||||
|
||||
for(i = 0; i < n; i++){
|
||||
ci = &cis[s[i]];
|
||||
ia.type = ci->type;
|
||||
ia.size = ci->uncsize;
|
||||
ia.addr = addrs[s[i]];
|
||||
ia.blocks = (ci->size + ClumpSize + (1 << ABlockLog) - 1) >> ABlockLog;
|
||||
|
||||
if(!check)
|
||||
goto Add;
|
||||
if(loadientry(ix, ci->score, ci->type, &ie) < 0){
|
||||
trace(TraceProc, "syncarenaindex missing block %V.%d", ci->score, ci->type);
|
||||
missing++;
|
||||
if(0) fprint(2, "missing block type=%d score=%V\n", ci->type, ci->score);
|
||||
}else if(iaddrcmp(&ia, &ie.ia) != 0){
|
||||
trace(TraceProc, "syncarenaindex mismatched entry");
|
||||
fprint(2, "\nmismatched index entry and clump at %d\n", clump + i);
|
||||
fprint(2, "\tclump: type=%d size=%d blocks=%d addr=%lld\n", ia.type, ia.size, ia.blocks, ia.addr);
|
||||
fprint(2, "\tindex: type=%d size=%d block=%d addr=%lld\n", ie.ia.type, ie.ia.size, ie.ia.blocks, ie.ia.addr);
|
||||
pack = readlump(ie.score, ie.ia.type, ie.ia.size, nil);
|
||||
packetfree(pack);
|
||||
if(pack != nil){
|
||||
fprint(2, "duplicated lump\n");
|
||||
continue;
|
||||
}
|
||||
wrong++;
|
||||
}else
|
||||
continue;
|
||||
Add:
|
||||
if(!fix){
|
||||
ok = -1;
|
||||
continue;
|
||||
}
|
||||
flush = 1;
|
||||
trace(TraceProc, "syncarenaindex insert %V", ci->score);
|
||||
insertscore(ci->score, &ia, IEDirty);
|
||||
}
|
||||
|
||||
if(0 && clump / 1000 != (clump + n) / 1000)
|
||||
fprint(2, ".");
|
||||
}
|
||||
free(cis);
|
||||
free(addrs);
|
||||
free(s);
|
||||
if(flush){
|
||||
flushdcache();
|
||||
*pflush = 1;
|
||||
as.stats.used += ClumpSize + ci.size;
|
||||
as.stats.uncsize += ia.size;
|
||||
as.stats.clumps++;
|
||||
if(ci.uncsize > ci.size)
|
||||
as.stats.cclumps++;
|
||||
as.aa = a;
|
||||
insertscore(ci.score, &ia, IEDirty, &as);
|
||||
}
|
||||
flushdcache();
|
||||
return ok;
|
||||
}
|
||||
|
||||
int
|
||||
syncindex(Index *ix, int fix, int mustflush, int check)
|
||||
syncindex(Index *ix)
|
||||
{
|
||||
Arena *arena;
|
||||
AState as;
|
||||
u64int a;
|
||||
int i, e, e1, ok, ok1, flush;
|
||||
int i, e, e1, ok;
|
||||
|
||||
ok = 0;
|
||||
flush = 0;
|
||||
for(i = 0; i < ix->narenas; i++){
|
||||
trace(TraceProc, "syncindex start %d", i);
|
||||
arena = ix->arenas[i];
|
||||
/*
|
||||
* Syncarena will scan through the arena looking for blocks
|
||||
* that have been forgotten. It will update arena->memstats.used,
|
||||
* so save the currenct copy as the place to start the
|
||||
* syncarenaindex scan.
|
||||
*/
|
||||
a = arena->memstats.used;
|
||||
e = syncarena(arena, ix->amap[i].start, TWID32, fix, fix);
|
||||
e = syncarena(arena, TWID32, 1, 1);
|
||||
e1 = e;
|
||||
if(fix)
|
||||
e1 &= ~(SyncHeader|SyncCIZero|SyncCIErr);
|
||||
if(e1 == SyncHeader)
|
||||
e1 &= ~(SyncHeader|SyncCIZero|SyncCIErr);
|
||||
if(e & SyncHeader)
|
||||
fprint(2, "arena %s: header is out-of-date\n", arena->name);
|
||||
if(e1)
|
||||
if(e1){
|
||||
fprint(2, "arena %s: %x\n", arena->name, e1);
|
||||
ok = -1;
|
||||
else{
|
||||
/*
|
||||
* use diskstats not memstats here, because diskstats
|
||||
* is what has been indexed; memstats is what has
|
||||
* made it to disk (confusing names).
|
||||
*/
|
||||
ok1 = syncarenaindex(ix, arena,
|
||||
arena->diskstats.clumps,
|
||||
ix->amap[i].start + arena->diskstats.used,
|
||||
fix, &flush, check);
|
||||
if(ok1 < 0)
|
||||
fprint(2, "syncarenaindex: %r\n");
|
||||
if(fix && ok1==0 && (e & SyncHeader) && wbarena(arena) < 0)
|
||||
fprint(2, "arena=%s header write failed: %r\n", arena->name);
|
||||
ok |= ok1;
|
||||
|
||||
as.arena = arena;
|
||||
as.aa = ix->amap[i].start + arena->memstats.used;
|
||||
as.stats = arena->memstats;
|
||||
setdcachestate(&as);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if(missing || wrong)
|
||||
fprint(2, "syncindex: %d missing entries, %d wrong entries (flush=%d)\n", missing, wrong, flush);
|
||||
if(fix && wbindex(ix) < 0){
|
||||
fprint(2, "can't write back index header for %s: %r\n", ix->name);
|
||||
return -1;
|
||||
}
|
||||
if(fix && flush){
|
||||
flushdcache();
|
||||
if(mustflush){
|
||||
flushicache();
|
||||
flushdcache();
|
||||
}else
|
||||
kickicache();
|
||||
|
||||
if(arena->memstats.clumps == arena->diskstats.clumps)
|
||||
continue;
|
||||
|
||||
fprint(2, "%T %s: indexing %d clumps...\n",
|
||||
arena->name,
|
||||
arena->memstats.clumps - arena->diskstats.clumps);
|
||||
|
||||
if(syncarenaindex(arena, ix->amap[i].start) < 0){
|
||||
fprint(2, "arena %s: syncarenaindex: %r\n", arena->name);
|
||||
ok = -1;
|
||||
continue;
|
||||
}
|
||||
if(wbarena(arena) < 0){
|
||||
fprint(2, "arena %s: wbarena: %r\n", arena->name);
|
||||
ok = -1;
|
||||
continue;
|
||||
}
|
||||
flushdcache();
|
||||
delaykickicache();
|
||||
}
|
||||
return ok;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -161,7 +161,7 @@ threadmain(int argc, char *argv[])
|
|||
startbloomproc(mainindex->bloom);
|
||||
|
||||
fprint(2, "sync...");
|
||||
if(!readonly && syncindex(mainindex, 1, 0, 0) < 0)
|
||||
if(!readonly && syncindex(mainindex) < 0)
|
||||
sysfatal("can't sync server: %r");
|
||||
|
||||
if(!readonly && queuewrites){
|
||||
|
|
|
|||
|
|
@ -133,7 +133,6 @@ threadmain(int argc, char *argv[])
|
|||
Arena *arena;
|
||||
u64int offset, aoffset;
|
||||
Part *part;
|
||||
Dir *d;
|
||||
uchar buf[8192];
|
||||
ArenaHead head;
|
||||
ZClump zerocl;
|
||||
|
|
@ -178,9 +177,6 @@ threadmain(int argc, char *argv[])
|
|||
|
||||
statsinit();
|
||||
|
||||
if((d = dirstat(file)) == nil)
|
||||
sysfatal("can't stat file %s: %r", file);
|
||||
|
||||
part = initpart(file, OREAD);
|
||||
if(part == nil)
|
||||
sysfatal("can't open file %s: %r", file);
|
||||
|
|
@ -190,9 +186,9 @@ threadmain(int argc, char *argv[])
|
|||
if(unpackarenahead(&head, buf) < 0)
|
||||
sysfatal("corrupted arena header: %r");
|
||||
|
||||
if(aoffset+head.size > d->length)
|
||||
if(aoffset+head.size > part->size)
|
||||
sysfatal("arena is truncated: want %llud bytes have %llud\n",
|
||||
head.size, d->length);
|
||||
head.size, part->size);
|
||||
|
||||
partblocksize(part, head.blocksize);
|
||||
initdcache(8 * MaxDiskBlock);
|
||||
|
|
|
|||
Loading…
Reference in a new issue