/*
  dbz.c  V6.1.1

Copyright 1988 Jon Zeeff (zeeff@b-tech.ann-arbor.mi.us)
You can use this code in any manner, as long as you leave my name on it
and don't hold me responsible for any problems with it.

Hacked on by gdb@ninja.UUCP (David Butler); Sun Jun  5 00:27:08 CDT 1988

Various improvments + INCORE by moraes@ai.toronto.edu (Mark Moraes)

Major reworking by Henry Spencer as part of the C News project.

Minor lint and CodeCenter (Saber) fluff removal by Rich $alz (March, 1991).
Non-portable CloseOnExec() calls added by Rich $alz (September, 1991).
Added "writethrough" and tagmask calculation code from
<rob@violet.berkeley.edu> and <leres@ee.lbl.gov> by Rich $alz (December, 1992).
Merged in MMAP code by David Robinson, formerly <david@elroy.jpl.nasa.gov>
now <david.robinson@sun.com> (January, 1993).

Major reworking by Clayton O'Neill (coneill@oneill.net).  Removed all the
C News and backwards compatible cruft.  Ripped out all the tagmask stuff
and replaced it with hashed .pag entries.  This removes the need for
base file access.  Primary bottleneck now appears to be the hash
algorithm and search().  You can change DBZ_INTERNAL_HASH_SIZE in
dbz.h to increase the size of the stored hash.

These routines replace dbm as used by the usenet news software
(it's not a full dbm replacement by any means).  It's fast and
simple.  It contains no AT&T code.

The dbz database exploits the fact that when news stores a <key,value>
tuple, the `value' part is a seek offset into a text file, pointing to
a copy of the `key' part.  This avoids the need to store a copy of
the key in the dbz files. 

The basic format of the database is two hash tables, each in it's own
file. One contains the offsets into the history text file , and the
other contains a hash of the message id.  A value is stored by
indexing into the tables using a hash value computed from the key;
collisions are resolved by linear probing (just search forward for an
empty slot, wrapping around to the beginning of the table if
necessary).  Linear probing is a performance disaster when the table
starts to get full, so a complication is introduced.  Each file actually
contains one *or more* tables, stored sequentially in the files, and
the length of the linear-probe sequences is limited.  The search (for
an existing item or an empy slot always starts in the first table of
the hash file, and whenever MAXRUN probes have been done in table N,
probing continues in table N+1.  It is best not to overflow into more
than 1-2 tables or else massive performance degradation may occur.
Choosing the size of the database is extremely important because of this.

The table size is fixed for any particular database, but is determined
dynamically when a database is rebuilt.  The strategy is to try to pick
the size so the first table will be no more than 2/3 full, that being
slightly before the point where performance starts to degrade.  (It is
desirable to be a bit conservative because the overflow strategy tends
to produce files with holes in them, which is a nuisance.)

Tagged hash + offset fuzzy technique merged by Sang-yong Suh (Nov, 1997)

Fixed a bug handling larger than 1Gb history offset by Sang-yong Suh(1998)
Similar fix was suggested by Mike Hucka <hucka@umich.edu> (Jan, 1998) for
dbz-3.3.2.

Limited can't tag warnings once per dbzinit() by Sang-yong Suh (May, 1998)

*/

#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <ctype.h>
#include <errno.h>
#include <netinet/in.h>
#include <sys/mman.h>
#include <configdata.h>
#include <clibrary.h>
#include <md5.h>
#include <libinn.h>
#include <macros.h>
#include <dbz.h>

/*
 * "LIA" = "leave it alone unless you know what you're doing".
 *
 * DBZDEBUG	enable debugging
 * DBZTEST      Generate a standalone program for testing and benchmarking
 * DEFSIZE	default table size (not as critical as in old dbz)
 * NMEMORY	number of days of memory for use in sizing new table (LIA)
 * MAXRUN	length of run which shifts to next table (see below) (LIA)
 */

static int dbzversion = 6;	/* for validating .dir file format */

#define of_t		long
#define	SOF		(sizeof(of_t))
#define	NOTFOUND	((OFFSET_T)-1)
#ifdef	DO_TAGGED_HASH

#define OVERFLOW
#ifdef OVERFLOW
#include <limits.h>
#endif

/* MAXDROPBITS is the maximum number of bits dropped from the offset value.
   The least significant bits are dropped.  The space is used to
   store hash additional bits, thereby increasing the possibility of the
   hash detection */
#define MAXDROPBITS	4	/* max # of bits to drop from the offset */

/* MAXFUZZYLENGTH is the maximum in the offset value due to the MAXDROPBITS */
#define MAXFUZZYLENGTH ((1 << MAXDROPBITS) - 1)

/*
 * We assume that unused areas of a binary file are zeros, and that the
 * bit pattern of `(of_t)0' is all zeros.  The alternative is rather
 * painful file initialization.  Note that okayvalue(), if OVERFLOW is
 * defined, knows what value of an offset would cause overflow.
 */
#define VACANT		((OFFSET_T)0)
#define BIAS(o)		((o)+1)		/* make any valid of_t non-VACANT */
#define UNBIAS(o)	((o)-1)		/* reverse BIAS() effect */

#define HASTAG(o)	((o)&taghere)
#define TAG(o)		((o)&tagbits)
#define NOTAG(o)	((o)&~tagboth)
#define CANTAG(o)	(((o)&tagboth) == 0)
#define MKTAG(v)	(((v)<<conf.tagshift)&tagbits)

#ifndef NOTAGS
#define TAGENB		0x80	/* tag enable is top bit, tag is next 7 */
#define TAGMASK 	0x7f
#define TAGSHIFT	24
#else
#define TAGENB		0	/* no tags */
#define TAGMASK 	0
#define TAGSHIFT	0
#endif

/*
 * Stdio buffer for base-file reads.  Message-IDs (all news ever needs to
 * read) are essentially never longer than 64 bytes, and the typical stdio
 * buffer is so much larger that it is much more expensive to fill.
 */
#ifndef SHISTBUF
#define SHISTBUF	64
#endif
#ifdef	_IOFBF
static char basebuf[SHISTBUF];          /* only needed if _IOFBF exists */
#endif

static OFFSET_T tagbits;		/* pre-shifted tag mask */
static OFFSET_T taghere;		/* pre-shifted tag-enable bit */
static OFFSET_T tagboth;		/* tagbits|taghere */
static int canttag_warned;		/* flag to control can't tag warning */

#endif	/* DO_TAGGED_HASH */

#ifdef MAP_FILE
#define MAP__ARG	(MAP_FILE | MAP_SHARED)
#else
#define MAP__ARG	(MAP_SHARED)
#endif

/* Old dbz used a long as the record type for dbz entries, which became
 * really gross in places because of mixed references.  We define these to
 * make it a bit easier if we want to store more in here.
 */

/* A new, from-scratch database, not built as a rebuild of an old one,
 * needs to know table size.  Normally the user supplies this info,
 * but there have to be defaults.
 */
#ifndef DEFSIZE

#ifdef	DO_TAGGED_HASH
#define DEFSIZE		500009		/* I need a prime number */
#else
#define	DEFSIZE		750000
#endif

#endif	/* DEFSIZE */
/*
 * We read configuration info from the .dir file into this structure,
 * so we can avoid wired-in assumptions for an existing database.
 *
 * Among the info is a record of recent peak usages, so that a new table
 * size can be chosen intelligently when rebuilding.  10 is a good
 * number of usages to keep, since news displays marked fluctuations
 * in volume on a 7-day cycle.
 */
#ifndef NMEMORY
#define	NMEMORY	10	/* # days of use info to remember */
#endif
#define	NUSEDS	(1+NMEMORY)

typedef struct {
    long tsize;		        /* table size */
    long used[NUSEDS];          /* entries used today, yesterday, ... */
    long vused[NUSEDS];		/* ditto for text size */
    int valuesize;		/* size of table values, == sizeof(dbzrec) */
    int fillpercent;            /* fillpercent/100 is the percent full we'll
				   try to keep the .pag file */
    of_t tagenb;		/* unshifted tag-enable bit */
    of_t tagmask;		/* unshifted tag mask */
    int tagshift;		/* shift count for tagmask and tagenb */
    int dropbits;		/* number of bits to discard from offset */
    int lenfuzzy;		/* num of fuzzy characters in offset */
} dbzconfig;
static dbzconfig conf;

/*
 * Default dbzoptions to
 */
static dbzoptions options = {
    FALSE,		/* write through off */
    INCORE_NO,		/* index/pag from disk */
#ifdef HAVE_MMAP
    INCORE_MMAP,	/* exists mmap'ed. ignored in tagged hash mode */
#else
    INCORE_NO,		/* exists from disk. ignored in tagged hash mode */
#endif
    TRUE		/* non-blocking writes */
};

/*
 * Data structure for recording info about searches.
 */
typedef struct {
    OFFSET_T place;		/* current location in file */
    int tabno;		        /* which table we're in */
    int run;		        /* how long we'll stay in this table */
#		ifndef MAXRUN
#		define	MAXRUN	100
#		endif
    HASH hash;			/* the key's hash code */
    unsigned long shorthash;    /* integer version of the hash, used for
				   determining the entries location.
				   Tagged_hash stores the 31-bit hash here */
    OFFSET_T tag;		/* tag we are looking for */
    int aborted;		/* has i/o error aborted search? */
} searcher;
#define	FRESH	((searcher *)NULL)

/*
 * Arguably the searcher struct for a given routine ought to be local to
 * it, but a fetch() is very often immediately followed by a store(), and
 * in some circumstances it is a useful performance win to remember where
 * the fetch() completed.  So we use a global struct and remember whether
 * it is current.
 */
static searcher srch;
static searcher *prevp;	/* &srch or FRESH */

/*
 * The double parentheses needed to make this work are ugly, but the
 * alternative (under most compilers) is to pack around 2K of unused
 * strings -- there's just no way to get rid of them.
 */
#ifdef DBZDEBUG
static int debug;			/* controlled by dbzdebug() */
#define DEBUG(args) if (debug) { printf args ; } else
#else
#define	DEBUG(args)	;
#endif

/* Structure for hash tables */
typedef struct {
    int fd;                     /* Non-blocking descriptor for writes */
    int pos;                    /* Current offset into the table */
    int reclen;                 /* Length of records in the table */
    dbz_incore_val incore;      /* What we're using core for */
    void *core;                 /* Pointer to in-core table */
} hash_table;

/* central data structures */
static BOOL opendb = FALSE;     /* Indicates if a database is currently open */
static FILE *dirf;		/* descriptor for .dir file */
static BOOL readonly;		/* database open read-only? */
#ifdef	DO_TAGGED_HASH
static FILE *basef;		/* descriptor for base file */
static char *basefname;		/* name for not-yet-opened base file */
static hash_table pagtab;       /* pag hash table, stores hash + offset */
#else
static hash_table idxtab;       /* index hash table, used for data retrieval */
static hash_table etab;         /* existance hash table, used for existance checks */
#endif
static BOOL dirty;		/* has a store() been done? */
static erec empty_rec;          /* empty rec to compare against
				   initalized in dbzinit */

/* misc. forwards */
static char *enstring(const char *s1, const char *s2);
static BOOL getcore(hash_table *tab);
static BOOL putcore(hash_table *tab);
static BOOL getconf(FILE *df, dbzconfig *cp);
static int  putconf(FILE *f, dbzconfig *cp);
static void start(searcher *sp, const HASH hash, searcher *osp);
#ifdef	DO_TAGGED_HASH
static OFFSET_T search(searcher *sp);
static BOOL set_pag(searcher *sp, OFFSET_T value);
#else
static BOOL search(searcher *sp);
#endif
static BOOL set(searcher *sp, hash_table *tab, void *value);

/* file-naming stuff */
static char dir[] = ".dir";
#ifdef	DO_TAGGED_HASH
static char pag[] = ".pag";
#else
static char idx[] = ".index";
static char exists[] = ".hash";
#endif

int dbzneedfilecount(void) {
#ifdef	DO_TAGGED_HASH
    return 2; /* basef and dirf are fopen()'ed and kept */
#else
    return 1; /* dirf is fopen()'ed and kept */
#endif
}

#ifdef	DO_TAGGED_HASH
/*
 - dbzconfbase - reconfigure dbzconf from base file size.
 */
static void config_by_text_size(dbzconfig *c, of_t basesize) {
    int i;
    unsigned long m;

    /* if no tag requested, just return. */
    if ((c->tagmask | c->tagenb) == 0)
	return;

    /* Use 10 % larger base file size.  Sometimes the offset overflows */
    basesize += basesize / 10;

    /* calculate tagging from old file */
    for (m = 1, i = 0; m < basesize; i++, m <<= 1)
	continue;
 
    /* if we had more tags than the default, use the new data */
    c->dropbits = 0;
    while (m > (1 << TAGSHIFT)) {
	if (c->dropbits >= MAXDROPBITS)
	    break;
	c->dropbits++;
	m >>= 1;
	i--;
    }
    c->tagenb = TAGENB;
    c->tagmask = TAGMASK;
    c->tagshift = TAGSHIFT;
    if ((c->tagmask | c->tagenb) && m > (1 << TAGSHIFT)) {
	c->tagshift = i;
	c->tagmask = (~(unsigned long)0) >> (i + 1);
	c->tagenb = (c->tagmask << 1) & ~c->tagmask;
    }
    c->lenfuzzy = (int)(1 << c->dropbits) - 1;

    m = (c->tagmask | c->tagenb) << c->tagshift;
    if (m & (basesize >> c->dropbits)) {
	fprintf(stderr, "m 0x%lx size 0x%lx\n", m, basesize);
	exit(1);
    }
}
#endif	/* DO_TAGGED_HASH */

/*
 - create and truncate .pag, .idx, or .hash files
 - return FALSE on error
 */
static BOOL create_truncate(const char *name, const char *pag) {
    char *fn;
    FILE *f;

    if ((fn = enstring(name, pag)) == NULL)
	return FALSE;
    f = Fopen(fn, "w", TEMPORARYOPEN);
    DISPOSE(fn);
    if (f == NULL) {
	DEBUG(("dbz.c create_truncate: unable to create/truncate %s\n", pag));
	return FALSE;
    } else
        Fclose(f);
    return TRUE;
}

/* dbzfresh - set up a new database, no historical info
 * Return TRUE for success, FALSE for failure
 * name - base name; .dir and .pag must exist
 * size - table size (0 means default)
 * fillpercent - target percentage full ***** inactive
 */
BOOL dbzfresh(const char *name, const long size, const int fillpercent)
{
    char *fn;
    dbzconfig c;
    FILE *f;
#ifdef	DO_TAGGED_HASH
    struct stat sb;
    of_t m;
#endif

    if (opendb) {
	DEBUG(("dbzfresh: database already open\n"));
	return FALSE;
    }
    if (size != 0 && size < 2) {
	DEBUG(("dbzfresh: preposterous size (%ld)\n", size));
	return FALSE;
    }

    /* get default configuration */
    if (!getconf((FILE *)NULL, &c))
	return FALSE;	/* "can't happen" */

#ifdef	DO_TAGGED_HASH
    /* and mess with it as specified */
    if (size != 0)
	c.tsize = size;
    m = c.tagmask;
    c.tagshift = 0;
    while (!(m & 1)) {
	m >>= 1;
	c.tagshift++;
    }
    c.tagmask = m;
    c.tagenb = (m << 1) & ~m;
    c.dropbits = 0;
    c.lenfuzzy = 0;

    /* if big enough basb file exists, update config */
    if (stat(name, &sb) != -1)
	config_by_text_size(&c, sb.st_size);
#else
    /* set the size as specified, make sure we get at least 2 bytes
       of implicit hash */
    if (size != 0)
	c.tsize = size > (64 * 1024) ? size : 64 * 1024;
#endif

    /* write it out */
    if ((fn = enstring(name, dir)) == NULL)
	return FALSE;
    f = Fopen(fn, "w", TEMPORARYOPEN);
    DISPOSE(fn);
    if (f == NULL) {
	DEBUG(("dbzfresh: unable to write config\n"));
	return FALSE;
    }
    if (putconf(f, &c) < 0) {
	Fclose(f);
	return FALSE;
    }
    if (Fclose(f) == EOF) {
	DEBUG(("dbzfresh: fclose failure\n"));
	return FALSE;
    }

    /* create and truncate .pag, or .index/.hash files */
#ifdef	DO_TAGGED_HASH
    if (!create_truncate(name, pag))
	return FALSE;
#else
    if (!create_truncate(name, idx))
	return FALSE;
    if (!create_truncate(name, exists))
	return FALSE;
#endif	/* DO_TAGGED_HASH */

    /* and punt to dbzinit for the hard work */
    return dbzinit(name);
}

#ifdef	DO_TAGGED_HASH
/*
 - isprime - is a number prime?
 */
static BOOL isprime(long x) {
    static int quick[] = {2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 0 };
    int *ip;
    long div, stop;

    /* hit the first few primes quickly to eliminate easy ones */
    /* this incidentally prevents ridiculously small tables */
    for (ip = quick; (div = *ip) != 0; ip++)
	if (x % div == 0) {
	    DEBUG(("isprime: quick result on %ld\n", (long)x));
	    return FALSE;
	}

    /* approximate square root of x */
    for (stop = x; x/stop < stop; stop >>= 1)
	continue;
    stop <<= 1;

    /* try odd numbers up to stop */
    for (div = *--ip; div < stop; div += 2)
	if (x%div == 0)
	    return FALSE;

    return TRUE;
}
#endif

/*
 * dbzsize  - what's a good table size to hold this many entries?
 * contents - size of table (0 means return the default)
 */
long dbzsize(const long contents) {
    long n;

    if (contents <= 0) {	/* foulup or default inquiry */
	DEBUG(("dbzsize: preposterous input (%ld)\n", contents));
	return DEFSIZE;
    }

    if ((conf.fillpercent > 0) && (conf.fillpercent < 100))
	n = (contents / conf.fillpercent) * 100;
    else 
	n = contents * 2;	/* try to keep table at most half full */

    /* Make sure that we get at least 2 bytes of implicit hash */
    if (n < (64 * 1024))
	n = 64 * 1024;

#ifdef	DO_TAGGED_HASH
    if (!(n & 1))
        n += 1;			/* make it odd */
    DEBUG(("dbzsize: tentative size %ld\n", n));
    while (!isprime(n))		/* look for a prime */
	n += 2;
#endif

    DEBUG(("dbzsize: final size %ld\n", n));
    return n;
}

/* dbzagain - set up a new database to be a rebuild of an old one
 * Returns TRUE on success, FALSE on failure
 * name - base name; .dir and .pag must exist
 * oldname - basename, all must exist
 */
BOOL dbzagain(const char *name, const char *oldname)
{
    char *fn;
    dbzconfig c;
    BOOL result;
    int i;
    long top;
    FILE *f;
    int newtable;
    OFFSET_T newsize;
#ifdef	DO_TAGGED_HASH
    long vtop;
    struct stat sb;
#endif

    if (opendb) {
	DEBUG(("dbzagain: database already open\n"));
	return FALSE;
    }

    /* pick up the old configuration */
    if ((fn = enstring(oldname, dir))== NULL)
	return FALSE;
    f = Fopen(fn, "r", TEMPORARYOPEN);
    DISPOSE(fn);
    if (f == NULL) {
	DEBUG(("dbzagain: cannot open old .dir file\n"));
	return FALSE;
    }
    result = getconf(f, &c);
    Fclose(f);
    if (!result) {
	DEBUG(("dbzagain: getconf failed\n"));
	return FALSE;
    }

    /* tinker with it */
    top = 0;
    newtable = 0;
    for (i = 0; i < NUSEDS; i++) {
	if (top < c.used[i])
	    top = c.used[i];
	if (c.used[i] == 0)
	    newtable = 1;	/* hasn't got full usage history yet */
    }
    if (top == 0) {
	DEBUG(("dbzagain: old table has no contents!\n"));
	newtable = 1;
    }
    for (i = NUSEDS-1; i > 0; i--)
	c.used[i] = c.used[i-1];
    c.used[0] = 0;

#ifdef	DO_TAGGED_HASH
    vtop = 0;
    for (i = 0; i < NUSEDS; i++) {
	if (vtop < c.vused[i])
	    vtop = c.vused[i];
	if (c.vused[i] == 0)
	    newtable = 1;	/* hasn't got full usage history yet */
    }
    if (top != 0 && vtop == 0) {
	DEBUG(("dbzagain: old table has no contents!\n"));
	newtable = 1;
    }
    for (i = NUSEDS-1; i > 0; i--)
	c.vused[i] = c.vused[i-1];
    c.vused[0] = 0;

    /* calculate tagging from old file */
    if (stat(oldname, &sb) != -1 && vtop < sb.st_size)
	vtop = sb.st_size;
    config_by_text_size(&c, vtop);
#endif

    newsize = dbzsize(top);
    if (!newtable || newsize > c.tsize)	/* don't shrink new table */
	c.tsize = newsize;

    /* write it out */
    fn = enstring(name, dir);
    if (fn == NULL)
	return FALSE;
    f = Fopen(fn, "w", TEMPORARYOPEN);
    DISPOSE(fn);
    if (f == NULL) {
	DEBUG(("dbzagain: unable to write new .dir\n"));
	return FALSE;
    }
    i = putconf(f, &c);
    Fclose(f);
    if (i < 0) {
	DEBUG(("dbzagain: putconf failed\n"));
	return FALSE;
    }

    /* create and truncate .pag, or .index/.hash files */
#ifdef	DO_TAGGED_HASH
    if (!create_truncate(name, pag))
	return FALSE;
#else
    if (!create_truncate(name, idx))
	return FALSE;
    if (!create_truncate(name, exists))
	return FALSE;
#endif

    /* and let dbzinit do the work */
    return dbzinit(name);
}

static BOOL openhashtable(const char *base, const char *ext, hash_table *tab,
			  const size_t reclen, const dbz_incore_val incore) {
    char *name;

    if ((name = enstring(base, ext)) == NULL)
	return FALSE;

    if ((tab->fd = open(name, readonly ? O_RDONLY : O_RDWR)) < 0) {
	DEBUG(("openhashtable: could not open raw\n"));
	DISPOSE(name);
	errno = EDOM;
	return FALSE;
    }
    DISPOSE(name);

    tab->reclen = reclen;
    CloseOnExec(tab->fd, 1);
    tab->pos = -1;

    /* get first table into core, if it looks desirable and feasible */
    tab->incore = incore;
    if (tab->incore != INCORE_NO) {
	if (!getcore(tab)) {
	    DEBUG(("openhashtable: getcore failure\n"));
	    close(tab->fd);
	    errno = EDOM;
	    return FALSE;
	}
    }

    if (options.nonblock && (SetNonBlocking(tab->fd, TRUE) < 0)) {
	DEBUG(("fcntl: could not set nonblock\n"));
	close(tab->fd);
	errno = EDOM;
	return FALSE;
    }
    return TRUE;
}

static void closehashtable(hash_table *tab) {
    close(tab->fd);
    if (tab->incore == INCORE_MEM)
	DISPOSE(tab->core);
    if (tab->incore == INCORE_MMAP) {
#if defined(HAVE_MMAP)
	if (munmap((MMAP_PTR) tab->core, (int)conf.tsize * tab->reclen) == -1) {
	    DEBUG(("closehashtable: munmap failed\n"));
	}
#else
	DEBUG(("closehashtable: can't mmap files\n"));
#endif
    }
}

#ifdef	DO_TAGGED_HASH
static BOOL openbasefile(const char *name) {
    basef = Fopen(name, "r", DBZ_BASE);
    if (basef == NULL) {
	DEBUG(("dbzinit: basefile open failed\n"));
	basefname = enstring(name, "");
	if (basefname == NULL) {
	    (void) Fclose(dirf);
	    return FALSE;
	}
    } else
	basefname = NULL;
    if (basef != NULL)
	CloseOnExec((int)fileno(basef), 1);
#ifdef _IOFBF
    if (basef != NULL)
	(void) setvbuf(basef, basebuf, _IOFBF, sizeof(basebuf));
#endif
    return TRUE;
}
#endif	/* DO_TAGGED_HASH */

/*
 * dbzinit - open a database, creating it (using defaults) if necessary
 *
 * We try to leave errno set plausibly, to the extent that underlying
 * functions permit this, since many people consult it if dbzinit() fails.
 * return TRUE for success, FALSE for failure
 */
int dbzinit(const char *name) {
    char *fname;

    if (opendb) {
	DEBUG(("dbzinit: dbzinit already called once\n"));
	errno = 0;
	return FALSE;
    }

    /* open the .dir file */
    if ((fname = enstring(name, dir)) == NULL)
	return FALSE;
    if ((dirf = Fopen(fname, "r+", DBZ_DIR)) == NULL) {
	dirf = Fopen(fname, "r", DBZ_DIR);
	readonly = TRUE;
    } else
	readonly = FALSE;
    DISPOSE(fname);
    if (dirf == NULL) {
	DEBUG(("dbzinit: can't open .dir file\n"));
	return FALSE;
    }
    CloseOnExec(fileno(dirf), 1);

    /* pick up configuration */
    if (!getconf(dirf, &conf)) {
	DEBUG(("dbzinit: getconf failure\n"));
	Fclose(dirf);
	errno = EDOM;	/* kind of a kludge, but very portable */
	return FALSE;
    }

    /* open pag or idx/exists file */
#ifdef	DO_TAGGED_HASH
    if (!openhashtable(name, pag, &pagtab, SOF, options.pag_incore)) {
	Fclose(dirf);
	return FALSE;
    }
    if (!openbasefile(name)) {
	close(pagtab.fd);
	Fclose(dirf);
	return FALSE;
    }
    tagbits = conf.tagmask << conf.tagshift;
    taghere = conf.tagenb << conf.tagshift;
    tagboth = tagbits | taghere;
    canttag_warned = 0;
#else
    if ((innconf == NULL) && (ReadInnConf() < 0)) {
	Fclose(dirf);
	return FALSE;
    }
    if (!openhashtable(name, idx, &idxtab, (innconf->extendeddbz ?
		sizeof(idxrecext) : sizeof(idxrec)), options.idx_incore)) {
	Fclose(dirf);
	return FALSE;
    }
    if (!openhashtable(name, exists, &etab, sizeof(erec),
		options.exists_incore)) {
	Fclose(dirf);
	return FALSE;
    }
#endif

    /* misc. setup */
    dirty = FALSE;
    opendb = TRUE;
    prevp = FRESH;
    memset(&empty_rec, '\0', sizeof(empty_rec));
    DEBUG(("dbzinit: succeeded\n"));
    return TRUE;
}

/* enstring - concatenate two strings into newly allocated memory
 * Returns NULL on failure
 */
static char *enstring(const char *s1, const char *s2)
{
    char *p;

    p = NEW(char, strlen(s1) + strlen(s2) + 1);
    strcpy(p, s1);
    strcat(p, s2);
    return p;
}

/* dbzclose - close a database
 */
BOOL dbzclose(void)
{
    BOOL ret = TRUE;

    if (!opendb) {
	DEBUG(("dbzclose: not opened!\n"));
	return FALSE;
    }

    if (dbzsync() < 0)
	ret = FALSE;

#ifdef	DO_TAGGED_HASH
    closehashtable(&pagtab);
    if (Fclose(basef) == EOF) {
	fprintf(stderr, "dbzclose: fclose(basef) failed\n");
	ret = FALSE;
    }
    if (basefname != NULL)
	DISPOSE(basefname);
    basef = NULL;
#else
    closehashtable(&idxtab);
    closehashtable(&etab);
#endif

    if (Fclose(dirf) == EOF) {
	DEBUG(("dbzclose: fclose(dirf) failed\n"));
	ret = FALSE;
    }

    DEBUG(("dbzclose: %s\n", (ret == 0) ? "succeeded" : "failed"));
    if (ret)
	opendb = FALSE;
    return ret;
}

/* dbzsync - push all in-core data out to disk
 */
BOOL dbzsync(void)
{
    BOOL ret = TRUE;

    if (!opendb) {
	DEBUG(("dbzsync: not opened!\n"));
	return FALSE;
    }

    if (!dirty)
	return TRUE;;

    if (!options.writethrough) {
#ifdef	DO_TAGGED_HASH
	if (!putcore(&pagtab)) {
#else
	if (!putcore(&idxtab) || !putcore(&etab)) {
#endif
	    DEBUG(("dbzsync: putcore failed\n"));
	    ret = FALSE;;
	}
    }

    if (putconf(dirf, &conf) < 0)
	ret = FALSE;

    DEBUG(("dbzsync: %s\n", ret ? "succeeded" : "failed"));
    return ret;
}

#ifdef	DO_TAGGED_HASH
/*
 - okayvalue - check that a value can be stored
 */
static int okayvalue(of_t value) {
    if (HASTAG(value))
	return(0);
#ifdef OVERFLOW
    if (value == LONG_MAX)	/* BIAS() and UNBIAS() will overflow */
	return(0);
#endif
    return(1);
}
#endif

/* dbzexists - check if the given message-id is in the database */
BOOL dbzexists(const HASH key) {

#ifdef	DO_TAGGED_HASH
    return (dbzfetch(key) != NOTFOUND);
#else
    
    if (!opendb) {
	DEBUG(("dbzexists: database not open!\n"));
	return FALSE;
    }

    prevp = FRESH;
    start(&srch, key, FRESH);
    return search(&srch);
#endif
}

/*
 * dbzfetch - get offset of an entry from the database
 *
 * Returns the offset of the text file for input key,
 * or -1 if NOTFOUND or error occurs.
 */
#ifdef	DO_TAGGED_HASH
OFFSET_T dbzfetch(const HASH key) {
#define	MAX_NB2RD	(DBZMAXKEY + MAXFUZZYLENGTH + 2)
#define MIN_KEY_LENGTH	6	/* strlen("<1@a>") + strlen("\t") */
    char *bp, buffer[MAX_NB2RD];
    int keylen, j, nb2r;
    HASH hishash;
    char *keytext = NULL;
    OFFSET_T offset = NOTFOUND;
#else
BOOL dbzfetch(const HASH key, void *ivalue) {
    idxrec	*ionevalue = (idxrec *)ivalue;
    idxrecext	*iextvalue = (idxrecext *)ivalue;
#endif

    prevp = FRESH;

    if (!opendb) {
	DEBUG(("dbzfetch: database not open!\n"));
#ifdef	DO_TAGGED_HASH
	return NOTFOUND;
#else
	return FALSE;
#endif
    }

    start(&srch, key, FRESH);
#ifdef	DO_TAGGED_HASH
    /*
     * nb2r: number of bytes to read from history file.
     *       It can be reduced if history text is written as hashes only.
     */
    nb2r = sizeof(buffer) - 1;

    while ((offset = search(&srch)) != NOTFOUND) {
	DEBUG(("got 0x%lx\n", key_ptr));

	/* fetch the key */
	offset <<= conf.dropbits;
	if (offset)		/* backspace 1 character to read '\n' */
	    offset--;
	if (fseek(basef, offset, SEEK_SET) != 0) {
	    DEBUG(("fetch: seek failed\n"));
	    return NOTFOUND;
	}
	keylen = fread((POINTER)buffer, 1, nb2r, basef);
	if (keylen < MIN_KEY_LENGTH) {
	    DEBUG(("fetch: read failed\n"));
	    return NOTFOUND;
	}
	buffer[keylen] = '\0';	/* terminate the string */

	if (offset) {		/* find the '\n', the previous EOL */
	    if (keylen > conf.lenfuzzy)
		keylen = conf.lenfuzzy; /* keylen is fuzzy distance now */
	    for (j=0,bp=buffer; j<keylen; j++,bp++)
		if (*bp == '\n')
		    break;
	    if (*bp != '\n') {
		DEBUG(("dbzfetch: can't locate EOL\n"));
		/* pag entry should be deleted, but I'm lazy... */
		continue;
	    }
	    offset++;
	    bp++;	/* now *bp is the start of key */
	} else {
	    j = 0;	/* We are looking for the first history line. */
	    bp = buffer;
	}

	/* Does this history line really have same key? */
	if (*bp == '[') {
	    if (!keytext)
		keytext = HashToText(key);
	    if (memcmp(keytext, bp+1, sizeof(HASH)*2) != 0)
		continue;
	} else if (*bp == '<') {
	    char *p = strchr(bp+1, '\t');
	    if (!p)		/* history has a corrupted line */
		continue;
	    *p = '\0';
	    hishash = HashMessageID(bp);
	    if (memcmp(&key, &hishash, sizeof(HASH)) != 0)
		continue;
	} else
	    continue;

	/* we found it */
	offset += j;
	DEBUG(("fetch: successful\n"));
	return offset;
    }

    /* we didn't find it */
    DEBUG(("fetch: failed\n"));
    prevp = &srch;			/* remember where we stopped */
    return NOTFOUND;
#else	/* DO_TAGGED_HASH */
    if (search(&srch) == TRUE) {
	/* Actually get the data now */
	if ((options.idx_incore != INCORE_NO) && (srch.place < conf.tsize)) {
	    if (innconf->extendeddbz)
		memcpy(ivalue, &((idxrecext *)idxtab.core)[srch.place], sizeof(idxrecext));
	    else
		memcpy(ivalue, &((idxrec *)idxtab.core)[srch.place], sizeof(idxrec));
	} else {
	    if (lseek(idxtab.fd, srch.place * idxtab.reclen, SEEK_SET) < 0) {
		DEBUG(("fetch: seek failed\n"));
		idxtab.pos = -1;
		srch.aborted = 1;
		return FALSE;
	    }
	    if (innconf->extendeddbz) {
		if (read(idxtab.fd, ivalue, sizeof(idxrecext)) != sizeof(idxrecext)) {
		    DEBUG(("fetch: read failed\n"));
		    idxtab.pos = -1;
		    srch.aborted = 1;
		    return FALSE;
		}
	    } else {
		if (read(idxtab.fd, ivalue, sizeof(idxrec)) != sizeof(idxrec)) {
		    DEBUG(("fetch: read failed\n"));
		    idxtab.pos = -1;
		    srch.aborted = 1;
		    return FALSE;
		}
	    }
	}
	if (innconf->extendeddbz) {
	    iextvalue->offset[HISTOFFSET] = ntohl(iextvalue->offset[HISTOFFSET]);
	    iextvalue->offset[OVEROFFSET] = ntohl(iextvalue->offset[OVEROFFSET]);
	} else
	    ionevalue->offset = ntohl(ionevalue->offset);
	DEBUG(("fetch: successful\n"));
	return TRUE;
    }

    /* we didn't find it */
    DEBUG(("fetch: failed\n"));
    prevp = &srch;			/* remember where we stopped */
    return FALSE;
#endif
}

/*
 * dbzstore - add an entry to the database
 * returns TRUE for success and FALSE for failure
 */
#ifdef	DO_TAGGED_HASH
DBZSTORE_RESULT dbzstore(const HASH key, const OFFSET_T data) {
    OFFSET_T value;
#else
DBZSTORE_RESULT dbzstore(const HASH key, void *ivalue) {
    idxrec	*ionevalue = (idxrec *)ivalue;
    idxrecext	*iextvalue = (idxrecext *)ivalue;
    erec     evalue;
    int	     offset;
#endif

    if (!opendb) {
	DEBUG(("dbzstore: database not open!\n"));
	return DBZSTORE_ERROR;
    }
    if (readonly) {
	DEBUG(("dbzstore: database open read-only\n"));
	return DBZSTORE_ERROR;
    }

#ifdef	DO_TAGGED_HASH
    /* copy the value in to ensure alignment */
    (void) memcpy((POINTER)&value, (POINTER)&data, SOF);

    /* update maximum offset value if necessary */
    if (value > conf.vused[0])
	conf.vused[0] = value;

    /* now value is in fuzzy format */
    value >>= conf.dropbits;
    DEBUG(("dbzstore: (%ld)\n", (long)value));

    if (!okayvalue(value)) {
	DEBUG(("dbzstore: reserved bit or overflow in 0x%lx\n", value));
	return DBZSTORE_ERROR;
    }

    /* find the place, exploiting previous search if possible */
    start(&srch, key, prevp);
    while (search(&srch) != NOTFOUND)
	continue;

    prevp = FRESH;
    conf.used[0]++;
    DEBUG(("store: used count %ld\n", conf.used[0]));
    dirty = 1;
    if (!set_pag(&srch, value))
	return DBZSTORE_ERROR;
    return DBZSTORE_OK;
#else	/* DO_TAGGED_HASH */

    /* find the place, exploiting previous search if possible */
    start(&srch, key, prevp);
    if (search(&srch) == TRUE)
	return DBZSTORE_EXISTS;

    prevp = FRESH;
    conf.used[0]++;
    DEBUG(("store: used count %ld\n", conf.used[0]));
    dirty = TRUE;

    if (innconf->extendeddbz) {
	iextvalue->offset[HISTOFFSET] = htonl(iextvalue->offset[HISTOFFSET]);
	iextvalue->offset[OVEROFFSET] = htonl(iextvalue->offset[OVEROFFSET]);
    } else
	ionevalue->offset = htonl(ionevalue->offset);
    memcpy(&evalue.hash, &srch.hash,
	   sizeof(evalue.hash) < sizeof(srch.hash) ? sizeof(evalue.hash) : sizeof(srch.hash));

    /* Set the value in the index first since we don't care if it's out of date */
    if (!set(&srch, &idxtab, ivalue))
	return DBZSTORE_ERROR;
    if (!set(&srch, &etab, &evalue))
	return DBZSTORE_ERROR;
    return DBZSTORE_OK;
#endif	/* DO_TAGGED_HASH */
}

/*
 * getconf - get configuration from .dir file
 *   df    - NULL means just give me the default 
 *   pf    - NULL means don't care about .pag 
 *   returns TRUE for success, FALSE for failure
 */
static BOOL getconf(FILE *df, dbzconfig *cp) {
    int i;

    /* empty file, no configuration known */
#ifdef	DO_TAGGED_HASH
    if (df == NULL) {
	cp->tsize = DEFSIZE;
	for (i = 0; i < NUSEDS; i++)
	    cp->used[i] = 0;
	for (i = 0; i < NUSEDS; i++)
	    cp->vused[i] = 0;
	cp->valuesize = sizeof(OFFSET_T);
	cp->fillpercent = 50;
	cp->tagenb = TAGENB;
	cp->tagmask = TAGMASK;
	cp->tagshift = TAGSHIFT;
	cp->dropbits = 0;
	cp->lenfuzzy = 0;
	DEBUG(("getconf: defaults (%ld, %c, (0x%lx/0x%lx<<%d %d))\n",
	cp->tsize, cp->casemap, cp->tagenb, 
	cp->tagmask, cp->tagshift, cp->dropbits));
	return TRUE;
    }

    i = fscanf(df, "dbz 6 %ld %d %d %ld %ld %d %d\n", &cp->tsize,
		&cp->valuesize, &cp->fillpercent, &cp->tagenb,
		&cp->tagmask, &cp->tagshift, &cp->dropbits);
    if (i != 7) {
	DEBUG(("getconf error"));
	return FALSE;
    }
    if (cp->valuesize != sizeof(OFFSET_T)) {
	DEBUG(("getconf: wrong of_t size (%d)\n", cp->valuesize));
	return FALSE;
    }
    cp->lenfuzzy = (int)(1 << cp->dropbits) - 1;
#else	/* DO_TAGGED_HASH */
    if (df == NULL) {
	cp->tsize = DEFSIZE;
	for (i = 0; i < NUSEDS; i++)
	    cp->used[i] = 0;
	cp->valuesize = sizeof(idxrec) + sizeof(erec);
	cp->fillpercent = 66;
	DEBUG(("getconf: defaults (%ld)\n", cp->tsize));
	return TRUE;
    }

    i = fscanf(df, "dbz 6 %ld %d %d\n", &cp->tsize,
		&cp->valuesize, &cp->fillpercent);
    if (i != 3) {
	DEBUG(("getconf error"));
	return FALSE;
    }
    if (cp->valuesize != (sizeof(idxrec) + sizeof(erec))) {
	DEBUG(("getconf: wrong of_t size (%d)\n", cp->valuesize));
	return FALSE;
    }
#endif	/* DO_TAGGED_HASH */
    DEBUG(("size %ld\n", cp->tsize));

    /* second line, the usages */
    for (i = 0; i < NUSEDS; i++)
	if (!fscanf(df, "%ld", &cp->used[i])) {
	    DEBUG(("getconf error\n"));
	    return FALSE;
	}
    DEBUG(("used %ld %ld %ld...\n", cp->used[0], cp->used[1], cp->used[2]));

#ifdef	DO_TAGGED_HASH
    /* third line, the text usages */
    for (i = 0; i < NUSEDS; i++)
	if (!fscanf(df, "%ld", &cp->vused[i])) {
	    DEBUG(("getconf error\n"));
	    return FALSE;
	}
    DEBUG(("vused %ld %ld %ld...\n", cp->vused[0], cp->vused[1], cp->vused[2]));
#endif	/* DO_TAGGED_HASH */

    return TRUE;
}

/* putconf - write configuration to .dir file
 * Returns: 0 for success, -1 for failure
 */
static int putconf(FILE *f, dbzconfig *cp) {
    int i;
    int ret = 0;

    if (fseek(f, (OFFSET_T)0, SEEK_SET) != 0) {
	DEBUG(("fseek failure in putconf\n"));
	ret = -1;
    }

#ifdef	DO_TAGGED_HASH
    fprintf(f, "dbz %d %ld %d %d %ld %ld %d %d\n", dbzversion, cp->tsize,
		cp->valuesize, cp->fillpercent, cp->tagenb,
		cp->tagmask, cp->tagshift, cp->dropbits);
#else	/* DO_TAGGED_HASH */
    fprintf(f, "dbz %d %ld %d %d\n", dbzversion, cp->tsize,
		   cp->valuesize, cp->fillpercent);
#endif	/* DO_TAGGED_HASH */

    for (i = 0; i < NUSEDS; i++)
	fprintf(f, "%ld%c", cp->used[i], (i < NUSEDS-1) ? ' ' : '\n');

#ifdef	DO_TAGGED_HASH
    for (i = 0; i < NUSEDS; i++)
	fprintf(f, "%ld%c", cp->vused[i], (i < NUSEDS-1) ? ' ' : '\n');
#endif

    fflush(f);
    if (ferror(f))
	ret = -1;

    DEBUG(("putconf status %d\n", ret));
    return ret;
}

/* getcore - try to set up an in-core copy of .pag file
 *
 * Returns: pointer to copy of .pag or NULL on errror
 */
static BOOL getcore(hash_table *tab) {
    char *it;
    int nread;
    int i;
    struct stat st;

    if (tab->incore == INCORE_MMAP) {
#if defined(HAVE_MMAP)
	if (fstat(tab->fd, &st) == -1) {
	    DEBUG(("getcore: fstat failed\n"));
	    return FALSE;
	}
	if ((conf.tsize * tab->reclen) > st.st_size) {
	    /* file too small; extend it */
	    if (ftruncate(tab->fd, conf.tsize * tab->reclen) == -1) {
		DEBUG(("getcore: ftruncate failed\n"));
		return FALSE;
	    }
	}
	it = mmap((caddr_t)0, (size_t)conf.tsize * tab->reclen,
		   readonly ? PROT_READ : PROT_WRITE | PROT_READ, MAP__ARG,
		   tab->fd, (OFFSET_T)0);
	if (it == (char *)-1) {
	    DEBUG(("getcore: mmap failed\n"));
	    return FALSE;
	}
#if defined (MADV_RANDOM) && !defined(_nec_ews)
	/* not present in all versions of mmap() */
	madvise(it, (size_t)conf.tsize * sizeof(tab->reclen), MADV_RANDOM);
#endif
#else
	DEBUG(("getcore: can't mmap files\n"));
	return FALSE;
#endif
    } else {
	it = NEW(char, conf.tsize * tab->reclen);
	
	nread = read(tab->fd, (POINTER)it, tab->reclen * conf.tsize);
	if (nread < 0) {
	    DEBUG(("getcore: read failed\n"));
	    DISPOSE(it);
	    return FALSE;
	}
	
	i = (size_t)(conf.tsize * tab->reclen) - nread;
	memset(it + nread, '\0', i);
    }

    tab->core = it;
    return TRUE;
}

/* putcore - try to rewrite an in-core table
 *
 * Returns TRUE on success, FALSE on failure
 */
static BOOL putcore(hash_table *tab) {
    int size;
    
    if (tab->incore == INCORE_MEM) {
	SetNonBlocking(tab->fd, FALSE);
	if (lseek(tab->fd, (OFFSET_T)0, SEEK_SET) != 0) {
	    DEBUG(("fseek failure in putcore\n"));
	    return FALSE;
	}
	size = tab->reclen * conf.tsize;
	if (write(tab->fd, (POINTER)tab->core, size) != size) {
	    SetNonBlocking(tab->fd, options.nonblock);
	    return FALSE;
	}
	SetNonBlocking(tab->fd, options.nonblock);
    }
    return TRUE;
}

#ifdef	DO_TAGGED_HASH
/*
 - makehash31 : make 31-bit hash from HASH
 */
static unsigned int makehash31(const HASH *hash) {
    unsigned int h;
    memcpy(&h, hash, sizeof(h));
    return (h >> 1);
}
#endif

/* start - set up to start or restart a search
 * osp == NULL is acceptable
 */
static void start(searcher *sp, const HASH hash, searcher *osp) {
#ifdef	DO_TAGGED_HASH
    long h;

    h = makehash31(&hash);
    if (osp != FRESH && osp->shorthash == h) {
	if (sp != osp)
	    *sp = *osp;
	sp->run--;
	DEBUG(("search restarted\n"));
    } else {
	sp->shorthash = h;
	sp->tag = MKTAG(h / conf.tsize);
	sp->place = h % conf.tsize;
	DEBUG(("hash %8.8lx tag %8.8lx place %ld\n",
	    sp->shorthash, sp->tag, sp->place));
	sp->tabno = 0;
	sp->run = -1;
	sp->aborted = 0;
    }

#else	/* DO_TAGGED_HASH */
    int tocopy;

    if (osp != FRESH && !memcmp(&osp->hash, &hash, sizeof(hash))) {
	if (sp != osp)
	    *sp = *osp;
	sp->run--;
	DEBUG(("search restarted\n"));
    } else {
	sp->hash = hash;
	tocopy = sizeof(hash) < sizeof(sp->shorthash) ? sizeof(hash) : sizeof(sp->shorthash);
	/* Copy the bottom half of thhe hash into sp->shorthash */
	memcpy(&sp->shorthash, (char *)&hash + (sizeof(hash) - tocopy), tocopy);
	sp->shorthash >>= 1;
	sp->tabno = 0;
	sp->run = -1;
	sp->aborted = 0;
    }
#endif	/* DO_TAGGED_HASH */
}

#ifdef	DO_TAGGED_HASH
/*
 - search - conduct part of a search
 */
static OFFSET_T			/* NOTFOUND if we hit VACANT or error */
search(searcher *sp)
{
    OFFSET_T value;
    unsigned long taboffset = 0;

    if (sp->aborted)
	return(NOTFOUND);

    for (;;) {
	/* go to next location */
	if (sp->run++ == MAXRUN) {
	    sp->tabno++;
	    sp->run = 0;
	    taboffset = sp->tabno * conf.tsize;
	}
	sp->place = ((sp->shorthash + sp->run) % conf.tsize) + taboffset;
	DEBUG(("search @ %ld\n", place));

	/* get the tagged value */
	if ((options.pag_incore != INCORE_NO) && (sp->place < conf.tsize)) {
	    DEBUG(("search: in core\n"));
	    value = ((OFFSET_T *)pagtab.core)[sp->place];
	} else {
	    OFFSET_T dest;
	    /* seek, if necessary */
	    dest = sp->place * SOF;
	    if (pagtab.pos != dest) {
		if (lseek(pagtab.fd, dest, SEEK_SET) == -1) {
		    DEBUG(("search: seek failed\n"));
		    pagtab.pos = -1;
		    sp->aborted = 1;
		    return(NOTFOUND);
		}
		pagtab.pos = dest;
	    }

	    /* read it */
	    errno = 0;
	    if (read(pagtab.fd, (POINTER)&value, sizeof(value)) != sizeof(value)) {
		if (errno != 0) {
		    DEBUG(("search: read failed\n"));
		    pagtab.pos = -1;
		    sp->aborted = 1;
		    return(NOTFOUND);
		} else
		    value = VACANT;
	    }

	    /* and finish up */
	    pagtab.pos += sizeof(value);
	}

	/* vacant slot is always cause to return */
	if (value == VACANT) {
	    DEBUG(("search: empty slot\n"));
	    return(NOTFOUND);
	};

	/* check the tag */
	value = UNBIAS(value);
	DEBUG(("got 0x%lx\n", value));
	if (!HASTAG(value)) {
	    DEBUG(("tagless\n"));
	    return(value);
	} else if (TAG(value) == sp->tag) {
	    DEBUG(("match\n"));
	    return(NOTAG(value));
	} else {
	    DEBUG(("mismatch 0x%lx\n", TAG(value)));
	}
    }
    /* NOTREACHED */
}

#else	/* DO_TAGGED_HASH */

/* search - conduct part of a search
 *
 * return FALSE if we hit vacant rec's or error
 */
static BOOL search(searcher *sp) {
    erec value;
    unsigned long taboffset = 0;

    if (sp->aborted)
	return FALSE;

    for (;;) {
	/* go to next location */
	if (sp->run++ == MAXRUN) {
	    sp->tabno++;
	    sp->run = 0;
	    taboffset = sp->tabno * conf.tsize;
	}

	sp->place = ((sp->shorthash + sp->run) % conf.tsize) + taboffset;
	DEBUG(("search @ %ld\n", sp->place));

	/* get the value */
	if ((options.exists_incore != INCORE_NO) && (sp->place < conf.tsize)) {
	    DEBUG(("search: in core\n"));
	    memcpy(&value, &((erec *)etab.core)[sp->place], sizeof(erec)); 
	} else {
	    OFFSET_T dest = 0;
	    /* seek, if necessary */
	    dest = sp->place * sizeof(erec);
	    if (etab.pos != dest) {
		if (lseek(etab.fd, dest, SEEK_SET) == -1) {
		    DEBUG(("search: seek failed\n"));
		    etab.pos = -1;
		    sp->aborted = 1;
		    return FALSE;
		}
		etab.pos = dest;
	    }

	    /* read it */
	    errno = 0;
	    if (read(etab.fd, (POINTER)&value, sizeof(erec)) != sizeof(erec)) {
		if (errno != 0) {
		    DEBUG(("search: read failed\n"));
		    etab.pos = -1;
		    sp->aborted = 1;
		    return FALSE;
		} else {
		    memset(&value, '\0', sizeof(erec));
		}
	    }

	    /* and finish up */
	    etab.pos += sizeof(erec);
	}

	/* Check for an empty record */
	if (!memcmp(&value, &empty_rec, sizeof(erec))) {
	    DEBUG(("search: empty slot\n"));
	    return FALSE;
	}

	/* check the value */
	DEBUG(("got 0x%lx\n", value));
	if (!memcmp(&value.hash, &sp->hash, DBZ_INTERNAL_HASH_SIZE)) {
	    return TRUE;
	}
    }
    /* NOTREACHED */
}
#endif	/* DO_TAGGED_HASH */

/* set - store a value into a location previously found by search
 *
 * Returns:  TRUE success, FALSE failure
 */
static BOOL set(searcher *sp, hash_table *tab, void *value) {
    if (sp->aborted)
	return FALSE;

    /* If we have the index file in memory, use it */
    if ((tab->incore != INCORE_NO) && (sp->place < conf.tsize)) {
	memcpy((void *)((char *)tab->core + (sp->place * tab->reclen)), value, tab->reclen);
	DEBUG(("set: incore\n"));
	if (tab->incore == INCORE_MMAP)
	    return TRUE;
	if (!options.writethrough)
	    return TRUE;
    }

    /* seek to spot */
    tab->pos = -1;		/* invalidate position memory */
    if (lseek(tab->fd, (OFFSET_T)(sp->place * tab->reclen), SEEK_SET) == -1) {
	DEBUG(("set: seek failed\n"));
	sp->aborted = 1;
	return FALSE;
    }

    /* write in data */
    while (write(tab->fd, (POINTER)value, tab->reclen) != tab->reclen) {
	if (errno == EINTR) {
	    if (lseek(tab->fd, (OFFSET_T)(sp->place * tab->reclen), SEEK_SET) == -1) {
		DEBUG(("set: seek failed\n"));
		sp->aborted = 1;
		return FALSE;
	    }
	    continue;
	}
	if (errno == EAGAIN) {
	    fd_set writeset;
	    
	    FD_ZERO(&writeset);
	    FD_SET(tab->fd, &writeset);
	    if (select(tab->fd + 1, NULL, &writeset, NULL, NULL) < 1) {
		DEBUG(("set: select failed\n"));
		sp->aborted = 1;
		return FALSE;
	    }
	    if (lseek(tab->fd, (OFFSET_T)(sp->place * tab->reclen), SEEK_SET) == -1) {
		DEBUG(("set: seek failed\n"));
		sp->aborted = 1;
		return FALSE;
	    }
	    continue;
	}
	DEBUG(("set: write failed\n"));
	sp->aborted = 1;
	return FALSE;
    }

    DEBUG(("set: succeeded\n"));
    return TRUE;
}

#ifdef	DO_TAGGED_HASH
/*
 - set_pag - store a value into a location previously found by search
 -       on the pag table.
 - Returns: TRUE success, FALSE failure
 */
static BOOL set_pag(searcher *sp, OFFSET_T value) {
    OFFSET_T v = value;

    if (CANTAG(v)) {
	v |= sp->tag | taghere;
	if (v != UNBIAS(VACANT))	/* BIAS(v) won't look VACANT */
#ifdef OVERFLOW
	    if (v != LONG_MAX)		/* and it won't overflow */
#endif
		value = v;
    } else if (canttag_warned == 0) {
	fprintf(stderr, "dbz.c(set): can't tag value 0x%lx", v);
	fprintf(stderr, " tagboth = 0x%lx\n", tagboth);
	canttag_warned = 1;
    }
    DEBUG(("tagged value is 0x%lx\n", value));
    value = BIAS(value);

    return set(sp, &pagtab, &value);
}
#endif	/* DO_TAGGED_HASH */

/* dbzsetoptions - set runtime options for the database.
 */
void dbzsetoptions(const dbzoptions o) {
    options = o;
#ifndef HAVE_MMAP
    /* Without a working mmap on files, we should avoid it. */
#ifdef	DO_TAGGED_HASH
    if (options.pag_incore == INCORE_MMAP) options.pag_incore = INCORE_NO;
#else
    if (options.idx_incore == INCORE_MMAP) options.idx_incore = INCORE_NO;
#endif
    if (options.exists_incore == INCORE_MMAP) options.exists_incore = INCORE_NO;
#endif
}

/* dbzgetoptions - get runtime options for the database.
 */
void dbzgetoptions(dbzoptions *o) {
    *o = options;
}


/* dbzdebug - control dbz debugging at run time
 * Returns: old value for dbzdebug
 */
#ifdef DBZDEBUG
int dbzdebug(const BOOL value) {
    BOOL old = debug;

    debug = value;
    return old;
}
#endif

#ifdef DBZTEST
void CloseOnExec(int i, int j) {}

int timediffms(struct timeval start, struct timeval end) {
    return (((end.tv_sec - start.tv_sec) * 1000) +
	((end.tv_usec - start.tv_usec)) / 1000);
}

void RemoveDBZ(char *filename) {
    char fn[1024];

#ifdef	DO_TAGGED_HASH
    sprintf(fn, "%s.pag", filename);
    unlink(fn);
#else
    sprintf(fn, "%s.exists", filename);
    unlink(fn);
    sprintf(fn, "%s.index", filename);
    unlink(fn);
#endif
    sprintf(fn, "%s.dir", filename);
    unlink(fn);
}

static void usage()
{
    fprintf(stderr, "usage: dbztest [-i] [-n|m] [-s size] <history>\n");
    fprintf(stderr, "  -i       initialize history. deletes .pag files\n");
    fprintf(stderr, "  -n or m  use INCORE_NO, INCORE_/MMAP. default = INCORE_MEM\n");
    fprintf(stderr, "  -s size  number of history lines[2500000]\n");
    fprintf(stderr, "  history  history text file\n");
    exit(1);
}

int main(argc, argv)
int  argc;
char *argv[];
{
    int  i, line;
    FILE *fpi;
    char ibuf[2048], *p;
    HASH key;
    long where, val;
    int  initialize = 0, size = 2500000;
    char *history = NULL;
    dbzoptions opt;
    dbz_incore_val incore = INCORE_MEM;
    struct timeval start, end;
#ifndef DO_TAGGED_HASH
    idxrec	ionevalue;
    idxrecext	iextvalue;
#endif

    for (i=1; i<argc; i++)
	if (strcmp(argv[i], "-i") == 0)
	    initialize = 1;
	else if (strcmp(argv[i], "-n") == 0)
	    incore = INCORE_NO;
	else if (strcmp(argv[i], "-m") == 0)
#if defined(HAVE_MMAP)
	    incore = INCORE_MMAP;
#else
            fprintf (stderr, "can't mmap files\n");
#endif
	else if (strcmp(argv[i], "-s") == 0)
	    size = atoi(argv[++i]);
	else if (*argv[i] != '-' && history == NULL)
	    history = argv[i];
	else
	    usage();

    if (history == NULL)
	usage();
    if ((fpi = fopen(history, "r")) == NULL) {
	fprintf(stderr, "can't open %s\n", history);
	usage();
    }

    dbzgetoptions(&opt);
#ifdef	DO_TAGGED_HASH
    opt.pag_incore = incore;
#else
    opt.idx_incore = incore;
#endif
    dbzsetoptions(opt);

    if (initialize) {
	RemoveDBZ(history);
	gettimeofday(&start, NULL);
	if (dbzfresh(history, dbzsize(size), 0) < 0) {
	    fprintf(stderr, "cant dbzfresh %s\n", history);
	    exit(1);
	}
	gettimeofday(&end, NULL);
	printf("dbzfresh: %d msec\n", timediffms(start, end));
    } else {
#ifdef	DBZDEBUG
	dbzdebug(1);
#endif
	gettimeofday(&start, NULL);
	if (dbzinit(history) < 0) {
	    fprintf(stderr, "cant dbzinit %s\n", history);
	    exit(1);
	}
	gettimeofday(&end, NULL);
	printf("dbzinit: %d msec\n", timediffms(start, end));
    }

    gettimeofday(&start, NULL);
    where = ftell(fpi);
    for (line=1; fgets(ibuf, sizeof(ibuf), fpi); line++, where=ftell(fpi)) {
	if (*ibuf == '<') {
	    if ((p = strchr(ibuf, '\t')) == NULL) {
		fprintf(stderr, "ignoreing bad line: %s\n", ibuf);
		continue;
	    }
	    *p = '\0';
	    key = HashMessageID(ibuf);
	} else if (*ibuf == '[')
	    key = TextToHash(ibuf+1);
	else
	    continue;
	if (initialize) {
	    if (dbzstore(key, where) < 0) {
		fprintf(stderr, "cant store %s\n", ibuf);
		exit(1);
	    }
	} else {
#ifdef	DO_TAGGED_HASH
	    val = dbzfetch(key);
	    if (val == NOTFOUND) {
		fprintf(stderr, "line %d can't fetch %s\n", line, ibuf);
		exit(1);
	    }
#else
	    if (innconf->extendeddbz) {
		if (!dbzfetch(key, &iextvalue)) {
		    fprintf(stderr, "line %d can't fetch %s\n", line, ibuf);
		    exit(1);
		}
	    } else {
		if (!dbzfetch(key, &ionevalue)) {
		    fprintf(stderr, "line %d can't fetch %s\n", line, ibuf);
		    exit(1);
		}
	    }
#endif
	}
    }
    gettimeofday(&end, NULL);
    i = timediffms(start, end);
    printf("%s: %d lines %.3f msec/id\n",
	(initialize) ? "dbzstore" : "dbzfetch",
	line, (double)i / (double)line);

    gettimeofday(&end, NULL);
    dbzclose();
    gettimeofday(&end, NULL);
    printf("dbzclose: %d msec\n", timediffms(start, end));
    return(0);
}
#endif	/* DBZTEST */
