tdb2: expand more slowly.

We took the original expansion heuristic from TDB1, and they just
fixed theirs, so copy that.

Before:

After:
time ./growtdb-bench 250000 10 > /dev/null && ls -l /tmp/growtdb.tdb && time ./tdbtorture -s 0 && ls -l torture.tdb && ./speed --transaction 2000000
growtdb-bench.c: In function ‘main’:
growtdb-bench.c:74:8: warning: ignoring return value of ‘system’, declared with attribute warn_unused_result
growtdb-bench.c:108:9: warning: ignoring return value of ‘system’, declared with attribute warn_unused_result

real	1m0.243s
user	0m13.677s
sys	0m4.336s
-rw------- 1 rusty rusty 683302864 2011-04-27 21:03 /tmp/growtdb.tdb
testing with 3 processes, 5000 loops, seed=0
OK

real	1m24.074s
user	0m0.344s
sys	0m0.468s
-rw------- 1 rusty rusty 836040 2011-04-27 21:04 torture.tdb
Adding 2000000 records:  1015 ns (110551992 bytes)
Finding 2000000 records:  641 ns (110551992 bytes)
Missing 2000000 records:  445 ns (110551992 bytes)
Traversing 2000000 records:  439 ns (110551992 bytes)
Deleting 2000000 records:  807 ns (199517112 bytes)
Re-adding 2000000 records:  851 ns (199517112 bytes)
Appending 2000000 records:  1301 ns (376542552 bytes)
Churning 2000000 records:  2423 ns (553641304 bytes)

ccan/tdb2/free.c

@@ -713,13 +713,10 @@ enum TDB_ERROR set_header(struct tdb_context *tdb,
 /* Expand the database. */
 static enum TDB_ERROR tdb_expand(struct tdb_context *tdb, tdb_len_t size)
 {
-	uint64_t old_size;
+	uint64_t old_size, rec_size, map_size;
 	tdb_len_t wanted;
 	enum TDB_ERROR ecode;
 
-	/* We need room for the record header too. */
-	wanted = sizeof(struct tdb_used_record) + size;
-
 	/* Need to hold a hash lock to expand DB: transactions rely on it. */
 	if (!(tdb->flags & TDB_NOLOCK)
 	    && !tdb->file->allrecord_lock.count && !tdb_has_hash_locks(tdb)) {
@@ -727,14 +724,6 @@ static enum TDB_ERROR tdb_expand(struct tdb_context *tdb, tdb_len_t size)
 				  "tdb_expand: must hold lock during expand");
 	}
 
-	/* always make room for at least 100 more records, and at
-           least 25% more space. */
-	if (size * TDB_EXTENSION_FACTOR > tdb->file->map_size / 4)
-		wanted = size * TDB_EXTENSION_FACTOR;
-	else
-		wanted = tdb->file->map_size / 4;
-	wanted = adjust_size(0, wanted);
-
 	/* Only one person can expand file at a time. */
 	ecode = tdb_lock_expand(tdb, F_WRLCK);
 	if (ecode != TDB_SUCCESS) {
@@ -749,6 +738,32 @@ static enum TDB_ERROR tdb_expand(struct tdb_context *tdb, tdb_len_t size)
 		return TDB_SUCCESS;
 	}
 
+	/* limit size in order to avoid using up huge amounts of memory for
+	 * in memory tdbs if an oddball huge record creeps in */
+	if (size > 100 * 1024) {
+		rec_size = size * 2;
+	} else {
+		rec_size = size * 100;
+	}
+
+	/* always make room for at least rec_size more records, and at
+	   least 25% more space. if the DB is smaller than 100MiB,
+	   otherwise grow it by 10% only. */
+	if (old_size > 100 * 1024 * 1024) {
+		map_size = old_size / 10;
+	} else {
+		map_size = old_size / 4;
+	}
+
+	if (map_size > rec_size) {
+		wanted = map_size;
+	} else {
+		wanted = rec_size;
+	}
+
+	/* We need room for the record header too. */
+	wanted = adjust_size(0, sizeof(struct tdb_used_record) + wanted);
+
 	ecode = tdb->methods->expand_file(tdb, wanted);
 	if (ecode != TDB_SUCCESS) {
 		tdb_unlock_expand(tdb, F_WRLCK);

ccan/tdb2/tools/Makefile

@@ -2,12 +2,13 @@ OBJS:=../../tdb2.o ../../hash.o ../../tally.o
 CFLAGS:=-I../../.. -Wall -g -O3 #-g -pg
 LDFLAGS:=-L../../..
 
-default: tdbtorture tdbtool mktdb speed
+default: tdbtorture tdbtool mktdb speed growtdb-bench
 
 tdbtorture: tdbtorture.c $(OBJS)
 tdbtool: tdbtool.c $(OBJS)
 mktdb: mktdb.c $(OBJS)
 speed: speed.c $(OBJS)
+growtdb-bench: growtdb-bench.c $(OBJS)
 
 clean:
-	rm -f tdbtorture tdbtool mktdb speed
+	rm -f tdbtorture tdbtool mktdb speed growtdb-bench

ccan/tdb2/tools/growtdb-bench.c

@@ -0,0 +1,112 @@
+#include <ccan/tdb2/tdb2.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <err.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+
+static void logfn(struct tdb_context *tdb,
+		  enum tdb_log_level level,
+		  const char *message,
+		  void *data)
+{
+	fprintf(stderr, "tdb:%s:%s\n", tdb_name(tdb), message);
+}
+
+int main(int argc, char *argv[])
+{
+	unsigned int i, j, users, groups;
+	TDB_DATA idxkey, idxdata;
+	TDB_DATA k, d, gk;
+	char cmd[100];
+	struct tdb_context *tdb;
+	enum TDB_ERROR ecode;
+	union tdb_attribute log;
+
+	if (argc != 3) {
+		printf("Usage: growtdb-bench <users> <groups>\n");
+		exit(1);
+	}
+	users = atoi(argv[1]);
+	groups = atoi(argv[2]);
+
+	sprintf(cmd, "cat /proc/%i/statm", getpid());
+
+	log.base.attr = TDB_ATTRIBUTE_LOG;
+	log.base.next = NULL;
+	log.log.fn = logfn;
+	
+	tdb = tdb_open("/tmp/growtdb.tdb", TDB_DEFAULT,
+		       O_RDWR|O_CREAT|O_TRUNC, 0600, &log);
+
+	idxkey.dptr = (unsigned char *)"User index";
+	idxkey.dsize = strlen("User index");
+	idxdata.dsize = 51;
+	idxdata.dptr = calloc(idxdata.dsize, 1);
+
+	/* Create users. */
+	k.dsize = 48;
+	k.dptr = calloc(k.dsize, 1);
+	d.dsize = 64;
+	d.dptr = calloc(d.dsize, 1);
+
+	tdb_transaction_start(tdb);
+	for (i = 0; i < users; i++) {
+		memcpy(k.dptr, &i, sizeof(i));
+		ecode = tdb_store(tdb, k, d, TDB_INSERT);
+		if (ecode != TDB_SUCCESS)
+			errx(1, "tdb insert failed: %s", tdb_errorstr(ecode));
+
+		/* This simulates a growing index record. */
+		ecode = tdb_append(tdb, idxkey, idxdata);
+		if (ecode != TDB_SUCCESS)
+			errx(1, "tdb append failed: %s", tdb_errorstr(ecode));
+	}
+	if ((ecode = tdb_transaction_commit(tdb)) != 0)
+		errx(1, "tdb commit1 failed: %s", tdb_errorstr(ecode));
+
+	if ((ecode = tdb_check(tdb, NULL, NULL)) != 0)
+		errx(1, "tdb_check failed after initial insert!");
+
+	system(cmd);
+
+	/* Now put them all in groups: add 32 bytes to each record for
+	 * a group. */
+	gk.dsize = 48;
+	gk.dptr = calloc(k.dsize, 1);
+	gk.dptr[gk.dsize-1] = 1;
+
+	d.dsize = 32;
+	for (i = 0; i < groups; i++) {
+		tdb_transaction_start(tdb);
+		/* Create the "group". */
+		memcpy(gk.dptr, &i, sizeof(i));
+		ecode = tdb_store(tdb, gk, d, TDB_INSERT);
+		if (ecode != TDB_SUCCESS)
+			errx(1, "tdb insert failed: %s", tdb_errorstr(ecode));
+
+		/* Now populate it. */
+		for (j = 0; j < users; j++) {
+			/* Append to the user. */
+			memcpy(k.dptr, &j, sizeof(j));
+			if ((ecode = tdb_append(tdb, k, d)) != 0)
+				errx(1, "tdb append failed: %s",
+				     tdb_errorstr(ecode));
+			
+			/* Append to the group. */
+			if ((ecode = tdb_append(tdb, gk, d)) != 0)
+				errx(1, "tdb append failed: %s",
+				     tdb_errorstr(ecode));
+		}
+		if ((ecode = tdb_transaction_commit(tdb)) != 0)
+			errx(1, "tdb commit2 failed: %s", tdb_errorstr(ecode));
+		if ((ecode = tdb_check(tdb, NULL, NULL)) != 0)
+			errx(1, "tdb_check failed after iteration %i!", i);
+		system(cmd);
+	}
+
+	return 0;
+}