tdb2: more stats

More recording of interesting events.  As we don't have an ABI yet, we
don't need to put these at the end.

ccan/tdb2/free.c

@@ -370,8 +370,10 @@ static tdb_len_t coalesce(struct tdb_context *tdb,
 		}
 
 		/* Did we just mess up a record you were hoping to use? */
-		if (end == *protect)
+		if (end == *protect) {
+			tdb->stats.alloc_coalesce_iterate_clash++;
 			*protect = TDB_ERR_NOEXIST;
+		}
 
 		ecode = remove_from_list(tdb, nb_off, end, &rec);
 		check_list(tdb, nb_off);
@@ -390,8 +392,10 @@ static tdb_len_t coalesce(struct tdb_context *tdb,
 		return 0;
 
 	/* Before we expand, check this isn't one you wanted protected? */
-	if (off == *protect)
+	if (off == *protect) {
 		*protect = TDB_ERR_EXISTS;
+		tdb->stats.alloc_coalesce_iterate_clash++;
+	}
 
 	/* OK, expand initial record */
 	ecode = tdb_read_convert(tdb, off, &rec, sizeof(rec));
@@ -416,6 +420,7 @@ static tdb_len_t coalesce(struct tdb_context *tdb,
 	ecode = add_free_record(tdb, off, end - off, TDB_LOCK_NOWAIT, false);
 	if (ecode != TDB_SUCCESS) {
 		/* Need to drop lock.  Can't rely on anything stable. */
+		tdb->stats.alloc_coalesce_lockfail++;
 		*protect = TDB_ERR_CORRUPT;
 
 		/* We have to drop this to avoid deadlocks, so make sure record

ccan/tdb2/lock.c

@@ -99,16 +99,21 @@ int tdb_fcntl_unlock(int fd, int rw, off_t off, off_t len, void *unused)
 static int lock(struct tdb_context *tdb,
 		      int rw, off_t off, off_t len, bool waitflag)
 {
+	int ret;
 	if (tdb->file->allrecord_lock.count == 0
 	    && tdb->file->num_lockrecs == 0) {
 		tdb->file->locker = getpid();
 	}
 
 	tdb->stats.lock_lowlevel++;
-	if (!waitflag)
+	ret = tdb->lock_fn(tdb->file->fd, rw, off, len, waitflag,
+			   tdb->lock_data);
+	if (!waitflag) {
 		tdb->stats.lock_nonblock++;
-	return tdb->lock_fn(tdb->file->fd, rw, off, len, waitflag,
-			    tdb->lock_data);
+		if (ret != 0)
+			tdb->stats.lock_nonblock_fail++;
+	}
+	return ret;
 }
 
 static int unlock(struct tdb_context *tdb, int rw, off_t off, off_t len)

ccan/tdb2/tdb2.h

@@ -757,21 +757,31 @@ struct tdb_attribute_stats {
 	uint64_t   alloc_bucket_max;
 	uint64_t   alloc_leftover;
 	uint64_t   alloc_coalesce_tried;
+	uint64_t     alloc_coalesce_iterate_clash;
 	uint64_t     alloc_coalesce_lockfail;
 	uint64_t     alloc_coalesce_race;
 	uint64_t     alloc_coalesce_succeeded;
-	uint64_t        alloc_coalesce_num_merged;
+	uint64_t       alloc_coalesce_num_merged;
 	uint64_t compares;
 	uint64_t   compare_wrong_bucket;
 	uint64_t   compare_wrong_offsetbits;
 	uint64_t   compare_wrong_keylen;
 	uint64_t   compare_wrong_rechash;
 	uint64_t   compare_wrong_keycmp;
+	uint64_t transactions;
+	uint64_t   transaction_cancel;
+	uint64_t   transaction_nest;
+	uint64_t   transaction_expand_file;
+	uint64_t   transaction_read_direct;
+	uint64_t      transaction_read_direct_fail;
+	uint64_t   transaction_write_direct;
+	uint64_t      transaction_write_direct_fail;
 	uint64_t expands;
 	uint64_t frees;
 	uint64_t locks;
-	uint64_t    lock_lowlevel;
-	uint64_t    lock_nonblock;
+	uint64_t   lock_lowlevel;
+	uint64_t   lock_nonblock;
+	uint64_t     lock_nonblock_fail;
 };
 
 /**

ccan/tdb2/tools/speed.c

@@ -70,13 +70,15 @@ static void dump_and_clear_stats(struct tdb_context **tdb,
 	       (unsigned long long)stats.stats.alloc_leftover);
 	printf("  alloc_coalesce_tried = %llu\n",
 	       (unsigned long long)stats.stats.alloc_coalesce_tried);
+	printf("    alloc_coalesce_iterate_clash = %llu\n",
+	       (unsigned long long)stats.stats.alloc_coalesce_iterate_clash);
 	printf("    alloc_coalesce_lockfail = %llu\n",
 	       (unsigned long long)stats.stats.alloc_coalesce_lockfail);
 	printf("    alloc_coalesce_race = %llu\n",
 	       (unsigned long long)stats.stats.alloc_coalesce_race);
 	printf("    alloc_coalesce_succeeded = %llu\n",
 	       (unsigned long long)stats.stats.alloc_coalesce_succeeded);
-	printf("       alloc_coalesce_num_merged = %llu\n",
+	printf("      alloc_coalesce_num_merged = %llu\n",
 	       (unsigned long long)stats.stats.alloc_coalesce_num_merged);
 	printf("compares = %llu\n",
 	       (unsigned long long)stats.stats.compares);
@@ -90,16 +92,34 @@ static void dump_and_clear_stats(struct tdb_context **tdb,
 	       (unsigned long long)stats.stats.compare_wrong_rechash);
 	printf("  compare_wrong_keycmp = %llu\n",
 	       (unsigned long long)stats.stats.compare_wrong_keycmp);
+	printf("transactions = %llu\n",
+	       (unsigned long long)stats.stats.transactions);
+	printf("  transaction_cancel = %llu\n",
+	       (unsigned long long)stats.stats.transaction_cancel);
+	printf("  transaction_nest = %llu\n",
+	       (unsigned long long)stats.stats.transaction_nest);
+	printf("  transaction_expand_file = %llu\n",
+	       (unsigned long long)stats.stats.transaction_expand_file);
+	printf("  transaction_read_direct = %llu\n",
+	       (unsigned long long)stats.stats.transaction_read_direct);
+	printf("    transaction_read_direct_fail = %llu\n",
+	       (unsigned long long)stats.stats.transaction_read_direct_fail);
+	printf("  transaction_write_direct = %llu\n",
+	       (unsigned long long)stats.stats.transaction_write_direct);
+	printf("    transaction_write_direct_fail = %llu\n",
+	       (unsigned long long)stats.stats.transaction_write_direct_fail);
 	printf("expands = %llu\n",
 	       (unsigned long long)stats.stats.expands);
 	printf("frees = %llu\n",
 	       (unsigned long long)stats.stats.frees);
 	printf("locks = %llu\n",
 	       (unsigned long long)stats.stats.locks);
-	printf("   lock_lowlevel = %llu\n",
+	printf("  lock_lowlevel = %llu\n",
 	       (unsigned long long)stats.stats.lock_lowlevel);
-	printf("   lock_nonblock = %llu\n",
+	printf("  lock_nonblock = %llu\n",
 	       (unsigned long long)stats.stats.lock_nonblock);
+	printf("    lock_nonblock_fail = %llu\n",
+	       (unsigned long long)stats.stats.lock_nonblock_fail);
 
 	/* Now clear. */
 	tdb_close(*tdb);

ccan/tdb2/transaction.c

@@ -387,15 +387,17 @@ static void *transaction_direct(struct tdb_context *tdb, tdb_off_t off,
 
 	/* Can only do direct if in single block and we've already copied. */
 	if (write_mode) {
-		if (blk != end_blk)
-			return NULL;
-		if (blk >= tdb->transaction->num_blocks)
-			return NULL;
-		if (tdb->transaction->blocks[blk] == NULL)
+		tdb->stats.transaction_write_direct++;
+		if (blk != end_blk
+		    || blk >= tdb->transaction->num_blocks
+		    || tdb->transaction->blocks[blk] == NULL) {
+			tdb->stats.transaction_write_direct_fail++;
 			return NULL;
+		}
 		return tdb->transaction->blocks[blk] + off % PAGESIZE;
 	}
 
+	tdb->stats.transaction_read_direct++;
 	/* Single which we have copied? */
 	if (blk == end_blk
 	    && blk < tdb->transaction->num_blocks
@@ -406,8 +408,10 @@ static void *transaction_direct(struct tdb_context *tdb, tdb_off_t off,
 	while (blk <= end_blk) {
 		if (blk >= tdb->transaction->num_blocks)
 			break;
-		if (tdb->transaction->blocks[blk])
+		if (tdb->transaction->blocks[blk]) {
+			tdb->stats.transaction_read_direct_fail++;
 			return NULL;
+		}
 		blk++;
 	}
 	return tdb->transaction->io_methods->direct(tdb, off, len, false);
@@ -518,6 +522,7 @@ enum TDB_ERROR tdb_transaction_start(struct tdb_context *tdb)
 {
 	enum TDB_ERROR ecode;
 
+	tdb->stats.transactions++;
 	/* some sanity checks */
 	if (tdb->read_only || (tdb->flags & TDB_INTERNAL)) {
 		return tdb->last_error = tdb_logerr(tdb, TDB_ERR_EINVAL,
@@ -538,6 +543,7 @@ enum TDB_ERROR tdb_transaction_start(struct tdb_context *tdb)
 					     " already inside transaction");
 		}
 		tdb->transaction->nesting++;
+		tdb->stats.transaction_nest++;
 		return 0;
 	}
 
@@ -603,6 +609,7 @@ fail_allrecord_lock:
 */
 void tdb_transaction_cancel(struct tdb_context *tdb)
 {
+	tdb->stats.transaction_cancel++;
 	_tdb_transaction_cancel(tdb);
 }
 
@@ -824,6 +831,7 @@ static tdb_off_t create_recovery_area(struct tdb_context *tdb,
 	addition = (tdb->file->map_size - tdb->transaction->old_map_size) +
 		sizeof(*rec) + rec->max_len;
 	tdb->file->map_size = tdb->transaction->old_map_size;
+	tdb->stats.transaction_expand_file++;
 	ecode = methods->expand_file(tdb, addition);
 	if (ecode != TDB_SUCCESS) {
 		return tdb_logerr(tdb, ecode, TDB_LOG_ERROR,