Make TDB 32 bit clean (not just 31 bit) for systems with 64 bit file offsets.

ccan/tdb/tdb_private.h

@@ -34,6 +34,7 @@
 #include "system/wait.h"
 #else
 #define _XOPEN_SOURCE 500
+#define _FILE_OFFSET_BITS 64
 #include <stdint.h>
 #include <stdbool.h>
 #include <stdlib.h>

ccan/tdb/test/run-3G-file.c

@@ -0,0 +1,127 @@
+/* We need this otherwise fcntl locking fails. */
+#define _FILE_OFFSET_BITS 64
+#define _XOPEN_SOURCE 500
+#include "tdb/tdb.h"
+#include "tdb/io.c"
+#include "tdb/tdb.c"
+#include "tdb/lock.c"
+#include "tdb/freelist.c"
+#include "tdb/traverse.c"
+#include "tdb/transaction.c"
+#include "tdb/error.c"
+#include "tdb/open.c"
+#include "tap/tap.h"
+#include <stdlib.h>
+#include <err.h>
+
+static int tdb_expand_file_sparse(struct tdb_context *tdb,
+				  tdb_off_t size,
+				  tdb_off_t addition)
+{
+	if (tdb->read_only || tdb->traverse_read) {
+		tdb->ecode = TDB_ERR_RDONLY;
+		return -1;
+	}
+
+	if (ftruncate(tdb->fd, size+addition) == -1) {
+		char b = 0;
+		ssize_t written = pwrite(tdb->fd,  &b, 1, (size+addition) - 1);
+		if (written == 0) {
+			/* try once more, potentially revealing errno */
+			written = pwrite(tdb->fd,  &b, 1, (size+addition) - 1);
+		}
+		if (written == 0) {
+			/* again - give up, guessing errno */
+			errno = ENOSPC;
+		}
+		if (written != 1) {
+			TDB_LOG((tdb, TDB_DEBUG_FATAL, "expand_file to %d failed (%s)\n", 
+				 size+addition, strerror(errno)));
+			return -1;
+		}
+	}
+
+	return 0;
+}
+
+static const struct tdb_methods large_io_methods = {
+	tdb_read,
+	tdb_write,
+	tdb_next_hash_chain,
+	tdb_oob,
+	tdb_expand_file_sparse,
+	tdb_brlock
+};
+
+static int test_traverse(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data,
+			 void *_data)
+{
+	TDB_DATA *expect = _data;
+	ok1(key.dsize == strlen("hi"));
+	ok1(memcmp(key.dptr, "hi", strlen("hi")) == 0);
+	ok1(data.dsize == expect->dsize);
+	ok1(memcmp(data.dptr, expect->dptr, data.dsize) == 0);
+	return 0;
+}
+
+int main(int argc, char *argv[])
+{
+	struct tdb_context *tdb;
+	TDB_DATA key, orig_data, data;
+	uint32_t hash;
+	tdb_off_t rec_ptr;
+	struct list_struct rec;
+
+	plan_tests(24);
+	tdb = tdb_open("/tmp/test.tdb", 1024, TDB_CLEAR_IF_FIRST,
+		       O_CREAT|O_TRUNC|O_RDWR, 0600);
+
+	ok1(tdb);
+	tdb->methods = &large_io_methods;
+
+	/* Enlarge the file (internally multiplies by 100). */
+	ok1(tdb_expand(tdb, 30000000) == 0);
+
+	/* Put an entry in, and check it. */
+	key.dsize = strlen("hi");
+	key.dptr = (void *)"hi";
+	orig_data.dsize = strlen("world");
+	orig_data.dptr = (void *)"world";
+
+	ok1(tdb_store(tdb, key, orig_data, TDB_INSERT) == 0);
+
+	data = tdb_fetch(tdb, key);
+	ok1(data.dsize == strlen("world"));
+	ok1(memcmp(data.dptr, "world", strlen("world")) == 0);
+	free(data.dptr);
+
+	/* That currently fills at the end, make sure that's true. */
+	hash = tdb->hash_fn(&key);
+	rec_ptr = tdb_find_lock_hash(tdb, key, hash, F_RDLCK, &rec);
+	ok1(rec_ptr);
+	ok1(rec_ptr > 2U*1024*1024*1024);
+	tdb_unlock(tdb, BUCKET(rec.full_hash), F_RDLCK);
+
+	/* Traverse must work. */
+	ok1(tdb_traverse(tdb, test_traverse, &orig_data) == 1);
+
+	/* Delete should work. */
+	ok1(tdb_delete(tdb, key) == 0);
+
+	ok1(tdb_traverse(tdb, test_traverse, NULL) == 0);
+
+	/* Transactions should work. */
+	ok1(tdb_transaction_start(tdb) == 0);
+	ok1(tdb_store(tdb, key, orig_data, TDB_INSERT) == 0);
+
+	data = tdb_fetch(tdb, key);
+	ok1(data.dsize == strlen("world"));
+	ok1(memcmp(data.dptr, "world", strlen("world")) == 0);
+	free(data.dptr);
+	ok1(tdb_transaction_commit(tdb) == 0);
+
+	ok1(tdb_traverse(tdb, test_traverse, &orig_data) == 1);
+	tdb_close(tdb);
+
+	return exit_status();
+}

ccan/tdb/traverse.c

@@ -27,8 +27,11 @@
 
 #include "tdb_private.h"
 
-/* Uses traverse lock: 0 = finish, -1 = error, other = record offset */
-static int tdb_next_lock(struct tdb_context *tdb, struct tdb_traverse_lock *tlock,
+#define TDB_NEXT_LOCK_ERR ((tdb_off_t)-1)
+
+/* Uses traverse lock: 0 = finish, TDB_NEXT_LOCK_ERR = error,
+   other = record offset */
+static tdb_off_t tdb_next_lock(struct tdb_context *tdb, struct tdb_traverse_lock *tlock,
 			 struct list_struct *rec)
 {
 	int want_next = (tlock->off != 0);
@@ -71,7 +74,7 @@ static int tdb_next_lock(struct tdb_context *tdb, struct tdb_traverse_lock *tloc
 		}
 
 		if (tdb_lock(tdb, tlock->hash, tlock->lock_rw) == -1)
-			return -1;
+			return TDB_NEXT_LOCK_ERR;
 
 		/* No previous record?  Start at top of chain. */
 		if (!tlock->off) {
@@ -99,6 +102,7 @@ static int tdb_next_lock(struct tdb_context *tdb, struct tdb_traverse_lock *tloc
 
 			/* Detect infinite loops. From "Shlomi Yaakobovich" <Shlomi@exanet.com>. */
 			if (tlock->off == rec->next) {
+				tdb->ecode = TDB_ERR_CORRUPT;
 				TDB_LOG((tdb, TDB_DEBUG_FATAL, "tdb_next_lock: loop detected.\n"));
 				goto fail;
 			}
@@ -128,7 +132,7 @@ static int tdb_next_lock(struct tdb_context *tdb, struct tdb_traverse_lock *tloc
 	tlock->off = 0;
 	if (tdb_unlock(tdb, tlock->hash, tlock->lock_rw) != 0)
 		TDB_LOG((tdb, TDB_DEBUG_FATAL, "tdb_next_lock: On error unlock failed!\n"));
-	return -1;
+	return TDB_NEXT_LOCK_ERR;
 }
 
 /* traverse the entire database - calling fn(tdb, key, data) on each element.
@@ -142,7 +146,8 @@ static int tdb_traverse_internal(struct tdb_context *tdb,
 {
 	TDB_DATA key, dbuf;
 	struct list_struct rec;
-	int ret, count = 0;
+	int ret = 0, count = 0;
+	tdb_off_t off;
 
 	/* This was in the initializaton, above, but the IRIX compiler
 	 * did not like it.  crh
@@ -153,7 +158,11 @@ static int tdb_traverse_internal(struct tdb_context *tdb,
 	tdb->travlocks.next = tl;
 
 	/* tdb_next_lock places locks on the record returned, and its chain */
-	while ((ret = tdb_next_lock(tdb, tl, &rec)) > 0) {
+	while ((off = tdb_next_lock(tdb, tl, &rec)) != 0) {
+		if (off == TDB_NEXT_LOCK_ERR) {
+			ret = -1;
+			goto out;
+		}
 		count++;
 		/* now read the full record */
 		key.dptr = tdb_alloc_read(tdb, tl->off + sizeof(rec), 
@@ -181,7 +190,6 @@ static int tdb_traverse_internal(struct tdb_context *tdb,
 		if (fn && fn(tdb, key, dbuf, private_data)) {
 			/* They want us to terminate traversal */
 			tdb_trace_ret(tdb, "tdb_traverse_end", count);
-			ret = count;
 			if (tdb_unlock_record(tdb, tl->off) != 0) {
 				TDB_LOG((tdb, TDB_DEBUG_FATAL, "tdb_traverse: unlock_record failed!\n"));;
 				ret = -1;
@@ -263,6 +271,7 @@ TDB_DATA tdb_firstkey(struct tdb_context *tdb)
 {
 	TDB_DATA key;
 	struct list_struct rec;
+	tdb_off_t off;
 
 	/* release any old lock */
 	if (tdb_unlock_record(tdb, tdb->travlocks.off) != 0)
@@ -271,7 +280,8 @@ TDB_DATA tdb_firstkey(struct tdb_context *tdb)
 	tdb->travlocks.lock_rw = F_RDLCK;
 
 	/* Grab first record: locks chain and returned record. */
-	if (tdb_next_lock(tdb, &tdb->travlocks, &rec) <= 0) {
+	off = tdb_next_lock(tdb, &tdb->travlocks, &rec);
+	if (off == 0 || off == TDB_NEXT_LOCK_ERR) {
 		tdb_trace_retrec(tdb, "tdb_firstkey", tdb_null);
 		return tdb_null;
 	}
@@ -294,6 +304,7 @@ TDB_DATA tdb_nextkey(struct tdb_context *tdb, TDB_DATA oldkey)
 	TDB_DATA key = tdb_null;
 	struct list_struct rec;
 	unsigned char *k = NULL;
+	tdb_off_t off;
 
 	/* Is locked key the old key?  If so, traverse will be reliable. */
 	if (tdb->travlocks.off) {
@@ -337,7 +348,8 @@ TDB_DATA tdb_nextkey(struct tdb_context *tdb, TDB_DATA oldkey)
 
 	/* Grab next record: locks chain and returned record,
 	   unlocks old record */
-	if (tdb_next_lock(tdb, &tdb->travlocks, &rec) > 0) {
+	off = tdb_next_lock(tdb, &tdb->travlocks, &rec);
+	if (off != TDB_NEXT_LOCK_ERR && off != 0) {
 		key.dsize = rec.key_len;
 		key.dptr = tdb_alloc_read(tdb, tdb->travlocks.off+sizeof(rec),
 					  key.dsize);