failtest: hook can return FAIL_PROBE

tdb2 has various places where it recovers from failure (eg. falling
back when it can't open /dev/urandom, or allocation for error
logging).  We want to test those paths, but doing so thoroughly causes
cominatorial explosion.

Add FAIL_PROBE for such cases: in this case it goes only 3 more calls
deep.

ccan/failtest/failtest.c

@@ -17,8 +17,7 @@
 #include <ccan/failtest/failtest.h>
 #include <ccan/build_assert/build_assert.h>
 
-bool (*failtest_hook)(struct failtest_call *history, unsigned num)
-= failtest_default_hook;
+enum failtest_result (*failtest_hook)(struct failtest_call *, unsigned);
 
 static int tracefd = -1;
 
@@ -48,6 +47,7 @@ static struct failtest_call *history = NULL;
 static unsigned int history_num = 0;
 static int control_fd = -1;
 static struct timeval start;
+static unsigned int probe_count = 0;
 
 static struct write_call *child_writes = NULL;
 static unsigned int child_writes_num = 0;
@@ -86,11 +86,6 @@ static struct failtest_call *add_history_(enum failtest_call_type type,
 #define set_cleanup(call, clean, type)			\
 	(call)->cleanup = (void *)((void)sizeof(clean((type *)NULL)), (clean))
 
-bool failtest_default_hook(struct failtest_call *history, unsigned num)
-{
-	return true;
-}
-
 static bool read_write_info(int fd)
 {
 	struct write_call *w;
@@ -305,6 +300,40 @@ static void restore_files(struct saved_file *s)
 	}
 }
 
+/* Free up memory, so valgrind doesn't report leaks. */
+static void free_everything(void)
+{
+	unsigned int i;
+
+	/* We don't do this in cleanup: needed even for failed opens. */
+	for (i = 0; i < history_num; i++) {
+		if (history[i].type == FAILTEST_OPEN)
+			free((char *)history[i].u.open.pathname);
+	}
+	free(history);
+}
+
+static NORETURN void failtest_cleanup(bool forced_cleanup, int status)
+{
+	int i;
+
+	if (forced_cleanup)
+		history_num--;
+
+	/* Cleanup everything, in reverse order. */
+	for (i = history_num - 1; i >= 0; i--)
+		if (history[i].cleanup)
+			history[i].cleanup(&history[i].u);
+
+	free_everything();
+
+	if (control_fd == -1)
+		exit(status);
+
+	tell_parent(SUCCESS);
+	exit(0);
+}
+
 static bool should_fail(struct failtest_call *call)
 {
 	int status;
@@ -314,6 +343,10 @@ static bool should_fail(struct failtest_call *call)
 	size_t outlen = 0;
 	struct saved_file *files;
 
+	/* Are we probing? */
+	if (probe_count && --probe_count == 0)
+		failtest_cleanup(true, 0);
+
 	if (call == &unrecorded_call)
 		return false;
 
@@ -353,9 +386,24 @@ static bool should_fail(struct failtest_call *call)
 		}
 	}
 
-	if (!failtest_hook(history, history_num)) {
-		call->fail = false;
-		return false;
+	if (failtest_hook) {
+		switch (failtest_hook(history, history_num)) {
+		case FAIL_OK:
+			break;
+		case FAIL_DONT_FAIL:
+			call->fail = false;
+			return false;
+		case FAIL_PROBE:
+			/* Already down probe path?  Stop now. */
+			if (probe_count)
+				failtest_cleanup(true, 0);
+			/* FIXME: We should run *parent* and run probe until
+			 * calls match up again. */
+			probe_count = 3;
+			break;
+		default:
+			abort();
+		}
 	}
 
 	files = save_files();
@@ -974,39 +1022,12 @@ void failtest_init(int argc, char *argv[])
 	gettimeofday(&start, NULL);
 }
 
-/* Free up memory, so valgrind doesn't report leaks. */
-static void free_everything(void)
-{
-	unsigned int i;
-
-	/* We don't do this in cleanup: needed even for failed opens. */
-	for (i = 0; i < history_num; i++) {
-		if (history[i].type == FAILTEST_OPEN)
-			free((char *)history[i].u.open.pathname);
-	}
-	free(history);
-}
-
 void failtest_exit(int status)
 {
-	int i;
-
 	if (failtest_exit_check) {
 		if (!failtest_exit_check(history, history_num))
 			child_fail(NULL, 0, "failtest_exit_check failed\n");
 	}
 
-	if (control_fd == -1) {
-		free_everything();
-		exit(status);
-	}
-
-	/* Cleanup everything, in reverse order. */
-	for (i = history_num - 1; i >= 0; i--)
-		if (history[i].cleanup)
-			history[i].cleanup(&history[i].u);
-
-	free_everything();
-	tell_parent(SUCCESS);
-	exit(0);
+	failtest_cleanup(false, status);
 }

ccan/failtest/failtest.h

@@ -143,30 +143,42 @@ struct failtest_call {
 	} u;
 };
 
+enum failtest_result {
+	/* Yes try failing this call. */
+	FAIL_OK,
+	/* No, don't try failing this call. */
+	FAIL_DONT_FAIL,
+	/* Try failing this call but don't go too far down that path. */
+	FAIL_PROBE,
+};
+
 /**
  * failtest_hook - whether a certain call should fail or not.
  * @history: the ordered history of all failtest calls.
  * @num: the number of elements in @history (greater than 0)
  *
  * The default value of this hook is failtest_default_hook(), which returns
- * true (ie. yes, fail the call).
+ * FAIL_OK (ie. yes, fail the call).
  *
  * You can override it, and avoid failing certain calls.  The parameters
  * of the call (but not the return value(s)) will be filled in for the last
  * call.
  *
  * Example:
- *	static bool dont_fail_allocations(struct failtest_call *history,
- *					  unsigned num)
+ *	static enum failtest_result dont_fail_alloc(struct failtest_call *hist,
+ *						    unsigned num)
  *	{
- *		return history[num-1].type != FAILTEST_MALLOC
- *			&& history[num-1].type != FAILTEST_CALLOC
- *			&& history[num-1].type != FAILTEST_REALLOC;
+ *		if (hist[num-1].type == FAILTEST_MALLOC
+ *			|| hist[num-1].type == FAILTEST_CALLOC
+ *			|| hist[num-1].type == FAILTEST_REALLOC)
+ *			return FAIL_DONT_FAIL;
+ *		return FAIL_OK;
  *	}
  *	...
- *		failtest_hook = dont_fail_allocations;
+ *		failtest_hook = dont_fail_alloc;
  */
-extern bool (*failtest_hook)(struct failtest_call *history, unsigned num);
+extern enum failtest_result
+(*failtest_hook)(struct failtest_call *history, unsigned num);
 
 /**
  * failtest_exit_check - hook for additional checks on a failed child.
@@ -183,9 +195,6 @@ extern bool (*failtest_hook)(struct failtest_call *history, unsigned num);
 extern bool (*failtest_exit_check)(struct failtest_call *history,
 				   unsigned num);
 
-/* This usually fails the call. */
-bool failtest_default_hook(struct failtest_call *history, unsigned num);
-
 /**
  * failtest_timeout_ms - how long to wait before killing child.
  *

ccan/failtest/test/run-locking.c

@@ -10,9 +10,10 @@
 #define SIZE 8
 
 /* We don't want to fork and fail; we're just testing lock recording. */
-static bool dont_fail(struct failtest_call *history, unsigned num)
+static enum failtest_result dont_fail(struct failtest_call *history,
+				      unsigned num)
 {
-	return false;
+	return FAIL_DONT_FAIL;
 }
 
 static bool place_lock(int fd, char lockarr[], unsigned pos, unsigned size,