ccanlint: try running example code.

Comments of form "// [given x] outputs y" we can check the examples do as expected.

tools/ccanlint/Makefile

@@ -11,6 +11,7 @@ CORE_OBJS := tools/ccanlint/ccanlint.o \
 	ccan/str_talloc/str_talloc.o ccan/grab_file/grab_file.o \
 	ccan/btree/btree.o \
 	ccan/talloc/talloc.o ccan/noerr/noerr.o \
+	ccan/foreach/foreach.o \
 	ccan/read_write_all/read_write_all.o \
 	ccan/opt/opt.o ccan/opt/usage.o ccan/opt/helpers.o ccan/opt/parse.o
 

tools/ccanlint/ccanlint.h

@@ -34,6 +34,7 @@ struct manifest {
 
 	struct list_head other_files;
 	struct list_head examples;
+	struct list_head mangled_examples;
 
 	/* From tests/check_depends_exist.c */
 	struct list_head dep_dirs;

tools/ccanlint/file_analysis.c

@@ -180,6 +180,7 @@ struct manifest *get_manifest(const void *ctx, const char *dir)
 	list_head_init(&m->other_test_files);
 	list_head_init(&m->other_files);
 	list_head_init(&m->examples);
+	list_head_init(&m->mangled_examples);
 	list_head_init(&m->dep_dirs);
 
 	olddir = talloc_getcwd(NULL);

tools/ccanlint/tests/examples_compile.c

@@ -122,6 +122,7 @@ static char *compile(const void *ctx,
 				  "", lib_list(m), file->compiled);
 	if (errmsg) {
 		talloc_free(file->compiled);
+		file->compiled = NULL;
 		return errmsg;
 	}
 	return NULL;
@@ -429,6 +430,7 @@ static struct ccan_file *mangle_example(struct manifest *m,
 	}
 	close(fd);
 	f->contents = talloc_steal(f, contents);
+	list_add(&m->mangled_examples, &f->list);
 	return f;
 }
 
@@ -450,35 +452,35 @@ static void *build_examples(struct manifest *m, bool keep,
 		examples_compile.total_score++;
 		/* Simplify our dumb parsing. */
 		strip_leading_whitespace(get_ccan_file_lines(i));
-		ret = compile(score, m, i, keep);
+		ret = compile(i, m, i, keep);
 		if (!ret) {
 			prev = get_ccan_file_lines(i);
 			score->score++;
 			continue;
 		}
 
+		/* Try standalone. */
+		mangle1 = mangle_example(m, i, get_ccan_file_lines(i), keep);
+		ret1 = compile(i, m, mangle1, keep);
+		if (!ret1) {
+			prev = get_ccan_file_lines(i);
+			score->score++;
+			continue;
+		}
+
 		/* Try combining with previous (successful) example... */
 		if (prev) {
 			char **new = combine(i, get_ccan_file_lines(i), prev);
 
-			mangle1 = mangle_example(m, i, new, keep);
-			ret1 = compile(score, m, mangle1, keep);
-			if (!ret1) {
+			mangle2 = mangle_example(m, i, new, keep);
+			ret2 = compile(i, m, mangle1, keep);
+			if (!ret2) {
 				prev = new;
 				score->score++;
 				continue;
 			}
 		}
 
-		/* Try standalone. */
-		mangle2 = mangle_example(m, i, get_ccan_file_lines(i), keep);
-		ret2 = compile(score, m, mangle2, keep);
-		if (!ret2) {
-			prev = get_ccan_file_lines(i);
-			score->score++;
-			continue;
-		}
-
 		score->errors = talloc_asprintf_append(score->errors,
 				       "%s: tried standalone example:\n"
 				       "%s\n"
@@ -486,23 +488,24 @@ static void *build_examples(struct manifest *m, bool keep,
 				       i->name,
 				       get_ccan_file_contents(i),
 				       ret);
-		if (mangle1) {
-			score->errors = talloc_asprintf_append(score->errors,
-					       "%s: tried combining with"
-					       " previous example:\n"
-					       "%s\n"
-					       "Errors: %s\n\n",
-					       i->name,
-					       get_ccan_file_contents(mangle1),
-					       ret1);
-		}
 		score->errors = talloc_asprintf_append(score->errors,
 				       "%s: tried adding headers, wrappers:\n"
 				       "%s\n"
 				       "Errors: %s\n\n",
 				       i->name,
+				       get_ccan_file_contents(mangle1),
+				       ret1);
+
+		if (mangle2) {
+			score->errors = talloc_asprintf_append(score->errors,
+				       "%s\n"
+				       "%s: tried combining with"
+				       " previous example:\n"
+				       "Errors: %s\n\n",
+				       i->name,
 				       get_ccan_file_contents(mangle2),
 				       ret2);
+		}
 		/* This didn't work, so not a candidate for combining. */
 		prev = NULL;
 	}

tools/ccanlint/tests/examples_run.c

@@ -0,0 +1,304 @@
+#include <tools/ccanlint/ccanlint.h>
+#include <tools/tools.h>
+#include <ccan/talloc/talloc.h>
+#include <ccan/foreach/foreach.h>
+#include <ccan/str/str.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <stdint.h>
+#include <string.h>
+#include <unistd.h>
+#include <ctype.h>
+#include <assert.h>
+
+static const char *can_run(struct manifest *m)
+{
+	if (safe_mode)
+		return "Safe mode enabled";
+	return NULL;
+}
+
+struct score {
+	unsigned int score;
+	char *errors;
+};
+
+/* Very dumb scanner, allocates %s-strings. */
+static bool scan_forv(const void *ctx,
+		      const char *input, const char *fmt, const va_list *args)
+{
+	va_list ap;
+	bool ret;
+
+	if (input[0] == '\0' || fmt[0] == '\0')
+		return input[0] == fmt[0];
+
+	va_copy(ap, *args);
+
+	if (isspace(fmt[0])) {
+		/* One format space can swallow many input spaces */
+		ret = false;
+		while (isspace(input[0])) {
+			if (scan_forv(ctx, ++input, fmt+1, &ap)) {
+				ret = true;
+				break;
+			}
+		}
+	} else if (fmt[0] != '%') {
+		if (toupper(input[0]) != toupper(fmt[0]))
+			ret = false;
+		else
+			ret = scan_forv(ctx, input+1, fmt+1, &ap);
+	} else {
+		char **p = va_arg(ap, char **);
+		unsigned int len;
+
+		ret = false;
+		assert(fmt[1] == 's');
+		for (len = 1; input[len-1]; len++) {
+			ret = scan_forv(ctx, input + len, fmt+2, &ap);
+			if (ret) {
+				*p = talloc_strndup(ctx, input, len);
+				ret = true;
+				break;
+			}
+		}
+	}
+	va_end(ap);
+	return ret;
+}
+
+static bool scan_for(const void *ctx, const char *input, const char *fmt, ...)
+{
+	bool ret;
+	va_list ap;
+
+	va_start(ap, fmt);
+	ret = scan_forv(ctx, input, fmt, &ap);
+	va_end(ap);
+	return ret;
+}
+
+static char *find_expect(struct ccan_file *file,
+			 char **lines, char **input, bool *exact,
+			 unsigned *line)
+{
+	char *expect;
+	const char *fmt;
+
+	for (; lines[*line]; (*line)++) {
+		char *p = lines[*line] + strspn(lines[*line], " \t");
+		if (!strstarts(p, "//"))
+			continue;
+		p += strspn(p, "/ ");
+		foreach_ptr(fmt,
+			    "given '%s', outputs '%s'",
+			    "given '%s' outputs '%s'",
+			    "given \"%s\", outputs \"%s\"",
+			    "given \"%s\" outputs \"%s\"") {
+			if (scan_for(file, p, fmt, input, &expect)) {
+				*exact = true;
+				return expect;
+			}
+		}
+
+		foreach_ptr(fmt,
+			    "given '%s', output contains '%s'",
+			    "given '%s' output contains '%s'",
+			    "given \"%s\", output contains \"%s\"",
+			    "given \"%s\" output contains \"%s\"") {
+			if (scan_for(file, p, fmt, input, &expect)) {
+				*exact = false;
+				return expect;
+			}
+		}
+
+		foreach_ptr(fmt, "outputs '%s'", "outputs \"%s\"") {
+			if (scan_for(file, p, fmt, &expect)) {
+				*input = "";
+				*exact = true;
+				return expect;
+			}
+		}
+
+		foreach_ptr(fmt,
+			    "given '%s', output contains '%s'",
+			    "given '%s' output contains '%s'",
+			    "given \"%s\", output contains \"%s\"",
+			    "given \"%s\" output contains \"%s\"") {
+			if (scan_for(file, p, fmt, input, &expect)) {
+				*exact = false;
+				return expect;
+			}
+		}
+
+		/* Unquoted versions... we can get this wrong! */
+		foreach_ptr(fmt,
+			    "given %s, outputs '%s'",
+			    "given '%s', outputs %s",
+			    "given %s, outputs \"%s\"",
+			    "given \"%s\", outputs %s",
+			    "given %s, outputs %s",
+			    "given %s outputs '%s'",
+			    "given '%s' outputs %s",
+			    "given %s outputs \"%s\"",
+			    "given \"%s\" outputs %s",
+			    "given %s outputs %s") {
+			if (scan_for(file, p, fmt, input, &expect)) {
+				*exact = true;
+				return expect;
+			}
+		}
+
+		foreach_ptr(fmt,
+			    "given %s, output contains '%s'",
+			    "given '%s', output contains %s",
+			    "given %s, output contains \"%s\"",
+			    "given \"%s\", output contains %s",
+			    "given %s, output contains %s",
+			    "given %s output contains '%s'",
+			    "given '%s' output contains %s",
+			    "given %s output contains \"%s\"",
+			    "given \"%s\" output contains %s",
+			    "given %s output contains %s") {
+			if (scan_for(file, p, fmt, input, &expect)) {
+				*exact = false;
+				return expect;
+			}
+		}
+
+		foreach_ptr(fmt,
+			    "outputs '%s'",
+			    "outputs \"%s\"",
+			    "outputs %s") {
+			if (scan_for(file, p, fmt, &expect)) {
+				*input = "";
+				*exact = true;
+				return expect;
+			}
+		}
+
+		foreach_ptr(fmt,
+			    "output contains '%s'",
+			    "output contains \"%s\"",
+			    "output contains %s") {
+			if (scan_for(file, p, fmt, &expect)) {
+				*input = "";
+				*exact = false;
+				return expect;
+			}
+		}
+	}		
+	return NULL;
+}
+
+static char *trim(char *string)
+{
+	while (strends(string, "\n"))
+	       string[strlen(string)-1] = '\0';
+	return string;
+}
+
+static char *unexpected(struct ccan_file *i, const char *input,
+			const char *expect, bool exact)
+{
+	char *output, *cmd;
+	bool ok;
+	unsigned int default_time = default_timeout_ms;
+
+	cmd = talloc_asprintf(i, "echo '%s' | %s %s",
+			      input, i->compiled, input);
+
+	output = run_with_timeout(i, cmd, &ok, &default_time);
+	if (!ok)
+		return talloc_asprintf(i, "Exited with non-zero status\n");
+
+	if (exact) {
+		if (streq(output, expect) || streq(trim(output), expect))
+			return NULL;
+	} else {
+		if (strstr(output, expect))
+			return NULL;
+	}
+	return output;
+}
+
+static void *run_examples(struct manifest *m, bool keep,
+			  unsigned int *timeleft)
+{
+	struct ccan_file *i;
+	struct list_head *list;
+	struct score *score = talloc(m, struct score);
+
+	score->score = 0;
+	score->errors = talloc_strdup(score, "");
+
+	examples_run.total_score = 0;
+	foreach_ptr(list, &m->examples, &m->mangled_examples) {
+		list_for_each(list, i, list) {
+			char **lines, *expect, *input, *output;
+			unsigned int linenum = 0;
+			bool exact;
+
+			if (i->compiled == NULL)
+				continue;
+
+			lines = get_ccan_file_lines(i);
+
+			for (expect = find_expect(i, lines, &input, &exact,
+						  &linenum);
+			     expect;
+			     linenum++,
+				     expect = find_expect(i, lines, &input,
+							  &exact, &linenum)) {
+				examples_run.total_score++;
+				output = unexpected(i, input, expect, exact);
+				if (!output)
+					score->score++;
+				else {
+					score->errors = talloc_asprintf_append(
+						score->errors,
+						"%s: output '%s' didn't"
+						" %s '%s'\n",
+						i->name, output,
+						exact ? "match" : "contain",
+						expect);
+				}
+			}
+		}
+	}
+
+	if (strcmp(score->errors, "") == 0) {
+		talloc_free(score);
+		return NULL;
+	}
+	return score;
+}
+
+static unsigned int score_examples(struct manifest *m, void *check_result)
+{
+	struct score *score = check_result;
+	return score->score;
+}
+
+static const char *describe(struct manifest *m, void *check_result)
+{
+	struct score *score = check_result;
+	if (verbose)
+		return talloc_asprintf(m, "Wrong output running examples:\n"
+				       "%s", score->errors);
+	return NULL;
+}
+
+struct ccanlint examples_run = {
+	.key = "examples-run",
+	.name = "Module examples with expected output give that output",
+	.score = score_examples,
+	.total_score = 3, /* This gets changed to # testable, if we run. */
+	.check = run_examples,
+	.describe = describe,
+	.can_run = can_run,
+};
+
+REGISTER_TEST(examples_run, &examples_compile, NULL);