ccanlint: exit with non-zero exit status if a test fails.

This means we change some minor tests to "never fail", eg. whitespace
or documentation tests.  Note that pass/fail is independent of the
score for a test.

tools/ccanlint/ccanlint.c

@@ -161,9 +161,9 @@ static bool run_test(struct ccanlint *i,
 			printf("%s%s", score->error,
 			       strends(score->error, "\n") ? "" : "\n");
 		}
-		if (!quiet && !score->pass && i->handle)
-			i->handle(m, score);
 	}
+	if (!quiet && score->score < score->total && i->handle)
+		i->handle(m, score);
 
 	*running_score += score->score;
 	*running_total += score->total;
@@ -584,7 +584,7 @@ static char *opt_set_const_charp(const char *arg, const char **p)
 
 int main(int argc, char *argv[])
 {
-	bool summary = false;
+	bool summary = false, pass = true;
 	unsigned int score = 0, total_score = 0;
 	struct manifest *m;
 	struct ccanlint *i;
@@ -675,8 +675,8 @@ int main(int argc, char *argv[])
 		add_info_options(m->info_file, !target);
 
 	while ((i = get_next_test(&normal_tests)) != NULL)
-		run_test(i, summary, &score, &total_score, m);
+		pass &= run_test(i, summary, &score, &total_score, m);
 
 	printf("%sTotal score: %u/%u\n", prefix, score, total_score);
-	return 0;
+	return pass ? 0 : 1;
 }

tools/ccanlint/ccanlint.h

@@ -44,18 +44,26 @@ struct manifest {
 	struct list_head deps;
 };
 
+/* Get the manifest for a given directory. */
 struct manifest *get_manifest(const void *ctx, const char *dir);
 
+/* Error in a particular file: stored off score->per_file_errors. */
 struct file_error {
 	struct list_node list;
 	struct ccan_file *file;
 	unsigned int line;
 };
 
+/* The score for an individual test. */
 struct score {
+	/* Starts as false: if not set to true, ccanlint exits non-zero.
+	 * Thus it is usually set for compilation or other serious failures. */
 	bool pass;
+	/* Starts at 0 and 1 respectively. */
 	unsigned int score, total;
+	/* The error message to print. */
 	char *error;
+	/* Per file errors, set by score_file_error() */
 	struct list_head per_file_errors;
 };
 
@@ -86,7 +94,7 @@ struct ccanlint {
 	/* If not set, we'll give an error if they try to set options. */
 	bool takes_options;
 
-	/* comma-separated list of dependency keys. */
+	/* Space-separated list of dependency keys. */
 	const char *needs;
 
 	/* Internal use fields: */

tools/ccanlint/tests/depends_accurate.c

@@ -37,6 +37,9 @@ static void check_depends_accurate(struct manifest *m,
 {
 	struct list_head *list;
 
+	/* FIXME: This isn't reliable enough with #ifdefs, so we don't fail. */
+	score->pass = true;
+
 	foreach_ptr(list, &m->c_files, &m->h_files,
 		    &m->run_tests, &m->api_tests,
 		    &m->compile_ok_tests, &m->compile_fail_tests,
@@ -63,7 +66,6 @@ static void check_depends_accurate(struct manifest *m,
 	}
 
 	if (!score->error) {
-		score->pass = true;
 		score->score = score->total;
 	}
 }

tools/ccanlint/tests/examples_exist.c

@@ -94,9 +94,10 @@ static void extract_examples(struct manifest *m,
 		}
 	}
 
+	/* We don't fail ccanlint for this. */
+	score->pass = true;
 	if (have_info_example && have_header_example) {
 		score->score = score->total;
-		score->pass = true;
 		return;
 	}
 
@@ -106,8 +107,6 @@ static void extract_examples(struct manifest *m,
 		score_file_error(score, mainh, 0, "No Example: section");
 
 	score->score = have_info_example + have_header_example;
-	/* We pass if we find any example. */
-	score->pass = score->score != 0;
 }
 
 struct ccanlint examples_exist = {

tools/ccanlint/tests/hash_if.c

@@ -25,6 +25,9 @@ static void check_hash_if(struct manifest *m,
 	"\n\t(#if works like #ifdef, but with gcc's -Wundef, we can detect\n"
 	"\tmistyped or unknown configuration options)";
 
+	/* We don't fail ccanlint for this. */ 
+	score->pass = true;
+
 	foreach_ptr(list, &m->c_files, &m->h_files,
 		    &m->run_tests, &m->api_tests,
 		    &m->compile_ok_tests, &m->compile_fail_tests,
@@ -60,7 +63,6 @@ static void check_hash_if(struct manifest *m,
 	}
 
 	if (!score->error) {
-		score->pass = true;
 		score->score = score->total;
 	}
 }

tools/ccanlint/tests/headers_idempotent.c

@@ -179,11 +179,13 @@ static void check_idempotent(struct manifest *m,
 {
 	struct ccan_file *f;
 
+	/* We don't fail ccanlint for this. */
+	score->pass = true;
+
 	list_for_each(&m->h_files, f, list) {
 		check_idem(f, score);
 	}
 	if (!score->error) {
-		score->pass = true;
 		score->score = score->total;
 	}
 }

tools/ccanlint/tests/info_documentation_exists.c

@@ -67,6 +67,9 @@ static void check_info_documentation_exists(struct manifest *m,
 	struct doc_section *d;
 	bool summary = false, description = false;
 
+	/* We don't fail ccanlint for this. */
+	score->pass = true;
+
 	list_for_each(infodocs, d, list) {
 		if (!streq(d->function, m->basename))
 			continue;
@@ -78,7 +81,6 @@ static void check_info_documentation_exists(struct manifest *m,
 
 	if (summary && description) {
 		score->score = score->total;
-		score->pass = true;
 	} else if (!summary) {
 		score->error = talloc_strdup(score,
 		"_info file has no module documentation.\n\n"

tools/ccanlint/tests/license_exists.c

@@ -11,8 +11,6 @@
 #include <ccan/talloc/talloc.h>
 #include <ccan/str/str.h>
 
-REGISTER_TEST(license_exists);
-
 static struct doc_section *find_license(const struct manifest *m)
 {
 	struct doc_section *d;
@@ -79,6 +77,8 @@ static void handle_license_link(struct manifest *m, struct score *score)
 	}
 }
 
+extern struct ccanlint license_exists;
+
 static void check_has_license(struct manifest *m,
 			      bool keep,
 			      unsigned int *timeleft, struct score *score)
@@ -94,6 +94,9 @@ static void check_has_license(struct manifest *m,
 		score->error = talloc_strdup(score, "No License: tag in _info");
 		return;
 	}
+	/* If they have a license tag at all, we pass. */
+	score->pass = true;
+
 	expected = expected_link(m, d);
 
 	len = readlink(license, buf, sizeof(buf));
@@ -157,3 +160,4 @@ struct ccanlint license_exists = {
 	.check = check_has_license,
 	.needs = "info_exists"
 };
+REGISTER_TEST(license_exists);

tools/ccanlint/tests/no_trailing_whitespace.c

@@ -30,6 +30,9 @@ static void check_trailing_whitespace(struct manifest *m,
 	struct ccan_file *f;
 	unsigned int i;
 
+	/* We don't fail ccanlint for this. */
+	score->pass = true;
+
 	foreach_ptr(list, &m->c_files, &m->h_files) {
 		list_for_each(list, f, list) {
 			char **lines = get_ccan_file_lines(f);
@@ -42,7 +45,6 @@ static void check_trailing_whitespace(struct manifest *m,
 		}
 	}
 	if (!score->error) {
-		score->pass = true;
 		score->score = score->total;
 	}
 }

tools/ccanlint/tests/objects_build_with_stringchecks.c

@@ -128,9 +128,11 @@ static void build_objects_with_stringchecks(struct manifest *m,
 		}
 	}
 
+	/* We don't fail ccanlint for this. */
+	score->pass = true;
+
 	score->total = 1;
 	if (!errors) {
-		score->pass = true;
 		score->score = !warnings;
 	}
 }

tools/ccanlint/tests/tests_exist.c

@@ -104,6 +104,8 @@ static void check_tests_exist(struct manifest *m,
 		if (errno != ENOENT)
 			err(1, "statting %s", test_dir);
 		tests_exist.handle = handle_no_tests;
+		/* We "pass" this. */
+		score->pass = true;
 		return;
 	}
 

tools/ccanlint/tests/tests_pass_valgrind.c

@@ -156,9 +156,10 @@ static void do_leakcheck_vg(struct manifest *m,
 		}
 	}
 
+	/* FIXME: We don't fail for this, since many tests leak. */ 
+	score->pass = true;
 	if (!leaks) {
 		score->score = 1;
-		score->pass = true;
 	}
 }