Merge branch 'ccantool'

tools/Makefile

@@ -8,6 +8,7 @@ DEP_OBJS = ccan/grab_file/grab_file.o \
 	ccan/str_talloc/str_talloc.o \
 	ccan/talloc/talloc.o \
 	ccan/time/time.o \
+	tools/read_config_header.o \
 	tools/compile.o \
 	tools/depends.o \
 	tools/tools.o

tools/ccan_dir.c

@@ -0,0 +1,31 @@
+#include <ccan/talloc/talloc.h>
+#include "tools.h"
+#include <assert.h>
+#include <string.h>
+#include <stdlib.h>
+
+/* Walk up to find /ccan/ => ccan directory. */
+static unsigned int ccan_dir_prefix(const char *fulldir)
+{
+	unsigned int i;
+
+	assert(fulldir[0] == '/');
+	for (i = strlen(fulldir) - 1; i > 0; i--) {
+		if (strncmp(fulldir+i, "/ccan", 5) != 0)
+			continue;
+		if (fulldir[i+5] != '\0' && fulldir[i+5] != '/')
+			continue;
+		return i + 1;
+	}
+	return 0;
+}
+
+const char *find_ccan_dir(const char *base)
+{
+	unsigned int prefix = ccan_dir_prefix(base);
+
+	if (!prefix)
+		return NULL;
+
+	return talloc_strndup(NULL, base, prefix);
+}

tools/ccanlint/Makefile

@@ -27,9 +27,12 @@ CORE_OBJS := \
 	tools/ccanlint/ccanlint.o \
 	tools/ccanlint/file_analysis.o \
 	tools/ccanlint/licenses.o \
+	tools/ccan_dir.o \
 	tools/compile.o \
 	tools/depends.o \
 	tools/doc_extract-core.o \
+	tools/manifest.o \
+	tools/read_config_header.o \
 	tools/tools.o
 
 OBJS := $(CORE_OBJS) $(TEST_OBJS)

tools/ccanlint/ccanlint.c

@@ -19,6 +19,7 @@
  */
 #include "ccanlint.h"
 #include "../tools.h"
+#include "../read_config_header.h"
 #include <unistd.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -52,6 +53,8 @@ const char *cflags = NULL;
 
 const char *config_header;
 
+const char *ccan_dir;
+
 #if 0
 static void indent_print(const char *string)
 {
@@ -557,100 +560,6 @@ char **per_file_options(const struct ccanlint *test, struct ccan_file *f)
 	return talloc_realloc(NULL, ret, char *, j + 1);
 }
 
-static char *demangle_string(char *string)
-{
-	unsigned int i;
-	const char mapfrom[] = "abfnrtv";
-	const char mapto[] = "\a\b\f\n\r\t\v";
-
-	if (!strchr(string, '"'))
-		return NULL;
-	string = strchr(string, '"') + 1;
-	if (!strrchr(string, '"'))
-		return NULL;
-	*strrchr(string, '"') = '\0';
-
-	for (i = 0; i < strlen(string); i++) {
-		if (string[i] == '\\') {
-			char repl;
-			unsigned len = 0;
-			const char *p = strchr(mapfrom, string[i+1]);
-			if (p) {
-				repl = mapto[p - mapfrom];
-				len = 1;
-			} else if (strlen(string+i+1) >= 3) {
-				if (string[i+1] == 'x') {
-					repl = (string[i+2]-'0')*16
-						+ string[i+3]-'0';
-					len = 3;
-				} else if (cisdigit(string[i+1])) {
-					repl = (string[i+2]-'0')*8*8
-						+ (string[i+3]-'0')*8
-						+ (string[i+4]-'0');
-					len = 3;
-				}
-			}
-			if (len == 0) {
-				repl = string[i+1];
-				len = 1;
-			}
-
-			string[i] = repl;
-			memmove(string + i + 1, string + i + len + 1,
-				strlen(string + i + len + 1) + 1);
-		}
-	}
-
-	return string;
-}
-
-
-static void read_config_header(void)
-{
-	char *fname = talloc_asprintf(NULL, "%s/config.h", ccan_dir);
-	char **lines;
-	unsigned int i;
-
-	config_header = grab_file(NULL, fname, NULL);
-	if (!config_header) {
-		talloc_free(fname);
-		return;
-	}
-
-	lines = strsplit(config_header, config_header, "\n");
-	for (i = 0; i < talloc_array_length(lines) - 1; i++) {
-		char *sym;
-		const char **line = (const char **)&lines[i];
-
-		if (!get_token(line, "#"))
-			continue;
-		if (!get_token(line, "define"))
-			continue;
-		sym = get_symbol_token(lines, line);
-		if (streq(sym, "CCAN_COMPILER") && !compiler) {
-			compiler = demangle_string(lines[i]);
-			if (!compiler)
-				errx(1, "%s:%u:could not parse CCAN_COMPILER",
-				     fname, i+1);
-			if (verbose > 1)
-				printf("%s: compiler set to '%s'\n",
-				       fname, compiler);
-		} else if (streq(sym, "CCAN_CFLAGS") && !cflags) {
-			cflags = demangle_string(lines[i]);
-			if (!cflags)
-				errx(1, "%s:%u:could not parse CCAN_CFLAGS",
-				     fname, i+1);
-			if (verbose > 1)
-				printf("%s: compiler flags set to '%s'\n",
-				       fname, cflags);
-		}
-	}
-	if (!compiler)
-		compiler = CCAN_COMPILER;
-	if (!cflags)
-		compiler = CCAN_CFLAGS;
-}
-
 static char *opt_set_const_charp(const char *arg, const char **p)
 {
 	return opt_set_charp(arg, cast_const2(char **, p));
@@ -689,7 +598,9 @@ static bool run_tests(struct dgraph_node *all,
 static bool add_to_all(const char *member, struct ccanlint *c,
 		       struct dgraph_node *all)
 {
-	dgraph_add_edge(&c->node, all);
+	/* If we're excluded on cmdline, don't add. */
+	if (!c->skip)
+		dgraph_add_edge(&c->node, all);
 	return true;
 }
 
@@ -775,16 +686,23 @@ int main(int argc, char *argv[])
 			dir[strlen(dir)-1] = '\0';
 
 	got_dir:
+		/* We assume there's a ccan/ in there somewhere... */
+		if (i == 1) {
+			ccan_dir = find_ccan_dir(dir);
+			if (!ccan_dir)
+				errx(1, "Cannot find ccan/ base directory in %s",
+				     dir);
+			config_header = read_config_header(ccan_dir,
+							   &compiler, &cflags,
+							   verbose > 1);
+		}
+
 		if (dir != base_dir)
 			prefix = talloc_append_string(talloc_basename(NULL,dir),
 						      ": ");
 
 		m = get_manifest(talloc_autofree_context(), dir);
 
-		/* FIXME: This has to come after we've got manifest. */
-		if (i == 1)
-			read_config_header();
-
 		/* Create a symlink from temp dir back to src dir's
 		 * test directory. */
 		unlink(testlink);

tools/ccanlint/ccanlint.h

@@ -6,6 +6,7 @@
 #include <ccan/autodata/autodata.h>
 #include <stdbool.h>
 #include "../doc_extract.h"
+#include "../manifest.h"
 #include "licenses.h"
 
 AUTODATA_TYPE(ccanlint_tests, struct ccanlint);
@@ -20,48 +21,6 @@ AUTODATA_TYPE(ccanlint_tests, struct ccanlint);
    4 == Describe every action. */
 extern int verbose;
 
-enum compile_type {
-	COMPILE_NORMAL,
-	COMPILE_NOFEAT,
-	COMPILE_COVERAGE,
-	COMPILE_TYPES
-};
-
-struct manifest {
-	char *dir;
-	/* The module name, ie. final element of dir name */
-	char *basename;
-	struct ccan_file *info_file;
-
-	/* Linked off deps. */
-	struct list_node list;
-	/* Where our final compiled output is */
-	char *compiled[COMPILE_TYPES];
-
-	struct list_head c_files;
-	struct list_head h_files;
-
-	struct list_head run_tests;
-	struct list_head api_tests;
-	struct list_head compile_ok_tests;
-	struct list_head compile_fail_tests;
-	struct list_head other_test_c_files;
-	struct list_head other_test_files;
-
-	struct list_head other_files;
-	struct list_head examples;
-	struct list_head mangled_examples;
-
-	/* From tests/check_depends_exist.c */
-	struct list_head deps;
-
-	/* From tests/license_exists.c */
-	enum license license;
-};
-
-/* Get the manifest for a given directory. */
-struct manifest *get_manifest(const void *ctx, const char *dir);
-
 /* Error in a particular file: stored off score->per_file_errors. */
 struct file_error {
 	struct list_node list;
@@ -158,49 +117,6 @@ struct line_info {
 	struct pp_conditions *cond;
 };
 
-struct ccan_file {
-	struct list_node list;
-
-	/* Name (usually, within m->dir). */
-	char *name;
-
-	/* Full path name. */
-	char *fullname;
-
-	/* Pristine version of the original file.
-	 * Use get_ccan_file_contents to fill this. */
-	const char *contents;
-	size_t contents_size;
-
-	/* Use get_ccan_file_lines / get_ccan_line_info to fill these. */
-	unsigned int num_lines;
-	char **lines;
-	struct line_info *line_info;
-
-	struct list_head *doc_sections;
-
-	/* If this file gets compiled (eg. .C file to .o file), result here. */
-	char *compiled[COMPILE_TYPES];
-
-	/* Filename containing output from valgrind. */
-	char *valgrind_log;
-
-	/* Leak output from valgrind. */
-	char *leak_info;
-
-	/* Simplified stream (lowercase letters and single spaces) */
-	char *simplified;
-};
-
-/* A new ccan_file, with the given name (talloc_steal onto returned value). */
-struct ccan_file *new_ccan_file(const void *ctx, const char *dir, char *name);
-
-/* Use this rather than accessing f->contents directly: loads on demand. */
-const char *get_ccan_file_contents(struct ccan_file *f);
-
-/* Use this rather than accessing f->lines directly: loads on demand. */
-char **get_ccan_file_lines(struct ccan_file *f);
-
 /* Use this rather than accessing f->lines directly: loads on demand. */
 struct line_info *get_ccan_line_info(struct ccan_file *f);
 
@@ -269,13 +185,13 @@ extern bool safe_mode;
 /* Did the user want to keep all the results? */
 extern bool keep_results;
 
-/* Where is the ccan dir?  Available after first manifest. */
-extern const char *ccan_dir;
-
 /* Compiler and CFLAGS, from config.h if available. */
 extern const char *compiler, *cflags;
 
 /* Contents of config.h (or NULL if not found) */
 extern const char *config_header;
 
+/* Where is the ccan dir?   */
+extern const char *ccan_dir;
+
 #endif /* CCAN_LINT_H */

tools/ccanlint/file_analysis.c

@@ -23,47 +23,6 @@
 #include <stdarg.h>
 #include <assert.h>
 
-const char *ccan_dir;
-
-static size_t dir_hash(const char *name)
-{
-	return hash(name, strlen(name), 0);
-}
-
-static const char *manifest_name(const struct manifest *m)
-{
-	return m->dir;
-}
-
-static bool dir_cmp(const struct manifest *m, const char *dir)
-{
-	return strcmp(m->dir, dir) == 0;
-}
-
-HTABLE_DEFINE_TYPE(struct manifest, manifest_name, dir_hash, dir_cmp,
-		   htable_manifest);
-static struct htable_manifest *manifests;
-
-const char *get_ccan_file_contents(struct ccan_file *f)
-{
-	if (!f->contents) {
-		f->contents = grab_file(f, f->fullname, &f->contents_size);
-		if (!f->contents)
-			err(1, "Reading file %s", f->fullname);
-	}
-	return f->contents;
-}
-
-char **get_ccan_file_lines(struct ccan_file *f)
-{
-	if (!f->lines)
-		f->lines = strsplit(f, get_ccan_file_contents(f), "\n");
-
-	/* FIXME: is f->num_lines necessary? */
-	f->num_lines = talloc_array_length(f->lines) - 1;
-	return f->lines;
-}
-
 struct list_head *get_ccan_file_docs(struct ccan_file *f)
 {
 	if (!f->doc_sections) {
@@ -73,223 +32,6 @@ struct list_head *get_ccan_file_docs(struct ccan_file *f)
 	return f->doc_sections;
 }
 
-struct ccan_file *new_ccan_file(const void *ctx, const char *dir, char *name)
-{
-	struct ccan_file *f;
-	unsigned int i;
-
-	assert(dir[0] == '/');
-
-	f = talloc(ctx, struct ccan_file);
-	f->lines = NULL;
-	f->line_info = NULL;
-	f->doc_sections = NULL;
-	for (i = 0; i < ARRAY_SIZE(f->compiled); i++)
-		f->compiled[i] = NULL;
-	f->name = talloc_steal(f, name);
-	f->fullname = talloc_asprintf(f, "%s/%s", dir, f->name);
-	f->contents = NULL;
-	f->simplified = NULL;
-	return f;
-}
-
-static void add_files(struct manifest *m, const char *dir)
-{
-	DIR *d;
-	struct dirent *ent;
-	char **subs = NULL;
-
-	if (dir[0])
-		d = opendir(dir);
-	else
-		d = opendir(".");
-	if (!d)
-		err(1, "Opening directory %s", dir[0] ? dir : ".");
-
-	while ((ent = readdir(d)) != NULL) {
-		struct stat st;
-		struct ccan_file *f;
-		struct list_head *dest;
-		bool is_c_src;
-
-		if (ent->d_name[0] == '.')
-			continue;
-
-		f = new_ccan_file(m, m->dir,
-				  talloc_asprintf(m, "%s%s",
-						  dir, ent->d_name));
-		if (lstat(f->name, &st) != 0)
-			err(1, "lstat %s", f->name);
-
-		if (S_ISDIR(st.st_mode)) {
-			size_t len = talloc_array_length(subs);
-			subs = talloc_realloc(m, subs, char *, len+1);
-			subs[len] = talloc_append_string(f->name, "/");
-			continue;
-		}
-		if (!S_ISREG(st.st_mode)) {
-			talloc_free(f);
-			continue;
-		}
-
-		if (streq(f->name, "_info")) {
-			m->info_file = f;
-			continue;
-		}
-
-		is_c_src = strends(f->name, ".c");
-		if (!is_c_src && !strends(f->name, ".h")) {
-			dest = &m->other_files;
-		} else if (!strchr(f->name, '/')) {
-			if (is_c_src)
-				dest = &m->c_files;
-			else
-				dest = &m->h_files;
-		} else if (strstarts(f->name, "test/")) {
-			if (is_c_src) {
-				if (strstarts(f->name, "test/api"))
-					dest = &m->api_tests;
-				else if (strstarts(f->name, "test/run"))
-					dest = &m->run_tests;
-				else if (strstarts(f->name, "test/compile_ok"))
-					dest = &m->compile_ok_tests;
-				else if (strstarts(f->name, "test/compile_fail"))
-					dest = &m->compile_fail_tests;
-				else
-					dest = &m->other_test_c_files;
-			} else
-				dest = &m->other_test_files;
-		} else
-			dest = &m->other_files;
-
-		list_add(dest, &f->list);
-	}
-	closedir(d);
-
-	/* Before we recurse, sanity check this is a ccan module. */ 
-	if (!dir[0]) {
-		size_t i;
-
-		if (!m->info_file
-		    && list_empty(&m->c_files)
-		    && list_empty(&m->h_files))
-			errx(1, "No _info, C or H files found here!");
-
-		for (i = 0; i < talloc_array_length(subs); i++)
-			add_files(m, subs[i]);
-	}
-	talloc_free(subs);
-}
-
-static int cmp_names(struct ccan_file *const *a, struct ccan_file *const *b,
-		     void *unused)
-{
-	return strcmp((*a)->name, (*b)->name);
-}
-
-static void sort_files(struct list_head *list)
-{
-	struct ccan_file **files = NULL, *f;
-	unsigned int i, num;
-
-	num = 0;
-	while ((f = list_top(list, struct ccan_file, list)) != NULL) {
-		files = talloc_realloc(NULL, files, struct ccan_file *, num+1);
-		files[num++] = f;
-		list_del(&f->list);
-	}
-	asort(files, num, cmp_names, NULL);
-
-	for (i = 0; i < num; i++)
-		list_add_tail(list, &files[i]->list);
-	talloc_free(files);
-}
-
-struct manifest *get_manifest(const void *ctx, const char *dir)
-{
-	struct manifest *m;
-	char *olddir, *canon_dir;
-	unsigned int len;
-	struct list_head *list;
-
-	if (!manifests) {
-		manifests = talloc(NULL, struct htable_manifest);
-		htable_manifest_init(manifests);
-	}
-
-	olddir = talloc_getcwd(NULL);
-	if (!olddir)
-		err(1, "Getting current directory");
-
-	if (chdir(dir) != 0)
-		err(1, "Failed to chdir to %s", dir);
-
-	canon_dir = talloc_getcwd(olddir);
-	if (!canon_dir)
-		err(1, "Getting current directory");
-
-	m = htable_manifest_get(manifests, canon_dir);
-	if (m)
-		goto done;
-
-	m = talloc_linked(ctx, talloc(NULL, struct manifest));
-	m->info_file = NULL;
-	m->compiled[COMPILE_NORMAL] = m->compiled[COMPILE_NOFEAT] = NULL;
-	m->dir = talloc_steal(m, canon_dir);
-	list_head_init(&m->c_files);
-	list_head_init(&m->h_files);
-	list_head_init(&m->api_tests);
-	list_head_init(&m->run_tests);
-	list_head_init(&m->compile_ok_tests);
-	list_head_init(&m->compile_fail_tests);
-	list_head_init(&m->other_test_c_files);
-	list_head_init(&m->other_test_files);
-	list_head_init(&m->other_files);
-	list_head_init(&m->examples);
-	list_head_init(&m->mangled_examples);
-	list_head_init(&m->deps);
-
-	len = strlen(m->dir);
-	while (len && m->dir[len-1] == '/')
-		m->dir[--len] = '\0';
-
-	m->basename = strrchr(m->dir, '/');
-	if (!m->basename)
-		errx(1, "I don't expect to be run from the root directory");
-	m->basename++;
-
-	/* We expect the ccan dir to be two levels above module dir. */
-	if (!ccan_dir) {
-		char *p, *dir;
-		dir = talloc_strdup(NULL, m->dir);
-		p = strrchr(dir, '/');
-		if (!p)
-			errx(1, "I expect the ccan root directory in ../..");
-		*p = '\0';
-		p = strrchr(dir, '/');
-		if (!p)
-			errx(1, "I expect the ccan root directory in ../..");
-		*p = '\0';
-		ccan_dir = dir;
-	}
-
-	add_files(m, "");
-
-	/* Nicer to run tests in a predictable order. */
-	foreach_ptr(list, &m->api_tests, &m->run_tests, &m->compile_ok_tests,
-		    &m->compile_fail_tests)
-		sort_files(list);
-
-	htable_manifest_add(manifests, m);
-
-done:
-	if (chdir(olddir) != 0)
-		err(1, "Returning to original directory '%s'", olddir);
-	talloc_free(olddir);
-
-	return m;
-}
-
 
 /**
  * remove_comments - strip comments from a line, return copy.
@@ -363,40 +105,6 @@ static bool continues(const char *line)
 	return strends(line, "\\");
 }
 
-/* Get token if it's equal to token. */
-bool get_token(const char **line, const char *token)
-{
-	unsigned int toklen;
-
-	*line += strspn(*line, " \t");
-	if (cisalnum(token[0]) || token[0] == '_')
-		toklen = strspn(*line, IDENT_CHARS);
-	else {
-		/* FIXME: real tokenizer handles ++ and other multi-chars.  */
-		toklen = strlen(token);
-	}
-
-	if (toklen == strlen(token) && !strncmp(*line, token, toklen)) {
-		*line += toklen;
-		return true;
-	}
-	return false;
-}
-
-char *get_symbol_token(void *ctx, const char **line)
-{
-	unsigned int toklen;
-	char *ret;
-
-	*line += strspn(*line, " \t");
-	toklen = strspn(*line, IDENT_CHARS);
-	if (!toklen)
-		return NULL;
-	ret = talloc_strndup(ctx, *line, toklen);
-	*line += toklen;
-	return ret;
-}
-
 static bool parse_hash_if(struct pp_conditions *cond, const char **line)
 {
 	bool brackets, defined;

tools/manifest.c

@@ -0,0 +1,265 @@
+#include "config.h"
+#include "manifest.h"
+#include "tools.h"
+#include <ccan/talloc/talloc.h>
+#include <ccan/str/str.h>
+#include <ccan/str_talloc/str_talloc.h>
+#include <ccan/talloc_link/talloc_link.h>
+#include <ccan/hash/hash.h>
+#include <ccan/htable/htable_type.h>
+#include <ccan/grab_file/grab_file.h>
+#include <ccan/noerr/noerr.h>
+#include <ccan/foreach/foreach.h>
+#include <ccan/asort/asort.h>
+#include <ccan/array_size/array_size.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <err.h>
+#include <errno.h>
+#include <dirent.h>
+#include <ctype.h>
+#include <stdarg.h>
+#include <assert.h>
+
+static size_t dir_hash(const char *name)
+{
+	return hash(name, strlen(name), 0);
+}
+
+static const char *manifest_name(const struct manifest *m)
+{
+	return m->dir;
+}
+
+static bool dir_cmp(const struct manifest *m, const char *dir)
+{
+	return strcmp(m->dir, dir) == 0;
+}
+
+HTABLE_DEFINE_TYPE(struct manifest, manifest_name, dir_hash, dir_cmp,
+		   htable_manifest);
+static struct htable_manifest *manifests;
+
+const char *get_ccan_file_contents(struct ccan_file *f)
+{
+	if (!f->contents) {
+		f->contents = grab_file(f, f->fullname, &f->contents_size);
+		if (!f->contents)
+			err(1, "Reading file %s", f->fullname);
+	}
+	return f->contents;
+}
+
+char **get_ccan_file_lines(struct ccan_file *f)
+{
+	if (!f->lines)
+		f->lines = strsplit(f, get_ccan_file_contents(f), "\n");
+
+	/* FIXME: is f->num_lines necessary? */
+	f->num_lines = talloc_array_length(f->lines) - 1;
+	return f->lines;
+}
+
+struct ccan_file *new_ccan_file(const void *ctx, const char *dir, char *name)
+{
+	struct ccan_file *f;
+	unsigned int i;
+
+	assert(dir[0] == '/');
+
+	f = talloc(ctx, struct ccan_file);
+	f->lines = NULL;
+	f->line_info = NULL;
+	f->doc_sections = NULL;
+	for (i = 0; i < ARRAY_SIZE(f->compiled); i++)
+		f->compiled[i] = NULL;
+	f->name = talloc_steal(f, name);
+	f->fullname = talloc_asprintf(f, "%s/%s", dir, f->name);
+	f->contents = NULL;
+	f->simplified = NULL;
+	return f;
+}
+
+static void add_files(struct manifest *m, const char *dir)
+{
+	DIR *d;
+	struct dirent *ent;
+	char **subs = NULL;
+
+	if (dir[0])
+		d = opendir(dir);
+	else
+		d = opendir(".");
+	if (!d)
+		err(1, "Opening directory %s", dir[0] ? dir : ".");
+
+	while ((ent = readdir(d)) != NULL) {
+		struct stat st;
+		struct ccan_file *f;
+		struct list_head *dest;
+		bool is_c_src;
+
+		if (ent->d_name[0] == '.')
+			continue;
+
+		f = new_ccan_file(m, m->dir,
+				  talloc_asprintf(m, "%s%s",
+						  dir, ent->d_name));
+		if (lstat(f->name, &st) != 0)
+			err(1, "lstat %s", f->name);
+
+		if (S_ISDIR(st.st_mode)) {
+			size_t len = talloc_array_length(subs);
+			subs = talloc_realloc(m, subs, char *, len+1);
+			subs[len] = talloc_append_string(f->name, "/");
+			continue;
+		}
+		if (!S_ISREG(st.st_mode)) {
+			talloc_free(f);
+			continue;
+		}
+
+		if (streq(f->name, "_info")) {
+			m->info_file = f;
+			continue;
+		}
+
+		is_c_src = strends(f->name, ".c");
+		if (!is_c_src && !strends(f->name, ".h")) {
+			dest = &m->other_files;
+		} else if (!strchr(f->name, '/')) {
+			if (is_c_src)
+				dest = &m->c_files;
+			else
+				dest = &m->h_files;
+		} else if (strstarts(f->name, "test/")) {
+			if (is_c_src) {
+				if (strstarts(f->name, "test/api"))
+					dest = &m->api_tests;
+				else if (strstarts(f->name, "test/run"))
+					dest = &m->run_tests;
+				else if (strstarts(f->name, "test/compile_ok"))
+					dest = &m->compile_ok_tests;
+				else if (strstarts(f->name, "test/compile_fail"))
+					dest = &m->compile_fail_tests;
+				else
+					dest = &m->other_test_c_files;
+			} else
+				dest = &m->other_test_files;
+		} else
+			dest = &m->other_files;
+
+		list_add(dest, &f->list);
+	}
+	closedir(d);
+
+	/* Before we recurse, sanity check this is a ccan module. */
+	if (!dir[0]) {
+		size_t i;
+
+		if (!m->info_file
+		    && list_empty(&m->c_files)
+		    && list_empty(&m->h_files))
+			errx(1, "No _info, C or H files found here!");
+
+		for (i = 0; i < talloc_array_length(subs); i++)
+			add_files(m, subs[i]);
+	}
+	talloc_free(subs);
+}
+
+static int cmp_names(struct ccan_file *const *a, struct ccan_file *const *b,
+		     void *unused)
+{
+	return strcmp((*a)->name, (*b)->name);
+}
+
+static void sort_files(struct list_head *list)
+{
+	struct ccan_file **files = NULL, *f;
+	unsigned int i, num;
+
+	num = 0;
+	while ((f = list_top(list, struct ccan_file, list)) != NULL) {
+		files = talloc_realloc(NULL, files, struct ccan_file *, num+1);
+		files[num++] = f;
+		list_del(&f->list);
+	}
+	asort(files, num, cmp_names, NULL);
+
+	for (i = 0; i < num; i++)
+		list_add_tail(list, &files[i]->list);
+	talloc_free(files);
+}
+
+struct manifest *get_manifest(const void *ctx, const char *dir)
+{
+	struct manifest *m;
+	char *olddir, *canon_dir;
+	unsigned int len;
+	struct list_head *list;
+
+	if (!manifests) {
+		manifests = talloc(NULL, struct htable_manifest);
+		htable_manifest_init(manifests);
+	}
+
+	olddir = talloc_getcwd(NULL);
+	if (!olddir)
+		err(1, "Getting current directory");
+
+	if (chdir(dir) != 0)
+		err(1, "Failed to chdir to %s", dir);
+
+	canon_dir = talloc_getcwd(olddir);
+	if (!canon_dir)
+		err(1, "Getting current directory");
+
+	m = htable_manifest_get(manifests, canon_dir);
+	if (m)
+		goto done;
+
+	m = talloc_linked(ctx, talloc(NULL, struct manifest));
+	m->info_file = NULL;
+	m->compiled[COMPILE_NORMAL] = m->compiled[COMPILE_NOFEAT] = NULL;
+	m->dir = talloc_steal(m, canon_dir);
+	list_head_init(&m->c_files);
+	list_head_init(&m->h_files);
+	list_head_init(&m->api_tests);
+	list_head_init(&m->run_tests);
+	list_head_init(&m->compile_ok_tests);
+	list_head_init(&m->compile_fail_tests);
+	list_head_init(&m->other_test_c_files);
+	list_head_init(&m->other_test_files);
+	list_head_init(&m->other_files);
+	list_head_init(&m->examples);
+	list_head_init(&m->mangled_examples);
+	list_head_init(&m->deps);
+
+	len = strlen(m->dir);
+	while (len && m->dir[len-1] == '/')
+		m->dir[--len] = '\0';
+
+	m->basename = strrchr(m->dir, '/');
+	if (!m->basename)
+		errx(1, "I don't expect to be run from the root directory");
+	m->basename++;
+
+	add_files(m, "");
+
+	/* Nicer to run tests in a predictable order. */
+	foreach_ptr(list, &m->api_tests, &m->run_tests, &m->compile_ok_tests,
+		    &m->compile_fail_tests)
+		sort_files(list);
+
+	htable_manifest_add(manifests, m);
+
+done:
+	if (chdir(olddir) != 0)
+		err(1, "Returning to original directory '%s'", olddir);
+	talloc_free(olddir);
+
+	return m;
+}

tools/manifest.h

@@ -0,0 +1,92 @@
+#ifndef CCAN_TOOLS_MANIFEST_H
+#define CCAN_TOOLS_MANIFEST_H
+#include "config.h"
+#include "ccanlint/licenses.h"
+#include <ccan/list/list.h>
+
+enum compile_type {
+	COMPILE_NORMAL,
+	COMPILE_NOFEAT,
+	COMPILE_COVERAGE,
+	COMPILE_TYPES
+};
+
+struct manifest {
+	char *dir;
+	/* The module name, ie. final element of dir name */
+	char *basename;
+	struct ccan_file *info_file;
+
+	/* Linked off deps. */
+	struct list_node list;
+	/* Where our final compiled output is */
+	char *compiled[COMPILE_TYPES];
+
+	struct list_head c_files;
+	struct list_head h_files;
+
+	struct list_head run_tests;
+	struct list_head api_tests;
+	struct list_head compile_ok_tests;
+	struct list_head compile_fail_tests;
+	struct list_head other_test_c_files;
+	struct list_head other_test_files;
+
+	struct list_head other_files;
+	struct list_head examples;
+	struct list_head mangled_examples;
+
+	/* From tests/check_depends_exist.c */
+	struct list_head deps;
+
+	/* From tests/license_exists.c */
+	enum license license;
+};
+
+/* Get the manifest for a given directory. */
+struct manifest *get_manifest(const void *ctx, const char *dir);
+
+struct ccan_file {
+	struct list_node list;
+
+	/* Name (usually, within m->dir). */
+	char *name;
+
+	/* Full path name. */
+	char *fullname;
+
+	/* Pristine version of the original file.
+	 * Use get_ccan_file_contents to fill this. */
+	const char *contents;
+	size_t contents_size;
+
+	/* Use get_ccan_file_lines / get_ccan_line_info to fill these. */
+	unsigned int num_lines;
+	char **lines;
+	struct line_info *line_info;
+
+	struct list_head *doc_sections;
+
+	/* If this file gets compiled (eg. .C file to .o file), result here. */
+	char *compiled[COMPILE_TYPES];
+
+	/* Filename containing output from valgrind. */
+	char *valgrind_log;
+
+	/* Leak output from valgrind. */
+	char *leak_info;
+
+	/* Simplified stream (lowercase letters and single spaces) */
+	char *simplified;
+};
+
+/* A new ccan_file, with the given name (talloc_steal onto returned value). */
+struct ccan_file *new_ccan_file(const void *ctx, const char *dir, char *name);
+
+/* Use this rather than accessing f->contents directly: loads on demand. */
+const char *get_ccan_file_contents(struct ccan_file *f);
+
+/* Use this rather than accessing f->lines directly: loads on demand. */
+char **get_ccan_file_lines(struct ccan_file *f);
+
+#endif /* CCAN_TOOLS_MANIFEST_H */

tools/read_config_header.c

@@ -0,0 +1,143 @@
+#include <ccan/grab_file/grab_file.h>
+#include <ccan/str/str.h>
+#include <ccan/str_talloc/str_talloc.h>
+#include <ccan/talloc/talloc.h>
+#include "read_config_header.h"
+#include "tools.h"
+#include <string.h>
+#include <err.h>
+
+/* Get an identifier token. */
+char *get_symbol_token(void *ctx, const char **line)
+{
+	unsigned int toklen;
+	char *ret;
+
+	*line += strspn(*line, " \t");
+	toklen = strspn(*line, IDENT_CHARS);
+	if (!toklen)
+		return NULL;
+	ret = talloc_strndup(ctx, *line, toklen);
+	*line += toklen;
+	return ret;
+}
+
+/* Get token if it's equal to token. */
+bool get_token(const char **line, const char *token)
+{
+	unsigned int toklen;
+
+	*line += strspn(*line, " \t");
+	if (cisalnum(token[0]) || token[0] == '_')
+		toklen = strspn(*line, IDENT_CHARS);
+	else {
+		/* FIXME: real tokenizer handles ++ and other multi-chars.  */
+		toklen = strlen(token);
+	}
+
+	if (toklen == strlen(token) && !strncmp(*line, token, toklen)) {
+		*line += toklen;
+		return true;
+	}
+	return false;
+}
+
+static char *demangle_string(char *string)
+{
+	unsigned int i;
+	const char mapfrom[] = "abfnrtv";
+	const char mapto[] = "\a\b\f\n\r\t\v";
+
+	if (!strchr(string, '"'))
+		return NULL;
+	string = strchr(string, '"') + 1;
+	if (!strrchr(string, '"'))
+		return NULL;
+	*strrchr(string, '"') = '\0';
+
+	for (i = 0; i < strlen(string); i++) {
+		if (string[i] == '\\') {
+			char repl;
+			unsigned len = 0;
+			const char *p = strchr(mapfrom, string[i+1]);
+			if (p) {
+				repl = mapto[p - mapfrom];
+				len = 1;
+			} else if (strlen(string+i+1) >= 3) {
+				if (string[i+1] == 'x') {
+					repl = (string[i+2]-'0')*16
+						+ string[i+3]-'0';
+					len = 3;
+				} else if (cisdigit(string[i+1])) {
+					repl = (string[i+2]-'0')*8*8
+						+ (string[i+3]-'0')*8
+						+ (string[i+4]-'0');
+					len = 3;
+				}
+			}
+			if (len == 0) {
+				repl = string[i+1];
+				len = 1;
+			}
+
+			string[i] = repl;
+			memmove(string + i + 1, string + i + len + 1,
+				strlen(string + i + len + 1) + 1);
+		}
+	}
+
+	return string;
+}
+
+char *read_config_header(const char *ccan_dir,
+			 const char **compiler, const char **cflags,
+			 bool verbose)
+{
+	char *fname = talloc_asprintf(NULL, "%s/config.h", ccan_dir);
+	char **lines;
+	unsigned int i;
+	char *config_header;
+
+	config_header = grab_file(NULL, fname, NULL);
+	talloc_free(fname);
+
+	if (!config_header)
+		goto out;
+
+	lines = strsplit(config_header, config_header, "\n");
+	for (i = 0; i < talloc_array_length(lines) - 1; i++) {
+		char *sym;
+		const char **line = (const char **)&lines[i];
+
+		if (!get_token(line, "#"))
+			continue;
+		if (!get_token(line, "define"))
+			continue;
+		sym = get_symbol_token(lines, line);
+		if (streq(sym, "CCAN_COMPILER") && !compiler) {
+			*compiler = demangle_string(lines[i]);
+			if (!*compiler)
+				errx(1, "%s:%u:could not parse CCAN_COMPILER",
+				     fname, i+1);
+			if (verbose)
+				printf("%s: compiler set to '%s'\n",
+				       fname, *compiler);
+		} else if (streq(sym, "CCAN_CFLAGS") && !cflags) {
+			*cflags = demangle_string(lines[i]);
+			if (!*cflags)
+				errx(1, "%s:%u:could not parse CCAN_CFLAGS",
+				     fname, i+1);
+			if (verbose)
+				printf("%s: compiler flags set to '%s'\n",
+				       fname, *cflags);
+		}
+	}
+
+out:
+	if (!*compiler)
+		*compiler = CCAN_COMPILER;
+	if (!*cflags)
+		*cflags = CCAN_CFLAGS;
+
+	return config_header;
+}

tools/read_config_header.h

@@ -0,0 +1,16 @@
+#ifndef CCAN_TOOLS_READ_CONFIG_HEADER_H
+#define CCAN_TOOLS_READ_CONFIG_HEADER_H
+#include <stdbool.h>
+
+/* Get token if it's equal to token. */
+bool get_token(const char **line, const char *token);
+
+/* Get an identifier token. */
+char *get_symbol_token(void *ctx, const char **line);
+
+/* Read config header from config_dir/config.h: set compiler/cflags. */
+char *read_config_header(const char *config_dir,
+			 const char **compiler, const char **cflags,
+			 bool verbose);
+
+#endif /* CCAN_TOOLS_READ_CONFIG_HEADER_H */

tools/tools.h

@@ -75,4 +75,6 @@ extern const unsigned int default_timeout_ms;
 /* Talloc destructor which unlinks file. */
 int unlink_file_destructor(char *filename);
 
+/* Get ccan/ top dir, given a directory within it. */
+const char *find_ccan_dir(const char *base);
 #endif /* CCAN_TOOLS_H */