Added submodule API and use in status

When processing status for a newly checked out repo, it is
possible that there will be submodules that have not yet been
initialized.  The only way to distinguish these from untracked
directories is to have some knowledge of submodules.  This
commit adds a new submodule API which, given a name or path,
can determine if it appears to be a submodule and can give
information about the submodule.

include/git2.h

@@ -40,7 +40,7 @@
 #include "git2/net.h"
 #include "git2/status.h"
 #include "git2/indexer.h"
-
+#include "git2/submodule.h"
 #include "git2/notes.h"
 
 #endif

include/git2/submodule.h

+/*
+ * Copyright (C) 2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#ifndef INCLUDE_git_submodule_h__
+#define INCLUDE_git_submodule_h__
+
+#include "common.h"
+#include "types.h"
+#include "oid.h"
+
+/**
+ * @file git2/submodule.h
+ * @brief Git submodule management utilities
+ * @defgroup git_submodule Git submodule management routines
+ * @ingroup Git
+ * @{
+ */
+GIT_BEGIN_DECL
+
+typedef enum {
+	GIT_SUBMODULE_UPDATE_CHECKOUT = 0,
+	GIT_SUBMOUDLE_UPDATE_REBASE = 1,
+	GIT_SUBMODULE_UPDATE_MERGE = 2
+} git_submodule_update_t;
+
+typedef enum {
+	GIT_SUBMODULE_IGNORE_ALL = 0,       /* never dirty */
+	GIT_SUBMODULE_IGNORE_DIRTY = 1,     /* only dirty if HEAD moved */
+	GIT_SUBMODULE_IGNORE_UNTRACKED = 2, /* dirty if tracked files change */
+	GIT_SUBMODULE_IGNORE_NONE = 3       /* any change or untracked == dirty */
+} git_submodule_ignore_t;
+
+/**
+ * Description of submodule
+ *
+ * This record describes a submodule found in a repository.  There
+ * should be an entry for every submodule found in the HEAD and for
+ * every submodule described in .gitmodules.  The fields are as follows:
+ *
+ * - `name` is the name of the submodule from .gitmodules.
+ * - `path` is the path to the submodule from the repo working directory.
+ *   It is almost always the same as `name`.
+ * - `url` is the url for the submodule.
+ * - `oid` is the HEAD SHA1 for the submodule.
+ * - `update` is a value from above - see gitmodules(5) update.
+ * - `ignore` is a value from above - see gitmodules(5) ignore.
+ * - `fetch_recurse` is 0 or 1 - see gitmodules(5) fetchRecurseSubmodules.
+ * - `refcount` is for internal use.
+ *
+ * If the submodule has been added to .gitmodules but not yet git added,
+ * then the `oid` will be zero.  If the submodule has been deleted, but
+ * the delete has not been committed yet, then the `oid` will be set, but
+ * the `url` will be NULL.
+ */
+typedef struct {
+	char *name;
+	char *path;
+	char *url;
+	git_oid oid; /* sha1 of submodule HEAD ref or zero if not committed */
+	git_submodule_update_t update;
+	git_submodule_ignore_t ignore;
+	int fetch_recurse;
+	int refcount;
+} git_submodule;
+
+/**
+ * Iterate over all submodules of a repository.
+ *
+ * @param repo The repository
+ * @param callback Function to be called with the name of each submodule.
+ *        Return a non-zero value to terminate the iteration.
+ * @param payload Extra data to pass to callback
+ * @return 0 on success, -1 on error, or non-zero return value of callback
+ */
+GIT_EXTERN(int) git_submodule_foreach(
+	git_repository *repo,
+	int (*callback)(const char *name, void *payload),
+	void *payload);
+
+#define GIT_SUBMODULE_HEAD "[internal]HEAD"
+
+/**
+ * Lookup submodule information by name or path.
+ *
+ * Given either the submodule name or path (they are ususally the same),
+ * this returns a structure describing the submodule.  If the submodule
+ * does not exist, this will return GIT_ENOTFOUND and set the submodule
+ * pointer to NULL.
+ *
+ * @param submodule Pointer to submodule description object pointer..
+ * @param repo The repository.
+ * @param name The name of the submodule.  Trailing slashes will be ignored.
+ * @return 0 on success, GIT_ENOTFOUND if submodule does not exist, -1 on error
+ */
+GIT_EXTERN(int) git_submodule_lookup(
+	git_submodule **submodule,
+	git_repository *repo,
+	const char *name);
+
+/** @} */
+GIT_END_DECL
+#endif

src/config.c

@@ -201,7 +201,7 @@ int git_config_set_string(git_config *cfg, const char *name, const char *value)
 	return file->set(file, name, value);
 }
 
-static int parse_bool(int *out, const char *value)
+int git_config_parse_bool(int *out, const char *value)
 {
 	/* A missing value means true */
 	if (value == NULL) {
@@ -301,7 +301,7 @@ int git_config_get_mapped(git_config *cfg, const char *name, git_cvar_map *maps,
 			case GIT_CVAR_TRUE: {
 				int bool_val;
 
-				if (parse_bool(&bool_val, value) == 0 && 
+				if (git_config_parse_bool(&bool_val, value) == 0 && 
 					bool_val == (int)m->cvar_type) {
 					*out = m->map_value;
 					return 0;
@@ -372,7 +372,7 @@ int git_config_get_bool(git_config *cfg, const char *name, int *out)
 	if (ret < 0)
 		return ret;
 
-	if (parse_bool(out, value) == 0)
+	if (git_config_parse_bool(out, value) == 0)
 		return 0;
 
 	if (parse_int32(out, value) == 0) {

src/config.h

@@ -25,4 +25,6 @@ struct git_config {
 extern int git_config_find_global_r(git_buf *global_config_path);
 extern int git_config_find_system_r(git_buf *system_config_path);
 
+extern int git_config_parse_bool(int *out, const char *bool_string);
+
 #endif

src/config_file.c

@@ -192,6 +192,9 @@ static int file_foreach(git_config_file *backend, int (*fn)(const char *, const
 	cvar_t *var;
 	const char *key;
 
+	if (!b->values)
+		return 0;
+
 	GIT_HASHTABLE_FOREACH(b->values, key, var,
 		do {
 			if (fn(key, var->value, data) < 0)

src/config_file.h

+/*
+ * Copyright (C) 2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#ifndef INCLUDE_config_file_h__
+#define INCLUDE_config_file_h__
+
+#include "git2/config.h"
+
+GIT_INLINE(int) git_config_file_open(git_config_file *cfg)
+{
+	return cfg->open(cfg);
+}
+
+GIT_INLINE(void) git_config_file_free(git_config_file *cfg)
+{
+	cfg->free(cfg);
+}
+
+GIT_INLINE(int) git_config_file_foreach(
+	git_config_file *cfg,
+	int (*fn)(const char *key, const char *value, void *data),
+	void *data)
+{
+	return cfg->foreach(cfg, fn, data);
+}
+
+#endif
+

src/iterator.c

@@ -9,6 +9,7 @@
 #include "tree.h"
 #include "ignore.h"
 #include "buffer.h"
+#include "git2/submodule.h"
 
 typedef struct tree_iterator_frame tree_iterator_frame;
 struct tree_iterator_frame {
@@ -424,14 +425,25 @@ static int workdir_iterator__update_entry(workdir_iterator *wi)
 		return 0; /* if error, ignore it and ignore file */
 
 	/* detect submodules */
-	if (S_ISDIR(wi->entry.mode) &&
-		git_path_contains(&wi->path, DOT_GIT) == true)
-	{
+	if (S_ISDIR(wi->entry.mode)) {
+		bool is_submodule = git_path_contains(&wi->path, DOT_GIT);
+
+		/* if there is no .git, still check submodules data */
+		if (!is_submodule) {
+			int res = git_submodule_lookup(NULL, wi->repo, wi->entry.path);
+			is_submodule = (res == 0);
+			if (res == GIT_ENOTFOUND)
+				giterr_clear();
+		}
+
+		/* if submodule, mark as GITLINK and remove trailing slash */
+		if (is_submodule) {
 			size_t len = strlen(wi->entry.path);
 			assert(wi->entry.path[len - 1] == '/');
 			wi->entry.path[len - 1] = '\0';
 			wi->entry.mode = S_IFGITLINK;
 		}
+	}
 
 	return 0;
 }
@@ -489,7 +501,9 @@ int git_iterator_advance_into_directory(
 	workdir_iterator *wi = (workdir_iterator *)iter;
 
 	if (iter->type == GIT_ITERATOR_WORKDIR &&
-		wi->entry.path && S_ISDIR(wi->entry.mode))
+		wi->entry.path &&
+		S_ISDIR(wi->entry.mode) &&
+		!S_ISGITLINK(wi->entry.mode))
 	{
 		if (workdir_iterator__expand_dir(wi) < 0)
 			/* if error loading or if empty, skip the directory. */

src/repository.c

@@ -64,6 +64,7 @@ void git_repository_free(git_repository *repo)
 	git_cache_free(&repo->objects);
 	git_repository__refcache_free(&repo->references);
 	git_attr_cache_flush(repo);
+	git_submodule_config_free(repo);
 
 	git__free(repo->path_repository);
 	git__free(repo->workdir);

src/repository.h

@@ -83,6 +83,7 @@ struct git_repository {
 	git_cache objects;
 	git_refcache references;
 	git_attr_cache attrcache;
+	git_hashtable *submodules;
 
 	char *path_repository;
 	char *workdir;
@@ -120,4 +121,9 @@ int git_repository_index__weakptr(git_index **out, git_repository *repo);
 int git_repository__cvar(int *out, git_repository *repo, git_cvar_cached cvar);
 void git_repository__cvar_cache_clear(git_repository *repo);
 
+/*
+ * Submodule cache
+ */
+extern void git_submodule_config_free(git_repository *repo);
+
 #endif

src/submodule.c

+/*
+ * Copyright (C) 2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+
+#include "common.h"
+#include "git2/config.h"
+#include "git2/types.h"
+#include "git2/repository.h"
+#include "git2/index.h"
+#include "git2/submodule.h"
+#include "buffer.h"
+#include "hashtable.h"
+#include "vector.h"
+#include "posix.h"
+#include "config_file.h"
+#include "config.h"
+#include "repository.h"
+
+static const char *sm_update_values[] = {
+	"checkout",
+	"rebase",
+	"merge",
+	NULL
+};
+
+static const char *sm_ignore_values[] = {
+	"all",
+	"dirty",
+	"untracked",
+	"none",
+	NULL
+};
+
+static int lookup_enum(const char **values, const char *str)
+{
+	int i;
+	for (i = 0; values[i]; ++i)
+		if (strcasecmp(str, values[i]) == 0)
+			return i;
+	return -1;
+}
+
+static uint32_t strhash_no_trailing_slash(const void *key, int hash_id)
+{
+	static uint32_t hash_seeds[GIT_HASHTABLE_HASHES] = {
+		0x01010101,
+		0x12345678,
+		0xFEDCBA98
+	};
+
+	size_t key_len = key ? strlen((const char *)key) : 0;
+	if (key_len > 0 && ((const char *)key)[key_len - 1] == '/')
+		key_len--;
+
+	return git__hash(key, (int)key_len, hash_seeds[hash_id]);
+}
+
+static int strcmp_no_trailing_slash(const void *a, const void *b)
+{
+	const char *astr = (const char *)a;
+	const char *bstr = (const char *)b;
+	size_t alen = a ? strlen(astr) : 0;
+	size_t blen = b ? strlen(bstr) : 0;
+	int cmp;
+
+	if (alen > 0 && astr[alen - 1] == '/')
+		alen--;
+	if (blen > 0 && bstr[blen - 1] == '/')
+		blen--;
+
+	cmp = strncmp(astr, bstr, min(alen, blen));
+	if (cmp == 0)
+		cmp = (alen < blen) ? -1 : (alen > blen) ? 1 : 0;
+
+	return cmp;
+}
+
+static git_submodule *submodule_alloc(const char *name)
+{
+	git_submodule *sm = git__calloc(1, sizeof(git_submodule));
+	if (sm == NULL)
+		return sm;
+
+	sm->path = sm->name = git__strdup(name);
+	if (!sm->name) {
+		git__free(sm);
+		return NULL;
+	}
+
+	return sm;
+}
+
+static void submodule_release(git_submodule *sm, int decr)
+{
+	if (!sm)
+		return;
+
+	sm->refcount -= decr;
+
+	if (sm->refcount == 0) {
+		if (sm->name != sm->path)
+			git__free(sm->path);
+		git__free(sm->name);
+		git__free(sm->url);
+		git__free(sm);
+	}
+}
+
+static int submodule_from_entry(
+	git_hashtable *smcfg, git_index_entry *entry)
+{
+	git_submodule *sm;
+	void *old_sm;
+
+	sm = git_hashtable_lookup(smcfg, entry->path);
+	if (!sm)
+		sm = submodule_alloc(entry->path);
+
+	git_oid_cpy(&sm->oid, &entry->oid);
+
+	if (strcmp(sm->path, entry->path) != 0) {
+		if (sm->path != sm->name) {
+			git__free(sm->path);
+			sm->path = sm->name;
+		}
+		sm->path = git__strdup(entry->path);
+		if (!sm->path)
+			goto fail;
+	}
+
+	if (git_hashtable_insert2(smcfg, sm->path, sm, &old_sm) < 0)
+		goto fail;
+	sm->refcount++;
+
+	if (old_sm && ((git_submodule *)old_sm) != sm) {
+		/* TODO: log warning about multiple entrys for same submodule path */
+		submodule_release(old_sm, 1);
+	}
+
+	return 0;
+
+fail:
+	submodule_release(sm, 0);
+	return -1;
+}
+
+static int submodule_from_config(
+	const char *key, const char *value, void *data)
+{
+	git_hashtable *smcfg = data;
+	const char *namestart;
+	const char *property;
+	git_buf name = GIT_BUF_INIT;
+	git_submodule *sm;
+	void *old_sm = NULL;
+	bool is_path;
+
+	if (git__prefixcmp(key, "submodule.") != 0)
+		return 0;
+
+	namestart = key + strlen("submodule.");
+	property  = strrchr(namestart, '.');
+	if (property == NULL)
+		return 0;
+	property++;
+	is_path = (strcmp(property, "path") == 0);
+
+	if (git_buf_set(&name, namestart, property - namestart - 1) < 0)
+		return -1;
+
+	sm = git_hashtable_lookup(smcfg, name.ptr);
+	if (!sm && is_path)
+		sm = git_hashtable_lookup(smcfg, value);
+	if (!sm)
+		sm = submodule_alloc(name.ptr);
+	if (!sm)
+		goto fail;
+
+	if (strcmp(sm->name, name.ptr) != 0) {
+		assert(sm->path == sm->name);
+		sm->name = git_buf_detach(&name);
+		if (git_hashtable_insert2(smcfg, sm->name, sm, &old_sm) < 0)
+			goto fail;
+		sm->refcount++;
+	}
+	else if (is_path && strcmp(sm->path, value) != 0) {
+		assert(sm->path == sm->name);
+		if ((sm->path = git__strdup(value)) == NULL ||
+			git_hashtable_insert2(smcfg, sm->path, sm, &old_sm) < 0)
+			goto fail;
+		sm->refcount++;
+	}
+
+	if (old_sm && ((git_submodule *)old_sm) != sm) {
+		/* TODO: log entry about multiple submodules with same path */
+		submodule_release(old_sm, 1);
+	}
+
+	if (is_path)
+		return 0;
+
+	/* copy other properties into submodule entry */
+	if (strcmp(property, "url") == 0) {
+		if (sm->url) {
+			git__free(sm->url);
+			sm->url = NULL;
+		}
+		if ((sm->url = git__strdup(value)) == NULL)
+			goto fail;
+	}
+	else if (strcmp(property, "update") == 0) {
+		int val = lookup_enum(sm_update_values, value);
+		if (val < 0) {
+			giterr_set(GITERR_INVALID,
+				"Invalid value for submodule update property: '%s'", value);
+			goto fail;
+		}
+		sm->update = (git_submodule_update_t)val;
+	}
+	else if (strcmp(property, "fetchRecurseSubmodules") == 0) {
+		if (git_config_parse_bool(&sm->fetch_recurse, value) < 0)
+			goto fail;
+	}
+	else if (strcmp(property, "ignore") == 0) {
+		int val = lookup_enum(sm_ignore_values, value);
+		if (val < 0) {
+			giterr_set(GITERR_INVALID,
+				"Invalid value for submodule ignore property: '%s'", value);
+			goto fail;
+		}
+		sm->ignore = (git_submodule_ignore_t)val;
+	}
+	/* ignore other unknown submodule properties */
+
+	return 0;
+
+fail:
+	submodule_release(sm, 0);
+	git_buf_free(&name);
+	return -1;
+}
+
+static int load_submodule_config(git_repository *repo)
+{
+	int error;
+	git_index *index;
+	unsigned int i, max_i;
+	git_oid gitmodules_oid;
+	git_hashtable *smcfg;
+	struct git_config_file *mods = NULL;
+
+	if (repo->submodules)
+		return 0;
+
+	/* submodule data is kept in a hashtable with each submodule stored
+	 * under both its name and its path.  These are usually the same, but
+	 * that is not guaranteed.
+	 */
+	smcfg = git_hashtable_alloc(
+		4, strhash_no_trailing_slash, strcmp_no_trailing_slash);
+	GITERR_CHECK_ALLOC(smcfg);
+
+	/* scan index for gitmodules (and .gitmodules entry) */
+	if ((error = git_repository_index(&index, repo)) < 0)
+		goto cleanup;
+	memset(&gitmodules_oid, 0, sizeof(gitmodules_oid));
+	max_i = git_index_entrycount(index);
+
+	for (i = 0; i < max_i; i++) {
+		git_index_entry *entry = git_index_get(index, i);
+		if (S_ISGITLINK(entry->mode)) {
+			if ((error = submodule_from_entry(smcfg, entry)) < 0)
+				goto cleanup;
+		}
+		else if (strcmp(entry->path, ".gitmodules") == 0)
+			git_oid_cpy(&gitmodules_oid, &entry->oid);
+	}
+
+	/* load .gitmodules from workdir if it exists */
+	if (git_repository_workdir(repo) != NULL) {
+		/* look in workdir for .gitmodules */
+		git_buf path = GIT_BUF_INIT;
+		if (!git_buf_joinpath(
+				&path, git_repository_workdir(repo), ".gitmodules") &&
+			git_path_isfile(path.ptr))
+		{
+			if (!(error = git_config_file__ondisk(&mods, path.ptr)))
+				error = git_config_file_open(mods);
+		}
+		git_buf_free(&path);
+	}
+
+	/* load .gitmodules from object cache if not in workdir */
+	if (!error && mods == NULL && !git_oid_iszero(&gitmodules_oid)) {
+		/* TODO: is it worth loading gitmodules from object cache? */
+	}
+
+	/* process .gitmodules info */
+	if (!error && mods != NULL)
+		error = git_config_file_foreach(mods, submodule_from_config, smcfg);
+
+	/* store submodule config in repo */
+	if (!error)
+		repo->submodules = smcfg;
+
+cleanup:
+	if (mods != NULL)
+		git_config_file_free(mods);
+	if (error)
+		git_hashtable_free(smcfg);
+	return error;
+}
+
+void git_submodule_config_free(git_repository *repo)
+{
+	git_hashtable *smcfg = repo->submodules;
+	git_submodule *sm;
+
+	repo->submodules = NULL;
+
+	if (smcfg == NULL)
+		return;
+
+	GIT_HASHTABLE_FOREACH_VALUE(smcfg, sm, { submodule_release(sm,1); });
+	git_hashtable_free(smcfg);
+}
+
+static int submodule_cmp(const void *a, const void *b)
+{
+	return strcmp(((git_submodule *)a)->name, ((git_submodule *)b)->name);
+}
+
+int git_submodule_foreach(
+	git_repository *repo,
+	int (*callback)(const char *name, void *payload),
+	void *payload)
+{
+	int error;
+	git_submodule *sm;
+	git_vector seen = GIT_VECTOR_INIT;
+	seen._cmp = submodule_cmp;
+
+	if ((error = load_submodule_config(repo)) < 0)
+		return error;
+
+	GIT_HASHTABLE_FOREACH_VALUE(
+		repo->submodules, sm, {
+			/* usually the following will not come into play */
+			if (sm->refcount > 1) {
+				if (git_vector_bsearch(&seen, sm) != GIT_ENOTFOUND)
+					continue;
+				if ((error = git_vector_insert(&seen, sm)) < 0)
+					break;
+			}
+
+			if ((error = callback(sm->name, payload)) < 0)
+				break;
+		});
+
+	git_vector_free(&seen);
+
+	return error;
+}
+
+int git_submodule_lookup(
+	git_submodule **sm_ptr, /* NULL allowed if user only wants to test */
+	git_repository *repo,
+	const char *name)       /* trailing slash is allowed */
+{
+	git_submodule *sm;
+
+	if (load_submodule_config(repo) < 0)
+		return -1;
+
+	sm = git_hashtable_lookup(repo->submodules, name);
+
+	if (sm_ptr)
+		*sm_ptr = sm;
+
+	return sm ? 0 : GIT_ENOTFOUND;
+}

tests-clar/status/submodules.c

@@ -28,6 +28,20 @@ void test_status_submodules__cleanup(void)
 	cl_git_sandbox_cleanup();
 }
 
+void test_status_submodules__api(void)
+{
+	git_submodule *sm;
+
+	cl_assert(git_submodule_lookup(NULL, g_repo, "nonexistent") == GIT_ENOTFOUND);
+
+	cl_assert(git_submodule_lookup(NULL, g_repo, "modified") == GIT_ENOTFOUND);
+
+	cl_git_pass(git_submodule_lookup(&sm, g_repo, "testrepo"));
+	cl_assert(sm != NULL);
+	cl_assert_equal_s("testrepo", sm->name);
+	cl_assert_equal_s("testrepo", sm->path);
+}
+
 static int
 cb_status__submodule_count(const char *p, unsigned int s, void *payload)
 {
@@ -79,7 +93,7 @@ cb_status__match(const char *p, unsigned int s, void *payload)
 {
 	volatile int *index = (int *)payload;
 
-	cl_assert_strequal(expected_files[*index], p);
+	cl_assert_equal_s(expected_files[*index], p);
 	cl_assert(expected_status[*index] == s);
 	(*index)++;