Add index pathspec-based operations

This adds three new public APIs for manipulating the index:

1. `git_index_add_all` is similar to `git add -A` and will add
   files in the working directory that match a pathspec to the
   index while honoring ignores, etc.
2. `git_index_remove_all` removes files from the index that match
   a pathspec.
3. `git_index_update_all` updates entries in the index based on
   the current contents of the working directory, either added
   the new information or removing the entry from the index.

include/git2/index.h

@@ -11,6 +11,7 @@
 #include "indexer.h"
 #include "types.h"
 #include "oid.h"
+#include "strarray.h"
 
 /**
  * @file git2/index.h
@@ -125,6 +126,18 @@ typedef enum {
 	GIT_INDEXCAP_FROM_OWNER  = ~0u
 } git_indexcap_t;
 
+/** Callback for APIs that add/remove/update files matching pathspec */
+typedef int (*git_index_matched_path_cb)(
+	const char *path, const char *matched_pathspec, void *payload);
+
+/** Flags for APIs that add files matching pathspec */
+typedef enum {
+	GIT_INDEX_ADD_DEFAULT = 0,
+	GIT_INDEX_ADD_FORCE = (1u << 0),
+	GIT_INDEX_ADD_DISABLE_PATHSPEC_MATCH = (1u << 1),
+	GIT_INDEX_ADD_CHECK_PATHSPEC = (1u << 2),
+} git_index_add_option_t;
+
 /** @name Index File Functions
  *
  * These functions work on the index file itself.
@@ -421,6 +434,108 @@ GIT_EXTERN(int) git_index_add_bypath(git_index *index, const char *path);
 GIT_EXTERN(int) git_index_remove_bypath(git_index *index, const char *path);
 
 /**
+ * Add or update index entries matching files in the working directory.
+ *
+ * This method will fail in bare index instances.
+ *
+ * The `pathspec` is a list of file names or shell glob patterns that will
+ * matched against files in the repository's working directory.  Each file
+ * that matches will be added to the index (either updating an existing
+ * entry or adding a new entry).  You can disable glob expansion and force
+ * exact matching with the `GIT_INDEX_ADD_DISABLE_PATHSPEC_MATCH` flag.
+ *
+ * Files that are ignored will be skipped (unlike `git_index_add_bypath`).
+ * If a file is already tracked in the index, then it *will* be updated
+ * even if it is ignored.  Pass the `GIT_INDEX_ADD_FORCE` flag to
+ * skip the checking of ignore rules.
+ *
+ * To emulate `git add -A` and generate an error if the pathspec contains
+ * the exact path of an ignored file (when not using FORCE), add the
+ * `GIT_INDEX_ADD_CHECK_PATHSPEC` flag.  This checks that each entry
+ * in the `pathspec` that is an exact match to a filename on disk is
+ * either not ignored or already in the index.  If this check fails, the
+ * function will return GIT_EINVALIDSPEC.
+ *
+ * To emulate `git add -A` with the "dry-run" option, just use a callback
+ * function that always returns a positive value.  See below for details.
+ *
+ * If any files are currently the result of a merge conflict, those files
+ * will no longer be marked as conflicting.  The data about the conflicts
+ * will be moved to the "resolve undo" (REUC) section.
+ *
+ * If you provide a callback function, it will be invoked on each matching
+ * item in the working directory immediately *before* it is added to /
+ * updated in the index.  Returning zero will add the item to the index,
+ * greater than zero will skip the item, and less than zero will abort the
+ * scan and cause GIT_EUSER to be returned.
+ *
+ * @param index an existing index object
+ * @param pathspec array of path patterns
+ * @param flags combination of git_index_add_option_t flags
+ * @param callback notification callback for each added/updated path (also
+ *                 gets index of matching pathspec entry); can be NULL;
+ *                 return 0 to add, >0 to skip, <0 to abort scan.
+ * @param payload payload passed through to callback function
+ * @return 0 or an error code
+ */
+GIT_EXTERN(int) git_index_add_all(
+	git_index *index,
+	const git_strarray *pathspec,
+	unsigned int flags,
+	git_index_matched_path_cb callback,
+	void *payload);
+
+/**
+ * Remove all matching index entries.
+ *
+ * If you provide a callback function, it will be invoked on each matching
+ * item in the index immediately *before* it is removed.  Return 0 to
+ * remove the item, > 0 to skip the item, and < 0 to abort the scan.
+ *
+ * @param index An existing index object
+ * @param pathspec array of path patterns
+ * @param callback notification callback for each removed path (also
+ *                 gets index of matching pathspec entry); can be NULL;
+ *                 return 0 to add, >0 to skip, <0 to abort scan.
+ * @param payload payload passed through to callback function
+ * @return 0 or an error code
+ */
+GIT_EXTERN(int) git_index_remove_all(
+	git_index *index,
+	const git_strarray *pathspec,
+	git_index_matched_path_cb callback,
+	void *payload);
+
+/**
+ * Update all index entries to match the working directory
+ *
+ * This method will fail in bare index instances.
+ *
+ * This scans the existing index entries and synchronizes them with the
+ * working directory, deleting them if the corresponding working directory
+ * file no longer exists otherwise updating the information (including
+ * adding the latest version of file to the ODB if needed).
+ *
+ * If you provide a callback function, it will be invoked on each matching
+ * item in the index immediately *before* it is updated (either refreshed
+ * or removed depending on working directory state).  Return 0 to proceed
+ * with updating the item, > 0 to skip the item, and < 0 to abort the scan.
+ *
+ * @param index An existing index object
+ * @param pathspec array of path patterns
+ * @param callback notification callback for each updated path (also
+ *                 gets index of matching pathspec entry); can be NULL;
+ *                 return 0 to add, >0 to skip, <0 to abort scan.
+ * @param payload payload passed through to callback function
+ * @return 0 or an error code
+ */
+GIT_EXTERN(int) git_index_update_all(
+	git_index *index,
+	const git_strarray *pathspec,
+	git_index_matched_path_cb callback,
+	void *payload);
+
+/**
  * Find the first position of any entries which point to given
  * path in the Git index.
  *

src/diff.c

@@ -675,8 +675,10 @@ static int maybe_modified(
 		}
 	}
 
-	/* if oids and modes match, then file is unmodified */
-	else if (git_oid_equal(&oitem->oid, &nitem->oid) && omode == nmode)
+	/* if oids and modes match (and are valid), then file is unmodified */
+	else if (git_oid_equal(&oitem->oid, &nitem->oid) &&
+			 omode == nmode &&
+			 !git_oid_iszero(&oitem->oid))
 		status = GIT_DELTA_UNMODIFIED;
 
 	/* if we have an unknown OID and a workdir iterator, then check some

src/index.c

@@ -15,6 +15,8 @@
 #include "hash.h"
 #include "iterator.h"
 #include "pathspec.h"
+#include "ignore.h"
+
 #include "git2/odb.h"
 #include "git2/oid.h"
 #include "git2/blob.h"
@@ -2044,3 +2046,212 @@ git_repository *git_index_owner(const git_index *index)
 {
 	return INDEX_OWNER(index);
 }
+
+int git_index_add_all(
+	git_index *index,
+	const git_strarray *paths,
+	unsigned int flags,
+	git_index_matched_path_cb cb,
+	void *payload)
+{
+	int error;
+	git_repository *repo;
+	git_iterator *wditer = NULL;
+	const git_index_entry *wd = NULL;
+	git_index_entry *entry;
+	git_pathspec_context ps;
+	const char *match;
+	size_t existing;
+	bool no_fnmatch = (flags & GIT_INDEX_ADD_DISABLE_PATHSPEC_MATCH) != 0;
+	int ignorecase;
+	git_oid blobid;
+
+	assert(index);
+
+	if (INDEX_OWNER(index) == NULL)
+		return create_index_error(-1,
+			"Could not add paths to index. "
+			"Index is not backed up by an existing repository.");
+
+	repo = INDEX_OWNER(index);
+	if ((error = git_repository__ensure_not_bare(repo, "index add all")) < 0)
+		return error;
+
+	if (git_repository__cvar(&ignorecase, repo, GIT_CVAR_IGNORECASE) < 0)
+		return -1;
+
+	if ((error = git_pathspec_context_init(&ps, paths)) < 0)
+		return error;
+
+	/* optionally check that pathspec doesn't mention any ignored files */
+	if ((flags & GIT_INDEX_ADD_CHECK_PATHSPEC) != 0 &&
+		(flags & GIT_INDEX_ADD_FORCE) == 0 &&
+		(error = git_ignore__check_pathspec_for_exact_ignores(
+			repo, &ps.pathspec, no_fnmatch)) < 0)
+		goto cleanup;
+
+	if ((error = git_iterator_for_workdir(
+			&wditer, repo, 0, ps.prefix, ps.prefix)) < 0)
+		goto cleanup;
+
+	while (!(error = git_iterator_advance(&wd, wditer))) {
+
+		/* check if path actually matches */
+		if (!git_pathspec_match_path(
+				&ps.pathspec, wd->path, no_fnmatch, ignorecase, &match))
+			continue;
+
+		/* skip ignored items that are not already in the index */
+		if ((flags & GIT_INDEX_ADD_FORCE) == 0 &&
+			git_iterator_current_is_ignored(wditer) &&
+			index_find(&existing, index, wd->path, 0) < 0)
+			continue;
+
+		/* issue notification callback if requested */
+		if (cb && (error = cb(wd->path, match, payload)) != 0) {
+			if (error > 0) /* return > 0 means skip this one */
+				continue;
+			if (error < 0) { /* return < 0 means abort */
+				giterr_clear();
+				error = GIT_EUSER;
+				break;
+			}
+		}
+
+		/* TODO: Should we check if the file on disk is already an exact
+		 * match to the file in the index and skip this work if it is?
+		 */
+
+		/* write the blob to disk and get the oid */
+		if ((error = git_blob_create_fromworkdir(&blobid, repo, wd->path)) < 0)
+			break;
+
+		/* make the new entry to insert */
+		if ((entry = index_entry_dup(wd)) == NULL) {
+			error = -1;
+			break;
+		}
+		entry->oid = blobid;
+
+		/* add working directory item to index */
+		if ((error = index_insert(index, entry, 1)) < 0) {
+			index_entry_free(entry);
+			break;
+		}
+
+		git_tree_cache_invalidate_path(index->tree, wd->path);
+
+		/* add implies conflict resolved, move conflict entries to REUC */
+		if ((error = index_conflict_to_reuc(index, wd->path)) < 0) {
+			if (error != GIT_ENOTFOUND)
+				break;
+			giterr_clear();
+		}
+	}
+
+	if (error == GIT_ITEROVER)
+		error = 0;
+
+cleanup:
+	git_iterator_free(wditer);
+	git_pathspec_context_free(&ps);
+
+	return error;
+}
+
+enum {
+	INDEX_ACTION_NONE = 0,
+	INDEX_ACTION_UPDATE = 1,
+	INDEX_ACTION_REMOVE = 2,
+};
+
+static int index_apply_to_all(
+	git_index *index,
+	int action,
+	const git_strarray *paths,
+	git_index_matched_path_cb cb,
+	void *payload)
+{
+	int error = 0;
+	size_t i;
+	git_pathspec_context ps;
+	const char *match;
+
+	assert(index);
+
+	if ((error = git_pathspec_context_init(&ps, paths)) < 0)
+		return error;
+
+	git_vector_sort(&index->entries);
+
+	for (i = 0; !error && i < index->entries.length; ++i) {
+		git_index_entry *entry = git_vector_get(&index->entries, i);
+
+		/* check if path actually matches */
+		if (!git_pathspec_match_path(
+				&ps.pathspec, entry->path, false, index->ignore_case, &match))
+			continue;
+
+		/* issue notification callback if requested */
+		if (cb && (error = cb(entry->path, match, payload)) != 0) {
+			if (error > 0) { /* return > 0 means skip this one */
+				error = 0;
+				continue;
+			}
+			if (error < 0) { /* return < 0 means abort */
+				giterr_clear();
+				error = GIT_EUSER;
+				break;
+			}
+		}
+
+		switch (action) {
+		case INDEX_ACTION_NONE:
+			break;
+		case INDEX_ACTION_UPDATE:
+			error = git_index_add_bypath(index, entry->path);
+
+			if (error == GIT_ENOTFOUND) {
+				giterr_clear();
+
+				error = git_index_remove_bypath(index, entry->path);
+
+				if (!error) /* back up foreach if we removed this */
+					i--;
+			}
+			break;
+		case INDEX_ACTION_REMOVE:
+			if (!(error = git_index_remove_bypath(index, entry->path)))
+				i--; /* back up foreach if we removed this */
+			break;
+		default:
+			giterr_set(GITERR_INVALID, "Unknown index action %d", action);
+			error = -1;
+			break;
+		}
+	}
+
+	git_pathspec_context_free(&ps);
+
+	return error;
+}
+
+int git_index_remove_all(
+	git_index *index,
+	const git_strarray *pathspec,
+	git_index_matched_path_cb cb,
+	void *payload)
+{
+	return index_apply_to_all(
+		index, INDEX_ACTION_REMOVE, pathspec, cb, payload);
+}
+
+int git_index_update_all(
+	git_index *index,
+	const git_strarray *pathspec,
+	git_index_matched_path_cb cb,
+	void *payload)
+{
+	return index_apply_to_all(
+		index, INDEX_ACTION_UPDATE, pathspec, cb, payload);
+}

tests-clar/index/addall.c

+#include "clar_libgit2.h"
+#include "../status/status_helpers.h"
+#include "posix.h"
+
+git_repository *g_repo = NULL;
+
+void test_index_addall__initialize(void)
+{
+}
+
+void test_index_addall__cleanup(void)
+{
+	git_repository_free(g_repo);
+	g_repo = NULL;
+}
+
+#define STATUS_INDEX_FLAGS \
+	(GIT_STATUS_INDEX_NEW | GIT_STATUS_INDEX_MODIFIED | \
+	 GIT_STATUS_INDEX_DELETED | GIT_STATUS_INDEX_RENAMED | \
+	 GIT_STATUS_INDEX_TYPECHANGE)
+
+#define STATUS_WT_FLAGS \
+	(GIT_STATUS_WT_NEW | GIT_STATUS_WT_MODIFIED | \
+	 GIT_STATUS_WT_DELETED | GIT_STATUS_WT_TYPECHANGE | \
+	 GIT_STATUS_WT_RENAMED)
+
+typedef struct {
+	size_t index_adds;
+	size_t index_dels;
+	size_t index_mods;
+	size_t wt_adds;
+	size_t wt_dels;
+	size_t wt_mods;
+	size_t ignores;
+} index_status_counts;
+
+static int index_status_cb(
+	const char *path, unsigned int status_flags, void *payload)
+{
+	index_status_counts *vals = payload;
+
+	/* cb_status__print(path, status_flags, NULL); */
+
+	GIT_UNUSED(path);
+
+	if (status_flags & GIT_STATUS_INDEX_NEW)
+		vals->index_adds++;
+	if (status_flags & GIT_STATUS_INDEX_MODIFIED)
+		vals->index_mods++;
+	if (status_flags & GIT_STATUS_INDEX_DELETED)
+		vals->index_dels++;
+	if (status_flags & GIT_STATUS_INDEX_TYPECHANGE)
+		vals->index_mods++;
+
+	if (status_flags & GIT_STATUS_WT_NEW)
+		vals->wt_adds++;
+	if (status_flags & GIT_STATUS_WT_MODIFIED)
+		vals->wt_mods++;
+	if (status_flags & GIT_STATUS_WT_DELETED)
+		vals->wt_dels++;
+	if (status_flags & GIT_STATUS_WT_TYPECHANGE)
+		vals->wt_mods++;
+
+	if (status_flags & GIT_STATUS_IGNORED)
+		vals->ignores++;
+
+	return 0;
+}
+
+static void check_status(
+	git_repository *repo,
+	size_t index_adds, size_t index_dels, size_t index_mods,
+	size_t wt_adds, size_t wt_dels, size_t wt_mods, size_t ignores)
+{
+	index_status_counts vals;
+
+	memset(&vals, 0, sizeof(vals));
+
+	cl_git_pass(git_status_foreach(repo, index_status_cb, &vals));
+
+	cl_assert_equal_sz(index_adds, vals.index_adds);
+	cl_assert_equal_sz(index_dels, vals.index_dels);
+	cl_assert_equal_sz(index_mods, vals.index_mods);
+	cl_assert_equal_sz(wt_adds, vals.wt_adds);
+	cl_assert_equal_sz(wt_dels, vals.wt_dels);
+	cl_assert_equal_sz(wt_mods, vals.wt_mods);
+	cl_assert_equal_sz(ignores, vals.ignores);
+}
+
+static void check_stat_data(git_index *index, const char *path, bool match)
+{
+	const git_index_entry *entry;
+	struct stat st;
+
+	cl_must_pass(p_lstat(path, &st));
+
+	/* skip repo base dir name */
+	while (*path != '/')
+		++path;
+	++path;
+
+	entry = git_index_get_bypath(index, path, 0);
+	cl_assert(entry);
+
+	if (match) {
+		cl_assert(st.st_ctime == entry->ctime.seconds);
+		cl_assert(st.st_mtime == entry->mtime.seconds);
+		cl_assert(st.st_size == entry->file_size);
+		cl_assert(st.st_uid  == entry->uid);
+		cl_assert(st.st_gid  == entry->gid);
+		cl_assert_equal_b(st.st_mode & ~0777, entry->mode & ~0777);
+		cl_assert_equal_b(st.st_mode &  0111, entry->mode &  0111);
+	} else {
+		/* most things will still match */
+		cl_assert(st.st_size != entry->file_size);
+		/* would check mtime, but with second resolution it won't work :( */
+	}
+}
+
+static void commit_index_to_head(
+	git_repository *repo,
+	const char *commit_message)
+{
+	git_index *index;
+	git_oid tree_id, commit_id;
+	git_tree *tree;
+	git_signature *sig;
+	git_commit *parent = NULL;
+
+	git_revparse_single((git_object **)&parent, repo, "HEAD");
+	/* it is okay if looking up the HEAD fails */
+
+	cl_git_pass(git_repository_index(&index, repo));
+	cl_git_pass(git_index_write_tree(&tree_id, index));
+	git_index_free(index);
+
+	cl_git_pass(git_tree_lookup(&tree, repo, &tree_id));
+
+	cl_git_pass(git_signature_now(&sig, "Testy McTester", "tt@tester.test"));
+
+	cl_git_pass(git_commit_create_v(
+		&commit_id, repo, "HEAD", sig, sig,
+		NULL, commit_message, tree, parent ? 1 : 0, parent));
+
+	git_commit_free(parent);
+	git_tree_free(tree);
+	git_signature_free(sig);
+}
+
+void test_index_addall__repo_lifecycle(void)
+{
+	int error;
+	git_index *index;
+	git_strarray paths = { NULL, 0 };
+	char *strs[1];
+
+	cl_git_pass(git_repository_init(&g_repo, "addall", false));
+	check_status(g_repo, 0, 0, 0, 0, 0, 0, 0);
+
+	cl_git_pass(git_repository_index(&index, g_repo));
+
+	cl_git_mkfile("addall/file.foo", "a file");
+	check_status(g_repo, 0, 0, 0, 1, 0, 0, 0);
+
+	cl_git_mkfile("addall/.gitignore", "*.foo\n");
+	check_status(g_repo, 0, 0, 0, 1, 0, 0, 1);
+
+	cl_git_mkfile("addall/file.bar", "another file");
+	check_status(g_repo, 0, 0, 0, 2, 0, 0, 1);
+
+	strs[0] = "file.*";
+	paths.strings = strs;
+	paths.count   = 1;
+
+	cl_git_pass(git_index_add_all(index, &paths, 0, NULL, NULL));
+	check_stat_data(index, "addall/file.bar", true);
+	check_status(g_repo, 1, 0, 0, 1, 0, 0, 1);
+
+	cl_git_rewritefile("addall/file.bar", "new content for file");
+	check_stat_data(index, "addall/file.bar", false);
+	check_status(g_repo, 1, 0, 0, 1, 0, 1, 1);
+
+	cl_git_mkfile("addall/file.zzz", "yet another one");
+	cl_git_mkfile("addall/other.zzz", "yet another one");
+	cl_git_mkfile("addall/more.zzz", "yet another one");
+	check_status(g_repo, 1, 0, 0, 4, 0, 1, 1);
+
+	cl_git_pass(git_index_update_all(index, NULL, NULL, NULL));
+	check_stat_data(index, "addall/file.bar", true);
+	check_status(g_repo, 1, 0, 0, 4, 0, 0, 1);
+
+	cl_git_pass(git_index_add_all(index, &paths, 0, NULL, NULL));
+	check_stat_data(index, "addall/file.zzz", true);
+	check_status(g_repo, 2, 0, 0, 3, 0, 0, 1);
+
+	commit_index_to_head(g_repo, "first commit");
+	check_status(g_repo, 0, 0, 0, 3, 0, 0, 1);
+
+	/* attempt to add an ignored file - does nothing */
+	strs[0] = "file.foo";
+	cl_git_pass(git_index_add_all(index, &paths, 0, NULL, NULL));
+	check_status(g_repo, 0, 0, 0, 3, 0, 0, 1);
+
+	/* add with check - should generate error */
+	error = git_index_add_all(
+		index, &paths, GIT_INDEX_ADD_CHECK_PATHSPEC, NULL, NULL);
+	cl_assert_equal_i(GIT_EINVALIDSPEC, error);
+	check_status(g_repo, 0, 0, 0, 3, 0, 0, 1);
+
+	/* add with force - should allow */
+	cl_git_pass(git_index_add_all(
+		index, &paths, GIT_INDEX_ADD_FORCE, NULL, NULL));
+	check_stat_data(index, "addall/file.foo", true);
+	check_status(g_repo, 1, 0, 0, 3, 0, 0, 0);
+
+	/* now it's in the index, so regular add should work */
+	cl_git_rewritefile("addall/file.foo", "new content for file");
+	check_stat_data(index, "addall/file.foo", false);
+	check_status(g_repo, 1, 0, 0, 3, 0, 1, 0);
+
+	cl_git_pass(git_index_add_all(index, &paths, 0, NULL, NULL));
+	check_stat_data(index, "addall/file.foo", true);
+	check_status(g_repo, 1, 0, 0, 3, 0, 0, 0);
+
+	cl_git_pass(git_index_add_bypath(index, "more.zzz"));
+	check_stat_data(index, "addall/more.zzz", true);
+	check_status(g_repo, 2, 0, 0, 2, 0, 0, 0);
+
+	cl_git_rewritefile("addall/file.zzz", "new content for file");
+	check_status(g_repo, 2, 0, 0, 2, 0, 1, 0);
+
+	cl_git_pass(git_index_add_bypath(index, "file.zzz"));
+	check_stat_data(index, "addall/file.zzz", true);
+	check_status(g_repo, 2, 0, 1, 2, 0, 0, 0);
+
+	strs[0] = "*.zzz";
+	cl_git_pass(git_index_remove_all(index, &paths, NULL, NULL));
+	check_status(g_repo, 1, 1, 0, 4, 0, 0, 0);
+
+	cl_git_pass(git_index_add_bypath(index, "file.zzz"));
+	check_status(g_repo, 1, 0, 1, 3, 0, 0, 0);
+
+	commit_index_to_head(g_repo, "second commit");
+	check_status(g_repo, 0, 0, 0, 3, 0, 0, 0);
+
+	cl_must_pass(p_unlink("addall/file.zzz"));
+	check_status(g_repo, 0, 0, 0, 3, 1, 0, 0);
+
+	/* update_all should be able to remove entries */
+	cl_git_pass(git_index_update_all(index, NULL, NULL, NULL));
+	check_status(g_repo, 0, 1, 0, 3, 0, 0, 0);
+
+	git_index_free(index);
+}