Recursing into ignored dirs for diff and status

This implements working versions of GIT_DIFF_RECURSE_IGNORED_DIRS
and GIT_STATUS_OPT_RECURSE_IGNORED_DIRS along with some tests for
the newly available behaviors.  This is not turned on by default
for status, but can be accessed via the options to the extended
version of the command.

include/git2/diff.h

@@ -123,7 +123,7 @@ typedef enum {
 	 *  will be marked with only a single entry in the diff list; this flag
 	 *  adds all files under the directory as IGNORED entries, too.
 	 */
-	GIT_DIFF_RECURSE_IGNORED_DIRS = (1 << 10),
+	GIT_DIFF_RECURSE_IGNORED_DIRS = (1 << 18),
 } git_diff_option_t;
 
 /**

include/git2/status.h

@@ -127,18 +127,22 @@ typedef enum {
  *   will.
  * - GIT_STATUS_OPT_DISABLE_PATHSPEC_MATCH indicates that the given path
  *   will be treated as a literal path, and not as a pathspec.
+ * - GIT_STATUS_OPT_RECURSE_IGNORED_DIRS indicates that the contents of
+ *   ignored directories should be included in the status.  This is like
+ *   doing `git ls-files -o -i --exclude-standard` with core git.
  *
  * Calling `git_status_foreach()` is like calling the extended version
  * with: GIT_STATUS_OPT_INCLUDE_IGNORED, GIT_STATUS_OPT_INCLUDE_UNTRACKED,
  * and GIT_STATUS_OPT_RECURSE_UNTRACKED_DIRS.
  */
 typedef enum {
-	GIT_STATUS_OPT_INCLUDE_UNTRACKED = (1 << 0),
-	GIT_STATUS_OPT_INCLUDE_IGNORED = (1 << 1),
-	GIT_STATUS_OPT_INCLUDE_UNMODIFIED = (1 << 2),
-	GIT_STATUS_OPT_EXCLUDE_SUBMODULES = (1 << 3),
-	GIT_STATUS_OPT_RECURSE_UNTRACKED_DIRS = (1 << 4),
-	GIT_STATUS_OPT_DISABLE_PATHSPEC_MATCH = (1 << 5),
+	GIT_STATUS_OPT_INCLUDE_UNTRACKED      = (1u << 0),
+	GIT_STATUS_OPT_INCLUDE_IGNORED        = (1u << 1),
+	GIT_STATUS_OPT_INCLUDE_UNMODIFIED     = (1u << 2),
+	GIT_STATUS_OPT_EXCLUDE_SUBMODULES     = (1u << 3),
+	GIT_STATUS_OPT_RECURSE_UNTRACKED_DIRS = (1u << 4),
+	GIT_STATUS_OPT_DISABLE_PATHSPEC_MATCH = (1u << 5),
+	GIT_STATUS_OPT_RECURSE_IGNORED_DIRS   = (1u << 6),
 } git_status_opt_t;
 
 /**

src/diff.c

@@ -12,6 +12,9 @@
 #include "filter.h"
 #include "pathspec.h"
 
+#define DIFF_FLAG_IS_SET(DIFF,FLAG) (((DIFF)->opts.flags & (FLAG)) != 0)
+#define DIFF_FLAG_ISNT_SET(DIFF,FLAG) (((DIFF)->opts.flags & (FLAG)) == 0)
+
 static git_diff_delta *diff_delta__alloc(
 	git_diff_list *diff,
 	git_delta_t status,
@@ -29,7 +32,7 @@ static git_diff_delta *diff_delta__alloc(
 
 	delta->new_file.path = delta->old_file.path;
 
-	if (diff->opts.flags & GIT_DIFF_REVERSE) {
+	if (DIFF_FLAG_IS_SET(diff, GIT_DIFF_REVERSE)) {
 		switch (status) {
 		case GIT_DELTA_ADDED:   status = GIT_DELTA_DELETED; break;
 		case GIT_DELTA_DELETED: status = GIT_DELTA_ADDED; break;
@@ -63,17 +66,18 @@ static int diff_delta__from_one(
 	int notify_res;
 
 	if (status == GIT_DELTA_IGNORED &&
-		(diff->opts.flags & GIT_DIFF_INCLUDE_IGNORED) == 0)
+		DIFF_FLAG_ISNT_SET(diff, GIT_DIFF_INCLUDE_IGNORED))
 		return 0;
 
 	if (status == GIT_DELTA_UNTRACKED &&
-		(diff->opts.flags & GIT_DIFF_INCLUDE_UNTRACKED) == 0)
+		DIFF_FLAG_ISNT_SET(diff, GIT_DIFF_INCLUDE_UNTRACKED))
 		return 0;
 
 	if (!git_pathspec_match_path(
 			&diff->pathspec, entry->path,
-			(diff->opts.flags & GIT_DIFF_DISABLE_PATHSPEC_MATCH) != 0,
-			(diff->opts.flags & GIT_DIFF_DELTAS_ARE_ICASE) != 0, &matched_pathspec))
+			DIFF_FLAG_IS_SET(diff, GIT_DIFF_DISABLE_PATHSPEC_MATCH),
+			DIFF_FLAG_IS_SET(diff, GIT_DIFF_DELTAS_ARE_ICASE),
+			&matched_pathspec))
 		return 0;
 
 	delta = diff_delta__alloc(diff, status, entry->path);
@@ -124,10 +128,10 @@ static int diff_delta__from_two(
 	int notify_res;
 
 	if (status == GIT_DELTA_UNMODIFIED &&
-		(diff->opts.flags & GIT_DIFF_INCLUDE_UNMODIFIED) == 0)
+		DIFF_FLAG_ISNT_SET(diff, GIT_DIFF_INCLUDE_UNMODIFIED))
 		return 0;
 
-	if ((diff->opts.flags & GIT_DIFF_REVERSE) != 0) {
+	if (DIFF_FLAG_IS_SET(diff, GIT_DIFF_REVERSE)) {
 		uint32_t temp_mode = old_mode;
 		const git_index_entry *temp_entry = old_entry;
 		old_entry = new_entry;
@@ -149,7 +153,7 @@ static int diff_delta__from_two(
 	delta->new_file.mode = new_mode;
 
 	if (new_oid) {
-		if ((diff->opts.flags & GIT_DIFF_REVERSE) != 0)
+		if (DIFF_FLAG_IS_SET(diff, GIT_DIFF_REVERSE))
 			git_oid_cpy(&delta->old_file.oid, new_oid);
 		else
 			git_oid_cpy(&delta->new_file.oid, new_oid);
@@ -316,14 +320,14 @@ static git_diff_list *git_diff_list_alloc(
 	if (!diff->opts.old_prefix || !diff->opts.new_prefix)
 		goto fail;
 
-	if (diff->opts.flags & GIT_DIFF_REVERSE) {
+	if (DIFF_FLAG_IS_SET(diff, GIT_DIFF_REVERSE)) {
 		const char *swap = diff->opts.old_prefix;
 		diff->opts.old_prefix = diff->opts.new_prefix;
 		diff->opts.new_prefix = swap;
 	}
 
 	/* INCLUDE_TYPECHANGE_TREES implies INCLUDE_TYPECHANGE */
-	if (diff->opts.flags & GIT_DIFF_INCLUDE_TYPECHANGE_TREES)
+	if (DIFF_FLAG_IS_SET(diff, GIT_DIFF_INCLUDE_TYPECHANGE_TREES))
 		diff->opts.flags |= GIT_DIFF_INCLUDE_TYPECHANGE;
 
 	return diff;
@@ -452,8 +456,9 @@ static int maybe_modified(
 
 	if (!git_pathspec_match_path(
 			&diff->pathspec, oitem->path,
-			(diff->opts.flags & GIT_DIFF_DISABLE_PATHSPEC_MATCH) != 0,
-			(diff->opts.flags & GIT_DIFF_DELTAS_ARE_ICASE) != 0, &matched_pathspec))
+			DIFF_FLAG_IS_SET(diff, GIT_DIFF_DISABLE_PATHSPEC_MATCH),
+			DIFF_FLAG_IS_SET(diff, GIT_DIFF_DELTAS_ARE_ICASE),
+			&matched_pathspec))
 		return 0;
 
 	/* on platforms with no symlinks, preserve mode of existing symlinks */
@@ -478,7 +483,7 @@ static int maybe_modified(
 
 	/* if basic type of file changed, then split into delete and add */
 	else if (GIT_MODE_TYPE(omode) != GIT_MODE_TYPE(nmode)) {
-		if ((diff->opts.flags & GIT_DIFF_INCLUDE_TYPECHANGE) != 0)
+		if (DIFF_FLAG_IS_SET(diff, GIT_DIFF_INCLUDE_TYPECHANGE))
 			status = GIT_DELTA_TYPECHANGE;
 		else {
 			if (diff_delta__from_one(diff, GIT_DELTA_DELETED, oitem) < 0 ||
@@ -515,7 +520,7 @@ static int maybe_modified(
 			int err;
 			git_submodule *sub;
 
-			if ((diff->opts.flags & GIT_DIFF_IGNORE_SUBMODULES) != 0)
+			if (DIFF_FLAG_IS_SET(diff, GIT_DIFF_IGNORE_SUBMODULES))
 				status = GIT_DELTA_UNMODIFIED;
 			else if ((err = git_submodule_lookup(&sub, diff->repo, nitem->path)) < 0) {
 				if (err == GIT_EEXISTS)
@@ -626,7 +631,7 @@ int git_diff__from_iterators(
 	if (!diff || diff_list_init_from_iterators(diff, old_iter, new_iter) < 0)
 		goto fail;
 
-	if (diff->opts.flags & GIT_DIFF_DELTAS_ARE_ICASE) {
+	if (DIFF_FLAG_IS_SET(diff, GIT_DIFF_DELTAS_ARE_ICASE)) {
 		if (git_iterator_set_ignore_case(old_iter, true) < 0 ||
 			git_iterator_set_ignore_case(new_iter, true) < 0)
 			goto fail;
@@ -648,7 +653,7 @@ int git_diff__from_iterators(
 			/* if we are generating TYPECHANGE records then check for that
 			 * instead of just generating a DELETE record
 			 */
-			if ((diff->opts.flags & GIT_DIFF_INCLUDE_TYPECHANGE_TREES) != 0 &&
+			if (DIFF_FLAG_IS_SET(diff, GIT_DIFF_INCLUDE_TYPECHANGE_TREES) &&
 				entry_is_prefixed(diff, nitem, oitem))
 			{
 				/* this entry has become a tree! convert to TYPECHANGE */
@@ -663,7 +668,7 @@ int git_diff__from_iterators(
 				 * Unless RECURSE_UNTRACKED_DIRS is set, skip over them...
 				 */
 				if (S_ISDIR(nitem->mode) &&
-					!(diff->opts.flags & GIT_DIFF_RECURSE_UNTRACKED_DIRS))
+					DIFF_FLAG_ISNT_SET(diff, GIT_DIFF_RECURSE_UNTRACKED_DIRS))
 				{
 					if (git_iterator_advance(&nitem, new_iter) < 0)
 						goto fail;
@@ -691,20 +696,22 @@ int git_diff__from_iterators(
 				 * it or if the user requested the contents of untracked
 				 * directories and it is not under an ignored directory.
 				 */
-				bool recurse_untracked =
+				bool recurse_into_dir =
 					(delta_type == GIT_DELTA_UNTRACKED &&
-					 (diff->opts.flags & GIT_DIFF_RECURSE_UNTRACKED_DIRS) != 0);
+					 DIFF_FLAG_IS_SET(diff, GIT_DIFF_RECURSE_UNTRACKED_DIRS)) ||
+					(delta_type == GIT_DELTA_IGNORED &&
+					 DIFF_FLAG_IS_SET(diff, GIT_DIFF_RECURSE_IGNORED_DIRS));
 
 				/* do not advance into directories that contain a .git file */
-				if (!contains_oitem && recurse_untracked) {
+				if (!contains_oitem && recurse_into_dir) {
 					git_buf *full = NULL;
 					if (git_iterator_current_workdir_path(&full, new_iter) < 0)
 						goto fail;
 					if (git_path_contains_dir(full, DOT_GIT))
-						recurse_untracked = false;
+						recurse_into_dir = false;
 				}
 
-				if (contains_oitem || recurse_untracked) {
+				if (contains_oitem || recurse_into_dir) {
 					/* if this directory is ignored, remember it as the
 					 * "ignore_prefix" for processing contained items
 					 */
@@ -744,7 +751,8 @@ int git_diff__from_iterators(
 			 * checked before container directory exclusions are used to
 			 * skip the file.
 			 */
-			else if (delta_type == GIT_DELTA_IGNORED) {
+			else if (delta_type == GIT_DELTA_IGNORED &&
+					 DIFF_FLAG_ISNT_SET(diff, GIT_DIFF_RECURSE_IGNORED_DIRS)) {
 				if (git_iterator_advance(&nitem, new_iter) < 0)
 					goto fail;
 				continue; /* ignored parent directory, so skip completely */
@@ -763,7 +771,7 @@ int git_diff__from_iterators(
 			 * instead of just generating an ADDED/UNTRACKED record
 			 */
 			if (delta_type != GIT_DELTA_IGNORED &&
-				(diff->opts.flags & GIT_DIFF_INCLUDE_TYPECHANGE_TREES) != 0 &&
+				DIFF_FLAG_IS_SET(diff, GIT_DIFF_INCLUDE_TYPECHANGE_TREES) &&
 				contains_oitem)
 			{
 				/* this entry was prefixed with a tree - make TYPECHANGE */

src/status.c

@@ -142,7 +142,8 @@ int git_status_foreach_ext(
 		diffopt.flags = diffopt.flags | GIT_DIFF_RECURSE_UNTRACKED_DIRS;
 	if ((opts->flags & GIT_STATUS_OPT_DISABLE_PATHSPEC_MATCH) != 0)
 		diffopt.flags = diffopt.flags | GIT_DIFF_DISABLE_PATHSPEC_MATCH;
-	/* TODO: support EXCLUDE_SUBMODULES flag */
+	if ((opts->flags & GIT_STATUS_OPT_RECURSE_IGNORED_DIRS) != 0)
+		diffopt.flags = diffopt.flags | GIT_DIFF_RECURSE_IGNORED_DIRS;
 
 	if (show != GIT_STATUS_SHOW_WORKDIR_ONLY &&
 		(err = git_diff_tree_to_index(&idx2head, repo, head, NULL, &diffopt)) < 0)

tests-clar/status/ignore.c

@@ -199,7 +199,6 @@ void test_status_ignore__subdirectories(void)
 	cl_git_pass(git_status_should_ignore(&ignored, g_repo, "ignore_me"));
 	cl_assert(ignored);
 
-
 	/* So, interestingly, as per the comment in diff_from_iterators() the
 	 * following file is ignored, but in a way so that it does not show up
 	 * in status even if INCLUDE_IGNORED is used.  This actually matches
@@ -207,11 +206,12 @@ void test_status_ignore__subdirectories(void)
 	 * status -uall --ignored" then the following file and directory will
 	 * not show up in the output at all.
 	 */
-
-	cl_git_pass(
-		git_futils_mkdir_r("empty_standard_repo/test/ignore_me", NULL, 0775));
+	cl_git_pass(git_futils_mkdir_r(
+		"empty_standard_repo/test/ignore_me", NULL, 0775));
 	cl_git_mkfile(
 		"empty_standard_repo/test/ignore_me/file", "I'm going to be ignored!");
+	cl_git_mkfile(
+		"empty_standard_repo/test/ignore_me/file2", "Me, too!");
 
 	memset(&st, 0, sizeof(st));
 	cl_git_pass(git_status_foreach(g_repo, cb_status__single, &st));
@@ -225,6 +225,61 @@ void test_status_ignore__subdirectories(void)
 	cl_assert(ignored);
 }
 
+void test_status_ignore__subdirectories_recursion(void)
+{
+	/* Let's try again with recursing into ignored dirs turned on */
+	git_status_options opts = GIT_STATUS_OPTIONS_INIT;
+	status_entry_counts counts;
+	static const char *paths[] = {
+		".gitignore",
+		"ignore_me",
+		"test/ignore_me/and_me/file",
+		"test/ignore_me/file",
+		"test/ignore_me/file2",
+	};
+	static const unsigned int statuses[] = {
+		GIT_STATUS_WT_NEW,
+		GIT_STATUS_IGNORED,
+		GIT_STATUS_IGNORED,
+		GIT_STATUS_IGNORED,
+		GIT_STATUS_IGNORED,
+	};
+
+	opts.flags = GIT_STATUS_OPT_INCLUDE_IGNORED |
+		GIT_STATUS_OPT_RECURSE_IGNORED_DIRS |
+		GIT_STATUS_OPT_INCLUDE_UNTRACKED |
+		GIT_STATUS_OPT_RECURSE_UNTRACKED_DIRS;
+
+	g_repo = cl_git_sandbox_init("empty_standard_repo");
+
+	cl_git_rewritefile("empty_standard_repo/.gitignore", "ignore_me\n");
+
+	cl_git_mkfile(
+		"empty_standard_repo/ignore_me", "I'm going to be ignored!");
+	cl_git_pass(git_futils_mkdir_r(
+		"empty_standard_repo/test/ignore_me", NULL, 0775));
+	cl_git_mkfile(
+		"empty_standard_repo/test/ignore_me/file", "I'm going to be ignored!");
+	cl_git_mkfile(
+		"empty_standard_repo/test/ignore_me/file2", "Me, too!");
+	cl_git_pass(git_futils_mkdir_r(
+		"empty_standard_repo/test/ignore_me/and_me", NULL, 0775));
+	cl_git_mkfile(
+		"empty_standard_repo/test/ignore_me/and_me/file", "Deeply ignored");
+
+	memset(&counts, 0x0, sizeof(status_entry_counts));
+	counts.expected_entry_count = 5;
+	counts.expected_paths = paths;
+	counts.expected_statuses = statuses;
+
+	cl_git_pass(git_status_foreach_ext(
+		g_repo, &opts, cb_status__normal, &counts));
+
+	cl_assert_equal_i(counts.expected_entry_count, counts.entry_count);
+	cl_assert_equal_i(0, counts.wrong_status_flags_count);
+	cl_assert_equal_i(0, counts.wrong_sorted_path);
+}
+
 void test_status_ignore__adding_internal_ignores(void)
 {
 	int ignored;