Merge pull request #1507 from arrbee/fix-look-inside-untracked-directory

Update diff handling of "untracked" directories

include/git2/diff.h

@@ -88,42 +88,61 @@ typedef enum {
 	GIT_DIFF_INCLUDE_UNTRACKED = (1 << 8),
 	/** Include unmodified files in the diff list */
 	GIT_DIFF_INCLUDE_UNMODIFIED = (1 << 9),
+
 	/** Even with GIT_DIFF_INCLUDE_UNTRACKED, an entire untracked directory
 	 *  will be marked with only a single entry in the diff list; this flag
 	 *  adds all files under the directory as UNTRACKED entries, too.
 	 */
 	GIT_DIFF_RECURSE_UNTRACKED_DIRS = (1 << 10),
+
 	/** If the pathspec is set in the diff options, this flags means to
 	 *  apply it as an exact match instead of as an fnmatch pattern.
 	 */
 	GIT_DIFF_DISABLE_PATHSPEC_MATCH = (1 << 11),
+
 	/** Use case insensitive filename comparisons */
 	GIT_DIFF_DELTAS_ARE_ICASE = (1 << 12),
+
 	/** When generating patch text, include the content of untracked files */
 	GIT_DIFF_INCLUDE_UNTRACKED_CONTENT = (1 << 13),
+
 	/** Disable updating of the `binary` flag in delta records.  This is
 	 *  useful when iterating over a diff if you don't need hunk and data
 	 *  callbacks and want to avoid having to load file completely.
 	 */
 	GIT_DIFF_SKIP_BINARY_CHECK = (1 << 14),
+
 	/** Normally, a type change between files will be converted into a
 	 *  DELETED record for the old and an ADDED record for the new; this
 	 *  options enabled the generation of TYPECHANGE delta records.
 	 */
 	GIT_DIFF_INCLUDE_TYPECHANGE = (1 << 15),
+
 	/** Even with GIT_DIFF_INCLUDE_TYPECHANGE, blob->tree changes still
 	 *  generally show as a DELETED blob.  This flag tries to correctly
 	 *  label blob->tree transitions as TYPECHANGE records with new_file's
 	 *  mode set to tree.  Note: the tree SHA will not be available.
 	 */
 	GIT_DIFF_INCLUDE_TYPECHANGE_TREES  = (1 << 16),
+
 	/** Ignore file mode changes */
 	GIT_DIFF_IGNORE_FILEMODE = (1 << 17),
+
 	/** Even with GIT_DIFF_INCLUDE_IGNORED, an entire ignored directory
 	 *  will be marked with only a single entry in the diff list; this flag
 	 *  adds all files under the directory as IGNORED entries, too.
 	 */
 	GIT_DIFF_RECURSE_IGNORED_DIRS = (1 << 18),
+
+	/** Core Git scans inside untracked directories, labeling them IGNORED
+	 *  if they are empty or only contain ignored files; a directory is
+	 *  consider UNTRACKED only if it has an actual untracked file in it.
+	 *  This scan is extra work for a case you often don't care about.  This
+	 *  flag makes libgit2 immediately label an untracked directory as
+	 *  UNTRACKED without looking insde it (which differs from core Git).
+	 *  Of course, ignore rules are still checked for the directory itself.
+	 */
+	GIT_DIFF_FAST_UNTRACKED_DIRS = (1 << 19),
 } git_diff_option_t;
 
 /**

include/git2/submodule.h

@@ -103,20 +103,20 @@ typedef enum {
  * * WD_UNTRACKED      - wd contains untracked files
  */
 typedef enum {
-	 GIT_SUBMODULE_STATUS_IN_HEAD           = (1u << 0),
-	 GIT_SUBMODULE_STATUS_IN_INDEX          = (1u << 1),
-	 GIT_SUBMODULE_STATUS_IN_CONFIG         = (1u << 2),
-	 GIT_SUBMODULE_STATUS_IN_WD             = (1u << 3),
-	 GIT_SUBMODULE_STATUS_INDEX_ADDED       = (1u << 4),
-	 GIT_SUBMODULE_STATUS_INDEX_DELETED     = (1u << 5),
-	 GIT_SUBMODULE_STATUS_INDEX_MODIFIED    = (1u << 6),
-	 GIT_SUBMODULE_STATUS_WD_UNINITIALIZED  = (1u << 7),
-	 GIT_SUBMODULE_STATUS_WD_ADDED          = (1u << 8),
-	 GIT_SUBMODULE_STATUS_WD_DELETED        = (1u << 9),
-	 GIT_SUBMODULE_STATUS_WD_MODIFIED       = (1u << 10),
-	 GIT_SUBMODULE_STATUS_WD_INDEX_MODIFIED = (1u << 11),
-	 GIT_SUBMODULE_STATUS_WD_WD_MODIFIED    = (1u << 12),
-	 GIT_SUBMODULE_STATUS_WD_UNTRACKED      = (1u << 13),
+	GIT_SUBMODULE_STATUS_IN_HEAD           = (1u << 0),
+	GIT_SUBMODULE_STATUS_IN_INDEX          = (1u << 1),
+	GIT_SUBMODULE_STATUS_IN_CONFIG         = (1u << 2),
+	GIT_SUBMODULE_STATUS_IN_WD             = (1u << 3),
+	GIT_SUBMODULE_STATUS_INDEX_ADDED       = (1u << 4),
+	GIT_SUBMODULE_STATUS_INDEX_DELETED     = (1u << 5),
+	GIT_SUBMODULE_STATUS_INDEX_MODIFIED    = (1u << 6),
+	GIT_SUBMODULE_STATUS_WD_UNINITIALIZED  = (1u << 7),
+	GIT_SUBMODULE_STATUS_WD_ADDED          = (1u << 8),
+	GIT_SUBMODULE_STATUS_WD_DELETED        = (1u << 9),
+	GIT_SUBMODULE_STATUS_WD_MODIFIED       = (1u << 10),
+	GIT_SUBMODULE_STATUS_WD_INDEX_MODIFIED = (1u << 11),
+	GIT_SUBMODULE_STATUS_WD_WD_MODIFIED    = (1u << 12),
+	GIT_SUBMODULE_STATUS_WD_UNTRACKED      = (1u << 13),
 } git_submodule_status_t;
 
 #define GIT_SUBMODULE_STATUS__IN_FLAGS \

src/diff.c

@@ -327,8 +327,7 @@ static git_diff_list *diff_list_alloc(
 	/* Use case-insensitive compare if either iterator has
 	 * the ignore_case bit set */
 	if (!git_iterator_ignore_case(old_iter) &&
-		!git_iterator_ignore_case(new_iter))
-	{
+		!git_iterator_ignore_case(new_iter)) {
 		diff->opts.flags &= ~GIT_DIFF_DELTAS_ARE_ICASE;
 
 		diff->strcomp    = git__strcmp;
@@ -530,24 +529,30 @@ cleanup:
 	return result;
 }
 
+typedef struct {
+	git_repository *repo;
+	git_iterator *old_iter;
+	git_iterator *new_iter;
+	const git_index_entry *oitem;
+	const git_index_entry *nitem;
+	git_buf ignore_prefix;
+} diff_in_progress;
+
 #define MODE_BITS_MASK 0000777
 
 static int maybe_modified(
-	git_iterator *old_iter,
-	const git_index_entry *oitem,
-	git_iterator *new_iter,
-	const git_index_entry *nitem,
-	git_diff_list *diff)
+	git_diff_list *diff,
+	diff_in_progress *info)
 {
 	git_oid noid, *use_noid = NULL;
 	git_delta_t status = GIT_DELTA_MODIFIED;
+	const git_index_entry *oitem = info->oitem;
+	const git_index_entry *nitem = info->nitem;
 	unsigned int omode = oitem->mode;
 	unsigned int nmode = nitem->mode;
-	bool new_is_workdir = (new_iter->type == GIT_ITERATOR_TYPE_WORKDIR);
+	bool new_is_workdir = (info->new_iter->type == GIT_ITERATOR_TYPE_WORKDIR);
 	const char *matched_pathspec;
 
-	GIT_UNUSED(old_iter);
-
 	if (!git_pathspec_match_path(
 			&diff->pathspec, oitem->path,
 			DIFF_FLAG_IS_SET(diff, GIT_DIFF_DISABLE_PATHSPEC_MATCH),
@@ -692,208 +697,311 @@ static bool entry_is_prefixed(
 			item->path[pathlen] == '/');
 }
 
-int git_diff__from_iterators(
-	git_diff_list **diff_ptr,
-	git_repository *repo,
-	git_iterator *old_iter,
-	git_iterator *new_iter,
-	const git_diff_options *opts)
+static int diff_scan_inside_untracked_dir(
+	git_diff_list *diff, diff_in_progress *info, git_delta_t *delta_type)
 {
 	int error = 0;
-	const git_index_entry *oitem, *nitem;
-	git_buf ignore_prefix = GIT_BUF_INIT;
-	git_diff_list *diff;
+	git_buf base = GIT_BUF_INIT;
+	bool is_ignored;
 
-	*diff_ptr = NULL;
+	*delta_type = GIT_DELTA_IGNORED;
+	git_buf_sets(&base, info->nitem->path);
 
-	diff = diff_list_alloc(repo, old_iter, new_iter);
-	GITERR_CHECK_ALLOC(diff);
+	/* advance into untracked directory */
+	if ((error = git_iterator_advance_into(&info->nitem, info->new_iter)) < 0) {
 
-	/* make iterators have matching icase behavior */
-	if (DIFF_FLAG_IS_SET(diff, GIT_DIFF_DELTAS_ARE_ICASE)) {
-		if (git_iterator_set_ignore_case(old_iter, true) < 0 ||
-			git_iterator_set_ignore_case(new_iter, true) < 0)
-			goto fail;
+		/* skip ahead if empty */
+		if (error == GIT_ENOTFOUND) {
+			giterr_clear();
+			error = git_iterator_advance(&info->nitem, info->new_iter);
+		}
+
+		return error;
 	}
 
-	if (diff_list_apply_options(diff, opts) < 0 ||
-		git_iterator_current(&oitem, old_iter) < 0 ||
-		git_iterator_current(&nitem, new_iter) < 0)
-		goto fail;
+	/* look for actual untracked file */
+	while (!diff->pfxcomp(info->nitem->path, git_buf_cstr(&base))) {
+		is_ignored = git_iterator_current_is_ignored(info->new_iter);
 
-	/* run iterators building diffs */
-	while (oitem || nitem) {
-		int cmp = oitem ? (nitem ? diff->entrycomp(oitem, nitem) : -1) : 1;
+		/* need to recurse into non-ignored directories */
+		if (!is_ignored && S_ISDIR(info->nitem->mode)) {
+			if ((error = git_iterator_advance_into(
+					 &info->nitem, info->new_iter)) < 0)
+				break;
+			continue;
+		}
 
-		/* create DELETED records for old items not matched in new */
-		if (cmp < 0) {
-			if (diff_delta__from_one(diff, GIT_DELTA_DELETED, oitem) < 0)
-				goto fail;
+		/* found a non-ignored item - treat parent dir as untracked */
+		if (!is_ignored) {
+			*delta_type = GIT_DELTA_UNTRACKED;
+			break;
+		}
 
-			/* if we are generating TYPECHANGE records then check for that
-			 * instead of just generating a DELETE record
-			 */
-			if (DIFF_FLAG_IS_SET(diff, GIT_DIFF_INCLUDE_TYPECHANGE_TREES) &&
-				entry_is_prefixed(diff, nitem, oitem))
-			{
-				/* this entry has become a tree! convert to TYPECHANGE */
-				git_diff_delta *last = diff_delta__last_for_item(diff, oitem);
-				if (last) {
-					last->status = GIT_DELTA_TYPECHANGE;
-					last->new_file.mode = GIT_FILEMODE_TREE;
-				}
+		if ((error = git_iterator_advance(&info->nitem, info->new_iter)) < 0)
+			break;
+	}
 
-				/* If new_iter is a workdir iterator, then this situation
-				 * will certainly be followed by a series of untracked items.
-				 * Unless RECURSE_UNTRACKED_DIRS is set, skip over them...
-				 */
-				if (S_ISDIR(nitem->mode) &&
-					DIFF_FLAG_ISNT_SET(diff, GIT_DIFF_RECURSE_UNTRACKED_DIRS))
-				{
-					if (git_iterator_advance(&nitem, new_iter) < 0)
-						goto fail;
-				}
-			}
+	/* finish off scan */
+	while (!diff->pfxcomp(info->nitem->path, git_buf_cstr(&base))) {
+		if ((error = git_iterator_advance(&info->nitem, info->new_iter)) < 0)
+			break;
+	}
 
-			if (git_iterator_advance(&oitem, old_iter) < 0)
-				goto fail;
-		}
+	git_buf_free(&base);
 
-		/* create ADDED, TRACKED, or IGNORED records for new items not
-		 * matched in old (and/or descend into directories as needed)
-		 */
-		else if (cmp > 0) {
-			git_delta_t delta_type = GIT_DELTA_UNTRACKED;
-			bool contains_oitem = entry_is_prefixed(diff, oitem, nitem);
-
-			/* check if contained in ignored parent directory */
-			if (git_buf_len(&ignore_prefix) &&
-				diff->pfxcomp(nitem->path, git_buf_cstr(&ignore_prefix)) == 0)
-				delta_type = GIT_DELTA_IGNORED;
-
-			if (S_ISDIR(nitem->mode)) {
-				/* recurse into directory only if there are tracked items in
-				 * it or if the user requested the contents of untracked
-				 * directories and it is not under an ignored directory.
-				 */
-				bool recurse_into_dir =
-					(delta_type == GIT_DELTA_UNTRACKED &&
-					 DIFF_FLAG_IS_SET(diff, GIT_DIFF_RECURSE_UNTRACKED_DIRS)) ||
-					(delta_type == GIT_DELTA_IGNORED &&
-					 DIFF_FLAG_IS_SET(diff, GIT_DIFF_RECURSE_IGNORED_DIRS));
-
-				/* do not advance into directories that contain a .git file */
-				if (!contains_oitem && recurse_into_dir) {
-					git_buf *full = NULL;
-					if (git_iterator_current_workdir_path(&full, new_iter) < 0)
-						goto fail;
-					if (git_path_contains_dir(full, DOT_GIT))
-						recurse_into_dir = false;
-				}
+	return error;
+}
 
-				/* if directory is ignored, remember ignore_prefix */
-				if ((contains_oitem || recurse_into_dir) &&
-					delta_type == GIT_DELTA_UNTRACKED &&
-					git_iterator_current_is_ignored(new_iter))
-				{
-					git_buf_sets(&ignore_prefix, nitem->path);
-					delta_type = GIT_DELTA_IGNORED;
-
-					/* skip recursion if we've just learned this is ignored */
-					if (DIFF_FLAG_ISNT_SET(diff, GIT_DIFF_RECURSE_IGNORED_DIRS))
-						recurse_into_dir = false;
-				}
+static int handle_unmatched_new_item(
+	git_diff_list *diff, diff_in_progress *info)
+{
+	int error = 0;
+	const git_index_entry *nitem = info->nitem;
+	git_delta_t delta_type = GIT_DELTA_UNTRACKED;
+	bool contains_oitem;
+
+	/* check if this is a prefix of the other side */
+	contains_oitem = entry_is_prefixed(diff, info->oitem, nitem);
 
-				if (contains_oitem || recurse_into_dir) {
-					/* advance into directory */
-					error = git_iterator_advance_into(&nitem, new_iter);
+	/* check if this is contained in an ignored parent directory */
+	if (git_buf_len(&info->ignore_prefix)) {
+		if (diff->pfxcomp(nitem->path, git_buf_cstr(&info->ignore_prefix)) == 0)
+			delta_type = GIT_DELTA_IGNORED;
+		else
+			git_buf_clear(&info->ignore_prefix);
+	}
 
-					/* if directory is empty, can't advance into it, so skip */
-					if (error == GIT_ENOTFOUND) {
-						giterr_clear();
-						error = git_iterator_advance(&nitem, new_iter);
+	if (S_ISDIR(nitem->mode)) {
+		bool recurse_into_dir = contains_oitem;
 
-						git_buf_clear(&ignore_prefix);
-					}
+		/* if not already inside an ignored dir, check if this is ignored */
+		if (delta_type != GIT_DELTA_IGNORED &&
+			git_iterator_current_is_ignored(info->new_iter)) {
+			delta_type = GIT_DELTA_IGNORED;
+			git_buf_sets(&info->ignore_prefix, nitem->path);
+		}
 
-					if (error < 0)
-						goto fail;
-					continue;
+		/* check if user requests recursion into this type of dir */
+		recurse_into_dir = contains_oitem ||
+			(delta_type == GIT_DELTA_UNTRACKED &&
+			 DIFF_FLAG_IS_SET(diff, GIT_DIFF_RECURSE_UNTRACKED_DIRS)) ||
+			(delta_type == GIT_DELTA_IGNORED &&
+			 DIFF_FLAG_IS_SET(diff, GIT_DIFF_RECURSE_IGNORED_DIRS));
+
+		/* do not advance into directories that contain a .git file */
+		if (recurse_into_dir) {
+			git_buf *full = NULL;
+			if (git_iterator_current_workdir_path(&full, info->new_iter) < 0)
+				return -1;
+			if (full && git_path_contains_dir(full, DOT_GIT))
+				recurse_into_dir = false;
+		}
+
+		/* still have to look into untracked directories to match core git -
+		 * with no untracked files, directory is treated as ignored
+		 */
+		if (!recurse_into_dir &&
+			delta_type == GIT_DELTA_UNTRACKED &&
+			DIFF_FLAG_ISNT_SET(diff, GIT_DIFF_FAST_UNTRACKED_DIRS))
+		{
+			git_diff_delta *last;
+
+			/* attempt to insert record for this directory */
+			if ((error = diff_delta__from_one(diff, delta_type, nitem)) < 0)
+				return error;
+
+			/* if delta wasn't created (because of rules), just skip ahead */
+			last = diff_delta__last_for_item(diff, nitem);
+			if (!last)
+				return git_iterator_advance(&info->nitem, info->new_iter);
+
+			/* iterate into dir looking for an actual untracked file */
+			if (diff_scan_inside_untracked_dir(diff, info, &delta_type) < 0)
+				return -1;
+
+			/* it iteration changed delta type, the update the record */
+			if (delta_type == GIT_DELTA_IGNORED) {
+				last->status = GIT_DELTA_IGNORED;
+
+				/* remove the record if we don't want ignored records */
+				if (DIFF_FLAG_ISNT_SET(diff, GIT_DIFF_INCLUDE_IGNORED)) {
+					git_vector_pop(&diff->deltas);
+					git__free(last);
 				}
 			}
 
-			/* In core git, the next two "else if" clauses are effectively
-			 * reversed -- i.e. when an untracked file contained in an
-			 * ignored directory is individually ignored, it shows up as an
-			 * ignored file in the diff list, even though other untracked
-			 * files in the same directory are skipped completely.
-			 *
-			 * To me, this is odd.  If the directory is ignored and the file
-			 * is untracked, we should skip it consistently, regardless of
-			 * whether it happens to match a pattern in the ignore file.
-			 *
-			 * To match the core git behavior, just reverse the following
-			 * two "else if" cases so that individual file ignores are
-			 * checked before container directory exclusions are used to
-			 * skip the file.
+			return 0;
+		}
+
+		/* try to advance into directory if necessary */
+		if (recurse_into_dir) {
+			error = git_iterator_advance_into(&info->nitem, info->new_iter);
+
+			/* if real error or no error, proceed with iteration */
+			if (error != GIT_ENOTFOUND)
+				return error;
+			giterr_clear();
+
+			/* if directory is empty, can't advance into it, so either skip
+			 * it or ignore it
 			 */
-			else if (delta_type == GIT_DELTA_IGNORED &&
-					 DIFF_FLAG_ISNT_SET(diff, GIT_DIFF_RECURSE_IGNORED_DIRS)) {
-				if (git_iterator_advance(&nitem, new_iter) < 0)
-					goto fail;
-				continue; /* ignored parent directory, so skip completely */
-			}
+			if (contains_oitem)
+				return git_iterator_advance(&info->nitem, info->new_iter);
+			delta_type = GIT_DELTA_IGNORED;
+		}
+	}
 
-			else if (git_iterator_current_is_ignored(new_iter))
-				delta_type = GIT_DELTA_IGNORED;
+	/* In core git, the next two checks are effectively reversed --
+	 * i.e. when an file contained in an ignored directory is explicitly
+	 * ignored, it shows up as an ignored file in the diff list, even though
+	 * other untracked files in the same directory are skipped completely.
+	 *
+	 * To me, this seems odd.  If the directory is ignored and the file is
+	 * untracked, we should skip it consistently, regardless of whether it
+	 * happens to match a pattern in the ignore file.
+	 *
+	 * To match the core git behavior, reverse the following two if checks
+	 * so that individual file ignores are checked before container
+	 * directory exclusions are used to skip the file.
+	 */
+	else if (delta_type == GIT_DELTA_IGNORED &&
+		DIFF_FLAG_ISNT_SET(diff, GIT_DIFF_RECURSE_IGNORED_DIRS))
+		/* item contained in ignored directory, so skip over it */
+		return git_iterator_advance(&info->nitem, info->new_iter);
 
-			else if (new_iter->type != GIT_ITERATOR_TYPE_WORKDIR)
-				delta_type = GIT_DELTA_ADDED;
+	else if (git_iterator_current_is_ignored(info->new_iter))
+		delta_type = GIT_DELTA_IGNORED;
 
-			if (diff_delta__from_one(diff, delta_type, nitem) < 0)
-				goto fail;
+	else if (info->new_iter->type != GIT_ITERATOR_TYPE_WORKDIR)
+		delta_type = GIT_DELTA_ADDED;
 
-			/* if we are generating TYPECHANGE records then check for that
-			 * instead of just generating an ADDED/UNTRACKED record
-			 */
-			if (delta_type != GIT_DELTA_IGNORED &&
-				DIFF_FLAG_IS_SET(diff, GIT_DIFF_INCLUDE_TYPECHANGE_TREES) &&
-				contains_oitem)
-			{
-				/* this entry was prefixed with a tree - make TYPECHANGE */
-				git_diff_delta *last = diff_delta__last_for_item(diff, nitem);
-				if (last) {
-					last->status = GIT_DELTA_TYPECHANGE;
-					last->old_file.mode = GIT_FILEMODE_TREE;
-				}
-			}
+	/* Actually create the record for this item if necessary */
+	if ((error = diff_delta__from_one(diff, delta_type, nitem)) < 0)
+		return error;
 
-			if (git_iterator_advance(&nitem, new_iter) < 0)
-				goto fail;
+	/* If user requested TYPECHANGE records, then check for that instead of
+	 * just generating an ADDED/UNTRACKED record
+	 */
+	if (delta_type != GIT_DELTA_IGNORED &&
+		DIFF_FLAG_IS_SET(diff, GIT_DIFF_INCLUDE_TYPECHANGE_TREES) &&
+		contains_oitem)
+	{
+		/* this entry was prefixed with a tree - make TYPECHANGE */
+		git_diff_delta *last = diff_delta__last_for_item(diff, nitem);
+		if (last) {
+			last->status = GIT_DELTA_TYPECHANGE;
+			last->old_file.mode = GIT_FILEMODE_TREE;
 		}
+	}
 
-		/* otherwise item paths match, so create MODIFIED record
-		 * (or ADDED and DELETED pair if type changed)
-		 */
-		else {
-			assert(oitem && nitem && cmp == 0);
+	return git_iterator_advance(&info->nitem, info->new_iter);
+}
 
-			if (maybe_modified(old_iter, oitem, new_iter, nitem, diff) < 0 ||
-				git_iterator_advance(&oitem, old_iter) < 0 ||
-				git_iterator_advance(&nitem, new_iter) < 0)
-				goto fail;
+static int handle_unmatched_old_item(
+	git_diff_list *diff, diff_in_progress *info)
+{
+	int error = diff_delta__from_one(diff, GIT_DELTA_DELETED, info->oitem);
+	if (error < 0)
+		return error;
+
+	/* if we are generating TYPECHANGE records then check for that
+	 * instead of just generating a DELETE record
+	 */
+	if (DIFF_FLAG_IS_SET(diff, GIT_DIFF_INCLUDE_TYPECHANGE_TREES) &&
+		entry_is_prefixed(diff, info->nitem, info->oitem))
+	{
+		/* this entry has become a tree! convert to TYPECHANGE */
+		git_diff_delta *last = diff_delta__last_for_item(diff, info->oitem);
+		if (last) {
+			last->status = GIT_DELTA_TYPECHANGE;
+			last->new_file.mode = GIT_FILEMODE_TREE;
 		}
+
+		/* If new_iter is a workdir iterator, then this situation
+		 * will certainly be followed by a series of untracked items.
+		 * Unless RECURSE_UNTRACKED_DIRS is set, skip over them...
+		 */
+		if (S_ISDIR(info->nitem->mode) &&
+			DIFF_FLAG_ISNT_SET(diff, GIT_DIFF_RECURSE_UNTRACKED_DIRS))
+			return git_iterator_advance(&info->nitem, info->new_iter);
 	}
 
-	*diff_ptr = diff;
+	return git_iterator_advance(&info->oitem, info->old_iter);
+}
+
+static int handle_matched_item(
+	git_diff_list *diff, diff_in_progress *info)
+{
+	int error = 0;
 
-fail:
-	if (!*diff_ptr) {
-		git_diff_list_free(diff);
-		error = -1;
+	if (!(error = maybe_modified(diff, info)) &&
+		!(error = git_iterator_advance(&info->oitem, info->old_iter)))
+		error = git_iterator_advance(&info->nitem, info->new_iter);
+
+	return error;
+}
+
+int git_diff__from_iterators(
+	git_diff_list **diff_ptr,
+	git_repository *repo,
+	git_iterator *old_iter,
+	git_iterator *new_iter,
+	const git_diff_options *opts)
+{
+	int error = 0;
+	diff_in_progress info;
+	git_diff_list *diff;
+
+	*diff_ptr = NULL;
+
+	diff = diff_list_alloc(repo, old_iter, new_iter);
+	GITERR_CHECK_ALLOC(diff);
+
+	info.repo = repo;
+	info.old_iter = old_iter;
+	info.new_iter = new_iter;
+	git_buf_init(&info.ignore_prefix, 0);
+
+	/* make iterators have matching icase behavior */
+	if (DIFF_FLAG_IS_SET(diff, GIT_DIFF_DELTAS_ARE_ICASE)) {
+		if (!(error = git_iterator_set_ignore_case(old_iter, true)))
+			error = git_iterator_set_ignore_case(new_iter, true);
+	}
+
+	/* finish initialization */
+	if (!error &&
+		!(error = diff_list_apply_options(diff, opts)) &&
+		!(error = git_iterator_current(&info.oitem, old_iter)))
+		error = git_iterator_current(&info.nitem, new_iter);
+
+	/* run iterators building diffs */
+	while (!error && (info.oitem || info.nitem)) {
+		int cmp = info.oitem ?
+			(info.nitem ? diff->entrycomp(info.oitem, info.nitem) : -1) : 1;
+
+		/* create DELETED records for old items not matched in new */
+		if (cmp < 0)
+			error = handle_unmatched_old_item(diff, &info);
+
+		/* create ADDED, TRACKED, or IGNORED records for new items not
+		 * matched in old (and/or descend into directories as needed)
+		 */
+		else if (cmp > 0)
+			error = handle_unmatched_new_item(diff, &info);
+
+		/* otherwise item paths match, so create MODIFIED record
+		 * (or ADDED and DELETED pair if type changed)
+		 */
+		else
+			error = handle_matched_item(diff, &info);
 	}
 
-	git_buf_free(&ignore_prefix);
+	if (!error)
+		*diff_ptr = diff;
+	else
+		git_diff_list_free(diff);
+
+	git_buf_free(&info.ignore_prefix);
 
 	return error;
 }

src/diff_output.c

@@ -101,8 +101,8 @@ static bool diff_delta_is_binary_forced(
 
 	/* make sure files are conceivably mmap-able */
 	if ((git_off_t)((size_t)delta->old_file.size) != delta->old_file.size ||
-		(git_off_t)((size_t)delta->new_file.size) != delta->new_file.size)
-	{
+		(git_off_t)((size_t)delta->new_file.size) != delta->new_file.size) {
+
 		delta->old_file.flags |= GIT_DIFF_FLAG_BINARY;
 		delta->new_file.flags |= GIT_DIFF_FLAG_BINARY;
 		delta->flags |= GIT_DIFF_FLAG_BINARY;
@@ -232,8 +232,7 @@ static int get_blob_content(
 	if (git_oid_iszero(&file->oid))
 		return 0;
 
-	if (file->mode == GIT_FILEMODE_COMMIT)
-	{
+	if (file->mode == GIT_FILEMODE_COMMIT) {
 		char oidstr[GIT_OID_HEXSZ+1];
 		git_buf content = GIT_BUF_INIT;
 
@@ -299,8 +298,8 @@ static int get_workdir_sm_content(
 	char oidstr[GIT_OID_HEXSZ+1];
 
 	if ((error = git_submodule_lookup(&sm, ctxt->repo, file->path)) < 0 ||
-		(error = git_submodule_status(&sm_status, sm)) < 0)
-	{
+		(error = git_submodule_status(&sm_status, sm)) < 0) {
+
 		/* GIT_EEXISTS means a "submodule" that has not been git added */
 		if (error == GIT_EEXISTS)
 			error = 0;
@@ -312,8 +311,8 @@ static int get_workdir_sm_content(
 		const git_oid* sm_head;
 
 		if ((sm_head = git_submodule_wd_id(sm)) != NULL ||
-			(sm_head = git_submodule_head_id(sm)) != NULL)
-		{
+			(sm_head = git_submodule_head_id(sm)) != NULL) {
+
 			git_oid_cpy(&file->oid, sm_head);
 			file->flags |= GIT_DIFF_FLAG_VALID_OID;
 		}
@@ -660,8 +659,8 @@ static int diff_patch_load(
 	 */
 	if (check_if_unmodified &&
 		delta->old_file.mode == delta->new_file.mode &&
-		!git_oid__cmp(&delta->old_file.oid, &delta->new_file.oid))
-	{
+		!git_oid__cmp(&delta->old_file.oid, &delta->new_file.oid)) {
+
 		delta->status = GIT_DELTA_UNMODIFIED;
 
 		if ((ctxt->opts->flags & GIT_DIFF_INCLUDE_UNMODIFIED) == 0)
@@ -1049,6 +1048,12 @@ char git_diff_status_char(git_delta_t status)
 	return code;
 }
 
+static int callback_error(void)
+{
+	giterr_clear();
+	return GIT_EUSER;
+}
+
 static int print_compact(
 	const git_diff_delta *delta, float progress, void *data)
 {
@@ -1083,10 +1088,7 @@ static int print_compact(
 
 	if (pi->print_cb(delta, NULL, GIT_DIFF_LINE_FILE_HDR,
 			git_buf_cstr(pi->buf), git_buf_len(pi->buf), pi->payload))
-	{
-		giterr_clear();
-		return GIT_EUSER;
-	}
+		return callback_error();
 
 	return 0;
 }
@@ -1200,10 +1202,7 @@ static int print_patch_file(
 
 	if (pi->print_cb(delta, NULL, GIT_DIFF_LINE_FILE_HDR,
 			git_buf_cstr(pi->buf), git_buf_len(pi->buf), pi->payload))
-	{
-		giterr_clear();
-		return GIT_EUSER;
-	}
+		return callback_error();
 
 	if ((delta->flags & GIT_DIFF_FLAG_BINARY) == 0)
 		return 0;
@@ -1217,10 +1216,7 @@ static int print_patch_file(
 
 	if (pi->print_cb(delta, NULL, GIT_DIFF_LINE_BINARY,
 			git_buf_cstr(pi->buf), git_buf_len(pi->buf), pi->payload))
-	{
-		giterr_clear();
-		return GIT_EUSER;
-	}
+		return callback_error();
 
 	return 0;
 }
@@ -1243,10 +1239,7 @@ static int print_patch_hunk(
 
 	if (pi->print_cb(d, r, GIT_DIFF_LINE_HUNK_HDR,
 			git_buf_cstr(pi->buf), git_buf_len(pi->buf), pi->payload))
-	{
-		giterr_clear();
-		return GIT_EUSER;
-	}
+		return callback_error();
 
 	return 0;
 }
@@ -1278,10 +1271,7 @@ static int print_patch_line(
 
 	if (pi->print_cb(delta, range, line_origin,
 			git_buf_cstr(pi->buf), git_buf_len(pi->buf), pi->payload))
-	{
-		giterr_clear();
-		return GIT_EUSER;
-	}
+		return callback_error();
 
 	return 0;
 }

src/vector.c

@@ -277,15 +277,13 @@ void git_vector_swap(git_vector *a, git_vector *b)
 
 int git_vector_resize_to(git_vector *v, size_t new_length)
 {
-	if (new_length <= v->length)
-		return 0;
-
 	if (new_length > v->_alloc_size &&
 		resize_vector(v, new_length) < 0)
 		return -1;
 
-	memset(&v->contents[v->length], 0,
-		sizeof(void *) * (new_length - v->length));
+	if (new_length > v->length)
+		memset(&v->contents[v->length], 0,
+			sizeof(void *) * (new_length - v->length));
 
 	v->length = new_length;
 

tests-clar/diff/diff_helpers.c

@@ -28,7 +28,15 @@ int diff_file_cb(
 {
 	diff_expects *e = payload;
 
-	GIT_UNUSED(progress);
+	if (e->debug)
+		fprintf(stderr, "%c %s (%.3f)\n",
+			git_diff_status_char(delta->status),
+			delta->old_file.path, progress);
+
+	if (e->names)
+		cl_assert_equal_s(e->names[e->files], delta->old_file.path);
+	if (e->statuses)
+		cl_assert_equal_i(e->statuses[e->files], (int)delta->status);
 
 	e->files++;
 

tests-clar/diff/diff_helpers.h

@@ -18,6 +18,13 @@ typedef struct {
 	int line_ctxt;
 	int line_adds;
 	int line_dels;
+
+	/* optional arrays of expected specific values */
+	const char **names;
+	int *statuses;
+
+	int debug;
+
 } diff_expects;
 
 typedef struct {

tests-clar/diff/workdir.c

@@ -1033,3 +1033,190 @@ void test_diff_workdir__to_tree_issue_1397(void)
 	git_diff_list_free(diff);
 	git_tree_free(a);
 }
+
+void test_diff_workdir__untracked_directory_scenarios(void)
+{
+	git_diff_options opts = GIT_DIFF_OPTIONS_INIT;
+	git_diff_list *diff = NULL;
+	diff_expects exp;
+	char *pathspec = NULL;
+	static const char *files0[] = {
+		"subdir/deleted_file",
+		"subdir/modified_file",
+		"subdir/new_file",
+		NULL
+	};
+	static const char *files1[] = {
+		"subdir/deleted_file",
+		"subdir/directory/",
+		"subdir/modified_file",
+		"subdir/new_file",
+		NULL
+	};
+	static const char *files2[] = {
+		"subdir/deleted_file",
+		"subdir/directory/more/notignored",
+		"subdir/modified_file",
+		"subdir/new_file",
+		NULL
+	};
+
+	g_repo = cl_git_sandbox_init("status");
+	cl_git_mkfile("status/.gitignore", "ignored\n");
+
+	opts.context_lines = 3;
+	opts.interhunk_lines = 1;
+	opts.flags |= GIT_DIFF_INCLUDE_IGNORED | GIT_DIFF_INCLUDE_UNTRACKED;
+	opts.pathspec.strings = &pathspec;
+	opts.pathspec.count   = 1;
+	pathspec = "subdir";
+
+	/* baseline for "subdir" pathspec */
+
+	memset(&exp, 0, sizeof(exp));
+	exp.names = files0;
+
+	cl_git_pass(git_diff_index_to_workdir(&diff, g_repo, NULL, &opts));
+
+	cl_git_pass(git_diff_foreach(diff, diff_file_cb, NULL, NULL, &exp));
+
+	cl_assert_equal_i(3, exp.files);
+	cl_assert_equal_i(0, exp.file_status[GIT_DELTA_ADDED]);
+	cl_assert_equal_i(1, exp.file_status[GIT_DELTA_DELETED]);
+	cl_assert_equal_i(1, exp.file_status[GIT_DELTA_MODIFIED]);
+	cl_assert_equal_i(0, exp.file_status[GIT_DELTA_IGNORED]);
+	cl_assert_equal_i(1, exp.file_status[GIT_DELTA_UNTRACKED]);
+
+	git_diff_list_free(diff);
+
+	/* empty directory */
+
+	cl_git_pass(p_mkdir("status/subdir/directory", 0777));
+
+	memset(&exp, 0, sizeof(exp));
+	exp.names = files1;
+
+	cl_git_pass(git_diff_index_to_workdir(&diff, g_repo, NULL, &opts));
+
+	cl_git_pass(git_diff_foreach(diff, diff_file_cb, NULL, NULL, &exp));
+
+	cl_assert_equal_i(4, exp.files);
+	cl_assert_equal_i(0, exp.file_status[GIT_DELTA_ADDED]);
+	cl_assert_equal_i(1, exp.file_status[GIT_DELTA_DELETED]);
+	cl_assert_equal_i(1, exp.file_status[GIT_DELTA_MODIFIED]);
+	cl_assert_equal_i(1, exp.file_status[GIT_DELTA_IGNORED]);
+	cl_assert_equal_i(1, exp.file_status[GIT_DELTA_UNTRACKED]);
+
+	git_diff_list_free(diff);
+
+	/* directory with only ignored files */
+
+	cl_git_pass(p_mkdir("status/subdir/directory/deeper", 0777));
+	cl_git_mkfile("status/subdir/directory/deeper/ignored", "ignore me\n");
+
+	cl_git_pass(p_mkdir("status/subdir/directory/another", 0777));
+	cl_git_mkfile("status/subdir/directory/another/ignored", "ignore me\n");
+
+	memset(&exp, 0, sizeof(exp));
+	exp.names = files1;
+
+	cl_git_pass(git_diff_index_to_workdir(&diff, g_repo, NULL, &opts));
+
+	cl_git_pass(git_diff_foreach(diff, diff_file_cb, NULL, NULL, &exp));
+
+	cl_assert_equal_i(4, exp.files);
+	cl_assert_equal_i(0, exp.file_status[GIT_DELTA_ADDED]);
+	cl_assert_equal_i(1, exp.file_status[GIT_DELTA_DELETED]);
+	cl_assert_equal_i(1, exp.file_status[GIT_DELTA_MODIFIED]);
+	cl_assert_equal_i(1, exp.file_status[GIT_DELTA_IGNORED]);
+	cl_assert_equal_i(1, exp.file_status[GIT_DELTA_UNTRACKED]);
+
+	git_diff_list_free(diff);
+
+	/* directory with ignored directory (contents irrelevant) */
+
+	cl_git_pass(p_mkdir("status/subdir/directory/more", 0777));
+	cl_git_pass(p_mkdir("status/subdir/directory/more/ignored", 0777));
+	cl_git_mkfile("status/subdir/directory/more/ignored/notignored",
+		"inside ignored dir\n");
+
+	memset(&exp, 0, sizeof(exp));
+	exp.names = files1;
+
+	cl_git_pass(git_diff_index_to_workdir(&diff, g_repo, NULL, &opts));
+
+	cl_git_pass(git_diff_foreach(diff, diff_file_cb, NULL, NULL, &exp));
+
+	cl_assert_equal_i(4, exp.files);
+	cl_assert_equal_i(0, exp.file_status[GIT_DELTA_ADDED]);
+	cl_assert_equal_i(1, exp.file_status[GIT_DELTA_DELETED]);
+	cl_assert_equal_i(1, exp.file_status[GIT_DELTA_MODIFIED]);
+	cl_assert_equal_i(1, exp.file_status[GIT_DELTA_IGNORED]);
+	cl_assert_equal_i(1, exp.file_status[GIT_DELTA_UNTRACKED]);
+
+	git_diff_list_free(diff);
+
+	/* quick version avoids directory scan */
+
+	opts.flags = opts.flags | GIT_DIFF_FAST_UNTRACKED_DIRS;
+
+	memset(&exp, 0, sizeof(exp));
+	exp.names = files1;
+
+	cl_git_pass(git_diff_index_to_workdir(&diff, g_repo, NULL, &opts));
+
+	cl_git_pass(git_diff_foreach(diff, diff_file_cb, NULL, NULL, &exp));
+
+	cl_assert_equal_i(4, exp.files);
+	cl_assert_equal_i(0, exp.file_status[GIT_DELTA_ADDED]);
+	cl_assert_equal_i(1, exp.file_status[GIT_DELTA_DELETED]);
+	cl_assert_equal_i(1, exp.file_status[GIT_DELTA_MODIFIED]);
+	cl_assert_equal_i(0, exp.file_status[GIT_DELTA_IGNORED]);
+	cl_assert_equal_i(2, exp.file_status[GIT_DELTA_UNTRACKED]);
+
+	git_diff_list_free(diff);
+
+	/* directory with nested non-ignored content */
+
+	opts.flags = opts.flags & ~GIT_DIFF_FAST_UNTRACKED_DIRS;
+
+	cl_git_mkfile("status/subdir/directory/more/notignored",
+		"not ignored deep under untracked\n");
+
+	memset(&exp, 0, sizeof(exp));
+	exp.names = files1;
+
+	cl_git_pass(git_diff_index_to_workdir(&diff, g_repo, NULL, &opts));
+
+	cl_git_pass(git_diff_foreach(diff, diff_file_cb, NULL, NULL, &exp));
+
+	cl_assert_equal_i(4, exp.files);
+	cl_assert_equal_i(0, exp.file_status[GIT_DELTA_ADDED]);
+	cl_assert_equal_i(1, exp.file_status[GIT_DELTA_DELETED]);
+	cl_assert_equal_i(1, exp.file_status[GIT_DELTA_MODIFIED]);
+	cl_assert_equal_i(0, exp.file_status[GIT_DELTA_IGNORED]);
+	cl_assert_equal_i(2, exp.file_status[GIT_DELTA_UNTRACKED]);
+
+	git_diff_list_free(diff);
+
+	/* use RECURSE_UNTRACKED_DIRS to get actual untracked files (no ignores) */
+
+	opts.flags = opts.flags & ~GIT_DIFF_INCLUDE_IGNORED;
+	opts.flags = opts.flags | GIT_DIFF_RECURSE_UNTRACKED_DIRS;
+
+	memset(&exp, 0, sizeof(exp));
+	exp.names = files2;
+
+	cl_git_pass(git_diff_index_to_workdir(&diff, g_repo, NULL, &opts));
+
+	cl_git_pass(git_diff_foreach(diff, diff_file_cb, NULL, NULL, &exp));
+
+	cl_assert_equal_i(4, exp.files);
+	cl_assert_equal_i(0, exp.file_status[GIT_DELTA_ADDED]);
+	cl_assert_equal_i(1, exp.file_status[GIT_DELTA_DELETED]);
+	cl_assert_equal_i(1, exp.file_status[GIT_DELTA_MODIFIED]);
+	cl_assert_equal_i(0, exp.file_status[GIT_DELTA_IGNORED]);
+	cl_assert_equal_i(2, exp.file_status[GIT_DELTA_UNTRACKED]);
+
+	git_diff_list_free(diff);
+}

tests-clar/status/status_helpers.c

@@ -40,7 +40,8 @@ int cb_status__single(const char *p, unsigned int s, void *payload)
 {
 	status_entry_single *data = (status_entry_single *)payload;
 
-	GIT_UNUSED(p);
+	if (data->debug)
+		fprintf(stderr, "%02d: %s (%04x)\n", data->count, p, s);
 
 	data->count++;
 	data->status = s;

tests-clar/status/status_helpers.h

@@ -24,6 +24,7 @@ extern int cb_status__count(const char *p, unsigned int s, void *payload);
 typedef struct {
 	int count;
 	unsigned int status;
+	bool debug;
 } status_entry_single;
 
 /* cb_status__single takes payload of "status_entry_single *" */

tests-clar/status/worktree.c

@@ -258,9 +258,8 @@ void test_status_worktree__ignores(void)
 
 static int cb_status__check_592(const char *p, unsigned int s, void *payload)
 {
-	GIT_UNUSED(payload);
-
-	if (s != GIT_STATUS_WT_DELETED || (payload != NULL && strcmp(p, (const char *)payload) != 0))
+	if (s != GIT_STATUS_WT_DELETED ||
+		(payload != NULL && strcmp(p, (const char *)payload) != 0))
 		return -1;
 
 	return 0;

tests-clar/submodule/status.c

@@ -383,3 +383,30 @@ void test_submodule_status__iterator(void)
 
 	cl_git_pass(git_status_foreach_ext(g_repo, &opts, confirm_submodule_status, &exp));
 }
+
+void test_submodule_status__untracked_dirs_containing_ignored_files(void)
+{
+	git_buf path = GIT_BUF_INIT;
+	unsigned int status, expected;
+	git_submodule *sm;
+
+	cl_git_pass(git_buf_joinpath(&path, git_repository_path(g_repo), "modules/sm_unchanged/info/exclude"));
+	cl_git_append2file(git_buf_cstr(&path), "\n*.ignored\n");
+
+	cl_git_pass(git_buf_joinpath(&path, git_repository_workdir(g_repo), "sm_unchanged/directory"));
+	cl_git_pass(git_futils_mkdir(git_buf_cstr(&path), NULL, 0755, 0));
+	cl_git_pass(git_buf_joinpath(&path, git_buf_cstr(&path), "i_am.ignored"));
+	cl_git_mkfile(git_buf_cstr(&path), "ignored this file, please\n");
+
+	cl_git_pass(git_submodule_lookup(&sm, g_repo, "sm_unchanged"));
+	cl_git_pass(git_submodule_status(&status, sm));
+
+	cl_assert(GIT_SUBMODULE_STATUS_IS_UNMODIFIED(status));
+
+	expected = GIT_SUBMODULE_STATUS_IN_HEAD |
+		GIT_SUBMODULE_STATUS_IN_INDEX |
+		GIT_SUBMODULE_STATUS_IN_CONFIG |
+		GIT_SUBMODULE_STATUS_IN_WD;
+
+	cl_assert(status == expected);
+}