Merge pull request #3691 from ethomson/iterators

Some FANTASTIC iterator refactoring

include/git2/blob.h

@@ -259,6 +259,15 @@ GIT_EXTERN(int) git_blob_create_frombuffer(
  */
 GIT_EXTERN(int) git_blob_is_binary(const git_blob *blob);
 
+/**
+ * Create an in-memory copy of a blob. The copy must be explicitly
+ * free'd or it will leak.
+ *
+ * @param out Pointer to store the copy of the object
+ * @param source Original object to copy
+ */
+GIT_EXTERN(int) git_blob_dup(git_blob **out, git_blob *source);
+
 /** @} */
 GIT_END_DECL
 #endif

include/git2/commit.h

@@ -461,6 +461,15 @@ GIT_EXTERN(int) git_commit_create_with_signature(
 	const char *signature,
 	const char *signature_field);
 
+/**
+ * Create an in-memory copy of a commit. The copy must be explicitly
+ * free'd or it will leak.
+ *
+ * @param out Pointer to store the copy of the commit
+ * @param source Original commit to copy
+ */
+GIT_EXTERN(int) git_commit_dup(git_commit **out, git_commit *source);
+
 /** @} */
 GIT_END_DECL
 #endif

include/git2/tag.h

@@ -347,6 +347,15 @@ GIT_EXTERN(int) git_tag_peel(
 	git_object **tag_target_out,
 	const git_tag *tag);
 
+/**
+ * Create an in-memory copy of a tag. The copy must be explicitly
+ * free'd or it will leak.
+ *
+ * @param out Pointer to store the copy of the tag
+ * @param source Original tag to copy
+ */
+GIT_EXTERN(int) git_tag_dup(git_tag **out, git_tag *source);
+
 /** @} */
 GIT_END_DECL
 #endif

include/git2/tree.h

@@ -409,6 +409,15 @@ GIT_EXTERN(int) git_tree_walk(
 	git_treewalk_cb callback,
 	void *payload);
 
+/**
+ * Create an in-memory copy of a tree. The copy must be explicitly
+ * free'd or it will leak.
+ *
+ * @param out Pointer to store the copy of the tree
+ * @param source Original tree to copy
+ */
+GIT_EXTERN(int) git_tree_dup(git_tree **out, git_tree *source);
+
 /** @} */
 
 GIT_END_DECL

src/checkout.c

@@ -66,8 +66,8 @@ typedef struct {
 	git_vector update_conflicts;
 	git_vector *update_reuc;
 	git_vector *update_names;
-	git_buf path;
-	size_t workdir_len;
+	git_buf target_path;
+	size_t target_len;
 	git_buf tmp;
 	unsigned int strategy;
 	int can_symlink;
@@ -294,14 +294,30 @@ static int checkout_action_no_wd(
 	return checkout_action_common(action, data, delta, NULL);
 }
 
-static bool wd_item_is_removable(git_iterator *iter, const git_index_entry *wd)
+static int checkout_target_fullpath(
+	git_buf **out, checkout_data *data, const char *path)
 {
-	git_buf *full = NULL;
+	git_buf_truncate(&data->target_path, data->target_len);
+
+	if (path && git_buf_puts(&data->target_path, path) < 0)
+		return -1;
+
+	*out = &data->target_path;
+
+	return 0;
+}
+
+static bool wd_item_is_removable(
+	checkout_data *data, const git_index_entry *wd)
+{
+	git_buf *full;
 
 	if (wd->mode != GIT_FILEMODE_TREE)
 		return true;
-	if (git_iterator_current_workdir_path(&full, iter) < 0)
-		return true;
+
+	if (checkout_target_fullpath(&full, data, wd->path) < 0)
+		return false;
+
 	return !full || !git_path_contains(full, DOT_GIT);
 }
 
@@ -363,14 +379,14 @@ static int checkout_action_wd_only(
 		if ((error = checkout_notify(data, notify, NULL, wd)) != 0)
 			return error;
 
-		if (remove && wd_item_is_removable(workdir, wd))
+		if (remove && wd_item_is_removable(data, wd))
 			error = checkout_queue_remove(data, wd->path);
 
 		if (!error)
 			error = git_iterator_advance(wditem, workdir);
 	} else {
 		/* untracked or ignored - can't know which until we advance through */
-		bool over = false, removable = wd_item_is_removable(workdir, wd);
+		bool over = false, removable = wd_item_is_removable(data, wd);
 		git_iterator_status_t untracked_state;
 
 		/* copy the entry for issuing notification callback later */
@@ -378,7 +394,7 @@ static int checkout_action_wd_only(
 		git_buf_sets(&data->tmp, wd->path);
 		saved_wd.path = data->tmp.ptr;
 
-		error = git_iterator_advance_over_with_status(
+		error = git_iterator_advance_over(
 			wditem, &untracked_state, workdir);
 		if (error == GIT_ITEROVER)
 			over = true;
@@ -428,10 +444,12 @@ static bool submodule_is_config_only(
 
 static bool checkout_is_empty_dir(checkout_data *data, const char *path)
 {
-	git_buf_truncate(&data->path, data->workdir_len);
-	if (git_buf_puts(&data->path, path) < 0)
+	git_buf *fullpath;
+
+	if (checkout_target_fullpath(&fullpath, data, path) < 0)
 		return false;
-	return git_path_is_empty_dir(data->path.ptr);
+
+	return git_path_is_empty_dir(fullpath->ptr);
 }
 
 static int checkout_action_with_wd(
@@ -639,7 +657,7 @@ static int checkout_action(
 			if (cmp == 0) {
 				if (wd->mode == GIT_FILEMODE_TREE) {
 					/* case 2 - entry prefixed by workdir tree */
-					error = git_iterator_advance_into_or_over(wditem, workdir);
+					error = git_iterator_advance_into(wditem, workdir);
 					if (error < 0 && error != GIT_ITEROVER)
 						goto done;
 					continue;
@@ -912,7 +930,7 @@ static int checkout_conflicts_load(checkout_data *data, git_iterator *workdir, g
 	git_index *index;
 
 	/* Only write conficts from sources that have them: indexes. */
-	if ((index = git_iterator_get_index(data->target)) == NULL)
+	if ((index = git_iterator_index(data->target)) == NULL)
 		return 0;
 
 	data->update_conflicts._cmp = checkout_conflictdata_cmp;
@@ -1062,7 +1080,7 @@ static int checkout_conflicts_coalesce_renames(
 	size_t i, names;
 	int error = 0;
 
-	if ((index = git_iterator_get_index(data->target)) == NULL)
+	if ((index = git_iterator_index(data->target)) == NULL)
 		return 0;
 
 	/* Juggle entries based on renames */
@@ -1120,7 +1138,7 @@ static int checkout_conflicts_mark_directoryfile(
 	const char *path;
 	int prefixed, error = 0;
 
-	if ((index = git_iterator_get_index(data->target)) == NULL)
+	if ((index = git_iterator_index(data->target)) == NULL)
 		return 0;
 
 	len = git_index_entrycount(index);
@@ -1582,18 +1600,18 @@ static int checkout_submodule_update_index(
 	checkout_data *data,
 	const git_diff_file *file)
 {
+	git_buf *fullpath;
 	struct stat st;
 
 	/* update the index unless prevented */
 	if ((data->strategy & GIT_CHECKOUT_DONT_UPDATE_INDEX) != 0)
 		return 0;
 
-	git_buf_truncate(&data->path, data->workdir_len);
-	if (git_buf_puts(&data->path, file->path) < 0)
+	if (checkout_target_fullpath(&fullpath, data, file->path) < 0)
 		return -1;
 
 	data->perfdata.stat_calls++;
-	if (p_stat(git_buf_cstr(&data->path), &st) < 0) {
+	if (p_stat(fullpath->ptr, &st) < 0) {
 		giterr_set(
 			GITERR_CHECKOUT, "Could not stat submodule %s\n", file->path);
 		return GIT_ENOTFOUND;
@@ -1718,22 +1736,23 @@ static int checkout_blob(
 	checkout_data *data,
 	const git_diff_file *file)
 {
-	int error = 0;
+	git_buf *fullpath;
 	struct stat st;
+	int error = 0;
 
-	git_buf_truncate(&data->path, data->workdir_len);
-	if (git_buf_puts(&data->path, file->path) < 0)
+	if (checkout_target_fullpath(&fullpath, data, file->path) < 0)
 		return -1;
 
 	if ((data->strategy & GIT_CHECKOUT_UPDATE_ONLY) != 0) {
 		int rval = checkout_safe_for_update_only(
-			data, git_buf_cstr(&data->path), file->mode);
+			data, fullpath->ptr, file->mode);
+
 		if (rval <= 0)
 			return rval;
 	}
 
 	error = checkout_write_content(
-		data, &file->id, git_buf_cstr(&data->path), NULL, file->mode, &st);
+		data, &file->id, fullpath->ptr, NULL, file->mode, &st);
 
 	/* update the index unless prevented */
 	if (!error && (data->strategy & GIT_CHECKOUT_DONT_UPDATE_INDEX) == 0)
@@ -1754,18 +1773,21 @@ static int checkout_remove_the_old(
 	git_diff_delta *delta;
 	const char *str;
 	size_t i;
-	const char *workdir = git_buf_cstr(&data->path);
+	git_buf *fullpath;
 	uint32_t flg = GIT_RMDIR_EMPTY_PARENTS |
 		GIT_RMDIR_REMOVE_FILES | GIT_RMDIR_REMOVE_BLOCKERS;
 
 	if (data->opts.checkout_strategy & GIT_CHECKOUT_SKIP_LOCKED_DIRECTORIES)
 		flg |= GIT_RMDIR_SKIP_NONEMPTY;
 
-	git_buf_truncate(&data->path, data->workdir_len);
+	if (checkout_target_fullpath(&fullpath, data, NULL) < 0)
+		return -1;
 
 	git_vector_foreach(&data->diff->deltas, i, delta) {
 		if (actions[i] & CHECKOUT_ACTION__REMOVE) {
-			error = git_futils_rmdir_r(delta->old_file.path, workdir, flg);
+			error = git_futils_rmdir_r(
+				delta->old_file.path, fullpath->ptr, flg);
+
 			if (error < 0)
 				return error;
 
@@ -1782,7 +1804,7 @@ static int checkout_remove_the_old(
 	}
 
 	git_vector_foreach(&data->removes, i, str) {
-		error = git_futils_rmdir_r(str, workdir, flg);
+		error = git_futils_rmdir_r(str, fullpath->ptr, flg);
 		if (error < 0)
 			return error;
 
@@ -1949,13 +1971,13 @@ static int checkout_write_entry(
 	const git_index_entry *side)
 {
 	const char *hint_path = NULL, *suffix;
+	git_buf *fullpath;
 	struct stat st;
 	int error;
 
 	assert (side == conflict->ours || side == conflict->theirs);
 
-	git_buf_truncate(&data->path, data->workdir_len);
-	if (git_buf_puts(&data->path, side->path) < 0)
+	if (checkout_target_fullpath(&fullpath, data, side->path) < 0)
 		return -1;
 
 	if ((conflict->name_collision || conflict->directoryfile) &&
@@ -1969,18 +1991,18 @@ static int checkout_write_entry(
 			suffix = data->opts.their_label ? data->opts.their_label :
 				"theirs";
 
-		if (checkout_path_suffixed(&data->path, suffix) < 0)
+		if (checkout_path_suffixed(fullpath, suffix) < 0)
 			return -1;
 
 		hint_path = side->path;
 	}
 
 	if ((data->strategy & GIT_CHECKOUT_UPDATE_ONLY) != 0 &&
-		(error = checkout_safe_for_update_only(data, git_buf_cstr(&data->path), side->mode)) <= 0)
+		(error = checkout_safe_for_update_only(data, fullpath->ptr, side->mode)) <= 0)
 		return error;
 
 	return checkout_write_content(data,
-		&side->id, git_buf_cstr(&data->path), hint_path, side->mode, &st);
+		&side->id, fullpath->ptr, hint_path, side->mode, &st);
 }
 
 static int checkout_write_entries(
@@ -2293,7 +2315,7 @@ static void checkout_data_clear(checkout_data *data)
 
 	git_strmap_free(data->mkdir_map);
 
-	git_buf_free(&data->path);
+	git_buf_free(&data->target_path);
 	git_buf_free(&data->tmp);
 
 	git_index_free(data->index);
@@ -2356,7 +2378,7 @@ static int checkout_data_init(
 		if ((error = git_repository_index(&data->index, data->repo)) < 0)
 			goto cleanup;
 
-		if (data->index != git_iterator_get_index(target)) {
+		if (data->index != git_iterator_index(target)) {
 			if ((error = git_index_read(data->index, true)) < 0)
 				goto cleanup;
 
@@ -2447,12 +2469,12 @@ static int checkout_data_init(
 	if ((error = git_vector_init(&data->removes, 0, git__strcmp_cb)) < 0 ||
 		(error = git_vector_init(&data->remove_conflicts, 0, NULL)) < 0 ||
 		(error = git_vector_init(&data->update_conflicts, 0, NULL)) < 0 ||
-		(error = git_buf_puts(&data->path, data->opts.target_directory)) < 0 ||
-		(error = git_path_to_dir(&data->path)) < 0 ||
+		(error = git_buf_puts(&data->target_path, data->opts.target_directory)) < 0 ||
+		(error = git_path_to_dir(&data->target_path)) < 0 ||
 		(error = git_strmap_alloc(&data->mkdir_map)) < 0)
 		goto cleanup;
 
-	data->workdir_len = git_buf_len(&data->path);
+	data->target_len = git_buf_len(&data->target_path);
 
 	git_attr_session__init(&data->attr_session, data->repo);
 
@@ -2508,7 +2530,7 @@ int git_checkout_iterator(
 	workdir_opts.start = data.pfx;
 	workdir_opts.end = data.pfx;
 
-	if ((error = git_iterator_reset(target, data.pfx, data.pfx)) < 0 ||
+	if ((error = git_iterator_reset_range(target, data.pfx, data.pfx)) < 0 ||
 		(error = git_iterator_for_workdir_ext(
 			&workdir, data.repo, data.opts.target_directory, index, NULL,
 			&workdir_opts)) < 0)
@@ -2578,7 +2600,7 @@ int git_checkout_iterator(
 		(error = checkout_create_conflicts(&data)) < 0)
 		goto cleanup;
 
-	if (data.index != git_iterator_get_index(target) &&
+	if (data.index != git_iterator_index(target) &&
 		(error = checkout_extensions_update_index(&data)) < 0)
 		goto cleanup;
 

src/commit.c

@@ -615,7 +615,7 @@ int git_commit_nth_gen_ancestor(
 
 	assert(ancestor && commit);
 
-	if (git_object_dup((git_object **) &current, (git_object *) commit) < 0)
+	if (git_commit_dup(&current, (git_commit *)commit) < 0)
 		return -1;
 
 	if (n == 0) {

src/describe.c

@@ -197,7 +197,7 @@ static int commit_name_dup(struct commit_name **out, struct commit_name *in)
 	name->tag = NULL;
 	name->path = NULL;
 
-	if (in->tag && git_object_dup((git_object **) &name->tag, (git_object *) in->tag) < 0)
+	if (in->tag && git_tag_dup(&name->tag, in->tag) < 0)
 		return -1;
 
 	name->path = git__strdup(in->path);

src/diff.c

@@ -826,8 +826,7 @@ static int maybe_modified(
 	 */
 	} else if (git_oid_iszero(&nitem->id) && new_is_workdir) {
 		bool use_ctime = ((diff->diffcaps & GIT_DIFFCAPS_TRUST_CTIME) != 0);
-		git_index *index;
-		git_iterator_index(&index, info->new_iter);
+		git_index *index = git_iterator_index(info->new_iter);
 
 		status = GIT_DELTA_UNMODIFIED;
 
@@ -980,15 +979,14 @@ static int iterator_advance_into(
 	return error;
 }
 
-static int iterator_advance_over_with_status(
+static int iterator_advance_over(
 	const git_index_entry **entry,
 	git_iterator_status_t *status,
 	git_iterator *iterator)
 {
-	int error;
+	int error = git_iterator_advance_over(entry, status, iterator);
 
-	if ((error = git_iterator_advance_over_with_status(
-			entry, status, iterator)) == GIT_ITEROVER) {
+	if (error == GIT_ITEROVER) {
 		*entry = NULL;
 		error = 0;
 	}
@@ -1056,7 +1054,7 @@ static int handle_unmatched_new_item(
 				return iterator_advance(&info->nitem, info->new_iter);
 
 			/* iterate into dir looking for an actual untracked file */
-			if ((error = iterator_advance_over_with_status(
+			if ((error = iterator_advance_over(
 					&info->nitem, &untracked_state, info->new_iter)) < 0)
 				return error;
 
@@ -1093,7 +1091,7 @@ static int handle_unmatched_new_item(
 			/* if directory is empty, can't advance into it, so either skip
 			 * it or ignore it
 			 */
-			if (contains_oitem)
+			if (error == GIT_ENOTFOUND || contains_oitem)
 				return iterator_advance(&info->nitem, info->new_iter);
 			delta_type = GIT_DELTA_IGNORED;
 		}
@@ -1231,9 +1229,8 @@ int git_diff__from_iterators(
 
 	/* make iterators have matching icase behavior */
 	if (DIFF_FLAG_IS_SET(diff, GIT_DIFF_IGNORE_CASE)) {
-		if ((error = git_iterator_set_ignore_case(old_iter, true)) < 0 ||
-			(error = git_iterator_set_ignore_case(new_iter, true)) < 0)
-			goto cleanup;
+		git_iterator_set_ignore_case(old_iter, true);
+		git_iterator_set_ignore_case(new_iter, true);
 	}
 
 	/* finish initialization */

src/iterator.h

@@ -34,10 +34,19 @@ typedef enum {
 	GIT_ITERATOR_DONT_AUTOEXPAND  = (1u << 3),
 	/** convert precomposed unicode to decomposed unicode */
 	GIT_ITERATOR_PRECOMPOSE_UNICODE = (1u << 4),
+	/** never convert precomposed unicode to decomposed unicode */
+	GIT_ITERATOR_DONT_PRECOMPOSE_UNICODE = (1u << 5),
 	/** include conflicts */
-	GIT_ITERATOR_INCLUDE_CONFLICTS = (1u << 5),
+	GIT_ITERATOR_INCLUDE_CONFLICTS = (1u << 6),
 } git_iterator_flag_t;
 
+typedef enum {
+	GIT_ITERATOR_STATUS_NORMAL = 0,
+	GIT_ITERATOR_STATUS_IGNORED = 1,
+	GIT_ITERATOR_STATUS_EMPTY = 2,
+	GIT_ITERATOR_STATUS_FILTERED = 3
+} git_iterator_status_t;
+
 typedef struct {
 	const char *start;
 	const char *end;
@@ -57,23 +66,33 @@ typedef struct {
 	int (*current)(const git_index_entry **, git_iterator *);
 	int (*advance)(const git_index_entry **, git_iterator *);
 	int (*advance_into)(const git_index_entry **, git_iterator *);
-	int (*seek)(git_iterator *, const char *prefix);
-	int (*reset)(git_iterator *, const char *start, const char *end);
-	int (*at_end)(git_iterator *);
+	int (*advance_over)(
+		const git_index_entry **, git_iterator_status_t *, git_iterator *);
+	int (*reset)(git_iterator *);
 	void (*free)(git_iterator *);
 } git_iterator_callbacks;
 
 struct git_iterator {
 	git_iterator_type_t type;
 	git_iterator_callbacks *cb;
+
 	git_repository *repo;
+	git_index *index;
+
 	char *start;
+	size_t start_len;
+
 	char *end;
+	size_t end_len;
+
+	bool started;
+	bool ended;
 	git_vector pathlist;
 	size_t pathlist_walk_idx;
 	int (*strcomp)(const char *a, const char *b);
 	int (*strncomp)(const char *a, const char *b, size_t n);
 	int (*prefixcomp)(const char *str, const char *prefix);
+	int (*entry_srch)(const void *key, const void *array_member);
 	size_t stat_calls;
 	unsigned int flags;
 };
@@ -181,54 +200,38 @@ GIT_INLINE(int) git_iterator_advance_into(
 	return iter->cb->advance_into(entry, iter);
 }
 
-/**
- * Advance into a tree or skip over it if it is empty.
+/* Advance over a directory and check if it contains no files or just
+ * ignored files.
  *
- * Because `git_iterator_advance_into` may return GIT_ENOTFOUND if the
- * directory is empty (only with filesystem and working directory
- * iterators) and a common response is to just call `git_iterator_advance`
- * when that happens, this bundles the two into a single simple call.
+ * In a tree or the index, all directories will contain files, but in the
+ * working directory it is possible to have an empty directory tree or a
+ * tree that only contains ignored files.  Many Git operations treat these
+ * cases specially.  This advances over a directory (presumably an
+ * untracked directory) but checks during the scan if there are any files
+ * and any non-ignored files.
  */
-GIT_INLINE(int) git_iterator_advance_into_or_over(
-	const git_index_entry **entry, git_iterator *iter)
-{
-	int error = iter->cb->advance_into(entry, iter);
-	if (error == GIT_ENOTFOUND) {
-		giterr_clear();
-		error = iter->cb->advance(entry, iter);
-	}
-	return error;
-}
-
-/* Seek is currently unimplemented */
-GIT_INLINE(int) git_iterator_seek(
-	git_iterator *iter, const char *prefix)
+GIT_INLINE(int) git_iterator_advance_over(
+	const git_index_entry **entry,
+	git_iterator_status_t *status,
+	git_iterator *iter)
 {
-	return iter->cb->seek(iter, prefix);
+	return iter->cb->advance_over(entry, status, iter);
 }
 
 /**
  * Go back to the start of the iteration.
- *
- * This resets the iterator to the start of the iteration.  It also allows
- * you to reset the `start` and `end` pathname boundaries of the iteration
- * when doing so.
  */
-GIT_INLINE(int) git_iterator_reset(
-	git_iterator *iter, const char *start, const char *end)
+GIT_INLINE(int) git_iterator_reset(git_iterator *iter)
 {
-	return iter->cb->reset(iter, start, end);
+	return iter->cb->reset(iter);
 }
 
 /**
- * Check if the iterator is at the end
- *
- * @return 0 if not at end, >0 if at end
+ * Go back to the start of the iteration after updating the `start` and
+ * `end` pathname boundaries of the iteration.
  */
-GIT_INLINE(int) git_iterator_at_end(git_iterator *iter)
-{
-	return iter->cb->at_end(iter);
-}
+extern int git_iterator_reset_range(
+	git_iterator *iter, const char *start, const char *end);
 
 GIT_INLINE(git_iterator_type_t) git_iterator_type(git_iterator *iter)
 {
@@ -240,6 +243,11 @@ GIT_INLINE(git_repository *) git_iterator_owner(git_iterator *iter)
 	return iter->repo;
 }
 
+GIT_INLINE(git_index *) git_iterator_index(git_iterator *iter)
+{
+	return iter->index;
+}
+
 GIT_INLINE(git_iterator_flag_t) git_iterator_flags(git_iterator *iter)
 {
 	return iter->flags;
@@ -250,21 +258,19 @@ GIT_INLINE(bool) git_iterator_ignore_case(git_iterator *iter)
 	return ((iter->flags & GIT_ITERATOR_IGNORE_CASE) != 0);
 }
 
-extern int git_iterator_set_ignore_case(git_iterator *iter, bool ignore_case);
+extern void git_iterator_set_ignore_case(
+	git_iterator *iter, bool ignore_case);
 
 extern int git_iterator_current_tree_entry(
 	const git_tree_entry **entry_out, git_iterator *iter);
 
 extern int git_iterator_current_parent_tree(
-	const git_tree **tree_out, git_iterator *iter, const char *parent_path);
+	const git_tree **tree_out, git_iterator *iter, size_t depth);
 
 extern bool git_iterator_current_is_ignored(git_iterator *iter);
 
 extern bool git_iterator_current_tree_is_ignored(git_iterator *iter);
 
-extern int git_iterator_cmp(
-	git_iterator *iter, const char *path_prefix);
-
 /**
  * Get full path of the current item from a workdir iterator.  This will
  * return NULL for a non-workdir iterator.  The git_buf is still owned by
@@ -273,35 +279,12 @@ extern int git_iterator_cmp(
 extern int git_iterator_current_workdir_path(
 	git_buf **path, git_iterator *iter);
 
-/* Return index pointer if index iterator, else NULL */
-extern git_index *git_iterator_get_index(git_iterator *iter);
-
-typedef enum {
-	GIT_ITERATOR_STATUS_NORMAL = 0,
-	GIT_ITERATOR_STATUS_IGNORED = 1,
-	GIT_ITERATOR_STATUS_EMPTY = 2,
-	GIT_ITERATOR_STATUS_FILTERED = 3
-} git_iterator_status_t;
-
-/* Advance over a directory and check if it contains no files or just
- * ignored files.
- *
- * In a tree or the index, all directories will contain files, but in the
- * working directory it is possible to have an empty directory tree or a
- * tree that only contains ignored files.  Many Git operations treat these
- * cases specially.  This advances over a directory (presumably an
- * untracked directory) but checks during the scan if there are any files
- * and any non-ignored files.
- */
-extern int git_iterator_advance_over_with_status(
-	const git_index_entry **entry, git_iterator_status_t *status, git_iterator *iter);
-
 /**
  * Retrieve the index stored in the iterator.
  *
- * Only implemented for the workdir iterator
+ * Only implemented for the workdir and index iterators.
  */
-extern int git_iterator_index(git_index **out, git_iterator *iter);
+extern git_index *git_iterator_index(git_iterator *iter);
 
 typedef int (*git_iterator_walk_cb)(
 	const git_index_entry **entries,

src/object_api.c

@@ -15,7 +15,7 @@
 #include "tag.h"
 
 /**
- * Blob
+ * Commit
  */
 int git_commit_lookup(git_commit **out, git_repository *repo, const git_oid *id)
 {
@@ -42,6 +42,10 @@ git_repository *git_commit_owner(const git_commit *obj)
 	return git_object_owner((const git_object *)obj);
 }
 
+int git_commit_dup(git_commit **out, git_commit *obj)
+{
+	return git_object_dup((git_object **)out, (git_object *)obj);
+}
 
 /**
  * Tree
@@ -71,6 +75,10 @@ git_repository *git_tree_owner(const git_tree *obj)
 	return git_object_owner((const git_object *)obj);
 }
 
+int git_tree_dup(git_tree **out, git_tree *obj)
+{
+	return git_object_dup((git_object **)out, (git_object *)obj);
+}
 
 /**
  * Tag
@@ -100,6 +108,11 @@ git_repository *git_tag_owner(const git_tag *obj)
 	return git_object_owner((const git_object *)obj);
 }
 
+int git_tag_dup(git_tag **out, git_tag *obj)
+{
+	return git_object_dup((git_object **)out, (git_object *)obj);
+}
+
 /**
  * Blob
  */
@@ -127,3 +140,8 @@ git_repository *git_blob_owner(const git_blob *obj)
 {
 	return git_object_owner((const git_object *)obj);
 }
+
+int git_blob_dup(git_blob **out, git_blob *obj)
+{
+	return git_object_dup((git_object **)out, (git_object *)obj);
+}

src/path.c

@@ -810,6 +810,20 @@ int git_path_cmp(
 	return (c1 < c2) ? -1 : (c1 > c2) ? 1 : 0;
 }
 
+size_t git_path_common_dirlen(const char *one, const char *two)
+{
+	const char *p, *q, *dirsep = NULL;
+
+	for (p = one, q = two; *p && *q; p++, q++) {
+		if (*p == '/' && *q == '/')
+			dirsep = p;
+		else if (*p != *q)
+			break;
+	}
+
+	return dirsep ? (dirsep - one) + 1 : 0;
+}
+
 int git_path_make_relative(git_buf *path, const char *parent)
 {
 	const char *p, *q, *p_dirsep, *q_dirsep;

src/path.h

@@ -203,6 +203,18 @@ extern bool git_path_contains(git_buf *dir, const char *item);
 extern bool git_path_contains_dir(git_buf *parent, const char *subdir);
 
 /**
+ * Determine the common directory length between two paths, including
+ * the final path separator.  For example, given paths 'a/b/c/1.txt
+ * and 'a/b/c/d/2.txt', the common directory is 'a/b/c/', and this
+ * will return the length of the string 'a/b/c/', which is 6.
+ *
+ * @param one The first path
+ * @param two The second path
+ * @return The length of the common directory
+ */
+extern size_t git_path_common_dirlen(const char *one, const char *two);
+
+/**
  * Make the path relative to the given parent path.
  *
  * @param path The path to make relative

src/pathspec.c

@@ -418,7 +418,7 @@ static int pathspec_match_from_iterator(
 		GITERR_CHECK_ALLOC(m);
 	}
 
-	if ((error = git_iterator_reset(iter, ps->prefix, ps->prefix)) < 0)
+	if ((error = git_iterator_reset_range(iter, ps->prefix, ps->prefix)) < 0)
 		goto done;
 
 	if (git_iterator_type(iter) == GIT_ITERATOR_TYPE_WORKDIR &&

tests/core/path.c

@@ -652,3 +652,23 @@ void test_core_path__15_resolve_relative(void)
 
 	git_buf_free(&buf);
 }
+
+#define assert_common_dirlen(i, p, q) \
+	cl_assert_equal_i((i), git_path_common_dirlen((p), (q)));
+
+void test_core_path__16_resolve_relative(void)
+{
+	assert_common_dirlen(0, "", "");
+	assert_common_dirlen(0, "", "bar.txt");
+	assert_common_dirlen(0, "foo.txt", "bar.txt");
+	assert_common_dirlen(0, "foo.txt", "");
+	assert_common_dirlen(0, "foo/bar.txt", "bar/foo.txt");
+	assert_common_dirlen(0, "foo/bar.txt", "../foo.txt");
+
+	assert_common_dirlen(1, "/one.txt", "/two.txt");
+	assert_common_dirlen(4, "foo/one.txt", "foo/two.txt");
+	assert_common_dirlen(5, "/foo/one.txt", "/foo/two.txt");
+
+	assert_common_dirlen(6, "a/b/c/foo.txt", "a/b/c/d/e/bar.txt");
+	assert_common_dirlen(7, "/a/b/c/foo.txt", "/a/b/c/d/e/bar.txt");
+}

tests/diff/racediffiter.c

+/* This test exercises the problem described in
+** https://github.com/libgit2/libgit2/pull/3568
+** where deleting a directory during a diff/status
+** operation can cause an access violation.
+**
+** The "test_diff_racediffiter__basic() test confirms
+** the normal operation of diff on the given repo.
+**
+** The "test_diff_racediffiter__racy_rmdir() test
+** uses the new diff progress callback to delete
+** a directory (after the initial readdir() and
+** before the directory itself is visited) causing
+** the recursion and iteration to fail.
+*/
+
+#include "clar_libgit2.h"
+#include "diff_helpers.h"
+
+#define ARRAY_LEN(a) (sizeof(a) / sizeof(a[0]))
+
+void test_diff_racediffiter__initialize(void)
+{
+}
+
+void test_diff_racediffiter__cleanup(void)
+{
+	cl_git_sandbox_cleanup();
+}
+
+typedef struct
+{
+	const char *path;
+	git_delta_t t;
+
+} basic_payload;
+
+static int notify_cb__basic(
+	const git_diff *diff_so_far,
+	const git_diff_delta *delta_to_add,
+	const char *matched_pathspec,
+	void *payload)
+{
+	basic_payload *exp = (basic_payload *)payload;
+	basic_payload *e;
+
+	GIT_UNUSED(diff_so_far);
+	GIT_UNUSED(matched_pathspec);
+
+	for (e = exp; e->path; e++) {
+		if (strcmp(e->path, delta_to_add->new_file.path) == 0) {
+			cl_assert_equal_i(e->t, delta_to_add->status);
+			return 0;
+		}
+	}
+	cl_assert(0);
+	return GIT_ENOTFOUND;
+}
+
+void test_diff_racediffiter__basic(void)
+{
+	git_diff_options opts = GIT_DIFF_OPTIONS_INIT;
+	git_repository *repo = cl_git_sandbox_init("diff");
+	git_diff *diff;
+
+	basic_payload exp_a[] = {
+		{ "another.txt", GIT_DELTA_MODIFIED },
+		{ "readme.txt", GIT_DELTA_MODIFIED },
+		{ "zzzzz/", GIT_DELTA_IGNORED },
+		{ NULL, 0 }
+	};
+
+	cl_must_pass(p_mkdir("diff/zzzzz", 0777));
+
+	opts.flags |= GIT_DIFF_INCLUDE_IGNORED | GIT_DIFF_RECURSE_UNTRACKED_DIRS;
+	opts.notify_cb = notify_cb__basic;
+	opts.payload = exp_a;
+
+	cl_git_pass(git_diff_index_to_workdir(&diff, repo, NULL, &opts));
+
+	git_diff_free(diff);
+}
+
+
+typedef struct {
+	bool first_time;
+	const char *dir;
+	basic_payload *basic_payload;
+} racy_payload;
+
+static int notify_cb__racy_rmdir(
+	const git_diff *diff_so_far,
+	const git_diff_delta *delta_to_add,
+	const char *matched_pathspec,
+	void *payload)
+{
+	racy_payload *pay = (racy_payload *)payload;
+
+	if (pay->first_time) {
+		cl_must_pass(p_rmdir(pay->dir));
+		pay->first_time = false;
+	}
+
+	return notify_cb__basic(diff_so_far, delta_to_add, matched_pathspec, pay->basic_payload);
+}
+
+void test_diff_racediffiter__racy(void)
+{
+	git_diff_options opts = GIT_DIFF_OPTIONS_INIT;
+	git_repository *repo = cl_git_sandbox_init("diff");
+	git_diff *diff;
+
+	basic_payload exp_a[] = {
+		{ "another.txt", GIT_DELTA_MODIFIED },
+		{ "readme.txt", GIT_DELTA_MODIFIED },
+		{ NULL, 0 }
+	};
+
+	racy_payload pay = { true, "diff/zzzzz", exp_a };
+
+	cl_must_pass(p_mkdir("diff/zzzzz", 0777));
+
+	opts.flags |= GIT_DIFF_INCLUDE_IGNORED | GIT_DIFF_RECURSE_UNTRACKED_DIRS;
+	opts.notify_cb = notify_cb__racy_rmdir;
+	opts.payload = &pay;
+
+	cl_git_pass(git_diff_index_to_workdir(&diff, repo, NULL, &opts));
+
+	git_diff_free(diff);
+}

tests/iterator/iterator_helpers.c

+#include "clar_libgit2.h"
+#include "iterator.h"
+#include "repository.h"
+#include "fileops.h"
+#include "iterator_helpers.h"
+#include <stdarg.h>
+
+static void assert_at_end(git_iterator *i, bool verbose)
+{
+	const git_index_entry *end;
+	int error = git_iterator_advance(&end, i);
+
+	if (verbose && error != GIT_ITEROVER)
+		fprintf(stderr, "Expected end of iterator, got '%s'\n", end->path);
+
+	cl_git_fail_with(GIT_ITEROVER, error);
+}
+
+void expect_iterator_items(
+	git_iterator *i,
+	int expected_flat,
+	const char **expected_flat_paths,
+	int expected_total,
+	const char **expected_total_paths)
+{
+	const git_index_entry *entry;
+	int count, error;
+	int no_trees = !(git_iterator_flags(i) & GIT_ITERATOR_INCLUDE_TREES);
+	bool v = false;
+
+	if (expected_flat < 0) { v = true; expected_flat = -expected_flat; }
+	if (expected_total < 0) { v = true; expected_total = -expected_total; }
+
+	if (v) fprintf(stderr, "== %s ==\n", no_trees ? "notrees" : "trees");
+
+	count = 0;
+
+	while (!git_iterator_advance(&entry, i)) {
+		if (v) fprintf(stderr, "  %s %07o\n", entry->path, (int)entry->mode);
+
+		if (no_trees)
+			cl_assert(entry->mode != GIT_FILEMODE_TREE);
+
+		if (expected_flat_paths) {
+			const char *expect_path = expected_flat_paths[count];
+			size_t expect_len = strlen(expect_path);
+
+			cl_assert_equal_s(expect_path, entry->path);
+
+			if (expect_path[expect_len - 1] == '/')
+				cl_assert_equal_i(GIT_FILEMODE_TREE, entry->mode);
+			else
+				cl_assert(entry->mode != GIT_FILEMODE_TREE);
+		}
+
+		if (++count >= expected_flat)
+			break;
+	}
+
+	assert_at_end(i, v);
+	cl_assert_equal_i(expected_flat, count);
+
+	cl_git_pass(git_iterator_reset(i));
+
+	count = 0;
+	cl_git_pass(git_iterator_current(&entry, i));
+
+	if (v) fprintf(stderr, "-- %s --\n", no_trees ? "notrees" : "trees");
+
+	while (entry != NULL) {
+		if (v) fprintf(stderr, "  %s %07o\n", entry->path, (int)entry->mode);
+
+		if (no_trees)
+			cl_assert(entry->mode != GIT_FILEMODE_TREE);
+
+		if (expected_total_paths) {
+			const char *expect_path = expected_total_paths[count];
+			size_t expect_len = strlen(expect_path);
+
+			cl_assert_equal_s(expect_path, entry->path);
+
+			if (expect_path[expect_len - 1] == '/')
+				cl_assert_equal_i(GIT_FILEMODE_TREE, entry->mode);
+			else
+				cl_assert(entry->mode != GIT_FILEMODE_TREE);
+		}
+
+		if (entry->mode == GIT_FILEMODE_TREE) {
+			error = git_iterator_advance_into(&entry, i);
+
+			/* could return NOTFOUND if directory is empty */
+			cl_assert(!error || error == GIT_ENOTFOUND);
+
+			if (error == GIT_ENOTFOUND) {
+				error = git_iterator_advance(&entry, i);
+				cl_assert(!error || error == GIT_ITEROVER);
+			}
+		} else {
+			error = git_iterator_advance(&entry, i);
+			cl_assert(!error || error == GIT_ITEROVER);
+		}
+
+		if (++count >= expected_total)
+			break;
+	}
+
+	assert_at_end(i, v);
+	cl_assert_equal_i(expected_total, count);
+}
+
+
+void expect_advance_over(
+	git_iterator *i,
+	const char *expected_path,
+	git_iterator_status_t expected_status)
+{
+	const git_index_entry *entry;
+	git_iterator_status_t status;
+	int error;
+
+	cl_git_pass(git_iterator_current(&entry, i));
+	cl_assert_equal_s(expected_path, entry->path);
+
+	error = git_iterator_advance_over(&entry, &status, i);
+	cl_assert(!error || error == GIT_ITEROVER);
+	cl_assert_equal_i(expected_status, status);
+}
+
+void expect_advance_into(
+	git_iterator *i,
+	const char *expected_path)
+{
+	const git_index_entry *entry;
+	int error;
+
+	cl_git_pass(git_iterator_current(&entry, i));
+	cl_assert_equal_s(expected_path, entry->path);
+
+	if (S_ISDIR(entry->mode))
+		error = git_iterator_advance_into(&entry, i);
+	else
+		error = git_iterator_advance(&entry, i);
+
+	cl_assert(!error || error == GIT_ITEROVER);
+}
+

tests/iterator/iterator_helpers.h

+
+extern void expect_iterator_items(
+	git_iterator *i,
+	int expected_flat,
+	const char **expected_flat_paths,
+	int expected_total,
+	const char **expected_total_paths);
+
+extern void expect_advance_over(
+	git_iterator *i,
+	const char *expected_path,
+	git_iterator_status_t expected_status);
+
+void expect_advance_into(
+	git_iterator *i,
+	const char *expected_path);

tests/status/worktree.c

@@ -218,6 +218,58 @@ void test_status_worktree__swap_subdir_with_recurse_and_pathspec(void)
 	cl_assert_equal_i(0, counts.wrong_sorted_path);
 }
 
+static void stage_and_commit(git_repository *repo, const char *path)
+{
+	git_index *index;
+
+	cl_git_pass(git_repository_index(&index, repo));
+	cl_git_pass(git_index_add_bypath(index, path));
+	cl_repo_commit_from_index(NULL, repo, NULL, 1323847743, "Initial commit\n");
+	git_index_free(index);
+}
+
+void test_status_worktree__within_subdir(void)
+{
+	status_entry_counts counts;
+	git_repository *repo = cl_git_sandbox_init("status");
+	git_status_options opts = GIT_STATUS_OPTIONS_INIT;
+	char *paths[] = { "zzz_new_dir" };
+	git_strarray pathsArray;
+
+	/* first alter the contents of the worktree */
+	cl_git_mkfile("status/.new_file", "dummy");
+	cl_git_pass(git_futils_mkdir_r("status/zzz_new_dir", 0777));
+	cl_git_mkfile("status/zzz_new_dir/new_file", "dummy");
+	cl_git_mkfile("status/zzz_new_file", "dummy");
+	cl_git_mkfile("status/wut", "dummy");
+
+	stage_and_commit(repo, "zzz_new_dir/new_file");
+
+	/* now get status */
+	memset(&counts, 0x0, sizeof(status_entry_counts));
+	counts.expected_entry_count = entry_count4;
+	counts.expected_paths = entry_paths4;
+	counts.expected_statuses = entry_statuses4;
+	counts.debug = true;
+
+	opts.flags = GIT_STATUS_OPT_INCLUDE_UNTRACKED |
+		GIT_STATUS_OPT_RECURSE_UNTRACKED_DIRS |
+		GIT_STATUS_OPT_DISABLE_PATHSPEC_MATCH;
+
+	pathsArray.count = 1;
+	pathsArray.strings = paths;
+	opts.pathspec = pathsArray;
+
+	// We committed zzz_new_dir/new_file above. It shouldn't be reported.
+	cl_git_pass(
+		git_status_foreach_ext(repo, &opts, cb_status__normal, &counts)
+	);
+
+	cl_assert_equal_i(0, counts.entry_count);
+	cl_assert_equal_i(0, counts.wrong_status_flags_count);
+	cl_assert_equal_i(0, counts.wrong_sorted_path);
+}
+
 /* this test is equivalent to t18-status.c:singlestatus0 */
 void test_status_worktree__single_file(void)
 {
@@ -657,7 +709,7 @@ void test_status_worktree__conflict_has_no_oid(void)
 
 	entry.mode = 0100644;
 	entry.path = "modified_file";
-	git_oid_fromstr(&entry.id, "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef");
+	git_oid_fromstr(&entry.id, "452e4244b5d083ddf0460acf1ecc74db9dcfa11a");
 
 	cl_git_pass(git_repository_index(&index, repo));
 	cl_git_pass(git_index_conflict_add(index, &entry, &entry, &entry));
@@ -692,16 +744,6 @@ void test_status_worktree__conflict_has_no_oid(void)
 	git_status_list_free(statuslist);
 }
 
-static void stage_and_commit(git_repository *repo, const char *path)
-{
-	git_index *index;
-
-	cl_git_pass(git_repository_index(&index, repo));
-	cl_git_pass(git_index_add_bypath(index, path));
-	cl_repo_commit_from_index(NULL, repo, NULL, 1323847743, "Initial commit\n");
-	git_index_free(index);
-}
-
 static void assert_ignore_case(
 	bool should_ignore_case,
 	int expected_lower_cased_file_status,
@@ -1146,3 +1188,86 @@ void test_status_worktree__update_index_with_symlink_doesnt_change_mode(void)
 	git_reference_free(head);
 }
 
+static const char *testrepo2_subdir_paths[] = {
+		"subdir/README",
+		"subdir/new.txt",
+		"subdir/subdir2/README",
+		"subdir/subdir2/new.txt",
+};
+
+static const char *testrepo2_subdir_paths_icase[] = {
+		"subdir/new.txt",
+		"subdir/README",
+		"subdir/subdir2/new.txt",
+		"subdir/subdir2/README"
+};
+
+void test_status_worktree__with_directory_in_pathlist(void)
+{
+	git_repository *repo = cl_git_sandbox_init("testrepo2");
+	git_index *index;
+	git_status_options opts = GIT_STATUS_OPTIONS_INIT;
+	git_status_list *statuslist;
+	const git_status_entry *status;
+	size_t i, entrycount;
+	bool native_ignore_case;
+
+	cl_git_pass(git_repository_index(&index, repo));
+	native_ignore_case =
+			(git_index_caps(index) & GIT_INDEXCAP_IGNORE_CASE) != 0;
+	git_index_free(index);
+
+	opts.pathspec.count = 1;
+	opts.pathspec.strings = malloc(opts.pathspec.count * sizeof(char *));
+	opts.pathspec.strings[0] = "subdir";
+	opts.flags =
+			GIT_STATUS_OPT_DEFAULTS |
+			GIT_STATUS_OPT_INCLUDE_UNMODIFIED |
+			GIT_STATUS_OPT_DISABLE_PATHSPEC_MATCH;
+
+	opts.show = GIT_STATUS_SHOW_WORKDIR_ONLY;
+	git_status_list_new(&statuslist, repo, &opts);
+
+	entrycount = git_status_list_entrycount(statuslist);
+	cl_assert_equal_i(4, entrycount);
+
+	for (i = 0; i < entrycount; i++) {
+		status = git_status_byindex(statuslist, i);
+		cl_assert_equal_i(0, status->status);
+		cl_assert_equal_s(native_ignore_case ?
+			testrepo2_subdir_paths_icase[i] :
+			testrepo2_subdir_paths[i],
+			status->index_to_workdir->old_file.path);
+	}
+
+	opts.show = GIT_STATUS_SHOW_INDEX_ONLY;
+	git_status_list_new(&statuslist, repo, &opts);
+
+	entrycount = git_status_list_entrycount(statuslist);
+	cl_assert_equal_i(4, entrycount);
+
+	for (i = 0; i < entrycount; i++) {
+		status = git_status_byindex(statuslist, i);
+		cl_assert_equal_i(0, status->status);
+		cl_assert_equal_s(native_ignore_case ?
+			testrepo2_subdir_paths_icase[i] :
+			testrepo2_subdir_paths[i],
+			status->head_to_index->old_file.path);
+	}
+
+	opts.show = GIT_STATUS_SHOW_INDEX_AND_WORKDIR;
+	git_status_list_new(&statuslist, repo, &opts);
+
+	entrycount = git_status_list_entrycount(statuslist);
+	cl_assert_equal_i(4, entrycount);
+
+	for (i = 0; i < entrycount; i++) {
+		status = git_status_byindex(statuslist, i);
+		cl_assert_equal_i(0, status->status);
+		cl_assert_equal_s(native_ignore_case ?
+			testrepo2_subdir_paths_icase[i] :
+			testrepo2_subdir_paths[i],
+			status->index_to_workdir->old_file.path);
+	}
+}
+