Index locking and entry allocation changes

This makes the lock management on the index a little bit broader,
having a number of routines hold the lock across looking up the
item to be modified and actually making the modification.  Still
not true thread safety, but more pure index modifications are now
safe which allows the simple cases (such as starting up a diff
while index modifications are underway) safe enough to get the
snapshot without hitting allocation problems.

As part of this, I simplified the allocation of index entries to
use a flex array and just put the path at the end of the index
entry.  This makes every entry self-contained and makes it a
little easier to feel sure that pointers to strings aren't
being accidentally copied and freed while other references are
still being held.

src/common.h

@@ -46,6 +46,7 @@
 # include <unistd.h>
 # ifdef GIT_THREADS
 #	include <pthread.h>
+#	include <sched.h>
 # endif
 #define GIT_STDLIB_CALL
 

src/index.c

@@ -90,13 +90,24 @@ struct entry_long {
 
 struct entry_srch_key {
 	const char *path;
-	size_t path_len;
+	size_t pathlen;
 	int stage;
 };
 
+struct entry_internal {
+	git_index_entry entry;
+	size_t pathlen;
+	char path[GIT_FLEX_ARRAY];
+};
+
+struct reuc_entry_internal {
+	git_index_reuc_entry entry;
+	size_t pathlen;
+	char path[GIT_FLEX_ARRAY];
+};
+
 /* local declarations */
 static size_t read_extension(git_index *index, const char *buffer, size_t buffer_size);
-static size_t read_entry(git_index_entry *dest, const void *buffer, size_t buffer_size);
 static int read_header(struct index_header *dest, const void *buffer);
 
 static int parse_index(git_index *index, const char *buffer, size_t buffer_size);
@@ -109,12 +120,12 @@ static void index_entry_reuc_free(git_index_reuc_entry *reuc);
 int git_index_entry_srch(const void *key, const void *array_member)
 {
 	const struct entry_srch_key *srch_key = key;
-	const git_index_entry *entry = array_member;
+	const struct entry_internal *entry = array_member;
 	int cmp;
 	size_t len1, len2, len;
 
-	len1 = srch_key->path_len;
-	len2 = strlen(entry->path);
+	len1 = srch_key->pathlen;
+	len2 = entry->pathlen;
 	len = len1 < len2 ? len1 : len2;
 
 	cmp = memcmp(srch_key->path, entry->path, len);
@@ -126,7 +137,7 @@ int git_index_entry_srch(const void *key, const void *array_member)
 		return 1;
 
 	if (srch_key->stage != GIT_INDEX_STAGE_ANY)
-		return srch_key->stage - GIT_IDXENTRY_STAGE(entry);
+		return srch_key->stage - GIT_IDXENTRY_STAGE(&entry->entry);
 
 	return 0;
 }
@@ -134,12 +145,12 @@ int git_index_entry_srch(const void *key, const void *array_member)
 int git_index_entry_isrch(const void *key, const void *array_member)
 {
 	const struct entry_srch_key *srch_key = key;
-	const git_index_entry *entry = array_member;
+	const struct entry_internal *entry = array_member;
 	int cmp;
 	size_t len1, len2, len;
 
-	len1 = srch_key->path_len;
-	len2 = strlen(entry->path);
+	len1 = srch_key->pathlen;
+	len2 = entry->pathlen;
 	len = len1 < len2 ? len1 : len2;
 
 	cmp = strncasecmp(srch_key->path, entry->path, len);
@@ -152,7 +163,7 @@ int git_index_entry_isrch(const void *key, const void *array_member)
 		return 1;
 
 	if (srch_key->stage != GIT_INDEX_STAGE_ANY)
-		return srch_key->stage - GIT_IDXENTRY_STAGE(entry);
+		return srch_key->stage - GIT_IDXENTRY_STAGE(&entry->entry);
 
 	return 0;
 }
@@ -276,17 +287,11 @@ static int reuc_icmp(const void *a, const void *b)
 
 static void index_entry_reuc_free(git_index_reuc_entry *reuc)
 {
-	if (!reuc)
-		return;
-	git__free(reuc->path);
 	git__free(reuc);
 }
 
 static void index_entry_free(git_index_entry *entry)
 {
-	if (!entry)
-		return;
-	git__free(entry->path);
 	git__free(entry);
 }
 
@@ -447,18 +452,22 @@ void git_index_free(git_index *index)
 /* call with locked index */
 static void index_free_deleted(git_index *index)
 {
+	int readers = (int)git_atomic_get(&index->readers);
 	size_t i;
 
-	if (git_atomic_get(&index->readers) > 0)
+	if (readers > 0)
 		return;
 
-	for (i = 0; i < index->deleted.length; ++i)
-		index_entry_free(git__swap(index->deleted.contents[i], NULL));
+	for (i = 0; i < index->deleted.length; ++i) {
+		git_index_entry *ie = git__swap(index->deleted.contents[i], NULL);
+		index_entry_free(ie);
+	}
 
 	git_vector_clear(&index->deleted);
 }
 
-static int index_remove_entry(git_index *index, size_t pos, bool need_lock)
+/* call with locked index */
+static int index_remove_entry(git_index *index, size_t pos)
 {
 	int error = 0;
 	git_index_entry *entry = git_vector_get(&index->entries, pos);
@@ -466,23 +475,17 @@ static int index_remove_entry(git_index *index, size_t pos, bool need_lock)
 	if (entry != NULL)
 		git_tree_cache_invalidate_path(index->tree, entry->path);
 
-	if (need_lock && git_mutex_lock(&index->lock) < 0) {
-		giterr_set(GITERR_OS, "Unable to lock index");
-		return -1;
-	}
-
 	error = git_vector_remove(&index->entries, pos);
 
 	if (!error) {
-		if (!git_atomic_get(&index->readers))
-			index_entry_free(entry);
-		else
+		int readers = (int)git_atomic_get(&index->readers);
+
+		if (readers > 0)
 			error = git_vector_insert(&index->deleted, entry);
+		else
+			index_entry_free(entry);
 	}
 
-	if (need_lock)
-		git_mutex_unlock(&index->lock);
-
 	return error;
 }
 
@@ -501,7 +504,7 @@ int git_index_clear(git_index *index)
 	}
 
 	while (!error && index->entries.length > 0)
-		error = index_remove_entry(index, index->entries.length - 1, false);
+		error = index_remove_entry(index, index->entries.length - 1);
 	index_free_deleted(index);
 
 	git_mutex_unlock(&index->lock);
@@ -684,6 +687,16 @@ size_t git_index_entrycount(const git_index *index)
 	return index->entries.length;
 }
 
+GIT_INLINE(int) git_index__find_internal(
+	size_t *out, git_index *index, const char *path, size_t path_len, int stage,
+	bool need_lock)
+{
+	if (index_sort_if_needed(index, need_lock) < 0)
+		return -1;
+	return git_index__find_in_entries(
+		out, &index->entries, index->entries_search, path, path_len, stage);
+}
+
 const git_index_entry *git_index_get_byindex(
 	git_index *index, size_t n)
 {
@@ -700,7 +713,7 @@ const git_index_entry *git_index_get_bypath(
 
 	assert(index);
 
-	if (git_index__find(&pos, index, path, 0, stage) < 0) {
+	if (git_index__find_internal(&pos, index, path, 0, stage, true) < 0) {
 		giterr_set(GITERR_INDEX, "Index does not contain %s", path);
 		return NULL;
 	}
@@ -724,6 +737,21 @@ void git_index_entry__init_from_stat(
 	entry->file_size = st->st_size;
 }
 
+static git_index_entry *index_entry_alloc(const char *path)
+{
+	size_t pathlen = strlen(path);
+	struct entry_internal *entry =
+		git__calloc(sizeof(struct entry_internal) + pathlen + 1, 1);
+	if (!entry)
+		return NULL;
+
+	entry->pathlen = pathlen;
+	memcpy(entry->path, path, pathlen);
+	entry->entry.path = entry->path;
+
+	return (git_index_entry *)entry;
+}
+
 static int index_entry_init(
 	git_index_entry **entry_out, git_index *index, const char *rel_path)
 {
@@ -743,22 +771,31 @@ static int index_entry_init(
 	if (error < 0)
 		return error;
 
-	entry = git__calloc(1, sizeof(git_index_entry));
+	entry = index_entry_alloc(rel_path);
 	GITERR_CHECK_ALLOC(entry);
 
-	git_index_entry__init_from_stat(entry, &st, !index->distrust_filemode);
-
 	entry->id = oid;
-	entry->path = git__strdup(rel_path);
-	if (!entry->path) {
-		git__free(entry);
-		return -1;
-	}
+	git_index_entry__init_from_stat(entry, &st, !index->distrust_filemode);
 
-	*entry_out = entry;
+	*entry_out = (git_index_entry *)entry;
 	return 0;
 }
 
+static git_index_reuc_entry *reuc_entry_alloc(const char *path)
+{
+	size_t pathlen = strlen(path);
+	struct reuc_entry_internal *entry =
+		git__calloc(sizeof(struct reuc_entry_internal) + pathlen + 1, 1);
+	if (!entry)
+		return NULL;
+
+	entry->pathlen = pathlen;
+	memcpy(entry->path, path, pathlen);
+	entry->entry.path = entry->path;
+
+	return (git_index_reuc_entry *)entry;
+}
+
 static int index_entry_reuc_init(git_index_reuc_entry **reuc_out,
 	const char *path,
 	int ancestor_mode, const git_oid *ancestor_oid,
@@ -769,17 +806,9 @@ static int index_entry_reuc_init(git_index_reuc_entry **reuc_out,
 
 	assert(reuc_out && path);
 
-	*reuc_out = NULL;
-
-	reuc = git__calloc(1, sizeof(git_index_reuc_entry));
+	*reuc_out = reuc = reuc_entry_alloc(path);
 	GITERR_CHECK_ALLOC(reuc);
 
-	reuc->path = git__strdup(path);
-	if (reuc->path == NULL) {
-		git__free(reuc);
-		return -1;
-	}
-
 	if ((reuc->mode[0] = ancestor_mode) > 0)
 		git_oid_cpy(&reuc->oid[0], ancestor_oid);
 
@@ -789,10 +818,16 @@ static int index_entry_reuc_init(git_index_reuc_entry **reuc_out,
 	if ((reuc->mode[2] = their_mode) > 0)
 		git_oid_cpy(&reuc->oid[2], their_oid);
 
-	*reuc_out = reuc;
 	return 0;
 }
 
+static void index_entry_cpy(git_index_entry *tgt, const git_index_entry *src)
+{
+	char *tgt_path = tgt->path;
+	memcpy(tgt, src, sizeof(*tgt));
+	tgt->path = tgt_path; /* reset to existing path data */
+}
+
 static int index_entry_dup(git_index_entry **out, const git_index_entry *src)
 {
 	git_index_entry *entry;
@@ -802,19 +837,10 @@ static int index_entry_dup(git_index_entry **out, const git_index_entry *src)
 		return 0;
 	}
 
-	entry = git__malloc(sizeof(git_index_entry));
+	*out = entry = index_entry_alloc(src->path);
 	GITERR_CHECK_ALLOC(entry);
 
-	memcpy(entry, src, sizeof(git_index_entry));
-
-	/* duplicate the path string so we own it */
-	entry->path = git__strdup(entry->path);
-	if (!entry->path) {
-		git__free(entry);
-		return -1;
-	}
-
-	*out = entry;
+	index_entry_cpy(entry, src);
 	return 0;
 }
 
@@ -827,13 +853,13 @@ static int has_file_name(git_index *index,
 	const char *name = entry->path;
 
 	while (pos < index->entries.length) {
-		git_index_entry *p = index->entries.contents[pos++];
+		struct entry_internal *p = index->entries.contents[pos++];
 
-		if (len >= strlen(p->path))
+		if (len >= p->pathlen)
 			break;
 		if (memcmp(name, p->path, len))
 			break;
-		if (GIT_IDXENTRY_STAGE(p) != stage)
+		if (GIT_IDXENTRY_STAGE(&p->entry) != stage)
 			continue;
 		if (p->path[len] != '/')
 			continue;
@@ -841,7 +867,7 @@ static int has_file_name(git_index *index,
 		if (!ok_to_replace)
 			break;
 
-		if (index_remove_entry(index, --pos, true) < 0)
+		if (index_remove_entry(index, --pos) < 0)
 			break;
 	}
 	return retval;
@@ -860,7 +886,7 @@ static int has_dir_name(git_index *index,
 	const char *slash = name + strlen(name);
 
 	for (;;) {
-		size_t len, position;
+		size_t len, pos;
 
 		for (;;) {
 			if (*--slash == '/')
@@ -870,12 +896,12 @@ static int has_dir_name(git_index *index,
 		}
 		len = slash - name;
 
-		if (git_index__find(&position, index, name, len, stage) == 0) {
+		if (!git_index__find_internal(&pos, index, name, len, stage, false)) {
 			retval = -1;
 			if (!ok_to_replace)
 				break;
 
-			if (index_remove_entry(index, position, true) < 0)
+			if (index_remove_entry(index, pos) < 0)
 				break;
 			continue;
 		}
@@ -885,20 +911,19 @@ static int has_dir_name(git_index *index,
 		 * already matches the sub-directory, then we know
 		 * we're ok, and we can exit.
 		 */
-		while (position < index->entries.length) {
-			git_index_entry *p = index->entries.contents[position];
+		for (; pos < index->entries.length; ++pos) {
+			struct entry_internal *p = index->entries.contents[pos];
 
-			if ((strlen(p->path) <= len) ||
-			    (p->path[len] != '/') ||
+			if (p->pathlen <= len ||
+			    p->path[len] != '/' ||
 			    memcmp(p->path, name, len))
 				break; /* not our subdirectory */
 
-			if (GIT_IDXENTRY_STAGE(p) == stage)
+			if (GIT_IDXENTRY_STAGE(&p->entry) == stage)
 				return retval;
-
-			position++;
 		}
 	}
+
 	return retval;
 }
 
@@ -909,7 +934,8 @@ static int check_file_directory_collision(git_index *index,
 	retval = retval + has_dir_name(index, entry, ok_to_replace);
 
 	if (retval) {
-		giterr_set(GITERR_INDEX, "'%s' appears as both a file an a directory", entry->path);
+		giterr_set(GITERR_INDEX,
+			"'%s' appears as both a file and a directory", entry->path);
 		return -1;
 	}
 
@@ -931,10 +957,9 @@ static int index_insert(
 	assert(index && entry_ptr);
 
 	entry = *entry_ptr;
-	assert(entry && entry->path);
 
 	/* make sure that the path length flag is correct */
-	path_length = strlen(entry->path);
+	path_length = ((struct entry_internal *)entry)->pathlen;
 
 	entry->flags &= ~GIT_IDXENTRY_NAMEMASK;
 
@@ -943,43 +968,43 @@ static int index_insert(
 	else
 		entry->flags |= GIT_IDXENTRY_NAMEMASK;
 
+	if ((error = git_mutex_lock(&index->lock)) < 0) {
+		giterr_set(GITERR_OS, "Unable to acquire index lock");
+		return error;
+	}
+
+	git_vector_sort(&index->entries);
+
 	/* look if an entry with this path already exists */
-	if (!git_index__find(
-			&position, index, entry->path, 0, GIT_IDXENTRY_STAGE(entry))) {
+	if (!git_index__find_internal(
+			&position, index, entry->path, 0, GIT_IDXENTRY_STAGE(entry), false)) {
 		existing = index->entries.contents[position];
 		/* update filemode to existing values if stat is not trusted */
 		entry->mode = index_merge_mode(index, existing, entry->mode);
 	}
 
+	/* look for tree / blob name collisions, removing conflicts if requested */
 	error = check_file_directory_collision(index, entry, position, replace);
 	if (error < 0)
-		goto done;
+		/* skip changes */;
 
 	/* if we are replacing an existing item, overwrite the existing entry
 	 * and return it in place of the passed in one.
 	 */
-	if (existing && replace) {
-		git__free(entry->path);
-		entry->path = existing->path;
-
-		memcpy(existing, entry, sizeof(*entry));
+	else if (existing && replace) {
+		index_entry_cpy(existing, entry);
+		index_entry_free(entry);
 		*entry_ptr = existing;
-
-		git__free(entry);
-		return 0;
 	}
-
+	else {
 		/* if replacing is not requested or no existing entry exists, just
 		 * insert entry at the end; the index is no longer sorted
 		 */
-	if ((error = git_mutex_lock(&index->lock)) < 0) {
-		giterr_set(GITERR_OS, "Unable to acquire index lock");
-	} else {
 		error = git_vector_insert(&index->entries, entry);
-		git_mutex_unlock(&index->lock);
 	}
 
-done:
+	git_mutex_unlock(&index->lock);
+
 	if (error < 0) {
 		index_entry_free(*entry_ptr);
 		*entry_ptr = NULL;
@@ -1053,7 +1078,7 @@ int git_index_add(git_index *index, const git_index_entry *source_entry)
 	git_index_entry *entry = NULL;
 	int ret;
 
-	assert(index && source_entry);
+	assert(index && source_entry && source_entry->path);
 
 	if ((ret = index_entry_dup(&entry, source_entry)) < 0 ||
 		(ret = index_insert(index, &entry, 1)) < 0)
@@ -1065,15 +1090,24 @@ int git_index_add(git_index *index, const git_index_entry *source_entry)
 
 int git_index_remove(git_index *index, const char *path, int stage)
 {
+	int error;
 	size_t position;
 
-	if (git_index__find(&position, index, path, 0, stage) < 0) {
-		giterr_set(GITERR_INDEX, "Index does not contain %s at stage %d",
-			path, stage);
-		return GIT_ENOTFOUND;
+	if (git_mutex_lock(&index->lock) < 0) {
+		giterr_set(GITERR_OS, "Failed to lock index");
+		return -1;
 	}
 
-	return index_remove_entry(index, position, true);
+	if (git_index__find_internal(&position, index, path, 0, stage, false) < 0) {
+		giterr_set(
+			GITERR_INDEX, "Index does not contain %s at stage %d", path, stage);
+		error = GIT_ENOTFOUND;
+	} else {
+		error = index_remove_entry(index, position);
+	}
+
+	git_mutex_unlock(&index->lock);
+	return error;
 }
 
 int git_index_remove_directory(git_index *index, const char *dir, int stage)
@@ -1083,12 +1117,17 @@ int git_index_remove_directory(git_index *index, const char *dir, int stage)
 	size_t pos;
 	git_index_entry *entry;
 
-	if (git_buf_sets(&pfx, dir) < 0 || git_path_to_dir(&pfx) < 0)
+	if (git_mutex_lock(&index->lock) < 0) {
+		giterr_set(GITERR_OS, "Failed to lock index");
 		return -1;
+	}
 
-	git_index__find(&pos, index, pfx.ptr, pfx.size, GIT_INDEX_STAGE_ANY);
+	if (!(error = git_buf_sets(&pfx, dir)) &&
+		!(error = git_path_to_dir(&pfx)))
+		git_index__find_internal(
+			&pos, index, pfx.ptr, pfx.size, GIT_INDEX_STAGE_ANY, false);
 
-	while (1) {
+	while (!error) {
 		entry = git_vector_get(&index->entries, pos);
 		if (!entry || git__prefixcmp(entry->path, pfx.ptr) != 0)
 			break;
@@ -1098,12 +1137,12 @@ int git_index_remove_directory(git_index *index, const char *dir, int stage)
 			continue;
 		}
 
-		if ((error = index_remove_entry(index, pos, true)) < 0)
-			break;
+		error = index_remove_entry(index, pos);
 
-		/* removed entry at 'pos' so we don't need to increment it */
+		/* removed entry at 'pos' so we don't need to increment */
 	}
 
+	git_mutex_unlock(&index->lock);
 	git_buf_free(&pfx);
 
 	return error;
@@ -1115,7 +1154,7 @@ int git_index__find_in_entries(
 {
 	struct entry_srch_key srch_key;
 	srch_key.path = path;
-	srch_key.path_len = !path_len ? strlen(path) : path_len;
+	srch_key.pathlen = !path_len ? strlen(path) : path_len;
 	srch_key.stage = stage;
 	return git_vector_bsearch2(out, entries, entry_srch, &srch_key);
 }
@@ -1124,12 +1163,7 @@ int git_index__find(
 	size_t *out, git_index *index, const char *path, size_t path_len, int stage)
 {
 	assert(index && path);
-
-	if (index_sort_if_needed(index, true) < 0)
-		return -1;
-
-	return git_index__find_in_entries(
-		out, &index->entries, index->entries_search, path, path_len, stage);
+	return git_index__find_internal(out, index, path, path_len, stage, true);
 }
 
 int git_index_find(size_t *at_pos, git_index *index, const char *path)
@@ -1138,7 +1172,13 @@ int git_index_find(size_t *at_pos, git_index *index, const char *path)
 
 	assert(index && path);
 
+	if (git_mutex_lock(&index->lock) < 0) {
+		giterr_set(GITERR_OS, "Failed to lock index");
+		return -1;
+	}
+
 	if (git_vector_bsearch2(&pos, &index->entries, index->entries_search_path, path) < 0) {
+		git_mutex_unlock(&index->lock);
 		giterr_set(GITERR_INDEX, "Index does not contain %s", path);
 		return GIT_ENOTFOUND;
 	}
@@ -1158,6 +1198,7 @@ int git_index_find(size_t *at_pos, git_index *index, const char *path)
 	if (at_pos)
 		*at_pos = pos;
 
+	git_mutex_unlock(&index->lock);
 	return 0;
 }
 
@@ -1288,6 +1329,11 @@ int git_index_conflict_remove(git_index *index, const char *path)
 	if (git_index_find(&pos, index, path) < 0)
 		return GIT_ENOTFOUND;
 
+	if (git_mutex_lock(&index->lock) < 0) {
+		giterr_set(GITERR_OS, "Unable to lock index");
+		return -1;
+	}
+
 	while ((conflict_entry = git_vector_get(&index->entries, pos)) != NULL) {
 
 		if (index->entries_cmp_path(conflict_entry->path, path) != 0)
@@ -1298,10 +1344,12 @@ int git_index_conflict_remove(git_index *index, const char *path)
 			continue;
 		}
 
-		if ((error = index_remove_entry(index, pos, true)) < 0)
+		if ((error = index_remove_entry(index, pos)) < 0)
 			break;
 	}
 
+	git_mutex_unlock(&index->lock);
+
 	return error;
 }
 
@@ -1311,7 +1359,7 @@ static int index_conflicts_match(const git_vector *v, size_t idx, void *p)
 	git_index_entry *entry = git_vector_get(v, idx);
 
 	if (GIT_IDXENTRY_STAGE(entry) > 0 &&
-		!index_remove_entry(index, idx, false))
+		!index_remove_entry(index, idx))
 		return 1;
 
 	return 0;
@@ -1488,7 +1536,6 @@ static int index_reuc_insert(
 		return git_vector_insert(&index->reuc, reuc);
 
 	/* exists, replace it */
-	git__free((*existing)->path);
 	git__free(*existing);
 	*existing = reuc;
 
@@ -1596,13 +1643,9 @@ static int read_reuc(git_index *index, const char *buffer, size_t size)
 		if (size <= len)
 			return index_error_invalid("reading reuc entries");
 
-		lost = git__calloc(1, sizeof(git_index_reuc_entry));
+		lost = reuc_entry_alloc(buffer);
 		GITERR_CHECK_ALLOC(lost);
 
-		/* read NUL-terminated pathname for entry */
-		lost->path = git__strdup(buffer);
-		GITERR_CHECK_ALLOC(lost->path);
-
 		size -= len;
 		buffer += len;
 
@@ -1700,41 +1743,41 @@ static int read_conflict_names(git_index *index, const char *buffer, size_t size
 	return 0;
 }
 
-static size_t read_entry(git_index_entry *dest, const void *buffer, size_t buffer_size)
+static size_t read_entry(
+	git_index_entry **out, const void *buffer, size_t buffer_size)
 {
 	size_t path_length, entry_size;
 	uint16_t flags_raw;
 	const char *path_ptr;
 	const struct entry_short *source = buffer;
+	git_index_entry entry = {{0}};
 
 	if (INDEX_FOOTER_SIZE + minimal_entry_size > buffer_size)
 		return 0;
 
-	memset(dest, 0x0, sizeof(git_index_entry));
-
-	dest->ctime.seconds = (git_time_t)ntohl(source->ctime.seconds);
-	dest->ctime.nanoseconds = ntohl(source->ctime.nanoseconds);
-	dest->mtime.seconds = (git_time_t)ntohl(source->mtime.seconds);
-	dest->mtime.nanoseconds = ntohl(source->mtime.nanoseconds);
-	dest->dev = ntohl(source->dev);
-	dest->ino = ntohl(source->ino);
-	dest->mode = ntohl(source->mode);
-	dest->uid = ntohl(source->uid);
-	dest->gid = ntohl(source->gid);
-	dest->file_size = ntohl(source->file_size);
-	git_oid_cpy(&dest->id, &source->oid);
-	dest->flags = ntohs(source->flags);
-
-	if (dest->flags & GIT_IDXENTRY_EXTENDED) {
+	entry.ctime.seconds = (git_time_t)ntohl(source->ctime.seconds);
+	entry.ctime.nanoseconds = ntohl(source->ctime.nanoseconds);
+	entry.mtime.seconds = (git_time_t)ntohl(source->mtime.seconds);
+	entry.mtime.nanoseconds = ntohl(source->mtime.nanoseconds);
+	entry.dev = ntohl(source->dev);
+	entry.ino = ntohl(source->ino);
+	entry.mode = ntohl(source->mode);
+	entry.uid = ntohl(source->uid);
+	entry.gid = ntohl(source->gid);
+	entry.file_size = ntohl(source->file_size);
+	git_oid_cpy(&entry.id, &source->oid);
+	entry.flags = ntohs(source->flags);
+
+	if (entry.flags & GIT_IDXENTRY_EXTENDED) {
 		const struct entry_long *source_l = (const struct entry_long *)source;
 		path_ptr = source_l->path;
 
 		flags_raw = ntohs(source_l->flags_extended);
-		memcpy(&dest->flags_extended, &flags_raw, 2);
+		memcpy(&entry.flags_extended, &flags_raw, 2);
 	} else
 		path_ptr = source->path;
 
-	path_length = dest->flags & GIT_IDXENTRY_NAMEMASK;
+	path_length = entry.flags & GIT_IDXENTRY_NAMEMASK;
 
 	/* if this is a very long string, we must find its
 	 * real length without overflowing */
@@ -1748,7 +1791,7 @@ static size_t read_entry(git_index_entry *dest, const void *buffer, size_t buffe
 		path_length = path_end - path_ptr;
 	}
 
-	if (dest->flags & GIT_IDXENTRY_EXTENDED)
+	if (entry.flags & GIT_IDXENTRY_EXTENDED)
 		entry_size = long_entry_size(path_length);
 	else
 		entry_size = short_entry_size(path_length);
@@ -1756,8 +1799,10 @@ static size_t read_entry(git_index_entry *dest, const void *buffer, size_t buffe
 	if (INDEX_FOOTER_SIZE + entry_size > buffer_size)
 		return 0;
 
-	dest->path = git__strdup(path_ptr);
-	assert(dest->path);
+	entry.path = (char *)path_ptr;
+
+	if (index_entry_dup(out, &entry) < 0)
+		return 0;
 
 	return entry_size;
 }
@@ -1853,20 +1898,12 @@ static int parse_index(git_index *index, const char *buffer, size_t buffer_size)
 		return -1;
 	}
 
-	git_vector_clear(&index->entries);
+	assert(!index->entries.length);
 
 	/* Parse all the entries */
 	for (i = 0; i < header.entry_count && buffer_size > INDEX_FOOTER_SIZE; ++i) {
-		size_t entry_size;
 		git_index_entry *entry;
-
-		entry = git__malloc(sizeof(git_index_entry));
-		if (!entry) {
-			error = -1;
-			goto done;
-		}
-
-		entry_size = read_entry(entry, buffer, buffer_size);
+		size_t entry_size = read_entry(&entry, buffer, buffer_size);
 
 		/* 0 bytes read means an object corruption */
 		if (entry_size == 0) {
@@ -1874,8 +1911,10 @@ static int parse_index(git_index *index, const char *buffer, size_t buffer_size)
 			goto done;
 		}
 
-		if ((error = git_vector_insert(&index->entries, entry)) < 0)
+		if ((error = git_vector_insert(&index->entries, entry)) < 0) {
+			index_entry_free(entry);
 			goto done;
+		}
 
 		seek_forward(entry_size);
 	}
@@ -1953,7 +1992,7 @@ static int write_disk_entry(git_filebuf *file, git_index_entry *entry)
 	size_t path_len, disk_size;
 	char *path;
 
-	path_len = strlen(entry->path);
+	path_len = ((struct entry_internal *)entry)->pathlen;
 
 	if (entry->flags & GIT_IDXENTRY_EXTENDED)
 		disk_size = long_entry_size(path_len);
@@ -2222,7 +2261,7 @@ static int read_tree_cb(
 	if (git_buf_joinpath(&path, root, tentry->filename) < 0)
 		return -1;
 
-	entry = git__calloc(1, sizeof(git_index_entry));
+	entry = index_entry_alloc(path.ptr);
 	GITERR_CHECK_ALLOC(entry);
 
 	entry->mode = tentry->attr;
@@ -2236,7 +2275,9 @@ static int read_tree_cb(
 		entry->mode == old_entry->mode &&
 		git_oid_equal(&entry->id, &old_entry->id))
 	{
+		char *oldpath = entry->path;
 		memcpy(entry, old_entry, sizeof(*entry));
+		entry->path = oldpath;
 		entry->flags_extended = 0;
 	}
 
@@ -2245,7 +2286,6 @@ static int read_tree_cb(
 	else
 		entry->flags = GIT_IDXENTRY_NAMEMASK;
 
-	entry->path = git_buf_detach(&path);
 	git_buf_free(&path);
 
 	if (git_vector_insert(data->new_entries, entry) < 0) {
@@ -2543,10 +2583,11 @@ int git_index__snapshot(git_vector *entries, git_index *index)
 void git_index__release_snapshot(git_index *index)
 {
 	git_atomic_dec(&index->readers);
-	git_index_free(index);
 
 	if (!git_mutex_lock(&index->lock)) {
 		index_free_deleted(index); /* try to free pending deleted items */
 		git_mutex_unlock(&index->lock);
 	}
+
+	git_index_free(index);
 }

src/merge.c

@@ -1230,7 +1230,7 @@ done:
 	return error;
 }
 
-GIT_INLINE(int) index_entry_dup(
+GIT_INLINE(int) index_entry_dup_pool(
 	git_index_entry *out,
 	git_pool *pool,
 	const git_index_entry *src)
@@ -1276,9 +1276,9 @@ static git_merge_diff *merge_diff_from_index_entries(
 	if ((conflict = git_pool_malloc(pool, sizeof(git_merge_diff))) == NULL)
 		return NULL;
 
-	if (index_entry_dup(&conflict->ancestor_entry, pool, entries[TREE_IDX_ANCESTOR]) < 0 ||
-		index_entry_dup(&conflict->our_entry, pool, entries[TREE_IDX_OURS]) < 0 ||
-		index_entry_dup(&conflict->their_entry, pool, entries[TREE_IDX_THEIRS]) < 0)
+	if (index_entry_dup_pool(&conflict->ancestor_entry, pool, entries[TREE_IDX_ANCESTOR]) < 0 ||
+		index_entry_dup_pool(&conflict->our_entry, pool, entries[TREE_IDX_OURS]) < 0 ||
+		index_entry_dup_pool(&conflict->their_entry, pool, entries[TREE_IDX_THEIRS]) < 0)
 		return NULL;
 
 	conflict->our_status = merge_delta_type_from_index_entries(
@@ -1318,7 +1318,7 @@ static int merge_diff_list_insert_unmodified(
 	entry = git_pool_malloc(&diff_list->pool, sizeof(git_index_entry));
 	GITERR_CHECK_ALLOC(entry);
 
-	if ((error = index_entry_dup(entry, &diff_list->pool, tree_items[0])) >= 0)
+	if ((error = index_entry_dup_pool(entry, &diff_list->pool, tree_items[0])) >= 0)
 		error = git_vector_insert(&diff_list->staged, entry);
 
 	return error;

src/thread-utils.h

@@ -47,6 +47,12 @@ typedef git_atomic git_atomic_ssize;
 #define git_thread_exit(status)	pthread_exit(status)
 #define git_thread_join(id, status) pthread_join(id, status)
 
+#if defined(GIT_WIN32)
+#define git_thread_yield() Sleep(0)
+#else
+#define git_thread_yield() sched_yield()
+#endif
+
 /* Pthreads Mutex */
 #define git_mutex pthread_mutex_t
 #define git_mutex_init(a)	pthread_mutex_init(a, NULL)
@@ -176,6 +182,7 @@ GIT_INLINE(int64_t) git_atomic64_add(git_atomic64 *a, int64_t addend)
 #define git_thread_kill(thread) (void)0
 #define git_thread_exit(status) (void)0
 #define git_thread_join(id, status) (void)0
+#define git_thread_yield() (void)0
 
 /* Pthreads Mutex */
 #define git_mutex unsigned int

tests/diff/iterator.c

@@ -363,9 +363,8 @@ static void index_iterator_test(
 	git_index *index;
 	git_iterator *i;
 	const git_index_entry *entry;
-	int error, count = 0;
+	int error, count = 0, caps;
 	git_repository *repo = cl_git_sandbox_init(sandbox);
-	unsigned int caps;
 
 	cl_git_pass(git_repository_index(&index, repo));
 	caps = git_index_caps(index);

tests/threads/diff.c

 #include "clar_libgit2.h"
 #include "thread-utils.h"
 
-static git_repository *g_repo;
-static git_tree *a, *b;
-static git_atomic counts[4];
+static git_repository *_repo;
+static git_tree *_a, *_b;
+static git_atomic _counts[4];
+static int _check_counts;
 
 void test_threads_diff__cleanup(void)
 {
@@ -24,7 +25,7 @@ static void run_in_parallel(
 	cl_assert(id != NULL && th != NULL);
 
 	for (r = 0; r < repeats; ++r) {
-		g_repo = cl_git_sandbox_reopen(); /* reopen sandbox to flush caches */
+		_repo = cl_git_sandbox_reopen(); /* reopen sandbox to flush caches */
 
 		if (before_test) before_test();
 
@@ -53,24 +54,26 @@ static void run_in_parallel(
 static void setup_trees(void)
 {
 	cl_git_pass(git_revparse_single(
-		(git_object **)&a, g_repo, "0017bd4ab1^{tree}"));
+		(git_object **)&_a, _repo, "0017bd4ab1^{tree}"));
 	cl_git_pass(git_revparse_single(
-		(git_object **)&b, g_repo, "26a125ee1b^{tree}"));
+		(git_object **)&_b, _repo, "26a125ee1b^{tree}"));
 
-	memset(counts, 0, sizeof(counts));
+	memset(_counts, 0, sizeof(_counts));
 }
 
 #define THREADS 20
 
 static void free_trees(void)
 {
-	git_tree_free(a); a = NULL;
-	git_tree_free(b); b = NULL;
-
-	cl_assert_equal_i(288, git_atomic_get(&counts[0]));
-	cl_assert_equal_i(112, git_atomic_get(&counts[1]));
-	cl_assert_equal_i( 80, git_atomic_get(&counts[2]));
-	cl_assert_equal_i( 96, git_atomic_get(&counts[3]));
+	git_tree_free(_a); _a = NULL;
+	git_tree_free(_b); _b = NULL;
+
+	if (_check_counts) {
+		cl_assert_equal_i(288, git_atomic_get(&_counts[0]));
+		cl_assert_equal_i(112, git_atomic_get(&_counts[1]));
+		cl_assert_equal_i( 80, git_atomic_get(&_counts[2]));
+		cl_assert_equal_i( 96, git_atomic_get(&_counts[3]));
+	}
 }
 
 static void *run_index_diffs(void *arg)
@@ -81,48 +84,41 @@ static void *run_index_diffs(void *arg)
 	size_t i;
 	int exp[4] = { 0, 0, 0, 0 };
 
-//	fprintf(stderr, "%d >>>\n", thread);
-
 	switch (thread & 0x03) {
 	case 0: /* diff index to workdir */;
-		cl_git_pass(git_diff_index_to_workdir(&diff, g_repo, NULL, &opts));
+		cl_git_pass(git_diff_index_to_workdir(&diff, _repo, NULL, &opts));
 		break;
 	case 1: /* diff tree 'a' to index */;
-		cl_git_pass(git_diff_tree_to_index(&diff, g_repo, a, NULL, &opts));
+		cl_git_pass(git_diff_tree_to_index(&diff, _repo, _a, NULL, &opts));
 		break;
 	case 2: /* diff tree 'b' to index */;
-		cl_git_pass(git_diff_tree_to_index(&diff, g_repo, b, NULL, &opts));
+		cl_git_pass(git_diff_tree_to_index(&diff, _repo, _b, NULL, &opts));
 		break;
 	case 3: /* diff index to workdir (explicit index) */;
 		{
 			git_index *idx;
-			cl_git_pass(git_repository_index(&idx, g_repo));
-			cl_git_pass(git_diff_index_to_workdir(&diff, g_repo, idx, &opts));
+			cl_git_pass(git_repository_index(&idx, _repo));
+			cl_git_pass(git_diff_index_to_workdir(&diff, _repo, idx, &opts));
 			git_index_free(idx);
 			break;
 		}
 	}
 
-//	fprintf(stderr, "%d <<<\n", thread);
-
 	/* keep some diff stats to make sure results are as expected */
 
 	i = git_diff_num_deltas(diff);
-	git_atomic_add(&counts[0], (int32_t)i);
+	git_atomic_add(&_counts[0], (int32_t)i);
 	exp[0] = (int)i;
 
 	while (i > 0) {
 		switch (git_diff_get_delta(diff, --i)->status) {
-		case GIT_DELTA_MODIFIED: exp[1]++; git_atomic_inc(&counts[1]); break;
-		case GIT_DELTA_ADDED:    exp[2]++; git_atomic_inc(&counts[2]); break;
-		case GIT_DELTA_DELETED:  exp[3]++; git_atomic_inc(&counts[3]); break;
+		case GIT_DELTA_MODIFIED: exp[1]++; git_atomic_inc(&_counts[1]); break;
+		case GIT_DELTA_ADDED:    exp[2]++; git_atomic_inc(&_counts[2]); break;
+		case GIT_DELTA_DELETED:  exp[3]++; git_atomic_inc(&_counts[3]); break;
 		default: break;
 		}
 	}
 
-//	fprintf(stderr, "%2d: [%d] total %d (M %d A %d D %d)\n",
-//			thread, (int)(thread & 0x03), exp[0], exp[1], exp[2], exp[3]);
-
 	switch (thread & 0x03) {
 	case 0: case 3:
 		cl_assert_equal_i(8, exp[0]); cl_assert_equal_i(4, exp[1]);
@@ -145,8 +141,71 @@ static void *run_index_diffs(void *arg)
 
 void test_threads_diff__concurrent_diffs(void)
 {
-	g_repo = cl_git_sandbox_init("status");
+	_repo = cl_git_sandbox_init("status");
+	_check_counts = 1;
 
 	run_in_parallel(
 		20, 32, run_index_diffs, setup_trees, free_trees);
 }
+
+static void *run_index_diffs_with_modifier(void *arg)
+{
+	int thread = *(int *)arg;
+	git_diff_options opts = GIT_DIFF_OPTIONS_INIT;
+	git_diff *diff = NULL;
+	git_index *idx = NULL;
+
+	cl_git_pass(git_repository_index(&idx, _repo));
+
+	/* have first thread altering the index as we go */
+	if (thread == 0) {
+		int i;
+
+		for (i = 0; i < 300; ++i) {
+			switch (i & 0x03) {
+			case 0: (void)git_index_add_bypath(idx, "new_file"); break;
+			case 1: (void)git_index_remove_bypath(idx, "modified_file"); break;
+			case 2: (void)git_index_remove_bypath(idx, "new_file"); break;
+			case 3: (void)git_index_add_bypath(idx, "modified_file"); break;
+			}
+			git_thread_yield();
+		}
+
+		git_index_free(idx);
+		return arg;
+	}
+
+	/* only use explicit index in this test to prevent reloading */
+
+	switch (thread & 0x03) {
+	case 0: /* diff index to workdir */;
+		cl_git_pass(git_diff_index_to_workdir(&diff, _repo, idx, &opts));
+		break;
+	case 1: /* diff tree 'a' to index */;
+		cl_git_pass(git_diff_tree_to_index(&diff, _repo, _a, idx, &opts));
+		break;
+	case 2: /* diff tree 'b' to index */;
+		cl_git_pass(git_diff_tree_to_index(&diff, _repo, _b, idx, &opts));
+		break;
+	case 3: /* diff index to workdir reversed */;
+		opts.flags |= GIT_DIFF_REVERSE;
+		cl_git_pass(git_diff_index_to_workdir(&diff, _repo, idx, &opts));
+		break;
+	}
+
+	/* results will be unpredictable with index modifier thread running */
+
+	git_diff_free(diff);
+	git_index_free(idx);
+
+	return arg;
+}
+
+void test_threads_diff__with_concurrent_index_modified(void)
+{
+	_repo = cl_git_sandbox_init("status");
+	_check_counts = 0;
+
+	run_in_parallel(
+		20, 32, run_index_diffs_with_modifier, setup_trees, free_trees);
+}