Merge pull request #1805 from libgit2/threading-packed-load

Thread safety for the refdb_fs

include/git2/commit.h

@@ -24,17 +24,24 @@ GIT_BEGIN_DECL
 /**
  * Lookup a commit object from a repository.
  *
+ * The returned object should be released with `git_commit_free` when no
+ * longer needed.
+ *
  * @param commit pointer to the looked up commit
  * @param repo the repo to use when locating the commit.
  * @param id identity of the commit to locate. If the object is
  *		an annotated tag it will be peeled back to the commit.
  * @return 0 or an error code
  */
-GIT_EXTERN(int) git_commit_lookup(git_commit **commit, git_repository *repo, const git_oid *id);
+GIT_EXTERN(int) git_commit_lookup(
+	git_commit **commit, git_repository *repo, const git_oid *id);
 
 /**
- * Lookup a commit object from a repository,
- * given a prefix of its identifier (short id).
+ * Lookup a commit object from a repository, given a prefix of its
+ * identifier (short id).
+ *
+ * The returned object should be released with `git_commit_free` when no
+ * longer needed.
  *
  * @see git_object_lookup_prefix
  *
@@ -45,7 +52,8 @@ GIT_EXTERN(int) git_commit_lookup(git_commit **commit, git_repository *repo, con
  * @param len the length of the short identifier
  * @return 0 or an error code
  */
-GIT_EXTERN(int) git_commit_lookup_prefix(git_commit **commit, git_repository *repo, const git_oid *id, size_t len);
+GIT_EXTERN(int) git_commit_lookup_prefix(
+	git_commit **commit, git_repository *repo, const git_oid *id, size_t len);
 
 /**
  * Close an open commit

include/git2/revparse.h

@@ -10,7 +10,6 @@
 #include "common.h"
 #include "types.h"
 
-
 /**
  * @file git2/revparse.h
  * @brief Git revision parsing routines
@@ -21,27 +20,37 @@
 GIT_BEGIN_DECL
 
 /**
- * Find a single object, as specified by a revision string. See `man gitrevisions`,
- * or http://git-scm.com/docs/git-rev-parse.html#_specifying_revisions for
+ * Find a single object, as specified by a revision string.
+ *
+ * See `man gitrevisions`, or
+ * http://git-scm.com/docs/git-rev-parse.html#_specifying_revisions for
  * information on the syntax accepted.
  *
+ * The returned object should be released with `git_object_free` when no
+ * longer needed.
+ *
  * @param out pointer to output object
  * @param repo the repository to search in
  * @param spec the textual specification for an object
  * @return 0 on success, GIT_ENOTFOUND, GIT_EAMBIGUOUS, GIT_EINVALIDSPEC or an error code
  */
-GIT_EXTERN(int) git_revparse_single(git_object **out, git_repository *repo, const char *spec);
+GIT_EXTERN(int) git_revparse_single(
+	git_object **out, git_repository *repo, const char *spec);
 
 /**
- * Find a single object, as specified by a revision string.
- * See `man gitrevisions`,
- * or http://git-scm.com/docs/git-rev-parse.html#_specifying_revisions for
+ * Find a single object and intermediate reference by a revision string.
+ *
+ * See `man gitrevisions`, or
+ * http://git-scm.com/docs/git-rev-parse.html#_specifying_revisions for
  * information on the syntax accepted.
  *
  * In some cases (`@{<-n>}` or `<branchname>@{upstream}`), the expression may
  * point to an intermediate reference. When such expressions are being passed
  * in, `reference_out` will be valued as well.
  *
+ * The returned object should be released with `git_object_free` and the
+ * returned reference with `git_reference_free` when no longer needed.
+ *
  * @param object_out pointer to output object
  * @param reference_out pointer to output reference or NULL
  * @param repo the repository to search in
@@ -76,25 +85,27 @@ typedef struct {
 	git_object *from;
 	/** The right element of the revspec; must be freed by the user */
 	git_object *to;
-	/** The intent of the revspec */
+	/** The intent of the revspec (i.e. `git_revparse_mode_t` flags) */
 	unsigned int flags;
 } git_revspec;
 
 /**
- * Parse a revision string for `from`, `to`, and intent. See `man gitrevisions` or
- * http://git-scm.com/docs/git-rev-parse.html#_specifying_revisions for information
- * on the syntax accepted.
+ * Parse a revision string for `from`, `to`, and intent.
+ *
+ * See `man gitrevisions` or
+ * http://git-scm.com/docs/git-rev-parse.html#_specifying_revisions for
+ * information on the syntax accepted.
  *
- * @param revspec Pointer to an user-allocated git_revspec struct where the result
- *	of the rev-parse will be stored
+ * @param revspec Pointer to an user-allocated git_revspec struct where
+ *	              the result of the rev-parse will be stored
  * @param repo the repository to search in
  * @param spec the rev-parse spec to parse
  * @return 0 on success, GIT_INVALIDSPEC, GIT_ENOTFOUND, GIT_EAMBIGUOUS or an error code
  */
 GIT_EXTERN(int) git_revparse(
-		git_revspec *revspec,
-		git_repository *repo,
-		const char *spec);
+	git_revspec *revspec,
+	git_repository *repo,
+	const char *spec);
 
 
 /** @} */

src/cc-compat.h

@@ -54,8 +54,12 @@
 
 #if defined (_MSC_VER)
 	typedef unsigned char bool;
-#	define true 1
-#	define false 0
+#	ifndef true
+#		define true 1
+#	endif
+#	ifndef false
+#		define false 0
+#	endif
 #else
 #	include <stdbool.h>
 #endif

src/diff_tform.c

@@ -859,9 +859,9 @@ find_best_matches:
 				}
 
 				/* write new mapping */
-				tgt2src[t].idx = s;
+				tgt2src[t].idx = (uint32_t)s;
 				tgt2src[t].similarity = (uint32_t)similarity;
-				src2tgt[s].idx = t;
+				src2tgt[s].idx = (uint32_t)t;
 				src2tgt[s].similarity = (uint32_t)similarity;
 			}
 
@@ -869,7 +869,7 @@ find_best_matches:
 			if (tgt2src_copy != NULL &&
 				tgt2src_copy[t].similarity < (uint32_t)similarity)
 			{
-				tgt2src_copy[t].idx = s;
+				tgt2src_copy[t].idx = (uint32_t)s;
 				tgt2src_copy[t].similarity = (uint32_t)similarity;
 			}
 

src/global.c

@@ -71,18 +71,22 @@ int git_threads_init(void)
 
 	GIT_MEMORY_BARRIER;
 
+	win32_pthread_initialize();
+
 	return error;
 }
 
 void git_threads_shutdown(void)
 {
+	/* Shut down any subsystems that have global state */
+	win32_pthread_shutdown();
+	git_futils_dirs_free();
+	git_hash_global_shutdown();
+
 	TlsFree(_tls_index);
 	_tls_init = 0;
-	git_mutex_free(&git__mwindow_mutex);
 
-	/* Shut down any subsystems that have global state */
-	git_hash_global_shutdown();
-	git_futils_dirs_free();
+	git_mutex_free(&git__mwindow_mutex);
 }
 
 git_global_st *git__global_state(void)

src/hash/hash_win32.c

@@ -28,7 +28,8 @@ GIT_INLINE(int) hash_cng_prov_init(void)
 		return -1;
 
 	/* Load bcrypt.dll explicitly from the system directory */
-	if ((dll_path_len = GetSystemDirectory(dll_path, MAX_PATH)) == 0 || dll_path_len > MAX_PATH ||
+	if ((dll_path_len = GetSystemDirectory(dll_path, MAX_PATH)) == 0 ||
+		dll_path_len > MAX_PATH ||
 		StringCchCat(dll_path, MAX_PATH, "\\") < 0 ||
 		StringCchCat(dll_path, MAX_PATH, GIT_HASH_CNG_DLL_NAME) < 0 ||
 		(hash_prov.prov.cng.dll = LoadLibrary(dll_path)) == NULL)

src/refs.c

@@ -88,6 +88,17 @@ git_reference *git_reference__alloc(
 	return ref;
 }
 
+git_reference *git_reference__set_name(
+	git_reference *ref, const char *name)
+{
+	size_t namelen = strlen(name);
+	git_reference *rewrite =
+		git__realloc(ref, sizeof(git_reference) + namelen + 1);
+	if (rewrite != NULL)
+		memcpy(rewrite->name, name, namelen + 1);
+	return rewrite;
+}
+
 void git_reference_free(git_reference *reference)
 {
 	if (reference == NULL)

src/refs.h

@@ -61,6 +61,8 @@ struct git_reference {
 	char name[0];
 };
 
+git_reference *git_reference__set_name(git_reference *ref, const char *name);
+
 int git_reference__normalize_name_lax(char *buffer_out, size_t out_size, const char *name);
 int git_reference__normalize_name(git_buf *buf, const char *name, unsigned int flags);
 int git_reference__update_terminal(git_repository *repo, const char *ref_name, const git_oid *oid);

src/sortedcache.c

+#include "sortedcache.h"
+
+GIT__USE_STRMAP;
+
+int git_sortedcache_new(
+	git_sortedcache **out,
+	size_t item_path_offset,
+	git_sortedcache_free_item_fn free_item,
+	void *free_item_payload,
+	git_vector_cmp item_cmp,
+	const char *path)
+{
+	git_sortedcache *sc;
+	size_t pathlen;
+
+	pathlen = path ? strlen(path) : 0;
+
+	sc = git__calloc(sizeof(git_sortedcache) + pathlen + 1, 1);
+	GITERR_CHECK_ALLOC(sc);
+
+	if (git_pool_init(&sc->pool, 1, 0) < 0 ||
+		git_vector_init(&sc->items, 4, item_cmp) < 0 ||
+		(sc->map = git_strmap_alloc()) == NULL)
+		goto fail;
+
+	if (git_rwlock_init(&sc->lock)) {
+		giterr_set(GITERR_OS, "Failed to initialize lock");
+		goto fail;
+	}
+
+	sc->item_path_offset  = item_path_offset;
+	sc->free_item         = free_item;
+	sc->free_item_payload = free_item_payload;
+	GIT_REFCOUNT_INC(sc);
+	if (pathlen)
+		memcpy(sc->path, path, pathlen);
+
+	*out = sc;
+	return 0;
+
+fail:
+	if (sc->map)
+		git_strmap_free(sc->map);
+	git_vector_free(&sc->items);
+	git_pool_clear(&sc->pool);
+	git__free(sc);
+	return -1;
+}
+
+void git_sortedcache_incref(git_sortedcache *sc)
+{
+	GIT_REFCOUNT_INC(sc);
+}
+
+const char *git_sortedcache_path(git_sortedcache *sc)
+{
+	return sc->path;
+}
+
+static void sortedcache_clear(git_sortedcache *sc)
+{
+	git_strmap_clear(sc->map);
+
+	if (sc->free_item) {
+		size_t i;
+		void *item;
+
+		git_vector_foreach(&sc->items, i, item) {
+			sc->free_item(sc->free_item_payload, item);
+		}
+	}
+
+	git_vector_clear(&sc->items);
+
+	git_pool_clear(&sc->pool);
+}
+
+static void sortedcache_free(git_sortedcache *sc)
+{
+	/* acquire write lock to make sure everyone else is done */
+	if (git_sortedcache_wlock(sc) < 0)
+		return;
+
+	sortedcache_clear(sc);
+	git_vector_free(&sc->items);
+	git_strmap_free(sc->map);
+
+	git_sortedcache_wunlock(sc);
+
+	git_rwlock_free(&sc->lock);
+	git__free(sc);
+}
+
+void git_sortedcache_free(git_sortedcache *sc)
+{
+	if (!sc)
+		return;
+	GIT_REFCOUNT_DEC(sc, sortedcache_free);
+}
+
+static int sortedcache_copy_item(void *payload, void *tgt_item, void *src_item)
+{
+	git_sortedcache *sc = payload;
+	/* path will already have been copied by upsert */
+	memcpy(tgt_item, src_item, sc->item_path_offset);
+	return 0;
+}
+
+/* copy a sorted cache */
+int git_sortedcache_copy(
+	git_sortedcache **out,
+	git_sortedcache *src,
+	bool lock,
+	int (*copy_item)(void *payload, void *tgt_item, void *src_item),
+	void *payload)
+{
+	int error = 0;
+	git_sortedcache *tgt;
+	size_t i;
+	void *src_item, *tgt_item;
+
+	/* just use memcpy if no special copy fn is passed in */
+	if (!copy_item) {
+		copy_item = sortedcache_copy_item;
+		payload   = src;
+	}
+
+	if ((error = git_sortedcache_new(
+			&tgt, src->item_path_offset,
+			src->free_item, src->free_item_payload,
+			src->items._cmp, src->path)) < 0)
+		return error;
+
+	if (lock && git_sortedcache_rlock(src) < 0) {
+		git_sortedcache_free(tgt);
+		return -1;
+	}
+
+	git_vector_foreach(&src->items, i, src_item) {
+		char *path = ((char *)src_item) + src->item_path_offset;
+
+		if ((error = git_sortedcache_upsert(&tgt_item, tgt, path)) < 0 ||
+			(error = copy_item(payload, tgt_item, src_item)) < 0)
+			break;
+	}
+
+	if (lock)
+		git_sortedcache_runlock(src);
+	if (error)
+		git_sortedcache_free(tgt);
+
+	*out = !error ? tgt : NULL;
+
+	return error;
+}
+
+/* lock sortedcache while making modifications */
+int git_sortedcache_wlock(git_sortedcache *sc)
+{
+	GIT_UNUSED(sc); /* prevent warning when compiled w/o threads */
+
+	if (git_rwlock_wrlock(&sc->lock) < 0) {
+		giterr_set(GITERR_OS, "Unable to acquire write lock on cache");
+		return -1;
+	}
+	return 0;
+}
+
+/* unlock sorted cache when done with modifications */
+void git_sortedcache_wunlock(git_sortedcache *sc)
+{
+	git_vector_sort(&sc->items);
+	git_rwlock_wrunlock(&sc->lock);
+}
+
+/* lock sortedcache for read */
+int git_sortedcache_rlock(git_sortedcache *sc)
+{
+	GIT_UNUSED(sc); /* prevent warning when compiled w/o threads */
+
+	if (git_rwlock_rdlock(&sc->lock) < 0) {
+		giterr_set(GITERR_OS, "Unable to acquire read lock on cache");
+		return -1;
+	}
+	return 0;
+}
+
+/* unlock sorted cache when done reading */
+void git_sortedcache_runlock(git_sortedcache *sc)
+{
+	GIT_UNUSED(sc); /* prevent warning when compiled w/o threads */
+	git_rwlock_rdunlock(&sc->lock);
+}
+
+/* if the file has changed, lock cache and load file contents into buf;
+ * returns <0 on error, >0 if file has not changed
+ */
+int git_sortedcache_lockandload(git_sortedcache *sc, git_buf *buf)
+{
+	int error, fd;
+
+	if ((error = git_sortedcache_wlock(sc)) < 0)
+		return error;
+
+	if ((error = git_futils_filestamp_check(&sc->stamp, sc->path)) <= 0)
+		goto unlock;
+
+	if (!git__is_sizet(sc->stamp.size)) {
+		giterr_set(GITERR_INVALID, "Unable to load file larger than size_t");
+		error = -1;
+		goto unlock;
+	}
+
+	if ((fd = git_futils_open_ro(sc->path)) < 0) {
+		error = fd;
+		goto unlock;
+	}
+
+	if (buf)
+		error = git_futils_readbuffer_fd(buf, fd, (size_t)sc->stamp.size);
+
+	(void)p_close(fd);
+
+	if (error < 0)
+		goto unlock;
+
+	return 1; /* return 1 -> file needs reload and was successfully loaded */
+
+unlock:
+	git_sortedcache_wunlock(sc);
+	return error;
+}
+
+void git_sortedcache_updated(git_sortedcache *sc)
+{
+	 /* update filestamp to latest value */
+	if (git_futils_filestamp_check(&sc->stamp, sc->path) < 0)
+		giterr_clear();
+}
+
+/* release all items in sorted cache */
+int git_sortedcache_clear(git_sortedcache *sc, bool wlock)
+{
+	if (wlock && git_sortedcache_wlock(sc) < 0)
+		return -1;
+
+	sortedcache_clear(sc);
+
+	if (wlock)
+		git_sortedcache_wunlock(sc);
+
+	return 0;
+}
+
+/* find and/or insert item, returning pointer to item data */
+int git_sortedcache_upsert(void **out, git_sortedcache *sc, const char *key)
+{
+	int error = 0;
+	khiter_t pos;
+	void *item;
+	size_t keylen, itemlen;
+	char *item_key;
+
+	pos = git_strmap_lookup_index(sc->map, key);
+	if (git_strmap_valid_index(sc->map, pos)) {
+		item = git_strmap_value_at(sc->map, pos);
+		goto done;
+	}
+
+	keylen  = strlen(key);
+	itemlen = sc->item_path_offset + keylen + 1;
+	itemlen = (itemlen + 7) & ~7;
+
+	if ((item = git_pool_mallocz(&sc->pool, (uint32_t)itemlen)) == NULL) {
+		/* don't use GITERR_CHECK_ALLOC b/c of lock */
+		error = -1;
+		goto done;
+	}
+
+	/* one strange thing is that even if the vector or hash table insert
+	 * fail, there is no way to free the pool item so we just abandon it
+	 */
+
+	item_key = ((char *)item) + sc->item_path_offset;
+	memcpy(item_key, key, keylen);
+
+	pos = kh_put(str, sc->map, item_key, &error);
+	if (error < 0)
+		goto done;
+
+	if (!error)
+		kh_key(sc->map, pos) = item_key;
+	kh_val(sc->map, pos) = item;
+
+	error = git_vector_insert(&sc->items, item);
+	if (error < 0)
+		git_strmap_delete_at(sc->map, pos);
+
+done:
+	if (out)
+		*out = !error ? item : NULL;
+	return error;
+}
+
+/* lookup item by key */
+void *git_sortedcache_lookup(const git_sortedcache *sc, const char *key)
+{
+	khiter_t pos = git_strmap_lookup_index(sc->map, key);
+	if (git_strmap_valid_index(sc->map, pos))
+		return git_strmap_value_at(sc->map, pos);
+	return NULL;
+}
+
+/* find out how many items are in the cache */
+size_t git_sortedcache_entrycount(const git_sortedcache *sc)
+{
+	return git_vector_length(&sc->items);
+}
+
+/* lookup item by index */
+void *git_sortedcache_entry(git_sortedcache *sc, size_t pos)
+{
+	/* make sure the items are sorted so this gets the correct item */
+	if (!sc->items.sorted)
+		git_vector_sort(&sc->items);
+
+	return git_vector_get(&sc->items, pos);
+}
+
+/* helper struct so bsearch callback can know offset + key value for cmp */
+struct sortedcache_magic_key {
+	size_t offset;
+	const char *key;
+};
+
+static int sortedcache_magic_cmp(const void *key, const void *value)
+{
+	const struct sortedcache_magic_key *magic = key;
+	const char *value_key = ((const char *)value) + magic->offset;
+	return strcmp(magic->key, value_key);
+}
+
+/* lookup index of item by key */
+int git_sortedcache_lookup_index(
+	size_t *out, git_sortedcache *sc, const char *key)
+{
+	struct sortedcache_magic_key magic;
+
+	magic.offset = sc->item_path_offset;
+	magic.key    = key;
+
+	return git_vector_bsearch2(out, &sc->items, sortedcache_magic_cmp, &magic);
+}
+
+/* remove entry from cache */
+int git_sortedcache_remove(git_sortedcache *sc, size_t pos)
+{
+	char *item;
+	khiter_t mappos;
+
+	/* because of pool allocation, this can't actually remove the item,
+	 * but we can remove it from the items vector and the hash table.
+	 */
+
+	if ((item = git_vector_get(&sc->items, pos)) == NULL) {
+		giterr_set(GITERR_INVALID, "Removing item out of range");
+		return GIT_ENOTFOUND;
+	}
+
+	(void)git_vector_remove(&sc->items, pos);
+
+	mappos = git_strmap_lookup_index(sc->map, item + sc->item_path_offset);
+	git_strmap_delete_at(sc->map, mappos);
+
+	if (sc->free_item)
+		sc->free_item(sc->free_item_payload, item);
+
+	return 0;
+}
+

src/sortedcache.h

+/*
+ * Copyright (C) the libgit2 contributors. All rights reserved.
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#ifndef INCLUDE_sorted_cache_h__
+#define INCLUDE_sorted_cache_h__
+
+#include "util.h"
+#include "fileops.h"
+#include "vector.h"
+#include "thread-utils.h"
+#include "pool.h"
+#include "strmap.h"
+
+/*
+ * The purpose of this data structure is to cache the parsed contents of a
+ * file (a.k.a. the backing file) where each item in the file can be
+ * identified by a key string and you want to both look them up by name
+ * and traverse them in sorted order.  Each item is assumed to itself end
+ * in a GIT_FLEX_ARRAY.
+ */
+
+typedef void (*git_sortedcache_free_item_fn)(void *payload, void *item);
+
+typedef struct {
+	git_refcount rc;
+	git_rwlock   lock;
+	size_t       item_path_offset;
+	git_sortedcache_free_item_fn free_item;
+	void         *free_item_payload;
+	git_pool     pool;
+	git_vector   items;
+	git_strmap   *map;
+	git_futils_filestamp stamp;
+	char         path[GIT_FLEX_ARRAY];
+} git_sortedcache;
+
+/* Create a new sortedcache
+ *
+ * Even though every sortedcache stores items with a GIT_FLEX_ARRAY at
+ * the end containing their key string, you have to provide the item_cmp
+ * sorting function because the sorting function doesn't get a payload
+ * and therefore can't know the offset to the item key string. :-(
+ *
+ * @param out The allocated git_sortedcache
+ * @param item_path_offset Offset to the GIT_FLEX_ARRAY item key in the
+ *        struct - use offsetof(struct mine, key-field) to get this
+ * @param free_item Optional callback to free each item
+ * @param free_item_payload Optional payload passed to free_item callback
+ * @param item_cmp Compare the keys of two items
+ * @param path The path to the backing store file for this cache; this
+ *        may be NULL.  The cache makes it easy to load this and check
+ *        if it has been modified since the last load and/or write.
+ */
+int git_sortedcache_new(
+	git_sortedcache **out,
+	size_t item_path_offset, /* use offsetof(struct, path-field) macro */
+	git_sortedcache_free_item_fn free_item,
+	void *free_item_payload,
+	git_vector_cmp item_cmp,
+	const char *path);
+
+/* Copy a sorted cache
+ *
+ * - `copy_item` can be NULL to just use memcpy
+ * - if `lock`, grabs read lock on `src` during copy and releases after
+ */
+int git_sortedcache_copy(
+	git_sortedcache **out,
+	git_sortedcache *src,
+	bool lock,
+	int (*copy_item)(void *payload, void *tgt_item, void *src_item),
+	void *payload);
+
+/* Free sorted cache (first calling `free_item` callbacks)
+ *
+ * Don't call on a locked collection - it may acquire a write lock
+ */
+void git_sortedcache_free(git_sortedcache *sc);
+
+/* Increment reference count - balance with call to free */
+void git_sortedcache_incref(git_sortedcache *sc);
+
+/* Get the pathname associated with this cache at creation time */
+const char *git_sortedcache_path(git_sortedcache *sc);
+
+/*
+ * CACHE WRITE FUNCTIONS
+ *
+ * The following functions require you to have a writer lock to make the
+ * modification.  Some of the functions take a `wlock` parameter and
+ * will optionally lock and unlock for you if that is passed as true.
+ *
+ */
+
+/* Lock sortedcache for write */
+int git_sortedcache_wlock(git_sortedcache *sc);
+
+/* Unlock sorted cache when done with write */
+void git_sortedcache_wunlock(git_sortedcache *sc);
+
+/* Lock cache and load backing file into a buffer.
+ *
+ * This grabs a write lock on the cache then looks at the modification
+ * time and size of the file on disk.
+ *
+ * If the file appears to have changed, this loads the file contents into
+ * the buffer and returns a positive value leaving the cache locked - the
+ * caller should parse the file content, update the cache as needed, then
+ * release the lock.  NOTE: In this case, the caller MUST unlock the cache.
+ *
+ * If the file appears to be unchanged, then this automatically releases
+ * the lock on the cache, clears the buffer, and returns 0.
+ *
+ * @return 0 if up-to-date, 1 if out-of-date, <0 on error
+ */
+int git_sortedcache_lockandload(git_sortedcache *sc, git_buf *buf);
+
+/* Refresh file timestamp after write completes
+ * You should already be holding the write lock when you call this.
+ */
+void git_sortedcache_updated(git_sortedcache *sc);
+
+/* Release all items in sorted cache
+ *
+ * If `wlock` is true, grabs write lock and releases when done, otherwise
+ * you should already be holding a write lock when you call this.
+ */
+int git_sortedcache_clear(git_sortedcache *sc, bool wlock);
+
+/* Find and/or insert item, returning pointer to item data.
+ * You should already be holding the write lock when you call this.
+ */
+int git_sortedcache_upsert(
+	void **out, git_sortedcache *sc, const char *key);
+
+/* Removes entry at pos from cache
+ * You should already be holding the write lock when you call this.
+ */
+int git_sortedcache_remove(git_sortedcache *sc, size_t pos);
+
+/*
+ * CACHE READ FUNCTIONS
+ *
+ * The following functions access items in the cache.  To prevent the
+ * results from being invalidated before they can be used, you should be
+ * holding either a read lock or a write lock when using these functions.
+ *
+ */
+
+/* Lock sortedcache for read */
+int git_sortedcache_rlock(git_sortedcache *sc);
+
+/* Unlock sorted cache when done with read */
+void git_sortedcache_runlock(git_sortedcache *sc);
+
+/* Lookup item by key - returns NULL if not found */
+void *git_sortedcache_lookup(const git_sortedcache *sc, const char *key);
+
+/* Get how many items are in the cache
+ *
+ * You can call this function without holding a lock, but be aware
+ * that it may change before you use it.
+ */
+size_t git_sortedcache_entrycount(const git_sortedcache *sc);
+
+/* Lookup item by index - returns NULL if out of range */
+void *git_sortedcache_entry(git_sortedcache *sc, size_t pos);
+
+/* Lookup index of item by key - returns GIT_ENOTFOUND if not found */
+int git_sortedcache_lookup_index(
+	size_t *out, git_sortedcache *sc, const char *key);
+
+#endif

src/thread-utils.h

@@ -41,7 +41,8 @@ typedef git_atomic git_atomic_ssize;
 #ifdef GIT_THREADS
 
 #define git_thread pthread_t
-#define git_thread_create(thread, attr, start_routine, arg) pthread_create(thread, attr, start_routine, arg)
+#define git_thread_create(thread, attr, start_routine, arg) \
+	pthread_create(thread, attr, start_routine, arg)
 #define git_thread_kill(thread) pthread_cancel(thread)
 #define git_thread_exit(status)	pthread_exit(status)
 #define git_thread_join(id, status) pthread_join(id, status)
@@ -61,6 +62,30 @@ typedef git_atomic git_atomic_ssize;
 #define git_cond_signal(c)	pthread_cond_signal(c)
 #define git_cond_broadcast(c)	pthread_cond_broadcast(c)
 
+/* Pthread (-ish) rwlock
+ *
+ * This differs from normal pthreads rwlocks in two ways:
+ * 1. Separate APIs for releasing read locks and write locks (as
+ *    opposed to the pure POSIX API which only has one unlock fn)
+ * 2. You should not use recursive read locks (i.e. grabbing a read
+ *    lock in a thread that already holds a read lock) because the
+ *    Windows implementation doesn't support it
+ */
+#define git_rwlock pthread_rwlock_t
+#define git_rwlock_init(a)		pthread_rwlock_init(a, NULL)
+#define git_rwlock_rdlock(a)	pthread_rwlock_rdlock(a)
+#define git_rwlock_rdunlock(a)	pthread_rwlock_rdunlock(a)
+#define git_rwlock_wrlock(a)	pthread_rwlock_wrlock(a)
+#define git_rwlock_wrunlock(a)	pthread_rwlock_wrunlock(a)
+#define git_rwlock_free(a)		pthread_rwlock_destroy(a)
+#define GIT_RWLOCK_STATIC_INIT	PTHREAD_RWLOCK_INITIALIZER
+
+#ifndef GIT_WIN32
+#define pthread_rwlock_rdunlock pthread_rwlock_unlock
+#define pthread_rwlock_wrunlock pthread_rwlock_unlock
+#endif
+
+
 GIT_INLINE(void) git_atomic_set(git_atomic *a, int val)
 {
 #if defined(GIT_WIN32)
@@ -147,7 +172,7 @@ GIT_INLINE(int64_t) git_atomic64_add(git_atomic64 *a, int64_t addend)
 #else
 
 #define git_thread unsigned int
-#define git_thread_create(thread, attr, start_routine, arg) (void)0
+#define git_thread_create(thread, attr, start_routine, arg) 0
 #define git_thread_kill(thread) (void)0
 #define git_thread_exit(status) (void)0
 #define git_thread_join(id, status) (void)0
@@ -167,6 +192,17 @@ GIT_INLINE(int64_t) git_atomic64_add(git_atomic64 *a, int64_t addend)
 #define git_cond_signal(c) (void)0
 #define git_cond_broadcast(c) (void)0
 
+/* Pthreads rwlock */
+#define git_rwlock unsigned int
+#define git_rwlock_init(a)		0
+#define git_rwlock_rdlock(a)	0
+#define git_rwlock_rdunlock(a)	(void)0
+#define git_rwlock_wrlock(a)	0
+#define git_rwlock_wrunlock(a)	(void)0
+#define git_rwlock_free(a)		(void)0
+#define GIT_RWLOCK_STATIC_INIT	0
+
+
 GIT_INLINE(void) git_atomic_set(git_atomic *a, int val)
 {
 	a->val = val;

src/vector.h

@@ -55,6 +55,11 @@ GIT_INLINE(void *) git_vector_get(const git_vector *v, size_t position)
 
 #define GIT_VECTOR_GET(V,I) ((I) < (V)->length ? (V)->contents[(I)] : NULL)
 
+GIT_INLINE(size_t) git_vector_length(const git_vector *v)
+{
+	return v->length;
+}
+
 GIT_INLINE(void *) git_vector_last(const git_vector *v)
 {
 	return (v->length > 0) ? git_vector_get(v, v->length - 1) : NULL;

src/win32/pthread.c

@@ -127,9 +127,10 @@ int pthread_cond_signal(pthread_cond_t *cond)
 	return 0;
 }
 
-/* pthread_cond_broadcast is not implemented because doing so with just Win32 events
- * is quite complicated, and no caller in libgit2 uses it yet. */
-
+/* pthread_cond_broadcast is not implemented because doing so with just
+ * Win32 events is quite complicated, and no caller in libgit2 uses it
+ * yet.
+ */
 int pthread_num_processors_np(void)
 {
 	DWORD_PTR p, s;
@@ -142,3 +143,111 @@ int pthread_num_processors_np(void)
 	return n ? n : 1;
 }
 
+
+static HINSTANCE win32_kernel32_dll;
+
+typedef void (WINAPI *win32_srwlock_fn)(GIT_SRWLOCK *);
+
+static win32_srwlock_fn win32_srwlock_initialize;
+static win32_srwlock_fn win32_srwlock_acquire_shared;
+static win32_srwlock_fn win32_srwlock_release_shared;
+static win32_srwlock_fn win32_srwlock_acquire_exclusive;
+static win32_srwlock_fn win32_srwlock_release_exclusive;
+
+int pthread_rwlock_init(
+	pthread_rwlock_t *GIT_RESTRICT lock,
+	const pthread_rwlockattr_t *GIT_RESTRICT attr)
+{
+	(void)attr;
+
+	if (win32_srwlock_initialize)
+		win32_srwlock_initialize(&lock->native.srwl);
+	else
+		InitializeCriticalSection(&lock->native.csec);
+
+	return 0;
+}
+
+int pthread_rwlock_rdlock(pthread_rwlock_t *lock)
+{
+	if (win32_srwlock_acquire_shared)
+		win32_srwlock_acquire_shared(&lock->native.srwl);
+	else
+		EnterCriticalSection(&lock->native.csec);
+
+	return 0;
+}
+
+int pthread_rwlock_rdunlock(pthread_rwlock_t *lock)
+{
+	if (win32_srwlock_release_shared)
+		win32_srwlock_release_shared(&lock->native.srwl);
+	else
+		LeaveCriticalSection(&lock->native.csec);
+
+	return 0;
+}
+
+int pthread_rwlock_wrlock(pthread_rwlock_t *lock)
+{
+	if (win32_srwlock_acquire_exclusive)
+		win32_srwlock_acquire_exclusive(&lock->native.srwl);
+	else
+		EnterCriticalSection(&lock->native.csec);
+
+	return 0;
+}
+
+int pthread_rwlock_wrunlock(pthread_rwlock_t *lock)
+{
+	if (win32_srwlock_release_exclusive)
+		win32_srwlock_release_exclusive(&lock->native.srwl);
+	else
+		LeaveCriticalSection(&lock->native.csec);
+
+	return 0;
+}
+
+int pthread_rwlock_destroy(pthread_rwlock_t *lock)
+{
+	if (!win32_srwlock_initialize)
+		DeleteCriticalSection(&lock->native.csec);
+	git__memzero(lock, sizeof(*lock));
+	return 0;
+}
+
+
+int win32_pthread_initialize(void)
+{
+	if (win32_kernel32_dll)
+		return 0;
+
+	win32_kernel32_dll = LoadLibrary("Kernel32.dll");
+	if (!win32_kernel32_dll) {
+		giterr_set(GITERR_OS, "Could not load Kernel32.dll!");
+		return -1;
+	}
+
+	win32_srwlock_initialize = (win32_srwlock_fn)
+		GetProcAddress(win32_kernel32_dll, "InitializeSRWLock");
+	win32_srwlock_acquire_shared = (win32_srwlock_fn)
+		GetProcAddress(win32_kernel32_dll, "AcquireSRWLockShared");
+	win32_srwlock_release_shared = (win32_srwlock_fn)
+		GetProcAddress(win32_kernel32_dll, "ReleaseSRWLockShared");
+	win32_srwlock_acquire_exclusive = (win32_srwlock_fn)
+		GetProcAddress(win32_kernel32_dll, "AcquireSRWLockExclusive");
+	win32_srwlock_release_exclusive = (win32_srwlock_fn)
+		GetProcAddress(win32_kernel32_dll, "ReleaseSRWLockExclusive");
+
+	return 0;
+}
+
+int win32_pthread_shutdown(void)
+{
+	if (win32_kernel32_dll) {
+		FreeLibrary(win32_kernel32_dll);
+		win32_kernel32_dll = NULL;
+	}
+
+	return 0;
+}

src/win32/pthread.h

@@ -19,22 +19,34 @@
 typedef int pthread_mutexattr_t;
 typedef int pthread_condattr_t;
 typedef int pthread_attr_t;
+typedef int pthread_rwlockattr_t;
+
 typedef CRITICAL_SECTION pthread_mutex_t;
 typedef HANDLE pthread_t;
 typedef HANDLE pthread_cond_t;
 
-#define PTHREAD_MUTEX_INITIALIZER {(void*)-1};
+typedef struct { void *Ptr; } GIT_SRWLOCK;
+
+typedef struct {
+	union {
+		GIT_SRWLOCK srwl;
+		CRITICAL_SECTION csec;
+	} native;
+} pthread_rwlock_t;
+
+#define PTHREAD_MUTEX_INITIALIZER  {(void*)-1}
 
 int pthread_create(
-	pthread_t *GIT_RESTRICT,
-	const pthread_attr_t *GIT_RESTRICT,
+	pthread_t *GIT_RESTRICT thread,
+	const pthread_attr_t *GIT_RESTRICT attr,
 	void *(*start_routine)(void*),
-	void *__restrict);
+	void *GIT_RESTRICT arg);
 
 int pthread_join(pthread_t, void **);
 
 int pthread_mutex_init(
-	pthread_mutex_t *GIT_RESTRICT, const pthread_mutexattr_t *GIT_RESTRICT);
+	pthread_mutex_t *GIT_RESTRICT mutex,
+	const pthread_mutexattr_t *GIT_RESTRICT mutexattr);
 int pthread_mutex_destroy(pthread_mutex_t *);
 int pthread_mutex_lock(pthread_mutex_t *);
 int pthread_mutex_unlock(pthread_mutex_t *);
@@ -47,4 +59,16 @@ int pthread_cond_signal(pthread_cond_t *);
 
 int pthread_num_processors_np(void);
 
+int pthread_rwlock_init(
+	pthread_rwlock_t *GIT_RESTRICT lock,
+	const pthread_rwlockattr_t *GIT_RESTRICT attr);
+int pthread_rwlock_rdlock(pthread_rwlock_t *);
+int pthread_rwlock_rdunlock(pthread_rwlock_t *);
+int pthread_rwlock_wrlock(pthread_rwlock_t *);
+int pthread_rwlock_wrunlock(pthread_rwlock_t *);
+int pthread_rwlock_destroy(pthread_rwlock_t *);
+
+extern int win32_pthread_initialize(void);
+extern int win32_pthread_shutdown(void);
+
 #endif

tests-clar/refs/pack.c

@@ -32,7 +32,7 @@ static void packall(void)
 
 void test_refs_pack__empty(void)
 {
-   // create a packfile for an empty folder
+	/* create a packfile for an empty folder */
 	git_buf temp_path = GIT_BUF_INIT;
 
 	cl_git_pass(git_buf_join_n(&temp_path, '/', 3, git_repository_path(g_repo), GIT_REFS_HEADS_DIR, "empty_dir"));
@@ -44,7 +44,7 @@ void test_refs_pack__empty(void)
 
 void test_refs_pack__loose(void)
 {
-   // create a packfile from all the loose rn a repo
+	/* create a packfile from all the loose refs in a repo */
 	git_reference *reference;
 	git_buf temp_path = GIT_BUF_INIT;
 
@@ -77,3 +77,29 @@ void test_refs_pack__loose(void)
 	git_reference_free(reference);
 	git_buf_free(&temp_path);
 }
+
+void test_refs_pack__symbolic(void)
+{
+	/* create a packfile from loose refs skipping symbolic refs */
+	int i;
+	git_oid head;
+	git_reference *ref;
+	char name[128];
+
+	cl_git_pass(git_reference_name_to_id(&head, g_repo, "HEAD"));
+
+	/* make a bunch of references */
+
+	for (i = 0; i < 100; ++i) {
+		snprintf(name, sizeof(name), "refs/heads/symbolic-%03d", i);
+		cl_git_pass(git_reference_symbolic_create(
+			&ref, g_repo, name, "refs/heads/master", 0));
+		git_reference_free(ref);
+
+		snprintf(name, sizeof(name), "refs/heads/direct-%03d", i);
+		cl_git_pass(git_reference_create(&ref, g_repo, name, &head, 0));
+		git_reference_free(ref);
+	}
+
+	packall();
+}

tests-clar/stress/diff.c

@@ -15,7 +15,7 @@ void test_stress_diff__cleanup(void)
 #define ANOTHER_POEM \
 "OH, glorious are the guarded heights\nWhere guardian souls abide—\nSelf-exiled from our gross delights—\nAbove, beyond, outside:\nAn ampler arc their spirit swings—\nCommands a juster view—\nWe have their word for all these things,\nNo doubt their words are true.\n\nYet we, the bond slaves of our day,\nWhom dirt and danger press—\nCo-heirs of insolence, delay,\nAnd leagued unfaithfulness—\nSuch is our need must seek indeed\nAnd, having found, engage\nThe men who merely do the work\nFor which they draw the wage.\n\nFrom forge and farm and mine and bench,\nDeck, altar, outpost lone—\nMill, school, battalion, counter, trench,\nRail, senate, sheepfold, throne—\nCreation's cry goes up on high\nFrom age to cheated age:\n\"Send us the men who do the work\n\"For which they draw the wage!\"\n"
 
-static void test_with_many(size_t expected_new)
+static void test_with_many(int expected_new)
 {
 	git_index *index;
 	git_tree *tree, *new_tree;

tests-clar/threads/refdb.c

+#include "clar_libgit2.h"
+#include "git2/refdb.h"
+#include "refdb.h"
+
+static git_repository *g_repo;
+static int g_expected = 0;
+
+void test_threads_refdb__initialize(void)
+{
+	g_repo = NULL;
+}
+
+void test_threads_refdb__cleanup(void)
+{
+	cl_git_sandbox_cleanup();
+	g_repo = NULL;
+}
+
+#define REPEAT 20
+#define THREADS 20
+
+static void *iterate_refs(void *arg)
+{
+	git_reference_iterator *i;
+	git_reference *ref;
+	int count = 0;
+
+	cl_git_pass(git_reference_iterator_new(&i, g_repo));
+
+	for (count = 0; !git_reference_next(&ref, i); ++count) {
+		cl_assert(ref != NULL);
+		git_reference_free(ref);
+	}
+
+	if (g_expected > 0)
+		cl_assert_equal_i(g_expected, count);
+
+	git_reference_iterator_free(i);
+
+	return arg;
+}
+
+void test_threads_refdb__iterator(void)
+{
+	int r, t;
+	git_thread th[THREADS];
+	int id[THREADS];
+	git_oid head;
+	git_reference *ref;
+	char name[128];
+	git_refdb *refdb;
+
+	g_repo = cl_git_sandbox_init("testrepo2");
+
+	cl_git_pass(git_reference_name_to_id(&head, g_repo, "HEAD"));
+
+	/* make a bunch of references */
+
+	for (r = 0; r < 200; ++r) {
+		snprintf(name, sizeof(name), "refs/heads/direct-%03d", r);
+		cl_git_pass(git_reference_create(&ref, g_repo, name, &head, 0));
+		git_reference_free(ref);
+	}
+
+	cl_git_pass(git_repository_refdb(&refdb, g_repo));
+	cl_git_pass(git_refdb_compress(refdb));
+	git_refdb_free(refdb);
+
+	g_expected = 206;
+
+	for (r = 0; r < REPEAT; ++r) {
+		g_repo = cl_git_sandbox_reopen(); /* reopen to flush caches */
+
+		for (t = 0; t < THREADS; ++t) {
+			id[t] = t;
+#ifdef GIT_THREADS
+			cl_git_pass(git_thread_create(&th[t], NULL, iterate_refs, &id[t]));
+#else
+			th[t] = t;
+			iterate_refs(&id[t]);
+#endif
+		}
+
+#ifdef GIT_THREADS
+		for (t = 0; t < THREADS; ++t) {
+			cl_git_pass(git_thread_join(th[t], NULL));
+		}
+#endif
+
+		memset(th, 0, sizeof(th));
+	}
+}
+
+static void *create_refs(void *arg)
+{
+	int *id = arg, i;
+	git_oid head;
+	char name[128];
+	git_reference *ref[10];
+
+	cl_git_pass(git_reference_name_to_id(&head, g_repo, "HEAD"));
+
+	for (i = 0; i < 10; ++i) {
+		snprintf(name, sizeof(name), "refs/heads/thread-%03d-%02d", *id, i);
+		cl_git_pass(git_reference_create(&ref[i], g_repo, name, &head, 0));
+
+		if (i == 5) {
+			git_refdb *refdb;
+			cl_git_pass(git_repository_refdb(&refdb, g_repo));
+			cl_git_pass(git_refdb_compress(refdb));
+			git_refdb_free(refdb);
+		}
+	}
+
+	for (i = 0; i < 10; ++i)
+		git_reference_free(ref[i]);
+
+	return arg;
+}
+
+static void *delete_refs(void *arg)
+{
+	int *id = arg, i;
+	git_reference *ref;
+	char name[128];
+
+	for (i = 0; i < 10; ++i) {
+		snprintf(
+			name, sizeof(name), "refs/heads/thread-%03d-%02d", (*id) & ~0x3, i);
+
+		if (!git_reference_lookup(&ref, g_repo, name)) {
+			cl_git_pass(git_reference_delete(ref));
+			git_reference_delete(ref);
+		}
+
+		if (i == 5) {
+			git_refdb *refdb;
+			cl_git_pass(git_repository_refdb(&refdb, g_repo));
+			cl_git_pass(git_refdb_compress(refdb));
+			git_refdb_free(refdb);
+		}
+	}
+
+	return arg;
+}
+
+void test_threads_refdb__edit_while_iterate(void)
+{
+	int r, t;
+	git_thread th[THREADS];
+	int id[THREADS];
+	git_oid head;
+	git_reference *ref;
+	char name[128];
+	git_refdb *refdb;
+
+	g_repo = cl_git_sandbox_init("testrepo2");
+
+	cl_git_pass(git_reference_name_to_id(&head, g_repo, "HEAD"));
+
+	/* make a bunch of references */
+
+	for (r = 0; r < 50; ++r) {
+		snprintf(name, sizeof(name), "refs/heads/starter-%03d", r);
+		cl_git_pass(git_reference_create(&ref, g_repo, name, &head, 0));
+		git_reference_free(ref);
+	}
+
+	cl_git_pass(git_repository_refdb(&refdb, g_repo));
+	cl_git_pass(git_refdb_compress(refdb));
+	git_refdb_free(refdb);
+
+	g_expected = -1;
+
+	g_repo = cl_git_sandbox_reopen(); /* reopen to flush caches */
+
+	for (t = 0; t < THREADS; ++t) {
+		void *(*fn)(void *arg);
+
+		switch (t & 0x3) {
+		case 0:  fn = create_refs;  break;
+		case 1:  fn = delete_refs;  break;
+		default: fn = iterate_refs; break;
+		}
+
+		id[t] = t;
+#ifdef GIT_THREADS
+		cl_git_pass(git_thread_create(&th[t], NULL, fn, &id[t]));
+#else
+		th[t] = t;
+		fn(&id[t]);
+#endif
+	}
+
+#ifdef GIT_THREADS
+	for (t = 0; t < THREADS; ++t) {
+		cl_git_pass(git_thread_join(th[t], NULL));
+	}
+
+	memset(th, 0, sizeof(th));
+
+	for (t = 0; t < THREADS; ++t) {
+		id[t] = t;
+		cl_git_pass(git_thread_create(&th[t], NULL, iterate_refs, &id[t]));
+	}
+
+	for (t = 0; t < THREADS; ++t) {
+		cl_git_pass(git_thread_join(th[t], NULL));
+	}
+#endif
+}