Merge pull request #1454 from libgit2/vmg/new-cache

New caching

include/git2/common.h

@@ -131,8 +131,9 @@ enum {
 	GIT_OPT_SET_MWINDOW_MAPPED_LIMIT,
 	GIT_OPT_GET_SEARCH_PATH,
 	GIT_OPT_SET_SEARCH_PATH,
-	GIT_OPT_GET_ODB_CACHE_SIZE,
-	GIT_OPT_SET_ODB_CACHE_SIZE,
+	GIT_OPT_SET_CACHE_OBJECT_LIMIT,
+	GIT_OPT_SET_CACHE_MAX_SIZE,
+	GIT_OPT_ENABLE_CACHING
 };
 
 /**
@@ -169,15 +170,6 @@ enum {
  *		- `level` must be GIT_CONFIG_LEVEL_SYSTEM, GIT_CONFIG_LEVEL_GLOBAL,
  *		  or GIT_CONFIG_LEVEL_XDG.
  *
- *	opts(GIT_OPT_GET_ODB_CACHE_SIZE):
- *		Get the size of the libgit2 odb cache.
- *
- *	opts(GIT_OPT_SET_ODB_CACHE_SIZE):
- *		Set the size of the of the libgit2 odb cache. This needs
- *		to be done before git_repository_open is called, since
- *		git_repository_open initializes the odb layer. Defaults
- *		to 128.
- *
  * @param option Option key
  * @param ... value to set the option
  * @return 0 on success, <0 on failure

src/blob.c

@@ -18,32 +18,34 @@
 const void *git_blob_rawcontent(const git_blob *blob)
 {
 	assert(blob);
-	return blob->odb_object->raw.data;
+	return git_odb_object_data(blob->odb_object);
 }
 
 git_off_t git_blob_rawsize(const git_blob *blob)
 {
 	assert(blob);
-	return (git_off_t)blob->odb_object->raw.len;
+	return (git_off_t)git_odb_object_size(blob->odb_object);
 }
 
 int git_blob__getbuf(git_buf *buffer, git_blob *blob)
 {
 	return git_buf_set(
-		buffer, blob->odb_object->raw.data, blob->odb_object->raw.len);
+		buffer,
+		git_odb_object_data(blob->odb_object),
+		git_odb_object_size(blob->odb_object));
 }
 
-void git_blob__free(git_blob *blob)
+void git_blob__free(void *blob)
 {
-	git_odb_object_free(blob->odb_object);
+	git_odb_object_free(((git_blob *)blob)->odb_object);
 	git__free(blob);
 }
 
-int git_blob__parse(git_blob *blob, git_odb_object *odb_obj)
+int git_blob__parse(void *blob, git_odb_object *odb_obj)
 {
 	assert(blob);
 	git_cached_obj_incref((git_cached_obj *)odb_obj);
-	blob->odb_object = odb_obj;
+	((git_blob *)blob)->odb_object = odb_obj;
 	return 0;
 }
 
@@ -315,8 +317,8 @@ int git_blob_is_binary(git_blob *blob)
 
 	assert(blob);
 
-	content.ptr = blob->odb_object->raw.data;
-	content.size = min(blob->odb_object->raw.len, 4000);
+	content.ptr = blob->odb_object->buffer;
+	content.size = min(blob->odb_object->cached.size, 4000);
 
 	return git_buf_text_is_binary(&content);
 }

src/blob.h

@@ -17,8 +17,8 @@ struct git_blob {
 	git_odb_object *odb_object;
 };
 
-void git_blob__free(git_blob *blob);
-int git_blob__parse(git_blob *blob, git_odb_object *obj);
+void git_blob__free(void *blob);
+int git_blob__parse(void *blob, git_odb_object *obj);
 int git_blob__getbuf(git_buf *buffer, git_blob *blob);
 
 #endif

src/cache.c

@@ -11,100 +11,251 @@
 #include "thread-utils.h"
 #include "util.h"
 #include "cache.h"
+#include "odb.h"
+#include "object.h"
 #include "git2/oid.h"
 
-int git_cache_init(git_cache *cache, size_t size, git_cached_obj_freeptr free_ptr)
+GIT__USE_OIDMAP
+
+bool git_cache__enabled = true;
+size_t git_cache__max_storage = (4 * 1024 * 1024);
+
+static size_t git_cache__max_object_size[8] = {
+	0,     /* GIT_OBJ__EXT1 */
+	4096,  /* GIT_OBJ_COMMIT */
+	4096,  /* GIT_OBJ_TREE */
+	0,     /* GIT_OBJ_BLOB */
+	4096,  /* GIT_OBJ_TAG */
+	0,     /* GIT_OBJ__EXT2 */
+	0,     /* GIT_OBJ_OFS_DELTA */
+	0      /* GIT_OBJ_REF_DELTA */
+};
+
+int git_cache_set_max_object_size(git_otype type, size_t size)
+{
+	if (type < 0 || (size_t)type >= ARRAY_SIZE(git_cache__max_object_size)) {
+		giterr_set(GITERR_INVALID, "type out of range");
+		return -1;
+	}
+
+	git_cache__max_object_size[type] = size;
+	return 0;
+}
+
+void git_cache_dump_stats(git_cache *cache)
 {
-	if (size < 8)
-		size = 8;
-	size = git__size_t_powerof2(size);
+	git_cached_obj *object;
 
-	cache->size_mask = size - 1;
-	cache->lru_count = 0;
-	cache->free_obj = free_ptr;
+	if (kh_size(cache->map) == 0)
+		return;
 
-	git_mutex_init(&cache->lock);
+	printf("Cache %p: %d items cached, %d bytes\n",
+		cache, kh_size(cache->map), (int)cache->used_memory);
 
-	cache->nodes = git__malloc(size * sizeof(git_cached_obj *));
-	GITERR_CHECK_ALLOC(cache->nodes);
+	kh_foreach_value(cache->map, object, {
+		char oid_str[9];
+		printf(" %s%c %s (%d)\n",
+			git_object_type2string(object->type),
+			object->flags == GIT_CACHE_STORE_PARSED ? '*' : ' ',
+			git_oid_tostr(oid_str, sizeof(oid_str), &object->oid),
+			(int)object->size
+		);
+	});
+}
 
-	memset(cache->nodes, 0x0, size * sizeof(git_cached_obj *));
+int git_cache_init(git_cache *cache)
+{
+	cache->used_memory = 0;
+	cache->map = git_oidmap_alloc();
+	git_mutex_init(&cache->lock);
 	return 0;
 }
 
+/* called with lock */
+static void clear_cache(git_cache *cache)
+{
+	git_cached_obj *evict = NULL;
+
+	kh_foreach_value(cache->map, evict, {
+		git_cached_obj_decref(evict);
+	});
+
+	kh_clear(oid, cache->map);
+	cache->used_memory = 0;
+}
+
+void git_cache_clear(git_cache *cache)
+{
+	if (git_mutex_lock(&cache->lock) < 0)
+		return;
+
+	clear_cache(cache);
+
+	git_mutex_unlock(&cache->lock);
+}
+
 void git_cache_free(git_cache *cache)
 {
-	size_t i;
+	git_cache_clear(cache);
+
+	git_oidmap_free(cache->map);
+	git_mutex_free(&cache->lock);
+}
+
+/* Called with lock */
+static void cache_evict_entries(git_cache *cache)
+{
+	uint32_t seed = rand();
+	size_t evict_count = 8;
 
-	for (i = 0; i < (cache->size_mask + 1); ++i) {
-		if (cache->nodes[i] != NULL)
-			git_cached_obj_decref(cache->nodes[i], cache->free_obj);
+	/* do not infinite loop if there's not enough entries to evict  */
+	if (evict_count > kh_size(cache->map)) {
+		clear_cache(cache);
+		return;
 	}
 
-	git_mutex_free(&cache->lock);
-	git__free(cache->nodes);
+	while (evict_count > 0) {
+		khiter_t pos = seed++ % kh_end(cache->map);
+
+		if (kh_exist(cache->map, pos)) {
+			git_cached_obj *evict = kh_val(cache->map, pos);
+
+			evict_count--;
+			cache->used_memory -= evict->size;
+			git_cached_obj_decref(evict);
+
+			kh_del(oid, cache->map, pos);
+		}
+	}
 }
 
-void *git_cache_get(git_cache *cache, const git_oid *oid)
+static bool cache_should_store(git_otype object_type, size_t object_size)
 {
-	uint32_t hash;
-	git_cached_obj *node = NULL, *result = NULL;
+	size_t max_size = git_cache__max_object_size[object_type];
+	return git_cache__enabled && object_size < max_size;
+}
 
-	memcpy(&hash, oid->id, sizeof(hash));
+static void *cache_get(git_cache *cache, const git_oid *oid, unsigned int flags)
+{
+	khiter_t pos;
+	git_cached_obj *entry = NULL;
 
-	if (git_mutex_lock(&cache->lock)) {
-		giterr_set(GITERR_THREAD, "unable to lock cache mutex");
+	if (!git_cache__enabled || git_mutex_lock(&cache->lock) < 0)
 		return NULL;
-	}
 
-	{
-		node = cache->nodes[hash & cache->size_mask];
+	pos = kh_get(oid, cache->map, oid);
+	if (pos != kh_end(cache->map)) {
+		entry = kh_val(cache->map, pos);
 
-		if (node != NULL && git_oid_cmp(&node->oid, oid) == 0) {
-			git_cached_obj_incref(node);
-			result = node;
+		if (flags && entry->flags != flags) {
+			entry = NULL;
+		} else {
+			git_cached_obj_incref(entry);
 		}
 	}
+
 	git_mutex_unlock(&cache->lock);
 
-	return result;
+	return entry;
 }
 
-void *git_cache_try_store(git_cache *cache, void *_entry)
+static void *cache_store(git_cache *cache, git_cached_obj *entry)
 {
-	git_cached_obj *entry = _entry;
-	uint32_t hash;
+	khiter_t pos;
 
-	memcpy(&hash, &entry->oid, sizeof(uint32_t));
+	git_cached_obj_incref(entry);
 
-	if (git_mutex_lock(&cache->lock)) {
-		giterr_set(GITERR_THREAD, "unable to lock cache mutex");
-		return NULL;
-	}
+	if (!cache_should_store(entry->type, entry->size))
+		return entry;
 
-	{
-		git_cached_obj *node = cache->nodes[hash & cache->size_mask];
+	if (git_mutex_lock(&cache->lock) < 0)
+		return entry;
 
-		/* increase the refcount on this object, because
-		 * the cache now owns it */
-		git_cached_obj_incref(entry);
+	if (cache->used_memory > git_cache__max_storage)
+		cache_evict_entries(cache);
 
-		if (node == NULL) {
-			cache->nodes[hash & cache->size_mask] = entry;
-		} else if (git_oid_cmp(&node->oid, &entry->oid) == 0) {
-			git_cached_obj_decref(entry, cache->free_obj);
-			entry = node;
-		} else {
-			git_cached_obj_decref(node, cache->free_obj);
-			cache->nodes[hash & cache->size_mask] = entry;
+	pos = kh_get(oid, cache->map, &entry->oid);
+
+	/* not found */
+	if (pos == kh_end(cache->map)) {
+		int rval;
+
+		pos = kh_put(oid, cache->map, &entry->oid, &rval);
+		if (rval >= 0) {
+			kh_key(cache->map, pos) = &entry->oid;
+			kh_val(cache->map, pos) = entry;
+			git_cached_obj_incref(entry);
+			cache->used_memory += entry->size;
+		}
 	}
+	/* found */
+	else {
+		git_cached_obj *stored_entry = kh_val(cache->map, pos);
 
-		/* increase the refcount again, because we are
-		 * returning it to the user */
+		if (stored_entry->flags == entry->flags) {
+			git_cached_obj_decref(entry);
+			git_cached_obj_incref(stored_entry);
+			entry = stored_entry;
+		} else if (stored_entry->flags == GIT_CACHE_STORE_RAW &&
+			entry->flags == GIT_CACHE_STORE_PARSED) {
+			git_cached_obj_decref(stored_entry);
 			git_cached_obj_incref(entry);
 
+			kh_key(cache->map, pos) = &entry->oid;
+			kh_val(cache->map, pos) = entry;
+		} else {
+			/* NO OP */
+		}
 	}
-	git_mutex_unlock(&cache->lock);
 
+	git_mutex_unlock(&cache->lock);
 	return entry;
 }
+
+void *git_cache_store_raw(git_cache *cache, git_odb_object *entry)
+{
+	entry->cached.flags = GIT_CACHE_STORE_RAW;
+	return cache_store(cache, (git_cached_obj *)entry);
+}
+
+void *git_cache_store_parsed(git_cache *cache, git_object *entry)
+{
+	entry->cached.flags = GIT_CACHE_STORE_PARSED;
+	return cache_store(cache, (git_cached_obj *)entry);
+}
+
+git_odb_object *git_cache_get_raw(git_cache *cache, const git_oid *oid)
+{
+	return cache_get(cache, oid, GIT_CACHE_STORE_RAW);
+}
+
+git_object *git_cache_get_parsed(git_cache *cache, const git_oid *oid)
+{
+	return cache_get(cache, oid, GIT_CACHE_STORE_PARSED);
+}
+
+void *git_cache_get_any(git_cache *cache, const git_oid *oid)
+{
+	return cache_get(cache, oid, GIT_CACHE_STORE_ANY);
+}
+
+void git_cached_obj_decref(void *_obj)
+{
+	git_cached_obj *obj = _obj;
+
+	if (git_atomic_dec(&obj->refcount) == 0) {
+		switch (obj->flags) {
+		case GIT_CACHE_STORE_RAW:
+			git_odb_object__free(_obj);
+			break;
+
+		case GIT_CACHE_STORE_PARSED:
+			git_object__free(_obj);
+			break;
+
+		default:
+			git__free(_obj);
+			break;
+		}
+	}
+}

src/cache.h

@@ -12,43 +12,54 @@
 #include "git2/odb.h"
 
 #include "thread-utils.h"
+#include "oidmap.h"
 
-#define GIT_DEFAULT_CACHE_SIZE 128
-
-typedef void (*git_cached_obj_freeptr)(void *);
+enum {
+	GIT_CACHE_STORE_ANY = 0,
+	GIT_CACHE_STORE_RAW = 1,
+	GIT_CACHE_STORE_PARSED = 2
+};
 
 typedef struct {
 	git_oid    oid;
+	int16_t    type;  /* git_otype value */
+	uint16_t   flags; /* GIT_CACHE_STORE value */
+	size_t     size;
 	git_atomic refcount;
 } git_cached_obj;
 
 typedef struct {
-	git_cached_obj **nodes;
+	git_oidmap *map;
 	git_mutex   lock;
-
-	unsigned int lru_count;
-	size_t size_mask;
-	git_cached_obj_freeptr free_obj;
+	size_t      used_memory;
 } git_cache;
 
-int git_cache_init(git_cache *cache, size_t size, git_cached_obj_freeptr free_ptr);
+extern bool git_cache__enabled;
+extern size_t git_cache__max_storage;
+
+int git_cache_set_max_object_size(git_otype type, size_t size);
+
+int git_cache_init(git_cache *cache);
 void git_cache_free(git_cache *cache);
 
-void *git_cache_try_store(git_cache *cache, void *entry);
-void *git_cache_get(git_cache *cache, const git_oid *oid);
+void *git_cache_store_raw(git_cache *cache, git_odb_object *entry);
+void *git_cache_store_parsed(git_cache *cache, git_object *entry);
 
-GIT_INLINE(void) git_cached_obj_incref(void *_obj)
+git_odb_object *git_cache_get_raw(git_cache *cache, const git_oid *oid);
+git_object *git_cache_get_parsed(git_cache *cache, const git_oid *oid);
+void *git_cache_get_any(git_cache *cache, const git_oid *oid);
+
+GIT_INLINE(size_t) git_cache_size(git_cache *cache)
 {
-	git_cached_obj *obj = _obj;
-	git_atomic_inc(&obj->refcount);
+	return (size_t)kh_size(cache->map);
 }
 
-GIT_INLINE(void) git_cached_obj_decref(void *_obj, git_cached_obj_freeptr free_obj)
+GIT_INLINE(void) git_cached_obj_incref(void *_obj)
 {
 	git_cached_obj *obj = _obj;
-
-	if (git_atomic_dec(&obj->refcount) == 0)
-		free_obj(obj);
+	git_atomic_inc(&obj->refcount);
 }
 
+void git_cached_obj_decref(void *_obj);
+
 #endif

src/checkout.c

@@ -710,8 +710,8 @@ static int blob_content_to_file(
 	git_vector filters = GIT_VECTOR_INIT;
 
 	/* Create a fake git_buf from the blob raw data... */
-	filtered.ptr = blob->odb_object->raw.data;
-	filtered.size = blob->odb_object->raw.len;
+	filtered.ptr  = (void *)git_blob_rawcontent(blob);
+	filtered.size = (size_t)git_blob_rawsize(blob);
 	/* ... and make sure it doesn't get unexpectedly freed */
 	dont_free_filtered = true;
 

src/commit.c

@@ -31,8 +31,10 @@ static void clear_parents(git_commit *commit)
 	git_vector_clear(&commit->parent_ids);
 }
 
-void git_commit__free(git_commit *commit)
+void git_commit__free(void *_commit)
 {
+	git_commit *commit = _commit;
+
 	clear_parents(commit);
 	git_vector_free(&commit->parent_ids);
 
@@ -166,10 +168,11 @@ int git_commit_create(
 	return retval;
 }
 
-int git_commit__parse_buffer(git_commit *commit, const void *data, size_t len)
+int git_commit__parse(void *_commit, git_odb_object *odb_obj)
 {
-	const char *buffer = data;
-	const char *buffer_end = (const char *)data + len;
+	git_commit *commit = _commit;
+	const char *buffer = git_odb_object_data(odb_obj);
+	const char *buffer_end = buffer + git_odb_object_size(odb_obj);
 	git_oid parent_id;
 
 	if (git_vector_init(&commit->parent_ids, 4, NULL) < 0)
@@ -241,12 +244,6 @@ bad_buffer:
 	return -1;
 }
 
-int git_commit__parse(git_commit *commit, git_odb_object *obj)
-{
-	assert(commit);
-	return git_commit__parse_buffer(commit, obj->raw.data, obj->raw.len);
-}
-
 #define GIT_COMMIT_GETTER(_rvalue, _name, _return) \
 	_rvalue git_commit_##_name(const git_commit *commit) \
 	{\

src/commit.h

@@ -27,8 +27,7 @@ struct git_commit {
 	char *message;
 };
 
-void git_commit__free(git_commit *c);
-int git_commit__parse(git_commit *commit, git_odb_object *obj);
+void git_commit__free(void *commit);
+int git_commit__parse(void *commit, git_odb_object *obj);
 
-int git_commit__parse_buffer(git_commit *commit, const void *data, size_t len);
 #endif

src/commit_list.c

@@ -100,12 +100,15 @@ git_commit_list_node *git_commit_list_pop(git_commit_list **stack)
 	return item;
 }
 
-static int commit_quick_parse(git_revwalk *walk, git_commit_list_node *commit, git_rawobj *raw)
+static int commit_quick_parse(
+	git_revwalk *walk,
+	git_commit_list_node *commit,
+	const uint8_t *buffer,
+	size_t buffer_len)
 {
 	const size_t parent_len = strlen("parent ") + GIT_OID_HEXSZ + 1;
-	unsigned char *buffer = raw->data;
-	unsigned char *buffer_end = buffer + raw->len;
-	unsigned char *parents_start, *committer_start;
+	const uint8_t *buffer_end = buffer + buffer_len;
+	const uint8_t *parents_start, *committer_start;
 	int i, parents = 0;
 	int commit_time;
 
@@ -124,7 +127,7 @@ static int commit_quick_parse(git_revwalk *walk, git_commit_list_node *commit, g
 	for (i = 0; i < parents; ++i) {
 		git_oid oid;
 
-		if (git_oid_fromstr(&oid, (char *)buffer + strlen("parent ")) < 0)
+		if (git_oid_fromstr(&oid, (const char *)buffer + strlen("parent ")) < 0)
 			return -1;
 
 		commit->parents[i] = git_revwalk__commit_lookup(walk, &oid);
@@ -182,11 +185,14 @@ int git_commit_list_parse(git_revwalk *walk, git_commit_list_node *commit)
 	if ((error = git_odb_read(&obj, walk->odb, &commit->oid)) < 0)
 		return error;
 
-	if (obj->raw.type != GIT_OBJ_COMMIT) {
+	if (obj->cached.type != GIT_OBJ_COMMIT) {
 		giterr_set(GITERR_INVALID, "Object is no commit object");
 		error = -1;
 	} else
-		error = commit_quick_parse(walk, commit, &obj->raw);
+		error = commit_quick_parse(
+			walk, commit,
+			(const uint8_t *)git_odb_object_data(obj),
+			git_odb_object_size(obj));
 
 	git_odb_object_free(obj);
 	return error;

src/global.c

@@ -135,6 +135,12 @@ int git_threads_init(void)
 
 void git_threads_shutdown(void)
 {
+	if (_tls_init) {
+		void *ptr = pthread_getspecific(_tls_key);
+		pthread_setspecific(_tls_key, NULL);
+		git__free(ptr);
+	}
+
 	pthread_key_delete(_tls_key);
 	_tls_init = 0;
 	git_mutex_free(&git__mwindow_mutex);

src/global.h

@@ -10,14 +10,6 @@
 #include "mwindow.h"
 #include "hash.h"
 
-#if defined(GIT_THREADS) && defined(_MSC_VER)
-# define GIT_MEMORY_BARRIER MemoryBarrier()
-#elif defined(GIT_THREADS)
-# define GIT_MEMORY_BARRIER __sync_synchronize()
-#else
-# define GIT_MEMORY_BARRIER /* noop */
-#endif
-
 typedef struct {
 	git_error *last_error;
 	git_error error_t;

src/indexer.c

@@ -60,36 +60,19 @@ const git_oid *git_indexer_stream_hash(const git_indexer_stream *idx)
 
 static int open_pack(struct git_pack_file **out, const char *filename)
 {
-	size_t namelen;
 	struct git_pack_file *pack;
-	struct stat st;
-	int fd;
 
-	namelen = strlen(filename);
-	pack = git__calloc(1, sizeof(struct git_pack_file) + namelen + 1);
-	GITERR_CHECK_ALLOC(pack);
-
-	memcpy(pack->pack_name, filename, namelen + 1);
-
-	if (p_stat(filename, &st) < 0) {
-		giterr_set(GITERR_OS, "Failed to stat packfile.");
-		goto cleanup;
-	}
+	if (git_packfile_alloc(&pack, filename) < 0)
+		return -1;
 
-	if ((fd = p_open(pack->pack_name, O_RDONLY)) < 0) {
+	if ((pack->mwf.fd = p_open(pack->pack_name, O_RDONLY)) < 0) {
 		giterr_set(GITERR_OS, "Failed to open packfile.");
-		goto cleanup;
+		git_packfile_free(pack);
+		return -1;
 	}
 
-	pack->mwf.fd = fd;
-	pack->mwf.size = (git_off_t)st.st_size;
-
 	*out = pack;
 	return 0;
-
-cleanup:
-	git__free(pack);
-	return -1;
 }
 
 static int parse_header(struct git_pack_header *hdr, struct git_pack_file *pack)
@@ -404,7 +387,6 @@ int git_indexer_stream_add(git_indexer_stream *idx, const void *data, size_t siz
 	/* Make sure we set the new size of the pack */
 	if (idx->opened_pack) {
 		idx->pack->mwf.size += size;
-		//printf("\nadding %zu for %zu\n", size, idx->pack->mwf.size);
 	} else {
 		if (open_pack(&idx->pack, idx->pack_file.path_lock) < 0)
 			return -1;

src/object.c

@@ -18,65 +18,38 @@
 
 static const int OBJECT_BASE_SIZE = 4096;
 
-static struct {
+typedef struct {
 	const char	*str;	/* type name string */
-	int			loose; /* valid loose object type flag */
 	size_t		size;	/* size in bytes of the object structure */
-} git_objects_table[] = {
+
+	int  (*parse)(void *self, git_odb_object *obj);
+	void (*free)(void *self);
+} git_object_def;
+
+static git_object_def git_objects_table[] = {
 	/* 0 = GIT_OBJ__EXT1 */
-	{ "", 0, 0},
+	{ "", 0, NULL, NULL },
 
 	/* 1 = GIT_OBJ_COMMIT */
-	{ "commit", 1, sizeof(struct git_commit)},
+	{ "commit", sizeof(git_commit), git_commit__parse, git_commit__free },
 
 	/* 2 = GIT_OBJ_TREE */
-	{ "tree", 1, sizeof(struct git_tree) },
+	{ "tree", sizeof(git_tree), git_tree__parse, git_tree__free },
 
 	/* 3 = GIT_OBJ_BLOB */
-	{ "blob", 1, sizeof(struct git_blob) },
+	{ "blob", sizeof(git_blob), git_blob__parse, git_blob__free },
 
 	/* 4 = GIT_OBJ_TAG */
-	{ "tag", 1, sizeof(struct git_tag) },
+	{ "tag", sizeof(git_tag), git_tag__parse, git_tag__free },
 
 	/* 5 = GIT_OBJ__EXT2 */
-	{ "", 0, 0 },
-
+	{ "", 0, NULL, NULL },
 	/* 6 = GIT_OBJ_OFS_DELTA */
-	{ "OFS_DELTA", 0, 0 },
-
+	{ "OFS_DELTA", 0, NULL, NULL },
 	/* 7 = GIT_OBJ_REF_DELTA */
-	{ "REF_DELTA", 0, 0	}
+	{ "REF_DELTA", 0, NULL, NULL },
 };
 
-static int create_object(git_object **object_out, git_otype type)
-{
-	git_object *object = NULL;
-
-	assert(object_out);
-
-	*object_out = NULL;
-
-	switch (type) {
-	case GIT_OBJ_COMMIT:
-	case GIT_OBJ_TAG:
-	case GIT_OBJ_BLOB:
-	case GIT_OBJ_TREE:
-		object = git__malloc(git_object__size(type));
-		GITERR_CHECK_ALLOC(object);
-		memset(object, 0x0, git_object__size(type));
-		break;
-
-	default:
-		giterr_set(GITERR_INVALID, "The given type is invalid");
-		return -1;
-	}
-
-	object->type = type;
-
-	*object_out = object;
-	return 0;
-}
-
 int git_object__from_odb_object(
 	git_object **object_out,
 	git_repository *repo,
@@ -84,49 +57,55 @@ int git_object__from_odb_object(
 	git_otype type)
 {
 	int error;
+	size_t object_size;
+	git_object_def *def;
 	git_object *object = NULL;
 
-	if (type != GIT_OBJ_ANY && type != odb_obj->raw.type) {
-		giterr_set(GITERR_INVALID, "The requested type does not match the type in the ODB");
+	assert(object_out);
+	*object_out = NULL;
+
+	/* Validate type match */
+	if (type != GIT_OBJ_ANY && type != odb_obj->cached.type) {
+		giterr_set(GITERR_INVALID,
+			"The requested type does not match the type in the ODB");
 		return GIT_ENOTFOUND;
 	}
 
-	type = odb_obj->raw.type;
+	if ((object_size = git_object__size(odb_obj->cached.type)) == 0) {
+		giterr_set(GITERR_INVALID, "The requested type is invalid");
+		return GIT_ENOTFOUND;
+	}
 
-	if ((error = create_object(&object, type)) < 0)
-		return error;
+	/* Allocate and initialize base object */
+	object = git__calloc(1, object_size);
+	GITERR_CHECK_ALLOC(object);
 
-	/* Initialize parent object */
 	git_oid_cpy(&object->cached.oid, &odb_obj->cached.oid);
+	object->cached.type = odb_obj->cached.type;
+	object->cached.size = odb_obj->cached.size;
 	object->repo = repo;
 
-	switch (type) {
-	case GIT_OBJ_COMMIT:
-		error = git_commit__parse((git_commit *)object, odb_obj);
-		break;
+	/* Parse raw object data */
+	def = &git_objects_table[odb_obj->cached.type];
+	assert(def->free && def->parse);
 
-	case GIT_OBJ_TREE:
-		error = git_tree__parse((git_tree *)object, odb_obj);
-		break;
-
-	case GIT_OBJ_TAG:
-		error = git_tag__parse((git_tag *)object, odb_obj);
-		break;
+	if ((error = def->parse(object, odb_obj)) < 0)
+		def->free(object);
+	else
+		*object_out = git_cache_store_parsed(&repo->objects, object);
 
-	case GIT_OBJ_BLOB:
-		error = git_blob__parse((git_blob *)object, odb_obj);
-		break;
+	return error;
+}
 
-	default:
-		break;
-	}
+void git_object__free(void *obj)
+{
+	git_otype type = ((git_object *)obj)->cached.type;
 
-	if (error < 0)
-		git_object__free(object);
+	if (type < 0 || ((size_t)type) >= ARRAY_SIZE(git_objects_table) ||
+		!git_objects_table[type].free)
+		git__free(obj);
 	else
-		*object_out = git_cache_try_store(&repo->objects, object);
-
-	return error;
+		git_objects_table[type].free(obj);
 }
 
 int git_object_lookup_prefix(
@@ -154,27 +133,38 @@ int git_object_lookup_prefix(
 		len = GIT_OID_HEXSZ;
 
 	if (len == GIT_OID_HEXSZ) {
+		git_cached_obj *cached = NULL;
+
 		/* We want to match the full id : we can first look up in the cache,
 		 * since there is no need to check for non ambiguousity
 		 */
-		object = git_cache_get(&repo->objects, id);
-		if (object != NULL) {
-			if (type != GIT_OBJ_ANY && type != object->type) {
+		cached = git_cache_get_any(&repo->objects, id);
+		if (cached != NULL) {
+			if (cached->flags == GIT_CACHE_STORE_PARSED) {
+				object = (git_object *)cached;
+
+				if (type != GIT_OBJ_ANY && type != object->cached.type) {
 					git_object_free(object);
-				giterr_set(GITERR_INVALID, "The requested type does not match the type in ODB");
+					giterr_set(GITERR_INVALID,
+						"The requested type does not match the type in ODB");
 					return GIT_ENOTFOUND;
 				}
 
 				*object_out = object;
 				return 0;
+			} else if (cached->flags == GIT_CACHE_STORE_RAW) {
+				odb_obj = (git_odb_object *)cached;
+			} else {
+				assert(!"Wrong caching type in the global object cache");
 			}
-
+		} else {
 			/* Object was not found in the cache, let's explore the backends.
 			 * We could just use git_odb_read_unique_short_oid,
 			 * it is the same cost for packed and loose object backends,
 			 * but it may be much more costly for sqlite and hiredis.
 			 */
 			error = git_odb_read(&odb_obj, odb, id);
+		}
 	} else {
 		git_oid short_oid;
 
@@ -211,41 +201,12 @@ int git_object_lookup(git_object **object_out, git_repository *repo, const git_o
 	return git_object_lookup_prefix(object_out, repo, id, GIT_OID_HEXSZ, type);
 }
 
-void git_object__free(void *_obj)
-{
-	git_object *object = (git_object *)_obj;
-
-	assert(object);
-
-	switch (object->type) {
-	case GIT_OBJ_COMMIT:
-		git_commit__free((git_commit *)object);
-		break;
-
-	case GIT_OBJ_TREE:
-		git_tree__free((git_tree *)object);
-		break;
-
-	case GIT_OBJ_TAG:
-		git_tag__free((git_tag *)object);
-		break;
-
-	case GIT_OBJ_BLOB:
-		git_blob__free((git_blob *)object);
-		break;
-
-	default:
-		git__free(object);
-		break;
-	}
-}
-
 void git_object_free(git_object *object)
 {
 	if (object == NULL)
 		return;
 
-	git_cached_obj_decref((git_cached_obj *)object, git_object__free);
+	git_cached_obj_decref(object);
 }
 
 const git_oid *git_object_id(const git_object *obj)
@@ -257,7 +218,7 @@ const git_oid *git_object_id(const git_object *obj)
 git_otype git_object_type(const git_object *obj)
 {
 	assert(obj);
-	return obj->type;
+	return obj->cached.type;
 }
 
 git_repository *git_object_owner(const git_object *obj)
@@ -293,7 +254,7 @@ int git_object_typeisloose(git_otype type)
 	if (type < 0 || ((size_t) type) >= ARRAY_SIZE(git_objects_table))
 		return 0;
 
-	return git_objects_table[type].loose;
+	return (git_objects_table[type].size > 0) ? 1 : 0;
 }
 
 size_t git_object__size(git_otype type)

src/object.h

@@ -11,7 +11,6 @@
 struct git_object {
 	git_cached_obj cached;
 	git_repository *repo;
-	git_otype type;
 };
 
 /* fully free the object; internal method, DO NOT EXPORT */

src/odb.c

@@ -14,6 +14,7 @@
 #include "odb.h"
 #include "delta-apply.h"
 #include "filter.h"
+#include "repository.h"
 
 #include "git2/odb_backend.h"
 #include "git2/oid.h"
@@ -34,7 +35,15 @@ typedef struct
 	ino_t disk_inode;
 } backend_internal;
 
-size_t git_odb__cache_size = GIT_DEFAULT_CACHE_SIZE;
+static git_cache *odb_cache(git_odb *odb)
+{
+	if (odb->rc.owner != NULL) {
+		git_repository *owner = odb->rc.owner;
+		return &owner->objects;
+	}
+
+	return &odb->own_cache;
+}
 
 static int load_alternates(git_odb *odb, const char *objects_dir, int alternate_depth);
 
@@ -56,6 +65,7 @@ int git_odb__hashobj(git_oid *id, git_rawobj *obj)
 
 	if (!git_object_typeisloose(obj->type))
 		return -1;
+
 	if (!obj->data && obj->len != 0)
 		return -1;
 
@@ -72,23 +82,24 @@ int git_odb__hashobj(git_oid *id, git_rawobj *obj)
 }
 
 
-static git_odb_object *new_odb_object(const git_oid *oid, git_rawobj *source)
+static git_odb_object *odb_object__alloc(const git_oid *oid, git_rawobj *source)
 {
-	git_odb_object *object = git__malloc(sizeof(git_odb_object));
-	memset(object, 0x0, sizeof(git_odb_object));
+	git_odb_object *object = git__calloc(1, sizeof(git_odb_object));
 
+	if (object != NULL) {
 		git_oid_cpy(&object->cached.oid, oid);
-	memcpy(&object->raw, source, sizeof(git_rawobj));
+		object->cached.type = source->type;
+		object->cached.size = source->len;
+		object->buffer      = source->data;
+	}
 
 	return object;
 }
 
-static void free_odb_object(void *o)
+void git_odb_object__free(void *object)
 {
-	git_odb_object *object = (git_odb_object *)o;
-
 	if (object != NULL) {
-		git__free(object->raw.data);
+		git__free(((git_odb_object *)object)->buffer);
 		git__free(object);
 	}
 }
@@ -100,17 +111,17 @@ const git_oid *git_odb_object_id(git_odb_object *object)
 
 const void *git_odb_object_data(git_odb_object *object)
 {
-	return object->raw.data;
+	return object->buffer;
 }
 
 size_t git_odb_object_size(git_odb_object *object)
 {
-	return object->raw.len;
+	return object->cached.size;
 }
 
 git_otype git_odb_object_type(git_odb_object *object)
 {
-	return object->raw.type;
+	return object->cached.type;
 }
 
 void git_odb_object_free(git_odb_object *object)
@@ -118,7 +129,7 @@ void git_odb_object_free(git_odb_object *object)
 	if (object == NULL)
 		return;
 
-	git_cached_obj_decref((git_cached_obj *)object, &free_odb_object);
+	git_cached_obj_decref(object);
 }
 
 int git_odb__hashfd(git_oid *out, git_file fd, size_t size, git_otype type)
@@ -355,9 +366,8 @@ int git_odb_new(git_odb **out)
 	git_odb *db = git__calloc(1, sizeof(*db));
 	GITERR_CHECK_ALLOC(db);
 
-	if (git_cache_init(&db->cache, git_odb__cache_size, &free_odb_object) < 0 ||
-		git_vector_init(&db->backends, 4, backend_sort_cmp) < 0)
-	{
+	if (git_cache_init(&db->own_cache) < 0 ||
+		git_vector_init(&db->backends, 4, backend_sort_cmp) < 0) {
 		git__free(db);
 		return -1;
 	}
@@ -559,7 +569,7 @@ static void odb_free(git_odb *db)
 	}
 
 	git_vector_free(&db->backends);
-	git_cache_free(&db->cache);
+	git_cache_free(&db->own_cache);
 	git__free(db);
 }
 
@@ -580,7 +590,7 @@ int git_odb_exists(git_odb *db, const git_oid *id)
 
 	assert(db && id);
 
-	if ((object = git_cache_get(&db->cache, id)) != NULL) {
+	if ((object = git_cache_get_raw(odb_cache(db), id)) != NULL) {
 		git_odb_object_free(object);
 		return (int)true;
 	}
@@ -630,9 +640,9 @@ int git_odb__read_header_or_object(
 
 	assert(db && id && out && len_p && type_p);
 
-	if ((object = git_cache_get(&db->cache, id)) != NULL) {
-		*len_p = object->raw.len;
-		*type_p = object->raw.type;
+	if ((object = git_cache_get_raw(odb_cache(db), id)) != NULL) {
+		*len_p = object->cached.size;
+		*type_p = object->cached.type;
 		*out = object;
 		return 0;
 	}
@@ -657,8 +667,8 @@ int git_odb__read_header_or_object(
 	if ((error = git_odb_read(&object, db, id)) < 0)
 		return error; /* error already set - pass along */
 
-	*len_p = object->raw.len;
-	*type_p = object->raw.type;
+	*len_p = object->cached.size;
+	*type_p = object->cached.type;
 	*out = object;
 
 	return 0;
@@ -670,6 +680,7 @@ int git_odb_read(git_odb_object **out, git_odb *db, const git_oid *id)
 	int error;
 	bool refreshed = false;
 	git_rawobj raw;
+	git_odb_object *object;
 
 	assert(out && db && id);
 
@@ -678,7 +689,7 @@ int git_odb_read(git_odb_object **out, git_odb *db, const git_oid *id)
 		return GIT_ENOTFOUND;
 	}
 
-	*out = git_cache_get(&db->cache, id);
+	*out = git_cache_get_raw(odb_cache(db), id);
 	if (*out != NULL)
 		return 0;
 
@@ -704,7 +715,10 @@ attempt_lookup:
 	if (error && error != GIT_PASSTHROUGH)
 		return error;
 
-	*out = git_cache_try_store(&db->cache, new_odb_object(id, &raw));
+	if ((object = odb_object__alloc(id, &raw)) == NULL)
+		return -1;
+
+	*out = git_cache_store_raw(odb_cache(db), object);
 	return 0;
 }
 
@@ -717,6 +731,7 @@ int git_odb_read_prefix(
 	git_rawobj raw;
 	void *data = NULL;
 	bool found = false, refreshed = false;
+	git_odb_object *object;
 
 	assert(out && db);
 
@@ -727,7 +742,7 @@ int git_odb_read_prefix(
 		len = GIT_OID_HEXSZ;
 
 	if (len == GIT_OID_HEXSZ) {
-		*out = git_cache_get(&db->cache, short_id);
+		*out = git_cache_get_raw(odb_cache(db), short_id);
 		if (*out != NULL)
 			return 0;
 	}
@@ -768,7 +783,10 @@ attempt_lookup:
 	if (!found)
 		return git_odb__error_notfound("no match for prefix", short_id);
 
-	*out = git_cache_try_store(&db->cache, new_odb_object(&found_full_oid, &raw));
+	if ((object = odb_object__alloc(&found_full_oid, &raw)) == NULL)
+		return -1;
+
+	*out = git_cache_store_raw(odb_cache(db), object);
 	return 0;
 }
 

src/odb.h

@@ -29,14 +29,14 @@ typedef struct {
 /* EXPORT */
 struct git_odb_object {
 	git_cached_obj cached;
-	git_rawobj raw;
+	void *buffer;
 };
 
 /* EXPORT */
 struct git_odb {
 	git_refcount rc;
 	git_vector backends;
-	git_cache cache;
+	git_cache own_cache;
 };
 
 /*
@@ -96,4 +96,7 @@ int git_odb__read_header_or_object(
 	git_odb_object **out, size_t *len_p, git_otype *type_p,
 	git_odb *db, const git_oid *id);
 
+/* fully free the object; internal method, DO NOT EXPORT */
+void git_odb_object__free(void *object);
+
 #endif

src/odb_pack.c

@@ -207,7 +207,7 @@ static int packfile_load__cb(void *_data, git_buf *path)
 			return 0;
 	}
 
-	error = git_packfile_check(&pack, path->ptr);
+	error = git_packfile_alloc(&pack, path->ptr);
 	if (error == GIT_ENOTFOUND)
 		/* ignore missing .pack file as git does */
 		return 0;
@@ -527,80 +527,75 @@ static void pack_backend__free(git_odb_backend *_backend)
 	git__free(backend);
 }
 
-int git_odb_backend_one_pack(git_odb_backend **backend_out, const char *idx)
+static int pack_backend__alloc(struct pack_backend **out, size_t initial_size)
 {
-	struct pack_backend *backend = NULL;
-	struct git_pack_file *packfile = NULL;
+	struct pack_backend *backend = git__calloc(1, sizeof(struct pack_backend));
+	GITERR_CHECK_ALLOC(backend);
 
-	if (git_packfile_check(&packfile, idx) < 0)
+	if (git_vector_init(&backend->packs, initial_size, packfile_sort__cb) < 0) {
+		git__free(backend);
 		return -1;
+	}
 
-	backend = git__calloc(1, sizeof(struct pack_backend));
-	GITERR_CHECK_ALLOC(backend);
 	backend->parent.version = GIT_ODB_BACKEND_VERSION;
 
-	if (git_vector_init(&backend->packs, 1, NULL) < 0)
-		goto on_error;
-
-	if (git_vector_insert(&backend->packs, packfile) < 0)
-		goto on_error;
-
 	backend->parent.read = &pack_backend__read;
 	backend->parent.read_prefix = &pack_backend__read_prefix;
 	backend->parent.read_header = &pack_backend__read_header;
 	backend->parent.exists = &pack_backend__exists;
 	backend->parent.refresh = &pack_backend__refresh;
 	backend->parent.foreach = &pack_backend__foreach;
+	backend->parent.writepack = &pack_backend__writepack;
 	backend->parent.free = &pack_backend__free;
 
-	*backend_out = (git_odb_backend *)backend;
-
+	*out = backend;
 	return 0;
-
-on_error:
-	git_vector_free(&backend->packs);
-	git__free(backend);
-	git__free(packfile);
-	return -1;
 }
 
-int git_odb_backend_pack(git_odb_backend **backend_out, const char *objects_dir)
+int git_odb_backend_one_pack(git_odb_backend **backend_out, const char *idx)
 {
 	struct pack_backend *backend = NULL;
-	git_buf path = GIT_BUF_INIT;
+	struct git_pack_file *packfile = NULL;
 
-	backend = git__calloc(1, sizeof(struct pack_backend));
-	GITERR_CHECK_ALLOC(backend);
-	backend->parent.version = GIT_ODB_BACKEND_VERSION;
+	if (pack_backend__alloc(&backend, 1) < 0)
+		return -1;
 
-	if (git_vector_init(&backend->packs, 8, packfile_sort__cb) < 0 ||
-		git_buf_joinpath(&path, objects_dir, "pack") < 0)
+	if (git_packfile_alloc(&packfile, idx) < 0 ||
+		git_vector_insert(&backend->packs, packfile) < 0)
 	{
-		git__free(backend);
+		pack_backend__free((git_odb_backend *)backend);
 		return -1;
 	}
 
-	if (git_path_isdir(git_buf_cstr(&path)) == true) {
-		int error;
+	*backend_out = (git_odb_backend *)backend;
+	return 0;
+}
 
+int git_odb_backend_pack(git_odb_backend **backend_out, const char *objects_dir)
+{
+	int error = 0;
+	struct pack_backend *backend = NULL;
+	git_buf path = GIT_BUF_INIT;
+
+	if (pack_backend__alloc(&backend, 8) < 0)
+		return -1;
+
+	if (!(error = git_buf_joinpath(&path, objects_dir, "pack")) &&
+		git_path_isdir(git_buf_cstr(&path)))
+	{
 		backend->pack_folder = git_buf_detach(&path);
+
 		error = pack_backend__refresh((git_odb_backend *)backend);
-		if (error < 0)
-			return error;
 	}
 
-	backend->parent.read = &pack_backend__read;
-	backend->parent.read_prefix = &pack_backend__read_prefix;
-	backend->parent.read_header = &pack_backend__read_header;
-	backend->parent.exists = &pack_backend__exists;
-	backend->parent.refresh = &pack_backend__refresh;
-	backend->parent.foreach = &pack_backend__foreach;
-	backend->parent.writepack = &pack_backend__writepack;
-	backend->parent.free = &pack_backend__free;
+	if (error < 0) {
+		pack_backend__free((git_odb_backend *)backend);
+		backend = NULL;
+	}
 
 	*backend_out = (git_odb_backend *)backend;
 
 	git_buf_free(&path);
 
-	return 0;
+	return error;
 }

src/oidmap.h

@@ -19,17 +19,15 @@
 __KHASH_TYPE(oid, const git_oid *, void *);
 typedef khash_t(oid) git_oidmap;
 
-GIT_INLINE(khint_t) hash_git_oid(const git_oid *oid)
+GIT_INLINE(khint_t) git_oidmap_hash(const git_oid *oid)
 {
-	int i;
-	khint_t h = 0;
-	for (i = 0; i < 20; ++i)
-		h = (h << 5) - h + oid->id[i];
+	khint_t h;
+	memcpy(&h, oid, sizeof(khint_t));
 	return h;
 }
 
 #define GIT__USE_OIDMAP \
-	__KHASH_IMPL(oid, static kh_inline, const git_oid *, void *, 1, hash_git_oid, git_oid_equal)
+	__KHASH_IMPL(oid, static kh_inline, const git_oid *, void *, 1, git_oidmap_hash, git_oid_equal)
 
 #define git_oidmap_alloc() kh_init(oid)
 #define git_oidmap_free(h) kh_destroy(oid,h), h = NULL

src/pack.c

@@ -296,24 +296,32 @@ static int pack_index_check(const char *path, struct git_pack_file *p)
 static int pack_index_open(struct git_pack_file *p)
 {
 	char *idx_name;
-	int error;
-	size_t name_len, offset;
+	int error = 0;
+	size_t name_len, base_len;
 
 	if (p->index_map.data)
 		return 0;
 
-	idx_name = git__strdup(p->pack_name);
-	GITERR_CHECK_ALLOC(idx_name);
+	name_len = strlen(p->pack_name);
+	assert(name_len > strlen(".pack")); /* checked by git_pack_file alloc */
+
+	if ((idx_name = git__malloc(name_len)) == NULL)
+		return -1;
 
-	name_len = strlen(idx_name);
-	offset = name_len - strlen(".pack");
-	assert(offset < name_len); /* make sure no underflow */
+	base_len = name_len - strlen(".pack");
+	memcpy(idx_name, p->pack_name, base_len);
+	memcpy(idx_name + base_len, ".idx", sizeof(".idx"));
 
-	strncpy(idx_name + offset, ".idx", name_len - offset);
+	if ((error = git_mutex_lock(&p->lock)) < 0)
+		return error;
 
+	if (!p->index_map.data)
 		error = pack_index_check(idx_name, p);
+
 	git__free(idx_name);
 
+	git_mutex_unlock(&p->lock);
+
 	return error;
 }
 
@@ -389,7 +397,7 @@ int git_packfile_unpack_header(
 	 * the maximum deflated object size is 2^137, which is just
 	 * insane, so we know won't exceed what we have been given.
 	 */
-//	base = pack_window_open(p, w_curs, *curpos, &left);
+/*	base = pack_window_open(p, w_curs, *curpos, &left); */
 	base = git_mwindow_open(mwf, w_curs, *curpos, 20, &left);
 	if (base == NULL)
 		return GIT_EBUFS;
@@ -786,23 +794,17 @@ git_off_t get_delta_base(
  *
  ***********************************************************/
 
-static struct git_pack_file *packfile_alloc(size_t extra)
-{
-	struct git_pack_file *p = git__calloc(1, sizeof(*p) + extra);
-	if (p != NULL)
-		p->mwf.fd = -1;
-	return p;
-}
-
-
 void git_packfile_free(struct git_pack_file *p)
 {
-	assert(p);
+	if (!p)
+		return;
+
+	if (git_mutex_lock(&p->lock) < 0)
+		return;
 
 	cache_free(&p->bases);
 
 	git_mwindow_free_all(&p->mwf);
-	git_mwindow_file_deregister(&p->mwf);
 
 	if (p->mwf.fd != -1)
 		p_close(p->mwf.fd);
@@ -810,6 +812,10 @@ void git_packfile_free(struct git_pack_file *p)
 	pack_index_free(p);
 
 	git__free(p->bad_object_sha1);
+
+	git_mutex_unlock(&p->lock);
+
+	git_mutex_free(&p->lock);
 	git__free(p);
 }
 
@@ -820,8 +826,6 @@ static int packfile_open(struct git_pack_file *p)
 	git_oid sha1;
 	unsigned char *idx_sha1;
 
-	assert(p->index_map.data);
-
 	if (!p->index_map.data && pack_index_open(p) < 0)
 		return git_odb__error_notfound("failed to open packfile", NULL);
 
@@ -881,34 +885,37 @@ cleanup:
 	return -1;
 }
 
-int git_packfile_check(struct git_pack_file **pack_out, const char *path)
+int git_packfile_alloc(struct git_pack_file **pack_out, const char *path)
 {
 	struct stat st;
 	struct git_pack_file *p;
-	size_t path_len;
+	size_t path_len = path ? strlen(path) : 0;
 
 	*pack_out = NULL;
-	path_len = strlen(path);
-	p = packfile_alloc(path_len + 2);
+
+	if (path_len < strlen(".idx"))
+		return git_odb__error_notfound("invalid packfile path", NULL);
+
+	p = git__calloc(1, sizeof(*p) + path_len + 2);
 	GITERR_CHECK_ALLOC(p);
 
+	memcpy(p->pack_name, path, path_len + 1);
+
 	/*
 	 * Make sure a corresponding .pack file exists and that
 	 * the index looks sane.
 	 */
-	path_len -= strlen(".idx");
-	if (path_len < 1) {
-		git__free(p);
-		return git_odb__error_notfound("invalid packfile path", NULL);
-	}
+	if (git__suffixcmp(path, ".idx") == 0) {
+		size_t root_len = path_len - strlen(".idx");
 
-	memcpy(p->pack_name, path, path_len);
-
-	strcpy(p->pack_name + path_len, ".keep");
+		memcpy(p->pack_name + root_len, ".keep", sizeof(".keep"));
 		if (git_path_exists(p->pack_name) == true)
 			p->pack_keep = 1;
 
-	strcpy(p->pack_name + path_len, ".pack");
+		memcpy(p->pack_name + root_len, ".pack", sizeof(".pack"));
+		path_len = path_len - strlen(".idx") + strlen(".pack");
+	}
+
 	if (p_stat(p->pack_name, &st) < 0 || !S_ISREG(st.st_mode)) {
 		git__free(p);
 		return git_odb__error_notfound("packfile not found", NULL);
@@ -917,10 +924,13 @@ int git_packfile_check(struct git_pack_file **pack_out, const char *path)
 	/* ok, it looks sane as far as we can check without
 	 * actually mapping the pack file.
 	 */
+	p->mwf.fd = -1;
 	p->mwf.size = st.st_size;
 	p->pack_local = 1;
 	p->mtime = (git_time_t)st.st_mtime;
 
+	git_mutex_init(&p->lock);
+
 	/* see if we can parse the sha1 oid in the packfile name */
 	if (path_len < 40 ||
 		git_oid_fromstr(&p->sha1, path + path_len - GIT_OID_HEXSZ) < 0)
@@ -1039,7 +1049,6 @@ static int pack_entry_find_offset(
 
 		if ((error = pack_index_open(p)) < 0)
 			return error;
-
 		assert(p->index_map.data);
 
 		index = p->index_map.data;
@@ -1099,6 +1108,7 @@ static int pack_entry_find_offset(
 		return git_odb__error_notfound("failed to find offset for pack entry", short_oid);
 	if (found > 1)
 		return git_odb__error_ambiguous("found multiple offsets for pack entry");
+
 	*offset_out = nth_packed_object_offset(p, pos);
 	git_oid_fromraw(found_oid, current);
 
@@ -1110,6 +1120,7 @@ static int pack_entry_find_offset(
 		printf("found lo=%d %s\n", lo, hex_sha1);
 	}
 #endif
+
 	return 0;
 }
 

src/pack.h

@@ -79,6 +79,7 @@ typedef struct {
 struct git_pack_file {
 	git_mwindow_file mwf;
 	git_map index_map;
+	git_mutex lock; /* protect updates to mwf and index_map */
 
 	uint32_t num_objects;
 	uint32_t num_bad_objects;
@@ -142,7 +143,8 @@ git_off_t get_delta_base(struct git_pack_file *p, git_mwindow **w_curs,
 		git_off_t delta_obj_offset);
 
 void git_packfile_free(struct git_pack_file *p);
-int git_packfile_check(struct git_pack_file **pack_out, const char *path);
+int git_packfile_alloc(struct git_pack_file **pack_out, const char *path);
+
 int git_pack_entry_find(
 		struct git_pack_entry *e,
 		struct git_pack_file *p,

src/refdb_fs.c

@@ -11,7 +11,6 @@
 #include "fileops.h"
 #include "pack.h"
 #include "reflog.h"
-#include "config.h"
 #include "refdb.h"
 #include "refdb_fs.h"
 

src/repository.c

@@ -32,41 +32,57 @@
 
 #define GIT_TEMPLATE_DIR "/usr/share/git-core/templates"
 
-static void drop_odb(git_repository *repo)
+static void set_odb(git_repository *repo, git_odb *odb)
 {
-	if (repo->_odb != NULL) {
-		GIT_REFCOUNT_OWN(repo->_odb, NULL);
-		git_odb_free(repo->_odb);
-		repo->_odb = NULL;
+	if (odb) {
+		GIT_REFCOUNT_OWN(odb, repo);
+		GIT_REFCOUNT_INC(odb);
+	}
+
+	if ((odb = git__swap(repo->_odb, odb)) != NULL) {
+		GIT_REFCOUNT_OWN(odb, NULL);
+		git_odb_free(odb);
 	}
 }
 
-static void drop_refdb(git_repository *repo)
+static void set_refdb(git_repository *repo, git_refdb *refdb)
 {
-	if (repo->_refdb != NULL) {
-		GIT_REFCOUNT_OWN(repo->_refdb, NULL);
-		git_refdb_free(repo->_refdb);
-		repo->_refdb = NULL;
+	if (refdb) {
+		GIT_REFCOUNT_OWN(refdb, repo);
+		GIT_REFCOUNT_INC(refdb);
+	}
+
+	if ((refdb = git__swap(repo->_refdb, refdb)) != NULL) {
+		GIT_REFCOUNT_OWN(refdb, NULL);
+		git_refdb_free(refdb);
 	}
 }
 
-static void drop_config(git_repository *repo)
+static void set_config(git_repository *repo, git_config *config)
 {
-	if (repo->_config != NULL) {
-		GIT_REFCOUNT_OWN(repo->_config, NULL);
-		git_config_free(repo->_config);
-		repo->_config = NULL;
+	if (config) {
+		GIT_REFCOUNT_OWN(config, repo);
+		GIT_REFCOUNT_INC(config);
+	}
+
+	if ((config = git__swap(repo->_config, config)) != NULL) {
+		GIT_REFCOUNT_OWN(config, NULL);
+		git_config_free(config);
 	}
 
 	git_repository__cvar_cache_clear(repo);
 }
 
-static void drop_index(git_repository *repo)
+static void set_index(git_repository *repo, git_index *index)
 {
-	if (repo->_index != NULL) {
-		GIT_REFCOUNT_OWN(repo->_index, NULL);
-		git_index_free(repo->_index);
-		repo->_index = NULL;
+	if (index) {
+		GIT_REFCOUNT_OWN(index, repo);
+		GIT_REFCOUNT_INC(index);
+	}
+
+	if ((index = git__swap(repo->_index, index)) != NULL) {
+		GIT_REFCOUNT_OWN(index, NULL);
+		git_index_free(index);
 	}
 }
 
@@ -79,14 +95,14 @@ void git_repository_free(git_repository *repo)
 	git_attr_cache_flush(repo);
 	git_submodule_config_free(repo);
 
+	set_config(repo, NULL);
+	set_index(repo, NULL);
+	set_odb(repo, NULL);
+	set_refdb(repo, NULL);
+
 	git__free(repo->path_repository);
 	git__free(repo->workdir);
 
-	drop_config(repo);
-	drop_index(repo);
-	drop_odb(repo);
-	drop_refdb(repo);
-
 	git__free(repo);
 }
 
@@ -119,7 +135,7 @@ static git_repository *repository_alloc(void)
 
 	memset(repo, 0x0, sizeof(git_repository));
 
-	if (git_cache_init(&repo->objects, GIT_DEFAULT_CACHE_SIZE, &git_object__free) < 0) {
+	if (git_cache_init(&repo->objects) < 0) {
 		git__free(repo);
 		return NULL;
 	}
@@ -549,39 +565,47 @@ on_error:
 	return error;
 }
 
-int git_repository_config__weakptr(git_config **out, git_repository *repo)
+static const char *path_unless_empty(git_buf *buf)
 {
-	if (repo->_config == NULL) {
-		git_buf global_buf = GIT_BUF_INIT, xdg_buf = GIT_BUF_INIT, system_buf = GIT_BUF_INIT;
-		int res;
+	return git_buf_len(buf) > 0 ? git_buf_cstr(buf) : NULL;
+}
 
-		const char *global_config_path = NULL;
-		const char *xdg_config_path = NULL;
-		const char *system_config_path = NULL;
+int git_repository_config__weakptr(git_config **out, git_repository *repo)
+{
+	int error = 0;
 
-		if (git_config_find_global_r(&global_buf) == 0)
-			global_config_path = global_buf.ptr;
+	if (repo->_config == NULL) {
+		git_buf global_buf = GIT_BUF_INIT;
+		git_buf xdg_buf = GIT_BUF_INIT;
+		git_buf system_buf = GIT_BUF_INIT;
+		git_config *config;
 
-		if (git_config_find_xdg_r(&xdg_buf) == 0)
-			xdg_config_path = xdg_buf.ptr;
+		git_config_find_global_r(&global_buf);
+		git_config_find_xdg_r(&xdg_buf);
+		git_config_find_system_r(&system_buf);
 
-		if (git_config_find_system_r(&system_buf) == 0)
-			system_config_path = system_buf.ptr;
+		error = load_config(
+			&config, repo,
+			path_unless_empty(&global_buf),
+			path_unless_empty(&xdg_buf),
+			path_unless_empty(&system_buf));
+		if (!error) {
+			GIT_REFCOUNT_OWN(config, repo);
 
-		res = load_config(&repo->_config, repo, global_config_path, xdg_config_path, system_config_path);
+			config = git__compare_and_swap(&repo->_config, NULL, config);
+			if (config != NULL) {
+				GIT_REFCOUNT_OWN(config, NULL);
+				git_config_free(config);
+			}
+		}
 
 		git_buf_free(&global_buf);
 		git_buf_free(&xdg_buf);
 		git_buf_free(&system_buf);
-
-		if (res < 0)
-			return -1;
-
-		GIT_REFCOUNT_OWN(repo->_config, repo);
 	}
 
 	*out = repo->_config;
-	return 0;
+	return error;
 }
 
 int git_repository_config(git_config **out, git_repository *repo)
@@ -596,36 +620,37 @@ int git_repository_config(git_config **out, git_repository *repo)
 void git_repository_set_config(git_repository *repo, git_config *config)
 {
 	assert(repo && config);
-
-	drop_config(repo);
-
-	repo->_config = config;
-	GIT_REFCOUNT_OWN(repo->_config, repo);
-	GIT_REFCOUNT_INC(repo->_config);
+	set_config(repo, config);
 }
 
 int git_repository_odb__weakptr(git_odb **out, git_repository *repo)
 {
+	int error = 0;
+
 	assert(repo && out);
 
 	if (repo->_odb == NULL) {
 		git_buf odb_path = GIT_BUF_INIT;
-		int res;
+		git_odb *odb;
 
-		if (git_buf_joinpath(&odb_path, repo->path_repository, GIT_OBJECTS_DIR) < 0)
-			return -1;
+		git_buf_joinpath(&odb_path, repo->path_repository, GIT_OBJECTS_DIR);
 
-		res = git_odb_open(&repo->_odb, odb_path.ptr);
-		git_buf_free(&odb_path); /* done with path */
+		error = git_odb_open(&odb, odb_path.ptr);
+		if (!error) {
+			GIT_REFCOUNT_OWN(odb, repo);
 
-		if (res < 0)
-			return -1;
+			odb = git__compare_and_swap(&repo->_odb, NULL, odb);
+			if (odb != NULL) {
+				GIT_REFCOUNT_OWN(odb, NULL);
+				git_odb_free(odb);
+			}
+		}
 
-		GIT_REFCOUNT_OWN(repo->_odb, repo);
+		git_buf_free(&odb_path);
 	}
 
 	*out = repo->_odb;
-	return 0;
+	return error;
 }
 
 int git_repository_odb(git_odb **out, git_repository *repo)
@@ -640,31 +665,32 @@ int git_repository_odb(git_odb **out, git_repository *repo)
 void git_repository_set_odb(git_repository *repo, git_odb *odb)
 {
 	assert(repo && odb);
-
-	drop_odb(repo);
-
-	repo->_odb = odb;
-	GIT_REFCOUNT_OWN(repo->_odb, repo);
-	GIT_REFCOUNT_INC(odb);
+	set_odb(repo, odb);
 }
 
 int git_repository_refdb__weakptr(git_refdb **out, git_repository *repo)
 {
+	int error = 0;
+
 	assert(out && repo);
 
 	if (repo->_refdb == NULL) {
-		int res;
-
-		res = git_refdb_open(&repo->_refdb, repo);
+		git_refdb *refdb;
 
-		if (res < 0)
-			return -1;
+		error = git_refdb_open(&refdb, repo);
+		if (!error) {
+			GIT_REFCOUNT_OWN(refdb, repo);
 
-		GIT_REFCOUNT_OWN(repo->_refdb, repo);
+			refdb = git__compare_and_swap(&repo->_refdb, NULL, refdb);
+			if (refdb != NULL) {
+				GIT_REFCOUNT_OWN(refdb, NULL);
+				git_refdb_free(refdb);
+			}
+		}
 	}
 
 	*out = repo->_refdb;
-	return 0;
+	return error;
 }
 
 int git_repository_refdb(git_refdb **out, git_repository *repo)
@@ -678,40 +704,40 @@ int git_repository_refdb(git_refdb **out, git_repository *repo)
 
 void git_repository_set_refdb(git_repository *repo, git_refdb *refdb)
 {
-	assert (repo && refdb);
-
-	drop_refdb(repo);
-
-	repo->_refdb = refdb;
-	GIT_REFCOUNT_OWN(repo->_refdb, repo);
-	GIT_REFCOUNT_INC(refdb);
+	assert(repo && refdb);
+	set_refdb(repo, refdb);
 }
 
 int git_repository_index__weakptr(git_index **out, git_repository *repo)
 {
+	int error = 0;
+
 	assert(out && repo);
 
 	if (repo->_index == NULL) {
-		int res;
 		git_buf index_path = GIT_BUF_INIT;
+		git_index *index;
 
-		if (git_buf_joinpath(&index_path, repo->path_repository, GIT_INDEX_FILE) < 0)
-			return -1;
+		git_buf_joinpath(&index_path, repo->path_repository, GIT_INDEX_FILE);
 
-		res = git_index_open(&repo->_index, index_path.ptr);
-		git_buf_free(&index_path); /* done with path */
+		error = git_index_open(&index, index_path.ptr);
+		if (!error) {
+			GIT_REFCOUNT_OWN(index, repo);
 
-		if (res < 0)
-			return -1;
+			index = git__compare_and_swap(&repo->_index, NULL, index);
+			if (index != NULL) {
+				GIT_REFCOUNT_OWN(index, NULL);
+				git_index_free(index);
+			}
 
-		GIT_REFCOUNT_OWN(repo->_index, repo);
+			error = git_index_set_caps(repo->_index, GIT_INDEXCAP_FROM_OWNER);
+		}
 
-		if (git_index_set_caps(repo->_index, GIT_INDEXCAP_FROM_OWNER) < 0)
-			return -1;
+		git_buf_free(&index_path);
 	}
 
 	*out = repo->_index;
-	return 0;
+	return error;
 }
 
 int git_repository_index(git_index **out, git_repository *repo)
@@ -726,12 +752,7 @@ int git_repository_index(git_index **out, git_repository *repo)
 void git_repository_set_index(git_repository *repo, git_index *index)
 {
 	assert(repo && index);
-
-	drop_index(repo);
-
-	repo->_index = index;
-	GIT_REFCOUNT_OWN(repo->_index, repo);
-	GIT_REFCOUNT_INC(index);
+	set_index(repo, index);
 }
 
 static int check_repositoryformatversion(git_config *config)
@@ -1421,14 +1442,13 @@ static int at_least_one_cb(const char *refname, void *payload)
 
 static int repo_contains_no_reference(git_repository *repo)
 {
-	int error;
-	
-	error = git_reference_foreach(repo, GIT_REF_LISTALL, at_least_one_cb, NULL);
+	int error = git_reference_foreach(repo, GIT_REF_LISTALL, at_least_one_cb, NULL);
 
 	if (error == GIT_EUSER)
 		return 0;
-
-	return error == 0 ? 1 : error;
+	if (!error)
+		return 1;
+	return error;
 }
 
 int git_repository_is_empty(git_repository *repo)

src/tag.c

@@ -15,8 +15,9 @@
 #include "git2/signature.h"
 #include "git2/odb_backend.h"
 
-void git_tag__free(git_tag *tag)
+void git_tag__free(void *_tag)
 {
+	git_tag *tag = _tag;
 	git_signature_free(tag->tagger);
 	git__free(tag->message);
 	git__free(tag->tag_name);
@@ -69,7 +70,7 @@ static int tag_error(const char *str)
 	return -1;
 }
 
-int git_tag__parse_buffer(git_tag *tag, const char *buffer, size_t length)
+static int tag_parse(git_tag *tag, const char *buffer, const char *buffer_end)
 {
 	static const char *tag_types[] = {
 		NULL, "commit\n", "tree\n", "blob\n", "tag\n"
@@ -79,8 +80,6 @@ int git_tag__parse_buffer(git_tag *tag, const char *buffer, size_t length)
 	size_t text_len;
 	char *search;
 
-	const char *buffer_end = buffer + length;
-
 	if (git_oid__parse(&tag->target, &buffer, buffer_end, "object ") < 0)
 		return tag_error("Object field invalid");
 
@@ -157,6 +156,15 @@ int git_tag__parse_buffer(git_tag *tag, const char *buffer, size_t length)
 	return 0;
 }
 
+int git_tag__parse(void *_tag, git_odb_object *odb_obj)
+{
+	git_tag *tag = _tag;
+	const char *buffer = git_odb_object_data(odb_obj);
+	const char *buffer_end = buffer + git_odb_object_size(odb_obj);
+
+	return tag_parse(tag, buffer, buffer_end);
+}
+
 static int retrieve_tag_reference(
 	git_reference **tag_reference_out,
 	git_buf *ref_name_out,
@@ -317,14 +325,14 @@ int git_tag_create_frombuffer(git_oid *oid, git_repository *repo, const char *bu
 		return -1;
 
 	/* validate the buffer */
-	if (git_tag__parse_buffer(&tag, buffer, strlen(buffer)) < 0)
+	if (tag_parse(&tag, buffer, buffer + strlen(buffer)) < 0)
 		return -1;
 
 	/* validate the target */
 	if (git_odb_read(&target_obj, odb, &tag.target) < 0)
 		goto on_error;
 
-	if (tag.type != target_obj->raw.type) {
+	if (tag.type != target_obj->cached.type) {
 		giterr_set(GITERR_TAG, "The type for the given target is invalid");
 		goto on_error;
 	}
@@ -394,12 +402,6 @@ int git_tag_delete(git_repository *repo, const char *tag_name)
 	return error;
 }
 
-int git_tag__parse(git_tag *tag, git_odb_object *obj)
-{
-	assert(tag);
-	return git_tag__parse_buffer(tag, obj->raw.data, obj->raw.len);
-}
-
 typedef struct {
 	git_repository *repo;
 	git_tag_foreach_cb cb;

src/tag.h

@@ -22,8 +22,7 @@ struct git_tag {
 	char *message;
 };
 
-void git_tag__free(git_tag *tag);
-int git_tag__parse(git_tag *tag, git_odb_object *obj);
-int git_tag__parse_buffer(git_tag *tag, const char *data, size_t len);
+void git_tag__free(void *tag);
+int git_tag__parse(void *tag, git_odb_object *obj);
 
 #endif

src/thread-utils.h

@@ -68,6 +68,20 @@ GIT_INLINE(int) git_atomic_dec(git_atomic *a)
 #endif
 }
 
+GIT_INLINE(void *) git___compare_and_swap(
+	volatile void **ptr, void *oldval, void *newval)
+{
+	volatile void *foundval;
+#if defined(GIT_WIN32)
+	foundval = InterlockedCompareExchangePointer(ptr, newval, oldval);
+#elif defined(__GNUC__)
+	foundval = __sync_val_compare_and_swap(ptr, oldval, newval);
+#else
+#	error "Unsupported architecture for atomic operations"
+#endif
+	return (foundval == oldval) ? oldval : newval;
+}
+
 #else
 
 #define git_thread unsigned int
@@ -101,8 +115,34 @@ GIT_INLINE(int) git_atomic_dec(git_atomic *a)
 	return --a->val;
 }
 
+GIT_INLINE(void *) git___compare_and_swap(
+	volatile void **ptr, void *oldval, void *newval)
+{
+	if (*ptr == oldval)
+		*ptr = newval;
+	else
+		oldval = newval;
+	return oldval;
+}
+
 #endif
 
+/* Atomically replace oldval with newval
+ * @return oldval if it was replaced or newval if it was not
+ */
+#define git__compare_and_swap(P,O,N) \
+	git___compare_and_swap((volatile void **)P, O, N)
+
+#define git__swap(ptr, val) git__compare_and_swap(&ptr, ptr, val)
+
 extern int git_online_cpus(void);
 
+#if defined(GIT_THREADS) && defined(GIT_WIN32)
+# define GIT_MEMORY_BARRIER MemoryBarrier()
+#elif defined(GIT_THREADS)
+# define GIT_MEMORY_BARRIER __sync_synchronize()
+#else
+# define GIT_MEMORY_BARRIER /* noop */
+#endif
+
 #endif /* INCLUDE_thread_utils_h__ */

src/tree.c

@@ -219,8 +219,9 @@ git_tree_entry *git_tree_entry_dup(const git_tree_entry *entry)
 	return copy;
 }
 
-void git_tree__free(git_tree *tree)
+void git_tree__free(void *_tree)
 {
+	git_tree *tree = _tree;
 	size_t i;
 	git_tree_entry *e;
 
@@ -371,8 +372,12 @@ static int tree_error(const char *str, const char *path)
 	return -1;
 }
 
-static int tree_parse_buffer(git_tree *tree, const char *buffer, const char *buffer_end)
+int git_tree__parse(void *_tree, git_odb_object *odb_obj)
 {
+	git_tree *tree = _tree;
+	const char *buffer = git_odb_object_data(odb_obj);
+	const char *buffer_end = buffer + git_odb_object_size(odb_obj);
+
 	if (git_vector_init(&tree->entries, DEFAULT_TREE_SIZE, entry_sort_cmp) < 0)
 		return -1;
 
@@ -416,12 +421,6 @@ static int tree_parse_buffer(git_tree *tree, const char *buffer, const char *buf
 	return 0;
 }
 
-int git_tree__parse(git_tree *tree, git_odb_object *obj)
-{
-	assert(tree);
-	return tree_parse_buffer(tree, (char *)obj->raw.data, (char *)obj->raw.data + obj->raw.len);
-}
-
 static size_t find_next_dir(const char *dirname, git_index *index, size_t start)
 {
 	size_t dirlen, i, entries = git_index_entrycount(index);

src/tree.h

@@ -37,8 +37,8 @@ GIT_INLINE(bool) git_tree_entry__is_tree(const struct git_tree_entry *e)
 
 extern int git_tree_entry_icmp(const git_tree_entry *e1, const git_tree_entry *e2);
 
-void git_tree__free(git_tree *tree);
-int git_tree__parse(git_tree *tree, git_odb_object *obj);
+void git_tree__free(void *tree);
+int git_tree__parse(void *tree, git_odb_object *obj);
 
 /**
  * Lookup the first position in the tree with a given prefix.

src/util.c

@@ -11,6 +11,7 @@
 #include <ctype.h>
 #include "posix.h"
 #include "fileops.h"
+#include "cache.h"
 
 #ifdef _MSC_VER
 # include <Shlwapi.h>
@@ -94,12 +95,20 @@ int git_libgit2_opts(int key, ...)
 			error = git_futils_dirs_set(error, va_arg(ap, const char *));
 		break;
 
-	case GIT_OPT_GET_ODB_CACHE_SIZE:
-		*(va_arg(ap, size_t *)) = git_odb__cache_size;
+	case GIT_OPT_SET_CACHE_OBJECT_LIMIT:
+		{
+			git_otype type = (git_otype)va_arg(ap, int);
+			size_t size = va_arg(ap, size_t);
+			error = git_cache_set_max_object_size(type, size);
+			break;
+		}
+
+	case GIT_OPT_SET_CACHE_MAX_SIZE:
+		git_cache__max_storage = va_arg(ap, size_t);
 		break;
 
-	case GIT_OPT_SET_ODB_CACHE_SIZE:
-		git_odb__cache_size = va_arg(ap, size_t);
+	case GIT_OPT_ENABLE_CACHING:
+		git_cache__enabled = (va_arg(ap, int) != 0);
 		break;
 	}
 

src/win32/pthread.c

@@ -14,21 +14,27 @@ int pthread_create(
 	void *GIT_RESTRICT arg)
 {
 	GIT_UNUSED(attr);
-	*thread = (pthread_t) CreateThread(
+	*thread = CreateThread(
 		NULL, 0, (LPTHREAD_START_ROUTINE)start_routine, arg, 0, NULL);
 	return *thread ? 0 : -1;
 }
 
 int pthread_join(pthread_t thread, void **value_ptr)
 {
-	int ret;
-	ret = WaitForSingleObject(thread, INFINITE);
-	if (ret && value_ptr)
-		GetExitCodeThread(thread, (void*) value_ptr);
-	return -(!!ret);
+	DWORD ret = WaitForSingleObject(thread, INFINITE);
+
+	if (ret == WAIT_OBJECT_0) {
+		if (value_ptr != NULL)
+			GetExitCodeThread(thread, (void *)value_ptr);
+		CloseHandle(thread);
+		return 0;
+	}
+
+	return -1;
 }
 
-int pthread_mutex_init(pthread_mutex_t *GIT_RESTRICT mutex,
+int pthread_mutex_init(
+	pthread_mutex_t *GIT_RESTRICT mutex,
 	const pthread_mutexattr_t *GIT_RESTRICT mutexattr)
 {
 	GIT_UNUSED(mutexattr);

src/win32/pthread.h

@@ -25,13 +25,16 @@ typedef HANDLE pthread_cond_t;
 
 #define PTHREAD_MUTEX_INITIALIZER {(void*)-1};
 
-int pthread_create(pthread_t *GIT_RESTRICT,
+int pthread_create(
+	pthread_t *GIT_RESTRICT,
 	const pthread_attr_t *GIT_RESTRICT,
-					void *(*start_routine)(void*), void *__restrict);
+	void *(*start_routine)(void*),
+	void *__restrict);
 
 int pthread_join(pthread_t, void **);
 
-int pthread_mutex_init(pthread_mutex_t *GIT_RESTRICT, const pthread_mutexattr_t *GIT_RESTRICT);
+int pthread_mutex_init(
+	pthread_mutex_t *GIT_RESTRICT, const pthread_mutexattr_t *GIT_RESTRICT);
 int pthread_mutex_destroy(pthread_mutex_t *);
 int pthread_mutex_lock(pthread_mutex_t *);
 int pthread_mutex_unlock(pthread_mutex_t *);

tests-clar/commit/parse.c

@@ -264,37 +264,40 @@ gpgsig -----BEGIN PGP SIGNATURE-----\n\
 a simple commit which works\n",
 };
 
-void test_commit_parse__entire_commit(void)
+static int parse_commit(git_commit **out, const char *buffer)
 {
-	const int broken_commit_count = sizeof(failing_commit_cases) / sizeof(*failing_commit_cases);
-	const int working_commit_count = sizeof(passing_commit_cases) / sizeof(*passing_commit_cases);
-	int i;
-
-	for (i = 0; i < broken_commit_count; ++i) {
 	git_commit *commit;
+	git_odb_object fake_odb_object;
+	int error;
+
 	commit = (git_commit*)git__malloc(sizeof(git_commit));
 	memset(commit, 0x0, sizeof(git_commit));
 	commit->object.repo = g_repo;
 
-		cl_git_fail(git_commit__parse_buffer(
-			commit, failing_commit_cases[i], strlen(failing_commit_cases[i]))
-		);
+	memset(&fake_odb_object, 0x0, sizeof(git_odb_object));
+	fake_odb_object.buffer = (char *)buffer;
+	fake_odb_object.cached.size = strlen(fake_odb_object.buffer);
 
-		git_commit__free(commit);
-	}
+	error = git_commit__parse(commit, &fake_odb_object);
+
+	*out = commit;
+	return error;
+}
 
-	for (i = 0; i < working_commit_count; ++i) {
+void test_commit_parse__entire_commit(void)
+{
+	const int failing_commit_count = ARRAY_SIZE(failing_commit_cases);
+	const int passing_commit_count = ARRAY_SIZE(passing_commit_cases);
+	int i;
 	git_commit *commit;
 
-		commit = (git_commit*)git__malloc(sizeof(git_commit));
-		memset(commit, 0x0, sizeof(git_commit));
-		commit->object.repo = g_repo;
+	for (i = 0; i < failing_commit_count; ++i) {
+		cl_git_fail(parse_commit(&commit, failing_commit_cases[i]));
+		git_commit__free(commit);
+	}
 
-		cl_git_pass(git_commit__parse_buffer(
-                     commit,
-                     passing_commit_cases[i],
-                     strlen(passing_commit_cases[i]))
-         );
+	for (i = 0; i < passing_commit_count; ++i) {
+		cl_git_pass(parse_commit(&commit, passing_commit_cases[i]));
 
 		if (!i)
 			cl_assert_equal_s("", git_commit_message(commit));
@@ -387,9 +390,7 @@ This commit has a few LF at the start of the commit message";
 	memset(commit, 0x0, sizeof(git_commit));
 	commit->object.repo = g_repo;
 
-	cl_git_pass(git_commit__parse_buffer(commit, buffer, strlen(buffer)));
-
+	cl_git_pass(parse_commit(&commit, buffer));
 	cl_assert_equal_s(message, git_commit_message(commit));
-
 	git_commit__free(commit);
 }

tests-clar/core/oidmap.c

+#include "clar_libgit2.h"
+#include "oidmap.h"
+
+GIT__USE_OIDMAP;
+
+typedef struct {
+	git_oid oid;
+	size_t extra;
+} oidmap_item;
+
+#define NITEMS 0x0fff
+
+void test_core_oidmap__basic(void)
+{
+	git_oidmap *map;
+	oidmap_item items[NITEMS];
+	uint32_t i, j;
+
+	for (i = 0; i < NITEMS; ++i) {
+		items[i].extra = i;
+		for (j = 0; j < GIT_OID_RAWSZ / 4; ++j) {
+			items[i].oid.id[j * 4    ] = (unsigned char)i;
+			items[i].oid.id[j * 4 + 1] = (unsigned char)(i >> 8);
+			items[i].oid.id[j * 4 + 2] = (unsigned char)(i >> 16);
+			items[i].oid.id[j * 4 + 3] = (unsigned char)(i >> 24);
+		}
+	}
+
+	map = git_oidmap_alloc();
+	cl_assert(map != NULL);
+
+	for (i = 0; i < NITEMS; ++i) {
+		khiter_t pos;
+		int ret;
+
+		pos = kh_get(oid, map, &items[i].oid);
+		cl_assert(pos == kh_end(map));
+
+		pos = kh_put(oid, map, &items[i].oid, &ret);
+		cl_assert(ret != 0);
+
+		kh_val(map, pos) = &items[i];
+	}
+
+
+	for (i = 0; i < NITEMS; ++i) {
+		khiter_t pos;
+
+		pos = kh_get(oid, map, &items[i].oid);
+		cl_assert(pos != kh_end(map));
+
+		cl_assert_equal_p(kh_val(map, pos), &items[i]);
+	}
+
+	git_oidmap_free(map);
+}
+
+void test_core_oidmap__hash_collision(void)
+{
+	git_oidmap *map;
+	oidmap_item items[NITEMS];
+	uint32_t i, j;
+
+	for (i = 0; i < NITEMS; ++i) {
+		uint32_t segment = i / 8;
+		int modi = i - (segment * 8);
+
+		items[i].extra = i;
+
+		for (j = 0; j < GIT_OID_RAWSZ / 4; ++j) {
+			items[i].oid.id[j * 4    ] = (unsigned char)modi;
+			items[i].oid.id[j * 4 + 1] = (unsigned char)(modi >> 8);
+			items[i].oid.id[j * 4 + 2] = (unsigned char)(modi >> 16);
+			items[i].oid.id[j * 4 + 3] = (unsigned char)(modi >> 24);
+		}
+
+		items[i].oid.id[ 8] = (unsigned char)i;
+		items[i].oid.id[ 9] = (unsigned char)(i >> 8);
+		items[i].oid.id[10] = (unsigned char)(i >> 16);
+		items[i].oid.id[11] = (unsigned char)(i >> 24);
+	}
+
+	map = git_oidmap_alloc();
+	cl_assert(map != NULL);
+
+	for (i = 0; i < NITEMS; ++i) {
+		khiter_t pos;
+		int ret;
+
+		pos = kh_get(oid, map, &items[i].oid);
+		cl_assert(pos == kh_end(map));
+
+		pos = kh_put(oid, map, &items[i].oid, &ret);
+		cl_assert(ret != 0);
+
+		kh_val(map, pos) = &items[i];
+	}
+
+
+	for (i = 0; i < NITEMS; ++i) {
+		khiter_t pos;
+
+		pos = kh_get(oid, map, &items[i].oid);
+		cl_assert(pos != kh_end(map));
+
+		cl_assert_equal_p(kh_val(map, pos), &items[i]);
+	}
+
+	git_oidmap_free(map);
+}

tests-clar/core/opts.c

@@ -16,15 +16,4 @@ void test_core_opts__readwrite(void)
 	git_libgit2_opts(GIT_OPT_GET_MWINDOW_SIZE, &new_val);
 
 	cl_assert(new_val == old_val);
-
-	git_libgit2_opts(GIT_OPT_GET_ODB_CACHE_SIZE, &old_val);
-
-	cl_assert(old_val == GIT_DEFAULT_CACHE_SIZE);
-
-	git_libgit2_opts(GIT_OPT_SET_ODB_CACHE_SIZE, (size_t)GIT_DEFAULT_CACHE_SIZE*2);
-	git_libgit2_opts(GIT_OPT_GET_ODB_CACHE_SIZE, &new_val);
-
-	cl_assert(new_val == (GIT_DEFAULT_CACHE_SIZE*2));
-
-	git_libgit2_opts(GIT_OPT_GET_ODB_CACHE_SIZE, &old_val);
 }

tests-clar/diff/workdir.c

@@ -908,7 +908,6 @@ void test_diff_workdir__can_diff_empty_file(void)
 	/* baseline - make sure there are no outstanding diffs */
 
 	cl_git_pass(git_diff_tree_to_workdir(&diff, g_repo, tree, &opts));
-	git_tree_free(tree);
 	cl_assert_equal_i(2, (int)git_diff_num_deltas(diff));
 	git_diff_list_free(diff);
 
@@ -935,6 +934,8 @@ void test_diff_workdir__can_diff_empty_file(void)
 	cl_git_pass(git_diff_get_patch(&patch, NULL, diff, 1));
 	git_diff_patch_free(patch);
 	git_diff_list_free(diff);
+
+	git_tree_free(tree);
 }
 
 void test_diff_workdir__to_index_issue_1397(void)

tests-clar/object/cache.c

+#include "clar_libgit2.h"
+#include "repository.h"
+
+static git_repository *g_repo;
+
+void test_object_cache__initialize(void)
+{
+   cl_git_pass(git_repository_open(&g_repo, cl_fixture("testrepo.git")));
+}
+
+void test_object_cache__cleanup(void)
+{
+	git_repository_free(g_repo);
+	g_repo = NULL;
+
+	git_libgit2_opts(GIT_OPT_SET_CACHE_OBJECT_LIMIT, (int)GIT_OBJ_BLOB, (size_t)0);
+}
+
+static struct {
+	git_otype type;
+	const char *sha;
+} g_data[] = {
+	/* HEAD */
+	{ GIT_OBJ_BLOB, "a8233120f6ad708f843d861ce2b7228ec4e3dec6" }, /* README */
+	{ GIT_OBJ_BLOB, "3697d64be941a53d4ae8f6a271e4e3fa56b022cc" }, /* branch_file.txt */
+	{ GIT_OBJ_BLOB, "a71586c1dfe8a71c6cbf6c129f404c5642ff31bd" }, /* new.txt */
+
+	/* refs/heads/subtrees */
+	{ GIT_OBJ_BLOB, "1385f264afb75a56a5bec74243be9b367ba4ca08" }, /* README */
+	{ GIT_OBJ_TREE, "f1425cef211cc08caa31e7b545ffb232acb098c3" }, /* ab */
+	{ GIT_OBJ_BLOB, "d6c93164c249c8000205dd4ec5cbca1b516d487f" }, /* ab/4.txt */
+	{ GIT_OBJ_TREE, "9a03079b8a8ee85a0bee58bf9be3da8b62414ed4" }, /* ab/c */
+	{ GIT_OBJ_BLOB, "270b8ea76056d5cad83af921837702d3e3c2924d" }, /* ab/c/3.txt */
+	{ GIT_OBJ_TREE, "b6361fc6a97178d8fc8639fdeed71c775ab52593" }, /* ab/de */
+	{ GIT_OBJ_BLOB, "e7b4ad382349ff96dd8199000580b9b1e2042eb0" }, /* ab/de/2.txt */
+	{ GIT_OBJ_TREE, "3259a6bd5b57fb9c1281bb7ed3167b50f224cb54" }, /* ab/de/fgh */
+	{ GIT_OBJ_BLOB, "1f67fc4386b2d171e0d21be1c447e12660561f9b" }, /* ab/de/fgh/1.txt */
+	{ GIT_OBJ_BLOB, "45b983be36b73c0788dc9cbcb76cbb80fc7bb057" }, /* branch_file.txt */
+	{ GIT_OBJ_BLOB, "fa49b077972391ad58037050f2a75f74e3671e92" }, /* new.txt */
+
+	/* refs/heads/chomped */
+	{ GIT_OBJ_BLOB, "0266163a49e280c4f5ed1e08facd36a2bd716bcf" }, /* readme.txt */
+
+	{ 0, NULL },
+	{ 0, NULL }
+};
+
+void test_object_cache__cache_everything(void)
+{
+	int i, start;
+	git_oid oid;
+	git_odb_object *odb_obj;
+	git_object *obj;
+	git_odb *odb;
+
+	git_libgit2_opts(
+		GIT_OPT_SET_CACHE_OBJECT_LIMIT, (int)GIT_OBJ_BLOB, (size_t)32767);
+
+	cl_git_pass(git_repository_odb(&odb, g_repo));
+
+	start = (int)git_cache_size(&g_repo->objects);
+
+	for (i = 0; g_data[i].sha != NULL; ++i) {
+		int count = (int)git_cache_size(&g_repo->objects);
+
+		cl_git_pass(git_oid_fromstr(&oid, g_data[i].sha));
+
+		/* alternate between loading raw and parsed objects */
+		if ((i & 1) == 0) {
+			cl_git_pass(git_odb_read(&odb_obj, odb, &oid));
+			cl_assert(g_data[i].type == git_odb_object_type(odb_obj));
+			git_odb_object_free(odb_obj);
+		} else {
+			cl_git_pass(git_object_lookup(&obj, g_repo, &oid, GIT_OBJ_ANY));
+			cl_assert(g_data[i].type == git_object_type(obj));
+			git_object_free(obj);
+		}
+
+		cl_assert_equal_i(count + 1, (int)git_cache_size(&g_repo->objects));
+	}
+
+	cl_assert_equal_i(i, git_cache_size(&g_repo->objects) - start);
+
+	git_odb_free(odb);
+
+	for (i = 0; g_data[i].sha != NULL; ++i) {
+		int count = (int)git_cache_size(&g_repo->objects);
+
+		cl_git_pass(git_oid_fromstr(&oid, g_data[i].sha));
+		cl_git_pass(git_object_lookup(&obj, g_repo, &oid, GIT_OBJ_ANY));
+		cl_assert(g_data[i].type == git_object_type(obj));
+		git_object_free(obj);
+
+		cl_assert_equal_i(count, (int)git_cache_size(&g_repo->objects));
+	}
+}
+
+void test_object_cache__cache_no_blobs(void)
+{
+	int i, start, nonblobs = 0;
+	git_oid oid;
+	git_odb_object *odb_obj;
+	git_object *obj;
+	git_odb *odb;
+
+	git_libgit2_opts(GIT_OPT_SET_CACHE_OBJECT_LIMIT, (int)GIT_OBJ_BLOB, (size_t)0);
+
+	cl_git_pass(git_repository_odb(&odb, g_repo));
+
+	start = (int)git_cache_size(&g_repo->objects);
+
+	for (i = 0; g_data[i].sha != NULL; ++i) {
+		int count = (int)git_cache_size(&g_repo->objects);
+
+		cl_git_pass(git_oid_fromstr(&oid, g_data[i].sha));
+
+		/* alternate between loading raw and parsed objects */
+		if ((i & 1) == 0) {
+			cl_git_pass(git_odb_read(&odb_obj, odb, &oid));
+			cl_assert(g_data[i].type == git_odb_object_type(odb_obj));
+			git_odb_object_free(odb_obj);
+		} else {
+			cl_git_pass(git_object_lookup(&obj, g_repo, &oid, GIT_OBJ_ANY));
+			cl_assert(g_data[i].type == git_object_type(obj));
+			git_object_free(obj);
+		}
+
+		if (g_data[i].type == GIT_OBJ_BLOB)
+			cl_assert_equal_i(count, (int)git_cache_size(&g_repo->objects));
+		else {
+			cl_assert_equal_i(count + 1, (int)git_cache_size(&g_repo->objects));
+			nonblobs++;
+		}
+	}
+
+	cl_assert_equal_i(nonblobs, git_cache_size(&g_repo->objects) - start);
+
+	git_odb_free(odb);
+}
+
+static void *cache_parsed(void *arg)
+{
+	int i;
+	git_oid oid;
+	git_object *obj;
+
+	for (i = ((int *)arg)[1]; g_data[i].sha != NULL; i += 2) {
+		cl_git_pass(git_oid_fromstr(&oid, g_data[i].sha));
+		cl_git_pass(git_object_lookup(&obj, g_repo, &oid, GIT_OBJ_ANY));
+		cl_assert(g_data[i].type == git_object_type(obj));
+		git_object_free(obj);
+	}
+
+	for (i = 0; i < ((int *)arg)[1]; i += 2) {
+		cl_git_pass(git_oid_fromstr(&oid, g_data[i].sha));
+		cl_git_pass(git_object_lookup(&obj, g_repo, &oid, GIT_OBJ_ANY));
+		cl_assert(g_data[i].type == git_object_type(obj));
+		git_object_free(obj);
+	}
+
+	return arg;
+}
+
+static void *cache_raw(void *arg)
+{
+	int i;
+	git_oid oid;
+	git_odb *odb;
+	git_odb_object *odb_obj;
+
+	cl_git_pass(git_repository_odb(&odb, g_repo));
+
+	for (i = ((int *)arg)[1]; g_data[i].sha != NULL; i += 2) {
+		cl_git_pass(git_oid_fromstr(&oid, g_data[i].sha));
+		cl_git_pass(git_odb_read(&odb_obj, odb, &oid));
+		cl_assert(g_data[i].type == git_odb_object_type(odb_obj));
+		git_odb_object_free(odb_obj);
+	}
+
+	for (i = 0; i < ((int *)arg)[1]; i += 2) {
+		cl_git_pass(git_oid_fromstr(&oid, g_data[i].sha));
+		cl_git_pass(git_odb_read(&odb_obj, odb, &oid));
+		cl_assert(g_data[i].type == git_odb_object_type(odb_obj));
+		git_odb_object_free(odb_obj);
+	}
+
+	git_odb_free(odb);
+
+	return arg;
+}
+
+#define REPEAT 50
+#define THREADCOUNT 20
+
+void test_object_cache__threadmania(void)
+{
+	int try, th, max_i;
+	void *data;
+	void *(*fn)(void *);
+
+#ifdef GIT_THREADS
+	git_thread t[THREADCOUNT];
+#endif
+
+	for (max_i = 0; g_data[max_i].sha != NULL; ++max_i)
+		/* count up */;
+
+	for (try = 0; try < REPEAT; ++try) {
+
+		for (th = 0; th < THREADCOUNT; ++th) {
+			data = git__malloc(2 * sizeof(int));
+
+			((int *)data)[0] = th;
+			((int *)data)[1] = th % max_i;
+
+			fn = (th & 1) ? cache_parsed : cache_raw;
+
+#ifdef GIT_THREADS
+			cl_git_pass(git_thread_create(&t[th], NULL, fn, data));
+#else
+			cl_assert(fn(data) == data);
+			git__free(data);
+#endif
+		}
+
+#ifdef GIT_THREADS
+		for (th = 0; th < THREADCOUNT; ++th) {
+			cl_git_pass(git_thread_join(t[th], &data));
+			cl_assert_equal_i(th, ((int *)data)[0]);
+			git__free(data);
+		}
+#endif
+
+	}
+}

tests-clar/object/raw/write.c

@@ -63,6 +63,7 @@ void test_body(object_data *d, git_rawobj *o)
    git_odb *db;
    git_oid id1, id2;
    git_odb_object *obj;
+   git_rawobj tmp;
 
    make_odb_dir();
    cl_git_pass(git_odb_open(&db, odb_dir));
@@ -73,7 +74,12 @@ void test_body(object_data *d, git_rawobj *o)
    check_object_files(d);
 
    cl_git_pass(git_odb_read(&obj, db, &id1));
-   cmp_objects(&obj->raw, o);
+
+   tmp.data = obj->buffer;
+   tmp.len = obj->cached.size;
+   tmp.type = obj->cached.type;
+
+   cmp_objects(&tmp, o);
 
    git_odb_object_free(obj);
    git_odb_free(db);

tests-clar/odb/loose.c

@@ -30,6 +30,7 @@ static void test_read_object(object_data *data)
     git_oid id;
     git_odb_object *obj;
 	git_odb *odb;
+	git_rawobj tmp;
 
     write_object_files(data);
 
@@ -37,7 +38,11 @@ static void test_read_object(object_data *data)
     cl_git_pass(git_oid_fromstr(&id, data->id));
     cl_git_pass(git_odb_read(&obj, odb, &id));
 
-    cmp_objects((git_rawobj *)&obj->raw, data);
+	tmp.data = obj->buffer;
+	tmp.len = obj->cached.size;
+	tmp.type = obj->cached.type;
+
+    cmp_objects(&tmp, data);
 
     git_odb_object_free(obj);
 	git_odb_free(odb);

tests-clar/odb/packed.c

@@ -46,8 +46,8 @@ void test_odb_packed__read_header_0(void)
 		cl_git_pass(git_odb_read(&obj, _odb, &id));
 		cl_git_pass(git_odb_read_header(&len, &type, _odb, &id));
 
-		cl_assert(obj->raw.len == len);
-		cl_assert(obj->raw.type == type);
+		cl_assert(obj->cached.size == len);
+		cl_assert(obj->cached.type == type);
 
 		git_odb_object_free(obj);
 	}
@@ -70,8 +70,8 @@ void test_odb_packed__read_header_1(void)
 		cl_git_pass(git_odb_read(&obj, _odb, &id));
 		cl_git_pass(git_odb_read_header(&len, &type, _odb, &id));
 
-		cl_assert(obj->raw.len == len);
-		cl_assert(obj->raw.type == type);
+		cl_assert(obj->cached.size == len);
+		cl_assert(obj->cached.type == type);
 
 		git_odb_object_free(obj);
 	}

tests-clar/odb/packed_one.c

@@ -52,8 +52,8 @@ void test_odb_packed_one__read_header_0(void)
 		cl_git_pass(git_odb_read(&obj, _odb, &id));
 		cl_git_pass(git_odb_read_header(&len, &type, _odb, &id));
 
-		cl_assert(obj->raw.len == len);
-		cl_assert(obj->raw.type == type);
+		cl_assert(obj->cached.size == len);
+		cl_assert(obj->cached.type == type);
 
 		git_odb_object_free(obj);
 	}