Merge branch 'cmn/global-mwf'

src/indexer.c

@@ -18,6 +18,8 @@
 #include "oidmap.h"
 #include "zstream.h"
 
+extern git_mutex git__mwindow_mutex;
+
 #define UINT31_MAX (0x7FFFFFFF)
 
 struct entry {
@@ -1044,6 +1046,11 @@ void git_indexer_free(git_indexer *idx)
 	}
 
 	git_vector_free_deep(&idx->deltas);
-	git_packfile_free(idx->pack);
+
+	if (!git_mutex_lock(&git__mwindow_mutex)) {
+		git_packfile_free(idx->pack);
+		git_mutex_unlock(&git__mwindow_mutex);
+	}
+
 	git__free(idx);
 }

src/mwindow.c

@@ -11,6 +11,10 @@
 #include "fileops.h"
 #include "map.h"
 #include "global.h"
+#include "strmap.h"
+#include "pack.h"
+
+GIT__USE_STRMAP;
 
 #define DEFAULT_WINDOW_SIZE \
 	(sizeof(void*) >= 8 \
@@ -26,20 +30,126 @@ size_t git_mwindow__mapped_limit = DEFAULT_MAPPED_LIMIT;
 /* Whenever you want to read or modify this, grab git__mwindow_mutex */
 static git_mwindow_ctl mem_ctl;
 
-/*
- * Free all the windows in a sequence, typically because we're done
- * with the file
+/* Global list of mwindow files, to open packs once across repos */
+git_strmap *git__pack_cache = NULL;
+
+/**
+ * Run under mwindow lock
  */
-void git_mwindow_free_all(git_mwindow_file *mwf)
+int git_mwindow_files_init(void)
 {
-	git_mwindow_ctl *ctl = &mem_ctl;
-	size_t i;
+	if (git__pack_cache)
+		return 0;
+
+	return git_strmap_alloc(&git__pack_cache);
+}
+
+void git_mwindow_files_free(void)
+{
+	git_strmap *tmp = git__pack_cache;
+
+	git__pack_cache = NULL;
+	git_strmap_free(tmp);
+}
+
+int git_mwindow_get_pack(struct git_pack_file **out, const char *path)
+{
+	int error;
+	char *packname;
+	git_strmap_iter pos;
+	struct git_pack_file *pack;
+
+	if ((error = git_packfile__name(&packname, path)) < 0)
+		return error;
+
+	if (git_mutex_lock(&git__mwindow_mutex) < 0)
+		return -1;
+
+	if (git_mwindow_files_init() < 0) {
+		git_mutex_unlock(&git__mwindow_mutex);
+		return -1;
+	}
+
+	pos = git_strmap_lookup_index(git__pack_cache, packname);
+	git__free(packname);
+
+	if (git_strmap_valid_index(git__pack_cache, pos)) {
+		pack = git_strmap_value_at(git__pack_cache, pos);
+		git_atomic_inc(&pack->refcount);
+
+		git_mutex_unlock(&git__mwindow_mutex);
+		*out = pack;
+		return 0;
+	}
+
+	/* If we didn't find it, we need to create it */
+	if ((error = git_packfile_alloc(&pack, path)) < 0) {
+		git_mutex_unlock(&git__mwindow_mutex);
+		return error;
+	}
+
+	git_atomic_inc(&pack->refcount);
+
+	git_strmap_insert(git__pack_cache, pack->pack_name, pack, error);
+	git_mutex_unlock(&git__mwindow_mutex);
+
+	if (error < 0)
+		return -1;
 
+	*out = pack;
+	return 0;
+}
+
+int git_mwindow_put_pack(struct git_pack_file *pack)
+{
+	int count;
+	git_strmap_iter pos;
+
+	if (git_mutex_lock(&git__mwindow_mutex) < 0)
+		return -1;
+
+	if (git_mwindow_files_init() < 0) {
+		git_mutex_unlock(&git__mwindow_mutex);
+		return -1;
+	}
+
+	pos = git_strmap_lookup_index(git__pack_cache, pack->pack_name);
+	if (!git_strmap_valid_index(git__pack_cache, pos)) {
+		git_mutex_unlock(&git__mwindow_mutex);
+		return GIT_ENOTFOUND;
+	}
+
+	count = git_atomic_dec(&pack->refcount);
+	if (count == 0) {
+		git_strmap_delete_at(git__pack_cache, pos);
+		git_packfile_free(pack);
+	}
+
+	git_mutex_unlock(&git__mwindow_mutex);
+	return 0;
+}
+
+void git_mwindow_free_all(git_mwindow_file *mwf)
+{
 	if (git_mutex_lock(&git__mwindow_mutex)) {
 		giterr_set(GITERR_THREAD, "unable to lock mwindow mutex");
 		return;
 	}
 
+	git_mwindow_free_all_locked(mwf);
+
+	git_mutex_unlock(&git__mwindow_mutex);
+}
+
+/*
+ * Free all the windows in a sequence, typically because we're done
+ * with the file
+ */
+void git_mwindow_free_all_locked(git_mwindow_file *mwf)
+{
+	git_mwindow_ctl *ctl = &mem_ctl;
+	size_t i;
+
 	/*
 	 * Remove these windows from the global list
 	 */
@@ -67,8 +177,6 @@ void git_mwindow_free_all(git_mwindow_file *mwf)
 		mwf->windows = w->next;
 		git__free(w);
 	}
-
-	git_mutex_unlock(&git__mwindow_mutex);
 }
 
 /*

src/mwindow.h

@@ -36,10 +36,18 @@ typedef struct git_mwindow_ctl {
 } git_mwindow_ctl;
 
 int git_mwindow_contains(git_mwindow *win, git_off_t offset);
-void git_mwindow_free_all(git_mwindow_file *mwf);
+void git_mwindow_free_all(git_mwindow_file *mwf); /* locks */
+void git_mwindow_free_all_locked(git_mwindow_file *mwf); /* run under lock */
 unsigned char *git_mwindow_open(git_mwindow_file *mwf, git_mwindow **cursor, git_off_t offset, size_t extra, unsigned int *left);
 int git_mwindow_file_register(git_mwindow_file *mwf);
 void git_mwindow_file_deregister(git_mwindow_file *mwf);
 void git_mwindow_close(git_mwindow **w_cursor);
 
+int git_mwindow_files_init(void);
+void git_mwindow_files_free(void);
+
+struct git_pack_file; /* just declaration to avoid cyclical includes */
+int git_mwindow_get_pack(struct git_pack_file **out, const char *path);
+int git_mwindow_put_pack(struct git_pack_file *pack);
+
 #endif

src/odb_pack.c

@@ -210,7 +210,7 @@ static int packfile_load__cb(void *data, git_buf *path)
 			return 0;
 	}
 
-	error = git_packfile_alloc(&pack, path->ptr);
+	error = git_mwindow_get_pack(&pack, path->ptr);
 
 	/* ignore missing .pack file as git does */
 	if (error == GIT_ENOTFOUND) {
@@ -605,7 +605,7 @@ static void pack_backend__free(git_odb_backend *_backend)
 
 	for (i = 0; i < backend->packs.length; ++i) {
 		struct git_pack_file *p = git_vector_get(&backend->packs, i);
-		git_packfile_free(p);
+		git_mwindow_put_pack(p);
 	}
 
 	git_vector_free(&backend->packs);
@@ -647,7 +647,7 @@ int git_odb_backend_one_pack(git_odb_backend **backend_out, const char *idx)
 	if (pack_backend__alloc(&backend, 1) < 0)
 		return -1;
 
-	if (git_packfile_alloc(&packfile, idx) < 0 ||
+	if (git_mwindow_get_pack(&packfile, idx) < 0 ||
 		git_vector_insert(&backend->packs, packfile) < 0)
 	{
 		pack_backend__free((git_odb_backend *)backend);
@@ -664,6 +664,9 @@ int git_odb_backend_pack(git_odb_backend **backend_out, const char *objects_dir)
 	struct pack_backend *backend = NULL;
 	git_buf path = GIT_BUF_INIT;
 
+	if (git_mwindow_files_init() < 0)
+		return -1;
+
 	if (pack_backend__alloc(&backend, 8) < 0)
 		return -1;
 

src/pack.c

@@ -968,7 +968,7 @@ void git_packfile_free(struct git_pack_file *p)
 
 	cache_free(&p->bases);
 
-	git_mwindow_free_all(&p->mwf);
+	git_mwindow_free_all_locked(&p->mwf);
 
 	if (p->mwf.fd >= 0)
 		p_close(p->mwf.fd);
@@ -1063,6 +1063,23 @@ cleanup:
 	return -1;
 }
 
+int git_packfile__name(char **out, const char *path)
+{
+	size_t path_len;
+	git_buf buf = GIT_BUF_INIT;
+
+	path_len = strlen(path);
+
+	if (path_len < strlen(".idx"))
+		return git_odb__error_notfound("invalid packfile path", NULL);
+
+	if (git_buf_printf(&buf, "%.*s.pack", (int)(path_len - strlen(".idx")), path) < 0)
+		return -1;
+
+	*out = git_buf_detach(&buf);
+	return 0;
+}
+
 int git_packfile_alloc(struct git_pack_file **pack_out, const char *path)
 {
 	struct stat st;

src/pack.h

@@ -90,6 +90,7 @@ struct git_pack_file {
 	git_mwindow_file mwf;
 	git_map index_map;
 	git_mutex lock; /* protect updates to mwf and index_map */
+	git_atomic refcount;
 
 	uint32_t num_objects;
 	uint32_t num_bad_objects;
@@ -123,6 +124,8 @@ typedef struct git_packfile_stream {
 
 size_t git_packfile__object_header(unsigned char *hdr, size_t size, git_otype type);
 
+int git_packfile__name(char **out, const char *path);
+
 int git_packfile_unpack_header(
 		size_t *size_p,
 		git_otype *type_p,

tests/pack/sharing.c

+#include "clar_libgit2.h"
+#include <git2.h>
+#include "strmap.h"
+#include "mwindow.h"
+#include "pack.h"
+
+extern git_strmap *git__pack_cache;
+
+void test_pack_sharing__open_two_repos(void)
+{
+	git_repository *repo1, *repo2;
+	git_object *obj1, *obj2;
+	git_oid id;
+	git_strmap_iter pos;
+	void *data;
+	int error;
+
+	cl_git_pass(git_repository_open(&repo1, cl_fixture("testrepo.git")));
+	cl_git_pass(git_repository_open(&repo2, cl_fixture("testrepo.git")));
+
+	git_oid_fromstr(&id, "a65fedf39aefe402d3bb6e24df4d4f5fe4547750");
+
+	cl_git_pass(git_object_lookup(&obj1, repo1, &id, GIT_OBJ_ANY));
+	cl_git_pass(git_object_lookup(&obj2, repo2, &id, GIT_OBJ_ANY));
+
+	pos = 0;
+	while ((error = git_strmap_next(&data, &pos, git__pack_cache)) == 0) {
+		struct git_pack_file *pack = (struct git_pack_file *) data;
+
+		cl_assert_equal_i(2, pack->refcount.val);
+	}
+
+	cl_assert_equal_i(3, git_strmap_num_entries(git__pack_cache));
+
+	git_object_free(obj1);
+	git_object_free(obj2);
+	git_repository_free(repo1);
+	git_repository_free(repo2);
+
+	/* we don't want to keep the packs open after the repos go away */
+	cl_assert_equal_i(0, git_strmap_num_entries(git__pack_cache));
+}