gsoc-pack-objects WIP

include/git2.h

@@ -51,5 +51,6 @@
 #include "git2/notes.h"
 #include "git2/reset.h"
 #include "git2/message.h"
+#include "git2/pack.h"
 
 #endif

include/git2/errors.h

@@ -56,6 +56,7 @@ typedef enum {
 	GITERR_INDEXER,
 	GITERR_SSL,
 	GITERR_SUBMODULE,
+	GITERR_THREAD,
 } git_error_t;
 
 /**

include/git2/pack.h

+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#ifndef INCLUDE_git_pack_h__
+#define INCLUDE_git_pack_h__
+
+#include "common.h"
+#include "oid.h"
+
+/**
+ * @file git2/pack.h
+ * @brief Git pack management routines
+ * @defgroup git_pack Git pack management routines
+ * @ingroup Git
+ * @{
+ */
+GIT_BEGIN_DECL
+
+/**
+ * Initialize a new packbuilder
+ *
+ * @param out The new packbuilder object
+ * @param repo The repository
+ *
+ * @return 0 or an error code
+ */
+GIT_EXTERN(int) git_packbuilder_new(git_packbuilder **out, git_repository *repo);
+
+/**
+ * Set number of threads to spawn
+ *
+ * By default, libgit2 won't spawn any threads at all;
+ * when set to 0, libgit2 will autodetect the number of
+ * CPUs.
+ *
+ * @param pb The packbuilder
+ * @param n Number of threads to spawn
+ */
+GIT_EXTERN(void) git_packbuilder_set_threads(git_packbuilder *pb, unsigned int n);
+
+/**
+ * Insert a single object
+ *
+ * For an optimal pack it's mandatory to insert objects in recency order,
+ * commits followed by trees and blobs.
+ *
+ * @param pb The packbuilder
+ * @param oid The oid of the commit
+ * @param oid The name; might be NULL
+ *
+ * @return 0 or an error code
+ */
+GIT_EXTERN(int) git_packbuilder_insert(git_packbuilder *pb, const git_oid *oid,	const char *name);
+
+/**
+ * Insert a root tree object
+ *
+ * This will add the tree as well as all referenced trees and blobs.
+ *
+ * @param pb The packbuilder
+ * @param oid The oid of the root tree
+ *
+ * @return 0 or an error code
+ */
+GIT_EXTERN(int) git_packbuilder_insert_tree(git_packbuilder *pb, const git_oid *oid);
+
+/**
+ * Write the new pack and the corresponding index to path
+ *
+ * @param pb The packbuilder
+ * @param path Directory to store the new pack and index
+ *
+ * @return 0 or an error code
+ */
+GIT_EXTERN(int) git_packbuilder_write(git_packbuilder *pb, const char *file);
+
+/**
+ * Free the packbuilder and all associated data
+ *
+ * @param pb The packbuilder
+ */
+GIT_EXTERN(void) git_packbuilder_free(git_packbuilder *pb);
+
+/** @} */
+GIT_END_DECL
+#endif

include/git2/types.h

@@ -137,6 +137,9 @@ typedef struct git_reflog git_reflog;
 /** Representation of a git note */
 typedef struct git_note git_note;
 
+/** Representation of a git packbuilder */
+typedef struct git_packbuilder git_packbuilder;
+
 /** Time in a signature */
 typedef struct git_time {
 	git_time_t time; /** time in seconds from epoch */

src/pack-objects.h

+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+
+#ifndef INCLUDE_pack_objects_h__
+#define INCLUDE_pack_objects_h__
+
+#include "common.h"
+
+#include "buffer.h"
+#include "hash.h"
+
+#include "git2/oid.h"
+
+#define GIT_PACK_WINDOW 10 /* number of objects to possibly delta against */
+#define GIT_PACK_DEPTH 50 /* max delta depth */
+#define GIT_PACK_DELTA_CACHE_SIZE (256 * 1024 * 1024)
+#define GIT_PACK_DELTA_CACHE_LIMIT 1000
+#define GIT_PACK_BIG_FILE_THRESHOLD (512 * 1024 * 1024)
+
+typedef struct git_pobject {
+	git_oid id;
+	git_otype type;
+	git_off_t offset;
+
+	size_t size;
+
+	unsigned int hash; /* name hint hash */
+
+	struct git_pobject *delta; /* delta base object */
+	struct git_pobject *delta_child; /* deltified objects who bases me */
+	struct git_pobject *delta_sibling; /* other deltified objects
+					    * who uses the same base as
+					    * me */
+
+	void *delta_data;
+	unsigned long delta_size;
+	unsigned long z_delta_size;
+
+	int written:1,
+	    recursing:1,
+	    no_try_delta:1,
+	    tagged:1,
+	    filled:1;
+} git_pobject;
+
+struct git_packbuilder {
+	git_repository *repo; /* associated repository */
+	git_odb *odb; /* associated object database */
+
+	git_hash_ctx *ctx;
+
+	uint32_t nr_objects,
+		 nr_alloc,
+		 nr_written,
+		 nr_remaining;
+
+	git_pobject *object_list;
+
+	int *object_ix;
+	int object_ix_hashsz;
+
+
+	git_oid pack_oid; /* hash of written pack */
+
+	/* configs */
+	unsigned long delta_cache_size;
+	unsigned long max_delta_cache_size;
+	unsigned long cache_max_small_delta_size;
+	unsigned long big_file_threshold;
+	unsigned long window_memory_limit;
+
+	int nr_threads; /* nr of threads to use */
+
+	bool done;
+};
+
+
+int git_packbuilder_send(git_packbuilder *pb, git_transport *t);
+int git_packbuilder_write_buf(git_buf *buf, git_packbuilder *pb);
+
+#endif

tests-clar/pack/packbuilder.c

+#include "clar_libgit2.h"
+#include "iterator.h"
+#include "vector.h"
+
+static git_repository *_repo;
+static git_revwalk *_revwalker;
+static git_packbuilder *_packbuilder;
+static git_indexer *_indexer;
+static git_vector _commits;
+
+void test_pack_packbuilder__initialize(void)
+{
+	cl_git_pass(git_repository_open(&_repo, cl_fixture("testrepo.git")));
+	cl_git_pass(git_revwalk_new(&_revwalker, _repo));
+	cl_git_pass(git_packbuilder_new(&_packbuilder, _repo));
+	cl_git_pass(git_vector_init(&_commits, 0, NULL));
+}
+
+void test_pack_packbuilder__cleanup(void)
+{
+	git_oid *o;
+	unsigned int i;
+
+	git_vector_foreach(&_commits, i, o) {
+		git__free(o);
+	}
+	git_vector_free(&_commits);
+	git_packbuilder_free(_packbuilder);
+	git_revwalk_free(_revwalker);
+	git_indexer_free(_indexer);
+	git_repository_free(_repo);
+}
+
+void test_pack_packbuilder__create_pack(void)
+{
+	git_indexer_stats stats;
+	git_oid oid, *o;
+	unsigned int i;
+
+	git_revwalk_sorting(_revwalker, GIT_SORT_TIME);
+	cl_git_pass(git_revwalk_push_ref(_revwalker, "HEAD"));
+
+	while (git_revwalk_next(&oid, _revwalker) == 0) {
+		o = git__malloc(GIT_OID_RAWSZ);
+		cl_assert(o != NULL);
+		git_oid_cpy(o, &oid);
+		cl_git_pass(git_vector_insert(&_commits, o));
+	}
+
+	git_vector_foreach(&_commits, i, o) {
+		cl_git_pass(git_packbuilder_insert(_packbuilder, o, NULL));
+	}
+
+	git_vector_foreach(&_commits, i, o) {
+		git_object *obj;
+		cl_git_pass(git_object_lookup(&obj, _repo, o, GIT_OBJ_COMMIT));
+		cl_git_pass(git_packbuilder_insert_tree(_packbuilder,
+					git_commit_tree_oid((git_commit *)obj)));
+		git_object_free(obj);
+	}
+
+	cl_git_pass(git_packbuilder_write(_packbuilder, "testpack.pack"));
+
+	cl_git_pass(git_indexer_new(&_indexer, "testpack.pack"));
+	cl_git_pass(git_indexer_run(_indexer, &stats));
+	cl_git_pass(git_indexer_write(_indexer));
+}