Merge pull request #1891 from libgit2/cmn/fix-thin-packs

Add support for thin packs

examples/network/clone.c

@@ -25,13 +25,19 @@ static void print_progress(const progress_data *pd)
 		: 0.f;
 	int kbytes = pd->fetch_progress.received_bytes / 1024;
 
-	printf("net %3d%% (%4d kb, %5d/%5d)  /  idx %3d%% (%5d/%5d)  /  chk %3d%% (%4" PRIuZ "/%4" PRIuZ ") %s\n",
+	if (pd->fetch_progress.received_objects == pd->fetch_progress.total_objects) {
+		printf("Resolving deltas %d/%d\r",
+		       pd->fetch_progress.indexed_deltas,
+		       pd->fetch_progress.total_deltas);
+	} else {
+		printf("net %3d%% (%4d kb, %5d/%5d)  /  idx %3d%% (%5d/%5d)  /  chk %3d%% (%4" PRIuZ "/%4" PRIuZ ") %s\n",
 		   network_percent, kbytes,
 		   pd->fetch_progress.received_objects, pd->fetch_progress.total_objects,
 		   index_percent, pd->fetch_progress.indexed_objects, pd->fetch_progress.total_objects,
 		   checkout_percent,
 		   pd->completed_steps, pd->total_steps,
 		   pd->path);
+	}
 }
 
 static int fetch_progress(const git_transfer_progress *stats, void *payload)

examples/network/fetch.c

@@ -72,6 +72,7 @@ int fetch(git_repository *repo, int argc, char **argv)
 	const git_transfer_progress *stats;
 	struct dl_data data;
 	git_remote_callbacks callbacks = GIT_REMOTE_CALLBACKS_INIT;
+	int resolve_deltas_ln = 0;
 #ifndef _WIN32
 	pthread_t worker;
 #endif
@@ -113,10 +114,14 @@ int fetch(git_repository *repo, int argc, char **argv)
 	do {
 		usleep(10000);
 
-		if (stats->total_objects > 0)
+		if (stats->received_objects == stats->total_objects) {
+			printf("Resolving deltas %d/%d\r",
+			       stats->indexed_deltas, stats->total_deltas);
+		} else if (stats->total_objects > 0) {
 			printf("Received %d/%d objects (%d) in %" PRIuZ " bytes\r",
 			       stats->received_objects, stats->total_objects,
 				   stats->indexed_objects, stats->received_bytes);
+		}
 	} while (!data.finished);
 
 	if (data.ret < 0)
@@ -125,8 +130,13 @@ int fetch(git_repository *repo, int argc, char **argv)
 	pthread_join(worker, NULL);
 #endif
 
-	printf("\rReceived %d/%d objects in %zu bytes\n",
+	if (stats->local_objects > 0) {
+		printf("\rReceived %d/%d objects in %zu bytes (used %d local objects)\n",
+		       stats->indexed_objects, stats->total_objects, stats->received_bytes, stats->local_objects);
+	} else{
+		printf("\rReceived %d/%d objects in %zu bytes\n",
 			stats->indexed_objects, stats->total_objects, stats->received_bytes);
+	}
 
 	// Disconnect the underlying connection to prevent from idling.
 	git_remote_disconnect(remote);

examples/network/index-pack.c

@@ -46,7 +46,7 @@ int index_pack(git_repository *repo, int argc, char **argv)
 		return EXIT_FAILURE;
 	}
 
-	if (git_indexer_stream_new(&idx, ".", NULL, NULL) < 0) {
+	if (git_indexer_stream_new(&idx, ".", NULL, NULL, NULL) < 0) {
 		puts("bad idx");
 		return -1;
 	}

include/git2/indexer.h

@@ -20,12 +20,16 @@ typedef struct git_indexer_stream git_indexer_stream;
  *
  * @param out where to store the indexer instance
  * @param path to the directory where the packfile should be stored
+ * @param odb object database from which to read base objects when
+ * fixing thin packs. Pass NULL if no thin pack is expected (an error
+ * will be returned if there are bases missing)
  * @param progress_cb function to call with progress information
  * @param progress_cb_payload payload for the progress callback
  */
 GIT_EXTERN(int) git_indexer_stream_new(
 		git_indexer_stream **out,
 		const char *path,
+		git_odb *odb,
 		git_transfer_progress_callback progress_cb,
 		void *progress_cb_payload);
 

include/git2/sys/odb_backend.h

@@ -80,7 +80,7 @@ struct git_odb_backend {
 		git_odb_backend *, git_odb_foreach_cb cb, void *payload);
 
 	int (* writepack)(
-		git_odb_writepack **, git_odb_backend *,
+		git_odb_writepack **, git_odb_backend *, git_odb *odb,
 		git_transfer_progress_callback progress_cb, void *progress_payload);
 
 	void (* free)(git_odb_backend *);

include/git2/types.h

@@ -212,11 +212,21 @@ typedef struct git_remote_callbacks git_remote_callbacks;
 /**
  * This is passed as the first argument to the callback to allow the
  * user to see the progress.
+ *
+ * - total_objects: number of objects in the packfile being downloaded
+ * - indexed_objects: received objects that have been hashed
+ * - received_objects: objects which have been downloaded
+ * - local_objects: locally-available objects that have been injected
+ *    in order to fix a thin pack.
+ * - received-bytes: size of the packfile received up to now
  */
 typedef struct git_transfer_progress {
 	unsigned int total_objects;
 	unsigned int indexed_objects;
 	unsigned int received_objects;
+	unsigned int local_objects;
+	unsigned int total_deltas;
+	unsigned int indexed_deltas;
 	size_t received_bytes;
 } git_transfer_progress;
 

src/odb.c

@@ -995,7 +995,7 @@ int git_odb_write_pack(struct git_odb_writepack **out, git_odb *db, git_transfer
 
 		if (b->writepack != NULL) {
 			++writes;
-			error = b->writepack(out, b, progress_cb, progress_payload);
+			error = b->writepack(out, b, db, progress_cb, progress_payload);
 		}
 	}
 

src/odb_pack.c

@@ -541,6 +541,7 @@ static void pack_backend__writepack_free(struct git_odb_writepack *_writepack)
 
 static int pack_backend__writepack(struct git_odb_writepack **out,
 	git_odb_backend *_backend,
+        git_odb *odb,
 	git_transfer_progress_callback progress_cb,
 	void *progress_payload)
 {
@@ -557,7 +558,7 @@ static int pack_backend__writepack(struct git_odb_writepack **out,
 	GITERR_CHECK_ALLOC(writepack);
 
 	if (git_indexer_stream_new(&writepack->indexer_stream,
-		backend->pack_folder, progress_cb, progress_payload) < 0) {
+		backend->pack_folder, odb, progress_cb, progress_payload) < 0) {
 		git__free(writepack);
 		return -1;
 	}

src/pack-objects.c

@@ -232,40 +232,6 @@ int git_packbuilder_insert(git_packbuilder *pb, const git_oid *oid,
 	return 0;
 }
 
-/*
- * The per-object header is a pretty dense thing, which is
- *  - first byte: low four bits are "size",
- *    then three bits of "type",
- *    with the high bit being "size continues".
- *  - each byte afterwards: low seven bits are size continuation,
- *    with the high bit being "size continues"
- */
-static int gen_pack_object_header(
-		unsigned char *hdr,
-		unsigned long size,
-		git_otype type)
-{
-	unsigned char *hdr_base;
-	unsigned char c;
-
-	assert(type >= GIT_OBJ_COMMIT && type <= GIT_OBJ_REF_DELTA);
-
-	/* TODO: add support for chunked objects; see git.git 6c0d19b1 */
-
-	c = (unsigned char)((type << 4) | (size & 15));
-	size >>= 4;
-	hdr_base = hdr;
-
-	while (size) {
-		*hdr++ = c | 0x80;
-		c = size & 0x7f;
-		size >>= 7;
-	}
-	*hdr++ = c;
-
-	return (int)(hdr - hdr_base);
-}
-
 static int get_delta(void **out, git_odb *odb, git_pobject *po)
 {
 	git_odb_object *src = NULL, *trg = NULL;
@@ -327,7 +293,7 @@ static int write_object(git_buf *buf, git_packbuilder *pb, git_pobject *po)
 	}
 
 	/* Write header */
-	hdr_len = gen_pack_object_header(hdr, size, type);
+	hdr_len = git_packfile__object_header(hdr, size, type);
 
 	if (git_buf_put(buf, (char *)hdr, hdr_len) < 0)
 		goto on_error;
@@ -1292,7 +1258,7 @@ int git_packbuilder_write(
 	PREPARE_PACK;
 
 	if (git_indexer_stream_new(
-		&indexer, path, progress_cb, progress_cb_payload) < 0)
+		&indexer, path, pb->odb, progress_cb, progress_cb_payload) < 0)
 		return -1;
 
 	ctx.indexer = indexer;

src/pack.c

@@ -364,6 +364,38 @@ static unsigned char *pack_window_open(
 	return git_mwindow_open(&p->mwf, w_cursor, offset, 20, left);
  }
 
+/*
+ * The per-object header is a pretty dense thing, which is
+ *  - first byte: low four bits are "size",
+ *    then three bits of "type",
+ *    with the high bit being "size continues".
+ *  - each byte afterwards: low seven bits are size continuation,
+ *    with the high bit being "size continues"
+ */
+int git_packfile__object_header(unsigned char *hdr, unsigned long size, git_otype type)
+{
+	unsigned char *hdr_base;
+	unsigned char c;
+
+	assert(type >= GIT_OBJ_COMMIT && type <= GIT_OBJ_REF_DELTA);
+
+	/* TODO: add support for chunked objects; see git.git 6c0d19b1 */
+
+	c = (unsigned char)((type << 4) | (size & 15));
+	size >>= 4;
+	hdr_base = hdr;
+
+	while (size) {
+		*hdr++ = c | 0x80;
+		c = size & 0x7f;
+		size >>= 7;
+	}
+	*hdr++ = c;
+
+	return (int)(hdr - hdr_base);
+}
+
+
 static int packfile_unpack_header1(
 		unsigned long *usedp,
 		size_t *sizep,

src/pack.h

@@ -112,6 +112,8 @@ typedef struct git_packfile_stream {
 	git_mwindow *mw;
 } git_packfile_stream;
 
+int git_packfile__object_header(unsigned char *hdr, unsigned long size, git_otype type);
+
 int git_packfile_unpack_header(
 		size_t *size_p,
 		git_otype *type_p,

src/transports/smart.h

@@ -21,6 +21,7 @@
 #define GIT_CAP_INCLUDE_TAG "include-tag"
 #define GIT_CAP_DELETE_REFS "delete-refs"
 #define GIT_CAP_REPORT_STATUS "report-status"
+#define GIT_CAP_THIN_PACK "thin-pack"
 
 enum git_pkt_type {
 	GIT_PKT_CMD,
@@ -116,7 +117,8 @@ typedef struct transport_smart_caps {
 		side_band_64k:1,
 		include_tag:1,
 		delete_refs:1,
-		report_status:1;
+		report_status:1,
+		thin_pack:1;
 } transport_smart_caps;
 
 typedef int (*packetsize_cb)(size_t received, void *payload);

src/transports/smart_pkt.c

@@ -472,6 +472,9 @@ static int buffer_want_with_caps(const git_remote_head *head, transport_smart_ca
 	if (caps->include_tag)
 		git_buf_puts(&str, GIT_CAP_INCLUDE_TAG " ");
 
+	if (caps->thin_pack)
+		git_buf_puts(&str, GIT_CAP_THIN_PACK " ");
+
 	if (git_buf_oom(&str))
 		return -1;
 

src/transports/smart_protocol.c

@@ -128,6 +128,12 @@ int git_smart__detect_caps(git_pkt_ref *pkt, transport_smart_caps *caps)
 			continue;
 		}
 
+		if (!git__prefixcmp(ptr, GIT_CAP_THIN_PACK)) {
+			caps->common = caps->thin_pack = 1;
+			ptr += strlen(GIT_CAP_THIN_PACK);
+			continue;
+		}
+
 		/* We don't know this capability, so skip it */
 		ptr = strchr(ptr, ' ');
 	}

tests-clar/pack/indexer.c

+#include "clar_libgit2.h"
+#include <git2.h>
+#include "fileops.h"
+#include "hash.h"
+#include "iterator.h"
+#include "vector.h"
+#include "posix.h"
+
+
+/*
+ * This is a packfile with three objects. The second is a delta which
+ * depends on the third, which is also a delta.
+ */
+unsigned char out_of_order_pack[] = {
+  0x50, 0x41, 0x43, 0x4b, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x03,
+  0x32, 0x78, 0x9c, 0x63, 0x67, 0x00, 0x00, 0x00, 0x10, 0x00, 0x08, 0x76,
+  0xe6, 0x8f, 0xe8, 0x12, 0x9b, 0x54, 0x6b, 0x10, 0x1a, 0xee, 0x95, 0x10,
+  0xc5, 0x32, 0x8e, 0x7f, 0x21, 0xca, 0x1d, 0x18, 0x78, 0x9c, 0x63, 0x62,
+  0x66, 0x4e, 0xcb, 0xcf, 0x07, 0x00, 0x02, 0xac, 0x01, 0x4d, 0x75, 0x01,
+  0xd7, 0x71, 0x36, 0x66, 0xf4, 0xde, 0x82, 0x27, 0x76, 0xc7, 0x62, 0x2c,
+  0x10, 0xf1, 0xb0, 0x7d, 0xe2, 0x80, 0xdc, 0x78, 0x9c, 0x63, 0x62, 0x62,
+  0x62, 0xb7, 0x03, 0x00, 0x00, 0x69, 0x00, 0x4c, 0xde, 0x7d, 0xaa, 0xe4,
+  0x19, 0x87, 0x58, 0x80, 0x61, 0x09, 0x9a, 0x33, 0xca, 0x7a, 0x31, 0x92,
+  0x6f, 0xae, 0x66, 0x75
+};
+unsigned int out_of_order_pack_len = 112;
+
+/*
+ * Packfile with two objects. The second is a delta against an object
+ * which is not in the packfile
+ */
+unsigned char thin_pack[] = {
+  0x50, 0x41, 0x43, 0x4b, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x02,
+  0x32, 0x78, 0x9c, 0x63, 0x67, 0x00, 0x00, 0x00, 0x10, 0x00, 0x08, 0x76,
+  0xe6, 0x8f, 0xe8, 0x12, 0x9b, 0x54, 0x6b, 0x10, 0x1a, 0xee, 0x95, 0x10,
+  0xc5, 0x32, 0x8e, 0x7f, 0x21, 0xca, 0x1d, 0x18, 0x78, 0x9c, 0x63, 0x62,
+  0x66, 0x4e, 0xcb, 0xcf, 0x07, 0x00, 0x02, 0xac, 0x01, 0x4d, 0x42, 0x52,
+  0x3a, 0x6f, 0x39, 0xd1, 0xfe, 0x66, 0x68, 0x6b, 0xa5, 0xe5, 0xe2, 0x97,
+  0xac, 0x94, 0x6c, 0x76, 0x0b, 0x04
+};
+unsigned int thin_pack_len = 78;
+
+unsigned char base_obj[] = { 07, 076 };
+unsigned int base_obj_len = 2;
+
+void test_pack_indexer__out_of_order(void)
+{
+	git_indexer_stream *idx;
+	git_transfer_progress stats;
+
+	cl_git_pass(git_indexer_stream_new(&idx, ".", NULL, NULL, NULL));
+	cl_git_pass(git_indexer_stream_add(idx, out_of_order_pack, out_of_order_pack_len, &stats));
+	cl_git_pass(git_indexer_stream_finalize(idx, &stats));
+
+	cl_assert_equal_i(stats.total_objects, 3);
+	cl_assert_equal_i(stats.received_objects, 3);
+	cl_assert_equal_i(stats.indexed_objects, 3);
+
+	git_indexer_stream_free(idx);
+}
+
+void test_pack_indexer__fix_thin(void)
+{
+	git_indexer_stream *idx;
+	git_transfer_progress stats;
+	git_repository *repo;
+	git_odb *odb;
+	git_oid id, should_id;
+
+	cl_git_pass(git_repository_init(&repo, "thin.git", true));
+	cl_git_pass(git_repository_odb(&odb, repo));
+
+	/* Store the missing base into your ODB so the indexer can fix the pack */
+	cl_git_pass(git_odb_write(&id, odb, base_obj, base_obj_len, GIT_OBJ_BLOB));
+	git_oid_fromstr(&should_id, "e68fe8129b546b101aee9510c5328e7f21ca1d18");
+	cl_assert(!git_oid_cmp(&id, &should_id));
+
+	cl_git_pass(git_indexer_stream_new(&idx, ".", odb, NULL, NULL));
+	cl_git_pass(git_indexer_stream_add(idx, thin_pack, thin_pack_len, &stats));
+	cl_git_pass(git_indexer_stream_finalize(idx, &stats));
+
+	cl_assert_equal_i(stats.total_objects, 2);
+	cl_assert_equal_i(stats.received_objects, 2);
+	cl_assert_equal_i(stats.indexed_objects, 2);
+	cl_assert_equal_i(stats.local_objects, 1);
+
+	git_oid_fromstr(&should_id, "11f0f69b334728fdd8bc86b80499f22f29d85b15");
+	cl_assert(!git_oid_cmp(git_indexer_stream_hash(idx), &should_id));
+
+	git_indexer_stream_free(idx);
+	git_odb_free(odb);
+	git_repository_free(repo);
+
+	/*
+	 * The pack's name/hash only tells us what objects there are,
+	 * so we need to go through the packfile again in order to
+	 * figure out whether we calculated the trailer correctly.
+	 */
+	{
+		unsigned char buffer[128];
+		int fd;
+		ssize_t read;
+		git_off_t left;
+		struct stat st;
+		const char *name = "pack-11f0f69b334728fdd8bc86b80499f22f29d85b15.pack";
+
+		fd = p_open(name, O_RDONLY);
+		cl_assert(fd != -1);
+
+		cl_git_pass(p_stat(name, &st));
+		left = st.st_size;
+
+		cl_git_pass(git_indexer_stream_new(&idx, ".", NULL, NULL, NULL));
+		read = p_read(fd, buffer, sizeof(buffer));
+		cl_assert(read != -1);
+		p_close(fd);
+
+		cl_git_pass(git_indexer_stream_add(idx, buffer, read, &stats));
+		cl_git_pass(git_indexer_stream_finalize(idx, &stats));
+
+		cl_assert_equal_i(stats.total_objects, 3);
+		cl_assert_equal_i(stats.received_objects, 3);
+		cl_assert_equal_i(stats.indexed_objects, 3);
+		cl_assert_equal_i(stats.local_objects, 0);
+
+		git_indexer_stream_free(idx);
+	}
+}

tests-clar/pack/packbuilder.c

@@ -92,7 +92,7 @@ void test_pack_packbuilder__create_pack(void)
 
 	seed_packbuilder();
 
-	cl_git_pass(git_indexer_stream_new(&_indexer, ".", NULL, NULL));
+	cl_git_pass(git_indexer_stream_new(&_indexer, ".", NULL, NULL, NULL));
 	cl_git_pass(git_packbuilder_foreach(_packbuilder, feed_indexer, &stats));
 	cl_git_pass(git_indexer_stream_finalize(_indexer, &stats));
 
@@ -141,7 +141,7 @@ void test_pack_packbuilder__foreach(void)
 	git_indexer_stream *idx;
 
 	seed_packbuilder();
-	cl_git_pass(git_indexer_stream_new(&idx, ".", NULL, NULL));
+	cl_git_pass(git_indexer_stream_new(&idx, ".", NULL, NULL, NULL));
 	cl_git_pass(git_packbuilder_foreach(_packbuilder, foreach_cb, idx));
 	cl_git_pass(git_indexer_stream_finalize(idx, &stats));
 	git_indexer_stream_free(idx);