indexer: fix thin packs

When given an ODB from which to read objects, the indexer will attempt
to inject the missing bases at the end of the pack and update the
header and trailer to reflect the new contents.

examples/network/index-pack.c

@@ -46,7 +46,7 @@ int index_pack(git_repository *repo, int argc, char **argv)
 		return EXIT_FAILURE;
 	}
 
-	if (git_indexer_stream_new(&idx, ".", NULL, NULL) < 0) {
+	if (git_indexer_stream_new(&idx, ".", NULL, NULL, NULL) < 0) {
 		puts("bad idx");
 		return -1;
 	}

include/git2/indexer.h

@@ -20,12 +20,16 @@ typedef struct git_indexer_stream git_indexer_stream;
  *
  * @param out where to store the indexer instance
  * @param path to the directory where the packfile should be stored
+ * @param odb object database from which to read base objects when
+ * fixing thin packs. Pass NULL if no thin pack is expected (an error
+ * will be returned if there are bases missing)
  * @param progress_cb function to call with progress information
  * @param progress_cb_payload payload for the progress callback
  */
 GIT_EXTERN(int) git_indexer_stream_new(
 		git_indexer_stream **out,
 		const char *path,
+		git_odb *odb,
 		git_transfer_progress_callback progress_cb,
 		void *progress_cb_payload);
 

include/git2/sys/odb_backend.h

@@ -80,7 +80,7 @@ struct git_odb_backend {
 		git_odb_backend *, git_odb_foreach_cb cb, void *payload);
 
 	int (* writepack)(
-		git_odb_writepack **, git_odb_backend *,
+		git_odb_writepack **, git_odb_backend *, git_odb *odb,
 		git_transfer_progress_callback progress_cb, void *progress_payload);
 
 	void (* free)(git_odb_backend *);

include/git2/types.h

@@ -217,6 +217,7 @@ typedef struct git_transfer_progress {
 	unsigned int total_objects;
 	unsigned int indexed_objects;
 	unsigned int received_objects;
+	unsigned int local_objects;
 	size_t received_bytes;
 } git_transfer_progress;
 

src/odb.c

@@ -988,7 +988,7 @@ int git_odb_write_pack(struct git_odb_writepack **out, git_odb *db, git_transfer
 
 		if (b->writepack != NULL) {
 			++writes;
-			error = b->writepack(out, b, progress_cb, progress_payload);
+			error = b->writepack(out, b, db, progress_cb, progress_payload);
 		}
 	}
 

src/odb_pack.c

@@ -541,6 +541,7 @@ static void pack_backend__writepack_free(struct git_odb_writepack *_writepack)
 
 static int pack_backend__writepack(struct git_odb_writepack **out,
 	git_odb_backend *_backend,
+        git_odb *odb,
 	git_transfer_progress_callback progress_cb,
 	void *progress_payload)
 {
@@ -557,7 +558,7 @@ static int pack_backend__writepack(struct git_odb_writepack **out,
 	GITERR_CHECK_ALLOC(writepack);
 
 	if (git_indexer_stream_new(&writepack->indexer_stream,
-		backend->pack_folder, progress_cb, progress_payload) < 0) {
+		backend->pack_folder, odb, progress_cb, progress_payload) < 0) {
 		git__free(writepack);
 		return -1;
 	}

src/pack-objects.c

@@ -1253,7 +1253,7 @@ int git_packbuilder_write(
 	PREPARE_PACK;
 
 	if (git_indexer_stream_new(
-		&indexer, path, progress_cb, progress_cb_payload) < 0)
+		&indexer, path, pb->odb, progress_cb, progress_cb_payload) < 0)
 		return -1;
 
 	ctx.indexer = indexer;

tests-clar/pack/indexer.c

 #include "clar_libgit2.h"
+#include <git2.h>
 #include "fileops.h"
 #include "hash.h"
 #include "iterator.h"
 #include "vector.h"
 #include "posix.h"
 
+
 /*
  * This is a packfile with three objects. The second is a delta which
  * depends on the third, which is also a delta.
@@ -23,6 +25,24 @@ unsigned char out_of_order_pack[] = {
 };
 unsigned int out_of_order_pack_len = 112;
 
+/*
+ * Packfile with two objects. The second is a delta against an object
+ * which is not in the packfile
+ */
+unsigned char thin_pack[] = {
+  0x50, 0x41, 0x43, 0x4b, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x02,
+  0x32, 0x78, 0x9c, 0x63, 0x67, 0x00, 0x00, 0x00, 0x10, 0x00, 0x08, 0x76,
+  0xe6, 0x8f, 0xe8, 0x12, 0x9b, 0x54, 0x6b, 0x10, 0x1a, 0xee, 0x95, 0x10,
+  0xc5, 0x32, 0x8e, 0x7f, 0x21, 0xca, 0x1d, 0x18, 0x78, 0x9c, 0x63, 0x62,
+  0x66, 0x4e, 0xcb, 0xcf, 0x07, 0x00, 0x02, 0xac, 0x01, 0x4d, 0x42, 0x52,
+  0x3a, 0x6f, 0x39, 0xd1, 0xfe, 0x66, 0x68, 0x6b, 0xa5, 0xe5, 0xe2, 0x97,
+  0xac, 0x94, 0x6c, 0x76, 0x0b, 0x04
+};
+unsigned int thin_pack_len = 78;
+
+unsigned char base_obj[] = { 07, 076 };
+unsigned int base_obj_len = 2;
+
 void test_pack_indexer__out_of_order(void)
 {
 	git_indexer_stream *idx;
@@ -38,3 +58,71 @@ void test_pack_indexer__out_of_order(void)
 
 	git_indexer_stream_free(idx);
 }
+
+void test_pack_indexer__fix_thin(void)
+{
+	git_indexer_stream *idx;
+	git_transfer_progress stats;
+	git_repository *repo;
+	git_odb *odb;
+	git_oid id, should_id;
+
+	cl_git_pass(git_repository_init(&repo, "thin.git", true));
+	cl_git_pass(git_repository_odb(&odb, repo));
+
+	/* Store the missing base into your ODB so the indexer can fix the pack */
+	cl_git_pass(git_odb_write(&id, odb, base_obj, base_obj_len, GIT_OBJ_BLOB));
+	git_oid_fromstr(&should_id, "e68fe8129b546b101aee9510c5328e7f21ca1d18");
+	cl_assert(!git_oid_cmp(&id, &should_id));
+
+	cl_git_pass(git_indexer_stream_new(&idx, ".", odb, NULL, NULL));
+	cl_git_pass(git_indexer_stream_add(idx, thin_pack, thin_pack_len, &stats));
+	cl_git_pass(git_indexer_stream_finalize(idx, &stats));
+
+	cl_assert_equal_i(stats.total_objects, 3);
+	cl_assert_equal_i(stats.received_objects, 2);
+	cl_assert_equal_i(stats.indexed_objects, 2);
+	cl_assert_equal_i(stats.local_objects, 1);
+
+	git_oid_fromstr(&should_id, "11f0f69b334728fdd8bc86b80499f22f29d85b15");
+	cl_assert(!git_oid_cmp(git_indexer_stream_hash(idx), &should_id));
+
+	git_indexer_stream_free(idx);
+	git_odb_free(odb);
+	git_repository_free(repo);
+
+	/*
+	 * The pack's name/hash only tells us what objects there are,
+	 * so we need to go through the packfile again in order to
+	 * figure out whether we calculated the trailer correctly.
+	 */
+	{
+		unsigned char buffer[128];
+		int fd;
+		ssize_t read;
+		git_off_t left;
+		struct stat st;
+		const char *name = "pack-11f0f69b334728fdd8bc86b80499f22f29d85b15.pack";
+
+		fd = p_open(name, O_RDONLY);
+		cl_assert(fd != -1);
+
+		cl_git_pass(p_stat(name, &st));
+		left = st.st_size;
+
+		cl_git_pass(git_indexer_stream_new(&idx, ".", NULL, NULL, NULL));
+		read = p_read(fd, buffer, sizeof(buffer));
+		cl_assert(read != -1);
+		p_close(fd);
+
+		cl_git_pass(git_indexer_stream_add(idx, buffer, read, &stats));
+		cl_git_pass(git_indexer_stream_finalize(idx, &stats));
+
+		cl_assert_equal_i(stats.total_objects, 3);
+		cl_assert_equal_i(stats.received_objects, 3);
+		cl_assert_equal_i(stats.indexed_objects, 3);
+		cl_assert_equal_i(stats.local_objects, 0);
+
+		git_indexer_stream_free(idx);
+	}
+}

tests-clar/pack/packbuilder.c

@@ -92,7 +92,7 @@ void test_pack_packbuilder__create_pack(void)
 
 	seed_packbuilder();
 
-	cl_git_pass(git_indexer_stream_new(&_indexer, ".", NULL, NULL));
+	cl_git_pass(git_indexer_stream_new(&_indexer, ".", NULL, NULL, NULL));
 	cl_git_pass(git_packbuilder_foreach(_packbuilder, feed_indexer, &stats));
 	cl_git_pass(git_indexer_stream_finalize(_indexer, &stats));
 
@@ -141,7 +141,7 @@ void test_pack_packbuilder__foreach(void)
 	git_indexer_stream *idx;
 
 	seed_packbuilder();
-	cl_git_pass(git_indexer_stream_new(&idx, ".", NULL, NULL));
+	cl_git_pass(git_indexer_stream_new(&idx, ".", NULL, NULL, NULL));
 	cl_git_pass(git_packbuilder_foreach(_packbuilder, foreach_cb, idx));
 	cl_git_pass(git_indexer_stream_finalize(idx, &stats));
 	git_indexer_stream_free(idx);