midx: Add a way to write multi-pack-index files

This change adds the git_midx_writer_* functions to allow to
write and create `multi-pack-index` files from `.idx`/`.pack` files.

Part of: #5399

include/git2/sys/midx.h

+/*
+ * Copyright (C) the libgit2 contributors. All rights reserved.
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#ifndef INCLUDE_sys_git_midx_h__
+#define INCLUDE_sys_git_midx_h__
+
+#include "git2/common.h"
+#include "git2/types.h"
+
+/**
+ * @file git2/midx.h
+ * @brief Git multi-pack-index routines
+ * @defgroup git_midx Git multi-pack-index routines
+ * @ingroup Git
+ * @{
+ */
+GIT_BEGIN_DECL
+
+/**
+ * Create a new writer for `multi-pack-index` files.
+ *
+ * @param out location to store the writer pointer.
+ * @param pack_dir the directory where the `.pack` and `.idx` files are. The
+ * `multi-pack-index` file will be written in this directory, too.
+ * @return 0 or an error code
+ */
+GIT_EXTERN(int) git_midx_writer_new(
+		git_midx_writer **out,
+		const char *pack_dir);
+
+/**
+ * Free the multi-pack-index writer and its resources.
+ *
+ * @param w the writer to free. If NULL no action is taken.
+ */
+GIT_EXTERN(void) git_midx_writer_free(git_midx_writer *w);
+
+/**
+ * Add an `.idx` file to the writer.
+ *
+ * @param w the writer
+ * @param idx_path the path of an `.idx` file.
+ * @return 0 or an error code
+ */
+GIT_EXTERN(int) git_midx_writer_add(
+		git_midx_writer *w,
+		const char *idx_path);
+
+/**
+ * Write a `multi-pack-index` file to a file.
+ *
+ * @param w the writer
+ * @return 0 or an error code
+ */
+GIT_EXTERN(int) git_midx_writer_commit(
+		git_midx_writer *w);
+
+/**
+ * Dump the contents of the `multi-pack-index` to an in-memory buffer.
+ *
+ * @param midx Buffer where to store the contents of the `multi-pack-index`.
+ * @param w the writer
+ * @return 0 or an error code
+ */
+GIT_EXTERN(int) git_midx_writer_dump(
+		git_buf *midx,
+		git_midx_writer *w);
+
+/** @} */
+GIT_END_DECL
+#endif

include/git2/types.h

@@ -96,6 +96,9 @@ typedef struct git_odb_stream git_odb_stream;
 /** A stream to write a packfile to the ODB */
 typedef struct git_odb_writepack git_odb_writepack;
 
+/** a writer for multi-pack-index files. */
+typedef struct git_midx_writer git_midx_writer;
+
 /** An open refs database handle. */
 typedef struct git_refdb git_refdb;
 

src/midx.h

@@ -12,6 +12,8 @@
 
 #include <ctype.h>
 
+#include "git2/sys/midx.h"
+
 #include "map.h"
 #include "mwindow.h"
 #include "odb.h"
@@ -67,6 +69,20 @@ typedef struct git_midx_entry {
 	git_oid sha1;
 } git_midx_entry;
 
+/*
+ * A writer for `multi-pack-index` files.
+ */
+struct git_midx_writer {
+	/*
+	 * The path of the directory where the .pack/.idx files are stored. The
+	 * `multi-pack-index` file will be written to the same directory.
+	 */
+	git_buf pack_dir;
+
+	/* The list of `git_pack_file`s. */
+	git_vector packs;
+};
+
 int git_midx_open(
 		git_midx_file **idx_out,
 		const char *path);

src/pack.c

@@ -1368,6 +1368,73 @@ int git_pack_foreach_entry(
 	return error;
 }
 
+int git_pack_foreach_entry_offset(
+	struct git_pack_file *p,
+	git_pack_foreach_entry_offset_cb cb,
+	void *data)
+{
+	const unsigned char *index;
+	off64_t current_offset;
+	const git_oid *current_oid;
+	uint32_t i;
+	int error = 0;
+
+	if (git_mutex_lock(&p->lock) < 0)
+		return packfile_error("failed to get lock for git_pack_foreach_entry_offset");
+
+	index = p->index_map.data;
+	if (index == NULL) {
+		if ((error = pack_index_open_locked(p)) < 0)
+			goto cleanup;
+
+		GIT_ASSERT(p->index_map.data);
+		index = p->index_map.data;
+	}
+
+	if (p->index_version > 1)
+		index += 8;
+
+	index += 4 * 256;
+
+	/* all offsets should have been validated by pack_index_check_locked */
+	if (p->index_version > 1) {
+		const unsigned char *offsets = index + 24 * p->num_objects;
+		const unsigned char *large_offset_ptr;
+		const unsigned char *large_offsets = index + 28 * p->num_objects;
+		const unsigned char *large_offsets_end = ((const unsigned char *)p->index_map.data) + p->index_map.len - 20;
+		for (i = 0; i < p->num_objects; i++) {
+			current_offset = ntohl(*(const uint32_t *)(offsets + 4 * i));
+			if (current_offset & 0x80000000) {
+				large_offset_ptr = large_offsets + (current_offset & 0x7fffffff) * 8;
+				if (large_offset_ptr >= large_offsets_end) {
+					error = packfile_error("invalid large offset");
+					goto cleanup;
+				}
+				current_offset = (((off64_t)ntohl(*((uint32_t *)(large_offset_ptr + 0)))) << 32) |
+						ntohl(*((uint32_t *)(large_offset_ptr + 4)));
+			}
+			current_oid = (const git_oid *)(index + 20 * i);
+			if ((error = cb(current_oid, current_offset, data)) != 0) {
+				error = git_error_set_after_callback(error);
+				goto cleanup;
+			}
+		}
+	} else {
+		for (i = 0; i < p->num_objects; i++) {
+			current_offset = ntohl(*(const uint32_t *)(index + 24 * i));
+			current_oid = (const git_oid *)(index + 24 * i + 4);
+			if ((error = cb(current_oid, current_offset, data)) != 0) {
+				error = git_error_set_after_callback(error);
+				goto cleanup;
+			}
+		}
+	}
+
+cleanup:
+	git_mutex_unlock(&p->lock);
+	return error;
+}
+
 int git_pack__lookup_sha1(const void *oid_lookup_table, size_t stride, unsigned lo,
 		unsigned hi, const unsigned char *oid_prefix)
 {

src/pack.h

@@ -20,6 +20,14 @@
 #include "oidmap.h"
 #include "zstream.h"
 
+/**
+ * Function type for callbacks from git_pack_foreach_entry_offset.
+ */
+typedef int git_pack_foreach_entry_offset_cb(
+		const git_oid *id,
+		off64_t offset,
+		void *payload);
+
 #define GIT_PACK_FILE_MODE 0444
 
 #define PACK_SIGNATURE 0x5041434b	/* "PACK" */
@@ -176,5 +184,16 @@ int git_pack_foreach_entry(
 		struct git_pack_file *p,
 		git_odb_foreach_cb cb,
 		void *data);
+/**
+ * Similar to git_pack_foreach_entry, but:
+ * - It also provides the offset of the object within the
+ *   packfile.
+ * - It does not sort the objects in any order.
+ * - It retains the lock while invoking the callback.
+ */
+int git_pack_foreach_entry_offset(
+		struct git_pack_file *p,
+		git_pack_foreach_entry_offset_cb cb,
+		void *data);
 
 #endif

tests/pack/midx.c

 #include "clar_libgit2.h"
 
 #include <git2.h>
+#include <git2/sys/midx.h>
 
 #include "midx.h"
 
@@ -44,3 +45,32 @@ void test_pack_midx__lookup(void)
 	git_commit_free(commit);
 	git_repository_free(repo);
 }
+
+void test_pack_midx__writer(void)
+{
+	git_repository *repo;
+	git_midx_writer *w = NULL;
+	git_buf midx = GIT_BUF_INIT, expected_midx = GIT_BUF_INIT, path = GIT_BUF_INIT;
+
+	cl_git_pass(git_repository_open(&repo, cl_fixture("testrepo.git")));
+
+	cl_git_pass(git_buf_joinpath(&path, git_repository_path(repo), "objects/pack"));
+	cl_git_pass(git_midx_writer_new(&w, git_buf_cstr(&path)));
+
+	cl_git_pass(git_midx_writer_add(w, "pack-d7c6adf9f61318f041845b01440d09aa7a91e1b5.idx"));
+	cl_git_pass(git_midx_writer_add(w, "pack-d85f5d483273108c9d8dd0e4728ccf0b2982423a.idx"));
+	cl_git_pass(git_midx_writer_add(w, "pack-a81e489679b7d3418f9ab594bda8ceb37dd4c695.idx"));
+
+	cl_git_pass(git_midx_writer_dump(&midx, w));
+	cl_git_pass(git_buf_joinpath(&path, git_repository_path(repo), "objects/pack/multi-pack-index"));
+	cl_git_pass(git_futils_readbuffer(&expected_midx, git_buf_cstr(&path)));
+
+	cl_assert_equal_i(git_buf_len(&midx), git_buf_len(&expected_midx));
+	cl_assert_equal_strn(git_buf_cstr(&midx), git_buf_cstr(&expected_midx), git_buf_len(&midx));
+
+	git_buf_dispose(&midx);
+	git_buf_dispose(&expected_midx);
+	git_buf_dispose(&path);
+	git_midx_writer_free(w);
+	git_repository_free(repo);
+}