midx: Support multi-pack-index files in odb_pack.c

This change adds support for reading multi-pack-index files from the
packfile odb backend. This also makes git_pack_file objects open their
backing failes lazily in more scenarios, since the multi-pack-index can
avoid having to open them in some cases (yay!).

This change also refreshes the documentation found in src/odb_pack.c to
match the updated code.

Part of: #5399

src/midx.c

@@ -13,8 +13,6 @@
 #include "odb.h"
 #include "pack.h"
 
-#define GIT_MIDX_FILE_MODE 0444
-
 #define MIDX_SIGNATURE 0x4d494458 /* "MIDX" */
 #define MIDX_VERSION 1
 #define MIDX_OBJECT_ID_VERSION 1
@@ -116,7 +114,7 @@ static int midx_parse_oid_lookup(
 		return midx_error("missing OID Lookup chunk");
 	if (chunk_oid_lookup->length == 0)
 		return midx_error("empty OID Lookup chunk");
-	if (chunk_oid_lookup->length != idx->num_objects * 20)
+	if (chunk_oid_lookup->length != idx->num_objects * GIT_OID_RAWSZ)
 		return midx_error("OID Lookup chunk has wrong length");
 
 	idx->oid_lookup = oid = (git_oid *)(data + chunk_oid_lookup->offset);
@@ -183,7 +181,7 @@ int git_midx_parse(
 
 	GIT_ASSERT_ARG(idx);
 
-	if (size < sizeof(struct git_midx_header) + 20)
+	if (size < sizeof(struct git_midx_header) + GIT_OID_RAWSZ)
 		return midx_error("multi-pack index is too short");
 
 	hdr = ((struct git_midx_header *)data);
@@ -203,7 +201,7 @@ int git_midx_parse(
 	last_chunk_offset =
 			sizeof(struct git_midx_header) +
 			(1 + hdr->chunks) * 12;
-	trailer_offset = size - 20;
+	trailer_offset = size - GIT_OID_RAWSZ;
 	if (trailer_offset < last_chunk_offset)
 		return midx_error("wrong index size");
 	git_oid_cpy(&idx->checksum, (git_oid *)(data + trailer_offset));
@@ -309,6 +307,10 @@ int git_midx_open(
 	idx = git__calloc(1, sizeof(git_midx_file));
 	GIT_ERROR_CHECK_ALLOC(idx);
 
+	error = git_buf_sets(&idx->filename, path);
+	if (error < 0)
+		return error;
+
 	error = git_futils_mmap_ro(&idx->index_map, fd, 0, idx_size);
 	p_close(fd);
 	if (error < 0) {
@@ -325,6 +327,46 @@ int git_midx_open(
 	return 0;
 }
 
+bool git_midx_needs_refresh(
+		const git_midx_file *idx,
+		const char *path)
+{
+	git_file fd = -1;
+	struct stat st;
+	ssize_t bytes_read;
+	git_oid idx_checksum = {{0}};
+
+	/* TODO: properly open the file without access time using O_NOATIME */
+	fd = git_futils_open_ro(path);
+	if (fd < 0)
+		return true;
+
+	if (p_fstat(fd, &st) < 0) {
+		p_close(fd);
+		return true;
+	}
+
+	if (!S_ISREG(st.st_mode) ||
+	    !git__is_sizet(st.st_size) ||
+	    (size_t)st.st_size != idx->index_map.len) {
+		p_close(fd);
+		return true;
+	}
+
+	if (p_lseek(fd, -GIT_OID_RAWSZ, SEEK_END) < 0) {
+		p_close(fd);
+		return true;
+	}
+
+	bytes_read = p_read(fd, &idx_checksum, GIT_OID_RAWSZ);
+	p_close(fd);
+
+	if (bytes_read != GIT_OID_RAWSZ)
+		return true;
+
+	return git_oid_cmp(&idx_checksum, &idx->checksum) == 0;
+}
+
 int git_midx_entry_find(
 		git_midx_entry *e,
 		git_midx_file *idx,
@@ -343,7 +385,7 @@ int git_midx_entry_find(
 	hi = ntohl(idx->oid_fanout[(int)short_oid->id[0]]);
 	lo = ((short_oid->id[0] == 0x0) ? 0 : ntohl(idx->oid_fanout[(int)short_oid->id[0] - 1]));
 
-	pos = git_pack__lookup_sha1(idx->oid_lookup, 20, lo, hi, short_oid->id);
+	pos = git_pack__lookup_sha1(idx->oid_lookup, GIT_OID_RAWSZ, lo, hi, short_oid->id);
 
 	if (pos >= 0) {
 		/* An object matching exactly the oid was found */
@@ -399,6 +441,24 @@ int git_midx_entry_find(
 	return 0;
 }
 
+int git_midx_foreach_entry(
+		git_midx_file *idx,
+		git_odb_foreach_cb cb,
+		void *data)
+{
+	size_t i;
+	int error;
+
+	GIT_ASSERT_ARG(idx);
+
+	for (i = 0; i < idx->num_objects; ++i) {
+		if ((error = cb(&idx->oid_lookup[i], data)) != 0)
+			return git_error_set_after_callback(error);
+	}
+
+	return error;
+}
+
 int git_midx_close(git_midx_file *idx)
 {
 	GIT_ASSERT_ARG(idx);
@@ -416,6 +476,7 @@ void git_midx_free(git_midx_file *idx)
 	if (!idx)
 		return;
 
+	git_buf_dispose(&idx->filename);
 	git_midx_close(idx);
 	git__free(idx);
 }

src/midx.h

@@ -14,6 +14,7 @@
 
 #include "map.h"
 #include "mwindow.h"
+#include "odb.h"
 
 /*
  * A multi-pack-index file.
@@ -49,6 +50,9 @@ typedef struct git_midx_file {
 
 	/* The trailer of the file. Contains the SHA1-checksum of the whole file. */
 	git_oid checksum;
+
+	/* something like ".git/objects/pack/multi-pack-index". */
+	git_buf filename;
 } git_midx_file;
 
 /*
@@ -66,11 +70,18 @@ typedef struct git_midx_entry {
 int git_midx_open(
 		git_midx_file **idx_out,
 		const char *path);
+bool git_midx_needs_refresh(
+		const git_midx_file *idx,
+		const char *path);
 int git_midx_entry_find(
 		git_midx_entry *e,
 		git_midx_file *idx,
 		const git_oid *short_oid,
 		size_t len);
+int git_midx_foreach_entry(
+		git_midx_file *idx,
+		git_odb_foreach_cb cb,
+		void *data);
 int git_midx_close(git_midx_file *idx);
 void git_midx_free(git_midx_file *idx);
 

src/pack.c

@@ -481,6 +481,9 @@ int git_packfile_resolve_header(
 	off64_t base_offset;
 	int error;
 
+	if (p->mwf.fd == -1 && (error = packfile_open(p)) < 0)
+		return error;
+
 	error = git_packfile_unpack_header(&size, &type, &p->mwf, &w_curs, &curpos);
 	if (error < 0)
 		return error;
@@ -631,6 +634,9 @@ int git_packfile_unpack(
 	size_t stack_size = 0, elem_pos, alloclen;
 	git_object_t base_type;
 
+	if (p->mwf.fd == -1 && (error = packfile_open(p)) < 0)
+		return error;
+
 	/*
 	 * TODO: optionally check the CRC on the packfile
 	 */

tests/pack/midx.c

@@ -27,3 +27,19 @@ void test_pack_midx__parse(void)
 	git_repository_free(repo);
 	git_buf_dispose(&midx_path);
 }
+
+void test_pack_midx__lookup(void)
+{
+	git_repository *repo;
+	git_commit *commit;
+	git_oid id;
+
+	cl_git_pass(git_repository_open(&repo, cl_fixture("testrepo.git")));
+
+	cl_git_pass(git_oid_fromstr(&id, "5001298e0c09ad9c34e4249bc5801c75e9754fa5"));
+	cl_git_pass(git_commit_lookup_prefix(&commit, repo, &id, GIT_OID_HEXSZ));
+	cl_assert_equal_s(git_commit_message(commit), "packed commit one\n");
+
+	git_commit_free(commit);
+	git_repository_free(repo);
+}