packfile: handle sha256 packfiles

Teach the packfile machinery to cope with SHA256.

include/git2/odb_backend.h

@@ -24,6 +24,26 @@ GIT_BEGIN_DECL
  * Constructors for in-box ODB backends.
  */
 
+/** Options for configuring a packfile object backend. */
+typedef struct {
+	unsigned int version; /**< version for the struct */
+
+	/**
+	 * Type of object IDs to use for this object database, or
+	 * 0 for default (currently SHA1).
+	 */
+	git_oid_t oid_type;
+} git_odb_backend_pack_options;
+
+/* The current version of the diff options structure */
+#define GIT_ODB_BACKEND_PACK_OPTIONS_VERSION 1
+
+/* Stack initializer for odb pack backend options.  Alternatively use
+ * `git_odb_backend_pack_options_init` programmatic initialization.
+ */
+#define GIT_ODB_BACKEND_PACK_OPTIONS_INIT \
+	{ GIT_ODB_BACKEND_PACK_OPTIONS_VERSION }
+
 /**
  * Create a backend for the packfiles.
  *
@@ -32,7 +52,38 @@ GIT_BEGIN_DECL
  *
  * @return 0 or an error code
  */
-GIT_EXTERN(int) git_odb_backend_pack(git_odb_backend **out, const char *objects_dir);
+#ifdef GIT_EXPERIMENTAL_SHA256
+GIT_EXTERN(int) git_odb_backend_pack(
+	git_odb_backend **out,
+	const char *objects_dir,
+	const git_odb_backend_pack_options *opts);
+#else
+GIT_EXTERN(int) git_odb_backend_pack(
+	git_odb_backend **out,
+	const char *objects_dir);
+#endif
+
+/**
+ * Create a backend out of a single packfile
+ *
+ * This can be useful for inspecting the contents of a single
+ * packfile.
+ *
+ * @param out location to store the odb backend pointer
+ * @param index_file path to the packfile's .idx file
+ *
+ * @return 0 or an error code
+ */
+#ifdef GIT_EXPERIMENTAL_SHA256
+GIT_EXTERN(int) git_odb_backend_one_pack(
+	git_odb_backend **out,
+	const char *index_file,
+	const git_odb_backend_pack_options *opts);
+#else
+GIT_EXTERN(int) git_odb_backend_one_pack(
+	git_odb_backend **out,
+	const char *index_file);
+#endif
 
 typedef enum {
 	GIT_ODB_BACKEND_LOOSE_FSYNC = (1 << 0)
@@ -100,19 +151,6 @@ GIT_EXTERN(int) git_odb_backend_loose(
 	unsigned int file_mode);
 #endif
 
-/**
- * Create a backend out of a single packfile
- *
- * This can be useful for inspecting the contents of a single
- * packfile.
- *
- * @param out location to store the odb backend pointer
- * @param index_file path to the packfile's .idx file
- *
- * @return 0 or an error code
- */
-GIT_EXTERN(int) git_odb_backend_one_pack(git_odb_backend **out, const char *index_file);
-
 /** Streaming mode */
 typedef enum {
 	GIT_STREAM_RDONLY = (1 << 1),

src/libgit2/commit_graph.c

@@ -538,7 +538,7 @@ int git_commit_graph_entry_find(
 	hi = ntohl(file->oid_fanout[(int)short_oid->id[0]]);
 	lo = ((short_oid->id[0] == 0x0) ? 0 : ntohl(file->oid_fanout[(int)short_oid->id[0] - 1]));
 
-	pos = git_pack__lookup_sha1(file->oid_lookup, GIT_OID_SHA1_SIZE, lo, hi, short_oid->id);
+	pos = git_pack__lookup_id(file->oid_lookup, GIT_OID_SHA1_SIZE, lo, hi, short_oid->id, GIT_OID_SHA1);
 
 	if (pos >= 0) {
 		/* An object matching exactly the oid was found */
@@ -726,7 +726,8 @@ int git_commit_graph_writer_add_index_file(
 	if (error < 0)
 		goto cleanup;
 
-	error = git_mwindow_get_pack(&p, idx_path);
+	/* TODO: SHA256 */
+	error = git_mwindow_get_pack(&p, idx_path, 0);
 	if (error < 0)
 		goto cleanup;
 

src/libgit2/indexer.c

@@ -179,7 +179,8 @@ int git_indexer_new(
 	if (fd < 0)
 		goto cleanup;
 
-	error = git_packfile_alloc(&idx->pack, git_str_cstr(&tmp_path));
+	/* TODO: SHA256 */
+	error = git_packfile_alloc(&idx->pack, git_str_cstr(&tmp_path), 0);
 	git_str_dispose(&tmp_path);
 
 	if (error < 0)
@@ -468,16 +469,16 @@ static int store_object(git_indexer *idx)
 			goto on_error;
 	}
 
-	git_oid_cpy(&pentry->sha1, &oid);
+	git_oid_cpy(&pentry->id, &oid);
 	pentry->offset = entry_start;
 
-	if (git_oidmap_exists(idx->pack->idx_cache, &pentry->sha1)) {
-		git_error_set(GIT_ERROR_INDEXER, "duplicate object %s found in pack", git_oid_tostr_s(&pentry->sha1));
+	if (git_oidmap_exists(idx->pack->idx_cache, &pentry->id)) {
+		git_error_set(GIT_ERROR_INDEXER, "duplicate object %s found in pack", git_oid_tostr_s(&pentry->id));
 		git__free(pentry);
 		goto on_error;
 	}
 
-	if ((error = git_oidmap_set(idx->pack->idx_cache, &pentry->sha1, pentry)) < 0) {
+	if ((error = git_oidmap_set(idx->pack->idx_cache, &pentry->id, pentry)) < 0) {
 		git__free(pentry);
 		git_error_set_oom();
 		goto on_error;
@@ -522,8 +523,8 @@ static int save_entry(git_indexer *idx, struct entry *entry, struct git_pack_ent
 
 	pentry->offset = entry_start;
 
-	if (git_oidmap_exists(idx->pack->idx_cache, &pentry->sha1) ||
-	    git_oidmap_set(idx->pack->idx_cache, &pentry->sha1, pentry) < 0) {
+	if (git_oidmap_exists(idx->pack->idx_cache, &pentry->id) ||
+	    git_oidmap_set(idx->pack->idx_cache, &pentry->id, pentry) < 0) {
 		git_error_set(GIT_ERROR_INDEXER, "cannot insert object into pack");
 		return -1;
 	}
@@ -557,7 +558,7 @@ static int hash_and_save(git_indexer *idx, git_rawobj *obj, off64_t entry_start)
 	pentry = git__calloc(1, sizeof(struct git_pack_entry));
 	GIT_ERROR_CHECK_ALLOC(pentry);
 
-	git_oid_cpy(&pentry->sha1, &oid);
+	git_oid_cpy(&pentry->id, &oid);
 	git_oid_cpy(&entry->oid, &oid);
 	entry->crc = crc32(0L, Z_NULL, 0);
 
@@ -987,7 +988,7 @@ static int inject_object(git_indexer *idx, git_oid *id)
 	pentry = git__calloc(1, sizeof(struct git_pack_entry));
 	GIT_ERROR_CHECK_ALLOC(pentry);
 
-	git_oid_cpy(&pentry->sha1, id);
+	git_oid_cpy(&pentry->id, id);
 	git_oid_cpy(&entry->oid, id);
 	idx->off = entry_start + hdr_len + len;
 

src/libgit2/midx.c

@@ -392,7 +392,7 @@ int git_midx_entry_find(
 	hi = ntohl(idx->oid_fanout[(int)short_oid->id[0]]);
 	lo = ((short_oid->id[0] == 0x0) ? 0 : ntohl(idx->oid_fanout[(int)short_oid->id[0] - 1]));
 
-	pos = git_pack__lookup_sha1(idx->oid_lookup, GIT_OID_SHA1_SIZE, lo, hi, short_oid->id);
+	pos = git_pack__lookup_id(idx->oid_lookup, GIT_OID_SHA1_SIZE, lo, hi, short_oid->id, GIT_OID_SHA1);
 
 	if (pos >= 0) {
 		/* An object matching exactly the oid was found */
@@ -549,7 +549,8 @@ int git_midx_writer_add(
 	if (error < 0)
 		return error;
 
-	error = git_mwindow_get_pack(&p, git_str_cstr(&idx_path_buf));
+	/* TODO: SHA256 */
+	error = git_mwindow_get_pack(&p, git_str_cstr(&idx_path_buf), 0);
 	git_str_dispose(&idx_path_buf);
 	if (error < 0)
 		return error;

src/libgit2/mwindow.c

@@ -61,7 +61,10 @@ int git_mwindow_global_init(void)
 	return git_runtime_shutdown_register(git_mwindow_global_shutdown);
 }
 
-int git_mwindow_get_pack(struct git_pack_file **out, const char *path)
+int git_mwindow_get_pack(
+	struct git_pack_file **out,
+	const char *path,
+	git_oid_t oid_type)
 {
 	struct git_pack_file *pack;
 	char *packname;
@@ -86,7 +89,7 @@ int git_mwindow_get_pack(struct git_pack_file **out, const char *path)
 	}
 
 	/* If we didn't find it, we need to create it */
-	if ((error = git_packfile_alloc(&pack, path)) < 0) {
+	if ((error = git_packfile_alloc(&pack, path, oid_type)) < 0) {
 		git_mutex_unlock(&git__mwindow_mutex);
 		return error;
 	}

src/libgit2/mwindow.h

@@ -48,7 +48,10 @@ void git_mwindow_close(git_mwindow **w_cursor);
 extern int git_mwindow_global_init(void);
 
 struct git_pack_file; /* just declaration to avoid cyclical includes */
-int git_mwindow_get_pack(struct git_pack_file **out, const char *path);
+int git_mwindow_get_pack(
+	struct git_pack_file **out,
+	const char *path,
+	git_oid_t oid_type);
 int git_mwindow_put_pack(struct git_pack_file *pack);
 
 #endif

src/libgit2/odb.c

@@ -684,6 +684,7 @@ int git_odb__add_default_backends(
 	ino_t inode;
 	git_odb_backend *loose, *packed;
 	git_odb_backend_loose_options loose_opts = GIT_ODB_BACKEND_LOOSE_OPTIONS_INIT;
+	git_odb_backend_pack_options pack_opts = GIT_ODB_BACKEND_PACK_OPTIONS_INIT;
 
 	/* TODO: inodes are not really relevant on Win32, so we need to find
 	 * a cross-platform workaround for this */
@@ -722,6 +723,7 @@ int git_odb__add_default_backends(
 		loose_opts.flags |= GIT_ODB_BACKEND_LOOSE_FSYNC;
 
 	loose_opts.oid_type = db->options.oid_type;
+	pack_opts.oid_type = db->options.oid_type;
 
 	/* add the loose object backend */
 	if (git_odb__backend_loose(&loose, objects_dir, &loose_opts) < 0 ||
@@ -729,8 +731,17 @@ int git_odb__add_default_backends(
 		return -1;
 
 	/* add the packed file backend */
-	if (git_odb_backend_pack(&packed, objects_dir) < 0 ||
-		add_backend_internal(db, packed, git_odb__packed_priority, as_alternates, inode) < 0)
+#ifdef GIT_EXPERIMENTAL_SHA256
+	if (git_odb_backend_pack(&packed, objects_dir, &pack_opts) < 0)
+		return -1;
+#else
+	GIT_UNUSED(pack_opts);
+
+	if (git_odb_backend_pack(&packed, objects_dir) < 0)
+		return -1;
+#endif
+
+	if (add_backend_internal(db, packed, git_odb__packed_priority, as_alternates, inode) < 0)
 		return -1;
 
 	if (git_mutex_lock(&db->lock) < 0) {

src/libgit2/pack.h

@@ -99,13 +99,19 @@ struct git_pack_file {
 
 	uint32_t num_objects;
 	uint32_t num_bad_objects;
-	git_oid *bad_object_sha1; /* array of git_oid */
+	git_oid *bad_object_ids; /* array of git_oid */
+
+	git_oid_t oid_type;
+	unsigned oid_hexsize:7,
+	         oid_size:6,
+	         pack_local:1,
+	         pack_keep:1,
+		 has_cache:1;
 
 	int index_version;
 	git_time_t mtime;
-	unsigned pack_local:1, pack_keep:1, has_cache:1;
 	git_oidmap *idx_cache;
-	unsigned char **oids;
+	unsigned char **ids;
 
 	git_pack_cache bases; /* delta base cache */
 
@@ -116,21 +122,26 @@ struct git_pack_file {
 };
 
 /**
- * Return the position where an OID (or a prefix) would be inserted within the
- * OID Lookup Table of an .idx file. This performs binary search between the lo
- * and hi indices.
+ * Return the position where an OID (or a prefix) would be inserted within
+ * the OID Lookup Table of an .idx file. This performs binary search
+ * between the lo and hi indices.
  *
- * The stride parameter is provided because .idx files version 1 store the OIDs
- * interleaved with the 4-byte file offsets of the objects within the .pack
- * file (stride = 24), whereas files with version 2 store them in a contiguous
- * flat array (stride = 20).
+ * The stride parameter is provided because .idx files version 1 store the
+ * OIDs interleaved with the 4-byte file offsets of the objects within the
+ * .pack file (stride = oid_size + 4), whereas files with version 2 store
+ * them in a contiguous flat array (stride = oid_size).
  */
-int git_pack__lookup_sha1(const void *oid_lookup_table, size_t stride, unsigned lo,
-		unsigned hi, const unsigned char *oid_prefix);
+int git_pack__lookup_id(
+	const void *id_lookup_table,
+	size_t stride,
+	unsigned lo,
+	unsigned hi,
+	const unsigned char *id_prefix,
+	const git_oid_t oid_type);
 
 struct git_pack_entry {
 	off64_t offset;
-	git_oid sha1;
+	git_oid id;
 	struct git_pack_file *p;
 };
 
@@ -174,12 +185,15 @@ int get_delta_base(
 		off64_t delta_obj_offset);
 
 void git_packfile_free(struct git_pack_file *p, bool unlink_packfile);
-int git_packfile_alloc(struct git_pack_file **pack_out, const char *path);
+int git_packfile_alloc(
+	struct git_pack_file **pack_out,
+	const char *path,
+	git_oid_t oid_type);
 
 int git_pack_entry_find(
 		struct git_pack_entry *e,
 		struct git_pack_file *p,
-		const git_oid *short_oid,
+		const git_oid *short_id,
 		size_t len);
 int git_pack_foreach_entry(
 		struct git_pack_file *p,

tests/libgit2/odb/foreach.c

@@ -52,7 +52,16 @@ void test_odb_foreach__one_pack(void)
 	int nobj = 0;
 
 	cl_git_pass(git_odb__new(&_odb, NULL));
-	cl_git_pass(git_odb_backend_one_pack(&backend, cl_fixture("testrepo.git/objects/pack/pack-a81e489679b7d3418f9ab594bda8ceb37dd4c695.idx")));
+
+#ifdef GIT_EXPERIMENTAL_SHA256
+	cl_git_pass(git_odb_backend_one_pack(&backend,
+		cl_fixture("testrepo.git/objects/pack/pack-a81e489679b7d3418f9ab594bda8ceb37dd4c695.idx"),
+		NULL));
+#else
+	cl_git_pass(git_odb_backend_one_pack(&backend,
+		cl_fixture("testrepo.git/objects/pack/pack-a81e489679b7d3418f9ab594bda8ceb37dd4c695.idx")));
+#endif
+
 	cl_git_pass(git_odb_add_backend(_odb, backend, 1));
 	_repo = NULL;
 

tests/libgit2/odb/pack_data_one256.h

+/* Just a few to make sure it's working, the rest is tested already */
+#ifdef GIT_EXPERIMENTAL_SHA256
+static const char *packed_objects_one256[] = {
+	"ea926306b1bab6d3f25f45609907eb6dff91a1460b25e63bf4a0494c70e7a269",
+	"d048ba2ef4fafa502a44cbc1a50cd58359b9bc243b84a08f541a08ca5f621137",
+	"a66bda0109d2b3c9bc87970da81bd91076b5f871febbc860f09ae997668b6800",
+	"3609a41c0506fe19d01fb8b4729923362675f191fe5f63fab3111ef804c48fdf",
+	"22b6705b86e4aa9120eff203af24709d483698d9f78695e86e82de121784b570",
+	"6f11d93bfb269ee8c7a506178f60c430abfac5d424acfd9c0b0b27b98e6ab49b",
+	"0aefd477d9e5b3f8d708a3cf6d78be6b670dfa2e2ec41244634f3b8f115d8e04",
+	"580474d948cd2ebd2e5ce7a5b81b872d87ba4639c1ac4c0fa7a11a8eddf9827c",
+	"0636b4292bfdd7274a977cb6f8b2ded8f315ea1bcd8dbedfca37964c2ed3d085",
+	"19fb1c78b11f0f8bda658d6fa6cc63c0b573c0f6760ee5a9c2df6ce2cde00c5c",
+	"7f2f7afccb317bb3fdd28555f126846dc4eebe5d9ae7b8d8a1456e8ff85422ce",
+	"4066249c68b0d3c8b39ebe02c9188935900465acad02a49269710f56720fa58e",
+	"a560d1fa1edf114f57b402e16d662c17b1e3b7e8601ff7dcea6615ba7f1e48ef",
+	"58923faa87c7d559d308a114ec2b164e5d6046c889420ed1def6eef2d55106a2",
+	"753ddabab8ae9c1e733cda15e8e3c83dd43f5a2d939ae32cc3b30b0be1e91f96",
+	"46333d32b3801cf11d9f80b557245c9e32b0e05deca61dae968914fde159f0e5"
+};
+#endif

tests/libgit2/odb/packedone.c

@@ -11,7 +11,14 @@ void test_odb_packedone__initialize(void)
 	git_odb_backend *backend = NULL;
 
 	cl_git_pass(git_odb__new(&_odb, NULL));
-	cl_git_pass(git_odb_backend_one_pack(&backend, cl_fixture("testrepo.git/objects/pack/pack-a81e489679b7d3418f9ab594bda8ceb37dd4c695.idx")));
+#ifdef GIT_EXPERIMENTAL_SHA256
+	cl_git_pass(git_odb_backend_one_pack(&backend,
+		cl_fixture("testrepo.git/objects/pack/pack-a81e489679b7d3418f9ab594bda8ceb37dd4c695.idx"),
+		NULL));
+#else
+	cl_git_pass(git_odb_backend_one_pack(&backend,
+		cl_fixture("testrepo.git/objects/pack/pack-a81e489679b7d3418f9ab594bda8ceb37dd4c695.idx")));
+#endif
 	cl_git_pass(git_odb_add_backend(_odb, backend, 1));
 }
 

tests/libgit2/odb/packedone256.c

+#include "clar_libgit2.h"
+#include "git2/odb_backend.h"
+
+#include "pack_data_one256.h"
+#include "pack.h"
+
+#ifdef GIT_EXPERIMENTAL_SHA256
+static git_odb *_odb;
+#endif
+
+void test_odb_packedone256__initialize(void)
+{
+#ifdef GIT_EXPERIMENTAL_SHA256
+	git_odb_backend *backend = NULL;
+	git_odb_options odb_opts = GIT_ODB_OPTIONS_INIT;
+	git_odb_backend_pack_options backend_opts = GIT_ODB_BACKEND_PACK_OPTIONS_INIT;
+
+	odb_opts.oid_type = GIT_OID_SHA256;
+	backend_opts.oid_type = GIT_OID_SHA256;
+
+	cl_git_pass(git_odb__new(&_odb, &odb_opts));
+	cl_git_pass(git_odb_backend_one_pack(
+		&backend,
+		cl_fixture("testrepo_256.git/objects/pack/pack-e2f07f30db7e480ea84a0e64ee791b9b270067124b2609019b74f33f256f33fa.idx"),
+		&backend_opts));
+	cl_git_pass(git_odb_add_backend(_odb, backend, 1));
+#endif
+}
+
+void test_odb_packedone256__cleanup(void)
+{
+#ifdef GIT_EXPERIMENTAL_SHA256
+	git_odb_free(_odb);
+	_odb = NULL;
+#endif
+}
+
+void test_odb_packedone256__mass_read(void)
+{
+#ifdef GIT_EXPERIMENTAL_SHA256
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(packed_objects_one256); ++i) {
+		git_oid id;
+		git_odb_object *obj;
+
+		cl_git_pass(git_oid__fromstr(&id, packed_objects_one256[i], GIT_OID_SHA256));
+		cl_assert(git_odb_exists(_odb, &id) == 1);
+		cl_git_pass(git_odb_read(&obj, _odb, &id));
+
+		git_odb_object_free(obj);
+	}
+#endif
+}
+
+void test_odb_packedone256__read_header_0(void)
+{
+#ifdef GIT_EXPERIMENTAL_SHA256
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(packed_objects_one256); ++i) {
+		git_oid id;
+		git_odb_object *obj;
+		size_t len;
+		git_object_t type;
+
+		cl_git_pass(git_oid__fromstr(&id, packed_objects_one256[i], GIT_OID_SHA256));
+
+		cl_git_pass(git_odb_read(&obj, _odb, &id));
+		cl_git_pass(git_odb_read_header(&len, &type, _odb, &id));
+
+		cl_assert(obj->cached.size == len);
+		cl_assert(obj->cached.type == type);
+
+		git_odb_object_free(obj);
+	}
+#endif
+}

tests/libgit2/odb/sorting.c

@@ -82,7 +82,11 @@ void test_odb_sorting__override_default_backend_priority(void)
 
 	cl_git_pass(git_libgit2_opts(GIT_OPT_SET_ODB_LOOSE_PRIORITY, 5));
 	cl_git_pass(git_libgit2_opts(GIT_OPT_SET_ODB_PACKED_PRIORITY, 3));
-	git_odb_backend_pack(&packed, "./testrepo.git/objects");
+	git_odb_backend_pack(&packed, "./testrepo.git/objects"
+#ifdef GIT_EXPERIMENTAL_SHA256
+		, NULL
+#endif
+	);
 	git_odb__backend_loose(&loose, "./testrepo.git/objects", NULL);
 
 	cl_git_pass(git_odb__open(&new_odb, cl_fixture("testrepo.git/objects"), NULL));