odb_loose: SHA256 support for loose object storage

Teach the loose object database how to cope with SHA256 objects.

include/git2/odb_backend.h

@@ -59,13 +59,19 @@ typedef struct {
 
 	/** Permissions to use creating a file or 0 for defaults */
 	unsigned int file_mode;
+
+	/**
+	 * Type of object IDs to use for this object database, or
+	 * 0 for default (currently SHA1).
+	 */
+	git_oid_t oid_type;
 } git_odb_backend_loose_options;
 
 /* The current version of the diff options structure */
 #define GIT_ODB_BACKEND_LOOSE_OPTIONS_VERSION 1
 
-/* Stack initializer for diff options.  Alternatively use
- * `git_diff_options_init` programmatic initialization.
+/* Stack initializer for odb loose backend options.  Alternatively use
+ * `git_odb_backend_loose_options_init` programmatic initialization.
  */
 #define GIT_ODB_BACKEND_LOOSE_OPTIONS_INIT \
 	{ GIT_ODB_BACKEND_LOOSE_OPTIONS_VERSION, 0, -1 }

src/libgit2/odb.c

@@ -663,6 +663,8 @@ int git_odb__add_default_backends(
 	if (db->do_fsync)
 		loose_opts.flags |= GIT_ODB_BACKEND_LOOSE_FSYNC;
 
+	loose_opts.oid_type = db->options.oid_type;
+
 	/* add the loose object backend */
 	if (git_odb_backend_loose(&loose, objects_dir, &loose_opts) < 0 ||
 		add_backend_internal(db, loose, git_odb__loose_priority, as_alternates, inode) < 0)

src/libgit2/odb_loose.c

@@ -47,6 +47,7 @@ typedef struct loose_backend {
 	git_odb_backend parent;
 
 	git_odb_backend_loose_options options;
+	size_t oid_hexsize;
 
 	size_t objects_dirlen;
 	char objects_dir[GIT_FLEX_ARRAY];
@@ -56,13 +57,19 @@ typedef struct loose_backend {
  * in order to locate objects matching a short oid.
  */
 typedef struct {
+	loose_backend *backend;
+
 	size_t dir_len;
-	unsigned char short_oid[GIT_OID_SHA1_HEXSIZE]; /* hex formatted oid to match */
+
+	/* Hex formatted oid to match (and its length) */
+	unsigned char short_oid[GIT_OID_MAX_HEXSIZE];
 	size_t short_oid_len;
-	int found;				/* number of matching
-						 * objects already found */
-	unsigned char res_oid[GIT_OID_SHA1_HEXSIZE];	/* hex formatted oid of
-						 * the object found */
+
+	/* Number of matching objects found so far */
+	int found;
+
+	/* Hex formatted oid of the object found */
+	unsigned char res_oid[GIT_OID_MAX_HEXSIZE];
 } loose_locate_object_state;
 
 
@@ -75,20 +82,17 @@ typedef struct {
 static int object_file_name(
 	git_str *name, const loose_backend *be, const git_oid *id)
 {
-	size_t alloclen;
-
-	/* expand length for object root + 40 hex sha1 chars + 2 * '/' + '\0' */
-	GIT_ERROR_CHECK_ALLOC_ADD(&alloclen, be->objects_dirlen, GIT_OID_SHA1_HEXSIZE);
-	GIT_ERROR_CHECK_ALLOC_ADD(&alloclen, alloclen, 3);
-	if (git_str_grow(name, alloclen) < 0)
-		return -1;
+	/* append loose object filename: aa/aaa... (41 bytes plus NUL) */
+	size_t path_size = be->oid_hexsize + 1;
 
 	git_str_set(name, be->objects_dir, be->objects_dirlen);
 	git_fs_path_to_dir(name);
 
-	/* loose object filename: aa/aaa... (41 bytes) */
+	if (git_str_grow_by(name, path_size + 1) < 0)
+		return -1;
+
 	git_oid_pathfmt(name->ptr + name->size, id);
-	name->size += GIT_OID_SHA1_HEXSIZE + 1;
+	name->size += path_size;
 	name->ptr[name->size] = '\0';
 
 	return 0;
@@ -460,8 +464,9 @@ static int locate_object(
 /* Explore an entry of a directory and see if it matches a short oid */
 static int fn_locate_object_short_oid(void *state, git_str *pathbuf) {
 	loose_locate_object_state *sstate = (loose_locate_object_state *)state;
+	size_t hex_size = sstate->backend->oid_hexsize;
 
-	if (git_str_len(pathbuf) - sstate->dir_len != GIT_OID_SHA1_HEXSIZE - 2) {
+	if (git_str_len(pathbuf) - sstate->dir_len != hex_size - 2) {
 		/* Entry cannot be an object. Continue to next entry */
 		return 0;
 	}
@@ -476,7 +481,9 @@ static int fn_locate_object_short_oid(void *state, git_str *pathbuf) {
 			if (!sstate->found) {
 				sstate->res_oid[0] = sstate->short_oid[0];
 				sstate->res_oid[1] = sstate->short_oid[1];
-				memcpy(sstate->res_oid+2, pathbuf->ptr+sstate->dir_len, GIT_OID_SHA1_HEXSIZE-2);
+				memcpy(sstate->res_oid + 2,
+				       pathbuf->ptr+sstate->dir_len,
+				       hex_size - 2);
 			}
 			sstate->found++;
 		}
@@ -502,7 +509,7 @@ static int locate_object_short_oid(
 	int error;
 
 	/* prealloc memory for OBJ_DIR/xx/xx..38x..xx */
-	GIT_ERROR_CHECK_ALLOC_ADD(&alloc_len, dir_len, GIT_OID_SHA1_HEXSIZE);
+	GIT_ERROR_CHECK_ALLOC_ADD(&alloc_len, dir_len, backend->oid_hexsize);
 	GIT_ERROR_CHECK_ALLOC_ADD(&alloc_len, alloc_len, 3);
 	if (git_str_grow(object_location, alloc_len) < 0)
 		return -1;
@@ -526,6 +533,7 @@ static int locate_object_short_oid(
 		return git_odb__error_notfound("no matching loose object for prefix",
 			short_oid, len);
 
+	state.backend = backend;
 	state.dir_len = git_str_len(object_location);
 	state.short_oid_len = len;
 	state.found = 0;
@@ -544,12 +552,12 @@ static int locate_object_short_oid(
 		return git_odb__error_ambiguous("multiple matches in loose objects");
 
 	/* Convert obtained hex formatted oid to raw */
-	error = git_oid_fromstr(res_oid, (char *)state.res_oid, GIT_OID_SHA1);
+	error = git_oid_fromstr(res_oid, (char *)state.res_oid, backend->options.oid_type);
 	if (error)
 		return error;
 
 	/* Update the location according to the oid obtained */
-	GIT_ERROR_CHECK_ALLOC_ADD(&alloc_len, dir_len, GIT_OID_SHA1_HEXSIZE);
+	GIT_ERROR_CHECK_ALLOC_ADD(&alloc_len, dir_len, backend->oid_hexsize);
 	GIT_ERROR_CHECK_ALLOC_ADD(&alloc_len, alloc_len, 2);
 
 	git_str_truncate(object_location, dir_len);
@@ -558,20 +566,12 @@ static int locate_object_short_oid(
 
 	git_oid_pathfmt(object_location->ptr + dir_len, res_oid);
 
-	object_location->size += GIT_OID_SHA1_HEXSIZE + 1;
+	object_location->size +=  backend->oid_hexsize + 1;
 	object_location->ptr[object_location->size] = '\0';
 
 	return 0;
 }
 
-
-
-
-
-
-
-
-
 /***********************************************************
  *
  * LOOSE BACKEND PUBLIC API
@@ -594,7 +594,7 @@ static int loose_backend__read_header(size_t *len_p, git_object_t *type_p, git_o
 
 	if (locate_object(&object_path, (loose_backend *)backend, oid) < 0) {
 		error = git_odb__error_notfound("no matching loose object",
-			oid, GIT_OID_SHA1_HEXSIZE);
+			oid, ((struct loose_backend *)backend)->oid_hexsize);
 	} else if ((error = read_header_loose(&raw, &object_path)) == 0) {
 		*len_p = raw.len;
 		*type_p = raw.type;
@@ -616,7 +616,7 @@ static int loose_backend__read(void **buffer_p, size_t *len_p, git_object_t *typ
 
 	if (locate_object(&object_path, (loose_backend *)backend, oid) < 0) {
 		error = git_odb__error_notfound("no matching loose object",
-			oid, GIT_OID_SHA1_HEXSIZE);
+			oid, ((struct loose_backend *)backend)->oid_hexsize);
 	} else if ((error = read_loose(&raw, &object_path)) == 0) {
 		*buffer_p = raw.data;
 		*len_p = raw.len;
@@ -633,17 +633,19 @@ static int loose_backend__read_prefix(
 	void **buffer_p,
 	size_t *len_p,
 	git_object_t *type_p,
-	git_odb_backend *backend,
+	git_odb_backend *_backend,
 	const git_oid *short_oid,
 	size_t len)
 {
+	struct loose_backend *backend = (struct loose_backend *)_backend;
 	int error = 0;
 
-	GIT_ASSERT_ARG(len >= GIT_OID_MINPREFIXLEN && len <= GIT_OID_SHA1_HEXSIZE);
+	GIT_ASSERT_ARG(len >= GIT_OID_MINPREFIXLEN &&
+	               len <= backend->oid_hexsize);
 
-	if (len == GIT_OID_SHA1_HEXSIZE) {
+	if (len == backend->oid_hexsize) {
 		/* We can fall back to regular read method */
-		error = loose_backend__read(buffer_p, len_p, type_p, backend, short_oid);
+		error = loose_backend__read(buffer_p, len_p, type_p, _backend, short_oid);
 		if (!error)
 			git_oid_cpy(out_oid, short_oid);
 	} else {
@@ -702,15 +704,18 @@ static int loose_backend__exists_prefix(
 }
 
 struct foreach_state {
+	struct loose_backend *backend;
 	size_t dir_len;
 	git_odb_foreach_cb cb;
 	void *data;
 };
 
-GIT_INLINE(int) filename_to_oid(git_oid *oid, const char *ptr)
+GIT_INLINE(int) filename_to_oid(struct loose_backend *backend, git_oid *oid, const char *ptr)
 {
-	int v, i = 0;
-	if (strlen(ptr) != GIT_OID_SHA1_HEXSIZE+1)
+	int v;
+	size_t i = 0;
+
+	if (strlen(ptr) != backend->oid_hexsize + 1)
 		return -1;
 
 	if (ptr[2] != '/') {
@@ -724,7 +729,7 @@ GIT_INLINE(int) filename_to_oid(git_oid *oid, const char *ptr)
 	oid->id[0] = (unsigned char) v;
 
 	ptr += 3;
-	for (i = 0; i < 38; i += 2) {
+	for (i = 0; i < backend->oid_hexsize - 2; i += 2) {
 		v = (git__fromhex(ptr[i]) << 4) | git__fromhex(ptr[i + 1]);
 		if (v < 0)
 			return -1;
@@ -732,7 +737,7 @@ GIT_INLINE(int) filename_to_oid(git_oid *oid, const char *ptr)
 		oid->id[1 + i/2] = (unsigned char) v;
 	}
 
-	oid->type = GIT_OID_SHA1;
+	oid->type = backend->options.oid_type;
 
 	return 0;
 }
@@ -742,7 +747,7 @@ static int foreach_object_dir_cb(void *_state, git_str *path)
 	git_oid oid;
 	struct foreach_state *state = (struct foreach_state *) _state;
 
-	if (filename_to_oid(&oid, path->ptr + state->dir_len) < 0)
+	if (filename_to_oid(state->backend, &oid, path->ptr + state->dir_len) < 0)
 		return 0;
 
 	return git_error_set_after_callback_function(
@@ -779,6 +784,7 @@ static int loose_backend__foreach(git_odb_backend *_backend, git_odb_foreach_cb
 		return -1;
 
 	memset(&state, 0, sizeof(state));
+	state.backend = backend;
 	state.cb = cb;
 	state.data = data;
 	state.dir_len = git_str_len(&buf);
@@ -999,6 +1005,7 @@ static int loose_backend__readstream(
 	loose_readstream *stream = NULL;
 	git_hash_ctx *hash_ctx = NULL;
 	git_str object_path = GIT_STR_INIT;
+	git_hash_algorithm_t algorithm;
 	obj_hdr hdr;
 	int error = 0;
 
@@ -1015,7 +1022,7 @@ static int loose_backend__readstream(
 
 	if (locate_object(&object_path, backend, oid) < 0) {
 		error = git_odb__error_notfound("no matching loose object",
-			oid, GIT_OID_SHA1_HEXSIZE);
+			oid, backend->oid_hexsize);
 		goto done;
 	}
 
@@ -1025,9 +1032,11 @@ static int loose_backend__readstream(
 	hash_ctx = git__malloc(sizeof(git_hash_ctx));
 	GIT_ERROR_CHECK_ALLOC(hash_ctx);
 
-	if ((error = git_hash_ctx_init(hash_ctx, GIT_HASH_ALGORITHM_SHA1)) < 0 ||
-		(error = git_futils_mmap_ro_file(&stream->map, object_path.ptr)) < 0 ||
-		(error = git_zstream_init(&stream->zstream, GIT_ZSTREAM_INFLATE)) < 0)
+	algorithm = git_oid_algorithm(backend->options.oid_type);
+
+	if ((error = git_hash_ctx_init(hash_ctx, algorithm)) < 0 ||
+	    (error = git_futils_mmap_ro_file(&stream->map, object_path.ptr)) < 0 ||
+	    (error = git_zstream_init(&stream->zstream, GIT_ZSTREAM_INFLATE)) < 0)
 		goto done;
 
 	/* check for a packlike loose object */
@@ -1145,6 +1154,9 @@ static void normalize_options(
 
 	if (opts->file_mode == 0)
 		opts->file_mode = GIT_OBJECT_FILE_MODE;
+
+	if (opts->oid_type == 0)
+		opts->oid_type = GIT_OID_DEFAULT;
 }
 
 int git_odb_backend_loose(
@@ -1173,6 +1185,7 @@ int git_odb_backend_loose(
 		backend->objects_dir[backend->objects_dirlen++] = '/';
 
 	normalize_options(&backend->options, opts);
+	backend->oid_hexsize = git_oid_hexsize(backend->options.oid_type);
 
 	backend->parent.read = &loose_backend__read;
 	backend->parent.write = &loose_backend__write;

tests/libgit2/odb/loose.c

@@ -34,24 +34,27 @@ static void cmp_objects(git_rawobj *o, object_data *d)
 
 static void test_read_object(object_data *data)
 {
-    git_oid id;
-    git_odb_object *obj;
+	git_oid id;
+	git_odb_object *obj;
 	git_odb *odb;
 	git_rawobj tmp;
+	git_odb_options opts = GIT_ODB_OPTIONS_INIT;
+
+	opts.oid_type = data->id_type;
 
-    write_object_files(data);
+	write_object_files(data);
 
-    cl_git_pass(git_odb_open(&odb, "test-objects", NULL));
-    cl_git_pass(git_oid_fromstr(&id, data->id, GIT_OID_SHA1));
-    cl_git_pass(git_odb_read(&obj, odb, &id));
+	cl_git_pass(git_odb_open(&odb, "test-objects", &opts));
+	cl_git_pass(git_oid_fromstr(&id, data->id, data->id_type));
+	cl_git_pass(git_odb_read(&obj, odb, &id));
 
 	tmp.data = obj->buffer;
 	tmp.len = obj->cached.size;
 	tmp.type = obj->cached.type;
 
-    cmp_objects(&tmp, data);
+	cmp_objects(&tmp, data);
 
-    git_odb_object_free(obj);
+	git_odb_object_free(obj);
 	git_odb_free(odb);
 }
 
@@ -61,11 +64,14 @@ static void test_read_header(object_data *data)
 	git_odb *odb;
 	size_t len;
 	git_object_t type;
+	git_odb_options opts = GIT_ODB_OPTIONS_INIT;
+
+	opts.oid_type = data->id_type;
 
 	write_object_files(data);
 
-	cl_git_pass(git_odb_open(&odb, "test-objects", NULL));
-	cl_git_pass(git_oid_fromstr(&id, data->id, GIT_OID_SHA1));
+	cl_git_pass(git_odb_open(&odb, "test-objects", &opts));
+	cl_git_pass(git_oid_fromstr(&id, data->id, data->id_type));
 	cl_git_pass(git_odb_read_header(&len, &type, odb, &id));
 
 	cl_assert_equal_sz(data->dlen, len);
@@ -83,11 +89,14 @@ static void test_readstream_object(object_data *data, size_t blocksize)
 	char buf[2048], *ptr = buf;
 	size_t remain;
 	int ret;
+	git_odb_options opts = GIT_ODB_OPTIONS_INIT;
+
+	opts.oid_type = data->id_type;
 
 	write_object_files(data);
 
-	cl_git_pass(git_odb_open(&odb, "test-objects", NULL));
-	cl_git_pass(git_oid_fromstr(&id, data->id, GIT_OID_SHA1));
+	cl_git_pass(git_odb_open(&odb, "test-objects", &opts));
+	cl_git_pass(git_oid_fromstr(&id, data->id, data->id_type));
 	cl_git_pass(git_odb_open_rstream(&stream, &tmp.len, &tmp.type, odb, &id));
 
 	remain = tmp.len;
@@ -124,7 +133,7 @@ void test_odb_loose__cleanup(void)
 	cl_fixture_cleanup("test-objects");
 }
 
-void test_odb_loose__exists(void)
+void test_odb_loose__exists_sha1(void)
 {
 	git_oid id, id2;
 	git_odb *odb;
@@ -149,7 +158,35 @@ void test_odb_loose__exists(void)
 	git_odb_free(odb);
 }
 
-void test_odb_loose__simple_reads(void)
+void test_odb_loose__exists_sha256(void)
+{
+	git_oid id, id2;
+	git_odb *odb;
+	git_odb_options odb_opts = GIT_ODB_OPTIONS_INIT;
+
+	odb_opts.oid_type = GIT_OID_SHA256;
+
+	write_object_files(&one_sha256);
+	cl_git_pass(git_odb_open(&odb, "test-objects", &odb_opts));
+
+	cl_git_pass(git_oid_fromstr(&id, one_sha256.id, GIT_OID_SHA256));
+	cl_assert(git_odb_exists(odb, &id));
+
+	cl_git_pass(git_oid_fromstrp(&id, "4c0d52d1", GIT_OID_SHA256));
+	cl_git_pass(git_odb_exists_prefix(&id2, odb, &id, 8));
+	cl_assert_equal_i(0, git_oid_streq(&id2, one_sha256.id));
+
+	/* Test for a missing object */
+	cl_git_pass(git_oid_fromstr(&id, "4c0d52d180c61d01ce1a91dec5ee58f0cbe65fd59433aea803ab927965493faa", GIT_OID_SHA256));
+	cl_assert(!git_odb_exists(odb, &id));
+
+	cl_git_pass(git_oid_fromstrp(&id, "4c0d52da", GIT_OID_SHA256));
+	cl_assert_equal_i(GIT_ENOTFOUND, git_odb_exists_prefix(&id2, odb, &id, 8));
+
+	git_odb_free(odb);
+}
+
+void test_odb_loose__simple_reads_sha1(void)
 {
 	test_read_object(&commit);
 	test_read_object(&tree);
@@ -160,7 +197,18 @@ void test_odb_loose__simple_reads(void)
 	test_read_object(&some);
 }
 
-void test_odb_loose__streaming_reads(void)
+void test_odb_loose__simple_reads_sha256(void)
+{
+	test_read_object(&commit_sha256);
+	test_read_object(&tree_sha256);
+	test_read_object(&tag_sha256);
+	test_read_object(&zero_sha256);
+	test_read_object(&one_sha256);
+	test_read_object(&two_sha256);
+	test_read_object(&some_sha256);
+}
+
+void test_odb_loose__streaming_reads_sha1(void)
 {
 	size_t blocksizes[] = { 1, 2, 4, 16, 99, 1024, 123456789 };
 	size_t i;
@@ -176,7 +224,23 @@ void test_odb_loose__streaming_reads(void)
 	}
 }
 
-void test_odb_loose__read_header(void)
+void test_odb_loose__streaming_reads_sha256(void)
+{
+	size_t blocksizes[] = { 1, 2, 4, 16, 99, 1024, 123456789 };
+	size_t i;
+
+	for (i = 0; i < ARRAY_SIZE(blocksizes); i++) {
+		test_readstream_object(&commit_sha256, blocksizes[i]);
+		test_readstream_object(&tree_sha256, blocksizes[i]);
+		test_readstream_object(&tag_sha256, blocksizes[i]);
+		test_readstream_object(&zero_sha256, blocksizes[i]);
+		test_readstream_object(&one_sha256, blocksizes[i]);
+		test_readstream_object(&two_sha256, blocksizes[i]);
+		test_readstream_object(&some_sha256, blocksizes[i]);
+	}
+}
+
+void test_odb_loose__read_header_sha1(void)
 {
 	test_read_header(&commit);
 	test_read_header(&tree);
@@ -187,6 +251,17 @@ void test_odb_loose__read_header(void)
 	test_read_header(&some);
 }
 
+void test_odb_loose__read_header_sha256(void)
+{
+	test_read_header(&commit_sha256);
+	test_read_header(&tree_sha256);
+	test_read_header(&tag_sha256);
+	test_read_header(&zero_sha256);
+	test_read_header(&one_sha256);
+	test_read_header(&two_sha256);
+	test_read_header(&some_sha256);
+}
+
 static void test_write_object_permission(
 	mode_t dir_mode, mode_t file_mode,
 	mode_t expected_dir_mode, mode_t expected_file_mode)