Skip to content

G
git2
Commit 04f34688 by Edward Thomson
Options

odb_loose: SHA256 support for loose object storage 

Teach the loose object database how to cope with SHA256 objects.
parent 162c996b
Showing with 158 additions and 62 deletions
include/git2/odb_backend.h
......@@ -59,13 +59,19 @@ typedef struct {
/** Permissions to use creating a file or 0 for defaults */
unsigned int file_mode;
/**
* Type of object IDs to use for this object database, or
* 0 for default (currently SHA1).
*/
git_oid_t oid_type;
} git_odb_backend_loose_options;
/* The current version of the diff options structure */
#define GIT_ODB_BACKEND_LOOSE_OPTIONS_VERSION 1
/* Stack initializer for diff options. Alternatively use
* `git_diff_options_init` programmatic initialization.
/* Stack initializer for odb loose backend options. Alternatively use
* `git_odb_backend_loose_options_init` programmatic initialization.
*/
#define GIT_ODB_BACKEND_LOOSE_OPTIONS_INIT \
{ GIT_ODB_BACKEND_LOOSE_OPTIONS_VERSION, 0, -1 }
......
src/libgit2/odb.c
......@@ -663,6 +663,8 @@ int git_odb__add_default_backends(
if (db->do_fsync)
loose_opts.flags |= GIT_ODB_BACKEND_LOOSE_FSYNC;
loose_opts.oid_type = db->options.oid_type;
/* add the loose object backend */
if (git_odb_backend_loose(&loose, objects_dir, &loose_opts) < 0 ||
add_backend_internal(db, loose, git_odb__loose_priority, as_alternates, inode) < 0)
......
src/libgit2/odb_loose.c
......@@ -47,6 +47,7 @@ typedef struct loose_backend {
git_odb_backend parent;
git_odb_backend_loose_options options;
size_t oid_hexsize;
size_t objects_dirlen;
char objects_dir[GIT_FLEX_ARRAY];
......@@ -56,13 +57,19 @@ typedef struct loose_backend {
* in order to locate objects matching a short oid.
*/
typedef struct {
loose_backend *backend;
size_t dir_len;
unsigned char short_oid[GIT_OID_SHA1_HEXSIZE]; /* hex formatted oid to match */
/* Hex formatted oid to match (and its length) */
unsigned char short_oid[GIT_OID_MAX_HEXSIZE];
size_t short_oid_len;
int found; /* number of matching
* objects already found */
unsigned char res_oid[GIT_OID_SHA1_HEXSIZE]; /* hex formatted oid of
* the object found */
/* Number of matching objects found so far */
int found;
/* Hex formatted oid of the object found */
unsigned char res_oid[GIT_OID_MAX_HEXSIZE];
} loose_locate_object_state;
......@@ -75,20 +82,17 @@ typedef struct {
static int object_file_name(
git_str *name, const loose_backend *be, const git_oid *id)
{
size_t alloclen;
/* expand length for object root + 40 hex sha1 chars + 2 * '/' + '\0' */
GIT_ERROR_CHECK_ALLOC_ADD(&alloclen, be->objects_dirlen, GIT_OID_SHA1_HEXSIZE);
GIT_ERROR_CHECK_ALLOC_ADD(&alloclen, alloclen, 3);
if (git_str_grow(name, alloclen) < 0)
return -1;
/* append loose object filename: aa/aaa... (41 bytes plus NUL) */
size_t path_size = be->oid_hexsize + 1;
git_str_set(name, be->objects_dir, be->objects_dirlen);
git_fs_path_to_dir(name);
/* loose object filename: aa/aaa... (41 bytes) */
if (git_str_grow_by(name, path_size + 1) < 0)
return -1;
git_oid_pathfmt(name->ptr + name->size, id);
name->size += GIT_OID_SHA1_HEXSIZE + 1;
name->size += path_size;
name->ptr[name->size] = '\0';
return 0;
......@@ -460,8 +464,9 @@ static int locate_object(
/* Explore an entry of a directory and see if it matches a short oid */
static int fn_locate_object_short_oid(void *state, git_str *pathbuf) {
loose_locate_object_state *sstate = (loose_locate_object_state *)state;
size_t hex_size = sstate->backend->oid_hexsize;
if (git_str_len(pathbuf) - sstate->dir_len != GIT_OID_SHA1_HEXSIZE - 2) {
if (git_str_len(pathbuf) - sstate->dir_len != hex_size - 2) {
/* Entry cannot be an object. Continue to next entry */
return 0;
}
......@@ -476,7 +481,9 @@ static int fn_locate_object_short_oid(void *state, git_str *pathbuf) {
if (!sstate->found) {
sstate->res_oid[0] = sstate->short_oid[0];
sstate->res_oid[1] = sstate->short_oid[1];
memcpy(sstate->res_oid+2, pathbuf->ptr+sstate->dir_len, GIT_OID_SHA1_HEXSIZE-2);
memcpy(sstate->res_oid + 2,
pathbuf->ptr+sstate->dir_len,
hex_size - 2);
}
sstate->found++;
}
......@@ -502,7 +509,7 @@ static int locate_object_short_oid(
int error;
/* prealloc memory for OBJ_DIR/xx/xx..38x..xx */
GIT_ERROR_CHECK_ALLOC_ADD(&alloc_len, dir_len, GIT_OID_SHA1_HEXSIZE);
GIT_ERROR_CHECK_ALLOC_ADD(&alloc_len, dir_len, backend->oid_hexsize);
GIT_ERROR_CHECK_ALLOC_ADD(&alloc_len, alloc_len, 3);
if (git_str_grow(object_location, alloc_len) < 0)
return -1;
......@@ -526,6 +533,7 @@ static int locate_object_short_oid(
return git_odb__error_notfound("no matching loose object for prefix",
short_oid, len);
state.backend = backend;
state.dir_len = git_str_len(object_location);
state.short_oid_len = len;
state.found = 0;
......@@ -544,12 +552,12 @@ static int locate_object_short_oid(
return git_odb__error_ambiguous("multiple matches in loose objects");
/* Convert obtained hex formatted oid to raw */
error = git_oid_fromstr(res_oid, (char *)state.res_oid, GIT_OID_SHA1);
error = git_oid_fromstr(res_oid, (char *)state.res_oid, backend->options.oid_type);
if (error)
return error;
/* Update the location according to the oid obtained */
GIT_ERROR_CHECK_ALLOC_ADD(&alloc_len, dir_len, GIT_OID_SHA1_HEXSIZE);
GIT_ERROR_CHECK_ALLOC_ADD(&alloc_len, dir_len, backend->oid_hexsize);
GIT_ERROR_CHECK_ALLOC_ADD(&alloc_len, alloc_len, 2);
git_str_truncate(object_location, dir_len);
......@@ -558,20 +566,12 @@ static int locate_object_short_oid(
git_oid_pathfmt(object_location->ptr + dir_len, res_oid);
object_location->size += GIT_OID_SHA1_HEXSIZE + 1;
object_location->size += backend->oid_hexsize + 1;
object_location->ptr[object_location->size] = '\0';
return 0;
}
/***********************************************************
*
* LOOSE BACKEND PUBLIC API
......@@ -594,7 +594,7 @@ static int loose_backend__read_header(size_t *len_p, git_object_t *type_p, git_o
if (locate_object(&object_path, (loose_backend *)backend, oid) < 0) {
error = git_odb__error_notfound("no matching loose object",
oid, GIT_OID_SHA1_HEXSIZE);
oid, ((struct loose_backend *)backend)->oid_hexsize);
} else if ((error = read_header_loose(&raw, &object_path)) == 0) {
*len_p = raw.len;
*type_p = raw.type;
......@@ -616,7 +616,7 @@ static int loose_backend__read(void **buffer_p, size_t *len_p, git_object_t *typ
if (locate_object(&object_path, (loose_backend *)backend, oid) < 0) {
error = git_odb__error_notfound("no matching loose object",
oid, GIT_OID_SHA1_HEXSIZE);
oid, ((struct loose_backend *)backend)->oid_hexsize);
} else if ((error = read_loose(&raw, &object_path)) == 0) {
*buffer_p = raw.data;
*len_p = raw.len;
......@@ -633,17 +633,19 @@ static int loose_backend__read_prefix(
void **buffer_p,
size_t *len_p,
git_object_t *type_p,
git_odb_backend *backend,
git_odb_backend *_backend,
const git_oid *short_oid,
size_t len)
{
struct loose_backend *backend = (struct loose_backend *)_backend;
int error = 0;
GIT_ASSERT_ARG(len >= GIT_OID_MINPREFIXLEN && len <= GIT_OID_SHA1_HEXSIZE);
GIT_ASSERT_ARG(len >= GIT_OID_MINPREFIXLEN &&
len <= backend->oid_hexsize);
if (len == GIT_OID_SHA1_HEXSIZE) {
if (len == backend->oid_hexsize) {
/* We can fall back to regular read method */
error = loose_backend__read(buffer_p, len_p, type_p, backend, short_oid);
error = loose_backend__read(buffer_p, len_p, type_p, _backend, short_oid);
if (!error)
git_oid_cpy(out_oid, short_oid);
} else {
......@@ -702,15 +704,18 @@ static int loose_backend__exists_prefix(
}
struct foreach_state {
struct loose_backend *backend;
size_t dir_len;
git_odb_foreach_cb cb;
void *data;
};
GIT_INLINE(int) filename_to_oid(git_oid *oid, const char *ptr)
GIT_INLINE(int) filename_to_oid(struct loose_backend *backend, git_oid *oid, const char *ptr)
{
int v, i = 0;
if (strlen(ptr) != GIT_OID_SHA1_HEXSIZE+1)
int v;
size_t i = 0;
if (strlen(ptr) != backend->oid_hexsize + 1)
return -1;
if (ptr[2] != '/') {
......@@ -724,7 +729,7 @@ GIT_INLINE(int) filename_to_oid(git_oid *oid, const char *ptr)
oid->id[0] = (unsigned char) v;
ptr += 3;
for (i = 0; i < 38; i += 2) {
for (i = 0; i < backend->oid_hexsize - 2; i += 2) {
v = (git__fromhex(ptr[i]) << 4) | git__fromhex(ptr[i + 1]);
if (v < 0)
return -1;
......@@ -732,7 +737,7 @@ GIT_INLINE(int) filename_to_oid(git_oid *oid, const char *ptr)
oid->id[1 + i/2] = (unsigned char) v;
}
oid->type = GIT_OID_SHA1;
oid->type = backend->options.oid_type;
return 0;
}
......@@ -742,7 +747,7 @@ static int foreach_object_dir_cb(void *_state, git_str *path)
git_oid oid;
struct foreach_state *state = (struct foreach_state *) _state;
if (filename_to_oid(&oid, path->ptr + state->dir_len) < 0)
if (filename_to_oid(state->backend, &oid, path->ptr + state->dir_len) < 0)
return 0;
return git_error_set_after_callback_function(
......@@ -779,6 +784,7 @@ static int loose_backend__foreach(git_odb_backend *_backend, git_odb_foreach_cb
return -1;
memset(&state, 0, sizeof(state));
state.backend = backend;
state.cb = cb;
state.data = data;
state.dir_len = git_str_len(&buf);
......@@ -999,6 +1005,7 @@ static int loose_backend__readstream(
loose_readstream *stream = NULL;
git_hash_ctx *hash_ctx = NULL;
git_str object_path = GIT_STR_INIT;
git_hash_algorithm_t algorithm;
obj_hdr hdr;
int error = 0;
......@@ -1015,7 +1022,7 @@ static int loose_backend__readstream(
if (locate_object(&object_path, backend, oid) < 0) {
error = git_odb__error_notfound("no matching loose object",
oid, GIT_OID_SHA1_HEXSIZE);
oid, backend->oid_hexsize);
goto done;
}
......@@ -1025,9 +1032,11 @@ static int loose_backend__readstream(
hash_ctx = git__malloc(sizeof(git_hash_ctx));
GIT_ERROR_CHECK_ALLOC(hash_ctx);
if ((error = git_hash_ctx_init(hash_ctx, GIT_HASH_ALGORITHM_SHA1)) < 0 ||
(error = git_futils_mmap_ro_file(&stream->map, object_path.ptr)) < 0 ||
(error = git_zstream_init(&stream->zstream, GIT_ZSTREAM_INFLATE)) < 0)
algorithm = git_oid_algorithm(backend->options.oid_type);
if ((error = git_hash_ctx_init(hash_ctx, algorithm)) < 0 ||
(error = git_futils_mmap_ro_file(&stream->map, object_path.ptr)) < 0 ||
(error = git_zstream_init(&stream->zstream, GIT_ZSTREAM_INFLATE)) < 0)
goto done;
/* check for a packlike loose object */
......@@ -1145,6 +1154,9 @@ static void normalize_options(
if (opts->file_mode == 0)
opts->file_mode = GIT_OBJECT_FILE_MODE;
if (opts->oid_type == 0)
opts->oid_type = GIT_OID_DEFAULT;
}
int git_odb_backend_loose(
......@@ -1173,6 +1185,7 @@ int git_odb_backend_loose(
backend->objects_dir[backend->objects_dirlen++] = '/';
normalize_options(&backend->options, opts);
backend->oid_hexsize = git_oid_hexsize(backend->options.oid_type);
backend->parent.read = &loose_backend__read;
backend->parent.write = &loose_backend__write;
......
tests/libgit2/odb/loose.c
......@@ -34,24 +34,27 @@ static void cmp_objects(git_rawobj *o, object_data *d)
static void test_read_object(object_data *data)
{
git_oid id;
git_odb_object *obj;
git_oid id;
git_odb_object *obj;
git_odb *odb;
git_rawobj tmp;
git_odb_options opts = GIT_ODB_OPTIONS_INIT;
opts.oid_type = data->id_type;
write_object_files(data);
write_object_files(data);
cl_git_pass(git_odb_open(&odb, "test-objects", NULL));
cl_git_pass(git_oid_fromstr(&id, data->id, GIT_OID_SHA1));
cl_git_pass(git_odb_read(&obj, odb, &id));
cl_git_pass(git_odb_open(&odb, "test-objects", &opts));
cl_git_pass(git_oid_fromstr(&id, data->id, data->id_type));
cl_git_pass(git_odb_read(&obj, odb, &id));
tmp.data = obj->buffer;
tmp.len = obj->cached.size;
tmp.type = obj->cached.type;
cmp_objects(&tmp, data);
cmp_objects(&tmp, data);
git_odb_object_free(obj);
git_odb_object_free(obj);
git_odb_free(odb);
}
......@@ -61,11 +64,14 @@ static void test_read_header(object_data *data)
git_odb *odb;
size_t len;
git_object_t type;
git_odb_options opts = GIT_ODB_OPTIONS_INIT;
opts.oid_type = data->id_type;
write_object_files(data);
cl_git_pass(git_odb_open(&odb, "test-objects", NULL));
cl_git_pass(git_oid_fromstr(&id, data->id, GIT_OID_SHA1));
cl_git_pass(git_odb_open(&odb, "test-objects", &opts));
cl_git_pass(git_oid_fromstr(&id, data->id, data->id_type));
cl_git_pass(git_odb_read_header(&len, &type, odb, &id));
cl_assert_equal_sz(data->dlen, len);
......@@ -83,11 +89,14 @@ static void test_readstream_object(object_data *data, size_t blocksize)
char buf[2048], *ptr = buf;
size_t remain;
int ret;
git_odb_options opts = GIT_ODB_OPTIONS_INIT;
opts.oid_type = data->id_type;
write_object_files(data);
cl_git_pass(git_odb_open(&odb, "test-objects", NULL));
cl_git_pass(git_oid_fromstr(&id, data->id, GIT_OID_SHA1));
cl_git_pass(git_odb_open(&odb, "test-objects", &opts));
cl_git_pass(git_oid_fromstr(&id, data->id, data->id_type));
cl_git_pass(git_odb_open_rstream(&stream, &tmp.len, &tmp.type, odb, &id));
remain = tmp.len;
......@@ -124,7 +133,7 @@ void test_odb_loose__cleanup(void)
cl_fixture_cleanup("test-objects");
}
void test_odb_loose__exists(void)
void test_odb_loose__exists_sha1(void)
{
git_oid id, id2;
git_odb *odb;
......@@ -149,7 +158,35 @@ void test_odb_loose__exists(void)
git_odb_free(odb);
}
void test_odb_loose__simple_reads(void)
void test_odb_loose__exists_sha256(void)
{
git_oid id, id2;
git_odb *odb;
git_odb_options odb_opts = GIT_ODB_OPTIONS_INIT;
odb_opts.oid_type = GIT_OID_SHA256;
write_object_files(&one_sha256);
cl_git_pass(git_odb_open(&odb, "test-objects", &odb_opts));
cl_git_pass(git_oid_fromstr(&id, one_sha256.id, GIT_OID_SHA256));
cl_assert(git_odb_exists(odb, &id));
cl_git_pass(git_oid_fromstrp(&id, "4c0d52d1", GIT_OID_SHA256));
cl_git_pass(git_odb_exists_prefix(&id2, odb, &id, 8));
cl_assert_equal_i(0, git_oid_streq(&id2, one_sha256.id));
/* Test for a missing object */
cl_git_pass(git_oid_fromstr(&id, "4c0d52d180c61d01ce1a91dec5ee58f0cbe65fd59433aea803ab927965493faa", GIT_OID_SHA256));
cl_assert(!git_odb_exists(odb, &id));
cl_git_pass(git_oid_fromstrp(&id, "4c0d52da", GIT_OID_SHA256));
cl_assert_equal_i(GIT_ENOTFOUND, git_odb_exists_prefix(&id2, odb, &id, 8));
git_odb_free(odb);
}
void test_odb_loose__simple_reads_sha1(void)
{
test_read_object(&commit);
test_read_object(&tree);
......@@ -160,7 +197,18 @@ void test_odb_loose__simple_reads(void)
test_read_object(&some);
}
void test_odb_loose__streaming_reads(void)
void test_odb_loose__simple_reads_sha256(void)
{
test_read_object(&commit_sha256);
test_read_object(&tree_sha256);
test_read_object(&tag_sha256);
test_read_object(&zero_sha256);
test_read_object(&one_sha256);
test_read_object(&two_sha256);
test_read_object(&some_sha256);
}
void test_odb_loose__streaming_reads_sha1(void)
{
size_t blocksizes[] = { 1, 2, 4, 16, 99, 1024, 123456789 };
size_t i;
......@@ -176,7 +224,23 @@ void test_odb_loose__streaming_reads(void)
}
}
void test_odb_loose__read_header(void)
void test_odb_loose__streaming_reads_sha256(void)
{
size_t blocksizes[] = { 1, 2, 4, 16, 99, 1024, 123456789 };
size_t i;
for (i = 0; i < ARRAY_SIZE(blocksizes); i++) {
test_readstream_object(&commit_sha256, blocksizes[i]);
test_readstream_object(&tree_sha256, blocksizes[i]);
test_readstream_object(&tag_sha256, blocksizes[i]);
test_readstream_object(&zero_sha256, blocksizes[i]);
test_readstream_object(&one_sha256, blocksizes[i]);
test_readstream_object(&two_sha256, blocksizes[i]);
test_readstream_object(&some_sha256, blocksizes[i]);
}
}
void test_odb_loose__read_header_sha1(void)
{
test_read_header(&commit);
test_read_header(&tree);
......@@ -187,6 +251,17 @@ void test_odb_loose__read_header(void)
test_read_header(&some);
}
void test_odb_loose__read_header_sha256(void)
{
test_read_header(&commit_sha256);
test_read_header(&tree_sha256);
test_read_header(&tag_sha256);
test_read_header(&zero_sha256);
test_read_header(&one_sha256);
test_read_header(&two_sha256);
test_read_header(&some_sha256);
}
static void test_write_object_permission(
mode_t dir_mode, mode_t file_mode,
mode_t expected_dir_mode, mode_t expected_file_mode)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment