Commit 479c8c8c by Edward Thomson

packfile: handle sha256 packfiles

Teach the packfile machinery to cope with SHA256.
parent e89e2e01
...@@ -24,6 +24,26 @@ GIT_BEGIN_DECL ...@@ -24,6 +24,26 @@ GIT_BEGIN_DECL
* Constructors for in-box ODB backends. * Constructors for in-box ODB backends.
*/ */
/** Options for configuring a packfile object backend. */
typedef struct {
unsigned int version; /**< version for the struct */
/**
* Type of object IDs to use for this object database, or
* 0 for default (currently SHA1).
*/
git_oid_t oid_type;
} git_odb_backend_pack_options;
/* The current version of the diff options structure */
#define GIT_ODB_BACKEND_PACK_OPTIONS_VERSION 1
/* Stack initializer for odb pack backend options. Alternatively use
* `git_odb_backend_pack_options_init` programmatic initialization.
*/
#define GIT_ODB_BACKEND_PACK_OPTIONS_INIT \
{ GIT_ODB_BACKEND_PACK_OPTIONS_VERSION }
/** /**
* Create a backend for the packfiles. * Create a backend for the packfiles.
* *
...@@ -32,7 +52,38 @@ GIT_BEGIN_DECL ...@@ -32,7 +52,38 @@ GIT_BEGIN_DECL
* *
* @return 0 or an error code * @return 0 or an error code
*/ */
GIT_EXTERN(int) git_odb_backend_pack(git_odb_backend **out, const char *objects_dir); #ifdef GIT_EXPERIMENTAL_SHA256
GIT_EXTERN(int) git_odb_backend_pack(
git_odb_backend **out,
const char *objects_dir,
const git_odb_backend_pack_options *opts);
#else
GIT_EXTERN(int) git_odb_backend_pack(
git_odb_backend **out,
const char *objects_dir);
#endif
/**
* Create a backend out of a single packfile
*
* This can be useful for inspecting the contents of a single
* packfile.
*
* @param out location to store the odb backend pointer
* @param index_file path to the packfile's .idx file
*
* @return 0 or an error code
*/
#ifdef GIT_EXPERIMENTAL_SHA256
GIT_EXTERN(int) git_odb_backend_one_pack(
git_odb_backend **out,
const char *index_file,
const git_odb_backend_pack_options *opts);
#else
GIT_EXTERN(int) git_odb_backend_one_pack(
git_odb_backend **out,
const char *index_file);
#endif
typedef enum { typedef enum {
GIT_ODB_BACKEND_LOOSE_FSYNC = (1 << 0) GIT_ODB_BACKEND_LOOSE_FSYNC = (1 << 0)
...@@ -100,19 +151,6 @@ GIT_EXTERN(int) git_odb_backend_loose( ...@@ -100,19 +151,6 @@ GIT_EXTERN(int) git_odb_backend_loose(
unsigned int file_mode); unsigned int file_mode);
#endif #endif
/**
* Create a backend out of a single packfile
*
* This can be useful for inspecting the contents of a single
* packfile.
*
* @param out location to store the odb backend pointer
* @param index_file path to the packfile's .idx file
*
* @return 0 or an error code
*/
GIT_EXTERN(int) git_odb_backend_one_pack(git_odb_backend **out, const char *index_file);
/** Streaming mode */ /** Streaming mode */
typedef enum { typedef enum {
GIT_STREAM_RDONLY = (1 << 1), GIT_STREAM_RDONLY = (1 << 1),
......
...@@ -538,7 +538,7 @@ int git_commit_graph_entry_find( ...@@ -538,7 +538,7 @@ int git_commit_graph_entry_find(
hi = ntohl(file->oid_fanout[(int)short_oid->id[0]]); hi = ntohl(file->oid_fanout[(int)short_oid->id[0]]);
lo = ((short_oid->id[0] == 0x0) ? 0 : ntohl(file->oid_fanout[(int)short_oid->id[0] - 1])); lo = ((short_oid->id[0] == 0x0) ? 0 : ntohl(file->oid_fanout[(int)short_oid->id[0] - 1]));
pos = git_pack__lookup_sha1(file->oid_lookup, GIT_OID_SHA1_SIZE, lo, hi, short_oid->id); pos = git_pack__lookup_id(file->oid_lookup, GIT_OID_SHA1_SIZE, lo, hi, short_oid->id, GIT_OID_SHA1);
if (pos >= 0) { if (pos >= 0) {
/* An object matching exactly the oid was found */ /* An object matching exactly the oid was found */
...@@ -726,7 +726,8 @@ int git_commit_graph_writer_add_index_file( ...@@ -726,7 +726,8 @@ int git_commit_graph_writer_add_index_file(
if (error < 0) if (error < 0)
goto cleanup; goto cleanup;
error = git_mwindow_get_pack(&p, idx_path); /* TODO: SHA256 */
error = git_mwindow_get_pack(&p, idx_path, 0);
if (error < 0) if (error < 0)
goto cleanup; goto cleanup;
......
...@@ -179,7 +179,8 @@ int git_indexer_new( ...@@ -179,7 +179,8 @@ int git_indexer_new(
if (fd < 0) if (fd < 0)
goto cleanup; goto cleanup;
error = git_packfile_alloc(&idx->pack, git_str_cstr(&tmp_path)); /* TODO: SHA256 */
error = git_packfile_alloc(&idx->pack, git_str_cstr(&tmp_path), 0);
git_str_dispose(&tmp_path); git_str_dispose(&tmp_path);
if (error < 0) if (error < 0)
...@@ -468,16 +469,16 @@ static int store_object(git_indexer *idx) ...@@ -468,16 +469,16 @@ static int store_object(git_indexer *idx)
goto on_error; goto on_error;
} }
git_oid_cpy(&pentry->sha1, &oid); git_oid_cpy(&pentry->id, &oid);
pentry->offset = entry_start; pentry->offset = entry_start;
if (git_oidmap_exists(idx->pack->idx_cache, &pentry->sha1)) { if (git_oidmap_exists(idx->pack->idx_cache, &pentry->id)) {
git_error_set(GIT_ERROR_INDEXER, "duplicate object %s found in pack", git_oid_tostr_s(&pentry->sha1)); git_error_set(GIT_ERROR_INDEXER, "duplicate object %s found in pack", git_oid_tostr_s(&pentry->id));
git__free(pentry); git__free(pentry);
goto on_error; goto on_error;
} }
if ((error = git_oidmap_set(idx->pack->idx_cache, &pentry->sha1, pentry)) < 0) { if ((error = git_oidmap_set(idx->pack->idx_cache, &pentry->id, pentry)) < 0) {
git__free(pentry); git__free(pentry);
git_error_set_oom(); git_error_set_oom();
goto on_error; goto on_error;
...@@ -522,8 +523,8 @@ static int save_entry(git_indexer *idx, struct entry *entry, struct git_pack_ent ...@@ -522,8 +523,8 @@ static int save_entry(git_indexer *idx, struct entry *entry, struct git_pack_ent
pentry->offset = entry_start; pentry->offset = entry_start;
if (git_oidmap_exists(idx->pack->idx_cache, &pentry->sha1) || if (git_oidmap_exists(idx->pack->idx_cache, &pentry->id) ||
git_oidmap_set(idx->pack->idx_cache, &pentry->sha1, pentry) < 0) { git_oidmap_set(idx->pack->idx_cache, &pentry->id, pentry) < 0) {
git_error_set(GIT_ERROR_INDEXER, "cannot insert object into pack"); git_error_set(GIT_ERROR_INDEXER, "cannot insert object into pack");
return -1; return -1;
} }
...@@ -557,7 +558,7 @@ static int hash_and_save(git_indexer *idx, git_rawobj *obj, off64_t entry_start) ...@@ -557,7 +558,7 @@ static int hash_and_save(git_indexer *idx, git_rawobj *obj, off64_t entry_start)
pentry = git__calloc(1, sizeof(struct git_pack_entry)); pentry = git__calloc(1, sizeof(struct git_pack_entry));
GIT_ERROR_CHECK_ALLOC(pentry); GIT_ERROR_CHECK_ALLOC(pentry);
git_oid_cpy(&pentry->sha1, &oid); git_oid_cpy(&pentry->id, &oid);
git_oid_cpy(&entry->oid, &oid); git_oid_cpy(&entry->oid, &oid);
entry->crc = crc32(0L, Z_NULL, 0); entry->crc = crc32(0L, Z_NULL, 0);
...@@ -987,7 +988,7 @@ static int inject_object(git_indexer *idx, git_oid *id) ...@@ -987,7 +988,7 @@ static int inject_object(git_indexer *idx, git_oid *id)
pentry = git__calloc(1, sizeof(struct git_pack_entry)); pentry = git__calloc(1, sizeof(struct git_pack_entry));
GIT_ERROR_CHECK_ALLOC(pentry); GIT_ERROR_CHECK_ALLOC(pentry);
git_oid_cpy(&pentry->sha1, id); git_oid_cpy(&pentry->id, id);
git_oid_cpy(&entry->oid, id); git_oid_cpy(&entry->oid, id);
idx->off = entry_start + hdr_len + len; idx->off = entry_start + hdr_len + len;
......
...@@ -392,7 +392,7 @@ int git_midx_entry_find( ...@@ -392,7 +392,7 @@ int git_midx_entry_find(
hi = ntohl(idx->oid_fanout[(int)short_oid->id[0]]); hi = ntohl(idx->oid_fanout[(int)short_oid->id[0]]);
lo = ((short_oid->id[0] == 0x0) ? 0 : ntohl(idx->oid_fanout[(int)short_oid->id[0] - 1])); lo = ((short_oid->id[0] == 0x0) ? 0 : ntohl(idx->oid_fanout[(int)short_oid->id[0] - 1]));
pos = git_pack__lookup_sha1(idx->oid_lookup, GIT_OID_SHA1_SIZE, lo, hi, short_oid->id); pos = git_pack__lookup_id(idx->oid_lookup, GIT_OID_SHA1_SIZE, lo, hi, short_oid->id, GIT_OID_SHA1);
if (pos >= 0) { if (pos >= 0) {
/* An object matching exactly the oid was found */ /* An object matching exactly the oid was found */
...@@ -549,7 +549,8 @@ int git_midx_writer_add( ...@@ -549,7 +549,8 @@ int git_midx_writer_add(
if (error < 0) if (error < 0)
return error; return error;
error = git_mwindow_get_pack(&p, git_str_cstr(&idx_path_buf)); /* TODO: SHA256 */
error = git_mwindow_get_pack(&p, git_str_cstr(&idx_path_buf), 0);
git_str_dispose(&idx_path_buf); git_str_dispose(&idx_path_buf);
if (error < 0) if (error < 0)
return error; return error;
......
...@@ -61,7 +61,10 @@ int git_mwindow_global_init(void) ...@@ -61,7 +61,10 @@ int git_mwindow_global_init(void)
return git_runtime_shutdown_register(git_mwindow_global_shutdown); return git_runtime_shutdown_register(git_mwindow_global_shutdown);
} }
int git_mwindow_get_pack(struct git_pack_file **out, const char *path) int git_mwindow_get_pack(
struct git_pack_file **out,
const char *path,
git_oid_t oid_type)
{ {
struct git_pack_file *pack; struct git_pack_file *pack;
char *packname; char *packname;
...@@ -86,7 +89,7 @@ int git_mwindow_get_pack(struct git_pack_file **out, const char *path) ...@@ -86,7 +89,7 @@ int git_mwindow_get_pack(struct git_pack_file **out, const char *path)
} }
/* If we didn't find it, we need to create it */ /* If we didn't find it, we need to create it */
if ((error = git_packfile_alloc(&pack, path)) < 0) { if ((error = git_packfile_alloc(&pack, path, oid_type)) < 0) {
git_mutex_unlock(&git__mwindow_mutex); git_mutex_unlock(&git__mwindow_mutex);
return error; return error;
} }
......
...@@ -48,7 +48,10 @@ void git_mwindow_close(git_mwindow **w_cursor); ...@@ -48,7 +48,10 @@ void git_mwindow_close(git_mwindow **w_cursor);
extern int git_mwindow_global_init(void); extern int git_mwindow_global_init(void);
struct git_pack_file; /* just declaration to avoid cyclical includes */ struct git_pack_file; /* just declaration to avoid cyclical includes */
int git_mwindow_get_pack(struct git_pack_file **out, const char *path); int git_mwindow_get_pack(
struct git_pack_file **out,
const char *path,
git_oid_t oid_type);
int git_mwindow_put_pack(struct git_pack_file *pack); int git_mwindow_put_pack(struct git_pack_file *pack);
#endif #endif
...@@ -684,6 +684,7 @@ int git_odb__add_default_backends( ...@@ -684,6 +684,7 @@ int git_odb__add_default_backends(
ino_t inode; ino_t inode;
git_odb_backend *loose, *packed; git_odb_backend *loose, *packed;
git_odb_backend_loose_options loose_opts = GIT_ODB_BACKEND_LOOSE_OPTIONS_INIT; git_odb_backend_loose_options loose_opts = GIT_ODB_BACKEND_LOOSE_OPTIONS_INIT;
git_odb_backend_pack_options pack_opts = GIT_ODB_BACKEND_PACK_OPTIONS_INIT;
/* TODO: inodes are not really relevant on Win32, so we need to find /* TODO: inodes are not really relevant on Win32, so we need to find
* a cross-platform workaround for this */ * a cross-platform workaround for this */
...@@ -722,6 +723,7 @@ int git_odb__add_default_backends( ...@@ -722,6 +723,7 @@ int git_odb__add_default_backends(
loose_opts.flags |= GIT_ODB_BACKEND_LOOSE_FSYNC; loose_opts.flags |= GIT_ODB_BACKEND_LOOSE_FSYNC;
loose_opts.oid_type = db->options.oid_type; loose_opts.oid_type = db->options.oid_type;
pack_opts.oid_type = db->options.oid_type;
/* add the loose object backend */ /* add the loose object backend */
if (git_odb__backend_loose(&loose, objects_dir, &loose_opts) < 0 || if (git_odb__backend_loose(&loose, objects_dir, &loose_opts) < 0 ||
...@@ -729,8 +731,17 @@ int git_odb__add_default_backends( ...@@ -729,8 +731,17 @@ int git_odb__add_default_backends(
return -1; return -1;
/* add the packed file backend */ /* add the packed file backend */
if (git_odb_backend_pack(&packed, objects_dir) < 0 || #ifdef GIT_EXPERIMENTAL_SHA256
add_backend_internal(db, packed, git_odb__packed_priority, as_alternates, inode) < 0) if (git_odb_backend_pack(&packed, objects_dir, &pack_opts) < 0)
return -1;
#else
GIT_UNUSED(pack_opts);
if (git_odb_backend_pack(&packed, objects_dir) < 0)
return -1;
#endif
if (add_backend_internal(db, packed, git_odb__packed_priority, as_alternates, inode) < 0)
return -1; return -1;
if (git_mutex_lock(&db->lock) < 0) { if (git_mutex_lock(&db->lock) < 0) {
......
...@@ -26,6 +26,7 @@ ...@@ -26,6 +26,7 @@
struct pack_backend { struct pack_backend {
git_odb_backend parent; git_odb_backend parent;
git_odb_backend_pack_options opts;
git_midx_file *midx; git_midx_file *midx;
git_vector midx_packs; git_vector midx_packs;
git_vector packs; git_vector packs;
...@@ -95,24 +96,24 @@ struct pack_writepack { ...@@ -95,24 +96,24 @@ struct pack_writepack {
* -------------------------------------------------- * --------------------------------------------------
* *
* # pack_backend__exists / pack_backend__exists_prefix * # pack_backend__exists / pack_backend__exists_prefix
* | Check if the given SHA1 oid (or a SHA1 oid prefix) exists in any of the * | Check if the given oid (or an oid prefix) exists in any of the
* | packs that have been loaded for our ODB. * | packs that have been loaded for our ODB.
* | * |
* |-# pack_entry_find / pack_entry_find_prefix * |-# pack_entry_find / pack_entry_find_prefix
* | If there is a multi-pack-index present, search the SHA1 oid in that * | If there is a multi-pack-index present, search the oid in that
* | index first. If it is not found there, iterate through all the unindexed * | index first. If it is not found there, iterate through all the unindexed
* | packs that have been preloaded (starting by the pack where the latest * | packs that have been preloaded (starting by the pack where the latest
* | object was found) to try to find the OID in one of them. * | object was found) to try to find the OID in one of them.
* | * |
* |-# git_midx_entry_find * |-# git_midx_entry_find
* | Search for the SHA1 oid in the multi-pack-index. See * | Search for the oid in the multi-pack-index. See
* | <https://github.com/git/git/blob/master/Documentation/technical/pack-format.txt> * | <https://github.com/git/git/blob/master/Documentation/technical/pack-format.txt>
* | for specifics on the multi-pack-index format and how do we find * | for specifics on the multi-pack-index format and how do we find
* | entries in it. * | entries in it.
* | * |
* |-# git_pack_entry_find * |-# git_pack_entry_find
* | Check the index of an individual unindexed pack to see if the SHA1 * | Check the index of an individual unindexed pack to see if the
* | OID can be found. If we can find the offset to that SHA1 inside of the * | OID can be found. If we can find the offset to that inside of the
* | index, that means the object is contained inside of the packfile and * | index, that means the object is contained inside of the packfile and
* | we can stop searching. Before returning, we verify that the * | we can stop searching. Before returning, we verify that the
* | packfile behind the index we are searching still exists on disk. * | packfile behind the index we are searching still exists on disk.
...@@ -141,13 +142,13 @@ struct pack_writepack { ...@@ -141,13 +142,13 @@ struct pack_writepack {
* -------------------------------------------------- * --------------------------------------------------
* *
* # pack_backend__read / pack_backend__read_prefix * # pack_backend__read / pack_backend__read_prefix
* | Check if the given SHA1 oid (or a SHA1 oid prefix) exists in any of the * | Check if the given oid (or an oid prefix) exists in any of the
* | packs that have been loaded for our ODB. If it does, open the packfile and * | packs that have been loaded for our ODB. If it does, open the packfile and
* | read from it. * | read from it.
* | * |
* |-# git_packfile_unpack * |-# git_packfile_unpack
* Armed with a packfile and the offset within it, we can finally unpack * Armed with a packfile and the offset within it, we can finally unpack
* the object pointed at by the SHA1 oid. This involves mmapping part of * the object pointed at by the oid. This involves mmapping part of
* the `.pack` file, and uncompressing the object within it (if it is * the `.pack` file, and uncompressing the object within it (if it is
* stored in the undelfitied representation), or finding a base object and * stored in the undelfitied representation), or finding a base object and
* applying some deltas to its uncompressed representation (if it is stored * applying some deltas to its uncompressed representation (if it is stored
...@@ -177,7 +178,7 @@ static int pack_entry_find(struct git_pack_entry *e, ...@@ -177,7 +178,7 @@ static int pack_entry_find(struct git_pack_entry *e,
* a prefix of an identifier. * a prefix of an identifier.
* Sets GIT_EAMBIGUOUS if short oid is ambiguous. * Sets GIT_EAMBIGUOUS if short oid is ambiguous.
* This method assumes that len is between * This method assumes that len is between
* GIT_OID_MINPREFIXLEN and GIT_OID_SHA1_HEXSIZE. * GIT_OID_MINPREFIXLEN and the hexsize for the hash type.
*/ */
static int pack_entry_find_prefix( static int pack_entry_find_prefix(
struct git_pack_entry *e, struct git_pack_entry *e,
...@@ -251,7 +252,7 @@ static int packfile_load__cb(void *data, git_str *path) ...@@ -251,7 +252,7 @@ static int packfile_load__cb(void *data, git_str *path)
if (git_vector_search2(NULL, &backend->packs, packfile_byname_search_cmp, &index_prefix) == 0) if (git_vector_search2(NULL, &backend->packs, packfile_byname_search_cmp, &index_prefix) == 0)
return 0; return 0;
error = git_mwindow_get_pack(&pack, path->ptr); error = git_mwindow_get_pack(&pack, path->ptr, backend->opts.oid_type);
/* ignore missing .pack file as git does */ /* ignore missing .pack file as git does */
if (error == GIT_ENOTFOUND) { if (error == GIT_ENOTFOUND) {
...@@ -270,33 +271,34 @@ static int pack_entry_find(struct git_pack_entry *e, struct pack_backend *backen ...@@ -270,33 +271,34 @@ static int pack_entry_find(struct git_pack_entry *e, struct pack_backend *backen
{ {
struct git_pack_file *last_found = backend->last_found, *p; struct git_pack_file *last_found = backend->last_found, *p;
git_midx_entry midx_entry; git_midx_entry midx_entry;
size_t oid_hexsize = git_oid_hexsize(backend->opts.oid_type);
size_t i; size_t i;
if (backend->midx && if (backend->midx &&
git_midx_entry_find(&midx_entry, backend->midx, oid, GIT_OID_SHA1_HEXSIZE) == 0 && git_midx_entry_find(&midx_entry, backend->midx, oid, oid_hexsize) == 0 &&
midx_entry.pack_index < git_vector_length(&backend->midx_packs)) { midx_entry.pack_index < git_vector_length(&backend->midx_packs)) {
e->offset = midx_entry.offset; e->offset = midx_entry.offset;
git_oid_cpy(&e->sha1, &midx_entry.sha1); git_oid_cpy(&e->id, &midx_entry.sha1);
e->p = git_vector_get(&backend->midx_packs, midx_entry.pack_index); e->p = git_vector_get(&backend->midx_packs, midx_entry.pack_index);
return 0; return 0;
} }
if (last_found && if (last_found &&
git_pack_entry_find(e, last_found, oid, GIT_OID_SHA1_HEXSIZE) == 0) git_pack_entry_find(e, last_found, oid, oid_hexsize) == 0)
return 0; return 0;
git_vector_foreach(&backend->packs, i, p) { git_vector_foreach(&backend->packs, i, p) {
if (p == last_found) if (p == last_found)
continue; continue;
if (git_pack_entry_find(e, p, oid, GIT_OID_SHA1_HEXSIZE) == 0) { if (git_pack_entry_find(e, p, oid, oid_hexsize) == 0) {
backend->last_found = p; backend->last_found = p;
return 0; return 0;
} }
} }
return git_odb__error_notfound( return git_odb__error_notfound(
"failed to find pack entry", oid, GIT_OID_SHA1_HEXSIZE); "failed to find pack entry", oid, oid_hexsize);
} }
static int pack_entry_find_prefix( static int pack_entry_find_prefix(
...@@ -318,9 +320,9 @@ static int pack_entry_find_prefix( ...@@ -318,9 +320,9 @@ static int pack_entry_find_prefix(
return error; return error;
if (!error && midx_entry.pack_index < git_vector_length(&backend->midx_packs)) { if (!error && midx_entry.pack_index < git_vector_length(&backend->midx_packs)) {
e->offset = midx_entry.offset; e->offset = midx_entry.offset;
git_oid_cpy(&e->sha1, &midx_entry.sha1); git_oid_cpy(&e->id, &midx_entry.sha1);
e->p = git_vector_get(&backend->midx_packs, midx_entry.pack_index); e->p = git_vector_get(&backend->midx_packs, midx_entry.pack_index);
git_oid_cpy(&found_full_oid, &e->sha1); git_oid_cpy(&found_full_oid, &e->id);
found = true; found = true;
} }
} }
...@@ -330,9 +332,9 @@ static int pack_entry_find_prefix( ...@@ -330,9 +332,9 @@ static int pack_entry_find_prefix(
if (error == GIT_EAMBIGUOUS) if (error == GIT_EAMBIGUOUS)
return error; return error;
if (!error) { if (!error) {
if (found && git_oid_cmp(&e->sha1, &found_full_oid)) if (found && git_oid_cmp(&e->id, &found_full_oid))
return git_odb__error_ambiguous("found multiple pack entries"); return git_odb__error_ambiguous("found multiple pack entries");
git_oid_cpy(&found_full_oid, &e->sha1); git_oid_cpy(&found_full_oid, &e->id);
found = true; found = true;
} }
} }
...@@ -345,9 +347,9 @@ static int pack_entry_find_prefix( ...@@ -345,9 +347,9 @@ static int pack_entry_find_prefix(
if (error == GIT_EAMBIGUOUS) if (error == GIT_EAMBIGUOUS)
return error; return error;
if (!error) { if (!error) {
if (found && git_oid_cmp(&e->sha1, &found_full_oid)) if (found && git_oid_cmp(&e->id, &found_full_oid))
return git_odb__error_ambiguous("found multiple pack entries"); return git_odb__error_ambiguous("found multiple pack entries");
git_oid_cpy(&found_full_oid, &e->sha1); git_oid_cpy(&found_full_oid, &e->id);
found = true; found = true;
backend->last_found = p; backend->last_found = p;
} }
...@@ -425,7 +427,10 @@ static int process_multi_pack_index_pack( ...@@ -425,7 +427,10 @@ static int process_multi_pack_index_pack(
} }
/* Pack was not found. Allocate a new one. */ /* Pack was not found. Allocate a new one. */
error = git_mwindow_get_pack(&pack, git_str_cstr(&pack_path)); error = git_mwindow_get_pack(
&pack,
git_str_cstr(&pack_path),
backend->opts.oid_type);
git_str_dispose(&pack_path); git_str_dispose(&pack_path);
if (error < 0) if (error < 0)
return error; return error;
...@@ -596,32 +601,33 @@ static int pack_backend__read_prefix( ...@@ -596,32 +601,33 @@ static int pack_backend__read_prefix(
void **buffer_p, void **buffer_p,
size_t *len_p, size_t *len_p,
git_object_t *type_p, git_object_t *type_p,
git_odb_backend *backend, git_odb_backend *_backend,
const git_oid *short_oid, const git_oid *short_oid,
size_t len) size_t len)
{ {
struct pack_backend *backend = (struct pack_backend *)_backend;
int error = 0; int error = 0;
if (len < GIT_OID_MINPREFIXLEN) if (len < GIT_OID_MINPREFIXLEN)
error = git_odb__error_ambiguous("prefix length too short"); error = git_odb__error_ambiguous("prefix length too short");
else if (len >= GIT_OID_SHA1_HEXSIZE) { else if (len >= git_oid_hexsize(backend->opts.oid_type)) {
/* We can fall back to regular read method */ /* We can fall back to regular read method */
error = pack_backend__read(buffer_p, len_p, type_p, backend, short_oid); error = pack_backend__read(buffer_p, len_p, type_p, _backend, short_oid);
if (!error) if (!error)
git_oid_cpy(out_oid, short_oid); git_oid_cpy(out_oid, short_oid);
} else { } else {
struct git_pack_entry e; struct git_pack_entry e;
git_rawobj raw = {NULL}; git_rawobj raw = {NULL};
if ((error = pack_entry_find_prefix( if ((error = pack_entry_find_prefix(&e,
&e, (struct pack_backend *)backend, short_oid, len)) == 0 && backend, short_oid, len)) == 0 &&
(error = git_packfile_unpack(&raw, e.p, &e.offset)) == 0) (error = git_packfile_unpack(&raw, e.p, &e.offset)) == 0)
{ {
*buffer_p = raw.data; *buffer_p = raw.data;
*len_p = raw.len; *len_p = raw.len;
*type_p = raw.type; *type_p = raw.type;
git_oid_cpy(out_oid, &e.sha1); git_oid_cpy(out_oid, &e.id);
} }
} }
...@@ -642,7 +648,7 @@ static int pack_backend__exists_prefix( ...@@ -642,7 +648,7 @@ static int pack_backend__exists_prefix(
struct git_pack_entry e = {0}; struct git_pack_entry e = {0};
error = pack_entry_find_prefix(&e, pb, short_id, len); error = pack_entry_find_prefix(&e, pb, short_id, len);
git_oid_cpy(out, &e.sha1); git_oid_cpy(out, &e.id);
return error; return error;
} }
...@@ -840,7 +846,10 @@ static void pack_backend__free(git_odb_backend *_backend) ...@@ -840,7 +846,10 @@ static void pack_backend__free(git_odb_backend *_backend)
git__free(backend); git__free(backend);
} }
static int pack_backend__alloc(struct pack_backend **out, size_t initial_size) static int pack_backend__alloc(
struct pack_backend **out,
size_t initial_size,
const git_odb_backend_pack_options *opts)
{ {
struct pack_backend *backend = git__calloc(1, sizeof(struct pack_backend)); struct pack_backend *backend = git__calloc(1, sizeof(struct pack_backend));
GIT_ERROR_CHECK_ALLOC(backend); GIT_ERROR_CHECK_ALLOC(backend);
...@@ -849,12 +858,19 @@ static int pack_backend__alloc(struct pack_backend **out, size_t initial_size) ...@@ -849,12 +858,19 @@ static int pack_backend__alloc(struct pack_backend **out, size_t initial_size)
git__free(backend); git__free(backend);
return -1; return -1;
} }
if (git_vector_init(&backend->packs, initial_size, packfile_sort__cb) < 0) { if (git_vector_init(&backend->packs, initial_size, packfile_sort__cb) < 0) {
git_vector_free(&backend->midx_packs); git_vector_free(&backend->midx_packs);
git__free(backend); git__free(backend);
return -1; return -1;
} }
if (opts)
memcpy(&backend->opts, opts, sizeof(git_odb_backend_pack_options));
if (!backend->opts.oid_type)
backend->opts.oid_type = GIT_OID_DEFAULT;
backend->parent.version = GIT_ODB_BACKEND_VERSION; backend->parent.version = GIT_ODB_BACKEND_VERSION;
backend->parent.read = &pack_backend__read; backend->parent.read = &pack_backend__read;
...@@ -873,17 +889,31 @@ static int pack_backend__alloc(struct pack_backend **out, size_t initial_size) ...@@ -873,17 +889,31 @@ static int pack_backend__alloc(struct pack_backend **out, size_t initial_size)
return 0; return 0;
} }
int git_odb_backend_one_pack(git_odb_backend **backend_out, const char *idx) #ifdef GIT_EXPERIMENTAL_SHA256
int git_odb_backend_one_pack(
git_odb_backend **backend_out,
const char *idx,
const git_odb_backend_pack_options *opts)
#else
int git_odb_backend_one_pack(
git_odb_backend **backend_out,
const char *idx)
#endif
{ {
struct pack_backend *backend = NULL; struct pack_backend *backend = NULL;
struct git_pack_file *packfile = NULL; struct git_pack_file *packfile = NULL;
if (pack_backend__alloc(&backend, 1) < 0) #ifndef GIT_EXPERIMENTAL_SHA256
git_odb_backend_pack_options *opts = NULL;
#endif
git_oid_t oid_type = opts ? opts->oid_type : 0;
if (pack_backend__alloc(&backend, 1, opts) < 0)
return -1; return -1;
if (git_mwindow_get_pack(&packfile, idx) < 0 || if (git_mwindow_get_pack(&packfile, idx, oid_type) < 0 ||
git_vector_insert(&backend->packs, packfile) < 0) git_vector_insert(&backend->packs, packfile) < 0) {
{
pack_backend__free((git_odb_backend *)backend); pack_backend__free((git_odb_backend *)backend);
return -1; return -1;
} }
...@@ -892,18 +922,30 @@ int git_odb_backend_one_pack(git_odb_backend **backend_out, const char *idx) ...@@ -892,18 +922,30 @@ int git_odb_backend_one_pack(git_odb_backend **backend_out, const char *idx)
return 0; return 0;
} }
int git_odb_backend_pack(git_odb_backend **backend_out, const char *objects_dir) #ifdef GIT_EXPERIMENTAL_SHA256
int git_odb_backend_pack(
git_odb_backend **backend_out,
const char *objects_dir,
const git_odb_backend_pack_options *opts)
#else
int git_odb_backend_pack(
git_odb_backend **backend_out,
const char *objects_dir)
#endif
{ {
int error = 0; int error = 0;
struct pack_backend *backend = NULL; struct pack_backend *backend = NULL;
git_str path = GIT_STR_INIT; git_str path = GIT_STR_INIT;
if (pack_backend__alloc(&backend, 8) < 0) #ifndef GIT_EXPERIMENTAL_SHA256
git_odb_backend_pack_options *opts = NULL;
#endif
if (pack_backend__alloc(&backend, 8, opts) < 0)
return -1; return -1;
if (!(error = git_str_joinpath(&path, objects_dir, "pack")) && if (!(error = git_str_joinpath(&path, objects_dir, "pack")) &&
git_fs_path_isdir(git_str_cstr(&path))) git_fs_path_isdir(git_str_cstr(&path))) {
{
backend->pack_folder = git_str_detach(&path); backend->pack_folder = git_str_detach(&path);
error = pack_backend__refresh((git_odb_backend *)backend); error = pack_backend__refresh((git_odb_backend *)backend);
} }
......
...@@ -32,7 +32,7 @@ static int packfile_unpack_compressed( ...@@ -32,7 +32,7 @@ static int packfile_unpack_compressed(
* Throws GIT_EAMBIGUOUSOIDPREFIX if short oid * Throws GIT_EAMBIGUOUSOIDPREFIX if short oid
* is ambiguous within the pack. * is ambiguous within the pack.
* This method assumes that len is between * This method assumes that len is between
* GIT_OID_MINPREFIXLEN and GIT_OID_SHA1_HEXSIZE. * GIT_OID_MINPREFIXLEN and the oid type's hexsize.
*/ */
static int pack_entry_find_offset( static int pack_entry_find_offset(
off64_t *offset_out, off64_t *offset_out,
...@@ -186,9 +186,9 @@ static int cache_add( ...@@ -186,9 +186,9 @@ static int cache_add(
static void pack_index_free(struct git_pack_file *p) static void pack_index_free(struct git_pack_file *p)
{ {
if (p->oids) { if (p->ids) {
git__free(p->oids); git__free(p->ids);
p->oids = NULL; p->ids = NULL;
} }
if (p->index_map.data) { if (p->index_map.data) {
git_futils_mmap_free(&p->index_map); git_futils_mmap_free(&p->index_map);
...@@ -205,6 +205,7 @@ static int pack_index_check_locked(const char *path, struct git_pack_file *p) ...@@ -205,6 +205,7 @@ static int pack_index_check_locked(const char *path, struct git_pack_file *p)
size_t idx_size; size_t idx_size;
struct stat st; struct stat st;
int error; int error;
/* TODO: properly open the file without access time using O_NOATIME */ /* TODO: properly open the file without access time using O_NOATIME */
git_file fd = git_futils_open_ro(path); git_file fd = git_futils_open_ro(path);
if (fd < 0) if (fd < 0)
...@@ -218,8 +219,7 @@ static int pack_index_check_locked(const char *path, struct git_pack_file *p) ...@@ -218,8 +219,7 @@ static int pack_index_check_locked(const char *path, struct git_pack_file *p)
if (!S_ISREG(st.st_mode) || if (!S_ISREG(st.st_mode) ||
!git__is_sizet(st.st_size) || !git__is_sizet(st.st_size) ||
(idx_size = (size_t)st.st_size) < 4 * 256 + 20 + 20) (idx_size = (size_t)st.st_size) < (size_t)((4 * 256) + (p->oid_size * 2))) {
{
p_close(fd); p_close(fd);
git_error_set(GIT_ERROR_ODB, "invalid pack index '%s'", path); git_error_set(GIT_ERROR_ODB, "invalid pack index '%s'", path);
return -1; return -1;
...@@ -242,8 +242,9 @@ static int pack_index_check_locked(const char *path, struct git_pack_file *p) ...@@ -242,8 +242,9 @@ static int pack_index_check_locked(const char *path, struct git_pack_file *p)
return packfile_error("unsupported index version"); return packfile_error("unsupported index version");
} }
} else } else {
version = 1; version = 1;
}
nr = 0; nr = 0;
index = idx_map; index = idx_map;
...@@ -264,11 +265,11 @@ static int pack_index_check_locked(const char *path, struct git_pack_file *p) ...@@ -264,11 +265,11 @@ static int pack_index_check_locked(const char *path, struct git_pack_file *p)
/* /*
* Total size: * Total size:
* - 256 index entries 4 bytes each * - 256 index entries 4 bytes each
* - 24-byte entries * nr (20-byte sha1 + 4-byte offset) * - 24/36-byte entries * nr (20/32 byte SHA + 4-byte offset)
* - 20-byte SHA1 of the packfile * - 20/32-byte SHA of the packfile
* - 20-byte SHA1 file checksum * - 20/32-byte SHA file checksum
*/ */
if (idx_size != 4*256 + nr * 24 + 20 + 20) { if (idx_size != (4 * 256 + (nr * (p->oid_size + 4)) + (p->oid_size * 2))) {
git_futils_mmap_free(&p->index_map); git_futils_mmap_free(&p->index_map);
return packfile_error("index is corrupted"); return packfile_error("index is corrupted");
} }
...@@ -277,16 +278,16 @@ static int pack_index_check_locked(const char *path, struct git_pack_file *p) ...@@ -277,16 +278,16 @@ static int pack_index_check_locked(const char *path, struct git_pack_file *p)
* Minimum size: * Minimum size:
* - 8 bytes of header * - 8 bytes of header
* - 256 index entries 4 bytes each * - 256 index entries 4 bytes each
* - 20-byte sha1 entry * nr * - 20/32-byte SHA entry * nr
* - 4-byte crc entry * nr * - 4-byte crc entry * nr
* - 4-byte offset entry * nr * - 4-byte offset entry * nr
* - 20-byte SHA1 of the packfile * - 20/32-byte SHA of the packfile
* - 20-byte SHA1 file checksum * - 20/32-byte SHA file checksum
* And after the 4-byte offset table might be a * And after the 4-byte offset table might be a
* variable sized table containing 8-byte entries * variable sized table containing 8-byte entries
* for offsets larger than 2^31. * for offsets larger than 2^31.
*/ */
unsigned long min_size = 8 + 4*256 + nr*(20 + 4 + 4) + 20 + 20; unsigned long min_size = 8 + (4 * 256) + (nr * (p->oid_size + 4 + 4)) + (p->oid_size * 2);
unsigned long max_size = min_size; unsigned long max_size = min_size;
if (nr) if (nr)
...@@ -365,12 +366,12 @@ static unsigned char *pack_window_open( ...@@ -365,12 +366,12 @@ static unsigned char *pack_window_open(
* Don't allow a negative offset, as that means we've wrapped * Don't allow a negative offset, as that means we've wrapped
* around. * around.
*/ */
if (offset > (p->mwf.size - 20)) if (offset > (p->mwf.size - p->oid_size))
goto cleanup; goto cleanup;
if (offset < 0) if (offset < 0)
goto cleanup; goto cleanup;
pack_data = git_mwindow_open(&p->mwf, w_cursor, offset, 20, left); pack_data = git_mwindow_open(&p->mwf, w_cursor, offset, p->oid_size, left);
cleanup: cleanup:
git_mutex_unlock(&p->mwf.lock); git_mutex_unlock(&p->mwf.lock);
...@@ -473,13 +474,13 @@ int git_packfile_unpack_header( ...@@ -473,13 +474,13 @@ int git_packfile_unpack_header(
return error; return error;
} }
/* pack_window_open() assures us we have [base, base + 20) available /* pack_window_open() assures us we have [base, base + oid_size)
* as a range that we can look at at. (Its actually the hash * available as a range that we can look at at. (It's actually
* size that is assured.) With our object header encoding * the hash size that is assured.) With our object header
* the maximum deflated object size is 2^137, which is just * encoding the maximum deflated object size is 2^137, which is
* insane, so we know won't exceed what we have been given. * just insane, so we know won't exceed what we have been given.
*/ */
base = git_mwindow_open(&p->mwf, w_curs, *curpos, 20, &left); base = git_mwindow_open(&p->mwf, w_curs, *curpos, p->oid_size, &left);
git_mutex_unlock(&p->lock); git_mutex_unlock(&p->lock);
git_mutex_unlock(&p->mwf.lock); git_mutex_unlock(&p->mwf.lock);
if (base == NULL) if (base == NULL)
...@@ -977,11 +978,12 @@ int get_delta_base( ...@@ -977,11 +978,12 @@ int get_delta_base(
/* Assumption: the only reason this would fail is because the file is too small */ /* Assumption: the only reason this would fail is because the file is too small */
if (base_info == NULL) if (base_info == NULL)
return GIT_EBUFS; return GIT_EBUFS;
/* pack_window_open() assured us we have [base_info, base_info + 20) /* pack_window_open() assured us we have
* as a range that we can look at without walking off the * [base_info, base_info + oid_size) as a range that we can look
* end of the mapped window. Its actually the hash size * at without walking off the end of the mapped window. Its
* that is assured. An OFS_DELTA longer than the hash size * actually the hash size that is assured. An OFS_DELTA longer
* is stupid, as then a REF_DELTA would be smaller to store. * than the hash size is stupid, as then a REF_DELTA would be
* smaller to store.
*/ */
if (type == GIT_OBJECT_OFS_DELTA) { if (type == GIT_OBJECT_OFS_DELTA) {
unsigned used = 0; unsigned used = 0;
...@@ -1002,7 +1004,7 @@ int get_delta_base( ...@@ -1002,7 +1004,7 @@ int get_delta_base(
*curpos += used; *curpos += used;
} else if (type == GIT_OBJECT_REF_DELTA) { } else if (type == GIT_OBJECT_REF_DELTA) {
git_oid base_oid; git_oid base_oid;
git_oid__fromraw(&base_oid, base_info, GIT_OID_SHA1); git_oid__fromraw(&base_oid, base_info, p->oid_type);
/* If we have the cooperative cache, search in it first */ /* If we have the cooperative cache, search in it first */
if (p->has_cache) { if (p->has_cache) {
...@@ -1012,7 +1014,7 @@ int get_delta_base( ...@@ -1012,7 +1014,7 @@ int get_delta_base(
if (entry->offset == 0) if (entry->offset == 0)
return packfile_error("delta offset is zero"); return packfile_error("delta offset is zero");
*curpos += 20; *curpos += p->oid_size;
*delta_base_out = entry->offset; *delta_base_out = entry->offset;
return 0; return 0;
} else { } else {
...@@ -1025,9 +1027,9 @@ int get_delta_base( ...@@ -1025,9 +1027,9 @@ int get_delta_base(
} }
/* The base entry _must_ be in the same pack */ /* The base entry _must_ be in the same pack */
if (pack_entry_find_offset(&base_offset, &unused, p, &base_oid, GIT_OID_SHA1_HEXSIZE) < 0) if (pack_entry_find_offset(&base_offset, &unused, p, &base_oid, p->oid_hexsize) < 0)
return packfile_error("base entry delta is not in the same pack"); return packfile_error("base entry delta is not in the same pack");
*curpos += 20; *curpos += p->oid_size;
} else } else
return packfile_error("unknown object type"); return packfile_error("unknown object type");
...@@ -1070,7 +1072,7 @@ void git_packfile_free(struct git_pack_file *p, bool unlink_packfile) ...@@ -1070,7 +1072,7 @@ void git_packfile_free(struct git_pack_file *p, bool unlink_packfile)
pack_index_free(p); pack_index_free(p);
git__free(p->bad_object_sha1); git__free(p->bad_object_ids);
git_mutex_free(&p->bases.lock); git_mutex_free(&p->bases.lock);
git_mutex_free(&p->mwf.lock); git_mutex_free(&p->mwf.lock);
...@@ -1083,8 +1085,8 @@ static int packfile_open_locked(struct git_pack_file *p) ...@@ -1083,8 +1085,8 @@ static int packfile_open_locked(struct git_pack_file *p)
{ {
struct stat st; struct stat st;
struct git_pack_header hdr; struct git_pack_header hdr;
unsigned char sha1[GIT_OID_SHA1_SIZE]; unsigned char checksum[GIT_OID_MAX_SIZE];
unsigned char *idx_sha1; unsigned char *idx_checksum;
if (pack_index_open_locked(p) < 0) if (pack_index_open_locked(p) < 0)
return git_odb__error_notfound("failed to open packfile", NULL, 0); return git_odb__error_notfound("failed to open packfile", NULL, 0);
...@@ -1131,12 +1133,13 @@ static int packfile_open_locked(struct git_pack_file *p) ...@@ -1131,12 +1133,13 @@ static int packfile_open_locked(struct git_pack_file *p)
/* Verify the pack matches its index. */ /* Verify the pack matches its index. */
if (p->num_objects != ntohl(hdr.hdr_entries) || if (p->num_objects != ntohl(hdr.hdr_entries) ||
p_pread(p->mwf.fd, sha1, GIT_OID_SHA1_SIZE, p->mwf.size - GIT_OID_SHA1_SIZE) < 0) p_pread(p->mwf.fd, checksum, p->oid_size, p->mwf.size - p->oid_size) < 0)
goto cleanup; goto cleanup;
idx_sha1 = ((unsigned char *)p->index_map.data) + p->index_map.len - 40; idx_checksum = ((unsigned char *)p->index_map.data) +
p->index_map.len - (p->oid_size * 2);
if (git_oid_raw_cmp(sha1, idx_sha1, GIT_OID_SHA1_SIZE) != 0) if (git_oid_raw_cmp(checksum, idx_checksum, p->oid_size) != 0)
goto cleanup; goto cleanup;
if (git_mwindow_file_register(&p->mwf) < 0) if (git_mwindow_file_register(&p->mwf) < 0)
...@@ -1171,7 +1174,10 @@ int git_packfile__name(char **out, const char *path) ...@@ -1171,7 +1174,10 @@ int git_packfile__name(char **out, const char *path)
return 0; return 0;
} }
int git_packfile_alloc(struct git_pack_file **pack_out, const char *path) int git_packfile_alloc(
struct git_pack_file **pack_out,
const char *path,
git_oid_t oid_type)
{ {
struct stat st; struct stat st;
struct git_pack_file *p; struct git_pack_file *p;
...@@ -1219,6 +1225,9 @@ int git_packfile_alloc(struct git_pack_file **pack_out, const char *path) ...@@ -1219,6 +1225,9 @@ int git_packfile_alloc(struct git_pack_file **pack_out, const char *path)
p->pack_local = 1; p->pack_local = 1;
p->mtime = (git_time_t)st.st_mtime; p->mtime = (git_time_t)st.st_mtime;
p->index_version = -1; p->index_version = -1;
p->oid_type = oid_type ? oid_type : GIT_OID_DEFAULT;
p->oid_size = (unsigned int)git_oid_size(p->oid_type);
p->oid_hexsize = (unsigned int)git_oid_hexsize(p->oid_type);
if (git_mutex_init(&p->lock) < 0) { if (git_mutex_init(&p->lock) < 0) {
git_error_set(GIT_ERROR_OS, "failed to initialize packfile mutex"); git_error_set(GIT_ERROR_OS, "failed to initialize packfile mutex");
...@@ -1260,9 +1269,9 @@ static off64_t nth_packed_object_offset_locked(struct git_pack_file *p, uint32_t ...@@ -1260,9 +1269,9 @@ static off64_t nth_packed_object_offset_locked(struct git_pack_file *p, uint32_t
end = index + p->index_map.len; end = index + p->index_map.len;
index += 4 * 256; index += 4 * 256;
if (p->index_version == 1) if (p->index_version == 1)
return ntohl(*((uint32_t *)(index + 24 * n))); return ntohl(*((uint32_t *)(index + (p->oid_size + 4) * n)));
index += 8 + p->num_objects * (20 + 4); index += 8 + p->num_objects * (p->oid_size + 4);
off32 = ntohl(*((uint32_t *)(index + 4 * n))); off32 = ntohl(*((uint32_t *)(index + 4 * n)));
if (!(off32 & 0x80000000)) if (!(off32 & 0x80000000))
return off32; return off32;
...@@ -1312,7 +1321,7 @@ int git_pack_foreach_entry( ...@@ -1312,7 +1321,7 @@ int git_pack_foreach_entry(
index += 4 * 256; index += 4 * 256;
if (p->oids == NULL) { if (p->ids == NULL) {
git_vector offsets, oids; git_vector offsets, oids;
if ((error = git_vector_init(&oids, p->num_objects, NULL))) { if ((error = git_vector_init(&oids, p->num_objects, NULL))) {
...@@ -1326,22 +1335,25 @@ int git_pack_foreach_entry( ...@@ -1326,22 +1335,25 @@ int git_pack_foreach_entry(
} }
if (p->index_version > 1) { if (p->index_version > 1) {
const unsigned char *off = index + 24 * p->num_objects; const unsigned char *off = index +
(p->oid_size + 4) * p->num_objects;
for (i = 0; i < p->num_objects; i++) for (i = 0; i < p->num_objects; i++)
git_vector_insert(&offsets, (void*)&off[4 * i]); git_vector_insert(&offsets, (void*)&off[4 * i]);
git_vector_sort(&offsets); git_vector_sort(&offsets);
git_vector_foreach(&offsets, i, current) git_vector_foreach(&offsets, i, current)
git_vector_insert(&oids, (void*)&index[5 * (current - off)]); git_vector_insert(&oids, (void*)&index[5 * (current - off)]);
} else { } else {
for (i = 0; i < p->num_objects; i++) for (i = 0; i < p->num_objects; i++)
git_vector_insert(&offsets, (void*)&index[24 * i]); git_vector_insert(&offsets, (void*)&index[(p->oid_size + 4) * i]);
git_vector_sort(&offsets); git_vector_sort(&offsets);
git_vector_foreach(&offsets, i, current) git_vector_foreach(&offsets, i, current)
git_vector_insert(&oids, (void*)&current[4]); git_vector_insert(&oids, (void*)&current[4]);
} }
git_vector_free(&offsets); git_vector_free(&offsets);
p->oids = (unsigned char **)git_vector_detach(NULL, NULL, &oids); p->ids = (unsigned char **)git_vector_detach(NULL, NULL, &oids);
} }
/* /*
...@@ -1362,7 +1374,7 @@ int git_pack_foreach_entry( ...@@ -1362,7 +1374,7 @@ int git_pack_foreach_entry(
git_array_clear(oids); git_array_clear(oids);
GIT_ERROR_CHECK_ALLOC(oid); GIT_ERROR_CHECK_ALLOC(oid);
} }
git_oid__fromraw(oid, p->oids[i], GIT_OID_SHA1); git_oid__fromraw(oid, p->ids[i], p->oid_type);
} }
git_mutex_unlock(&p->lock); git_mutex_unlock(&p->lock);
...@@ -1412,10 +1424,13 @@ int git_pack_foreach_entry_offset( ...@@ -1412,10 +1424,13 @@ int git_pack_foreach_entry_offset(
/* all offsets should have been validated by pack_index_check_locked */ /* all offsets should have been validated by pack_index_check_locked */
if (p->index_version > 1) { if (p->index_version > 1) {
const unsigned char *offsets = index + 24 * p->num_objects; const unsigned char *offsets = index +
(p->oid_size + 4) * p->num_objects;
const unsigned char *large_offset_ptr; const unsigned char *large_offset_ptr;
const unsigned char *large_offsets = index + 28 * p->num_objects; const unsigned char *large_offsets = index +
const unsigned char *large_offsets_end = ((const unsigned char *)p->index_map.data) + p->index_map.len - 20; (p->oid_size + 8) * p->num_objects;
const unsigned char *large_offsets_end = ((const unsigned char *)p->index_map.data) + p->index_map.len - p->oid_size;
for (i = 0; i < p->num_objects; i++) { for (i = 0; i < p->num_objects; i++) {
current_offset = ntohl(*(const uint32_t *)(offsets + 4 * i)); current_offset = ntohl(*(const uint32_t *)(offsets + 4 * i));
if (current_offset & 0x80000000) { if (current_offset & 0x80000000) {
...@@ -1428,7 +1443,7 @@ int git_pack_foreach_entry_offset( ...@@ -1428,7 +1443,7 @@ int git_pack_foreach_entry_offset(
ntohl(*((uint32_t *)(large_offset_ptr + 4))); ntohl(*((uint32_t *)(large_offset_ptr + 4)));
} }
git_oid__fromraw(&current_oid, (index + 20 * i), GIT_OID_SHA1); git_oid__fromraw(&current_oid, (index + p->oid_size * i), p->oid_type);
if ((error = cb(&current_oid, current_offset, data)) != 0) { if ((error = cb(&current_oid, current_offset, data)) != 0) {
error = git_error_set_after_callback(error); error = git_error_set_after_callback(error);
goto cleanup; goto cleanup;
...@@ -1436,8 +1451,8 @@ int git_pack_foreach_entry_offset( ...@@ -1436,8 +1451,8 @@ int git_pack_foreach_entry_offset(
} }
} else { } else {
for (i = 0; i < p->num_objects; i++) { for (i = 0; i < p->num_objects; i++) {
current_offset = ntohl(*(const uint32_t *)(index + 24 * i)); current_offset = ntohl(*(const uint32_t *)(index + (p->oid_size + 4) * i));
git_oid__fromraw(&current_oid, (index + 24 * i + 4), GIT_OID_SHA1); git_oid__fromraw(&current_oid, (index + (p->oid_size + 4) * i + 4), p->oid_type);
if ((error = cb(&current_oid, current_offset, data)) != 0) { if ((error = cb(&current_oid, current_offset, data)) != 0) {
error = git_error_set_after_callback(error); error = git_error_set_after_callback(error);
goto cleanup; goto cleanup;
...@@ -1450,14 +1465,20 @@ cleanup: ...@@ -1450,14 +1465,20 @@ cleanup:
return error; return error;
} }
int git_pack__lookup_sha1(const void *oid_lookup_table, size_t stride, unsigned lo, int git_pack__lookup_id(
unsigned hi, const unsigned char *oid_prefix) const void *oid_lookup_table,
size_t stride,
unsigned lo,
unsigned hi,
const unsigned char *oid_prefix,
const git_oid_t oid_type)
{ {
const unsigned char *base = oid_lookup_table; const unsigned char *base = oid_lookup_table;
size_t oid_size = git_oid_size(oid_type);
while (lo < hi) { while (lo < hi) {
unsigned mi = (lo + hi) / 2; unsigned mi = (lo + hi) / 2;
int cmp = git_oid_raw_cmp(base + mi * stride, oid_prefix, GIT_OID_SHA1_SIZE); int cmp = git_oid_raw_cmp(base + mi * stride, oid_prefix, oid_size);
if (!cmp) if (!cmp)
return mi; return mi;
...@@ -1512,9 +1533,9 @@ static int pack_entry_find_offset( ...@@ -1512,9 +1533,9 @@ static int pack_entry_find_offset(
lo = ((short_oid->id[0] == 0x0) ? 0 : ntohl(level1_ofs[(int)short_oid->id[0] - 1])); lo = ((short_oid->id[0] == 0x0) ? 0 : ntohl(level1_ofs[(int)short_oid->id[0] - 1]));
if (p->index_version > 1) { if (p->index_version > 1) {
stride = 20; stride = p->oid_size;
} else { } else {
stride = 24; stride = p->oid_size + 4;
index += 4; index += 4;
} }
...@@ -1523,7 +1544,8 @@ static int pack_entry_find_offset( ...@@ -1523,7 +1544,8 @@ static int pack_entry_find_offset(
short_oid->id[0], short_oid->id[1], short_oid->id[2], lo, hi, p->num_objects); short_oid->id[0], short_oid->id[1], short_oid->id[2], lo, hi, p->num_objects);
#endif #endif
pos = git_pack__lookup_sha1(index, stride, lo, hi, short_oid->id); pos = git_pack__lookup_id(index, stride, lo, hi,
short_oid->id, p->oid_type);
if (pos >= 0) { if (pos >= 0) {
/* An object matching exactly the oid was found */ /* An object matching exactly the oid was found */
...@@ -1541,7 +1563,9 @@ static int pack_entry_find_offset( ...@@ -1541,7 +1563,9 @@ static int pack_entry_find_offset(
} }
} }
if (found && len != GIT_OID_SHA1_HEXSIZE && pos + 1 < (int)p->num_objects) { if (found &&
len != p->oid_hexsize &&
pos + 1 < (int)p->num_objects) {
/* Check for ambiguousity */ /* Check for ambiguousity */
const unsigned char *next = current + stride; const unsigned char *next = current + stride;
...@@ -1566,13 +1590,13 @@ static int pack_entry_find_offset( ...@@ -1566,13 +1590,13 @@ static int pack_entry_find_offset(
} }
*offset_out = offset; *offset_out = offset;
git_oid__fromraw(found_oid, current, GIT_OID_SHA1); git_oid__fromraw(found_oid, current, p->oid_type);
#ifdef INDEX_DEBUG_LOOKUP #ifdef INDEX_DEBUG_LOOKUP
{ {
unsigned char hex_sha1[GIT_OID_SHA1_HEXSIZE + 1]; char hex_sha1[p->oid_hexsize + 1];
git_oid_fmt(hex_sha1, found_oid); git_oid_fmt(hex_sha1, found_oid);
hex_sha1[GIT_OID_SHA1_HEXSIZE] = '\0'; hex_sha1[p->oid_hexsize] = '\0';
printf("found lo=%d %s\n", lo, hex_sha1); printf("found lo=%d %s\n", lo, hex_sha1);
} }
#endif #endif
...@@ -1594,10 +1618,10 @@ int git_pack_entry_find( ...@@ -1594,10 +1618,10 @@ int git_pack_entry_find(
GIT_ASSERT_ARG(p); GIT_ASSERT_ARG(p);
if (len == GIT_OID_SHA1_HEXSIZE && p->num_bad_objects) { if (len == p->oid_hexsize && p->num_bad_objects) {
unsigned i; unsigned i;
for (i = 0; i < p->num_bad_objects; i++) for (i = 0; i < p->num_bad_objects; i++)
if (git_oid__cmp(short_oid, &p->bad_object_sha1[i]) == 0) if (git_oid__cmp(short_oid, &p->bad_object_ids[i]) == 0)
return packfile_error("bad object found in packfile"); return packfile_error("bad object found in packfile");
} }
...@@ -1630,6 +1654,6 @@ int git_pack_entry_find( ...@@ -1630,6 +1654,6 @@ int git_pack_entry_find(
e->offset = offset; e->offset = offset;
e->p = p; e->p = p;
git_oid_cpy(&e->sha1, &found_oid); git_oid_cpy(&e->id, &found_oid);
return 0; return 0;
} }
...@@ -99,13 +99,19 @@ struct git_pack_file { ...@@ -99,13 +99,19 @@ struct git_pack_file {
uint32_t num_objects; uint32_t num_objects;
uint32_t num_bad_objects; uint32_t num_bad_objects;
git_oid *bad_object_sha1; /* array of git_oid */ git_oid *bad_object_ids; /* array of git_oid */
git_oid_t oid_type;
unsigned oid_hexsize:7,
oid_size:6,
pack_local:1,
pack_keep:1,
has_cache:1;
int index_version; int index_version;
git_time_t mtime; git_time_t mtime;
unsigned pack_local:1, pack_keep:1, has_cache:1;
git_oidmap *idx_cache; git_oidmap *idx_cache;
unsigned char **oids; unsigned char **ids;
git_pack_cache bases; /* delta base cache */ git_pack_cache bases; /* delta base cache */
...@@ -116,21 +122,26 @@ struct git_pack_file { ...@@ -116,21 +122,26 @@ struct git_pack_file {
}; };
/** /**
* Return the position where an OID (or a prefix) would be inserted within the * Return the position where an OID (or a prefix) would be inserted within
* OID Lookup Table of an .idx file. This performs binary search between the lo * the OID Lookup Table of an .idx file. This performs binary search
* and hi indices. * between the lo and hi indices.
* *
* The stride parameter is provided because .idx files version 1 store the OIDs * The stride parameter is provided because .idx files version 1 store the
* interleaved with the 4-byte file offsets of the objects within the .pack * OIDs interleaved with the 4-byte file offsets of the objects within the
* file (stride = 24), whereas files with version 2 store them in a contiguous * .pack file (stride = oid_size + 4), whereas files with version 2 store
* flat array (stride = 20). * them in a contiguous flat array (stride = oid_size).
*/ */
int git_pack__lookup_sha1(const void *oid_lookup_table, size_t stride, unsigned lo, int git_pack__lookup_id(
unsigned hi, const unsigned char *oid_prefix); const void *id_lookup_table,
size_t stride,
unsigned lo,
unsigned hi,
const unsigned char *id_prefix,
const git_oid_t oid_type);
struct git_pack_entry { struct git_pack_entry {
off64_t offset; off64_t offset;
git_oid sha1; git_oid id;
struct git_pack_file *p; struct git_pack_file *p;
}; };
...@@ -174,12 +185,15 @@ int get_delta_base( ...@@ -174,12 +185,15 @@ int get_delta_base(
off64_t delta_obj_offset); off64_t delta_obj_offset);
void git_packfile_free(struct git_pack_file *p, bool unlink_packfile); void git_packfile_free(struct git_pack_file *p, bool unlink_packfile);
int git_packfile_alloc(struct git_pack_file **pack_out, const char *path); int git_packfile_alloc(
struct git_pack_file **pack_out,
const char *path,
git_oid_t oid_type);
int git_pack_entry_find( int git_pack_entry_find(
struct git_pack_entry *e, struct git_pack_entry *e,
struct git_pack_file *p, struct git_pack_file *p,
const git_oid *short_oid, const git_oid *short_id,
size_t len); size_t len);
int git_pack_foreach_entry( int git_pack_foreach_entry(
struct git_pack_file *p, struct git_pack_file *p,
......
...@@ -52,7 +52,16 @@ void test_odb_foreach__one_pack(void) ...@@ -52,7 +52,16 @@ void test_odb_foreach__one_pack(void)
int nobj = 0; int nobj = 0;
cl_git_pass(git_odb__new(&_odb, NULL)); cl_git_pass(git_odb__new(&_odb, NULL));
cl_git_pass(git_odb_backend_one_pack(&backend, cl_fixture("testrepo.git/objects/pack/pack-a81e489679b7d3418f9ab594bda8ceb37dd4c695.idx")));
#ifdef GIT_EXPERIMENTAL_SHA256
cl_git_pass(git_odb_backend_one_pack(&backend,
cl_fixture("testrepo.git/objects/pack/pack-a81e489679b7d3418f9ab594bda8ceb37dd4c695.idx"),
NULL));
#else
cl_git_pass(git_odb_backend_one_pack(&backend,
cl_fixture("testrepo.git/objects/pack/pack-a81e489679b7d3418f9ab594bda8ceb37dd4c695.idx")));
#endif
cl_git_pass(git_odb_add_backend(_odb, backend, 1)); cl_git_pass(git_odb_add_backend(_odb, backend, 1));
_repo = NULL; _repo = NULL;
......
/* Just a few to make sure it's working, the rest is tested already */
#ifdef GIT_EXPERIMENTAL_SHA256
static const char *packed_objects_one256[] = {
"ea926306b1bab6d3f25f45609907eb6dff91a1460b25e63bf4a0494c70e7a269",
"d048ba2ef4fafa502a44cbc1a50cd58359b9bc243b84a08f541a08ca5f621137",
"a66bda0109d2b3c9bc87970da81bd91076b5f871febbc860f09ae997668b6800",
"3609a41c0506fe19d01fb8b4729923362675f191fe5f63fab3111ef804c48fdf",
"22b6705b86e4aa9120eff203af24709d483698d9f78695e86e82de121784b570",
"6f11d93bfb269ee8c7a506178f60c430abfac5d424acfd9c0b0b27b98e6ab49b",
"0aefd477d9e5b3f8d708a3cf6d78be6b670dfa2e2ec41244634f3b8f115d8e04",
"580474d948cd2ebd2e5ce7a5b81b872d87ba4639c1ac4c0fa7a11a8eddf9827c",
"0636b4292bfdd7274a977cb6f8b2ded8f315ea1bcd8dbedfca37964c2ed3d085",
"19fb1c78b11f0f8bda658d6fa6cc63c0b573c0f6760ee5a9c2df6ce2cde00c5c",
"7f2f7afccb317bb3fdd28555f126846dc4eebe5d9ae7b8d8a1456e8ff85422ce",
"4066249c68b0d3c8b39ebe02c9188935900465acad02a49269710f56720fa58e",
"a560d1fa1edf114f57b402e16d662c17b1e3b7e8601ff7dcea6615ba7f1e48ef",
"58923faa87c7d559d308a114ec2b164e5d6046c889420ed1def6eef2d55106a2",
"753ddabab8ae9c1e733cda15e8e3c83dd43f5a2d939ae32cc3b30b0be1e91f96",
"46333d32b3801cf11d9f80b557245c9e32b0e05deca61dae968914fde159f0e5"
};
#endif
...@@ -11,7 +11,14 @@ void test_odb_packedone__initialize(void) ...@@ -11,7 +11,14 @@ void test_odb_packedone__initialize(void)
git_odb_backend *backend = NULL; git_odb_backend *backend = NULL;
cl_git_pass(git_odb__new(&_odb, NULL)); cl_git_pass(git_odb__new(&_odb, NULL));
cl_git_pass(git_odb_backend_one_pack(&backend, cl_fixture("testrepo.git/objects/pack/pack-a81e489679b7d3418f9ab594bda8ceb37dd4c695.idx"))); #ifdef GIT_EXPERIMENTAL_SHA256
cl_git_pass(git_odb_backend_one_pack(&backend,
cl_fixture("testrepo.git/objects/pack/pack-a81e489679b7d3418f9ab594bda8ceb37dd4c695.idx"),
NULL));
#else
cl_git_pass(git_odb_backend_one_pack(&backend,
cl_fixture("testrepo.git/objects/pack/pack-a81e489679b7d3418f9ab594bda8ceb37dd4c695.idx")));
#endif
cl_git_pass(git_odb_add_backend(_odb, backend, 1)); cl_git_pass(git_odb_add_backend(_odb, backend, 1));
} }
......
#include "clar_libgit2.h"
#include "git2/odb_backend.h"
#include "pack_data_one256.h"
#include "pack.h"
#ifdef GIT_EXPERIMENTAL_SHA256
static git_odb *_odb;
#endif
void test_odb_packedone256__initialize(void)
{
#ifdef GIT_EXPERIMENTAL_SHA256
git_odb_backend *backend = NULL;
git_odb_options odb_opts = GIT_ODB_OPTIONS_INIT;
git_odb_backend_pack_options backend_opts = GIT_ODB_BACKEND_PACK_OPTIONS_INIT;
odb_opts.oid_type = GIT_OID_SHA256;
backend_opts.oid_type = GIT_OID_SHA256;
cl_git_pass(git_odb__new(&_odb, &odb_opts));
cl_git_pass(git_odb_backend_one_pack(
&backend,
cl_fixture("testrepo_256.git/objects/pack/pack-e2f07f30db7e480ea84a0e64ee791b9b270067124b2609019b74f33f256f33fa.idx"),
&backend_opts));
cl_git_pass(git_odb_add_backend(_odb, backend, 1));
#endif
}
void test_odb_packedone256__cleanup(void)
{
#ifdef GIT_EXPERIMENTAL_SHA256
git_odb_free(_odb);
_odb = NULL;
#endif
}
void test_odb_packedone256__mass_read(void)
{
#ifdef GIT_EXPERIMENTAL_SHA256
unsigned int i;
for (i = 0; i < ARRAY_SIZE(packed_objects_one256); ++i) {
git_oid id;
git_odb_object *obj;
cl_git_pass(git_oid__fromstr(&id, packed_objects_one256[i], GIT_OID_SHA256));
cl_assert(git_odb_exists(_odb, &id) == 1);
cl_git_pass(git_odb_read(&obj, _odb, &id));
git_odb_object_free(obj);
}
#endif
}
void test_odb_packedone256__read_header_0(void)
{
#ifdef GIT_EXPERIMENTAL_SHA256
unsigned int i;
for (i = 0; i < ARRAY_SIZE(packed_objects_one256); ++i) {
git_oid id;
git_odb_object *obj;
size_t len;
git_object_t type;
cl_git_pass(git_oid__fromstr(&id, packed_objects_one256[i], GIT_OID_SHA256));
cl_git_pass(git_odb_read(&obj, _odb, &id));
cl_git_pass(git_odb_read_header(&len, &type, _odb, &id));
cl_assert(obj->cached.size == len);
cl_assert(obj->cached.type == type);
git_odb_object_free(obj);
}
#endif
}
...@@ -82,7 +82,11 @@ void test_odb_sorting__override_default_backend_priority(void) ...@@ -82,7 +82,11 @@ void test_odb_sorting__override_default_backend_priority(void)
cl_git_pass(git_libgit2_opts(GIT_OPT_SET_ODB_LOOSE_PRIORITY, 5)); cl_git_pass(git_libgit2_opts(GIT_OPT_SET_ODB_LOOSE_PRIORITY, 5));
cl_git_pass(git_libgit2_opts(GIT_OPT_SET_ODB_PACKED_PRIORITY, 3)); cl_git_pass(git_libgit2_opts(GIT_OPT_SET_ODB_PACKED_PRIORITY, 3));
git_odb_backend_pack(&packed, "./testrepo.git/objects"); git_odb_backend_pack(&packed, "./testrepo.git/objects"
#ifdef GIT_EXPERIMENTAL_SHA256
, NULL
#endif
);
git_odb__backend_loose(&loose, "./testrepo.git/objects", NULL); git_odb__backend_loose(&loose, "./testrepo.git/objects", NULL);
cl_git_pass(git_odb__open(&new_odb, cl_fixture("testrepo.git/objects"), NULL)); cl_git_pass(git_odb__open(&new_odb, cl_fixture("testrepo.git/objects"), NULL));
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment