Skip to content

G
git2
Commit 29ab8774 by Vicent Martí
Options

Merge pull request #1387 from carlosmn/kill-indexer 

indexer: kill git_indexer
parents bb19532c 447ae791
Showing with 19 additions and 383 deletions
include/git2/indexer.h
......@@ -33,7 +33,6 @@ typedef struct git_transfer_progress {
*/
typedef int (*git_transfer_progress_callback)(const git_transfer_progress *stats, void *payload);
typedef struct git_indexer git_indexer;
typedef struct git_indexer_stream git_indexer_stream;
/**
......@@ -86,53 +85,6 @@ GIT_EXTERN(const git_oid *) git_indexer_stream_hash(const git_indexer_stream *id
*/
GIT_EXTERN(void) git_indexer_stream_free(git_indexer_stream *idx);
/**
* Create a new indexer instance
*
* @param out where to store the indexer instance
* @param packname the absolute filename of the packfile to index
*/
GIT_EXTERN(int) git_indexer_new(git_indexer **out, const char *packname);
/**
* Iterate over the objects in the packfile and extract the information
*
* Indexing a packfile can be very expensive so this function is
* expected to be run in a worker thread and the stats used to provide
* feedback the user.
*
* @param idx the indexer instance
* @param stats storage for the running state
*/
GIT_EXTERN(int) git_indexer_run(git_indexer *idx, git_transfer_progress *stats);
/**
* Write the index file to disk.
*
* The file will be stored as pack-$hash.idx in the same directory as
* the packfile.
*
* @param idx the indexer instance
*/
GIT_EXTERN(int) git_indexer_write(git_indexer *idx);
/**
* Get the packfile's hash
*
* A packfile's name is derived from the sorted hashing of all object
* names. This is only correct after the index has been written to disk.
*
* @param idx the indexer instance
*/
GIT_EXTERN(const git_oid *) git_indexer_hash(const git_indexer *idx);
/**
* Free the indexer and its resources
*
* @param idx the indexer to free
*/
GIT_EXTERN(void) git_indexer_free(git_indexer *idx);
GIT_END_DECL
#endif
src/indexer.c
......@@ -27,15 +27,6 @@ struct entry {
uint64_t offset_long;
};
struct git_indexer {
struct git_pack_file *pack;
size_t nr_objects;
git_vector objects;
git_filebuf file;
unsigned int fanout[256];
git_oid hash;
};
struct git_indexer_stream {
unsigned int parsed_header :1,
opened_pack :1,
......@@ -61,11 +52,6 @@ struct delta_info {
git_off_t delta_off;
};
const git_oid *git_indexer_hash(const git_indexer *idx)
{
return &idx->hash;
}
const git_oid *git_indexer_stream_hash(const git_indexer_stream *idx)
{
return &idx->hash;
......@@ -451,7 +437,7 @@ int git_indexer_stream_add(git_indexer_stream *idx, const void *data, size_t siz
return -1;
stats->received_objects = 0;
stats->indexed_objects = 0;
processed = stats->indexed_objects = 0;
stats->total_objects = (unsigned int)idx->nr_objects;
do_progress_callback(idx, stats);
}
......@@ -755,315 +741,3 @@ void git_indexer_stream_free(git_indexer_stream *idx)
git_filebuf_cleanup(&idx->pack_file);
git__free(idx);
}
int git_indexer_new(git_indexer **out, const char *packname)
{
git_indexer *idx;
struct git_pack_header hdr;
int error;
assert(out && packname);
idx = git__calloc(1, sizeof(git_indexer));
GITERR_CHECK_ALLOC(idx);
open_pack(&idx->pack, packname);
if ((error = parse_header(&hdr, idx->pack)) < 0)
goto cleanup;
idx->nr_objects = ntohl(hdr.hdr_entries);
/* for now, limit to 2^32 objects */
assert(idx->nr_objects == (size_t)((unsigned int)idx->nr_objects));
error = git_vector_init(&idx->pack->cache, (unsigned int)idx->nr_objects, cache_cmp);
if (error < 0)
goto cleanup;
idx->pack->has_cache = 1;
error = git_vector_init(&idx->objects, (unsigned int)idx->nr_objects, objects_cmp);
if (error < 0)
goto cleanup;
*out = idx;
return 0;
cleanup:
git_indexer_free(idx);
return -1;
}
static int index_path(git_buf *path, git_indexer *idx)
{
const char prefix[] = "pack-", suffix[] = ".idx";
size_t slash = (size_t)path->size;
/* search backwards for '/' */
while (slash > 0 && path->ptr[slash - 1] != '/')
slash--;
if (git_buf_grow(path, slash + 1 + strlen(prefix) +
GIT_OID_HEXSZ + strlen(suffix) + 1) < 0)
return -1;
git_buf_truncate(path, slash);
git_buf_puts(path, prefix);
git_oid_fmt(path->ptr + git_buf_len(path), &idx->hash);
path->size += GIT_OID_HEXSZ;
git_buf_puts(path, suffix);
return git_buf_oom(path) ? -1 : 0;
}
int git_indexer_write(git_indexer *idx)
{
git_mwindow *w = NULL;
int error;
unsigned int i, long_offsets = 0, left;
struct git_pack_idx_header hdr;
git_buf filename = GIT_BUF_INIT;
struct entry *entry;
void *packfile_hash;
git_oid file_hash;
git_hash_ctx ctx;
if (git_hash_ctx_init(&ctx) < 0)
return -1;
git_vector_sort(&idx->objects);
git_buf_sets(&filename, idx->pack->pack_name);
git_buf_truncate(&filename, filename.size - strlen("pack"));
git_buf_puts(&filename, "idx");
if (git_buf_oom(&filename))
return -1;
error = git_filebuf_open(&idx->file, filename.ptr, GIT_FILEBUF_HASH_CONTENTS);
if (error < 0)
goto cleanup;
/* Write out the header */
hdr.idx_signature = htonl(PACK_IDX_SIGNATURE);
hdr.idx_version = htonl(2);
error = git_filebuf_write(&idx->file, &hdr, sizeof(hdr));
if (error < 0)
goto cleanup;
/* Write out the fanout table */
for (i = 0; i < 256; ++i) {
uint32_t n = htonl(idx->fanout[i]);
error = git_filebuf_write(&idx->file, &n, sizeof(n));
if (error < 0)
goto cleanup;
}
/* Write out the object names (SHA-1 hashes) */
git_vector_foreach(&idx->objects, i, entry) {
if ((error = git_filebuf_write(&idx->file, &entry->oid, sizeof(git_oid))) < 0 ||
(error = git_hash_update(&ctx, &entry->oid, GIT_OID_RAWSZ)) < 0)
goto cleanup;
}
if ((error = git_hash_final(&idx->hash, &ctx)) < 0)
goto cleanup;
/* Write out the CRC32 values */
git_vector_foreach(&idx->objects, i, entry) {
error = git_filebuf_write(&idx->file, &entry->crc, sizeof(uint32_t));
if (error < 0)
goto cleanup;
}
/* Write out the offsets */
git_vector_foreach(&idx->objects, i, entry) {
uint32_t n;
if (entry->offset == UINT32_MAX)
n = htonl(0x80000000 | long_offsets++);
else
n = htonl(entry->offset);
error = git_filebuf_write(&idx->file, &n, sizeof(uint32_t));
if (error < 0)
goto cleanup;
}
/* Write out the long offsets */
git_vector_foreach(&idx->objects, i, entry) {
uint32_t split[2];
if (entry->offset != UINT32_MAX)
continue;
split[0] = htonl(entry->offset_long >> 32);
split[1] = htonl(entry->offset_long & 0xffffffff);
error = git_filebuf_write(&idx->file, &split, sizeof(uint32_t) * 2);
if (error < 0)
goto cleanup;
}
/* Write out the packfile trailer */
packfile_hash = git_mwindow_open(&idx->pack->mwf, &w, idx->pack->mwf.size - GIT_OID_RAWSZ, GIT_OID_RAWSZ, &left);
git_mwindow_close(&w);
if (packfile_hash == NULL) {
error = -1;
goto cleanup;
}
memcpy(&file_hash, packfile_hash, GIT_OID_RAWSZ);
git_mwindow_close(&w);
error = git_filebuf_write(&idx->file, &file_hash, sizeof(git_oid));
if (error < 0)
goto cleanup;
/* Write out the index sha */
error = git_filebuf_hash(&file_hash, &idx->file);
if (error < 0)
goto cleanup;
error = git_filebuf_write(&idx->file, &file_hash, sizeof(git_oid));
if (error < 0)
goto cleanup;
/* Figure out what the final name should be */
error = index_path(&filename, idx);
if (error < 0)
goto cleanup;
/* Commit file */
error = git_filebuf_commit_at(&idx->file, filename.ptr, GIT_PACK_FILE_MODE);
cleanup:
git_mwindow_free_all(&idx->pack->mwf);
git_mwindow_file_deregister(&idx->pack->mwf);
if (error < 0)
git_filebuf_cleanup(&idx->file);
git_buf_free(&filename);
git_hash_ctx_cleanup(&ctx);
return error;
}
int git_indexer_run(git_indexer *idx, git_transfer_progress *stats)
{
git_mwindow_file *mwf;
git_off_t off = sizeof(struct git_pack_header);
int error;
struct entry *entry;
unsigned int left, processed;
assert(idx && stats);
mwf = &idx->pack->mwf;
error = git_mwindow_file_register(mwf);
if (error < 0)
return error;
stats->total_objects = (unsigned int)idx->nr_objects;
stats->indexed_objects = processed = 0;
while (processed < idx->nr_objects) {
git_rawobj obj;
git_oid oid;
struct git_pack_entry *pentry;
git_mwindow *w = NULL;
int i;
git_off_t entry_start = off;
void *packed;
size_t entry_size;
char fmt[GIT_OID_HEXSZ] = {0};
entry = git__calloc(1, sizeof(*entry));
GITERR_CHECK_ALLOC(entry);
if (off > UINT31_MAX) {
entry->offset = UINT32_MAX;
entry->offset_long = off;
} else {
entry->offset = (uint32_t)off;
}
error = git_packfile_unpack(&obj, idx->pack, &off);
if (error < 0)
goto cleanup;
/* FIXME: Parse the object instead of hashing it */
error = git_odb__hashobj(&oid, &obj);
if (error < 0) {
giterr_set(GITERR_INDEXER, "Failed to hash object");
goto cleanup;
}
pentry = git__malloc(sizeof(struct git_pack_entry));
if (pentry == NULL) {
error = -1;
goto cleanup;
}
git_oid_cpy(&pentry->sha1, &oid);
pentry->offset = entry_start;
git_oid_fmt(fmt, &oid);
error = git_vector_insert(&idx->pack->cache, pentry);
if (error < 0)
goto cleanup;
git_oid_cpy(&entry->oid, &oid);
entry->crc = crc32(0L, Z_NULL, 0);
entry_size = (size_t)(off - entry_start);
packed = git_mwindow_open(mwf, &w, entry_start, entry_size, &left);
if (packed == NULL) {
error = -1;
goto cleanup;
}
entry->crc = htonl(crc32(entry->crc, packed, (uInt)entry_size));
git_mwindow_close(&w);
/* Add the object to the list */
error = git_vector_insert(&idx->objects, entry);
if (error < 0)
goto cleanup;
for (i = oid.id[0]; i < 256; ++i) {
idx->fanout[i]++;
}
git__free(obj.data);
stats->indexed_objects = ++processed;
}
cleanup:
git_mwindow_free_all(mwf);
return error;
}
void git_indexer_free(git_indexer *idx)
{
unsigned int i;
struct entry *e;
struct git_pack_entry *pe;
if (idx == NULL)
return;
git_mwindow_file_deregister(&idx->pack->mwf);
git_vector_foreach(&idx->objects, i, e)
git__free(e);
git_vector_free(&idx->objects);
git_vector_foreach(&idx->pack->cache, i, pe)
git__free(pe);
git_vector_free(&idx->pack->cache);
git_packfile_free(idx->pack);
git__free(idx);
}
tests-clar/pack/packbuilder.c
......@@ -8,7 +8,7 @@
static git_repository *_repo;
static git_revwalk *_revwalker;
static git_packbuilder *_packbuilder;
static git_indexer *_indexer;
static git_indexer_stream *_indexer;
static git_vector _commits;
static int _commits_is_initialized;
......@@ -40,7 +40,7 @@ void test_pack_packbuilder__cleanup(void)
git_revwalk_free(_revwalker);
_revwalker = NULL;
git_indexer_free(_indexer);
git_indexer_stream_free(_indexer);
_indexer = NULL;
cl_git_sandbox_cleanup();
......@@ -75,20 +75,29 @@ static void seed_packbuilder(void)
}
}
static int feed_indexer(void *ptr, size_t len, void *payload)
{
git_transfer_progress *stats = (git_transfer_progress *)payload;
return git_indexer_stream_add(_indexer, ptr, len, stats);
}
void test_pack_packbuilder__create_pack(void)
{
git_transfer_progress stats;
git_buf buf = GIT_BUF_INIT;
git_buf buf = GIT_BUF_INIT, path = GIT_BUF_INIT;
git_hash_ctx ctx;
git_oid hash;
char hex[41]; hex[40] = '\0';
seed_packbuilder();
cl_git_pass(git_packbuilder_write(_packbuilder, "testpack.pack"));
cl_git_pass(git_indexer_new(&_indexer, "testpack.pack"));
cl_git_pass(git_indexer_run(_indexer, &stats));
cl_git_pass(git_indexer_write(_indexer));
cl_git_pass(git_indexer_stream_new(&_indexer, ".", NULL, NULL));
cl_git_pass(git_packbuilder_foreach(_packbuilder, feed_indexer, &stats));
cl_git_pass(git_indexer_stream_finalize(_indexer, &stats));
git_oid_fmt(hex, git_indexer_stream_hash(_indexer));
git_buf_printf(&path, "pack-%s.pack", hex);
/*
* By default, packfiles are created with only one thread.
......@@ -104,13 +113,14 @@ void test_pack_packbuilder__create_pack(void)
*
*/
cl_git_pass(git_futils_readbuffer(&buf, "testpack.pack"));
cl_git_pass(git_futils_readbuffer(&buf, git_buf_cstr(&path)));
cl_git_pass(git_hash_ctx_init(&ctx));
cl_git_pass(git_hash_update(&ctx, buf.ptr, buf.size));
cl_git_pass(git_hash_final(&hash, &ctx));
git_hash_ctx_cleanup(&ctx);
git_buf_free(&path);
git_buf_free(&buf);
git_oid_fmt(hex, &hash);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment