Skip to content

G
git2
Commit be41c384 by Patrick Steinhardt
Options

indexer: extract function reading stream objects 

The loop inside of `git_indexer_append` iterates over every object that
is to be stored as part of the index. While the logic to retrieve every
object from the packfile stream is rather involved, it currently just
part of the loop, making it unnecessarily hard to follow.

Move the logic into its own function `read_stream_object`, which unpacks
a single object from the stream. Note that there is some subtletly here
involving the special error `GIT_EBUFS`, which indicates to the indexer
that no more data is currently available. So instead of returning an
error and aborting the whole loop in that case, we do have to catch that
value and return successfully to wait for more data to be read.
parent 6568f374
Showing with 86 additions and 73 deletions
src/indexer.c
......@@ -525,86 +525,26 @@ static int append_to_pack(git_indexer *idx, const void *data, size_t size)
return write_at(idx, data, idx->pack->mwf.size, size);
}
int git_indexer_append(git_indexer *idx, const void *data, size_t size, git_transfer_progress *stats)
static int read_stream_object(git_indexer *idx, git_transfer_progress *stats)
{
int error = -1;
struct git_pack_header *hdr = &idx->hdr;
git_mwindow_file *mwf = &idx->pack->mwf;
assert(idx && data && stats);
if ((error = append_to_pack(idx, data, size)) < 0)
return error;
hash_partially(idx, data, (int)size);
/* Make sure we set the new size of the pack */
idx->pack->mwf.size += size;
if (!idx->parsed_header) {
unsigned int total_objects;
if ((unsigned)idx->pack->mwf.size < sizeof(struct git_pack_header))
return 0;
if ((error = parse_header(&idx->hdr, idx->pack)) < 0)
return error;
idx->parsed_header = 1;
idx->nr_objects = ntohl(hdr->hdr_entries);
idx->off = sizeof(struct git_pack_header);
/* for now, limit to 2^32 objects */
assert(idx->nr_objects == (size_t)((unsigned int)idx->nr_objects));
if (idx->nr_objects == (size_t)((unsigned int)idx->nr_objects))
total_objects = (unsigned int)idx->nr_objects;
else
total_objects = UINT_MAX;
idx->pack->idx_cache = git_oidmap_alloc();
GITERR_CHECK_ALLOC(idx->pack->idx_cache);
idx->pack->has_cache = 1;
if (git_vector_init(&idx->objects, total_objects, objects_cmp) < 0)
return -1;
if (git_vector_init(&idx->deltas, total_objects / 2, NULL) < 0)
return -1;
stats->received_objects = 0;
stats->local_objects = 0;
stats->total_deltas = 0;
stats->indexed_deltas = 0;
stats->indexed_objects = 0;
stats->total_objects = total_objects;
if ((error = do_progress_callback(idx, stats)) != 0)
return error;
}
/* Now that we have data in the pack, let's try to parse it */
/* As the file grows any windows we try to use will be out of date */
git_mwindow_free_all(mwf);
while (stats->indexed_objects < idx->nr_objects) {
git_packfile_stream *stream = &idx->stream;
git_off_t entry_start = idx->off;
size_t entry_size;
git_otype type;
git_mwindow *w = NULL;
int error;
if (idx->pack->mwf.size <= idx->off + 20)
return 0;
return GIT_EBUFS;
if (!idx->have_stream) {
error = git_packfile_unpack_header(&entry_size, &type, mwf, &w, &idx->off);
error = git_packfile_unpack_header(&entry_size, &type, &idx->pack->mwf, &w, &idx->off);
if (error == GIT_EBUFS) {
idx->off = entry_start;
return 0;
return error;
}
if (error < 0)
goto on_error;
return error;
git_mwindow_close(&w);
idx->entry_start = entry_start;
......@@ -614,10 +554,10 @@ int git_indexer_append(git_indexer *idx, const void *data, size_t size, git_tran
error = advance_delta_offset(idx, type);
if (error == GIT_EBUFS) {
idx->off = entry_start;
return 0;
return error;
}
if (error < 0)
goto on_error;
return error;
idx->have_delta = 1;
} else {
......@@ -625,14 +565,14 @@ int git_indexer_append(git_indexer *idx, const void *data, size_t size, git_tran
error = hash_header(&idx->hash_ctx, entry_size, type);
if (error < 0)
goto on_error;
return error;
}
idx->have_stream = 1;
error = git_packfile_stream_open(stream, idx->pack, idx->off);
if (error < 0)
goto on_error;
return error;
}
if (idx->have_delta) {
......@@ -643,14 +583,14 @@ int git_indexer_append(git_indexer *idx, const void *data, size_t size, git_tran
idx->off = stream->curpos;
if (error == GIT_EBUFS)
return 0;
return error;
/* We want to free the stream reasorces no matter what here */
idx->have_stream = 0;
git_packfile_stream_dispose(stream);
if (error < 0)
goto on_error;
return error;
if (idx->have_delta) {
error = store_delta(idx);
......@@ -659,7 +599,7 @@ int git_indexer_append(git_indexer *idx, const void *data, size_t size, git_tran
}
if (error < 0)
goto on_error;
return error;
if (!idx->have_delta) {
stats->indexed_objects++;
......@@ -667,8 +607,81 @@ int git_indexer_append(git_indexer *idx, const void *data, size_t size, git_tran
stats->received_objects++;
if ((error = do_progress_callback(idx, stats)) != 0)
return error;
return 0;
}
int git_indexer_append(git_indexer *idx, const void *data, size_t size, git_transfer_progress *stats)
{
int error = -1;
struct git_pack_header *hdr = &idx->hdr;
git_mwindow_file *mwf = &idx->pack->mwf;
assert(idx && data && stats);
if ((error = append_to_pack(idx, data, size)) < 0)
return error;
hash_partially(idx, data, (int)size);
/* Make sure we set the new size of the pack */
idx->pack->mwf.size += size;
if (!idx->parsed_header) {
unsigned int total_objects;
if ((unsigned)idx->pack->mwf.size < sizeof(struct git_pack_header))
return 0;
if ((error = parse_header(&idx->hdr, idx->pack)) < 0)
return error;
idx->parsed_header = 1;
idx->nr_objects = ntohl(hdr->hdr_entries);
idx->off = sizeof(struct git_pack_header);
/* for now, limit to 2^32 objects */
assert(idx->nr_objects == (size_t)((unsigned int)idx->nr_objects));
if (idx->nr_objects == (size_t)((unsigned int)idx->nr_objects))
total_objects = (unsigned int)idx->nr_objects;
else
total_objects = UINT_MAX;
idx->pack->idx_cache = git_oidmap_alloc();
GITERR_CHECK_ALLOC(idx->pack->idx_cache);
idx->pack->has_cache = 1;
if (git_vector_init(&idx->objects, total_objects, objects_cmp) < 0)
return -1;
if (git_vector_init(&idx->deltas, total_objects / 2, NULL) < 0)
return -1;
stats->received_objects = 0;
stats->local_objects = 0;
stats->total_deltas = 0;
stats->indexed_deltas = 0;
stats->indexed_objects = 0;
stats->total_objects = total_objects;
if ((error = do_progress_callback(idx, stats)) != 0)
return error;
}
/* Now that we have data in the pack, let's try to parse it */
/* As the file grows any windows we try to use will be out of date */
git_mwindow_free_all(mwf);
while (stats->indexed_objects < idx->nr_objects) {
if ((error = read_stream_object(idx, stats)) != 0) {
if (error == GIT_EBUFS)
break;
else
goto on_error;
}
}
return 0;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment