Skip to content

G
git2
Commit 7d0cdf82 by Carlos Martín Nieto
Options

Make packfile_unpack_header more generic 

On the way, store the fd and the size in the mwindow file.

Signed-off-by: Carlos Martín Nieto <carlos@cmartin.tk>
parent ab525a74
Showing with 360 additions and 374 deletions
src/indexer.c
...@@ -24,6 +24,7 @@ ...@@ -24,6 +24,7 @@
*/ */
#include "git2/indexer.h" #include "git2/indexer.h"
#include "git2/object.h"
#include "git2/zlib.h" #include "git2/zlib.h"
#include "common.h" #include "common.h"
...@@ -45,7 +46,7 @@ static int parse_header(git_indexer *idx) ...@@ -45,7 +46,7 @@ static int parse_header(git_indexer *idx)
int error; int error;
/* Verify we recognize this pack file format. */ /* Verify we recognize this pack file format. */
if ((error = p_read(idx->pack->pack_fd, &hdr, sizeof(hdr))) < GIT_SUCCESS) if ((error = p_read(idx->pack->mwf.fd, &hdr, sizeof(hdr))) < GIT_SUCCESS)
goto cleanup; goto cleanup;
if (hdr.hdr_signature != htonl(PACK_SIGNATURE)) { if (hdr.hdr_signature != htonl(PACK_SIGNATURE)) {
...@@ -118,7 +119,7 @@ int git_indexer_new(git_indexer **out, const char *packname) ...@@ -118,7 +119,7 @@ int git_indexer_new(git_indexer **out, const char *packname)
goto cleanup; goto cleanup;
} }
idx->pack->pack_fd = ret; idx->pack->mwf.fd = ret;
error = parse_header(idx); error = parse_header(idx);
if (error < GIT_SUCCESS) { if (error < GIT_SUCCESS) {
...@@ -177,7 +178,7 @@ static git_otype entry_type(const char *buf) ...@@ -177,7 +178,7 @@ static git_otype entry_type(const char *buf)
* (something has been parse or resolved), the callback gets called * (something has been parse or resolved), the callback gets called
* with some stats so it can tell the user how hard we're working * with some stats so it can tell the user how hard we're working
*/ */
int git_indexer_run(git_indexer *idx, int (*cb)(const git_indexer_stats *, void *), void *data) int git_indexer_run(git_indexer *idx, int (*cb)(const git_indexer_stats *, void *), void *cb_data)
{ {
git_mwindow_file *mwf = &idx->pack->mwf; git_mwindow_file *mwf = &idx->pack->mwf;
git_mwindow *w = NULL; git_mwindow *w = NULL;
...@@ -192,25 +193,13 @@ int git_indexer_run(git_indexer *idx, int (*cb)(const git_indexer_stats *, void ...@@ -192,25 +193,13 @@ int git_indexer_run(git_indexer *idx, int (*cb)(const git_indexer_stats *, void
/* Notify before the first one */ /* Notify before the first one */
if (cb) if (cb)
cb(&idx->stats, data); cb(&idx->stats, cb_data);
while (idx->stats.processed < idx->stats.total) { while (idx->stats.processed < idx->stats.total) {
unsigned long size; size_t size;
git_otype type; git_otype type;
/* 4k is a bit magic for the moment */ error = git_packfile_unpack_header(&size, &type, mwf, &w, &off);
ptr = git_mwindow_open(mwf, &w, idx->pack->pack_fd, 4096, off, 0, NULL);
if (ptr == NULL) {
error = GIT_ENOMEM;
goto cleanup;
}
/*
* The size is when expanded, so we need to inflate the object
* so we know where the next one ist.
*/
type = entry_type(ptr);
size = entry_len(&data, ptr);
switch (type) { switch (type) {
case GIT_OBJ_COMMIT: case GIT_OBJ_COMMIT:
...@@ -234,7 +223,7 @@ int git_indexer_run(git_indexer *idx, int (*cb)(const git_indexer_stats *, void ...@@ -234,7 +223,7 @@ int git_indexer_run(git_indexer *idx, int (*cb)(const git_indexer_stats *, void
idx->stats.processed++; idx->stats.processed++;
if (cb) if (cb)
cb(&idx->stats, data); cb(&idx->stats, cb_data);
} }
...@@ -247,7 +236,7 @@ cleanup: ...@@ -247,7 +236,7 @@ cleanup:
void git_indexer_free(git_indexer *idx) void git_indexer_free(git_indexer *idx)
{ {
p_close(idx->pack->pack_fd); p_close(idx->pack->mwf.fd);
git_vector_free(&idx->objects); git_vector_free(&idx->objects);
git_vector_free(&idx->deltas); git_vector_free(&idx->deltas);
free(idx->pack); free(idx->pack);
......
src/mwindow.c
...@@ -203,8 +203,8 @@ cleanup: ...@@ -203,8 +203,8 @@ cleanup:
* Open a new window, closing the least recenty used until we have * Open a new window, closing the least recenty used until we have
* enough space. Don't forget to add it to your list * enough space. Don't forget to add it to your list
*/ */
unsigned char *git_mwindow_open(git_mwindow_file *mwf, git_mwindow **cursor, git_file fd, unsigned char *git_mwindow_open(git_mwindow_file *mwf, git_mwindow **cursor,
size_t size, off_t offset, int extra, unsigned int *left) off_t offset, int extra, unsigned int *left)
{ {
git_mwindow *w = *cursor; git_mwindow *w = *cursor;
...@@ -223,7 +223,7 @@ unsigned char *git_mwindow_open(git_mwindow_file *mwf, git_mwindow **cursor, git ...@@ -223,7 +223,7 @@ unsigned char *git_mwindow_open(git_mwindow_file *mwf, git_mwindow **cursor, git
* one. * one.
*/ */
if (!w) { if (!w) {
w = new_window(mwf, fd, size, offset); w = new_window(mwf, mwf->fd, mwf->size, offset);
if (w == NULL) if (w == NULL)
return NULL; return NULL;
w->next = mwf->windows; w->next = mwf->windows;
......
src/mwindow.h
...@@ -40,6 +40,8 @@ typedef struct git_mwindow { ...@@ -40,6 +40,8 @@ typedef struct git_mwindow {
typedef struct git_mwindow_file { typedef struct git_mwindow_file {
git_mwindow *windows; git_mwindow *windows;
int fd;
off_t size;
} git_mwindow_file; } git_mwindow_file;
typedef struct git_mwindow_ctl { typedef struct git_mwindow_ctl {
...@@ -56,7 +58,7 @@ typedef struct git_mwindow_ctl { ...@@ -56,7 +58,7 @@ typedef struct git_mwindow_ctl {
int git_mwindow_contains(git_mwindow *win, off_t offset); int git_mwindow_contains(git_mwindow *win, off_t offset);
void git_mwindow_free_all(git_mwindow_file *mwf); void git_mwindow_free_all(git_mwindow_file *mwf);
unsigned char *git_mwindow_open(git_mwindow_file *mwf, git_mwindow **cursor, git_file fd, size_t size, off_t offset, int extra, unsigned int *left); unsigned char *git_mwindow_open(git_mwindow_file *mwf, git_mwindow **cursor, off_t offset, int extra, unsigned int *left);
void git_mwindow_scan_lru(git_mwindow_file *mwf, git_mwindow **lru_w, git_mwindow **lru_l); void git_mwindow_scan_lru(git_mwindow_file *mwf, git_mwindow **lru_w, git_mwindow **lru_l);
int git_mwindow_file_register(git_mwindow_file *mwf); int git_mwindow_file_register(git_mwindow_file *mwf);
void git_mwindow_close(git_mwindow **w_cursor); void git_mwindow_close(git_mwindow **w_cursor);
......
src/odb_pack.c
...@@ -149,9 +149,6 @@ struct pack_backend { ...@@ -149,9 +149,6 @@ struct pack_backend {
static void pack_window_free_all(struct pack_backend *backend, struct pack_file *p); static void pack_window_free_all(struct pack_backend *backend, struct pack_file *p);
static int pack_window_contains(git_mwindow *win, off_t offset); static int pack_window_contains(git_mwindow *win, off_t offset);
static unsigned char *pack_window_open(struct pack_file *p,
git_mwindow **w_cursor, off_t offset, unsigned int *left);
static int packfile_sort__cb(const void *a_, const void *b_); static int packfile_sort__cb(const void *a_, const void *b_);
static void pack_index_free(struct pack_file *p); static void pack_index_free(struct pack_file *p);
...@@ -202,46 +199,6 @@ static int pack_entry_find_prefix(struct pack_entry *e, ...@@ -202,46 +199,6 @@ static int pack_entry_find_prefix(struct pack_entry *e,
const git_oid *short_oid, const git_oid *short_oid,
unsigned int len); unsigned int len);
static off_t get_delta_base(struct pack_file *p, git_mwindow **w_curs,
off_t *curpos, git_otype type,
off_t delta_obj_offset);
static unsigned long packfile_unpack_header1(
size_t *sizep,
git_otype *type,
const unsigned char *buf,
unsigned long len);
static int packfile_unpack_header(
size_t *size_p,
git_otype *type_p,
struct pack_file *p,
git_mwindow **w_curs,
off_t *curpos);
static int packfile_unpack_compressed(
git_rawobj *obj,
struct pack_file *p,
git_mwindow **w_curs,
off_t curpos,
size_t size,
git_otype type);
static int packfile_unpack_delta(
git_rawobj *obj,
struct pack_backend *backend,
struct pack_file *p,
git_mwindow **w_curs,
off_t curpos,
size_t delta_size,
git_otype delta_type,
off_t obj_offset);
static int packfile_unpack(git_rawobj *obj, struct pack_backend *backend,
struct pack_file *p, off_t obj_offset);
/*********************************************************** /***********************************************************
...@@ -266,27 +223,6 @@ GIT_INLINE(int) pack_window_contains(git_mwindow *win, off_t offset) ...@@ -266,27 +223,6 @@ GIT_INLINE(int) pack_window_contains(git_mwindow *win, off_t offset)
return git_mwindow_contains(win, offset + 20); return git_mwindow_contains(win, offset + 20);
} }
static unsigned char *pack_window_open(
struct pack_file *p,
git_mwindow **w_cursor,
off_t offset,
unsigned int *left)
{
if (p->pack_fd == -1 && packfile_open(p) < GIT_SUCCESS)
return NULL;
/* Since packfiles end in a hash of their content and it's
* pointless to ask for an offset into the middle of that
* hash, and the pack_window_contains function above wouldn't match
* don't allow an offset too close to the end of the file.
*/
if (offset > (p->pack_size - 20))
return NULL;
return git_mwindow_open(&p->mwf, w_cursor, p->pack_fd, p->pack_size, offset, 20, left);
}
/*********************************************************** /***********************************************************
* *
...@@ -483,7 +419,7 @@ static struct pack_file *packfile_alloc(int extra) ...@@ -483,7 +419,7 @@ static struct pack_file *packfile_alloc(int extra)
{ {
struct pack_file *p = git__malloc(sizeof(*p) + extra); struct pack_file *p = git__malloc(sizeof(*p) + extra);
memset(p, 0, sizeof(*p)); memset(p, 0, sizeof(*p));
p->pack_fd = -1; p->mwf.fd = -1;
return p; return p;
} }
...@@ -495,8 +431,8 @@ static void packfile_free(struct pack_backend *backend, struct pack_file *p) ...@@ -495,8 +431,8 @@ static void packfile_free(struct pack_backend *backend, struct pack_file *p)
/* clear_delta_base_cache(); */ /* clear_delta_base_cache(); */
pack_window_free_all(backend, p); pack_window_free_all(backend, p);
if (p->pack_fd != -1) if (p->mwf.fd != -1)
p_close(p->pack_fd); p_close(p->mwf.fd);
pack_index_free(p); pack_index_free(p);
...@@ -515,28 +451,28 @@ static int packfile_open(struct pack_file *p) ...@@ -515,28 +451,28 @@ static int packfile_open(struct pack_file *p)
return git__throw(GIT_ENOTFOUND, "Failed to open packfile. File not found"); return git__throw(GIT_ENOTFOUND, "Failed to open packfile. File not found");
/* TODO: open with noatime */ /* TODO: open with noatime */
p->pack_fd = p_open(p->pack_name, O_RDONLY); p->mwf.fd = p_open(p->pack_name, O_RDONLY);
if (p->pack_fd < 0 || p_fstat(p->pack_fd, &st) < GIT_SUCCESS) if (p->mwf.fd < 0 || p_fstat(p->mwf.fd, &st) < GIT_SUCCESS)
return git__throw(GIT_EOSERR, "Failed to open packfile. File appears to be corrupted"); return git__throw(GIT_EOSERR, "Failed to open packfile. File appears to be corrupted");
if (git_mwindow_file_register(&p->mwf) < GIT_SUCCESS) { if (git_mwindow_file_register(&p->mwf) < GIT_SUCCESS) {
p_close(p->pack_fd); p_close(p->mwf.fd);
return git__throw(GIT_ERROR, "Failed to register packfile windows"); return git__throw(GIT_ERROR, "Failed to register packfile windows");
} }
/* If we created the struct before we had the pack we lack size. */ /* If we created the struct before we had the pack we lack size. */
if (!p->pack_size) { if (!p->mwf.size) {
if (!S_ISREG(st.st_mode)) if (!S_ISREG(st.st_mode))
goto cleanup; goto cleanup;
p->pack_size = (off_t)st.st_size; p->mwf.size = (off_t)st.st_size;
} else if (p->pack_size != st.st_size) } else if (p->mwf.size != st.st_size)
goto cleanup; goto cleanup;
#if 0 #if 0
/* We leave these file descriptors open with sliding mmap; /* We leave these file descriptors open with sliding mmap;
* there is no point keeping them open across exec(), though. * there is no point keeping them open across exec(), though.
*/ */
fd_flag = fcntl(p->pack_fd, F_GETFD, 0); fd_flag = fcntl(p->mwf.fd, F_GETFD, 0);
if (fd_flag < 0) if (fd_flag < 0)
return error("cannot determine file descriptor flags"); return error("cannot determine file descriptor flags");
...@@ -546,7 +482,7 @@ static int packfile_open(struct pack_file *p) ...@@ -546,7 +482,7 @@ static int packfile_open(struct pack_file *p)
#endif #endif
/* Verify we recognize this pack file format. */ /* Verify we recognize this pack file format. */
if (p_read(p->pack_fd, &hdr, sizeof(hdr)) < GIT_SUCCESS) if (p_read(p->mwf.fd, &hdr, sizeof(hdr)) < GIT_SUCCESS)
goto cleanup; goto cleanup;
if (hdr.hdr_signature != htonl(PACK_SIGNATURE)) if (hdr.hdr_signature != htonl(PACK_SIGNATURE))
...@@ -559,10 +495,10 @@ static int packfile_open(struct pack_file *p) ...@@ -559,10 +495,10 @@ static int packfile_open(struct pack_file *p)
if (p->num_objects != ntohl(hdr.hdr_entries)) if (p->num_objects != ntohl(hdr.hdr_entries))
goto cleanup; goto cleanup;
if (p_lseek(p->pack_fd, p->pack_size - GIT_OID_RAWSZ, SEEK_SET) == -1) if (p_lseek(p->mwf.fd, p->mwf.size - GIT_OID_RAWSZ, SEEK_SET) == -1)
goto cleanup; goto cleanup;
if (p_read(p->pack_fd, sha1.id, GIT_OID_RAWSZ) < GIT_SUCCESS) if (p_read(p->mwf.fd, sha1.id, GIT_OID_RAWSZ) < GIT_SUCCESS)
goto cleanup; goto cleanup;
idx_sha1 = ((unsigned char *)p->index_map.data) + p->index_map.len - 40; idx_sha1 = ((unsigned char *)p->index_map.data) + p->index_map.len - 40;
...@@ -573,8 +509,8 @@ static int packfile_open(struct pack_file *p) ...@@ -573,8 +509,8 @@ static int packfile_open(struct pack_file *p)
return GIT_SUCCESS; return GIT_SUCCESS;
cleanup: cleanup:
p_close(p->pack_fd); p_close(p->mwf.fd);
p->pack_fd = -1; p->mwf.fd = -1;
return git__throw(GIT_EPACKCORRUPTED, "Failed to packfile. Pack is corrupted"); return git__throw(GIT_EPACKCORRUPTED, "Failed to packfile. Pack is corrupted");
} }
...@@ -613,7 +549,7 @@ static int packfile_check(struct pack_file **pack_out, const char *path) ...@@ -613,7 +549,7 @@ static int packfile_check(struct pack_file **pack_out, const char *path)
/* ok, it looks sane as far as we can check without /* ok, it looks sane as far as we can check without
* actually mapping the pack file. * actually mapping the pack file.
*/ */
p->pack_size = (off_t)st.st_size; p->mwf.size = (off_t)st.st_size;
p->pack_local = 1; p->pack_local = 1;
p->mtime = (git_time_t)st.st_mtime; p->mtime = (git_time_t)st.st_mtime;
...@@ -833,7 +769,7 @@ static int pack_entry_find1( ...@@ -833,7 +769,7 @@ static int pack_entry_find1(
/* we found a unique entry in the index; /* we found a unique entry in the index;
* make sure the packfile backing the index * make sure the packfile backing the index
* still exists on disk */ * still exists on disk */
if (p->pack_fd == -1 && packfile_open(p) < GIT_SUCCESS) if (p->mwf.fd == -1 && packfile_open(p) < GIT_SUCCESS)
return git__throw(GIT_EOSERR, "Failed to find pack entry. Packfile doesn't exist on disk"); return git__throw(GIT_EOSERR, "Failed to find pack entry. Packfile doesn't exist on disk");
e->offset = offset; e->offset = offset;
...@@ -922,271 +858,6 @@ static int pack_entry_find_prefix( ...@@ -922,271 +858,6 @@ static int pack_entry_find_prefix(
} }
/***********************************************************
*
* PACKFILE ENTRY UNPACK INTERNALS
*
***********************************************************/
static unsigned long packfile_unpack_header1(
size_t *sizep,
git_otype *type,
const unsigned char *buf,
unsigned long len)
{
unsigned shift;
unsigned long size, c;
unsigned long used = 0;
c = buf[used++];
*type = (c >> 4) & 7;
size = c & 15;
shift = 4;
while (c & 0x80) {
if (len <= used || bitsizeof(long) <= shift)
return 0;
c = buf[used++];
size += (c & 0x7f) << shift;
shift += 7;
}
*sizep = (size_t)size;
return used;
}
static int packfile_unpack_header(
size_t *size_p,
git_otype *type_p,
struct pack_file *p,
git_mwindow **w_curs,
off_t *curpos)
{
unsigned char *base;
unsigned int left;
unsigned long used;
/* pack_window_open() assures us we have [base, base + 20) available
* as a range that we can look at at. (Its actually the hash
* size that is assured.) With our object header encoding
* the maximum deflated object size is 2^137, which is just
* insane, so we know won't exceed what we have been given.
*/
base = pack_window_open(p, w_curs, *curpos, &left);
if (base == NULL)
return GIT_ENOMEM;
used = packfile_unpack_header1(size_p, type_p, base, left);
if (used == 0)
return git__throw(GIT_EOBJCORRUPTED, "Header length is zero");
*curpos += used;
return GIT_SUCCESS;
}
static int packfile_unpack_compressed(
git_rawobj *obj,
struct pack_file *p,
git_mwindow **w_curs,
off_t curpos,
size_t size,
git_otype type)
{
int st;
z_stream stream;
unsigned char *buffer, *in;
buffer = git__malloc(size + 1);
memset(buffer, 0x0, size + 1);
memset(&stream, 0, sizeof(stream));
stream.next_out = buffer;
stream.avail_out = size + 1;
st = inflateInit(&stream);
if (st != Z_OK) {
free(buffer);
return git__throw(GIT_EZLIB, "Error in zlib");
}
do {
in = pack_window_open(p, w_curs, curpos, &stream.avail_in);
stream.next_in = in;
st = inflate(&stream, Z_FINISH);
if (!stream.avail_out)
break; /* the payload is larger than it should be */
curpos += stream.next_in - in;
} while (st == Z_OK || st == Z_BUF_ERROR);
inflateEnd(&stream);
if ((st != Z_STREAM_END) || stream.total_out != size) {
free(buffer);
return git__throw(GIT_EZLIB, "Error in zlib");
}
obj->type = type;
obj->len = size;
obj->data = buffer;
return GIT_SUCCESS;
}
static off_t get_delta_base(
struct pack_file *p,
git_mwindow **w_curs,
off_t *curpos,
git_otype type,
off_t delta_obj_offset)
{
unsigned char *base_info = pack_window_open(p, w_curs, *curpos, NULL);
off_t base_offset;
git_oid unused;
/* pack_window_open() assured us we have [base_info, base_info + 20)
* as a range that we can look at without walking off the
* end of the mapped window. Its actually the hash size
* that is assured. An OFS_DELTA longer than the hash size
* is stupid, as then a REF_DELTA would be smaller to store.
*/
if (type == GIT_OBJ_OFS_DELTA) {
unsigned used = 0;
unsigned char c = base_info[used++];
base_offset = c & 127;
while (c & 128) {
base_offset += 1;
if (!base_offset || MSB(base_offset, 7))
return 0; /* overflow */
c = base_info[used++];
base_offset = (base_offset << 7) + (c & 127);
}
base_offset = delta_obj_offset - base_offset;
if (base_offset <= 0 || base_offset >= delta_obj_offset)
return 0; /* out of bound */
*curpos += used;
} else if (type == GIT_OBJ_REF_DELTA) {
/* The base entry _must_ be in the same pack */
if (pack_entry_find_offset(&base_offset, &unused, p, (git_oid *)base_info, GIT_OID_HEXSZ) < GIT_SUCCESS)
return git__throw(GIT_EPACKCORRUPTED, "Base entry delta is not in the same pack");
*curpos += 20;
} else
return 0;
return base_offset;
}
static int packfile_unpack_delta(
git_rawobj *obj,
struct pack_backend *backend,
struct pack_file *p,
git_mwindow **w_curs,
off_t curpos,
size_t delta_size,
git_otype delta_type,
off_t obj_offset)
{
off_t base_offset;
git_rawobj base, delta;
int error;
base_offset = get_delta_base(p, w_curs, &curpos, delta_type, obj_offset);
if (base_offset == 0)
return git__throw(GIT_EOBJCORRUPTED, "Delta offset is zero");
git_mwindow_close(w_curs);
error = packfile_unpack(&base, backend, p, base_offset);
/* TODO: git.git tries to load the base from other packfiles
* or loose objects */
if (error < GIT_SUCCESS)
return git__rethrow(error, "Corrupted delta");
error = packfile_unpack_compressed(&delta, p, w_curs, curpos, delta_size, delta_type);
if (error < GIT_SUCCESS) {
free(base.data);
return git__rethrow(error, "Corrupted delta");
}
obj->type = base.type;
error = git__delta_apply(obj,
base.data, base.len,
delta.data, delta.len);
free(base.data);
free(delta.data);
/* TODO: we might want to cache this shit. eventually */
//add_delta_base_cache(p, base_offset, base, base_size, *type);
return error; /* error set by git__delta_apply */
}
static int packfile_unpack(
git_rawobj *obj,
struct pack_backend *backend,
struct pack_file *p,
off_t obj_offset)
{
git_mwindow *w_curs = NULL;
off_t curpos = obj_offset;
int error;
size_t size = 0;
git_otype type;
/*
* TODO: optionally check the CRC on the packfile
*/
obj->data = NULL;
obj->len = 0;
obj->type = GIT_OBJ_BAD;
error = packfile_unpack_header(&size, &type, p, &w_curs, &curpos);
if (error < GIT_SUCCESS)
return git__rethrow(error, "Failed to unpack packfile");
switch (type) {
case GIT_OBJ_OFS_DELTA:
case GIT_OBJ_REF_DELTA:
error = packfile_unpack_delta(
obj, backend, p, &w_curs, curpos,
size, type, obj_offset);
break;
case GIT_OBJ_COMMIT:
case GIT_OBJ_TREE:
case GIT_OBJ_BLOB:
case GIT_OBJ_TAG:
error = packfile_unpack_compressed(
obj, p, &w_curs, curpos,
size, type);
break;
default:
error = GIT_EOBJCORRUPTED;
break;
}
git_mwindow_close(&w_curs);
return error == GIT_SUCCESS ? GIT_SUCCESS : git__rethrow(error, "Failed to unpack packfile");
}
/*********************************************************** /***********************************************************
* *
* PACKED BACKEND PUBLIC API * PACKED BACKEND PUBLIC API
...@@ -1218,7 +889,7 @@ int pack_backend__read(void **buffer_p, size_t *len_p, git_otype *type_p, git_od ...@@ -1218,7 +889,7 @@ int pack_backend__read(void **buffer_p, size_t *len_p, git_otype *type_p, git_od
if ((error = pack_entry_find(&e, (struct pack_backend *)backend, oid)) < GIT_SUCCESS) if ((error = pack_entry_find(&e, (struct pack_backend *)backend, oid)) < GIT_SUCCESS)
return git__rethrow(error, "Failed to read pack backend"); return git__rethrow(error, "Failed to read pack backend");
if ((error = packfile_unpack(&raw, (struct pack_backend *)backend, e.p, e.offset)) < GIT_SUCCESS) if ((error = packfile_unpack(&raw, e.p, e.offset)) < GIT_SUCCESS)
return git__rethrow(error, "Failed to read pack backend"); return git__rethrow(error, "Failed to read pack backend");
*buffer_p = raw.data; *buffer_p = raw.data;
...@@ -1255,7 +926,7 @@ int pack_backend__read_prefix( ...@@ -1255,7 +926,7 @@ int pack_backend__read_prefix(
if ((error = pack_entry_find_prefix(&e, (struct pack_backend *)backend, short_oid, len)) < GIT_SUCCESS) if ((error = pack_entry_find_prefix(&e, (struct pack_backend *)backend, short_oid, len)) < GIT_SUCCESS)
return git__rethrow(error, "Failed to read pack backend"); return git__rethrow(error, "Failed to read pack backend");
if ((error = packfile_unpack(&raw, (struct pack_backend *)backend, e.p, e.offset)) < GIT_SUCCESS) if ((error = packfile_unpack(&raw, e.p, e.offset)) < GIT_SUCCESS)
return git__rethrow(error, "Failed to read pack backend"); return git__rethrow(error, "Failed to read pack backend");
*buffer_p = raw.data; *buffer_p = raw.data;
......
src/pack.c 0 → 100644
/*
* This file is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License, version 2,
* as published by the Free Software Foundation.
*
* In addition to the permissions in the GNU General Public License,
* the authors give you unlimited permission to link the compiled
* version of this file into combinations with other programs,
* and to distribute those combinations without any restriction
* coming from the use of this file. (The General Public License
* restrictions do apply in other respects; for example, they cover
* modification of the file, and distribution when not linked into
* a combined executable.)
*
* This file is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; see the file COPYING. If not, write to
* the Free Software Foundation, 51 Franklin Street, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
#include "mwindow.h"
#include "odb.h"
#include "pack.h"
#include "delta-apply.h"
#include "git2/oid.h"
#include "git2/zlib.h"
unsigned char *pack_window_open(
struct pack_file *p,
git_mwindow **w_cursor,
off_t offset,
unsigned int *left)
{
if (p->mwf.fd == -1 && packfile_open(p) < GIT_SUCCESS)
return NULL;
/* Since packfiles end in a hash of their content and it's
* pointless to ask for an offset into the middle of that
* hash, and the pack_window_contains function above wouldn't match
* don't allow an offset too close to the end of the file.
*/
if (offset > (p->mwf.size - 20))
return NULL;
return git_mwindow_open(&p->mwf, w_cursor, offset, 20, left);
}
static unsigned long packfile_unpack_header1(
size_t *sizep,
git_otype *type,
const unsigned char *buf,
unsigned long len)
{
unsigned shift;
unsigned long size, c;
unsigned long used = 0;
c = buf[used++];
*type = (c >> 4) & 7;
size = c & 15;
shift = 4;
while (c & 0x80) {
if (len <= used || bitsizeof(long) <= shift)
return 0;
c = buf[used++];
size += (c & 0x7f) << shift;
shift += 7;
}
*sizep = (size_t)size;
return used;
}
int git_packfile_unpack_header(
size_t *size_p,
git_otype *type_p,
git_mwindow_file *mwf,
git_mwindow **w_curs,
off_t *curpos)
{
unsigned char *base;
unsigned int left;
unsigned long used;
/* pack_window_open() assures us we have [base, base + 20) available
* as a range that we can look at at. (Its actually the hash
* size that is assured.) With our object header encoding
* the maximum deflated object size is 2^137, which is just
* insane, so we know won't exceed what we have been given.
*/
// base = pack_window_open(p, w_curs, *curpos, &left);
base = git_mwindow_open(mwf, w_curs, *curpos, 20, &left);
if (base == NULL)
return GIT_ENOMEM;
used = packfile_unpack_header1(size_p, type_p, base, left);
if (used == 0)
return git__throw(GIT_EOBJCORRUPTED, "Header length is zero");
*curpos += used;
return GIT_SUCCESS;
}
int packfile_unpack_delta(
git_rawobj *obj,
struct pack_file *p,
git_mwindow **w_curs,
off_t curpos,
size_t delta_size,
git_otype delta_type,
off_t obj_offset)
{
off_t base_offset;
git_rawobj base, delta;
int error;
base_offset = get_delta_base(p, w_curs, &curpos, delta_type, obj_offset);
if (base_offset == 0)
return git__throw(GIT_EOBJCORRUPTED, "Delta offset is zero");
git_mwindow_close(w_curs);
error = packfile_unpack(&base, p, base_offset);
/*
* TODO: git.git tries to load the base from other packfiles
* or loose objects.
*
* We'll need to do this in order to support thin packs.
*/
if (error < GIT_SUCCESS)
return git__rethrow(error, "Corrupted delta");
error = packfile_unpack_compressed(&delta, p, w_curs, curpos, delta_size, delta_type);
if (error < GIT_SUCCESS) {
free(base.data);
return git__rethrow(error, "Corrupted delta");
}
obj->type = base.type;
error = git__delta_apply(obj,
base.data, base.len,
delta.data, delta.len);
free(base.data);
free(delta.data);
/* TODO: we might want to cache this shit. eventually */
//add_delta_base_cache(p, base_offset, base, base_size, *type);
return error; /* error set by git__delta_apply */
}
int packfile_unpack(
git_rawobj *obj,
struct pack_file *p,
off_t obj_offset)
{
git_mwindow *w_curs = NULL;
off_t curpos = obj_offset;
int error;
size_t size = 0;
git_otype type;
/*
* TODO: optionally check the CRC on the packfile
*/
obj->data = NULL;
obj->len = 0;
obj->type = GIT_OBJ_BAD;
error = git_packfile_unpack_header(&size, &type, &p->mwf, &w_curs, &curpos);
if (error < GIT_SUCCESS)
return git__rethrow(error, "Failed to unpack packfile");
switch (type) {
case GIT_OBJ_OFS_DELTA:
case GIT_OBJ_REF_DELTA:
error = packfile_unpack_delta(
obj, p, &w_curs, curpos,
size, type, obj_offset);
break;
case GIT_OBJ_COMMIT:
case GIT_OBJ_TREE:
case GIT_OBJ_BLOB:
case GIT_OBJ_TAG:
error = packfile_unpack_compressed(
obj, p, &w_curs, curpos,
size, type);
break;
default:
error = GIT_EOBJCORRUPTED;
break;
}
git_mwindow_close(&w_curs);
return error == GIT_SUCCESS ? GIT_SUCCESS : git__rethrow(error, "Failed to unpack packfile");
}
int packfile_unpack_compressed(
git_rawobj *obj,
struct pack_file *p,
git_mwindow **w_curs,
off_t curpos,
size_t size,
git_otype type)
{
int st;
z_stream stream;
unsigned char *buffer, *in;
buffer = git__malloc(size + 1);
memset(buffer, 0x0, size + 1);
memset(&stream, 0, sizeof(stream));
stream.next_out = buffer;
stream.avail_out = size + 1;
st = inflateInit(&stream);
if (st != Z_OK) {
free(buffer);
return git__throw(GIT_EZLIB, "Error in zlib");
}
do {
in = pack_window_open(p, w_curs, curpos, &stream.avail_in);
stream.next_in = in;
st = inflate(&stream, Z_FINISH);
if (!stream.avail_out)
break; /* the payload is larger than it should be */
curpos += stream.next_in - in;
} while (st == Z_OK || st == Z_BUF_ERROR);
inflateEnd(&stream);
if ((st != Z_STREAM_END) || stream.total_out != size) {
free(buffer);
return git__throw(GIT_EZLIB, "Error in zlib");
}
obj->type = type;
obj->len = size;
obj->data = buffer;
return GIT_SUCCESS;
}
off_t get_delta_base(
struct pack_file *p,
git_mwindow **w_curs,
off_t *curpos,
git_otype type,
off_t delta_obj_offset)
{
unsigned char *base_info = pack_window_open(p, w_curs, *curpos, NULL);
off_t base_offset;
git_oid unused;
/* pack_window_open() assured us we have [base_info, base_info + 20)
* as a range that we can look at without walking off the
* end of the mapped window. Its actually the hash size
* that is assured. An OFS_DELTA longer than the hash size
* is stupid, as then a REF_DELTA would be smaller to store.
*/
if (type == GIT_OBJ_OFS_DELTA) {
unsigned used = 0;
unsigned char c = base_info[used++];
base_offset = c & 127;
while (c & 128) {
base_offset += 1;
if (!base_offset || MSB(base_offset, 7))
return 0; /* overflow */
c = base_info[used++];
base_offset = (base_offset << 7) + (c & 127);
}
base_offset = delta_obj_offset - base_offset;
if (base_offset <= 0 || base_offset >= delta_obj_offset)
return 0; /* out of bound */
*curpos += used;
} else if (type == GIT_OBJ_REF_DELTA) {
/* The base entry _must_ be in the same pack */
if (pack_entry_find_offset(&base_offset, &unused, p, (git_oid *)base_info, GIT_OID_HEXSZ) < GIT_SUCCESS)
return git__throw(GIT_EPACKCORRUPTED, "Base entry delta is not in the same pack");
*curpos += 20;
} else
return 0;
return base_offset;
}
src/pack.h
...@@ -31,6 +31,7 @@ ...@@ -31,6 +31,7 @@
#include "common.h" #include "common.h"
#include "map.h" #include "map.h"
#include "mwindow.h" #include "mwindow.h"
#include "odb.h"
#define PACK_SIGNATURE 0x5041434b /* "PACK" */ #define PACK_SIGNATURE 0x5041434b /* "PACK" */
#define PACK_VERSION 2 #define PACK_VERSION 2
...@@ -67,9 +68,7 @@ struct pack_idx_header { ...@@ -67,9 +68,7 @@ struct pack_idx_header {
}; };
struct pack_file { struct pack_file {
int pack_fd;
git_mwindow_file mwf; git_mwindow_file mwf;
off_t pack_size;
git_map index_map; git_map index_map;
uint32_t num_objects; uint32_t num_objects;
...@@ -91,4 +90,29 @@ struct pack_entry { ...@@ -91,4 +90,29 @@ struct pack_entry {
struct pack_file *p; struct pack_file *p;
}; };
static unsigned char *pack_window_open(struct pack_file *p,
git_mwindow **w_cursor, off_t offset, unsigned int *left);
int git_packfile_unpack_header(
size_t *size_p,
git_otype *type_p,
git_mwindow_file *mwf,
git_mwindow **w_curs,
off_t *curpos);
int packfile_unpack_delta(
git_rawobj *obj,
struct pack_file *p,
git_mwindow **w_curs,
off_t curpos,
size_t delta_size,
git_otype delta_type,
off_t obj_offset);
int packfile_unpack(git_rawobj *obj, struct pack_file *p, off_t obj_offset);
off_t get_delta_base(struct pack_file *p, git_mwindow **w_curs,
off_t *curpos, git_otype type,
off_t delta_obj_offset);
#endif #endif
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment