Skip to content

G
git2
Commit 7e4f56a5 by Vicent Marti
Options

Add packfile reading 

Packed objects inside packfiles are now properly unpacked when calling
the git_odb__read_packed() method; delta'ed objects are also properly
generated when needed.

A new unit test 0204-readpack tries to read a couple hundred packed
objects from a standard packed repository.

Signed-off-by: Vicent Marti <tanoku@gmail.com>
parent d8603ed9
Showing with 311 additions and 19 deletions
src/odb.c
......@@ -29,6 +29,7 @@
#include "fileops.h"
#include "hash.h"
#include "odb.h"
#include "delta-apply.h"
#define GIT_PACK_NAME_MAX (5 + 40 + 1)
......@@ -74,6 +75,9 @@ struct git_pack {
/** File descriptor for the .pack file. */
git_file pack_fd;
/** Memory map of the pack's contents */
git_map pack_map;
/** The size of the .pack file. */
off_t pack_size;
......@@ -437,7 +441,7 @@ static int inflate_buffer(void *in, size_t inlen, void *out, size_t outlen)
inflateEnd(&zs);
if ((status != Z_STREAM_END) || (zs.avail_in != 0))
if ((status != Z_STREAM_END) /*|| (zs.avail_in != 0) */)
return GIT_ERROR;
if (zs.total_out != outlen)
......@@ -1160,23 +1164,26 @@ static int open_pack(git_pack *p)
if (pack_openidx(p))
return GIT_ERROR;
if ((p->pack_fd = gitfo_open(pb, O_RDONLY)) < 0) {
p->pack_fd = -1;
pack_decidx(p);
return GIT_ERROR;
}
if ((p->pack_fd = gitfo_open(pb, O_RDONLY)) < 0)
goto error_cleanup;
if (gitfo_fstat(p->pack_fd, &sb)
|| !S_ISREG(sb.st_mode) || p->pack_size != sb.st_size
|| check_pack_hdr(p) || check_pack_sha1(p)) {
gitfo_close(p->pack_fd);
p->pack_fd = -1;
pack_decidx(p);
return GIT_ERROR;
}
|| check_pack_hdr(p) || check_pack_sha1(p))
goto error_cleanup;
if (!git__is_sizet(p->pack_size) ||
gitfo_map_ro(&p->pack_map, p->pack_fd, 0, (size_t)p->pack_size) < 0)
goto error_cleanup;
pack_decidx(p);
return GIT_SUCCESS;
error_cleanup:
gitfo_close(p->pack_fd);
p->pack_fd = -1;
pack_decidx(p);
return GIT_ERROR;
}
static void pack_dec(git_pack *p)
......@@ -1194,8 +1201,10 @@ static void pack_dec(git_pack *p)
free(p->im_fanout);
free(p->im_off_idx);
free(p->im_off_next);
if (p->pack_fd != -1)
if (p->pack_fd != -1) {
gitfo_close(p->pack_fd);
gitfo_free_map(&p->pack_map);
}
}
gitlck_free(&p->lock);
......@@ -1475,14 +1484,141 @@ int git_odb__read_loose(git_obj *out, git_odb *db, const git_oid *id)
return GIT_SUCCESS;
}
static int unpack_object(git_obj *out, git_pack *p, index_entry *e);
static int unpack_object_delta(git_obj *out, git_pack *p,
index_entry *base_entry,
uint8_t *delta_buffer,
size_t delta_deflated_size,
size_t delta_inflated_size)
{
int res = 0;
uint8_t *delta = NULL;
git_obj base_obj;
base_obj.data = NULL;
base_obj.type = GIT_OBJ_BAD;
base_obj.len = 0;
if ((res = unpack_object(&base_obj, p, base_entry)) < 0)
goto cleanup;
delta = git__malloc(delta_inflated_size + 1);
if ((res = inflate_buffer(delta_buffer, delta_deflated_size,
delta, delta_inflated_size)) < 0)
goto cleanup;
res = git__delta_apply(out, base_obj.data, base_obj.len, delta, delta_inflated_size);
out->type = base_obj.type;
cleanup:
free(delta);
git_obj_close(&base_obj);
return res;
}
static int unpack_object(git_obj *out, git_pack *p, index_entry *e)
{
assert(out && p && e);
git_otype object_type;
size_t inflated_size, deflated_size, shift;
uint8_t *buffer, byte;
assert(out && p && e && git__is_sizet(e->size));
if (open_pack(p))
return GIT_ERROR;
/* TODO - actually unpack the data! */
buffer = (uint8_t *)p->pack_map.data + e->offset;
deflated_size = (size_t)e->size;
if (deflated_size == 0)
deflated_size = p->pack_size - e->offset;
byte = *buffer++ & 0xFF;
deflated_size--;
object_type = (byte >> 4) & 0x7;
inflated_size = byte & 0xF;
shift = 4;
while (byte & 0x80) {
byte = *buffer++ & 0xFF;
deflated_size--;
inflated_size += (byte & 0x7F) << shift;
shift += 7;
}
switch (object_type) {
case GIT_OBJ_COMMIT:
case GIT_OBJ_TREE:
case GIT_OBJ_BLOB:
case GIT_OBJ_TAG: {
/* Handle a normal zlib stream */
out->len = inflated_size;
out->type = object_type;
out->data = git__malloc(inflated_size + 1);
if (inflate_buffer(buffer, deflated_size, out->data, out->len) < 0) {
free(out->data);
out->data = NULL;
return GIT_ERROR;
}
return GIT_SUCCESS;
}
case GIT_OBJ_OFS_DELTA: {
off_t delta_offset;
index_entry entry;
byte = *buffer++ & 0xFF;
delta_offset = byte & 0x7F;
while (byte & 0x80) {
delta_offset += 1;
byte = *buffer++ & 0xFF;
delta_offset <<= 7;
delta_offset += (byte & 0x7F);
}
entry.n = 0;
entry.oid = NULL;
entry.offset = e->offset - delta_offset;
entry.size = 0;
if (unpack_object_delta(out, p, &entry,
buffer, deflated_size, inflated_size) < 0)
return GIT_ERROR;
return GIT_SUCCESS;
}
case GIT_OBJ_REF_DELTA: {
git_oid base_id;
uint32_t n;
index_entry entry;
int res = GIT_ERROR;
git_oid_mkraw(&base_id, buffer);
if (!p->idx_search(&n, p, &base_id) &&
!p->idx_get(&entry, p, n)) {
/* FIXME: deflated_size - 20 ? */
res = unpack_object_delta(out, p, &entry,
buffer + GIT_OID_RAWSZ, deflated_size, inflated_size);
}
return res;
}
default:
return GIT_EOBJCORRUPTED;
}
return GIT_SUCCESS;
}
......@@ -1497,15 +1633,15 @@ static int read_packed(git_obj *out, git_pack *p, const git_oid *id)
if (pack_openidx(p))
return GIT_ERROR;
res = p->idx_search(&n, p, id);
if (!res)
res = p->idx_get(&e, p, n);
pack_decidx(p);
if (!res) {
/* TODO unpack object */
if (!res)
res = unpack_object(out, p, &e);
}
pack_decidx(p);
return res;
}
......
tests/resources/pack-odb/info/packs 0 → 100644
P pack-d85f5d483273108c9d8dd0e4728ccf0b2982423a.pack
tests/t0204-readpack.c 0 → 100644
#include "test_lib.h"
#include "test_helpers.h"
#include <git/odb.h>
#define ODB_FOLDER "../resources/pack-odb"
static const char *packed_objects[] = {
"0266163a49e280c4f5ed1e08facd36a2bd716bcf",
"53fc32d17276939fc79ed05badaef2db09990016",
"6336846bd5c88d32f93ae57d846683e61ab5c530",
"6dcf9bf7541ee10456529833502442f385010c3d",
"bed08a0b30b72a9d4aed7f1af8c8ca124e8d64b9",
"e90810b8df3e80c413d903f631643c716887138d",
"fc3c3a2083e9f6f89e6bd53e9420e70d1e357c9b",
"fc58168adf502d0c0ef614c3111a7038fc8c09c8",
"fd0ec0333948dfe23265ac46be0205a436a8c3a5",
"fd8430bc864cfcd5f10e5590f8a447e01b942bfe",
"fd899f45951c15c1c5f7c34b1c864e91bd6556c6",
"fda23b974899e7e1f938619099280bfda13bdca9",
"fdbec189efb657c8325962b494875987881a356b",
"fe1ca6bd22b5d8353ce6c2f3aba80805c438a7a5",
"fe3a6a42c87ff1239370c741a265f3997add87c1",
"deb106bfd2d36ecf9f0079224c12022201a39ad1",
"dec93efc79e60f2680de3e666755d335967eec30",
"def425bf8568b9c1e20879bf5be6f9c52b7361c4",
"df48000ac4f48570054e3a71a81916357997b680",
"dfae6ed8f6dd8acc3b40a31811ea316239223559",
"dff79e27d3d2cdc09790ded80fe2ea8ff5d61034",
"e00e46abe4c542e17c8bc83d72cf5be8018d7b0e",
"e01b107b4f77f8f98645adac0206a504f2d29d7c",
"e032d863f512c47b479bd984f8b6c8061f66b7d4",
"e044baa468a1c74f9f9da36805445f6888358b49",
"e04529998989ba8ae3419538dd57969af819b241",
"e0637ddfbea67c8d7f557c709e095af8906e9176",
"e0743ad4031231e71700abdc6fdbe94f189d20e5",
"cf33ac7a3d8b2b8f6bb266518aadbf59de397608",
"cf5f7235b9c9689b133f6ea12015720b411329bd",
"cf6cccf1297284833a9a03138a1f5738fa1c6c94",
"cf7992bde17ce7a79cab5f0c1fcbe8a0108721ed",
"cfe3a027ab12506d4144ee8a35669ae8fc4b7ab1",
"cfe96f31dfad7bab49977aa1df7302f7fafcb025",
"cff54d138945ef4de384e9d2759291d0c13ea90a",
"d01f7573ac34c2f502bd1cf18cde73480c741151",
"d03f567593f346a1ca96a57f8191def098d126e3",
"d047b47aadf88501238f36f5c17dd0a50dc62087",
"d0a0d63086fae3b0682af7261df21f7d0f7f066d",
"d0a44bd6ed0be21b725a96c0891bbc79bc1a540c",
"d0d7e736e536a41bcb885005f8bf258c61cad682",
"d0e7959d4b95ffec6198df6f5a7ae259b23a5f50",
"bf2fe2acca17d13356ce802ba9dc8343f710dfb7",
"bf55f407d6d9418e51f42ea7a3a6aadf17388349",
"bf92206f8b633b88a66dca4a911777630b06fbac",
"bfaf8c42eb8842abe206179fee864cfba87e3ca9",
"bfe05675d4e8f6b59d50932add8790f1a06b10ee",
"bff8618112330763327cfa6ce6e914db84f51ddf",
"bff873e9853ed99fed52c25f7ad29f78b27dcec2",
"c01c3fae7251098d7af1b459bcd0786e81d4616d",
"c0220fca67f48b8a5d4163d53b1486224be3a198",
"c02d0b160b82ee72469c269f13de4c26a7ea09cb",
"c059510ad1b45ab58390e042d7dee1ac46703854",
"c07204a1897aeeaa3c248d29dbfa9b033baf9755",
"c073337a4dd7276931b4b3fdbc3f0040e9441793",
"0fd7e4bfba5b3a82be88d1057757ca8b2c5e6d26",
"100746511cc45c9f1ad6721c4ef5be49222fee4d",
"1088490171d9b984d68b8b9be9ca003f4eafff59",
"1093c8ff4cb78fcf5f79dbbeedcb6e824bd4e253",
"10aa3fa72afab7ee31e116ae06442fe0f7b79df2",
"10b759e734e8299aa0dca08be935d95d886127b6",
"111d5ccf0bb010c4e8d7af3eedfa12ef4c5e265b",
"11261fbff21758444d426356ff6327ee01e90752",
"112998d425717bb922ce74e8f6f0f831d8dc4510",
"2ef4e5d838b6507bd61d457cf6466662b791c5c0",
"2ef4faa0f82efa00eeac6cae9e8b2abccc8566ee",
"2f06098183b0d7be350acbe39cdbaccff2df0c4a",
"2f1c5d509ac5bffb3c62f710a1c2c542e126dfd1",
"2f205b20fc16423c42b3ba51b2ea78d7b9ff3578",
"2f9b6b6e3d9250ba09360734aa47973a993b59d1",
"30c62a2d5a8d644f1311d4f7fe3f6a788e4c8188",
"31438e245492d85fd6da4d1406eba0fbde8332a4",
"3184a3abdfea231992254929ff4e275898e5bbf6",
"3188ffdbb3a3d52e0f78f30c484533899224436e",
"32581d0093429770d044a60eb0e9cc0462bedb13",
"32679a9544d83e5403202c4d5efb61ad02492847",
"4e7e9f60b7e2049b7f5697daf133161a18ef688f",
"4e8cda27ddc8be7db875ceb0f360c37734724c6d",
"4ea481c61c59ab55169b7cbaae536ad50b49d6f0",
"4f0adcd0e61eabe06fe32be66b16559537124b7a",
"4f1355c91100d12f9e7202f91b245df0c110867c",
"4f6eadeb08b9d0d1e8b1b3eac8a34940adf29a2d",
"4f9339df943c53117a5fc8e86e2f38716ff3a668",
"4fc3874b118752e40de556b1c3e7b4a9f1737d00",
"4ff1dd0992dd6baafdb5e166be6f9f23b59bdf87",
"5018a35e0b7e2eec7ce5050baf9c7343f3f74164",
"50298f44a45eda3a29dae82dbe911b5aa176ac07",
"502acd164fb115768d723144da2e7bb5a24891bb",
"50330c02bd4fd95c9db1fcf2f97f4218e42b7226",
"5052bf355d9f8c52446561a39733a8767bf31e37",
"6f2cd729ae42988c1dd43588d3a6661ba48ad7a0",
"6f4e2c42d9138bfbf3e0f908f1308828cc6f2178",
"6f6a17db05a83620cef4572761831c20a70ba9b9",
"6faad60901e36538634f0d8b8ff3f21f83503c71",
"6fc72e46de3df0c3842dab302bbacf697a63abab",
"6fdccd49f442a7204399ca9b418f017322dbded8",
"6fe7568fc3861c334cb008fd85d57d9647249ef5",
"700f55d91d7b55665594676a4bada1f1457a0598",
"702bd70595a7b19afc48a1f784a6505be68469d4",
"7033f9ee0e52b08cb5679cd49b7b7999eaf9eaf8",
"70957110ce446c4e250f865760fb3da513cdcc92",
"8ec696a4734f16479d091bc70574d23dd9fe7443",
"8ed341c55ed4d6f4cdc8bf4f0ca18a08c93f6962",
"8edc2805f1f11b63e44bf81f4557f8b473612b69",
"8ef9060a954118a698fc10e20acdc430566a100f",
"8f0c4b543f4bb6eb1518ecfc3d4699e43108d393",
"8fac94df3035405c2e60b3799153ce7c428af6b9",
"904c0ac12b23548de524adae712241b423d765a3",
"90bbaa9a809c3a768d873a9cc7d52b4f3bf3d1b9",
"90d4d2f0fc362beabbbf76b4ffda0828229c198d",
"90f9ff6755330b685feff6c3d81782ee3592ab04",
"91822c50ebe4f9bf5bbb8308ecf9f6557062775c",
"91d973263a55708fa8255867b3202d81ef9c2868",
"af292c99c6148d772af3315a1c74e83330e7ead7",
"af3b99d5be330dbbce0b9250c3a5fb05911908cc",
"af55d0cdeb280af2db8697e5afa506e081012719",
"af795e498d411142ddb073e8ca2c5447c3295a4c",
"afadc73a392f8cc8e2cc77dd62a7433dd3bafa8c",
"affd84ed8ec7ce67612fe3c12a80f8164b101f6a",
"b0941f9c70ffe67f0387a827b338e64ecf3190f0",
"b0a3077f9ef6e093f8d9869bdb0c07095bd722cb",
"b0a8568a7614806378a54db5706ee3b06ae58693",
"b0fb7372f242233d1d35ce7d8e74d3990cbc5841",
"b10489944b9ead17427551759d180d10203e06ba",
"b196a807b323f2748ffc6b1d42cd0812d04c9a40",
"b1bb1d888f0c5e19278536d49fa77db035fac7ae"
};
BEGIN_TEST(readpacked_test)
unsigned int i;
git_odb *db;
must_pass(git_odb_open(&db, ODB_FOLDER));
for (i = 0; i < ARRAY_SIZE(packed_objects); ++i) {
git_oid id;
git_obj obj;
must_pass(git_oid_mkstr(&id, packed_objects[i]));
must_be_true(git_odb_exists(db, &id) == 1);
must_pass(git_odb__read_packed(&obj, db, &id));
git_obj_close(&obj);
}
git_odb_close(db);
END_TEST
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment