indexer.c 9.23 KB
Newer Older
1
/*
Vicent Marti committed
2
 * Copyright (C) 2009-2011 the libgit2 contributors
3
 *
Vicent Marti committed
4 5
 * This file is part of libgit2, distributed under the GNU GPL v2 with
 * a Linking Exception. For full terms see the included COPYING file.
6 7
 */

Carlos Martín Nieto committed
8
#include "git2/indexer.h"
9
#include "git2/object.h"
10
#include "git2/zlib.h"
11
#include "git2/oid.h"
Carlos Martín Nieto committed
12

13 14
#include "common.h"
#include "pack.h"
Carlos Martín Nieto committed
15
#include "mwindow.h"
16
#include "posix.h"
17 18 19 20 21
#include "pack.h"
#include "filebuf.h"
#include "sha1.h"

#define UINT31_MAX (0x7FFFFFFF)
22

23
struct entry {
24
	git_oid oid;
25 26 27 28 29
	uint32_t crc;
	uint32_t offset;
	uint64_t offset_long;
};

30
struct git_indexer {
31
	struct git_pack_file *pack;
Carlos Martín Nieto committed
32
	struct stat st;
33
	struct git_pack_header hdr;
34 35 36 37 38
	size_t nr_objects;
	git_vector objects;
	git_filebuf file;
	unsigned int fanout[256];
	git_oid hash;
39
};
Carlos Martín Nieto committed
40

41 42 43 44 45
const git_oid *git_indexer_hash(git_indexer *idx)
{
	return &idx->hash;
}

46
static int parse_header(git_indexer *idx)
47 48 49 50
{
	int error;

	/* Verify we recognize this pack file format. */
51 52
	if ((error = p_read(idx->pack->mwf.fd, &idx->hdr, sizeof(idx->hdr))) < GIT_SUCCESS)
		return git__rethrow(error, "Failed to read in pack header");
53

54
	if (idx->hdr.hdr_signature != ntohl(PACK_SIGNATURE))
55
		return git__throw(GIT_EOBJCORRUPTED, "Wrong pack signature");
56

57 58
	if (!pack_version_ok(idx->hdr.hdr_version))
		return git__throw(GIT_EOBJCORRUPTED, "Wrong pack version");
59

Carlos Martín Nieto committed
60

61 62 63
	return GIT_SUCCESS;
}

64
static int objects_cmp(const void *a, const void *b)
65 66 67 68 69 70 71
{
	const struct entry *entrya = a;
	const struct entry *entryb = b;

	return git_oid_cmp(&entrya->oid, &entryb->oid);
}

72 73 74 75 76 77 78 79 80
static int cache_cmp(const void *a, const void *b)
{
	const struct git_pack_entry *ea = a;
	const struct git_pack_entry *eb = b;

	return git_oid_cmp(&ea->sha1, &eb->sha1);
}


81
int git_indexer_new(git_indexer **out, const char *packname)
82
{
83
	git_indexer *idx;
84
	size_t namelen;
85 86
	int ret, error;

87 88
	assert(out && packname);

89 90 91
	if (git_path_root(packname) < 0)
		return git__throw(GIT_EINVALIDPATH, "Path is not absolute");

92
	idx = git__malloc(sizeof(git_indexer));
93 94 95 96 97 98
	if (idx == NULL)
		return GIT_ENOMEM;

	memset(idx, 0x0, sizeof(*idx));

	namelen = strlen(packname);
99
	idx->pack = git__malloc(sizeof(struct git_pack_file) + namelen + 1);
100 101
	if (idx->pack == NULL) {
		error = GIT_ENOMEM;
102
		goto cleanup;
103
	}
104

105
	memset(idx->pack, 0x0, sizeof(struct git_pack_file));
106
	memcpy(idx->pack->pack_name, packname, namelen + 1);
107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123

	ret = p_stat(packname, &idx->st);
	if (ret < 0) {
		if (errno == ENOENT)
			error = git__throw(GIT_ENOTFOUND, "Failed to stat packfile. File not found");
		else
			error = git__throw(GIT_EOSERR, "Failed to stat packfile.");

		goto cleanup;
	}

	ret = p_open(idx->pack->pack_name, O_RDONLY);
	if (ret < 0) {
		error = git__throw(GIT_EOSERR, "Failed to open packfile");
		goto cleanup;
	}

124
	idx->pack->mwf.fd = ret;
125
	idx->pack->mwf.size = (git_off_t)idx->st.st_size;
126 127 128 129 130 131 132

	error = parse_header(idx);
	if (error < GIT_SUCCESS) {
		error = git__rethrow(error, "Failed to parse packfile header");
		goto cleanup;
	}

133 134
	idx->nr_objects = ntohl(idx->hdr.hdr_entries);

135 136 137 138 139
	error = git_vector_init(&idx->pack->cache, idx->nr_objects, cache_cmp);
	if (error < GIT_SUCCESS)
		goto cleanup;

	idx->pack->has_cache = 1;
140
	error = git_vector_init(&idx->objects, idx->nr_objects, objects_cmp);
141
	if (error < GIT_SUCCESS)
142 143
		goto cleanup;

144 145 146 147 148
	*out = idx;

	return GIT_SUCCESS;

cleanup:
149
	git_indexer_free(idx);
150 151 152 153

	return error;
}

154
static void index_path(char *path, git_indexer *idx)
Carlos Martín Nieto committed
155
{
156
	char *ptr;
157
	const char prefix[] = "pack-", suffix[] = ".idx";
158 159 160

	ptr = strrchr(path, '/') + 1;

161 162
	memcpy(ptr, prefix, strlen(prefix));
	ptr += strlen(prefix);
163 164
	git_oid_fmt(ptr, &idx->hash);
	ptr += GIT_OID_HEXSZ;
165
	memcpy(ptr, suffix, strlen(suffix) + 1);
166 167
}

168
int git_indexer_write(git_indexer *idx)
169 170
{
	git_mwindow *w = NULL;
171 172
	int error;
	size_t namelen;
173
	unsigned int i, long_offsets = 0, left;
174 175 176 177 178 179 180 181 182 183 184
	struct git_pack_idx_header hdr;
	char filename[GIT_PATH_MAX];
	struct entry *entry;
	void *packfile_hash;
	git_oid file_hash;
	SHA_CTX ctx;

	git_vector_sort(&idx->objects);

	namelen = strlen(idx->pack->pack_name);
	memcpy(filename, idx->pack->pack_name, namelen);
185
	memcpy(filename + namelen - strlen("pack"), "idx", strlen("idx") + 1);
186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250

	error = git_filebuf_open(&idx->file, filename, GIT_FILEBUF_HASH_CONTENTS);

	/* Write out the header */
	hdr.idx_signature = htonl(PACK_IDX_SIGNATURE);
	hdr.idx_version = htonl(2);
	error = git_filebuf_write(&idx->file, &hdr, sizeof(hdr));

	/* Write out the fanout table */
	for (i = 0; i < 256; ++i) {
		uint32_t n = htonl(idx->fanout[i]);
		error = git_filebuf_write(&idx->file, &n, sizeof(n));
		if (error < GIT_SUCCESS)
			goto cleanup;
	}

	/* Write out the object names (SHA-1 hashes) */
	SHA1_Init(&ctx);
	git_vector_foreach(&idx->objects, i, entry) {
		error = git_filebuf_write(&idx->file, &entry->oid, sizeof(git_oid));
		SHA1_Update(&ctx, &entry->oid, GIT_OID_RAWSZ);
		if (error < GIT_SUCCESS)
			goto cleanup;
	}
	SHA1_Final(idx->hash.id, &ctx);

	/* Write out the CRC32 values */
	git_vector_foreach(&idx->objects, i, entry) {
		error = git_filebuf_write(&idx->file, &entry->crc, sizeof(uint32_t));
		if (error < GIT_SUCCESS)
			goto cleanup;
	}

	/* Write out the offsets */
	git_vector_foreach(&idx->objects, i, entry) {
		uint32_t n;

		if (entry->offset == UINT32_MAX)
			n = htonl(0x80000000 | long_offsets++);
		else
			n = htonl(entry->offset);

		error = git_filebuf_write(&idx->file, &n, sizeof(uint32_t));
		if (error < GIT_SUCCESS)
			goto cleanup;
	}

	/* Write out the long offsets */
	git_vector_foreach(&idx->objects, i, entry) {
		uint32_t split[2];

		if (entry->offset != UINT32_MAX)
			continue;

		split[0] = htonl(entry->offset_long >> 32);
		split[1] = htonl(entry->offset_long & 0xffffffff);

		error = git_filebuf_write(&idx->file, &split, sizeof(uint32_t) * 2);
		if (error < GIT_SUCCESS)
			goto cleanup;
	}

	/* Write out the packfile trailer */

	packfile_hash = git_mwindow_open(&idx->pack->mwf, &w, idx->st.st_size - GIT_OID_RAWSZ, GIT_OID_RAWSZ, &left);
251
	git_mwindow_close(&w);
252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274
	if (packfile_hash == NULL) {
		error = git__rethrow(GIT_ENOMEM, "Failed to open window to packfile hash");
		goto cleanup;
	}

	memcpy(&file_hash, packfile_hash, GIT_OID_RAWSZ);

	git_mwindow_close(&w);

	error = git_filebuf_write(&idx->file, &file_hash, sizeof(git_oid));

	/* Write out the index sha */
	error = git_filebuf_hash(&file_hash, &idx->file);
	if (error < GIT_SUCCESS)
		goto cleanup;

	error = git_filebuf_write(&idx->file, &file_hash, sizeof(git_oid));
	if (error < GIT_SUCCESS)
		goto cleanup;

	/* Figure out what the final name should be */
	index_path(filename, idx);
	/* Commit file */
275
	error = git_filebuf_commit_at(&idx->file, filename, GIT_PACK_FILE_MODE);
276 277

cleanup:
278
	git_mwindow_free_all(&idx->pack->mwf);
279 280 281 282 283 284 285 286 287 288
	if (error < GIT_SUCCESS)
		git_filebuf_cleanup(&idx->file);

	return error;
}

int git_indexer_run(git_indexer *idx, git_indexer_stats *stats)
{
	git_mwindow_file *mwf;
	off_t off = sizeof(struct git_pack_header);
Carlos Martín Nieto committed
289
	int error;
290 291
	struct entry *entry;
	unsigned int left, processed;
Carlos Martín Nieto committed
292

293
	assert(idx && stats);
294

295
	mwf = &idx->pack->mwf;
Carlos Martín Nieto committed
296 297 298 299
	error = git_mwindow_file_register(mwf);
	if (error < GIT_SUCCESS)
		return git__rethrow(error, "Failed to register mwindow file");

300 301
	stats->total = idx->nr_objects;
	stats->processed = processed = 0;
Carlos Martín Nieto committed
302

303
	while (processed < idx->nr_objects) {
304 305
		git_rawobj obj;
		git_oid oid;
306
		struct git_pack_entry *pentry;
307
		git_mwindow *w = NULL;
308
		int i;
309 310 311
		off_t entry_start = off;
		void *packed;
		size_t entry_size;
312

313 314
		entry = git__malloc(sizeof(struct entry));
		memset(entry, 0x0, sizeof(struct entry));
315 316

		if (off > UINT31_MAX) {
317 318
			entry->offset = UINT32_MAX;
			entry->offset_long = off;
319
		} else {
320
			entry->offset = off;
321 322 323 324 325 326 327 328
		}

		error = git_packfile_unpack(&obj, idx->pack, &off);
		if (error < GIT_SUCCESS) {
			error = git__rethrow(error, "Failed to unpack object");
			goto cleanup;
		}

329
		/* FIXME: Parse the object instead of hashing it */
330
		error = git_odb__hash_obj(&oid, &obj);
331 332
		if (error < GIT_SUCCESS) {
			error = git__rethrow(error, "Failed to hash object");
333 334 335
			goto cleanup;
		}

336 337 338 339 340 341 342 343 344 345 346
		pentry = git__malloc(sizeof(struct git_pack_entry));
		if (pentry == NULL) {
			error = GIT_ENOMEM;
			goto cleanup;
		}
		git_oid_cpy(&pentry->sha1, &oid);
		pentry->offset = entry_start;
		error = git_vector_insert(&idx->pack->cache, pentry);
		if (error < GIT_SUCCESS)
			goto cleanup;

347 348 349 350 351 352 353 354 355 356 357
		git_oid_cpy(&entry->oid, &oid);
		entry->crc = crc32(0L, Z_NULL, 0);

		entry_size = off - entry_start;
		packed = git_mwindow_open(mwf, &w, entry_start, entry_size, &left);
		if (packed == NULL) {
			error = git__rethrow(error, "Failed to open window to read packed data");
			goto cleanup;
		}
		entry->crc = htonl(crc32(entry->crc, packed, entry_size));
		git_mwindow_close(&w);
358

359
		/* Add the object to the list */
360 361 362 363 364
		error = git_vector_insert(&idx->objects, entry);
		if (error < GIT_SUCCESS) {
			error = git__rethrow(error, "Failed to add entry to list");
			goto cleanup;
		}
365 366

		for (i = oid.id[0]; i < 256; ++i) {
367
			idx->fanout[i]++;
368 369
		}

370
		git__free(obj.data);
371

372
		stats->processed = ++processed;
373 374 375 376 377
	}

cleanup:
	git_mwindow_free_all(mwf);

Carlos Martín Nieto committed
378
	return error;
379

Carlos Martín Nieto committed
380 381
}

382
void git_indexer_free(git_indexer *idx)
383
{
384 385
	unsigned int i;
	struct entry *e;
386
	struct git_pack_entry *pe;
387

388 389 390
	if (idx == NULL)
		return;

391
	p_close(idx->pack->mwf.fd);
392
	git_vector_foreach(&idx->objects, i, e)
393
		git__free(e);
394
	git_vector_free(&idx->objects);
395
	git_vector_foreach(&idx->pack->cache, i, pe)
396
		git__free(pe);
397
	git_vector_free(&idx->pack->cache);
398 399
	git__free(idx->pack);
	git__free(idx);
400
}
401