indexer.c 27.2 KB
Newer Older
1
/*
Edward Thomson committed
2
 * Copyright (C) the libgit2 contributors. All rights reserved.
3
 *
Vicent Marti committed
4 5
 * This file is part of libgit2, distributed under the GNU GPL v2 with
 * a Linking Exception. For full terms see the included COPYING file.
6 7
 */

8 9
#include "indexer.h"

Carlos Martín Nieto committed
10
#include "git2/indexer.h"
11
#include "git2/object.h"
Carlos Martín Nieto committed
12

13
#include "pack.h"
Carlos Martín Nieto committed
14
#include "mwindow.h"
15
#include "posix.h"
16 17
#include "pack.h"
#include "filebuf.h"
18
#include "oid.h"
19
#include "oidmap.h"
20
#include "zstream.h"
21
#include "object.h"
22

23 24
extern git_mutex git__mwindow_mutex;

25
#define UINT31_MAX (0x7FFFFFFF)
26

27
struct entry {
28
	git_oid oid;
29 30 31 32 33
	uint32_t crc;
	uint32_t offset;
	uint64_t offset_long;
};

34
struct git_indexer {
35
	unsigned int parsed_header :1,
36
		pack_committed :1,
37
		have_stream :1,
38 39
		have_delta :1,
		do_fsync :1;
40
	struct git_pack_header hdr;
41
	struct git_pack_file *pack;
42
	unsigned int mode;
43
	git_off_t off;
44 45
	git_off_t entry_start;
	git_packfile_stream stream;
46 47 48 49
	size_t nr_objects;
	git_vector objects;
	git_vector deltas;
	unsigned int fanout[256];
50
	git_hash_ctx hash_ctx;
51
	git_oid hash;
52
	git_transfer_progress_cb progress_cb;
53
	void *progress_payload;
54
	char objbuf[8*1024];
55

56 57 58
	/* Needed to look up objects which we want to inject to fix a thin pack */
	git_odb *odb;

59 60
	/* Fields for calculating the packfile trailer (hash of everything before it) */
	char inbuf[GIT_OID_RAWSZ];
61
	size_t inbuf_len;
62
	git_hash_ctx trailer;
63 64 65
};

struct delta_info {
66
	git_off_t delta_off;
67 68
};

69
const git_oid *git_indexer_hash(const git_indexer *idx)
70 71 72 73
{
	return &idx->hash;
}

74
static int parse_header(struct git_pack_header *hdr, struct git_pack_file *pack)
75 76
{
	int error;
77
	git_map map;
78

79
	if ((error = p_mmap(&map, sizeof(*hdr), GIT_PROT_READ, GIT_MAP_SHARED, pack->mwf.fd, 0)) < 0)
80
		return error;
81

82 83 84 85
	memcpy(hdr, map.data, sizeof(*hdr));
	p_munmap(&map);

	/* Verify we recognize this pack file format. */
86
	if (hdr->hdr_signature != ntohl(PACK_SIGNATURE)) {
87
		giterr_set(GITERR_INDEXER, "wrong pack signature");
88 89
		return -1;
	}
90

91
	if (!pack_version_ok(hdr->hdr_version)) {
92
		giterr_set(GITERR_INDEXER, "wrong pack version");
93 94
		return -1;
	}
Carlos Martín Nieto committed
95

96
	return 0;
97 98
}

99
static int objects_cmp(const void *a, const void *b)
100 101 102 103
{
	const struct entry *entrya = a;
	const struct entry *entryb = b;

104
	return git_oid__cmp(&entrya->oid, &entryb->oid);
105 106
}

107 108
int git_indexer_new(
		git_indexer **out,
109
		const char *prefix,
110
		unsigned int mode,
111
		git_odb *odb,
112
		git_transfer_progress_cb progress_cb,
113
		void *progress_payload)
114
{
115
	git_indexer *idx;
116
	git_buf path = GIT_BUF_INIT, tmp_path = GIT_BUF_INIT;
117
	static const char suff[] = "/pack";
118
	int error, fd = -1;
119

120
	idx = git__calloc(1, sizeof(git_indexer));
121
	GITERR_CHECK_ALLOC(idx);
122
	idx->odb = odb;
123 124
	idx->progress_cb = progress_cb;
	idx->progress_payload = progress_payload;
125
	idx->mode = mode ? mode : GIT_PACK_FILE_MODE;
126
	git_hash_ctx_init(&idx->hash_ctx);
127
	git_hash_ctx_init(&idx->trailer);
128

129
	if (git_repository__fsync_gitdir)
130 131
		idx->do_fsync = 1;

132 133 134 135
	error = git_buf_joinpath(&path, prefix, suff);
	if (error < 0)
		goto cleanup;

136
	fd = git_futils_mktmp(&tmp_path, git_buf_cstr(&path), idx->mode);
137
	git_buf_dispose(&path);
138 139 140 141
	if (fd < 0)
		goto cleanup;

	error = git_packfile_alloc(&idx->pack, git_buf_cstr(&tmp_path));
142
	git_buf_dispose(&tmp_path);
143

144 145 146
	if (error < 0)
		goto cleanup;

147 148 149 150
	idx->pack->mwf.fd = fd;
	if ((error = git_mwindow_file_register(&idx->pack->mwf)) < 0)
		goto cleanup;

151 152 153 154
	*out = idx;
	return 0;

cleanup:
155 156 157
	if (fd != -1)
		p_close(fd);

lhchavez committed
158 159
	if (git_buf_len(&tmp_path) > 0)
		p_unlink(git_buf_cstr(&tmp_path));
160 161

	if (idx->pack != NULL)
lhchavez committed
162
		p_unlink(idx->pack->pack_name);
163

164 165
	git_buf_dispose(&path);
	git_buf_dispose(&tmp_path);
166 167 168 169
	git__free(idx);
	return -1;
}

170 171 172 173 174
void git_indexer__set_fsync(git_indexer *idx, int do_fsync)
{
	idx->do_fsync = !!do_fsync;
}

175
/* Try to store the delta so we can try to resolve it later */
176
static int store_delta(git_indexer *idx)
177
{
178 179
	struct delta_info *delta;

180 181
	delta = git__calloc(1, sizeof(struct delta_info));
	GITERR_CHECK_ALLOC(delta);
182
	delta->delta_off = idx->entry_start;
183

184
	if (git_vector_insert(&idx->deltas, delta) < 0)
185 186 187 188 189
		return -1;

	return 0;
}

190
static int hash_header(git_hash_ctx *ctx, git_off_t len, git_otype type)
191 192 193
{
	char buffer[64];
	size_t hdrlen;
194 195 196 197 198
	int error;

	if ((error = git_odb__format_object_header(&hdrlen,
		buffer, sizeof(buffer), (size_t)len, type)) < 0)
		return error;
199

200
	return git_hash_update(ctx, buffer, hdrlen);
201 202
}

203
static int hash_object_stream(git_indexer*idx, git_packfile_stream *stream)
204 205 206
{
	ssize_t read;

207
	assert(idx && stream);
208 209

	do {
210
		if ((read = git_packfile_stream_read(stream, idx->objbuf, sizeof(idx->objbuf))) < 0)
211 212
			break;

213
		git_hash_update(&idx->hash_ctx, idx->objbuf, read);
214 215 216 217 218 219 220 221
	} while (read > 0);

	if (read < 0)
		return (int)read;

	return 0;
}

222
/* In order to create the packfile stream, we need to skip over the delta base description */
223
static int advance_delta_offset(git_indexer *idx, git_otype type)
224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241
{
	git_mwindow *w = NULL;

	assert(type == GIT_OBJ_REF_DELTA || type == GIT_OBJ_OFS_DELTA);

	if (type == GIT_OBJ_REF_DELTA) {
		idx->off += GIT_OID_RAWSZ;
	} else {
		git_off_t base_off = get_delta_base(idx->pack, &w, &idx->off, type, idx->entry_start);
		git_mwindow_close(&w);
		if (base_off < 0)
			return (int)base_off;
	}

	return 0;
}

/* Read from the stream and discard any output */
242
static int read_object_stream(git_indexer *idx, git_packfile_stream *stream)
243 244 245 246 247 248
{
	ssize_t read;

	assert(stream);

	do {
249
		read = git_packfile_stream_read(stream, idx->objbuf, sizeof(idx->objbuf));
250 251 252 253 254 255 256 257
	} while (read > 0);

	if (read < 0)
		return (int)read;

	return 0;
}

258 259 260 261 262 263 264 265 266
static int crc_object(uint32_t *crc_out, git_mwindow_file *mwf, git_off_t start, git_off_t size)
{
	void *ptr;
	uint32_t crc;
	unsigned int left, len;
	git_mwindow *w = NULL;

	crc = crc32(0L, Z_NULL, 0);
	while (size) {
267
		ptr = git_mwindow_open(mwf, &w, start, (size_t)size, &left);
268 269 270
		if (ptr == NULL)
			return -1;

271
		len = min(left, (unsigned int)size);
272 273 274 275 276 277 278 279 280 281
		crc = crc32(crc, ptr, len);
		size -= len;
		start += len;
		git_mwindow_close(&w);
	}

	*crc_out = htonl(crc);
	return 0;
}

282
static int store_object(git_indexer *idx)
283
{
284 285
	int i, error;
	khiter_t k;
286 287 288 289
	git_oid oid;
	struct entry *entry;
	git_off_t entry_size;
	struct git_pack_entry *pentry;
290
	git_off_t entry_start = idx->entry_start;
291 292 293 294

	entry = git__calloc(1, sizeof(*entry));
	GITERR_CHECK_ALLOC(entry);

Linquize committed
295
	pentry = git__calloc(1, sizeof(struct git_pack_entry));
296 297
	GITERR_CHECK_ALLOC(pentry);

298
	git_hash_final(&oid, &idx->hash_ctx);
299 300 301 302 303 304 305 306 307 308
	entry_size = idx->off - entry_start;
	if (entry_start > UINT31_MAX) {
		entry->offset = UINT32_MAX;
		entry->offset_long = entry_start;
	} else {
		entry->offset = (uint32_t)entry_start;
	}

	git_oid_cpy(&pentry->sha1, &oid);
	pentry->offset = entry_start;
309

310
	k = git_oidmap_put(idx->pack->idx_cache, &pentry->sha1, &error);
311
	if (error == -1) {
312
		git__free(pentry);
313
		giterr_set_oom();
314
		goto on_error;
315
	}
316

317 318 319 320 321 322 323
	if (error == 0) {
		giterr_set(GITERR_INDEXER, "duplicate object %s found in pack", git_oid_tostr_s(&pentry->sha1));
		git__free(pentry);
		goto on_error;
	}


324
	git_oidmap_set_value_at(idx->pack->idx_cache, k, pentry);
325

326 327
	git_oid_cpy(&entry->oid, &oid);

328
	if (crc_object(&entry->crc, &idx->pack->mwf, entry_start, entry_size) < 0)
329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346
		goto on_error;

	/* Add the object to the list */
	if (git_vector_insert(&idx->objects, entry) < 0)
		goto on_error;

	for (i = oid.id[0]; i < 256; ++i) {
		idx->fanout[i]++;
	}

	return 0;

on_error:
	git__free(entry);

	return -1;
}

347 348
GIT_INLINE(bool) has_entry(git_indexer *idx, git_oid *id)
{
349
	return git_oidmap_exists(idx->pack->idx_cache, id);
350 351
}

352
static int save_entry(git_indexer *idx, struct entry *entry, struct git_pack_entry *pentry, git_off_t entry_start)
353
{
354 355
	int i, error;
	khiter_t k;
356 357 358 359 360 361 362 363

	if (entry_start > UINT31_MAX) {
		entry->offset = UINT32_MAX;
		entry->offset_long = entry_start;
	} else {
		entry->offset = (uint32_t)entry_start;
	}

364
	pentry->offset = entry_start;
365
	k = git_oidmap_put(idx->pack->idx_cache, &pentry->sha1, &error);
366 367 368

	if (error <= 0) {
		giterr_set(GITERR_INDEXER, "cannot insert object into pack");
369
		return -1;
370
	}
371

372
	git_oidmap_set_value_at(idx->pack->idx_cache, k, pentry);
373 374 375 376 377 378 379 380 381 382 383 384

	/* Add the object to the list */
	if (git_vector_insert(&idx->objects, entry) < 0)
		return -1;

	for (i = entry->oid.id[0]; i < 256; ++i) {
		idx->fanout[i]++;
	}

	return 0;
}

385
static int hash_and_save(git_indexer *idx, git_rawobj *obj, git_off_t entry_start)
386 387 388 389
{
	git_oid oid;
	size_t entry_size;
	struct entry *entry;
390
	struct git_pack_entry *pentry = NULL;
391 392 393 394

	entry = git__calloc(1, sizeof(*entry));
	GITERR_CHECK_ALLOC(entry);

395
	if (git_odb__hashobj(&oid, obj) < 0) {
396
		giterr_set(GITERR_INDEXER, "failed to hash object");
397
		goto on_error;
398 399
	}

Linquize committed
400
	pentry = git__calloc(1, sizeof(struct git_pack_entry));
401 402 403 404 405 406 407
	GITERR_CHECK_ALLOC(pentry);

	git_oid_cpy(&pentry->sha1, &oid);
	git_oid_cpy(&entry->oid, &oid);
	entry->crc = crc32(0L, Z_NULL, 0);

	entry_size = (size_t)(idx->off - entry_start);
408
	if (crc_object(&entry->crc, &idx->pack->mwf, entry_start, entry_size) < 0)
409 410
		goto on_error;

411
	return save_entry(idx, entry, pentry, entry_start);
412

413
on_error:
414
	git__free(pentry);
415 416
	git__free(entry);
	git__free(obj->data);
417 418
	return -1;
}
419

420
static int do_progress_callback(git_indexer *idx, git_transfer_progress *stats)
421
{
422
	if (idx->progress_cb)
423
		return giterr_set_after_callback_function(
424 425
			idx->progress_cb(stats, idx->progress_payload),
			"indexer progress");
426
	return 0;
427 428
}

429
/* Hash everything but the last 20B of input */
430
static void hash_partially(git_indexer *idx, const uint8_t *data, size_t size)
431
{
432
	size_t to_expell, to_keep;
433 434 435 436 437

	if (size == 0)
		return;

	/* Easy case, dump the buffer and the data minus the last 20 bytes */
438
	if (size >= GIT_OID_RAWSZ) {
439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455
		git_hash_update(&idx->trailer, idx->inbuf, idx->inbuf_len);
		git_hash_update(&idx->trailer, data, size - GIT_OID_RAWSZ);

		data += size - GIT_OID_RAWSZ;
		memcpy(idx->inbuf, data, GIT_OID_RAWSZ);
		idx->inbuf_len = GIT_OID_RAWSZ;
		return;
	}

	/* We can just append */
	if (idx->inbuf_len + size <= GIT_OID_RAWSZ) {
		memcpy(idx->inbuf + idx->inbuf_len, data, size);
		idx->inbuf_len += size;
		return;
	}

	/* We need to partially drain the buffer and then append */
456 457
	to_keep   = GIT_OID_RAWSZ - size;
	to_expell = idx->inbuf_len - to_keep;
458 459 460 461 462 463 464 465

	git_hash_update(&idx->trailer, idx->inbuf, to_expell);

	memmove(idx->inbuf, idx->inbuf + to_expell, to_keep);
	memcpy(idx->inbuf + to_keep, data, size);
	idx->inbuf_len += size - to_expell;
}

466 467 468
static int write_at(git_indexer *idx, const void *data, git_off_t offset, size_t size)
{
	git_file fd = idx->pack->mwf.fd;
469
	size_t mmap_alignment;
470 471
	size_t page_offset;
	git_off_t page_start;
472
	unsigned char *map_data;
473 474 475
	git_map map;
	int error;

476 477
	assert(data && size);

478
	if ((error = git__mmap_alignment(&mmap_alignment)) < 0)
479 480
		return error;

481 482
	/* the offset needs to be at the mmap boundary for the platform */
	page_offset = offset % mmap_alignment;
483
	page_start = offset - page_offset;
484 485 486 487

	if ((error = p_mmap(&map, page_offset + size, GIT_PROT_WRITE, GIT_MAP_SHARED, fd, page_start)) < 0)
		return error;

488 489
	map_data = (unsigned char *)map.data;
	memcpy(map_data + page_offset, data, size);
490 491 492 493 494 495 496
	p_munmap(&map);

	return 0;
}

static int append_to_pack(git_indexer *idx, const void *data, size_t size)
{
497 498 499 500
	git_off_t new_size;
	size_t mmap_alignment;
	size_t page_offset;
	git_off_t page_start;
501
	git_off_t current_size = idx->pack->mwf.size;
502
	int fd = idx->pack->mwf.fd;
503
	int error;
504

505 506 507
	if (!size)
		return 0;

508 509 510 511 512 513 514 515 516 517 518 519
	if ((error = git__mmap_alignment(&mmap_alignment)) < 0)
		return error;

	/* Write a single byte to force the file system to allocate space now or
	 * report an error, since we can't report errors when writing using mmap.
	 * Round the size up to the nearest page so that we only need to perform file
	 * I/O when we add a page, instead of whenever we write even a single byte. */
	new_size = current_size + size;
	page_offset = new_size % mmap_alignment;
	page_start = new_size - page_offset;

	if (p_lseek(fd, page_start + mmap_alignment - 1, SEEK_SET) < 0 ||
520 521
	    p_write(idx->pack->mwf.fd, data, 1) < 0) {
		giterr_set(GITERR_OS, "cannot extend packfile '%s'", idx->pack->pack_name);
522 523 524 525 526 527
		return -1;
	}

	return write_at(idx, data, idx->pack->mwf.size, size);
}

528
int git_indexer_append(git_indexer *idx, const void *data, size_t size, git_transfer_progress *stats)
529
{
530
	int error = -1;
531
	size_t processed;
532
	struct git_pack_header *hdr = &idx->hdr;
533
	git_mwindow_file *mwf = &idx->pack->mwf;
534

535 536
	assert(idx && data && stats);

537
	processed = stats->indexed_objects;
538

539
	if ((error = append_to_pack(idx, data, size)) < 0)
540
		return error;
541

nulltoken committed
542
	hash_partially(idx, data, (int)size);
543

544
	/* Make sure we set the new size of the pack */
545
	idx->pack->mwf.size += size;
546 547

	if (!idx->parsed_header) {
548 549
		unsigned int total_objects;

550
		if ((unsigned)idx->pack->mwf.size < sizeof(struct git_pack_header))
551 552
			return 0;

553 554
		if ((error = parse_header(&idx->hdr, idx->pack)) < 0)
			return error;
555 556

		idx->parsed_header = 1;
557
		idx->nr_objects = ntohl(hdr->hdr_entries);
558 559 560 561
		idx->off = sizeof(struct git_pack_header);

		/* for now, limit to 2^32 objects */
		assert(idx->nr_objects == (size_t)((unsigned int)idx->nr_objects));
562 563 564 565
		if (idx->nr_objects == (size_t)((unsigned int)idx->nr_objects))
			total_objects = (unsigned int)idx->nr_objects;
		else
			total_objects = UINT_MAX;
566

567 568
		idx->pack->idx_cache = git_oidmap_alloc();
		GITERR_CHECK_ALLOC(idx->pack->idx_cache);
569 570

		idx->pack->has_cache = 1;
571
		if (git_vector_init(&idx->objects, total_objects, objects_cmp) < 0)
572 573
			return -1;

574
		if (git_vector_init(&idx->deltas, total_objects / 2, NULL) < 0)
575 576
			return -1;

577
		stats->received_objects = 0;
578
		stats->local_objects = 0;
579 580
		stats->total_deltas = 0;
		stats->indexed_deltas = 0;
581
		processed = stats->indexed_objects = 0;
582
		stats->total_objects = total_objects;
583

584
		if ((error = do_progress_callback(idx, stats)) != 0)
585
			return error;
586 587 588 589 590 591
	}

	/* Now that we have data in the pack, let's try to parse it */

	/* As the file grows any windows we try to use will be out of date */
	git_mwindow_free_all(mwf);
592

593
	while (processed < idx->nr_objects) {
594
		git_packfile_stream *stream = &idx->stream;
595
		git_off_t entry_start = idx->off;
596 597 598
		size_t entry_size;
		git_otype type;
		git_mwindow *w = NULL;
599

600 601 602
		if (idx->pack->mwf.size <= idx->off + 20)
			return 0;

603 604
		if (!idx->have_stream) {
			error = git_packfile_unpack_header(&entry_size, &type, mwf, &w, &idx->off);
605 606
			if (error == GIT_EBUFS) {
				idx->off = entry_start;
607
				return 0;
608
			}
609
			if (error < 0)
610
				goto on_error;
611 612 613

			git_mwindow_close(&w);
			idx->entry_start = entry_start;
614
			git_hash_init(&idx->hash_ctx);
615 616

			if (type == GIT_OBJ_REF_DELTA || type == GIT_OBJ_OFS_DELTA) {
617
				error = advance_delta_offset(idx, type);
618 619 620 621 622
				if (error == GIT_EBUFS) {
					idx->off = entry_start;
					return 0;
				}
				if (error < 0)
623
					goto on_error;
624

625 626 627
				idx->have_delta = 1;
			} else {
				idx->have_delta = 0;
628 629 630 631

				error = hash_header(&idx->hash_ctx, entry_size, type);
				if (error < 0)
					goto on_error;
632
			}
633

634
			idx->have_stream = 1;
635

636 637 638
			error = git_packfile_stream_open(stream, idx->pack, idx->off);
			if (error < 0)
				goto on_error;
639 640 641
		}

		if (idx->have_delta) {
642
			error = read_object_stream(idx, stream);
643
		} else {
644
			error = hash_object_stream(idx, stream);
645 646
		}

647
		idx->off = stream->curpos;
648
		if (error == GIT_EBUFS)
649
			return 0;
650 651 652

		/* We want to free the stream reasorces no matter what here */
		idx->have_stream = 0;
653
		git_packfile_stream_dispose(stream);
654

655
		if (error < 0)
656 657
			goto on_error;

658 659 660 661 662 663 664
		if (idx->have_delta) {
			error = store_delta(idx);
		} else {
			error = store_object(idx);
		}

		if (error < 0)
665
			goto on_error;
666

667 668 669
		if (!idx->have_delta) {
			stats->indexed_objects = (unsigned int)++processed;
		}
670
		stats->received_objects++;
671

672
		if ((error = do_progress_callback(idx, stats)) != 0)
673
			goto on_error;
674
	}
675

676
	return 0;
677

678 679
on_error:
	git_mwindow_free_all(mwf);
680
	return error;
681
}
682

683
static int index_path(git_buf *path, git_indexer *idx, const char *suffix)
684 685 686
{
	const char prefix[] = "pack-";
	size_t slash = (size_t)path->size;
687

688 689 690
	/* search backwards for '/' */
	while (slash > 0 && path->ptr[slash - 1] != '/')
		slash--;
691

692 693 694 695 696 697
	if (git_buf_grow(path, slash + 1 + strlen(prefix) +
					 GIT_OID_HEXSZ + strlen(suffix) + 1) < 0)
		return -1;

	git_buf_truncate(path, slash);
	git_buf_puts(path, prefix);
nulltoken committed
698
	git_oid_fmt(path->ptr + git_buf_len(path), &idx->hash);
699 700 701 702 703 704
	path->size += GIT_OID_HEXSZ;
	git_buf_puts(path, suffix);

	return git_buf_oom(path) ? -1 : 0;
}

705 706 707 708
/**
 * Rewind the packfile by the trailer, as we might need to fix the
 * packfile by injecting objects at the tail and must overwrite it.
 */
709
static void seek_back_trailer(git_indexer *idx)
710 711 712 713 714
{
	idx->pack->mwf.size -= GIT_OID_RAWSZ;
	git_mwindow_free_all(&idx->pack->mwf);
}

715
static int inject_object(git_indexer *idx, git_oid *id)
716
{
717 718
	git_odb_object *obj;
	struct entry *entry;
719
	struct git_pack_entry *pentry = NULL;
720 721 722 723 724 725 726 727
	git_oid foo = {{0}};
	unsigned char hdr[64];
	git_buf buf = GIT_BUF_INIT;
	git_off_t entry_start;
	const void *data;
	size_t len, hdr_len;
	int error;

728 729
	seek_back_trailer(idx);
	entry_start = idx->pack->mwf.size;
730

731 732
	if (git_odb_read(&obj, idx->odb, id) < 0) {
		giterr_set(GITERR_INDEXER, "missing delta bases");
733
		return -1;
734
	}
735 736 737 738

	data = git_odb_object_data(obj);
	len = git_odb_object_size(obj);

739 740 741
	entry = git__calloc(1, sizeof(*entry));
	GITERR_CHECK_ALLOC(entry);

742 743 744 745
	entry->crc = crc32(0L, Z_NULL, 0);

	/* Write out the object header */
	hdr_len = git_packfile__object_header(hdr, len, git_odb_object_type(obj));
746 747 748
	if ((error = append_to_pack(idx, hdr, hdr_len)) < 0)
		goto cleanup;

749
	idx->pack->mwf.size += hdr_len;
750
	entry->crc = crc32(entry->crc, hdr, (uInt)hdr_len);
751

752
	if ((error = git_zstream_deflatebuf(&buf, data, len)) < 0)
753 754 755
		goto cleanup;

	/* And then the compressed object */
756 757 758
	if ((error = append_to_pack(idx, buf.ptr, buf.size)) < 0)
		goto cleanup;

759
	idx->pack->mwf.size += buf.size;
Linquize committed
760
	entry->crc = htonl(crc32(entry->crc, (unsigned char *)buf.ptr, (uInt)buf.size));
761
	git_buf_dispose(&buf);
762 763

	/* Write a fake trailer so the pack functions play ball */
764 765

	if ((error = append_to_pack(idx, &foo, GIT_OID_RAWSZ)) < 0)
766 767 768 769 770 771 772 773 774 775 776
		goto cleanup;

	idx->pack->mwf.size += GIT_OID_RAWSZ;

	pentry = git__calloc(1, sizeof(struct git_pack_entry));
	GITERR_CHECK_ALLOC(pentry);

	git_oid_cpy(&pentry->sha1, id);
	git_oid_cpy(&entry->oid, id);
	idx->off = entry_start + hdr_len + len;

777
	error = save_entry(idx, entry, pentry, entry_start);
778 779

cleanup:
780 781 782 783
	if (error) {
		git__free(entry);
		git__free(pentry);
	}
784

785 786 787 788
	git_odb_object_free(obj);
	return error;
}

789
static int fix_thin_pack(git_indexer *idx, git_transfer_progress *stats)
790
{
791
	int error, found_ref_delta = 0;
792 793
	unsigned int i;
	struct delta_info *delta;
794 795 796
	size_t size;
	git_otype type;
	git_mwindow *w = NULL;
Linquize committed
797
	git_off_t curpos = 0;
798 799 800 801 802
	unsigned char *base_info;
	unsigned int left = 0;
	git_oid base;

	assert(git_vector_length(&idx->deltas) > 0);
803 804 805 806 807

	if (idx->odb == NULL) {
		giterr_set(GITERR_INDEXER, "cannot fix a thin pack without an ODB");
		return -1;
	}
808

809
	/* Loop until we find the first REF delta */
810
	git_vector_foreach(&idx->deltas, i, delta) {
811 812 813
		if (!delta)
			continue;

814
		curpos = delta->delta_off;
815 816 817 818
		error = git_packfile_unpack_header(&size, &type, &idx->pack->mwf, &w, &curpos);
		if (error < 0)
			return error;

819 820 821
		if (type == GIT_OBJ_REF_DELTA) {
			found_ref_delta = 1;
			break;
822
		}
823
	}
824

825 826 827 828
	if (!found_ref_delta) {
		giterr_set(GITERR_INDEXER, "no REF_DELTA found, cannot inject object");
		return -1;
	}
829

830 831 832 833 834 835
	/* curpos now points to the base information, which is an OID */
	base_info = git_mwindow_open(&idx->pack->mwf, &w, curpos, GIT_OID_RAWSZ, &left);
	if (base_info == NULL) {
		giterr_set(GITERR_INDEXER, "failed to map delta information");
		return -1;
	}
836

837 838
	git_oid_fromraw(&base, base_info);
	git_mwindow_close(&w);
839

840 841 842
	if (has_entry(idx, &base))
		return 0;

843 844 845 846
	if (inject_object(idx, &base) < 0)
		return -1;

	stats->local_objects++;
847 848 849 850

	return 0;
}

851
static int resolve_deltas(git_indexer *idx, git_transfer_progress *stats)
852 853
{
	unsigned int i;
lhchavez committed
854
	int error;
855
	struct delta_info *delta;
856
	int progressed = 0, non_null = 0, progress_cb_result;
857 858 859

	while (idx->deltas.length > 0) {
		progressed = 0;
860
		non_null = 0;
861
		git_vector_foreach(&idx->deltas, i, delta) {
862
			git_rawobj obj = {NULL};
863

864 865 866 867
			if (!delta)
				continue;

			non_null = 1;
868
			idx->off = delta->delta_off;
lhchavez committed
869 870 871 872 873 874 875
			if ((error = git_packfile_unpack(&obj, idx->pack, &idx->off)) < 0) {
				if (error == GIT_PASSTHROUGH) {
					/* We have not seen the base object, we'll try again later. */
					continue;
				}
				return -1;
			}
876 877 878 879 880 881

			if (hash_and_save(idx, &obj, delta->delta_off) < 0)
				continue;

			git__free(obj.data);
			stats->indexed_objects++;
882
			stats->indexed_deltas++;
883
			progressed = 1;
884 885
			if ((progress_cb_result = do_progress_callback(idx, stats)) < 0)
				return progress_cb_result;
886

887 888
			/* remove from the list */
			git_vector_set(NULL, &idx->deltas, i, NULL);
889
			git__free(delta);
890
		}
891

892 893 894 895
		/* if none were actually set, we're done */
		if (!non_null)
			break;

896
		if (!progressed && (fix_thin_pack(idx, stats) < 0)) {
897
			return -1;
898
		}
899 900 901 902 903
	}

	return 0;
}

904
static int update_header_and_rehash(git_indexer *idx, git_transfer_progress *stats)
905 906 907 908 909 910 911 912 913 914
{
	void *ptr;
	size_t chunk = 1024*1024;
	git_off_t hashed = 0;
	git_mwindow *w = NULL;
	git_mwindow_file *mwf;
	unsigned int left;

	mwf = &idx->pack->mwf;

915
	git_hash_init(&idx->trailer);
916

917 918

	/* Update the header to include the numer of local objects we injected */
919
	idx->hdr.hdr_entries = htonl(stats->total_objects + stats->local_objects);
920
	if (write_at(idx, &idx->hdr, 0, sizeof(struct git_pack_header)) < 0)
921
		return -1;
922

923 924 925 926 927 928
	/*
	 * We now use the same technique as before to determine the
	 * hash. We keep reading up to the end and let
	 * hash_partially() keep the existing trailer out of the
	 * calculation.
	 */
929
	git_mwindow_free_all(mwf);
930 931 932 933
	idx->inbuf_len = 0;
	while (hashed < mwf->size) {
		ptr = git_mwindow_open(mwf, &w, hashed, chunk, &left);
		if (ptr == NULL)
934
			return -1;
935

936 937 938 939
		hash_partially(idx, ptr, left);
		hashed += left;

		git_mwindow_close(&w);
940
	}
941

942 943 944
	return 0;
}

945
int git_indexer_commit(git_indexer *idx, git_transfer_progress *stats)
946 947 948
{
	git_mwindow *w = NULL;
	unsigned int i, long_offsets = 0, left;
949
	int error;
950 951 952
	struct git_pack_idx_header hdr;
	git_buf filename = GIT_BUF_INIT;
	struct entry *entry;
953
	git_oid trailer_hash, file_hash;
954
	git_filebuf index_file = {0};
955
	void *packfile_trailer;
956

957 958 959 960 961
	if (!idx->parsed_header) {
		giterr_set(GITERR_INDEXER, "incomplete pack header");
		return -1;
	}

962
	/* Test for this before resolve_deltas(), as it plays with idx->off */
963 964
	if (idx->off + 20 < idx->pack->mwf.size) {
		giterr_set(GITERR_INDEXER, "unexpected data at the end of the pack");
965 966
		return -1;
	}
967 968 969 970
	if (idx->off + 20 > idx->pack->mwf.size) {
		giterr_set(GITERR_INDEXER, "missing trailer at the end of the pack");
		return -1;
	}
971

972 973 974 975 976 977 978 979 980 981 982 983
	packfile_trailer = git_mwindow_open(&idx->pack->mwf, &w, idx->pack->mwf.size - GIT_OID_RAWSZ, GIT_OID_RAWSZ, &left);
	if (packfile_trailer == NULL) {
		git_mwindow_close(&w);
		goto on_error;
	}

	/* Compare the packfile trailer as it was sent to us and what we calculated */
	git_oid_fromraw(&file_hash, packfile_trailer);
	git_mwindow_close(&w);

	git_hash_final(&trailer_hash, &idx->trailer);
	if (git_oid_cmp(&file_hash, &trailer_hash)) {
984
		giterr_set(GITERR_INDEXER, "packfile trailer mismatch");
985 986 987
		return -1;
	}

988 989 990
	/* Freeze the number of deltas */
	stats->total_deltas = stats->total_objects - stats->indexed_objects;

991 992
	if ((error = resolve_deltas(idx, stats)) < 0)
		return error;
993

994
	if (stats->indexed_objects != stats->total_objects) {
995
		giterr_set(GITERR_INDEXER, "early EOF");
996 997 998
		return -1;
	}

999 1000 1001 1002 1003
	if (stats->local_objects > 0) {
		if (update_header_and_rehash(idx, stats) < 0)
			return -1;

		git_hash_final(&trailer_hash, &idx->trailer);
1004
		write_at(idx, &trailer_hash, idx->pack->mwf.size - GIT_OID_RAWSZ, GIT_OID_RAWSZ);
1005 1006
	}

1007 1008
	git_vector_sort(&idx->objects);

1009 1010 1011 1012
	/* Use the trailer hash as the pack file name to ensure
	 * files with different contents have different names */
	git_oid_cpy(&idx->hash, &trailer_hash);

1013
	git_buf_sets(&filename, idx->pack->pack_name);
1014
	git_buf_shorten(&filename, strlen("pack"));
1015 1016 1017 1018
	git_buf_puts(&filename, "idx");
	if (git_buf_oom(&filename))
		return -1;

1019
	if (git_filebuf_open(&index_file, filename.ptr,
1020
		GIT_FILEBUF_HASH_CONTENTS |
1021
		(idx->do_fsync ? GIT_FILEBUF_FSYNC : 0),
1022
		idx->mode) < 0)
1023 1024 1025 1026 1027
		goto on_error;

	/* Write out the header */
	hdr.idx_signature = htonl(PACK_IDX_SIGNATURE);
	hdr.idx_version = htonl(2);
1028
	git_filebuf_write(&index_file, &hdr, sizeof(hdr));
1029 1030 1031 1032

	/* Write out the fanout table */
	for (i = 0; i < 256; ++i) {
		uint32_t n = htonl(idx->fanout[i]);
1033
		git_filebuf_write(&index_file, &n, sizeof(n));
1034 1035
	}

1036 1037
	/* Write out the object names (SHA-1 hashes) */
	git_vector_foreach(&idx->objects, i, entry) {
1038
		git_filebuf_write(&index_file, &entry->oid, sizeof(git_oid));
1039 1040 1041 1042
	}

	/* Write out the CRC32 values */
	git_vector_foreach(&idx->objects, i, entry) {
1043
		git_filebuf_write(&index_file, &entry->crc, sizeof(uint32_t));
1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054
	}

	/* Write out the offsets */
	git_vector_foreach(&idx->objects, i, entry) {
		uint32_t n;

		if (entry->offset == UINT32_MAX)
			n = htonl(0x80000000 | long_offsets++);
		else
			n = htonl(entry->offset);

1055
		git_filebuf_write(&index_file, &n, sizeof(uint32_t));
1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067
	}

	/* Write out the long offsets */
	git_vector_foreach(&idx->objects, i, entry) {
		uint32_t split[2];

		if (entry->offset != UINT32_MAX)
			continue;

		split[0] = htonl(entry->offset_long >> 32);
		split[1] = htonl(entry->offset_long & 0xffffffff);

1068
		git_filebuf_write(&index_file, &split, sizeof(uint32_t) * 2);
1069 1070
	}

1071 1072
	/* Write out the packfile trailer to the index */
	if (git_filebuf_write(&index_file, &trailer_hash, GIT_OID_RAWSZ) < 0)
1073 1074
		goto on_error;

1075 1076
	/* Write out the hash of the idx */
	if (git_filebuf_hash(&trailer_hash, &index_file) < 0)
1077 1078
		goto on_error;

1079
	git_filebuf_write(&index_file, &trailer_hash, sizeof(git_oid));
1080 1081

	/* Figure out what the final name should be */
1082
	if (index_path(&filename, idx, ".idx") < 0)
1083 1084 1085
		goto on_error;

	/* Commit file */
1086
	if (git_filebuf_commit_at(&index_file, filename.ptr) < 0)
1087 1088 1089
		goto on_error;

	git_mwindow_free_all(&idx->pack->mwf);
1090 1091 1092 1093 1094 1095 1096

	/* Truncate file to undo rounding up to next page_size in append_to_pack */
	if (p_ftruncate(idx->pack->mwf.fd, idx->pack->mwf.size) < 0) {
		giterr_set(GITERR_OS, "failed to truncate pack file '%s'", idx->pack->pack_name);
		return -1;
	}

1097
	if (idx->do_fsync && p_fsync(idx->pack->mwf.fd) < 0) {
1098 1099 1100 1101
		giterr_set(GITERR_OS, "failed to fsync packfile");
		goto on_error;
	}

1102
	/* We need to close the descriptor here so Windows doesn't choke on commit_at */
1103 1104 1105 1106 1107
	if (p_close(idx->pack->mwf.fd) < 0) {
		giterr_set(GITERR_OS, "failed to close packfile");
		goto on_error;
	}

1108
	idx->pack->mwf.fd = -1;
1109

1110
	if (index_path(&filename, idx, ".pack") < 0)
1111
		goto on_error;
1112

1113
	/* And don't forget to rename the packfile to its new place. */
1114 1115 1116 1117
	if (p_rename(idx->pack->pack_name, git_buf_cstr(&filename)) < 0)
		goto on_error;

	/* And fsync the parent directory if we're asked to. */
1118
	if (idx->do_fsync &&
1119 1120 1121
		git_futils_fsync_parent(git_buf_cstr(&filename)) < 0)
		goto on_error;

1122
	idx->pack_committed = 1;
1123

1124
	git_buf_dispose(&filename);
1125 1126 1127
	return 0;

on_error:
1128
	git_mwindow_free_all(&idx->pack->mwf);
1129
	git_filebuf_cleanup(&index_file);
1130
	git_buf_dispose(&filename);
1131 1132 1133
	return -1;
}

1134
void git_indexer_free(git_indexer *idx)
1135 1136 1137 1138
{
	if (idx == NULL)
		return;

1139
	if (idx->have_stream)
1140
		git_packfile_stream_dispose(&idx->stream);
1141

1142
	git_vector_free_deep(&idx->objects);
1143

1144
	if (idx->pack->idx_cache) {
Russell Belfer committed
1145
		struct git_pack_entry *pentry;
1146 1147 1148
		git_oidmap_foreach_value(idx->pack->idx_cache, pentry, {
			git__free(pentry);
		});
1149 1150

		git_oidmap_free(idx->pack->idx_cache);
1151
	}
1152

1153
	git_vector_free_deep(&idx->deltas);
1154 1155

	if (!git_mutex_lock(&git__mwindow_mutex)) {
1156 1157 1158
		if (!idx->pack_committed)
			git_packfile_close(idx->pack, true);

1159 1160 1161 1162
		git_packfile_free(idx->pack);
		git_mutex_unlock(&git__mwindow_mutex);
	}

1163 1164
	git_hash_ctx_cleanup(&idx->trailer);
	git_hash_ctx_cleanup(&idx->hash_ctx);
1165 1166
	git__free(idx);
}