odb.c 26.1 KB
Newer Older
1
/*
Edward Thomson committed
2
 * Copyright (C) the libgit2 contributors. All rights reserved.
3
 *
Vicent Marti committed
4 5
 * This file is part of libgit2, distributed under the GNU GPL v2 with
 * a Linking Exception. For full terms see the included COPYING file.
6 7
 */

8
#include "common.h"
9
#include <zlib.h>
10
#include "git2/object.h"
11
#include "git2/sys/odb_backend.h"
12
#include "fileops.h"
13
#include "hash.h"
14
#include "odb.h"
Vicent Marti committed
15
#include "delta-apply.h"
16
#include "filter.h"
17
#include "repository.h"
18

19
#include "git2/odb_backend.h"
20
#include "git2/oid.h"
21

22 23
#define GIT_ALTERNATES_FILE "info/alternates"

24 25 26 27 28 29
/*
 * We work under the assumption that most objects for long-running
 * operations will be packed
 */
#define GIT_LOOSE_PRIORITY 1
#define GIT_PACKED_PRIORITY 2
30

31 32
#define GIT_ALTERNATES_MAX_DEPTH 5

33 34 35 36
typedef struct
{
	git_odb_backend *backend;
	int priority;
37 38
	bool is_alternate;
	ino_t disk_inode;
39 40
} backend_internal;

41 42 43 44 45 46 47 48 49
static git_cache *odb_cache(git_odb *odb)
{
	if (odb->rc.owner != NULL) {
		git_repository *owner = odb->rc.owner;
		return &owner->objects;
	}

	return &odb->own_cache;
}
50

51 52
static int load_alternates(git_odb *odb, const char *objects_dir, int alternate_depth);

53
int git_odb__format_object_header(char *hdr, size_t n, git_off_t obj_len, git_otype obj_type)
54
{
Vicent Marti committed
55
	const char *type_str = git_object_type2string(obj_type);
56
	int len = p_snprintf(hdr, n, "%s %lld", type_str, obj_len);
57
	assert(len > 0 && len <= (int)n);
58 59 60
	return len+1;
}

61
int git_odb__hashobj(git_oid *id, git_rawobj *obj)
62 63
{
	git_buf_vec vec[2];
64 65
	char header[64];
	int hdrlen;
66

67
	assert(id && obj);
68

69
	if (!git_object_typeisloose(obj->type))
70
		return -1;
Vicent Marti committed
71

72
	if (!obj->data && obj->len != 0)
73
		return -1;
74

75
	hdrlen = git_odb__format_object_header(header, sizeof(header), obj->len, obj->type);
76

77
	vec[0].data = header;
Vicent Marti committed
78
	vec[0].len = hdrlen;
79
	vec[1].data = obj->data;
Vicent Marti committed
80
	vec[1].len = obj->len;
81 82 83

	git_hash_vec(id, vec, 2);

84
	return 0;
85 86
}

87

88
static git_odb_object *odb_object__alloc(const git_oid *oid, git_rawobj *source)
Ramsay Jones committed
89
{
90
	git_odb_object *object = git__calloc(1, sizeof(git_odb_object));
Ramsay Jones committed
91

92 93 94 95 96 97
	if (object != NULL) {
		git_oid_cpy(&object->cached.oid, oid);
		object->cached.type = source->type;
		object->cached.size = source->len;
		object->buffer      = source->data;
	}
Ramsay Jones committed
98

Vicent Marti committed
99
	return object;
100 101
}

102
void git_odb_object__free(void *object)
103
{
Vicent Marti committed
104
	if (object != NULL) {
105
		git__free(((git_odb_object *)object)->buffer);
106
		git__free(object);
Vicent Marti committed
107 108
	}
}
109

110 111 112 113 114 115 116
const git_oid *git_odb_object_id(git_odb_object *object)
{
	return &object->cached.oid;
}

const void *git_odb_object_data(git_odb_object *object)
{
Vicent Marti committed
117
	return object->buffer;
118 119 120 121
}

size_t git_odb_object_size(git_odb_object *object)
{
Vicent Marti committed
122
	return object->cached.size;
123 124 125 126
}

git_otype git_odb_object_type(git_odb_object *object)
{
Vicent Marti committed
127
	return object->cached.type;
128 129
}

130 131 132 133 134 135 136
int git_odb_object_dup(git_odb_object **dest, git_odb_object *source)
{
	git_cached_obj_incref(source);
	*dest = source;
	return 0;
}

137
void git_odb_object_free(git_odb_object *object)
Vicent Marti committed
138
{
139 140 141
	if (object == NULL)
		return;

142
	git_cached_obj_decref(object);
Vicent Marti committed
143
}
144

145
int git_odb__hashfd(git_oid *out, git_file fd, size_t size, git_otype type)
Vicent Marti committed
146
{
147
	int hdr_len;
148
	char hdr[64], buffer[FILEIO_BUFSIZE];
149
	git_hash_ctx ctx;
150
	ssize_t read_len = 0;
151
	int error = 0;
Vicent Marti committed
152

153 154 155 156 157
	if (!git_object_typeisloose(type)) {
		giterr_set(GITERR_INVALID, "Invalid object type for hash");
		return -1;
	}

158 159
	if ((error = git_hash_ctx_init(&ctx)) < 0)
		return -1;
Vicent Marti committed
160

161
	hdr_len = git_odb__format_object_header(hdr, sizeof(hdr), size, type);
162

163
	if ((error = git_hash_update(&ctx, hdr, hdr_len)) < 0)
164
		goto done;
Vicent Marti committed
165

Vicent Marti committed
166
	while (size > 0 && (read_len = p_read(fd, buffer, sizeof(buffer))) > 0) {
167
		if ((error = git_hash_update(&ctx, buffer, read_len)) < 0)
168 169
			goto done;

Vicent Marti committed
170 171 172
		size -= read_len;
	}

Vicent Marti committed
173 174 175 176 177
	/* If p_read returned an error code, the read obviously failed.
	 * If size is not zero, the file was truncated after we originally
	 * stat'd it, so we consider this a read failure too */
	if (read_len < 0 || size > 0) {
		giterr_set(GITERR_OS, "Error reading file for hashing");
178 179 180
		error = -1;

		goto done;
Vicent Marti committed
181 182
	}

183
	error = git_hash_final(out, &ctx);
Vicent Marti committed
184

185
done:
186
	git_hash_ctx_cleanup(&ctx);
187
	return error;
Vicent Marti committed
188 189
}

190
int git_odb__hashfd_filtered(
191
	git_oid *out, git_file fd, size_t size, git_otype type, git_filter_list *fl)
192 193 194 195
{
	int error;
	git_buf raw = GIT_BUF_INIT;

196
	if (!fl)
197 198 199 200 201 202
		return git_odb__hashfd(out, fd, size, type);

	/* size of data is used in header, so we have to read the whole file
	 * into memory to apply filters before beginning to calculate the hash
	 */

203
	if (!(error = git_futils_readbuffer_fd(&raw, fd, size))) {
204
		git_buf post = GIT_BUF_INIT;
205

206
		error = git_filter_list_apply_to_data(&post, fl, &raw);
207

208
		git_buf_free(&raw);
209

210 211 212
		if (!error)
			error = git_odb_hash(out, post.ptr, post.size, type);

213
		git_buf_free(&post);
214
	}
215 216 217 218

	return error;
}

219 220 221
int git_odb__hashlink(git_oid *out, const char *path)
{
	struct stat st;
222
	int size;
223
	int result;
224

225
	if (git_path_lstat(path, &st) < 0)
226
		return -1;
227

228
	if (!git__is_int(st.st_size) || (int)st.st_size < 0) {
229
		giterr_set(GITERR_FILESYSTEM, "File size overflow for 32-bit systems");
230 231
		return -1;
	}
232

233
	size = (int)st.st_size;
234

235 236
	if (S_ISLNK(st.st_mode)) {
		char *link_data;
237 238
		int read_len;
		size_t alloc_size;
239

240 241
		GITERR_CHECK_ALLOC_ADD(&alloc_size, size, 1);
		link_data = git__malloc(alloc_size);
242
		GITERR_CHECK_ALLOC(link_data);
243

244
		read_len = p_readlink(path, link_data, size);
245
		link_data[size] = '\0';
246
		if (read_len != size) {
247
			giterr_set(GITERR_OS, "Failed to read symlink data for '%s'", path);
248
			git__free(link_data);
249 250
			return -1;
		}
251

252
		result = git_odb_hash(out, link_data, size, GIT_OBJ_BLOB);
253
		git__free(link_data);
254
	} else {
255 256 257
		int fd = git_futils_open_ro(path);
		if (fd < 0)
			return -1;
258
		result = git_odb__hashfd(out, fd, size, GIT_OBJ_BLOB);
259 260 261
		p_close(fd);
	}

262
	return result;
263 264
}

265 266 267
int git_odb_hashfile(git_oid *out, const char *path, git_otype type)
{
	git_off_t size;
268 269
	int result, fd = git_futils_open_ro(path);
	if (fd < 0)
270
		return fd;
271 272

	if ((size = git_futils_filesize(fd)) < 0 || !git__is_sizet(size)) {
273
		giterr_set(GITERR_OS, "File size overflow for 32-bit systems");
274
		p_close(fd);
275
		return -1;
276 277
	}

278
	result = git_odb__hashfd(out, fd, (size_t)size, type);
279
	p_close(fd);
280
	return result;
281 282
}

Vicent Marti committed
283 284 285
int git_odb_hash(git_oid *id, const void *data, size_t len, git_otype type)
{
	git_rawobj raw;
286

Vicent Marti committed
287
	assert(id);
288

Vicent Marti committed
289 290 291
	raw.data = (void *)data;
	raw.len = len;
	raw.type = type;
292

293
	return git_odb__hashobj(id, &raw);
294 295
}

296 297 298 299 300 301 302 303 304 305 306
/**
 * FAKE WSTREAM
 */

typedef struct {
	git_odb_stream stream;
	char *buffer;
	size_t size, written;
	git_otype type;
} fake_wstream;

307
static int fake_wstream__fwrite(git_odb_stream *_stream, const git_oid *oid)
308 309
{
	fake_wstream *stream = (fake_wstream *)_stream;
310
	return _stream->backend->write(_stream->backend, oid, stream->buffer, stream->size, stream->type);
311 312 313 314 315 316
}

static int fake_wstream__write(git_odb_stream *_stream, const char *data, size_t len)
{
	fake_wstream *stream = (fake_wstream *)_stream;

317
	if (stream->written + len > stream->size)
318
		return -1;
319 320 321

	memcpy(stream->buffer + stream->written, data, len);
	stream->written += len;
322
	return 0;
323 324 325 326 327 328
}

static void fake_wstream__free(git_odb_stream *_stream)
{
	fake_wstream *stream = (fake_wstream *)_stream;

329 330
	git__free(stream->buffer);
	git__free(stream);
331 332
}

333
static int init_fake_wstream(git_odb_stream **stream_p, git_odb_backend *backend, git_off_t size, git_otype type)
334 335 336
{
	fake_wstream *stream;

337 338 339 340 341
	if (!git__is_ssizet(size)) {
		giterr_set(GITERR_ODB, "object size too large to keep in memory");
		return -1;
	}

342
	stream = git__calloc(1, sizeof(fake_wstream));
343
	GITERR_CHECK_ALLOC(stream);
344 345 346 347 348

	stream->size = size;
	stream->type = type;
	stream->buffer = git__malloc(size);
	if (stream->buffer == NULL) {
349
		git__free(stream);
350
		return -1;
351 352 353 354 355 356 357 358 359 360
	}

	stream->stream.backend = backend;
	stream->stream.read = NULL; /* read only */
	stream->stream.write = &fake_wstream__write;
	stream->stream.finalize_write = &fake_wstream__fwrite;
	stream->stream.free = &fake_wstream__free;
	stream->stream.mode = GIT_STREAM_WRONLY;

	*stream_p = (git_odb_stream *)stream;
361
	return 0;
362
}
363

364 365 366 367 368 369 370
/***********************************************************
 *
 * OBJECT DATABASE PUBLIC API
 *
 * Public calls for the ODB functionality
 *
 ***********************************************************/
371

372
static int backend_sort_cmp(const void *a, const void *b)
373
{
374 375
	const backend_internal *backend_a = (const backend_internal *)(a);
	const backend_internal *backend_b = (const backend_internal *)(b);
376 377 378

	if (backend_a->is_alternate == backend_b->is_alternate)
		return (backend_b->priority - backend_a->priority);
379

380
	return backend_a->is_alternate ? 1 : -1;
381 382
}

383
int git_odb_new(git_odb **out)
384
{
385
	git_odb *db = git__calloc(1, sizeof(*db));
386
	GITERR_CHECK_ALLOC(db);
387

388 389
	if (git_cache_init(&db->own_cache) < 0 ||
		git_vector_init(&db->backends, 4, backend_sort_cmp) < 0) {
390
		git__free(db);
391
		return -1;
392
	}
393

394
	*out = db;
395
	GIT_REFCOUNT_INC(db);
396
	return 0;
397 398
}

399 400 401
static int add_backend_internal(
	git_odb *odb, git_odb_backend *backend,
	int priority, bool is_alternate, ino_t disk_inode)
Ramsay Jones committed
402
{
403 404
	backend_internal *internal;

405
	assert(odb && backend);
Ramsay Jones committed
406

Ben Straub committed
407
	GITERR_CHECK_VERSION(backend, GIT_ODB_BACKEND_VERSION, "git_odb_backend");
408

409 410
	/* Check if the backend is already owned by another ODB */
	assert(!backend->odb || backend->odb == odb);
Ramsay Jones committed
411

412
	internal = git__malloc(sizeof(backend_internal));
413
	GITERR_CHECK_ALLOC(internal);
414 415 416 417

	internal->backend = backend;
	internal->priority = priority;
	internal->is_alternate = is_alternate;
418
	internal->disk_inode = disk_inode;
Ramsay Jones committed
419

420
	if (git_vector_insert(&odb->backends, internal) < 0) {
421
		git__free(internal);
422
		return -1;
423
	}
Ramsay Jones committed
424

425
	git_vector_sort(&odb->backends);
426
	internal->backend->odb = odb;
427
	return 0;
Ramsay Jones committed
428 429
}

430 431
int git_odb_add_backend(git_odb *odb, git_odb_backend *backend, int priority)
{
432
	return add_backend_internal(odb, backend, priority, false, 0);
433 434 435 436
}

int git_odb_add_alternate(git_odb *odb, git_odb_backend *backend, int priority)
{
437
	return add_backend_internal(odb, backend, priority, true, 0);
438 439
}

440 441 442 443 444 445
size_t git_odb_num_backends(git_odb *odb)
{
	assert(odb);
	return odb->backends.length;
}

446 447 448 449 450 451 452 453
static int git_odb__error_unsupported_in_backend(const char *action)
{
	giterr_set(GITERR_ODB,
		"Cannot %s - unsupported in the loaded odb backends", action);
	return -1;
}


454 455 456 457
int git_odb_get_backend(git_odb_backend **out, git_odb *odb, size_t pos)
{
	backend_internal *internal;

Linquize committed
458
	assert(out && odb);
459 460 461 462 463 464 465
	internal = git_vector_get(&odb->backends, pos);

	if (internal && internal->backend) {
		*out = internal->backend;
		return 0;
	}

466
	giterr_set(GITERR_ODB, "No ODB backend loaded at index %" PRIuZ, pos);
467 468 469
	return GIT_ENOTFOUND;
}

470 471 472
static int add_default_backends(
	git_odb *db, const char *objects_dir,
	bool as_alternates, int alternate_depth)
473
{
474 475
	size_t i;
	struct stat st;
476
	ino_t inode;
477 478
	git_odb_backend *loose, *packed;

479 480
	/* TODO: inodes are not really relevant on Win32, so we need to find
	 * a cross-platform workaround for this */
481 482 483 484 485 486
#ifdef GIT_WIN32
	GIT_UNUSED(i);
	GIT_UNUSED(st);

	inode = 0;
#else
487
	if (p_stat(objects_dir, &st) < 0) {
488 489 490
		if (as_alternates)
			return 0;

491 492 493 494
		giterr_set(GITERR_ODB, "Failed to load object database in '%s'", objects_dir);
		return -1;
	}

495 496
	inode = st.st_ino;

497 498
	for (i = 0; i < db->backends.length; ++i) {
		backend_internal *backend = git_vector_get(&db->backends, i);
499
		if (backend->disk_inode == inode)
500 501
			return 0;
	}
502
#endif
503

504
	/* add the loose object backend */
505
	if (git_odb_backend_loose(&loose, objects_dir, -1, 0, 0, 0) < 0 ||
506
		add_backend_internal(db, loose, GIT_LOOSE_PRIORITY, as_alternates, inode) < 0)
507
		return -1;
508 509

	/* add the packed file backend */
510
	if (git_odb_backend_pack(&packed, objects_dir) < 0 ||
511
		add_backend_internal(db, packed, GIT_PACKED_PRIORITY, as_alternates, inode) < 0)
512
		return -1;
513

514
	return load_alternates(db, objects_dir, alternate_depth);
515 516
}

517
static int load_alternates(git_odb *odb, const char *objects_dir, int alternate_depth)
518
{
519
	git_buf alternates_path = GIT_BUF_INIT;
520
	git_buf alternates_buf = GIT_BUF_INIT;
521 522
	char *buffer;
	const char *alternate;
523
	int result = 0;
524

525
	/* Git reports an error, we just ignore anything deeper */
526
	if (alternate_depth > GIT_ALTERNATES_MAX_DEPTH)
527 528
		return 0;

529 530
	if (git_buf_joinpath(&alternates_path, objects_dir, GIT_ALTERNATES_FILE) < 0)
		return -1;
531

532
	if (git_path_exists(alternates_path.ptr) == false) {
533
		git_buf_free(&alternates_path);
534
		return 0;
535
	}
536

537
	if (git_futils_readbuffer(&alternates_buf, alternates_path.ptr) < 0) {
538
		git_buf_free(&alternates_path);
539
		return -1;
540
	}
541

542
	buffer = (char *)alternates_buf.ptr;
543 544

	/* add each alternate as a new backend; one alternate per line */
545 546 547 548
	while ((alternate = git__strtok(&buffer, "\r\n")) != NULL) {
		if (*alternate == '\0' || *alternate == '#')
			continue;

549 550 551 552 553 554
		/*
		 * Relative path: build based on the current `objects`
		 * folder. However, relative paths are only allowed in
		 * the current repository.
		 */
		if (*alternate == '.' && !alternate_depth) {
555
			if ((result = git_buf_joinpath(&alternates_path, objects_dir, alternate)) < 0)
556 557
				break;
			alternate = git_buf_cstr(&alternates_path);
558 559
		}

560
		if ((result = add_default_backends(odb, alternate, true, alternate_depth + 1)) < 0)
561 562
			break;
	}
563

564
	git_buf_free(&alternates_path);
565 566
	git_buf_free(&alternates_buf);

567
	return result;
568
}
Ramsay Jones committed
569

570 571
int git_odb_add_disk_alternate(git_odb *odb, const char *path)
{
572
	return add_default_backends(odb, path, true, 0);
573 574
}

575
int git_odb_open(git_odb **out, const char *objects_dir)
Ramsay Jones committed
576
{
577
	git_odb *db;
Ramsay Jones committed
578

579 580 581 582
	assert(out && objects_dir);

	*out = NULL;

583 584
	if (git_odb_new(&db) < 0)
		return -1;
Ramsay Jones committed
585

586
	if (add_default_backends(db, objects_dir, 0, 0) < 0) {
587 588 589
		git_odb_free(db);
		return -1;
	}
Ramsay Jones committed
590

591
	*out = db;
592
	return 0;
593
}
Ramsay Jones committed
594

595
static void odb_free(git_odb *db)
596
{
597
	size_t i;
598

599
	for (i = 0; i < db->backends.length; ++i) {
600 601
		backend_internal *internal = git_vector_get(&db->backends, i);
		git_odb_backend *backend = internal->backend;
602

603
		if (backend->free) backend->free(backend);
604
		else git__free(backend);
605

606
		git__free(internal);
607 608
	}

609
	git_vector_free(&db->backends);
610
	git_cache_free(&db->own_cache);
611

612
	git__memzero(db, sizeof(*db));
613
	git__free(db);
614 615
}

616 617 618 619 620 621 622 623
void git_odb_free(git_odb *db)
{
	if (db == NULL)
		return;

	GIT_REFCOUNT_DEC(db, odb_free);
}

624
int git_odb_exists(git_odb *db, const git_oid *id)
625
{
Vicent Marti committed
626
	git_odb_object *object;
627
	size_t i;
628
	bool found = false;
629

630
	assert(db && id);
631

632
	if ((object = git_cache_get_raw(odb_cache(db), id)) != NULL) {
633
		git_odb_object_free(object);
634
		return (int)true;
Vicent Marti committed
635 636
	}

637
	for (i = 0; i < db->backends.length && !found; ++i) {
638 639
		backend_internal *internal = git_vector_get(&db->backends, i);
		git_odb_backend *b = internal->backend;
640

641
		if (b->exists != NULL)
Linquize committed
642
			found = (bool)b->exists(b, id);
643 644
	}

645
	return (int)found;
646 647
}

648 649 650 651 652
int git_odb_exists_prefix(
	git_oid *out, git_odb *db, const git_oid *short_id, size_t len)
{
	int error = GIT_ENOTFOUND, num_found = 0;
	size_t i;
653
	git_oid key = {{0}}, last_found = {{0}}, found;
654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671

	assert(db && short_id);

	if (len < GIT_OID_MINPREFIXLEN)
		return git_odb__error_ambiguous("prefix length too short");
	if (len > GIT_OID_HEXSZ)
		len = GIT_OID_HEXSZ;

	if (len == GIT_OID_HEXSZ) {
		if (git_odb_exists(db, short_id)) {
			if (out)
				git_oid_cpy(out, short_id);
			return 0;
		} else {
			return git_odb__error_notfound("no match for id prefix", short_id);
		}
	}

672 673 674 675 676
	/* just copy valid part of short_id */
	memcpy(&key.id, short_id->id, (len + 1) / 2);
	if (len & 1)
		key.id[len / 2] &= 0xF0;

677 678 679 680 681 682 683
	for (i = 0; i < db->backends.length; ++i) {
		backend_internal *internal = git_vector_get(&db->backends, i);
		git_odb_backend *b = internal->backend;

		if (!b->exists_prefix)
			continue;

684
		error = b->exists_prefix(&found, b, &key, len);
685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700
		if (error == GIT_ENOTFOUND || error == GIT_PASSTHROUGH)
			continue;
		if (error)
			return error;

		/* make sure found item doesn't introduce ambiguity */
		if (num_found) {
			if (git_oid__cmp(&last_found, &found))
				return git_odb__error_ambiguous("multiple matches for prefix");
		} else {
			git_oid_cpy(&last_found, &found);
			num_found++;
		}
	}

	if (!num_found)
701
		return git_odb__error_notfound("no match for id prefix", &key);
702 703 704
	if (out)
		git_oid_cpy(out, &last_found);

705
	return 0;
706 707
}

Vicent Marti committed
708
int git_odb_read_header(size_t *len_p, git_otype *type_p, git_odb *db, const git_oid *id)
709
{
710 711 712 713 714 715 716 717 718 719 720 721 722 723 724
	int error;
	git_odb_object *object;

	error = git_odb__read_header_or_object(&object, len_p, type_p, db, id);

	if (object)
		git_odb_object_free(object);

	return error;
}

int git_odb__read_header_or_object(
	git_odb_object **out, size_t *len_p, git_otype *type_p,
	git_odb *db, const git_oid *id)
{
725
	size_t i;
726
	int error = GIT_ENOTFOUND;
Vicent Marti committed
727
	git_odb_object *object;
728

729
	assert(db && id && out && len_p && type_p);
Vicent Marti committed
730

731
	if ((object = git_cache_get_raw(odb_cache(db), id)) != NULL) {
Vicent Marti committed
732 733
		*len_p = object->cached.size;
		*type_p = object->cached.type;
734
		*out = object;
735
		return 0;
Vicent Marti committed
736
	}
737

738 739
	*out = NULL;

740
	for (i = 0; i < db->backends.length && error < 0; ++i) {
741 742
		backend_internal *internal = git_vector_get(&db->backends, i);
		git_odb_backend *b = internal->backend;
743

744
		if (b->read_header != NULL)
Vicent Marti committed
745
			error = b->read_header(len_p, type_p, b, id);
746 747
	}

748
	if (!error || error == GIT_PASSTHROUGH)
749
		return 0;
Vicent Marti committed
750

751 752 753 754
	/*
	 * no backend could read only the header.
	 * try reading the whole object and freeing the contents
	 */
755 756
	if ((error = git_odb_read(&object, db, id)) < 0)
		return error; /* error already set - pass along */
757

Vicent Marti committed
758 759
	*len_p = object->cached.size;
	*type_p = object->cached.type;
760 761
	*out = object;

762
	return 0;
763 764
}

765 766 767 768 769 770 771 772 773 774
static git_oid empty_blob = {{ 0xe6, 0x9d, 0xe2, 0x9b, 0xb2, 0xd1, 0xd6, 0x43, 0x4b, 0x8b,
			       0x29, 0xae, 0x77, 0x5a, 0xd8, 0xc2, 0xe4, 0x8c, 0x53, 0x91 }};
static git_oid empty_tree = {{ 0x4b, 0x82, 0x5d, 0xc6, 0x42, 0xcb, 0x6e, 0xb9, 0xa0, 0x60,
			       0xe5, 0x4b, 0xf8, 0xd6, 0x92, 0x88, 0xfb, 0xee, 0x49, 0x04 }};

static int hardcoded_objects(git_rawobj *raw, const git_oid *id)
{
	if (!git_oid_cmp(id, &empty_blob)) {
		raw->type = GIT_OBJ_BLOB;
		raw->len = 0;
775
		raw->data = git__calloc(1, sizeof(uint8_t));
776 777 778 779
		return 0;
	} else if (!git_oid_cmp(id, &empty_tree)) {
		raw->type = GIT_OBJ_TREE;
		raw->len = 0;
780
		raw->data = git__calloc(1, sizeof(uint8_t));
781 782 783 784 785 786
		return 0;
	} else {
		return GIT_ENOTFOUND;
	}
}

Vicent Marti committed
787
int git_odb_read(git_odb_object **out, git_odb *db, const git_oid *id)
788
{
789
	size_t i, reads = 0;
Vicent Marti committed
790
	int error;
Vicent Marti committed
791
	git_rawobj raw;
792
	git_odb_object *object;
793

794
	assert(out && db && id);
795

796
	*out = git_cache_get_raw(odb_cache(db), id);
Vicent Marti committed
797
	if (*out != NULL)
798
		return 0;
Vicent Marti committed
799

800
	error = hardcoded_objects(&raw, id);
Vicent Marti committed
801

802
	for (i = 0; i < db->backends.length && error < 0; ++i) {
803 804
		backend_internal *internal = git_vector_get(&db->backends, i);
		git_odb_backend *b = internal->backend;
805

806
		if (b->read != NULL) {
807
			++reads;
Vicent Marti committed
808
			error = b->read(&raw.data, &raw.len, &raw.type, b, id);
809
		}
Vicent Marti committed
810 811
	}

812 813 814
	if (error && error != GIT_PASSTHROUGH) {
		if (!reads)
			return git_odb__error_notfound("no match for id", id);
815
		return error;
816
	}
817

818
	giterr_clear();
819 820 821 822
	if ((object = odb_object__alloc(id, &raw)) == NULL)
		return -1;

	*out = git_cache_store_raw(odb_cache(db), object);
823
	return 0;
824 825
}

826
int git_odb_read_prefix(
827
	git_odb_object **out, git_odb *db, const git_oid *short_id, size_t len)
828
{
829
	size_t i;
830
	int error = GIT_ENOTFOUND;
831
	git_oid key = {{0}}, found_full_oid = {{0}};
832
	git_rawobj raw;
833
	void *data = NULL;
834
	bool found = false;
835
	git_odb_object *object;
836

Vicent Marti committed
837
	assert(out && db);
838

839
	if (len < GIT_OID_MINPREFIXLEN)
840
		return git_odb__error_ambiguous("prefix length too short");
841 842 843 844
	if (len > GIT_OID_HEXSZ)
		len = GIT_OID_HEXSZ;

	if (len == GIT_OID_HEXSZ) {
845
		*out = git_cache_get_raw(odb_cache(db), short_id);
Vicent Marti committed
846
		if (*out != NULL)
847
			return 0;
848 849
	}

850 851 852 853 854
	/* just copy valid part of short_id */
	memcpy(&key.id, short_id->id, (len + 1) / 2);
	if (len & 1)
		key.id[len / 2] &= 0xF0;

855
	for (i = 0; i < db->backends.length; ++i) {
856 857 858
		backend_internal *internal = git_vector_get(&db->backends, i);
		git_odb_backend *b = internal->backend;

859
		if (b->read_prefix != NULL) {
860
			git_oid full_oid;
861
			error = b->read_prefix(&full_oid, &raw.data, &raw.len, &raw.type, b, &key, len);
862
			if (error == GIT_ENOTFOUND || error == GIT_PASSTHROUGH)
863 864 865
				continue;

			if (error)
866
				return error;
867

868 869
			git__free(data);
			data = raw.data;
Vicent Marti committed
870

871 872
			if (found && git_oid__cmp(&full_oid, &found_full_oid)) {
				git__free(raw.data);
873
				return git_odb__error_ambiguous("multiple matches for prefix");
874
			}
Vicent Marti committed
875

876 877
			found_full_oid = full_oid;
			found = true;
878 879 880
		}
	}

881
	if (!found)
882
		return git_odb__error_notfound("no match for prefix", &key);
883

884 885 886 887
	if ((object = odb_object__alloc(&found_full_oid, &raw)) == NULL)
		return -1;

	*out = git_cache_store_raw(odb_cache(db), object);
888
	return 0;
889 890
}

Ben Straub committed
891
int git_odb_foreach(git_odb *db, git_odb_foreach_cb cb, void *payload)
892 893 894
{
	unsigned int i;
	backend_internal *internal;
895

896 897
	git_vector_foreach(&db->backends, i, internal) {
		git_odb_backend *b = internal->backend;
Ben Straub committed
898
		int error = b->foreach(b, cb, payload);
899 900
		if (error < 0)
			return error;
901 902 903 904 905
	}

	return 0;
}

906 907
int git_odb_write(
	git_oid *oid, git_odb *db, const void *data, size_t len, git_otype type)
908
{
909
	size_t i;
910
	int error = GIT_ERROR;
Vicent Marti committed
911
	git_odb_stream *stream;
912 913 914

	assert(oid && db);

915 916 917 918
	git_odb_hash(oid, data, len, type);
	if (git_odb_exists(db, oid))
		return 0;

919 920 921 922 923 924 925 926 927
	for (i = 0; i < db->backends.length && error < 0; ++i) {
		backend_internal *internal = git_vector_get(&db->backends, i);
		git_odb_backend *b = internal->backend;

		/* we don't write in alternates! */
		if (internal->is_alternate)
			continue;

		if (b->write != NULL)
928
			error = b->write(b, oid, data, len, type);
929 930
	}

931
	if (!error || error == GIT_PASSTHROUGH)
932
		return 0;
Vicent Marti committed
933

934 935 936 937
	/* if no backends were able to write the object directly, we try a
	 * streaming write to the backends; just write the whole object into the
	 * stream in one push
	 */
938 939
	if ((error = git_odb_open_wstream(&stream, db, len, type)) != 0)
		return error;
940

941 942
	stream->write(stream, data, len);
	error = stream->finalize_write(stream, oid);
943
	git_odb_stream_free(stream);
944 945

	return error;
946 947
}

948
static void hash_header(git_hash_ctx *ctx, git_off_t size, git_otype type)
949 950 951 952 953 954 955 956
{
	char header[64];
	int hdrlen;

	hdrlen = git_odb__format_object_header(header, sizeof(header), size, type);
	git_hash_update(ctx, header, hdrlen);
}

957
int git_odb_open_wstream(
958
	git_odb_stream **stream, git_odb *db, git_off_t size, git_otype type)
959
{
960
	size_t i, writes = 0;
961
	int error = GIT_ERROR;
962
	git_hash_ctx *ctx = NULL;
963

Vicent Marti committed
964
	assert(stream && db);
965

966
	for (i = 0; i < db->backends.length && error < 0; ++i) {
967 968 969 970 971 972
		backend_internal *internal = git_vector_get(&db->backends, i);
		git_odb_backend *b = internal->backend;

		/* we don't write in alternates! */
		if (internal->is_alternate)
			continue;
973

974 975
		if (b->writestream != NULL) {
			++writes;
Vicent Marti committed
976
			error = b->writestream(stream, b, size, type);
977 978
		} else if (b->write != NULL) {
			++writes;
979
			error = init_fake_wstream(stream, b, size, type);
980
		}
Vicent Marti committed
981 982
	}

983 984 985 986 987 988 989 990
	if (error < 0) {
		if (error == GIT_PASSTHROUGH)
			error = 0;
		else if (!writes)
			error = git_odb__error_unsupported_in_backend("write object");

		goto done;
	}
Vicent Marti committed
991

992 993 994
	ctx = git__malloc(sizeof(git_hash_ctx));
	GITERR_CHECK_ALLOC(ctx);

995 996
	if ((error = git_hash_ctx_init(ctx)) < 0)
		goto done;
997 998 999 1000

	hash_header(ctx, size, type);
	(*stream)->hash_ctx = ctx;

1001 1002 1003
	(*stream)->declared_size = size;
	(*stream)->received_bytes = 0;

1004
done:
1005
	return error;
Vicent Marti committed
1006 1007
}

1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020
static int git_odb_stream__invalid_length(
	const git_odb_stream *stream,
	const char *action)
{
	giterr_set(GITERR_ODB,
		"Cannot %s - "
		"Invalid length. %"PRIuZ" was expected. The "
		"total size of the received chunks amounts to %"PRIuZ".",
		action, stream->declared_size, stream->received_bytes);		

	return -1;
}

1021 1022
int git_odb_stream_write(git_odb_stream *stream, const char *buffer, size_t len)
{
1023
	git_hash_update(stream->hash_ctx, buffer, len);
1024 1025 1026 1027 1028 1029 1030

	stream->received_bytes += len;

	if (stream->received_bytes > stream->declared_size)
		return git_odb_stream__invalid_length(stream,
			"stream_write()");

1031 1032 1033 1034 1035
	return stream->write(stream, buffer, len);
}

int git_odb_stream_finalize_write(git_oid *out, git_odb_stream *stream)
{
1036 1037 1038 1039
	if (stream->received_bytes != stream->declared_size)
		return git_odb_stream__invalid_length(stream,
			"stream_finalize_write()");

1040
	git_hash_final(out, stream->hash_ctx);
1041 1042 1043 1044

	if (git_odb_exists(stream->backend->odb, out))
		return 0;

1045
	return stream->finalize_write(stream, out);
1046 1047 1048 1049 1050 1051 1052 1053 1054
}

int git_odb_stream_read(git_odb_stream *stream, char *buffer, size_t len)
{
	return stream->read(stream, buffer, len);
}

void git_odb_stream_free(git_odb_stream *stream)
{
1055 1056 1057
	if (stream == NULL)
		return;

1058
	git_hash_ctx_cleanup(stream->hash_ctx);
1059
	git__free(stream->hash_ctx);
1060 1061 1062
	stream->free(stream);
}

1063
int git_odb_open_rstream(git_odb_stream **stream, git_odb *db, const git_oid *oid)
Vicent Marti committed
1064
{
1065
	size_t i, reads = 0;
Vicent Marti committed
1066 1067 1068 1069 1070 1071 1072 1073
	int error = GIT_ERROR;

	assert(stream && db);

	for (i = 0; i < db->backends.length && error < 0; ++i) {
		backend_internal *internal = git_vector_get(&db->backends, i);
		git_odb_backend *b = internal->backend;

1074 1075
		if (b->readstream != NULL) {
			++reads;
Vicent Marti committed
1076
			error = b->readstream(stream, b, oid);
1077
		}
1078 1079
	}

1080
	if (error == GIT_PASSTHROUGH)
1081
		error = 0;
1082 1083
	if (error < 0 && !reads)
		error = git_odb__error_unsupported_in_backend("read object streamed");
Vicent Marti committed
1084

1085 1086 1087
	return error;
}

1088
int git_odb_write_pack(struct git_odb_writepack **out, git_odb *db, git_transfer_progress_cb progress_cb, void *progress_payload)
1089
{
1090
	size_t i, writes = 0;
1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102
	int error = GIT_ERROR;

	assert(out && db);

	for (i = 0; i < db->backends.length && error < 0; ++i) {
		backend_internal *internal = git_vector_get(&db->backends, i);
		git_odb_backend *b = internal->backend;

		/* we don't write in alternates! */
		if (internal->is_alternate)
			continue;

1103 1104
		if (b->writepack != NULL) {
			++writes;
1105
			error = b->writepack(out, b, db, progress_cb, progress_payload);
1106
		}
1107 1108 1109 1110
	}

	if (error == GIT_PASSTHROUGH)
		error = 0;
1111 1112
	if (error < 0 && !writes)
		error = git_odb__error_unsupported_in_backend("write pack");
1113 1114 1115 1116

	return error;
}

Vicent Marti committed
1117
void *git_odb_backend_malloc(git_odb_backend *backend, size_t len)
1118
{
1119
	GIT_UNUSED(backend);
1120 1121 1122
	return git__malloc(len);
}

Vicent Marti committed
1123 1124
int git_odb_refresh(struct git_odb *db)
{
1125
	size_t i;
Vicent Marti committed
1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141
	assert(db);

	for (i = 0; i < db->backends.length; ++i) {
		backend_internal *internal = git_vector_get(&db->backends, i);
		git_odb_backend *b = internal->backend;

		if (b->refresh != NULL) {
			int error = b->refresh(b);
			if (error < 0)
				return error;
		}
	}

	return 0;
}

Russell Belfer committed
1142
int git_odb__error_notfound(const char *message, const git_oid *oid)
1143
{
Russell Belfer committed
1144 1145 1146 1147 1148 1149 1150
	if (oid != NULL) {
		char oid_str[GIT_OID_HEXSZ + 1];
		git_oid_tostr(oid_str, sizeof(oid_str), oid);
		giterr_set(GITERR_ODB, "Object not found - %s (%s)", message, oid_str);
	} else
		giterr_set(GITERR_ODB, "Object not found - %s", message);

1151
	return GIT_ENOTFOUND;
1152 1153 1154 1155 1156
}

int git_odb__error_ambiguous(const char *message)
{
	giterr_set(GITERR_ODB, "Ambiguous SHA1 prefix - %s", message);
1157
	return GIT_EAMBIGUOUS;
1158 1159
}

1160
int git_odb_init_backend(git_odb_backend *backend, unsigned int version)
1161
{
1162 1163 1164
	GIT_INIT_STRUCTURE_FROM_TEMPLATE(
		backend, version, git_odb_backend, GIT_ODB_BACKEND_INIT);
	return 0;
1165
}