odb.c 40.6 KB
Newer Older
1
/*
Edward Thomson committed
2
 * Copyright (C) the libgit2 contributors. All rights reserved.
3
 *
Vicent Marti committed
4 5
 * This file is part of libgit2, distributed under the GNU GPL v2 with
 * a Linking Exception. For full terms see the included COPYING file.
6 7
 */

8 9
#include "odb.h"

10
#include <zlib.h>
11
#include "git2/object.h"
12
#include "git2/sys/odb_backend.h"
13
#include "futils.h"
14
#include "hash.h"
15
#include "delta.h"
16
#include "filter.h"
17
#include "repository.h"
18
#include "blob.h"
19

20
#include "git2/odb_backend.h"
21
#include "git2/oid.h"
22
#include "git2/oidarray.h"
23

24 25
#define GIT_ALTERNATES_FILE "info/alternates"

26 27
#define GIT_ALTERNATES_MAX_DEPTH 5

28 29 30 31
/*
 * We work under the assumption that most objects for long-running
 * operations will be packed
 */
32 33
int git_odb__loose_priority = GIT_ODB_DEFAULT_LOOSE_PRIORITY;
int git_odb__packed_priority = GIT_ODB_DEFAULT_PACKED_PRIORITY;
34

35 36
bool git_odb__strict_hash_verification = true;

37 38 39 40
typedef struct
{
	git_odb_backend *backend;
	int priority;
41 42
	bool is_alternate;
	ino_t disk_inode;
43 44
} backend_internal;

45 46
static git_cache *odb_cache(git_odb *odb)
{
47 48
	git_repository *owner = GIT_REFCOUNT_OWNER(odb);
	if (owner != NULL) {
49 50 51 52 53
		return &owner->objects;
	}

	return &odb->own_cache;
}
54

55
static int odb_otype_fast(git_object_t *type_p, git_odb *db, const git_oid *id);
56
static int load_alternates(git_odb *odb, const char *objects_dir, int alternate_depth);
57
static int error_null_oid(int error, const char *message);
58

59
static git_object_t odb_hardcoded_type(const git_oid *id)
60 61 62 63 64
{
	static git_oid empty_tree = {{ 0x4b, 0x82, 0x5d, 0xc6, 0x42, 0xcb, 0x6e, 0xb9, 0xa0, 0x60,
					   0xe5, 0x4b, 0xf8, 0xd6, 0x92, 0x88, 0xfb, 0xee, 0x49, 0x04 }};

	if (!git_oid_cmp(id, &empty_tree))
65
		return GIT_OBJECT_TREE;
66

67
	return GIT_OBJECT_INVALID;
68 69
}

70
static int odb_read_hardcoded(bool *found, git_rawobj *raw, const git_oid *id)
71
{
72
	git_object_t type;
73 74 75

	*found = false;

76
	if ((type = odb_hardcoded_type(id)) == GIT_OBJECT_INVALID)
77
		return 0;
78 79 80 81

	raw->type = type;
	raw->len = 0;
	raw->data = git__calloc(1, sizeof(uint8_t));
82
	GIT_ERROR_CHECK_ALLOC(raw->data);
83 84

	*found = true;
85 86 87
	return 0;
}

88 89 90 91
int git_odb__format_object_header(
	size_t *written,
	char *hdr,
	size_t hdr_size,
92
	git_object_size_t obj_len,
93
	git_object_t obj_type)
94
{
Vicent Marti committed
95
	const char *type_str = git_object_type2string(obj_type);
96 97 98
	int hdr_max = (hdr_size > INT_MAX-2) ? (INT_MAX-2) : (int)hdr_size;
	int len;

lhchavez committed
99
	len = p_snprintf(hdr, hdr_max, "%s %"PRId64, type_str, (int64_t)obj_len);
100 101

	if (len < 0 || len >= hdr_max) {
102
		git_error_set(GIT_ERROR_OS, "object header creation failed");
103 104 105 106 107
		return -1;
	}

	*written = (size_t)(len + 1);
	return 0;
108 109
}

110
int git_odb__hashobj(git_oid *id, git_rawobj *obj)
111 112
{
	git_buf_vec vec[2];
113
	char header[64];
114 115
	size_t hdrlen;
	int error;
116

Edward Thomson committed
117 118
	GIT_ASSERT_ARG(id);
	GIT_ASSERT_ARG(obj);
119

120
	if (!git_object_typeisloose(obj->type)) {
121
		git_error_set(GIT_ERROR_INVALID, "invalid object type");
122
		return -1;
123
	}
Vicent Marti committed
124

125
	if (!obj->data && obj->len != 0) {
126
		git_error_set(GIT_ERROR_INVALID, "invalid object");
127
		return -1;
128
	}
129

130 131 132
	if ((error = git_odb__format_object_header(&hdrlen,
		header, sizeof(header), obj->len, obj->type)) < 0)
		return error;
133

134
	vec[0].data = header;
Vicent Marti committed
135
	vec[0].len = hdrlen;
136
	vec[1].data = obj->data;
Vicent Marti committed
137
	vec[1].len = obj->len;
138

139
	return git_hash_vec(id, vec, 2);
140 141
}

142

143
static git_odb_object *odb_object__alloc(const git_oid *oid, git_rawobj *source)
Ramsay Jones committed
144
{
145
	git_odb_object *object = git__calloc(1, sizeof(git_odb_object));
Ramsay Jones committed
146

147 148 149 150 151 152
	if (object != NULL) {
		git_oid_cpy(&object->cached.oid, oid);
		object->cached.type = source->type;
		object->cached.size = source->len;
		object->buffer      = source->data;
	}
Ramsay Jones committed
153

Vicent Marti committed
154
	return object;
155 156
}

157
void git_odb_object__free(void *object)
158
{
Vicent Marti committed
159
	if (object != NULL) {
160
		git__free(((git_odb_object *)object)->buffer);
161
		git__free(object);
Vicent Marti committed
162 163
	}
}
164

165 166 167 168 169 170 171
const git_oid *git_odb_object_id(git_odb_object *object)
{
	return &object->cached.oid;
}

const void *git_odb_object_data(git_odb_object *object)
{
Vicent Marti committed
172
	return object->buffer;
173 174 175 176
}

size_t git_odb_object_size(git_odb_object *object)
{
Vicent Marti committed
177
	return object->cached.size;
178 179
}

180
git_object_t git_odb_object_type(git_odb_object *object)
181
{
Vicent Marti committed
182
	return object->cached.type;
183 184
}

185 186 187 188 189 190 191
int git_odb_object_dup(git_odb_object **dest, git_odb_object *source)
{
	git_cached_obj_incref(source);
	*dest = source;
	return 0;
}

192
void git_odb_object_free(git_odb_object *object)
Vicent Marti committed
193
{
194 195 196
	if (object == NULL)
		return;

197
	git_cached_obj_decref(object);
Vicent Marti committed
198
}
199

200
int git_odb__hashfd(git_oid *out, git_file fd, size_t size, git_object_t type)
Vicent Marti committed
201
{
202
	size_t hdr_len;
203
	char hdr[64], buffer[FILEIO_BUFSIZE];
204
	git_hash_ctx ctx;
205
	ssize_t read_len = 0;
206
	int error = 0;
Vicent Marti committed
207

208
	if (!git_object_typeisloose(type)) {
209
		git_error_set(GIT_ERROR_INVALID, "invalid object type for hash");
210 211 212
		return -1;
	}

213
	if ((error = git_hash_ctx_init(&ctx)) < 0)
214
		return error;
Vicent Marti committed
215

216 217 218
	if ((error = git_odb__format_object_header(&hdr_len, hdr,
		sizeof(hdr), size, type)) < 0)
		goto done;
219

220
	if ((error = git_hash_update(&ctx, hdr, hdr_len)) < 0)
221
		goto done;
Vicent Marti committed
222

Vicent Marti committed
223
	while (size > 0 && (read_len = p_read(fd, buffer, sizeof(buffer))) > 0) {
224
		if ((error = git_hash_update(&ctx, buffer, read_len)) < 0)
225 226
			goto done;

Vicent Marti committed
227 228 229
		size -= read_len;
	}

Vicent Marti committed
230 231 232 233
	/* If p_read returned an error code, the read obviously failed.
	 * If size is not zero, the file was truncated after we originally
	 * stat'd it, so we consider this a read failure too */
	if (read_len < 0 || size > 0) {
234
		git_error_set(GIT_ERROR_OS, "error reading file for hashing");
235 236 237
		error = -1;

		goto done;
Vicent Marti committed
238 239
	}

240
	error = git_hash_final(out, &ctx);
Vicent Marti committed
241

242
done:
243
	git_hash_ctx_cleanup(&ctx);
244
	return error;
Vicent Marti committed
245 246
}

247
int git_odb__hashfd_filtered(
248
	git_oid *out, git_file fd, size_t size, git_object_t type, git_filter_list *fl)
249 250 251 252
{
	int error;
	git_buf raw = GIT_BUF_INIT;

253
	if (!fl)
254 255 256 257 258 259
		return git_odb__hashfd(out, fd, size, type);

	/* size of data is used in header, so we have to read the whole file
	 * into memory to apply filters before beginning to calculate the hash
	 */

260
	if (!(error = git_futils_readbuffer_fd(&raw, fd, size))) {
261
		git_buf post = GIT_BUF_INIT;
262

263
		error = git_filter_list__convert_buf(&post, fl, &raw);
264

265 266 267
		if (!error)
			error = git_odb_hash(out, post.ptr, post.size, type);

268
		git_buf_dispose(&post);
269
	}
270 271 272 273

	return error;
}

274 275 276
int git_odb__hashlink(git_oid *out, const char *path)
{
	struct stat st;
277
	int size;
278
	int result;
279

280
	if (git_path_lstat(path, &st) < 0)
281
		return -1;
282

283
	if (!git__is_int(st.st_size) || (int)st.st_size < 0) {
284
		git_error_set(GIT_ERROR_FILESYSTEM, "file size overflow for 32-bit systems");
285 286
		return -1;
	}
287

288
	size = (int)st.st_size;
289

290 291
	if (S_ISLNK(st.st_mode)) {
		char *link_data;
292 293
		int read_len;
		size_t alloc_size;
294

295
		GIT_ERROR_CHECK_ALLOC_ADD(&alloc_size, size, 1);
296
		link_data = git__malloc(alloc_size);
297
		GIT_ERROR_CHECK_ALLOC(link_data);
298

299
		read_len = p_readlink(path, link_data, size);
300
		if (read_len == -1) {
301
			git_error_set(GIT_ERROR_OS, "failed to read symlink data for '%s'", path);
302
			git__free(link_data);
303 304
			return -1;
		}
305 306
		GIT_ASSERT(read_len <= size);
		link_data[read_len] = '\0';
307

308
		result = git_odb_hash(out, link_data, read_len, GIT_OBJECT_BLOB);
309
		git__free(link_data);
310
	} else {
311 312 313
		int fd = git_futils_open_ro(path);
		if (fd < 0)
			return -1;
314
		result = git_odb__hashfd(out, fd, size, GIT_OBJECT_BLOB);
315 316 317
		p_close(fd);
	}

318
	return result;
319 320
}

321
int git_odb_hashfile(git_oid *out, const char *path, git_object_t type)
322
{
323 324 325 326
	uint64_t size;
	int fd, error = 0;

	if ((fd = git_futils_open_ro(path)) < 0)
327
		return fd;
328

329 330 331 332
	if ((error = git_futils_filesize(&size, fd)) < 0)
		goto done;

	if (!git__is_sizet(size)) {
333
		git_error_set(GIT_ERROR_OS, "file size overflow for 32-bit systems");
334 335
		error = -1;
		goto done;
336 337
	}

338 339 340
	error = git_odb__hashfd(out, fd, (size_t)size, type);

done:
341
	p_close(fd);
342
	return error;
343 344
}

345
int git_odb_hash(git_oid *id, const void *data, size_t len, git_object_t type)
Vicent Marti committed
346 347
{
	git_rawobj raw;
348

Edward Thomson committed
349
	GIT_ASSERT_ARG(id);
350

Vicent Marti committed
351 352 353
	raw.data = (void *)data;
	raw.len = len;
	raw.type = type;
354

355
	return git_odb__hashobj(id, &raw);
356 357
}

358 359 360 361 362 363 364 365
/**
 * FAKE WSTREAM
 */

typedef struct {
	git_odb_stream stream;
	char *buffer;
	size_t size, written;
366
	git_object_t type;
367 368
} fake_wstream;

369
static int fake_wstream__fwrite(git_odb_stream *_stream, const git_oid *oid)
370 371
{
	fake_wstream *stream = (fake_wstream *)_stream;
372
	return _stream->backend->write(_stream->backend, oid, stream->buffer, stream->size, stream->type);
373 374 375 376 377 378
}

static int fake_wstream__write(git_odb_stream *_stream, const char *data, size_t len)
{
	fake_wstream *stream = (fake_wstream *)_stream;

Edward Thomson committed
379
	GIT_ASSERT(stream->written + len <= stream->size);
380 381 382

	memcpy(stream->buffer + stream->written, data, len);
	stream->written += len;
383
	return 0;
384 385 386 387 388 389
}

static void fake_wstream__free(git_odb_stream *_stream)
{
	fake_wstream *stream = (fake_wstream *)_stream;

390 391
	git__free(stream->buffer);
	git__free(stream);
392 393
}

394
static int init_fake_wstream(git_odb_stream **stream_p, git_odb_backend *backend, git_object_size_t size, git_object_t type)
395 396
{
	fake_wstream *stream;
397
	size_t blobsize;
398

399 400
	GIT_ERROR_CHECK_BLOBSIZE(size);
	blobsize = (size_t)size;
401

402
	stream = git__calloc(1, sizeof(fake_wstream));
403
	GIT_ERROR_CHECK_ALLOC(stream);
404

405
	stream->size = blobsize;
406
	stream->type = type;
407
	stream->buffer = git__malloc(blobsize);
408
	if (stream->buffer == NULL) {
409
		git__free(stream);
410
		return -1;
411 412 413 414 415 416 417 418 419 420
	}

	stream->stream.backend = backend;
	stream->stream.read = NULL; /* read only */
	stream->stream.write = &fake_wstream__write;
	stream->stream.finalize_write = &fake_wstream__fwrite;
	stream->stream.free = &fake_wstream__free;
	stream->stream.mode = GIT_STREAM_WRONLY;

	*stream_p = (git_odb_stream *)stream;
421
	return 0;
422
}
423

424 425 426 427 428 429 430
/***********************************************************
 *
 * OBJECT DATABASE PUBLIC API
 *
 * Public calls for the ODB functionality
 *
 ***********************************************************/
431

432
static int backend_sort_cmp(const void *a, const void *b)
433
{
434 435
	const backend_internal *backend_a = (const backend_internal *)(a);
	const backend_internal *backend_b = (const backend_internal *)(b);
436

437 438 439 440 441 442 443 444
	if (backend_b->priority == backend_a->priority) {
		if (backend_a->is_alternate)
			return -1;
		if (backend_b->is_alternate)
			return 1;
		return 0;
	}
	return (backend_b->priority - backend_a->priority);
445 446
}

447
int git_odb_new(git_odb **out)
448
{
449
	git_odb *db = git__calloc(1, sizeof(*db));
450
	GIT_ERROR_CHECK_ALLOC(db);
451

lhchavez committed
452 453 454 455
	if (git_mutex_init(&db->lock) < 0) {
		git__free(db);
		return -1;
	}
456
	if (git_cache_init(&db->own_cache) < 0) {
lhchavez committed
457
		git_mutex_free(&db->lock);
458 459 460 461
		git__free(db);
		return -1;
	}
	if (git_vector_init(&db->backends, 4, backend_sort_cmp) < 0) {
462
		git_cache_dispose(&db->own_cache);
lhchavez committed
463
		git_mutex_free(&db->lock);
464
		git__free(db);
465
		return -1;
466
	}
467

468
	*out = db;
469
	GIT_REFCOUNT_INC(db);
470
	return 0;
471 472
}

473 474 475
static int add_backend_internal(
	git_odb *odb, git_odb_backend *backend,
	int priority, bool is_alternate, ino_t disk_inode)
Ramsay Jones committed
476
{
477 478
	backend_internal *internal;

Edward Thomson committed
479 480
	GIT_ASSERT_ARG(odb);
	GIT_ASSERT_ARG(backend);
Ramsay Jones committed
481

482
	GIT_ERROR_CHECK_VERSION(backend, GIT_ODB_BACKEND_VERSION, "git_odb_backend");
483

484
	/* Check if the backend is already owned by another ODB */
Edward Thomson committed
485
	GIT_ASSERT(!backend->odb || backend->odb == odb);
Ramsay Jones committed
486

487
	internal = git__malloc(sizeof(backend_internal));
488
	GIT_ERROR_CHECK_ALLOC(internal);
489 490 491 492

	internal->backend = backend;
	internal->priority = priority;
	internal->is_alternate = is_alternate;
493
	internal->disk_inode = disk_inode;
Ramsay Jones committed
494

lhchavez committed
495 496 497 498
	if (git_mutex_lock(&odb->lock) < 0) {
		git_error_set(GIT_ERROR_ODB, "failed to acquire the odb lock");
		return -1;
	}
499
	if (git_vector_insert(&odb->backends, internal) < 0) {
lhchavez committed
500
		git_mutex_unlock(&odb->lock);
501
		git__free(internal);
502
		return -1;
503
	}
504
	git_vector_sort(&odb->backends);
505
	internal->backend->odb = odb;
lhchavez committed
506
	git_mutex_unlock(&odb->lock);
507
	return 0;
Ramsay Jones committed
508 509
}

510 511
int git_odb_add_backend(git_odb *odb, git_odb_backend *backend, int priority)
{
512
	return add_backend_internal(odb, backend, priority, false, 0);
513 514 515 516
}

int git_odb_add_alternate(git_odb *odb, git_odb_backend *backend, int priority)
{
517
	return add_backend_internal(odb, backend, priority, true, 0);
518 519
}

520 521
size_t git_odb_num_backends(git_odb *odb)
{
lhchavez committed
522 523 524
	size_t length;
	bool locked = true;

Edward Thomson committed
525
	GIT_ASSERT_ARG(odb);
lhchavez committed
526 527 528 529 530 531 532 533 534

	if (git_mutex_lock(&odb->lock) < 0) {
		git_error_set(GIT_ERROR_ODB, "failed to acquire the odb lock");
		locked = false;
	}
	length = odb->backends.length;
	if (locked)
		git_mutex_unlock(&odb->lock);
	return length;
535 536
}

537 538
static int git_odb__error_unsupported_in_backend(const char *action)
{
539
	git_error_set(GIT_ERROR_ODB,
540
		"cannot %s - unsupported in the loaded odb backends", action);
541 542 543 544
	return -1;
}


545 546 547
int git_odb_get_backend(git_odb_backend **out, git_odb *odb, size_t pos)
{
	backend_internal *internal;
lhchavez committed
548
	int error;
549

Edward Thomson committed
550 551 552
	GIT_ASSERT_ARG(out);
	GIT_ASSERT_ARG(odb);

lhchavez committed
553 554 555 556 557

	if ((error = git_mutex_lock(&odb->lock)) < 0) {
		git_error_set(GIT_ERROR_ODB, "failed to acquire the odb lock");
		return error;
	}
558 559
	internal = git_vector_get(&odb->backends, pos);

lhchavez committed
560 561 562 563 564
	if (!internal || !internal->backend) {
		git_mutex_unlock(&odb->lock);

		git_error_set(GIT_ERROR_ODB, "no ODB backend loaded at index %" PRIuZ, pos);
		return GIT_ENOTFOUND;
565
	}
lhchavez committed
566 567
	*out = internal->backend;
	git_mutex_unlock(&odb->lock);
568

lhchavez committed
569
	return 0;
570 571
}

572
int git_odb__add_default_backends(
573 574
	git_odb *db, const char *objects_dir,
	bool as_alternates, int alternate_depth)
575
{
576
	size_t i = 0;
577
	struct stat st;
578
	ino_t inode;
579 580
	git_odb_backend *loose, *packed;

581 582
	/* TODO: inodes are not really relevant on Win32, so we need to find
	 * a cross-platform workaround for this */
583 584
#ifdef GIT_WIN32
	GIT_UNUSED(i);
585
	GIT_UNUSED(&st);
586 587 588

	inode = 0;
#else
589
	if (p_stat(objects_dir, &st) < 0) {
590
		if (as_alternates)
Etienne Samson committed
591
			/* this should warn */
592 593
			return 0;

594
		git_error_set(GIT_ERROR_ODB, "failed to load object database in '%s'", objects_dir);
595 596 597
		return -1;
	}

598 599
	inode = st.st_ino;

lhchavez committed
600 601 602 603
	if (git_mutex_lock(&db->lock) < 0) {
		git_error_set(GIT_ERROR_ODB, "failed to acquire the odb lock");
		return -1;
	}
604 605
	for (i = 0; i < db->backends.length; ++i) {
		backend_internal *backend = git_vector_get(&db->backends, i);
lhchavez committed
606 607
		if (backend->disk_inode == inode) {
			git_mutex_unlock(&db->lock);
608
			return 0;
lhchavez committed
609
		}
610
	}
lhchavez committed
611
	git_mutex_unlock(&db->lock);
612
#endif
613

614
	/* add the loose object backend */
615
	if (git_odb_backend_loose(&loose, objects_dir, -1, db->do_fsync, 0, 0) < 0 ||
616
		add_backend_internal(db, loose, git_odb__loose_priority, as_alternates, inode) < 0)
617
		return -1;
618 619

	/* add the packed file backend */
620
	if (git_odb_backend_pack(&packed, objects_dir) < 0 ||
621
		add_backend_internal(db, packed, git_odb__packed_priority, as_alternates, inode) < 0)
622
		return -1;
623

624 625 626 627 628 629 630 631 632 633
	if (git_mutex_lock(&db->lock) < 0) {
		git_error_set(GIT_ERROR_ODB, "failed to acquire the odb lock");
		return -1;
	}
	if (!db->cgraph && git_commit_graph_new(&db->cgraph, objects_dir, false) < 0) {
		git_mutex_unlock(&db->lock);
		return -1;
	}
	git_mutex_unlock(&db->lock);

634
	return load_alternates(db, objects_dir, alternate_depth);
635 636
}

637
static int load_alternates(git_odb *odb, const char *objects_dir, int alternate_depth)
638
{
639
	git_buf alternates_path = GIT_BUF_INIT;
640
	git_buf alternates_buf = GIT_BUF_INIT;
641 642
	char *buffer;
	const char *alternate;
643
	int result = 0;
644

645
	/* Git reports an error, we just ignore anything deeper */
646
	if (alternate_depth > GIT_ALTERNATES_MAX_DEPTH)
647 648
		return 0;

649 650
	if (git_buf_joinpath(&alternates_path, objects_dir, GIT_ALTERNATES_FILE) < 0)
		return -1;
651

652
	if (git_path_exists(alternates_path.ptr) == false) {
653
		git_buf_dispose(&alternates_path);
654
		return 0;
655
	}
656

657
	if (git_futils_readbuffer(&alternates_buf, alternates_path.ptr) < 0) {
658
		git_buf_dispose(&alternates_path);
659
		return -1;
660
	}
661

662
	buffer = (char *)alternates_buf.ptr;
663 664

	/* add each alternate as a new backend; one alternate per line */
665 666 667 668
	while ((alternate = git__strtok(&buffer, "\r\n")) != NULL) {
		if (*alternate == '\0' || *alternate == '#')
			continue;

669 670 671 672 673 674
		/*
		 * Relative path: build based on the current `objects`
		 * folder. However, relative paths are only allowed in
		 * the current repository.
		 */
		if (*alternate == '.' && !alternate_depth) {
675
			if ((result = git_buf_joinpath(&alternates_path, objects_dir, alternate)) < 0)
676 677
				break;
			alternate = git_buf_cstr(&alternates_path);
678 679
		}

680
		if ((result = git_odb__add_default_backends(odb, alternate, true, alternate_depth + 1)) < 0)
681 682
			break;
	}
683

684 685
	git_buf_dispose(&alternates_path);
	git_buf_dispose(&alternates_buf);
686

687
	return result;
688
}
Ramsay Jones committed
689

690 691
int git_odb_add_disk_alternate(git_odb *odb, const char *path)
{
692
	return git_odb__add_default_backends(odb, path, true, 0);
693 694
}

695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711
int git_odb_set_commit_graph(git_odb *odb, git_commit_graph *cgraph)
{
	int error = 0;

	GIT_ASSERT_ARG(odb);

	if ((error = git_mutex_lock(&odb->lock)) < 0) {
		git_error_set(GIT_ERROR_ODB, "failed to acquire the db lock");
		return error;
	}
	git_commit_graph_free(odb->cgraph);
	odb->cgraph = cgraph;
	git_mutex_unlock(&odb->lock);

	return error;
}

712
int git_odb_open(git_odb **out, const char *objects_dir)
Ramsay Jones committed
713
{
714
	git_odb *db;
Ramsay Jones committed
715

Edward Thomson committed
716 717
	GIT_ASSERT_ARG(out);
	GIT_ASSERT_ARG(objects_dir);
718 719 720

	*out = NULL;

721 722
	if (git_odb_new(&db) < 0)
		return -1;
Ramsay Jones committed
723

724
	if (git_odb__add_default_backends(db, objects_dir, 0, 0) < 0) {
725 726 727
		git_odb_free(db);
		return -1;
	}
Ramsay Jones committed
728

729
	*out = db;
730
	return 0;
731
}
Ramsay Jones committed
732

733 734 735
int git_odb__set_caps(git_odb *odb, int caps)
{
	if (caps == GIT_ODB_CAP_FROM_OWNER) {
736
		git_repository *repo = GIT_REFCOUNT_OWNER(odb);
737 738 739
		int val;

		if (!repo) {
740
			git_error_set(GIT_ERROR_ODB, "cannot access repository to set odb caps");
741 742 743
			return -1;
		}

744
		if (!git_repository__configmap_lookup(&val, repo, GIT_CONFIGMAP_FSYNCOBJECTFILES))
745 746 747 748 749 750
			odb->do_fsync = !!val;
	}

	return 0;
}

751
static void odb_free(git_odb *db)
752
{
753
	size_t i;
lhchavez committed
754
	bool locked = true;
755

lhchavez committed
756 757 758 759
	if (git_mutex_lock(&db->lock) < 0) {
		git_error_set(GIT_ERROR_ODB, "failed to acquire the odb lock");
		locked = false;
	}
760
	for (i = 0; i < db->backends.length; ++i) {
761 762
		backend_internal *internal = git_vector_get(&db->backends, i);
		git_odb_backend *backend = internal->backend;
763

764
		backend->free(backend);
765

766
		git__free(internal);
767
	}
lhchavez committed
768 769
	if (locked)
		git_mutex_unlock(&db->lock);
770

771
	git_commit_graph_free(db->cgraph);
772
	git_vector_free(&db->backends);
773
	git_cache_dispose(&db->own_cache);
lhchavez committed
774
	git_mutex_free(&db->lock);
775

776
	git__memzero(db, sizeof(*db));
777
	git__free(db);
778 779
}

780 781 782 783 784 785 786 787
void git_odb_free(git_odb *db)
{
	if (db == NULL)
		return;

	GIT_REFCOUNT_DEC(db, odb_free);
}

788 789 790 791
static int odb_exists_1(
	git_odb *db,
	const git_oid *id,
	bool only_refreshed)
792
{
793
	size_t i;
794
	bool found = false;
lhchavez committed
795
	int error;
796

lhchavez committed
797 798 799 800
	if ((error = git_mutex_lock(&db->lock)) < 0) {
		git_error_set(GIT_ERROR_ODB, "failed to acquire the odb lock");
		return error;
	}
801
	for (i = 0; i < db->backends.length && !found; ++i) {
802 803
		backend_internal *internal = git_vector_get(&db->backends, i);
		git_odb_backend *b = internal->backend;
804

805 806 807
		if (only_refreshed && !b->refresh)
			continue;

808
		if (b->exists != NULL)
Linquize committed
809
			found = (bool)b->exists(b, id);
810
	}
lhchavez committed
811
	git_mutex_unlock(&db->lock);
812

813
	return (int)found;
814 815
}

816
int git_odb__get_commit_graph_file(git_commit_graph_file **out, git_odb *db)
817 818
{
	int error = 0;
819
	git_commit_graph_file *result = NULL;
820 821 822 823 824

	if ((error = git_mutex_lock(&db->lock)) < 0) {
		git_error_set(GIT_ERROR_ODB, "failed to acquire the db lock");
		return error;
	}
825 826 827
	if (!db->cgraph) {
		error = GIT_ENOTFOUND;
		goto done;
828
	}
829 830 831 832
	error = git_commit_graph_get_file(&result, db->cgraph);
	if (error)
		goto done;
	*out = result;
833 834 835

done:
	git_mutex_unlock(&db->lock);
836
	return error;
837 838
}

839 840 841 842 843 844 845
static int odb_freshen_1(
	git_odb *db,
	const git_oid *id,
	bool only_refreshed)
{
	size_t i;
	bool found = false;
lhchavez committed
846
	int error;
847

lhchavez committed
848 849 850 851
	if ((error = git_mutex_lock(&db->lock)) < 0) {
		git_error_set(GIT_ERROR_ODB, "failed to acquire the odb lock");
		return error;
	}
852 853 854 855 856 857 858 859 860 861 862 863
	for (i = 0; i < db->backends.length && !found; ++i) {
		backend_internal *internal = git_vector_get(&db->backends, i);
		git_odb_backend *b = internal->backend;

		if (only_refreshed && !b->refresh)
			continue;

		if (b->freshen != NULL)
			found = !b->freshen(b, id);
		else if (b->exists != NULL)
			found = b->exists(b, id);
	}
lhchavez committed
864
	git_mutex_unlock(&db->lock);
865 866 867 868

	return (int)found;
}

869
int git_odb__freshen(git_odb *db, const git_oid *id)
870
{
Edward Thomson committed
871 872
	GIT_ASSERT_ARG(db);
	GIT_ASSERT_ARG(id);
873 874 875 876 877 878 879 880 881 882 883

	if (odb_freshen_1(db, id, false))
		return 1;

	if (!git_odb_refresh(db))
		return odb_freshen_1(db, id, true);

	/* Failed to refresh, hence not found */
	return 0;
}

884
int git_odb_exists(git_odb *db, const git_oid *id)
885
{
886
	git_odb_object *object;
887

Edward Thomson committed
888 889
	GIT_ASSERT_ARG(db);
	GIT_ASSERT_ARG(id);
890

891
	if (git_oid_is_zero(id))
892 893
		return 0;

894 895
	if ((object = git_cache_get_raw(odb_cache(db), id)) != NULL) {
		git_odb_object_free(object);
896
		return 1;
897 898
	}

899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914
	if (odb_exists_1(db, id, false))
		return 1;

	if (!git_odb_refresh(db))
		return odb_exists_1(db, id, true);

	/* Failed to refresh, hence not found */
	return 0;
}

static int odb_exists_prefix_1(git_oid *out, git_odb *db,
	const git_oid *key, size_t len, bool only_refreshed)
{
	size_t i;
	int error = GIT_ENOTFOUND, num_found = 0;
	git_oid last_found = {{0}}, found;
915

lhchavez committed
916 917 918 919 920
	if ((error = git_mutex_lock(&db->lock)) < 0) {
		git_error_set(GIT_ERROR_ODB, "failed to acquire the odb lock");
		return error;
	}
	error = GIT_ENOTFOUND;
921 922 923 924
	for (i = 0; i < db->backends.length; ++i) {
		backend_internal *internal = git_vector_get(&db->backends, i);
		git_odb_backend *b = internal->backend;

925 926 927
		if (only_refreshed && !b->refresh)
			continue;

928 929 930
		if (!b->exists_prefix)
			continue;

931
		error = b->exists_prefix(&found, b, key, len);
932 933
		if (error == GIT_ENOTFOUND || error == GIT_PASSTHROUGH)
			continue;
lhchavez committed
934 935
		if (error) {
			git_mutex_unlock(&db->lock);
936
			return error;
lhchavez committed
937
		}
938 939 940

		/* make sure found item doesn't introduce ambiguity */
		if (num_found) {
lhchavez committed
941 942
			if (git_oid__cmp(&last_found, &found)) {
				git_mutex_unlock(&db->lock);
943
				return git_odb__error_ambiguous("multiple matches for prefix");
lhchavez committed
944
			}
945 946 947 948 949
		} else {
			git_oid_cpy(&last_found, &found);
			num_found++;
		}
	}
lhchavez committed
950
	git_mutex_unlock(&db->lock);
951 952

	if (!num_found)
953 954
		return GIT_ENOTFOUND;

955 956 957
	if (out)
		git_oid_cpy(out, &last_found);

958
	return 0;
959 960
}

961 962 963 964
int git_odb_exists_prefix(
	git_oid *out, git_odb *db, const git_oid *short_id, size_t len)
{
	int error;
965
	git_oid key = {{0}};
966

Edward Thomson committed
967 968
	GIT_ASSERT_ARG(db);
	GIT_ASSERT_ARG(short_id);
969 970 971 972

	if (len < GIT_OID_MINPREFIXLEN)
		return git_odb__error_ambiguous("prefix length too short");

973
	if (len >= GIT_OID_HEXSZ) {
974 975 976 977 978
		if (git_odb_exists(db, short_id)) {
			if (out)
				git_oid_cpy(out, short_id);
			return 0;
		} else {
979 980
			return git_odb__error_notfound(
				"no match for id prefix", short_id, len);
981 982 983
		}
	}

984
	git_oid__cpy_prefix(&key, short_id, len);
985 986 987 988 989 990 991

	error = odb_exists_prefix_1(out, db, &key, len, false);

	if (error == GIT_ENOTFOUND && !git_odb_refresh(db))
		error = odb_exists_prefix_1(out, db, &key, len, true);

	if (error == GIT_ENOTFOUND)
992
		return git_odb__error_notfound("no match for id prefix", &key, len);
993 994 995 996

	return error;
}

997
int git_odb_expand_ids(
998
	git_odb *db,
999 1000
	git_odb_expand_id *ids,
	size_t count)
1001
{
1002
	size_t i;
1003

Edward Thomson committed
1004 1005
	GIT_ASSERT_ARG(db);
	GIT_ASSERT_ARG(ids);
1006

1007 1008
	for (i = 0; i < count; i++) {
		git_odb_expand_id *query = &ids[i];
1009
		int error = GIT_EAMBIGUOUS;
1010

1011
		if (!query->type)
1012
			query->type = GIT_OBJECT_ANY;
1013 1014 1015 1016 1017 1018 1019 1020 1021 1022

		/* if we have a short OID, expand it first */
		if (query->length >= GIT_OID_MINPREFIXLEN && query->length < GIT_OID_HEXSZ) {
			git_oid actual_id;

			error = odb_exists_prefix_1(&actual_id, db, &query->id, query->length, false);
			if (!error) {
				git_oid_cpy(&query->id, &actual_id);
				query->length = GIT_OID_HEXSZ;
			}
1023 1024
		}

1025
		/*
1026 1027
		 * now we ought to have a 40-char OID, either because we've expanded it
		 * or because the user passed a full OID. Ensure its type is right.
1028
		 */
1029
		if (query->length >= GIT_OID_HEXSZ) {
1030
			git_object_t actual_type;
1031

1032 1033
			error = odb_otype_fast(&actual_type, db, &query->id);
			if (!error) {
1034
				if (query->type != GIT_OBJECT_ANY && query->type != actual_type)
1035 1036 1037 1038 1039
					error = GIT_ENOTFOUND;
				else
					query->type = actual_type;
			}
		}
1040

1041
		switch (error) {
1042
		/* no errors, so we've successfully expanded the OID */
1043
		case 0:
1044
			continue;
1045 1046 1047 1048

		/* the object is missing or ambiguous */
		case GIT_ENOTFOUND:
		case GIT_EAMBIGUOUS:
1049 1050 1051
			memset(&query->id, 0, sizeof(git_oid));
			query->length = 0;
			query->type = 0;
1052 1053 1054 1055 1056
			break;

		/* something went very wrong with the ODB; bail hard */
		default:
			return error;
1057 1058 1059
		}
	}

1060
	git_error_clear();
1061
	return 0;
1062 1063
}

1064
int git_odb_read_header(size_t *len_p, git_object_t *type_p, git_odb *db, const git_oid *id)
1065
{
1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076
	int error;
	git_odb_object *object;

	error = git_odb__read_header_or_object(&object, len_p, type_p, db, id);

	if (object)
		git_odb_object_free(object);

	return error;
}

1077
static int odb_read_header_1(
1078
	size_t *len_p, git_object_t *type_p, git_odb *db,
1079 1080 1081
	const git_oid *id, bool only_refreshed)
{
	size_t i;
1082
	git_object_t ht;
1083 1084
	bool passthrough = false;
	int error;
1085

1086
	if (!only_refreshed && (ht = odb_hardcoded_type(id)) != GIT_OBJECT_INVALID) {
1087 1088 1089 1090 1091
		*type_p = ht;
		*len_p = 0;
		return 0;
	}

lhchavez committed
1092 1093 1094 1095
	if ((error = git_mutex_lock(&db->lock)) < 0) {
		git_error_set(GIT_ERROR_ODB, "failed to acquire the odb lock");
		return error;
	}
1096
	for (i = 0; i < db->backends.length; ++i) {
1097 1098 1099 1100 1101 1102
		backend_internal *internal = git_vector_get(&db->backends, i);
		git_odb_backend *b = internal->backend;

		if (only_refreshed && !b->refresh)
			continue;

1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116
		if (!b->read_header) {
			passthrough = true;
			continue;
		}

		error = b->read_header(len_p, type_p, b, id);

		switch (error) {
		case GIT_PASSTHROUGH:
			passthrough = true;
			break;
		case GIT_ENOTFOUND:
			break;
		default:
lhchavez committed
1117
			git_mutex_unlock(&db->lock);
1118 1119
			return error;
		}
1120
	}
lhchavez committed
1121
	git_mutex_unlock(&db->lock);
1122

1123
	return passthrough ? GIT_PASSTHROUGH : GIT_ENOTFOUND;
1124 1125
}

1126
int git_odb__read_header_or_object(
1127
	git_odb_object **out, size_t *len_p, git_object_t *type_p,
1128 1129
	git_odb *db, const git_oid *id)
{
1130
	int error = GIT_ENOTFOUND;
Vicent Marti committed
1131
	git_odb_object *object;
1132

Edward Thomson committed
1133 1134 1135 1136 1137
	GIT_ASSERT_ARG(db);
	GIT_ASSERT_ARG(id);
	GIT_ASSERT_ARG(out);
	GIT_ASSERT_ARG(len_p);
	GIT_ASSERT_ARG(type_p);
Vicent Marti committed
1138

1139 1140
	*out = NULL;

1141
	if (git_oid_is_zero(id))
1142 1143
		return error_null_oid(GIT_ENOTFOUND, "cannot read object");

1144
	if ((object = git_cache_get_raw(odb_cache(db), id)) != NULL) {
Vicent Marti committed
1145 1146
		*len_p = object->cached.size;
		*type_p = object->cached.type;
1147
		*out = object;
1148
		return 0;
Vicent Marti committed
1149
	}
1150

1151
	error = odb_read_header_1(len_p, type_p, db, id, false);
1152

1153 1154
	if (error == GIT_ENOTFOUND && !git_odb_refresh(db))
		error = odb_read_header_1(len_p, type_p, db, id, true);
1155

1156 1157
	if (error == GIT_ENOTFOUND)
		return git_odb__error_notfound("cannot read header for", id, GIT_OID_HEXSZ);
1158

1159 1160
	/* we found the header; return early */
	if (!error)
1161
		return 0;
Vicent Marti committed
1162

1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173
	if (error == GIT_PASSTHROUGH) {
		/*
		 * no backend has header-reading functionality
		 * so try using `git_odb_read` instead
		 */
		error = git_odb_read(&object, db, id);
		if (!error) {
			*len_p = object->cached.size;
			*type_p = object->cached.type;
			*out = object;
		}
1174
	}
1175 1176

	return error;
1177 1178
}

1179 1180
static int odb_read_1(git_odb_object **out, git_odb *db, const git_oid *id,
		bool only_refreshed)
1181
{
1182
	size_t i;
Vicent Marti committed
1183
	git_rawobj raw;
1184
	git_odb_object *object;
1185
	git_oid hashed;
1186
	bool found = false;
1187
	int error = 0;
1188

1189 1190 1191 1192
	if (!only_refreshed) {
		if ((error = odb_read_hardcoded(&found, &raw, id)) < 0)
			return error;
	}
Vicent Marti committed
1193

lhchavez committed
1194 1195 1196 1197
	if ((error = git_mutex_lock(&db->lock)) < 0) {
		git_error_set(GIT_ERROR_ODB, "failed to acquire the odb lock");
		return error;
	}
1198
	for (i = 0; i < db->backends.length && !found; ++i) {
1199 1200
		backend_internal *internal = git_vector_get(&db->backends, i);
		git_odb_backend *b = internal->backend;
1201

1202 1203 1204
		if (only_refreshed && !b->refresh)
			continue;

1205
		if (b->read != NULL) {
1206
			error = b->read(&raw.data, &raw.len, &raw.type, b, id);
1207 1208 1209
			if (error == GIT_PASSTHROUGH || error == GIT_ENOTFOUND)
				continue;

lhchavez committed
1210 1211
			if (error < 0) {
				git_mutex_unlock(&db->lock);
1212
				return error;
lhchavez committed
1213
			}
1214 1215

			found = true;
1216
		}
Vicent Marti committed
1217
	}
lhchavez committed
1218
	git_mutex_unlock(&db->lock);
Vicent Marti committed
1219

1220 1221
	if (!found)
		return GIT_ENOTFOUND;
1222

1223 1224 1225
	if (git_odb__strict_hash_verification) {
		if ((error = git_odb_hash(&hashed, raw.data, raw.len, raw.type)) < 0)
			goto out;
1226

1227 1228 1229 1230
		if (!git_oid_equal(id, &hashed)) {
			error = git_odb__error_mismatch(id, &hashed);
			goto out;
		}
1231 1232
	}

1233
	git_error_clear();
1234 1235
	if ((object = odb_object__alloc(id, &raw)) == NULL) {
		error = -1;
1236
		goto out;
1237
	}
1238 1239

	*out = git_cache_store_raw(odb_cache(db), object);
1240 1241 1242 1243 1244

out:
	if (error)
		git__free(raw.data);
	return error;
1245 1246
}

1247 1248 1249 1250
int git_odb_read(git_odb_object **out, git_odb *db, const git_oid *id)
{
	int error;

Edward Thomson committed
1251 1252 1253
	GIT_ASSERT_ARG(out);
	GIT_ASSERT_ARG(db);
	GIT_ASSERT_ARG(id);
1254

1255
	if (git_oid_is_zero(id))
1256 1257
		return error_null_oid(GIT_ENOTFOUND, "cannot read object");

1258 1259 1260 1261 1262 1263 1264 1265 1266 1267
	*out = git_cache_get_raw(odb_cache(db), id);
	if (*out != NULL)
		return 0;

	error = odb_read_1(out, db, id, false);

	if (error == GIT_ENOTFOUND && !git_odb_refresh(db))
		error = odb_read_1(out, db, id, true);

	if (error == GIT_ENOTFOUND)
1268
		return git_odb__error_notfound("no match for id", id, GIT_OID_HEXSZ);
1269 1270 1271 1272

	return error;
}

1273
static int odb_otype_fast(git_object_t *type_p, git_odb *db, const git_oid *id)
1274 1275 1276 1277 1278
{
	git_odb_object *object;
	size_t _unused;
	int error;

1279
	if (git_oid_is_zero(id))
1280 1281
		return error_null_oid(GIT_ENOTFOUND, "cannot get object type");

1282 1283
	if ((object = git_cache_get_raw(odb_cache(db), id)) != NULL) {
		*type_p = object->cached.type;
1284
		git_odb_object_free(object);
1285 1286
		return 0;
	}
1287

1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299
	error = odb_read_header_1(&_unused, type_p, db, id, false);

	if (error == GIT_PASSTHROUGH) {
		error = odb_read_1(&object, db, id, false);
		if (!error)
			*type_p = object->cached.type;
		git_odb_object_free(object);
	}

	return error;
}

1300 1301
static int read_prefix_1(git_odb_object **out, git_odb *db,
		const git_oid *key, size_t len, bool only_refreshed)
1302
{
1303
	size_t i;
1304
	int error = 0;
1305
	git_oid found_full_oid = {{0}};
1306
	git_rawobj raw = {0};
1307
	void *data = NULL;
1308
	bool found = false;
1309
	git_odb_object *object;
1310

lhchavez committed
1311 1312 1313 1314
	if ((error = git_mutex_lock(&db->lock)) < 0) {
		git_error_set(GIT_ERROR_ODB, "failed to acquire the odb lock");
		return error;
	}
1315
	for (i = 0; i < db->backends.length; ++i) {
1316 1317 1318
		backend_internal *internal = git_vector_get(&db->backends, i);
		git_odb_backend *b = internal->backend;

1319 1320 1321
		if (only_refreshed && !b->refresh)
			continue;

1322
		if (b->read_prefix != NULL) {
1323
			git_oid full_oid;
1324
			error = b->read_prefix(&full_oid, &raw.data, &raw.len, &raw.type, b, key, len);
1325 1326 1327

			if (error == GIT_ENOTFOUND || error == GIT_PASSTHROUGH) {
				error = 0;
1328
				continue;
1329
			}
1330

lhchavez committed
1331 1332
			if (error) {
				git_mutex_unlock(&db->lock);
1333
				goto out;
lhchavez committed
1334
			}
1335

1336 1337
			git__free(data);
			data = raw.data;
Vicent Marti committed
1338

1339
			if (found && git_oid__cmp(&full_oid, &found_full_oid)) {
1340 1341 1342 1343 1344 1345 1346 1347
				git_buf buf = GIT_BUF_INIT;

				git_buf_printf(&buf, "multiple matches for prefix: %s",
					git_oid_tostr_s(&full_oid));
				git_buf_printf(&buf, " %s",
					git_oid_tostr_s(&found_full_oid));

				error = git_odb__error_ambiguous(buf.ptr);
1348
				git_buf_dispose(&buf);
lhchavez committed
1349
				git_mutex_unlock(&db->lock);
1350
				goto out;
1351
			}
Vicent Marti committed
1352

1353 1354
			found_full_oid = full_oid;
			found = true;
1355 1356
		}
	}
lhchavez committed
1357
	git_mutex_unlock(&db->lock);
1358

1359
	if (!found)
1360
		return GIT_ENOTFOUND;
1361

1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373
	if (git_odb__strict_hash_verification) {
		git_oid hash;

		if ((error = git_odb_hash(&hash, raw.data, raw.len, raw.type)) < 0)
			goto out;

		if (!git_oid_equal(&found_full_oid, &hash)) {
			error = git_odb__error_mismatch(&found_full_oid, &hash);
			goto out;
		}
	}

1374 1375
	if ((object = odb_object__alloc(&found_full_oid, &raw)) == NULL) {
		error = -1;
1376
		goto out;
1377
	}
1378 1379

	*out = git_cache_store_raw(odb_cache(db), object);
1380 1381 1382 1383 1384 1385

out:
	if (error)
		git__free(raw.data);

	return error;
1386 1387
}

1388 1389 1390 1391 1392 1393
int git_odb_read_prefix(
	git_odb_object **out, git_odb *db, const git_oid *short_id, size_t len)
{
	git_oid key = {{0}};
	int error;

Edward Thomson committed
1394 1395
	GIT_ASSERT_ARG(out);
	GIT_ASSERT_ARG(db);
1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408

	if (len < GIT_OID_MINPREFIXLEN)
		return git_odb__error_ambiguous("prefix length too short");

	if (len > GIT_OID_HEXSZ)
		len = GIT_OID_HEXSZ;

	if (len == GIT_OID_HEXSZ) {
		*out = git_cache_get_raw(odb_cache(db), short_id);
		if (*out != NULL)
			return 0;
	}

1409
	git_oid__cpy_prefix(&key, short_id, len);
1410 1411 1412 1413 1414 1415 1416

	error = read_prefix_1(out, db, &key, len, false);

	if (error == GIT_ENOTFOUND && !git_odb_refresh(db))
		error = read_prefix_1(out, db, &key, len, true);

	if (error == GIT_ENOTFOUND)
1417
		return git_odb__error_notfound("no match for prefix", &key, len);
1418 1419 1420 1421

	return error;
}

Ben Straub committed
1422
int git_odb_foreach(git_odb *db, git_odb_foreach_cb cb, void *payload)
1423 1424
{
	unsigned int i;
lhchavez committed
1425
	git_vector backends = GIT_VECTOR_INIT;
1426
	backend_internal *internal;
lhchavez committed
1427 1428 1429 1430 1431 1432 1433 1434 1435
	int error = 0;

	/* Make a copy of the backends vector to invoke the callback without holding the lock. */
	if ((error = git_mutex_lock(&db->lock)) < 0) {
		git_error_set(GIT_ERROR_ODB, "failed to acquire the odb lock");
		goto cleanup;
	}
	error = git_vector_dup(&backends, &db->backends, NULL);
	git_mutex_unlock(&db->lock);
1436

lhchavez committed
1437 1438 1439 1440
	if (error < 0)
		goto cleanup;

	git_vector_foreach(&backends, i, internal) {
1441
		git_odb_backend *b = internal->backend;
lhchavez committed
1442
		error = b->foreach(b, cb, payload);
1443
		if (error != 0)
lhchavez committed
1444
			goto cleanup;
1445 1446
	}

lhchavez committed
1447 1448 1449 1450
cleanup:
	git_vector_free(&backends);

	return error;
1451 1452
}

1453
int git_odb_write(
1454
	git_oid *oid, git_odb *db, const void *data, size_t len, git_object_t type)
1455
{
1456
	size_t i;
1457
	int error;
Vicent Marti committed
1458
	git_odb_stream *stream;
1459

Edward Thomson committed
1460 1461
	GIT_ASSERT_ARG(oid);
	GIT_ASSERT_ARG(db);
1462

1463 1464
	if ((error = git_odb_hash(oid, data, len, type)) < 0)
		return error;
1465

1466
	if (git_oid_is_zero(oid))
1467 1468
		return error_null_oid(GIT_EINVALID, "cannot write object");

1469
	if (git_odb__freshen(db, oid))
1470 1471
		return 0;

lhchavez committed
1472 1473 1474 1475
	if ((error = git_mutex_lock(&db->lock)) < 0) {
		git_error_set(GIT_ERROR_ODB, "failed to acquire the odb lock");
		return error;
	}
1476
	for (i = 0, error = GIT_ERROR; i < db->backends.length && error < 0; ++i) {
1477 1478 1479 1480 1481 1482 1483 1484
		backend_internal *internal = git_vector_get(&db->backends, i);
		git_odb_backend *b = internal->backend;

		/* we don't write in alternates! */
		if (internal->is_alternate)
			continue;

		if (b->write != NULL)
1485
			error = b->write(b, oid, data, len, type);
1486
	}
lhchavez committed
1487
	git_mutex_unlock(&db->lock);
1488

1489
	if (!error || error == GIT_PASSTHROUGH)
1490
		return 0;
Vicent Marti committed
1491

1492 1493 1494 1495
	/* if no backends were able to write the object directly, we try a
	 * streaming write to the backends; just write the whole object into the
	 * stream in one push
	 */
1496 1497
	if ((error = git_odb_open_wstream(&stream, db, len, type)) != 0)
		return error;
1498

1499 1500
	stream->write(stream, data, len);
	error = stream->finalize_write(stream, oid);
1501
	git_odb_stream_free(stream);
1502 1503

	return error;
1504 1505
}

1506
static int hash_header(git_hash_ctx *ctx, git_object_size_t size, git_object_t type)
1507 1508
{
	char header[64];
1509 1510
	size_t hdrlen;
	int error;
1511

1512 1513 1514 1515 1516
	 if ((error = git_odb__format_object_header(&hdrlen,
		header, sizeof(header), size, type)) < 0)
		return error;

	return git_hash_update(ctx, header, hdrlen);
1517 1518
}

1519
int git_odb_open_wstream(
1520
	git_odb_stream **stream, git_odb *db, git_object_size_t size, git_object_t type)
1521
{
1522
	size_t i, writes = 0;
1523
	int error = GIT_ERROR;
1524
	git_hash_ctx *ctx = NULL;
1525

Edward Thomson committed
1526 1527
	GIT_ASSERT_ARG(stream);
	GIT_ASSERT_ARG(db);
1528

lhchavez committed
1529 1530 1531 1532 1533
	if ((error = git_mutex_lock(&db->lock)) < 0) {
		git_error_set(GIT_ERROR_ODB, "failed to acquire the odb lock");
		return error;
	}
	error = GIT_ERROR;
1534
	for (i = 0; i < db->backends.length && error < 0; ++i) {
1535 1536 1537 1538 1539 1540
		backend_internal *internal = git_vector_get(&db->backends, i);
		git_odb_backend *b = internal->backend;

		/* we don't write in alternates! */
		if (internal->is_alternate)
			continue;
1541

1542 1543
		if (b->writestream != NULL) {
			++writes;
Vicent Marti committed
1544
			error = b->writestream(stream, b, size, type);
1545 1546
		} else if (b->write != NULL) {
			++writes;
1547
			error = init_fake_wstream(stream, b, size, type);
1548
		}
Vicent Marti committed
1549
	}
lhchavez committed
1550
	git_mutex_unlock(&db->lock);
Vicent Marti committed
1551

1552 1553 1554 1555 1556 1557 1558 1559
	if (error < 0) {
		if (error == GIT_PASSTHROUGH)
			error = 0;
		else if (!writes)
			error = git_odb__error_unsupported_in_backend("write object");

		goto done;
	}
Vicent Marti committed
1560

1561
	ctx = git__malloc(sizeof(git_hash_ctx));
1562
	GIT_ERROR_CHECK_ALLOC(ctx);
1563

1564 1565
	if ((error = git_hash_ctx_init(ctx)) < 0 ||
		(error = hash_header(ctx, size, type)) < 0)
1566
		goto done;
1567 1568

	(*stream)->hash_ctx = ctx;
1569 1570 1571
	(*stream)->declared_size = size;
	(*stream)->received_bytes = 0;

1572
done:
1573 1574
	if (error)
		git__free(ctx);
1575
	return error;
Vicent Marti committed
1576 1577
}

1578 1579 1580 1581
static int git_odb_stream__invalid_length(
	const git_odb_stream *stream,
	const char *action)
{
1582
	git_error_set(GIT_ERROR_ODB,
1583
		"cannot %s - "
1584 1585
		"Invalid length. %"PRId64" was expected. The "
		"total size of the received chunks amounts to %"PRId64".",
1586
		action, stream->declared_size, stream->received_bytes);
1587 1588 1589 1590

	return -1;
}

1591 1592
int git_odb_stream_write(git_odb_stream *stream, const char *buffer, size_t len)
{
1593
	git_hash_update(stream->hash_ctx, buffer, len);
1594 1595 1596 1597 1598 1599 1600

	stream->received_bytes += len;

	if (stream->received_bytes > stream->declared_size)
		return git_odb_stream__invalid_length(stream,
			"stream_write()");

1601 1602 1603 1604 1605
	return stream->write(stream, buffer, len);
}

int git_odb_stream_finalize_write(git_oid *out, git_odb_stream *stream)
{
1606 1607 1608 1609
	if (stream->received_bytes != stream->declared_size)
		return git_odb_stream__invalid_length(stream,
			"stream_finalize_write()");

1610
	git_hash_final(out, stream->hash_ctx);
1611

1612
	if (git_odb__freshen(stream->backend->odb, out))
1613 1614
		return 0;

1615
	return stream->finalize_write(stream, out);
1616 1617 1618 1619 1620 1621 1622 1623 1624
}

int git_odb_stream_read(git_odb_stream *stream, char *buffer, size_t len)
{
	return stream->read(stream, buffer, len);
}

void git_odb_stream_free(git_odb_stream *stream)
{
1625 1626 1627
	if (stream == NULL)
		return;

1628
	git_hash_ctx_cleanup(stream->hash_ctx);
1629
	git__free(stream->hash_ctx);
1630 1631 1632
	stream->free(stream);
}

1633 1634 1635
int git_odb_open_rstream(
	git_odb_stream **stream,
	size_t *len,
1636
	git_object_t *type,
1637 1638
	git_odb *db,
	const git_oid *oid)
Vicent Marti committed
1639
{
1640
	size_t i, reads = 0;
Vicent Marti committed
1641 1642
	int error = GIT_ERROR;

Edward Thomson committed
1643 1644
	GIT_ASSERT_ARG(stream);
	GIT_ASSERT_ARG(db);
Vicent Marti committed
1645

lhchavez committed
1646 1647 1648 1649 1650
	if ((error = git_mutex_lock(&db->lock)) < 0) {
		git_error_set(GIT_ERROR_ODB, "failed to acquire the odb lock");
		return error;
	}
	error = GIT_ERROR;
Vicent Marti committed
1651 1652 1653 1654
	for (i = 0; i < db->backends.length && error < 0; ++i) {
		backend_internal *internal = git_vector_get(&db->backends, i);
		git_odb_backend *b = internal->backend;

1655 1656
		if (b->readstream != NULL) {
			++reads;
1657
			error = b->readstream(stream, len, type, b, oid);
1658
		}
1659
	}
lhchavez committed
1660
	git_mutex_unlock(&db->lock);
1661

1662
	if (error == GIT_PASSTHROUGH)
1663
		error = 0;
1664 1665
	if (error < 0 && !reads)
		error = git_odb__error_unsupported_in_backend("read object streamed");
Vicent Marti committed
1666

1667 1668 1669
	return error;
}

1670
int git_odb_write_pack(struct git_odb_writepack **out, git_odb *db, git_indexer_progress_cb progress_cb, void *progress_payload)
1671
{
1672
	size_t i, writes = 0;
1673 1674
	int error = GIT_ERROR;

Edward Thomson committed
1675 1676
	GIT_ASSERT_ARG(out);
	GIT_ASSERT_ARG(db);
1677

lhchavez committed
1678 1679 1680 1681 1682
	if ((error = git_mutex_lock(&db->lock)) < 0) {
		git_error_set(GIT_ERROR_ODB, "failed to acquire the odb lock");
		return error;
	}
	error = GIT_ERROR;
1683 1684 1685 1686 1687 1688 1689 1690
	for (i = 0; i < db->backends.length && error < 0; ++i) {
		backend_internal *internal = git_vector_get(&db->backends, i);
		git_odb_backend *b = internal->backend;

		/* we don't write in alternates! */
		if (internal->is_alternate)
			continue;

1691 1692
		if (b->writepack != NULL) {
			++writes;
1693
			error = b->writepack(out, b, db, progress_cb, progress_payload);
1694
		}
1695
	}
lhchavez committed
1696
	git_mutex_unlock(&db->lock);
1697 1698 1699

	if (error == GIT_PASSTHROUGH)
		error = 0;
1700 1701
	if (error < 0 && !writes)
		error = git_odb__error_unsupported_in_backend("write pack");
1702 1703 1704 1705

	return error;
}

1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734
int git_odb_write_multi_pack_index(git_odb *db)
{
	size_t i, writes = 0;
	int error = GIT_ERROR;

	GIT_ASSERT_ARG(db);

	for (i = 0; i < db->backends.length && error < 0; ++i) {
		backend_internal *internal = git_vector_get(&db->backends, i);
		git_odb_backend *b = internal->backend;

		/* we don't write in alternates! */
		if (internal->is_alternate)
			continue;

		if (b->writemidx != NULL) {
			++writes;
			error = b->writemidx(b);
		}
	}

	if (error == GIT_PASSTHROUGH)
		error = 0;
	if (error < 0 && !writes)
		error = git_odb__error_unsupported_in_backend("write multi-pack-index");

	return error;
}

1735
void *git_odb_backend_data_alloc(git_odb_backend *backend, size_t len)
1736
{
1737
	GIT_UNUSED(backend);
1738 1739 1740
	return git__malloc(len);
}

1741
#ifndef GIT_DEPRECATE_HARD
1742 1743 1744 1745
void *git_odb_backend_malloc(git_odb_backend *backend, size_t len)
{
	return git_odb_backend_data_alloc(backend, len);
}
1746
#endif
1747

1748 1749 1750 1751 1752 1753
void git_odb_backend_data_free(git_odb_backend *backend, void *data)
{
	GIT_UNUSED(backend);
	git__free(data);
}

Vicent Marti committed
1754 1755
int git_odb_refresh(struct git_odb *db)
{
1756
	size_t i;
lhchavez committed
1757
	int error;
Edward Thomson committed
1758 1759

	GIT_ASSERT_ARG(db);
Vicent Marti committed
1760

lhchavez committed
1761 1762 1763 1764
	if ((error = git_mutex_lock(&db->lock)) < 0) {
		git_error_set(GIT_ERROR_ODB, "failed to acquire the odb lock");
		return error;
	}
Vicent Marti committed
1765 1766 1767 1768 1769 1770
	for (i = 0; i < db->backends.length; ++i) {
		backend_internal *internal = git_vector_get(&db->backends, i);
		git_odb_backend *b = internal->backend;

		if (b->refresh != NULL) {
			int error = b->refresh(b);
lhchavez committed
1771 1772
			if (error < 0) {
				git_mutex_unlock(&db->lock);
Vicent Marti committed
1773
				return error;
lhchavez committed
1774
			}
Vicent Marti committed
1775 1776
		}
	}
1777 1778
	if (db->cgraph)
		git_commit_graph_refresh(db->cgraph);
lhchavez committed
1779
	git_mutex_unlock(&db->lock);
Vicent Marti committed
1780 1781 1782 1783

	return 0;
}

1784 1785 1786 1787 1788 1789 1790
int git_odb__error_mismatch(const git_oid *expected, const git_oid *actual)
{
	char expected_oid[GIT_OID_HEXSZ + 1], actual_oid[GIT_OID_HEXSZ + 1];

	git_oid_tostr(expected_oid, sizeof(expected_oid), expected);
	git_oid_tostr(actual_oid, sizeof(actual_oid), actual);

1791
	git_error_set(GIT_ERROR_ODB, "object hash mismatch - expected %s but got %s",
1792 1793 1794 1795 1796
		expected_oid, actual_oid);

	return GIT_EMISMATCH;
}

1797 1798
int git_odb__error_notfound(
	const char *message, const git_oid *oid, size_t oid_len)
1799
{
Russell Belfer committed
1800 1801
	if (oid != NULL) {
		char oid_str[GIT_OID_HEXSZ + 1];
1802
		git_oid_tostr(oid_str, oid_len+1, oid);
1803
		git_error_set(GIT_ERROR_ODB, "object not found - %s (%.*s)",
1804
			message, (int) oid_len, oid_str);
Russell Belfer committed
1805
	} else
1806
		git_error_set(GIT_ERROR_ODB, "object not found - %s", message);
Russell Belfer committed
1807

1808
	return GIT_ENOTFOUND;
1809 1810
}

1811 1812
static int error_null_oid(int error, const char *message)
{
1813
	git_error_set(GIT_ERROR_ODB, "odb: %s: null OID cannot exist", message);
1814 1815 1816
	return error;
}

1817 1818
int git_odb__error_ambiguous(const char *message)
{
1819
	git_error_set(GIT_ERROR_ODB, "ambiguous SHA1 prefix - %s", message);
1820
	return GIT_EAMBIGUOUS;
1821 1822
}

1823
int git_odb_init_backend(git_odb_backend *backend, unsigned int version)
1824
{
1825 1826 1827
	GIT_INIT_STRUCTURE_FROM_TEMPLATE(
		backend, version, git_odb_backend, GIT_ODB_BACKEND_INIT);
	return 0;
1828
}