odb.c 44.2 KB
Newer Older
1
/*
Edward Thomson committed
2
 * Copyright (C) the libgit2 contributors. All rights reserved.
3
 *
Vicent Marti committed
4 5
 * This file is part of libgit2, distributed under the GNU GPL v2 with
 * a Linking Exception. For full terms see the included COPYING file.
6 7
 */

8 9
#include "odb.h"

10
#include <zlib.h>
11
#include "git2/object.h"
12
#include "git2/sys/odb_backend.h"
13
#include "futils.h"
14
#include "hash.h"
15
#include "delta.h"
16
#include "filter.h"
17
#include "repository.h"
18
#include "blob.h"
19
#include "oid.h"
20

21
#include "git2/odb_backend.h"
22
#include "git2/oid.h"
23
#include "git2/oidarray.h"
24

25 26
#define GIT_ALTERNATES_FILE "info/alternates"

27 28
#define GIT_ALTERNATES_MAX_DEPTH 5

29 30 31 32
/*
 * We work under the assumption that most objects for long-running
 * operations will be packed
 */
33 34
int git_odb__loose_priority = GIT_ODB_DEFAULT_LOOSE_PRIORITY;
int git_odb__packed_priority = GIT_ODB_DEFAULT_PACKED_PRIORITY;
35

36 37
bool git_odb__strict_hash_verification = true;

38 39 40 41
typedef struct
{
	git_odb_backend *backend;
	int priority;
42 43
	bool is_alternate;
	ino_t disk_inode;
44 45
} backend_internal;

46 47
static git_cache *odb_cache(git_odb *odb)
{
48 49
	git_repository *owner = GIT_REFCOUNT_OWNER(odb);
	if (owner != NULL) {
50 51 52 53 54
		return &owner->objects;
	}

	return &odb->own_cache;
}
55

56
static int odb_otype_fast(git_object_t *type_p, git_odb *db, const git_oid *id);
57
static int load_alternates(git_odb *odb, const char *objects_dir, int alternate_depth);
58
static int error_null_oid(int error, const char *message);
59

60
static git_object_t odb_hardcoded_type(const git_oid *id)
61
{
62
	if (!git_oid_cmp(id, &git_oid__empty_tree_sha1))
63
		return GIT_OBJECT_TREE;
64

65
	return GIT_OBJECT_INVALID;
66 67
}

68
static int odb_read_hardcoded(bool *found, git_rawobj *raw, const git_oid *id)
69
{
70
	git_object_t type;
71 72 73

	*found = false;

74
	if ((type = odb_hardcoded_type(id)) == GIT_OBJECT_INVALID)
75
		return 0;
76 77 78 79

	raw->type = type;
	raw->len = 0;
	raw->data = git__calloc(1, sizeof(uint8_t));
80
	GIT_ERROR_CHECK_ALLOC(raw->data);
81 82

	*found = true;
83 84 85
	return 0;
}

86 87 88 89
int git_odb__format_object_header(
	size_t *written,
	char *hdr,
	size_t hdr_size,
90
	git_object_size_t obj_len,
91
	git_object_t obj_type)
92
{
Vicent Marti committed
93
	const char *type_str = git_object_type2string(obj_type);
94 95 96
	int hdr_max = (hdr_size > INT_MAX-2) ? (INT_MAX-2) : (int)hdr_size;
	int len;

lhchavez committed
97
	len = p_snprintf(hdr, hdr_max, "%s %"PRId64, type_str, (int64_t)obj_len);
98 99

	if (len < 0 || len >= hdr_max) {
100
		git_error_set(GIT_ERROR_OS, "object header creation failed");
101 102 103 104 105
		return -1;
	}

	*written = (size_t)(len + 1);
	return 0;
106 107
}

108
int git_odb__hashobj(git_oid *id, git_rawobj *obj, git_oid_t oid_type)
109
{
110
	git_str_vec vec[2];
111
	char header[64];
112
	size_t hdrlen;
113
	git_hash_algorithm_t algorithm;
114
	int error;
115

Edward Thomson committed
116 117
	GIT_ASSERT_ARG(id);
	GIT_ASSERT_ARG(obj);
118

119
	if (!git_object_typeisloose(obj->type)) {
120
		git_error_set(GIT_ERROR_INVALID, "invalid object type");
121
		return -1;
122
	}
Vicent Marti committed
123

124 125 126 127 128
	if (!(algorithm = git_oid_algorithm(oid_type))) {
		git_error_set(GIT_ERROR_INVALID, "unknown oid type");
		return -1;
	}

129
	if (!obj->data && obj->len != 0) {
130
		git_error_set(GIT_ERROR_INVALID, "invalid object");
131
		return -1;
132
	}
133

134 135 136
	if ((error = git_odb__format_object_header(&hdrlen,
		header, sizeof(header), obj->len, obj->type)) < 0)
		return error;
137

138
	vec[0].data = header;
Vicent Marti committed
139
	vec[0].len = hdrlen;
140
	vec[1].data = obj->data;
Vicent Marti committed
141
	vec[1].len = obj->len;
142

143
#ifdef GIT_EXPERIMENTAL_SHA256
144
	id->type = oid_type;
145 146
#endif

147
	return git_hash_vec(id->id, vec, 2, algorithm);
148 149
}

150

151
static git_odb_object *odb_object__alloc(const git_oid *oid, git_rawobj *source)
Ramsay Jones committed
152
{
153
	git_odb_object *object = git__calloc(1, sizeof(git_odb_object));
Ramsay Jones committed
154

155 156 157 158 159 160
	if (object != NULL) {
		git_oid_cpy(&object->cached.oid, oid);
		object->cached.type = source->type;
		object->cached.size = source->len;
		object->buffer      = source->data;
	}
Ramsay Jones committed
161

Vicent Marti committed
162
	return object;
163 164
}

165
void git_odb_object__free(void *object)
166
{
Vicent Marti committed
167
	if (object != NULL) {
168
		git__free(((git_odb_object *)object)->buffer);
169
		git__free(object);
Vicent Marti committed
170 171
	}
}
172

173 174 175 176 177 178 179
const git_oid *git_odb_object_id(git_odb_object *object)
{
	return &object->cached.oid;
}

const void *git_odb_object_data(git_odb_object *object)
{
Vicent Marti committed
180
	return object->buffer;
181 182 183 184
}

size_t git_odb_object_size(git_odb_object *object)
{
Vicent Marti committed
185
	return object->cached.size;
186 187
}

188
git_object_t git_odb_object_type(git_odb_object *object)
189
{
Vicent Marti committed
190
	return object->cached.type;
191 192
}

193 194 195 196 197 198 199
int git_odb_object_dup(git_odb_object **dest, git_odb_object *source)
{
	git_cached_obj_incref(source);
	*dest = source;
	return 0;
}

200
void git_odb_object_free(git_odb_object *object)
Vicent Marti committed
201
{
202 203 204
	if (object == NULL)
		return;

205
	git_cached_obj_decref(object);
Vicent Marti committed
206
}
207

208 209 210 211 212 213
int git_odb__hashfd(
	git_oid *out,
	git_file fd,
	size_t size,
	git_object_t object_type,
	git_oid_t oid_type)
Vicent Marti committed
214
{
215
	size_t hdr_len;
216
	char hdr[64], buffer[GIT_BUFSIZE_FILEIO];
217
	git_hash_ctx ctx;
218
	git_hash_algorithm_t algorithm;
219
	ssize_t read_len = 0;
220
	int error = 0;
Vicent Marti committed
221

222
	if (!git_object_typeisloose(object_type)) {
223
		git_error_set(GIT_ERROR_INVALID, "invalid object type for hash");
224 225 226
		return -1;
	}

227 228 229 230 231 232
	if (!(algorithm = git_oid_algorithm(oid_type))) {
		git_error_set(GIT_ERROR_INVALID, "unknown oid type");
		return -1;
	}

	if ((error = git_hash_ctx_init(&ctx, algorithm)) < 0)
233
		return error;
Vicent Marti committed
234

235
	if ((error = git_odb__format_object_header(&hdr_len, hdr,
236
		sizeof(hdr), size, object_type)) < 0)
237
		goto done;
238

239
	if ((error = git_hash_update(&ctx, hdr, hdr_len)) < 0)
240
		goto done;
Vicent Marti committed
241

Vicent Marti committed
242
	while (size > 0 && (read_len = p_read(fd, buffer, sizeof(buffer))) > 0) {
243
		if ((error = git_hash_update(&ctx, buffer, read_len)) < 0)
244 245
			goto done;

Vicent Marti committed
246 247 248
		size -= read_len;
	}

Vicent Marti committed
249 250 251 252
	/* If p_read returned an error code, the read obviously failed.
	 * If size is not zero, the file was truncated after we originally
	 * stat'd it, so we consider this a read failure too */
	if (read_len < 0 || size > 0) {
253
		git_error_set(GIT_ERROR_OS, "error reading file for hashing");
254 255 256
		error = -1;

		goto done;
Vicent Marti committed
257 258
	}

259
	error = git_hash_final(out->id, &ctx);
260 261

#ifdef GIT_EXPERIMENTAL_SHA256
262
	out->type = oid_type;
263
#endif
Vicent Marti committed
264

265
done:
266
	git_hash_ctx_cleanup(&ctx);
267
	return error;
Vicent Marti committed
268 269
}

270
int git_odb__hashfd_filtered(
271 272 273 274 275 276
	git_oid *out,
	git_file fd,
	size_t size,
	git_object_t object_type,
	git_oid_t oid_type,
	git_filter_list *fl)
277 278
{
	int error;
279
	git_str raw = GIT_STR_INIT;
280

281
	if (!fl)
282
		return git_odb__hashfd(out, fd, size, object_type, oid_type);
283 284 285 286 287

	/* size of data is used in header, so we have to read the whole file
	 * into memory to apply filters before beginning to calculate the hash
	 */

288
	if (!(error = git_futils_readbuffer_fd(&raw, fd, size))) {
289
		git_str post = GIT_STR_INIT;
290

291
		error = git_filter_list__convert_buf(&post, fl, &raw);
292

293
		if (!error)
294
			error = git_odb__hash(out, post.ptr, post.size, object_type, oid_type);
295

296
		git_str_dispose(&post);
297
	}
298 299 300 301

	return error;
}

302
int git_odb__hashlink(git_oid *out, const char *path, git_oid_t oid_type)
303 304
{
	struct stat st;
305
	int size;
306
	int result;
307

308
	if (git_fs_path_lstat(path, &st) < 0)
309
		return -1;
310

311
	if (!git__is_int(st.st_size) || (int)st.st_size < 0) {
312
		git_error_set(GIT_ERROR_FILESYSTEM, "file size overflow for 32-bit systems");
313 314
		return -1;
	}
315

316
	size = (int)st.st_size;
317

318 319
	if (S_ISLNK(st.st_mode)) {
		char *link_data;
320 321
		int read_len;
		size_t alloc_size;
322

323
		GIT_ERROR_CHECK_ALLOC_ADD(&alloc_size, size, 1);
324
		link_data = git__malloc(alloc_size);
325
		GIT_ERROR_CHECK_ALLOC(link_data);
326

327
		read_len = p_readlink(path, link_data, size);
328
		if (read_len == -1) {
329
			git_error_set(GIT_ERROR_OS, "failed to read symlink data for '%s'", path);
330
			git__free(link_data);
331 332
			return -1;
		}
333 334
		GIT_ASSERT(read_len <= size);
		link_data[read_len] = '\0';
335

336
		result = git_odb__hash(out, link_data, read_len, GIT_OBJECT_BLOB, oid_type);
337
		git__free(link_data);
338
	} else {
339 340 341
		int fd = git_futils_open_ro(path);
		if (fd < 0)
			return -1;
342
		result = git_odb__hashfd(out, fd, size, GIT_OBJECT_BLOB, oid_type);
343 344 345
		p_close(fd);
	}

346
	return result;
347 348
}

349
int git_odb__hashfile(
350 351 352 353
	git_oid *out,
	const char *path,
	git_object_t object_type,
	git_oid_t oid_type)
354
{
355 356 357 358
	uint64_t size;
	int fd, error = 0;

	if ((fd = git_futils_open_ro(path)) < 0)
359
		return fd;
360

361 362 363 364
	if ((error = git_futils_filesize(&size, fd)) < 0)
		goto done;

	if (!git__is_sizet(size)) {
365
		git_error_set(GIT_ERROR_OS, "file size overflow for 32-bit systems");
366 367
		error = -1;
		goto done;
368 369
	}

370
	error = git_odb__hashfd(out, fd, (size_t)size, object_type, oid_type);
371 372

done:
373
	p_close(fd);
374
	return error;
375 376
}

377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396
#ifdef GIT_EXPERIMENTAL_SHA256
int git_odb_hashfile(
	git_oid *out,
	const char *path,
	git_object_t object_type,
	git_oid_t oid_type)
{
	return git_odb__hashfile(out, path, object_type, oid_type);
}
#else
int git_odb_hashfile(
	git_oid *out,
	const char *path,
	git_object_t object_type)
{
	return git_odb__hashfile(out, path, object_type, GIT_OID_SHA1);
}
#endif

int git_odb__hash(
397 398 399 400 401
	git_oid *id,
	const void *data,
	size_t len,
	git_object_t object_type,
	git_oid_t oid_type)
Vicent Marti committed
402 403
{
	git_rawobj raw;
404

Edward Thomson committed
405
	GIT_ASSERT_ARG(id);
406

Vicent Marti committed
407 408
	raw.data = (void *)data;
	raw.len = len;
409
	raw.type = object_type;
410

411
	return git_odb__hashobj(id, &raw, oid_type);
412 413
}

414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434
#ifdef GIT_EXPERIMENTAL_SHA256
int git_odb_hash(
	git_oid *out,
	const void *data,
	size_t len,
	git_object_t object_type,
	git_oid_t oid_type)
{
	return git_odb__hash(out, data, len, object_type, oid_type);
}
#else
int git_odb_hash(
	git_oid *out,
	const void *data,
	size_t len,
	git_object_t type)
{
	return git_odb__hash(out, data, len, type, GIT_OID_SHA1);
}
#endif

435 436 437 438 439 440 441 442
/**
 * FAKE WSTREAM
 */

typedef struct {
	git_odb_stream stream;
	char *buffer;
	size_t size, written;
443
	git_object_t type;
444 445
} fake_wstream;

446
static int fake_wstream__fwrite(git_odb_stream *_stream, const git_oid *oid)
447 448
{
	fake_wstream *stream = (fake_wstream *)_stream;
449
	return _stream->backend->write(_stream->backend, oid, stream->buffer, stream->size, stream->type);
450 451 452 453 454 455
}

static int fake_wstream__write(git_odb_stream *_stream, const char *data, size_t len)
{
	fake_wstream *stream = (fake_wstream *)_stream;

Edward Thomson committed
456
	GIT_ASSERT(stream->written + len <= stream->size);
457 458 459

	memcpy(stream->buffer + stream->written, data, len);
	stream->written += len;
460
	return 0;
461 462 463 464 465 466
}

static void fake_wstream__free(git_odb_stream *_stream)
{
	fake_wstream *stream = (fake_wstream *)_stream;

467 468
	git__free(stream->buffer);
	git__free(stream);
469 470
}

471
static int init_fake_wstream(git_odb_stream **stream_p, git_odb_backend *backend, git_object_size_t size, git_object_t type)
472 473
{
	fake_wstream *stream;
474
	size_t blobsize;
475

476 477
	GIT_ERROR_CHECK_BLOBSIZE(size);
	blobsize = (size_t)size;
478

479
	stream = git__calloc(1, sizeof(fake_wstream));
480
	GIT_ERROR_CHECK_ALLOC(stream);
481

482
	stream->size = blobsize;
483
	stream->type = type;
484
	stream->buffer = git__malloc(blobsize);
485
	if (stream->buffer == NULL) {
486
		git__free(stream);
487
		return -1;
488 489 490 491 492 493 494 495 496 497
	}

	stream->stream.backend = backend;
	stream->stream.read = NULL; /* read only */
	stream->stream.write = &fake_wstream__write;
	stream->stream.finalize_write = &fake_wstream__fwrite;
	stream->stream.free = &fake_wstream__free;
	stream->stream.mode = GIT_STREAM_WRONLY;

	*stream_p = (git_odb_stream *)stream;
498
	return 0;
499
}
500

501 502 503 504 505 506 507
/***********************************************************
 *
 * OBJECT DATABASE PUBLIC API
 *
 * Public calls for the ODB functionality
 *
 ***********************************************************/
508

509
static int backend_sort_cmp(const void *a, const void *b)
510
{
511 512
	const backend_internal *backend_a = (const backend_internal *)(a);
	const backend_internal *backend_b = (const backend_internal *)(b);
513

514 515 516 517 518 519 520 521
	if (backend_b->priority == backend_a->priority) {
		if (backend_a->is_alternate)
			return -1;
		if (backend_b->is_alternate)
			return 1;
		return 0;
	}
	return (backend_b->priority - backend_a->priority);
522 523
}

524 525 526 527 528 529 530 531 532 533
static void normalize_options(
	git_odb_options *opts,
	const git_odb_options *given_opts)
{
	git_odb_options init = GIT_ODB_OPTIONS_INIT;

	if (given_opts)
		memcpy(opts, given_opts, sizeof(git_odb_options));
	else
		memcpy(opts, &init, sizeof(git_odb_options));
534 535 536

	if (!opts->oid_type)
		opts->oid_type = GIT_OID_DEFAULT;
537 538
}

539
int git_odb__new(git_odb **out, const git_odb_options *opts)
540
{
541
	git_odb *db = git__calloc(1, sizeof(*db));
542
	GIT_ERROR_CHECK_ALLOC(db);
543

544 545
	normalize_options(&db->options, opts);

lhchavez committed
546 547 548 549
	if (git_mutex_init(&db->lock) < 0) {
		git__free(db);
		return -1;
	}
550
	if (git_cache_init(&db->own_cache) < 0) {
lhchavez committed
551
		git_mutex_free(&db->lock);
552 553 554 555
		git__free(db);
		return -1;
	}
	if (git_vector_init(&db->backends, 4, backend_sort_cmp) < 0) {
556
		git_cache_dispose(&db->own_cache);
lhchavez committed
557
		git_mutex_free(&db->lock);
558
		git__free(db);
559
		return -1;
560
	}
561

562
	*out = db;
563
	GIT_REFCOUNT_INC(db);
564
	return 0;
565 566
}

567 568 569 570 571 572 573 574 575 576 577 578
#ifdef GIT_EXPERIMENTAL_SHA256
int git_odb_new(git_odb **out, const git_odb_options *opts)
{
	return git_odb__new(out, opts);
}
#else
int git_odb_new(git_odb **out)
{
	return git_odb__new(out, NULL);
}
#endif

579 580 581
static int add_backend_internal(
	git_odb *odb, git_odb_backend *backend,
	int priority, bool is_alternate, ino_t disk_inode)
Ramsay Jones committed
582
{
583 584
	backend_internal *internal;

Edward Thomson committed
585 586
	GIT_ASSERT_ARG(odb);
	GIT_ASSERT_ARG(backend);
Ramsay Jones committed
587

588
	GIT_ERROR_CHECK_VERSION(backend, GIT_ODB_BACKEND_VERSION, "git_odb_backend");
589

590
	/* Check if the backend is already owned by another ODB */
Edward Thomson committed
591
	GIT_ASSERT(!backend->odb || backend->odb == odb);
Ramsay Jones committed
592

593
	internal = git__malloc(sizeof(backend_internal));
594
	GIT_ERROR_CHECK_ALLOC(internal);
595 596 597 598

	internal->backend = backend;
	internal->priority = priority;
	internal->is_alternate = is_alternate;
599
	internal->disk_inode = disk_inode;
Ramsay Jones committed
600

lhchavez committed
601 602 603 604
	if (git_mutex_lock(&odb->lock) < 0) {
		git_error_set(GIT_ERROR_ODB, "failed to acquire the odb lock");
		return -1;
	}
605
	if (git_vector_insert(&odb->backends, internal) < 0) {
lhchavez committed
606
		git_mutex_unlock(&odb->lock);
607
		git__free(internal);
608
		return -1;
609
	}
610
	git_vector_sort(&odb->backends);
611
	internal->backend->odb = odb;
lhchavez committed
612
	git_mutex_unlock(&odb->lock);
613
	return 0;
Ramsay Jones committed
614 615
}

616 617
int git_odb_add_backend(git_odb *odb, git_odb_backend *backend, int priority)
{
618
	return add_backend_internal(odb, backend, priority, false, 0);
619 620 621 622
}

int git_odb_add_alternate(git_odb *odb, git_odb_backend *backend, int priority)
{
623
	return add_backend_internal(odb, backend, priority, true, 0);
624 625
}

626 627
size_t git_odb_num_backends(git_odb *odb)
{
lhchavez committed
628 629 630
	size_t length;
	bool locked = true;

Edward Thomson committed
631
	GIT_ASSERT_ARG(odb);
lhchavez committed
632 633 634 635 636 637 638 639 640

	if (git_mutex_lock(&odb->lock) < 0) {
		git_error_set(GIT_ERROR_ODB, "failed to acquire the odb lock");
		locked = false;
	}
	length = odb->backends.length;
	if (locked)
		git_mutex_unlock(&odb->lock);
	return length;
641 642
}

643 644
static int git_odb__error_unsupported_in_backend(const char *action)
{
645
	git_error_set(GIT_ERROR_ODB,
646
		"cannot %s - unsupported in the loaded odb backends", action);
647 648 649 650
	return -1;
}


651 652 653
int git_odb_get_backend(git_odb_backend **out, git_odb *odb, size_t pos)
{
	backend_internal *internal;
lhchavez committed
654
	int error;
655

Edward Thomson committed
656 657 658
	GIT_ASSERT_ARG(out);
	GIT_ASSERT_ARG(odb);

lhchavez committed
659 660 661 662 663

	if ((error = git_mutex_lock(&odb->lock)) < 0) {
		git_error_set(GIT_ERROR_ODB, "failed to acquire the odb lock");
		return error;
	}
664 665
	internal = git_vector_get(&odb->backends, pos);

lhchavez committed
666 667 668 669 670
	if (!internal || !internal->backend) {
		git_mutex_unlock(&odb->lock);

		git_error_set(GIT_ERROR_ODB, "no ODB backend loaded at index %" PRIuZ, pos);
		return GIT_ENOTFOUND;
671
	}
lhchavez committed
672 673
	*out = internal->backend;
	git_mutex_unlock(&odb->lock);
674

lhchavez committed
675
	return 0;
676 677
}

678
int git_odb__add_default_backends(
679 680
	git_odb *db, const char *objects_dir,
	bool as_alternates, int alternate_depth)
681
{
682
	size_t i = 0;
683
	struct stat st;
684
	ino_t inode;
685
	git_odb_backend *loose, *packed;
686
	git_odb_backend_loose_options loose_opts = GIT_ODB_BACKEND_LOOSE_OPTIONS_INIT;
687
	git_odb_backend_pack_options pack_opts = GIT_ODB_BACKEND_PACK_OPTIONS_INIT;
688

689 690
	/* TODO: inodes are not really relevant on Win32, so we need to find
	 * a cross-platform workaround for this */
691 692
#ifdef GIT_WIN32
	GIT_UNUSED(i);
693
	GIT_UNUSED(&st);
694 695 696

	inode = 0;
#else
697
	if (p_stat(objects_dir, &st) < 0) {
698
		if (as_alternates)
Etienne Samson committed
699
			/* this should warn */
700 701
			return 0;

702
		git_error_set(GIT_ERROR_ODB, "failed to load object database in '%s'", objects_dir);
703 704 705
		return -1;
	}

706 707
	inode = st.st_ino;

lhchavez committed
708 709 710 711
	if (git_mutex_lock(&db->lock) < 0) {
		git_error_set(GIT_ERROR_ODB, "failed to acquire the odb lock");
		return -1;
	}
712 713
	for (i = 0; i < db->backends.length; ++i) {
		backend_internal *backend = git_vector_get(&db->backends, i);
lhchavez committed
714 715
		if (backend->disk_inode == inode) {
			git_mutex_unlock(&db->lock);
716
			return 0;
lhchavez committed
717
		}
718
	}
lhchavez committed
719
	git_mutex_unlock(&db->lock);
720
#endif
721

722 723 724
	if (db->do_fsync)
		loose_opts.flags |= GIT_ODB_BACKEND_LOOSE_FSYNC;

725
	loose_opts.oid_type = db->options.oid_type;
726
	pack_opts.oid_type = db->options.oid_type;
727

728
	/* add the loose object backend */
729
	if (git_odb__backend_loose(&loose, objects_dir, &loose_opts) < 0 ||
730
		add_backend_internal(db, loose, git_odb__loose_priority, as_alternates, inode) < 0)
731
		return -1;
732 733

	/* add the packed file backend */
734 735 736 737 738 739 740 741 742 743 744
#ifdef GIT_EXPERIMENTAL_SHA256
	if (git_odb_backend_pack(&packed, objects_dir, &pack_opts) < 0)
		return -1;
#else
	GIT_UNUSED(pack_opts);

	if (git_odb_backend_pack(&packed, objects_dir) < 0)
		return -1;
#endif

	if (add_backend_internal(db, packed, git_odb__packed_priority, as_alternates, inode) < 0)
745
		return -1;
746

747 748 749 750
	if (git_mutex_lock(&db->lock) < 0) {
		git_error_set(GIT_ERROR_ODB, "failed to acquire the odb lock");
		return -1;
	}
751 752
	if (!db->cgraph &&
	    git_commit_graph_new(&db->cgraph, objects_dir, false, db->options.oid_type) < 0) {
753 754 755 756 757
		git_mutex_unlock(&db->lock);
		return -1;
	}
	git_mutex_unlock(&db->lock);

758
	return load_alternates(db, objects_dir, alternate_depth);
759 760
}

761
static int load_alternates(git_odb *odb, const char *objects_dir, int alternate_depth)
762
{
763 764
	git_str alternates_path = GIT_STR_INIT;
	git_str alternates_buf = GIT_STR_INIT;
765 766
	char *buffer;
	const char *alternate;
767
	int result = 0;
768

769
	/* Git reports an error, we just ignore anything deeper */
770
	if (alternate_depth > GIT_ALTERNATES_MAX_DEPTH)
771 772
		return 0;

773
	if (git_str_joinpath(&alternates_path, objects_dir, GIT_ALTERNATES_FILE) < 0)
774
		return -1;
775

776
	if (git_fs_path_exists(alternates_path.ptr) == false) {
777
		git_str_dispose(&alternates_path);
778
		return 0;
779
	}
780

781
	if (git_futils_readbuffer(&alternates_buf, alternates_path.ptr) < 0) {
782
		git_str_dispose(&alternates_path);
783
		return -1;
784
	}
785

786
	buffer = (char *)alternates_buf.ptr;
787 788

	/* add each alternate as a new backend; one alternate per line */
789 790 791 792
	while ((alternate = git__strtok(&buffer, "\r\n")) != NULL) {
		if (*alternate == '\0' || *alternate == '#')
			continue;

793 794 795 796 797 798
		/*
		 * Relative path: build based on the current `objects`
		 * folder. However, relative paths are only allowed in
		 * the current repository.
		 */
		if (*alternate == '.' && !alternate_depth) {
799
			if ((result = git_str_joinpath(&alternates_path, objects_dir, alternate)) < 0)
800
				break;
801
			alternate = git_str_cstr(&alternates_path);
802 803
		}

804
		if ((result = git_odb__add_default_backends(odb, alternate, true, alternate_depth + 1)) < 0)
805 806
			break;
	}
807

808 809
	git_str_dispose(&alternates_path);
	git_str_dispose(&alternates_buf);
810

811
	return result;
812
}
Ramsay Jones committed
813

814 815
int git_odb_add_disk_alternate(git_odb *odb, const char *path)
{
816
	return git_odb__add_default_backends(odb, path, true, 0);
817 818
}

819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835
int git_odb_set_commit_graph(git_odb *odb, git_commit_graph *cgraph)
{
	int error = 0;

	GIT_ASSERT_ARG(odb);

	if ((error = git_mutex_lock(&odb->lock)) < 0) {
		git_error_set(GIT_ERROR_ODB, "failed to acquire the db lock");
		return error;
	}
	git_commit_graph_free(odb->cgraph);
	odb->cgraph = cgraph;
	git_mutex_unlock(&odb->lock);

	return error;
}

836
int git_odb__open(
837 838 839
	git_odb **out,
	const char *objects_dir,
	const git_odb_options *opts)
Ramsay Jones committed
840
{
841
	git_odb *db;
Ramsay Jones committed
842

Edward Thomson committed
843 844
	GIT_ASSERT_ARG(out);
	GIT_ASSERT_ARG(objects_dir);
845 846 847

	*out = NULL;

848
	if (git_odb__new(&db, opts) < 0)
849
		return -1;
Ramsay Jones committed
850

851
	if (git_odb__add_default_backends(db, objects_dir, 0, 0) < 0) {
852 853 854
		git_odb_free(db);
		return -1;
	}
Ramsay Jones committed
855

856
	*out = db;
857
	return 0;
858
}
Ramsay Jones committed
859

860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878
#ifdef GIT_EXPERIMENTAL_SHA256

int git_odb_open(
	git_odb **out,
	const char *objects_dir,
	const git_odb_options *opts)
{
	return git_odb__open(out, objects_dir, opts);
}

#else

int git_odb_open(git_odb **out, const char *objects_dir)
{
	return git_odb__open(out, objects_dir, NULL);
}

#endif

879 880 881
int git_odb__set_caps(git_odb *odb, int caps)
{
	if (caps == GIT_ODB_CAP_FROM_OWNER) {
882
		git_repository *repo = GIT_REFCOUNT_OWNER(odb);
883 884 885
		int val;

		if (!repo) {
886
			git_error_set(GIT_ERROR_ODB, "cannot access repository to set odb caps");
887 888 889
			return -1;
		}

890
		if (!git_repository__configmap_lookup(&val, repo, GIT_CONFIGMAP_FSYNCOBJECTFILES))
891 892 893 894 895 896
			odb->do_fsync = !!val;
	}

	return 0;
}

897
static void odb_free(git_odb *db)
898
{
899
	size_t i;
lhchavez committed
900
	bool locked = true;
901

lhchavez committed
902 903 904 905
	if (git_mutex_lock(&db->lock) < 0) {
		git_error_set(GIT_ERROR_ODB, "failed to acquire the odb lock");
		locked = false;
	}
906
	for (i = 0; i < db->backends.length; ++i) {
907 908
		backend_internal *internal = git_vector_get(&db->backends, i);
		git_odb_backend *backend = internal->backend;
909

910
		backend->free(backend);
911

912
		git__free(internal);
913
	}
lhchavez committed
914 915
	if (locked)
		git_mutex_unlock(&db->lock);
916

917
	git_commit_graph_free(db->cgraph);
918
	git_vector_free(&db->backends);
919
	git_cache_dispose(&db->own_cache);
lhchavez committed
920
	git_mutex_free(&db->lock);
921

922
	git__memzero(db, sizeof(*db));
923
	git__free(db);
924 925
}

926 927 928 929 930 931 932 933
void git_odb_free(git_odb *db)
{
	if (db == NULL)
		return;

	GIT_REFCOUNT_DEC(db, odb_free);
}

934 935 936 937
static int odb_exists_1(
	git_odb *db,
	const git_oid *id,
	bool only_refreshed)
938
{
939
	size_t i;
940
	bool found = false;
lhchavez committed
941
	int error;
942

lhchavez committed
943 944 945 946
	if ((error = git_mutex_lock(&db->lock)) < 0) {
		git_error_set(GIT_ERROR_ODB, "failed to acquire the odb lock");
		return error;
	}
947
	for (i = 0; i < db->backends.length && !found; ++i) {
948 949
		backend_internal *internal = git_vector_get(&db->backends, i);
		git_odb_backend *b = internal->backend;
950

951 952 953
		if (only_refreshed && !b->refresh)
			continue;

954
		if (b->exists != NULL)
Linquize committed
955
			found = (bool)b->exists(b, id);
956
	}
lhchavez committed
957
	git_mutex_unlock(&db->lock);
958

959
	return (int)found;
960 961
}

962
int git_odb__get_commit_graph_file(git_commit_graph_file **out, git_odb *db)
963 964
{
	int error = 0;
965
	git_commit_graph_file *result = NULL;
966 967 968 969 970

	if ((error = git_mutex_lock(&db->lock)) < 0) {
		git_error_set(GIT_ERROR_ODB, "failed to acquire the db lock");
		return error;
	}
971 972 973
	if (!db->cgraph) {
		error = GIT_ENOTFOUND;
		goto done;
974
	}
975 976 977 978
	error = git_commit_graph_get_file(&result, db->cgraph);
	if (error)
		goto done;
	*out = result;
979 980 981

done:
	git_mutex_unlock(&db->lock);
982
	return error;
983 984
}

985 986 987 988 989 990 991
static int odb_freshen_1(
	git_odb *db,
	const git_oid *id,
	bool only_refreshed)
{
	size_t i;
	bool found = false;
lhchavez committed
992
	int error;
993

lhchavez committed
994 995 996 997
	if ((error = git_mutex_lock(&db->lock)) < 0) {
		git_error_set(GIT_ERROR_ODB, "failed to acquire the odb lock");
		return error;
	}
998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009
	for (i = 0; i < db->backends.length && !found; ++i) {
		backend_internal *internal = git_vector_get(&db->backends, i);
		git_odb_backend *b = internal->backend;

		if (only_refreshed && !b->refresh)
			continue;

		if (b->freshen != NULL)
			found = !b->freshen(b, id);
		else if (b->exists != NULL)
			found = b->exists(b, id);
	}
lhchavez committed
1010
	git_mutex_unlock(&db->lock);
1011 1012 1013 1014

	return (int)found;
}

1015
int git_odb__freshen(git_odb *db, const git_oid *id)
1016
{
Edward Thomson committed
1017 1018
	GIT_ASSERT_ARG(db);
	GIT_ASSERT_ARG(id);
1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029

	if (odb_freshen_1(db, id, false))
		return 1;

	if (!git_odb_refresh(db))
		return odb_freshen_1(db, id, true);

	/* Failed to refresh, hence not found */
	return 0;
}

1030
int git_odb_exists(git_odb *db, const git_oid *id)
1031
{
1032 1033 1034 1035 1036
    return git_odb_exists_ext(db, id, 0);
}

int git_odb_exists_ext(git_odb *db, const git_oid *id, unsigned int flags)
{
1037
	git_odb_object *object;
1038

Edward Thomson committed
1039 1040
	GIT_ASSERT_ARG(db);
	GIT_ASSERT_ARG(id);
1041

1042
	if (git_oid_is_zero(id))
1043 1044
		return 0;

1045 1046
	if ((object = git_cache_get_raw(odb_cache(db), id)) != NULL) {
		git_odb_object_free(object);
1047
		return 1;
1048 1049
	}

1050 1051 1052
	if (odb_exists_1(db, id, false))
		return 1;

1053
	if (!(flags & GIT_ODB_LOOKUP_NO_REFRESH) && !git_odb_refresh(db))
1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064
		return odb_exists_1(db, id, true);

	/* Failed to refresh, hence not found */
	return 0;
}

static int odb_exists_prefix_1(git_oid *out, git_odb *db,
	const git_oid *key, size_t len, bool only_refreshed)
{
	size_t i;
	int error = GIT_ENOTFOUND, num_found = 0;
Edward Thomson committed
1065
	git_oid last_found = GIT_OID_NONE, found;
1066

lhchavez committed
1067 1068 1069 1070 1071
	if ((error = git_mutex_lock(&db->lock)) < 0) {
		git_error_set(GIT_ERROR_ODB, "failed to acquire the odb lock");
		return error;
	}
	error = GIT_ENOTFOUND;
1072 1073 1074 1075
	for (i = 0; i < db->backends.length; ++i) {
		backend_internal *internal = git_vector_get(&db->backends, i);
		git_odb_backend *b = internal->backend;

1076 1077 1078
		if (only_refreshed && !b->refresh)
			continue;

1079 1080 1081
		if (!b->exists_prefix)
			continue;

1082
		error = b->exists_prefix(&found, b, key, len);
1083 1084
		if (error == GIT_ENOTFOUND || error == GIT_PASSTHROUGH)
			continue;
lhchavez committed
1085 1086
		if (error) {
			git_mutex_unlock(&db->lock);
1087
			return error;
lhchavez committed
1088
		}
1089 1090 1091

		/* make sure found item doesn't introduce ambiguity */
		if (num_found) {
lhchavez committed
1092 1093
			if (git_oid__cmp(&last_found, &found)) {
				git_mutex_unlock(&db->lock);
1094
				return git_odb__error_ambiguous("multiple matches for prefix");
lhchavez committed
1095
			}
1096 1097 1098 1099 1100
		} else {
			git_oid_cpy(&last_found, &found);
			num_found++;
		}
	}
lhchavez committed
1101
	git_mutex_unlock(&db->lock);
1102 1103

	if (!num_found)
1104 1105
		return GIT_ENOTFOUND;

1106 1107 1108
	if (out)
		git_oid_cpy(out, &last_found);

1109
	return 0;
1110 1111
}

1112 1113 1114 1115
int git_odb_exists_prefix(
	git_oid *out, git_odb *db, const git_oid *short_id, size_t len)
{
	int error;
Edward Thomson committed
1116
	git_oid key = GIT_OID_NONE;
1117

Edward Thomson committed
1118 1119
	GIT_ASSERT_ARG(db);
	GIT_ASSERT_ARG(short_id);
1120 1121 1122 1123

	if (len < GIT_OID_MINPREFIXLEN)
		return git_odb__error_ambiguous("prefix length too short");

1124
	if (len >= git_oid_hexsize(db->options.oid_type)) {
1125 1126 1127 1128 1129
		if (git_odb_exists(db, short_id)) {
			if (out)
				git_oid_cpy(out, short_id);
			return 0;
		} else {
1130 1131
			return git_odb__error_notfound(
				"no match for id prefix", short_id, len);
1132 1133 1134
		}
	}

1135
	git_oid__cpy_prefix(&key, short_id, len);
1136 1137 1138 1139 1140 1141 1142

	error = odb_exists_prefix_1(out, db, &key, len, false);

	if (error == GIT_ENOTFOUND && !git_odb_refresh(db))
		error = odb_exists_prefix_1(out, db, &key, len, true);

	if (error == GIT_ENOTFOUND)
1143
		return git_odb__error_notfound("no match for id prefix", &key, len);
1144 1145 1146 1147

	return error;
}

1148
int git_odb_expand_ids(
1149
	git_odb *db,
1150 1151
	git_odb_expand_id *ids,
	size_t count)
1152
{
1153
	size_t hex_size, i;
1154

Edward Thomson committed
1155 1156
	GIT_ASSERT_ARG(db);
	GIT_ASSERT_ARG(ids);
1157

1158 1159
	hex_size = git_oid_hexsize(db->options.oid_type);

1160 1161
	for (i = 0; i < count; i++) {
		git_odb_expand_id *query = &ids[i];
1162
		int error = GIT_EAMBIGUOUS;
1163

1164
		if (!query->type)
1165
			query->type = GIT_OBJECT_ANY;
1166 1167

		/* if we have a short OID, expand it first */
1168
		if (query->length >= GIT_OID_MINPREFIXLEN && query->length < hex_size) {
1169 1170 1171 1172 1173
			git_oid actual_id;

			error = odb_exists_prefix_1(&actual_id, db, &query->id, query->length, false);
			if (!error) {
				git_oid_cpy(&query->id, &actual_id);
1174
				query->length = (unsigned short)hex_size;
1175
			}
1176 1177
		}

1178
		/*
1179 1180
		 * now we ought to have a 40-char OID, either because we've expanded it
		 * or because the user passed a full OID. Ensure its type is right.
1181
		 */
1182
		if (query->length >= hex_size) {
1183
			git_object_t actual_type;
1184

1185 1186
			error = odb_otype_fast(&actual_type, db, &query->id);
			if (!error) {
1187
				if (query->type != GIT_OBJECT_ANY && query->type != actual_type)
1188 1189 1190 1191 1192
					error = GIT_ENOTFOUND;
				else
					query->type = actual_type;
			}
		}
1193

1194
		switch (error) {
1195
		/* no errors, so we've successfully expanded the OID */
1196
		case 0:
1197
			continue;
1198 1199 1200 1201

		/* the object is missing or ambiguous */
		case GIT_ENOTFOUND:
		case GIT_EAMBIGUOUS:
1202
			git_oid_clear(&query->id, db->options.oid_type);
1203 1204
			query->length = 0;
			query->type = 0;
1205 1206 1207 1208 1209
			break;

		/* something went very wrong with the ODB; bail hard */
		default:
			return error;
1210 1211 1212
		}
	}

1213
	git_error_clear();
1214
	return 0;
1215 1216
}

1217
int git_odb_read_header(size_t *len_p, git_object_t *type_p, git_odb *db, const git_oid *id)
1218
{
1219
	int error;
1220
	git_odb_object *object = NULL;
1221 1222 1223 1224 1225 1226 1227 1228 1229

	error = git_odb__read_header_or_object(&object, len_p, type_p, db, id);

	if (object)
		git_odb_object_free(object);

	return error;
}

1230
static int odb_read_header_1(
1231
	size_t *len_p, git_object_t *type_p, git_odb *db,
1232 1233 1234
	const git_oid *id, bool only_refreshed)
{
	size_t i;
1235
	git_object_t ht;
1236 1237
	bool passthrough = false;
	int error;
1238

1239
	if (!only_refreshed && (ht = odb_hardcoded_type(id)) != GIT_OBJECT_INVALID) {
1240 1241 1242 1243 1244
		*type_p = ht;
		*len_p = 0;
		return 0;
	}

lhchavez committed
1245 1246 1247 1248
	if ((error = git_mutex_lock(&db->lock)) < 0) {
		git_error_set(GIT_ERROR_ODB, "failed to acquire the odb lock");
		return error;
	}
1249
	for (i = 0; i < db->backends.length; ++i) {
1250 1251 1252 1253 1254 1255
		backend_internal *internal = git_vector_get(&db->backends, i);
		git_odb_backend *b = internal->backend;

		if (only_refreshed && !b->refresh)
			continue;

1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269
		if (!b->read_header) {
			passthrough = true;
			continue;
		}

		error = b->read_header(len_p, type_p, b, id);

		switch (error) {
		case GIT_PASSTHROUGH:
			passthrough = true;
			break;
		case GIT_ENOTFOUND:
			break;
		default:
lhchavez committed
1270
			git_mutex_unlock(&db->lock);
1271 1272
			return error;
		}
1273
	}
lhchavez committed
1274
	git_mutex_unlock(&db->lock);
1275

1276
	return passthrough ? GIT_PASSTHROUGH : GIT_ENOTFOUND;
1277 1278
}

1279
int git_odb__read_header_or_object(
1280
	git_odb_object **out, size_t *len_p, git_object_t *type_p,
1281 1282
	git_odb *db, const git_oid *id)
{
1283
	int error = GIT_ENOTFOUND;
Vicent Marti committed
1284
	git_odb_object *object;
1285

Edward Thomson committed
1286 1287 1288 1289 1290
	GIT_ASSERT_ARG(db);
	GIT_ASSERT_ARG(id);
	GIT_ASSERT_ARG(out);
	GIT_ASSERT_ARG(len_p);
	GIT_ASSERT_ARG(type_p);
Vicent Marti committed
1291

1292 1293
	*out = NULL;

1294
	if (git_oid_is_zero(id))
1295 1296
		return error_null_oid(GIT_ENOTFOUND, "cannot read object");

1297
	if ((object = git_cache_get_raw(odb_cache(db), id)) != NULL) {
Vicent Marti committed
1298 1299
		*len_p = object->cached.size;
		*type_p = object->cached.type;
1300
		*out = object;
1301
		return 0;
Vicent Marti committed
1302
	}
1303

1304
	error = odb_read_header_1(len_p, type_p, db, id, false);
1305

1306 1307
	if (error == GIT_ENOTFOUND && !git_odb_refresh(db))
		error = odb_read_header_1(len_p, type_p, db, id, true);
1308

1309
	if (error == GIT_ENOTFOUND)
1310
		return git_odb__error_notfound("cannot read header for", id, git_oid_hexsize(db->options.oid_type));
1311

1312 1313
	/* we found the header; return early */
	if (!error)
1314
		return 0;
Vicent Marti committed
1315

1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326
	if (error == GIT_PASSTHROUGH) {
		/*
		 * no backend has header-reading functionality
		 * so try using `git_odb_read` instead
		 */
		error = git_odb_read(&object, db, id);
		if (!error) {
			*len_p = object->cached.size;
			*type_p = object->cached.type;
			*out = object;
		}
1327
	}
1328 1329

	return error;
1330 1331
}

1332 1333 1334 1335 1336
static int odb_read_1(
	git_odb_object **out,
	git_odb *db,
	const git_oid *id,
	bool only_refreshed)
1337
{
1338
	size_t i;
Vicent Marti committed
1339
	git_rawobj raw;
1340
	git_odb_object *object;
1341
	git_oid hashed;
1342
	bool found = false;
1343
	int error = 0;
1344

1345 1346 1347 1348
	if (!only_refreshed) {
		if ((error = odb_read_hardcoded(&found, &raw, id)) < 0)
			return error;
	}
Vicent Marti committed
1349

lhchavez committed
1350 1351 1352 1353
	if ((error = git_mutex_lock(&db->lock)) < 0) {
		git_error_set(GIT_ERROR_ODB, "failed to acquire the odb lock");
		return error;
	}
1354
	for (i = 0; i < db->backends.length && !found; ++i) {
1355 1356
		backend_internal *internal = git_vector_get(&db->backends, i);
		git_odb_backend *b = internal->backend;
1357

1358 1359 1360
		if (only_refreshed && !b->refresh)
			continue;

1361
		if (b->read != NULL) {
1362
			error = b->read(&raw.data, &raw.len, &raw.type, b, id);
1363 1364 1365
			if (error == GIT_PASSTHROUGH || error == GIT_ENOTFOUND)
				continue;

lhchavez committed
1366 1367
			if (error < 0) {
				git_mutex_unlock(&db->lock);
1368
				return error;
lhchavez committed
1369
			}
1370 1371

			found = true;
1372
		}
Vicent Marti committed
1373
	}
lhchavez committed
1374
	git_mutex_unlock(&db->lock);
Vicent Marti committed
1375

1376 1377
	if (!found)
		return GIT_ENOTFOUND;
1378

1379
	if (git_odb__strict_hash_verification) {
1380
		if ((error = git_odb__hash(&hashed, raw.data, raw.len, raw.type, db->options.oid_type)) < 0)
1381
			goto out;
1382

1383 1384 1385 1386
		if (!git_oid_equal(id, &hashed)) {
			error = git_odb__error_mismatch(id, &hashed);
			goto out;
		}
1387 1388
	}

1389
	git_error_clear();
1390 1391
	if ((object = odb_object__alloc(id, &raw)) == NULL) {
		error = -1;
1392
		goto out;
1393
	}
1394 1395

	*out = git_cache_store_raw(odb_cache(db), object);
1396 1397 1398 1399 1400

out:
	if (error)
		git__free(raw.data);
	return error;
1401 1402
}

1403 1404 1405 1406
int git_odb_read(git_odb_object **out, git_odb *db, const git_oid *id)
{
	int error;

Edward Thomson committed
1407 1408 1409
	GIT_ASSERT_ARG(out);
	GIT_ASSERT_ARG(db);
	GIT_ASSERT_ARG(id);
1410

1411
	if (git_oid_is_zero(id))
1412 1413
		return error_null_oid(GIT_ENOTFOUND, "cannot read object");

1414 1415 1416 1417 1418 1419 1420 1421 1422 1423
	*out = git_cache_get_raw(odb_cache(db), id);
	if (*out != NULL)
		return 0;

	error = odb_read_1(out, db, id, false);

	if (error == GIT_ENOTFOUND && !git_odb_refresh(db))
		error = odb_read_1(out, db, id, true);

	if (error == GIT_ENOTFOUND)
1424
		return git_odb__error_notfound("no match for id", id, git_oid_hexsize(git_oid_type(id)));
1425 1426 1427 1428

	return error;
}

1429
static int odb_otype_fast(git_object_t *type_p, git_odb *db, const git_oid *id)
1430 1431 1432 1433 1434
{
	git_odb_object *object;
	size_t _unused;
	int error;

1435
	if (git_oid_is_zero(id))
1436 1437
		return error_null_oid(GIT_ENOTFOUND, "cannot get object type");

1438 1439
	if ((object = git_cache_get_raw(odb_cache(db), id)) != NULL) {
		*type_p = object->cached.type;
1440
		git_odb_object_free(object);
1441 1442
		return 0;
	}
1443

1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455
	error = odb_read_header_1(&_unused, type_p, db, id, false);

	if (error == GIT_PASSTHROUGH) {
		error = odb_read_1(&object, db, id, false);
		if (!error)
			*type_p = object->cached.type;
		git_odb_object_free(object);
	}

	return error;
}

1456 1457
static int read_prefix_1(git_odb_object **out, git_odb *db,
		const git_oid *key, size_t len, bool only_refreshed)
1458
{
1459
	size_t i;
1460
	int error = 0;
Edward Thomson committed
1461
	git_oid found_full_oid = GIT_OID_NONE;
1462
	git_rawobj raw = {0};
1463
	void *data = NULL;
1464
	bool found = false;
1465
	git_odb_object *object;
1466

lhchavez committed
1467 1468 1469 1470
	if ((error = git_mutex_lock(&db->lock)) < 0) {
		git_error_set(GIT_ERROR_ODB, "failed to acquire the odb lock");
		return error;
	}
1471
	for (i = 0; i < db->backends.length; ++i) {
1472 1473 1474
		backend_internal *internal = git_vector_get(&db->backends, i);
		git_odb_backend *b = internal->backend;

1475 1476 1477
		if (only_refreshed && !b->refresh)
			continue;

1478
		if (b->read_prefix != NULL) {
1479
			git_oid full_oid;
1480
			error = b->read_prefix(&full_oid, &raw.data, &raw.len, &raw.type, b, key, len);
1481 1482 1483

			if (error == GIT_ENOTFOUND || error == GIT_PASSTHROUGH) {
				error = 0;
1484
				continue;
1485
			}
1486

lhchavez committed
1487 1488
			if (error) {
				git_mutex_unlock(&db->lock);
1489
				goto out;
lhchavez committed
1490
			}
1491

1492 1493
			git__free(data);
			data = raw.data;
Vicent Marti committed
1494

1495
			if (found && git_oid__cmp(&full_oid, &found_full_oid)) {
1496
				git_str buf = GIT_STR_INIT;
1497
				const char *idstr;
1498

1499 1500 1501 1502 1503 1504 1505 1506
				if ((idstr = git_oid_tostr_s(&full_oid)) == NULL) {
					git_str_puts(&buf, "failed to parse object id");
				} else {
					git_str_printf(&buf, "multiple matches for prefix: %s", idstr);

					if ((idstr = git_oid_tostr_s(&found_full_oid)) != NULL)
						git_str_printf(&buf, " %s", idstr);
				}
1507 1508

				error = git_odb__error_ambiguous(buf.ptr);
1509
				git_str_dispose(&buf);
lhchavez committed
1510
				git_mutex_unlock(&db->lock);
1511
				goto out;
1512
			}
Vicent Marti committed
1513

1514 1515
			found_full_oid = full_oid;
			found = true;
1516 1517
		}
	}
lhchavez committed
1518
	git_mutex_unlock(&db->lock);
1519

1520
	if (!found)
1521
		return GIT_ENOTFOUND;
1522

1523 1524 1525
	if (git_odb__strict_hash_verification) {
		git_oid hash;

1526
		if ((error = git_odb__hash(&hash, raw.data, raw.len, raw.type, db->options.oid_type)) < 0)
1527 1528 1529 1530 1531 1532 1533 1534
			goto out;

		if (!git_oid_equal(&found_full_oid, &hash)) {
			error = git_odb__error_mismatch(&found_full_oid, &hash);
			goto out;
		}
	}

1535 1536
	if ((object = odb_object__alloc(&found_full_oid, &raw)) == NULL) {
		error = -1;
1537
		goto out;
1538
	}
1539 1540

	*out = git_cache_store_raw(odb_cache(db), object);
1541 1542 1543 1544 1545 1546

out:
	if (error)
		git__free(raw.data);

	return error;
1547 1548
}

1549 1550 1551
int git_odb_read_prefix(
	git_odb_object **out, git_odb *db, const git_oid *short_id, size_t len)
{
Edward Thomson committed
1552 1553
	git_oid key = GIT_OID_NONE;
	size_t hex_size;
1554 1555
	int error;

Edward Thomson committed
1556 1557
	GIT_ASSERT_ARG(out);
	GIT_ASSERT_ARG(db);
1558

1559 1560
	hex_size = git_oid_hexsize(db->options.oid_type);

1561 1562 1563
	if (len < GIT_OID_MINPREFIXLEN)
		return git_odb__error_ambiguous("prefix length too short");

1564 1565
	if (len > hex_size)
		len = hex_size;
1566

1567
	if (len == hex_size) {
1568 1569 1570 1571 1572
		*out = git_cache_get_raw(odb_cache(db), short_id);
		if (*out != NULL)
			return 0;
	}

1573
	git_oid__cpy_prefix(&key, short_id, len);
1574 1575 1576 1577 1578 1579 1580

	error = read_prefix_1(out, db, &key, len, false);

	if (error == GIT_ENOTFOUND && !git_odb_refresh(db))
		error = read_prefix_1(out, db, &key, len, true);

	if (error == GIT_ENOTFOUND)
1581
		return git_odb__error_notfound("no match for prefix", &key, len);
1582 1583 1584 1585

	return error;
}

Ben Straub committed
1586
int git_odb_foreach(git_odb *db, git_odb_foreach_cb cb, void *payload)
1587 1588
{
	unsigned int i;
lhchavez committed
1589
	git_vector backends = GIT_VECTOR_INIT;
1590
	backend_internal *internal;
lhchavez committed
1591 1592 1593 1594 1595 1596 1597 1598 1599
	int error = 0;

	/* Make a copy of the backends vector to invoke the callback without holding the lock. */
	if ((error = git_mutex_lock(&db->lock)) < 0) {
		git_error_set(GIT_ERROR_ODB, "failed to acquire the odb lock");
		goto cleanup;
	}
	error = git_vector_dup(&backends, &db->backends, NULL);
	git_mutex_unlock(&db->lock);
1600

lhchavez committed
1601 1602 1603 1604
	if (error < 0)
		goto cleanup;

	git_vector_foreach(&backends, i, internal) {
1605
		git_odb_backend *b = internal->backend;
lhchavez committed
1606
		error = b->foreach(b, cb, payload);
1607
		if (error != 0)
lhchavez committed
1608
			goto cleanup;
1609 1610
	}

lhchavez committed
1611 1612 1613 1614
cleanup:
	git_vector_free(&backends);

	return error;
1615 1616
}

1617
int git_odb_write(
1618
	git_oid *oid, git_odb *db, const void *data, size_t len, git_object_t type)
1619
{
1620
	size_t i;
1621
	int error;
Vicent Marti committed
1622
	git_odb_stream *stream;
1623

Edward Thomson committed
1624 1625
	GIT_ASSERT_ARG(oid);
	GIT_ASSERT_ARG(db);
1626

1627
	if ((error = git_odb__hash(oid, data, len, type, db->options.oid_type)) < 0)
1628
		return error;
1629

1630
	if (git_oid_is_zero(oid))
1631 1632
		return error_null_oid(GIT_EINVALID, "cannot write object");

1633
	if (git_odb__freshen(db, oid))
1634 1635
		return 0;

lhchavez committed
1636 1637 1638 1639
	if ((error = git_mutex_lock(&db->lock)) < 0) {
		git_error_set(GIT_ERROR_ODB, "failed to acquire the odb lock");
		return error;
	}
1640
	for (i = 0, error = GIT_ERROR; i < db->backends.length && error < 0; ++i) {
1641 1642 1643 1644 1645 1646 1647 1648
		backend_internal *internal = git_vector_get(&db->backends, i);
		git_odb_backend *b = internal->backend;

		/* we don't write in alternates! */
		if (internal->is_alternate)
			continue;

		if (b->write != NULL)
1649
			error = b->write(b, oid, data, len, type);
1650
	}
lhchavez committed
1651
	git_mutex_unlock(&db->lock);
1652

1653
	if (!error || error == GIT_PASSTHROUGH)
1654
		return 0;
Vicent Marti committed
1655

1656 1657 1658 1659
	/* if no backends were able to write the object directly, we try a
	 * streaming write to the backends; just write the whole object into the
	 * stream in one push
	 */
1660 1661
	if ((error = git_odb_open_wstream(&stream, db, len, type)) != 0)
		return error;
1662

1663 1664
	if ((error = stream->write(stream, data, len)) == 0)
		error = stream->finalize_write(stream, oid);
1665

1666
	git_odb_stream_free(stream);
1667
	return error;
1668 1669
}

1670
static int hash_header(git_hash_ctx *ctx, git_object_size_t size, git_object_t type)
1671 1672
{
	char header[64];
1673 1674
	size_t hdrlen;
	int error;
1675

1676 1677 1678 1679 1680
	 if ((error = git_odb__format_object_header(&hdrlen,
		header, sizeof(header), size, type)) < 0)
		return error;

	return git_hash_update(ctx, header, hdrlen);
1681 1682
}

1683
int git_odb_open_wstream(
1684
	git_odb_stream **stream, git_odb *db, git_object_size_t size, git_object_t type)
1685
{
1686
	size_t i, writes = 0;
1687
	int error = GIT_ERROR;
1688
	git_hash_ctx *ctx = NULL;
1689

Edward Thomson committed
1690 1691
	GIT_ASSERT_ARG(stream);
	GIT_ASSERT_ARG(db);
1692

lhchavez committed
1693 1694 1695 1696 1697
	if ((error = git_mutex_lock(&db->lock)) < 0) {
		git_error_set(GIT_ERROR_ODB, "failed to acquire the odb lock");
		return error;
	}
	error = GIT_ERROR;
1698
	for (i = 0; i < db->backends.length && error < 0; ++i) {
1699 1700 1701 1702 1703 1704
		backend_internal *internal = git_vector_get(&db->backends, i);
		git_odb_backend *b = internal->backend;

		/* we don't write in alternates! */
		if (internal->is_alternate)
			continue;
1705

1706 1707
		if (b->writestream != NULL) {
			++writes;
Vicent Marti committed
1708
			error = b->writestream(stream, b, size, type);
1709 1710
		} else if (b->write != NULL) {
			++writes;
1711
			error = init_fake_wstream(stream, b, size, type);
1712
		}
Vicent Marti committed
1713
	}
lhchavez committed
1714
	git_mutex_unlock(&db->lock);
Vicent Marti committed
1715

1716 1717 1718 1719 1720 1721 1722 1723
	if (error < 0) {
		if (error == GIT_PASSTHROUGH)
			error = 0;
		else if (!writes)
			error = git_odb__error_unsupported_in_backend("write object");

		goto done;
	}
Vicent Marti committed
1724

1725
	ctx = git__malloc(sizeof(git_hash_ctx));
1726
	GIT_ERROR_CHECK_ALLOC(ctx);
1727

1728
	if ((error = git_hash_ctx_init(ctx, git_oid_algorithm(db->options.oid_type))) < 0 ||
1729
	    (error = hash_header(ctx, size, type)) < 0)
1730
		goto done;
1731

1732
#ifdef GIT_EXPERIMENTAL_SHA256
1733
	(*stream)->oid_type = db->options.oid_type;
1734
#endif
1735
	(*stream)->hash_ctx = ctx;
1736 1737 1738
	(*stream)->declared_size = size;
	(*stream)->received_bytes = 0;

1739
done:
1740 1741
	if (error)
		git__free(ctx);
1742
	return error;
Vicent Marti committed
1743 1744
}

1745 1746 1747 1748
static int git_odb_stream__invalid_length(
	const git_odb_stream *stream,
	const char *action)
{
1749
	git_error_set(GIT_ERROR_ODB,
1750
		"cannot %s - "
1751 1752
		"Invalid length. %"PRId64" was expected. The "
		"total size of the received chunks amounts to %"PRId64".",
1753
		action, stream->declared_size, stream->received_bytes);
1754 1755 1756 1757

	return -1;
}

1758 1759
int git_odb_stream_write(git_odb_stream *stream, const char *buffer, size_t len)
{
1760
	git_hash_update(stream->hash_ctx, buffer, len);
1761 1762 1763 1764 1765 1766 1767

	stream->received_bytes += len;

	if (stream->received_bytes > stream->declared_size)
		return git_odb_stream__invalid_length(stream,
			"stream_write()");

1768 1769 1770 1771 1772
	return stream->write(stream, buffer, len);
}

int git_odb_stream_finalize_write(git_oid *out, git_odb_stream *stream)
{
1773 1774 1775 1776
	if (stream->received_bytes != stream->declared_size)
		return git_odb_stream__invalid_length(stream,
			"stream_finalize_write()");

1777
	git_hash_final(out->id, stream->hash_ctx);
1778 1779

#ifdef GIT_EXPERIMENTAL_SHA256
1780
	out->type = stream->oid_type;
1781
#endif
1782

1783
	if (git_odb__freshen(stream->backend->odb, out))
1784 1785
		return 0;

1786
	return stream->finalize_write(stream, out);
1787 1788 1789 1790 1791 1792 1793 1794 1795
}

int git_odb_stream_read(git_odb_stream *stream, char *buffer, size_t len)
{
	return stream->read(stream, buffer, len);
}

void git_odb_stream_free(git_odb_stream *stream)
{
1796 1797 1798
	if (stream == NULL)
		return;

1799
	git_hash_ctx_cleanup(stream->hash_ctx);
1800
	git__free(stream->hash_ctx);
1801 1802 1803
	stream->free(stream);
}

1804 1805 1806
int git_odb_open_rstream(
	git_odb_stream **stream,
	size_t *len,
1807
	git_object_t *type,
1808 1809
	git_odb *db,
	const git_oid *oid)
Vicent Marti committed
1810
{
1811
	size_t i, reads = 0;
Vicent Marti committed
1812 1813
	int error = GIT_ERROR;

Edward Thomson committed
1814 1815
	GIT_ASSERT_ARG(stream);
	GIT_ASSERT_ARG(db);
Vicent Marti committed
1816

lhchavez committed
1817 1818 1819 1820 1821
	if ((error = git_mutex_lock(&db->lock)) < 0) {
		git_error_set(GIT_ERROR_ODB, "failed to acquire the odb lock");
		return error;
	}
	error = GIT_ERROR;
Vicent Marti committed
1822 1823 1824 1825
	for (i = 0; i < db->backends.length && error < 0; ++i) {
		backend_internal *internal = git_vector_get(&db->backends, i);
		git_odb_backend *b = internal->backend;

1826 1827
		if (b->readstream != NULL) {
			++reads;
1828
			error = b->readstream(stream, len, type, b, oid);
1829
		}
1830
	}
lhchavez committed
1831
	git_mutex_unlock(&db->lock);
1832

1833
	if (error == GIT_PASSTHROUGH)
1834
		error = 0;
1835 1836
	if (error < 0 && !reads)
		error = git_odb__error_unsupported_in_backend("read object streamed");
Vicent Marti committed
1837

1838 1839 1840
	return error;
}

1841
int git_odb_write_pack(struct git_odb_writepack **out, git_odb *db, git_indexer_progress_cb progress_cb, void *progress_payload)
1842
{
1843
	size_t i, writes = 0;
1844 1845
	int error = GIT_ERROR;

Edward Thomson committed
1846 1847
	GIT_ASSERT_ARG(out);
	GIT_ASSERT_ARG(db);
1848

lhchavez committed
1849 1850 1851 1852 1853
	if ((error = git_mutex_lock(&db->lock)) < 0) {
		git_error_set(GIT_ERROR_ODB, "failed to acquire the odb lock");
		return error;
	}
	error = GIT_ERROR;
1854 1855 1856 1857 1858 1859 1860 1861
	for (i = 0; i < db->backends.length && error < 0; ++i) {
		backend_internal *internal = git_vector_get(&db->backends, i);
		git_odb_backend *b = internal->backend;

		/* we don't write in alternates! */
		if (internal->is_alternate)
			continue;

1862 1863
		if (b->writepack != NULL) {
			++writes;
1864
			error = b->writepack(out, b, db, progress_cb, progress_payload);
1865
		}
1866
	}
lhchavez committed
1867
	git_mutex_unlock(&db->lock);
1868 1869 1870

	if (error == GIT_PASSTHROUGH)
		error = 0;
1871 1872
	if (error < 0 && !writes)
		error = git_odb__error_unsupported_in_backend("write pack");
1873 1874 1875 1876

	return error;
}

1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905
int git_odb_write_multi_pack_index(git_odb *db)
{
	size_t i, writes = 0;
	int error = GIT_ERROR;

	GIT_ASSERT_ARG(db);

	for (i = 0; i < db->backends.length && error < 0; ++i) {
		backend_internal *internal = git_vector_get(&db->backends, i);
		git_odb_backend *b = internal->backend;

		/* we don't write in alternates! */
		if (internal->is_alternate)
			continue;

		if (b->writemidx != NULL) {
			++writes;
			error = b->writemidx(b);
		}
	}

	if (error == GIT_PASSTHROUGH)
		error = 0;
	if (error < 0 && !writes)
		error = git_odb__error_unsupported_in_backend("write multi-pack-index");

	return error;
}

1906
void *git_odb_backend_data_alloc(git_odb_backend *backend, size_t len)
1907
{
1908
	GIT_UNUSED(backend);
1909 1910 1911
	return git__malloc(len);
}

1912
#ifndef GIT_DEPRECATE_HARD
1913 1914 1915 1916
void *git_odb_backend_malloc(git_odb_backend *backend, size_t len)
{
	return git_odb_backend_data_alloc(backend, len);
}
1917
#endif
1918

1919 1920 1921 1922 1923 1924
void git_odb_backend_data_free(git_odb_backend *backend, void *data)
{
	GIT_UNUSED(backend);
	git__free(data);
}

Vicent Marti committed
1925 1926
int git_odb_refresh(struct git_odb *db)
{
1927
	size_t i;
lhchavez committed
1928
	int error;
Edward Thomson committed
1929 1930

	GIT_ASSERT_ARG(db);
Vicent Marti committed
1931

lhchavez committed
1932 1933 1934 1935
	if ((error = git_mutex_lock(&db->lock)) < 0) {
		git_error_set(GIT_ERROR_ODB, "failed to acquire the odb lock");
		return error;
	}
Vicent Marti committed
1936 1937 1938 1939 1940 1941
	for (i = 0; i < db->backends.length; ++i) {
		backend_internal *internal = git_vector_get(&db->backends, i);
		git_odb_backend *b = internal->backend;

		if (b->refresh != NULL) {
			int error = b->refresh(b);
lhchavez committed
1942 1943
			if (error < 0) {
				git_mutex_unlock(&db->lock);
Vicent Marti committed
1944
				return error;
lhchavez committed
1945
			}
Vicent Marti committed
1946 1947
		}
	}
1948 1949
	if (db->cgraph)
		git_commit_graph_refresh(db->cgraph);
lhchavez committed
1950
	git_mutex_unlock(&db->lock);
Vicent Marti committed
1951 1952 1953 1954

	return 0;
}

1955 1956
int git_odb__error_mismatch(const git_oid *expected, const git_oid *actual)
{
1957 1958
	char expected_oid[GIT_OID_MAX_HEXSIZE + 1],
	     actual_oid[GIT_OID_MAX_HEXSIZE + 1];
1959

1960 1961
	git_oid_tostr(expected_oid, git_oid_hexsize(git_oid_type(expected)) + 1, expected);
	git_oid_tostr(actual_oid, git_oid_hexsize(git_oid_type(actual)) + 1, actual);
1962

1963
	git_error_set(GIT_ERROR_ODB, "object hash mismatch - expected %s but got %s",
1964 1965 1966 1967 1968
		expected_oid, actual_oid);

	return GIT_EMISMATCH;
}

1969 1970
int git_odb__error_notfound(
	const char *message, const git_oid *oid, size_t oid_len)
1971
{
Russell Belfer committed
1972
	if (oid != NULL) {
1973
		char oid_str[GIT_OID_MAX_HEXSIZE + 1];
1974
		git_oid_tostr(oid_str, oid_len+1, oid);
1975
		git_error_set(GIT_ERROR_ODB, "object not found - %s (%.*s)",
1976
			message, (int) oid_len, oid_str);
Russell Belfer committed
1977
	} else
1978
		git_error_set(GIT_ERROR_ODB, "object not found - %s", message);
Russell Belfer committed
1979

1980
	return GIT_ENOTFOUND;
1981 1982
}

1983 1984
static int error_null_oid(int error, const char *message)
{
1985
	git_error_set(GIT_ERROR_ODB, "odb: %s: null OID cannot exist", message);
1986 1987 1988
	return error;
}

1989 1990
int git_odb__error_ambiguous(const char *message)
{
1991
	git_error_set(GIT_ERROR_ODB, "ambiguous OID prefix - %s", message);
1992
	return GIT_EAMBIGUOUS;
1993 1994
}

1995
int git_odb_init_backend(git_odb_backend *backend, unsigned int version)
1996
{
1997 1998 1999
	GIT_INIT_STRUCTURE_FROM_TEMPLATE(
		backend, version, git_odb_backend, GIT_ODB_BACKEND_INIT);
	return 0;
2000
}