odb_loose.c 22.9 KB
Newer Older
1
/*
Edward Thomson committed
2
 * Copyright (C) the libgit2 contributors. All rights reserved.
3
 *
Vicent Marti committed
4 5
 * This file is part of libgit2, distributed under the GNU GPL v2 with
 * a Linking Exception. For full terms see the included COPYING file.
6 7 8
 */

#include "common.h"
9
#include <zlib.h>
10
#include "git2/object.h"
11
#include "git2/sys/odb_backend.h"
12 13 14 15
#include "fileops.h"
#include "hash.h"
#include "odb.h"
#include "delta-apply.h"
Vicent Marti committed
16
#include "filebuf.h"
17

18
#include "git2/odb_backend.h"
Vicent Marti committed
19
#include "git2/types.h"
20

Vicent Marti committed
21 22 23
typedef struct { /* object header data */
	git_otype type; /* object type */
	size_t	size; /* object size */
24 25
} obj_hdr;

Vicent Marti committed
26 27 28 29 30
typedef struct {
	git_odb_stream stream;
	git_filebuf fbuf;
} loose_writestream;

31 32 33 34 35
typedef struct loose_backend {
	git_odb_backend parent;

	int object_zlib_level; /** loose object zlib compression level. */
	int fsync_object_files; /** loose object file fsync flag. */
36 37
	mode_t object_file_mode;
	mode_t object_dir_mode;
38 39 40

	size_t objects_dirlen;
	char objects_dir[GIT_FLEX_ARRAY];
41 42
} loose_backend;

43 44 45 46 47 48
/* State structure for exploring directories,
 * in order to locate objects matching a short oid.
 */
typedef struct {
	size_t dir_len;
	unsigned char short_oid[GIT_OID_HEXSZ]; /* hex formatted oid to match */
49
	size_t short_oid_len;
50 51 52 53 54 55 56
	int found;				/* number of matching
						 * objects already found */
	unsigned char res_oid[GIT_OID_HEXSZ];	/* hex formatted oid of
						 * the object found */
} loose_locate_object_state;


57 58
/***********************************************************
 *
Will Stamper committed
59
 * MISCELLANEOUS HELPER FUNCTIONS
60 61 62
 *
 ***********************************************************/

63 64
static int object_file_name(
	git_buf *name, const loose_backend *be, const git_oid *id)
65
{
66 67
	size_t alloclen;

68
	/* expand length for object root + 40 hex sha1 chars + 2 * '/' + '\0' */
69 70 71
	GITERR_CHECK_ALLOC_ADD(&alloclen, be->objects_dirlen, GIT_OID_HEXSZ);
	GITERR_CHECK_ALLOC_ADD(&alloclen, alloclen, 3);
	if (git_buf_grow(name, alloclen) < 0)
72
		return -1;
73

74
	git_buf_set(name, be->objects_dir, be->objects_dirlen);
75
	git_path_to_dir(name);
76 77

	/* loose object filename: aa/aaa... (41 bytes) */
78
	git_oid_pathfmt(name->ptr + name->size, id);
79 80
	name->size += GIT_OID_HEXSZ + 1;
	name->ptr[name->size] = '\0';
81

82
	return 0;
83 84
}

85 86
static int object_mkdir(const git_buf *name, const loose_backend *be)
{
87
	return git_futils_mkdir_relative(
88
		name->ptr + be->objects_dirlen, be->objects_dir, be->object_dir_mode,
89
		GIT_MKDIR_PATH | GIT_MKDIR_SKIP_LAST | GIT_MKDIR_VERIFY_DIR, NULL);
90
}
91

92
static size_t get_binary_object_header(obj_hdr *hdr, git_buf *obj)
93
{
94
	unsigned long c;
95
	unsigned char *data = (unsigned char *)obj->ptr;
96 97
	size_t shift, size, used = 0;

nulltoken committed
98
	if (git_buf_len(obj) == 0)
99 100 101 102 103 104 105 106
		return 0;

	c = data[used++];
	hdr->type = (c >> 4) & 7;

	size = c & 15;
	shift = 4;
	while (c & 0x80) {
nulltoken committed
107
		if (git_buf_len(obj) <= used)
108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135
			return 0;
		if (sizeof(size_t) * 8 <= shift)
			return 0;
		c = data[used++];
		size += (c & 0x7f) << shift;
		shift += 7;
	}
	hdr->size = size;

	return used;
}

static size_t get_object_header(obj_hdr *hdr, unsigned char *data)
{
	char c, typename[10];
	size_t size, used = 0;

	/*
	 * type name string followed by space.
	 */
	while ((c = data[used]) != ' ') {
		typename[used++] = c;
		if (used >= sizeof(typename))
			return 0;
	}
	typename[used] = 0;
	if (used == 0)
		return 0;
136
	hdr->type = git_object_string2type(typename);
Vicent Marti committed
137
	used++; /* consume the space */
138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176

	/*
	 * length follows immediately in decimal (without
	 * leading zeros).
	 */
	size = data[used++] - '0';
	if (size > 9)
		return 0;
	if (size) {
		while ((c = data[used]) != '\0') {
			size_t d = c - '0';
			if (d > 9)
				break;
			used++;
			size = size * 10 + d;
		}
	}
	hdr->size = size;

	/*
	 * the length must be followed by a zero byte
	 */
	if (data[used++] != '\0')
		return 0;

	return used;
}



/***********************************************************
 *
 * ZLIB RELATED FUNCTIONS
 *
 ***********************************************************/

static void init_stream(z_stream *s, void *out, size_t len)
{
	memset(s, 0, sizeof(*s));
Vicent Marti committed
177
	s->next_out = out;
178
	s->avail_out = (uInt)len;
179 180 181 182
}

static void set_stream_input(z_stream *s, void *in, size_t len)
{
Vicent Marti committed
183
	s->next_in = in;
184
	s->avail_in = (uInt)len;
185 186 187 188
}

static void set_stream_output(z_stream *s, void *out, size_t len)
{
Vicent Marti committed
189
	s->next_out = out;
190
	s->avail_out = (uInt)len;
191 192 193
}


194
static int start_inflate(z_stream *s, git_buf *obj, void *out, size_t len)
195 196 197 198
{
	int status;

	init_stream(s, out, len);
nulltoken committed
199
	set_stream_input(s, obj->ptr, git_buf_len(obj));
200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215

	if ((status = inflateInit(s)) < Z_OK)
		return status;

	return inflate(s, 0);
}

static int finish_inflate(z_stream *s)
{
	int status = Z_OK;

	while (status == Z_OK)
		status = inflate(s, Z_FINISH);

	inflateEnd(s);

216 217 218 219
	if ((status != Z_STREAM_END) || (s->avail_in != 0)) {
		giterr_set(GITERR_ZLIB, "Failed to finish ZLib inflation. Stream aborted prematurely");
		return -1;
	}
220

221
	return 0;
222 223
}

Vicent Marti committed
224
static int is_zlib_compressed_data(unsigned char *data)
225
{
Vicent Marti committed
226
	unsigned int w;
227

Vicent Marti committed
228
	w = ((unsigned int)(data[0]) << 8) + data[1];
229
	return (data[0] & 0x8F) == 0x08 && !(w % 31);
230 231
}

Vicent Marti committed
232
static int inflate_buffer(void *in, size_t inlen, void *out, size_t outlen)
233 234
{
	z_stream zs;
Vicent Marti committed
235
	int status = Z_OK;
236

Vicent Marti committed
237
	memset(&zs, 0x0, sizeof(zs));
238

Vicent Marti committed
239
	zs.next_out = out;
240
	zs.avail_out = (uInt)outlen;
241

Vicent Marti committed
242
	zs.next_in = in;
243
	zs.avail_in = (uInt)inlen;
244

245 246 247 248
	if (inflateInit(&zs) < Z_OK) {
		giterr_set(GITERR_ZLIB, "Failed to inflate buffer");
		return -1;
	}
249

Vicent Marti committed
250 251
	while (status == Z_OK)
		status = inflate(&zs, Z_FINISH);
252

Vicent Marti committed
253
	inflateEnd(&zs);
254

255 256 257 258 259 260
	if (status != Z_STREAM_END /* || zs.avail_in != 0 */ ||
		zs.total_out != outlen)
	{
		giterr_set(GITERR_ZLIB, "Failed to inflate buffer. Stream aborted prematurely");
		return -1;
	}
261

262
	return 0;
263 264 265 266 267
}

static void *inflate_tail(z_stream *s, void *hb, size_t used, obj_hdr *hdr)
{
	unsigned char *buf, *head = hb;
268
	size_t tail, alloc_size;
269 270 271 272 273 274

	/*
	 * allocate a buffer to hold the inflated data and copy the
	 * initial sequence of inflated data from the tail of the
	 * head buffer, if any.
	 */
275 276
	if (GIT_ADD_SIZET_OVERFLOW(&alloc_size, hdr->size, 1) ||
		(buf = git__malloc(alloc_size)) == NULL) {
277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295
		inflateEnd(s);
		return NULL;
	}
	tail = s->total_out - used;
	if (used > 0 && tail > 0) {
		if (tail > hdr->size)
			tail = hdr->size;
		memcpy(buf, head + used, tail);
	}
	used = tail;

	/*
	 * inflate the remainder of the object data, if any
	 */
	if (hdr->size < used)
		inflateEnd(s);
	else {
		set_stream_output(s, buf + used, hdr->size - used);
		if (finish_inflate(s)) {
296
			git__free(buf);
297 298 299 300 301 302 303 304 305 306 307 308 309
			return NULL;
		}
	}

	return buf;
}

/*
 * At one point, there was a loose object format that was intended to
 * mimic the format used in pack-files. This was to allow easy copying
 * of loose object data into packs. This format is no longer used, but
 * we must still read it.
 */
310
static int inflate_packlike_loose_disk_obj(git_rawobj *out, git_buf *obj)
311 312 313
{
	unsigned char *in, *buf;
	obj_hdr hdr;
314
	size_t len, used, alloclen;
315 316 317 318 319

	/*
	 * read the object header, which is an (uncompressed)
	 * binary encoding of the object type and size.
	 */
320 321 322 323 324
	if ((used = get_binary_object_header(&hdr, obj)) == 0 ||
		!git_object_typeisloose(hdr.type)) {
		giterr_set(GITERR_ODB, "Failed to inflate loose object.");
		return -1;
	}
325 326 327 328

	/*
	 * allocate a buffer and inflate the data into it
	 */
329 330
	GITERR_CHECK_ALLOC_ADD(&alloclen, hdr.size, 1);
	buf = git__malloc(alloclen);
331
	GITERR_CHECK_ALLOC(buf);
332

333 334
	in = ((unsigned char *)obj->ptr) + used;
	len = obj->size - used;
335
	if (inflate_buffer(in, len, buf, hdr.size) < 0) {
336
		git__free(buf);
337
		return -1;
338 339 340 341
	}
	buf[hdr.size] = '\0';

	out->data = buf;
Vicent Marti committed
342
	out->len = hdr.size;
343 344
	out->type = hdr.type;

345
	return 0;
346 347
}

348
static int inflate_disk_obj(git_rawobj *out, git_buf *obj)
349 350 351 352 353 354 355 356 357
{
	unsigned char head[64], *buf;
	z_stream zs;
	obj_hdr hdr;
	size_t used;

	/*
	 * check for a pack-like loose object
	 */
358
	if (!is_zlib_compressed_data((unsigned char *)obj->ptr))
359 360 361 362 363 364
		return inflate_packlike_loose_disk_obj(out, obj);

	/*
	 * inflate the initial part of the io buffer in order
	 * to parse the object header (type and size).
	 */
365 366 367 368 369 370 371
	if (start_inflate(&zs, obj, head, sizeof(head)) < Z_OK ||
		(used = get_object_header(&hdr, head)) == 0 ||
		!git_object_typeisloose(hdr.type))
	{
		giterr_set(GITERR_ODB, "Failed to inflate disk object.");
		return -1;
	}
372 373 374 375 376 377

	/*
	 * allocate a buffer and inflate the object data into it
	 * (including the initial sequence in the head buffer).
	 */
	if ((buf = inflate_tail(&zs, head, used, &hdr)) == NULL)
378
		return -1;
379 380 381
	buf[hdr.size] = '\0';

	out->data = buf;
Vicent Marti committed
382
	out->len = hdr.size;
383 384
	out->type = hdr.type;

385
	return 0;
386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401
}






/***********************************************************
 *
 * ODB OBJECT READING & WRITING
 *
 * Backend for the public API; read headers and full objects
 * from the ODB. Write raw data to the ODB.
 *
 ***********************************************************/

402
static int read_loose(git_rawobj *out, git_buf *loc)
403 404
{
	int error;
405
	git_buf obj = GIT_BUF_INIT;
406 407 408

	assert(out && loc);

409
	if (git_buf_oom(loc))
410
		return -1;
411

412
	out->data = NULL;
Vicent Marti committed
413
	out->len = 0;
414 415
	out->type = GIT_OBJ_BAD;

416 417
	if (!(error = git_futils_readbuffer(&obj, loc->ptr)))
		error = inflate_disk_obj(out, &obj);
418

419
	git_buf_free(&obj);
420

421
	return error;
422 423
}

424
static int read_header_loose(git_rawobj *out, git_buf *loc)
425
{
426
	int error = 0, z_return = Z_ERRNO, read_bytes;
427 428 429 430 431 432 433
	git_file fd;
	z_stream zs;
	obj_hdr header_obj;
	unsigned char raw_buffer[16], inflated_buffer[64];

	assert(out && loc);

434
	if (git_buf_oom(loc))
435
		return -1;
436

437 438
	out->data = NULL;

439 440
	if ((fd = git_futils_open_ro(loc->ptr)) < 0)
		return fd;
441 442 443

	init_stream(&zs, inflated_buffer, sizeof(inflated_buffer));

444
	z_return = inflateInit(&zs);
445

446 447
	while (z_return == Z_OK) {
		if ((read_bytes = p_read(fd, raw_buffer, sizeof(raw_buffer))) > 0) {
448 449
			set_stream_input(&zs, raw_buffer, read_bytes);
			z_return = inflate(&zs, 0);
450
		} else
451
			z_return = Z_STREAM_END;
452
	}
453 454 455

	if ((z_return != Z_STREAM_END && z_return != Z_BUF_ERROR)
		|| get_object_header(&header_obj, inflated_buffer) == 0
456 457 458 459 460 461 462
		|| git_object_typeisloose(header_obj.type) == 0)
	{
		giterr_set(GITERR_ZLIB, "Failed to read loose object header");
		error = -1;
	} else {
		out->len = header_obj.size;
		out->type = header_obj.type;
463 464 465
	}

	finish_inflate(&zs);
Vicent Marti committed
466
	p_close(fd);
Vicent Marti committed
467

468
	return error;
469 470
}

471 472 473 474
static int locate_object(
	git_buf *object_location,
	loose_backend *backend,
	const git_oid *oid)
475
{
476
	int error = object_file_name(object_location, backend, oid);
477

478 479
	if (!error && !git_path_exists(object_location->ptr))
		return GIT_ENOTFOUND;
480 481

	return error;
482 483
}

484
/* Explore an entry of a directory and see if it matches a short oid */
485
static int fn_locate_object_short_oid(void *state, git_buf *pathbuf) {
486 487
	loose_locate_object_state *sstate = (loose_locate_object_state *)state;

nulltoken committed
488
	if (git_buf_len(pathbuf) - sstate->dir_len != GIT_OID_HEXSZ - 2) {
489
		/* Entry cannot be an object. Continue to next entry */
490
		return 0;
491 492
	}

493
	if (git_path_isdir(pathbuf->ptr) == false) {
494 495 496
		/* We are already in the directory matching the 2 first hex characters,
		 * compare the first ncmp characters of the oids */
		if (!memcmp(sstate->short_oid + 2,
497
			(unsigned char *)pathbuf->ptr + sstate->dir_len,
498 499
			sstate->short_oid_len - 2)) {

500 501 502
			if (!sstate->found) {
				sstate->res_oid[0] = sstate->short_oid[0];
				sstate->res_oid[1] = sstate->short_oid[1];
503
				memcpy(sstate->res_oid+2, pathbuf->ptr+sstate->dir_len, GIT_OID_HEXSZ-2);
504 505 506 507
			}
			sstate->found++;
		}
	}
508

Vicent Marti committed
509
	if (sstate->found > 1)
510
		return GIT_EAMBIGUOUS;
Vicent Marti committed
511

512
	return 0;
513 514 515
}

/* Locate an object matching a given short oid */
516 517 518 519 520
static int locate_object_short_oid(
	git_buf *object_location,
	git_oid *res_oid,
	loose_backend *backend,
	const git_oid *short_oid,
521
	size_t len)
522 523
{
	char *objects_dir = backend->objects_dir;
524
	size_t dir_len = strlen(objects_dir), alloc_len;
525 526 527
	loose_locate_object_state state;
	int error;

528
	/* prealloc memory for OBJ_DIR/xx/xx..38x..xx */
529 530 531
	GITERR_CHECK_ALLOC_ADD(&alloc_len, dir_len, GIT_OID_HEXSZ);
	GITERR_CHECK_ALLOC_ADD(&alloc_len, alloc_len, 3);
	if (git_buf_grow(object_location, alloc_len) < 0)
532
		return -1;
533

534
	git_buf_set(object_location, objects_dir, dir_len);
535
	git_path_to_dir(object_location);
536

537
	/* save adjusted position at end of dir so it can be restored later */
nulltoken committed
538
	dir_len = git_buf_len(object_location);
539 540

	/* Convert raw oid to hex formatted oid */
Vicent Marti committed
541
	git_oid_fmt((char *)state.short_oid, short_oid);
542

543
	/* Explore OBJ_DIR/xx/ where xx is the beginning of hex formatted short oid */
544
	if (git_buf_put(object_location, (char *)state.short_oid, 3) < 0)
545
		return -1;
546
	object_location->ptr[object_location->size - 1] = '/';
547 548

	/* Check that directory exists */
549
	if (git_path_isdir(object_location->ptr) == false)
550 551
		return git_odb__error_notfound("no matching loose object for prefix",
			short_oid, len);
552

nulltoken committed
553
	state.dir_len = git_buf_len(object_location);
554 555
	state.short_oid_len = len;
	state.found = 0;
556

557
	/* Explore directory to find a unique object matching short_oid */
558
	error = git_path_direach(
559
		object_location, 0, fn_locate_object_short_oid, &state);
560
	if (error < 0 && error != GIT_EAMBIGUOUS)
561
		return error;
562

563
	if (!state.found)
564 565
		return git_odb__error_notfound("no matching loose object for prefix",
			short_oid, len);
566

567 568 569
	if (state.found > 1)
		return git_odb__error_ambiguous("multiple matches in loose objects");

570
	/* Convert obtained hex formatted oid to raw */
Vicent Marti committed
571
	error = git_oid_fromstr(res_oid, (char *)state.res_oid);
572 573
	if (error)
		return error;
574 575

	/* Update the location according to the oid obtained */
576 577
	GITERR_CHECK_ALLOC_ADD(&alloc_len, dir_len, GIT_OID_HEXSZ);
	GITERR_CHECK_ALLOC_ADD(&alloc_len, alloc_len, 2);
578 579

	git_buf_truncate(object_location, dir_len);
580
	if (git_buf_grow(object_location, alloc_len) < 0)
581
		return -1;
582 583 584 585 586

	git_oid_pathfmt(object_location->ptr + dir_len, res_oid);

	object_location->size += GIT_OID_HEXSZ + 1;
	object_location->ptr[object_location->size] = '\0';
587

588
	return 0;
589 590
}

591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606








/***********************************************************
 *
 * LOOSE BACKEND PUBLIC API
 *
 * Implement the git_odb_backend API calls
 *
 ***********************************************************/

607
static int loose_backend__read_header(size_t *len_p, git_otype *type_p, git_odb_backend *backend, const git_oid *oid)
608
{
609
	git_buf object_path = GIT_BUF_INIT;
Vicent Marti committed
610
	git_rawobj raw;
611
	int error;
612

Vicent Marti committed
613
	assert(backend && oid);
614

Vicent Marti committed
615 616 617
	raw.len = 0;
	raw.type = GIT_OBJ_BAD;

618 619 620 621
	if (locate_object(&object_path, (loose_backend *)backend, oid) < 0) {
		error = git_odb__error_notfound("no matching loose object",
			oid, GIT_OID_HEXSZ);
	} else if ((error = read_header_loose(&raw, &object_path)) == 0) {
622 623 624
		*len_p = raw.len;
		*type_p = raw.type;
	}
625

626
	git_buf_free(&object_path);
627

628
	return error;
Vicent Marti committed
629
}
630

631
static int loose_backend__read(void **buffer_p, size_t *len_p, git_otype *type_p, git_odb_backend *backend, const git_oid *oid)
632
{
633
	git_buf object_path = GIT_BUF_INIT;
Vicent Marti committed
634
	git_rawobj raw;
635
	int error = 0;
636

Vicent Marti committed
637
	assert(backend && oid);
638

639 640 641 642
	if (locate_object(&object_path, (loose_backend *)backend, oid) < 0) {
		error = git_odb__error_notfound("no matching loose object",
			oid, GIT_OID_HEXSZ);
	} else if ((error = read_loose(&raw, &object_path)) == 0) {
643 644 645 646
		*buffer_p = raw.data;
		*len_p = raw.len;
		*type_p = raw.type;
	}
Vicent Marti committed
647

648
	git_buf_free(&object_path);
Vicent Marti committed
649

650
	return error;
651 652
}

653
static int loose_backend__read_prefix(
Vicent Marti committed
654 655 656 657 658 659
	git_oid *out_oid,
	void **buffer_p,
	size_t *len_p,
	git_otype *type_p,
	git_odb_backend *backend,
	const git_oid *short_oid,
660
	size_t len)
661
{
662
	int error = 0;
663

664
	assert(len >= GIT_OID_MINPREFIXLEN && len <= GIT_OID_HEXSZ);
665

666
	if (len == GIT_OID_HEXSZ) {
667
		/* We can fall back to regular read method */
668
		error = loose_backend__read(buffer_p, len_p, type_p, backend, short_oid);
669
		if (!error)
670
			git_oid_cpy(out_oid, short_oid);
671
	} else {
672
		git_buf object_path = GIT_BUF_INIT;
673 674 675 676
		git_rawobj raw;

		assert(backend && short_oid);

677
		if ((error = locate_object_short_oid(&object_path, out_oid,
678 679 680
				(loose_backend *)backend, short_oid, len)) == 0 &&
			(error = read_loose(&raw, &object_path)) == 0)
		{
681 682 683
			*buffer_p = raw.data;
			*len_p = raw.len;
			*type_p = raw.type;
684 685
		}

686
		git_buf_free(&object_path);
687
	}
688

689
	return error;
690 691
}

692
static int loose_backend__exists(git_odb_backend *backend, const git_oid *oid)
693
{
694 695
	git_buf object_path = GIT_BUF_INIT;
	int error;
696 697 698

	assert(backend && oid);

699 700 701 702
	error = locate_object(&object_path, (loose_backend *)backend, oid);

	git_buf_free(&object_path);

703
	return !error;
704 705
}

706 707 708 709 710 711
static int loose_backend__exists_prefix(
	git_oid *out, git_odb_backend *backend, const git_oid *short_id, size_t len)
{
	git_buf object_path = GIT_BUF_INIT;
	int error;

712
	assert(backend && out && short_id && len >= GIT_OID_MINPREFIXLEN);
713 714 715 716 717 718 719 720 721

	error = locate_object_short_oid(
		&object_path, out, (loose_backend *)backend, short_id, len);

	git_buf_free(&object_path);

	return error;
}

722 723
struct foreach_state {
	size_t dir_len;
724
	git_odb_foreach_cb cb;
725 726 727
	void *data;
};

728
GIT_INLINE(int) filename_to_oid(git_oid *oid, const char *ptr)
729 730
{
	int v, i = 0;
731
	if (strlen(ptr) != GIT_OID_HEXSZ+1)
732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763
		return -1;

	if (ptr[2] != '/') {
		return -1;
	}

	v = (git__fromhex(ptr[i]) << 4) | git__fromhex(ptr[i+1]);
	if (v < 0)
		return -1;

	oid->id[0] = (unsigned char) v;

	ptr += 3;
	for (i = 0; i < 38; i += 2) {
		v = (git__fromhex(ptr[i]) << 4) | git__fromhex(ptr[i + 1]);
		if (v < 0)
			return -1;

		oid->id[1 + i/2] = (unsigned char) v;
	}

	return 0;
}

static int foreach_object_dir_cb(void *_state, git_buf *path)
{
	git_oid oid;
	struct foreach_state *state = (struct foreach_state *) _state;

	if (filename_to_oid(&oid, path->ptr + state->dir_len) < 0)
		return 0;

764
	return giterr_set_after_callback_function(
765
		state->cb(&oid, state->data), "git_odb_foreach");
766 767 768 769 770 771
}

static int foreach_cb(void *_state, git_buf *path)
{
	struct foreach_state *state = (struct foreach_state *) _state;

772 773 774 775
	/* non-dir is some stray file, ignore it */
	if (!git_path_isdir(git_buf_cstr(path)))
		return 0;

776
	return git_path_direach(path, 0, foreach_object_dir_cb, state);
777 778
}

779
static int loose_backend__foreach(git_odb_backend *_backend, git_odb_foreach_cb cb, void *data)
780 781 782 783 784 785 786 787 788 789 790
{
	char *objects_dir;
	int error;
	git_buf buf = GIT_BUF_INIT;
	struct foreach_state state;
	loose_backend *backend = (loose_backend *) _backend;

	assert(backend && cb);

	objects_dir = backend->objects_dir;

791
	git_buf_sets(&buf, objects_dir);
792
	git_path_to_dir(&buf);
793 794
	if (git_buf_oom(&buf))
		return -1;
795

796
	memset(&state, 0, sizeof(state));
797 798 799 800
	state.cb = cb;
	state.data = data;
	state.dir_len = git_buf_len(&buf);

801
	error = git_path_direach(&buf, 0, foreach_cb, &state);
802

803 804
	git_buf_free(&buf);

805
	return error;
806 807
}

808
static int loose_backend__stream_fwrite(git_odb_stream *_stream, const git_oid *oid)
Vicent Marti committed
809 810 811
{
	loose_writestream *stream = (loose_writestream *)_stream;
	loose_backend *backend = (loose_backend *)_stream->backend;
812
	git_buf final_path = GIT_BUF_INIT;
813
	int error = 0;
Vicent Marti committed
814

815
	if (object_file_name(&final_path, backend, oid) < 0 ||
816
		object_mkdir(&final_path, backend) < 0)
817 818 819
		error = -1;
	else
		error = git_filebuf_commit_at(
820
			&stream->fbuf, final_path.ptr);
821 822 823 824

	git_buf_free(&final_path);

	return error;
Vicent Marti committed
825 826
}

827
static int loose_backend__stream_write(git_odb_stream *_stream, const char *data, size_t len)
828
{
Vicent Marti committed
829 830 831 832
	loose_writestream *stream = (loose_writestream *)_stream;
	return git_filebuf_write(&stream->fbuf, data, len);
}

833
static void loose_backend__stream_free(git_odb_stream *_stream)
Vicent Marti committed
834 835 836
{
	loose_writestream *stream = (loose_writestream *)_stream;

837
	git_filebuf_cleanup(&stream->fbuf);
838
	git__free(stream);
Vicent Marti committed
839 840
}

841
static int loose_backend__stream(git_odb_stream **stream_out, git_odb_backend *_backend, git_off_t length, git_otype type)
Vicent Marti committed
842 843
{
	loose_backend *backend;
844
	loose_writestream *stream = NULL;
845 846
	char hdr[64];
	git_buf tmp_path = GIT_BUF_INIT;
Vicent Marti committed
847
	int hdrlen;
848

849
	assert(_backend && length >= 0);
850 851

	backend = (loose_backend *)_backend;
Vicent Marti committed
852
	*stream_out = NULL;
853

854
	hdrlen = git_odb__format_object_header(hdr, sizeof(hdr), length, type);
Vicent Marti committed
855 856

	stream = git__calloc(1, sizeof(loose_writestream));
857
	GITERR_CHECK_ALLOC(stream);
858

Vicent Marti committed
859 860 861 862 863 864
	stream->stream.backend = _backend;
	stream->stream.read = NULL; /* read only */
	stream->stream.write = &loose_backend__stream_write;
	stream->stream.finalize_write = &loose_backend__stream_fwrite;
	stream->stream.free = &loose_backend__stream_free;
	stream->stream.mode = GIT_STREAM_WRONLY;
865

866 867 868
	if (git_buf_joinpath(&tmp_path, backend->objects_dir, "tmp_object") < 0 ||
		git_filebuf_open(&stream->fbuf, tmp_path.ptr,
			GIT_FILEBUF_TEMPORARY |
869 870
			(backend->object_zlib_level << GIT_FILEBUF_DEFLATE_SHIFT),
			backend->object_file_mode) < 0 ||
871 872 873 874 875 876
		stream->stream.write((git_odb_stream *)stream, hdr, hdrlen) < 0)
	{
		git_filebuf_cleanup(&stream->fbuf);
		git__free(stream);
		stream = NULL;
	}
877
	git_buf_free(&tmp_path);
Vicent Marti committed
878
	*stream_out = (git_odb_stream *)stream;
879

880
	return !stream ? -1 : 0;
881 882
}

883
static int loose_backend__write(git_odb_backend *_backend, const git_oid *oid, const void *data, size_t len, git_otype type)
884
{
885
	int error = 0, header_len;
886 887
	git_buf final_path = GIT_BUF_INIT;
	char header[64];
888
	git_filebuf fbuf = GIT_FILEBUF_INIT;
889 890 891 892 893
	loose_backend *backend;

	backend = (loose_backend *)_backend;

	/* prepare the header for the file */
894
	header_len = git_odb__format_object_header(header, sizeof(header), len, type);
895

896 897 898
	if (git_buf_joinpath(&final_path, backend->objects_dir, "tmp_object") < 0 ||
		git_filebuf_open(&fbuf, final_path.ptr,
			GIT_FILEBUF_TEMPORARY |
899 900
			(backend->object_zlib_level << GIT_FILEBUF_DEFLATE_SHIFT),
			backend->object_file_mode) < 0)
901 902
	{
		error = -1;
903
		goto cleanup;
904
	}
905 906 907 908

	git_filebuf_write(&fbuf, header, header_len);
	git_filebuf_write(&fbuf, data, len);

909 910
	if (object_file_name(&final_path, backend, oid) < 0 ||
		object_mkdir(&final_path, backend) < 0 ||
911
		git_filebuf_commit_at(&fbuf, final_path.ptr) < 0)
912
		error = -1;
913 914

cleanup:
915
	if (error < 0)
916 917
		git_filebuf_cleanup(&fbuf);
	git_buf_free(&final_path);
918 919 920
	return error;
}

921
static void loose_backend__free(git_odb_backend *_backend)
922 923 924 925 926
{
	loose_backend *backend;
	assert(_backend);
	backend = (loose_backend *)_backend;

927
	git__free(backend);
928 929
}

930 931 932 933
int git_odb_backend_loose(
	git_odb_backend **backend_out,
	const char *objects_dir,
	int compression_level,
934
	int do_fsync,
935 936
	unsigned int dir_mode,
	unsigned int file_mode)
937 938
{
	loose_backend *backend;
939
	size_t objects_dirlen, alloclen;
940 941 942 943

	assert(backend_out && objects_dir);

	objects_dirlen = strlen(objects_dir);
944

945 946 947
	GITERR_CHECK_ALLOC_ADD(&alloclen, sizeof(loose_backend), objects_dirlen);
	GITERR_CHECK_ALLOC_ADD(&alloclen, alloclen, 2);
	backend = git__calloc(1, alloclen);
948
	GITERR_CHECK_ALLOC(backend);
949

950
	backend->parent.version = GIT_ODB_BACKEND_VERSION;
951 952 953 954
	backend->objects_dirlen = objects_dirlen;
	memcpy(backend->objects_dir, objects_dir, objects_dirlen);
	if (backend->objects_dir[backend->objects_dirlen - 1] != '/')
		backend->objects_dir[backend->objects_dirlen++] = '/';
955

956 957 958
	if (compression_level < 0)
		compression_level = Z_BEST_SPEED;

959 960 961 962 963 964
	if (dir_mode == 0)
		dir_mode = GIT_OBJECT_DIR_MODE;

	if (file_mode == 0)
		file_mode = GIT_OBJECT_FILE_MODE;

965 966
	backend->object_zlib_level = compression_level;
	backend->fsync_object_files = do_fsync;
967 968
	backend->object_dir_mode = dir_mode;
	backend->object_file_mode = file_mode;
969 970

	backend->parent.read = &loose_backend__read;
971
	backend->parent.write = &loose_backend__write;
Vicent Marti committed
972
	backend->parent.read_prefix = &loose_backend__read_prefix;
973
	backend->parent.read_header = &loose_backend__read_header;
Vicent Marti committed
974
	backend->parent.writestream = &loose_backend__stream;
975
	backend->parent.exists = &loose_backend__exists;
976
	backend->parent.exists_prefix = &loose_backend__exists_prefix;
977
	backend->parent.foreach = &loose_backend__foreach;
978 979 980
	backend->parent.free = &loose_backend__free;

	*backend_out = (git_odb_backend *)backend;
981
	return 0;
982
}