odb_loose.c 22.9 KB
Newer Older
1
/*
Edward Thomson committed
2
 * Copyright (C) the libgit2 contributors. All rights reserved.
3
 *
Vicent Marti committed
4 5
 * This file is part of libgit2, distributed under the GNU GPL v2 with
 * a Linking Exception. For full terms see the included COPYING file.
6 7 8
 */

#include "common.h"
9
#include <zlib.h>
10
#include "git2/object.h"
11
#include "git2/sys/odb_backend.h"
12 13 14 15
#include "fileops.h"
#include "hash.h"
#include "odb.h"
#include "delta-apply.h"
Vicent Marti committed
16
#include "filebuf.h"
17

18
#include "git2/odb_backend.h"
Vicent Marti committed
19
#include "git2/types.h"
20

Vicent Marti committed
21 22 23
typedef struct { /* object header data */
	git_otype type; /* object type */
	size_t	size; /* object size */
24 25
} obj_hdr;

Vicent Marti committed
26 27 28 29 30
typedef struct {
	git_odb_stream stream;
	git_filebuf fbuf;
} loose_writestream;

31 32 33 34 35
typedef struct loose_backend {
	git_odb_backend parent;

	int object_zlib_level; /** loose object zlib compression level. */
	int fsync_object_files; /** loose object file fsync flag. */
36 37
	mode_t object_file_mode;
	mode_t object_dir_mode;
38 39 40

	size_t objects_dirlen;
	char objects_dir[GIT_FLEX_ARRAY];
41 42
} loose_backend;

43 44 45 46 47 48
/* State structure for exploring directories,
 * in order to locate objects matching a short oid.
 */
typedef struct {
	size_t dir_len;
	unsigned char short_oid[GIT_OID_HEXSZ]; /* hex formatted oid to match */
49
	size_t short_oid_len;
50 51 52 53 54 55 56
	int found;				/* number of matching
						 * objects already found */
	unsigned char res_oid[GIT_OID_HEXSZ];	/* hex formatted oid of
						 * the object found */
} loose_locate_object_state;


57 58
/***********************************************************
 *
Will Stamper committed
59
 * MISCELLANEOUS HELPER FUNCTIONS
60 61 62
 *
 ***********************************************************/

63 64
static int object_file_name(
	git_buf *name, const loose_backend *be, const git_oid *id)
65
{
66 67
	size_t alloclen;

68
	/* expand length for object root + 40 hex sha1 chars + 2 * '/' + '\0' */
69 70 71
	GITERR_CHECK_ALLOC_ADD(&alloclen, be->objects_dirlen, GIT_OID_HEXSZ);
	GITERR_CHECK_ALLOC_ADD(&alloclen, alloclen, 3);
	if (git_buf_grow(name, alloclen) < 0)
72
		return -1;
73

74
	git_buf_set(name, be->objects_dir, be->objects_dirlen);
75
	git_path_to_dir(name);
76 77

	/* loose object filename: aa/aaa... (41 bytes) */
78
	git_oid_pathfmt(name->ptr + name->size, id);
79 80
	name->size += GIT_OID_HEXSZ + 1;
	name->ptr[name->size] = '\0';
81

82
	return 0;
83 84
}

85 86
static int object_mkdir(const git_buf *name, const loose_backend *be)
{
87
	return git_futils_mkdir_relative(
88
		name->ptr + be->objects_dirlen, be->objects_dir, be->object_dir_mode,
89
		GIT_MKDIR_PATH | GIT_MKDIR_SKIP_LAST | GIT_MKDIR_VERIFY_DIR, NULL);
90
}
91

92
static size_t get_binary_object_header(obj_hdr *hdr, git_buf *obj)
93 94
{
	unsigned char c;
95
	unsigned char *data = (unsigned char *)obj->ptr;
96 97
	size_t shift, size, used = 0;

nulltoken committed
98
	if (git_buf_len(obj) == 0)
99 100 101 102 103 104 105 106
		return 0;

	c = data[used++];
	hdr->type = (c >> 4) & 7;

	size = c & 15;
	shift = 4;
	while (c & 0x80) {
nulltoken committed
107
		if (git_buf_len(obj) <= used)
108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135
			return 0;
		if (sizeof(size_t) * 8 <= shift)
			return 0;
		c = data[used++];
		size += (c & 0x7f) << shift;
		shift += 7;
	}
	hdr->size = size;

	return used;
}

static size_t get_object_header(obj_hdr *hdr, unsigned char *data)
{
	char c, typename[10];
	size_t size, used = 0;

	/*
	 * type name string followed by space.
	 */
	while ((c = data[used]) != ' ') {
		typename[used++] = c;
		if (used >= sizeof(typename))
			return 0;
	}
	typename[used] = 0;
	if (used == 0)
		return 0;
136
	hdr->type = git_object_string2type(typename);
Vicent Marti committed
137
	used++; /* consume the space */
138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176

	/*
	 * length follows immediately in decimal (without
	 * leading zeros).
	 */
	size = data[used++] - '0';
	if (size > 9)
		return 0;
	if (size) {
		while ((c = data[used]) != '\0') {
			size_t d = c - '0';
			if (d > 9)
				break;
			used++;
			size = size * 10 + d;
		}
	}
	hdr->size = size;

	/*
	 * the length must be followed by a zero byte
	 */
	if (data[used++] != '\0')
		return 0;

	return used;
}



/***********************************************************
 *
 * ZLIB RELATED FUNCTIONS
 *
 ***********************************************************/

static void init_stream(z_stream *s, void *out, size_t len)
{
	memset(s, 0, sizeof(*s));
Vicent Marti committed
177
	s->next_out = out;
178
	s->avail_out = (uInt)len;
179 180 181 182
}

static void set_stream_input(z_stream *s, void *in, size_t len)
{
Vicent Marti committed
183
	s->next_in = in;
184
	s->avail_in = (uInt)len;
185 186 187 188
}

static void set_stream_output(z_stream *s, void *out, size_t len)
{
Vicent Marti committed
189
	s->next_out = out;
190
	s->avail_out = (uInt)len;
191 192 193
}


194
static int start_inflate(z_stream *s, git_buf *obj, void *out, size_t len)
195 196 197 198
{
	int status;

	init_stream(s, out, len);
nulltoken committed
199
	set_stream_input(s, obj->ptr, git_buf_len(obj));
200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215

	if ((status = inflateInit(s)) < Z_OK)
		return status;

	return inflate(s, 0);
}

static int finish_inflate(z_stream *s)
{
	int status = Z_OK;

	while (status == Z_OK)
		status = inflate(s, Z_FINISH);

	inflateEnd(s);

216 217 218 219
	if ((status != Z_STREAM_END) || (s->avail_in != 0)) {
		giterr_set(GITERR_ZLIB, "Failed to finish ZLib inflation. Stream aborted prematurely");
		return -1;
	}
220

221
	return 0;
222 223
}

Vicent Marti committed
224
static int is_zlib_compressed_data(unsigned char *data)
225
{
Vicent Marti committed
226
	unsigned int w;
227

Vicent Marti committed
228
	w = ((unsigned int)(data[0]) << 8) + data[1];
229
	return (data[0] & 0x8F) == 0x08 && !(w % 31);
230 231
}

Vicent Marti committed
232
static int inflate_buffer(void *in, size_t inlen, void *out, size_t outlen)
233 234
{
	z_stream zs;
Vicent Marti committed
235
	int status = Z_OK;
236

Vicent Marti committed
237
	memset(&zs, 0x0, sizeof(zs));
238

Vicent Marti committed
239
	zs.next_out = out;
240
	zs.avail_out = (uInt)outlen;
241

Vicent Marti committed
242
	zs.next_in = in;
243
	zs.avail_in = (uInt)inlen;
244

245 246 247 248
	if (inflateInit(&zs) < Z_OK) {
		giterr_set(GITERR_ZLIB, "Failed to inflate buffer");
		return -1;
	}
249

Vicent Marti committed
250 251
	while (status == Z_OK)
		status = inflate(&zs, Z_FINISH);
252

Vicent Marti committed
253
	inflateEnd(&zs);
254

255 256 257 258 259 260
	if (status != Z_STREAM_END /* || zs.avail_in != 0 */ ||
		zs.total_out != outlen)
	{
		giterr_set(GITERR_ZLIB, "Failed to inflate buffer. Stream aborted prematurely");
		return -1;
	}
261

262
	return 0;
263 264 265 266 267
}

static void *inflate_tail(z_stream *s, void *hb, size_t used, obj_hdr *hdr)
{
	unsigned char *buf, *head = hb;
268
	size_t tail, alloc_size;
269 270 271 272 273 274

	/*
	 * allocate a buffer to hold the inflated data and copy the
	 * initial sequence of inflated data from the tail of the
	 * head buffer, if any.
	 */
275 276
	if (GIT_ADD_SIZET_OVERFLOW(&alloc_size, hdr->size, 1) ||
		(buf = git__malloc(alloc_size)) == NULL) {
277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295
		inflateEnd(s);
		return NULL;
	}
	tail = s->total_out - used;
	if (used > 0 && tail > 0) {
		if (tail > hdr->size)
			tail = hdr->size;
		memcpy(buf, head + used, tail);
	}
	used = tail;

	/*
	 * inflate the remainder of the object data, if any
	 */
	if (hdr->size < used)
		inflateEnd(s);
	else {
		set_stream_output(s, buf + used, hdr->size - used);
		if (finish_inflate(s)) {
296
			git__free(buf);
297 298 299 300 301 302 303 304 305 306 307 308 309
			return NULL;
		}
	}

	return buf;
}

/*
 * At one point, there was a loose object format that was intended to
 * mimic the format used in pack-files. This was to allow easy copying
 * of loose object data into packs. This format is no longer used, but
 * we must still read it.
 */
310
static int inflate_packlike_loose_disk_obj(git_rawobj *out, git_buf *obj)
311 312 313
{
	unsigned char *in, *buf;
	obj_hdr hdr;
314
	size_t len, used, alloclen;
315 316 317 318 319

	/*
	 * read the object header, which is an (uncompressed)
	 * binary encoding of the object type and size.
	 */
320 321 322 323 324
	if ((used = get_binary_object_header(&hdr, obj)) == 0 ||
		!git_object_typeisloose(hdr.type)) {
		giterr_set(GITERR_ODB, "Failed to inflate loose object.");
		return -1;
	}
325 326 327 328

	/*
	 * allocate a buffer and inflate the data into it
	 */
329 330
	GITERR_CHECK_ALLOC_ADD(&alloclen, hdr.size, 1);
	buf = git__malloc(alloclen);
331
	GITERR_CHECK_ALLOC(buf);
332

333 334
	in = ((unsigned char *)obj->ptr) + used;
	len = obj->size - used;
335
	if (inflate_buffer(in, len, buf, hdr.size) < 0) {
336
		git__free(buf);
337
		return -1;
338 339 340 341
	}
	buf[hdr.size] = '\0';

	out->data = buf;
Vicent Marti committed
342
	out->len = hdr.size;
343 344
	out->type = hdr.type;

345
	return 0;
346 347
}

348
static int inflate_disk_obj(git_rawobj *out, git_buf *obj)
349 350 351 352 353 354 355 356 357
{
	unsigned char head[64], *buf;
	z_stream zs;
	obj_hdr hdr;
	size_t used;

	/*
	 * check for a pack-like loose object
	 */
358
	if (!is_zlib_compressed_data((unsigned char *)obj->ptr))
359 360 361 362 363 364
		return inflate_packlike_loose_disk_obj(out, obj);

	/*
	 * inflate the initial part of the io buffer in order
	 * to parse the object header (type and size).
	 */
365 366 367 368 369 370 371
	if (start_inflate(&zs, obj, head, sizeof(head)) < Z_OK ||
		(used = get_object_header(&hdr, head)) == 0 ||
		!git_object_typeisloose(hdr.type))
	{
		giterr_set(GITERR_ODB, "Failed to inflate disk object.");
		return -1;
	}
372 373 374 375 376 377

	/*
	 * allocate a buffer and inflate the object data into it
	 * (including the initial sequence in the head buffer).
	 */
	if ((buf = inflate_tail(&zs, head, used, &hdr)) == NULL)
378
		return -1;
379 380 381
	buf[hdr.size] = '\0';

	out->data = buf;
Vicent Marti committed
382
	out->len = hdr.size;
383 384
	out->type = hdr.type;

385
	return 0;
386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401
}






/***********************************************************
 *
 * ODB OBJECT READING & WRITING
 *
 * Backend for the public API; read headers and full objects
 * from the ODB. Write raw data to the ODB.
 *
 ***********************************************************/

402
static int read_loose(git_rawobj *out, git_buf *loc)
403 404
{
	int error;
405
	git_buf obj = GIT_BUF_INIT;
406 407 408

	assert(out && loc);

409
	if (git_buf_oom(loc))
410
		return -1;
411

412
	out->data = NULL;
Vicent Marti committed
413
	out->len = 0;
414 415
	out->type = GIT_OBJ_BAD;

416 417
	if (!(error = git_futils_readbuffer(&obj, loc->ptr)))
		error = inflate_disk_obj(out, &obj);
418

419
	git_buf_free(&obj);
420

421
	return error;
422 423
}

424
static int read_header_loose(git_rawobj *out, git_buf *loc)
425
{
426
	int error = 0, z_return = Z_ERRNO, read_bytes;
427 428 429 430 431 432 433
	git_file fd;
	z_stream zs;
	obj_hdr header_obj;
	unsigned char raw_buffer[16], inflated_buffer[64];

	assert(out && loc);

434
	if (git_buf_oom(loc))
435
		return -1;
436

437 438
	out->data = NULL;

439 440
	if ((fd = git_futils_open_ro(loc->ptr)) < 0)
		return fd;
441 442 443

	init_stream(&zs, inflated_buffer, sizeof(inflated_buffer));

444
	z_return = inflateInit(&zs);
445

446 447
	while (z_return == Z_OK) {
		if ((read_bytes = p_read(fd, raw_buffer, sizeof(raw_buffer))) > 0) {
448 449
			set_stream_input(&zs, raw_buffer, read_bytes);
			z_return = inflate(&zs, 0);
450
		} else
451
			z_return = Z_STREAM_END;
452
	}
453 454 455

	if ((z_return != Z_STREAM_END && z_return != Z_BUF_ERROR)
		|| get_object_header(&header_obj, inflated_buffer) == 0
456 457 458 459 460 461 462
		|| git_object_typeisloose(header_obj.type) == 0)
	{
		giterr_set(GITERR_ZLIB, "Failed to read loose object header");
		error = -1;
	} else {
		out->len = header_obj.size;
		out->type = header_obj.type;
463 464 465
	}

	finish_inflate(&zs);
Vicent Marti committed
466
	p_close(fd);
Vicent Marti committed
467

468
	return error;
469 470
}

471 472 473 474
static int locate_object(
	git_buf *object_location,
	loose_backend *backend,
	const git_oid *oid)
475
{
476
	int error = object_file_name(object_location, backend, oid);
477

478 479
	if (!error && !git_path_exists(object_location->ptr))
		return GIT_ENOTFOUND;
480 481

	return error;
482 483
}

484
/* Explore an entry of a directory and see if it matches a short oid */
485
static int fn_locate_object_short_oid(void *state, git_buf *pathbuf) {
486 487
	loose_locate_object_state *sstate = (loose_locate_object_state *)state;

nulltoken committed
488
	if (git_buf_len(pathbuf) - sstate->dir_len != GIT_OID_HEXSZ - 2) {
489
		/* Entry cannot be an object. Continue to next entry */
490
		return 0;
491 492
	}

493
	if (git_path_isdir(pathbuf->ptr) == false) {
494 495 496
		/* We are already in the directory matching the 2 first hex characters,
		 * compare the first ncmp characters of the oids */
		if (!memcmp(sstate->short_oid + 2,
497
			(unsigned char *)pathbuf->ptr + sstate->dir_len,
498 499
			sstate->short_oid_len - 2)) {

500 501 502
			if (!sstate->found) {
				sstate->res_oid[0] = sstate->short_oid[0];
				sstate->res_oid[1] = sstate->short_oid[1];
503
				memcpy(sstate->res_oid+2, pathbuf->ptr+sstate->dir_len, GIT_OID_HEXSZ-2);
504 505 506 507
			}
			sstate->found++;
		}
	}
508

Vicent Marti committed
509
	if (sstate->found > 1)
510
		return GIT_EAMBIGUOUS;
Vicent Marti committed
511

512
	return 0;
513 514 515
}

/* Locate an object matching a given short oid */
516 517 518 519 520
static int locate_object_short_oid(
	git_buf *object_location,
	git_oid *res_oid,
	loose_backend *backend,
	const git_oid *short_oid,
521
	size_t len)
522 523
{
	char *objects_dir = backend->objects_dir;
524
	size_t dir_len = strlen(objects_dir), alloc_len;
525 526 527
	loose_locate_object_state state;
	int error;

528
	/* prealloc memory for OBJ_DIR/xx/xx..38x..xx */
529 530 531
	GITERR_CHECK_ALLOC_ADD(&alloc_len, dir_len, GIT_OID_HEXSZ);
	GITERR_CHECK_ALLOC_ADD(&alloc_len, alloc_len, 3);
	if (git_buf_grow(object_location, alloc_len) < 0)
532
		return -1;
533

534
	git_buf_set(object_location, objects_dir, dir_len);
535
	git_path_to_dir(object_location);
536

537
	/* save adjusted position at end of dir so it can be restored later */
nulltoken committed
538
	dir_len = git_buf_len(object_location);
539 540

	/* Convert raw oid to hex formatted oid */
Vicent Marti committed
541
	git_oid_fmt((char *)state.short_oid, short_oid);
542

543
	/* Explore OBJ_DIR/xx/ where xx is the beginning of hex formatted short oid */
544
	if (git_buf_put(object_location, (char *)state.short_oid, 3) < 0)
545
		return -1;
546
	object_location->ptr[object_location->size - 1] = '/';
547 548

	/* Check that directory exists */
549
	if (git_path_isdir(object_location->ptr) == false)
Russell Belfer committed
550
		return git_odb__error_notfound("no matching loose object for prefix", short_oid);
551

nulltoken committed
552
	state.dir_len = git_buf_len(object_location);
553 554
	state.short_oid_len = len;
	state.found = 0;
555

556
	/* Explore directory to find a unique object matching short_oid */
557
	error = git_path_direach(
558
		object_location, 0, fn_locate_object_short_oid, &state);
559
	if (error < 0 && error != GIT_EAMBIGUOUS)
560
		return error;
561

562
	if (!state.found)
Russell Belfer committed
563
		return git_odb__error_notfound("no matching loose object for prefix", short_oid);
564

565 566 567
	if (state.found > 1)
		return git_odb__error_ambiguous("multiple matches in loose objects");

568
	/* Convert obtained hex formatted oid to raw */
Vicent Marti committed
569
	error = git_oid_fromstr(res_oid, (char *)state.res_oid);
570 571
	if (error)
		return error;
572 573

	/* Update the location according to the oid obtained */
574 575
	GITERR_CHECK_ALLOC_ADD(&alloc_len, dir_len, GIT_OID_HEXSZ);
	GITERR_CHECK_ALLOC_ADD(&alloc_len, alloc_len, 2);
576 577

	git_buf_truncate(object_location, dir_len);
578
	if (git_buf_grow(object_location, alloc_len) < 0)
579
		return -1;
580 581 582 583 584

	git_oid_pathfmt(object_location->ptr + dir_len, res_oid);

	object_location->size += GIT_OID_HEXSZ + 1;
	object_location->ptr[object_location->size] = '\0';
585

586
	return 0;
587 588
}

589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604








/***********************************************************
 *
 * LOOSE BACKEND PUBLIC API
 *
 * Implement the git_odb_backend API calls
 *
 ***********************************************************/

605
static int loose_backend__read_header(size_t *len_p, git_otype *type_p, git_odb_backend *backend, const git_oid *oid)
606
{
607
	git_buf object_path = GIT_BUF_INIT;
Vicent Marti committed
608
	git_rawobj raw;
609
	int error;
610

Vicent Marti committed
611
	assert(backend && oid);
612

Vicent Marti committed
613 614 615
	raw.len = 0;
	raw.type = GIT_OBJ_BAD;

616
	if (locate_object(&object_path, (loose_backend *)backend, oid) < 0)
Russell Belfer committed
617
		error = git_odb__error_notfound("no matching loose object", oid);
618
	else if ((error = read_header_loose(&raw, &object_path)) == 0) {
619 620 621
		*len_p = raw.len;
		*type_p = raw.type;
	}
622

623
	git_buf_free(&object_path);
624

625
	return error;
Vicent Marti committed
626
}
627

628
static int loose_backend__read(void **buffer_p, size_t *len_p, git_otype *type_p, git_odb_backend *backend, const git_oid *oid)
629
{
630
	git_buf object_path = GIT_BUF_INIT;
Vicent Marti committed
631
	git_rawobj raw;
632
	int error = 0;
633

Vicent Marti committed
634
	assert(backend && oid);
635

636
	if (locate_object(&object_path, (loose_backend *)backend, oid) < 0)
Russell Belfer committed
637
		error = git_odb__error_notfound("no matching loose object", oid);
638
	else if ((error = read_loose(&raw, &object_path)) == 0) {
639 640 641 642
		*buffer_p = raw.data;
		*len_p = raw.len;
		*type_p = raw.type;
	}
Vicent Marti committed
643

644
	git_buf_free(&object_path);
Vicent Marti committed
645

646
	return error;
647 648
}

649
static int loose_backend__read_prefix(
Vicent Marti committed
650 651 652 653 654 655
	git_oid *out_oid,
	void **buffer_p,
	size_t *len_p,
	git_otype *type_p,
	git_odb_backend *backend,
	const git_oid *short_oid,
656
	size_t len)
657
{
658
	int error = 0;
659

660
	assert(len >= GIT_OID_MINPREFIXLEN && len <= GIT_OID_HEXSZ);
661

662
	if (len == GIT_OID_HEXSZ) {
663
		/* We can fall back to regular read method */
664
		error = loose_backend__read(buffer_p, len_p, type_p, backend, short_oid);
665
		if (!error)
666
			git_oid_cpy(out_oid, short_oid);
667
	} else {
668
		git_buf object_path = GIT_BUF_INIT;
669 670 671 672
		git_rawobj raw;

		assert(backend && short_oid);

673
		if ((error = locate_object_short_oid(&object_path, out_oid,
674 675 676
				(loose_backend *)backend, short_oid, len)) == 0 &&
			(error = read_loose(&raw, &object_path)) == 0)
		{
677 678 679
			*buffer_p = raw.data;
			*len_p = raw.len;
			*type_p = raw.type;
680 681
		}

682
		git_buf_free(&object_path);
683
	}
684

685
	return error;
686 687
}

688
static int loose_backend__exists(git_odb_backend *backend, const git_oid *oid)
689
{
690 691
	git_buf object_path = GIT_BUF_INIT;
	int error;
692 693 694

	assert(backend && oid);

695 696 697 698
	error = locate_object(&object_path, (loose_backend *)backend, oid);

	git_buf_free(&object_path);

699
	return !error;
700 701
}

702 703 704 705 706 707
static int loose_backend__exists_prefix(
	git_oid *out, git_odb_backend *backend, const git_oid *short_id, size_t len)
{
	git_buf object_path = GIT_BUF_INIT;
	int error;

708
	assert(backend && out && short_id && len >= GIT_OID_MINPREFIXLEN);
709 710 711 712 713 714 715 716 717

	error = locate_object_short_oid(
		&object_path, out, (loose_backend *)backend, short_id, len);

	git_buf_free(&object_path);

	return error;
}

718 719
struct foreach_state {
	size_t dir_len;
720
	git_odb_foreach_cb cb;
721 722 723
	void *data;
};

724
GIT_INLINE(int) filename_to_oid(git_oid *oid, const char *ptr)
725 726
{
	int v, i = 0;
727
	if (strlen(ptr) != GIT_OID_HEXSZ+1)
728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759
		return -1;

	if (ptr[2] != '/') {
		return -1;
	}

	v = (git__fromhex(ptr[i]) << 4) | git__fromhex(ptr[i+1]);
	if (v < 0)
		return -1;

	oid->id[0] = (unsigned char) v;

	ptr += 3;
	for (i = 0; i < 38; i += 2) {
		v = (git__fromhex(ptr[i]) << 4) | git__fromhex(ptr[i + 1]);
		if (v < 0)
			return -1;

		oid->id[1 + i/2] = (unsigned char) v;
	}

	return 0;
}

static int foreach_object_dir_cb(void *_state, git_buf *path)
{
	git_oid oid;
	struct foreach_state *state = (struct foreach_state *) _state;

	if (filename_to_oid(&oid, path->ptr + state->dir_len) < 0)
		return 0;

760
	return giterr_set_after_callback_function(
761
		state->cb(&oid, state->data), "git_odb_foreach");
762 763 764 765 766 767
}

static int foreach_cb(void *_state, git_buf *path)
{
	struct foreach_state *state = (struct foreach_state *) _state;

768 769 770 771
	/* non-dir is some stray file, ignore it */
	if (!git_path_isdir(git_buf_cstr(path)))
		return 0;

772
	return git_path_direach(path, 0, foreach_object_dir_cb, state);
773 774
}

775
static int loose_backend__foreach(git_odb_backend *_backend, git_odb_foreach_cb cb, void *data)
776 777 778 779 780 781 782 783 784 785 786
{
	char *objects_dir;
	int error;
	git_buf buf = GIT_BUF_INIT;
	struct foreach_state state;
	loose_backend *backend = (loose_backend *) _backend;

	assert(backend && cb);

	objects_dir = backend->objects_dir;

787
	git_buf_sets(&buf, objects_dir);
788
	git_path_to_dir(&buf);
789 790
	if (git_buf_oom(&buf))
		return -1;
791

792
	memset(&state, 0, sizeof(state));
793 794 795 796
	state.cb = cb;
	state.data = data;
	state.dir_len = git_buf_len(&buf);

797
	error = git_path_direach(&buf, 0, foreach_cb, &state);
798

799 800
	git_buf_free(&buf);

801
	return error;
802 803
}

804
static int loose_backend__stream_fwrite(git_odb_stream *_stream, const git_oid *oid)
Vicent Marti committed
805 806 807
{
	loose_writestream *stream = (loose_writestream *)_stream;
	loose_backend *backend = (loose_backend *)_stream->backend;
808
	git_buf final_path = GIT_BUF_INIT;
809
	int error = 0;
Vicent Marti committed
810

811
	if (object_file_name(&final_path, backend, oid) < 0 ||
812
		object_mkdir(&final_path, backend) < 0)
813 814 815
		error = -1;
	else
		error = git_filebuf_commit_at(
816
			&stream->fbuf, final_path.ptr);
817 818 819 820

	git_buf_free(&final_path);

	return error;
Vicent Marti committed
821 822
}

823
static int loose_backend__stream_write(git_odb_stream *_stream, const char *data, size_t len)
824
{
Vicent Marti committed
825 826 827 828
	loose_writestream *stream = (loose_writestream *)_stream;
	return git_filebuf_write(&stream->fbuf, data, len);
}

829
static void loose_backend__stream_free(git_odb_stream *_stream)
Vicent Marti committed
830 831 832
{
	loose_writestream *stream = (loose_writestream *)_stream;

833
	git_filebuf_cleanup(&stream->fbuf);
834
	git__free(stream);
Vicent Marti committed
835 836
}

837
static int loose_backend__stream(git_odb_stream **stream_out, git_odb_backend *_backend, git_off_t length, git_otype type)
Vicent Marti committed
838 839
{
	loose_backend *backend;
840
	loose_writestream *stream = NULL;
841 842
	char hdr[64];
	git_buf tmp_path = GIT_BUF_INIT;
Vicent Marti committed
843
	int hdrlen;
844

845
	assert(_backend && length >= 0);
846 847

	backend = (loose_backend *)_backend;
Vicent Marti committed
848
	*stream_out = NULL;
849

850
	hdrlen = git_odb__format_object_header(hdr, sizeof(hdr), length, type);
Vicent Marti committed
851 852

	stream = git__calloc(1, sizeof(loose_writestream));
853
	GITERR_CHECK_ALLOC(stream);
854

Vicent Marti committed
855 856 857 858 859 860
	stream->stream.backend = _backend;
	stream->stream.read = NULL; /* read only */
	stream->stream.write = &loose_backend__stream_write;
	stream->stream.finalize_write = &loose_backend__stream_fwrite;
	stream->stream.free = &loose_backend__stream_free;
	stream->stream.mode = GIT_STREAM_WRONLY;
861

862 863 864
	if (git_buf_joinpath(&tmp_path, backend->objects_dir, "tmp_object") < 0 ||
		git_filebuf_open(&stream->fbuf, tmp_path.ptr,
			GIT_FILEBUF_TEMPORARY |
865 866
			(backend->object_zlib_level << GIT_FILEBUF_DEFLATE_SHIFT),
			backend->object_file_mode) < 0 ||
867 868 869 870 871 872
		stream->stream.write((git_odb_stream *)stream, hdr, hdrlen) < 0)
	{
		git_filebuf_cleanup(&stream->fbuf);
		git__free(stream);
		stream = NULL;
	}
873
	git_buf_free(&tmp_path);
Vicent Marti committed
874
	*stream_out = (git_odb_stream *)stream;
875

876
	return !stream ? -1 : 0;
877 878
}

879
static int loose_backend__write(git_odb_backend *_backend, const git_oid *oid, const void *data, size_t len, git_otype type)
880
{
881
	int error = 0, header_len;
882 883
	git_buf final_path = GIT_BUF_INIT;
	char header[64];
884
	git_filebuf fbuf = GIT_FILEBUF_INIT;
885 886 887 888 889
	loose_backend *backend;

	backend = (loose_backend *)_backend;

	/* prepare the header for the file */
890
	header_len = git_odb__format_object_header(header, sizeof(header), len, type);
891

892 893 894
	if (git_buf_joinpath(&final_path, backend->objects_dir, "tmp_object") < 0 ||
		git_filebuf_open(&fbuf, final_path.ptr,
			GIT_FILEBUF_TEMPORARY |
895 896
			(backend->object_zlib_level << GIT_FILEBUF_DEFLATE_SHIFT),
			backend->object_file_mode) < 0)
897 898
	{
		error = -1;
899
		goto cleanup;
900
	}
901 902 903 904

	git_filebuf_write(&fbuf, header, header_len);
	git_filebuf_write(&fbuf, data, len);

905 906
	if (object_file_name(&final_path, backend, oid) < 0 ||
		object_mkdir(&final_path, backend) < 0 ||
907
		git_filebuf_commit_at(&fbuf, final_path.ptr) < 0)
908
		error = -1;
909 910

cleanup:
911
	if (error < 0)
912 913
		git_filebuf_cleanup(&fbuf);
	git_buf_free(&final_path);
914 915 916
	return error;
}

917
static void loose_backend__free(git_odb_backend *_backend)
918 919 920 921 922
{
	loose_backend *backend;
	assert(_backend);
	backend = (loose_backend *)_backend;

923
	git__free(backend);
924 925
}

926 927 928 929
int git_odb_backend_loose(
	git_odb_backend **backend_out,
	const char *objects_dir,
	int compression_level,
930
	int do_fsync,
931 932
	unsigned int dir_mode,
	unsigned int file_mode)
933 934
{
	loose_backend *backend;
935
	size_t objects_dirlen, alloclen;
936 937 938 939

	assert(backend_out && objects_dir);

	objects_dirlen = strlen(objects_dir);
940

941 942 943
	GITERR_CHECK_ALLOC_ADD(&alloclen, sizeof(loose_backend), objects_dirlen);
	GITERR_CHECK_ALLOC_ADD(&alloclen, alloclen, 2);
	backend = git__calloc(1, alloclen);
944
	GITERR_CHECK_ALLOC(backend);
945

946
	backend->parent.version = GIT_ODB_BACKEND_VERSION;
947 948 949 950
	backend->objects_dirlen = objects_dirlen;
	memcpy(backend->objects_dir, objects_dir, objects_dirlen);
	if (backend->objects_dir[backend->objects_dirlen - 1] != '/')
		backend->objects_dir[backend->objects_dirlen++] = '/';
951

952 953 954
	if (compression_level < 0)
		compression_level = Z_BEST_SPEED;

955 956 957 958 959 960
	if (dir_mode == 0)
		dir_mode = GIT_OBJECT_DIR_MODE;

	if (file_mode == 0)
		file_mode = GIT_OBJECT_FILE_MODE;

961 962
	backend->object_zlib_level = compression_level;
	backend->fsync_object_files = do_fsync;
963 964
	backend->object_dir_mode = dir_mode;
	backend->object_file_mode = file_mode;
965 966

	backend->parent.read = &loose_backend__read;
967
	backend->parent.write = &loose_backend__write;
Vicent Marti committed
968
	backend->parent.read_prefix = &loose_backend__read_prefix;
969
	backend->parent.read_header = &loose_backend__read_header;
Vicent Marti committed
970
	backend->parent.writestream = &loose_backend__stream;
971
	backend->parent.exists = &loose_backend__exists;
972
	backend->parent.exists_prefix = &loose_backend__exists_prefix;
973
	backend->parent.foreach = &loose_backend__foreach;
974 975 976
	backend->parent.free = &loose_backend__free;

	*backend_out = (git_odb_backend *)backend;
977
	return 0;
978
}