smart_protocol.c 28.7 KB
Newer Older
1
/*
Edward Thomson committed
2
 * Copyright (C) the libgit2 contributors. All rights reserved.
3 4 5 6
 *
 * This file is part of libgit2, distributed under the GNU GPL v2 with
 * a Linking Exception. For full terms see the included COPYING file.
 */
7 8 9

#include "common.h"

10
#include "git2.h"
11
#include "git2/odb_backend.h"
12

13 14
#include "smart.h"
#include "refs.h"
15
#include "repository.h"
16 17 18
#include "push.h"
#include "pack-objects.h"
#include "remote.h"
19
#include "util.h"
20
#include "revwalk.h"
21 22

#define NETWORK_XFER_THRESHOLD (100*1024)
23 24
/* The minimal interval between progress updates (in seconds). */
#define MIN_PROGRESS_UPDATE_INTERVAL 0.5
25

26 27
bool git_smart__ofs_delta_enabled = true;

28 29 30 31
int git_smart__store_refs(transport_smart *t, int flushes)
{
	git_vector *refs = &t->refs;
	int error, flush = 0, recvd;
32 33
	const char *line_end = NULL;
	git_pkt *pkt = NULL;
Edward Thomson committed
34
	git_pkt_parse_data pkt_parse_data = { 0 };
Edward Thomson committed
35
	size_t i;
36

37 38 39
	/* Clear existing refs in case git_remote_connect() is called again
	 * after git_remote_disconnect().
	 */
40 41
	git_vector_foreach(refs, i, pkt) {
		git_pkt_free(pkt);
Edward Thomson committed
42
	}
43
	git_vector_clear(refs);
44
	pkt = NULL;
45

46
	do {
47 48 49 50
		if (t->buffer.len > 0)
			error = git_pkt_parse_line(&pkt, &line_end,
				t->buffer.data, t->buffer.len,
				&pkt_parse_data);
51 52 53 54
		else
			error = GIT_EBUFS;

		if (error < 0 && error != GIT_EBUFS)
55
			return error;
56 57

		if (error == GIT_EBUFS) {
58
			if ((recvd = git_smart__recv(t)) < 0)
59
				return recvd;
60

61
			if (recvd == 0) {
62
				git_error_set(GIT_ERROR_NET, "could not read refs from remote repository");
63
				return GIT_EEOF;
64 65 66 67 68
			}

			continue;
		}

69
		git_staticstr_consume(&t->buffer, line_end);
70

71
		if (pkt->type == GIT_PKT_ERR) {
72
			git_error_set(GIT_ERROR_NET, "remote error: %s", ((git_pkt_err *)pkt)->error);
73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88
			git__free(pkt);
			return -1;
		}

		if (pkt->type != GIT_PKT_FLUSH && git_vector_insert(refs, pkt) < 0)
			return -1;

		if (pkt->type == GIT_PKT_FLUSH) {
			flush++;
			git_pkt_free(pkt);
		}
	} while (flush < flushes);

	return flush;
}

89 90 91 92
static int append_symref(const char **out, git_vector *symrefs, const char *ptr)
{
	int error;
	const char *end;
93
	git_str buf = GIT_STR_INIT;
Jacques Germishuys committed
94
	git_refspec *mapping = NULL;
95 96 97 98 99 100 101 102 103 104

	ptr += strlen(GIT_CAP_SYMREF);
	if (*ptr != '=')
		goto on_invalid;

	ptr++;
	if (!(end = strchr(ptr, ' ')) &&
	    !(end = strchr(ptr, '\0')))
		goto on_invalid;

105
	if ((error = git_str_put(&buf, ptr, end - ptr)) < 0)
106 107 108
		return error;

	/* symref mapping has refspec format */
Jacques Germishuys committed
109
	mapping = git__calloc(1, sizeof(git_refspec));
110
	GIT_ERROR_CHECK_ALLOC(mapping);
111

112 113
	error = git_refspec__parse(mapping, git_str_cstr(&buf), true);
	git_str_dispose(&buf);
114 115 116

	/* if the error isn't OOM, then it's a parse error; let's use a nicer message */
	if (error < 0) {
117
		if (git_error_last()->klass != GIT_ERROR_NOMEMORY)
118 119
			goto on_invalid;

120
		git__free(mapping);
121 122 123 124 125 126 127 128 129 130
		return error;
	}

	if ((error = git_vector_insert(symrefs, mapping)) < 0)
		return error;

	*out = end;
	return 0;

on_invalid:
131
	git_error_set(GIT_ERROR_NET, "remote sent invalid symref");
132
	git_refspec__dispose(mapping);
133
	git__free(mapping);
134 135 136
	return -1;
}

137 138 139 140
int git_smart__detect_caps(
	git_pkt_ref *pkt,
	transport_smart_caps *caps,
	git_vector *symrefs)
141
{
142
	const char *ptr, *start;
143

Dimitris Apostolou committed
144
	/* No refs or capabilities, odd but not a problem */
145
	if (pkt == NULL || pkt->capabilities == NULL)
146
		return GIT_ENOTFOUND;
147 148 149 150 151 152

	ptr = pkt->capabilities;
	while (ptr != NULL && *ptr != '\0') {
		if (*ptr == ' ')
			ptr++;

153
		if (git_smart__ofs_delta_enabled && !git__prefixcmp(ptr, GIT_CAP_OFS_DELTA)) {
154 155 156 157 158
			caps->common = caps->ofs_delta = 1;
			ptr += strlen(GIT_CAP_OFS_DELTA);
			continue;
		}

159 160 161 162 163 164 165
		/* Keep multi_ack_detailed before multi_ack */
		if (!git__prefixcmp(ptr, GIT_CAP_MULTI_ACK_DETAILED)) {
			caps->common = caps->multi_ack_detailed = 1;
			ptr += strlen(GIT_CAP_MULTI_ACK_DETAILED);
			continue;
		}

166
		if (!git__prefixcmp(ptr, GIT_CAP_MULTI_ACK)) {
167 168 169 170 171
			caps->common = caps->multi_ack = 1;
			ptr += strlen(GIT_CAP_MULTI_ACK);
			continue;
		}

172
		if (!git__prefixcmp(ptr, GIT_CAP_INCLUDE_TAG)) {
173 174 175 176 177 178
			caps->common = caps->include_tag = 1;
			ptr += strlen(GIT_CAP_INCLUDE_TAG);
			continue;
		}

		/* Keep side-band check after side-band-64k */
179
		if (!git__prefixcmp(ptr, GIT_CAP_SIDE_BAND_64K)) {
180 181 182 183 184
			caps->common = caps->side_band_64k = 1;
			ptr += strlen(GIT_CAP_SIDE_BAND_64K);
			continue;
		}

185
		if (!git__prefixcmp(ptr, GIT_CAP_SIDE_BAND)) {
186 187 188 189 190
			caps->common = caps->side_band = 1;
			ptr += strlen(GIT_CAP_SIDE_BAND);
			continue;
		}

191 192 193 194 195 196
		if (!git__prefixcmp(ptr, GIT_CAP_DELETE_REFS)) {
			caps->common = caps->delete_refs = 1;
			ptr += strlen(GIT_CAP_DELETE_REFS);
			continue;
		}

197 198 199 200 201 202
		if (!git__prefixcmp(ptr, GIT_CAP_THIN_PACK)) {
			caps->common = caps->thin_pack = 1;
			ptr += strlen(GIT_CAP_THIN_PACK);
			continue;
		}

203 204 205 206 207 208 209 210 211
		if (!git__prefixcmp(ptr, GIT_CAP_SYMREF)) {
			int error;

			if ((error = append_symref(&ptr, symrefs, ptr)) < 0)
				return error;

			continue;
		}

212 213
		if (!git__prefixcmp(ptr, GIT_CAP_WANT_TIP_SHA1)) {
			caps->common = caps->want_tip_sha1 = 1;
214
			ptr += strlen(GIT_CAP_WANT_TIP_SHA1);
215 216 217 218 219
			continue;
		}

		if (!git__prefixcmp(ptr, GIT_CAP_WANT_REACHABLE_SHA1)) {
			caps->common = caps->want_reachable_sha1 = 1;
220
			ptr += strlen(GIT_CAP_WANT_REACHABLE_SHA1);
221 222 223
			continue;
		}

224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243
		if (!git__prefixcmp(ptr, GIT_CAP_OBJECT_FORMAT)) {
			ptr += strlen(GIT_CAP_OBJECT_FORMAT);

			start = ptr;
			ptr = strchr(ptr, ' ');

			if ((caps->object_format = git__strndup(start, (ptr - start))) == NULL)
				return -1;
			continue;
		}

		if (!git__prefixcmp(ptr, GIT_CAP_AGENT)) {
			ptr += strlen(GIT_CAP_AGENT);

			start = ptr;
			ptr = strchr(ptr, ' ');

			if ((caps->agent = git__strndup(start, (ptr - start))) == NULL)
				return -1;
			continue;
244
		}
245

246 247 248
		if (!git__prefixcmp(ptr, GIT_CAP_SHALLOW)) {
			caps->common = caps->shallow = 1;
			ptr += strlen(GIT_CAP_SHALLOW);
249 250 251
			continue;
		}

252 253 254 255 256 257 258
		/* We don't know this capability, so skip it */
		ptr = strchr(ptr, ' ');
	}

	return 0;
}

259 260 261
static int recv_pkt(
	git_pkt **out_pkt,
	git_pkt_type *out_type,
262
	transport_smart *t)
263
{
264
	const char *ptr = t->buffer.data, *line_end = ptr;
265
	git_pkt *pkt = NULL;
Edward Thomson committed
266
	git_pkt_parse_data pkt_parse_data = { 0 };
267
	int error = 0, ret;
268

269 270 271
	pkt_parse_data.oid_type = t->owner->repo->oid_type;
	pkt_parse_data.seen_capabilities = 1;

272
	do {
273 274 275
		if (t->buffer.len > 0)
			error = git_pkt_parse_line(&pkt, &line_end, ptr,
				t->buffer.len, &pkt_parse_data);
276 277 278 279 280 281 282
		else
			error = GIT_EBUFS;

		if (error == 0)
			break; /* return the pkt */

		if (error < 0 && error != GIT_EBUFS)
283
			return error;
284

285
		if ((ret = git_smart__recv(t)) < 0) {
286
			return ret;
287
		} else if (ret == 0) {
288
			git_error_set(GIT_ERROR_NET, "could not read from remote repository");
289 290
			return GIT_EEOF;
		}
291 292
	} while (error);

293
	git_staticstr_consume(&t->buffer, line_end);
294

295 296 297 298
	if (out_type != NULL)
		*out_type = pkt->type;
	if (out_pkt != NULL)
		*out_pkt = pkt;
299 300 301
	else
		git__free(pkt);

302
	return error;
303 304 305 306 307
}

static int store_common(transport_smart *t)
{
	git_pkt *pkt = NULL;
308
	int error;
309 310

	do {
311
		if ((error = recv_pkt(&pkt, NULL, t)) < 0)
312
			return error;
313

314
		if (pkt->type != GIT_PKT_ACK) {
315 316 317 318
			git__free(pkt);
			return 0;
		}

319 320 321 322
		if (git_vector_insert(&t->common, pkt) < 0) {
			git__free(pkt);
			return -1;
		}
323 324 325 326 327
	} while (1);

	return 0;
}

328
static int wait_while_ack(transport_smart *t)
329 330
{
	int error;
331 332
	git_pkt *pkt = NULL;
	git_pkt_ack *ack = NULL;
333 334

	while (1) {
335
		git_pkt_free(pkt);
336

337
		if ((error = recv_pkt(&pkt, NULL, t)) < 0)
338 339 340 341
			return error;

		if (pkt->type == GIT_PKT_NAK)
			break;
342 343
		if (pkt->type != GIT_PKT_ACK)
			continue;
344

345 346 347 348 349 350
		ack = (git_pkt_ack*)pkt;

		if (ack->status != GIT_ACK_CONTINUE &&
		    ack->status != GIT_ACK_COMMON &&
		    ack->status != GIT_ACK_READY) {
			break;
351 352 353
		}
	}

354
	git_pkt_free(pkt);
355 356 357
	return 0;
}

358 359 360 361 362 363 364
static int cap_not_sup_err(const char *cap_name)
{
	git_error_set(GIT_ERROR_NET, "server doesn't support %s", cap_name);
	return GIT_EINVALID;
}

/* Disables server capabilities we're not interested in */
365 366 367
static int setup_caps(
	transport_smart_caps *caps,
	const git_fetch_negotiation *wants)
368
{
369
	if (wants->depth > 0) {
370 371 372 373 374 375 376 377 378
		if (!caps->shallow)
			return cap_not_sup_err(GIT_CAP_SHALLOW);
	} else {
		caps->shallow = 0;
	}

	return 0;
}

379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399
static int setup_shallow_roots(
	git_array_oid_t *out,
	const git_fetch_negotiation *wants)
{
	git_array_clear(*out);

	if (wants->shallow_roots_len > 0) {
		git_array_init_to_size(*out, wants->shallow_roots_len);
		GIT_ERROR_CHECK_ALLOC(out->ptr);

		memcpy(out->ptr, wants->shallow_roots,
		       sizeof(git_oid) * wants->shallow_roots_len);
	}

	return 0;
}

int git_smart__negotiate_fetch(
	git_transport *transport,
	git_repository *repo,
	const git_fetch_negotiation *wants)
400 401
{
	transport_smart *t = (transport_smart *)transport;
402
	git_revwalk__push_options opts = GIT_REVWALK__PUSH_OPTIONS_INIT;
403
	git_str data = GIT_STR_INIT;
404
	git_revwalk *walk = NULL;
405 406
	int error = -1;
	git_pkt_type pkt_type;
407 408 409
	unsigned int i;
	git_oid oid;

410 411
	if ((error = setup_caps(&t->caps, wants)) < 0 ||
	    (error = setup_shallow_roots(&t->shallow_roots, wants)) < 0)
412 413 414
		return error;

	if ((error = git_pkt_buffer_wants(wants, &t->caps, &data)) < 0)
415
		return error;
416

417 418 419
	if ((error = git_revwalk_new(&walk, repo)) < 0)
		goto on_error;

420 421
	opts.insert_by_date = 1;
	if ((error = git_revwalk__push_glob(walk, "refs/*", &opts)) < 0)
422
		goto on_error;
423

424 425 426 427 428 429
	if (wants->depth > 0) {
		git_pkt_shallow *pkt;

		if ((error = git_smart__negotiation_step(&t->parent, data.ptr, data.size)) < 0)
			goto on_error;

430
		while ((error = recv_pkt((git_pkt **)&pkt, NULL, t)) == 0) {
431 432
			bool complete = false;

433
			if (pkt->type == GIT_PKT_SHALLOW) {
434
				error = git_oidarray__add(&t->shallow_roots, &pkt->oid);
435
			} else if (pkt->type == GIT_PKT_UNSHALLOW) {
436
				git_oidarray__remove(&t->shallow_roots, &pkt->oid);
437 438
			} else if (pkt->type == GIT_PKT_FLUSH) {
				/* Server is done, stop processing shallow oids */
439
				complete = true;
440
			} else {
441 442
				git_error_set(GIT_ERROR_NET, "unexpected packet type");
				error = -1;
443
			}
444 445

			git_pkt_free((git_pkt *) pkt);
446

447 448 449
			if (complete || error < 0)
				break;
		}
450

451
		if (error < 0)
452 453
			goto on_error;
	}
454

455
	/*
456 457 458 459
	 * Our support for ACK extensions is simply to parse them. On
	 * the first ACK we will accept that as enough common
	 * objects. We give up if we haven't found an answer in the
	 * first 256 we send.
460 461
	 */
	i = 0;
462
	while (i < 256) {
463 464 465 466 467 468 469 470 471
		error = git_revwalk_next(&oid, walk);

		if (error < 0) {
			if (GIT_ITEROVER == error)
				break;

			goto on_error;
		}

472 473 474 475
		git_pkt_buffer_have(&oid, &data);
		i++;
		if (i % 20 == 0) {
			if (t->cancelled.val) {
476
				git_error_set(GIT_ERROR_NET, "The fetch was cancelled by the user");
477 478 479 480 481
				error = GIT_EUSER;
				goto on_error;
			}

			git_pkt_buffer_flush(&data);
482
			if (git_str_oom(&data)) {
483
				error = -1;
484
				goto on_error;
485
			}
486

487
			if ((error = git_smart__negotiation_step(&t->parent, data.ptr, data.size)) < 0)
488 489
				goto on_error;

490
			git_str_clear(&data);
491
			if (t->caps.multi_ack || t->caps.multi_ack_detailed) {
492
				if ((error = store_common(t)) < 0)
493 494
					goto on_error;
			} else {
495
				if ((error = recv_pkt(NULL, &pkt_type, t)) < 0)
496
					goto on_error;
497 498

				if (pkt_type == GIT_PKT_ACK) {
499 500 501 502
					break;
				} else if (pkt_type == GIT_PKT_NAK) {
					continue;
				} else {
503
					git_error_set(GIT_ERROR_NET, "unexpected pkt type");
504
					error = -1;
505 506 507 508 509 510 511 512 513 514
					goto on_error;
				}
			}
		}

		if (t->common.length > 0)
			break;

		if (i % 20 == 0 && t->rpc) {
			git_pkt_ack *pkt;
515
			unsigned int j;
516

517
			if ((error = git_pkt_buffer_wants(wants, &t->caps, &data)) < 0)
518 519
				goto on_error;

520
			git_vector_foreach(&t->common, j, pkt) {
521 522
				if ((error = git_pkt_buffer_have(&pkt->oid, &data)) < 0)
					goto on_error;
523 524
			}

525
			if (git_str_oom(&data)) {
526
				error = -1;
527
				goto on_error;
528
			}
529 530 531 532 533 534
		}
	}

	/* Tell the other end that we're done negotiating */
	if (t->rpc && t->common.length > 0) {
		git_pkt_ack *pkt;
535
		unsigned int j;
536

537
		if ((error = git_pkt_buffer_wants(wants, &t->caps, &data)) < 0)
538 539
			goto on_error;

540
		git_vector_foreach(&t->common, j, pkt) {
541 542
			if ((error = git_pkt_buffer_have(&pkt->oid, &data)) < 0)
				goto on_error;
543 544
		}

545
		if (git_str_oom(&data)) {
546
			error = -1;
547
			goto on_error;
548
		}
549 550
	}

551 552 553
	if ((error = git_pkt_buffer_done(&data)) < 0)
		goto on_error;

554
	if (t->cancelled.val) {
555
		git_error_set(GIT_ERROR_NET, "the fetch was cancelled");
556 557 558
		error = GIT_EUSER;
		goto on_error;
	}
559

560
	if ((error = git_smart__negotiation_step(&t->parent, data.ptr, data.size)) < 0)
561 562
		goto on_error;

563
	git_str_dispose(&data);
564 565 566
	git_revwalk_free(walk);

	/* Now let's eat up whatever the server gives us */
567
	if (!t->caps.multi_ack && !t->caps.multi_ack_detailed) {
568
		if ((error = recv_pkt(NULL, &pkt_type, t)) < 0)
569
			return error;
570 571

		if (pkt_type != GIT_PKT_ACK && pkt_type != GIT_PKT_NAK) {
572
			git_error_set(GIT_ERROR_NET, "unexpected pkt type");
573 574 575
			return -1;
		}
	} else {
576
		error = wait_while_ack(t);
577 578
	}

579
	return error;
580 581 582

on_error:
	git_revwalk_free(walk);
583
	git_str_dispose(&data);
584 585 586
	return error;
}

587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605
int git_smart__shallow_roots(git_oidarray *out, git_transport *transport)
{
	transport_smart *t = (transport_smart *)transport;
	size_t len;

	GIT_ERROR_CHECK_ALLOC_MULTIPLY(&len, t->shallow_roots.size, sizeof(git_oid));

	out->count = t->shallow_roots.size;

	if (len) {
		out->ids = git__malloc(len);
		memcpy(out->ids, t->shallow_roots.ptr, len);
	} else {
		out->ids = NULL;
	}

	return 0;
}

606 607 608 609
static int no_sideband(
	transport_smart *t,
	struct git_odb_writepack *writepack,
	git_indexer_progress *stats)
610 611 612 613 614
{
	int recvd;

	do {
		if (t->cancelled.val) {
615
			git_error_set(GIT_ERROR_NET, "the fetch was cancelled by the user");
616 617 618
			return GIT_EUSER;
		}

619
		if (writepack->append(writepack, t->buffer.data, t->buffer.len, stats) < 0)
620 621
			return -1;

622
		git_staticstr_clear(&t->buffer);
623

624
		if ((recvd = git_smart__recv(t)) < 0)
625
			return recvd;
626 627
	} while(recvd > 0);

628
	if (writepack->commit(writepack, stats) < 0)
629 630 631 632 633
		return -1;

	return 0;
}

634
struct network_packetsize_payload
635
{
636
	git_indexer_progress_cb callback;
637
	void *payload;
638
	git_indexer_progress *stats;
639
	size_t last_fired_bytes;
640 641
};

642
static int network_packetsize(size_t received, void *payload)
643 644 645 646 647 648 649 650
{
	struct network_packetsize_payload *npp = (struct network_packetsize_payload*)payload;

	/* Accumulate bytes */
	npp->stats->received_bytes += received;

	/* Fire notification if the threshold is reached */
	if ((npp->stats->received_bytes - npp->last_fired_bytes) > NETWORK_XFER_THRESHOLD) {
651
		npp->last_fired_bytes = npp->stats->received_bytes;
652

653
		if (npp->callback(npp->stats, npp->payload))
654
			return GIT_EUSER;
655
	}
656 657

	return 0;
658 659 660 661 662
}

int git_smart__download_pack(
	git_transport *transport,
	git_repository *repo,
663
	git_indexer_progress *stats)
664 665
{
	transport_smart *t = (transport_smart *)transport;
666 667
	git_odb *odb;
	struct git_odb_writepack *writepack = NULL;
668
	int error = 0;
669 670
	struct network_packetsize_payload npp = {0};

671 672 673
	git_indexer_progress_cb progress_cb = t->connect_opts.callbacks.transfer_progress;
	void *progress_payload = t->connect_opts.callbacks.payload;

674
	memset(stats, 0, sizeof(git_indexer_progress));
675

676 677
	if (progress_cb) {
		npp.callback = progress_cb;
678 679 680 681
		npp.payload = progress_payload;
		npp.stats = stats;
		t->packetsize_cb = &network_packetsize;
		t->packetsize_payload = &npp;
682 683

		/* We might have something in the buffer already from negotiate_fetch */
684 685
		if (t->buffer.len > 0 && !t->cancelled.val) {
			if (t->packetsize_cb(t->buffer.len, t->packetsize_payload))
686
				git_atomic32_set(&t->cancelled, 1);
687
		}
688 689
	}

690
	if ((error = git_repository_odb__weakptr(&odb, repo)) < 0 ||
691
		((error = git_odb_write_pack(&writepack, odb, progress_cb, progress_payload)) != 0))
692
		goto done;
693 694 695

	/*
	 * If the remote doesn't support the side-band, we can feed
696
	 * the data directly to the pack writer. Otherwise, we need to
697 698 699
	 * check which one belongs there.
	 */
	if (!t->caps.side_band && !t->caps.side_band_64k) {
700
		error = no_sideband(t, writepack, stats);
701
		goto done;
702 703 704
	}

	do {
705
		git_pkt *pkt = NULL;
706

707
		/* Check cancellation before network call */
708
		if (t->cancelled.val) {
709
			git_error_clear();
710
			error = GIT_EUSER;
711
			goto done;
712 713
		}

714
		if ((error = recv_pkt(&pkt, NULL, t)) >= 0) {
715 716
			/* Check cancellation after network call */
			if (t->cancelled.val) {
717
				git_error_clear();
718 719
				error = GIT_EUSER;
			} else if (pkt->type == GIT_PKT_PROGRESS) {
720
				if (t->connect_opts.callbacks.sideband_progress) {
721
					git_pkt_progress *p = (git_pkt_progress *) pkt;
722 723 724 725 726 727 728

					if (p->len > INT_MAX) {
						git_error_set(GIT_ERROR_NET, "oversized progress message");
						error = GIT_ERROR;
						goto done;
					}

729
					error = t->connect_opts.callbacks.sideband_progress(p->data, (int)p->len, t->connect_opts.callbacks.payload);
730 731 732
				}
			} else if (pkt->type == GIT_PKT_DATA) {
				git_pkt_data *p = (git_pkt_data *) pkt;
733 734 735

				if (p->len)
					error = writepack->append(writepack, p->data, p->len, stats);
736 737 738 739 740
			} else if (pkt->type == GIT_PKT_FLUSH) {
				/* A flush indicates the end of the packfile */
				git__free(pkt);
				break;
			}
741
		}
742

743 744
		git_pkt_free(pkt);

745 746
		if (error < 0)
			goto done;
747 748 749

	} while (1);

750
	/*
751
	 * Trailing execution of progress_cb, if necessary...
752 753
	 * Only the callback through the npp datastructure currently
	 * updates the last_fired_bytes value. It is possible that
754
	 * progress has already been reported with the correct
755 756 757 758 759
	 * "received_bytes" value, but until (if?) this is unified
	 * then we will report progress again to be sure that the
	 * correct last received_bytes value is reported.
	 */
	if (npp.callback && npp.stats->received_bytes > npp.last_fired_bytes) {
760 761
		error = npp.callback(npp.stats, npp.payload);
		if (error != 0)
762 763 764
			goto done;
	}

765
	error = writepack->commit(writepack, stats);
766

767
done:
768 769
	if (writepack)
		writepack->free(writepack);
770
	if (progress_cb) {
771 772 773
		t->packetsize_cb = NULL;
		t->packetsize_payload = NULL;
	}
774

775 776
	return error;
}
777

778
static int gen_pktline(git_str *buf, git_push *push)
779 780
{
	push_spec *spec;
781
	size_t i, len;
782
	char old_id[GIT_OID_SHA1_HEXSIZE+1], new_id[GIT_OID_SHA1_HEXSIZE+1];
783

784
	old_id[GIT_OID_SHA1_HEXSIZE] = '\0'; new_id[GIT_OID_SHA1_HEXSIZE] = '\0';
785 786

	git_vector_foreach(&push->specs, i, spec) {
787
		len = 2*GIT_OID_SHA1_HEXSIZE + 7 + strlen(spec->refspec.dst);
788 789

		if (i == 0) {
790
			++len; /* '\0' */
791
			if (push->report_status)
792 793
				len += strlen(GIT_CAP_REPORT_STATUS) + 1;
			len += strlen(GIT_CAP_SIDE_BAND_64K) + 1;
794 795
		}

796 797
		git_oid_fmt(old_id, &spec->roid);
		git_oid_fmt(new_id, &spec->loid);
798

799
		git_str_printf(buf, "%04"PRIxZ"%s %s %s", len, old_id, new_id, spec->refspec.dst);
800 801

		if (i == 0) {
802
			git_str_putc(buf, '\0');
803 804
			/* Core git always starts their capabilities string with a space */
			if (push->report_status) {
805 806
				git_str_putc(buf, ' ');
				git_str_printf(buf, GIT_CAP_REPORT_STATUS);
807
			}
808 809
			git_str_putc(buf, ' ');
			git_str_printf(buf, GIT_CAP_SIDE_BAND_64K);
810 811
		}

812
		git_str_putc(buf, '\n');
813
	}
814

815 816
	git_str_puts(buf, "0000");
	return git_str_oom(buf) ? -1 : 0;
817 818
}

819 820 821 822 823 824
static int add_push_report_pkt(git_push *push, git_pkt *pkt)
{
	push_status *status;

	switch (pkt->type) {
		case GIT_PKT_OK:
825
			status = git__calloc(1, sizeof(push_status));
826
			GIT_ERROR_CHECK_ALLOC(status);
827 828 829 830 831 832 833 834 835
			status->msg = NULL;
			status->ref = git__strdup(((git_pkt_ok *)pkt)->ref);
			if (!status->ref ||
				git_vector_insert(&push->status, status) < 0) {
				git_push_status_free(status);
				return -1;
			}
			break;
		case GIT_PKT_NG:
836
			status = git__calloc(1, sizeof(push_status));
837
			GIT_ERROR_CHECK_ALLOC(status);
838 839 840 841 842 843 844 845 846 847 848 849 850 851
			status->ref = git__strdup(((git_pkt_ng *)pkt)->ref);
			status->msg = git__strdup(((git_pkt_ng *)pkt)->msg);
			if (!status->ref || !status->msg ||
				git_vector_insert(&push->status, status) < 0) {
				git_push_status_free(status);
				return -1;
			}
			break;
		case GIT_PKT_UNPACK:
			push->unpack_ok = ((git_pkt_unpack *)pkt)->unpack_ok;
			break;
		case GIT_PKT_FLUSH:
			return GIT_ITEROVER;
		default:
852
			git_error_set(GIT_ERROR_NET, "report-status: protocol error");
853 854 855 856 857 858
			return -1;
	}

	return 0;
}

859
static int add_push_report_sideband_pkt(git_push *push, git_pkt_data *data_pkt, git_str *data_pkt_buf)
860 861
{
	git_pkt *pkt;
Edward Thomson committed
862
	git_pkt_parse_data pkt_parse_data = { 0 };
863
	const char *line, *line_end = NULL;
864
	size_t line_len;
865
	int error;
866 867 868 869 870
	int reading_from_buf = data_pkt_buf->size > 0;

	if (reading_from_buf) {
		/* We had an existing partial packet, so add the new
		 * packet to the buffer and parse the whole thing */
871
		git_str_put(data_pkt_buf, data_pkt->data, data_pkt->len);
872 873 874 875 876 877 878
		line = data_pkt_buf->ptr;
		line_len = data_pkt_buf->size;
	}
	else {
		line = data_pkt->data;
		line_len = data_pkt->len;
	}
879 880

	while (line_len > 0) {
Edward Thomson committed
881
		error = git_pkt_parse_line(&pkt, &line_end, line, line_len, &pkt_parse_data);
882

883 884 885 886
		if (error == GIT_EBUFS) {
			/* Buffer the data when the inner packet is split
			 * across multiple sideband packets */
			if (!reading_from_buf)
887
				git_str_put(data_pkt_buf, line, line_len);
888 889 890 891 892
			error = 0;
			goto done;
		}
		else if (error < 0)
			goto done;
893 894 895 896 897 898 899 900 901

		/* Advance in the buffer */
		line_len -= (line_end - line);
		line = line_end;

		error = add_push_report_pkt(push, pkt);

		git_pkt_free(pkt);

902
		if (error < 0 && error != GIT_ITEROVER)
903
			goto done;
904 905
	}

906 907 908 909
	error = 0;

done:
	if (reading_from_buf)
910
		git_str_consume(data_pkt_buf, line_end);
911
	return error;
912 913
}

914
static int parse_report(transport_smart *transport, git_push *push)
915
{
916
	git_pkt *pkt = NULL;
Edward Thomson committed
917
	git_pkt_parse_data pkt_parse_data = { 0 };
918
	const char *line_end = NULL;
919
	int error, recvd;
920
	git_str data_pkt_buf = GIT_STR_INIT;
921 922

	for (;;) {
923
		if (transport->buffer.len > 0)
924
			error = git_pkt_parse_line(&pkt, &line_end,
925 926 927
				   transport->buffer.data,
				   transport->buffer.len,
				   &pkt_parse_data);
928 929 930
		else
			error = GIT_EBUFS;

931 932 933 934
		if (error < 0 && error != GIT_EBUFS) {
			error = -1;
			goto done;
		}
935 936

		if (error == GIT_EBUFS) {
937
			if ((recvd = git_smart__recv(transport)) < 0) {
938 939 940
				error = recvd;
				goto done;
			}
941 942

			if (recvd == 0) {
943
				git_error_set(GIT_ERROR_NET, "could not read report from remote repository");
944 945
				error = GIT_EEOF;
				goto done;
946 947 948 949
			}
			continue;
		}

950
		git_staticstr_consume(&transport->buffer, line_end);
951
		error = 0;
952

953 954 955
		switch (pkt->type) {
			case GIT_PKT_DATA:
				/* This is a sideband packet which contains other packets */
956
				error = add_push_report_sideband_pkt(push, (git_pkt_data *)pkt, &data_pkt_buf);
957 958
				break;
			case GIT_PKT_ERR:
959
				git_error_set(GIT_ERROR_NET, "report-status: Error reported: %s",
960 961 962 963
					((git_pkt_err *)pkt)->error);
				error = -1;
				break;
			case GIT_PKT_PROGRESS:
964
				if (transport->connect_opts.callbacks.sideband_progress) {
965
					git_pkt_progress *p = (git_pkt_progress *) pkt;
966 967 968 969 970 971 972

					if (p->len > INT_MAX) {
						git_error_set(GIT_ERROR_NET, "oversized progress message");
						error = GIT_ERROR;
						goto done;
					}

973
					error = transport->connect_opts.callbacks.sideband_progress(p->data, (int)p->len, transport->connect_opts.callbacks.payload);
974
				}
975 976 977 978
				break;
			default:
				error = add_push_report_pkt(push, pkt);
				break;
979 980
		}

981
		git_pkt_free(pkt);
982

983
		/* add_push_report_pkt returns GIT_ITEROVER when it receives a flush */
984 985 986 987 988
		if (error == GIT_ITEROVER) {
			error = 0;
			if (data_pkt_buf.size > 0) {
				/* If there was data remaining in the pack data buffer,
				 * then the server sent a partial pkt-line */
989
				git_error_set(GIT_ERROR_NET, "incomplete pack data pkt-line");
990 991 992 993
				error = GIT_ERROR;
			}
			goto done;
		}
994

995 996 997
		if (error < 0) {
			goto done;
		}
998
	}
999
done:
1000
	git_str_dispose(&data_pkt_buf);
1001
	return error;
1002 1003
}

1004 1005 1006
static int add_ref_from_push_spec(git_vector *refs, push_spec *push_spec)
{
	git_pkt_ref *added = git__calloc(1, sizeof(git_pkt_ref));
1007
	GIT_ERROR_CHECK_ALLOC(added);
1008 1009 1010

	added->type = GIT_PKT_REF;
	git_oid_cpy(&added->head.oid, &push_spec->loid);
1011
	added->head.name = git__strdup(push_spec->refspec.dst);
1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028

	if (!added->head.name ||
		git_vector_insert(refs, added) < 0) {
		git_pkt_free((git_pkt *)added);
		return -1;
	}

	return 0;
}

static int update_refs_from_report(
	git_vector *refs,
	git_vector *push_specs,
	git_vector *push_report)
{
	git_pkt_ref *ref;
	push_spec *push_spec;
1029
	push_status *push_status;
1030 1031 1032 1033 1034 1035
	size_t i, j, refs_len;
	int cmp;

	/* For each push spec we sent to the server, we should have
	 * gotten back a status packet in the push report */
	if (push_specs->length != push_report->length) {
1036
		git_error_set(GIT_ERROR_NET, "report-status: protocol error");
1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049
		return -1;
	}

	/* We require that push_specs be sorted with push_spec_rref_cmp,
	 * and that push_report be sorted with push_status_ref_cmp */
	git_vector_sort(push_specs);
	git_vector_sort(push_report);

	git_vector_foreach(push_specs, i, push_spec) {
		push_status = git_vector_get(push_report, i);

		/* For each push spec we sent to the server, we should have
		 * gotten back a status packet in the push report which matches */
1050
		if (strcmp(push_spec->refspec.dst, push_status->ref)) {
1051
			git_error_set(GIT_ERROR_NET, "report-status: protocol error");
1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063
			return -1;
		}
	}

	/* We require that refs be sorted with ref_name_cmp */
	git_vector_sort(refs);
	i = j = 0;
	refs_len = refs->length;

	/* Merge join push_specs with refs */
	while (i < push_specs->length && j < refs_len) {
		push_spec = git_vector_get(push_specs, i);
1064
		push_status = git_vector_get(push_report, i);
1065 1066
		ref = git_vector_get(refs, j);

1067
		cmp = strcmp(push_spec->refspec.dst, ref->head.name);
1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086

		/* Iterate appropriately */
		if (cmp <= 0) i++;
		if (cmp >= 0) j++;

		/* Add case */
		if (cmp < 0 &&
			!push_status->msg &&
			add_ref_from_push_spec(refs, push_spec) < 0)
			return -1;

		/* Update case, delete case */
		if (cmp == 0 &&
			!push_status->msg)
			git_oid_cpy(&ref->head.oid, &push_spec->loid);
	}

	for (; i < push_specs->length; i++) {
		push_spec = git_vector_get(push_specs, i);
1087
		push_status = git_vector_get(push_report, i);
1088 1089 1090 1091 1092 1093 1094 1095 1096

		/* Add case */
		if (!push_status->msg &&
			add_ref_from_push_spec(refs, push_spec) < 0)
			return -1;
	}

	/* Remove any refs which we updated to have a zero OID. */
	git_vector_rforeach(refs, i, ref) {
1097
		if (git_oid_is_zero(&ref->head.oid)) {
1098 1099 1100 1101 1102 1103 1104 1105 1106 1107
			git_vector_remove(refs, i);
			git_pkt_free((git_pkt *)ref);
		}
	}

	git_vector_sort(refs);

	return 0;
}

1108 1109 1110 1111
struct push_packbuilder_payload
{
	git_smart_subtransport_stream *stream;
	git_packbuilder *pb;
1112
	git_push_transfer_progress_cb cb;
1113 1114
	void *cb_payload;
	size_t last_bytes;
1115
	uint64_t last_progress_report_time;
1116 1117
};

1118 1119
static int stream_thunk(void *buf, size_t size, void *data)
{
1120 1121 1122 1123 1124 1125 1126
	int error = 0;
	struct push_packbuilder_payload *payload = data;

	if ((error = payload->stream->write(payload->stream, (const char *)buf, size)) < 0)
		return error;

	if (payload->cb) {
1127 1128
		uint64_t current_time = git_time_monotonic();
		uint64_t elapsed = current_time - payload->last_progress_report_time;
1129
		payload->last_bytes += size;
1130

1131
		if (elapsed >= MIN_PROGRESS_UPDATE_INTERVAL) {
1132
			payload->last_progress_report_time = current_time;
1133
			error = payload->cb(payload->pb->nr_written, payload->pb->nr_objects, payload->last_bytes, payload->cb_payload);
1134 1135 1136 1137
		}
	}

	return error;
1138 1139
}

1140
int git_smart__push(git_transport *transport, git_push *push)
1141 1142
{
	transport_smart *t = (transport_smart *)transport;
1143
	git_remote_callbacks *cbs = &t->connect_opts.callbacks;
1144
	struct push_packbuilder_payload packbuilder_payload = {0};
1145
	git_str pktline = GIT_STR_INIT;
1146
	int error = 0, need_pack = 0;
1147 1148
	push_spec *spec;
	unsigned int i;
1149

1150 1151
	packbuilder_payload.pb = push->pb;

1152
	if (cbs && cbs->push_transfer_progress) {
1153 1154
		packbuilder_payload.cb = cbs->push_transfer_progress;
		packbuilder_payload.cb_payload = cbs->payload;
1155 1156
	}

1157 1158 1159
#ifdef PUSH_DEBUG
{
	git_remote_head *head;
1160
	char hex[GIT_OID_SHA1_HEXSIZE+1]; hex[GIT_OID_SHA1_HEXSIZE] = '\0';
1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176

	git_vector_foreach(&push->remote->refs, i, head) {
		git_oid_fmt(hex, &head->oid);
		fprintf(stderr, "%s (%s)\n", hex, head->name);
	}

	git_vector_foreach(&push->specs, i, spec) {
		git_oid_fmt(hex, &spec->roid);
		fprintf(stderr, "%s (%s) -> ", hex, spec->lref);
		git_oid_fmt(hex, &spec->loid);
		fprintf(stderr, "%s (%s)\n", hex, spec->rref ?
			spec->rref : spec->lref);
	}
}
#endif

1177 1178 1179 1180 1181
	/*
	 * Figure out if we need to send a packfile; which is in all
	 * cases except when we only send delete commands
	 */
	git_vector_foreach(&push->specs, i, spec) {
1182
		if (spec->refspec.src && spec->refspec.src[0] != '\0') {
1183 1184 1185 1186 1187
			need_pack = 1;
			break;
		}
	}

1188
	/* prepare pack before sending pack header to avoid timeouts */
1189
	if (need_pack && ((error = git_packbuilder__prepare(push->pb))) < 0)
1190 1191
		goto done;

1192 1193
	if ((error = git_smart__get_push_stream(t, &packbuilder_payload.stream)) < 0 ||
		(error = gen_pktline(&pktline, push)) < 0 ||
1194
		(error = packbuilder_payload.stream->write(packbuilder_payload.stream, git_str_cstr(&pktline), git_str_len(&pktline))) < 0)
1195
		goto done;
1196

1197 1198 1199
	if (need_pack &&
		(error = git_packbuilder_foreach(push->pb, &stream_thunk, &packbuilder_payload)) < 0)
		goto done;
1200 1201 1202 1203 1204

	/* If we sent nothing or the server doesn't support report-status, then
	 * we consider the pack to have been unpacked successfully */
	if (!push->specs.length || !push->report_status)
		push->unpack_ok = 1;
1205
	else if ((error = parse_report(t, push)) < 0)
1206
		goto done;
1207

1208
	/* If progress is being reported write the final report */
1209
	if (cbs && cbs->push_transfer_progress) {
1210
		error = cbs->push_transfer_progress(
1211 1212 1213
					push->pb->nr_written,
					push->pb->nr_objects,
					packbuilder_payload.last_bytes,
1214
					cbs->payload);
1215 1216 1217

		if (error < 0)
			goto done;
1218 1219
	}

1220
	if (push->status.length) {
1221
		error = update_refs_from_report(&t->refs, &push->specs, &push->status);
1222 1223 1224
		if (error < 0)
			goto done;

1225
		error = git_smart__update_heads(t, NULL);
1226
	}
1227

1228
done:
1229
	git_str_dispose(&pktline);
1230 1231
	return error;
}