smart_protocol.c 25.6 KB
Newer Older
1
/*
Edward Thomson committed
2
 * Copyright (C) the libgit2 contributors. All rights reserved.
3 4 5 6
 *
 * This file is part of libgit2, distributed under the GNU GPL v2 with
 * a Linking Exception. For full terms see the included COPYING file.
 */
7 8 9

#include "common.h"

10
#include "git2.h"
11
#include "git2/odb_backend.h"
12

13 14
#include "smart.h"
#include "refs.h"
15
#include "repository.h"
16 17 18
#include "push.h"
#include "pack-objects.h"
#include "remote.h"
19
#include "util.h"
20 21

#define NETWORK_XFER_THRESHOLD (100*1024)
22 23
/* The minimal interval between progress updates (in seconds). */
#define MIN_PROGRESS_UPDATE_INTERVAL 0.5
24

25 26
bool git_smart__ofs_delta_enabled = true;

27 28 29 30 31
int git_smart__store_refs(transport_smart *t, int flushes)
{
	gitno_buffer *buf = &t->buffer;
	git_vector *refs = &t->refs;
	int error, flush = 0, recvd;
32 33
	const char *line_end = NULL;
	git_pkt *pkt = NULL;
Edward Thomson committed
34
	size_t i;
35

36 37 38
	/* Clear existing refs in case git_remote_connect() is called again
	 * after git_remote_disconnect().
	 */
39 40
	git_vector_foreach(refs, i, pkt) {
		git_pkt_free(pkt);
Edward Thomson committed
41
	}
42
	git_vector_clear(refs);
43
	pkt = NULL;
44

45 46
	do {
		if (buf->offset > 0)
47
			error = git_pkt_parse_line(&pkt, &line_end, buf->data, buf->offset);
48 49 50 51
		else
			error = GIT_EBUFS;

		if (error < 0 && error != GIT_EBUFS)
52
			return error;
53 54 55

		if (error == GIT_EBUFS) {
			if ((recvd = gitno_recv(buf)) < 0)
56
				return recvd;
57

58
			if (recvd == 0) {
59
				git_error_set(GIT_ERROR_NET, "early EOF");
60
				return GIT_EEOF;
61 62 63 64 65 66 67
			}

			continue;
		}

		gitno_consume(buf, line_end);
		if (pkt->type == GIT_PKT_ERR) {
68
			git_error_set(GIT_ERROR_NET, "remote error: %s", ((git_pkt_err *)pkt)->error);
69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84
			git__free(pkt);
			return -1;
		}

		if (pkt->type != GIT_PKT_FLUSH && git_vector_insert(refs, pkt) < 0)
			return -1;

		if (pkt->type == GIT_PKT_FLUSH) {
			flush++;
			git_pkt_free(pkt);
		}
	} while (flush < flushes);

	return flush;
}

85 86 87 88 89
static int append_symref(const char **out, git_vector *symrefs, const char *ptr)
{
	int error;
	const char *end;
	git_buf buf = GIT_BUF_INIT;
Jacques Germishuys committed
90
	git_refspec *mapping = NULL;
91 92 93 94 95 96 97 98 99 100 101 102 103 104

	ptr += strlen(GIT_CAP_SYMREF);
	if (*ptr != '=')
		goto on_invalid;

	ptr++;
	if (!(end = strchr(ptr, ' ')) &&
	    !(end = strchr(ptr, '\0')))
		goto on_invalid;

	if ((error = git_buf_put(&buf, ptr, end - ptr)) < 0)
		return error;

	/* symref mapping has refspec format */
Jacques Germishuys committed
105
	mapping = git__calloc(1, sizeof(git_refspec));
106
	GIT_ERROR_CHECK_ALLOC(mapping);
107 108

	error = git_refspec__parse(mapping, git_buf_cstr(&buf), true);
109
	git_buf_dispose(&buf);
110 111 112

	/* if the error isn't OOM, then it's a parse error; let's use a nicer message */
	if (error < 0) {
113
		if (git_error_last()->klass != GIT_ERROR_NOMEMORY)
114 115
			goto on_invalid;

116
		git__free(mapping);
117 118 119 120 121 122 123 124 125 126
		return error;
	}

	if ((error = git_vector_insert(symrefs, mapping)) < 0)
		return error;

	*out = end;
	return 0;

on_invalid:
127
	git_error_set(GIT_ERROR_NET, "remote sent invalid symref");
128
	git_refspec__dispose(mapping);
129
	git__free(mapping);
130 131 132 133
	return -1;
}

int git_smart__detect_caps(git_pkt_ref *pkt, transport_smart_caps *caps, git_vector *symrefs)
134 135 136 137 138
{
	const char *ptr;

	/* No refs or capabilites, odd but not a problem */
	if (pkt == NULL || pkt->capabilities == NULL)
139
		return GIT_ENOTFOUND;
140 141 142 143 144 145

	ptr = pkt->capabilities;
	while (ptr != NULL && *ptr != '\0') {
		if (*ptr == ' ')
			ptr++;

146
		if (git_smart__ofs_delta_enabled && !git__prefixcmp(ptr, GIT_CAP_OFS_DELTA)) {
147 148 149 150 151
			caps->common = caps->ofs_delta = 1;
			ptr += strlen(GIT_CAP_OFS_DELTA);
			continue;
		}

152 153 154 155 156 157 158
		/* Keep multi_ack_detailed before multi_ack */
		if (!git__prefixcmp(ptr, GIT_CAP_MULTI_ACK_DETAILED)) {
			caps->common = caps->multi_ack_detailed = 1;
			ptr += strlen(GIT_CAP_MULTI_ACK_DETAILED);
			continue;
		}

159
		if (!git__prefixcmp(ptr, GIT_CAP_MULTI_ACK)) {
160 161 162 163 164
			caps->common = caps->multi_ack = 1;
			ptr += strlen(GIT_CAP_MULTI_ACK);
			continue;
		}

165
		if (!git__prefixcmp(ptr, GIT_CAP_INCLUDE_TAG)) {
166 167 168 169 170 171
			caps->common = caps->include_tag = 1;
			ptr += strlen(GIT_CAP_INCLUDE_TAG);
			continue;
		}

		/* Keep side-band check after side-band-64k */
172
		if (!git__prefixcmp(ptr, GIT_CAP_SIDE_BAND_64K)) {
173 174 175 176 177
			caps->common = caps->side_band_64k = 1;
			ptr += strlen(GIT_CAP_SIDE_BAND_64K);
			continue;
		}

178
		if (!git__prefixcmp(ptr, GIT_CAP_SIDE_BAND)) {
179 180 181 182 183
			caps->common = caps->side_band = 1;
			ptr += strlen(GIT_CAP_SIDE_BAND);
			continue;
		}

184 185 186 187 188 189
		if (!git__prefixcmp(ptr, GIT_CAP_DELETE_REFS)) {
			caps->common = caps->delete_refs = 1;
			ptr += strlen(GIT_CAP_DELETE_REFS);
			continue;
		}

190 191 192 193 194 195
		if (!git__prefixcmp(ptr, GIT_CAP_THIN_PACK)) {
			caps->common = caps->thin_pack = 1;
			ptr += strlen(GIT_CAP_THIN_PACK);
			continue;
		}

196 197 198 199 200 201 202 203 204
		if (!git__prefixcmp(ptr, GIT_CAP_SYMREF)) {
			int error;

			if ((error = append_symref(&ptr, symrefs, ptr)) < 0)
				return error;

			continue;
		}

205 206 207 208 209 210 211
		/* We don't know this capability, so skip it */
		ptr = strchr(ptr, ' ');
	}

	return 0;
}

212
static int recv_pkt(git_pkt **out_pkt, git_pkt_type *out_type, gitno_buffer *buf)
213 214
{
	const char *ptr = buf->data, *line_end = ptr;
215
	git_pkt *pkt = NULL;
216
	int error = 0, ret;
217 218 219

	do {
		if (buf->offset > 0)
220
			error = git_pkt_parse_line(&pkt, &line_end, ptr, buf->offset);
221 222 223 224 225 226 227
		else
			error = GIT_EBUFS;

		if (error == 0)
			break; /* return the pkt */

		if (error < 0 && error != GIT_EBUFS)
228
			return error;
229

230
		if ((ret = gitno_recv(buf)) < 0) {
231
			return ret;
232
		} else if (ret == 0) {
233
			git_error_set(GIT_ERROR_NET, "early EOF");
234 235
			return GIT_EEOF;
		}
236 237 238
	} while (error);

	gitno_consume(buf, line_end);
239 240 241 242
	if (out_type != NULL)
		*out_type = pkt->type;
	if (out_pkt != NULL)
		*out_pkt = pkt;
243 244 245
	else
		git__free(pkt);

246
	return error;
247 248 249 250 251 252
}

static int store_common(transport_smart *t)
{
	git_pkt *pkt = NULL;
	gitno_buffer *buf = &t->buffer;
253
	int error;
254 255

	do {
256
		if ((error = recv_pkt(&pkt, NULL, buf)) < 0)
257
			return error;
258

259
		if (pkt->type != GIT_PKT_ACK) {
260 261 262 263
			git__free(pkt);
			return 0;
		}

264 265 266 267
		if (git_vector_insert(&t->common, pkt) < 0) {
			git__free(pkt);
			return -1;
		}
268 269 270 271 272 273 274
	} while (1);

	return 0;
}

static int fetch_setup_walk(git_revwalk **out, git_repository *repo)
{
Jacques Germishuys committed
275
	git_revwalk *walk = NULL;
276 277
	git_strarray refs;
	unsigned int i;
278
	git_reference *ref = NULL;
279
	int error;
280

281 282
	if ((error = git_reference_list(&refs, repo)) < 0)
		return error;
283

284 285
	if ((error = git_revwalk_new(&walk, repo)) < 0)
		return error;
286 287 288 289

	git_revwalk_sorting(walk, GIT_SORT_TIME);

	for (i = 0; i < refs.count; ++i) {
290 291 292
		git_reference_free(ref);
		ref = NULL;

293 294 295 296
		/* No tags */
		if (!git__prefixcmp(refs.strings[i], GIT_REFS_TAGS_DIR))
			continue;

297
		if ((error = git_reference_lookup(&ref, repo, refs.strings[i])) < 0)
298 299
			goto on_error;

300
		if (git_reference_type(ref) == GIT_REFERENCE_SYMBOLIC)
301
			continue;
302

303
		if ((error = git_revwalk_push(walk, git_reference_target(ref))) < 0)
304 305 306 307 308 309
			goto on_error;
	}

	*out = walk;

on_error:
310 311
	if (error)
		git_revwalk_free(walk);
312 313
	git_reference_free(ref);
	git_strarray_free(&refs);
314
	return error;
315 316
}

317 318 319
static int wait_while_ack(gitno_buffer *buf)
{
	int error;
320 321
	git_pkt *pkt = NULL;
	git_pkt_ack *ack = NULL;
322 323

	while (1) {
324
		git_pkt_free(pkt);
325

326
		if ((error = recv_pkt(&pkt, NULL, buf)) < 0)
327 328 329 330
			return error;

		if (pkt->type == GIT_PKT_NAK)
			break;
331 332
		if (pkt->type != GIT_PKT_ACK)
			continue;
333

334 335 336 337 338 339
		ack = (git_pkt_ack*)pkt;

		if (ack->status != GIT_ACK_CONTINUE &&
		    ack->status != GIT_ACK_COMMON &&
		    ack->status != GIT_ACK_READY) {
			break;
340 341 342
		}
	}

343
	git_pkt_free(pkt);
344 345 346
	return 0;
}

347
int git_smart__negotiate_fetch(git_transport *transport, git_repository *repo, const git_remote_head * const *wants, size_t count)
348 349 350 351 352
{
	transport_smart *t = (transport_smart *)transport;
	gitno_buffer *buf = &t->buffer;
	git_buf data = GIT_BUF_INIT;
	git_revwalk *walk = NULL;
353 354
	int error = -1;
	git_pkt_type pkt_type;
355 356 357
	unsigned int i;
	git_oid oid;

358
	if ((error = git_pkt_buffer_wants(wants, count, &t->caps, &data)) < 0)
359
		return error;
360

361
	if ((error = fetch_setup_walk(&walk, repo)) < 0)
362
		goto on_error;
363

364
	/*
365 366 367 368
	 * Our support for ACK extensions is simply to parse them. On
	 * the first ACK we will accept that as enough common
	 * objects. We give up if we haven't found an answer in the
	 * first 256 we send.
369 370
	 */
	i = 0;
371
	while (i < 256) {
372 373 374 375 376 377 378 379 380
		error = git_revwalk_next(&oid, walk);

		if (error < 0) {
			if (GIT_ITEROVER == error)
				break;

			goto on_error;
		}

381 382 383 384
		git_pkt_buffer_have(&oid, &data);
		i++;
		if (i % 20 == 0) {
			if (t->cancelled.val) {
385
				git_error_set(GIT_ERROR_NET, "The fetch was cancelled by the user");
386 387 388 389 390
				error = GIT_EUSER;
				goto on_error;
			}

			git_pkt_buffer_flush(&data);
391 392
			if (git_buf_oom(&data)) {
				error = -1;
393
				goto on_error;
394
			}
395

396
			if ((error = git_smart__negotiation_step(&t->parent, data.ptr, data.size)) < 0)
397 398 399
				goto on_error;

			git_buf_clear(&data);
400
			if (t->caps.multi_ack || t->caps.multi_ack_detailed) {
401
				if ((error = store_common(t)) < 0)
402 403
					goto on_error;
			} else {
404
				if ((error = recv_pkt(NULL, &pkt_type, buf)) < 0)
405
					goto on_error;
406 407

				if (pkt_type == GIT_PKT_ACK) {
408 409 410 411
					break;
				} else if (pkt_type == GIT_PKT_NAK) {
					continue;
				} else {
412
					git_error_set(GIT_ERROR_NET, "Unexpected pkt type");
413
					error = -1;
414 415 416 417 418 419 420 421 422 423
					goto on_error;
				}
			}
		}

		if (t->common.length > 0)
			break;

		if (i % 20 == 0 && t->rpc) {
			git_pkt_ack *pkt;
424
			unsigned int j;
425

426
			if ((error = git_pkt_buffer_wants(wants, count, &t->caps, &data)) < 0)
427 428
				goto on_error;

429
			git_vector_foreach(&t->common, j, pkt) {
430 431
				if ((error = git_pkt_buffer_have(&pkt->oid, &data)) < 0)
					goto on_error;
432 433
			}

434 435
			if (git_buf_oom(&data)) {
				error = -1;
436
				goto on_error;
437
			}
438 439 440 441 442 443
		}
	}

	/* Tell the other end that we're done negotiating */
	if (t->rpc && t->common.length > 0) {
		git_pkt_ack *pkt;
444
		unsigned int j;
445

446
		if ((error = git_pkt_buffer_wants(wants, count, &t->caps, &data)) < 0)
447 448
			goto on_error;

449
		git_vector_foreach(&t->common, j, pkt) {
450 451
			if ((error = git_pkt_buffer_have(&pkt->oid, &data)) < 0)
				goto on_error;
452 453
		}

454 455
		if (git_buf_oom(&data)) {
			error = -1;
456
			goto on_error;
457
		}
458 459
	}

460 461 462
	if ((error = git_pkt_buffer_done(&data)) < 0)
		goto on_error;

463
	if (t->cancelled.val) {
464
		git_error_set(GIT_ERROR_NET, "The fetch was cancelled by the user");
465 466 467
		error = GIT_EUSER;
		goto on_error;
	}
468
	if ((error = git_smart__negotiation_step(&t->parent, data.ptr, data.size)) < 0)
469 470
		goto on_error;

471
	git_buf_dispose(&data);
472 473 474
	git_revwalk_free(walk);

	/* Now let's eat up whatever the server gives us */
475
	if (!t->caps.multi_ack && !t->caps.multi_ack_detailed) {
476
		if ((error = recv_pkt(NULL, &pkt_type, buf)) < 0)
477
			return error;
478 479

		if (pkt_type != GIT_PKT_ACK && pkt_type != GIT_PKT_NAK) {
480
			git_error_set(GIT_ERROR_NET, "Unexpected pkt type");
481 482 483
			return -1;
		}
	} else {
484
		error = wait_while_ack(buf);
485 486
	}

487
	return error;
488 489 490

on_error:
	git_revwalk_free(walk);
491
	git_buf_dispose(&data);
492 493 494
	return error;
}

495
static int no_sideband(transport_smart *t, struct git_odb_writepack *writepack, gitno_buffer *buf, git_indexer_progress *stats)
496 497 498 499 500
{
	int recvd;

	do {
		if (t->cancelled.val) {
501
			git_error_set(GIT_ERROR_NET, "The fetch was cancelled by the user");
502 503 504
			return GIT_EUSER;
		}

505
		if (writepack->append(writepack, buf->data, buf->offset, stats) < 0)
506 507 508 509 510
			return -1;

		gitno_consume_n(buf, buf->offset);

		if ((recvd = gitno_recv(buf)) < 0)
511
			return recvd;
512 513
	} while(recvd > 0);

514
	if (writepack->commit(writepack, stats) < 0)
515 516 517 518 519
		return -1;

	return 0;
}

520
struct network_packetsize_payload
521
{
522
	git_indexer_progress_cb callback;
523
	void *payload;
524
	git_indexer_progress *stats;
525
	size_t last_fired_bytes;
526 527
};

528
static int network_packetsize(size_t received, void *payload)
529 530 531 532 533 534 535 536
{
	struct network_packetsize_payload *npp = (struct network_packetsize_payload*)payload;

	/* Accumulate bytes */
	npp->stats->received_bytes += received;

	/* Fire notification if the threshold is reached */
	if ((npp->stats->received_bytes - npp->last_fired_bytes) > NETWORK_XFER_THRESHOLD) {
537
		npp->last_fired_bytes = npp->stats->received_bytes;
538

539
		if (npp->callback(npp->stats, npp->payload))
540
			return GIT_EUSER;
541
	}
542 543

	return 0;
544 545 546 547 548
}

int git_smart__download_pack(
	git_transport *transport,
	git_repository *repo,
549 550
	git_indexer_progress *stats,
	git_indexer_progress_cb progress_cb,
551 552 553 554
	void *progress_payload)
{
	transport_smart *t = (transport_smart *)transport;
	gitno_buffer *buf = &t->buffer;
555 556
	git_odb *odb;
	struct git_odb_writepack *writepack = NULL;
557
	int error = 0;
558 559
	struct network_packetsize_payload npp = {0};

560
	memset(stats, 0, sizeof(git_indexer_progress));
561

562 563
	if (progress_cb) {
		npp.callback = progress_cb;
564 565 566 567
		npp.payload = progress_payload;
		npp.stats = stats;
		t->packetsize_cb = &network_packetsize;
		t->packetsize_payload = &npp;
568 569

		/* We might have something in the buffer already from negotiate_fetch */
570
		if (t->buffer.offset > 0 && !t->cancelled.val)
571
			if (t->packetsize_cb(t->buffer.offset, t->packetsize_payload))
572
				git_atomic_set(&t->cancelled, 1);
573 574
	}

575
	if ((error = git_repository_odb__weakptr(&odb, repo)) < 0 ||
576
		((error = git_odb_write_pack(&writepack, odb, progress_cb, progress_payload)) != 0))
577
		goto done;
578 579 580

	/*
	 * If the remote doesn't support the side-band, we can feed
581
	 * the data directly to the pack writer. Otherwise, we need to
582 583 584
	 * check which one belongs there.
	 */
	if (!t->caps.side_band && !t->caps.side_band_64k) {
585 586
		error = no_sideband(t, writepack, buf, stats);
		goto done;
587 588 589
	}

	do {
590
		git_pkt *pkt = NULL;
591

592
		/* Check cancellation before network call */
593
		if (t->cancelled.val) {
594
			git_error_clear();
595
			error = GIT_EUSER;
596
			goto done;
597 598
		}

599
		if ((error = recv_pkt(&pkt, NULL, buf)) >= 0) {
600 601
			/* Check cancellation after network call */
			if (t->cancelled.val) {
602
				git_error_clear();
603 604 605 606
				error = GIT_EUSER;
			} else if (pkt->type == GIT_PKT_PROGRESS) {
				if (t->progress_cb) {
					git_pkt_progress *p = (git_pkt_progress *) pkt;
607 608 609 610 611 612 613 614

					if (p->len > INT_MAX) {
						git_error_set(GIT_ERROR_NET, "oversized progress message");
						error = GIT_ERROR;
						goto done;
					}

					error = t->progress_cb(p->data, (int)p->len, t->message_cb_payload);
615 616 617
				}
			} else if (pkt->type == GIT_PKT_DATA) {
				git_pkt_data *p = (git_pkt_data *) pkt;
618 619 620

				if (p->len)
					error = writepack->append(writepack, p->data, p->len, stats);
621 622 623 624 625
			} else if (pkt->type == GIT_PKT_FLUSH) {
				/* A flush indicates the end of the packfile */
				git__free(pkt);
				break;
			}
626
		}
627

628 629
		git_pkt_free(pkt);

630 631
		if (error < 0)
			goto done;
632 633 634

	} while (1);

635
	/*
636
	 * Trailing execution of progress_cb, if necessary...
637 638
	 * Only the callback through the npp datastructure currently
	 * updates the last_fired_bytes value. It is possible that
639
	 * progress has already been reported with the correct
640 641 642 643 644
	 * "received_bytes" value, but until (if?) this is unified
	 * then we will report progress again to be sure that the
	 * correct last received_bytes value is reported.
	 */
	if (npp.callback && npp.stats->received_bytes > npp.last_fired_bytes) {
645 646
		error = npp.callback(npp.stats, npp.payload);
		if (error != 0)
647 648 649
			goto done;
	}

650
	error = writepack->commit(writepack, stats);
651

652
done:
653 654
	if (writepack)
		writepack->free(writepack);
655
	if (progress_cb) {
656 657 658
		t->packetsize_cb = NULL;
		t->packetsize_payload = NULL;
	}
659

660 661
	return error;
}
662 663 664 665

static int gen_pktline(git_buf *buf, git_push *push)
{
	push_spec *spec;
666
	size_t i, len;
667
	char old_id[GIT_OID_HEXSZ+1], new_id[GIT_OID_HEXSZ+1];
668

669
	old_id[GIT_OID_HEXSZ] = '\0'; new_id[GIT_OID_HEXSZ] = '\0';
670 671

	git_vector_foreach(&push->specs, i, spec) {
672
		len = 2*GIT_OID_HEXSZ + 7 + strlen(spec->refspec.dst);
673 674

		if (i == 0) {
675
			++len; /* '\0' */
676
			if (push->report_status)
677 678
				len += strlen(GIT_CAP_REPORT_STATUS) + 1;
			len += strlen(GIT_CAP_SIDE_BAND_64K) + 1;
679 680
		}

681 682
		git_oid_fmt(old_id, &spec->roid);
		git_oid_fmt(new_id, &spec->loid);
683

684
		git_buf_printf(buf, "%04"PRIxZ"%s %s %s", len, old_id, new_id, spec->refspec.dst);
685 686 687

		if (i == 0) {
			git_buf_putc(buf, '\0');
688 689 690
			/* Core git always starts their capabilities string with a space */
			if (push->report_status) {
				git_buf_putc(buf, ' ');
691
				git_buf_printf(buf, GIT_CAP_REPORT_STATUS);
692 693 694
			}
			git_buf_putc(buf, ' ');
			git_buf_printf(buf, GIT_CAP_SIDE_BAND_64K);
695 696 697 698
		}

		git_buf_putc(buf, '\n');
	}
699

700 701 702 703
	git_buf_puts(buf, "0000");
	return git_buf_oom(buf) ? -1 : 0;
}

704 705 706 707 708 709
static int add_push_report_pkt(git_push *push, git_pkt *pkt)
{
	push_status *status;

	switch (pkt->type) {
		case GIT_PKT_OK:
710
			status = git__calloc(1, sizeof(push_status));
711
			GIT_ERROR_CHECK_ALLOC(status);
712 713 714 715 716 717 718 719 720
			status->msg = NULL;
			status->ref = git__strdup(((git_pkt_ok *)pkt)->ref);
			if (!status->ref ||
				git_vector_insert(&push->status, status) < 0) {
				git_push_status_free(status);
				return -1;
			}
			break;
		case GIT_PKT_NG:
721
			status = git__calloc(1, sizeof(push_status));
722
			GIT_ERROR_CHECK_ALLOC(status);
723 724 725 726 727 728 729 730 731 732 733 734 735 736
			status->ref = git__strdup(((git_pkt_ng *)pkt)->ref);
			status->msg = git__strdup(((git_pkt_ng *)pkt)->msg);
			if (!status->ref || !status->msg ||
				git_vector_insert(&push->status, status) < 0) {
				git_push_status_free(status);
				return -1;
			}
			break;
		case GIT_PKT_UNPACK:
			push->unpack_ok = ((git_pkt_unpack *)pkt)->unpack_ok;
			break;
		case GIT_PKT_FLUSH:
			return GIT_ITEROVER;
		default:
737
			git_error_set(GIT_ERROR_NET, "report-status: protocol error");
738 739 740 741 742 743
			return -1;
	}

	return 0;
}

744
static int add_push_report_sideband_pkt(git_push *push, git_pkt_data *data_pkt, git_buf *data_pkt_buf)
745 746
{
	git_pkt *pkt;
747
	const char *line, *line_end = NULL;
748
	size_t line_len;
749
	int error;
750 751 752 753 754 755 756 757 758 759 760 761 762
	int reading_from_buf = data_pkt_buf->size > 0;

	if (reading_from_buf) {
		/* We had an existing partial packet, so add the new
		 * packet to the buffer and parse the whole thing */
		git_buf_put(data_pkt_buf, data_pkt->data, data_pkt->len);
		line = data_pkt_buf->ptr;
		line_len = data_pkt_buf->size;
	}
	else {
		line = data_pkt->data;
		line_len = data_pkt->len;
	}
763 764

	while (line_len > 0) {
765
		error = git_pkt_parse_line(&pkt, &line_end, line, line_len);
766

767 768 769 770 771 772 773 774 775 776
		if (error == GIT_EBUFS) {
			/* Buffer the data when the inner packet is split
			 * across multiple sideband packets */
			if (!reading_from_buf)
				git_buf_put(data_pkt_buf, line, line_len);
			error = 0;
			goto done;
		}
		else if (error < 0)
			goto done;
777 778 779 780 781 782 783 784 785

		/* Advance in the buffer */
		line_len -= (line_end - line);
		line = line_end;

		error = add_push_report_pkt(push, pkt);

		git_pkt_free(pkt);

786
		if (error < 0 && error != GIT_ITEROVER)
787
			goto done;
788 789
	}

790 791 792 793 794 795
	error = 0;

done:
	if (reading_from_buf)
		git_buf_consume(data_pkt_buf, line_end);
	return error;
796 797
}

798
static int parse_report(transport_smart *transport, git_push *push)
799
{
800 801
	git_pkt *pkt = NULL;
	const char *line_end = NULL;
802
	gitno_buffer *buf = &transport->buffer;
803
	int error, recvd;
804
	git_buf data_pkt_buf = GIT_BUF_INIT;
805 806 807

	for (;;) {
		if (buf->offset > 0)
808 809
			error = git_pkt_parse_line(&pkt, &line_end,
						   buf->data, buf->offset);
810 811 812
		else
			error = GIT_EBUFS;

813 814 815 816
		if (error < 0 && error != GIT_EBUFS) {
			error = -1;
			goto done;
		}
817 818

		if (error == GIT_EBUFS) {
819 820 821 822
			if ((recvd = gitno_recv(buf)) < 0) {
				error = recvd;
				goto done;
			}
823 824

			if (recvd == 0) {
825
				git_error_set(GIT_ERROR_NET, "early EOF");
826 827
				error = GIT_EEOF;
				goto done;
828 829 830 831 832 833
			}
			continue;
		}

		gitno_consume(buf, line_end);

834
		error = 0;
835

836 837 838
		switch (pkt->type) {
			case GIT_PKT_DATA:
				/* This is a sideband packet which contains other packets */
839
				error = add_push_report_sideband_pkt(push, (git_pkt_data *)pkt, &data_pkt_buf);
840 841
				break;
			case GIT_PKT_ERR:
842
				git_error_set(GIT_ERROR_NET, "report-status: Error reported: %s",
843 844 845 846
					((git_pkt_err *)pkt)->error);
				error = -1;
				break;
			case GIT_PKT_PROGRESS:
847 848
				if (transport->progress_cb) {
					git_pkt_progress *p = (git_pkt_progress *) pkt;
849 850 851 852 853 854 855 856

					if (p->len > INT_MAX) {
						git_error_set(GIT_ERROR_NET, "oversized progress message");
						error = GIT_ERROR;
						goto done;
					}

					error = transport->progress_cb(p->data, (int)p->len, transport->message_cb_payload);
857
				}
858 859 860 861
				break;
			default:
				error = add_push_report_pkt(push, pkt);
				break;
862 863
		}

864
		git_pkt_free(pkt);
865

866
		/* add_push_report_pkt returns GIT_ITEROVER when it receives a flush */
867 868 869 870 871
		if (error == GIT_ITEROVER) {
			error = 0;
			if (data_pkt_buf.size > 0) {
				/* If there was data remaining in the pack data buffer,
				 * then the server sent a partial pkt-line */
872
				git_error_set(GIT_ERROR_NET, "Incomplete pack data pkt-line");
873 874 875 876
				error = GIT_ERROR;
			}
			goto done;
		}
877

878 879 880
		if (error < 0) {
			goto done;
		}
881
	}
882
done:
883
	git_buf_dispose(&data_pkt_buf);
884
	return error;
885 886
}

887 888 889
static int add_ref_from_push_spec(git_vector *refs, push_spec *push_spec)
{
	git_pkt_ref *added = git__calloc(1, sizeof(git_pkt_ref));
890
	GIT_ERROR_CHECK_ALLOC(added);
891 892 893

	added->type = GIT_PKT_REF;
	git_oid_cpy(&added->head.oid, &push_spec->loid);
894
	added->head.name = git__strdup(push_spec->refspec.dst);
895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911

	if (!added->head.name ||
		git_vector_insert(refs, added) < 0) {
		git_pkt_free((git_pkt *)added);
		return -1;
	}

	return 0;
}

static int update_refs_from_report(
	git_vector *refs,
	git_vector *push_specs,
	git_vector *push_report)
{
	git_pkt_ref *ref;
	push_spec *push_spec;
912
	push_status *push_status;
913 914 915 916 917 918
	size_t i, j, refs_len;
	int cmp;

	/* For each push spec we sent to the server, we should have
	 * gotten back a status packet in the push report */
	if (push_specs->length != push_report->length) {
919
		git_error_set(GIT_ERROR_NET, "report-status: protocol error");
920 921 922 923 924 925 926 927 928 929 930 931 932
		return -1;
	}

	/* We require that push_specs be sorted with push_spec_rref_cmp,
	 * and that push_report be sorted with push_status_ref_cmp */
	git_vector_sort(push_specs);
	git_vector_sort(push_report);

	git_vector_foreach(push_specs, i, push_spec) {
		push_status = git_vector_get(push_report, i);

		/* For each push spec we sent to the server, we should have
		 * gotten back a status packet in the push report which matches */
933
		if (strcmp(push_spec->refspec.dst, push_status->ref)) {
934
			git_error_set(GIT_ERROR_NET, "report-status: protocol error");
935 936 937 938 939 940 941 942 943 944 945 946
			return -1;
		}
	}

	/* We require that refs be sorted with ref_name_cmp */
	git_vector_sort(refs);
	i = j = 0;
	refs_len = refs->length;

	/* Merge join push_specs with refs */
	while (i < push_specs->length && j < refs_len) {
		push_spec = git_vector_get(push_specs, i);
947
		push_status = git_vector_get(push_report, i);
948 949
		ref = git_vector_get(refs, j);

950
		cmp = strcmp(push_spec->refspec.dst, ref->head.name);
951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969

		/* Iterate appropriately */
		if (cmp <= 0) i++;
		if (cmp >= 0) j++;

		/* Add case */
		if (cmp < 0 &&
			!push_status->msg &&
			add_ref_from_push_spec(refs, push_spec) < 0)
			return -1;

		/* Update case, delete case */
		if (cmp == 0 &&
			!push_status->msg)
			git_oid_cpy(&ref->head.oid, &push_spec->loid);
	}

	for (; i < push_specs->length; i++) {
		push_spec = git_vector_get(push_specs, i);
970
		push_status = git_vector_get(push_report, i);
971 972 973 974 975 976 977 978 979

		/* Add case */
		if (!push_status->msg &&
			add_ref_from_push_spec(refs, push_spec) < 0)
			return -1;
	}

	/* Remove any refs which we updated to have a zero OID. */
	git_vector_rforeach(refs, i, ref) {
980
		if (git_oid_is_zero(&ref->head.oid)) {
981 982 983 984 985 986 987 988 989 990
			git_vector_remove(refs, i);
			git_pkt_free((git_pkt *)ref);
		}
	}

	git_vector_sort(refs);

	return 0;
}

991 992 993 994
struct push_packbuilder_payload
{
	git_smart_subtransport_stream *stream;
	git_packbuilder *pb;
995
	git_push_transfer_progress_cb cb;
996 997 998 999 1000
	void *cb_payload;
	size_t last_bytes;
	double last_progress_report_time;
};

1001 1002
static int stream_thunk(void *buf, size_t size, void *data)
{
1003 1004 1005 1006 1007 1008 1009 1010 1011
	int error = 0;
	struct push_packbuilder_payload *payload = data;

	if ((error = payload->stream->write(payload->stream, (const char *)buf, size)) < 0)
		return error;

	if (payload->cb) {
		double current_time = git__timer();
		payload->last_bytes += size;
1012

1013 1014
		if ((current_time - payload->last_progress_report_time) >= MIN_PROGRESS_UPDATE_INTERVAL) {
			payload->last_progress_report_time = current_time;
1015
			error = payload->cb(payload->pb->nr_written, payload->pb->nr_objects, payload->last_bytes, payload->cb_payload);
1016 1017 1018 1019
		}
	}

	return error;
1020 1021
}

1022
int git_smart__push(git_transport *transport, git_push *push, const git_remote_callbacks *cbs)
1023 1024
{
	transport_smart *t = (transport_smart *)transport;
1025
	struct push_packbuilder_payload packbuilder_payload = {0};
1026
	git_buf pktline = GIT_BUF_INIT;
1027
	int error = 0, need_pack = 0;
1028 1029
	push_spec *spec;
	unsigned int i;
1030

1031 1032
	packbuilder_payload.pb = push->pb;

1033
	if (cbs && cbs->push_transfer_progress) {
1034 1035
		packbuilder_payload.cb = cbs->push_transfer_progress;
		packbuilder_payload.cb_payload = cbs->payload;
1036 1037
	}

1038 1039 1040
#ifdef PUSH_DEBUG
{
	git_remote_head *head;
1041
	char hex[GIT_OID_HEXSZ+1]; hex[GIT_OID_HEXSZ] = '\0';
1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057

	git_vector_foreach(&push->remote->refs, i, head) {
		git_oid_fmt(hex, &head->oid);
		fprintf(stderr, "%s (%s)\n", hex, head->name);
	}

	git_vector_foreach(&push->specs, i, spec) {
		git_oid_fmt(hex, &spec->roid);
		fprintf(stderr, "%s (%s) -> ", hex, spec->lref);
		git_oid_fmt(hex, &spec->loid);
		fprintf(stderr, "%s (%s)\n", hex, spec->rref ?
			spec->rref : spec->lref);
	}
}
#endif

1058 1059 1060 1061 1062
	/*
	 * Figure out if we need to send a packfile; which is in all
	 * cases except when we only send delete commands
	 */
	git_vector_foreach(&push->specs, i, spec) {
1063
		if (spec->refspec.src && spec->refspec.src[0] != '\0') {
1064 1065 1066 1067 1068
			need_pack = 1;
			break;
		}
	}

1069 1070 1071 1072
	if ((error = git_smart__get_push_stream(t, &packbuilder_payload.stream)) < 0 ||
		(error = gen_pktline(&pktline, push)) < 0 ||
		(error = packbuilder_payload.stream->write(packbuilder_payload.stream, git_buf_cstr(&pktline), git_buf_len(&pktline))) < 0)
		goto done;
1073

1074 1075 1076
	if (need_pack &&
		(error = git_packbuilder_foreach(push->pb, &stream_thunk, &packbuilder_payload)) < 0)
		goto done;
1077 1078 1079 1080 1081

	/* If we sent nothing or the server doesn't support report-status, then
	 * we consider the pack to have been unpacked successfully */
	if (!push->specs.length || !push->report_status)
		push->unpack_ok = 1;
1082
	else if ((error = parse_report(t, push)) < 0)
1083
		goto done;
1084

1085
	/* If progress is being reported write the final report */
1086
	if (cbs && cbs->push_transfer_progress) {
1087
		error = cbs->push_transfer_progress(
1088 1089 1090
					push->pb->nr_written,
					push->pb->nr_objects,
					packbuilder_payload.last_bytes,
1091
					cbs->payload);
1092 1093 1094

		if (error < 0)
			goto done;
1095 1096
	}

1097
	if (push->status.length) {
1098
		error = update_refs_from_report(&t->refs, &push->specs, &push->status);
1099 1100 1101
		if (error < 0)
			goto done;

1102
		error = git_smart__update_heads(t, NULL);
1103
	}
1104

1105
done:
1106
	git_buf_dispose(&pktline);
1107 1108
	return error;
}