Merge pull request #840 from carlosmn/remote-unify

Unify the transport code

examples/network/fetch.c

@@ -40,9 +40,8 @@ exit:
 	pthread_exit(&data->ret);
 }
 
-int update_cb(const char *refname, const git_oid *a, const git_oid *b, void *data)
+static int update_cb(const char *refname, const git_oid *a, const git_oid *b, void *data)
 {
-	const char *action;
 	char a_str[GIT_OID_HEXSZ+1], b_str[GIT_OID_HEXSZ+1];
 
 	git_oid_fmt(b_str, b);
@@ -68,6 +67,7 @@ int fetch(git_repository *repo, int argc, char **argv)
 	struct dl_data data;
 	git_remote_callbacks callbacks;
 
+	argc = argc;
 	// Figure out whether it's a named remote or a URL
 	printf("Fetching %s\n", argv[1]);
 	if (git_remote_load(&remote, repo, argv[1]) < 0) {
@@ -96,10 +96,14 @@ int fetch(git_repository *repo, int argc, char **argv)
 	// the download rate.
 	do {
 		usleep(10000);
-		printf("\rReceived %d/%d objects in %d bytes", stats.processed, stats.total, bytes);
+		printf("\rReceived %d/%d objects in %zu bytes", stats.processed, stats.total, bytes);
 	} while (!data.finished);
 
-	printf("\rReceived %d/%d objects in %d bytes\n", stats.processed, stats.total, bytes);
+	if (data.ret < 0)
+		goto on_error;
+
+	pthread_join(worker, NULL);
+	printf("\rReceived %d/%d objects in %zu bytes\n", stats.processed, stats.total, bytes);
 
 	// Disconnect the underlying connection to prevent from idling.
 	git_remote_disconnect(remote);

examples/network/git2.c

 #include <stdlib.h>
 #include <stdio.h>
+#include <string.h>
 
 #include "common.h"
 
@@ -16,7 +17,7 @@ struct {
 	{ NULL, NULL}
 };
 
-int run_command(git_cb fn, int argc, char **argv)
+static int run_command(git_cb fn, int argc, char **argv)
 {
 	int error;
 	git_repository *repo;
@@ -45,7 +46,7 @@ int run_command(git_cb fn, int argc, char **argv)
 
 int main(int argc, char **argv)
 {
-	int i, error;
+	int i;
 
 	if (argc < 2) {
 		fprintf(stderr, "usage: %s <cmd> [repo]\n", argv[0]);

examples/network/index-pack.c

@@ -2,13 +2,20 @@
 #include <stdlib.h>
 #include <stdio.h>
 #include <string.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <unistd.h>
 #include "common.h"
 
 // This could be run in the main loop whilst the application waits for
 // the indexing to finish in a worker thread
-int index_cb(const git_indexer_stats *stats, void *data)
+static int index_cb(const git_indexer_stats *stats, void *data)
 {
+	data = data;
 	printf("\rProcessing %d of %d", stats->processed, stats->total);
+
+	return 0;
 }
 
 int index_pack(git_repository *repo, int argc, char **argv)
@@ -20,6 +27,7 @@ int index_pack(git_repository *repo, int argc, char **argv)
 	ssize_t read_bytes;
 	char buf[512];
 
+	repo = repo;
 	if (argc < 2) {
 		fprintf(stderr, "I need a packfile\n");
 		return EXIT_FAILURE;
@@ -43,7 +51,7 @@ int index_pack(git_repository *repo, int argc, char **argv)
 		if ((error = git_indexer_stream_add(idx, buf, read_bytes, &stats)) < 0)
 			goto cleanup;
 
-		printf("\rIndexing %d of %d", stats.processed, stats.total);
+		index_cb(&stats, NULL);
 	} while (read_bytes > 0);
 
 	if (read_bytes < 0) {
@@ -65,38 +73,3 @@ int index_pack(git_repository *repo, int argc, char **argv)
 	git_indexer_stream_free(idx);
 	return error;
 }
-
-int index_pack_old(git_repository *repo, int argc, char **argv)
-{
-	git_indexer *indexer;
-	git_indexer_stats stats;
-	int error;
-	char hash[GIT_OID_HEXSZ + 1] = {0};
-
-	if (argc < 2) {
-		fprintf(stderr, "I need a packfile\n");
-		return EXIT_FAILURE;
-	}
-
-	// Create a new indexer
-	error = git_indexer_new(&indexer, argv[1]);
-	if (error < 0)
-		return error;
-
-	// Index the packfile. This function can take a very long time and
-	// should be run in a worker thread.
-	error = git_indexer_run(indexer, &stats);
-	if (error < 0)
-		return error;
-
-	// Write the information out to an index file
-	error = git_indexer_write(indexer);
-
-	// Get the packfile's hash (which should become it's filename)
-	git_oid_fmt(hash, git_indexer_hash(indexer));
-	puts(hash);
-
-	git_indexer_free(indexer);
-
-	return 0;
-}

examples/network/ls-remote.c

@@ -7,12 +7,14 @@
 static int show_ref__cb(git_remote_head *head, void *payload)
 {
 	char oid[GIT_OID_HEXSZ + 1] = {0};
+
+	payload = payload;
 	git_oid_fmt(oid, &head->oid);
 	printf("%s\t%s\n", oid, head->name);
 	return 0;
 }
 
-int use_unnamed(git_repository *repo, const char *url)
+static int use_unnamed(git_repository *repo, const char *url)
 {
 	git_remote *remote = NULL;
 	int error;
@@ -37,7 +39,7 @@ cleanup:
 	return error;
 }
 
-int use_remote(git_repository *repo, char *name)
+static int use_remote(git_repository *repo, char *name)
 {
 	git_remote *remote = NULL;
 	int error;
@@ -63,8 +65,9 @@ cleanup:
 
 int ls_remote(git_repository *repo, int argc, char **argv)
 {
-	int error, i;
+	int error;
 
+	argc = argc;
 	/* If there's a ':' in the name, assume it's an URL */
 	if (strchr(argv[1], ':') != NULL) {
 		error = use_unnamed(repo, argv[1]);

src/common.h

@@ -59,4 +59,7 @@ void giterr_set_regex(const regex_t *regex, int error_code);
 
 #include "util.h"
 
+typedef struct git_transport git_transport;
+typedef struct gitno_buffer gitno_buffer;
+
 #endif /* INCLUDE_common_h__ */

src/fetch.c

@@ -18,6 +18,7 @@
 #include "pack.h"
 #include "fetch.h"
 #include "netops.h"
+#include "pkt.h"
 
 struct filter_payload {
 	git_remote *remote;
@@ -70,7 +71,62 @@ static int filter_wants(git_remote *remote)
 	if (git_repository_odb__weakptr(&p.odb, remote->repo) < 0)
 		return -1;
 
-	return remote->transport->ls(remote->transport, &filter_ref__cb, &p);
+	return git_remote_ls(remote, filter_ref__cb, &p);
+}
+
+/* Wait until we get an ack from the */
+static int recv_pkt(git_pkt **out, gitno_buffer *buf)
+{
+	const char *ptr = buf->data, *line_end = ptr;
+	git_pkt *pkt;
+	int pkt_type, error = 0, ret;
+
+	do {
+		if (buf->offset > 0)
+			error = git_pkt_parse_line(&pkt, ptr, &line_end, buf->offset);
+		else
+			error = GIT_EBUFS;
+
+		if (error == 0)
+			break; /* return the pkt */
+
+		if (error < 0 && error != GIT_EBUFS)
+			return -1;
+
+		if ((ret = gitno_recv(buf)) < 0)
+			return -1;
+	} while (error);
+
+	gitno_consume(buf, line_end);
+	pkt_type = pkt->type;
+	if (out != NULL)
+		*out = pkt;
+	else
+		git__free(pkt);
+
+	return pkt_type;
+}
+
+static int store_common(git_transport *t)
+{
+	git_pkt *pkt = NULL;
+	gitno_buffer *buf = &t->buffer;
+
+	do {
+		if (recv_pkt(&pkt, buf) < 0)
+			return -1;
+
+		if (pkt->type == GIT_PKT_ACK) {
+			if (git_vector_insert(&t->common, pkt) < 0)
+				return -1;
+		} else {
+			git__free(pkt);
+			return 0;
+		}
+
+	} while (1);
+
+	return 0;
 }
 
 /*
@@ -81,6 +137,12 @@ static int filter_wants(git_remote *remote)
 int git_fetch_negotiate(git_remote *remote)
 {
 	git_transport *t = remote->transport;
+	gitno_buffer *buf = &t->buffer;
+	git_buf data = GIT_BUF_INIT;
+	git_revwalk *walk = NULL;
+	int error, pkt_type;
+	unsigned int i;
+	git_oid oid;
 
 	if (filter_wants(remote) < 0) {
 		giterr_set(GITERR_NET, "Failed to filter the reference list for wants");
@@ -92,60 +154,174 @@ int git_fetch_negotiate(git_remote *remote)
 		return 0;
 
 	/*
-	 * Now we have everything set up so we can start tell the server
-	 * what we want and what we have.
+	 * Now we have everything set up so we can start tell the
+	 * server what we want and what we have. Call the function if
+	 * the transport has its own logic. This is transitional and
+	 * will be removed once this function can support git and http.
 	 */
-	return t->negotiate_fetch(t, remote->repo, &remote->refs);
+	if (t->own_logic)
+		return t->negotiate_fetch(t, remote->repo, &remote->refs);
+
+	/* No own logic, do our thing */
+	if (git_pkt_buffer_wants(&remote->refs, &t->caps, &data) < 0)
+		return -1;
+
+	if (git_fetch_setup_walk(&walk, remote->repo) < 0)
+		goto on_error;
+	/*
+	 * We don't support any kind of ACK extensions, so the negotiation
+	 * boils down to sending what we have and listening for an ACK
+	 * every once in a while.
+	 */
+	i = 0;
+	while ((error = git_revwalk_next(&oid, walk)) == 0) {
+		git_pkt_buffer_have(&oid, &data);
+		i++;
+		if (i % 20 == 0) {
+			git_pkt_buffer_flush(&data);
+			if (git_buf_oom(&data))
+				goto on_error;
+
+			if (t->negotiation_step(t, data.ptr, data.size) < 0)
+				goto on_error;
+
+			git_buf_clear(&data);
+			if (t->caps.multi_ack) {
+				if (store_common(t) < 0)
+					goto on_error;
+			} else {
+				pkt_type = recv_pkt(NULL, buf);
+
+				if (pkt_type == GIT_PKT_ACK) {
+					break;
+				} else if (pkt_type == GIT_PKT_NAK) {
+					continue;
+				} else {
+					giterr_set(GITERR_NET, "Unexpected pkt type");
+					goto on_error;
+				}
+			}
+		}
+
+		if (t->common.length > 0)
+			break;
+
+		if (i % 20 == 0 && t->rpc) {
+			git_pkt_ack *pkt;
+			unsigned int i;
+
+			if (git_pkt_buffer_wants(&remote->refs, &t->caps, &data) < 0)
+				goto on_error;
+
+			git_vector_foreach(&t->common, i, pkt) {
+				git_pkt_buffer_have(&pkt->oid, &data);
+			}
+
+			if (git_buf_oom(&data))
+				goto on_error;
+		}
+	}
+
+	if (error < 0 && error != GIT_REVWALKOVER)
+		goto on_error;
+
+	/* Tell the other end that we're done negotiating */
+	if (t->rpc && t->common.length > 0) {
+		git_pkt_ack *pkt;
+		unsigned int i;
+
+		if (git_pkt_buffer_wants(&remote->refs, &t->caps, &data) < 0)
+			goto on_error;
+
+		git_vector_foreach(&t->common, i, pkt) {
+			git_pkt_buffer_have(&pkt->oid, &data);
+		}
+
+		if (git_buf_oom(&data))
+			goto on_error;
+	}
+
+	git_pkt_buffer_done(&data);
+	if (t->negotiation_step(t, data.ptr, data.size) < 0)
+		goto on_error;
+
+	git_buf_free(&data);
+	git_revwalk_free(walk);
+
+	/* Now let's eat up whatever the server gives us */
+	if (!t->caps.multi_ack) {
+		pkt_type = recv_pkt(NULL, buf);
+		if (pkt_type != GIT_PKT_ACK && pkt_type != GIT_PKT_NAK) {
+			giterr_set(GITERR_NET, "Unexpected pkt type");
+			return -1;
+		}
+	} else {
+		git_pkt_ack *pkt;
+		do {
+			if (recv_pkt((git_pkt **)&pkt, buf) < 0)
+				return -1;
+
+			if (pkt->type == GIT_PKT_NAK ||
+			    (pkt->type == GIT_PKT_ACK && pkt->status != GIT_ACK_CONTINUE)) {
+				git__free(pkt);
+				break;
+			}
+
+			git__free(pkt);
+		} while (1);
+	}
+
+	return 0;
+
+on_error:
+	git_revwalk_free(walk);
+	git_buf_free(&data);
+	return -1;
 }
 
 int git_fetch_download_pack(git_remote *remote, git_off_t *bytes, git_indexer_stats *stats)
 {
+	git_transport *t = remote->transport;
+
 	if(!remote->need_pack)
 		return 0;
 
-	return remote->transport->download_pack(remote->transport, remote->repo, bytes, stats);
+	if (t->own_logic)
+		return t->download_pack(t, remote->repo, bytes, stats);
+
+	return git_fetch__download_pack(t, remote->repo, bytes, stats);
+
 }
 
 /* Receiving data from a socket and storing it is pretty much the same for git and HTTP */
 int git_fetch__download_pack(
-	const char *buffered,
-	size_t buffered_size,
 	git_transport *t,
 	git_repository *repo,
 	git_off_t *bytes,
 	git_indexer_stats *stats)
 {
 	int recvd;
-	char buff[1024];
-	gitno_buffer buf;
 	git_buf path = GIT_BUF_INIT;
+	gitno_buffer *buf = &t->buffer;
 	git_indexer_stream *idx = NULL;
 
-	gitno_buffer_setup(t, &buf, buff, sizeof(buff));
-
-	if (memcmp(buffered, "PACK", strlen("PACK"))) {
-		giterr_set(GITERR_NET, "The pack doesn't start with the signature");
-		return -1;
-	}
-
 	if (git_buf_joinpath(&path, git_repository_path(repo), "objects/pack") < 0)
 		return -1;
 
 	if (git_indexer_stream_new(&idx, git_buf_cstr(&path)) < 0)
 		goto on_error;
 
+	git_buf_free(&path);
 	memset(stats, 0, sizeof(git_indexer_stats));
-	if (git_indexer_stream_add(idx, buffered, buffered_size, stats) < 0)
-		goto on_error;
-
-	*bytes = buffered_size;
+	*bytes = 0;
 
 	do {
-		if (git_indexer_stream_add(idx, buf.data, buf.offset, stats) < 0)
+		if (git_indexer_stream_add(idx, buf->data, buf->offset, stats) < 0)
 			goto on_error;
 
-		gitno_consume_n(&buf, buf.offset);
-		if ((recvd = gitno_recv(&buf)) < 0)
+		gitno_consume_n(buf, buf->offset);
+
+		if ((recvd = gitno_recv(buf)) < 0)
 			goto on_error;
 
 		*bytes += recvd;

src/fetch.h

@@ -12,8 +12,7 @@
 int git_fetch_negotiate(git_remote *remote);
 int git_fetch_download_pack(git_remote *remote, git_off_t *bytes, git_indexer_stats *stats);
 
-int git_fetch__download_pack(const char *buffered, size_t buffered_size, git_transport *t,
-			     git_repository *repo, git_off_t *bytes, git_indexer_stats *stats);
+int git_fetch__download_pack(git_transport *t, git_repository *repo, git_off_t *bytes, git_indexer_stats *stats);
 int git_fetch_setup_walk(git_revwalk **out, git_repository *repo);
 
 #endif

src/netops.c

@@ -61,63 +61,67 @@ static int ssl_set_error(gitno_ssl *ssl, int error)
 }
 #endif
 
-void gitno_buffer_setup(git_transport *t, gitno_buffer *buf, char *data, unsigned int len)
+int gitno_recv(gitno_buffer *buf)
 {
-	memset(buf, 0x0, sizeof(gitno_buffer));
-	memset(data, 0x0, len);
-	buf->data = data;
-	buf->len = len;
-	buf->offset = 0;
-	buf->fd = t->socket;
-#ifdef GIT_SSL
-	if (t->encrypt)
-		buf->ssl = &t->ssl;
-#endif
+	return buf->recv(buf);
 }
 
 #ifdef GIT_SSL
-static int ssl_recv(gitno_ssl *ssl, void *data, size_t len)
+static int gitno__recv_ssl(gitno_buffer *buf)
 {
 	int ret;
 
 	do {
-		ret = SSL_read(ssl->ssl, data, len);
-	} while (SSL_get_error(ssl->ssl, ret) == SSL_ERROR_WANT_READ);
+		ret = SSL_read(buf->ssl->ssl, buf->data + buf->offset, buf->len - buf->offset);
+	} while (SSL_get_error(buf->ssl->ssl, ret) == SSL_ERROR_WANT_READ);
 
-	if (ret < 0)
-		return ssl_set_error(ssl, ret);
+	if (ret < 0) {
+		net_set_error("Error receiving socket data");
+		return -1;
+	}
 
+	buf->offset += ret;
 	return ret;
 }
 #endif
 
-int gitno_recv(gitno_buffer *buf)
+int gitno__recv(gitno_buffer *buf)
 {
 	int ret;
 
-#ifdef GIT_SSL
-	if (buf->ssl != NULL) {
-		if ((ret = ssl_recv(buf->ssl, buf->data + buf->offset, buf->len - buf->offset)) < 0)
-			return -1;
-	} else {
-		ret = p_recv(buf->fd, buf->data + buf->offset, buf->len - buf->offset, 0);
-		if (ret < 0) {
-			net_set_error("Error receiving socket data");
-			return -1;
-		}
-	}
-#else
 	ret = p_recv(buf->fd, buf->data + buf->offset, buf->len - buf->offset, 0);
 	if (ret < 0) {
 		net_set_error("Error receiving socket data");
 		return -1;
 	}
-#endif
 
 	buf->offset += ret;
 	return ret;
 }
 
+void gitno_buffer_setup_callback(git_transport *t, gitno_buffer *buf, char *data, unsigned int len, int (*recv)(gitno_buffer *buf), void *cb_data)
+{
+	memset(buf, 0x0, sizeof(gitno_buffer));
+	memset(data, 0x0, len);
+	buf->data = data;
+	buf->len = len;
+	buf->offset = 0;
+	buf->fd = t->socket;
+	buf->recv = recv;
+	buf->cb_data = cb_data;
+}
+
+void gitno_buffer_setup(git_transport *t, gitno_buffer *buf, char *data, unsigned int len)
+{
+#ifdef GIT_SSL
+	if (t->use_ssl) {
+		gitno_buffer_setup_callback(t, buf, data, len, gitno__recv_ssl, NULL);
+		buf->ssl = &t->ssl;
+	} else
+#endif
+		gitno_buffer_setup_callback(t, buf, data, len, gitno__recv, NULL);
+}
+
 /* Consume up to ptr and move the rest of the buffer to the beginning */
 void gitno_consume(gitno_buffer *buf, const char *ptr)
 {
@@ -147,7 +151,7 @@ int gitno_ssl_teardown(git_transport *t)
 	int ret;
 #endif
 
-	if (!t->encrypt)
+	if (!t->use_ssl)
 		return 0;
 
 #ifdef GIT_SSL
@@ -415,7 +419,7 @@ int gitno_connect(git_transport *t, const char *host, const char *port)
 	t->socket = s;
 	p_freeaddrinfo(info);
 
-	if (t->encrypt && ssl_setup(t, host) < 0)
+	if (t->use_ssl && ssl_setup(t, host) < 0)
 		return -1;
 
 	return 0;
@@ -445,7 +449,7 @@ int gitno_send(git_transport *t, const char *msg, size_t len, int flags)
 	size_t off = 0;
 
 #ifdef GIT_SSL
-	if (t->encrypt)
+	if (t->use_ssl)
 		return send_ssl(&t->ssl, msg, len);
 #endif
 

src/netops.h

@@ -8,10 +8,9 @@
 #define INCLUDE_netops_h__
 
 #include "posix.h"
-#include "transport.h"
 #include "common.h"
 
-typedef struct gitno_buffer {
+struct gitno_buffer {
 	char *data;
 	size_t len;
 	size_t offset;
@@ -19,10 +18,14 @@ typedef struct gitno_buffer {
 #ifdef GIT_SSL
 	struct gitno_ssl *ssl;
 #endif
-} gitno_buffer;
+	int (*recv)(gitno_buffer *buffer);
+	void *cb_data;
+};
 
 void gitno_buffer_setup(git_transport *t, gitno_buffer *buf, char *data, unsigned int len);
+void gitno_buffer_setup_callback(git_transport *t, gitno_buffer *buf, char *data, unsigned int len, int (*recv)(gitno_buffer *buf), void *cb_data);
 int gitno_recv(gitno_buffer *buf);
+int gitno__recv(gitno_buffer *buf);
 
 void gitno_consume(gitno_buffer *buf, const char *ptr);
 void gitno_consume_n(gitno_buffer *buf, size_t cons);

src/pkt.c

@@ -42,15 +42,29 @@ static int flush_pkt(git_pkt **out)
 /* the rest of the line will be useful for multi_ack */
 static int ack_pkt(git_pkt **out, const char *line, size_t len)
 {
-	git_pkt *pkt;
+	git_pkt_ack *pkt;
 	GIT_UNUSED(line);
 	GIT_UNUSED(len);
 
-	pkt = git__malloc(sizeof(git_pkt));
+	pkt = git__calloc(1, sizeof(git_pkt_ack));
 	GITERR_CHECK_ALLOC(pkt);
 
 	pkt->type = GIT_PKT_ACK;
-	*out = pkt;
+	line += 3;
+	len -= 3;
+
+	if (len >= GIT_OID_HEXSZ) {
+		git_oid_fromstr(&pkt->oid, line + 1);
+		line += GIT_OID_HEXSZ + 1;
+		len -= GIT_OID_HEXSZ + 1;
+	}
+
+	if (len >= 7) {
+		if (!git__prefixcmp(line + 1, "continue"))
+			pkt->status = GIT_ACK_CONTINUE;
+	}
+
+	*out = (git_pkt *) pkt;
 
 	return 0;
 }
@@ -283,20 +297,28 @@ int git_pkt_buffer_flush(git_buf *buf)
 
 static int buffer_want_with_caps(git_remote_head *head, git_transport_caps *caps, git_buf *buf)
 {
-	char capstr[20];
+	git_buf str = GIT_BUF_INIT;
 	char oid[GIT_OID_HEXSZ +1] = {0};
 	unsigned int len;
 
 	if (caps->ofs_delta)
-		strncpy(capstr, GIT_CAP_OFS_DELTA, sizeof(capstr));
+		git_buf_puts(&str, GIT_CAP_OFS_DELTA " ");
+
+	if (caps->multi_ack)
+		git_buf_puts(&str, GIT_CAP_MULTI_ACK " ");
+
+	if (git_buf_oom(&str))
+		return -1;
 
 	len = (unsigned int)
 		(strlen("XXXXwant ") + GIT_OID_HEXSZ + 1 /* NUL */ +
-		 strlen(capstr) + 1 /* LF */);
+		 git_buf_len(&str) + 1 /* LF */);
 	git_buf_grow(buf, git_buf_len(buf) + len);
-
 	git_oid_fmt(oid, &head->oid);
-	return git_buf_printf(buf, "%04xwant %s %s\n", len, oid, capstr);
+	git_buf_printf(buf, "%04xwant %s %s\n", len, oid, git_buf_cstr(&str));
+	git_buf_free(&str);
+
+	return git_buf_oom(buf);
 }
 
 /*

src/protocol.c

@@ -9,38 +9,36 @@
 #include "pkt.h"
 #include "buffer.h"
 
-int git_protocol_store_refs(git_protocol *p, const char *data, size_t len)
+int git_protocol_store_refs(git_transport *t, int flushes)
 {
-	git_buf *buf = &p->buf;
-	git_vector *refs = p->refs;
-	int error;
-	const char *line_end, *ptr;
-
-	if (len == 0) { /* EOF */
-		if (git_buf_len(buf) != 0) {
-			giterr_set(GITERR_NET, "Unexpected EOF");
-			return p->error = -1;
-		} else {
-			return 0;
-		}
-	}
+	gitno_buffer *buf = &t->buffer;
+	git_vector *refs = &t->refs;
+	int error, flush = 0, recvd;
+	const char *line_end;
+	git_pkt *pkt;
 
-	git_buf_put(buf, data, len);
-	ptr = buf->ptr;
-	while (1) {
-		git_pkt *pkt;
+	do {
+		if (buf->offset > 0)
+			error = git_pkt_parse_line(&pkt, buf->data, &line_end, buf->offset);
+		else
+			error = GIT_EBUFS;
 
-		if (git_buf_len(buf) == 0)
-			return 0;
+		if (error < 0 && error != GIT_EBUFS)
+			return -1;
 
-		error = git_pkt_parse_line(&pkt, ptr, &line_end, git_buf_len(buf));
-		if (error == GIT_EBUFS)
-			return 0; /* Ask for more */
-		if (error < 0)
-			return p->error = -1;
+		if (error == GIT_EBUFS) {
+			if ((recvd = gitno_recv(buf)) < 0)
+				return -1;
 
-		git_buf_consume(buf, line_end);
+			if (recvd == 0 && !flush) {
+				giterr_set(GITERR_NET, "Early EOF");
+				return -1;
+			}
+
+			continue;
+		}
 
+		gitno_consume(buf, line_end);
 		if (pkt->type == GIT_PKT_ERR) {
 			giterr_set(GITERR_NET, "Remote error: %s", ((git_pkt_err *)pkt)->error);
 			git__free(pkt);
@@ -48,10 +46,42 @@ int git_protocol_store_refs(git_protocol *p, const char *data, size_t len)
 		}
 
 		if (git_vector_insert(refs, pkt) < 0)
-			return p->error = -1;
+			return -1;
 
 		if (pkt->type == GIT_PKT_FLUSH)
-			p->flush = 1;
+			flush++;
+	} while (flush < flushes);
+
+	return flush;
+}
+
+int git_protocol_detect_caps(git_pkt_ref *pkt, git_transport_caps *caps)
+{
+	const char *ptr;
+
+	/* No refs or capabilites, odd but not a problem */
+	if (pkt == NULL || pkt->capabilities == NULL)
+		return 0;
+
+	ptr = pkt->capabilities;
+	while (ptr != NULL && *ptr != '\0') {
+		if (*ptr == ' ')
+			ptr++;
+
+		if(!git__prefixcmp(ptr, GIT_CAP_OFS_DELTA)) {
+			caps->common = caps->ofs_delta = 1;
+			ptr += strlen(GIT_CAP_OFS_DELTA);
+			continue;
+		}
+
+		if(!git__prefixcmp(ptr, GIT_CAP_MULTI_ACK)) {
+			caps->common = caps->multi_ack = 1;
+			ptr += strlen(GIT_CAP_MULTI_ACK);
+			continue;
+		}
+
+		/* We don't know this capability, so skip it */
+		ptr = strchr(ptr, ' ');
 	}
 
 	return 0;

src/protocol.h

@@ -9,15 +9,9 @@
 
 #include "transport.h"
 #include "buffer.h"
+#include "pkt.h"
 
-typedef struct {
-	git_transport *transport;
-	git_vector *refs;
-	git_buf buf;
-	int error;
-	unsigned int flush :1;
-} git_protocol;
-
-int git_protocol_store_refs(git_protocol *p, const char *data, size_t len);
+int git_protocol_store_refs(git_transport *t, int flushes);
+int git_protocol_detect_caps(git_pkt_ref *pkt, git_transport_caps *caps);
 
 #endif

src/remote.c

@@ -14,6 +14,7 @@
 #include "remote.h"
 #include "fetch.h"
 #include "refs.h"
+#include "pkt.h"
 
 #include <regex.h>
 
@@ -401,6 +402,10 @@ on_error:
 
 int git_remote_ls(git_remote *remote, git_headlist_cb list_cb, void *payload)
 {
+	git_vector *refs = &remote->transport->refs;
+	unsigned int i;
+	git_pkt *p = NULL;
+
 	assert(remote);
 
 	if (!remote->transport || !remote->transport->connected) {
@@ -408,7 +413,21 @@ int git_remote_ls(git_remote *remote, git_headlist_cb list_cb, void *payload)
 		return -1;
 	}
 
-	return remote->transport->ls(remote->transport, list_cb, payload);
+	git_vector_foreach(refs, i, p) {
+		git_pkt_ref *pkt = NULL;
+
+		if (p->type != GIT_PKT_REF)
+			continue;
+
+		pkt = (git_pkt_ref *)p;
+
+		if (list_cb(&pkt->head, payload) < 0) {
+			giterr_set(GITERR_NET, "User callback returned error");
+			return -1;
+		}
+	}
+
+	return 0;
 }
 
 int git_remote_download(git_remote *remote, git_off_t *bytes, git_indexer_stats *stats)

src/transport.h

@@ -12,6 +12,7 @@
 #include "vector.h"
 #include "posix.h"
 #include "common.h"
+#include "netops.h"
 #ifdef GIT_SSL
 # include <openssl/ssl.h>
 # include <openssl/err.h>
@@ -19,10 +20,12 @@
 
 
 #define GIT_CAP_OFS_DELTA "ofs-delta"
+#define GIT_CAP_MULTI_ACK "multi_ack"
 
 typedef struct git_transport_caps {
 	int common:1,
-		ofs_delta:1;
+		ofs_delta:1,
+		multi_ack: 1;
 } git_transport_caps;
 
 #ifdef GIT_SSL
@@ -70,19 +73,25 @@ struct git_transport {
 	int direction : 1, /* 0 fetch, 1 push */
 		connected : 1,
 		check_cert: 1,
-		encrypt : 1;
+		use_ssl : 1,
+		own_logic: 1, /* transitional */
+		rpc: 1; /* git-speak for the HTTP transport */
 #ifdef GIT_SSL
 	struct gitno_ssl ssl;
 #endif
+	git_vector refs;
+	git_vector common;
+	gitno_buffer buffer;
 	GIT_SOCKET socket;
+	git_transport_caps caps;
 	/**
 	 * Connect and store the remote heads
 	 */
 	int (*connect)(struct git_transport *transport, int dir);
 	/**
-	 * Give a list of references, useful for ls-remote
+	 * Send our side of a negotiation
 	 */
-	int (*ls)(struct git_transport *transport, git_headlist_cb list_cb, void *opaque);
+	int (*negotiation_step)(struct git_transport *transport, void *data, size_t len);
 	/**
 	 * Push the changes over
 	 */
@@ -97,10 +106,6 @@ struct git_transport {
 	 */
 	int (*download_pack)(struct git_transport *transport, git_repository *repo, git_off_t *bytes, git_indexer_stats *stats);
 	/**
-	 * Fetch the changes
-	 */
-	int (*fetch)(struct git_transport *transport);
-	/**
 	 * Close the connection
 	 */
 	int (*close)(struct git_transport *transport);
@@ -124,7 +129,6 @@ int git_transport_dummy(struct git_transport **transport);
 */
 int git_transport_valid_url(const char *url);
 
-typedef struct git_transport git_transport;
 typedef int (*git_transport_cb)(git_transport **transport);
 
 #endif

src/transports/git.c

@@ -24,12 +24,7 @@
 
 typedef struct {
 	git_transport parent;
-	git_protocol proto;
-	git_vector refs;
-	git_remote_head **heads;
-	git_transport_caps caps;
 	char buff[1024];
-	gitno_buffer buf;
 #ifdef GIT_WIN32
 	WSADATA wsd;
 #endif
@@ -127,68 +122,6 @@ on_error:
 }
 
 /*
- * Read from the socket and store the references in the vector
- */
-static int store_refs(transport_git *t)
-{
-	gitno_buffer *buf = &t->buf;
-	int ret = 0;
-
-	while (1) {
-		if ((ret = gitno_recv(buf)) < 0)
-			return -1;
-		if (ret == 0) /* Orderly shutdown, so exit */
-			return 0;
-
-		ret = git_protocol_store_refs(&t->proto, buf->data, buf->offset);
-		if (ret == GIT_EBUFS) {
-			gitno_consume_n(buf, buf->len);
-			continue;
-		}
-
-		if (ret < 0)
-			return ret;
-
-		gitno_consume_n(buf, buf->offset);
-
-		if (t->proto.flush) { /* No more refs */
-			t->proto.flush = 0;
-			return 0;
-		}
-	}
-}
-
-static int detect_caps(transport_git *t)
-{
-	git_vector *refs = &t->refs;
-	git_pkt_ref *pkt;
-	git_transport_caps *caps = &t->caps;
-	const char *ptr;
-
-	pkt = git_vector_get(refs, 0);
-	/* No refs or capabilites, odd but not a problem */
-	if (pkt == NULL || pkt->capabilities == NULL)
-		return 0;
-
-	ptr = pkt->capabilities;
-	while (ptr != NULL && *ptr != '\0') {
-		if (*ptr == ' ')
-			ptr++;
-
-		if(!git__prefixcmp(ptr, GIT_CAP_OFS_DELTA)) {
-			caps->common = caps->ofs_delta = 1;
-			ptr += strlen(GIT_CAP_OFS_DELTA);
-			continue;
-		}
-
-		/* We don't know this capability, so skip it */
-		ptr = strchr(ptr, ' ');
-	}
-
-	return 0;
-}
-
-/*
  * Since this is a network connection, we need to parse and store the
  * pkt-lines at this stage and keep them there.
  */
@@ -202,202 +135,26 @@ static int git_connect(git_transport *transport, int direction)
 	}
 
 	t->parent.direction = direction;
-	if (git_vector_init(&t->refs, 16, NULL) < 0)
-		return -1;
 
 	/* Connect and ask for the refs */
 	if (do_connect(t, transport->url) < 0)
-		goto cleanup;
+		return -1;
 
-	gitno_buffer_setup(transport, &t->buf, t->buff, sizeof(t->buff));
+	gitno_buffer_setup(transport, &transport->buffer, t->buff, sizeof(t->buff));
 
 	t->parent.connected = 1;
-	if (store_refs(t) < 0)
-		goto cleanup;
-
-	if (detect_caps(t) < 0)
-		goto cleanup;
-
-	return 0;
-cleanup:
-	git_vector_free(&t->refs);
-	return -1;
-}
-
-static int git_ls(git_transport *transport, git_headlist_cb list_cb, void *opaque)
-{
-	transport_git *t = (transport_git *) transport;
-	git_vector *refs = &t->refs;
-	unsigned int i;
-	git_pkt *p = NULL;
-
-	git_vector_foreach(refs, i, p) {
-		git_pkt_ref *pkt = NULL;
-
-		if (p->type != GIT_PKT_REF)
-			continue;
-
-		pkt = (git_pkt_ref *)p;
-
-		if (list_cb(&pkt->head, opaque) < 0) {
-			giterr_set(GITERR_NET, "User callback returned error");
-			return -1;
-		}
-	}
-
-	return 0;
-}
-
-/* Wait until we get an ack from the */
-static int recv_pkt(gitno_buffer *buf)
-{
-	const char *ptr = buf->data, *line_end;
-	git_pkt *pkt;
-	int pkt_type, error;
-
-	do {
-		/* Wait for max. 1 second */
-		if ((error = gitno_select_in(buf, 1, 0)) < 0) {
-			return -1;
-		} else if (error == 0) {
-			/*
-			 * Some servers don't respond immediately, so if this
-			 * happens, we keep sending information until it
-			 * answers. Pretend we received a NAK to convince higher
-			 * layers to do so.
-			 */
-			return GIT_PKT_NAK;
-		}
-
-		if ((error = gitno_recv(buf)) < 0)
-			return -1;
-
-		error = git_pkt_parse_line(&pkt, ptr, &line_end, buf->offset);
-		if (error == GIT_EBUFS)
-			continue;
-		if (error < 0)
-			return -1;
-	} while (error);
-
-	gitno_consume(buf, line_end);
-	pkt_type = pkt->type;
-	git__free(pkt);
-
-	return pkt_type;
-}
-
-static int git_negotiate_fetch(git_transport *transport, git_repository *repo, const git_vector *wants)
-{
-	transport_git *t = (transport_git *) transport;
-	git_revwalk *walk;
-	git_oid oid;
-	int error;
-	unsigned int i;
-	git_buf data = GIT_BUF_INIT;
-	gitno_buffer *buf = &t->buf;
-
-	if (git_pkt_buffer_wants(wants, &t->caps, &data) < 0)
+	if (git_protocol_store_refs(transport, 1) < 0)
 		return -1;
 
-	if (git_fetch_setup_walk(&walk, repo) < 0)
-		goto on_error;
-
-	if (gitno_send(transport, data.ptr, data.size, 0) < 0)
-		goto on_error;
-
-	git_buf_clear(&data);
-	/*
-	 * We don't support any kind of ACK extensions, so the negotiation
-	 * boils down to sending what we have and listening for an ACK
-	 * every once in a while.
-	 */
-	i = 0;
-	while ((error = git_revwalk_next(&oid, walk)) == 0) {
-		git_pkt_buffer_have(&oid, &data);
-		i++;
-		if (i % 20 == 0) {
-			int pkt_type;
-
-			git_pkt_buffer_flush(&data);
-			if (git_buf_oom(&data))
-				goto on_error;
-
-			if (gitno_send(transport, data.ptr, data.size, 0) < 0)
-				goto on_error;
-
-			pkt_type = recv_pkt(buf);
-
-			if (pkt_type == GIT_PKT_ACK) {
-				break;
-			} else if (pkt_type == GIT_PKT_NAK) {
-				continue;
-			} else {
-				giterr_set(GITERR_NET, "Unexpected pkt type");
-				goto on_error;
-			}
-
-		}
-	}
-	if (error < 0 && error != GIT_REVWALKOVER)
-		goto on_error;
-
-	/* Tell the other end that we're done negotiating */
-	git_buf_clear(&data);
-	git_pkt_buffer_flush(&data);
-	git_pkt_buffer_done(&data);
-	if (gitno_send(transport, data.ptr, data.size, 0) < 0)
-		goto on_error;
+	if (git_protocol_detect_caps(git_vector_get(&transport->refs, 0), &transport->caps) < 0)
+		return -1;
 
-	git_buf_free(&data);
-	git_revwalk_free(walk);
 	return 0;
-
-on_error:
-	git_buf_free(&data);
-	git_revwalk_free(walk);
-	return -1;
 }
 
-static int git_download_pack(git_transport *transport, git_repository *repo, git_off_t *bytes, git_indexer_stats *stats)
+static int git_negotiation_step(struct git_transport *transport, void *data, size_t len)
 {
-	transport_git *t = (transport_git *) transport;
-	int error = 0, read_bytes;
-	gitno_buffer *buf = &t->buf;
-	git_pkt *pkt;
-	const char *line_end, *ptr;
-
-	/*
-	 * For now, we ignore everything and wait for the pack
-	 */
-	do {
-		ptr = buf->data;
-		/* Whilst we're searching for the pack */
-		while (1) {
-			if (buf->offset == 0) {
-				break;
-			}
-
-			error = git_pkt_parse_line(&pkt, ptr, &line_end, buf->offset);
-			if (error == GIT_EBUFS)
-				break;
-
-			if (error < 0)
-				return error;
-
-			if (pkt->type == GIT_PKT_PACK) {
-				git__free(pkt);
-				return git_fetch__download_pack(buf->data, buf->offset, transport, repo, bytes, stats);
-			}
-
-			/* For now we don't care about anything */
-			git__free(pkt);
-			gitno_consume(buf, line_end);
-		}
-
-		read_bytes = gitno_recv(buf);
-	} while (read_bytes);
-
-	return read_bytes;
+	return gitno_send(transport, data, len, 0);
 }
 
 static int git_close(git_transport *t)
@@ -408,6 +165,7 @@ static int git_close(git_transport *t)
 		return -1;
 	/* Can't do anything if there's an error, so don't bother checking  */
 	gitno_send(t, buf.ptr, buf.size, 0);
+	git_buf_free(&buf);
 
 	if (gitno_close(t->socket) < 0) {
 		giterr_set(GITERR_NET, "Failed to close socket");
@@ -426,17 +184,22 @@ static int git_close(git_transport *t)
 static void git_free(git_transport *transport)
 {
 	transport_git *t = (transport_git *) transport;
-	git_vector *refs = &t->refs;
+	git_vector *refs = &transport->refs;
 	unsigned int i;
 
 	for (i = 0; i < refs->length; ++i) {
 		git_pkt *p = git_vector_get(refs, i);
 		git_pkt_free(p);
 	}
+	git_vector_free(refs);
 
+	refs = &transport->common;
+	for (i = 0; i < refs->length; ++i) {
+		git_pkt *p = git_vector_get(refs, i);
+		git_pkt_free(p);
+	}
 	git_vector_free(refs);
-	git__free(t->heads);
-	git_buf_free(&t->proto.buf);
+
 	git__free(t->parent.url);
 	git__free(t);
 }
@@ -452,15 +215,16 @@ int git_transport_git(git_transport **out)
 	GITERR_CHECK_ALLOC(t);
 
 	memset(t, 0x0, sizeof(transport_git));
+	if (git_vector_init(&t->parent.common, 8, NULL))
+		goto on_error;
+
+	if (git_vector_init(&t->parent.refs, 16, NULL) < 0)
+		goto on_error;
 
 	t->parent.connect = git_connect;
-	t->parent.ls = git_ls;
-	t->parent.negotiate_fetch = git_negotiate_fetch;
-	t->parent.download_pack = git_download_pack;
+	t->parent.negotiation_step = git_negotiation_step;
 	t->parent.close = git_close;
 	t->parent.free = git_free;
-	t->proto.refs = &t->refs;
-	t->proto.transport = (git_transport *) t;
 
 	*out = (git_transport *) t;
 
@@ -474,4 +238,8 @@ int git_transport_git(git_transport **out)
 #endif
 
 	return 0;
+
+on_error:
+	git__free(t);
+	return -1;
 }

src/transports/local.c

@@ -15,11 +15,11 @@
 #include "posix.h"
 #include "path.h"
 #include "buffer.h"
+#include "pkt.h"
 
 typedef struct {
 	git_transport parent;
 	git_repository *repo;
-	git_vector refs;
 } transport_local;
 
 static int add_ref(transport_local *t, const char *name)
@@ -27,19 +27,32 @@ static int add_ref(transport_local *t, const char *name)
 	const char peeled[] = "^{}";
 	git_remote_head *head;
 	git_object *obj = NULL, *target = NULL;
+	git_transport *transport = (git_transport *) t;
 	git_buf buf = GIT_BUF_INIT;
+	git_pkt_ref *pkt;
 
 	head = git__malloc(sizeof(git_remote_head));
 	GITERR_CHECK_ALLOC(head);
+	pkt = git__malloc(sizeof(git_pkt_ref));
+	GITERR_CHECK_ALLOC(pkt);
 
 	head->name = git__strdup(name);
 	GITERR_CHECK_ALLOC(head->name);
 
-	if (git_reference_name_to_oid(&head->oid, t->repo, name) < 0 ||
-		git_vector_insert(&t->refs, head) < 0)
-	{
-		git__free(head->name);
+	if (git_reference_name_to_oid(&head->oid, t->repo, name) < 0) {
 		git__free(head);
+		git__free(pkt->head.name);
+		git__free(pkt);
+	}
+
+	pkt->type = GIT_PKT_REF;
+	memcpy(&pkt->head, head, sizeof(git_remote_head));
+	git__free(head);
+
+	if (git_vector_insert(&transport->refs, pkt) < 0)
+	{
+		git__free(pkt->head.name);
+		git__free(pkt);
 		return -1;
 	}
 
@@ -47,7 +60,7 @@ static int add_ref(transport_local *t, const char *name)
 	if (git__prefixcmp(name, GIT_REFS_TAGS_DIR))
 		return 0;
 
-	if (git_object_lookup(&obj, t->repo, &head->oid, GIT_OBJ_ANY) < 0)
+	if (git_object_lookup(&obj, t->repo, &pkt->head.oid, GIT_OBJ_ANY) < 0)
 		return -1;
 
 	head = NULL;
@@ -66,14 +79,20 @@ static int add_ref(transport_local *t, const char *name)
 
 	head->name = git_buf_detach(&buf);
 
+	pkt = git__malloc(sizeof(git_pkt_ref));
+	GITERR_CHECK_ALLOC(pkt);
+	pkt->type = GIT_PKT_REF;
+
 	if (git_tag_peel(&target, (git_tag *) obj) < 0)
 		goto on_error;
 
 	git_oid_cpy(&head->oid, git_object_id(target));
 	git_object_free(obj);
 	git_object_free(target);
+	memcpy(&pkt->head, head, sizeof(git_remote_head));
+	git__free(head);
 
-	if (git_vector_insert(&t->refs, head) < 0)
+	if (git_vector_insert(&transport->refs, pkt) < 0)
 		return -1;
 
 	return 0;
@@ -88,11 +107,12 @@ static int store_refs(transport_local *t)
 {
 	unsigned int i;
 	git_strarray ref_names = {0};
+	git_transport *transport = (git_transport *) t;
 
 	assert(t);
 
 	if (git_reference_list(&ref_names, t->repo, GIT_REF_LISTALL) < 0 ||
-		git_vector_init(&t->refs, (unsigned int)ref_names.count, NULL) < 0)
+		git_vector_init(&transport->refs, (unsigned int)ref_names.count, NULL) < 0)
 		goto on_error;
 
 	/* Sort the references first */
@@ -111,28 +131,11 @@ static int store_refs(transport_local *t)
 	return 0;
 
 on_error:
-	git_vector_free(&t->refs);
+	git_vector_free(&transport->refs);
 	git_strarray_free(&ref_names);
 	return -1;
 }
 
-static int local_ls(git_transport *transport, git_headlist_cb list_cb, void *payload)
-{
-	transport_local  *t = (transport_local *) transport;
-	git_vector *refs = &t->refs;
-	unsigned int i;
-	git_remote_head *h;
-
-	assert(transport && transport->connected);
-
-	git_vector_foreach(refs, i, h) {
-		if (list_cb(h, payload) < 0)
-			return -1;
-	}
-
-	return 0;
-}
-
 /*
  * Try to open the url as a git directory. The direction doesn't
  * matter in this case because we're calulating the heads ourselves.
@@ -201,14 +204,14 @@ static void local_free(git_transport *transport)
 {
 	unsigned int i;
 	transport_local *t = (transport_local *) transport;
-	git_vector *vec = &t->refs;
-	git_remote_head *h;
+	git_vector *vec = &transport->refs;
+	git_pkt_ref *pkt;
 
 	assert(transport);
 
-	git_vector_foreach (vec, i, h) {
-		git__free(h->name);
-		git__free(h);
+	git_vector_foreach (vec, i, pkt) {
+		git__free(pkt->head.name);
+		git__free(pkt);
 	}
 	git_vector_free(vec);
 
@@ -229,8 +232,8 @@ int git_transport_local(git_transport **out)
 
 	memset(t, 0x0, sizeof(transport_local));
 
+	t->parent.own_logic = 1;
 	t->parent.connect = local_connect;
-	t->parent.ls = local_ls;
 	t->parent.negotiate_fetch = local_negotiate_fetch;
 	t->parent.close = local_close;
 	t->parent.free = local_free;