Merge pull request #641 from carlosmn/networking

More Networking updates

examples/network/Makefile

@@ -2,7 +2,7 @@ default: all
 
 CC = gcc
 CFLAGS += -g
-CFLAGS += -I../../include -L../../ -lgit2
+CFLAGS += -I../../include -L../../ -lgit2 -lpthread
 
 OBJECTS = \
   git2.o \

examples/network/fetch.c

@@ -3,95 +3,108 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <pthread.h>
 
-static int rename_packfile(char *packname, git_indexer *idx)
-{
-  char path[GIT_PATH_MAX], oid[GIT_OID_HEXSZ + 1], *slash;
+struct dl_data {
+	git_remote *remote;
+	git_off_t *bytes;
+	git_indexer_stats *stats;
 	int ret;
+	int finished;
+};
+
+static void *download(void *ptr)
+{
+	struct dl_data *data = (struct dl_data *)ptr;
+
+	// Connect to the remote end specifying that we want to fetch
+	// information from it.
+	if (git_remote_connect(data->remote, GIT_DIR_FETCH) < 0) {
+		data->ret = -1;
+		goto exit;
+	}
+
+	// Download the packfile and index it. This function updates the
+	// amount of received data and the indexer stats which lets you
+	// inform the user about progress.
+	if (git_remote_download(data->remote, data->bytes, data->stats) < 0) {
+		data->ret = -1;
+		goto exit;
+	}
+
+	data->ret = 0;
 
-  strcpy(path, packname);
-  slash = strrchr(path, '/');
-
-  if (!slash)
-	  return GIT_EINVALIDARGS;
-
-  memset(oid, 0x0, sizeof(oid));
-  // The name of the packfile is given by it's hash which you can get
-  // with git_indexer_hash after the index has been written out to
-  // disk. Rename the packfile to its "real" name in the same
-  // directory as it was originally (libgit2 stores it in the folder
-  // where the packs go, so a rename in place is the right thing to do here
-  git_oid_fmt(oid, git_indexer_hash(idx));
-  ret = sprintf(slash + 1, "pack-%s.pack", oid);
-  if(ret < 0)
-	  return GIT_EOSERR;
-
-  printf("Renaming pack to %s\n", path);
-  return rename(packname, path);
+exit:
+	data->finished = 1;
+	pthread_exit(&data->ret);
+}
+
+int update_cb(const char *refname, const git_oid *a, const git_oid *b)
+{
+	const char *action;
+	char a_str[GIT_OID_HEXSZ+1], b_str[GIT_OID_HEXSZ+1];
+
+	git_oid_fmt(b_str, b);
+	b_str[GIT_OID_HEXSZ] = '\0';
+
+	if (git_oid_iszero(a)) {
+		printf("[new]     %.20s %s\n", b_str, refname);
+	} else {
+		git_oid_fmt(a_str, a);
+		a_str[GIT_OID_HEXSZ] = '\0';
+		printf("[updated] %.10s..%.10s %s\n", a_str, b_str, refname);
+	}
+
+	return 0;
 }
 
 int fetch(git_repository *repo, int argc, char **argv)
 {
   git_remote *remote = NULL;
-  git_indexer *idx = NULL;
+  git_off_t bytes = 0;
   git_indexer_stats stats;
-  int error;
-  char *packname = NULL;
+  pthread_t worker;
+  struct dl_data data;
 
-  // Get the remote and connect to it
+  // Figure out whether it's a named remote or a URL
   printf("Fetching %s\n", argv[1]);
-  error = git_remote_new(&remote, repo, argv[1], NULL);
-  if (error < GIT_SUCCESS)
-    return error;
-
-  error = git_remote_connect(remote, GIT_DIR_FETCH);
-  if (error < GIT_SUCCESS)
-    return error;
-
-  // Download the packfile from the server. As we don't know its hash
-  // yet, it will get a temporary filename
-  error = git_remote_download(&packname, remote);
-  if (error < GIT_SUCCESS)
-    return error;
-
-  // No error and a NULL packname means no packfile was needed
-  if (packname != NULL) {
-	  printf("The packname is %s\n", packname);
-
-	  // Create a new instance indexer
-	  error = git_indexer_new(&idx, packname);
-	  if (error < GIT_SUCCESS)
-		  return error;
-
-	  // This should be run in paralel, but it'd be too complicated for the example
-	  error = git_indexer_run(idx, &stats);
-	  if (error < GIT_SUCCESS)
-		  return error;
-
-	  printf("Received %d objects\n", stats.total);
-
-	  // Write the index file. The index will be stored with the
-	  // correct filename
-	  error = git_indexer_write(idx);
-	  if (error < GIT_SUCCESS)
-		  return error;
-
-	  error = rename_packfile(packname, idx);
-	  if (error < GIT_SUCCESS)
-		  return error;
+  if (git_remote_load(&remote, repo, argv[1]) < 0) {
+	  if (git_remote_new(&remote, repo, argv[1], NULL) < 0)
+		  return -1;
   }
 
+  // Set up the information for the background worker thread
+  data.remote = remote;
+  data.bytes = &bytes;
+  data.stats = &stats;
+  data.ret = 0;
+  data.finished = 0;
+  memset(&stats, 0, sizeof(stats));
+
+  pthread_create(&worker, NULL, download, &data);
+
+  // Loop while the worker thread is still running. Here we show processed
+  // and total objects in the pack and the amount of received
+  // data. Most frontends will probably want to show a percentage and
+  // the download rate.
+  do {
+	usleep(10000);
+	printf("\rReceived %d/%d objects in %d bytes", stats.processed, stats.total, bytes);
+  } while (!data.finished);
+  printf("\rReceived %d/%d objects in %d bytes\n", stats.processed, stats.total, bytes);
+
   // Update the references in the remote's namespace to point to the
   // right commits. This may be needed even if there was no packfile
   // to download, which can happen e.g. when the branches have been
   // changed but all the neede objects are available locally.
-  error = git_remote_update_tips(remote);
-  if (error < GIT_SUCCESS)
-    return error;
+  if (git_remote_update_tips(remote, update_cb) < 0)
+	  return -1;
 
-  free(packname);
-  git_indexer_free(idx);
   git_remote_free(remote);
 
-  return GIT_SUCCESS;
+  return 0;
+
+on_error:
+  git_remote_free(remote);
+  return -1;
 }

include/git2/indexer.h

@@ -41,7 +41,7 @@ GIT_EXTERN(int) git_indexer_stream_new(git_indexer_stream **out, const char *git
  * @param size the size of the data
  * @param stats stat storage
  */
-GIT_EXTERN(int) git_indexer_stream_add(git_indexer_stream *idx, void *data, size_t size, git_indexer_stats *stats);
+GIT_EXTERN(int) git_indexer_stream_add(git_indexer_stream *idx, const void *data, size_t size, git_indexer_stats *stats);
 
 /**
  * Finalize the pack and index

include/git2/remote.h

@@ -11,6 +11,7 @@
 #include "repository.h"
 #include "refspec.h"
 #include "net.h"
+#include "indexer.h"
 
 /**
  * @file git2/remote.h
@@ -150,7 +151,7 @@ GIT_EXTERN(int) git_remote_ls(git_remote *remote, git_headlist_cb list_cb, void
  * @param filename where to store the temproray filename
  * @return GIT_SUCCESS or an error code
  */
-GIT_EXTERN(int) git_remote_download(char **filename, git_remote *remote);
+GIT_EXTERN(int) git_remote_download(git_remote *remote, git_off_t *bytes, git_indexer_stats *stats);
 
 /**
  * Check whether the remote is connected
@@ -182,12 +183,10 @@ GIT_EXTERN(void) git_remote_free(git_remote *remote);
 /**
  * Update the tips to the new state
  *
- * Make sure that you only call this once you've successfully indexed
- * or expanded the packfile.
- *
  * @param remote the remote to update
+ * @param cb callback to run on each ref update. 'a' is the old value, 'b' is then new value
  */
-GIT_EXTERN(int) git_remote_update_tips(git_remote *remote);
+GIT_EXTERN(int) git_remote_update_tips(git_remote *remote, int (*cb)(const char *refname, const git_oid *a, const git_oid *b));
 
 /**
  * Return whether a string is a valid remote URL

src/fetch.c

@@ -9,6 +9,7 @@
 #include "git2/oid.h"
 #include "git2/refs.h"
 #include "git2/revwalk.h"
+#include "git2/indexer.h"
 
 #include "common.h"
 #include "transport.h"
@@ -101,30 +102,27 @@ int git_fetch_negotiate(git_remote *remote)
 	return t->negotiate_fetch(t, remote->repo, &remote->refs);
 }
 
-int git_fetch_download_pack(char **out, git_remote *remote)
+int git_fetch_download_pack(git_remote *remote, git_off_t *bytes, git_indexer_stats *stats)
 {
-	if(!remote->need_pack) {
-		*out = NULL;
+	if(!remote->need_pack)
 		return 0;
-	}
 
-	return remote->transport->download_pack(out, remote->transport, remote->repo);
+	return remote->transport->download_pack(remote->transport, remote->repo, bytes, stats);
 }
 
 /* Receiving data from a socket and storing it is pretty much the same for git and HTTP */
 int git_fetch__download_pack(
-	char **out,
 	const char *buffered,
 	size_t buffered_size,
 	GIT_SOCKET fd,
-	git_repository *repo)
+	git_repository *repo,
+	git_off_t *bytes,
+	git_indexer_stats *stats)
 {
-	git_filebuf file = GIT_FILEBUF_INIT;
-	int error;
+	int recvd;
 	char buff[1024];
-	git_buf path = GIT_BUF_INIT;
-	static const char suff[] = "/objects/pack/pack-received";
 	gitno_buffer buf;
+	git_indexer_stream *idx;
 
 	gitno_buffer_setup(&buf, buff, sizeof(buff), fd);
 
@@ -133,41 +131,74 @@ int git_fetch__download_pack(
 		return -1;
 	}
 
-	if (git_buf_joinpath(&path, repo->path_repository, suff) < 0)
-		goto on_error;
+	if (git_indexer_stream_new(&idx, git_repository_path(repo)) < 0)
+		return -1;
 
-	if (git_filebuf_open(&file, path.ptr, GIT_FILEBUF_TEMPORARY) < 0)
+	memset(stats, 0, sizeof(git_indexer_stats));
+	if (git_indexer_stream_add(idx, buffered, buffered_size, stats) < 0)
 		goto on_error;
 
-	/* Part of the packfile has been received, don't loose it */
-	if (git_filebuf_write(&file, buffered, buffered_size) < 0)
-		goto on_error;
+	*bytes = buffered_size;
 
-	while (1) {
-		if (git_filebuf_write(&file, buf.data, buf.offset) < 0)
+	do {
+		if (git_indexer_stream_add(idx, buf.data, buf.offset, stats) < 0)
 			goto on_error;
 
 		gitno_consume_n(&buf, buf.offset);
-		error = gitno_recv(&buf);
-		if (error < GIT_SUCCESS)
+		if ((recvd = gitno_recv(&buf)) < 0)
 			goto on_error;
-		if (error == 0) /* Orderly shutdown */
-			break;
-	}
 
-	*out = git__strdup(file.path_lock);
-	if (*out == NULL)
+		*bytes += recvd;
+	} while(recvd > 0);
+
+	if (git_indexer_stream_finalize(idx, stats))
+		goto on_error;
+
+	git_indexer_stream_free(idx);
+	return 0;
+
+on_error:
+	git_indexer_stream_free(idx);
+	return -1;
+}
+
+int git_fetch_setup_walk(git_revwalk **out, git_repository *repo)
+{
+	git_revwalk *walk;
+	git_strarray refs;
+	unsigned int i;
+	git_reference *ref;
+
+	if (git_reference_listall(&refs, repo, GIT_REF_LISTALL) < 0)
+		return -1;
+
+	if (git_revwalk_new(&walk, repo) < 0)
+		return -1;
+
+	git_revwalk_sorting(walk, GIT_SORT_TIME);
+
+	for (i = 0; i < refs.count; ++i) {
+		/* No tags */
+		if (!git__prefixcmp(refs.strings[i], GIT_REFS_TAGS_DIR))
+			continue;
+
+		if (git_reference_lookup(&ref, repo, refs.strings[i]) < 0)
 			goto on_error;
 
-	/* A bit dodgy, but we need to keep the pack at the temporary path */
-	if (git_filebuf_commit_at(&file, file.path_lock, GIT_PACK_FILE_MODE) < 0)
+		if (git_reference_type(ref) == GIT_REF_SYMBOLIC)
+			continue;
+		if (git_revwalk_push(walk, git_reference_oid(ref)) < 0)
 			goto on_error;
 
-	git_buf_free(&path);
+		git_reference_free(ref);
+	}
 
+	git_strarray_free(&refs);
+	*out = walk;
 	return 0;
+
 on_error:
-	git_buf_free(&path);
-	git_filebuf_cleanup(&file);
+	git_reference_free(ref);
+	git_strarray_free(&refs);
 	return -1;
 }

src/fetch.h

@@ -10,9 +10,10 @@
 #include "netops.h"
 
 int git_fetch_negotiate(git_remote *remote);
-int git_fetch_download_pack(char **out, git_remote *remote);
+int git_fetch_download_pack(git_remote *remote, git_off_t *bytes, git_indexer_stats *stats);
 
-int git_fetch__download_pack(char **out, const char *buffered, size_t buffered_size,
-                             GIT_SOCKET fd, git_repository *repo);
+int git_fetch__download_pack(const char *buffered, size_t buffered_size, GIT_SOCKET fd,
+			     git_repository *repo, git_off_t *bytes, git_indexer_stats *stats);
+int git_fetch_setup_walk(git_revwalk **out, git_repository *repo);
 
 #endif

src/indexer.c

@@ -288,7 +288,7 @@ on_error:
 	return -1;
 }
 
-int git_indexer_stream_add(git_indexer_stream *idx, void *data, size_t size, git_indexer_stats *stats)
+int git_indexer_stream_add(git_indexer_stream *idx, const void *data, size_t size, git_indexer_stats *stats)
 {
 	int error;
 	struct git_pack_header hdr;

src/pkt.c

@@ -287,19 +287,6 @@ static int buffer_want_with_caps(git_remote_head *head, git_transport_caps *caps
 	return git_buf_printf(buf, "%04xwant %s%c%s\n", len, oid, 0, capstr);
 }
 
-static int send_want_with_caps(git_remote_head *head, git_transport_caps *caps, GIT_SOCKET fd)
-{
-	git_buf buf = GIT_BUF_INIT;
-	int ret;
-
-	if (buffer_want_with_caps(head, caps, &buf) < 0)
-		return -1;
-
-	ret = gitno_send(fd, buf.ptr, buf.size, 0);
-	git_buf_free(&buf);
-	return ret;
-}
-
 /*
  * All "want" packets have the same length and format, so what we do
  * is overwrite the OID each time.
@@ -341,47 +328,6 @@ int git_pkt_buffer_wants(const git_vector *refs, git_transport_caps *caps, git_b
 	return git_pkt_buffer_flush(buf);
 }
 
-int git_pkt_send_wants(const git_vector *refs, git_transport_caps *caps, GIT_SOCKET fd)
-{
-	unsigned int i = 0;
-	char buf[sizeof(pkt_want_prefix) + GIT_OID_HEXSZ + 1];
-	git_remote_head *head;
-
-	memcpy(buf, pkt_want_prefix, strlen(pkt_want_prefix));
-	buf[sizeof(buf) - 2] = '\n';
-	buf[sizeof(buf) - 1] = '\0';
-
-	/* If there are common caps, find the first one */
-	if (caps->common) {
-		for (; i < refs->length; ++i) {
-			head = refs->contents[i];
-			if (head->local)
-				continue;
-			else
-				break;
-		}
-
-		if (send_want_with_caps(refs->contents[i], caps, fd) < 0)
-			return -1;
-
-		/* Increase it here so it's correct whether we run this or not */
-		i++;
-	}
-
-	/* Continue from where we left off */
-	for (; i < refs->length; ++i) {
-		head = refs->contents[i];
-		if (head->local)
-			continue;
-
-		git_oid_fmt(buf + strlen(pkt_want_prefix), &head->oid);
-		if (gitno_send(fd, buf, strlen(buf), 0) < 0)
-			return -1;
-	}
-
-	return git_pkt_send_flush(fd);
-}
-
 int git_pkt_buffer_have(git_oid *oid, git_buf *buf)
 {
 	char oidhex[GIT_OID_HEXSZ + 1];
@@ -391,21 +337,7 @@ int git_pkt_buffer_have(git_oid *oid, git_buf *buf)
 	return git_buf_printf(buf, "%s%s\n", pkt_have_prefix, oidhex);
 }
 
-int git_pkt_send_have(git_oid *oid, GIT_SOCKET fd)
-{
-	char buf[] = "0032have 0000000000000000000000000000000000000000\n";
-
-	git_oid_fmt(buf + strlen(pkt_have_prefix), oid);
-	return gitno_send(fd, buf, strlen(buf), 0);
-}
-
-
 int git_pkt_buffer_done(git_buf *buf)
 {
 	return git_buf_puts(buf, pkt_done_str);
 }
-
-int git_pkt_send_done(GIT_SOCKET fd)
-{
-	return gitno_send(fd, pkt_done_str, strlen(pkt_done_str), 0);
-}

src/pkt.h

@@ -68,11 +68,8 @@ int git_pkt_parse_line(git_pkt **head, const char *line, const char **out, size_
 int git_pkt_buffer_flush(git_buf *buf);
 int git_pkt_send_flush(GIT_SOCKET s);
 int git_pkt_buffer_done(git_buf *buf);
-int git_pkt_send_done(GIT_SOCKET s);
 int git_pkt_buffer_wants(const git_vector *refs, git_transport_caps *caps, git_buf *buf);
-int git_pkt_send_wants(const git_vector *refs, git_transport_caps *caps, GIT_SOCKET fd);
 int git_pkt_buffer_have(git_oid *oid, git_buf *buf);
-int git_pkt_send_have(git_oid *oid, GIT_SOCKET fd);
 void git_pkt_free(git_pkt *pkt);
 
 #endif

src/remote.c

@@ -297,23 +297,24 @@ int git_remote_ls(git_remote *remote, git_headlist_cb list_cb, void *payload)
 	return remote->transport->ls(remote->transport, list_cb, payload);
 }
 
-int git_remote_download(char **filename, git_remote *remote)
+int git_remote_download(git_remote *remote, git_off_t *bytes, git_indexer_stats *stats)
 {
 	int error;
 
-	assert(filename && remote);
+	assert(remote && bytes && stats);
 
 	if ((error = git_fetch_negotiate(remote)) < 0)
 		return error;
 
-	return git_fetch_download_pack(filename, remote);
+	return git_fetch_download_pack(remote, bytes, stats);
 }
 
-int git_remote_update_tips(git_remote *remote)
+int git_remote_update_tips(git_remote *remote, int (*cb)(const char *refname, const git_oid *a, const git_oid *b))
 {
 	int error = 0;
 	unsigned int i = 0;
 	git_buf refname = GIT_BUF_INIT;
+	git_oid old;
 	git_vector *refs = &remote->refs;
 	git_remote_head *head;
 	git_reference *ref;
@@ -338,17 +339,36 @@ int git_remote_update_tips(git_remote *remote)
 		head = refs->contents[i];
 
 		if (git_refspec_transform_r(&refname, spec, head->name) < 0)
-			break;
+			goto on_error;
+
+		error = git_reference_name_to_oid(&old, remote->repo, refname.ptr);
+		if (error < 0 && error != GIT_ENOTFOUND)
+			goto on_error;
+
+		if (error == GIT_ENOTFOUND)
+			memset(&old, 0, GIT_OID_RAWSZ);
+
+		if (!git_oid_cmp(&old, &head->oid))
+			continue;
 
 		if (git_reference_create_oid(&ref, remote->repo, refname.ptr, &head->oid, 1) < 0)
 			break;
 
 		git_reference_free(ref);
+
+		if (cb != NULL) {
+			if (cb(refname.ptr, &old, &head->oid) < 0)
+				goto on_error;
+		}
 	}
 
 	git_buf_free(&refname);
+	return 0;
+
+on_error:
+	git_buf_free(&refname);
+	return -1;
 
-	return error;
 }
 
 int git_remote_connected(git_remote *remote)

src/revwalk.c

@@ -689,6 +689,13 @@ static int prepare_walk(git_revwalk *walk)
 	commit_object *next, *two;
 	commit_list *bases = NULL;
 
+	/*
+	 * If walk->one is NULL, there were no positive references,
+	 * so we know that the walk is already over.
+	 */
+	if (walk->one == NULL)
+		return GIT_EREVWALKOVER;
+
 	/* first figure out what the merge bases are */
 	if (merge_bases_many(&bases, walk, walk->one, &walk->twos) < 0)
 		return -1;

src/transport.h

@@ -8,6 +8,7 @@
 #define INCLUDE_transport_h__
 
 #include "git2/net.h"
+#include "git2/indexer.h"
 #include "vector.h"
 
 #define GIT_CAP_OFS_DELTA "ofs-delta"
@@ -66,22 +67,14 @@ struct git_transport {
 	 */
 	int (*push)(struct git_transport *transport);
 	/**
-	 * Send a 'done' message
-	 */
-	int (*send_done)(struct git_transport *transport);
-	/**
 	 * Negotiate the minimal amount of objects that need to be
 	 * retrieved
 	 */
 	int (*negotiate_fetch)(struct git_transport *transport, git_repository *repo, const git_vector *wants);
 	/**
-	 * Send a flush
-	 */
-	int (*send_flush)(struct git_transport *transport);
-	/**
 	 * Download the packfile
 	 */
-	int (*download_pack)(char **out, struct git_transport *transport, git_repository *repo);
+	int (*download_pack)(struct git_transport *transport, git_repository *repo, git_off_t *bytes, git_indexer_stats *stats);
 	/**
 	 * Fetch the changes
 	 */

src/transports/git.c

@@ -293,41 +293,22 @@ static int git_negotiate_fetch(git_transport *transport, git_repository *repo, c
 {
 	transport_git *t = (transport_git *) transport;
 	git_revwalk *walk;
-	git_reference *ref;
-	git_strarray refs;
 	git_oid oid;
 	int error;
 	unsigned int i;
+	git_buf data = GIT_BUF_INIT;
 	gitno_buffer *buf = &t->buf;
 
-	if (git_pkt_send_wants(wants, &t->caps, t->socket) < 0)
+	if (git_pkt_buffer_wants(wants, &t->caps, &data) < 0)
 		return -1;
 
-	if (git_reference_listall(&refs, repo, GIT_REF_LISTALL) < 0)
-		return -1;
-
-	if (git_revwalk_new(&walk, repo) < 0)
-		return -1;
-
-	git_revwalk_sorting(walk, GIT_SORT_TIME);
-
-	for (i = 0; i < refs.count; ++i) {
-		/* No tags */
-		if (!git__prefixcmp(refs.strings[i], GIT_REFS_TAGS_DIR))
-			continue;
-
-		if (git_reference_lookup(&ref, repo, refs.strings[i]) < 0)
+	if (git_fetch_setup_walk(&walk, repo) < 0)
 		goto on_error;
 
-		if (git_reference_type(ref) == GIT_REF_SYMBOLIC)
-			continue;
-
-		if ((error = git_revwalk_push(walk, git_reference_oid(ref))) < 0)
+	if (gitno_send(t->socket, data.ptr, data.size, 0) < 0)
 		goto on_error;
 
-	}
-	git_strarray_free(&refs);
-
+	git_buf_clear(&data);
 	/*
 	 * We don't support any kind of ACK extensions, so the negotiation
 	 * boils down to sending what we have and listening for an ACK
@@ -335,12 +316,18 @@ static int git_negotiate_fetch(git_transport *transport, git_repository *repo, c
 	 */
 	i = 0;
 	while ((error = git_revwalk_next(&oid, walk)) == 0) {
-		error = git_pkt_send_have(&oid, t->socket);
+		git_pkt_buffer_have(&oid, &data);
 		i++;
 		if (i % 20 == 0) {
 			int pkt_type;
 
-			git_pkt_send_flush(t->socket);
+			git_pkt_buffer_flush(&data);
+			if (git_buf_oom(&data))
+				goto on_error;
+
+			if (gitno_send(t->socket, data.ptr, data.size, 0) < 0)
+				goto on_error;
+
 			pkt_type = recv_pkt(buf);
 
 			if (pkt_type == GIT_PKT_ACK) {
@@ -354,35 +341,27 @@ static int git_negotiate_fetch(git_transport *transport, git_repository *repo, c
 
 		}
 	}
-	if (error != GIT_EREVWALKOVER)
+	if (error < 0 && error != GIT_EREVWALKOVER)
 		goto on_error;
 
-	git_pkt_send_flush(t->socket);
-	git_pkt_send_done(t->socket);
+	/* Tell the other end that we're done negotiating */
+	git_buf_clear(&data);
+	git_pkt_buffer_flush(&data);
+	git_pkt_buffer_done(&data);
+	if (gitno_send(t->socket, data.ptr, data.size, 0) < 0)
+		goto on_error;
 
+	git_buf_free(&data);
 	git_revwalk_free(walk);
 	return 0;
 
 on_error:
+	git_buf_free(&data);
 	git_revwalk_free(walk);
 	return -1;
 }
 
-static int git_send_flush(git_transport *transport)
-{
-	transport_git *t = (transport_git *) transport;
-
-	return git_pkt_send_flush(t->socket);
-}
-
-static int git_send_done(git_transport *transport)
-{
-	transport_git *t = (transport_git *) transport;
-
-	return git_pkt_send_done(t->socket);
-}
-
-static int git_download_pack(char **out, git_transport *transport, git_repository *repo)
+static int git_download_pack(git_transport *transport, git_repository *repo, git_off_t *bytes, git_indexer_stats *stats)
 {
 	transport_git *t = (transport_git *) transport;
 	int error = 0, read_bytes;
@@ -410,7 +389,7 @@ static int git_download_pack(char **out, git_transport *transport, git_repositor
 
 			if (pkt->type == GIT_PKT_PACK) {
 				git__free(pkt);
-				return git_fetch__download_pack(out, buf->data, buf->offset, t->socket, repo);
+				return git_fetch__download_pack(buf->data, buf->offset, t->socket, repo, bytes, stats);
 			}
 
 			/* For now we don't care about anything */
@@ -424,7 +403,6 @@ static int git_download_pack(char **out, git_transport *transport, git_repositor
 	return read_bytes;
 }
 
-
 static int git_close(git_transport *transport)
 {
 	transport_git *t = (transport_git*) transport;
@@ -476,8 +454,6 @@ int git_transport_git(git_transport **out)
 	t->parent.connect = git_connect;
 	t->parent.ls = git_ls;
 	t->parent.negotiate_fetch = git_negotiate_fetch;
-	t->parent.send_flush = git_send_flush;
-	t->parent.send_done = git_send_done;
 	t->parent.download_pack = git_download_pack;
 	t->parent.close = git_close;
 	t->parent.free = git_free;

src/transports/http.c

@@ -410,44 +410,6 @@ static int parse_response(transport_http *t)
 	return ret;
 }
 
-static int setup_walk(git_revwalk **out, git_repository *repo)
-{
-	git_revwalk *walk;
-	git_strarray refs;
-	unsigned int i;
-	git_reference *ref;
-
-	if (git_reference_listall(&refs, repo, GIT_REF_LISTALL) < 0)
-		return -1;
-
-	if (git_revwalk_new(&walk, repo) < 0)
-		return -1;
-
-	git_revwalk_sorting(walk, GIT_SORT_TIME);
-
-	for (i = 0; i < refs.count; ++i) {
-		/* No tags */
-		if (!git__prefixcmp(refs.strings[i], GIT_REFS_TAGS_DIR))
-			continue;
-
-		if (git_reference_lookup(&ref, repo, refs.strings[i]) < 0)
-			goto on_error;
-
-		if (git_reference_type(ref) == GIT_REF_SYMBOLIC)
-			continue;
-		if (git_revwalk_push(walk, git_reference_oid(ref)) < 0)
-			goto on_error;
-	}
-
-	git_strarray_free(&refs);
-	*out = walk;
-	return 0;
-
-on_error:
-	git_strarray_free(&refs);
-	return -1;
-}
-
 static int http_negotiate_fetch(git_transport *transport, git_repository *repo, const git_vector *wants)
 {
 	transport_http *t = (transport_http *) transport;
@@ -470,7 +432,7 @@ static int http_negotiate_fetch(git_transport *transport, git_repository *repo,
 	if (git_vector_init(common, 16, NULL) < 0)
 		return -1;
 
-	if (setup_walk(&walk, repo) < 0)
+	if (git_fetch_setup_walk(&walk, repo) < 0)
 		return -1;
 
 	do {
@@ -529,7 +491,8 @@ cleanup:
 }
 
 typedef struct {
-	git_filebuf *file;
+	git_indexer_stream *idx;
+	git_indexer_stats *stats;
 	transport_http *transport;
 } download_pack_cbdata;
 
@@ -545,10 +508,10 @@ static int on_body_download_pack(http_parser *parser, const char *str, size_t le
 {
 	download_pack_cbdata *data = (download_pack_cbdata *) parser->data;
 	transport_http *t = data->transport;
-	git_filebuf *file = data->file;
+	git_indexer_stream *idx = data->idx;
+	git_indexer_stats *stats = data->stats;
 
-
-	return t->error = git_filebuf_write(file, str, len);
+	return t->error = git_indexer_stream_add(idx, str, len, stats);
 }
 
 /*
@@ -557,80 +520,68 @@ static int on_body_download_pack(http_parser *parser, const char *str, size_t le
  * the simple downloader. Furthermore, we're using keep-alive
  * connections, so the simple downloader would just hang.
  */
-static int http_download_pack(char **out, git_transport *transport, git_repository *repo)
+static int http_download_pack(git_transport *transport, git_repository *repo, git_off_t *bytes, git_indexer_stats *stats)
 {
 	transport_http *t = (transport_http *) transport;
 	git_buf *oldbuf = &t->buf;
-	int ret = 0;
+	int recvd;
 	http_parser_settings settings;
 	char buffer[1024];
 	gitno_buffer buf;
+	git_indexer_stream *idx = NULL;
 	download_pack_cbdata data;
-	git_filebuf file = GIT_FILEBUF_INIT;
-	git_buf path = GIT_BUF_INIT;
-	char suff[] = "/objects/pack/pack-received\0";
+
+	gitno_buffer_setup(&buf, buffer, sizeof(buffer), t->socket);
+
+	if (memcmp(oldbuf->ptr, "PACK", strlen("PACK"))) {
+		giterr_set(GITERR_NET, "The pack doesn't start with a pack signature");
+		return -1;
+	}
+
+	if (git_indexer_stream_new(&idx, git_repository_path(repo)) < 0)
+		return -1;
+
 
 	/*
 	 * This is part of the previous response, so we don't want to
 	 * re-init the parser, just set these two callbacks.
 	 */
-	data.file = &file;
+	memset(stats, 0, sizeof(git_indexer_stats));
+	data.stats = stats;
+	data.idx = idx;
 	data.transport = t;
 	t->parser.data = &data;
 	t->transfer_finished = 0;
 	memset(&settings, 0x0, sizeof(settings));
 	settings.on_message_complete = on_message_complete_download_pack;
 	settings.on_body = on_body_download_pack;
+	*bytes = oldbuf->size;
 
-	gitno_buffer_setup(&buf, buffer, sizeof(buffer), t->socket);
-
-	if (memcmp(oldbuf->ptr, "PACK", strlen("PACK"))) {
-		giterr_set(GITERR_NET, "The pack doesn't start with a pack signature");
-		return -1;
-	}
-
-	if (git_buf_joinpath(&path, repo->path_repository, suff) < 0)
-		goto on_error;
-
-	if (git_filebuf_open(&file, path.ptr, GIT_FILEBUF_TEMPORARY) < 0)
+	if (git_indexer_stream_add(idx, oldbuf->ptr, oldbuf->size, stats) < 0)
 		goto on_error;
 
-	/* Part of the packfile has been received, don't loose it */
-	if (git_filebuf_write(&file, oldbuf->ptr, oldbuf->size) < 0)
-		goto on_error;
-
-	while(1) {
+	do {
 		size_t parsed;
 
-		ret = gitno_recv(&buf);
-		if (ret < 0)
+		if ((recvd = gitno_recv(&buf)) < 0)
 			goto on_error;
 
 		parsed = http_parser_execute(&t->parser, &settings, buf.data, buf.offset);
-		/* Both should happen at the same time */
 		if (parsed != buf.offset || t->error < 0)
-			return t->error;
+			goto on_error;
 
+		*bytes += recvd;
 		gitno_consume_n(&buf, parsed);
+	} while (recvd > 0 && !t->transfer_finished);
 
-		if (ret == 0 || t->transfer_finished) {
-			break;
-		}
-	}
-
-	*out = git__strdup(file.path_lock);
-	GITERR_CHECK_ALLOC(*out);
-
-	/* A bit dodgy, but we need to keep the pack at the temporary path */
-	ret = git_filebuf_commit_at(&file, file.path_lock, GIT_PACK_FILE_MODE);
-
-	git_buf_free(&path);
+	if (git_indexer_stream_finalize(idx, stats) < 0)
+		goto on_error;
 
+	git_indexer_stream_free(idx);
 	return 0;
 
 on_error:
-	git_filebuf_cleanup(&file);
-	git_buf_free(&path);
+	git_indexer_stream_free(idx);
 	return -1;
 }