Merge pull request #990 from ben/clone-callbacks

Progress callbacks

examples/network/clone.c

@@ -7,62 +7,72 @@
 #include <pthread.h>
 #include <unistd.h>
 
-struct dl_data {
-	git_indexer_stats fetch_stats;
-	git_indexer_stats checkout_stats;
-	git_checkout_opts opts;
-	int ret;
-	int finished;
-	const char *url;
+typedef struct progress_data {
+	git_transfer_progress fetch_progress;
+	size_t completed_steps;
+	size_t total_steps;
 	const char *path;
-};
+} progress_data;
 
-static void *clone_thread(void *ptr)
+static void print_progress(const progress_data *pd)
 {
-	struct dl_data *data = (struct dl_data *)ptr;
-	git_repository *repo = NULL;
-
-	// Kick off the clone
-	data->ret = git_clone(&repo, data->url, data->path,
-								 &data->fetch_stats, &data->checkout_stats, 
-								 &data->opts);
-	if (repo) git_repository_free(repo);
-	data->finished = 1;
+	int network_percent = (100*pd->fetch_progress.received_objects) / pd->fetch_progress.total_objects;
+	int index_percent = (100*pd->fetch_progress.indexed_objects) / pd->fetch_progress.total_objects;
+	int checkout_percent = pd->total_steps > 0
+		? (100 * pd->completed_steps) / pd->total_steps
+		: 0.f;
+	int kbytes = pd->fetch_progress.received_bytes / 1024;
+	printf("net %3d%% (%4d kb, %5d/%5d)  /  idx %3d%% (%5d/%5d)  /  chk %3d%% (%4lu/%4lu) %s\n",
+			network_percent, kbytes,
+			pd->fetch_progress.received_objects, pd->fetch_progress.total_objects,
+			index_percent, pd->fetch_progress.indexed_objects, pd->fetch_progress.total_objects,
+			checkout_percent, pd->completed_steps, pd->total_steps,
+			pd->path);
+}
 
-	pthread_exit(&data->ret);
+static void fetch_progress(const git_transfer_progress *stats, void *payload)
+{
+	progress_data *pd = (progress_data*)payload;
+	pd->fetch_progress = *stats;
+	print_progress(pd);
+}
+static void checkout_progress(const char *path, size_t cur, size_t tot, void *payload)
+{
+	progress_data *pd = (progress_data*)payload;
+	pd->completed_steps = cur;
+	pd->total_steps = tot;
+	pd->path = path;
+	print_progress(pd);
 }
 
 int do_clone(git_repository *repo, int argc, char **argv)
 {
-	struct dl_data data = {0};
-	pthread_t worker;
+	progress_data pd = {0};
+	git_repository *cloned_repo = NULL;
+	git_checkout_opts checkout_opts = {0};
+	const char *url = argv[1];
+	const char *path = argv[2];
+	int error;
 
 	// Validate args
 	if (argc < 3) {
-		printf("USAGE: %s <url> <path>\n", argv[0]);
+		printf ("USAGE: %s <url> <path>\n", argv[0]);
 		return -1;
 	}
 
-	// Data for background thread
-	data.url = argv[1];
-	data.path = argv[2];
-	data.opts.disable_filters = 1;
-	printf("Cloning '%s' to '%s'\n", data.url, data.path);
+	// Set up options
+	checkout_opts.checkout_strategy = GIT_CHECKOUT_CREATE_MISSING;
+	checkout_opts.progress_cb = checkout_progress;
+	checkout_opts.progress_payload = &pd;
 
-	// Create the worker thread
-	pthread_create(&worker, NULL, clone_thread, &data);
-
-	// Watch for progress information
-	do {
-		usleep(10000);
-		printf("Fetch %d/%d  –  Checkout %d/%d\n",
-				 data.fetch_stats.processed, data.fetch_stats.total,
-				 data.checkout_stats.processed, data.checkout_stats.total);
-	} while (!data.finished);
-	printf("Fetch %d/%d  –  Checkout %d/%d\n",
-			 data.fetch_stats.processed, data.fetch_stats.total,
-			 data.checkout_stats.processed, data.checkout_stats.total);
-
-	return data.ret;
+	// Do the clone
+	error = git_clone(&cloned_repo, url, path, &fetch_progress, &pd, &checkout_opts);
+	printf("\n");
+	if (error != 0) {
+		const git_error *err = giterr_last();
+		if (err) printf("ERROR %d: %s\n", err->klass, err->message);
+		else printf("ERROR %d: no detailed info\n", error);
+	}
+	else if (cloned_repo) git_repository_free(cloned_repo);
+	return error;
 }
-

examples/network/fetch.c

@@ -8,8 +8,6 @@
 
 struct dl_data {
 	git_remote *remote;
-	git_off_t *bytes;
-	git_indexer_stats *stats;
 	int ret;
 	int finished;
 };
@@ -35,7 +33,7 @@ static void *download(void *ptr)
 	// Download the packfile and index it. This function updates the
 	// amount of received data and the indexer stats which lets you
 	// inform the user about progress.
-	if (git_remote_download(data->remote, data->bytes, data->stats) < 0) {
+	if (git_remote_download(data->remote, NULL, NULL) < 0) {
 		data->ret = -1;
 		goto exit;
 	}
@@ -69,15 +67,14 @@ static int update_cb(const char *refname, const git_oid *a, const git_oid *b, vo
 int fetch(git_repository *repo, int argc, char **argv)
 {
 	git_remote *remote = NULL;
-	git_off_t bytes = 0;
-	git_indexer_stats stats;
+	const git_transfer_progress *stats;
 	pthread_t worker;
 	struct dl_data data;
 	git_remote_callbacks callbacks;
 
 	argc = argc;
 	// Figure out whether it's a named remote or a URL
-	printf("Fetching %s\n", argv[1]);
+	printf("Fetching %s for repo %p\n", argv[1], repo);
 	if (git_remote_load(&remote, repo, argv[1]) < 0) {
 		if (git_remote_new(&remote, repo, NULL, argv[1], NULL) < 0)
 			return -1;
@@ -91,11 +88,10 @@ int fetch(git_repository *repo, int argc, char **argv)
 
 	// Set up the information for the background worker thread
 	data.remote = remote;
-	data.bytes = &bytes;
-	data.stats = &stats;
 	data.ret = 0;
 	data.finished = 0;
-	memset(&stats, 0, sizeof(stats));
+
+	stats = git_remote_stats(remote);
 
 	pthread_create(&worker, NULL, download, &data);
 
@@ -106,16 +102,18 @@ int fetch(git_repository *repo, int argc, char **argv)
 	do {
 		usleep(10000);
 
-		if (stats.total > 0)
+		if (stats->total_objects > 0)
 			printf("Received %d/%d objects (%d) in %d bytes\r",
-			       stats.received, stats.total, stats.processed, bytes);
+			       stats->received_objects, stats->total_objects,
+					 stats->indexed_objects, stats->received_bytes);
 	} while (!data.finished);
 
 	if (data.ret < 0)
 		goto on_error;
 
 	pthread_join(worker, NULL);
-	printf("\rReceived %d/%d objects in %zu bytes\n", stats.processed, stats.total, bytes);
+	printf("\rReceived %d/%d objects in %zu bytes\n",
+			stats->indexed_objects, stats->total_objects, stats->received_bytes);
 
 	// Disconnect the underlying connection to prevent from idling.
 	git_remote_disconnect(remote);

examples/network/index-pack.c

@@ -10,10 +10,10 @@
 
 // This could be run in the main loop whilst the application waits for
 // the indexing to finish in a worker thread
-static int index_cb(const git_indexer_stats *stats, void *data)
+static int index_cb(const git_transfer_progress *stats, void *data)
 {
 	data = data;
-	printf("\rProcessing %d of %d", stats->processed, stats->total);
+	printf("\rProcessing %d of %d", stats->indexed_objects, stats->total_objects);
 
 	return 0;
 }
@@ -21,7 +21,7 @@ static int index_cb(const git_indexer_stats *stats, void *data)
 int index_pack(git_repository *repo, int argc, char **argv)
 {
 	git_indexer_stream *idx;
-	git_indexer_stats stats = {0, 0};
+	git_transfer_progress stats = {0, 0};
 	int error, fd;
 	char hash[GIT_OID_HEXSZ + 1] = {0};
 	ssize_t read_bytes;
@@ -33,7 +33,7 @@ int index_pack(git_repository *repo, int argc, char **argv)
 		return EXIT_FAILURE;
 	}
 
-	if (git_indexer_stream_new(&idx, ".") < 0) {
+	if (git_indexer_stream_new(&idx, ".", NULL, NULL) < 0) {
 		puts("bad idx");
 		return -1;
 	}
@@ -63,7 +63,7 @@ int index_pack(git_repository *repo, int argc, char **argv)
 	if ((error = git_indexer_stream_finalize(idx, &stats)) < 0)
 		goto cleanup;
 
-	printf("\rIndexing %d of %d\n", stats.processed, stats.total);
+	printf("\rIndexing %d of %d\n", stats.indexed_objects, stats.total_objects);
 
 	git_oid_fmt(hash, git_indexer_stream_hash(idx));
 	puts(hash);

include/git2/checkout.h

@@ -65,9 +65,16 @@ typedef struct git_checkout_opts {
 		const git_oid *blob_oid,
 		int file_mode,
 		void *payload);
-
 	void *notify_payload;
 
+	/* Optional callback to notify the consumer of checkout progress. */
+	void (* progress_cb)(
+			const char *path,
+			size_t completed_steps,
+			size_t total_steps,
+			void *payload);
+	void *progress_payload;
+
 	/** When not NULL, array of fnmatch patterns specifying
 	 * which paths should be taken into account
 	 */
@@ -80,29 +87,25 @@ typedef struct git_checkout_opts {
  *
  * @param repo repository to check out (must be non-bare)
  * @param opts specifies checkout options (may be NULL)
- * @param stats structure through which progress information is reported
  * @return 0 on success, GIT_EORPHANEDHEAD when HEAD points to a non existing
  * branch, GIT_ERROR otherwise (use giterr_last for information
  * about the error)
  */
 GIT_EXTERN(int) git_checkout_head(
 	git_repository *repo,
-	git_checkout_opts *opts,
-	git_indexer_stats *stats);
+	git_checkout_opts *opts);
 
 /**
  * Updates files in the working tree to match the content of the index.
  *
  * @param repo repository to check out (must be non-bare)
  * @param opts specifies checkout options (may be NULL)
- * @param stats structure through which progress information is reported
  * @return 0 on success, GIT_ERROR otherwise (use giterr_last for information
  * about the error)
  */
 GIT_EXTERN(int) git_checkout_index(
 	git_repository *repo,
-	git_checkout_opts *opts,
-	git_indexer_stats *stats);
+	git_checkout_opts *opts);
 
 /**
  * Updates files in the index and working tree to match the content of the
@@ -112,15 +115,13 @@ GIT_EXTERN(int) git_checkout_index(
  * @param treeish a commit, tag or tree which content will be used to update
  * the working directory
  * @param opts specifies checkout options (may be NULL)
- * @param stats structure through which progress information is reported
  * @return 0 on success, GIT_ERROR otherwise (use giterr_last for information
  * about the error)
  */
 GIT_EXTERN(int) git_checkout_tree(
 	git_repository *repo,
 	git_object *treeish,
-	git_checkout_opts *opts,
-	git_indexer_stats *stats);
+	git_checkout_opts *opts);
 
 /** @} */
 GIT_END_DECL

include/git2/clone.h

@@ -29,19 +29,22 @@ GIT_BEGIN_DECL
  * @param out pointer that will receive the resulting repository object
  * @param origin_url repository to clone from
  * @param workdir_path local directory to clone to
- * @param fetch_stats pointer to structure that receives fetch progress
- * information (may be NULL)
+ * @param fetch_progress_cb optional callback for fetch progress. Be aware that
+ * this is called inline with network and indexing operations, so performance
+ * may be affected.
+ * @param fetch_progress_payload payload for fetch_progress_cb
  * @param checkout_opts options for the checkout step. If NULL, no checkout
  * is performed
  * @return 0 on success, GIT_ERROR otherwise (use giterr_last for information
  * about the error)
  */
-GIT_EXTERN(int) git_clone(git_repository **out,
-								  const char *origin_url,
-								  const char *workdir_path,
-								  git_indexer_stats *fetch_stats,
-								  git_indexer_stats *checkout_stats,
-								  git_checkout_opts *checkout_opts);
+GIT_EXTERN(int) git_clone(
+		git_repository **out,
+		const char *origin_url,
+		const char *workdir_path,
+		git_transfer_progress_callback fetch_progress_cb,
+		void *fetch_progress_payload,
+		git_checkout_opts *checkout_opts);
 
 /**
  * Create a bare clone of a remote repository.
@@ -49,13 +52,18 @@ GIT_EXTERN(int) git_clone(git_repository **out,
  * @param out pointer that will receive the resulting repository object
  * @param origin_url repository to clone from
  * @param dest_path local directory to clone to
- * @param fetch_stats pointer to structure that receives fetch progress information (may be NULL)
+ * @param fetch_progress_cb optional callback for fetch progress. Be aware that
+ * this is called inline with network and indexing operations, so performance
+ * may be affected.
+ * @param fetch_progress_payload payload for fetch_progress_cb
  * @return 0 on success, GIT_ERROR otherwise (use giterr_last for information about the error)
  */
-GIT_EXTERN(int) git_clone_bare(git_repository **out,
-										 const char *origin_url,
-										 const char *dest_path,
-										 git_indexer_stats *fetch_stats);
+GIT_EXTERN(int) git_clone_bare(
+		git_repository **out,
+		const char *origin_url,
+		const char *dest_path,
+		git_transfer_progress_callback fetch_progress_cb,
+		void *fetch_progress_payload);
 
 /** @} */
 GIT_END_DECL

include/git2/index.h

@@ -343,10 +343,9 @@ GIT_EXTERN(int) git_index_entry_stage(const git_index_entry *entry);
  *
  * @param index an existing index object
  * @param tree tree to read
- * @param stats structure that receives the total node count (may be NULL)
  * @return 0 or an error code
  */
-GIT_EXTERN(int) git_index_read_tree(git_index *index, git_tree *tree, git_indexer_stats *stats);
+GIT_EXTERN(int) git_index_read_tree(git_index *index, git_tree *tree);
 
 /** @} */
 GIT_END_DECL

include/git2/indexer.h

@@ -16,13 +16,19 @@ GIT_BEGIN_DECL
  * This is passed as the first argument to the callback to allow the
  * user to see the progress.
  */
-typedef struct git_indexer_stats {
-	unsigned int total;
-	unsigned int processed;
-	unsigned int received;
-} git_indexer_stats;
+typedef struct git_transfer_progress {
+	unsigned int total_objects;
+	unsigned int indexed_objects;
+	unsigned int received_objects;
+	size_t received_bytes;
+} git_transfer_progress;
 
 
+/**
+ * Type for progress callbacks during indexing
+ */
+typedef void (*git_transfer_progress_callback)(const git_transfer_progress *stats, void *payload);
+
 typedef struct git_indexer git_indexer;
 typedef struct git_indexer_stream git_indexer_stream;
 
@@ -31,8 +37,14 @@ typedef struct git_indexer_stream git_indexer_stream;
  *
  * @param out where to store the indexer instance
  * @param path to the directory where the packfile should be stored
+ * @param progress_cb function to call with progress information
+ * @param progress_payload payload for the progress callback
  */
-GIT_EXTERN(int) git_indexer_stream_new(git_indexer_stream **out, const char *path);
+GIT_EXTERN(int) git_indexer_stream_new(
+		git_indexer_stream **out,
+		const char *path,
+		git_transfer_progress_callback progress_cb,
+		void *progress_callback_payload);
 
 /**
  * Add data to the indexer
@@ -42,7 +54,7 @@ GIT_EXTERN(int) git_indexer_stream_new(git_indexer_stream **out, const char *pat
  * @param size the size of the data
  * @param stats stat storage
  */
-GIT_EXTERN(int) git_indexer_stream_add(git_indexer_stream *idx, const void *data, size_t size, git_indexer_stats *stats);
+GIT_EXTERN(int) git_indexer_stream_add(git_indexer_stream *idx, const void *data, size_t size, git_transfer_progress *stats);
 
 /**
  * Finalize the pack and index
@@ -51,7 +63,7 @@ GIT_EXTERN(int) git_indexer_stream_add(git_indexer_stream *idx, const void *data
  *
  * @param idx the indexer
  */
-GIT_EXTERN(int) git_indexer_stream_finalize(git_indexer_stream *idx, git_indexer_stats *stats);
+GIT_EXTERN(int) git_indexer_stream_finalize(git_indexer_stream *idx, git_transfer_progress *stats);
 
 /**
  * Get the packfile's hash
@@ -88,7 +100,7 @@ GIT_EXTERN(int) git_indexer_new(git_indexer **out, const char *packname);
  * @param idx the indexer instance
  * @param stats storage for the running state
  */
-GIT_EXTERN(int) git_indexer_run(git_indexer *idx, git_indexer_stats *stats);
+GIT_EXTERN(int) git_indexer_run(git_indexer *idx, git_transfer_progress *stats);
 
 /**
  * Write the index file to disk.

include/git2/remote.h

@@ -183,10 +183,16 @@ GIT_EXTERN(int) git_remote_ls(git_remote *remote, git_headlist_cb list_cb, void
  * filename will be NULL and the function will return success.
  *
  * @param remote the remote to download from
- * @param filename where to store the temporary filename
+ * @param progress_cb function to call with progress information.  Be aware that
+ * this is called inline with network and indexing operations, so performance
+ * may be affected.
+ * @param progress_payload payload for the progress callback
  * @return 0 or an error code
  */
-GIT_EXTERN(int) git_remote_download(git_remote *remote, git_off_t *bytes, git_indexer_stats *stats);
+GIT_EXTERN(int) git_remote_download(
+		git_remote *remote,
+		git_transfer_progress_callback progress_cb,
+		void *progress_payload);
 
 /**
  * Check whether the remote is connected
@@ -313,6 +319,11 @@ struct git_remote_callbacks {
  */
 GIT_EXTERN(void) git_remote_set_callbacks(git_remote *remote, git_remote_callbacks *callbacks);
 
+/**
+ * Get the statistics structure that is filled in by the fetch operation.
+ */
+GIT_EXTERN(const git_transfer_progress *) git_remote_stats(git_remote *remote);
+
 enum {
 	GIT_REMOTE_DOWNLOAD_TAGS_UNSET,
 	GIT_REMOTE_DOWNLOAD_TAGS_NONE,

src/checkout.c

@@ -27,12 +27,13 @@ struct checkout_diff_data
 	git_buf *path;
 	size_t workdir_len;
 	git_checkout_opts *checkout_opts;
-	git_indexer_stats *stats;
 	git_repository *owner;
 	bool can_symlink;
 	bool found_submodules;
 	bool create_submodules;
 	int error;
+	size_t total_steps;
+	size_t completed_steps;
 };
 
 static int buffer_to_file(
@@ -158,6 +159,18 @@ static int checkout_submodule(
 	return 0;
 }
 
+static void report_progress(
+		struct checkout_diff_data *data,
+		const char *path)
+{
+	if (data->checkout_opts->progress_cb)
+		data->checkout_opts->progress_cb(
+				path,
+				data->completed_steps,
+				data->total_steps,
+				data->checkout_opts->progress_payload);
+}
+
 static int checkout_blob(
 	struct checkout_diff_data *data,
 	const git_diff_file *file)
@@ -191,7 +204,6 @@ static int checkout_remove_the_old(
 	git_checkout_opts *opts = data->checkout_opts;
 
 	GIT_UNUSED(progress);
-	data->stats->processed++;
 
 	if ((delta->status == GIT_DELTA_UNTRACKED &&
 		 (opts->checkout_strategy & GIT_CHECKOUT_REMOVE_UNTRACKED) != 0) ||
@@ -202,6 +214,9 @@ static int checkout_remove_the_old(
 			delta->new_file.path,
 			git_repository_workdir(data->owner),
 			GIT_DIRREMOVAL_FILES_AND_DIRS);
+
+		data->completed_steps++;
+		report_progress(data, delta->new_file.path);
 	}
 
 	return data->error;
@@ -216,7 +231,6 @@ static int checkout_create_the_new(
 	bool do_checkout = false, do_notify = false;
 
 	GIT_UNUSED(progress);
-	data->stats->processed++;
 
 	if (delta->status == GIT_DELTA_MODIFIED ||
 		delta->status == GIT_DELTA_TYPECHANGE)
@@ -243,14 +257,22 @@ static int checkout_create_the_new(
 	if (do_checkout) {
 		bool is_submodule = S_ISGITLINK(delta->old_file.mode);
 
-		if (is_submodule)
+		if (is_submodule) {
 			data->found_submodules = true;
+		}
 
-		if (!is_submodule && !data->create_submodules)
+		if (!is_submodule && !data->create_submodules) {
 			error = checkout_blob(data, &delta->old_file);
+			data->completed_steps++;
+			report_progress(data, delta->old_file.path);
+		}
 
-		else if (is_submodule && data->create_submodules)
+		else if (is_submodule && data->create_submodules) {
 			error = checkout_submodule(data, &delta->old_file);
+			data->completed_steps++;
+			report_progress(data, delta->old_file.path);
+		}
+
 	}
 
 	if (error)
@@ -304,11 +326,9 @@ static void normalize_options(git_checkout_opts *normalized, git_checkout_opts *
 
 int git_checkout_index(
 	git_repository *repo,
-	git_checkout_opts *opts,
-	git_indexer_stats *stats)
+	git_checkout_opts *opts)
 {
 	git_diff_list *diff = NULL;
-	git_indexer_stats dummy_stats;
 
 	git_diff_options diff_opts = {0};
 	git_checkout_opts checkout_opts;
@@ -339,20 +359,13 @@ int git_checkout_index(
 
 	normalize_options(&checkout_opts, opts);
 
-	if (!stats)
-		stats = &dummy_stats;
-
-	stats->processed = 0;
-	/* total based on 3 passes, but it might be 2 if no submodules */
-	stats->total = (unsigned int)git_diff_num_deltas(diff) * 3;
-
 	memset(&data, 0, sizeof(data));
 
 	data.path = &workdir;
 	data.workdir_len = git_buf_len(&workdir);
 	data.checkout_opts = &checkout_opts;
-	data.stats = stats;
 	data.owner = repo;
+	data.total_steps = (size_t)git_diff_num_deltas(diff);
 
 	if ((error = retrieve_symlink_capabilities(repo, &data.can_symlink)) < 0)
 		goto cleanup;
@@ -367,6 +380,8 @@ int git_checkout_index(
 	 *    checked out during pass #2.
 	 */
 
+	report_progress(&data, NULL);
+
 	if (!(error = git_diff_foreach(
 			diff, &data, checkout_remove_the_old, NULL, NULL)) &&
 		!(error = git_diff_foreach(
@@ -378,7 +393,7 @@ int git_checkout_index(
 			diff, &data, checkout_create_the_new, NULL, NULL);
 	}
 
-	stats->processed = stats->total;
+	report_progress(&data, NULL);
 
 cleanup:
 	if (error == GIT_EUSER)
@@ -393,8 +408,7 @@ cleanup:
 int git_checkout_tree(
 	git_repository *repo,
 	git_object *treeish,
-	git_checkout_opts *opts,
-	git_indexer_stats *stats)
+	git_checkout_opts *opts)
 {
 	git_index *index = NULL;
 	git_tree *tree = NULL;
@@ -411,13 +425,13 @@ int git_checkout_tree(
 	if ((error = git_repository_index(&index, repo)) < 0)
 		goto cleanup;
 
-	if ((error = git_index_read_tree(index, tree, NULL)) < 0)
+	if ((error = git_index_read_tree(index, tree)) < 0)
 		goto cleanup;
 
 	if ((error = git_index_write(index)) < 0)
 		goto cleanup;
 
-	error = git_checkout_index(repo, opts, stats);
+	error = git_checkout_index(repo, opts);
 
 cleanup:
 	git_index_free(index);
@@ -427,8 +441,7 @@ cleanup:
 
 int git_checkout_head(
 	git_repository *repo,
-	git_checkout_opts *opts,
-	git_indexer_stats *stats)
+	git_checkout_opts *opts)
 {
 	git_reference *head;
 	int error;
@@ -442,7 +455,7 @@ int git_checkout_head(
 	if ((error = git_reference_peel(&tree, head, GIT_OBJ_TREE)) < 0)
 		goto cleanup;
 
-	error = git_checkout_tree(repo, tree, opts, stats);
+	error = git_checkout_tree(repo, tree, opts);
 
 cleanup:
 	git_reference_free(head);

src/clone.c

@@ -248,22 +248,20 @@ cleanup:
 
 
 
-static int setup_remotes_and_fetch(git_repository *repo,
-											  const char *origin_url,
-											  git_indexer_stats *fetch_stats)
+static int setup_remotes_and_fetch(
+		git_repository *repo,
+		const char *origin_url,
+		git_transfer_progress_callback progress_cb,
+		void *progress_payload)
 {
 	int retcode = GIT_ERROR;
 	git_remote *origin = NULL;
-	git_off_t bytes = 0;
-	git_indexer_stats dummy_stats;
-
-	if (!fetch_stats) fetch_stats = &dummy_stats;
 
 	/* Create the "origin" remote */
 	if (!git_remote_add(&origin, repo, GIT_REMOTE_ORIGIN, origin_url)) {
 		/* Connect and download everything */
 		if (!git_remote_connect(origin, GIT_DIR_FETCH)) {
-			if (!git_remote_download(origin, &bytes, fetch_stats)) {
+			if (!git_remote_download(origin, progress_cb, progress_payload)) {
 				/* Create "origin/foo" branches for all remote branches */
 				if (!git_remote_update_tips(origin)) {
 					/* Point HEAD to the same ref as the remote's head */
@@ -311,23 +309,21 @@ static int clone_internal(
 	git_repository **out,
 	const char *origin_url,
 	const char *path,
-	git_indexer_stats *fetch_stats,
-	git_indexer_stats *checkout_stats,
+	git_transfer_progress_callback fetch_progress_cb,
+	void *fetch_progress_payload,
 	git_checkout_opts *checkout_opts,
 	bool is_bare)
 {
 	int retcode = GIT_ERROR;
 	git_repository *repo = NULL;
-	git_indexer_stats dummy_stats;
-
-	if (!fetch_stats) fetch_stats = &dummy_stats;
 
 	if (!path_is_okay(path)) {
 		return GIT_ERROR;
 	}
 
 	if (!(retcode = git_repository_init(&repo, path, is_bare))) {
-		if ((retcode = setup_remotes_and_fetch(repo, origin_url, fetch_stats)) < 0) {
+		if ((retcode = setup_remotes_and_fetch(repo, origin_url,
+						fetch_progress_cb, fetch_progress_payload)) < 0) {
 			/* Failed to fetch; clean up */
 			git_repository_free(repo);
 			git_futils_rmdir_r(path, NULL, GIT_DIRREMOVAL_FILES_AND_DIRS);
@@ -338,15 +334,17 @@ static int clone_internal(
 	}
 
 	if (!retcode && should_checkout(repo, is_bare, checkout_opts))
-		retcode = git_checkout_head(*out, checkout_opts, checkout_stats);
+		retcode = git_checkout_head(*out, checkout_opts);
 
 	return retcode;
 }
 
-int git_clone_bare(git_repository **out,
-						 const char *origin_url,
-						 const char *dest_path,
-						 git_indexer_stats *fetch_stats)
+int git_clone_bare(
+		git_repository **out,
+		const char *origin_url,
+		const char *dest_path,
+		git_transfer_progress_callback fetch_progress_cb,
+		void *fetch_progress_payload)
 {
 	assert(out && origin_url && dest_path);
 
@@ -354,19 +352,20 @@ int git_clone_bare(git_repository **out,
 		out,
 		origin_url,
 		dest_path,
-		fetch_stats,
-		NULL,
+		fetch_progress_cb,
+		fetch_progress_payload,
 		NULL,
 		1);
 }
 
 
-int git_clone(git_repository **out,
-				  const char *origin_url,
-				  const char *workdir_path,
-				  git_indexer_stats *fetch_stats,
-				  git_indexer_stats *checkout_stats,
-				  git_checkout_opts *checkout_opts)
+int git_clone(
+		git_repository **out,
+		const char *origin_url,
+		const char *workdir_path,
+		git_transfer_progress_callback fetch_progress_cb,
+		void *fetch_progress_payload,
+		git_checkout_opts *checkout_opts)
 {
 	assert(out && origin_url && workdir_path);
 
@@ -374,8 +373,8 @@ int git_clone(git_repository **out,
 		out,
 		origin_url,
 		workdir_path,
-		fetch_stats,
-		checkout_stats,
+		fetch_progress_cb,
+		fetch_progress_payload,
 		checkout_opts,
 		0);
 }

src/fetch.c

@@ -19,6 +19,8 @@
 #include "netops.h"
 #include "pkt.h"
 
+#define NETWORK_XFER_THRESHOLD (100*1024)
+
 struct filter_payload {
 	git_remote *remote;
 	const git_refspec *spec, *tagspec;
@@ -302,7 +304,10 @@ on_error:
 	return error;
 }
 
-int git_fetch_download_pack(git_remote *remote, git_off_t *bytes, git_indexer_stats *stats)
+int git_fetch_download_pack(
+		git_remote *remote,
+		git_transfer_progress_callback progress_cb,
+		void *progress_payload)
 {
 	git_transport *t = remote->transport;
 
@@ -310,13 +315,14 @@ int git_fetch_download_pack(git_remote *remote, git_off_t *bytes, git_indexer_st
 		return 0;
 
 	if (t->own_logic)
-		return t->download_pack(t, remote->repo, bytes, stats);
+		return t->download_pack(t, remote->repo, &remote->stats);
 
-	return git_fetch__download_pack(t, remote->repo, bytes, stats);
+	return git_fetch__download_pack(t, remote->repo, &remote->stats,
+			progress_cb, progress_payload);
 
 }
 
-static int no_sideband(git_transport *t, git_indexer_stream *idx, gitno_buffer *buf, git_off_t *bytes, git_indexer_stats *stats)
+static int no_sideband(git_transport *t, git_indexer_stream *idx, gitno_buffer *buf, git_transfer_progress *stats)
 {
 	int recvd;
 
@@ -333,8 +339,6 @@ static int no_sideband(git_transport *t, git_indexer_stream *idx, gitno_buffer *
 
 		if ((recvd = gitno_recv(buf)) < 0)
 			return -1;
-
-		*bytes += recvd;
 	} while(recvd > 0);
 
 	if (git_indexer_stream_finalize(idx, stats))
@@ -343,27 +347,58 @@ static int no_sideband(git_transport *t, git_indexer_stream *idx, gitno_buffer *
 	return 0;
 }
 
+struct network_packetsize_payload
+{
+	git_transfer_progress_callback callback;
+	void *payload;
+	git_transfer_progress *stats;
+	git_off_t last_fired_bytes;
+};
+
+static void network_packetsize(int received, void *payload)
+{
+	struct network_packetsize_payload *npp = (struct network_packetsize_payload*)payload;
+
+	/* Accumulate bytes */
+	npp->stats->received_bytes += received;
+
+	/* Fire notification if the threshold is reached */
+	if ((npp->stats->received_bytes - npp->last_fired_bytes) > NETWORK_XFER_THRESHOLD) {
+		npp->last_fired_bytes = npp->stats->received_bytes;
+		npp->callback(npp->stats, npp->payload);
+	}
+}
+
 /* Receiving data from a socket and storing it is pretty much the same for git and HTTP */
 int git_fetch__download_pack(
 	git_transport *t,
 	git_repository *repo,
-	git_off_t *bytes,
-	git_indexer_stats *stats)
+	git_transfer_progress *stats,
+	git_transfer_progress_callback progress_cb,
+	void *progress_payload)
 {
 	git_buf path = GIT_BUF_INIT;
 	gitno_buffer *buf = &t->buffer;
 	git_indexer_stream *idx = NULL;
 	int error = -1;
+	struct network_packetsize_payload npp = {0};
+
+	if (progress_cb) {
+		npp.callback = progress_cb;
+		npp.payload = progress_payload;
+		npp.stats = stats;
+		buf->packetsize_cb = &network_packetsize;
+		buf->packetsize_payload = &npp;
+	}
 
 	if (git_buf_joinpath(&path, git_repository_path(repo), "objects/pack") < 0)
 		return -1;
 
-	if (git_indexer_stream_new(&idx, git_buf_cstr(&path)) < 0)
+	if (git_indexer_stream_new(&idx, git_buf_cstr(&path), progress_cb, progress_payload) < 0)
 		goto on_error;
 
 	git_buf_free(&path);
-	memset(stats, 0, sizeof(git_indexer_stats));
-	*bytes = 0;
+	memset(stats, 0, sizeof(git_transfer_progress));
 
 	/*
 	 * If the remote doesn't support the side-band, we can feed
@@ -371,7 +406,7 @@ int git_fetch__download_pack(
 	 * check which one belongs there.
 	 */
 	if (!t->caps.side_band && !t->caps.side_band_64k) {
-		if (no_sideband(t, idx, buf, bytes, stats) < 0)
+		if (no_sideband(t, idx, buf, stats) < 0)
 			goto on_error;
 
 		git_indexer_stream_free(idx);
@@ -398,7 +433,6 @@ int git_fetch__download_pack(
 			git__free(pkt);
 		} else if (pkt->type == GIT_PKT_DATA) {
 			git_pkt_data *p = (git_pkt_data *) pkt;
-			*bytes += p->len;
 			if (git_indexer_stream_add(idx, p->data, p->len, stats) < 0)
 				goto on_error;
 

src/fetch.h

@@ -10,9 +10,19 @@
 #include "netops.h"
 
 int git_fetch_negotiate(git_remote *remote);
-int git_fetch_download_pack(git_remote *remote, git_off_t *bytes, git_indexer_stats *stats);
 
-int git_fetch__download_pack(git_transport *t, git_repository *repo, git_off_t *bytes, git_indexer_stats *stats);
+int git_fetch_download_pack(
+		git_remote *remote,
+		git_transfer_progress_callback progress_cb,
+		void *progress_payload);
+
+int git_fetch__download_pack(
+		git_transport *t,
+		git_repository *repo,
+		git_transfer_progress *stats,
+		git_transfer_progress_callback progress_cb,
+		void *progress_payload);
+
 int git_fetch_setup_walk(git_revwalk **out, git_repository *repo);
 
 #endif

src/index.c

@@ -1034,17 +1034,15 @@ int git_index_entry_stage(const git_index_entry *entry)
 
 typedef struct read_tree_data {
 	git_index *index;
-	git_indexer_stats *stats;
+	git_transfer_progress *stats;
 } read_tree_data;
 
 static int read_tree_cb(const char *root, const git_tree_entry *tentry, void *data)
 {
-	read_tree_data *rtd = data;
+	git_index *index = (git_index *)data;
 	git_index_entry *entry = NULL;
 	git_buf path = GIT_BUF_INIT;
 
-	rtd->stats->total++;
-
 	if (git_tree_entry__is_tree(tentry))
 		return 0;
 
@@ -1059,7 +1057,7 @@ static int read_tree_cb(const char *root, const git_tree_entry *tentry, void *da
 	entry->path = git_buf_detach(&path);
 	git_buf_free(&path);
 
-	if (index_insert(rtd->index, entry, 0) < 0) {
+	if (index_insert(index, entry, 0) < 0) {
 		index_entry_free(entry);
 		return -1;
 	}
@@ -1067,16 +1065,9 @@ static int read_tree_cb(const char *root, const git_tree_entry *tentry, void *da
 	return 0;
 }
 
-int git_index_read_tree(git_index *index, git_tree *tree, git_indexer_stats *stats)
+int git_index_read_tree(git_index *index, git_tree *tree)
 {
-	git_indexer_stats dummy_stats;
-	read_tree_data rtd = {index, NULL};
-
-	if (!stats) stats = &dummy_stats;
-	stats->total = 0;
-	rtd.stats = stats;
-
 	git_index_clear(index);
 
-	return git_tree_walk(tree, read_tree_cb, GIT_TREEWALK_POST, &rtd);
+	return git_tree_walk(tree, read_tree_cb, GIT_TREEWALK_POST, index);
 }

src/indexer.c

@@ -49,6 +49,8 @@ struct git_indexer_stream {
 	git_vector deltas;
 	unsigned int fanout[256];
 	git_oid hash;
+	git_transfer_progress_callback progress_cb;
+	void *progress_payload;
 };
 
 struct delta_info {
@@ -138,7 +140,11 @@ static int cache_cmp(const void *a, const void *b)
 	return git_oid_cmp(&ea->sha1, &eb->sha1);
 }
 
-int git_indexer_stream_new(git_indexer_stream **out, const char *prefix)
+int git_indexer_stream_new(
+		git_indexer_stream **out,
+		const char *prefix,
+		git_transfer_progress_callback progress_cb,
+		void *progress_payload)
 {
 	git_indexer_stream *idx;
 	git_buf path = GIT_BUF_INIT;
@@ -147,6 +153,8 @@ int git_indexer_stream_new(git_indexer_stream **out, const char *prefix)
 
 	idx = git__calloc(1, sizeof(git_indexer_stream));
 	GITERR_CHECK_ALLOC(idx);
+	idx->progress_cb = progress_cb;
+	idx->progress_payload = progress_payload;
 
 	error = git_buf_joinpath(&path, prefix, suff);
 	if (error < 0)
@@ -273,7 +281,13 @@ on_error:
 	return -1;
 }
 
-int git_indexer_stream_add(git_indexer_stream *idx, const void *data, size_t size, git_indexer_stats *stats)
+static void do_progress_callback(git_indexer_stream *idx, git_transfer_progress *stats)
+{
+	if (!idx->progress_cb) return;
+	idx->progress_cb(stats, idx->progress_payload);
+}
+
+int git_indexer_stream_add(git_indexer_stream *idx, const void *data, size_t size, git_transfer_progress *stats)
 {
 	int error;
 	struct git_pack_header hdr;
@@ -282,7 +296,7 @@ int git_indexer_stream_add(git_indexer_stream *idx, const void *data, size_t siz
 
 	assert(idx && data && stats);
 
-	processed = stats->processed;
+	processed = stats->indexed_objects;
 
 	if (git_filebuf_write(&idx->pack_file, data, size) < 0)
 		return -1;
@@ -324,8 +338,9 @@ int git_indexer_stream_add(git_indexer_stream *idx, const void *data, size_t siz
 		if (git_vector_init(&idx->deltas, (unsigned int)(idx->nr_objects / 2), NULL) < 0)
 			return -1;
 
-		memset(stats, 0, sizeof(git_indexer_stats));
-		stats->total = (unsigned int)idx->nr_objects;
+		memset(stats, 0, sizeof(git_transfer_progress));
+		stats->total_objects = (unsigned int)idx->nr_objects;
+		do_progress_callback(idx, stats);
 	}
 
 	/* Now that we have data in the pack, let's try to parse it */
@@ -361,7 +376,8 @@ int git_indexer_stream_add(git_indexer_stream *idx, const void *data, size_t siz
 			if (error < 0)
 				return error;
 
-			stats->received++;
+			stats->received_objects++;
+			do_progress_callback(idx, stats);
 			continue;
 		}
 
@@ -379,8 +395,9 @@ int git_indexer_stream_add(git_indexer_stream *idx, const void *data, size_t siz
 
 		git__free(obj.data);
 
-		stats->processed = (unsigned int)++processed;
-		stats->received++;
+		stats->indexed_objects = (unsigned int)++processed;
+		stats->received_objects++;
+		do_progress_callback(idx, stats);
 	}
 
 	return 0;
@@ -412,7 +429,7 @@ static int index_path_stream(git_buf *path, git_indexer_stream *idx, const char
 	return git_buf_oom(path) ? -1 : 0;
 }
 
-static int resolve_deltas(git_indexer_stream *idx, git_indexer_stats *stats)
+static int resolve_deltas(git_indexer_stream *idx, git_transfer_progress *stats)
 {
 	unsigned int i;
 	struct delta_info *delta;
@@ -428,13 +445,14 @@ static int resolve_deltas(git_indexer_stream *idx, git_indexer_stats *stats)
 			return -1;
 
 		git__free(obj.data);
-		stats->processed++;
+		stats->indexed_objects++;
+		do_progress_callback(idx, stats);
 	}
 
 	return 0;
 }
 
-int git_indexer_stream_finalize(git_indexer_stream *idx, git_indexer_stats *stats)
+int git_indexer_stream_finalize(git_indexer_stream *idx, git_transfer_progress *stats)
 {
 	git_mwindow *w = NULL;
 	unsigned int i, long_offsets = 0, left;
@@ -455,7 +473,7 @@ int git_indexer_stream_finalize(git_indexer_stream *idx, git_indexer_stats *stat
 		if (resolve_deltas(idx, stats) < 0)
 			return -1;
 
-	if (stats->processed != stats->total) {
+	if (stats->indexed_objects != stats->total_objects) {
 		giterr_set(GITERR_INDEXER, "Indexing error: early EOF");
 		return -1;
 	}
@@ -782,7 +800,7 @@ cleanup:
 	return error;
 }
 
-int git_indexer_run(git_indexer *idx, git_indexer_stats *stats)
+int git_indexer_run(git_indexer *idx, git_transfer_progress *stats)
 {
 	git_mwindow_file *mwf;
 	git_off_t off = sizeof(struct git_pack_header);
@@ -797,8 +815,8 @@ int git_indexer_run(git_indexer *idx, git_indexer_stats *stats)
 	if (error < 0)
 		return error;
 
-	stats->total = (unsigned int)idx->nr_objects;
-	stats->processed = processed = 0;
+	stats->total_objects = (unsigned int)idx->nr_objects;
+	stats->indexed_objects = processed = 0;
 
 	while (processed < idx->nr_objects) {
 		git_rawobj obj;
@@ -868,7 +886,7 @@ int git_indexer_run(git_indexer *idx, git_indexer_stats *stats)
 
 		git__free(obj.data);
 
-		stats->processed = ++processed;
+		stats->indexed_objects = ++processed;
 	}
 
 cleanup:

src/netops.c

@@ -117,6 +117,7 @@ static int gitno__recv_ssl(gitno_buffer *buf)
 	}
 
 	buf->offset += ret;
+	if (buf->packetsize_cb) buf->packetsize_cb(ret, buf->packetsize_payload);
 	return ret;
 }
 #endif
@@ -132,6 +133,7 @@ int gitno__recv(gitno_buffer *buf)
 	}
 
 	buf->offset += ret;
+	if (buf->packetsize_cb) buf->packetsize_cb(ret, buf->packetsize_payload);
 	return ret;
 }
 
@@ -142,7 +144,6 @@ void gitno_buffer_setup_callback(
 	size_t len,
 	int (*recv)(gitno_buffer *buf), void *cb_data)
 {
-	memset(buf, 0x0, sizeof(gitno_buffer));
 	memset(data, 0x0, len);
 	buf->data = data;
 	buf->len = len;

src/netops.h

@@ -20,6 +20,8 @@ struct gitno_buffer {
 #endif
 	int (*recv)(gitno_buffer *buffer);
 	void *cb_data;
+	void (*packetsize_cb)(int received, void *payload);
+	void *packetsize_payload;
 };
 
 void gitno_buffer_setup(git_transport *t, gitno_buffer *buf, char *data, size_t len);

src/remote.c

@@ -472,16 +472,19 @@ int git_remote_ls(git_remote *remote, git_headlist_cb list_cb, void *payload)
 	return 0;
 }
 
-int git_remote_download(git_remote *remote, git_off_t *bytes, git_indexer_stats *stats)
+int git_remote_download(
+		git_remote *remote,
+		git_transfer_progress_callback progress_cb,
+		void *progress_payload)
 {
 	int error;
 
-	assert(remote && bytes && stats);
+	assert(remote);
 
 	if ((error = git_fetch_negotiate(remote)) < 0)
 		return error;
 
-	return git_fetch_download_pack(remote, bytes, stats);
+	return git_fetch_download_pack(remote, progress_cb, progress_payload);
 }
 
 int git_remote_update_tips(git_remote *remote)
@@ -742,6 +745,12 @@ void git_remote_set_callbacks(git_remote *remote, git_remote_callbacks *callback
 	}
 }
 
+inline const git_transfer_progress* git_remote_stats(git_remote *remote)
+{
+	assert(remote);
+	return &remote->stats;
+}
+
 int git_remote_autotag(git_remote *remote)
 {
 	return remote->download_tags;

src/remote.h

@@ -25,6 +25,7 @@ struct git_remote {
 	git_transport *transport;
 	git_repository *repo;
 	git_remote_callbacks callbacks;
+	git_transfer_progress stats;
 	unsigned int need_pack:1,
 		download_tags:2, /* There are four possible values */
 		check_cert:1;

src/reset.c

@@ -116,7 +116,7 @@ int git_reset(
 		goto cleanup;
 	}
 
-	if (git_index_read_tree(index, tree, NULL) < 0) {
+	if (git_index_read_tree(index, tree) < 0) {
 		giterr_set(GITERR_INDEX, "%s - Failed to update the index.", ERROR_MSG);
 		goto cleanup;
 	}
@@ -142,7 +142,7 @@ int git_reset(
 		| GIT_CHECKOUT_OVERWRITE_MODIFIED
 		| GIT_CHECKOUT_REMOVE_UNTRACKED;
 
-	if (git_checkout_index(repo, &opts, NULL) < 0) {
+	if (git_checkout_index(repo, &opts) < 0) {
 		giterr_set(GITERR_INDEX, "%s - Failed to checkout the index.", ERROR_MSG);
 		goto cleanup;
 	}

src/transport.h

@@ -113,7 +113,7 @@ struct git_transport {
 	/**
 	 * Download the packfile
 	 */
-	int (*download_pack)(struct git_transport *transport, git_repository *repo, git_off_t *bytes, git_indexer_stats *stats);
+	int (*download_pack)(struct git_transport *transport, git_repository *repo, git_transfer_progress *stats);
 	/**
 	 * Close the connection
 	 */

src/transports/http.c

@@ -379,6 +379,8 @@ static int http_recv_cb(gitno_buffer *buf)
 
 #ifndef GIT_WINHTTP
 	gitno_buffer_setup(transport, &inner, buffer, sizeof(buffer));
+	inner.packetsize_cb = buf->packetsize_cb;
+	inner.packetsize_payload = buf->packetsize_payload;
 
 	if ((error = gitno_recv(&inner)) < 0)
 		return -1;

tests-clar/checkout/head.c

@@ -18,5 +18,5 @@ void test_checkout_head__checking_out_an_orphaned_head_returns_GIT_EORPHANEDHEAD
 {
 	make_head_orphaned(g_repo, NON_EXISTING_HEAD);
 
-	cl_assert_equal_i(GIT_EORPHANEDHEAD, git_checkout_head(g_repo, NULL, NULL));
+	cl_assert_equal_i(GIT_EORPHANEDHEAD, git_checkout_head(g_repo, NULL));
 }

tests-clar/checkout/index.c

@@ -14,7 +14,7 @@ static void reset_index_to_treeish(git_object *treeish)
 	cl_git_pass(git_object_peel(&tree, treeish, GIT_OBJ_TREE));
 
 	cl_git_pass(git_repository_index(&index, g_repo));
-	cl_git_pass(git_index_read_tree(index, (git_tree *)tree, NULL));
+	cl_git_pass(git_index_read_tree(index, (git_tree *)tree));
 	cl_git_pass(git_index_write(index));
 
 	git_object_free(tree);
@@ -69,7 +69,7 @@ void test_checkout_index__cannot_checkout_a_bare_repository(void)
 	memset(&g_opts, 0, sizeof(g_opts));
 	g_repo = cl_git_sandbox_init("testrepo.git");
 
-	cl_git_fail(git_checkout_index(g_repo, NULL, NULL));
+	cl_git_fail(git_checkout_index(g_repo, NULL));
 }
 
 void test_checkout_index__can_create_missing_files(void)
@@ -79,7 +79,7 @@ void test_checkout_index__can_create_missing_files(void)
 	cl_assert_equal_i(false, git_path_isfile("./testrepo/new.txt"));
 
 	g_opts.checkout_strategy = GIT_CHECKOUT_CREATE_MISSING;
-	cl_git_pass(git_checkout_index(g_repo, &g_opts, NULL));
+	cl_git_pass(git_checkout_index(g_repo, &g_opts));
 
 	test_file_contents("./testrepo/README", "hey there\n");
 	test_file_contents("./testrepo/branch_file.txt", "hi\nbye!\n");
@@ -95,7 +95,7 @@ void test_checkout_index__can_remove_untracked_files(void)
 	cl_assert_equal_i(true, git_path_isdir("./testrepo/dir/subdir/subsubdir"));
 
 	g_opts.checkout_strategy = GIT_CHECKOUT_REMOVE_UNTRACKED;
-	cl_git_pass(git_checkout_index(g_repo, &g_opts, NULL));
+	cl_git_pass(git_checkout_index(g_repo, &g_opts));
 
 	cl_assert_equal_i(false, git_path_isdir("./testrepo/dir"));
 }
@@ -111,7 +111,7 @@ void test_checkout_index__honor_the_specified_pathspecs(void)
 	cl_assert_equal_i(false, git_path_isfile("./testrepo/branch_file.txt"));
 	cl_assert_equal_i(false, git_path_isfile("./testrepo/new.txt"));
 
-	cl_git_pass(git_checkout_index(g_repo, &g_opts, NULL));
+	cl_git_pass(git_checkout_index(g_repo, &g_opts));
 
 	cl_assert_equal_i(false, git_path_isfile("./testrepo/README"));
 	test_file_contents("./testrepo/branch_file.txt", "hi\nbye!\n");
@@ -142,7 +142,7 @@ void test_checkout_index__honor_the_gitattributes_directives(void)
 	cl_git_mkfile("./testrepo/.gitattributes", attributes);
 	set_core_autocrlf_to(false);
 
-	cl_git_pass(git_checkout_index(g_repo, &g_opts, NULL));
+	cl_git_pass(git_checkout_index(g_repo, &g_opts));
 
 	test_file_contents("./testrepo/README", "hey there\n");
 	test_file_contents("./testrepo/new.txt", "my new file\n");
@@ -157,7 +157,7 @@ void test_checkout_index__honor_coreautocrlf_setting_set_to_true(void)
 	cl_git_pass(p_unlink("./testrepo/.gitattributes"));
 	set_core_autocrlf_to(true);
 
-	cl_git_pass(git_checkout_index(g_repo, &g_opts, NULL));
+	cl_git_pass(git_checkout_index(g_repo, &g_opts));
 
 	test_file_contents("./testrepo/README", expected_readme_text);
 #endif
@@ -172,7 +172,7 @@ void test_checkout_index__honor_coresymlinks_setting_set_to_true(void)
 {
 	set_repo_symlink_handling_cap_to(true);
 
-	cl_git_pass(git_checkout_index(g_repo, &g_opts, NULL));
+	cl_git_pass(git_checkout_index(g_repo, &g_opts));
 
 #ifdef GIT_WIN32
 	test_file_contents("./testrepo/link_to_new.txt", "new.txt");
@@ -194,7 +194,7 @@ void test_checkout_index__honor_coresymlinks_setting_set_to_false(void)
 {
 	set_repo_symlink_handling_cap_to(false);
 
-	cl_git_pass(git_checkout_index(g_repo, &g_opts, NULL));
+	cl_git_pass(git_checkout_index(g_repo, &g_opts));
 
 	test_file_contents("./testrepo/link_to_new.txt", "new.txt");
 }
@@ -204,7 +204,7 @@ void test_checkout_index__donot_overwrite_modified_file_by_default(void)
 	cl_git_mkfile("./testrepo/new.txt", "This isn't what's stored!");
 
 	g_opts.checkout_strategy = 0;
-	cl_git_pass(git_checkout_index(g_repo, &g_opts, NULL));
+	cl_git_pass(git_checkout_index(g_repo, &g_opts));
 
 	test_file_contents("./testrepo/new.txt", "This isn't what's stored!");
 }
@@ -214,7 +214,7 @@ void test_checkout_index__can_overwrite_modified_file(void)
 	cl_git_mkfile("./testrepo/new.txt", "This isn't what's stored!");
 
 	g_opts.checkout_strategy = GIT_CHECKOUT_OVERWRITE_MODIFIED;
-	cl_git_pass(git_checkout_index(g_repo, &g_opts, NULL));
+	cl_git_pass(git_checkout_index(g_repo, &g_opts));
 
 	test_file_contents("./testrepo/new.txt", "my new file\n");
 }
@@ -224,14 +224,14 @@ void test_checkout_index__options_disable_filters(void)
 	cl_git_mkfile("./testrepo/.gitattributes", "*.txt text eol=crlf\n");
 
 	g_opts.disable_filters = false;
-	cl_git_pass(git_checkout_index(g_repo, &g_opts, NULL));
+	cl_git_pass(git_checkout_index(g_repo, &g_opts));
 
 	test_file_contents("./testrepo/new.txt", "my new file\r\n");
 
 	p_unlink("./testrepo/new.txt");
 
 	g_opts.disable_filters = true;
-	cl_git_pass(git_checkout_index(g_repo, &g_opts, NULL));
+	cl_git_pass(git_checkout_index(g_repo, &g_opts));
 
 	test_file_contents("./testrepo/new.txt", "my new file\n");
 }
@@ -249,7 +249,7 @@ void test_checkout_index__options_dir_modes(void)
 	reset_index_to_treeish((git_object *)commit);
 
 	g_opts.dir_mode = 0701;
-	cl_git_pass(git_checkout_index(g_repo, &g_opts, NULL));
+	cl_git_pass(git_checkout_index(g_repo, &g_opts));
 
 	cl_git_pass(p_stat("./testrepo/a", &st));
 	cl_assert_equal_i(st.st_mode & 0777, 0701);
@@ -269,7 +269,7 @@ void test_checkout_index__options_override_file_modes(void)
 
 	g_opts.file_mode = 0700;
 
-	cl_git_pass(git_checkout_index(g_repo, &g_opts, NULL));
+	cl_git_pass(git_checkout_index(g_repo, &g_opts));
 
 	cl_git_pass(p_stat("./testrepo/new.txt", &st));
 	cl_assert_equal_i(st.st_mode & 0777, 0700);
@@ -283,7 +283,7 @@ void test_checkout_index__options_open_flags(void)
 	g_opts.file_open_flags = O_CREAT | O_RDWR | O_APPEND;
 
 	g_opts.checkout_strategy |= GIT_CHECKOUT_OVERWRITE_MODIFIED;
-	cl_git_pass(git_checkout_index(g_repo, &g_opts, NULL));
+	cl_git_pass(git_checkout_index(g_repo, &g_opts));
 
 	test_file_contents("./testrepo/new.txt", "hi\nmy new file\n");
 }
@@ -328,7 +328,7 @@ void test_checkout_index__can_notify_of_skipped_files(void)
 	g_opts.skipped_notify_cb = notify_cb;
 	g_opts.notify_payload = &data;
 
-	cl_git_pass(git_checkout_index(g_repo, &g_opts, NULL));
+	cl_git_pass(git_checkout_index(g_repo, &g_opts));
 }
 
 static int dont_notify_cb(
@@ -358,5 +358,22 @@ void test_checkout_index__wont_notify_of_expected_line_ending_changes(void)
 	g_opts.skipped_notify_cb = dont_notify_cb;
 	g_opts.notify_payload = NULL;
 
-	cl_git_pass(git_checkout_index(g_repo, &g_opts, NULL));
+	cl_git_pass(git_checkout_index(g_repo, &g_opts));
+}
+
+static void progress(const char *path, size_t cur, size_t tot, void *payload)
+{
+	GIT_UNUSED(path); GIT_UNUSED(cur); GIT_UNUSED(tot);
+	bool *was_called = (bool*)payload;
+	*was_called = true;
+}
+
+void test_checkout_index__calls_progress_callback(void)
+{
+	bool was_called = 0;
+	g_opts.progress_cb = progress;
+	g_opts.progress_payload = &was_called;
+
+	cl_git_pass(git_checkout_index(g_repo, &g_opts));
+	cl_assert_equal_i(was_called, true);
 }

tests-clar/checkout/tree.c

@@ -27,7 +27,7 @@ void test_checkout_tree__cannot_checkout_a_non_treeish(void)
 	/* blob */
 	cl_git_pass(git_revparse_single(&g_object, g_repo, "a71586c1dfe8a71c6cbf6c129f404c5642ff31bd"));
 
-	cl_git_fail(git_checkout_tree(g_repo, g_object, NULL, NULL));
+	cl_git_fail(git_checkout_tree(g_repo, g_object, NULL));
 }
 
 void test_checkout_tree__can_checkout_a_subdirectory_from_a_commit(void)
@@ -41,7 +41,7 @@ void test_checkout_tree__can_checkout_a_subdirectory_from_a_commit(void)
 
 	cl_assert_equal_i(false, git_path_isdir("./testrepo/ab/"));
 
-	cl_git_pass(git_checkout_tree(g_repo, g_object, &g_opts, NULL));
+	cl_git_pass(git_checkout_tree(g_repo, g_object, &g_opts));
 
 	cl_assert_equal_i(true, git_path_isfile("./testrepo/ab/de/2.txt"));
 	cl_assert_equal_i(true, git_path_isfile("./testrepo/ab/de/fgh/1.txt"));
@@ -58,8 +58,26 @@ void test_checkout_tree__can_checkout_a_subdirectory_from_a_subtree(void)
 
 	cl_assert_equal_i(false, git_path_isdir("./testrepo/de/"));
 
-	cl_git_pass(git_checkout_tree(g_repo, g_object, &g_opts, NULL));
+	cl_git_pass(git_checkout_tree(g_repo, g_object, &g_opts));
 
 	cl_assert_equal_i(true, git_path_isfile("./testrepo/de/2.txt"));
 	cl_assert_equal_i(true, git_path_isfile("./testrepo/de/fgh/1.txt"));
 }
+
+static void progress(const char *path, size_t cur, size_t tot, void *payload)
+{
+	GIT_UNUSED(path); GIT_UNUSED(cur); GIT_UNUSED(tot);
+	bool *was_called = (bool*)payload;
+	*was_called = true;
+}
+
+void test_checkout_tree__calls_progress_callback(void)
+{
+	bool was_called = 0;
+	g_opts.progress_cb = progress;
+	g_opts.progress_payload = &was_called;
+
+	cl_git_pass(git_revparse_single(&g_object, g_repo, "master"));
+	cl_git_pass(git_checkout_tree(g_repo, g_object, &g_opts));
+	cl_assert_equal_i(was_called, true);
+}

tests-clar/checkout/typechange.c

@@ -49,7 +49,7 @@ void test_checkout_typechange__checkout_typechanges(void)
 		cl_git_pass(git_revparse_single(&obj, g_repo, g_typechange_oids[i]));
 		/* fprintf(stderr, "checking out '%s'\n", g_typechange_oids[i]); */
 
-		cl_git_pass(git_checkout_tree(g_repo, obj, &opts, NULL));
+		cl_git_pass(git_checkout_tree(g_repo, obj, &opts));
 
 		git_object_free(obj);
 

tests-clar/clone/network.c

@@ -43,7 +43,7 @@ void test_clone_network__network_bare(void)
 
 	cl_set_cleanup(&cleanup_repository, "./test");
 
-	cl_git_pass(git_clone_bare(&g_repo, LIVE_REPO_URL, "./test", NULL));
+	cl_git_pass(git_clone_bare(&g_repo, LIVE_REPO_URL, "./test", NULL, NULL));
 	cl_assert(git_repository_is_bare(g_repo));
 	cl_git_pass(git_remote_load(&origin, g_repo, "origin"));
 
@@ -91,18 +91,36 @@ void test_clone_network__can_prevent_the_checkout_of_a_standard_repo(void)
 	git_buf_free(&path);
 }
 
+static void checkout_progress(const char *path, size_t cur, size_t tot, void *payload)
+{
+	GIT_UNUSED(path); GIT_UNUSED(cur); GIT_UNUSED(tot);
+	bool *was_called = (bool*)payload;
+	(*was_called) = true;
+}
+
+static void fetch_progress(const git_transfer_progress *stats, void *payload)
+{
+	GIT_UNUSED(stats);
+	bool *was_called = (bool*)payload;
+	(*was_called) = true;
+}
+
 void test_clone_network__can_checkout_a_cloned_repo(void)
 {
-	git_checkout_opts opts;
+	git_checkout_opts opts = {0};
 	git_buf path = GIT_BUF_INIT;
 	git_reference *head;
+	bool checkout_progress_cb_was_called = false,
+		  fetch_progress_cb_was_called = false;
 
-	memset(&opts, 0, sizeof(opts));
 	opts.checkout_strategy = GIT_CHECKOUT_CREATE_MISSING;
+	opts.progress_cb = &checkout_progress;
+	opts.progress_payload = &checkout_progress_cb_was_called;
 
 	cl_set_cleanup(&cleanup_repository, "./default-checkout");
 
-	cl_git_pass(git_clone(&g_repo, LIVE_REPO_URL, "./default-checkout", NULL, NULL, &opts));
+	cl_git_pass(git_clone(&g_repo, LIVE_REPO_URL, "./default-checkout", 
+				&fetch_progress, &fetch_progress_cb_was_called, &opts));
 
 	cl_git_pass(git_buf_joinpath(&path, git_repository_workdir(g_repo), "master.txt"));
 	cl_assert_equal_i(true, git_path_isfile(git_buf_cstr(&path)));
@@ -111,6 +129,9 @@ void test_clone_network__can_checkout_a_cloned_repo(void)
 	cl_assert_equal_i(GIT_REF_SYMBOLIC, git_reference_type(head));
 	cl_assert_equal_s("refs/heads/master", git_reference_target(head));
 
+	cl_assert_equal_i(true, checkout_progress_cb_was_called);
+	cl_assert_equal_i(true, fetch_progress_cb_was_called);
+
 	git_reference_free(head);
 	git_buf_free(&path);
 }

tests-clar/clone/nonetwork.c

@@ -65,7 +65,7 @@ void test_clone_nonetwork__bad_url(void)
 	/* Clone should clean up the mess if the URL isn't a git repository */
 	cl_git_fail(git_clone(&g_repo, "not_a_repo", "./foo", NULL, NULL, NULL));
 	cl_assert(!git_path_exists("./foo"));
-	cl_git_fail(git_clone_bare(&g_repo, "not_a_repo", "./foo.git", NULL));
+	cl_git_fail(git_clone_bare(&g_repo, "not_a_repo", "./foo.git", NULL, NULL));
 	cl_assert(!git_path_exists("./foo.git"));
 }
 
@@ -91,7 +91,7 @@ void test_clone_nonetwork__local_bare(void)
 #if DO_LOCAL_TEST
 	cl_set_cleanup(&cleanup_repository, "./local.git");
 
-	cl_git_pass(git_clone_bare(&g_repo, git_buf_cstr(&src), "./local.git", NULL));
+	cl_git_pass(git_clone_bare(&g_repo, git_buf_cstr(&src), "./local.git", NULL, NULL));
 #endif
 
 	git_buf_free(&src);

tests-clar/index/read_tree.c

@@ -33,7 +33,7 @@ void test_index_read_tree__read_write_involution(void)
 
 	/* read-tree */
 	git_tree_lookup(&tree, repo, &expected);
-	cl_git_pass(git_index_read_tree(index, tree, NULL));
+	cl_git_pass(git_index_read_tree(index, tree));
 	git_tree_free(tree);
 
 	cl_git_pass(git_tree_create_fromindex(&tree_oid, index));

tests-clar/network/fetch.c

@@ -28,12 +28,18 @@ static int update_tips(const char *refname, const git_oid *a, const git_oid *b,
 	return 0;
 }
 
+static void progress(const git_transfer_progress *stats, void *payload)
+{
+	GIT_UNUSED(stats);
+	bool *was_called = (bool*)payload;
+	*was_called = true;
+}
+
 static void do_fetch(const char *url, int flag, int n)
 {
 	git_remote *remote;
-	git_off_t bytes;
-	git_indexer_stats stats;
 	git_remote_callbacks callbacks;
+	bool progress_was_called = false;
 
 	memset(&callbacks, 0, sizeof(git_remote_callbacks));
 	callbacks.update_tips = update_tips;
@@ -43,10 +49,11 @@ static void do_fetch(const char *url, int flag, int n)
 	git_remote_set_callbacks(remote, &callbacks);
 	git_remote_set_autotag(remote, flag);
 	cl_git_pass(git_remote_connect(remote, GIT_DIR_FETCH));
-	cl_git_pass(git_remote_download(remote, &bytes, &stats));
+	cl_git_pass(git_remote_download(remote, progress, &progress_was_called));
 	git_remote_disconnect(remote);
 	cl_git_pass(git_remote_update_tips(remote));
 	cl_assert_equal_i(counter, n);
+	cl_assert_equal_i(progress_was_called, true);
 
 	git_remote_free(remote);
 }

tests-clar/pack/packbuilder.c

@@ -33,7 +33,7 @@ void test_pack_packbuilder__cleanup(void)
 
 void test_pack_packbuilder__create_pack(void)
 {
-	git_indexer_stats stats;
+	git_transfer_progress stats;
 	git_oid oid, *o;
 	unsigned int i;
 

tests-clar/status/worktree.c

@@ -486,7 +486,7 @@ static void fill_index_wth_head_entries(git_repository *repo, git_index *index)
 	cl_git_pass(git_commit_lookup(&commit, repo, &oid));
 	cl_git_pass(git_commit_tree(&tree, commit));
 
-	cl_git_pass(git_index_read_tree(index, tree, NULL));
+	cl_git_pass(git_index_read_tree(index, tree));
 	cl_git_pass(git_index_write(index));
 
 	git_tree_free(tree);