Merge pull request #2761 from libgit2/cmn/fetch-prune

Remote-tracking branch prunning

CHANGELOG.md

@@ -137,6 +137,11 @@ v0.21 + 1
   has been changed to match git 1.9.0 and later. In this mode, libgit2 now
   fetches all tags in addition to whatever else needs to be fetched.
 
+* The remote object has learnt to prune remote-tracking branches. If
+  the remote is configured to do so, this will happen via
+  git_remote_fetch(). You can also call git_remote_prune() after
+  connecting or fetching to perform the prune.
+
 * git_threads_init() and git_threads_shutdown() have been renamed to
   git_libgit2_init() and git_libgit2_shutdown() to better explain what
   their purpose is, as it's grown to be more than just about threads.

include/git2/remote.h

@@ -388,6 +388,14 @@ GIT_EXTERN(int) git_remote_update_tips(
 		const char *reflog_message);
 
 /**
+ * Prune tracking refs that are no longer present on remote
+ *
+ * @param remote the remote to prune
+ * @return 0 or an error code
+ */
+GIT_EXTERN(int) git_remote_prune(git_remote *remote);
+
+/**
  * Download new data and update tips
  *
  * Convenience function to connect to a remote, download the data,
@@ -598,6 +606,14 @@ GIT_EXTERN(void) git_remote_set_autotag(
 	git_remote_autotag_option_t value);
 
 /**
+ * Retrieve the ref-prune setting
+ *
+ * @param remote the remote to query
+ * @return the ref-prune setting
+ */
+GIT_EXTERN(int) git_remote_prune_refs(const git_remote *remote);
+
+/**
  * Give the remote a new name
  *
  * All remote-tracking branches and configuration settings

src/remote.c

@@ -288,6 +288,7 @@ int git_remote_dup(git_remote **dest, git_remote *source)
 	remote->repo = source->repo;
 	remote->download_tags = source->download_tags;
 	remote->update_fetchhead = source->update_fetchhead;
+	remote->prune_refs = source->prune_refs;
 
 	if (git_vector_init(&remote->refs, 32, NULL) < 0 ||
 	    git_vector_init(&remote->refspecs, 2, NULL) < 0 ||
@@ -443,6 +444,22 @@ int git_remote_lookup(git_remote **out, git_repository *repo, const char *name)
 	if (download_tags_value(remote, config) < 0)
 		goto cleanup;
 
+	git_buf_clear(&buf);
+	git_buf_printf(&buf, "remote.%s.prune", name);
+
+	if ((error = git_config_get_bool(&remote->prune_refs, config, git_buf_cstr(&buf))) < 0) {
+		if (error == GIT_ENOTFOUND) {
+			giterr_clear();
+
+			if ((error = git_config_get_bool(&remote->prune_refs, config, "fetch.prune")) < 0) {
+				if (error == GIT_ENOTFOUND) {
+					giterr_clear();
+					error = 0;
+				}
+			}
+		}
+	}
+
 	/* Move the data over to where the matching functions can find them */
 	if (dwim_refspecs(&remote->active_refspecs, &remote->refspecs, &remote->refs) < 0)
 		goto cleanup;
@@ -913,6 +930,12 @@ int git_remote_fetch(
 	/* Create "remote/foo" branches for all remote branches */
 	error = git_remote_update_tips(remote, signature, git_buf_cstr(&reflog_msg_buf));
 	git_buf_free(&reflog_msg_buf);
+	if (error < 0)
+		return error;
+
+	if (remote->prune_refs)
+		error = git_remote_prune(remote);
+
 	return error;
 }
 
@@ -1067,6 +1090,145 @@ cleanup:
 	return error;
 }
 
+/**
+ * Generate a list of candidates for pruning by getting a list of
+ * references which match the rhs of an active refspec.
+ */
+static int prune_candidates(git_vector *candidates, git_remote *remote)
+{
+	git_strarray arr = { 0 };
+	size_t i;
+	int error;
+
+	if ((error = git_reference_list(&arr, remote->repo)) < 0)
+		return error;
+
+	for (i = 0; i < arr.count; i++) {
+		const char *refname = arr.strings[i];
+		char *refname_dup;
+
+		if (!git_remote__matching_dst_refspec(remote, refname))
+			continue;
+
+		refname_dup = git__strdup(refname);
+		GITERR_CHECK_ALLOC(refname_dup);
+
+		if ((error = git_vector_insert(candidates, refname_dup)) < 0)
+			goto out;
+	}
+
+out:
+	git_strarray_free(&arr);
+	return error;
+}
+
+static int find_head(const void *_a, const void *_b)
+{
+	git_remote_head *a = (git_remote_head *) _a;
+	git_remote_head *b = (git_remote_head *) _b;
+
+	return strcmp(a->name, b->name);
+}
+
+int git_remote_prune(git_remote *remote)
+{
+	size_t i, j;
+	git_vector remote_refs = GIT_VECTOR_INIT;
+	git_vector candidates = GIT_VECTOR_INIT;
+	const git_refspec *spec;
+	const char *refname;
+	int error;
+	git_oid zero_id = {{ 0 }};
+
+	if ((error = ls_to_vector(&remote_refs, remote)) < 0)
+		goto cleanup;
+
+	git_vector_set_cmp(&remote_refs, find_head);
+
+	if ((error = prune_candidates(&candidates, remote)) < 0)
+		goto cleanup;
+
+	/*
+	 * Remove those entries from the candidate list for which we
+	 * can find a remote reference in at least one refspec.
+	 */
+	git_vector_foreach(&candidates, i, refname) {
+		git_vector_foreach(&remote->active_refspecs, j, spec) {
+			git_buf buf = GIT_BUF_INIT;
+			size_t pos;
+			char *src_name;
+			git_remote_head key = {0};
+
+			if (!git_refspec_dst_matches(spec, refname))
+				continue;
+
+			if ((error = git_refspec_rtransform(&buf, spec, refname)) < 0)
+				goto cleanup;
+
+			key.name = (char *) git_buf_cstr(&buf);
+			error = git_vector_search(&pos, &remote_refs, &key);
+			git_buf_free(&buf);
+
+			if (error < 0 && error != GIT_ENOTFOUND)
+				goto cleanup;
+
+			if (error == GIT_ENOTFOUND)
+				continue;
+
+			/* if we did find a source, remove it from the candiates */
+			if ((error = git_vector_set((void **) &src_name, &candidates, i, NULL)) < 0)
+				goto cleanup;
+
+			git__free(src_name);
+			break;
+		}
+	}
+
+	/*
+	 * For those candidates still left in the list, we need to
+	 * remove them. We do not remove symrefs, as those are for
+	 * stuff like origin/HEAD which will never match, but we do
+	 * not want to remove them.
+	 */
+	git_vector_foreach(&candidates, i, refname) {
+		git_reference *ref;
+		git_oid id;
+
+		if (refname == NULL)
+			continue;
+
+		error = git_reference_lookup(&ref, remote->repo, refname);
+		/* as we want it gone, let's not consider this an error */
+		if (error == GIT_ENOTFOUND)
+			continue;
+
+		if (error < 0)
+			goto cleanup;
+
+		if (git_reference_type(ref) == GIT_REF_SYMBOLIC) {
+			git_reference_free(ref);
+			continue;
+		}
+
+		git_oid_cpy(&id, git_reference_target(ref));
+		error = git_reference_delete(ref);
+		git_reference_free(ref);
+		if (error < 0)
+			goto cleanup;
+
+		if (remote->callbacks.update_tips)
+			error = remote->callbacks.update_tips(refname, &id, &zero_id, remote->callbacks.payload);
+
+		if (error < 0)
+			goto cleanup;
+	}
+
+cleanup:
+	git_vector_free(&remote_refs);
+	git_vector_free_deep(&candidates);
+	return error;
+}
+
 static int update_tips_for_spec(
 		git_remote *remote,
 		git_refspec *spec,
@@ -1472,6 +1634,11 @@ void git_remote_set_autotag(git_remote *remote, git_remote_autotag_option_t valu
 	remote->download_tags = value;
 }
 
+int git_remote_prune_refs(const git_remote *remote)
+{
+	return remote->prune_refs;
+}
+
 static int rename_remote_config_section(
 	git_repository *repo,
 	const char *old_name,

src/remote.h

@@ -34,6 +34,7 @@ struct git_remote {
 	unsigned int need_pack;
 	git_remote_autotag_option_t download_tags;
 	int update_fetchhead;
+	int prune_refs;
 	int passed_refspecs;
 };
 

tests/network/fetchlocal.c

@@ -4,6 +4,10 @@
 #include "path.h"
 #include "remote.h"
 
+static const char* tagger_name = "Vicent Marti";
+static const char* tagger_email = "vicent@github.com";
+static const char* tagger_message = "This is my tag.\n\nThere are many tags, but this one is mine\n";
+
 static int transfer_cb(const git_transfer_progress *stats, void *payload)
 {
 	int *callcount = (int*)payload;
@@ -17,6 +21,11 @@ static void cleanup_local_repo(void *path)
 	cl_fixture_cleanup((char *)path);
 }
 
+void test_network_fetchlocal__cleanup(void)
+{
+	cl_git_sandbox_cleanup();
+}
+
 void test_network_fetchlocal__complete(void)
 {
 	git_repository *repo;
@@ -48,6 +57,282 @@ void test_network_fetchlocal__complete(void)
 	git_repository_free(repo);
 }
 
+void test_network_fetchlocal__prune(void)
+{
+	git_repository *repo;
+	git_remote *origin;
+	int callcount = 0;
+	git_strarray refnames = {0};
+	git_reference *ref;
+	git_repository *remote_repo = cl_git_sandbox_init("testrepo.git");
+	const char *url = cl_git_path_url(git_repository_path(remote_repo));
+	git_remote_callbacks callbacks = GIT_REMOTE_CALLBACKS_INIT;
+
+	callbacks.transfer_progress = transfer_cb;
+	callbacks.payload = &callcount;
+
+	cl_set_cleanup(&cleanup_local_repo, "foo");
+	cl_git_pass(git_repository_init(&repo, "foo", true));
+
+	cl_git_pass(git_remote_create(&origin, repo, GIT_REMOTE_ORIGIN, url));
+	git_remote_set_callbacks(origin, &callbacks);
+	cl_git_pass(git_remote_fetch(origin, NULL, NULL, NULL));
+
+	cl_git_pass(git_reference_list(&refnames, repo));
+	cl_assert_equal_i(19, (int)refnames.count);
+	cl_assert(callcount > 0);
+	git_strarray_free(&refnames);
+	git_remote_free(origin);
+
+	cl_git_pass(git_reference_lookup(&ref, remote_repo, "refs/heads/br2"));
+	cl_git_pass(git_reference_delete(ref));
+	git_reference_free(ref);
+
+	cl_git_pass(git_remote_lookup(&origin, repo, GIT_REMOTE_ORIGIN));
+	git_remote_set_callbacks(origin, &callbacks);
+	cl_git_pass(git_remote_connect(origin, GIT_DIRECTION_FETCH));
+	cl_git_pass(git_remote_download(origin, NULL));
+	cl_git_pass(git_remote_prune(origin));
+	cl_git_pass(git_remote_update_tips(origin, NULL, NULL));
+
+	cl_git_pass(git_reference_list(&refnames, repo));
+	cl_assert_equal_i(18, (int)refnames.count);
+	git_strarray_free(&refnames);
+	git_remote_free(origin);
+
+	cl_git_pass(git_reference_lookup(&ref, remote_repo, "refs/heads/packed"));
+	cl_git_pass(git_reference_delete(ref));
+	git_reference_free(ref);
+
+	cl_git_pass(git_remote_lookup(&origin, repo, GIT_REMOTE_ORIGIN));
+	git_remote_set_callbacks(origin, &callbacks);
+	cl_git_pass(git_remote_connect(origin, GIT_DIRECTION_FETCH));
+	cl_git_pass(git_remote_download(origin, NULL));
+	cl_git_pass(git_remote_prune(origin));
+	cl_git_pass(git_remote_update_tips(origin, NULL, NULL));
+
+	cl_git_pass(git_reference_list(&refnames, repo));
+	cl_assert_equal_i(17, (int)refnames.count);
+	git_strarray_free(&refnames);
+	git_remote_free(origin);
+
+	git_repository_free(repo);
+}
+
+int update_tips_fail_on_call(const char *ref, const git_oid *old, const git_oid *new, void *data)
+{
+	GIT_UNUSED(ref);
+	GIT_UNUSED(old);
+	GIT_UNUSED(new);
+	GIT_UNUSED(data);
+
+	cl_fail("update tips called");
+	return 0;
+}
+
+void assert_ref_exists(git_repository *repo, const char *name)
+{
+	git_reference *ref;
+
+	cl_git_pass(git_reference_lookup(&ref, repo, name));
+	git_reference_free(ref);
+}
+
+void test_network_fetchlocal__prune_overlapping(void)
+{
+	git_repository *repo;
+	git_remote *origin;
+	int callcount = 0;
+	git_strarray refnames = {0};
+	git_reference *ref;
+	git_config *config;
+	git_oid target;
+
+	git_repository *remote_repo = cl_git_sandbox_init("testrepo.git");
+	const char *url = cl_git_path_url(git_repository_path(remote_repo));
+
+	git_remote_callbacks callbacks = GIT_REMOTE_CALLBACKS_INIT;
+	callbacks.transfer_progress = transfer_cb;
+	callbacks.payload = &callcount;
+
+	cl_git_pass(git_reference_lookup(&ref, remote_repo, "refs/heads/master"));
+	git_oid_cpy(&target, git_reference_target(ref));
+	git_reference_free(ref);
+	cl_git_pass(git_reference_create(&ref, remote_repo, "refs/pull/42/head", &target, 1, NULL, NULL));
+	git_reference_free(ref);
+
+	cl_set_cleanup(&cleanup_local_repo, "foo");
+	cl_git_pass(git_repository_init(&repo, "foo", true));
+
+	cl_git_pass(git_remote_create(&origin, repo, GIT_REMOTE_ORIGIN, url));
+	git_remote_set_callbacks(origin, &callbacks);
+
+	cl_git_pass(git_repository_config(&config, repo));
+	cl_git_pass(git_config_set_bool(config, "remote.origin.prune", true));
+	cl_git_pass(git_config_set_multivar(config, "remote.origin.fetch", "^$", "refs/pull/*/head:refs/remotes/origin/pr/*"));
+
+	git_remote_free(origin);
+	cl_git_pass(git_remote_lookup(&origin, repo, GIT_REMOTE_ORIGIN));
+	git_remote_set_callbacks(origin, &callbacks);
+	cl_git_pass(git_remote_fetch(origin, NULL, NULL, NULL));
+
+	assert_ref_exists(repo, "refs/remotes/origin/master");
+	assert_ref_exists(repo, "refs/remotes/origin/pr/42");
+	cl_git_pass(git_reference_list(&refnames, repo));
+	cl_assert_equal_i(20, (int)refnames.count);
+	git_strarray_free(&refnames);
+
+	cl_git_pass(git_config_delete_multivar(config, "remote.origin.fetch", "refs"));
+	cl_git_pass(git_config_set_multivar(config, "remote.origin.fetch", "^$", "refs/pull/*/head:refs/remotes/origin/pr/*"));
+	cl_git_pass(git_config_set_multivar(config, "remote.origin.fetch", "^$", "refs/heads/*:refs/remotes/origin/*"));
+
+	git_remote_free(origin);
+	cl_git_pass(git_remote_lookup(&origin, repo, GIT_REMOTE_ORIGIN));
+	callbacks.update_tips = update_tips_fail_on_call;
+	git_remote_set_callbacks(origin, &callbacks);
+	cl_git_pass(git_remote_fetch(origin, NULL, NULL, NULL));
+
+	assert_ref_exists(repo, "refs/remotes/origin/master");
+	assert_ref_exists(repo, "refs/remotes/origin/pr/42");
+	cl_git_pass(git_reference_list(&refnames, repo));
+	cl_assert_equal_i(20, (int)refnames.count);
+	git_strarray_free(&refnames);
+
+	cl_git_pass(git_config_delete_multivar(config, "remote.origin.fetch", "refs"));
+	cl_git_pass(git_config_set_multivar(config, "remote.origin.fetch", "^$", "refs/heads/*:refs/remotes/origin/*"));
+	cl_git_pass(git_config_set_multivar(config, "remote.origin.fetch", "^$", "refs/pull/*/head:refs/remotes/origin/pr/*"));
+
+	git_remote_free(origin);
+	cl_git_pass(git_remote_lookup(&origin, repo, GIT_REMOTE_ORIGIN));
+	callbacks.update_tips = update_tips_fail_on_call;
+	git_remote_set_callbacks(origin, &callbacks);
+	cl_git_pass(git_remote_fetch(origin, NULL, NULL, NULL));
+
+	git_config_free(config);
+	git_strarray_free(&refnames);
+	git_remote_free(origin);
+	git_repository_free(repo);
+}
+
+void test_network_fetchlocal__fetchprune(void)
+{
+	git_repository *repo;
+	git_remote *origin;
+	int callcount = 0;
+	git_strarray refnames = {0};
+	git_reference *ref;
+	git_config *config;
+	git_repository *remote_repo = cl_git_sandbox_init("testrepo.git");
+	const char *url = cl_git_path_url(git_repository_path(remote_repo));
+	git_remote_callbacks callbacks = GIT_REMOTE_CALLBACKS_INIT;
+
+	callbacks.transfer_progress = transfer_cb;
+	callbacks.payload = &callcount;
+
+	cl_set_cleanup(&cleanup_local_repo, "foo");
+	cl_git_pass(git_repository_init(&repo, "foo", true));
+
+	cl_git_pass(git_remote_create(&origin, repo, GIT_REMOTE_ORIGIN, url));
+	git_remote_set_callbacks(origin, &callbacks);
+	cl_git_pass(git_remote_fetch(origin, NULL, NULL, NULL));
+
+	cl_git_pass(git_reference_list(&refnames, repo));
+	cl_assert_equal_i(19, (int)refnames.count);
+	cl_assert(callcount > 0);
+	git_strarray_free(&refnames);
+	git_remote_free(origin);
+
+	cl_git_pass(git_reference_lookup(&ref, remote_repo, "refs/heads/br2"));
+	cl_git_pass(git_reference_delete(ref));
+	git_reference_free(ref);
+
+	cl_git_pass(git_remote_lookup(&origin, repo, GIT_REMOTE_ORIGIN));
+	git_remote_set_callbacks(origin, &callbacks);
+	cl_git_pass(git_remote_fetch(origin, NULL, NULL, NULL));
+	cl_git_pass(git_remote_prune(origin));
+
+	cl_git_pass(git_reference_list(&refnames, repo));
+	cl_assert_equal_i(18, (int)refnames.count);
+	git_strarray_free(&refnames);
+	git_remote_free(origin);
+
+	cl_git_pass(git_reference_lookup(&ref, remote_repo, "refs/heads/packed"));
+	cl_git_pass(git_reference_delete(ref));
+	git_reference_free(ref);
+
+	cl_git_pass(git_repository_config(&config, repo));
+	cl_git_pass(git_config_set_bool(config, "remote.origin.prune", 1));
+	git_config_free(config);
+	cl_git_pass(git_remote_lookup(&origin, repo, GIT_REMOTE_ORIGIN));
+	cl_assert_equal_i(1, git_remote_prune_refs(origin));
+	git_remote_set_callbacks(origin, &callbacks);
+	cl_git_pass(git_remote_fetch(origin, NULL, NULL, NULL));
+
+	cl_git_pass(git_reference_list(&refnames, repo));
+	cl_assert_equal_i(17, (int)refnames.count);
+	git_strarray_free(&refnames);
+	git_remote_free(origin);
+
+	git_repository_free(repo);
+}
+
+void test_network_fetchlocal__prune_tag(void)
+{
+	git_repository *repo;
+	git_remote *origin;
+	int callcount = 0;
+	git_reference *ref;
+	git_config *config;
+	git_oid tag_id;
+	git_signature *tagger;
+	git_object *obj;
+
+	git_repository *remote_repo = cl_git_sandbox_init("testrepo.git");
+	const char *url = cl_git_path_url(git_repository_path(remote_repo));
+	git_remote_callbacks callbacks = GIT_REMOTE_CALLBACKS_INIT;
+
+	callbacks.transfer_progress = transfer_cb;
+	callbacks.payload = &callcount;
+
+	cl_set_cleanup(&cleanup_local_repo, "foo");
+	cl_git_pass(git_repository_init(&repo, "foo", true));
+
+	cl_git_pass(git_remote_create(&origin, repo, GIT_REMOTE_ORIGIN, url));
+	git_remote_set_callbacks(origin, &callbacks);
+	cl_git_pass(git_remote_fetch(origin, NULL, NULL, NULL));
+	git_remote_free(origin);
+
+	cl_git_pass(git_revparse_single(&obj, repo, "origin/master"));
+
+	cl_git_pass(git_reference_create(&ref, repo, "refs/remotes/origin/fake-remote", git_object_id(obj), 1, NULL, NULL));
+	git_reference_free(ref);
+
+	/* create signature */
+	cl_git_pass(git_signature_new(&tagger, tagger_name, tagger_email, 123456789, 60));
+
+	cl_git_pass(
+		git_tag_create(&tag_id, repo,
+		  "some-tag", obj, tagger, tagger_message, 0)
+	);
+	git_signature_free(tagger);
+
+	cl_git_pass(git_repository_config(&config, repo));
+	cl_git_pass(git_config_set_bool(config, "remote.origin.prune", 1));
+	git_config_free(config);
+	cl_git_pass(git_remote_lookup(&origin, repo, GIT_REMOTE_ORIGIN));
+	cl_assert_equal_i(1, git_remote_prune_refs(origin));
+	git_remote_set_callbacks(origin, &callbacks);
+	cl_git_pass(git_remote_fetch(origin, NULL, NULL, NULL));
+
+	assert_ref_exists(repo, "refs/tags/some-tag");
+	cl_git_fail_with(GIT_ENOTFOUND, git_reference_lookup(&ref, repo, "refs/remotes/origin/fake-remote"));
+
+	git_object_free(obj);
+	git_remote_free(origin);
+
+	git_repository_free(repo);
+}
+
 static void cleanup_sandbox(void *unused)
 {
 	GIT_UNUSED(unused);