Merge pull request #6203 from libgit2/ethomson/fetch_by_oid

Fetch by object id

include/git2/sys/remote.h

+/*
+ * Copyright (C) the libgit2 contributors. All rights reserved.
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+
+#ifndef INCLUDE_sys_git_remote_h
+#define INCLUDE_sys_git_remote_h
+
+/**
+ * @file git2/sys/remote.h
+ * @brief Low-level remote functionality for custom transports
+ * @defgroup git_remote Low-level remote functionality
+ * @ingroup Git
+ * @{
+*/
+
+GIT_BEGIN_DECL
+
+typedef enum {
+	/** Remote supports fetching an advertised object by ID. */
+	GIT_REMOTE_CAPABILITY_TIP_OID = (1 << 0),
+
+	/** Remote supports fetching an individual reachable object. */
+	GIT_REMOTE_CAPABILITY_REACHABLE_OID = (1 << 1),
+} git_remote_capability_t;
+
+/** @} */
+GIT_END_DECL
+#endif

include/git2/sys/transport.h

@@ -47,6 +47,16 @@ struct git_transport {
 		const git_remote_connect_options *connect_opts);
 
 	/**
+	 * Gets the capabilities for this remote repository.
+	 *
+	 * This function may be called after a successful call to
+	 * `connect()`.
+	 */
+	int GIT_CALLBACK(capabilities)(
+		unsigned int *capabilities,
+		git_transport *transport);
+
+	/**
 	 * Get the list of available references in the remote repository.
 	 *
 	 * This function may be called after a successful call to

src/fetch.c

@@ -11,6 +11,7 @@
 #include "git2/refs.h"
 #include "git2/revwalk.h"
 #include "git2/transport.h"
+#include "git2/sys/remote.h"
 
 #include "remote.h"
 #include "refspec.h"
@@ -19,7 +20,7 @@
 #include "repository.h"
 #include "refs.h"
 
-static int maybe_want(git_remote *remote, git_remote_head *head, git_odb *odb, git_refspec *tagspec, git_remote_autotag_option_t tagopt)
+static int maybe_want(git_remote *remote, git_remote_head *head, git_refspec *tagspec, git_remote_autotag_option_t tagopt)
 {
 	int match = 0, valid;
 
@@ -44,23 +45,57 @@ static int maybe_want(git_remote *remote, git_remote_head *head, git_odb *odb, g
 	if (!match)
 		return 0;
 
-	/* If we have the object, mark it so we don't ask for it */
-	if (git_odb_exists(odb, &head->oid)) {
-		head->local = 1;
+	return git_vector_insert(&remote->refs, head);
+}
+
+static int mark_local(git_remote *remote)
+{
+	git_remote_head *head;
+	git_odb *odb;
+	size_t i;
+
+	if (git_repository_odb__weakptr(&odb, remote->repo) < 0)
+		return -1;
+
+	git_vector_foreach(&remote->refs, i, head) {
+		/* If we have the object, mark it so we don't ask for it */
+		if (git_odb_exists(odb, &head->oid))
+			head->local = 1;
+		else
+			remote->need_pack = 1;
 	}
-	else
-		remote->need_pack = 1;
 
-	return git_vector_insert(&remote->refs, head);
+	return 0;
+}
+
+static int maybe_want_oid(git_remote *remote, git_refspec *spec)
+{
+	git_remote_head *oid_head;
+
+	oid_head = git__calloc(1, sizeof(git_remote_head));
+	GIT_ERROR_CHECK_ALLOC(oid_head);
+
+	git_oid_fromstr(&oid_head->oid, spec->src);
+	oid_head->name = git__strdup(spec->dst);
+	GIT_ERROR_CHECK_ALLOC(oid_head->name);
+
+	if (git_vector_insert(&remote->local_heads, oid_head) < 0 ||
+	    git_vector_insert(&remote->refs, oid_head) < 0)
+		return -1;
+
+	return 0;
 }
 
 static int filter_wants(git_remote *remote, const git_fetch_options *opts)
 {
 	git_remote_head **heads;
-	git_refspec tagspec, head;
+	git_refspec tagspec, head, *spec;
 	int error = 0;
 	git_odb *odb;
 	size_t i, heads_len;
+	unsigned int remote_caps;
+	unsigned int oid_mask = GIT_REMOTE_CAPABILITY_TIP_OID |
+	                        GIT_REMOTE_CAPABILITY_REACHABLE_OID;
 	git_remote_autotag_option_t tagopt = remote->download_tags;
 
 	if (opts && opts->download_tags != GIT_REMOTE_DOWNLOAD_TAGS_UNSPECIFIED)
@@ -90,14 +125,33 @@ static int filter_wants(git_remote *remote, const git_fetch_options *opts)
 	if ((error = git_repository_odb__weakptr(&odb, remote->repo)) < 0)
 		goto cleanup;
 
-	if ((error = git_remote_ls((const git_remote_head ***)&heads, &heads_len, remote)) < 0)
+	if ((error = git_remote_ls((const git_remote_head ***)&heads, &heads_len, remote)) < 0 ||
+	    (error = git_remote_capabilities(&remote_caps, remote)) < 0)
 		goto cleanup;
 
+	/* Handle remote heads */
 	for (i = 0; i < heads_len; i++) {
-		if ((error = maybe_want(remote, heads[i], odb, &tagspec, tagopt)) < 0)
-			break;
+		if ((error = maybe_want(remote, heads[i], &tagspec, tagopt)) < 0)
+			goto cleanup;
+	}
+
+	/* Handle explicitly specified OID specs */
+	git_vector_foreach(&remote->active_refspecs, i, spec) {
+		if (!git_oid__is_hexstr(spec->src))
+			continue;
+
+		if (!(remote_caps & oid_mask)) {
+			git_error_set(GIT_ERROR_INVALID, "cannot fetch a specific object from the remote repository");
+			error = -1;
+			goto cleanup;
+		}
+
+		if ((error = maybe_want_oid(remote, spec)) < 0)
+			goto cleanup;
 	}
 
+	error = mark_local(remote);
+
 cleanup:
 	git_refspec__dispose(&tagspec);
 
@@ -115,10 +169,8 @@ int git_fetch_negotiate(git_remote *remote, const git_fetch_options *opts)
 
 	remote->need_pack = 0;
 
-	if (filter_wants(remote, opts) < 0) {
-		git_error_set(GIT_ERROR_NET, "failed to filter the reference list for wants");
+	if (filter_wants(remote, opts) < 0)
 		return -1;
-	}
 
 	/* Don't try to negotiate when we don't want anything */
 	if (!remote->need_pack)

src/oid.h

@@ -48,4 +48,16 @@ GIT_INLINE(void) git_oid__cpy_prefix(
 		out->id[len / 2] &= 0xF0;
 }
 
+GIT_INLINE(bool) git_oid__is_hexstr(const char *str)
+{
+	size_t i;
+
+	for (i = 0; str[i] != '\0'; i++) {
+		if (git__fromhex(str[i]) < 0)
+			return false;
+	}
+
+	return (i == GIT_OID_HEXSZ);
+}
+
 #endif

src/remote.h

@@ -27,6 +27,7 @@ struct git_remote {
 	git_vector refspecs;
 	git_vector active_refspecs;
 	git_vector passive_refspecs;
+	git_vector local_heads;
 	git_transport *transport;
 	git_repository *repo;
 	git_push *push;
@@ -54,4 +55,6 @@ int git_remote_connect_options_normalize(
 	const git_remote_connect_options *src);
 void git_remote_connect_options_dispose(git_remote_connect_options *opts);
 
+int git_remote_capabilities(unsigned int *out, git_remote *remote);
+
 #endif

src/transports/local.c

@@ -28,6 +28,7 @@
 #include "git2/pack.h"
 #include "git2/commit.h"
 #include "git2/revparse.h"
+#include "git2/sys/remote.h"
 
 typedef struct {
 	git_transport parent;
@@ -256,6 +257,15 @@ static int local_set_connect_opts(
 	return git_remote_connect_options_normalize(&t->connect_opts, t->owner->repo, connect_opts);
 }
 
+static int local_capabilities(unsigned int *capabilities, git_transport *transport)
+{
+	GIT_UNUSED(transport);
+
+	*capabilities = GIT_REMOTE_CAPABILITY_TIP_OID |
+	                GIT_REMOTE_CAPABILITY_REACHABLE_OID;
+	return 0;
+}
+
 static int local_ls(const git_remote_head ***out, size_t *size, git_transport *transport)
 {
 	transport_local *t = (transport_local *)transport;
@@ -721,6 +731,7 @@ int git_transport_local(git_transport **out, git_remote *owner, void *param)
 	t->parent.version = GIT_TRANSPORT_VERSION;
 	t->parent.connect = local_connect;
 	t->parent.set_connect_opts = local_set_connect_opts;
+	t->parent.capabilities = local_capabilities;
 	t->parent.negotiate_fetch = local_negotiate_fetch;
 	t->parent.download_pack = local_download_pack;
 	t->parent.push = local_push;

src/transports/smart.c

@@ -8,6 +8,7 @@
 #include "smart.h"
 
 #include "git2.h"
+#include "git2/sys/remote.h"
 #include "refs.h"
 #include "refspec.h"
 #include "proxy.h"
@@ -226,6 +227,21 @@ static int git_smart__set_connect_opts(
 	return git_remote_connect_options_normalize(&t->connect_opts, t->owner->repo, opts);
 }
 
+static int git_smart__capabilities(unsigned int *capabilities, git_transport *transport)
+{
+	transport_smart *t = GIT_CONTAINER_OF(transport, transport_smart, parent);
+
+	*capabilities = 0;
+
+	if (t->caps.want_tip_sha1)
+		*capabilities |= GIT_REMOTE_CAPABILITY_TIP_OID;
+
+	if (t->caps.want_reachable_sha1)
+		*capabilities |= GIT_REMOTE_CAPABILITY_REACHABLE_OID;
+
+	return 0;
+}
+
 static int git_smart__ls(const git_remote_head ***out, size_t *size, git_transport *transport)
 {
 	transport_smart *t = GIT_CONTAINER_OF(transport, transport_smart, parent);
@@ -423,6 +439,7 @@ int git_transport_smart(git_transport **out, git_remote *owner, void *param)
 	t->parent.version = GIT_TRANSPORT_VERSION;
 	t->parent.connect = git_smart__connect;
 	t->parent.set_connect_opts = git_smart__set_connect_opts;
+	t->parent.capabilities = git_smart__capabilities;
 	t->parent.close = git_smart__close;
 	t->parent.free = git_smart__free;
 	t->parent.negotiate_fetch = git_smart__negotiate_fetch;

src/transports/smart.h

@@ -30,6 +30,8 @@
 #define GIT_CAP_REPORT_STATUS "report-status"
 #define GIT_CAP_THIN_PACK "thin-pack"
 #define GIT_CAP_SYMREF "symref"
+#define GIT_CAP_WANT_TIP_SHA1 "allow-tip-sha1-in-want"
+#define GIT_CAP_WANT_REACHABLE_SHA1 "allow-reachable-sha1-in-want"
 
 extern bool git_smart__ofs_delta_enabled;
 
@@ -128,7 +130,9 @@ typedef struct transport_smart_caps {
 	             include_tag:1,
 	             delete_refs:1,
 	             report_status:1,
-	             thin_pack:1;
+	             thin_pack:1,
+	             want_tip_sha1:1,
+	             want_reachable_sha1:1;
 } transport_smart_caps;
 
 typedef int (*packetsize_cb)(size_t received, void *payload);

src/transports/smart_protocol.c

@@ -205,6 +205,18 @@ int git_smart__detect_caps(git_pkt_ref *pkt, transport_smart_caps *caps, git_vec
 			continue;
 		}
 
+		if (!git__prefixcmp(ptr, GIT_CAP_WANT_TIP_SHA1)) {
+			caps->common = caps->want_tip_sha1 = 1;
+			ptr += strlen(GIT_CAP_DELETE_REFS);
+			continue;
+		}
+
+		if (!git__prefixcmp(ptr, GIT_CAP_WANT_REACHABLE_SHA1)) {
+			caps->common = caps->want_reachable_sha1 = 1;
+			ptr += strlen(GIT_CAP_DELETE_REFS);
+			continue;
+		}
+
 		/* We don't know this capability, so skip it */
 		ptr = strchr(ptr, ' ');
 	}

tests/core/oid.c

 #include "clar_libgit2.h"
+#include "oid.h"
 
 static git_oid id;
 static git_oid idp;
@@ -68,3 +69,11 @@ void test_core_oid__ncmp(void)
 	cl_assert(!git_oid_ncmp(&id, &id, 40));
 	cl_assert(!git_oid_ncmp(&id, &id, 41));
 }
+
+void test_core_oid__is_hexstr(void)
+{
+	cl_assert(git_oid__is_hexstr("deadbeefdeadbeefdeadbeefdeadbeefdeadbeef"));
+	cl_assert(!git_oid__is_hexstr("deadbeefdeadbeef"));
+	cl_assert(!git_oid__is_hexstr("zeadbeefdeadbeefdeadbeefdeadbeefdeadbeef"));
+	cl_assert(!git_oid__is_hexstr("deadbeefdeadbeefdeadbeefdeadbeefdeadbeef1"));
+}

tests/fetch/local.c

+#include "clar_libgit2.h"
+#include "futils.h"
+
+static git_repository *repo;
+
+void test_fetch_local__initialize(void)
+{
+	cl_git_pass(git_repository_init(&repo, "./fetch", 0));
+}
+
+void test_fetch_local__cleanup(void)
+{
+	git_repository_free(repo);
+	repo = NULL;
+
+	cl_fixture_cleanup("./fetch");
+}
+
+void test_fetch_local__defaults(void)
+{
+	git_remote *remote;
+	git_object *obj;
+	git_oid expected_id;
+
+	cl_git_pass(git_remote_create(&remote, repo, "test",
+		cl_fixture("testrepo.git")));
+	cl_git_pass(git_remote_fetch(remote, NULL, NULL, NULL));
+
+	git_oid_fromstr(&expected_id, "258f0e2a959a364e40ed6603d5d44fbb24765b10");
+
+	cl_git_pass(git_revparse_single(&obj, repo, "refs/remotes/test/haacked"));
+	cl_assert_equal_oid(&expected_id, git_object_id(obj));
+
+	git_object_free(obj);
+	git_remote_free(remote);
+}
+
+void test_fetch_local__reachable_commit(void)
+{
+	git_remote *remote;
+	git_strarray refspecs;
+	git_object *obj;
+	git_oid expected_id;
+	git_str fetchhead = GIT_STR_INIT;
+	char *refspec = "+5b5b025afb0b4c913b4c338a42934a3863bf3644:refs/success";
+
+	refspecs.strings = &refspec;
+	refspecs.count = 1;
+
+	git_oid_fromstr(&expected_id, "5b5b025afb0b4c913b4c338a42934a3863bf3644");
+
+	cl_git_pass(git_remote_create(&remote, repo, "test",
+		cl_fixture("testrepo.git")));
+	cl_git_pass(git_remote_fetch(remote, &refspecs, NULL, NULL));
+
+	cl_git_pass(git_revparse_single(&obj, repo, "refs/success"));
+	cl_assert_equal_oid(&expected_id, git_object_id(obj));
+
+	cl_git_pass(git_futils_readbuffer(&fetchhead, "./fetch/.git/FETCH_HEAD"));
+	cl_assert_equal_strn(fetchhead.ptr,
+		"5b5b025afb0b4c913b4c338a42934a3863bf3644\t\t'5b5b025afb0b4c913b4c338a42934a3863bf3644' of ",
+		strlen("5b5b025afb0b4c913b4c338a42934a3863bf3644\t\t'5b5b025afb0b4c913b4c338a42934a3863bf3644' of "));
+
+	git_str_dispose(&fetchhead);
+	git_object_free(obj);
+	git_remote_free(remote);
+}

tests/online/fetch.c

 #include "clar_libgit2.h"
+#include "futils.h"
 
 static git_repository *_repo;
 static int counter;
@@ -290,3 +291,33 @@ void test_online_fetch__redirect_config(void)
 	cl_git_fail(do_redirected_fetch(_remote_redirect_initial, "initial", "false"));
 	cl_git_fail(do_redirected_fetch(_remote_redirect_subsequent, "subsequent", "false"));
 }
+
+void test_online_fetch__reachable_commit(void)
+{
+	git_remote *remote;
+	git_strarray refspecs;
+	git_object *obj;
+	git_oid expected_id;
+	git_str fetchhead = GIT_STR_INIT;
+	char *refspec = "+2c349335b7f797072cf729c4f3bb0914ecb6dec9:refs/success";
+
+	refspecs.strings = &refspec;
+	refspecs.count = 1;
+
+	git_oid_fromstr(&expected_id, "2c349335b7f797072cf729c4f3bb0914ecb6dec9");
+
+	cl_git_pass(git_remote_create(&remote, _repo, "test",
+		"https://github.com/libgit2/TestGitRepository"));
+	cl_git_pass(git_remote_fetch(remote, &refspecs, NULL, NULL));
+
+	cl_git_pass(git_revparse_single(&obj, _repo, "refs/success"));
+	cl_assert_equal_oid(&expected_id, git_object_id(obj));
+
+	cl_git_pass(git_futils_readbuffer(&fetchhead, "./fetch/.git/FETCH_HEAD"));
+	cl_assert_equal_s(fetchhead.ptr,
+		"2c349335b7f797072cf729c4f3bb0914ecb6dec9\t\t'2c349335b7f797072cf729c4f3bb0914ecb6dec9' of https://github.com/libgit2/TestGitRepository\n");
+
+	git_str_dispose(&fetchhead);
+	git_object_free(obj);
+	git_remote_free(remote);
+}