Merge pull request #2011 from libgit2/cmn/clone-local

Local clone

include/git2/clone.h

@@ -23,6 +23,13 @@
  */
 GIT_BEGIN_DECL
 
+typedef enum {
+	GIT_CLONE_LOCAL_AUTO,
+	GIT_CLONE_LOCAL,
+	GIT_CLONE_NO_LOCAL,
+	GIT_CLONE_LOCAL_NO_LINKS,
+} git_clone_local_t;
+
 /**
  * Clone options structure
  *
@@ -57,6 +64,7 @@ typedef struct git_clone_options {
 
 	int bare;
 	int ignore_cert_errors;
+	git_clone_local_t local;
 	const char *remote_name;
 	const char* checkout_branch;
 	git_signature *signature;
@@ -123,6 +131,35 @@ GIT_EXTERN(int) git_clone_into(
 	const char *branch,
 	const git_signature *signature);
 
+/**
+ * Perform a local clone into a repository
+ *
+ * A "local clone" bypasses any git-aware protocols and simply copies
+ * over the object database from the source repository. It is often
+ * faster than a git-aware clone, but no verification of the data is
+ * performed, and can copy over too much data.
+ *
+ * @param repo the repository to use
+ * @param remote the remote repository to clone from
+ * @param co_opts options to use during checkout
+ * @param branch the branch to checkout after the clone, pass NULL for the
+ *        remote's default branch
+ * @param link wether to use hardlinks instead of copying
+ * objects. This is only possible if both repositories are on the same
+ * filesystem.
+ * @param signature the identity used when updating the reflog
+ * @return 0 on success, any non-zero return value from a callback
+ *         function, or a negative value to indicate an error (use
+ *         `giterr_last` for a detailed error message)
+ */
+GIT_EXTERN(int) git_clone_local_into(
+	git_repository *repo,
+	git_remote *remote,
+	const git_checkout_options *co_opts,
+	const char *branch,
+	int link,
+	const git_signature *signature);
+
 /** @} */
 GIT_END_DECL
 #endif

src/clone.c

@@ -22,6 +22,7 @@
 #include "refs.h"
 #include "path.h"
 #include "repository.h"
+#include "odb.h"
 
 static int create_branch(
 	git_reference **branch,
@@ -241,6 +242,15 @@ static int create_and_configure_origin(
 	int error;
 	git_remote *origin = NULL;
 	const char *name;
+	char buf[GIT_PATH_MAX];
+
+	/* If the path exists and is a dir, the url should be the absolute path */
+	if (git_path_root(url) < 0 && git_path_exists(url) && git_path_isdir(url)) {
+		if (p_realpath(url, buf) == NULL)
+			return -1;
+
+		url = buf;
+	}
 
 	name = options->remote_name ? options->remote_name : "origin";
 	if ((error = git_remote_create(&origin, repo, name, url)) < 0)
@@ -280,6 +290,23 @@ static bool should_checkout(
 	return !git_repository_head_unborn(repo);
 }
 
+static int checkout_branch(git_repository *repo, git_remote *remote, const git_checkout_options *co_opts, const char *branch, const git_signature *signature, const char *reflog_message)
+{
+	int error;
+
+	if (branch)
+		error = update_head_to_branch(repo, git_remote_name(remote), branch,
+				signature, reflog_message);
+	/* Point HEAD to the same ref as the remote's head */
+	else
+		error = update_head_to_remote(repo, remote, signature, reflog_message);
+
+	if (!error && should_checkout(repo, git_repository_is_bare(repo), co_opts))
+		error = git_checkout_head(repo, co_opts);
+
+	return error;
+}
+
 int git_clone_into(git_repository *repo, git_remote *_remote, const git_checkout_options *co_opts, const char *branch, const git_signature *signature)
 {
 	int error;
@@ -311,15 +338,7 @@ int git_clone_into(git_repository *repo, git_remote *_remote, const git_checkout
 	if ((error = git_remote_fetch(remote, signature, git_buf_cstr(&reflog_message))) != 0)
 		goto cleanup;
 
-	if (branch)
-		error = update_head_to_branch(repo, git_remote_name(remote), branch,
-				signature, git_buf_cstr(&reflog_message));
-	/* Point HEAD to the same ref as the remote's head */
-	else
-		error = update_head_to_remote(repo, remote, signature, git_buf_cstr(&reflog_message));
-
-	if (!error && should_checkout(repo, git_repository_is_bare(repo), co_opts))
-		error = git_checkout_head(repo, co_opts);
+	error = checkout_branch(repo, remote, co_opts, branch, signature, git_buf_cstr(&reflog_message));
 
 cleanup:
 	git_remote_free(remote);
@@ -328,6 +347,29 @@ cleanup:
 	return error;
 }
 
+int git_clone__should_clone_local(const char *url, git_clone_local_t local)
+{
+	const char *path;
+	int is_url;
+
+	if (local == GIT_CLONE_NO_LOCAL)
+		return false;
+
+	is_url = !git__prefixcmp(url, "file://");
+
+	if (is_url && local != GIT_CLONE_LOCAL && local != GIT_CLONE_LOCAL_NO_LINKS )
+		return false;
+
+	path = url;
+	if (is_url)
+		path = url + strlen("file://");
+
+	if ((git_path_exists(path) && git_path_isdir(path)) && local != GIT_CLONE_NO_LOCAL)
+		return true;
+
+	return false;
+}
+
 int git_clone(
 	git_repository **out,
 	const char *url,
@@ -362,8 +404,16 @@ int git_clone(
 		return error;
 
 	if (!(error = create_and_configure_origin(&origin, repo, url, &options))) {
-		error = git_clone_into(
-			repo, origin, &options.checkout_opts, options.checkout_branch, options.signature);
+		if (git_clone__should_clone_local(url, options.local)) {
+			int link = options.local != GIT_CLONE_LOCAL_NO_LINKS;
+			error = git_clone_local_into(
+				repo, origin, &options.checkout_opts,
+				options.checkout_branch, link, options.signature);
+		} else {
+			error = git_clone_into(
+				repo, origin, &options.checkout_opts,
+				options.checkout_branch, options.signature);
+		}
 
 		git_remote_free(origin);
 	}
@@ -390,3 +440,103 @@ int git_clone_init_options(git_clone_options *opts, unsigned int version)
 		opts, version, git_clone_options, GIT_CLONE_OPTIONS_INIT);
 	return 0;
 }
+
+static const char *repository_base(git_repository *repo)
+{
+	if (git_repository_is_bare(repo))
+		return git_repository_path(repo);
+
+	return git_repository_workdir(repo);
+}
+
+static bool can_link(const char *src, const char *dst, int link)
+{
+#ifdef GIT_WIN32
+	return false;
+#else
+
+	struct stat st_src, st_dst;
+
+	if (!link)
+		return false;
+
+	if (p_stat(src, &st_src) < 0)
+		return false;
+
+	if (p_stat(dst, &st_dst) < 0)
+		return false;
+
+	return st_src.st_dev == st_dst.st_dev;
+#endif
+}
+
+int git_clone_local_into(git_repository *repo, git_remote *remote, const git_checkout_options *co_opts, const char *branch, int link, const git_signature *signature)
+{
+	int error, root, flags;
+	git_repository *src;
+	git_buf src_odb = GIT_BUF_INIT, dst_odb = GIT_BUF_INIT, src_path = GIT_BUF_INIT;
+	git_buf reflog_message = GIT_BUF_INIT;
+	const char *url;
+
+	assert(repo && remote);
+
+	if (!git_repository_is_empty(repo)) {
+		giterr_set(GITERR_INVALID, "the repository is not empty");
+		return -1;
+	}
+
+	/*
+	 * Let's figure out what path we should use for the source
+	 * repo, if it's not rooted, the path should be relative to
+	 * the repository's worktree/gitdir.
+	 */
+	url = git_remote_url(remote);
+	if (!git__prefixcmp(url, "file://"))
+		root = strlen("file://");
+	else
+		root = git_path_root(url);
+
+	if (root >= 0)
+		git_buf_puts(&src_path, url + root);
+	else
+		git_buf_joinpath(&src_path, repository_base(repo), url);
+
+	if (git_buf_oom(&src_path))
+		return -1;
+
+	/* Copy .git/objects/ from the source to the target */
+	if ((error = git_repository_open(&src, git_buf_cstr(&src_path))) < 0) {
+		git_buf_free(&src_path);
+		return error;
+	}
+
+	git_buf_joinpath(&src_odb, git_repository_path(src), GIT_OBJECTS_DIR);
+	git_buf_joinpath(&dst_odb, git_repository_path(repo), GIT_OBJECTS_DIR);
+	if (git_buf_oom(&src_odb) || git_buf_oom(&dst_odb)) {
+		error = -1;
+		goto cleanup;
+	}
+
+	flags = 0;
+	if (can_link(git_repository_path(src), git_repository_path(repo), link))
+		flags |= GIT_CPDIR_LINK_FILES;
+
+	if ((error = git_futils_cp_r(git_buf_cstr(&src_odb), git_buf_cstr(&dst_odb),
+				     flags, GIT_OBJECT_DIR_MODE)) < 0)
+		goto cleanup;
+
+	git_buf_printf(&reflog_message, "clone: from %s", git_remote_url(remote));
+
+	if ((error = git_remote_fetch(remote, signature, git_buf_cstr(&reflog_message))) != 0)
+		goto cleanup;
+
+	error = checkout_branch(repo, remote, co_opts, branch, signature, git_buf_cstr(&reflog_message));
+
+cleanup:
+	git_buf_free(&reflog_message);
+	git_buf_free(&src_path);
+	git_buf_free(&src_odb);
+	git_buf_free(&dst_odb);
+	git_repository_free(src);
+	return error;
+}

src/clone.h

+/*
+ * Copyright (C) the libgit2 contributors. All rights reserved.
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#ifndef INCLUDE_clone_h__
+#define INCLUDE_clone_h__
+
+extern int git_clone__should_clone_local(const char *url, git_clone_local_t local);
+
+#endif

src/fileops.c

@@ -740,9 +740,11 @@ static int _cp_r_callback(void *ref, git_buf *from)
 		return error;
 
 	/* make symlink or regular file */
-	if (S_ISLNK(from_st.st_mode))
+	if (info->flags & GIT_CPDIR_LINK_FILES) {
+		error = p_link(from->ptr, info->to.ptr);
+	} else if (S_ISLNK(from_st.st_mode)) {
 		error = cp_link(from->ptr, info->to.ptr, (size_t)from_st.st_size);
-	else {
+	} else {
 		mode_t usemode = from_st.st_mode;
 
 		if ((info->flags & GIT_CPDIR_SIMPLE_TO_MODE) != 0)

src/fileops.h

@@ -173,6 +173,7 @@ extern int git_futils_cp(
  * - GIT_CPDIR_SIMPLE_TO_MODE: default tries to replicate the mode of the
  *   source file to the target; with this flag, always use 0666 (or 0777 if
  *   source has exec bits set) for target.
+ * - GIT_CPDIR_LINK_FILES will try to use hardlinks for the files
  */
 typedef enum {
 	GIT_CPDIR_CREATE_EMPTY_DIRS = (1u << 0),
@@ -181,6 +182,7 @@ typedef enum {
 	GIT_CPDIR_OVERWRITE         = (1u << 3),
 	GIT_CPDIR_CHMOD_DIRS        = (1u << 4),
 	GIT_CPDIR_SIMPLE_TO_MODE    = (1u << 5),
+	GIT_CPDIR_LINK_FILES        = (1u << 6),
 } git_futils_cpdir_flags;
 
 /**

tests/clone/local.c

+#include "clar_libgit2.h"
+
+#include "git2/clone.h"
+#include "clone.h"
+#include "buffer.h"
+#include "path.h"
+#include "posix.h"
+#include "fileops.h"
+
+void test_clone_local__should_clone_local(void)
+{
+	git_buf buf = GIT_BUF_INIT;
+	const char *path;
+
+	/* we use a fixture path because it needs to exist for us to want to clone */
+	
+	cl_git_pass(git_buf_printf(&buf, "file://%s", cl_fixture("testrepo.git")));
+	cl_assert_equal_i(false, git_clone__should_clone_local(buf.ptr, GIT_CLONE_LOCAL_AUTO));
+	cl_assert_equal_i(true,  git_clone__should_clone_local(buf.ptr, GIT_CLONE_LOCAL));
+	cl_assert_equal_i(true,  git_clone__should_clone_local(buf.ptr, GIT_CLONE_LOCAL_NO_LINKS));
+	cl_assert_equal_i(false, git_clone__should_clone_local(buf.ptr, GIT_CLONE_NO_LOCAL));
+	git_buf_free(&buf);
+
+	path = cl_fixture("testrepo.git");
+	cl_assert_equal_i(true,  git_clone__should_clone_local(path, GIT_CLONE_LOCAL_AUTO));
+	cl_assert_equal_i(true,  git_clone__should_clone_local(path, GIT_CLONE_LOCAL));
+	cl_assert_equal_i(true,  git_clone__should_clone_local(path, GIT_CLONE_LOCAL_NO_LINKS));
+	cl_assert_equal_i(false, git_clone__should_clone_local(path, GIT_CLONE_NO_LOCAL));
+}
+
+void test_clone_local__hardlinks(void)
+{
+	git_repository *repo;
+	git_remote *remote;
+	git_signature *sig;
+	git_buf buf = GIT_BUF_INIT;
+	struct stat st;
+
+
+	/*
+	 * In this first clone, we just copy over, since the temp dir
+	 * will often be in a different filesystem, so we cannot
+	 * link. It also allows us to control the number of links
+	 */
+	cl_git_pass(git_repository_init(&repo, "./clone.git", true));
+	cl_git_pass(git_remote_create(&remote, repo, "origin", cl_fixture("testrepo.git")));
+	cl_git_pass(git_signature_now(&sig, "foo", "bar"));
+	cl_git_pass(git_clone_local_into(repo, remote, NULL, NULL, false, sig));
+
+	git_remote_free(remote);
+	git_repository_free(repo);
+
+	/* This second clone is in the same filesystem, so we can hardlink */
+
+	cl_git_pass(git_repository_init(&repo, "./clone2.git", true));
+	cl_git_pass(git_buf_puts(&buf, cl_git_path_url("clone.git")));
+	cl_git_pass(git_remote_create(&remote, repo, "origin", buf.ptr));
+	cl_git_pass(git_clone_local_into(repo, remote, NULL, NULL, true, sig));
+
+#ifndef GIT_WIN32
+	git_buf_clear(&buf);
+	cl_git_pass(git_buf_join_n(&buf, '/', 4, git_repository_path(repo), "objects", "08", "b041783f40edfe12bb406c9c9a8a040177c125"));
+
+	cl_git_pass(p_stat(buf.ptr, &st));
+	cl_assert_equal_i(2, st.st_nlink);
+#endif
+
+	git_remote_free(remote);
+	git_repository_free(repo);
+	git_buf_clear(&buf);
+
+	cl_git_pass(git_repository_init(&repo, "./clone3.git", true));
+	cl_git_pass(git_buf_puts(&buf, cl_git_path_url("clone.git")));
+	cl_git_pass(git_remote_create(&remote, repo, "origin", buf.ptr));
+	cl_git_pass(git_clone_local_into(repo, remote, NULL, NULL, false, sig));
+
+	git_buf_clear(&buf);
+	cl_git_pass(git_buf_join_n(&buf, '/', 4, git_repository_path(repo), "objects", "08", "b041783f40edfe12bb406c9c9a8a040177c125"));
+
+	cl_git_pass(p_stat(buf.ptr, &st));
+	cl_assert_equal_i(1, st.st_nlink);
+
+	git_remote_free(remote);
+	git_repository_free(repo);
+
+	/* this one should automatically use links */
+	cl_git_pass(git_clone(&repo, "./clone.git", "./clone4.git", NULL));
+
+#ifndef GIT_WIN32
+	git_buf_clear(&buf);
+	cl_git_pass(git_buf_join_n(&buf, '/', 4, git_repository_path(repo), "objects", "08", "b041783f40edfe12bb406c9c9a8a040177c125"));
+
+	cl_git_pass(p_stat(buf.ptr, &st));
+	cl_assert_equal_i(3, st.st_nlink);
+#endif
+
+	git_buf_free(&buf);
+	git_signature_free(sig);
+	git_repository_free(repo);
+
+	cl_git_pass(git_futils_rmdir_r("./clone.git", NULL, GIT_RMDIR_REMOVE_FILES));
+	cl_git_pass(git_futils_rmdir_r("./clone2.git", NULL, GIT_RMDIR_REMOVE_FILES));
+	cl_git_pass(git_futils_rmdir_r("./clone3.git", NULL, GIT_RMDIR_REMOVE_FILES));
+	cl_git_pass(git_futils_rmdir_r("./clone4.git", NULL, GIT_RMDIR_REMOVE_FILES));
+}

tests/core/copy.c

@@ -45,6 +45,16 @@ void test_core_copy__file_in_dir(void)
 	cl_assert(!git_path_isdir("an_dir"));
 }
 
+void assert_hard_link(const char *path)
+{
+	/* we assert this by checking that there's more than one link to the file */
+	struct stat st;
+
+	cl_assert(git_path_isfile(path));
+	cl_git_pass(p_stat(path, &st));
+	cl_assert(st.st_nlink > 1);
+}
+
 void test_core_copy__tree(void)
 {
 	struct stat st;
@@ -122,5 +132,21 @@ void test_core_copy__tree(void)
 	cl_git_pass(git_futils_rmdir_r("t2", NULL, GIT_RMDIR_REMOVE_FILES));
 	cl_assert(!git_path_isdir("t2"));
 
+#ifndef GIT_WIN32
+	cl_git_pass(git_futils_cp_r("src", "t3", GIT_CPDIR_CREATE_EMPTY_DIRS | GIT_CPDIR_LINK_FILES, 0));
+	cl_assert(git_path_isdir("t3"));
+
+	cl_assert(git_path_isdir("t3"));
+	cl_assert(git_path_isdir("t3/b"));
+	cl_assert(git_path_isdir("t3/c"));
+	cl_assert(git_path_isdir("t3/c/d"));
+	cl_assert(git_path_isdir("t3/c/e"));
+
+	assert_hard_link("t3/f1");
+	assert_hard_link("t3/b/f2");
+	assert_hard_link("t3/c/f3");
+	assert_hard_link("t3/c/d/f4");
+#endif
+
 	cl_git_pass(git_futils_rmdir_r("src", NULL, GIT_RMDIR_REMOVE_FILES));
 }