Merge pull request #1431 from libgit2/autocrlf-fixes

Fix crlf handling, particularly when autocrlf=true

src/blob.c

@@ -12,6 +12,7 @@
 #include "common.h"
 #include "blob.h"
 #include "filter.h"
+#include "buf_text.h"
 
 const void *git_blob_rawcontent(const git_blob *blob)
 {
@@ -221,7 +222,9 @@ int git_blob_create_fromworkdir(git_oid *oid, git_repository *repo, const char *
 		return -1;
 	}
 
-	error = blob_create_internal(oid, repo, git_buf_cstr(&full_path), git_buf_cstr(&full_path), true);
+	error = blob_create_internal(
+		oid, repo, git_buf_cstr(&full_path),
+		git_buf_cstr(&full_path) + strlen(workdir), true);
 
 	git_buf_free(&full_path);
 	return error;
@@ -231,13 +234,21 @@ int git_blob_create_fromdisk(git_oid *oid, git_repository *repo, const char *pat
 {
 	int error;
 	git_buf full_path = GIT_BUF_INIT;
+	const char *workdir, *hintpath;
 
 	if ((error = git_path_prettify(&full_path, path, NULL)) < 0) {
 		git_buf_free(&full_path);
 		return error;
 	}
 
-	error = blob_create_internal(oid, repo, git_buf_cstr(&full_path), git_buf_cstr(&full_path), true);
+	hintpath = git_buf_cstr(&full_path);
+	workdir  = git_repository_workdir(repo);
+
+	if (workdir && !git__prefixcmp(hintpath, workdir))
+		hintpath += strlen(workdir);
+
+	error = blob_create_internal(
+		oid, repo, git_buf_cstr(&full_path), hintpath, true);
 
 	git_buf_free(&full_path);
 	return error;

src/buf_text.c

@@ -60,6 +60,83 @@ void git_buf_text_unescape(git_buf *buf)
 	buf->size = git__unescape(buf->ptr);
 }
 
+int git_buf_text_crlf_to_lf(git_buf *tgt, const git_buf *src)
+{
+	const char *scan = src->ptr;
+	const char *scan_end = src->ptr + src->size;
+	const char *next = memchr(scan, '\r', src->size);
+	char *out;
+
+	assert(tgt != src);
+
+	if (!next)
+		return GIT_ENOTFOUND;
+
+	/* reduce reallocs while in the loop */
+	if (git_buf_grow(tgt, src->size) < 0)
+		return -1;
+	out = tgt->ptr;
+	tgt->size = 0;
+
+	/* Find the next \r and copy whole chunk up to there to tgt */
+	for (; next; scan = next + 1, next = memchr(scan, '\r', scan_end - scan)) {
+		if (next > scan) {
+			size_t copylen = next - scan;
+			memcpy(out, scan, copylen);
+			out += copylen;
+		}
+
+		/* Do not drop \r unless it is followed by \n */
+		if (next[1] != '\n')
+			*out++ = '\r';
+	}
+
+	/* Copy remaining input into dest */
+	memcpy(out, scan, scan_end - scan + 1); /* +1 for NUL byte */
+	out += (scan_end - scan);
+	tgt->size = out - tgt->ptr;
+
+	return 0;
+}
+
+int git_buf_text_lf_to_crlf(git_buf *tgt, const git_buf *src)
+{
+	const char *start = src->ptr;
+	const char *end = start + src->size;
+	const char *scan = start;
+	const char *next = memchr(scan, '\n', src->size);
+
+	assert(tgt != src);
+
+	if (!next)
+		return GIT_ENOTFOUND;
+
+	/* attempt to reduce reallocs while in the loop */
+	if (git_buf_grow(tgt, src->size + (src->size >> 4) + 1) < 0)
+		return -1;
+	tgt->size = 0;
+
+	for (; next; scan = next + 1, next = memchr(scan, '\n', end - scan)) {
+		size_t copylen = next - scan;
+		/* don't convert existing \r\n to \r\r\n */
+		size_t extralen = (next > start && next[-1] == '\r') ? 1 : 2;
+		size_t needsize = tgt->size + copylen + extralen + 1;
+
+		if (tgt->asize < needsize && git_buf_grow(tgt, needsize) < 0)
+			return -1;
+
+		if (next > scan) {
+			memcpy(tgt->ptr + tgt->size, scan, copylen);
+			tgt->size += copylen;
+		}
+		if (extralen == 2)
+			tgt->ptr[tgt->size++] = '\r';
+		tgt->ptr[tgt->size++] = '\n';
+	}
+
+	return git_buf_put(tgt, scan, end - scan);
+}
+
 int git_buf_text_common_prefix(git_buf *buf, const git_strarray *strings)
 {
 	size_t i;

src/buf_text.h

@@ -56,6 +56,20 @@ GIT_INLINE(int) git_buf_text_puts_escape_regex(git_buf *buf, const char *string)
 extern void git_buf_text_unescape(git_buf *buf);
 
 /**
+ * Replace all \r\n with \n (or do nothing if no \r\n are found)
+ *
+ * @return 0 on success, GIT_ENOTFOUND if no \r\n, -1 on memory error
+ */
+extern int git_buf_text_crlf_to_lf(git_buf *tgt, const git_buf *src);
+
+/**
+ * Replace all \n with \r\n (or do nothing if no \n are found)
+ *
+ * @return 0 on success, GIT_ENOTFOUND if no \n, -1 on memory error
+ */
+extern int git_buf_text_lf_to_crlf(git_buf *tgt, const git_buf *src);
+
+/**
  * Fill buffer with the common prefix of a array of strings
  *
  * Buffer will be set to empty if there is no common prefix

src/checkout.c

@@ -23,6 +23,7 @@
 #include "blob.h"
 #include "diff.h"
 #include "pathspec.h"
+#include "buf_text.h"
 
 /* See docs/checkout-internals.md for more information */
 

src/crlf.c

@@ -9,9 +9,10 @@
 #include "fileops.h"
 #include "hash.h"
 #include "filter.h"
+#include "buf_text.h"
 #include "repository.h"
-
 #include "git2/attr.h"
+#include "git2/blob.h"
 
 struct crlf_attrs {
 	int crlf_action;
@@ -21,6 +22,8 @@ struct crlf_attrs {
 struct crlf_filter {
 	git_filter f;
 	struct crlf_attrs attrs;
+	git_repository *repo;
+	char path[GIT_FLEX_ARRAY];
 };
 
 static int check_crlf(const char *value)
@@ -103,36 +106,46 @@ static int crlf_load_attributes(struct crlf_attrs *ca, git_repository *repo, con
 	return -1;
 }
 
-static int drop_crlf(git_buf *dest, const git_buf *source)
+static int has_cr_in_index(git_filter *self)
 {
-	const char *scan = source->ptr, *next;
-	const char *scan_end = git_buf_cstr(source) + git_buf_len(source);
+	struct crlf_filter *filter = (struct crlf_filter *)self;
+	git_index *index;
+	const git_index_entry *entry;
+	git_blob *blob;
+	const void *blobcontent;
+	git_off_t blobsize;
+	bool found_cr;
+
+	if (git_repository_index__weakptr(&index, filter->repo) < 0) {
+		giterr_clear();
+		return false;
+	}
 
-	/* Main scan loop.  Find the next carriage return and copy the
-	 * whole chunk up to that point to the destination buffer.
-	 */
-	while ((next = memchr(scan, '\r', scan_end - scan)) != NULL) {
-		/* copy input up to \r */
-		if (next > scan)
-			git_buf_put(dest, scan, next - scan);
+	if (!(entry = git_index_get_bypath(index, filter->path, 0)) &&
+		!(entry = git_index_get_bypath(index, filter->path, 1)))
+		return false;
 
-		/* Do not drop \r unless it is followed by \n */
-		if (*(next + 1) != '\n')
-			git_buf_putc(dest, '\r');
+	if (!S_ISREG(entry->mode)) /* don't crlf filter non-blobs */
+		return true;
 
-		scan = next + 1;
-	}
+	if (git_blob_lookup(&blob, filter->repo, &entry->oid) < 0)
+		return false;
 
-	/* If there was no \r, then tell the library to skip this filter */
-	if (scan == source->ptr)
-		return -1;
+	blobcontent = git_blob_rawcontent(blob);
+	blobsize    = git_blob_rawsize(blob);
+	if (!git__is_sizet(blobsize))
+		blobsize = (size_t)-1;
+
+	found_cr = (blobcontent != NULL &&
+		blobsize > 0 &&
+		memchr(blobcontent, '\r', (size_t)blobsize) != NULL);
 
-	/* Copy remaining input into dest */
-	git_buf_put(dest, scan, scan_end - scan);
-	return 0;
+	git_blob_free(blob);
+	return found_cr;
 }
 
-static int crlf_apply_to_odb(git_filter *self, git_buf *dest, const git_buf *source)
+static int crlf_apply_to_odb(
+	git_filter *self, git_buf *dest, const git_buf *source)
 {
 	struct crlf_filter *filter = (struct crlf_filter *)self;
 
@@ -162,40 +175,21 @@ static int crlf_apply_to_odb(git_filter *self, git_buf *dest, const git_buf *sou
 		if (stats.cr != stats.crlf)
 			return -1;
 
-#if 0
-		if (crlf_action == CRLF_GUESS) {
+		if (filter->attrs.crlf_action == GIT_CRLF_GUESS) {
 			/*
 			 * If the file in the index has any CR in it, do not convert.
 			 * This is the new safer autocrlf handling.
 			 */
-			if (has_cr_in_index(path))
-				return 0;
+			if (has_cr_in_index(self))
+				return -1;
 		}
-#endif
 
 		if (!stats.cr)
 			return -1;
 	}
 
 	/* Actually drop the carriage returns */
-	return drop_crlf(dest, source);
-}
-
-static int convert_line_endings(git_buf *dest, const git_buf *source, const char *ending)
-{
-	const char *scan = git_buf_cstr(source),
-				  *next,
-				  *scan_end = git_buf_cstr(source) + git_buf_len(source);
-
-	while ((next = memchr(scan, '\n', scan_end - scan)) != NULL) {
-		if (next > scan)
-			git_buf_put(dest, scan, next-scan);
-		git_buf_puts(dest, ending);
-		scan = next + 1;
-	}
-
-	git_buf_put(dest, scan, scan_end - scan);
-	return 0;
+	return git_buf_text_crlf_to_lf(dest, source);
 }
 
 static const char *line_ending(struct crlf_filter *filter)
@@ -238,26 +232,28 @@ line_ending_error:
 	return NULL;
 }
 
-static int crlf_apply_to_workdir(git_filter *self, git_buf *dest, const git_buf *source)
+static int crlf_apply_to_workdir(
+	git_filter *self, git_buf *dest, const git_buf *source)
 {
 	struct crlf_filter *filter = (struct crlf_filter *)self;
 	const char *workdir_ending = NULL;
 
-	assert (self && dest && source);
+	assert(self && dest && source);
 
 	/* Empty file? Nothing to do. */
 	if (git_buf_len(source) == 0)
-		return 0;
+		return -1;
 
 	/* Determine proper line ending */
 	workdir_ending = line_ending(filter);
-	if (!workdir_ending) return -1;
-
-	/* If the line ending is '\n', just copy the input */
-	if (!strcmp(workdir_ending, "\n"))
-		return git_buf_puts(dest, git_buf_cstr(source));
+	if (!workdir_ending)
+		return -1;
+	if (!strcmp("\n", workdir_ending)) /* do nothing for \n ending */
+		return -1;
 
-	return convert_line_endings(dest, source, workdir_ending);
+	/* for now, only lf->crlf conversion is supported here */
+	assert(!strcmp("\r\n", workdir_ending));
+	return git_buf_text_lf_to_crlf(dest, source);
 }
 
 static int find_and_add_filter(
@@ -266,6 +262,7 @@ static int find_and_add_filter(
 {
 	struct crlf_attrs ca;
 	struct crlf_filter *filter;
+	size_t pathlen;
 	int error;
 
 	/* Load gitattributes for the path */
@@ -293,22 +290,27 @@ static int find_and_add_filter(
 
 	/* If we're good, we create a new filter object and push it
 	 * into the filters array */
-	filter = git__malloc(sizeof(struct crlf_filter));
+	pathlen = strlen(path);
+	filter = git__malloc(sizeof(struct crlf_filter) + pathlen + 1);
 	GITERR_CHECK_ALLOC(filter);
 
 	filter->f.apply = apply;
 	filter->f.do_free = NULL;
 	memcpy(&filter->attrs, &ca, sizeof(struct crlf_attrs));
+	filter->repo = repo;
+	memcpy(filter->path, path, pathlen + 1);
 
 	return git_vector_insert(filters, filter);
 }
 
-int git_filter_add__crlf_to_odb(git_vector *filters, git_repository *repo, const char *path)
+int git_filter_add__crlf_to_odb(
+	git_vector *filters, git_repository *repo, const char *path)
 {
 	return find_and_add_filter(filters, repo, path, &crlf_apply_to_odb);
 }
 
-int git_filter_add__crlf_to_workdir(git_vector *filters, git_repository *repo, const char *path)
+int git_filter_add__crlf_to_workdir(
+	git_vector *filters, git_repository *repo, const char *path)
 {
 	return find_and_add_filter(filters, repo, path, &crlf_apply_to_workdir);
 }

src/diff_output.c

@@ -12,6 +12,7 @@
 #include <ctype.h>
 #include "fileops.h"
 #include "filter.h"
+#include "buf_text.h"
 
 static int read_next_int(const char **str, int *value)
 {

src/filter.h

@@ -9,7 +9,6 @@
 
 #include "common.h"
 #include "buffer.h"
-#include "buf_text.h"
 #include "git2/odb.h"
 #include "git2/repository.h"
 

tests-clar/checkout/checkout_helpers.c

@@ -50,35 +50,44 @@ void reset_index_to_treeish(git_object *treeish)
 	git_index_free(index);
 }
 
-static void test_file_contents_internal(
-	const char *path, const char *expectedcontents, bool strip_cr)
+static void check_file_contents_internal(
+	const char *path,
+	const char *expected_content,
+	bool strip_cr,
+	const char *file,
+	int line,
+	const char *msg)
 {
 	int fd;
 	char data[1024] = {0};
 	git_buf buf = GIT_BUF_INIT;
-	size_t expectedlen = strlen(expectedcontents);
+	size_t expected_len = expected_content ? strlen(expected_content) : 0;
 
 	fd = p_open(path, O_RDONLY);
 	cl_assert(fd >= 0);
 
 	buf.ptr = data;
-	buf.size = p_read(fd, buf.ptr, 1024);
+	buf.size = p_read(fd, buf.ptr, sizeof(data));
 
 	cl_git_pass(p_close(fd));
 
 	if (strip_cr)
 		strip_cr_from_buf(&buf);
 
-	cl_assert_equal_i((int)expectedlen, (int)buf.size);
-	cl_assert_equal_s(expectedcontents, buf.ptr);
+	clar__assert_equal_i((int)expected_len, (int)buf.size, file, line, "strlen(expected_content) != strlen(actual_content)", 1);
+	clar__assert_equal_s(expected_content, buf.ptr, file, line, msg, 1);
 }
 
-void test_file_contents(const char *path, const char *expected)
+void check_file_contents_at_line(
+	const char *path, const char *expected,
+	const char *file, int line, const char *msg)
 {
-	test_file_contents_internal(path, expected, false);
+	check_file_contents_internal(path, expected, false, file, line, msg);
 }
 
-void test_file_contents_nocr(const char *path, const char *expected)
+void check_file_contents_nocr_at_line(
+	const char *path, const char *expected,
+	const char *file, int line, const char *msg)
 {
-	test_file_contents_internal(path, expected, true);
+	check_file_contents_internal(path, expected, true, file, line, msg);
 }

tests-clar/checkout/checkout_helpers.h

@@ -5,5 +5,17 @@
 extern void strip_cr_from_buf(git_buf *buf);
 extern void assert_on_branch(git_repository *repo, const char *branch);
 extern void reset_index_to_treeish(git_object *treeish);
-extern void test_file_contents(const char *path, const char *expected);
-extern void test_file_contents_nocr(const char *path, const char *expected);
+
+extern void check_file_contents_at_line(
+	const char *path, const char *expected,
+	const char *file, int line, const char *msg);
+
+extern void check_file_contents_nocr_at_line(
+	const char *path, const char *expected,
+	const char *file, int line, const char *msg);
+
+#define check_file_contents(PATH,EXP) \
+	check_file_contents_at_line(PATH,EXP,__FILE__,__LINE__,"String mismatch: " #EXP " != " #PATH)
+
+#define check_file_contents_nocr(PATH,EXP) \
+	check_file_contents_nocr_at_line(PATH,EXP,__FILE__,__LINE__,"String mismatch: " #EXP " != " #PATH)

tests-clar/checkout/crlf.c

@@ -10,7 +10,9 @@
 #define MORE_CRLF_TEXT_RAW	"crlf\r\ncrlf\r\nlf\ncrlf\r\ncrlf\r\n"
 #define MORE_LF_TEXT_RAW	"lf\nlf\ncrlf\r\nlf\nlf\n"
 
-#define ALL_LF_TEXT_AS_CRLF	"lf\r\nlf\r\nlf\r\nlf\r\nlf\r\n"
+#define ALL_LF_TEXT_AS_CRLF		"lf\r\nlf\r\nlf\r\nlf\r\nlf\r\n"
+#define MORE_CRLF_TEXT_AS_CRLF	"crlf\r\ncrlf\r\nlf\r\ncrlf\r\ncrlf\r\n"
+#define MORE_LF_TEXT_AS_CRLF	"lf\r\nlf\r\ncrlf\r\nlf\r\nlf\r\n"
 
 static git_repository *g_repo;
 
@@ -26,7 +28,6 @@ void test_checkout_crlf__cleanup(void)
 
 void test_checkout_crlf__detect_crlf_autocrlf_false(void)
 {
-#ifdef GIT_WIN32
 	git_checkout_opts opts = GIT_CHECKOUT_OPTS_INIT;
 	opts.checkout_strategy = GIT_CHECKOUT_SAFE_CREATE;
 
@@ -34,13 +35,12 @@ void test_checkout_crlf__detect_crlf_autocrlf_false(void)
 
 	git_checkout_head(g_repo, &opts);
 
-	test_file_contents("./crlf/all-lf", ALL_LF_TEXT_RAW);
-#endif
+	check_file_contents("./crlf/all-lf", ALL_LF_TEXT_RAW);
+	check_file_contents("./crlf/all-crlf", ALL_CRLF_TEXT_RAW);
 }
 
 void test_checkout_crlf__autocrlf_false_index_size_is_unfiltered_size(void)
 {
-#ifdef GIT_WIN32
 	git_index *index;
 	const git_index_entry *entry;
 	git_checkout_opts opts = GIT_CHECKOUT_OPTS_INIT;
@@ -55,13 +55,14 @@ void test_checkout_crlf__autocrlf_false_index_size_is_unfiltered_size(void)
 	cl_assert((entry = git_index_get_bypath(index, "all-lf", 0)) != NULL);
 	cl_assert(entry->file_size == strlen(ALL_LF_TEXT_RAW));
 
+	cl_assert((entry = git_index_get_bypath(index, "all-crlf", 0)) != NULL);
+	cl_assert(entry->file_size == strlen(ALL_CRLF_TEXT_RAW));
+
 	git_index_free(index);
-#endif
 }
 
 void test_checkout_crlf__detect_crlf_autocrlf_true(void)
 {
-#ifdef GIT_WIN32
 	git_checkout_opts opts = GIT_CHECKOUT_OPTS_INIT;
 	opts.checkout_strategy = GIT_CHECKOUT_SAFE_CREATE;
 
@@ -69,13 +70,58 @@ void test_checkout_crlf__detect_crlf_autocrlf_true(void)
 
 	git_checkout_head(g_repo, &opts);
 
-	test_file_contents("./crlf/all-lf", ALL_LF_TEXT_AS_CRLF);
-#endif
+	if (GIT_EOL_NATIVE == GIT_EOL_LF)
+		check_file_contents("./crlf/all-lf", ALL_LF_TEXT_RAW);
+	else
+		check_file_contents("./crlf/all-lf", ALL_LF_TEXT_AS_CRLF);
+
+	check_file_contents("./crlf/all-crlf", ALL_CRLF_TEXT_RAW);
+}
+
+void test_checkout_crlf__more_lf_autocrlf_true(void)
+{
+	git_checkout_opts opts = GIT_CHECKOUT_OPTS_INIT;
+	opts.checkout_strategy = GIT_CHECKOUT_SAFE_CREATE;
+
+	cl_repo_set_bool(g_repo, "core.autocrlf", true);
+
+	git_checkout_head(g_repo, &opts);
+
+	if (GIT_EOL_NATIVE == GIT_EOL_LF)
+		check_file_contents("./crlf/more-lf", MORE_LF_TEXT_RAW);
+	else
+		check_file_contents("./crlf/more-lf", MORE_LF_TEXT_AS_CRLF);
+}
+
+void test_checkout_crlf__more_crlf_autocrlf_true(void)
+{
+	git_checkout_opts opts = GIT_CHECKOUT_OPTS_INIT;
+	opts.checkout_strategy = GIT_CHECKOUT_SAFE_CREATE;
+
+	cl_repo_set_bool(g_repo, "core.autocrlf", true);
+
+	git_checkout_head(g_repo, &opts);
+
+	if (GIT_EOL_NATIVE == GIT_EOL_LF)
+		check_file_contents("./crlf/more-crlf", MORE_CRLF_TEXT_RAW);
+	else
+		check_file_contents("./crlf/more-crlf", MORE_CRLF_TEXT_AS_CRLF);
+}
+
+void test_checkout_crlf__all_crlf_autocrlf_true(void)
+{
+	git_checkout_opts opts = GIT_CHECKOUT_OPTS_INIT;
+	opts.checkout_strategy = GIT_CHECKOUT_SAFE_CREATE;
+
+	cl_repo_set_bool(g_repo, "core.autocrlf", true);
+
+	git_checkout_head(g_repo, &opts);
+
+	check_file_contents("./crlf/all-crlf", ALL_CRLF_TEXT_RAW);
 }
 
 void test_checkout_crlf__autocrlf_true_index_size_is_filtered_size(void)
 {
-#ifdef GIT_WIN32
 	git_index *index;
 	const git_index_entry *entry;
 	git_checkout_opts opts = GIT_CHECKOUT_OPTS_INIT;
@@ -88,8 +134,14 @@ void test_checkout_crlf__autocrlf_true_index_size_is_filtered_size(void)
 	git_repository_index(&index, g_repo);
 
 	cl_assert((entry = git_index_get_bypath(index, "all-lf", 0)) != NULL);
-	cl_assert(entry->file_size == strlen(ALL_LF_TEXT_AS_CRLF));
+
+	if (GIT_EOL_NATIVE == GIT_EOL_LF)
+		cl_assert_equal_sz(strlen(ALL_LF_TEXT_RAW), entry->file_size);
+	else
+		cl_assert_equal_sz(strlen(ALL_LF_TEXT_AS_CRLF), entry->file_size);
+
+	cl_assert((entry = git_index_get_bypath(index, "all-crlf", 0)) != NULL);
+	cl_assert_equal_sz(strlen(ALL_CRLF_TEXT_RAW), entry->file_size);
 
 	git_index_free(index);
-#endif
 }

tests-clar/checkout/index.c

@@ -48,9 +48,9 @@ void test_checkout_index__can_create_missing_files(void)
 
 	cl_git_pass(git_checkout_index(g_repo, NULL, &opts));
 
-	test_file_contents("./testrepo/README", "hey there\n");
-	test_file_contents("./testrepo/branch_file.txt", "hi\nbye!\n");
-	test_file_contents("./testrepo/new.txt", "my new file\n");
+	check_file_contents("./testrepo/README", "hey there\n");
+	check_file_contents("./testrepo/branch_file.txt", "hi\nbye!\n");
+	check_file_contents("./testrepo/new.txt", "my new file\n");
 }
 
 void test_checkout_index__can_remove_untracked_files(void)
@@ -88,8 +88,8 @@ void test_checkout_index__honor_the_specified_pathspecs(void)
 	cl_git_pass(git_checkout_index(g_repo, NULL, &opts));
 
 	cl_assert_equal_i(false, git_path_isfile("./testrepo/README"));
-	test_file_contents("./testrepo/branch_file.txt", "hi\nbye!\n");
-	test_file_contents("./testrepo/new.txt", "my new file\n");
+	check_file_contents("./testrepo/branch_file.txt", "hi\nbye!\n");
+	check_file_contents("./testrepo/new.txt", "my new file\n");
 }
 
 void test_checkout_index__honor_the_gitattributes_directives(void)
@@ -106,9 +106,9 @@ void test_checkout_index__honor_the_gitattributes_directives(void)
 
 	cl_git_pass(git_checkout_index(g_repo, NULL, &opts));
 
-	test_file_contents("./testrepo/README", "hey there\n");
-	test_file_contents("./testrepo/new.txt", "my new file\n");
-	test_file_contents("./testrepo/branch_file.txt", "hi\r\nbye!\r\n");
+	check_file_contents("./testrepo/README", "hey there\n");
+	check_file_contents("./testrepo/new.txt", "my new file\n");
+	check_file_contents("./testrepo/branch_file.txt", "hi\r\nbye!\r\n");
 }
 
 void test_checkout_index__honor_coreautocrlf_setting_set_to_true(void)
@@ -124,7 +124,7 @@ void test_checkout_index__honor_coreautocrlf_setting_set_to_true(void)
 
 	cl_git_pass(git_checkout_index(g_repo, NULL, &opts));
 
-	test_file_contents("./testrepo/README", expected_readme_text);
+	check_file_contents("./testrepo/README", expected_readme_text);
 #endif
 }
 
@@ -139,7 +139,7 @@ void test_checkout_index__honor_coresymlinks_setting_set_to_true(void)
 	cl_git_pass(git_checkout_index(g_repo, NULL, &opts));
 
 #ifdef GIT_WIN32
-	test_file_contents("./testrepo/link_to_new.txt", "new.txt");
+	check_file_contents("./testrepo/link_to_new.txt", "new.txt");
 #else
 	{
 		char link_data[1024];
@@ -149,7 +149,7 @@ void test_checkout_index__honor_coresymlinks_setting_set_to_true(void)
 		link_data[link_size] = '\0';
 		cl_assert_equal_i(link_size, strlen("new.txt"));
 		cl_assert_equal_s(link_data, "new.txt");
-		test_file_contents("./testrepo/link_to_new.txt", "my new file\n");
+		check_file_contents("./testrepo/link_to_new.txt", "my new file\n");
 	}
 #endif
 }
@@ -164,7 +164,7 @@ void test_checkout_index__honor_coresymlinks_setting_set_to_false(void)
 
 	cl_git_pass(git_checkout_index(g_repo, NULL, &opts));
 
-	test_file_contents("./testrepo/link_to_new.txt", "new.txt");
+	check_file_contents("./testrepo/link_to_new.txt", "new.txt");
 }
 
 void test_checkout_index__donot_overwrite_modified_file_by_default(void)
@@ -180,7 +180,7 @@ void test_checkout_index__donot_overwrite_modified_file_by_default(void)
 
 	cl_git_pass(git_checkout_index(g_repo, NULL, &opts));
 
-	test_file_contents("./testrepo/new.txt", "This isn't what's stored!");
+	check_file_contents("./testrepo/new.txt", "This isn't what's stored!");
 }
 
 void test_checkout_index__can_overwrite_modified_file(void)
@@ -193,7 +193,7 @@ void test_checkout_index__can_overwrite_modified_file(void)
 
 	cl_git_pass(git_checkout_index(g_repo, NULL, &opts));
 
-	test_file_contents("./testrepo/new.txt", "my new file\n");
+	check_file_contents("./testrepo/new.txt", "my new file\n");
 }
 
 void test_checkout_index__options_disable_filters(void)
@@ -207,14 +207,14 @@ void test_checkout_index__options_disable_filters(void)
 
 	cl_git_pass(git_checkout_index(g_repo, NULL, &opts));
 
-	test_file_contents("./testrepo/new.txt", "my new file\r\n");
+	check_file_contents("./testrepo/new.txt", "my new file\r\n");
 
 	p_unlink("./testrepo/new.txt");
 
 	opts.disable_filters = true;
 	cl_git_pass(git_checkout_index(g_repo, NULL, &opts));
 
-	test_file_contents("./testrepo/new.txt", "my new file\n");
+	check_file_contents("./testrepo/new.txt", "my new file\n");
 }
 
 void test_checkout_index__options_dir_modes(void)
@@ -274,7 +274,7 @@ void test_checkout_index__options_open_flags(void)
 	opts.checkout_strategy = GIT_CHECKOUT_FORCE;
 	cl_git_pass(git_checkout_index(g_repo, NULL, &opts));
 
-	test_file_contents("./testrepo/new.txt", "hi\nmy new file\n");
+	check_file_contents("./testrepo/new.txt", "hi\nmy new file\n");
 }
 
 struct notify_data {
@@ -469,9 +469,9 @@ void test_checkout_index__can_update_prefixed_files(void)
 	/* remove untracked will remove the .gitattributes file before the blobs
 	 * were created, so they will have had crlf filtering applied on Windows
 	 */
-	test_file_contents_nocr("./testrepo/README", "hey there\n");
-	test_file_contents_nocr("./testrepo/branch_file.txt", "hi\nbye!\n");
-	test_file_contents_nocr("./testrepo/new.txt", "my new file\n");
+	check_file_contents_nocr("./testrepo/README", "hey there\n");
+	check_file_contents_nocr("./testrepo/branch_file.txt", "hi\nbye!\n");
+	check_file_contents_nocr("./testrepo/new.txt", "my new file\n");
 
 	cl_assert(!git_path_exists("testrepo/READ"));
 	cl_assert(!git_path_exists("testrepo/README.after"));
@@ -488,3 +488,20 @@ void test_checkout_index__can_checkout_a_newly_initialized_repository(void)
 
 	cl_git_pass(git_checkout_index(g_repo, NULL, NULL));
 }
+
+void test_checkout_index__issue_1397(void)
+{
+	git_checkout_opts opts = GIT_CHECKOUT_OPTS_INIT;
+
+	test_checkout_index__cleanup();
+
+	g_repo = cl_git_sandbox_init("issue_1397");
+
+	cl_repo_set_bool(g_repo, "core.autocrlf", true);
+
+	opts.checkout_strategy = GIT_CHECKOUT_FORCE;
+
+	cl_git_pass(git_checkout_index(g_repo, NULL, &opts));
+
+	check_file_contents("./issue_1397/crlf_file.txt", "first line\r\nsecond line\r\nboth with crlf");
+}

tests-clar/checkout/tree.c

@@ -248,7 +248,7 @@ void test_checkout_tree__can_update_only(void)
 
 	cl_assert(!git_path_isdir("testrepo/a"));
 
-	test_file_contents_nocr("testrepo/branch_file.txt", "hi\nbye!\n");
+	check_file_contents_nocr("testrepo/branch_file.txt", "hi\nbye!\n");
 
 	/* now checkout branch but with update only */
 
@@ -269,7 +269,7 @@ void test_checkout_tree__can_update_only(void)
 	cl_assert(!git_path_isdir("testrepo/a"));
 
 	/* but this file still should have been updated */
-	test_file_contents_nocr("testrepo/branch_file.txt", "hi\n");
+	check_file_contents_nocr("testrepo/branch_file.txt", "hi\n");
 
 	git_object_free(obj);
 }
@@ -481,3 +481,26 @@ void test_checkout_tree__can_checkout_with_last_workdir_item_missing(void)
 	git_commit_free(commit);
 	git_index_free(index);
 }
+
+void test_checkout_tree__issue_1397(void)
+{
+	git_checkout_opts opts = GIT_CHECKOUT_OPTS_INIT;
+	const char *partial_oid = "8a7ef04";
+	git_object *tree = NULL;
+
+	test_checkout_tree__cleanup(); /* cleanup default checkout */
+
+	g_repo = cl_git_sandbox_init("issue_1397");
+
+	cl_repo_set_bool(g_repo, "core.autocrlf", true);
+
+	cl_git_pass(git_revparse_single(&tree, g_repo, partial_oid));
+
+	opts.checkout_strategy = GIT_CHECKOUT_FORCE;
+
+	cl_git_pass(git_checkout_tree(g_repo, tree, &opts));
+
+	check_file_contents("./issue_1397/crlf_file.txt", "first line\r\nsecond line\r\nboth with crlf");
+
+	git_object_free(tree);
+}

tests-clar/core/buffer.c

@@ -904,3 +904,85 @@ void test_core_buffer__similarity_metric_whitespace(void)
 
 	git_buf_free(&buf);
 }
+
+#define check_buf(expected,buf) do { \
+	cl_assert_equal_s(expected, buf.ptr); \
+	cl_assert_equal_sz(strlen(expected), buf.size); } while (0)
+
+void test_core_buffer__lf_and_crlf_conversions(void)
+{
+	git_buf src = GIT_BUF_INIT, tgt = GIT_BUF_INIT;
+
+	/* LF source */
+
+	git_buf_sets(&src, "lf\nlf\nlf\nlf\n");
+
+	cl_git_pass(git_buf_text_lf_to_crlf(&tgt, &src));
+	check_buf("lf\r\nlf\r\nlf\r\nlf\r\n", tgt);
+
+	cl_assert_equal_i(GIT_ENOTFOUND, git_buf_text_crlf_to_lf(&tgt, &src));
+	/* no conversion needed if all LFs already */
+
+	git_buf_sets(&src, "\nlf\nlf\nlf\nlf\nlf");
+
+	cl_git_pass(git_buf_text_lf_to_crlf(&tgt, &src));
+	check_buf("\r\nlf\r\nlf\r\nlf\r\nlf\r\nlf", tgt);
+
+	cl_assert_equal_i(GIT_ENOTFOUND, git_buf_text_crlf_to_lf(&tgt, &src));
+	/* no conversion needed if all LFs already */
+
+	/* CRLF source */
+
+	git_buf_sets(&src, "crlf\r\ncrlf\r\ncrlf\r\ncrlf\r\n");
+
+	cl_git_pass(git_buf_text_lf_to_crlf(&tgt, &src));
+	check_buf("crlf\r\ncrlf\r\ncrlf\r\ncrlf\r\n", tgt);
+	check_buf(src.ptr, tgt);
+
+	cl_git_pass(git_buf_text_crlf_to_lf(&tgt, &src));
+	check_buf("crlf\ncrlf\ncrlf\ncrlf\n", tgt);
+
+	git_buf_sets(&src, "\r\ncrlf\r\ncrlf\r\ncrlf\r\ncrlf\r\ncrlf");
+
+	cl_git_pass(git_buf_text_lf_to_crlf(&tgt, &src));
+	check_buf("\r\ncrlf\r\ncrlf\r\ncrlf\r\ncrlf\r\ncrlf", tgt);
+	check_buf(src.ptr, tgt);
+
+	cl_git_pass(git_buf_text_crlf_to_lf(&tgt, &src));
+	check_buf("\ncrlf\ncrlf\ncrlf\ncrlf\ncrlf", tgt);
+
+	/* CRLF in LF text */
+
+	git_buf_sets(&src, "\nlf\nlf\ncrlf\r\nlf\nlf\ncrlf\r\n");
+
+	cl_git_pass(git_buf_text_lf_to_crlf(&tgt, &src));
+	check_buf("\r\nlf\r\nlf\r\ncrlf\r\nlf\r\nlf\r\ncrlf\r\n", tgt);
+	cl_git_pass(git_buf_text_crlf_to_lf(&tgt, &src));
+	check_buf("\nlf\nlf\ncrlf\nlf\nlf\ncrlf\n", tgt);
+
+	/* LF in CRLF text */
+
+	git_buf_sets(&src, "\ncrlf\r\ncrlf\r\nlf\ncrlf\r\ncrlf");
+
+	cl_git_pass(git_buf_text_lf_to_crlf(&tgt, &src));
+	check_buf("\r\ncrlf\r\ncrlf\r\nlf\r\ncrlf\r\ncrlf", tgt);
+	cl_git_pass(git_buf_text_crlf_to_lf(&tgt, &src));
+	check_buf("\ncrlf\ncrlf\nlf\ncrlf\ncrlf", tgt);
+
+	/* bare CR test */
+
+	git_buf_sets(&src, "\rcrlf\r\nlf\nlf\ncr\rcrlf\r\nlf\ncr\r");
+
+	cl_git_pass(git_buf_text_lf_to_crlf(&tgt, &src));
+	check_buf("\rcrlf\r\nlf\r\nlf\r\ncr\rcrlf\r\nlf\r\ncr\r", tgt);
+	cl_git_pass(git_buf_text_crlf_to_lf(&tgt, &src));
+	check_buf("\rcrlf\nlf\nlf\ncr\rcrlf\nlf\ncr\r", tgt);
+
+	git_buf_sets(&src, "\rcr\r");
+	cl_assert_equal_i(GIT_ENOTFOUND, git_buf_text_lf_to_crlf(&tgt, &src));
+	cl_git_pass(git_buf_text_crlf_to_lf(&tgt, &src));
+	check_buf("\rcr\r", tgt);
+
+	git_buf_free(&src);
+	git_buf_free(&tgt);
+}

tests-clar/diff/tree.c

@@ -431,3 +431,26 @@ void test_diff_tree__regular_blob_mode_changed_to_executable_file(void)
 	cl_assert_equal_i(0, expect.file_status[GIT_DELTA_ADDED]);
 	cl_assert_equal_i(0, expect.file_status[GIT_DELTA_TYPECHANGE]);
 }
+
+void test_diff_tree__issue_1397(void)
+{
+	/* this test shows that it is not needed */
+
+	g_repo = cl_git_sandbox_init("issue_1397");
+
+	cl_repo_set_bool(g_repo, "core.autocrlf", true);
+
+	cl_assert((a = resolve_commit_oid_to_tree(g_repo, "8a7ef04")) != NULL);
+	cl_assert((b = resolve_commit_oid_to_tree(g_repo, "7f483a7")) != NULL);
+
+	cl_git_pass(git_diff_tree_to_tree(&diff, g_repo, a, b, &opts));
+
+	cl_git_pass(git_diff_foreach(
+		diff, diff_file_cb, diff_hunk_cb, diff_line_cb, &expect));
+
+	cl_assert_equal_i(1, expect.files);
+	cl_assert_equal_i(0, expect.file_status[GIT_DELTA_DELETED]);
+	cl_assert_equal_i(1, expect.file_status[GIT_DELTA_MODIFIED]);
+	cl_assert_equal_i(0, expect.file_status[GIT_DELTA_ADDED]);
+	cl_assert_equal_i(0, expect.file_status[GIT_DELTA_TYPECHANGE]);
+}

tests-clar/diff/workdir.c

@@ -1084,3 +1084,99 @@ void test_diff_workdir__can_diff_empty_file(void)
 	git_diff_patch_free(patch);
 	git_diff_list_free(diff);
 }
+
+void test_diff_workdir__to_index_issue_1397(void)
+{
+	git_diff_options opts = GIT_DIFF_OPTIONS_INIT;
+	git_diff_list *diff = NULL;
+	diff_expects exp;
+
+	g_repo = cl_git_sandbox_init("issue_1397");
+
+	cl_repo_set_bool(g_repo, "core.autocrlf", true);
+
+	opts.context_lines = 3;
+	opts.interhunk_lines = 1;
+
+	cl_git_pass(git_diff_index_to_workdir(&diff, g_repo, NULL, &opts));
+
+	memset(&exp, 0, sizeof(exp));
+	cl_git_pass(git_diff_foreach(
+		diff, diff_file_cb, diff_hunk_cb, diff_line_cb, &exp));
+
+	cl_assert_equal_i(0, exp.files);
+	cl_assert_equal_i(0, exp.hunks);
+	cl_assert_equal_i(0, exp.lines);
+
+	git_diff_list_free(diff);
+	diff = NULL;
+
+	cl_git_rewritefile("issue_1397/crlf_file.txt",
+		"first line\r\nsecond line modified\r\nboth with crlf");
+
+	cl_git_pass(git_diff_index_to_workdir(&diff, g_repo, NULL, &opts));
+
+	memset(&exp, 0, sizeof(exp));
+	cl_git_pass(git_diff_foreach(
+		diff, diff_file_cb, diff_hunk_cb, diff_line_cb, &exp));
+
+	cl_assert_equal_i(1, exp.files);
+	cl_assert_equal_i(1, exp.file_status[GIT_DELTA_MODIFIED]);
+
+	cl_assert_equal_i(1, exp.hunks);
+
+	cl_assert_equal_i(5, exp.lines);
+	cl_assert_equal_i(3, exp.line_ctxt);
+	cl_assert_equal_i(1, exp.line_adds);
+	cl_assert_equal_i(1, exp.line_dels);
+
+	git_diff_list_free(diff);
+}
+
+void test_diff_workdir__to_tree_issue_1397(void)
+{
+	const char *a_commit = "7f483a738"; /* the current HEAD */
+	git_tree *a;
+	git_diff_options opts = GIT_DIFF_OPTIONS_INIT;
+	git_diff_list *diff = NULL;
+	git_diff_list *diff2 = NULL;
+	diff_expects exp;
+
+	g_repo = cl_git_sandbox_init("issue_1397");
+
+	cl_repo_set_bool(g_repo, "core.autocrlf", true);
+
+	a = resolve_commit_oid_to_tree(g_repo, a_commit);
+
+	opts.context_lines = 3;
+	opts.interhunk_lines = 1;
+
+	cl_git_pass(git_diff_tree_to_workdir(&diff, g_repo, a, &opts));
+
+	memset(&exp, 0, sizeof(exp));
+	cl_git_pass(git_diff_foreach(
+		diff, diff_file_cb, diff_hunk_cb, diff_line_cb, &exp));
+
+	cl_assert_equal_i(0, exp.files);
+	cl_assert_equal_i(0, exp.hunks);
+	cl_assert_equal_i(0, exp.lines);
+
+	git_diff_list_free(diff);
+	diff = NULL;
+
+	cl_git_pass(git_diff_tree_to_index(&diff, g_repo, a, NULL, &opts));
+	cl_git_pass(git_diff_index_to_workdir(&diff2, g_repo, NULL, &opts));
+	cl_git_pass(git_diff_merge(diff, diff2));
+	git_diff_list_free(diff2);
+
+	memset(&exp, 0, sizeof(exp));
+	cl_git_pass(git_diff_foreach(
+		diff, diff_file_cb, diff_hunk_cb, diff_line_cb, &exp));
+
+	cl_assert_equal_i(0, exp.files);
+	cl_assert_equal_i(0, exp.hunks);
+	cl_assert_equal_i(0, exp.lines);
+
+	git_diff_list_free(diff);
+	git_tree_free(a);
+}

tests-clar/index/tests.c

@@ -251,6 +251,48 @@ void test_index_tests__add(void)
 	git_repository_free(repo);
 }
 
+static void cleanup_1397(void *opaque)
+{
+	GIT_UNUSED(opaque);
+	cl_git_sandbox_cleanup();
+}
+
+void test_index_tests__add_issue_1397(void)
+{
+	git_index *index;
+	git_repository *repo;
+	const git_index_entry *entry;
+	git_oid id1;
+
+	cl_set_cleanup(&cleanup_1397, NULL);
+
+	repo = cl_git_sandbox_init("issue_1397");
+
+	cl_repo_set_bool(repo, "core.autocrlf", true);
+
+	/* Ensure we're the only guy in the room */
+	cl_git_pass(git_repository_index(&index, repo));
+
+	/* Store the expected hash of the file/blob
+	 * This has been generated by executing the following
+	 * $ git hash-object crlf_file.txt
+	 */
+	cl_git_pass(git_oid_fromstr(&id1, "8312e0889a9cbab77c732b6bc39b51a683e3a318"));
+
+	/* Make sure the initial SHA-1 is correct */
+	cl_assert((entry = git_index_get_bypath(index, "crlf_file.txt", 0)) != NULL);
+	cl_assert_(git_oid_cmp(&id1, &entry->oid) == 0, "first oid check");
+
+	/* Update the index */
+	cl_git_pass(git_index_add_bypath(index, "crlf_file.txt"));
+
+	/* Check the new SHA-1 */
+	cl_assert((entry = git_index_get_bypath(index, "crlf_file.txt", 0)) != NULL);
+	cl_assert_(git_oid_cmp(&id1, &entry->oid) == 0, "second oid check");
+
+	git_index_free(index);
+}
+
 void test_index_tests__add_bypath_to_a_bare_repository_returns_EBAREPO(void)
 {
 	git_repository *bare_repo;

tests-clar/object/blob/filter.c

@@ -2,6 +2,7 @@
 #include "posix.h"
 #include "blob.h"
 #include "filter.h"
+#include "buf_text.h"
 
 static git_repository *g_repo = NULL;
 #define NUM_TEST_OBJECTS 8

tests-clar/resources/issue_1397/.gitted/HEAD

+ref: refs/heads/master

tests-clar/resources/issue_1397/.gitted/config

+[core]
+	bare = false
+	repositoryformatversion = 0
+	filemode = false
+	logallrefupdates = true
+	ignorecase = true

tests-clar/resources/issue_1397/.gitted/refs/heads/master

+7f483a738f867e5b21c8f377d70311f011eb48b5

tests-clar/resources/issue_1397/crlf_file.txt

+first line
+second line
+both with crlf
\ No newline at end of file

tests-clar/resources/issue_1397/some_other_crlf_file.txt

+first line
+second line with some change
+both with crlf
\ No newline at end of file

tests-clar/status/worktree.c

@@ -541,6 +541,18 @@ void test_status_worktree__line_endings_dont_count_as_changes_with_autocrlf(void
 	cl_assert_equal_i(GIT_STATUS_CURRENT, status);
 }
 
+void test_status_worktree__line_endings_dont_count_as_changes_with_autocrlf_issue_1397(void)
+{
+	git_repository *repo = cl_git_sandbox_init("issue_1397");
+	unsigned int status;
+
+	cl_repo_set_bool(repo, "core.autocrlf", true);
+
+	cl_git_pass(git_status_file(&status, repo, "crlf_file.txt"));
+
+	cl_assert_equal_i(GIT_STATUS_CURRENT, status);
+}
+
 void test_status_worktree__conflicted_item(void)
 {
 	git_repository *repo = cl_git_sandbox_init("status");