Merge pull request #1233 from arrbee/file-similarity-metric

Add file similarity scoring to diff rename/copy detection

include/git2/diff.h

@@ -88,10 +88,9 @@ typedef enum {
 	GIT_DIFF_INCLUDE_UNTRACKED = (1 << 8),
 	/** Include unmodified files in the diff list */
 	GIT_DIFF_INCLUDE_UNMODIFIED = (1 << 9),
-	/** Even with the GIT_DIFF_INCLUDE_UNTRACKED flag, when an untracked
-	 *  directory is found, only a single entry for the directory is added
-	 *  to the diff list; with this flag, all files under the directory will
-	 *  be included, too.
+	/** Even with GIT_DIFF_INCLUDE_UNTRACKED, an entire untracked directory
+	 *  will be marked with only a single entry in the diff list; this flag
+	 *  adds all files under the directory as UNTRACKED entries, too.
 	 */
 	GIT_DIFF_RECURSE_UNTRACKED_DIRS = (1 << 10),
 	/** If the pathspec is set in the diff options, this flags means to
@@ -120,6 +119,11 @@ typedef enum {
 	GIT_DIFF_INCLUDE_TYPECHANGE_TREES  = (1 << 16),
 	/** Ignore file mode changes */
 	GIT_DIFF_IGNORE_FILEMODE = (1 << 17),
+	/** Even with GIT_DIFF_INCLUDE_IGNORED, an entire ignored directory
+	 *  will be marked with only a single entry in the diff list; this flag
+	 *  adds all files under the directory as IGNORED entries, too.
+	 */
+	GIT_DIFF_RECURSE_IGNORED_DIRS = (1 << 10),
 } git_diff_option_t;
 
 /**
@@ -133,20 +137,18 @@ typedef enum {
 typedef struct git_diff_list git_diff_list;
 
 /**
- * Flags for the file object on each side of a diff.
+ * Flags for the delta object and the file objects on each side.
  *
- * Note: most of these flags are just for **internal** consumption by
- * libgit2, but some of them may be interesting to external users.
+ * These flags are used for both the `flags` value of the `git_diff_delta`
+ * and the flags for the `git_diff_file` objects representing the old and
+ * new sides of the delta.  Values outside of this public range should be
+ * considered reserved for internal or future use.
  */
 typedef enum {
-	GIT_DIFF_FILE_VALID_OID  = (1 << 0), /** `oid` value is known correct */
-	GIT_DIFF_FILE_FREE_PATH  = (1 << 1), /** `path` is allocated memory */
-	GIT_DIFF_FILE_BINARY     = (1 << 2), /** should be considered binary data */
-	GIT_DIFF_FILE_NOT_BINARY = (1 << 3), /** should be considered text data */
-	GIT_DIFF_FILE_FREE_DATA  = (1 << 4), /** internal file data is allocated */
-	GIT_DIFF_FILE_UNMAP_DATA = (1 << 5), /** internal file data is mmap'ed */
-	GIT_DIFF_FILE_NO_DATA    = (1 << 6), /** file data should not be loaded */
-} git_diff_file_flag_t;
+	GIT_DIFF_FLAG_BINARY     = (1 << 0), /** file(s) treated as binary data */
+	GIT_DIFF_FLAG_NOT_BINARY = (1 << 1), /** file(s) treated as text data */
+	GIT_DIFF_FLAG_VALID_OID  = (1 << 2), /** `oid` value is known correct */
+} git_diff_flag_t;
 
 /**
  * What type of change is described by a git_diff_delta?
@@ -186,18 +188,17 @@ typedef enum {
  *
  * `size` is the size of the entry in bytes.
  *
- * `flags` is a combination of the `git_diff_file_flag_t` types, but those
- * are largely internal values.
+ * `flags` is a combination of the `git_diff_flag_t` types
  *
  * `mode` is, roughly, the stat() `st_mode` value for the item.  This will
  * be restricted to one of the `git_filemode_t` values.
  */
 typedef struct {
-	git_oid oid;
+	git_oid     oid;
 	const char *path;
-	git_off_t size;
-	unsigned int flags;
-	uint16_t mode;
+	git_off_t   size;
+	uint32_t    flags;
+	uint16_t    mode;
 } git_diff_file;
 
 /**
@@ -219,16 +220,17 @@ typedef struct {
  *
  * Under some circumstances, in the name of efficiency, not all fields will
  * be filled in, but we generally try to fill in as much as possible.  One
- * example is that the "binary" field will not examine file contents if you
- * do not pass in hunk and/or line callbacks to the diff foreach iteration
- * function.  It will just use the git attributes for those files.
+ * example is that the "flags" field may not have either the `BINARY` or the
+ * `NOT_BINARY` flag set to avoid examining file contents if you do not pass
+ * in hunk and/or line callbacks to the diff foreach iteration function.  It
+ * will just use the git attributes for those files.
  */
 typedef struct {
 	git_diff_file old_file;
 	git_diff_file new_file;
 	git_delta_t   status;
-	unsigned int  similarity; /**< for RENAMED and COPIED, value 0-100 */
-	int           binary;
+	uint32_t      similarity; /**< for RENAMED and COPIED, value 0-100 */
+	uint32_t      flags;
 } git_diff_delta;
 
 /**
@@ -377,7 +379,7 @@ typedef struct git_diff_patch git_diff_patch;
 typedef enum {
 	/** look for renames? (`--find-renames`) */
 	GIT_DIFF_FIND_RENAMES = (1 << 0),
-	/** consider old size of modified for renames? (`--break-rewrites=N`) */
+	/** consider old side of modified for renames? (`--break-rewrites=N`) */
 	GIT_DIFF_FIND_RENAMES_FROM_REWRITES = (1 << 1),
 
 	/** look for copies? (a la `--find-copies`) */
@@ -387,10 +389,49 @@ typedef enum {
 
 	/** split large rewrites into delete/add pairs (`--break-rewrites=/M`) */
 	GIT_DIFF_FIND_AND_BREAK_REWRITES = (1 << 4),
+
+	/** turn on all finding features */
+	GIT_DIFF_FIND_ALL = (0x1f),
+
+	/** measure similarity ignoring leading whitespace (default) */
+	GIT_DIFF_FIND_IGNORE_LEADING_WHITESPACE = 0,
+	/** measure similarity ignoring all whitespace */
+	GIT_DIFF_FIND_IGNORE_WHITESPACE = (1 << 6),
+	/** measure similarity including all data */
+	GIT_DIFF_FIND_DONT_IGNORE_WHITESPACE = (1 << 7),
 } git_diff_find_t;
 
 /**
+ * Pluggable similarity metric
+ */
+typedef struct {
+	int (*file_signature)(
+		void **out, const git_diff_file *file,
+		const char *fullpath, void *payload);
+	int (*buffer_signature)(
+		void **out, const git_diff_file *file,
+		const char *buf, size_t buflen, void *payload);
+	void (*free_signature)(void *sig, void *payload);
+	int (*similarity)(int *score, void *siga, void *sigb, void *payload);
+	void *payload;
+} git_diff_similarity_metric;
+
+/**
  * Control behavior of rename and copy detection
+ *
+ * These options mostly mimic parameters that can be passed to git-diff.
+ *
+ * - `rename_threshold` is the same as the -M option with a value
+ * - `copy_threshold` is the same as the -C option with a value
+ * - `rename_from_rewrite_threshold` matches the top of the -B option
+ * - `break_rewrite_threshold` matches the bottom of the -B option
+ * - `target_limit` matches the -l option
+ *
+ * The `metric` option allows you to plug in a custom similarity metric.
+ * Set it to NULL for the default internal metric which is based on sampling
+ * hashes of ranges of data in the file.  The default metric is a pretty
+ * good similarity approximation that should work fairly well for both text
+ * and binary data, and is pretty fast with fixed memory overhead.
  */
 typedef struct {
 	unsigned int version;
@@ -411,6 +452,9 @@ typedef struct {
 	 *  the `diff.renameLimit` config) (default 200)
 	 */
 	unsigned int target_limit;
+
+	/** Pluggable similarity metric; pass NULL to use internal metric */
+	git_diff_similarity_metric *metric;
 } git_diff_find_options;
 
 #define GIT_DIFF_FIND_OPTIONS_VERSION 1
@@ -856,11 +900,12 @@ GIT_EXTERN(int) git_diff_patch_to_str(
  *
  * NULL is allowed for either `old_blob` or `new_blob` and will be treated
  * as an empty blob, with the `oid` set to NULL in the `git_diff_file` data.
+ * Passing NULL for both blobs is a noop; no callbacks will be made at all.
  *
- * We do run a binary content check on the two blobs and if either of the
- * blobs looks like binary data, the `git_diff_delta` binary attribute will
- * be set to 1 and no call to the hunk_cb nor line_cb will be made (unless
- * you pass `GIT_DIFF_FORCE_TEXT` of course).
+ * We do run a binary content check on the blob content and if either blob
+ * looks like binary data, the `git_diff_delta` binary attribute will be set
+ * to 1 and no call to the hunk_cb nor line_cb will be made (unless you pass
+ * `GIT_DIFF_FORCE_TEXT` of course).
  *
  * @return 0 on success, GIT_EUSER on non-zero callback, or error code
  */
@@ -880,6 +925,11 @@ GIT_EXTERN(int) git_diff_blobs(
  * so the `git_diff_file` parameters to the callbacks will be faked a la the
  * rules for `git_diff_blobs()`.
  *
+ * Passing NULL for `old_blob` will be treated as an empty blob (i.e. the
+ * `file_cb` will be invoked with GIT_DELTA_ADDED and the diff will be the
+ * entire content of the buffer added).  Passing NULL to the buffer will do
+ * the reverse, with GIT_DELTA_REMOVED and blob content removed.
+ *
  * @return 0 on success, GIT_EUSER on non-zero callback, or error code
  */
 GIT_EXTERN(int) git_diff_blob_to_buffer(

src/checkout.c

@@ -78,7 +78,7 @@ static int checkout_notify(
 		git_oid_cpy(&wdfile.oid, &wditem->oid);
 		wdfile.path = wditem->path;
 		wdfile.size = wditem->file_size;
-		wdfile.flags = GIT_DIFF_FILE_VALID_OID;
+		wdfile.flags = GIT_DIFF_FLAG_VALID_OID;
 		wdfile.mode = wditem->mode;
 
 		workdir = &wdfile;

src/diff.c

@@ -92,11 +92,11 @@ static int diff_delta__from_one(
 		git_oid_cpy(&delta->new_file.oid, &entry->oid);
 	}
 
-	delta->old_file.flags |= GIT_DIFF_FILE_VALID_OID;
+	delta->old_file.flags |= GIT_DIFF_FLAG_VALID_OID;
 
 	if (delta->status == GIT_DELTA_DELETED ||
 		!git_oid_iszero(&delta->new_file.oid))
-		delta->new_file.flags |= GIT_DIFF_FILE_VALID_OID;
+		delta->new_file.flags |= GIT_DIFF_FLAG_VALID_OID;
 
 	notify_res = diff_notify(diff, delta, matched_pathspec);
 
@@ -142,7 +142,7 @@ static int diff_delta__from_two(
 	git_oid_cpy(&delta->old_file.oid, &old_entry->oid);
 	delta->old_file.size = old_entry->file_size;
 	delta->old_file.mode = old_mode;
-	delta->old_file.flags |= GIT_DIFF_FILE_VALID_OID;
+	delta->old_file.flags |= GIT_DIFF_FLAG_VALID_OID;
 
 	git_oid_cpy(&delta->new_file.oid, &new_entry->oid);
 	delta->new_file.size = new_entry->file_size;
@@ -156,7 +156,7 @@ static int diff_delta__from_two(
 	}
 
 	if (new_oid || !git_oid_iszero(&new_entry->oid))
-		delta->new_file.flags |= GIT_DIFF_FILE_VALID_OID;
+		delta->new_file.flags |= GIT_DIFF_FLAG_VALID_OID;
 
 	notify_res = diff_notify(diff, delta, matched_pathspec);
 

src/diff.h

@@ -28,8 +28,14 @@ enum {
 	GIT_DIFFCAPS_USE_DEV          = (1 << 4), /* use st_dev? */
 };
 
-#define GIT_DELTA__TO_DELETE 10
-#define GIT_DELTA__TO_SPLIT  11
+enum {
+	GIT_DIFF_FLAG__FREE_PATH  = (1 << 7),  /* `path` is allocated memory */
+	GIT_DIFF_FLAG__FREE_DATA  = (1 << 8),  /* internal file data is allocated */
+	GIT_DIFF_FLAG__UNMAP_DATA = (1 << 9),  /* internal file data is mmap'ed */
+	GIT_DIFF_FLAG__NO_DATA    = (1 << 10), /* file data should not be loaded */
+	GIT_DIFF_FLAG__TO_DELETE  = (1 << 11), /* delete entry during rename det. */
+	GIT_DIFF_FLAG__TO_SPLIT   = (1 << 12), /* split entry during rename det. */
+};
 
 struct git_diff_list {
 	git_refcount     rc;

src/hashsig.c

+/*
+ * Copyright (C) the libgit2 contributors. All rights reserved.
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#include "hashsig.h"
+#include "fileops.h"
+
+typedef uint32_t hashsig_t;
+typedef uint64_t hashsig_state;
+
+#define HASHSIG_SCALE 100
+
+#define HASHSIG_HASH_WINDOW 32
+#define HASHSIG_HASH_START	0
+#define HASHSIG_HASH_SHIFT  5
+#define HASHSIG_HASH_MASK   0x7FFFFFFF
+
+#define HASHSIG_HEAP_SIZE ((1 << 7) - 1)
+
+typedef int (*hashsig_cmp)(const void *a, const void *b);
+
+typedef struct {
+	int size, asize;
+	hashsig_cmp cmp;
+	hashsig_t values[HASHSIG_HEAP_SIZE];
+} hashsig_heap;
+
+typedef struct {
+	hashsig_state state, shift_n;
+	char window[HASHSIG_HASH_WINDOW];
+	int win_len, win_pos, saw_lf;
+} hashsig_in_progress;
+
+#define HASHSIG_IN_PROGRESS_INIT { HASHSIG_HASH_START, 1, {0}, 0, 0, 1 }
+
+struct git_hashsig {
+	hashsig_heap mins;
+	hashsig_heap maxs;
+	git_hashsig_option_t opt;
+	int considered;
+};
+
+#define HEAP_LCHILD_OF(I) (((I)*2)+1)
+#define HEAP_RCHILD_OF(I) (((I)*2)+2)
+#define HEAP_PARENT_OF(I) (((I)-1)>>1)
+
+static void hashsig_heap_init(hashsig_heap *h, hashsig_cmp cmp)
+{
+	h->size  = 0;
+	h->asize = HASHSIG_HEAP_SIZE;
+	h->cmp   = cmp;
+}
+
+static int hashsig_cmp_max(const void *a, const void *b)
+{
+	hashsig_t av = *(const hashsig_t *)a, bv = *(const hashsig_t *)b;
+	return (av < bv) ? -1 : (av > bv) ? 1 : 0;
+}
+
+static int hashsig_cmp_min(const void *a, const void *b)
+{
+	hashsig_t av = *(const hashsig_t *)a, bv = *(const hashsig_t *)b;
+	return (av > bv) ? -1 : (av < bv) ? 1 : 0;
+}
+
+static void hashsig_heap_up(hashsig_heap *h, int el)
+{
+	int parent_el = HEAP_PARENT_OF(el);
+
+	while (el > 0 && h->cmp(&h->values[parent_el], &h->values[el]) > 0) {
+		hashsig_t t = h->values[el];
+		h->values[el] = h->values[parent_el];
+		h->values[parent_el] = t;
+
+		el = parent_el;
+		parent_el = HEAP_PARENT_OF(el);
+	}
+}
+
+static void hashsig_heap_down(hashsig_heap *h, int el)
+{
+	hashsig_t v, lv, rv;
+
+	/* 'el < h->size / 2' tests if el is bottom row of heap */
+
+	while (el < h->size / 2) {
+		int lel = HEAP_LCHILD_OF(el), rel = HEAP_RCHILD_OF(el), swapel;
+
+		v  = h->values[el];
+		lv = h->values[lel];
+		rv = h->values[rel];
+
+		if (h->cmp(&v, &lv) < 0 && h->cmp(&v, &rv) < 0)
+			break;
+
+		swapel = (h->cmp(&lv, &rv) < 0) ? lel : rel;
+
+		h->values[el] = h->values[swapel];
+		h->values[swapel] = v;
+
+		el = swapel;
+	}
+}
+
+static void hashsig_heap_sort(hashsig_heap *h)
+{
+	/* only need to do this at the end for signature comparison */
+	qsort(h->values, h->size, sizeof(hashsig_t), h->cmp);
+}
+
+static void hashsig_heap_insert(hashsig_heap *h, hashsig_t val)
+{
+	/* if heap is full, pop top if new element should replace it */
+	if (h->size == h->asize && h->cmp(&val, &h->values[0]) > 0) {
+		h->size--;
+		h->values[0] = h->values[h->size];
+		hashsig_heap_down(h, 0);
+	}
+
+	/* if heap is not full, insert new element */
+	if (h->size < h->asize) {
+		h->values[h->size++] = val;
+		hashsig_heap_up(h, h->size - 1);
+	}
+}
+
+GIT_INLINE(bool) hashsig_include_char(
+	char ch, git_hashsig_option_t opt, int *saw_lf)
+{
+	if ((opt & GIT_HASHSIG_IGNORE_WHITESPACE) && git__isspace(ch))
+		return false;
+
+	if (opt & GIT_HASHSIG_SMART_WHITESPACE) {
+		if (ch == '\r' || (*saw_lf && git__isspace(ch)))
+			return false;
+
+		*saw_lf = (ch == '\n');
+	}
+
+	return true;
+}
+
+static void hashsig_initial_window(
+	git_hashsig *sig,
+	const char **data,
+	size_t size,
+	hashsig_in_progress *prog)
+{
+	hashsig_state state, shift_n;
+	int win_len;
+	const char *scan, *end;
+
+	/* init until we have processed at least HASHSIG_HASH_WINDOW data */
+
+	if (prog->win_len >= HASHSIG_HASH_WINDOW)
+		return;
+
+	state   = prog->state;
+	win_len = prog->win_len;
+	shift_n = prog->shift_n;
+
+	scan = *data;
+	end  = scan + size;
+
+	while (scan < end && win_len < HASHSIG_HASH_WINDOW) {
+		char ch = *scan++;
+
+		if (!hashsig_include_char(ch, sig->opt, &prog->saw_lf))
+			continue;
+
+		state = (state * HASHSIG_HASH_SHIFT + ch) & HASHSIG_HASH_MASK;
+
+		if (!win_len)
+			shift_n = 1;
+		else
+			shift_n = (shift_n * HASHSIG_HASH_SHIFT) & HASHSIG_HASH_MASK;
+
+		prog->window[win_len++] = ch;
+	}
+
+	/* insert initial hash if we just finished */
+
+	if (win_len == HASHSIG_HASH_WINDOW) {
+		hashsig_heap_insert(&sig->mins, state);
+		hashsig_heap_insert(&sig->maxs, state);
+		sig->considered = 1;
+	}
+
+	prog->state   = state;
+	prog->win_len = win_len;
+	prog->shift_n = shift_n;
+
+	*data = scan;
+}
+
+static int hashsig_add_hashes(
+	git_hashsig *sig,
+	const char *data,
+	size_t size,
+	hashsig_in_progress *prog)
+{
+	const char *scan = data, *end = data + size;
+	hashsig_state state, shift_n, rmv;
+
+	if (prog->win_len < HASHSIG_HASH_WINDOW)
+		hashsig_initial_window(sig, &scan, size, prog);
+
+	state   = prog->state;
+	shift_n = prog->shift_n;
+
+	/* advance window, adding new chars and removing old */
+
+	for (; scan < end; ++scan) {
+		char ch = *scan;
+
+		if (!hashsig_include_char(ch, sig->opt, &prog->saw_lf))
+			continue;
+
+		rmv = shift_n * prog->window[prog->win_pos];
+
+		state = (state - rmv) & HASHSIG_HASH_MASK;
+		state = (state * HASHSIG_HASH_SHIFT) & HASHSIG_HASH_MASK;
+		state = (state + ch) & HASHSIG_HASH_MASK;
+
+		hashsig_heap_insert(&sig->mins, state);
+		hashsig_heap_insert(&sig->maxs, state);
+		sig->considered++;
+
+		prog->window[prog->win_pos] = ch;
+		prog->win_pos = (prog->win_pos + 1) % HASHSIG_HASH_WINDOW;
+	}
+
+	prog->state = state;
+
+	return 0;
+}
+
+static int hashsig_finalize_hashes(git_hashsig *sig)
+{
+	if (sig->mins.size < HASHSIG_HEAP_SIZE) {
+		giterr_set(GITERR_INVALID,
+			"File too small for similarity signature calculation");
+		return GIT_EBUFS;
+	}
+
+	hashsig_heap_sort(&sig->mins);
+	hashsig_heap_sort(&sig->maxs);
+
+	return 0;
+}
+
+static git_hashsig *hashsig_alloc(git_hashsig_option_t opts)
+{
+	git_hashsig *sig = git__calloc(1, sizeof(git_hashsig));
+	if (!sig)
+		return NULL;
+
+	hashsig_heap_init(&sig->mins, hashsig_cmp_min);
+	hashsig_heap_init(&sig->maxs, hashsig_cmp_max);
+	sig->opt = opts;
+
+	return sig;
+}
+
+int git_hashsig_create(
+	git_hashsig **out,
+	const char *buf,
+	size_t buflen,
+	git_hashsig_option_t opts)
+{
+	int error;
+	hashsig_in_progress prog = HASHSIG_IN_PROGRESS_INIT;
+	git_hashsig *sig = hashsig_alloc(opts);
+	GITERR_CHECK_ALLOC(sig);
+
+	error = hashsig_add_hashes(sig, buf, buflen, &prog);
+
+	if (!error)
+		error = hashsig_finalize_hashes(sig);
+
+	if (!error)
+		*out = sig;
+	else
+		git_hashsig_free(sig);
+
+	return error;
+}
+
+int git_hashsig_create_fromfile(
+	git_hashsig **out,
+	const char *path,
+	git_hashsig_option_t opts)
+{
+	char buf[4096];
+	ssize_t buflen = 0;
+	int error = 0, fd;
+	hashsig_in_progress prog = HASHSIG_IN_PROGRESS_INIT;
+	git_hashsig *sig = hashsig_alloc(opts);
+	GITERR_CHECK_ALLOC(sig);
+
+	if ((fd = git_futils_open_ro(path)) < 0) {
+		git__free(sig);
+		return fd;
+	}
+
+	while (!error) {
+		if ((buflen = p_read(fd, buf, sizeof(buf))) <= 0) {
+			if ((error = buflen) < 0)
+				giterr_set(GITERR_OS,
+					"Read error on '%s' calculating similarity hashes", path);
+			break;
+		}
+
+		error = hashsig_add_hashes(sig, buf, buflen, &prog);
+	}
+
+	p_close(fd);
+
+	if (!error)
+		error = hashsig_finalize_hashes(sig);
+
+	if (!error)
+		*out = sig;
+	else
+		git_hashsig_free(sig);
+
+	return error;
+}
+
+void git_hashsig_free(git_hashsig *sig)
+{
+	git__free(sig);
+}
+
+static int hashsig_heap_compare(const hashsig_heap *a, const hashsig_heap *b)
+{
+	int matches = 0, i, j, cmp;
+
+	assert(a->cmp == b->cmp);
+
+	/* hash heaps are sorted - just look for overlap vs total */
+
+	for (i = 0, j = 0; i < a->size && j < b->size; ) {
+		cmp = a->cmp(&a->values[i], &b->values[j]);
+
+		if (cmp < 0)
+			++i;
+		else if (cmp > 0)
+			++j;
+		else {
+			++i; ++j; ++matches;
+		}
+	}
+
+	return HASHSIG_SCALE * (matches * 2) / (a->size + b->size);
+}
+
+int git_hashsig_compare(const git_hashsig *a, const git_hashsig *b)
+{
+	return (hashsig_heap_compare(&a->mins, &b->mins) +
+			hashsig_heap_compare(&a->maxs, &b->maxs)) / 2;
+}
+

src/hashsig.h

+/*
+ * Copyright (C) the libgit2 contributors. All rights reserved.
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#ifndef INCLUDE_hashsig_h__
+#define INCLUDE_hashsig_h__
+
+#include "common.h"
+
+/**
+ * Similarity signature of line hashes for a buffer
+ */
+typedef struct git_hashsig git_hashsig;
+
+typedef enum {
+	GIT_HASHSIG_NORMAL = 0, /* use all data */
+	GIT_HASHSIG_IGNORE_WHITESPACE = 1, /* ignore whitespace */
+	GIT_HASHSIG_SMART_WHITESPACE = 2, /* ignore \r and all space after \n */
+} git_hashsig_option_t;
+
+/**
+ * Build a similarity signature for a buffer
+ *
+ * If you have passed a whitespace-ignoring buffer, then the whitespace
+ * will be removed from the buffer while it is being processed, modifying
+ * the buffer in place.  Sorry about that!
+ *
+ * This will return an error if the buffer doesn't contain enough data to
+ * compute a valid signature.
+ *
+ * @param out The array of hashed runs representing the file content
+ * @param buf The contents of the file to hash
+ * @param buflen The length of the data at `buf`
+ * @param generate_pairwise_hashes Should pairwise runs be hashed
+ */
+extern int git_hashsig_create(
+	git_hashsig **out,
+	const char *buf,
+	size_t buflen,
+	git_hashsig_option_t opts);
+
+/**
+ * Build a similarity signature from a file
+ *
+ * This walks through the file, only loading a maximum of 4K of file data at
+ * a time.  Otherwise, it acts just like `git_hashsig_create`.
+ *
+ * This will return an error if the file doesn't contain enough data to
+ * compute a valid signature.
+ */
+extern int git_hashsig_create_fromfile(
+	git_hashsig **out,
+	const char *path,
+	git_hashsig_option_t opts);
+
+/**
+ * Release memory for a content similarity signature
+ */
+extern void git_hashsig_free(git_hashsig *sig);
+
+/**
+ * Measure similarity between two files
+ *
+ * @return <0 for error, [0 to 100] as similarity score
+ */
+extern int git_hashsig_compare(
+	const git_hashsig *a,
+	const git_hashsig *b);
+
+#endif

tests-clar/core/buffer.c

 #include "clar_libgit2.h"
 #include "buffer.h"
 #include "buf_text.h"
+#include "hashsig.h"
+#include "fileops.h"
 
 #define TESTSTR "Have you seen that? Have you seeeen that??"
 const char *test_string = TESTSTR;
@@ -730,3 +732,175 @@ void test_core_buffer__classify_with_utf8(void)
 	cl_assert(git_buf_text_is_binary(&b));
 	cl_assert(git_buf_text_contains_nul(&b));
 }
+
+#define SIMILARITY_TEST_DATA_1 \
+	"test data\nright here\ninline\ntada\nneeds more data\nlots of data\n" \
+	"is this enough?\nthere has to be enough data to fill the hash array!\n" \
+	"Apparently 191 bytes is the minimum amount of data needed.\nHere goes!\n" \
+	"Let's make sure we've got plenty to go with here.\n   smile   \n"
+
+void test_core_buffer__similarity_metric(void)
+{
+	git_hashsig *a, *b;
+	git_buf buf = GIT_BUF_INIT;
+	int sim;
+
+	/* in the first case, we compare data to itself and expect 100% match */
+
+	cl_git_pass(git_buf_sets(&buf, SIMILARITY_TEST_DATA_1));
+	cl_git_pass(git_hashsig_create(&a, buf.ptr, buf.size, GIT_HASHSIG_NORMAL));
+	cl_git_pass(git_hashsig_create(&b, buf.ptr, buf.size, GIT_HASHSIG_NORMAL));
+
+	cl_assert_equal_i(100, git_hashsig_compare(a, b));
+
+	git_hashsig_free(a);
+	git_hashsig_free(b);
+
+	/* if we change just a single byte, how much does that change magnify? */
+
+	cl_git_pass(git_buf_sets(&buf, SIMILARITY_TEST_DATA_1));
+	cl_git_pass(git_hashsig_create(&a, buf.ptr, buf.size, GIT_HASHSIG_NORMAL));
+	cl_git_pass(git_buf_sets(&buf,
+		"Test data\nright here\ninline\ntada\nneeds more data\nlots of data\n"
+		"is this enough?\nthere has to be enough data to fill the hash array!\n"
+		"Apparently 191 bytes is the minimum amount of data needed.\nHere goes!\n"
+		 "Let's make sure we've got plenty to go with here.\n   smile   \n"));
+	cl_git_pass(git_hashsig_create(&b, buf.ptr, buf.size, GIT_HASHSIG_NORMAL));
+
+	sim = git_hashsig_compare(a, b);
+
+	cl_assert(95 < sim && sim < 100); /* expect >95% similarity */
+
+	git_hashsig_free(a);
+	git_hashsig_free(b);
+
+	/* let's try comparing data to a superset of itself */
+
+	cl_git_pass(git_buf_sets(&buf, SIMILARITY_TEST_DATA_1));
+	cl_git_pass(git_hashsig_create(&a, buf.ptr, buf.size, GIT_HASHSIG_NORMAL));
+	cl_git_pass(git_buf_sets(&buf, SIMILARITY_TEST_DATA_1
+		"and if I add some more, it should still be pretty similar, yes?\n"));
+	cl_git_pass(git_hashsig_create(&b, buf.ptr, buf.size, GIT_HASHSIG_NORMAL));
+
+	sim = git_hashsig_compare(a, b);
+
+	cl_assert(70 < sim && sim < 80); /* expect in the 70-80% similarity range */
+
+	git_hashsig_free(a);
+	git_hashsig_free(b);
+
+	/* what if we keep about half the original data and add half new */
+
+	cl_git_pass(git_buf_sets(&buf, SIMILARITY_TEST_DATA_1));
+	cl_git_pass(git_hashsig_create(&a, buf.ptr, buf.size, GIT_HASHSIG_NORMAL));
+	cl_git_pass(git_buf_sets(&buf,
+		"test data\nright here\ninline\ntada\nneeds more data\nlots of data\n"
+		"is this enough?\nthere has to be enough data to fill the hash array!\n"
+		"okay, that's half the original\nwhat else can we add?\nmore data\n"
+		 "one more line will complete this\nshort\nlines\ndon't\nmatter\n"));
+	cl_git_pass(git_hashsig_create(&b, buf.ptr, buf.size, GIT_HASHSIG_NORMAL));
+
+	sim = git_hashsig_compare(a, b);
+
+	cl_assert(40 < sim && sim < 60); /* expect in the 40-60% similarity range */
+
+	git_hashsig_free(a);
+	git_hashsig_free(b);
+
+	/* lastly, let's check that we can hash file content as well */
+
+	cl_git_pass(git_buf_sets(&buf, SIMILARITY_TEST_DATA_1));
+	cl_git_pass(git_hashsig_create(&a, buf.ptr, buf.size, GIT_HASHSIG_NORMAL));
+
+	cl_git_pass(git_futils_mkdir("scratch", NULL, 0755, GIT_MKDIR_PATH));
+	cl_git_mkfile("scratch/testdata", SIMILARITY_TEST_DATA_1);
+	cl_git_pass(git_hashsig_create_fromfile(
+		&b, "scratch/testdata", GIT_HASHSIG_NORMAL));
+
+	cl_assert_equal_i(100, git_hashsig_compare(a, b));
+
+	git_hashsig_free(a);
+	git_hashsig_free(b);
+
+	git_buf_free(&buf);
+	git_futils_rmdir_r("scratch", NULL, GIT_RMDIR_REMOVE_FILES);
+}
+
+
+void test_core_buffer__similarity_metric_whitespace(void)
+{
+	git_hashsig *a, *b;
+	git_buf buf = GIT_BUF_INIT;
+	int sim, i, j;
+	git_hashsig_option_t opt;
+	const char *tabbed =
+		"	for (s = 0; s < sizeof(sep) / sizeof(char); ++s) {\n"
+		"		separator = sep[s];\n"
+		"		expect = expect_values[s];\n"
+		"\n"
+		"		for (j = 0; j < sizeof(b) / sizeof(char*); ++j) {\n"
+		"			for (i = 0; i < sizeof(a) / sizeof(char*); ++i) {\n"
+		"				git_buf_join(&buf, separator, a[i], b[j]);\n"
+		"				cl_assert_equal_s(*expect, buf.ptr);\n"
+		"				expect++;\n"
+		"			}\n"
+		"		}\n"
+		"	}\n";
+	const char *spaced =
+		"   for (s = 0; s < sizeof(sep) / sizeof(char); ++s) {\n"
+		"       separator = sep[s];\n"
+		"       expect = expect_values[s];\n"
+		"\n"
+		"       for (j = 0; j < sizeof(b) / sizeof(char*); ++j) {\n"
+		"           for (i = 0; i < sizeof(a) / sizeof(char*); ++i) {\n"
+		"               git_buf_join(&buf, separator, a[i], b[j]);\n"
+		"               cl_assert_equal_s(*expect, buf.ptr);\n"
+		"               expect++;\n"
+		"           }\n"
+		"       }\n"
+		"   }\n";
+	const char *crlf_spaced2 =
+		"  for (s = 0; s < sizeof(sep) / sizeof(char); ++s) {\r\n"
+		"    separator = sep[s];\r\n"
+		"    expect = expect_values[s];\r\n"
+		"\r\n"
+		"    for (j = 0; j < sizeof(b) / sizeof(char*); ++j) {\r\n"
+		"      for (i = 0; i < sizeof(a) / sizeof(char*); ++i) {\r\n"
+		"        git_buf_join(&buf, separator, a[i], b[j]);\r\n"
+		"        cl_assert_equal_s(*expect, buf.ptr);\r\n"
+		"        expect++;\r\n"
+		"      }\r\n"
+		"    }\r\n"
+		"  }\r\n";
+	const char *text[3] = { tabbed, spaced, crlf_spaced2 };
+
+	/* let's try variations of our own code with whitespace changes */
+
+	for (opt = GIT_HASHSIG_NORMAL; opt <= GIT_HASHSIG_SMART_WHITESPACE; ++opt) {
+		for (i = 0; i < 3; ++i) {
+			for (j = 0; j < 3; ++j) {
+				cl_git_pass(git_buf_sets(&buf, text[i]));
+				cl_git_pass(git_hashsig_create(&a, buf.ptr, buf.size, opt));
+
+				cl_git_pass(git_buf_sets(&buf, text[j]));
+				cl_git_pass(git_hashsig_create(&b, buf.ptr, buf.size, opt));
+
+				sim = git_hashsig_compare(a, b);
+
+				if (opt == GIT_HASHSIG_NORMAL) {
+					if (i == j)
+						cl_assert_equal_i(100, sim);
+					else
+						cl_assert(sim < 30); /* expect pretty different */
+				} else {
+					cl_assert_equal_i(100, sim);
+				}
+
+				git_hashsig_free(a);
+				git_hashsig_free(b);
+			}
+		}
+	}
+
+	git_buf_free(&buf);
+}

tests-clar/diff/blob.c

@@ -196,7 +196,7 @@ void test_diff_blob__can_compare_identical_blobs(void)
 		NULL, NULL, &opts, diff_file_cb, diff_hunk_cb, diff_line_cb, &expected));
 
 	cl_assert_equal_i(0, expected.files_binary);
-	assert_identical_blobs_comparison(&expected);
+	cl_assert_equal_i(0, expected.files); /* NULLs mean no callbacks, period */
 
 	memset(&expected, 0, sizeof(expected));
 	cl_git_pass(git_diff_blobs(
@@ -399,7 +399,7 @@ void test_diff_blob__can_compare_blob_to_buffer(void)
 
 	assert_identical_blobs_comparison(&expected);
 
-	/* diff from NULL blob to content of b */
+	/* diff from NULL blob to content of a */
 	memset(&expected, 0, sizeof(expected));
 	cl_git_pass(git_diff_blob_to_buffer(
 		NULL, a_content, strlen(a_content),

tests-clar/diff/diff_helpers.c

@@ -32,7 +32,7 @@ int diff_file_cb(
 
 	e->files++;
 
-	if (delta->binary)
+	if ((delta->flags & GIT_DIFF_FLAG_BINARY) != 0)
 		e->files_binary++;
 
 	cl_assert(delta->status <= GIT_DELTA_TYPECHANGE);
@@ -126,7 +126,8 @@ int diff_foreach_via_iterator(
 
 		/* if there are no changes, then the patch will be NULL */
 		if (!patch) {
-			cl_assert(delta->status == GIT_DELTA_UNMODIFIED || delta->binary == 1);
+			cl_assert(delta->status == GIT_DELTA_UNMODIFIED ||
+					  (delta->flags & GIT_DIFF_FLAG_BINARY) != 0);
 			continue;
 		}
 

tests-clar/diff/diffiter.c

@@ -152,8 +152,8 @@ void test_diff_diffiter__max_size_threshold(void)
 		file_count++;
 		hunk_count += (int)git_diff_patch_num_hunks(patch);
 
-		assert(delta->binary == 0 || delta->binary == 1);
-		binary_count += delta->binary;
+		assert((delta->flags & (GIT_DIFF_FLAG_BINARY|GIT_DIFF_FLAG_NOT_BINARY)) != 0);
+		binary_count += ((delta->flags & GIT_DIFF_FLAG_BINARY) != 0);
 
 		git_diff_patch_free(patch);
 	}
@@ -185,8 +185,8 @@ void test_diff_diffiter__max_size_threshold(void)
 		file_count++;
 		hunk_count += (int)git_diff_patch_num_hunks(patch);
 
-		assert(delta->binary == 0 || delta->binary == 1);
-		binary_count += delta->binary;
+		assert((delta->flags & (GIT_DIFF_FLAG_BINARY|GIT_DIFF_FLAG_NOT_BINARY)) != 0);
+		binary_count += ((delta->flags & GIT_DIFF_FLAG_BINARY) != 0);
 
 		git_diff_patch_free(patch);
 	}

tests-clar/diff/patch.c

@@ -144,11 +144,11 @@ void test_diff_patch__hunks_have_correct_line_numbers(void)
 	size_t hdrlen, hunklen, textlen;
 	char origin;
 	int oldno, newno;
-	const char *new_content = "The Song of Seven Cities\n========================\n\nI WAS Lord of Cities very sumptuously builded.\nSeven roaring Cities paid me tribute from afar.\nIvory their outposts were—the guardrooms of them gilded,\nAnd garrisoned with Amazons invincible in war.\n\nThis is some new text;\nNot as good as the old text;\nBut here it is.\n\nSo they warred and trafficked only yesterday, my Cities.\nTo-day there is no mark or mound of where my Cities stood.\nFor the River rose at midnight and it washed away my Cities.\nThey are evened with Atlantis and the towns before the Flood.\n\nRain on rain-gorged channels raised the water-levels round them,\nFreshet backed on freshet swelled and swept their world from sight,\nTill the emboldened floods linked arms and, flashing forward, drowned them—\nDrowned my Seven Cities and their peoples in one night!\n\nLow among the alders lie their derelict foundations,\nThe beams wherein they trusted and the plinths whereon they built—\nMy rulers and their treasure and their unborn populations,\nDead, destroyed, aborted, and defiled with mud and silt!\n\nAnother replacement;\nBreaking up the poem;\nGenerating some hunks.\n\nTo the sound of trumpets shall their seed restore my Cities\nWealthy and well-weaponed, that once more may I behold\nAll the world go softly when it walks before my Cities,\nAnd the horses and the chariots fleeing from them as of old!\n\n                -- Rudyard Kipling\n";
+	const char *new_content = "The Song of Seven Cities\n------------------------\n\nI WAS Lord of Cities very sumptuously builded.\nSeven roaring Cities paid me tribute from afar.\nIvory their outposts were--the guardrooms of them gilded,\nAnd garrisoned with Amazons invincible in war.\n\nThis is some new text;\nNot as good as the old text;\nBut here it is.\n\nSo they warred and trafficked only yesterday, my Cities.\nTo-day there is no mark or mound of where my Cities stood.\nFor the River rose at midnight and it washed away my Cities.\nThey are evened with Atlantis and the towns before the Flood.\n\nRain on rain-gorged channels raised the water-levels round them,\nFreshet backed on freshet swelled and swept their world from sight,\nTill the emboldened floods linked arms and, flashing forward, drowned them--\nDrowned my Seven Cities and their peoples in one night!\n\nLow among the alders lie their derelict foundations,\nThe beams wherein they trusted and the plinths whereon they built--\nMy rulers and their treasure and their unborn populations,\nDead, destroyed, aborted, and defiled with mud and silt!\n\nAnother replacement;\nBreaking up the poem;\nGenerating some hunks.\n\nTo the sound of trumpets shall their seed restore my Cities\nWealthy and well-weaponed, that once more may I behold\nAll the world go softly when it walks before my Cities,\nAnd the horses and the chariots fleeing from them as of old!\n\n  -- Rudyard Kipling\n";
 
 	g_repo = cl_git_sandbox_init("renames");
 
-	cl_git_rewritefile("renames/songofseven.txt", new_content);
+	cl_git_rewritefile("renames/songof7cities.txt", new_content);
 
 	cl_git_pass(git_repository_head_tree(&head, g_repo));
 
@@ -178,14 +178,14 @@ void test_diff_patch__hunks_have_correct_line_numbers(void)
 	cl_git_pass(git_diff_patch_get_line_in_hunk(
 		&origin, &text, &textlen, &oldno, &newno, patch, 0, 0));
 	cl_assert_equal_i(GIT_DIFF_LINE_CONTEXT, (int)origin);
-	cl_assert(strncmp("Ivory their outposts were—the guardrooms of them gilded,\n", text, textlen) == 0);
+	cl_assert(strncmp("Ivory their outposts were--the guardrooms of them gilded,\n", text, textlen) == 0);
 	cl_assert_equal_i(6, oldno);
 	cl_assert_equal_i(6, newno);
 
 	cl_git_pass(git_diff_patch_get_line_in_hunk(
 		&origin, &text, &textlen, &oldno, &newno, patch, 0, 3));
 	cl_assert_equal_i(GIT_DIFF_LINE_DELETION, (int)origin);
-	cl_assert(strncmp("All the world went softly when it walked before my Cities—\n", text, textlen) == 0);
+	cl_assert(strncmp("All the world went softly when it walked before my Cities--\n", text, textlen) == 0);
 	cl_assert_equal_i(9, oldno);
 	cl_assert_equal_i(-1, newno);
 
@@ -271,32 +271,32 @@ void test_diff_patch__line_counts_with_eofnl(void)
 
 	g_repo = cl_git_sandbox_init("renames");
 
-	cl_git_pass(git_futils_readbuffer(&content, "renames/songofseven.txt"));
+	cl_git_pass(git_futils_readbuffer(&content, "renames/songof7cities.txt"));
 
 	/* remove first line */
 
 	end = git_buf_cstr(&content) + git_buf_find(&content, '\n') + 1;
 	git_buf_consume(&content, end);
-	cl_git_rewritefile("renames/songofseven.txt", content.ptr);
+	cl_git_rewritefile("renames/songof7cities.txt", content.ptr);
 
 	check_single_patch_stats(g_repo, 1, 0, 1);
 
 	/* remove trailing whitespace */
 
 	git_buf_rtrim(&content);
-	cl_git_rewritefile("renames/songofseven.txt", content.ptr);
+	cl_git_rewritefile("renames/songof7cities.txt", content.ptr);
 
 	check_single_patch_stats(g_repo, 2, 1, 2);
 
 	/* add trailing whitespace */
 
 	cl_git_pass(git_repository_index(&index, g_repo));
-	cl_git_pass(git_index_add_bypath(index, "songofseven.txt"));
+	cl_git_pass(git_index_add_bypath(index, "songof7cities.txt"));
 	cl_git_pass(git_index_write(index));
 	git_index_free(index);
 
 	cl_git_pass(git_buf_putc(&content, '\n'));
-	cl_git_rewritefile("renames/songofseven.txt", content.ptr);
+	cl_git_rewritefile("renames/songof7cities.txt", content.ptr);
 
 	check_single_patch_stats(g_repo, 1, 1, 1);
 

tests-clar/diff/rename.c

@@ -23,8 +23,16 @@ void test_diff_rename__cleanup(void)
  *   serving.txt     -> sixserving.txt  (rename, no change, 100% match)
  *   sevencities.txt -> sevencities.txt (no change)
  *   sevencities.txt -> songofseven.txt (copy, no change, 100% match)
- *
- * TODO: add commits with various % changes of copy / rename
+ * commit 1c068dee5790ef1580cfc4cd670915b48d790084
+ *   songofseven.txt -> songofseven.txt (major rewrite, <20% match - split)
+ *   sixserving.txt  -> sixserving.txt  (indentation change)
+ *   sixserving.txt  -> ikeepsix.txt    (copy, add title, >80% match)
+ *   sevencities.txt                    (no change)
+ * commit 19dd32dfb1520a64e5bbaae8dce6ef423dfa2f13
+ *   songofseven.txt -> untimely.txt    (rename, convert to crlf)
+ *   ikeepsix.txt    -> ikeepsix.txt    (reorder sections in file)
+ *   sixserving.txt  -> sixserving.txt  (whitespace change - not just indent)
+ *   sevencities.txt -> songof7cities.txt (rename, small text changes)
  */
 
 void test_diff_rename__match_oid(void)
@@ -133,3 +141,219 @@ void test_diff_rename__checks_options_version(void)
 	git_tree_free(old_tree);
 	git_tree_free(new_tree);
 }
+
+void test_diff_rename__not_exact_match(void)
+{
+	const char *sha0 = "2bc7f351d20b53f1c72c16c4b036e491c478c49a";
+	const char *sha1 = "1c068dee5790ef1580cfc4cd670915b48d790084";
+	const char *sha2 = "19dd32dfb1520a64e5bbaae8dce6ef423dfa2f13";
+	git_tree *old_tree, *new_tree;
+	git_diff_list *diff;
+	git_diff_options diffopts = GIT_DIFF_OPTIONS_INIT;
+	git_diff_find_options opts = GIT_DIFF_FIND_OPTIONS_INIT;
+	diff_expects exp;
+
+	/* == Changes =====================================================
+	 * songofseven.txt -> songofseven.txt (major rewrite, <20% match - split)
+	 * sixserving.txt  -> sixserving.txt  (indentation change)
+	 * sixserving.txt  -> ikeepsix.txt    (copy, add title, >80% match)
+	 * sevencities.txt                    (no change)
+	 */
+
+	old_tree = resolve_commit_oid_to_tree(g_repo, sha0);
+	new_tree = resolve_commit_oid_to_tree(g_repo, sha1);
+
+	/* Must pass GIT_DIFF_INCLUDE_UNMODIFIED if you expect to emulate
+	 * --find-copies-harder during rename transformion...
+	 */
+	diffopts.flags |= GIT_DIFF_INCLUDE_UNMODIFIED;
+
+	cl_git_pass(git_diff_tree_to_tree(
+		&diff, g_repo, old_tree, new_tree, &diffopts));
+
+	/* git diff --no-renames \
+	 *          2bc7f351d20b53f1c72c16c4b036e491c478c49a \
+	 *          1c068dee5790ef1580cfc4cd670915b48d790084
+	 */
+	memset(&exp, 0, sizeof(exp));
+	cl_git_pass(git_diff_foreach(
+		diff, diff_file_cb, diff_hunk_cb, diff_line_cb, &exp));
+
+	cl_assert_equal_i(4, exp.files);
+	cl_assert_equal_i(1, exp.file_status[GIT_DELTA_UNMODIFIED]);
+	cl_assert_equal_i(2, exp.file_status[GIT_DELTA_MODIFIED]);
+	cl_assert_equal_i(1, exp.file_status[GIT_DELTA_ADDED]);
+
+	/* git diff -M 2bc7f351d20b53f1c72c16c4b036e491c478c49a \
+	 *          1c068dee5790ef1580cfc4cd670915b48d790084
+	 *
+	 * must not pass NULL for opts because it will pick up environment
+	 * values for "diff.renames" and test won't be consistent.
+	 */
+	opts.flags = GIT_DIFF_FIND_RENAMES;
+	cl_git_pass(git_diff_find_similar(diff, &opts));
+
+	memset(&exp, 0, sizeof(exp));
+	cl_git_pass(git_diff_foreach(
+		diff, diff_file_cb, diff_hunk_cb, diff_line_cb, &exp));
+
+	cl_assert_equal_i(4, exp.files);
+	cl_assert_equal_i(1, exp.file_status[GIT_DELTA_UNMODIFIED]);
+	cl_assert_equal_i(2, exp.file_status[GIT_DELTA_MODIFIED]);
+	cl_assert_equal_i(1, exp.file_status[GIT_DELTA_ADDED]);
+
+	git_diff_list_free(diff);
+
+	/* git diff -M -C \
+	 *          2bc7f351d20b53f1c72c16c4b036e491c478c49a \
+	 *          1c068dee5790ef1580cfc4cd670915b48d790084
+	 */
+	cl_git_pass(git_diff_tree_to_tree(
+		&diff, g_repo, old_tree, new_tree, &diffopts));
+
+	opts.flags = GIT_DIFF_FIND_RENAMES | GIT_DIFF_FIND_COPIES;
+	cl_git_pass(git_diff_find_similar(diff, &opts));
+
+	memset(&exp, 0, sizeof(exp));
+	cl_git_pass(git_diff_foreach(
+		diff, diff_file_cb, diff_hunk_cb, diff_line_cb, &exp));
+
+	cl_assert_equal_i(4, exp.files);
+	cl_assert_equal_i(1, exp.file_status[GIT_DELTA_UNMODIFIED]);
+	cl_assert_equal_i(2, exp.file_status[GIT_DELTA_MODIFIED]);
+	cl_assert_equal_i(1, exp.file_status[GIT_DELTA_COPIED]);
+
+	git_diff_list_free(diff);
+
+	/* git diff -M -C --find-copies-harder --break-rewrites \
+	 *          2bc7f351d20b53f1c72c16c4b036e491c478c49a \
+	 *          1c068dee5790ef1580cfc4cd670915b48d790084
+	 */
+	cl_git_pass(git_diff_tree_to_tree(
+		&diff, g_repo, old_tree, new_tree, &diffopts));
+
+	opts.flags = GIT_DIFF_FIND_ALL;
+	cl_git_pass(git_diff_find_similar(diff, &opts));
+
+	memset(&exp, 0, sizeof(exp));
+	cl_git_pass(git_diff_foreach(
+		diff, diff_file_cb, diff_hunk_cb, diff_line_cb, &exp));
+
+	cl_assert_equal_i(5, exp.files);
+	cl_assert_equal_i(1, exp.file_status[GIT_DELTA_UNMODIFIED]);
+	cl_assert_equal_i(1, exp.file_status[GIT_DELTA_MODIFIED]);
+	cl_assert_equal_i(1, exp.file_status[GIT_DELTA_DELETED]);
+	cl_assert_equal_i(1, exp.file_status[GIT_DELTA_ADDED]);
+	cl_assert_equal_i(1, exp.file_status[GIT_DELTA_COPIED]);
+
+	git_diff_list_free(diff);
+
+	/* == Changes =====================================================
+	 * songofseven.txt -> untimely.txt    (rename, convert to crlf)
+	 * ikeepsix.txt    -> ikeepsix.txt    (reorder sections in file)
+	 * sixserving.txt  -> sixserving.txt  (whitespace - not just indent)
+	 * sevencities.txt -> songof7cities.txt (rename, small text changes)
+	 */
+
+	git_tree_free(old_tree);
+	old_tree = new_tree;
+	new_tree = resolve_commit_oid_to_tree(g_repo, sha2);
+
+	cl_git_pass(git_diff_tree_to_tree(
+		&diff, g_repo, old_tree, new_tree, &diffopts));
+
+	/* git diff --no-renames \
+	 *          1c068dee5790ef1580cfc4cd670915b48d790084 \
+	 *          19dd32dfb1520a64e5bbaae8dce6ef423dfa2f13
+	 */
+	memset(&exp, 0, sizeof(exp));
+	cl_git_pass(git_diff_foreach(
+		diff, diff_file_cb, diff_hunk_cb, diff_line_cb, &exp));
+
+	cl_assert_equal_i(6, exp.files);
+	cl_assert_equal_i(2, exp.file_status[GIT_DELTA_MODIFIED]);
+	cl_assert_equal_i(2, exp.file_status[GIT_DELTA_ADDED]);
+	cl_assert_equal_i(2, exp.file_status[GIT_DELTA_DELETED]);
+
+	/* git diff -M -C \
+	 *          1c068dee5790ef1580cfc4cd670915b48d790084 \
+	 *          19dd32dfb1520a64e5bbaae8dce6ef423dfa2f13
+	 */
+	cl_git_pass(git_diff_tree_to_tree(
+		&diff, g_repo, old_tree, new_tree, &diffopts));
+
+	opts.flags = GIT_DIFF_FIND_RENAMES | GIT_DIFF_FIND_COPIES;
+	cl_git_pass(git_diff_find_similar(diff, &opts));
+
+	memset(&exp, 0, sizeof(exp));
+	cl_git_pass(git_diff_foreach(
+		diff, diff_file_cb, diff_hunk_cb, diff_line_cb, &exp));
+
+	cl_assert_equal_i(4, exp.files);
+	cl_assert_equal_i(2, exp.file_status[GIT_DELTA_MODIFIED]);
+	cl_assert_equal_i(2, exp.file_status[GIT_DELTA_RENAMED]);
+
+	git_diff_list_free(diff);
+
+	/* git diff -M -C --find-copies-harder --break-rewrites \
+	 *          1c068dee5790ef1580cfc4cd670915b48d790084 \
+	 *          19dd32dfb1520a64e5bbaae8dce6ef423dfa2f13
+	 * with libgit2 default similarity comparison...
+	 */
+	cl_git_pass(git_diff_tree_to_tree(
+		&diff, g_repo, old_tree, new_tree, &diffopts));
+
+	opts.flags = GIT_DIFF_FIND_ALL;
+	cl_git_pass(git_diff_find_similar(diff, &opts));
+
+	/* the default match algorithm is going to find the internal
+	 * whitespace differences in the lines of sixserving.txt to be
+	 * significant enough that this will decide to split it into
+	 * an ADD and a DELETE
+	 */
+
+	memset(&exp, 0, sizeof(exp));
+	cl_git_pass(git_diff_foreach(
+		diff, diff_file_cb, diff_hunk_cb, diff_line_cb, &exp));
+
+	cl_assert_equal_i(5, exp.files);
+	cl_assert_equal_i(1, exp.file_status[GIT_DELTA_MODIFIED]);
+	cl_assert_equal_i(1, exp.file_status[GIT_DELTA_ADDED]);
+	cl_assert_equal_i(1, exp.file_status[GIT_DELTA_DELETED]);
+	cl_assert_equal_i(2, exp.file_status[GIT_DELTA_RENAMED]);
+
+	git_diff_list_free(diff);
+
+	/* git diff -M -C --find-copies-harder --break-rewrites \
+	 *          1c068dee5790ef1580cfc4cd670915b48d790084 \
+	 *          19dd32dfb1520a64e5bbaae8dce6ef423dfa2f13
+	 * with ignore_space whitespace comparision
+	 */
+	cl_git_pass(git_diff_tree_to_tree(
+		&diff, g_repo, old_tree, new_tree, &diffopts));
+
+	opts.flags = GIT_DIFF_FIND_ALL | GIT_DIFF_FIND_IGNORE_WHITESPACE;
+	cl_git_pass(git_diff_find_similar(diff, &opts));
+
+	/* Ignoring whitespace, this should no longer split sixserver.txt */
+
+	memset(&exp, 0, sizeof(exp));
+	cl_git_pass(git_diff_foreach(
+		diff, diff_file_cb, diff_hunk_cb, diff_line_cb, &exp));
+
+	cl_assert_equal_i(4, exp.files);
+	cl_assert_equal_i(2, exp.file_status[GIT_DELTA_MODIFIED]);
+	cl_assert_equal_i(2, exp.file_status[GIT_DELTA_RENAMED]);
+
+	git_diff_list_free(diff);
+
+	git_tree_free(old_tree);
+	git_tree_free(new_tree);
+}
+
+void test_diff_rename__working_directory_changes(void)
+{
+	/* let's rewrite some files in the working directory on demand */
+
+	/* and with / without CRLF changes */
+}

tests-clar/resources/renames/.gitted/logs/HEAD

 0000000000000000000000000000000000000000 31e47d8c1fa36d7f8d537b96158e3f024de0a9f2 Russell Belfer <rb@github.com> 1351024687 -0700	commit (initial): Initial commit
 31e47d8c1fa36d7f8d537b96158e3f024de0a9f2 2bc7f351d20b53f1c72c16c4b036e491c478c49a Russell Belfer <rb@github.com> 1351024817 -0700	commit: copy and rename with no change
+2bc7f351d20b53f1c72c16c4b036e491c478c49a 1c068dee5790ef1580cfc4cd670915b48d790084 Russell Belfer <rb@github.com> 1361485758 -0800	commit: rewrites, copies with changes, etc.
+1c068dee5790ef1580cfc4cd670915b48d790084 19dd32dfb1520a64e5bbaae8dce6ef423dfa2f13 Russell Belfer <rb@github.com> 1361486360 -0800	commit: more renames and smallish modifications

tests-clar/resources/renames/.gitted/logs/refs/heads/master

 0000000000000000000000000000000000000000 31e47d8c1fa36d7f8d537b96158e3f024de0a9f2 Russell Belfer <rb@github.com> 1351024687 -0700	commit (initial): Initial commit
 31e47d8c1fa36d7f8d537b96158e3f024de0a9f2 2bc7f351d20b53f1c72c16c4b036e491c478c49a Russell Belfer <rb@github.com> 1351024817 -0700	commit: copy and rename with no change
+2bc7f351d20b53f1c72c16c4b036e491c478c49a 1c068dee5790ef1580cfc4cd670915b48d790084 Russell Belfer <rb@github.com> 1361485758 -0800	commit: rewrites, copies with changes, etc.
+1c068dee5790ef1580cfc4cd670915b48d790084 19dd32dfb1520a64e5bbaae8dce6ef423dfa2f13 Russell Belfer <rb@github.com> 1361486360 -0800	commit: more renames and smallish modifications

tests-clar/resources/renames/.gitted/refs/heads/master

-2bc7f351d20b53f1c72c16c4b036e491c478c49a
+19dd32dfb1520a64e5bbaae8dce6ef423dfa2f13

tests-clar/resources/renames/ikeepsix.txt

+I Keep Six Honest Serving-Men
+=============================
+
+She sends'em abroad on her own affairs,
+ From the second she opens her eyes—
+One million Hows, two million Wheres,
+And seven million Whys!
+
+I let them rest from nine till five,
+ For I am busy then,
+As well as breakfast, lunch, and tea,
+ For they are hungry men.
+But different folk have different views;
+I know a person small—
+She keeps ten million serving-men,
+Who get no rest at all!
+
+  -- Rudyard Kipling
+
+I KEEP six honest serving-men
+ (They taught me all I knew);
+Their names are What and Why and When
+ And How and Where and Who.
+I send them over land and sea,
+ I send them east and west;
+But after they have worked for me,
+ I give them all a rest.

tests-clar/resources/renames/sixserving.txt

-I KEEP six honest serving-men
- (They taught me all I knew);
-Their names are What and Why and When
- And How and Where and Who.
-I send them over land and sea,
- I send them east and west;
-But after they have worked for me,
- I give them all a rest.
+I  KEEP  six  honest  serving-men
+  (They  taught  me  all  I  knew);
+Their  names  are  What  and  Why  and  When
+  And  How  and  Where  and  Who.
+I  send  them  over  land  and  sea,
+  I  send  them  east  and  west;
+But  after  they  have  worked  for  me,
+  I  give  them  all  a  rest.
 
-I let them rest from nine till five,
- For I am busy then,
-As well as breakfast, lunch, and tea,
- For they are hungry men.
-But different folk have different views;
-I know a person small—
-She keeps ten million serving-men,
-Who get no rest at all!
+I  let  them  rest  from  nine  till  five,
+  For  I  am  busy  then,
+As  well  as  breakfast,  lunch,  and  tea,
+  For  they  are  hungry  men.
+But  different  folk  have  different  views;
+I  know  a  person  small—
+She  keeps  ten  million  serving-men,
+Who  get  no  rest  at  all!
 
-She sends'em abroad on her own affairs,
- From the second she opens her eyes—
-One million Hows, two million Wheres,
-And seven million Whys!
+She  sends'em  abroad  on  her  own  affairs,
+  From  the  second  she  opens  her  eyes—
+One  million  Hows,  two  million  Wheres,
+And  seven  million  Whys!
+
+    --  Rudyard  Kipling
 
-                -- Rudyard Kipling

tests-clar/resources/renames/sevencities.txt → tests-clar/resources/renames/songof7cities.txt

 The Song of Seven Cities
-========================
+------------------------
 
 I WAS Lord of Cities very sumptuously builded.
 Seven roaring Cities paid me tribute from afar.
-Ivory their outposts were—the guardrooms of them gilded,
+Ivory their outposts were--the guardrooms of them gilded,
 And garrisoned with Amazons invincible in war.
 
-All the world went softly when it walked before my Cities—
+All the world went softly when it walked before my Cities--
 Neither King nor Army vexed my peoples at their toil,
 Never horse nor chariot irked or overbore my Cities,
 Never Mob nor Ruler questioned whence they drew their spoil.
@@ -23,20 +23,20 @@ They are evened with Atlantis and the towns before the Flood.
 
 Rain on rain-gorged channels raised the water-levels round them,
 Freshet backed on freshet swelled and swept their world from sight,
-Till the emboldened floods linked arms and, flashing forward, drowned them—
+Till the emboldened floods linked arms and, flashing forward, drowned them--
 Drowned my Seven Cities and their peoples in one night!
 
 Low among the alders lie their derelict foundations,
-The beams wherein they trusted and the plinths whereon they built—
+The beams wherein they trusted and the plinths whereon they built--
 My rulers and their treasure and their unborn populations,
 Dead, destroyed, aborted, and defiled with mud and silt!
 
 The Daughters of the Palace whom they cherished in my Cities,
-My silver-tongued Princesses, and the promise of their May—
-Their bridegrooms of the June-tide—all have perished in my Cities,
+My silver-tongued Princesses, and the promise of their May--
+Their bridegrooms of the June-tide--all have perished in my Cities,
 With the harsh envenomed virgins that can neither love nor play.
 
-I was Lord of Cities—I will build anew my Cities,
+I was Lord of Cities--I will build anew my Cities,
 Seven, set on rocks, above the wrath of any flood.
 Nor will I rest from search till I have filled anew my Cities
 With peoples undefeated of the dark, enduring blood.
@@ -46,4 +46,4 @@ Wealthy and well-weaponed, that once more may I behold
 All the world go softly when it walks before my Cities,
 And the horses and the chariots fleeing from them as of old!
 
-                -- Rudyard Kipling
+  -- Rudyard Kipling

tests-clar/resources/renames/songofseven.txt

-The Song of Seven Cities
-========================
-
-I WAS Lord of Cities very sumptuously builded.
-Seven roaring Cities paid me tribute from afar.
-Ivory their outposts were—the guardrooms of them gilded,
-And garrisoned with Amazons invincible in war.
-
-All the world went softly when it walked before my Cities—
-Neither King nor Army vexed my peoples at their toil,
-Never horse nor chariot irked or overbore my Cities,
-Never Mob nor Ruler questioned whence they drew their spoil.
-
-Banded, mailed and arrogant from sunrise unto sunset;
-Singing while they sacked it, they possessed the land at large.
-Yet when men would rob them, they resisted, they made onset
-And pierced the smoke of battle with a thousand-sabred charge.
-
-So they warred and trafficked only yesterday, my Cities.
-To-day there is no mark or mound of where my Cities stood.
-For the River rose at midnight and it washed away my Cities.
-They are evened with Atlantis and the towns before the Flood.
-
-Rain on rain-gorged channels raised the water-levels round them,
-Freshet backed on freshet swelled and swept their world from sight,
-Till the emboldened floods linked arms and, flashing forward, drowned them—
-Drowned my Seven Cities and their peoples in one night!
-
-Low among the alders lie their derelict foundations,
-The beams wherein they trusted and the plinths whereon they built—
-My rulers and their treasure and their unborn populations,
-Dead, destroyed, aborted, and defiled with mud and silt!
-
-The Daughters of the Palace whom they cherished in my Cities,
-My silver-tongued Princesses, and the promise of their May—
-Their bridegrooms of the June-tide—all have perished in my Cities,
-With the harsh envenomed virgins that can neither love nor play.
-
-I was Lord of Cities—I will build anew my Cities,
-Seven, set on rocks, above the wrath of any flood.
-Nor will I rest from search till I have filled anew my Cities
-With peoples undefeated of the dark, enduring blood.
-
-To the sound of trumpets shall their seed restore my Cities
-Wealthy and well-weaponed, that once more may I behold
-All the world go softly when it walks before my Cities,
-And the horses and the chariots fleeing from them as of old!
-
-                -- Rudyard Kipling

tests-clar/resources/renames/untimely.txt

+Untimely
+========
+
+Nothing in life has been made by man for man's using
+But it was shown long since to man in ages
+Lost as the name of the maker of it,
+Who received oppression and shame for his wages--
+Hate, avoidance, and scorn in his daily dealings--
+Until he perished, wholly confounded
+
+More to be pitied than he are the wise
+Souls which foresaw the evil of loosing
+Knowledge or Art before time, and aborted
+Noble devices and deep-wrought healings,
+Lest offense should arise.
+
+Heaven delivers on earth the Hour that cannot be
+  thwarted,
+Neither advanced, at the price of a world nor a soul,
+  and its Prophet
+Comes through the blood of the vanguards who
+  dreamed--too soon--it had sounded.
+
+                -- Rudyard Kipling