Add diff rename tests with partial similarity

This adds some new tests that actually exercise the similarity
metric between files to detect renames, copies, and split modified
files that are too heavily modified.

There is still more testing to do - these tests are just partially
covering the cases.

There is also one bug fix in this where a change set with only
MODIFY being broken into ADD/DELETE (due to low self-similarity)
without any additional RENAMED entries would end up not processing
the split requests (because the num_rewrites counter got reset).

include/git2/diff.h

@@ -390,6 +390,9 @@ typedef enum {
 	/** split large rewrites into delete/add pairs (`--break-rewrites=/M`) */
 	GIT_DIFF_FIND_AND_BREAK_REWRITES = (1 << 4),
 
+	/** turn on all finding features */
+	GIT_DIFF_FIND_ALL = (0x1f),
+
 	/** measure similarity ignoring leading whitespace (default) */
 	GIT_DIFF_FIND_IGNORE_LEADING_WHITESPACE = 0,
 	/** measure similarity ignoring all whitespace */

src/diff_tform.c

@@ -561,8 +561,6 @@ int git_diff_find_similar(
 
 	/* next rewrite the diffs with renames / copies */
 
-	num_rewrites = 0;
-
 	git_vector_foreach(&diff->deltas, j, to) {
 		if (!matches[j]) {
 			assert(to->similarity == 0);

tests-clar/diff/rename.c

@@ -23,8 +23,16 @@ void test_diff_rename__cleanup(void)
  *   serving.txt     -> sixserving.txt  (rename, no change, 100% match)
  *   sevencities.txt -> sevencities.txt (no change)
  *   sevencities.txt -> songofseven.txt (copy, no change, 100% match)
- *
- * TODO: add commits with various % changes of copy / rename
+ * commit 1c068dee5790ef1580cfc4cd670915b48d790084
+ *   songofseven.txt -> songofseven.txt (major rewrite, <20% match - split)
+ *   sixserving.txt  -> sixserving.txt  (indentation change)
+ *   sixserving.txt  -> ikeepsix.txt    (copy, add title, >80% match)
+ *   sevencities.txt                    (no change)
+ * commit 19dd32dfb1520a64e5bbaae8dce6ef423dfa2f13
+ *   songofseven.txt -> untimely.txt    (rename, convert to crlf)
+ *   ikeepsix.txt    -> ikeepsix.txt    (reorder sections in file)
+ *   sixserving.txt  -> sixserving.txt  (whitespace change - not just indent)
+ *   sevencities.txt -> songof7cities.txt (rename, small text changes)
  */
 
 void test_diff_rename__match_oid(void)
@@ -133,3 +141,108 @@ void test_diff_rename__checks_options_version(void)
 	git_tree_free(old_tree);
 	git_tree_free(new_tree);
 }
+
+void test_diff_rename__not_exact_match(void)
+{
+	const char *sha0 = "2bc7f351d20b53f1c72c16c4b036e491c478c49a";
+	const char *sha1 = "1c068dee5790ef1580cfc4cd670915b48d790084";
+	const char *sha2 = "19dd32dfb1520a64e5bbaae8dce6ef423dfa2f13";
+	git_tree *old_tree, *new_tree;
+	git_diff_list *diff;
+	git_diff_options diffopts = GIT_DIFF_OPTIONS_INIT;
+	git_diff_find_options opts = GIT_DIFF_FIND_OPTIONS_INIT;
+	diff_expects exp;
+
+	/* Changes:
+	 *   songofseven.txt -> songofseven.txt (major rewrite, <20% match - split)
+	 *   sixserving.txt  -> sixserving.txt  (indentation change)
+	 *   sixserving.txt  -> ikeepsix.txt    (copy, add title, >80% match)
+	 *   sevencities.txt                    (no change)
+	 */
+
+	old_tree = resolve_commit_oid_to_tree(g_repo, sha0);
+	new_tree = resolve_commit_oid_to_tree(g_repo, sha1);
+
+	/* Must pass GIT_DIFF_INCLUDE_UNMODIFIED if you expect to emulate
+	 * --find-copies-harder during rename transformion...
+	 */
+	diffopts.flags |= GIT_DIFF_INCLUDE_UNMODIFIED;
+
+	cl_git_pass(git_diff_tree_to_tree(
+		&diff, g_repo, old_tree, new_tree, &diffopts));
+
+	/* git diff --no-renames \
+	 *          2bc7f351d20b53f1c72c16c4b036e491c478c49a \
+	 *          1c068dee5790ef1580cfc4cd670915b48d790084
+	 */
+	memset(&exp, 0, sizeof(exp));
+	cl_git_pass(git_diff_foreach(
+		diff, diff_file_cb, diff_hunk_cb, diff_line_cb, &exp));
+
+	cl_assert_equal_i(4, exp.files);
+	cl_assert_equal_i(1, exp.file_status[GIT_DELTA_UNMODIFIED]);
+	cl_assert_equal_i(2, exp.file_status[GIT_DELTA_MODIFIED]);
+	cl_assert_equal_i(1, exp.file_status[GIT_DELTA_ADDED]);
+
+	/* git diff 31e47d8c1fa36d7f8d537b96158e3f024de0a9f2 \
+	 *          2bc7f351d20b53f1c72c16c4b036e491c478c49a
+	 */
+	cl_git_pass(git_diff_find_similar(diff, NULL));
+
+	memset(&exp, 0, sizeof(exp));
+	cl_git_pass(git_diff_foreach(
+		diff, diff_file_cb, diff_hunk_cb, diff_line_cb, &exp));
+
+	cl_assert_equal_i(4, exp.files);
+	cl_assert_equal_i(1, exp.file_status[GIT_DELTA_UNMODIFIED]);
+	cl_assert_equal_i(2, exp.file_status[GIT_DELTA_MODIFIED]);
+	cl_assert_equal_i(1, exp.file_status[GIT_DELTA_COPIED]);
+
+	git_diff_list_free(diff);
+
+	cl_git_pass(git_diff_tree_to_tree(
+		&diff, g_repo, old_tree, new_tree, &diffopts));
+
+	/* git diff --find-copies-harder --break-rewrites \
+	 *          2bc7f351d20b53f1c72c16c4b036e491c478c49a \
+	 *          1c068dee5790ef1580cfc4cd670915b48d790084
+	 */
+	opts.flags = GIT_DIFF_FIND_ALL;
+	cl_git_pass(git_diff_find_similar(diff, &opts));
+
+	memset(&exp, 0, sizeof(exp));
+	cl_git_pass(git_diff_foreach(
+		diff, diff_file_cb, diff_hunk_cb, diff_line_cb, &exp));
+
+	cl_assert_equal_i(5, exp.files);
+	cl_assert_equal_i(1, exp.file_status[GIT_DELTA_UNMODIFIED]);
+	cl_assert_equal_i(1, exp.file_status[GIT_DELTA_MODIFIED]);
+	cl_assert_equal_i(1, exp.file_status[GIT_DELTA_DELETED]);
+	cl_assert_equal_i(1, exp.file_status[GIT_DELTA_ADDED]);
+	cl_assert_equal_i(1, exp.file_status[GIT_DELTA_COPIED]);
+
+	git_diff_list_free(diff);
+
+	/* Changes:
+	 *   songofseven.txt -> untimely.txt    (rename, convert to crlf)
+	 *   ikeepsix.txt    -> ikeepsix.txt    (reorder sections in file)
+	 *   sixserving.txt  -> sixserving.txt  (whitespace - not just indent)
+	 *   sevencities.txt -> songof7cities.txt (rename, small text changes)
+	 */
+
+	git_tree_free(old_tree);
+	old_tree = new_tree;
+	new_tree = resolve_commit_oid_to_tree(g_repo, sha2);
+
+	/* moar tests needed */
+
+	git_tree_free(old_tree);
+	git_tree_free(new_tree);
+}
+
+void test_diff_rename__working_directory_changes(void)
+{
+	/* let's rewrite some files in the working directory on demand */
+
+	/* and with / without CRLF changes */
+}

tests-clar/resources/renames/.gitted/logs/HEAD

 0000000000000000000000000000000000000000 31e47d8c1fa36d7f8d537b96158e3f024de0a9f2 Russell Belfer <rb@github.com> 1351024687 -0700	commit (initial): Initial commit
 31e47d8c1fa36d7f8d537b96158e3f024de0a9f2 2bc7f351d20b53f1c72c16c4b036e491c478c49a Russell Belfer <rb@github.com> 1351024817 -0700	commit: copy and rename with no change
+2bc7f351d20b53f1c72c16c4b036e491c478c49a 1c068dee5790ef1580cfc4cd670915b48d790084 Russell Belfer <rb@github.com> 1361485758 -0800	commit: rewrites, copies with changes, etc.
+1c068dee5790ef1580cfc4cd670915b48d790084 19dd32dfb1520a64e5bbaae8dce6ef423dfa2f13 Russell Belfer <rb@github.com> 1361486360 -0800	commit: more renames and smallish modifications

tests-clar/resources/renames/.gitted/logs/refs/heads/master

 0000000000000000000000000000000000000000 31e47d8c1fa36d7f8d537b96158e3f024de0a9f2 Russell Belfer <rb@github.com> 1351024687 -0700	commit (initial): Initial commit
 31e47d8c1fa36d7f8d537b96158e3f024de0a9f2 2bc7f351d20b53f1c72c16c4b036e491c478c49a Russell Belfer <rb@github.com> 1351024817 -0700	commit: copy and rename with no change
+2bc7f351d20b53f1c72c16c4b036e491c478c49a 1c068dee5790ef1580cfc4cd670915b48d790084 Russell Belfer <rb@github.com> 1361485758 -0800	commit: rewrites, copies with changes, etc.
+1c068dee5790ef1580cfc4cd670915b48d790084 19dd32dfb1520a64e5bbaae8dce6ef423dfa2f13 Russell Belfer <rb@github.com> 1361486360 -0800	commit: more renames and smallish modifications

tests-clar/resources/renames/.gitted/refs/heads/master

-2bc7f351d20b53f1c72c16c4b036e491c478c49a
+19dd32dfb1520a64e5bbaae8dce6ef423dfa2f13

tests-clar/resources/renames/ikeepsix.txt

+I Keep Six Honest Serving-Men
+=============================
+
+She sends'em abroad on her own affairs,
+ From the second she opens her eyes—
+One million Hows, two million Wheres,
+And seven million Whys!
+
+I let them rest from nine till five,
+ For I am busy then,
+As well as breakfast, lunch, and tea,
+ For they are hungry men.
+But different folk have different views;
+I know a person small—
+She keeps ten million serving-men,
+Who get no rest at all!
+
+  -- Rudyard Kipling
+
+I KEEP six honest serving-men
+ (They taught me all I knew);
+Their names are What and Why and When
+ And How and Where and Who.
+I send them over land and sea,
+ I send them east and west;
+But after they have worked for me,
+ I give them all a rest.

tests-clar/resources/renames/sixserving.txt

-I KEEP six honest serving-men
- (They taught me all I knew);
-Their names are What and Why and When
- And How and Where and Who.
-I send them over land and sea,
- I send them east and west;
-But after they have worked for me,
- I give them all a rest.
+I  KEEP  six  honest  serving-men
+  (They  taught  me  all  I  knew);
+Their  names  are  What  and  Why  and  When
+  And  How  and  Where  and  Who.
+I  send  them  over  land  and  sea,
+  I  send  them  east  and  west;
+But  after  they  have  worked  for  me,
+  I  give  them  all  a  rest.
 
-I let them rest from nine till five,
- For I am busy then,
-As well as breakfast, lunch, and tea,
- For they are hungry men.
-But different folk have different views;
-I know a person small—
-She keeps ten million serving-men,
-Who get no rest at all!
+I  let  them  rest  from  nine  till  five,
+  For  I  am  busy  then,
+As  well  as  breakfast,  lunch,  and  tea,
+  For  they  are  hungry  men.
+But  different  folk  have  different  views;
+I  know  a  person  small—
+She  keeps  ten  million  serving-men,
+Who  get  no  rest  at  all!
 
-She sends'em abroad on her own affairs,
- From the second she opens her eyes—
-One million Hows, two million Wheres,
-And seven million Whys!
+She  sends'em  abroad  on  her  own  affairs,
+  From  the  second  she  opens  her  eyes—
+One  million  Hows,  two  million  Wheres,
+And  seven  million  Whys!
+
+    --  Rudyard  Kipling
 
-                -- Rudyard Kipling

tests-clar/resources/renames/sevencities.txt → tests-clar/resources/renames/songof7cities.txt

 The Song of Seven Cities
-========================
+------------------------
 
 I WAS Lord of Cities very sumptuously builded.
 Seven roaring Cities paid me tribute from afar.
-Ivory their outposts were—the guardrooms of them gilded,
+Ivory their outposts were--the guardrooms of them gilded,
 And garrisoned with Amazons invincible in war.
 
-All the world went softly when it walked before my Cities—
+All the world went softly when it walked before my Cities--
 Neither King nor Army vexed my peoples at their toil,
 Never horse nor chariot irked or overbore my Cities,
 Never Mob nor Ruler questioned whence they drew their spoil.
@@ -23,20 +23,20 @@ They are evened with Atlantis and the towns before the Flood.
 
 Rain on rain-gorged channels raised the water-levels round them,
 Freshet backed on freshet swelled and swept their world from sight,
-Till the emboldened floods linked arms and, flashing forward, drowned them—
+Till the emboldened floods linked arms and, flashing forward, drowned them--
 Drowned my Seven Cities and their peoples in one night!
 
 Low among the alders lie their derelict foundations,
-The beams wherein they trusted and the plinths whereon they built—
+The beams wherein they trusted and the plinths whereon they built--
 My rulers and their treasure and their unborn populations,
 Dead, destroyed, aborted, and defiled with mud and silt!
 
 The Daughters of the Palace whom they cherished in my Cities,
-My silver-tongued Princesses, and the promise of their May—
-Their bridegrooms of the June-tide—all have perished in my Cities,
+My silver-tongued Princesses, and the promise of their May--
+Their bridegrooms of the June-tide--all have perished in my Cities,
 With the harsh envenomed virgins that can neither love nor play.
 
-I was Lord of Cities—I will build anew my Cities,
+I was Lord of Cities--I will build anew my Cities,
 Seven, set on rocks, above the wrath of any flood.
 Nor will I rest from search till I have filled anew my Cities
 With peoples undefeated of the dark, enduring blood.
@@ -46,4 +46,4 @@ Wealthy and well-weaponed, that once more may I behold
 All the world go softly when it walks before my Cities,
 And the horses and the chariots fleeing from them as of old!
 
-                -- Rudyard Kipling
+  -- Rudyard Kipling

tests-clar/resources/renames/songofseven.txt

-The Song of Seven Cities
-========================
-
-I WAS Lord of Cities very sumptuously builded.
-Seven roaring Cities paid me tribute from afar.
-Ivory their outposts were—the guardrooms of them gilded,
-And garrisoned with Amazons invincible in war.
-
-All the world went softly when it walked before my Cities—
-Neither King nor Army vexed my peoples at their toil,
-Never horse nor chariot irked or overbore my Cities,
-Never Mob nor Ruler questioned whence they drew their spoil.
-
-Banded, mailed and arrogant from sunrise unto sunset;
-Singing while they sacked it, they possessed the land at large.
-Yet when men would rob them, they resisted, they made onset
-And pierced the smoke of battle with a thousand-sabred charge.
-
-So they warred and trafficked only yesterday, my Cities.
-To-day there is no mark or mound of where my Cities stood.
-For the River rose at midnight and it washed away my Cities.
-They are evened with Atlantis and the towns before the Flood.
-
-Rain on rain-gorged channels raised the water-levels round them,
-Freshet backed on freshet swelled and swept their world from sight,
-Till the emboldened floods linked arms and, flashing forward, drowned them—
-Drowned my Seven Cities and their peoples in one night!
-
-Low among the alders lie their derelict foundations,
-The beams wherein they trusted and the plinths whereon they built—
-My rulers and their treasure and their unborn populations,
-Dead, destroyed, aborted, and defiled with mud and silt!
-
-The Daughters of the Palace whom they cherished in my Cities,
-My silver-tongued Princesses, and the promise of their May—
-Their bridegrooms of the June-tide—all have perished in my Cities,
-With the harsh envenomed virgins that can neither love nor play.
-
-I was Lord of Cities—I will build anew my Cities,
-Seven, set on rocks, above the wrath of any flood.
-Nor will I rest from search till I have filled anew my Cities
-With peoples undefeated of the dark, enduring blood.
-
-To the sound of trumpets shall their seed restore my Cities
-Wealthy and well-weaponed, that once more may I behold
-All the world go softly when it walks before my Cities,
-And the horses and the chariots fleeing from them as of old!
-
-                -- Rudyard Kipling

tests-clar/resources/renames/untimely.txt

+Untimely
+========
+
+Nothing in life has been made by man for man's using
+But it was shown long since to man in ages
+Lost as the name of the maker of it,
+Who received oppression and shame for his wages--
+Hate, avoidance, and scorn in his daily dealings--
+Until he perished, wholly confounded
+
+More to be pitied than he are the wise
+Souls which foresaw the evil of loosing
+Knowledge or Art before time, and aborted
+Noble devices and deep-wrought healings,
+Lest offense should arise.
+
+Heaven delivers on earth the Hour that cannot be
+  thwarted,
+Neither advanced, at the price of a world nor a soul,
+  and its Prophet
+Comes through the blood of the vanguards who
+  dreamed--too soon--it had sounded.
+
+                -- Rudyard Kipling