Cleanup TYPECHANGE support

This is just some cleanup code, rearranging some of the checkout
code where TYPECHANGE support was added and adding some comments
to the diff header regarding the constants.

include/git2/diff.h

@@ -32,6 +32,30 @@ GIT_BEGIN_DECL
 /**
  * Flags for diff options.  A combination of these flags can be passed
  * in via the `flags` value in the `git_diff_options`.
+ *
+ * The flags are:
+ *
+ * - GIT_DIFF_NORMAL: normal diff, the default.
+ * - GIT_DIFF_REVERSE: reverse the sides of the diff.
+ * - GIT_DIFF_FORCE_TEXT: treat all as text, no binary attributes / detection.
+ * - GIT_DIFF_IGNORE_WHITESPACE: ignore all whitespace.
+ * - GIT_DIFF_IGNORE_WHITESPACE_CHANGE: ignore changes in amount of whitespace.
+ * - GIT_DIFF_IGNORE_WHITESPACE_EOL: ignore whitespace only at end-of-line.
+ * - GIT_DIFF_IGNORE_SUBMODULES: exclude submodules from diff completely.
+ * - GIT_DIFF_PATIENCE: use "patience diff" algorithm
+ * - GIT_DIFF_INCLUDE_IGNORED: include ignored files in the diff list
+ * - GIT_DIFF_INCLUDE_UNTRACKED: include untracked files in the diff list
+ * - GIT_DIFF_INCLUDE_UNMODIFIED: include unmodified files in the diff list
+ * - GIT_DIFF_RECURSE_UNTRACKED_DIRS: even with the INCLUDE_UNTRACKED flag,
+ *   when am untracked directory is found, only a single entry for the directory
+ *   will be included in the diff list; with this flag, all files under the
+ *   directory will be included, too.
+ * - GIT_DIFF_DISABLE_PATHSPEC_MATCH: if the pathspec is set in the diff
+ *   options, this flags means to interpret it exactly instead of fnmatch.
+ * - GIT_DIFF_DELTAS_ARE_ICASE: use case insensitive filename comparisons
+ * - GIT_DIFF_DONT_SPLIT_TYPECHANGE: normally, a type change between files
+ *   will be converted into a DELETED record for the old file and an ADDED
+ *   record for the new one; this option enabled TYPECHANGE records.
  */
 enum {
 	GIT_DIFF_NORMAL = 0,

src/checkout.c

@@ -156,71 +156,74 @@ static int checkout_diff_fn(
 {
 	struct checkout_diff_data *data = cb_data;
 	int error = 0;
-	git_checkout_opts *opts;
+	git_checkout_opts *opts = data->checkout_opts;
+	bool do_delete = false, do_checkout_blob = false;
 
 	data->stats->processed = (unsigned int)(data->stats->total * progress);
 
-	git_buf_truncate(data->path, data->workdir_len);
-	if (git_buf_joinpath(data->path, git_buf_cstr(data->path), delta->new_file.path) < 0)
-		return -1;
-
-	opts = data->checkout_opts;
-
 	switch (delta->status) {
 	case GIT_DELTA_UNTRACKED:
-		if (!(opts->checkout_strategy & GIT_CHECKOUT_REMOVE_UNTRACKED))
-			return 0;
-
-		if (!git__suffixcmp(delta->new_file.path, "/"))
-			error = git_futils_rmdir_r(git_buf_cstr(data->path), GIT_DIRREMOVAL_FILES_AND_DIRS);
-		else
-			error = p_unlink(git_buf_cstr(data->path));
+		if ((opts->checkout_strategy & GIT_CHECKOUT_REMOVE_UNTRACKED) != 0)
+			do_delete = true;
 		break;
 
 	case GIT_DELTA_MODIFIED:
 	case GIT_DELTA_TYPECHANGE:
 		if (!(opts->checkout_strategy & GIT_CHECKOUT_OVERWRITE_MODIFIED)) {
 
-			if ((opts->skipped_notify_cb != NULL)
-				&& (opts->skipped_notify_cb(
+			if (opts->skipped_notify_cb != NULL &&
+				opts->skipped_notify_cb(
 					delta->new_file.path,
 					&delta->old_file.oid,
 					delta->old_file.mode,
-					opts->notify_payload))) {
-						giterr_clear();
-						error = GIT_EUSER;
+					opts->notify_payload) != 0)
+			{
+				giterr_clear();
+				error = GIT_EUSER;
 			}
+
+			goto cleanup;
 		}
-		else
-			error = checkout_blob(
-				data->owner,
-				&delta->old_file.oid,
-				git_buf_cstr(data->path),
-				delta->old_file.mode,
-				data->can_symlink,
-				opts);
 
+		do_checkout_blob = true;
+
+		if (delta->status == GIT_DELTA_TYPECHANGE)
+			do_delete = true;
 		break;
 
 	case GIT_DELTA_DELETED:
-		if (!(opts->checkout_strategy & GIT_CHECKOUT_CREATE_MISSING))
-			return 0;
-
-		error = checkout_blob(
-				data->owner,
-				&delta->old_file.oid,
-				git_buf_cstr(data->path),
-				delta->old_file.mode,
-				data->can_symlink,
-				opts);
+		if ((opts->checkout_strategy & GIT_CHECKOUT_CREATE_MISSING) != 0)
+			do_checkout_blob = true;
 		break;
 
 	default:
 		giterr_set(GITERR_INVALID, "Unexpected status (%d) for path '%s'.",
 			delta->status, delta->new_file.path);
 		error = -1;
+		goto cleanup;
 	}
 
+	git_buf_truncate(data->path, data->workdir_len);
+
+	if ((error = git_buf_joinpath(
+			data->path, git_buf_cstr(data->path), delta->new_file.path)) < 0)
+		goto cleanup;
+
+	if (do_delete &&
+		(error = git_futils_rmdir_r(
+			git_buf_cstr(data->path), GIT_DIRREMOVAL_FILES_AND_DIRS)) < 0)
+		goto cleanup;
+
+	if (do_checkout_blob)
+		error = checkout_blob(
+			data->owner,
+			&delta->old_file.oid,
+			git_buf_cstr(data->path),
+			delta->old_file.mode,
+			data->can_symlink,
+			opts);
+
+cleanup:
 	if (error)
 		data->error = error; /* preserve real error */