Merge pull request #1642 from arrbee/diff-function-context

Diff code reorg plus function context in diff headers

docs/diff-internals.md

+Diff is broken into four phases:
+
+1. Building a list of things that have changed.  These changes are called
+   deltas (git_diff_delta objects) and are grouped into a git_diff_list.
+2. Applying file similarity measurement for rename and copy detection (and
+   to potentially split files that have changed radically).  This step is
+   optional.
+3. Computing the textual diff for each delta.  Not all deltas have a
+   meaningful textual diff.  For those that do, the textual diff can
+   either be generated on the fly and passed to output callbacks or can be
+   turned into a git_diff_patch object.
+4. Formatting the diff and/or patch into standard text formats (such as
+   patches, raw lists, etc).
+
+In the source code, step 1 is implemented in `src/diff.c`, step 2 in
+`src/diff_tform.c`, step 3 in `src/diff_patch.c`, and step 4 in
+`src/diff_print.c`.  Additionally, when it comes to accessing file
+content, everything goes through diff drivers that are implemented in
+`src/diff_driver.c`.
+
+External Objects
+----------------
+
+* `git_diff_options` repesents user choices about how a diff should be
+  performed and is passed to most diff generating functions.
+* `git_diff_file` represents an item on one side of a possible delta
+* `git_diff_delta` represents a pair of items that have changed in some
+  way - it contains two `git_diff_file` plus a status and other stuff.
+* `git_diff_list` is a list of deltas along with information about how
+  those particular deltas were found.
+* `git_diff_patch` represents the actual diff between a pair of items.  In
+  some cases, a delta may not have a corresponding patch, if the objects
+  are binary, for example.  The content of a patch will be a set of hunks
+  and lines.
+* A `hunk` is range of lines described by a `git_diff_range` (i.e.  "lines
+  10-20 in the old file became lines 12-23 in the new").  It will have a
+  header that compactly represents that information, and it will have a
+  number of lines of context surrounding added and deleted lines.
+* A `line` is simple a line of data along with a `git_diff_line_t` value
+  that tells how the data should be interpretted (e.g. context or added).
+
+Internal Objects
+----------------
+
+* `git_diff_file_content` is an internal structure that represents the
+  data on one side of an item to be diffed; it is an augmented
+  `git_diff_file` with more flags and the actual file data.
+** it is created from a repository plus a) a git_diff_file, b) a git_blob,
+   or c) raw data and size
+** there are three main operations on git_diff_file_content:
+*** _initialization_ sets up the data structure and does what it can up to,
+    but not including loading and looking at the actual data
+*** _loading_ loads the data, preprocesses it (i.e. applies filters) and
+    potentially analyzes it (to decide if binary)
+*** _free_ releases loaded data and frees any allocated memory
+
+* The internal structure of a `git_diff_patch` stores the actual diff
+  between a pair of `git_diff_file_content` items
+** it may be "unset" if the items are not diffable
+** "empty" if the items are the same
+** otherwise it will consist of a set of hunks each of which covers some
+   number of lines of context, additions and deletions
+** a patch is created from two git_diff_file_content items
+** a patch is fully instantiated in three phases:
+*** initial creation and initialization
+*** loading of data and preliminary data examination
+*** diffing of data and optional storage of diffs
+** (TBD) if a patch is asked to store the diffs and the size of the diff
+   is significantly smaller than the raw data of the two sides, then the
+   patch may be flattened using a pool of string data
+
+* `git_diff_output` is an internal structure that represents an output
+  target for a `git_diff_patch`
+** It consists of file, hunk, and line callbacks, plus a payload
+** There is a standard flattened output that can be used for plain text output
+** Typically we use a `git_xdiff_output` which drives the callbacks via the
+   xdiff code taken from core Git.
+
+* `git_diff_driver` is an internal structure that encapsulates the logic
+  for a given type of file
+** a driver is looked up based on the name and mode of a file.
+** the driver can then be used to:
+*** determine if a file is binary (by attributes, by git_diff_options
+    settings, or by examining the content)
+*** give you a function pointer that is used to evaluate function context
+    for hunk headers
+** At some point, the logic for getting a filtered version of file content
+   or calculating the OID of a file may be moved into the driver.

include/git2/diff.h

@@ -148,6 +148,9 @@ typedef enum {
 	 *  Of course, ignore rules are still checked for the directory itself.
 	 */
 	GIT_DIFF_FAST_UNTRACKED_DIRS = (1 << 19),
+
+	/** Treat all files as binary, disabling text diffs */
+	GIT_DIFF_FORCE_BINARY = (1 << 20),
 } git_diff_option_t;
 
 /**
@@ -857,7 +860,7 @@ GIT_EXTERN(size_t) git_diff_patch_num_hunks(
  * @param total_additions Count of addition lines in output, can be NULL.
  * @param total_deletions Count of deletion lines in output, can be NULL.
  * @param patch The git_diff_patch object
- * @return Number of lines in hunk or -1 if invalid hunk index
+ * @return 0 on success, <0 on error
  */
 GIT_EXTERN(int) git_diff_patch_line_stats(
 	size_t *total_context,
@@ -998,6 +1001,26 @@ GIT_EXTERN(int) git_diff_blobs(
 	void *payload);
 
 /**
+ * Directly generate a patch from the difference between two blobs.
+ *
+ * This is just like `git_diff_blobs()` except it generates a patch object
+ * for the difference instead of directly making callbacks.  You can use the
+ * standard `git_diff_patch` accessor functions to read the patch data, and
+ * you must call `git_diff_patch_free()` on the patch when done.
+ *
+ * @param out The generated patch; NULL on error
+ * @param old_blob Blob for old side of diff, or NULL for empty blob
+ * @param new_blob Blob for new side of diff, or NULL for empty blob
+ * @param options Options for diff, or NULL for default options
+ * @return 0 on success or error code < 0
+ */
+GIT_EXTERN(int) git_diff_patch_from_blobs(
+	git_diff_patch **out,
+	const git_blob *old_blob,
+	const git_blob *new_blob,
+	const git_diff_options *opts);
+
+/**
  * Directly run a diff between a blob and a buffer.
  *
  * As with `git_diff_blobs`, comparing a blob and buffer lacks some context,
@@ -1010,7 +1033,7 @@ GIT_EXTERN(int) git_diff_blobs(
  * the reverse, with GIT_DELTA_REMOVED and blob content removed.
  *
  * @param old_blob Blob for old side of diff, or NULL for empty blob
- * @param buffer Raw data for new side of diff
+ * @param buffer Raw data for new side of diff, or NULL for empty
  * @param buffer_len Length of raw data for new side of diff
  * @param options Options for diff, or NULL for default options
  * @param file_cb Callback for "file"; made once if there is a diff; can be NULL
@@ -1029,6 +1052,29 @@ GIT_EXTERN(int) git_diff_blob_to_buffer(
 	git_diff_data_cb data_cb,
 	void *payload);
 
+/**
+ * Directly generate a patch from the difference between a blob and a buffer.
+ *
+ * This is just like `git_diff_blob_to_buffer()` except it generates a patch
+ * object for the difference instead of directly making callbacks.  You can
+ * use the standard `git_diff_patch` accessor functions to read the patch
+ * data, and you must call `git_diff_patch_free()` on the patch when done.
+ *
+ * @param out The generated patch; NULL on error
+ * @param old_blob Blob for old side of diff, or NULL for empty blob
+ * @param buffer Raw data for new side of diff, or NULL for empty
+ * @param buffer_len Length of raw data for new side of diff
+ * @param options Options for diff, or NULL for default options
+ * @return 0 on success or error code < 0
+ */
+GIT_EXTERN(int) git_diff_patch_from_blob_and_buffer(
+	git_diff_patch **out,
+	const git_blob *old_blob,
+	const char *buf,
+	size_t buflen,
+	const git_diff_options *opts);
+
+
 GIT_END_DECL
 
 /** @} */

src/array.h

+/*
+ * Copyright (C) the libgit2 contributors. All rights reserved.
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#ifndef INCLUDE_array_h__
+#define INCLUDE_array_h__
+
+#include "util.h"
+
+/*
+ * Use this to declare a typesafe resizable array of items, a la:
+ *
+ *     git_array_t(int) my_ints = GIT_ARRAY_INIT;
+ *     ...
+ *     int *i = git_array_alloc(my_ints);
+ *     GITERR_CHECK_ALLOC(i);
+ *     ...
+ *     git_array_clear(my_ints);
+ *
+ * You may also want to do things like:
+ *
+ *     typedef git_array_t(my_struct) my_struct_array_t;
+ */
+#define git_array_t(type) struct { type *ptr; uint32_t size, asize; }
+
+#define GIT_ARRAY_INIT { NULL, 0, 0 }
+
+#define git_array_init(a) \
+	do { (a).size = (a).asize = 0; (a).ptr = NULL; } while (0)
+
+#define git_array_clear(a) \
+	do { git__free((a).ptr); git_array_init(a); } while (0)
+
+#define GITERR_CHECK_ARRAY(a) GITERR_CHECK_ALLOC((a).ptr)
+
+
+typedef git_array_t(void) git_array_generic_t;
+
+/* use a generic array for growth so this can return the new item */
+GIT_INLINE(void *) git_array_grow(git_array_generic_t *a, size_t item_size)
+{
+	uint32_t new_size = (a->size < 8) ? 8 : a->asize * 3 / 2;
+	void *new_array = git__realloc(a->ptr, new_size * item_size);
+	if (!new_array) {
+		git_array_clear(*a);
+		return NULL;
+	} else {
+		a->ptr = new_array; a->asize = new_size; a->size++;
+		return (((char *)a->ptr) + (a->size - 1) * item_size);
+	}
+}
+
+#define git_array_alloc(a) \
+	((a).size >= (a).asize) ? \
+	git_array_grow((git_array_generic_t *)&(a), sizeof(*(a).ptr)) :	\
+	(a).ptr ? &(a).ptr[(a).size++] : NULL
+
+#define git_array_last(a) ((a).size ? &(a).ptr[(a).size - 1] : NULL)
+
+#define git_array_get(a, i) (((i) < (a).size) ? &(a).ptr[(i)] : NULL)
+
+#define git_array_size(a) (a).size
+
+#endif

src/blob.c

@@ -11,6 +11,7 @@
 #include "git2/odb_backend.h"
 
 #include "common.h"
+#include "filebuf.h"
 #include "blob.h"
 #include "filter.h"
 #include "buf_text.h"

src/checkout.c

@@ -20,6 +20,7 @@
 
 #include "refs.h"
 #include "repository.h"
+#include "index.h"
 #include "filter.h"
 #include "blob.h"
 #include "diff.h"

src/clone.c

@@ -21,6 +21,7 @@
 #include "fileops.h"
 #include "refs.h"
 #include "path.h"
+#include "repository.h"
 
 static int create_branch(
 	git_reference **branch,

src/crlf.c

@@ -5,14 +5,16 @@
  * a Linking Exception. For full terms see the included COPYING file.
  */
 
+#include "git2/attr.h"
+#include "git2/blob.h"
+#include "git2/index.h"
+
 #include "common.h"
 #include "fileops.h"
 #include "hash.h"
 #include "filter.h"
 #include "buf_text.h"
 #include "repository.h"
-#include "git2/attr.h"
-#include "git2/blob.h"
 
 struct crlf_attrs {
 	int crlf_action;

src/diff.c

@@ -11,6 +11,8 @@
 #include "attr_file.h"
 #include "filter.h"
 #include "pathspec.h"
+#include "index.h"
+#include "odb.h"
 
 #define DIFF_FLAG_IS_SET(DIFF,FLAG) (((DIFF)->opts.flags & (FLAG)) != 0)
 #define DIFF_FLAG_ISNT_SET(DIFF,FLAG) (((DIFF)->opts.flags & (FLAG)) == 0)
@@ -1170,3 +1172,73 @@ int git_diff_tree_to_workdir(
 
 	return error;
 }
+
+size_t git_diff_num_deltas(git_diff_list *diff)
+{
+	assert(diff);
+	return (size_t)diff->deltas.length;
+}
+
+size_t git_diff_num_deltas_of_type(git_diff_list *diff, git_delta_t type)
+{
+	size_t i, count = 0;
+	git_diff_delta *delta;
+
+	assert(diff);
+
+	git_vector_foreach(&diff->deltas, i, delta) {
+		count += (delta->status == type);
+	}
+
+	return count;
+}
+
+int git_diff__paired_foreach(
+	git_diff_list *idx2head,
+	git_diff_list *wd2idx,
+	int (*cb)(git_diff_delta *i2h, git_diff_delta *w2i, void *payload),
+	void *payload)
+{
+	int cmp;
+	git_diff_delta *i2h, *w2i;
+	size_t i, j, i_max, j_max;
+	int (*strcomp)(const char *, const char *);
+
+	i_max = idx2head ? idx2head->deltas.length : 0;
+	j_max = wd2idx   ? wd2idx->deltas.length   : 0;
+
+	/* Get appropriate strcmp function */
+	strcomp = idx2head ? idx2head->strcomp : wd2idx ? wd2idx->strcomp : NULL;
+
+	/* Assert both iterators use matching ignore-case. If this function ever
+	* supports merging diffs that are not sorted by the same function, then
+	* it will need to spool and sort on one of the results before merging
+	*/
+	if (idx2head && wd2idx) {
+		assert(idx2head->strcomp == wd2idx->strcomp);
+	}
+
+	for (i = 0, j = 0; i < i_max || j < j_max; ) {
+		i2h = idx2head ? GIT_VECTOR_GET(&idx2head->deltas,i) : NULL;
+		w2i = wd2idx   ? GIT_VECTOR_GET(&wd2idx->deltas,j)   : NULL;
+
+		cmp = !w2i ? -1 : !i2h ? 1 :
+			strcomp(i2h->old_file.path, w2i->old_file.path);
+
+		if (cmp < 0) {
+			if (cb(i2h, NULL, payload))
+				return GIT_EUSER;
+			i++;
+		} else if (cmp > 0) {
+			if (cb(NULL, w2i, payload))
+				return GIT_EUSER;
+			j++;
+		} else {
+			if (cb(i2h, w2i, payload))
+				return GIT_EUSER;
+			i++; j++;
+		}
+	}
+
+	return 0;
+}

src/diff.h

@@ -29,11 +29,16 @@ enum {
 	GIT_DIFFCAPS_TRUST_NANOSECS   = (1 << 5), /* use stat time nanoseconds */
 };
 
+#define DIFF_FLAGS_KNOWN_BINARY (GIT_DIFF_FLAG_BINARY|GIT_DIFF_FLAG_NOT_BINARY)
+#define DIFF_FLAGS_NOT_BINARY   (GIT_DIFF_FLAG_NOT_BINARY|GIT_DIFF_FLAG__NO_DATA)
+
 enum {
 	GIT_DIFF_FLAG__FREE_PATH  = (1 << 7),  /* `path` is allocated memory */
 	GIT_DIFF_FLAG__FREE_DATA  = (1 << 8),  /* internal file data is allocated */
 	GIT_DIFF_FLAG__UNMAP_DATA = (1 << 9),  /* internal file data is mmap'ed */
 	GIT_DIFF_FLAG__NO_DATA    = (1 << 10), /* file data should not be loaded */
+	GIT_DIFF_FLAG__FREE_BLOB  = (1 << 11), /* release the blob when done */
+	GIT_DIFF_FLAG__LOADED     = (1 << 12), /* file data has been loaded */
 
 	GIT_DIFF_FLAG__TO_DELETE  = (1 << 16), /* delete entry during rename det. */
 	GIT_DIFF_FLAG__TO_SPLIT   = (1 << 17), /* split entry during rename det. */
@@ -83,6 +88,12 @@ extern int git_diff__from_iterators(
 	git_iterator *new_iter,
 	const git_diff_options *opts);
 
+extern int git_diff__paired_foreach(
+	git_diff_list *idx2head,
+	git_diff_list *wd2idx,
+	int (*cb)(git_diff_delta *i2h, git_diff_delta *w2i, void *payload),
+	void *payload);
+
 int git_diff_find_similar__hashsig_for_file(
 	void **out, const git_diff_file *f, const char *path, void *p);
 

src/diff_driver.h

+/*
+ * Copyright (C) the libgit2 contributors. All rights reserved.
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#ifndef INCLUDE_diff_driver_h__
+#define INCLUDE_diff_driver_h__
+
+#include "common.h"
+#include "buffer.h"
+
+typedef struct git_diff_driver_registry git_diff_driver_registry;
+
+git_diff_driver_registry *git_diff_driver_registry_new(void);
+void git_diff_driver_registry_free(git_diff_driver_registry *);
+
+typedef struct git_diff_driver git_diff_driver;
+
+int git_diff_driver_lookup(git_diff_driver **, git_repository *, const char *);
+void git_diff_driver_free(git_diff_driver *);
+
+/* diff option flags to force off and on for this driver */
+void git_diff_driver_update_options(uint32_t *option_flags, git_diff_driver *);
+
+/* returns -1 meaning "unknown", 0 meaning not binary, 1 meaning binary */
+int git_diff_driver_content_is_binary(
+	git_diff_driver *, const char *content, size_t content_len);
+
+typedef long (*git_diff_find_context_fn)(
+	const char *, long, char *, long, void *);
+
+typedef int (*git_diff_find_context_line)(
+	git_diff_driver *, const char *, long);
+
+typedef struct {
+	git_diff_driver *driver;
+	git_diff_find_context_line match_line;
+	git_buf line;
+} git_diff_find_context_payload;
+
+void git_diff_find_context_init(
+	git_diff_find_context_fn *findfn_out,
+	git_diff_find_context_payload *payload_out,
+	git_diff_driver *driver);
+
+void git_diff_find_context_clear(git_diff_find_context_payload *);
+
+#endif

src/diff_file.h

+/*
+ * Copyright (C) the libgit2 contributors. All rights reserved.
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#ifndef INCLUDE_diff_file_h__
+#define INCLUDE_diff_file_h__
+
+#include "common.h"
+#include "diff.h"
+#include "diff_driver.h"
+#include "map.h"
+
+/* expanded information for one side of a delta */
+typedef struct {
+	git_repository *repo;
+	git_diff_file file;
+	git_diff_driver *driver;
+	uint32_t opts_flags;
+	git_off_t opts_max_size;
+	git_iterator_type_t src;
+	const git_blob *blob;
+	git_map map;
+} git_diff_file_content;
+
+extern int git_diff_file_content__init_from_diff(
+	git_diff_file_content *fc,
+	git_diff_list *diff,
+	size_t delta_index,
+	bool use_old);
+
+extern int git_diff_file_content__init_from_blob(
+	git_diff_file_content *fc,
+	git_repository *repo,
+	const git_diff_options *opts,
+	const git_blob *blob);
+
+extern int git_diff_file_content__init_from_raw(
+	git_diff_file_content *fc,
+	git_repository *repo,
+	const git_diff_options *opts,
+	const char *buf,
+	size_t buflen);
+
+/* this loads the blob/file-on-disk as needed */
+extern int git_diff_file_content__load(git_diff_file_content *fc);
+
+/* this releases the blob/file-in-memory */
+extern void git_diff_file_content__unload(git_diff_file_content *fc);
+
+/* this unloads and also releases any other resources */
+extern void git_diff_file_content__clear(git_diff_file_content *fc);
+
+#endif

src/diff_output.h

-/*
- * Copyright (C) the libgit2 contributors. All rights reserved.
- *
- * This file is part of libgit2, distributed under the GNU GPL v2 with
- * a Linking Exception. For full terms see the included COPYING file.
- */
-#ifndef INCLUDE_diff_output_h__
-#define INCLUDE_diff_output_h__
-
-#include "git2/blob.h"
-#include "diff.h"
-#include "map.h"
-#include "xdiff/xdiff.h"
-
-#define MAX_DIFF_FILESIZE 0x20000000
-
-enum {
-	GIT_DIFF_PATCH_ALLOCATED  = (1 << 0),
-	GIT_DIFF_PATCH_PREPPED    = (1 << 1),
-	GIT_DIFF_PATCH_LOADED     = (1 << 2),
-	GIT_DIFF_PATCH_DIFFABLE   = (1 << 3),
-	GIT_DIFF_PATCH_DIFFED     = (1 << 4),
-};
-
-/* context for performing diffs */
-typedef struct {
-	git_repository   *repo;
-	git_diff_list    *diff;
-	const git_diff_options *opts;
-	git_diff_file_cb  file_cb;
-	git_diff_hunk_cb  hunk_cb;
-	git_diff_data_cb  data_cb;
-	void *payload;
-	int   error;
-	git_diff_range range;
-	xdemitconf_t xdiff_config;
-	xpparam_t    xdiff_params;
-} diff_context;
-
-/* cached information about a single span in a diff */
-typedef struct diff_patch_line diff_patch_line;
-struct diff_patch_line {
-	const char *ptr;
-	size_t len;
-	size_t lines, oldno, newno;
-	char origin;
-};
-
-/* cached information about a hunk in a diff */
-typedef struct diff_patch_hunk diff_patch_hunk;
-struct diff_patch_hunk {
-	git_diff_range range;
-	char   header[128];
-	size_t header_len;
-	size_t line_start;
-	size_t line_count;
-};
-
-struct git_diff_patch {
-	git_refcount rc;
-	git_diff_list *diff; /* for refcount purposes, maybe NULL for blob diffs */
-	git_diff_delta *delta;
-	diff_context *ctxt; /* only valid while generating patch */
-	git_iterator_type_t old_src;
-	git_iterator_type_t new_src;
-	git_blob *old_blob;
-	git_blob *new_blob;
-	git_map  old_data;
-	git_map  new_data;
-	uint32_t flags;
-	diff_patch_hunk *hunks;
-	size_t hunks_asize, hunks_size;
-	diff_patch_line *lines;
-	size_t lines_asize, lines_size;
-	size_t oldno, newno;
-};
-
-/* context for performing diff on a single delta */
-typedef struct {
-	git_diff_patch *patch;
-	uint32_t prepped  : 1;
-	uint32_t loaded   : 1;
-	uint32_t diffable : 1;
-	uint32_t diffed   : 1;
-} diff_delta_context;
-
-extern int git_diff__paired_foreach(
-	git_diff_list *idx2head,
-	git_diff_list *wd2idx,
-	int (*cb)(git_diff_delta *i2h, git_diff_delta *w2i, void *payload),
-	void *payload);
-
-#endif

src/diff_patch.h

+/*
+ * Copyright (C) the libgit2 contributors. All rights reserved.
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#ifndef INCLUDE_diff_patch_h__
+#define INCLUDE_diff_patch_h__
+
+#include "common.h"
+#include "diff.h"
+#include "diff_file.h"
+#include "array.h"
+
+extern git_diff_list *git_diff_patch__diff(git_diff_patch *);
+
+extern git_diff_driver *git_diff_patch__driver(git_diff_patch *);
+
+extern void git_diff_patch__old_data(char **, size_t *, git_diff_patch *);
+extern void git_diff_patch__new_data(char **, size_t *, git_diff_patch *);
+
+extern int git_diff_patch__invoke_callbacks(
+	git_diff_patch *patch,
+	git_diff_file_cb file_cb,
+	git_diff_hunk_cb hunk_cb,
+	git_diff_data_cb line_cb,
+	void *payload);
+
+typedef struct git_diff_output git_diff_output;
+struct git_diff_output {
+	/* these callbacks are issued with the diff data */
+	git_diff_file_cb file_cb;
+	git_diff_hunk_cb hunk_cb;
+	git_diff_data_cb data_cb;
+	void *payload;
+
+	/* this records the actual error in cases where it may be obscured */
+	int error;
+
+	/* this callback is used to do the diff and drive the other callbacks.
+	 * see diff_xdiff.h for how to use this in practice for now.
+	 */
+	int (*diff_cb)(git_diff_output *output, git_diff_patch *patch);
+};
+
+#endif

src/diff_tform.c

@@ -5,10 +5,14 @@
  * a Linking Exception. For full terms see the included COPYING file.
  */
 #include "common.h"
-#include "diff.h"
+
 #include "git2/config.h"
 #include "git2/blob.h"
+
+#include "diff.h"
 #include "hashsig.h"
+#include "path.h"
+#include "fileops.h"
 
 static git_diff_delta *diff_delta__dup(
 	const git_diff_delta *d, git_pool *pool)

src/diff_xdiff.c

+/*
+ * Copyright (C) the libgit2 contributors. All rights reserved.
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#include "common.h"
+#include "diff.h"
+#include "diff_driver.h"
+#include "diff_patch.h"
+#include "diff_xdiff.h"
+
+static int git_xdiff_scan_int(const char **str, int *value)
+{
+	const char *scan = *str;
+	int v = 0, digits = 0;
+	/* find next digit */
+	for (scan = *str; *scan && !git__isdigit(*scan); scan++);
+	/* parse next number */
+	for (; git__isdigit(*scan); scan++, digits++)
+		v = (v * 10) + (*scan - '0');
+	*str = scan;
+	*value = v;
+	return (digits > 0) ? 0 : -1;
+}
+
+static int git_xdiff_parse_hunk(git_diff_range *range, const char *header)
+{
+	/* expect something of the form "@@ -%d[,%d] +%d[,%d] @@" */
+	if (*header != '@')
+		return -1;
+	if (git_xdiff_scan_int(&header, &range->old_start) < 0)
+		return -1;
+	if (*header == ',') {
+		if (git_xdiff_scan_int(&header, &range->old_lines) < 0)
+			return -1;
+	} else
+		range->old_lines = 1;
+	if (git_xdiff_scan_int(&header, &range->new_start) < 0)
+		return -1;
+	if (*header == ',') {
+		if (git_xdiff_scan_int(&header, &range->new_lines) < 0)
+			return -1;
+	} else
+		range->new_lines = 1;
+	if (range->old_start < 0 || range->new_start < 0)
+		return -1;
+
+	return 0;
+}
+
+typedef struct {
+	git_xdiff_output *xo;
+	git_diff_patch *patch;
+	git_diff_range range;
+} git_xdiff_info;
+
+static int git_xdiff_cb(void *priv, mmbuffer_t *bufs, int len)
+{
+	git_xdiff_info *info = priv;
+	git_diff_patch *patch = info->patch;
+	const git_diff_delta *delta = git_diff_patch_delta(patch);
+	git_diff_output *output = &info->xo->output;
+
+	if (len == 1) {
+		output->error = git_xdiff_parse_hunk(&info->range, bufs[0].ptr);
+		if (output->error < 0)
+			return output->error;
+
+		if (output->hunk_cb != NULL &&
+			output->hunk_cb(delta, &info->range,
+				bufs[0].ptr, bufs[0].size, output->payload))
+			output->error = GIT_EUSER;
+	}
+
+	if (len == 2 || len == 3) {
+		/* expect " "/"-"/"+", then data */
+		char origin =
+			(*bufs[0].ptr == '+') ? GIT_DIFF_LINE_ADDITION :
+			(*bufs[0].ptr == '-') ? GIT_DIFF_LINE_DELETION :
+			GIT_DIFF_LINE_CONTEXT;
+
+		if (output->data_cb != NULL &&
+			output->data_cb(delta, &info->range,
+				origin, bufs[1].ptr, bufs[1].size, output->payload))
+			output->error = GIT_EUSER;
+	}
+
+	if (len == 3 && !output->error) {
+		/* If we have a '+' and a third buf, then we have added a line
+		 * without a newline and the old code had one, so DEL_EOFNL.
+		 * If we have a '-' and a third buf, then we have removed a line
+		 * with out a newline but added a blank line, so ADD_EOFNL.
+		 */
+		char origin =
+			(*bufs[0].ptr == '+') ? GIT_DIFF_LINE_DEL_EOFNL :
+			(*bufs[0].ptr == '-') ? GIT_DIFF_LINE_ADD_EOFNL :
+			GIT_DIFF_LINE_CONTEXT_EOFNL;
+
+		if (output->data_cb != NULL &&
+			output->data_cb(delta, &info->range,
+				origin, bufs[2].ptr, bufs[2].size, output->payload))
+			output->error = GIT_EUSER;
+	}
+
+	return output->error;
+}
+
+static int git_xdiff(git_diff_output *output, git_diff_patch *patch)
+{
+	git_xdiff_output *xo = (git_xdiff_output *)output;
+	git_xdiff_info info;
+	git_diff_find_context_payload findctxt;
+	mmfile_t xd_old_data, xd_new_data;
+
+	memset(&info, 0, sizeof(info));
+	info.patch = patch;
+	info.xo    = xo;
+
+	xo->callback.priv = &info;
+
+	git_diff_find_context_init(
+		&xo->config.find_func, &findctxt, git_diff_patch__driver(patch));
+	xo->config.find_func_priv = &findctxt;
+
+	if (xo->config.find_func != NULL)
+		xo->config.flags |= XDL_EMIT_FUNCNAMES;
+	else
+		xo->config.flags &= ~XDL_EMIT_FUNCNAMES;
+
+	/* TODO: check ofile.opts_flags to see if driver-specific per-file
+	 * updates are needed to xo->params.flags
+	 */
+
+	git_diff_patch__old_data(&xd_old_data.ptr, &xd_old_data.size, patch);
+	git_diff_patch__new_data(&xd_new_data.ptr, &xd_new_data.size, patch);
+
+	xdl_diff(&xd_old_data, &xd_new_data,
+		&xo->params, &xo->config, &xo->callback);
+
+	git_diff_find_context_clear(&findctxt);
+
+	return xo->output.error;
+}
+
+void git_xdiff_init(git_xdiff_output *xo, const git_diff_options *opts)
+{
+	uint32_t flags = opts ? opts->flags : GIT_DIFF_NORMAL;
+
+	xo->output.diff_cb = git_xdiff;
+
+	memset(&xo->config, 0, sizeof(xo->config));
+	xo->config.ctxlen = opts ? opts->context_lines : 3;
+	xo->config.interhunkctxlen = opts ? opts->interhunk_lines : 0;
+
+	memset(&xo->params, 0, sizeof(xo->params));
+	if (flags & GIT_DIFF_IGNORE_WHITESPACE)
+		xo->params.flags |= XDF_WHITESPACE_FLAGS;
+	if (flags & GIT_DIFF_IGNORE_WHITESPACE_CHANGE)
+		xo->params.flags |= XDF_IGNORE_WHITESPACE_CHANGE;
+	if (flags & GIT_DIFF_IGNORE_WHITESPACE_EOL)
+		xo->params.flags |= XDF_IGNORE_WHITESPACE_AT_EOL;
+
+	memset(&xo->callback, 0, sizeof(xo->callback));
+	xo->callback.outf = git_xdiff_cb;
+}

src/diff_xdiff.h

+/*
+ * Copyright (C) the libgit2 contributors. All rights reserved.
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#ifndef INCLUDE_diff_xdiff_h__
+#define INCLUDE_diff_xdiff_h__
+
+#include "diff.h"
+#include "diff_patch.h"
+#include "xdiff/xdiff.h"
+
+/* A git_xdiff_output is a git_diff_output with extra fields necessary
+ * to use libxdiff.  Calling git_xdiff_init() will set the diff_cb field
+ * of the output to use xdiff to generate the diffs.
+ */
+typedef struct {
+	git_diff_output output;
+
+	xdemitconf_t config;
+	xpparam_t    params;
+	xdemitcb_t   callback;
+} git_xdiff_output;
+
+void git_xdiff_init(git_xdiff_output *xo, const git_diff_options *opts);
+
+#endif

src/fetch.c

@@ -16,6 +16,8 @@
 #include "pack.h"
 #include "fetch.h"
 #include "netops.h"
+#include "repository.h"
+#include "refs.h"
 
 struct filter_payload {
 	git_remote *remote;

src/index.c

@@ -2030,6 +2030,8 @@ int git_index_read_tree(git_index *index, const git_tree *tree)
 	error = git_tree_walk(tree, GIT_TREEWALK_POST, read_tree_cb, &data);
 
 	index_entries_free(&entries);
+	git_vector_free(&entries);
+
 	git_vector_sort(&index->entries);
 
 	return error;

src/iterator.c

@@ -7,6 +7,7 @@
 
 #include "iterator.h"
 #include "tree.h"
+#include "index.h"
 #include "ignore.h"
 #include "buffer.h"
 #include "git2/submodule.h"

src/merge.c

@@ -24,6 +24,8 @@
 #include "blob.h"
 #include "hashsig.h"
 #include "oid.h"
+#include "index.h"
+#include "filebuf.h"
 
 #include "git2/types.h"
 #include "git2/repository.h"

src/refdb_fs.c

@@ -9,6 +9,7 @@
 #include "hash.h"
 #include "repository.h"
 #include "fileops.h"
+#include "filebuf.h"
 #include "pack.h"
 #include "reflog.h"
 #include "refdb.h"

src/refs.c

@@ -9,6 +9,7 @@
 #include "hash.h"
 #include "repository.h"
 #include "fileops.h"
+#include "filebuf.h"
 #include "pack.h"
 #include "reflog.h"
 #include "refdb.h"

src/remote.c

@@ -1267,8 +1267,10 @@ static int rename_remote_references(
 		return -1;
 
 	while ((error = git_reference_next(&ref, iter)) == 0) {
-		if (git__prefixcmp(ref->name, GIT_REFS_REMOTES_DIR))
+		if (git__prefixcmp(ref->name, GIT_REFS_REMOTES_DIR)) {
+			git_reference_free(ref);
 			continue;
+		}
 
 		if ((error = rename_one_remote_reference(ref, old_name, new_name)) < 0) {
 			git_reference_iterator_free(iter);

src/remote.h

@@ -11,7 +11,7 @@
 #include "git2/transport.h"
 
 #include "refspec.h"
-#include "repository.h"
+#include "vector.h"
 
 #define GIT_REMOTE_ORIGIN "origin"
 

src/repository.c

@@ -17,12 +17,15 @@
 #include "tag.h"
 #include "blob.h"
 #include "fileops.h"
+#include "filebuf.h"
+#include "index.h"
 #include "config.h"
 #include "refs.h"
 #include "filter.h"
 #include "odb.h"
 #include "remote.h"
 #include "merge.h"
+#include "diff_driver.h"
 
 #define GIT_FILE_CONTENT_PREFIX "gitdir:"
 
@@ -109,6 +112,9 @@ void git_repository_free(git_repository *repo)
 	git_cache_free(&repo->objects);
 	git_submodule_config_free(repo);
 
+	git_diff_driver_registry_free(repo->diff_drivers);
+	repo->diff_drivers = NULL;
+
 	git__free(repo->path_repository);
 	git__free(repo->workdir);
 	git__free(repo->namespace);

src/repository.h

@@ -14,15 +14,13 @@
 #include "git2/object.h"
 #include "git2/config.h"
 
-#include "index.h"
 #include "cache.h"
 #include "refs.h"
 #include "buffer.h"
-#include "odb.h"
 #include "object.h"
 #include "attrcache.h"
 #include "strmap.h"
-#include "refdb.h"
+#include "diff_driver.h"
 
 #define DOT_GIT ".git"
 #define GIT_DIR DOT_GIT "/"
@@ -108,6 +106,7 @@ struct git_repository {
 	git_cache objects;
 	git_attr_cache attrcache;
 	git_strmap *submodules;
+	git_diff_driver_registry *diff_drivers;
 
 	char *path_repository;
 	char *workdir;

src/signature.c

@@ -9,6 +9,7 @@
 #include "signature.h"
 #include "repository.h"
 #include "git2/common.h"
+#include "posix.h"
 
 void git_signature_free(git_signature *sig)
 {

src/stash.c

@@ -14,6 +14,7 @@
 #include "git2/stash.h"
 #include "git2/status.h"
 #include "git2/checkout.h"
+#include "git2/index.h"
 #include "signature.h"
 
 static int create_error(int error, const char *msg)

src/status.c

@@ -14,10 +14,10 @@
 #include "git2/status.h"
 #include "repository.h"
 #include "ignore.h"
+#include "index.h"
 
 #include "git2/diff.h"
 #include "diff.h"
-#include "diff_output.h"
 
 static unsigned int index_delta2status(git_delta_t index_status)
 {

src/submodule.c

@@ -22,6 +22,8 @@
 #include "submodule.h"
 #include "tree.h"
 #include "iterator.h"
+#include "path.h"
+#include "index.h"
 
 #define GIT_MODULES_FILE ".gitmodules"
 

src/thread-utils.h

@@ -7,8 +7,6 @@
 #ifndef INCLUDE_thread_utils_h__
 #define INCLUDE_thread_utils_h__
 
-#include "common.h"
-
 /* Common operations even if threading has been disabled */
 typedef struct {
 #if defined(GIT_WIN32)

src/tree.c

@@ -10,6 +10,9 @@
 #include "tree.h"
 #include "git2/repository.h"
 #include "git2/object.h"
+#include "path.h"
+#include "tree-cache.h"
+#include "index.h"
 
 #define DEFAULT_TREE_SIZE 16
 #define MAX_FILEMODE_BYTES 6

src/util.h

@@ -194,6 +194,8 @@ extern int git__strcasecmp(const char *a, const char *b);
 extern int git__strncmp(const char *a, const char *b, size_t sz);
 extern int git__strncasecmp(const char *a, const char *b, size_t sz);
 
+#include "thread-utils.h"
+
 typedef struct {
 	git_atomic refcount;
 	void *owner;

tests-clar/checkout/index.c

@@ -2,6 +2,7 @@
 #include "checkout_helpers.h"
 
 #include "git2/checkout.h"
+#include "fileops.h"
 #include "repository.h"
 
 static git_repository *g_repo;

tests-clar/clar.c

@@ -183,10 +183,10 @@ clar_run_test(
 }
 
 static void
-clar_run_suite(const struct clar_suite *suite)
+clar_run_suite(const struct clar_suite *suite, const char *name)
 {
 	const struct clar_func *test = suite->tests;
-	size_t i;
+	size_t i, namelen;
 
 	if (!suite->enabled)
 		return;
@@ -200,7 +200,23 @@ clar_run_suite(const struct clar_suite *suite)
 	_clar.active_suite = suite->name;
 	_clar.suite_errors = 0;
 
+	if (name) {
+		size_t suitelen = strlen(suite->name);
+		namelen = strlen(name);
+		if (namelen <= suitelen) {
+			name = NULL;
+		} else {
+			name += suitelen;
+			while (*name == ':')
+				++name;
+			namelen = strlen(name);
+		}
+	}
+
 	for (i = 0; i < suite->test_count; ++i) {
+		if (name && strncmp(test[i].name, name, namelen))
+			continue;
+
 		_clar.active_test = test[i].name;
 		clar_run_test(&test[i], &suite->initialize, &suite->cleanup);
 
@@ -240,7 +256,7 @@ clar_parse_args(int argc, char **argv)
 		case 'x': { /* given suite name */
 			int offset = (argument[2] == '=') ? 3 : 2, found = 0;
 			char action = argument[1];
-			size_t j, len;
+			size_t j, len, cmplen;
 
 			argument += offset;
 			len = strlen(argument);
@@ -249,7 +265,11 @@ clar_parse_args(int argc, char **argv)
 				clar_usage(argv[0]);
 
 			for (j = 0; j < _clar_suite_count; ++j) {
-				if (strncmp(argument, _clar_suites[j].name, len) == 0) {
+				cmplen = strlen(_clar_suites[j].name);
+				if (cmplen > len)
+					cmplen = len;
+
+				if (strncmp(argument, _clar_suites[j].name, cmplen) == 0) {
 					int exact = !strcmp(argument, _clar_suites[j].name);
 
 					++found;
@@ -258,7 +278,7 @@ clar_parse_args(int argc, char **argv)
 						_clar.report_suite_names = 1;
 
 					switch (action) {
-						case 's': clar_run_suite(&_clar_suites[j]); break;
+					case 's': clar_run_suite(&_clar_suites[j], argument); break;
 					case 'i': _clar_suites[j].enabled = 1; break;
 					case 'x': _clar_suites[j].enabled = 0; break;
 					}
@@ -318,7 +338,7 @@ clar_test(int argc, char **argv)
 	if (!_clar.suites_ran) {
 		size_t i;
 		for (i = 0; i < _clar_suite_count; ++i)
-			clar_run_suite(&_clar_suites[i]);
+			clar_run_suite(&_clar_suites[i], NULL);
 	}
 
 	clar_print_shutdown(

tests-clar/clone/nonetwork.c

 #include "clar_libgit2.h"
 
 #include "git2/clone.h"
-#include "repository.h"
 #include "remote.h"
+#include "fileops.h"
+#include "repository.h"
 
 #define LIVE_REPO_URL "git://github.com/libgit2/TestGitRepository"
 

tests-clar/diff/blob.c

@@ -120,6 +120,93 @@ void test_diff_blob__can_compare_text_blobs(void)
 	git_blob_free(c);
 }
 
+void test_diff_blob__can_compare_text_blobs_with_patch(void)
+{
+	git_blob *a, *b, *c;
+	git_oid a_oid, b_oid, c_oid;
+	git_diff_patch *p;
+	size_t tc, ta, td;
+
+	/* tests/resources/attr/root_test1 */
+	cl_git_pass(git_oid_fromstrn(&a_oid, "45141a79", 8));
+	cl_git_pass(git_blob_lookup_prefix(&a, g_repo, &a_oid, 4));
+
+	/* tests/resources/attr/root_test2 */
+	cl_git_pass(git_oid_fromstrn(&b_oid, "4d713dc4", 8));
+	cl_git_pass(git_blob_lookup_prefix(&b, g_repo, &b_oid, 4));
+
+	/* tests/resources/attr/root_test3 */
+	cl_git_pass(git_oid_fromstrn(&c_oid, "c96bbb2c2557a832", 16));
+	cl_git_pass(git_blob_lookup_prefix(&c, g_repo, &c_oid, 8));
+
+	/* Doing the equivalent of a `git diff -U1` on these files */
+
+	/* diff on tests/resources/attr/root_test1 */
+	cl_git_pass(git_diff_patch_from_blobs(&p, a, b, &opts));
+
+	cl_assert(p != NULL);
+	cl_assert_equal_i(GIT_DELTA_MODIFIED, git_diff_patch_delta(p)->status);
+	cl_assert_equal_i(1, (int)git_diff_patch_num_hunks(p));
+	cl_assert_equal_i(6, git_diff_patch_num_lines_in_hunk(p, 0));
+
+	cl_git_pass(git_diff_patch_line_stats(&tc, &ta, &td, p));
+	cl_assert_equal_i(1, (int)tc);
+	cl_assert_equal_i(5, (int)ta);
+	cl_assert_equal_i(0, (int)td);
+
+	git_diff_patch_free(p);
+
+	/* diff on tests/resources/attr/root_test2 */
+	cl_git_pass(git_diff_patch_from_blobs(&p, b, c, &opts));
+
+	cl_assert(p != NULL);
+	cl_assert_equal_i(GIT_DELTA_MODIFIED, git_diff_patch_delta(p)->status);
+	cl_assert_equal_i(1, (int)git_diff_patch_num_hunks(p));
+	cl_assert_equal_i(15, git_diff_patch_num_lines_in_hunk(p, 0));
+
+	cl_git_pass(git_diff_patch_line_stats(&tc, &ta, &td, p));
+	cl_assert_equal_i(3, (int)tc);
+	cl_assert_equal_i(9, (int)ta);
+	cl_assert_equal_i(3, (int)td);
+
+	git_diff_patch_free(p);
+
+	/* diff on tests/resources/attr/root_test3 */
+	cl_git_pass(git_diff_patch_from_blobs(&p, a, c, &opts));
+
+	cl_assert(p != NULL);
+	cl_assert_equal_i(GIT_DELTA_MODIFIED, git_diff_patch_delta(p)->status);
+	cl_assert_equal_i(1, (int)git_diff_patch_num_hunks(p));
+	cl_assert_equal_i(13, git_diff_patch_num_lines_in_hunk(p, 0));
+
+	cl_git_pass(git_diff_patch_line_stats(&tc, &ta, &td, p));
+	cl_assert_equal_i(0, (int)tc);
+	cl_assert_equal_i(12, (int)ta);
+	cl_assert_equal_i(1, (int)td);
+
+	git_diff_patch_free(p);
+
+	/* one more */
+	cl_git_pass(git_diff_patch_from_blobs(&p, c, d, &opts));
+
+	cl_assert(p != NULL);
+	cl_assert_equal_i(GIT_DELTA_MODIFIED, git_diff_patch_delta(p)->status);
+	cl_assert_equal_i(2, (int)git_diff_patch_num_hunks(p));
+	cl_assert_equal_i(5, git_diff_patch_num_lines_in_hunk(p, 0));
+	cl_assert_equal_i(9, git_diff_patch_num_lines_in_hunk(p, 1));
+
+	cl_git_pass(git_diff_patch_line_stats(&tc, &ta, &td, p));
+	cl_assert_equal_i(4, (int)tc);
+	cl_assert_equal_i(6, (int)ta);
+	cl_assert_equal_i(4, (int)td);
+
+	git_diff_patch_free(p);
+
+	git_blob_free(a);
+	git_blob_free(b);
+	git_blob_free(c);
+}
+
 void test_diff_blob__can_compare_against_null_blobs(void)
 {
 	git_blob *e = NULL;
@@ -175,6 +262,66 @@ void test_diff_blob__can_compare_against_null_blobs(void)
 	cl_assert_equal_i(0, expected.lines);
 }
 
+void test_diff_blob__can_compare_against_null_blobs_with_patch(void)
+{
+	git_blob *e = NULL;
+	git_diff_patch *p;
+	int line;
+	char origin;
+
+	cl_git_pass(git_diff_patch_from_blobs(&p, d, e, &opts));
+
+	cl_assert(p != NULL);
+	cl_assert_equal_i(GIT_DELTA_DELETED, git_diff_patch_delta(p)->status);
+	cl_assert_equal_i(1, (int)git_diff_patch_num_hunks(p));
+	cl_assert_equal_i(14, git_diff_patch_num_lines_in_hunk(p, 0));
+
+	for (line = 0; line < git_diff_patch_num_lines_in_hunk(p, 0); ++line) {
+		cl_git_pass(git_diff_patch_get_line_in_hunk(
+			&origin, NULL, NULL, NULL, NULL, p, 0, line));
+		cl_assert_equal_i(GIT_DIFF_LINE_DELETION, (int)origin);
+	}
+
+	git_diff_patch_free(p);
+
+	opts.flags |= GIT_DIFF_REVERSE;
+
+	cl_git_pass(git_diff_patch_from_blobs(&p, d, e, &opts));
+
+	cl_assert(p != NULL);
+	cl_assert_equal_i(GIT_DELTA_ADDED, git_diff_patch_delta(p)->status);
+	cl_assert_equal_i(1, (int)git_diff_patch_num_hunks(p));
+	cl_assert_equal_i(14, git_diff_patch_num_lines_in_hunk(p, 0));
+
+	for (line = 0; line < git_diff_patch_num_lines_in_hunk(p, 0); ++line) {
+		cl_git_pass(git_diff_patch_get_line_in_hunk(
+			&origin, NULL, NULL, NULL, NULL, p, 0, line));
+		cl_assert_equal_i(GIT_DIFF_LINE_ADDITION, (int)origin);
+	}
+
+	git_diff_patch_free(p);
+
+	opts.flags ^= GIT_DIFF_REVERSE;
+
+	cl_git_pass(git_diff_patch_from_blobs(&p, alien, NULL, &opts));
+
+	cl_assert(p != NULL);
+	cl_assert_equal_i(GIT_DELTA_DELETED, git_diff_patch_delta(p)->status);
+	cl_assert((git_diff_patch_delta(p)->flags & GIT_DIFF_FLAG_BINARY) != 0);
+	cl_assert_equal_i(0, (int)git_diff_patch_num_hunks(p));
+
+	git_diff_patch_free(p);
+
+	cl_git_pass(git_diff_patch_from_blobs(&p, NULL, alien, &opts));
+
+	cl_assert(p != NULL);
+	cl_assert_equal_i(GIT_DELTA_ADDED, git_diff_patch_delta(p)->status);
+	cl_assert((git_diff_patch_delta(p)->flags & GIT_DIFF_FLAG_BINARY) != 0);
+	cl_assert_equal_i(0, (int)git_diff_patch_num_hunks(p));
+
+	git_diff_patch_free(p);
+}
+
 static void assert_identical_blobs_comparison(diff_expects *expected)
 {
 	cl_assert_equal_i(1, expected->files);
@@ -206,6 +353,29 @@ void test_diff_blob__can_compare_identical_blobs(void)
 	assert_identical_blobs_comparison(&expected);
 }
 
+void test_diff_blob__can_compare_identical_blobs_with_patch(void)
+{
+	git_diff_patch *p;
+
+	cl_git_pass(git_diff_patch_from_blobs(&p, d, d, &opts));
+	cl_assert(p != NULL);
+	cl_assert_equal_i(GIT_DELTA_UNMODIFIED, git_diff_patch_delta(p)->status);
+	cl_assert_equal_i(0, (int)git_diff_patch_num_hunks(p));
+	git_diff_patch_free(p);
+
+	cl_git_pass(git_diff_patch_from_blobs(&p, NULL, NULL, &opts));
+	cl_assert(p != NULL);
+	cl_assert_equal_i(GIT_DELTA_UNMODIFIED, git_diff_patch_delta(p)->status);
+	cl_assert_equal_i(0, (int)git_diff_patch_num_hunks(p));
+	git_diff_patch_free(p);
+
+	cl_git_pass(git_diff_patch_from_blobs(&p, alien, alien, &opts));
+	cl_assert(p != NULL);
+	cl_assert_equal_i(GIT_DELTA_UNMODIFIED, git_diff_patch_delta(p)->status);
+	cl_assert_equal_i(0, (int)git_diff_patch_num_hunks(p));
+	git_diff_patch_free(p);
+}
+
 static void assert_binary_blobs_comparison(diff_expects *expected)
 {
 	cl_assert(expected->files_binary > 0);
@@ -428,6 +598,74 @@ void test_diff_blob__can_compare_blob_to_buffer(void)
 	git_blob_free(a);
 }
 
+void test_diff_blob__can_compare_blob_to_buffer_with_patch(void)
+{
+	git_diff_patch *p;
+	git_blob *a;
+	git_oid a_oid;
+	const char *a_content = "Hello from the root\n";
+	const char *b_content = "Hello from the root\n\nSome additional lines\n\nDown here below\n\n";
+	size_t tc, ta, td;
+
+	/* tests/resources/attr/root_test1 */
+	cl_git_pass(git_oid_fromstrn(&a_oid, "45141a79", 8));
+	cl_git_pass(git_blob_lookup_prefix(&a, g_repo, &a_oid, 4));
+
+	/* diff from blob a to content of b */
+	cl_git_pass(git_diff_patch_from_blob_and_buffer(
+		&p, a, b_content, strlen(b_content), &opts));
+
+	cl_assert(p != NULL);
+	cl_assert_equal_i(GIT_DELTA_MODIFIED, git_diff_patch_delta(p)->status);
+	cl_assert_equal_i(1, (int)git_diff_patch_num_hunks(p));
+	cl_assert_equal_i(6, git_diff_patch_num_lines_in_hunk(p, 0));
+
+	cl_git_pass(git_diff_patch_line_stats(&tc, &ta, &td, p));
+	cl_assert_equal_i(1, (int)tc);
+	cl_assert_equal_i(5, (int)ta);
+	cl_assert_equal_i(0, (int)td);
+
+	git_diff_patch_free(p);
+
+	/* diff from blob a to content of a */
+	cl_git_pass(git_diff_patch_from_blob_and_buffer(
+		&p, a, a_content, strlen(a_content), &opts));
+	cl_assert(p != NULL);
+	cl_assert_equal_i(GIT_DELTA_UNMODIFIED, git_diff_patch_delta(p)->status);
+	cl_assert_equal_i(0, (int)git_diff_patch_num_hunks(p));
+	git_diff_patch_free(p);
+
+	/* diff from NULL blob to content of a */
+	cl_git_pass(git_diff_patch_from_blob_and_buffer(
+		&p, NULL, a_content, strlen(a_content), &opts));
+	cl_assert(p != NULL);
+	cl_assert_equal_i(GIT_DELTA_ADDED, git_diff_patch_delta(p)->status);
+	cl_assert_equal_i(1, (int)git_diff_patch_num_hunks(p));
+	cl_assert_equal_i(1, git_diff_patch_num_lines_in_hunk(p, 0));
+	git_diff_patch_free(p);
+
+	/* diff from blob a to NULL buffer */
+	cl_git_pass(git_diff_patch_from_blob_and_buffer(
+		&p, a, NULL, 0, &opts));
+	cl_assert(p != NULL);
+	cl_assert_equal_i(GIT_DELTA_DELETED, git_diff_patch_delta(p)->status);
+	cl_assert_equal_i(1, (int)git_diff_patch_num_hunks(p));
+	cl_assert_equal_i(1, git_diff_patch_num_lines_in_hunk(p, 0));
+	git_diff_patch_free(p);
+
+	/* diff with reverse */
+	opts.flags ^= GIT_DIFF_REVERSE;
+
+	cl_git_pass(git_diff_patch_from_blob_and_buffer(
+		&p, a, NULL, 0, &opts));
+	cl_assert(p != NULL);
+	cl_assert_equal_i(GIT_DELTA_ADDED, git_diff_patch_delta(p)->status);
+	cl_assert_equal_i(1, (int)git_diff_patch_num_hunks(p));
+	cl_assert_equal_i(1, git_diff_patch_num_lines_in_hunk(p, 0));
+	git_diff_patch_free(p);
+
+	git_blob_free(a);
+}
 
 static void assert_one_modified_with_lines(diff_expects *expected, int lines)
 {

tests-clar/diff/drivers.c

+#include "clar_libgit2.h"
+#include "diff_helpers.h"
+#include "repository.h"
+#include "diff_driver.h"
+
+static git_repository *g_repo = NULL;
+
+void test_diff_drivers__initialize(void)
+{
+}
+
+void test_diff_drivers__cleanup(void)
+{
+	cl_git_sandbox_cleanup();
+	g_repo = NULL;
+}
+
+void test_diff_drivers__patterns(void)
+{
+	git_config *cfg;
+	const char *one_sha = "19dd32dfb1520a64e5bbaae8dce6ef423dfa2f13";
+	git_tree *one;
+	git_diff_list *diff;
+	git_diff_patch *patch;
+	char *text;
+	const char *expected0 = "diff --git a/untimely.txt b/untimely.txt\nindex 9a69d96..57fd0cf 100644\n--- a/untimely.txt\n+++ b/untimely.txt\n@@ -22,3 +22,5 @@ Comes through the blood of the vanguards who\n   dreamed--too soon--it had sounded.\r\n \r\n                 -- Rudyard Kipling\r\n+\r\n+Some new stuff\r\n";
+	const char *expected1 = "diff --git a/untimely.txt b/untimely.txt\nindex 9a69d96..57fd0cf 100644\nBinary files a/untimely.txt and b/untimely.txt differ\n";
+	const char *expected2 = "diff --git a/untimely.txt b/untimely.txt\nindex 9a69d96..57fd0cf 100644\n--- a/untimely.txt\n+++ b/untimely.txt\n@@ -22,3 +22,5 @@ Heaven delivers on earth the Hour that cannot be\n   dreamed--too soon--it had sounded.\r\n \r\n                 -- Rudyard Kipling\r\n+\r\n+Some new stuff\r\n";
+
+	g_repo = cl_git_sandbox_init("renames");
+
+	one = resolve_commit_oid_to_tree(g_repo, one_sha);
+
+	/* no diff */
+
+	cl_git_pass(git_diff_tree_to_workdir(&diff, g_repo, one, NULL));
+	cl_assert_equal_i(0, (int)git_diff_num_deltas(diff));
+	git_diff_list_free(diff);
+
+	/* default diff */
+
+	cl_git_append2file("renames/untimely.txt", "\r\nSome new stuff\r\n");
+
+	cl_git_pass(git_diff_tree_to_workdir(&diff, g_repo, one, NULL));
+	cl_assert_equal_i(1, (int)git_diff_num_deltas(diff));
+
+	cl_git_pass(git_diff_get_patch(&patch, NULL, diff, 0));
+	cl_git_pass(git_diff_patch_to_str(&text, patch));
+	cl_assert_equal_s(expected0, text);
+
+	git__free(text);
+	git_diff_patch_free(patch);
+	git_diff_list_free(diff);
+
+	/* attribute diff set to false */
+
+	cl_git_rewritefile("renames/.gitattributes", "untimely.txt -diff\n");
+
+	cl_git_pass(git_diff_tree_to_workdir(&diff, g_repo, one, NULL));
+	cl_assert_equal_i(1, (int)git_diff_num_deltas(diff));
+
+	cl_git_pass(git_diff_get_patch(&patch, NULL, diff, 0));
+	cl_git_pass(git_diff_patch_to_str(&text, patch));
+	cl_assert_equal_s(expected1, text);
+
+	git__free(text);
+	git_diff_patch_free(patch);
+	git_diff_list_free(diff);
+
+	/* attribute diff set to unconfigured value (should use default) */
+
+	cl_git_rewritefile("renames/.gitattributes", "untimely.txt diff=kipling0\n");
+
+	cl_git_pass(git_diff_tree_to_workdir(&diff, g_repo, one, NULL));
+	cl_assert_equal_i(1, (int)git_diff_num_deltas(diff));
+
+	cl_git_pass(git_diff_get_patch(&patch, NULL, diff, 0));
+	cl_git_pass(git_diff_patch_to_str(&text, patch));
+	cl_assert_equal_s(expected0, text);
+
+	git__free(text);
+	git_diff_patch_free(patch);
+	git_diff_list_free(diff);
+
+	/* let's define that driver */
+
+	cl_git_pass(git_repository_config(&cfg, g_repo));
+	cl_git_pass(git_config_set_bool(cfg, "diff.kipling0.binary", 1));
+	git_config_free(cfg);
+
+	cl_git_pass(git_diff_tree_to_workdir(&diff, g_repo, one, NULL));
+	cl_assert_equal_i(1, (int)git_diff_num_deltas(diff));
+
+	cl_git_pass(git_diff_get_patch(&patch, NULL, diff, 0));
+	cl_git_pass(git_diff_patch_to_str(&text, patch));
+	cl_assert_equal_s(expected1, text);
+
+	git__free(text);
+	git_diff_patch_free(patch);
+	git_diff_list_free(diff);
+
+	/* let's use a real driver with some regular expressions */
+
+	git_diff_driver_registry_free(g_repo->diff_drivers);
+	g_repo->diff_drivers = NULL;
+
+	cl_git_pass(git_repository_config(&cfg, g_repo));
+	cl_git_pass(git_config_set_bool(cfg, "diff.kipling0.binary", 0));
+	cl_git_pass(git_config_set_string(cfg, "diff.kipling0.xfuncname", "^H"));
+	git_config_free(cfg);
+
+	cl_git_pass(git_diff_tree_to_workdir(&diff, g_repo, one, NULL));
+	cl_assert_equal_i(1, (int)git_diff_num_deltas(diff));
+
+	cl_git_pass(git_diff_get_patch(&patch, NULL, diff, 0));
+	cl_git_pass(git_diff_patch_to_str(&text, patch));
+	cl_assert_equal_s(expected2, text);
+
+	git__free(text);
+	git_diff_patch_free(patch);
+	git_diff_list_free(diff);
+
+	git_tree_free(one);
+}
+

tests-clar/diff/patch.c

@@ -543,7 +543,7 @@ void test_diff_patch__line_counts_with_eofnl(void)
 		"index 378a7d9..3d0154e 100644\n"
 		"--- a/songof7cities.txt\n"
 		"+++ b/songof7cities.txt\n"
-		"@@ -42,7 +42,7 @@\n"
+		"@@ -42,7 +42,7 @@ With peoples undefeated of the dark, enduring blood.\n"
 		" \n"
 		" To the sound of trumpets shall their seed restore my Cities\n"
 		" Wealthy and well-weaponed, that once more may I behold\n"

tests-clar/diff/rename.c

@@ -558,7 +558,7 @@ void test_diff_rename__patch(void)
 	git_diff_patch *patch;
 	const git_diff_delta *delta;
 	char *text;
-	const char *expected = "diff --git a/sixserving.txt b/ikeepsix.txt\nindex ad0a8e5..36020db 100644\n--- a/sixserving.txt\n+++ b/ikeepsix.txt\n@@ -1,3 +1,6 @@\n+I Keep Six Honest Serving-Men\n+=============================\n+\n I KEEP six honest serving-men\n  (They taught me all I knew);\n Their names are What and Why and When\n@@ -21,4 +24,4 @@\n One million Hows, two million Wheres,\n And seven million Whys!\n \n-                -- Rudyard Kipling\n+  -- Rudyard Kipling\n";
+	const char *expected = "diff --git a/sixserving.txt b/ikeepsix.txt\nindex ad0a8e5..36020db 100644\n--- a/sixserving.txt\n+++ b/ikeepsix.txt\n@@ -1,3 +1,6 @@\n+I Keep Six Honest Serving-Men\n+=============================\n+\n I KEEP six honest serving-men\n  (They taught me all I knew);\n Their names are What and Why and When\n@@ -21,4 +24,4 @@ She sends'em abroad on her own affairs,\n One million Hows, two million Wheres,\n And seven million Whys!\n \n-                -- Rudyard Kipling\n+  -- Rudyard Kipling\n";
 
 	old_tree = resolve_commit_oid_to_tree(g_repo, sha0);
 	new_tree = resolve_commit_oid_to_tree(g_repo, sha1);

tests-clar/diff/submodules.c

 #include "clar_libgit2.h"
 #include "repository.h"
+#include "posix.h"
 #include "../submodule/submodule_helpers.h"
 
 static git_repository *g_repo = NULL;

tests-clar/fetchhead/nonetwork.c

 #include "clar_libgit2.h"
 
-#include "repository.h"
+#include "fileops.h"
 #include "fetchhead.h"
 
 #include "fetchhead_data.h"

tests-clar/merge/merge_helpers.c

 #include "clar_libgit2.h"
-#include "buffer.h"
+#include "fileops.h"
 #include "refs.h"
 #include "tree.h"
 #include "merge_helpers.h"

tests-clar/odb/alternates.c

 #include "clar_libgit2.h"
 #include "odb.h"
-#include "repository.h"
+#include "filebuf.h"
 
 static git_buf destpath, filepath;
 static const char *paths[] = {

tests-clar/online/clone.c

@@ -2,8 +2,9 @@
 
 #include "git2/clone.h"
 #include "git2/cred_helpers.h"
-#include "repository.h"
 #include "remote.h"
+#include "fileops.h"
+#include "refs.h"
 
 #define LIVE_REPO_URL "http://github.com/libgit2/TestGitRepository"
 #define LIVE_EMPTYREPO_URL "http://github.com/libgit2/TestEmptyRepository"

tests-clar/online/fetchhead.c

 #include "clar_libgit2.h"
 
-#include "repository.h"
+#include "fileops.h"
 #include "fetchhead.h"
 #include "../fetchhead/fetchhead_data.h"
 #include "git2/clone.h"

tests-clar/refs/delete.c

 #include "clar_libgit2.h"
 
-#include "repository.h"
+#include "fileops.h"
 #include "git2/reflog.h"
+#include "git2/refdb.h"
 #include "reflog.h"
 #include "ref_helpers.h"
 
@@ -31,7 +32,7 @@ void test_refs_delete__packed_loose(void)
 	git_buf temp_path = GIT_BUF_INIT;
 
 	/* Ensure the loose reference exists on the file system */
-	cl_git_pass(git_buf_joinpath(&temp_path, g_repo->path_repository, packed_test_head_name));
+	cl_git_pass(git_buf_joinpath(&temp_path, git_repository_path(g_repo), packed_test_head_name));
 	cl_assert(git_path_exists(temp_path.ptr));
 
 	/* Lookup the reference */

tests-clar/refs/pack.c

 #include "clar_libgit2.h"
 
-#include "repository.h"
+#include "fileops.h"
 #include "git2/reflog.h"
+#include "git2/refdb.h"
 #include "reflog.h"
+#include "refs.h"
 #include "ref_helpers.h"
 
 static const char *loose_tag_ref_name = "refs/tags/e90810b";
@@ -33,7 +35,7 @@ void test_refs_pack__empty(void)
    // create a packfile for an empty folder
 	git_buf temp_path = GIT_BUF_INIT;
 
-	cl_git_pass(git_buf_join_n(&temp_path, '/', 3, g_repo->path_repository, GIT_REFS_HEADS_DIR, "empty_dir"));
+	cl_git_pass(git_buf_join_n(&temp_path, '/', 3, git_repository_path(g_repo), GIT_REFS_HEADS_DIR, "empty_dir"));
 	cl_git_pass(git_futils_mkdir_r(temp_path.ptr, NULL, GIT_REFS_DIR_MODE));
 	git_buf_free(&temp_path);
 
@@ -60,7 +62,7 @@ void test_refs_pack__loose(void)
 	packall();
 
 	/* Ensure the packed-refs file exists */
-	cl_git_pass(git_buf_joinpath(&temp_path, g_repo->path_repository, GIT_PACKEDREFS_FILE));
+	cl_git_pass(git_buf_joinpath(&temp_path, git_repository_path(g_repo), GIT_PACKEDREFS_FILE));
 	cl_assert(git_path_exists(temp_path.ptr));
 
 	/* Ensure the known ref can still be looked up but is now packed */
@@ -69,7 +71,7 @@ void test_refs_pack__loose(void)
 	cl_assert_equal_s(reference->name, loose_tag_ref_name);
 
 	/* Ensure the known ref has been removed from the loose folder structure */
-	cl_git_pass(git_buf_joinpath(&temp_path, g_repo->path_repository, loose_tag_ref_name));
+	cl_git_pass(git_buf_joinpath(&temp_path, git_repository_path(g_repo), loose_tag_ref_name));
 	cl_assert(!git_path_exists(temp_path.ptr));
 
 	git_reference_free(reference);

tests-clar/refs/reflog/reflog.c

 #include "clar_libgit2.h"
 
-#include "repository.h"
+#include "fileops.h"
 #include "git2/reflog.h"
 #include "reflog.h"
 

tests-clar/refs/rename.c

 #include "clar_libgit2.h"
 
-#include "repository.h"
+#include "fileops.h"
 #include "git2/reflog.h"
 #include "reflog.h"
+#include "refs.h"
 #include "ref_helpers.h"
 
 static const char *loose_tag_ref_name = "refs/tags/e90810b";
@@ -38,7 +39,7 @@ void test_refs_rename__loose(void)
 	const char *new_name = "refs/tags/Nemo/knows/refs.kung-fu";
 
 	/* Ensure the ref doesn't exist on the file system */
-	cl_git_pass(git_buf_joinpath(&temp_path, g_repo->path_repository, new_name));
+	cl_git_pass(git_buf_joinpath(&temp_path, git_repository_path(g_repo), new_name));
 	cl_assert(!git_path_exists(temp_path.ptr));
 
 	/* Retrieval of the reference to rename */
@@ -64,7 +65,7 @@ void test_refs_rename__loose(void)
 	cl_assert(reference_is_packed(new_ref) == 0);
 
 	/* ...and the ref can be found in the file system */
-	cl_git_pass(git_buf_joinpath(&temp_path, g_repo->path_repository, new_name));
+	cl_git_pass(git_buf_joinpath(&temp_path, git_repository_path(g_repo), new_name));
 	cl_assert(git_path_exists(temp_path.ptr));
 
 	git_reference_free(new_ref);
@@ -80,7 +81,7 @@ void test_refs_rename__packed(void)
 	const char *brand_new_name = "refs/heads/brand_new_name";
 
 	/* Ensure the ref doesn't exist on the file system */
-	cl_git_pass(git_buf_joinpath(&temp_path, g_repo->path_repository, packed_head_name));
+	cl_git_pass(git_buf_joinpath(&temp_path, git_repository_path(g_repo), packed_head_name));
 	cl_assert(!git_path_exists(temp_path.ptr));
 
 	/* The reference can however be looked-up... */
@@ -106,7 +107,7 @@ void test_refs_rename__packed(void)
 	cl_assert(reference_is_packed(new_ref) == 0);
 
 	/* ...and the ref now happily lives in the file system */
-	cl_git_pass(git_buf_joinpath(&temp_path, g_repo->path_repository, brand_new_name));
+	cl_git_pass(git_buf_joinpath(&temp_path, git_repository_path(g_repo), brand_new_name));
 	cl_assert(git_path_exists(temp_path.ptr));
 
 	git_reference_free(new_ref);
@@ -122,7 +123,7 @@ void test_refs_rename__packed_doesnt_pack_others(void)
 	const char *brand_new_name = "refs/heads/brand_new_name";
 
 	/* Ensure the other reference exists on the file system */
-	cl_git_pass(git_buf_joinpath(&temp_path, g_repo->path_repository, packed_test_head_name));
+	cl_git_pass(git_buf_joinpath(&temp_path, git_repository_path(g_repo), packed_test_head_name));
 	cl_assert(git_path_exists(temp_path.ptr));
 
 	/* Lookup the other reference */

tests-clar/repo/discover.c

 #include "clar_libgit2.h"
 
 #include "odb.h"
+#include "fileops.h"
 #include "repository.h"
 
-
 #define TEMP_REPO_FOLDER "temprepo/"
 #define DISCOVER_FOLDER TEMP_REPO_FOLDER "discover.git"
 

tests-clar/status/ignore.c

@@ -169,7 +169,7 @@ void test_status_ignore__ignore_pattern_ignorecase(void)
 	cl_git_mkfile("empty_standard_repo/A.txt", "Differs in case");
 
 	cl_git_pass(git_repository_index(&index, g_repo));
-	ignore_case = index->ignore_case;
+	ignore_case = (git_index_caps(index) & GIT_INDEXCAP_IGNORE_CASE) != 0;
 	git_index_free(index);
 
 	cl_git_pass(git_status_file(&flags, g_repo, "A.txt"));

tests-clar/status/worktree.c

@@ -105,7 +105,7 @@ void test_status_worktree__swap_subdir_and_file(void)
 	bool ignore_case;
 
 	cl_git_pass(git_repository_index(&index, repo));
-	ignore_case = index->ignore_case;
+	ignore_case = (git_index_caps(index) & GIT_INDEXCAP_IGNORE_CASE) != 0;
 	git_index_free(index);
 
 	/* first alter the contents of the worktree */