Merge pull request #1847 from libgit2/filters-alternative

Alternative proposal for filter API

examples/showindex.c

@@ -12,6 +12,8 @@ int main (int argc, char** argv)
 	char out[41];
 	out[40] = '\0';
 
+	git_threads_init();
+
 	if (argc > 1)
 		dir = argv[1];
 	if (!dir || argc > 2) {
@@ -62,6 +64,8 @@ int main (int argc, char** argv)
 	git_index_free(index);
 	git_repository_free(repo);
 
+	git_threads_shutdown();
+
 	return 0;
 }
 

include/git2.h

@@ -58,4 +58,7 @@
 #include "git2/stash.h"
 #include "git2/pathspec.h"
 
+#include "git2/buffer.h"
+#include "git2/filter.h"
+
 #endif

include/git2/blob.h

@@ -11,6 +11,7 @@
 #include "types.h"
 #include "oid.h"
 #include "object.h"
+#include "buffer.h"
 
 /**
  * @file git2/blob.h
@@ -96,6 +97,37 @@ GIT_EXTERN(const void *) git_blob_rawcontent(const git_blob *blob);
 GIT_EXTERN(git_off_t) git_blob_rawsize(const git_blob *blob);
 
 /**
+ * Get a buffer with the filtered content of a blob.
+ *
+ * This applies filters as if the blob was being checked out to the
+ * working directory under the specified filename.  This may apply
+ * CRLF filtering or other types of changes depending on the file
+ * attributes set for the blob and the content detected in it.
+ *
+ * The output is written into a `git_buf` which the caller must free
+ * when done (via `git_buf_free`).
+ *
+ * If no filters need to be applied, then the `out` buffer will just be
+ * populated with a pointer to the raw content of the blob.  In that case,
+ * be careful to *not* free the blob until done with the buffer.  To keep
+ * the data detached from the blob, call `git_buf_grow` on the buffer
+ * with a `want_size` of 0 and the buffer will be reallocated to be
+ * detached from the blob.
+ *
+ * @param out The git_buf to be filled in
+ * @param blob Pointer to the blob
+ * @param as_path Path used for file attribute lookups, etc.
+ * @param check_for_binary_data Should this test if blob content contains
+ *        NUL bytes / looks like binary data before applying filters?
+ * @return 0 on success or an error code
+ */
+GIT_EXTERN(int) git_blob_filtered_content(
+	git_buf *out,
+	git_blob *blob,
+	const char *as_path,
+	int check_for_binary_data);
+
+/**
  * Read a file from the working folder of a repository
  * and write it to the Object Database as a loose blob
  *

include/git2/buffer.h

+/*
+ * Copyright (C) the libgit2 contributors. All rights reserved.
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#ifndef INCLUDE_git_buf_h__
+#define INCLUDE_git_buf_h__
+
+#include "common.h"
+
+/**
+ * @file git2/buffer.h
+ * @brief Buffer export structure
+ *
+ * @ingroup Git
+ * @{
+ */
+GIT_BEGIN_DECL
+
+/**
+ * A data buffer for exporting data from libgit2
+ *
+ * Sometimes libgit2 wants to return an allocated data buffer to the
+ * caller and have the caller take responsibility for freeing that memory.
+ * This can be awkward if the caller does not have easy access to the same
+ * allocation functions that libgit2 is using.  In those cases, libgit2
+ * will fill in a `git_buf` and the caller can use `git_buf_free()` to
+ * release it when they are done.
+ *
+ * A `git_buf` may also be used for the caller to pass in a reference to
+ * a block of memory they hold.  In this case, libgit2 will not resize or
+ * free the memory, but will read from it as needed.
+ *
+ * A `git_buf` is a public structure with three fields:
+ *
+ * - `ptr` points to the start of the allocated memory.  If it is NULL,
+ *   then the `git_buf` is considered empty and libgit2 will feel free
+ *   to overwrite it with new data.
+ *
+ * - `size` holds the size (in bytes) of the data that is actually used.
+ *
+ * - `asize` holds the known total amount of allocated memory if the `ptr`
+ *    was allocated by libgit2.  It may be larger than `size`.  If `ptr`
+ *    was not allocated by libgit2 and should not be resized and/or freed,
+ *    then `asize` will be set to zero.
+ *
+ * Some APIs may occasionally do something slightly unusual with a buffer,
+ * such as setting `ptr` to a value that was passed in by the user.  In
+ * those cases, the behavior will be clearly documented by the API.
+ */
+typedef struct {
+	char   *ptr;
+	size_t asize, size;
+} git_buf;
+
+/**
+ * Static initializer for git_buf from static buffer
+ */
+#define GIT_BUF_INIT_CONST(STR,LEN) { (char *)(STR), 0, (size_t)(LEN) }
+
+/**
+ * Free the memory referred to by the git_buf.
+ *
+ * Note that this does not free the `git_buf` itself, just the memory
+ * pointed to by `buffer->ptr`.  This will not free the memory if it looks
+ * like it was not allocated internally, but it will clear the buffer back
+ * to the empty state.
+ *
+ * @param buffer The buffer to deallocate
+ */
+GIT_EXTERN(void) git_buf_free(git_buf *buffer);
+
+/**
+ * Resize the buffer allocation to make more space.
+ *
+ * This will attempt to grow the buffer to accomodate the target size.
+ *
+ * If the buffer refers to memory that was not allocated by libgit2 (i.e.
+ * the `asize` field is zero), then `ptr` will be replaced with a newly
+ * allocated block of data.  Be careful so that memory allocated by the
+ * caller is not lost.  As a special variant, if you pass `target_size` as
+ * 0 and the memory is not allocated by libgit2, this will allocate a new
+ * buffer of size `size` and copy the external data into it.
+ *
+ * Currently, this will never shrink a buffer, only expand it.
+ *
+ * If the allocation fails, this will return an error and the buffer will be
+ * marked as invalid for future operations, invaliding the contents.
+ *
+ * @param buffer The buffer to be resized; may or may not be allocated yet
+ * @param target_size The desired available size
+ * @return 0 on success, -1 on allocation failure
+ */
+GIT_EXTERN(int) git_buf_grow(git_buf *buffer, size_t target_size);
+
+/**
+ * Set buffer to a copy of some raw data.
+ *
+ * @param buffer The buffer to set
+ * @param data The data to copy into the buffer
+ * @param datalen The length of the data to copy into the buffer
+ * @return 0 on success, -1 on allocation failure
+ */
+GIT_EXTERN(int) git_buf_set(
+	git_buf *buffer, const void *data, size_t datalen);
+
+GIT_END_DECL
+
+/** @} */
+
+#endif

include/git2/errors.h

@@ -68,6 +68,7 @@ typedef enum {
 	GITERR_FETCHHEAD,
 	GITERR_MERGE,
 	GITERR_SSH,
+	GITERR_FILTER,
 } git_error_t;
 
 /**

include/git2/filter.h

+/*
+ * Copyright (C) the libgit2 contributors. All rights reserved.
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#ifndef INCLUDE_git_filter_h__
+#define INCLUDE_git_filter_h__
+
+#include "common.h"
+#include "types.h"
+#include "oid.h"
+#include "buffer.h"
+
+/**
+ * @file git2/filter.h
+ * @brief Git filter APIs
+ *
+ * @ingroup Git
+ * @{
+ */
+GIT_BEGIN_DECL
+
+/**
+ * Filters are applied in one of two directions: smudging - which is
+ * exporting a file from the Git object database to the working directory,
+ * and cleaning - which is importing a file from the working directory to
+ * the Git object database.  These values control which direction of
+ * change is being applied.
+ */
+typedef enum {
+	GIT_FILTER_TO_WORKTREE = 0,
+	GIT_FILTER_SMUDGE = GIT_FILTER_TO_WORKTREE,
+	GIT_FILTER_TO_ODB = 1,
+	GIT_FILTER_CLEAN = GIT_FILTER_TO_ODB,
+} git_filter_mode_t;
+
+/**
+ * A filter that can transform file data
+ *
+ * This represents a filter that can be used to transform or even replace
+ * file data.  Libgit2 includes one built in filter and it is possible to
+ * write your own (see git2/sys/filter.h for information on that).
+ *
+ * The two builtin filters are:
+ *
+ * * "crlf" which uses the complex rules with the "text", "eol", and
+ *   "crlf" file attributes to decide how to convert between LF and CRLF
+ *   line endings
+ * * "ident" which replaces "$Id$" in a blob with "$Id: <blob OID>$" upon
+ *   checkout and replaced "$Id: <anything>$" with "$Id$" on checkin.
+ */
+typedef struct git_filter git_filter;
+
+/**
+ * List of filters to be applied
+ *
+ * This represents a list of filters to be applied to a file / blob.  You
+ * can build the list with one call, apply it with another, and dispose it
+ * with a third.  In typical usage, there are not many occasions where a
+ * git_filter_list is needed directly since the library will generally
+ * handle conversions for you, but it can be convenient to be able to
+ * build and apply the list sometimes.
+ */
+typedef struct git_filter_list git_filter_list;
+
+/**
+ * Load the filter list for a given path.
+ *
+ * This will return 0 (success) but set the output git_filter_list to NULL
+ * if no filters are requested for the given file.
+ *
+ * @param filters Output newly created git_filter_list (or NULL)
+ * @param repo Repository object that contains `path`
+ * @param blob The blob to which the filter will be applied (if known)
+ * @param path Relative path of the file to be filtered
+ * @param mode Filtering direction (WT->ODB or ODB->WT)
+ * @return 0 on success (which could still return NULL if no filters are
+ *         needed for the requested file), <0 on error
+ */
+GIT_EXTERN(int) git_filter_list_load(
+	git_filter_list **filters,
+	git_repository *repo,
+	git_blob *blob, /* can be NULL */
+	const char *path,
+	git_filter_mode_t mode);
+
+/**
+ * Apply filter list to a data buffer.
+ *
+ * See `git2/buffer.h` for background on `git_buf` objects.
+ *
+ * If the `in` buffer holds data allocated by libgit2 (i.e. `in->asize` is
+ * not zero), then it will be overwritten when applying the filters.  If
+ * not, then it will be left untouched.
+ *
+ * If there are no filters to apply (or `filters` is NULL), then the `out`
+ * buffer will reference the `in` buffer data (with `asize` set to zero)
+ * instead of allocating data.  This keeps allocations to a minimum, but
+ * it means you have to be careful about freeing the `in` data since `out`
+ * may be pointing to it!
+ *
+ * @param out Buffer to store the result of the filtering
+ * @param filters A loaded git_filter_list (or NULL)
+ * @param in Buffer containing the data to filter
+ * @return 0 on success, an error code otherwise
+ */
+GIT_EXTERN(int) git_filter_list_apply_to_data(
+	git_buf *out,
+	git_filter_list *filters,
+	git_buf *in);
+
+/**
+ * Apply filter list to the contents of a file on disk
+ */
+GIT_EXTERN(int) git_filter_list_apply_to_file(
+	git_buf *out,
+	git_filter_list *filters,
+	git_repository *repo,
+	const char *path);
+
+/**
+ * Apply filter list to the contents of a blob
+ */
+GIT_EXTERN(int) git_filter_list_apply_to_blob(
+	git_buf *out,
+	git_filter_list *filters,
+	git_blob *blob);
+
+/**
+ * Free a git_filter_list
+ *
+ * @param filters A git_filter_list created by `git_filter_list_load`
+ */
+GIT_EXTERN(void) git_filter_list_free(git_filter_list *filters);
+
+
+GIT_END_DECL
+
+/** @} */
+
+#endif

src/array.h

@@ -59,7 +59,7 @@ GIT_INLINE(void *) git_array_grow(void *_a, size_t item_size)
 #define git_array_alloc(a) \
 	((a).size >= (a).asize) ? \
 	git_array_grow(&(a), sizeof(*(a).ptr)) : \
-	(a).ptr ? &(a).ptr[(a).size++] : NULL
+	((a).ptr ? &(a).ptr[(a).size++] : NULL)
 
 #define git_array_last(a) ((a).size ? &(a).ptr[(a).size - 1] : NULL)
 

src/attr.c

@@ -26,7 +26,6 @@ git_attr_t git_attr_value(const char *attr)
 	return GIT_ATTR_VALUE_T;
 }
 
-
 static int collect_attr_files(
 	git_repository *repo,
 	uint32_t flags,
@@ -103,8 +102,6 @@ int git_attr_get_many(
 	attr_get_many_info *info = NULL;
 	size_t num_found = 0;
 
-	memset((void *)values, 0, sizeof(const char *) * num_attr);
-
 	if (git_attr_path__init(&path, pathname, git_repository_workdir(repo)) < 0)
 		return -1;
 
@@ -141,6 +138,11 @@ int git_attr_get_many(
 		}
 	}
 
+	for (k = 0; k < num_attr; k++) {
+		if (!info[k].found)
+			values[k] = NULL;
+	}
+
 cleanup:
 	git_vector_free(&files);
 	git_attr_path__free(&path);

src/blob.c

@@ -108,29 +108,21 @@ static int write_file_filtered(
 	git_off_t *size,
 	git_odb *odb,
 	const char *full_path,
-	git_vector *filters)
+	git_filter_list *fl)
 {
 	int error;
-	git_buf source = GIT_BUF_INIT;
-	git_buf dest = GIT_BUF_INIT;
+	git_buf tgt = GIT_BUF_INIT;
 
-	if ((error = git_futils_readbuffer(&source, full_path)) < 0)
-		return error;
-
-	error = git_filters_apply(&dest, &source, filters);
-
-	/* Free the source as soon as possible. This can be big in memory,
-	 * and we don't want to ODB write to choke */
-	git_buf_free(&source);
+	error = git_filter_list_apply_to_file(&tgt, fl, NULL, full_path);
 
 	/* Write the file to disk if it was properly filtered */
 	if (!error) {
-		*size = dest.size;
+		*size = tgt.size;
 
-		error = git_odb_write(oid, odb, dest.ptr, dest.size, GIT_OBJ_BLOB);
+		error = git_odb_write(oid, odb, tgt.ptr, tgt.size, GIT_OBJ_BLOB);
 	}
 
-	git_buf_free(&dest);
+	git_buf_free(&tgt);
 	return error;
 }
 
@@ -198,29 +190,25 @@ int git_blob__create_from_paths(
 	if (S_ISLNK(mode)) {
 		error = write_symlink(oid, odb, content_path, (size_t)size);
 	} else {
-		git_vector write_filters = GIT_VECTOR_INIT;
-		int filter_count = 0;
+		git_filter_list *fl = NULL;
 
-		if (try_load_filters) {
+		if (try_load_filters)
 			/* Load the filters for writing this file to the ODB */
-			filter_count = git_filters_load(
-				&write_filters, repo, hint_path, GIT_FILTER_TO_ODB);
-		}
+			error = git_filter_list_load(
+				&fl, repo, NULL, hint_path, GIT_FILTER_TO_ODB);
 
-		if (filter_count < 0) {
-			/* Negative value means there was a critical error */
-			error = filter_count;
-		} else if (filter_count == 0) {
+		if (error < 0)
+			/* well, that didn't work */;
+		else if (fl == NULL)
 			/* No filters need to be applied to the document: we can stream
 			 * directly from disk */
 			error = write_file_stream(oid, odb, content_path, size);
-		} else {
+		else {
 			/* We need to apply one or more filters */
-			error = write_file_filtered(
-				oid, &size, odb, content_path, &write_filters);
-		}
+			error = write_file_filtered(oid, &size, odb, content_path, fl);
 
-		git_filters_free(&write_filters);
+			git_filter_list_free(fl);
+		}
 
 		/*
 		 * TODO: eventually support streaming filtered files, for files
@@ -333,8 +321,34 @@ int git_blob_is_binary(git_blob *blob)
 
 	assert(blob);
 
-	content.ptr = blob->odb_object->buffer;
-	content.size = min(blob->odb_object->cached.size, 4000);
+	content.ptr   = blob->odb_object->buffer;
+	content.size  = min(blob->odb_object->cached.size, 4000);
+	content.asize = 0;
 
 	return git_buf_text_is_binary(&content);
 }
+
+int git_blob_filtered_content(
+	git_buf *out,
+	git_blob *blob,
+	const char *path,
+	int check_for_binary_data)
+{
+	int error = 0;
+	git_filter_list *fl = NULL;
+
+	assert(blob && path && out);
+
+	if (check_for_binary_data && git_blob_is_binary(blob))
+		return 0;
+
+	if (!(error = git_filter_list_load(
+			&fl, git_blob_owner(blob), blob, path, GIT_FILTER_TO_WORKTREE))) {
+
+		error = git_filter_list_apply_to_blob(out, fl, blob);
+
+		git_filter_list_free(fl);
+	}
+
+	return error;
+}

src/buf_text.c

@@ -70,10 +70,10 @@ int git_buf_text_crlf_to_lf(git_buf *tgt, const git_buf *src)
 	assert(tgt != src);
 
 	if (!next)
-		return GIT_ENOTFOUND;
+		return git_buf_set(tgt, src->ptr, src->size);
 
 	/* reduce reallocs while in the loop */
-	if (git_buf_grow(tgt, src->size) < 0)
+	if (git_buf_grow(tgt, src->size + 1) < 0)
 		return -1;
 	out = tgt->ptr;
 	tgt->size = 0;
@@ -81,20 +81,25 @@ int git_buf_text_crlf_to_lf(git_buf *tgt, const git_buf *src)
 	/* Find the next \r and copy whole chunk up to there to tgt */
 	for (; next; scan = next + 1, next = memchr(scan, '\r', scan_end - scan)) {
 		if (next > scan) {
-			size_t copylen = next - scan;
+			size_t copylen = (size_t)(next - scan);
 			memcpy(out, scan, copylen);
 			out += copylen;
 		}
 
 		/* Do not drop \r unless it is followed by \n */
-		if (next[1] != '\n')
+		if (next + 1 == scan_end || next[1] != '\n')
 			*out++ = '\r';
 	}
 
 	/* Copy remaining input into dest */
-	memcpy(out, scan, scan_end - scan + 1); /* +1 for NUL byte */
-	out += (scan_end - scan);
-	tgt->size = out - tgt->ptr;
+	if (scan < scan_end) {
+		size_t remaining = (size_t)(scan_end - scan);
+		memcpy(out, scan, remaining);
+		out += remaining;
+	}
+
+	tgt->size = (size_t)(out - tgt->ptr);
+	tgt->ptr[tgt->size] = '\0';
 
 	return 0;
 }
@@ -109,7 +114,7 @@ int git_buf_text_lf_to_crlf(git_buf *tgt, const git_buf *src)
 	assert(tgt != src);
 
 	if (!next)
-		return GIT_ENOTFOUND;
+		return git_buf_set(tgt, src->ptr, src->size);
 
 	/* attempt to reduce reallocs while in the loop */
 	if (git_buf_grow(tgt, src->size + (src->size >> 4) + 1) < 0)

src/buf_text.h

@@ -56,16 +56,16 @@ GIT_INLINE(int) git_buf_text_puts_escape_regex(git_buf *buf, const char *string)
 extern void git_buf_text_unescape(git_buf *buf);
 
 /**
- * Replace all \r\n with \n (or do nothing if no \r\n are found)
+ * Replace all \r\n with \n. Does not modify \r without trailing \n.
  *
- * @return 0 on success, GIT_ENOTFOUND if no \r\n, -1 on memory error
+ * @return 0 on success, -1 on memory error
  */
 extern int git_buf_text_crlf_to_lf(git_buf *tgt, const git_buf *src);
 
 /**
- * Replace all \n with \r\n (or do nothing if no \n are found)
+ * Replace all \n with \r\n. Does not modify existing \r\n.
  *
- * @return 0 on success, GIT_ENOTFOUND if no \n, -1 on memory error
+ * @return 0 on success, -1 on memory error
  */
 extern int git_buf_text_lf_to_crlf(git_buf *tgt, const git_buf *src);
 

src/buffer.c

@@ -6,6 +6,7 @@
  */
 #include "buffer.h"
 #include "posix.h"
+#include "git2/buffer.h"
 #include <stdarg.h>
 #include <ctype.h>
 
@@ -31,7 +32,8 @@ void git_buf_init(git_buf *buf, size_t initial_size)
 		git_buf_grow(buf, initial_size);
 }
 
-int git_buf_try_grow(git_buf *buf, size_t target_size, bool mark_oom)
+int git_buf_try_grow(
+	git_buf *buf, size_t target_size, bool mark_oom, bool preserve_external)
 {
 	char *new_ptr;
 	size_t new_size;
@@ -39,6 +41,9 @@ int git_buf_try_grow(git_buf *buf, size_t target_size, bool mark_oom)
 	if (buf->ptr == git_buf__oom)
 		return -1;
 
+	if (!target_size)
+		target_size = buf->size;
+
 	if (target_size <= buf->asize)
 		return 0;
 
@@ -66,6 +71,9 @@ int git_buf_try_grow(git_buf *buf, size_t target_size, bool mark_oom)
 		return -1;
 	}
 
+	if (preserve_external && !buf->asize && buf->ptr != NULL && buf->size > 0)
+		memcpy(new_ptr, buf->ptr, min(buf->size, new_size));
+
 	buf->asize = new_size;
 	buf->ptr   = new_ptr;
 
@@ -77,11 +85,16 @@ int git_buf_try_grow(git_buf *buf, size_t target_size, bool mark_oom)
 	return 0;
 }
 
+int git_buf_grow(git_buf *buffer, size_t target_size)
+{
+	return git_buf_try_grow(buffer, target_size, true, true);
+}
+
 void git_buf_free(git_buf *buf)
 {
 	if (!buf) return;
 
-	if (buf->ptr != git_buf__initbuf && buf->ptr != git_buf__oom)
+	if (buf->asize > 0 && buf->ptr != NULL && buf->ptr != git_buf__oom)
 		git__free(buf->ptr);
 
 	git_buf_init(buf, 0);
@@ -90,11 +103,15 @@ void git_buf_free(git_buf *buf)
 void git_buf_clear(git_buf *buf)
 {
 	buf->size = 0;
+
+	if (!buf->ptr)
+		buf->ptr = git_buf__initbuf;
+
 	if (buf->asize > 0)
 		buf->ptr[0] = '\0';
 }
 
-int git_buf_set(git_buf *buf, const char *data, size_t len)
+int git_buf_set(git_buf *buf, const void *data, size_t len)
 {
 	if (len == 0 || data == NULL) {
 		git_buf_clear(buf);

src/buffer.h

@@ -9,18 +9,26 @@
 
 #include "common.h"
 #include "git2/strarray.h"
+#include "git2/buffer.h"
 #include <stdarg.h>
 
-typedef struct {
-	char *ptr;
-	size_t asize, size;
-} git_buf;
+/* typedef struct {
+ *  	char   *ptr;
+ *  	size_t asize, size;
+ * } git_buf;
+ */
 
 extern char git_buf__initbuf[];
 extern char git_buf__oom[];
 
+/* Use to initialize buffer structure when git_buf is on stack */
 #define GIT_BUF_INIT { git_buf__initbuf, 0, 0 }
 
+GIT_INLINE(bool) git_buf_is_allocated(const git_buf *buf)
+{
+	return (buf->ptr != NULL && buf->asize > 0);
+}
+
 /**
  * Initialize a git_buf structure.
  *
@@ -32,27 +40,16 @@ extern void git_buf_init(git_buf *buf, size_t initial_size);
 /**
  * Attempt to grow the buffer to hold at least `target_size` bytes.
  *
- * If the allocation fails, this will return an error.  If mark_oom is true,
+ * If the allocation fails, this will return an error.  If `mark_oom` is true,
  * this will mark the buffer as invalid for future operations; if false,
  * existing buffer content will be preserved, but calling code must handle
- * that buffer was not expanded.
+ * that buffer was not expanded.  If `preserve_external` is true, then any
+ * existing data pointed to be `ptr` even if `asize` is zero will be copied
+ * into the newly allocated buffer.
  */
-extern int git_buf_try_grow(git_buf *buf, size_t target_size, bool mark_oom);
-
-/**
- * Grow the buffer to hold at least `target_size` bytes.
- *
- * If the allocation fails, this will return an error and the buffer will be
- * marked as invalid for future operations, invaliding contents.
- *
- * @return 0 on success or -1 on failure
- */
-GIT_INLINE(int) git_buf_grow(git_buf *buf, size_t target_size)
-{
-	return git_buf_try_grow(buf, target_size, true);
-}
+extern int git_buf_try_grow(
+	git_buf *buf, size_t target_size, bool mark_oom, bool preserve_external);
 
-extern void git_buf_free(git_buf *buf);
 extern void git_buf_swap(git_buf *buf_a, git_buf *buf_b);
 extern char *git_buf_detach(git_buf *buf);
 extern void git_buf_attach(git_buf *buf, char *ptr, size_t asize);
@@ -81,7 +78,6 @@ GIT_INLINE(bool) git_buf_oom(const git_buf *buf)
  * return code of these functions and call them in a series then just call
  * git_buf_oom at the end.
  */
-int git_buf_set(git_buf *buf, const char *data, size_t len);
 int git_buf_sets(git_buf *buf, const char *string);
 int git_buf_putc(git_buf *buf, char c);
 int git_buf_put(git_buf *buf, const char *data, size_t len);

src/checkout.c

@@ -678,7 +678,7 @@ fail:
 
 static int buffer_to_file(
 	struct stat *st,
-	git_buf *buffer,
+	git_buf *buf,
 	const char *path,
 	mode_t dir_mode,
 	int file_open_flags,
@@ -690,7 +690,7 @@ static int buffer_to_file(
 		return error;
 
 	if ((error = git_futils_writebuffer(
-			buffer, path, file_open_flags, file_mode)) < 0)
+			buf, path, file_open_flags, file_mode)) < 0)
 		return error;
 
 	if (st != NULL && (error = p_stat(path, st)) < 0)
@@ -710,57 +710,28 @@ static int blob_content_to_file(
 	mode_t entry_filemode,
 	git_checkout_opts *opts)
 {
-	int error = -1, nb_filters = 0;
-	mode_t file_mode = opts->file_mode;
-	bool dont_free_filtered;
-	git_buf unfiltered = GIT_BUF_INIT, filtered = GIT_BUF_INIT;
-	git_vector filters = GIT_VECTOR_INIT;
-
-	/* Create a fake git_buf from the blob raw data... */
-	filtered.ptr  = (void *)git_blob_rawcontent(blob);
-	filtered.size = (size_t)git_blob_rawsize(blob);
-	/* ... and make sure it doesn't get unexpectedly freed */
-	dont_free_filtered = true;
-
-	if (!opts->disable_filters &&
-		!git_buf_text_is_binary(&filtered) &&
-		(nb_filters = git_filters_load(
-			&filters,
-			git_object_owner((git_object *)blob),
-			path,
-			GIT_FILTER_TO_WORKTREE)) > 0)
-	{
-		/* reset 'filtered' so it can be a filter target */
-		git_buf_init(&filtered, 0);
-		dont_free_filtered = false;
-	}
-
-	if (nb_filters < 0)
-		return nb_filters;
+	int error = 0;
+	mode_t file_mode = opts->file_mode ? opts->file_mode : entry_filemode;
+	git_buf out = GIT_BUF_INIT;
+	git_filter_list *fl = NULL;
 
-	if (nb_filters > 0)	 {
-		if ((error = git_blob__getbuf(&unfiltered, blob)) < 0)
-			goto cleanup;
+	if (!opts->disable_filters)
+		error = git_filter_list_load(
+			&fl, git_blob_owner(blob), blob, path, GIT_FILTER_TO_WORKTREE);
 
-		if ((error = git_filters_apply(&filtered, &unfiltered, &filters)) < 0)
-			goto cleanup;
-	}
+	if (!error)
+		error = git_filter_list_apply_to_blob(&out, fl, blob);
 
-	/* Allow overriding of file mode */
-	if (!file_mode)
-		file_mode = entry_filemode;
+	git_filter_list_free(fl);
 
-	error = buffer_to_file(
-		st, &filtered, path, opts->dir_mode, opts->file_open_flags, file_mode);
+	if (!error) {
+		error = buffer_to_file(
+			st, &out, path, opts->dir_mode, opts->file_open_flags, file_mode);
 
-	if (!error)
 		st->st_mode = entry_filemode;
 
-cleanup:
-	git_filters_free(&filters);
-	git_buf_free(&unfiltered);
-	if (!dont_free_filtered)
-		git_buf_free(&filtered);
+		git_buf_free(&out);
+	}
 
 	return error;
 }

src/config_file.c

@@ -293,6 +293,8 @@ static int config_iterator_new(
 	diskfile_backend *b = (diskfile_backend *)backend;
 	git_config_file_iter *it = git__calloc(1, sizeof(git_config_file_iter));
 
+	GIT_UNUSED(b);
+
 	GITERR_CHECK_ALLOC(it);
 
 	it->parent.backend = backend;

src/crlf.c

@@ -8,6 +8,7 @@
 #include "git2/attr.h"
 #include "git2/blob.h"
 #include "git2/index.h"
+#include "git2/sys/filter.h"
 
 #include "common.h"
 #include "fileops.h"
@@ -19,13 +20,11 @@
 struct crlf_attrs {
 	int crlf_action;
 	int eol;
+	int auto_crlf;
 };
 
 struct crlf_filter {
 	git_filter f;
-	struct crlf_attrs attrs;
-	git_repository *repo;
-	char path[GIT_FLEX_ARRAY];
 };
 
 static int check_crlf(const char *value)
@@ -76,41 +75,10 @@ static int crlf_input_action(struct crlf_attrs *ca)
 	return ca->crlf_action;
 }
 
-static int crlf_load_attributes(struct crlf_attrs *ca, git_repository *repo, const char *path)
+static int has_cr_in_index(const git_filter_source *src)
 {
-#define NUM_CONV_ATTRS 3
-
-	static const char *attr_names[NUM_CONV_ATTRS] = {
-		"crlf", "eol", "text",
-	};
-
-	const char *attr_vals[NUM_CONV_ATTRS];
-	int error;
-
-	error = git_attr_get_many(attr_vals,
-		repo, 0, path, NUM_CONV_ATTRS, attr_names);
-
-	if (error == GIT_ENOTFOUND) {
-		ca->crlf_action = GIT_CRLF_GUESS;
-		ca->eol = GIT_EOL_UNSET;
-		return 0;
-	}
-
-	if (error == 0) {
-		ca->crlf_action = check_crlf(attr_vals[2]); /* text */
-		if (ca->crlf_action == GIT_CRLF_GUESS)
-			ca->crlf_action = check_crlf(attr_vals[0]); /* clrf */
-
-		ca->eol = check_eol(attr_vals[1]); /* eol */
-		return 0;
-	}
-
-	return -1;
-}
-
-static int has_cr_in_index(git_filter *self)
-{
-	struct crlf_filter *filter = (struct crlf_filter *)self;
+	git_repository *repo = git_filter_source_repo(src);
+	const char *path = git_filter_source_path(src);
 	git_index *index;
 	const git_index_entry *entry;
 	git_blob *blob;
@@ -118,19 +86,22 @@ static int has_cr_in_index(git_filter *self)
 	git_off_t blobsize;
 	bool found_cr;
 
-	if (git_repository_index__weakptr(&index, filter->repo) < 0) {
+	if (!path)
+		return false;
+
+	if (git_repository_index__weakptr(&index, repo) < 0) {
 		giterr_clear();
 		return false;
 	}
 
-	if (!(entry = git_index_get_bypath(index, filter->path, 0)) &&
-		!(entry = git_index_get_bypath(index, filter->path, 1)))
+	if (!(entry = git_index_get_bypath(index, path, 0)) &&
+		!(entry = git_index_get_bypath(index, path, 1)))
 		return false;
 
 	if (!S_ISREG(entry->mode)) /* don't crlf filter non-blobs */
 		return true;
 
-	if (git_blob_lookup(&blob, filter->repo, &entry->oid) < 0)
+	if (git_blob_lookup(&blob, repo, &entry->oid) < 0)
 		return false;
 
 	blobcontent = git_blob_rawcontent(blob);
@@ -147,26 +118,23 @@ static int has_cr_in_index(git_filter *self)
 }
 
 static int crlf_apply_to_odb(
-	git_filter *self, git_buf *dest, const git_buf *source)
+	struct crlf_attrs *ca,
+	git_buf *to,
+	const git_buf *from,
+	const git_filter_source *src)
 {
-	struct crlf_filter *filter = (struct crlf_filter *)self;
-
-	assert(self && dest && source);
-
 	/* Empty file? Nothing to do */
-	if (git_buf_len(source) == 0)
+	if (!git_buf_len(from))
 		return 0;
 
 	/* Heuristics to see if we can skip the conversion.
 	 * Straight from Core Git.
 	 */
-	if (filter->attrs.crlf_action == GIT_CRLF_AUTO ||
-		filter->attrs.crlf_action == GIT_CRLF_GUESS) {
-
+	if (ca->crlf_action == GIT_CRLF_AUTO || ca->crlf_action == GIT_CRLF_GUESS) {
 		git_buf_text_stats stats;
 
 		/* Check heuristics for binary vs text... */
-		if (git_buf_text_gather_stats(&stats, source, false))
+		if (git_buf_text_gather_stats(&stats, from, false))
 			return -1;
 
 		/*
@@ -175,28 +143,28 @@ static int crlf_apply_to_odb(
 		 * stuff?
 		 */
 		if (stats.cr != stats.crlf)
-			return -1;
+			return GIT_PASSTHROUGH;
 
-		if (filter->attrs.crlf_action == GIT_CRLF_GUESS) {
+		if (ca->crlf_action == GIT_CRLF_GUESS) {
 			/*
 			 * If the file in the index has any CR in it, do not convert.
 			 * This is the new safer autocrlf handling.
 			 */
-			if (has_cr_in_index(self))
-				return -1;
+			if (has_cr_in_index(src))
+				return GIT_PASSTHROUGH;
 		}
 
 		if (!stats.cr)
-			return -1;
+			return GIT_PASSTHROUGH;
 	}
 
 	/* Actually drop the carriage returns */
-	return git_buf_text_crlf_to_lf(dest, source);
+	return git_buf_text_crlf_to_lf(to, from);
 }
 
-static const char *line_ending(struct crlf_filter *filter)
+static const char *line_ending(struct crlf_attrs *ca)
 {
-	switch (filter->attrs.crlf_action) {
+	switch (ca->crlf_action) {
 	case GIT_CRLF_BINARY:
 	case GIT_CRLF_INPUT:
 		return "\n";
@@ -213,11 +181,9 @@ static const char *line_ending(struct crlf_filter *filter)
 		goto line_ending_error;
 	}
 
-	switch (filter->attrs.eol) {
+	switch (ca->eol) {
 	case GIT_EOL_UNSET:
-		return GIT_EOL_NATIVE == GIT_EOL_CRLF
-			? "\r\n"
-			: "\n";
+		return GIT_EOL_NATIVE == GIT_EOL_CRLF ? "\r\n" : "\n";
 
 	case GIT_EOL_CRLF:
 		return "\r\n";
@@ -235,41 +201,64 @@ line_ending_error:
 }
 
 static int crlf_apply_to_workdir(
-	git_filter *self, git_buf *dest, const git_buf *source)
+	struct crlf_attrs *ca, git_buf *to, const git_buf *from)
 {
-	struct crlf_filter *filter = (struct crlf_filter *)self;
 	const char *workdir_ending = NULL;
 
-	assert(self && dest && source);
-
 	/* Empty file? Nothing to do. */
-	if (git_buf_len(source) == 0)
-		return -1;
+	if (git_buf_len(from) == 0)
+		return 0;
+
+	/* Don't filter binary files */
+	if (git_buf_text_is_binary(from))
+		return GIT_PASSTHROUGH;
 
 	/* Determine proper line ending */
-	workdir_ending = line_ending(filter);
+	workdir_ending = line_ending(ca);
 	if (!workdir_ending)
 		return -1;
-	if (!strcmp("\n", workdir_ending)) /* do nothing for \n ending */
-		return -1;
 
-	/* for now, only lf->crlf conversion is supported here */
-	assert(!strcmp("\r\n", workdir_ending));
-	return git_buf_text_lf_to_crlf(dest, source);
+	if (!strcmp("\n", workdir_ending)) {
+		if (ca->crlf_action == GIT_CRLF_GUESS && ca->auto_crlf)
+			return GIT_PASSTHROUGH;
+
+		if (git_buf_find(from, '\r') < 0)
+			return GIT_PASSTHROUGH;
+
+		if (git_buf_text_crlf_to_lf(to, from) < 0)
+			return -1;
+	} else {
+		/* only other supported option is lf->crlf conversion */
+		assert(!strcmp("\r\n", workdir_ending));
+
+		if (git_buf_text_lf_to_crlf(to, from) < 0)
+			return -1;
+	}
+
+	return 0;
 }
 
-static int find_and_add_filter(
-	git_vector *filters, git_repository *repo, const char *path,
-	int (*apply)(struct git_filter *self, git_buf *dest, const git_buf *source))
+static int crlf_check(
+	git_filter        *self,
+	void              **payload, /* points to NULL ptr on entry, may be set */
+	const git_filter_source *src,
+	const char **attr_values)
 {
-	struct crlf_attrs ca;
-	struct crlf_filter *filter;
-	size_t pathlen;
 	int error;
+	struct crlf_attrs ca;
 
-	/* Load gitattributes for the path */
-	if ((error = crlf_load_attributes(&ca, repo, path)) < 0)
-		return error;
+	GIT_UNUSED(self);
+
+	if (!attr_values) {
+		ca.crlf_action = GIT_CRLF_GUESS;
+		ca.eol = GIT_EOL_UNSET;
+	} else {
+		ca.crlf_action = check_crlf(attr_values[2]); /* text */
+		if (ca.crlf_action == GIT_CRLF_GUESS)
+			ca.crlf_action = check_crlf(attr_values[0]); /* clrf */
+		ca.eol = check_eol(attr_values[1]); /* eol */
+	}
+	ca.auto_crlf = GIT_AUTO_CRLF_DEFAULT;
 
 	/*
 	 * Use the core Git logic to see if we should perform CRLF for this file
@@ -278,41 +267,64 @@ static int find_and_add_filter(
 	ca.crlf_action = crlf_input_action(&ca);
 
 	if (ca.crlf_action == GIT_CRLF_BINARY)
-		return 0;
+		return GIT_PASSTHROUGH;
 
 	if (ca.crlf_action == GIT_CRLF_GUESS) {
-		int auto_crlf;
-
-		if ((error = git_repository__cvar(&auto_crlf, repo, GIT_CVAR_AUTO_CRLF)) < 0)
+		error = git_repository__cvar(
+			&ca.auto_crlf, git_filter_source_repo(src), GIT_CVAR_AUTO_CRLF);
+		if (error < 0)
 			return error;
 
-		if (auto_crlf == GIT_AUTO_CRLF_FALSE)
-			return 0;
+		if (ca.auto_crlf == GIT_AUTO_CRLF_FALSE)
+			return GIT_PASSTHROUGH;
 	}
 
-	/* If we're good, we create a new filter object and push it
-	 * into the filters array */
-	pathlen = strlen(path);
-	filter = git__malloc(sizeof(struct crlf_filter) + pathlen + 1);
-	GITERR_CHECK_ALLOC(filter);
+	*payload = git__malloc(sizeof(ca));
+	GITERR_CHECK_ALLOC(*payload);
+	memcpy(*payload, &ca, sizeof(ca));
 
-	filter->f.apply = apply;
-	filter->f.do_free = NULL;
-	memcpy(&filter->attrs, &ca, sizeof(struct crlf_attrs));
-	filter->repo = repo;
-	memcpy(filter->path, path, pathlen + 1);
+	return 0;
+}
 
-	return git_vector_insert(filters, filter);
+static int crlf_apply(
+	git_filter    *self,
+	void         **payload, /* may be read and/or set */
+	git_buf       *to,
+	const git_buf *from,
+	const git_filter_source *src)
+{
+	/* initialize payload in case `check` was bypassed */
+	if (!*payload) {
+		int error = crlf_check(self, payload, src, NULL);
+		if (error < 0 && error != GIT_PASSTHROUGH)
+			return error;
+	}
+
+	if (git_filter_source_mode(src) == GIT_FILTER_SMUDGE)
+		return crlf_apply_to_workdir(*payload, to, from);
+	else
+		return crlf_apply_to_odb(*payload, to, from, src);
 }
 
-int git_filter_add__crlf_to_odb(
-	git_vector *filters, git_repository *repo, const char *path)
+static void crlf_cleanup(
+	git_filter *self,
+	void       *payload)
 {
-	return find_and_add_filter(filters, repo, path, &crlf_apply_to_odb);
+	GIT_UNUSED(self);
+	git__free(payload);
 }
 
-int git_filter_add__crlf_to_workdir(
-	git_vector *filters, git_repository *repo, const char *path)
+git_filter *git_crlf_filter_new(void)
 {
-	return find_and_add_filter(filters, repo, path, &crlf_apply_to_workdir);
+	struct crlf_filter *f = git__calloc(1, sizeof(struct crlf_filter));
+
+	f->f.version = GIT_FILTER_VERSION;
+	f->f.attributes = "crlf eol text";
+	f->f.initialize = NULL;
+	f->f.shutdown = git_filter_free;
+	f->f.check    = crlf_check;
+	f->f.apply    = crlf_apply;
+	f->f.cleanup  = crlf_cleanup;
+
+	return (git_filter *)f;
 }

src/diff.c

@@ -568,21 +568,21 @@ int git_diff__oid_for_file(
 		giterr_set(GITERR_OS, "File size overflow (for 32-bits) on '%s'", path);
 		result = -1;
 	} else {
-		git_vector filters = GIT_VECTOR_INIT;
+		git_filter_list *fl = NULL;
 
-		result = git_filters_load(&filters, repo, path, GIT_FILTER_TO_ODB);
-		if (result >= 0) {
+		result = git_filter_list_load(&fl, repo, NULL, path, GIT_FILTER_TO_ODB);
+		if (!result) {
 			int fd = git_futils_open_ro(full_path.ptr);
 			if (fd < 0)
 				result = fd;
 			else {
 				result = git_odb__hashfd_filtered(
-					oid, fd, (size_t)size, GIT_OBJ_BLOB, &filters);
+					oid, fd, (size_t)size, GIT_OBJ_BLOB, fl);
 				p_close(fd);
 			}
-		}
 
-		git_filters_free(&filters);
+			git_filter_list_free(fl);
+		}
 	}
 
 cleanup:

src/diff_file.c

@@ -296,9 +296,9 @@ static int diff_file_content_load_workdir_file(
 	git_diff_file_content *fc, git_buf *path)
 {
 	int error = 0;
-	git_vector filters = GIT_VECTOR_INIT;
-	git_buf raw = GIT_BUF_INIT, filtered = GIT_BUF_INIT;
+	git_filter_list *fl = NULL;
 	git_file fd = git_futils_open_ro(git_buf_cstr(path));
+	git_buf raw = GIT_BUF_INIT;
 
 	if (fd < 0)
 		return fd;
@@ -310,41 +310,38 @@ static int diff_file_content_load_workdir_file(
 	if (diff_file_content_binary_by_size(fc))
 		goto cleanup;
 
-	if ((error = git_filters_load(
-			&filters, fc->repo, fc->file->path, GIT_FILTER_TO_ODB)) < 0)
+	if ((error = git_filter_list_load(
+			&fl, fc->repo, NULL, fc->file->path, GIT_FILTER_TO_ODB)) < 0)
 		goto cleanup;
-	/* error >= is a filter count */
 
-	if (error == 0) {
+	/* if there are no filters, try to mmap the file */
+	if (fl == NULL) {
 		if (!(error = git_futils_mmap_ro(
-				&fc->map, fd, 0, (size_t)fc->file->size)))
+				&fc->map, fd, 0, (size_t)fc->file->size))) {
 			fc->flags |= GIT_DIFF_FLAG__UNMAP_DATA;
-		else /* fall through to try readbuffer below */
-			giterr_clear();
+			goto cleanup;
+		}
+
+		/* if mmap failed, fall through to try readbuffer below */
+		giterr_clear();
 	}
 
-	if (error != 0) {
-		error = git_futils_readbuffer_fd(&raw, fd, (size_t)fc->file->size);
-		if (error < 0)
-			goto cleanup;
+	if (!(error = git_futils_readbuffer_fd(&raw, fd, (size_t)fc->file->size))) {
+		git_buf out = GIT_BUF_INIT;
 
-		if (!filters.length)
-			git_buf_swap(&filtered, &raw);
-		else
-			error = git_filters_apply(&filtered, &raw, &filters);
+		error = git_filter_list_apply_to_data(&out, fl, &raw);
+
+		git_buf_free(&raw);
 
 		if (!error) {
-			fc->map.len  = git_buf_len(&filtered);
-			fc->map.data = git_buf_detach(&filtered);
+			fc->map.len  = out.size;
+			fc->map.data = out.ptr;
 			fc->flags |= GIT_DIFF_FLAG__FREE_DATA;
 		}
-
-		git_buf_free(&raw);
-		git_buf_free(&filtered);
 	}
 
 cleanup:
-	git_filters_free(&filters);
+	git_filter_list_free(fl);
 	p_close(fd);
 
 	return error;

src/fileops.c

@@ -6,6 +6,7 @@
  */
 #include "common.h"
 #include "fileops.h"
+#include "global.h"
 #include <ctype.h>
 #if GIT_WIN32
 #include "win32/findfile.h"
@@ -55,18 +56,8 @@ int git_futils_creat_withpath(const char *path, const mode_t dirmode, const mode
 
 int git_futils_creat_locked(const char *path, const mode_t mode)
 {
-	int fd;
-
-#ifdef GIT_WIN32
-	git_win32_path buf;
-
-	git_win32_path_from_c(buf, path);
-	fd = _wopen(buf, O_WRONLY | O_CREAT | O_TRUNC |
-		O_EXCL | O_BINARY | O_CLOEXEC, mode);
-#else
-	fd = open(path, O_WRONLY | O_CREAT | O_TRUNC |
+	int fd = p_open(path, O_WRONLY | O_CREAT | O_TRUNC |
 		O_EXCL | O_BINARY | O_CLOEXEC, mode);
-#endif
 
 	if (fd < 0) {
 		giterr_set(GITERR_OS, "Failed to create locked file '%s'", path);
@@ -635,6 +626,13 @@ static git_futils_dirs_guess_cb git_futils__dir_guess[GIT_FUTILS_DIR__MAX] = {
 	git_futils_guess_xdg_dirs,
 };
 
+static void git_futils_dirs_global_shutdown(void)
+{
+	int i;
+	for (i = 0; i < GIT_FUTILS_DIR__MAX; ++i)
+		git_buf_free(&git_futils__dirs[i]);
+}
+
 int git_futils_dirs_global_init(void)
 {
 	git_futils_dir_t i;
@@ -644,6 +642,8 @@ int git_futils_dirs_global_init(void)
 	for (i = 0; !error && i < GIT_FUTILS_DIR__MAX; i++)
 		error = git_futils_dirs_get(&path, i);
 
+	git__on_shutdown(git_futils_dirs_global_shutdown);
+
 	return error;
 }
 
@@ -726,13 +726,6 @@ int git_futils_dirs_set(git_futils_dir_t which, const char *search_path)
 	return git_buf_oom(&git_futils__dirs[which]) ? -1 : 0;
 }
 
-void git_futils_dirs_free(void)
-{
-	int i;
-	for (i = 0; i < GIT_FUTILS_DIR__MAX; ++i)
-		git_buf_free(&git_futils__dirs[i]);
-}
-
 static int git_futils_find_in_dirlist(
 	git_buf *path, const char *name, git_futils_dir_t which, const char *label)
 {

src/fileops.h

@@ -354,11 +354,6 @@ extern int git_futils_dirs_get_str(
 extern int git_futils_dirs_set(git_futils_dir_t which, const char *paths);
 
 /**
- * Release / reset all search paths
- */
-extern void git_futils_dirs_free(void);
-
-/**
  * Create a "fake" symlink (text file containing the target path).
  *
  * @param new symlink file to be created

src/filter.h

@@ -8,19 +8,7 @@
 #define INCLUDE_filter_h__
 
 #include "common.h"
-#include "buffer.h"
-#include "git2/odb.h"
-#include "git2/repository.h"
-
-typedef struct git_filter {
-	int (*apply)(struct git_filter *self, git_buf *dest, const git_buf *source);
-	void (*do_free)(struct git_filter *self);
-} git_filter;
-
-typedef enum {
-	GIT_FILTER_TO_WORKTREE,
-	GIT_FILTER_TO_ODB
-} git_filter_mode;
+#include "git2/filter.h"
 
 typedef enum {
 	GIT_CRLF_GUESS = -1,
@@ -31,64 +19,13 @@ typedef enum {
 	GIT_CRLF_AUTO,
 } git_crlf_t;
 
-/*
- * FILTER API
- */
-
-/*
- * For any given path in the working directory, fill the `filters`
- * array with the relevant filters that need to be applied.
- *
- * Mode is either `GIT_FILTER_TO_WORKTREE` if you need to load the
- * filters that will be used when checking out a file to the working
- * directory, or `GIT_FILTER_TO_ODB` for the filters used when writing
- * a file to the ODB.
- *
- * @param filters Vector where to store all the loaded filters
- * @param repo Repository object that contains `path`
- * @param path Relative path of the file to be filtered
- * @param mode Filtering direction (WT->ODB or ODB->WT)
- * @return the number of filters loaded for the file (0 if the file
- *	doesn't need filtering), or a negative error code
- */
-extern int git_filters_load(git_vector *filters, git_repository *repo, const char *path, int mode);
-
-/*
- * Apply one or more filters to a file.
- *
- * The file must have been loaded as a `git_buf` object. Both the `source`
- * and `dest` buffers are owned by the caller and must be freed once
- * they are no longer needed.
- *
- * NOTE: Because of the double-buffering schema, the `source` buffer that contains
- * the original file may be tampered once the filtering is complete. Regardless, 
- * the `dest` buffer will always contain the final result of the filtering
- *
- * @param dest Buffer to store the result of the filtering
- * @param source Buffer containing the document to filter
- * @param filters A non-empty vector of filters as supplied by `git_filters_load`
- * @return 0 on success, an error code otherwise
- */
-extern int git_filters_apply(git_buf *dest, git_buf *source, git_vector *filters);
-
-/*
- * Free the `filters` array generated by `git_filters_load`.
- *
- * Note that this frees both the array and its contents. The array will
- * be clean/reusable after this call.
- *
- * @param filters A filters array as supplied by `git_filters_load`
- */
-extern void git_filters_free(git_vector *filters);
+extern void git_filter_free(git_filter *filter);
 
 /*
  * Available filters
  */
 
-/* Strip CRLF, from Worktree to ODB */
-extern int git_filter_add__crlf_to_odb(git_vector *filters, git_repository *repo, const char *path);
-
-/* Add CRLF, from ODB to worktree */
-extern int git_filter_add__crlf_to_workdir(git_vector *filters, git_repository *repo, const char *path);
+extern git_filter *git_crlf_filter_new(void);
+extern git_filter *git_ident_filter_new(void);
 
 #endif

src/global.c

@@ -14,6 +14,28 @@
 
 git_mutex git__mwindow_mutex;
 
+#define MAX_SHUTDOWN_CB 8
+
+git_global_shutdown_fn git__shutdown_callbacks[MAX_SHUTDOWN_CB];
+git_atomic git__n_shutdown_callbacks;
+
+void git__on_shutdown(git_global_shutdown_fn callback)
+{
+	int count = git_atomic_inc(&git__n_shutdown_callbacks);
+	assert(count <= MAX_SHUTDOWN_CB);
+	git__shutdown_callbacks[count - 1] = callback;
+}
+
+static void git__shutdown(void)
+{
+	int pos;
+
+	while ((pos = git_atomic_dec(&git__n_shutdown_callbacks)) >= 0) {
+		if (git__shutdown_callbacks[pos])
+			git__shutdown_callbacks[pos]();
+	}
+}
+
 /**
  * Handle the global state with TLS
  *
@@ -79,9 +101,7 @@ int git_threads_init(void)
 void git_threads_shutdown(void)
 {
 	/* Shut down any subsystems that have global state */
-	win32_pthread_shutdown();
-	git_futils_dirs_free();
-	git_hash_global_shutdown();
+	git__shutdown();
 
 	TlsFree(_tls_index);
 	_tls_init = 0;
@@ -140,6 +160,9 @@ int git_threads_init(void)
 
 void git_threads_shutdown(void)
 {
+	/* Shut down any subsystems that have global state */
+	git__shutdown();
+
 	if (_tls_init) {
 		void *ptr = pthread_getspecific(_tls_key);
 		pthread_setspecific(_tls_key, NULL);
@@ -149,10 +172,6 @@ void git_threads_shutdown(void)
 	pthread_key_delete(_tls_key);
 	_tls_init = 0;
 	git_mutex_free(&git__mwindow_mutex);
-
-	/* Shut down any subsystems that have global state */
-	git_hash_global_shutdown();
-	git_futils_dirs_free();
 }
 
 git_global_st *git__global_state(void)
@@ -179,15 +198,14 @@ static git_global_st __state;
 
 int git_threads_init(void)
 {
-	/* noop */ 
+	/* noop */
 	return 0;
 }
 
 void git_threads_shutdown(void)
 {
 	/* Shut down any subsystems that have global state */
-	git_hash_global_shutdown();
-	git_futils_dirs_free();
+	git__shutdown();
 }
 
 git_global_st *git__global_state(void)

src/global.h

@@ -21,4 +21,8 @@ extern git_mutex git__mwindow_mutex;
 
 #define GIT_GLOBAL (git__global_state())
 
+typedef void (*git_global_shutdown_fn)(void);
+
+extern void git__on_shutdown(git_global_shutdown_fn callback);
+
 #endif

src/hash.h

@@ -13,8 +13,6 @@ typedef struct git_hash_prov git_hash_prov;
 typedef struct git_hash_ctx git_hash_ctx;
 
 int git_hash_global_init(void);
-void git_hash_global_shutdown(void);
-
 int git_hash_ctx_init(git_hash_ctx *ctx);
 void git_hash_ctx_cleanup(git_hash_ctx *ctx);
 

src/hash/hash_generic.h

@@ -17,7 +17,6 @@ struct git_hash_ctx {
 };
 
 #define git_hash_global_init() 0
-#define git_hash_global_shutdown() /* noop */
 #define git_hash_ctx_init(ctx) git_hash_init(ctx)
 #define git_hash_ctx_cleanup(ctx)
 

src/hash/hash_openssl.h

@@ -17,7 +17,6 @@ struct git_hash_ctx {
 };
 
 #define git_hash_global_init() 0
-#define git_hash_global_shutdown() /* noop */
 #define git_hash_ctx_init(ctx) git_hash_init(ctx)
 #define git_hash_ctx_cleanup(ctx)
 

src/hash/hash_win32.c

@@ -89,7 +89,15 @@ GIT_INLINE(void) hash_cryptoapi_prov_shutdown(void)
 	hash_prov.type = INVALID;
 }
 
-int git_hash_global_init()
+static void git_hash_global_shutdown(void)
+{
+	if (hash_prov.type == CNG)
+		hash_cng_prov_shutdown();
+	else if(hash_prov.type == CRYPTOAPI)
+		hash_cryptoapi_prov_shutdown();
+}
+
+int git_hash_global_init(void)
 {
 	int error = 0;
 
@@ -99,15 +107,9 @@ int git_hash_global_init()
 	if ((error = hash_cng_prov_init()) < 0)
 		error = hash_cryptoapi_prov_init();
 
-	return error;	
-}
+	git__on_shutdown(git_hash_global_shutdown);
 
-void git_hash_global_shutdown()
-{
-	if (hash_prov.type == CNG)
-		hash_cng_prov_shutdown();
-	else if(hash_prov.type == CRYPTOAPI)
-		hash_cryptoapi_prov_shutdown();
+	return error;
 }
 
 /* CryptoAPI: available in Windows XP and newer */

src/ident.c

+/*
+ * Copyright (C) the libgit2 contributors. All rights reserved.
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+
+#include "git2/sys/filter.h"
+#include "filter.h"
+#include "buffer.h"
+#include "buf_text.h"
+
+static int ident_find_id(
+	const char **id_start, const char **id_end, const char *start, size_t len)
+{
+	const char *end = start + len, *found = NULL;
+
+	while (len > 3 && (found = memchr(start, '$', len)) != NULL) {
+		size_t remaining = (size_t)(end - found) - 1;
+		if (remaining < 3)
+			return GIT_ENOTFOUND;
+
+		start = found + 1;
+		len   = remaining;
+
+		if (start[0] == 'I' && start[1] == 'd')
+			break;
+	}
+
+	if (len < 3 || !found)
+		return GIT_ENOTFOUND;
+	*id_start = found;
+
+	if ((found = memchr(start + 2, '$', len - 2)) == NULL)
+		return GIT_ENOTFOUND;
+
+	*id_end = found + 1;
+	return 0;
+}
+
+static int ident_insert_id(
+	git_buf *to, const git_buf *from, const git_filter_source *src)
+{
+	char oid[GIT_OID_HEXSZ+1];
+	const char *id_start, *id_end, *from_end = from->ptr + from->size;
+	size_t need_size;
+
+	/* replace $Id$ with blob id */
+
+	if (!git_filter_source_id(src))
+		return GIT_PASSTHROUGH;
+
+	git_oid_tostr(oid, sizeof(oid), git_filter_source_id(src));
+
+	if (ident_find_id(&id_start, &id_end, from->ptr, from->size) < 0)
+		return GIT_PASSTHROUGH;
+
+	need_size = (size_t)(id_start - from->ptr) +
+		5 /* "$Id: " */ + GIT_OID_HEXSZ + 1 /* "$" */ +
+		(size_t)(from_end - id_end);
+
+	if (git_buf_grow(to, need_size) < 0)
+		return -1;
+
+	git_buf_set(to, from->ptr, (size_t)(id_start - from->ptr));
+	git_buf_put(to, "$Id: ", 5);
+	git_buf_put(to, oid, GIT_OID_HEXSZ);
+	git_buf_putc(to, '$');
+	git_buf_put(to, id_end, (size_t)(from_end - id_end));
+
+	return git_buf_oom(to) ? -1 : 0;
+}
+
+static int ident_remove_id(
+	git_buf *to, const git_buf *from)
+{
+	const char *id_start, *id_end, *from_end = from->ptr + from->size;
+	size_t need_size;
+
+	if (ident_find_id(&id_start, &id_end, from->ptr, from->size) < 0)
+		return GIT_PASSTHROUGH;
+
+	need_size = (size_t)(id_start - from->ptr) +
+		4 /* "$Id$" */ + (size_t)(from_end - id_end);
+
+	if (git_buf_grow(to, need_size) < 0)
+		return -1;
+
+	git_buf_set(to, from->ptr, (size_t)(id_start - from->ptr));
+	git_buf_put(to, "$Id$", 4);
+	git_buf_put(to, id_end, (size_t)(from_end - id_end));
+
+	return git_buf_oom(to) ? -1 : 0;
+}
+
+static int ident_apply(
+	git_filter     *self,
+	void          **payload,
+	git_buf        *to,
+	const git_buf  *from,
+	const git_filter_source *src)
+{
+	GIT_UNUSED(self); GIT_UNUSED(payload);
+
+	/* Don't filter binary files */
+	if (git_buf_text_is_binary(from))
+		return GIT_PASSTHROUGH;
+
+	if (git_filter_source_mode(src) == GIT_FILTER_SMUDGE)
+		return ident_insert_id(to, from, src);
+	else
+		return ident_remove_id(to, from);
+}
+
+git_filter *git_ident_filter_new(void)
+{
+	git_filter *f = git__calloc(1, sizeof(git_filter));
+
+	f->version = GIT_FILTER_VERSION;
+	f->attributes = "+ident"; /* apply to files with ident attribute set */
+	f->shutdown = git_filter_free;
+	f->apply    = ident_apply;
+
+	return f;
+}

src/odb.c

@@ -179,28 +179,30 @@ done:
 }
 
 int git_odb__hashfd_filtered(
-	git_oid *out, git_file fd, size_t size, git_otype type, git_vector *filters)
+	git_oid *out, git_file fd, size_t size, git_otype type, git_filter_list *fl)
 {
 	int error;
 	git_buf raw = GIT_BUF_INIT;
-	git_buf filtered = GIT_BUF_INIT;
 
-	if (!filters || !filters->length)
+	if (!fl)
 		return git_odb__hashfd(out, fd, size, type);
 
 	/* size of data is used in header, so we have to read the whole file
 	 * into memory to apply filters before beginning to calculate the hash
 	 */
 
-	if (!(error = git_futils_readbuffer_fd(&raw, fd, size)))
-		error = git_filters_apply(&filtered, &raw, filters);
+	if (!(error = git_futils_readbuffer_fd(&raw, fd, size))) {
+		git_buf post = GIT_BUF_INIT;
 
-	git_buf_free(&raw);
+		error = git_filter_list_apply_to_data(&post, fl, &raw);
 
-	if (!error)
-		error = git_odb_hash(out, filtered.ptr, filtered.size, type);
+		git_buf_free(&raw);
 
-	git_buf_free(&filtered);
+		if (!error)
+			error = git_odb_hash(out, post.ptr, post.size, type);
+
+		git_buf_free(&post);
+	}
 
 	return error;
 }

src/odb.h

@@ -14,6 +14,7 @@
 #include "vector.h"
 #include "cache.h"
 #include "posix.h"
+#include "filter.h"
 
 #define GIT_OBJECTS_DIR "objects/"
 #define GIT_OBJECT_DIR_MODE 0777
@@ -66,7 +67,7 @@ int git_odb__hashfd(git_oid *out, git_file fd, size_t size, git_otype type);
  * Acts just like git_odb__hashfd with the addition of filters...
  */
 int git_odb__hashfd_filtered(
-	git_oid *out, git_file fd, size_t len, git_otype type, git_vector *filters);
+	git_oid *out, git_file fd, size_t len, git_otype type, git_filter_list *fl);
 
 /*
  * Hash a `path`, assuming it could be a POSIX symlink: if the path is a

src/path.c

@@ -565,7 +565,7 @@ static bool _check_dir_contents(
 	size_t sub_size = strlen(sub);
 
 	/* leave base valid even if we could not make space for subdir */
-	if (git_buf_try_grow(dir, dir_size + sub_size + 2, false) < 0)
+	if (git_buf_try_grow(dir, dir_size + sub_size + 2, false, false) < 0)
 		return false;
 
 	/* save excursion */

src/repository.c

@@ -1649,7 +1649,7 @@ int git_repository_hashfile(
 	const char *as_path)
 {
 	int error;
-	git_vector filters = GIT_VECTOR_INIT;
+	git_filter_list *fl = NULL;
 	git_file fd = -1;
 	git_off_t len;
 	git_buf full_path = GIT_BUF_INIT;
@@ -1671,7 +1671,8 @@ int git_repository_hashfile(
 
 	/* passing empty string for "as_path" indicated --no-filters */
 	if (strlen(as_path) > 0) {
-		error = git_filters_load(&filters, repo, as_path, GIT_FILTER_TO_ODB);
+		error = git_filter_list_load(
+			&fl, repo, NULL, as_path, GIT_FILTER_TO_ODB);
 		if (error < 0)
 			return error;
 	} else {
@@ -1698,12 +1699,12 @@ int git_repository_hashfile(
 		goto cleanup;
 	}
 
-	error = git_odb__hashfd_filtered(out, fd, (size_t)len, type, &filters);
+	error = git_odb__hashfd_filtered(out, fd, (size_t)len, type, fl);
 
 cleanup:
 	if (fd >= 0)
 		p_close(fd);
-	git_filters_free(&filters);
+	git_filter_list_free(fl);
 	git_buf_free(&full_path);
 
 	return error;

src/util.c

@@ -679,6 +679,9 @@ size_t git__unescape(char *str)
 {
 	char *scan, *pos = str;
 
+	if (!str)
+		return 0;
+
 	for (scan = str; *scan; pos++, scan++) {
 		if (*scan == '\\' && *(scan + 1) != '\0')
 			scan++; /* skip '\' but include next char */

src/win32/pthread.c

@@ -6,6 +6,7 @@
  */
 
 #include "pthread.h"
+#include "../global.h"
 
 int pthread_create(
 	pthread_t *GIT_RESTRICT thread,
@@ -217,6 +218,14 @@ int pthread_rwlock_destroy(pthread_rwlock_t *lock)
 }
 
 
+static void win32_pthread_shutdown(void)
+{
+	if (win32_kernel32_dll) {
+		FreeLibrary(win32_kernel32_dll);
+		win32_kernel32_dll = NULL;
+	}
+}
+
 int win32_pthread_initialize(void)
 {
 	if (win32_kernel32_dll)
@@ -239,15 +248,7 @@ int win32_pthread_initialize(void)
 	win32_srwlock_release_exclusive = (win32_srwlock_fn)
 		GetProcAddress(win32_kernel32_dll, "ReleaseSRWLockExclusive");
 
-	return 0;
-}
-
-int win32_pthread_shutdown(void)
-{
-	if (win32_kernel32_dll) {
-		FreeLibrary(win32_kernel32_dll);
-		win32_kernel32_dll = NULL;
-	}
+	git__on_shutdown(win32_pthread_shutdown);
 
 	return 0;
 }

src/win32/pthread.h

@@ -69,6 +69,5 @@ int pthread_rwlock_wrunlock(pthread_rwlock_t *);
 int pthread_rwlock_destroy(pthread_rwlock_t *);
 
 extern int win32_pthread_initialize(void);
-extern int win32_pthread_shutdown(void);
 
 #endif

tests-clar/attr/repo.c

@@ -100,6 +100,22 @@ void test_attr_repo__get_many(void)
 	cl_assert_equal_s("yes", values[3]);
 }
 
+void test_attr_repo__get_many_in_place(void)
+{
+	const char *vals[4] = { "repoattr", "rootattr", "missingattr", "subattr" };
+
+	/* it should be legal to look up values into the same array that has
+	 * the attribute names, overwriting each name as the value is found.
+	 */
+
+	cl_git_pass(git_attr_get_many(vals, g_repo, 0, "sub/subdir_test1", 4, vals));
+
+	cl_assert(GIT_ATTR_TRUE(vals[0]));
+	cl_assert(GIT_ATTR_TRUE(vals[1]));
+	cl_assert(GIT_ATTR_UNSPECIFIED(vals[2]));
+	cl_assert_equal_s("yes", vals[3]);
+}
+
 static int count_attrs(
 	const char *name,
 	const char *value,

tests-clar/checkout/checkout_helpers.c

@@ -3,22 +3,6 @@
 #include "refs.h"
 #include "fileops.h"
 
-/* this is essentially the code from git__unescape modified slightly */
-void strip_cr_from_buf(git_buf *buf)
-{
-	char *scan, *pos = buf->ptr, *end = pos + buf->size;
-
-	for (scan = pos; scan < end; pos++, scan++) {
-		if (*scan == '\r')
-			scan++; /* skip '\r' */
-		if (pos != scan)
-			*pos = *scan;
-	}
-
-	*pos = '\0';
-	buf->size = (pos - buf->ptr);
-}
-
 void assert_on_branch(git_repository *repo, const char *branch)
 {
 	git_reference *head;
@@ -50,48 +34,6 @@ void reset_index_to_treeish(git_object *treeish)
 	git_index_free(index);
 }
 
-static void check_file_contents_internal(
-	const char *path,
-	const char *expected_content,
-	bool strip_cr,
-	const char *file,
-	int line,
-	const char *msg)
-{
-	int fd;
-	char data[1024] = {0};
-	git_buf buf = GIT_BUF_INIT;
-	size_t expected_len = expected_content ? strlen(expected_content) : 0;
-
-	fd = p_open(path, O_RDONLY);
-	cl_assert(fd >= 0);
-
-	buf.ptr = data;
-	buf.size = p_read(fd, buf.ptr, sizeof(data));
-
-	cl_git_pass(p_close(fd));
-
-	if (strip_cr)
-		strip_cr_from_buf(&buf);
-
-	clar__assert_equal(file, line, "strlen(expected_content) != strlen(actual_content)", 1, PRIuZ, expected_len, (size_t)buf.size);
-	clar__assert_equal(file, line, msg, 1, "%s", expected_content, buf.ptr);
-}
-
-void check_file_contents_at_line(
-	const char *path, const char *expected,
-	const char *file, int line, const char *msg)
-{
-	check_file_contents_internal(path, expected, false, file, line, msg);
-}
-
-void check_file_contents_nocr_at_line(
-	const char *path, const char *expected,
-	const char *file, int line, const char *msg)
-{
-	check_file_contents_internal(path, expected, true, file, line, msg);
-}
-
 int checkout_count_callback(
 	git_checkout_notify_t why,
 	const char *path,

tests-clar/checkout/checkout_helpers.h

@@ -2,23 +2,14 @@
 #include "git2/object.h"
 #include "git2/repository.h"
 
-extern void strip_cr_from_buf(git_buf *buf);
 extern void assert_on_branch(git_repository *repo, const char *branch);
 extern void reset_index_to_treeish(git_object *treeish);
 
-extern void check_file_contents_at_line(
-	const char *path, const char *expected,
-	const char *file, int line, const char *msg);
-
-extern void check_file_contents_nocr_at_line(
-	const char *path, const char *expected,
-	const char *file, int line, const char *msg);
-
 #define check_file_contents(PATH,EXP) \
-	check_file_contents_at_line(PATH,EXP,__FILE__,__LINE__,"String mismatch: " #EXP " != " #PATH)
+	cl_assert_equal_file(EXP,0,PATH)
 
 #define check_file_contents_nocr(PATH,EXP) \
-	check_file_contents_nocr_at_line(PATH,EXP,__FILE__,__LINE__,"String mismatch: " #EXP " != " #PATH)
+	cl_assert_equal_file_ignore_cr(EXP,0,PATH)
 
 typedef struct {
 	int n_conflicts;

tests-clar/checkout/crlf.c

 #include "clar_libgit2.h"
 #include "checkout_helpers.h"
+#include "../filter/crlf.h"
 
 #include "git2/checkout.h"
 #include "repository.h"
-
-#define UTF8_BOM "\xEF\xBB\xBF"
-#define ALL_CRLF_TEXT_RAW	"crlf\r\ncrlf\r\ncrlf\r\ncrlf\r\n"
-#define ALL_LF_TEXT_RAW		"lf\nlf\nlf\nlf\nlf\n"
-#define MORE_CRLF_TEXT_RAW	"crlf\r\ncrlf\r\nlf\ncrlf\r\ncrlf\r\n"
-#define MORE_LF_TEXT_RAW	"lf\nlf\ncrlf\r\nlf\nlf\n"
-
-#define ALL_LF_TEXT_AS_CRLF		"lf\r\nlf\r\nlf\r\nlf\r\nlf\r\n"
-#define MORE_CRLF_TEXT_AS_CRLF	"crlf\r\ncrlf\r\nlf\r\ncrlf\r\ncrlf\r\n"
-#define MORE_LF_TEXT_AS_CRLF	"lf\r\nlf\r\ncrlf\r\nlf\r\nlf\r\n"
+#include "posix.h"
 
 static git_repository *g_repo;
 
@@ -145,3 +137,95 @@ void test_checkout_crlf__autocrlf_true_index_size_is_filtered_size(void)
 
 	git_index_free(index);
 }
+
+void test_checkout_crlf__with_ident(void)
+{
+	git_index *index;
+	git_blob *blob;
+	git_checkout_opts opts = GIT_CHECKOUT_OPTS_INIT;
+	opts.checkout_strategy = GIT_CHECKOUT_SAFE_CREATE;
+
+	cl_git_mkfile("crlf/.gitattributes",
+		"*.txt text\n*.bin binary\n"
+		"*.crlf text eol=crlf\n"
+		"*.lf text eol=lf\n"
+		"*.ident text ident\n"
+		"*.identcrlf ident text eol=crlf\n"
+		"*.identlf ident text eol=lf\n");
+
+	cl_repo_set_bool(g_repo, "core.autocrlf", true);
+
+	/* add files with $Id$ */
+
+	cl_git_mkfile("crlf/lf.ident", ALL_LF_TEXT_RAW "\n$Id: initial content$\n");
+	cl_git_mkfile("crlf/crlf.ident", ALL_CRLF_TEXT_RAW "\r\n$Id$\r\n\r\n");
+	cl_git_mkfile("crlf/more1.identlf", "$Id$\n" MORE_LF_TEXT_RAW);
+	cl_git_mkfile("crlf/more2.identcrlf", "\r\n$Id: $\r\n" MORE_CRLF_TEXT_RAW);
+
+	cl_git_pass(git_repository_index(&index, g_repo));
+	cl_git_pass(git_index_add_bypath(index, "lf.ident"));
+	cl_git_pass(git_index_add_bypath(index, "crlf.ident"));
+	cl_git_pass(git_index_add_bypath(index, "more1.identlf"));
+	cl_git_pass(git_index_add_bypath(index, "more2.identcrlf"));
+	cl_repo_commit_from_index(NULL, g_repo, NULL, 0, "Some ident files\n");
+
+	git_checkout_head(g_repo, &opts);
+
+	/* check that blobs have $Id$ */
+
+	cl_git_pass(git_blob_lookup(&blob, g_repo,
+		& git_index_get_bypath(index, "lf.ident", 0)->oid));
+	cl_assert_equal_s(
+		ALL_LF_TEXT_RAW "\n$Id$\n", git_blob_rawcontent(blob));
+	git_blob_free(blob);
+
+	cl_git_pass(git_blob_lookup(&blob, g_repo,
+		& git_index_get_bypath(index, "more2.identcrlf", 0)->oid));
+	cl_assert_equal_s(
+		"\n$Id$\n" MORE_CRLF_TEXT_AS_LF, git_blob_rawcontent(blob));
+	git_blob_free(blob);
+
+	/* check that filesystem is initially untouched - matching core Git */
+
+	cl_assert_equal_file(
+		ALL_LF_TEXT_RAW "\n$Id: initial content$\n", 0, "crlf/lf.ident");
+
+	/* check that forced checkout rewrites correctly */
+
+	p_unlink("crlf/lf.ident");
+	p_unlink("crlf/crlf.ident");
+	p_unlink("crlf/more1.identlf");
+	p_unlink("crlf/more2.identcrlf");
+
+	git_checkout_head(g_repo, &opts);
+
+	if (GIT_EOL_NATIVE == GIT_EOL_LF) {
+		cl_assert_equal_file(
+			ALL_LF_TEXT_RAW
+			"\n$Id: fcf6d4d9c212dc66563b1171b1cd99953c756467$\n",
+			0, "crlf/lf.ident");
+		cl_assert_equal_file(
+			ALL_CRLF_TEXT_AS_LF
+			"\n$Id: f2c66ad9b2b5a734d9bf00d5000cc10a62b8a857$\n\n",
+			0, "crlf/crlf.ident");
+	} else {
+		cl_assert_equal_file(
+			ALL_LF_TEXT_AS_CRLF
+			"\r\n$Id: fcf6d4d9c212dc66563b1171b1cd99953c756467$\r\n",
+			0, "crlf/lf.ident");
+		cl_assert_equal_file(
+			ALL_CRLF_TEXT_RAW
+			"\r\n$Id: f2c66ad9b2b5a734d9bf00d5000cc10a62b8a857$\r\n\r\n",
+			0, "crlf/crlf.ident");
+	}
+
+	cl_assert_equal_file(
+		"$Id: f7830382dac1f1583422be5530fdfbd26289431b$\n"
+		MORE_LF_TEXT_AS_LF, 0, "crlf/more1.identlf");
+
+	cl_assert_equal_file(
+		"\r\n$Id: 74677a68413012ce8d7e7cfc3f12603df3a3eac4$\r\n"
+		MORE_CRLF_TEXT_AS_CRLF, 0, "crlf/more2.identcrlf");
+
+	git_index_free(index);
+}

tests-clar/clar.h

@@ -68,7 +68,6 @@ void cl_fixture_cleanup(const char *fixture_name);
 
 #define cl_assert_equal_p(p1,p2) clar__assert_equal(__FILE__,__LINE__,"Pointer mismatch: " #p1 " != " #p2, 1, "%p", (p1), (p2))
 
-#define cl_assert_equal_sz(sz1,sz2) clar__assert_equal(__FILE__,__LINE__,#sz1 " != " #sz2, 1, "%"PRIuZ, (size_t)(sz1), (size_t)(sz2))
 
 void clar__fail(
 	const char *file,

tests-clar/clar_libgit2.c

@@ -337,6 +337,65 @@ int cl_git_remove_placeholders(const char *directory_path, const char *filename)
 	return error;
 }
 
+#define CL_COMMIT_NAME "Libgit2 Tester"
+#define CL_COMMIT_EMAIL "libgit2-test@github.com"
+#define CL_COMMIT_MSG "Test commit of tree "
+
+void cl_repo_commit_from_index(
+	git_oid *out,
+	git_repository *repo,
+	git_signature *sig,
+	git_time_t time,
+	const char *msg)
+{
+	git_index *index;
+	git_oid commit_id, tree_id;
+	git_object *parent = NULL;
+	git_reference *ref = NULL;
+	git_tree *tree = NULL;
+	char buf[128];
+	int free_sig = (sig == NULL);
+
+	/* it is fine if looking up HEAD fails - we make this the first commit */
+	git_revparse_ext(&parent, &ref, repo, "HEAD");
+
+	/* write the index content as a tree */
+	cl_git_pass(git_repository_index(&index, repo));
+	cl_git_pass(git_index_write_tree(&tree_id, index));
+	cl_git_pass(git_index_write(index));
+	git_index_free(index);
+
+	cl_git_pass(git_tree_lookup(&tree, repo, &tree_id));
+
+	if (sig)
+		cl_assert(sig->name && sig->email);
+	else if (!time)
+		cl_git_pass(git_signature_now(&sig, CL_COMMIT_NAME, CL_COMMIT_EMAIL));
+	else
+		cl_git_pass(git_signature_new(
+			&sig, CL_COMMIT_NAME, CL_COMMIT_EMAIL, time, 0));
+
+	if (!msg) {
+		strcpy(buf, CL_COMMIT_MSG);
+		git_oid_tostr(buf + strlen(CL_COMMIT_MSG),
+			sizeof(buf) - strlen(CL_COMMIT_MSG), &tree_id);
+		msg = buf;
+	}
+
+	cl_git_pass(git_commit_create_v(
+		&commit_id, repo, ref ? git_reference_name(ref) : "HEAD",
+		sig, sig, NULL, msg, tree, parent ? 1 : 0, parent));
+
+	if (out)
+		git_oid_cpy(out, &commit_id);
+
+	git_object_free(parent);
+	git_reference_free(ref);
+	if (free_sig)
+		git_signature_free(sig);
+	git_tree_free(tree);
+}
+
 void cl_repo_set_bool(git_repository *repo, const char *cfg, int value)
 {
 	git_config *config;
@@ -354,3 +413,65 @@ int cl_repo_get_bool(git_repository *repo, const char *cfg)
 	git_config_free(config);
 	return val;
 }
+
+/* this is essentially the code from git__unescape modified slightly */
+static size_t strip_cr_from_buf(char *start, size_t len)
+{
+	char *scan, *trail, *end = start + len;
+
+	for (scan = trail = start; scan < end; trail++, scan++) {
+		while (*scan == '\r')
+			scan++; /* skip '\r' */
+
+		if (trail != scan)
+			*trail = *scan;
+	}
+
+	*trail = '\0';
+
+	return (trail - start);
+}
+
+void clar__assert_equal_file(
+	const char *expected_data,
+	size_t expected_bytes,
+	int ignore_cr,
+	const char *path,
+	const char *file,
+	size_t line)
+{
+	char buf[4000];
+	ssize_t bytes, total_bytes = 0;
+	int fd = p_open(path, O_RDONLY | O_BINARY);
+	cl_assert(fd >= 0);
+
+	if (expected_data && !expected_bytes)
+		expected_bytes = strlen(expected_data);
+
+	while ((bytes = p_read(fd, buf, sizeof(buf))) != 0) {
+		clar__assert(
+			bytes > 0, file, line, "error reading from file", path, 1);
+
+		if (ignore_cr)
+			bytes = strip_cr_from_buf(buf, bytes);
+
+		if (memcmp(expected_data, buf, bytes) != 0) {
+			int pos;
+			for (pos = 0; pos < bytes && expected_data[pos] == buf[pos]; ++pos)
+				/* find differing byte offset */;
+			p_snprintf(
+				buf, sizeof(buf), "file content mismatch at byte %d",
+				(int)(total_bytes + pos));
+			clar__fail(file, line, buf, path, 1);
+		}
+
+		expected_data += bytes;
+		total_bytes   += bytes;
+	}
+
+	p_close(fd);
+
+	clar__assert(!bytes, file, line, "error reading from file", path, 1);
+	clar__assert_equal(file, line, "mismatched file length", 1, "%"PRIuZ,
+		(size_t)expected_bytes, (size_t)total_bytes);
+}

tests-clar/clar_libgit2.h

@@ -43,9 +43,28 @@ GIT_INLINE(void) clar__assert_in_range(
 	}
 }
 
+#define cl_assert_equal_sz(sz1,sz2) do { \
+	size_t __sz1 = (sz1), __sz2 = (sz2); \
+	clar__assert_equal(__FILE__,__LINE__,#sz1 " != " #sz2, 1, "%"PRIuZ, __sz1, __sz2); \
+} while (0)
+
 #define cl_assert_in_range(L,V,H) \
 	clar__assert_in_range((L),(V),(H),__FILE__,__LINE__,"Range check: " #V " in [" #L "," #H "]", 1)
 
+#define cl_assert_equal_file(DATA,SIZE,PATH) \
+	clar__assert_equal_file(DATA,SIZE,0,PATH,__FILE__,__LINE__)
+
+#define cl_assert_equal_file_ignore_cr(DATA,SIZE,PATH) \
+	clar__assert_equal_file(DATA,SIZE,1,PATH,__FILE__,__LINE__)
+
+void clar__assert_equal_file(
+	const char *expected_data,
+	size_t expected_size,
+	int ignore_cr,
+	const char *path,
+	const char *file,
+	size_t line);
+
 /*
  * Some utility macros for building long strings
  */
@@ -84,6 +103,14 @@ const char* cl_git_path_url(const char *path);
 /* Test repository cleaner */
 int cl_git_remove_placeholders(const char *directory_path, const char *filename);
 
+/* commit creation helpers */
+void cl_repo_commit_from_index(
+	git_oid *out,
+	git_repository *repo,
+	git_signature *sig,
+	git_time_t time,
+	const char *msg);
+
 /* config setting helpers */
 void cl_repo_set_bool(git_repository *repo, const char *cfg, int value);
 int cl_repo_get_bool(git_repository *repo, const char *cfg);

tests-clar/core/buffer.c

@@ -919,6 +919,8 @@ void test_core_buffer__similarity_metric_whitespace(void)
 	git_buf_free(&buf);
 }
 
+#include "../filter/crlf.h"
+
 #define check_buf(expected,buf) do { \
 	cl_assert_equal_s(expected, buf.ptr); \
 	cl_assert_equal_sz(strlen(expected), buf.size); } while (0)
@@ -934,16 +936,16 @@ void test_core_buffer__lf_and_crlf_conversions(void)
 	cl_git_pass(git_buf_text_lf_to_crlf(&tgt, &src));
 	check_buf("lf\r\nlf\r\nlf\r\nlf\r\n", tgt);
 
-	cl_assert_equal_i(GIT_ENOTFOUND, git_buf_text_crlf_to_lf(&tgt, &src));
-	/* no conversion needed if all LFs already */
+	cl_git_pass(git_buf_text_crlf_to_lf(&tgt, &src));
+	check_buf(src.ptr, tgt);
 
 	git_buf_sets(&src, "\nlf\nlf\nlf\nlf\nlf");
 
 	cl_git_pass(git_buf_text_lf_to_crlf(&tgt, &src));
 	check_buf("\r\nlf\r\nlf\r\nlf\r\nlf\r\nlf", tgt);
 
-	cl_assert_equal_i(GIT_ENOTFOUND, git_buf_text_crlf_to_lf(&tgt, &src));
-	/* no conversion needed if all LFs already */
+	cl_git_pass(git_buf_text_crlf_to_lf(&tgt, &src));
+	check_buf(src.ptr, tgt);
 
 	/* CRLF source */
 
@@ -993,10 +995,45 @@ void test_core_buffer__lf_and_crlf_conversions(void)
 	check_buf("\rcrlf\nlf\nlf\ncr\rcrlf\nlf\ncr\r", tgt);
 
 	git_buf_sets(&src, "\rcr\r");
-	cl_assert_equal_i(GIT_ENOTFOUND, git_buf_text_lf_to_crlf(&tgt, &src));
+	cl_git_pass(git_buf_text_lf_to_crlf(&tgt, &src));
+	check_buf(src.ptr, tgt);
 	cl_git_pass(git_buf_text_crlf_to_lf(&tgt, &src));
 	check_buf("\rcr\r", tgt);
 
 	git_buf_free(&src);
 	git_buf_free(&tgt);
+
+	/* blob correspondence tests */
+
+	git_buf_sets(&src, ALL_CRLF_TEXT_RAW);
+	cl_git_pass(git_buf_text_lf_to_crlf(&tgt, &src));
+	check_buf(ALL_CRLF_TEXT_AS_CRLF, tgt);
+	cl_git_pass(git_buf_text_crlf_to_lf(&tgt, &src));
+	check_buf(ALL_CRLF_TEXT_AS_LF, tgt);
+	git_buf_free(&src);
+	git_buf_free(&tgt);
+
+	git_buf_sets(&src, ALL_LF_TEXT_RAW);
+	cl_git_pass(git_buf_text_lf_to_crlf(&tgt, &src));
+	check_buf(ALL_LF_TEXT_AS_CRLF, tgt);
+	cl_git_pass(git_buf_text_crlf_to_lf(&tgt, &src));
+	check_buf(ALL_LF_TEXT_AS_LF, tgt);
+	git_buf_free(&src);
+	git_buf_free(&tgt);
+
+	git_buf_sets(&src, MORE_CRLF_TEXT_RAW);
+	cl_git_pass(git_buf_text_lf_to_crlf(&tgt, &src));
+	check_buf(MORE_CRLF_TEXT_AS_CRLF, tgt);
+	cl_git_pass(git_buf_text_crlf_to_lf(&tgt, &src));
+	check_buf(MORE_CRLF_TEXT_AS_LF, tgt);
+	git_buf_free(&src);
+	git_buf_free(&tgt);
+
+	git_buf_sets(&src, MORE_LF_TEXT_RAW);
+	cl_git_pass(git_buf_text_lf_to_crlf(&tgt, &src));
+	check_buf(MORE_LF_TEXT_AS_CRLF, tgt);
+	cl_git_pass(git_buf_text_crlf_to_lf(&tgt, &src));
+	check_buf(MORE_LF_TEXT_AS_LF, tgt);
+	git_buf_free(&src);
+	git_buf_free(&tgt);
 }

tests-clar/diff/rename.c

@@ -7,6 +7,8 @@ static git_repository *g_repo = NULL;
 void test_diff_rename__initialize(void)
 {
 	g_repo = cl_git_sandbox_init("renames");
+
+	cl_repo_set_bool(g_repo, "core.autocrlf", false);
 }
 
 void test_diff_rename__cleanup(void)

tests-clar/diff/submodules.c

@@ -11,7 +11,6 @@ void test_diff_submodules__initialize(void)
 
 void test_diff_submodules__cleanup(void)
 {
-	cleanup_fixture_submodules();
 }
 
 static void check_diff_patches_at_line(
@@ -229,11 +228,11 @@ void test_diff_submodules__invalid_cache(void)
 		"<END>"
 	};
 	static const char *expected_moved[] = {
-		"diff --git a/sm_changed_head b/sm_changed_head\nindex 3d9386c..0910a13 160000\n--- a/sm_changed_head\n+++ b/sm_changed_head\n@@ -1 +1 @@\n-Subproject commit 3d9386c507f6b093471a3e324085657a3c2b4247\n+Subproject commit 0910a13dfa2210496f6c590d75bc360dd11b2a1b\n",
+		"diff --git a/sm_changed_head b/sm_changed_head\nindex 3d9386c..7002348 160000\n--- a/sm_changed_head\n+++ b/sm_changed_head\n@@ -1 +1 @@\n-Subproject commit 3d9386c507f6b093471a3e324085657a3c2b4247\n+Subproject commit 700234833f6ccc20d744b238612646be071acaae\n",
 		"<END>"
 	};
 	static const char *expected_moved_dirty[] = {
-		"diff --git a/sm_changed_head b/sm_changed_head\nindex 3d9386c..0910a13 160000\n--- a/sm_changed_head\n+++ b/sm_changed_head\n@@ -1 +1 @@\n-Subproject commit 3d9386c507f6b093471a3e324085657a3c2b4247\n+Subproject commit 0910a13dfa2210496f6c590d75bc360dd11b2a1b-dirty\n",
+		"diff --git a/sm_changed_head b/sm_changed_head\nindex 3d9386c..7002348 160000\n--- a/sm_changed_head\n+++ b/sm_changed_head\n@@ -1 +1 @@\n-Subproject commit 3d9386c507f6b093471a3e324085657a3c2b4247\n+Subproject commit 700234833f6ccc20d744b238612646be071acaae-dirty\n",
 		"<END>"
 	};
 
@@ -310,26 +309,7 @@ void test_diff_submodules__invalid_cache(void)
 	git_diff_list_free(diff);
 
 	/* commit changed index of submodule */
-	{
-		git_object *parent;
-		git_oid tree_id, commit_id;
-		git_tree *tree;
-		git_signature *sig;
-		git_reference *ref;
-
-		cl_git_pass(git_revparse_ext(&parent, &ref, smrepo, "HEAD"));
-		cl_git_pass(git_index_write_tree(&tree_id, smindex));
-		cl_git_pass(git_index_write(smindex));
-		cl_git_pass(git_tree_lookup(&tree, smrepo, &tree_id));
-		cl_git_pass(git_signature_new(&sig, "Sm Test", "sm@tester.test", 1372350000, 480));
-		cl_git_pass(git_commit_create_v(
-			&commit_id, smrepo, git_reference_name(ref), sig, sig,
-			NULL, "Move it", tree, 1, parent));
-		git_object_free(parent);
-		git_tree_free(tree);
-		git_reference_free(ref);
-		git_signature_free(sig);
-	}
+	cl_repo_commit_from_index(NULL, smrepo, NULL, 1372350000, "Move it");
 
 	git_submodule_set_ignore(sm, GIT_SUBMODULE_IGNORE_DIRTY);
 

tests-clar/filter/blob.c

+#include "clar_libgit2.h"
+#include "crlf.h"
+
+static git_repository *g_repo = NULL;
+
+void test_filter_blob__initialize(void)
+{
+	g_repo = cl_git_sandbox_init("crlf");
+	cl_git_mkfile("crlf/.gitattributes",
+		"*.txt text\n*.bin binary\n"
+		"*.crlf text eol=crlf\n"
+		"*.lf text eol=lf\n"
+		"*.ident text ident\n"
+		"*.identcrlf ident text eol=crlf\n"
+		"*.identlf ident text eol=lf\n");
+}
+
+void test_filter_blob__cleanup(void)
+{
+	cl_git_sandbox_cleanup();
+}
+
+void test_filter_blob__all_crlf(void)
+{
+	git_blob *blob;
+	git_buf buf = { 0 };
+
+	cl_git_pass(git_revparse_single(
+		(git_object **)&blob, g_repo, "a9a2e891")); /* all-crlf */
+
+	cl_assert_equal_s(ALL_CRLF_TEXT_RAW, git_blob_rawcontent(blob));
+
+	cl_git_pass(git_blob_filtered_content(&buf, blob, "file.bin", 1));
+
+	cl_assert_equal_s(ALL_CRLF_TEXT_RAW, buf.ptr);
+
+	cl_git_pass(git_blob_filtered_content(&buf, blob, "file.crlf", 1));
+
+	/* in this case, raw content has crlf in it already */
+	cl_assert_equal_s(ALL_CRLF_TEXT_AS_CRLF, buf.ptr);
+
+	cl_git_pass(git_blob_filtered_content(&buf, blob, "file.lf", 1));
+
+	cl_assert_equal_s(ALL_CRLF_TEXT_AS_LF, buf.ptr);
+
+	git_buf_free(&buf);
+	git_blob_free(blob);
+}
+
+void test_filter_blob__ident(void)
+{
+	git_oid id;
+	git_blob *blob;
+	git_buf buf = { 0 };
+
+	cl_git_mkfile("crlf/test.ident", "Some text\n$Id$\nGoes there\n");
+	cl_git_pass(git_blob_create_fromworkdir(&id, g_repo, "test.ident"));
+	cl_git_pass(git_blob_lookup(&blob, g_repo, &id));
+	cl_assert_equal_s(
+		"Some text\n$Id$\nGoes there\n", git_blob_rawcontent(blob));
+	git_blob_free(blob);
+
+	cl_git_mkfile("crlf/test.ident", "Some text\n$Id: Any old just you want$\nGoes there\n");
+	cl_git_pass(git_blob_create_fromworkdir(&id, g_repo, "test.ident"));
+	cl_git_pass(git_blob_lookup(&blob, g_repo, &id));
+	cl_assert_equal_s(
+		"Some text\n$Id$\nGoes there\n", git_blob_rawcontent(blob));
+
+	cl_git_pass(git_blob_filtered_content(&buf, blob, "filter.bin", 1));
+	cl_assert_equal_s(
+		"Some text\n$Id$\nGoes there\n", buf.ptr);
+
+	cl_git_pass(git_blob_filtered_content(&buf, blob, "filter.identcrlf", 1));
+	cl_assert_equal_s(
+		"Some text\r\n$Id: 3164f585d548ac68027d22b104f2d8100b2b6845$\r\nGoes there\r\n", buf.ptr);
+
+	cl_git_pass(git_blob_filtered_content(&buf, blob, "filter.identlf", 1));
+	cl_assert_equal_s(
+		"Some text\n$Id: 3164f585d548ac68027d22b104f2d8100b2b6845$\nGoes there\n", buf.ptr);
+
+	git_buf_free(&buf);
+	git_blob_free(blob);
+
+}

tests-clar/filter/crlf.c

+#include "clar_libgit2.h"
+#include "git2/sys/filter.h"
+
+static git_repository *g_repo = NULL;
+
+void test_filter_crlf__initialize(void)
+{
+	g_repo = cl_git_sandbox_init("crlf");
+
+	cl_git_mkfile("crlf/.gitattributes",
+		"*.txt text\n*.bin binary\n*.crlf text eol=crlf\n*.lf text eol=lf\n");
+
+	cl_repo_set_bool(g_repo, "core.autocrlf", true);
+}
+
+void test_filter_crlf__cleanup(void)
+{
+	cl_git_sandbox_cleanup();
+}
+
+void test_filter_crlf__to_worktree(void)
+{
+	git_filter_list *fl;
+	git_filter *crlf;
+	git_buf in = { 0 }, out = { 0 };
+
+	cl_git_pass(git_filter_list_new(&fl, g_repo, GIT_FILTER_TO_WORKTREE));
+
+	crlf = git_filter_lookup(GIT_FILTER_CRLF);
+	cl_assert(crlf != NULL);
+
+	cl_git_pass(git_filter_list_push(fl, crlf, NULL));
+
+	in.ptr = "Some text\nRight here\n";
+	in.size = strlen(in.ptr);
+
+	cl_git_pass(git_filter_list_apply_to_data(&out, fl, &in));
+
+#ifdef GIT_WIN32
+	cl_assert_equal_s("Some text\r\nRight here\r\n", out.ptr);
+#else
+	cl_assert_equal_s("Some text\nRight here\n", out.ptr);
+#endif
+
+	git_filter_list_free(fl);
+	git_buf_free(&out);
+}
+
+void test_filter_crlf__to_odb(void)
+{
+	git_filter_list *fl;
+	git_filter *crlf;
+	git_buf in = { 0 }, out = { 0 };
+
+	cl_git_pass(git_filter_list_new(&fl, g_repo, GIT_FILTER_TO_ODB));
+
+	crlf = git_filter_lookup(GIT_FILTER_CRLF);
+	cl_assert(crlf != NULL);
+
+	cl_git_pass(git_filter_list_push(fl, crlf, NULL));
+
+	in.ptr = "Some text\r\nRight here\r\n";
+	in.size = strlen(in.ptr);
+
+	cl_git_pass(git_filter_list_apply_to_data(&out, fl, &in));
+
+	cl_assert_equal_s("Some text\nRight here\n", out.ptr);
+
+	git_filter_list_free(fl);
+	git_buf_free(&out);
+}

tests-clar/filter/crlf.h

+#ifndef INCLUDE_filter_crlf_h__
+#define INCLUDE_filter_crlf_h__
+
+/*
+ * file content for files in the resources/crlf repository
+ */
+
+#define UTF8_BOM "\xEF\xBB\xBF"
+
+#define ALL_CRLF_TEXT_RAW		"crlf\r\ncrlf\r\ncrlf\r\ncrlf\r\n"
+#define ALL_LF_TEXT_RAW			"lf\nlf\nlf\nlf\nlf\n"
+#define MORE_CRLF_TEXT_RAW		"crlf\r\ncrlf\r\nlf\ncrlf\r\ncrlf\r\n"
+#define MORE_LF_TEXT_RAW		"lf\nlf\ncrlf\r\nlf\nlf\n"
+
+#define ALL_CRLF_TEXT_AS_CRLF	ALL_CRLF_TEXT_RAW
+#define ALL_LF_TEXT_AS_CRLF		"lf\r\nlf\r\nlf\r\nlf\r\nlf\r\n"
+#define MORE_CRLF_TEXT_AS_CRLF	"crlf\r\ncrlf\r\nlf\r\ncrlf\r\ncrlf\r\n"
+#define MORE_LF_TEXT_AS_CRLF	"lf\r\nlf\r\ncrlf\r\nlf\r\nlf\r\n"
+
+#define ALL_CRLF_TEXT_AS_LF		"crlf\ncrlf\ncrlf\ncrlf\n"
+#define ALL_LF_TEXT_AS_LF		ALL_LF_TEXT_RAW
+#define MORE_CRLF_TEXT_AS_LF	"crlf\ncrlf\nlf\ncrlf\ncrlf\n"
+#define MORE_LF_TEXT_AS_LF		"lf\nlf\ncrlf\nlf\nlf\n"
+
+#endif

tests-clar/filter/custom.c

+#include "clar_libgit2.h"
+#include "posix.h"
+#include "blob.h"
+#include "filter.h"
+#include "buf_text.h"
+#include "git2/sys/filter.h"
+#include "git2/sys/repository.h"
+
+/* going TO_WORKDIR, filters are executed low to high
+ * going TO_ODB, filters are executed high to low
+ */
+#define BITFLIP_FILTER_PRIORITY -1
+#define REVERSE_FILTER_PRIORITY -2
+
+#define VERY_SECURE_ENCRYPTION(b) ((b) ^ 0xff)
+
+#ifdef GIT_WIN32
+# define NEWLINE "\r\n"
+#else
+# define NEWLINE "\n"
+#endif
+
+static char workdir_data[] =
+	"some simple" NEWLINE
+	"data" NEWLINE
+	"that will be" NEWLINE
+	"trivially" NEWLINE
+	"scrambled." NEWLINE;
+
+/* Represents the data above scrambled (bits flipped) after \r\n -> \n
+ * conversion, then bytewise reversed
+ */
+static unsigned char bitflipped_and_reversed_data[] =
+	{ 0xf5, 0xd1, 0x9b, 0x9a, 0x93, 0x9d, 0x92, 0x9e, 0x8d, 0x9c, 0x8c,
+	  0xf5, 0x86, 0x93, 0x93, 0x9e, 0x96, 0x89, 0x96, 0x8d, 0x8b, 0xf5,
+	  0x9a, 0x9d, 0xdf, 0x93, 0x93, 0x96, 0x88, 0xdf, 0x8b, 0x9e, 0x97,
+	  0x8b, 0xf5, 0x9e, 0x8b, 0x9e, 0x9b, 0xf5, 0x9a, 0x93, 0x8f, 0x92,
+	  0x96, 0x8c, 0xdf, 0x9a, 0x92, 0x90, 0x8c };
+
+#define BITFLIPPED_AND_REVERSED_DATA_LEN 51
+
+static git_repository *g_repo = NULL;
+
+static void register_custom_filters(void);
+
+void test_filter_custom__initialize(void)
+{
+	register_custom_filters();
+
+	g_repo = cl_git_sandbox_init("empty_standard_repo");
+
+	cl_git_mkfile(
+		"empty_standard_repo/.gitattributes",
+		"hero* bitflip reverse\n"
+		"herofile text\n"
+		"heroflip -reverse binary\n"
+		"*.bin binary\n");
+}
+
+void test_filter_custom__cleanup(void)
+{
+	cl_git_sandbox_cleanup();
+	g_repo = NULL;
+}
+
+static int bitflip_filter_apply(
+	git_filter     *self,
+	void          **payload,
+	git_buf        *to,
+	const git_buf  *from,
+	const git_filter_source *source)
+{
+	const unsigned char *src = (const unsigned char *)from->ptr;
+	unsigned char *dst;
+	size_t i;
+
+	GIT_UNUSED(self); GIT_UNUSED(payload);
+
+	/* verify that attribute path match worked as expected */
+	cl_assert_equal_i(
+		0, git__strncmp("hero", git_filter_source_path(source), 4));
+
+	if (!from->size)
+		return 0;
+
+	cl_git_pass(git_buf_grow(to, from->size));
+
+	dst = (unsigned char *)to->ptr;
+
+	for (i = 0; i < from->size; i++)
+		dst[i] = VERY_SECURE_ENCRYPTION(src[i]);
+
+	to->size = from->size;
+
+	return 0;
+}
+
+static void bitflip_filter_free(git_filter *f)
+{
+	git__free(f);
+}
+
+static git_filter *create_bitflip_filter(void)
+{
+	git_filter *filter = git__calloc(1, sizeof(git_filter));
+	cl_assert(filter);
+
+	filter->version = GIT_FILTER_VERSION;
+	filter->attributes = "+bitflip";
+	filter->shutdown = bitflip_filter_free;
+	filter->apply = bitflip_filter_apply;
+
+	return filter;
+}
+
+
+static int reverse_filter_apply(
+	git_filter     *self,
+	void          **payload,
+	git_buf        *to,
+	const git_buf  *from,
+	const git_filter_source *source)
+{
+	const unsigned char *src = (const unsigned char *)from->ptr;
+	const unsigned char *end = src + from->size;
+	unsigned char *dst;
+
+	GIT_UNUSED(self); GIT_UNUSED(payload); GIT_UNUSED(source);
+
+	/* verify that attribute path match worked as expected */
+	cl_assert_equal_i(
+		0, git__strncmp("hero", git_filter_source_path(source), 4));
+
+	if (!from->size)
+		return 0;
+
+	cl_git_pass(git_buf_grow(to, from->size));
+
+	dst = (unsigned char *)to->ptr + from->size - 1;
+
+	while (src < end)
+		*dst-- = *src++;
+
+	to->size = from->size;
+
+	return 0;
+}
+
+static void reverse_filter_free(git_filter *f)
+{
+	git__free(f);
+}
+
+static git_filter *create_reverse_filter(const char *attrs)
+{
+	git_filter *filter = git__calloc(1, sizeof(git_filter));
+	cl_assert(filter);
+
+	filter->version = GIT_FILTER_VERSION;
+	filter->attributes = attrs;
+	filter->shutdown = reverse_filter_free;
+	filter->apply = reverse_filter_apply;
+
+	return filter;
+}
+
+static void register_custom_filters(void)
+{
+	static int filters_registered = 0;
+
+	if (!filters_registered) {
+		cl_git_pass(git_filter_register(
+			"bitflip", create_bitflip_filter(), BITFLIP_FILTER_PRIORITY));
+
+		cl_git_pass(git_filter_register(
+			"reverse", create_reverse_filter("+reverse"),
+			REVERSE_FILTER_PRIORITY));
+
+		/* re-register reverse filter with standard filter=xyz priority */
+		cl_git_pass(git_filter_register(
+			"pre-reverse",
+			create_reverse_filter("+prereverse"),
+			GIT_FILTER_DRIVER_PRIORITY));
+
+		filters_registered = 1;
+	}
+}
+
+
+void test_filter_custom__to_odb(void)
+{
+	git_filter_list *fl;
+	git_buf out = { 0 };
+	git_buf in = GIT_BUF_INIT_CONST(workdir_data, strlen(workdir_data));
+
+	cl_git_pass(git_filter_list_load(
+		&fl, g_repo, NULL, "herofile", GIT_FILTER_TO_ODB));
+
+	cl_git_pass(git_filter_list_apply_to_data(&out, fl, &in));
+
+	cl_assert_equal_i(BITFLIPPED_AND_REVERSED_DATA_LEN, out.size);
+
+	cl_assert_equal_i(
+		0, memcmp(bitflipped_and_reversed_data, out.ptr, out.size));
+
+	git_filter_list_free(fl);
+	git_buf_free(&out);
+}
+
+void test_filter_custom__to_workdir(void)
+{
+	git_filter_list *fl;
+	git_buf out = { 0 };
+	git_buf in = GIT_BUF_INIT_CONST(
+		bitflipped_and_reversed_data, BITFLIPPED_AND_REVERSED_DATA_LEN);
+
+	cl_git_pass(git_filter_list_load(
+		&fl, g_repo, NULL, "herofile", GIT_FILTER_TO_WORKTREE));
+
+	cl_git_pass(git_filter_list_apply_to_data(&out, fl, &in));
+
+	cl_assert_equal_i(strlen(workdir_data), out.size);
+
+	cl_assert_equal_i(
+		0, memcmp(workdir_data, out.ptr, out.size));
+
+	git_filter_list_free(fl);
+	git_buf_free(&out);
+}
+
+void test_filter_custom__can_register_a_custom_filter_in_the_repository(void)
+{
+	git_filter_list *fl;
+
+	cl_git_pass(git_filter_list_load(
+		&fl, g_repo, NULL, "herofile", GIT_FILTER_TO_WORKTREE));
+	/* expect: bitflip, reverse, crlf */
+	cl_assert_equal_sz(3, git_filter_list_length(fl));
+	git_filter_list_free(fl);
+
+	cl_git_pass(git_filter_list_load(
+		&fl, g_repo, NULL, "herocorp", GIT_FILTER_TO_WORKTREE));
+	/* expect: bitflip, reverse - possibly crlf depending on global config */
+	{
+		size_t flen = git_filter_list_length(fl);
+		cl_assert(flen == 2 || flen == 3);
+	}
+	git_filter_list_free(fl);
+
+	cl_git_pass(git_filter_list_load(
+		&fl, g_repo, NULL, "hero.bin", GIT_FILTER_TO_WORKTREE));
+	/* expect: bitflip, reverse */
+	cl_assert_equal_sz(2, git_filter_list_length(fl));
+	git_filter_list_free(fl);
+
+	cl_git_pass(git_filter_list_load(
+		&fl, g_repo, NULL, "heroflip", GIT_FILTER_TO_WORKTREE));
+	/* expect: bitflip (because of -reverse) */
+	cl_assert_equal_sz(1, git_filter_list_length(fl));
+	git_filter_list_free(fl);
+
+	cl_git_pass(git_filter_list_load(
+		&fl, g_repo, NULL, "doesntapplytome.bin", GIT_FILTER_TO_WORKTREE));
+	/* expect: none */
+	cl_assert_equal_sz(0, git_filter_list_length(fl));
+	git_filter_list_free(fl);
+}
+
+void test_filter_custom__order_dependency(void)
+{
+	git_index *index;
+	git_blob *blob;
+	git_buf buf = { 0 };
+
+	/* so if ident and reverse are used together, an interesting thing
+	 * happens - a reversed "$Id$" string is no longer going to trigger
+	 * ident correctly.  When checking out, the filters should be applied
+	 * in order CLRF, then ident, then reverse, so ident expansion should
+	 * work correctly.  On check in, the content should be reversed, then
+	 * ident, then CRLF filtered.  Let's make sure that works...
+	 */
+
+	cl_git_mkfile(
+		"empty_standard_repo/.gitattributes",
+		"hero.*.rev-ident text ident prereverse eol=lf\n");
+
+	cl_git_mkfile(
+		"empty_standard_repo/hero.1.rev-ident",
+		"This is a test\n$Id$\nHave fun!\n");
+
+	cl_git_mkfile(
+		"empty_standard_repo/hero.2.rev-ident",
+		"Another test\n$dI$\nCrazy!\n");
+
+	cl_git_pass(git_repository_index(&index, g_repo));
+	cl_git_pass(git_index_add_bypath(index, "hero.1.rev-ident"));
+	cl_git_pass(git_index_add_bypath(index, "hero.2.rev-ident"));
+	cl_repo_commit_from_index(NULL, g_repo, NULL, 0, "Filter chains\n");
+	git_index_free(index);
+
+	cl_git_pass(git_blob_lookup(&blob, g_repo,
+		& git_index_get_bypath(index, "hero.1.rev-ident", 0)->oid));
+	cl_assert_equal_s(
+		"\n!nuf evaH\n$dI$\ntset a si sihT", git_blob_rawcontent(blob));
+	cl_git_pass(git_blob_filtered_content(&buf, blob, "hero.1.rev-ident", 0));
+	/* no expansion because id was reversed at checkin and now at ident
+	 * time, reverse is not applied yet */
+	cl_assert_equal_s(
+		"This is a test\n$Id$\nHave fun!\n", buf.ptr);
+	git_blob_free(blob);
+
+	cl_git_pass(git_blob_lookup(&blob, g_repo,
+		& git_index_get_bypath(index, "hero.2.rev-ident", 0)->oid));
+	cl_assert_equal_s(
+		"\n!yzarC\n$Id$\ntset rehtonA", git_blob_rawcontent(blob));
+	cl_git_pass(git_blob_filtered_content(&buf, blob, "hero.2.rev-ident", 0));
+	/* expansion because reverse was applied at checkin and at ident time,
+	 * reverse is not applied yet */
+	cl_assert_equal_s(
+		"Another test\n$59001fe193103b1016b27027c0c827d036fd0ac8 :dI$\nCrazy!\n", buf.ptr);
+	cl_assert_equal_i(0, git_oid_strcmp(
+		git_blob_id(blob), "8ca0df630d728c0c72072b6101b301391ef10095"));
+	git_blob_free(blob);
+
+	git_buf_free(&buf);
+}
+
+void test_filter_custom__filter_registry_failure_cases(void)
+{
+	git_filter fake = { GIT_FILTER_VERSION, 0 };
+
+	cl_assert_equal_i(GIT_EEXISTS, git_filter_register("bitflip", &fake, 0));
+
+	cl_git_fail(git_filter_unregister(GIT_FILTER_CRLF));
+	cl_git_fail(git_filter_unregister(GIT_FILTER_IDENT));
+	cl_assert_equal_i(GIT_ENOTFOUND, git_filter_unregister("not-a-filter"));
+}

tests-clar/filter/ident.c

+#include "clar_libgit2.h"
+#include "git2/sys/filter.h"
+
+static git_repository *g_repo = NULL;
+
+void test_filter_ident__initialize(void)
+{
+	g_repo = cl_git_sandbox_init("crlf");
+}
+
+void test_filter_ident__cleanup(void)
+{
+	cl_git_sandbox_cleanup();
+}
+
+static void add_blob_and_filter(
+	const char *data,
+	git_filter_list *fl,
+	const char *expected)
+{
+	git_oid id;
+	git_blob *blob;
+	git_buf out = { 0 };
+
+	cl_git_mkfile("crlf/identtest", data);
+	cl_git_pass(git_blob_create_fromworkdir(&id, g_repo, "identtest"));
+	cl_git_pass(git_blob_lookup(&blob, g_repo, &id));
+
+	cl_git_pass(git_filter_list_apply_to_blob(&out, fl, blob));
+
+	cl_assert_equal_s(expected, out.ptr);
+
+	git_blob_free(blob);
+	git_buf_free(&out);
+}
+
+void test_filter_ident__to_worktree(void)
+{
+	git_filter_list *fl;
+	git_filter *ident;
+
+	cl_git_pass(git_filter_list_new(&fl, g_repo, GIT_FILTER_TO_WORKTREE));
+
+	ident = git_filter_lookup(GIT_FILTER_IDENT);
+	cl_assert(ident != NULL);
+
+	cl_git_pass(git_filter_list_push(fl, ident, NULL));
+
+	add_blob_and_filter(
+		"Hello\n$Id$\nFun stuff\n", fl,
+		"Hello\n$Id: b69e2387aafcaf73c4de5b9ab59abe27fdadee30$\nFun stuff\n");
+	add_blob_and_filter(
+		"Hello\n$Id: Junky$\nFun stuff\n", fl,
+		"Hello\n$Id: 45cd107a7102911cb2a7df08404674327fa050b9$\nFun stuff\n");
+	add_blob_and_filter(
+		"$Id$\nAt the start\n", fl,
+		"$Id: b13415c767abc196fb95bd17070e8c1113e32160$\nAt the start\n");
+	add_blob_and_filter(
+		"At the end\n$Id$", fl,
+		"At the end\n$Id: 1344925c6bc65b34c5a7b50f86bf688e48e9a272$");
+	add_blob_and_filter(
+		"$Id$", fl,
+		"$Id: b3f5ebfb5843bc43ceecff6d4f26bb37c615beb1$");
+	add_blob_and_filter(
+		"$Id: Some sort of junk goes here$", fl,
+		"$Id: ab2dd3853c7c9a4bff55aca2bea077a73c32ac06$");
+
+	add_blob_and_filter("$Id: ", fl, "$Id: ");
+	add_blob_and_filter("$Id", fl, "$Id");
+	add_blob_and_filter("$I", fl, "$I");
+	add_blob_and_filter("Id$", fl, "Id$");
+
+	git_filter_list_free(fl);
+}
+
+void test_filter_ident__to_odb(void)
+{
+	git_filter_list *fl;
+	git_filter *ident;
+
+	cl_git_pass(git_filter_list_new(&fl, g_repo, GIT_FILTER_TO_ODB));
+
+	ident = git_filter_lookup(GIT_FILTER_IDENT);
+	cl_assert(ident != NULL);
+
+	cl_git_pass(git_filter_list_push(fl, ident, NULL));
+
+	add_blob_and_filter(
+		"Hello\n$Id$\nFun stuff\n",
+		fl, "Hello\n$Id$\nFun stuff\n");
+	add_blob_and_filter(
+		"Hello\n$Id: b69e2387aafcaf73c4de5b9ab59abe27fdadee30$\nFun stuff\n",
+		fl, "Hello\n$Id$\nFun stuff\n");
+	add_blob_and_filter(
+		"Hello\n$Id: Any junk you may have left here$\nFun stuff\n",
+		fl, "Hello\n$Id$\nFun stuff\n");
+	add_blob_and_filter(
+		"Hello\n$Id:$\nFun stuff\n",
+		fl, "Hello\n$Id$\nFun stuff\n");
+	add_blob_and_filter(
+		"Hello\n$Id:x$\nFun stuff\n",
+		fl, "Hello\n$Id$\nFun stuff\n");
+
+	add_blob_and_filter(
+		"$Id$\nAt the start\n", fl, "$Id$\nAt the start\n");
+	add_blob_and_filter(
+		"$Id: lots of random text that should be removed from here$\nAt the start\n", fl, "$Id$\nAt the start\n");
+	add_blob_and_filter(
+		"$Id: lots of random text that should not be removed without a terminator\nAt the start\n", fl, "$Id: lots of random text that should not be removed without a terminator\nAt the start\n");
+
+	add_blob_and_filter(
+		"At the end\n$Id$", fl, "At the end\n$Id$");
+	add_blob_and_filter(
+		"At the end\n$Id:$", fl, "At the end\n$Id$");
+	add_blob_and_filter(
+		"At the end\n$Id:asdfasdf$", fl, "At the end\n$Id$");
+	add_blob_and_filter(
+		"At the end\n$Id", fl, "At the end\n$Id");
+	add_blob_and_filter(
+		"At the end\n$IddI", fl, "At the end\n$IddI");
+
+	add_blob_and_filter("$Id$", fl, "$Id$");
+	add_blob_and_filter("$Id: any$", fl, "$Id$");
+	add_blob_and_filter("$Id: any long stuff goes here you see$", fl, "$Id$");
+	add_blob_and_filter("$Id: ", fl, "$Id: ");
+	add_blob_and_filter("$Id", fl, "$Id");
+	add_blob_and_filter("$I", fl, "$I");
+	add_blob_and_filter("Id$", fl, "Id$");
+
+	git_filter_list_free(fl);
+}

tests-clar/index/addall.c

@@ -120,37 +120,6 @@ static void check_stat_data(git_index *index, const char *path, bool match)
 	}
 }
 
-static void commit_index_to_head(
-	git_repository *repo,
-	const char *commit_message)
-{
-	git_index *index;
-	git_oid tree_id, commit_id;
-	git_tree *tree;
-	git_signature *sig;
-	git_commit *parent = NULL;
-
-	git_revparse_single((git_object **)&parent, repo, "HEAD");
-	/* it is okay if looking up the HEAD fails */
-
-	cl_git_pass(git_repository_index(&index, repo));
-	cl_git_pass(git_index_write_tree(&tree_id, index));
-	cl_git_pass(git_index_write(index)); /* not needed, but might as well */
-	git_index_free(index);
-
-	cl_git_pass(git_tree_lookup(&tree, repo, &tree_id));
-
-	cl_git_pass(git_signature_now(&sig, "Testy McTester", "tt@tester.test"));
-
-	cl_git_pass(git_commit_create_v(
-		&commit_id, repo, "HEAD", sig, sig,
-		NULL, commit_message, tree, parent ? 1 : 0, parent));
-
-	git_commit_free(parent);
-	git_tree_free(tree);
-	git_signature_free(sig);
-}
-
 void test_index_addall__repo_lifecycle(void)
 {
 	int error;
@@ -197,7 +166,7 @@ void test_index_addall__repo_lifecycle(void)
 	check_stat_data(index, "addall/file.zzz", true);
 	check_status(g_repo, 2, 0, 0, 3, 0, 0, 1);
 
-	commit_index_to_head(g_repo, "first commit");
+	cl_repo_commit_from_index(NULL, g_repo, NULL, 0, "first commit");
 	check_status(g_repo, 0, 0, 0, 3, 0, 0, 1);
 
 	/* attempt to add an ignored file - does nothing */
@@ -244,7 +213,7 @@ void test_index_addall__repo_lifecycle(void)
 	cl_git_pass(git_index_add_bypath(index, "file.zzz"));
 	check_status(g_repo, 1, 0, 1, 3, 0, 0, 0);
 
-	commit_index_to_head(g_repo, "second commit");
+	cl_repo_commit_from_index(NULL, g_repo, NULL, 0, "second commit");
 	check_status(g_repo, 0, 0, 0, 3, 0, 0, 0);
 
 	cl_must_pass(p_unlink("addall/file.zzz"));

tests-clar/object/blob/filter.c

 #include "clar_libgit2.h"
 #include "posix.h"
 #include "blob.h"
-#include "filter.h"
 #include "buf_text.h"
 
 static git_repository *g_repo = NULL;
-#define NUM_TEST_OBJECTS 9
-static git_oid g_oids[NUM_TEST_OBJECTS];
-static const char *g_raw[NUM_TEST_OBJECTS] = {
+
+#define CRLF_NUM_TEST_OBJECTS	9
+
+static const char *g_crlf_raw[CRLF_NUM_TEST_OBJECTS] = {
 	"",
 	"foo\nbar\n",
 	"foo\rbar\r",
@@ -18,19 +18,14 @@ static const char *g_raw[NUM_TEST_OBJECTS] = {
 	"\xEF\xBB\xBF\xE3\x81\xBB\xE3\x81\x92\xE3\x81\xBB\xE3\x81\x92\r\n\xE3\x81\xBB\xE3\x81\x92\xE3\x81\xBB\xE3\x81\x92\r\n",
 	"\xFE\xFF\x00T\x00h\x00i\x00s\x00!"
 };
-static git_off_t g_len[NUM_TEST_OBJECTS] = { -1, -1, -1, -1, -1, 17, -1, -1, 12 };
-static git_buf_text_stats g_stats[NUM_TEST_OBJECTS] = {
-	{ 0, 0, 0, 0, 0, 0, 0 },
-	{ 0, 0, 0, 2, 0, 6, 0 },
-	{ 0, 0, 2, 0, 0, 6, 0 },
-	{ 0, 0, 2, 2, 2, 6, 0 },
-	{ 0, 0, 4, 4, 1, 31, 0 },
-	{ 0, 1, 1, 2, 1, 9, 5 },
-	{ GIT_BOM_UTF8, 0, 0, 1, 0, 16, 0 },
-	{ GIT_BOM_UTF8, 0, 2, 2, 2, 27, 0 },
-	{ GIT_BOM_UTF16_BE, 5, 0, 0, 0, 7, 5 },
+
+static git_off_t g_crlf_raw_len[CRLF_NUM_TEST_OBJECTS] = {
+	-1, -1, -1, -1, -1, 17, -1, -1, 12
 };
-static git_buf g_crlf_filtered[NUM_TEST_OBJECTS] = {
+
+static git_oid g_crlf_oids[CRLF_NUM_TEST_OBJECTS];
+
+static git_buf g_crlf_filtered[CRLF_NUM_TEST_OBJECTS] = {
 	{ "", 0, 0 },
 	{ "foo\nbar\n", 0, 8 },
 	{ "foo\rbar\r", 0, 8 },
@@ -42,30 +37,36 @@ static git_buf g_crlf_filtered[NUM_TEST_OBJECTS] = {
 	{ "\xFE\xFF\x00T\x00h\x00i\x00s\x00!", 0, 12 }
 };
 
+static git_buf_text_stats g_crlf_filtered_stats[CRLF_NUM_TEST_OBJECTS] = {
+	{ 0, 0, 0, 0, 0, 0, 0 },
+	{ 0, 0, 0, 2, 0, 6, 0 },
+	{ 0, 0, 2, 0, 0, 6, 0 },
+	{ 0, 0, 2, 2, 2, 6, 0 },
+	{ 0, 0, 4, 4, 1, 31, 0 },
+	{ 0, 1, 1, 2, 1, 9, 5 },
+	{ GIT_BOM_UTF8, 0, 0, 1, 0, 16, 0 },
+	{ GIT_BOM_UTF8, 0, 2, 2, 2, 27, 0 },
+	{ GIT_BOM_UTF16_BE, 5, 0, 0, 0, 7, 5 },
+};
+
 void test_object_blob_filter__initialize(void)
 {
 	int i;
 
-	cl_fixture_sandbox("empty_standard_repo");
-	cl_git_pass(p_rename(
-		"empty_standard_repo/.gitted", "empty_standard_repo/.git"));
-	cl_git_pass(git_repository_open(&g_repo, "empty_standard_repo"));
+	g_repo = cl_git_sandbox_init("empty_standard_repo");
 
-	for (i = 0; i < NUM_TEST_OBJECTS; i++) {
-		size_t len = (g_len[i] < 0) ? strlen(g_raw[i]) : (size_t)g_len[i];
-		g_len[i] = (git_off_t)len;
+	for (i = 0; i < CRLF_NUM_TEST_OBJECTS; i++) {
+		if (g_crlf_raw_len[i] < 0)
+			g_crlf_raw_len[i] = strlen(g_crlf_raw[i]);
 
-		cl_git_pass(
-			git_blob_create_frombuffer(&g_oids[i], g_repo, g_raw[i], len)
-		);
+		cl_git_pass(git_blob_create_frombuffer(
+			&g_crlf_oids[i], g_repo, g_crlf_raw[i], (size_t)g_crlf_raw_len[i]));
 	}
 }
 
 void test_object_blob_filter__cleanup(void)
 {
-	git_repository_free(g_repo);
-	g_repo = NULL;
-	cl_fixture_cleanup("empty_standard_repo");
+	cl_git_sandbox_cleanup();
 }
 
 void test_object_blob_filter__unfiltered(void)
@@ -73,10 +74,15 @@ void test_object_blob_filter__unfiltered(void)
 	int i;
 	git_blob *blob;
 
-	for (i = 0; i < NUM_TEST_OBJECTS; i++) {
-		cl_git_pass(git_blob_lookup(&blob, g_repo, &g_oids[i]));
-		cl_assert(g_len[i] == git_blob_rawsize(blob));
-		cl_assert(memcmp(git_blob_rawcontent(blob), g_raw[i], (size_t)g_len[i]) == 0);
+	for (i = 0; i < CRLF_NUM_TEST_OBJECTS; i++) {
+		size_t raw_len = (size_t)g_crlf_raw_len[i];
+
+		cl_git_pass(git_blob_lookup(&blob, g_repo, &g_crlf_oids[i]));
+
+		cl_assert_equal_sz(raw_len, (size_t)git_blob_rawsize(blob));
+		cl_assert_equal_i(
+			0, memcmp(g_crlf_raw[i], git_blob_rawcontent(blob), raw_len));
+
 		git_blob_free(blob);
 	}
 }
@@ -88,11 +94,12 @@ void test_object_blob_filter__stats(void)
 	git_buf buf = GIT_BUF_INIT;
 	git_buf_text_stats stats;
 
-	for (i = 0; i < NUM_TEST_OBJECTS; i++) {
-		cl_git_pass(git_blob_lookup(&blob, g_repo, &g_oids[i]));
+	for (i = 0; i < CRLF_NUM_TEST_OBJECTS; i++) {
+		cl_git_pass(git_blob_lookup(&blob, g_repo, &g_crlf_oids[i]));
 		cl_git_pass(git_blob__getbuf(&buf, blob));
 		git_buf_text_gather_stats(&stats, &buf, false);
-		cl_assert(memcmp(&g_stats[i], &stats, sizeof(stats)) == 0);
+		cl_assert_equal_i(
+			0, memcmp(&g_crlf_filtered_stats[i], &stats, sizeof(stats)));
 		git_blob_free(blob);
 	}
 
@@ -101,11 +108,11 @@ void test_object_blob_filter__stats(void)
 
 void test_object_blob_filter__to_odb(void)
 {
-	git_vector filters = GIT_VECTOR_INIT;
+	git_filter_list *fl = NULL;
 	git_config *cfg;
 	int i;
 	git_blob *blob;
-	git_buf orig = GIT_BUF_INIT, out = GIT_BUF_INIT;
+	git_buf out = GIT_BUF_INIT;
 
 	cl_git_pass(git_repository_config(&cfg, g_repo));
 	cl_assert(cfg);
@@ -113,23 +120,24 @@ void test_object_blob_filter__to_odb(void)
 	git_attr_cache_flush(g_repo);
 	cl_git_append2file("empty_standard_repo/.gitattributes", "*.txt text\n");
 
-	cl_assert(git_filters_load(
-		&filters, g_repo, "filename.txt", GIT_FILTER_TO_ODB) > 0);
-	cl_assert(filters.length == 1);
+	cl_git_pass(git_filter_list_load(
+		&fl, g_repo, NULL, "filename.txt", GIT_FILTER_TO_ODB));
+	cl_assert(fl != NULL);
 
-	for (i = 0; i < NUM_TEST_OBJECTS; i++) {
-		cl_git_pass(git_blob_lookup(&blob, g_repo, &g_oids[i]));
-		cl_git_pass(git_blob__getbuf(&orig, blob));
+	for (i = 0; i < CRLF_NUM_TEST_OBJECTS; i++) {
+		cl_git_pass(git_blob_lookup(&blob, g_repo, &g_crlf_oids[i]));
 
-		cl_git_pass(git_filters_apply(&out, &orig, &filters));
-		cl_assert(git_buf_cmp(&out, &g_crlf_filtered[i]) == 0);
+		cl_git_pass(git_filter_list_apply_to_blob(&out, fl, blob));
+
+		cl_assert_equal_sz(g_crlf_filtered[i].size, out.size);
+
+		cl_assert_equal_i(
+			0, memcmp(out.ptr, g_crlf_filtered[i].ptr, out.size));
 
 		git_blob_free(blob);
 	}
 
-	git_filters_free(&filters);
-	git_buf_free(&orig);
+	git_filter_list_free(fl);
 	git_buf_free(&out);
 	git_config_free(cfg);
 }
-

tests-clar/repo/init.c

@@ -565,6 +565,11 @@ void test_repo_init__init_with_initial_commit(void)
 	cl_git_pass(git_index_add_bypath(index, "file.txt"));
 	cl_git_pass(git_index_write(index));
 
+	/* Intentionally not using cl_repo_commit_from_index here so this code
+	 * can be used as an example of how an initial commit is typically
+	 * made to a repository...
+	 */
+
 	/* Make sure we're ready to use git_signature_default :-) */
 	{
 		git_config *cfg, *local;

tests-clar/revwalk/simplify.c

 #include "clar_libgit2.h"
 
+void test_revwalk_simplify__cleanup(void)
+{
+	cl_git_sandbox_cleanup();
+}
+
 /*
 	*   a4a7dce [0] Merge branch 'master' into br2
 	|\
@@ -47,5 +52,4 @@ void test_revwalk_simplify__first_parent(void)
 	cl_assert_equal_i(error, GIT_ITEROVER);
 
 	git_revwalk_free(walk);
-	git_repository_free(repo);
 }

tests-clar/stash/drop.c

@@ -36,25 +36,27 @@ static void push_three_states(void)
 	cl_git_mkfile("stash/zero.txt", "content\n");
 	cl_git_pass(git_repository_index(&index, repo));
 	cl_git_pass(git_index_add_bypath(index, "zero.txt"));
-	commit_staged_files(&oid, index, signature);
+	cl_repo_commit_from_index(NULL, repo, signature, 0, "Initial commit");
 	cl_assert(git_path_exists("stash/zero.txt"));
+	git_index_free(index);
 
 	cl_git_mkfile("stash/one.txt", "content\n");
-	cl_git_pass(git_stash_save(&oid, repo, signature, "First", GIT_STASH_INCLUDE_UNTRACKED));
+	cl_git_pass(git_stash_save(
+		&oid, repo, signature, "First", GIT_STASH_INCLUDE_UNTRACKED));
 	cl_assert(!git_path_exists("stash/one.txt"));
 	cl_assert(git_path_exists("stash/zero.txt"));
 
 	cl_git_mkfile("stash/two.txt", "content\n");
-	cl_git_pass(git_stash_save(&oid, repo, signature, "Second", GIT_STASH_INCLUDE_UNTRACKED));
+	cl_git_pass(git_stash_save(
+		&oid, repo, signature, "Second", GIT_STASH_INCLUDE_UNTRACKED));
 	cl_assert(!git_path_exists("stash/two.txt"));
 	cl_assert(git_path_exists("stash/zero.txt"));
 
 	cl_git_mkfile("stash/three.txt", "content\n");
-	cl_git_pass(git_stash_save(&oid, repo, signature, "Third", GIT_STASH_INCLUDE_UNTRACKED));
+	cl_git_pass(git_stash_save(
+		&oid, repo, signature, "Third", GIT_STASH_INCLUDE_UNTRACKED));
 	cl_assert(!git_path_exists("stash/three.txt"));
 	cl_assert(git_path_exists("stash/zero.txt"));
-
-	git_index_free(index);
 }
 
 void test_stash_drop__cannot_drop_a_non_existing_stashed_state(void)
@@ -160,7 +162,7 @@ void test_stash_drop__dropping_the_top_stash_updates_the_stash_reference(void)
 	retrieve_top_stash_id(&oid);
 
 	cl_git_pass(git_revparse_single(&next_top_stash, repo, "stash@{1}"));
-	cl_assert_equal_i(false, git_oid_cmp(&oid, git_object_id(next_top_stash)) == 0);
+	cl_assert(git_oid_cmp(&oid, git_object_id(next_top_stash)) != 0);
 
 	cl_git_pass(git_stash_drop(repo, 0));
 

tests-clar/stash/save.c

@@ -241,7 +241,7 @@ void test_stash_save__stashing_updates_the_reflog(void)
 void test_stash_save__cannot_stash_when_there_are_no_local_change(void)
 {
 	git_index *index;
-	git_oid commit_oid, stash_tip_oid;
+	git_oid stash_tip_oid;
 
 	cl_git_pass(git_repository_index(&index, repo));
 
@@ -251,8 +251,7 @@ void test_stash_save__cannot_stash_when_there_are_no_local_change(void)
 	 */
 	cl_git_pass(git_index_add_bypath(index, "what"));
 	cl_git_pass(git_index_add_bypath(index, "who"));
-	cl_git_pass(git_index_write(index));
-	commit_staged_files(&commit_oid, index, signature);
+	cl_repo_commit_from_index(NULL, repo, signature, 0, "Initial commit");
 	git_index_free(index);
 
 	cl_assert_equal_i(GIT_ENOTFOUND,

tests-clar/stash/stash_helpers.c

@@ -2,38 +2,8 @@
 #include "fileops.h"
 #include "stash_helpers.h"
 
-void commit_staged_files(
-	git_oid *commit_oid,
-	git_index *index,
-	git_signature *signature)
-{
-	git_tree *tree;
-	git_oid tree_oid;
-	git_repository *repo;
-
-	repo = git_index_owner(index);
-
-	cl_git_pass(git_index_write_tree(&tree_oid, index));
-
-	cl_git_pass(git_tree_lookup(&tree, repo, &tree_oid));
-
-	cl_git_pass(git_commit_create_v(
-		commit_oid,
-		repo,
-		"HEAD",
-		signature,
-		signature,
-		NULL,
-		"Initial commit",
-		tree,
-		0));
-
-	git_tree_free(tree);
-}
-
 void setup_stash(git_repository *repo, git_signature *signature)
 {
-	git_oid commit_oid;
 	git_index *index;
 
 	cl_git_pass(git_repository_index(&index, repo));
@@ -50,9 +20,8 @@ void setup_stash(git_repository *repo, git_signature *signature)
 	cl_git_pass(git_index_add_bypath(index, "how"));
 	cl_git_pass(git_index_add_bypath(index, "who"));
 	cl_git_pass(git_index_add_bypath(index, ".gitignore"));
-	cl_git_pass(git_index_write(index));
 
-	commit_staged_files(&commit_oid, index, signature);
+	cl_repo_commit_from_index(NULL, repo, signature, 0, "Initial commit");
 
 	cl_git_rewritefile("stash/what", "goodbye\n");			/* dd7e1c6f0fefe118f0b63d9f10908c460aa317a6 */
 	cl_git_rewritefile("stash/how", "not so small and\n");	/* e6d64adb2c7f3eb8feb493b556cc8070dca379a3 */

tests-clar/stash/stash_helpers.h

 void setup_stash(
 	git_repository *repo,
 	git_signature *signature);
-
-void commit_staged_files(
-	git_oid *commit_oid,
-	git_index *index,
-	git_signature *signature);
\ No newline at end of file

tests-clar/status/renames.c

@@ -11,6 +11,8 @@ static git_repository *g_repo = NULL;
 void test_status_renames__initialize(void)
 {
 	g_repo = cl_git_sandbox_init("renames");
+
+	cl_repo_set_bool(g_repo, "core.autocrlf", false);
 }
 
 void test_status_renames__cleanup(void)
@@ -67,7 +69,7 @@ static void test_status(
 		actual = git_status_byindex(status_list, i);
 		expected = &expected_list[i];
 
-		cl_assert_equal_i((int)expected->status, (int)actual->status);
+		cl_assert_equal_i_fmt(expected->status, actual->status, "%04x");
 
 		oldname = actual->head_to_index ? actual->head_to_index->old_file.path :
 			actual->index_to_workdir ? actual->index_to_workdir->old_file.path : NULL;

tests-clar/status/submodules.c

@@ -13,7 +13,6 @@ void test_status_submodules__initialize(void)
 
 void test_status_submodules__cleanup(void)
 {
-	cleanup_fixture_submodules();
 }
 
 void test_status_submodules__api(void)

tests-clar/status/worktree.c

@@ -632,35 +632,12 @@ void test_status_worktree__conflicted_item(void)
 
 static void stage_and_commit(git_repository *repo, const char *path)
 {
-	git_oid tree_oid, commit_oid;
-	git_tree *tree;
-	git_signature *signature;
 	git_index *index;
 
 	cl_git_pass(git_repository_index(&index, repo));
 	cl_git_pass(git_index_add_bypath(index, path));
-	cl_git_pass(git_index_write(index));
-
-	cl_git_pass(git_index_write_tree(&tree_oid, index));
+	cl_repo_commit_from_index(NULL, repo, NULL, 1323847743, "Initial commit\n");
 	git_index_free(index);
-
-	cl_git_pass(git_tree_lookup(&tree, repo, &tree_oid));
-
-	cl_git_pass(git_signature_new(&signature, "nulltoken", "emeric.fermas@gmail.com", 1323847743, 60));
-
-	cl_git_pass(git_commit_create_v(
-		&commit_oid,
-		repo,
-		"HEAD",
-		signature,
-		signature,
-		NULL,
-		"Initial commit\n\0",
-		tree,
-		0));
-
-	git_tree_free(tree);
-	git_signature_free(signature);
 }
 
 static void assert_ignore_case(

tests-clar/stress/diff.c

@@ -54,27 +54,9 @@ static void test_with_many(int expected_new)
 
 	git_diff_list_free(diff);
 
-	{
-		git_object *parent;
-		git_signature *sig;
-		git_oid tree_id, commit_id;
-		git_reference *ref;
-
-		cl_git_pass(git_index_write_tree(&tree_id, index));
-		cl_git_pass(git_tree_lookup(&new_tree, g_repo, &tree_id));
-
-		cl_git_pass(git_revparse_ext(&parent, &ref, g_repo, "HEAD"));
-		cl_git_pass(git_signature_new(
-			&sig, "Sm Test", "sm@tester.test", 1372350000, 480));
-
-		cl_git_pass(git_commit_create_v(
-			&commit_id, g_repo, git_reference_name(ref), sig, sig,
-			NULL, "yoyoyo", new_tree, 1, parent));
-
-		git_object_free(parent);
-		git_reference_free(ref);
-		git_signature_free(sig);
-	}
+	cl_repo_commit_from_index(NULL, g_repo, NULL, 1372350000, "yoyoyo");
+	cl_git_pass(git_revparse_single(
+		(git_object **)&new_tree, g_repo, "HEAD^{tree}"));
 
 	cl_git_pass(git_diff_tree_to_tree(
 		&diff, g_repo, tree, new_tree, &diffopts));

tests-clar/submodule/status.c

@@ -14,7 +14,6 @@ void test_submodule_status__initialize(void)
 
 void test_submodule_status__cleanup(void)
 {
-	cleanup_fixture_submodules();
 }
 
 void test_submodule_status__unchanged(void)

tests-clar/submodule/submodule_helpers.c

@@ -83,6 +83,14 @@ void rewrite_gitmodules(const char *workdir)
 	git_buf_free(&path);
 }
 
+static void cleanup_fixture_submodules(void *payload)
+{
+	cl_git_sandbox_cleanup(); /* either "submodules" or "submod2" */
+
+	if (payload)
+		cl_fixture_cleanup(payload);
+}
+
 git_repository *setup_fixture_submodules(void)
 {
 	git_repository *repo = cl_git_sandbox_init("submodules");
@@ -92,6 +100,8 @@ git_repository *setup_fixture_submodules(void)
 	rewrite_gitmodules(git_repository_workdir(repo));
 	p_rename("submodules/testrepo/.gitted", "submodules/testrepo/.git");
 
+	cl_set_cleanup(cleanup_fixture_submodules, "testrepo.git");
+
 	return repo;
 }
 
@@ -106,14 +116,7 @@ git_repository *setup_fixture_submod2(void)
 	p_rename("submod2/not-submodule/.gitted", "submod2/not-submodule/.git");
 	p_rename("submod2/not/.gitted", "submod2/not/.git");
 
-	return repo;
-}
+	cl_set_cleanup(cleanup_fixture_submodules, "submod2_target");
 
-void cleanup_fixture_submodules(void)
-{
-	cl_git_sandbox_cleanup();
-
-	/* just try to clean up both possible extras */
-	cl_fixture_cleanup("testrepo.git");
-	cl_fixture_cleanup("submod2_target");
+	return repo;
 }

tests-clar/submodule/submodule_helpers.h

 extern void rewrite_gitmodules(const char *workdir);
 
+/* these will automatically set a cleanup callback */
 extern git_repository *setup_fixture_submodules(void);
 extern git_repository *setup_fixture_submod2(void);
-extern void cleanup_fixture_submodules(void);