Add ident filter

This adds the ident filter (that knows how to replace $Id$) and
tweaks the filter APIs and code so that git_filter_source objects
actually have the updated OID of the object being filtered when
it is a known value.

include/git2/filter.h

@@ -42,11 +42,13 @@ typedef enum {
  * file data.  Libgit2 includes one built in filter and it is possible to
  * write your own (see git2/sys/filter.h for information on that).
  *
- * The built in filter is:
+ * The two builtin filters are:
  *
  * * "crlf" which uses the complex rules with the "text", "eol", and
  *   "crlf" file attributes to decide how to convert between LF and CRLF
  *   line endings
+ * * "ident" which replaces "$Id$" in a blob with "$Id: <blob OID>$" upon
+ *   checkout and replaced "$Id: <anything>$" with "$Id$" on checkin.
  */
 typedef struct git_filter git_filter;
 
@@ -70,6 +72,7 @@ typedef struct git_filter_list git_filter_list;
  *
  * @param filters Output newly created git_filter_list (or NULL)
  * @param repo Repository object that contains `path`
+ * @param blob The blob to which the filter will be applied (if known)
  * @param path Relative path of the file to be filtered
  * @param mode Filtering direction (WT->ODB or ODB->WT)
  * @return 0 on success (which could still return NULL if no filters are
@@ -78,6 +81,7 @@ typedef struct git_filter_list git_filter_list;
 GIT_EXTERN(int) git_filter_list_load(
 	git_filter_list **filters,
 	git_repository *repo,
+	git_blob *blob, /* can be NULL */
 	const char *path,
 	git_filter_mode_t mode);
 

include/git2/sys/filter.h

@@ -26,7 +26,11 @@ GIT_BEGIN_DECL
  */
 GIT_EXTERN(git_filter *) git_filter_lookup(const char *name);
 
-#define GIT_FILTER_CRLF "crlf"
+#define GIT_FILTER_CRLF  "crlf"
+#define GIT_FILTER_IDENT "ident"
+
+#define GIT_FILTER_CRLF_PRIORITY 0
+#define GIT_FILTER_IDENT_PRIORITY 100
 
 /**
  * Create a new empty filter list
@@ -199,8 +203,9 @@ struct git_filter {
  * issued in order of `priority` on smudge (to workdir), and in reverse
  * order of `priority` on clean (to odb).
  *
- * One filter will be preregistered with libgit2:
- * - GIT_FILTER_CRLF with priority of 0.
+ * Two filters are preregistered with libgit2:
+ * - GIT_FILTER_CRLF with priority 0
+ * - GIT_FILTER_IDENT with priority 100
  *
  * Currently the filter registry is not thread safe, so any registering or
  * deregistering of filters must be done outside of any possible usage of

src/blob.c

@@ -195,7 +195,7 @@ int git_blob__create_from_paths(
 		if (try_load_filters)
 			/* Load the filters for writing this file to the ODB */
 			error = git_filter_list_load(
-				&fl, repo, hint_path, GIT_FILTER_TO_ODB);
+				&fl, repo, NULL, hint_path, GIT_FILTER_TO_ODB);
 
 		if (error < 0)
 			/* well, that didn't work */;
@@ -331,19 +331,19 @@ int git_blob_is_binary(git_blob *blob)
 int git_blob_filtered_content(
 	git_buffer *out,
 	git_blob *blob,
-	const char *as_path,
+	const char *path,
 	int check_for_binary_data)
 {
 	int error = 0;
 	git_filter_list *fl = NULL;
 
-	assert(blob && as_path && out);
+	assert(blob && path && out);
 
 	if (check_for_binary_data && git_blob_is_binary(blob))
 		return 0;
 
 	if (!(error = git_filter_list_load(
-			&fl, git_blob_owner(blob), as_path, GIT_FILTER_TO_WORKTREE))) {
+			&fl, git_blob_owner(blob), blob, path, GIT_FILTER_TO_WORKTREE))) {
 
 		error = git_filter_list_apply_to_blob(out, fl, blob);
 

src/checkout.c

@@ -718,7 +718,7 @@ static int blob_content_to_file(
 
 	if (!opts->disable_filters && !git_blob_is_binary(blob))
 		error = git_filter_list_load(
-			&fl, git_blob_owner(blob), path, GIT_FILTER_TO_WORKTREE);
+			&fl, git_blob_owner(blob), blob, path, GIT_FILTER_TO_WORKTREE);
 
 	if (!error)
 		error = git_filter_list_apply_to_blob(&out, fl, blob);

src/crlf.c

@@ -324,11 +324,6 @@ static void crlf_cleanup(
 	git__free(payload);
 }
 
-static void crlf_shutdown(git_filter *self)
-{
-	git__free(self);
-}
-
 git_filter *git_crlf_filter_new(void)
 {
 	struct crlf_filter *f = git__calloc(1, sizeof(struct crlf_filter));
@@ -336,7 +331,7 @@ git_filter *git_crlf_filter_new(void)
 	f->f.version = GIT_FILTER_VERSION;
 	f->f.attributes = "crlf eol text";
 	f->f.initialize = NULL;
-	f->f.shutdown = crlf_shutdown;
+	f->f.shutdown = git_filter_free;
 	f->f.check    = crlf_check;
 	f->f.apply    = crlf_apply;
 	f->f.cleanup  = crlf_cleanup;

src/diff.c

@@ -570,7 +570,7 @@ int git_diff__oid_for_file(
 	} else {
 		git_filter_list *fl = NULL;
 
-		result = git_filter_list_load(&fl, repo, path, GIT_FILTER_TO_ODB);
+		result = git_filter_list_load(&fl, repo, NULL, path, GIT_FILTER_TO_ODB);
 		if (!result) {
 			int fd = git_futils_open_ro(full_path.ptr);
 			if (fd < 0)

src/diff_file.c

@@ -311,7 +311,7 @@ static int diff_file_content_load_workdir_file(
 		goto cleanup;
 
 	if ((error = git_filter_list_load(
-			&fl, fc->repo, fc->file->path, GIT_FILTER_TO_ODB)) < 0)
+			&fl, fc->repo, NULL, fc->file->path, GIT_FILTER_TO_ODB)) < 0)
 		goto cleanup;
 
 	/* if there are no filters, try to mmap the file */

src/filter.c

@@ -103,7 +103,23 @@ static int filter_registry_initialize(void)
 
 	git__on_shutdown(filter_registry_shutdown);
 
-	return git_filter_register(GIT_FILTER_CRLF, git_crlf_filter_new(), 0);
+	/* try to register both default filters */
+	{
+		git_filter *crlf = git_crlf_filter_new();
+		git_filter *ident = git_ident_filter_new();
+
+		if (crlf && git_filter_register(
+				GIT_FILTER_CRLF, crlf, GIT_FILTER_CRLF_PRIORITY) < 0)
+			crlf = NULL;
+		if (ident && git_filter_register(
+				GIT_FILTER_IDENT, ident, GIT_FILTER_IDENT_PRIORITY) < 0)
+			ident = NULL;
+
+		if (!crlf || !ident)
+			return -1;
+	}
+
+	return 0;
 
 cleanup:
 	git_vector_free(&reg->filters);
@@ -132,7 +148,7 @@ static int filter_def_scan_attrs(
 
 		if (scan > start) {
 			(*nattr)++;
-			if (has_eq || *scan == '-' || *scan == '+' || *scan == '!')
+			if (has_eq || *start == '-' || *start == '+' || *start == '!')
 				(*nmatch)++;
 
 			if (has_eq)
@@ -312,6 +328,11 @@ git_filter *git_filter_lookup(const char *name)
 	return fdef->filter;
 }
 
+void git_filter_free(git_filter *filter)
+{
+	git__free(filter);
+}
+
 git_repository *git_filter_source_repo(const git_filter_source *src)
 {
 	return src->repo;
@@ -410,6 +431,7 @@ int git_filter_list_new(
 int git_filter_list_load(
 	git_filter_list **filters,
 	git_repository *repo,
+	git_blob *blob, /* can be NULL */
 	const char *path,
 	git_filter_mode_t mode)
 {
@@ -426,6 +448,8 @@ int git_filter_list_load(
 	src.repo = repo;
 	src.path = path;
 	src.mode = mode;
+	if (blob)
+		git_oid_cpy(&src.oid, git_blob_id(blob));
 
 	git_vector_foreach(&git__filter_registry->filters, idx, fdef) {
 		const char **values = NULL;
@@ -630,5 +654,8 @@ int git_filter_list_apply_to_blob(
 		(char *)git_blob_rawcontent(blob), git_blob_rawsize(blob), 0
 	};
 
+	if (filters)
+		git_oid_cpy(&filters->source.oid, git_blob_id(blob));
+
 	return git_filter_list_apply_to_data(out, filters, &in);
 }

src/filter.h

@@ -19,10 +19,13 @@ typedef enum {
 	GIT_CRLF_AUTO,
 } git_crlf_t;
 
+extern void git_filter_free(git_filter *filter);
+
 /*
  * Available filters
  */
 
 extern git_filter *git_crlf_filter_new(void);
+extern git_filter *git_ident_filter_new(void);
 
 #endif

src/ident.c

+/*
+ * Copyright (C) the libgit2 contributors. All rights reserved.
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+
+#include "git2/sys/filter.h"
+#include "filter.h"
+#include "buffer.h"
+
+int ident_find_id(
+	const char **id_start, const char **id_end, const char *start, size_t len)
+{
+	const char *found;
+
+	while (len > 0 && (found = memchr(start, '$', len)) != NULL) {
+		size_t remaining = len - (size_t)(found - start);
+		if (remaining < 3)
+			return GIT_ENOTFOUND;
+		if (found[1] == 'I' && found[2] == 'd')
+			break;
+		start = found + 1;
+		len = remaining - 1;
+	}
+
+	if (len < 3)
+		return GIT_ENOTFOUND;
+	*id_start = found;
+
+	if ((found = memchr(found + 3, '$', len - 3)) == NULL)
+		return GIT_ENOTFOUND;
+
+	*id_end = found + 1;
+	return 0;
+}
+
+static int ident_insert_id(
+	git_buffer *to, const git_buffer *from, const git_filter_source *src)
+{
+	char oid[GIT_OID_HEXSZ+1];
+	const char *id_start, *id_end, *from_end = from->ptr + from->size;
+	size_t need_size;
+	git_buf to_buf = GIT_BUF_FROM_BUFFER(to);
+
+	/* replace $Id$ with blob id */
+
+	if (!git_filter_source_id(src))
+		return GIT_ENOTFOUND;
+
+	git_oid_tostr(oid, sizeof(oid), git_filter_source_id(src));
+
+	if (ident_find_id(&id_start, &id_end, from->ptr, from->size) < 0)
+		return GIT_ENOTFOUND;
+
+	need_size = (size_t)(id_start - from->ptr) +
+		5 /* "$Id: " */ + GIT_OID_HEXSZ + 1 /* "$" */ +
+		(size_t)(from_end - id_end);
+
+	if (git_buf_grow(&to_buf, need_size) < 0)
+		return -1;
+
+	git_buf_set(&to_buf, from->ptr, (size_t)(id_start - from->ptr));
+	git_buf_put(&to_buf, "$Id: ", 5);
+	git_buf_put(&to_buf, oid, GIT_OID_HEXSZ);
+	git_buf_putc(&to_buf, '$');
+	git_buf_put(&to_buf, id_end, (size_t)(from_end - id_end));
+
+	if (git_buf_oom(&to_buf))
+		return -1;
+
+	git_buffer_from_buf(to, &to_buf);
+	return 0;
+}
+
+static int ident_remove_id(
+	git_buffer *to, const git_buffer *from)
+{
+	const char *id_start, *id_end, *from_end = from->ptr + from->size;
+	size_t need_size;
+	git_buf to_buf = GIT_BUF_FROM_BUFFER(to);
+
+	if (ident_find_id(&id_start, &id_end, from->ptr, from->size) < 0)
+		return GIT_ENOTFOUND;
+
+	need_size = (size_t)(id_start - from->ptr) +
+		4 /* "$Id$" */ + (size_t)(from_end - id_end);
+
+	if (git_buf_grow(&to_buf, need_size) < 0)
+		return -1;
+
+	git_buf_set(&to_buf, from->ptr, (size_t)(id_start - from->ptr));
+	git_buf_put(&to_buf, "$Id$", 4);
+	git_buf_put(&to_buf, id_end, (size_t)(from_end - id_end));
+
+	if (git_buf_oom(&to_buf))
+		return -1;
+
+	git_buffer_from_buf(to, &to_buf);
+	return 0;
+}
+
+static int ident_apply(
+	git_filter        *self,
+	void              **payload,
+	git_buffer        *to,
+	const git_buffer  *from,
+	const git_filter_source *src)
+{
+	GIT_UNUSED(self); GIT_UNUSED(payload);
+
+	if (git_filter_source_mode(src) == GIT_FILTER_SMUDGE)
+		return ident_insert_id(to, from, src);
+	else
+		return ident_remove_id(to, from);
+}
+
+git_filter *git_ident_filter_new(void)
+{
+	git_filter *f = git__calloc(1, sizeof(git_filter));
+
+	f->version = GIT_FILTER_VERSION;
+	f->attributes = "+ident"; /* apply to files with ident attribute set */
+	f->shutdown = git_filter_free;
+	f->apply    = ident_apply;
+
+	return f;
+}

src/repository.c

@@ -1671,7 +1671,8 @@ int git_repository_hashfile(
 
 	/* passing empty string for "as_path" indicated --no-filters */
 	if (strlen(as_path) > 0) {
-		error = git_filter_list_load(&fl, repo, as_path, GIT_FILTER_TO_ODB);
+		error = git_filter_list_load(
+			&fl, repo, NULL, as_path, GIT_FILTER_TO_ODB);
 		if (error < 0)
 			return error;
 	} else {

tests-clar/filter/blob.c

@@ -7,7 +7,12 @@ void test_filter_blob__initialize(void)
 {
 	g_repo = cl_git_sandbox_init("crlf");
 	cl_git_mkfile("crlf/.gitattributes",
-		"*.txt text\n*.bin binary\n*.crlf text eol=crlf\n*.lf text eol=lf\n");
+		"*.txt text\n*.bin binary\n"
+		"*.crlf text eol=crlf\n"
+		"*.lf text eol=lf\n"
+		"*.ident text ident\n"
+		"*.identcrlf ident text eol=crlf\n"
+		"*.identlf ident text eol.lf\n");
 }
 
 void test_filter_blob__cleanup(void)
@@ -41,3 +46,36 @@ void test_filter_blob__all_crlf(void)
 	git_buffer_free(&buf);
 	git_blob_free(blob);
 }
+
+void test_filter_blob__ident(void)
+{
+	git_oid id;
+	git_blob *blob;
+	git_buffer buf = GIT_BUFFER_INIT;
+
+	cl_git_mkfile("crlf/test.ident", "Some text\n$Id$\nGoes there\n");
+	cl_git_pass(git_blob_create_fromworkdir(&id, g_repo, "test.ident"));
+	cl_git_pass(git_blob_lookup(&blob, g_repo, &id));
+	cl_assert_equal_s(
+		"Some text\n$Id$\nGoes there\n", git_blob_rawcontent(blob));
+	git_blob_free(blob);
+
+	cl_git_mkfile("crlf/test.ident", "Some text\n$Id: Any old just you want$\nGoes there\n");
+	cl_git_pass(git_blob_create_fromworkdir(&id, g_repo, "test.ident"));
+	cl_git_pass(git_blob_lookup(&blob, g_repo, &id));
+	cl_assert_equal_s(
+		"Some text\n$Id$\nGoes there\n", git_blob_rawcontent(blob));
+
+	cl_git_pass(git_blob_filtered_content(&buf, blob, "filter.bin", 1));
+	cl_assert_equal_s(
+		"Some text\n$Id$\nGoes there\n", buf.ptr);
+
+	cl_git_pass(git_blob_filtered_content(&buf, blob, "filter.identcrlf", 1));
+	cl_assert_equal_s(
+		"Some text\r\n$Id: 3164f585d548ac68027d22b104f2d8100b2b6845$\r\nGoes there\r\n", buf.ptr);
+
+	cl_git_pass(git_blob_filtered_content(&buf, blob, "filter.identlf", 1));
+	cl_assert_equal_s(
+		"Some text\n$Id: 3164f585d548ac68027d22b104f2d8100b2b6845$\nGoes there\n", buf.ptr);
+
+}

tests-clar/filter/ident.c

+#include "clar_libgit2.h"
+#include "git2/sys/filter.h"
+
+static git_repository *g_repo = NULL;
+
+void test_filter_ident__initialize(void)
+{
+	g_repo = cl_git_sandbox_init("crlf");
+}
+
+void test_filter_ident__cleanup(void)
+{
+	cl_git_sandbox_cleanup();
+}
+
+static void add_blob_and_filter(
+	const char *data,
+	git_filter_list *fl,
+	const char *expected)
+{
+	git_oid id;
+	git_blob *blob;
+	git_buffer out = GIT_BUFFER_INIT;
+
+	cl_git_mkfile("crlf/identtest", data);
+	cl_git_pass(git_blob_create_fromworkdir(&id, g_repo, "identtest"));
+	cl_git_pass(git_blob_lookup(&blob, g_repo, &id));
+
+	cl_git_pass(git_filter_list_apply_to_blob(&out, fl, blob));
+
+	cl_assert_equal_s(expected, out.ptr);
+
+	git_blob_free(blob);
+	git_buffer_free(&out);
+}
+
+void test_filter_ident__to_worktree(void)
+{
+	git_filter_list *fl;
+	git_filter *ident;
+
+	cl_git_pass(git_filter_list_new(&fl, g_repo, GIT_FILTER_TO_WORKTREE));
+
+	ident = git_filter_lookup(GIT_FILTER_IDENT);
+	cl_assert(ident != NULL);
+
+	cl_git_pass(git_filter_list_push(fl, ident, NULL));
+
+	add_blob_and_filter(
+		"Hello\n$Id$\nFun stuff\n", fl,
+		"Hello\n$Id: b69e2387aafcaf73c4de5b9ab59abe27fdadee30$\nFun stuff\n");
+	add_blob_and_filter(
+		"Hello\n$Id: Junky$\nFun stuff\n", fl,
+		"Hello\n$Id: 45cd107a7102911cb2a7df08404674327fa050b9$\nFun stuff\n");
+	add_blob_and_filter(
+		"$Id$\nAt the start\n", fl,
+		"$Id: b13415c767abc196fb95bd17070e8c1113e32160$\nAt the start\n");
+	add_blob_and_filter(
+		"At the end\n$Id$", fl,
+		"At the end\n$Id: 1344925c6bc65b34c5a7b50f86bf688e48e9a272$");
+	add_blob_and_filter(
+		"$Id$", fl,
+		"$Id: b3f5ebfb5843bc43ceecff6d4f26bb37c615beb1$");
+	add_blob_and_filter(
+		"$Id: Some sort of junk goes here$", fl,
+		"$Id: ab2dd3853c7c9a4bff55aca2bea077a73c32ac06$");
+
+	add_blob_and_filter("$Id: ", fl, "$Id: ");
+	add_blob_and_filter("$Id", fl, "$Id");
+	add_blob_and_filter("$I", fl, "$I");
+	add_blob_and_filter("Id$", fl, "Id$");
+
+	git_filter_list_free(fl);
+}
+
+void test_filter_ident__to_odb(void)
+{
+	git_filter_list *fl;
+	git_filter *ident;
+
+	cl_git_pass(git_filter_list_new(&fl, g_repo, GIT_FILTER_TO_ODB));
+
+	ident = git_filter_lookup(GIT_FILTER_IDENT);
+	cl_assert(ident != NULL);
+
+	cl_git_pass(git_filter_list_push(fl, ident, NULL));
+
+	add_blob_and_filter(
+		"Hello\n$Id$\nFun stuff\n",
+		fl, "Hello\n$Id$\nFun stuff\n");
+	add_blob_and_filter(
+		"Hello\n$Id: b69e2387aafcaf73c4de5b9ab59abe27fdadee30$\nFun stuff\n",
+		fl, "Hello\n$Id$\nFun stuff\n");
+	add_blob_and_filter(
+		"Hello\n$Id: Any junk you may have left here$\nFun stuff\n",
+		fl, "Hello\n$Id$\nFun stuff\n");
+	add_blob_and_filter(
+		"Hello\n$Id:$\nFun stuff\n",
+		fl, "Hello\n$Id$\nFun stuff\n");
+	add_blob_and_filter(
+		"Hello\n$Id:x$\nFun stuff\n",
+		fl, "Hello\n$Id$\nFun stuff\n");
+
+	add_blob_and_filter(
+		"$Id$\nAt the start\n", fl, "$Id$\nAt the start\n");
+	add_blob_and_filter(
+		"$Id: lots of random text that should be removed from here$\nAt the start\n", fl, "$Id$\nAt the start\n");
+	add_blob_and_filter(
+		"$Id: lots of random text that should not be removed without a terminator\nAt the start\n", fl, "$Id: lots of random text that should not be removed without a terminator\nAt the start\n");
+
+	add_blob_and_filter(
+		"At the end\n$Id$", fl, "At the end\n$Id$");
+	add_blob_and_filter(
+		"At the end\n$Id:$", fl, "At the end\n$Id$");
+	add_blob_and_filter(
+		"At the end\n$Id:asdfasdf$", fl, "At the end\n$Id$");
+	add_blob_and_filter(
+		"At the end\n$Id", fl, "At the end\n$Id");
+	add_blob_and_filter(
+		"At the end\n$IddI", fl, "At the end\n$IddI");
+
+	add_blob_and_filter("$Id$", fl, "$Id$");
+	add_blob_and_filter("$Id: any$", fl, "$Id$");
+	add_blob_and_filter("$Id: any long stuff goes here you see$", fl, "$Id$");
+	add_blob_and_filter("$Id: ", fl, "$Id: ");
+	add_blob_and_filter("$Id", fl, "$Id");
+	add_blob_and_filter("$I", fl, "$I");
+	add_blob_and_filter("Id$", fl, "Id$");
+
+	git_filter_list_free(fl);
+}

tests-clar/object/blob/filter.c

@@ -112,8 +112,8 @@ void test_object_blob_filter__to_odb(void)
 	git_attr_cache_flush(g_repo);
 	cl_git_append2file("empty_standard_repo/.gitattributes", "*.txt text\n");
 
-	cl_git_pass(
-		git_filter_list_load(&fl, g_repo, "filename.txt", GIT_FILTER_TO_ODB));
+	cl_git_pass(git_filter_list_load(
+		&fl, g_repo, NULL, "filename.txt", GIT_FILTER_TO_ODB));
 	cl_assert(fl != NULL);
 
 	for (i = 0; i < NUM_TEST_OBJECTS; i++) {