filters: custom filters with wildcard attributes

Allow custom filters with wildcard attributes, so that clients
can support some random `filter=foo` in a .gitattributes and look
up the corresponding smudge/clean commands in the configuration file.

CHANGELOG.md

@@ -75,6 +75,10 @@ support for HTTPS connections insead of OpenSSL.
 * If libcurl is installed, we will use it to connect to HTTP(S)
   servers.
 
+* Custom filters can now be registered with wildcard attributes, for
+  example `filter=*`.  Consumers should examine the attributes parameter
+  of the `check` function for details.
+
 ### API additions
 
 * The `git_merge_options` gained a `file_flags` member.

include/git2/sys/filter.h

@@ -240,7 +240,10 @@ typedef void (*git_filter_cleanup_fn)(
  * for this filter (e.g. "eol crlf text").  If the attribute name is bare,
  * it will be simply loaded and passed to the `check` callback.  If it has
  * a value (i.e. "name=value"), the attribute must match that value for
- * the filter to be applied.
+ * the filter to be applied.  The value may be a wildcard (eg, "name=*"),
+ * in which case the filter will be invoked for any value for the given
+ * attribute name.  See the attribute parameter of the `check` callback
+ * for the attribute value that was specified.
  *
  * The `initialize`, `shutdown`, `check`, `apply`, and `cleanup` callbacks
  * are all documented above with the respective function pointer typedefs.

src/filter.c

@@ -433,8 +433,11 @@ static int filter_list_check_attributes(
 		want_type  = git_attr_value(want);
 		found_type = git_attr_value(strs[i]);
 
-		if (want_type != found_type ||
-			(want_type == GIT_ATTR_VALUE_T && strcmp(want, strs[i])))
+		if (want_type != found_type)
+			error = GIT_ENOTFOUND;
+		else if (want_type == GIT_ATTR_VALUE_T &&
+				strcmp(want, strs[i]) &&
+				strcmp(want, "*"))
 			error = GIT_ENOTFOUND;
 	}
 

tests/filter/custom.c

@@ -5,6 +5,7 @@
 #include "buf_text.h"
 #include "git2/sys/filter.h"
 #include "git2/sys/repository.h"
+#include "custom_helpers.h"
 
 /* going TO_WORKDIR, filters are executed low to high
  * going TO_ODB, filters are executed high to low
@@ -12,8 +13,6 @@
 #define BITFLIP_FILTER_PRIORITY -1
 #define REVERSE_FILTER_PRIORITY -2
 
-#define VERY_SECURE_ENCRYPTION(b) ((b) ^ 0xff)
-
 #ifdef GIT_WIN32
 # define NEWLINE "\r\n"
 #else
@@ -27,6 +26,8 @@ static char workdir_data[] =
 	"trivially" NEWLINE
 	"scrambled." NEWLINE;
 
+#define REVERSED_DATA_LEN 51
+
 /* Represents the data above scrambled (bits flipped) after \r\n -> \n
  * conversion, then bytewise reversed
  */
@@ -63,107 +64,6 @@ void test_filter_custom__cleanup(void)
 	g_repo = NULL;
 }
 
-static int bitflip_filter_apply(
-	git_filter     *self,
-	void          **payload,
-	git_buf        *to,
-	const git_buf  *from,
-	const git_filter_source *source)
-{
-	const unsigned char *src = (const unsigned char *)from->ptr;
-	unsigned char *dst;
-	size_t i;
-
-	GIT_UNUSED(self); GIT_UNUSED(payload);
-
-	/* verify that attribute path match worked as expected */
-	cl_assert_equal_i(
-		0, git__strncmp("hero", git_filter_source_path(source), 4));
-
-	if (!from->size)
-		return 0;
-
-	cl_git_pass(git_buf_grow(to, from->size));
-
-	dst = (unsigned char *)to->ptr;
-
-	for (i = 0; i < from->size; i++)
-		dst[i] = VERY_SECURE_ENCRYPTION(src[i]);
-
-	to->size = from->size;
-
-	return 0;
-}
-
-static void bitflip_filter_free(git_filter *f)
-{
-	git__free(f);
-}
-
-static git_filter *create_bitflip_filter(void)
-{
-	git_filter *filter = git__calloc(1, sizeof(git_filter));
-	cl_assert(filter);
-
-	filter->version = GIT_FILTER_VERSION;
-	filter->attributes = "+bitflip";
-	filter->shutdown = bitflip_filter_free;
-	filter->apply = bitflip_filter_apply;
-
-	return filter;
-}
-
-
-static int reverse_filter_apply(
-	git_filter     *self,
-	void          **payload,
-	git_buf        *to,
-	const git_buf  *from,
-	const git_filter_source *source)
-{
-	const unsigned char *src = (const unsigned char *)from->ptr;
-	const unsigned char *end = src + from->size;
-	unsigned char *dst;
-
-	GIT_UNUSED(self); GIT_UNUSED(payload); GIT_UNUSED(source);
-
-	/* verify that attribute path match worked as expected */
-	cl_assert_equal_i(
-		0, git__strncmp("hero", git_filter_source_path(source), 4));
-
-	if (!from->size)
-		return 0;
-
-	cl_git_pass(git_buf_grow(to, from->size));
-
-	dst = (unsigned char *)to->ptr + from->size - 1;
-
-	while (src < end)
-		*dst-- = *src++;
-
-	to->size = from->size;
-
-	return 0;
-}
-
-static void reverse_filter_free(git_filter *f)
-{
-	git__free(f);
-}
-
-static git_filter *create_reverse_filter(const char *attrs)
-{
-	git_filter *filter = git__calloc(1, sizeof(git_filter));
-	cl_assert(filter);
-
-	filter->version = GIT_FILTER_VERSION;
-	filter->attributes = attrs;
-	filter->shutdown = reverse_filter_free;
-	filter->apply = reverse_filter_apply;
-
-	return filter;
-}
-
 static void register_custom_filters(void)
 {
 	static int filters_registered = 0;
@@ -186,7 +86,6 @@ static void register_custom_filters(void)
 	}
 }
 
-
 void test_filter_custom__to_odb(void)
 {
 	git_filter_list *fl;

tests/filter/custom_helpers.c

+#include "clar_libgit2.h"
+#include "posix.h"
+#include "filter.h"
+#include "buf_text.h"
+#include "git2/sys/filter.h"
+
+#define VERY_SECURE_ENCRYPTION(b) ((b) ^ 0xff)
+
+int bitflip_filter_apply(
+	git_filter     *self,
+	void          **payload,
+	git_buf        *to,
+	const git_buf  *from,
+	const git_filter_source *source)
+{
+	const unsigned char *src = (const unsigned char *)from->ptr;
+	unsigned char *dst;
+	size_t i;
+
+	GIT_UNUSED(self); GIT_UNUSED(payload);
+
+	/* verify that attribute path match worked as expected */
+	cl_assert_equal_i(
+		0, git__strncmp("hero", git_filter_source_path(source), 4));
+
+	if (!from->size)
+		return 0;
+
+	cl_git_pass(git_buf_grow(to, from->size));
+
+	dst = (unsigned char *)to->ptr;
+
+	for (i = 0; i < from->size; i++)
+		dst[i] = VERY_SECURE_ENCRYPTION(src[i]);
+
+	to->size = from->size;
+
+	return 0;
+}
+
+static void bitflip_filter_free(git_filter *f)
+{
+	git__free(f);
+}
+
+git_filter *create_bitflip_filter(void)
+{
+	git_filter *filter = git__calloc(1, sizeof(git_filter));
+	cl_assert(filter);
+
+	filter->version = GIT_FILTER_VERSION;
+	filter->attributes = "+bitflip";
+	filter->shutdown = bitflip_filter_free;
+	filter->apply = bitflip_filter_apply;
+
+	return filter;
+}
+
+
+int reverse_filter_apply(
+	git_filter     *self,
+	void          **payload,
+	git_buf        *to,
+	const git_buf  *from,
+	const git_filter_source *source)
+{
+	const unsigned char *src = (const unsigned char *)from->ptr;
+	const unsigned char *end = src + from->size;
+	unsigned char *dst;
+
+	GIT_UNUSED(self); GIT_UNUSED(payload); GIT_UNUSED(source);
+
+	/* verify that attribute path match worked as expected */
+	cl_assert_equal_i(
+		0, git__strncmp("hero", git_filter_source_path(source), 4));
+
+	if (!from->size)
+		return 0;
+
+	cl_git_pass(git_buf_grow(to, from->size));
+
+	dst = (unsigned char *)to->ptr + from->size - 1;
+
+	while (src < end)
+		*dst-- = *src++;
+
+	to->size = from->size;
+
+	return 0;
+}
+
+static void reverse_filter_free(git_filter *f)
+{
+	git__free(f);
+}
+
+git_filter *create_reverse_filter(const char *attrs)
+{
+	git_filter *filter = git__calloc(1, sizeof(git_filter));
+	cl_assert(filter);
+
+	filter->version = GIT_FILTER_VERSION;
+	filter->attributes = attrs;
+	filter->shutdown = reverse_filter_free;
+	filter->apply = reverse_filter_apply;
+
+	return filter;
+}

tests/filter/custom_helpers.h

+#include "git2/sys/filter.h"
+
+extern git_filter *create_bitflip_filter(void);
+extern git_filter *create_reverse_filter(const char *attr);
+
+extern int bitflip_filter_apply(
+	git_filter     *self,
+	void          **payload,
+	git_buf        *to,
+	const git_buf  *from,
+	const git_filter_source *source);
+
+extern int reverse_filter_apply(
+	git_filter     *self,
+	void          **payload,
+	git_buf        *to,
+	const git_buf  *from,
+	const git_filter_source *source);

tests/filter/wildcard.c

+#include "clar_libgit2.h"
+#include "posix.h"
+#include "blob.h"
+#include "filter.h"
+#include "buf_text.h"
+#include "git2/sys/filter.h"
+#include "git2/sys/repository.h"
+#include "custom_helpers.h"
+
+static git_repository *g_repo = NULL;
+
+static git_filter *create_wildcard_filter();
+
+#define DATA_LEN 32
+
+static unsigned char input[] = {
+	0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+	0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
+	0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17,
+	0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f,
+};
+
+static unsigned char reversed[] = {
+	0x1f, 0x1e, 0x1d, 0x1c, 0x1b, 0x1a, 0x19, 0x18,
+	0x17, 0x16, 0x15, 0x14, 0x13, 0x12, 0x11, 0x10,
+	0x0f, 0x0e, 0x0d, 0x0c, 0x0b, 0x0a, 0x09, 0x08,
+	0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00,
+};
+
+static unsigned char flipped[] = {
+	0xff, 0xfe, 0xfd, 0xfc, 0xfb, 0xfa, 0xf9, 0xf8,
+	0xf7, 0xf6, 0xf5, 0xf4, 0xf3, 0xf2, 0xf1, 0xf0,
+	0xef, 0xee, 0xed, 0xec, 0xeb, 0xea, 0xe9, 0xe8,
+	0xe7, 0xe6, 0xe5, 0xe4, 0xe3, 0xe2, 0xe1, 0xe0,
+};
+
+void test_filter_wildcard__initialize(void)
+{
+	cl_git_pass(git_filter_register(
+		"wildcard", create_wildcard_filter(), GIT_FILTER_DRIVER_PRIORITY));
+
+	g_repo = cl_git_sandbox_init("empty_standard_repo");
+
+	cl_git_rewritefile(
+		"empty_standard_repo/.gitattributes",
+		"* binary\n"
+		"hero-flip-* filter=wcflip\n"
+		"hero-reverse-* filter=wcreverse\n"
+		"none-* filter=unregistered\n");
+}
+
+void test_filter_wildcard__cleanup(void)
+{
+	cl_git_pass(git_filter_unregister("wildcard"));
+
+	cl_git_sandbox_cleanup();
+	g_repo = NULL;
+}
+
+static int wildcard_filter_check(
+	git_filter  *self,
+	void **payload,
+	const git_filter_source *src,
+	const char **attr_values)
+{
+	if (strcmp(attr_values[0], "wcflip") == 0 ||
+		strcmp(attr_values[0], "wcreverse") == 0) {
+		*payload = git__strdup(attr_values[0]);
+		GITERR_CHECK_ALLOC(*payload);
+		return 0;
+	}
+
+	return GIT_PASSTHROUGH;
+}
+
+static int wildcard_filter_apply(
+	git_filter     *self,
+	void          **payload,
+	git_buf        *to,
+	const git_buf  *from,
+	const git_filter_source *source)
+{
+	const char *filtername = *payload;
+
+	if (filtername && strcmp(filtername, "wcflip") == 0)
+		return bitflip_filter_apply(self, payload, to, from, source);
+	else if (filtername && strcmp(filtername, "wcreverse") == 0)
+		return reverse_filter_apply(self, payload, to, from, source);
+
+	cl_fail("Unexpected attribute");
+	return GIT_PASSTHROUGH;
+}
+
+static int wildcard_filter_cleanup(git_filter *self, void *payload)
+{
+	git__free(payload);
+	return 0;
+}
+
+static void wildcard_filter_free(git_filter *f)
+{
+	git__free(f);
+}
+
+static git_filter *create_wildcard_filter()
+{
+	git_filter *filter = git__calloc(1, sizeof(git_filter));
+	cl_assert(filter);
+
+	filter->version = GIT_FILTER_VERSION;
+	filter->attributes = "filter=*";
+	filter->check = wildcard_filter_check;
+	filter->apply = wildcard_filter_apply;
+	filter->cleanup = wildcard_filter_cleanup;
+	filter->shutdown = wildcard_filter_free;
+
+	return filter;
+}
+
+void test_filter_wildcard__reverse(void)
+{
+	git_filter_list *fl;
+	git_buf in = GIT_BUF_INIT, out = GIT_BUF_INIT;
+
+	cl_git_pass(git_filter_list_load(
+		&fl, g_repo, NULL, "hero-reverse-foo", GIT_FILTER_TO_ODB, 0));
+
+	cl_git_pass(git_buf_put(&in, input, DATA_LEN));
+	cl_git_pass(git_filter_list_apply_to_data(&out, fl, &in));
+
+	cl_assert_equal_i(DATA_LEN, out.size);
+
+	cl_assert_equal_i(
+		0, memcmp(reversed, out.ptr, out.size));
+
+	git_filter_list_free(fl);
+	git_buf_free(&out);
+	git_buf_free(&in);
+}
+
+void test_filter_wildcard__flip(void)
+{
+	git_filter_list *fl;
+	git_buf in = GIT_BUF_INIT, out = GIT_BUF_INIT;
+
+	cl_git_pass(git_filter_list_load(
+		&fl, g_repo, NULL, "hero-flip-foo", GIT_FILTER_TO_ODB, 0));
+
+	cl_git_pass(git_buf_put(&in, input, DATA_LEN));
+	cl_git_pass(git_filter_list_apply_to_data(&out, fl, &in));
+
+	cl_assert_equal_i(DATA_LEN, out.size);
+
+	cl_assert_equal_i(
+		0, memcmp(flipped, out.ptr, out.size));
+
+	git_filter_list_free(fl);
+	git_buf_free(&out);
+	git_buf_free(&in);
+}
+
+void test_filter_wildcard__none(void)
+{
+	git_filter_list *fl;
+	git_buf in = GIT_BUF_INIT, out = GIT_BUF_INIT;
+
+	cl_git_pass(git_filter_list_load(
+		&fl, g_repo, NULL, "none-foo", GIT_FILTER_TO_ODB, 0));
+
+	cl_git_pass(git_buf_put(&in, input, DATA_LEN));
+	cl_git_pass(git_filter_list_apply_to_data(&out, fl, &in));
+
+	cl_assert_equal_i(DATA_LEN, out.size);
+
+	cl_assert_equal_i(
+		0, memcmp(input, out.ptr, out.size));
+
+	git_filter_list_free(fl);
+	git_buf_free(&out);
+	git_buf_free(&in);
+}