Merge pull request #2291 from ethomson/patch_binary

patch: emit deflated binary patches (optionally)

include/git2/diff.h

@@ -180,6 +180,10 @@ typedef enum {
 	/** Take extra time to find minimal diff */
 	GIT_DIFF_MINIMAL = (1 << 29),
 
+	/** Include the necessary deflate / delta information so that `git-apply`
+	 *  can apply given diff information to binary files.
+	 */
+	GIT_DIFF_SHOW_BINARY = (1 << 30),
 } git_diff_option_t;
 
 /**

src/buffer.c

@@ -212,6 +212,42 @@ int git_buf_put_base64(git_buf *buf, const char *data, size_t len)
 	return 0;
 }
 
+static const char b85str[] =
+	"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz!#$%&()*+-;<=>?@^_`{|}~";
+
+int git_buf_put_base85(git_buf *buf, const char *data, size_t len)
+{
+	ENSURE_SIZE(buf, buf->size + (5 * ((len / 4) + !!(len % 4))) + 1);
+
+	while (len) {
+		uint32_t acc = 0;
+		char b85[5];
+		int i;
+
+		for (i = 24; i >= 0; i -= 8) {
+			uint8_t ch = *data++;
+			acc |= ch << i;
+
+			if (--len == 0)
+				break;
+		}
+
+		for (i = 4; i >= 0; i--) {
+			int val = acc % 85;
+			acc /= 85;
+
+			b85[i] = b85str[val];
+		}
+
+		for (i = 0; i < 5; i++)
+			buf->ptr[buf->size++] = b85[i];
+	}
+
+	buf->ptr[buf->size] = '\0';
+
+	return 0;
+}
+
 int git_buf_vprintf(git_buf *buf, const char *format, va_list ap)
 {
 	int len;

src/buffer.h

@@ -158,6 +158,9 @@ int git_buf_cmp(const git_buf *a, const git_buf *b);
 /* Write data as base64 encoded in buffer */
 int git_buf_put_base64(git_buf *buf, const char *data, size_t len);
 
+/* Write data as "base85" encoded in buffer */
+int git_buf_put_base85(git_buf *buf, const char *data, size_t len);
+
 /*
  * Insert, remove or replace a portion of the buffer.
  *

src/diff_print.c

@@ -8,6 +8,9 @@
 #include "diff.h"
 #include "diff_patch.h"
 #include "fileops.h"
+#include "zstream.h"
+#include "blob.h"
+#include "delta.h"
 #include "git2/sys/diff.h"
 
 typedef struct {
@@ -37,6 +40,8 @@ static int diff_print_info_init(
 
 	if (diff)
 		pi->flags = diff->opts.flags;
+	else
+		pi->flags = 0;
 
 	if (diff && diff->opts.id_abbrev != 0)
 		pi->oid_strlen = diff->opts.id_abbrev;
@@ -277,6 +282,118 @@ int git_diff_delta__format_file_header(
 	return git_buf_oom(out) ? -1 : 0;
 }
 
+static int print_binary_hunk(diff_print_info *pi, git_blob *old, git_blob *new)
+{
+	git_buf deflate = GIT_BUF_INIT, delta = GIT_BUF_INIT, *out = NULL;
+	const void *old_data, *new_data;
+	size_t old_data_len, new_data_len, delta_data_len, inflated_len, remain;
+	const char *out_type = "literal";
+	char *ptr;
+	int error;
+
+	old_data = old ? git_blob_rawcontent(old) : NULL;
+	new_data = new ? git_blob_rawcontent(new) : NULL;
+
+	old_data_len = old ? (size_t)git_blob_rawsize(old) : 0;
+	new_data_len = new ? (size_t)git_blob_rawsize(new) : 0;
+
+	out = &deflate;
+	inflated_len = new_data_len;
+
+	if ((error = git_zstream_deflatebuf(
+		&deflate, new_data, new_data_len)) < 0)
+		goto done;
+
+	if (old && new) {
+		void *delta_data;
+
+		delta_data = git_delta(old_data, old_data_len, new_data,
+			new_data_len, &delta_data_len, deflate.size);
+
+		if (delta_data) {
+			error = git_zstream_deflatebuf(&delta, delta_data, delta_data_len);
+			free(delta_data);
+
+			if (error < 0)
+				goto done;
+
+			if (delta.size < deflate.size) {
+				out = &delta;
+				out_type = "delta";
+				inflated_len = delta_data_len;
+			}
+		}
+	}
+
+	git_buf_printf(pi->buf, "%s %" PRIuZ "\n", out_type, inflated_len);
+	pi->line.num_lines++;
+
+	for (ptr = out->ptr, remain = out->size; remain > 0; ) {
+		size_t chunk_len = (52 < remain) ? 52 : remain;
+
+		if (chunk_len <= 26)
+			git_buf_putc(pi->buf, chunk_len + 'A' - 1);
+		else
+			git_buf_putc(pi->buf, chunk_len - 26 + 'a' - 1);
+
+		git_buf_put_base85(pi->buf, ptr, chunk_len);
+		git_buf_putc(pi->buf, '\n');
+
+		if (git_buf_oom(pi->buf)) {
+			error = -1;
+			goto done;
+		}
+
+		ptr += chunk_len;
+		remain -= chunk_len;
+		pi->line.num_lines++;
+	}
+
+done:
+	git_buf_free(&deflate);
+	git_buf_free(&delta);
+
+	return error;
+}
+
+/* git diff --binary 8d7523f~2 8d7523f~1 */
+static int diff_print_patch_file_binary(
+	diff_print_info *pi, const git_diff_delta *delta,
+	const char *oldpfx, const char *newpfx)
+{
+	git_blob *old = NULL, *new = NULL;
+	const git_oid *old_id, *new_id;
+	int error;
+
+	if ((pi->flags & GIT_DIFF_SHOW_BINARY) == 0) {
+		pi->line.num_lines = 1;
+		return diff_delta_format_with_paths(
+			pi->buf, delta, oldpfx, newpfx,
+				"Binary files %s%s and %s%s differ\n");
+	}
+
+	git_buf_printf(pi->buf, "GIT binary patch\n");
+	pi->line.num_lines++;
+
+	old_id = (delta->status != GIT_DELTA_ADDED) ? &delta->old_file.id : NULL;
+	new_id = (delta->status != GIT_DELTA_DELETED) ? &delta->new_file.id : NULL;
+
+	if ((old_id && (error = git_blob_lookup(&old, pi->diff->repo, old_id)) < 0) ||
+		(new_id && (error = git_blob_lookup(&new, pi->diff->repo,new_id)) < 0) ||
+		(error = print_binary_hunk(pi, old, new)) < 0 ||
+		(error = git_buf_putc(pi->buf, '\n')) < 0 ||
+		(error = print_binary_hunk(pi, new, old)) < 0)
+		goto done;
+
+	pi->line.num_lines++;
+
+done:
+	git_blob_free(old);
+	git_blob_free(new);
+
+	return error;
+}
+
 static int diff_print_patch_file(
 	const git_diff_delta *delta, float progress, void *data)
 {
@@ -287,6 +404,11 @@ static int diff_print_patch_file(
 	const char *newpfx =
 		pi->diff ? pi->diff->opts.new_prefix : DIFF_NEW_PREFIX_DEFAULT;
 
+	bool binary = !!(delta->flags & GIT_DIFF_FLAG_BINARY);
+	bool show_binary = !!(pi->flags & GIT_DIFF_SHOW_BINARY);
+	int oid_strlen = binary && show_binary ?
+		GIT_OID_HEXSZ + 1 : pi->oid_strlen;
+
 	GIT_UNUSED(progress);
 
 	if (S_ISDIR(delta->new_file.mode) ||
@@ -297,7 +419,7 @@ static int diff_print_patch_file(
 		return 0;
 
 	if ((error = git_diff_delta__format_file_header(
-			pi->buf, delta, oldpfx, newpfx, pi->oid_strlen)) < 0)
+			pi->buf, delta, oldpfx, newpfx, oid_strlen)) < 0)
 		return error;
 
 	pi->line.origin      = GIT_DIFF_LINE_FILE_HDR;
@@ -307,20 +429,17 @@ static int diff_print_patch_file(
 	if ((error = pi->print_cb(delta, NULL, &pi->line, pi->payload)) != 0)
 		return error;
 
-	if ((delta->flags & GIT_DIFF_FLAG_BINARY) == 0)
+	if (!binary)
 		return 0;
 
 	git_buf_clear(pi->buf);
 
-	if ((error = diff_delta_format_with_paths(
-			pi->buf, delta, oldpfx, newpfx,
-			"Binary files %s%s and %s%s differ\n")) < 0)
+	if ((error = diff_print_patch_file_binary(pi, delta, oldpfx, newpfx)) < 0)
 		return error;
 
 	pi->line.origin      = GIT_DIFF_LINE_BINARY;
 	pi->line.content     = git_buf_cstr(pi->buf);
 	pi->line.content_len = git_buf_len(pi->buf);
-	pi->line.num_lines   = 1;
 
 	return pi->print_cb(delta, NULL, &pi->line, pi->payload);
 }

tests/core/buffer.c

@@ -773,6 +773,26 @@ void test_core_buffer__base64(void)
 	git_buf_free(&buf);
 }
 
+void test_core_buffer__base85(void)
+{
+	git_buf buf = GIT_BUF_INIT;
+
+	cl_git_pass(git_buf_put_base85(&buf, "this", 4));
+	cl_assert_equal_s("bZBXF", buf.ptr);
+	git_buf_clear(&buf);
+
+	cl_git_pass(git_buf_put_base85(&buf, "two rnds", 8));
+	cl_assert_equal_s("ba!tca&BaE", buf.ptr);
+	git_buf_clear(&buf);
+
+	cl_git_pass(git_buf_put_base85(&buf, "this is base 85 encoded",
+		strlen("this is base 85 encoded")));
+	cl_assert_equal_s("bZBXFAZc?TVqtS-AUHK3Wo~0{WMyOk", buf.ptr);
+	git_buf_clear(&buf);
+
+	git_buf_free(&buf);
+}
+
 void test_core_buffer__classify_with_utf8(void)
 {
 	char *data0 = "Simple text\n";

tests/diff/binary.c

+#include "clar_libgit2.h"
+
+#include "buffer.h"
+#include "filebuf.h"
+
+static git_repository *repo;
+
+void test_diff_binary__initialize(void)
+{
+}
+
+void test_diff_binary__cleanup(void)
+{
+	cl_git_sandbox_cleanup();
+}
+
+void test_patch(
+	const char *one,
+	const char *two,
+	const git_diff_options *opts,
+	const char *expected)
+{
+	git_oid id_one, id_two;
+	git_index *index = NULL;
+	git_commit *commit_one, *commit_two = NULL;
+	git_tree *tree_one, *tree_two;
+	git_diff *diff;
+	git_patch *patch;
+	git_buf actual = GIT_BUF_INIT;
+
+	cl_git_pass(git_oid_fromstr(&id_one, one));
+	cl_git_pass(git_commit_lookup(&commit_one, repo, &id_one));
+	cl_git_pass(git_commit_tree(&tree_one, commit_one));
+
+	if (two) {
+		cl_git_pass(git_oid_fromstr(&id_two, two));
+		cl_git_pass(git_commit_lookup(&commit_two, repo, &id_two));
+		cl_git_pass(git_commit_tree(&tree_two, commit_two));
+	} else {
+		cl_git_pass(git_repository_index(&index, repo));
+		cl_git_pass(git_index_write_tree(&id_two, index));
+		cl_git_pass(git_tree_lookup(&tree_two, repo, &id_two));
+	}
+
+	cl_git_pass(git_diff_tree_to_tree(&diff, repo, tree_one, tree_two, opts));
+
+	cl_git_pass(git_patch_from_diff(&patch, diff, 0));
+	cl_git_pass(git_patch_to_buf(&actual, patch));
+
+	cl_assert_equal_s(expected, actual.ptr);
+
+	git_buf_free(&actual);
+	git_patch_free(patch);
+	git_diff_free(diff);
+	git_tree_free(tree_one);
+	git_tree_free(tree_two);
+	git_commit_free(commit_one);
+	git_commit_free(commit_two);
+	git_index_free(index);
+}
+
+void test_diff_binary__add_normal(void)
+{
+	git_diff_options opts = GIT_DIFF_OPTIONS_INIT;
+	const char *expected =
+		"diff --git a/binary.bin b/binary.bin\n" \
+		"new file mode 100644\n" \
+		"index 0000000..bd474b2\n" \
+		"Binary files /dev/null and b/binary.bin differ\n";
+
+	repo = cl_git_sandbox_init("diff_format_email");
+	test_patch(
+		"873806f6f27e631eb0b23e4b56bea2bfac14a373",
+		"897d3af16ca9e420cd071b1c4541bd2b91d04c8c",
+		&opts,
+		expected);
+}
+
+void test_diff_binary__add(void)
+{
+	git_diff_options opts = GIT_DIFF_OPTIONS_INIT;
+	const char *expected =
+		"diff --git a/binary.bin b/binary.bin\n" \
+		"new file mode 100644\n" \
+		"index 0000000000000000000000000000000000000000..bd474b2519cc15eab801ff851cc7d50f0dee49a1\n" \
+		"GIT binary patch\n" \
+		"literal 3\n" \
+		"Kc${Nk-~s>u4FC%O\n"
+		"\n" \
+		"literal 0\n" \
+		"Hc$@<O00001\n";
+
+	opts.flags = GIT_DIFF_SHOW_BINARY;
+	opts.id_abbrev = GIT_OID_HEXSZ;
+
+	repo = cl_git_sandbox_init("diff_format_email");
+	test_patch(
+		"873806f6f27e631eb0b23e4b56bea2bfac14a373",
+		"897d3af16ca9e420cd071b1c4541bd2b91d04c8c",
+		&opts,
+		expected);
+}
+
+void test_diff_binary__modify_normal(void)
+{
+	git_diff_options opts = GIT_DIFF_OPTIONS_INIT;
+	const char *expected =
+		"diff --git a/binary.bin b/binary.bin\n" \
+		"index bd474b2..9ac35ff 100644\n" \
+		"Binary files a/binary.bin and b/binary.bin differ\n";
+
+	repo = cl_git_sandbox_init("diff_format_email");
+	test_patch(
+		"897d3af16ca9e420cd071b1c4541bd2b91d04c8c",
+		"8d7523f6fcb2404257889abe0d96f093d9f524f9",
+		&opts,
+		expected);
+}
+
+void test_diff_binary__modify(void)
+{
+	git_diff_options opts = GIT_DIFF_OPTIONS_INIT;
+	const char *expected =
+		"diff --git a/binary.bin b/binary.bin\n" \
+		"index bd474b2519cc15eab801ff851cc7d50f0dee49a1..9ac35ff15cd8864aeafd889e4826a3150f0b06c4 100644\n" \
+		"GIT binary patch\n" \
+		"literal 5\n" \
+		"Mc${NkU}WL~000&M4gdfE\n" \
+		"\n" \
+		"literal 3\n" \
+		"Kc${Nk-~s>u4FC%O\n";
+
+	opts.flags = GIT_DIFF_SHOW_BINARY;
+
+	repo = cl_git_sandbox_init("diff_format_email");
+	test_patch(
+		"897d3af16ca9e420cd071b1c4541bd2b91d04c8c",
+		"8d7523f6fcb2404257889abe0d96f093d9f524f9",
+		&opts,
+		expected);
+}
+
+void test_diff_binary__delete_normal(void)
+{
+	git_diff_options opts = GIT_DIFF_OPTIONS_INIT;
+	const char *expected =
+		"diff --git a/binary.bin b/binary.bin\n" \
+		"deleted file mode 100644\n" \
+		"index bd474b2..0000000\n" \
+		"Binary files a/binary.bin and /dev/null differ\n";
+
+	repo = cl_git_sandbox_init("diff_format_email");
+	test_patch(
+		"897d3af16ca9e420cd071b1c4541bd2b91d04c8c",
+		"873806f6f27e631eb0b23e4b56bea2bfac14a373",
+		&opts,
+		expected);
+}
+
+void test_diff_binary__delete(void)
+{
+	git_diff_options opts = GIT_DIFF_OPTIONS_INIT;
+	const char *expected =
+		"diff --git a/binary.bin b/binary.bin\n" \
+		"deleted file mode 100644\n" \
+		"index bd474b2519cc15eab801ff851cc7d50f0dee49a1..0000000000000000000000000000000000000000\n" \
+		"GIT binary patch\n" \
+		"literal 0\n" \
+		"Hc$@<O00001\n" \
+		"\n" \
+		"literal 3\n" \
+		"Kc${Nk-~s>u4FC%O\n";
+
+	opts.flags = GIT_DIFF_SHOW_BINARY;
+	opts.id_abbrev = GIT_OID_HEXSZ;
+
+	repo = cl_git_sandbox_init("diff_format_email");
+	test_patch(
+		"897d3af16ca9e420cd071b1c4541bd2b91d04c8c",
+		"873806f6f27e631eb0b23e4b56bea2bfac14a373",
+		&opts,
+		expected);
+}
+
+void test_diff_binary__delta(void)
+{
+	git_index *index;
+	git_buf contents = GIT_BUF_INIT;
+	size_t i;
+	git_diff_options opts = GIT_DIFF_OPTIONS_INIT;
+	const char *expected =
+		"diff --git a/songof7cities.txt b/songof7cities.txt\n" \
+		"index 4210ffd5c390b21dd5483375e75288dea9ede512..cc84ec183351c9944ed90a619ca08911924055b5 100644\n" \
+		"GIT binary patch\n" \
+		"delta 198\n" \
+		"zc$}LmI8{(0BqLQJI6p64AwNwaIJGP_Pa)Ye#M3o+qJ$<Jl;sX*mF<MGCYv&*L7AHu\n" \
+		"zGA1*^gt?gYVN82wTbPO_W)+x<&1+cP;HrPHR>PQ;Y(X&QMK*C5^Br3bjG4d=XI^5@\n" \
+		"JfH567LIG)KJdFSV\n" \
+		"\n" \
+		"delta 198\n" \
+		"zc$}LmI8{(0BqLQJI6p64AwNwaIJGP_Pr*5}Br~;mqJ$<Jl;sX*mF<MGCYv&*L7AHu\n" \
+		"zGA1*^gt?gYVN82wTbPO_W)+x<&1+cP;HrPHR>PQ;Y(X&QMK*C5^Br3bjG4d=XI^5@\n" \
+		"JfH567LIF3FM2!Fd\n";
+
+	opts.flags = GIT_DIFF_SHOW_BINARY | GIT_DIFF_FORCE_BINARY;
+	opts.id_abbrev = GIT_OID_HEXSZ;
+
+	repo = cl_git_sandbox_init("renames");
+	cl_git_pass(git_repository_index(&index, repo));
+
+	cl_git_pass(git_futils_readbuffer(&contents, "renames/songof7cities.txt"));
+
+	for (i = 0; i < contents.size - 6; i++) {
+		if (strncmp(&contents.ptr[i], "Cities", 6) == 0)
+			memcpy(&contents.ptr[i], "cITIES", 6);
+	}
+
+	cl_git_rewritefile("renames/songof7cities.txt", contents.ptr);
+	cl_git_pass(git_index_add_bypath(index, "songof7cities.txt"));
+	cl_git_pass(git_index_write(index));
+
+	test_patch(
+		"19dd32dfb1520a64e5bbaae8dce6ef423dfa2f13",
+		NULL,
+		&opts,
+		expected);
+
+	git_index_free(index);
+	git_buf_free(&contents);
+}
+
+void test_diff_binary__delta_append(void)
+{
+	git_index *index;
+	git_diff_options opts = GIT_DIFF_OPTIONS_INIT;
+	const char *expected =
+		"diff --git a/untimely.txt b/untimely.txt\n" \
+		"index 9a69d960ae94b060f56c2a8702545e2bb1abb935..1111d4f11f4b35bf6759e0fb714fe09731ef0840 100644\n" \
+		"GIT binary patch\n" \
+		"delta 32\n" \
+		"nc%1vf+QYWt3zLL@hC)e3Vu?a>QDRl4f_G*?PG(-ZA}<#J$+QbW\n" \
+		"\n" \
+		"delta 7\n" \
+		"Oc%18D`@*{63ljhg(E~C7\n";
+
+	opts.flags = GIT_DIFF_SHOW_BINARY | GIT_DIFF_FORCE_BINARY;
+	opts.id_abbrev = GIT_OID_HEXSZ;
+
+	repo = cl_git_sandbox_init("renames");
+	cl_git_pass(git_repository_index(&index, repo));
+
+	cl_git_append2file("renames/untimely.txt", "Oh that crazy Kipling!\r\n");
+	cl_git_pass(git_index_add_bypath(index, "untimely.txt"));
+	cl_git_pass(git_index_write(index));
+
+	test_patch(
+		"19dd32dfb1520a64e5bbaae8dce6ef423dfa2f13",
+		NULL,
+		&opts,
+		expected);
+
+	git_index_free(index);
+}

tests/diff/patch.c

@@ -2,6 +2,7 @@
 #include "git2/sys/repository.h"
 
 #include "diff_helpers.h"
+#include "diff.h"
 #include "repository.h"
 #include "buf_text.h"