Merge pull request #4136 from libgit2/ethomson/sha1dc

Introduce (optional) SHA1 collision attack detection

CMakeLists.txt

@@ -37,6 +37,7 @@ OPTION( PROFILE				"Generate profiling information"		OFF )
 OPTION( ENABLE_TRACE		"Enables tracing support"				OFF )
 OPTION( LIBGIT2_FILENAME	"Name of the produced binary"			OFF )
 
+OPTION( USE_SHA1DC			"Use SHA-1 with collision detection"	OFF )
 OPTION( USE_ICONV			"Link with and use iconv library" 		OFF )
 OPTION( USE_SSH				"Link with libssh to enable SSH support" ON )
 OPTION( USE_GSSAPI			"Link with libgssapi for SPNEGO auth"   OFF )
@@ -292,13 +293,16 @@ ELSE ()
 ENDIF()
 
 # Specify sha1 implementation
-IF (WIN32 AND NOT MINGW AND NOT SHA1_TYPE STREQUAL "builtin")
-	ADD_DEFINITIONS(-DWIN32_SHA1)
+IF (USE_SHA1DC)
+	ADD_DEFINITIONS(-DGIT_SHA1_COLLISIONDETECT)
+	FILE(GLOB SRC_SHA1 src/hash/hash_collisiondetect.c src/hash/sha1dc/*)
+ELSEIF (WIN32 AND NOT MINGW)
+	ADD_DEFINITIONS(-DGIT_SHA1_WIN32)
 	FILE(GLOB SRC_SHA1 src/hash/hash_win32.c)
 ELSEIF (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
-	ADD_DEFINITIONS(-DGIT_COMMON_CRYPTO)
-ELSEIF (OPENSSL_FOUND AND NOT SHA1_TYPE STREQUAL "builtin")
-	ADD_DEFINITIONS(-DOPENSSL_SHA1)
+	ADD_DEFINITIONS(-DGIT_SHA1_COMMON_CRYPTO)
+ELSEIF (OPENSSL_FOUND)
+	ADD_DEFINITIONS(-DGIT_SHA1_OPENSSL)
 	IF (CMAKE_SYSTEM_NAME MATCHES "FreeBSD")
 		LIST(APPEND LIBGIT2_PC_LIBS "-lssl")
 	ELSE()

COPYING

@@ -958,3 +958,36 @@ necessary.  Here is a sample; alter the names:
 That's all there is to it!
 
 ----------------------------------------------------------------------
+
+The bundled SHA1 collision detection code is licensed under the MIT license:
+
+MIT License
+
+Copyright (c) 2017:
+    Marc Stevens
+    Cryptology Group
+    Centrum Wiskunde & Informatica
+    P.O. Box 94079, 1090 GB Amsterdam, Netherlands
+    marc@marc-stevens.nl
+
+    Dan Shumow
+    Microsoft Research
+    danshu@microsoft.com
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

include/git2/errors.h

@@ -100,7 +100,8 @@ typedef enum {
 	GITERR_REBASE,
 	GITERR_FILESYSTEM,
 	GITERR_PATCH,
-	GITERR_WORKTREE
+	GITERR_WORKTREE,
+	GITERR_SHA1
 } git_error_t;
 
 /**

src/hash.h

@@ -16,11 +16,13 @@ int git_hash_global_init(void);
 int git_hash_ctx_init(git_hash_ctx *ctx);
 void git_hash_ctx_cleanup(git_hash_ctx *ctx);
 
-#if defined(GIT_COMMON_CRYPTO)
+#if defined(GIT_SHA1_COLLISIONDETECT)
+# include "hash/hash_collisiondetect.h"
+#elif defined(GIT_SHA1_COMMON_CRYPTO)
 # include "hash/hash_common_crypto.h"
-#elif defined(OPENSSL_SHA1)
+#elif defined(GIT_SHA1_OPENSSL)
 # include "hash/hash_openssl.h"
-#elif defined(WIN32_SHA1)
+#elif defined(GIT_SHA1_WIN32)
 # include "hash/hash_win32.h"
 #else
 # include "hash/hash_generic.h"

src/hash/hash_collisiondetect.h

+/*
+ * Copyright (C) the libgit2 contributors. All rights reserved.
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+
+#ifndef INCLUDE_hash_collisiondetect_h__
+#define INCLUDE_hash_collisiondetect_h__
+
+#include "hash.h"
+#include "sha1dc/sha1.h"
+
+struct git_hash_ctx {
+	SHA1_CTX c;
+};
+
+#define git_hash_global_init() 0
+#define git_hash_ctx_init(ctx) git_hash_init(ctx)
+#define git_hash_ctx_cleanup(ctx)
+
+GIT_INLINE(int) git_hash_init(git_hash_ctx *ctx)
+{
+	assert(ctx);
+	SHA1DCInit(&ctx->c);
+	return 0;
+}
+
+GIT_INLINE(int) git_hash_update(git_hash_ctx *ctx, const void *data, size_t len)
+{
+    const char *p = data;
+
+	assert(ctx);
+
+	/* We expect a size_t, but sha1dc only takes an int */
+	while (len > INT_MAX) {
+		SHA1DCUpdate(&ctx->c, p, INT_MAX);
+		p += INT_MAX;
+		len -= INT_MAX;
+	}
+
+	SHA1DCUpdate(&ctx->c, p, len);
+	return 0;
+}
+
+GIT_INLINE(int) git_hash_final(git_oid *out, git_hash_ctx *ctx)
+{
+	assert(ctx);
+	if (SHA1DCFinal(out->id, &ctx->c)) {
+		giterr_set(GITERR_SHA1, "SHA1 collision attack detected");
+		return -1;
+	}
+
+	return 0;
+}
+
+#endif /* INCLUDE_hash_collisiondetect_h__ */

src/hash/sha1dc/sha1.h

+/***
+* Copyright 2017 Marc Stevens <marc@marc-stevens.nl>, Dan Shumow <danshu@microsoft.com>
+* Distributed under the MIT Software License.
+* See accompanying file LICENSE.txt or copy at
+* https://opensource.org/licenses/MIT
+***/
+
+#include <stdint.h>
+
+// uses SHA-1 message expansion to expand the first 16 words of W[] to 80 words
+void sha1_message_expansion(uint32_t W[80]);
+
+// sha-1 compression function; first version takes a message block pre-parsed as 16 32-bit integers, second version takes an already expanded message)
+void sha1_compression(uint32_t ihv[5], const uint32_t m[16]);
+void sha1_compression_W(uint32_t ihv[5], const uint32_t W[80]);
+
+// same as sha1_compression_W, but additionally store intermediate states
+// only stores states ii (the state between step ii-1 and step ii) when DOSTORESTATEii is defined in ubc_check.h
+void sha1_compression_states(uint32_t ihv[5], const uint32_t W[80], uint32_t states[80][5]);
+
+// function type for sha1_recompression_step_T (uint32_t ihvin[5], uint32_t ihvout[5], const uint32_t me2[80], const uint32_t state[5])
+// where 0 <= T < 80
+//       me2 is an expanded message (the expansion of an original message block XOR'ed with a disturbance vector's message block difference)
+//       state is the internal state (a,b,c,d,e) before step T of the SHA-1 compression function while processing the original message block
+// the function will return:
+//       ihvin: the reconstructed input chaining value
+//       ihvout: the reconstructed output chaining value
+typedef void(*sha1_recompression_type)(uint32_t*, uint32_t*, const uint32_t*, const uint32_t*);
+
+// table of sha1_recompression_step_0, ... , sha1_recompression_step_79
+extern sha1_recompression_type sha1_recompression_step[80];
+
+// a callback function type that can be set to be called when a collision block has been found:
+// void collision_block_callback(uint64_t byteoffset, const uint32_t ihvin1[5], const uint32_t ihvin2[5], const uint32_t m1[80], const uint32_t m2[80])
+typedef void(*collision_block_callback)(uint64_t, const uint32_t*, const uint32_t*, const uint32_t*, const uint32_t*);
+
+// the SHA-1 context
+typedef struct {
+	uint64_t total;
+	uint32_t ihv[5];
+	unsigned char buffer[64];
+	int bigendian;
+	int found_collision;
+	int safe_hash;
+	int detect_coll;
+	int ubc_check;
+	int reduced_round_coll;
+	collision_block_callback callback;
+
+	uint32_t ihv1[5];
+	uint32_t ihv2[5];
+	uint32_t m1[80];
+	uint32_t m2[80];
+	uint32_t states[80][5];
+} SHA1_CTX;
+
+// initialize SHA-1 context
+void SHA1DCInit(SHA1_CTX*); 
+
+// function to enable safe SHA-1 hashing:
+// collision attacks are thwarted by hashing a detected near-collision block 3 times
+// think of it as extending SHA-1 from 80-steps to 240-steps for such blocks:
+//   the best collision attacks against SHA-1 have complexity about 2^60, 
+//   thus for 240-steps an immediate lower-bound for the best cryptanalytic attacks would 2^180
+//   an attacker would be better off using a generic birthday search of complexity 2^80
+//
+// enabling safe SHA-1 hashing will result in the correct SHA-1 hash for messages where no collision attack was detected
+// but it will result in a different SHA-1 hash for messages where a collision attack was detected 
+// this will automatically invalidate SHA-1 based digital signature forgeries
+// enabled by default
+void SHA1DCSetSafeHash(SHA1_CTX*, int);
+
+// function to disable or enable the use of Unavoidable Bitconditions (provides a significant speed up)
+// enabled by default
+void SHA1DCSetUseUBC(SHA1_CTX*, int);
+
+// function to disable or enable the use of Collision Detection
+// enabled by default
+void SHA1DCSetUseDetectColl(SHA1_CTX* ctx, int detect_coll);
+
+// function to disable or enable the detection of reduced-round SHA-1 collisions
+// disabled by default
+void SHA1DCSetDetectReducedRoundCollision(SHA1_CTX*, int);
+
+// function to set a callback function, pass NULL to disable
+// by default no callback set
+void SHA1DCSetCallback(SHA1_CTX*, collision_block_callback);
+
+// update SHA-1 context with buffer contents
+void SHA1DCUpdate(SHA1_CTX*, const char*, unsigned);
+
+// obtain SHA-1 hash from SHA-1 context
+// returns: 0 = no collision detected, otherwise = collision found => warn user for active attack
+int  SHA1DCFinal(unsigned char[20], SHA1_CTX*); 

src/hash/sha1dc/ubc_check.h

+/***
+* Copyright 2017 Marc Stevens <marc@marc-stevens.nl>, Dan Shumow <danshu@microsoft.com>
+* Distributed under the MIT Software License.
+* See accompanying file LICENSE.txt or copy at
+* https://opensource.org/licenses/MIT
+***/
+
+// this file was generated by the 'parse_bitrel' program in the tools section
+// using the data files from directory 'tools/data/3565'
+//
+// sha1_dvs contains a list of SHA-1 Disturbance Vectors (DV) to check
+// dvType, dvK and dvB define the DV: I(K,B) or II(K,B) (see the paper)
+// dm[80] is the expanded message block XOR-difference defined by the DV
+// testt is the step to do the recompression from for collision detection
+// maski and maskb define the bit to check for each DV in the dvmask returned by ubc_check
+//
+// ubc_check takes as input an expanded message block and verifies the unavoidable bitconditions for all listed DVs
+// it returns a dvmask where each bit belonging to a DV is set if all unavoidable bitconditions for that DV have been met
+// thus one needs to do the recompression check for each DV that has its bit set
+
+#ifndef UBC_CHECK_H
+#define UBC_CHECK_H
+
+#include <stdint.h>
+
+#define DVMASKSIZE 1
+typedef struct { int dvType; int dvK; int dvB; int testt; int maski; int maskb; uint32_t dm[80]; } dv_info_t;
+extern dv_info_t sha1_dvs[];
+void ubc_check(const uint32_t W[80], uint32_t dvmask[DVMASKSIZE]);
+
+#define DOSTORESTATE58
+#define DOSTORESTATE65
+
+
+#endif // UBC_CHECK_H

tests/core/sha1.c

+#include "clar_libgit2.h"
+#include "hash.h"
+
+#define FIXTURE_DIR "sha1"
+
+void test_core_sha1__initialize(void)
+{
+	cl_fixture_sandbox(FIXTURE_DIR);
+}
+
+void test_core_sha1__cleanup(void)
+{
+	cl_fixture_cleanup(FIXTURE_DIR);
+}
+
+static int sha1_file(git_oid *oid, const char *filename)
+{
+	git_hash_ctx ctx;
+	char buf[2048];
+	int fd, ret;
+	ssize_t read_len;
+
+	fd = p_open(filename, O_RDONLY);
+	cl_assert(fd >= 0);
+
+	cl_git_pass(git_hash_ctx_init(&ctx));
+
+	while ((read_len = p_read(fd, buf, 2048)) > 0)
+		cl_git_pass(git_hash_update(&ctx, buf, (size_t)read_len));
+
+	cl_assert_equal_i(0, read_len);
+	p_close(fd);
+
+	ret = git_hash_final(oid, &ctx);
+	git_hash_ctx_cleanup(&ctx);
+
+	return ret;
+}
+
+void test_core_sha1__sum(void)
+{
+	git_oid oid, expected;
+
+	cl_git_pass(sha1_file(&oid, FIXTURE_DIR "/hello_c"));
+	git_oid_fromstr(&expected, "4e72679e3ea4d04e0c642f029e61eb8056c7ed94");
+	cl_assert_equal_oid(&expected, &oid);
+}
+
+/* test that sha1 collision detection works when enabled */
+void test_core_sha1__detect_collision_attack(void)
+{
+	git_oid oid, expected;
+
+#ifdef GIT_SHA1_COLLISIONDETECT
+	GIT_UNUSED(expected);
+	cl_git_fail(sha1_file(&oid, FIXTURE_DIR "/shattered-1.pdf"));
+	cl_assert_equal_s("SHA1 collision attack detected", giterr_last()->message);
+#else
+	cl_git_pass(sha1_file(&oid, FIXTURE_DIR "/shattered-1.pdf"));
+	git_oid_fromstr(&expected, "38762cf7f55934b34d179ae6a4c80cadccbb7f0a");
+	cl_assert_equal_oid(&expected, &oid);
+#endif
+}
+

tests/resources/sha1/hello_c

+#include <stdio.h>
+
+int main(int argc, char **argv)
+{
+	printf("Hello, %s\n", "world");
+}