Introduce git__add_sizet_overflow and friends

Add some helper functions to check for overflow in a type-specific
manner.

src/common.h

@@ -58,6 +58,7 @@
 #include "git2/types.h"
 #include "git2/errors.h"
 #include "thread-utils.h"
+#include "integer.h"
 
 #include <regex.h>
 
@@ -174,13 +175,14 @@ GIT_INLINE(void) git__init_structure(void *structure, size_t len, unsigned int v
 	GITERR_CHECK_VERSION(&(VERSION), _tmpl.version, #TYPE);	\
 	memcpy((PTR), &_tmpl, sizeof(_tmpl)); } while (0)
 
+
 /** Check for integer overflow from addition or multiplication */
 #define GIT_ALLOC_OVERFLOW_ADD(one, two) \
-	(((one) + (two) < (one)) ? (giterr_set_oom(), 1) : 0)
+	(!git__add_sizet_overflow(NULL, (one), (two)) ? (giterr_set_oom(), 1) : 0)
 
 /** Check for integer overflow from multiplication */
 #define GIT_ALLOC_OVERFLOW_MULTIPLY(one, two) \
-	((one && ((one) * (two)) / (one) != (two)) ? (giterr_set_oom(), 1) : 0)
+	(!git__multiply_sizet_overflow(NULL, (one), (two)) ? (giterr_set_oom(), 1) : 0)
 
 /** Check for additive overflow, failing if it would occur. */
 #define GITERR_CHECK_ALLOC_ADD(one, two) \

src/integer.h

+/*
+ * Copyright (C) the libgit2 contributors. All rights reserved.
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#ifndef INCLUDE_integer_h__
+#define INCLUDE_integer_h__
+
+/** @return true if p fits into the range of a size_t */
+GIT_INLINE(int) git__is_sizet(git_off_t p)
+{
+	size_t r = (size_t)p;
+	return p == (git_off_t)r;
+}
+
+/** @return true if p fits into the range of an ssize_t */
+GIT_INLINE(int) git__is_ssizet(size_t p)
+{
+	ssize_t r = (ssize_t)p;
+	return p == (size_t)r;
+}
+
+/** @return true if p fits into the range of a uint32_t */
+GIT_INLINE(int) git__is_uint32(size_t p)
+{
+	uint32_t r = (uint32_t)p;
+	return p == (size_t)r;
+}
+
+/** @return true if p fits into the range of an unsigned long */
+GIT_INLINE(int) git__is_ulong(git_off_t p)
+{
+	unsigned long r = (unsigned long)p;
+	return p == (git_off_t)r;
+}
+
+/**
+ * Sets `one + two` into `out`, unless the arithmetic would overflow.
+ * @return true if the result fits in a `uint64_t`, false on overflow.
+ */
+GIT_INLINE(bool) git__add_uint64_overflow(uint64_t *out, uint64_t one, uint64_t two)
+{
+	if (UINT64_MAX - one < two)
+		return false;
+	if (out)
+		*out = one + two;
+	return true;
+}
+
+/**
+ * Sets `one + two` into `out`, unless the arithmetic would overflow.
+ * @return true if the result fits in a `size_t`, false on overflow.
+ */
+GIT_INLINE(bool) git__add_sizet_overflow(size_t *out, size_t one, size_t two)
+{
+	if (SIZE_MAX - one < two)
+		return false;
+	if (out)
+		*out = one + two;
+	return true;
+}
+
+/**
+ * Sets `one * two` into `out`, unless the arithmetic would overflow.
+ * @return true if the result fits in a `size_t`, false on overflow.
+ */
+GIT_INLINE(bool) git__multiply_sizet_overflow(size_t *out, size_t one, size_t two)
+{
+	if (one && SIZE_MAX / one < two)
+		return false;
+	if (out)
+		*out = one * two;
+	return true;
+}
+
+#endif /* INCLUDE_integer_h__ */

src/pack-objects.c

@@ -833,8 +833,8 @@ static int try_delta(git_packbuilder *pb, struct unpacked *trg,
 		trg_object->delta_data = NULL;
 	}
 	if (delta_cacheable(pb, src_size, trg_size, delta_size)) {
-		GITERR_CHECK_ALLOC_ADD(pb->delta_cache_size, delta_size);
-		pb->delta_cache_size += delta_size;
+		if (!git__add_uint64_overflow(&pb->delta_cache_size, pb->delta_cache_size, delta_size))
+			return -1;
 
 		git_packbuilder__cache_unlock(pb);
 

src/util.h

@@ -146,34 +146,6 @@ extern int git__strtol64(int64_t *n, const char *buff, const char **end_buf, int
 extern void git__hexdump(const char *buffer, size_t n);
 extern uint32_t git__hash(const void *key, int len, uint32_t seed);
 
-/** @return true if p fits into the range of a size_t */
-GIT_INLINE(int) git__is_sizet(git_off_t p)
-{
-	size_t r = (size_t)p;
-	return p == (git_off_t)r;
-}
-
-/** @return true if p fits into the range of an ssize_t */
-GIT_INLINE(int) git__is_ssizet(size_t p)
-{
-	ssize_t r = (ssize_t)p;
-	return p == (size_t)r;
-}
-
-/** @return true if p fits into the range of a uint32_t */
-GIT_INLINE(int) git__is_uint32(size_t p)
-{
-	uint32_t r = (uint32_t)p;
-	return p == (size_t)r;
-}
-
-/** @return true if p fits into the range of an unsigned long */
-GIT_INLINE(int) git__is_ulong(git_off_t p)
-{
-	unsigned long r = (unsigned long)p;
-	return p == (git_off_t)r;
-}
-
 /* 32-bit cross-platform rotl */
 #ifdef _MSC_VER /* use built-in method in MSVC */
 #	define git__rotl(v, s) (uint32_t)_rotl(v, s)