alloc: make memory allocators use function pointers

Currently, our memory allocators are being redirected to the correct
implementation at compile time by simply using macros. In order to make
them swappable at runtime, this commit reshuffles that by instead making
use of a global "git_allocator" structure, whose pointers are set up to
reference the allocator functions. Like this, it becomes easy to swap
out allocators by simply setting these function pointers.

In order to initialize a "git_allocator", our provided allocators
"stdalloc" and "crtdbg" both provide an init function. This is being
called to initialize a passed in allocator struct and set up its members
correctly.

No support is yet included to enable users of libgit2 to switch out the
memory allocator at a global level.

include/git2/sys/alloc.h

+/*
+ * Copyright (C) the libgit2 contributors. All rights reserved.
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+
+#ifndef INCLUDE_sys_git_alloc_h__
+#define INCLUDE_sys_git_alloc_h__
+
+#include "git2/common.h"
+
+GIT_BEGIN_DECL
+
+/**
+ * An instance for a custom memory allocator
+ *
+ * Setting the pointers of this structure allows the developer to implement
+ * custom memory allocators. The global memory allocator can be set by using
+ * "GIT_OPT_SET_ALLOCATOR" with the `git_libgit2_opts` function. Keep in mind
+ * that all fields need to be set to a proper function.
+ */
+typedef struct {
+	/* Allocate `n` bytes of memory */
+	void *(*gmalloc)(size_t n, const char *file, int line);
+
+	/*
+	 * Allocate memory for an array of `nelem` elements, where each element
+	 * has a size of `elsize`. Returned memory shall be initialized to
+	 * all-zeroes
+	 */
+	void *(*gcalloc)(size_t nelem, size_t elsize, const char *file, int line);
+
+	/* Allocate memory for the string `str` and duplicate its contents. */
+	char *(*gstrdup)(const char *str, const char *file, int line);
+
+	/*
+	 * Equivalent to the `gstrdup` function, but only duplicating at most
+	 * `n + 1` bytes
+	 */
+	char *(*gstrndup)(const char *str, size_t n, const char *file, int line);
+
+	/*
+	 * Equivalent to `gstrndup`, but will always duplicate exactly `n` bytes
+	 * of `str`. Thus, out of bounds reads at `str` may happen.
+	 */
+	char *(*gsubstrdup)(const char *str, size_t n, const char *file, int line);
+
+	/*
+	 * This function shall deallocate the old object `ptr` and return a
+	 * pointer to a new object that has the size specified by `size`. In
+	 * case `ptr` is `NULL`, a new array shall be allocated.
+	 */
+	void *(*grealloc)(void *ptr, size_t size, const char *file, int line);
+
+	/*
+	 * This function shall be equivalent to `grealloc`, but allocating
+	 * `neleme * elsize` bytes.
+	 */
+	void *(*greallocarray)(void *ptr, size_t nelem, size_t elsize, const char *file, int line);
+
+	/*
+	 * This function shall allocate a new array of `nelem` elements, where
+	 * each element has a size of `elsize` bytes.
+	 */
+	void *(*gmallocarray)(size_t nelem, size_t elsize, const char *file, int line);
+
+	/*
+	 * This function shall free the memory pointed to by `ptr`. In case
+	 * `ptr` is `NULL`, this shall be a no-op.
+	 */
+	void (*gfree)(void *ptr);
+} git_allocator;
+
+GIT_END_DECL
+
+#endif

src/alloc.c

+/*
+ * Copyright (C) the libgit2 contributors. All rights reserved.
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+
+#include "alloc.h"
+
+#if defined(GIT_MSVC_CRTDBG)
+# include "win32/w32_crtdbg_stacktrace.h"
+#else
+# include "stdalloc.h"
+#endif
+
+git_allocator git__allocator;
+
+int git_allocator_global_init(void)
+{
+#if defined(GIT_MSVC_CRTDBG)
+	return git_win32_crtdbg_init_allocator(&git__allocator);
+#else
+	return git_stdalloc_init_allocator(&git__allocator);
+#endif
+}
+
+int git_allocator_setup(git_allocator *allocator)
+{
+	memcpy(&git__allocator, allocator, sizeof(*allocator));
+	return 0;
+}

src/alloc.h

@@ -8,62 +8,33 @@
 #ifndef INCLUDE_alloc_h__
 #define INCLUDE_alloc_h__
 
-#include "common.h"
-
-#if defined(GIT_MSVC_CRTDBG)
+#include "git2/sys/alloc.h"
+
+extern git_allocator git__allocator;
+
+#define git__malloc(len)                      git__allocator.gmalloc(len, __FILE__, __LINE__)
+#define git__calloc(nelem, elsize)            git__allocator.gcalloc(nelem, elsize, __FILE__, __LINE__)
+#define git__strdup(str)                      git__allocator.gstrdup(str, __FILE__, __LINE__)
+#define git__strndup(str, n)                  git__allocator.gstrndup(str, n, __FILE__, __LINE__)
+#define git__substrdup(str, n)                git__allocator.gsubstrdup(str, n, __FILE__, __LINE__)
+#define git__realloc(ptr, size)               git__allocator.grealloc(ptr, size, __FILE__, __LINE__)
+#define git__reallocarray(ptr, nelem, elsize) git__allocator.greallocarray(ptr, nelem, elsize, __FILE__, __LINE__)
+#define git__mallocarray(nelem, elsize)       git__allocator.gmallocarray(nelem, elsize, __FILE__, __LINE__)
+#define git__free                             git__allocator.gfree
+
+/**
+ * This function is being called by our global setup routines to
+ * initialize the standard allocator.
+ */
+int git_allocator_global_init(void);
 
-/* Enable MSVC CRTDBG memory leak reporting.
- *
- * We DO NOT use the "_CRTDBG_MAP_ALLOC" macro described in the MSVC
- * documentation because all allocs/frees in libgit2 already go through
- * the "git__" routines defined in this file.  Simply using the normal
- * reporting mechanism causes all leaks to be attributed to a routine
- * here in util.h (ie, the actual call to calloc()) rather than the
- * caller of git__calloc().
- *
- * Therefore, we declare a set of "git__crtdbg__" routines to replace
- * the corresponding "git__" routines and re-define the "git__" symbols
- * as macros.  This allows us to get and report the file:line info of
- * the real caller.
- *
- * We DO NOT replace the "git__free" routine because it needs to remain
- * a function pointer because it is used as a function argument when
- * setting up various structure "destructors".
- *
- * We also DO NOT use the "_CRTDBG_MAP_ALLOC" macro because it causes
- * "free" to be remapped to "_free_dbg" and this causes problems for
- * structures which define a field named "free".
+/**
+ * Switch out libgit2's global memory allocator
  *
- * Finally, CRTDBG must be explicitly enabled and configured at program
- * startup.  See tests/main.c for an example.
+ * @param allocator The new allocator that should be used. All function pointers
+ *                  of it need to be set correctly.
+ * @return An error code or 0.
  */
-
-#include "win32/w32_crtdbg_stacktrace.h"
-
-#define git__malloc(len)                      git__crtdbg__malloc(len, __FILE__, __LINE__)
-#define git__calloc(nelem, elsize)            git__crtdbg__calloc(nelem, elsize, __FILE__, __LINE__)
-#define git__strdup(str)                      git__crtdbg__strdup(str, __FILE__, __LINE__)
-#define git__strndup(str, n)                  git__crtdbg__strndup(str, n, __FILE__, __LINE__)
-#define git__substrdup(str, n)                git__crtdbg__substrdup(str, n, __FILE__, __LINE__)
-#define git__realloc(ptr, size)               git__crtdbg__realloc(ptr, size, __FILE__, __LINE__)
-#define git__reallocarray(ptr, nelem, elsize) git__crtdbg__reallocarray(ptr, nelem, elsize, __FILE__, __LINE__)
-#define git__mallocarray(nelem, elsize)       git__crtdbg__mallocarray(nelem, elsize, __FILE__, __LINE__)
-#define git__free                             git__crtdbg__free
-
-#else
-
-#include "stdalloc.h"
-
-#define git__malloc(len)                      git__stdalloc__malloc(len, __FILE__, __LINE__)
-#define git__calloc(nelem, elsize)            git__stdalloc__calloc(nelem, elsize, __FILE__, __LINE__)
-#define git__strdup(str)                      git__stdalloc__strdup(str, __FILE__, __LINE__)
-#define git__strndup(str, n)                  git__stdalloc__strndup(str, n, __FILE__, __LINE__)
-#define git__substrdup(str, n)                git__stdalloc__substrdup(str, n, __FILE__, __LINE__)
-#define git__realloc(ptr, size)               git__stdalloc__realloc(ptr, size, __FILE__, __LINE__)
-#define git__reallocarray(ptr, nelem, elsize) git__stdalloc__reallocarray(ptr, nelem, elsize, __FILE__, __LINE__)
-#define git__mallocarray(nelem, elsize)       git__stdalloc__mallocarray(nelem, elsize, __FILE__, __LINE__)
-#define git__free                             git__stdalloc__free
-
-#endif /* !MSVC_CTRDBG */
+int git_allocator_setup(git_allocator *allocator);
 
 #endif

src/global.c

@@ -7,6 +7,7 @@
 
 #include "global.h"
 
+#include "alloc.h"
 #include "hash.h"
 #include "sysdir.h"
 #include "filter.h"
@@ -60,7 +61,8 @@ static int init_common(void)
 #endif
 
 	/* Initialize any other subsystems that have global state */
-	if ((ret = git_hash_global_init()) == 0 &&
+	if ((ret = git_allocator_global_init()) == 0 &&
+		(ret = git_hash_global_init()) == 0 &&
 		(ret = git_sysdir_global_init()) == 0 &&
 		(ret = git_filter_global_init()) == 0 &&
 		(ret = git_merge_driver_global_init()) == 0 &&

src/stdalloc.c

@@ -7,7 +7,7 @@
 
 #include "stdalloc.h"
 
-void *git__stdalloc__malloc(size_t len, const char *file, int line)
+static void *stdalloc__malloc(size_t len, const char *file, int line)
 {
 	void *ptr = malloc(len);
 
@@ -18,7 +18,7 @@ void *git__stdalloc__malloc(size_t len, const char *file, int line)
 	return ptr;
 }
 
-void *git__stdalloc__calloc(size_t nelem, size_t elsize, const char *file, int line)
+static void *stdalloc__calloc(size_t nelem, size_t elsize, const char *file, int line)
 {
 	void *ptr = calloc(nelem, elsize);
 
@@ -29,7 +29,7 @@ void *git__stdalloc__calloc(size_t nelem, size_t elsize, const char *file, int l
 	return ptr;
 }
 
-char *git__stdalloc__strdup(const char *str, const char *file, int line)
+static char *stdalloc__strdup(const char *str, const char *file, int line)
 {
 	char *ptr = strdup(str);
 
@@ -40,7 +40,7 @@ char *git__stdalloc__strdup(const char *str, const char *file, int line)
 	return ptr;
 }
 
-char *git__stdalloc__strndup(const char *str, size_t n, const char *file, int line)
+static char *stdalloc__strndup(const char *str, size_t n, const char *file, int line)
 {
 	size_t length = 0, alloclength;
 	char *ptr;
@@ -48,7 +48,7 @@ char *git__stdalloc__strndup(const char *str, size_t n, const char *file, int li
 	length = p_strnlen(str, n);
 
 	if (GIT_ADD_SIZET_OVERFLOW(&alloclength, length, 1) ||
-		!(ptr = git__stdalloc__malloc(alloclength, file, line)))
+		!(ptr = stdalloc__malloc(alloclength, file, line)))
 		return NULL;
 
 	if (length)
@@ -59,13 +59,13 @@ char *git__stdalloc__strndup(const char *str, size_t n, const char *file, int li
 	return ptr;
 }
 
-char *git__stdalloc__substrdup(const char *start, size_t n, const char *file, int line)
+static char *stdalloc__substrdup(const char *start, size_t n, const char *file, int line)
 {
 	char *ptr;
 	size_t alloclen;
 
 	if (GIT_ADD_SIZET_OVERFLOW(&alloclen, n, 1) ||
-		!(ptr = git__stdalloc__malloc(alloclen, file, line)))
+		!(ptr = stdalloc__malloc(alloclen, file, line)))
 		return NULL;
 
 	memcpy(ptr, start, n);
@@ -73,7 +73,7 @@ char *git__stdalloc__substrdup(const char *start, size_t n, const char *file, in
 	return ptr;
 }
 
-void *git__stdalloc__realloc(void *ptr, size_t size, const char *file, int line)
+static void *stdalloc__realloc(void *ptr, size_t size, const char *file, int line)
 {
 	void *new_ptr = realloc(ptr, size);
 
@@ -84,7 +84,7 @@ void *git__stdalloc__realloc(void *ptr, size_t size, const char *file, int line)
 	return new_ptr;
 }
 
-void *git__stdalloc__reallocarray(void *ptr, size_t nelem, size_t elsize, const char *file, int line)
+static void *stdalloc__reallocarray(void *ptr, size_t nelem, size_t elsize, const char *file, int line)
 {
 	size_t newsize;
 
@@ -95,12 +95,26 @@ void *git__stdalloc__reallocarray(void *ptr, size_t nelem, size_t elsize, const
 		NULL : realloc(ptr, newsize);
 }
 
-void *git__stdalloc__mallocarray(size_t nelem, size_t elsize, const char *file, int line)
+static void *stdalloc__mallocarray(size_t nelem, size_t elsize, const char *file, int line)
 {
-	return git__stdalloc__reallocarray(NULL, nelem, elsize, file, line);
+	return stdalloc__reallocarray(NULL, nelem, elsize, file, line);
 }
 
-void git__stdalloc__free(void *ptr)
+static void stdalloc__free(void *ptr)
 {
 	free(ptr);
 }
+
+int git_stdalloc_init_allocator(git_allocator *allocator)
+{
+	allocator->gmalloc = stdalloc__malloc;
+	allocator->gcalloc = stdalloc__calloc;
+	allocator->gstrdup = stdalloc__strdup;
+	allocator->gstrndup = stdalloc__strndup;
+	allocator->gsubstrdup = stdalloc__substrdup;
+	allocator->grealloc = stdalloc__realloc;
+	allocator->greallocarray = stdalloc__reallocarray;
+	allocator->gmallocarray = stdalloc__mallocarray;
+	allocator->gfree = stdalloc__free;
+	return 0;
+}

src/stdalloc.h

@@ -8,33 +8,10 @@
 #ifndef INCLUDE_stdalloc_h__
 #define INCLUDE_stdalloc_h__
 
-#include "common.h"
-
-/*
- * Custom memory allocation wrappers
- * that set error code and error message
- * on allocation failure
- */
-void *git__stdalloc__malloc(size_t len, const char *file, int line);
-void *git__stdalloc__calloc(size_t nelem, size_t elsize, const char *file, int line);
-char *git__stdalloc__strdup(const char *str, const char *file, int line);
-char *git__stdalloc__strndup(const char *str, size_t n, const char *file, int line);
-/* NOTE: This doesn't do null or '\0' checking.  Watch those boundaries! */
-char *git__stdalloc__substrdup(const char *start, size_t n, const char *file, int line);
-void *git__stdalloc__realloc(void *ptr, size_t size, const char *file, int line);
+#include "alloc.h"
 
-/**
- * Similar to `git__stdalloc__realloc`, except that it is suitable for reallocing an
- * array to a new number of elements of `nelem`, each of size `elsize`.
- * The total size calculation is checked for overflow.
- */
-void *git__stdalloc__reallocarray(void *ptr, size_t nelem, size_t elsize, const char *file, int line);
-
-/**
- * Similar to `git__stdalloc__calloc`, except that it does not zero memory.
- */
-void *git__stdalloc__mallocarray(size_t nelem, size_t elsize, const char *file, int line);
+#include "common.h"
 
-void git__stdalloc__free(void *ptr);
+int git_stdalloc_init_allocator(git_allocator *allocator);
 
 #endif

src/win32/w32_crtdbg_stacktrace.c

@@ -71,28 +71,28 @@ static bool g_limit_reached = false; /* had allocs after we filled row table */
 static unsigned int g_checkpoint_id = 0; /* to better label leak checkpoints */
 static bool g_transient_leaks_since_mark = false; /* payload for hook */
 
-void *git__crtdbg__malloc(size_t len, const char *file, int line)
+static void *crtdbg__malloc(size_t len, const char *file, int line)
 {
 	void *ptr = _malloc_dbg(len, _NORMAL_BLOCK, git_win32__crtdbg_stacktrace(1,file), line);
 	if (!ptr) giterr_set_oom();
 	return ptr;
 }
 
-void *git__crtdbg__calloc(size_t nelem, size_t elsize, const char *file, int line)
+static void *crtdbg__calloc(size_t nelem, size_t elsize, const char *file, int line)
 {
 	void *ptr = _calloc_dbg(nelem, elsize, _NORMAL_BLOCK, git_win32__crtdbg_stacktrace(1,file), line);
 	if (!ptr) giterr_set_oom();
 	return ptr;
 }
 
-char *git__crtdbg__strdup(const char *str, const char *file, int line)
+static char *crtdbg__strdup(const char *str, const char *file, int line)
 {
 	char *ptr = _strdup_dbg(str, _NORMAL_BLOCK, git_win32__crtdbg_stacktrace(1,file), line);
 	if (!ptr) giterr_set_oom();
 	return ptr;
 }
 
-char *git__crtdbg__strndup(const char *str, size_t n, const char *file, int line)
+static char *crtdbg__strndup(const char *str, size_t n, const char *file, int line)
 {
 	size_t length = 0, alloclength;
 	char *ptr;
@@ -100,7 +100,7 @@ char *git__crtdbg__strndup(const char *str, size_t n, const char *file, int line
 	length = p_strnlen(str, n);
 
 	if (GIT_ADD_SIZET_OVERFLOW(&alloclength, length, 1) ||
-		!(ptr = git__crtdbg__malloc(alloclength, file, line)))
+		!(ptr = crtdbg__malloc(alloclength, file, line)))
 		return NULL;
 
 	if (length)
@@ -111,13 +111,13 @@ char *git__crtdbg__strndup(const char *str, size_t n, const char *file, int line
 	return ptr;
 }
 
-char *git__crtdbg__substrdup(const char *start, size_t n, const char *file, int line)
+static char *crtdbg__substrdup(const char *start, size_t n, const char *file, int line)
 {
 	char *ptr;
 	size_t alloclen;
 
 	if (GIT_ADD_SIZET_OVERFLOW(&alloclen, n, 1) ||
-		!(ptr = git__crtdbg__malloc(alloclen, file, line)))
+		!(ptr = crtdbg__malloc(alloclen, file, line)))
 		return NULL;
 
 	memcpy(ptr, start, n);
@@ -125,14 +125,14 @@ char *git__crtdbg__substrdup(const char *start, size_t n, const char *file, int
 	return ptr;
 }
 
-void *git__crtdbg__realloc(void *ptr, size_t size, const char *file, int line)
+static void *crtdbg__realloc(void *ptr, size_t size, const char *file, int line)
 {
 	void *new_ptr = _realloc_dbg(ptr, size, _NORMAL_BLOCK, git_win32__crtdbg_stacktrace(1,file), line);
 	if (!new_ptr) giterr_set_oom();
 	return new_ptr;
 }
 
-void *git__crtdbg__reallocarray(void *ptr, size_t nelem, size_t elsize, const char *file, int line)
+static void *crtdbg__reallocarray(void *ptr, size_t nelem, size_t elsize, const char *file, int line)
 {
 	size_t newsize;
 
@@ -140,16 +140,30 @@ void *git__crtdbg__reallocarray(void *ptr, size_t nelem, size_t elsize, const ch
 		NULL : _realloc_dbg(ptr, newsize, _NORMAL_BLOCK, git_win32__crtdbg_stacktrace(1,file), line);
 }
 
-void *git__crtdbg__mallocarray(size_t nelem, size_t elsize, const char *file, int line)
+static void *crtdbg__mallocarray(size_t nelem, size_t elsize, const char *file, int line)
 {
-	return git__crtdbg__reallocarray(NULL, nelem, elsize, file, line);
+	return crtdbg__reallocarray(NULL, nelem, elsize, file, line);
 }
 
-void git__crtdbg__free(void *ptr)
+static void crtdbg__free(void *ptr)
 {
 	free(ptr);
 }
 
+int git_win32_crtdbg_init_allocator(git_allocator *allocator)
+{
+	allocator->gmalloc = crtdbg__malloc;
+	allocator->gcalloc = crtdbg__calloc;
+	allocator->gstrdup = crtdbg__strdup;
+	allocator->gstrndup = crtdbg__strndup;
+	allocator->gsubstrdup = crtdbg__substrdup;
+	allocator->grealloc = crtdbg__realloc;
+	allocator->greallocarray = crtdbg__reallocarray;
+	allocator->gmallocarray = crtdbg__mallocarray;
+	allocator->gfree = crtdbg__free;
+	return 0;
+}
+
 /**
  * Compare function for bsearch on g_cs_index table.
  */

src/win32/w32_crtdbg_stacktrace.h

@@ -17,6 +17,34 @@
 #include "git2/errors.h"
 #include "strnlen.h"
 
+/* MSVC CRTDBG memory leak reporting.
+ *
+ * We DO NOT use the "_CRTDBG_MAP_ALLOC" macro described in the MSVC
+ * documentation because all allocs/frees in libgit2 already go through
+ * the "git__" routines defined in this file.  Simply using the normal
+ * reporting mechanism causes all leaks to be attributed to a routine
+ * here in util.h (ie, the actual call to calloc()) rather than the
+ * caller of git__calloc().
+ *
+ * Therefore, we declare a set of "git__crtdbg__" routines to replace
+ * the corresponding "git__" routines and re-define the "git__" symbols
+ * as macros.  This allows us to get and report the file:line info of
+ * the real caller.
+ *
+ * We DO NOT replace the "git__free" routine because it needs to remain
+ * a function pointer because it is used as a function argument when
+ * setting up various structure "destructors".
+ *
+ * We also DO NOT use the "_CRTDBG_MAP_ALLOC" macro because it causes
+ * "free" to be remapped to "_free_dbg" and this causes problems for
+ * structures which define a field named "free".
+ *
+ * Finally, CRTDBG must be explicitly enabled and configured at program
+ * startup.  See tests/main.c for an example.
+ */
+
+int git_win32_crtdbg_init_allocator(git_allocator *allocator);
+
 /**
  * Initialize our memory leak tracking and de-dup data structures.
  * This should ONLY be called by git_libgit2_init().
@@ -97,15 +125,5 @@ GIT_EXTERN(int) git_win32__crtdbg_stacktrace__dump(
  */
 const char *git_win32__crtdbg_stacktrace(int skip, const char *file);
 
-void *git__crtdbg__malloc(size_t len, const char *file, int line);
-void *git__crtdbg__calloc(size_t nelem, size_t elsize, const char *file, int line);
-char *git__crtdbg__strdup(const char *str, const char *file, int line);
-char *git__crtdbg__strndup(const char *str, size_t n, const char *file, int line);
-char *git__crtdbg__substrdup(const char *start, size_t n, const char *file, int line);
-void *git__crtdbg__realloc(void *ptr, size_t size, const char *file, int line);
-void *git__crtdbg__reallocarray(void *ptr, size_t nelem, size_t elsize, const char *file, int line);
-void *git__crtdbg__mallocarray(size_t nelem, size_t elsize, const char *file, int line);
-void *git__crtdbg__free(void *ptr);
-
 #endif
 #endif

tests/trace/windows/stacktrace.c

 #include "clar_libgit2.h"
 #include "win32/w32_stack.h"
+#include "win32/w32_crtdbg_stacktrace.h"
 
 #if defined(GIT_MSVC_CRTDBG)
 static void a(void)