Merge pull request #5594 from lhchavez/git-atomics

Improve the support of atomics

src/cache.c

@@ -168,7 +168,7 @@ static void *cache_store(git_cache *cache, git_cached_obj *entry)
 		return entry;
 
 	/* soften the load on the cache */
-	if (git_cache__current_storage.val > git_cache__max_storage)
+	if (git_atomic_ssize_get(&git_cache__current_storage) > git_cache__max_storage)
 		cache_evict_entries(cache);
 
 	/* not found */

src/global.c

@@ -12,6 +12,7 @@
 #include "sysdir.h"
 #include "filter.h"
 #include "merge_driver.h"
+#include "pool.h"
 #include "streams/registry.h"
 #include "streams/mbedtls.h"
 #include "streams/openssl.h"
@@ -38,7 +39,8 @@ static git_global_init_fn git__init_callbacks[] = {
 	git_stream_registry_global_init,
 	git_openssl_stream_global_init,
 	git_mbedtls_stream_global_init,
-	git_mwindow_global_init
+	git_mwindow_global_init,
+	git_pool_global_init
 };
 
 static git_global_shutdown_fn git__shutdown_callbacks[ARRAY_SIZE(git__init_callbacks)];

src/odb.c

@@ -44,8 +44,8 @@ typedef struct
 
 static git_cache *odb_cache(git_odb *odb)
 {
-	if (odb->rc.owner != NULL) {
-		git_repository *owner = odb->rc.owner;
+	git_repository *owner = GIT_REFCOUNT_OWNER(odb);
+	if (owner != NULL) {
 		return &owner->objects;
 	}
 
@@ -664,7 +664,7 @@ int git_odb_open(git_odb **out, const char *objects_dir)
 int git_odb__set_caps(git_odb *odb, int caps)
 {
 	if (caps == GIT_ODB_CAP_FROM_OWNER) {
-		git_repository *repo = odb->rc.owner;
+		git_repository *repo = GIT_REFCOUNT_OWNER(odb);
 		int val;
 
 		if (!repo) {

src/pool.c

@@ -21,22 +21,19 @@ struct git_pool_page {
 
 static void *pool_alloc_page(git_pool *pool, size_t size);
 
-static size_t pool_system_page_size(void)
-{
-	static size_t size = 0;
+#ifndef GIT_DEBUG_POOL
 
-	if (!size) {
-		size_t page_size;
-		if (git__page_size(&page_size) < 0)
-			page_size = 4096;
-		/* allow space for malloc overhead */
-		size = (page_size - (2 * sizeof(void *)) - sizeof(git_pool_page));
-	}
+static size_t system_page_size = 0;
 
-	return size;
+int git_pool_global_init(void)
+{
+	if (git__page_size(&system_page_size) < 0)
+		system_page_size = 4096;
+	/* allow space for malloc overhead */
+	system_page_size -= (2 * sizeof(void *)) + sizeof(git_pool_page);
+	return 0;
 }
 
-#ifndef GIT_DEBUG_POOL
 int git_pool_init(git_pool *pool, size_t item_size)
 {
 	assert(pool);
@@ -44,7 +41,7 @@ int git_pool_init(git_pool *pool, size_t item_size)
 
 	memset(pool, 0, sizeof(git_pool));
 	pool->item_size = item_size;
-	pool->page_size = pool_system_page_size();
+	pool->page_size = system_page_size;
 
 	return 0;
 }
@@ -114,6 +111,11 @@ bool git_pool__ptr_in_pool(git_pool *pool, void *ptr)
 
 #else
 
+int git_pool_global_init(void)
+{
+	return 0;
+}
+
 static int git_pool__ptr_cmp(const void * a, const void * b)
 {
 	if(a > b) {

src/pool.h

@@ -135,4 +135,12 @@ extern uint32_t git_pool__open_pages(git_pool *pool);
 #endif
 extern bool git_pool__ptr_in_pool(git_pool *pool, void *ptr);
 
+/**
+ * This function is being called by our global setup routines to
+ * initialize the system pool size.
+ *
+ * @return 0 on success, <0 on failure
+ */
+extern int git_pool_global_init(void);
+
 #endif

src/thread-utils.h

@@ -7,12 +7,30 @@
 #ifndef INCLUDE_thread_utils_h__
 #define INCLUDE_thread_utils_h__
 
-#if defined(__GNUC__) && defined(GIT_THREADS)
+#if defined(GIT_THREADS)
+
+#if defined(__clang__)
+
+# if (__clang_major__ < 3 || (__clang_major__ == 3 && __clang_minor__ < 1))
+#  error Atomic primitives do not exist on this version of clang; configure libgit2 with -DTHREADSAFE=OFF
+# else
+#  define GIT_BUILTIN_ATOMIC
+# endif
+
+#elif defined(__GNUC__)
+
 # if (__GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 1))
 #  error Atomic primitives do not exist on this version of gcc; configure libgit2 with -DTHREADSAFE=OFF
+# elif (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 7))
+#  define GIT_BUILTIN_ATOMIC
+# else
+#  define GIT_BUILTIN_SYNC
 # endif
+
 #endif
 
+#endif /* GIT_THREADS */
+
 /* Common operations even if threading has been disabled */
 typedef struct {
 #if defined(GIT_WIN32)
@@ -26,21 +44,25 @@ typedef struct {
 
 typedef struct {
 #if defined(GIT_WIN32)
-	__int64 val;
+	volatile __int64 val;
 #else
-	int64_t val;
+	volatile int64_t val;
 #endif
 } git_atomic64;
 
 typedef git_atomic64 git_atomic_ssize;
 
+#define git_atomic_ssize_set git_atomic64_set
 #define git_atomic_ssize_add git_atomic64_add
+#define git_atomic_ssize_get git_atomic64_get
 
 #else
 
 typedef git_atomic git_atomic_ssize;
 
+#define git_atomic_ssize_set git_atomic_set
 #define git_atomic_ssize_add git_atomic_add
+#define git_atomic_ssize_get git_atomic_get
 
 #endif
 
@@ -56,7 +78,9 @@ GIT_INLINE(void) git_atomic_set(git_atomic *a, int val)
 {
 #if defined(GIT_WIN32)
 	InterlockedExchange(&a->val, (LONG)val);
-#elif defined(__GNUC__)
+#elif defined(GIT_BUILTIN_ATOMIC)
+	__atomic_store_n(&a->val, val, __ATOMIC_SEQ_CST);
+#elif defined(GIT_BUILTIN_SYNC)
 	__sync_lock_test_and_set(&a->val, val);
 #else
 #	error "Unsupported architecture for atomic operations"
@@ -67,7 +91,9 @@ GIT_INLINE(int) git_atomic_inc(git_atomic *a)
 {
 #if defined(GIT_WIN32)
 	return InterlockedIncrement(&a->val);
-#elif defined(__GNUC__)
+#elif defined(GIT_BUILTIN_ATOMIC)
+	return __atomic_add_fetch(&a->val, 1, __ATOMIC_SEQ_CST);
+#elif defined(GIT_BUILTIN_SYNC)
 	return __sync_add_and_fetch(&a->val, 1);
 #else
 #	error "Unsupported architecture for atomic operations"
@@ -78,7 +104,9 @@ GIT_INLINE(int) git_atomic_add(git_atomic *a, int32_t addend)
 {
 #if defined(GIT_WIN32)
 	return InterlockedExchangeAdd(&a->val, addend);
-#elif defined(__GNUC__)
+#elif defined(GIT_BUILTIN_ATOMIC)
+	return __atomic_add_fetch(&a->val, addend, __ATOMIC_SEQ_CST);
+#elif defined(GIT_BUILTIN_SYNC)
 	return __sync_add_and_fetch(&a->val, addend);
 #else
 #	error "Unsupported architecture for atomic operations"
@@ -89,25 +117,45 @@ GIT_INLINE(int) git_atomic_dec(git_atomic *a)
 {
 #if defined(GIT_WIN32)
 	return InterlockedDecrement(&a->val);
-#elif defined(__GNUC__)
+#elif defined(GIT_BUILTIN_ATOMIC)
+	return __atomic_sub_fetch(&a->val, 1, __ATOMIC_SEQ_CST);
+#elif defined(GIT_BUILTIN_SYNC)
 	return __sync_sub_and_fetch(&a->val, 1);
 #else
 #	error "Unsupported architecture for atomic operations"
 #endif
 }
 
+GIT_INLINE(int) git_atomic_get(git_atomic *a)
+{
+#if defined(GIT_WIN32)
+	return (int)InterlockedCompareExchange(&a->val, 0, 0);
+#elif defined(GIT_BUILTIN_ATOMIC)
+	return __atomic_load_n(&a->val, __ATOMIC_SEQ_CST);
+#elif defined(GIT_BUILTIN_SYNC)
+	return __sync_val_compare_and_swap(&a->val, 0, 0);
+#else
+#	error "Unsupported architecture for atomic operations"
+#endif
+}
+
 GIT_INLINE(void *) git___compare_and_swap(
 	void * volatile *ptr, void *oldval, void *newval)
 {
-	volatile void *foundval;
 #if defined(GIT_WIN32)
+	volatile void *foundval;
 	foundval = InterlockedCompareExchangePointer((volatile PVOID *)ptr, newval, oldval);
-#elif defined(__GNUC__)
+	return (foundval == oldval) ? oldval : newval;
+#elif defined(GIT_BUILTIN_ATOMIC)
+	bool success = __atomic_compare_exchange(ptr, &oldval, &newval, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
+	return success ? oldval : newval;
+#elif defined(GIT_BUILTIN_SYNC)
+	volatile void *foundval;
 	foundval = __sync_val_compare_and_swap(ptr, oldval, newval);
+	return (foundval == oldval) ? oldval : newval;
 #else
 #	error "Unsupported architecture for atomic operations"
 #endif
-	return (foundval == oldval) ? oldval : newval;
 }
 
 GIT_INLINE(volatile void *) git___swap(
@@ -115,8 +163,30 @@ GIT_INLINE(volatile void *) git___swap(
 {
 #if defined(GIT_WIN32)
 	return InterlockedExchangePointer(ptr, newval);
-#else
+#elif defined(GIT_BUILTIN_ATOMIC)
+	void * volatile foundval;
+	__atomic_exchange(ptr, &newval, &foundval, __ATOMIC_SEQ_CST);
+	return foundval;
+#elif defined(GIT_BUILTIN_SYNC)
 	return __sync_lock_test_and_set(ptr, newval);
+#else
+#	error "Unsupported architecture for atomic operations"
+#endif
+}
+
+GIT_INLINE(volatile void *) git___load(void * volatile *ptr)
+{
+#if defined(GIT_WIN32)
+	void *newval = NULL, *oldval = NULL;
+	volatile void *foundval = NULL;
+	foundval = InterlockedCompareExchangePointer((volatile PVOID *)ptr, newval, oldval);
+	return foundval;
+#elif defined(GIT_BUILTIN_ATOMIC)
+	return (volatile void *)__atomic_load_n(ptr, __ATOMIC_SEQ_CST);
+#elif defined(GIT_BUILTIN_SYNC)
+	return (volatile void *)__sync_val_compare_and_swap(ptr, 0, 0);
+#else
+#	error "Unsupported architecture for atomic operations"
 #endif
 }
 
@@ -126,13 +196,41 @@ GIT_INLINE(int64_t) git_atomic64_add(git_atomic64 *a, int64_t addend)
 {
 #if defined(GIT_WIN32)
 	return InterlockedExchangeAdd64(&a->val, addend);
-#elif defined(__GNUC__)
+#elif defined(GIT_BUILTIN_ATOMIC)
+	return __atomic_add_fetch(&a->val, addend, __ATOMIC_SEQ_CST);
+#elif defined(GIT_BUILTIN_SYNC)
 	return __sync_add_and_fetch(&a->val, addend);
 #else
 #	error "Unsupported architecture for atomic operations"
 #endif
 }
 
+GIT_INLINE(void) git_atomic64_set(git_atomic64 *a, int64_t val)
+{
+#if defined(GIT_WIN32)
+	InterlockedExchange64(&a->val, val);
+#elif defined(GIT_BUILTIN_ATOMIC)
+	__atomic_store_n(&a->val, val, __ATOMIC_SEQ_CST);
+#elif defined(GIT_BUILTIN_SYNC)
+	__sync_lock_test_and_set(&a->val, val);
+#else
+#	error "Unsupported architecture for atomic operations"
+#endif
+}
+
+GIT_INLINE(int64_t) git_atomic64_get(git_atomic64 *a)
+{
+#if defined(GIT_WIN32)
+	return (int64_t)InterlockedCompareExchange64(&a->val, 0, 0);
+#elif defined(GIT_BUILTIN_ATOMIC)
+	return __atomic_load_n(&a->val, __ATOMIC_SEQ_CST);
+#elif defined(GIT_BUILTIN_SYNC)
+	return __sync_val_compare_and_swap(&a->val, 0, 0);
+#else
+#	error "Unsupported architecture for atomic operations"
+#endif
+}
+
 #endif
 
 #else
@@ -190,6 +288,11 @@ GIT_INLINE(int) git_atomic_dec(git_atomic *a)
 	return --a->val;
 }
 
+GIT_INLINE(int) git_atomic_get(git_atomic *a)
+{
+	return (int)a->val;
+}
+
 GIT_INLINE(void *) git___compare_and_swap(
 	void * volatile *ptr, void *oldval, void *newval)
 {
@@ -216,15 +319,20 @@ GIT_INLINE(int64_t) git_atomic64_add(git_atomic64 *a, int64_t addend)
 	return a->val;
 }
 
-#endif
-
-#endif
+GIT_INLINE(void) git_atomic64_set(git_atomic64 *a, int64_t val)
+{
+	a->val = val;
+}
 
-GIT_INLINE(int) git_atomic_get(git_atomic *a)
+GIT_INLINE(int64_t) git_atomic64_get(git_atomic64 *a)
 {
-	return (int)a->val;
+	return (int64_t)a->val;
 }
 
+#endif
+
+#endif
+
 /* Atomically replace oldval with newval
  * @return oldval if it was replaced or newval if it was not
  */
@@ -233,14 +341,24 @@ GIT_INLINE(int) git_atomic_get(git_atomic *a)
 
 #define git__swap(ptr, val) (void *)git___swap((void * volatile *)&ptr, val)
 
+#define git__load(ptr) (void *)git___load((void * volatile *)&ptr)
+
 extern int git_online_cpus(void);
 
-#if defined(GIT_THREADS) && defined(_MSC_VER)
-# define GIT_MEMORY_BARRIER MemoryBarrier()
-#elif defined(GIT_THREADS)
-# define GIT_MEMORY_BARRIER __sync_synchronize()
+#if defined(GIT_THREADS)
+
+# if defined(GIT_WIN32)
+#  define GIT_MEMORY_BARRIER MemoryBarrier()
+# elif defined(GIT_BUILTIN_ATOMIC)
+#  define GIT_MEMORY_BARRIER __atomic_thread_fence(__ATOMIC_SEQ_CST)
+# elif defined(GIT_BUILTIN_SYNC)
+#  define GIT_MEMORY_BARRIER __sync_synchronize()
+# endif
+
 #else
+
 # define GIT_MEMORY_BARRIER /* noop */
+
 #endif
 
 #endif

src/util.h

@@ -186,10 +186,10 @@ typedef void (*git_refcount_freeptr)(void *r);
 }
 
 #define GIT_REFCOUNT_OWN(r, o) { \
-	(r)->rc.owner = o; \
+	(void)git__swap((r)->rc.owner, o); \
 }
 
-#define GIT_REFCOUNT_OWNER(r) ((r)->rc.owner)
+#define GIT_REFCOUNT_OWNER(r) git__load((r)->rc.owner)
 
 #define GIT_REFCOUNT_VAL(r) git_atomic_get((r)->rc.refcount)