util: make the qsort_r check work on macOS

This performs a compile-check by using CMake support, to differentiate the GNU
version from the BSD version of qsort_r.

Module taken from 4f252abea5f1d17c60f6ff115c9c44cc0b6f1df6, which I've checked
against CMake 2.8.11.

CMakeLists.txt

@@ -28,6 +28,7 @@ INCLUDE(CheckLibraryExists)
 INCLUDE(CheckFunctionExists)
 INCLUDE(CheckSymbolExists)
 INCLUDE(CheckStructHasMember)
+INCLUDE(CheckPrototypeDefinition) # Added in CMake 3.0
 INCLUDE(AddCFlagIfSupported)
 INCLUDE(FindPkgLibraries)
 INCLUDE(FindThreads)

cmake/Modules/CheckPrototypeDefinition.c.in

+@CHECK_PROTOTYPE_DEFINITION_HEADER@
+
+static void cmakeRequireSymbol(int dummy, ...) {
+  (void) dummy;
+}
+
+static void checkSymbol(void) {
+#ifndef @CHECK_PROTOTYPE_DEFINITION_SYMBOL@
+  cmakeRequireSymbol(0, &@CHECK_PROTOTYPE_DEFINITION_SYMBOL@);
+#endif
+}
+
+@CHECK_PROTOTYPE_DEFINITION_PROTO@ {
+  return @CHECK_PROTOTYPE_DEFINITION_RETURN@;
+}
+
+#ifdef __CLASSIC_C__
+int main() {
+  int ac;
+  char*av[];
+#else
+int main(int ac, char *av[]) {
+#endif
+  checkSymbol();
+  if (ac > 1000) {
+    return *av[0];
+  }
+  return 0;
+}

cmake/Modules/CheckPrototypeDefinition.cmake

+# - Check if the protoype we expect is correct.
+# check_prototype_definition(FUNCTION PROTOTYPE RETURN HEADER VARIABLE)
+#  FUNCTION - The name of the function (used to check if prototype exists)
+#  PROTOTYPE- The prototype to check.
+#  RETURN - The return value of the function.
+#  HEADER - The header files required.
+#  VARIABLE - The variable to store the result.
+# Example:
+#  check_prototype_definition(getpwent_r
+#   "struct passwd *getpwent_r(struct passwd *src, char *buf, int buflen)"
+#   "NULL"
+#   "unistd.h;pwd.h"
+#   SOLARIS_GETPWENT_R)
+# The following variables may be set before calling this macro to
+# modify the way the check is run:
+#
+#  CMAKE_REQUIRED_FLAGS = string of compile command line flags
+#  CMAKE_REQUIRED_DEFINITIONS = list of macros to define (-DFOO=bar)
+#  CMAKE_REQUIRED_INCLUDES = list of include directories
+#  CMAKE_REQUIRED_LIBRARIES = list of libraries to link
+
+#=============================================================================
+# Copyright 2005-2009 Kitware, Inc.
+# Copyright 2010-2011 Andreas Schneider <asn@cryptomilk.org>
+#
+# Distributed under the OSI-approved BSD License (the "License");
+# see accompanying file Copyright.txt for details.
+#
+# This software is distributed WITHOUT ANY WARRANTY; without even the
+# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See the License for more information.
+#=============================================================================
+# (To distribute this file outside of CMake, substitute the full
+#  License text for the above reference.)
+#
+
+get_filename_component(__check_proto_def_dir "${CMAKE_CURRENT_LIST_FILE}" PATH)
+
+function(CHECK_PROTOTYPE_DEFINITION _FUNCTION _PROTOTYPE _RETURN _HEADER _VARIABLE)
+
+  if ("${_VARIABLE}" MATCHES "^${_VARIABLE}$")
+    set(CHECK_PROTOTYPE_DEFINITION_CONTENT "/* */\n")
+
+    set(CHECK_PROTOTYPE_DEFINITION_FLAGS ${CMAKE_REQUIRED_FLAGS})
+    if (CMAKE_REQUIRED_LIBRARIES)
+      set(CHECK_PROTOTYPE_DEFINITION_LIBS
+        "-DLINK_LIBRARIES:STRING=${CMAKE_REQUIRED_LIBRARIES}")
+    else(CMAKE_REQUIRED_LIBRARIES)
+      set(CHECK_PROTOTYPE_DEFINITION_LIBS)
+    endif(CMAKE_REQUIRED_LIBRARIES)
+    if (CMAKE_REQUIRED_INCLUDES)
+      set(CMAKE_SYMBOL_EXISTS_INCLUDES
+        "-DINCLUDE_DIRECTORIES:STRING=${CMAKE_REQUIRED_INCLUDES}")
+    else(CMAKE_REQUIRED_INCLUDES)
+      set(CMAKE_SYMBOL_EXISTS_INCLUDES)
+    endif(CMAKE_REQUIRED_INCLUDES)
+
+    foreach(_FILE ${_HEADER})
+      set(CHECK_PROTOTYPE_DEFINITION_HEADER
+        "${CHECK_PROTOTYPE_DEFINITION_HEADER}#include <${_FILE}>\n")
+    endforeach(_FILE)
+
+    set(CHECK_PROTOTYPE_DEFINITION_SYMBOL ${_FUNCTION})
+    set(CHECK_PROTOTYPE_DEFINITION_PROTO ${_PROTOTYPE})
+    set(CHECK_PROTOTYPE_DEFINITION_RETURN ${_RETURN})
+
+    configure_file("${__check_proto_def_dir}/CheckPrototypeDefinition.c.in"
+      "${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp/CheckPrototypeDefinition.c" @ONLY)
+
+    file(READ ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp/CheckPrototypeDefinition.c _SOURCE)
+
+    try_compile(${_VARIABLE}
+      ${CMAKE_BINARY_DIR}
+      ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp/CheckPrototypeDefinition.c
+      COMPILE_DEFINITIONS ${CMAKE_REQUIRED_DEFINITIONS}
+      CMAKE_FLAGS -DCOMPILE_DEFINITIONS:STRING=${CHECK_PROTOTYPE_DEFINITION_FLAGS}
+      "${CHECK_PROTOTYPE_DEFINITION_LIBS}"
+      "${CMAKE_SYMBOL_EXISTS_INCLUDES}"
+      OUTPUT_VARIABLE OUTPUT)
+
+    if (${_VARIABLE})
+      set(${_VARIABLE} 1 CACHE INTERNAL "Have correct prototype for ${_FUNCTION}")
+      message(STATUS "Checking prototype ${_FUNCTION} for ${_VARIABLE} - True")
+      file(APPEND ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeOutput.log
+        "Determining if the prototype ${_FUNCTION} exists for ${_VARIABLE} passed with the following output:\n"
+        "${OUTPUT}\n\n")
+    else (${_VARIABLE})
+      message(STATUS "Checking prototype ${_FUNCTION} for ${_VARIABLE} - False")
+      set(${_VARIABLE} 0 CACHE INTERNAL "Have correct prototype for ${_FUNCTION}")
+      file(APPEND ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeError.log
+        "Determining if the prototype ${_FUNCTION} exists for ${_VARIABLE} failed with the following output:\n"
+        "${OUTPUT}\n\n${_SOURCE}\n\n")
+    endif (${_VARIABLE})
+  endif("${_VARIABLE}" MATCHES "^${_VARIABLE}$")
+
+endfunction(CHECK_PROTOTYPE_DEFINITION)

src/CMakeLists.txt

@@ -58,10 +58,19 @@ IF (HAVE_FUTIMENS)
 	SET(GIT_USE_FUTIMENS 1)
 ENDIF ()
 
-CHECK_FUNCTION_EXISTS(qsort_r HAVE_QSORT_R)
-IF (HAVE_QSORT_R)
-	ADD_DEFINITIONS(-DHAVE_QSORT_R)
-ENDIF ()
+CHECK_PROTOTYPE_DEFINITION(qsort_r
+	"void qsort_r(void *base, size_t nmemb, size_t size, void *thunk, int (*compar)(void *, const void *, const void *))"
+	"" "stdlib.h" HAVE_QSORT_R_BSD)
+IF (HAVE_QSORT_R_BSD)
+	ADD_DEFINITIONS(-DHAVE_QSORT_R_BSD)
+ENDIF()
+
+CHECK_PROTOTYPE_DEFINITION(qsort_r
+	"void qsort_r(void *base, size_t nmemb, size_t size, int (*compar)(const void *, const void *, void *), void *arg)"
+	"" "stdlib.h" HAVE_QSORT_R_GNU)
+IF (HAVE_QSORT_R_GNU)
+	ADD_DEFINITIONS(-DHAVE_QSORT_R_GNU)
+ENDIF()
 
 CHECK_FUNCTION_EXISTS(qsort_s HAVE_QSORT_S)
 IF (HAVE_QSORT_S)

src/util.c

@@ -647,7 +647,7 @@ size_t git__unescape(char *str)
 	return (pos - str);
 }
 
-#if defined(HAVE_QSORT_S) || (defined(HAVE_QSORT_R) && defined(BSD))
+#if defined(HAVE_QSORT_S) || defined(HAVE_QSORT_R_BSD)
 typedef struct {
 	git__sort_r_cmp cmp;
 	void *payload;
@@ -664,10 +664,10 @@ static int GIT_STDLIB_CALL git__qsort_r_glue_cmp(
 void git__qsort_r(
 	void *els, size_t nel, size_t elsize, git__sort_r_cmp cmp, void *payload)
 {
-#if defined(HAVE_QSORT_R) && defined(BSD)
+#if defined(HAVE_QSORT_R_BSD)
 	git__qsort_r_glue glue = { cmp, payload };
 	qsort_r(els, nel, elsize, &glue, git__qsort_r_glue_cmp);
-#elif defined(HAVE_QSORT_R) && defined(__GLIBC__)
+#elif defined(HAVE_QSORT_R_GNU)
 	qsort_r(els, nel, elsize, cmp, payload);
 #elif defined(HAVE_QSORT_S)
 	git__qsort_r_glue glue = { cmp, payload };