Merge pull request #6207 from libgit2/ethomson/prng

mktmp: improve our temp file creation

COPYING

@@ -1132,3 +1132,15 @@ STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
 OF THE POSSIBILITY OF SUCH DAMAGE.
 
+----------------------------------------------------------------------
+
+The xoroshiro256** implementation is licensed in the public domain:
+
+Written in 2018 by David Blackman and Sebastiano Vigna (vigna@acm.org)
+
+To the extent possible under law, the author has dedicated all copyright
+and related and neighboring rights to this software to the public domain
+worldwide. This software is distributed without any warranty.
+
+See <http://creativecommons.org/publicdomain/zero/1.0/>.
+

src/CMakeLists.txt

@@ -55,6 +55,8 @@ check_prototype_definition(qsort_r
 
 check_function_exists(qsort_s GIT_QSORT_S)
 
+check_function_exists(getentropy GIT_RAND_GETENTROPY)
+
 # Find required dependencies
 
 if(WIN32)

src/features.h.in

@@ -48,4 +48,6 @@
 #cmakedefine GIT_SHA1_OPENSSL 1
 #cmakedefine GIT_SHA1_MBEDTLS 1
 
+#cmakedefine GIT_RAND_GETENTROPY 1
+
 #endif

src/fs_path.c

@@ -1040,8 +1040,8 @@ fail:
 	return -1;
 }
 
-static const char *nfc_file = "\xC3\x85\x73\x74\x72\xC3\xB6\x6D.XXXXXX";
-static const char *nfd_file = "\x41\xCC\x8A\x73\x74\x72\x6F\xCC\x88\x6D.XXXXXX";
+static const char *nfc_file = "\xC3\x85\x73\x74\x72\xC3\xB6\x6D";
+static const char *nfd_file = "\x41\xCC\x8A\x73\x74\x72\x6F\xCC\x88\x6D";
 
 /* Check if the platform is decomposing unicode data for us.  We will
  * emulate core Git and prefer to use precomposed unicode data internally
@@ -1054,39 +1054,42 @@ static const char *nfd_file = "\x41\xCC\x8A\x73\x74\x72\x6F\xCC\x88\x6D.XXXXXX";
  */
 bool git_fs_path_does_decompose_unicode(const char *root)
 {
-	git_str path = GIT_STR_INIT;
+	git_str nfc_path = GIT_STR_INIT;
+	git_str nfd_path = GIT_STR_INIT;
 	int fd;
 	bool found_decomposed = false;
-	char tmp[6];
+	size_t orig_len;
+	const char *trailer;
 
 	/* Create a file using a precomposed path and then try to find it
 	 * using the decomposed name.  If the lookup fails, then we will mark
 	 * that we should precompose unicode for this repository.
 	 */
-	if (git_str_joinpath(&path, root, nfc_file) < 0 ||
-		(fd = p_mkstemp(path.ptr)) < 0)
+	if (git_str_joinpath(&nfc_path, root, nfc_file) < 0)
+		goto done;
+
+	/* record original path length before trailer */
+	orig_len = nfc_path.size;
+
+	if ((fd = git_futils_mktmp(&nfc_path, nfc_path.ptr, 0666)) < 0)
 		goto done;
 	p_close(fd);
 
-	/* record trailing digits generated by mkstemp */
-	memcpy(tmp, path.ptr + path.size - sizeof(tmp), sizeof(tmp));
+	trailer = nfc_path.ptr + orig_len;
 
 	/* try to look up as NFD path */
-	if (git_str_joinpath(&path, root, nfd_file) < 0)
+	if (git_str_joinpath(&nfd_path, root, nfd_file) < 0 ||
+	    git_str_puts(&nfd_path, trailer) < 0)
 		goto done;
-	memcpy(path.ptr + path.size - sizeof(tmp), tmp, sizeof(tmp));
 
-	found_decomposed = git_fs_path_exists(path.ptr);
+	found_decomposed = git_fs_path_exists(nfd_path.ptr);
 
 	/* remove temporary file (using original precomposed path) */
-	if (git_str_joinpath(&path, root, nfc_file) < 0)
-		goto done;
-	memcpy(path.ptr + path.size - sizeof(tmp), tmp, sizeof(tmp));
-
-	(void)p_unlink(path.ptr);
+	(void)p_unlink(nfc_path.ptr);
 
 done:
-	git_str_dispose(&path);
+	git_str_dispose(&nfc_path);
+	git_str_dispose(&nfd_path);
 	return found_decomposed;
 }
 

src/futils.c

@@ -10,6 +10,8 @@
 #include "runtime.h"
 #include "strmap.h"
 #include "hash.h"
+#include "rand.h"
+
 #include <ctype.h>
 #if GIT_WIN32
 #include "win32/findfile.h"
@@ -27,23 +29,20 @@ int git_futils_mkpath2file(const char *file_path, const mode_t mode)
 int git_futils_mktmp(git_str *path_out, const char *filename, mode_t mode)
 {
 	const int open_flags = O_RDWR | O_CREAT | O_EXCL | O_BINARY | O_CLOEXEC;
-	/* TMP_MAX is unrelated to mktemp but should provide a reasonable amount */
-	unsigned int tries = TMP_MAX;
+	unsigned int tries = 32;
 	int fd;
 
 	while (tries--) {
+		uint64_t rand = git_rand_next();
+
 		git_str_sets(path_out, filename);
-		git_str_puts(path_out, "_git2_XXXXXX");
+		git_str_puts(path_out, "_git2_");
+		git_str_encode_hexstr(path_out, (void *)&rand, sizeof(uint64_t));
 
 		if (git_str_oom(path_out))
 			return -1;
 
-		/* Using mktemp is safe when we open with O_CREAT | O_EXCL */
-		p_mktemp(path_out->ptr);
-		/* mktemp sets template to empty string on failure */
-		if (path_out->ptr[0] == '\0')
-			break;
-
+		/* Note that we open with O_CREAT | O_EXCL */
 		if ((fd = p_open(path_out->ptr, open_flags, mode)) >= 0)
 			return fd;
 	}

src/futils.h

@@ -177,11 +177,12 @@ extern int git_futils_rmdir_r(const char *path, const char *base, uint32_t flags
  * protected directory; the file created will created will honor
  * the current `umask`.  Writes the filename into path_out.
  *
- * This function is *NOT* suitable for use in temporary directories
- * that are world writable.  It uses `mktemp` (for portability) and
- * many `mktemp` implementations use weak random characters.  It
- * should only be assumed to be suitable for atomically writing
- * a new file in a directory that you control.
+ * This function uses a high-quality PRNG seeded by the system's
+ * entropy pool _where available_ and falls back to a simple seed
+ * (time plus system information) when not.  This is suitable for
+ * writing within a protected directory, but the system's safe
+ * temporary file creation functions should be preferred where
+ * available when writing into world-writable (temp) directories.
  *
  * @return On success, an open file descriptor, else an error code < 0.
  */

src/libgit2.c

@@ -20,6 +20,7 @@
 #include "mwindow.h"
 #include "object.h"
 #include "odb.h"
+#include "rand.h"
 #include "refs.h"
 #include "runtime.h"
 #include "sysdir.h"
@@ -70,6 +71,7 @@ int git_libgit2_init(void)
 		git_allocator_global_init,
 		git_threadstate_global_init,
 		git_threads_global_init,
+		git_rand_global_init,
 		git_hash_global_init,
 		git_sysdir_global_init,
 		git_filter_global_init,

src/posix.h

@@ -131,7 +131,6 @@ extern ssize_t p_pwrite(int fd, const void *data, size_t size, off64_t offset);
 
 #define p_close(fd) close(fd)
 #define p_umask(m) umask(m)
-#define p_mktemp(p) mktemp(p)
 
 extern int p_open(const char *path, int flags, ...);
 extern int p_creat(const char *path, mode_t mode);

src/rand.c

+/*  Written in 2018 by David Blackman and Sebastiano Vigna (vigna@acm.org)
+
+To the extent possible under law, the author has dedicated all copyright
+and related and neighboring rights to this software to the public domain
+worldwide. This software is distributed without any warranty.
+
+See <http://creativecommons.org/publicdomain/zero/1.0/>. */
+
+#include "common.h"
+#include "rand.h"
+#include "runtime.h"
+
+#if defined(GIT_RAND_GETENTROPY)
+# include <sys/random.h>
+#endif
+
+static uint64_t state[4];
+static git_mutex state_lock;
+
+typedef union {
+	double f;
+	uint64_t d;
+} bits;
+
+#if defined(GIT_WIN32)
+GIT_INLINE(int) getseed(uint64_t *seed)
+{
+	HCRYPTPROV provider;
+	SYSTEMTIME systemtime;
+	FILETIME filetime, idletime, kerneltime, usertime;
+	bits convert;
+
+	if (CryptAcquireContext(&provider, 0, 0, PROV_RSA_FULL,
+	                        CRYPT_VERIFYCONTEXT|CRYPT_SILENT)) {
+		BOOL success = CryptGenRandom(provider, sizeof(uint64_t), (void *)seed);
+		CryptReleaseContext(provider, 0);
+
+		if (success)
+			return 0;
+	}
+
+	GetSystemTime(&systemtime);
+	if (!SystemTimeToFileTime(&systemtime, &filetime)) {
+		git_error_set(GIT_ERROR_OS, "could not get time for random seed");
+		return -1;
+	}
+
+	/* Fall-through: generate a seed from the time and system state */
+	*seed = 0;
+	*seed |= ((uint64_t)filetime.dwLowDateTime << 32);
+	*seed |= ((uint64_t)filetime.dwHighDateTime);
+
+	GetSystemTimes(&idletime, &kerneltime, &usertime);
+
+	*seed ^= ((uint64_t)idletime.dwLowDateTime << 32);
+	*seed ^= ((uint64_t)kerneltime.dwLowDateTime);
+	*seed ^= ((uint64_t)usertime.dwLowDateTime << 32);
+
+	*seed ^= ((uint64_t)idletime.dwHighDateTime);
+	*seed ^= ((uint64_t)kerneltime.dwHighDateTime << 12);
+	*seed ^= ((uint64_t)usertime.dwHighDateTime << 24);
+
+	*seed ^= ((uint64_t)GetCurrentProcessId() << 32);
+	*seed ^= ((uint64_t)GetCurrentThreadId() << 48);
+
+	convert.f = git__timer(); *seed ^= (convert.d);
+
+	/* Mix in the addresses of some functions and variables */
+	*seed ^= (((uint64_t)((uintptr_t)seed) << 32));
+	*seed ^= (((uint64_t)((uintptr_t)&errno)));
+
+	return 0;
+}
+
+#else
+
+GIT_INLINE(int) getseed(uint64_t *seed)
+{
+	struct timeval tv;
+	double loadavg[3];
+	bits convert;
+	int fd;
+
+# if defined(GIT_RAND_GETENTROPY)
+	GIT_UNUSED((fd = 0));
+
+	if (getentropy(seed, sizeof(uint64_t)) == 0)
+		return 0;
+# else
+	/*
+	 * Try to read from /dev/urandom; most modern systems will have
+	 * this, but we may be chrooted, etc, so it's not a fatal error
+	 */
+	if ((fd = open("/dev/urandom", O_RDONLY)) >= 0) {
+		ssize_t ret = read(fd, seed, sizeof(uint64_t));
+		close(fd);
+
+		if (ret == (ssize_t)sizeof(uint64_t))
+			return 0;
+	}
+# endif
+
+	/* Fall-through: generate a seed from the time and system state */
+	if (gettimeofday(&tv, NULL) < 0) {
+		git_error_set(GIT_ERROR_OS, "could get time for random seed");
+		return -1;
+	}
+
+	getloadavg(loadavg, 3);
+
+	*seed = 0;
+	*seed |= ((uint64_t)tv.tv_usec << 40);
+	*seed |= ((uint64_t)tv.tv_sec);
+
+	*seed ^= ((uint64_t)getpid() << 48);
+	*seed ^= ((uint64_t)getppid() << 32);
+	*seed ^= ((uint64_t)getpgid(0) << 28);
+	*seed ^= ((uint64_t)getsid(0) << 16);
+	*seed ^= ((uint64_t)getuid() << 8);
+	*seed ^= ((uint64_t)getgid());
+
+	convert.f = loadavg[0]; *seed ^= (convert.d >> 36);
+	convert.f = loadavg[1]; *seed ^= (convert.d);
+	convert.f = loadavg[2]; *seed ^= (convert.d >> 16);
+
+	convert.f = git__timer(); *seed ^= (convert.d);
+
+	/* Mix in the addresses of some variables */
+	*seed ^= ((uint64_t)((size_t)((void *)seed)) << 32);
+	*seed ^= ((uint64_t)((size_t)((void *)&errno)));
+
+	return 0;
+}
+#endif
+
+static void git_rand_global_shutdown(void)
+{
+	git_mutex_free(&state_lock);
+}
+
+int git_rand_global_init(void)
+{
+	uint64_t seed = 0;
+
+	if (git_mutex_init(&state_lock) < 0 || getseed(&seed) < 0)
+		return -1;
+
+	if (!seed) {
+		git_error_set(GIT_ERROR_INTERNAL, "failed to generate random seed");
+		return -1;
+	}
+
+	git_rand_seed(seed);
+	git_runtime_shutdown_register(git_rand_global_shutdown);
+
+	return 0;
+}
+
+/*
+ * This is splitmix64. xoroshiro256** uses 256 bit seed; this is used
+ * to generate 256 bits of seed from the given 64, per the author's
+ * recommendation.
+ */
+GIT_INLINE(uint64_t) splitmix64(uint64_t *in)
+{
+	uint64_t z;
+
+	*in += 0x9e3779b97f4a7c15;
+
+	z = *in;
+	z = (z ^ (z >> 30)) * 0xbf58476d1ce4e5b9;
+	z = (z ^ (z >> 27)) * 0x94d049bb133111eb;
+	return z ^ (z >> 31);
+}
+
+void git_rand_seed(uint64_t seed)
+{
+	uint64_t mixer;
+
+	mixer = seed;
+
+	git_mutex_lock(&state_lock);
+	state[0] = splitmix64(&mixer);
+	state[1] = splitmix64(&mixer);
+	state[2] = splitmix64(&mixer);
+	state[3] = splitmix64(&mixer);
+	git_mutex_unlock(&state_lock);
+}
+
+/* This is xoshiro256** 1.0, one of our all-purpose, rock-solid
+   generators. It has excellent (sub-ns) speed, a state (256 bits) that is
+   large enough for any parallel application, and it passes all tests we
+   are aware of.
+
+   For generating just floating-point numbers, xoshiro256+ is even faster.
+
+   The state must be seeded so that it is not everywhere zero. If you have
+   a 64-bit seed, we suggest to seed a splitmix64 generator and use its
+   output to fill s. */
+
+GIT_INLINE(uint64_t) rotl(const uint64_t x, int k) {
+	return (x << k) | (x >> (64 - k));
+}
+
+uint64_t git_rand_next(void) {
+	uint64_t t, result;
+
+	git_mutex_lock(&state_lock);
+
+	result = rotl(state[1] * 5, 7) * 9;
+
+	t = state[1] << 17;
+
+	state[2] ^= state[0];
+	state[3] ^= state[1];
+	state[1] ^= state[2];
+	state[0] ^= state[3];
+
+	state[2] ^= t;
+
+	state[3] = rotl(state[3], 45);
+
+	git_mutex_unlock(&state_lock);
+
+	return result;
+}

src/rand.h

+/*
+ * Copyright (C) the libgit2 contributors. All rights reserved.
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#ifndef INCLUDE_rand_h__
+#define INCLUDE_rand_h__
+
+#include "common.h"
+
+/**
+ * Initialize the random number generation subsystem.  This will
+ * seed the random number generator with the system's entropy pool,
+ * if available, and will fall back to the current time and
+ * system information if not.
+ */
+int git_rand_global_init(void);
+
+/**
+ * Seed the pseudo-random number generator.  This is not needed to be
+ * called; the PRNG is seeded by `git_rand_global_init`, but it may
+ * be useful for testing.  When the same seed is specified, the same
+ * sequence of random numbers from `git_rand_next` is emitted.
+ *
+ * @param seed the seed to use
+ */
+void git_rand_seed(uint64_t seed);
+
+/**
+ * Get the next pseudo-random number in the sequence.
+ *
+ * @return a 64-bit pseudo-random number
+ */
+uint64_t git_rand_next(void);
+
+#endif

src/str.c

@@ -217,6 +217,32 @@ int git_str_puts(git_str *buf, const char *string)
 	return git_str_put(buf, string, strlen(string));
 }
 
+static char hex_encode[] = "0123456789abcdef";
+
+int git_str_encode_hexstr(git_str *str, const char *data, size_t len)
+{
+	size_t new_size, i;
+	char *s;
+
+	GIT_ERROR_CHECK_ALLOC_MULTIPLY(&new_size, len, 2);
+	GIT_ERROR_CHECK_ALLOC_ADD(&new_size, new_size, 1);
+
+	if (git_str_grow_by(str, new_size) < 0)
+		return -1;
+
+	s = str->ptr + str->size;
+
+	for (i = 0; i < len; i++) {
+		*s++ = hex_encode[(data[i] & 0xf0) >> 4];
+		*s++ = hex_encode[(data[i] & 0x0f)];
+	}
+
+	str->size += (len * 2);
+	str->ptr[str->size] = '\0';
+
+	return 0;
+}
+
 static const char base64_encode[] =
 	"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
 

src/str.h

@@ -217,6 +217,9 @@ int git_str_cmp(const git_str *a, const git_str *b);
 int git_str_quote(git_str *str);
 int git_str_unquote(git_str *str);
 
+/* Write data as a hex string */
+int git_str_encode_hexstr(git_str *str, const char *data, size_t len);
+
 /* Write data as base64 encoded in string buffer */
 int git_str_encode_base64(git_str *str, const char *data, size_t len);
 /* Decode the given bas64 and write the result to the string buffer */

src/unix/posix.h

@@ -58,7 +58,6 @@ GIT_INLINE(int) p_fsync(int fd)
 #define p_strncasecmp(s1, s2, c) strncasecmp(s1, s2, c)
 #define p_vsnprintf(b, c, f, a) vsnprintf(b, c, f, a)
 #define p_snprintf snprintf
-#define p_mkstemp(p) mkstemp(p)
 #define p_chdir(p) chdir(p)
 #define p_rmdir(p) rmdir(p)
 #define p_access(p,m) access(p,m)

src/win32/posix.h

@@ -42,7 +42,6 @@ extern int p_inet_pton(int af, const char *src, void* dst);
 
 extern int p_vsnprintf(char *buffer, size_t count, const char *format, va_list argptr);
 extern int p_snprintf(char *buffer, size_t count, const char *format, ...) GIT_FORMAT_PRINTF(3, 4);
-extern int p_mkstemp(char *tmp_path);
 extern int p_chdir(const char *path);
 extern int p_chmod(const char *path, mode_t mode);
 extern int p_rmdir(const char *path);

src/win32/posix_w32.c

@@ -860,20 +860,6 @@ int p_snprintf(char *buffer, size_t count, const char *format, ...)
 	return r;
 }
 
-/* TODO: wut? */
-int p_mkstemp(char *tmp_path)
-{
-#if defined(_MSC_VER) && _MSC_VER >= 1500
-	if (_mktemp_s(tmp_path, strlen(tmp_path) + 1) != 0)
-		return -1;
-#else
-	if (_mktemp(tmp_path) == NULL)
-		return -1;
-#endif
-
-	return p_open(tmp_path, O_RDWR | O_CREAT | O_EXCL, 0744); /* -V536 */
-}
-
 int p_access(const char *path, mode_t mode)
 {
 	git_win32_path buf;