Merge pull request #5583 from 0xdky/dhruva/build-with-nommap

Build with NO_MMAP

.github/workflows/nightly.yml

@@ -96,6 +96,17 @@ jobs:
             ASAN_SYMBOLIZER_PATH: /usr/bin/llvm-symbolizer-10
             TSAN_OPTIONS: suppressions=/home/libgit2/source/script/thread-sanitizer.supp second_deadlock_stack=1
           os: ubuntu-latest
+        - # Focal, Clang 10, mmap emulation (NO_MMAP)
+          container:
+            name: focal
+          env:
+            CC: clang-10
+            CFLAGS: -DNO_MMAP
+            CMAKE_OPTIONS: -DCMAKE_PREFIX_PATH=/usr/local
+            CMAKE_GENERATOR: Ninja
+            SKIP_SSH_TESTS: true
+            SKIP_NEGOTIATE_TESTS: true
+          os: ubuntu-latest
         - # macOS
           os: macos-10.15
           env:
@@ -114,6 +125,15 @@ jobs:
             CMAKE_OPTIONS: -A x64 -DWIN32_LEAKCHECK=ON -DDEPRECATE_HARD=ON
             SKIP_SSH_TESTS: true
             SKIP_NEGOTIATE_TESTS: true
+        - # Windows amd64 Visual Studio (NO_MMAP)
+          os: windows-2019
+          env:
+            ARCH: amd64
+            CMAKE_GENERATOR: Visual Studio 16 2019
+            CFLAGS: -DNO_MMAP
+            CMAKE_OPTIONS: -A x64 -DDEPRECATE_HARD=ON
+            SKIP_SSH_TESTS: true
+            SKIP_NEGOTIATE_TESTS: true
         - # Windows x86 Visual Studio
           os: windows-2019
           env:

src/indexer.c

@@ -603,6 +603,23 @@ static void hash_partially(git_indexer *idx, const uint8_t *data, size_t size)
 
 static int write_at(git_indexer *idx, const void *data, off64_t offset, size_t size)
 {
+#ifdef NO_MMAP
+	size_t remaining_size = size;
+	const char *ptr = (const char *)data;
+
+	/* Handle data size larger that ssize_t */
+	while (remaining_size > 0) {
+		ssize_t nb;
+		HANDLE_EINTR(nb, p_pwrite(idx->pack->mwf.fd, (void *)ptr,
+					  remaining_size, offset));
+		if (nb <= 0)
+			return -1;
+
+		ptr += nb;
+		offset += nb;
+		remaining_size -= nb;
+	}
+#else
 	git_file fd = idx->pack->mwf.fd;
 	size_t mmap_alignment;
 	size_t page_offset;
@@ -627,6 +644,7 @@ static int write_at(git_indexer *idx, const void *data, off64_t offset, size_t s
 	map_data = (unsigned char *)map.data;
 	memcpy(map_data + page_offset, data, size);
 	p_munmap(&map);
+#endif
 
 	return 0;
 }

src/posix.c

@@ -238,24 +238,43 @@ int git__mmap_alignment(size_t *alignment)
 
 int p_mmap(git_map *out, size_t len, int prot, int flags, int fd, off64_t offset)
 {
+	const char *ptr;
+	size_t remaining_len;
+
 	GIT_MMAP_VALIDATE(out, len, prot, flags);
 
-	out->data = NULL;
-	out->len = 0;
+	/* writes cannot be emulated without handling pagefaults since write happens by
+	 * writing to mapped memory */
+	if (prot & GIT_PROT_WRITE) {
+		git_error_set(GIT_ERROR_OS, "trying to map %s-writeable",
+				((flags & GIT_MAP_TYPE) == GIT_MAP_SHARED) ? "shared": "private");
+		return -1;
+	}
 
-	if ((prot & GIT_PROT_WRITE) && ((flags & GIT_MAP_TYPE) == GIT_MAP_SHARED)) {
-		git_error_set(GIT_ERROR_OS, "trying to map shared-writeable");
+	if (!git__is_ssizet(len)) {
+		errno = EINVAL;
 		return -1;
 	}
 
+	out->len = 0;
 	out->data = git__malloc(len);
 	GIT_ERROR_CHECK_ALLOC(out->data);
 
-	if (!git__is_ssizet(len) ||
-		(p_lseek(fd, offset, SEEK_SET) < 0) ||
-		(p_read(fd, out->data, len) != (ssize_t)len)) {
-		git_error_set(GIT_ERROR_OS, "mmap emulation failed");
-		return -1;
+	remaining_len = len;
+	ptr = (const char *)out->data;
+	while (remaining_len > 0) {
+		ssize_t nb;
+		HANDLE_EINTR(nb, p_pread(fd, (void *)ptr, remaining_len, offset));
+		if (nb <= 0) {
+			git_error_set(GIT_ERROR_OS, "mmap emulation failed");
+			git__free(out->data);
+			out->data = NULL;
+			return -1;
+		}
+
+		ptr += nb;
+		offset += nb;
+		remaining_len -= nb;
 	}
 
 	out->len = len;
@@ -267,6 +286,10 @@ int p_munmap(git_map *map)
 	GIT_ASSERT_ARG(map);
 	git__free(map->data);
 
+	/* Initializing will help debug use-after-free */
+	map->len = 0;
+	map->data = NULL;
+
 	return 0;
 }
 

src/posix.h

@@ -89,6 +89,12 @@
 #define EAFNOSUPPORT (INT_MAX-1)
 #endif
 
+/* Compiler independent macro to handle signal interrpted system calls */
+#define HANDLE_EINTR(result, x) do {					\
+		result = (x);						\
+	} while (result == -1 && errno == EINTR);
+
+
 /* Provide a 64-bit size for offsets. */
 
 #if defined(_MSC_VER)
@@ -119,6 +125,9 @@ typedef int git_file;
 extern ssize_t p_read(git_file fd, void *buf, size_t cnt);
 extern int p_write(git_file fd, const void *buf, size_t cnt);
 
+extern ssize_t p_pread(int fd, void *data, size_t size, off64_t offset);
+extern ssize_t p_pwrite(int fd, const void *data, size_t size, off64_t offset);
+
 #define p_close(fd) close(fd)
 #define p_umask(m) umask(m)
 

src/unix/posix.h

@@ -101,4 +101,7 @@ GIT_INLINE(int) p_futimes(int f, const struct p_timeval t[2])
 # define p_futimes futimes
 #endif
 
+#define p_pread(f, d, s, o) pread(f, d, s, o)
+#define p_pwrite(f, d, s, o) pwrite(f, d, s, o)
+
 #endif

src/win32/posix_w32.c

@@ -981,3 +981,73 @@ int p_inet_pton(int af, const char *src, void *dst)
 	errno = EINVAL;
 	return -1;
 }
+
+ssize_t p_pread(int fd, void *data, size_t size, off64_t offset)
+{
+	HANDLE fh;
+	DWORD rsize = 0;
+	OVERLAPPED ov = {0};
+	LARGE_INTEGER pos = {0};
+	off64_t final_offset = 0;
+
+	/* Fail if the final offset would have overflowed to match POSIX semantics. */
+	if (!git__is_ssizet(size) || git__add_int64_overflow(&final_offset, offset, (int64_t)size)) {
+		errno = EINVAL;
+		return -1;	
+	}
+
+	/*
+	 * Truncate large writes to the maximum allowable size: the caller
+	 * needs to always call this in a loop anyways.
+	 */
+	if (size > INT32_MAX) {
+		size = INT32_MAX;
+	}
+
+	pos.QuadPart = offset;
+	ov.Offset = pos.LowPart;
+	ov.OffsetHigh = pos.HighPart;
+	fh = (HANDLE)_get_osfhandle(fd);
+
+	if (ReadFile(fh, data, (DWORD)size, &rsize, &ov)) {
+		return (ssize_t)rsize;
+	}
+
+	set_errno();
+	return -1;
+}
+
+ssize_t p_pwrite(int fd, const void *data, size_t size, off64_t offset)
+{
+	HANDLE fh;
+	DWORD wsize = 0;
+	OVERLAPPED ov = {0};
+	LARGE_INTEGER pos = {0};
+	off64_t final_offset = 0;
+
+	/* Fail if the final offset would have overflowed to match POSIX semantics. */
+	if (!git__is_ssizet(size) || git__add_int64_overflow(&final_offset, offset, (int64_t)size)) {
+		errno = EINVAL;
+		return -1;	
+	}
+
+	/*
+	 * Truncate large writes to the maximum allowable size: the caller
+	 * needs to always call this in a loop anyways.
+	 */
+	if (size > INT32_MAX) {
+		size = INT32_MAX;
+	}
+
+	pos.QuadPart = offset;
+	ov.Offset = pos.LowPart;
+	ov.OffsetHigh = pos.HighPart;
+	fh = (HANDLE)_get_osfhandle(fd);
+
+	if (WriteFile(fh, data, (DWORD)size, &wsize, &ov)) {
+		return (ssize_t)wsize;
+	}
+
+	set_errno();
+	return -1;
+}