Merge pull request #413 from libgit2/utf8-paths-win32

Rewrite p_* functions to use Unicode and marshal to UTF8 internally, take 2

src/dir.h

@@ -22,7 +22,7 @@ struct git__dirent {
 
 typedef struct {
 	HANDLE h;
-	WIN32_FIND_DATA f;
+	WIN32_FIND_DATAW f;
 	struct git__dirent entry;
 	char *dir;
 	int first;

src/posix.c

@@ -10,6 +10,8 @@
 #include <stdio.h>
 #include <ctype.h>
 
+#ifndef GIT_WIN32
+
 int p_open(const char *path, int flags)
 {
 	return open(path, flags | O_BINARY);
@@ -20,6 +22,25 @@ int p_creat(const char *path, int mode)
 	return open(path, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, mode);
 }
 
+int p_getcwd(char *buffer_out, size_t size)
+{
+	char *cwd_buffer;
+
+	assert(buffer_out && size > 0);
+
+	cwd_buffer = getcwd(buffer_out, size);
+
+	if (cwd_buffer == NULL)
+		return git__throw(GIT_EOSERR, "Failed to retrieve current working directory");
+
+	git_path_mkposix(buffer_out);
+
+	git_path_join(buffer_out, buffer_out, "");	//Ensure the path ends with a trailing slash
+	return GIT_SUCCESS;
+}
+
+#endif
+
 int p_read(git_file fd, void *buf, size_t cnt)
 {
 	char *b = buf;
@@ -57,24 +78,3 @@ int p_write(git_file fd, const void *buf, size_t cnt)
 	}
 	return GIT_SUCCESS;
 }
-
-int p_getcwd(char *buffer_out, size_t size)
-{
-	char *cwd_buffer;
-
-	assert(buffer_out && size > 0);
-
-#ifdef GIT_WIN32
-	cwd_buffer = _getcwd(buffer_out, size);
-#else
-	cwd_buffer = getcwd(buffer_out, size);
-#endif
-
-	if (cwd_buffer == NULL)
-		return git__throw(GIT_EOSERR, "Failed to retrieve current working directory");
-
-	git_path_mkposix(buffer_out);
-
-	git_path_join(buffer_out, buffer_out, "");	//Ensure the path ends with a trailing slash
-	return GIT_SUCCESS;
-}

src/posix.h

@@ -33,19 +33,26 @@ typedef int git_file;
  * Use your manpages to check the docs on these.
  * Straightforward 
  */
-extern int p_open(const char *path, int flags);
-extern int p_creat(const char *path, int mode);
+
 extern int p_read(git_file fd, void *buf, size_t cnt);
 extern int p_write(git_file fd, const void *buf, size_t cnt);
-extern int p_getcwd(char *buffer_out, size_t size);
 
+#define p_fstat(f,b) fstat(f, b)
 #define p_lseek(f,n,w) lseek(f, n, w)
+#define p_close(fd) close(fd)
+
+extern int p_open(const char *path, int flags);
+extern int p_creat(const char *path, int mode);
+extern int p_getcwd(char *buffer_out, size_t size);
+
+#ifndef GIT_WIN32
+
 #define p_stat(p,b) stat(p, b)
-#define p_fstat(f,b) fstat(f, b)
 #define p_chdir(p) chdir(p)
 #define p_rmdir(p) rmdir(p)
 #define p_chmod(p,m) chmod(p, m)
-#define p_close(fd) close(fd)
+
+#endif
 
 /**
  * Platform-dependent methods

src/win32/dir.c

@@ -6,6 +6,7 @@
  */
 #define GIT__WIN32_NO_WRAP_DIR
 #include "dir.h"
+#include "utf8-conv.h"
 
 static int init_filter(char *filter, size_t n, const char *dir)
 {
@@ -25,6 +26,7 @@ static int init_filter(char *filter, size_t n, const char *dir)
 git__DIR *git__opendir(const char *dir)
 {
 	char filter[4096];
+	wchar_t* filter_w;
 	git__DIR *new;
 
 	if (!dir || !init_filter(filter, sizeof(filter), dir))
@@ -41,7 +43,10 @@ git__DIR *git__opendir(const char *dir)
 	}
 	strcpy(new->dir, dir);
 
-	new->h = FindFirstFile(filter, &new->f);
+	filter_w = conv_utf8_to_utf16(filter);
+	new->h = FindFirstFileW(filter_w, &new->f);
+	free(filter_w);
+
 	if (new->h == INVALID_HANDLE_VALUE) {
 		free(new->dir);
 		free(new);
@@ -60,15 +65,15 @@ struct git__dirent *git__readdir(git__DIR *d)
 	if (d->first)
 		d->first = 0;
 	else {
-		if (!FindNextFile(d->h, &d->f))
+		if (!FindNextFileW(d->h, &d->f))
 			return NULL;
 	}
 
-	if (strlen(d->f.cFileName) >= sizeof(d->entry.d_name))
+	if (wcslen(d->f.cFileName) >= sizeof(d->entry.d_name))
 		return NULL;
 
 	d->entry.d_ino = 0;
-	strcpy(d->entry.d_name, d->f.cFileName);
+	WideCharToMultiByte(CP_UTF8, 0, d->f.cFileName, -1, d->entry.d_name, GIT_PATH_MAX, NULL, NULL);
 
 	return &d->entry;
 }
@@ -76,14 +81,19 @@ struct git__dirent *git__readdir(git__DIR *d)
 void git__rewinddir(git__DIR *d)
 {
 	char filter[4096];
+	wchar_t* filter_w;
 
 	if (d) {
 		if (d->h != INVALID_HANDLE_VALUE)
 			FindClose(d->h);
 		d->h = INVALID_HANDLE_VALUE;
 		d->first = 0;
+
 		if (init_filter(filter, sizeof(filter), d->dir)) {
-			d->h = FindFirstFile(filter, &d->f);
+			filter_w = conv_utf8_to_utf16(filter);
+			d->h = FindFirstFileW(filter_w, &d->f);
+			free(filter_w);
+
 			if (d->h != INVALID_HANDLE_VALUE)
 				d->first = 1;
 		}

src/win32/posix.c

@@ -6,13 +6,23 @@
  */
 #include "posix.h"
 #include "path.h"
+#include "utf8-conv.h"
 #include <errno.h>
 #include <io.h>
+#include <fcntl.h>
+
 
 int p_unlink(const char *path)
 {
-	chmod(path, 0666);
-	return unlink(path);
+	int ret = 0;
+	wchar_t* buf;
+
+	buf = conv_utf8_to_utf16(path);
+	_wchmod(buf, 0666);
+	ret = _wunlink(buf);
+	free(buf);
+
+	return ret;
 }
 
 int p_fsync(int fd)
@@ -49,8 +59,9 @@ GIT_INLINE(time_t) filetime_to_time_t(const FILETIME *ft)
 static int do_lstat(const char *file_name, struct stat *buf)
 {
 	WIN32_FILE_ATTRIBUTE_DATA fdata;
+	wchar_t* fbuf = conv_utf8_to_utf16(file_name);
 
-	if (GetFileAttributesExA(file_name, GetFileExInfoStandard, &fdata)) {
+	if (GetFileAttributesExW(fbuf, GetFileExInfoStandard, &fdata)) {
 		int fMode = S_IREAD;
 
 		if (fdata.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)
@@ -74,9 +85,13 @@ static int do_lstat(const char *file_name, struct stat *buf)
 		buf->st_atime = filetime_to_time_t(&(fdata.ftLastAccessTime));
 		buf->st_mtime = filetime_to_time_t(&(fdata.ftLastWriteTime));
 		buf->st_ctime = filetime_to_time_t(&(fdata.ftCreationTime));
+
+		free(fbuf);
 		return GIT_SUCCESS;
 	}
 
+	free(fbuf);
+
 	switch (GetLastError()) {
 		case ERROR_ACCESS_DENIED:
 		case ERROR_SHARING_VIOLATION:
@@ -124,10 +139,12 @@ int p_lstat(const char *file_name, struct stat *buf)
 
 int p_readlink(const char *link, char *target, size_t target_len)
 {
-	typedef DWORD (WINAPI *fpath_func)(HANDLE, LPTSTR, DWORD, DWORD);
+	typedef DWORD (WINAPI *fpath_func)(HANDLE, LPWSTR, DWORD, DWORD);
 	static fpath_func pGetFinalPath = NULL;
 	HANDLE hFile;
 	DWORD dwRet;
+	wchar_t* link_w;
+	wchar_t* target_w;
 
 	/*
 	 * Try to load the pointer to pGetFinalPath dynamically, because
@@ -137,14 +154,16 @@ int p_readlink(const char *link, char *target, size_t target_len)
 		HINSTANCE library = LoadLibrary("kernel32");
 
 		if (library != NULL)
-			pGetFinalPath = (fpath_func)GetProcAddress(library, "GetFinalPathNameByHandleA");
+			pGetFinalPath = (fpath_func)GetProcAddress(library, "GetFinalPathNameByHandleW");
 
 		if (pGetFinalPath == NULL)
 			return git__throw(GIT_EOSERR,
-				"'GetFinalPathNameByHandleA' is not available in this platform");
+				"'GetFinalPathNameByHandleW' is not available in this platform");
 	}
 
-	hFile = CreateFile(link,			// file to open
+	link_w = conv_utf8_to_utf16(link);
+
+	hFile = CreateFileW(link_w,			// file to open
 			GENERIC_READ,			// open for reading
 			FILE_SHARE_READ,		// share for reading
 			NULL,					// default security
@@ -152,13 +171,30 @@ int p_readlink(const char *link, char *target, size_t target_len)
 			FILE_FLAG_BACKUP_SEMANTICS, // normal file
 			NULL);					// no attr. template
 
+	free(link_w);
+
 	if (hFile == INVALID_HANDLE_VALUE)
 		return GIT_EOSERR;
 
-	dwRet = pGetFinalPath(hFile, target, target_len, 0x0);
-	if (dwRet >= target_len)
+	if (target_len <= 0) {
+		return GIT_EINVALIDARGS;
+	}
+
+	target_w = (wchar_t*)git__malloc(target_len * sizeof(wchar_t));
+
+	dwRet = pGetFinalPath(hFile, target_w, target_len, 0x0);
+	if (dwRet >= target_len) {
+		free(target_w);
+		CloseHandle(hFile);
 		return GIT_ENOMEM;
+	}
+
+	if (!WideCharToMultiByte(CP_UTF8, 0, target_w, -1, target, target_len * sizeof(char), NULL, NULL)) {
+		free(target_w);
+		return GIT_EOSERR;
+	}
 
+	free(target_w);
 	CloseHandle(hFile);
 
 	if (dwRet > 4) {
@@ -184,13 +220,82 @@ int p_readlink(const char *link, char *target, size_t target_len)
 	return dwRet;
 }
 
+int p_open(const char *path, int flags)
+{
+	int fd;
+	wchar_t* buf = conv_utf8_to_utf16(path);
+	fd = _wopen(buf, flags | _O_BINARY);
+
+	free(buf);
+	return fd;
+}
+
+int p_creat(const char *path, int mode)
+{
+	int fd;
+	wchar_t* buf = conv_utf8_to_utf16(path);
+	fd = _wopen(buf, _O_WRONLY | _O_CREAT | _O_TRUNC | _O_BINARY, mode);
+
+	free(buf);
+	return fd;
+}
+
+int p_getcwd(char *buffer_out, size_t size)
+{
+	wchar_t* buf = (wchar_t*)git__malloc(sizeof(wchar_t) * (int)size);
+	_wgetcwd(buf, (int)size);
+
+	if (!WideCharToMultiByte(CP_UTF8, 0, buf, -1, buffer_out, size, NULL, NULL)) {
+		free(buf);
+		return GIT_EOSERR;
+	}
+
+	free(buf);
+	return GIT_SUCCESS;
+}
+
+int p_stat(const char* path, struct stat* buf)
+{
+	return do_lstat(path, buf);
+}
+
+int p_chdir(const char* path)
+{
+	wchar_t* buf = conv_utf8_to_utf16(path);
+	int ret = _wchdir(buf);
+
+	free(buf);
+	return ret;
+}
+
+int p_chmod(const char* path, int mode)
+{
+	wchar_t* buf = conv_utf8_to_utf16(path);
+	int ret = _wchmod(buf, mode);
+
+	free(buf);
+	return ret;
+}
+
+int p_rmdir(const char* path)
+{
+	wchar_t* buf = conv_utf8_to_utf16(path);
+	int ret = _wrmdir(buf);
+
+	free(buf);
+	return ret;
+}
+
 int p_hide_directory__w32(const char *path)
 {
 	int error;
+	wchar_t* buf = conv_utf8_to_utf16(path);
 
-	error = SetFileAttributes(path, FILE_ATTRIBUTE_HIDDEN) != 0 ?
+	error = SetFileAttributesW(buf, FILE_ATTRIBUTE_HIDDEN) != 0 ?
 		GIT_SUCCESS : GIT_ERROR; /* MSDN states a "non zero" value indicates a success */
 
+	free(buf);
+
 	if (error < GIT_SUCCESS)
 		error = git__throw(GIT_EOSERR, "Failed to hide directory '%s'", path);
 
@@ -200,18 +305,30 @@ int p_hide_directory__w32(const char *path)
 char *p_realpath(const char *orig_path, char *buffer)
 {
 	int ret, alloc = 0;
+	wchar_t* orig_path_w = conv_utf8_to_utf16(orig_path);
+	wchar_t* buffer_w = (wchar_t*)git__malloc(GIT_PATH_MAX * sizeof(wchar_t));
 
 	if (buffer == NULL) {
 		buffer = (char *)git__malloc(GIT_PATH_MAX);
 		alloc = 1;
 	}
 
-	ret = GetFullPathName(orig_path, GIT_PATH_MAX, buffer, NULL);
+	ret = GetFullPathNameW(orig_path_w, GIT_PATH_MAX, buffer_w, NULL);
+	free(orig_path_w);
+
 	if (!ret || ret > GIT_PATH_MAX) {
+		free(buffer_w);
 		if (alloc) free(buffer);
+
 		return NULL;
 	}
 
+	if (!WideCharToMultiByte(CP_UTF8, 0, buffer_w, -1, buffer, GIT_PATH_MAX, NULL, NULL)) {
+		free(buffer_w);
+		if (alloc) free(buffer);
+	}
+	
+	free(buffer_w);
 	git_path_mkposix(buffer);
 	return buffer;
 }

src/win32/posix.h

@@ -9,6 +9,7 @@
 
 #include "common.h"
 #include "fnmatch.h"
+#include "utf8-conv.h"
 
 GIT_INLINE(int) p_link(const char *GIT_UNUSED(old), const char *GIT_UNUSED(new))
 {
@@ -20,8 +21,13 @@ GIT_INLINE(int) p_link(const char *GIT_UNUSED(old), const char *GIT_UNUSED(new))
 
 GIT_INLINE(int) p_mkdir(const char *path, int GIT_UNUSED(mode))
 {
+	wchar_t* buf = conv_utf8_to_utf16(path);
+	int ret = _wmkdir(buf);
+
 	GIT_UNUSED_ARG(mode)
-	return mkdir(path);
+
+	free(buf);
+	return ret;
 }
 
 extern int p_unlink(const char *path);
@@ -33,5 +39,10 @@ extern int p_vsnprintf(char *buffer, size_t count, const char *format, va_list a
 extern int p_snprintf(char *buffer, size_t count, const char *format, ...) GIT_FORMAT_PRINTF(3, 4);
 extern int p_mkstemp(char *tmp_path);
 extern int p_setenv(const char* name, const char* value, int overwrite);
+extern int p_stat(const char* path, struct stat* buf);
+extern int p_chdir(const char* path);
+extern int p_chmod(const char* path, int mode);
+extern int p_rmdir(const char* path);
+
 
 #endif

src/win32/utf8-conv.c

+/*
+ * Copyright (C) 2009-2011 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+
+#include "common.h"
+#include "utf8-conv.h"
+
+wchar_t* conv_utf8_to_utf16(const char* str)
+{
+	wchar_t* ret;
+	int cb;
+
+	if (!str) {
+		return NULL;
+	}
+
+	cb = strlen(str) * sizeof(wchar_t);
+	if (cb == 0) {
+		ret = (wchar_t*)git__malloc(sizeof(wchar_t));
+		ret[0] = 0;
+		return ret;
+	}
+
+	/* Add space for null terminator */
+	cb += sizeof(wchar_t);
+
+	ret = (wchar_t*)git__malloc(cb);
+
+	if (MultiByteToWideChar(CP_UTF8, 0, str, -1, ret, cb) == 0) {
+		free(ret);
+		ret = NULL;
+	}
+
+	return ret;
+}
+
+char* conv_utf16_to_utf8(const wchar_t* str)
+{
+	char* ret;
+	int cb;
+
+	if (!str) {
+		return NULL;
+	}
+
+	cb = wcslen(str) * sizeof(char);
+	if (cb == 0) {
+		ret = (char*)git__malloc(sizeof(char));
+		ret[0] = 0;
+		return ret;
+	}
+
+	/* Add space for null terminator */
+	cb += sizeof(char);
+
+	ret = (char*)git__malloc(cb);
+
+	if (WideCharToMultiByte(CP_UTF8, 0, str, -1, ret, cb, NULL, NULL) == 0) {
+		free(ret);
+		ret = NULL;
+	}
+
+	return ret;
+
+}

src/win32/utf8-conv.h

+/*
+ * Copyright (C) 2009-2011 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+
+#include <wchar.h>
+
+#ifndef INCLUDE_git_utf8conv_h__
+#define INCLUDE_git_utf8conv_h__
+
+wchar_t* conv_utf8_to_utf16(const char* str);
+char* conv_utf16_to_utf8(const wchar_t* str);
+
+#endif
+