Merge pull request #3183 from libgit2/cmn/curl-stream

Implement a cURL stream

CHANGELOG.md

@@ -72,6 +72,9 @@ support for HTTPS connections insead of OpenSSL.
 * The race condition mitigations described in `racy-git.txt` have been
   implemented.
 
+* If libcurl is installed, we will use it to connect to HTTP(S)
+  servers.
+
 ### API additions
 
 * The `git_merge_options` gained a `file_flags` member.

CMakeLists.txt

@@ -38,6 +38,7 @@ OPTION( USE_ICONV			"Link with and use iconv library" 		OFF )
 OPTION( USE_SSH				"Link with libssh to enable SSH support" ON )
 OPTION( USE_GSSAPI			"Link with libgssapi for SPNEGO auth"   OFF )
 OPTION( VALGRIND			"Configure build for valgrind"			OFF )
+OPTION( CURL			"User curl for HTTP if available" ON)
 
 IF(${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
 	SET( USE_ICONV ON )
@@ -199,10 +200,20 @@ IF (WIN32 AND WINHTTP)
 
 	LINK_LIBRARIES(winhttp rpcrt4 crypt32)
 ELSE ()
+	IF (CURL)
+		FIND_PACKAGE(CURL)
+	ENDIF ()
+
 	IF (NOT AMIGA AND USE_OPENSSL)
 		FIND_PACKAGE(OpenSSL)
 	ENDIF ()
 
+	IF (CURL_FOUND)
+		ADD_DEFINITIONS(-DGIT_CURL)
+		INCLUDE_DIRECTORIES(${CURL_INCLUDE_DIRS})
+		LINK_LIBRARIES(${CURL_LIBRARIES})
+	ENDIF()
+
 	FIND_PACKAGE(HTTP_Parser)
 	IF (HTTP_PARSER_FOUND AND HTTP_PARSER_VERSION_MAJOR EQUAL 2)
 		INCLUDE_DIRECTORIES(${HTTP_PARSER_INCLUDE_DIRS})

THREADING.md

@@ -47,9 +47,14 @@ you.
 On Mac OS X
 -----------
 
-On OS X, the library makes use of CommonCrypto and SecureTransport for
-cryptographic support. These are thread-safe and you do not need to do
-anything special.
+By default we use libcurl to perform the encryption. The
+system-provided libcurl uses SecureTransport, so no special steps are
+necessary. If you link against another libcurl (e.g. from homebrew)
+refer to the general case.
+
+If the option to use libcurl was deactivated, the library makes use of
+CommonCrypto and SecureTransport for cryptographic support. These are
+thread-safe and you do not need to do anything special.
 
 Note that libssh2 may still use OpenSSL itself. In that case, the
 general case still affects you if you use ssh.
@@ -57,12 +62,15 @@ general case still affects you if you use ssh.
 General Case
 ------------
 
-On the rest of the platforms, libgit2 uses OpenSSL to be able to use
-HTTPS as a transport. This library is made to be thread-implementation
-agnostic, and the users of the library must set which locking function
-it should use. This means that libgit2 cannot know what to set as the
-user of libgit2 may use OpenSSL independently and the locking settings
-must survive libgit2 shutting down.
+By default we use libcurl, which has its own ![recommendations for
+thread safety](http://curl.haxx.se/libcurl/c/libcurl-tutorial.html#Multi-threading).
+
+If libcurl was not found or was disabled, libgit2 uses OpenSSL to be
+able to use HTTPS as a transport. This library is made to be
+thread-implementation agnostic, and the users of the library must set
+which locking function it should use. This means that libgit2 cannot
+know what to set as the user of libgit2 may use OpenSSL independently
+and the locking settings must survive libgit2 shutting down.
 
 libgit2 does provide a last-resort convenience function
 `git_openssl_set_locking()` (available in `sys/openssl.h`) to use the

include/git2/sys/stream.h

@@ -29,8 +29,10 @@ typedef struct git_stream {
 	int version;
 
 	int encrypted;
+	int proxy_support;
 	int (*connect)(struct git_stream *);
 	int (*certificate)(git_cert **, struct git_stream *);
+	int (*set_proxy)(struct git_stream *, const char *proxy_url);
 	ssize_t (*read)(struct git_stream *, void *, size_t);
 	ssize_t (*write)(struct git_stream *, const char *, size_t, int);
 	int (*close)(struct git_stream *);

include/git2/types.h

@@ -284,6 +284,11 @@ typedef int (*git_transport_message_cb)(const char *str, int len, void *payload)
  * Type of host certificate structure that is passed to the check callback
  */
 typedef enum git_cert_t {
+	/**
+	 * No information about the certificate is available. This may
+	 * happen when using curl.
+	 */
+	GIT_CERT_NONE,
         /**
          * The `data` argument to the callback will be a pointer to
          * the DER-encoded data.
@@ -294,6 +299,13 @@ typedef enum git_cert_t {
          * `git_cert_hostkey` structure.
          */
 	GIT_CERT_HOSTKEY_LIBSSH2,
+	/**
+	 * The `data` argument to the callback will be a pointer to a
+	 * `git_strarray` with `name:content` strings containing
+	 * information about the certificate. This is used when using
+	 * curl.
+	 */
+	GIT_CERT_STRARRAY,
 } git_cert_t;
 
 /**

src/curl_stream.c

+/*
+ * Copyright (C) the libgit2 contributors. All rights reserved.
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+
+#ifdef GIT_CURL
+
+#include <curl/curl.h>
+
+#include "stream.h"
+#include "git2/transport.h"
+#include "buffer.h"
+#include "vector.h"
+
+typedef struct {
+	git_stream parent;
+	CURL *handle;
+	curl_socket_t socket;
+	char curl_error[CURL_ERROR_SIZE + 1];
+	git_cert_x509 cert_info;
+	git_strarray cert_info_strings;
+} curl_stream;
+
+static int seterr_curl(curl_stream *s)
+{
+	giterr_set(GITERR_NET, "curl error: %s\n", s->curl_error);
+	return -1;
+}
+
+static int curls_connect(git_stream *stream)
+{
+	curl_stream *s = (curl_stream *) stream;
+	long sockextr;
+	int failed_cert = 0;
+	CURLcode res;
+	res = curl_easy_perform(s->handle);
+
+	if (res != CURLE_OK && res != CURLE_PEER_FAILED_VERIFICATION)
+		return seterr_curl(s);
+	if (res == CURLE_PEER_FAILED_VERIFICATION)
+		failed_cert = 1;
+
+	if ((res = curl_easy_getinfo(s->handle, CURLINFO_LASTSOCKET, &sockextr)) != CURLE_OK)
+		return seterr_curl(s);
+
+	s->socket = sockextr;
+
+	if (s->parent.encrypted && failed_cert)
+		return GIT_ECERTIFICATE;
+
+	return 0;
+}
+
+static int curls_certificate(git_cert **out, git_stream *stream)
+{
+	int error;
+	CURLcode res;
+	struct curl_slist *slist;
+	struct curl_certinfo *certinfo;
+	git_vector strings = GIT_VECTOR_INIT;
+	curl_stream *s = (curl_stream *) stream;
+
+	if ((res = curl_easy_getinfo(s->handle, CURLINFO_CERTINFO, &certinfo)) != CURLE_OK)
+		return seterr_curl(s);
+
+	/* No information is available, can happen with SecureTransport */
+	if (certinfo->num_of_certs == 0) {
+		s->cert_info.cert_type = GIT_CERT_NONE;
+		s->cert_info.data      = NULL;
+		s->cert_info.len       = 0;
+		return 0;
+	}
+
+	if ((error = git_vector_init(&strings, 8, NULL)) < 0)
+		return error;
+
+	for (slist = certinfo->certinfo[0]; slist; slist = slist->next) {
+		char *str = git__strdup(slist->data);
+		GITERR_CHECK_ALLOC(str);
+	}
+
+	/* Copy the contents of the vector into a strarray so we can expose them */
+	s->cert_info_strings.strings = (char **) strings.contents;
+	s->cert_info_strings.count   = strings.length;
+
+	s->cert_info.cert_type = GIT_CERT_STRARRAY;
+	s->cert_info.data      = &s->cert_info_strings;
+	s->cert_info.len       = strings.length;
+
+	*out = (git_cert *) &s->cert_info;
+
+	return 0;
+}
+
+static int curls_set_proxy(git_stream *stream, const char *proxy_url)
+{
+	CURLcode res;
+	curl_stream *s = (curl_stream *) stream;
+
+	if ((res = curl_easy_setopt(s->handle, CURLOPT_PROXY, proxy_url)) != CURLE_OK)
+		return seterr_curl(s);
+
+	return 0;
+}
+
+static int wait_for(curl_socket_t fd, bool reading)
+{
+	int ret;
+	fd_set infd, outfd, errfd;
+
+	FD_ZERO(&infd);
+	FD_ZERO(&outfd);
+	FD_ZERO(&errfd);
+
+	FD_SET(fd, &errfd);
+	if (reading)
+		FD_SET(fd, &infd);
+	else
+		FD_SET(fd, &outfd);
+
+	if ((ret = select(fd + 1, &infd, &outfd, &errfd, NULL)) < 0) {
+		giterr_set(GITERR_OS, "error in select");
+		return -1;
+	}
+
+	return 0;
+}
+
+static ssize_t curls_write(git_stream *stream, const char *data, size_t len, int flags)
+{
+	int error;
+	size_t off = 0, sent;
+	CURLcode res;
+	curl_stream *s = (curl_stream *) stream;
+
+	GIT_UNUSED(flags);
+
+	do {
+		if ((error = wait_for(s->socket, false)) < 0)
+			return error;
+
+		res = curl_easy_send(s->handle, data + off, len - off, &sent);
+		if (res == CURLE_OK)
+			off += sent;
+	} while ((res == CURLE_OK || res == CURLE_AGAIN) && off < len);
+
+	if (res != CURLE_OK)
+		return seterr_curl(s);
+
+	return len;
+}
+
+static ssize_t curls_read(git_stream *stream, void *data, size_t len)
+{
+	int error;
+	size_t read;
+	CURLcode res;
+	curl_stream *s = (curl_stream *) stream;
+
+	do {
+		if ((error = wait_for(s->socket, true)) < 0)
+			return error;
+
+		res = curl_easy_recv(s->handle, data, len, &read);
+	} while (res == CURLE_AGAIN);
+
+	if (res != CURLE_OK)
+		return seterr_curl(s);
+
+	return read;
+}
+
+static int curls_close(git_stream *stream)
+{
+	curl_stream *s = (curl_stream *) stream;
+
+	if (!s->handle)
+		return 0;
+
+	curl_easy_cleanup(s->handle);
+	s->handle = NULL;
+	s->socket = 0;
+
+	return 0;
+}
+
+static void curls_free(git_stream *stream)
+{
+	curl_stream *s = (curl_stream *) stream;
+
+	curls_close(stream);
+	git_strarray_free(&s->cert_info_strings);
+	git__free(s);
+}
+
+int git_curl_stream_new(git_stream **out, const char *host, const char *port)
+{
+	curl_stream *st;
+	CURL *handle;
+	int iport = 0, error;
+
+	st = git__calloc(1, sizeof(curl_stream));
+	GITERR_CHECK_ALLOC(st);
+
+	handle = curl_easy_init();
+	if (handle == NULL) {
+		giterr_set(GITERR_NET, "failed to create curl handle");
+		return -1;
+	}
+
+	if ((error = git__strtol32(&iport, port, NULL, 10)) < 0)
+		return error;
+
+	curl_easy_setopt(handle, CURLOPT_URL, host);
+	curl_easy_setopt(handle, CURLOPT_ERRORBUFFER, st->curl_error);
+	curl_easy_setopt(handle, CURLOPT_PORT, iport);
+	curl_easy_setopt(handle, CURLOPT_CONNECT_ONLY, 1);
+	curl_easy_setopt(handle, CURLOPT_SSL_VERIFYPEER, 1);
+	curl_easy_setopt(handle, CURLOPT_CERTINFO, 1);
+	curl_easy_setopt(handle, CURLOPT_HTTPPROXYTUNNEL, 1);
+
+	/* curl_easy_setopt(handle, CURLOPT_VERBOSE, 1); */
+
+	st->parent.version = GIT_STREAM_VERSION;
+	st->parent.encrypted = 0; /* we don't encrypt ourselves */
+	st->parent.proxy_support = 1;
+	st->parent.connect = curls_connect;
+	st->parent.certificate = curls_certificate;
+	st->parent.set_proxy = curls_set_proxy;
+	st->parent.read = curls_read;
+	st->parent.write = curls_write;
+	st->parent.close = curls_close;
+	st->parent.free = curls_free;
+	st->handle = handle;
+
+	*out = (git_stream *) st;
+	return 0;
+}
+
+#else
+
+#include "stream.h"
+
+int git_curl_stream_new(git_stream **out, const char *host, const char *port)
+{
+	GIT_UNUSED(out);
+	GIT_UNUSED(host);
+	GIT_UNUSED(port);
+
+	giterr_set(GITERR_NET, "curl is not supported in this version");
+	return -1;
+}
+
+
+#endif

src/curl_stream.h

+/*
+ * Copyright (C) the libgit2 contributors. All rights reserved.
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#ifndef INCLUDE_curl_stream_h__
+#define INCLUDE_curl_stream_h__
+
+#include "git2/sys/stream.h"
+
+extern int git_curl_stream_new(git_stream **out, const char *host, const char *port);
+
+#endif

src/openssl_stream.c

@@ -16,6 +16,10 @@
 #include "netops.h"
 #include "git2/transport.h"
 
+#ifdef GIT_CURL
+# include "curl_stream.h"
+#endif
+
 #ifndef GIT_WIN32
 # include <sys/types.h>
 # include <sys/socket.h>
@@ -25,6 +29,79 @@
 #include <openssl/ssl.h>
 #include <openssl/err.h>
 #include <openssl/x509v3.h>
+#include <openssl/bio.h>
+
+static int bio_create(BIO *b)
+{
+	b->init = 1;
+	b->num = 0;
+	b->ptr = NULL;
+	b->flags = 0;
+
+	return 1;
+}
+
+static int bio_destroy(BIO *b)
+{
+	if (!b)
+		return 0;
+
+	b->init = 0;
+	b->num = 0;
+	b->ptr = NULL;
+	b->flags = 0;
+
+	return 1;
+}
+
+static int bio_read(BIO *b, char *buf, int len)
+{
+	git_stream *io = (git_stream *) b->ptr;
+	return (int) git_stream_read(io, buf, len);
+}
+
+static int bio_write(BIO *b, const char *buf, int len)
+{
+	git_stream *io = (git_stream *) b->ptr;
+	return (int) git_stream_write(io, buf, len, 0);
+}
+
+static long bio_ctrl(BIO *b, int cmd, long num, void *ptr)
+{
+	GIT_UNUSED(b);
+	GIT_UNUSED(num);
+	GIT_UNUSED(ptr);
+
+	if (cmd == BIO_CTRL_FLUSH)
+		return 1;
+
+	return 0;
+}
+
+static int bio_gets(BIO *b, char *buf, int len)
+{
+	GIT_UNUSED(b);
+	GIT_UNUSED(buf);
+	GIT_UNUSED(len);
+	return -1;
+}
+
+static int bio_puts(BIO *b, const char *str)
+{
+	return bio_write(b, str, strlen(str));
+}
+
+static BIO_METHOD git_stream_bio_method = {
+	BIO_TYPE_SOURCE_SINK,
+	"git_stream",
+	bio_write,
+	bio_read,
+	bio_puts,
+	bio_gets,
+	bio_ctrl,
+	bio_create,
+	bio_destroy
+};
 
 static int ssl_set_error(SSL *ssl, int error)
 {
@@ -224,7 +301,8 @@ cert_fail_name:
 
 typedef struct {
 	git_stream parent;
-	git_socket_stream *socket;
+	git_stream *io;
+	char *host;
 	SSL *ssl;
 	git_cert_x509 cert_info;
 } openssl_stream;
@@ -234,23 +312,24 @@ int openssl_close(git_stream *stream);
 int openssl_connect(git_stream *stream)
 {
 	int ret;
+	BIO *bio;
 	openssl_stream *st = (openssl_stream *) stream;
 
-	if ((ret = git_stream_connect((git_stream *)st->socket)) < 0)
+	if ((ret = git_stream_connect(st->io)) < 0)
 		return ret;
 
-	if ((ret = SSL_set_fd(st->ssl, st->socket->s)) <= 0) {
-		openssl_close((git_stream *) st);
-		return ssl_set_error(st->ssl, ret);
-	}
+	bio = BIO_new(&git_stream_bio_method);
+	GITERR_CHECK_ALLOC(bio);
+	bio->ptr = st->io;
 
+	SSL_set_bio(st->ssl, bio, bio);
 	/* specify the host in case SNI is needed */
-	SSL_set_tlsext_host_name(st->ssl, st->socket->host);
+	SSL_set_tlsext_host_name(st->ssl, st->host);
 
 	if ((ret = SSL_connect(st->ssl)) <= 0)
 		return ssl_set_error(st->ssl, ret);
 
-	return verify_server_cert(st->ssl, st->socket->host);
+	return verify_server_cert(st->ssl, st->host);
 }
 
 int openssl_certificate(git_cert **out, git_stream *stream)
@@ -287,6 +366,13 @@ int openssl_certificate(git_cert **out, git_stream *stream)
 	return 0;
 }
 
+static int openssl_set_proxy(git_stream *stream, const char *proxy_url)
+{
+	openssl_stream *st = (openssl_stream *) stream;
+
+	return git_stream_set_proxy(st->io, proxy_url);
+}
+
 ssize_t openssl_write(git_stream *stream, const char *data, size_t len, int flags)
 {
 	openssl_stream *st = (openssl_stream *) stream;
@@ -320,7 +406,7 @@ int openssl_close(git_stream *stream)
 	if ((ret = ssl_teardown(st->ssl)) < 0)
 		return -1;
 
-	return git_stream_close((git_stream *)st->socket);
+	return git_stream_close(st->io);
 }
 
 void openssl_free(git_stream *stream)
@@ -328,19 +414,26 @@ void openssl_free(git_stream *stream)
 	openssl_stream *st = (openssl_stream *) stream;
 
 	git__free(st->cert_info.data);
-	git_stream_free((git_stream *) st->socket);
+	git_stream_free(st->io);
 	git__free(st);
 }
 
 int git_openssl_stream_new(git_stream **out, const char *host, const char *port)
 {
+	int error;
 	openssl_stream *st;
 
 	st = git__calloc(1, sizeof(openssl_stream));
 	GITERR_CHECK_ALLOC(st);
 
-	if (git_socket_stream_new((git_stream **) &st->socket, host, port))
-		return -1;
+#ifdef GIT_CURL
+	error = git_curl_stream_new(&st->io, host, port);
+#else
+	error = git_socket_stream_new(&st->io, host, port)
+#endif
+
+	if (error < 0)
+		return error;
 
 	st->ssl = SSL_new(git__ssl_ctx);
 	if (st->ssl == NULL) {
@@ -348,10 +441,15 @@ int git_openssl_stream_new(git_stream **out, const char *host, const char *port)
 		return -1;
 	}
 
+	st->host = git__strdup(host);
+	GITERR_CHECK_ALLOC(st->host);
+
 	st->parent.version = GIT_STREAM_VERSION;
 	st->parent.encrypted = 1;
+	st->parent.proxy_support = git_stream_supports_proxy(st->io);
 	st->parent.connect = openssl_connect;
 	st->parent.certificate = openssl_certificate;
+	st->parent.set_proxy = openssl_set_proxy;
 	st->parent.read = openssl_read;
 	st->parent.write = openssl_write;
 	st->parent.close = openssl_close;

src/stransport_stream.c

@@ -14,6 +14,7 @@
 #include "git2/transport.h"
 
 #include "socket_stream.h"
+#include "curl_stream.h"
 
 int stransport_error(OSStatus ret)
 {
@@ -115,6 +116,13 @@ int stransport_certificate(git_cert **out, git_stream *stream)
 	return 0;
 }
 
+int stransport_set_proxy(git_stream *stream, const char *proxy)
+{
+	stransport_stream *st = (stransport_stream *) stream;
+
+	return git_stream_set_proxy(st->io, proxy);
+}
+
 /*
  * Contrary to typical network IO callbacks, Secure Transport write callback is
  * expected to write *all* passed data, not just as much as it can, and any
@@ -233,7 +241,13 @@ int git_stransport_stream_new(git_stream **out, const char *host, const char *po
 	st = git__calloc(1, sizeof(stransport_stream));
 	GITERR_CHECK_ALLOC(st);
 
-	if ((error = git_socket_stream_new(&st->io, host, port)) < 0){
+#ifdef GIT_CURL
+	error = git_curl_stream_new(&st->io, host, port);
+#else
+	error = git_socket_stream_new(&st->io, host, port)
+#endif
+
+	if (error < 0){
 		git__free(st);
 		return error;
 	}
@@ -256,8 +270,10 @@ int git_stransport_stream_new(git_stream **out, const char *host, const char *po
 
 	st->parent.version = GIT_STREAM_VERSION;
 	st->parent.encrypted = 1;
+	st->parent.proxy_support = git_stream_supports_proxy(st->io);
 	st->parent.connect = stransport_connect;
 	st->parent.certificate = stransport_certificate;
+	st->parent.set_proxy = stransport_set_proxy;
 	st->parent.read = stransport_read;
 	st->parent.write = stransport_write;
 	st->parent.close = stransport_close;

src/stream.h

@@ -30,6 +30,21 @@ GIT_INLINE(int) git_stream_certificate(git_cert **out, git_stream *st)
 	return st->certificate(out, st);
 }
 
+GIT_INLINE(int) git_stream_supports_proxy(git_stream *st)
+{
+	return st->proxy_support;
+}
+
+GIT_INLINE(int) git_stream_set_proxy(git_stream *st, const char *proxy_url)
+{
+	if (!st->proxy_support) {
+		giterr_set(GITERR_INVALID, "proxy not supported on this stream");
+		return -1;
+	}
+
+	return st->set_proxy(st, proxy_url);
+}
+
 GIT_INLINE(ssize_t) git_stream_read(git_stream *st, void *data, size_t len)
 {
 	return st->read(st, data, len);

src/transports/http.c

@@ -10,11 +10,13 @@
 #include "http_parser.h"
 #include "buffer.h"
 #include "netops.h"
+#include "remote.h"
 #include "smart.h"
 #include "auth.h"
 #include "auth_negotiate.h"
 #include "tls_stream.h"
 #include "socket_stream.h"
+#include "curl_stream.h"
 
 git_http_auth_scheme auth_schemes[] = {
 	{ GIT_AUTHTYPE_NEGOTIATE, "Negotiate", GIT_CREDTYPE_DEFAULT, git_http_auth_negotiate },
@@ -532,6 +534,7 @@ static int write_chunk(git_stream *io, const char *buffer, size_t len)
 static int http_connect(http_subtransport *t)
 {
 	int error;
+	char *proxy_url;
 
 	if (t->connected &&
 		http_should_keep_alive(&t->parser) &&
@@ -547,7 +550,11 @@ static int http_connect(http_subtransport *t)
 	if (t->connection_data.use_ssl) {
 		error = git_tls_stream_new(&t->io, t->connection_data.host, t->connection_data.port);
 	} else {
+#ifdef GIT_CURL
+		error = git_curl_stream_new(&t->io, t->connection_data.host, t->connection_data.port);
+#else
 		error = git_socket_stream_new(&t->io,  t->connection_data.host, t->connection_data.port);
+#endif
 	}
 
 	if (error < 0)
@@ -555,9 +562,18 @@ static int http_connect(http_subtransport *t)
 
 	GITERR_CHECK_VERSION(t->io, GIT_STREAM_VERSION, "git_stream");
 
+	if (git_stream_supports_proxy(t->io) &&
+	    !git_remote__get_http_proxy(t->owner->owner, !!t->connection_data.use_ssl, &proxy_url)) {
+		error = git_stream_set_proxy(t->io, proxy_url);
+		git__free(proxy_url);
+
+		if (error < 0)
+			return error;
+	}
+
 	error = git_stream_connect(t->io);
 
-#if defined(GIT_OPENSSL) || defined(GIT_SECURE_TRANSPORT)
+#if defined(GIT_OPENSSL) || defined(GIT_SECURE_TRANSPORT) || defined(GIT_CURL)
 	if ((!error || error == GIT_ECERTIFICATE) && t->owner->certificate_check_cb != NULL &&
 	    git_stream_is_encrypted(t->io)) {
 		git_cert *cert;