httpclient: support authentication

Store the last-seen credential challenges (eg, all the
'WWW-Authenticate' headers in a response message).  Given some
credentials, find the best (first) challenge whose mechanism supports
these credentials.  (eg, 'Basic' supports username/password credentials,
'Negotiate' supports default credentials).

Set up an authentication context for this mechanism and these
credentials.  Continue exchanging challenge/responses until we're
authenticated.

src/transports/httpclient.c

@@ -13,17 +13,30 @@
 #include "global.h"
 #include "httpclient.h"
 #include "http.h"
+#include "auth.h"
+#include "auth_negotiate.h"
+#include "auth_ntlm.h"
+#include "git2/sys/cred.h"
 #include "net.h"
 #include "stream.h"
 #include "streams/socket.h"
 #include "streams/tls.h"
 #include "auth.h"
 
+static git_http_auth_scheme auth_schemes[] = {
+	{ GIT_HTTP_AUTH_NEGOTIATE, "Negotiate", GIT_CREDTYPE_DEFAULT, git_http_auth_negotiate },
+	{ GIT_HTTP_AUTH_NTLM, "NTLM", GIT_CREDTYPE_USERPASS_PLAINTEXT, git_http_auth_ntlm },
+	{ GIT_HTTP_AUTH_BASIC, "Basic", GIT_CREDTYPE_USERPASS_PLAINTEXT, git_http_auth_basic },
+};
+
 #define GIT_READ_BUFFER_SIZE 8192
 
 typedef struct {
 	git_net_url url;
 	git_stream *stream;
+
+	git_vector auth_challenges;
+	git_http_auth_context *auth_context;
 } git_http_server;
 
 typedef enum {
@@ -45,6 +58,7 @@ typedef enum {
 
 typedef enum {
 	PARSE_STATUS_OK,
+	PARSE_STATUS_NO_OUTPUT,
 	PARSE_STATUS_ERROR
 } parse_status;
 
@@ -116,6 +130,7 @@ void git_http_response_dispose(git_http_response *response)
 static int on_header_complete(http_parser *parser)
 {
 	http_parser_context *ctx = (http_parser_context *) parser->data;
+	git_http_client *client = ctx->client;
 	git_http_response *response = ctx->response;
 
 	git_buf *name = &ctx->parse_header_name;
@@ -148,6 +163,18 @@ static int on_header_complete(http_parser *parser)
 		}
 
 		response->content_length = (size_t)len;
+	} else if (!strcasecmp("Proxy-Authenticate", git_buf_cstr(name))) {
+		char *dup = git__strndup(value->ptr, value->size);
+		GIT_ERROR_CHECK_ALLOC(dup);
+
+		if (git_vector_insert(&client->proxy.auth_challenges, dup) < 0)
+			return -1;
+	} else if (!strcasecmp("WWW-Authenticate", name->ptr)) {
+		char *dup = git__strndup(value->ptr, value->size);
+		GIT_ERROR_CHECK_ALLOC(dup);
+
+		if (git_vector_insert(&client->server.auth_challenges, dup) < 0)
+			return -1;
 	} else if (!strcasecmp("Location", name->ptr)) {
 		if (response->location) {
 			git_error_set(GIT_ERROR_NET,
@@ -220,6 +247,71 @@ static int on_header_value(http_parser *parser, const char *str, size_t len)
 	return 0;
 }
 
+GIT_INLINE(bool) challenge_matches_scheme(
+	const char *challenge,
+	git_http_auth_scheme *scheme)
+{
+	const char *scheme_name = scheme->name;
+	size_t scheme_len = strlen(scheme_name);
+
+	if (!strncasecmp(challenge, scheme_name, scheme_len) &&
+	    (challenge[scheme_len] == '\0' || challenge[scheme_len] == ' '))
+		return true;
+
+	return false;
+}
+
+static git_http_auth_scheme *scheme_for_challenge(const char *challenge)
+{
+	size_t i;
+
+	for (i = 0; i < ARRAY_SIZE(auth_schemes); i++) {
+		if (challenge_matches_scheme(challenge, &auth_schemes[i]))
+			return &auth_schemes[i];
+	}
+
+	return NULL;
+}
+
+GIT_INLINE(void) collect_authinfo(
+	unsigned int *schemetypes,
+	unsigned int *credtypes,
+	git_vector *challenges)
+{
+	git_http_auth_scheme *scheme;
+	const char *challenge;
+	size_t i;
+
+	*schemetypes = 0;
+	*credtypes = 0;
+
+	git_vector_foreach(challenges, i, challenge) {
+		if ((scheme = scheme_for_challenge(challenge)) != NULL) {
+			*schemetypes |= scheme->type;
+			*credtypes |= scheme->credtypes;
+		}
+	}
+}
+
+static int resend_needed(git_http_client *client, git_http_response *response)
+{
+	git_http_auth_context *auth_context;
+
+	if (response->status == 401 &&
+	    (auth_context = client->server.auth_context) &&
+	    auth_context->is_complete &&
+	    !auth_context->is_complete(auth_context))
+		return 1;
+
+	if (response->status == 407 &&
+	    (auth_context = client->proxy.auth_context) &&
+	    auth_context->is_complete &&
+	    !auth_context->is_complete(auth_context))
+		return 1;
+
+	return 0;
+}
+
 static int on_headers_complete(http_parser *parser)
 {
 	http_parser_context *ctx = (http_parser_context *) parser->data;
@@ -245,6 +337,17 @@ static int on_headers_complete(http_parser *parser)
 	ctx->response->status = parser->status_code;
 	ctx->client->keepalive = http_should_keep_alive(parser);
 
+	/* Prepare for authentication */
+	collect_authinfo(&ctx->response->server_auth_schemetypes,
+	                 &ctx->response->server_auth_credtypes,
+	                 &ctx->client->server.auth_challenges);
+	collect_authinfo(&ctx->response->proxy_auth_schemetypes,
+	                 &ctx->response->proxy_auth_credtypes,
+	                 &ctx->client->proxy.auth_challenges);
+
+	ctx->response->resend_credentials = resend_needed(ctx->client,
+	                                                  ctx->response);
+
 	/* Stop parsing. */
 	http_parser_pause(parser, 1);
 
@@ -305,26 +408,201 @@ const char *name_for_method(git_http_method method)
 	return NULL;
 }
 
+/*
+ * Find the scheme that is suitable for the given credentials, based on the
+ * server's auth challenges.
+ */
+static bool best_scheme_and_challenge(
+	git_http_auth_scheme **scheme_out,
+	const char **challenge_out,
+	git_vector *challenges,
+	git_cred *credentials)
+{
+	const char *challenge;
+	size_t i, j;
+
+	for (i = 0; i < ARRAY_SIZE(auth_schemes); i++) {
+		git_vector_foreach(challenges, j, challenge) {
+			git_http_auth_scheme *scheme = &auth_schemes[i];
+
+			if (challenge_matches_scheme(challenge, scheme) &&
+			    (scheme->credtypes & credentials->credtype)) {
+				*scheme_out = scheme;
+				*challenge_out = challenge;
+				return true;
+			}
+		}
+	}
+
+	return false;
+}
+
+/*
+ * Find the challenge from the server for our current auth context.
+ */
+static const char *challenge_for_context(
+	git_vector *challenges,
+	git_http_auth_context *auth_ctx)
+{
+	const char *challenge;
+	size_t i, j;
+
+	for (i = 0; i < ARRAY_SIZE(auth_schemes); i++) {
+		if (auth_schemes[i].type == auth_ctx->type) {
+			git_http_auth_scheme *scheme = &auth_schemes[i];
+
+			git_vector_foreach(challenges, j, challenge) {
+				if (challenge_matches_scheme(challenge, scheme))
+					return challenge;
+			}
+		}
+	}
+
+	return NULL;
+}
+
+static const char *init_auth_context(
+	git_http_server *server,
+	git_vector *challenges,
+	git_cred *credentials)
+{
+	git_http_auth_scheme *scheme;
+	const char *challenge;
+	int error;
+
+	if (!best_scheme_and_challenge(&scheme, &challenge, challenges, credentials)) {
+		git_error_set(GIT_ERROR_NET, "could not find appropriate mechanism for credentials");
+		return NULL;
+	}
+
+	error = scheme->init_context(&server->auth_context, &server->url);
+
+	if (error == GIT_PASSTHROUGH) {
+		git_error_set(GIT_ERROR_NET, "'%s' authentication is not supported", scheme->name);
+		return NULL;
+	}
+
+	return challenge;
+}
+
+static void free_auth_context(git_http_server *server)
+{
+	if (!server->auth_context)
+		return;
+
+	if (server->auth_context->free)
+		server->auth_context->free(server->auth_context);
+
+	server->auth_context = NULL;
+}
+
+static int apply_credentials(
+	git_buf *buf,
+	git_http_server *server,
+	const char *header_name,
+	git_cred *credentials)
+{
+	git_http_auth_context *auth = server->auth_context;
+	git_vector *challenges = &server->auth_challenges;
+	const char *challenge;
+	git_buf token = GIT_BUF_INIT;
+	int error = 0;
+
+	/* We've started a new request without creds; free the context. */
+	if (auth && !credentials) {
+		free_auth_context(server);
+		return 0;
+	}
+
+	/* We haven't authenticated, nor were we asked to.  Nothing to do. */
+	if (!auth && !git_vector_length(challenges))
+		return 0;
+
+	if (!auth) {
+		challenge = init_auth_context(server, challenges, credentials);
+		auth = server->auth_context;
+
+		if (!challenge || !auth) {
+			error = -1;
+			goto done;
+		}
+	} else if (auth->set_challenge) {
+		challenge = challenge_for_context(challenges, auth);
+	}
+
+	if (auth->set_challenge && challenge &&
+	    (error = auth->set_challenge(auth, challenge)) < 0)
+		goto done;
+
+	if ((error = auth->next_token(&token, auth, credentials)) < 0)
+		goto done;
+
+	if (auth->is_complete && auth->is_complete(auth)) {
+		/*
+		 * If we're done with an auth mechanism with connection affinity,
+		 * we don't need to send any more headers and can dispose the context.
+		 */
+		if (auth->connection_affinity)
+			free_auth_context(server);
+	} else if (!token.size) {
+		git_error_set(GIT_ERROR_NET, "failed to respond to authentication challange");
+		error = -1;
+		goto done;
+	}
+
+	if (token.size > 0)
+		error = git_buf_printf(buf, "%s: %s\r\n", header_name, token.ptr);
+
+done:
+	git_buf_dispose(&token);
+	return error;
+}
+
+GIT_INLINE(int) apply_server_credentials(
+	git_buf *buf,
+	git_http_client *client,
+	git_http_request *request)
+{
+	return apply_credentials(buf,
+	                         &client->server,
+	                         "Authorization",
+	                         request->credentials);
+}
+
+GIT_INLINE(int) apply_proxy_credentials(
+	git_buf *buf,
+	git_http_client *client,
+	git_http_request *request)
+{
+	return apply_credentials(buf,
+	                         &client->proxy,
+	                         "Proxy-Authorization",
+	                         request->proxy_credentials);
+}
+
 static int generate_request(
 	git_http_client *client,
 	git_http_request *request)
 {
-	const char *method, *path, *sep, *query;
 	git_buf *buf;
 	size_t i;
+	int error;
 
 	assert(client && request);
 
 	git_buf_clear(&client->request_msg);
 	buf = &client->request_msg;
 
-	method = name_for_method(request->method);
-	path = request->url->path ? request->url->path : "/";
-	sep = request->url->query ? "?" : "";
-	query = request->url->query ? request->url->query : "";
+	/* GET|POST path HTTP/1.1 */
+	git_buf_puts(buf, name_for_method(request->method));
+	git_buf_putc(buf, ' ');
+
+	if (request->proxy && strcmp(request->url->scheme, "https"))
+		git_net_url_fmt(buf, request->url);
+	else
+		git_net_url_fmt_path(buf, request->url);
 
-	git_buf_printf(buf, "%s %s%s%s HTTP/1.1\r\n",
-		method, path, sep, query);
+	git_buf_puts(buf, " HTTP/1.1\r\n");
 
 	git_buf_puts(buf, "User-Agent: ");
 	git_http__user_agent(buf);
@@ -355,6 +633,10 @@ static int generate_request(
 	if (request->expect_continue)
 		git_buf_printf(buf, "Expect: 100-continue\r\n");
 
+	if ((error = apply_server_credentials(buf, client, request)) < 0 ||
+	    (error = apply_proxy_credentials(buf, client, request)) < 0)
+		return error;
+
 	if (request->custom_headers) {
 		for (i = 0; i < request->custom_headers->count; i++) {
 			const char *hdr = request->custom_headers->strings[i];
@@ -424,6 +706,24 @@ static int stream_connect(
 	return error;
 }
 
+static void reset_auth_connection(git_http_server *server)
+{
+	/*
+	 * If we've authenticated and we're doing "normal"
+	 * authentication with a request affinity (Basic, Digest)
+	 * then we want to _keep_ our context, since authentication
+	 * survives even through non-keep-alive connections.  If
+	 * we've authenticated and we're doing connection-based
+	 * authentication (NTLM, Negotiate) - indicated by the presence
+	 * of an `is_complete` callback - then we need to restart
+	 * authentication on a new connection.
+	 */
+
+	if (server->auth_context &&
+	    server->auth_context->connection_affinity)
+		free_auth_context(server);
+}
+
 /*
  * Updates the server data structure with the new URL; returns 1 if the server
  * has changed and we need to reconnect, returns 0 otherwise.
@@ -511,6 +811,9 @@ static int http_client_connect(git_http_client *client)
 		client->proxy.stream = NULL;
 	}
 
+	reset_auth_connection(&client->server);
+	reset_auth_connection(&client->proxy);
+
 	reset_parser(client);
 
 	client->connected = 0;
@@ -611,7 +914,6 @@ GIT_INLINE(http_parser_settings *) http_client_parser_settings(void)
 	return &parser_settings;
 }
 
-
 GIT_INLINE(int) client_read_and_parse(git_http_client *client)
 {
 	http_parser *parser = &client->parser;
@@ -743,6 +1045,7 @@ int git_http_client_send_request(
 		complete_response_body(client);
 
 	http_parser_init(&client->parser, HTTP_RESPONSE);
+	git_buf_clear(&client->read_buf);
 
 	if (git_trace_level() >= GIT_TRACE_DEBUG) {
 		git_buf url = GIT_BUF_INIT;
@@ -844,6 +1147,11 @@ int git_http_client_read_response(
 		goto done;
 	}
 
+	git_http_response_dispose(response);
+
+	git_vector_free_deep(&client->server.auth_challenges);
+	git_vector_free_deep(&client->proxy.auth_challenges);
+
 	client->state = READING_RESPONSE;
 	client->parser.data = &parser_context;
 
@@ -913,6 +1221,40 @@ done:
 	return error;
 }
 
+int git_http_client_skip_body(git_http_client *client)
+{
+	http_parser_context parser_context = {0};
+	int error;
+
+	if (client->state == DONE)
+		return 0;
+
+	if (client->state != READING_BODY) {
+		git_error_set(GIT_ERROR_NET, "client is in invalid state");
+		return -1;
+	}
+
+	parser_context.client = client;
+	client->parser.data = &parser_context;
+
+	do {
+		error = client_read_and_parse(client);
+
+		if (parser_context.error != HPE_OK ||
+		    (parser_context.parse_status != PARSE_STATUS_OK &&
+		     parser_context.parse_status != PARSE_STATUS_NO_OUTPUT)) {
+			git_error_set(GIT_ERROR_NET,
+			              "unexpected data handled in callback");
+			error = -1;
+		}
+	} while (!error);
+
+	if (error < 0)
+		client->connected = 0;
+
+	return error;
+}
+
 /*
  * Create an http_client capable of communicating with the given remote
  * host.
@@ -947,6 +1289,9 @@ GIT_INLINE(void) http_server_close(git_http_server *server)
 	}
 
 	git_net_url_dispose(&server->url);
+
+	git_vector_free_deep(&server->auth_challenges);
+	free_auth_context(server);
 }
 
 static void http_client_close(git_http_client *client)

src/transports/httpclient.h

@@ -28,6 +28,8 @@ typedef struct {
 	/* Headers */
 	const char *accept;           /**< Contents of the Accept header */
 	const char *content_type;     /**< Content-Type header (for POST) */
+	git_cred *credentials;        /**< Credentials to authenticate with */
+	git_cred *proxy_credentials;  /**< Credentials for proxy */
 	git_strarray *custom_headers; /**< Additional headers to deliver */
 
 	/* To POST a payload, either set content_length OR set chunked. */
@@ -43,6 +45,15 @@ typedef struct {
 	char *content_type;
 	size_t content_length;
 	char *location;
+
+	/* Authentication headers */
+	unsigned server_auth_schemetypes; /**< Schemes requested by remote */
+	unsigned server_auth_credtypes;   /**< Supported cred types for remote */
+
+	unsigned proxy_auth_schemetypes;  /**< Schemes requested by proxy */
+	unsigned proxy_auth_credtypes;    /**< Supported cred types for proxy */
+
+	unsigned resend_credentials : 1;  /**< Resend with authentication */
 } git_http_response;
 
 typedef struct {
@@ -121,6 +132,15 @@ extern int git_http_client_read_body(
 	size_t buffer_size);
 
 /**
+ * Reads all of the (remainder of the) body of the response and ignores it.
+ * None of the data from the body will be returned to the caller.
+ *
+ * @param client the client to read the response from
+ * @return 0 or an error code
+ */
+extern int git_http_client_skip_body(git_http_client *client);
+
+/**
  * Examines the status code of the response to determine if it is a
  * redirect of any type (eg, 301, 302, etc).
  *