Merge pull request #5308 from libgit2/ethomson/cifix

CI Build Updates

azure-pipelines.yml

@@ -19,7 +19,7 @@ jobs:
       environmentVariables: |
        CC=gcc
        CMAKE_GENERATOR=Ninja
-       CMAKE_OPTIONS=-DUSE_HTTPS=OpenSSL -DREGEX_BACKEND=builtin -DDEPRECATE_HARD=ON -DUSE_LEAK_CHECKER=valgrind
+       CMAKE_OPTIONS=-DUSE_HTTPS=OpenSSL -DREGEX_BACKEND=builtin -DDEPRECATE_HARD=ON -DUSE_LEAK_CHECKER=valgrind -DVALGRIND=on
 
 - job: linux_amd64_xenial_gcc_mbedtls
   displayName: 'Linux (amd64; Xenial; GCC; mbedTLS)'
@@ -34,7 +34,7 @@ jobs:
       environmentVariables: |
        CC=gcc
        CMAKE_GENERATOR=Ninja
-       CMAKE_OPTIONS=-DUSE_HTTPS=mbedTLS -DUSE_SHA1=HTTPS -DDEPRECATE_HARD=ON -DUSE_LEAK_CHECKER=valgrind
+       CMAKE_OPTIONS=-DUSE_HTTPS=mbedTLS -DUSE_SHA1=HTTPS -DDEPRECATE_HARD=ON -DUSE_LEAK_CHECKER=valgrind -DVALGRIND=on
 
 - job: linux_amd64_xenial_clang_openssl
   displayName: 'Linux (amd64; Xenial; Clang; OpenSSL)'
@@ -49,7 +49,7 @@ jobs:
       environmentVariables: |
        CC=clang
        CMAKE_GENERATOR=Ninja
-       CMAKE_OPTIONS=-DUSE_HTTPS=OpenSSL -DDEPRECATE_HARD=ON -DUSE_LEAK_CHECKER=valgrind
+       CMAKE_OPTIONS=-DUSE_HTTPS=OpenSSL -DDEPRECATE_HARD=ON -DUSE_LEAK_CHECKER=valgrind -DVALGRIND=on
 
 - job: linux_amd64_xenial_clang_mbedtls
   displayName: 'Linux (amd64; Xenial; Clang; mbedTLS)'
@@ -64,7 +64,7 @@ jobs:
       environmentVariables: |
        CC=clang
        CMAKE_GENERATOR=Ninja
-       CMAKE_OPTIONS=-DUSE_HTTPS=mbedTLS -DUSE_SHA1=HTTPS -DREGEX_BACKEND=pcre -DDEPRECATE_HARD=ON -DUSE_LEAK_CHECKER=valgrind
+       CMAKE_OPTIONS=-DUSE_HTTPS=mbedTLS -DUSE_SHA1=HTTPS -DREGEX_BACKEND=pcre -DDEPRECATE_HARD=ON -DUSE_LEAK_CHECKER=valgrind -DVALGRIND=on
 
 - job: macos
   displayName: 'macOS'

azure-pipelines/build.sh

@@ -11,6 +11,7 @@ SOURCE_DIR=${SOURCE_DIR:-$( cd "$( dirname "${BASH_SOURCE[0]}" )" && dirname $(
 BUILD_DIR=$(pwd)
 BUILD_PATH=${BUILD_PATH:=$PATH}
 CMAKE=$(which cmake)
+CMAKE_GENERATOR=${CMAKE_GENERATOR:-Unix Makefiles}
 
 indent() { sed "s/^/    /"; }
 
@@ -25,7 +26,7 @@ fi
 
 if [ -f "/etc/debian_version" ]; then
 	echo "Debian version:"
-	lsb_release -a | indent
+	(source /etc/lsb-release && echo "${DISTRIB_DESCRIPTION}") | indent
 fi
 
 echo "Kernel version:"

azure-pipelines/docker.yml

@@ -4,9 +4,19 @@ steps:
   - script: docker run --rm --privileged multiarch/qemu-user-static:register --reset
     displayName: 'Register Docker QEMU'
 
+- task: cache@2
+  displayName: Cache Docker layers
+  inputs:
+    key: docker
+    path: /tmp/dockercache
+- script: |
+    if [ -f /tmp/dockercache/${{parameters.docker.image}}.tar ]; then docker load < /tmp/dockercache/${{parameters.docker.image}}.tar; fi
+  displayName: 'Load Docker cache'
 - script: |
     cd $(Build.SourcesDirectory)/azure-pipelines/docker
     docker build -t libgit2/${{parameters.docker.image}} --build-arg BASE=${{parameters.docker.base}} -f ${{parameters.docker.image}} .
+    if [ ! -d /tmp/dockercache ]; then mkdir /tmp/dockercache; fi
+    docker save libgit2/${{parameters.docker.image}} $(docker history -q libgit2/${{parameters.docker.image}} | grep -v '<missing>') > /tmp/dockercache/${{parameters.docker.image}}.tar
   displayName: 'Build Docker image'
 - task: docker@0
   displayName: Build

azure-pipelines/docker/xenial

 ARG BASE
-FROM $BASE
-RUN echo 'deb http://ppa.launchpad.net/hola-launchpad/valgrind/ubuntu xenial main' >/etc/apt/sources.list.d/valgrind.list && \
-    apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 8A0303A7544D59A08EBD1D621BCFD9273D292CF6 && \
-    apt-get update && \
+FROM $BASE AS apt
+RUN apt-get update && \
     apt-get install -y --no-install-recommends \
+        bzip2 \
         clang \
         cmake \
         curl \
@@ -11,21 +10,20 @@ RUN echo 'deb http://ppa.launchpad.net/hola-launchpad/valgrind/ubuntu xenial mai
         git \
         gosu \
         libcurl4-gnutls-dev \
+        libgcrypt20-dev \
         libpcre3-dev \
-        libssh2-1-dev \
         libssl-dev \
         libz-dev \
+        make \
         ninja-build \
         openjdk-8-jre-headless \
         openssh-server \
         openssl \
         pkgconf \
         python \
-        valgrind \
-        && \
-    rm -rf /var/lib/apt/lists/*
-RUN mkdir /var/run/sshd
+        valgrind
 
+FROM apt AS mbedtls
 RUN cd /tmp && \
     curl -LO https://tls.mbed.org/download/mbedtls-2.16.2-apache.tgz && \
     tar -xf mbedtls-2.16.2-apache.tgz && \
@@ -37,17 +35,32 @@ RUN cd /tmp && \
     cd .. && \
     rm -rf mbedtls-2.16.2
 
+FROM mbedtls AS libssh2
 RUN cd /tmp && \
     curl -LO https://www.libssh2.org/download/libssh2-1.8.2.tar.gz && \
     tar -xf libssh2-1.8.2.tar.gz && \
     rm -f libssh2-1.8.2.tar.gz && \
     cd libssh2-1.8.2 && \
-    CFLAGS=-fPIC cmake -G Ninja -DCRYPTO_BACKEND=Libgcrypt . && \
+    CFLAGS=-fPIC cmake -G Ninja -DBUILD_SHARED_LIBS=ON -DCRYPTO_BACKEND=Libgcrypt . && \
     ninja install && \
     cd .. && \
     rm -rf libssh2-1.8.2
 
+FROM libssh2 AS valgrind
+RUN cd /tmp && \
+    curl -LO https://sourceware.org/pub/valgrind/valgrind-3.15.0.tar.bz2 && \
+    tar -xf valgrind-3.15.0.tar.bz2 && \
+    rm -f valgrind-3.15.0.tar.bz2 && \
+    cd valgrind-3.15.0 && \
+    ./configure && \
+    make && \
+    make install && \
+    cd .. && \
+    rm -rf valgrind-3.15.0
+
+FROM valgrind AS configure
 COPY entrypoint.sh /usr/local/bin/entrypoint.sh
 RUN chmod a+x /usr/local/bin/entrypoint.sh
+RUN mkdir /var/run/sshd
 
 ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]

script/valgrind.supp

@@ -86,6 +86,7 @@
 	...
 	fun:gcry_mpi_scan
 	obj:*libssh2.so*
+	...
 }
 
 {
@@ -116,7 +117,6 @@
 	ignore-libssh2-gcrypt-session-handshake
 	Memcheck:Leak
 	...
-	obj:*libgcrypt.so*
 	obj:*libssh2.so*
 	obj:*libssh2.so*
 	fun:libssh2_session_handshake
@@ -124,6 +124,44 @@
 }
 
 {
+	ignore-openssl-undefined-in-read
+	Memcheck:Cond
+	...
+	obj:*libssl.so*
+	...
+	fun:openssl_read
+	...
+}
+
+{
+	ignore-openssl-undefined-in-connect
+	Memcheck:Cond
+	...
+	obj:*libssl.so*
+	...
+	fun:openssl_connect
+	...
+}
+
+{
+	ignore-libssh2-rsa-sha1-sign
+	Memcheck:Leak
+	...
+	obj:*libgcrypt.so*
+	fun:_libssh2_rsa_sha1_sign
+	...
+}
+
+{
+	ignore-libssh2-kexinit
+	Memcheck:Leak
+	...
+	obj:*libssh2.so*
+	fun:kexinit
+	...
+}
+
+{
 	ignore-noai6ai_cached-double-free
 	Memcheck:Free
 	fun:free
@@ -132,3 +170,11 @@
 	fun:exit
 	...
 }
+
+{
+	ignore-libcrypto-uninitialized-read-for-entropy
+	Memcheck:Value8
+	...
+	obj:*libcrypto.so*
+	...
+}

src/CMakeLists.txt

@@ -288,9 +288,11 @@ IF (WIN32 AND NOT CYGWIN)
 ELSEIF (AMIGA)
 	ADD_DEFINITIONS(-DNO_ADDRINFO -DNO_READDIR_R -DNO_MMAP)
 ELSE()
+	ADD_FEATURE_INFO(valgrind VALGRIND "valgrind hints")
 	IF (VALGRIND)
-		ADD_DEFINITIONS(-DNO_MMAP)
+		ADD_DEFINITIONS(-DVALGRIND)
 	ENDIF()
+
 	FILE(GLOB SRC_OS unix/*.c unix/*.h)
 ENDIF()
 

src/streams/openssl.c

@@ -30,6 +30,10 @@
 #include <openssl/x509v3.h>
 #include <openssl/bio.h>
 
+#ifdef VALGRIND
+# include <valgrind/memcheck.h>
+#endif
+
 SSL_CTX *git__ssl_ctx;
 
 #define GIT_SSL_DEFAULT_CIPHERS "ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-DSS-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA:DHE-DSS-AES128-SHA256:DHE-DSS-AES256-SHA256:DHE-DSS-AES128-SHA:DHE-DSS-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA"
@@ -311,6 +315,10 @@ static int bio_write(BIO *b, const char *buf, int len)
 {
 	git_stream *io = (git_stream *) BIO_get_data(b);
 
+#ifdef VALGRIND
+	VALGRIND_MAKE_MEM_DEFINED(buf, len);
+#endif
+
 	return (int) git_stream_write(io, buf, len, 0);
 }
 
@@ -587,6 +595,10 @@ static int openssl_connect(git_stream *stream)
 	BIO_set_data(bio, st->io);
 	SSL_set_bio(st->ssl, bio, bio);
 
+#ifdef VALGRIND
+	VALGRIND_MAKE_MEM_DEFINED(st->ssl, sizeof(SSL));
+#endif
+
 	/* specify the host in case SNI is needed */
 #ifdef SSL_CTRL_SET_TLSEXT_HOSTNAME
 	SSL_set_tlsext_host_name(st->ssl, st->host);
@@ -597,6 +609,10 @@ static int openssl_connect(git_stream *stream)
 
 	st->connected = true;
 
+#ifdef VALGRIND
+	VALGRIND_MAKE_MEM_DEFINED(st->ssl, sizeof(SSL));
+#endif
+
 	return verify_server_cert(st->ssl, st->host);
 }
 
@@ -663,6 +679,10 @@ static ssize_t openssl_read(git_stream *stream, void *data, size_t len)
 	if ((ret = SSL_read(st->ssl, data, len)) <= 0)
 		return ssl_set_error(st->ssl, ret);
 
+#ifdef VALGRIND
+	VALGRIND_MAKE_MEM_DEFINED(data, ret);
+#endif
+
 	return ret;
 }
 

tests/online/clone.c

@@ -864,6 +864,12 @@ void test_online_clone__proxy_cred_callback_after_failed_url_creds(void)
 	git_buf_dispose(&url);
 }
 
+void test_online_clone__azurerepos(void)
+{
+	cl_git_pass(git_clone(&g_repo, "https://libgit2@dev.azure.com/libgit2/test/_git/test", "./foo", &g_options));
+	cl_assert(git_path_exists("./foo/master.txt"));
+}
+
 void test_online_clone__path_whitespace(void)
 {
 	cl_git_pass(git_clone(&g_repo, "https://libgit2@dev.azure.com/libgit2/test/_git/spaces%20in%20the%20name", "./foo", &g_options));