Merge pull request #4812 from libgit2/ethomson/ci-refactor

CI: refactoring

CMakeLists.txt

@@ -62,8 +62,6 @@ OPTION(USE_EXT_HTTP_PARSER		"Use system HTTP_Parser if available"			 ON)
 OPTION(DEBUG_POOL			"Enable debug pool allocator"				OFF)
 OPTION(ENABLE_WERROR			"Enable compilation with -Werror"			OFF)
 OPTION(USE_BUNDLED_ZLIB    		"Use the bundled version of zlib"			OFF)
-   SET(CLAR_XML 			"OFF"					       CACHE STRING
-       "Writes test results in XML format.  One of ON, OFF or the directory to write to; this does not affect the output executables, this only affects the behavior of the ctest command.")
 
 IF (UNIX AND NOT APPLE)
 	OPTION(ENABLE_REPRODUCIBLE_BUILDS "Enable reproducible builds"				OFF)

ci/build.ps1

@@ -18,9 +18,7 @@ Write-Host "####################################################################
 Write-Host "## Configuring build environment"
 Write-Host "##############################################################################"
 
-$TestOutputDirectory = $BuildDirectory -replace "\\", "/"
-
-Invoke-Expression "cmake ${SourceDirectory} -DBUILD_EXAMPLES=ON -DCLAR_XML=${TestOutputDirectory} ${Env:CMAKE_OPTIONS}"
+Invoke-Expression "cmake ${SourceDirectory} -DBUILD_EXAMPLES=ON ${Env:CMAKE_OPTIONS}"
 if ($LastExitCode -ne 0) { [Environment]::Exit($LastExitCode) }
 
 Write-Host ""

ci/build.sh

@@ -28,8 +28,8 @@ echo "##########################################################################
 echo "## Configuring build environment"
 echo "##############################################################################"
 
-echo cmake ${SOURCE_DIR} -DENABLE_WERROR=ON -DBUILD_EXAMPLES=ON -DBUILD_FUZZERS=ON -DUSE_STANDALONE_FUZZERS=ON -DCLAR_XML=\"${BUILD_DIR}\" ${CMAKE_OPTIONS}
-cmake ${SOURCE_DIR} -DENABLE_WERROR=ON -DBUILD_EXAMPLES=ON -DBUILD_FUZZERS=ON -DUSE_STANDALONE_FUZZERS=ON -DCLAR_XML="${BUILD_DIR}" ${CMAKE_OPTIONS}
+echo cmake ${SOURCE_DIR} -DENABLE_WERROR=ON -DBUILD_EXAMPLES=ON -DBUILD_FUZZERS=ON -DUSE_STANDALONE_FUZZERS=ON ${CMAKE_OPTIONS}
+cmake ${SOURCE_DIR} -DENABLE_WERROR=ON -DBUILD_EXAMPLES=ON -DBUILD_FUZZERS=ON -DUSE_STANDALONE_FUZZERS=ON ${CMAKE_OPTIONS}
 
 echo ""
 echo "##############################################################################"

ci/test.ps1

@@ -5,41 +5,68 @@ $PSDefaultParameterValues['*:ErrorAction'] = 'Stop'
 
 [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
 
+$SourceDir = Split-Path (Split-Path (Get-Variable MyInvocation).Value.MyCommand.Path)
+$BuildDir = Get-Location
+$Success = $true
+
 if ($Env:SKIP_TESTS) { exit }
 
+# Ask ctest what it would run if we were to invoke it directly.  This lets
+# us manage the test configuration in a single place (tests/CMakeLists.txt)
+# instead of running clar here as well.  But it allows us to wrap our test
+# harness with a leak checker like valgrind.  Append the option to write
+# JUnit-style XML files.
+function run_test {
+	$TestName = $args[0]
+
+	$TestCommand = (ctest -N -V -R "^$TestName$") -join "`n" -replace "(?ms).*\n^[0-9]*: Test command: ","" -replace "\n.*",""
+	$TestCommand += " -r${BuildDir}\results_${TestName}.xml"
+
+	Write-Host $TestCommand
+	Invoke-Expression $TestCommand
+
+	if ($LastExitCode -ne 0) { $Success = $false }
+}
+
 Write-Host "##############################################################################"
 Write-Host "## Configuring test environment"
 Write-Host "##############################################################################"
 
-Write-Host ""
-Write-Host "Starting HTTP proxy..."
-Invoke-WebRequest -Method GET -Uri https://github.com/ethomson/poxyproxy/releases/download/v0.1.0/poxyproxy-0.1.0.jar -OutFile poxyproxy.jar
-javaw -jar poxyproxy.jar -d --port 8080 --credentials foo:bar
+if (-not $Env:SKIP_PROXY_TESTS) {
+	Write-Host ""
+	Write-Host "Starting HTTP proxy..."
+	Invoke-WebRequest -Method GET -Uri https://github.com/ethomson/poxyproxy/releases/download/v0.1.0/poxyproxy-0.1.0.jar -OutFile poxyproxy.jar
+	javaw -jar poxyproxy.jar -d --port 8080 --credentials foo:bar
+}
 
 Write-Host ""
 Write-Host "##############################################################################"
 Write-Host "## Running (offline) tests"
 Write-Host "##############################################################################"
 
-ctest -V -R offline
-if ($LastExitCode -ne 0) { [Environment]::Exit($LastExitCode) }
+run_test offline
 
-Write-Host ""
-Write-Host "##############################################################################"
-Write-Host "## Running (online) tests"
-Write-Host "##############################################################################"
+if (-not $Env:SKIP_ONLINE_TESTS) {
+	Write-Host ""
+	Write-Host "##############################################################################"
+	Write-Host "## Running (online) tests"
+	Write-Host "##############################################################################"
 
-ctest -V -R online
-if ($LastExitCode -ne 0) { [Environment]::Exit($LastExitCode) }
+	run_test online
+}
 
-Write-Host ""
-Write-Host "Running proxy tests"
-Write-Host ""
+if (-not $Env:SKIP_PROXY_TESTS) {
+	Write-Host ""
+	Write-Host "Running proxy tests"
+	Write-Host ""
+
+	$Env:GITTEST_REMOTE_PROXY_URL="localhost:8080"
+	$Env:GITTEST_REMOTE_PROXY_USER="foo"
+	$Env:GITTEST_REMOTE_PROXY_PASS="bar"
+
+	run_test proxy
 
-$Env:GITTEST_REMOTE_PROXY_URL="localhost:8080"
-$Env:GITTEST_REMOTE_PROXY_USER="foo"
-$Env:GITTEST_REMOTE_PROXY_PASS="bar"
-ctest -V -R proxy
-if ($LastExitCode -ne 0) { [Environment]::Exit($LastExitCode) }
+	taskkill /F /IM javaw.exe
+}
 
-taskkill /F /IM javaw.exe
+if (-not $Success) { exit 1 }

ci/test.sh

@@ -11,6 +11,8 @@ BUILD_DIR=$(pwd)
 TMPDIR=${TMPDIR:-/tmp}
 USER=${USER:-$(whoami)}
 
+SUCCESS=1
+
 VALGRIND="valgrind --leak-check=full --show-reachable=yes --error-exitcode=125 --num-callers=50 --suppressions=\"$SOURCE_DIR/libgit2_clar.supp\""
 LEAKS="MallocStackLogging=1 MallocScribble=1 leaks -quiet -atExit -- nohup"
 
@@ -30,18 +32,19 @@ cleanup() {
 	echo "Done."
 }
 
-die() {
+failure() {
 	echo "Test exited with code: $1"
-
-	cleanup
-	exit $1
+	SUCCESS=0
 }
 
-# Ask ctest what it would run if we were to invoke it directly.  This lets us manage the
-# test configuration in a single place (tests/CMakeLists.txt) instead of running clar
-# here as well.  But it allows us to wrap our test harness with a leak checker like valgrind.
+# Ask ctest what it would run if we were to invoke it directly.  This lets
+# us manage the test configuration in a single place (tests/CMakeLists.txt)
+# instead of running clar here as well.  But it allows us to wrap our test
+# harness with a leak checker like valgrind.  Append the option to write
+# JUnit-style XML files.
 run_test() {
-	TEST_CMD=$(ctest -N -V -R $1 | sed -n 's/^[0-9]*: Test command: //p')
+	TEST_CMD=$(ctest -N -V -R "^${1}$" | sed -n 's/^[0-9]*: Test command: //p')
+	TEST_CMD="${TEST_CMD} -r${BUILD_DIR}/results_${1}.xml"
 
 	if [ "$LEAK_CHECK" = "valgrind" ]; then
 		RUNNER="$VALGRIND $TEST_CMD"
@@ -51,7 +54,7 @@ run_test() {
 		RUNNER="$TEST_CMD"
 	fi
 
-	eval $RUNNER || die $?
+	eval $RUNNER || failure
 }
 
 # Configure the test environment; run them early so that we're certain
@@ -191,10 +194,16 @@ if [ -z "$SKIP_FUZZERS" ]; then
 	echo "##############################################################################"
 
 	for fuzzer in fuzzers/*_fuzzer; do
-		"${fuzzer}" "${SOURCE_DIR}/fuzzers/corpora/$(basename "${fuzzer%_fuzzer}")" || die $?
+		"${fuzzer}" "${SOURCE_DIR}/fuzzers/corpora/$(basename "${fuzzer%_fuzzer}")" || failure
 	done
 fi
 
-echo "Success."
 cleanup
+
+if [ "$SUCCESS" -ne "1" ]; then
+	echo "Some tests failed."
+	exit 1
+fi
+
+echo "Success."
 exit 0

tests/CMakeLists.txt

@@ -54,22 +54,8 @@ IF (MSVC_IDE)
 	SET_SOURCE_FILES_PROPERTIES("precompiled.c" COMPILE_FLAGS "/Ycprecompiled.h")
 ENDIF ()
 
-IF (CLAR_XML)
-	IF (CLAR_XML STREQUAL "ON")
-		SET(XML_PATH "")
-	ELSE ()
-		SET(XML_PATH "${CLAR_XML}/")
-	ENDIF ()
-
-	SET(TESTS_OFFLINE_XML   "-r${XML_PATH}results_offline.xml")
-	SET(TESTS_ONLINE_XML    "-r${XML_PATH}results_online.xml")
-	SET(TESTS_GITDAEMON_XML "-r${XML_PATH}results_gitdaemon.xml")
-	SET(TESTS_SSH_XML       "-r${XML_PATH}results_ssh.xml")
-	SET(TESTS_PROXY_XML     "-r${XML_PATH}results_proxy.xml")
-ENDIF ()
-
-ADD_TEST(offline   "${libgit2_BINARY_DIR}/libgit2_clar" -v ${TESTS_OFFLINE_XML} -xonline)
-ADD_TEST(online    "${libgit2_BINARY_DIR}/libgit2_clar" -v ${TESTS_ONLINE_XML} -sonline)
-ADD_TEST(gitdaemon "${libgit2_BINARY_DIR}/libgit2_clar" -v ${TESTS_GITDAEMON_XML} -sonline::push)
-ADD_TEST(ssh       "${libgit2_BINARY_DIR}/libgit2_clar" -v ${TESTS_SSH_XML} -sonline::push -sonline::clone::ssh_cert -sonline::clone::ssh_with_paths)
-ADD_TEST(proxy     "${libgit2_BINARY_DIR}/libgit2_clar" -v ${TESTS_PROXY_XML} -sonline::clone::proxy_credentials_in_url -sonline::clone::proxy_credentials_request)
+ADD_TEST(offline   "${libgit2_BINARY_DIR}/libgit2_clar" -v -xonline)
+ADD_TEST(online    "${libgit2_BINARY_DIR}/libgit2_clar" -v -sonline)
+ADD_TEST(gitdaemon "${libgit2_BINARY_DIR}/libgit2_clar" -v -sonline::push)
+ADD_TEST(ssh       "${libgit2_BINARY_DIR}/libgit2_clar" -v -sonline::push -sonline::clone::ssh_cert -sonline::clone::ssh_with_paths)
+ADD_TEST(proxy     "${libgit2_BINARY_DIR}/libgit2_clar" -v -sonline::clone::proxy_credentials_in_url -sonline::clone::proxy_credentials_request)