Merge pull request #5140 from libgit2/ethomson/flaky_ci

Re-run flaky tests

ci/test.ps1

@@ -29,8 +29,32 @@ function run_test {
 	$TestCommand = (ctest -N -V -R "^$TestName$") -join "`n" -replace "(?ms).*\n^[0-9]*: Test command: ","" -replace "\n.*",""
 	$TestCommand += " -r${BuildDir}\results_${TestName}.xml"
 
-	Invoke-Expression $TestCommand
-	if ($LastExitCode -ne 0) { $global:Success = $false }
+	if ($Env:GITTEST_FLAKY_RETRY -gt 0) {
+		$AttemptsRemain = $Env:GITTEST_FLAKY_RETRY
+	} else {
+		$AttemptsRemain = 1
+	}
+
+	$Failed = 0
+	while ($AttemptsRemain -ne 0) {
+		if ($Failed -eq 1) {
+			Write-Host ""
+			Write-Host "Re-running flaky $TestName tests..."
+			Write-Host ""
+		}
+
+		Invoke-Expression $TestCommand
+		if ($LastExitCode -eq 0) {
+			$Failed = 0
+			break
+		} else {
+			$Failed = 1
+		}
+
+		$AttemptsRemain = $AttemptsRemain - 1
+	}
+
+	if ($Failed -eq 1) { $global:Success = $false }
 }
 
 Write-Host "##############################################################################"
@@ -79,7 +103,9 @@ if (-not $Env:SKIP_ONLINE_TESTS) {
 	Write-Host "## Running (online) tests"
 	Write-Host "##############################################################################"
 
+	$Env:GITTEST_FLAKY_RETRY=5
 	run_test online
+	$Env:GITTEST_FLAKY_RETRY=0
 }
 
 if (-not $Env:SKIP_PROXY_TESTS) {

ci/test.sh

@@ -32,11 +32,6 @@ cleanup() {
 	echo "Done."
 }
 
-failure() {
-	echo "Test exited with code: $1"
-	SUCCESS=0
-}
-
 # Ask ctest what it would run if we were to invoke it directly.  This lets
 # us manage the test configuration in a single place (tests/CMakeLists.txt)
 # instead of running clar here as well.  But it allows us to wrap our test
@@ -60,7 +55,35 @@ run_test() {
 		RUNNER="$TEST_CMD"
 	fi
 
-	eval $RUNNER || failure
+	if [[ "$GITTEST_FLAKY_RETRY" > 0 ]]; then
+		ATTEMPTS_REMAIN=$GITTEST_FLAKY_RETRY
+	else
+		ATTEMPTS_REMAIN=1
+	fi
+
+	FAILED=0
+	while [[ "$ATTEMPTS_REMAIN" > 0 ]]; do
+		if [ "$FAILED" -eq 1 ]; then
+			echo ""
+			echo "Re-running flaky ${1} tests..."
+			echo ""
+		fi
+
+		RETURN_CODE=0
+		eval $RUNNER || RETURN_CODE=$? && true
+
+		if [ "$RETURN_CODE" -eq 0 ]; then
+			break
+		fi
+
+		echo "Test exited with code: $RETURN_CODE"
+		ATTEMPTS_REMAIN="$(($ATTEMPTS_REMAIN-1))"
+		FAILED=1
+	done
+
+	if [ "$FAILED" -ne 0 ]; then
+		SUCCESS=0
+	fi
 }
 
 # Configure the test environment; run them early so that we're certain
@@ -166,7 +189,9 @@ if [ -z "$SKIP_ONLINE_TESTS" ]; then
 	echo "## Running (online) tests"
 	echo "##############################################################################"
 
+	export GITTEST_FLAKY_RETRY=5
 	run_test online
+	unset GITTEST_FLAKY_RETRY
 fi
 
 if [ -z "$SKIP_GITDAEMON_TESTS" ]; then
@@ -238,7 +263,7 @@ fi
 
 cleanup
 
-if [ "$SUCCESS" -ne "1" ]; then
+if [ "$SUCCESS" -ne 1 ]; then
 	echo "Some tests failed."
 	exit 1
 fi