Merge pull request #4778 from libgit2/ethomson/clar-xml

Clar XML output redux

.vsts-ci.yml

@@ -11,128 +11,50 @@ jobs:
   pool:
     vmImage: 'Ubuntu 16.04'
   steps:
-  - task: Docker@0
-    displayName: Build
-    inputs:
-      action: 'Run an image'
+  - template: ci/vsts-docker.yml
+    parameters:
       imageName: 'libgit2/trusty-openssl:latest'
-      volumes: |
-       $(Build.SourcesDirectory):/src
-       $(Build.BinariesDirectory):/build
-      workDir: '/build'
-      containerCommand: '/src/ci/build.sh'
-      detached: false
-  - task: Docker@0
-    displayName: Test
-    inputs:
-      action: 'Run an image'
-      imageName: 'libgit2/trusty-openssl:latest'
-      volumes: |
-       $(Build.SourcesDirectory):/src
-       $(Build.BinariesDirectory):/build
-      envVars: |
+      environmentVariables: |
        CC=gcc
        LEAK_CHECK=valgrind
-      workDir: '/build'
-      containerCommand: '/src/ci/test.sh'
-      detached: false
 
 - job: linux_trusty_gcc_mbedtls
   displayName: 'Linux (Trusty; GCC; mbedTLS)'
   pool:
     vmImage: 'Ubuntu 16.04'
   steps:
-  - task: Docker@0
-    displayName: Build
-    inputs:
-      action: 'Run an image'
+  - template: ci/vsts-docker.yml
+    parameters:
       imageName: 'libgit2/trusty-mbedtls:latest'
-      volumes: |
-       $(Build.SourcesDirectory):/src
-       $(Build.BinariesDirectory):/build
-      envVars: |
+      environmentVariables: |
        CC=gcc
        CMAKE_OPTIONS=-DUSE_HTTPS=mbedTLS -DSHA1_BACKEND=mbedTLS
        LEAK_CHECK=valgrind
-      workDir: '/build'
-      containerCommand: '/src/ci/build.sh'
-      detached: false
-  - task: Docker@0
-    displayName: Test
-    inputs:
-      action: 'Run an image'
-      imageName: 'libgit2/trusty-mbedtls:latest'
-      volumes: |
-       $(Build.SourcesDirectory):/src
-       $(Build.BinariesDirectory):/build
-      envVars: 'LEAK_CHECK=valgrind'
-      workDir: '/build'
-      containerCommand: '/src/ci/test.sh'
-      detached: false
 
 - job: linux_trusty_clang_openssl
   displayName: 'Linux (Trusty; Clang; OpenSSL)'
   pool:
     vmImage: 'Ubuntu 16.04'
   steps:
-  - task: Docker@0
-    displayName: Build
-    inputs:
-      action: 'Run an image'
-      imageName: 'libgit2/trusty-openssl:latest'
-      volumes: |
-       $(Build.SourcesDirectory):/src
-       $(Build.BinariesDirectory):/build
-      workDir: '/build'
-      containerCommand: '/src/ci/build.sh'
-      detached: false
-  - task: Docker@0
-    displayName: Test
-    inputs:
-      action: 'Run an image'
+  - template: ci/vsts-docker.yml
+    parameters:
       imageName: 'libgit2/trusty-openssl:latest'
-      volumes: |
-       $(Build.SourcesDirectory):/src
-       $(Build.BinariesDirectory):/build
-      envVars: |
+      environmentVariables: |
        CC=clang
        LEAK_CHECK=valgrind
-      workDir: '/build'
-      containerCommand: '/src/ci/test.sh'
-      detached: false
 
 - job: linux_trusty_clang_mbedtls
   displayName: 'Linux (Trusty; Clang; mbedTLS)'
   pool:
     vmImage: 'Ubuntu 16.04'
   steps:
-  - task: Docker@0
-    displayName: Build
-    inputs:
-      action: 'Run an image'
+  - template: ci/vsts-docker.yml
+    parameters:
       imageName: 'libgit2/trusty-mbedtls:latest'
-      volumes: |
-       $(Build.SourcesDirectory):/src
-       $(Build.BinariesDirectory):/build
-      envVars: |
+      environmentVariables: |
        CC=clang
        CMAKE_OPTIONS=-DUSE_HTTPS=mbedTLS -DSHA1_BACKEND=mbedTLS
        LEAK_CHECK=valgrind
-      workDir: '/build'
-      containerCommand: '/src/ci/build.sh'
-      detached: false
-  - task: Docker@0
-    displayName: Test
-    inputs:
-      action: 'Run an image'
-      imageName: 'libgit2/trusty-mbedtls:latest'
-      volumes: |
-       $(Build.SourcesDirectory):/src
-       $(Build.BinariesDirectory):/build
-      envVars: 'LEAK_CHECK=valgrind'
-      workDir: '/build'
-      containerCommand: '/src/ci/test.sh'
-      detached: false
 
 - job: macos
   displayName: 'macOS'
@@ -141,37 +63,30 @@ jobs:
   steps:
   - bash: . '$(Build.SourcesDirectory)/ci/setup-osx.sh'
     displayName: Setup
-  - bash: . '$(Build.SourcesDirectory)/ci/build.sh'
-    displayName: Build
-    env:
-      PKG_CONFIG_PATH: /usr/local/opt/openssl/lib/pkgconfig
-  - bash: . '$(Build.SourcesDirectory)/ci/test.sh'
-    displayName: Test
-    env:
-      TMPDIR: $(Agent.TempDirectory)
-      LEAK_CHECK: leaks
+  - template: ci/vsts-bash.yml
+    parameters:
+      environmentVariables:
+        TMPDIR: $(Agent.TempDirectory)
+        PKG_CONFIG_PATH: /usr/local/opt/openssl/lib/pkgconfig
+        LEAK_CHECK: leaks
 
 - job: windows_vs_amd64
   displayName: 'Windows (Visual Studio; amd64)'
   pool: Hosted
   steps:
-  - powershell: . '$(Build.SourcesDirectory)\ci\build.ps1'
-    displayName: Build
-    env:
-      CMAKE_OPTIONS: -DMSVC_CRTDBG=ON -G"Visual Studio 12 2013 Win64"
-  - powershell: . '$(Build.SourcesDirectory)\ci\test.ps1'
-    displayName: Test
+  - template: ci/vsts-powershell.yml
+    parameters:
+      environmentVariables:
+        CMAKE_OPTIONS: -DMSVC_CRTDBG=ON -G"Visual Studio 12 2013 Win64"
 
 - job: windows_vs_x86
   displayName: 'Windows (Visual Studio; x86)'
   pool: Hosted
   steps:
-  - powershell: . '$(Build.SourcesDirectory)\ci\build.ps1'
-    displayName: Build
-    env:
-      CMAKE_OPTIONS: -DMSVC_CRTDBG=ON -G"Visual Studio 12 2013"
-  - powershell: . '$(Build.SourcesDirectory)\ci\test.ps1'
-    displayName: Test
+  - template: ci/vsts-powershell.yml
+    parameters:
+      environmentVariables:
+        CMAKE_OPTIONS: -DMSVC_CRTDBG=ON -G"Visual Studio 12 2013"
 
 - job: windows_mingw_amd64
   displayName: 'Windows (MinGW; amd64)'
@@ -182,13 +97,11 @@ jobs:
     env:
       TEMP: $(Agent.TempDirectory)
       ARCH: amd64
-  - powershell: . '$(Build.SourcesDirectory)\ci\build.ps1'
-    displayName: Build
-    env:
-      CMAKE_OPTIONS: -G"MinGW Makefiles"
-      PATH: $(Agent.TempDirectory)\mingw64\bin;C:\ProgramData\Oracle\Java\javapath;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Windows\System32\WindowsPowerShell\v1.0\;C:\Program Files (x86)\CMake\bin
-  - powershell: . '$(Build.SourcesDirectory)\ci\test.ps1'
-    displayName: Test
+  - template: ci/vsts-powershell.yml
+    parameters:
+      environmentVariables:
+        CMAKE_OPTIONS: -G"MinGW Makefiles"
+        PATH: $(Agent.TempDirectory)\mingw64\bin;C:\ProgramData\Oracle\Java\javapath;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Windows\System32\WindowsPowerShell\v1.0\;C:\Program Files (x86)\CMake\bin
 
 - job: windows_mingw_x86
   displayName: 'Windows (MinGW; x86)'
@@ -196,13 +109,12 @@ jobs:
   steps:
   - powershell: . '$(Build.SourcesDirectory)\ci\setup-mingw.ps1'
     displayName: Setup
+    workingDirectory: '$(Build.BinariesDirectory)'
     env:
       TEMP: $(Agent.TempDirectory)
       ARCH: x86
-  - powershell: . '$(Build.SourcesDirectory)\ci\build.ps1'
-    displayName: Build
-    env:
-      CMAKE_OPTIONS: -G"MinGW Makefiles"
-      PATH: $(Agent.TempDirectory)\mingw32\bin;C:\ProgramData\Oracle\Java\javapath;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Windows\System32\WindowsPowerShell\v1.0\;C:\Program Files (x86)\CMake\bin
-  - powershell: . '$(Build.SourcesDirectory)\ci\test.ps1'
-    displayName: Test
+  - template: ci/vsts-powershell.yml
+    parameters:
+      environmentVariables:
+        CMAKE_OPTIONS: -G"MinGW Makefiles"
+        PATH: $(Agent.TempDirectory)\mingw32\bin;C:\ProgramData\Oracle\Java\javapath;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Windows\System32\WindowsPowerShell\v1.0\;C:\Program Files (x86)\CMake\bin

CMakeLists.txt

@@ -62,6 +62,8 @@ OPTION(USE_EXT_HTTP_PARSER		"Use system HTTP_Parser if available"			 ON)
 OPTION(DEBUG_POOL			"Enable debug pool allocator"				OFF)
 OPTION(ENABLE_WERROR			"Enable compilation with -Werror"			OFF)
 OPTION(USE_BUNDLED_ZLIB    		"Use the bundled version of zlib"			OFF)
+   SET(CLAR_XML 			"OFF"					       CACHE STRING
+       "Writes test results in XML format.  One of ON, OFF or the directory to write to; this does not affect the output executables, this only affects the behavior of the ctest command.")
 
 IF (UNIX AND NOT APPLE)
 	OPTION(ENABLE_REPRODUCIBLE_BUILDS "Enable reproducible builds"				OFF)

ci/build.ps1

@@ -18,7 +18,9 @@ Write-Host "####################################################################
 Write-Host "## Configuring build environment"
 Write-Host "##############################################################################"
 
-Invoke-Expression "cmake ${SourceDirectory} -DBUILD_EXAMPLES=ON ${Env:CMAKE_OPTIONS}"
+$TestOutputDirectory = $BuildDirectory -replace "\\", "/"
+
+Invoke-Expression "cmake ${SourceDirectory} -DBUILD_EXAMPLES=ON -DCLAR_XML=${TestOutputDirectory} ${Env:CMAKE_OPTIONS}"
 if ($LastExitCode -ne 0) { [Environment]::Exit($LastExitCode) }
 
 Write-Host ""

ci/build.sh

@@ -28,8 +28,8 @@ echo "##########################################################################
 echo "## Configuring build environment"
 echo "##############################################################################"
 
-echo cmake ${SOURCE_DIR} -DBUILD_EXAMPLES=ON ${CMAKE_OPTIONS}
-cmake ${SOURCE_DIR} -DENABLE_WERROR=ON -DBUILD_EXAMPLES=ON -DBUILD_FUZZERS=ON -DUSE_STANDALONE_FUZZERS=ON ${CMAKE_OPTIONS}
+echo cmake ${SOURCE_DIR} -DENABLE_WERROR=ON -DBUILD_EXAMPLES=ON -DBUILD_FUZZERS=ON -DUSE_STANDALONE_FUZZERS=ON -DCLAR_XML=\"${BUILD_DIR}\" ${CMAKE_OPTIONS}
+cmake ${SOURCE_DIR} -DENABLE_WERROR=ON -DBUILD_EXAMPLES=ON -DBUILD_FUZZERS=ON -DUSE_STANDALONE_FUZZERS=ON -DCLAR_XML="${BUILD_DIR}" ${CMAKE_OPTIONS}
 
 echo ""
 echo "##############################################################################"

ci/vsts-bash.yml

+# These are the steps used for building on machines with bash.
+steps:
+- bash: . '$(Build.SourcesDirectory)/ci/build.sh'
+  displayName: Build
+  workingDirectory: '$(Build.BinariesDirectory)'
+  env: ${{ parameters.environmentVariables }}
+- bash: . '$(Build.SourcesDirectory)/ci/test.sh'
+  displayName: Test
+  workingDirectory: '$(Build.BinariesDirectory)'
+  env: ${{ parameters.environmentVariables }}
+- task: PublishTestResults@2
+  displayName: Publish Test Results
+  condition: succeededOrFailed()
+  inputs:
+    testResultsFiles: 'results_*.xml'
+    searchFolder: '$(Build.BinariesDirectory)'
+    mergeTestResults: true

ci/vsts-docker.yml

+# These are the steps used in a container-based build in VSTS.
+steps:
+- task: docker@0
+  displayName: Build
+  inputs:
+    action: 'Run an image'
+    imageName: ${{ parameters.imageName }}
+    volumes: |
+     $(Build.SourcesDirectory):/src
+     $(Build.BinariesDirectory):/build
+    envVars: ${{ parameters.environmentVariables }}
+    workDir: '/build'
+    containerCommand: '/src/ci/build.sh'
+    detached: false
+- task: docker@0
+  displayName: Test
+  inputs:
+    action: 'Run an image'
+    imageName: ${{ parameters.imageName }}
+    volumes: |
+     $(Build.SourcesDirectory):/src
+     $(Build.BinariesDirectory):/build
+    envVars: ${{ parameters.environmentVariables }}
+    workDir: '/build'
+    containerCommand: '/src/ci/test.sh'
+    detached: false
+- task: publishtestresults@2
+  displayName: Publish Test Results
+  condition: succeededOrFailed()
+  inputs:
+    testResultsFiles: 'results_*.xml'
+    searchFolder: '$(Build.BinariesDirectory)'
+    mergeTestResults: true

ci/vsts-powershell.yml

+# These are the steps used for building on machines with PowerShell.
+steps:
+- powershell: . '$(Build.SourcesDirectory)\ci\build.ps1'
+  displayName: Build
+  workingDirectory: '$(Build.BinariesDirectory)'
+  env: ${{ parameters.environmentVariables }}
+- powershell: . '$(Build.SourcesDirectory)\ci\test.ps1'
+  displayName: Test
+  workingDirectory: '$(Build.BinariesDirectory)'
+  env: ${{ parameters.environmentVariables }}
+- task: PublishTestResults@2
+  displayName: Publish Test Results
+  condition: succeededOrFailed()
+  inputs:
+    testResultsFiles: 'results_*.xml'
+    searchFolder: '$(Build.BinariesDirectory)'
+    mergeTestResults: true

tests/CMakeLists.txt

@@ -54,8 +54,22 @@ IF (MSVC_IDE)
 	SET_SOURCE_FILES_PROPERTIES("precompiled.c" COMPILE_FLAGS "/Ycprecompiled.h")
 ENDIF ()
 
-ADD_TEST(offline   "${libgit2_BINARY_DIR}/libgit2_clar" -v -xonline)
-ADD_TEST(online    "${libgit2_BINARY_DIR}/libgit2_clar" -v -sonline)
-ADD_TEST(gitdaemon "${libgit2_BINARY_DIR}/libgit2_clar" -v -sonline::push)
-ADD_TEST(ssh       "${libgit2_BINARY_DIR}/libgit2_clar" -v -sonline::push -sonline::clone::ssh_cert -sonline::clone::ssh_with_paths)
-ADD_TEST(proxy     "${libgit2_BINARY_DIR}/libgit2_clar" -v -sonline::clone::proxy_credentials_in_url -sonline::clone::proxy_credentials_request)
+IF (CLAR_XML)
+	IF (CLAR_XML STREQUAL "ON")
+		SET(XML_PATH "")
+	ELSE ()
+		SET(XML_PATH "${CLAR_XML}/")
+	ENDIF ()
+
+	SET(TESTS_OFFLINE_XML   "-r${XML_PATH}results_offline.xml")
+	SET(TESTS_ONLINE_XML    "-r${XML_PATH}results_online.xml")
+	SET(TESTS_GITDAEMON_XML "-r${XML_PATH}results_gitdaemon.xml")
+	SET(TESTS_SSH_XML       "-r${XML_PATH}results_ssh.xml")
+	SET(TESTS_PROXY_XML     "-r${XML_PATH}results_proxy.xml")
+ENDIF ()
+
+ADD_TEST(offline   "${libgit2_BINARY_DIR}/libgit2_clar" -v ${TESTS_OFFLINE_XML} -xonline)
+ADD_TEST(online    "${libgit2_BINARY_DIR}/libgit2_clar" -v ${TESTS_ONLINE_XML} -sonline)
+ADD_TEST(gitdaemon "${libgit2_BINARY_DIR}/libgit2_clar" -v ${TESTS_GITDAEMON_XML} -sonline::push)
+ADD_TEST(ssh       "${libgit2_BINARY_DIR}/libgit2_clar" -v ${TESTS_SSH_XML} -sonline::push -sonline::clone::ssh_cert -sonline::clone::ssh_with_paths)
+ADD_TEST(proxy     "${libgit2_BINARY_DIR}/libgit2_clar" -v ${TESTS_PROXY_XML} -sonline::clone::proxy_credentials_in_url -sonline::clone::proxy_credentials_request)

tests/clar.h

@@ -12,13 +12,16 @@
 enum cl_test_status {
 	CL_TEST_OK,
 	CL_TEST_FAILURE,
-	CL_TEST_SKIP
+	CL_TEST_SKIP,
+	CL_TEST_NOTRUN,
 };
 
+/** Setup clar environment */
 void clar_test_init(int argc, char *argv[]);
 int clar_test_run(void);
 void clar_test_shutdown(void);
 
+/** One shot setup & run */
 int clar_test(int argc, char *argv[]);
 
 const char *clar_sandbox_path(void);

tests/clar/print.h

@@ -13,16 +13,16 @@ static void clar_print_shutdown(int test_count, int suite_count, int error_count
 	(void)error_count;
 
 	printf("\n\n");
-	clar_report_errors();
+	clar_report_all();
 }
 
-static void clar_print_error(int num, const struct clar_error *error)
+static void clar_print_error(int num, const struct clar_report *report, const struct clar_error *error)
 {
 	printf("  %d) Failure:\n", num);
 
 	printf("%s::%s [%s:%d]\n",
-		error->suite,
-		error->test,
+		report->suite,
+		report->test,
 		error->file,
 		error->line_number);
 
@@ -44,6 +44,7 @@ static void clar_print_ontest(const char *test_name, int test_number, enum cl_te
 	case CL_TEST_OK: printf("."); break;
 	case CL_TEST_FAILURE: printf("F"); break;
 	case CL_TEST_SKIP: printf("S"); break;
+	case CL_TEST_NOTRUN: printf("N"); break;
 	}
 
 	fflush(stdout);

tests/clar/summary.h

+
+#include <stdio.h>
+#include <time.h>
+
+int clar_summary_close_tag(
+    struct clar_summary *summary, const char *tag, int indent)
+{
+	const char *indt;
+
+	if (indent == 0) indt = "";
+	else if (indent == 1) indt = "\t";
+	else indt = "\t\t";
+
+	return fprintf(summary->fp, "%s</%s>\n", indt, tag);
+}
+
+int clar_summary_testsuites(struct clar_summary *summary)
+{
+	return fprintf(summary->fp, "<testsuites>\n");
+}
+
+int clar_summary_testsuite(struct clar_summary *summary,
+    int idn, const char *name, const char *pkg, time_t timestamp,
+    double elapsed, int test_count, int fail_count, int error_count)
+{
+	struct tm *tm = localtime(&timestamp);
+	char iso_dt[20];
+
+	if (strftime(iso_dt, sizeof(iso_dt), "%Y-%m-%dT%H:%M:%S", tm) == 0)
+		return -1;
+
+	return fprintf(summary->fp, "\t<testsuite "
+		       " id=\"%d\""
+		       " name=\"%s\""
+		       " package=\"%s\""
+		       " hostname=\"localhost\""
+		       " timestamp=\"%s\""
+		       " time=\"%.2f\""
+		       " tests=\"%d\""
+		       " failures=\"%d\""
+		       " errors=\"%d\">\n",
+		       idn, name, pkg, iso_dt, elapsed, test_count, fail_count, error_count);
+}
+
+int clar_summary_testcase(struct clar_summary *summary,
+    const char *name, const char *classname, double elapsed)
+{
+	return fprintf(summary->fp,
+	    "\t\t<testcase name=\"%s\" classname=\"%s\" time=\"%.2f\">\n",
+		name, classname, elapsed);
+}
+
+int clar_summary_failure(struct clar_summary *summary,
+    const char *type, const char *message, const char *desc)
+{
+	return fprintf(summary->fp,
+	    "\t\t\t<failure type=\"%s\"><![CDATA[%s\n%s]]></failure>\n",
+	    type, message, desc);
+}
+
+struct clar_summary *clar_summary_init(const char *filename)
+{
+	struct clar_summary *summary;
+	FILE *fp;
+
+	if ((fp = fopen(filename, "w")) == NULL)
+		return NULL;
+
+	if ((summary = malloc(sizeof(struct clar_summary))) == NULL) {
+		fclose(fp);
+		return NULL;
+	}
+
+	summary->filename = filename;
+	summary->fp = fp;
+
+	return summary;
+}
+
+int clar_summary_shutdown(struct clar_summary *summary)
+{
+	struct clar_report *report;
+	const char *last_suite = NULL;
+
+	if (clar_summary_testsuites(summary) < 0)
+		goto on_error;
+
+	report = _clar.reports;
+	while (report != NULL) {
+		struct clar_error *error = report->errors;
+
+		if (last_suite == NULL || strcmp(last_suite, report->suite) != 0) {
+			if (clar_summary_testsuite(summary, 0, report->suite, "",
+			    time(NULL), 0, _clar.tests_ran, _clar.total_errors, 0) < 0)
+				goto on_error;
+		}
+
+		last_suite = report->suite;
+
+		clar_summary_testcase(summary, report->test, "what", 0);
+
+		while (error != NULL) {
+			if (clar_summary_failure(summary, "assert",
+			    error->error_msg, error->description) < 0)
+				goto on_error;
+
+			error = error->next;
+		}
+
+		if (clar_summary_close_tag(summary, "testcase", 2) < 0)
+			goto on_error;
+
+		report = report->next;
+
+		if (!report || strcmp(last_suite, report->suite) != 0) {
+			if (clar_summary_close_tag(summary, "testsuite", 1) < 0)
+				goto on_error;
+		}
+	}
+
+	if (clar_summary_close_tag(summary, "testsuites", 0) < 0 ||
+	    fclose(summary->fp) != 0)
+		goto on_error;
+
+	printf("written summary file to %s\n", summary->filename);
+
+	free(summary);
+	return 0;
+
+on_error:
+	fclose(summary->fp);
+	free(summary);
+	return -1;
+}