ci: updates for CI

.github/workflows/main.yml

@@ -10,7 +10,7 @@ on:
   workflow_dispatch:
 
 env:
-  docker-registry: docker.pkg.github.com
+  docker-registry: ghcr.io
   docker-config-path: source/ci/docker
 
 jobs:
@@ -69,7 +69,7 @@ jobs:
         if [ "${{ matrix.container.base }}" != "" ]; then
           BASE_ARG="--build-arg BASE=${{ matrix.container.base }}"
         fi
-        docker build -t ${{ env.docker-registry-container-sha }} ${BASE_ARG} -f ${{ env.dockerfile }} .
+        docker build -t ${{ env.docker-registry-container-sha }} --build-arg UID=$(id -u) --build-arg GID=$(id -g) ${BASE_ARG} -f ${{ env.dockerfile }} .
         docker tag ${{ env.docker-registry-container-sha }} ${{ env.docker-registry-container-latest }}
         docker push ${{ env.docker-registry-container-sha }}
         docker push ${{ env.docker-registry-container-latest }}
@@ -85,7 +85,7 @@ jobs:
     strategy:
       matrix:
         platform:
-        - # Xenial, GCC, OpenSSL
+        - name: "Linux (Xenial, GCC, OpenSSL)"
           container:
             name: xenial
           env:
@@ -93,7 +93,7 @@ jobs:
             CMAKE_GENERATOR: Ninja
             CMAKE_OPTIONS: -DUSE_HTTPS=OpenSSL -DREGEX_BACKEND=builtin -DDEPRECATE_HARD=ON -DUSE_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DDEBUG_STRICT_ALLOC=ON -DDEBUG_STRICT_OPEN=ON
           os: ubuntu-latest
-        - # Xenial, GCC, mbedTLS
+        - name: "Linux (Xenial, GCC, mbedTLS)"
           container:
             name: xenial
           env:
@@ -101,7 +101,7 @@ jobs:
             CMAKE_GENERATOR: Ninja
             CMAKE_OPTIONS: -DUSE_HTTPS=mbedTLS -DUSE_SHA1=HTTPS -DDEPRECATE_HARD=ON -DUSE_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON
           os: ubuntu-latest
-        - # Xenial, Clang, OpenSSL
+        - name: "Linux (Xenial, Clang, OpenSSL)"
           container:
             name: xenial
           env:
@@ -109,7 +109,7 @@ jobs:
             CMAKE_GENERATOR: Ninja
             CMAKE_OPTIONS: -DUSE_HTTPS=OpenSSL -DDEPRECATE_HARD=ON -DUSE_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON
           os: ubuntu-latest
-        - # Xenial, Clang, mbedTLS
+        - name: "Linux (Xenial, Clang, mbedTLS)"
           container:
             name: xenial
           env:
@@ -117,7 +117,7 @@ jobs:
             CMAKE_OPTIONS: -DUSE_HTTPS=mbedTLS -DUSE_SHA1=HTTPS -DREGEX_BACKEND=pcre -DDEPRECATE_HARD=ON -DUSE_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON
             CMAKE_GENERATOR: Ninja
           os: ubuntu-latest
-        - # Focal, Clang 10, mbedTLS, MemorySanitizer
+        - name: "Linux (MemorySanitizer)"
           container:
             name: focal
           env:
@@ -130,7 +130,7 @@ jobs:
             ASAN_SYMBOLIZER_PATH: /usr/bin/llvm-symbolizer-10
             UBSAN_OPTIONS: print_stacktrace=1
           os: ubuntu-latest
-        - # Focal, Clang 10, OpenSSL, UndefinedBehaviorSanitizer
+        - name: "Linux (UndefinedBehaviorSanitizer)"
           container:
             name: focal
           env:
@@ -143,7 +143,7 @@ jobs:
             ASAN_SYMBOLIZER_PATH: /usr/bin/llvm-symbolizer-10
             UBSAN_OPTIONS: print_stacktrace=1
           os: ubuntu-latest
-        - # Focal, Clang 10, OpenSSL, ThreadSanitizer
+        - name: "Linux (ThreadSanitizer)"
           container:
             name: focal
           env:
@@ -157,7 +157,7 @@ jobs:
             UBSAN_OPTIONS: print_stacktrace=1
             TSAN_OPTIONS: suppressions=/home/libgit2/source/script/thread-sanitizer.supp second_deadlock_stack=1
           os: ubuntu-latest
-        - # macOS
+        - name: "macOS"
           os: macos-10.15
           env:
             CC: clang
@@ -166,7 +166,7 @@ jobs:
             SKIP_SSH_TESTS: true
             SKIP_NEGOTIATE_TESTS: true
           setup-script: osx
-        - # Windows amd64 Visual Studio
+        - name: "Windows (amd64, Visual Studio)"
           os: windows-2019
           env:
             ARCH: amd64
@@ -174,7 +174,7 @@ jobs:
             CMAKE_OPTIONS: -A x64 -DWIN32_LEAKCHECK=ON -DDEPRECATE_HARD=ON
             SKIP_SSH_TESTS: true
             SKIP_NEGOTIATE_TESTS: true
-        - # Windows x86 Visual Studio
+        - name: "Windows (x86, Visual Studio)"
           os: windows-2019
           env:
             ARCH: x86
@@ -182,7 +182,7 @@ jobs:
             CMAKE_OPTIONS: -A Win32 -DWIN32_LEAKCHECK=ON -DDEPRECATE_HARD=ON -DUSE_SHA1=HTTPS -DUSE_BUNDLED_ZLIB=ON
             SKIP_SSH_TESTS: true
             SKIP_NEGOTIATE_TESTS: true
-        - # Windows amd64 mingw
+        - name: "Windows (amd64, mingw)"
           os: windows-2019
           setup-script: mingw
           env:
@@ -193,7 +193,7 @@ jobs:
             BUILD_PATH: D:\Temp\mingw64\bin;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Program Files (x86)\CMake\bin
             SKIP_SSH_TESTS: true
             SKIP_NEGOTIATE_TESTS: true
-        - # Windows x86 mingw
+        - name: "Windows (x86, mingw)"
           os: windows-2019
           setup-script: mingw
           env:
@@ -214,7 +214,7 @@ jobs:
         path: source
         fetch-depth: 0
     - name: Set up build environment
-      run: source/ci/setup-${{ matrix.platform.setup-script }}.sh
+      run: source/ci/setup-${{ matrix.platform.setup-script }}-build.sh
       shell: bash
       if: matrix.platform.setup-script != ''
     - name: Setup QEMU
@@ -237,10 +237,12 @@ jobs:
         export GITTEST_NEGOTIATE_PASSWORD="${{ secrets.GITTEST_NEGOTIATE_PASSWORD }}"
 
         if [ -n "${{ matrix.platform.container.name }}" ]; then
+          mkdir build
           docker run \
               --rm \
-              --user libgit2:libgit2 \
+              --user "$(id -u):$(id -g)" \
               -v "$(pwd)/source:/home/libgit2/source" \
+              -v "$(pwd)/build:/home/libgit2/build" \
               -w /home/libgit2 \
               -e ASAN_SYMBOLIZER_PATH \
               -e CC \
@@ -248,19 +250,40 @@ jobs:
               -e CMAKE_GENERATOR \
               -e CMAKE_OPTIONS \
               -e GITTEST_NEGOTIATE_PASSWORD \
+              -e GITTEST_FLAKY_STAT \
               -e PKG_CONFIG_PATH \
               -e SKIP_NEGOTIATE_TESTS \
               -e SKIP_SSH_TESTS \
               -e TSAN_OPTIONS \
               -e UBSAN_OPTIONS \
               ${{ env.docker-registry-container-sha }} \
-              /bin/bash -c "mkdir build && cd build && ../source/ci/build.sh && ../source/ci/test.sh"
+              /bin/bash -c "cd build && ../source/ci/build.sh && ../source/ci/test.sh"
         else
-          mkdir build && cd build
+          mkdir build
+          cd build
           ../source/ci/build.sh
           ../source/ci/test.sh
         fi
       shell: bash
+    - name: Upload test results
+      uses: actions/upload-artifact@v3
+      if: success() || failure()
+      with:
+        name: test-results-${{ matrix.platform.id }}
+        path: build/results_*.xml
+
+  test_results:
+    name: Test results
+    needs: [ build ]
+    runs-on: ubuntu-latest
+    steps:
+    - name: Download test results
+      uses: actions/download-artifact@v3
+    - name: Generate test summary
+      uses: test-summary/action@v1
+      with:
+        paths: 'test-results-*/*.xml'
+
 
   # Generate documentation using docurium.  We'll upload the documentation
   # as a build artifact so that it can be reviewed as part of a pull

ci/build.sh

@@ -59,7 +59,7 @@ echo "##########################################################################
 echo "## Configuring build environment"
 echo "##############################################################################"
 
-echo cmake -DENABLE_WERROR=ON -DBUILD_EXAMPLES=ON -DBUILD_FUZZERS=ON -DUSE_STANDALONE_FUZZERS=ON -G \"${CMAKE_GENERATOR}\" ${CMAKE_OPTIONS} -S \"${SOURCE_DIR}\"
+echo "${CMAKE}" -DENABLE_WERROR=ON -DBUILD_EXAMPLES=ON -DBUILD_FUZZERS=ON -DUSE_STANDALONE_FUZZERS=ON -G \"${CMAKE_GENERATOR}\" ${CMAKE_OPTIONS} -S \"${SOURCE_DIR}\"
 env PATH="${BUILD_PATH}" "${CMAKE}" -DENABLE_WERROR=ON -DBUILD_EXAMPLES=ON -DBUILD_FUZZERS=ON -DUSE_STANDALONE_FUZZERS=ON -G "${CMAKE_GENERATOR}" ${CMAKE_OPTIONS} -S "${SOURCE_DIR}"
 
 echo ""
@@ -69,10 +69,11 @@ echo "##########################################################################
 
 # Determine parallelism; newer cmake supports `--build --parallel` but
 # we cannot yet rely on that.
-if [ "${CMAKE_GENERATOR}" = "Unix Makefiles" -a "${CORES}" != "" ]; then
+if [ "${CMAKE_GENERATOR}" = "Unix Makefiles" -a "${CORES}" != "" -a "${CMAKE_BUILD_OPTIONS}" = "" ]; then
 	BUILDER=(make -j ${CORES})
 else
-	BUILDER=("${CMAKE}" --build .)
+	BUILDER=("${CMAKE}" --build . ${CMAKE_BUILD_OPTIONS})
 fi
 
+echo "${BUILDER[@]}"
 env PATH="${BUILD_PATH}" "${BUILDER[@]}"

ci/docker/bionic

@@ -38,7 +38,12 @@ RUN cd /tmp && \
     rm -rf mbedtls-2.16.2
 
 FROM mbedtls AS adduser
-RUN useradd --shell /bin/bash libgit2 --create-home
+ARG UID=""
+ARG GID=""
+RUN if [ "${UID}" != "" ]; then USER_ARG="--uid ${UID}"; fi && \
+    if [ "${GID}" != "" ]; then GROUP_ARG="--gid ${GID}"; fi && \
+    groupadd ${GROUP_ARG} libgit2 && \
+    useradd ${USER_ARG} --gid libgit2 --shell /bin/bash --create-home libgit2
 
 FROM adduser AS configure
 RUN mkdir /var/run/sshd

ci/docker/centos7

@@ -48,7 +48,12 @@ RUN cd /tmp && \
     rm -rf cmake-3.21.1
 
 FROM cmake AS adduser
-RUN useradd --shell /bin/bash libgit2 --create-home
+ARG UID=""
+ARG GID=""
+RUN if [ "${UID}" != "" ]; then USER_ARG="--uid ${UID}"; fi && \
+    if [ "${GID}" != "" ]; then GROUP_ARG="--gid ${GID}"; fi && \
+    groupadd ${GROUP_ARG} libgit2 && \
+    useradd ${USER_ARG} --gid libgit2 --shell /bin/bash --create-home libgit2
 
 FROM adduser AS configure
 ENV PKG_CONFIG_PATH /usr/local/lib/pkgconfig

ci/docker/centos8

 ARG BASE=centos:8
 
-FROM ${BASE} AS yum
+FROM ${BASE} AS stream
+RUN dnf -y --disablerepo '*' --enablerepo=extras swap centos-linux-repos centos-stream-repos && \
+    dnf -y distro-sync
+
+FROM stream AS yum
 RUN yum install -y \
 	which \
 	bzip2 \
@@ -40,7 +44,12 @@ RUN cd /tmp && \
     rm -rf valgrind-3.15.0
 
 FROM valgrind AS adduser
-RUN useradd --shell /bin/bash libgit2 --create-home
+ARG UID=""
+ARG GID=""
+RUN if [ "${UID}" != "" ]; then USER_ARG="--uid ${UID}"; fi && \
+    if [ "${GID}" != "" ]; then GROUP_ARG="--gid ${GID}"; fi && \
+    groupadd ${GROUP_ARG} libgit2 && \
+    useradd ${USER_ARG} --gid libgit2 --shell /bin/bash --create-home libgit2
 
 FROM adduser AS configure
 ENV PKG_CONFIG_PATH /usr/local/lib/pkgconfig

ci/docker/focal

@@ -73,7 +73,13 @@ RUN cd /tmp && \
     rm -rf valgrind-3.15.0
 
 FROM valgrind AS adduser
-RUN useradd --shell /bin/bash libgit2 --create-home
+ARG UID=""
+ARG GID=""
+RUN if [ "${UID}" != "" ]; then USER_ARG="--uid ${UID}"; fi && \
+    if [ "${GID}" != "" ]; then GROUP_ARG="--gid ${GID}"; fi && \
+    groupadd ${GROUP_ARG} libgit2 && \
+    useradd ${USER_ARG} --gid libgit2 --shell /bin/bash --create-home libgit2
+
 
 FROM adduser AS configure
 RUN mkdir /var/run/sshd

ci/docker/xenial

@@ -60,7 +60,13 @@ RUN cd /tmp && \
     rm -rf valgrind-3.15.0
 
 FROM valgrind AS adduser
-RUN useradd --shell /bin/bash libgit2 --create-home
+ARG UID=""
+ARG GID=""
+RUN if [ "${UID}" != "" ]; then USER_ARG="--uid ${UID}"; fi && \
+    if [ "${GID}" != "" ]; then GROUP_ARG="--gid ${GID}"; fi && \
+    groupadd ${GROUP_ARG} libgit2 && \
+    useradd ${USER_ARG} --gid libgit2 --shell /bin/bash --create-home libgit2
+
 
 FROM adduser AS configure
 RUN mkdir /var/run/sshd

ci/getcontainer.sh

@@ -37,9 +37,13 @@ DOCKER_REGISTRY_CONTAINER_SHA="${DOCKER_REGISTRY_CONTAINER}:${DOCKER_SHA}"
 echo "docker-registry-container-sha=${DOCKER_REGISTRY_CONTAINER_SHA}" >> $GITHUB_ENV
 echo "docker-registry-container-latest=${DOCKER_REGISTRY_CONTAINER}:latest" >> $GITHUB_ENV
 
+echo "::: logging in to ${DOCKER_REGISTRY} as ${GITHUB_ACTOR}"
+
 exists="true"
 docker login https://${DOCKER_REGISTRY} -u ${GITHUB_ACTOR} -p ${GITHUB_TOKEN} || exists="false"
 
+echo "::: pulling ${DOCKER_REGISTRY_CONTAINER_SHA}"
+
 if [ "${exists}" != "false" ]; then
 	docker pull ${DOCKER_REGISTRY_CONTAINER_SHA} || exists="false"
 fi

ci/setup-mingw-build.sh

+#!/bin/sh
+
+set -ex
+
+echo "##############################################################################"
+echo "## Downloading mingw"
+echo "##############################################################################"
+
+BUILD_TEMP=${BUILD_TEMP:=$TEMP}
+BUILD_TEMP=$(cygpath $BUILD_TEMP)
+
+case "$ARCH" in
+	amd64)
+		MINGW_URI="https://github.com/libgit2/ci-dependencies/releases/download/2021-05-04/mingw-x86_64-8.1.0-release-win32-sjlj-rt_v6-rev0.zip";;
+	x86)
+		MINGW_URI="https://github.com/libgit2/ci-dependencies/releases/download/2021-05-04/mingw-i686-8.1.0-release-win32-sjlj-rt_v6-rev0.zip";;
+esac
+
+if [ -z "$MINGW_URI" ]; then
+	echo "No URL"
+	exit 1
+fi
+
+mkdir -p "$BUILD_TEMP"
+
+curl -s -L "$MINGW_URI" -o "$BUILD_TEMP"/mingw-"$ARCH".zip
+unzip -q "$BUILD_TEMP"/mingw-"$ARCH".zip -d "$BUILD_TEMP"

ci/setup-osx-benchmark.sh

+#!/bin/sh
+
+set -ex
+
+brew update
+brew install hyperfine

ci/setup-osx-build.sh

+#!/bin/sh
+
+set -ex
+
+brew update
+brew install pkgconfig zlib curl openssl libssh2 ninja
+
+ln -s /Applications/Xcode.app/Contents/Developer/usr/lib/libLeaksAtExit.dylib /usr/local/lib

ci/setup-ubuntu-benchmark.sh

+#!/bin/sh
+
+set -ex
+
+sudo apt-get update
+sudo apt-get install -y --no-install-recommends \
+	cargo \
+	cmake \
+	gcc \
+	git \
+	krb5-user \
+	libkrb5-dev \
+	libssl-dev \
+	libz-dev \
+	make \
+	ninja-build \
+	pkgconf
+
+wget https://github.com/sharkdp/hyperfine/releases/download/v1.12.0/hyperfine_1.12.0_amd64.deb
+sudo dpkg -i hyperfine_1.12.0_amd64.deb

ci/setup-win32-benchmark.sh

+#!/bin/sh
+
+set -ex
+
+choco install hyperfine zip
+
+CHOCO_PATH=$(mktemp -d)
+curl -L https://github.com/ethomson/PurgeStandbyList/releases/download/v1.0/purgestandbylist.1.0.0.nupkg -o "${CHOCO_PATH}/purgestandbylist.1.0.0.nupkg"
+choco install purgestandbylist -s $(cygpath -w "${CHOCO_PATH}")

ci/test.sh

@@ -158,10 +158,18 @@ fi
 if [ -z "$SKIP_OFFLINE_TESTS" ]; then
 	echo ""
 	echo "##############################################################################"
-	echo "## Running (offline) tests"
+	echo "## Running core tests"
 	echo "##############################################################################"
 
+	echo ""
+	echo "Running libgit2 integration (offline) tests"
+	echo ""
 	run_test offline
+
+	echo ""
+	echo "Running utility tests"
+	echo ""
+	run_test util
 fi
 
 if [ -n "$RUN_INVASIVE_TESTS" ]; then
@@ -185,7 +193,7 @@ if [ -z "$SKIP_ONLINE_TESTS" ]; then
 
 	echo ""
 	echo "##############################################################################"
-	echo "## Running (online) tests"
+	echo "## Running networking (online) tests"
 	echo "##############################################################################"
 
 	export GITTEST_FLAKY_RETRY=5
@@ -195,9 +203,9 @@ if [ -z "$SKIP_ONLINE_TESTS" ]; then
 	# Run the online tests that immutably change global state separately
 	# to avoid polluting the test environment.
 	echo ""
-	echo "##############################################################################"
-	echo "## Running (online_customcert) tests"
-	echo "##############################################################################"
+	echo "Running custom certificate (online_customcert) tests"
+	echo ""
+
 	run_test online_customcert
 fi