[CI]: breakout 7: integration tests reports

.github/workflows/job-build.yml

@@ -42,10 +42,11 @@ jobs:
       - if: ${{ inputs.canary }}
         name: "Init (canary): retrieve GO_VERSION"
         run: |
+          . ./hack/github/action-helpers.sh
           latest_go="$(. ./hack/provisioning/version/fetch.sh; go::canary::for::go-setup)"
           printf "GO_VERSION=%s\n" "$latest_go" >> "$GITHUB_ENV"
           [ "$latest_go" != "" ] || \
-            echo "::warning title=No canary go::There is currently no canary go version to test. Steps will not run."
+            github::log::warning "No canary go" "There is currently no canary go version to test. Steps will not run."
 
       - if: ${{ env.GO_VERSION != '' }}
         name: "Init: install go"
@@ -57,13 +58,41 @@ jobs:
       - if: ${{ env.GO_VERSION != '' }}
         name: "Run: make binaries"
         run: |
+          . ./hack/github/action-helpers.sh
+
+          github::md::table::header "OS" "Arch" "Result" "Time" >> $GITHUB_STEP_SUMMARY
+
+          failure=
+
+          build(){
+            local goos="$1"
+            local goarch="${2:-amd64}"
+            local goarm="${3:-}"
+            local result
+
+            github::timer::begin
+
+            GOOS="$goos" GOARCH="$goarch" GOARM="$goarm" make binaries \
+              && result="$decorator_success" \
+              || {
+                failure=true
+                result="$decorator_failure"
+              }
+
+            [ ! "$goarm" ] || goarch="$goarch/v$goarm"
+            github::md::table::line "$goos" "$goarch" "$result" "$(github::timer::format <(github::timer::tick))" >> $GITHUB_STEP_SUMMARY
+          }
+
           # We officially support these
-          GOOS=linux make binaries
-          GOOS=windows make binaries
-          GOOS=freebsd make binaries
-          GOOS=darwin make binaries
-          GOARCH=arm GOARM=6 make binaries
+          build linux
+          build linux arm64
+          build windows
+          build freebsd
+          build darwin
+          build linux arm 6
           # These architectures are not released, but we still verify that we can at least compile
-          GOARCH=ppc64le make binaries
-          GOARCH=riscv64 make binaries
-          GOARCH=s390x make binaries
+          build linux ppc64le
+          build linux riscv64
+          build linux s390x
+
+          [ ! "$failure" ] || exit 1

.github/workflows/job-test-in-container.yml

@@ -146,9 +146,12 @@ jobs:
           if echo "${ROOTLESSKIT_VERSION}" | grep -q v1; then
             WORKAROUND_ISSUE_622=1
           fi
-          echo "WORKAROUND_ISSUE_622=${WORKAROUND_ISSUE_622}" >> "$GITHUB_ENV"
+          echo "WORKAROUND_ISSUE_622=$WORKAROUND_ISSUE_622" >> "$GITHUB_ENV"
       - name: "Run: integration tests"
         run: |
+          . ./hack/github/action-helpers.sh
+          github::md::h2 "non-flaky" >> "$GITHUB_STEP_SUMMARY"
+
           # IPV6 note: nested IPv6 network inside docker and qemu is complex and needs a bunch of sysctl config.
           # Therefore, it's hard to debug why the IPv6 tests fail in such an isolation layer.
           # On the other side, using the host network is easier at configuration.
@@ -158,18 +161,21 @@ jobs:
             && args=(test-integration ./hack/test-integration.sh) \
             || args=(test-integration-${{ inputs.target }} /test-integration-rootless.sh ./hack/test-integration.sh)
           if [ "${{ inputs.ipv6 }}" == true ]; then
-            docker run --network host -t --rm --privileged -e WORKAROUND_ISSUE_622=${WORKAROUND_ISSUE_622} "${args[@]}" -test.only-flaky=false -test.only-ipv6 -test.target=${{ inputs.binary }}
+            docker run --network host -t --rm --privileged -e GITHUB_STEP_SUMMARY="$GITHUB_STEP_SUMMARY" -v "$GITHUB_STEP_SUMMARY":"$GITHUB_STEP_SUMMARY" -e WORKAROUND_ISSUE_622=${WORKAROUND_ISSUE_622:-} "${args[@]}" -test.only-flaky=false -test.only-ipv6 -test.target=${{ inputs.binary }}
           else
-            docker run -t --rm --privileged -e WORKAROUND_ISSUE_622=${WORKAROUND_ISSUE_622} "${args[@]}" -test.only-flaky=false -test.target=${{ inputs.binary }}
+            docker run -t --rm --privileged -e GITHUB_STEP_SUMMARY="$GITHUB_STEP_SUMMARY" -v "$GITHUB_STEP_SUMMARY":"$GITHUB_STEP_SUMMARY" -e WORKAROUND_ISSUE_622=${WORKAROUND_ISSUE_622:-} "${args[@]}" -test.only-flaky=false -test.target=${{ inputs.binary }}
           fi
       # FIXME: this NEEDS to go away
       - name: "Run: integration tests (flaky)"
         run: |
+          . ./hack/github/action-helpers.sh
+          github::md::h2 "flaky" >> "$GITHUB_STEP_SUMMARY"
+
           [ "${{ inputs.target }}" == "rootful" ] \
             && args=(test-integration ./hack/test-integration.sh) \
             || args=(test-integration-${{ inputs.target }} /test-integration-rootless.sh ./hack/test-integration.sh)
           if [ "${{ inputs.ipv6 }}" == true ]; then
-            docker run --network host -t --rm --privileged -e WORKAROUND_ISSUE_622=${WORKAROUND_ISSUE_622} "${args[@]}" -test.only-flaky=true -test.only-ipv6 -test.target=${{ inputs.binary }}
+            docker run --network host -t --rm --privileged -e GITHUB_STEP_SUMMARY="$GITHUB_STEP_SUMMARY" -v "$GITHUB_STEP_SUMMARY":"$GITHUB_STEP_SUMMARY" -e WORKAROUND_ISSUE_622=${WORKAROUND_ISSUE_622:-} "${args[@]}" -test.only-flaky=true -test.only-ipv6 -test.target=${{ inputs.binary }}
           else
-            docker run -t --rm --privileged -e WORKAROUND_ISSUE_622=${WORKAROUND_ISSUE_622} "${args[@]}" -test.only-flaky=true -test.target=${{ inputs.binary }}
+            docker run -t --rm --privileged -e GITHUB_STEP_SUMMARY="$GITHUB_STEP_SUMMARY" -v "$GITHUB_STEP_SUMMARY":"$GITHUB_STEP_SUMMARY" -e WORKAROUND_ISSUE_622=${WORKAROUND_ISSUE_622:-} "${args[@]}" -test.only-flaky=true -test.target=${{ inputs.binary }}
           fi

.github/workflows/job-test-in-host.yml

@@ -156,6 +156,8 @@ jobs:
           go install ./cmd/nerdctl
           echo "::endgroup::"
 
+          choco install jq
+
       - if: ${{ env.SHOULD_RUN == 'yes' }}
         name: "Init: install dev tools"
         run: |
@@ -167,15 +169,24 @@ jobs:
       - if: ${{ contains(inputs.runner, 'ubuntu') && env.SHOULD_RUN == 'yes' }}
         name: "Run (linux): integration tests (IPv6)"
         run: |
+          . ./hack/github/action-helpers.sh
+          github::md::h2 "ipv6" >> "$GITHUB_STEP_SUMMARY"
+
           ./hack/test-integration.sh -test.target=${{ inputs.binary }} -test.only-ipv6
 
       - if: ${{ env.SHOULD_RUN == 'yes' }}
         name: "Run: integration tests"
         run: |
+          . ./hack/github/action-helpers.sh
+          github::md::h2 "non-flaky" >> "$GITHUB_STEP_SUMMARY"
+
           ./hack/test-integration.sh -test.target=${{ inputs.binary }} -test.only-flaky=false
 
       # FIXME: this must go
       - if: ${{ env.SHOULD_RUN == 'yes' }}
         name: "Run: integration tests (flaky)"
         run: |
+          . ./hack/github/action-helpers.sh
+          github::md::h2 "flaky" >> "$GITHUB_STEP_SUMMARY"
+
           ./hack/test-integration.sh -test.target=${{ inputs.binary }} -test.only-flaky=true

.github/workflows/workflow-lint.yml

@@ -74,7 +74,7 @@ jobs:
           - go-version: "1.24"
             canary: true
     with:
-      timeout: 5
+      timeout: 10
       go-version: ${{ matrix.go-version }}
       runner: ubuntu-24.04
       canary: ${{ matrix.canary && true || false }}

Dockerfile

@@ -295,8 +295,10 @@ CMD ["bash", "--login", "-i"]
 FROM base AS test-integration
 ARG DEBIAN_FRONTEND=noninteractive
 # `expect` package contains `unbuffer(1)`, which is used for emulating TTY for testing
+# `jq` is required to generate test summaries
 RUN apt-get update -qq && apt-get install -qq --no-install-recommends \
   expect \
+  jq \
   git \
   make
 # We wouldn't need this if Docker Hub could have "golang:${GO_VERSION}-ubuntu"

hack/github/action-helpers.sh

@@ -0,0 +1,119 @@
+#!/usr/bin/env bash
+
+#   Copyright The containerd Authors.
+
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+
+#       http://www.apache.org/licenses/LICENSE-2.0
+
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+# shellcheck disable=SC2034
+set -o errexit -o errtrace -o functrace -o nounset -o pipefail
+
+readonly decorator_success="✅"
+readonly decorator_failure="❌"
+
+github::md::h1(){
+  printf "# %s\n" "$1"
+}
+
+github::md::h2(){
+  printf "## %s\n" "$1"
+}
+
+github::md::h3(){
+  printf "### %s\n" "$1"
+}
+
+github::md::bq(){
+  local x
+  for x in "$@"; do
+    printf "> %s\n" "$1"
+  done
+}
+
+github::md::table::header(){
+  printf "|"
+  for x in "$@"; do
+    printf " %s |" "$x"
+  done
+  printf "\n"
+  printf "|"
+  for x in "$@"; do
+    printf "%s|" "---"
+  done
+  printf "\n"
+}
+
+github::md::table::line(){
+  printf "|"
+  for x in "$@"; do
+    printf " %s |" "$x"
+  done
+  printf "\n"
+}
+
+github::md::pie(){
+  local title="$1"
+  local label
+  local value
+  shift
+
+  printf '```mermaid\npie\n    title %s\n' "$title"
+  while [ "$#" -gt 0 ]; do
+    label="$1"
+    value="$2"
+    shift
+    shift
+    printf '    "%s" : %s\n' "$label" "$value"
+  done
+  printf '```\n\n'
+
+}
+
+github::log::group(){
+  echo "::group::$*"
+}
+
+github::log::endgroup(){
+  echo "::endgroup::"
+}
+
+github::log::warning(){
+  local title="$1"
+  local msg="$2"
+
+  echo "::warning title=$title::$msg"
+}
+
+_begin=${_begin:-}
+_duration=
+github::timer::begin(){
+  _begin="$(date +%s)"
+}
+
+github::timer::tick(){
+  local tick
+
+  tick="$(date +%s)"
+  printf "%s" "$((tick - _begin))"
+}
+
+github::timer::format() {
+  local t
+  t="$(cat "$1")"
+  local h=$((t/60/60%24))
+  local m=$((t/60%60))
+  local s=$((t%60))
+
+  [[ "$h" == 0 ]] || printf "%d hours " "$h"
+  [[ "$m" == 0 ]] || printf "%d minutes " "$m"
+  printf '%d seconds' "$s"
+}

hack/github/gotestsum-reporter.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+
+#   Copyright The containerd Authors.
+
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+
+#       http://www.apache.org/licenses/LICENSE-2.0
+
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+# shellcheck disable=SC2034,SC2015
+set -o errexit -o errtrace -o functrace -o nounset -o pipefail
+root="$(cd "$(dirname "${BASH_SOURCE[0]:-$PWD}")" 2>/dev/null 1>&2 && pwd)"
+readonly root
+
+# shellcheck source=/dev/null
+. "$root"/action-helpers.sh
+
+GITHUB_STEP_SUMMARY="${GITHUB_STEP_SUMMARY:-/dev/null}"
+
+{
+  github::md::h3 "Total number of tests: $TESTS_TOTAL"
+  github::md::pie "Status" "Skipped" "$TESTS_SKIPPED" "Failed" "$TESTS_FAILED" "Passed" "$(( TESTS_TOTAL - TESTS_FAILED - TESTS_SKIPPED ))"
+
+  # shellcheck disable=SC2207
+  pie=($(jq -rc 'select(has("Test") | not) | select(.Elapsed) | select(.Elapsed > 0) | "\(.Package) \(.Elapsed) "' < "$GOTESTSUM_JSONFILE"))
+  github::md::pie "Time spent per package" "${pie[@]}"
+
+  github::md::h3 "Failing tests"
+  echo '```'
+  jq -rc 'select(.Action == "fail") | select(.Test) | .Test' < "$GOTESTSUM_JSONFILE"
+  echo '```'
+
+  github::md::h3 "Tests taking more than 15 seconds"
+  echo '```'
+  gotestsum tool slowest --threshold 15s --jsonfile "$GOTESTSUM_JSONFILE"
+  echo '```'
+} >> "$GITHUB_STEP_SUMMARY"

hack/test-integration.sh

@@ -32,6 +32,9 @@ readonly needsudo="${WITH_SUDO:-}"
 
 # See https://github.com/containerd/nerdctl/blob/main/docs/testing/README.md#about-parallelization
 args=(--format=testname --jsonfile /tmp/test-integration.log --packages="$root"/../cmd/nerdctl/...)
+# FIXME: not working on windows. Need to change approach: move away from --post-run-command and
+# just process the log file. This might also allow multi-steps/multi-target results aggregation.
+[ "$(uname -s)" != "Linux" ] || args+=(--post-run-command "$root"/github/gotestsum-reporter.sh)
 
 if [ "$#" == 0 ]; then
   "$root"/test-integration.sh -test.only-flaky=false
@@ -51,6 +54,3 @@ if [ "$needsudo" == "true" ] || [ "$needsudo" == "yes" ] || [ "$needsudo" == "1"
 else
   gotestsum "${args[@]}" -- -timeout="$timeout" -p 1 -args -test.allow-kill-daemon "$@"
 fi
-
-echo "These are the tests that took more than 10 seconds:"
-gotestsum tool slowest --threshold 10s --jsonfile /tmp/test-integration.log