Merge pull request #2508 from crazy-max/integration-tests-coverage

test: setup integration tests coverage

.github/workflows/build.yml

@@ -36,6 +36,7 @@ jobs:
       TESTFLAGS_DOCKER: "-v --parallel=1 --timeout=30m"
       TEST_IMAGE_BUILD: "0"
       TEST_IMAGE_ID: "buildx-tests"
+      TEST_COVERAGE: "1"
     strategy:
       fail-fast: false
       matrix:
@@ -125,6 +126,7 @@ jobs:
           directory: ./bin/testreports
           flags: integration
           token: ${{ secrets.CODECOV_TOKEN }}
+          disable_file_fixes: true
       -
         name: Generate annotations
         if: always()
@@ -197,6 +199,7 @@ jobs:
           env_vars: RUNNER_OS
           flags: unit
           token: ${{ secrets.CODECOV_TOKEN }}
+          disable_file_fixes: true
       -
         name: Generate annotations
         if: always()

Dockerfile

@@ -27,10 +27,36 @@ WORKDIR /src
 
 FROM gobase AS gotestsum
 ARG GOTESTSUM_VERSION
-ENV GOFLAGS=
-RUN --mount=target=/root/.cache,type=cache \
-  GOBIN=/out/ go install "gotest.tools/gotestsum@${GOTESTSUM_VERSION}" && \
-  /out/gotestsum --version
+ENV GOFLAGS=""
+RUN --mount=target=/root/.cache,type=cache <<EOT
+  set -ex
+  go install "gotest.tools/gotestsum@${GOTESTSUM_VERSION}"
+  go install "github.com/wadey/gocovmerge@latest"
+  mkdir /out
+  /go/bin/gotestsum --version
+  mv /go/bin/gotestsum /out
+  mv /go/bin/gocovmerge /out
+EOT
+COPY --chmod=755 <<"EOF" /out/gotestsumandcover
+#!/bin/sh
+set -x
+if [ -z "$GO_TEST_COVERPROFILE" ]; then
+  exec gotestsum "$@"
+fi
+coverdir="$(dirname "$GO_TEST_COVERPROFILE")"
+mkdir -p "$coverdir/helpers"
+gotestsum "$@" "-coverprofile=$GO_TEST_COVERPROFILE"
+ecode=$?
+go tool covdata textfmt -i=$coverdir/helpers -o=$coverdir/helpers-report.txt
+gocovmerge "$coverdir/helpers-report.txt" "$GO_TEST_COVERPROFILE" > "$coverdir/merged-report.txt"
+mv "$coverdir/merged-report.txt" "$GO_TEST_COVERPROFILE"
+rm "$coverdir/helpers-report.txt"
+for f in "$coverdir/helpers"/*; do
+  rm "$f"
+done
+rmdir "$coverdir/helpers"
+exit $ecode
+EOF
 
 FROM gobase AS buildx-version
 RUN --mount=type=bind,target=. <<EOT
@@ -42,6 +68,7 @@ EOT
 
 FROM gobase AS buildx-build
 ARG TARGETPLATFORM
+ARG GO_EXTRA_FLAGS
 RUN --mount=type=bind,target=. \
   --mount=type=cache,target=/root/.cache \
   --mount=type=cache,target=/go/pkg/mod \
@@ -88,7 +115,7 @@ RUN apk add --no-cache \
       shadow-uidmap \
       xfsprogs \
       xz
-COPY --link --from=gotestsum /out/gotestsum /usr/bin/
+COPY --link --from=gotestsum /out /usr/bin/
 COPY --link --from=registry /bin/registry /usr/bin/
 COPY --link --from=docker-engine / /usr/bin/
 COPY --link --from=docker-cli / /usr/bin/

codecov.yml

@@ -1 +1,4 @@
 comment: false
+
+ignore:
+  - "**/*.pb.go"

docker-bake.hcl

@@ -7,6 +7,9 @@ variable "DOCS_FORMATS" {
 variable "DESTDIR" {
   default = "./bin"
 }
+variable "TEST_COVERAGE" {
+  default = null
+}
 variable "GOLANGCI_LINT_MULTIPLATFORM" {
   default = ""
 }
@@ -192,6 +195,7 @@ variable "TEST_BUILDKIT_TAG" {
 target "integration-test-base" {
   inherits = ["_common"]
   args = {
+    GO_EXTRA_FLAGS = TEST_COVERAGE == "1" ? "-cover" : null
     HTTP_PROXY = HTTP_PROXY
     HTTPS_PROXY = HTTPS_PROXY
     NO_PROXY = NO_PROXY

hack/test

@@ -4,6 +4,7 @@ set -eu -o pipefail
 
 : "${BUILDX_CMD=docker buildx}"
 
+: "${TEST_COVERAGE=}"
 : "${TEST_IMAGE_BUILD=1}"
 : "${TEST_IMAGE_ID=buildx-tests}"
 
@@ -22,7 +23,7 @@ testReportsDir="$(pwd)/bin/testreports"
 mkdir -p "$testReportsDir"
 testReportsVol="-v $testReportsDir:/testreports"
 gotestsumArgs="--format=standard-verbose --jsonfile=/testreports/go-test-report$TEST_REPORT_SUFFIX.json --junitfile=/testreports/junit-report$TEST_REPORT_SUFFIX.xml"
-gotestArgs="-mod=vendor -coverprofile=/testreports/coverage-report$TEST_REPORT_SUFFIX.txt -covermode=atomic"
+gotestArgs="-mod=vendor"
 
 cacheVolume="buildx-test-cache"
 if ! docker container inspect "$cacheVolume" >/dev/null 2>/dev/null; then
@@ -32,5 +33,23 @@ if [ "$TEST_KEEP_CACHE" != "1" ]; then
   trap 'docker rm -v $cacheVolume' EXIT
 fi
 
-cid=$(docker create --rm -v /tmp $testReportsVol --volumes-from=$cacheVolume -e GITHUB_REF -e TEST_DOCKERD -e TEST_BUILDKIT_IMAGE -e TEST_BUILDKIT_TAG -e TEST_BUILDX_EXPERIMENTAL -e SKIP_INTEGRATION_TESTS -e GOTESTSUM_FORMAT ${BUILDKIT_INTEGRATION_SNAPSHOTTER:+"-eBUILDKIT_INTEGRATION_SNAPSHOTTER"} -e BUILDKIT_REGISTRY_MIRROR_DIR=/root/.cache/registry --privileged $TEST_IMAGE_ID gotestsum $gotestsumArgs --packages="${TESTPKGS:-./...}" -- $gotestArgs ${TESTFLAGS:--v})
+if [ "$TEST_COVERAGE" = "1" ]; then
+  export GO_TEST_COVERPROFILE="/testreports/coverage-report$TEST_REPORT_SUFFIX.txt"
+fi
+
+cid=$(docker create --rm --privileged \
+  -v /tmp $testReportsVol \
+  --volumes-from=$cacheVolume \
+  -e GITHUB_REF \
+  -e TEST_DOCKERD \
+  -e TEST_BUILDKIT_IMAGE \
+  -e TEST_BUILDKIT_TAG \
+  -e TEST_BUILDX_EXPERIMENTAL \
+  -e SKIP_INTEGRATION_TESTS \
+  -e GOTESTSUM_FORMAT \
+  -e GO_TEST_COVERPROFILE \
+  ${BUILDKIT_INTEGRATION_SNAPSHOTTER:+"-eBUILDKIT_INTEGRATION_SNAPSHOTTER"} \
+  -e BUILDKIT_REGISTRY_MIRROR_DIR=/root/.cache/registry \
+  $TEST_IMAGE_ID gotestsumandcover $gotestsumArgs --packages="${TESTPKGS:-./...}" -- $gotestArgs ${TESTFLAGS:--v})
+
 docker start -a -i $cid

tests/integration.go

@@ -66,6 +66,10 @@ func buildxCmd(sb integration.Sandbox, opts ...cmdOpt) *exec.Cmd {
 	if isExperimental() {
 		cmd.Env = append(cmd.Env, "BUILDX_EXPERIMENTAL=1")
 	}
+	if v := os.Getenv("GO_TEST_COVERPROFILE"); v != "" {
+		coverDir := filepath.Join(filepath.Dir(v), "helpers")
+		cmd.Env = append(cmd.Env, "GOCOVERDIR="+coverDir)
+	}
 
 	return cmd
 }