tests: matrix with buildkit versions

Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com>

.github/workflows/build.yml

@@ -30,35 +30,8 @@ env:
   GOTESTSUM_VERSION: "v1.9.0"  # same as one in Dockerfile
 
 jobs:
-  prepare-test-integration:
-    runs-on: ubuntu-22.04
-    steps:
-      -
-        name: Checkout
-        uses: actions/checkout@v4
-      -
-        name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-      -
-        name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-        with:
-          version: ${{ env.BUILDX_VERSION }}
-          driver-opts: image=${{ env.BUILDKIT_IMAGE }}
-          buildkitd-flags: --debug
-      -
-        name: Build
-        uses: docker/bake-action@v4
-        with:
-          targets: integration-test-base
-          set: |
-            *.cache-from=type=gha,scope=${{ env.TEST_CACHE_SCOPE }}
-            *.cache-to=type=gha,scope=${{ env.TEST_CACHE_SCOPE }}
-
   test-integration:
     runs-on: ubuntu-22.04
-    needs:
-      - prepare-test-integration
     env:
       TESTFLAGS_DOCKER: "-v --parallel=1 --timeout=30m"
       TEST_IMAGE_BUILD: "0"
@@ -66,6 +39,12 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
+        buildkit:
+          - master
+          - latest
+          - buildx-stable-1
+          - v0.11.6
+          - v0.10.6
         worker:
           - docker
           - docker\+containerd  # same as docker, but with containerd snapshotter
@@ -80,7 +59,10 @@ jobs:
       -
         name: Prepare
         run: |
-          echo "TESTREPORTS_NAME=${{ github.job }}-$(echo "${{ matrix.pkg }}-${{ matrix.worker }}-${{ matrix.experimental }}" | tr -dc '[:alnum:]-\n\r' | tr '[:upper:]' '[:lower:]')" >> $GITHUB_ENV
+          echo "TESTREPORTS_NAME=${{ github.job }}-$(echo "${{ matrix.pkg }}-${{ matrix.buildkit }}-${{ matrix.worker }}-${{ matrix.experimental }}" | tr -dc '[:alnum:]-\n\r' | tr '[:upper:]' '[:lower:]')" >> $GITHUB_ENV
+          if [ -n "${{ matrix.buildkit }}" ]; then
+            echo "TEST_BUILDKIT_TAG=${{ matrix.buildkit }}" >> $GITHUB_ENV
+          fi
       -
         name: Checkout
         uses: actions/checkout@v4
@@ -102,7 +84,6 @@ jobs:
         with:
           targets: integration-test
           set: |
-            *.cache-from=type=gha,scope=${{ env.TEST_CACHE_SCOPE }}
             *.output=type=docker,name=${{ env.TEST_IMAGE_ID }}
       -
         name: Test

docker-bake.hcl

@@ -180,6 +180,9 @@ variable "HTTPS_PROXY" {
 variable "NO_PROXY" {
   default = ""
 }
+variable "TEST_BUILDKIT_TAG" {
+  default = null
+}
 
 target "integration-test-base" {
   inherits = ["_common"]
@@ -187,6 +190,7 @@ target "integration-test-base" {
     HTTP_PROXY = HTTP_PROXY
     HTTPS_PROXY = HTTPS_PROXY
     NO_PROXY = NO_PROXY
+    BUILDKIT_VERSION = TEST_BUILDKIT_TAG
   }
   target = "integration-test-base"
   output = ["type=cacheonly"]

hack/test

@@ -10,6 +10,7 @@ set -eu -o pipefail
 : "${TEST_REPORT_SUFFIX=}"
 : "${TEST_KEEP_CACHE=}"
 : "${TEST_DOCKERD=}"
+: "${TEST_BUILDKIT_TAG=}"
 : "${TEST_BUILDKIT_IMAGE=}"
 : "${TEST_BUILDX_EXPERIMENTAL=}"
 
@@ -31,5 +32,5 @@ if [ "$TEST_KEEP_CACHE" != "1" ]; then
   trap 'docker rm -v $cacheVolume' EXIT
 fi
 
-cid=$(docker create --rm -v /tmp $testReportsVol --volumes-from=$cacheVolume -e GITHUB_REF -e TEST_DOCKERD -e TEST_BUILDKIT_IMAGE -e TEST_BUILDX_EXPERIMENTAL -e SKIP_INTEGRATION_TESTS -e GOTESTSUM_FORMAT ${BUILDKIT_INTEGRATION_SNAPSHOTTER:+"-eBUILDKIT_INTEGRATION_SNAPSHOTTER"} -e BUILDKIT_REGISTRY_MIRROR_DIR=/root/.cache/registry --privileged $TEST_IMAGE_ID gotestsum $gotestsumArgs --packages="${TESTPKGS:-./...}" -- $gotestArgs ${TESTFLAGS:--v})
+cid=$(docker create --rm -v /tmp $testReportsVol --volumes-from=$cacheVolume -e GITHUB_REF -e TEST_DOCKERD -e TEST_BUILDKIT_IMAGE -e TEST_BUILDKIT_TAG -e TEST_BUILDX_EXPERIMENTAL -e SKIP_INTEGRATION_TESTS -e GOTESTSUM_FORMAT ${BUILDKIT_INTEGRATION_SNAPSHOTTER:+"-eBUILDKIT_INTEGRATION_SNAPSHOTTER"} -e BUILDKIT_REGISTRY_MIRROR_DIR=/root/.cache/registry --privileged $TEST_IMAGE_ID gotestsum $gotestsumArgs --packages="${TESTPKGS:-./...}" -- $gotestArgs ${TESTFLAGS:--v})
 docker start -a -i $cid

tests/integration.go

@@ -12,6 +12,8 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+const defaultBuildKitTag = "buildx-stable-1"
+
 func tmpdir(t *testing.T, appliers ...fstest.Applier) string {
 	t.Helper()
 	tmpdir := t.TempDir()
@@ -107,3 +109,10 @@ func isExperimental() bool {
 	}
 	return false
 }
+
+func buildkitTag() string {
+	if v := os.Getenv("TEST_BUILDKIT_TAG"); v != "" {
+		return v
+	}
+	return defaultBuildKitTag
+}

tests/integration_test.go

@@ -35,7 +35,7 @@ func TestIntegration(t *testing.T) {
 
 func testIntegration(t *testing.T, funcs ...func(t *testing.T, sb integration.Sandbox)) {
 	mirroredImages := integration.OfficialImages("busybox:latest", "alpine:latest")
-	buildkitImage := "docker.io/moby/buildkit:buildx-stable-1"
+	buildkitImage := "docker.io/moby/buildkit:" + buildkitTag()
 	if bkworkers.IsTestDockerd() {
 		if img, ok := os.LookupEnv("TEST_BUILDKIT_IMAGE"); ok {
 			ref, err := reference.ParseNormalizedNamed(img)