builder: move kube config handling to k8s driver package

Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com>

builder/node.go

@@ -8,7 +8,6 @@ import (
 
 	"github.com/containerd/platforms"
 	"github.com/docker/buildx/driver"
-	ctxkube "github.com/docker/buildx/driver/kubernetes/context"
 	"github.com/docker/buildx/store"
 	"github.com/docker/buildx/store/storeutil"
 	"github.com/docker/buildx/util/dockerutil"
@@ -18,7 +17,6 @@ import (
 	"github.com/moby/buildkit/util/grpcerrors"
 	ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
-	"github.com/sirupsen/logrus"
 	"golang.org/x/sync/errgroup"
 	"google.golang.org/grpc/codes"
 )
@@ -119,47 +117,18 @@ func (b *Builder) LoadNodes(ctx context.Context, opts ...LoadNodesOption) (_ []N
 					return nil
 				}
 
-				contextStore := b.opts.dockerCli.ContextStore()
-
-				var kcc driver.KubeClientConfig
-				kcc, err = ctxkube.ConfigFromEndpoint(n.Endpoint, contextStore)
-				if err != nil {
-					// err is returned if n.Endpoint is non-context name like "unix:///var/run/docker.sock".
-					// try again with name="default".
-					// FIXME(@AkihiroSuda): n should retain real context name.
-					kcc, err = ctxkube.ConfigFromEndpoint("default", contextStore)
-					if err != nil {
-						logrus.Error(err)
-					}
-				}
-
-				tryToUseKubeConfigInCluster := false
-				if kcc == nil {
-					tryToUseKubeConfigInCluster = true
-				} else {
-					if _, err := kcc.ClientConfig(); err != nil {
-						tryToUseKubeConfigInCluster = true
-					}
-				}
-				if tryToUseKubeConfigInCluster {
-					kccInCluster := driver.KubeClientConfigInCluster{}
-					if _, err := kccInCluster.ClientConfig(); err == nil {
-						logrus.Debug("using kube config in cluster")
-						kcc = kccInCluster
-					}
-				}
 				d, err := driver.GetDriver(ctx, factory, driver.InitConfig{
-					Name:             driver.BuilderName(n.Name),
+					Name:            driver.BuilderName(n.Name),
-					EndpointAddr:     n.Endpoint,
+					EndpointAddr:    n.Endpoint,
-					DockerAPI:        dockerapi,
+					DockerAPI:       dockerapi,
-					KubeClientConfig: kcc,
+					ContextStore:    b.opts.dockerCli.ContextStore(),
-					BuildkitdFlags:   n.BuildkitdFlags,
+					BuildkitdFlags:  n.BuildkitdFlags,
-					Files:            n.Files,
+					Files:           n.Files,
-					DriverOpts:       n.DriverOpts,
+					DriverOpts:      n.DriverOpts,
-					Auth:             imageopt.Auth,
+					Auth:            imageopt.Auth,
-					Platforms:        n.Platforms,
+					Platforms:       n.Platforms,
-					ContextPathHash:  b.opts.contextPathHash,
+					ContextPathHash: b.opts.contextPathHash,
-					DialMeta:         lno.dialMeta,
+					DialMeta:        lno.dialMeta,
 				})
 				if err != nil {
 					node.Err = err

driver/kubernetes/driver.go

@@ -38,7 +38,8 @@ const (
 
 type Driver struct {
 	driver.InitConfig
-	factory driver.Factory
+	factory      driver.Factory
+	clientConfig ClientConfig
 
 	// if you add fields, remember to update docs:
 	// https://github.com/docker/docs/blob/main/content/build/drivers/kubernetes.md
@@ -198,7 +199,7 @@ func (d *Driver) Rm(ctx context.Context, force, rmVolume, rmDaemon bool) error {
 
 func (d *Driver) Dial(ctx context.Context) (net.Conn, error) {
 	restClient := d.clientset.CoreV1().RESTClient()
-	restClientConfig, err := d.KubeClientConfig.ClientConfig()
+	restClientConfig, err := d.clientConfig.ClientConfig()
 	if err != nil {
 		return nil, err
 	}

driver/kubernetes/factory.go

@@ -2,19 +2,22 @@ package kubernetes
 
 import (
 	"context"
+	"os"
 	"strconv"
 	"strings"
 	"time"
 
-	corev1 "k8s.io/api/core/v1"
-
 	"github.com/docker/buildx/driver"
 	"github.com/docker/buildx/driver/bkimage"
+	ctxkube "github.com/docker/buildx/driver/kubernetes/context"
 	"github.com/docker/buildx/driver/kubernetes/manifest"
 	"github.com/docker/buildx/driver/kubernetes/podchooser"
 	dockerclient "github.com/docker/docker/client"
 	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
 )
 
 const (
@@ -23,11 +26,31 @@ const (
 	defaultTimeout      = 120 * time.Second
 )
 
+type ClientConfig interface {
+	ClientConfig() (*rest.Config, error)
+	Namespace() (string, bool, error)
+}
+
+type ClientConfigInCluster struct{}
+
+func (k ClientConfigInCluster) ClientConfig() (*rest.Config, error) {
+	return rest.InClusterConfig()
+}
+
+func (k ClientConfigInCluster) Namespace() (string, bool, error) {
+	namespace, err := os.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace")
+	if err != nil {
+		return "", false, err
+	}
+	return strings.TrimSpace(string(namespace)), true, nil
+}
+
 func init() {
 	driver.Register(&factory{})
 }
 
 type factory struct {
+	cc ClientConfig // used for testing
 }
 
 func (*factory) Name() string {
@@ -46,18 +69,50 @@ func (*factory) Priority(ctx context.Context, endpoint string, api dockerclient.
 }
 
 func (f *factory) New(ctx context.Context, cfg driver.InitConfig) (driver.Driver, error) {
-	if cfg.KubeClientConfig == nil {
+	var err error
-		return nil, errors.Errorf("%s driver requires kubernetes API access", DriverName)
+	var cc ClientConfig
+	if f.cc != nil {
+		cc = f.cc
+	} else {
+		cc, err = ctxkube.ConfigFromEndpoint(cfg.EndpointAddr, cfg.ContextStore)
+		if err != nil {
+			// err is returned if cfg.EndpointAddr is non-context name like "unix:///var/run/docker.sock".
+			// try again with name="default".
+			// FIXME(@AkihiroSuda): cfg should retain real context name.
+			cc, err = ctxkube.ConfigFromEndpoint("default", cfg.ContextStore)
+			if err != nil {
+				logrus.Error(err)
+			}
+		}
+		tryToUseConfigInCluster := false
+		if cc == nil {
+			tryToUseConfigInCluster = true
+		} else {
+			if _, err := cc.ClientConfig(); err != nil {
+				tryToUseConfigInCluster = true
+			}
+		}
+		if tryToUseConfigInCluster {
+			ccInCluster := ClientConfigInCluster{}
+			if _, err := ccInCluster.ClientConfig(); err == nil {
+				logrus.Debug("using kube config in cluster")
+				cc = ccInCluster
+			}
+		}
+		if cc == nil {
+			return nil, errors.Errorf("%s driver requires kubernetes API access", DriverName)
+		}
 	}
+
 	deploymentName, err := buildxNameToDeploymentName(cfg.Name)
 	if err != nil {
 		return nil, err
 	}
-	namespace, _, err := cfg.KubeClientConfig.Namespace()
+	namespace, _, err := cc.Namespace()
 	if err != nil {
 		return nil, errors.Wrap(err, "cannot determine Kubernetes namespace, specify manually")
 	}
-	restClientConfig, err := cfg.KubeClientConfig.ClientConfig()
+	restClientConfig, err := cc.ClientConfig()
 	if err != nil {
 		return nil, err
 	}
@@ -67,9 +122,10 @@ func (f *factory) New(ctx context.Context, cfg driver.InitConfig) (driver.Driver
 	}
 
 	d := &Driver{
-		factory:    f,
+		factory:      f,
-		InitConfig: cfg,
+		clientConfig: cc,
-		clientset:  clientset,
+		InitConfig:   cfg,
+		clientset:    clientset,
 	}
 
 	deploymentOpt, loadbalance, namespace, defaultLoad, timeout, err := f.processDriverOpts(deploymentName, namespace, cfg)

driver/kubernetes/factory_test.go

@@ -11,29 +11,28 @@ import (
 	"k8s.io/client-go/rest"
 )
 
-type mockKubeClientConfig struct {
+type mockClientConfig struct {
 	clientConfig *rest.Config
 	namespace    string
 }
 
-func (r *mockKubeClientConfig) ClientConfig() (*rest.Config, error) {
+func (r *mockClientConfig) ClientConfig() (*rest.Config, error) {
 	return r.clientConfig, nil
 }
 
-func (r *mockKubeClientConfig) Namespace() (string, bool, error) {
+func (r *mockClientConfig) Namespace() (string, bool, error) {
 	return r.namespace, true, nil
 }
 
 func TestFactory_processDriverOpts(t *testing.T) {
-	kcc := mockKubeClientConfig{
-		clientConfig: &rest.Config{},
-	}
-
 	cfg := driver.InitConfig{
-		Name:             driver.BuilderName("test"),
+		Name: driver.BuilderName("test"),
-		KubeClientConfig: &kcc,
+	}
+	f := factory{
+		cc: &mockClientConfig{
+			clientConfig: &rest.Config{},
+		},
 	}
-	f := factory{}
 
 	t.Run(
 		"ValidOptions", func(t *testing.T) {

driver/manager.go

@@ -2,17 +2,15 @@ package driver
 
 import (
 	"context"
-	"os"
 	"sort"
-	"strings"
 	"sync"
 
+	"github.com/docker/cli/cli/context/store"
 	dockerclient "github.com/docker/docker/client"
 	"github.com/moby/buildkit/client"
 	"github.com/moby/buildkit/util/tracing/delegated"
 	specs "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
-	"k8s.io/client-go/rest"
 )
 
 type Factory interface {
@@ -28,38 +26,18 @@ type BuildkitConfig struct {
 	// Rootless bool
 }
 
-type KubeClientConfig interface {
-	ClientConfig() (*rest.Config, error)
-	Namespace() (string, bool, error)
-}
-
-type KubeClientConfigInCluster struct{}
-
-func (k KubeClientConfigInCluster) ClientConfig() (*rest.Config, error) {
-	return rest.InClusterConfig()
-}
-
-func (k KubeClientConfigInCluster) Namespace() (string, bool, error) {
-	namespace, err := os.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace")
-	if err != nil {
-		return "", false, err
-	}
-	return strings.TrimSpace(string(namespace)), true, nil
-}
-
 type InitConfig struct {
-	// TODO: This object needs updates to be generic for different drivers
+	Name            string
-	Name             string
+	EndpointAddr    string
-	EndpointAddr     string
+	DockerAPI       dockerclient.APIClient
-	DockerAPI        dockerclient.APIClient
+	ContextStore    store.Reader
-	KubeClientConfig KubeClientConfig
+	BuildkitdFlags  []string
-	BuildkitdFlags   []string
+	Files           map[string][]byte
-	Files            map[string][]byte
+	DriverOpts      map[string]string
-	DriverOpts       map[string]string
+	Auth            Auth
-	Auth             Auth
+	Platforms       []specs.Platform
-	Platforms        []specs.Platform
+	ContextPathHash string
-	ContextPathHash  string // can be used for determining pods in the driver instance
+	DialMeta        map[string][]string
-	DialMeta         map[string][]string
 }
 
 var drivers map[string]Factory

driver/remote/factory.go

@@ -7,15 +7,15 @@ import (
 	"strconv"
 	"strings"
 
-	// import connhelpers for special url schemes
-	_ "github.com/moby/buildkit/client/connhelper/dockercontainer"
-	_ "github.com/moby/buildkit/client/connhelper/kubepod"
-	_ "github.com/moby/buildkit/client/connhelper/ssh"
-
 	"github.com/docker/buildx/driver"
 	util "github.com/docker/buildx/driver/remote/util"
 	dockerclient "github.com/docker/docker/client"
 	"github.com/pkg/errors"
+
+	// import connhelpers for special url schemes
+	_ "github.com/moby/buildkit/client/connhelper/dockercontainer"
+	_ "github.com/moby/buildkit/client/connhelper/kubepod"
+	_ "github.com/moby/buildkit/client/connhelper/ssh"
 )
 
 const prioritySupported = 20