diff --git a/builder/node.go b/builder/node.go index 271b488b..e5a8e66d 100644 --- a/builder/node.go +++ b/builder/node.go @@ -8,7 +8,6 @@ import ( "github.com/containerd/platforms" "github.com/docker/buildx/driver" - ctxkube "github.com/docker/buildx/driver/kubernetes/context" "github.com/docker/buildx/store" "github.com/docker/buildx/store/storeutil" "github.com/docker/buildx/util/dockerutil" @@ -18,7 +17,6 @@ import ( "github.com/moby/buildkit/util/grpcerrors" ocispecs "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" - "github.com/sirupsen/logrus" "golang.org/x/sync/errgroup" "google.golang.org/grpc/codes" ) @@ -119,37 +117,19 @@ func (b *Builder) LoadNodes(ctx context.Context, opts ...LoadNodesOption) (_ []N return nil } - contextStore := b.opts.dockerCli.ContextStore() - - var kcc driver.KubeClientConfig - kcc, err = ctxkube.ConfigFromEndpoint(n.Endpoint, contextStore) - if err != nil { - // err is returned if n.Endpoint is non-context name like "unix:///var/run/docker.sock". - // try again with name="default". - // FIXME(@AkihiroSuda): n should retain real context name. - kcc, err = ctxkube.ConfigFromEndpoint("default", contextStore) - if err != nil { - logrus.Error(err) - } - } - - tryToUseKubeConfigInCluster := false - if kcc == nil { - tryToUseKubeConfigInCluster = true - } else { - if _, err := kcc.ClientConfig(); err != nil { - tryToUseKubeConfigInCluster = true - } - } - if tryToUseKubeConfigInCluster { - kccInCluster := driver.KubeClientConfigInCluster{} - if _, err := kccInCluster.ClientConfig(); err == nil { - logrus.Debug("using kube config in cluster") - kcc = kccInCluster - } - } - - d, err := driver.GetDriver(ctx, driver.BuilderName(n.Name), factory, n.Endpoint, dockerapi, imageopt.Auth, kcc, n.BuildkitdFlags, n.Files, n.DriverOpts, n.Platforms, b.opts.contextPathHash, lno.dialMeta) + d, err := driver.GetDriver(ctx, factory, driver.InitConfig{ + Name: driver.BuilderName(n.Name), + EndpointAddr: n.Endpoint, + DockerAPI: dockerapi, + ContextStore: b.opts.dockerCli.ContextStore(), + BuildkitdFlags: n.BuildkitdFlags, + Files: n.Files, + DriverOpts: n.DriverOpts, + Auth: imageopt.Auth, + Platforms: n.Platforms, + ContextPathHash: b.opts.contextPathHash, + DialMeta: lno.dialMeta, + }) if err != nil { node.Err = err return nil diff --git a/driver/kubernetes/driver.go b/driver/kubernetes/driver.go index de03c04e..289f17cd 100644 --- a/driver/kubernetes/driver.go +++ b/driver/kubernetes/driver.go @@ -38,7 +38,8 @@ const ( type Driver struct { driver.InitConfig - factory driver.Factory + factory driver.Factory + clientConfig ClientConfig // if you add fields, remember to update docs: // https://github.com/docker/docs/blob/main/content/build/drivers/kubernetes.md @@ -198,7 +199,7 @@ func (d *Driver) Rm(ctx context.Context, force, rmVolume, rmDaemon bool) error { func (d *Driver) Dial(ctx context.Context) (net.Conn, error) { restClient := d.clientset.CoreV1().RESTClient() - restClientConfig, err := d.KubeClientConfig.ClientConfig() + restClientConfig, err := d.clientConfig.ClientConfig() if err != nil { return nil, err } diff --git a/driver/kubernetes/factory.go b/driver/kubernetes/factory.go index c75e4544..b25abb89 100644 --- a/driver/kubernetes/factory.go +++ b/driver/kubernetes/factory.go @@ -2,19 +2,22 @@ package kubernetes import ( "context" + "os" "strconv" "strings" "time" - corev1 "k8s.io/api/core/v1" - "github.com/docker/buildx/driver" "github.com/docker/buildx/driver/bkimage" + ctxkube "github.com/docker/buildx/driver/kubernetes/context" "github.com/docker/buildx/driver/kubernetes/manifest" "github.com/docker/buildx/driver/kubernetes/podchooser" dockerclient "github.com/docker/docker/client" "github.com/pkg/errors" + "github.com/sirupsen/logrus" + corev1 "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" ) const ( @@ -23,11 +26,31 @@ const ( defaultTimeout = 120 * time.Second ) +type ClientConfig interface { + ClientConfig() (*rest.Config, error) + Namespace() (string, bool, error) +} + +type ClientConfigInCluster struct{} + +func (k ClientConfigInCluster) ClientConfig() (*rest.Config, error) { + return rest.InClusterConfig() +} + +func (k ClientConfigInCluster) Namespace() (string, bool, error) { + namespace, err := os.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace") + if err != nil { + return "", false, err + } + return strings.TrimSpace(string(namespace)), true, nil +} + func init() { driver.Register(&factory{}) } type factory struct { + cc ClientConfig // used for testing } func (*factory) Name() string { @@ -46,18 +69,50 @@ func (*factory) Priority(ctx context.Context, endpoint string, api dockerclient. } func (f *factory) New(ctx context.Context, cfg driver.InitConfig) (driver.Driver, error) { - if cfg.KubeClientConfig == nil { - return nil, errors.Errorf("%s driver requires kubernetes API access", DriverName) + var err error + var cc ClientConfig + if f.cc != nil { + cc = f.cc + } else { + cc, err = ctxkube.ConfigFromEndpoint(cfg.EndpointAddr, cfg.ContextStore) + if err != nil { + // err is returned if cfg.EndpointAddr is non-context name like "unix:///var/run/docker.sock". + // try again with name="default". + // FIXME(@AkihiroSuda): cfg should retain real context name. + cc, err = ctxkube.ConfigFromEndpoint("default", cfg.ContextStore) + if err != nil { + logrus.Error(err) + } + } + tryToUseConfigInCluster := false + if cc == nil { + tryToUseConfigInCluster = true + } else { + if _, err := cc.ClientConfig(); err != nil { + tryToUseConfigInCluster = true + } + } + if tryToUseConfigInCluster { + ccInCluster := ClientConfigInCluster{} + if _, err := ccInCluster.ClientConfig(); err == nil { + logrus.Debug("using kube config in cluster") + cc = ccInCluster + } + } + if cc == nil { + return nil, errors.Errorf("%s driver requires kubernetes API access", DriverName) + } } + deploymentName, err := buildxNameToDeploymentName(cfg.Name) if err != nil { return nil, err } - namespace, _, err := cfg.KubeClientConfig.Namespace() + namespace, _, err := cc.Namespace() if err != nil { return nil, errors.Wrap(err, "cannot determine Kubernetes namespace, specify manually") } - restClientConfig, err := cfg.KubeClientConfig.ClientConfig() + restClientConfig, err := cc.ClientConfig() if err != nil { return nil, err } @@ -67,9 +122,10 @@ func (f *factory) New(ctx context.Context, cfg driver.InitConfig) (driver.Driver } d := &Driver{ - factory: f, - InitConfig: cfg, - clientset: clientset, + factory: f, + clientConfig: cc, + InitConfig: cfg, + clientset: clientset, } deploymentOpt, loadbalance, namespace, defaultLoad, timeout, err := f.processDriverOpts(deploymentName, namespace, cfg) diff --git a/driver/kubernetes/factory_test.go b/driver/kubernetes/factory_test.go index 3224b250..bae29327 100644 --- a/driver/kubernetes/factory_test.go +++ b/driver/kubernetes/factory_test.go @@ -11,29 +11,28 @@ import ( "k8s.io/client-go/rest" ) -type mockKubeClientConfig struct { +type mockClientConfig struct { clientConfig *rest.Config namespace string } -func (r *mockKubeClientConfig) ClientConfig() (*rest.Config, error) { +func (r *mockClientConfig) ClientConfig() (*rest.Config, error) { return r.clientConfig, nil } -func (r *mockKubeClientConfig) Namespace() (string, bool, error) { +func (r *mockClientConfig) Namespace() (string, bool, error) { return r.namespace, true, nil } func TestFactory_processDriverOpts(t *testing.T) { - kcc := mockKubeClientConfig{ - clientConfig: &rest.Config{}, - } - cfg := driver.InitConfig{ - Name: driver.BuilderName("test"), - KubeClientConfig: &kcc, + Name: driver.BuilderName("test"), + } + f := factory{ + cc: &mockClientConfig{ + clientConfig: &rest.Config{}, + }, } - f := factory{} t.Run( "ValidOptions", func(t *testing.T) { diff --git a/driver/manager.go b/driver/manager.go index 509db1ca..400a0cdd 100644 --- a/driver/manager.go +++ b/driver/manager.go @@ -2,17 +2,15 @@ package driver import ( "context" - "os" "sort" - "strings" "sync" + "github.com/docker/cli/cli/context/store" dockerclient "github.com/docker/docker/client" "github.com/moby/buildkit/client" "github.com/moby/buildkit/util/tracing/delegated" specs "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" - "k8s.io/client-go/rest" ) type Factory interface { @@ -28,38 +26,18 @@ type BuildkitConfig struct { // Rootless bool } -type KubeClientConfig interface { - ClientConfig() (*rest.Config, error) - Namespace() (string, bool, error) -} - -type KubeClientConfigInCluster struct{} - -func (k KubeClientConfigInCluster) ClientConfig() (*rest.Config, error) { - return rest.InClusterConfig() -} - -func (k KubeClientConfigInCluster) Namespace() (string, bool, error) { - namespace, err := os.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace") - if err != nil { - return "", false, err - } - return strings.TrimSpace(string(namespace)), true, nil -} - type InitConfig struct { - // This object needs updates to be generic for different drivers - Name string - EndpointAddr string - DockerAPI dockerclient.APIClient - KubeClientConfig KubeClientConfig - BuildkitdFlags []string - Files map[string][]byte - DriverOpts map[string]string - Auth Auth - Platforms []specs.Platform - ContextPathHash string // can be used for determining pods in the driver instance - DialMeta map[string][]string + Name string + EndpointAddr string + DockerAPI dockerclient.APIClient + ContextStore store.Reader + BuildkitdFlags []string + Files map[string][]byte + DriverOpts map[string]string + Auth Auth + Platforms []specs.Platform + ContextPathHash string + DialMeta map[string][]string } var drivers map[string]Factory @@ -104,28 +82,15 @@ func GetFactory(name string, instanceRequired bool) (Factory, error) { return nil, errors.Errorf("failed to find driver %q", name) } -func GetDriver(ctx context.Context, name string, f Factory, endpointAddr string, api dockerclient.APIClient, auth Auth, kcc KubeClientConfig, buildkitdFlags []string, files map[string][]byte, do map[string]string, platforms []specs.Platform, contextPathHash string, dialMeta map[string][]string) (*DriverHandle, error) { - ic := InitConfig{ - EndpointAddr: endpointAddr, - DockerAPI: api, - KubeClientConfig: kcc, - Name: name, - BuildkitdFlags: buildkitdFlags, - DriverOpts: do, - Auth: auth, - Platforms: platforms, - ContextPathHash: contextPathHash, - DialMeta: dialMeta, - Files: files, - } +func GetDriver(ctx context.Context, f Factory, cfg InitConfig) (*DriverHandle, error) { if f == nil { var err error - f, err = GetDefaultFactory(ctx, endpointAddr, api, false, dialMeta) + f, err = GetDefaultFactory(ctx, cfg.EndpointAddr, cfg.DockerAPI, false, cfg.DialMeta) if err != nil { return nil, err } } - d, err := f.New(ctx, ic) + d, err := f.New(ctx, cfg) if err != nil { return nil, err } diff --git a/driver/remote/factory.go b/driver/remote/factory.go index 1c30baad..c07c08f2 100644 --- a/driver/remote/factory.go +++ b/driver/remote/factory.go @@ -7,15 +7,15 @@ import ( "strconv" "strings" - // import connhelpers for special url schemes - _ "github.com/moby/buildkit/client/connhelper/dockercontainer" - _ "github.com/moby/buildkit/client/connhelper/kubepod" - _ "github.com/moby/buildkit/client/connhelper/ssh" - "github.com/docker/buildx/driver" util "github.com/docker/buildx/driver/remote/util" dockerclient "github.com/docker/docker/client" "github.com/pkg/errors" + + // import connhelpers for special url schemes + _ "github.com/moby/buildkit/client/connhelper/dockercontainer" + _ "github.com/moby/buildkit/client/connhelper/kubepod" + _ "github.com/moby/buildkit/client/connhelper/ssh" ) const prioritySupported = 20