diff --git a/build/build.go b/build/build.go index 9cf2bc54..89e7d194 100644 --- a/build/build.go +++ b/build/build.go @@ -23,6 +23,7 @@ import ( "github.com/containerd/containerd/images" "github.com/containerd/containerd/platforms" "github.com/docker/buildx/driver" + "github.com/docker/buildx/util/dockerutil" "github.com/docker/buildx/util/imagetools" "github.com/docker/buildx/util/progress" "github.com/docker/buildx/util/resolver" @@ -31,7 +32,6 @@ import ( "github.com/docker/distribution/reference" "github.com/docker/docker/api/types" "github.com/docker/docker/builder/remotecontext/urlutil" - dockerclient "github.com/docker/docker/client" "github.com/docker/docker/pkg/jsonmessage" "github.com/moby/buildkit/client" "github.com/moby/buildkit/client/llb" @@ -118,10 +118,6 @@ type DriverInfo struct { ProxyConfig map[string]string } -type DockerAPI interface { - DockerAPI(name string) (dockerclient.APIClient, error) -} - func filterAvailableDrivers(drivers []DriverInfo) ([]DriverInfo, error) { out := make([]DriverInfo, 0, len(drivers)) err := errors.Errorf("no drivers found") @@ -782,11 +778,11 @@ func Invoke(ctx context.Context, cfg ContainerConfig) error { return err } -func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, docker DockerAPI, configDir string, w progress.Writer) (resp map[string]*client.SolveResponse, err error) { +func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, docker *dockerutil.Client, configDir string, w progress.Writer) (resp map[string]*client.SolveResponse, err error) { return BuildWithResultHandler(ctx, drivers, opt, docker, configDir, w, nil, false) } -func BuildWithResultHandler(ctx context.Context, drivers []DriverInfo, opt map[string]Options, docker DockerAPI, configDir string, w progress.Writer, resultHandleFunc func(driverIndex int, rCtx *ResultContext), allowNoOutput bool) (resp map[string]*client.SolveResponse, err error) { +func BuildWithResultHandler(ctx context.Context, drivers []DriverInfo, opt map[string]Options, docker *dockerutil.Client, configDir string, w progress.Writer, resultHandleFunc func(driverIndex int, rCtx *ResultContext), allowNoOutput bool) (resp map[string]*client.SolveResponse, err error) { if len(drivers) == 0 { return nil, errors.Errorf("driver required for build") } @@ -864,7 +860,7 @@ func BuildWithResultHandler(ctx context.Context, drivers []DriverInfo, opt map[s } opt.Platforms = dp.platforms so, release, err := toSolveOpt(ctx, di, multiDriver, opt, dp.bopts, configDir, w, func(name string) (io.WriteCloser, func(), error) { - return newDockerLoader(ctx, docker, name, w) + return docker.LoadImage(ctx, name, w) }) if err != nil { return nil, err @@ -1631,40 +1627,6 @@ func notSupported(d driver.Driver, f driver.Feature) error { type dockerLoadCallback func(name string) (io.WriteCloser, func(), error) -func newDockerLoader(ctx context.Context, d DockerAPI, name string, status progress.Writer) (io.WriteCloser, func(), error) { - c, err := d.DockerAPI(name) - if err != nil { - return nil, nil, err - } - - pr, pw := io.Pipe() - done := make(chan struct{}) - - ctx, cancel := context.WithCancel(ctx) - var w *waitingWriter - w = &waitingWriter{ - PipeWriter: pw, - f: func() { - resp, err := c.ImageLoad(ctx, pr, false) - defer close(done) - if err != nil { - pr.CloseWithError(err) - w.mu.Lock() - w.err = err - w.mu.Unlock() - return - } - prog := progress.WithPrefix(status, "", false) - progress.FromReader(prog, "importing to docker", resp.Body) - }, - done: done, - cancel: cancel, - } - return w, func() { - pr.Close() - }, nil -} - func noDefaultLoad() bool { v, ok := os.LookupEnv("BUILDX_NO_DEFAULT_LOAD") if !ok { @@ -1677,34 +1639,6 @@ func noDefaultLoad() bool { return b } -type waitingWriter struct { - *io.PipeWriter - f func() - once sync.Once - mu sync.Mutex - err error - done chan struct{} - cancel func() -} - -func (w *waitingWriter) Write(dt []byte) (int, error) { - w.once.Do(func() { - go w.f() - }) - return w.PipeWriter.Write(dt) -} - -func (w *waitingWriter) Close() error { - err := w.PipeWriter.Close() - <-w.done - if err == nil { - w.mu.Lock() - defer w.mu.Unlock() - return w.err - } - return err -} - // handle https://github.com/moby/moby/pull/10858 func handleLowercaseDockerfile(dir, p string) string { if filepath.Base(p) != "Dockerfile" { diff --git a/commands/bake.go b/commands/bake.go index 0629df5b..317f2381 100644 --- a/commands/bake.go +++ b/commands/bake.go @@ -10,6 +10,7 @@ import ( "github.com/docker/buildx/bake" "github.com/docker/buildx/build" "github.com/docker/buildx/util/confutil" + "github.com/docker/buildx/util/dockerutil" "github.com/docker/buildx/util/progress" "github.com/docker/buildx/util/tracing" "github.com/docker/cli/cli/command" @@ -146,7 +147,7 @@ func runBake(dockerCli command.Cli, targets []string, in bakeOptions) (err error return nil } - resp, err := build.Build(ctx, dis, bo, dockerAPI(dockerCli), confutil.ConfigDir(dockerCli), printer) + resp, err := build.Build(ctx, dis, bo, dockerutil.NewClient(dockerCli), confutil.ConfigDir(dockerCli), printer) if err != nil { return wrapBuildError(err, true) } diff --git a/commands/build.go b/commands/build.go index ddcdbb09..b46b4bb3 100644 --- a/commands/build.go +++ b/commands/build.go @@ -19,6 +19,7 @@ import ( "github.com/docker/buildx/monitor" "github.com/docker/buildx/util/buildflags" "github.com/docker/buildx/util/confutil" + "github.com/docker/buildx/util/dockerutil" "github.com/docker/buildx/util/platformutil" "github.com/docker/buildx/util/progress" "github.com/docker/buildx/util/tracing" @@ -291,7 +292,7 @@ func buildTargets(ctx context.Context, dockerCli command.Cli, opts map[string]bu var mu sync.Mutex var idx int - resp, err := build.BuildWithResultHandler(ctx, dis, opts, dockerAPI(dockerCli), confutil.ConfigDir(dockerCli), printer, func(driverIndex int, gotRes *build.ResultContext) { + resp, err := build.BuildWithResultHandler(ctx, dis, opts, dockerutil.NewClient(dockerCli), confutil.ConfigDir(dockerCli), printer, func(driverIndex int, gotRes *build.ResultContext) { mu.Lock() defer mu.Unlock() if res == nil || driverIndex < idx { diff --git a/commands/create.go b/commands/create.go index cc0af47c..3eeae50e 100644 --- a/commands/create.go +++ b/commands/create.go @@ -15,6 +15,7 @@ import ( "github.com/docker/buildx/store/storeutil" "github.com/docker/buildx/util/cobrautil" "github.com/docker/buildx/util/confutil" + "github.com/docker/buildx/util/dockerutil" "github.com/docker/cli/cli" "github.com/docker/cli/cli/command" "github.com/google/shlex" @@ -204,7 +205,7 @@ func runCreate(dockerCli command.Cli, in createOptions, args []string) error { if dockerCli.CurrentContext() == "default" && dockerCli.DockerEndpoint().TLSData != nil { return errors.Errorf("could not create a builder instance with TLS data loaded from environment. Please use `docker context create ` to create a context for current environment and then create a builder instance with `docker buildx create `") } - ep, err = storeutil.GetCurrentEndpoint(dockerCli) + ep, err = dockerutil.GetCurrentEndpoint(dockerCli) if err != nil { return err } @@ -259,7 +260,7 @@ func runCreate(dockerCli command.Cli, in createOptions, args []string) error { } if in.use && ep != "" { - current, err := storeutil.GetCurrentEndpoint(dockerCli) + current, err := dockerutil.GetCurrentEndpoint(dockerCli) if err != nil { return err } diff --git a/commands/use.go b/commands/use.go index 71320135..7e52ce85 100644 --- a/commands/use.go +++ b/commands/use.go @@ -4,6 +4,7 @@ import ( "os" "github.com/docker/buildx/store/storeutil" + "github.com/docker/buildx/util/dockerutil" "github.com/docker/cli/cli" "github.com/docker/cli/cli/command" "github.com/pkg/errors" @@ -29,7 +30,7 @@ func runUse(dockerCli command.Cli, in useOptions) error { return errors.Errorf("run `docker context use default` to switch to default context") } if in.builder == "default" || in.builder == dockerCli.CurrentContext() { - ep, err := storeutil.GetCurrentEndpoint(dockerCli) + ep, err := dockerutil.GetCurrentEndpoint(dockerCli) if err != nil { return err } @@ -52,7 +53,7 @@ func runUse(dockerCli command.Cli, in useOptions) error { return errors.Wrapf(err, "failed to find instance %q", in.builder) } - ep, err := storeutil.GetCurrentEndpoint(dockerCli) + ep, err := dockerutil.GetCurrentEndpoint(dockerCli) if err != nil { return err } diff --git a/commands/util.go b/commands/util.go index 571f2731..e15c069d 100644 --- a/commands/util.go +++ b/commands/util.go @@ -10,12 +10,11 @@ import ( remoteutil "github.com/docker/buildx/driver/remote/util" "github.com/docker/buildx/store" "github.com/docker/buildx/store/storeutil" + "github.com/docker/buildx/util/dockerutil" "github.com/docker/buildx/util/platformutil" "github.com/docker/buildx/util/progress" "github.com/docker/cli/cli/command" - "github.com/docker/cli/cli/context/docker" dopts "github.com/docker/cli/opts" - dockerclient "github.com/docker/docker/client" "github.com/moby/buildkit/util/grpcerrors" specs "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" @@ -26,10 +25,10 @@ import ( // validateEndpoint validates that endpoint is either a context or a docker host func validateEndpoint(dockerCli command.Cli, ep string) (string, error) { - de, err := storeutil.GetDockerEndpoint(dockerCli, ep) - if err == nil && de != "" { + dem, err := dockerutil.GetDockerEndpoint(dockerCli, ep) + if err == nil && dem != nil { if ep == "default" { - return de, nil + return dem.Host, nil } return ep, nil } @@ -67,7 +66,7 @@ func driversForNodeGroup(ctx context.Context, dockerCli command.Cli, ng *store.N // docker-container driver for older daemon that doesn't support // buildkit (< 18.06). ep := ng.Nodes[0].Endpoint - dockerapi, err := clientForEndpoint(dockerCli, ep) + dockerapi, err := dockerutil.NewClientAPI(dockerCli, ep) if err != nil { return nil, err } @@ -99,7 +98,7 @@ func driversForNodeGroup(ctx context.Context, dockerCli command.Cli, ng *store.N dis[i] = di }() - dockerapi, err := clientForEndpoint(dockerCli, n.Endpoint) + dockerapi, err := dockerutil.NewClientAPI(dockerCli, n.Endpoint) if err != nil { di.Err = err return nil @@ -154,44 +153,6 @@ func driversForNodeGroup(ctx context.Context, dockerCli command.Cli, ng *store.N return dis, nil } -// clientForEndpoint returns a docker client for an endpoint -func clientForEndpoint(dockerCli command.Cli, name string) (dockerclient.APIClient, error) { - list, err := dockerCli.ContextStore().List() - if err != nil { - return nil, err - } - for _, l := range list { - if l.Name == name { - epm, err := docker.EndpointFromContext(l) - if err != nil { - return nil, err - } - ep, err := docker.WithTLSData(dockerCli.ContextStore(), name, epm) - if err != nil { - return nil, err - } - clientOpts, err := ep.ClientOpts() - if err != nil { - return nil, err - } - return dockerclient.NewClientWithOpts(clientOpts...) - } - } - - ep := docker.Endpoint{ - EndpointMeta: docker.EndpointMeta{ - Host: name, - }, - } - - clientOpts, err := ep.ClientOpts() - if err != nil { - return nil, err - } - - return dockerclient.NewClientWithOpts(clientOpts...) -} - func getInstanceOrDefault(ctx context.Context, dockerCli command.Cli, instance, contextPathHash string) ([]build.DriverInfo, error) { var defaultOnly bool @@ -380,21 +341,6 @@ func hasNodeGroup(list []*nginfo, ngi *nginfo) bool { return false } -func dockerAPI(dockerCli command.Cli) *api { - return &api{dockerCli: dockerCli} -} - -type api struct { - dockerCli command.Cli -} - -func (a *api) DockerAPI(name string) (dockerclient.APIClient, error) { - if name == "" { - name = a.dockerCli.CurrentContext() - } - return clientForEndpoint(a.dockerCli, name) -} - type dinfo struct { di *build.DriverInfo info *driver.Info diff --git a/store/storeutil/storeutil.go b/store/storeutil/storeutil.go index 494a878f..99e53a3d 100644 --- a/store/storeutil/storeutil.go +++ b/store/storeutil/storeutil.go @@ -7,10 +7,10 @@ import ( "github.com/docker/buildx/store" "github.com/docker/buildx/util/confutil" + "github.com/docker/buildx/util/dockerutil" "github.com/docker/buildx/util/imagetools" "github.com/docker/buildx/util/resolver" "github.com/docker/cli/cli/command" - "github.com/docker/cli/cli/context/docker" buildkitdconfig "github.com/moby/buildkit/cmd/buildkitd/config" "github.com/pkg/errors" ) @@ -24,19 +24,6 @@ func GetStore(dockerCli command.Cli) (*store.Txn, func(), error) { return s.Txn() } -// GetCurrentEndpoint returns the current default endpoint value -func GetCurrentEndpoint(dockerCli command.Cli) (string, error) { - name := dockerCli.CurrentContext() - if name != "default" { - return name, nil - } - de, err := GetDockerEndpoint(dockerCli, name) - if err != nil { - return "", errors.Errorf("docker endpoint for %q not found", name) - } - return de, nil -} - func GetProxyConfig(dockerCli command.Cli) map[string]string { cfg := dockerCli.ConfigFile() host := dockerCli.Client().DaemonHost() @@ -63,31 +50,9 @@ func GetProxyConfig(dockerCli command.Cli) map[string]string { return m } -// GetDockerEndpoint returns docker endpoint string for given context -func GetDockerEndpoint(dockerCli command.Cli, name string) (string, error) { - list, err := dockerCli.ContextStore().List() - if err != nil { - return "", err - } - for _, l := range list { - if l.Name == name { - ep, ok := l.Endpoints["docker"] - if !ok { - return "", errors.Errorf("context %q does not have a Docker endpoint", name) - } - typed, ok := ep.(docker.EndpointMeta) - if !ok { - return "", errors.Errorf("endpoint %q is not of type EndpointMeta, %T", ep, ep) - } - return typed.Host, nil - } - } - return "", nil -} - // GetCurrentInstance finds the current builder instance func GetCurrentInstance(txn *store.Txn, dockerCli command.Cli) (*store.NodeGroup, error) { - ep, err := GetCurrentEndpoint(dockerCli) + ep, err := dockerutil.GetCurrentEndpoint(dockerCli) if err != nil { return nil, err } diff --git a/util/dockerutil/api.go b/util/dockerutil/api.go new file mode 100644 index 00000000..5fd9b652 --- /dev/null +++ b/util/dockerutil/api.go @@ -0,0 +1,45 @@ +package dockerutil + +import ( + "github.com/docker/cli/cli/command" + "github.com/docker/cli/cli/context/docker" + "github.com/docker/docker/client" +) + +// ClientAPI represents an active docker API object. +type ClientAPI struct { + client.APIClient +} + +func NewClientAPI(cli command.Cli, ep string) (*ClientAPI, error) { + ca := &ClientAPI{} + + var dep docker.Endpoint + dem, err := GetDockerEndpoint(cli, ep) + if err != nil { + return nil, err + } else if dem != nil { + dep, err = docker.WithTLSData(cli.ContextStore(), ep, *dem) + if err != nil { + return nil, err + } + } else { + dep = docker.Endpoint{ + EndpointMeta: docker.EndpointMeta{ + Host: ep, + }, + } + } + + clientOpts, err := dep.ClientOpts() + if err != nil { + return nil, err + } + + ca.APIClient, err = client.NewClientWithOpts(clientOpts...) + if err != nil { + return nil, err + } + + return ca, nil +} diff --git a/util/dockerutil/client.go b/util/dockerutil/client.go new file mode 100644 index 00000000..04b09ede --- /dev/null +++ b/util/dockerutil/client.go @@ -0,0 +1,92 @@ +package dockerutil + +import ( + "context" + "io" + "sync" + + "github.com/docker/buildx/util/progress" + "github.com/docker/cli/cli/command" + "github.com/docker/docker/client" +) + +// Client represents an active docker object. +type Client struct { + cli command.Cli +} + +// NewClient initializes a new docker client. +func NewClient(cli command.Cli) *Client { + return &Client{cli: cli} +} + +// API returns a new docker API client. +func (c *Client) API(name string) (client.APIClient, error) { + if name == "" { + name = c.cli.CurrentContext() + } + return NewClientAPI(c.cli, name) +} + +// LoadImage imports an image to docker. +func (c *Client) LoadImage(ctx context.Context, name string, status progress.Writer) (io.WriteCloser, func(), error) { + dapi, err := c.API(name) + if err != nil { + return nil, nil, err + } + + pr, pw := io.Pipe() + done := make(chan struct{}) + + ctx, cancel := context.WithCancel(ctx) + var w *waitingWriter + w = &waitingWriter{ + PipeWriter: pw, + f: func() { + resp, err := dapi.ImageLoad(ctx, pr, false) + defer close(done) + if err != nil { + pr.CloseWithError(err) + w.mu.Lock() + w.err = err + w.mu.Unlock() + return + } + prog := progress.WithPrefix(status, "", false) + progress.FromReader(prog, "importing to docker", resp.Body) + }, + done: done, + cancel: cancel, + } + return w, func() { + pr.Close() + }, nil +} + +type waitingWriter struct { + *io.PipeWriter + f func() + once sync.Once + mu sync.Mutex + err error + done chan struct{} + cancel func() +} + +func (w *waitingWriter) Write(dt []byte) (int, error) { + w.once.Do(func() { + go w.f() + }) + return w.PipeWriter.Write(dt) +} + +func (w *waitingWriter) Close() error { + err := w.PipeWriter.Close() + <-w.done + if err == nil { + w.mu.Lock() + defer w.mu.Unlock() + return w.err + } + return err +} diff --git a/util/dockerutil/context.go b/util/dockerutil/context.go new file mode 100644 index 00000000..e4e0c697 --- /dev/null +++ b/util/dockerutil/context.go @@ -0,0 +1,40 @@ +package dockerutil + +import ( + "github.com/docker/cli/cli/command" + "github.com/docker/cli/cli/context/docker" + "github.com/pkg/errors" +) + +// GetDockerEndpoint returns docker endpoint meta for given context +func GetDockerEndpoint(dockerCli command.Cli, name string) (*docker.EndpointMeta, error) { + list, err := dockerCli.ContextStore().List() + if err != nil { + return nil, err + } + for _, l := range list { + if l.Name == name { + epm, err := docker.EndpointFromContext(l) + if err != nil { + return nil, err + } + return &epm, nil + } + } + return nil, nil +} + +// GetCurrentEndpoint returns the current default endpoint value +func GetCurrentEndpoint(dockerCli command.Cli) (string, error) { + name := dockerCli.CurrentContext() + if name != "default" { + return name, nil + } + dem, err := GetDockerEndpoint(dockerCli, name) + if err != nil { + return "", errors.Errorf("docker endpoint for %q not found", name) + } else if dem != nil { + return dem.Host, nil + } + return "", nil +}