docker
/
buildx
mirror of https://github.com/docker/buildx.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #37 from tonistiigi/multi-driver 

build: multi-node build support
This commit is contained in:
Tõnis Tiigi 2019-04-24 09:14:36 -07:00 committed by GitHub
parent 2f668d555c e7eac2b909
commit e7e57b626d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 1071 additions and 232 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

770
build/build.go
View File

@ -3,6 +3,7 @@ package build
import (
"bufio"
"context"
"fmt"
"io"
"io/ioutil"
"os"
@ -11,17 +12,21 @@ import (
"strings"
"sync"
"github.com/containerd/containerd/images"
"github.com/containerd/containerd/platforms"
clitypes "github.com/docker/cli/cli/config/types"
"github.com/docker/distribution/reference"
dockerclient "github.com/docker/docker/client"
"github.com/docker/docker/pkg/urlutil"
"github.com/moby/buildkit/client"
"github.com/moby/buildkit/session"
"github.com/moby/buildkit/session/upload/uploadprovider"
"github.com/opencontainers/go-digest"
specs "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"github.com/tonistiigi/buildx/driver"
"github.com/tonistiigi/buildx/util/imagetools"
"github.com/tonistiigi/buildx/util/progress"
"golang.org/x/sync/errgroup"
)
@ -66,245 +71,620 @@ type DriverInfo struct {
Err error
}
type Auth interface {
GetAuthConfig(registryHostname string) (clitypes.AuthConfig, error)
}
type DockerAPI interface {
DockerAPI(name string) (dockerclient.APIClient, error)
}
func getFirstDriver(drivers []DriverInfo) (driver.Driver, error) {
func filterAvailableDrivers(drivers []DriverInfo) ([]DriverInfo, error) {
out := make([]DriverInfo, 0, len(drivers))
err := errors.Errorf("no drivers found")
for _, di := range drivers {
if di.Driver != nil {
return di.Driver, nil
if di.Err == nil && di.Driver != nil {
out = append(out, di)
}
if di.Err != nil {
err = di.Err
}
}
if len(out) > 0 {
return out, nil
}
return nil, err
}
func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, docker DockerAPI, pw progress.Writer) (map[string]*client.SolveResponse, error) {
if len(drivers) == 0 {
return nil, errors.Errorf("driver required for build")
type driverPair struct {
driverIndex int
platforms []specs.Platform
so *client.SolveOpt
}
func driverIndexes(m map[string][]driverPair) []int {
out := make([]int, 0, len(m))
visited := map[int]struct{}{}
for _, dp := range m {
for _, d := range dp {
if _, ok := visited[d.driverIndex]; ok {
continue
}
visited[d.driverIndex] = struct{}{}
out = append(out, d.driverIndex)
}
}
return out
}
func allIndexes(l int) []int {
out := make([]int, 0, l)
for i := 0; i < l; i++ {
out = append(out, i)
}
return out
}
func ensureBooted(ctx context.Context, drivers []DriverInfo, idxs []int, pw progress.Writer) ([]*client.Client, error) {
clients := make([]*client.Client, len(drivers))
eg, ctx := errgroup.WithContext(ctx)
for _, i := range idxs {
func(i int) {
eg.Go(func() error {
c, err := driver.Boot(ctx, drivers[i].Driver, pw)
if err != nil {
return err
}
clients[i] = c
return nil
})
}(i)
}
if len(drivers) > 1 {
return nil, errors.Errorf("multiple drivers currently not supported")
}
d, err := getFirstDriver(drivers)
if err != nil {
if err := eg.Wait(); err != nil {
return nil, err
}
return clients, nil
}
func splitToDriverPairs(availablePlatforms map[string]int, opt map[string]Options) map[string][]driverPair {
m := map[string][]driverPair{}
for k, opt := range opt {
mm := map[int][]specs.Platform{}
for _, p := range opt.Platforms {
k := platforms.Format(p)
idx := availablePlatforms[k] // default 0
pp := mm[idx]
pp = append(pp, p)
mm[idx] = pp
}
dps := make([]driverPair, 0, 2)
for idx, pp := range mm {
dps = append(dps, driverPair{driverIndex: idx, platforms: pp})
}
m[k] = dps
}
return m
}
func resolveDrivers(ctx context.Context, drivers []DriverInfo, opt map[string]Options, pw progress.Writer) (map[string][]driverPair, []*client.Client, error) {
availablePlatforms := map[string]int{}
for i, d := range drivers {
for _, p := range d.Platform {
availablePlatforms[platforms.Format(p)] = i
}
}
undetectedPlatform := false
allPlatforms := map[string]int{}
for _, opt := range opt {
for _, p := range opt.Platforms {
k := platforms.Format(p)
allPlatforms[k] = -1
if _, ok := availablePlatforms[k]; !ok {
undetectedPlatform = true
}
}
}
// fast path
if len(drivers) == 1 || len(allPlatforms) == 0 {
m := map[string][]driverPair{}
for k, opt := range opt {
m[k] = []driverPair{{driverIndex: 0, platforms: opt.Platforms}}
}
clients, err := ensureBooted(ctx, drivers, driverIndexes(m), pw)
if err != nil {
return nil, nil, err
}
return m, clients, nil
}
// map based on existing platforms
if !undetectedPlatform {
m := splitToDriverPairs(availablePlatforms, opt)
clients, err := ensureBooted(ctx, drivers, driverIndexes(m), pw)
if err != nil {
return nil, nil, err
}
return m, clients, nil
}
// boot all drivers in k
clients, err := ensureBooted(ctx, drivers, allIndexes(len(drivers)), pw)
if err != nil {
return nil, nil, err
}
eg, ctx := errgroup.WithContext(ctx)
workers := make([][]*client.WorkerInfo, len(clients))
for i, c := range clients {
if c == nil {
continue
}
func(i int) {
eg.Go(func() error {
ww, err := clients[i].ListWorkers(ctx)
if err != nil {
return errors.Wrap(err, "listing workers")
}
workers[i] = ww
return nil
})
}(i)
}
if err := eg.Wait(); err != nil {
return nil, nil, err
}
for i, ww := range workers {
for _, w := range ww {
for _, p := range w.Platforms {
p = platforms.Normalize(p)
ps := platforms.Format(p)
if _, ok := availablePlatforms[ps]; !ok {
availablePlatforms[ps] = i
}
}
}
}
return splitToDriverPairs(availablePlatforms, opt), clients, nil
}
func toRepoOnly(in string) (string, error) {
m := map[string]struct{}{}
p := strings.Split(in, ",")
for _, pp := range p {
n, err := reference.ParseNormalizedNamed(pp)
if err != nil {
return "", err
}
m[n.Name()] = struct{}{}
}
out := make([]string, 0, len(m))
for k := range m {
out = append(out, k)
}
return strings.Join(out, ","), nil
}
func isDefaultMobyDriver(d driver.Driver) bool {
_, ok := d.(interface {
IsDefaultMobyDriver()
})
return ok
}
func toSolveOpt(d driver.Driver, multiDriver bool, opt Options, dl dockerLoadCallback) (solveOpt *client.SolveOpt, release func(), err error) {
defers := make([]func(), 0, 2)
releaseF := func() {
for _, f := range defers {
f()
}
}
defer func() {
if err != nil {
releaseF()
}
}()
if opt.ImageIDFile != "" {
if multiDriver || len(opt.Platforms) != 0 {
return nil, nil, errors.Errorf("image ID file cannot be specified when building for multiple platforms")
}
// Avoid leaving a stale file if we eventually fail
if err := os.Remove(opt.ImageIDFile); err != nil && !os.IsNotExist(err) {
return nil, nil, errors.Wrap(err, "removing image ID file")
}
}
// inline cache from build arg
if v, ok := opt.BuildArgs["BUILDKIT_INLINE_CACHE"]; ok {
if v, _ := strconv.ParseBool(v); v {
opt.CacheTo = append(opt.CacheTo, client.CacheOptionsEntry{
Type: "inline",
Attrs: map[string]string{},
})
}
}
for _, e := range opt.CacheTo {
if e.Type != "inline" && !d.Features()[driver.CacheExport] {
return nil, nil, notSupported(d, driver.CacheExport)
}
}
so := client.SolveOpt{
Frontend: "dockerfile.v0",
FrontendAttrs: map[string]string{},
LocalDirs: map[string]string{},
CacheExports: opt.CacheTo,
CacheImports: opt.CacheFrom,
}
if multiDriver {
// force creation of manifest list
so.FrontendAttrs["multi-platform"] = "true"
}
_, isDefaultMobyDriver := d.(interface {
IsDefaultMobyDriver()
})
for _, opt := range opt {
if !isDefaultMobyDriver && len(opt.Exports) == 0 {
logrus.Warnf("No output specified for %s driver. Build result will only remain in the build cache. To push result image into registry use --push or to load image into docker use --load", d.Factory().Name())
switch len(opt.Exports) {
case 1:
// valid
case 0:
if isDefaultMobyDriver {
// backwards compat for docker driver only:
// this ensures the build results in a docker image.
opt.Exports = []client.ExportEntry{{Type: "image", Attrs: map[string]string{}}}
}
default:
return nil, nil, errors.Errorf("multiple outputs currently unsupported")
}
// fill in image exporter names from tags
if len(opt.Tags) > 0 {
tags := make([]string, len(opt.Tags))
for i, tag := range opt.Tags {
ref, err := reference.Parse(tag)
if err != nil {
return nil, nil, errors.Wrapf(err, "invalid tag %q", tag)
}
tags[i] = ref.String()
}
for i, e := range opt.Exports {
switch e.Type {
case "image", "oci", "docker":
opt.Exports[i].Attrs["name"] = strings.Join(tags, ",")
}
}
} else {
for _, e := range opt.Exports {
if e.Type == "image" && e.Attrs["name"] == "" && e.Attrs["push"] != "" {
if ok, _ := strconv.ParseBool(e.Attrs["push"]); ok {
return nil, nil, errors.Errorf("tag is needed when pushing to registry")
}
}
}
}
// set up exporters
for i, e := range opt.Exports {
if (e.Type == "local" || e.Type == "tar") && opt.ImageIDFile != "" {
return nil, nil, errors.Errorf("local and tar exporters are incompatible with image ID file")
}
if e.Type == "oci" && !d.Features()[driver.OCIExporter] {
return nil, nil, notSupported(d, driver.OCIExporter)
}
if e.Type == "docker" {
if e.Output == nil {
if isDefaultMobyDriver {
e.Type = "image"
} else {
w, cancel, err := dl(e.Attrs["context"])
if err != nil {
return nil, nil, err
}
defers = append(defers, cancel)
opt.Exports[i].Output = w
}
} else if !d.Features()[driver.DockerExporter] {
return nil, nil, notSupported(d, driver.DockerExporter)
}
}
if e.Type == "image" && isDefaultMobyDriver {
opt.Exports[i].Type = "moby"
if e.Attrs["push"] != "" {
if ok, _ := strconv.ParseBool(e.Attrs["push"]); ok {
return nil, nil, errors.Errorf("auto-push is currently not implemented for docker driver")
}
}
}
}
so.Exports = opt.Exports
so.Session = opt.Session
releaseLoad, err := LoadInputs(opt.Inputs, &so)
if err != nil {
return nil, nil, err
}
defers = append(defers, releaseLoad)
if opt.Pull {
so.FrontendAttrs["image-resolve-mode"] = "pull"
}
if opt.Target != "" {
so.FrontendAttrs["target"] = opt.Target
}
if opt.NoCache {
so.FrontendAttrs["no-cache"] = ""
}
for k, v := range opt.BuildArgs {
so.FrontendAttrs["build-arg:"+k] = v
}
for k, v := range opt.Labels {
so.FrontendAttrs["label:"+k] = v
}
// set platforms
if len(opt.Platforms) != 0 {
pp := make([]string, len(opt.Platforms))
for i, p := range opt.Platforms {
pp[i] = platforms.Format(p)
}
if len(pp) > 1 && !d.Features()[driver.MultiPlatform] {
return nil, nil, notSupported(d, driver.MultiPlatform)
}
so.FrontendAttrs["platform"] = strings.Join(pp, ",")
}
// setup networkmode
switch opt.NetworkMode {
case "host", "none":
so.FrontendAttrs["force-network-mode"] = opt.NetworkMode
case "", "default":
default:
return nil, nil, errors.Errorf("network mode %q not supported by buildkit", opt.NetworkMode)
}
// setup extrahosts
extraHosts, err := toBuildkitExtraHosts(opt.ExtraHosts)
if err != nil {
return nil, nil, err
}
so.FrontendAttrs["add-hosts"] = extraHosts
return &so, releaseF, nil
}
func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, docker DockerAPI, auth Auth, pw progress.Writer) (resp map[string]*client.SolveResponse, err error) {
if len(drivers) == 0 {
return nil, errors.Errorf("driver required for build")
}
drivers, err = filterAvailableDrivers(drivers)
if err != nil {
return nil, errors.Wrapf(err, "no valid drivers found")
}
var noMobyDriver driver.Driver
for _, d := range drivers {
if !isDefaultMobyDriver(d.Driver) {
noMobyDriver = d.Driver
break
}
}
c, err := driver.Boot(ctx, d, pw)
if noMobyDriver != nil {
for _, opt := range opt {
if len(opt.Exports) == 0 {
logrus.Warnf("No output specified for %s driver. Build result will only remain in the build cache. To push result image into registry use --push or to load image into docker use --load", noMobyDriver.Factory().Name())
break
}
}
}
m, clients, err := resolveDrivers(ctx, drivers, opt, pw)
if err != nil {
close(pw.Status())
<-pw.Done()
return nil, err
}
withPrefix := len(opt) > 1
defers := make([]func(), 0, 2)
defer func() {
if err != nil {
for _, f := range defers {
f()
}
}
}()
mw := progress.NewMultiWriter(pw)
eg, ctx := errgroup.WithContext(ctx)
resp := map[string]*client.SolveResponse{}
var mu sync.Mutex
for k, opt := range opt {
multiDriver := len(m[k]) > 1
for i, dp := range m[k] {
d := drivers[dp.driverIndex].Driver
opt.Platforms = dp.platforms
so, release, err := toSolveOpt(d, multiDriver, opt, func(name string) (io.WriteCloser, func(), error) {
return newDockerLoader(ctx, docker, name, mw)
})
if err != nil {
return nil, err
}
defers = append(defers, release)
m[k][i].so = so
}
}
resp = map[string]*client.SolveResponse{}
var respMu sync.Mutex
multiTarget := len(opt) > 1
for k, opt := range opt {
pw := mw.WithPrefix(k, withPrefix)
err := func() error {
opt := opt
dps := m[k]
multiDriver := len(m[k]) > 1
if opt.ImageIDFile != "" {
if len(opt.Platforms) != 0 {
return nil, errors.Errorf("image ID file cannot be specified when building for multiple platforms")
}
// Avoid leaving a stale file if we eventually fail
if err := os.Remove(opt.ImageIDFile); err != nil && !os.IsNotExist(err) {
return nil, errors.Wrap(err, "removing image ID file")
}
}
res := make([]*client.SolveResponse, len(dps))
wg := &sync.WaitGroup{}
wg.Add(len(dps))
if v, ok := opt.BuildArgs["BUILDKIT_INLINE_CACHE"]; ok {
if v, _ := strconv.ParseBool(v); v {
opt.CacheTo = append(opt.CacheTo, client.CacheOptionsEntry{
Type: "inline",
Attrs: map[string]string{},
})
}
}
var pushNames string
for _, e := range opt.CacheTo {
if e.Type != "inline" && !d.Features()[driver.CacheExport] {
return nil, notSupported(d, driver.CacheExport)
}
}
so := client.SolveOpt{
Frontend: "dockerfile.v0",
FrontendAttrs: map[string]string{},
LocalDirs: map[string]string{},
CacheExports: opt.CacheTo,
CacheImports: opt.CacheFrom,
}
switch len(opt.Exports) {
case 1:
// valid
case 0:
if isDefaultMobyDriver {
// backwards compat for docker driver only:
// this ensures the build results in a docker image.
opt.Exports = []client.ExportEntry{{Type: "image", Attrs: map[string]string{}}}
}
default:
return nil, errors.Errorf("multiple outputs currently unsupported")
}
if len(opt.Tags) > 0 {
tags := make([]string, len(opt.Tags))
for i, tag := range opt.Tags {
ref, err := reference.Parse(tag)
if err != nil {
return nil, errors.Wrapf(err, "invalid tag %q", tag)
}
tags[i] = ref.String()
}
for i, e := range opt.Exports {
switch e.Type {
case "image", "oci", "docker":
opt.Exports[i].Attrs["name"] = strings.Join(tags, ",")
}
}
} else {
for _, e := range opt.Exports {
if e.Type == "image" && e.Attrs["name"] == "" && e.Attrs["push"] != "" {
if ok, _ := strconv.ParseBool(e.Attrs["push"]); ok {
return nil, errors.Errorf("tag is needed when pushing to registry")
}
}
}
}
for i, e := range opt.Exports {
if (e.Type == "local" || e.Type == "tar") && opt.ImageIDFile != "" {
return nil, errors.Errorf("local and tar exporters are incompatible with image ID file")
}
if e.Type == "oci" && !d.Features()[driver.OCIExporter] {
return nil, notSupported(d, driver.OCIExporter)
}
if e.Type == "docker" {
if e.Output == nil {
if isDefaultMobyDriver {
e.Type = "image"
} else {
w, cancel, err := newDockerLoader(ctx, docker, e.Attrs["context"], mw)
if err != nil {
return nil, err
}
defer cancel()
opt.Exports[i].Output = w
}
} else if !d.Features()[driver.DockerExporter] {
return nil, notSupported(d, driver.DockerExporter)
}
}
if e.Type == "image" && isDefaultMobyDriver {
opt.Exports[i].Type = "moby"
if e.Attrs["push"] != "" {
if ok, _ := strconv.ParseBool(e.Attrs["push"]); ok {
return nil, errors.Errorf("auto-push is currently not implemented for docker driver")
}
}
}
}
// TODO: handle loading to docker daemon
so.Exports = opt.Exports
so.Session = opt.Session
release, err := LoadInputs(opt.Inputs, &so)
if err != nil {
return nil, err
}
defer release()
if opt.Pull {
so.FrontendAttrs["image-resolve-mode"] = "pull"
}
if opt.Target != "" {
so.FrontendAttrs["target"] = opt.Target
}
if opt.NoCache {
so.FrontendAttrs["no-cache"] = ""
}
for k, v := range opt.BuildArgs {
so.FrontendAttrs["build-arg:"+k] = v
}
for k, v := range opt.Labels {
so.FrontendAttrs["label:"+k] = v
}
if len(opt.Platforms) != 0 {
pp := make([]string, len(opt.Platforms))
for i, p := range opt.Platforms {
pp[i] = platforms.Format(p)
}
if len(pp) > 1 && !d.Features()[driver.MultiPlatform] {
return nil, notSupported(d, driver.MultiPlatform)
}
so.FrontendAttrs["platform"] = strings.Join(pp, ",")
}
switch opt.NetworkMode {
case "host", "none":
so.FrontendAttrs["force-network-mode"] = opt.NetworkMode
case "", "default":
default:
return nil, errors.Errorf("network mode %q not supported by buildkit", opt.NetworkMode)
}
extraHosts, err := toBuildkitExtraHosts(opt.ExtraHosts)
if err != nil {
return nil, err
}
so.FrontendAttrs["add-hosts"] = extraHosts
var statusCh chan *client.SolveStatus
if pw != nil {
pw = progress.ResetTime(pw)
statusCh = pw.Status()
eg.Go(func() error {
<-pw.Done()
return pw.Err()
})
}
pw := mw.WithPrefix("default", false)
defer close(pw.Status())
wg.Wait()
select {
case <-ctx.Done():
return ctx.Err()
default:
}
eg.Go(func() error {
rr, err := c.Solve(ctx, nil, so, statusCh)
if err != nil {
return err
}
mu.Lock()
resp[k] = rr
mu.Unlock()
if opt.ImageIDFile != "" {
return ioutil.WriteFile(opt.ImageIDFile, []byte(rr.ExporterResponse["containerimage.digest"]), 0644)
respMu.Lock()
resp[k] = res[0]
respMu.Unlock()
if len(res) == 1 {
if opt.ImageIDFile != "" {
return ioutil.WriteFile(opt.ImageIDFile, []byte(res[0].ExporterResponse["containerimage.digest"]), 0644)
}
return nil
}
if pushNames != "" {
progress.Write(pw, fmt.Sprintf("merging manifest list %s", pushNames), func() error {
descs := make([]specs.Descriptor, 0, len(res))
for _, r := range res {
s, ok := r.ExporterResponse["containerimage.digest"]
if ok {
descs = append(descs, specs.Descriptor{
Digest: digest.Digest(s),
MediaType: images.MediaTypeDockerSchema2ManifestList,
Size: -1,
})
}
}
if len(descs) > 0 {
itpull := imagetools.New(imagetools.Opt{
Auth: auth,
})
names := strings.Split(pushNames, ",")
dt, desc, err := itpull.Combine(ctx, names[0], descs)
if err != nil {
return err
}
if opt.ImageIDFile != "" {
return ioutil.WriteFile(opt.ImageIDFile, []byte(desc.Digest), 0644)
}
itpush := imagetools.New(imagetools.Opt{
Auth: auth,
})
for _, n := range names {
nn, err := reference.ParseNormalizedNamed(n)
if err != nil {
return err
}
if err := itpush.Push(ctx, nn, desc, dt); err != nil {
return err
}
}
respMu.Lock()
resp[k] = &client.SolveResponse{
ExporterResponse: map[string]string{
"containerimage.digest": desc.Digest.String(),
},
}
respMu.Unlock()
}
return nil
})
}
return nil
})
for i, dp := range dps {
so := *dp.so
if multiDriver {
for i, e := range so.Exports {
switch e.Type {
case "oci", "tar":
return errors.Errorf("%s for multi-node builds currently not supported", e.Type)
case "image":
if pushNames == "" && e.Attrs["push"] != "" {
if ok, _ := strconv.ParseBool(e.Attrs["push"]); ok {
pushNames = e.Attrs["name"]
if pushNames == "" {
return errors.Errorf("tag is needed when pushing to registry")
}
names, err := toRepoOnly(e.Attrs["name"])
if err != nil {
return err
}
e.Attrs["name"] = names
e.Attrs["push-by-digest"] = "true"
so.Exports[i].Attrs = e.Attrs
}
}
}
}
}
func(i int, dp driverPair, so client.SolveOpt) {
pw := mw.WithPrefix(k, multiTarget)
c := clients[dp.driverIndex]
var statusCh chan *client.SolveStatus
if pw != nil {
pw = progress.ResetTime(pw)
statusCh = pw.Status()
eg.Go(func() error {
<-pw.Done()
return pw.Err()
})
}
eg.Go(func() error {
defer wg.Done()
rr, err := c.Solve(ctx, nil, so, statusCh)
if err != nil {
return err
}
res[i] = rr
return nil
})
}(i, dp, so)
}
return nil
})
}()
if err != nil {
return nil, err
}
}
if err := eg.Wait(); err != nil {
@ -423,6 +803,8 @@ func notSupported(d driver.Driver, f driver.Feature) error {
return errors.Errorf("%s feature is currently not supported for %s driver. Please switch to a different driver (eg. \"docker buildx create --use\")", f, d.Factory().Name())
}
type dockerLoadCallback func(name string) (io.WriteCloser, func(), error)
func newDockerLoader(ctx context.Context, d DockerAPI, name string, mw *progress.MultiWriter) (io.WriteCloser, func(), error) {
c, err := d.DockerAPI(name)
if err != nil {

2
commands/build.go
View File

@ -180,7 +180,7 @@ func buildTargets(ctx context.Context, dockerCli command.Cli, opts map[string]bu
defer cancel()
pw := progress.NewPrinter(ctx2, os.Stderr, progressMode)
_, err = build.Build(ctx, dis, opts, dockerAPI(dockerCli), pw)
_, err = build.Build(ctx, dis, opts, dockerAPI(dockerCli), dockerCli.ConfigFile(), pw)
return err
}

213
commands/imagetools/create.go
View File

@ -1,27 +1,226 @@
package commands
import (
"encoding/json"
"fmt"
"io/ioutil"
"strings"
"github.com/docker/cli/cli/command"
"github.com/docker/distribution/reference"
"github.com/moby/buildkit/util/appcontext"
"github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
"github.com/spf13/cobra"
"github.com/tonistiigi/buildx/util/imagetools"
"golang.org/x/sync/errgroup"
)
type createOptions struct {
files []string
tags []string
dryrun bool
append bool
files []string
tags []string
dryrun bool
actionAppend bool
}
func runCreate(dockerCli command.Cli, in createOptions, args []string) error {
return errors.Errorf("not-implemented")
if len(args) == 0 && len(in.files) == 0 {
return errors.Errorf("no sources specified")
}
if !in.dryrun && len(in.tags) == 0 {
return errors.Errorf("can't push with no tags specified, please set --tag or --dry-run")
}
fileArgs := make([]string, len(in.files))
for i, f := range in.files {
dt, err := ioutil.ReadFile(f)
if err != nil {
return err
}
fileArgs[i] = string(dt)
}
args = append(fileArgs, args...)
tags, err := parseRefs(in.tags)
if err != nil {
return err
}
if in.actionAppend && len(in.tags) > 0 {
args = append([]string{in.tags[0]}, args...)
}
srcs, err := parseSources(args)
if err != nil {
return err
}
repos := map[string]struct{}{}
for _, t := range tags {
repos[t.Name()] = struct{}{}
}
sourceRefs := false
for _, s := range srcs {
if s.Ref != nil {
repos[s.Ref.Name()] = struct{}{}
sourceRefs = true
}
}
if len(repos) == 0 {
return errors.Errorf("no repositories specified, please set a reference in tag or source")
}
if len(repos) > 1 {
return errors.Errorf("multiple repositories currently not supported, found %v", repos)
}
var repo string
for r := range repos {
repo = r
}
for i, s := range srcs {
if s.Ref == nil && s.Desc.MediaType == "" && s.Desc.Digest != "" {
n, err := reference.ParseNormalizedNamed(repo)
if err != nil {
return err
}
r, err := reference.WithDigest(n, s.Desc.Digest)
if err != nil {
return err
}
srcs[i].Ref = r
sourceRefs = true
}
}
ctx := appcontext.Context()
r := imagetools.New(imagetools.Opt{
Auth: dockerCli.ConfigFile(),
})
if sourceRefs {
eg, ctx2 := errgroup.WithContext(ctx)
for i, s := range srcs {
if s.Ref == nil {
continue
}
func(i int) {
eg.Go(func() error {
_, desc, err := r.Resolve(ctx2, srcs[i].Ref.String())
if err != nil {
return err
}
srcs[i].Ref = nil
srcs[i].Desc = desc
return nil
})
}(i)
}
if err := eg.Wait(); err != nil {
return err
}
}
descs := make([]ocispec.Descriptor, len(srcs))
for i := range descs {
descs[i] = srcs[i].Desc
}
dt, desc, err := r.Combine(ctx, repo, descs)
if err != nil {
return err
}
if in.dryrun {
fmt.Printf("%s\n", dt)
return nil
}
// new resolver cause need new auth
r = imagetools.New(imagetools.Opt{
Auth: dockerCli.ConfigFile(),
})
for _, t := range tags {
if err := r.Push(ctx, t, desc, dt); err != nil {
return err
}
fmt.Println(t.String())
}
return nil
}
type src struct {
Desc ocispec.Descriptor
Ref reference.Named
}
func parseSources(in []string) ([]*src, error) {
out := make([]*src, len(in))
for i, in := range in {
s, err := parseSource(in)
if err != nil {
return nil, errors.Wrapf(err, "failed to parse source %q, valid sources are digests, refereces and descriptors", in)
}
out[i] = s
}
return out, nil
}
func parseRefs(in []string) ([]reference.Named, error) {
refs := make([]reference.Named, len(in))
for i, in := range in {
n, err := reference.ParseNormalizedNamed(in)
if err != nil {
return nil, err
}
refs[i] = n
}
return refs, nil
}
func parseSource(in string) (*src, error) {
// source can be a digest, reference or a descriptor JSON
dgst, err := digest.Parse(in)
if err == nil {
return &src{
Desc: ocispec.Descriptor{
Digest: dgst,
},
}, nil
} else if strings.HasPrefix(in, "sha256") {
return nil, err
}
ref, err := reference.ParseNormalizedNamed(in)
if err == nil {
return &src{
Ref: ref,
}, nil
} else if !strings.HasPrefix(in, "{") {
return nil, err
}
var s src
if err := json.Unmarshal([]byte(in), &s.Desc); err != nil {
return nil, errors.WithStack(err)
}
return &s, nil
}
func createCmd(dockerCli command.Cli) *cobra.Command {
var options createOptions
cmd := &cobra.Command{
Use: "create [OPTIONS] [SOURCE...]",
Use: "create [OPTIONS] [SOURCE] [SOURCE...]",
Short: "Create a new image based on source images",
RunE: func(cmd *cobra.Command, args []string) error {
return runCreate(dockerCli, options, args)
@ -33,7 +232,7 @@ func createCmd(dockerCli command.Cli) *cobra.Command {
flags.StringArrayVarP(&options.files, "file", "f", []string{}, "Read source descriptor from file")
flags.StringArrayVarP(&options.tags, "tag", "t", []string{}, "Set reference for new image")
flags.BoolVar(&options.dryrun, "dry-run", false, "Show final image instead of pushing")
flags.BoolVar(&options.append, "append", false, "Append to existing manifest")
flags.BoolVar(&options.actionAppend, "append", false, "Append to existing manifest")
_ = flags

1
driver/docker-container/driver.go
View File

@ -25,7 +25,6 @@ var buildkitImage = "moby/buildkit:master" // TODO: make this verified and confi
type Driver struct {
driver.InitConfig
factory driver.Factory
version dockertypes.Version
}
func (d *Driver) Bootstrap(ctx context.Context, l progress.Logger) error {

7
driver/docker-container/factory.go
View File

@ -38,12 +38,7 @@ func (f *factory) New(ctx context.Context, cfg driver.InitConfig) (driver.Driver
return nil, errors.Errorf("%s driver requires docker API access", f.Name())
}
v, err := cfg.DockerAPI.ServerVersion(ctx)
if err != nil {
return nil, errors.Wrapf(driver.ErrNotConnecting, err.Error())
}
return &Driver{factory: f, InitConfig: cfg, version: v}, nil
return &Driver{factory: f, InitConfig: cfg}, nil
}
func (f *factory) AllowsInstances() bool {

2
driver/docker/driver.go
View File

@ -5,7 +5,6 @@ import (
"net"
"time"
dockertypes "github.com/docker/docker/api/types"
"github.com/moby/buildkit/client"
"github.com/pkg/errors"
"github.com/tonistiigi/buildx/driver"
@ -15,7 +14,6 @@ import (
type Driver struct {
factory driver.Factory
driver.InitConfig
version dockertypes.Version
}
func (d *Driver) Bootstrap(ctx context.Context, l progress.Logger) error {

7
driver/docker/factory.go
View File

@ -45,12 +45,7 @@ func (f *factory) New(ctx context.Context, cfg driver.InitConfig) (driver.Driver
return nil, errors.Errorf("docker driver requires docker API access")
}
v, err := cfg.DockerAPI.ServerVersion(ctx)
if err != nil {
return nil, errors.Wrapf(driver.ErrNotConnecting, err.Error())
}
return &Driver{factory: f, InitConfig: cfg, version: v}, nil
return &Driver{factory: f, InitConfig: cfg}, nil
}
func (f *factory) AllowsInstances() bool {

2
driver/driver.go
View File

@ -72,7 +72,7 @@ func Boot(ctx context.Context, d Driver, pw progress.Writer) (*client.Client, er
}
}
c, err := d.Client(ctx)
c, err := d.Client(context.TODO())
if err != nil {
if errors.Cause(err) == ErrNotRunning && try <= 2 {
continue

212
util/imagetools/create.go Normal file
View File

@ -0,0 +1,212 @@
package imagetools
import (
"bytes"
"context"
"encoding/json"
"github.com/containerd/containerd/content"
"github.com/containerd/containerd/errdefs"
"github.com/containerd/containerd/images"
"github.com/docker/distribution/reference"
"github.com/opencontainers/go-digest"
"github.com/opencontainers/image-spec/specs-go"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
"golang.org/x/sync/errgroup"
)
func (r *Resolver) Combine(ctx context.Context, in string, descs []ocispec.Descriptor) ([]byte, ocispec.Descriptor, error) {
ref, err := parseRef(in)
if err != nil {
return nil, ocispec.Descriptor{}, err
}
eg, ctx := errgroup.WithContext(ctx)
dts := make([][]byte, len(descs))
for i := range dts {
func(i int) {
eg.Go(func() error {
dt, err := r.GetDescriptor(ctx, ref.String(), descs[i])
if err != nil {
return err
}
dts[i] = dt
if descs[i].MediaType == "" {
mt, err := detectMediaType(dt)
if err != nil {
return err
}
descs[i].MediaType = mt
}
mt := descs[i].MediaType
switch mt {
case images.MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest:
if descs[i].Platform == nil {
cfg, err := r.loadConfig(ctx, in, dt)
if err != nil {
return err
}
descs[i].Platform = &ocispec.Platform{
OS: cfg.OS,
Architecture: cfg.Architecture,
}
}
case images.MediaTypeDockerSchema1Manifest:
return errors.Errorf("schema1 manifests are not allowed in manifest lists")
}
return nil
})
}(i)
}
if err := eg.Wait(); err != nil {
return nil, ocispec.Descriptor{}, err
}
// on single source, return original bytes
if len(descs) == 1 {
if mt := descs[0].MediaType; mt == images.MediaTypeDockerSchema2ManifestList || mt == ocispec.MediaTypeImageIndex {
return dts[0], descs[0], nil
}
}
m := map[digest.Digest]int{}
newDescs := make([]ocispec.Descriptor, 0, len(descs))
addDesc := func(d ocispec.Descriptor) {
idx, ok := m[d.Digest]
if ok {
old := newDescs[idx]
if old.MediaType == "" {
old.MediaType = d.MediaType
}
if d.Platform != nil {
old.Platform = d.Platform
}
if old.Annotations == nil {
old.Annotations = map[string]string{}
}
for k, v := range d.Annotations {
old.Annotations[k] = v
}
newDescs[idx] = old
} else {
m[d.Digest] = len(newDescs)
newDescs = append(newDescs, d)
}
}
for i, desc := range descs {
switch desc.MediaType {
case images.MediaTypeDockerSchema2ManifestList, ocispec.MediaTypeImageIndex:
var mfst ocispec.Index
if err := json.Unmarshal(dts[i], &mfst); err != nil {
return nil, ocispec.Descriptor{}, errors.WithStack(err)
}
for _, d := range mfst.Manifests {
addDesc(d)
}
default:
addDesc(desc)
}
}
mt := images.MediaTypeDockerSchema2ManifestList //ocispec.MediaTypeImageIndex
idx := struct {
// MediaType is reserved in the OCI spec but
// excluded from go types.
MediaType string `json:"mediaType,omitempty"`
ocispec.Index
}{
MediaType: mt,
Index: ocispec.Index{
Versioned: specs.Versioned{
SchemaVersion: 2,
},
Manifests: newDescs,
},
}
idxBytes, err := json.MarshalIndent(idx, "", " ")
if err != nil {
return nil, ocispec.Descriptor{}, errors.Wrap(err, "failed to marshal index")
}
return idxBytes, ocispec.Descriptor{
MediaType: mt,
Size: int64(len(idxBytes)),
Digest: digest.FromBytes(idxBytes),
}, nil
}
func (r *Resolver) Push(ctx context.Context, ref reference.Named, desc ocispec.Descriptor, dt []byte) error {
ref = reference.TagNameOnly(ref)
p, err := r.r.Pusher(ctx, ref.String())
if err != nil {
return err
}
cw, err := p.Push(ctx, desc)
if err != nil {
if errdefs.IsAlreadyExists(err) {
return nil
}
return err
}
err = content.Copy(ctx, cw, bytes.NewReader(dt), desc.Size, desc.Digest)
if errdefs.IsAlreadyExists(err) {
return nil
}
return err
}
func (r *Resolver) loadConfig(ctx context.Context, in string, dt []byte) (*ocispec.Image, error) {
var manifest ocispec.Manifest
if err := json.Unmarshal(dt, &manifest); err != nil {
return nil, errors.WithStack(err)
}
dt, err := r.GetDescriptor(ctx, in, manifest.Config)
if err != nil {
return nil, err
}
var img ocispec.Image
if err := json.Unmarshal(dt, &img); err != nil {
return nil, errors.WithStack(err)
}
return &img, nil
}
func detectMediaType(dt []byte) (string, error) {
var mfst struct {
MediaType string `json:"mediaType"`
Config json.RawMessage `json:"config"`
FSLayers []string `json:"fsLayers"`
}
if err := json.Unmarshal(dt, &mfst); err != nil {
return "", errors.WithStack(err)
}
if mfst.MediaType != "" {
return mfst.MediaType, nil
}
if mfst.Config != nil {
return images.MediaTypeDockerSchema2Manifest, nil
}
if len(mfst.FSLayers) > 0 {
return images.MediaTypeDockerSchema1Manifest, nil
}
return images.MediaTypeDockerSchema2ManifestList, nil
}

29
util/imagetools/inspect.go
View File

@ -35,35 +35,52 @@ func New(opt Opt) *Resolver {
}
}
func (r *Resolver) Get(ctx context.Context, in string) ([]byte, ocispec.Descriptor, error) {
func (r *Resolver) Resolve(ctx context.Context, in string) (string, ocispec.Descriptor, error) {
ref, err := parseRef(in)
if err != nil {
return nil, ocispec.Descriptor{}, err
return "", ocispec.Descriptor{}, err
}
in, desc, err := r.r.Resolve(ctx, ref.String())
if err != nil {
return "", ocispec.Descriptor{}, err
}
return in, desc, nil
}
func (r *Resolver) Get(ctx context.Context, in string) ([]byte, ocispec.Descriptor, error) {
in, desc, err := r.Resolve(ctx, in)
if err != nil {
return nil, ocispec.Descriptor{}, err
}
fetcher, err := r.r.Fetcher(ctx, in)
dt, err := r.GetDescriptor(ctx, in, desc)
if err != nil {
return nil, ocispec.Descriptor{}, err
}
return dt, desc, nil
}
func (r *Resolver) GetDescriptor(ctx context.Context, in string, desc ocispec.Descriptor) ([]byte, error) {
fetcher, err := r.r.Fetcher(ctx, in)
if err != nil {
return nil, err
}
rc, err := fetcher.Fetch(ctx, desc)
if err != nil {
return nil, ocispec.Descriptor{}, err
return nil, err
}
buf := &bytes.Buffer{}
_, err = io.Copy(buf, rc)
rc.Close()
if err != nil {
return nil, ocispec.Descriptor{}, err
return nil, err
}
return buf.Bytes(), desc, nil
return buf.Bytes(), nil
}
func parseRef(s string) (reference.Named, error) {

18
util/progress/multiwriter.go
View File

@ -3,14 +3,17 @@ package progress
import (
"context"
"strings"
"sync"
"github.com/moby/buildkit/client"
"golang.org/x/sync/errgroup"
)
type MultiWriter struct {
w Writer
eg *errgroup.Group
w Writer
eg *errgroup.Group
once sync.Once
ready chan struct{}
}
func (mw *MultiWriter) WithPrefix(pfx string, force bool) Writer {
@ -21,6 +24,9 @@ func (mw *MultiWriter) WithPrefix(pfx string, force bool) Writer {
in: in,
}
mw.eg.Go(func() error {
mw.once.Do(func() {
close(mw.ready)
})
for {
select {
case v, ok := <-in:
@ -77,14 +83,18 @@ func NewMultiWriter(pw Writer) *MultiWriter {
}
eg, _ := errgroup.WithContext(context.TODO())
ready := make(chan struct{})
go func() {
<-ready
eg.Wait()
close(pw.Status())
}()
return &MultiWriter{
w: pw,
eg: eg,
w: pw,
eg: eg,
ready: ready,
}
}

32
util/progress/writer.go
View File

@ -1,7 +1,11 @@
package progress
import (
"time"
"github.com/moby/buildkit/client"
"github.com/moby/buildkit/identity"
"github.com/opencontainers/go-digest"
)
type Writer interface {
@ -9,3 +13,31 @@ type Writer interface {
Err() error
Status() chan *client.SolveStatus
}
func Write(w Writer, name string, f func() error) {
status := w.Status()
dgst := digest.FromBytes([]byte(identity.NewID()))
tm := time.Now()
vtx := client.Vertex{
Digest: dgst,
Name: name,
Started: &tm,
}
status <- &client.SolveStatus{
Vertexes: []*client.Vertex{&vtx},
}
err := f()
tm2 := time.Now()
vtx2 := vtx
vtx2.Completed = &tm2
if err != nil {
vtx2.Error = err.Error()
}
status <- &client.SolveStatus{
Vertexes: []*client.Vertex{&vtx2},
}
}

8
vendor/modules.txt vendored
View File

@ -27,13 +27,13 @@ github.com/beorn7/perks/quantile
# github.com/containerd/console v0.0.0-20181022165439-0650fd9eeb50
github.com/containerd/console
# github.com/containerd/containerd v1.3.0-0.20190321141026-ceba56893a76
github.com/containerd/containerd/platforms
github.com/containerd/containerd/images
github.com/containerd/containerd/platforms
github.com/containerd/containerd/content
github.com/containerd/containerd/errdefs
github.com/containerd/containerd/remotes
github.com/containerd/containerd/remotes/docker
github.com/containerd/containerd/errdefs
github.com/containerd/containerd/log
github.com/containerd/containerd/content
github.com/containerd/containerd/content/local
github.com/containerd/containerd/labels
github.com/containerd/containerd/reference
@ -98,6 +98,7 @@ github.com/davecgh/go-spew/spew
# github.com/docker/cli v0.0.0-20190321234815-f40f9c240ab0 => github.com/tiborvass/cli v0.0.0-20190419012645-1ed02c40fe68
github.com/docker/cli/cli/compose/loader
github.com/docker/cli/cli/compose/types
github.com/docker/cli/cli/config/types
github.com/docker/cli/cli-plugins/manager
github.com/docker/cli/cli-plugins/plugin
github.com/docker/cli/cli/command
@ -106,7 +107,6 @@ github.com/docker/cli/cli
github.com/docker/cli/cli/config
github.com/docker/cli/cli/context/docker
github.com/docker/cli/opts
github.com/docker/cli/cli/config/types
github.com/docker/cli/cli/compose/interpolation
github.com/docker/cli/cli/compose/schema
github.com/docker/cli/cli/compose/template