docker/buildx
1
0
Fork 0
mirror of https://github.com/docker/buildx.git synced 2024-11-22 15:37:16 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #825 from tonistiigi/multi-node-registry-conf

Browse Source 
allow multi-node push and imagetools to use custom registry config
This commit is contained in:
CrazyMax 2021-11-04 18:43:31 +01:00 committed by GitHub
commit 4c938c77ba
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
24 changed files with 747 additions and 168 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
build/build.go
View File

@ -21,7 +21,7 @@ import (
"github.com/docker/buildx/driver"
"github.com/docker/buildx/util/imagetools"
"github.com/docker/buildx/util/progress"
clitypes "github.com/docker/cli/cli/config/types"
"github.com/docker/buildx/util/resolver"
"github.com/docker/cli/opts"
"github.com/docker/distribution/reference"
"github.com/docker/docker/api/types"
@ -86,10 +86,7 @@ type DriverInfo struct {
Name string
Platform []specs.Platform
Err error
}
type Auth interface {
GetAuthConfig(registryHostname string) (clitypes.AuthConfig, error)
ImageOpt imagetools.Opt
}
type DockerAPI interface {
@ -189,8 +186,8 @@ func splitToDriverPairs(availablePlatforms map[string]int, opt map[string]Option
return m
}
func resolveDrivers(ctx context.Context, drivers []DriverInfo, auth Auth, opt map[string]Options, pw progress.Writer) (map[string][]driverPair, []*client.Client, error) {
dps, clients, err := resolveDriversBase(ctx, drivers, auth, opt, pw)
func resolveDrivers(ctx context.Context, drivers []DriverInfo, opt map[string]Options, pw progress.Writer) (map[string][]driverPair, []*client.Client, error) {
dps, clients, err := resolveDriversBase(ctx, drivers, opt, pw)
if err != nil {
return nil, nil, err
}
@ -230,7 +227,7 @@ func resolveDrivers(ctx context.Context, drivers []DriverInfo, auth Auth, opt ma
return dps, clients, nil
}
func resolveDriversBase(ctx context.Context, drivers []DriverInfo, auth Auth, opt map[string]Options, pw progress.Writer) (map[string][]driverPair, []*client.Client, error) {
func resolveDriversBase(ctx context.Context, drivers []DriverInfo, opt map[string]Options, pw progress.Writer) (map[string][]driverPair, []*client.Client, error) {
availablePlatforms := map[string]int{}
for i, d := range drivers {
for _, p := range d.Platform {
@ -583,7 +580,7 @@ func toSolveOpt(ctx context.Context, d driver.Driver, multiDriver bool, opt Opti
return &so, releaseF, nil
}
func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, docker DockerAPI, auth Auth, configDir string, w progress.Writer) (resp map[string]*client.SolveResponse, err error) {
func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, docker DockerAPI, configDir string, w progress.Writer) (resp map[string]*client.SolveResponse, err error) {
if len(drivers) == 0 {
return nil, errors.Errorf("driver required for build")
}
@ -610,7 +607,7 @@ func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, do
}
}
m, clients, err := resolveDrivers(ctx, drivers, auth, opt, w)
m, clients, err := resolveDrivers(ctx, drivers, opt, w)
if err != nil {
return nil, err
}
@ -690,6 +687,7 @@ func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, do
wg.Add(len(dps))
var pushNames string
var insecurePush bool
eg.Go(func() (err error) {
defer func() {
@ -731,11 +729,30 @@ func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, do
}
}
if len(descs) > 0 {
itpull := imagetools.New(imagetools.Opt{
Auth: auth,
})
var imageopt imagetools.Opt
for _, dp := range dps {
imageopt = drivers[dp.driverIndex].ImageOpt
break
}
names := strings.Split(pushNames, ",")
if insecurePush {
insecureTrue := true
httpTrue := true
nn, err := reference.ParseNormalizedNamed(names[0])
if err != nil {
return err
}
imageopt.RegistryConfig = map[string]resolver.RegistryConfig{
reference.Domain(nn): {
Insecure: &insecureTrue,
PlainHTTP: &httpTrue,
},
}
}
itpull := imagetools.New(imageopt)
dt, desc, err := itpull.Combine(ctx, names[0], descs)
if err != nil {
return err
@ -746,9 +763,7 @@ func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, do
}
}
itpush := imagetools.New(imagetools.Opt{
Auth: auth,
})
itpush := imagetools.New(imageopt)
for _, n := range names {
nn, err := reference.ParseNormalizedNamed(n)
@ -793,6 +808,9 @@ func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, do
if err != nil {
return err
}
if ok, _ := strconv.ParseBool(e.Attrs["registry.insecure"]); ok {
insecurePush = true
}
e.Attrs["name"] = names
e.Attrs["push-by-digest"] = "true"
so.Exports[i].Attrs = e.Attrs

2
commands/bake.go
View File

@ -156,7 +156,7 @@ func runBake(dockerCli command.Cli, targets []string, in bakeOptions) (err error
return nil
}
resp, err := build.Build(ctx, dis, bo, dockerAPI(dockerCli), dockerCli.ConfigFile(), confutil.ConfigDir(dockerCli), printer)
resp, err := build.Build(ctx, dis, bo, dockerAPI(dockerCli), confutil.ConfigDir(dockerCli), printer)
if err != nil {
return err
}

2
commands/build.go
View File

@ -225,7 +225,7 @@ func buildTargets(ctx context.Context, dockerCli command.Cli, opts map[string]bu
printer := progress.NewPrinter(ctx2, os.Stderr, progressMode)
resp, err := build.Build(ctx, dis, opts, dockerAPI(dockerCli), dockerCli.ConfigFile(), confutil.ConfigDir(dockerCli), printer)
resp, err := build.Build(ctx, dis, opts, dockerAPI(dockerCli), confutil.ConfigDir(dockerCli), printer)
err1 := printer.Wait()
if err == nil {
err = err1

7
commands/create.go
View File

@ -12,6 +12,7 @@ import (
"github.com/docker/buildx/driver"
"github.com/docker/buildx/store"
"github.com/docker/buildx/store/storeutil"
"github.com/docker/cli/cli"
"github.com/docker/cli/cli/command"
"github.com/google/shlex"
@ -74,7 +75,7 @@ func runCreate(dockerCli command.Cli, in createOptions, args []string) error {
return errors.Errorf("failed to find driver %q", in.driver)
}
txn, release, err := getStore(dockerCli)
txn, release, err := storeutil.GetStore(dockerCli)
if err != nil {
return err
}
@ -142,7 +143,7 @@ func runCreate(dockerCli command.Cli, in createOptions, args []string) error {
return errors.Errorf("could not create a builder instance with TLS data loaded from environment. Please use `docker context create <context-name>` to create a context for current environment and then create a builder instance with `docker buildx create <context-name>`")
}
ep, err = getCurrentEndpoint(dockerCli)
ep, err = storeutil.GetCurrentEndpoint(dockerCli)
if err != nil {
return err
}
@ -174,7 +175,7 @@ func runCreate(dockerCli command.Cli, in createOptions, args []string) error {
}
if in.use && ep != "" {
current, err := getCurrentEndpoint(dockerCli)
current, err := storeutil.GetCurrentEndpoint(dockerCli)
if err != nil {
return err
}

41
commands/imagetools/create.go
View File

@ -6,6 +6,8 @@ import (
"io/ioutil"
"strings"
"github.com/docker/buildx/store"
"github.com/docker/buildx/store/storeutil"
"github.com/docker/buildx/util/imagetools"
"github.com/docker/cli/cli/command"
"github.com/docker/distribution/reference"
@ -18,6 +20,7 @@ import (
)
type createOptions struct {
builder string
files []string
tags []string
dryrun bool
@ -101,9 +104,32 @@ func runCreate(dockerCli command.Cli, in createOptions, args []string) error {
ctx := appcontext.Context()
r := imagetools.New(imagetools.Opt{
Auth: dockerCli.ConfigFile(),
})
txn, release, err := storeutil.GetStore(dockerCli)
if err != nil {
return err
}
defer release()
var ng *store.NodeGroup
if in.builder != "" {
ng, err = storeutil.GetNodeGroup(txn, dockerCli, in.builder)
if err != nil {
return err
}
} else {
ng, err = storeutil.GetCurrentInstance(txn, dockerCli)
if err != nil {
return err
}
}
imageopt, err := storeutil.GetImageConfig(dockerCli, ng)
if err != nil {
return err
}
r := imagetools.New(imageopt)
if sourceRefs {
eg, ctx2 := errgroup.WithContext(ctx)
@ -152,9 +178,7 @@ func runCreate(dockerCli command.Cli, in createOptions, args []string) error {
}
// new resolver cause need new auth
r = imagetools.New(imagetools.Opt{
Auth: dockerCli.ConfigFile(),
})
r = imagetools.New(imageopt)
for _, t := range tags {
if err := r.Push(ctx, t, desc, dt); err != nil {
@ -224,13 +248,14 @@ func parseSource(in string) (*src, error) {
return &s, nil
}
func createCmd(dockerCli command.Cli) *cobra.Command {
func createCmd(dockerCli command.Cli, opts RootOptions) *cobra.Command {
var options createOptions
cmd := &cobra.Command{
Use: "create [OPTIONS] [SOURCE] [SOURCE...]",
Short: "Create a new image based on source images",
RunE: func(cmd *cobra.Command, args []string) error {
options.builder = opts.Builder
return runCreate(dockerCli, options, args)
},
}
@ -242,8 +267,6 @@ func createCmd(dockerCli command.Cli) *cobra.Command {
flags.BoolVar(&options.dryrun, "dry-run", false, "Show final image instead of pushing")
flags.BoolVar(&options.actionAppend, "append", false, "Append to existing manifest")
_ = flags
return cmd
}

38
commands/imagetools/inspect.go
View File

@ -5,6 +5,8 @@ import (
"os"
"github.com/containerd/containerd/images"
"github.com/docker/buildx/store"
"github.com/docker/buildx/store/storeutil"
"github.com/docker/buildx/util/imagetools"
"github.com/docker/cli/cli"
"github.com/docker/cli/cli/command"
@ -14,15 +16,38 @@ import (
)
type inspectOptions struct {
raw bool
raw bool
builder string
}
func runInspect(dockerCli command.Cli, in inspectOptions, name string) error {
ctx := appcontext.Context()
r := imagetools.New(imagetools.Opt{
Auth: dockerCli.ConfigFile(),
})
txn, release, err := storeutil.GetStore(dockerCli)
if err != nil {
return err
}
defer release()
var ng *store.NodeGroup
if in.builder != "" {
ng, err = storeutil.GetNodeGroup(txn, dockerCli, in.builder)
if err != nil {
return err
}
} else {
ng, err = storeutil.GetCurrentInstance(txn, dockerCli)
if err != nil {
return err
}
}
imageopt, err := storeutil.GetImageConfig(dockerCli, ng)
if err != nil {
return err
}
r := imagetools.New(imageopt)
dt, desc, err := r.Get(ctx, name)
if err != nil {
@ -46,7 +71,7 @@ func runInspect(dockerCli command.Cli, in inspectOptions, name string) error {
return nil
}
func inspectCmd(dockerCli command.Cli) *cobra.Command {
func inspectCmd(dockerCli command.Cli, rootOpts RootOptions) *cobra.Command {
var options inspectOptions
cmd := &cobra.Command{
@ -54,6 +79,7 @@ func inspectCmd(dockerCli command.Cli) *cobra.Command {
Short: "Show details of image in the registry",
Args: cli.ExactArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
options.builder = rootOpts.Builder
return runInspect(dockerCli, options, args[0])
},
}
@ -62,7 +88,5 @@ func inspectCmd(dockerCli command.Cli) *cobra.Command {
flags.BoolVar(&options.raw, "raw", false, "Show original JSON manifest")
_ = flags
return cmd
}

10
commands/imagetools/root.go
View File

@ -5,15 +5,19 @@ import (
"github.com/spf13/cobra"
)
func RootCmd(dockerCli command.Cli) *cobra.Command {
type RootOptions struct {
Builder string
}
func RootCmd(dockerCli command.Cli, opts RootOptions) *cobra.Command {
cmd := &cobra.Command{
Use: "imagetools",
Short: "Commands to work on images in registry",
}
cmd.AddCommand(
inspectCmd(dockerCli),
createCmd(dockerCli),
inspectCmd(dockerCli, opts),
createCmd(dockerCli, opts),
)
return cmd

7
commands/inspect.go
View File

@ -9,6 +9,7 @@ import (
"time"
"github.com/docker/buildx/store"
"github.com/docker/buildx/store/storeutil"
"github.com/docker/buildx/util/platformutil"
"github.com/docker/cli/cli"
"github.com/docker/cli/cli/command"
@ -24,7 +25,7 @@ type inspectOptions struct {
func runInspect(dockerCli command.Cli, in inspectOptions) error {
ctx := appcontext.Context()
txn, release, err := getStore(dockerCli)
txn, release, err := storeutil.GetStore(dockerCli)
if err != nil {
return err
}
@ -33,12 +34,12 @@ func runInspect(dockerCli command.Cli, in inspectOptions) error {
var ng *store.NodeGroup
if in.builder != "" {
ng, err = getNodeGroup(txn, dockerCli, in.builder)
ng, err = storeutil.GetNodeGroup(txn, dockerCli, in.builder)
if err != nil {
return err
}
} else {
ng, err = getCurrentInstance(txn, dockerCli)
ng, err = storeutil.GetCurrentInstance(txn, dockerCli)
if err != nil {
return err
}

5
commands/ls.go
View File

@ -10,6 +10,7 @@ import (
"time"
"github.com/docker/buildx/store"
"github.com/docker/buildx/store/storeutil"
"github.com/docker/buildx/util/platformutil"
"github.com/docker/cli/cli"
"github.com/docker/cli/cli/command"
@ -24,7 +25,7 @@ type lsOptions struct {
func runLs(dockerCli command.Cli, in lsOptions) error {
ctx := appcontext.Context()
txn, release, err := getStore(dockerCli)
txn, release, err := storeutil.GetStore(dockerCli)
if err != nil {
return err
}
@ -79,7 +80,7 @@ func runLs(dockerCli command.Cli, in lsOptions) error {
}
currentName := "default"
current, err := getCurrentInstance(txn, dockerCli)
current, err := storeutil.GetCurrentInstance(txn, dockerCli)
if err != nil {
return err
}

7
commands/rm.go
View File

@ -4,6 +4,7 @@ import (
"context"
"github.com/docker/buildx/store"
"github.com/docker/buildx/store/storeutil"
"github.com/docker/cli/cli"
"github.com/docker/cli/cli/command"
"github.com/moby/buildkit/util/appcontext"
@ -18,14 +19,14 @@ type rmOptions struct {
func runRm(dockerCli command.Cli, in rmOptions) error {
ctx := appcontext.Context()
txn, release, err := getStore(dockerCli)
txn, release, err := storeutil.GetStore(dockerCli)
if err != nil {
return err
}
defer release()
if in.builder != "" {
ng, err := getNodeGroup(txn, dockerCli, in.builder)
ng, err := storeutil.GetNodeGroup(txn, dockerCli, in.builder)
if err != nil {
return err
}
@ -36,7 +37,7 @@ func runRm(dockerCli command.Cli, in rmOptions) error {
return err1
}
ng, err := getCurrentInstance(txn, dockerCli)
ng, err := storeutil.GetCurrentInstance(txn, dockerCli)
if err != nil {
return err
}

2
commands/root.go
View File

@ -48,7 +48,7 @@ func addCommands(cmd *cobra.Command, dockerCli command.Cli) {
versionCmd(dockerCli),
pruneCmd(dockerCli, opts),
duCmd(dockerCli, opts),
imagetoolscmd.RootCmd(dockerCli),
imagetoolscmd.RootCmd(dockerCli, imagetoolscmd.RootOptions{Builder: opts.builder}),
)
}

7
commands/stop.go
View File

@ -4,6 +4,7 @@ import (
"context"
"github.com/docker/buildx/store"
"github.com/docker/buildx/store/storeutil"
"github.com/docker/cli/cli"
"github.com/docker/cli/cli/command"
"github.com/moby/buildkit/util/appcontext"
@ -17,14 +18,14 @@ type stopOptions struct {
func runStop(dockerCli command.Cli, in stopOptions) error {
ctx := appcontext.Context()
txn, release, err := getStore(dockerCli)
txn, release, err := storeutil.GetStore(dockerCli)
if err != nil {
return err
}
defer release()
if in.builder != "" {
ng, err := getNodeGroup(txn, dockerCli, in.builder)
ng, err := storeutil.GetNodeGroup(txn, dockerCli, in.builder)
if err != nil {
return err
}
@ -34,7 +35,7 @@ func runStop(dockerCli command.Cli, in stopOptions) error {
return nil
}
ng, err := getCurrentInstance(txn, dockerCli)
ng, err := storeutil.GetCurrentInstance(txn, dockerCli)
if err != nil {
return err
}

7
commands/use.go
View File

@ -3,6 +3,7 @@ package commands
import (
"os"
"github.com/docker/buildx/store/storeutil"
"github.com/docker/cli/cli"
"github.com/docker/cli/cli/command"
"github.com/pkg/errors"
@ -16,7 +17,7 @@ type useOptions struct {
}
func runUse(dockerCli command.Cli, in useOptions) error {
txn, release, err := getStore(dockerCli)
txn, release, err := storeutil.GetStore(dockerCli)
if err != nil {
return err
}
@ -28,7 +29,7 @@ func runUse(dockerCli command.Cli, in useOptions) error {
return errors.Errorf("run `docker context use default` to switch to default context")
}
if in.builder == "default" || in.builder == dockerCli.CurrentContext() {
ep, err := getCurrentEndpoint(dockerCli)
ep, err := storeutil.GetCurrentEndpoint(dockerCli)
if err != nil {
return err
}
@ -51,7 +52,7 @@ func runUse(dockerCli command.Cli, in useOptions) error {
return errors.Wrapf(err, "failed to find instance %q", in.builder)
}
ep, err := getCurrentEndpoint(dockerCli)
ep, err := storeutil.GetCurrentEndpoint(dockerCli)
if err != nil {
return err
}

127
commands/util.go
View File

@ -9,7 +9,7 @@ import (
"github.com/docker/buildx/build"
"github.com/docker/buildx/driver"
"github.com/docker/buildx/store"
"github.com/docker/buildx/util/confutil"
"github.com/docker/buildx/store/storeutil"
"github.com/docker/buildx/util/platformutil"
"github.com/docker/buildx/util/progress"
"github.com/docker/cli/cli/command"
@ -25,53 +25,9 @@ import (
"k8s.io/client-go/tools/clientcmd"
)
// getStore returns current builder instance store
func getStore(dockerCli command.Cli) (*store.Txn, func(), error) {
s, err := store.New(confutil.ConfigDir(dockerCli))
if err != nil {
return nil, nil, err
}
return s.Txn()
}
// getCurrentEndpoint returns the current default endpoint value
func getCurrentEndpoint(dockerCli command.Cli) (string, error) {
name := dockerCli.CurrentContext()
if name != "default" {
return name, nil
}
de, err := getDockerEndpoint(dockerCli, name)
if err != nil {
return "", errors.Errorf("docker endpoint for %q not found", name)
}
return de, nil
}
// getDockerEndpoint returns docker endpoint string for given context
func getDockerEndpoint(dockerCli command.Cli, name string) (string, error) {
list, err := dockerCli.ContextStore().List()
if err != nil {
return "", err
}
for _, l := range list {
if l.Name == name {
ep, ok := l.Endpoints["docker"]
if !ok {
return "", errors.Errorf("context %q does not have a Docker endpoint", name)
}
typed, ok := ep.(docker.EndpointMeta)
if !ok {
return "", errors.Errorf("endpoint %q is not of type EndpointMeta, %T", ep, ep)
}
return typed.Host, nil
}
}
return "", nil
}
// validateEndpoint validates that endpoint is either a context or a docker host
func validateEndpoint(dockerCli command.Cli, ep string) (string, error) {
de, err := getDockerEndpoint(dockerCli, ep)
de, err := storeutil.GetDockerEndpoint(dockerCli, ep)
if err == nil && de != "" {
if ep == "default" {
return de, nil
@ -85,60 +41,6 @@ func validateEndpoint(dockerCli command.Cli, ep string) (string, error) {
return h, nil
}
// getCurrentInstance finds the current builder instance
func getCurrentInstance(txn *store.Txn, dockerCli command.Cli) (*store.NodeGroup, error) {
ep, err := getCurrentEndpoint(dockerCli)
if err != nil {
return nil, err
}
ng, err := txn.Current(ep)
if err != nil {
return nil, err
}
if ng == nil {
ng, _ = getNodeGroup(txn, dockerCli, dockerCli.CurrentContext())
}
return ng, nil
}
// getNodeGroup returns nodegroup based on the name
func getNodeGroup(txn *store.Txn, dockerCli command.Cli, name string) (*store.NodeGroup, error) {
ng, err := txn.NodeGroupByName(name)
if err != nil {
if !os.IsNotExist(errors.Cause(err)) {
return nil, err
}
}
if ng != nil {
return ng, nil
}
if name == "default" {
name = dockerCli.CurrentContext()
}
list, err := dockerCli.ContextStore().List()
if err != nil {
return nil, err
}
for _, l := range list {
if l.Name == name {
return &store.NodeGroup{
Name: "default",
Nodes: []store.Node{
{
Name: "default",
Endpoint: name,
},
},
}, nil
}
}
return nil, errors.Errorf("no builder %q found", name)
}
// driversForNodeGroup returns drivers for a nodegroup instance
func driversForNodeGroup(ctx context.Context, dockerCli command.Cli, ng *store.NodeGroup, contextPathHash string) ([]build.DriverInfo, error) {
eg, _ := errgroup.WithContext(ctx)
@ -162,6 +64,10 @@ func driversForNodeGroup(ctx context.Context, dockerCli command.Cli, ng *store.N
}
ng.Driver = f.Name()
}
imageopt, err := storeutil.GetImageConfig(dockerCli, ng)
if err != nil {
return nil, err
}
for i, n := range ng.Nodes {
func(i int, n store.Node) {
@ -211,12 +117,13 @@ func driversForNodeGroup(ctx context.Context, dockerCli command.Cli, ng *store.N
}
}
d, err := driver.GetDriver(ctx, "buildx_buildkit_"+n.Name, f, dockerapi, dockerCli.ConfigFile(), kcc, n.Flags, n.Files, n.DriverOpts, n.Platforms, contextPathHash)
d, err := driver.GetDriver(ctx, "buildx_buildkit_"+n.Name, f, dockerapi, imageopt.Auth, kcc, n.Flags, n.Files, n.DriverOpts, n.Platforms, contextPathHash)
if err != nil {
di.Err = err
return nil
}
di.Driver = d
di.ImageOpt = imageopt
return nil
})
}(i, n)
@ -314,7 +221,7 @@ func getInstanceOrDefault(ctx context.Context, dockerCli command.Cli, instance,
}
func getInstanceByName(ctx context.Context, dockerCli command.Cli, instance, contextPathHash string) ([]build.DriverInfo, error) {
txn, release, err := getStore(dockerCli)
txn, release, err := storeutil.GetStore(dockerCli)
if err != nil {
return nil, err
}
@ -329,14 +236,14 @@ func getInstanceByName(ctx context.Context, dockerCli command.Cli, instance, con
// getDefaultDrivers returns drivers based on current cli config
func getDefaultDrivers(ctx context.Context, dockerCli command.Cli, defaultOnly bool, contextPathHash string) ([]build.DriverInfo, error) {
txn, release, err := getStore(dockerCli)
txn, release, err := storeutil.GetStore(dockerCli)
if err != nil {
return nil, err
}
defer release()
if !defaultOnly {
ng, err := getCurrentInstance(txn, dockerCli)
ng, err := storeutil.GetCurrentInstance(txn, dockerCli)
if err != nil {
return nil, err
}
@ -346,14 +253,20 @@ func getDefaultDrivers(ctx context.Context, dockerCli command.Cli, defaultOnly b
}
}
d, err := driver.GetDriver(ctx, "buildx_buildkit_default", nil, dockerCli.Client(), dockerCli.ConfigFile(), nil, nil, nil, nil, nil, contextPathHash)
imageopt, err := storeutil.GetImageConfig(dockerCli, nil)
if err != nil {
return nil, err
}
d, err := driver.GetDriver(ctx, "buildx_buildkit_default", nil, dockerCli.Client(), imageopt.Auth, nil, nil, nil, nil, nil, contextPathHash)
if err != nil {
return nil, err
}
return []build.DriverInfo{
{
Name: "default",
Driver: d,
Name: "default",
Driver: d,
ImageOpt: imageopt,
},
}, nil
}

166
store/storeutil/storeutil.go Normal file
View File

@ -0,0 +1,166 @@
package storeutil
import (
"bytes"
"os"
"strings"
"github.com/docker/buildx/store"
"github.com/docker/buildx/util/confutil"
"github.com/docker/buildx/util/imagetools"
"github.com/docker/buildx/util/resolver"
"github.com/docker/cli/cli/command"
"github.com/docker/cli/cli/context/docker"
buildkitdconfig "github.com/moby/buildkit/cmd/buildkitd/config"
"github.com/pkg/errors"
)
// GetStore returns current builder instance store
func GetStore(dockerCli command.Cli) (*store.Txn, func(), error) {
s, err := store.New(confutil.ConfigDir(dockerCli))
if err != nil {
return nil, nil, err
}
return s.Txn()
}
// GetCurrentEndpoint returns the current default endpoint value
func GetCurrentEndpoint(dockerCli command.Cli) (string, error) {
name := dockerCli.CurrentContext()
if name != "default" {
return name, nil
}
de, err := GetDockerEndpoint(dockerCli, name)
if err != nil {
return "", errors.Errorf("docker endpoint for %q not found", name)
}
return de, nil
}
// GetDockerEndpoint returns docker endpoint string for given context
func GetDockerEndpoint(dockerCli command.Cli, name string) (string, error) {
list, err := dockerCli.ContextStore().List()
if err != nil {
return "", err
}
for _, l := range list {
if l.Name == name {
ep, ok := l.Endpoints["docker"]
if !ok {
return "", errors.Errorf("context %q does not have a Docker endpoint", name)
}
typed, ok := ep.(docker.EndpointMeta)
if !ok {
return "", errors.Errorf("endpoint %q is not of type EndpointMeta, %T", ep, ep)
}
return typed.Host, nil
}
}
return "", nil
}
// GetCurrentInstance finds the current builder instance
func GetCurrentInstance(txn *store.Txn, dockerCli command.Cli) (*store.NodeGroup, error) {
ep, err := GetCurrentEndpoint(dockerCli)
if err != nil {
return nil, err
}
ng, err := txn.Current(ep)
if err != nil {
return nil, err
}
if ng == nil {
ng, _ = GetNodeGroup(txn, dockerCli, dockerCli.CurrentContext())
}
return ng, nil
}
// GetNodeGroup returns nodegroup based on the name
func GetNodeGroup(txn *store.Txn, dockerCli command.Cli, name string) (*store.NodeGroup, error) {
ng, err := txn.NodeGroupByName(name)
if err != nil {
if !os.IsNotExist(errors.Cause(err)) {
return nil, err
}
}
if ng != nil {
return ng, nil
}
if name == "default" {
name = dockerCli.CurrentContext()
}
list, err := dockerCli.ContextStore().List()
if err != nil {
return nil, err
}
for _, l := range list {
if l.Name == name {
return &store.NodeGroup{
Name: "default",
Nodes: []store.Node{
{
Name: "default",
Endpoint: name,
},
},
}, nil
}
}
return nil, errors.Errorf("no builder %q found", name)
}
func GetImageConfig(dockerCli command.Cli, ng *store.NodeGroup) (opt imagetools.Opt, err error) {
opt.Auth = dockerCli.ConfigFile()
if ng == nil || len(ng.Nodes) == 0 {
return opt, nil
}
files := ng.Nodes[0].Files
dt, ok := files["buildkitd.toml"]
if !ok {
return opt, nil
}
config, err := buildkitdconfig.Load(bytes.NewReader(dt))
if err != nil {
return opt, err
}
regconfig := make(map[string]resolver.RegistryConfig)
for k, v := range config.Registries {
rc := resolver.RegistryConfig{
Mirrors: v.Mirrors,
PlainHTTP: v.PlainHTTP,
Insecure: v.Insecure,
}
for _, ca := range v.RootCAs {
dt, ok := files[strings.TrimPrefix(ca, confutil.DefaultBuildKitConfigDir+"/")]
if ok {
rc.RootCAs = append(rc.RootCAs, dt)
}
}
for _, kp := range v.KeyPairs {
key, keyok := files[strings.TrimPrefix(kp.Key, confutil.DefaultBuildKitConfigDir+"/")]
cert, certok := files[strings.TrimPrefix(kp.Certificate, confutil.DefaultBuildKitConfigDir+"/")]
if keyok && certok {
rc.KeyPairs = append(rc.KeyPairs, resolver.TLSKeyPair{
Key: key,
Certificate: cert,
})
}
}
regconfig[k] = rc
}
opt.RegistryConfig = regconfig
return opt, nil
}

24
util/imagetools/inspect.go
View File

@ -10,8 +10,10 @@ import (
"github.com/containerd/containerd/remotes"
"github.com/containerd/containerd/remotes/docker"
"github.com/docker/buildx/util/resolver"
clitypes "github.com/docker/cli/cli/config/types"
"github.com/docker/distribution/reference"
"github.com/moby/buildkit/util/tracing"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
)
@ -20,23 +22,35 @@ type Auth interface {
}
type Opt struct {
Auth Auth
Auth Auth
RegistryConfig map[string]resolver.RegistryConfig
}
type Resolver struct {
auth docker.Authorizer
auth docker.Authorizer
hosts docker.RegistryHosts
}
func New(opt Opt) *Resolver {
return &Resolver{
auth: docker.NewDockerAuthorizer(docker.WithAuthCreds(toCredentialsFunc(opt.Auth)), docker.WithAuthClient(http.DefaultClient)),
auth: docker.NewDockerAuthorizer(docker.WithAuthCreds(toCredentialsFunc(opt.Auth)), docker.WithAuthClient(http.DefaultClient)),
hosts: resolver.NewRegistryConfig(opt.RegistryConfig),
}
}
func (r *Resolver) resolver() remotes.Resolver {
return docker.NewResolver(docker.ResolverOptions{
Authorizer: r.auth,
Client: http.DefaultClient,
Hosts: func(domain string) ([]docker.RegistryHost, error) {
res, err := r.hosts(domain)
if err != nil {
return nil, err
}
for i := range res {
res[i].Authorizer = r.auth
}
return res, nil
},
Client: tracing.DefaultClient,
})
}

188
util/resolver/resolver.go Normal file
View File

@ -0,0 +1,188 @@
package resolver
import (
"crypto/tls"
"crypto/x509"
"net"
"net/http"
"runtime"
"time"
"github.com/containerd/containerd/remotes/docker"
"github.com/moby/buildkit/util/tracing"
"github.com/pkg/errors"
)
// TODO: copied from buildkit/util/resolver. Update upstream so we can use the same code.
type RegistryConfig struct {
Mirrors []string
PlainHTTP *bool
Insecure *bool
RootCAs [][]byte
KeyPairs []TLSKeyPair
}
type TLSKeyPair struct {
Key []byte
Certificate []byte
}
func fillInsecureOpts(host string, c RegistryConfig, h docker.RegistryHost) ([]docker.RegistryHost, error) {
var hosts []docker.RegistryHost
tc, err := loadTLSConfig(c)
if err != nil {
return nil, err
}
var isHTTP bool
if c.PlainHTTP != nil && *c.PlainHTTP {
isHTTP = true
}
if c.PlainHTTP == nil {
if ok, _ := docker.MatchLocalhost(host); ok {
isHTTP = true
}
}
if isHTTP {
h2 := h
h2.Scheme = "http"
hosts = append(hosts, h2)
}
if c.Insecure != nil && *c.Insecure {
h2 := h
transport := newDefaultTransport()
transport.TLSClientConfig = tc
h2.Client = &http.Client{
Transport: tracing.NewTransport(transport),
}
tc.InsecureSkipVerify = true
hosts = append(hosts, h2)
}
if len(hosts) == 0 {
transport := newDefaultTransport()
transport.TLSClientConfig = tc
h.Client = &http.Client{
Transport: tracing.NewTransport(transport),
}
hosts = append(hosts, h)
}
return hosts, nil
}
func loadTLSConfig(c RegistryConfig) (*tls.Config, error) {
tc := &tls.Config{}
if len(c.RootCAs) > 0 {
systemPool, err := x509.SystemCertPool()
if err != nil {
if runtime.GOOS == "windows" {
systemPool = x509.NewCertPool()
} else {
return nil, errors.Wrapf(err, "unable to get system cert pool")
}
}
tc.RootCAs = systemPool
}
for _, p := range c.RootCAs {
tc.RootCAs.AppendCertsFromPEM(p)
}
for _, kp := range c.KeyPairs {
cert, err := tls.X509KeyPair(kp.Certificate, kp.Key)
if err != nil {
return nil, errors.Wrapf(err, "failed to load keypair for %s", kp.Certificate)
}
tc.Certificates = append(tc.Certificates, cert)
}
return tc, nil
}
// NewRegistryConfig converts registry config to docker.RegistryHosts callback
func NewRegistryConfig(m map[string]RegistryConfig) docker.RegistryHosts {
return docker.Registries(
func(host string) ([]docker.RegistryHost, error) {
c, ok := m[host]
if !ok {
return nil, nil
}
var out []docker.RegistryHost
for _, mirror := range c.Mirrors {
h := docker.RegistryHost{
Scheme: "https",
Client: newDefaultClient(),
Host: mirror,
Path: "/v2",
Capabilities: docker.HostCapabilityPull | docker.HostCapabilityResolve,
}
hosts, err := fillInsecureOpts(mirror, m[mirror], h)
if err != nil {
return nil, err
}
out = append(out, hosts...)
}
if host == "docker.io" {
host = "registry-1.docker.io"
}
h := docker.RegistryHost{
Scheme: "https",
Client: newDefaultClient(),
Host: host,
Path: "/v2",
Capabilities: docker.HostCapabilityPush | docker.HostCapabilityPull | docker.HostCapabilityResolve,
}
hosts, err := fillInsecureOpts(host, c, h)
if err != nil {
return nil, err
}
out = append(out, hosts...)
return out, nil
},
docker.ConfigureDefaultRegistries(
docker.WithClient(newDefaultClient()),
docker.WithPlainHTTP(docker.MatchLocalhost),
),
)
}
func newDefaultClient() *http.Client {
return &http.Client{
Transport: tracing.NewTransport(newDefaultTransport()),
}
}
// newDefaultTransport is for pull or push client
//
// NOTE: For push, there must disable http2 for https because the flow control
// will limit data transfer. The net/http package doesn't provide http2 tunable
// settings which limits push performance.
//
// REF: https://github.com/golang/go/issues/14077
func newDefaultTransport() *http.Transport {
return &http.Transport{
Proxy: http.ProxyFromEnvironment,
DialContext: (&net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 60 * time.Second,
}).DialContext,
MaxIdleConns: 30,
IdleConnTimeout: 120 * time.Second,
MaxIdleConnsPerHost: 4,
TLSHandshakeTimeout: 10 * time.Second,
ExpectContinueTimeout: 5 * time.Second,
TLSNextProto: make(map[string]func(authority string, c *tls.Conn) http.RoundTripper),
}
}

115
vendor/github.com/moby/buildkit/cmd/buildkitd/config/config.go generated vendored Normal file
View File

@ -0,0 +1,115 @@
package config
import (
resolverconfig "github.com/moby/buildkit/util/resolver/config"
)
// Config provides containerd configuration data for the server
type Config struct {
Debug bool `toml:"debug"`
// Root is the path to a directory where buildkit will store persistent data
Root string `toml:"root"`
// Entitlements e.g. security.insecure, network.host
Entitlements []string `toml:"insecure-entitlements"`
// GRPC configuration settings
GRPC GRPCConfig `toml:"grpc"`
Workers struct {
OCI OCIConfig `toml:"oci"`
Containerd ContainerdConfig `toml:"containerd"`
} `toml:"worker"`
Registries map[string]resolverconfig.RegistryConfig `toml:"registry"`
DNS *DNSConfig `toml:"dns"`
}
type GRPCConfig struct {
Address []string `toml:"address"`
DebugAddress string `toml:"debugAddress"`
UID *int `toml:"uid"`
GID *int `toml:"gid"`
TLS TLSConfig `toml:"tls"`
// MaxRecvMsgSize int `toml:"max_recv_message_size"`
// MaxSendMsgSize int `toml:"max_send_message_size"`
}
type TLSConfig struct {
Cert string `toml:"cert"`
Key string `toml:"key"`
CA string `toml:"ca"`
}
type GCConfig struct {
GC *bool `toml:"gc"`
GCKeepStorage int64 `toml:"gckeepstorage"`
GCPolicy []GCPolicy `toml:"gcpolicy"`
}
type NetworkConfig struct {
Mode string `toml:"networkMode"`
CNIConfigPath string `toml:"cniConfigPath"`
CNIBinaryPath string `toml:"cniBinaryPath"`
}
type OCIConfig struct {
Enabled *bool `toml:"enabled"`
Labels map[string]string `toml:"labels"`
Platforms []string `toml:"platforms"`
Snapshotter string `toml:"snapshotter"`
Rootless bool `toml:"rootless"`
NoProcessSandbox bool `toml:"noProcessSandbox"`
GCConfig
NetworkConfig
// UserRemapUnsupported is unsupported key for testing. The feature is
// incomplete and the intention is to make it default without config.
UserRemapUnsupported string `toml:"userRemapUnsupported"`
// For use in storing the OCI worker binary name that will replace buildkit-runc
Binary string `toml:"binary"`
ProxySnapshotterPath string `toml:"proxySnapshotterPath"`
// StargzSnapshotterConfig is configuration for stargz snapshotter.
// We use a generic map[string]interface{} in order to remove the dependency
// on stargz snapshotter's config pkg from our config.
StargzSnapshotterConfig map[string]interface{} `toml:"stargzSnapshotter"`
// ApparmorProfile is the name of the apparmor profile that should be used to constrain build containers.
// The profile should already be loaded (by a higher level system) before creating a worker.
ApparmorProfile string `toml:"apparmor-profile"`
// MaxParallelism is the maximum number of parallel build steps that can be run at the same time.
MaxParallelism int `toml:"max-parallelism"`
}
type ContainerdConfig struct {
Address string `toml:"address"`
Enabled *bool `toml:"enabled"`
Labels map[string]string `toml:"labels"`
Platforms []string `toml:"platforms"`
Namespace string `toml:"namespace"`
GCConfig
NetworkConfig
Snapshotter string `toml:"snapshotter"`
// ApparmorProfile is the name of the apparmor profile that should be used to constrain build containers.
// The profile should already be loaded (by a higher level system) before creating a worker.
ApparmorProfile string `toml:"apparmor-profile"`
MaxParallelism int `toml:"max-parallelism"`
}
type GCPolicy struct {
All bool `toml:"all"`
KeepBytes int64 `toml:"keepBytes"`
KeepDuration int64 `toml:"keepDuration"`
Filters []string `toml:"filters"`
}
type DNSConfig struct {
Nameservers []string `toml:"nameservers"`
Options []string `toml:"options"`
SearchDomains []string `toml:"searchDomains"`
}

31
vendor/github.com/moby/buildkit/cmd/buildkitd/config/gcpolicy.go generated vendored Normal file
View File

@ -0,0 +1,31 @@
package config
const defaultCap int64 = 2e9 // 2GB
func DefaultGCPolicy(p string, keep int64) []GCPolicy {
if keep == 0 {
keep = DetectDefaultGCCap(p)
}
return []GCPolicy{
// if build cache uses more than 512MB delete the most easily reproducible data after it has not been used for 2 days
{
Filters: []string{"type==source.local,type==exec.cachemount,type==source.git.checkout"},
KeepDuration: 48 * 3600, // 48h
KeepBytes: 512 * 1e6, // 512MB
},
// remove any data not used for 60 days
{
KeepDuration: 60 * 24 * 3600, // 60d
KeepBytes: keep,
},
// keep the unshared build cache under cap
{
KeepBytes: keep,
},
// if previous policies were insufficient start deleting internal data to keep build cache under cap
{
All: true,
KeepBytes: keep,
},
}
}

17
vendor/github.com/moby/buildkit/cmd/buildkitd/config/gcpolicy_unix.go generated vendored Normal file
View File

@ -0,0 +1,17 @@
// +build !windows
package config
import (
"syscall"
)
func DetectDefaultGCCap(root string) int64 {
var st syscall.Statfs_t
if err := syscall.Statfs(root, &st); err != nil {
return defaultCap
}
diskSize := int64(st.Bsize) * int64(st.Blocks)
avail := diskSize / 10
return (avail/(1<<30) + 1) * 1e9 // round up
}

7
vendor/github.com/moby/buildkit/cmd/buildkitd/config/gcpolicy_windows.go generated vendored Normal file
View File

@ -0,0 +1,7 @@
// +build windows
package config
func DetectDefaultGCCap(root string) int64 {
return defaultCap
}

36
vendor/github.com/moby/buildkit/cmd/buildkitd/config/load.go generated vendored Normal file
View File

@ -0,0 +1,36 @@
package config
import (
"io"
"os"
"github.com/pelletier/go-toml"
"github.com/pkg/errors"
)
// Load loads buildkitd config
func Load(r io.Reader) (Config, error) {
var c Config
t, err := toml.LoadReader(r)
if err != nil {
return c, errors.Wrap(err, "failed to parse config")
}
err = t.Unmarshal(&c)
if err != nil {
return c, errors.Wrap(err, "failed to parse config")
}
return c, nil
}
// LoadFile loads buildkitd config file
func LoadFile(fp string) (Config, error) {
f, err := os.Open(fp)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return Config{}, nil
}
return Config{}, errors.Wrapf(err, "failed to load config from %s", fp)
}
defer f.Close()
return Load(f)
}

15
vendor/github.com/moby/buildkit/util/resolver/config/config.go generated vendored Normal file
View File

@ -0,0 +1,15 @@
package config
type RegistryConfig struct {
Mirrors []string `toml:"mirrors"`
PlainHTTP *bool `toml:"http"`
Insecure *bool `toml:"insecure"`
RootCAs []string `toml:"ca"`
KeyPairs []TLSKeyPair `toml:"keypair"`
TLSConfigDir []string `toml:"tlsconfigdir"`
}
type TLSKeyPair struct {
Key string `toml:"key"`
Certificate string `toml:"cert"`
}

2
vendor/modules.txt vendored
View File

@ -295,6 +295,7 @@ github.com/moby/buildkit/client/buildid
github.com/moby/buildkit/client/connhelper
github.com/moby/buildkit/client/llb
github.com/moby/buildkit/client/ociindex
github.com/moby/buildkit/cmd/buildkitd/config
github.com/moby/buildkit/frontend/gateway/client
github.com/moby/buildkit/frontend/gateway/grpcclient
github.com/moby/buildkit/frontend/gateway/pb
@ -325,6 +326,7 @@ github.com/moby/buildkit/util/grpcerrors
github.com/moby/buildkit/util/progress
github.com/moby/buildkit/util/progress/progressui
github.com/moby/buildkit/util/progress/progresswriter
github.com/moby/buildkit/util/resolver/config
github.com/moby/buildkit/util/sshutil
github.com/moby/buildkit/util/stack
github.com/moby/buildkit/util/system