build: basis of build command

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>
This commit is contained in:
Tonis Tiigi 2019-03-23 21:30:29 -07:00
parent 8b7c38e61a
commit 4b0c0468d0
10 changed files with 620 additions and 88 deletions

View File

@ -53,12 +53,19 @@ COPY --from=buildx-build /usr/bin/buildx /buildx.exe
FROM binaries-$TARGETOS AS binaries
FROM alpine AS demo-env
RUN apk add --no-cache iptables tmux
RUN apk add --no-cache iptables tmux git
RUN mkdir -p /usr/local/lib/docker/cli-plugins && ln -s /usr/local/bin/buildx /usr/local/lib/docker/cli-plugins/docker-buildx
COPY ./hack/demo-env/entrypoint.sh /usr/local/bin
COPY ./hack/demo-env/tmux.conf /root/.tmux.conf
COPY --from=dockerd-release /usr/local/bin /usr/local/bin
COPY --from=docker-cli-build /go/src/github.com/docker/cli/build/docker /usr/local/bin
# Temporary buildkitd binaries. To be removed.
COPY --from=moby/buildkit /usr/bin/build* /usr/local/bin
VOLUME /var/lib/buildkit
WORKDIR /work
COPY ./hack/demo-env/examples .
COPY --from=binaries / /usr/local/bin/
VOLUME /var/lib/docker
ENTRYPOINT ["entrypoint.sh"]

193
build/build.go Normal file
View File

@ -0,0 +1,193 @@
package build
import (
"context"
"io"
"os"
"path/filepath"
"strconv"
"strings"
"github.com/containerd/console"
"github.com/containerd/containerd/platforms"
"github.com/moby/buildkit/client"
"github.com/moby/buildkit/session"
"github.com/moby/buildkit/util/progress/progressui"
specs "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
"golang.org/x/sync/errgroup"
)
type Options struct {
Inputs Inputs
Tags []string
Labels map[string]string
BuildArgs map[string]string
Pull bool
NoCache bool
Target string
Platforms []specs.Platform
Exports []client.ExportEntry
Session []session.Attachable
// DockerTarget
}
type Inputs struct {
ContextPath string
DockerfilePath string
InStream io.Reader
}
func Build(ctx context.Context, c *client.Client, opt Options, pw *ProgressWriter) (*client.SolveResponse, error) {
so := client.SolveOpt{
Frontend: "dockerfile.v0",
FrontendAttrs: map[string]string{},
}
if len(opt.Exports) > 1 {
return nil, errors.Errorf("multiple outputs currently unsupported")
}
if len(opt.Tags) > 0 {
for i, e := range opt.Exports {
switch e.Type {
case "image", "oci", "docker":
opt.Exports[i].Attrs["name"] = strings.Join(opt.Tags, ",")
}
}
} else {
for _, e := range opt.Exports {
if e.Type == "image" && e.Attrs["name"] == "" && e.Attrs["push"] != "" {
if ok, _ := strconv.ParseBool(e.Attrs["push"]); ok {
return nil, errors.Errorf("tag is needed when pushing to registry")
}
}
}
}
// TODO: handle loading to docker daemon
so.Exports = opt.Exports
so.Session = opt.Session
if err := LoadInputs(opt.Inputs, &so); err != nil {
return nil, err
}
if opt.Pull {
so.FrontendAttrs["image-resolve-mode"] = "pull"
}
if opt.Target != "" {
so.FrontendAttrs["target"] = opt.Target
}
if opt.NoCache {
so.FrontendAttrs["no-cache"] = ""
}
for k, v := range opt.BuildArgs {
so.FrontendAttrs["build-arg:"+k] = v
}
for k, v := range opt.Labels {
so.FrontendAttrs["label:"+k] = v
}
if len(opt.Platforms) != 0 {
pp := make([]string, len(opt.Platforms))
for i, p := range opt.Platforms {
pp[i] = platforms.Format(p)
}
so.FrontendAttrs["platform"] = strings.Join(pp, ",")
}
eg, ctx := errgroup.WithContext(ctx)
var statusCh chan *client.SolveStatus
if pw != nil {
statusCh = pw.Status()
eg.Go(func() error {
<-pw.Done()
return pw.Err()
})
}
var resp *client.SolveResponse
eg.Go(func() error {
var err error
resp, err = c.Solve(ctx, nil, so, statusCh)
if err != nil {
return err
}
return nil
})
if err := eg.Wait(); err != nil {
return nil, err
}
return resp, nil
}
type ProgressWriter struct {
status chan *client.SolveStatus
done <-chan struct{}
err error
}
func (pw *ProgressWriter) Done() <-chan struct{} {
return pw.done
}
func (pw *ProgressWriter) Err() error {
return pw.err
}
func (pw *ProgressWriter) Status() chan *client.SolveStatus {
return pw.status
}
func NewProgressWriter(ctx context.Context, out *os.File, mode string) *ProgressWriter {
statusCh := make(chan *client.SolveStatus)
doneCh := make(chan struct{})
pw := &ProgressWriter{
status: statusCh,
done: doneCh,
}
go func() {
var c console.Console
if cons, err := console.ConsoleFromFile(out); err == nil && (mode == "auto" || mode == "tty") {
c = cons
}
// not using shared context to not disrupt display but let is finish reporting errors
pw.err = progressui.DisplaySolveStatus(ctx, "", c, out, statusCh)
close(doneCh)
}()
return pw
}
func LoadInputs(inp Inputs, target *client.SolveOpt) error {
if inp.ContextPath == "" {
return errors.New("please specify build context (e.g. \".\" for the current directory)")
}
// TODO: handle stdin, symlinks, remote contexts, check files exist
if inp.DockerfilePath == "" {
inp.DockerfilePath = filepath.Join(inp.ContextPath, "Dockerfile")
}
if target.LocalDirs == nil {
target.LocalDirs = map[string]string{}
}
target.LocalDirs["context"] = inp.ContextPath
target.LocalDirs["dockerfile"] = filepath.Dir(inp.DockerfilePath)
if target.FrontendAttrs == nil {
target.FrontendAttrs = map[string]string{}
}
target.FrontendAttrs["filename"] = filepath.Base(inp.DockerfilePath)
return nil
}

86
build/output.go Normal file
View File

@ -0,0 +1,86 @@
package build
import (
"encoding/csv"
"os"
"strings"
"github.com/moby/buildkit/client"
"github.com/pkg/errors"
)
func ParseOutputs(inp []string) ([]client.ExportEntry, error) {
var outs []client.ExportEntry
if len(inp) == 0 {
return nil, nil
}
for _, s := range inp {
csvReader := csv.NewReader(strings.NewReader(s))
fields, err := csvReader.Read()
if err != nil {
return nil, err
}
if len(fields) == 1 && fields[0] == s {
outs = append(outs, client.ExportEntry{
Type: "local",
OutputDir: s,
})
continue
}
out := client.ExportEntry{
Attrs: map[string]string{},
}
for _, field := range fields {
parts := strings.SplitN(field, "=", 2)
if len(parts) != 2 {
return nil, errors.Errorf("invalid value %s", field)
}
key := strings.ToLower(parts[0])
value := parts[1]
switch key {
case "type":
out.Type = value
default:
out.Attrs[key] = value
}
}
if out.Type == "" {
return nil, errors.Errorf("type is required for output")
}
// handle client side
switch out.Type {
case "local":
dest, ok := out.Attrs["dest"]
if !ok {
return nil, errors.Errorf("dest is required for local output")
}
out.OutputDir = dest
delete(out.Attrs, "dest")
case "oci", "dest":
dest, ok := out.Attrs["dest"]
if !ok {
if out.Type != "docker" {
return nil, errors.Errorf("dest is required for %s output", out.Type)
}
} else {
if dest == "-" {
out.Output = os.Stdout
} else {
f, err := os.Open(dest)
if err != nil {
out.Output = f
}
}
delete(out.Attrs, "dest")
}
case "registry":
out.Type = "iamge"
out.Attrs["push"] = "true"
}
outs = append(outs, out)
}
return outs, nil
}

32
build/platform.go Normal file
View File

@ -0,0 +1,32 @@
package build
import (
"strings"
"github.com/containerd/containerd/platforms"
specs "github.com/opencontainers/image-spec/specs-go/v1"
)
func ParsePlatformSpecs(platformsStr []string) ([]specs.Platform, error) {
if len(platformsStr) == 0 {
return nil, nil
}
out := make([]specs.Platform, 0, len(platformsStr))
for _, s := range platformsStr {
parts := strings.Split(s, ",")
if len(parts) > 1 {
p, err := ParsePlatformSpecs(parts)
if err != nil {
return nil, err
}
out = append(out, p...)
continue
}
p, err := platforms.Parse(s)
if err != nil {
return nil, err
}
out = append(out, platforms.Normalize(p))
}
return out, nil
}

60
build/secrets.go Normal file
View File

@ -0,0 +1,60 @@
package build
import (
"encoding/csv"
"strings"
"github.com/moby/buildkit/session"
"github.com/moby/buildkit/session/secrets/secretsprovider"
"github.com/pkg/errors"
)
func ParseSecretSpecs(sl []string) (session.Attachable, error) {
fs := make([]secretsprovider.FileSource, 0, len(sl))
for _, v := range sl {
s, err := parseSecret(v)
if err != nil {
return nil, err
}
fs = append(fs, *s)
}
store, err := secretsprovider.NewFileStore(fs)
if err != nil {
return nil, err
}
return secretsprovider.NewSecretProvider(store), nil
}
func parseSecret(value string) (*secretsprovider.FileSource, error) {
csvReader := csv.NewReader(strings.NewReader(value))
fields, err := csvReader.Read()
if err != nil {
return nil, errors.Wrap(err, "failed to parse csv secret")
}
fs := secretsprovider.FileSource{}
for _, field := range fields {
parts := strings.SplitN(field, "=", 2)
key := strings.ToLower(parts[0])
if len(parts) != 2 {
return nil, errors.Errorf("invalid field '%s' must be a key=value pair", field)
}
value := parts[1]
switch key {
case "type":
if value != "file" {
return nil, errors.Errorf("unsupported secret type %q", value)
}
case "id":
fs.ID = value
case "source", "src":
fs.FilePath = value
default:
return nil, errors.Errorf("unexpected key '%s' in '%s'", key, field)
}
}
return &fs, nil
}

31
build/ssh.go Normal file
View File

@ -0,0 +1,31 @@
package build
import (
"strings"
"github.com/moby/buildkit/session"
"github.com/moby/buildkit/session/sshforward/sshprovider"
)
func ParseSSHSpecs(sl []string) (session.Attachable, error) {
configs := make([]sshprovider.AgentConfig, 0, len(sl))
for _, v := range sl {
c, err := parseSSH(v)
if err != nil {
return nil, err
}
configs = append(configs, *c)
}
return sshprovider.NewSSHAgentProvider(configs)
}
func parseSSH(value string) (*sshprovider.AgentConfig, error) {
parts := strings.SplitN(value, "=", 2)
cfg := sshprovider.AgentConfig{
ID: parts[0],
}
if len(parts) > 1 {
cfg.Paths = strings.Split(parts[1], ",")
}
return &cfg, nil
}

View File

@ -1,102 +1,17 @@
package main
import (
"context"
"fmt"
"os"
"github.com/docker/cli/cli-plugins/manager"
"github.com/docker/cli/cli-plugins/plugin"
"github.com/docker/cli/cli/command"
"github.com/spf13/cobra"
"github.com/tonistiigi/buildx/commands"
"github.com/tonistiigi/buildx/version"
)
func main() {
plugin.Run(func(dockerCli command.Cli) *cobra.Command {
goodbye := &cobra.Command{
Use: "goodbye",
Short: "Say Goodbye instead of Hello",
Run: func(cmd *cobra.Command, _ []string) {
fmt.Fprintln(dockerCli.Out(), "Goodbye World!")
},
}
apiversion := &cobra.Command{
Use: "apiversion",
Short: "Print the API version of the server",
RunE: func(_ *cobra.Command, _ []string) error {
cli := dockerCli.Client()
ping, err := cli.Ping(context.Background())
if err != nil {
return err
}
fmt.Println(ping.APIVersion)
return nil
},
}
exitStatus2 := &cobra.Command{
Use: "exitstatus2",
Short: "Exit with status 2",
RunE: func(_ *cobra.Command, _ []string) error {
fmt.Fprintln(dockerCli.Err(), "Exiting with error status 2")
os.Exit(2)
return nil
},
}
var (
who, context string
preRun, debug bool
)
cmd := &cobra.Command{
Use: "helloworld",
Short: "A basic Hello World plugin for tests",
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := plugin.PersistentPreRunE(cmd, args); err != nil {
return err
}
if preRun {
fmt.Fprintf(dockerCli.Err(), "Plugin PersistentPreRunE called")
}
return nil
},
RunE: func(cmd *cobra.Command, args []string) error {
if debug {
fmt.Fprintf(dockerCli.Err(), "Plugin debug mode enabled")
}
switch context {
case "Christmas":
fmt.Fprintf(dockerCli.Out(), "Merry Christmas!\n")
return nil
case "":
// nothing
}
if who == "" {
who, _ = dockerCli.ConfigFile().PluginConfig("helloworld", "who")
}
if who == "" {
who = "World"
}
fmt.Fprintf(dockerCli.Out(), "Hello %s!\n", who)
dockerCli.ConfigFile().SetPluginConfig("helloworld", "lastwho", who)
return dockerCli.ConfigFile().Save()
},
}
flags := cmd.Flags()
flags.StringVar(&who, "who", "", "Who are we addressing?")
flags.BoolVar(&preRun, "pre-run", false, "Log from prerun hook")
// These are intended to deliberately clash with the CLIs own top
// level arguments.
flags.BoolVarP(&debug, "debug", "D", false, "Enable debug")
flags.StringVarP(&context, "context", "c", "", "Is it Christmas?")
cmd.AddCommand(goodbye, apiversion, exitStatus2)
return cmd
return commands.NewRootCmd(dockerCli)
},
manager.Metadata{
SchemaVersion: "0.1.0",

185
commands/build.go Normal file
View File

@ -0,0 +1,185 @@
package commands
import (
"context"
"os"
"strings"
"github.com/docker/cli/cli"
"github.com/docker/cli/cli/command"
"github.com/moby/buildkit/client"
"github.com/moby/buildkit/session/auth/authprovider"
"github.com/moby/buildkit/util/appcontext"
bkappdefaults "github.com/moby/buildkit/util/appdefaults"
"github.com/spf13/cobra"
"github.com/tonistiigi/buildx/build"
)
type buildOptions struct {
contextPath string
dockerfileName string
tags []string
labels []string
buildArgs []string
// extraHosts opts.ListOpts
// ulimits *opts.UlimitOpt
// memory opts.MemBytes
// memorySwap opts.MemSwapBytes
// shmSize opts.MemBytes
// cpuShares int64
// cpuPeriod int64
// cpuQuota int64
// cpuSetCpus string
// cpuSetMems string
// cgroupParent string
// isolation string
// quiet bool
noCache bool
progress string
pull bool
cacheFrom []string
// compress bool
// securityOpt []string
// networkMode string
// squash bool
target string
// imageIDFile string
platforms []string
// untrusted bool
secrets []string
ssh []string
outputs []string
}
func runBuild(dockerCli command.Cli, in buildOptions) error {
ctx := appcontext.Context()
opts := build.Options{
Inputs: build.Inputs{
ContextPath: in.contextPath,
DockerfilePath: in.dockerfileName,
InStream: os.Stdin,
},
Tags: in.tags,
Labels: listToMap(in.labels),
BuildArgs: listToMap(in.buildArgs),
Pull: in.pull,
NoCache: in.noCache,
Target: in.target,
}
platforms, err := build.ParsePlatformSpecs(in.platforms)
if err != nil {
return err
}
opts.Platforms = platforms
opts.Session = append(opts.Session, authprovider.NewDockerAuthProvider())
secrets, err := build.ParseSecretSpecs(in.secrets)
if err != nil {
return err
}
opts.Session = append(opts.Session, secrets)
ssh, err := build.ParseSSHSpecs(in.ssh)
if err != nil {
return err
}
opts.Session = append(opts.Session, ssh)
outputs, err := build.ParseOutputs(in.outputs)
if err != nil {
return err
}
opts.Exports = outputs
// TODO: temporary
c, err := client.New(ctx, bkappdefaults.Address, client.WithFailFast())
if err != nil {
return err
}
ctx2, cancel := context.WithCancel(context.TODO())
defer cancel()
pw := build.NewProgressWriter(ctx2, os.Stderr, in.progress)
_, err = build.Build(ctx, c, opts, pw)
return err
}
func buildCmd(dockerCli command.Cli) *cobra.Command {
var options buildOptions
cmd := &cobra.Command{
Use: "build [OPTIONS] PATH | URL | -",
Aliases: []string{"b"},
Short: "Start a build",
Args: cli.ExactArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
options.contextPath = args[0]
return runBuild(dockerCli, options)
},
}
flags := cmd.Flags()
flags.StringArrayVarP(&options.tags, "tag", "t", []string{}, "Name and optionally a tag in the 'name:tag' format")
flags.StringArrayVar(&options.buildArgs, "build-arg", []string{}, "Set build-time variables")
// flags.Var(options.ulimits, "ulimit", "Ulimit options")
flags.StringVarP(&options.dockerfileName, "file", "f", "", "Name of the Dockerfile (Default is 'PATH/Dockerfile')")
// flags.VarP(&options.memory, "memory", "m", "Memory limit")
// flags.Var(&options.memorySwap, "memory-swap", "Swap limit equal to memory plus swap: '-1' to enable unlimited swap")
// flags.Var(&options.shmSize, "shm-size", "Size of /dev/shm")
// flags.Int64VarP(&options.cpuShares, "cpu-shares", "c", 0, "CPU shares (relative weight)")
// flags.Int64Var(&options.cpuPeriod, "cpu-period", 0, "Limit the CPU CFS (Completely Fair Scheduler) period")
// flags.Int64Var(&options.cpuQuota, "cpu-quota", 0, "Limit the CPU CFS (Completely Fair Scheduler) quota")
// flags.StringVar(&options.cpuSetCpus, "cpuset-cpus", "", "CPUs in which to allow execution (0-3, 0,1)")
// flags.StringVar(&options.cpuSetMems, "cpuset-mems", "", "MEMs in which to allow execution (0-3, 0,1)")
// flags.StringVar(&options.cgroupParent, "cgroup-parent", "", "Optional parent cgroup for the container")
// flags.StringVar(&options.isolation, "isolation", "", "Container isolation technology")
flags.StringArrayVar(&options.labels, "label", []string{}, "Set metadata for an image")
flags.BoolVar(&options.noCache, "no-cache", false, "Do not use cache when building the image")
// flags.BoolVarP(&options.quiet, "quiet", "q", false, "Suppress the build output and print image ID on success")
flags.BoolVar(&options.pull, "pull", false, "Always attempt to pull a newer version of the image")
flags.StringSliceVar(&options.cacheFrom, "cache-from", []string{}, "Images to consider as cache sources")
// flags.BoolVar(&options.compress, "compress", false, "Compress the build context using gzip")
// flags.StringSliceVar(&options.securityOpt, "security-opt", []string{}, "Security options")
// flags.StringVar(&options.networkMode, "network", "default", "Set the networking mode for the RUN instructions during build")
// flags.Var(&options.extraHosts, "add-host", "Add a custom host-to-IP mapping (host:ip)")
flags.StringVar(&options.target, "target", "", "Set the target build stage to build.")
// flags.StringVar(&options.imageIDFile, "iidfile", "", "Write the image ID to the file")
platformsDefault := []string{}
if v := os.Getenv("DOCKER_DEFAULT_PLATFORM"); v != "" {
platformsDefault = []string{v}
}
flags.StringArrayVar(&options.platforms, "platform", platformsDefault, "Set target platform for build")
// flags.BoolVar(&options.squash, "squash", false, "Squash newly built layers into a single new layer")
flags.StringVar(&options.progress, "progress", "auto", "Set type of progress output (auto, plain, tty). Use plain to show container output")
flags.StringArrayVar(&options.secrets, "secret", []string{}, "Secret file to expose to the build: id=mysecret,src=/local/secret")
flags.StringArrayVar(&options.ssh, "ssh", []string{}, "SSH agent socket or keys to expose to the build (format: default|<id>[=<socket>|<key>[,<key>]])")
flags.StringArrayVarP(&options.outputs, "output", "o", []string{}, "Output destination (format: type=local,dest=path)")
return cmd
}
func listToMap(values []string) map[string]string {
result := make(map[string]string, len(values))
for _, value := range values {
kv := strings.SplitN(value, "=", 2)
if len(kv) == 1 {
result[kv[0]] = ""
} else {
result[kv[0]] = kv[1]
}
}
return result
}

21
commands/root.go Normal file
View File

@ -0,0 +1,21 @@
package commands
import (
"github.com/docker/cli/cli/command"
"github.com/spf13/cobra"
)
func NewRootCmd(dockerCli command.Cli) *cobra.Command {
cmd := &cobra.Command{
Short: "Build with BuildKit",
Use: "buildx",
}
addCommands(cmd, dockerCli)
return cmd
}
func addCommands(cmd *cobra.Command, dockerCli command.Cli) {
cmd.AddCommand(
buildCmd(dockerCli),
)
}

View File

@ -0,0 +1,2 @@
FROM alpine
RUN touch foo