diff --git a/Dockerfile b/Dockerfile index e59518b5..510cd458 100644 --- a/Dockerfile +++ b/Dockerfile @@ -53,12 +53,19 @@ COPY --from=buildx-build /usr/bin/buildx /buildx.exe FROM binaries-$TARGETOS AS binaries FROM alpine AS demo-env -RUN apk add --no-cache iptables tmux +RUN apk add --no-cache iptables tmux git RUN mkdir -p /usr/local/lib/docker/cli-plugins && ln -s /usr/local/bin/buildx /usr/local/lib/docker/cli-plugins/docker-buildx COPY ./hack/demo-env/entrypoint.sh /usr/local/bin COPY ./hack/demo-env/tmux.conf /root/.tmux.conf COPY --from=dockerd-release /usr/local/bin /usr/local/bin COPY --from=docker-cli-build /go/src/github.com/docker/cli/build/docker /usr/local/bin + +# Temporary buildkitd binaries. To be removed. +COPY --from=moby/buildkit /usr/bin/build* /usr/local/bin +VOLUME /var/lib/buildkit + +WORKDIR /work +COPY ./hack/demo-env/examples . COPY --from=binaries / /usr/local/bin/ VOLUME /var/lib/docker ENTRYPOINT ["entrypoint.sh"] diff --git a/build/build.go b/build/build.go new file mode 100644 index 00000000..90bb31d7 --- /dev/null +++ b/build/build.go @@ -0,0 +1,193 @@ +package build + +import ( + "context" + "io" + "os" + "path/filepath" + "strconv" + "strings" + + "github.com/containerd/console" + "github.com/containerd/containerd/platforms" + "github.com/moby/buildkit/client" + "github.com/moby/buildkit/session" + "github.com/moby/buildkit/util/progress/progressui" + specs "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" + "golang.org/x/sync/errgroup" +) + +type Options struct { + Inputs Inputs + Tags []string + Labels map[string]string + BuildArgs map[string]string + Pull bool + + NoCache bool + Target string + Platforms []specs.Platform + Exports []client.ExportEntry + Session []session.Attachable + + // DockerTarget +} + +type Inputs struct { + ContextPath string + DockerfilePath string + InStream io.Reader +} + +func Build(ctx context.Context, c *client.Client, opt Options, pw *ProgressWriter) (*client.SolveResponse, error) { + so := client.SolveOpt{ + Frontend: "dockerfile.v0", + FrontendAttrs: map[string]string{}, + } + + if len(opt.Exports) > 1 { + return nil, errors.Errorf("multiple outputs currently unsupported") + } + + if len(opt.Tags) > 0 { + for i, e := range opt.Exports { + switch e.Type { + case "image", "oci", "docker": + opt.Exports[i].Attrs["name"] = strings.Join(opt.Tags, ",") + } + } + } else { + for _, e := range opt.Exports { + if e.Type == "image" && e.Attrs["name"] == "" && e.Attrs["push"] != "" { + if ok, _ := strconv.ParseBool(e.Attrs["push"]); ok { + return nil, errors.Errorf("tag is needed when pushing to registry") + } + } + } + } + // TODO: handle loading to docker daemon + + so.Exports = opt.Exports + so.Session = opt.Session + + if err := LoadInputs(opt.Inputs, &so); err != nil { + return nil, err + } + + if opt.Pull { + so.FrontendAttrs["image-resolve-mode"] = "pull" + } + if opt.Target != "" { + so.FrontendAttrs["target"] = opt.Target + } + if opt.NoCache { + so.FrontendAttrs["no-cache"] = "" + } + for k, v := range opt.BuildArgs { + so.FrontendAttrs["build-arg:"+k] = v + } + for k, v := range opt.Labels { + so.FrontendAttrs["label:"+k] = v + } + + if len(opt.Platforms) != 0 { + pp := make([]string, len(opt.Platforms)) + for i, p := range opt.Platforms { + pp[i] = platforms.Format(p) + } + so.FrontendAttrs["platform"] = strings.Join(pp, ",") + } + + eg, ctx := errgroup.WithContext(ctx) + + var statusCh chan *client.SolveStatus + if pw != nil { + statusCh = pw.Status() + eg.Go(func() error { + <-pw.Done() + return pw.Err() + }) + } + + var resp *client.SolveResponse + eg.Go(func() error { + var err error + resp, err = c.Solve(ctx, nil, so, statusCh) + if err != nil { + return err + } + return nil + }) + + if err := eg.Wait(); err != nil { + return nil, err + } + + return resp, nil +} + +type ProgressWriter struct { + status chan *client.SolveStatus + done <-chan struct{} + err error +} + +func (pw *ProgressWriter) Done() <-chan struct{} { + return pw.done +} + +func (pw *ProgressWriter) Err() error { + return pw.err +} + +func (pw *ProgressWriter) Status() chan *client.SolveStatus { + return pw.status +} + +func NewProgressWriter(ctx context.Context, out *os.File, mode string) *ProgressWriter { + statusCh := make(chan *client.SolveStatus) + doneCh := make(chan struct{}) + + pw := &ProgressWriter{ + status: statusCh, + done: doneCh, + } + + go func() { + var c console.Console + if cons, err := console.ConsoleFromFile(out); err == nil && (mode == "auto" || mode == "tty") { + c = cons + } + // not using shared context to not disrupt display but let is finish reporting errors + pw.err = progressui.DisplaySolveStatus(ctx, "", c, out, statusCh) + close(doneCh) + }() + return pw +} + +func LoadInputs(inp Inputs, target *client.SolveOpt) error { + if inp.ContextPath == "" { + return errors.New("please specify build context (e.g. \".\" for the current directory)") + } + + // TODO: handle stdin, symlinks, remote contexts, check files exist + + if inp.DockerfilePath == "" { + inp.DockerfilePath = filepath.Join(inp.ContextPath, "Dockerfile") + } + + if target.LocalDirs == nil { + target.LocalDirs = map[string]string{} + } + + target.LocalDirs["context"] = inp.ContextPath + target.LocalDirs["dockerfile"] = filepath.Dir(inp.DockerfilePath) + + if target.FrontendAttrs == nil { + target.FrontendAttrs = map[string]string{} + } + + target.FrontendAttrs["filename"] = filepath.Base(inp.DockerfilePath) + return nil +} diff --git a/build/output.go b/build/output.go new file mode 100644 index 00000000..bb61a33b --- /dev/null +++ b/build/output.go @@ -0,0 +1,86 @@ +package build + +import ( + "encoding/csv" + "os" + "strings" + + "github.com/moby/buildkit/client" + "github.com/pkg/errors" +) + +func ParseOutputs(inp []string) ([]client.ExportEntry, error) { + var outs []client.ExportEntry + if len(inp) == 0 { + return nil, nil + } + for _, s := range inp { + csvReader := csv.NewReader(strings.NewReader(s)) + fields, err := csvReader.Read() + if err != nil { + return nil, err + } + if len(fields) == 1 && fields[0] == s { + outs = append(outs, client.ExportEntry{ + Type: "local", + OutputDir: s, + }) + continue + } + + out := client.ExportEntry{ + Attrs: map[string]string{}, + } + for _, field := range fields { + parts := strings.SplitN(field, "=", 2) + if len(parts) != 2 { + return nil, errors.Errorf("invalid value %s", field) + } + key := strings.ToLower(parts[0]) + value := parts[1] + switch key { + case "type": + out.Type = value + default: + out.Attrs[key] = value + } + } + if out.Type == "" { + return nil, errors.Errorf("type is required for output") + } + + // handle client side + switch out.Type { + case "local": + dest, ok := out.Attrs["dest"] + if !ok { + return nil, errors.Errorf("dest is required for local output") + } + out.OutputDir = dest + delete(out.Attrs, "dest") + case "oci", "dest": + dest, ok := out.Attrs["dest"] + if !ok { + if out.Type != "docker" { + return nil, errors.Errorf("dest is required for %s output", out.Type) + } + } else { + if dest == "-" { + out.Output = os.Stdout + } else { + f, err := os.Open(dest) + if err != nil { + out.Output = f + } + } + delete(out.Attrs, "dest") + } + case "registry": + out.Type = "iamge" + out.Attrs["push"] = "true" + } + + outs = append(outs, out) + } + return outs, nil +} diff --git a/build/platform.go b/build/platform.go new file mode 100644 index 00000000..e24dad41 --- /dev/null +++ b/build/platform.go @@ -0,0 +1,32 @@ +package build + +import ( + "strings" + + "github.com/containerd/containerd/platforms" + specs "github.com/opencontainers/image-spec/specs-go/v1" +) + +func ParsePlatformSpecs(platformsStr []string) ([]specs.Platform, error) { + if len(platformsStr) == 0 { + return nil, nil + } + out := make([]specs.Platform, 0, len(platformsStr)) + for _, s := range platformsStr { + parts := strings.Split(s, ",") + if len(parts) > 1 { + p, err := ParsePlatformSpecs(parts) + if err != nil { + return nil, err + } + out = append(out, p...) + continue + } + p, err := platforms.Parse(s) + if err != nil { + return nil, err + } + out = append(out, platforms.Normalize(p)) + } + return out, nil +} diff --git a/build/secrets.go b/build/secrets.go new file mode 100644 index 00000000..b8ea68c7 --- /dev/null +++ b/build/secrets.go @@ -0,0 +1,60 @@ +package build + +import ( + "encoding/csv" + "strings" + + "github.com/moby/buildkit/session" + "github.com/moby/buildkit/session/secrets/secretsprovider" + "github.com/pkg/errors" +) + +func ParseSecretSpecs(sl []string) (session.Attachable, error) { + fs := make([]secretsprovider.FileSource, 0, len(sl)) + for _, v := range sl { + s, err := parseSecret(v) + if err != nil { + return nil, err + } + fs = append(fs, *s) + } + store, err := secretsprovider.NewFileStore(fs) + if err != nil { + return nil, err + } + return secretsprovider.NewSecretProvider(store), nil +} + +func parseSecret(value string) (*secretsprovider.FileSource, error) { + csvReader := csv.NewReader(strings.NewReader(value)) + fields, err := csvReader.Read() + if err != nil { + return nil, errors.Wrap(err, "failed to parse csv secret") + } + + fs := secretsprovider.FileSource{} + + for _, field := range fields { + parts := strings.SplitN(field, "=", 2) + key := strings.ToLower(parts[0]) + + if len(parts) != 2 { + return nil, errors.Errorf("invalid field '%s' must be a key=value pair", field) + } + + value := parts[1] + switch key { + case "type": + if value != "file" { + return nil, errors.Errorf("unsupported secret type %q", value) + } + case "id": + fs.ID = value + case "source", "src": + fs.FilePath = value + default: + return nil, errors.Errorf("unexpected key '%s' in '%s'", key, field) + } + } + return &fs, nil +} diff --git a/build/ssh.go b/build/ssh.go new file mode 100644 index 00000000..0e4f5076 --- /dev/null +++ b/build/ssh.go @@ -0,0 +1,31 @@ +package build + +import ( + "strings" + + "github.com/moby/buildkit/session" + "github.com/moby/buildkit/session/sshforward/sshprovider" +) + +func ParseSSHSpecs(sl []string) (session.Attachable, error) { + configs := make([]sshprovider.AgentConfig, 0, len(sl)) + for _, v := range sl { + c, err := parseSSH(v) + if err != nil { + return nil, err + } + configs = append(configs, *c) + } + return sshprovider.NewSSHAgentProvider(configs) +} + +func parseSSH(value string) (*sshprovider.AgentConfig, error) { + parts := strings.SplitN(value, "=", 2) + cfg := sshprovider.AgentConfig{ + ID: parts[0], + } + if len(parts) > 1 { + cfg.Paths = strings.Split(parts[1], ",") + } + return &cfg, nil +} diff --git a/cmd/buildx/main.go b/cmd/buildx/main.go index da61c611..c991dca0 100644 --- a/cmd/buildx/main.go +++ b/cmd/buildx/main.go @@ -1,102 +1,17 @@ package main import ( - "context" - "fmt" - "os" - "github.com/docker/cli/cli-plugins/manager" "github.com/docker/cli/cli-plugins/plugin" "github.com/docker/cli/cli/command" "github.com/spf13/cobra" + "github.com/tonistiigi/buildx/commands" "github.com/tonistiigi/buildx/version" ) func main() { plugin.Run(func(dockerCli command.Cli) *cobra.Command { - goodbye := &cobra.Command{ - Use: "goodbye", - Short: "Say Goodbye instead of Hello", - Run: func(cmd *cobra.Command, _ []string) { - fmt.Fprintln(dockerCli.Out(), "Goodbye World!") - }, - } - apiversion := &cobra.Command{ - Use: "apiversion", - Short: "Print the API version of the server", - RunE: func(_ *cobra.Command, _ []string) error { - cli := dockerCli.Client() - ping, err := cli.Ping(context.Background()) - if err != nil { - return err - } - fmt.Println(ping.APIVersion) - return nil - }, - } - - exitStatus2 := &cobra.Command{ - Use: "exitstatus2", - Short: "Exit with status 2", - RunE: func(_ *cobra.Command, _ []string) error { - fmt.Fprintln(dockerCli.Err(), "Exiting with error status 2") - os.Exit(2) - return nil - }, - } - - var ( - who, context string - preRun, debug bool - ) - cmd := &cobra.Command{ - Use: "helloworld", - Short: "A basic Hello World plugin for tests", - PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - if err := plugin.PersistentPreRunE(cmd, args); err != nil { - return err - } - if preRun { - fmt.Fprintf(dockerCli.Err(), "Plugin PersistentPreRunE called") - } - return nil - }, - RunE: func(cmd *cobra.Command, args []string) error { - if debug { - fmt.Fprintf(dockerCli.Err(), "Plugin debug mode enabled") - } - - switch context { - case "Christmas": - fmt.Fprintf(dockerCli.Out(), "Merry Christmas!\n") - return nil - case "": - // nothing - } - - if who == "" { - who, _ = dockerCli.ConfigFile().PluginConfig("helloworld", "who") - } - if who == "" { - who = "World" - } - - fmt.Fprintf(dockerCli.Out(), "Hello %s!\n", who) - dockerCli.ConfigFile().SetPluginConfig("helloworld", "lastwho", who) - return dockerCli.ConfigFile().Save() - }, - } - - flags := cmd.Flags() - flags.StringVar(&who, "who", "", "Who are we addressing?") - flags.BoolVar(&preRun, "pre-run", false, "Log from prerun hook") - // These are intended to deliberately clash with the CLIs own top - // level arguments. - flags.BoolVarP(&debug, "debug", "D", false, "Enable debug") - flags.StringVarP(&context, "context", "c", "", "Is it Christmas?") - - cmd.AddCommand(goodbye, apiversion, exitStatus2) - return cmd + return commands.NewRootCmd(dockerCli) }, manager.Metadata{ SchemaVersion: "0.1.0", diff --git a/commands/build.go b/commands/build.go new file mode 100644 index 00000000..ec638fe8 --- /dev/null +++ b/commands/build.go @@ -0,0 +1,185 @@ +package commands + +import ( + "context" + "os" + "strings" + + "github.com/docker/cli/cli" + "github.com/docker/cli/cli/command" + "github.com/moby/buildkit/client" + "github.com/moby/buildkit/session/auth/authprovider" + "github.com/moby/buildkit/util/appcontext" + bkappdefaults "github.com/moby/buildkit/util/appdefaults" + "github.com/spf13/cobra" + "github.com/tonistiigi/buildx/build" +) + +type buildOptions struct { + contextPath string + dockerfileName string + tags []string + labels []string + buildArgs []string + // extraHosts opts.ListOpts + // ulimits *opts.UlimitOpt + // memory opts.MemBytes + // memorySwap opts.MemSwapBytes + // shmSize opts.MemBytes + // cpuShares int64 + // cpuPeriod int64 + // cpuQuota int64 + // cpuSetCpus string + // cpuSetMems string + // cgroupParent string + // isolation string + // quiet bool + noCache bool + progress string + pull bool + cacheFrom []string + // compress bool + // securityOpt []string + // networkMode string + // squash bool + target string + // imageIDFile string + platforms []string + // untrusted bool + secrets []string + ssh []string + outputs []string +} + +func runBuild(dockerCli command.Cli, in buildOptions) error { + ctx := appcontext.Context() + + opts := build.Options{ + Inputs: build.Inputs{ + ContextPath: in.contextPath, + DockerfilePath: in.dockerfileName, + InStream: os.Stdin, + }, + Tags: in.tags, + Labels: listToMap(in.labels), + BuildArgs: listToMap(in.buildArgs), + Pull: in.pull, + NoCache: in.noCache, + Target: in.target, + } + + platforms, err := build.ParsePlatformSpecs(in.platforms) + if err != nil { + return err + } + opts.Platforms = platforms + + opts.Session = append(opts.Session, authprovider.NewDockerAuthProvider()) + + secrets, err := build.ParseSecretSpecs(in.secrets) + if err != nil { + return err + } + opts.Session = append(opts.Session, secrets) + + ssh, err := build.ParseSSHSpecs(in.ssh) + if err != nil { + return err + } + opts.Session = append(opts.Session, ssh) + + outputs, err := build.ParseOutputs(in.outputs) + if err != nil { + return err + } + opts.Exports = outputs + + // TODO: temporary + c, err := client.New(ctx, bkappdefaults.Address, client.WithFailFast()) + if err != nil { + return err + } + + ctx2, cancel := context.WithCancel(context.TODO()) + defer cancel() + pw := build.NewProgressWriter(ctx2, os.Stderr, in.progress) + + _, err = build.Build(ctx, c, opts, pw) + + return err +} + +func buildCmd(dockerCli command.Cli) *cobra.Command { + var options buildOptions + + cmd := &cobra.Command{ + Use: "build [OPTIONS] PATH | URL | -", + Aliases: []string{"b"}, + Short: "Start a build", + Args: cli.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + options.contextPath = args[0] + return runBuild(dockerCli, options) + }, + } + + flags := cmd.Flags() + + flags.StringArrayVarP(&options.tags, "tag", "t", []string{}, "Name and optionally a tag in the 'name:tag' format") + flags.StringArrayVar(&options.buildArgs, "build-arg", []string{}, "Set build-time variables") + // flags.Var(options.ulimits, "ulimit", "Ulimit options") + flags.StringVarP(&options.dockerfileName, "file", "f", "", "Name of the Dockerfile (Default is 'PATH/Dockerfile')") + // flags.VarP(&options.memory, "memory", "m", "Memory limit") + // flags.Var(&options.memorySwap, "memory-swap", "Swap limit equal to memory plus swap: '-1' to enable unlimited swap") + // flags.Var(&options.shmSize, "shm-size", "Size of /dev/shm") + // flags.Int64VarP(&options.cpuShares, "cpu-shares", "c", 0, "CPU shares (relative weight)") + // flags.Int64Var(&options.cpuPeriod, "cpu-period", 0, "Limit the CPU CFS (Completely Fair Scheduler) period") + // flags.Int64Var(&options.cpuQuota, "cpu-quota", 0, "Limit the CPU CFS (Completely Fair Scheduler) quota") + // flags.StringVar(&options.cpuSetCpus, "cpuset-cpus", "", "CPUs in which to allow execution (0-3, 0,1)") + // flags.StringVar(&options.cpuSetMems, "cpuset-mems", "", "MEMs in which to allow execution (0-3, 0,1)") + // flags.StringVar(&options.cgroupParent, "cgroup-parent", "", "Optional parent cgroup for the container") + // flags.StringVar(&options.isolation, "isolation", "", "Container isolation technology") + flags.StringArrayVar(&options.labels, "label", []string{}, "Set metadata for an image") + flags.BoolVar(&options.noCache, "no-cache", false, "Do not use cache when building the image") + // flags.BoolVarP(&options.quiet, "quiet", "q", false, "Suppress the build output and print image ID on success") + flags.BoolVar(&options.pull, "pull", false, "Always attempt to pull a newer version of the image") + flags.StringSliceVar(&options.cacheFrom, "cache-from", []string{}, "Images to consider as cache sources") + // flags.BoolVar(&options.compress, "compress", false, "Compress the build context using gzip") + + // flags.StringSliceVar(&options.securityOpt, "security-opt", []string{}, "Security options") + // flags.StringVar(&options.networkMode, "network", "default", "Set the networking mode for the RUN instructions during build") + // flags.Var(&options.extraHosts, "add-host", "Add a custom host-to-IP mapping (host:ip)") + flags.StringVar(&options.target, "target", "", "Set the target build stage to build.") + // flags.StringVar(&options.imageIDFile, "iidfile", "", "Write the image ID to the file") + + platformsDefault := []string{} + if v := os.Getenv("DOCKER_DEFAULT_PLATFORM"); v != "" { + platformsDefault = []string{v} + } + flags.StringArrayVar(&options.platforms, "platform", platformsDefault, "Set target platform for build") + + // flags.BoolVar(&options.squash, "squash", false, "Squash newly built layers into a single new layer") + + flags.StringVar(&options.progress, "progress", "auto", "Set type of progress output (auto, plain, tty). Use plain to show container output") + + flags.StringArrayVar(&options.secrets, "secret", []string{}, "Secret file to expose to the build: id=mysecret,src=/local/secret") + + flags.StringArrayVar(&options.ssh, "ssh", []string{}, "SSH agent socket or keys to expose to the build (format: default|[=|[,]])") + + flags.StringArrayVarP(&options.outputs, "output", "o", []string{}, "Output destination (format: type=local,dest=path)") + + return cmd +} + +func listToMap(values []string) map[string]string { + result := make(map[string]string, len(values)) + for _, value := range values { + kv := strings.SplitN(value, "=", 2) + if len(kv) == 1 { + result[kv[0]] = "" + } else { + result[kv[0]] = kv[1] + } + } + return result +} diff --git a/commands/root.go b/commands/root.go new file mode 100644 index 00000000..d693160f --- /dev/null +++ b/commands/root.go @@ -0,0 +1,21 @@ +package commands + +import ( + "github.com/docker/cli/cli/command" + "github.com/spf13/cobra" +) + +func NewRootCmd(dockerCli command.Cli) *cobra.Command { + cmd := &cobra.Command{ + Short: "Build with BuildKit", + Use: "buildx", + } + addCommands(cmd, dockerCli) + return cmd +} + +func addCommands(cmd *cobra.Command, dockerCli command.Cli) { + cmd.AddCommand( + buildCmd(dockerCli), + ) +} diff --git a/hack/demo-env/examples/simple1/Dockerfile b/hack/demo-env/examples/simple1/Dockerfile new file mode 100644 index 00000000..cf147d51 --- /dev/null +++ b/hack/demo-env/examples/simple1/Dockerfile @@ -0,0 +1,2 @@ +FROM alpine +RUN touch foo \ No newline at end of file