package commands import ( "bytes" "context" "encoding/base64" "encoding/csv" "encoding/json" "fmt" "io" "os" "path/filepath" "strconv" "strings" "sync" "github.com/containerd/console" "github.com/docker/buildx/build" "github.com/docker/buildx/monitor" "github.com/docker/buildx/util/buildflags" "github.com/docker/buildx/util/confutil" "github.com/docker/buildx/util/platformutil" "github.com/docker/buildx/util/progress" "github.com/docker/buildx/util/tracing" "github.com/docker/cli-docs-tool/annotation" "github.com/docker/cli/cli" "github.com/docker/cli/cli/command" "github.com/docker/cli/cli/config" dockeropts "github.com/docker/cli/opts" "github.com/docker/distribution/reference" "github.com/docker/docker/pkg/ioutils" "github.com/docker/go-units" "github.com/moby/buildkit/client" "github.com/moby/buildkit/session/auth/authprovider" "github.com/moby/buildkit/solver/errdefs" "github.com/moby/buildkit/util/appcontext" "github.com/moby/buildkit/util/grpcerrors" "github.com/morikuni/aec" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/spf13/pflag" "google.golang.org/grpc/codes" ) const defaultTargetName = "default" type buildOptions struct { contextPath string dockerfileName string printFunc string allow []string buildArgs []string cacheFrom []string cacheTo []string cgroupParent string contexts []string extraHosts []string imageIDFile string labels []string networkMode string noCacheFilter []string outputs []string platforms []string quiet bool secrets []string shmSize dockeropts.MemBytes ssh []string tags []string target string ulimits *dockeropts.UlimitOpt invoke string commonOptions } type commonOptions struct { builder string metadataFile string noCache *bool progress string pull *bool exportPush bool exportLoad bool } func runBuild(dockerCli command.Cli, in buildOptions) (err error) { ctx := appcontext.Context() ctx, end, err := tracing.TraceCurrentCommand(ctx, "build") if err != nil { return err } defer func() { end(err) }() noCache := false if in.noCache != nil { noCache = *in.noCache } pull := false if in.pull != nil { pull = *in.pull } if noCache && len(in.noCacheFilter) > 0 { return errors.Errorf("--no-cache and --no-cache-filter cannot currently be used together") } if in.quiet && in.progress != progress.PrinterModeAuto && in.progress != progress.PrinterModeQuiet { return errors.Errorf("progress=%s and quiet cannot be used together", in.progress) } else if in.quiet { in.progress = "quiet" } contexts, err := parseContextNames(in.contexts) if err != nil { return err } printFunc, err := parsePrintFunc(in.printFunc) if err != nil { return err } opts := build.Options{ Inputs: build.Inputs{ ContextPath: in.contextPath, DockerfilePath: in.dockerfileName, InStream: os.Stdin, NamedContexts: contexts, }, BuildArgs: listToMap(in.buildArgs, true), ExtraHosts: in.extraHosts, ImageIDFile: in.imageIDFile, Labels: listToMap(in.labels, false), NetworkMode: in.networkMode, NoCache: noCache, NoCacheFilter: in.noCacheFilter, Pull: pull, ShmSize: in.shmSize, Tags: in.tags, Target: in.target, Ulimits: in.ulimits, PrintFunc: printFunc, } platforms, err := platformutil.Parse(in.platforms) if err != nil { return err } opts.Platforms = platforms dockerConfig := config.LoadDefaultConfigFile(os.Stderr) opts.Session = append(opts.Session, authprovider.NewDockerAuthProvider(dockerConfig)) secrets, err := buildflags.ParseSecretSpecs(in.secrets) if err != nil { return err } opts.Session = append(opts.Session, secrets) sshSpecs := in.ssh if len(sshSpecs) == 0 && buildflags.IsGitSSH(in.contextPath) { sshSpecs = []string{"default"} } ssh, err := buildflags.ParseSSHSpecs(sshSpecs) if err != nil { return err } opts.Session = append(opts.Session, ssh) outputs, err := buildflags.ParseOutputs(in.outputs) if err != nil { return err } if in.exportPush { if in.exportLoad { return errors.Errorf("push and load may not be set together at the moment") } if len(outputs) == 0 { outputs = []client.ExportEntry{{ Type: "image", Attrs: map[string]string{ "push": "true", }, }} } else { switch outputs[0].Type { case "image": outputs[0].Attrs["push"] = "true" default: return errors.Errorf("push and %q output can't be used together", outputs[0].Type) } } } if in.exportLoad { if len(outputs) == 0 { outputs = []client.ExportEntry{{ Type: "docker", Attrs: map[string]string{}, }} } else { switch outputs[0].Type { case "docker": default: return errors.Errorf("load and %q output can't be used together", outputs[0].Type) } } } opts.Exports = outputs cacheImports, err := buildflags.ParseCacheEntry(in.cacheFrom) if err != nil { return err } opts.CacheFrom = cacheImports cacheExports, err := buildflags.ParseCacheEntry(in.cacheTo) if err != nil { return err } opts.CacheTo = cacheExports allow, err := buildflags.ParseEntitlements(in.allow) if err != nil { return err } opts.Allow = allow // key string used for kubernetes "sticky" mode contextPathHash, err := filepath.Abs(in.contextPath) if err != nil { contextPathHash = in.contextPath } imageID, res, err := buildTargets(ctx, dockerCli, map[string]build.Options{defaultTargetName: opts}, in.progress, contextPathHash, in.builder, in.metadataFile, in.invoke != "") err = wrapBuildError(err, false) if err != nil { return err } if in.invoke != "" { cfg, err := parseInvokeConfig(in.invoke) if err != nil { return err } cfg.ResultCtx = res con := console.Current() if err := con.SetRaw(); err != nil { return errors.Errorf("failed to configure terminal: %v", err) } err = monitor.RunMonitor(ctx, cfg, func(ctx context.Context) (*build.ResultContext, error) { _, rr, err := buildTargets(ctx, dockerCli, map[string]build.Options{defaultTargetName: opts}, in.progress, contextPathHash, in.builder, in.metadataFile, true) return rr, err }, io.NopCloser(os.Stdin), nopCloser{os.Stdout}, nopCloser{os.Stderr}) if err != nil { logrus.Warnf("failed to run monitor: %v", err) } con.Reset() } if in.quiet { fmt.Println(imageID) } return nil } type nopCloser struct { io.WriteCloser } func (c nopCloser) Close() error { return nil } func buildTargets(ctx context.Context, dockerCli command.Cli, opts map[string]build.Options, progressMode, contextPathHash, instance string, metadataFile string, allowNoOutput bool) (imageID string, res *build.ResultContext, err error) { dis, err := getInstanceOrDefault(ctx, dockerCli, instance, contextPathHash) if err != nil { return "", nil, err } ctx2, cancel := context.WithCancel(context.TODO()) defer cancel() printer, err := progress.NewPrinter(ctx2, os.Stderr, os.Stderr, progressMode) if err != nil { return "", nil, err } var mu sync.Mutex var idx int resp, err := build.BuildWithResultHandler(ctx, dis, opts, dockerAPI(dockerCli), confutil.ConfigDir(dockerCli), printer, func(driverIndex int, gotRes *build.ResultContext) { mu.Lock() defer mu.Unlock() if res == nil || driverIndex < idx { idx, res = driverIndex, gotRes } }, allowNoOutput) err1 := printer.Wait() if err == nil { err = err1 } if err != nil { return "", nil, err } if len(metadataFile) > 0 && resp != nil { if err := writeMetadataFile(metadataFile, decodeExporterResponse(resp[defaultTargetName].ExporterResponse)); err != nil { return "", nil, err } } printWarnings(os.Stderr, printer.Warnings(), progressMode) for k := range resp { if opts[k].PrintFunc != nil { if err := printResult(opts[k].PrintFunc, resp[k].ExporterResponse); err != nil { return "", nil, err } } } return resp[defaultTargetName].ExporterResponse["containerimage.digest"], res, err } func parseInvokeConfig(invoke string) (cfg build.ContainerConfig, err error) { cfg.Tty = true if invoke == "default" { return cfg, nil } csvReader := csv.NewReader(strings.NewReader(invoke)) fields, err := csvReader.Read() if err != nil { return cfg, err } if len(fields) == 1 && !strings.Contains(fields[0], "=") { cfg.Cmd = []string{fields[0]} return cfg, nil } for _, field := range fields { parts := strings.SplitN(field, "=", 2) if len(parts) != 2 { return cfg, errors.Errorf("invalid value %s", field) } key := strings.ToLower(parts[0]) value := parts[1] switch key { case "args": cfg.Cmd = append(cfg.Cmd, value) // TODO: support JSON case "entrypoint": cfg.Entrypoint = append(cfg.Entrypoint, value) // TODO: support JSON case "env": cfg.Env = append(cfg.Env, value) case "user": cfg.User = &value case "cwd": cfg.Cwd = &value case "tty": cfg.Tty, err = strconv.ParseBool(value) if err != nil { return cfg, errors.Errorf("failed to parse tty: %v", err) } default: return cfg, errors.Errorf("unknown key %q", key) } } return cfg, nil } func printWarnings(w io.Writer, warnings []client.VertexWarning, mode string) { if len(warnings) == 0 || mode == progress.PrinterModeQuiet { return } fmt.Fprintf(w, "\n ") sb := &bytes.Buffer{} if len(warnings) == 1 { fmt.Fprintf(sb, "1 warning found") } else { fmt.Fprintf(sb, "%d warnings found", len(warnings)) } if logrus.GetLevel() < logrus.DebugLevel { fmt.Fprintf(sb, " (use --debug to expand)") } fmt.Fprintf(sb, ":\n") fmt.Fprint(w, aec.Apply(sb.String(), aec.YellowF)) for _, warn := range warnings { fmt.Fprintf(w, " - %s\n", warn.Short) if logrus.GetLevel() < logrus.DebugLevel { continue } for _, d := range warn.Detail { fmt.Fprintf(w, "%s\n", d) } if warn.URL != "" { fmt.Fprintf(w, "More info: %s\n", warn.URL) } if warn.SourceInfo != nil && warn.Range != nil { src := errdefs.Source{ Info: warn.SourceInfo, Ranges: warn.Range, } src.Print(w) } fmt.Fprintf(w, "\n") } } func newBuildOptions() buildOptions { ulimits := make(map[string]*units.Ulimit) return buildOptions{ ulimits: dockeropts.NewUlimitOpt(&ulimits), } } func buildCmd(dockerCli command.Cli, rootOpts *rootOptions) *cobra.Command { options := newBuildOptions() cmd := &cobra.Command{ Use: "build [OPTIONS] PATH | URL | -", Aliases: []string{"b"}, Short: "Start a build", Args: cli.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { options.contextPath = args[0] options.builder = rootOpts.builder cmd.Flags().VisitAll(checkWarnedFlags) return runBuild(dockerCli, options) }, } var platformsDefault []string if v := os.Getenv("DOCKER_DEFAULT_PLATFORM"); v != "" { platformsDefault = []string{v} } flags := cmd.Flags() flags.StringSliceVar(&options.extraHosts, "add-host", []string{}, `Add a custom host-to-IP mapping (format: "host:ip")`) flags.SetAnnotation("add-host", annotation.ExternalURL, []string{"https://docs.docker.com/engine/reference/commandline/build/#add-entries-to-container-hosts-file---add-host"}) flags.StringSliceVar(&options.allow, "allow", []string{}, `Allow extra privileged entitlement (e.g., "network.host", "security.insecure")`) flags.StringArrayVar(&options.buildArgs, "build-arg", []string{}, "Set build-time variables") flags.StringArrayVar(&options.cacheFrom, "cache-from", []string{}, `External cache sources (e.g., "user/app:cache", "type=local,src=path/to/dir")`) flags.StringArrayVar(&options.cacheTo, "cache-to", []string{}, `Cache export destinations (e.g., "user/app:cache", "type=local,dest=path/to/dir")`) flags.StringVar(&options.cgroupParent, "cgroup-parent", "", "Optional parent cgroup for the container") flags.SetAnnotation("cgroup-parent", annotation.ExternalURL, []string{"https://docs.docker.com/engine/reference/commandline/build/#use-a-custom-parent-cgroup---cgroup-parent"}) flags.StringArrayVar(&options.contexts, "build-context", []string{}, "Additional build contexts (e.g., name=path)") flags.StringVarP(&options.dockerfileName, "file", "f", "", `Name of the Dockerfile (default: "PATH/Dockerfile")`) flags.SetAnnotation("file", annotation.ExternalURL, []string{"https://docs.docker.com/engine/reference/commandline/build/#specify-a-dockerfile--f"}) flags.StringVar(&options.imageIDFile, "iidfile", "", "Write the image ID to the file") flags.StringArrayVar(&options.labels, "label", []string{}, "Set metadata for an image") flags.BoolVar(&options.exportLoad, "load", false, `Shorthand for "--output=type=docker"`) flags.StringVar(&options.networkMode, "network", "default", `Set the networking mode for the "RUN" instructions during build`) flags.StringArrayVar(&options.noCacheFilter, "no-cache-filter", []string{}, "Do not cache specified stages") flags.StringArrayVarP(&options.outputs, "output", "o", []string{}, `Output destination (format: "type=local,dest=path")`) flags.StringArrayVar(&options.platforms, "platform", platformsDefault, "Set target platform for build") if isExperimental() { flags.StringVar(&options.printFunc, "print", "", "Print result of information request (e.g., outline, targets) [experimental]") } flags.BoolVar(&options.exportPush, "push", false, `Shorthand for "--output=type=registry"`) flags.BoolVarP(&options.quiet, "quiet", "q", false, "Suppress the build output and print image ID on success") flags.StringArrayVar(&options.secrets, "secret", []string{}, `Secret to expose to the build (format: "id=mysecret[,src=/local/secret]")`) flags.Var(&options.shmSize, "shm-size", `Size of "/dev/shm"`) flags.StringArrayVar(&options.ssh, "ssh", []string{}, `SSH agent socket or keys to expose to the build (format: "default|[=|[,]]")`) flags.StringArrayVarP(&options.tags, "tag", "t", []string{}, `Name and optionally a tag (format: "name:tag")`) flags.SetAnnotation("tag", annotation.ExternalURL, []string{"https://docs.docker.com/engine/reference/commandline/build/#tag-an-image--t"}) flags.StringVar(&options.target, "target", "", "Set the target build stage to build") flags.SetAnnotation("target", annotation.ExternalURL, []string{"https://docs.docker.com/engine/reference/commandline/build/#specifying-target-build-stage---target"}) flags.Var(options.ulimits, "ulimit", "Ulimit options") if isExperimental() { flags.StringVar(&options.invoke, "invoke", "", "Invoke a command after the build [experimental]") } // hidden flags var ignore string var ignoreSlice []string var ignoreBool bool var ignoreInt int64 flags.BoolVar(&ignoreBool, "compress", false, "Compress the build context using gzip") flags.MarkHidden("compress") flags.StringVar(&ignore, "isolation", "", "Container isolation technology") flags.MarkHidden("isolation") flags.SetAnnotation("isolation", "flag-warn", []string{"isolation flag is deprecated with BuildKit."}) flags.StringSliceVar(&ignoreSlice, "security-opt", []string{}, "Security options") flags.MarkHidden("security-opt") flags.SetAnnotation("security-opt", "flag-warn", []string{`security-opt flag is deprecated. "RUN --security=insecure" should be used with BuildKit.`}) flags.BoolVar(&ignoreBool, "squash", false, "Squash newly built layers into a single new layer") flags.MarkHidden("squash") flags.SetAnnotation("squash", "flag-warn", []string{"experimental flag squash is removed with BuildKit. You should squash inside build using a multi-stage Dockerfile for efficiency."}) flags.StringVarP(&ignore, "memory", "m", "", "Memory limit") flags.MarkHidden("memory") flags.StringVar(&ignore, "memory-swap", "", `Swap limit equal to memory plus swap: "-1" to enable unlimited swap`) flags.MarkHidden("memory-swap") flags.Int64VarP(&ignoreInt, "cpu-shares", "c", 0, "CPU shares (relative weight)") flags.MarkHidden("cpu-shares") flags.Int64Var(&ignoreInt, "cpu-period", 0, "Limit the CPU CFS (Completely Fair Scheduler) period") flags.MarkHidden("cpu-period") flags.Int64Var(&ignoreInt, "cpu-quota", 0, "Limit the CPU CFS (Completely Fair Scheduler) quota") flags.MarkHidden("cpu-quota") flags.StringVar(&ignore, "cpuset-cpus", "", `CPUs in which to allow execution ("0-3", "0,1")`) flags.MarkHidden("cpuset-cpus") flags.StringVar(&ignore, "cpuset-mems", "", `MEMs in which to allow execution ("0-3", "0,1")`) flags.MarkHidden("cpuset-mems") flags.BoolVar(&ignoreBool, "rm", true, "Remove intermediate containers after a successful build") flags.MarkHidden("rm") flags.BoolVar(&ignoreBool, "force-rm", false, "Always remove intermediate containers") flags.MarkHidden("force-rm") commonBuildFlags(&options.commonOptions, flags) return cmd } func commonBuildFlags(options *commonOptions, flags *pflag.FlagSet) { options.noCache = flags.Bool("no-cache", false, "Do not use cache when building the image") flags.StringVar(&options.progress, "progress", "auto", `Set type of progress output ("auto", "plain", "tty"). Use plain to show container output`) options.pull = flags.Bool("pull", false, "Always attempt to pull all referenced images") flags.StringVar(&options.metadataFile, "metadata-file", "", "Write build result metadata to the file") } func checkWarnedFlags(f *pflag.Flag) { if !f.Changed { return } for t, m := range f.Annotations { switch t { case "flag-warn": logrus.Warn(m[0]) } } } func listToMap(values []string, defaultEnv bool) map[string]string { result := make(map[string]string, len(values)) for _, value := range values { kv := strings.SplitN(value, "=", 2) if len(kv) == 1 { if defaultEnv { v, ok := os.LookupEnv(kv[0]) if ok { result[kv[0]] = v } } else { result[kv[0]] = "" } } else { result[kv[0]] = kv[1] } } return result } func parseContextNames(values []string) (map[string]build.NamedContext, error) { if len(values) == 0 { return nil, nil } result := make(map[string]build.NamedContext, len(values)) for _, value := range values { kv := strings.SplitN(value, "=", 2) if len(kv) != 2 { return nil, errors.Errorf("invalid context value: %s, expected key=value", value) } named, err := reference.ParseNormalizedNamed(kv[0]) if err != nil { return nil, errors.Wrapf(err, "invalid context name %s", kv[0]) } name := strings.TrimSuffix(reference.FamiliarString(named), ":latest") result[name] = build.NamedContext{Path: kv[1]} } return result, nil } func parsePrintFunc(str string) (*build.PrintFunc, error) { if str == "" { return nil, nil } csvReader := csv.NewReader(strings.NewReader(str)) fields, err := csvReader.Read() if err != nil { return nil, err } f := &build.PrintFunc{} for _, field := range fields { parts := strings.SplitN(field, "=", 2) if len(parts) == 2 { if parts[0] == "format" { f.Format = parts[1] } else { return nil, errors.Errorf("invalid print field: %s", field) } } else { if f.Name != "" { return nil, errors.Errorf("invalid print value: %s", str) } f.Name = field } } return f, nil } func writeMetadataFile(filename string, dt interface{}) error { b, err := json.MarshalIndent(dt, "", " ") if err != nil { return err } return ioutils.AtomicWriteFile(filename, b, 0644) } func decodeExporterResponse(exporterResponse map[string]string) map[string]interface{} { out := make(map[string]interface{}) for k, v := range exporterResponse { dt, err := base64.StdEncoding.DecodeString(v) if err != nil { out[k] = v continue } var raw map[string]interface{} if err = json.Unmarshal(dt, &raw); err != nil || len(raw) == 0 { out[k] = v continue } out[k] = json.RawMessage(dt) } return out } func wrapBuildError(err error, bake bool) error { if err == nil { return nil } st, ok := grpcerrors.AsGRPCStatus(err) if ok { if st.Code() == codes.Unimplemented && strings.Contains(st.Message(), "unsupported frontend capability moby.buildkit.frontend.contexts") { msg := "current frontend does not support --build-context." if bake { msg = "current frontend does not support defining additional contexts for targets." } msg += " Named contexts are supported since Dockerfile v1.4. Use #syntax directive in Dockerfile or update to latest BuildKit." return &wrapped{err, msg} } } return err } type wrapped struct { err error msg string } func (w *wrapped) Error() string { return w.msg } func (w *wrapped) Unwrap() error { return w.err } func isExperimental() bool { if v, ok := os.LookupEnv("BUILDX_EXPERIMENTAL"); ok { vv, _ := strconv.ParseBool(v) return vv } return false }