Merge pull request #1890 from jedevc/tests-share-docker-container-backend

.github/workflows/build.yml

@@ -102,7 +102,7 @@ jobs:
           ./hack/test
         env:
           TEST_DOCKERD: "${{ (matrix.worker == 'docker' || matrix.worker == 'docker-container') && '1' || '0' }}"
-          TESTFLAGS: "${{ (matrix.worker == 'docker' || matrix.worker == 'docker-container') && env.TESTFLAGS_DOCKER || env.TESTFLAGS }} --run=//worker=${{ matrix.worker }}$"
+          TESTFLAGS: "${{ (matrix.worker == 'docker') && env.TESTFLAGS_DOCKER || env.TESTFLAGS }} --run=//worker=${{ matrix.worker }}$"
           TESTPKGS: "${{ matrix.pkg }}"
           SKIP_INTEGRATION_TESTS: "${{ matrix.skip-integration-tests }}"
       -

go.mod

@@ -22,7 +22,7 @@ require (
 	github.com/google/uuid v1.3.0
 	github.com/hashicorp/go-cty-funcs v0.0.0-20200930094925-2721b1e36840
 	github.com/hashicorp/hcl/v2 v2.8.2
-	github.com/moby/buildkit v0.11.0-rc3.0.20230609092854-67a08623b95a
+	github.com/moby/buildkit v0.11.0-rc3.0.20230620112432-2d91ddcceedc
 	github.com/moby/sys/mountinfo v0.6.2
 	github.com/moby/sys/signal v0.7.0
 	github.com/morikuni/aec v1.0.0

go.sum

@@ -371,8 +371,8 @@ github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7 h1:DpOJ2HYzC
 github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo=
 github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
 github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
-github.com/moby/buildkit v0.11.0-rc3.0.20230609092854-67a08623b95a h1:1k3bAXwxC2N1FncWijq/43sLj2OVIZ11FT0APIXWhMg=
+github.com/moby/buildkit v0.11.0-rc3.0.20230620112432-2d91ddcceedc h1:79EnLqFEkPn6sTBXDHE546AHmYeb/QzXgGyIpli8w34=
-github.com/moby/buildkit v0.11.0-rc3.0.20230609092854-67a08623b95a/go.mod h1:4sM7BBBqXOQ+vV6LrVAOAMhZI9cVNYV5RhZCl906a64=
+github.com/moby/buildkit v0.11.0-rc3.0.20230620112432-2d91ddcceedc/go.mod h1:6Y1HYDrxg3sY5gBY2FVaEvQpswBj3g/ck7aKYCjOkk0=
 github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg=
 github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc=
 github.com/moby/patternmatcher v0.5.0 h1:YCZgJOeULcxLw1Q+sVR636pmS7sPEn1Qo2iAN6M7DBo=

tests/integration.go

@@ -47,6 +47,7 @@ func buildxCmd(sb integration.Sandbox, opts ...cmdOpt) *exec.Cmd {
 
 	if builder := sb.Address(); builder != "" {
 		cmd.Args = append(cmd.Args, "--builder="+builder)
+		cmd.Env = append(cmd.Env, "BUILDX_CONFIG=/tmp/buildx-"+builder)
 	}
 	if context := sb.DockerAddress(); context != "" {
 		cmd.Env = append(cmd.Env, "DOCKER_CONTEXT="+context)

tests/workers/docker-container.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"os"
 	"os/exec"
+	"sync"
 
 	"github.com/moby/buildkit/identity"
 	"github.com/moby/buildkit/util/testutil/integration"
@@ -18,6 +19,11 @@ func InitDockerContainerWorker() {
 
 type containerWorker struct {
 	id string
+
+	docker      integration.Backend
+	dockerClose func() error
+	dockerErr   error
+	dockerOnce  sync.Once
 }
 
 func (w *containerWorker) Name() string {
@@ -29,9 +35,11 @@ func (w *containerWorker) Rootless() bool {
 }
 
 func (w *containerWorker) New(ctx context.Context, cfg *integration.BackendConfig) (integration.Backend, func() error, error) {
-	bk, bkclose, err := dockerWorker{id: w.id}.New(ctx, cfg)
+	w.dockerOnce.Do(func() {
-	if err != nil {
+		w.docker, w.dockerClose, w.dockerErr = dockerWorker{id: w.id}.New(ctx, cfg)
-		return bk, bkclose, err
+	})
+	if w.dockerErr != nil {
+		return w.docker, w.dockerClose, w.dockerErr
 	}
 
 	name := "integration-container-" + identity.NewID()
@@ -42,25 +50,36 @@ func (w *containerWorker) New(ctx context.Context, cfg *integration.BackendConfi
 		"--driver=docker-container",
 		"--driver-opt=network=host",
 	)
-	cmd.Env = append(os.Environ(), "DOCKER_CONTEXT="+bk.DockerAddress())
+	cmd.Env = append(
+		os.Environ(),
+		"BUILDX_CONFIG=/tmp/buildx-"+name,
+		"DOCKER_CONTEXT="+w.docker.DockerAddress(),
+	)
 	if err := cmd.Run(); err != nil {
 		return nil, nil, errors.Wrapf(err, "failed to create buildx instance %s", name)
 	}
 
 	cl := func() error {
-		var err error
-		if err1 := bkclose(); err == nil {
-			err = err1
-		}
 		cmd := exec.Command("buildx", "rm", "-f", name)
-		if err1 := cmd.Run(); err == nil {
+		return cmd.Run()
-			err = err1
-		}
-		return err
 	}
 
 	return &backend{
-		context: bk.DockerAddress(),
+		context: w.docker.DockerAddress(),
 		builder: name,
 	}, cl, nil
 }
+
+func (w *containerWorker) Close() error {
+	if close := w.dockerClose; close != nil {
+		return close()
+	}
+
+	// reset the worker to be ready to go again
+	w.docker = nil
+	w.dockerClose = nil
+	w.dockerErr = nil
+	w.dockerOnce = sync.Once{}
+
+	return nil
+}

tests/workers/docker.go

@@ -2,6 +2,7 @@ package workers
 
 import (
 	"context"
+	"os"
 	"os/exec"
 
 	"github.com/moby/buildkit/identity"
@@ -41,8 +42,9 @@ func (c dockerWorker) New(ctx context.Context, cfg *integration.BackendConfig) (
 		name,
 		"--docker", "host="+bk.DockerAddress(),
 	)
+	cmd.Env = append(os.Environ(), "BUILDX_CONFIG=/tmp/buildx-"+name)
 	if err := cmd.Run(); err != nil {
-		return nil, cl, errors.Wrapf(err, "failed to create buildx instance %s", name)
+		return bk, cl, errors.Wrapf(err, "failed to create buildx instance %s", name)
 	}
 
 	cl = func() error {
@@ -62,3 +64,7 @@ func (c dockerWorker) New(ctx context.Context, cfg *integration.BackendConfig) (
 		context: name,
 	}, cl, nil
 }
+
+func (c dockerWorker) Close() error {
+	return nil
+}

tests/workers/remote.go

@@ -2,6 +2,7 @@ package workers
 
 import (
 	"context"
+	"os"
 	"os/exec"
 
 	"github.com/moby/buildkit/identity"
@@ -41,6 +42,7 @@ func (w remoteWorker) New(ctx context.Context, cfg *integration.BackendConfig) (
 		"--driver=remote",
 		bk.Address(),
 	)
+	cmd.Env = append(os.Environ(), "BUILDX_CONFIG=/tmp/buildx-"+name)
 	if err := cmd.Run(); err != nil {
 		return nil, nil, errors.Wrapf(err, "failed to create buildx instance %s", name)
 	}
@@ -61,3 +63,7 @@ func (w remoteWorker) New(ctx context.Context, cfg *integration.BackendConfig) (
 		builder: name,
 	}, cl, nil
 }
+
+func (w remoteWorker) Close() error {
+	return nil
+}

vendor/github.com/moby/buildkit/client/llb/fileop.go

@@ -61,6 +61,10 @@ type capAdder interface {
 	addCaps(*FileOp)
 }
 
+// FileAction is used to specify a file operation on a [State].
+// It can be used to create a directory, create a file, or remove a file, etc.
+// This is used by [State.File]
+// Typically a FileAction is created by calling one of the helper functions such as [Mkdir], [Copy], [Rm], [Mkfile]
 type FileAction struct {
 	state  *State
 	prev   *FileAction

vendor/github.com/moby/buildkit/client/llb/meta.go

@@ -29,10 +29,15 @@ var (
 	keySecurity = contextKeyT("llb.security")
 )
 
+// AddEnvf is the same as [AddEnv] but allows for a format string.
+// This is the equivalent of `[State.AddEnvf]`
 func AddEnvf(key, value string, v ...interface{}) StateOption {
 	return addEnvf(key, value, true, v...)
 }
 
+// AddEnv returns a [StateOption] whichs adds an environment variable to the state.
+// Use this with [State.With] to create a new state with the environment variable set.
+// This is the equivalent of `[State.AddEnv]`
 func AddEnv(key, value string) StateOption {
 	return addEnvf(key, value, false)
 }
@@ -52,10 +57,14 @@ func addEnvf(key, value string, replace bool, v ...interface{}) StateOption {
 	}
 }
 
+// Dir returns a [StateOption] sets the working directory for the state which will be used to resolve
+// relative paths as well as the working directory for [State.Run].
+// See [State.With] for where to use this.
 func Dir(str string) StateOption {
 	return dirf(str, false)
 }
 
+// Dirf is the same as [Dir] but allows for a format string.
 func Dirf(str string, v ...interface{}) StateOption {
 	return dirf(str, true, v...)
 }
@@ -81,12 +90,18 @@ func dirf(value string, replace bool, v ...interface{}) StateOption {
 	}
 }
 
+// User returns a [StateOption] which sets the user for the state which will be used by [State.Run].
+// This is the equivalent of [State.User]
+// See [State.With] for where to use this.
 func User(str string) StateOption {
 	return func(s State) State {
 		return s.WithValue(keyUser, str)
 	}
 }
 
+// Reset returns a [StateOption] which creates a new [State] with just the
+// output of the current [State] and the provided [State] is set as the parent.
+// This is the equivalent of [State.Reset]
 func Reset(other State) StateOption {
 	return func(s State) State {
 		s = NewState(s.Output())
@@ -147,6 +162,9 @@ func getUser(s State) func(context.Context, *Constraints) (string, error) {
 	}
 }
 
+// Hostname returns a [StateOption] which sets the hostname used for containers created by [State.Run].
+// This is the equivalent of [State.Hostname]
+// See [State.With] for where to use this.
 func Hostname(str string) StateOption {
 	return func(s State) State {
 		return s.WithValue(keyHostname, str)
@@ -283,6 +301,9 @@ func getCgroupParent(s State) func(context.Context, *Constraints) (string, error
 	}
 }
 
+// Network returns a [StateOption] which sets the network mode used for containers created by [State.Run].
+// This is the equivalent of [State.Network]
+// See [State.With] for where to use this.
 func Network(v pb.NetMode) StateOption {
 	return func(s State) State {
 		return s.WithValue(keyNetwork, v)
@@ -302,6 +323,9 @@ func getNetwork(s State) func(context.Context, *Constraints) (pb.NetMode, error)
 	}
 }
 
+// Security returns a [StateOption] which sets the security mode used for containers created by [State.Run].
+// This is the equivalent of [State.Security]
+// See [State.With] for where to use this.
 func Security(v pb.SecurityMode) StateOption {
 	return func(s State) State {
 		return s.WithValue(keySecurity, v)

vendor/github.com/moby/buildkit/client/llb/source.go

@@ -230,6 +230,9 @@ type ImageInfo struct {
 // Other URL formats are supported such as "git@github.com:moby/buildkit.git", "git://...", "ssh://..."
 // Formats that utilize SSH may need to supply credentials as a [GitOption].
 // You may need to check the source code for a full list of supported formats.
+//
+// By default the git repository is cloned with `--depth=1` to reduce the amount of data downloaded.
+// Additionally the ".git" directory is removed after the clone, you can keep ith with the [KeepGitDir] [GitOption].
 func Git(remote, ref string, opts ...GitOption) State {
 	url := strings.Split(remote, "#")[0]
 

vendor/github.com/moby/buildkit/client/llb/state.go

@@ -53,6 +53,8 @@ func NewState(o Output) State {
 // States are immutable, and all operations return a new state linked to the previous one.
 // State is the core type of the LLB API and is used to build a graph of operations.
 // The graph is then marshaled into a definition that can be executed by a backend (such as buildkitd).
+//
+// Operations performed on a State are executed lazily after the entire state graph is marshalled and sent to the backend.
 type State struct {
 	out   Output
 	prev  *State
@@ -127,6 +129,7 @@ func (s State) SetMarshalDefaults(co ...ConstraintsOpt) State {
 	return s
 }
 
+// Marshal marshals the state and all its parents into a [Definition].
 func (s State) Marshal(ctx context.Context, co ...ConstraintsOpt) (*Definition, error) {
 	c := NewConstraints(append(s.opts, co...)...)
 	def := &Definition{
@@ -212,10 +215,13 @@ func marshal(ctx context.Context, v Vertex, def *Definition, s *sourceMapCollect
 	return def, nil
 }
 
+// Validate validates the state.
+// This validation, unlike most other operations on [State], is not lazily performed.
 func (s State) Validate(ctx context.Context, c *Constraints) error {
 	return s.Output().Vertex(ctx, c).Validate(ctx, c)
 }
 
+// Output returns the output of the state.
 func (s State) Output() Output {
 	if s.async != nil {
 		return s.async.Output()
@@ -223,6 +229,7 @@ func (s State) Output() Output {
 	return s.out
 }
 
+// WithOutput creats a new state with the output set to the given output.
 func (s State) WithOutput(o Output) State {
 	prev := s
 	s = State{
@@ -233,6 +240,7 @@ func (s State) WithOutput(o Output) State {
 	return s
 }
 
+// WithImageConfig adds the environment variables, working directory, and platform specified in the image config to the state.
 func (s State) WithImageConfig(c []byte) (State, error) {
 	var img ocispecs.Image
 	if err := json.Unmarshal(c, &img); err != nil {
@@ -259,6 +267,12 @@ func (s State) WithImageConfig(c []byte) (State, error) {
 	return s, nil
 }
 
+// Run performs the command specified by the arguments within the contexst of the current [State].
+// The command is executed as a container with the [State]'s filesystem as the root filesystem.
+// As such any command you run must be present in the [State]'s filesystem.
+// Constraints such as [State.Ulimit], [State.ParentCgroup], [State.Network], etc. are applied to the container.
+//
+// Run is useful when none of the LLB ops are sufficient for the operation that you want to perform.
 func (s State) Run(ro ...RunOption) ExecState {
 	ei := &ExecInfo{State: s}
 	for _, o := range ro {
@@ -277,6 +291,8 @@ func (s State) Run(ro ...RunOption) ExecState {
 	}
 }
 
+// File performs a file operation on the current state.
+// See [FileAction] for details on the operations that can be performed.
 func (s State) File(a *FileAction, opts ...ConstraintsOpt) State {
 	var c Constraints
 	for _, o := range opts {
@@ -286,21 +302,29 @@ func (s State) File(a *FileAction, opts ...ConstraintsOpt) State {
 	return s.WithOutput(NewFileOp(s, a, c).Output())
 }
 
+// AddEnv returns a new [State] with the provided environment variable set.
+// See [AddEnv]
 func (s State) AddEnv(key, value string) State {
 	return AddEnv(key, value)(s)
 }
 
+// AddEnvf is the same as [State.AddEnv] but with a format string.
 func (s State) AddEnvf(key, value string, v ...interface{}) State {
 	return AddEnvf(key, value, v...)(s)
 }
 
+// Dir returns a new [State] with the provided working directory set.
+// See [Dir]
 func (s State) Dir(str string) State {
 	return Dir(str)(s)
 }
+
+// Dirf is the same as [State.Dir] but with a format string.
 func (s State) Dirf(str string, v ...interface{}) State {
 	return Dirf(str, v...)(s)
 }
 
+// GetEnv returns the value of the environment variable with the provided key.
 func (s State) GetEnv(ctx context.Context, key string, co ...ConstraintsOpt) (string, bool, error) {
 	c := &Constraints{}
 	for _, f := range co {
@@ -314,6 +338,8 @@ func (s State) GetEnv(ctx context.Context, key string, co ...ConstraintsOpt) (st
 	return v, ok, nil
 }
 
+// Env returns a new [State] with the provided environment variable set.
+// See [Env]
 func (s State) Env(ctx context.Context, co ...ConstraintsOpt) ([]string, error) {
 	c := &Constraints{}
 	for _, f := range co {
@@ -326,6 +352,7 @@ func (s State) Env(ctx context.Context, co ...ConstraintsOpt) ([]string, error)
 	return env.ToArray(), nil
 }
 
+// GetDir returns the current working directory for the state.
 func (s State) GetDir(ctx context.Context, co ...ConstraintsOpt) (string, error) {
 	c := &Constraints{}
 	for _, f := range co {
@@ -342,18 +369,28 @@ func (s State) GetArgs(ctx context.Context, co ...ConstraintsOpt) ([]string, err
 	return getArgs(s)(ctx, c)
 }
 
+// Reset is used to return a new [State] with all of the current state and the
+// provided [State] as the parent. In effect you can think of this as creating
+// a new state with all the output from the current state but reparented to the
+// provided state.  See [Reset] for more details.
 func (s State) Reset(s2 State) State {
 	return Reset(s2)(s)
 }
 
+// User sets the user for this state.
+// See [User] for more details.
 func (s State) User(v string) State {
 	return User(v)(s)
 }
 
+// Hostname sets the hostname for this state.
+// See [Hostname] for more details.
 func (s State) Hostname(v string) State {
 	return Hostname(v)(s)
 }
 
+// GetHostname returns the hostname set on the state.
+// See [Hostname] for more details.
 func (s State) GetHostname(ctx context.Context, co ...ConstraintsOpt) (string, error) {
 	c := &Constraints{}
 	for _, f := range co {
@@ -362,10 +399,14 @@ func (s State) GetHostname(ctx context.Context, co ...ConstraintsOpt) (string, e
 	return getHostname(s)(ctx, c)
 }
 
+// Platform sets the platform for the state. Platforms are used to determine
+// image variants to pull and run as well as the platform metadata to set on the
+// image config.
 func (s State) Platform(p ocispecs.Platform) State {
 	return platform(p)(s)
 }
 
+// GetPlatform returns the platform for the state.
 func (s State) GetPlatform(ctx context.Context, co ...ConstraintsOpt) (*ocispecs.Platform, error) {
 	c := &Constraints{}
 	for _, f := range co {
@@ -374,10 +415,14 @@ func (s State) GetPlatform(ctx context.Context, co ...ConstraintsOpt) (*ocispecs
 	return getPlatform(s)(ctx, c)
 }
 
+// Network sets the network mode for the state.
+// Network modes are used by [State.Run] to determine the network mode used when running the container.
+// Network modes are not applied to image configs.
 func (s State) Network(n pb.NetMode) State {
 	return Network(n)(s)
 }
 
+// GetNetwork returns the network mode for the state.
 func (s State) GetNetwork(ctx context.Context, co ...ConstraintsOpt) (pb.NetMode, error) {
 	c := &Constraints{}
 	for _, f := range co {
@@ -385,10 +430,15 @@ func (s State) GetNetwork(ctx context.Context, co ...ConstraintsOpt) (pb.NetMode
 	}
 	return getNetwork(s)(ctx, c)
 }
+
+// Security sets the security mode for the state.
+// Security modes are used by [State.Run] to the privileges that processes in the container will run with.
+// Security modes are not applied to image configs.
 func (s State) Security(n pb.SecurityMode) State {
 	return Security(n)(s)
 }
 
+// GetSecurity returns the security mode for the state.
 func (s State) GetSecurity(ctx context.Context, co ...ConstraintsOpt) (pb.SecurityMode, error) {
 	c := &Constraints{}
 	for _, f := range co {
@@ -397,6 +447,8 @@ func (s State) GetSecurity(ctx context.Context, co ...ConstraintsOpt) (pb.Securi
 	return getSecurity(s)(ctx, c)
 }
 
+// With applies [StateOption]s to the [State].
+// Each applied [StateOption] creates a new [State] object with the previous as its parent.
 func (s State) With(so ...StateOption) State {
 	for _, o := range so {
 		s = o(s)
@@ -404,14 +456,23 @@ func (s State) With(so ...StateOption) State {
 	return s
 }
 
+// AddExtraHost adds a host name to IP mapping to any containers created from this state.
 func (s State) AddExtraHost(host string, ip net.IP) State {
 	return extraHost(host, ip)(s)
 }
 
+// AddUlimit sets the hard/soft for the given ulimit.
+// The ulimit is applied to containers created from this state.
+// Ulimits are Linux specific and only applies to containers created from this state such as via `[State.Run]`
+// Ulimits do not apply to image configs.
 func (s State) AddUlimit(name UlimitName, soft int64, hard int64) State {
 	return ulimit(name, soft, hard)(s)
 }
 
+// WithCgroupParent sets the parent cgroup for any containers created from this state.
+// This is useful when you want to apply resource constraints to a group of containers.
+// Cgroups are Linux specific and only applies to containers created from this state such as via `[State.Run]`
+// Cgroups do not apply to image configs.
 func (s State) WithCgroupParent(cp string) State {
 	return cgroupParent(cp)(s)
 }

vendor/github.com/moby/buildkit/util/testutil/integration/containerd.go

@@ -230,6 +230,10 @@ disabled_plugins = ["cri"]
 	}, cl, nil
 }
 
+func (c *Containerd) Close() error {
+	return nil
+}
+
 func formatLogs(m map[string]*bytes.Buffer) string {
 	var ss []string
 	for k, b := range m {

vendor/github.com/moby/buildkit/util/testutil/integration/dockerd.go

@@ -219,6 +219,10 @@ func (c Moby) New(ctx context.Context, cfg *BackendConfig) (b Backend, cl func()
 	}, cl, nil
 }
 
+func (c Moby) Close() error {
+	return nil
+}
+
 func waitForAPI(ctx context.Context, apiClient *client.Client, d time.Duration) error {
 	step := 50 * time.Millisecond
 	i := 0

vendor/github.com/moby/buildkit/util/testutil/integration/oci.go

@@ -84,3 +84,7 @@ func (s *OCI) New(ctx context.Context, cfg *BackendConfig) (Backend, func() erro
 		snapshotter: s.Snapshotter,
 	}, stop, nil
 }
+
+func (s *OCI) Close() error {
+	return nil
+}

vendor/github.com/moby/buildkit/util/testutil/integration/run.go

@@ -62,6 +62,7 @@ type BackendConfig struct {
 
 type Worker interface {
 	New(context.Context, *BackendConfig) (Backend, func() error, error)
+	Close() error
 	Name() string
 	Rootless() bool
 }
@@ -168,6 +169,11 @@ func Run(t *testing.T, testCases []Test, opt ...TestOpt) {
 		rng := rand.New(rand.NewSource(time.Now().UnixNano())) //nolint:gosec // using math/rand is fine in a test utility
 		list = []Worker{list[rng.Intn(len(list))]}
 	}
+	t.Cleanup(func() {
+		for _, br := range list {
+			_ = br.Close()
+		}
+	})
 
 	for _, br := range list {
 		for _, tc := range testCases {

vendor/modules.txt

@@ -498,7 +498,7 @@ github.com/mitchellh/go-wordwrap
 # github.com/mitchellh/mapstructure v1.5.0
 ## explicit; go 1.14
 github.com/mitchellh/mapstructure
-# github.com/moby/buildkit v0.11.0-rc3.0.20230609092854-67a08623b95a
+# github.com/moby/buildkit v0.11.0-rc3.0.20230620112432-2d91ddcceedc
 ## explicit; go 1.20
 github.com/moby/buildkit/api/services/control
 github.com/moby/buildkit/api/types