mirror of https://github.com/docker/buildx.git
Merge pull request #2242 from crazy-max/bake-ulimits-shmsize
bake: ulimits and shm-size support
This commit is contained in:
Tõnis Tiigi
GitHub
commit
daaa8f2482
29
bake/bake.go
29
bake/bake.go
|
|
@ -21,6 +21,7 @@ import (
|
|||
"github.com/docker/buildx/util/platformutil"
|
||||
"github.com/docker/buildx/util/progress"
|
||||
"github.com/docker/cli/cli/config"
|
||||
dockeropts "github.com/docker/cli/opts"
|
||||
hcl "github.com/hashicorp/hcl/v2"
|
||||
"github.com/moby/buildkit/client"
|
||||
"github.com/moby/buildkit/client/llb"
|
||||
|
|
@ -699,6 +700,8 @@ type Target struct {
|
|||
NoCache *bool `json:"no-cache,omitempty" hcl:"no-cache,optional" cty:"no-cache"`
|
||||
NetworkMode *string `json:"-" hcl:"-" cty:"-"`
|
||||
NoCacheFilter []string `json:"no-cache-filter,omitempty" hcl:"no-cache-filter,optional" cty:"no-cache-filter"`
|
||||
ShmSize *string `json:"shm-size,omitempty" hcl:"shm-size,optional"`
|
||||
Ulimits []string `json:"ulimits,omitempty" hcl:"ulimits,optional"`
|
||||
// IMPORTANT: if you add more fields here, do not forget to update newOverrides and docs/bake-reference.md.
|
||||
|
||||
// linked is a private field to mark a target used as a linked one
|
||||
|
|
@ -721,6 +724,7 @@ func (t *Target) normalize() {
|
|||
t.CacheTo = removeDupes(t.CacheTo)
|
||||
t.Outputs = removeDupes(t.Outputs)
|
||||
t.NoCacheFilter = removeDupes(t.NoCacheFilter)
|
||||
t.Ulimits = removeDupes(t.Ulimits)
|
||||
|
||||
for k, v := range t.Contexts {
|
||||
if v == "" {
|
||||
|
|
@ -809,6 +813,12 @@ func (t *Target) Merge(t2 *Target) {
|
|||
if t2.NoCacheFilter != nil { // merge
|
||||
t.NoCacheFilter = append(t.NoCacheFilter, t2.NoCacheFilter...)
|
||||
}
|
||||
if t2.ShmSize != nil { // no merge
|
||||
t.ShmSize = t2.ShmSize
|
||||
}
|
||||
if t2.Ulimits != nil { // merge
|
||||
t.Ulimits = append(t.Ulimits, t2.Ulimits...)
|
||||
}
|
||||
t.Inherits = append(t.Inherits, t2.Inherits...)
|
||||
}
|
||||
|
||||
|
|
@ -873,6 +883,10 @@ func (t *Target) AddOverrides(overrides map[string]Override) error {
|
|||
t.NoCache = &noCache
|
||||
case "no-cache-filter":
|
||||
t.NoCacheFilter = o.ArrValue
|
||||
case "shm-size":
|
||||
t.ShmSize = &value
|
||||
case "ulimits":
|
||||
t.Ulimits = o.ArrValue
|
||||
case "pull":
|
||||
pull, err := strconv.ParseBool(value)
|
||||
if err != nil {
|
||||
|
|
@ -1233,6 +1247,12 @@ func toBuildOpt(t *Target, inp *Input) (*build.Options, error) {
|
|||
if t.NetworkMode != nil {
|
||||
networkMode = *t.NetworkMode
|
||||
}
|
||||
shmSize := new(dockeropts.MemBytes)
|
||||
if t.ShmSize != nil {
|
||||
if err := shmSize.Set(*t.ShmSize); err != nil {
|
||||
return nil, errors.Errorf("invalid value %s for membytes key shm-size", *t.ShmSize)
|
||||
}
|
||||
}
|
||||
|
||||
bo := &build.Options{
|
||||
Inputs: bi,
|
||||
|
|
@ -1244,6 +1264,7 @@ func toBuildOpt(t *Target, inp *Input) (*build.Options, error) {
|
|||
Pull: pull,
|
||||
NetworkMode: networkMode,
|
||||
Linked: t.linked,
|
||||
ShmSize: *shmSize,
|
||||
}
|
||||
|
||||
platforms, err := platformutil.Parse(t.Platforms)
|
||||
|
|
@ -1321,6 +1342,14 @@ func toBuildOpt(t *Target, inp *Input) (*build.Options, error) {
|
|||
return nil, err
|
||||
}
|
||||
|
||||
ulimits := dockeropts.NewUlimitOpt(nil)
|
||||
for _, field := range t.Ulimits {
|
||||
if err := ulimits.Set(field); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
bo.Ulimits = ulimits
|
||||
|
||||
return bo, nil
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -22,6 +22,8 @@ target "webDEP" {
|
|||
VAR_BOTH = "webDEP"
|
||||
}
|
||||
no-cache = true
|
||||
shm-size = "128m"
|
||||
ulimits = ["nofile=1024:1024"]
|
||||
}
|
||||
|
||||
target "webapp" {
|
||||
|
|
@ -45,6 +47,8 @@ target "webapp" {
|
|||
require.Equal(t, ".", *m["webapp"].Context)
|
||||
require.Equal(t, ptrstr("webDEP"), m["webapp"].Args["VAR_INHERITED"])
|
||||
require.Equal(t, true, *m["webapp"].NoCache)
|
||||
require.Equal(t, "128m", *m["webapp"].ShmSize)
|
||||
require.Equal(t, []string{"nofile=1024:1024"}, m["webapp"].Ulimits)
|
||||
require.Nil(t, m["webapp"].Pull)
|
||||
|
||||
require.Equal(t, 1, len(g))
|
||||
|
|
@ -129,6 +133,12 @@ target "webapp" {
|
|||
require.Equal(t, []string{"webapp"}, g["default"].Targets)
|
||||
})
|
||||
|
||||
t.Run("ShmSizeOverride", func(t *testing.T) {
|
||||
m, _, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, []string{"webapp.shm-size=256m"}, nil)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "256m", *m["webapp"].ShmSize)
|
||||
})
|
||||
|
||||
t.Run("PullOverride", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
m, g, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, []string{"webapp.pull=false"}, nil)
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ package bake
|
|||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
|
@ -9,6 +10,8 @@ import (
|
|||
"github.com/compose-spec/compose-go/v2/dotenv"
|
||||
"github.com/compose-spec/compose-go/v2/loader"
|
||||
composetypes "github.com/compose-spec/compose-go/v2/types"
|
||||
dockeropts "github.com/docker/cli/opts"
|
||||
"github.com/docker/go-units"
|
||||
"github.com/pkg/errors"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
|
@ -86,6 +89,24 @@ func ParseCompose(cfgs []composetypes.ConfigFile, envs map[string]string) (*Conf
|
|||
}
|
||||
}
|
||||
|
||||
var shmSize *string
|
||||
if s.Build.ShmSize > 0 {
|
||||
shmSizeBytes := dockeropts.MemBytes(s.Build.ShmSize)
|
||||
shmSizeStr := shmSizeBytes.String()
|
||||
shmSize = &shmSizeStr
|
||||
}
|
||||
|
||||
var ulimits []string
|
||||
if s.Build.Ulimits != nil {
|
||||
for n, u := range s.Build.Ulimits {
|
||||
ulimit, err := units.ParseUlimit(fmt.Sprintf("%s=%d:%d", n, u.Soft, u.Hard))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ulimits = append(ulimits, ulimit.String())
|
||||
}
|
||||
}
|
||||
|
||||
var secrets []string
|
||||
for _, bs := range s.Build.Secrets {
|
||||
secret, err := composeToBuildkitSecret(bs, cfg.Secrets[bs.Source])
|
||||
|
|
@ -122,6 +143,8 @@ func ParseCompose(cfgs []composetypes.ConfigFile, envs map[string]string) (*Conf
|
|||
CacheTo: s.Build.CacheTo,
|
||||
NetworkMode: &s.Build.Network,
|
||||
Secrets: secrets,
|
||||
ShmSize: shmSize,
|
||||
Ulimits: ulimits,
|
||||
}
|
||||
if err = t.composeExtTarget(s.Build.Extensions); err != nil {
|
||||
return nil, err
|
||||
|
|
|
|||
|
|
@ -303,6 +303,11 @@ services:
|
|||
args:
|
||||
CT_ECR: foo
|
||||
CT_TAG: bar
|
||||
shm_size: 128m
|
||||
ulimits:
|
||||
nofile:
|
||||
soft: 1024
|
||||
hard: 1024
|
||||
x-bake:
|
||||
secret:
|
||||
- id=mysecret,src=/local/secret
|
||||
|
|
@ -332,6 +337,8 @@ services:
|
|||
require.Equal(t, []string{"linux/arm64"}, c.Targets[1].Platforms)
|
||||
require.Equal(t, []string{"type=docker"}, c.Targets[1].Outputs)
|
||||
require.Equal(t, newBool(true), c.Targets[1].NoCache)
|
||||
require.Equal(t, ptrstr("128MiB"), c.Targets[1].ShmSize)
|
||||
require.Equal(t, []string{"nofile=1024:1024"}, c.Targets[1].Ulimits)
|
||||
}
|
||||
|
||||
func TestComposeExtDedup(t *testing.T) {
|
||||
|
|
|
|||
|
|
@ -599,7 +599,7 @@ func buildCmd(dockerCli command.Cli, rootOpts *rootOptions, debugConfig *debug.D
|
|||
|
||||
flags.StringArrayVar(&options.secrets, "secret", []string{}, `Secret to expose to the build (format: "id=mysecret[,src=/local/secret]")`)
|
||||
|
||||
flags.Var(&options.shmSize, "shm-size", `Size of "/dev/shm"`)
|
||||
flags.Var(&options.shmSize, "shm-size", `Shared memory size for build containers`)
|
||||
|
||||
flags.StringArrayVar(&options.ssh, "ssh", []string{}, `SSH agent socket or keys to expose to the build (format: "default|<id>[=<socket>|<key>[,<key>]]")`)
|
||||
|
||||
|
|
|
|||
|
|
@ -213,7 +213,7 @@ target "webapp" {
|
|||
The following table shows the complete list of attributes that you can assign to a target:
|
||||
|
||||
| Name | Type | Description |
|
||||
| ----------------------------------------------- | ------- | -------------------------------------------------------------------- |
|
||||
|-------------------------------------------------|---------|----------------------------------------------------------------------|
|
||||
| [`args`](#targetargs) | Map | Build arguments |
|
||||
| [`annotations`](#targetannotations) | List | Exporter annotations |
|
||||
| [`attest`](#targetattest) | List | Build attestations |
|
||||
|
|
@ -233,9 +233,11 @@ The following table shows the complete list of attributes that you can assign to
|
|||
| [`platforms`](#targetplatforms) | List | Target platforms |
|
||||
| [`pull`](#targetpull) | Boolean | Always pull images |
|
||||
| [`secret`](#targetsecret) | List | Secrets to expose to the build |
|
||||
| [`shm-size`](#targetshm-size) | List | Size of `/dev/shm` |
|
||||
| [`ssh`](#targetssh) | List | SSH agent sockets or keys to expose to the build |
|
||||
| [`tags`](#targettags) | List | Image names and tags |
|
||||
| [`target`](#targettarget) | String | Target build stage |
|
||||
| [`ulimits`](#targetulimits) | List | Ulimit options |
|
||||
|
||||
### `target.args`
|
||||
|
||||
|
|
@ -832,6 +834,29 @@ RUN --mount=type=secret,id=KUBECONFIG \
|
|||
KUBECONFIG=$(cat /run/secrets/KUBECONFIG) helm upgrade --install
|
||||
```
|
||||
|
||||
### `target.shm-size`
|
||||
|
||||
Sets the size of the shared memory allocated for build containers when using
|
||||
`RUN` instructions.
|
||||
|
||||
The format is `<number><unit>`. `number` must be greater than `0`. Unit is
|
||||
optional and can be `b` (bytes), `k` (kilobytes), `m` (megabytes), or `g`
|
||||
(gigabytes). If you omit the unit, the system uses bytes.
|
||||
|
||||
This is the same as the `--shm-size` flag for `docker build`.
|
||||
|
||||
```hcl
|
||||
target "default" {
|
||||
shm-size = "128m"
|
||||
}
|
||||
```
|
||||
|
||||
> **Note**
|
||||
>
|
||||
> In most cases, it is recommended to let the builder automatically determine
|
||||
> the appropriate configurations. Manual adjustments should only be considered
|
||||
> when specific performance tuning is required for complex build scenarios.
|
||||
|
||||
### `target.ssh`
|
||||
|
||||
Defines SSH agent sockets or keys to expose to the build.
|
||||
|
|
@ -878,6 +903,32 @@ target "default" {
|
|||
}
|
||||
```
|
||||
|
||||
### `target.ulimits`
|
||||
|
||||
Ulimits overrides the default ulimits of build's containers when using `RUN`
|
||||
instructions and are specified with a soft and hard limit as such:
|
||||
`<type>=<soft limit>[:<hard limit>]`, for example:
|
||||
|
||||
```hcl
|
||||
target "app" {
|
||||
ulimits = [
|
||||
"nofile=1024:1024"
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
> **Note**
|
||||
>
|
||||
> If you do not provide a `hard limit`, the `soft limit` is used
|
||||
> for both values. If no `ulimits` are set, they are inherited from
|
||||
> the default `ulimits` set on the daemon.
|
||||
|
||||
> **Note**
|
||||
>
|
||||
> In most cases, it is recommended to let the builder automatically determine
|
||||
> the appropriate configurations. Manual adjustments should only be considered
|
||||
> when specific performance tuning is required for complex build scenarios.
|
||||
|
||||
## Group
|
||||
|
||||
Groups allow you to invoke multiple builds (targets) at once.
|
||||
|
|
|
|||
|
|
@ -46,7 +46,7 @@ Start a build
|
|||
| [`--sbom`](#sbom) | `string` | | Shorthand for `--attest=type=sbom` |
|
||||
| [`--secret`](#secret) | `stringArray` | | Secret to expose to the build (format: `id=mysecret[,src=/local/secret]`) |
|
||||
| `--server-config` | `string` | | Specify buildx server config file (used only when launching new server) (EXPERIMENTAL) |
|
||||
| [`--shm-size`](#shm-size) | `bytes` | `0` | Size of `/dev/shm` |
|
||||
| [`--shm-size`](#shm-size) | `bytes` | `0` | Shared memory size for build containers |
|
||||
| [`--ssh`](#ssh) | `stringArray` | | SSH agent socket or keys to expose to the build (format: `default\|<id>[=<socket>\|<key>[,<key>]]`) |
|
||||
| [`-t`](https://docs.docker.com/reference/cli/docker/image/build/#tag), [`--tag`](https://docs.docker.com/reference/cli/docker/image/build/#tag) | `stringArray` | | Name and optionally a tag (format: `name:tag`) |
|
||||
| [`--target`](https://docs.docker.com/reference/cli/docker/image/build/#target) | `string` | | Set the target build stage to build |
|
||||
|
|
@ -653,12 +653,21 @@ RUN --mount=type=bind,target=. \
|
|||
$ SECRET_TOKEN=token docker buildx build --secret id=SECRET_TOKEN .
|
||||
```
|
||||
|
||||
### <a name="shm-size"></a> Size of /dev/shm (--shm-size)
|
||||
### <a name="shm-size"></a> Shared memory size for build containers (--shm-size)
|
||||
|
||||
Sets the size of the shared memory allocated for build containers when using
|
||||
`RUN` instructions.
|
||||
|
||||
The format is `<number><unit>`. `number` must be greater than `0`. Unit is
|
||||
optional and can be `b` (bytes), `k` (kilobytes), `m` (megabytes), or `g`
|
||||
(gigabytes). If you omit the unit, the system uses bytes.
|
||||
|
||||
> **Note**
|
||||
>
|
||||
> In most cases, it is recommended to let the builder automatically determine
|
||||
> the appropriate configurations. Manual adjustments should only be considered
|
||||
> when specific performance tuning is required for complex build scenarios.
|
||||
|
||||
### <a name="ssh"></a> SSH agent socket or keys to expose to the build (--ssh)
|
||||
|
||||
```text
|
||||
|
|
@ -692,7 +701,8 @@ $ docker buildx build --ssh default=$SSH_AUTH_SOCK .
|
|||
|
||||
### <a name="ulimit"></a> Set ulimits (--ulimit)
|
||||
|
||||
`--ulimit` is specified with a soft and hard limit as such:
|
||||
`--ulimit` overrides the default ulimits of build's containers when using `RUN`
|
||||
instructions and are specified with a soft and hard limit as such:
|
||||
`<type>=<soft limit>[:<hard limit>]`, for example:
|
||||
|
||||
```console
|
||||
|
|
@ -704,3 +714,9 @@ $ docker buildx build --ulimit nofile=1024:1024 .
|
|||
> If you don't provide a `hard limit`, the `soft limit` is used
|
||||
> for both values. If no `ulimits` are set, they're inherited from
|
||||
> the default `ulimits` set on the daemon.
|
||||
|
||||
> **Note**
|
||||
>
|
||||
> In most cases, it is recommended to let the builder automatically determine
|
||||
> the appropriate configurations. Manual adjustments should only be considered
|
||||
> when specific performance tuning is required for complex build scenarios.
|
||||
|
|
|
|||
|
|
@ -42,7 +42,7 @@ Start a build
|
|||
| `--sbom` | `string` | | Shorthand for `--attest=type=sbom` |
|
||||
| `--secret` | `stringArray` | | Secret to expose to the build (format: `id=mysecret[,src=/local/secret]`) |
|
||||
| `--server-config` | `string` | | Specify buildx server config file (used only when launching new server) (EXPERIMENTAL) |
|
||||
| `--shm-size` | `bytes` | `0` | Size of `/dev/shm` |
|
||||
| `--shm-size` | `bytes` | `0` | Shared memory size for build containers |
|
||||
| `--ssh` | `stringArray` | | SSH agent socket or keys to expose to the build (format: `default\|<id>[=<socket>\|<key>[,<key>]]`) |
|
||||
| [`-t`](https://docs.docker.com/reference/cli/docker/image/build/#tag), [`--tag`](https://docs.docker.com/reference/cli/docker/image/build/#tag) | `stringArray` | | Name and optionally a tag (format: `name:tag`) |
|
||||
| [`--target`](https://docs.docker.com/reference/cli/docker/image/build/#target) | `string` | | Set the target build stage to build |
|
||||
|
|
|
|||
|
|
@ -32,6 +32,8 @@ var bakeTests = []func(t *testing.T, sb integration.Sandbox){
|
|||
testBakeRemoteDockerfileCwd,
|
||||
testBakeRemoteLocalContextRemoteDockerfile,
|
||||
testBakeEmpty,
|
||||
testBakeShmSize,
|
||||
testBakeUlimits,
|
||||
}
|
||||
|
||||
func testBakeLocal(t *testing.T, sb integration.Sandbox) {
|
||||
|
|
@ -54,7 +56,7 @@ target "default" {
|
|||
|
||||
cmd := buildxCmd(sb, withDir(dir), withArgs("bake", "--progress=plain", "--set", "*.output=type=local,dest="+dirDest))
|
||||
out, err := cmd.CombinedOutput()
|
||||
require.NoError(t, err, out)
|
||||
require.NoError(t, err, string(out))
|
||||
require.Contains(t, string(out), `#1 [internal] load local bake definitions`)
|
||||
require.Contains(t, string(out), `#1 reading docker-bake.hcl`)
|
||||
|
||||
|
|
@ -520,3 +522,67 @@ func testBakeEmpty(t *testing.T, sb integration.Sandbox) {
|
|||
require.Error(t, err, out)
|
||||
require.Contains(t, out, "couldn't find a bake definition")
|
||||
}
|
||||
|
||||
func testBakeShmSize(t *testing.T, sb integration.Sandbox) {
|
||||
dockerfile := []byte(`
|
||||
FROM busybox AS build
|
||||
RUN mount | grep /dev/shm > /shmsize
|
||||
FROM scratch
|
||||
COPY --from=build /shmsize /
|
||||
`)
|
||||
bakefile := []byte(`
|
||||
target "default" {
|
||||
shm-size = "128m"
|
||||
}
|
||||
`)
|
||||
dir := tmpdir(
|
||||
t,
|
||||
fstest.CreateFile("docker-bake.hcl", bakefile, 0600),
|
||||
fstest.CreateFile("Dockerfile", dockerfile, 0600),
|
||||
)
|
||||
|
||||
dirDest := t.TempDir()
|
||||
|
||||
out, err := bakeCmd(
|
||||
sb,
|
||||
withDir(dir),
|
||||
withArgs("--set", "*.output=type=local,dest="+dirDest),
|
||||
)
|
||||
require.NoError(t, err, out)
|
||||
|
||||
dt, err := os.ReadFile(filepath.Join(dirDest, "shmsize"))
|
||||
require.NoError(t, err)
|
||||
require.Contains(t, string(dt), `size=131072k`)
|
||||
}
|
||||
|
||||
func testBakeUlimits(t *testing.T, sb integration.Sandbox) {
|
||||
dockerfile := []byte(`
|
||||
FROM busybox AS build
|
||||
RUN ulimit -n > first > /ulimit
|
||||
FROM scratch
|
||||
COPY --from=build /ulimit /
|
||||
`)
|
||||
bakefile := []byte(`
|
||||
target "default" {
|
||||
ulimits = ["nofile=1024:1024"]
|
||||
}
|
||||
`)
|
||||
dir := tmpdir(
|
||||
t,
|
||||
fstest.CreateFile("docker-bake.hcl", bakefile, 0600),
|
||||
fstest.CreateFile("Dockerfile", dockerfile, 0600),
|
||||
)
|
||||
|
||||
dirDest := t.TempDir()
|
||||
|
||||
out, err := bakeCmd(
|
||||
sb,
|
||||
withDir(dir),
|
||||
withArgs("--set", "*.output=type=local,dest="+dirDest),
|
||||
)
|
||||
require.NoError(t, err, out)
|
||||
|
||||
dt, err := os.ReadFile(filepath.Join(dirDest, "ulimit"))
|
||||
require.NoError(t, err)
|
||||
require.Contains(t, string(dt), `1024`)
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue