From 0808747add96562cd425396c6dd162f910a4bd9d Mon Sep 17 00:00:00 2001 From: Tim Smith Date: Tue, 28 May 2024 23:01:14 -0400 Subject: [PATCH 1/6] Added application/vnd.docker.distribution.manifest.v2+json mediatype to the list of mediatypes we return the original bytes for when calling *Resolver.Combine rather than adding it to a newly created manifest list Signed-off-by: Tim Smith --- tests/imagetools.go | 36 +++++++++++++----------------------- util/imagetools/create.go | 2 +- 2 files changed, 14 insertions(+), 24 deletions(-) diff --git a/tests/imagetools.go b/tests/imagetools.go index b6d21b49..cca1c75b 100644 --- a/tests/imagetools.go +++ b/tests/imagetools.go @@ -61,23 +61,7 @@ func testImagetoolsCopyManifest(t *testing.T, sb integration.Sandbox) { var idx2 ocispecs.Index err = json.Unmarshal(dt, &idx2) require.NoError(t, err) - require.Equal(t, images.MediaTypeDockerSchema2ManifestList, idx2.MediaType) - require.Equal(t, 1, len(idx2.Manifests)) - - cmd = buildxCmd(sb, withArgs("imagetools", "inspect", target2+"@"+string(idx2.Manifests[0].Digest), "--raw")) - dt, err = cmd.CombinedOutput() - require.NoError(t, err, string(dt)) - - var mfst2 ocispecs.Manifest - err = json.Unmarshal(dt, &mfst2) - require.NoError(t, err) - require.Equal(t, images.MediaTypeDockerSchema2Manifest, mfst2.MediaType) - - require.Equal(t, mfst.Config.Digest, mfst2.Config.Digest) - require.Equal(t, len(mfst.Layers), len(mfst2.Layers)) - for i := range mfst.Layers { - require.Equal(t, mfst.Layers[i].Digest, mfst2.Layers[i].Digest) - } + require.Equal(t, images.MediaTypeDockerSchema2Manifest, idx2.MediaType) } func testImagetoolsCopyIndex(t *testing.T, sb integration.Sandbox) { @@ -161,7 +145,15 @@ func testImagetoolsInspectAndFilter(t *testing.T, sb integration.Sandbox) { mfst = idx.Manifests[1] require.Equal(t, "linux/arm64", platforms.Format(*mfst.Platform)) - // create amd64 only image + cmd = buildxCmd(sb, withArgs("imagetools", "inspect", target+"@"+string(idx.Manifests[1].Digest), "--raw")) + dt, err = cmd.CombinedOutput() + require.NoError(t, err, string(dt)) + + var armMfst ocispecs.Manifest + err = json.Unmarshal(dt, &armMfst) + require.NoError(t, err) + + // create arm64 only image cmd = buildxCmd(sb, withArgs("imagetools", "create", "-t", target+"-arm64", target+"@"+string(idx.Manifests[1].Digest))) dt, err = cmd.CombinedOutput() require.NoError(t, err, string(dt)) @@ -170,14 +162,12 @@ func testImagetoolsInspectAndFilter(t *testing.T, sb integration.Sandbox) { dt, err = cmd.CombinedOutput() require.NoError(t, err, string(dt)) - var idx2 ocispecs.Index + var idx2 ocispecs.Manifest err = json.Unmarshal(dt, &idx2) require.NoError(t, err) - require.Equal(t, 1, len(idx2.Manifests)) - - require.Equal(t, idx.Manifests[1].Digest, idx2.Manifests[0].Digest) - require.Equal(t, platforms.Format(*idx.Manifests[1].Platform), platforms.Format(*idx2.Manifests[0].Platform)) + require.Equal(t, armMfst.Config.Digest, idx2.Config.Digest) + require.Equal(t, armMfst.MediaType, idx2.MediaType) } func testImagetoolsAnnotation(t *testing.T, sb integration.Sandbox) { diff --git a/util/imagetools/create.go b/util/imagetools/create.go index d1e8dfaa..a8c3241d 100644 --- a/util/imagetools/create.go +++ b/util/imagetools/create.go @@ -79,7 +79,7 @@ func (r *Resolver) Combine(ctx context.Context, srcs []*Source, ann []string) ([ // on single source, return original bytes if len(srcs) == 1 && len(ann) == 0 { - if mt := srcs[0].Desc.MediaType; mt == images.MediaTypeDockerSchema2ManifestList || mt == ocispec.MediaTypeImageIndex { + if mt := srcs[0].Desc.MediaType; mt == images.MediaTypeDockerSchema2ManifestList || mt == images.MediaTypeDockerSchema2Manifest || mt == ocispec.MediaTypeImageIndex { return dts[0], srcs[0].Desc, nil } } From abf6c77d919f181e1600f0f42aef9cc9fa6be416 Mon Sep 17 00:00:00 2001 From: Tim Smith Date: Wed, 29 May 2024 14:07:28 -0400 Subject: [PATCH 2/6] Add a --prefer-index flag that allows you to specify the preferred behavior when deciding on how to create an image/manifest from a single source. Signed-off-by: Tim Smith --- build/build.go | 2 +- commands/imagetools/create.go | 4 +++- util/imagetools/create.go | 9 +++++++-- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/build/build.go b/build/build.go index 3fd2e581..42ec32a1 100644 --- a/build/build.go +++ b/build/build.go @@ -608,7 +608,7 @@ func BuildWithResultHandler(ctx context.Context, nodes []builder.Node, opt map[s } } - dt, desc, err := itpull.Combine(ctx, srcs, nil) + dt, desc, err := itpull.Combine(ctx, srcs, nil, false) if err != nil { return err } diff --git a/commands/imagetools/create.go b/commands/imagetools/create.go index f05571b0..5239bd0f 100644 --- a/commands/imagetools/create.go +++ b/commands/imagetools/create.go @@ -29,6 +29,7 @@ type createOptions struct { dryrun bool actionAppend bool progress string + preferIndex bool } func runCreate(ctx context.Context, dockerCli command.Cli, in createOptions, args []string) error { @@ -153,7 +154,7 @@ func runCreate(ctx context.Context, dockerCli command.Cli, in createOptions, arg } } - dt, desc, err := r.Combine(ctx, srcs, in.annotations) + dt, desc, err := r.Combine(ctx, srcs, in.annotations, in.preferIndex) if err != nil { return err } @@ -283,6 +284,7 @@ func createCmd(dockerCli command.Cli, opts RootOptions) *cobra.Command { flags.BoolVar(&options.actionAppend, "append", false, "Append to existing manifest") flags.StringVar(&options.progress, "progress", "auto", `Set type of progress output ("auto", "plain", "tty"). Use plain to show container output`) flags.StringArrayVarP(&options.annotations, "annotation", "", []string{}, "Add annotation to the image") + flags.BoolVar(&options.preferIndex, "prefer-index", true, "When only a single source is specified, prefer outputting an image index or manifest list instead of performing a carbon copy") return cmd } diff --git a/util/imagetools/create.go b/util/imagetools/create.go index a8c3241d..fa168b1a 100644 --- a/util/imagetools/create.go +++ b/util/imagetools/create.go @@ -29,7 +29,7 @@ type Source struct { Ref reference.Named } -func (r *Resolver) Combine(ctx context.Context, srcs []*Source, ann []string) ([]byte, ocispec.Descriptor, error) { +func (r *Resolver) Combine(ctx context.Context, srcs []*Source, ann []string, preferIndex bool) ([]byte, ocispec.Descriptor, error) { eg, ctx := errgroup.WithContext(ctx) dts := make([][]byte, len(srcs)) @@ -79,7 +79,12 @@ func (r *Resolver) Combine(ctx context.Context, srcs []*Source, ann []string) ([ // on single source, return original bytes if len(srcs) == 1 && len(ann) == 0 { - if mt := srcs[0].Desc.MediaType; mt == images.MediaTypeDockerSchema2ManifestList || mt == images.MediaTypeDockerSchema2Manifest || mt == ocispec.MediaTypeImageIndex { + switch srcs[0].Desc.MediaType { + case images.MediaTypeDockerSchema2Manifest: + if !preferIndex { + return dts[0], srcs[0].Desc, nil + } + case images.MediaTypeDockerSchema2ManifestList, ocispec.MediaTypeImageIndex: return dts[0], srcs[0].Desc, nil } } From 2061550bc178afd85b02fc74304d93ef9ad1b2bc Mon Sep 17 00:00:00 2001 From: Tim Smith Date: Wed, 29 May 2024 14:20:53 -0400 Subject: [PATCH 3/6] Slightly refactored the mediaType check on single source so that now we return original bytes without filtering on mediaType, based on the preferIndex preference. Signed-off-by: Tim Smith --- util/imagetools/create.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/util/imagetools/create.go b/util/imagetools/create.go index fa168b1a..71853e07 100644 --- a/util/imagetools/create.go +++ b/util/imagetools/create.go @@ -80,12 +80,15 @@ func (r *Resolver) Combine(ctx context.Context, srcs []*Source, ann []string, pr // on single source, return original bytes if len(srcs) == 1 && len(ann) == 0 { switch srcs[0].Desc.MediaType { - case images.MediaTypeDockerSchema2Manifest: + // if the source is already an image index or manifest list, there is no need to consider the value + // of preferIndex since if set to true then the source is already in the preferred format, and if false + // it doesn't matter since we're not going to split it into separate manifests + case images.MediaTypeDockerSchema2ManifestList, ocispec.MediaTypeImageIndex: + return dts[0], srcs[0].Desc, nil + default: if !preferIndex { return dts[0], srcs[0].Desc, nil } - case images.MediaTypeDockerSchema2ManifestList, ocispec.MediaTypeImageIndex: - return dts[0], srcs[0].Desc, nil } } From 388af3576a3fad78be79afac44a122abb3e77ac4 Mon Sep 17 00:00:00 2001 From: Tim Smith Date: Wed, 29 May 2024 21:39:14 -0400 Subject: [PATCH 4/6] Updated tests to test new --prefer-index flag Signed-off-by: Tim Smith --- tests/imagetools.go | 49 +++++++++++++++++++++++++++++++++------------ 1 file changed, 36 insertions(+), 13 deletions(-) diff --git a/tests/imagetools.go b/tests/imagetools.go index cca1c75b..2f423acb 100644 --- a/tests/imagetools.go +++ b/tests/imagetools.go @@ -61,7 +61,36 @@ func testImagetoolsCopyManifest(t *testing.T, sb integration.Sandbox) { var idx2 ocispecs.Index err = json.Unmarshal(dt, &idx2) require.NoError(t, err) - require.Equal(t, images.MediaTypeDockerSchema2Manifest, idx2.MediaType) + require.Equal(t, images.MediaTypeDockerSchema2ManifestList, idx2.MediaType) + require.Equal(t, 1, len(idx2.Manifests)) + + cmd = buildxCmd(sb, withArgs("imagetools", "inspect", target2+"@"+string(idx2.Manifests[0].Digest), "--raw")) + dt, err = cmd.CombinedOutput() + require.NoError(t, err, string(dt)) + + var mfst2 ocispecs.Manifest + err = json.Unmarshal(dt, &mfst2) + require.NoError(t, err) + require.Equal(t, images.MediaTypeDockerSchema2Manifest, mfst2.MediaType) + + require.Equal(t, mfst.Config.Digest, mfst2.Config.Digest) + require.Equal(t, len(mfst.Layers), len(mfst2.Layers)) + for i := range mfst.Layers { + require.Equal(t, mfst.Layers[i].Digest, mfst2.Layers[i].Digest) + } + + cmd = buildxCmd(sb, withArgs("imagetools", "create", "--prefer-index=false", "-t", target2+"-not-index", target)) + dt, err = cmd.CombinedOutput() + require.NoError(t, err, string(dt)) + + cmd = buildxCmd(sb, withArgs("imagetools", "inspect", target2+"-not-index", "--raw")) + dt, err = cmd.CombinedOutput() + require.NoError(t, err, string(dt)) + + var idx3 ocispecs.Manifest + err = json.Unmarshal(dt, &idx3) + require.NoError(t, err) + require.Equal(t, images.MediaTypeDockerSchema2Manifest, idx3.MediaType) } func testImagetoolsCopyIndex(t *testing.T, sb integration.Sandbox) { @@ -145,15 +174,7 @@ func testImagetoolsInspectAndFilter(t *testing.T, sb integration.Sandbox) { mfst = idx.Manifests[1] require.Equal(t, "linux/arm64", platforms.Format(*mfst.Platform)) - cmd = buildxCmd(sb, withArgs("imagetools", "inspect", target+"@"+string(idx.Manifests[1].Digest), "--raw")) - dt, err = cmd.CombinedOutput() - require.NoError(t, err, string(dt)) - - var armMfst ocispecs.Manifest - err = json.Unmarshal(dt, &armMfst) - require.NoError(t, err) - - // create arm64 only image + // create amd64 only image cmd = buildxCmd(sb, withArgs("imagetools", "create", "-t", target+"-arm64", target+"@"+string(idx.Manifests[1].Digest))) dt, err = cmd.CombinedOutput() require.NoError(t, err, string(dt)) @@ -162,12 +183,14 @@ func testImagetoolsInspectAndFilter(t *testing.T, sb integration.Sandbox) { dt, err = cmd.CombinedOutput() require.NoError(t, err, string(dt)) - var idx2 ocispecs.Manifest + var idx2 ocispecs.Index err = json.Unmarshal(dt, &idx2) require.NoError(t, err) - require.Equal(t, armMfst.Config.Digest, idx2.Config.Digest) - require.Equal(t, armMfst.MediaType, idx2.MediaType) + require.Equal(t, 1, len(idx2.Manifests)) + + require.Equal(t, idx.Manifests[1].Digest, idx2.Manifests[0].Digest) + require.Equal(t, platforms.Format(*idx.Manifests[1].Platform), platforms.Format(*idx2.Manifests[0].Platform)) } func testImagetoolsAnnotation(t *testing.T, sb integration.Sandbox) { From 117c9016e1dc4e851c4fb6db6d41b1ef02237331 Mon Sep 17 00:00:00 2001 From: Tim Smith Date: Wed, 29 May 2024 21:56:22 -0400 Subject: [PATCH 5/6] Updated tests further to make sure the new flag doesn't affect copy an index regardless of what value you specify. Signed-off-by: Tim Smith --- tests/imagetools.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/imagetools.go b/tests/imagetools.go index 2f423acb..aa378bda 100644 --- a/tests/imagetools.go +++ b/tests/imagetools.go @@ -140,6 +140,24 @@ func testImagetoolsCopyIndex(t *testing.T, sb integration.Sandbox) { for i := range idx.Manifests { require.Equal(t, idx.Manifests[i].Digest, idx2.Manifests[i].Digest) } + + cmd = buildxCmd(sb, withArgs("imagetools", "create", "--prefer-index=false", "-t", target2+"-still-index", target)) + dt, err = cmd.CombinedOutput() + require.NoError(t, err, string(dt)) + + cmd = buildxCmd(sb, withArgs("imagetools", "inspect", target2+"-still-index", "--raw")) + dt, err = cmd.CombinedOutput() + require.NoError(t, err, string(dt)) + + var idx3 ocispecs.Index + err = json.Unmarshal(dt, &idx3) + require.NoError(t, err) + require.Equal(t, images.MediaTypeDockerSchema2ManifestList, idx3.MediaType) + + require.Equal(t, len(idx.Manifests), len(idx3.Manifests)) + for i := range idx.Manifests { + require.Equal(t, idx.Manifests[i].Digest, idx3.Manifests[i].Digest) + } } func testImagetoolsInspectAndFilter(t *testing.T, sb integration.Sandbox) { From 7bfae2b8090ecd3473a4abcb9ce17ecf20ca9b4d Mon Sep 17 00:00:00 2001 From: Tim Smith Date: Wed, 29 May 2024 22:54:09 -0400 Subject: [PATCH 6/6] Updated the docs. Signed-off-by: Tim Smith --- docs/reference/buildx_imagetools_create.md | 28 +++++++++++++--------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/docs/reference/buildx_imagetools_create.md b/docs/reference/buildx_imagetools_create.md index cfce7478..1b3bc06a 100644 --- a/docs/reference/buildx_imagetools_create.md +++ b/docs/reference/buildx_imagetools_create.md @@ -9,15 +9,16 @@ Create a new image based on source images ### Options -| Name | Type | Default | Description | -|:---------------------------------|:--------------|:--------|:-----------------------------------------------------------------------------------------| -| [`--annotation`](#annotation) | `stringArray` | | Add annotation to the image | -| [`--append`](#append) | | | Append to existing manifest | -| [`--builder`](#builder) | `string` | | Override the configured builder instance | -| [`--dry-run`](#dry-run) | | | Show final image instead of pushing | -| [`-f`](#file), [`--file`](#file) | `stringArray` | | Read source descriptor from file | -| `--progress` | `string` | `auto` | Set type of progress output (`auto`, `plain`, `tty`). Use plain to show container output | -| [`-t`](#tag), [`--tag`](#tag) | `stringArray` | | Set reference for new image | +| Name | Type | Default | Description | +|:---------------------------------|:--------------|:--------|:------------------------------------------------------------------------------------------------------------------------------| +| [`--annotation`](#annotation) | `stringArray` | | Add annotation to the image | +| [`--append`](#append) | | | Append to existing manifest | +| [`--builder`](#builder) | `string` | | Override the configured builder instance | +| [`--dry-run`](#dry-run) | | | Show final image instead of pushing | +| [`-f`](#file), [`--file`](#file) | `stringArray` | | Read source descriptor from file | +| `--prefer-index` | `bool` | `true` | When only a single source is specified, prefer outputting an image index or manifest list instead of performing a carbon copy | +| `--progress` | `string` | `auto` | Set type of progress output (`auto`, `plain`, `tty`). Use plain to show container output | +| [`-t`](#tag), [`--tag`](#tag) | `stringArray` | | Set reference for new image | @@ -26,8 +27,13 @@ Create a new image based on source images Create a new manifest list based on source manifests. The source manifests can be manifest lists or single platform distribution manifests and must already -exist in the registry where the new manifest is created. If only one source is -specified, create performs a carbon copy. +exist in the registry where the new manifest is created. + +If only one source is specified and that source is a manifest list or image index, +create performs a carbon copy. If one source is specified and that source is *not* +a list or index, the output will be a manifest list, however you can disable this +behavior with `--prefer-index=false` which attempts to preserve the source manifest +format in the output. ## Examples