docker/buildx
1
0
Fork 0
mirror of https://github.com/docker/buildx.git synced 2024-11-22 15:37:16 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1137 from jedevc/imagetools-multiple-repositories

Browse Source 
Imagetools multiple repositories
This commit is contained in:
Tõnis Tiigi 2022-08-01 16:30:53 -07:00 committed by GitHub
commit 1f6612b118
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 128 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
build/build.go
View File

@ -925,7 +925,21 @@ func BuildWithResultHandler(ctx context.Context, drivers []DriverInfo, opt map[s
itpull := imagetools.New(imageopt)
dt, desc, err := itpull.Combine(ctx, names[0], descs)
ref, err := reference.ParseNormalizedNamed(names[0])
if err != nil {
return err
}
ref = reference.TagNameOnly(ref)
srcs := make([]*imagetools.Source, len(descs))
for i, desc := range descs {
srcs[i] = &imagetools.Source{
Desc: desc,
Ref: ref,
}
}
dt, desc, err := itpull.Combine(ctx, srcs)
if err != nil {
return err
}

87
commands/imagetools/create.go
View File

@ -1,6 +1,7 @@
package commands
import (
"context"
"encoding/json"
"fmt"
"os"
@ -9,6 +10,7 @@ import (
"github.com/docker/buildx/store"
"github.com/docker/buildx/store/storeutil"
"github.com/docker/buildx/util/imagetools"
"github.com/docker/buildx/util/progress"
"github.com/docker/cli/cli/command"
"github.com/docker/distribution/reference"
"github.com/moby/buildkit/util/appcontext"
@ -25,6 +27,7 @@ type createOptions struct {
tags []string
dryrun bool
actionAppend bool
progress string
}
func runCreate(dockerCli command.Cli, in createOptions, args []string) error {
@ -78,18 +81,21 @@ func runCreate(dockerCli command.Cli, in createOptions, args []string) error {
if len(repos) == 0 {
return errors.Errorf("no repositories specified, please set a reference in tag or source")
}
if len(repos) > 1 {
return errors.Errorf("multiple repositories currently not supported, found %v", repos)
}
var repo string
for r := range repos {
repo = r
var defaultRepo *string
if len(repos) == 1 {
for repo := range repos {
defaultRepo = &repo
}
}
for i, s := range srcs {
if s.Ref == nil && s.Desc.MediaType == "" && s.Desc.Digest != "" {
n, err := reference.ParseNormalizedNamed(repo)
if defaultRepo == nil {
return errors.Errorf("multiple repositories specified, cannot infer repository for %q", args[i])
}
n, err := reference.ParseNormalizedNamed(*defaultRepo)
if err != nil {
return err
}
@ -143,7 +149,6 @@ func runCreate(dockerCli command.Cli, in createOptions, args []string) error {
if err != nil {
return err
}
srcs[i].Ref = nil
if srcs[i].Desc.Digest == "" {
srcs[i].Desc = desc
} else {
@ -162,12 +167,7 @@ func runCreate(dockerCli command.Cli, in createOptions, args []string) error {
}
}
descs := make([]ocispec.Descriptor, len(srcs))
for i := range descs {
descs[i] = srcs[i].Desc
}
dt, desc, err := r.Combine(ctx, repo, descs)
dt, desc, err := r.Combine(ctx, srcs)
if err != nil {
return err
}
@ -180,23 +180,49 @@ func runCreate(dockerCli command.Cli, in createOptions, args []string) error {
// new resolver cause need new auth
r = imagetools.New(imageopt)
ctx2, cancel := context.WithCancel(context.TODO())
defer cancel()
printer := progress.NewPrinter(ctx2, os.Stderr, os.Stderr, in.progress)
eg, _ := errgroup.WithContext(ctx)
pw := progress.WithPrefix(printer, "internal", true)
for _, t := range tags {
if err := r.Push(ctx, t, desc, dt); err != nil {
return err
}
fmt.Println(t.String())
t := t
eg.Go(func() error {
return progress.Wrap(fmt.Sprintf("pushing %s", t.String()), pw.Write, func(sub progress.SubLogger) error {
eg2, _ := errgroup.WithContext(ctx)
for _, s := range srcs {
if reference.Domain(s.Ref) == reference.Domain(t) && reference.Path(s.Ref) == reference.Path(t) {
continue
}
s := s
eg2.Go(func() error {
sub.Log(1, []byte(fmt.Sprintf("copying %s from %s to %s\n", s.Desc.Digest.String(), s.Ref.String(), t.String())))
return r.Copy(ctx, s, t)
})
}
if err := eg2.Wait(); err != nil {
return err
}
sub.Log(1, []byte(fmt.Sprintf("pushing %s to %s\n", desc.Digest.String(), t.String())))
return r.Push(ctx, t, desc, dt)
})
})
}
return nil
err = eg.Wait()
err1 := printer.Wait()
if err == nil {
err = err1
}
return err
}
type src struct {
Desc ocispec.Descriptor
Ref reference.Named
}
func parseSources(in []string) ([]*src, error) {
out := make([]*src, len(in))
func parseSources(in []string) ([]*imagetools.Source, error) {
out := make([]*imagetools.Source, len(in))
for i, in := range in {
s, err := parseSource(in)
if err != nil {
@ -219,11 +245,11 @@ func parseRefs(in []string) ([]reference.Named, error) {
return refs, nil
}
func parseSource(in string) (*src, error) {
func parseSource(in string) (*imagetools.Source, error) {
// source can be a digest, reference or a descriptor JSON
dgst, err := digest.Parse(in)
if err == nil {
return &src{
return &imagetools.Source{
Desc: ocispec.Descriptor{
Digest: dgst,
},
@ -234,14 +260,14 @@ func parseSource(in string) (*src, error) {
ref, err := reference.ParseNormalizedNamed(in)
if err == nil {
return &src{
return &imagetools.Source{
Ref: ref,
}, nil
} else if !strings.HasPrefix(in, "{") {
return nil, err
}
var s src
var s imagetools.Source
if err := json.Unmarshal([]byte(in), &s.Desc); err != nil {
return nil, errors.WithStack(err)
}
@ -265,6 +291,7 @@ func createCmd(dockerCli command.Cli, opts RootOptions) *cobra.Command {
flags.StringArrayVarP(&options.tags, "tag", "t", []string{}, "Set reference for new image")
flags.BoolVar(&options.dryrun, "dry-run", false, "Show final image instead of pushing")
flags.BoolVar(&options.actionAppend, "append", false, "Append to existing manifest")
flags.StringVar(&options.progress, "progress", "auto", `Set type of progress output ("auto", "plain", "tty"). Use plain to show container output`)
return cmd
}

1
docs/reference/buildx_imagetools_create.md
View File

@ -15,6 +15,7 @@ Create a new image based on source images
| [`--builder`](#builder) | `string` | | Override the configured builder instance |
| [`--dry-run`](#dry-run) | | | Show final image instead of pushing |
| [`-f`](#file), [`--file`](#file) | `stringArray` | | Read source descriptor from file |
| `--progress` | `string` | `auto` | Set type of progress output (`auto`, `plain`, `tty`). Use plain to show container output |
| [`-t`](#tag), [`--tag`](#tag) | `stringArray` | | Set reference for new image |

76
util/imagetools/create.go
View File

@ -4,12 +4,15 @@ import (
"bytes"
"context"
"encoding/json"
"net/url"
"strings"
"github.com/containerd/containerd/content"
"github.com/containerd/containerd/errdefs"
"github.com/containerd/containerd/images"
"github.com/containerd/containerd/platforms"
"github.com/docker/distribution/reference"
"github.com/moby/buildkit/util/contentutil"
"github.com/opencontainers/go-digest"
"github.com/opencontainers/image-spec/specs-go"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
@ -17,46 +20,46 @@ import (
"golang.org/x/sync/errgroup"
)
func (r *Resolver) Combine(ctx context.Context, in string, descs []ocispec.Descriptor) ([]byte, ocispec.Descriptor, error) {
ref, err := parseRef(in)
if err != nil {
return nil, ocispec.Descriptor{}, err
}
type Source struct {
Desc ocispec.Descriptor
Ref reference.Named
}
func (r *Resolver) Combine(ctx context.Context, srcs []*Source) ([]byte, ocispec.Descriptor, error) {
eg, ctx := errgroup.WithContext(ctx)
dts := make([][]byte, len(descs))
dts := make([][]byte, len(srcs))
for i := range dts {
func(i int) {
eg.Go(func() error {
dt, err := r.GetDescriptor(ctx, ref.String(), descs[i])
dt, err := r.GetDescriptor(ctx, srcs[i].Ref.String(), srcs[i].Desc)
if err != nil {
return err
}
dts[i] = dt
if descs[i].MediaType == "" {
if srcs[i].Desc.MediaType == "" {
mt, err := detectMediaType(dt)
if err != nil {
return err
}
descs[i].MediaType = mt
srcs[i].Desc.MediaType = mt
}
mt := descs[i].MediaType
mt := srcs[i].Desc.MediaType
switch mt {
case images.MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest:
p := descs[i].Platform
if descs[i].Platform == nil {
p := srcs[i].Desc.Platform
if srcs[i].Desc.Platform == nil {
p = &ocispec.Platform{}
}
if p.OS == "" || p.Architecture == "" {
if err := r.loadPlatform(ctx, p, in, dt); err != nil {
if err := r.loadPlatform(ctx, p, srcs[i].Ref.String(), dt); err != nil {
return err
}
}
descs[i].Platform = p
srcs[i].Desc.Platform = p
case images.MediaTypeDockerSchema1Manifest:
return errors.Errorf("schema1 manifests are not allowed in manifest lists")
}
@ -71,14 +74,14 @@ func (r *Resolver) Combine(ctx context.Context, in string, descs []ocispec.Descr
}
// on single source, return original bytes
if len(descs) == 1 {
if mt := descs[0].MediaType; mt == images.MediaTypeDockerSchema2ManifestList || mt == ocispec.MediaTypeImageIndex {
return dts[0], descs[0], nil
if len(srcs) == 1 {
if mt := srcs[0].Desc.MediaType; mt == images.MediaTypeDockerSchema2ManifestList || mt == ocispec.MediaTypeImageIndex {
return dts[0], srcs[0].Desc, nil
}
}
m := map[digest.Digest]int{}
newDescs := make([]ocispec.Descriptor, 0, len(descs))
newDescs := make([]ocispec.Descriptor, 0, len(srcs))
addDesc := func(d ocispec.Descriptor) {
idx, ok := m[d.Digest]
@ -103,8 +106,8 @@ func (r *Resolver) Combine(ctx context.Context, in string, descs []ocispec.Descr
}
}
for i, desc := range descs {
switch desc.MediaType {
for i, src := range srcs {
switch src.Desc.MediaType {
case images.MediaTypeDockerSchema2ManifestList, ocispec.MediaTypeImageIndex:
var mfst ocispec.Index
if err := json.Unmarshal(dts[i], &mfst); err != nil {
@ -114,7 +117,7 @@ func (r *Resolver) Combine(ctx context.Context, in string, descs []ocispec.Descr
addDesc(d)
}
default:
addDesc(desc)
addDesc(src.Desc)
}
}
@ -169,6 +172,37 @@ func (r *Resolver) Push(ctx context.Context, ref reference.Named, desc ocispec.D
return err
}
func (r *Resolver) Copy(ctx context.Context, src *Source, dest reference.Named) error {
dest = reference.TagNameOnly(dest)
p, err := r.resolver().Pusher(ctx, dest.String())
if err != nil {
return err
}
srcRef := reference.TagNameOnly(src.Ref)
f, err := r.resolver().Fetcher(ctx, srcRef.String())
if err != nil {
return err
}
refspec := reference.TrimNamed(src.Ref).String()
u, err := url.Parse("dummy://" + refspec)
if err != nil {
return err
}
source, repo := u.Hostname(), strings.TrimPrefix(u.Path, "/")
if src.Desc.Annotations == nil {
src.Desc.Annotations = make(map[string]string)
}
src.Desc.Annotations["containerd.io/distribution.source."+source] = repo
err = contentutil.CopyChain(ctx, contentutil.FromPusher(p), contentutil.FromFetcher(f), src.Desc)
if err != nil {
return err
}
return nil
}
func (r *Resolver) loadPlatform(ctx context.Context, p2 *ocispec.Platform, in string, dt []byte) error {
var manifest ocispec.Manifest
if err := json.Unmarshal(dt, &manifest); err != nil {