diff --git a/build/build.go b/build/build.go
index 1d06824a..75373e64 100644
--- a/build/build.go
+++ b/build/build.go
@@ -61,6 +61,7 @@ type Options struct {
 
 	Ref           string
 	Allow         []entitlements.Entitlement
+	Annotations   map[exptypes.AnnotationKey]string
 	Attests       map[string]*string
 	BuildArgs     map[string]string
 	CacheFrom     []client.CacheOptionsEntry
@@ -607,7 +608,8 @@ func BuildWithResultHandler(ctx context.Context, nodes []builder.Node, opt map[s
 								}
 							}
 
-							dt, desc, err := itpull.Combine(ctx, srcs, nil, false)
+							filteredAnnotations := filterIndexAnnotations(opt.Annotations)
+							dt, desc, err := itpull.Combine(ctx, srcs, filteredAnnotations, false)
 							if err != nil {
 								return err
 							}
@@ -655,6 +657,17 @@ func BuildWithResultHandler(ctx context.Context, nodes []builder.Node, opt map[s
 	return resp, nil
 }
 
+func filterIndexAnnotations(annotations map[exptypes.AnnotationKey]string) map[exptypes.AnnotationKey]string {
+	filteredAnnotations := map[exptypes.AnnotationKey]string{}
+	for k, v := range annotations {
+		switch k.Type {
+		case exptypes.AnnotationIndex, exptypes.AnnotationManifestDescriptor:
+			filteredAnnotations[k] = v
+		}
+	}
+	return filteredAnnotations
+}
+
 func pushWithMoby(ctx context.Context, d *driver.DriverHandle, name string, l progress.SubLogger) error {
 	api := d.Config().DockerAPI
 	if api == nil {
diff --git a/commands/imagetools/create.go b/commands/imagetools/create.go
index 934602ee..05f460d7 100644
--- a/commands/imagetools/create.go
+++ b/commands/imagetools/create.go
@@ -9,6 +9,7 @@ import (
 
 	"github.com/distribution/reference"
 	"github.com/docker/buildx/builder"
+	"github.com/docker/buildx/util/buildflags"
 	"github.com/docker/buildx/util/cobrautil/completion"
 	"github.com/docker/buildx/util/imagetools"
 	"github.com/docker/buildx/util/progress"
@@ -154,7 +155,12 @@ func runCreate(ctx context.Context, dockerCli command.Cli, in createOptions, arg
 		}
 	}
 
-	dt, desc, err := r.Combine(ctx, srcs, in.annotations, in.preferIndex)
+	annotations, err := buildflags.ParseAnnotations(in.annotations)
+	if err != nil {
+		return errors.Wrapf(err, "failed to parse annotations")
+	}
+
+	dt, desc, err := r.Combine(ctx, srcs, annotations, in.preferIndex)
 	if err != nil {
 		return err
 	}
diff --git a/controller/build/build.go b/controller/build/build.go
index 1cae9b6c..57b4353c 100644
--- a/controller/build/build.go
+++ b/controller/build/build.go
@@ -136,8 +136,9 @@ func RunBuild(ctx context.Context, dockerCli command.Cli, in controllerapi.Build
 
 	annotations, err := buildflags.ParseAnnotations(in.Annotations)
 	if err != nil {
-		return nil, nil, err
+		return nil, nil, errors.Wrap(err, "parse annotations")
 	}
+
 	for _, o := range outputs {
 		for k, v := range annotations {
 			o.Attrs[k.String()] = v
@@ -146,6 +147,8 @@ func RunBuild(ctx context.Context, dockerCli command.Cli, in controllerapi.Build
 
 	opts.Exports = outputs
 
+	opts.Annotations = annotations
+
 	opts.CacheFrom = controllerapi.CreateCaches(in.CacheFrom)
 	opts.CacheTo = controllerapi.CreateCaches(in.CacheTo)
 
diff --git a/util/imagetools/create.go b/util/imagetools/create.go
index 08148ad2..54cbf344 100644
--- a/util/imagetools/create.go
+++ b/util/imagetools/create.go
@@ -14,7 +14,6 @@ import (
 	"github.com/containerd/containerd/remotes"
 	"github.com/containerd/errdefs"
 	"github.com/distribution/reference"
-	"github.com/docker/buildx/util/buildflags"
 	"github.com/moby/buildkit/exporter/containerimage/exptypes"
 	"github.com/moby/buildkit/util/contentutil"
 	"github.com/opencontainers/go-digest"
@@ -29,7 +28,7 @@ type Source struct {
 	Ref  reference.Named
 }
 
-func (r *Resolver) Combine(ctx context.Context, srcs []*Source, ann []string, preferIndex bool) ([]byte, ocispec.Descriptor, error) {
+func (r *Resolver) Combine(ctx context.Context, srcs []*Source, ann map[exptypes.AnnotationKey]string, preferIndex bool) ([]byte, ocispec.Descriptor, error) {
 	eg, ctx := errgroup.WithContext(ctx)
 
 	dts := make([][]byte, len(srcs))
@@ -152,11 +151,7 @@ func (r *Resolver) Combine(ctx context.Context, srcs []*Source, ann []string, pr
 	// annotations are only allowed on OCI indexes
 	indexAnnotation := make(map[string]string)
 	if mt == ocispec.MediaTypeImageIndex {
-		annotations, err := buildflags.ParseAnnotations(ann)
-		if err != nil {
-			return nil, ocispec.Descriptor{}, err
-		}
-		for k, v := range annotations {
+		for k, v := range ann {
 			switch k.Type {
 			case exptypes.AnnotationIndex:
 				indexAnnotation[k.Key] = v