docker
/
buildx
mirror of https://github.com/docker/buildx.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1434 from jedevc/resource-interpolation 

Resource interpolation support
This commit is contained in:
Tõnis Tiigi 2022-12-13 18:06:51 -08:00 committed by GitHub
parent 2a6ff4cbfc e51b55e03c
commit e21f56e801
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 500 additions and 89 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
bake/bake.go
View File

@ -270,8 +270,8 @@ func ParseFile(dt []byte, fn string) (*Config, error) {
}
type Config struct {
Groups []*Group `json:"group" hcl:"group,block"`
Targets []*Target `json:"target" hcl:"target,block"`
Groups []*Group `json:"group" hcl:"group,block" cty:"group"`
Targets []*Target `json:"target" hcl:"target,block" cty:"target"`
}
func mergeConfig(c1, c2 Config) Config {
@ -547,36 +547,36 @@ func (c Config) target(name string, visited map[string]*Target, overrides map[st
}
type Group struct {
Name string `json:"-" hcl:"name,label"`
Targets []string `json:"targets" hcl:"targets"`
Name string `json:"-" hcl:"name,label" cty:"name"`
Targets []string `json:"targets" hcl:"targets" cty:"targets"`
// Target // TODO?
}
type Target struct {
Name string `json:"-" hcl:"name,label"`
Name string `json:"-" hcl:"name,label" cty:"name"`
// Inherits is the only field that cannot be overridden with --set
Inherits []string `json:"inherits,omitempty" hcl:"inherits,optional"`
Attest []string `json:"attest,omitempty" hcl:"attest,optional" cty:"attest"`
Inherits []string `json:"inherits,omitempty" hcl:"inherits,optional" cty:"inherits"`
Attest []string `json:"attest,omitempty" hcl:"attest,optional"`
Context *string `json:"context,omitempty" hcl:"context,optional"`
Contexts map[string]string `json:"contexts,omitempty" hcl:"contexts,optional"`
Dockerfile *string `json:"dockerfile,omitempty" hcl:"dockerfile,optional"`
DockerfileInline *string `json:"dockerfile-inline,omitempty" hcl:"dockerfile-inline,optional"`
Args map[string]string `json:"args,omitempty" hcl:"args,optional"`
Labels map[string]string `json:"labels,omitempty" hcl:"labels,optional"`
Tags []string `json:"tags,omitempty" hcl:"tags,optional"`
CacheFrom []string `json:"cache-from,omitempty" hcl:"cache-from,optional"`
CacheTo []string `json:"cache-to,omitempty" hcl:"cache-to,optional"`
Target *string `json:"target,omitempty" hcl:"target,optional"`
Secrets []string `json:"secret,omitempty" hcl:"secret,optional"`
SSH []string `json:"ssh,omitempty" hcl:"ssh,optional"`
Platforms []string `json:"platforms,omitempty" hcl:"platforms,optional"`
Outputs []string `json:"output,omitempty" hcl:"output,optional"`
Pull *bool `json:"pull,omitempty" hcl:"pull,optional"`
NoCache *bool `json:"no-cache,omitempty" hcl:"no-cache,optional"`
NetworkMode *string `json:"-" hcl:"-"`
NoCacheFilter []string `json:"no-cache-filter,omitempty" hcl:"no-cache-filter,optional"`
Context *string `json:"context,omitempty" hcl:"context,optional" cty:"context"`
Contexts map[string]string `json:"contexts,omitempty" hcl:"contexts,optional" cty:"contexts"`
Dockerfile *string `json:"dockerfile,omitempty" hcl:"dockerfile,optional" cty:"dockerfile"`
DockerfileInline *string `json:"dockerfile-inline,omitempty" hcl:"dockerfile-inline,optional" cty:"dockerfile-inline"`
Args map[string]string `json:"args,omitempty" hcl:"args,optional" cty:"args"`
Labels map[string]string `json:"labels,omitempty" hcl:"labels,optional" cty:"labels"`
Tags []string `json:"tags,omitempty" hcl:"tags,optional" cty:"tags"`
CacheFrom []string `json:"cache-from,omitempty" hcl:"cache-from,optional" cty:"cache-from"`
CacheTo []string `json:"cache-to,omitempty" hcl:"cache-to,optional" cty:"cache-to"`
Target *string `json:"target,omitempty" hcl:"target,optional" cty:"target"`
Secrets []string `json:"secret,omitempty" hcl:"secret,optional" cty:"secret"`
SSH []string `json:"ssh,omitempty" hcl:"ssh,optional" cty:"ssh"`
Platforms []string `json:"platforms,omitempty" hcl:"platforms,optional" cty:"platforms"`
Outputs []string `json:"output,omitempty" hcl:"output,optional" cty:"output"`
Pull *bool `json:"pull,omitempty" hcl:"pull,optional" cty:"pull"`
NoCache *bool `json:"no-cache,omitempty" hcl:"no-cache,optional" cty:"no-cache"`
NetworkMode *string `json:"-" hcl:"-" cty:"-"`
NoCacheFilter []string `json:"no-cache-filter,omitempty" hcl:"no-cache-filter,optional" cty:"no-cache-filter"`
// IMPORTANT: if you add more fields here, do not forget to update newOverrides and docs/manuals/bake/file-definition.md.
// linked is a private field to mark a target used as a linked one

88
bake/hcl_test.go
View File

@ -444,6 +444,94 @@ func TestHCLAttrs(t *testing.T) {
// attr-multifile
}
func TestHCLTargetAttrs(t *testing.T) {
dt := []byte(`
target "foo" {
dockerfile = "xxx"
context = target.bar.context
target = target.foo.dockerfile
}
target "bar" {
dockerfile = target.foo.dockerfile
context = "yyy"
target = target.bar.context
}
`)
c, err := ParseFile(dt, "docker-bake.hcl")
require.NoError(t, err)
require.Equal(t, 2, len(c.Targets))
require.Equal(t, "foo", c.Targets[0].Name)
require.Equal(t, "bar", c.Targets[1].Name)
require.Equal(t, "xxx", *c.Targets[0].Dockerfile)
require.Equal(t, "yyy", *c.Targets[0].Context)
require.Equal(t, "xxx", *c.Targets[0].Target)
require.Equal(t, "xxx", *c.Targets[1].Dockerfile)
require.Equal(t, "yyy", *c.Targets[1].Context)
require.Equal(t, "yyy", *c.Targets[1].Target)
}
func TestHCLTargetGlobal(t *testing.T) {
dt := []byte(`
target "foo" {
dockerfile = "x"
}
x = target.foo.dockerfile
y = x
target "bar" {
dockerfile = y
}
`)
c, err := ParseFile(dt, "docker-bake.hcl")
require.NoError(t, err)
require.Equal(t, 2, len(c.Targets))
require.Equal(t, "foo", c.Targets[0].Name)
require.Equal(t, "bar", c.Targets[1].Name)
require.Equal(t, "x", *c.Targets[0].Dockerfile)
require.Equal(t, "x", *c.Targets[1].Dockerfile)
}
func TestHCLTargetAttrName(t *testing.T) {
dt := []byte(`
target "foo" {
dockerfile = target.foo.name
}
`)
c, err := ParseFile(dt, "docker-bake.hcl")
require.NoError(t, err)
require.Equal(t, 1, len(c.Targets))
require.Equal(t, "foo", c.Targets[0].Name)
require.Equal(t, "foo", *c.Targets[0].Dockerfile)
}
func TestHCLTargetAttrEmptyChain(t *testing.T) {
dt := []byte(`
target "foo" {
# dockerfile = Dockerfile
context = target.foo.dockerfile
target = target.foo.context
}
`)
c, err := ParseFile(dt, "docker-bake.hcl")
require.NoError(t, err)
require.Equal(t, 1, len(c.Targets))
require.Equal(t, "foo", c.Targets[0].Name)
require.Nil(t, c.Targets[0].Dockerfile)
require.Nil(t, c.Targets[0].Context)
require.Nil(t, c.Targets[0].Target)
}
func TestHCLAttrsCustomType(t *testing.T) {
dt := []byte(`
platforms=["linux/arm64", "linux/amd64"]

103
bake/hclparser/body.go Normal file
View File

@ -0,0 +1,103 @@
package hclparser
import (
"github.com/hashicorp/hcl/v2"
)
type filterBody struct {
body hcl.Body
schema *hcl.BodySchema
exclude bool
}
func FilterIncludeBody(body hcl.Body, schema *hcl.BodySchema) hcl.Body {
return &filterBody{
body: body,
schema: schema,
}
}
func FilterExcludeBody(body hcl.Body, schema *hcl.BodySchema) hcl.Body {
return &filterBody{
body: body,
schema: schema,
exclude: true,
}
}
func (b *filterBody) Content(schema *hcl.BodySchema) (*hcl.BodyContent, hcl.Diagnostics) {
if b.exclude {
schema = subtractSchemas(schema, b.schema)
} else {
schema = intersectSchemas(schema, b.schema)
}
content, _, diag := b.body.PartialContent(schema)
return content, diag
}
func (b *filterBody) PartialContent(schema *hcl.BodySchema) (*hcl.BodyContent, hcl.Body, hcl.Diagnostics) {
if b.exclude {
schema = subtractSchemas(schema, b.schema)
} else {
schema = intersectSchemas(schema, b.schema)
}
return b.body.PartialContent(schema)
}
func (b *filterBody) JustAttributes() (hcl.Attributes, hcl.Diagnostics) {
return b.body.JustAttributes()
}
func (b *filterBody) MissingItemRange() hcl.Range {
return b.body.MissingItemRange()
}
func intersectSchemas(a, b *hcl.BodySchema) *hcl.BodySchema {
result := &hcl.BodySchema{}
for _, blockA := range a.Blocks {
for _, blockB := range b.Blocks {
if blockA.Type == blockB.Type {
result.Blocks = append(result.Blocks, blockA)
break
}
}
}
for _, attrA := range a.Attributes {
for _, attrB := range b.Attributes {
if attrA.Name == attrB.Name {
result.Attributes = append(result.Attributes, attrA)
break
}
}
}
return result
}
func subtractSchemas(a, b *hcl.BodySchema) *hcl.BodySchema {
result := &hcl.BodySchema{}
for _, blockA := range a.Blocks {
found := false
for _, blockB := range b.Blocks {
if blockA.Type == blockB.Type {
found = true
break
}
}
if !found {
result.Blocks = append(result.Blocks, blockA)
}
}
for _, attrA := range a.Attributes {
found := false
for _, attrB := range b.Attributes {
if attrA.Name == attrB.Name {
found = true
break
}
}
if !found {
result.Attributes = append(result.Attributes, attrA)
}
}
return result
}

348
bake/hclparser/hclparser.go
View File

@ -13,6 +13,7 @@ import (
"github.com/hashicorp/hcl/v2/gohcl"
"github.com/pkg/errors"
"github.com/zclconf/go-cty/cty"
"github.com/zclconf/go-cty/cty/gocty"
)
type Opt struct {
@ -48,11 +49,17 @@ type parser struct {
attrs map[string]*hcl.Attribute
funcs map[string]*functionDef
blocks map[string]map[string][]*hcl.Block
blockValues map[*hcl.Block]reflect.Value
blockTypes map[string]reflect.Type
ectx *hcl.EvalContext
progress map[string]struct{}
progressF map[string]struct{}
progressB map[*hcl.Block]map[string]struct{}
doneF map[string]struct{}
doneB map[*hcl.Block]map[string]struct{}
}
func (p *parser) loadDeps(exp hcl.Expression, exclude map[string]struct{}) hcl.Diagnostics {
@ -79,15 +86,69 @@ func (p *parser) loadDeps(exp hcl.Expression, exclude map[string]struct{}) hcl.D
if _, ok := exclude[v.RootName()]; ok {
continue
}
if err := p.resolveValue(v.RootName()); err != nil {
return hcl.Diagnostics{
&hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Invalid expression",
Detail: err.Error(),
Subject: v.SourceRange().Ptr(),
Context: v.SourceRange().Ptr(),
},
if _, ok := p.blockTypes[v.RootName()]; ok {
blockType := v.RootName()
split := v.SimpleSplit().Rel
if len(split) == 0 {
return hcl.Diagnostics{
&hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Invalid expression",
Detail: fmt.Sprintf("cannot access %s as a variable", blockType),
Subject: exp.Range().Ptr(),
Context: exp.Range().Ptr(),
},
}
}
blockName, ok := split[0].(hcl.TraverseAttr)
if !ok {
return hcl.Diagnostics{
&hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Invalid expression",
Detail: fmt.Sprintf("cannot traverse %s without attribute", blockType),
Subject: exp.Range().Ptr(),
Context: exp.Range().Ptr(),
},
}
}
blocks := p.blocks[blockType][blockName.Name]
if len(blocks) == 0 {
continue
}
var target *hcl.BodySchema
if len(split) > 1 {
if attr, ok := split[1].(hcl.TraverseAttr); ok {
target = &hcl.BodySchema{
Attributes: []hcl.AttributeSchema{{Name: attr.Name}},
Blocks: []hcl.BlockHeaderSchema{{Type: attr.Name}},
}
}
}
if err := p.resolveBlock(blocks[0], target); err != nil {
return hcl.Diagnostics{
&hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Invalid expression",
Detail: err.Error(),
Subject: v.SourceRange().Ptr(),
Context: v.SourceRange().Ptr(),
},
}
}
} else {
if err := p.resolveValue(v.RootName()); err != nil {
return hcl.Diagnostics{
&hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Invalid expression",
Detail: err.Error(),
Subject: v.SourceRange().Ptr(),
Context: v.SourceRange().Ptr(),
},
}
}
}
}
@ -95,6 +156,8 @@ func (p *parser) loadDeps(exp hcl.Expression, exclude map[string]struct{}) hcl.D
return nil
}
// resolveFunction forces evaluation of a function, storing the result into the
// parser.
func (p *parser) resolveFunction(name string) error {
if _, ok := p.doneF[name]; ok {
return nil
@ -170,6 +233,8 @@ func (p *parser) resolveFunction(name string) error {
return nil
}
// resolveValue forces evaluation of a named value, storing the result into the
// parser.
func (p *parser) resolveValue(name string) (err error) {
if _, ok := p.ectx.Variables[name]; ok {
return nil
@ -248,6 +313,157 @@ func (p *parser) resolveValue(name string) (err error) {
return nil
}
// resolveBlock force evaluates a block, storing the result in the parser. If a
// target schema is provided, only the attributes and blocks present in the
// schema will be evaluated.
func (p *parser) resolveBlock(block *hcl.Block, target *hcl.BodySchema) (err error) {
name := block.Labels[0]
if err := p.opt.ValidateLabel(name); err != nil {
return hcl.Diagnostics{
&hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Invalid name",
Detail: err.Error(),
Subject: &block.LabelRanges[0],
},
}
}
if _, ok := p.doneB[block]; !ok {
p.doneB[block] = map[string]struct{}{}
}
if _, ok := p.progressB[block]; !ok {
p.progressB[block] = map[string]struct{}{}
}
if target != nil {
// filter out attributes and blocks that are already evaluated
original := target
target = &hcl.BodySchema{}
for _, a := range original.Attributes {
if _, ok := p.doneB[block][a.Name]; !ok {
target.Attributes = append(target.Attributes, a)
}
}
for _, b := range original.Blocks {
if _, ok := p.doneB[block][b.Type]; !ok {
target.Blocks = append(target.Blocks, b)
}
}
if len(target.Attributes) == 0 && len(target.Blocks) == 0 {
return nil
}
}
if target != nil {
// detect reference cycles
for _, a := range target.Attributes {
if _, ok := p.progressB[block][a.Name]; ok {
return errors.Errorf("reference cycle not allowed for %s.%s.%s", block.Type, name, a.Name)
}
}
for _, b := range target.Blocks {
if _, ok := p.progressB[block][b.Type]; ok {
return errors.Errorf("reference cycle not allowed for %s.%s.%s", block.Type, name, b.Type)
}
}
for _, a := range target.Attributes {
p.progressB[block][a.Name] = struct{}{}
}
for _, b := range target.Blocks {
p.progressB[block][b.Type] = struct{}{}
}
}
// create a filtered body that contains only the target properties
body := func() hcl.Body {
if target != nil {
return FilterIncludeBody(block.Body, target)
}
filter := &hcl.BodySchema{}
for k := range p.doneB[block] {
filter.Attributes = append(filter.Attributes, hcl.AttributeSchema{Name: k})
filter.Blocks = append(filter.Blocks, hcl.BlockHeaderSchema{Type: k})
}
return FilterExcludeBody(block.Body, filter)
}
// load dependencies from all targeted properties
t, ok := p.blockTypes[block.Type]
if !ok {
return nil
}
schema, _ := gohcl.ImpliedBodySchema(reflect.New(t).Interface())
content, _, diag := body().PartialContent(schema)
if diag.HasErrors() {
return diag
}
for _, a := range content.Attributes {
diag := p.loadDeps(a.Expr, nil)
if diag.HasErrors() {
return diag
}
}
for _, b := range content.Blocks {
err := p.resolveBlock(b, nil)
if err != nil {
return err
}
}
// decode!
var output reflect.Value
if prev, ok := p.blockValues[block]; ok {
output = prev
} else {
output = reflect.New(t)
setLabel(output, block.Labels[0]) // early attach labels, so we can reference them
}
diag = gohcl.DecodeBody(body(), p.ectx, output.Interface())
if diag.HasErrors() {
return diag
}
p.blockValues[block] = output
// mark all targeted properties as done
for _, a := range content.Attributes {
p.doneB[block][a.Name] = struct{}{}
}
for _, b := range content.Blocks {
p.doneB[block][b.Type] = struct{}{}
}
if target != nil {
for _, a := range target.Attributes {
p.doneB[block][a.Name] = struct{}{}
}
for _, b := range target.Blocks {
p.doneB[block][b.Type] = struct{}{}
}
}
// store the result into the evaluation context (so if can be referenced)
outputType, err := gocty.ImpliedType(output.Interface())
if err != nil {
return err
}
outputValue, err := gocty.ToCtyValue(output.Interface(), outputType)
if err != nil {
return err
}
var m map[string]cty.Value
if m2, ok := p.ectx.Variables[block.Type]; ok {
m = m2.AsValueMap()
}
if m == nil {
m = map[string]cty.Value{}
}
m[name] = outputValue
p.ectx.Variables[block.Type] = cty.MapVal(m)
return nil
}
func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics {
reserved := map[string]struct{}{}
schema, _ := gohcl.ImpliedBodySchema(val)
@ -284,9 +500,16 @@ func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics {
attrs: map[string]*hcl.Attribute{},
funcs: map[string]*functionDef{},
blocks: map[string]map[string][]*hcl.Block{},
blockValues: map[*hcl.Block]reflect.Value{},
blockTypes: map[string]reflect.Type{},
progress: map[string]struct{}{},
progressF: map[string]struct{}{},
doneF: map[string]struct{}{},
progressB: map[*hcl.Block]map[string]struct{}{},
doneF: map[string]struct{}{},
doneB: map[*hcl.Block]map[string]struct{}{},
ectx: &hcl.EvalContext{
Variables: map[string]cty.Value{},
Functions: stdlibFunctions,
@ -337,20 +560,15 @@ func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics {
_ = p.resolveValue(k)
}
for k := range p.attrs {
if err := p.resolveValue(k); err != nil {
if diags, ok := err.(hcl.Diagnostics); ok {
return diags
}
return hcl.Diagnostics{
&hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Invalid attribute",
Detail: err.Error(),
Subject: &p.attrs[k].Range,
Context: &p.attrs[k].Range,
},
}
for _, a := range content.Attributes {
return hcl.Diagnostics{
&hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Invalid attribute",
Detail: "global attributes currently not supported",
Subject: &a.Range,
Context: &a.Range,
},
}
}
@ -403,19 +621,6 @@ func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics {
}
}
for _, a := range content.Attributes {
return hcl.Diagnostics{
&hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Invalid attribute",
Detail: "global attributes currently not supported",
Subject: &a.Range,
Context: &a.Range,
},
}
}
m := map[string]map[string][]*hcl.Block{}
for _, b := range content.Blocks {
if len(b.Labels) == 0 || len(b.Labels) > 1 {
return hcl.Diagnostics{
@ -428,19 +633,16 @@ func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics {
},
}
}
bm, ok := m[b.Type]
bm, ok := p.blocks[b.Type]
if !ok {
bm = map[string][]*hcl.Block{}
m[b.Type] = bm
p.blocks[b.Type] = bm
}
lbl := b.Labels[0]
bm[lbl] = append(bm[lbl], b)
}
vt := reflect.ValueOf(val).Elem().Type()
numFields := vt.NumField()
type value struct {
reflect.Value
idx int
@ -452,9 +654,11 @@ func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics {
}
types := map[string]field{}
for i := 0; i < numFields; i++ {
vt := reflect.ValueOf(val).Elem().Type()
for i := 0; i < vt.NumField(); i++ {
tags := strings.Split(vt.Field(i).Tag.Get("hcl"), ",")
p.blockTypes[tags[0]] = vt.Field(i).Type.Elem().Elem()
types[tags[0]] = field{
idx: i,
typ: vt.Field(i).Type,
@ -466,29 +670,29 @@ func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics {
for _, b := range content.Blocks {
v := reflect.ValueOf(val)
t, ok := types[b.Type]
if !ok {
continue
}
vv := reflect.New(t.typ.Elem().Elem())
diag := gohcl.DecodeBody(b.Body, p.ectx, vv.Interface())
if diag.HasErrors() {
diags = append(diags, diag...)
continue
}
if err := opt.ValidateLabel(b.Labels[0]); err != nil {
return hcl.Diagnostics{
&hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Invalid name",
Detail: err.Error(),
Subject: &b.LabelRanges[0],
},
err := p.resolveBlock(b, nil)
if err != nil {
if diag, ok := err.(hcl.Diagnostics); ok {
if diag.HasErrors() {
diags = append(diags, diag...)
continue
}
} else {
return hcl.Diagnostics{
&hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Invalid attribute",
Detail: err.Error(),
Subject: &b.LabelRanges[0],
Context: &b.DefRange,
},
}
}
}
vv := p.blockValues[b]
t := types[b.Type]
lblIndex := setLabel(vv, b.Labels[0])
oldValue, exists := t.values[b.Labels[0]]
@ -502,7 +706,6 @@ func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics {
}
}
}
}
if exists {
if m := oldValue.Value.MethodByName("Merge"); m.IsValid() {
@ -523,6 +726,23 @@ func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics {
return diags
}
for k := range p.attrs {
if err := p.resolveValue(k); err != nil {
if diags, ok := err.(hcl.Diagnostics); ok {
return diags
}
return hcl.Diagnostics{
&hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Invalid attribute",
Detail: err.Error(),
Subject: &p.attrs[k].Range,
Context: &p.attrs[k].Range,
},
}
}
}
return nil
}