Merge pull request #1153 from crazy-max/update-compose-go

bake: support compose build tags and secrets env
This commit is contained in:
Tõnis Tiigi 2022-06-14 12:45:33 -07:00 committed by GitHub
commit 91fec23f5d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 305 additions and 109 deletions

View File

@ -89,6 +89,7 @@ func ParseCompose(dt []byte) (*Config, error) {
Name: s.Name,
Context: contextPathP,
Dockerfile: dockerfilePathP,
Tags: s.Build.Tags,
Labels: s.Build.Labels,
Args: flatten(s.Build.Args.Resolve(func(val string) (string, bool) {
if val, ok := s.Environment[val]; ok && val != nil {
@ -235,6 +236,9 @@ func composeToBuildkitSecret(inp compose.ServiceSecretConfig, psecret compose.Se
if psecret.File != "" {
bkattrs = append(bkattrs, "src="+psecret.File)
}
if psecret.Environment != "" {
bkattrs = append(bkattrs, "env="+psecret.Environment)
}
return strings.Join(bkattrs, ","), nil
}

View File

@ -24,10 +24,11 @@ services:
args:
buildno: 123
secrets:
- ENV_TOKEN
- token
- aws
secrets:
ENV_TOKEN: {}
token:
environment: ENV_TOKEN
aws:
file: /root/.aws/credentials
`)
@ -46,6 +47,7 @@ secrets:
})
require.Equal(t, "db", c.Targets[0].Name)
require.Equal(t, "./db", *c.Targets[0].Context)
require.Equal(t, []string{"docker.io/tonistiigi/db"}, c.Targets[0].Tags)
require.Equal(t, "webapp", c.Targets[1].Name)
require.Equal(t, "./dir", *c.Targets[1].Context)
@ -54,7 +56,7 @@ secrets:
require.Equal(t, "123", c.Targets[1].Args["buildno"])
require.Equal(t, "none", *c.Targets[1].NetworkMode)
require.Equal(t, []string{
"id=ENV_TOKEN",
"id=token,env=ENV_TOKEN",
"id=aws,src=/root/.aws/credentials",
}, c.Targets[1].Secrets)
}
@ -193,6 +195,24 @@ networks:
require.NoError(t, err)
}
func TestTags(t *testing.T) {
var dt = []byte(`
services:
example:
image: example
build:
context: .
dockerfile: Dockerfile
tags:
- foo
- bar
`)
c, err := ParseCompose(dt)
require.NoError(t, err)
require.Equal(t, c.Targets[0].Tags, []string{"foo", "bar"})
}
func TestDependsOnList(t *testing.T) {
var dt = []byte(`
version: "3.8"
@ -239,6 +259,8 @@ services:
dockerfile: ./Dockerfile
cache_from:
- user/app:cache
tags:
- ct-addon:baz
args:
CT_ECR: foo
CT_TAG: bar
@ -278,7 +300,7 @@ services:
return c.Targets[i].Name < c.Targets[j].Name
})
require.Equal(t, c.Targets[0].Args, map[string]string{"CT_ECR": "foo", "CT_TAG": "bar"})
require.Equal(t, c.Targets[0].Tags, []string{"ct-addon:foo", "ct-addon:alp"})
require.Equal(t, c.Targets[0].Tags, []string{"ct-addon:baz", "ct-addon:foo", "ct-addon:alp"})
require.Equal(t, c.Targets[0].Platforms, []string{"linux/amd64", "linux/arm64"})
require.Equal(t, c.Targets[0].CacheFrom, []string{"type=local,src=path/to/cache"})
require.Equal(t, c.Targets[0].CacheTo, []string{"local,dest=path/to/cache"})

8
go.mod
View File

@ -3,7 +3,7 @@ module github.com/docker/buildx
go 1.17
require (
github.com/compose-spec/compose-go v1.2.4
github.com/compose-spec/compose-go v1.2.7
github.com/containerd/console v1.0.3
github.com/containerd/containerd v1.6.4
github.com/docker/cli v20.10.14+incompatible
@ -80,7 +80,7 @@ require (
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect
github.com/imdario/mergo v0.3.12 // indirect
github.com/imdario/mergo v0.3.13 // indirect
github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/jinzhu/gorm v1.9.2 // indirect
github.com/jinzhu/inflection v0.0.0-20180308033659-04140366298a // indirect
@ -91,7 +91,7 @@ require (
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
github.com/miekg/pkcs11 v1.1.1 // indirect
github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7 // indirect
github.com/mitchellh/mapstructure v1.4.3 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/moby/locker v1.0.1 // indirect
github.com/moby/spdystream v0.2.0 // indirect
github.com/moby/sys/mount v0.3.0 // indirect
@ -139,7 +139,7 @@ require (
gopkg.in/gorethink/gorethink.v3 v3.0.5 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
gopkg.in/yaml.v3 v3.0.0 // indirect
k8s.io/klog/v2 v2.30.0 // indirect
k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.1.2 // indirect

17
go.sum
View File

@ -128,8 +128,8 @@ github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWH
github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/compose-spec/compose-go v1.2.4 h1:nzTFqM8+2J7Veao5Pq5U451thinv3U1wChIvcjX59/A=
github.com/compose-spec/compose-go v1.2.4/go.mod h1:pAy7Mikpeft4pxkFU565/DRHEbDfR84G6AQuiL+Hdg8=
github.com/compose-spec/compose-go v1.2.7 h1:eqKGZhdOQEGKW/FmFDt4xyEPhCpbA5dr0PkcWD895aI=
github.com/compose-spec/compose-go v1.2.7/go.mod h1:Jl9L8zJrt4aGY1XAz03DvHAu8V3/f00TK+uJL4BayDU=
github.com/containerd/cgroups v1.0.3 h1:ADZftAkglvCiD44c77s5YmMqaP2pzVCFZvBmAlBdAP4=
github.com/containerd/console v1.0.3 h1:lIr7SlA5PxZyMV30bDW0MGbiOPXwc63yRuCP0ARubLw=
github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U=
@ -384,8 +384,8 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU=
github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk=
github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg=
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/jinzhu/gorm v1.9.2 h1:lCvgEaqe/HVE+tjAR2mt4HbbHAZsQOv3XAZiEZV37iw=
@ -454,8 +454,8 @@ github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0Qu
github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs=
github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A=
github.com/moby/buildkit v0.10.1-0.20220513101920-55f0ecfdafaa h1:Q8d1P3Ot+Z0eoynWkiWAfSyXRkvzPXqvUjE5y/Ycxw4=
github.com/moby/buildkit v0.10.1-0.20220513101920-55f0ecfdafaa/go.mod h1:grddVHNUy3u8m1v5ZEbFKhpIwyGmLsSd8rRwJ5ccg7c=
@ -1123,11 +1123,12 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0 h1:hjy8E9ON/egN1tAYqKb61G10WtihqetD4sz2H+8nIeA=
gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=
gotest.tools/v3 v3.1.0 h1:rVV8Tcg/8jHUkPUorwjaMTtemIMVXfIPKiOqnhEhakk=
gotest.tools/v3 v3.2.0 h1:I0DwBVMGAx26dttAj1BtJLAkVGncrkkUXfJLC4Flt/I=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

View File

@ -18,6 +18,9 @@ const (
func parseBytes(src []byte, out map[string]string, lookupFn LookupFn) error {
cutset := src
if lookupFn == nil {
lookupFn = noLookupFn
}
for {
cutset = getStatementStart(cutset)
if cutset == nil {
@ -34,9 +37,6 @@ func parseBytes(src []byte, out map[string]string, lookupFn LookupFn) error {
}
if inherited {
if lookupFn == nil {
lookupFn = noLookupFn
}
value, ok := lookupFn(key)
if ok {
@ -50,6 +50,9 @@ func parseBytes(src []byte, out map[string]string, lookupFn LookupFn) error {
if err != nil {
return err
}
if lookUpValue, ok := lookupFn(key); ok {
value = lookUpValue
}
out[key] = value
cutset = left

View File

@ -22,6 +22,9 @@ services:
uid: '103'
gid: '103'
mode: 0440
tags:
- foo:v1.0.0
- docker.io/username/foo:my-other-tag
cap_add:
@ -402,6 +405,7 @@ configs:
external: true
config4:
name: foo
file: ~/config_data
x-bar: baz
x-foo: bar
@ -417,6 +421,7 @@ secrets:
external: true
secret4:
name: bar
environment: BAR
x-bar: baz
x-foo: bar
x-bar: baz

View File

@ -24,7 +24,6 @@ import (
"path/filepath"
"reflect"
"regexp"
"sort"
"strconv"
"strings"
"time"
@ -70,7 +69,7 @@ type Options struct {
}
func (o *Options) SetProjectName(name string, imperativelySet bool) {
o.projectName = normalizeProjectName(name)
o.projectName = NormalizeProjectName(name)
o.projectNameImperativelySet = imperativelySet
}
@ -209,7 +208,7 @@ func Load(configDetails types.ConfigDetails, options ...func(*Options)) (*types.
}
projectName, projectNameImperativelySet := opts.GetProjectName()
model.Name = normalizeProjectName(model.Name)
model.Name = NormalizeProjectName(model.Name)
if !projectNameImperativelySet && model.Name != "" {
projectName = model.Name
}
@ -246,7 +245,7 @@ func Load(configDetails types.ConfigDetails, options ...func(*Options)) (*types.
return project, nil
}
func normalizeProjectName(s string) string {
func NormalizeProjectName(s string) string {
r := regexp.MustCompile("[a-z0-9_-]")
s = strings.ToLower(s)
s = strings.Join(r.FindAllString(s, -1), "")
@ -385,7 +384,7 @@ func createTransformHook(additionalTransformers ...Transformer) mapstructure.Dec
reflect.TypeOf(types.MappingWithEquals{}): transformMappingOrListFunc("=", true),
reflect.TypeOf(types.Labels{}): transformMappingOrListFunc("=", false),
reflect.TypeOf(types.MappingWithColon{}): transformMappingOrListFunc(":", false),
reflect.TypeOf(types.HostsList{}): transformListOrMappingFunc(":", false),
reflect.TypeOf(types.HostsList{}): transformMappingOrListFunc(":", false),
reflect.TypeOf(types.ServiceVolumeConfig{}): transformServiceVolumeConfig,
reflect.TypeOf(types.BuildConfig{}): transformBuildConfig,
reflect.TypeOf(types.Duration(0)): transformStringToDuration,
@ -571,18 +570,18 @@ func LoadService(name string, serviceDict map[string]interface{}, workingDir str
if volume.Type != types.VolumeTypeBind {
continue
}
if volume.Source == "" {
return nil, errors.New(`invalid mount config for type "bind": field Source must not be empty`)
}
if resolvePaths {
serviceConfig.Volumes[i] = resolveVolumePath(volume, workingDir, lookupEnv)
if resolvePaths || convertPaths {
volume = resolveVolumePath(volume, workingDir, lookupEnv)
}
if convertPaths {
serviceConfig.Volumes[i] = convertVolumePath(volume)
volume = convertVolumePath(volume)
}
serviceConfig.Volumes[i] = volume
}
return serviceConfig, nil
@ -805,7 +804,7 @@ func loadFileObjectConfig(name string, objType string, obj types.FileObjectConfi
return obj, errors.Errorf("%[1]s %[2]s: %[1]s.driver and %[1]s.file conflict; only use %[1]s.driver", objType, name)
}
default:
if resolvePaths {
if obj.File != "" && resolvePaths {
obj.File = absPath(details.WorkingDir, obj.File)
}
}
@ -1060,22 +1059,6 @@ func transformMappingOrListFunc(sep string, allowNil bool) TransformerFunc {
}
}
func transformListOrMappingFunc(sep string, allowNil bool) TransformerFunc {
return func(data interface{}) (interface{}, error) {
return transformListOrMapping(data, sep, allowNil)
}
}
func transformListOrMapping(listOrMapping interface{}, sep string, allowNil bool) (interface{}, error) {
switch value := listOrMapping.(type) {
case map[string]interface{}:
return toStringList(value, sep, allowNil), nil
case []interface{}:
return listOrMapping, nil
}
return nil, errors.Errorf("expected a map or a list, got %T: %#v", listOrMapping, listOrMapping)
}
func transformMappingOrList(mappingOrList interface{}, sep string, allowNil bool) (interface{}, error) {
switch value := mappingOrList.(type) {
case map[string]interface{}:
@ -1168,15 +1151,3 @@ func toString(value interface{}, allowNil bool) interface{} {
return ""
}
}
func toStringList(value map[string]interface{}, separator string, allowNil bool) []string {
var output []string
for key, value := range value {
if value == nil && !allowNil {
continue
}
output = append(output, fmt.Sprintf("%s%s%s", key, separator, value))
}
sort.Strings(output)
return output
}

View File

@ -68,5 +68,15 @@ func checkConsistency(project *types.Project) error {
}
}
}
for name, secret := range project.Secrets {
if secret.External.External {
continue
}
if secret.File == "" && secret.Environment == "" {
return errors.Wrap(errdefs.ErrInvalid, fmt.Sprintf("secret %q must declare either `file` or `environment`", name))
}
}
return nil
}

View File

@ -102,7 +102,8 @@
"shm_size": {"type": ["integer", "string"]},
"extra_hosts": {"$ref": "#/definitions/list_or_dict"},
"isolation": {"type": "string"},
"secrets": {"$ref": "#/definitions/service_config_or_secret"}
"secrets": {"$ref": "#/definitions/service_config_or_secret"},
"tags": {"type": "array", "items": {"type": "string"}}
},
"additionalProperties": false,
"patternProperties": {"^x-": {}}
@ -684,6 +685,7 @@
"type": "object",
"properties": {
"name": {"type": "string"},
"environment": {"type": "string"},
"file": {"type": "string"},
"external": {
"type": ["boolean", "object"],

View File

@ -18,10 +18,17 @@ package types
import (
"encoding/json"
"runtime"
"strings"
"github.com/mitchellh/mapstructure"
)
var (
// isCaseInsensitiveEnvVars is true on platforms where environment variable names are treated case-insensitively.
isCaseInsensitiveEnvVars = (runtime.GOOS == "windows")
)
// ConfigDetails are the details about a group of ConfigFiles
type ConfigDetails struct {
Version string
@ -33,7 +40,21 @@ type ConfigDetails struct {
// LookupEnv provides a lookup function for environment variables
func (cd ConfigDetails) LookupEnv(key string) (string, bool) {
v, ok := cd.Environment[key]
return v, ok
if !isCaseInsensitiveEnvVars || ok {
return v, ok
}
// variable names must be treated case-insensitively on some platforms (that is, Windows).
// Resolves in this way:
// * Return the value if its name matches with the passed name case-sensitively.
// * Otherwise, return the value if its lower-cased name matches lower-cased passed name.
// * The value is indefinite if multiple variables match.
lowerKey := strings.ToLower(key)
for k, v := range cd.Environment {
if strings.ToLower(k) == lowerKey {
return v, true
}
}
return "", false
}
// ConfigFile is a filename and the contents of the file as a Dict

View File

@ -305,6 +305,7 @@ type BuildConfig struct {
Network string `yaml:",omitempty" json:"network,omitempty"`
Target string `yaml:",omitempty" json:"target,omitempty"`
Secrets []ServiceSecretConfig `yaml:",omitempty" json:"secrets,omitempty"`
Tags StringList `mapstructure:"tags" yaml:"tags,omitempty" json:"tags,omitempty"`
Extensions map[string]interface{} `yaml:",inline" json:"-"`
}
@ -468,7 +469,16 @@ func (s SSHKey) MarshalJSON() ([]byte, error) {
type MappingWithColon map[string]string
// HostsList is a list of colon-separated host-ip mappings
type HostsList []string
type HostsList map[string]string
// AsList return host-ip mappings as a list of colon-separated strings
func (h HostsList) AsList() []string {
l := make([]string, 0, len(h))
for k, v := range h {
l = append(l, fmt.Sprintf("%s:%s", k, v))
}
return l
}
// LoggingConfig the logging configuration for a service
type LoggingConfig struct {
@ -881,6 +891,7 @@ type CredentialSpecConfig struct {
type FileObjectConfig struct {
Name string `yaml:",omitempty" json:"name,omitempty"`
File string `yaml:",omitempty" json:"file,omitempty"`
Environment string `yaml:",omitempty" json:"environment,omitempty"`
External External `yaml:",omitempty" json:"external,omitempty"`
Labels Labels `yaml:",omitempty" json:"labels,omitempty"`
Driver string `yaml:",omitempty" json:"driver,omitempty"`

View File

@ -8,8 +8,7 @@
[![Coverage Status][9]][10]
[![Sourcegraph][11]][12]
[![FOSSA Status][13]][14]
[![GoCenter Kudos][15]][16]
[![Become my sponsor][15]][16]
[1]: https://travis-ci.org/imdario/mergo.png
[2]: https://travis-ci.org/imdario/mergo
@ -25,8 +24,8 @@
[12]: https://sourcegraph.com/github.com/imdario/mergo?badge
[13]: https://app.fossa.io/api/projects/git%2Bgithub.com%2Fimdario%2Fmergo.svg?type=shield
[14]: https://app.fossa.io/projects/git%2Bgithub.com%2Fimdario%2Fmergo?ref=badge_shield
[15]: https://search.gocenter.io/api/ui/badge/github.com%2Fimdario%2Fmergo
[16]: https://search.gocenter.io/github.com/imdario/mergo
[15]: https://img.shields.io/github/sponsors/imdario
[16]: https://github.com/sponsors/imdario
A helper to merge structs and maps in Golang. Useful for configuration default values, avoiding messy if-statements.
@ -36,11 +35,11 @@ Also a lovely [comune](http://en.wikipedia.org/wiki/Mergo) (municipality) in the
## Status
It is ready for production use. [It is used in several projects by Docker, Google, The Linux Foundation, VMWare, Shopify, etc](https://github.com/imdario/mergo#mergo-in-the-wild).
It is ready for production use. [It is used in several projects by Docker, Google, The Linux Foundation, VMWare, Shopify, Microsoft, etc](https://github.com/imdario/mergo#mergo-in-the-wild).
### Important note
Please keep in mind that a problematic PR broke [0.3.9](//github.com/imdario/mergo/releases/tag/0.3.9). I reverted it in [0.3.10](//github.com/imdario/mergo/releases/tag/0.3.10), and I consider it stable but not bug-free. Also, this version adds suppot for go modules.
Please keep in mind that a problematic PR broke [0.3.9](//github.com/imdario/mergo/releases/tag/0.3.9). I reverted it in [0.3.10](//github.com/imdario/mergo/releases/tag/0.3.10), and I consider it stable but not bug-free. Also, this version adds support for go modules.
Keep in mind that in [0.3.2](//github.com/imdario/mergo/releases/tag/0.3.2), Mergo changed `Merge()`and `Map()` signatures to support [transformers](#transformers). I added an optional/variadic argument so that it won't break the existing code.
@ -51,12 +50,12 @@ If you were using Mergo before April 6th, 2015, please check your project works
If Mergo is useful to you, consider buying me a coffee, a beer, or making a monthly donation to allow me to keep building great free software. :heart_eyes:
<a href='https://ko-fi.com/B0B58839' target='_blank'><img height='36' style='border:0px;height:36px;' src='https://az743702.vo.msecnd.net/cdn/kofi1.png?v=0' border='0' alt='Buy Me a Coffee at ko-fi.com' /></a>
[![Beerpay](https://beerpay.io/imdario/mergo/badge.svg)](https://beerpay.io/imdario/mergo)
[![Beerpay](https://beerpay.io/imdario/mergo/make-wish.svg)](https://beerpay.io/imdario/mergo)
<a href="https://liberapay.com/dario/donate"><img alt="Donate using Liberapay" src="https://liberapay.com/assets/widgets/donate.svg"></a>
<a href='https://github.com/sponsors/imdario' target='_blank'><img alt="Become my sponsor" src="https://img.shields.io/github/sponsors/imdario?style=for-the-badge" /></a>
### Mergo in the wild
- [cli/cli](https://github.com/cli/cli)
- [moby/moby](https://github.com/moby/moby)
- [kubernetes/kubernetes](https://github.com/kubernetes/kubernetes)
- [vmware/dispatch](https://github.com/vmware/dispatch)
@ -98,6 +97,8 @@ If Mergo is useful to you, consider buying me a coffee, a beer, or making a mont
- [jnuthong/item_search](https://github.com/jnuthong/item_search)
- [bukalapak/snowboard](https://github.com/bukalapak/snowboard)
- [containerssh/containerssh](https://github.com/containerssh/containerssh)
- [goreleaser/goreleaser](https://github.com/goreleaser/goreleaser)
- [tjpnz/structbot](https://github.com/tjpnz/structbot)
## Install
@ -168,7 +169,7 @@ func main() {
Note: if test are failing due missing package, please execute:
go get gopkg.in/yaml.v2
go get gopkg.in/yaml.v3
### Transformers
@ -218,7 +219,6 @@ func main() {
}
```
## Contact me
If I can help you, you have an idea or you are using Mergo in your projects, don't hesitate to drop me a line (or a pull request): [@im_dario](https://twitter.com/im_dario)
@ -227,18 +227,6 @@ If I can help you, you have an idea or you are using Mergo in your projects, don
Written by [Dario Castañé](http://dario.im).
## Top Contributors
[![0](https://sourcerer.io/fame/imdario/imdario/mergo/images/0)](https://sourcerer.io/fame/imdario/imdario/mergo/links/0)
[![1](https://sourcerer.io/fame/imdario/imdario/mergo/images/1)](https://sourcerer.io/fame/imdario/imdario/mergo/links/1)
[![2](https://sourcerer.io/fame/imdario/imdario/mergo/images/2)](https://sourcerer.io/fame/imdario/imdario/mergo/links/2)
[![3](https://sourcerer.io/fame/imdario/imdario/mergo/images/3)](https://sourcerer.io/fame/imdario/imdario/mergo/links/3)
[![4](https://sourcerer.io/fame/imdario/imdario/mergo/images/4)](https://sourcerer.io/fame/imdario/imdario/mergo/links/4)
[![5](https://sourcerer.io/fame/imdario/imdario/mergo/images/5)](https://sourcerer.io/fame/imdario/imdario/mergo/links/5)
[![6](https://sourcerer.io/fame/imdario/imdario/mergo/images/6)](https://sourcerer.io/fame/imdario/imdario/mergo/links/6)
[![7](https://sourcerer.io/fame/imdario/imdario/mergo/images/7)](https://sourcerer.io/fame/imdario/imdario/mergo/links/7)
## License
[BSD 3-Clause](http://opensource.org/licenses/BSD-3-Clause) license, as [Go language](http://golang.org/LICENSE).

View File

@ -79,7 +79,7 @@ func deepMerge(dst, src reflect.Value, visited map[uintptr]*visit, depth int, co
visited[h] = &visit{addr, typ, seen}
}
if config.Transformers != nil && !isEmptyValue(dst) {
if config.Transformers != nil && !isReflectNil(dst) && dst.IsValid() {
if fn := config.Transformers.Transformer(dst.Type()); fn != nil {
err = fn(dst, src)
return

View File

@ -17,7 +17,7 @@ import (
var (
ErrNilArguments = errors.New("src and dst must not be nil")
ErrDifferentArgumentsTypes = errors.New("src and dst must be of same type")
ErrNotSupported = errors.New("only structs and maps are supported")
ErrNotSupported = errors.New("only structs, maps, and slices are supported")
ErrExpectedMapAsDestination = errors.New("dst was expected to be a map")
ErrExpectedStructAsDestination = errors.New("dst was expected to be a struct")
ErrNonPointerAgument = errors.New("dst must be a pointer")
@ -65,7 +65,7 @@ func resolveValues(dst, src interface{}) (vDst, vSrc reflect.Value, err error) {
return
}
vDst = reflect.ValueOf(dst).Elem()
if vDst.Kind() != reflect.Struct && vDst.Kind() != reflect.Map {
if vDst.Kind() != reflect.Struct && vDst.Kind() != reflect.Map && vDst.Kind() != reflect.Slice {
err = ErrNotSupported
return
}

View File

@ -1,3 +1,16 @@
## 1.5.0
* New option `IgnoreUntaggedFields` to ignore decoding to any fields
without `mapstructure` (or the configured tag name) set [GH-277]
* New option `ErrorUnset` which makes it an error if any fields
in a target struct are not set by the decoding process. [GH-225]
* New function `OrComposeDecodeHookFunc` to help compose decode hooks. [GH-240]
* Decoding to slice from array no longer crashes [GH-265]
* Decode nested struct pointers to map [GH-271]
* Fix issue where `,squash` was ignored if `Squash` option was set. [GH-280]
* Fix issue where fields with `,omitempty` would sometimes decode
into a map with an empty string key [GH-281]
## 1.4.3
* Fix cases where `json.Number` didn't decode properly [GH-261]

View File

@ -77,6 +77,28 @@ func ComposeDecodeHookFunc(fs ...DecodeHookFunc) DecodeHookFunc {
}
}
// OrComposeDecodeHookFunc executes all input hook functions until one of them returns no error. In that case its value is returned.
// If all hooks return an error, OrComposeDecodeHookFunc returns an error concatenating all error messages.
func OrComposeDecodeHookFunc(ff ...DecodeHookFunc) DecodeHookFunc {
return func(a, b reflect.Value) (interface{}, error) {
var allErrs string
var out interface{}
var err error
for _, f := range ff {
out, err = DecodeHookExec(f, a, b)
if err != nil {
allErrs += err.Error() + "\n"
continue
}
return out, nil
}
return nil, errors.New(allErrs)
}
}
// StringToSliceHookFunc returns a DecodeHookFunc that converts
// string to []string by splitting on the given sep.
func StringToSliceHookFunc(sep string) DecodeHookFunc {

View File

@ -122,7 +122,7 @@
// field value is zero and a numeric type, the field is empty, and it won't
// be encoded into the destination type.
//
// type Source {
// type Source struct {
// Age int `mapstructure:",omitempty"`
// }
//
@ -215,6 +215,12 @@ type DecoderConfig struct {
// (extra keys).
ErrorUnused bool
// If ErrorUnset is true, then it is an error for there to exist
// fields in the result that were not set in the decoding process
// (extra fields). This only applies to decoding to a struct. This
// will affect all nested structs as well.
ErrorUnset bool
// ZeroFields, if set to true, will zero fields before writing them.
// For example, a map will be emptied before decoded values are put in
// it. If this is false, a map will be merged.
@ -259,6 +265,10 @@ type DecoderConfig struct {
// defaults to "mapstructure"
TagName string
// IgnoreUntaggedFields ignores all struct fields without explicit
// TagName, comparable to `mapstructure:"-"` as default behaviour.
IgnoreUntaggedFields bool
// MatchName is the function used to match the map key to the struct
// field name or tag. Defaults to `strings.EqualFold`. This can be used
// to implement case-sensitive tag values, support snake casing, etc.
@ -284,6 +294,11 @@ type Metadata struct {
// Unused is a slice of keys that were found in the raw value but
// weren't decoded since there was no matching field in the result interface
Unused []string
// Unset is a slice of field names that were found in the result interface
// but weren't set in the decoding process since there was no matching value
// in the input
Unset []string
}
// Decode takes an input structure and uses reflection to translate it to
@ -375,6 +390,10 @@ func NewDecoder(config *DecoderConfig) (*Decoder, error) {
if config.Metadata.Unused == nil {
config.Metadata.Unused = make([]string, 0)
}
if config.Metadata.Unset == nil {
config.Metadata.Unset = make([]string, 0)
}
}
if config.TagName == "" {
@ -906,9 +925,15 @@ func (d *Decoder) decodeMapFromStruct(name string, dataVal reflect.Value, val re
tagValue := f.Tag.Get(d.config.TagName)
keyName := f.Name
if tagValue == "" && d.config.IgnoreUntaggedFields {
continue
}
// If Squash is set in the config, we squash the field down.
squash := d.config.Squash && v.Kind() == reflect.Struct && f.Anonymous
v = dereferencePtrToStructIfNeeded(v, d.config.TagName)
// Determine the name of the key in the map
if index := strings.Index(tagValue, ","); index != -1 {
if tagValue[:index] == "-" {
@ -920,7 +945,7 @@ func (d *Decoder) decodeMapFromStruct(name string, dataVal reflect.Value, val re
}
// If "squash" is specified in the tag, we squash the field down.
squash = !squash && strings.Index(tagValue[index+1:], "squash") != -1
squash = squash || strings.Index(tagValue[index+1:], "squash") != -1
if squash {
// When squashing, the embedded type can be a pointer to a struct.
if v.Kind() == reflect.Ptr && v.Elem().Kind() == reflect.Struct {
@ -932,7 +957,9 @@ func (d *Decoder) decodeMapFromStruct(name string, dataVal reflect.Value, val re
return fmt.Errorf("cannot squash non-struct type '%s'", v.Type())
}
}
keyName = tagValue[:index]
if keyNameTagValue := tagValue[:index]; keyNameTagValue != "" {
keyName = keyNameTagValue
}
} else if len(tagValue) > 0 {
if tagValue == "-" {
continue
@ -1088,7 +1115,7 @@ func (d *Decoder) decodeSlice(name string, data interface{}, val reflect.Value)
}
// If the input value is nil, then don't allocate since empty != nil
if dataVal.IsNil() {
if dataValKind != reflect.Array && dataVal.IsNil() {
return nil
}
@ -1250,6 +1277,7 @@ func (d *Decoder) decodeStructFromMap(name string, dataVal, val reflect.Value) e
dataValKeysUnused[dataValKey.Interface()] = struct{}{}
}
targetValKeysUnused := make(map[interface{}]struct{})
errors := make([]string, 0)
// This slice will keep track of all the structs we'll be decoding.
@ -1354,7 +1382,8 @@ func (d *Decoder) decodeStructFromMap(name string, dataVal, val reflect.Value) e
if !rawMapVal.IsValid() {
// There was no matching key in the map for the value in
// the struct. Just ignore.
// the struct. Remember it for potential errors and metadata.
targetValKeysUnused[fieldName] = struct{}{}
continue
}
}
@ -1414,6 +1443,17 @@ func (d *Decoder) decodeStructFromMap(name string, dataVal, val reflect.Value) e
errors = appendErrors(errors, err)
}
if d.config.ErrorUnset && len(targetValKeysUnused) > 0 {
keys := make([]string, 0, len(targetValKeysUnused))
for rawKey := range targetValKeysUnused {
keys = append(keys, rawKey.(string))
}
sort.Strings(keys)
err := fmt.Errorf("'%s' has unset fields: %s", name, strings.Join(keys, ", "))
errors = appendErrors(errors, err)
}
if len(errors) > 0 {
return &Error{errors}
}
@ -1428,6 +1468,14 @@ func (d *Decoder) decodeStructFromMap(name string, dataVal, val reflect.Value) e
d.config.Metadata.Unused = append(d.config.Metadata.Unused, key)
}
for rawKey := range targetValKeysUnused {
key := rawKey.(string)
if name != "" {
key = name + "." + key
}
d.config.Metadata.Unset = append(d.config.Metadata.Unset, key)
}
}
return nil
@ -1465,3 +1513,28 @@ func getKind(val reflect.Value) reflect.Kind {
return kind
}
}
func isStructTypeConvertibleToMap(typ reflect.Type, checkMapstructureTags bool, tagName string) bool {
for i := 0; i < typ.NumField(); i++ {
f := typ.Field(i)
if f.PkgPath == "" && !checkMapstructureTags { // check for unexported fields
return true
}
if checkMapstructureTags && f.Tag.Get(tagName) != "" { // check for mapstructure tags inside
return true
}
}
return false
}
func dereferencePtrToStructIfNeeded(v reflect.Value, tagName string) reflect.Value {
if v.Kind() != reflect.Ptr || v.Elem().Kind() != reflect.Struct {
return v
}
deref := v.Elem()
derefT := deref.Type()
if isStructTypeConvertibleToMap(derefT, true, tagName) {
return deref
}
return v
}

78
vendor/gopkg.in/yaml.v3/decode.go generated vendored
View File

@ -100,7 +100,10 @@ func (p *parser) peek() yaml_event_type_t {
if p.event.typ != yaml_NO_EVENT {
return p.event.typ
}
if !yaml_parser_parse(&p.parser, &p.event) {
// It's curious choice from the underlying API to generally return a
// positive result on success, but on this case return true in an error
// scenario. This was the source of bugs in the past (issue #666).
if !yaml_parser_parse(&p.parser, &p.event) || p.parser.error != yaml_NO_ERROR {
p.fail()
}
return p.event.typ
@ -320,6 +323,8 @@ type decoder struct {
decodeCount int
aliasCount int
aliasDepth int
mergedFields map[interface{}]bool
}
var (
@ -808,6 +813,11 @@ func (d *decoder) mapping(n *Node, out reflect.Value) (good bool) {
}
}
mergedFields := d.mergedFields
d.mergedFields = nil
var mergeNode *Node
mapIsNew := false
if out.IsNil() {
out.Set(reflect.MakeMap(outt))
@ -815,11 +825,18 @@ func (d *decoder) mapping(n *Node, out reflect.Value) (good bool) {
}
for i := 0; i < l; i += 2 {
if isMerge(n.Content[i]) {
d.merge(n.Content[i+1], out)
mergeNode = n.Content[i+1]
continue
}
k := reflect.New(kt).Elem()
if d.unmarshal(n.Content[i], k) {
if mergedFields != nil {
ki := k.Interface()
if mergedFields[ki] {
continue
}
mergedFields[ki] = true
}
kkind := k.Kind()
if kkind == reflect.Interface {
kkind = k.Elem().Kind()
@ -833,6 +850,12 @@ func (d *decoder) mapping(n *Node, out reflect.Value) (good bool) {
}
}
}
d.mergedFields = mergedFields
if mergeNode != nil {
d.merge(n, mergeNode, out)
}
d.stringMapType = stringMapType
d.generalMapType = generalMapType
return true
@ -844,7 +867,8 @@ func isStringMap(n *Node) bool {
}
l := len(n.Content)
for i := 0; i < l; i += 2 {
if n.Content[i].ShortTag() != strTag {
shortTag := n.Content[i].ShortTag()
if shortTag != strTag && shortTag != mergeTag {
return false
}
}
@ -861,7 +885,6 @@ func (d *decoder) mappingStruct(n *Node, out reflect.Value) (good bool) {
var elemType reflect.Type
if sinfo.InlineMap != -1 {
inlineMap = out.Field(sinfo.InlineMap)
inlineMap.Set(reflect.New(inlineMap.Type()).Elem())
elemType = inlineMap.Type().Elem()
}
@ -870,6 +893,9 @@ func (d *decoder) mappingStruct(n *Node, out reflect.Value) (good bool) {
d.prepare(n, field)
}
mergedFields := d.mergedFields
d.mergedFields = nil
var mergeNode *Node
var doneFields []bool
if d.uniqueKeys {
doneFields = make([]bool, len(sinfo.FieldsList))
@ -879,13 +905,20 @@ func (d *decoder) mappingStruct(n *Node, out reflect.Value) (good bool) {
for i := 0; i < l; i += 2 {
ni := n.Content[i]
if isMerge(ni) {
d.merge(n.Content[i+1], out)
mergeNode = n.Content[i+1]
continue
}
if !d.unmarshal(ni, name) {
continue
}
if info, ok := sinfo.FieldsMap[name.String()]; ok {
sname := name.String()
if mergedFields != nil {
if mergedFields[sname] {
continue
}
mergedFields[sname] = true
}
if info, ok := sinfo.FieldsMap[sname]; ok {
if d.uniqueKeys {
if doneFields[info.Id] {
d.terrors = append(d.terrors, fmt.Sprintf("line %d: field %s already set in type %s", ni.Line, name.String(), out.Type()))
@ -911,6 +944,11 @@ func (d *decoder) mappingStruct(n *Node, out reflect.Value) (good bool) {
d.terrors = append(d.terrors, fmt.Sprintf("line %d: field %s not found in type %s", ni.Line, name.String(), out.Type()))
}
}
d.mergedFields = mergedFields
if mergeNode != nil {
d.merge(n, mergeNode, out)
}
return true
}
@ -918,19 +956,29 @@ func failWantMap() {
failf("map merge requires map or sequence of maps as the value")
}
func (d *decoder) merge(n *Node, out reflect.Value) {
switch n.Kind {
func (d *decoder) merge(parent *Node, merge *Node, out reflect.Value) {
mergedFields := d.mergedFields
if mergedFields == nil {
d.mergedFields = make(map[interface{}]bool)
for i := 0; i < len(parent.Content); i += 2 {
k := reflect.New(ifaceType).Elem()
if d.unmarshal(parent.Content[i], k) {
d.mergedFields[k.Interface()] = true
}
}
}
switch merge.Kind {
case MappingNode:
d.unmarshal(n, out)
d.unmarshal(merge, out)
case AliasNode:
if n.Alias != nil && n.Alias.Kind != MappingNode {
if merge.Alias != nil && merge.Alias.Kind != MappingNode {
failWantMap()
}
d.unmarshal(n, out)
d.unmarshal(merge, out)
case SequenceNode:
// Step backwards as earlier nodes take precedence.
for i := len(n.Content) - 1; i >= 0; i-- {
ni := n.Content[i]
for i := 0; i < len(merge.Content); i++ {
ni := merge.Content[i]
if ni.Kind == AliasNode {
if ni.Alias != nil && ni.Alias.Kind != MappingNode {
failWantMap()
@ -943,6 +991,8 @@ func (d *decoder) merge(n *Node, out reflect.Value) {
default:
failWantMap()
}
d.mergedFields = mergedFields
}
func isMerge(n *Node) bool {

10
vendor/modules.txt vendored
View File

@ -42,8 +42,8 @@ github.com/cenkalti/backoff/v4
github.com/cespare/xxhash/v2
# github.com/cloudflare/cfssl v0.0.0-20181213083726-b94e044bb51e
## explicit
# github.com/compose-spec/compose-go v1.2.4
## explicit; go 1.17
# github.com/compose-spec/compose-go v1.2.7
## explicit; go 1.18
github.com/compose-spec/compose-go/consts
github.com/compose-spec/compose-go/dotenv
github.com/compose-spec/compose-go/errdefs
@ -296,7 +296,7 @@ github.com/hashicorp/hcl/v2/hclparse
github.com/hashicorp/hcl/v2/hclsyntax
github.com/hashicorp/hcl/v2/hclwrite
github.com/hashicorp/hcl/v2/json
# github.com/imdario/mergo v0.3.12
# github.com/imdario/mergo v0.3.13
## explicit; go 1.13
github.com/imdario/mergo
# github.com/inconshreveable/mousetrap v1.0.0
@ -331,7 +331,7 @@ github.com/miekg/pkcs11
# github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7
## explicit
github.com/mitchellh/go-wordwrap
# github.com/mitchellh/mapstructure v1.4.3
# github.com/mitchellh/mapstructure v1.5.0
## explicit; go 1.14
github.com/mitchellh/mapstructure
# github.com/moby/buildkit v0.10.1-0.20220513101920-55f0ecfdafaa
@ -761,7 +761,7 @@ gopkg.in/inf.v0
# gopkg.in/yaml.v2 v2.4.0
## explicit; go 1.15
gopkg.in/yaml.v2
# gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
# gopkg.in/yaml.v3 v3.0.0
## explicit
gopkg.in/yaml.v3
# k8s.io/api v0.23.4 => k8s.io/api v0.22.4