From 21ac4c34fb91a8ddd174c6951597f3f8b0e50a5e Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Fri, 3 Feb 2023 03:19:59 +0100 Subject: [PATCH] update github.com/compose-spec/compose-go to v1.9.0 Signed-off-by: CrazyMax --- bake/compose.go | 2 +- go.mod | 4 +- go.sum | 10 +- .../compose-go/dotenv/godotenv.go | 106 +------------ .../compose-spec/compose-go/dotenv/parser.go | 50 +++--- .../compose-go/loader/full-example.yml | 6 +- .../compose-spec/compose-go/loader/loader.go | 142 +++++++++--------- .../compose-spec/compose-go/loader/merge.go | 51 ++++--- .../compose-go/loader/normalize.go | 11 ++ .../compose-go/loader/validate.go | 14 ++ .../compose-go/schema/compose-spec.json | 6 +- .../compose-spec/compose-go/types/project.go | 77 ++++++++-- .../compose-spec/compose-go/types/types.go | 34 ++++- vendor/modules.txt | 4 +- 14 files changed, 284 insertions(+), 233 deletions(-) diff --git a/bake/compose.go b/bake/compose.go index 8c4e5fe4..bb62765c 100644 --- a/bake/compose.go +++ b/bake/compose.go @@ -185,7 +185,7 @@ func loadDotEnv(curenv map[string]string, workingDir string) (map[string]string, return nil, err } - envs, err := dotenv.UnmarshalBytes(dt) + envs, err := dotenv.UnmarshalBytesWithLookup(dt, nil) if err != nil { return nil, err } diff --git a/go.mod b/go.mod index d6cc78f6..4fdcc252 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.17 require ( github.com/aws/aws-sdk-go-v2/config v1.15.5 - github.com/compose-spec/compose-go v1.6.0 + github.com/compose-spec/compose-go v1.9.0 github.com/containerd/console v1.0.3 github.com/containerd/containerd v1.6.16-0.20230124210447-1709cfe273d9 github.com/docker/cli v23.0.0-rc.1+incompatible @@ -80,7 +80,7 @@ require ( github.com/containerd/ttrpc v1.1.0 // indirect github.com/containerd/typeurl v1.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/distribution/distribution/v3 v3.0.0-20220725133111-4bf3547399eb // indirect + github.com/distribution/distribution/v3 v3.0.0-20221103125252-ebfa2a0ac0a9 // indirect github.com/docker/docker-credential-helpers v0.7.0 // indirect github.com/docker/go v1.5.1-1.0.20160303222718-d30aec9fd63c // indirect github.com/docker/go-connections v0.4.0 // indirect diff --git a/go.sum b/go.sum index eb06d899..bc8669e0 100644 --- a/go.sum +++ b/go.sum @@ -135,8 +135,8 @@ github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XP github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/compose-spec/compose-go v1.6.0 h1:7Ol/UULMUtbPmB0EYrETASRoum821JpOh/XaEf+hN+Q= -github.com/compose-spec/compose-go v1.6.0/go.mod h1:os+Ulh2jlZxY1XT1hbciERadjSUU/BtZ6+gcN7vD7J0= +github.com/compose-spec/compose-go v1.9.0 h1:oaewhNhUP/AClVs6ytHzcjw1xwK+2EMWuvHXj6tYvRc= +github.com/compose-spec/compose-go v1.9.0/go.mod h1:Tb5Ae2PsYN3GTqYqzl2IRbTPiJtPZZjMw8UKUvmehFk= github.com/containerd/cgroups v1.0.4 h1:jN/mbWBEaz+T1pi5OFtnkQ+8qnmEbAr1Oo1FRm5B0dA= github.com/containerd/console v1.0.3 h1:lIr7SlA5PxZyMV30bDW0MGbiOPXwc63yRuCP0ARubLw= github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= @@ -161,8 +161,8 @@ github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxG github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/distribution/distribution/v3 v3.0.0-20220725133111-4bf3547399eb h1:oCCuuU3kMO3sjZH/p7LamvQNW9SWoT4yQuMGcdSxGAE= -github.com/distribution/distribution/v3 v3.0.0-20220725133111-4bf3547399eb/go.mod h1:28YO/VJk9/64+sTGNuYaBjWxrXTPrj0C0XmgTIOjxX4= +github.com/distribution/distribution/v3 v3.0.0-20221103125252-ebfa2a0ac0a9 h1:doprs/RuXCuN864IfxC3h2qocrt158wGv3A5mcqSZQw= +github.com/distribution/distribution/v3 v3.0.0-20221103125252-ebfa2a0ac0a9/go.mod h1:6rIc5NMSjXjjnwzWWy3HAm9gDBu+X7aCzL8VrHIKgxM= github.com/docker/cli v23.0.0-rc.1+incompatible h1:Vl3pcUK4/LFAD56Ys3BrqgAtuwpWd/IO3amuSL0ZbP0= github.com/docker/cli v23.0.0-rc.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/cli-docs-tool v0.5.1 h1:jIk/cCZurZERhALPVKhqlNxTQGxn2kcI+56gE57PQXg= @@ -979,7 +979,7 @@ gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools/v3 v3.3.0 h1:MfDY1b1/0xN1CyMlQDac0ziEy9zJQd9CXBRRDHw2jJo= +gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/vendor/github.com/compose-spec/compose-go/dotenv/godotenv.go b/vendor/github.com/compose-spec/compose-go/dotenv/godotenv.go index c1c12eaf..ae34ffa9 100644 --- a/vendor/github.com/compose-spec/compose-go/dotenv/godotenv.go +++ b/vendor/github.com/compose-spec/compose-go/dotenv/godotenv.go @@ -15,13 +15,9 @@ package dotenv import ( "bytes" - "fmt" "io" "os" - "os/exec" "regexp" - "sort" - "strconv" "strings" "github.com/compose-spec/compose-go/template" @@ -72,21 +68,6 @@ func Load(filenames ...string) error { return load(false, filenames...) } -// Overload will read your env file(s) and load them into ENV for this process. -// -// Call this function as close as possible to the start of your program (ideally in main). -// -// If you call Overload without any args it will default to loading .env in the current path. -// -// You can otherwise tell it which files to load (there can be more than one) like: -// -// godotenv.Overload("fileone", "filetwo") -// -// It's important to note this WILL OVERRIDE an env variable that already exists - consider the .env file to forcefilly set all vars. -func Overload(filenames ...string) error { - return load(true, filenames...) -} - func load(overload bool, filenames ...string) error { filenames = filenamesOrDefault(filenames) for _, filename := range filenames { @@ -128,82 +109,13 @@ func Read(filenames ...string) (map[string]string, error) { return ReadWithLookup(nil, filenames...) } -// Unmarshal reads an env file from a string, returning a map of keys and values. -func Unmarshal(str string) (map[string]string, error) { - return UnmarshalBytes([]byte(str)) -} - -// UnmarshalBytes parses env file from byte slice of chars, returning a map of keys and values. -func UnmarshalBytes(src []byte) (map[string]string, error) { - return UnmarshalBytesWithLookup(src, nil) -} - // UnmarshalBytesWithLookup parses env file from byte slice of chars, returning a map of keys and values. func UnmarshalBytesWithLookup(src []byte, lookupFn LookupFn) (map[string]string, error) { out := make(map[string]string) - err := parseBytes(src, out, lookupFn) + err := newParser().parseBytes(src, out, lookupFn) return out, err } -// Exec loads env vars from the specified filenames (empty map falls back to default) -// then executes the cmd specified. -// -// Simply hooks up os.Stdin/err/out to the command and calls Run() -// -// If you want more fine grained control over your command it's recommended -// that you use `Load()` or `Read()` and the `os/exec` package yourself. -// -// Deprecated: Use the `os/exec` package directly. -func Exec(filenames []string, cmd string, cmdArgs []string) error { - if err := Load(filenames...); err != nil { - return err - } - - command := exec.Command(cmd, cmdArgs...) - command.Stdin = os.Stdin - command.Stdout = os.Stdout - command.Stderr = os.Stderr - return command.Run() -} - -// Write serializes the given environment and writes it to a file -// -// Deprecated: The serialization functions are untested and unmaintained. -func Write(envMap map[string]string, filename string) error { - //goland:noinspection GoDeprecation - content, err := Marshal(envMap) - if err != nil { - return err - } - file, err := os.Create(filename) - if err != nil { - return err - } - defer file.Close() - _, err = file.WriteString(content + "\n") - if err != nil { - return err - } - return file.Sync() -} - -// Marshal outputs the given environment as a dotenv-formatted environment file. -// Each line is in the format: KEY="VALUE" where VALUE is backslash-escaped. -// -// Deprecated: The serialization functions are untested and unmaintained. -func Marshal(envMap map[string]string) (string, error) { - lines := make([]string, 0, len(envMap)) - for k, v := range envMap { - if d, err := strconv.Atoi(v); err == nil { - lines = append(lines, fmt.Sprintf(`%s=%d`, k, d)) - } else { - lines = append(lines, fmt.Sprintf(`%s="%s"`, k, doubleQuoteEscape(v))) // nolint // Cannot use %q here - } - } - sort.Strings(lines) - return strings.Join(lines, "\n"), nil -} - func filenamesOrDefault(filenames []string) []string { if len(filenames) == 0 { return []string{".env"} @@ -255,19 +167,3 @@ func expandVariables(value string, envMap map[string]string, lookupFn LookupFn) } return retVal, nil } - -// Deprecated: only used by unsupported/untested code for Marshal/Write. -func doubleQuoteEscape(line string) string { - const doubleQuoteSpecialChars = "\\\n\r\"!$`" - for _, c := range doubleQuoteSpecialChars { - toReplace := "\\" + string(c) - if c == '\n' { - toReplace = `\n` - } - if c == '\r' { - toReplace = `\r` - } - line = strings.ReplaceAll(line, string(c), toReplace) - } - return line -} diff --git a/vendor/github.com/compose-spec/compose-go/dotenv/parser.go b/vendor/github.com/compose-spec/compose-go/dotenv/parser.go index 9a8d0f60..cc6f9331 100644 --- a/vendor/github.com/compose-spec/compose-go/dotenv/parser.go +++ b/vendor/github.com/compose-spec/compose-go/dotenv/parser.go @@ -21,24 +21,34 @@ var ( exportRegex = regexp.MustCompile(`^export\s+`) ) -func parseBytes(src []byte, out map[string]string, lookupFn LookupFn) error { +type parser struct { + line int +} + +func newParser() *parser { + return &parser{ + line: 1, + } +} + +func (p *parser) parseBytes(src []byte, out map[string]string, lookupFn LookupFn) error { cutset := src if lookupFn == nil { lookupFn = noLookupFn } for { - cutset = getStatementStart(cutset) + cutset = p.getStatementStart(cutset) if cutset == nil { // reached end of file break } - key, left, inherited, err := locateKeyName(cutset) + key, left, inherited, err := p.locateKeyName(cutset) if err != nil { return err } if strings.Contains(key, " ") { - return errors.New("key cannot contain a space") + return fmt.Errorf("line %d: key cannot contain a space", p.line) } if inherited { @@ -50,7 +60,7 @@ func parseBytes(src []byte, out map[string]string, lookupFn LookupFn) error { continue } - value, left, err := extractVarValue(left, out, lookupFn) + value, left, err := p.extractVarValue(left, out, lookupFn) if err != nil { return err } @@ -65,8 +75,8 @@ func parseBytes(src []byte, out map[string]string, lookupFn LookupFn) error { // getStatementPosition returns position of statement begin. // // It skips any comment line or non-whitespace character. -func getStatementStart(src []byte) []byte { - pos := indexOfNonSpaceChar(src) +func (p *parser) getStatementStart(src []byte) []byte { + pos := p.indexOfNonSpaceChar(src) if pos == -1 { return nil } @@ -81,12 +91,11 @@ func getStatementStart(src []byte) []byte { if pos == -1 { return nil } - - return getStatementStart(src[pos:]) + return p.getStatementStart(src[pos:]) } // locateKeyName locates and parses key name and returns rest of slice -func locateKeyName(src []byte) (string, []byte, bool, error) { +func (p *parser) locateKeyName(src []byte) (string, []byte, bool, error) { var key string var inherited bool // trim "export" and space at beginning @@ -108,16 +117,16 @@ loop: offset = i + 1 inherited = char == '\n' break loop - case '_', '.': + case '_', '.', '-', '[', ']': default: - // variable name should match [A-Za-z0-9_.] + // variable name should match [A-Za-z0-9_.-] if unicode.IsLetter(rchar) || unicode.IsNumber(rchar) { continue } return "", nil, inherited, fmt.Errorf( - `unexpected character %q in variable name near %q`, - string(char), string(src)) + `line %d: unexpected character %q in variable name`, + p.line, string(char)) } } @@ -132,11 +141,12 @@ loop: } // extractVarValue extracts variable value and returns rest of slice -func extractVarValue(src []byte, envMap map[string]string, lookupFn LookupFn) (string, []byte, error) { +func (p *parser) extractVarValue(src []byte, envMap map[string]string, lookupFn LookupFn) (string, []byte, error) { quote, isQuoted := hasQuotePrefix(src) if !isQuoted { // unquoted value - read until new line value, rest, _ := bytes.Cut(src, []byte("\n")) + p.line++ // Remove inline comments on unquoted lines value, _, _ = bytes.Cut(value, []byte(" #")) @@ -147,6 +157,9 @@ func extractVarValue(src []byte, envMap map[string]string, lookupFn LookupFn) (s // lookup quoted string terminator for i := 1; i < len(src); i++ { + if src[i] == '\n' { + p.line++ + } if char := src[i]; char != quote { continue } @@ -177,7 +190,7 @@ func extractVarValue(src []byte, envMap map[string]string, lookupFn LookupFn) (s valEndIndex = len(src) } - return "", nil, fmt.Errorf("unterminated quoted value %s", src[:valEndIndex]) + return "", nil, fmt.Errorf("line %d: unterminated quoted value %s", p.line, src[:valEndIndex]) } func expandEscapes(str string) string { @@ -212,8 +225,11 @@ func expandEscapes(str string) string { return out } -func indexOfNonSpaceChar(src []byte) int { +func (p *parser) indexOfNonSpaceChar(src []byte) int { return bytes.IndexFunc(src, func(r rune) bool { + if r == '\n' { + p.line++ + } return !unicode.IsSpace(r) }) } diff --git a/vendor/github.com/compose-spec/compose-go/loader/full-example.yml b/vendor/github.com/compose-spec/compose-go/loader/full-example.yml index cb300c9f..5c62e815 100644 --- a/vendor/github.com/compose-spec/compose-go/loader/full-example.yml +++ b/vendor/github.com/compose-spec/compose-go/loader/full-example.yml @@ -160,8 +160,8 @@ services: # somehost: "162.242.195.82" # otherhost: "50.31.209.229" extra_hosts: - - "somehost:162.242.195.82" - "otherhost:50.31.209.229" + - "somehost:162.242.195.82" hostname: foo @@ -182,6 +182,8 @@ services: ipc: host + uts: host + # Mapping or list # Mapping values can be strings, numbers or null labels: @@ -428,6 +430,8 @@ secrets: environment: BAR x-bar: baz x-foo: bar + secret5: + file: /abs/secret_data x-bar: baz x-foo: bar x-nested: diff --git a/vendor/github.com/compose-spec/compose-go/loader/loader.go b/vendor/github.com/compose-spec/compose-go/loader/loader.go index acf11a95..a7a8ca9f 100644 --- a/vendor/github.com/compose-spec/compose-go/loader/loader.go +++ b/vendor/github.com/compose-spec/compose-go/loader/loader.go @@ -17,9 +17,7 @@ package loader import ( - "bytes" "fmt" - "io" "os" paths "path" "path/filepath" @@ -30,7 +28,6 @@ import ( "time" "github.com/compose-spec/compose-go/consts" - "github.com/compose-spec/compose-go/dotenv" interp "github.com/compose-spec/compose-go/interpolation" "github.com/compose-spec/compose-go/schema" "github.com/compose-spec/compose-go/template" @@ -67,6 +64,8 @@ type Options struct { projectName string // Indicates when the projectName was imperatively set or guessed from path projectNameImperativelySet bool + // Profiles set profiles to enable + Profiles []string } func (o *Options) SetProjectName(name string, imperativelySet bool) { @@ -125,6 +124,13 @@ func WithSkipValidation(opts *Options) { opts.SkipValidation = true } +// WithProfiles sets profiles to be activated +func WithProfiles(profiles []string) func(*Options) { + return func(opts *Options) { + opts.Profiles = profiles + } +} + // ParseYAML reads the bytes from a file, parses the bytes into a mapping // structure, and returns it. func ParseYAML(source []byte) (map[string]interface{}, error) { @@ -161,12 +167,22 @@ func Load(configDetails types.ConfigDetails, options ...func(*Options)) (*types. op(opts) } - projectName := projectName(configDetails, opts) + projectName, err := projectName(configDetails, opts) + if err != nil { + return nil, err + } var configs []*types.Config for i, file := range configDetails.ConfigFiles { configDict := file.Config if configDict == nil { + if len(file.Content) == 0 { + content, err := os.ReadFile(file.Filename) + if err != nil { + return nil, err + } + file.Content = content + } dict, err := parseConfig(file.Content, opts) if err != nil { return nil, err @@ -188,12 +204,6 @@ func Load(configDetails types.ConfigDetails, options ...func(*Options)) (*types. if err != nil { return nil, err } - if opts.discardEnvFiles { - for i := range cfg.Services { - cfg.Services[i].EnvFile = nil - } - } - configs = append(configs, cfg) } @@ -236,22 +246,35 @@ func Load(configDetails types.ConfigDetails, options ...func(*Options)) (*types. } } - return project, nil + if len(opts.Profiles) > 0 { + project.ApplyProfiles(opts.Profiles) + } + + err = project.ResolveServicesEnvironment(opts.discardEnvFiles) + + return project, err } -func projectName(details types.ConfigDetails, opts *Options) string { +func projectName(details types.ConfigDetails, opts *Options) (string, error) { projectName, projectNameImperativelySet := opts.GetProjectName() var pjNameFromConfigFile string for _, configFile := range details.ConfigFiles { yml, err := ParseYAML(configFile.Content) if err != nil { - return "" + return "", nil } if val, ok := yml["name"]; ok && val != "" { pjNameFromConfigFile = yml["name"].(string) } } + if !opts.SkipInterpolation { + interpolated, err := interp.Interpolate(map[string]interface{}{"name": pjNameFromConfigFile}, *opts.Interpolate) + if err != nil { + return "", err + } + pjNameFromConfigFile = interpolated["name"].(string) + } pjNameFromConfigFile = NormalizeProjectName(pjNameFromConfigFile) if !projectNameImperativelySet && pjNameFromConfigFile != "" { projectName = pjNameFromConfigFile @@ -260,7 +283,7 @@ func projectName(details types.ConfigDetails, opts *Options) string { if _, ok := details.Environment[consts.ComposeProjectName]; !ok && projectName != "" { details.Environment[consts.ComposeProjectName] = projectName } - return projectName + return projectName, nil } func NormalizeProjectName(s string) string { @@ -508,6 +531,10 @@ func loadServiceWithExtends(filename, name string, servicesDict map[string]inter return nil, fmt.Errorf("cannot extend service %q in %s: service not found", name, filename) } + if target == nil { + target = map[string]interface{}{} + } + serviceConfig, err := LoadService(name, target.(map[string]interface{}), workingDir, lookupEnv, opts.ResolvePaths, opts.ConvertWindowsPaths) if err != nil { return nil, err @@ -547,16 +574,14 @@ func loadServiceWithExtends(filename, name string, servicesDict map[string]inter // absolute path. baseFileParent := filepath.Dir(*file) if baseService.Build != nil { - // Note that the Dockerfile is always defined relative to the - // build context, so there's no need to update the Dockerfile field. - baseService.Build.Context = absPath(baseFileParent, baseService.Build.Context) + baseService.Build.Context = resolveBuildContextPath(baseFileParent, baseService.Build.Context) } for i, vol := range baseService.Volumes { if vol.Type != types.VolumeTypeBind { continue } - baseService.Volumes[i].Source = absPath(baseFileParent, vol.Source) + baseService.Volumes[i].Source = resolveMaybeUnixPath(vol.Source, baseFileParent, lookupEnv) } } @@ -569,6 +594,19 @@ func loadServiceWithExtends(filename, name string, servicesDict map[string]inter return serviceConfig, nil } +func resolveBuildContextPath(baseFileParent string, context string) string { + // Checks if the context is an HTTP(S) URL or a remote git repository URL + for _, prefix := range []string{"https://", "http://", "git://", "github.com/", "git@"} { + if strings.HasPrefix(context, prefix) { + return context + } + } + + // Note that the Dockerfile is always defined relative to the + // build context, so there's no need to update the Dockerfile field. + return absPath(baseFileParent, context) +} + // LoadService produces a single ServiceConfig from a compose file Dict // the serviceDict is not validated if directly used. Use Load() to enable validation func LoadService(name string, serviceDict map[string]interface{}, workingDir string, lookupEnv template.Mapping, resolvePaths bool, convertPaths bool) (*types.ServiceConfig, error) { @@ -580,10 +618,6 @@ func LoadService(name string, serviceDict map[string]interface{}, workingDir str } serviceConfig.Name = name - if err := resolveEnvironment(serviceConfig, workingDir, lookupEnv); err != nil { - return nil, err - } - for i, volume := range serviceConfig.Volumes { if volume.Type != types.VolumeTypeBind { continue @@ -620,48 +654,8 @@ func convertVolumePath(volume types.ServiceVolumeConfig) types.ServiceVolumeConf return volume } -func resolveEnvironment(serviceConfig *types.ServiceConfig, workingDir string, lookupEnv template.Mapping) error { - environment := types.MappingWithEquals{} - - if len(serviceConfig.EnvFile) > 0 { - if serviceConfig.Environment == nil { - serviceConfig.Environment = types.MappingWithEquals{} - } - for _, envFile := range serviceConfig.EnvFile { - filePath := absPath(workingDir, envFile) - file, err := os.Open(filePath) - if err != nil { - return err - } - - b, err := io.ReadAll(file) - if err != nil { - return err - } - - // Do not defer to avoid it inside a loop - file.Close() //nolint:errcheck - - fileVars, err := dotenv.ParseWithLookup(bytes.NewBuffer(b), dotenv.LookupFn(lookupEnv)) - if err != nil { - return err - } - env := types.MappingWithEquals{} - for k, v := range fileVars { - v := v - env[k] = &v - } - environment.OverrideBy(env.Resolve(lookupEnv).RemoveEmpty()) - } - } - - environment.OverrideBy(serviceConfig.Environment.Resolve(lookupEnv)) - serviceConfig.Environment = environment - return nil -} - -func resolveVolumePath(volume types.ServiceVolumeConfig, workingDir string, lookupEnv template.Mapping) types.ServiceVolumeConfig { - filePath := expandUser(volume.Source, lookupEnv) +func resolveMaybeUnixPath(path string, workingDir string, lookupEnv template.Mapping) string { + filePath := expandUser(path, lookupEnv) // Check if source is an absolute path (either Unix or Windows), to // handle a Windows client with a Unix daemon or vice-versa. // @@ -671,10 +665,21 @@ func resolveVolumePath(volume types.ServiceVolumeConfig, workingDir string, look if !paths.IsAbs(filePath) && !isAbs(filePath) { filePath = absPath(workingDir, filePath) } - volume.Source = filePath + return filePath +} + +func resolveVolumePath(volume types.ServiceVolumeConfig, workingDir string, lookupEnv template.Mapping) types.ServiceVolumeConfig { + volume.Source = resolveMaybeUnixPath(volume.Source, workingDir, lookupEnv) return volume } +func resolveSecretsPath(secret types.SecretConfig, workingDir string, lookupEnv template.Mapping) types.SecretConfig { + if !secret.External.External && secret.File != "" { + secret.File = resolveMaybeUnixPath(secret.File, workingDir, lookupEnv) + } + return secret +} + // TODO: make this more robust func expandUser(path string, lookupEnv template.Mapping) string { if strings.HasPrefix(path, "~") { @@ -723,7 +728,7 @@ func LoadNetworks(source map[string]interface{}) (map[string]types.NetworkConfig if network.Name != "" { return nil, errors.Errorf("network %s: network.external.name and network.name conflict; only use network.name", name) } - logrus.Warnf("network %s: network.external.name is deprecated in favor of network.name", name) + logrus.Warnf("network %s: network.external.name is deprecated. Please set network.name with external: true", name) network.Name = network.External.Name network.External.Name = "" case network.Name == "": @@ -782,11 +787,14 @@ func LoadSecrets(source map[string]interface{}, details types.ConfigDetails, res return secrets, err } for name, secret := range secrets { - obj, err := loadFileObjectConfig(name, "secret", types.FileObjectConfig(secret), details, resolvePaths) + obj, err := loadFileObjectConfig(name, "secret", types.FileObjectConfig(secret), details, false) if err != nil { return nil, err } secretConfig := types.SecretConfig(obj) + if resolvePaths { + secretConfig = resolveSecretsPath(secretConfig, details.WorkingDir, details.LookupEnv) + } secrets[name] = secretConfig } return secrets, nil diff --git a/vendor/github.com/compose-spec/compose-go/loader/merge.go b/vendor/github.com/compose-spec/compose-go/loader/merge.go index bf10cf77..3c24008d 100644 --- a/vendor/github.com/compose-spec/compose-go/loader/merge.go +++ b/vendor/github.com/compose-spec/compose-go/loader/merge.go @@ -38,11 +38,19 @@ var serviceSpecials = &specials{ reflect.TypeOf([]types.ServiceSecretConfig{}): mergeSlice(toServiceSecretConfigsMap, toServiceSecretConfigsSlice), reflect.TypeOf([]types.ServiceConfigObjConfig{}): mergeSlice(toServiceConfigObjConfigsMap, toSServiceConfigObjConfigsSlice), reflect.TypeOf(&types.UlimitsConfig{}): mergeUlimitsConfig, - reflect.TypeOf(&types.ServiceNetworkConfig{}): mergeServiceNetworkConfig, }, } func (s *specials) Transformer(t reflect.Type) func(dst, src reflect.Value) error { + // TODO this is a workaround waiting for imdario/mergo#131 + if t.Kind() == reflect.Pointer && t.Elem().Kind() == reflect.Bool { + return func(dst, src reflect.Value) error { + if dst.CanSet() && !src.IsNil() { + dst.Set(src) + } + return nil + } + } if fn, ok := s.m[t]; ok { return fn } @@ -113,12 +121,18 @@ func mergeServices(base, override []types.ServiceConfig) ([]types.ServiceConfig, } func _merge(baseService *types.ServiceConfig, overrideService *types.ServiceConfig) (*types.ServiceConfig, error) { - if err := mergo.Merge(baseService, overrideService, mergo.WithAppendSlice, mergo.WithOverride, mergo.WithTransformers(serviceSpecials)); err != nil { + if err := mergo.Merge(baseService, overrideService, + mergo.WithAppendSlice, + mergo.WithOverride, + mergo.WithTransformers(serviceSpecials)); err != nil { return nil, err } if overrideService.Command != nil { baseService.Command = overrideService.Command } + if overrideService.HealthCheck != nil { + baseService.HealthCheck.Test = overrideService.HealthCheck.Test + } if overrideService.Entrypoint != nil { baseService.Entrypoint = overrideService.Entrypoint } @@ -127,9 +141,26 @@ func _merge(baseService *types.ServiceConfig, overrideService *types.ServiceConf } else { baseService.Environment = overrideService.Environment } + baseService.Expose = unique(baseService.Expose) return baseService, nil } +func unique(slice []string) []string { + if slice == nil { + return nil + } + uniqMap := make(map[string]struct{}) + for _, v := range slice { + uniqMap[v] = struct{}{} + } + + uniqSlice := make([]string, 0, len(uniqMap)) + for v := range uniqMap { + uniqSlice = append(uniqSlice, v) + } + return uniqSlice +} + func toServiceSecretConfigsMap(s interface{}) (map[interface{}]interface{}, error) { secrets, ok := s.([]types.ServiceSecretConfig) if !ok { @@ -299,7 +330,7 @@ func mergeLoggingConfig(dst, src reflect.Value) error { return nil } -//nolint: unparam +// nolint: unparam func mergeUlimitsConfig(dst, src reflect.Value) error { if src.Interface() != reflect.Zero(reflect.TypeOf(src.Interface())).Interface() { dst.Elem().Set(src.Elem()) @@ -307,20 +338,6 @@ func mergeUlimitsConfig(dst, src reflect.Value) error { return nil } -//nolint: unparam -func mergeServiceNetworkConfig(dst, src reflect.Value) error { - if src.Interface() != reflect.Zero(reflect.TypeOf(src.Interface())).Interface() { - dst.Elem().FieldByName("Aliases").Set(src.Elem().FieldByName("Aliases")) - if ipv4 := src.Elem().FieldByName("Ipv4Address").Interface().(string); ipv4 != "" { - dst.Elem().FieldByName("Ipv4Address").SetString(ipv4) - } - if ipv6 := src.Elem().FieldByName("Ipv6Address").Interface().(string); ipv6 != "" { - dst.Elem().FieldByName("Ipv6Address").SetString(ipv6) - } - } - return nil -} - func getLoggingDriver(v reflect.Value) string { return v.FieldByName("Driver").String() } diff --git a/vendor/github.com/compose-spec/compose-go/loader/normalize.go b/vendor/github.com/compose-spec/compose-go/loader/normalize.go index 4b98d624..b4c3acc1 100644 --- a/vendor/github.com/compose-spec/compose-go/loader/normalize.go +++ b/vendor/github.com/compose-spec/compose-go/loader/normalize.go @@ -85,6 +85,9 @@ func normalize(project *types.Project, resolvePaths bool) error { } s.Build.Args = s.Build.Args.Resolve(fn) } + for j, f := range s.EnvFile { + s.EnvFile[j] = absPath(project.WorkingDir, f) + } s.Environment = s.Environment.Resolve(fn) err := relocateLogDriver(&s) @@ -110,6 +113,14 @@ func normalize(project *types.Project, resolvePaths bool) error { project.Services[i] = s } + for name, config := range project.Volumes { + if config.Driver == "local" && config.DriverOpts["o"] == "bind" { + // This is actually a bind mount + config.DriverOpts["device"] = absPath(project.WorkingDir, config.DriverOpts["device"]) + project.Volumes[name] = config + } + } + setNameFromKey(project) return nil diff --git a/vendor/github.com/compose-spec/compose-go/loader/validate.go b/vendor/github.com/compose-spec/compose-go/loader/validate.go index 4d635889..a6a2c30a 100644 --- a/vendor/github.com/compose-spec/compose-go/loader/validate.go +++ b/vendor/github.com/compose-spec/compose-go/loader/validate.go @@ -38,6 +38,14 @@ func checkConsistency(project *types.Project) error { } } + if s.HealthCheck != nil && len(s.HealthCheck.Test) > 0 { + switch s.HealthCheck.Test[0] { + case "CMD", "CMD-SHELL", "NONE": + default: + return errors.New(`healthcheck.test must start either by "CMD", "CMD-SHELL" or "NONE"`) + } + } + for dependedService := range s.DependsOn { if _, err := project.GetService(dependedService); err != nil { return errors.Wrap(errdefs.ErrInvalid, fmt.Sprintf("service %q depends on undefined service %s", s.Name, dependedService)) @@ -70,6 +78,12 @@ func checkConsistency(project *types.Project) error { return errors.Wrap(errdefs.ErrInvalid, fmt.Sprintf("service %q refers to undefined config %s", s.Name, config.Source)) } } + + for _, secret := range s.Secrets { + if _, ok := project.Secrets[secret.Source]; !ok { + return errors.Wrap(errdefs.ErrInvalid, fmt.Sprintf("service %q refers to undefined secret %s", s.Name, secret.Source)) + } + } } for name, secret := range project.Secrets { diff --git a/vendor/github.com/compose-spec/compose-go/schema/compose-spec.json b/vendor/github.com/compose-spec/compose-go/schema/compose-spec.json index d444e71d..6f827971 100644 --- a/vendor/github.com/compose-spec/compose-go/schema/compose-spec.json +++ b/vendor/github.com/compose-spec/compose-go/schema/compose-spec.json @@ -102,6 +102,7 @@ "shm_size": {"type": ["integer", "string"]}, "extra_hosts": {"$ref": "#/definitions/list_or_dict"}, "isolation": {"type": "string"}, + "privileged": {"type": "boolean"}, "secrets": {"$ref": "#/definitions/service_config_or_secret"}, "tags": {"type": "array", "items": {"type": "string"}}, "platforms": {"type": "array", "items": {"type": "string"}} @@ -140,6 +141,7 @@ }, "cap_add": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, "cap_drop": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "cgroup": {"type": "string", "enum": ["host", "private"]}, "cgroup_parent": {"type": "string"}, "command": { "oneOf": [ @@ -365,6 +367,7 @@ } }, "user": {"type": "string"}, + "uts": {"type": "string"}, "userns_mode": {"type": "string"}, "volumes": { "type": "array", @@ -406,7 +409,8 @@ {"type": "integer", "minimum": 0}, {"type": "string"} ] - } + }, + "mode": {"type": "number"} }, "additionalProperties": false, "patternProperties": {"^x-": {}} diff --git a/vendor/github.com/compose-spec/compose-go/types/project.go b/vendor/github.com/compose-spec/compose-go/types/project.go index 6b28e863..fd53d44f 100644 --- a/vendor/github.com/compose-spec/compose-go/types/project.go +++ b/vendor/github.com/compose-spec/compose-go/types/project.go @@ -17,28 +17,31 @@ package types import ( + "bytes" "fmt" "os" "path/filepath" "sort" + "github.com/compose-spec/compose-go/dotenv" "github.com/distribution/distribution/v3/reference" godigest "github.com/opencontainers/go-digest" + "github.com/pkg/errors" "golang.org/x/sync/errgroup" ) // Project is the result of loading a set of compose files type Project struct { - Name string `yaml:"name,omitempty" json:"name,omitempty"` - WorkingDir string `yaml:"-" json:"-"` - Services Services `json:"services"` - Networks Networks `yaml:",omitempty" json:"networks,omitempty"` - Volumes Volumes `yaml:",omitempty" json:"volumes,omitempty"` - Secrets Secrets `yaml:",omitempty" json:"secrets,omitempty"` - Configs Configs `yaml:",omitempty" json:"configs,omitempty"` - Extensions Extensions `yaml:",inline" json:"-"` // https://github.com/golang/go/issues/6213 - ComposeFiles []string `yaml:"-" json:"-"` - Environment map[string]string `yaml:"-" json:"-"` + Name string `yaml:"name,omitempty" json:"name,omitempty"` + WorkingDir string `yaml:"-" json:"-"` + Services Services `json:"services"` + Networks Networks `yaml:",omitempty" json:"networks,omitempty"` + Volumes Volumes `yaml:",omitempty" json:"volumes,omitempty"` + Secrets Secrets `yaml:",omitempty" json:"secrets,omitempty"` + Configs Configs `yaml:",omitempty" json:"configs,omitempty"` + Extensions Extensions `yaml:",inline" json:"-"` // https://github.com/golang/go/issues/6213 + ComposeFiles []string `yaml:"-" json:"-"` + Environment Mapping `yaml:"-" json:"-"` // DisabledServices track services which have been disable as profile is not active DisabledServices Services `yaml:"-" json:"-"` @@ -258,25 +261,33 @@ func (p *Project) WithoutUnnecessaryResources() { networks := Networks{} for k := range requiredNetworks { - networks[k] = p.Networks[k] + if value, ok := p.Networks[k]; ok { + networks[k] = value + } } p.Networks = networks volumes := Volumes{} for k := range requiredVolumes { - volumes[k] = p.Volumes[k] + if value, ok := p.Volumes[k]; ok { + volumes[k] = value + } } p.Volumes = volumes secrets := Secrets{} for k := range requiredSecrets { - secrets[k] = p.Secrets[k] + if value, ok := p.Secrets[k]; ok { + secrets[k] = value + } } p.Secrets = secrets configs := Configs{} for k := range requiredConfigs { - configs[k] = p.Configs[k] + if value, ok := p.Configs[k]; ok { + configs[k] = value + } } p.Configs = configs } @@ -345,3 +356,41 @@ func (p *Project) ResolveImages(resolver func(named reference.Named) (godigest.D } return eg.Wait() } + +// ResolveServicesEnvironment parse env_files set for services to resolve the actual environment map for services +func (p Project) ResolveServicesEnvironment(discardEnvFiles bool) error { + for i, service := range p.Services { + service.Environment = service.Environment.Resolve(p.Environment.Resolve) + + environment := MappingWithEquals{} + // resolve variables based on other files we already parsed, + project's environment + var resolve dotenv.LookupFn = func(s string) (string, bool) { + v, ok := environment[s] + if ok && v != nil { + return *v, ok + } + return p.Environment.Resolve(s) + } + + for _, envFile := range service.EnvFile { + b, err := os.ReadFile(envFile) + if err != nil { + return errors.Wrapf(err, "Failed to load %s", envFile) + } + + fileVars, err := dotenv.ParseWithLookup(bytes.NewBuffer(b), resolve) + if err != nil { + return err + } + environment.OverrideBy(Mapping(fileVars).ToMappingWithEquals()) + } + + service.Environment = environment.OverrideBy(service.Environment) + + if discardEnvFiles { + service.EnvFile = nil + } + p.Services[i] = service + } + return nil +} diff --git a/vendor/github.com/compose-spec/compose-go/types/types.go b/vendor/github.com/compose-spec/compose-go/types/types.go index cb42c6e9..e920e635 100644 --- a/vendor/github.com/compose-spec/compose-go/types/types.go +++ b/vendor/github.com/compose-spec/compose-go/types/types.go @@ -93,6 +93,7 @@ type ServiceConfig struct { CapAdd []string `mapstructure:"cap_add" yaml:"cap_add,omitempty" json:"cap_add,omitempty"` CapDrop []string `mapstructure:"cap_drop" yaml:"cap_drop,omitempty" json:"cap_drop,omitempty"` CgroupParent string `mapstructure:"cgroup_parent" yaml:"cgroup_parent,omitempty" json:"cgroup_parent,omitempty"` + Cgroup string `mapstructure:"cgroup" yaml:"cgroup,omitempty" json:"cgroup,omitempty"` CPUCount int64 `mapstructure:"cpu_count" yaml:"cpu_count,omitempty" json:"cpu_count,omitempty"` CPUPercent float32 `mapstructure:"cpu_percent" yaml:"cpu_percent,omitempty" json:"cpu_percent,omitempty"` CPUPeriod int64 `mapstructure:"cpu_period" yaml:"cpu_period,omitempty" json:"cpu_period,omitempty"` @@ -278,7 +279,8 @@ func (s ServiceConfig) GetDependencies() []string { } for _, vol := range s.VolumesFrom { if !strings.HasPrefix(s.Pid, ContainerPrefix) { - dependencies.append(vol) + spec := strings.Split(vol, ":") + dependencies.append(spec[0]) } } @@ -319,6 +321,7 @@ type BuildConfig struct { Secrets []ServiceSecretConfig `yaml:",omitempty" json:"secrets,omitempty"` Tags StringList `mapstructure:"tags" yaml:"tags,omitempty" json:"tags,omitempty"` Platforms StringList `mapstructure:"platforms" yaml:"platforms,omitempty" json:"platforms,omitempty"` + Privileged bool `yaml:",omitempty" json:"privileged,omitempty"` Extensions map[string]interface{} `yaml:",inline" json:"-"` } @@ -479,6 +482,21 @@ func NewMapping(values []string) Mapping { return mapping } +// ToMappingWithEquals converts Mapping into a MappingWithEquals with pointer references +func (m Mapping) ToMappingWithEquals() MappingWithEquals { + mapping := MappingWithEquals{} + for k, v := range m { + v := v + mapping[k] = &v + } + return mapping +} + +func (m Mapping) Resolve(s string) (string, bool) { + v, ok := m[s] + return v, ok +} + // Labels is a mapping type for labels type Labels map[string]string @@ -539,6 +557,18 @@ func (h HostsList) AsList() []string { return l } +func (h HostsList) MarshalYAML() (interface{}, error) { + list := h.AsList() + sort.Strings(list) + return list, nil +} + +func (h HostsList) MarshalJSON() ([]byte, error) { + list := h.AsList() + sort.Strings(list) + return json.Marshal(list) +} + // LoggingConfig the logging configuration for a service type LoggingConfig struct { Driver string `yaml:",omitempty" json:"driver,omitempty"` @@ -828,6 +858,8 @@ type ServiceVolumeVolume struct { type ServiceVolumeTmpfs struct { Size UnitBytes `yaml:",omitempty" json:"size,omitempty"` + Mode uint32 `yaml:",omitempty" json:"mode,omitempty"` + Extensions map[string]interface{} `yaml:",inline" json:"-"` } diff --git a/vendor/modules.txt b/vendor/modules.txt index a397db75..4c9d78e0 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -137,7 +137,7 @@ github.com/cenkalti/backoff/v4 github.com/cespare/xxhash/v2 # github.com/cloudflare/cfssl v0.0.0-20181213083726-b94e044bb51e ## explicit -# github.com/compose-spec/compose-go v1.6.0 +# github.com/compose-spec/compose-go v1.9.0 ## explicit; go 1.18 github.com/compose-spec/compose-go/consts github.com/compose-spec/compose-go/dotenv @@ -191,7 +191,7 @@ github.com/containerd/typeurl # github.com/davecgh/go-spew v1.1.1 ## explicit github.com/davecgh/go-spew/spew -# github.com/distribution/distribution/v3 v3.0.0-20220725133111-4bf3547399eb +# github.com/distribution/distribution/v3 v3.0.0-20221103125252-ebfa2a0ac0a9 ## explicit; go 1.18 github.com/distribution/distribution/v3/digestset github.com/distribution/distribution/v3/reference