mirror of https://github.com/docker/buildx.git
vendor: github.com/moby/buildkit 5ae9b23c40a9 (master / v0.13.0-dev)
full diff: -36ef4d8c0d...f098008783-d5c1d785b0...5ae9b23c40Signed-off-by: Sebastiaan van Stijn <github@gone.nl>
This commit is contained in:
Sebastiaan van Stijn
parent
898a8eeddf
commit
c855277d53
4
go.mod
4
go.mod
|
|
@ -24,7 +24,7 @@ require (
|
|||
github.com/google/uuid v1.3.0
|
||||
github.com/hashicorp/go-cty-funcs v0.0.0-20230405223818-a090f58aa992
|
||||
github.com/hashicorp/hcl/v2 v2.19.1
|
||||
github.com/moby/buildkit v0.13.0-beta1.0.20231023114302-d5c1d785b042
|
||||
github.com/moby/buildkit v0.13.0-beta1.0.20231114164402-5ae9b23c40a9 // master (v0.13.0-dev)
|
||||
github.com/moby/sys/mountinfo v0.6.2
|
||||
github.com/moby/sys/signal v0.7.0
|
||||
github.com/morikuni/aec v1.0.0
|
||||
|
|
@ -126,7 +126,7 @@ require (
|
|||
github.com/secure-systems-lab/go-securesystemslib v0.4.0 // indirect
|
||||
github.com/shibumi/go-pathspec v1.3.0 // indirect
|
||||
github.com/theupdateframework/notary v0.7.0 // indirect
|
||||
github.com/tonistiigi/fsutil v0.0.0-20230629203738-36ef4d8c0dbb // indirect
|
||||
github.com/tonistiigi/fsutil v0.0.0-20230825212630-f09800878302 // indirect
|
||||
github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea // indirect
|
||||
github.com/tonistiigi/vt100 v0.0.0-20230623042737-f9a4f7ef6531 // indirect
|
||||
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
|
||||
|
|
|
|||
16
go.sum
16
go.sum
|
|
@ -137,7 +137,7 @@ github.com/containerd/continuity v0.4.2/go.mod h1:F6PTNCKepoxEaXLQp3wDAjygEnImnZ
|
|||
github.com/containerd/fifo v1.1.0 h1:4I2mbh5stb1u6ycIABlBw9zgtlK8viPI9QkQNRQEEmY=
|
||||
github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
|
||||
github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
|
||||
github.com/containerd/nydus-snapshotter v0.8.2 h1:7SOrMU2YmLzfbsr5J7liMZJlNi5WT6vtIOxLGv+iz7E=
|
||||
github.com/containerd/nydus-snapshotter v0.13.1 h1:5XNkCZ9ivLXCcyx3Jbbfh/fntkcls69uBg0x9VE8zlk=
|
||||
github.com/containerd/stargz-snapshotter v0.14.3 h1:OTUVZoPSPs8mGgmQUE1dqw3WX/3nrsmsurW7UPLWl1U=
|
||||
github.com/containerd/stargz-snapshotter/estargz v0.14.3 h1:OqlDCK3ZVUO6C3B/5FSkDwbkEETK84kQgEeFwDC+62k=
|
||||
github.com/containerd/ttrpc v1.2.2 h1:9vqZr0pxwOF5koz6N0N3kJ0zDHokrcPxIR/ZR2YFtOs=
|
||||
|
|
@ -388,8 +388,8 @@ github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7/go.mod h1:ZX
|
|||
github.com/mitchellh/mapstructure v0.0.0-20150613213606-2caf8efc9366/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
|
||||
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
|
||||
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
|
||||
github.com/moby/buildkit v0.13.0-beta1.0.20231023114302-d5c1d785b042 h1:1J+fRIucIeyl1gvSYOlTcN0gmsZ8SMlLdkwB01PEn94=
|
||||
github.com/moby/buildkit v0.13.0-beta1.0.20231023114302-d5c1d785b042/go.mod h1:3sbzGMUHhpx+6++efVlHhvcarzusX1+QbGTR/S4y9gI=
|
||||
github.com/moby/buildkit v0.13.0-beta1.0.20231114164402-5ae9b23c40a9 h1:9BTYsENJ9teIBzqRRFIs7YX3ZAMwruBud1CL+nqoqVs=
|
||||
github.com/moby/buildkit v0.13.0-beta1.0.20231114164402-5ae9b23c40a9/go.mod h1:bhWy+9dHWVXaUaVzSTBhVFiNNzK0uKPWfF2taj6ln2o=
|
||||
github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg=
|
||||
github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc=
|
||||
github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk=
|
||||
|
|
@ -432,8 +432,8 @@ github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3I
|
|||
github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
|
||||
github.com/opencontainers/image-spec v1.1.0-rc5 h1:Ygwkfw9bpDvs+c9E34SdgGOj41dX/cbdlwvlWt0pnFI=
|
||||
github.com/opencontainers/image-spec v1.1.0-rc5/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8=
|
||||
github.com/opencontainers/runc v1.1.9 h1:XR0VIHTGce5eWPkaPesqTBrhW2yAcaraWfsEalNwQLM=
|
||||
github.com/opencontainers/runtime-spec v1.1.0-rc.2 h1:ucBtEms2tamYYW/SvGpvq9yUN0NEVL6oyLEwDcTSrk8=
|
||||
github.com/opencontainers/runc v1.1.10 h1:EaL5WeO9lv9wmS6SASjszOeQdSctvpbu0DdBQBizE40=
|
||||
github.com/opencontainers/runtime-spec v1.1.0 h1:HHUyrt9mwHUjtasSbXSMvs4cyFxh+Bll4AjJ9odEGpg=
|
||||
github.com/opencontainers/selinux v1.11.0 h1:+5Zbo97w3Lbmb3PeqQtpmTkMwsW5nRI3YaLpt7tQ7oU=
|
||||
github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU=
|
||||
github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
|
||||
|
|
@ -514,13 +514,13 @@ github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcU
|
|||
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
|
||||
github.com/theupdateframework/notary v0.7.0 h1:QyagRZ7wlSpjT5N2qQAh/pN+DVqgekv4DzbAiAiEL3c=
|
||||
github.com/theupdateframework/notary v0.7.0/go.mod h1:c9DRxcmhHmVLDay4/2fUYdISnHqbFDGRSlXPO0AhYWw=
|
||||
github.com/tonistiigi/fsutil v0.0.0-20230629203738-36ef4d8c0dbb h1:uUe8rNyVXM8moActoBol6Xf6xX2GMr7SosR2EywMvGg=
|
||||
github.com/tonistiigi/fsutil v0.0.0-20230629203738-36ef4d8c0dbb/go.mod h1:SxX/oNQ/ag6Vaoli547ipFK9J7BZn5JqJG0JE8lf8bA=
|
||||
github.com/tonistiigi/fsutil v0.0.0-20230825212630-f09800878302 h1:ZT8ibgassurSISJ1Pj26NsM3vY2jxFZn63Nd/TpHmRw=
|
||||
github.com/tonistiigi/fsutil v0.0.0-20230825212630-f09800878302/go.mod h1:9kMVqMyQ/Sx2df5LtnGG+nbrmiZzCS7V6gjW3oGHsvI=
|
||||
github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea h1:SXhTLE6pb6eld/v/cCndK0AMpt1wiVFb/YYmqB3/QG0=
|
||||
github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea/go.mod h1:WPnis/6cRcDZSUvVmezrxJPkiO87ThFYsoUiMwWNDJk=
|
||||
github.com/tonistiigi/vt100 v0.0.0-20230623042737-f9a4f7ef6531 h1:Y/M5lygoNPKwVNLMPXgVfsRT40CSFKXCxuU8LoHySjs=
|
||||
github.com/tonistiigi/vt100 v0.0.0-20230623042737-f9a4f7ef6531/go.mod h1:ulncasL3N9uLrVann0m+CDlJKWsIAP34MPcOJF6VRvc=
|
||||
github.com/vbatts/tar-split v0.11.2 h1:Via6XqJr0hceW4wff3QRzD5gAk/tatMw/4ZA7cTlIME=
|
||||
github.com/vbatts/tar-split v0.11.3 h1:hLFqsOLQ1SsppQNTMpkpPXClLDfC2A3Zgy9OUU+RVck=
|
||||
github.com/vmihailenco/msgpack v3.3.3+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk=
|
||||
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
|
||||
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo=
|
||||
|
|
|
|||
|
|
@ -95,14 +95,18 @@ func MarshalConstraints(base, override *Constraints) (*pb.Op, *pb.OpMetadata) {
|
|||
c.Platform = &defaultPlatform
|
||||
}
|
||||
|
||||
opPlatform := pb.Platform{
|
||||
OS: c.Platform.OS,
|
||||
Architecture: c.Platform.Architecture,
|
||||
Variant: c.Platform.Variant,
|
||||
OSVersion: c.Platform.OSVersion,
|
||||
}
|
||||
if c.Platform.OSFeatures != nil {
|
||||
opPlatform.OSFeatures = append([]string{}, c.Platform.OSFeatures...)
|
||||
}
|
||||
|
||||
return &pb.Op{
|
||||
Platform: &pb.Platform{
|
||||
OS: c.Platform.OS,
|
||||
Architecture: c.Platform.Architecture,
|
||||
Variant: c.Platform.Variant,
|
||||
OSVersion: c.Platform.OSVersion,
|
||||
OSFeatures: c.Platform.OSFeatures,
|
||||
},
|
||||
Platform: &opPlatform,
|
||||
Constraints: &pb.WorkerConstraints{
|
||||
Filter: c.WorkerConstraints,
|
||||
},
|
||||
|
|
|
|||
|
|
@ -258,11 +258,16 @@ func (s State) WithImageConfig(c []byte) (State, error) {
|
|||
}
|
||||
s = s.Dir(img.Config.WorkingDir)
|
||||
if img.Architecture != "" && img.OS != "" {
|
||||
s = s.Platform(ocispecs.Platform{
|
||||
plat := ocispecs.Platform{
|
||||
OS: img.OS,
|
||||
Architecture: img.Architecture,
|
||||
Variant: img.Variant,
|
||||
})
|
||||
OSVersion: img.OSVersion,
|
||||
}
|
||||
if img.OSFeatures != nil {
|
||||
plat.OSFeatures = append([]string{}, img.OSFeatures...)
|
||||
}
|
||||
s = s.Platform(plat)
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -35,7 +35,8 @@ import (
|
|||
|
||||
type SolveOpt struct {
|
||||
Exports []ExportEntry
|
||||
LocalDirs map[string]string
|
||||
LocalDirs map[string]string // Deprecated: use LocalMounts
|
||||
LocalMounts map[string]fsutil.FS
|
||||
OCIStores map[string]content.Store
|
||||
SharedKey string
|
||||
Frontend string
|
||||
|
|
@ -90,7 +91,11 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG
|
|||
return nil, errors.New("invalid with def and cb")
|
||||
}
|
||||
|
||||
syncedDirs, err := prepareSyncedDirs(def, opt.LocalDirs)
|
||||
mounts, err := prepareMounts(&opt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
syncedDirs, err := prepareSyncedFiles(def, mounts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -361,26 +366,23 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG
|
|||
return res, nil
|
||||
}
|
||||
|
||||
func prepareSyncedDirs(def *llb.Definition, localDirs map[string]string) (filesync.StaticDirSource, error) {
|
||||
for _, d := range localDirs {
|
||||
fi, err := os.Stat(d)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "could not find %s", d)
|
||||
}
|
||||
if !fi.IsDir() {
|
||||
return nil, errors.Errorf("%s not a directory", d)
|
||||
}
|
||||
}
|
||||
func prepareSyncedFiles(def *llb.Definition, localMounts map[string]fsutil.FS) (filesync.StaticDirSource, error) {
|
||||
resetUIDAndGID := func(p string, st *fstypes.Stat) fsutil.MapResult {
|
||||
st.Uid = 0
|
||||
st.Gid = 0
|
||||
return fsutil.MapResultKeep
|
||||
}
|
||||
|
||||
dirs := make(filesync.StaticDirSource, len(localDirs))
|
||||
result := make(filesync.StaticDirSource, len(localMounts))
|
||||
if def == nil {
|
||||
for name, d := range localDirs {
|
||||
dirs[name] = filesync.SyncedDir{Dir: d, Map: resetUIDAndGID}
|
||||
for name, mount := range localMounts {
|
||||
mount, err := fsutil.NewFilterFS(mount, &fsutil.FilterOpt{
|
||||
Map: resetUIDAndGID,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result[name] = mount
|
||||
}
|
||||
} else {
|
||||
for _, dt := range def.Def {
|
||||
|
|
@ -391,16 +393,22 @@ func prepareSyncedDirs(def *llb.Definition, localDirs map[string]string) (filesy
|
|||
if src := op.GetSource(); src != nil {
|
||||
if strings.HasPrefix(src.Identifier, "local://") {
|
||||
name := strings.TrimPrefix(src.Identifier, "local://")
|
||||
d, ok := localDirs[name]
|
||||
mount, ok := localMounts[name]
|
||||
if !ok {
|
||||
return nil, errors.Errorf("local directory %s not enabled", name)
|
||||
}
|
||||
dirs[name] = filesync.SyncedDir{Dir: d, Map: resetUIDAndGID}
|
||||
mount, err := fsutil.NewFilterFS(mount, &fsutil.FilterOpt{
|
||||
Map: resetUIDAndGID,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result[name] = mount
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return dirs, nil
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func defaultSessionName() string {
|
||||
|
|
@ -523,3 +531,22 @@ func parseCacheOptions(ctx context.Context, isGateway bool, opt SolveOpt) (*cach
|
|||
}
|
||||
return &res, nil
|
||||
}
|
||||
|
||||
func prepareMounts(opt *SolveOpt) (map[string]fsutil.FS, error) {
|
||||
// merge local mounts and fallback local directories together
|
||||
mounts := make(map[string]fsutil.FS)
|
||||
for k, mount := range opt.LocalMounts {
|
||||
mounts[k] = mount
|
||||
}
|
||||
for k, dir := range opt.LocalDirs {
|
||||
mount, err := fsutil.NewFS(dir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, ok := mounts[k]; ok {
|
||||
return nil, errors.Errorf("local mount %s already exists", k)
|
||||
}
|
||||
mounts[k] = mount
|
||||
}
|
||||
return mounts, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -57,7 +57,7 @@ func (bc *Client) Build(ctx context.Context, fn BuildFunc) (*ResultBuilder, erro
|
|||
p.OSVersion = img.OSVersion
|
||||
}
|
||||
if p.OSFeatures == nil && len(img.OSFeatures) > 0 {
|
||||
p.OSFeatures = img.OSFeatures
|
||||
p.OSFeatures = append([]string{}, img.OSFeatures...)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -47,6 +47,22 @@ type streamWriterCloser struct {
|
|||
}
|
||||
|
||||
func (wc *streamWriterCloser) Write(dt []byte) (int, error) {
|
||||
// grpc-go has a 4MB limit on messages by default. Split large messages
|
||||
// so we don't get close to that limit.
|
||||
const maxChunkSize = 3 * 1024 * 1024
|
||||
if len(dt) > maxChunkSize {
|
||||
n1, err := wc.Write(dt[:maxChunkSize])
|
||||
if err != nil {
|
||||
return n1, err
|
||||
}
|
||||
dt = dt[maxChunkSize:]
|
||||
var n2 int
|
||||
if n2, err = wc.Write(dt); err != nil {
|
||||
return n1 + n2, err
|
||||
}
|
||||
return n1 + n2, nil
|
||||
}
|
||||
|
||||
if err := wc.ClientStream.SendMsg(&BytesMessage{Data: dt}); err != nil {
|
||||
// SendMsg return EOF on remote errors
|
||||
if errors.Is(err, io.EOF) {
|
||||
|
|
|
|||
|
|
@ -35,20 +35,15 @@ type fsSyncProvider struct {
|
|||
doneCh chan error
|
||||
}
|
||||
|
||||
type SyncedDir struct {
|
||||
Dir string
|
||||
Map func(string, *fstypes.Stat) fsutil.MapResult
|
||||
}
|
||||
|
||||
type DirSource interface {
|
||||
LookupDir(string) (SyncedDir, bool)
|
||||
LookupDir(string) (fsutil.FS, bool)
|
||||
}
|
||||
|
||||
type StaticDirSource map[string]SyncedDir
|
||||
type StaticDirSource map[string]fsutil.FS
|
||||
|
||||
var _ DirSource = StaticDirSource{}
|
||||
|
||||
func (dirs StaticDirSource) LookupDir(name string) (SyncedDir, bool) {
|
||||
func (dirs StaticDirSource) LookupDir(name string) (fsutil.FS, bool) {
|
||||
dir, found := dirs[name]
|
||||
return dir, found
|
||||
}
|
||||
|
|
@ -92,15 +87,22 @@ func (sp *fsSyncProvider) handle(method string, stream grpc.ServerStream) (retEr
|
|||
dirName = name[0]
|
||||
}
|
||||
|
||||
excludes := opts[keyExcludePatterns]
|
||||
includes := opts[keyIncludePatterns]
|
||||
followPaths := opts[keyFollowPaths]
|
||||
|
||||
dir, ok := sp.dirs.LookupDir(dirName)
|
||||
if !ok {
|
||||
return InvalidSessionError{status.Errorf(codes.NotFound, "no access allowed to dir %q", dirName)}
|
||||
}
|
||||
|
||||
excludes := opts[keyExcludePatterns]
|
||||
includes := opts[keyIncludePatterns]
|
||||
|
||||
followPaths := opts[keyFollowPaths]
|
||||
dir, err := fsutil.NewFilterFS(dir, &fsutil.FilterOpt{
|
||||
ExcludePatterns: excludes,
|
||||
IncludePatterns: includes,
|
||||
FollowPaths: followPaths,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var progress progressCb
|
||||
if sp.p != nil {
|
||||
|
|
@ -113,12 +115,7 @@ func (sp *fsSyncProvider) handle(method string, stream grpc.ServerStream) (retEr
|
|||
doneCh = sp.doneCh
|
||||
sp.doneCh = nil
|
||||
}
|
||||
err := pr.sendFn(stream, fsutil.NewFS(dir.Dir, &fsutil.WalkOpt{
|
||||
ExcludePatterns: excludes,
|
||||
IncludePatterns: includes,
|
||||
FollowPaths: followPaths,
|
||||
Map: dir.Map,
|
||||
}), progress)
|
||||
err = pr.sendFn(stream, dir, progress)
|
||||
if doneCh != nil {
|
||||
if err != nil {
|
||||
doneCh <- err
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import (
|
|||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/containerd/containerd/defaults"
|
||||
grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware"
|
||||
"github.com/moby/buildkit/util/bklog"
|
||||
"github.com/moby/buildkit/util/grpcerrors"
|
||||
|
|
@ -44,6 +45,8 @@ func grpcClientConn(ctx context.Context, conn net.Conn) (context.Context, *grpc.
|
|||
dialOpts := []grpc.DialOption{
|
||||
dialer,
|
||||
grpc.WithTransportCredentials(insecure.NewCredentials()),
|
||||
grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(defaults.DefaultMaxRecvMsgSize)),
|
||||
grpc.WithDefaultCallOptions(grpc.MaxCallSendMsgSize(defaults.DefaultMaxSendMsgSize)),
|
||||
}
|
||||
|
||||
if span := trace.SpanFromContext(ctx); span.SpanContext().IsValid() {
|
||||
|
|
|
|||
|
|
@ -59,6 +59,20 @@ type writer struct {
|
|||
}
|
||||
|
||||
func (w *writer) Write(dt []byte) (int, error) {
|
||||
// avoid sending too big messages on grpc stream
|
||||
const maxChunkSize = 3 * 1024 * 1024
|
||||
if len(dt) > maxChunkSize {
|
||||
n1, err := w.Write(dt[:maxChunkSize])
|
||||
if err != nil {
|
||||
return n1, err
|
||||
}
|
||||
dt = dt[maxChunkSize:]
|
||||
var n2 int
|
||||
if n2, err := w.Write(dt); err != nil {
|
||||
return n1 + n2, err
|
||||
}
|
||||
return n1 + n2, nil
|
||||
}
|
||||
if err := w.SendMsg(&upload.BytesMessage{Data: dt}); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,96 @@
|
|||
package pb
|
||||
|
||||
import "encoding/json"
|
||||
|
||||
func (m *Op) UnmarshalJSON(data []byte) error {
|
||||
var v struct {
|
||||
Inputs []*Input `json:"inputs,omitempty"`
|
||||
Op struct {
|
||||
*Op_Exec
|
||||
*Op_Source
|
||||
*Op_File
|
||||
*Op_Build
|
||||
*Op_Merge
|
||||
*Op_Diff
|
||||
}
|
||||
Platform *Platform `json:"platform,omitempty"`
|
||||
Constraints *WorkerConstraints `json:"constraints,omitempty"`
|
||||
}
|
||||
|
||||
if err := json.Unmarshal(data, &v); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
m.Inputs = v.Inputs
|
||||
switch {
|
||||
case v.Op.Op_Exec != nil:
|
||||
m.Op = v.Op.Op_Exec
|
||||
case v.Op.Op_Source != nil:
|
||||
m.Op = v.Op.Op_Source
|
||||
case v.Op.Op_File != nil:
|
||||
m.Op = v.Op.Op_File
|
||||
case v.Op.Op_Build != nil:
|
||||
m.Op = v.Op.Op_Build
|
||||
case v.Op.Op_Merge != nil:
|
||||
m.Op = v.Op.Op_Merge
|
||||
case v.Op.Op_Diff != nil:
|
||||
m.Op = v.Op.Op_Diff
|
||||
}
|
||||
m.Platform = v.Platform
|
||||
m.Constraints = v.Constraints
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *FileAction) UnmarshalJSON(data []byte) error {
|
||||
var v struct {
|
||||
Input InputIndex `json:"input"`
|
||||
SecondaryInput InputIndex `json:"secondaryInput"`
|
||||
Output OutputIndex `json:"output"`
|
||||
Action struct {
|
||||
*FileAction_Copy
|
||||
*FileAction_Mkfile
|
||||
*FileAction_Mkdir
|
||||
*FileAction_Rm
|
||||
}
|
||||
}
|
||||
|
||||
if err := json.Unmarshal(data, &v); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
m.Input = v.Input
|
||||
m.SecondaryInput = v.SecondaryInput
|
||||
m.Output = v.Output
|
||||
switch {
|
||||
case v.Action.FileAction_Copy != nil:
|
||||
m.Action = v.Action.FileAction_Copy
|
||||
case v.Action.FileAction_Mkfile != nil:
|
||||
m.Action = v.Action.FileAction_Mkfile
|
||||
case v.Action.FileAction_Mkdir != nil:
|
||||
m.Action = v.Action.FileAction_Mkdir
|
||||
case v.Action.FileAction_Rm != nil:
|
||||
m.Action = v.Action.FileAction_Rm
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *UserOpt) UnmarshalJSON(data []byte) error {
|
||||
var v struct {
|
||||
User struct {
|
||||
*UserOpt_ByName
|
||||
*UserOpt_ByID
|
||||
}
|
||||
}
|
||||
|
||||
if err := json.Unmarshal(data, &v); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
switch {
|
||||
case v.User.UserOpt_ByName != nil:
|
||||
m.User = v.User.UserOpt_ByName
|
||||
case v.User.UserOpt_ByID != nil:
|
||||
m.User = v.User.UserOpt_ByID
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
@ -151,6 +151,7 @@ func (CacheSharingOpt) EnumDescriptor() ([]byte, []int) {
|
|||
|
||||
// Op represents a vertex of the LLB DAG.
|
||||
type Op struct {
|
||||
// changes to this structure must be represented in json.go.
|
||||
// inputs is a set of input edges.
|
||||
Inputs []*Input `protobuf:"bytes,1,rep,name=inputs,proto3" json:"inputs,omitempty"`
|
||||
// Types that are valid to be assigned to Op:
|
||||
|
|
@ -1961,6 +1962,7 @@ func (m *FileOp) GetActions() []*FileAction {
|
|||
}
|
||||
|
||||
type FileAction struct {
|
||||
// changes to this structure must be represented in json.go.
|
||||
Input InputIndex `protobuf:"varint,1,opt,name=input,proto3,customtype=InputIndex" json:"input"`
|
||||
SecondaryInput InputIndex `protobuf:"varint,2,opt,name=secondaryInput,proto3,customtype=InputIndex" json:"secondaryInput"`
|
||||
Output OutputIndex `protobuf:"varint,3,opt,name=output,proto3,customtype=OutputIndex" json:"output"`
|
||||
|
|
@ -2482,6 +2484,8 @@ func (m *ChownOpt) GetGroup() *UserOpt {
|
|||
}
|
||||
|
||||
type UserOpt struct {
|
||||
// changes to this structure must be represented in json.go.
|
||||
//
|
||||
// Types that are valid to be assigned to User:
|
||||
//
|
||||
// *UserOpt_ByName
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ option (gogoproto.stable_marshaler_all) = true;
|
|||
|
||||
// Op represents a vertex of the LLB DAG.
|
||||
message Op {
|
||||
// changes to this structure must be represented in json.go.
|
||||
// inputs is a set of input edges.
|
||||
repeated Input inputs = 1;
|
||||
oneof op {
|
||||
|
|
@ -29,7 +30,7 @@ message Platform {
|
|||
string Architecture = 1;
|
||||
string OS = 2;
|
||||
string Variant = 3;
|
||||
string OSVersion = 4; // unused
|
||||
string OSVersion = 4;
|
||||
repeated string OSFeatures = 5; // unused
|
||||
}
|
||||
|
||||
|
|
@ -288,6 +289,7 @@ message FileOp {
|
|||
}
|
||||
|
||||
message FileAction {
|
||||
// changes to this structure must be represented in json.go.
|
||||
int64 input = 1 [(gogoproto.customtype) = "InputIndex", (gogoproto.nullable) = false]; // could be real input or target (target index + max input index)
|
||||
int64 secondaryInput = 2 [(gogoproto.customtype) = "InputIndex", (gogoproto.nullable) = false]; // --//--
|
||||
int64 output = 3 [(gogoproto.customtype) = "OutputIndex", (gogoproto.nullable) = false];
|
||||
|
|
@ -373,6 +375,7 @@ message ChownOpt {
|
|||
}
|
||||
|
||||
message UserOpt {
|
||||
// changes to this structure must be represented in json.go.
|
||||
oneof user {
|
||||
NamedUserOpt byName = 1;
|
||||
uint32 byID = 2;
|
||||
|
|
|
|||
|
|
@ -5,23 +5,29 @@ import (
|
|||
)
|
||||
|
||||
func (p *Platform) Spec() ocispecs.Platform {
|
||||
return ocispecs.Platform{
|
||||
result := ocispecs.Platform{
|
||||
OS: p.OS,
|
||||
Architecture: p.Architecture,
|
||||
Variant: p.Variant,
|
||||
OSVersion: p.OSVersion,
|
||||
OSFeatures: p.OSFeatures,
|
||||
}
|
||||
if p.OSFeatures != nil {
|
||||
result.OSFeatures = append([]string{}, p.OSFeatures...)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func PlatformFromSpec(p ocispecs.Platform) Platform {
|
||||
return Platform{
|
||||
result := Platform{
|
||||
OS: p.OS,
|
||||
Architecture: p.Architecture,
|
||||
Variant: p.Variant,
|
||||
OSVersion: p.OSVersion,
|
||||
OSFeatures: p.OSFeatures,
|
||||
}
|
||||
if p.OSFeatures != nil {
|
||||
result.OSFeatures = append([]string{}, p.OSFeatures...)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func ToSpecPlatforms(p []Platform) []ocispecs.Platform {
|
||||
|
|
|
|||
|
|
@ -18,6 +18,8 @@ type MultiWriter struct {
|
|||
meta map[string]interface{}
|
||||
}
|
||||
|
||||
var _ rawProgressWriter = &MultiWriter{}
|
||||
|
||||
func NewMultiWriter(opts ...WriterOption) *MultiWriter {
|
||||
mw := &MultiWriter{
|
||||
writers: map[rawProgressWriter]struct{}{},
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
// Code generated by protoc-gen-go. DO NOT EDIT.
|
||||
// versions:
|
||||
// protoc-gen-go v1.30.0
|
||||
// protoc-gen-go v1.31.0
|
||||
// protoc v3.11.4
|
||||
// source: stack.proto
|
||||
|
||||
|
|
|
|||
|
|
@ -1,16 +1,17 @@
|
|||
package system
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
iofs "io/fs"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
func Atime(st iofs.FileInfo) (time.Time, error) {
|
||||
stSys, ok := st.Sys().(*syscall.Win32FileAttributeData)
|
||||
if !ok {
|
||||
return time.Time{}, fmt.Errorf("expected st.Sys() to be *syscall.Win32FileAttributeData, got %T", st.Sys())
|
||||
return time.Time{}, errors.Errorf("expected st.Sys() to be *syscall.Win32FileAttributeData, got %T", st.Sys())
|
||||
}
|
||||
// ref: https://github.com/golang/go/blob/go1.19.2/src/os/types_windows.go#L230
|
||||
return time.Unix(0, stSys.LastAccessTime.Nanoseconds()), nil
|
||||
|
|
|
|||
|
|
@ -2,25 +2,35 @@ package fsutil
|
|||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
gofs "io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/moby/patternmatcher"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/tonistiigi/fsutil/types"
|
||||
)
|
||||
|
||||
type WalkOpt struct {
|
||||
type FilterOpt struct {
|
||||
// IncludePatterns requires that the path matches at least one of the
|
||||
// specified patterns.
|
||||
IncludePatterns []string
|
||||
|
||||
// ExcludePatterns requires that the path does not match any of the
|
||||
// specified patterns.
|
||||
ExcludePatterns []string
|
||||
// FollowPaths contains symlinks that are resolved into include patterns
|
||||
// before performing the fs walk
|
||||
|
||||
// FollowPaths contains symlinks that are resolved into IncludePatterns
|
||||
// at the time of the call to NewFilterFS.
|
||||
FollowPaths []string
|
||||
Map MapFunc
|
||||
|
||||
// Map is called for each path that is included in the result.
|
||||
// The function can modify the stat info for each element, while the result
|
||||
// of the function controls both how Walk continues.
|
||||
Map MapFunc
|
||||
}
|
||||
|
||||
type MapFunc func(string, *types.Stat) MapResult
|
||||
|
|
@ -43,33 +53,41 @@ const (
|
|||
MapResultSkipDir
|
||||
)
|
||||
|
||||
func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) error {
|
||||
root, err := filepath.EvalSymlinks(p)
|
||||
if err != nil {
|
||||
return errors.WithStack(&os.PathError{Op: "resolve", Path: root, Err: err})
|
||||
}
|
||||
rootFI, err := os.Stat(root)
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
if !rootFI.IsDir() {
|
||||
return errors.WithStack(&os.PathError{Op: "walk", Path: root, Err: syscall.ENOTDIR})
|
||||
type filterFS struct {
|
||||
fs FS
|
||||
|
||||
includeMatcher *patternmatcher.PatternMatcher
|
||||
excludeMatcher *patternmatcher.PatternMatcher
|
||||
onlyPrefixIncludes bool
|
||||
onlyPrefixExcludeExceptions bool
|
||||
|
||||
mapFn MapFunc
|
||||
}
|
||||
|
||||
// NewFilterFS creates a new FS that filters the given FS using the given
|
||||
// FilterOpt.
|
||||
|
||||
// The returned FS will not contain any paths that do not match the provided
|
||||
// include and exclude patterns, or that are are exlcluded using the mapping
|
||||
// function.
|
||||
//
|
||||
// The FS is assumed to be a snapshot of the filesystem at the time of the
|
||||
// call to NewFilterFS. If the underlying filesystem changes, calls to the
|
||||
// underlying FS may be inconsistent.
|
||||
func NewFilterFS(fs FS, opt *FilterOpt) (FS, error) {
|
||||
if opt == nil {
|
||||
return fs, nil
|
||||
}
|
||||
|
||||
var (
|
||||
includePatterns []string
|
||||
includeMatcher *patternmatcher.PatternMatcher
|
||||
excludeMatcher *patternmatcher.PatternMatcher
|
||||
)
|
||||
|
||||
if opt != nil && opt.IncludePatterns != nil {
|
||||
var includePatterns []string
|
||||
if opt.IncludePatterns != nil {
|
||||
includePatterns = make([]string, len(opt.IncludePatterns))
|
||||
copy(includePatterns, opt.IncludePatterns)
|
||||
}
|
||||
if opt != nil && opt.FollowPaths != nil {
|
||||
targets, err := FollowLinks(p, opt.FollowPaths)
|
||||
if opt.FollowPaths != nil {
|
||||
targets, err := FollowLinks(fs, opt.FollowPaths)
|
||||
if err != nil {
|
||||
return err
|
||||
return nil, err
|
||||
}
|
||||
if targets != nil {
|
||||
includePatterns = append(includePatterns, targets...)
|
||||
|
|
@ -82,11 +100,18 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err
|
|||
patternChars += `\`
|
||||
}
|
||||
|
||||
onlyPrefixIncludes := true
|
||||
if len(includePatterns) != 0 {
|
||||
var (
|
||||
includeMatcher *patternmatcher.PatternMatcher
|
||||
excludeMatcher *patternmatcher.PatternMatcher
|
||||
err error
|
||||
onlyPrefixIncludes = true
|
||||
onlyPrefixExcludeExceptions = true
|
||||
)
|
||||
|
||||
if len(includePatterns) > 0 {
|
||||
includeMatcher, err = patternmatcher.New(includePatterns)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "invalid includepatterns: %s", opt.IncludePatterns)
|
||||
return nil, errors.Wrapf(err, "invalid includepatterns: %s", opt.IncludePatterns)
|
||||
}
|
||||
|
||||
for _, p := range includeMatcher.Patterns() {
|
||||
|
|
@ -98,11 +123,10 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err
|
|||
|
||||
}
|
||||
|
||||
onlyPrefixExcludeExceptions := true
|
||||
if opt != nil && opt.ExcludePatterns != nil {
|
||||
if len(opt.ExcludePatterns) > 0 {
|
||||
excludeMatcher, err = patternmatcher.New(opt.ExcludePatterns)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "invalid excludepatterns: %s", opt.ExcludePatterns)
|
||||
return nil, errors.Wrapf(err, "invalid excludepatterns: %s", opt.ExcludePatterns)
|
||||
}
|
||||
|
||||
for _, p := range excludeMatcher.Patterns() {
|
||||
|
|
@ -113,10 +137,41 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err
|
|||
}
|
||||
}
|
||||
|
||||
return &filterFS{
|
||||
fs: fs,
|
||||
includeMatcher: includeMatcher,
|
||||
excludeMatcher: excludeMatcher,
|
||||
onlyPrefixIncludes: onlyPrefixIncludes,
|
||||
onlyPrefixExcludeExceptions: onlyPrefixExcludeExceptions,
|
||||
mapFn: opt.Map,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (fs *filterFS) Open(p string) (io.ReadCloser, error) {
|
||||
if fs.includeMatcher != nil {
|
||||
m, err := fs.includeMatcher.MatchesOrParentMatches(p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !m {
|
||||
return nil, errors.Wrapf(os.ErrNotExist, "open %s", p)
|
||||
}
|
||||
}
|
||||
if fs.excludeMatcher != nil {
|
||||
m, err := fs.excludeMatcher.MatchesOrParentMatches(p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if m {
|
||||
return nil, errors.Wrapf(os.ErrNotExist, "open %s", p)
|
||||
}
|
||||
}
|
||||
return fs.fs.Open(p)
|
||||
}
|
||||
|
||||
func (fs *filterFS) Walk(ctx context.Context, target string, fn gofs.WalkDirFunc) error {
|
||||
type visitedDir struct {
|
||||
fi os.FileInfo
|
||||
path string
|
||||
origpath string
|
||||
entry gofs.DirEntry
|
||||
pathWithSep string
|
||||
includeMatchInfo patternmatcher.MatchInfo
|
||||
excludeMatchInfo patternmatcher.MatchInfo
|
||||
|
|
@ -126,34 +181,22 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err
|
|||
// used only for include/exclude handling
|
||||
var parentDirs []visitedDir
|
||||
|
||||
seenFiles := make(map[uint64]string)
|
||||
return filepath.WalkDir(root, func(path string, dirEntry gofs.DirEntry, walkErr error) (retErr error) {
|
||||
return fs.fs.Walk(ctx, target, func(path string, dirEntry gofs.DirEntry, walkErr error) (retErr error) {
|
||||
defer func() {
|
||||
if retErr != nil && isNotExist(retErr) {
|
||||
retErr = filepath.SkipDir
|
||||
}
|
||||
}()
|
||||
|
||||
origpath := path
|
||||
path, err = filepath.Rel(root, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Skip root
|
||||
if path == "." {
|
||||
return nil
|
||||
}
|
||||
|
||||
var (
|
||||
dir visitedDir
|
||||
isDir bool
|
||||
fi gofs.FileInfo
|
||||
)
|
||||
if dirEntry != nil {
|
||||
isDir = dirEntry.IsDir()
|
||||
}
|
||||
|
||||
if includeMatcher != nil || excludeMatcher != nil {
|
||||
if fs.includeMatcher != nil || fs.excludeMatcher != nil {
|
||||
for len(parentDirs) != 0 {
|
||||
lastParentDir := parentDirs[len(parentDirs)-1].pathWithSep
|
||||
if strings.HasPrefix(path, lastParentDir) {
|
||||
|
|
@ -163,15 +206,8 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err
|
|||
}
|
||||
|
||||
if isDir {
|
||||
fi, err = dirEntry.Info()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
dir = visitedDir{
|
||||
fi: fi,
|
||||
path: path,
|
||||
origpath: origpath,
|
||||
entry: dirEntry,
|
||||
pathWithSep: path + string(filepath.Separator),
|
||||
}
|
||||
}
|
||||
|
|
@ -179,12 +215,12 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err
|
|||
|
||||
skip := false
|
||||
|
||||
if includeMatcher != nil {
|
||||
if fs.includeMatcher != nil {
|
||||
var parentIncludeMatchInfo patternmatcher.MatchInfo
|
||||
if len(parentDirs) != 0 {
|
||||
parentIncludeMatchInfo = parentDirs[len(parentDirs)-1].includeMatchInfo
|
||||
}
|
||||
m, matchInfo, err := includeMatcher.MatchesUsingParentResults(path, parentIncludeMatchInfo)
|
||||
m, matchInfo, err := fs.includeMatcher.MatchesUsingParentResults(path, parentIncludeMatchInfo)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to match includepatterns")
|
||||
}
|
||||
|
|
@ -194,11 +230,11 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err
|
|||
}
|
||||
|
||||
if !m {
|
||||
if isDir && onlyPrefixIncludes {
|
||||
if isDir && fs.onlyPrefixIncludes {
|
||||
// Optimization: we can skip walking this dir if no include
|
||||
// patterns could match anything inside it.
|
||||
dirSlash := path + string(filepath.Separator)
|
||||
for _, pat := range includeMatcher.Patterns() {
|
||||
for _, pat := range fs.includeMatcher.Patterns() {
|
||||
if pat.Exclusion() {
|
||||
continue
|
||||
}
|
||||
|
|
@ -214,12 +250,12 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err
|
|||
}
|
||||
}
|
||||
|
||||
if excludeMatcher != nil {
|
||||
if fs.excludeMatcher != nil {
|
||||
var parentExcludeMatchInfo patternmatcher.MatchInfo
|
||||
if len(parentDirs) != 0 {
|
||||
parentExcludeMatchInfo = parentDirs[len(parentDirs)-1].excludeMatchInfo
|
||||
}
|
||||
m, matchInfo, err := excludeMatcher.MatchesUsingParentResults(path, parentExcludeMatchInfo)
|
||||
m, matchInfo, err := fs.excludeMatcher.MatchesUsingParentResults(path, parentExcludeMatchInfo)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to match excludepatterns")
|
||||
}
|
||||
|
|
@ -229,16 +265,16 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err
|
|||
}
|
||||
|
||||
if m {
|
||||
if isDir && onlyPrefixExcludeExceptions {
|
||||
if isDir && fs.onlyPrefixExcludeExceptions {
|
||||
// Optimization: we can skip walking this dir if no
|
||||
// exceptions to exclude patterns could match anything
|
||||
// inside it.
|
||||
if !excludeMatcher.Exclusions() {
|
||||
if !fs.excludeMatcher.Exclusions() {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
|
||||
dirSlash := path + string(filepath.Separator)
|
||||
for _, pat := range excludeMatcher.Patterns() {
|
||||
for _, pat := range fs.excludeMatcher.Patterns() {
|
||||
if !pat.Exclusion() {
|
||||
continue
|
||||
}
|
||||
|
|
@ -261,7 +297,7 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err
|
|||
return walkErr
|
||||
}
|
||||
|
||||
if includeMatcher != nil || excludeMatcher != nil {
|
||||
if fs.includeMatcher != nil || fs.excludeMatcher != nil {
|
||||
defer func() {
|
||||
if isDir {
|
||||
parentDirs = append(parentDirs, dir)
|
||||
|
|
@ -275,25 +311,21 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err
|
|||
|
||||
dir.calledFn = true
|
||||
|
||||
// The FileInfo might have already been read further up.
|
||||
if fi == nil {
|
||||
fi, err = dirEntry.Info()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
stat, err := mkstat(origpath, path, fi, seenFiles)
|
||||
fi, err := dirEntry.Info()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
stat, ok := fi.Sys().(*types.Stat)
|
||||
if !ok {
|
||||
return errors.WithStack(&os.PathError{Path: path, Err: syscall.EBADMSG, Op: "fileinfo without stat info"})
|
||||
}
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
default:
|
||||
if opt != nil && opt.Map != nil {
|
||||
result := opt.Map(stat.Path, stat)
|
||||
if fs.mapFn != nil {
|
||||
result := fs.mapFn(stat.Path, stat)
|
||||
if result == MapResultSkipDir {
|
||||
return filepath.SkipDir
|
||||
} else if result == MapResultExclude {
|
||||
|
|
@ -304,29 +336,33 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err
|
|||
if parentDir.calledFn {
|
||||
continue
|
||||
}
|
||||
parentStat, err := mkstat(parentDir.origpath, parentDir.path, parentDir.fi, seenFiles)
|
||||
parentFi, err := parentDir.entry.Info()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
parentStat, ok := parentFi.Sys().(*types.Stat)
|
||||
if !ok {
|
||||
return errors.WithStack(&os.PathError{Path: path, Err: syscall.EBADMSG, Op: "fileinfo without stat info"})
|
||||
}
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
default:
|
||||
}
|
||||
if opt != nil && opt.Map != nil {
|
||||
result := opt.Map(parentStat.Path, parentStat)
|
||||
if fs.mapFn != nil {
|
||||
result := fs.mapFn(parentStat.Path, parentStat)
|
||||
if result == MapResultSkipDir || result == MapResultExclude {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
if err := fn(parentStat.Path, &StatInfo{parentStat}, nil); err != nil {
|
||||
if err := fn(parentStat.Path, &DirEntryInfo{Stat: parentStat}, nil); err != nil {
|
||||
return err
|
||||
}
|
||||
parentDirs[i].calledFn = true
|
||||
}
|
||||
if err := fn(stat.Path, &StatInfo{stat}, nil); err != nil {
|
||||
if err := fn(stat.Path, &DirEntryInfo{Stat: stat}, nil); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
|
@ -334,6 +370,40 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err
|
|||
})
|
||||
}
|
||||
|
||||
func Walk(ctx context.Context, p string, opt *FilterOpt, fn filepath.WalkFunc) error {
|
||||
f, err := NewFS(p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
f, err = NewFilterFS(f, opt)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return f.Walk(ctx, "/", func(path string, d gofs.DirEntry, err error) error {
|
||||
var info gofs.FileInfo
|
||||
if d != nil {
|
||||
var err2 error
|
||||
info, err2 = d.Info()
|
||||
if err == nil {
|
||||
err = err2
|
||||
}
|
||||
}
|
||||
return fn(path, info, err)
|
||||
})
|
||||
}
|
||||
|
||||
func WalkDir(ctx context.Context, p string, opt *FilterOpt, fn gofs.WalkDirFunc) error {
|
||||
f, err := NewFS(p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
f, err = NewFilterFS(f, opt)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return f.Walk(ctx, "/", fn)
|
||||
}
|
||||
|
||||
func patternWithoutTrailingGlob(p *patternmatcher.Pattern) string {
|
||||
patStr := p.String()
|
||||
// We use filepath.Separator here because patternmatcher.Pattern patterns
|
||||
|
|
@ -344,29 +414,6 @@ func patternWithoutTrailingGlob(p *patternmatcher.Pattern) string {
|
|||
return patStr
|
||||
}
|
||||
|
||||
type StatInfo struct {
|
||||
*types.Stat
|
||||
}
|
||||
|
||||
func (s *StatInfo) Name() string {
|
||||
return filepath.Base(s.Stat.Path)
|
||||
}
|
||||
func (s *StatInfo) Size() int64 {
|
||||
return s.Stat.Size_
|
||||
}
|
||||
func (s *StatInfo) Mode() os.FileMode {
|
||||
return os.FileMode(s.Stat.Mode)
|
||||
}
|
||||
func (s *StatInfo) ModTime() time.Time {
|
||||
return time.Unix(s.Stat.ModTime/1e9, s.Stat.ModTime%1e9)
|
||||
}
|
||||
func (s *StatInfo) IsDir() bool {
|
||||
return s.Mode().IsDir()
|
||||
}
|
||||
func (s *StatInfo) Sys() interface{} {
|
||||
return s.Stat
|
||||
}
|
||||
|
||||
func isNotExist(err error) bool {
|
||||
return errors.Is(err, os.ErrNotExist) || errors.Is(err, syscall.ENOTDIR)
|
||||
}
|
||||
|
|
@ -1,17 +1,21 @@
|
|||
package fsutil
|
||||
|
||||
import (
|
||||
"context"
|
||||
gofs "io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"sort"
|
||||
strings "strings"
|
||||
"syscall"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/tonistiigi/fsutil/types"
|
||||
)
|
||||
|
||||
func FollowLinks(root string, paths []string) ([]string, error) {
|
||||
r := &symlinkResolver{root: root, resolved: map[string]struct{}{}}
|
||||
func FollowLinks(fs FS, paths []string) ([]string, error) {
|
||||
r := &symlinkResolver{fs: fs, resolved: map[string]struct{}{}}
|
||||
for _, p := range paths {
|
||||
if err := r.append(p); err != nil {
|
||||
return nil, err
|
||||
|
|
@ -26,7 +30,7 @@ func FollowLinks(root string, paths []string) ([]string, error) {
|
|||
}
|
||||
|
||||
type symlinkResolver struct {
|
||||
root string
|
||||
fs FS
|
||||
resolved map[string]struct{}
|
||||
}
|
||||
|
||||
|
|
@ -76,10 +80,9 @@ func (r *symlinkResolver) append(p string) error {
|
|||
}
|
||||
|
||||
func (r *symlinkResolver) readSymlink(p string, allowWildcard bool) ([]string, error) {
|
||||
realPath := filepath.Join(r.root, p)
|
||||
base := filepath.Base(p)
|
||||
if allowWildcard && containsWildcards(base) {
|
||||
fis, err := os.ReadDir(filepath.Dir(realPath))
|
||||
fis, err := readDir(r.fs, filepath.Dir(p))
|
||||
if err != nil {
|
||||
if isNotFound(err) {
|
||||
return nil, nil
|
||||
|
|
@ -99,21 +102,30 @@ func (r *symlinkResolver) readSymlink(p string, allowWildcard bool) ([]string, e
|
|||
return out, nil
|
||||
}
|
||||
|
||||
fi, err := os.Lstat(realPath)
|
||||
entry, err := statFile(r.fs, p)
|
||||
if err != nil {
|
||||
if isNotFound(err) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
if fi.Mode()&os.ModeSymlink == 0 {
|
||||
if entry == nil {
|
||||
return nil, nil
|
||||
}
|
||||
link, err := os.Readlink(realPath)
|
||||
if entry.Type()&os.ModeSymlink == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
fi, err := entry.Info()
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
link = filepath.Clean(link)
|
||||
stat, ok := fi.Sys().(*types.Stat)
|
||||
if !ok {
|
||||
return nil, errors.WithStack(&os.PathError{Path: p, Err: syscall.EBADMSG, Op: "fileinfo without stat info"})
|
||||
}
|
||||
|
||||
link := filepath.Clean(stat.Linkname)
|
||||
if filepath.IsAbs(link) {
|
||||
return []string{link}, nil
|
||||
}
|
||||
|
|
@ -122,6 +134,76 @@ func (r *symlinkResolver) readSymlink(p string, allowWildcard bool) ([]string, e
|
|||
}, nil
|
||||
}
|
||||
|
||||
func statFile(fs FS, root string) (os.DirEntry, error) {
|
||||
var out os.DirEntry
|
||||
|
||||
root = filepath.Clean(root)
|
||||
if root == "/" || root == "." {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
err := fs.Walk(context.TODO(), root, func(p string, entry os.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if p != root {
|
||||
return errors.Errorf("expected single entry %q but got %q", root, p)
|
||||
}
|
||||
out = entry
|
||||
if entry.IsDir() {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if out == nil {
|
||||
return nil, errors.Wrapf(os.ErrNotExist, "readFile %s", root)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func readDir(fs FS, root string) ([]os.DirEntry, error) {
|
||||
var out []os.DirEntry
|
||||
|
||||
root = filepath.Clean(root)
|
||||
if root == "/" || root == "." {
|
||||
root = "."
|
||||
out = make([]gofs.DirEntry, 0)
|
||||
}
|
||||
|
||||
err := fs.Walk(context.TODO(), root, func(p string, entry os.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if p == root {
|
||||
if !entry.IsDir() {
|
||||
return errors.WithStack(&os.PathError{Op: "walk", Path: root, Err: syscall.ENOTDIR})
|
||||
}
|
||||
out = make([]gofs.DirEntry, 0)
|
||||
return nil
|
||||
}
|
||||
if out == nil {
|
||||
return errors.Errorf("expected to read parent entry %q before child %q", root, p)
|
||||
}
|
||||
out = append(out, entry)
|
||||
if entry.IsDir() {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if out == nil && root != "." {
|
||||
return nil, errors.Wrapf(os.ErrNotExist, "readDir %s", root)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func containsWildcards(name string) bool {
|
||||
isWindows := runtime.GOOS == "windows"
|
||||
for i := 0; i < len(name); i++ {
|
||||
|
|
|
|||
|
|
@ -3,36 +3,86 @@ package fsutil
|
|||
import (
|
||||
"context"
|
||||
"io"
|
||||
gofs "io/fs"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/tonistiigi/fsutil/types"
|
||||
)
|
||||
|
||||
type FS interface {
|
||||
Walk(context.Context, filepath.WalkFunc) error
|
||||
Walk(context.Context, string, gofs.WalkDirFunc) error
|
||||
Open(string) (io.ReadCloser, error)
|
||||
}
|
||||
|
||||
func NewFS(root string, opt *WalkOpt) FS {
|
||||
// NewFS creates a new FS from a root directory on the host filesystem.
|
||||
func NewFS(root string) (FS, error) {
|
||||
root, err := filepath.EvalSymlinks(root)
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(&os.PathError{Op: "resolve", Path: root, Err: err})
|
||||
}
|
||||
fi, err := os.Stat(root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !fi.IsDir() {
|
||||
return nil, errors.WithStack(&os.PathError{Op: "stat", Path: root, Err: syscall.ENOTDIR})
|
||||
}
|
||||
|
||||
return &fs{
|
||||
root: root,
|
||||
opt: opt,
|
||||
}
|
||||
}, nil
|
||||
}
|
||||
|
||||
type fs struct {
|
||||
root string
|
||||
opt *WalkOpt
|
||||
}
|
||||
|
||||
func (fs *fs) Walk(ctx context.Context, fn filepath.WalkFunc) error {
|
||||
return Walk(ctx, fs.root, fs.opt, fn)
|
||||
func (fs *fs) Walk(ctx context.Context, target string, fn gofs.WalkDirFunc) error {
|
||||
seenFiles := make(map[uint64]string)
|
||||
return filepath.WalkDir(filepath.Join(fs.root, target), func(path string, dirEntry gofs.DirEntry, walkErr error) (retErr error) {
|
||||
defer func() {
|
||||
if retErr != nil && isNotExist(retErr) {
|
||||
retErr = filepath.SkipDir
|
||||
}
|
||||
}()
|
||||
|
||||
origpath := path
|
||||
path, err := filepath.Rel(fs.root, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Skip root
|
||||
if path == "." {
|
||||
return nil
|
||||
}
|
||||
|
||||
var entry gofs.DirEntry
|
||||
if dirEntry != nil {
|
||||
entry = &DirEntryInfo{
|
||||
path: path,
|
||||
origpath: origpath,
|
||||
entry: dirEntry,
|
||||
seenFiles: seenFiles,
|
||||
}
|
||||
}
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
default:
|
||||
if err := fn(path, entry, walkErr); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func (fs *fs) Open(p string) (io.ReadCloser, error) {
|
||||
|
|
@ -67,16 +117,31 @@ type subDirFS struct {
|
|||
dirs []Dir
|
||||
}
|
||||
|
||||
func (fs *subDirFS) Walk(ctx context.Context, fn filepath.WalkFunc) error {
|
||||
func (fs *subDirFS) Walk(ctx context.Context, target string, fn gofs.WalkDirFunc) error {
|
||||
first, rest, _ := strings.Cut(target, string(filepath.Separator))
|
||||
|
||||
for _, d := range fs.dirs {
|
||||
fi := &StatInfo{Stat: &d.Stat}
|
||||
if first != "" && first != d.Stat.Path {
|
||||
continue
|
||||
}
|
||||
|
||||
fi := &StatInfo{&d.Stat}
|
||||
if !fi.IsDir() {
|
||||
return errors.WithStack(&os.PathError{Path: d.Stat.Path, Err: syscall.ENOTDIR, Op: "walk subdir"})
|
||||
}
|
||||
if err := fn(d.Stat.Path, fi, nil); err != nil {
|
||||
dStat := d.Stat
|
||||
if err := fn(d.Stat.Path, &DirEntryInfo{Stat: &dStat}, nil); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := d.FS.Walk(ctx, func(p string, fi os.FileInfo, err error) error {
|
||||
if err := d.FS.Walk(ctx, rest, func(p string, entry gofs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fi, err := entry.Info()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
stat, ok := fi.Sys().(*types.Stat)
|
||||
if !ok {
|
||||
return errors.WithStack(&os.PathError{Path: d.Stat.Path, Err: syscall.EBADMSG, Op: "fileinfo without stat info"})
|
||||
|
|
@ -91,7 +156,7 @@ func (fs *subDirFS) Walk(ctx context.Context, fn filepath.WalkFunc) error {
|
|||
stat.Linkname = path.Join(d.Stat.Path, stat.Linkname)
|
||||
}
|
||||
}
|
||||
return fn(filepath.Join(d.Stat.Path, p), &StatInfo{stat}, nil)
|
||||
return fn(filepath.Join(d.Stat.Path, p), &DirEntryInfo{Stat: stat}, nil)
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -117,3 +182,70 @@ type emptyReader struct {
|
|||
func (*emptyReader) Read([]byte) (int, error) {
|
||||
return 0, io.EOF
|
||||
}
|
||||
|
||||
type StatInfo struct {
|
||||
*types.Stat
|
||||
}
|
||||
|
||||
func (s *StatInfo) Name() string {
|
||||
return filepath.Base(s.Stat.Path)
|
||||
}
|
||||
func (s *StatInfo) Size() int64 {
|
||||
return s.Stat.Size_
|
||||
}
|
||||
func (s *StatInfo) Mode() os.FileMode {
|
||||
return os.FileMode(s.Stat.Mode)
|
||||
}
|
||||
func (s *StatInfo) ModTime() time.Time {
|
||||
return time.Unix(s.Stat.ModTime/1e9, s.Stat.ModTime%1e9)
|
||||
}
|
||||
func (s *StatInfo) IsDir() bool {
|
||||
return s.Mode().IsDir()
|
||||
}
|
||||
func (s *StatInfo) Sys() interface{} {
|
||||
return s.Stat
|
||||
}
|
||||
|
||||
type DirEntryInfo struct {
|
||||
*types.Stat
|
||||
|
||||
entry gofs.DirEntry
|
||||
path string
|
||||
origpath string
|
||||
seenFiles map[uint64]string
|
||||
}
|
||||
|
||||
func (s *DirEntryInfo) Name() string {
|
||||
if s.Stat != nil {
|
||||
return filepath.Base(s.Stat.Path)
|
||||
}
|
||||
return s.entry.Name()
|
||||
}
|
||||
func (s *DirEntryInfo) IsDir() bool {
|
||||
if s.Stat != nil {
|
||||
return s.Stat.IsDir()
|
||||
}
|
||||
return s.entry.IsDir()
|
||||
}
|
||||
func (s *DirEntryInfo) Type() gofs.FileMode {
|
||||
if s.Stat != nil {
|
||||
return gofs.FileMode(s.Stat.Mode)
|
||||
}
|
||||
return s.entry.Type()
|
||||
}
|
||||
func (s *DirEntryInfo) Info() (gofs.FileInfo, error) {
|
||||
if s.Stat == nil {
|
||||
fi, err := s.entry.Info()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stat, err := mkstat(s.origpath, s.path, fi, s.seenFiles)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s.Stat = stat
|
||||
}
|
||||
|
||||
st := *s.Stat
|
||||
return &StatInfo{&st}, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -144,7 +144,11 @@ func (s *sender) sendFile(h *sendHandle) error {
|
|||
|
||||
func (s *sender) walk(ctx context.Context) error {
|
||||
var i uint32 = 0
|
||||
err := s.fs.Walk(ctx, func(path string, fi os.FileInfo, err error) error {
|
||||
err := s.fs.Walk(ctx, "/", func(path string, entry os.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fi, err := entry.Info()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,10 +15,15 @@ import (
|
|||
|
||||
func WriteTar(ctx context.Context, fs FS, w io.Writer) error {
|
||||
tw := tar.NewWriter(w)
|
||||
err := fs.Walk(ctx, func(path string, fi os.FileInfo, err error) error {
|
||||
err := fs.Walk(ctx, "/", func(path string, entry os.DirEntry, err error) error {
|
||||
if err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return err
|
||||
}
|
||||
|
||||
fi, err := entry.Info()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
stat, ok := fi.Sys().(*types.Stat)
|
||||
if !ok {
|
||||
return errors.WithStack(&os.PathError{Path: path, Err: syscall.EBADMSG, Op: "fileinfo without stat info"})
|
||||
|
|
|
|||
|
|
@ -479,7 +479,7 @@ github.com/mitchellh/go-wordwrap
|
|||
# github.com/mitchellh/mapstructure v1.5.0
|
||||
## explicit; go 1.14
|
||||
github.com/mitchellh/mapstructure
|
||||
# github.com/moby/buildkit v0.13.0-beta1.0.20231023114302-d5c1d785b042
|
||||
# github.com/moby/buildkit v0.13.0-beta1.0.20231114164402-5ae9b23c40a9
|
||||
## explicit; go 1.20
|
||||
github.com/moby/buildkit/api/services/control
|
||||
github.com/moby/buildkit/api/types
|
||||
|
|
@ -666,7 +666,7 @@ github.com/theupdateframework/notary/tuf/data
|
|||
github.com/theupdateframework/notary/tuf/signed
|
||||
github.com/theupdateframework/notary/tuf/utils
|
||||
github.com/theupdateframework/notary/tuf/validation
|
||||
# github.com/tonistiigi/fsutil v0.0.0-20230629203738-36ef4d8c0dbb
|
||||
# github.com/tonistiigi/fsutil v0.0.0-20230825212630-f09800878302
|
||||
## explicit; go 1.19
|
||||
github.com/tonistiigi/fsutil
|
||||
github.com/tonistiigi/fsutil/types
|
||||
|
|
|
|||
Loading…
Reference in New Issue