diff --git a/build/build.go b/build/build.go
index 86095371..92e4f318 100644
--- a/build/build.go
+++ b/build/build.go
@@ -25,6 +25,7 @@ import (
 	"github.com/containerd/containerd/platforms"
 	"github.com/docker/buildx/builder"
 	"github.com/docker/buildx/driver"
+	"github.com/docker/buildx/localstate"
 	"github.com/docker/buildx/util/dockerutil"
 	"github.com/docker/buildx/util/imagetools"
 	"github.com/docker/buildx/util/progress"
@@ -650,6 +651,12 @@ func toSolveOpt(ctx context.Context, node builder.Node, multiDriver bool, opt Op
 		so.FrontendAttrs["ulimit"] = ulimits
 	}
 
+	// remember local state like directory path that is not sent to buildkit
+	so.Ref = identity.NewID()
+	if err := saveLocalState(so, opt, node, configDir); err != nil {
+		return nil, nil, err
+	}
+
 	return &so, releaseF, nil
 }
 
@@ -1612,3 +1619,40 @@ func noPrintFunc(opt map[string]Options) bool {
 	}
 	return true
 }
+
+func saveLocalState(so client.SolveOpt, opt Options, node builder.Node, configDir string) error {
+	var err error
+
+	if so.Ref == "" {
+		return nil
+	}
+
+	lp := opt.Inputs.ContextPath
+	dp := opt.Inputs.DockerfilePath
+	if lp != "" || dp != "" {
+		if lp != "" {
+			lp, err = filepath.Abs(lp)
+			if err != nil {
+				return err
+			}
+		}
+		if dp != "" {
+			dp, err = filepath.Abs(dp)
+			if err != nil {
+				return err
+			}
+		}
+		ls, err := localstate.New(configDir)
+		if err != nil {
+			return err
+		}
+		if err := ls.SaveRef(node.Builder, node.Name, so.Ref, localstate.State{
+			LocalPath:      lp,
+			DockerfilePath: dp,
+		}); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
diff --git a/builder/node.go b/builder/node.go
index 459dd282..68acb214 100644
--- a/builder/node.go
+++ b/builder/node.go
@@ -20,6 +20,7 @@ import (
 
 type Node struct {
 	store.Node
+	Builder     string
 	Driver      driver.Driver
 	DriverInfo  *driver.Info
 	Platforms   []ocispecs.Platform
@@ -63,6 +64,7 @@ func (b *Builder) LoadNodes(ctx context.Context, withData bool) (_ []Node, err e
 					Node:        n,
 					ProxyConfig: storeutil.GetProxyConfig(b.opts.dockerCli),
 					Platforms:   n.Platforms,
+					Builder:     b.Name,
 				}
 				defer func() {
 					b.nodes[i] = node
diff --git a/commands/create.go b/commands/create.go
index 1870e39d..e1def29a 100644
--- a/commands/create.go
+++ b/commands/create.go
@@ -14,6 +14,7 @@ import (
 	"github.com/docker/buildx/driver"
 	k8sutil "github.com/docker/buildx/driver/kubernetes/util"
 	remoteutil "github.com/docker/buildx/driver/remote/util"
+	"github.com/docker/buildx/localstate"
 	"github.com/docker/buildx/store"
 	"github.com/docker/buildx/store/storeutil"
 	"github.com/docker/buildx/util/cobrautil"
@@ -170,6 +171,13 @@ func runCreate(dockerCli command.Cli, in createOptions, args []string) error {
 		if err := ng.Leave(in.nodeName); err != nil {
 			return err
 		}
+		ls, err := localstate.New(confutil.ConfigDir(dockerCli))
+		if err != nil {
+			return err
+		}
+		if err := ls.RemoveBuilderNode(ng.Name, in.nodeName); err != nil {
+			return err
+		}
 	} else {
 		switch {
 		case driverName == "kubernetes":
diff --git a/localstate/localstate.go b/localstate/localstate.go
new file mode 100644
index 00000000..49b7a0b6
--- /dev/null
+++ b/localstate/localstate.go
@@ -0,0 +1,93 @@
+package localstate
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+
+	"github.com/docker/docker/pkg/ioutils"
+	"github.com/pkg/errors"
+)
+
+const refsDir = "refs"
+
+type State struct {
+	LocalPath      string
+	DockerfilePath string
+}
+
+type LocalState struct {
+	root string
+}
+
+func New(root string) (*LocalState, error) {
+	if root == "" {
+		return nil, errors.Errorf("root dir empty")
+	}
+	if err := os.MkdirAll(filepath.Join(root, refsDir), 0700); err != nil {
+		return nil, err
+	}
+	return &LocalState{
+		root: root,
+	}, nil
+}
+
+func (ls *LocalState) ReadRef(builderName, nodeName, id string) (*State, error) {
+	if err := ls.validate(builderName, nodeName, id); err != nil {
+		return nil, err
+	}
+	dt, err := os.ReadFile(filepath.Join(ls.root, refsDir, builderName, nodeName, id))
+	if err != nil {
+		return nil, err
+	}
+	var st State
+	if err := json.Unmarshal(dt, &st); err != nil {
+		return nil, err
+	}
+	return &st, nil
+}
+
+func (ls *LocalState) SaveRef(builderName, nodeName, id string, st State) error {
+	if err := ls.validate(builderName, nodeName, id); err != nil {
+		return err
+	}
+	refDir := filepath.Join(ls.root, refsDir, builderName, nodeName)
+	if err := os.MkdirAll(refDir, 0700); err != nil {
+		return err
+	}
+	dt, err := json.Marshal(st)
+	if err != nil {
+		return err
+	}
+	return ioutils.AtomicWriteFile(filepath.Join(refDir, id), dt, 0600)
+}
+
+func (ls *LocalState) RemoveBuilder(builderName string) error {
+	if builderName == "" {
+		return errors.Errorf("builder name empty")
+	}
+	return os.RemoveAll(filepath.Join(ls.root, refsDir, builderName))
+}
+
+func (ls *LocalState) RemoveBuilderNode(builderName string, nodeName string) error {
+	if builderName == "" {
+		return errors.Errorf("builder name empty")
+	}
+	if nodeName == "" {
+		return errors.Errorf("node name empty")
+	}
+	return os.RemoveAll(filepath.Join(ls.root, refsDir, builderName, nodeName))
+}
+
+func (ls *LocalState) validate(builderName, nodeName, id string) error {
+	if builderName == "" {
+		return errors.Errorf("builder name empty")
+	}
+	if nodeName == "" {
+		return errors.Errorf("node name empty")
+	}
+	if id == "" {
+		return errors.Errorf("ref ID empty")
+	}
+	return nil
+}
diff --git a/store/store.go b/store/store.go
index 72d1811e..2d4b0b12 100644
--- a/store/store.go
+++ b/store/store.go
@@ -7,6 +7,7 @@ import (
 	"sort"
 	"time"
 
+	"github.com/docker/buildx/localstate"
 	"github.com/docker/docker/pkg/ioutils"
 	"github.com/gofrs/flock"
 	"github.com/opencontainers/go-digest"
@@ -120,6 +121,13 @@ func (t *Txn) Remove(name string) error {
 	if err := t.RemoveLastActivity(name); err != nil {
 		return err
 	}
+	ls, err := localstate.New(t.s.root)
+	if err != nil {
+		return err
+	}
+	if err := ls.RemoveBuilder(name); err != nil {
+		return err
+	}
 	return os.RemoveAll(filepath.Join(t.s.root, instanceDir, name))
 }