From 6ad5e2fcf3dcd1eb585e7ab170426ef8ccbb25e7 Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Tue, 13 Dec 2022 11:20:04 +0100 Subject: [PATCH] build: set provenance vcs details Signed-off-by: CrazyMax --- build/build.go | 19 +--- build/git.go | 135 ++++++++++++----------- build/git_test.go | 203 ++++++++++++++++++----------------- util/gitutil/gitutil.go | 123 +++++++++++++++++++++ util/gitutil/gitutil_test.go | 69 ++++++++++++ util/gitutil/testutil.go | 76 +++++++++++++ 6 files changed, 451 insertions(+), 174 deletions(-) create mode 100644 util/gitutil/gitutil.go create mode 100644 util/gitutil/gitutil_test.go create mode 100644 util/gitutil/testutil.go diff --git a/build/build.go b/build/build.go index 5bb2767b..a0e2974d 100644 --- a/build/build.go +++ b/build/build.go @@ -595,6 +595,10 @@ func toSolveOpt(ctx context.Context, node builder.Node, multiDriver bool, opt Op so.FrontendAttrs["attest:provenance"] = "mode=min,inline-only=true" } + for k, v := range getGitAttributes(ctx, opt.Inputs.ContextPath, opt.Inputs.DockerfilePath) { + so.FrontendAttrs[k] = v + } + // set platforms if len(opt.Platforms) != 0 { pp := make([]string, len(opt.Platforms)) @@ -846,21 +850,6 @@ func BuildWithResultHandler(ctx context.Context, nodes []builder.Node, opt map[s eg, ctx := errgroup.WithContext(ctx) - for _, opt := range opt { - gitLabels, err := addGitProvenance(ctx, opt.Inputs.ContextPath, opt.Inputs.DockerfilePath) - if err != nil { - return nil, err - } - for n, v := range gitLabels { - if _, ok := opt.Labels[n]; !ok { - if opt.Labels == nil { - opt.Labels = map[string]string{} - } - opt.Labels[n] = v - } - } - } - for k, opt := range opt { multiDriver := len(m[k]) > 1 hasMobyDriver := false diff --git a/build/git.go b/build/git.go index d85bfe85..67aea527 100644 --- a/build/git.go +++ b/build/git.go @@ -3,23 +3,41 @@ package build import ( "context" "os" - "os/exec" "path/filepath" + "strconv" "strings" - ocispecs "github.com/opencontainers/image-spec/specs-go/v1" - "github.com/pkg/errors" + "github.com/docker/buildx/util/gitutil" + specs "github.com/opencontainers/image-spec/specs-go/v1" "github.com/sirupsen/logrus" ) const DockerfileLabel = "com.docker.image.source.entrypoint" -func addGitProvenance(ctx context.Context, contextPath string, dockerfilePath string) (map[string]string, error) { - v := os.Getenv("BUILDX_GIT_LABELS") - if (v != "1" && v != "full") || contextPath == "" { - return nil, nil +func getGitAttributes(ctx context.Context, contextPath string, dockerfilePath string) (res map[string]string) { + res = make(map[string]string) + if contextPath == "" { + return + } + + setGitLabels := false + if v, ok := os.LookupEnv("BUILDX_GIT_LABELS"); ok { + if v == "full" { // backward compatibility with old "full" mode + setGitLabels = true + } else if v, _ := strconv.ParseBool(v); v { + setGitLabels = v + } + } + setGitInfo := true + if v, ok := os.LookupEnv("BUILDX_GIT_INFO"); ok { + if v, _ := strconv.ParseBool(v); v { + setGitInfo = v + } + } + + if !setGitLabels && !setGitInfo { + return } - labels := make(map[string]string, 0) // figure out in which directory the git command needs to run in var wd string @@ -30,69 +48,62 @@ func addGitProvenance(ctx context.Context, contextPath string, dockerfilePath st wd, _ = filepath.Abs(filepath.Join(cwd, contextPath)) } - // check if inside git working tree - cmd := exec.CommandContext(ctx, "git", "rev-parse", "--is-inside-work-tree") - cmd.Dir = wd - err := cmd.Run() - if err != nil { + gitc := gitutil.New(gitutil.WithContext(ctx), gitutil.WithWorkingDir(wd)) + if !gitc.IsInsideWorkTree() { logrus.Warnf("Unable to determine Git information") - return nil, nil + return } - // obtain Git sha of current HEAD - cmd = exec.CommandContext(ctx, "git", "rev-parse", "HEAD") - cmd.Dir = wd - out, err := cmd.Output() - if err != nil { - return nil, errors.Wrap(err, "error obtaining git head") - } - sha := strings.TrimSpace(string(out)) + var resRevision, resSource, resDockerfilePath string - // check if the current HEAD is clean - cmd = exec.CommandContext(ctx, "git", "status", "--porcelain", "--ignored") - cmd.Dir = wd - out, err = cmd.Output() - if err != nil { - return nil, errors.Wrap(err, "error obtaining git status") - } - if len(strings.TrimSpace(string(out))) != 0 { - sha += "-dirty" - } - labels[ocispecs.AnnotationRevision] = sha - - // add a remote url if full Git details are requested; if there aren't any remotes don't fail - if v == "full" { - cmd = exec.CommandContext(ctx, "git", "ls-remote", "--get-url") - cmd.Dir = wd - out, _ := cmd.Output() - if len(out) > 0 { - labels[ocispecs.AnnotationSource] = strings.TrimSpace(string(out)) + if sha, err := gitc.FullCommit(); err == nil && sha != "" { + resRevision = sha + if gitc.IsDirty() { + resRevision += "-dirty" } } - // add Dockerfile path; there is no org.opencontainers annotation for this - if dockerfilePath == "" { - dockerfilePath = filepath.Join(wd, "Dockerfile") + if rurl, err := gitc.RemoteURL(); err == nil && rurl != "" { + resSource = rurl } - // obtain Git root directory - cmd = exec.CommandContext(ctx, "git", "rev-parse", "--show-toplevel") - cmd.Dir = wd - out, err = cmd.Output() - if err != nil { - return nil, errors.Wrap(err, "failed to get git root") - } - root := strings.TrimSpace(string(out)) - - // record only Dockerfile paths that are within the Git root - if !filepath.IsAbs(dockerfilePath) { - cwd, _ := os.Getwd() - dockerfilePath = filepath.Join(cwd, dockerfilePath) - } - dockerfilePath, _ = filepath.Rel(root, dockerfilePath) - if !strings.HasPrefix(dockerfilePath, "..") { - labels[DockerfileLabel] = dockerfilePath + if setGitLabels { + if root, err := gitc.RootDir(); err == nil && root != "" { + if dockerfilePath == "" { + dockerfilePath = filepath.Join(wd, "Dockerfile") + } + if !filepath.IsAbs(dockerfilePath) { + cwd, _ := os.Getwd() + dockerfilePath = filepath.Join(cwd, dockerfilePath) + } + dockerfilePath, _ = filepath.Rel(root, dockerfilePath) + if !strings.HasPrefix(dockerfilePath, "..") { + resDockerfilePath = dockerfilePath + } + } } - return labels, nil + if resSource != "" { + if setGitLabels { + res["label:"+specs.AnnotationSource] = resSource + } + if setGitInfo { + res["vcs:source"] = resSource + } + } + if resRevision != "" { + if setGitLabels { + res["label:"+specs.AnnotationRevision] = resRevision + } + if setGitInfo { + res["vcs:revision"] = resRevision + } + } + if resDockerfilePath != "" { + if setGitLabels { + res["label:"+DockerfileLabel] = resDockerfilePath + } + } + + return } diff --git a/build/git_test.go b/build/git_test.go index 6c05d8af..5dc630f4 100644 --- a/build/git_test.go +++ b/build/git_test.go @@ -2,122 +2,131 @@ package build import ( "context" - "fmt" "os" - "os/exec" "path/filepath" - "runtime" "strings" "testing" - ocispecs "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/docker/buildx/util/gitutil" + specs "github.com/opencontainers/image-spec/specs-go/v1" "github.com/stretchr/testify/assert" ) -var repoDir string - -func setupTest(tb testing.TB) func(tb testing.TB) { - repoDir = tb.TempDir() - // required for local testing on mac to avoid strange /private symlinks - if runtime.GOOS == "darwin" { - repoDir, _ = filepath.EvalSymlinks(repoDir) - } - cmd := exec.Command("git", "init") - cmd.Dir = repoDir - err := cmd.Run() - assert.Nilf(tb, err, "failed to init git repo: %v", err) - +func setupTest(tb testing.TB) { + gitutil.Mktmp(tb) + gitutil.GitInit(tb) df := []byte("FROM alpine:latest\n") - err = os.WriteFile(filepath.Join(repoDir, "Dockerfile"), df, 0644) - assert.Nilf(tb, err, "failed to write file: %v", err) + assert.NoError(tb, os.WriteFile("Dockerfile", df, 0644)) + gitutil.GitAdd(tb, "Dockerfile") + gitutil.GitCommit(tb, "initial commit") +} - cmd = exec.Command("git", "add", "Dockerfile") - cmd.Dir = repoDir - err = cmd.Run() - assert.Nilf(tb, err, "failed to add file: %v", err) +func TestGetGitAttributesNoContext(t *testing.T) { + setupTest(t) - cmd = exec.Command("git", "config", "user.name", "buildx") - cmd.Dir = repoDir - err = cmd.Run() - assert.Nilf(tb, err, "failed to set git user.name: %v", err) + gitattrs := getGitAttributes(context.Background(), "", "Dockerfile") + assert.Empty(t, gitattrs) +} - cmd = exec.Command("git", "config", "user.email", "buildx@docker.com") - cmd.Dir = repoDir - err = cmd.Run() - assert.Nilf(tb, err, "failed to set git user.email: %v", err) - - cmd = exec.Command("git", "commit", "-m", "Initial commit") - cmd.Dir = repoDir - err = cmd.Run() - assert.Nilf(tb, err, "failed to commit: %v", err) - - return func(tb testing.TB) { - os.Unsetenv("BUILDX_GIT_LABELS") - os.RemoveAll(repoDir) +func TestGetGitAttributes(t *testing.T) { + cases := []struct { + name string + envGitLabels string + envGitInfo string + expected []string + }{ + { + name: "default", + envGitLabels: "", + envGitInfo: "", + expected: []string{ + "vcs:revision", + }, + }, + { + name: "gitinfo", + envGitLabels: "false", + envGitInfo: "true", + expected: []string{ + "vcs:revision", + }, + }, + { + name: "gitlabels", + envGitLabels: "true", + envGitInfo: "false", + expected: []string{ + "label:" + DockerfileLabel, + "label:" + specs.AnnotationRevision, + }, + }, + { + name: "both", + envGitLabels: "true", + envGitInfo: "", + expected: []string{ + "label:" + DockerfileLabel, + "label:" + specs.AnnotationRevision, + "vcs:revision", + }, + }, + } + for _, tt := range cases { + tt := tt + t.Run(tt.name, func(t *testing.T) { + setupTest(t) + if tt.envGitLabels != "" { + t.Setenv("BUILDX_GIT_LABELS", tt.envGitLabels) + } + if tt.envGitInfo != "" { + t.Setenv("BUILDX_GIT_INFO", tt.envGitInfo) + } + gitattrs := getGitAttributes(context.Background(), ".", "Dockerfile") + for _, e := range tt.expected { + assert.Contains(t, gitattrs, e) + assert.NotEmpty(t, gitattrs[e]) + if e == "label:"+DockerfileLabel { + assert.Equal(t, "Dockerfile", gitattrs[e]) + } + } + }) } } -func TestAddGitProvenanceDataWithoutEnv(t *testing.T) { - defer setupTest(t)(t) - labels, err := addGitProvenance(context.Background(), repoDir, filepath.Join(repoDir, "Dockerfile")) - assert.Nilf(t, err, "No error expected") - assert.Nilf(t, labels, "No labels expected") +func TestGetGitAttributesWithRemote(t *testing.T) { + setupTest(t) + gitutil.GitSetRemote(t, "git@github.com:docker/buildx.git") + + t.Setenv("BUILDX_GIT_LABELS", "true") + gitattrs := getGitAttributes(context.Background(), ".", "Dockerfile") + assert.Equal(t, 5, len(gitattrs)) + assert.Contains(t, gitattrs, "label:"+DockerfileLabel) + assert.Equal(t, "Dockerfile", gitattrs["label:"+DockerfileLabel]) + assert.Contains(t, gitattrs, "label:"+specs.AnnotationRevision) + assert.NotEmpty(t, gitattrs["label:"+specs.AnnotationRevision]) + assert.Contains(t, gitattrs, "label:"+specs.AnnotationSource) + assert.Equal(t, "git@github.com:docker/buildx.git", gitattrs["label:"+specs.AnnotationSource]) + assert.Contains(t, gitattrs, "vcs:revision") + assert.NotEmpty(t, gitattrs["vcs:revision"]) + assert.Contains(t, gitattrs, "vcs:source") + assert.Equal(t, "git@github.com:docker/buildx.git", gitattrs["vcs:source"]) } -func TestAddGitProvenanceDataWitEmptyEnv(t *testing.T) { - defer setupTest(t)(t) - os.Setenv("BUILDX_GIT_LABELS", "") - labels, err := addGitProvenance(context.Background(), repoDir, filepath.Join(repoDir, "Dockerfile")) - assert.Nilf(t, err, "No error expected") - assert.Nilf(t, labels, "No labels expected") -} +func TestGetGitAttributesDirty(t *testing.T) { + setupTest(t) -func TestAddGitProvenanceDataWithoutLabels(t *testing.T) { - defer setupTest(t)(t) - os.Setenv("BUILDX_GIT_LABELS", "full") - labels, err := addGitProvenance(context.Background(), repoDir, filepath.Join(repoDir, "Dockerfile")) - assert.Nilf(t, err, "No error expected") - assert.Equal(t, 2, len(labels), "Exactly 2 git provenance labels expected") - assert.Equal(t, "Dockerfile", labels[DockerfileLabel], "Expected a dockerfile path provenance label") - - cmd := exec.Command("git", "rev-parse", "HEAD") - cmd.Dir = repoDir - out, _ := cmd.Output() - assert.Equal(t, strings.TrimSpace(string(out)), labels[ocispecs.AnnotationRevision], "Expected a sha provenance label") -} - -func TestAddGitProvenanceDataWithLabels(t *testing.T) { - defer setupTest(t)(t) // make a change to test dirty flag df := []byte("FROM alpine:edge\n") - os.Mkdir(filepath.Join(repoDir, "dir"), 0755) - os.WriteFile(filepath.Join(repoDir, "dir", "Dockerfile"), df, 0644) - // add a remote - cmd := exec.Command("git", "remote", "add", "origin", "git@github.com:docker/buildx.git") - cmd.Dir = repoDir - cmd.Run() + assert.NoError(t, os.Mkdir("dir", 0755)) + assert.NoError(t, os.WriteFile(filepath.Join("dir", "Dockerfile"), df, 0644)) - os.Setenv("BUILDX_GIT_LABELS", "full") - labels, err := addGitProvenance(context.Background(), repoDir, filepath.Join(repoDir, "Dockerfile")) - assert.Nilf(t, err, "No error expected") - assert.Equal(t, 3, len(labels), "Exactly 3 git provenance labels expected") - assert.Equal(t, "Dockerfile", labels[DockerfileLabel], "Expected a dockerfile path provenance label") - assert.Equal(t, "git@github.com:docker/buildx.git", labels[ocispecs.AnnotationSource], "Expected a remote provenance label") - - cmd = exec.Command("git", "rev-parse", "HEAD") - cmd.Dir = repoDir - out, _ := cmd.Output() - assert.Equal(t, fmt.Sprintf("%s-dirty", strings.TrimSpace(string(out))), labels[ocispecs.AnnotationRevision], "Expected a sha provenance label") -} - -func TestAddGitProvenanceDataOutsideOfGitRepository(t *testing.T) { - defer setupTest(t)(t) - os.Setenv("BUILDX_GIT_LABELS", "full") - parentDir := filepath.Dir(repoDir) - cwd, _ := os.Getwd() - os.Chdir(parentDir) - labels, err := addGitProvenance(context.Background(), filepath.Base(repoDir), "") - assert.Nilf(t, err, "No error expected") - assert.Equal(t, "Dockerfile", labels[DockerfileLabel], "Expected a dockerfile path provenance label") - os.Chdir(cwd) + t.Setenv("BUILDX_GIT_LABELS", "true") + gitattrs := getGitAttributes(context.Background(), ".", "Dockerfile") + assert.Equal(t, 3, len(gitattrs)) + assert.Contains(t, gitattrs, "label:"+DockerfileLabel) + assert.Equal(t, "Dockerfile", gitattrs["label:"+DockerfileLabel]) + assert.Contains(t, gitattrs, "label:"+specs.AnnotationRevision) + assert.True(t, strings.HasSuffix(gitattrs["label:"+specs.AnnotationRevision], "-dirty")) + assert.Contains(t, gitattrs, "vcs:revision") + assert.True(t, strings.HasSuffix(gitattrs["vcs:revision"], "-dirty")) } diff --git a/util/gitutil/gitutil.go b/util/gitutil/gitutil.go new file mode 100644 index 00000000..953d282f --- /dev/null +++ b/util/gitutil/gitutil.go @@ -0,0 +1,123 @@ +package gitutil + +import ( + "bytes" + "context" + "os/exec" + "strings" + + "github.com/pkg/errors" +) + +// Git represents an active git object +type Git struct { + ctx context.Context + wd string +} + +// Option provides a variadic option for configuring the git client. +type Option func(b *Git) + +// WithContext sets context. +func WithContext(ctx context.Context) Option { + return func(b *Git) { + b.ctx = ctx + } +} + +// WithWorkingDir sets working directory. +func WithWorkingDir(wd string) Option { + return func(b *Git) { + b.wd = wd + } +} + +// New initializes a new git client +func New(opts ...Option) *Git { + c := &Git{ + ctx: context.Background(), + } + for _, opt := range opts { + opt(c) + } + return c +} + +func (c *Git) IsInsideWorkTree() bool { + out, err := c.clean(c.run("rev-parse", "--is-inside-work-tree")) + return out == "true" && err == nil +} + +func (c *Git) IsDirty() bool { + out, err := c.run("status", "--porcelain", "--ignored") + return strings.TrimSpace(out) != "" || err != nil +} + +func (c *Git) RootDir() (string, error) { + return c.clean(c.run("rev-parse", "--show-toplevel")) +} + +func (c *Git) RemoteURL() (string, error) { + return c.clean(c.run("ls-remote", "--get-url")) +} + +func (c *Git) FullCommit() (string, error) { + return c.clean(c.run("show", "--format=%H", "HEAD", "--quiet")) +} + +func (c *Git) ShortCommit() (string, error) { + return c.clean(c.run("show", "--format=%h", "HEAD", "--quiet")) +} + +func (c *Git) Tag() (string, error) { + var tag string + var err error + for _, fn := range []func() (string, error){ + func() (string, error) { + return c.clean(c.run("tag", "--points-at", "HEAD", "--sort", "-version:creatordate")) + }, + func() (string, error) { + return c.clean(c.run("describe", "--tags", "--abbrev=0")) + }, + } { + tag, err = fn() + if tag != "" || err != nil { + return tag, err + } + } + return tag, err +} + +func (c *Git) run(args ...string) (string, error) { + if _, err := exec.LookPath("git"); err != nil { + return "", errors.New("git not present in PATH") + } + + var extraArgs = []string{ + "-c", "log.showSignature=false", + } + + args = append(extraArgs, args...) + cmd := exec.Command("git", args...) + if c.wd != "" { + cmd.Dir = c.wd + } + + stdout := bytes.Buffer{} + stderr := bytes.Buffer{} + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + if err := cmd.Run(); err != nil { + return "", errors.New(stderr.String()) + } + return stdout.String(), nil +} + +func (c *Git) clean(out string, err error) (string, error) { + out = strings.ReplaceAll(strings.Split(out, "\n")[0], "'", "") + if err != nil { + err = errors.New(strings.TrimSuffix(err.Error(), "\n")) + } + return out, err +} diff --git a/util/gitutil/gitutil_test.go b/util/gitutil/gitutil_test.go new file mode 100644 index 00000000..b1113a8a --- /dev/null +++ b/util/gitutil/gitutil_test.go @@ -0,0 +1,69 @@ +package gitutil + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestGit(t *testing.T) { + c := New() + out, err := c.run("status") + require.NoError(t, err) + require.NotEmpty(t, out) + + out, err = c.clean(c.run("not-exist")) + require.Error(t, err) + require.Empty(t, out) + require.Equal(t, "git: 'not-exist' is not a git command. See 'git --help'.", err.Error()) +} + +func TestGitFullCommit(t *testing.T) { + Mktmp(t) + GitInit(t) + GitCommit(t, "bar") + + c := New() + out, err := c.FullCommit() + require.NoError(t, err) + require.Equal(t, 40, len(out)) +} + +func TestGitShortCommit(t *testing.T) { + Mktmp(t) + GitInit(t) + GitCommit(t, "bar") + + c := New() + out, err := c.ShortCommit() + require.NoError(t, err) + require.Equal(t, 7, len(out)) +} + +func TestGitTagsPointsAt(t *testing.T) { + Mktmp(t) + GitInit(t) + GitCommit(t, "bar") + GitTag(t, "v0.8.0") + GitCommit(t, "foo") + GitTag(t, "v0.9.0") + + c := New() + out, err := c.clean(c.run("tag", "--points-at", "HEAD", "--sort", "-version:creatordate")) + require.NoError(t, err) + require.Equal(t, "v0.9.0", out) +} + +func TestGitDescribeTags(t *testing.T) { + Mktmp(t) + GitInit(t) + GitCommit(t, "bar") + GitTag(t, "v0.8.0") + GitCommit(t, "foo") + GitTag(t, "v0.9.0") + + c := New() + out, err := c.clean(c.run("describe", "--tags", "--abbrev=0")) + require.NoError(t, err) + require.Equal(t, "v0.9.0", out) +} diff --git a/util/gitutil/testutil.go b/util/gitutil/testutil.go new file mode 100644 index 00000000..7560e5b0 --- /dev/null +++ b/util/gitutil/testutil.go @@ -0,0 +1,76 @@ +package gitutil + +import ( + "os" + "testing" + + "github.com/stretchr/testify/require" +) + +func GitInit(tb testing.TB) { + tb.Helper() + out, err := fakeGit("init") + require.NoError(tb, err) + require.Contains(tb, out, "Initialized empty Git repository") + require.NoError(tb, err) + GitCheckoutBranch(tb, "main") + _, _ = fakeGit("branch", "-D", "master") +} + +func GitCommit(tb testing.TB, msg string) { + tb.Helper() + out, err := fakeGit("commit", "--allow-empty", "-m", msg) + require.NoError(tb, err) + require.Contains(tb, out, "main", msg) +} + +func GitTag(tb testing.TB, tag string) { + tb.Helper() + out, err := fakeGit("tag", tag) + require.NoError(tb, err) + require.Empty(tb, out) +} + +func GitCheckoutBranch(tb testing.TB, name string) { + tb.Helper() + out, err := fakeGit("checkout", "-b", name) + require.NoError(tb, err) + require.Empty(tb, out) +} + +func GitAdd(tb testing.TB, file string) { + tb.Helper() + _, err := fakeGit("add", file) + require.NoError(tb, err) +} + +func GitSetRemote(tb testing.TB, url string) { + tb.Helper() + _, err := fakeGit("remote", "add", "origin", url) + require.NoError(tb, err) +} + +func Mktmp(tb testing.TB) string { + tb.Helper() + folder := tb.TempDir() + current, err := os.Getwd() + require.NoError(tb, err) + require.NoError(tb, os.Chdir(folder)) + tb.Cleanup(func() { + require.NoError(tb, os.Chdir(current)) + }) + return folder +} + +func fakeGit(args ...string) (string, error) { + allArgs := []string{ + "-c", "user.name=buildx", + "-c", "user.email=buildx@docker.com", + "-c", "commit.gpgSign=false", + "-c", "tag.gpgSign=false", + "-c", "log.showSignature=false", + } + allArgs = append(allArgs, args...) + c := New() + return c.clean(c.run(allArgs...)) +}