diff --git a/build/git_test.go b/build/git_test.go
index 9f2f7324..18d995b0 100644
--- a/build/git_test.go
+++ b/build/git_test.go
@@ -26,7 +26,7 @@ func setupTest(tb testing.TB) {
 
 	gitutil.GitAdd(c, tb, "Dockerfile")
 	gitutil.GitCommit(c, tb, "initial commit")
-	gitutil.GitSetRemote(c, tb, "git@github.com:docker/buildx.git")
+	gitutil.GitSetRemote(c, tb, "origin", "git@github.com:docker/buildx.git")
 }
 
 func TestGetGitAttributesNotGitRepo(t *testing.T) {
diff --git a/util/gitutil/gitutil.go b/util/gitutil/gitutil.go
index abf591d4..3aa33013 100644
--- a/util/gitutil/gitutil.go
+++ b/util/gitutil/gitutil.go
@@ -67,7 +67,15 @@ func (c *Git) RootDir() (string, error) {
 }
 
 func (c *Git) RemoteURL() (string, error) {
-	return c.clean(c.run("ls-remote", "--get-url"))
+	// Try to get the remote URL from the origin remote first
+	if ru, err := c.clean(c.run("remote", "get-url", "origin")); err == nil && ru != "" {
+		return ru, nil
+	}
+	// If that fails, try to get the remote URL from the upstream remote
+	if ru, err := c.clean(c.run("remote", "get-url", "upstream")); err == nil && ru != "" {
+		return ru, nil
+	}
+	return "", errors.New("no remote URL found for either origin or upstream")
 }
 
 func (c *Git) FullCommit() (string, error) {
diff --git a/util/gitutil/gitutil_test.go b/util/gitutil/gitutil_test.go
index 9948089c..28992646 100644
--- a/util/gitutil/gitutil_test.go
+++ b/util/gitutil/gitutil_test.go
@@ -77,3 +77,89 @@ func TestGitDescribeTags(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, "v0.9.0", out)
 }
+
+func TestGitRemoteURL(t *testing.T) {
+	type remote struct {
+		name string
+		url  string
+	}
+
+	cases := []struct {
+		name     string
+		remotes  []remote
+		expected string
+		fail     bool
+	}{
+		{
+			name:    "no remotes",
+			remotes: []remote{},
+			fail:    true,
+		},
+		{
+			name: "origin",
+			remotes: []remote{
+				{
+					name: "origin",
+					url:  "git@github.com:crazy-max/buildx.git",
+				},
+			},
+			expected: "git@github.com:crazy-max/buildx.git",
+		},
+		{
+			name: "upstream",
+			remotes: []remote{
+				{
+					name: "upstream",
+					url:  "git@github.com:docker/buildx.git",
+				},
+			},
+			expected: "git@github.com:docker/buildx.git",
+		},
+		{
+			name: "origin and upstream",
+			remotes: []remote{
+				{
+					name: "upstream",
+					url:  "git@github.com:docker/buildx.git",
+				},
+				{
+					name: "origin",
+					url:  "git@github.com:crazy-max/buildx.git",
+				},
+			},
+			expected: "git@github.com:crazy-max/buildx.git",
+		},
+		{
+			name: "not found",
+			remotes: []remote{
+				{
+					name: "foo",
+					url:  "git@github.com:docker/buildx.git",
+				},
+			},
+			fail: true,
+		},
+	}
+	for _, tt := range cases {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			Mktmp(t)
+			c, err := New()
+			require.NoError(t, err)
+
+			GitInit(c, t)
+			GitCommit(c, t, "initial commit")
+			for _, r := range tt.remotes {
+				GitSetRemote(c, t, r.name, r.url)
+			}
+
+			ru, err := c.RemoteURL()
+			if tt.fail {
+				require.Error(t, err)
+				return
+			}
+			require.NoError(t, err)
+			require.Equal(t, tt.expected, ru)
+		})
+	}
+}
diff --git a/util/gitutil/testutil.go b/util/gitutil/testutil.go
index 91448a0d..2b0d514b 100644
--- a/util/gitutil/testutil.go
+++ b/util/gitutil/testutil.go
@@ -44,9 +44,9 @@ func GitAdd(c *Git, tb testing.TB, file string) {
 	require.NoError(tb, err)
 }
 
-func GitSetRemote(c *Git, tb testing.TB, url string) {
+func GitSetRemote(c *Git, tb testing.TB, name string, url string) {
 	tb.Helper()
-	_, err := fakeGit(c, "remote", "add", "origin", url)
+	_, err := fakeGit(c, "remote", "add", name, url)
 	require.NoError(tb, err)
 }