ibnezzoubayr/Atay-Makhzan
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix mirror sync parser and fix mirror messages (#36504)

Browse Source 
Fix #36474 

It also fixed a bug when sync deleted branches.
This commit is contained in:
Lunny Xiao
2026-02-10 16:16:05 -08:00
committed by GitHub
parent 2d70d37bff
commit 18ccee0f2f
11 changed files with 119 additions and 234 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/migrations/gitea_uploader.go
+3 -2
View File
@@ -364,11 +364,12 @@ func (g *GiteaLocalUploader) CreateReleases(ctx context.Context, releases ...*ba
// SyncTags syncs releases with tags in the database
func (g *GiteaLocalUploader) SyncTags(ctx context.Context) error {
return repo_module.SyncReleasesWithTags(ctx, g.repo, g.gitRepo)
_, err := repo_module.SyncReleasesWithTags(ctx, g.repo, g.gitRepo)
return err
}
func (g *GiteaLocalUploader) SyncBranches(ctx context.Context) error {
_, err := repo_module.SyncRepoBranchesWithRepo(ctx, g.repo, g.gitRepo, g.doer.ID)
_, _, err := repo_module.SyncRepoBranchesWithRepo(ctx, g.repo, g.gitRepo, g.doer.ID)
return err
}
services/mirror/mirror_pull.go
+23 -147
View File
@@ -29,9 +29,6 @@ import (
repo_service "code.gitea.io/gitea/services/repository"
)
// gitShortEmptySha Git short empty SHA
const gitShortEmptySha = "0000000"
// UpdateAddress writes new address to Git repository and database
func UpdateAddress(ctx context.Context, m *repo_model.Mirror, addr string) error {
u, err := giturl.ParseGitURL(addr)
@@ -72,127 +69,6 @@ func UpdateAddress(ctx context.Context, m *repo_model.Mirror, addr string) error
return repo_model.UpdateRepositoryColsNoAutoTime(ctx, m.Repo, "original_url")
}
// mirrorSyncResult contains information of a updated reference.
// If the oldCommitID is "0000000", it means a new reference, the value of newCommitID is empty.
// If the newCommitID is "0000000", it means the reference is deleted, the value of oldCommitID is empty.
type mirrorSyncResult struct {
refName git.RefName
oldCommitID string
newCommitID string
}
// parseRemoteUpdateOutput detects create, update and delete operations of references from upstream.
// possible output example:
/*
// * [new tag] v0.1.8 -> v0.1.8
// * [new branch] master -> origin/master
// * [new ref] refs/pull/2/head -> refs/pull/2/head"
// - [deleted] (none) -> origin/test // delete a branch
// - [deleted] (none) -> 1 // delete a tag
// 957a993..a87ba5f test -> origin/test
// + f895a1e...957a993 test -> origin/test (forced update)
*/
// TODO: return whether it's a force update
func parseRemoteUpdateOutput(output, remoteName string) []*mirrorSyncResult {
results := make([]*mirrorSyncResult, 0, 3)
lines := strings.Split(output, "\n")
for i := range lines {
// Make sure reference name is presented before continue
idx := strings.Index(lines[i], "-> ")
if idx == -1 {
continue
}
refName := strings.TrimSpace(lines[i][idx+3:])
switch {
case strings.HasPrefix(lines[i], " * [new tag]"): // new tag
results = append(results, &mirrorSyncResult{
refName: git.RefNameFromTag(refName),
oldCommitID: gitShortEmptySha,
})
case strings.HasPrefix(lines[i], " * [new branch]"): // new branch
refName = strings.TrimPrefix(refName, remoteName+"/")
results = append(results, &mirrorSyncResult{
refName: git.RefNameFromBranch(refName),
oldCommitID: gitShortEmptySha,
})
case strings.HasPrefix(lines[i], " * [new ref]"): // new reference
results = append(results, &mirrorSyncResult{
refName: git.RefName(refName),
oldCommitID: gitShortEmptySha,
})
case strings.HasPrefix(lines[i], " - "): // Delete reference
isTag := !strings.HasPrefix(refName, remoteName+"/")
var refFullName git.RefName
if strings.HasPrefix(refName, "refs/") {
refFullName = git.RefName(refName)
} else if isTag {
refFullName = git.RefNameFromTag(refName)
} else {
refFullName = git.RefNameFromBranch(strings.TrimPrefix(refName, remoteName+"/"))
}
results = append(results, &mirrorSyncResult{
refName: refFullName,
newCommitID: gitShortEmptySha,
})
case strings.HasPrefix(lines[i], " + "): // Force update
if idx := strings.Index(refName, " "); idx > -1 {
refName = refName[:idx]
}
delimIdx := strings.Index(lines[i][3:], " ")
if delimIdx == -1 {
log.Error("SHA delimiter not found: %q", lines[i])
continue
}
shas := strings.Split(lines[i][3:delimIdx+3], "...")
if len(shas) != 2 {
log.Error("Expect two SHAs but not what found: %q", lines[i])
continue
}
var refFullName git.RefName
if strings.HasPrefix(refName, "refs/") {
refFullName = git.RefName(refName)
} else {
refFullName = git.RefNameFromBranch(strings.TrimPrefix(refName, remoteName+"/"))
}
results = append(results, &mirrorSyncResult{
refName: refFullName,
oldCommitID: shas[0],
newCommitID: shas[1],
})
case strings.HasPrefix(lines[i], " "): // New commits of a reference
delimIdx := strings.Index(lines[i][3:], " ")
if delimIdx == -1 {
log.Error("SHA delimiter not found: %q", lines[i])
continue
}
shas := strings.Split(lines[i][3:delimIdx+3], "..")
if len(shas) != 2 {
log.Error("Expect two SHAs but not what found: %q", lines[i])
continue
}
var refFullName git.RefName
if strings.HasPrefix(refName, "refs/") {
refFullName = git.RefName(refName)
} else {
refFullName = git.RefNameFromBranch(strings.TrimPrefix(refName, remoteName+"/"))
}
results = append(results, &mirrorSyncResult{
refName: refFullName,
oldCommitID: shas[0],
newCommitID: shas[1],
})
default:
log.Warn("parseRemoteUpdateOutput: unexpected update line %q", lines[i])
}
}
return results
}
func pruneBrokenReferences(ctx context.Context, m *repo_model.Mirror, gitRepo gitrepo.Repository, timeout time.Duration) error {
cmd := gitcmd.NewCommand("remote", "prune").AddDynamicArguments(m.GetRemoteName()).WithTimeout(timeout)
stdout, _, pruneErr := gitrepo.RunCmdString(ctx, gitRepo, cmd)
@@ -229,7 +105,7 @@ func checkRecoverableSyncError(stderrMessage string) bool {
}
// runSync returns true if sync finished without error.
func runSync(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bool) {
func runSync(ctx context.Context, m *repo_model.Mirror) ([]*repo_module.SyncResult, bool) {
log.Trace("SyncMirrors [repo: %-v]: running git remote update...", m.Repo)
remoteURL, remoteErr := gitrepo.GitRemoteGetURL(ctx, m.Repo, m.GetRemoteName())
@@ -250,7 +126,6 @@ func runSync(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bo
}
var err error
var fetchOutput string // it is from fetch's stderr
fetchStdout, fetchStderr, err := gitrepo.RunCmdString(ctx, m.Repo, cmdFetch())
if err != nil {
// sanitize the output, since it may contain the remote address, which may contain a password
@@ -284,8 +159,6 @@ func runSync(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bo
return nil, false
}
}
fetchOutput = fetchStderr // the result of "git fetch" is in stderr
if err := gitrepo.WriteCommitGraph(ctx, m.Repo); err != nil {
log.Error("SyncMirrors [repo: %-v]: %v", m.Repo, err)
}
@@ -306,14 +179,17 @@ func runSync(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bo
}
log.Trace("SyncMirrors [repo: %-v]: syncing branches...", m.Repo)
if _, err = repo_module.SyncRepoBranchesWithRepo(ctx, m.Repo, gitRepo, 0); err != nil {
_, results, err := repo_module.SyncRepoBranchesWithRepo(ctx, m.Repo, gitRepo, 0)
if err != nil {
log.Error("SyncMirrors [repo: %-v]: failed to synchronize branches: %v", m.Repo, err)
}
log.Trace("SyncMirrors [repo: %-v]: syncing releases with tags...", m.Repo)
if err = repo_module.SyncReleasesWithTags(ctx, m.Repo, gitRepo); err != nil {
tagResults, err := repo_module.SyncReleasesWithTags(ctx, m.Repo, gitRepo)
if err != nil {
log.Error("SyncMirrors [repo: %-v]: failed to synchronize tags to releases: %v", m.Repo, err)
}
results = append(results, tagResults...)
gitRepo.Close()
log.Trace("SyncMirrors [repo: %-v]: updating size of repository", m.Repo)
@@ -381,7 +257,7 @@ func runSync(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bo
}
m.UpdatedUnix = timeutil.TimeStampNow()
return parseRemoteUpdateOutput(fetchOutput, m.GetRemoteName()), true
return results, true
}
func getRepoPullMirrorLockKey(repoID int64) string {
@@ -450,42 +326,42 @@ func SyncPullMirror(ctx context.Context, repoID int64) bool {
for _, result := range results {
// Discard GitHub pull requests, i.e. refs/pull/*
if result.refName.IsPull() {
if result.RefName.IsPull() {
continue
}
// Create reference
if result.oldCommitID == gitShortEmptySha {
commitID, err := gitRepo.GetRefCommitID(result.refName.String())
if result.OldCommitID == "" {
commitID, err := gitRepo.GetRefCommitID(result.RefName.String())
if err != nil {
log.Error("SyncMirrors [repo: %-v]: unable to GetRefCommitID [ref_name: %s]: %v", m.Repo, result.refName, err)
log.Error("SyncMirrors [repo: %-v]: unable to GetRefCommitID [ref_name: %s]: %v", m.Repo, result.RefName, err)
continue
}
objectFormat := git.ObjectFormatFromName(m.Repo.ObjectFormatName)
notify_service.SyncPushCommits(ctx, m.Repo.MustOwner(ctx), m.Repo, &repo_module.PushUpdateOptions{
RefFullName: result.refName,
RefFullName: result.RefName,
OldCommitID: objectFormat.EmptyObjectID().String(),
NewCommitID: commitID,
}, repo_module.NewPushCommits())
notify_service.SyncCreateRef(ctx, m.Repo.MustOwner(ctx), m.Repo, result.refName, commitID)
notify_service.SyncCreateRef(ctx, m.Repo.MustOwner(ctx), m.Repo, result.RefName, commitID)
continue
}
// Delete reference
if result.newCommitID == gitShortEmptySha {
notify_service.SyncDeleteRef(ctx, m.Repo.MustOwner(ctx), m.Repo, result.refName)
if result.NewCommitID == "" {
notify_service.SyncDeleteRef(ctx, m.Repo.MustOwner(ctx), m.Repo, result.RefName)
continue
}
// Push commits
oldCommitID, err := gitrepo.GetFullCommitID(ctx, repo, result.oldCommitID)
oldCommitID, err := gitrepo.GetFullCommitID(ctx, repo, result.OldCommitID)
if err != nil {
log.Error("SyncMirrors [repo: %-v]: unable to get GetFullCommitID[%s]: %v", m.Repo, result.oldCommitID, err)
log.Error("SyncMirrors [repo: %-v]: unable to get GetFullCommitID[%s]: %v", m.Repo, result.OldCommitID, err)
continue
}
newCommitID, err := gitrepo.GetFullCommitID(ctx, repo, result.newCommitID)
newCommitID, err := gitrepo.GetFullCommitID(ctx, repo, result.NewCommitID)
if err != nil {
log.Error("SyncMirrors [repo: %-v]: unable to get GetFullCommitID [%s]: %v", m.Repo, result.newCommitID, err)
log.Error("SyncMirrors [repo: %-v]: unable to get GetFullCommitID [%s]: %v", m.Repo, result.NewCommitID, err)
continue
}
commits, err := gitRepo.CommitsBetweenIDs(newCommitID, oldCommitID)
@@ -509,7 +385,7 @@ func SyncPullMirror(ctx context.Context, repoID int64) bool {
theCommits.CompareURL = m.Repo.ComposeCompareURL(oldCommitID, newCommitID)
notify_service.SyncPushCommits(ctx, m.Repo.MustOwner(ctx), m.Repo, &repo_module.PushUpdateOptions{
RefFullName: result.refName,
RefFullName: result.RefName,
OldCommitID: oldCommitID,
NewCommitID: newCommitID,
}, theCommits)
@@ -548,7 +424,7 @@ func SyncPullMirror(ctx context.Context, repoID int64) bool {
return true
}
func checkAndUpdateEmptyRepository(ctx context.Context, m *repo_model.Mirror, results []*mirrorSyncResult) bool {
func checkAndUpdateEmptyRepository(ctx context.Context, m *repo_model.Mirror, results []*repo_module.SyncResult) bool {
if !m.Repo.IsEmpty {
return true
}
@@ -562,11 +438,11 @@ func checkAndUpdateEmptyRepository(ctx context.Context, m *repo_model.Mirror, re
}
firstName := ""
for _, result := range results {
if !result.refName.IsBranch() {
if !result.RefName.IsBranch() {
continue
}
name := result.refName.BranchName()
name := result.RefName.BranchName()
if len(firstName) == 0 {
firstName = name
}
services/mirror/mirror_pull_test.go
-56
View File
@@ -9,62 +9,6 @@ import (
"github.com/stretchr/testify/assert"
)
func Test_parseRemoteUpdateOutput(t *testing.T) {
output := `
* [new tag] v0.1.8 -> v0.1.8
* [new branch] master -> origin/master
- [deleted] (none) -> origin/test1
- [deleted] (none) -> tag1
+ f895a1e...957a993 test2 -> origin/test2 (forced update)
957a993..a87ba5f test3 -> origin/test3
* [new ref] refs/pull/26595/head -> refs/pull/26595/head
* [new ref] refs/pull/26595/merge -> refs/pull/26595/merge
e0639e38fb..6db2410489 refs/pull/25873/head -> refs/pull/25873/head
+ 1c97ebc746...976d27d52f refs/pull/25873/merge -> refs/pull/25873/merge (forced update)
`
results := parseRemoteUpdateOutput(output, "origin")
assert.Len(t, results, 10)
assert.Equal(t, "refs/tags/v0.1.8", results[0].refName.String())
assert.Equal(t, gitShortEmptySha, results[0].oldCommitID)
assert.Empty(t, results[0].newCommitID)
assert.Equal(t, "refs/heads/master", results[1].refName.String())
assert.Equal(t, gitShortEmptySha, results[1].oldCommitID)
assert.Empty(t, results[1].newCommitID)
assert.Equal(t, "refs/heads/test1", results[2].refName.String())
assert.Empty(t, results[2].oldCommitID)
assert.Equal(t, gitShortEmptySha, results[2].newCommitID)
assert.Equal(t, "refs/tags/tag1", results[3].refName.String())
assert.Empty(t, results[3].oldCommitID)
assert.Equal(t, gitShortEmptySha, results[3].newCommitID)
assert.Equal(t, "refs/heads/test2", results[4].refName.String())
assert.Equal(t, "f895a1e", results[4].oldCommitID)
assert.Equal(t, "957a993", results[4].newCommitID)
assert.Equal(t, "refs/heads/test3", results[5].refName.String())
assert.Equal(t, "957a993", results[5].oldCommitID)
assert.Equal(t, "a87ba5f", results[5].newCommitID)
assert.Equal(t, "refs/pull/26595/head", results[6].refName.String())
assert.Equal(t, gitShortEmptySha, results[6].oldCommitID)
assert.Empty(t, results[6].newCommitID)
assert.Equal(t, "refs/pull/26595/merge", results[7].refName.String())
assert.Equal(t, gitShortEmptySha, results[7].oldCommitID)
assert.Empty(t, results[7].newCommitID)
assert.Equal(t, "refs/pull/25873/head", results[8].refName.String())
assert.Equal(t, "e0639e38fb", results[8].oldCommitID)
assert.Equal(t, "6db2410489", results[8].newCommitID)
assert.Equal(t, "refs/pull/25873/merge", results[9].refName.String())
assert.Equal(t, "1c97ebc746", results[9].oldCommitID)
assert.Equal(t, "976d27d52f", results[9].newCommitID)
}
func Test_checkRecoverableSyncError(t *testing.T) {
cases := []struct {
recoverable bool
services/repository/adopt.go
+2 -2
View File
@@ -147,11 +147,11 @@ func adoptRepository(ctx context.Context, repo *repo_model.Repository, defaultBr
}
defer gitRepo.Close()
if _, err = repo_module.SyncRepoBranchesWithRepo(ctx, repo, gitRepo, 0); err != nil {
if _, _, err = repo_module.SyncRepoBranchesWithRepo(ctx, repo, gitRepo, 0); err != nil {
return fmt.Errorf("SyncRepoBranchesWithRepo: %w", err)
}
if err = repo_module.SyncReleasesWithTags(ctx, repo, gitRepo); err != nil {
if _, err = repo_module.SyncReleasesWithTags(ctx, repo, gitRepo); err != nil {
return fmt.Errorf("SyncReleasesWithTags: %w", err)
}
services/repository/fork.go
+2 -2
View File
@@ -177,10 +177,10 @@ func ForkRepository(ctx context.Context, doer, owner *user_model.User, opts Fork
}
defer gitRepo.Close()
if _, err = repo_module.SyncRepoBranchesWithRepo(ctx, repo, gitRepo, doer.ID); err != nil {
if _, _, err = repo_module.SyncRepoBranchesWithRepo(ctx, repo, gitRepo, doer.ID); err != nil {
return nil, fmt.Errorf("SyncRepoBranchesWithRepo: %w", err)
}
if err = repo_module.SyncReleasesWithTags(ctx, repo, gitRepo); err != nil {
if _, err = repo_module.SyncReleasesWithTags(ctx, repo, gitRepo); err != nil {
return nil, fmt.Errorf("Sync releases from git tags failed: %v", err)
}
services/repository/migrate.go
+2 -2
View File
@@ -145,7 +145,7 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User,
}
}
if _, err := repo_module.SyncRepoBranchesWithRepo(ctx, repo, gitRepo, u.ID); err != nil {
if _, _, err := repo_module.SyncRepoBranchesWithRepo(ctx, repo, gitRepo, u.ID); err != nil {
return repo, fmt.Errorf("SyncRepoBranchesWithRepo: %v", err)
}
@@ -153,7 +153,7 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User,
// otherwise, the releases sync will be done out of this function
if !opts.Releases {
repo.IsMirror = opts.Mirror
if err = repo_module.SyncReleasesWithTags(ctx, repo, gitRepo); err != nil {
if _, err = repo_module.SyncReleasesWithTags(ctx, repo, gitRepo); err != nil {
log.Error("Failed to synchronize tags to releases for repository: %v", err)
}
}