Fix mirror sync parser and fix mirror messages (#36504)

Fix #36474 

It also fixed a bug when sync deleted branches.
This commit is contained in:
Lunny Xiao
2026-02-10 16:16:05 -08:00
committed by GitHub
parent 2d70d37bff
commit 18ccee0f2f
11 changed files with 119 additions and 234 deletions
+1 -1
View File
@@ -134,7 +134,7 @@ func runRepoSyncReleases(ctx context.Context, _ *cli.Command) error {
} }
log.Trace(" currentNumReleases is %d, running SyncReleasesWithTags", oldnum) log.Trace(" currentNumReleases is %d, running SyncReleasesWithTags", oldnum)
if err = repo_module.SyncReleasesWithTags(ctx, repo, gitRepo); err != nil { if _, err = repo_module.SyncReleasesWithTags(ctx, repo, gitRepo); err != nil {
log.Warn(" SyncReleasesWithTags: %v", err) log.Warn(" SyncReleasesWithTags: %v", err)
gitRepo.Close() gitRepo.Close()
continue continue
+35 -11
View File
@@ -17,6 +17,13 @@ import (
"code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/timeutil"
) )
// SyncResult describes a reference update detected during sync.
type SyncResult struct {
RefName git.RefName
OldCommitID string
NewCommitID string
}
// SyncRepoBranches synchronizes branch table with repository branches // SyncRepoBranches synchronizes branch table with repository branches
func SyncRepoBranches(ctx context.Context, repoID, doerID int64) (int64, error) { func SyncRepoBranches(ctx context.Context, repoID, doerID int64) (int64, error) {
repo, err := repo_model.GetRepositoryByID(ctx, repoID) repo, err := repo_model.GetRepositoryByID(ctx, repoID)
@@ -33,18 +40,19 @@ func SyncRepoBranches(ctx context.Context, repoID, doerID int64) (int64, error)
} }
defer gitRepo.Close() defer gitRepo.Close()
return SyncRepoBranchesWithRepo(ctx, repo, gitRepo, doerID) count, _, err := SyncRepoBranchesWithRepo(ctx, repo, gitRepo, doerID)
return count, err
} }
func SyncRepoBranchesWithRepo(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, doerID int64) (int64, error) { func SyncRepoBranchesWithRepo(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, doerID int64) (int64, []*SyncResult, error) {
objFmt, err := gitRepo.GetObjectFormat() objFmt, err := gitRepo.GetObjectFormat()
if err != nil { if err != nil {
return 0, fmt.Errorf("GetObjectFormat: %w", err) return 0, nil, fmt.Errorf("GetObjectFormat: %w", err)
} }
if objFmt.Name() != repo.ObjectFormatName { if objFmt.Name() != repo.ObjectFormatName {
repo.ObjectFormatName = objFmt.Name() repo.ObjectFormatName = objFmt.Name()
if err = repo_model.UpdateRepositoryColsWithAutoTime(ctx, repo, "object_format_name"); err != nil { if err = repo_model.UpdateRepositoryColsWithAutoTime(ctx, repo, "object_format_name"); err != nil {
return 0, fmt.Errorf("UpdateRepositoryColsWithAutoTime: %w", err) return 0, nil, fmt.Errorf("UpdateRepositoryColsWithAutoTime: %w", err)
} }
} }
@@ -52,7 +60,7 @@ func SyncRepoBranchesWithRepo(ctx context.Context, repo *repo_model.Repository,
{ {
branches, _, err := gitRepo.GetBranchNames(0, 0) branches, _, err := gitRepo.GetBranchNames(0, 0)
if err != nil { if err != nil {
return 0, err return 0, nil, err
} }
log.Trace("SyncRepoBranches[%s]: branches[%d]: %v", repo.FullName(), len(branches), branches) log.Trace("SyncRepoBranches[%s]: branches[%d]: %v", repo.FullName(), len(branches), branches)
for _, branch := range branches { for _, branch := range branches {
@@ -67,7 +75,7 @@ func SyncRepoBranchesWithRepo(ctx context.Context, repo *repo_model.Repository,
RepoID: repo.ID, RepoID: repo.ID,
}) })
if err != nil { if err != nil {
return 0, err return 0, nil, err
} }
for _, branch := range branches { for _, branch := range branches {
dbBranches[branch.Name] = branch dbBranches[branch.Name] = branch
@@ -77,11 +85,12 @@ func SyncRepoBranchesWithRepo(ctx context.Context, repo *repo_model.Repository,
var toAdd []*git_model.Branch var toAdd []*git_model.Branch
var toUpdate []*git_model.Branch var toUpdate []*git_model.Branch
var toRemove []int64 var toRemove []int64
var syncResults []*SyncResult
for branch := range allBranches { for branch := range allBranches {
dbb := dbBranches[branch] dbb := dbBranches[branch]
commit, err := gitRepo.GetBranchCommit(branch) commit, err := gitRepo.GetBranchCommit(branch)
if err != nil { if err != nil {
return 0, err return 0, nil, err
} }
if dbb == nil { if dbb == nil {
toAdd = append(toAdd, &git_model.Branch{ toAdd = append(toAdd, &git_model.Branch{
@@ -92,7 +101,12 @@ func SyncRepoBranchesWithRepo(ctx context.Context, repo *repo_model.Repository,
PusherID: doerID, PusherID: doerID,
CommitTime: timeutil.TimeStamp(commit.Committer.When.Unix()), CommitTime: timeutil.TimeStamp(commit.Committer.When.Unix()),
}) })
} else if commit.ID.String() != dbb.CommitID { syncResults = append(syncResults, &SyncResult{
RefName: git.RefNameFromBranch(branch),
OldCommitID: "",
NewCommitID: commit.ID.String(),
})
} else if commit.ID.String() != dbb.CommitID || dbb.IsDeleted {
toUpdate = append(toUpdate, &git_model.Branch{ toUpdate = append(toUpdate, &git_model.Branch{
ID: dbb.ID, ID: dbb.ID,
RepoID: repo.ID, RepoID: repo.ID,
@@ -102,19 +116,29 @@ func SyncRepoBranchesWithRepo(ctx context.Context, repo *repo_model.Repository,
PusherID: doerID, PusherID: doerID,
CommitTime: timeutil.TimeStamp(commit.Committer.When.Unix()), CommitTime: timeutil.TimeStamp(commit.Committer.When.Unix()),
}) })
syncResults = append(syncResults, &SyncResult{
RefName: git.RefNameFromBranch(branch),
OldCommitID: dbb.CommitID,
NewCommitID: commit.ID.String(),
})
} }
} }
for _, dbBranch := range dbBranches { for _, dbBranch := range dbBranches {
if !allBranches.Contains(dbBranch.Name) && !dbBranch.IsDeleted { if !allBranches.Contains(dbBranch.Name) && !dbBranch.IsDeleted {
toRemove = append(toRemove, dbBranch.ID) toRemove = append(toRemove, dbBranch.ID)
syncResults = append(syncResults, &SyncResult{
RefName: git.RefNameFromBranch(dbBranch.Name),
OldCommitID: dbBranch.CommitID,
NewCommitID: "",
})
} }
} }
log.Trace("SyncRepoBranches[%s]: toAdd: %v, toUpdate: %v, toRemove: %v", repo.FullName(), toAdd, toUpdate, toRemove) log.Trace("SyncRepoBranches[%s]: toAdd: %v, toUpdate: %v, toRemove: %v", repo.FullName(), toAdd, toUpdate, toRemove)
if len(toAdd) == 0 && len(toRemove) == 0 && len(toUpdate) == 0 { if len(toAdd) == 0 && len(toRemove) == 0 && len(toUpdate) == 0 {
return int64(len(allBranches)), nil return int64(len(allBranches)), syncResults, nil
} }
if err := db.WithTx(ctx, func(ctx context.Context) error { if err := db.WithTx(ctx, func(ctx context.Context) error {
@@ -140,7 +164,7 @@ func SyncRepoBranchesWithRepo(ctx context.Context, repo *repo_model.Repository,
return nil return nil
}); err != nil { }); err != nil {
return 0, err return 0, nil, err
} }
return int64(len(allBranches)), nil return int64(len(allBranches)), syncResults, nil
} }
+45 -5
View File
@@ -53,7 +53,8 @@ func SyncRepoTags(ctx context.Context, repoID int64) error {
} }
defer gitRepo.Close() defer gitRepo.Close()
return SyncReleasesWithTags(ctx, repo, gitRepo) _, err = SyncReleasesWithTags(ctx, repo, gitRepo)
return err
} }
// StoreMissingLfsObjectsInRepository downloads missing LFS objects // StoreMissingLfsObjectsInRepository downloads missing LFS objects
@@ -178,13 +179,14 @@ func (shortRelease) TableName() string {
// upstream. Hence, after each sync we want the release set to be // upstream. Hence, after each sync we want the release set to be
// identical to the upstream tag set. This is much more efficient for // identical to the upstream tag set. This is much more efficient for
// repositories like https://github.com/vim/vim (with over 13000 tags). // repositories like https://github.com/vim/vim (with over 13000 tags).
func SyncReleasesWithTags(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository) error { func SyncReleasesWithTags(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository) ([]*SyncResult, error) {
log.Debug("SyncReleasesWithTags: in Repo[%d:%s/%s]", repo.ID, repo.OwnerName, repo.Name) log.Debug("SyncReleasesWithTags: in Repo[%d:%s/%s]", repo.ID, repo.OwnerName, repo.Name)
tags, _, err := gitRepo.GetTagInfos(0, 0) tags, _, err := gitRepo.GetTagInfos(0, 0)
if err != nil { if err != nil {
return fmt.Errorf("unable to GetTagInfos in pull-mirror Repo[%d:%s/%s]: %w", repo.ID, repo.OwnerName, repo.Name, err) return nil, fmt.Errorf("unable to GetTagInfos in pull-mirror Repo[%d:%s/%s]: %w", repo.ID, repo.OwnerName, repo.Name, err)
} }
var added, deleted, updated int var added, deleted, updated int
var syncResults []*SyncResult
err = db.WithTx(ctx, func(ctx context.Context) error { err = db.WithTx(ctx, func(ctx context.Context) error {
dbReleases, err := db.Find[shortRelease](ctx, repo_model.FindReleasesOptions{ dbReleases, err := db.Find[shortRelease](ctx, repo_model.FindReleasesOptions{
RepoID: repo.ID, RepoID: repo.ID,
@@ -195,7 +197,45 @@ func SyncReleasesWithTags(ctx context.Context, repo *repo_model.Repository, gitR
return fmt.Errorf("unable to FindReleases in pull-mirror Repo[%d:%s/%s]: %w", repo.ID, repo.OwnerName, repo.Name, err) return fmt.Errorf("unable to FindReleases in pull-mirror Repo[%d:%s/%s]: %w", repo.ID, repo.OwnerName, repo.Name, err)
} }
dbReleasesByID := make(map[int64]*shortRelease, len(dbReleases))
dbReleasesByTag := make(map[string]*shortRelease, len(dbReleases))
for _, release := range dbReleases {
dbReleasesByID[release.ID] = release
dbReleasesByTag[release.TagName] = release
}
inserts, deletes, updates := calcSync(tags, dbReleases) inserts, deletes, updates := calcSync(tags, dbReleases)
syncResults = make([]*SyncResult, 0, len(inserts)+len(deletes)+len(updates))
for _, tag := range inserts {
syncResults = append(syncResults, &SyncResult{
RefName: git.RefNameFromTag(tag.Name),
OldCommitID: "",
NewCommitID: tag.Object.String(),
})
}
for _, deleteID := range deletes {
release := dbReleasesByID[deleteID]
if release == nil {
continue
}
syncResults = append(syncResults, &SyncResult{
RefName: git.RefNameFromTag(release.TagName),
OldCommitID: release.Sha1,
NewCommitID: "",
})
}
for _, tag := range updates {
release := dbReleasesByTag[tag.Name]
oldSha := ""
if release != nil {
oldSha = release.Sha1
}
syncResults = append(syncResults, &SyncResult{
RefName: git.RefNameFromTag(tag.Name),
OldCommitID: oldSha,
NewCommitID: tag.Object.String(),
})
}
// //
// make release set identical to upstream tags // make release set identical to upstream tags
// //
@@ -238,11 +278,11 @@ func SyncReleasesWithTags(ctx context.Context, repo *repo_model.Repository, gitR
return nil return nil
}) })
if err != nil { if err != nil {
return fmt.Errorf("unable to rebuild release table for pull-mirror Repo[%d:%s/%s]: %w", repo.ID, repo.OwnerName, repo.Name, err) return nil, fmt.Errorf("unable to rebuild release table for pull-mirror Repo[%d:%s/%s]: %w", repo.ID, repo.OwnerName, repo.Name, err)
} }
log.Trace("SyncReleasesWithTags: %d tags added, %d tags deleted, %d tags updated", added, deleted, updated) log.Trace("SyncReleasesWithTags: %d tags added, %d tags deleted, %d tags updated", added, deleted, updated)
return nil return syncResults, nil
} }
func calcSync(destTags []*git.Tag, dbTags []*shortRelease) ([]*git.Tag, []int64, []*git.Tag) { func calcSync(destTags []*git.Tag, dbTags []*shortRelease) ([]*git.Tag, []int64, []*git.Tag) {
+3 -3
View File
@@ -158,16 +158,16 @@ func feedActionsToFeedItems(ctx *context.Context, actions activities_model.Actio
if link.Href == "#" { if link.Href == "#" {
link.Href = srcLink link.Href = srcLink
} }
titleExtra = ctx.Locale.Tr("action.mirror_sync_push", act.GetRepoAbsoluteLink(ctx), srcLink, act.GetBranch(), act.ShortRepoPath(ctx)) titleExtra = ctx.Locale.Tr("action.mirror_sync_push", act.GetRepoAbsoluteLink(ctx), srcLink, act.RefName, act.ShortRepoPath(ctx))
case activities_model.ActionMirrorSyncCreate: case activities_model.ActionMirrorSyncCreate:
srcLink := toSrcLink(ctx, act) srcLink := toSrcLink(ctx, act)
if link.Href == "#" { if link.Href == "#" {
link.Href = srcLink link.Href = srcLink
} }
titleExtra = ctx.Locale.Tr("action.mirror_sync_create", act.GetRepoAbsoluteLink(ctx), srcLink, act.GetBranch(), act.ShortRepoPath(ctx)) titleExtra = ctx.Locale.Tr("action.mirror_sync_create", act.GetRepoAbsoluteLink(ctx), srcLink, act.RefName, act.ShortRepoPath(ctx))
case activities_model.ActionMirrorSyncDelete: case activities_model.ActionMirrorSyncDelete:
link.Href = act.GetRepoAbsoluteLink(ctx) link.Href = act.GetRepoAbsoluteLink(ctx)
titleExtra = ctx.Locale.Tr("action.mirror_sync_delete", act.GetRepoAbsoluteLink(ctx), act.GetBranch(), act.ShortRepoPath(ctx)) titleExtra = ctx.Locale.Tr("action.mirror_sync_delete", act.GetRepoAbsoluteLink(ctx), act.RefName, act.ShortRepoPath(ctx))
case activities_model.ActionApprovePullRequest: case activities_model.ActionApprovePullRequest:
pullLink := toPullLink(ctx, act) pullLink := toPullLink(ctx, act)
titleExtra = ctx.Locale.Tr("action.approve_pull_request", pullLink, act.GetIssueInfos()[0], act.ShortRepoPath(ctx)) titleExtra = ctx.Locale.Tr("action.approve_pull_request", pullLink, act.GetIssueInfos()[0], act.ShortRepoPath(ctx))
+3 -2
View File
@@ -364,11 +364,12 @@ func (g *GiteaLocalUploader) CreateReleases(ctx context.Context, releases ...*ba
// SyncTags syncs releases with tags in the database // SyncTags syncs releases with tags in the database
func (g *GiteaLocalUploader) SyncTags(ctx context.Context) error { func (g *GiteaLocalUploader) SyncTags(ctx context.Context) error {
return repo_module.SyncReleasesWithTags(ctx, g.repo, g.gitRepo) _, err := repo_module.SyncReleasesWithTags(ctx, g.repo, g.gitRepo)
return err
} }
func (g *GiteaLocalUploader) SyncBranches(ctx context.Context) error { func (g *GiteaLocalUploader) SyncBranches(ctx context.Context) error {
_, err := repo_module.SyncRepoBranchesWithRepo(ctx, g.repo, g.gitRepo, g.doer.ID) _, _, err := repo_module.SyncRepoBranchesWithRepo(ctx, g.repo, g.gitRepo, g.doer.ID)
return err return err
} }
+23 -147
View File
@@ -29,9 +29,6 @@ import (
repo_service "code.gitea.io/gitea/services/repository" repo_service "code.gitea.io/gitea/services/repository"
) )
// gitShortEmptySha Git short empty SHA
const gitShortEmptySha = "0000000"
// UpdateAddress writes new address to Git repository and database // UpdateAddress writes new address to Git repository and database
func UpdateAddress(ctx context.Context, m *repo_model.Mirror, addr string) error { func UpdateAddress(ctx context.Context, m *repo_model.Mirror, addr string) error {
u, err := giturl.ParseGitURL(addr) u, err := giturl.ParseGitURL(addr)
@@ -72,127 +69,6 @@ func UpdateAddress(ctx context.Context, m *repo_model.Mirror, addr string) error
return repo_model.UpdateRepositoryColsNoAutoTime(ctx, m.Repo, "original_url") return repo_model.UpdateRepositoryColsNoAutoTime(ctx, m.Repo, "original_url")
} }
// mirrorSyncResult contains information of a updated reference.
// If the oldCommitID is "0000000", it means a new reference, the value of newCommitID is empty.
// If the newCommitID is "0000000", it means the reference is deleted, the value of oldCommitID is empty.
type mirrorSyncResult struct {
refName git.RefName
oldCommitID string
newCommitID string
}
// parseRemoteUpdateOutput detects create, update and delete operations of references from upstream.
// possible output example:
/*
// * [new tag] v0.1.8 -> v0.1.8
// * [new branch] master -> origin/master
// * [new ref] refs/pull/2/head -> refs/pull/2/head"
// - [deleted] (none) -> origin/test // delete a branch
// - [deleted] (none) -> 1 // delete a tag
// 957a993..a87ba5f test -> origin/test
// + f895a1e...957a993 test -> origin/test (forced update)
*/
// TODO: return whether it's a force update
func parseRemoteUpdateOutput(output, remoteName string) []*mirrorSyncResult {
results := make([]*mirrorSyncResult, 0, 3)
lines := strings.Split(output, "\n")
for i := range lines {
// Make sure reference name is presented before continue
idx := strings.Index(lines[i], "-> ")
if idx == -1 {
continue
}
refName := strings.TrimSpace(lines[i][idx+3:])
switch {
case strings.HasPrefix(lines[i], " * [new tag]"): // new tag
results = append(results, &mirrorSyncResult{
refName: git.RefNameFromTag(refName),
oldCommitID: gitShortEmptySha,
})
case strings.HasPrefix(lines[i], " * [new branch]"): // new branch
refName = strings.TrimPrefix(refName, remoteName+"/")
results = append(results, &mirrorSyncResult{
refName: git.RefNameFromBranch(refName),
oldCommitID: gitShortEmptySha,
})
case strings.HasPrefix(lines[i], " * [new ref]"): // new reference
results = append(results, &mirrorSyncResult{
refName: git.RefName(refName),
oldCommitID: gitShortEmptySha,
})
case strings.HasPrefix(lines[i], " - "): // Delete reference
isTag := !strings.HasPrefix(refName, remoteName+"/")
var refFullName git.RefName
if strings.HasPrefix(refName, "refs/") {
refFullName = git.RefName(refName)
} else if isTag {
refFullName = git.RefNameFromTag(refName)
} else {
refFullName = git.RefNameFromBranch(strings.TrimPrefix(refName, remoteName+"/"))
}
results = append(results, &mirrorSyncResult{
refName: refFullName,
newCommitID: gitShortEmptySha,
})
case strings.HasPrefix(lines[i], " + "): // Force update
if idx := strings.Index(refName, " "); idx > -1 {
refName = refName[:idx]
}
delimIdx := strings.Index(lines[i][3:], " ")
if delimIdx == -1 {
log.Error("SHA delimiter not found: %q", lines[i])
continue
}
shas := strings.Split(lines[i][3:delimIdx+3], "...")
if len(shas) != 2 {
log.Error("Expect two SHAs but not what found: %q", lines[i])
continue
}
var refFullName git.RefName
if strings.HasPrefix(refName, "refs/") {
refFullName = git.RefName(refName)
} else {
refFullName = git.RefNameFromBranch(strings.TrimPrefix(refName, remoteName+"/"))
}
results = append(results, &mirrorSyncResult{
refName: refFullName,
oldCommitID: shas[0],
newCommitID: shas[1],
})
case strings.HasPrefix(lines[i], " "): // New commits of a reference
delimIdx := strings.Index(lines[i][3:], " ")
if delimIdx == -1 {
log.Error("SHA delimiter not found: %q", lines[i])
continue
}
shas := strings.Split(lines[i][3:delimIdx+3], "..")
if len(shas) != 2 {
log.Error("Expect two SHAs but not what found: %q", lines[i])
continue
}
var refFullName git.RefName
if strings.HasPrefix(refName, "refs/") {
refFullName = git.RefName(refName)
} else {
refFullName = git.RefNameFromBranch(strings.TrimPrefix(refName, remoteName+"/"))
}
results = append(results, &mirrorSyncResult{
refName: refFullName,
oldCommitID: shas[0],
newCommitID: shas[1],
})
default:
log.Warn("parseRemoteUpdateOutput: unexpected update line %q", lines[i])
}
}
return results
}
func pruneBrokenReferences(ctx context.Context, m *repo_model.Mirror, gitRepo gitrepo.Repository, timeout time.Duration) error { func pruneBrokenReferences(ctx context.Context, m *repo_model.Mirror, gitRepo gitrepo.Repository, timeout time.Duration) error {
cmd := gitcmd.NewCommand("remote", "prune").AddDynamicArguments(m.GetRemoteName()).WithTimeout(timeout) cmd := gitcmd.NewCommand("remote", "prune").AddDynamicArguments(m.GetRemoteName()).WithTimeout(timeout)
stdout, _, pruneErr := gitrepo.RunCmdString(ctx, gitRepo, cmd) stdout, _, pruneErr := gitrepo.RunCmdString(ctx, gitRepo, cmd)
@@ -229,7 +105,7 @@ func checkRecoverableSyncError(stderrMessage string) bool {
} }
// runSync returns true if sync finished without error. // runSync returns true if sync finished without error.
func runSync(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bool) { func runSync(ctx context.Context, m *repo_model.Mirror) ([]*repo_module.SyncResult, bool) {
log.Trace("SyncMirrors [repo: %-v]: running git remote update...", m.Repo) log.Trace("SyncMirrors [repo: %-v]: running git remote update...", m.Repo)
remoteURL, remoteErr := gitrepo.GitRemoteGetURL(ctx, m.Repo, m.GetRemoteName()) remoteURL, remoteErr := gitrepo.GitRemoteGetURL(ctx, m.Repo, m.GetRemoteName())
@@ -250,7 +126,6 @@ func runSync(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bo
} }
var err error var err error
var fetchOutput string // it is from fetch's stderr
fetchStdout, fetchStderr, err := gitrepo.RunCmdString(ctx, m.Repo, cmdFetch()) fetchStdout, fetchStderr, err := gitrepo.RunCmdString(ctx, m.Repo, cmdFetch())
if err != nil { if err != nil {
// sanitize the output, since it may contain the remote address, which may contain a password // sanitize the output, since it may contain the remote address, which may contain a password
@@ -284,8 +159,6 @@ func runSync(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bo
return nil, false return nil, false
} }
} }
fetchOutput = fetchStderr // the result of "git fetch" is in stderr
if err := gitrepo.WriteCommitGraph(ctx, m.Repo); err != nil { if err := gitrepo.WriteCommitGraph(ctx, m.Repo); err != nil {
log.Error("SyncMirrors [repo: %-v]: %v", m.Repo, err) log.Error("SyncMirrors [repo: %-v]: %v", m.Repo, err)
} }
@@ -306,14 +179,17 @@ func runSync(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bo
} }
log.Trace("SyncMirrors [repo: %-v]: syncing branches...", m.Repo) log.Trace("SyncMirrors [repo: %-v]: syncing branches...", m.Repo)
if _, err = repo_module.SyncRepoBranchesWithRepo(ctx, m.Repo, gitRepo, 0); err != nil { _, results, err := repo_module.SyncRepoBranchesWithRepo(ctx, m.Repo, gitRepo, 0)
if err != nil {
log.Error("SyncMirrors [repo: %-v]: failed to synchronize branches: %v", m.Repo, err) log.Error("SyncMirrors [repo: %-v]: failed to synchronize branches: %v", m.Repo, err)
} }
log.Trace("SyncMirrors [repo: %-v]: syncing releases with tags...", m.Repo) log.Trace("SyncMirrors [repo: %-v]: syncing releases with tags...", m.Repo)
if err = repo_module.SyncReleasesWithTags(ctx, m.Repo, gitRepo); err != nil { tagResults, err := repo_module.SyncReleasesWithTags(ctx, m.Repo, gitRepo)
if err != nil {
log.Error("SyncMirrors [repo: %-v]: failed to synchronize tags to releases: %v", m.Repo, err) log.Error("SyncMirrors [repo: %-v]: failed to synchronize tags to releases: %v", m.Repo, err)
} }
results = append(results, tagResults...)
gitRepo.Close() gitRepo.Close()
log.Trace("SyncMirrors [repo: %-v]: updating size of repository", m.Repo) log.Trace("SyncMirrors [repo: %-v]: updating size of repository", m.Repo)
@@ -381,7 +257,7 @@ func runSync(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bo
} }
m.UpdatedUnix = timeutil.TimeStampNow() m.UpdatedUnix = timeutil.TimeStampNow()
return parseRemoteUpdateOutput(fetchOutput, m.GetRemoteName()), true return results, true
} }
func getRepoPullMirrorLockKey(repoID int64) string { func getRepoPullMirrorLockKey(repoID int64) string {
@@ -450,42 +326,42 @@ func SyncPullMirror(ctx context.Context, repoID int64) bool {
for _, result := range results { for _, result := range results {
// Discard GitHub pull requests, i.e. refs/pull/* // Discard GitHub pull requests, i.e. refs/pull/*
if result.refName.IsPull() { if result.RefName.IsPull() {
continue continue
} }
// Create reference // Create reference
if result.oldCommitID == gitShortEmptySha { if result.OldCommitID == "" {
commitID, err := gitRepo.GetRefCommitID(result.refName.String()) commitID, err := gitRepo.GetRefCommitID(result.RefName.String())
if err != nil { if err != nil {
log.Error("SyncMirrors [repo: %-v]: unable to GetRefCommitID [ref_name: %s]: %v", m.Repo, result.refName, err) log.Error("SyncMirrors [repo: %-v]: unable to GetRefCommitID [ref_name: %s]: %v", m.Repo, result.RefName, err)
continue continue
} }
objectFormat := git.ObjectFormatFromName(m.Repo.ObjectFormatName) objectFormat := git.ObjectFormatFromName(m.Repo.ObjectFormatName)
notify_service.SyncPushCommits(ctx, m.Repo.MustOwner(ctx), m.Repo, &repo_module.PushUpdateOptions{ notify_service.SyncPushCommits(ctx, m.Repo.MustOwner(ctx), m.Repo, &repo_module.PushUpdateOptions{
RefFullName: result.refName, RefFullName: result.RefName,
OldCommitID: objectFormat.EmptyObjectID().String(), OldCommitID: objectFormat.EmptyObjectID().String(),
NewCommitID: commitID, NewCommitID: commitID,
}, repo_module.NewPushCommits()) }, repo_module.NewPushCommits())
notify_service.SyncCreateRef(ctx, m.Repo.MustOwner(ctx), m.Repo, result.refName, commitID) notify_service.SyncCreateRef(ctx, m.Repo.MustOwner(ctx), m.Repo, result.RefName, commitID)
continue continue
} }
// Delete reference // Delete reference
if result.newCommitID == gitShortEmptySha { if result.NewCommitID == "" {
notify_service.SyncDeleteRef(ctx, m.Repo.MustOwner(ctx), m.Repo, result.refName) notify_service.SyncDeleteRef(ctx, m.Repo.MustOwner(ctx), m.Repo, result.RefName)
continue continue
} }
// Push commits // Push commits
oldCommitID, err := gitrepo.GetFullCommitID(ctx, repo, result.oldCommitID) oldCommitID, err := gitrepo.GetFullCommitID(ctx, repo, result.OldCommitID)
if err != nil { if err != nil {
log.Error("SyncMirrors [repo: %-v]: unable to get GetFullCommitID[%s]: %v", m.Repo, result.oldCommitID, err) log.Error("SyncMirrors [repo: %-v]: unable to get GetFullCommitID[%s]: %v", m.Repo, result.OldCommitID, err)
continue continue
} }
newCommitID, err := gitrepo.GetFullCommitID(ctx, repo, result.newCommitID) newCommitID, err := gitrepo.GetFullCommitID(ctx, repo, result.NewCommitID)
if err != nil { if err != nil {
log.Error("SyncMirrors [repo: %-v]: unable to get GetFullCommitID [%s]: %v", m.Repo, result.newCommitID, err) log.Error("SyncMirrors [repo: %-v]: unable to get GetFullCommitID [%s]: %v", m.Repo, result.NewCommitID, err)
continue continue
} }
commits, err := gitRepo.CommitsBetweenIDs(newCommitID, oldCommitID) commits, err := gitRepo.CommitsBetweenIDs(newCommitID, oldCommitID)
@@ -509,7 +385,7 @@ func SyncPullMirror(ctx context.Context, repoID int64) bool {
theCommits.CompareURL = m.Repo.ComposeCompareURL(oldCommitID, newCommitID) theCommits.CompareURL = m.Repo.ComposeCompareURL(oldCommitID, newCommitID)
notify_service.SyncPushCommits(ctx, m.Repo.MustOwner(ctx), m.Repo, &repo_module.PushUpdateOptions{ notify_service.SyncPushCommits(ctx, m.Repo.MustOwner(ctx), m.Repo, &repo_module.PushUpdateOptions{
RefFullName: result.refName, RefFullName: result.RefName,
OldCommitID: oldCommitID, OldCommitID: oldCommitID,
NewCommitID: newCommitID, NewCommitID: newCommitID,
}, theCommits) }, theCommits)
@@ -548,7 +424,7 @@ func SyncPullMirror(ctx context.Context, repoID int64) bool {
return true return true
} }
func checkAndUpdateEmptyRepository(ctx context.Context, m *repo_model.Mirror, results []*mirrorSyncResult) bool { func checkAndUpdateEmptyRepository(ctx context.Context, m *repo_model.Mirror, results []*repo_module.SyncResult) bool {
if !m.Repo.IsEmpty { if !m.Repo.IsEmpty {
return true return true
} }
@@ -562,11 +438,11 @@ func checkAndUpdateEmptyRepository(ctx context.Context, m *repo_model.Mirror, re
} }
firstName := "" firstName := ""
for _, result := range results { for _, result := range results {
if !result.refName.IsBranch() { if !result.RefName.IsBranch() {
continue continue
} }
name := result.refName.BranchName() name := result.RefName.BranchName()
if len(firstName) == 0 { if len(firstName) == 0 {
firstName = name firstName = name
} }
-56
View File
@@ -9,62 +9,6 @@ import (
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
) )
func Test_parseRemoteUpdateOutput(t *testing.T) {
output := `
* [new tag] v0.1.8 -> v0.1.8
* [new branch] master -> origin/master
- [deleted] (none) -> origin/test1
- [deleted] (none) -> tag1
+ f895a1e...957a993 test2 -> origin/test2 (forced update)
957a993..a87ba5f test3 -> origin/test3
* [new ref] refs/pull/26595/head -> refs/pull/26595/head
* [new ref] refs/pull/26595/merge -> refs/pull/26595/merge
e0639e38fb..6db2410489 refs/pull/25873/head -> refs/pull/25873/head
+ 1c97ebc746...976d27d52f refs/pull/25873/merge -> refs/pull/25873/merge (forced update)
`
results := parseRemoteUpdateOutput(output, "origin")
assert.Len(t, results, 10)
assert.Equal(t, "refs/tags/v0.1.8", results[0].refName.String())
assert.Equal(t, gitShortEmptySha, results[0].oldCommitID)
assert.Empty(t, results[0].newCommitID)
assert.Equal(t, "refs/heads/master", results[1].refName.String())
assert.Equal(t, gitShortEmptySha, results[1].oldCommitID)
assert.Empty(t, results[1].newCommitID)
assert.Equal(t, "refs/heads/test1", results[2].refName.String())
assert.Empty(t, results[2].oldCommitID)
assert.Equal(t, gitShortEmptySha, results[2].newCommitID)
assert.Equal(t, "refs/tags/tag1", results[3].refName.String())
assert.Empty(t, results[3].oldCommitID)
assert.Equal(t, gitShortEmptySha, results[3].newCommitID)
assert.Equal(t, "refs/heads/test2", results[4].refName.String())
assert.Equal(t, "f895a1e", results[4].oldCommitID)
assert.Equal(t, "957a993", results[4].newCommitID)
assert.Equal(t, "refs/heads/test3", results[5].refName.String())
assert.Equal(t, "957a993", results[5].oldCommitID)
assert.Equal(t, "a87ba5f", results[5].newCommitID)
assert.Equal(t, "refs/pull/26595/head", results[6].refName.String())
assert.Equal(t, gitShortEmptySha, results[6].oldCommitID)
assert.Empty(t, results[6].newCommitID)
assert.Equal(t, "refs/pull/26595/merge", results[7].refName.String())
assert.Equal(t, gitShortEmptySha, results[7].oldCommitID)
assert.Empty(t, results[7].newCommitID)
assert.Equal(t, "refs/pull/25873/head", results[8].refName.String())
assert.Equal(t, "e0639e38fb", results[8].oldCommitID)
assert.Equal(t, "6db2410489", results[8].newCommitID)
assert.Equal(t, "refs/pull/25873/merge", results[9].refName.String())
assert.Equal(t, "1c97ebc746", results[9].oldCommitID)
assert.Equal(t, "976d27d52f", results[9].newCommitID)
}
func Test_checkRecoverableSyncError(t *testing.T) { func Test_checkRecoverableSyncError(t *testing.T) {
cases := []struct { cases := []struct {
recoverable bool recoverable bool
+2 -2
View File
@@ -147,11 +147,11 @@ func adoptRepository(ctx context.Context, repo *repo_model.Repository, defaultBr
} }
defer gitRepo.Close() defer gitRepo.Close()
if _, err = repo_module.SyncRepoBranchesWithRepo(ctx, repo, gitRepo, 0); err != nil { if _, _, err = repo_module.SyncRepoBranchesWithRepo(ctx, repo, gitRepo, 0); err != nil {
return fmt.Errorf("SyncRepoBranchesWithRepo: %w", err) return fmt.Errorf("SyncRepoBranchesWithRepo: %w", err)
} }
if err = repo_module.SyncReleasesWithTags(ctx, repo, gitRepo); err != nil { if _, err = repo_module.SyncReleasesWithTags(ctx, repo, gitRepo); err != nil {
return fmt.Errorf("SyncReleasesWithTags: %w", err) return fmt.Errorf("SyncReleasesWithTags: %w", err)
} }
+2 -2
View File
@@ -177,10 +177,10 @@ func ForkRepository(ctx context.Context, doer, owner *user_model.User, opts Fork
} }
defer gitRepo.Close() defer gitRepo.Close()
if _, err = repo_module.SyncRepoBranchesWithRepo(ctx, repo, gitRepo, doer.ID); err != nil { if _, _, err = repo_module.SyncRepoBranchesWithRepo(ctx, repo, gitRepo, doer.ID); err != nil {
return nil, fmt.Errorf("SyncRepoBranchesWithRepo: %w", err) return nil, fmt.Errorf("SyncRepoBranchesWithRepo: %w", err)
} }
if err = repo_module.SyncReleasesWithTags(ctx, repo, gitRepo); err != nil { if _, err = repo_module.SyncReleasesWithTags(ctx, repo, gitRepo); err != nil {
return nil, fmt.Errorf("Sync releases from git tags failed: %v", err) return nil, fmt.Errorf("Sync releases from git tags failed: %v", err)
} }
+2 -2
View File
@@ -145,7 +145,7 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User,
} }
} }
if _, err := repo_module.SyncRepoBranchesWithRepo(ctx, repo, gitRepo, u.ID); err != nil { if _, _, err := repo_module.SyncRepoBranchesWithRepo(ctx, repo, gitRepo, u.ID); err != nil {
return repo, fmt.Errorf("SyncRepoBranchesWithRepo: %v", err) return repo, fmt.Errorf("SyncRepoBranchesWithRepo: %v", err)
} }
@@ -153,7 +153,7 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User,
// otherwise, the releases sync will be done out of this function // otherwise, the releases sync will be done out of this function
if !opts.Releases { if !opts.Releases {
repo.IsMirror = opts.Mirror repo.IsMirror = opts.Mirror
if err = repo_module.SyncReleasesWithTags(ctx, repo, gitRepo); err != nil { if _, err = repo_module.SyncReleasesWithTags(ctx, repo, gitRepo); err != nil {
log.Error("Failed to synchronize tags to releases for repository: %v", err) log.Error("Failed to synchronize tags to releases for repository: %v", err)
} }
} }
+3 -3
View File
@@ -56,11 +56,11 @@
{{$index := index .GetIssueInfos 0}} {{$index := index .GetIssueInfos 0}}
{{ctx.Locale.Tr "action.delete_branch" (.GetRepoLink ctx) .GetBranch (.ShortRepoPath ctx)}} {{ctx.Locale.Tr "action.delete_branch" (.GetRepoLink ctx) .GetBranch (.ShortRepoPath ctx)}}
{{else if .GetOpType.InActions "mirror_sync_push"}} {{else if .GetOpType.InActions "mirror_sync_push"}}
{{ctx.Locale.Tr "action.mirror_sync_push" (.GetRepoLink ctx) (.GetRefLink ctx) .GetBranch (.ShortRepoPath ctx)}} {{ctx.Locale.Tr "action.mirror_sync_push" (.GetRepoLink ctx) (.GetRefLink ctx) .RefName (.ShortRepoPath ctx)}}
{{else if .GetOpType.InActions "mirror_sync_create"}} {{else if .GetOpType.InActions "mirror_sync_create"}}
{{ctx.Locale.Tr "action.mirror_sync_create" (.GetRepoLink ctx) (.GetRefLink ctx) .GetBranch (.ShortRepoPath ctx)}} {{ctx.Locale.Tr "action.mirror_sync_create" (.GetRepoLink ctx) (.GetRefLink ctx) .RefName (.ShortRepoPath ctx)}}
{{else if .GetOpType.InActions "mirror_sync_delete"}} {{else if .GetOpType.InActions "mirror_sync_delete"}}
{{ctx.Locale.Tr "action.mirror_sync_delete" (.GetRepoLink ctx) .GetBranch (.ShortRepoPath ctx)}} {{ctx.Locale.Tr "action.mirror_sync_delete" (.GetRepoLink ctx) .RefName (.ShortRepoPath ctx)}}
{{else if .GetOpType.InActions "approve_pull_request"}} {{else if .GetOpType.InActions "approve_pull_request"}}
{{$index := index .GetIssueInfos 0}} {{$index := index .GetIssueInfos 0}}
{{ctx.Locale.Tr "action.approve_pull_request" (printf "%s/pulls/%s" (.GetRepoLink ctx) $index) $index (.ShortRepoPath ctx)}} {{ctx.Locale.Tr "action.approve_pull_request" (printf "%s/pulls/%s" (.GetRepoLink ctx) $index) $index (.ShortRepoPath ctx)}}