姚章浩
013ce4a24e
build / build (push) Successful in 1m3s
收录 main、注释开发快照和完整运行 SHA；开发说明绑定原提交，补记 0.1.1–0.1.25。 进行中同步按仓库合并，完成后重新执行；启动入队，部署脚本读回版本证据。 开发快照独立于正式标签与 Release，既有冲突标签不移动。 已运行：整仓 make verify，Go 673 通过、前端 274 通过，92 个已声明可选跳过，必需跳过 0。 隔离 PostgreSQL 重复同步队列用例及 -race 通过；未更新现有 8088、未修改生产问答。
320 lines
10 KiB
Go
320 lines
10 KiB
Go
package versionhistory
|
||
|
||
import (
|
||
"context"
|
||
"errors"
|
||
"fmt"
|
||
"sort"
|
||
"strings"
|
||
"time"
|
||
|
||
"knowledge-governance/internal/governance"
|
||
)
|
||
|
||
// 同步作业的错误码：页面与同步记录直接展示，**不暴露内部细节**。
|
||
const (
|
||
ErrorCodeUnreachable = "gitea_unreachable"
|
||
ErrorCodeStatus = "gitea_status"
|
||
ErrorCodeTooLarge = "gitea_response_too_large"
|
||
ErrorCodeNotFound = "gitea_not_found"
|
||
ErrorCodeParse = "gitea_parse"
|
||
ErrorCodeRedirect = "gitea_redirect"
|
||
ErrorCodeCountMismatch = "count_mismatch"
|
||
ErrorCodeIncompletePages = "incomplete_history"
|
||
ErrorCodeUnexpected = "unexpected"
|
||
)
|
||
|
||
// ErrorKind 是客户端错误的机器可读类别（同步器据此给 error_code，不去猜字符串）。
|
||
type ErrorKind string
|
||
|
||
const (
|
||
KindUnreachable ErrorKind = "unreachable"
|
||
KindStatus ErrorKind = "status"
|
||
KindTooLarge ErrorKind = "too_large"
|
||
KindNotFound ErrorKind = "not_found"
|
||
KindParse ErrorKind = "parse"
|
||
KindRedirect ErrorKind = "redirect"
|
||
)
|
||
|
||
// GiteaError 携带类别与 HTTP 状态码。
|
||
type GiteaError struct {
|
||
Kind ErrorKind
|
||
Status int
|
||
Message string
|
||
Wrapped error
|
||
}
|
||
|
||
func (e *GiteaError) Error() string {
|
||
if e.Status > 0 {
|
||
return fmt.Sprintf("%s（HTTP %d）", e.Message, e.Status)
|
||
}
|
||
return e.Message
|
||
}
|
||
|
||
func (e *GiteaError) Unwrap() error { return e.Wrapped }
|
||
|
||
func newGiteaError(kind ErrorKind, status int, message string, wrapped error) *GiteaError {
|
||
return &GiteaError{Kind: kind, Status: status, Message: message, Wrapped: wrapped}
|
||
}
|
||
|
||
// errorCode 把任意错误翻成同步错误码；未识别的错误归为 unexpected（不静默吞掉）。
|
||
func errorCode(err error) string {
|
||
var gitea *GiteaError
|
||
if errors.As(err, &gitea) {
|
||
switch gitea.Kind {
|
||
case KindUnreachable:
|
||
return ErrorCodeUnreachable
|
||
case KindStatus:
|
||
if gitea.Status > 0 {
|
||
return fmt.Sprintf("%s_%d", ErrorCodeStatus, gitea.Status)
|
||
}
|
||
return ErrorCodeStatus
|
||
case KindTooLarge:
|
||
return ErrorCodeTooLarge
|
||
case KindNotFound:
|
||
return ErrorCodeNotFound
|
||
case KindParse:
|
||
return ErrorCodeParse
|
||
case KindRedirect:
|
||
return ErrorCodeRedirect
|
||
}
|
||
}
|
||
var typed interface{ ErrorCode() string }
|
||
if errors.As(err, &typed) {
|
||
return typed.ErrorCode()
|
||
}
|
||
return ErrorCodeUnexpected
|
||
}
|
||
|
||
// codeError 让同步器自己的失败也带机器可读码。
|
||
type codeError struct {
|
||
code string
|
||
message string
|
||
}
|
||
|
||
func (e codeError) Error() string { return e.message }
|
||
func (e codeError) ErrorCode() string { return e.code }
|
||
|
||
// JobType 是同步作业的类型名：HTTP 侧入队与 worker 侧注册必须用同一个值。
|
||
const JobType = "versionhistory.sync"
|
||
|
||
// Source 是版本记录同步需要的只读数据源。
|
||
type Source interface {
|
||
PageSize() int
|
||
CommitURLBase() string
|
||
ListCommits(ctx context.Context, page, limit int) ([]RawCommit, int, error)
|
||
ListTags(ctx context.Context) ([]RawTag, error)
|
||
// TagURLBase 是 tag 链接的基址（给人看的 origin，可能与抓取 origin 不同）。
|
||
TagURLBase() string
|
||
ReadFile(ctx context.Context, sha, path string) (string, error)
|
||
// BranchHeadSHA 读分支 HEAD；读不到不算致命（退回"提交列表里最新的那条"）。
|
||
BranchHeadSHA(ctx context.Context, branch string) (string, error)
|
||
}
|
||
|
||
// SyncStore 是同步器需要的持久化能力。
|
||
type SyncStore interface {
|
||
BeginVersionHistorySync(ctx context.Context, repoKey, jobID, triggeredBy string, now time.Time) (int64, error)
|
||
SaveVersionHistorySnapshot(ctx context.Context, snapshotID int64, repoKey string,
|
||
records []governance.VersionHistoryCommit, expected int, complete bool, now time.Time) error
|
||
FinishVersionHistorySync(ctx context.Context, runID int64, outcome SyncOutcome, now time.Time) error
|
||
FailVersionHistorySync(ctx context.Context, runID int64, errorCode string, now time.Time) error
|
||
}
|
||
|
||
// SyncOutcome 是一次成功同步的结果（「同步成功」与「历史完整」分开表达）。
|
||
type SyncOutcome struct {
|
||
SourceHeadSHA string
|
||
FetchedCount int
|
||
ExpectedCount int
|
||
Complete bool
|
||
}
|
||
|
||
// SyncerConfig 装配同步器。
|
||
type SyncerConfig struct {
|
||
Source Source
|
||
Store SyncStore
|
||
RepoKey string
|
||
Changelog string // 提交内的变更记录文件路径，默认 CHANGELOG.md
|
||
Now func() time.Time
|
||
MaxPages int
|
||
TargetSHAs []string
|
||
}
|
||
|
||
// Syncer 抓取远端 main 可达的全部提交并换算版本记录。
|
||
type Syncer struct {
|
||
source Source
|
||
store SyncStore
|
||
repoKey string
|
||
changelog string
|
||
now func() time.Time
|
||
maxPages int
|
||
targetSHAs []string
|
||
}
|
||
|
||
const syncerDefaultMaxPages = 200
|
||
|
||
// NewSyncer 校验依赖；缺一即拒绝装配（不允许"半接线"跑起来）。
|
||
func NewSyncer(config SyncerConfig) (*Syncer, error) {
|
||
if config.Source == nil || config.Store == nil {
|
||
return nil, errors.New("版本记录同步器：数据源与存储都必需")
|
||
}
|
||
if strings.TrimSpace(config.RepoKey) == "" {
|
||
return nil, errors.New("版本记录同步器：仓库标识必需")
|
||
}
|
||
changelog := config.Changelog
|
||
if changelog == "" {
|
||
changelog = "CHANGELOG.md"
|
||
}
|
||
now := config.Now
|
||
if now == nil {
|
||
now = time.Now
|
||
}
|
||
maxPages := config.MaxPages
|
||
if maxPages <= 0 {
|
||
maxPages = syncerDefaultMaxPages
|
||
}
|
||
for _, sha := range config.TargetSHAs {
|
||
if !fullSHA.MatchString(sha) {
|
||
return nil, errors.New("版本记录：运行身份必须是完整 SHA")
|
||
}
|
||
}
|
||
return &Syncer{targetSHAs: append([]string{}, config.TargetSHAs...), source: config.Source, store: config.Store, repoKey: config.RepoKey,
|
||
changelog: changelog, now: now, maxPages: maxPages}, nil
|
||
}
|
||
|
||
// Sync 跑一次同步。任何失败都会把这次 run 标成 failed 并带上错误码；
|
||
// **失败绝不切换对外快照**，页面继续看上一次成功的数据。
|
||
func (s *Syncer) Sync(ctx context.Context, jobID, triggeredBy string) (governance.VersionHistorySyncRun, error) {
|
||
now := s.now().UTC()
|
||
runID, err := s.store.BeginVersionHistorySync(ctx, s.repoKey, jobID, triggeredBy, now)
|
||
if err != nil {
|
||
return governance.VersionHistorySyncRun{}, fmt.Errorf("版本记录：开始同步失败：%w", err)
|
||
}
|
||
fail := func(cause error) (governance.VersionHistorySyncRun, error) {
|
||
code := errorCode(cause)
|
||
if markErr := s.store.FailVersionHistorySync(ctx, runID, code, s.now().UTC()); markErr != nil {
|
||
return governance.VersionHistorySyncRun{}, fmt.Errorf("版本记录：标记同步失败时又出错：%w", markErr)
|
||
}
|
||
return governance.VersionHistorySyncRun{ID: runID, Status: "failed", ErrorCode: code},
|
||
fmt.Errorf("版本记录：同步失败（%s）：%w", code, cause)
|
||
}
|
||
|
||
commits, total, err := s.fetchAllCommits(ctx)
|
||
if err != nil {
|
||
return fail(err)
|
||
}
|
||
if total > 0 && len(commits) != total {
|
||
return fail(codeError{code: ErrorCodeCountMismatch,
|
||
message: fmt.Sprintf("版本记录：提交数与远端宣称不一致（抓到 %d，远端 X-Total-Count=%d）", len(commits), total)})
|
||
}
|
||
tags, err := s.source.ListTags(ctx)
|
||
if err != nil {
|
||
return fail(err)
|
||
}
|
||
|
||
commits, err = s.includeDeliveredCommits(ctx, commits, tags)
|
||
if err != nil {
|
||
return fail(err)
|
||
}
|
||
total = len(commits)
|
||
deliveryArchive := ""
|
||
if len(s.targetSHAs) > 0 {
|
||
deliveryArchive, err = s.source.ReadFile(ctx, s.targetSHAs[0], s.changelog)
|
||
if err != nil {
|
||
return fail(err)
|
||
}
|
||
}
|
||
derived := DeriveVersions(DeriveInput{RepoKey: s.repoKey, CommitURLBase: s.source.CommitURLBase(),
|
||
TagURLBase: s.source.TagURLBase(),
|
||
DeliveryArchive: deliveryArchive,
|
||
Commits: commits, Tags: tags, SnapshotID: runID,
|
||
Changelog: func(sha string) (string, error) {
|
||
return s.source.ReadFile(ctx, sha, s.changelog)
|
||
}})
|
||
|
||
if err := s.store.SaveVersionHistorySnapshot(ctx, runID, s.repoKey, derived.Records, total, true, s.now().UTC()); err != nil {
|
||
return fail(err)
|
||
}
|
||
sourceHead := ""
|
||
if head, err := s.source.BranchHeadSHA(ctx, "main"); err == nil {
|
||
sourceHead = head
|
||
} else {
|
||
sourceHead = headSHA(commits) // 读分支失败不致命：退回提交列表最新一条
|
||
}
|
||
if len(s.targetSHAs) > 0 {
|
||
sourceHead = s.targetSHAs[0]
|
||
}
|
||
outcome := SyncOutcome{SourceHeadSHA: sourceHead, FetchedCount: len(commits),
|
||
ExpectedCount: total, Complete: derived.GraphComplete && (total == 0 || len(commits) == total)}
|
||
if err := s.store.FinishVersionHistorySync(ctx, runID, outcome, s.now().UTC()); err != nil {
|
||
return fail(err)
|
||
}
|
||
return governance.VersionHistorySyncRun{ID: runID, Status: "completed", SourceHeadSHA: outcome.SourceHeadSHA,
|
||
FetchedCount: outcome.FetchedCount, ExpectedCount: &total, Complete: outcome.Complete,
|
||
LastSuccessAt: s.now().UTC(), StartedAt: now}, nil
|
||
}
|
||
|
||
// fetchAllCommits 逐页抓取并按 SHA 去重；短页即终止（也防住服务端忽略 page 参数）。
|
||
func (s *Syncer) fetchAllCommits(ctx context.Context) ([]RawCommit, int, error) {
|
||
return s.fetchCommitsAt(ctx, "")
|
||
}
|
||
func (s *Syncer) fetchCommitsAt(ctx context.Context, ref string) ([]RawCommit, int, error) {
|
||
pageSize := s.source.PageSize()
|
||
if pageSize <= 0 {
|
||
pageSize = giteaDefaultPageSize
|
||
}
|
||
seen := map[string]bool{}
|
||
commits := make([]RawCommit, 0, pageSize)
|
||
total := 0
|
||
for page := 1; page <= s.maxPages; page++ {
|
||
var pageCommits []RawCommit
|
||
var pageTotal int
|
||
var err error
|
||
if ref == "" {
|
||
pageCommits, pageTotal, err = s.source.ListCommits(ctx, page, pageSize)
|
||
} else if source, ok := s.source.(interface {
|
||
ListCommitsAtRef(context.Context, string, int, int) ([]RawCommit, int, error)
|
||
}); ok {
|
||
pageCommits, pageTotal, err = source.ListCommitsAtRef(ctx, ref, page, pageSize)
|
||
} else {
|
||
return nil, 0, errors.New("版本来源不支持开发快照历史")
|
||
}
|
||
if err != nil {
|
||
return nil, 0, err
|
||
}
|
||
if pageTotal > 0 {
|
||
total = pageTotal
|
||
}
|
||
for _, commit := range pageCommits {
|
||
if seen[commit.SHA] {
|
||
continue
|
||
}
|
||
seen[commit.SHA] = true
|
||
commits = append(commits, commit)
|
||
}
|
||
if len(pageCommits) < pageSize {
|
||
return commits, total, nil
|
||
}
|
||
if total > 0 && len(commits) >= total {
|
||
return commits, total, nil
|
||
}
|
||
}
|
||
return nil, 0, codeError{code: ErrorCodeIncompletePages,
|
||
message: fmt.Sprintf("版本记录：分页超过 %d 页仍未取完（疑似服务端分页异常）", s.maxPages)}
|
||
}
|
||
|
||
// headSHA 取远端 main 的 HEAD（提交列表按时间倒序，最新一条即 HEAD；取不到就空）。
|
||
func headSHA(commits []RawCommit) string {
|
||
sorted := make([]RawCommit, len(commits))
|
||
copy(sorted, commits)
|
||
sort.Slice(sorted, func(i, j int) bool {
|
||
if !sorted[i].CommittedAt.Equal(sorted[j].CommittedAt) {
|
||
return sorted[i].CommittedAt.After(sorted[j].CommittedAt)
|
||
}
|
||
return sorted[i].SHA > sorted[j].SHA
|
||
})
|
||
if len(sorted) == 0 {
|
||
return ""
|
||
}
|
||
return sorted[0].SHA
|
||
}
|