Files
knowledge-governance/internal/versionhistory/sync.go
T
姚章浩 4cb5110845
build / build (push) Successful in 1m1s
fix(version-history): 页面链接改走公网 origin(抓取仍走内网)+ tag 补链接
现象(主人反馈):点「查看提交」打开是空白页。原因:链接用的是**抓取用的内网地址**
(`http://192.168.50.223:3000/...`),内网外的浏览器到不了;而公网地址
`https://gitea.zheromiku.top/...` 实测 200。另外 tag 证据里**根本没有 url**,
页面渲染出空 `href`,点下去像"没反应"。

修复:

- 新增 `VERSION_HISTORY_LINK_ORIGIN`(页面链接 origin,公网;留空 = 与抓取 origin 相同):
  抓取仍匿名走内网白名单地址,**给人点的链接**用公网 origin 拼 —— 两者本就不该共用一个地址;
  校验必须是干净的 http(s) 绝对地址。
- tag 证据补 `url`(`<link origin>/<repo>/src/tag/<name>`),并在前端**没有链接就不渲染**该按钮。
- 测试:客户端(LinkOrigin 生效 / 未配时退回 / 非法拒绝)、规则(tag 链接写入证据 / 无基址不编链接)、
  前端(有链接才渲染「查看 tag」)。

验证:make verify 15/15;真实链路重同步后库里 commit_url = `https://gitea.zheromiku.top/...`(200)、
release_tags 带 url。
2026-10-14 11:06:13 +08:00

283 lines
9.3 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package versionhistory
import (
"context"
"errors"
"fmt"
"sort"
"strings"
"time"
"knowledge-governance/internal/governance"
)
// 同步作业的错误码:页面与同步记录直接展示,**不暴露内部细节**。
const (
ErrorCodeUnreachable = "gitea_unreachable"
ErrorCodeStatus = "gitea_status"
ErrorCodeTooLarge = "gitea_response_too_large"
ErrorCodeNotFound = "gitea_not_found"
ErrorCodeParse = "gitea_parse"
ErrorCodeRedirect = "gitea_redirect"
ErrorCodeCountMismatch = "count_mismatch"
ErrorCodeIncompletePages = "incomplete_history"
ErrorCodeUnexpected = "unexpected"
)
// ErrorKind 是客户端错误的机器可读类别(同步器据此给 error_code,不去猜字符串)。
type ErrorKind string
const (
KindUnreachable ErrorKind = "unreachable"
KindStatus ErrorKind = "status"
KindTooLarge ErrorKind = "too_large"
KindNotFound ErrorKind = "not_found"
KindParse ErrorKind = "parse"
KindRedirect ErrorKind = "redirect"
)
// GiteaError 携带类别与 HTTP 状态码。
type GiteaError struct {
Kind ErrorKind
Status int
Message string
Wrapped error
}
func (e *GiteaError) Error() string {
if e.Status > 0 {
return fmt.Sprintf("%s(HTTP %d)", e.Message, e.Status)
}
return e.Message
}
func (e *GiteaError) Unwrap() error { return e.Wrapped }
func newGiteaError(kind ErrorKind, status int, message string, wrapped error) *GiteaError {
return &GiteaError{Kind: kind, Status: status, Message: message, Wrapped: wrapped}
}
// errorCode 把任意错误翻成同步错误码;未识别的错误归为 unexpected(不静默吞掉)。
func errorCode(err error) string {
var gitea *GiteaError
if errors.As(err, &gitea) {
switch gitea.Kind {
case KindUnreachable:
return ErrorCodeUnreachable
case KindStatus:
if gitea.Status > 0 {
return fmt.Sprintf("%s_%d", ErrorCodeStatus, gitea.Status)
}
return ErrorCodeStatus
case KindTooLarge:
return ErrorCodeTooLarge
case KindNotFound:
return ErrorCodeNotFound
case KindParse:
return ErrorCodeParse
case KindRedirect:
return ErrorCodeRedirect
}
}
var typed interface{ ErrorCode() string }
if errors.As(err, &typed) {
return typed.ErrorCode()
}
return ErrorCodeUnexpected
}
// codeError 让同步器自己的失败也带机器可读码。
type codeError struct {
code string
message string
}
func (e codeError) Error() string { return e.message }
func (e codeError) ErrorCode() string { return e.code }
// JobType 是同步作业的类型名:HTTP 侧入队与 worker 侧注册必须用同一个值。
const JobType = "versionhistory.sync"
// Source 是版本记录同步需要的只读数据源。
type Source interface {
PageSize() int
CommitURLBase() string
ListCommits(ctx context.Context, page, limit int) ([]RawCommit, int, error)
ListTags(ctx context.Context) ([]RawTag, error)
// TagURLBase 是 tag 链接的基址(给人看的 origin,可能与抓取 origin 不同)。
TagURLBase() string
ReadFile(ctx context.Context, sha, path string) (string, error)
// BranchHeadSHA 读分支 HEAD;读不到不算致命(退回"提交列表里最新的那条")。
BranchHeadSHA(ctx context.Context, branch string) (string, error)
}
// SyncStore 是同步器需要的持久化能力。
type SyncStore interface {
BeginVersionHistorySync(ctx context.Context, repoKey, jobID, triggeredBy string, now time.Time) (int64, error)
SaveVersionHistorySnapshot(ctx context.Context, snapshotID int64, repoKey string,
records []governance.VersionHistoryCommit, expected int, complete bool, now time.Time) error
FinishVersionHistorySync(ctx context.Context, runID int64, outcome SyncOutcome, now time.Time) error
FailVersionHistorySync(ctx context.Context, runID int64, errorCode string, now time.Time) error
}
// SyncOutcome 是一次成功同步的结果(「同步成功」与「历史完整」分开表达)。
type SyncOutcome struct {
SourceHeadSHA string
FetchedCount int
ExpectedCount int
Complete bool
}
// SyncerConfig 装配同步器。
type SyncerConfig struct {
Source Source
Store SyncStore
RepoKey string
Changelog string // 提交内的变更记录文件路径,默认 CHANGELOG.md
Now func() time.Time
MaxPages int
}
// Syncer 抓取远端 main 可达的全部提交并换算版本记录。
type Syncer struct {
source Source
store SyncStore
repoKey string
changelog string
now func() time.Time
maxPages int
}
const syncerDefaultMaxPages = 200
// NewSyncer 校验依赖;缺一即拒绝装配(不允许"半接线"跑起来)。
func NewSyncer(config SyncerConfig) (*Syncer, error) {
if config.Source == nil || config.Store == nil {
return nil, errors.New("版本记录同步器:数据源与存储都必需")
}
if strings.TrimSpace(config.RepoKey) == "" {
return nil, errors.New("版本记录同步器:仓库标识必需")
}
changelog := config.Changelog
if changelog == "" {
changelog = "CHANGELOG.md"
}
now := config.Now
if now == nil {
now = time.Now
}
maxPages := config.MaxPages
if maxPages <= 0 {
maxPages = syncerDefaultMaxPages
}
return &Syncer{source: config.Source, store: config.Store, repoKey: config.RepoKey,
changelog: changelog, now: now, maxPages: maxPages}, nil
}
// Sync 跑一次同步。任何失败都会把这次 run 标成 failed 并带上错误码;
// **失败绝不切换对外快照**,页面继续看上一次成功的数据。
func (s *Syncer) Sync(ctx context.Context, jobID, triggeredBy string) (governance.VersionHistorySyncRun, error) {
now := s.now().UTC()
runID, err := s.store.BeginVersionHistorySync(ctx, s.repoKey, jobID, triggeredBy, now)
if err != nil {
return governance.VersionHistorySyncRun{}, fmt.Errorf("版本记录:开始同步失败:%w", err)
}
fail := func(cause error) (governance.VersionHistorySyncRun, error) {
code := errorCode(cause)
if markErr := s.store.FailVersionHistorySync(ctx, runID, code, s.now().UTC()); markErr != nil {
return governance.VersionHistorySyncRun{}, fmt.Errorf("版本记录:标记同步失败时又出错:%w", markErr)
}
return governance.VersionHistorySyncRun{ID: runID, Status: "failed", ErrorCode: code},
fmt.Errorf("版本记录:同步失败(%s):%w", code, cause)
}
commits, total, err := s.fetchAllCommits(ctx)
if err != nil {
return fail(err)
}
if total > 0 && len(commits) != total {
return fail(codeError{code: ErrorCodeCountMismatch,
message: fmt.Sprintf("版本记录:提交数与远端宣称不一致(抓到 %d,远端 X-Total-Count=%d)", len(commits), total)})
}
tags, err := s.source.ListTags(ctx)
if err != nil {
return fail(err)
}
derived := DeriveVersions(DeriveInput{RepoKey: s.repoKey, CommitURLBase: s.source.CommitURLBase(),
TagURLBase: s.source.TagURLBase(),
Commits: commits, Tags: tags, SnapshotID: runID,
Changelog: func(sha string) (string, error) {
return s.source.ReadFile(ctx, sha, s.changelog)
}})
if err := s.store.SaveVersionHistorySnapshot(ctx, runID, s.repoKey, derived.Records, total, true, s.now().UTC()); err != nil {
return fail(err)
}
sourceHead := ""
if head, err := s.source.BranchHeadSHA(ctx, "main"); err == nil {
sourceHead = head
} else {
sourceHead = headSHA(commits) // 读分支失败不致命:退回提交列表最新一条
}
outcome := SyncOutcome{SourceHeadSHA: sourceHead, FetchedCount: len(commits),
ExpectedCount: total, Complete: derived.GraphComplete && (total == 0 || len(commits) == total)}
if err := s.store.FinishVersionHistorySync(ctx, runID, outcome, s.now().UTC()); err != nil {
return fail(err)
}
return governance.VersionHistorySyncRun{ID: runID, Status: "completed", SourceHeadSHA: outcome.SourceHeadSHA,
FetchedCount: outcome.FetchedCount, ExpectedCount: &total, Complete: outcome.Complete,
LastSuccessAt: s.now().UTC(), StartedAt: now}, nil
}
// fetchAllCommits 逐页抓取并按 SHA 去重;短页即终止(也防住服务端忽略 page 参数)。
func (s *Syncer) fetchAllCommits(ctx context.Context) ([]RawCommit, int, error) {
pageSize := s.source.PageSize()
if pageSize <= 0 {
pageSize = giteaDefaultPageSize
}
seen := map[string]bool{}
commits := make([]RawCommit, 0, pageSize)
total := 0
for page := 1; page <= s.maxPages; page++ {
pageCommits, pageTotal, err := s.source.ListCommits(ctx, page, pageSize)
if err != nil {
return nil, 0, err
}
if pageTotal > 0 {
total = pageTotal
}
for _, commit := range pageCommits {
if seen[commit.SHA] {
continue
}
seen[commit.SHA] = true
commits = append(commits, commit)
}
if len(pageCommits) < pageSize {
return commits, total, nil
}
if total > 0 && len(commits) >= total {
return commits, total, nil
}
}
return nil, 0, codeError{code: ErrorCodeIncompletePages,
message: fmt.Sprintf("版本记录:分页超过 %d 页仍未取完(疑似服务端分页异常)", s.maxPages)}
}
// headSHA 取远端 main 的 HEAD(提交列表按时间倒序,最新一条即 HEAD;取不到就空)。
func headSHA(commits []RawCommit) string {
sorted := make([]RawCommit, len(commits))
copy(sorted, commits)
sort.Slice(sorted, func(i, j int) bool {
if !sorted[i].CommittedAt.Equal(sorted[j].CommittedAt) {
return sorted[i].CommittedAt.After(sorted[j].CommittedAt)
}
return sorted[i].SHA > sorted[j].SHA
})
if len(sorted) == 0 {
return ""
}
return sorted[0].SHA
}