feat: 知识治理平台(含已验证发布/回滚契约)

全栈知识治理平台首次入库(P1-P10 + 发布链路模拟验证闭环):

- 治理核心:源目录/变更集/人工审核/发布批次/发布执行(fail-closed 闸口)
- 测试通道:WeKnora 测试 KB 上传/轮询/验收;poll 完成时写入
  parser_generation/index_generation,修复 manual/seed 路径发布闸口永久拦截
- 发布适配器 v071(本地 source=测试角色 / 云端 target=生产角色 模拟验证通过):
  * ES 导出/导入 mapping 自适应(.keyword 子字段、可选 tenant_id)
  * OSS locator 支持嵌套 scheme;chunks.seq_id 由目标序列分配
  * bundle 归档同身份重试原子替换;verifier 按 bundle chunk_type 计数
  * 回滚 job payload 支持 prior_execution_id(strict 解码)
- 三通道发布+回滚演练实测:publish 4/4 verified,检索命中,
  rollback 4/4 rolled_back,目标端 PG is_enabled=false、ES is_enabled=false
- 前端 A·Data-console 风格全站统一,120 用例通过
- 门禁:go vet / go test 全包通过,gofmt 干净
This commit is contained in:
2026-09-13 12:02:58 +08:00
committed by 姚章浩
parent ed48c4ceda
commit dcdc8d1ddd
410 changed files with 56661 additions and 1 deletions
@@ -0,0 +1,191 @@
package catalogrefresh
import (
"bufio"
"bytes"
"context"
"encoding/json"
"errors"
"io"
"os/exec"
"path/filepath"
"strings"
"time"
"unicode"
"knowledge-governance/internal/jobs"
)
type Request struct {
ConnectionID string `json:"connection_id"`
SourceType string `json:"source_type"`
SourceName string `json:"source_name"`
Stage string `json:"stage"`
Progress int `json:"progress"`
Message string `json:"message"`
Processed int `json:"processed"`
Total int `json:"total"`
}
type Progress struct {
Stage string `json:"stage"`
Progress int `json:"progress"`
Message string `json:"message"`
Processed int `json:"processed"`
Total int `json:"total"`
SourceName string `json:"source_name,omitempty"`
CurrentDirectory string `json:"current_directory,omitempty"`
ScannedDirectories int `json:"scanned_directories,omitempty"`
DiscoveredDirectories int `json:"discovered_directories,omitempty"`
ScannedFiles int `json:"scanned_files,omitempty"`
EstimatedTotalFiles int `json:"estimated_total_files,omitempty"`
TotalIsEstimate bool `json:"total_is_estimate,omitempty"`
}
type Store interface {
UpdateCatalogRefreshProgress(context.Context, string, string, Progress, time.Time) error
}
var progressForbiddenSubstrings = []string{"http://", "https://", "token", "secret", "password", "authorization", "bearer", "cookie", "/run/source-secrets", "api-key", "apikey"}
type Runner interface {
Run(context.Context, Request, func(Progress) error) error
}
type Config struct {
Store Store
Runner Runner
Now func() time.Time
}
type Handler struct {
store Store
runner Runner
now func() time.Time
}
func NewHandler(config Config) (*Handler, error) {
if config.Store == nil || config.Runner == nil {
return nil, errors.New("catalog refresh dependencies are required")
}
if config.Now == nil {
config.Now = time.Now
}
return &Handler{store: config.Store, runner: config.Runner, now: config.Now}, nil
}
func (handler *Handler) Handle(ctx context.Context, job jobs.Job) error {
var request Request
decoder := json.NewDecoder(bytes.NewReader(job.Payload))
decoder.DisallowUnknownFields()
if err := decoder.Decode(&request); err != nil || request.ConnectionID == "" || request.SourceName == "" || (request.SourceType != "pp" && request.SourceType != "dingtalk" && request.SourceType != "alist") || job.LeaseOwner == "" {
return errors.New("catalog_refresh_invalid_payload")
}
var last Progress
return handler.runner.Run(ctx, request, func(progress Progress) error {
progress = clampProgress(last, progress)
if !validProgress(progress) {
return errors.New("catalog_refresh_invalid_progress")
}
last = progress
return handler.store.UpdateCatalogRefreshProgress(ctx, job.ID, job.LeaseOwner, progress, handler.now().UTC())
})
}
func validProgress(progress Progress) bool {
if progress.Progress < 0 || progress.Progress > 100 || progress.Processed < 0 || progress.Total < 0 ||
progress.ScannedDirectories < 0 || progress.DiscoveredDirectories < 0 || progress.ScannedFiles < 0 || progress.EstimatedTotalFiles < 0 ||
strings.TrimSpace(progress.Stage) == "" || len(progress.Stage) > 32 ||
strings.TrimSpace(progress.Message) == "" || len(progress.Message) > 300 ||
len(progress.CurrentDirectory) > 1024 || len(progress.SourceName) > 255 {
return false
}
for _, text := range []string{progress.Stage, progress.Message, progress.CurrentDirectory, progress.SourceName} {
for _, r := range text {
if unicode.IsControl(r) && r != ' ' {
return false
}
}
lowered := strings.ToLower(text)
for _, forbidden := range progressForbiddenSubstrings {
if strings.Contains(lowered, forbidden) {
return false
}
}
}
return true
}
// clampProgress enforces that display progress never regresses when discovered
// totals shift mid-scan (BFS estimates grow), and never clamps an explicit
// failure snapshot.
func clampProgress(previous, next Progress) Progress {
if next.Stage == "failed" {
return next
}
if next.Total < previous.Total && previous.Total >= next.Processed {
next.Total = previous.Total
}
if next.Total > 0 {
estimated := next.Processed * 100 / next.Total
if next.Progress < previous.Progress && previous.Progress <= estimated {
next.Progress = previous.Progress
}
}
if next.ScannedDirectories < previous.ScannedDirectories {
next.ScannedDirectories = previous.ScannedDirectories
}
if next.ScannedFiles < previous.ScannedFiles {
next.ScannedFiles = previous.ScannedFiles
}
if next.DiscoveredDirectories < previous.DiscoveredDirectories {
next.DiscoveredDirectories = previous.DiscoveredDirectories
}
if next.EstimatedTotalFiles < previous.EstimatedTotalFiles {
next.EstimatedTotalFiles = previous.EstimatedTotalFiles
}
return next
}
type CommandRunner struct {
PythonPath string
ScriptPath string
}
func NewCommandRunner(pythonPath, scriptPath string) (*CommandRunner, error) {
if !filepath.IsAbs(pythonPath) || !filepath.IsAbs(scriptPath) {
return nil, errors.New("catalog refresh command paths must be absolute")
}
return &CommandRunner{PythonPath: pythonPath, ScriptPath: scriptPath}, nil
}
func (runner *CommandRunner) Run(ctx context.Context, request Request, report func(Progress) error) error {
command := exec.CommandContext(ctx, runner.PythonPath, runner.ScriptPath, "--connection-id", request.ConnectionID, "--source-type", request.SourceType, "--source-name", request.SourceName)
stdout, err := command.StdoutPipe()
if err != nil {
return errors.New("catalog_refresh_start_failed")
}
command.Stderr = io.Discard
if err = command.Start(); err != nil {
return errors.New("catalog_refresh_start_failed")
}
scanner := bufio.NewScanner(stdout)
scanner.Buffer(make([]byte, 4096), 1<<20)
for scanner.Scan() {
decoder := json.NewDecoder(bytes.NewReader(scanner.Bytes()))
decoder.DisallowUnknownFields()
var progress Progress
if decoder.Decode(&progress) != nil || !validProgress(progress) {
_ = command.Process.Kill()
_ = command.Wait()
return errors.New("catalog_refresh_invalid_output")
}
if err = report(progress); err != nil {
_ = command.Process.Kill()
_ = command.Wait()
return err
}
}
if scanner.Err() != nil {
_ = command.Process.Kill()
_ = command.Wait()
return errors.New("catalog_refresh_output_failed")
}
if err = command.Wait(); err != nil {
return errors.New("catalog_refresh_failed")
}
return nil
}
@@ -0,0 +1,126 @@
package catalogrefresh
import (
"context"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"time"
"knowledge-governance/internal/jobs"
)
type fakeStore struct{ updates []Progress }
func (s *fakeStore) UpdateCatalogRefreshProgress(_ context.Context, _ string, _ string, p Progress, _ time.Time) error {
s.updates = append(s.updates, p)
return nil
}
func TestHandlerClampsMonotonicProgressAndKeepsFailuresVisible(t *testing.T) {
store := &fakeStore{}
handler, err := NewHandler(Config{Store: store, Runner: fakeRunner{}, Now: func() time.Time { return time.Now() }})
if err != nil {
t.Fatal(err)
}
runner := func(ctx context.Context, _ Request, report func(Progress) error) error {
_ = report(Progress{Stage: "scanning", Progress: 40, Message: "扫描中", Processed: 60, Total: 100})
// Discovered more directories than estimated: total must not shrink, percent must not regress.
_ = report(Progress{Stage: "scanning", Progress: 30, Message: "发现更多目录", Processed: 70, Total: 50})
return report(Progress{Stage: "failed", Progress: 0, Message: "来源不可用"})
}
custom := *handler
custom.runner = runnerFunc(runner)
job := jobs.Job{ID: "job-1", LeaseOwner: "worker-1", Payload: []byte(`{"connection_id":"conn-1","source_type":"alist","source_name":"国内工业","stage":"queued","progress":0,"message":"等待后台刷新","processed":0,"total":0}`)}
if err = custom.Handle(context.Background(), job); err != nil {
t.Fatal(err)
}
if len(store.updates) != 3 {
t.Fatalf("updates=%#v", store.updates)
}
if store.updates[1].Progress < 40 || store.updates[1].Total < 70 {
t.Fatalf("regression not clamped: %#v", store.updates[1])
}
if store.updates[2].Stage != "failed" || store.updates[2].Progress != 0 {
t.Fatalf("failure must keep its own state: %#v", store.updates[2])
}
}
func TestValidProgressRejectsControlCharsURLsAndCredentials(t *testing.T) {
base := Progress{Stage: "scanning", Progress: 50, Message: "正在扫描", Processed: 10, Total: 20}
if !validProgress(base) {
t.Fatal("baseline rejected")
}
for name, mutate := range map[string]func(*Progress){
"control": func(p *Progress) { p.CurrentDirectory = "/产品\a资料" },
"url": func(p *Progress) { p.CurrentDirectory = "https://evil.invalid/x" },
"url-message": func(p *Progress) { p.Message = "访问 https://evil.invalid 失败" },
"token": func(p *Progress) { p.Message = "token=abc123 已过期" },
"too-long": func(p *Progress) { p.CurrentDirectory = strings.Repeat("目", 1100) },
"negative": func(p *Progress) { p.ScannedFiles = -1 },
"secret-dir": func(p *Progress) { p.CurrentDirectory = "/run/source-secrets/pp" },
} {
item := base
mutate(&item)
if validProgress(item) {
t.Errorf("%s accepted", name)
}
}
}
func TestCommandRunnerRejectsUnknownProgressFields(t *testing.T) {
python, err := exec.LookPath("python3")
if err != nil {
t.Skip("python3 unavailable")
}
runner, err := NewCommandRunner(python, writeFakeProgressScript(t))
if err != nil {
t.Fatal(err)
}
err = runner.Run(context.Background(), Request{ConnectionID: "c", SourceType: "pp", SourceName: "n"}, func(Progress) error { return nil })
if err == nil || err.Error() != "catalog_refresh_invalid_output" {
t.Fatalf("unknown field must fail: %v", err)
}
}
func writeFakeProgressScript(t *testing.T) string {
t.Helper()
directory := t.TempDir()
path := filepath.Join(directory, "progress.py")
body := "import json\nprint(json.dumps({'stage':'scanning','progress':10,'message':'ok','processed':1,'total':2,'unexpected_field':'leak'}),flush=True)\n"
if err := os.WriteFile(path, []byte(body), 0o644); err != nil {
t.Fatal(err)
}
return path
}
type runnerFunc func(context.Context, Request, func(Progress) error) error
func (f runnerFunc) Run(ctx context.Context, request Request, report func(Progress) error) error {
return f(ctx, request, report)
}
type fakeRunner struct{}
func (fakeRunner) Run(_ context.Context, _ Request, report func(Progress) error) error {
if err := report(Progress{Stage: "scanning", Progress: 50, Message: "正在扫描"}); err != nil {
return err
}
return report(Progress{Stage: "completed", Progress: 100, Message: "完成"})
}
func TestHandlerRunsLiveRefreshAndPersistsProgress(t *testing.T) {
store := &fakeStore{}
handler, err := NewHandler(Config{Store: store, Runner: fakeRunner{}, Now: func() time.Time { return time.Now() }})
if err != nil {
t.Fatal(err)
}
job := jobs.Job{ID: "job-1", LeaseOwner: "worker-1", Payload: []byte(`{"connection_id":"conn-1","source_type":"pp","source_name":"PP资料下载中心","stage":"queued","progress":0,"message":"等待后台刷新","processed":0,"total":0}`)}
if err = handler.Handle(context.Background(), job); err != nil {
t.Fatal(err)
}
if len(store.updates) != 2 || store.updates[0].Stage != "scanning" || store.updates[1].Progress != 100 {
t.Fatalf("updates=%#v", store.updates)
}
}