feat: 知识治理平台(含已验证发布/回滚契约)
全栈知识治理平台首次入库(P1-P10 + 发布链路模拟验证闭环): - 治理核心:源目录/变更集/人工审核/发布批次/发布执行(fail-closed 闸口) - 测试通道:WeKnora 测试 KB 上传/轮询/验收;poll 完成时写入 parser_generation/index_generation,修复 manual/seed 路径发布闸口永久拦截 - 发布适配器 v071(本地 source=测试角色 / 云端 target=生产角色 模拟验证通过): * ES 导出/导入 mapping 自适应(.keyword 子字段、可选 tenant_id) * OSS locator 支持嵌套 scheme;chunks.seq_id 由目标序列分配 * bundle 归档同身份重试原子替换;verifier 按 bundle chunk_type 计数 * 回滚 job payload 支持 prior_execution_id(strict 解码) - 三通道发布+回滚演练实测:publish 4/4 verified,检索命中, rollback 4/4 rolled_back,目标端 PG is_enabled=false、ES is_enabled=false - 前端 A·Data-console 风格全站统一,120 用例通过 - 门禁:go vet / go test 全包通过,gofmt 干净
This commit is contained in:
@@ -0,0 +1,191 @@
|
||||
package catalogrefresh
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode"
|
||||
|
||||
"knowledge-governance/internal/jobs"
|
||||
)
|
||||
|
||||
type Request struct {
|
||||
ConnectionID string `json:"connection_id"`
|
||||
SourceType string `json:"source_type"`
|
||||
SourceName string `json:"source_name"`
|
||||
Stage string `json:"stage"`
|
||||
Progress int `json:"progress"`
|
||||
Message string `json:"message"`
|
||||
Processed int `json:"processed"`
|
||||
Total int `json:"total"`
|
||||
}
|
||||
type Progress struct {
|
||||
Stage string `json:"stage"`
|
||||
Progress int `json:"progress"`
|
||||
Message string `json:"message"`
|
||||
Processed int `json:"processed"`
|
||||
Total int `json:"total"`
|
||||
SourceName string `json:"source_name,omitempty"`
|
||||
CurrentDirectory string `json:"current_directory,omitempty"`
|
||||
ScannedDirectories int `json:"scanned_directories,omitempty"`
|
||||
DiscoveredDirectories int `json:"discovered_directories,omitempty"`
|
||||
ScannedFiles int `json:"scanned_files,omitempty"`
|
||||
EstimatedTotalFiles int `json:"estimated_total_files,omitempty"`
|
||||
TotalIsEstimate bool `json:"total_is_estimate,omitempty"`
|
||||
}
|
||||
type Store interface {
|
||||
UpdateCatalogRefreshProgress(context.Context, string, string, Progress, time.Time) error
|
||||
}
|
||||
|
||||
var progressForbiddenSubstrings = []string{"http://", "https://", "token", "secret", "password", "authorization", "bearer", "cookie", "/run/source-secrets", "api-key", "apikey"}
|
||||
|
||||
type Runner interface {
|
||||
Run(context.Context, Request, func(Progress) error) error
|
||||
}
|
||||
type Config struct {
|
||||
Store Store
|
||||
Runner Runner
|
||||
Now func() time.Time
|
||||
}
|
||||
type Handler struct {
|
||||
store Store
|
||||
runner Runner
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
func NewHandler(config Config) (*Handler, error) {
|
||||
if config.Store == nil || config.Runner == nil {
|
||||
return nil, errors.New("catalog refresh dependencies are required")
|
||||
}
|
||||
if config.Now == nil {
|
||||
config.Now = time.Now
|
||||
}
|
||||
return &Handler{store: config.Store, runner: config.Runner, now: config.Now}, nil
|
||||
}
|
||||
func (handler *Handler) Handle(ctx context.Context, job jobs.Job) error {
|
||||
var request Request
|
||||
decoder := json.NewDecoder(bytes.NewReader(job.Payload))
|
||||
decoder.DisallowUnknownFields()
|
||||
if err := decoder.Decode(&request); err != nil || request.ConnectionID == "" || request.SourceName == "" || (request.SourceType != "pp" && request.SourceType != "dingtalk" && request.SourceType != "alist") || job.LeaseOwner == "" {
|
||||
return errors.New("catalog_refresh_invalid_payload")
|
||||
}
|
||||
var last Progress
|
||||
return handler.runner.Run(ctx, request, func(progress Progress) error {
|
||||
progress = clampProgress(last, progress)
|
||||
if !validProgress(progress) {
|
||||
return errors.New("catalog_refresh_invalid_progress")
|
||||
}
|
||||
last = progress
|
||||
return handler.store.UpdateCatalogRefreshProgress(ctx, job.ID, job.LeaseOwner, progress, handler.now().UTC())
|
||||
})
|
||||
}
|
||||
func validProgress(progress Progress) bool {
|
||||
if progress.Progress < 0 || progress.Progress > 100 || progress.Processed < 0 || progress.Total < 0 ||
|
||||
progress.ScannedDirectories < 0 || progress.DiscoveredDirectories < 0 || progress.ScannedFiles < 0 || progress.EstimatedTotalFiles < 0 ||
|
||||
strings.TrimSpace(progress.Stage) == "" || len(progress.Stage) > 32 ||
|
||||
strings.TrimSpace(progress.Message) == "" || len(progress.Message) > 300 ||
|
||||
len(progress.CurrentDirectory) > 1024 || len(progress.SourceName) > 255 {
|
||||
return false
|
||||
}
|
||||
for _, text := range []string{progress.Stage, progress.Message, progress.CurrentDirectory, progress.SourceName} {
|
||||
for _, r := range text {
|
||||
if unicode.IsControl(r) && r != ' ' {
|
||||
return false
|
||||
}
|
||||
}
|
||||
lowered := strings.ToLower(text)
|
||||
for _, forbidden := range progressForbiddenSubstrings {
|
||||
if strings.Contains(lowered, forbidden) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// clampProgress enforces that display progress never regresses when discovered
|
||||
// totals shift mid-scan (BFS estimates grow), and never clamps an explicit
|
||||
// failure snapshot.
|
||||
func clampProgress(previous, next Progress) Progress {
|
||||
if next.Stage == "failed" {
|
||||
return next
|
||||
}
|
||||
if next.Total < previous.Total && previous.Total >= next.Processed {
|
||||
next.Total = previous.Total
|
||||
}
|
||||
if next.Total > 0 {
|
||||
estimated := next.Processed * 100 / next.Total
|
||||
if next.Progress < previous.Progress && previous.Progress <= estimated {
|
||||
next.Progress = previous.Progress
|
||||
}
|
||||
}
|
||||
if next.ScannedDirectories < previous.ScannedDirectories {
|
||||
next.ScannedDirectories = previous.ScannedDirectories
|
||||
}
|
||||
if next.ScannedFiles < previous.ScannedFiles {
|
||||
next.ScannedFiles = previous.ScannedFiles
|
||||
}
|
||||
if next.DiscoveredDirectories < previous.DiscoveredDirectories {
|
||||
next.DiscoveredDirectories = previous.DiscoveredDirectories
|
||||
}
|
||||
if next.EstimatedTotalFiles < previous.EstimatedTotalFiles {
|
||||
next.EstimatedTotalFiles = previous.EstimatedTotalFiles
|
||||
}
|
||||
return next
|
||||
}
|
||||
|
||||
type CommandRunner struct {
|
||||
PythonPath string
|
||||
ScriptPath string
|
||||
}
|
||||
|
||||
func NewCommandRunner(pythonPath, scriptPath string) (*CommandRunner, error) {
|
||||
if !filepath.IsAbs(pythonPath) || !filepath.IsAbs(scriptPath) {
|
||||
return nil, errors.New("catalog refresh command paths must be absolute")
|
||||
}
|
||||
return &CommandRunner{PythonPath: pythonPath, ScriptPath: scriptPath}, nil
|
||||
}
|
||||
func (runner *CommandRunner) Run(ctx context.Context, request Request, report func(Progress) error) error {
|
||||
command := exec.CommandContext(ctx, runner.PythonPath, runner.ScriptPath, "--connection-id", request.ConnectionID, "--source-type", request.SourceType, "--source-name", request.SourceName)
|
||||
stdout, err := command.StdoutPipe()
|
||||
if err != nil {
|
||||
return errors.New("catalog_refresh_start_failed")
|
||||
}
|
||||
command.Stderr = io.Discard
|
||||
if err = command.Start(); err != nil {
|
||||
return errors.New("catalog_refresh_start_failed")
|
||||
}
|
||||
scanner := bufio.NewScanner(stdout)
|
||||
scanner.Buffer(make([]byte, 4096), 1<<20)
|
||||
for scanner.Scan() {
|
||||
decoder := json.NewDecoder(bytes.NewReader(scanner.Bytes()))
|
||||
decoder.DisallowUnknownFields()
|
||||
var progress Progress
|
||||
if decoder.Decode(&progress) != nil || !validProgress(progress) {
|
||||
_ = command.Process.Kill()
|
||||
_ = command.Wait()
|
||||
return errors.New("catalog_refresh_invalid_output")
|
||||
}
|
||||
if err = report(progress); err != nil {
|
||||
_ = command.Process.Kill()
|
||||
_ = command.Wait()
|
||||
return err
|
||||
}
|
||||
}
|
||||
if scanner.Err() != nil {
|
||||
_ = command.Process.Kill()
|
||||
_ = command.Wait()
|
||||
return errors.New("catalog_refresh_output_failed")
|
||||
}
|
||||
if err = command.Wait(); err != nil {
|
||||
return errors.New("catalog_refresh_failed")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
package catalogrefresh
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"knowledge-governance/internal/jobs"
|
||||
)
|
||||
|
||||
type fakeStore struct{ updates []Progress }
|
||||
|
||||
func (s *fakeStore) UpdateCatalogRefreshProgress(_ context.Context, _ string, _ string, p Progress, _ time.Time) error {
|
||||
s.updates = append(s.updates, p)
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestHandlerClampsMonotonicProgressAndKeepsFailuresVisible(t *testing.T) {
|
||||
store := &fakeStore{}
|
||||
handler, err := NewHandler(Config{Store: store, Runner: fakeRunner{}, Now: func() time.Time { return time.Now() }})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runner := func(ctx context.Context, _ Request, report func(Progress) error) error {
|
||||
_ = report(Progress{Stage: "scanning", Progress: 40, Message: "扫描中", Processed: 60, Total: 100})
|
||||
// Discovered more directories than estimated: total must not shrink, percent must not regress.
|
||||
_ = report(Progress{Stage: "scanning", Progress: 30, Message: "发现更多目录", Processed: 70, Total: 50})
|
||||
return report(Progress{Stage: "failed", Progress: 0, Message: "来源不可用"})
|
||||
}
|
||||
custom := *handler
|
||||
custom.runner = runnerFunc(runner)
|
||||
job := jobs.Job{ID: "job-1", LeaseOwner: "worker-1", Payload: []byte(`{"connection_id":"conn-1","source_type":"alist","source_name":"国内工业","stage":"queued","progress":0,"message":"等待后台刷新","processed":0,"total":0}`)}
|
||||
if err = custom.Handle(context.Background(), job); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(store.updates) != 3 {
|
||||
t.Fatalf("updates=%#v", store.updates)
|
||||
}
|
||||
if store.updates[1].Progress < 40 || store.updates[1].Total < 70 {
|
||||
t.Fatalf("regression not clamped: %#v", store.updates[1])
|
||||
}
|
||||
if store.updates[2].Stage != "failed" || store.updates[2].Progress != 0 {
|
||||
t.Fatalf("failure must keep its own state: %#v", store.updates[2])
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidProgressRejectsControlCharsURLsAndCredentials(t *testing.T) {
|
||||
base := Progress{Stage: "scanning", Progress: 50, Message: "正在扫描", Processed: 10, Total: 20}
|
||||
if !validProgress(base) {
|
||||
t.Fatal("baseline rejected")
|
||||
}
|
||||
for name, mutate := range map[string]func(*Progress){
|
||||
"control": func(p *Progress) { p.CurrentDirectory = "/产品\a资料" },
|
||||
"url": func(p *Progress) { p.CurrentDirectory = "https://evil.invalid/x" },
|
||||
"url-message": func(p *Progress) { p.Message = "访问 https://evil.invalid 失败" },
|
||||
"token": func(p *Progress) { p.Message = "token=abc123 已过期" },
|
||||
"too-long": func(p *Progress) { p.CurrentDirectory = strings.Repeat("目", 1100) },
|
||||
"negative": func(p *Progress) { p.ScannedFiles = -1 },
|
||||
"secret-dir": func(p *Progress) { p.CurrentDirectory = "/run/source-secrets/pp" },
|
||||
} {
|
||||
item := base
|
||||
mutate(&item)
|
||||
if validProgress(item) {
|
||||
t.Errorf("%s accepted", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCommandRunnerRejectsUnknownProgressFields(t *testing.T) {
|
||||
python, err := exec.LookPath("python3")
|
||||
if err != nil {
|
||||
t.Skip("python3 unavailable")
|
||||
}
|
||||
runner, err := NewCommandRunner(python, writeFakeProgressScript(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = runner.Run(context.Background(), Request{ConnectionID: "c", SourceType: "pp", SourceName: "n"}, func(Progress) error { return nil })
|
||||
if err == nil || err.Error() != "catalog_refresh_invalid_output" {
|
||||
t.Fatalf("unknown field must fail: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func writeFakeProgressScript(t *testing.T) string {
|
||||
t.Helper()
|
||||
directory := t.TempDir()
|
||||
path := filepath.Join(directory, "progress.py")
|
||||
body := "import json\nprint(json.dumps({'stage':'scanning','progress':10,'message':'ok','processed':1,'total':2,'unexpected_field':'leak'}),flush=True)\n"
|
||||
if err := os.WriteFile(path, []byte(body), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
type runnerFunc func(context.Context, Request, func(Progress) error) error
|
||||
|
||||
func (f runnerFunc) Run(ctx context.Context, request Request, report func(Progress) error) error {
|
||||
return f(ctx, request, report)
|
||||
}
|
||||
|
||||
type fakeRunner struct{}
|
||||
|
||||
func (fakeRunner) Run(_ context.Context, _ Request, report func(Progress) error) error {
|
||||
if err := report(Progress{Stage: "scanning", Progress: 50, Message: "正在扫描"}); err != nil {
|
||||
return err
|
||||
}
|
||||
return report(Progress{Stage: "completed", Progress: 100, Message: "完成"})
|
||||
}
|
||||
func TestHandlerRunsLiveRefreshAndPersistsProgress(t *testing.T) {
|
||||
store := &fakeStore{}
|
||||
handler, err := NewHandler(Config{Store: store, Runner: fakeRunner{}, Now: func() time.Time { return time.Now() }})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
job := jobs.Job{ID: "job-1", LeaseOwner: "worker-1", Payload: []byte(`{"connection_id":"conn-1","source_type":"pp","source_name":"PP资料下载中心","stage":"queued","progress":0,"message":"等待后台刷新","processed":0,"total":0}`)}
|
||||
if err = handler.Handle(context.Background(), job); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(store.updates) != 2 || store.updates[0].Stage != "scanning" || store.updates[1].Progress != 100 {
|
||||
t.Fatalf("updates=%#v", store.updates)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user