dcdc8d1ddd
全栈知识治理平台首次入库(P1-P10 + 发布链路模拟验证闭环): - 治理核心:源目录/变更集/人工审核/发布批次/发布执行(fail-closed 闸口) - 测试通道:WeKnora 测试 KB 上传/轮询/验收;poll 完成时写入 parser_generation/index_generation,修复 manual/seed 路径发布闸口永久拦截 - 发布适配器 v071(本地 source=测试角色 / 云端 target=生产角色 模拟验证通过): * ES 导出/导入 mapping 自适应(.keyword 子字段、可选 tenant_id) * OSS locator 支持嵌套 scheme;chunks.seq_id 由目标序列分配 * bundle 归档同身份重试原子替换;verifier 按 bundle chunk_type 计数 * 回滚 job payload 支持 prior_execution_id(strict 解码) - 三通道发布+回滚演练实测:publish 4/4 verified,检索命中, rollback 4/4 rolled_back,目标端 PG is_enabled=false、ES is_enabled=false - 前端 A·Data-console 风格全站统一,120 用例通过 - 门禁:go vet / go test 全包通过,gofmt 干净
197 lines
5.1 KiB
Go
197 lines
5.1 KiB
Go
package testweknora
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"sort"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
|
|
"knowledge-governance/internal/platform/weknora"
|
|
)
|
|
|
|
type LegacyCandidate struct {
|
|
ID string
|
|
KnowledgeID string
|
|
TestKBID string
|
|
}
|
|
|
|
type LegacyVerifyClient interface {
|
|
GetKnowledge(context.Context, string) (weknora.Knowledge, error)
|
|
ListChunks(context.Context, string, int, int, []string) ([]weknora.Chunk, int64, error)
|
|
}
|
|
|
|
type LegacyVerificationReport struct {
|
|
Total int
|
|
Existing int
|
|
Missing int
|
|
WrongKB int
|
|
Pending int
|
|
Processing int
|
|
Finalizing int
|
|
Completed int
|
|
Failed int
|
|
ChunkChecked int
|
|
ChunkEmpty int
|
|
HTTPErrors int
|
|
}
|
|
|
|
func VerifyLegacy(ctx context.Context, client LegacyVerifyClient, candidates []LegacyCandidate, chunkSampleEvery int) (LegacyVerificationReport, error) {
|
|
report := LegacyVerificationReport{Total: len(candidates)}
|
|
completedSeen := 0
|
|
for _, candidate := range candidates {
|
|
knowledge, err := client.GetKnowledge(ctx, candidate.KnowledgeID)
|
|
if err != nil {
|
|
var httpErr *weknora.HTTPError
|
|
if errors.As(err, &httpErr) && httpErr.StatusCode == 404 {
|
|
report.Missing++
|
|
} else {
|
|
report.HTTPErrors++
|
|
}
|
|
continue
|
|
}
|
|
if knowledge.KnowledgeBaseID != candidate.TestKBID {
|
|
report.WrongKB++
|
|
continue
|
|
}
|
|
report.Existing++
|
|
switch knowledge.ParseStatus {
|
|
case "pending":
|
|
report.Pending++
|
|
case "processing":
|
|
report.Processing++
|
|
case "finalizing":
|
|
report.Finalizing++
|
|
case "completed":
|
|
report.Completed++
|
|
completedSeen++
|
|
if chunkSampleEvery > 0 && completedSeen%chunkSampleEvery == 0 {
|
|
if err := verifyLegacyChunk(ctx, client, candidate.KnowledgeID, &report); err != nil {
|
|
report.HTTPErrors++
|
|
}
|
|
}
|
|
case "failed":
|
|
report.Failed++
|
|
}
|
|
}
|
|
return report, nil
|
|
}
|
|
|
|
func VerifyLegacyConcurrent(ctx context.Context, client LegacyVerifyClient, candidates []LegacyCandidate, chunkSampleEvery, concurrency int) (LegacyVerificationReport, error) {
|
|
if client == nil || concurrency < 1 || concurrency > 32 || chunkSampleEvery < 0 {
|
|
return LegacyVerificationReport{}, errors.New("invalid legacy verification configuration")
|
|
}
|
|
jobs := make(chan LegacyCandidate)
|
|
results := make(chan LegacyVerificationReport)
|
|
var workers sync.WaitGroup
|
|
var completedSeen atomic.Int64
|
|
workerCount := min(concurrency, len(candidates))
|
|
workers.Add(workerCount)
|
|
for range workerCount {
|
|
go func() {
|
|
defer workers.Done()
|
|
for candidate := range jobs {
|
|
partial, _ := VerifyLegacy(ctx, client, []LegacyCandidate{candidate}, 0)
|
|
if partial.Completed == 1 {
|
|
completed := completedSeen.Add(1)
|
|
if chunkSampleEvery > 0 && completed%int64(chunkSampleEvery) == 0 {
|
|
if err := verifyLegacyChunk(ctx, client, candidate.KnowledgeID, &partial); err != nil {
|
|
partial.HTTPErrors++
|
|
}
|
|
}
|
|
}
|
|
results <- partial
|
|
}
|
|
}()
|
|
}
|
|
go func() {
|
|
defer close(jobs)
|
|
for _, candidate := range candidates {
|
|
select {
|
|
case jobs <- candidate:
|
|
case <-ctx.Done():
|
|
return
|
|
}
|
|
}
|
|
}()
|
|
go func() {
|
|
workers.Wait()
|
|
close(results)
|
|
}()
|
|
|
|
report := LegacyVerificationReport{}
|
|
for partial := range results {
|
|
mergeLegacyVerificationReport(&report, partial)
|
|
}
|
|
if err := ctx.Err(); err != nil {
|
|
return report, err
|
|
}
|
|
return report, nil
|
|
}
|
|
|
|
func mergeLegacyVerificationReport(report *LegacyVerificationReport, partial LegacyVerificationReport) {
|
|
report.Total += partial.Total
|
|
report.Existing += partial.Existing
|
|
report.Missing += partial.Missing
|
|
report.WrongKB += partial.WrongKB
|
|
report.Pending += partial.Pending
|
|
report.Processing += partial.Processing
|
|
report.Finalizing += partial.Finalizing
|
|
report.Completed += partial.Completed
|
|
report.Failed += partial.Failed
|
|
report.ChunkChecked += partial.ChunkChecked
|
|
report.ChunkEmpty += partial.ChunkEmpty
|
|
report.HTTPErrors += partial.HTTPErrors
|
|
}
|
|
|
|
func verifyLegacyChunk(ctx context.Context, client LegacyVerifyClient, knowledgeID string, report *LegacyVerificationReport) error {
|
|
_, total, err := client.ListChunks(ctx, knowledgeID, 1, 1, nil)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
report.ChunkChecked++
|
|
if total == 0 {
|
|
report.ChunkEmpty++
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func FormatDryRunReport(candidates []LegacyCandidate, concurrency int) string {
|
|
kbCounts := make(map[string]int)
|
|
for _, candidate := range candidates {
|
|
kbCounts[candidate.TestKBID]++
|
|
}
|
|
kbs := make([]string, 0, len(kbCounts))
|
|
for kb := range kbCounts {
|
|
kbs = append(kbs, kb)
|
|
}
|
|
sort.Strings(kbs)
|
|
var builder strings.Builder
|
|
fmt.Fprintf(&builder, "total=%d\n", len(candidates))
|
|
fmt.Fprintf(&builder, "concurrency=%d\n", concurrency)
|
|
for _, kb := range kbs {
|
|
fmt.Fprintf(&builder, "knowledge_base=%s count=%d\n", kb, kbCounts[kb])
|
|
}
|
|
return builder.String()
|
|
}
|
|
|
|
func FormatLegacyVerificationReport(report LegacyVerificationReport) string {
|
|
return fmt.Sprintf(
|
|
"total=%d\nexisting=%d\nmissing=%d\nwrong_kb=%d\npending=%d\nprocessing=%d\nfinalizing=%d\ncompleted=%d\nfailed=%d\nchunk_checked=%d\nchunk_empty=%d\nhttp_errors=%d\n",
|
|
report.Total,
|
|
report.Existing,
|
|
report.Missing,
|
|
report.WrongKB,
|
|
report.Pending,
|
|
report.Processing,
|
|
report.Finalizing,
|
|
report.Completed,
|
|
report.Failed,
|
|
report.ChunkChecked,
|
|
report.ChunkEmpty,
|
|
report.HTTPErrors,
|
|
)
|
|
}
|