package testweknora import ( "context" "errors" "fmt" "sort" "strings" "sync" "sync/atomic" "knowledge-governance/internal/platform/weknora" ) type LegacyCandidate struct { ID string KnowledgeID string TestKBID string } type LegacyVerifyClient interface { GetKnowledge(context.Context, string) (weknora.Knowledge, error) ListChunks(context.Context, string, int, int, []string) ([]weknora.Chunk, int64, error) } type LegacyVerificationReport struct { Total int Existing int Missing int WrongKB int Pending int Processing int Finalizing int Completed int Failed int ChunkChecked int ChunkEmpty int HTTPErrors int } func VerifyLegacy(ctx context.Context, client LegacyVerifyClient, candidates []LegacyCandidate, chunkSampleEvery int) (LegacyVerificationReport, error) { report := LegacyVerificationReport{Total: len(candidates)} completedSeen := 0 for _, candidate := range candidates { knowledge, err := client.GetKnowledge(ctx, candidate.KnowledgeID) if err != nil { var httpErr *weknora.HTTPError if errors.As(err, &httpErr) && httpErr.StatusCode == 404 { report.Missing++ } else { report.HTTPErrors++ } continue } if knowledge.KnowledgeBaseID != candidate.TestKBID { report.WrongKB++ continue } report.Existing++ switch knowledge.ParseStatus { case "pending": report.Pending++ case "processing": report.Processing++ case "finalizing": report.Finalizing++ case "completed": report.Completed++ completedSeen++ if chunkSampleEvery > 0 && completedSeen%chunkSampleEvery == 0 { if err := verifyLegacyChunk(ctx, client, candidate.KnowledgeID, &report); err != nil { report.HTTPErrors++ } } case "failed": report.Failed++ } } return report, nil } func VerifyLegacyConcurrent(ctx context.Context, client LegacyVerifyClient, candidates []LegacyCandidate, chunkSampleEvery, concurrency int) (LegacyVerificationReport, error) { if client == nil || concurrency < 1 || concurrency > 32 || chunkSampleEvery < 0 { return LegacyVerificationReport{}, errors.New("invalid legacy verification configuration") } jobs := make(chan LegacyCandidate) results := make(chan LegacyVerificationReport) var workers sync.WaitGroup var completedSeen atomic.Int64 workerCount := min(concurrency, len(candidates)) workers.Add(workerCount) for range workerCount { go func() { defer workers.Done() for candidate := range jobs { partial, _ := VerifyLegacy(ctx, client, []LegacyCandidate{candidate}, 0) if partial.Completed == 1 { completed := completedSeen.Add(1) if chunkSampleEvery > 0 && completed%int64(chunkSampleEvery) == 0 { if err := verifyLegacyChunk(ctx, client, candidate.KnowledgeID, &partial); err != nil { partial.HTTPErrors++ } } } results <- partial } }() } go func() { defer close(jobs) for _, candidate := range candidates { select { case jobs <- candidate: case <-ctx.Done(): return } } }() go func() { workers.Wait() close(results) }() report := LegacyVerificationReport{} for partial := range results { mergeLegacyVerificationReport(&report, partial) } if err := ctx.Err(); err != nil { return report, err } return report, nil } func mergeLegacyVerificationReport(report *LegacyVerificationReport, partial LegacyVerificationReport) { report.Total += partial.Total report.Existing += partial.Existing report.Missing += partial.Missing report.WrongKB += partial.WrongKB report.Pending += partial.Pending report.Processing += partial.Processing report.Finalizing += partial.Finalizing report.Completed += partial.Completed report.Failed += partial.Failed report.ChunkChecked += partial.ChunkChecked report.ChunkEmpty += partial.ChunkEmpty report.HTTPErrors += partial.HTTPErrors } func verifyLegacyChunk(ctx context.Context, client LegacyVerifyClient, knowledgeID string, report *LegacyVerificationReport) error { _, total, err := client.ListChunks(ctx, knowledgeID, 1, 1, nil) if err != nil { return err } report.ChunkChecked++ if total == 0 { report.ChunkEmpty++ } return nil } func FormatDryRunReport(candidates []LegacyCandidate, concurrency int) string { kbCounts := make(map[string]int) for _, candidate := range candidates { kbCounts[candidate.TestKBID]++ } kbs := make([]string, 0, len(kbCounts)) for kb := range kbCounts { kbs = append(kbs, kb) } sort.Strings(kbs) var builder strings.Builder fmt.Fprintf(&builder, "total=%d\n", len(candidates)) fmt.Fprintf(&builder, "concurrency=%d\n", concurrency) for _, kb := range kbs { fmt.Fprintf(&builder, "knowledge_base=%s count=%d\n", kb, kbCounts[kb]) } return builder.String() } func FormatLegacyVerificationReport(report LegacyVerificationReport) string { return fmt.Sprintf( "total=%d\nexisting=%d\nmissing=%d\nwrong_kb=%d\npending=%d\nprocessing=%d\nfinalizing=%d\ncompleted=%d\nfailed=%d\nchunk_checked=%d\nchunk_empty=%d\nhttp_errors=%d\n", report.Total, report.Existing, report.Missing, report.WrongKB, report.Pending, report.Processing, report.Finalizing, report.Completed, report.Failed, report.ChunkChecked, report.ChunkEmpty, report.HTTPErrors, ) }