Files
Enterprise-Knowledge-Base/internal/application/service/wiki_lint.go
T
wizardchen fdccb68ee4 feat: Improve wiki cleanup and ingest handling for deleted knowledge
- Enhanced the `cleanupWikiOnKnowledgeDelete` function to pass the full knowledge object, allowing for more accurate title and summary retrieval during wiki cleanup.
- Introduced a tombstone mechanism in Redis to mark recently deleted knowledge, enabling in-flight wiki ingest tasks to skip processing for deleted documents.
- Updated the `isKnowledgeGone` function to check for deleted knowledge using the tombstone, improving efficiency in handling concurrent delete and ingest operations.
- Added logic in the `wikiIngestService` to filter out updates for deleted knowledge, ensuring that no ghost references are created during the ingest process.
- Implemented lint checks for stale source references in the `WikiLintService`, allowing for automatic cleanup of pages referencing deleted knowledge.

These changes significantly enhance the robustness of the wiki system, ensuring consistency and accuracy in handling knowledge deletions and related wiki pages.
2026-04-22 21:18:20 +08:00

399 lines
11 KiB
Go

package service
import (
"context"
"fmt"
"strings"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
)
// WikiLintIssueType defines the type of lint issue
type WikiLintIssueType string
const (
LintIssueOrphanPage WikiLintIssueType = "orphan_page"
LintIssueBrokenLink WikiLintIssueType = "broken_link"
LintIssueStaleRef WikiLintIssueType = "stale_ref"
LintIssueMissingCrossRef WikiLintIssueType = "missing_cross_ref"
LintIssueEmptyContent WikiLintIssueType = "empty_content"
LintIssueDuplicateSlug WikiLintIssueType = "duplicate_slug"
)
// WikiLintIssueSeverity defines the severity of a lint issue
type WikiLintIssueSeverity string
const (
SeverityInfo WikiLintIssueSeverity = "info"
SeverityWarning WikiLintIssueSeverity = "warning"
SeverityError WikiLintIssueSeverity = "error"
)
// WikiLintIssue represents a single lint finding
type WikiLintIssue struct {
Type WikiLintIssueType `json:"type"`
Severity WikiLintIssueSeverity `json:"severity"`
PageSlug string `json:"page_slug"`
// TargetSlug identifies the other page involved in the issue (e.g. the
// broken link target, or the entity slug for a missing cross-ref). It is
// the structured field used by AutoFix instead of parsing Description.
TargetSlug string `json:"target_slug,omitempty"`
Description string `json:"description"`
AutoFixable bool `json:"auto_fixable"`
}
// WikiLintReport is the complete lint report for a wiki KB
type WikiLintReport struct {
KnowledgeBaseID string `json:"knowledge_base_id"`
Issues []WikiLintIssue `json:"issues"`
HealthScore int `json:"health_score"` // 0-100
Stats *types.WikiStats `json:"stats"`
Summary string `json:"summary"`
}
// WikiLintService provides wiki health checking capabilities
type WikiLintService struct {
wikiService interfaces.WikiPageService
kbService interfaces.KnowledgeBaseService
knowledgeService interfaces.KnowledgeService
}
// NewWikiLintService creates a new wiki lint service
func NewWikiLintService(
wikiService interfaces.WikiPageService,
kbService interfaces.KnowledgeBaseService,
knowledgeService interfaces.KnowledgeService,
) *WikiLintService {
return &WikiLintService{
wikiService: wikiService,
kbService: kbService,
knowledgeService: knowledgeService,
}
}
// RunLint performs a comprehensive health check on a wiki knowledge base
func (s *WikiLintService) RunLint(ctx context.Context, kbID string) (*WikiLintReport, error) {
// Validate KB
kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, kbID)
if err != nil {
return nil, fmt.Errorf("get KB: %w", err)
}
if !kb.IsWikiEnabled() {
return nil, fmt.Errorf("KB %s is not a wiki type", kbID)
}
// Get stats
stats, err := s.wikiService.GetStats(ctx, kbID)
if err != nil {
return nil, fmt.Errorf("get stats: %w", err)
}
// Get graph for link analysis
graph, err := s.wikiService.GetGraph(ctx, kbID)
if err != nil {
return nil, fmt.Errorf("get graph: %w", err)
}
// Get all pages for detailed analysis
resp, err := s.wikiService.ListPages(ctx, &types.WikiPageListRequest{
KnowledgeBaseID: kbID,
PageSize: 500,
})
if err != nil {
return nil, fmt.Errorf("list pages: %w", err)
}
var issues []WikiLintIssue
healthScore := 100
// Build slug set for link validation
slugSet := make(map[string]bool)
for _, node := range graph.Nodes {
slugSet[node.Slug] = true
}
// Check 1: Orphan pages (no inbound links, excluding index/log)
for _, page := range resp.Pages {
if page.PageType == types.WikiPageTypeIndex || page.PageType == types.WikiPageTypeLog {
continue
}
if len(page.InLinks) == 0 {
issues = append(issues, WikiLintIssue{
Type: LintIssueOrphanPage,
Severity: SeverityWarning,
PageSlug: page.Slug,
Description: fmt.Sprintf("Page '%s' has no inbound links — it's disconnected from the wiki", page.Title),
AutoFixable: false,
})
}
}
// Check 2: Broken links
for _, page := range resp.Pages {
for _, outLink := range page.OutLinks {
if !slugSet[outLink] {
issues = append(issues, WikiLintIssue{
Type: LintIssueBrokenLink,
Severity: SeverityError,
PageSlug: page.Slug,
TargetSlug: outLink,
Description: fmt.Sprintf("Page '%s' links to [[%s]] which does not exist", page.Title, outLink),
AutoFixable: true,
})
}
}
}
// Check 3: Empty content
for _, page := range resp.Pages {
content := strings.TrimSpace(page.Content)
if len(content) < 50 {
issues = append(issues, WikiLintIssue{
Type: LintIssueEmptyContent,
Severity: SeverityWarning,
PageSlug: page.Slug,
Description: fmt.Sprintf("Page '%s' has very little content (%d chars)", page.Title, len(content)),
AutoFixable: true,
})
}
}
// Check 4: Stale source refs — source_refs pointing at soft-deleted
// knowledge. This is the primary self-heal for the ingest/delete race
// condition: if a wiki_ingest task managed to slip past the in-flight
// guards and wrote a page for a knowledge that has since been deleted,
// the page becomes a dead-end (wiki_read_source_doc returns "knowledge
// not found"). Flag it so AutoFix can strip the ref, and delete the
// page if no live refs remain.
if s.knowledgeService != nil {
knowledgeLive := make(map[string]bool) // kid -> exists
for _, page := range resp.Pages {
// Skip wiki-intrinsic pages: index/log never carry SourceRefs
// anyway, and accidentally flagging them would risk AutoFix
// deleting system pages.
if page.PageType == types.WikiPageTypeIndex || page.PageType == types.WikiPageTypeLog {
continue
}
for _, ref := range page.SourceRefs {
kid := ref
if i := strings.Index(ref, "|"); i > 0 {
kid = ref[:i]
}
if kid == "" {
continue
}
live, seen := knowledgeLive[kid]
if !seen {
kn, err := s.knowledgeService.GetKnowledgeByIDOnly(ctx, kid)
live = err == nil && kn != nil
knowledgeLive[kid] = live
}
if !live {
issues = append(issues, WikiLintIssue{
Type: LintIssueStaleRef,
Severity: SeverityError,
PageSlug: page.Slug,
TargetSlug: kid,
Description: fmt.Sprintf("Page '%s' references deleted knowledge %s", page.Title, kid),
AutoFixable: true,
})
}
}
}
}
// Check 5: Missing cross-references (entities mentioned in content but not linked)
entitySlugs := make(map[string]string) // slug -> title
for _, page := range resp.Pages {
if page.PageType == types.WikiPageTypeEntity || page.PageType == types.WikiPageTypeConcept {
entitySlugs[page.Slug] = page.Title
}
}
for _, page := range resp.Pages {
for slug, title := range entitySlugs {
if slug == page.Slug {
continue
}
// Check if title is mentioned in content but not linked
if strings.Contains(strings.ToLower(page.Content), strings.ToLower(title)) {
linked := false
for _, l := range page.OutLinks {
if l == slug {
linked = true
break
}
}
if !linked {
issues = append(issues, WikiLintIssue{
Type: LintIssueMissingCrossRef,
Severity: SeverityInfo,
PageSlug: page.Slug,
TargetSlug: slug,
Description: fmt.Sprintf("Page '%s' mentions '%s' but doesn't link to [[%s]]", page.Title, title, slug),
AutoFixable: false,
})
}
}
}
}
// Calculate health score
if stats.TotalPages > 0 {
// Penalize for orphans
orphanPct := float64(stats.OrphanCount) / float64(stats.TotalPages) * 100
if orphanPct > 50 {
healthScore -= 25
} else if orphanPct > 25 {
healthScore -= 10
}
// Penalize for broken links
brokenCount := 0
for _, issue := range issues {
if issue.Type == LintIssueBrokenLink {
brokenCount++
}
}
healthScore -= brokenCount * 5
// Penalize for no links at all
if stats.TotalLinks == 0 && stats.TotalPages > 2 {
healthScore -= 15
}
// Penalize for empty pages
emptyCount := 0
for _, issue := range issues {
if issue.Type == LintIssueEmptyContent {
emptyCount++
}
}
healthScore -= emptyCount * 3
}
if healthScore < 0 {
healthScore = 0
}
// Generate summary
var summary strings.Builder
errorCount := 0
warningCount := 0
infoCount := 0
for _, issue := range issues {
switch issue.Severity {
case SeverityError:
errorCount++
case SeverityWarning:
warningCount++
case SeverityInfo:
infoCount++
}
}
if len(issues) == 0 {
summary.WriteString("Wiki is healthy! No issues found.")
} else {
fmt.Fprintf(&summary, "Found %d issues: %d errors, %d warnings, %d suggestions.",
len(issues), errorCount, warningCount, infoCount)
}
report := &WikiLintReport{
KnowledgeBaseID: kbID,
Issues: issues,
HealthScore: healthScore,
Stats: stats,
Summary: summary.String(),
}
logger.Infof(ctx, "wiki lint: KB %s — health score %d/100, %d issues", kbID, healthScore, len(issues))
return report, nil
}
// AutoFix attempts to automatically fix fixable issues
func (s *WikiLintService) AutoFix(ctx context.Context, kbID string) (int, error) {
report, err := s.RunLint(ctx, kbID)
if err != nil {
return 0, err
}
fixed := 0
for _, issue := range report.Issues {
if !issue.AutoFixable {
continue
}
switch issue.Type {
case LintIssueBrokenLink:
// Replace [[broken-slug]] with plain text so the reference text is
// preserved but no longer renders as a dangling wiki link.
if issue.TargetSlug == "" {
continue
}
page, err := s.wikiService.GetPageBySlug(ctx, kbID, issue.PageSlug)
if err != nil {
continue
}
target := issue.TargetSlug
page.Content = strings.ReplaceAll(page.Content, "[["+target+"]]", target)
if _, err := s.wikiService.UpdatePage(ctx, page); err == nil {
fixed++
}
case LintIssueEmptyContent:
// Archive pages with very little content instead of deleting
page, err := s.wikiService.GetPageBySlug(ctx, kbID, issue.PageSlug)
if err != nil {
continue
}
// Don't archive index or log pages
if page.PageType == types.WikiPageTypeIndex || page.PageType == types.WikiPageTypeLog {
continue
}
page.Status = types.WikiPageStatusArchived
if _, err := s.wikiService.UpdatePage(ctx, page); err == nil {
fixed++
}
case LintIssueStaleRef:
// Strip source_refs that point at soft-deleted knowledge. If the
// page has no other live sources, delete it outright — leaving
// an orphan summary page is worse than removing it, because the
// model would still link to it from other pages and the
// wiki_read_source_doc drill-down would always fail.
if issue.TargetSlug == "" {
continue
}
page, err := s.wikiService.GetPageBySlug(ctx, kbID, issue.PageSlug)
if err != nil || page == nil {
continue
}
if page.PageType == types.WikiPageTypeIndex || page.PageType == types.WikiPageTypeLog {
continue
}
remaining := removeSourceRef(page.SourceRefs, issue.TargetSlug)
if len(remaining) == 0 {
if err := s.wikiService.DeletePage(ctx, kbID, page.Slug); err == nil {
fixed++
}
} else if len(remaining) != len(page.SourceRefs) {
page.SourceRefs = remaining
if err := s.wikiService.UpdatePageMeta(ctx, page); err == nil {
fixed++
}
}
}
}
// Rebuild links after fixes
if fixed > 0 {
_ = s.wikiService.RebuildLinks(ctx, kbID)
}
logger.Infof(ctx, "wiki auto-fix: KB %s — fixed %d issues", kbID, fixed)
return fixed, nil
}