feat: Enhance wiki extraction and configuration options

- Introduced extraction granularity settings in the wiki configuration, allowing users to control the level of detail in entity and concept extraction (focused, standard, exhaustive).
- Updated the UI to reflect these new options, including tooltips for better user guidance.
- Refactored the knowledge base editor to support the new extraction granularity feature, ensuring a seamless user experience.
- Improved the backend logic for wiki ingest tasks to utilize the new granularity settings, enhancing the accuracy of extracted content.
- Added tests to validate the new extraction granularity functionality and ensure robust performance across different configurations.
This commit is contained in:
wizardchen
2026-04-22 20:27:19 +08:00
parent 623add89f1
commit a8a3d9e694
23 changed files with 2110 additions and 174 deletions
+84 -30
View File
@@ -65,8 +65,19 @@ type WikiPage struct {
Summary string `json:"summary" gorm:"type:text"`
// Alternate names, abbreviations, acronyms or translated names
Aliases StringArray `json:"aliases" gorm:"type:json"`
// References to source knowledge IDs that contributed to this page
// References to source knowledge IDs that contributed to this page.
// Format matches the legacy "<knowledge_id>|<doc_title>" convention used
// across the ingest pipeline, so retract / display code can split on `|`
// to recover the title. Document-level granularity.
SourceRefs StringArray `json:"source_refs" gorm:"type:json"`
// ChunkRefs records the specific source-document chunks this page was
// built from — one UUID per cited chunk. Populated during ingest from
// the chunk-citation pass; refreshed wholesale whenever the page is
// re-materialized. Empty for summary pages (they are document-level
// synopses and don't carry chunk-level citations). Use this when you
// need to surface the underlying evidence for a wiki page, or to
// retract citations when a source document is deleted.
ChunkRefs StringArray `json:"chunk_refs" gorm:"type:json"`
// Slugs of pages that link TO this page (backlinks)
InLinks StringArray `json:"in_links" gorm:"type:json"`
// Slugs of pages this page links to (outbound links)
@@ -88,18 +99,61 @@ func (WikiPage) TableName() string {
return "wiki_pages"
}
// WikiExtractionGranularity controls how aggressive Pass 0 (candidate slug
// extraction) is. Higher granularity = more slugs, lower = tighter focus on
// the document's main subjects.
type WikiExtractionGranularity string
const (
// WikiExtractionFocused keeps only the document's main subjects (e.g.
// a resume yields the person + their projects, nothing else). Most
// aggressive slug pruning; avoids index bloat from incidental technology
// names and generic concepts.
WikiExtractionFocused WikiExtractionGranularity = "focused"
// WikiExtractionStandard is the default: main subjects plus entities /
// concepts that are substantively discussed (a dedicated paragraph or
// multiple bullet points). Skips one-off mentions and commodity terms.
WikiExtractionStandard WikiExtractionGranularity = "standard"
// WikiExtractionExhaustive extracts every named entity and recognizable
// concept, including stacks/libs mentioned in passing. Matches the
// pre-granularity behavior. Useful when the KB is being used as a
// glossary rather than a curated wiki.
WikiExtractionExhaustive WikiExtractionGranularity = "exhaustive"
)
// IsValid reports whether g is one of the three recognized levels.
func (g WikiExtractionGranularity) IsValid() bool {
switch g {
case WikiExtractionFocused, WikiExtractionStandard, WikiExtractionExhaustive:
return true
}
return false
}
// Normalize returns g if valid, otherwise WikiExtractionStandard. Callers
// pipe config through this so historical rows with empty / unknown values
// don't surprise the extraction prompt.
func (g WikiExtractionGranularity) Normalize() WikiExtractionGranularity {
if g.IsValid() {
return g
}
return WikiExtractionStandard
}
// WikiConfig stores wiki-specific configuration for a knowledge base.
// Applicable to document-type knowledge bases with wiki feature enabled.
// When Enabled is true, document ingestion triggers automatic wiki page generation.
// Whether the wiki feature is turned on is controlled by IndexingStrategy.WikiEnabled;
// this struct only carries wiki-specific tunables.
type WikiConfig struct {
// Enabled activates the wiki feature for this knowledge base
Enabled bool `yaml:"enabled" json:"enabled"`
// AutoIngest triggers wiki page generation/update when new documents are added
AutoIngest bool `yaml:"auto_ingest" json:"auto_ingest"`
// SynthesisModelID is the LLM model ID used for wiki page generation and updates
SynthesisModelID string `yaml:"synthesis_model_id" json:"synthesis_model_id"`
// MaxPagesPerIngest limits pages created/updated per ingest operation (0 = no limit)
MaxPagesPerIngest int `yaml:"max_pages_per_ingest" json:"max_pages_per_ingest"`
// ExtractionGranularity controls how many candidate slugs Pass 0 extracts
// per document. Empty / unknown value is treated as WikiExtractionStandard.
ExtractionGranularity WikiExtractionGranularity `yaml:"extraction_granularity" json:"extraction_granularity,omitempty"`
}
// Value implements the driver.Valuer interface
@@ -122,13 +176,13 @@ func (c *WikiConfig) Scan(value interface{}) error {
// WikiPageListRequest represents a request to list wiki pages with filtering
type WikiPageListRequest struct {
KnowledgeBaseID string `json:"knowledge_base_id"`
PageType string `json:"page_type,omitempty"` // filter by type
Status string `json:"status,omitempty"` // filter by status
Query string `json:"query,omitempty"` // full-text search
Page int `json:"page,omitempty"` // pagination page (1-based)
PageSize int `json:"page_size,omitempty"` // pagination size
SortBy string `json:"sort_by,omitempty"` // "updated_at", "created_at", "title"
SortOrder string `json:"sort_order,omitempty"` // "asc" or "desc"
PageType string `json:"page_type,omitempty"` // filter by type
Status string `json:"status,omitempty"` // filter by status
Query string `json:"query,omitempty"` // full-text search
Page int `json:"page,omitempty"` // pagination page (1-based)
PageSize int `json:"page_size,omitempty"` // pagination size
SortBy string `json:"sort_by,omitempty"` // "updated_at", "created_at", "title"
SortOrder string `json:"sort_order,omitempty"` // "asc" or "desc"
}
// WikiPageListResponse represents a paginated list of wiki pages
@@ -166,28 +220,28 @@ type WikiStats struct {
TotalPages int64 `json:"total_pages"`
PagesByType map[string]int64 `json:"pages_by_type"`
TotalLinks int64 `json:"total_links"`
OrphanCount int64 `json:"orphan_count"` // pages with no inbound links
RecentUpdates []*WikiPage `json:"recent_updates"` // last N updated pages
PendingTasks int64 `json:"pending_tasks"` // number of documents waiting to be ingested
PendingIssues int64 `json:"pending_issues"` // number of pending wiki issues
IsActive bool `json:"is_active"` // whether wiki ingestion is currently running
OrphanCount int64 `json:"orphan_count"` // pages with no inbound links
RecentUpdates []*WikiPage `json:"recent_updates"` // last N updated pages
PendingTasks int64 `json:"pending_tasks"` // number of documents waiting to be ingested
PendingIssues int64 `json:"pending_issues"` // number of pending wiki issues
IsActive bool `json:"is_active"` // whether wiki ingestion is currently running
}
// WikiPageIssue represents an issue flagged on a specific wiki page.
// These issues are typically identified by agents or linters and stored for review.
type WikiPageIssue struct {
ID string `json:"id" gorm:"type:varchar(36);primaryKey"`
TenantID uint64 `json:"tenant_id" gorm:"index"`
KnowledgeBaseID string `json:"knowledge_base_id" gorm:"type:varchar(36);index"`
Slug string `json:"slug" gorm:"type:varchar(255);index"`
IssueType string `json:"issue_type" gorm:"type:varchar(50)"`
Description string `json:"description" gorm:"type:text"`
SuspectedKnowledgeIDs StringArray `json:"suspected_knowledge_ids" gorm:"type:json"`
Status string `json:"status" gorm:"type:varchar(20);default:'pending';index"`
ReportedBy string `json:"reported_by" gorm:"type:varchar(100)"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
DeletedAt gorm.DeletedAt `json:"deleted_at" gorm:"index"`
ID string `json:"id" gorm:"type:varchar(36);primaryKey"`
TenantID uint64 `json:"tenant_id" gorm:"index"`
KnowledgeBaseID string `json:"knowledge_base_id" gorm:"type:varchar(36);index"`
Slug string `json:"slug" gorm:"type:varchar(255);index"`
IssueType string `json:"issue_type" gorm:"type:varchar(50)"`
Description string `json:"description" gorm:"type:text"`
SuspectedKnowledgeIDs StringArray `json:"suspected_knowledge_ids" gorm:"type:json"`
Status string `json:"status" gorm:"type:varchar(20);default:'pending';index"`
ReportedBy string `json:"reported_by" gorm:"type:varchar(100)"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
DeletedAt gorm.DeletedAt `json:"deleted_at" gorm:"index"`
}
// TableName specifies the database table name
+87 -28
View File
@@ -29,11 +29,8 @@ func TestWikiPageTypes(t *testing.T) {
}
func TestWikiConfigValueScan(t *testing.T) {
// Test Value (serialize)
config := WikiConfig{
Enabled: true,
AutoIngest: true,
SynthesisModelID: "model-123",
SynthesisModelID: "model-123",
MaxPagesPerIngest: 20,
}
@@ -42,7 +39,6 @@ func TestWikiConfigValueScan(t *testing.T) {
t.Fatalf("WikiConfig.Value() error: %v", err)
}
// Test Scan (deserialize)
var restored WikiConfig
b, ok := val.([]byte)
if !ok {
@@ -52,9 +48,6 @@ func TestWikiConfigValueScan(t *testing.T) {
t.Fatalf("WikiConfig.Scan() error: %v", err)
}
if restored.AutoIngest != true {
t.Error("AutoIngest mismatch")
}
if restored.SynthesisModelID != "model-123" {
t.Error("SynthesisModelID mismatch")
}
@@ -115,25 +108,21 @@ func TestStringArrayEmpty(t *testing.T) {
}
func TestKnowledgeBaseEnsureDefaultsWiki(t *testing.T) {
kb := &KnowledgeBase{Type: KnowledgeBaseTypeDocument, WikiConfig: &WikiConfig{Enabled: true, AutoIngest: true}}
kb := &KnowledgeBase{
Type: KnowledgeBaseTypeDocument,
WikiConfig: &WikiConfig{SynthesisModelID: "m-1"},
}
kb.EnsureDefaults()
if kb.WikiConfig == nil {
t.Fatal("EnsureDefaults should preserve WikiConfig for wiki-enabled KB")
}
if kb.WikiConfig.AutoIngest != true {
t.Error("AutoIngest should be preserved")
if kb.WikiConfig.SynthesisModelID != "m-1" {
t.Error("SynthesisModelID should be preserved")
}
if kb.FAQConfig != nil {
t.Error("Document KB should not have FAQConfig")
}
// Test that user can disable AutoIngest
kb2 := &KnowledgeBase{Type: KnowledgeBaseTypeDocument, WikiConfig: &WikiConfig{Enabled: true, AutoIngest: false}}
kb2.EnsureDefaults()
if kb2.WikiConfig.AutoIngest != false {
t.Error("EnsureDefaults should not override user's AutoIngest=false setting")
}
}
func TestKnowledgeBaseEnsureDefaultsDocumentWithoutWiki(t *testing.T) {
@@ -150,16 +139,16 @@ func TestKnowledgeBaseEnsureDefaultsDocumentWithoutWiki(t *testing.T) {
func TestWikiPageJSON(t *testing.T) {
page := WikiPage{
ID: "test-id",
Slug: "entity/test",
Title: "Test Entity",
PageType: WikiPageTypeEntity,
Content: "# Test\n\nSome content with [[concept/related]]",
Summary: "A test entity page",
SourceRefs: StringArray{"source-1", "source-2"},
OutLinks: StringArray{"concept/related"},
InLinks: StringArray{"summary/doc1"},
Version: 3,
ID: "test-id",
Slug: "entity/test",
Title: "Test Entity",
PageType: WikiPageTypeEntity,
Content: "# Test\n\nSome content with [[concept/related]]",
Summary: "A test entity page",
SourceRefs: StringArray{"source-1", "source-2"},
OutLinks: StringArray{"concept/related"},
InLinks: StringArray{"summary/doc1"},
Version: 3,
}
// Serialize
@@ -225,3 +214,73 @@ func TestChunkTypeWikiPage(t *testing.T) {
t.Errorf("ChunkTypeWikiPage should be 'wiki_page', got '%s'", ChunkTypeWikiPage)
}
}
func TestWikiExtractionGranularity_IsValid(t *testing.T) {
valid := []WikiExtractionGranularity{
WikiExtractionFocused,
WikiExtractionStandard,
WikiExtractionExhaustive,
}
for _, g := range valid {
if !g.IsValid() {
t.Errorf("%q should be valid", g)
}
}
invalid := []WikiExtractionGranularity{"", "FOCUSED", "strict", "none", "FULL"}
for _, g := range invalid {
if g.IsValid() {
t.Errorf("%q should NOT be valid", g)
}
}
}
func TestWikiExtractionGranularity_Normalize(t *testing.T) {
cases := map[WikiExtractionGranularity]WikiExtractionGranularity{
"": WikiExtractionStandard, // legacy / unset
"unknown": WikiExtractionStandard,
"FOCUSED": WikiExtractionStandard, // case-sensitive on purpose
"focused": WikiExtractionFocused,
"standard": WikiExtractionStandard,
"exhaustive": WikiExtractionExhaustive,
}
for in, want := range cases {
if got := in.Normalize(); got != want {
t.Errorf("Normalize(%q) = %q, want %q", in, got, want)
}
}
}
func TestWikiConfig_JSONRoundTrip_WithGranularity(t *testing.T) {
original := WikiConfig{
SynthesisModelID: "m-1",
MaxPagesPerIngest: 20,
ExtractionGranularity: WikiExtractionFocused,
}
data, err := json.Marshal(original)
if err != nil {
t.Fatalf("Marshal error: %v", err)
}
var restored WikiConfig
if err := json.Unmarshal(data, &restored); err != nil {
t.Fatalf("Unmarshal error: %v", err)
}
if restored.ExtractionGranularity != WikiExtractionFocused {
t.Errorf("granularity round-trip failed: got %q", restored.ExtractionGranularity)
}
// Legacy row without the granularity field. We also include the removed
// "enabled" / "auto_ingest" keys to verify the decoder tolerates unknown
// fields on rows written before those columns were retired.
legacy := []byte(`{"enabled":true,"auto_ingest":true,"synthesis_model_id":"","max_pages_per_ingest":0}`)
var oldCfg WikiConfig
if err := json.Unmarshal(legacy, &oldCfg); err != nil {
t.Fatalf("Unmarshal legacy error: %v", err)
}
if oldCfg.ExtractionGranularity != "" {
t.Errorf("legacy row should decode to empty granularity, got %q", oldCfg.ExtractionGranularity)
}
if oldCfg.ExtractionGranularity.Normalize() != WikiExtractionStandard {
t.Errorf("legacy row should normalize to standard")
}
}