feat: Enhance wiki extraction and configuration options
- Introduced extraction granularity settings in the wiki configuration, allowing users to control the level of detail in entity and concept extraction (focused, standard, exhaustive). - Updated the UI to reflect these new options, including tooltips for better user guidance. - Refactored the knowledge base editor to support the new extraction granularity feature, ensuring a seamless user experience. - Improved the backend logic for wiki ingest tasks to utilize the new granularity settings, enhancing the accuracy of extracted content. - Added tests to validate the new extraction granularity functionality and ensure robust performance across different configurations.
This commit is contained in:
+84
-30
@@ -65,8 +65,19 @@ type WikiPage struct {
|
||||
Summary string `json:"summary" gorm:"type:text"`
|
||||
// Alternate names, abbreviations, acronyms or translated names
|
||||
Aliases StringArray `json:"aliases" gorm:"type:json"`
|
||||
// References to source knowledge IDs that contributed to this page
|
||||
// References to source knowledge IDs that contributed to this page.
|
||||
// Format matches the legacy "<knowledge_id>|<doc_title>" convention used
|
||||
// across the ingest pipeline, so retract / display code can split on `|`
|
||||
// to recover the title. Document-level granularity.
|
||||
SourceRefs StringArray `json:"source_refs" gorm:"type:json"`
|
||||
// ChunkRefs records the specific source-document chunks this page was
|
||||
// built from — one UUID per cited chunk. Populated during ingest from
|
||||
// the chunk-citation pass; refreshed wholesale whenever the page is
|
||||
// re-materialized. Empty for summary pages (they are document-level
|
||||
// synopses and don't carry chunk-level citations). Use this when you
|
||||
// need to surface the underlying evidence for a wiki page, or to
|
||||
// retract citations when a source document is deleted.
|
||||
ChunkRefs StringArray `json:"chunk_refs" gorm:"type:json"`
|
||||
// Slugs of pages that link TO this page (backlinks)
|
||||
InLinks StringArray `json:"in_links" gorm:"type:json"`
|
||||
// Slugs of pages this page links to (outbound links)
|
||||
@@ -88,18 +99,61 @@ func (WikiPage) TableName() string {
|
||||
return "wiki_pages"
|
||||
}
|
||||
|
||||
// WikiExtractionGranularity controls how aggressive Pass 0 (candidate slug
|
||||
// extraction) is. Higher granularity = more slugs, lower = tighter focus on
|
||||
// the document's main subjects.
|
||||
type WikiExtractionGranularity string
|
||||
|
||||
const (
|
||||
// WikiExtractionFocused keeps only the document's main subjects (e.g.
|
||||
// a resume yields the person + their projects, nothing else). Most
|
||||
// aggressive slug pruning; avoids index bloat from incidental technology
|
||||
// names and generic concepts.
|
||||
WikiExtractionFocused WikiExtractionGranularity = "focused"
|
||||
|
||||
// WikiExtractionStandard is the default: main subjects plus entities /
|
||||
// concepts that are substantively discussed (a dedicated paragraph or
|
||||
// multiple bullet points). Skips one-off mentions and commodity terms.
|
||||
WikiExtractionStandard WikiExtractionGranularity = "standard"
|
||||
|
||||
// WikiExtractionExhaustive extracts every named entity and recognizable
|
||||
// concept, including stacks/libs mentioned in passing. Matches the
|
||||
// pre-granularity behavior. Useful when the KB is being used as a
|
||||
// glossary rather than a curated wiki.
|
||||
WikiExtractionExhaustive WikiExtractionGranularity = "exhaustive"
|
||||
)
|
||||
|
||||
// IsValid reports whether g is one of the three recognized levels.
|
||||
func (g WikiExtractionGranularity) IsValid() bool {
|
||||
switch g {
|
||||
case WikiExtractionFocused, WikiExtractionStandard, WikiExtractionExhaustive:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Normalize returns g if valid, otherwise WikiExtractionStandard. Callers
|
||||
// pipe config through this so historical rows with empty / unknown values
|
||||
// don't surprise the extraction prompt.
|
||||
func (g WikiExtractionGranularity) Normalize() WikiExtractionGranularity {
|
||||
if g.IsValid() {
|
||||
return g
|
||||
}
|
||||
return WikiExtractionStandard
|
||||
}
|
||||
|
||||
// WikiConfig stores wiki-specific configuration for a knowledge base.
|
||||
// Applicable to document-type knowledge bases with wiki feature enabled.
|
||||
// When Enabled is true, document ingestion triggers automatic wiki page generation.
|
||||
// Whether the wiki feature is turned on is controlled by IndexingStrategy.WikiEnabled;
|
||||
// this struct only carries wiki-specific tunables.
|
||||
type WikiConfig struct {
|
||||
// Enabled activates the wiki feature for this knowledge base
|
||||
Enabled bool `yaml:"enabled" json:"enabled"`
|
||||
// AutoIngest triggers wiki page generation/update when new documents are added
|
||||
AutoIngest bool `yaml:"auto_ingest" json:"auto_ingest"`
|
||||
// SynthesisModelID is the LLM model ID used for wiki page generation and updates
|
||||
SynthesisModelID string `yaml:"synthesis_model_id" json:"synthesis_model_id"`
|
||||
// MaxPagesPerIngest limits pages created/updated per ingest operation (0 = no limit)
|
||||
MaxPagesPerIngest int `yaml:"max_pages_per_ingest" json:"max_pages_per_ingest"`
|
||||
// ExtractionGranularity controls how many candidate slugs Pass 0 extracts
|
||||
// per document. Empty / unknown value is treated as WikiExtractionStandard.
|
||||
ExtractionGranularity WikiExtractionGranularity `yaml:"extraction_granularity" json:"extraction_granularity,omitempty"`
|
||||
}
|
||||
|
||||
// Value implements the driver.Valuer interface
|
||||
@@ -122,13 +176,13 @@ func (c *WikiConfig) Scan(value interface{}) error {
|
||||
// WikiPageListRequest represents a request to list wiki pages with filtering
|
||||
type WikiPageListRequest struct {
|
||||
KnowledgeBaseID string `json:"knowledge_base_id"`
|
||||
PageType string `json:"page_type,omitempty"` // filter by type
|
||||
Status string `json:"status,omitempty"` // filter by status
|
||||
Query string `json:"query,omitempty"` // full-text search
|
||||
Page int `json:"page,omitempty"` // pagination page (1-based)
|
||||
PageSize int `json:"page_size,omitempty"` // pagination size
|
||||
SortBy string `json:"sort_by,omitempty"` // "updated_at", "created_at", "title"
|
||||
SortOrder string `json:"sort_order,omitempty"` // "asc" or "desc"
|
||||
PageType string `json:"page_type,omitempty"` // filter by type
|
||||
Status string `json:"status,omitempty"` // filter by status
|
||||
Query string `json:"query,omitempty"` // full-text search
|
||||
Page int `json:"page,omitempty"` // pagination page (1-based)
|
||||
PageSize int `json:"page_size,omitempty"` // pagination size
|
||||
SortBy string `json:"sort_by,omitempty"` // "updated_at", "created_at", "title"
|
||||
SortOrder string `json:"sort_order,omitempty"` // "asc" or "desc"
|
||||
}
|
||||
|
||||
// WikiPageListResponse represents a paginated list of wiki pages
|
||||
@@ -166,28 +220,28 @@ type WikiStats struct {
|
||||
TotalPages int64 `json:"total_pages"`
|
||||
PagesByType map[string]int64 `json:"pages_by_type"`
|
||||
TotalLinks int64 `json:"total_links"`
|
||||
OrphanCount int64 `json:"orphan_count"` // pages with no inbound links
|
||||
RecentUpdates []*WikiPage `json:"recent_updates"` // last N updated pages
|
||||
PendingTasks int64 `json:"pending_tasks"` // number of documents waiting to be ingested
|
||||
PendingIssues int64 `json:"pending_issues"` // number of pending wiki issues
|
||||
IsActive bool `json:"is_active"` // whether wiki ingestion is currently running
|
||||
OrphanCount int64 `json:"orphan_count"` // pages with no inbound links
|
||||
RecentUpdates []*WikiPage `json:"recent_updates"` // last N updated pages
|
||||
PendingTasks int64 `json:"pending_tasks"` // number of documents waiting to be ingested
|
||||
PendingIssues int64 `json:"pending_issues"` // number of pending wiki issues
|
||||
IsActive bool `json:"is_active"` // whether wiki ingestion is currently running
|
||||
}
|
||||
|
||||
// WikiPageIssue represents an issue flagged on a specific wiki page.
|
||||
// These issues are typically identified by agents or linters and stored for review.
|
||||
type WikiPageIssue struct {
|
||||
ID string `json:"id" gorm:"type:varchar(36);primaryKey"`
|
||||
TenantID uint64 `json:"tenant_id" gorm:"index"`
|
||||
KnowledgeBaseID string `json:"knowledge_base_id" gorm:"type:varchar(36);index"`
|
||||
Slug string `json:"slug" gorm:"type:varchar(255);index"`
|
||||
IssueType string `json:"issue_type" gorm:"type:varchar(50)"`
|
||||
Description string `json:"description" gorm:"type:text"`
|
||||
SuspectedKnowledgeIDs StringArray `json:"suspected_knowledge_ids" gorm:"type:json"`
|
||||
Status string `json:"status" gorm:"type:varchar(20);default:'pending';index"`
|
||||
ReportedBy string `json:"reported_by" gorm:"type:varchar(100)"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
DeletedAt gorm.DeletedAt `json:"deleted_at" gorm:"index"`
|
||||
ID string `json:"id" gorm:"type:varchar(36);primaryKey"`
|
||||
TenantID uint64 `json:"tenant_id" gorm:"index"`
|
||||
KnowledgeBaseID string `json:"knowledge_base_id" gorm:"type:varchar(36);index"`
|
||||
Slug string `json:"slug" gorm:"type:varchar(255);index"`
|
||||
IssueType string `json:"issue_type" gorm:"type:varchar(50)"`
|
||||
Description string `json:"description" gorm:"type:text"`
|
||||
SuspectedKnowledgeIDs StringArray `json:"suspected_knowledge_ids" gorm:"type:json"`
|
||||
Status string `json:"status" gorm:"type:varchar(20);default:'pending';index"`
|
||||
ReportedBy string `json:"reported_by" gorm:"type:varchar(100)"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
DeletedAt gorm.DeletedAt `json:"deleted_at" gorm:"index"`
|
||||
}
|
||||
|
||||
// TableName specifies the database table name
|
||||
|
||||
@@ -29,11 +29,8 @@ func TestWikiPageTypes(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestWikiConfigValueScan(t *testing.T) {
|
||||
// Test Value (serialize)
|
||||
config := WikiConfig{
|
||||
Enabled: true,
|
||||
AutoIngest: true,
|
||||
SynthesisModelID: "model-123",
|
||||
SynthesisModelID: "model-123",
|
||||
MaxPagesPerIngest: 20,
|
||||
}
|
||||
|
||||
@@ -42,7 +39,6 @@ func TestWikiConfigValueScan(t *testing.T) {
|
||||
t.Fatalf("WikiConfig.Value() error: %v", err)
|
||||
}
|
||||
|
||||
// Test Scan (deserialize)
|
||||
var restored WikiConfig
|
||||
b, ok := val.([]byte)
|
||||
if !ok {
|
||||
@@ -52,9 +48,6 @@ func TestWikiConfigValueScan(t *testing.T) {
|
||||
t.Fatalf("WikiConfig.Scan() error: %v", err)
|
||||
}
|
||||
|
||||
if restored.AutoIngest != true {
|
||||
t.Error("AutoIngest mismatch")
|
||||
}
|
||||
if restored.SynthesisModelID != "model-123" {
|
||||
t.Error("SynthesisModelID mismatch")
|
||||
}
|
||||
@@ -115,25 +108,21 @@ func TestStringArrayEmpty(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestKnowledgeBaseEnsureDefaultsWiki(t *testing.T) {
|
||||
kb := &KnowledgeBase{Type: KnowledgeBaseTypeDocument, WikiConfig: &WikiConfig{Enabled: true, AutoIngest: true}}
|
||||
kb := &KnowledgeBase{
|
||||
Type: KnowledgeBaseTypeDocument,
|
||||
WikiConfig: &WikiConfig{SynthesisModelID: "m-1"},
|
||||
}
|
||||
kb.EnsureDefaults()
|
||||
|
||||
if kb.WikiConfig == nil {
|
||||
t.Fatal("EnsureDefaults should preserve WikiConfig for wiki-enabled KB")
|
||||
}
|
||||
if kb.WikiConfig.AutoIngest != true {
|
||||
t.Error("AutoIngest should be preserved")
|
||||
if kb.WikiConfig.SynthesisModelID != "m-1" {
|
||||
t.Error("SynthesisModelID should be preserved")
|
||||
}
|
||||
if kb.FAQConfig != nil {
|
||||
t.Error("Document KB should not have FAQConfig")
|
||||
}
|
||||
|
||||
// Test that user can disable AutoIngest
|
||||
kb2 := &KnowledgeBase{Type: KnowledgeBaseTypeDocument, WikiConfig: &WikiConfig{Enabled: true, AutoIngest: false}}
|
||||
kb2.EnsureDefaults()
|
||||
if kb2.WikiConfig.AutoIngest != false {
|
||||
t.Error("EnsureDefaults should not override user's AutoIngest=false setting")
|
||||
}
|
||||
}
|
||||
|
||||
func TestKnowledgeBaseEnsureDefaultsDocumentWithoutWiki(t *testing.T) {
|
||||
@@ -150,16 +139,16 @@ func TestKnowledgeBaseEnsureDefaultsDocumentWithoutWiki(t *testing.T) {
|
||||
|
||||
func TestWikiPageJSON(t *testing.T) {
|
||||
page := WikiPage{
|
||||
ID: "test-id",
|
||||
Slug: "entity/test",
|
||||
Title: "Test Entity",
|
||||
PageType: WikiPageTypeEntity,
|
||||
Content: "# Test\n\nSome content with [[concept/related]]",
|
||||
Summary: "A test entity page",
|
||||
SourceRefs: StringArray{"source-1", "source-2"},
|
||||
OutLinks: StringArray{"concept/related"},
|
||||
InLinks: StringArray{"summary/doc1"},
|
||||
Version: 3,
|
||||
ID: "test-id",
|
||||
Slug: "entity/test",
|
||||
Title: "Test Entity",
|
||||
PageType: WikiPageTypeEntity,
|
||||
Content: "# Test\n\nSome content with [[concept/related]]",
|
||||
Summary: "A test entity page",
|
||||
SourceRefs: StringArray{"source-1", "source-2"},
|
||||
OutLinks: StringArray{"concept/related"},
|
||||
InLinks: StringArray{"summary/doc1"},
|
||||
Version: 3,
|
||||
}
|
||||
|
||||
// Serialize
|
||||
@@ -225,3 +214,73 @@ func TestChunkTypeWikiPage(t *testing.T) {
|
||||
t.Errorf("ChunkTypeWikiPage should be 'wiki_page', got '%s'", ChunkTypeWikiPage)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWikiExtractionGranularity_IsValid(t *testing.T) {
|
||||
valid := []WikiExtractionGranularity{
|
||||
WikiExtractionFocused,
|
||||
WikiExtractionStandard,
|
||||
WikiExtractionExhaustive,
|
||||
}
|
||||
for _, g := range valid {
|
||||
if !g.IsValid() {
|
||||
t.Errorf("%q should be valid", g)
|
||||
}
|
||||
}
|
||||
|
||||
invalid := []WikiExtractionGranularity{"", "FOCUSED", "strict", "none", "FULL"}
|
||||
for _, g := range invalid {
|
||||
if g.IsValid() {
|
||||
t.Errorf("%q should NOT be valid", g)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestWikiExtractionGranularity_Normalize(t *testing.T) {
|
||||
cases := map[WikiExtractionGranularity]WikiExtractionGranularity{
|
||||
"": WikiExtractionStandard, // legacy / unset
|
||||
"unknown": WikiExtractionStandard,
|
||||
"FOCUSED": WikiExtractionStandard, // case-sensitive on purpose
|
||||
"focused": WikiExtractionFocused,
|
||||
"standard": WikiExtractionStandard,
|
||||
"exhaustive": WikiExtractionExhaustive,
|
||||
}
|
||||
for in, want := range cases {
|
||||
if got := in.Normalize(); got != want {
|
||||
t.Errorf("Normalize(%q) = %q, want %q", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestWikiConfig_JSONRoundTrip_WithGranularity(t *testing.T) {
|
||||
original := WikiConfig{
|
||||
SynthesisModelID: "m-1",
|
||||
MaxPagesPerIngest: 20,
|
||||
ExtractionGranularity: WikiExtractionFocused,
|
||||
}
|
||||
data, err := json.Marshal(original)
|
||||
if err != nil {
|
||||
t.Fatalf("Marshal error: %v", err)
|
||||
}
|
||||
var restored WikiConfig
|
||||
if err := json.Unmarshal(data, &restored); err != nil {
|
||||
t.Fatalf("Unmarshal error: %v", err)
|
||||
}
|
||||
if restored.ExtractionGranularity != WikiExtractionFocused {
|
||||
t.Errorf("granularity round-trip failed: got %q", restored.ExtractionGranularity)
|
||||
}
|
||||
|
||||
// Legacy row without the granularity field. We also include the removed
|
||||
// "enabled" / "auto_ingest" keys to verify the decoder tolerates unknown
|
||||
// fields on rows written before those columns were retired.
|
||||
legacy := []byte(`{"enabled":true,"auto_ingest":true,"synthesis_model_id":"","max_pages_per_ingest":0}`)
|
||||
var oldCfg WikiConfig
|
||||
if err := json.Unmarshal(legacy, &oldCfg); err != nil {
|
||||
t.Fatalf("Unmarshal legacy error: %v", err)
|
||||
}
|
||||
if oldCfg.ExtractionGranularity != "" {
|
||||
t.Errorf("legacy row should decode to empty granularity, got %q", oldCfg.ExtractionGranularity)
|
||||
}
|
||||
if oldCfg.ExtractionGranularity.Normalize() != WikiExtractionStandard {
|
||||
t.Errorf("legacy row should normalize to standard")
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user