feat: Refactor knowledge base service and router for improved wiki handling
- Updated the knowledge base service to ensure WikiConfig is created when wiki indexing is enabled, enhancing backward compatibility. - Removed legacy syncing of WikiConfig.Enabled to IndexingStrategy.WikiEnabled, streamlining the logic for wiki enablement. - Introduced a custom retry delay function for asynq tasks to handle wiki ingest lock conflicts more effectively, improving task management. - Cleaned up the container file by removing unnecessary whitespace, contributing to better code readability.
This commit is contained in:
@@ -297,17 +297,16 @@ func (s *knowledgeBaseService) UpdateKnowledgeBase(ctx context.Context,
|
||||
if config.WikiConfig != nil {
|
||||
kb.WikiConfig = config.WikiConfig
|
||||
}
|
||||
// Update indexing strategy — syncs to WikiConfig and ExtractConfig for backward compat
|
||||
// Update indexing strategy — syncs to ExtractConfig for backward compat
|
||||
if config.IndexingStrategy != nil {
|
||||
if !config.IndexingStrategy.HasAnyIndexing() {
|
||||
return nil, errors.New("at least one indexing strategy must be enabled")
|
||||
}
|
||||
kb.IndexingStrategy = *config.IndexingStrategy
|
||||
// Sync WikiEnabled → WikiConfig
|
||||
if kb.WikiConfig != nil {
|
||||
kb.WikiConfig.Enabled = config.IndexingStrategy.WikiEnabled
|
||||
} else if config.IndexingStrategy.WikiEnabled {
|
||||
kb.WikiConfig = &types.WikiConfig{Enabled: true, AutoIngest: true}
|
||||
// Ensure WikiConfig exists when wiki indexing is enabled so that
|
||||
// wiki-specific tunables (synthesis model, granularity, …) have a home.
|
||||
if kb.WikiConfig == nil && config.IndexingStrategy.WikiEnabled {
|
||||
kb.WikiConfig = &types.WikiConfig{}
|
||||
}
|
||||
// Sync GraphEnabled → ExtractConfig
|
||||
if kb.ExtractConfig != nil {
|
||||
|
||||
@@ -1,11 +1,13 @@
|
||||
package router
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"log"
|
||||
"os"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/application/service"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
"github.com/hibiken/asynq"
|
||||
@@ -55,6 +57,29 @@ func NewAsyncqClient() (*asynq.Client, error) {
|
||||
return client, nil
|
||||
}
|
||||
|
||||
// wikiIngestRetryDelay is a fixed, short backoff for wiki ingest lock
|
||||
// conflicts. Must be slightly longer than the active-lock TTL's worst-case
|
||||
// "just got set" window so the retry is highly likely to succeed without
|
||||
// burning through retries; but short enough that users don't feel the stall.
|
||||
const wikiIngestRetryDelay = 15 * time.Second
|
||||
|
||||
// asynqRetryDelayFunc customizes per-task retry backoff.
|
||||
//
|
||||
// Default asynq backoff is exponential (≈10s, 40s, 90s, 2.5m, ...), which
|
||||
// is appropriate for transient errors like remote HTTP failures. But for
|
||||
// wiki ingest lock conflicts (ErrWikiIngestConcurrent), exponential
|
||||
// backoff is harmful: a freshly orphaned lock expires in ≤60s, so a 15s
|
||||
// fixed retry virtually guarantees the next attempt succeeds. Without
|
||||
// this override, a crash-restart cycle can leave a KB unable to make
|
||||
// progress for 7–10 minutes while the orphan lock expires AND the retry
|
||||
// schedule catches up.
|
||||
func asynqRetryDelayFunc(n int, e error, t *asynq.Task) time.Duration {
|
||||
if errors.Is(e, service.ErrWikiIngestConcurrent) {
|
||||
return wikiIngestRetryDelay
|
||||
}
|
||||
return asynq.DefaultRetryDelayFunc(n, e, t)
|
||||
}
|
||||
|
||||
func NewAsynqServer() *asynq.Server {
|
||||
opt := getAsynqRedisClientOpt()
|
||||
srv := asynq.NewServer(
|
||||
@@ -65,6 +90,7 @@ func NewAsynqServer() *asynq.Server {
|
||||
"default": 3, // Default priority queue
|
||||
"low": 1, // Lowest priority queue
|
||||
},
|
||||
RetryDelayFunc: asynqRetryDelayFunc,
|
||||
},
|
||||
)
|
||||
return srv
|
||||
|
||||
@@ -89,10 +89,6 @@ type KnowledgeService interface {
|
||||
) (*types.Knowledge, error)
|
||||
// ReparseKnowledge deletes existing document content and re-parses the knowledge asynchronously.
|
||||
ReparseKnowledge(ctx context.Context, knowledgeID string) (*types.Knowledge, error)
|
||||
// RebuildKnowledgeBaseIndex re-processes all documents in a knowledge base.
|
||||
// Used when the indexing strategy changes and existing documents need to be re-indexed.
|
||||
// Returns the number of documents enqueued for re-processing.
|
||||
RebuildKnowledgeBaseIndex(ctx context.Context, kbID string) (int, error)
|
||||
// CloneKnowledgeBase clones knowledge to another knowledge base.
|
||||
CloneKnowledgeBase(ctx context.Context, srcID, dstID string) error
|
||||
// UpdateImageInfo updates image information for a knowledge chunk.
|
||||
|
||||
@@ -496,8 +496,6 @@ func (kb *KnowledgeBase) EnsureDefaults() {
|
||||
if kb.Type != KnowledgeBaseTypeFAQ {
|
||||
kb.FAQConfig = nil
|
||||
}
|
||||
// Wiki defaults are handled at creation time, not in EnsureDefaults,
|
||||
// because AutoIngest is a user-configurable toggle (not a missing-field default).
|
||||
// Set defaults for FAQ
|
||||
if kb.Type == KnowledgeBaseTypeFAQ {
|
||||
if kb.FAQConfig == nil {
|
||||
@@ -522,10 +520,6 @@ func (kb *KnowledgeBase) EnsureDefaults() {
|
||||
if kb.IndexingStrategy.IsZero() {
|
||||
kb.IndexingStrategy = DefaultIndexingStrategy()
|
||||
}
|
||||
// Sync legacy WikiConfig.Enabled → IndexingStrategy.WikiEnabled
|
||||
if kb.WikiConfig != nil && kb.WikiConfig.Enabled && !kb.IndexingStrategy.WikiEnabled {
|
||||
kb.IndexingStrategy.WikiEnabled = true
|
||||
}
|
||||
// Sync legacy ExtractConfig.Enabled → IndexingStrategy.GraphEnabled
|
||||
if kb.ExtractConfig != nil && kb.ExtractConfig.Enabled && !kb.IndexingStrategy.GraphEnabled {
|
||||
kb.IndexingStrategy.GraphEnabled = true
|
||||
@@ -580,17 +574,12 @@ func (kb *KnowledgeBase) MarshalJSON() ([]byte, error) {
|
||||
}
|
||||
|
||||
// IsWikiEnabled checks if the wiki feature is enabled for this knowledge base.
|
||||
// Wiki requires both IndexingStrategy.WikiEnabled and WikiConfig with Enabled=true.
|
||||
// Wiki enablement is the single source of truth on IndexingStrategy.WikiEnabled.
|
||||
func (kb *KnowledgeBase) IsWikiEnabled() bool {
|
||||
if kb == nil {
|
||||
return false
|
||||
}
|
||||
// Primary: check IndexingStrategy (canonical source after migration)
|
||||
if kb.IndexingStrategy.WikiEnabled {
|
||||
return kb.WikiConfig != nil && kb.WikiConfig.Enabled
|
||||
}
|
||||
// Legacy fallback: direct WikiConfig check
|
||||
return kb.WikiConfig != nil && kb.WikiConfig.Enabled
|
||||
return kb.IndexingStrategy.WikiEnabled
|
||||
}
|
||||
|
||||
// IsVectorEnabled checks if vector (semantic) search is enabled.
|
||||
|
||||
Reference in New Issue
Block a user