feat: Add pending tasks and active status to WikiStats

- Introduced `pending_tasks` and `is_active` fields in the `WikiStats` interface to track the number of tasks waiting for processing and the current activity status of the wiki ingestion.
- Updated localization files for English, Korean, Russian, and Chinese to include a new message for displaying the number of pending tasks in the wiki queue.
- Enhanced the `WikiBrowser` component to display the queue status, improving user awareness of ongoing tasks.
- Implemented polling logic to refresh wiki stats periodically when there are pending tasks or active ingestion, ensuring up-to-date information is presented.

These changes enhance the functionality and user experience of the wiki management system by providing real-time insights into task status.
This commit is contained in:
wizardchen
2026-04-09 13:35:46 +08:00
parent a81ef5fbb2
commit 8daf6b368c
17 changed files with 682 additions and 229 deletions
+2
View File
@@ -40,6 +40,8 @@ export interface WikiStats {
total_links: number;
orphan_count: number;
recent_updates: WikiPage[];
pending_tasks: number;
is_active: boolean;
}
// Wiki API Functions
+1
View File
@@ -1636,6 +1636,7 @@ export default {
showArrows: 'Show arrows',
hideArrows: 'Hide arrows',
healthCheck: 'Health Check',
queueStatus: '{count} pending tasks in Wiki queue',
},
buttons: {
create: 'Create Knowledge Base',
+1
View File
@@ -2159,6 +2159,7 @@ export default {
showArrows: '화살표 표시',
hideArrows: '화살표 숨기기',
healthCheck: "상태 점검",
queueStatus: "Wiki 대기열에 {count}개의 보류 중인 작업",
},
buttons: {
create: "지식베이스 생성",
+1
View File
@@ -1838,6 +1838,7 @@ export default {
showArrows: 'Показать стрелки',
hideArrows: 'Скрыть стрелки',
healthCheck: 'Проверка здоровья',
queueStatus: '{count} ожидающих задач в очереди Wiki',
},
buttons: {
create: 'Создать базу знаний',
+1
View File
@@ -2144,6 +2144,7 @@ export default {
showArrows: '显示箭头',
hideArrows: '隐藏箭头',
healthCheck: "健康检查",
queueStatus: "Wiki 队列中 {count} 个待处理任务",
},
buttons: {
create: "创建知识库",
@@ -91,6 +91,10 @@
<!-- Left Panel: Page List -->
<aside class="wiki-sidebar">
<div class="wiki-sidebar-header">
<div v-if="stats && (stats.pending_tasks > 0 || stats.is_active)" class="wiki-queue-status">
<t-loading size="small" />
<span class="queue-text">{{ $t('knowledgeEditor.wikiBrowser.queueStatus', { count: stats.pending_tasks || 0 }) }}</span>
</div>
<t-input
v-model="searchQuery"
:placeholder="$t('knowledgeEditor.wikiBrowser.searchPlaceholder')"
@@ -255,7 +259,7 @@
</template>
<script setup lang="ts">
import { ref, computed, onMounted, watch, nextTick, reactive } from 'vue'
import { ref, computed, onMounted, onUnmounted, watch, nextTick, reactive } from 'vue'
import { useI18n } from 'vue-i18n'
import { marked } from 'marked'
import { hydrateProtectedFileImages } from '@/utils/security'
@@ -477,10 +481,26 @@ async function loadPages() {
}
}
let statsTimer: ReturnType<typeof setInterval> | null = null
async function loadStats() {
try {
const res = await getWikiStats(props.knowledgeBaseId)
stats.value = (res as any).data || res as any
// Poll if there are pending tasks or wiki ingest is active
if (stats.value && (stats.value.pending_tasks > 0 || stats.value.is_active)) {
if (!statsTimer) {
statsTimer = setInterval(() => {
loadStats()
}, 5000)
}
} else if (statsTimer) {
// If completed, clear timer and reload pages once to get new content
clearInterval(statsTimer)
statsTimer = null
loadPages()
}
} catch (e) { /* ignore */ }
}
@@ -1300,6 +1320,12 @@ onMounted(() => {
loadStats()
if (props.view === 'graph') loadGraph()
})
onUnmounted(() => {
if (statsTimer) {
clearInterval(statsTimer)
}
})
</script>
<style scoped lang="less">
@@ -1323,6 +1349,24 @@ onMounted(() => {
.wiki-sidebar-header {
padding: 16px 16px 12px;
display: flex;
flex-direction: column;
gap: 12px;
}
.wiki-queue-status {
display: flex;
align-items: center;
gap: 8px;
padding: 8px 12px;
background: var(--td-bg-color-secondarycontainer);
border-radius: 6px;
color: var(--td-text-color-secondary);
font-size: 13px;
.queue-text {
line-height: 1.2;
}
}
.wiki-page-list {
+14 -6
View File
@@ -85,6 +85,9 @@ Only include concepts that are substantively discussed. Skip trivial or overly g
- If something is a specific named thing (person, company, product, place), put it ONLY in "entities".
- If something is an abstract idea, methodology, or theory, put it ONLY in "concepts".
- Never duplicate items across the two arrays.
### JSON Formatting Rules
- **CRITICAL**: Do NOT use literal newline characters inside JSON string values. If you need a newline in a string, you MUST use the escaped sequence \n.
</instructions>
Output ONLY valid JSON. Example:
@@ -139,6 +142,8 @@ const WikiPageUpdatePrompt = `You are a wiki editor tasked with updating an exis
Output the SUMMARY line first, then the updated Markdown content. Do not include any other preamble.`
// WikiPageRetractPrompt removes information contributed by a deleted document from an existing wiki page.
// It includes the actual content of remaining source documents (from their summary pages) so the LLM
// can accurately determine which facts are still supported and which should be removed.
const WikiPageRetractPrompt = `You are a wiki editor. A source document has been DELETED from the knowledge base. You must update the existing wiki page to remove any information that came exclusively from this deleted document.
<existing_page_content>
@@ -152,9 +157,9 @@ const WikiPageRetractPrompt = `You are a wiki editor. A source document has been
</content>
</deleted_document>
<remaining_sources>
{{.RemainingSources}}
</remaining_sources>
<remaining_source_documents>
{{.RemainingSourcesContent}}
</remaining_source_documents>
<valid_wiki_links>
{{.AvailableSlugs}}
@@ -162,9 +167,9 @@ const WikiPageRetractPrompt = `You are a wiki editor. A source document has been
<instructions>
1. The FIRST line of your output MUST be: SUMMARY: {one sentence, 15-40 words, describing what this page is about after retraction — for wiki index listing}
2. Carefully review the existing page content and compare it with the deleted document content above.
3. Remove any facts, claims, or details that were ONLY sourced from the deleted document.
4. If a fact is also supported by the remaining source documents, KEEP it.
2. Carefully review the existing page content. Compare it against BOTH the deleted document and the remaining source documents above.
3. Remove any facts, claims, or details that were ONLY sourced from the deleted document and are NOT present in any remaining source document.
4. If a fact appears in both the deleted document and a remaining source, KEEP it.
5. If you are unsure whether a fact came from the deleted document, keep it as-is — do NOT add any review notes or annotations.
6. Update or remove the "Source: {{.DeletedDocTitle}}" reference line if present.
7. Keep [[slug|name]] wiki-link references ONLY if the slug appears in the <valid_wiki_links> list above. Remove any [[slug|name]] whose slug is NOT in that list.
@@ -284,6 +289,9 @@ For each newly extracted item, check if it refers to the same real-world entity
Return a JSON object with a "merges" map. The key is the NEW item's slug, the value is the EXISTING page's slug that it should merge into. Only include items that have a match.
If no items match any existing pages, return: {"merges": {}}
### JSON Formatting Rules
- **CRITICAL**: Do NOT use literal newline characters inside JSON string values. If you need a newline in a string, you MUST use the escaped sequence \n.
</instructions>
Output ONLY valid JSON. Example:
@@ -154,9 +154,6 @@ func (r *wikiPageRepository) List(ctx context.Context, req *types.WikiPageListRe
if pageSize < 1 {
pageSize = 20
}
if pageSize > 100 {
pageSize = 100
}
offset := (page - 1) * pageSize
query = query.Offset(offset).Limit(pageSize)
+74 -11
View File
@@ -1331,7 +1331,7 @@ func (s *knowledgeService) cleanupWikiOnKnowledgeDelete(ctx context.Context, kbI
if len(retractSlugs) > 0 {
lang, _ := types.LanguageFromContext(ctx)
tenantID, _ := types.TenantIDFromContext(ctx)
EnqueueWikiRetract(ctx, s.task, WikiRetractPayload{
EnqueueWikiRetract(ctx, s.task, s.redisClient, WikiRetractPayload{
TenantID: tenantID,
KnowledgeBaseID: kbID,
KnowledgeID: knowledgeID,
@@ -2494,11 +2494,53 @@ func (s *knowledgeService) ProcessSummaryGeneration(ctx context.Context, t *asyn
// ProcessQuestionGeneration handles async question generation task
func (s *knowledgeService) ProcessQuestionGeneration(ctx context.Context, t *asynq.Task) error {
ctx, span := tracing.ContextWithSpan(ctx, "knowledgeService.ProcessQuestionGeneration")
defer span.End()
taskStartedAt := time.Now()
retryCount, _ := asynq.GetRetryCount(ctx)
maxRetry, _ := asynq.GetMaxRetry(ctx)
var payload types.QuestionGenerationPayload
exitStatus := "success"
totalChunks := 0
totalTextChunks := 0
emptyContentChunks := 0
llmCallAttempts := 0
llmCallSuccess := 0
llmCallFailed := 0
llmCallEmpty := 0
generatedQuestionsTotal := 0
chunkMetadataSetFailed := 0
chunkUpdateFailed := 0
indexEntriesPrepared := 0
indexBatchAttempted := false
indexBatchSucceeded := false
defer func() {
logger.Infof(
ctx,
"Question generation stats: knowledge=%s kb=%s retry=%d/%d status=%s elapsed=%s chunks(total=%d,text=%d,empty_text=%d) llm(attempt=%d,success=%d,empty=%d,failed=%d) generated_questions=%d chunk_update_failed=%d metadata_set_failed=%d index(prepared=%d,attempted=%v,succeeded=%v)",
payload.KnowledgeID,
payload.KnowledgeBaseID,
retryCount,
maxRetry,
exitStatus,
time.Since(taskStartedAt).Round(time.Millisecond),
totalChunks,
totalTextChunks,
emptyContentChunks,
llmCallAttempts,
llmCallSuccess,
llmCallEmpty,
llmCallFailed,
generatedQuestionsTotal,
chunkUpdateFailed,
chunkMetadataSetFailed,
indexEntriesPrepared,
indexBatchAttempted,
indexBatchSucceeded,
)
}()
if err := json.Unmarshal(t.Payload(), &payload); err != nil {
exitStatus = "invalid_payload"
logger.Errorf(ctx, "Failed to unmarshal question generation payload: %v", err)
return nil // Don't retry on unmarshal error
}
@@ -2512,6 +2554,7 @@ func (s *knowledgeService) ProcessQuestionGeneration(ctx context.Context, t *asy
}
if strings.TrimSpace(s.config.Conversation.GenerateQuestionsPrompt) == "" {
exitStatus = "prompt_not_configured"
logger.Errorf(ctx, "GenerateQuestionsPrompt is empty: configure conversation.generate_questions_prompt_id")
return fmt.Errorf("generate questions prompt not configured")
}
@@ -2519,6 +2562,7 @@ func (s *knowledgeService) ProcessQuestionGeneration(ctx context.Context, t *asy
// Get knowledge base
kb, err := s.kbService.GetKnowledgeBaseByID(ctx, payload.KnowledgeBaseID)
if err != nil {
exitStatus = "kb_not_found"
logger.Errorf(ctx, "Failed to get knowledge base: %v", err)
return nil
}
@@ -2526,6 +2570,7 @@ func (s *knowledgeService) ProcessQuestionGeneration(ctx context.Context, t *asy
// Get knowledge
knowledge, err := s.repo.GetKnowledgeByID(ctx, payload.TenantID, payload.KnowledgeID)
if err != nil {
exitStatus = "knowledge_not_found"
logger.Errorf(ctx, "Failed to get knowledge: %v", err)
return nil
}
@@ -2533,9 +2578,11 @@ func (s *knowledgeService) ProcessQuestionGeneration(ctx context.Context, t *asy
// Get text chunks for this knowledge
chunks, err := s.chunkService.ListChunksByKnowledgeID(ctx, payload.KnowledgeID)
if err != nil {
exitStatus = "list_chunks_failed"
logger.Errorf(ctx, "Failed to get chunks: %v", err)
return nil
}
totalChunks = len(chunks)
// Filter text chunks only
textChunks := make([]*types.Chunk, 0)
@@ -2544,8 +2591,10 @@ func (s *knowledgeService) ProcessQuestionGeneration(ctx context.Context, t *asy
textChunks = append(textChunks, chunk)
}
}
totalTextChunks = len(textChunks)
if len(textChunks) == 0 {
exitStatus = "no_text_chunks"
logger.Infof(ctx, "No text chunks found for knowledge: %s", payload.KnowledgeID)
return nil
}
@@ -2558,6 +2607,7 @@ func (s *knowledgeService) ProcessQuestionGeneration(ctx context.Context, t *asy
// Initialize chat model
chatModel, err := s.modelService.GetChatModel(ctx, kb.SummaryModelID)
if err != nil {
exitStatus = "get_chat_model_failed"
logger.Errorf(ctx, "Failed to get chat model: %v", err)
return fmt.Errorf("failed to get chat model: %w", err)
}
@@ -2565,12 +2615,14 @@ func (s *knowledgeService) ProcessQuestionGeneration(ctx context.Context, t *asy
// Initialize embedding model and retrieval engine
embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, kb.EmbeddingModelID)
if err != nil {
exitStatus = "get_embedding_model_failed"
logger.Errorf(ctx, "Failed to get embedding model: %v", err)
return fmt.Errorf("failed to get embedding model: %w", err)
}
tenantInfo, err := s.tenantRepo.GetTenantByID(ctx, payload.TenantID)
if err != nil {
exitStatus = "get_tenant_failed"
logger.Errorf(ctx, "Failed to get tenant info: %v", err)
return fmt.Errorf("failed to get tenant info: %w", err)
}
@@ -2578,6 +2630,7 @@ func (s *knowledgeService) ProcessQuestionGeneration(ctx context.Context, t *asy
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines())
if err != nil {
exitStatus = "init_retrieve_engine_failed"
logger.Errorf(ctx, "Failed to init retrieve engine: %v", err)
return fmt.Errorf("failed to init retrieve engine: %w", err)
}
@@ -2608,30 +2661,34 @@ func (s *knowledgeService) ProcessQuestionGeneration(ctx context.Context, t *asy
// Generate questions for each chunk with context
var indexInfoList []*types.IndexInfo
for i, chunk := range textChunks {
if strings.TrimSpace(chunk.Content) == "" {
emptyContentChunks++
continue
}
// Build context from adjacent chunks
var prevContent, nextContent string
if i > 0 {
prevContent = enrichContent(textChunks[i-1])
if len(prevContent) > 500 {
prevContent = prevContent[len(prevContent)-500:]
}
}
if i < len(textChunks)-1 {
nextContent = enrichContent(textChunks[i+1])
if len(nextContent) > 500 {
nextContent = nextContent[:500]
}
}
llmCallAttempts++
questions, err := s.generateQuestionsWithContext(ctx, chatModel, enrichContent(chunk), prevContent, nextContent, knowledge.Title, questionCount)
if err != nil {
llmCallFailed++
logger.Warnf(ctx, "Failed to generate questions for chunk %s: %v", chunk.ID, err)
continue
}
if len(questions) == 0 {
llmCallEmpty++
continue
}
llmCallSuccess++
generatedQuestionsTotal += len(questions)
// Update chunk metadata with unique IDs for each question
generatedQuestions := make([]types.GeneratedQuestion, len(questions))
@@ -2646,12 +2703,14 @@ func (s *knowledgeService) ProcessQuestionGeneration(ctx context.Context, t *asy
GeneratedQuestions: generatedQuestions,
}
if err := chunk.SetDocumentMetadata(meta); err != nil {
chunkMetadataSetFailed++
logger.Warnf(ctx, "Failed to set document metadata for chunk %s: %v", chunk.ID, err)
continue
}
// Update chunk in database
if err := s.chunkService.UpdateChunk(ctx, chunk); err != nil {
chunkUpdateFailed++
logger.Warnf(ctx, "Failed to update chunk %s: %v", chunk.ID, err)
continue
}
@@ -2671,13 +2730,17 @@ func (s *knowledgeService) ProcessQuestionGeneration(ctx context.Context, t *asy
}
logger.Debugf(ctx, "Generated %d questions for chunk %s", len(questions), chunk.ID)
}
indexEntriesPrepared = len(indexInfoList)
// Index generated questions
if len(indexInfoList) > 0 {
indexBatchAttempted = true
if err := retrieveEngine.BatchIndex(ctx, embeddingModel, indexInfoList); err != nil {
exitStatus = "index_questions_failed"
logger.Errorf(ctx, "Failed to index generated questions: %v", err)
return fmt.Errorf("failed to index questions: %w", err)
}
indexBatchSucceeded = true
logger.Infof(ctx, "Successfully indexed %d generated questions for knowledge: %s", len(indexInfoList), payload.KnowledgeID)
}
@@ -2702,10 +2765,10 @@ func (s *knowledgeService) generateQuestionsWithContext(ctx context.Context,
if prevContent != "" || nextContent != "" {
contextSection = "<surrounding_context>\n"
if prevContent != "" {
contextSection += fmt.Sprintf("[Preceding Content]\n%s\n\n", prevContent)
contextSection += fmt.Sprintf("<preceding_content>\n%s\n\n</preceding_content>\n\n", prevContent)
}
if nextContent != "" {
contextSection += fmt.Sprintf("[Following Content]\n%s\n\n", nextContent)
contextSection += fmt.Sprintf("<following_content>\n%s\n\n</following_content>\n\n", nextContent)
}
contextSection += "</surrounding_context>\n\n"
}
File diff suppressed because it is too large Load Diff
+20 -6
View File
@@ -13,6 +13,7 @@ import (
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/google/uuid"
"github.com/redis/go-redis/v9"
)
// wikiLinkRegex matches [[wiki-link]] syntax in markdown content
@@ -20,9 +21,10 @@ var wikiLinkRegex = regexp.MustCompile(`\[\[([^\]]+)\]\]`)
// wikiPageService implements the WikiPageService interface
type wikiPageService struct {
repo interfaces.WikiPageRepository
chunkRepo interfaces.ChunkRepository
kbService interfaces.KnowledgeBaseService
repo interfaces.WikiPageRepository
chunkRepo interfaces.ChunkRepository
kbService interfaces.KnowledgeBaseService
redisClient *redis.Client
}
// NewWikiPageService creates a new wiki page service
@@ -30,11 +32,13 @@ func NewWikiPageService(
repo interfaces.WikiPageRepository,
chunkRepo interfaces.ChunkRepository,
kbService interfaces.KnowledgeBaseService,
redisClient *redis.Client,
) interfaces.WikiPageService {
return &wikiPageService{
repo: repo,
chunkRepo: chunkRepo,
kbService: kbService,
repo: repo,
chunkRepo: chunkRepo,
kbService: kbService,
redisClient: redisClient,
}
}
@@ -283,12 +287,22 @@ func (s *wikiPageService) GetStats(ctx context.Context, kbID string) (*types.Wik
return nil, err
}
var pendingTasks int64
var isActive bool
if s.redisClient != nil {
pendingTasks, _ = s.redisClient.LLen(ctx, "wiki:pending:"+kbID).Result()
activeFlag, _ := s.redisClient.Exists(ctx, "wiki:active:"+kbID).Result()
isActive = activeFlag > 0
}
return &types.WikiStats{
TotalPages: total,
PagesByType: counts,
TotalLinks: totalLinks,
OrphanCount: orphans,
RecentUpdates: recentPages,
PendingTasks: pendingTasks,
IsActive: isActive,
}, nil
}
@@ -37,8 +37,8 @@ func TestIsIconImage(t *testing.T) {
expect bool
}{
{
name: "tiny bytes (< 2KB)",
data: make([]byte, 1024),
name: "tiny bytes (< 512B)",
data: make([]byte, 256),
expect: true,
},
{
-1
View File
@@ -102,6 +102,5 @@ func RegisterSyncHandlers(params SyncTaskParams) {
params.Executor.RegisterHandler(types.TypeKnowledgePostProcess, params.KnowledgePostProcess.Handle)
params.Executor.RegisterHandler(types.TypeDataSourceSync, params.DataSourceService.ProcessSync)
params.Executor.RegisterHandler(types.TypeWikiIngest, params.WikiIngest.Handle)
params.Executor.RegisterHandler(types.TypeWikiRetract, params.WikiIngest.Handle)
logger.Infof(context.Background(), "[SyncTask] All task handlers registered (Lite mode, no Redis)")
}
-1
View File
@@ -119,7 +119,6 @@ func RunAsynqServer(params AsynqTaskParams) *asynq.ServeMux {
// Register wiki ingest handler
mux.HandleFunc(types.TypeWikiIngest, params.WikiIngest.Handle)
mux.HandleFunc(types.TypeWikiRetract, params.WikiIngest.Handle)
go func() {
// Start the server
-1
View File
@@ -17,7 +17,6 @@ const (
TypeManualProcess = "manual:process" // 手工知识更新任务(cleanup + 重新索引)
TypeDataSourceSync = "datasource:sync" // 数据源同步任务
TypeWikiIngest = "wiki:ingest" // Wiki 页面同步任务
TypeWikiRetract = "wiki:retract" // Wiki 内容回收任务(文档删除时)
)
// ExtractChunkPayload represents the extract chunk task payload
+2
View File
@@ -168,4 +168,6 @@ type WikiStats struct {
TotalLinks int64 `json:"total_links"`
OrphanCount int64 `json:"orphan_count"` // pages with no inbound links
RecentUpdates []*WikiPage `json:"recent_updates"` // last N updated pages
PendingTasks int64 `json:"pending_tasks"` // number of documents waiting to be ingested
IsActive bool `json:"is_active"` // whether wiki ingestion is currently running
}
@@ -23,6 +23,7 @@ CREATE TABLE IF NOT EXISTS wiki_pages (
in_links JSONB DEFAULT '[]'::JSONB,
out_links JSONB DEFAULT '[]'::JSONB,
page_metadata JSONB DEFAULT '{}'::JSONB,
aliases JSONB DEFAULT '[]'::JSONB,
version INT NOT NULL DEFAULT 1,
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),