Add instruction fields to knowledge base configuration

This commit is contained in:
wizardchen
2026-07-13 11:57:30 +08:00
committed by lyingbug
parent def92bb74f
commit c1cddb4d4d
28 changed files with 634 additions and 81 deletions
+21 -6
View File
@@ -319,8 +319,9 @@ func (s *ChunkExtractService) Handle(ctx context.Context, t *asynq.Task) error {
}
template := &types.PromptTemplateStructured{
Description: s.template.Description,
Tags: extractCfg.Tags,
Description: types.AppendCustomPromptInstructions(
s.template.Description, extractCfg.CustomInstructions, "graph_extraction"),
Tags: extractCfg.Tags,
Examples: []types.GraphData{
{
Text: extractCfg.Text,
@@ -473,6 +474,7 @@ func (s *DataTableSummaryService) Handle(ctx context.Context, t *asynq.Task) err
// extractionResources 封装提取过程所需的所有资源
type extractionResources struct {
knowledge *types.Knowledge
knowledgeBase *types.KnowledgeBase
tenant *types.Tenant
chatModel chat.Chat
embeddingModel embedding.Embedder
@@ -543,6 +545,7 @@ func (s *DataTableSummaryService) prepareResources(ctx context.Context, payload
return &extractionResources{
knowledge: knowledge,
knowledgeBase: kb,
tenant: tenantInfo,
chatModel: chatModel,
embeddingModel: embeddingModel,
@@ -621,14 +624,20 @@ func (s *DataTableSummaryService) processTableData(ctx context.Context, resource
sampleDesc := s.buildSampleDataDescription(sampleResult, 10)
// 使用AI生成表格摘要和列描述
tableDescription, err := s.generateTableDescription(ctx, resources.chatModel, tableSchema.TableName, schemaDesc, sampleDesc)
customInstructions := ""
if resources.knowledgeBase != nil {
customInstructions = resources.knowledgeBase.ChunkingConfig.TableMetadataInstructions
}
tableDescription, err := s.generateTableDescription(ctx, resources.chatModel, tableSchema.TableName,
schemaDesc, sampleDesc, customInstructions)
if err != nil {
logger.Errorf(ctx, "failed to generate table description: %v", err)
return nil, err
}
logger.Debugf(ctx, "table describe of knowledge %s: %s", resources.knowledge.ID, tableDescription)
columnDescription, err := s.generateColumnDescriptions(ctx, resources.chatModel, tableSchema.TableName, schemaDesc, sampleDesc)
columnDescription, err := s.generateColumnDescriptions(ctx, resources.chatModel, tableSchema.TableName,
schemaDesc, sampleDesc, customInstructions)
if err != nil {
logger.Errorf(ctx, "failed to generate column descriptions: %v", err)
return nil, err
@@ -771,8 +780,11 @@ func (s *DataTableSummaryService) cleanupOnFailure(ctx context.Context, resource
}
// generateTableDescription generates a summary description for the entire table
func (s *DataTableSummaryService) generateTableDescription(ctx context.Context, chatModel chat.Chat, tableName, schemaDesc, sampleDesc string) (string, error) {
func (s *DataTableSummaryService) generateTableDescription(ctx context.Context, chatModel chat.Chat,
tableName, schemaDesc, sampleDesc, customInstructions string,
) (string, error) {
prompt := fmt.Sprintf(tableDescriptionPromptTemplate, tableName, schemaDesc, sampleDesc)
prompt = types.AppendCustomPromptInstructions(prompt, customInstructions, "table_metadata")
// logger.Debugf(ctx, "generateTableDescription prompt: %s", prompt)
thinking := false
@@ -791,9 +803,12 @@ func (s *DataTableSummaryService) generateTableDescription(ctx context.Context,
}
// generateColumnDescriptions generates descriptions for each column in batch
func (s *DataTableSummaryService) generateColumnDescriptions(ctx context.Context, chatModel chat.Chat, tableName, schemaDesc, sampleDesc string) (string, error) {
func (s *DataTableSummaryService) generateColumnDescriptions(ctx context.Context, chatModel chat.Chat,
tableName, schemaDesc, sampleDesc, customInstructions string,
) (string, error) {
// Build batch prompt for all columns
prompt := fmt.Sprintf(columnDescriptionsPromptTemplate, tableName, schemaDesc, sampleDesc)
prompt = types.AppendCustomPromptInstructions(prompt, customInstructions, "table_metadata")
// logger.Debugf(ctx, "generateColumnDescriptions prompt: %s", prompt)
// Call LLM once for all columns
@@ -47,9 +47,17 @@ const (
"5. Output ONLY the extracted text content. Do NOT include any HTML tags, reasoning, or unrelated comments.\n" +
"6. If there is absolutely no recognizable text content in the image, reply ONLY with: No text content.\n" +
"</instructions>"
vlmCaptionPrompt = "Provide a brief and concise description of the main content of the image in Chinese"
)
func buildVLMCaptionPrompt(ctx context.Context, cfg types.VLMConfig) string {
language := strings.TrimSpace(cfg.DescriptionLanguage)
if language == "" {
language = types.LanguageNameFromContext(ctx)
}
prompt := fmt.Sprintf("Provide a brief and concise description of the main content of the image in %s.", language)
return types.AppendCustomPromptInstructions(prompt, cfg.CustomInstructions, "image_description")
}
// ImageMultimodalService handles image:multimodal asynq tasks.
// It reads images from storage (via FileService for provider:// URLs),
// performs OCR and VLM caption, and creates child chunks.
@@ -243,6 +251,7 @@ func (s *ImageMultimodalService) Handle(ctx context.Context, task *asynq.Task) e
} else {
imgOut["ocr_prompt"] = "default"
}
prompt = types.AppendCustomPromptInstructions(prompt, vlmCfg.CustomInstructions, "image_ocr")
ocrText, ocrErr := vlmModel.Predict(ctx, [][]byte{imgBytes}, prompt)
if ocrErr != nil {
@@ -262,7 +271,7 @@ func (s *ImageMultimodalService) Handle(ctx context.Context, task *asynq.Task) e
}
}
caption, capErr := vlmModel.Predict(ctx, [][]byte{imgBytes}, vlmCaptionPrompt)
caption, capErr := vlmModel.Predict(ctx, [][]byte{imgBytes}, buildVLMCaptionPrompt(ctx, vlmCfg))
if capErr != nil {
logger.Warnf(ctx, "[ImageMultimodal] Caption failed for %s: %v", payload.ImageURL, capErr)
imgOut["caption_error"] = capErr.Error()
@@ -0,0 +1,29 @@
package service
import (
"context"
"strings"
"testing"
"github.com/Tencent/WeKnora/internal/types"
)
func TestBuildVLMCaptionPrompt(t *testing.T) {
t.Run("uses configured language and custom instructions", func(t *testing.T) {
got := buildVLMCaptionPrompt(context.Background(), types.VLMConfig{
DescriptionLanguage: "English",
CustomInstructions: "Focus on alarm codes.",
})
if !strings.Contains(got, "in English") || !strings.Contains(got, "Focus on alarm codes.") {
t.Fatalf("unexpected prompt: %s", got)
}
})
t.Run("defaults to context language", func(t *testing.T) {
ctx := context.WithValue(context.Background(), types.LanguageContextKey, "ko-KR")
got := buildVLMCaptionPrompt(ctx, types.VLMConfig{})
if !strings.Contains(got, "in Korean") {
t.Fatalf("unexpected prompt: %s", got)
}
})
}
@@ -1490,7 +1490,12 @@ func (s *knowledgeService) processQuestionGenerationForKnowledge(ctx context.Con
}
llmCallAttempts++
questions, err := s.generateQuestionsWithContext(ctx, chatModel, enrichContent(chunk), prevContent, nextContent, knowledge.Title, questionCount)
customInstructions := ""
if kb.QuestionGenerationConfig != nil {
customInstructions = kb.QuestionGenerationConfig.CustomInstructions
}
questions, err := s.generateQuestionsWithContext(ctx, chatModel, enrichContent(chunk), prevContent, nextContent,
knowledge.Title, questionCount, customInstructions)
if err != nil {
llmCallFailed++
logger.Warnf(ctx, "Failed to generate questions for chunk %s: %v", chunk.ID, err)
@@ -1815,8 +1820,13 @@ func (s *knowledgeService) processQuestionGenerationForChunks(ctx context.Contex
continue
}
customInstructions := ""
if kb.QuestionGenerationConfig != nil {
customInstructions = kb.QuestionGenerationConfig.CustomInstructions
}
questions, gerr := s.generateQuestionsWithContext(
ctx, chatModel, enrich(chunk), prevContentAt(i), nextContentAt(i), knowledge.Title, questionCount)
ctx, chatModel, enrich(chunk), prevContentAt(i), nextContentAt(i), knowledge.Title, questionCount,
customInstructions)
if gerr != nil {
llmCallFailed++
logger.Warnf(ctx, "Failed to generate questions for chunk %s: %v", chunk.ID, gerr)
@@ -1878,6 +1888,7 @@ func (s *knowledgeService) processQuestionGenerationForChunks(ctx context.Contex
// generateQuestionsWithContext generates questions for a chunk with surrounding context
func (s *knowledgeService) generateQuestionsWithContext(ctx context.Context,
chatModel chat.Chat, content, prevContent, nextContent, docName string, questionCount int,
customInstructions string,
) ([]string, error) {
if content == "" || questionCount <= 0 {
return nil, nil
@@ -1909,6 +1920,7 @@ func (s *knowledgeService) generateQuestionsWithContext(ctx context.Context,
"doc_name": docName,
"language": langName,
})
prompt = types.AppendCustomPromptInstructions(prompt, customInstructions, "question_generation")
thinking := false
response, err := chatModel.Chat(ctx, []chat.Message{
@@ -34,13 +34,24 @@ func ResolveProcessConfig(kb *types.KnowledgeBase, overrides *types.KnowledgePro
eff.EnableMultimodel = *overrides.EnableMultimodel
}
if overrides.VLMConfig != nil {
base := eff.VLMConfig
eff.VLMConfig = *overrides.VLMConfig
if eff.VLMConfig.DescriptionLanguage == "" {
eff.VLMConfig.DescriptionLanguage = base.DescriptionLanguage
}
if eff.VLMConfig.CustomInstructions == "" {
eff.VLMConfig.CustomInstructions = base.CustomInstructions
}
}
if overrides.ASRConfig != nil {
eff.ASRConfig = *overrides.ASRConfig
}
if overrides.QuestionGenerationConfig != nil {
base := eff.QuestionGenerationConfig
eff.QuestionGenerationConfig = *overrides.QuestionGenerationConfig
if eff.QuestionGenerationConfig.CustomInstructions == "" {
eff.QuestionGenerationConfig.CustomInstructions = base.CustomInstructions
}
}
if overrides.GraphEnabled != nil {
eff.GraphEnabled = *overrides.GraphEnabled
@@ -189,6 +200,9 @@ func mergeChunkingConfig(base types.ChunkingConfig, override *types.ChunkingConf
if len(override.Languages) > 0 {
result.Languages = override.Languages
}
if override.TableMetadataInstructions != "" {
result.TableMetadataInstructions = override.TableMetadataInstructions
}
return result
}
@@ -210,6 +224,9 @@ func mergeExtractConfig(base types.ExtractConfig, override *types.ExtractConfig)
if len(override.Relations) > 0 {
result.Relations = override.Relations
}
if override.CustomInstructions != "" {
result.CustomInstructions = override.CustomInstructions
}
return result
}
@@ -199,6 +199,34 @@ func TestResolveProcessConfig_ExtractConfigFieldMerge(t *testing.T) {
require.Equal(t, []string{"override-tag"}, eff.ExtractConfig.Tags)
}
func TestResolveProcessConfig_PreservesKnowledgeBasePromptInstructions(t *testing.T) {
t.Parallel()
kb := &types.KnowledgeBase{
ChunkingConfig: types.ChunkingConfig{TableMetadataInstructions: "table context"},
VLMConfig: types.VLMConfig{
Enabled: true, ModelID: "vlm-1", DescriptionLanguage: "English", CustomInstructions: "read labels",
},
QuestionGenerationConfig: &types.QuestionGenerationConfig{
Enabled: true, QuestionCount: 3, CustomInstructions: "customer questions",
},
ExtractConfig: &types.ExtractConfig{Enabled: true, CustomInstructions: "contract entities"},
}
overrides := &types.KnowledgeProcessOverrides{
ChunkingConfig: &types.ChunkingConfig{ChunkSize: 256},
VLMConfig: &types.VLMConfig{Enabled: true, ModelID: "vlm-2"},
QuestionGenerationConfig: &types.QuestionGenerationConfig{Enabled: true, QuestionCount: 5},
ExtractConfig: &types.ExtractConfig{Enabled: true},
}
eff := ResolveProcessConfig(kb, overrides)
require.Equal(t, "table context", eff.ChunkingConfig.TableMetadataInstructions)
require.Equal(t, "English", eff.VLMConfig.DescriptionLanguage)
require.Equal(t, "read labels", eff.VLMConfig.CustomInstructions)
require.Equal(t, "customer questions", eff.QuestionGenerationConfig.CustomInstructions)
require.Equal(t, "contract entities", eff.ExtractConfig.CustomInstructions)
}
func TestValidateProcessOverrides_NilOverrides(t *testing.T) {
t.Parallel()
+21 -7
View File
@@ -1087,6 +1087,12 @@ type WikiBatchContext struct {
// three valid values.
ExtractionGranularity types.WikiExtractionGranularity
// ContentInstructions and ExtractionInstructions are KB-scoped business
// guidance. Stable citation, merge, taxonomy and JSON rules remain in the
// system templates and cannot be replaced by these fields.
ContentInstructions string
ExtractionInstructions string
// PlannedFolderID holds the per-slug wiki_folders.id assigned by the batch
// taxonomy planning pass (planBatchTaxonomy + folder resolution), keyed by
// page slug. Reduce applies it only to pages that aren't already filed
@@ -1729,6 +1735,7 @@ func formatExistingTaxonomyForPrompt(paths [][]string) string {
}
return strings.TrimSpace(buf.String())
}
// getExistingPageSlugsForKnowledge returns all page slugs that currently
// reference a given knowledge ID in their source_refs. Used to snapshot
// state before re-ingest so the reduce phase can reconcile additions vs
@@ -1848,7 +1855,9 @@ const indexIntroSummaryCap = 200
//
// The intro is written to both Content and Summary so legacy readers
// that fall through to Summary stay in sync.
func (s *wikiIngestService) rebuildIndexPage(ctx context.Context, chatModel chat.Chat, payload WikiIngestPayload, changeDesc, lang string) error {
func (s *wikiIngestService) rebuildIndexPage(ctx context.Context, chatModel chat.Chat, payload WikiIngestPayload,
changeDesc, lang, customInstructions string,
) error {
indexPage, _ := s.wikiService.GetIndex(ctx, payload.KnowledgeBaseID)
if indexPage == nil {
return nil
@@ -1902,8 +1911,10 @@ func (s *wikiIngestService) rebuildIndexPage(ctx context.Context, chatModel chat
docSummaries.WriteString("(no documents yet)")
}
generatedIntro, genErr := s.generateWithTemplate(ctx, chatModel, agent.WikiIndexIntroPrompt, map[string]string{
"DocumentSummaries": framing + docSummaries.String(),
"Language": lang,
"DocumentSummaries": framing + docSummaries.String(),
"Language": lang,
"CustomInstructions": customInstructions,
"InstructionScope": "wiki_content",
})
if genErr != nil {
intro = "# Wiki Index\n\nThis wiki contains knowledge extracted from uploaded documents.\n"
@@ -1918,10 +1929,12 @@ func (s *wikiIngestService) rebuildIndexPage(ctx context.Context, chatModel chat
// change-description block already encodes the "what just
// changed" signal the prompt is asking for.
updatedIntro, genErr := s.generateWithTemplate(ctx, chatModel, agent.WikiIndexIntroUpdatePrompt, map[string]string{
"ExistingIntro": existingIntro,
"ChangeDescription": changeDesc,
"DocumentSummaries": "",
"Language": lang,
"ExistingIntro": existingIntro,
"ChangeDescription": changeDesc,
"DocumentSummaries": "",
"Language": lang,
"CustomInstructions": customInstructions,
"InstructionScope": "wiki_content",
})
if genErr != nil {
intro = existingIntro // keep existing on error
@@ -2233,6 +2246,7 @@ func (s *wikiIngestService) generateWithTemplate(ctx context.Context, chatModel
}
prompt := buf.String()
prompt = types.AppendCustomPromptInstructions(prompt, data["CustomInstructions"], data["InstructionScope"])
thinking := false
var lastErr error
@@ -70,7 +70,7 @@ func (s *wikiIngestService) scheduleFollowUp(ctx context.Context, payload WikiIn
// isolated view.
func (s *wikiIngestService) newWikiBatchContext(
kbID string,
granularity types.WikiExtractionGranularity,
wikiConfig *types.WikiConfig,
) *WikiBatchContext {
var (
fetchMu sync.Mutex
@@ -165,6 +165,14 @@ func (s *wikiIngestService) newWikiBatchContext(
return out
}
granularity := types.WikiExtractionStandard
contentInstructions := ""
extractionInstructions := ""
if wikiConfig != nil {
granularity = wikiConfig.ExtractionGranularity.Normalize()
contentInstructions = wikiConfig.ContentInstructions
extractionInstructions = wikiConfig.ExtractionInstructions
}
return &WikiBatchContext{
SlugTitle: func(ctx context.Context, slug string) string {
m := resolveSlugs(ctx, []string{slug})
@@ -175,7 +183,9 @@ func (s *wikiIngestService) newWikiBatchContext(
m := resolveSummaries(ctx, []string{kid})
return m[kid]
},
ExtractionGranularity: granularity,
ExtractionGranularity: granularity,
ContentInstructions: contentInstructions,
ExtractionInstructions: extractionInstructions,
}
}
@@ -387,12 +397,12 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
// empty/unknown values fall back to Standard via Normalize(). Failures
// to load the KB (unlikely since we're already acting on it) also
// degrade gracefully to Standard.
granularity := types.WikiExtractionStandard
var wikiConfig *types.WikiConfig
if kb, kbErr := s.kbService.GetKnowledgeBaseByID(ctx, payload.KnowledgeBaseID); kbErr == nil && kb != nil && kb.WikiConfig != nil {
granularity = kb.WikiConfig.ExtractionGranularity.Normalize()
wikiConfig = kb.WikiConfig
}
batchCtx := s.newWikiBatchContext(payload.KnowledgeBaseID, granularity)
batchCtx := s.newWikiBatchContext(payload.KnowledgeBaseID, wikiConfig)
// 1. MAP PHASE (Parallel extraction and generation of updates)
var mapMu sync.Mutex
@@ -1025,11 +1035,7 @@ func (s *wikiIngestService) ProcessWikiFinalize(ctx context.Context, t *asynq.Ta
logger.Warnf(ctx, "wiki finalize: no synthesis model for KB %s, skipping index rebuild", payload.KnowledgeBaseID)
}
granularity := types.WikiExtractionStandard
if kb.WikiConfig != nil {
granularity = kb.WikiConfig.ExtractionGranularity.Normalize()
}
batchCtx := s.newWikiBatchContext(payload.KnowledgeBaseID, granularity)
batchCtx := s.newWikiBatchContext(payload.KnowledgeBaseID, kb.WikiConfig)
lang := types.LanguageNameFromContext(ctx)
indexRebuilt := false
@@ -1037,7 +1043,8 @@ func (s *wikiIngestService) ProcessWikiFinalize(ctx context.Context, t *asynq.Ta
chatModel, mErr := s.modelService.GetChatModel(ctx, synthesisModelID)
if mErr != nil {
logger.Warnf(ctx, "wiki finalize: get chat model failed: %v", mErr)
} else if err := s.rebuildIndexPage(ctx, chatModel, payload, changeDesc.String(), lang); err != nil {
} else if err := s.rebuildIndexPage(ctx, chatModel, payload, changeDesc.String(), lang,
batchCtx.ContentInstructions); err != nil {
logger.Warnf(ctx, "wiki finalize: rebuild index failed: %v", err)
} else {
indexRebuilt = true
@@ -1244,9 +1251,11 @@ func (s *wikiIngestService) mapOneDocument(
go func() {
defer wg.Done()
summaryContent, summaryErr = s.generateWithTemplate(ctx, chatModel, agent.WikiSummaryPrompt, map[string]string{
"Content": content,
"Language": lang,
"ExtractedSlugs": slugListing,
"Content": content,
"Language": lang,
"ExtractedSlugs": slugListing,
"CustomInstructions": batchCtx.ContentInstructions,
"InstructionScope": "wiki_content",
})
if summaryErr != nil {
s.tracker().FailSpan(ctx, summarySpan, "SUMMARY_FAILED", summaryErr.Error(), summaryErr)
@@ -1541,9 +1550,11 @@ func (s *wikiIngestService) extractEntitiesAndConceptsNoUpsert(
}
extractionJSON, err := s.generateWithTemplate(ctx, chatModel, agent.WikiKnowledgeExtractPrompt, map[string]string{
"Content": content,
"Language": lang,
"PreviousSlugs": prevSlugsText,
"Content": content,
"Language": lang,
"PreviousSlugs": prevSlugsText,
"CustomInstructions": batchCtx.ExtractionInstructions,
"InstructionScope": "wiki_extraction",
})
if err != nil {
return nil, nil, nil, fmt.Errorf("combined extraction failed: %w", err)
@@ -1908,6 +1919,8 @@ func (s *wikiIngestService) reduceSlugUpdates(
"RemainingSourcesContent": remainingSourcesContent.String(),
"AvailableSlugs": relatedSlugs.String(),
"Language": language,
"CustomInstructions": batchCtx.ContentInstructions,
"InstructionScope": "wiki_content",
})
if err == nil && updatedContent != "" {
@@ -98,6 +98,8 @@ func (s *wikiIngestService) extractCandidateSlugs(
"PreviousSlugs": prevSlugsText,
"Granularity": string(granularity),
"GranularityGuidance": agent.WikiGranularityGuidance(string(granularity)),
"CustomInstructions": batchCtx.ExtractionInstructions,
"InstructionScope": "wiki_extraction",
})
if err != nil {
return nil, nil, nil, fmt.Errorf("candidate slug extraction failed: %w", err)