Add instruction fields to knowledge base configuration
This commit is contained in:
@@ -319,8 +319,9 @@ func (s *ChunkExtractService) Handle(ctx context.Context, t *asynq.Task) error {
|
||||
}
|
||||
|
||||
template := &types.PromptTemplateStructured{
|
||||
Description: s.template.Description,
|
||||
Tags: extractCfg.Tags,
|
||||
Description: types.AppendCustomPromptInstructions(
|
||||
s.template.Description, extractCfg.CustomInstructions, "graph_extraction"),
|
||||
Tags: extractCfg.Tags,
|
||||
Examples: []types.GraphData{
|
||||
{
|
||||
Text: extractCfg.Text,
|
||||
@@ -473,6 +474,7 @@ func (s *DataTableSummaryService) Handle(ctx context.Context, t *asynq.Task) err
|
||||
// extractionResources 封装提取过程所需的所有资源
|
||||
type extractionResources struct {
|
||||
knowledge *types.Knowledge
|
||||
knowledgeBase *types.KnowledgeBase
|
||||
tenant *types.Tenant
|
||||
chatModel chat.Chat
|
||||
embeddingModel embedding.Embedder
|
||||
@@ -543,6 +545,7 @@ func (s *DataTableSummaryService) prepareResources(ctx context.Context, payload
|
||||
|
||||
return &extractionResources{
|
||||
knowledge: knowledge,
|
||||
knowledgeBase: kb,
|
||||
tenant: tenantInfo,
|
||||
chatModel: chatModel,
|
||||
embeddingModel: embeddingModel,
|
||||
@@ -621,14 +624,20 @@ func (s *DataTableSummaryService) processTableData(ctx context.Context, resource
|
||||
sampleDesc := s.buildSampleDataDescription(sampleResult, 10)
|
||||
|
||||
// 使用AI生成表格摘要和列描述
|
||||
tableDescription, err := s.generateTableDescription(ctx, resources.chatModel, tableSchema.TableName, schemaDesc, sampleDesc)
|
||||
customInstructions := ""
|
||||
if resources.knowledgeBase != nil {
|
||||
customInstructions = resources.knowledgeBase.ChunkingConfig.TableMetadataInstructions
|
||||
}
|
||||
tableDescription, err := s.generateTableDescription(ctx, resources.chatModel, tableSchema.TableName,
|
||||
schemaDesc, sampleDesc, customInstructions)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "failed to generate table description: %v", err)
|
||||
return nil, err
|
||||
}
|
||||
logger.Debugf(ctx, "table describe of knowledge %s: %s", resources.knowledge.ID, tableDescription)
|
||||
|
||||
columnDescription, err := s.generateColumnDescriptions(ctx, resources.chatModel, tableSchema.TableName, schemaDesc, sampleDesc)
|
||||
columnDescription, err := s.generateColumnDescriptions(ctx, resources.chatModel, tableSchema.TableName,
|
||||
schemaDesc, sampleDesc, customInstructions)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "failed to generate column descriptions: %v", err)
|
||||
return nil, err
|
||||
@@ -771,8 +780,11 @@ func (s *DataTableSummaryService) cleanupOnFailure(ctx context.Context, resource
|
||||
}
|
||||
|
||||
// generateTableDescription generates a summary description for the entire table
|
||||
func (s *DataTableSummaryService) generateTableDescription(ctx context.Context, chatModel chat.Chat, tableName, schemaDesc, sampleDesc string) (string, error) {
|
||||
func (s *DataTableSummaryService) generateTableDescription(ctx context.Context, chatModel chat.Chat,
|
||||
tableName, schemaDesc, sampleDesc, customInstructions string,
|
||||
) (string, error) {
|
||||
prompt := fmt.Sprintf(tableDescriptionPromptTemplate, tableName, schemaDesc, sampleDesc)
|
||||
prompt = types.AppendCustomPromptInstructions(prompt, customInstructions, "table_metadata")
|
||||
// logger.Debugf(ctx, "generateTableDescription prompt: %s", prompt)
|
||||
|
||||
thinking := false
|
||||
@@ -791,9 +803,12 @@ func (s *DataTableSummaryService) generateTableDescription(ctx context.Context,
|
||||
}
|
||||
|
||||
// generateColumnDescriptions generates descriptions for each column in batch
|
||||
func (s *DataTableSummaryService) generateColumnDescriptions(ctx context.Context, chatModel chat.Chat, tableName, schemaDesc, sampleDesc string) (string, error) {
|
||||
func (s *DataTableSummaryService) generateColumnDescriptions(ctx context.Context, chatModel chat.Chat,
|
||||
tableName, schemaDesc, sampleDesc, customInstructions string,
|
||||
) (string, error) {
|
||||
// Build batch prompt for all columns
|
||||
prompt := fmt.Sprintf(columnDescriptionsPromptTemplate, tableName, schemaDesc, sampleDesc)
|
||||
prompt = types.AppendCustomPromptInstructions(prompt, customInstructions, "table_metadata")
|
||||
// logger.Debugf(ctx, "generateColumnDescriptions prompt: %s", prompt)
|
||||
|
||||
// Call LLM once for all columns
|
||||
|
||||
@@ -47,9 +47,17 @@ const (
|
||||
"5. Output ONLY the extracted text content. Do NOT include any HTML tags, reasoning, or unrelated comments.\n" +
|
||||
"6. If there is absolutely no recognizable text content in the image, reply ONLY with: No text content.\n" +
|
||||
"</instructions>"
|
||||
vlmCaptionPrompt = "Provide a brief and concise description of the main content of the image in Chinese"
|
||||
)
|
||||
|
||||
func buildVLMCaptionPrompt(ctx context.Context, cfg types.VLMConfig) string {
|
||||
language := strings.TrimSpace(cfg.DescriptionLanguage)
|
||||
if language == "" {
|
||||
language = types.LanguageNameFromContext(ctx)
|
||||
}
|
||||
prompt := fmt.Sprintf("Provide a brief and concise description of the main content of the image in %s.", language)
|
||||
return types.AppendCustomPromptInstructions(prompt, cfg.CustomInstructions, "image_description")
|
||||
}
|
||||
|
||||
// ImageMultimodalService handles image:multimodal asynq tasks.
|
||||
// It reads images from storage (via FileService for provider:// URLs),
|
||||
// performs OCR and VLM caption, and creates child chunks.
|
||||
@@ -243,6 +251,7 @@ func (s *ImageMultimodalService) Handle(ctx context.Context, task *asynq.Task) e
|
||||
} else {
|
||||
imgOut["ocr_prompt"] = "default"
|
||||
}
|
||||
prompt = types.AppendCustomPromptInstructions(prompt, vlmCfg.CustomInstructions, "image_ocr")
|
||||
|
||||
ocrText, ocrErr := vlmModel.Predict(ctx, [][]byte{imgBytes}, prompt)
|
||||
if ocrErr != nil {
|
||||
@@ -262,7 +271,7 @@ func (s *ImageMultimodalService) Handle(ctx context.Context, task *asynq.Task) e
|
||||
}
|
||||
}
|
||||
|
||||
caption, capErr := vlmModel.Predict(ctx, [][]byte{imgBytes}, vlmCaptionPrompt)
|
||||
caption, capErr := vlmModel.Predict(ctx, [][]byte{imgBytes}, buildVLMCaptionPrompt(ctx, vlmCfg))
|
||||
if capErr != nil {
|
||||
logger.Warnf(ctx, "[ImageMultimodal] Caption failed for %s: %v", payload.ImageURL, capErr)
|
||||
imgOut["caption_error"] = capErr.Error()
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
func TestBuildVLMCaptionPrompt(t *testing.T) {
|
||||
t.Run("uses configured language and custom instructions", func(t *testing.T) {
|
||||
got := buildVLMCaptionPrompt(context.Background(), types.VLMConfig{
|
||||
DescriptionLanguage: "English",
|
||||
CustomInstructions: "Focus on alarm codes.",
|
||||
})
|
||||
if !strings.Contains(got, "in English") || !strings.Contains(got, "Focus on alarm codes.") {
|
||||
t.Fatalf("unexpected prompt: %s", got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("defaults to context language", func(t *testing.T) {
|
||||
ctx := context.WithValue(context.Background(), types.LanguageContextKey, "ko-KR")
|
||||
got := buildVLMCaptionPrompt(ctx, types.VLMConfig{})
|
||||
if !strings.Contains(got, "in Korean") {
|
||||
t.Fatalf("unexpected prompt: %s", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -1490,7 +1490,12 @@ func (s *knowledgeService) processQuestionGenerationForKnowledge(ctx context.Con
|
||||
}
|
||||
|
||||
llmCallAttempts++
|
||||
questions, err := s.generateQuestionsWithContext(ctx, chatModel, enrichContent(chunk), prevContent, nextContent, knowledge.Title, questionCount)
|
||||
customInstructions := ""
|
||||
if kb.QuestionGenerationConfig != nil {
|
||||
customInstructions = kb.QuestionGenerationConfig.CustomInstructions
|
||||
}
|
||||
questions, err := s.generateQuestionsWithContext(ctx, chatModel, enrichContent(chunk), prevContent, nextContent,
|
||||
knowledge.Title, questionCount, customInstructions)
|
||||
if err != nil {
|
||||
llmCallFailed++
|
||||
logger.Warnf(ctx, "Failed to generate questions for chunk %s: %v", chunk.ID, err)
|
||||
@@ -1815,8 +1820,13 @@ func (s *knowledgeService) processQuestionGenerationForChunks(ctx context.Contex
|
||||
continue
|
||||
}
|
||||
|
||||
customInstructions := ""
|
||||
if kb.QuestionGenerationConfig != nil {
|
||||
customInstructions = kb.QuestionGenerationConfig.CustomInstructions
|
||||
}
|
||||
questions, gerr := s.generateQuestionsWithContext(
|
||||
ctx, chatModel, enrich(chunk), prevContentAt(i), nextContentAt(i), knowledge.Title, questionCount)
|
||||
ctx, chatModel, enrich(chunk), prevContentAt(i), nextContentAt(i), knowledge.Title, questionCount,
|
||||
customInstructions)
|
||||
if gerr != nil {
|
||||
llmCallFailed++
|
||||
logger.Warnf(ctx, "Failed to generate questions for chunk %s: %v", chunk.ID, gerr)
|
||||
@@ -1878,6 +1888,7 @@ func (s *knowledgeService) processQuestionGenerationForChunks(ctx context.Contex
|
||||
// generateQuestionsWithContext generates questions for a chunk with surrounding context
|
||||
func (s *knowledgeService) generateQuestionsWithContext(ctx context.Context,
|
||||
chatModel chat.Chat, content, prevContent, nextContent, docName string, questionCount int,
|
||||
customInstructions string,
|
||||
) ([]string, error) {
|
||||
if content == "" || questionCount <= 0 {
|
||||
return nil, nil
|
||||
@@ -1909,6 +1920,7 @@ func (s *knowledgeService) generateQuestionsWithContext(ctx context.Context,
|
||||
"doc_name": docName,
|
||||
"language": langName,
|
||||
})
|
||||
prompt = types.AppendCustomPromptInstructions(prompt, customInstructions, "question_generation")
|
||||
|
||||
thinking := false
|
||||
response, err := chatModel.Chat(ctx, []chat.Message{
|
||||
|
||||
@@ -34,13 +34,24 @@ func ResolveProcessConfig(kb *types.KnowledgeBase, overrides *types.KnowledgePro
|
||||
eff.EnableMultimodel = *overrides.EnableMultimodel
|
||||
}
|
||||
if overrides.VLMConfig != nil {
|
||||
base := eff.VLMConfig
|
||||
eff.VLMConfig = *overrides.VLMConfig
|
||||
if eff.VLMConfig.DescriptionLanguage == "" {
|
||||
eff.VLMConfig.DescriptionLanguage = base.DescriptionLanguage
|
||||
}
|
||||
if eff.VLMConfig.CustomInstructions == "" {
|
||||
eff.VLMConfig.CustomInstructions = base.CustomInstructions
|
||||
}
|
||||
}
|
||||
if overrides.ASRConfig != nil {
|
||||
eff.ASRConfig = *overrides.ASRConfig
|
||||
}
|
||||
if overrides.QuestionGenerationConfig != nil {
|
||||
base := eff.QuestionGenerationConfig
|
||||
eff.QuestionGenerationConfig = *overrides.QuestionGenerationConfig
|
||||
if eff.QuestionGenerationConfig.CustomInstructions == "" {
|
||||
eff.QuestionGenerationConfig.CustomInstructions = base.CustomInstructions
|
||||
}
|
||||
}
|
||||
if overrides.GraphEnabled != nil {
|
||||
eff.GraphEnabled = *overrides.GraphEnabled
|
||||
@@ -189,6 +200,9 @@ func mergeChunkingConfig(base types.ChunkingConfig, override *types.ChunkingConf
|
||||
if len(override.Languages) > 0 {
|
||||
result.Languages = override.Languages
|
||||
}
|
||||
if override.TableMetadataInstructions != "" {
|
||||
result.TableMetadataInstructions = override.TableMetadataInstructions
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
@@ -210,6 +224,9 @@ func mergeExtractConfig(base types.ExtractConfig, override *types.ExtractConfig)
|
||||
if len(override.Relations) > 0 {
|
||||
result.Relations = override.Relations
|
||||
}
|
||||
if override.CustomInstructions != "" {
|
||||
result.CustomInstructions = override.CustomInstructions
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
|
||||
@@ -199,6 +199,34 @@ func TestResolveProcessConfig_ExtractConfigFieldMerge(t *testing.T) {
|
||||
require.Equal(t, []string{"override-tag"}, eff.ExtractConfig.Tags)
|
||||
}
|
||||
|
||||
func TestResolveProcessConfig_PreservesKnowledgeBasePromptInstructions(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
kb := &types.KnowledgeBase{
|
||||
ChunkingConfig: types.ChunkingConfig{TableMetadataInstructions: "table context"},
|
||||
VLMConfig: types.VLMConfig{
|
||||
Enabled: true, ModelID: "vlm-1", DescriptionLanguage: "English", CustomInstructions: "read labels",
|
||||
},
|
||||
QuestionGenerationConfig: &types.QuestionGenerationConfig{
|
||||
Enabled: true, QuestionCount: 3, CustomInstructions: "customer questions",
|
||||
},
|
||||
ExtractConfig: &types.ExtractConfig{Enabled: true, CustomInstructions: "contract entities"},
|
||||
}
|
||||
overrides := &types.KnowledgeProcessOverrides{
|
||||
ChunkingConfig: &types.ChunkingConfig{ChunkSize: 256},
|
||||
VLMConfig: &types.VLMConfig{Enabled: true, ModelID: "vlm-2"},
|
||||
QuestionGenerationConfig: &types.QuestionGenerationConfig{Enabled: true, QuestionCount: 5},
|
||||
ExtractConfig: &types.ExtractConfig{Enabled: true},
|
||||
}
|
||||
|
||||
eff := ResolveProcessConfig(kb, overrides)
|
||||
require.Equal(t, "table context", eff.ChunkingConfig.TableMetadataInstructions)
|
||||
require.Equal(t, "English", eff.VLMConfig.DescriptionLanguage)
|
||||
require.Equal(t, "read labels", eff.VLMConfig.CustomInstructions)
|
||||
require.Equal(t, "customer questions", eff.QuestionGenerationConfig.CustomInstructions)
|
||||
require.Equal(t, "contract entities", eff.ExtractConfig.CustomInstructions)
|
||||
}
|
||||
|
||||
func TestValidateProcessOverrides_NilOverrides(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
@@ -1087,6 +1087,12 @@ type WikiBatchContext struct {
|
||||
// three valid values.
|
||||
ExtractionGranularity types.WikiExtractionGranularity
|
||||
|
||||
// ContentInstructions and ExtractionInstructions are KB-scoped business
|
||||
// guidance. Stable citation, merge, taxonomy and JSON rules remain in the
|
||||
// system templates and cannot be replaced by these fields.
|
||||
ContentInstructions string
|
||||
ExtractionInstructions string
|
||||
|
||||
// PlannedFolderID holds the per-slug wiki_folders.id assigned by the batch
|
||||
// taxonomy planning pass (planBatchTaxonomy + folder resolution), keyed by
|
||||
// page slug. Reduce applies it only to pages that aren't already filed
|
||||
@@ -1729,6 +1735,7 @@ func formatExistingTaxonomyForPrompt(paths [][]string) string {
|
||||
}
|
||||
return strings.TrimSpace(buf.String())
|
||||
}
|
||||
|
||||
// getExistingPageSlugsForKnowledge returns all page slugs that currently
|
||||
// reference a given knowledge ID in their source_refs. Used to snapshot
|
||||
// state before re-ingest so the reduce phase can reconcile additions vs
|
||||
@@ -1848,7 +1855,9 @@ const indexIntroSummaryCap = 200
|
||||
//
|
||||
// The intro is written to both Content and Summary so legacy readers
|
||||
// that fall through to Summary stay in sync.
|
||||
func (s *wikiIngestService) rebuildIndexPage(ctx context.Context, chatModel chat.Chat, payload WikiIngestPayload, changeDesc, lang string) error {
|
||||
func (s *wikiIngestService) rebuildIndexPage(ctx context.Context, chatModel chat.Chat, payload WikiIngestPayload,
|
||||
changeDesc, lang, customInstructions string,
|
||||
) error {
|
||||
indexPage, _ := s.wikiService.GetIndex(ctx, payload.KnowledgeBaseID)
|
||||
if indexPage == nil {
|
||||
return nil
|
||||
@@ -1902,8 +1911,10 @@ func (s *wikiIngestService) rebuildIndexPage(ctx context.Context, chatModel chat
|
||||
docSummaries.WriteString("(no documents yet)")
|
||||
}
|
||||
generatedIntro, genErr := s.generateWithTemplate(ctx, chatModel, agent.WikiIndexIntroPrompt, map[string]string{
|
||||
"DocumentSummaries": framing + docSummaries.String(),
|
||||
"Language": lang,
|
||||
"DocumentSummaries": framing + docSummaries.String(),
|
||||
"Language": lang,
|
||||
"CustomInstructions": customInstructions,
|
||||
"InstructionScope": "wiki_content",
|
||||
})
|
||||
if genErr != nil {
|
||||
intro = "# Wiki Index\n\nThis wiki contains knowledge extracted from uploaded documents.\n"
|
||||
@@ -1918,10 +1929,12 @@ func (s *wikiIngestService) rebuildIndexPage(ctx context.Context, chatModel chat
|
||||
// change-description block already encodes the "what just
|
||||
// changed" signal the prompt is asking for.
|
||||
updatedIntro, genErr := s.generateWithTemplate(ctx, chatModel, agent.WikiIndexIntroUpdatePrompt, map[string]string{
|
||||
"ExistingIntro": existingIntro,
|
||||
"ChangeDescription": changeDesc,
|
||||
"DocumentSummaries": "",
|
||||
"Language": lang,
|
||||
"ExistingIntro": existingIntro,
|
||||
"ChangeDescription": changeDesc,
|
||||
"DocumentSummaries": "",
|
||||
"Language": lang,
|
||||
"CustomInstructions": customInstructions,
|
||||
"InstructionScope": "wiki_content",
|
||||
})
|
||||
if genErr != nil {
|
||||
intro = existingIntro // keep existing on error
|
||||
@@ -2233,6 +2246,7 @@ func (s *wikiIngestService) generateWithTemplate(ctx context.Context, chatModel
|
||||
}
|
||||
|
||||
prompt := buf.String()
|
||||
prompt = types.AppendCustomPromptInstructions(prompt, data["CustomInstructions"], data["InstructionScope"])
|
||||
thinking := false
|
||||
|
||||
var lastErr error
|
||||
|
||||
@@ -70,7 +70,7 @@ func (s *wikiIngestService) scheduleFollowUp(ctx context.Context, payload WikiIn
|
||||
// isolated view.
|
||||
func (s *wikiIngestService) newWikiBatchContext(
|
||||
kbID string,
|
||||
granularity types.WikiExtractionGranularity,
|
||||
wikiConfig *types.WikiConfig,
|
||||
) *WikiBatchContext {
|
||||
var (
|
||||
fetchMu sync.Mutex
|
||||
@@ -165,6 +165,14 @@ func (s *wikiIngestService) newWikiBatchContext(
|
||||
return out
|
||||
}
|
||||
|
||||
granularity := types.WikiExtractionStandard
|
||||
contentInstructions := ""
|
||||
extractionInstructions := ""
|
||||
if wikiConfig != nil {
|
||||
granularity = wikiConfig.ExtractionGranularity.Normalize()
|
||||
contentInstructions = wikiConfig.ContentInstructions
|
||||
extractionInstructions = wikiConfig.ExtractionInstructions
|
||||
}
|
||||
return &WikiBatchContext{
|
||||
SlugTitle: func(ctx context.Context, slug string) string {
|
||||
m := resolveSlugs(ctx, []string{slug})
|
||||
@@ -175,7 +183,9 @@ func (s *wikiIngestService) newWikiBatchContext(
|
||||
m := resolveSummaries(ctx, []string{kid})
|
||||
return m[kid]
|
||||
},
|
||||
ExtractionGranularity: granularity,
|
||||
ExtractionGranularity: granularity,
|
||||
ContentInstructions: contentInstructions,
|
||||
ExtractionInstructions: extractionInstructions,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -387,12 +397,12 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
|
||||
// empty/unknown values fall back to Standard via Normalize(). Failures
|
||||
// to load the KB (unlikely since we're already acting on it) also
|
||||
// degrade gracefully to Standard.
|
||||
granularity := types.WikiExtractionStandard
|
||||
var wikiConfig *types.WikiConfig
|
||||
if kb, kbErr := s.kbService.GetKnowledgeBaseByID(ctx, payload.KnowledgeBaseID); kbErr == nil && kb != nil && kb.WikiConfig != nil {
|
||||
granularity = kb.WikiConfig.ExtractionGranularity.Normalize()
|
||||
wikiConfig = kb.WikiConfig
|
||||
}
|
||||
|
||||
batchCtx := s.newWikiBatchContext(payload.KnowledgeBaseID, granularity)
|
||||
batchCtx := s.newWikiBatchContext(payload.KnowledgeBaseID, wikiConfig)
|
||||
|
||||
// 1. MAP PHASE (Parallel extraction and generation of updates)
|
||||
var mapMu sync.Mutex
|
||||
@@ -1025,11 +1035,7 @@ func (s *wikiIngestService) ProcessWikiFinalize(ctx context.Context, t *asynq.Ta
|
||||
logger.Warnf(ctx, "wiki finalize: no synthesis model for KB %s, skipping index rebuild", payload.KnowledgeBaseID)
|
||||
}
|
||||
|
||||
granularity := types.WikiExtractionStandard
|
||||
if kb.WikiConfig != nil {
|
||||
granularity = kb.WikiConfig.ExtractionGranularity.Normalize()
|
||||
}
|
||||
batchCtx := s.newWikiBatchContext(payload.KnowledgeBaseID, granularity)
|
||||
batchCtx := s.newWikiBatchContext(payload.KnowledgeBaseID, kb.WikiConfig)
|
||||
lang := types.LanguageNameFromContext(ctx)
|
||||
|
||||
indexRebuilt := false
|
||||
@@ -1037,7 +1043,8 @@ func (s *wikiIngestService) ProcessWikiFinalize(ctx context.Context, t *asynq.Ta
|
||||
chatModel, mErr := s.modelService.GetChatModel(ctx, synthesisModelID)
|
||||
if mErr != nil {
|
||||
logger.Warnf(ctx, "wiki finalize: get chat model failed: %v", mErr)
|
||||
} else if err := s.rebuildIndexPage(ctx, chatModel, payload, changeDesc.String(), lang); err != nil {
|
||||
} else if err := s.rebuildIndexPage(ctx, chatModel, payload, changeDesc.String(), lang,
|
||||
batchCtx.ContentInstructions); err != nil {
|
||||
logger.Warnf(ctx, "wiki finalize: rebuild index failed: %v", err)
|
||||
} else {
|
||||
indexRebuilt = true
|
||||
@@ -1244,9 +1251,11 @@ func (s *wikiIngestService) mapOneDocument(
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
summaryContent, summaryErr = s.generateWithTemplate(ctx, chatModel, agent.WikiSummaryPrompt, map[string]string{
|
||||
"Content": content,
|
||||
"Language": lang,
|
||||
"ExtractedSlugs": slugListing,
|
||||
"Content": content,
|
||||
"Language": lang,
|
||||
"ExtractedSlugs": slugListing,
|
||||
"CustomInstructions": batchCtx.ContentInstructions,
|
||||
"InstructionScope": "wiki_content",
|
||||
})
|
||||
if summaryErr != nil {
|
||||
s.tracker().FailSpan(ctx, summarySpan, "SUMMARY_FAILED", summaryErr.Error(), summaryErr)
|
||||
@@ -1541,9 +1550,11 @@ func (s *wikiIngestService) extractEntitiesAndConceptsNoUpsert(
|
||||
}
|
||||
|
||||
extractionJSON, err := s.generateWithTemplate(ctx, chatModel, agent.WikiKnowledgeExtractPrompt, map[string]string{
|
||||
"Content": content,
|
||||
"Language": lang,
|
||||
"PreviousSlugs": prevSlugsText,
|
||||
"Content": content,
|
||||
"Language": lang,
|
||||
"PreviousSlugs": prevSlugsText,
|
||||
"CustomInstructions": batchCtx.ExtractionInstructions,
|
||||
"InstructionScope": "wiki_extraction",
|
||||
})
|
||||
if err != nil {
|
||||
return nil, nil, nil, fmt.Errorf("combined extraction failed: %w", err)
|
||||
@@ -1908,6 +1919,8 @@ func (s *wikiIngestService) reduceSlugUpdates(
|
||||
"RemainingSourcesContent": remainingSourcesContent.String(),
|
||||
"AvailableSlugs": relatedSlugs.String(),
|
||||
"Language": language,
|
||||
"CustomInstructions": batchCtx.ContentInstructions,
|
||||
"InstructionScope": "wiki_content",
|
||||
})
|
||||
|
||||
if err == nil && updatedContent != "" {
|
||||
|
||||
@@ -98,6 +98,8 @@ func (s *wikiIngestService) extractCandidateSlugs(
|
||||
"PreviousSlugs": prevSlugsText,
|
||||
"Granularity": string(granularity),
|
||||
"GranularityGuidance": agent.WikiGranularityGuidance(string(granularity)),
|
||||
"CustomInstructions": batchCtx.ExtractionInstructions,
|
||||
"InstructionScope": "wiki_extraction",
|
||||
})
|
||||
if err != nil {
|
||||
return nil, nil, nil, fmt.Errorf("candidate slug extraction failed: %w", err)
|
||||
|
||||
Reference in New Issue
Block a user