Enhance knowledge base configuration by normalizing and validating custom instruction fields. Implemented length checks for prompt instructions and added tests for validation logic. Updated GraphSettings to retain custom instructions when extraction is disabled.

This commit is contained in:
wizardchen
2026-07-13 12:14:36 +08:00
committed by lyingbug
parent c1cddb4d4d
commit 5eefa70e6f
9 changed files with 158 additions and 35 deletions
@@ -403,13 +403,12 @@ const handleConfigChange = () => {
// 处理启用/禁用切换
const handleEnabledChange = () => {
// 当关闭提取功能时,清空所有数据
// 当关闭提取功能时,清空示例数据,但保留自定义指令以便再次启用时恢复。
if (!localGraphExtract.value.enabled) {
localGraphExtract.value.text = ''
localGraphExtract.value.tags = []
localGraphExtract.value.nodes = []
localGraphExtract.value.relations = []
localGraphExtract.value.customInstructions = ''
}
handleConfigChange()
}
+5 -1
View File
@@ -626,7 +626,11 @@ func (s *DataTableSummaryService) processTableData(ctx context.Context, resource
// 使用AI生成表格摘要和列描述
customInstructions := ""
if resources.knowledgeBase != nil {
customInstructions = resources.knowledgeBase.ChunkingConfig.TableMetadataInstructions
var processOverrides *types.KnowledgeProcessOverrides
if resources.knowledge != nil {
processOverrides, _ = resources.knowledge.ProcessOverrides()
}
customInstructions = ResolveProcessConfig(resources.knowledgeBase, processOverrides).ChunkingConfig.TableMetadataInstructions
}
tableDescription, err := s.generateTableDescription(ctx, resources.chatModel, tableSchema.TableName,
schemaDesc, sampleDesc, customInstructions)
@@ -1457,6 +1457,10 @@ func (s *knowledgeService) processQuestionGenerationForKnowledge(ctx context.Con
questionCount = 10
}
processOverrides, _ := knowledge.ProcessOverrides()
questionGenCfg := ResolveProcessConfig(kb, processOverrides).QuestionGenerationConfig
customInstructions := questionGenCfg.CustomInstructions
// Collect image info for all text chunks so question generation can
// see caption / OCR text instead of bare image links.
textChunkIDs := make([]string, len(textChunks))
@@ -1490,10 +1494,6 @@ func (s *knowledgeService) processQuestionGenerationForKnowledge(ctx context.Con
}
llmCallAttempts++
customInstructions := ""
if kb.QuestionGenerationConfig != nil {
customInstructions = kb.QuestionGenerationConfig.CustomInstructions
}
questions, err := s.generateQuestionsWithContext(ctx, chatModel, enrichContent(chunk), prevContent, nextContent,
knowledge.Title, questionCount, customInstructions)
if err != nil {
@@ -1758,6 +1758,10 @@ func (s *knowledgeService) processQuestionGenerationForChunks(ctx context.Contex
questionCount = 10
}
processOverrides, _ := knowledge.ProcessOverrides()
questionGenCfg := ResolveProcessConfig(kb, processOverrides).QuestionGenerationConfig
customInstructions := questionGenCfg.CustomInstructions
// Fetch the batch chunks (in payload order) plus the two boundary
// neighbors so we can rebuild the same surrounding context the legacy
// loop used, all enriched with image OCR / caption info. A vanished
@@ -1820,10 +1824,6 @@ func (s *knowledgeService) processQuestionGenerationForChunks(ctx context.Contex
continue
}
customInstructions := ""
if kb.QuestionGenerationConfig != nil {
customInstructions = kb.QuestionGenerationConfig.CustomInstructions
}
questions, gerr := s.generateQuestionsWithContext(
ctx, chatModel, enrich(chunk), prevContentAt(i), nextContentAt(i), knowledge.Title, questionCount,
customInstructions)
@@ -103,6 +103,10 @@ func ValidateProcessOverrides(
return werrors.NewBadRequestError("上传音频文件需要设置ASR语音识别模型")
}
if err := types.ValidateEffectiveProcessPromptInstructions(eff); err != nil {
return werrors.NewBadRequestError(err.Error())
}
return nil
}
@@ -2,6 +2,7 @@ package service
import (
"context"
"strings"
"testing"
werrors "github.com/Tencent/WeKnora/internal/errors"
@@ -227,6 +228,21 @@ func TestResolveProcessConfig_PreservesKnowledgeBasePromptInstructions(t *testin
require.Equal(t, "contract entities", eff.ExtractConfig.CustomInstructions)
}
func TestValidateProcessOverrides_RejectsOversizedInstructions(t *testing.T) {
t.Parallel()
kb := &types.KnowledgeBase{}
overrides := &types.KnowledgeProcessOverrides{
VLMConfig: &types.VLMConfig{
CustomInstructions: strings.Repeat("x", types.MaxCustomPromptInstructionsLength+1),
},
}
err := ValidateProcessOverrides(context.Background(), kb, overrides, []string{"txt"})
require.Error(t, err)
var badReq *werrors.AppError
require.ErrorAs(t, err, &badReq)
}
func TestValidateProcessOverrides_NilOverrides(t *testing.T) {
t.Parallel()
+37 -1
View File
@@ -404,6 +404,8 @@ func (h *InitializationHandler) UpdateKBConfig(c *gin.Context) {
Relations: relations,
CustomInstructions: strings.TrimSpace(req.NodeExtract.CustomInstructions),
}
} else if kb.ExtractConfig != nil {
kb.ExtractConfig.Enabled = false
} else {
kb.ExtractConfig = &types.ExtractConfig{Enabled: false}
}
@@ -433,6 +435,7 @@ func (h *InitializationHandler) UpdateKBConfig(c *gin.Context) {
CustomInstructions: strings.TrimSpace(req.QuestionGeneration.CustomInstructions),
}
}
types.NormalizeKnowledgeBasePromptInstructions(kb)
if err := validateKnowledgeBasePromptInstructions(kb); err != nil {
c.Error(err)
return
@@ -1470,6 +1473,20 @@ func (h *InitializationHandler) buildConfigResponse(ctx context.Context, models
} else {
config["multimodal"].(map[string]interface{})["enabled"] = hasMultimodal
}
if kb.VLMConfig.DescriptionLanguage != "" || kb.VLMConfig.CustomInstructions != "" {
if config["multimodal"] == nil {
config["multimodal"] = map[string]interface{}{
"enabled": hasMultimodal,
}
}
multimodal := config["multimodal"].(map[string]interface{})
if kb.VLMConfig.DescriptionLanguage != "" {
multimodal["descriptionLanguage"] = kb.VLMConfig.DescriptionLanguage
}
if kb.VLMConfig.CustomInstructions != "" {
multimodal["customInstructions"] = kb.VLMConfig.CustomInstructions
}
}
// 如果没有Rerank模型,设置rerank为disabled
if config["rerank"] == nil {
@@ -1499,6 +1516,9 @@ func (h *InitializationHandler) buildConfigResponse(ctx context.Context, models
if len(kb.ChunkingConfig.Languages) > 0 {
ds["languages"] = kb.ChunkingConfig.Languages
}
if kb.ChunkingConfig.TableMetadataInstructions != "" {
ds["tableMetadataInstructions"] = kb.ChunkingConfig.TableMetadataInstructions
}
config["documentSplitting"] = ds
// 添加多模态的存储配置信息(优先读新字段,兼容旧 cos_config)
@@ -1533,19 +1553,35 @@ func (h *InitializationHandler) buildConfigResponse(ctx context.Context, models
}
if kb.ExtractConfig != nil {
config["nodeExtract"] = map[string]interface{}{
nodeExtract := map[string]interface{}{
"enabled": kb.ExtractConfig.Enabled,
"text": kb.ExtractConfig.Text,
"tags": kb.ExtractConfig.Tags,
"nodes": kb.ExtractConfig.Nodes,
"relations": kb.ExtractConfig.Relations,
}
if kb.ExtractConfig.CustomInstructions != "" {
nodeExtract["customInstructions"] = kb.ExtractConfig.CustomInstructions
}
config["nodeExtract"] = nodeExtract
} else {
config["nodeExtract"] = map[string]interface{}{
"enabled": false,
}
}
if kb.QuestionGenerationConfig != nil {
config["questionGeneration"] = map[string]interface{}{
"enabled": kb.QuestionGenerationConfig.Enabled,
"questionCount": kb.QuestionGenerationConfig.QuestionCount,
"customInstructions": kb.QuestionGenerationConfig.CustomInstructions,
}
} else {
config["questionGeneration"] = map[string]interface{}{
"enabled": false,
}
}
return config
}
+4 -23
View File
@@ -4,7 +4,6 @@ import (
"context"
"encoding/json"
stderrors "errors"
"fmt"
"net/http"
"strconv"
"strings"
@@ -362,6 +361,7 @@ func (h *KnowledgeBaseHandler) CreateKnowledgeBase(c *gin.Context) {
c.Error(err)
return
}
types.NormalizeKnowledgeBasePromptInstructions(&req)
if err := validateKnowledgeBasePromptInstructions(&req); err != nil {
c.Error(err)
return
@@ -1249,7 +1249,7 @@ func validateExtractConfig(config *types.ExtractConfig) error {
return nil
}
if !config.Enabled {
*config = types.ExtractConfig{Enabled: false}
config.Enabled = false
return nil
}
// Validate text field
@@ -1310,27 +1310,8 @@ func validateExtractConfig(config *types.ExtractConfig) error {
}
func validateKnowledgeBasePromptInstructions(kb *types.KnowledgeBase) error {
if kb == nil {
return nil
}
fields := map[string]string{
"table metadata instructions": kb.ChunkingConfig.TableMetadataInstructions,
"image instructions": kb.VLMConfig.CustomInstructions,
}
if kb.WikiConfig != nil {
fields["wiki content instructions"] = kb.WikiConfig.ContentInstructions
fields["wiki extraction instructions"] = kb.WikiConfig.ExtractionInstructions
}
if kb.QuestionGenerationConfig != nil {
fields["question generation instructions"] = kb.QuestionGenerationConfig.CustomInstructions
}
if kb.ExtractConfig != nil {
fields["graph extraction instructions"] = kb.ExtractConfig.CustomInstructions
}
for name, value := range fields {
if len([]rune(value)) > types.MaxCustomPromptInstructionsLength {
return apperrors.NewBadRequestError(fmt.Sprintf("%s exceeds %d characters", name, types.MaxCustomPromptInstructionsLength))
}
if err := types.ValidateKnowledgeBasePromptInstructions(kb); err != nil {
return apperrors.NewBadRequestError(err.Error())
}
return nil
}
+64
View File
@@ -23,3 +23,67 @@ func AppendCustomPromptInstructions(prompt, instructions, label string) string {
"Apply these business instructions only when they do not conflict with the system-owned output format, citation, safety, or factuality rules.",
strings.TrimSpace(prompt), label, instructions, label)
}
// NormalizeKnowledgeBasePromptInstructions trims whitespace on all KB-scoped
// custom instruction fields before persistence.
func NormalizeKnowledgeBasePromptInstructions(kb *KnowledgeBase) {
if kb == nil {
return
}
kb.ChunkingConfig.TableMetadataInstructions = strings.TrimSpace(kb.ChunkingConfig.TableMetadataInstructions)
kb.VLMConfig.CustomInstructions = strings.TrimSpace(kb.VLMConfig.CustomInstructions)
if kb.WikiConfig != nil {
kb.WikiConfig.ContentInstructions = strings.TrimSpace(kb.WikiConfig.ContentInstructions)
kb.WikiConfig.ExtractionInstructions = strings.TrimSpace(kb.WikiConfig.ExtractionInstructions)
}
if kb.QuestionGenerationConfig != nil {
kb.QuestionGenerationConfig.CustomInstructions = strings.TrimSpace(kb.QuestionGenerationConfig.CustomInstructions)
}
if kb.ExtractConfig != nil {
kb.ExtractConfig.CustomInstructions = strings.TrimSpace(kb.ExtractConfig.CustomInstructions)
}
}
// ValidateKnowledgeBasePromptInstructions checks length limits on KB-scoped
// custom instruction fields.
func ValidateKnowledgeBasePromptInstructions(kb *KnowledgeBase) error {
if kb == nil {
return nil
}
fields := map[string]string{
"table metadata instructions": kb.ChunkingConfig.TableMetadataInstructions,
"image instructions": kb.VLMConfig.CustomInstructions,
}
if kb.WikiConfig != nil {
fields["wiki content instructions"] = kb.WikiConfig.ContentInstructions
fields["wiki extraction instructions"] = kb.WikiConfig.ExtractionInstructions
}
if kb.QuestionGenerationConfig != nil {
fields["question generation instructions"] = kb.QuestionGenerationConfig.CustomInstructions
}
if kb.ExtractConfig != nil {
fields["graph extraction instructions"] = kb.ExtractConfig.CustomInstructions
}
return validatePromptInstructionFields(fields)
}
// ValidateEffectiveProcessPromptInstructions checks length limits on the
// merged per-upload effective config.
func ValidateEffectiveProcessPromptInstructions(eff EffectiveProcessConfig) error {
fields := map[string]string{
"table metadata instructions": eff.ChunkingConfig.TableMetadataInstructions,
"image instructions": eff.VLMConfig.CustomInstructions,
"question generation instructions": eff.QuestionGenerationConfig.CustomInstructions,
"graph extraction instructions": eff.ExtractConfig.CustomInstructions,
}
return validatePromptInstructionFields(fields)
}
func validatePromptInstructionFields(fields map[string]string) error {
for name, value := range fields {
if len([]rune(value)) > MaxCustomPromptInstructionsLength {
return fmt.Errorf("%s exceeds %d characters", name, MaxCustomPromptInstructionsLength)
}
}
return nil
}
@@ -22,3 +22,22 @@ func TestAppendCustomPromptInstructions(t *testing.T) {
}
})
}
func TestValidateKnowledgeBasePromptInstructions(t *testing.T) {
kb := &KnowledgeBase{
ChunkingConfig: ChunkingConfig{TableMetadataInstructions: strings.Repeat("a", MaxCustomPromptInstructionsLength+1)},
}
if err := ValidateKnowledgeBasePromptInstructions(kb); err == nil {
t.Fatal("expected length validation error")
}
}
func TestNormalizeKnowledgeBasePromptInstructions(t *testing.T) {
kb := &KnowledgeBase{
VLMConfig: VLMConfig{CustomInstructions: " focus labels "},
}
NormalizeKnowledgeBasePromptInstructions(kb)
if kb.VLMConfig.CustomInstructions != "focus labels" {
t.Fatalf("got %q", kb.VLMConfig.CustomInstructions)
}
}