From c25741673cad8a3fb3cfcb735cacc18463fa0114 Mon Sep 17 00:00:00 2001 From: wizardchen Date: Thu, 19 Mar 2026 15:44:03 +0800 Subject: [PATCH] feat(chunk): implement batch deletion for chunks to avoid MySQL placeholder limit - Updated DeleteChunks method to split IDs into batches of 5000 to prevent MySQL Error 1390. - Added unit tests for CalculateFAQContentHash to ensure consistent hashing behavior across various scenarios, including normalization and case sensitivity. --- internal/application/repository/chunk.go | 15 +- internal/types/faq.go | 8 +- internal/types/faq_test.go | 180 +++++++++++++++++++++++ 3 files changed, 198 insertions(+), 5 deletions(-) create mode 100644 internal/types/faq_test.go diff --git a/internal/application/repository/chunk.go b/internal/application/repository/chunk.go index b2f3eb6a..61789a48 100644 --- a/internal/application/repository/chunk.go +++ b/internal/application/repository/chunk.go @@ -398,12 +398,23 @@ func (r *chunkRepository) DeleteChunk(ctx context.Context, tenantID uint64, id s return r.db.WithContext(ctx).Where("tenant_id = ? AND id = ?", tenantID, id).Delete(&types.Chunk{}).Error } -// DeleteChunks deletes chunks by IDs in batch +// DeleteChunks deletes chunks by IDs in batch. +// To avoid MySQL Error 1390 (too many placeholders), IDs are split into batches. func (r *chunkRepository) DeleteChunks(ctx context.Context, tenantID uint64, ids []string) error { if len(ids) == 0 { return nil } - return r.db.WithContext(ctx).Where("tenant_id = ? AND id IN ?", tenantID, ids).Delete(&types.Chunk{}).Error + const batchSize = 5000 + for i := 0; i < len(ids); i += batchSize { + end := i + batchSize + if end > len(ids) { + end = len(ids) + } + if err := r.db.WithContext(ctx).Where("tenant_id = ? AND id IN ?", tenantID, ids[i:end]).Delete(&types.Chunk{}).Error; err != nil { + return err + } + } + return nil } // DeleteChunksByKnowledgeID deletes all chunks for a knowledge ID diff --git a/internal/types/faq.go b/internal/types/faq.go index 1a9dc452..663f376d 100644 --- a/internal/types/faq.go +++ b/internal/types/faq.go @@ -131,9 +131,11 @@ func CalculateFAQContentHash(meta *FAQChunkMetadata) string { return "" } - // 创建副本并标准化 - normalized := *meta - normalized.Normalize() + // Normalize() returns a new copy; the old code discarded the return value. + normalized := meta.Normalize() + if normalized == nil { + return "" + } // 对数组进行排序(确保相同内容产生相同 hash) similarQuestions := make([]string, len(normalized.SimilarQuestions)) diff --git a/internal/types/faq_test.go b/internal/types/faq_test.go new file mode 100644 index 00000000..2b4a2a68 --- /dev/null +++ b/internal/types/faq_test.go @@ -0,0 +1,180 @@ +package types + +import ( + "testing" +) + +func TestCalculateFAQContentHash_NormalizeIsApplied(t *testing.T) { + // The core bug: CalculateFAQContentHash must normalize the input so that + // sanitized-only data and pre-normalized data produce the same hash. + meta := &FAQChunkMetadata{ + StandardQuestion: " 你好,World? ", + SimilarQuestions: []string{"Hello World", "hello world"}, + Answers: []string{"answer1"}, + AnswerStrategy: AnswerStrategyAll, + Version: 1, + } + + // Path 1: what SetFAQMetadata does (normalize first, then hash) + normalized := meta.Normalize() + hashFromNormalized := CalculateFAQContentHash(normalized) + + // Path 2: what calculateReplaceOperations does (hash directly from sanitized data) + sanitized := &FAQChunkMetadata{ + StandardQuestion: " 你好,World? ", + SimilarQuestions: []string{"Hello World", "hello world"}, + Answers: []string{"answer1"}, + AnswerStrategy: AnswerStrategyAll, + Version: 1, + } + sanitized.Sanitize() + hashFromSanitized := CalculateFAQContentHash(sanitized) + + if hashFromNormalized != hashFromSanitized { + t.Errorf("Hash mismatch between write and read paths:\n write (normalized first): %s\n read (sanitized only): %s", + hashFromNormalized, hashFromSanitized) + } +} + +func TestCalculateFAQContentHash_ConsistentViaSetFAQMetadata(t *testing.T) { + // Simulate the full write path then read-path comparison + meta := &FAQChunkMetadata{ + StandardQuestion: "如何退款?", + SimilarQuestions: []string{"怎么退款", "退款流程"}, + Answers: []string{"请联系客服"}, + AnswerStrategy: AnswerStrategyAll, + Version: 1, + Source: "faq", + } + + // Write path: SetFAQMetadata stores ContentHash + chunk := &Chunk{} + if err := chunk.SetFAQMetadata(meta); err != nil { + t.Fatalf("SetFAQMetadata failed: %v", err) + } + if chunk.ContentHash == "" { + t.Fatal("SetFAQMetadata did not set ContentHash") + } + + // Read path: calculateReplaceOperations calls sanitize + CalculateFAQContentHash + readMeta := &FAQChunkMetadata{ + StandardQuestion: "如何退款?", + SimilarQuestions: []string{"怎么退款", "退款流程"}, + Answers: []string{"请联系客服"}, + AnswerStrategy: AnswerStrategyAll, + Version: 1, + Source: "faq", + } + readMeta.Sanitize() + readHash := CalculateFAQContentHash(readMeta) + + if chunk.ContentHash != readHash { + t.Errorf("Hash mismatch between SetFAQMetadata and direct CalculateFAQContentHash:\n SetFAQMetadata: %s\n CalculateFAQContentHash: %s", + chunk.ContentHash, readHash) + } +} + +func TestCalculateFAQContentHash_CaseAndPunctuationInvariant(t *testing.T) { + meta1 := &FAQChunkMetadata{ + StandardQuestion: "Hello World?", + Answers: []string{"answer"}, + } + meta2 := &FAQChunkMetadata{ + StandardQuestion: "hello world?", + Answers: []string{"answer"}, + } + + hash1 := CalculateFAQContentHash(meta1) + hash2 := CalculateFAQContentHash(meta2) + + if hash1 != hash2 { + t.Errorf("Hash should be case/punctuation invariant after normalization:\n %q -> %s\n %q -> %s", + meta1.StandardQuestion, hash1, meta2.StandardQuestion, hash2) + } +} + +func TestCalculateFAQContentHash_TraditionalSimplifiedInvariant(t *testing.T) { + meta1 := &FAQChunkMetadata{ + StandardQuestion: "如何退款", + Answers: []string{"请联系客服"}, + } + meta2 := &FAQChunkMetadata{ + StandardQuestion: "如何退款", // simplified + Answers: []string{"請聯繫客服"}, // traditional in answers — answers only sanitize, not normalize + } + + // Questions should normalize, but answers only sanitize. + // So answers in traditional vs simplified WILL produce different hashes (by design). + // But standard questions with t2s should match. + metaTraditionalQ := &FAQChunkMetadata{ + StandardQuestion: "開發環境", + Answers: []string{"answer"}, + } + metaSimplifiedQ := &FAQChunkMetadata{ + StandardQuestion: "开发环境", + Answers: []string{"answer"}, + } + + hashTrad := CalculateFAQContentHash(metaTraditionalQ) + hashSimp := CalculateFAQContentHash(metaSimplifiedQ) + + if hashTrad != hashSimp { + t.Errorf("Hash should be traditional/simplified invariant for questions:\n traditional: %s\n simplified: %s", + hashTrad, hashSimp) + } + + _ = meta1 + _ = meta2 +} + +func TestCalculateFAQContentHash_SortInvariant(t *testing.T) { + meta1 := &FAQChunkMetadata{ + StandardQuestion: "问题", + SimilarQuestions: []string{"a", "b", "c"}, + Answers: []string{"x", "y", "z"}, + } + meta2 := &FAQChunkMetadata{ + StandardQuestion: "问题", + SimilarQuestions: []string{"c", "a", "b"}, + Answers: []string{"z", "x", "y"}, + } + + hash1 := CalculateFAQContentHash(meta1) + hash2 := CalculateFAQContentHash(meta2) + + if hash1 != hash2 { + t.Errorf("Hash should be order-invariant for similar questions and answers:\n order1: %s\n order2: %s", + hash1, hash2) + } +} + +func TestCalculateFAQContentHash_NilAndEmpty(t *testing.T) { + if h := CalculateFAQContentHash(nil); h != "" { + t.Errorf("Expected empty hash for nil, got %s", h) + } + + meta := &FAQChunkMetadata{} + h := CalculateFAQContentHash(meta) + if h == "" { + t.Error("Expected non-empty hash for empty metadata (still has delimiters)") + } +} + +func TestCalculateFAQContentHash_FullWidthHalfWidthInvariant(t *testing.T) { + metaFull := &FAQChunkMetadata{ + StandardQuestion: "Hello World", + Answers: []string{"answer"}, + } + metaHalf := &FAQChunkMetadata{ + StandardQuestion: "hello world", + Answers: []string{"answer"}, + } + + hashFull := CalculateFAQContentHash(metaFull) + hashHalf := CalculateFAQContentHash(metaHalf) + + if hashFull != hashHalf { + t.Errorf("Hash should be fullwidth/halfwidth invariant:\n fullwidth: %s\n halfwidth: %s", + hashFull, hashHalf) + } +}