feat(chunk): implement batch deletion for chunks to avoid MySQL placeholder limit

- Updated DeleteChunks method to split IDs into batches of 5000 to prevent MySQL Error 1390.
- Added unit tests for CalculateFAQContentHash to ensure consistent hashing behavior across various scenarios, including normalization and case sensitivity.
This commit is contained in:
wizardchen
2026-03-19 15:44:03 +08:00
committed by lyingbug
parent 8db4b6cdea
commit c25741673c
3 changed files with 198 additions and 5 deletions
+13 -2
View File
@@ -398,12 +398,23 @@ func (r *chunkRepository) DeleteChunk(ctx context.Context, tenantID uint64, id s
return r.db.WithContext(ctx).Where("tenant_id = ? AND id = ?", tenantID, id).Delete(&types.Chunk{}).Error
}
// DeleteChunks deletes chunks by IDs in batch
// DeleteChunks deletes chunks by IDs in batch.
// To avoid MySQL Error 1390 (too many placeholders), IDs are split into batches.
func (r *chunkRepository) DeleteChunks(ctx context.Context, tenantID uint64, ids []string) error {
if len(ids) == 0 {
return nil
}
return r.db.WithContext(ctx).Where("tenant_id = ? AND id IN ?", tenantID, ids).Delete(&types.Chunk{}).Error
const batchSize = 5000
for i := 0; i < len(ids); i += batchSize {
end := i + batchSize
if end > len(ids) {
end = len(ids)
}
if err := r.db.WithContext(ctx).Where("tenant_id = ? AND id IN ?", tenantID, ids[i:end]).Delete(&types.Chunk{}).Error; err != nil {
return err
}
}
return nil
}
// DeleteChunksByKnowledgeID deletes all chunks for a knowledge ID
+5 -3
View File
@@ -131,9 +131,11 @@ func CalculateFAQContentHash(meta *FAQChunkMetadata) string {
return ""
}
// 创建副本并标准化
normalized := *meta
normalized.Normalize()
// Normalize() returns a new copy; the old code discarded the return value.
normalized := meta.Normalize()
if normalized == nil {
return ""
}
// 对数组进行排序(确保相同内容产生相同 hash)
similarQuestions := make([]string, len(normalized.SimilarQuestions))
+180
View File
@@ -0,0 +1,180 @@
package types
import (
"testing"
)
func TestCalculateFAQContentHash_NormalizeIsApplied(t *testing.T) {
// The core bug: CalculateFAQContentHash must normalize the input so that
// sanitized-only data and pre-normalized data produce the same hash.
meta := &FAQChunkMetadata{
StandardQuestion: " 你好,World? ",
SimilarQuestions: []string{"Hello World", "hello world"},
Answers: []string{"answer1"},
AnswerStrategy: AnswerStrategyAll,
Version: 1,
}
// Path 1: what SetFAQMetadata does (normalize first, then hash)
normalized := meta.Normalize()
hashFromNormalized := CalculateFAQContentHash(normalized)
// Path 2: what calculateReplaceOperations does (hash directly from sanitized data)
sanitized := &FAQChunkMetadata{
StandardQuestion: " 你好,World? ",
SimilarQuestions: []string{"Hello World", "hello world"},
Answers: []string{"answer1"},
AnswerStrategy: AnswerStrategyAll,
Version: 1,
}
sanitized.Sanitize()
hashFromSanitized := CalculateFAQContentHash(sanitized)
if hashFromNormalized != hashFromSanitized {
t.Errorf("Hash mismatch between write and read paths:\n write (normalized first): %s\n read (sanitized only): %s",
hashFromNormalized, hashFromSanitized)
}
}
func TestCalculateFAQContentHash_ConsistentViaSetFAQMetadata(t *testing.T) {
// Simulate the full write path then read-path comparison
meta := &FAQChunkMetadata{
StandardQuestion: "如何退款?",
SimilarQuestions: []string{"怎么退款", "退款流程"},
Answers: []string{"请联系客服"},
AnswerStrategy: AnswerStrategyAll,
Version: 1,
Source: "faq",
}
// Write path: SetFAQMetadata stores ContentHash
chunk := &Chunk{}
if err := chunk.SetFAQMetadata(meta); err != nil {
t.Fatalf("SetFAQMetadata failed: %v", err)
}
if chunk.ContentHash == "" {
t.Fatal("SetFAQMetadata did not set ContentHash")
}
// Read path: calculateReplaceOperations calls sanitize + CalculateFAQContentHash
readMeta := &FAQChunkMetadata{
StandardQuestion: "如何退款?",
SimilarQuestions: []string{"怎么退款", "退款流程"},
Answers: []string{"请联系客服"},
AnswerStrategy: AnswerStrategyAll,
Version: 1,
Source: "faq",
}
readMeta.Sanitize()
readHash := CalculateFAQContentHash(readMeta)
if chunk.ContentHash != readHash {
t.Errorf("Hash mismatch between SetFAQMetadata and direct CalculateFAQContentHash:\n SetFAQMetadata: %s\n CalculateFAQContentHash: %s",
chunk.ContentHash, readHash)
}
}
func TestCalculateFAQContentHash_CaseAndPunctuationInvariant(t *testing.T) {
meta1 := &FAQChunkMetadata{
StandardQuestion: "Hello World?",
Answers: []string{"answer"},
}
meta2 := &FAQChunkMetadata{
StandardQuestion: "hello world?",
Answers: []string{"answer"},
}
hash1 := CalculateFAQContentHash(meta1)
hash2 := CalculateFAQContentHash(meta2)
if hash1 != hash2 {
t.Errorf("Hash should be case/punctuation invariant after normalization:\n %q -> %s\n %q -> %s",
meta1.StandardQuestion, hash1, meta2.StandardQuestion, hash2)
}
}
func TestCalculateFAQContentHash_TraditionalSimplifiedInvariant(t *testing.T) {
meta1 := &FAQChunkMetadata{
StandardQuestion: "如何退款",
Answers: []string{"请联系客服"},
}
meta2 := &FAQChunkMetadata{
StandardQuestion: "如何退款", // simplified
Answers: []string{"請聯繫客服"}, // traditional in answers — answers only sanitize, not normalize
}
// Questions should normalize, but answers only sanitize.
// So answers in traditional vs simplified WILL produce different hashes (by design).
// But standard questions with t2s should match.
metaTraditionalQ := &FAQChunkMetadata{
StandardQuestion: "開發環境",
Answers: []string{"answer"},
}
metaSimplifiedQ := &FAQChunkMetadata{
StandardQuestion: "开发环境",
Answers: []string{"answer"},
}
hashTrad := CalculateFAQContentHash(metaTraditionalQ)
hashSimp := CalculateFAQContentHash(metaSimplifiedQ)
if hashTrad != hashSimp {
t.Errorf("Hash should be traditional/simplified invariant for questions:\n traditional: %s\n simplified: %s",
hashTrad, hashSimp)
}
_ = meta1
_ = meta2
}
func TestCalculateFAQContentHash_SortInvariant(t *testing.T) {
meta1 := &FAQChunkMetadata{
StandardQuestion: "问题",
SimilarQuestions: []string{"a", "b", "c"},
Answers: []string{"x", "y", "z"},
}
meta2 := &FAQChunkMetadata{
StandardQuestion: "问题",
SimilarQuestions: []string{"c", "a", "b"},
Answers: []string{"z", "x", "y"},
}
hash1 := CalculateFAQContentHash(meta1)
hash2 := CalculateFAQContentHash(meta2)
if hash1 != hash2 {
t.Errorf("Hash should be order-invariant for similar questions and answers:\n order1: %s\n order2: %s",
hash1, hash2)
}
}
func TestCalculateFAQContentHash_NilAndEmpty(t *testing.T) {
if h := CalculateFAQContentHash(nil); h != "" {
t.Errorf("Expected empty hash for nil, got %s", h)
}
meta := &FAQChunkMetadata{}
h := CalculateFAQContentHash(meta)
if h == "" {
t.Error("Expected non-empty hash for empty metadata (still has delimiters)")
}
}
func TestCalculateFAQContentHash_FullWidthHalfWidthInvariant(t *testing.T) {
metaFull := &FAQChunkMetadata{
StandardQuestion: "Hello World",
Answers: []string{"answer"},
}
metaHalf := &FAQChunkMetadata{
StandardQuestion: "hello world",
Answers: []string{"answer"},
}
hashFull := CalculateFAQContentHash(metaFull)
hashHalf := CalculateFAQContentHash(metaHalf)
if hashFull != hashHalf {
t.Errorf("Hash should be fullwidth/halfwidth invariant:\n fullwidth: %s\n halfwidth: %s",
hashFull, hashHalf)
}
}