feat(chunk): implement batch deletion for chunks to avoid MySQL placeholder limit
- Updated DeleteChunks method to split IDs into batches of 5000 to prevent MySQL Error 1390. - Added unit tests for CalculateFAQContentHash to ensure consistent hashing behavior across various scenarios, including normalization and case sensitivity.
This commit is contained in:
@@ -398,12 +398,23 @@ func (r *chunkRepository) DeleteChunk(ctx context.Context, tenantID uint64, id s
|
||||
return r.db.WithContext(ctx).Where("tenant_id = ? AND id = ?", tenantID, id).Delete(&types.Chunk{}).Error
|
||||
}
|
||||
|
||||
// DeleteChunks deletes chunks by IDs in batch
|
||||
// DeleteChunks deletes chunks by IDs in batch.
|
||||
// To avoid MySQL Error 1390 (too many placeholders), IDs are split into batches.
|
||||
func (r *chunkRepository) DeleteChunks(ctx context.Context, tenantID uint64, ids []string) error {
|
||||
if len(ids) == 0 {
|
||||
return nil
|
||||
}
|
||||
return r.db.WithContext(ctx).Where("tenant_id = ? AND id IN ?", tenantID, ids).Delete(&types.Chunk{}).Error
|
||||
const batchSize = 5000
|
||||
for i := 0; i < len(ids); i += batchSize {
|
||||
end := i + batchSize
|
||||
if end > len(ids) {
|
||||
end = len(ids)
|
||||
}
|
||||
if err := r.db.WithContext(ctx).Where("tenant_id = ? AND id IN ?", tenantID, ids[i:end]).Delete(&types.Chunk{}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeleteChunksByKnowledgeID deletes all chunks for a knowledge ID
|
||||
|
||||
@@ -131,9 +131,11 @@ func CalculateFAQContentHash(meta *FAQChunkMetadata) string {
|
||||
return ""
|
||||
}
|
||||
|
||||
// 创建副本并标准化
|
||||
normalized := *meta
|
||||
normalized.Normalize()
|
||||
// Normalize() returns a new copy; the old code discarded the return value.
|
||||
normalized := meta.Normalize()
|
||||
if normalized == nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
// 对数组进行排序(确保相同内容产生相同 hash)
|
||||
similarQuestions := make([]string, len(normalized.SimilarQuestions))
|
||||
|
||||
@@ -0,0 +1,180 @@
|
||||
package types
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCalculateFAQContentHash_NormalizeIsApplied(t *testing.T) {
|
||||
// The core bug: CalculateFAQContentHash must normalize the input so that
|
||||
// sanitized-only data and pre-normalized data produce the same hash.
|
||||
meta := &FAQChunkMetadata{
|
||||
StandardQuestion: " 你好,World? ",
|
||||
SimilarQuestions: []string{"Hello World", "hello world"},
|
||||
Answers: []string{"answer1"},
|
||||
AnswerStrategy: AnswerStrategyAll,
|
||||
Version: 1,
|
||||
}
|
||||
|
||||
// Path 1: what SetFAQMetadata does (normalize first, then hash)
|
||||
normalized := meta.Normalize()
|
||||
hashFromNormalized := CalculateFAQContentHash(normalized)
|
||||
|
||||
// Path 2: what calculateReplaceOperations does (hash directly from sanitized data)
|
||||
sanitized := &FAQChunkMetadata{
|
||||
StandardQuestion: " 你好,World? ",
|
||||
SimilarQuestions: []string{"Hello World", "hello world"},
|
||||
Answers: []string{"answer1"},
|
||||
AnswerStrategy: AnswerStrategyAll,
|
||||
Version: 1,
|
||||
}
|
||||
sanitized.Sanitize()
|
||||
hashFromSanitized := CalculateFAQContentHash(sanitized)
|
||||
|
||||
if hashFromNormalized != hashFromSanitized {
|
||||
t.Errorf("Hash mismatch between write and read paths:\n write (normalized first): %s\n read (sanitized only): %s",
|
||||
hashFromNormalized, hashFromSanitized)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCalculateFAQContentHash_ConsistentViaSetFAQMetadata(t *testing.T) {
|
||||
// Simulate the full write path then read-path comparison
|
||||
meta := &FAQChunkMetadata{
|
||||
StandardQuestion: "如何退款?",
|
||||
SimilarQuestions: []string{"怎么退款", "退款流程"},
|
||||
Answers: []string{"请联系客服"},
|
||||
AnswerStrategy: AnswerStrategyAll,
|
||||
Version: 1,
|
||||
Source: "faq",
|
||||
}
|
||||
|
||||
// Write path: SetFAQMetadata stores ContentHash
|
||||
chunk := &Chunk{}
|
||||
if err := chunk.SetFAQMetadata(meta); err != nil {
|
||||
t.Fatalf("SetFAQMetadata failed: %v", err)
|
||||
}
|
||||
if chunk.ContentHash == "" {
|
||||
t.Fatal("SetFAQMetadata did not set ContentHash")
|
||||
}
|
||||
|
||||
// Read path: calculateReplaceOperations calls sanitize + CalculateFAQContentHash
|
||||
readMeta := &FAQChunkMetadata{
|
||||
StandardQuestion: "如何退款?",
|
||||
SimilarQuestions: []string{"怎么退款", "退款流程"},
|
||||
Answers: []string{"请联系客服"},
|
||||
AnswerStrategy: AnswerStrategyAll,
|
||||
Version: 1,
|
||||
Source: "faq",
|
||||
}
|
||||
readMeta.Sanitize()
|
||||
readHash := CalculateFAQContentHash(readMeta)
|
||||
|
||||
if chunk.ContentHash != readHash {
|
||||
t.Errorf("Hash mismatch between SetFAQMetadata and direct CalculateFAQContentHash:\n SetFAQMetadata: %s\n CalculateFAQContentHash: %s",
|
||||
chunk.ContentHash, readHash)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCalculateFAQContentHash_CaseAndPunctuationInvariant(t *testing.T) {
|
||||
meta1 := &FAQChunkMetadata{
|
||||
StandardQuestion: "Hello World?",
|
||||
Answers: []string{"answer"},
|
||||
}
|
||||
meta2 := &FAQChunkMetadata{
|
||||
StandardQuestion: "hello world?",
|
||||
Answers: []string{"answer"},
|
||||
}
|
||||
|
||||
hash1 := CalculateFAQContentHash(meta1)
|
||||
hash2 := CalculateFAQContentHash(meta2)
|
||||
|
||||
if hash1 != hash2 {
|
||||
t.Errorf("Hash should be case/punctuation invariant after normalization:\n %q -> %s\n %q -> %s",
|
||||
meta1.StandardQuestion, hash1, meta2.StandardQuestion, hash2)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCalculateFAQContentHash_TraditionalSimplifiedInvariant(t *testing.T) {
|
||||
meta1 := &FAQChunkMetadata{
|
||||
StandardQuestion: "如何退款",
|
||||
Answers: []string{"请联系客服"},
|
||||
}
|
||||
meta2 := &FAQChunkMetadata{
|
||||
StandardQuestion: "如何退款", // simplified
|
||||
Answers: []string{"請聯繫客服"}, // traditional in answers — answers only sanitize, not normalize
|
||||
}
|
||||
|
||||
// Questions should normalize, but answers only sanitize.
|
||||
// So answers in traditional vs simplified WILL produce different hashes (by design).
|
||||
// But standard questions with t2s should match.
|
||||
metaTraditionalQ := &FAQChunkMetadata{
|
||||
StandardQuestion: "開發環境",
|
||||
Answers: []string{"answer"},
|
||||
}
|
||||
metaSimplifiedQ := &FAQChunkMetadata{
|
||||
StandardQuestion: "开发环境",
|
||||
Answers: []string{"answer"},
|
||||
}
|
||||
|
||||
hashTrad := CalculateFAQContentHash(metaTraditionalQ)
|
||||
hashSimp := CalculateFAQContentHash(metaSimplifiedQ)
|
||||
|
||||
if hashTrad != hashSimp {
|
||||
t.Errorf("Hash should be traditional/simplified invariant for questions:\n traditional: %s\n simplified: %s",
|
||||
hashTrad, hashSimp)
|
||||
}
|
||||
|
||||
_ = meta1
|
||||
_ = meta2
|
||||
}
|
||||
|
||||
func TestCalculateFAQContentHash_SortInvariant(t *testing.T) {
|
||||
meta1 := &FAQChunkMetadata{
|
||||
StandardQuestion: "问题",
|
||||
SimilarQuestions: []string{"a", "b", "c"},
|
||||
Answers: []string{"x", "y", "z"},
|
||||
}
|
||||
meta2 := &FAQChunkMetadata{
|
||||
StandardQuestion: "问题",
|
||||
SimilarQuestions: []string{"c", "a", "b"},
|
||||
Answers: []string{"z", "x", "y"},
|
||||
}
|
||||
|
||||
hash1 := CalculateFAQContentHash(meta1)
|
||||
hash2 := CalculateFAQContentHash(meta2)
|
||||
|
||||
if hash1 != hash2 {
|
||||
t.Errorf("Hash should be order-invariant for similar questions and answers:\n order1: %s\n order2: %s",
|
||||
hash1, hash2)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCalculateFAQContentHash_NilAndEmpty(t *testing.T) {
|
||||
if h := CalculateFAQContentHash(nil); h != "" {
|
||||
t.Errorf("Expected empty hash for nil, got %s", h)
|
||||
}
|
||||
|
||||
meta := &FAQChunkMetadata{}
|
||||
h := CalculateFAQContentHash(meta)
|
||||
if h == "" {
|
||||
t.Error("Expected non-empty hash for empty metadata (still has delimiters)")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCalculateFAQContentHash_FullWidthHalfWidthInvariant(t *testing.T) {
|
||||
metaFull := &FAQChunkMetadata{
|
||||
StandardQuestion: "Hello World",
|
||||
Answers: []string{"answer"},
|
||||
}
|
||||
metaHalf := &FAQChunkMetadata{
|
||||
StandardQuestion: "hello world",
|
||||
Answers: []string{"answer"},
|
||||
}
|
||||
|
||||
hashFull := CalculateFAQContentHash(metaFull)
|
||||
hashHalf := CalculateFAQContentHash(metaHalf)
|
||||
|
||||
if hashFull != hashHalf {
|
||||
t.Errorf("Hash should be fullwidth/halfwidth invariant:\n fullwidth: %s\n halfwidth: %s",
|
||||
hashFull, hashHalf)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user