feat(knowledge): implement AminusB function for multiset comparison of knowledge bases

- Added AminusB method to the knowledgeRepository, which returns IDs of knowledge in A that have no counterpart in B, comparing by file_hash as a multiset.
- This implementation addresses issues with handling duplicate entries and NULL file_hash values, ensuring accurate synchronization between knowledge bases.
- Updated the KnowledgeRepository interface to reflect the new functionality and improved documentation for clarity.
This commit is contained in:
wizardchen
2026-07-21 17:00:20 +08:00
committed by lyingbug
parent 516ef01cf2
commit 0d545c8c6a
2 changed files with 75 additions and 12 deletions
+72 -11
View File
@@ -286,23 +286,84 @@ func (r *knowledgeRepository) CheckKnowledgeExists(
return false, nil, nil
}
// AminusB returns the IDs of knowledge in A that have no counterpart in B,
// comparing by file_hash as a MULTISET rather than a plain set.
//
// A plain "file_hash NOT IN (SELECT file_hash FROM B)" only asks whether a
// hash exists in B at all, so once a KB accumulates several rows sharing the
// same file_hash (e.g. the same file ingested multiple times), the diff can
// never reconcile the *count* difference: two KBs with identical distinct-hash
// sets but different row counts produce an empty diff in both directions, and
// a clone target can never converge to the source. This also breaks on MySQL
// when B contains a NULL file_hash, because NOT IN then yields no rows at all.
//
// The multiset diff is computed in Go rather than SQL: we only pull
// (id, file_hash) for A plus per-hash counts for B, then keep A's surplus
// copies. This avoids window functions (unsupported on MySQL 5.7 / MariaDB)
// and the O(n^2) correlated-subquery ranking that would otherwise be needed
// there. Clone is a background job over at most a few thousand rows, so the
// two lightweight two-column reads are cheap.
//
// Rows with a NULL/empty file_hash carry no reliable identity (unparsed /
// passage knowledge), so they are always treated as present-only-in-A to
// avoid collapsing distinct rows into one.
func (r *knowledgeRepository) AminusB(
ctx context.Context,
Atenant uint64, A string,
Btenant uint64, B string,
) ([]string, error) {
knowledgeIDs := []string{}
subQuery := r.db.Model(&types.Knowledge{}).
Where("tenant_id = ? AND knowledge_base_id = ?", Btenant, B).Select("file_hash")
err := r.db.Model(&types.Knowledge{}).
Where("tenant_id = ? AND knowledge_base_id = ?", Atenant, A).
Where("file_hash NOT IN (?)", subQuery).
Pluck("id", &knowledgeIDs).
Error
if errors.Is(err, gorm.ErrRecordNotFound) {
return knowledgeIDs, nil
type hashRow struct {
ID string
FileHash string
}
return knowledgeIDs, err
// Order so the retained (matched) copies are the earliest ones and the
// surplus we return is deterministic across runs.
var aRows []hashRow
if err := r.db.WithContext(ctx).
Model(&types.Knowledge{}).
Select("id, file_hash").
Where("tenant_id = ? AND knowledge_base_id = ?", Atenant, A).
Order("file_hash, created_at, id").
Find(&aRows).Error; err != nil {
return nil, err
}
type hashCount struct {
FileHash string
Cnt int
}
var bCounts []hashCount
if err := r.db.WithContext(ctx).
Model(&types.Knowledge{}).
Select("file_hash, COUNT(*) AS cnt").
Where("tenant_id = ? AND knowledge_base_id = ?", Btenant, B).
Group("file_hash").
Find(&bCounts).Error; err != nil {
return nil, err
}
// remaining[h] is how many copies of hash h in B are still unmatched.
remaining := make(map[string]int, len(bCounts))
for _, c := range bCounts {
if c.FileHash != "" {
remaining[c.FileHash] = c.Cnt
}
}
knowledgeIDs := make([]string, 0)
for _, row := range aRows {
// NULL scans into "" here, so this also covers NULL hashes.
if row.FileHash == "" {
knowledgeIDs = append(knowledgeIDs, row.ID)
continue
}
if remaining[row.FileHash] > 0 {
remaining[row.FileHash]-- // matched by an existing copy in B
continue
}
knowledgeIDs = append(knowledgeIDs, row.ID) // surplus copy in A
}
return knowledgeIDs, nil
}
func (r *knowledgeRepository) UpdateKnowledgeColumn(
+3 -1
View File
@@ -232,7 +232,9 @@ type KnowledgeRepository interface {
kbID string,
params *types.KnowledgeCheckParams,
) (bool, *types.Knowledge, error)
// AminusB returns the difference set of A and B.
// AminusB returns the IDs of knowledge in A that have no counterpart in B,
// comparing file_hash as a multiset (so duplicate-count differences and
// NULL/empty hashes are handled correctly, letting a clone converge).
AminusB(ctx context.Context, Atenant uint64, A string, Btenant uint64, B string) ([]string, error)
UpdateKnowledgeColumn(ctx context.Context, id string, column string, value interface{}) error
// UpdateKnowledgeColumns updates multiple columns of a knowledge row in a single