Files
Enterprise-Knowledge-Base/internal/application/repository/retriever/sqlite/repository.go
T
wizardchen 931bd38138 feat(repository): add TagID and IsRecommended fields to vector embeddings
- Introduced `TagID` and `IsRecommended` fields across various repository implementations (Elasticsearch, Milvus, Postgres, Qdrant, SQLite) to enhance document categorization and recommendation capabilities.
- Updated the logic for handling `IsEnabled` to ensure it reflects the actual state from the source data.
- Adjusted the knowledge service to incorporate the new `IsRecommended` field, improving the indexing process for knowledge chunks.

These changes enhance the flexibility and functionality of document retrieval and indexing across the application.
2026-03-02 21:43:30 +08:00

614 lines
18 KiB
Go

package sqlite
import (
"context"
"fmt"
"strings"
"time"
"unicode"
"github.com/Tencent/WeKnora/internal/common"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
sqlite_vec "github.com/asg017/sqlite-vec-go-bindings/cgo"
"github.com/google/uuid"
"gorm.io/gorm"
"gorm.io/gorm/clause"
)
// sqliteEmbedding stores metadata alongside the vec0 virtual table rows
type sqliteEmbedding struct {
ID uint `gorm:"primarykey;autoIncrement"`
CreatedAt time.Time `gorm:"column:created_at"`
UpdatedAt time.Time `gorm:"column:updated_at"`
SourceID string `gorm:"column:source_id;not null;uniqueIndex:idx_sqlite_emb_source"`
SourceType int `gorm:"column:source_type;not null;uniqueIndex:idx_sqlite_emb_source"`
ChunkID string `gorm:"column:chunk_id;index"`
KnowledgeID string `gorm:"column:knowledge_id;index"`
KnowledgeBaseID string `gorm:"column:knowledge_base_id;index"`
TagID string `gorm:"column:tag_id;index"`
Content string `gorm:"column:content;not null"`
Dimension int `gorm:"column:dimension;not null"`
IsEnabled *bool `gorm:"column:is_enabled;default:true;index"`
}
func (sqliteEmbedding) TableName() string { return "lite_embeddings" }
type sqliteRepository struct {
db *gorm.DB
vecTables map[int]bool // tracks which vec0 tables have been created (keyed by dimension)
}
func NewSQLiteRetrieveEngineRepository(db *gorm.DB) interfaces.RetrieveEngineRepository {
logger.GetLogger(context.Background()).Info("[SQLite] Initializing SQLite retriever engine repository with sqlite-vec")
if err := db.AutoMigrate(&sqliteEmbedding{}); err != nil {
logger.GetLogger(context.Background()).Errorf("[SQLite] Failed to auto-migrate lite_embeddings: %v", err)
}
initFTS5(db)
repo := &sqliteRepository{
db: db,
vecTables: make(map[int]bool),
}
repo.ensureExistingVecTables()
return repo
}
func initFTS5(db *gorm.DB) {
sql := `CREATE VIRTUAL TABLE IF NOT EXISTS lite_embeddings_fts USING fts5(
content, source_id, chunk_id, knowledge_id, knowledge_base_id,
content='lite_embeddings', content_rowid='id',
tokenize='unicode61'
)`
if err := db.Exec(sql).Error; err != nil {
logger.GetLogger(context.Background()).Warnf("[SQLite] Failed to create FTS5 table: %v", err)
}
}
func vecTableName(dim int) string {
return fmt.Sprintf("vec_embeddings_%d", dim)
}
func (r *sqliteRepository) ensureVecTable(dim int) {
if dim <= 0 || r.vecTables[dim] {
return
}
tbl := vecTableName(dim)
createSQL := fmt.Sprintf(
`CREATE VIRTUAL TABLE IF NOT EXISTS %s USING vec0(embedding float[%d] distance_metric=cosine)`,
tbl, dim,
)
if err := r.db.Exec(createSQL).Error; err != nil {
if strings.Contains(err.Error(), "already exists") {
r.vecTables[dim] = true
return
}
logger.GetLogger(context.Background()).Errorf("[SQLite] Failed to create vec0 table for dim %d: %v", dim, err)
return
}
r.vecTables[dim] = true
}
func (r *sqliteRepository) ensureExistingVecTables() {
var dims []int
r.db.Model(&sqliteEmbedding{}).Distinct("dimension").Where("dimension > 0").Pluck("dimension", &dims)
for _, dim := range dims {
r.ensureVecTable(dim)
}
}
func (r *sqliteRepository) EngineType() types.RetrieverEngineType {
return types.SQLiteRetrieverEngineType
}
func (r *sqliteRepository) Support() []types.RetrieverType {
return []types.RetrieverType{types.KeywordsRetrieverType, types.VectorRetrieverType}
}
func (r *sqliteRepository) Save(ctx context.Context, indexInfo *types.IndexInfo, params map[string]any) error {
row := toSQLiteEmbedding(indexInfo)
emb := extractEmbedding(params, indexInfo.SourceID)
if len(emb) > 0 {
row.Dimension = len(emb)
}
if err := r.db.WithContext(ctx).Clauses(clause.OnConflict{DoNothing: true}).Create(row).Error; err != nil {
return err
}
r.syncFTS5Insert(ctx, row)
if len(emb) > 0 && row.ID > 0 {
r.insertVec(ctx, row.ID, row.Dimension, emb)
}
return nil
}
func (r *sqliteRepository) BatchSave(ctx context.Context, indexInfoList []*types.IndexInfo, params map[string]any) error {
if len(indexInfoList) == 0 {
return nil
}
rows := make([]*sqliteEmbedding, len(indexInfoList))
embs := make([][]float32, len(indexInfoList))
for i, info := range indexInfoList {
rows[i] = toSQLiteEmbedding(info)
emb := extractEmbedding(params, info.SourceID)
embs[i] = emb
if len(emb) > 0 {
rows[i].Dimension = len(emb)
}
}
if err := r.db.WithContext(ctx).Clauses(clause.OnConflict{DoNothing: true}).Create(rows).Error; err != nil {
return err
}
for i, row := range rows {
r.syncFTS5Insert(ctx, row)
if len(embs[i]) > 0 && row.ID > 0 {
r.insertVec(ctx, row.ID, row.Dimension, embs[i])
}
}
return nil
}
func (r *sqliteRepository) EstimateStorageSize(_ context.Context, indexInfoList []*types.IndexInfo, _ map[string]any) int64 {
var total int64
for _, info := range indexInfoList {
total += int64(len(info.Content)) + 200
}
return total
}
func (r *sqliteRepository) DeleteByChunkIDList(ctx context.Context, chunkIDList []string, _ int, _ string) error {
var rows []sqliteEmbedding
r.db.WithContext(ctx).Where("chunk_id IN ?", chunkIDList).Find(&rows)
r.deleteRowsAndVecs(ctx, rows)
return r.db.WithContext(ctx).Where("chunk_id IN ?", chunkIDList).Delete(&sqliteEmbedding{}).Error
}
func (r *sqliteRepository) DeleteBySourceIDList(ctx context.Context, sourceIDList []string, _ int, _ string) error {
var rows []sqliteEmbedding
r.db.WithContext(ctx).Where("source_id IN ?", sourceIDList).Find(&rows)
r.deleteRowsAndVecs(ctx, rows)
return r.db.WithContext(ctx).Where("source_id IN ?", sourceIDList).Delete(&sqliteEmbedding{}).Error
}
func (r *sqliteRepository) DeleteByKnowledgeIDList(ctx context.Context, knowledgeIDList []string, _ int, _ string) error {
var rows []sqliteEmbedding
r.db.WithContext(ctx).Where("knowledge_id IN ?", knowledgeIDList).Find(&rows)
r.deleteRowsAndVecs(ctx, rows)
return r.db.WithContext(ctx).Where("knowledge_id IN ?", knowledgeIDList).Delete(&sqliteEmbedding{}).Error
}
func (r *sqliteRepository) CopyIndices(ctx context.Context,
_ string,
sourceToTargetKBIDMap map[string]string,
sourceToTargetChunkIDMap map[string]string,
targetKnowledgeBaseID string,
_ int, _ string,
) error {
for sourceChunkID, targetChunkID := range sourceToTargetChunkIDMap {
var src sqliteEmbedding
if err := r.db.WithContext(ctx).Where("chunk_id = ?", sourceChunkID).First(&src).Error; err != nil {
continue
}
newRow := sqliteEmbedding{
SourceID: uuid.New().String(),
SourceType: src.SourceType,
ChunkID: targetChunkID,
KnowledgeID: sourceToTargetKBIDMap[src.KnowledgeID],
KnowledgeBaseID: targetKnowledgeBaseID,
TagID: src.TagID,
Content: src.Content,
Dimension: src.Dimension,
IsEnabled: src.IsEnabled,
}
if err := r.db.WithContext(ctx).Create(&newRow).Error; err != nil {
logger.GetLogger(ctx).Warnf("[SQLite] CopyIndices: failed to copy chunk %s: %v", sourceChunkID, err)
continue
}
r.syncFTS5Insert(ctx, &newRow)
if src.Dimension > 0 && newRow.ID > 0 {
r.copyVec(ctx, src.ID, newRow.ID, src.Dimension)
}
}
return nil
}
func (r *sqliteRepository) BatchUpdateChunkEnabledStatus(ctx context.Context, chunkStatusMap map[string]bool) error {
for chunkID, enabled := range chunkStatusMap {
r.db.WithContext(ctx).Model(&sqliteEmbedding{}).Where("chunk_id = ?", chunkID).Update("is_enabled", enabled)
}
return nil
}
func (r *sqliteRepository) BatchUpdateChunkTagID(ctx context.Context, chunkTagMap map[string]string) error {
for chunkID, tagID := range chunkTagMap {
r.db.WithContext(ctx).Model(&sqliteEmbedding{}).Where("chunk_id = ?", chunkID).Update("tag_id", tagID)
}
return nil
}
// --- Retrieve ---
func (r *sqliteRepository) Retrieve(ctx context.Context, params types.RetrieveParams) ([]*types.RetrieveResult, error) {
var results []*types.RetrieveResult
if params.RetrieverType == types.KeywordsRetrieverType || params.RetrieverType == "" {
res, err := r.keywordsRetrieve(ctx, params)
if err != nil {
results = append(results, &types.RetrieveResult{
RetrieverEngineType: types.SQLiteRetrieverEngineType,
RetrieverType: types.KeywordsRetrieverType,
Error: err,
})
} else {
results = append(results, res...)
}
}
if params.RetrieverType == types.VectorRetrieverType || params.RetrieverType == "" {
res, err := r.vectorRetrieve(ctx, params)
if err != nil {
results = append(results, &types.RetrieveResult{
RetrieverEngineType: types.SQLiteRetrieverEngineType,
RetrieverType: types.VectorRetrieverType,
Error: err,
})
} else {
results = append(results, res...)
}
}
return results, nil
}
// --- Keywords retrieval via FTS5 ---
func (r *sqliteRepository) keywordsRetrieve(ctx context.Context, params types.RetrieveParams) ([]*types.RetrieveResult, error) {
if params.Query == "" {
return nil, nil
}
ftsQuery := sanitizeFTS5Query(params.Query)
sql := `
SELECT e.id, e.source_id, e.source_type, e.chunk_id,
e.knowledge_id, e.knowledge_base_id, e.tag_id,
e.content,
bm25(lite_embeddings_fts) AS score
FROM lite_embeddings_fts
JOIN lite_embeddings e ON e.id = lite_embeddings_fts.rowid
WHERE lite_embeddings_fts MATCH ?
AND (e.is_enabled IS NULL OR e.is_enabled = 1)
`
args := []interface{}{ftsQuery}
for _, wp := range buildFilterWhere(params) {
sql += " AND " + wp.clause
args = append(args, wp.args...)
}
sql += " ORDER BY score ASC LIMIT ?"
args = append(args, params.TopK)
type ftsResult struct {
ID uint
SourceID string
SourceType int
ChunkID string
KnowledgeID string
KnowledgeBaseID string
TagID string
Content string
Score float64
}
var rows []ftsResult
if err := r.db.WithContext(ctx).Raw(sql, args...).Scan(&rows).Error; err != nil {
return nil, fmt.Errorf("FTS5 query failed: %w", err)
}
logger.GetLogger(ctx).Infof("[SQLite] keywordsRetrieve: query=%q, ftsQuery=%q, matched=%d rows", params.Query, ftsQuery, len(rows))
items := make([]*types.IndexWithScore, len(rows))
for i, row := range rows {
// bm25 越小越相关 → 转成正向分数
score := -row.Score
logger.GetLogger(ctx).Infof("[SQLite] keywordsRetrieve: #%d chunk_id=%s, bm25_raw=%.4f, score=%.4f, content_preview=%.60s",
i+1, row.ChunkID, row.Score, score, row.Content)
items[i] = &types.IndexWithScore{
ID: fmt.Sprintf("%d", row.ID),
SourceID: row.SourceID,
SourceType: types.SourceType(row.SourceType),
ChunkID: row.ChunkID,
KnowledgeID: row.KnowledgeID,
KnowledgeBaseID: row.KnowledgeBaseID,
TagID: row.TagID,
Content: row.Content,
Score: score,
MatchType: types.MatchTypeKeywords,
}
}
return []*types.RetrieveResult{{
Results: items,
RetrieverEngineType: types.SQLiteRetrieverEngineType,
RetrieverType: types.KeywordsRetrieverType,
}}, nil
}
func (r *sqliteRepository) vectorRetrieve(ctx context.Context, params types.RetrieveParams) ([]*types.RetrieveResult, error) {
if len(params.Embedding) == 0 {
return nil, nil
}
dim := len(params.Embedding)
r.ensureVecTable(dim)
queryBlob, err := sqlite_vec.SerializeFloat32(params.Embedding)
if err != nil {
return nil, fmt.Errorf("serialize query vector failed: %w", err)
}
tbl := vecTableName(dim)
// ⚠️ sqlite-vec 要求必须有 k = ?
vecSQL := fmt.Sprintf(`
SELECT v.rowid, v.distance,
e.source_id, e.source_type, e.chunk_id,
e.knowledge_id, e.knowledge_base_id,
e.tag_id, e.content
FROM %s v
JOIN lite_embeddings e ON e.id = v.rowid
WHERE v.embedding MATCH ?
AND k = ?
AND (e.is_enabled IS NULL OR e.is_enabled = 1)
`, tbl)
args := []interface{}{
queryBlob,
params.TopK, // 这里就是 k
}
// 追加过滤条件
for _, wp := range buildFilterWhere(params) {
vecSQL += " AND " + wp.clause
args = append(args, wp.args...)
}
// ⚠️ 这里仍然建议加 ORDER BY,虽然 vec0 已经按距离返回
vecSQL += " ORDER BY v.distance ASC"
type row struct {
Rowid uint
Distance float64
SourceID string
SourceType int
ChunkID string
KnowledgeID string
KnowledgeBaseID string
TagID string
Content string
}
var rows []row
if err := r.db.WithContext(ctx).
Raw(vecSQL, args...).
Scan(&rows).Error; err != nil {
return nil, fmt.Errorf("sqlite-vec query failed: %w", err)
}
logger.GetLogger(ctx).Infof("[SQLite] vectorRetrieve: query_dim=%d, threshold=%.4f, matched=%d rows", dim, params.Threshold, len(rows))
items := make([]*types.IndexWithScore, 0, len(rows))
for i, v := range rows {
// cosine distance = 1 - cosine_similarity
score := 1 - v.Distance
logger.GetLogger(ctx).Infof("[SQLite] vectorRetrieve: #%d chunk_id=%s, distance=%.4f, score=%.4f, content_preview=%.60s",
i+1, v.ChunkID, v.Distance, score, v.Content)
items = append(items, &types.IndexWithScore{
ID: fmt.Sprintf("%d", v.Rowid),
SourceID: v.SourceID,
SourceType: types.SourceType(v.SourceType),
ChunkID: v.ChunkID,
KnowledgeID: v.KnowledgeID,
KnowledgeBaseID: v.KnowledgeBaseID,
TagID: v.TagID,
Content: v.Content,
Score: score,
MatchType: types.MatchTypeEmbedding,
})
}
return []*types.RetrieveResult{{
Results: items,
RetrieverEngineType: types.SQLiteRetrieverEngineType,
RetrieverType: types.VectorRetrieverType,
}}, nil
}
// --- Internal helpers ---
func toSQLiteEmbedding(info *types.IndexInfo) *sqliteEmbedding {
enabled := info.IsEnabled
return &sqliteEmbedding{
SourceID: info.SourceID,
SourceType: int(info.SourceType),
ChunkID: info.ChunkID,
KnowledgeID: info.KnowledgeID,
KnowledgeBaseID: info.KnowledgeBaseID,
TagID: info.TagID,
Content: common.CleanInvalidUTF8(info.Content),
Dimension: 0,
IsEnabled: &enabled,
}
}
func extractEmbedding(params map[string]any, sourceID string) []float32 {
if params == nil {
return nil
}
embMap, ok := params["embedding"].(map[string][]float32)
if !ok {
return nil
}
return embMap[sourceID]
}
func (r *sqliteRepository) insertVec(_ context.Context, rowID uint, dim int, emb []float32) {
r.ensureVecTable(dim)
blob, err := sqlite_vec.SerializeFloat32(emb)
if err != nil {
return
}
sql := fmt.Sprintf("INSERT INTO %s(rowid, embedding) VALUES (?, ?)", vecTableName(dim))
r.db.Exec(sql, rowID, blob)
}
func (r *sqliteRepository) deleteRowsAndVecs(_ context.Context, rows []sqliteEmbedding) {
dimIDs := make(map[int][]uint)
for _, row := range rows {
if row.Dimension > 0 {
dimIDs[row.Dimension] = append(dimIDs[row.Dimension], row.ID)
}
}
for dim, ids := range dimIDs {
if !r.vecTables[dim] {
continue
}
tbl := vecTableName(dim)
for _, id := range ids {
r.db.Exec(fmt.Sprintf("DELETE FROM %s WHERE rowid = ?", tbl), id)
}
}
}
func (r *sqliteRepository) copyVec(_ context.Context, srcID, dstID uint, dim int) {
if !r.vecTables[dim] {
return
}
tbl := vecTableName(dim)
r.db.Exec(fmt.Sprintf(
"INSERT INTO %s(rowid, embedding) SELECT ?, embedding FROM %s WHERE rowid = ?",
tbl, tbl,
), dstID, srcID)
}
func (r *sqliteRepository) syncFTS5Insert(_ context.Context, row *sqliteEmbedding) {
if row.ID == 0 {
return
}
sql := `INSERT INTO lite_embeddings_fts(rowid, content, source_id, chunk_id, knowledge_id, knowledge_base_id) VALUES(?, ?, ?, ?, ?, ?)`
r.db.Exec(sql, row.ID, row.Content, row.SourceID, row.ChunkID, row.KnowledgeID, row.KnowledgeBaseID)
}
type whereClause struct {
clause string
args []interface{}
}
func buildFilterWhere(params types.RetrieveParams) []whereClause {
var parts []whereClause
if len(params.KnowledgeBaseIDs) > 0 {
parts = append(parts, whereClause{
clause: "e.knowledge_base_id IN (" + placeholders(len(params.KnowledgeBaseIDs)) + ")",
args: toInterfaceSlice(params.KnowledgeBaseIDs),
})
}
if len(params.KnowledgeIDs) > 0 {
parts = append(parts, whereClause{
clause: "e.knowledge_id IN (" + placeholders(len(params.KnowledgeIDs)) + ")",
args: toInterfaceSlice(params.KnowledgeIDs),
})
}
if len(params.TagIDs) > 0 {
parts = append(parts, whereClause{
clause: "e.tag_id IN (" + placeholders(len(params.TagIDs)) + ")",
args: toInterfaceSlice(params.TagIDs),
})
}
return parts
}
func placeholders(n int) string {
p := make([]string, n)
for i := range p {
p[i] = "?"
}
return strings.Join(p, ",")
}
func toInterfaceSlice(ss []string) []interface{} {
out := make([]interface{}, len(ss))
for i, s := range ss {
out[i] = s
}
return out
}
// sanitizeFTS5Query builds an FTS5 query from user input.
// CJK characters are split into overlapping bigrams (mimicking CJK analyzers used
// by Elasticsearch etc.) and joined with OR for broad partial matching.
// Non-CJK words are kept intact. BM25 ranking naturally boosts documents
// that match more tokens.
func sanitizeFTS5Query(q string) string {
q = strings.TrimSpace(q)
if q == "" {
return q
}
var cjkRunes []rune
var nonCJKWords []string
var buf strings.Builder
flushNonCJK := func() {
if buf.Len() > 0 {
nonCJKWords = append(nonCJKWords, buf.String())
buf.Reset()
}
}
for _, r := range q {
if unicode.Is(unicode.Han, r) {
flushNonCJK()
cjkRunes = append(cjkRunes, r)
} else if unicode.IsSpace(r) || r == '|' {
flushNonCJK()
} else if r == '"' || r == '*' || r == '(' || r == ')' || r == '{' || r == '}' {
// skip FTS5 special characters
} else {
buf.WriteRune(r)
}
}
flushNonCJK()
var parts []string
// CJK bigrams for better matching (e.g. "苹果第四季度" → "苹果" OR "果第" OR "第四" OR ...)
if len(cjkRunes) == 1 {
parts = append(parts, `"`+string(cjkRunes[0])+`"`)
} else {
for i := 0; i < len(cjkRunes)-1; i++ {
bigram := string(cjkRunes[i]) + string(cjkRunes[i+1])
parts = append(parts, `"`+bigram+`"`)
}
}
for _, w := range nonCJKWords {
w = strings.ReplaceAll(w, `"`, `""`)
parts = append(parts, `"`+w+`"`)
}
if len(parts) == 0 {
return `"` + strings.ReplaceAll(q, `"`, `""`) + `"`
}
return strings.Join(parts, " OR ")
}