397689d2f3
- Added a new `.env.lite.example` file for the Lite version, providing a minimal configuration template. - Updated `.env.example` to remove deprecated variables and include new Docreader settings. - Enhanced Docker configurations to support the Lite version, including a new Dockerfile for the Docreader service. - Introduced a Makefile target for building and running the Lite version, along with packaging capabilities. - Created GitHub workflows for building and releasing Lite binaries, including Homebrew formula support. - Implemented a new service file for managing the Lite version as a system service. This update enables a streamlined, single-binary deployment of WeKnora, reducing external dependencies and simplifying setup.
614 lines
18 KiB
Go
614 lines
18 KiB
Go
package sqlite
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"strings"
|
|
"time"
|
|
"unicode"
|
|
|
|
"github.com/Tencent/WeKnora/internal/common"
|
|
"github.com/Tencent/WeKnora/internal/logger"
|
|
"github.com/Tencent/WeKnora/internal/types"
|
|
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
|
sqlite_vec "github.com/asg017/sqlite-vec-go-bindings/cgo"
|
|
"github.com/google/uuid"
|
|
"gorm.io/gorm"
|
|
"gorm.io/gorm/clause"
|
|
)
|
|
|
|
// sqliteEmbedding stores metadata alongside the vec0 virtual table rows
|
|
type sqliteEmbedding struct {
|
|
ID uint `gorm:"primarykey;autoIncrement"`
|
|
CreatedAt time.Time `gorm:"column:created_at"`
|
|
UpdatedAt time.Time `gorm:"column:updated_at"`
|
|
SourceID string `gorm:"column:source_id;not null;uniqueIndex:idx_sqlite_emb_source"`
|
|
SourceType int `gorm:"column:source_type;not null;uniqueIndex:idx_sqlite_emb_source"`
|
|
ChunkID string `gorm:"column:chunk_id;index"`
|
|
KnowledgeID string `gorm:"column:knowledge_id;index"`
|
|
KnowledgeBaseID string `gorm:"column:knowledge_base_id;index"`
|
|
TagID string `gorm:"column:tag_id;index"`
|
|
Content string `gorm:"column:content;not null"`
|
|
Dimension int `gorm:"column:dimension;not null"`
|
|
IsEnabled *bool `gorm:"column:is_enabled;default:true;index"`
|
|
}
|
|
|
|
func (sqliteEmbedding) TableName() string { return "lite_embeddings" }
|
|
|
|
type sqliteRepository struct {
|
|
db *gorm.DB
|
|
vecTables map[int]bool // tracks which vec0 tables have been created (keyed by dimension)
|
|
}
|
|
|
|
func NewSQLiteRetrieveEngineRepository(db *gorm.DB) interfaces.RetrieveEngineRepository {
|
|
logger.GetLogger(context.Background()).Info("[SQLite] Initializing SQLite retriever engine repository with sqlite-vec")
|
|
|
|
if err := db.AutoMigrate(&sqliteEmbedding{}); err != nil {
|
|
logger.GetLogger(context.Background()).Errorf("[SQLite] Failed to auto-migrate lite_embeddings: %v", err)
|
|
}
|
|
|
|
initFTS5(db)
|
|
|
|
repo := &sqliteRepository{
|
|
db: db,
|
|
vecTables: make(map[int]bool),
|
|
}
|
|
|
|
repo.ensureExistingVecTables()
|
|
|
|
return repo
|
|
}
|
|
|
|
func initFTS5(db *gorm.DB) {
|
|
sql := `CREATE VIRTUAL TABLE IF NOT EXISTS lite_embeddings_fts USING fts5(
|
|
content, source_id, chunk_id, knowledge_id, knowledge_base_id,
|
|
content='lite_embeddings', content_rowid='id',
|
|
tokenize='unicode61'
|
|
)`
|
|
if err := db.Exec(sql).Error; err != nil {
|
|
logger.GetLogger(context.Background()).Warnf("[SQLite] Failed to create FTS5 table: %v", err)
|
|
}
|
|
}
|
|
|
|
func vecTableName(dim int) string {
|
|
return fmt.Sprintf("vec_embeddings_%d", dim)
|
|
}
|
|
|
|
func (r *sqliteRepository) ensureVecTable(dim int) {
|
|
if dim <= 0 || r.vecTables[dim] {
|
|
return
|
|
}
|
|
tbl := vecTableName(dim)
|
|
createSQL := fmt.Sprintf(
|
|
`CREATE VIRTUAL TABLE IF NOT EXISTS %s USING vec0(embedding float[%d] distance_metric=cosine)`,
|
|
tbl, dim,
|
|
)
|
|
if err := r.db.Exec(createSQL).Error; err != nil {
|
|
if strings.Contains(err.Error(), "already exists") {
|
|
r.vecTables[dim] = true
|
|
return
|
|
}
|
|
logger.GetLogger(context.Background()).Errorf("[SQLite] Failed to create vec0 table for dim %d: %v", dim, err)
|
|
return
|
|
}
|
|
r.vecTables[dim] = true
|
|
}
|
|
|
|
func (r *sqliteRepository) ensureExistingVecTables() {
|
|
var dims []int
|
|
r.db.Model(&sqliteEmbedding{}).Distinct("dimension").Where("dimension > 0").Pluck("dimension", &dims)
|
|
for _, dim := range dims {
|
|
r.ensureVecTable(dim)
|
|
}
|
|
}
|
|
|
|
func (r *sqliteRepository) EngineType() types.RetrieverEngineType {
|
|
return types.SQLiteRetrieverEngineType
|
|
}
|
|
|
|
func (r *sqliteRepository) Support() []types.RetrieverType {
|
|
return []types.RetrieverType{types.KeywordsRetrieverType, types.VectorRetrieverType}
|
|
}
|
|
|
|
func (r *sqliteRepository) Save(ctx context.Context, indexInfo *types.IndexInfo, params map[string]any) error {
|
|
row := toSQLiteEmbedding(indexInfo)
|
|
emb := extractEmbedding(params, indexInfo.SourceID)
|
|
if len(emb) > 0 {
|
|
row.Dimension = len(emb)
|
|
}
|
|
if err := r.db.WithContext(ctx).Clauses(clause.OnConflict{DoNothing: true}).Create(row).Error; err != nil {
|
|
return err
|
|
}
|
|
r.syncFTS5Insert(ctx, row)
|
|
if len(emb) > 0 && row.ID > 0 {
|
|
r.insertVec(ctx, row.ID, row.Dimension, emb)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (r *sqliteRepository) BatchSave(ctx context.Context, indexInfoList []*types.IndexInfo, params map[string]any) error {
|
|
if len(indexInfoList) == 0 {
|
|
return nil
|
|
}
|
|
rows := make([]*sqliteEmbedding, len(indexInfoList))
|
|
embs := make([][]float32, len(indexInfoList))
|
|
for i, info := range indexInfoList {
|
|
rows[i] = toSQLiteEmbedding(info)
|
|
emb := extractEmbedding(params, info.SourceID)
|
|
embs[i] = emb
|
|
if len(emb) > 0 {
|
|
rows[i].Dimension = len(emb)
|
|
}
|
|
}
|
|
if err := r.db.WithContext(ctx).Clauses(clause.OnConflict{DoNothing: true}).Create(rows).Error; err != nil {
|
|
return err
|
|
}
|
|
for i, row := range rows {
|
|
r.syncFTS5Insert(ctx, row)
|
|
if len(embs[i]) > 0 && row.ID > 0 {
|
|
r.insertVec(ctx, row.ID, row.Dimension, embs[i])
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (r *sqliteRepository) EstimateStorageSize(_ context.Context, indexInfoList []*types.IndexInfo, _ map[string]any) int64 {
|
|
var total int64
|
|
for _, info := range indexInfoList {
|
|
total += int64(len(info.Content)) + 200
|
|
}
|
|
return total
|
|
}
|
|
|
|
func (r *sqliteRepository) DeleteByChunkIDList(ctx context.Context, chunkIDList []string, _ int, _ string) error {
|
|
var rows []sqliteEmbedding
|
|
r.db.WithContext(ctx).Where("chunk_id IN ?", chunkIDList).Find(&rows)
|
|
r.deleteRowsAndVecs(ctx, rows)
|
|
return r.db.WithContext(ctx).Where("chunk_id IN ?", chunkIDList).Delete(&sqliteEmbedding{}).Error
|
|
}
|
|
|
|
func (r *sqliteRepository) DeleteBySourceIDList(ctx context.Context, sourceIDList []string, _ int, _ string) error {
|
|
var rows []sqliteEmbedding
|
|
r.db.WithContext(ctx).Where("source_id IN ?", sourceIDList).Find(&rows)
|
|
r.deleteRowsAndVecs(ctx, rows)
|
|
return r.db.WithContext(ctx).Where("source_id IN ?", sourceIDList).Delete(&sqliteEmbedding{}).Error
|
|
}
|
|
|
|
func (r *sqliteRepository) DeleteByKnowledgeIDList(ctx context.Context, knowledgeIDList []string, _ int, _ string) error {
|
|
var rows []sqliteEmbedding
|
|
r.db.WithContext(ctx).Where("knowledge_id IN ?", knowledgeIDList).Find(&rows)
|
|
r.deleteRowsAndVecs(ctx, rows)
|
|
return r.db.WithContext(ctx).Where("knowledge_id IN ?", knowledgeIDList).Delete(&sqliteEmbedding{}).Error
|
|
}
|
|
|
|
func (r *sqliteRepository) CopyIndices(ctx context.Context,
|
|
_ string,
|
|
sourceToTargetKBIDMap map[string]string,
|
|
sourceToTargetChunkIDMap map[string]string,
|
|
targetKnowledgeBaseID string,
|
|
_ int, _ string,
|
|
) error {
|
|
for sourceChunkID, targetChunkID := range sourceToTargetChunkIDMap {
|
|
var src sqliteEmbedding
|
|
if err := r.db.WithContext(ctx).Where("chunk_id = ?", sourceChunkID).First(&src).Error; err != nil {
|
|
continue
|
|
}
|
|
newRow := sqliteEmbedding{
|
|
SourceID: uuid.New().String(),
|
|
SourceType: src.SourceType,
|
|
ChunkID: targetChunkID,
|
|
KnowledgeID: sourceToTargetKBIDMap[src.KnowledgeID],
|
|
KnowledgeBaseID: targetKnowledgeBaseID,
|
|
TagID: src.TagID,
|
|
Content: src.Content,
|
|
Dimension: src.Dimension,
|
|
IsEnabled: src.IsEnabled,
|
|
}
|
|
if err := r.db.WithContext(ctx).Create(&newRow).Error; err != nil {
|
|
logger.GetLogger(ctx).Warnf("[SQLite] CopyIndices: failed to copy chunk %s: %v", sourceChunkID, err)
|
|
continue
|
|
}
|
|
r.syncFTS5Insert(ctx, &newRow)
|
|
if src.Dimension > 0 && newRow.ID > 0 {
|
|
r.copyVec(ctx, src.ID, newRow.ID, src.Dimension)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (r *sqliteRepository) BatchUpdateChunkEnabledStatus(ctx context.Context, chunkStatusMap map[string]bool) error {
|
|
for chunkID, enabled := range chunkStatusMap {
|
|
r.db.WithContext(ctx).Model(&sqliteEmbedding{}).Where("chunk_id = ?", chunkID).Update("is_enabled", enabled)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (r *sqliteRepository) BatchUpdateChunkTagID(ctx context.Context, chunkTagMap map[string]string) error {
|
|
for chunkID, tagID := range chunkTagMap {
|
|
r.db.WithContext(ctx).Model(&sqliteEmbedding{}).Where("chunk_id = ?", chunkID).Update("tag_id", tagID)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// --- Retrieve ---
|
|
|
|
func (r *sqliteRepository) Retrieve(ctx context.Context, params types.RetrieveParams) ([]*types.RetrieveResult, error) {
|
|
var results []*types.RetrieveResult
|
|
|
|
if params.RetrieverType == types.KeywordsRetrieverType || params.RetrieverType == "" {
|
|
res, err := r.keywordsRetrieve(ctx, params)
|
|
if err != nil {
|
|
results = append(results, &types.RetrieveResult{
|
|
RetrieverEngineType: types.SQLiteRetrieverEngineType,
|
|
RetrieverType: types.KeywordsRetrieverType,
|
|
Error: err,
|
|
})
|
|
} else {
|
|
results = append(results, res...)
|
|
}
|
|
}
|
|
|
|
if params.RetrieverType == types.VectorRetrieverType || params.RetrieverType == "" {
|
|
res, err := r.vectorRetrieve(ctx, params)
|
|
if err != nil {
|
|
results = append(results, &types.RetrieveResult{
|
|
RetrieverEngineType: types.SQLiteRetrieverEngineType,
|
|
RetrieverType: types.VectorRetrieverType,
|
|
Error: err,
|
|
})
|
|
} else {
|
|
results = append(results, res...)
|
|
}
|
|
}
|
|
|
|
return results, nil
|
|
}
|
|
|
|
// --- Keywords retrieval via FTS5 ---
|
|
func (r *sqliteRepository) keywordsRetrieve(ctx context.Context, params types.RetrieveParams) ([]*types.RetrieveResult, error) {
|
|
if params.Query == "" {
|
|
return nil, nil
|
|
}
|
|
|
|
ftsQuery := sanitizeFTS5Query(params.Query)
|
|
|
|
sql := `
|
|
SELECT e.id, e.source_id, e.source_type, e.chunk_id,
|
|
e.knowledge_id, e.knowledge_base_id, e.tag_id,
|
|
e.content,
|
|
bm25(lite_embeddings_fts) AS score
|
|
FROM lite_embeddings_fts
|
|
JOIN lite_embeddings e ON e.id = lite_embeddings_fts.rowid
|
|
WHERE lite_embeddings_fts MATCH ?
|
|
AND (e.is_enabled IS NULL OR e.is_enabled = 1)
|
|
`
|
|
|
|
args := []interface{}{ftsQuery}
|
|
|
|
for _, wp := range buildFilterWhere(params) {
|
|
sql += " AND " + wp.clause
|
|
args = append(args, wp.args...)
|
|
}
|
|
|
|
sql += " ORDER BY score ASC LIMIT ?"
|
|
args = append(args, params.TopK)
|
|
|
|
type ftsResult struct {
|
|
ID uint
|
|
SourceID string
|
|
SourceType int
|
|
ChunkID string
|
|
KnowledgeID string
|
|
KnowledgeBaseID string
|
|
TagID string
|
|
Content string
|
|
Score float64
|
|
}
|
|
|
|
var rows []ftsResult
|
|
if err := r.db.WithContext(ctx).Raw(sql, args...).Scan(&rows).Error; err != nil {
|
|
return nil, fmt.Errorf("FTS5 query failed: %w", err)
|
|
}
|
|
|
|
logger.GetLogger(ctx).Infof("[SQLite] keywordsRetrieve: query=%q, ftsQuery=%q, matched=%d rows", params.Query, ftsQuery, len(rows))
|
|
|
|
items := make([]*types.IndexWithScore, len(rows))
|
|
for i, row := range rows {
|
|
|
|
// bm25 越小越相关 → 转成正向分数
|
|
score := -row.Score
|
|
|
|
logger.GetLogger(ctx).Infof("[SQLite] keywordsRetrieve: #%d chunk_id=%s, bm25_raw=%.4f, score=%.4f, content_preview=%.60s",
|
|
i+1, row.ChunkID, row.Score, score, row.Content)
|
|
|
|
items[i] = &types.IndexWithScore{
|
|
ID: fmt.Sprintf("%d", row.ID),
|
|
SourceID: row.SourceID,
|
|
SourceType: types.SourceType(row.SourceType),
|
|
ChunkID: row.ChunkID,
|
|
KnowledgeID: row.KnowledgeID,
|
|
KnowledgeBaseID: row.KnowledgeBaseID,
|
|
TagID: row.TagID,
|
|
Content: row.Content,
|
|
Score: score,
|
|
MatchType: types.MatchTypeKeywords,
|
|
}
|
|
}
|
|
|
|
return []*types.RetrieveResult{{
|
|
Results: items,
|
|
RetrieverEngineType: types.SQLiteRetrieverEngineType,
|
|
RetrieverType: types.KeywordsRetrieverType,
|
|
}}, nil
|
|
}
|
|
func (r *sqliteRepository) vectorRetrieve(ctx context.Context, params types.RetrieveParams) ([]*types.RetrieveResult, error) {
|
|
if len(params.Embedding) == 0 {
|
|
return nil, nil
|
|
}
|
|
|
|
dim := len(params.Embedding)
|
|
r.ensureVecTable(dim)
|
|
|
|
queryBlob, err := sqlite_vec.SerializeFloat32(params.Embedding)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("serialize query vector failed: %w", err)
|
|
}
|
|
|
|
tbl := vecTableName(dim)
|
|
|
|
// ⚠️ sqlite-vec 要求必须有 k = ?
|
|
vecSQL := fmt.Sprintf(`
|
|
SELECT v.rowid, v.distance,
|
|
e.source_id, e.source_type, e.chunk_id,
|
|
e.knowledge_id, e.knowledge_base_id,
|
|
e.tag_id, e.content
|
|
FROM %s v
|
|
JOIN lite_embeddings e ON e.id = v.rowid
|
|
WHERE v.embedding MATCH ?
|
|
AND k = ?
|
|
AND (e.is_enabled IS NULL OR e.is_enabled = 1)
|
|
`, tbl)
|
|
|
|
args := []interface{}{
|
|
queryBlob,
|
|
params.TopK, // 这里就是 k
|
|
}
|
|
|
|
// 追加过滤条件
|
|
for _, wp := range buildFilterWhere(params) {
|
|
vecSQL += " AND " + wp.clause
|
|
args = append(args, wp.args...)
|
|
}
|
|
|
|
// ⚠️ 这里仍然建议加 ORDER BY,虽然 vec0 已经按距离返回
|
|
vecSQL += " ORDER BY v.distance ASC"
|
|
|
|
type row struct {
|
|
Rowid uint
|
|
Distance float64
|
|
SourceID string
|
|
SourceType int
|
|
ChunkID string
|
|
KnowledgeID string
|
|
KnowledgeBaseID string
|
|
TagID string
|
|
Content string
|
|
}
|
|
|
|
var rows []row
|
|
if err := r.db.WithContext(ctx).
|
|
Raw(vecSQL, args...).
|
|
Scan(&rows).Error; err != nil {
|
|
return nil, fmt.Errorf("sqlite-vec query failed: %w", err)
|
|
}
|
|
|
|
logger.GetLogger(ctx).Infof("[SQLite] vectorRetrieve: query_dim=%d, threshold=%.4f, matched=%d rows", dim, params.Threshold, len(rows))
|
|
|
|
items := make([]*types.IndexWithScore, 0, len(rows))
|
|
|
|
for i, v := range rows {
|
|
// cosine distance = 1 - cosine_similarity
|
|
score := 1 - v.Distance
|
|
|
|
logger.GetLogger(ctx).Infof("[SQLite] vectorRetrieve: #%d chunk_id=%s, distance=%.4f, score=%.4f, content_preview=%.60s",
|
|
i+1, v.ChunkID, v.Distance, score, v.Content)
|
|
|
|
items = append(items, &types.IndexWithScore{
|
|
ID: fmt.Sprintf("%d", v.Rowid),
|
|
SourceID: v.SourceID,
|
|
SourceType: types.SourceType(v.SourceType),
|
|
ChunkID: v.ChunkID,
|
|
KnowledgeID: v.KnowledgeID,
|
|
KnowledgeBaseID: v.KnowledgeBaseID,
|
|
TagID: v.TagID,
|
|
Content: v.Content,
|
|
Score: score,
|
|
MatchType: types.MatchTypeEmbedding,
|
|
})
|
|
}
|
|
|
|
return []*types.RetrieveResult{{
|
|
Results: items,
|
|
RetrieverEngineType: types.SQLiteRetrieverEngineType,
|
|
RetrieverType: types.VectorRetrieverType,
|
|
}}, nil
|
|
}
|
|
|
|
// --- Internal helpers ---
|
|
|
|
func toSQLiteEmbedding(info *types.IndexInfo) *sqliteEmbedding {
|
|
enabled := true
|
|
return &sqliteEmbedding{
|
|
SourceID: info.SourceID,
|
|
SourceType: int(info.SourceType),
|
|
ChunkID: info.ChunkID,
|
|
KnowledgeID: info.KnowledgeID,
|
|
KnowledgeBaseID: info.KnowledgeBaseID,
|
|
TagID: info.TagID,
|
|
Content: common.CleanInvalidUTF8(info.Content),
|
|
Dimension: 0,
|
|
IsEnabled: &enabled,
|
|
}
|
|
}
|
|
|
|
func extractEmbedding(params map[string]any, sourceID string) []float32 {
|
|
if params == nil {
|
|
return nil
|
|
}
|
|
embMap, ok := params["embedding"].(map[string][]float32)
|
|
if !ok {
|
|
return nil
|
|
}
|
|
return embMap[sourceID]
|
|
}
|
|
|
|
func (r *sqliteRepository) insertVec(_ context.Context, rowID uint, dim int, emb []float32) {
|
|
r.ensureVecTable(dim)
|
|
blob, err := sqlite_vec.SerializeFloat32(emb)
|
|
if err != nil {
|
|
return
|
|
}
|
|
sql := fmt.Sprintf("INSERT INTO %s(rowid, embedding) VALUES (?, ?)", vecTableName(dim))
|
|
r.db.Exec(sql, rowID, blob)
|
|
}
|
|
|
|
func (r *sqliteRepository) deleteRowsAndVecs(_ context.Context, rows []sqliteEmbedding) {
|
|
dimIDs := make(map[int][]uint)
|
|
for _, row := range rows {
|
|
if row.Dimension > 0 {
|
|
dimIDs[row.Dimension] = append(dimIDs[row.Dimension], row.ID)
|
|
}
|
|
}
|
|
for dim, ids := range dimIDs {
|
|
if !r.vecTables[dim] {
|
|
continue
|
|
}
|
|
tbl := vecTableName(dim)
|
|
for _, id := range ids {
|
|
r.db.Exec(fmt.Sprintf("DELETE FROM %s WHERE rowid = ?", tbl), id)
|
|
}
|
|
}
|
|
}
|
|
|
|
func (r *sqliteRepository) copyVec(_ context.Context, srcID, dstID uint, dim int) {
|
|
if !r.vecTables[dim] {
|
|
return
|
|
}
|
|
tbl := vecTableName(dim)
|
|
r.db.Exec(fmt.Sprintf(
|
|
"INSERT INTO %s(rowid, embedding) SELECT ?, embedding FROM %s WHERE rowid = ?",
|
|
tbl, tbl,
|
|
), dstID, srcID)
|
|
}
|
|
|
|
func (r *sqliteRepository) syncFTS5Insert(_ context.Context, row *sqliteEmbedding) {
|
|
if row.ID == 0 {
|
|
return
|
|
}
|
|
sql := `INSERT INTO lite_embeddings_fts(rowid, content, source_id, chunk_id, knowledge_id, knowledge_base_id) VALUES(?, ?, ?, ?, ?, ?)`
|
|
r.db.Exec(sql, row.ID, row.Content, row.SourceID, row.ChunkID, row.KnowledgeID, row.KnowledgeBaseID)
|
|
}
|
|
|
|
type whereClause struct {
|
|
clause string
|
|
args []interface{}
|
|
}
|
|
|
|
func buildFilterWhere(params types.RetrieveParams) []whereClause {
|
|
var parts []whereClause
|
|
if len(params.KnowledgeBaseIDs) > 0 {
|
|
parts = append(parts, whereClause{
|
|
clause: "e.knowledge_base_id IN (" + placeholders(len(params.KnowledgeBaseIDs)) + ")",
|
|
args: toInterfaceSlice(params.KnowledgeBaseIDs),
|
|
})
|
|
}
|
|
if len(params.KnowledgeIDs) > 0 {
|
|
parts = append(parts, whereClause{
|
|
clause: "e.knowledge_id IN (" + placeholders(len(params.KnowledgeIDs)) + ")",
|
|
args: toInterfaceSlice(params.KnowledgeIDs),
|
|
})
|
|
}
|
|
if len(params.TagIDs) > 0 {
|
|
parts = append(parts, whereClause{
|
|
clause: "e.tag_id IN (" + placeholders(len(params.TagIDs)) + ")",
|
|
args: toInterfaceSlice(params.TagIDs),
|
|
})
|
|
}
|
|
return parts
|
|
}
|
|
|
|
func placeholders(n int) string {
|
|
p := make([]string, n)
|
|
for i := range p {
|
|
p[i] = "?"
|
|
}
|
|
return strings.Join(p, ",")
|
|
}
|
|
|
|
func toInterfaceSlice(ss []string) []interface{} {
|
|
out := make([]interface{}, len(ss))
|
|
for i, s := range ss {
|
|
out[i] = s
|
|
}
|
|
return out
|
|
}
|
|
|
|
// sanitizeFTS5Query builds an FTS5 query from user input.
|
|
// CJK characters are split into overlapping bigrams (mimicking CJK analyzers used
|
|
// by Elasticsearch etc.) and joined with OR for broad partial matching.
|
|
// Non-CJK words are kept intact. BM25 ranking naturally boosts documents
|
|
// that match more tokens.
|
|
func sanitizeFTS5Query(q string) string {
|
|
q = strings.TrimSpace(q)
|
|
if q == "" {
|
|
return q
|
|
}
|
|
|
|
var cjkRunes []rune
|
|
var nonCJKWords []string
|
|
var buf strings.Builder
|
|
|
|
flushNonCJK := func() {
|
|
if buf.Len() > 0 {
|
|
nonCJKWords = append(nonCJKWords, buf.String())
|
|
buf.Reset()
|
|
}
|
|
}
|
|
|
|
for _, r := range q {
|
|
if unicode.Is(unicode.Han, r) {
|
|
flushNonCJK()
|
|
cjkRunes = append(cjkRunes, r)
|
|
} else if unicode.IsSpace(r) || r == '|' {
|
|
flushNonCJK()
|
|
} else if r == '"' || r == '*' || r == '(' || r == ')' || r == '{' || r == '}' {
|
|
// skip FTS5 special characters
|
|
} else {
|
|
buf.WriteRune(r)
|
|
}
|
|
}
|
|
flushNonCJK()
|
|
|
|
var parts []string
|
|
|
|
// CJK bigrams for better matching (e.g. "苹果第四季度" → "苹果" OR "果第" OR "第四" OR ...)
|
|
if len(cjkRunes) == 1 {
|
|
parts = append(parts, `"`+string(cjkRunes[0])+`"`)
|
|
} else {
|
|
for i := 0; i < len(cjkRunes)-1; i++ {
|
|
bigram := string(cjkRunes[i]) + string(cjkRunes[i+1])
|
|
parts = append(parts, `"`+bigram+`"`)
|
|
}
|
|
}
|
|
|
|
for _, w := range nonCJKWords {
|
|
w = strings.ReplaceAll(w, `"`, `""`)
|
|
parts = append(parts, `"`+w+`"`)
|
|
}
|
|
|
|
if len(parts) == 0 {
|
|
return `"` + strings.ReplaceAll(q, `"`, `""`) + `"`
|
|
}
|
|
return strings.Join(parts, " OR ")
|
|
}
|