feat: introduce WeKnora Lite edition with lightweight configuration and deployment
- Added a new `.env.lite.example` file for the Lite version, providing a minimal configuration template. - Updated `.env.example` to remove deprecated variables and include new Docreader settings. - Enhanced Docker configurations to support the Lite version, including a new Dockerfile for the Docreader service. - Introduced a Makefile target for building and running the Lite version, along with packaging capabilities. - Created GitHub workflows for building and releasing Lite binaries, including Homebrew formula support. - Implemented a new service file for managing the Lite version as a system service. This update enables a streamlined, single-binary deployment of WeKnora, reducing external dependencies and simplifying setup.
This commit is contained in:
@@ -246,11 +246,11 @@ func (r *chunkRepository) ListChunkByParentID(
|
||||
return chunks, nil
|
||||
}
|
||||
|
||||
// UpdateChunk updates a chunk using GORM Save, which updates ALL fields.
|
||||
// Note: This will update all fields including metadata and content_hash.
|
||||
// UpdateChunk updates a chunk using GORM Save, which updates ALL fields
|
||||
// except SeqID (auto-increment, must not be overwritten).
|
||||
// Make sure the chunk object is complete (e.g., fetched from DB) before calling this method.
|
||||
func (r *chunkRepository) UpdateChunk(ctx context.Context, chunk *types.Chunk) error {
|
||||
return r.db.WithContext(ctx).Save(chunk).Error
|
||||
return r.db.WithContext(ctx).Omit("SeqID").Save(chunk).Error
|
||||
}
|
||||
|
||||
// UpdateChunks updates chunks in batch using raw SQL for efficiency.
|
||||
@@ -332,23 +332,46 @@ func (r *chunkRepository) UpdateChunks(ctx context.Context, chunks []*types.Chun
|
||||
args = append(args, id)
|
||||
}
|
||||
|
||||
sql := fmt.Sprintf(`
|
||||
UPDATE chunks SET
|
||||
content = CASE %s END,
|
||||
is_enabled = (CASE %s END)::boolean,
|
||||
tag_id = CASE %s END,
|
||||
flags = (CASE %s END)::integer,
|
||||
status = (CASE %s END)::integer,
|
||||
updated_at = NOW()
|
||||
WHERE id IN (%s)
|
||||
`,
|
||||
strings.Join(contentCases, " "),
|
||||
strings.Join(isEnabledCases, " "),
|
||||
strings.Join(tagIDCases, " "),
|
||||
strings.Join(flagsCases, " "),
|
||||
strings.Join(statusCases, " "),
|
||||
strings.Join(inPlaceholders, ","),
|
||||
)
|
||||
isPostgres := r.db.Dialector.Name() == "postgres"
|
||||
|
||||
var sql string
|
||||
if isPostgres {
|
||||
sql = fmt.Sprintf(`
|
||||
UPDATE chunks SET
|
||||
content = CASE %s END,
|
||||
is_enabled = (CASE %s END)::boolean,
|
||||
tag_id = CASE %s END,
|
||||
flags = (CASE %s END)::integer,
|
||||
status = (CASE %s END)::integer,
|
||||
updated_at = NOW()
|
||||
WHERE id IN (%s)
|
||||
`,
|
||||
strings.Join(contentCases, " "),
|
||||
strings.Join(isEnabledCases, " "),
|
||||
strings.Join(tagIDCases, " "),
|
||||
strings.Join(flagsCases, " "),
|
||||
strings.Join(statusCases, " "),
|
||||
strings.Join(inPlaceholders, ","),
|
||||
)
|
||||
} else {
|
||||
sql = fmt.Sprintf(`
|
||||
UPDATE chunks SET
|
||||
content = CASE %s END,
|
||||
is_enabled = CASE %s END,
|
||||
tag_id = CASE %s END,
|
||||
flags = CASE %s END,
|
||||
status = CASE %s END,
|
||||
updated_at = datetime('now')
|
||||
WHERE id IN (%s)
|
||||
`,
|
||||
strings.Join(contentCases, " "),
|
||||
strings.Join(isEnabledCases, " "),
|
||||
strings.Join(tagIDCases, " "),
|
||||
strings.Join(flagsCases, " "),
|
||||
strings.Join(statusCases, " "),
|
||||
strings.Join(inPlaceholders, ","),
|
||||
)
|
||||
}
|
||||
|
||||
return r.db.WithContext(ctx).Exec(sql, args...).Error
|
||||
}
|
||||
|
||||
@@ -0,0 +1,613 @@
|
||||
package sqlite
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/common"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
sqlite_vec "github.com/asg017/sqlite-vec-go-bindings/cgo"
|
||||
"github.com/google/uuid"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
)
|
||||
|
||||
// sqliteEmbedding stores metadata alongside the vec0 virtual table rows
|
||||
type sqliteEmbedding struct {
|
||||
ID uint `gorm:"primarykey;autoIncrement"`
|
||||
CreatedAt time.Time `gorm:"column:created_at"`
|
||||
UpdatedAt time.Time `gorm:"column:updated_at"`
|
||||
SourceID string `gorm:"column:source_id;not null;uniqueIndex:idx_sqlite_emb_source"`
|
||||
SourceType int `gorm:"column:source_type;not null;uniqueIndex:idx_sqlite_emb_source"`
|
||||
ChunkID string `gorm:"column:chunk_id;index"`
|
||||
KnowledgeID string `gorm:"column:knowledge_id;index"`
|
||||
KnowledgeBaseID string `gorm:"column:knowledge_base_id;index"`
|
||||
TagID string `gorm:"column:tag_id;index"`
|
||||
Content string `gorm:"column:content;not null"`
|
||||
Dimension int `gorm:"column:dimension;not null"`
|
||||
IsEnabled *bool `gorm:"column:is_enabled;default:true;index"`
|
||||
}
|
||||
|
||||
func (sqliteEmbedding) TableName() string { return "lite_embeddings" }
|
||||
|
||||
type sqliteRepository struct {
|
||||
db *gorm.DB
|
||||
vecTables map[int]bool // tracks which vec0 tables have been created (keyed by dimension)
|
||||
}
|
||||
|
||||
func NewSQLiteRetrieveEngineRepository(db *gorm.DB) interfaces.RetrieveEngineRepository {
|
||||
logger.GetLogger(context.Background()).Info("[SQLite] Initializing SQLite retriever engine repository with sqlite-vec")
|
||||
|
||||
if err := db.AutoMigrate(&sqliteEmbedding{}); err != nil {
|
||||
logger.GetLogger(context.Background()).Errorf("[SQLite] Failed to auto-migrate lite_embeddings: %v", err)
|
||||
}
|
||||
|
||||
initFTS5(db)
|
||||
|
||||
repo := &sqliteRepository{
|
||||
db: db,
|
||||
vecTables: make(map[int]bool),
|
||||
}
|
||||
|
||||
repo.ensureExistingVecTables()
|
||||
|
||||
return repo
|
||||
}
|
||||
|
||||
func initFTS5(db *gorm.DB) {
|
||||
sql := `CREATE VIRTUAL TABLE IF NOT EXISTS lite_embeddings_fts USING fts5(
|
||||
content, source_id, chunk_id, knowledge_id, knowledge_base_id,
|
||||
content='lite_embeddings', content_rowid='id',
|
||||
tokenize='unicode61'
|
||||
)`
|
||||
if err := db.Exec(sql).Error; err != nil {
|
||||
logger.GetLogger(context.Background()).Warnf("[SQLite] Failed to create FTS5 table: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func vecTableName(dim int) string {
|
||||
return fmt.Sprintf("vec_embeddings_%d", dim)
|
||||
}
|
||||
|
||||
func (r *sqliteRepository) ensureVecTable(dim int) {
|
||||
if dim <= 0 || r.vecTables[dim] {
|
||||
return
|
||||
}
|
||||
tbl := vecTableName(dim)
|
||||
createSQL := fmt.Sprintf(
|
||||
`CREATE VIRTUAL TABLE IF NOT EXISTS %s USING vec0(embedding float[%d] distance_metric=cosine)`,
|
||||
tbl, dim,
|
||||
)
|
||||
if err := r.db.Exec(createSQL).Error; err != nil {
|
||||
if strings.Contains(err.Error(), "already exists") {
|
||||
r.vecTables[dim] = true
|
||||
return
|
||||
}
|
||||
logger.GetLogger(context.Background()).Errorf("[SQLite] Failed to create vec0 table for dim %d: %v", dim, err)
|
||||
return
|
||||
}
|
||||
r.vecTables[dim] = true
|
||||
}
|
||||
|
||||
func (r *sqliteRepository) ensureExistingVecTables() {
|
||||
var dims []int
|
||||
r.db.Model(&sqliteEmbedding{}).Distinct("dimension").Where("dimension > 0").Pluck("dimension", &dims)
|
||||
for _, dim := range dims {
|
||||
r.ensureVecTable(dim)
|
||||
}
|
||||
}
|
||||
|
||||
func (r *sqliteRepository) EngineType() types.RetrieverEngineType {
|
||||
return types.SQLiteRetrieverEngineType
|
||||
}
|
||||
|
||||
func (r *sqliteRepository) Support() []types.RetrieverType {
|
||||
return []types.RetrieverType{types.KeywordsRetrieverType, types.VectorRetrieverType}
|
||||
}
|
||||
|
||||
func (r *sqliteRepository) Save(ctx context.Context, indexInfo *types.IndexInfo, params map[string]any) error {
|
||||
row := toSQLiteEmbedding(indexInfo)
|
||||
emb := extractEmbedding(params, indexInfo.SourceID)
|
||||
if len(emb) > 0 {
|
||||
row.Dimension = len(emb)
|
||||
}
|
||||
if err := r.db.WithContext(ctx).Clauses(clause.OnConflict{DoNothing: true}).Create(row).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
r.syncFTS5Insert(ctx, row)
|
||||
if len(emb) > 0 && row.ID > 0 {
|
||||
r.insertVec(ctx, row.ID, row.Dimension, emb)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *sqliteRepository) BatchSave(ctx context.Context, indexInfoList []*types.IndexInfo, params map[string]any) error {
|
||||
if len(indexInfoList) == 0 {
|
||||
return nil
|
||||
}
|
||||
rows := make([]*sqliteEmbedding, len(indexInfoList))
|
||||
embs := make([][]float32, len(indexInfoList))
|
||||
for i, info := range indexInfoList {
|
||||
rows[i] = toSQLiteEmbedding(info)
|
||||
emb := extractEmbedding(params, info.SourceID)
|
||||
embs[i] = emb
|
||||
if len(emb) > 0 {
|
||||
rows[i].Dimension = len(emb)
|
||||
}
|
||||
}
|
||||
if err := r.db.WithContext(ctx).Clauses(clause.OnConflict{DoNothing: true}).Create(rows).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
for i, row := range rows {
|
||||
r.syncFTS5Insert(ctx, row)
|
||||
if len(embs[i]) > 0 && row.ID > 0 {
|
||||
r.insertVec(ctx, row.ID, row.Dimension, embs[i])
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *sqliteRepository) EstimateStorageSize(_ context.Context, indexInfoList []*types.IndexInfo, _ map[string]any) int64 {
|
||||
var total int64
|
||||
for _, info := range indexInfoList {
|
||||
total += int64(len(info.Content)) + 200
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
func (r *sqliteRepository) DeleteByChunkIDList(ctx context.Context, chunkIDList []string, _ int, _ string) error {
|
||||
var rows []sqliteEmbedding
|
||||
r.db.WithContext(ctx).Where("chunk_id IN ?", chunkIDList).Find(&rows)
|
||||
r.deleteRowsAndVecs(ctx, rows)
|
||||
return r.db.WithContext(ctx).Where("chunk_id IN ?", chunkIDList).Delete(&sqliteEmbedding{}).Error
|
||||
}
|
||||
|
||||
func (r *sqliteRepository) DeleteBySourceIDList(ctx context.Context, sourceIDList []string, _ int, _ string) error {
|
||||
var rows []sqliteEmbedding
|
||||
r.db.WithContext(ctx).Where("source_id IN ?", sourceIDList).Find(&rows)
|
||||
r.deleteRowsAndVecs(ctx, rows)
|
||||
return r.db.WithContext(ctx).Where("source_id IN ?", sourceIDList).Delete(&sqliteEmbedding{}).Error
|
||||
}
|
||||
|
||||
func (r *sqliteRepository) DeleteByKnowledgeIDList(ctx context.Context, knowledgeIDList []string, _ int, _ string) error {
|
||||
var rows []sqliteEmbedding
|
||||
r.db.WithContext(ctx).Where("knowledge_id IN ?", knowledgeIDList).Find(&rows)
|
||||
r.deleteRowsAndVecs(ctx, rows)
|
||||
return r.db.WithContext(ctx).Where("knowledge_id IN ?", knowledgeIDList).Delete(&sqliteEmbedding{}).Error
|
||||
}
|
||||
|
||||
func (r *sqliteRepository) CopyIndices(ctx context.Context,
|
||||
_ string,
|
||||
sourceToTargetKBIDMap map[string]string,
|
||||
sourceToTargetChunkIDMap map[string]string,
|
||||
targetKnowledgeBaseID string,
|
||||
_ int, _ string,
|
||||
) error {
|
||||
for sourceChunkID, targetChunkID := range sourceToTargetChunkIDMap {
|
||||
var src sqliteEmbedding
|
||||
if err := r.db.WithContext(ctx).Where("chunk_id = ?", sourceChunkID).First(&src).Error; err != nil {
|
||||
continue
|
||||
}
|
||||
newRow := sqliteEmbedding{
|
||||
SourceID: uuid.New().String(),
|
||||
SourceType: src.SourceType,
|
||||
ChunkID: targetChunkID,
|
||||
KnowledgeID: sourceToTargetKBIDMap[src.KnowledgeID],
|
||||
KnowledgeBaseID: targetKnowledgeBaseID,
|
||||
TagID: src.TagID,
|
||||
Content: src.Content,
|
||||
Dimension: src.Dimension,
|
||||
IsEnabled: src.IsEnabled,
|
||||
}
|
||||
if err := r.db.WithContext(ctx).Create(&newRow).Error; err != nil {
|
||||
logger.GetLogger(ctx).Warnf("[SQLite] CopyIndices: failed to copy chunk %s: %v", sourceChunkID, err)
|
||||
continue
|
||||
}
|
||||
r.syncFTS5Insert(ctx, &newRow)
|
||||
if src.Dimension > 0 && newRow.ID > 0 {
|
||||
r.copyVec(ctx, src.ID, newRow.ID, src.Dimension)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *sqliteRepository) BatchUpdateChunkEnabledStatus(ctx context.Context, chunkStatusMap map[string]bool) error {
|
||||
for chunkID, enabled := range chunkStatusMap {
|
||||
r.db.WithContext(ctx).Model(&sqliteEmbedding{}).Where("chunk_id = ?", chunkID).Update("is_enabled", enabled)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *sqliteRepository) BatchUpdateChunkTagID(ctx context.Context, chunkTagMap map[string]string) error {
|
||||
for chunkID, tagID := range chunkTagMap {
|
||||
r.db.WithContext(ctx).Model(&sqliteEmbedding{}).Where("chunk_id = ?", chunkID).Update("tag_id", tagID)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// --- Retrieve ---
|
||||
|
||||
func (r *sqliteRepository) Retrieve(ctx context.Context, params types.RetrieveParams) ([]*types.RetrieveResult, error) {
|
||||
var results []*types.RetrieveResult
|
||||
|
||||
if params.RetrieverType == types.KeywordsRetrieverType || params.RetrieverType == "" {
|
||||
res, err := r.keywordsRetrieve(ctx, params)
|
||||
if err != nil {
|
||||
results = append(results, &types.RetrieveResult{
|
||||
RetrieverEngineType: types.SQLiteRetrieverEngineType,
|
||||
RetrieverType: types.KeywordsRetrieverType,
|
||||
Error: err,
|
||||
})
|
||||
} else {
|
||||
results = append(results, res...)
|
||||
}
|
||||
}
|
||||
|
||||
if params.RetrieverType == types.VectorRetrieverType || params.RetrieverType == "" {
|
||||
res, err := r.vectorRetrieve(ctx, params)
|
||||
if err != nil {
|
||||
results = append(results, &types.RetrieveResult{
|
||||
RetrieverEngineType: types.SQLiteRetrieverEngineType,
|
||||
RetrieverType: types.VectorRetrieverType,
|
||||
Error: err,
|
||||
})
|
||||
} else {
|
||||
results = append(results, res...)
|
||||
}
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
// --- Keywords retrieval via FTS5 ---
|
||||
func (r *sqliteRepository) keywordsRetrieve(ctx context.Context, params types.RetrieveParams) ([]*types.RetrieveResult, error) {
|
||||
if params.Query == "" {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
ftsQuery := sanitizeFTS5Query(params.Query)
|
||||
|
||||
sql := `
|
||||
SELECT e.id, e.source_id, e.source_type, e.chunk_id,
|
||||
e.knowledge_id, e.knowledge_base_id, e.tag_id,
|
||||
e.content,
|
||||
bm25(lite_embeddings_fts) AS score
|
||||
FROM lite_embeddings_fts
|
||||
JOIN lite_embeddings e ON e.id = lite_embeddings_fts.rowid
|
||||
WHERE lite_embeddings_fts MATCH ?
|
||||
AND (e.is_enabled IS NULL OR e.is_enabled = 1)
|
||||
`
|
||||
|
||||
args := []interface{}{ftsQuery}
|
||||
|
||||
for _, wp := range buildFilterWhere(params) {
|
||||
sql += " AND " + wp.clause
|
||||
args = append(args, wp.args...)
|
||||
}
|
||||
|
||||
sql += " ORDER BY score ASC LIMIT ?"
|
||||
args = append(args, params.TopK)
|
||||
|
||||
type ftsResult struct {
|
||||
ID uint
|
||||
SourceID string
|
||||
SourceType int
|
||||
ChunkID string
|
||||
KnowledgeID string
|
||||
KnowledgeBaseID string
|
||||
TagID string
|
||||
Content string
|
||||
Score float64
|
||||
}
|
||||
|
||||
var rows []ftsResult
|
||||
if err := r.db.WithContext(ctx).Raw(sql, args...).Scan(&rows).Error; err != nil {
|
||||
return nil, fmt.Errorf("FTS5 query failed: %w", err)
|
||||
}
|
||||
|
||||
logger.GetLogger(ctx).Infof("[SQLite] keywordsRetrieve: query=%q, ftsQuery=%q, matched=%d rows", params.Query, ftsQuery, len(rows))
|
||||
|
||||
items := make([]*types.IndexWithScore, len(rows))
|
||||
for i, row := range rows {
|
||||
|
||||
// bm25 越小越相关 → 转成正向分数
|
||||
score := -row.Score
|
||||
|
||||
logger.GetLogger(ctx).Infof("[SQLite] keywordsRetrieve: #%d chunk_id=%s, bm25_raw=%.4f, score=%.4f, content_preview=%.60s",
|
||||
i+1, row.ChunkID, row.Score, score, row.Content)
|
||||
|
||||
items[i] = &types.IndexWithScore{
|
||||
ID: fmt.Sprintf("%d", row.ID),
|
||||
SourceID: row.SourceID,
|
||||
SourceType: types.SourceType(row.SourceType),
|
||||
ChunkID: row.ChunkID,
|
||||
KnowledgeID: row.KnowledgeID,
|
||||
KnowledgeBaseID: row.KnowledgeBaseID,
|
||||
TagID: row.TagID,
|
||||
Content: row.Content,
|
||||
Score: score,
|
||||
MatchType: types.MatchTypeKeywords,
|
||||
}
|
||||
}
|
||||
|
||||
return []*types.RetrieveResult{{
|
||||
Results: items,
|
||||
RetrieverEngineType: types.SQLiteRetrieverEngineType,
|
||||
RetrieverType: types.KeywordsRetrieverType,
|
||||
}}, nil
|
||||
}
|
||||
func (r *sqliteRepository) vectorRetrieve(ctx context.Context, params types.RetrieveParams) ([]*types.RetrieveResult, error) {
|
||||
if len(params.Embedding) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
dim := len(params.Embedding)
|
||||
r.ensureVecTable(dim)
|
||||
|
||||
queryBlob, err := sqlite_vec.SerializeFloat32(params.Embedding)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("serialize query vector failed: %w", err)
|
||||
}
|
||||
|
||||
tbl := vecTableName(dim)
|
||||
|
||||
// ⚠️ sqlite-vec 要求必须有 k = ?
|
||||
vecSQL := fmt.Sprintf(`
|
||||
SELECT v.rowid, v.distance,
|
||||
e.source_id, e.source_type, e.chunk_id,
|
||||
e.knowledge_id, e.knowledge_base_id,
|
||||
e.tag_id, e.content
|
||||
FROM %s v
|
||||
JOIN lite_embeddings e ON e.id = v.rowid
|
||||
WHERE v.embedding MATCH ?
|
||||
AND k = ?
|
||||
AND (e.is_enabled IS NULL OR e.is_enabled = 1)
|
||||
`, tbl)
|
||||
|
||||
args := []interface{}{
|
||||
queryBlob,
|
||||
params.TopK, // 这里就是 k
|
||||
}
|
||||
|
||||
// 追加过滤条件
|
||||
for _, wp := range buildFilterWhere(params) {
|
||||
vecSQL += " AND " + wp.clause
|
||||
args = append(args, wp.args...)
|
||||
}
|
||||
|
||||
// ⚠️ 这里仍然建议加 ORDER BY,虽然 vec0 已经按距离返回
|
||||
vecSQL += " ORDER BY v.distance ASC"
|
||||
|
||||
type row struct {
|
||||
Rowid uint
|
||||
Distance float64
|
||||
SourceID string
|
||||
SourceType int
|
||||
ChunkID string
|
||||
KnowledgeID string
|
||||
KnowledgeBaseID string
|
||||
TagID string
|
||||
Content string
|
||||
}
|
||||
|
||||
var rows []row
|
||||
if err := r.db.WithContext(ctx).
|
||||
Raw(vecSQL, args...).
|
||||
Scan(&rows).Error; err != nil {
|
||||
return nil, fmt.Errorf("sqlite-vec query failed: %w", err)
|
||||
}
|
||||
|
||||
logger.GetLogger(ctx).Infof("[SQLite] vectorRetrieve: query_dim=%d, threshold=%.4f, matched=%d rows", dim, params.Threshold, len(rows))
|
||||
|
||||
items := make([]*types.IndexWithScore, 0, len(rows))
|
||||
|
||||
for i, v := range rows {
|
||||
// cosine distance = 1 - cosine_similarity
|
||||
score := 1 - v.Distance
|
||||
|
||||
logger.GetLogger(ctx).Infof("[SQLite] vectorRetrieve: #%d chunk_id=%s, distance=%.4f, score=%.4f, content_preview=%.60s",
|
||||
i+1, v.ChunkID, v.Distance, score, v.Content)
|
||||
|
||||
items = append(items, &types.IndexWithScore{
|
||||
ID: fmt.Sprintf("%d", v.Rowid),
|
||||
SourceID: v.SourceID,
|
||||
SourceType: types.SourceType(v.SourceType),
|
||||
ChunkID: v.ChunkID,
|
||||
KnowledgeID: v.KnowledgeID,
|
||||
KnowledgeBaseID: v.KnowledgeBaseID,
|
||||
TagID: v.TagID,
|
||||
Content: v.Content,
|
||||
Score: score,
|
||||
MatchType: types.MatchTypeEmbedding,
|
||||
})
|
||||
}
|
||||
|
||||
return []*types.RetrieveResult{{
|
||||
Results: items,
|
||||
RetrieverEngineType: types.SQLiteRetrieverEngineType,
|
||||
RetrieverType: types.VectorRetrieverType,
|
||||
}}, nil
|
||||
}
|
||||
|
||||
// --- Internal helpers ---
|
||||
|
||||
func toSQLiteEmbedding(info *types.IndexInfo) *sqliteEmbedding {
|
||||
enabled := true
|
||||
return &sqliteEmbedding{
|
||||
SourceID: info.SourceID,
|
||||
SourceType: int(info.SourceType),
|
||||
ChunkID: info.ChunkID,
|
||||
KnowledgeID: info.KnowledgeID,
|
||||
KnowledgeBaseID: info.KnowledgeBaseID,
|
||||
TagID: info.TagID,
|
||||
Content: common.CleanInvalidUTF8(info.Content),
|
||||
Dimension: 0,
|
||||
IsEnabled: &enabled,
|
||||
}
|
||||
}
|
||||
|
||||
func extractEmbedding(params map[string]any, sourceID string) []float32 {
|
||||
if params == nil {
|
||||
return nil
|
||||
}
|
||||
embMap, ok := params["embedding"].(map[string][]float32)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return embMap[sourceID]
|
||||
}
|
||||
|
||||
func (r *sqliteRepository) insertVec(_ context.Context, rowID uint, dim int, emb []float32) {
|
||||
r.ensureVecTable(dim)
|
||||
blob, err := sqlite_vec.SerializeFloat32(emb)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
sql := fmt.Sprintf("INSERT INTO %s(rowid, embedding) VALUES (?, ?)", vecTableName(dim))
|
||||
r.db.Exec(sql, rowID, blob)
|
||||
}
|
||||
|
||||
func (r *sqliteRepository) deleteRowsAndVecs(_ context.Context, rows []sqliteEmbedding) {
|
||||
dimIDs := make(map[int][]uint)
|
||||
for _, row := range rows {
|
||||
if row.Dimension > 0 {
|
||||
dimIDs[row.Dimension] = append(dimIDs[row.Dimension], row.ID)
|
||||
}
|
||||
}
|
||||
for dim, ids := range dimIDs {
|
||||
if !r.vecTables[dim] {
|
||||
continue
|
||||
}
|
||||
tbl := vecTableName(dim)
|
||||
for _, id := range ids {
|
||||
r.db.Exec(fmt.Sprintf("DELETE FROM %s WHERE rowid = ?", tbl), id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (r *sqliteRepository) copyVec(_ context.Context, srcID, dstID uint, dim int) {
|
||||
if !r.vecTables[dim] {
|
||||
return
|
||||
}
|
||||
tbl := vecTableName(dim)
|
||||
r.db.Exec(fmt.Sprintf(
|
||||
"INSERT INTO %s(rowid, embedding) SELECT ?, embedding FROM %s WHERE rowid = ?",
|
||||
tbl, tbl,
|
||||
), dstID, srcID)
|
||||
}
|
||||
|
||||
func (r *sqliteRepository) syncFTS5Insert(_ context.Context, row *sqliteEmbedding) {
|
||||
if row.ID == 0 {
|
||||
return
|
||||
}
|
||||
sql := `INSERT INTO lite_embeddings_fts(rowid, content, source_id, chunk_id, knowledge_id, knowledge_base_id) VALUES(?, ?, ?, ?, ?, ?)`
|
||||
r.db.Exec(sql, row.ID, row.Content, row.SourceID, row.ChunkID, row.KnowledgeID, row.KnowledgeBaseID)
|
||||
}
|
||||
|
||||
type whereClause struct {
|
||||
clause string
|
||||
args []interface{}
|
||||
}
|
||||
|
||||
func buildFilterWhere(params types.RetrieveParams) []whereClause {
|
||||
var parts []whereClause
|
||||
if len(params.KnowledgeBaseIDs) > 0 {
|
||||
parts = append(parts, whereClause{
|
||||
clause: "e.knowledge_base_id IN (" + placeholders(len(params.KnowledgeBaseIDs)) + ")",
|
||||
args: toInterfaceSlice(params.KnowledgeBaseIDs),
|
||||
})
|
||||
}
|
||||
if len(params.KnowledgeIDs) > 0 {
|
||||
parts = append(parts, whereClause{
|
||||
clause: "e.knowledge_id IN (" + placeholders(len(params.KnowledgeIDs)) + ")",
|
||||
args: toInterfaceSlice(params.KnowledgeIDs),
|
||||
})
|
||||
}
|
||||
if len(params.TagIDs) > 0 {
|
||||
parts = append(parts, whereClause{
|
||||
clause: "e.tag_id IN (" + placeholders(len(params.TagIDs)) + ")",
|
||||
args: toInterfaceSlice(params.TagIDs),
|
||||
})
|
||||
}
|
||||
return parts
|
||||
}
|
||||
|
||||
func placeholders(n int) string {
|
||||
p := make([]string, n)
|
||||
for i := range p {
|
||||
p[i] = "?"
|
||||
}
|
||||
return strings.Join(p, ",")
|
||||
}
|
||||
|
||||
func toInterfaceSlice(ss []string) []interface{} {
|
||||
out := make([]interface{}, len(ss))
|
||||
for i, s := range ss {
|
||||
out[i] = s
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// sanitizeFTS5Query builds an FTS5 query from user input.
|
||||
// CJK characters are split into overlapping bigrams (mimicking CJK analyzers used
|
||||
// by Elasticsearch etc.) and joined with OR for broad partial matching.
|
||||
// Non-CJK words are kept intact. BM25 ranking naturally boosts documents
|
||||
// that match more tokens.
|
||||
func sanitizeFTS5Query(q string) string {
|
||||
q = strings.TrimSpace(q)
|
||||
if q == "" {
|
||||
return q
|
||||
}
|
||||
|
||||
var cjkRunes []rune
|
||||
var nonCJKWords []string
|
||||
var buf strings.Builder
|
||||
|
||||
flushNonCJK := func() {
|
||||
if buf.Len() > 0 {
|
||||
nonCJKWords = append(nonCJKWords, buf.String())
|
||||
buf.Reset()
|
||||
}
|
||||
}
|
||||
|
||||
for _, r := range q {
|
||||
if unicode.Is(unicode.Han, r) {
|
||||
flushNonCJK()
|
||||
cjkRunes = append(cjkRunes, r)
|
||||
} else if unicode.IsSpace(r) || r == '|' {
|
||||
flushNonCJK()
|
||||
} else if r == '"' || r == '*' || r == '(' || r == ')' || r == '{' || r == '}' {
|
||||
// skip FTS5 special characters
|
||||
} else {
|
||||
buf.WriteRune(r)
|
||||
}
|
||||
}
|
||||
flushNonCJK()
|
||||
|
||||
var parts []string
|
||||
|
||||
// CJK bigrams for better matching (e.g. "苹果第四季度" → "苹果" OR "果第" OR "第四" OR ...)
|
||||
if len(cjkRunes) == 1 {
|
||||
parts = append(parts, `"`+string(cjkRunes[0])+`"`)
|
||||
} else {
|
||||
for i := 0; i < len(cjkRunes)-1; i++ {
|
||||
bigram := string(cjkRunes[i]) + string(cjkRunes[i+1])
|
||||
parts = append(parts, `"`+bigram+`"`)
|
||||
}
|
||||
}
|
||||
|
||||
for _, w := range nonCJKWords {
|
||||
w = strings.ReplaceAll(w, `"`, `""`)
|
||||
parts = append(parts, `"`+w+`"`)
|
||||
}
|
||||
|
||||
if len(parts) == 0 {
|
||||
return `"` + strings.ReplaceAll(q, `"`, `""`) + `"`
|
||||
}
|
||||
return strings.Join(parts, " OR ")
|
||||
}
|
||||
@@ -114,7 +114,7 @@ func (r *tenantRepository) AdjustStorageUsed(ctx context.Context, tenantID uint6
|
||||
tenant.StorageUsed += delta
|
||||
// 保存更新并验证业务规则
|
||||
if tenant.StorageUsed < 0 {
|
||||
logger.Error(ctx, "tenant storage used is negative %s: %d", tenant.ID, tenant.StorageUsed)
|
||||
logger.Errorf(ctx, "tenant storage used is negative %d: %d", tenant.ID, tenant.StorageUsed)
|
||||
tenant.StorageUsed = 0
|
||||
}
|
||||
|
||||
|
||||
@@ -183,6 +183,9 @@ func (p *PluginSearch) OnEvent(ctx context.Context,
|
||||
return
|
||||
}
|
||||
if len(res) > 0 {
|
||||
for _, r := range res {
|
||||
r.KnowledgeBaseID = t.KnowledgeBaseID
|
||||
}
|
||||
pipelineInfo(ctx, "Search", "expansion_hits", map[string]interface{}{
|
||||
"kb_id": t.KnowledgeBaseID,
|
||||
"query": q,
|
||||
@@ -338,6 +341,9 @@ func (p *PluginSearch) searchByTargets(
|
||||
directResults, skippedIDs := p.tryDirectChunkLoading(ctx, chatManage.TenantID, t.KnowledgeIDs)
|
||||
|
||||
if len(directResults) > 0 {
|
||||
for _, r := range directResults {
|
||||
r.KnowledgeBaseID = t.KnowledgeBaseID
|
||||
}
|
||||
pipelineInfo(ctx, "Search", "direct_load", map[string]interface{}{
|
||||
"kb_id": t.KnowledgeBaseID,
|
||||
"loaded_count": len(directResults),
|
||||
@@ -383,6 +389,9 @@ func (p *PluginSearch) searchByTargets(
|
||||
})
|
||||
return
|
||||
}
|
||||
for _, r := range res {
|
||||
r.KnowledgeBaseID = t.KnowledgeBaseID
|
||||
}
|
||||
pipelineInfo(ctx, "Search", "kb_result", map[string]interface{}{
|
||||
"kb_id": t.KnowledgeBaseID,
|
||||
"target_type": t.Type,
|
||||
|
||||
@@ -222,5 +222,6 @@ func chunk2SearchResult(chunk *types.Chunk, knowledge *types.Knowledge) *types.S
|
||||
KnowledgeFilename: knowledge.FileName,
|
||||
KnowledgeSource: knowledge.Source,
|
||||
ChunkMetadata: chunk.Metadata,
|
||||
KnowledgeBaseID: knowledge.KnowledgeBaseID,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -79,7 +79,7 @@ const (
|
||||
// NewChunkExtractTask creates a new chunk extract task
|
||||
func NewChunkExtractTask(
|
||||
ctx context.Context,
|
||||
client *asynq.Client,
|
||||
client interfaces.TaskEnqueuer,
|
||||
tenantID uint64,
|
||||
chunkID string,
|
||||
modelID string,
|
||||
@@ -109,7 +109,7 @@ func NewChunkExtractTask(
|
||||
// NewTableExtractTask creates a new table extract task
|
||||
func NewDataTableSummaryTask(
|
||||
ctx context.Context,
|
||||
client *asynq.Client,
|
||||
client interfaces.TaskEnqueuer,
|
||||
tenantID uint64,
|
||||
knowledgeID string,
|
||||
summaryModel string,
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
"github.com/Tencent/WeKnora/internal/utils"
|
||||
"github.com/google/uuid"
|
||||
@@ -23,11 +24,28 @@ type cosFileService struct {
|
||||
client *cos.Client
|
||||
bucketURL string
|
||||
cosPathPrefix string
|
||||
// 临时桶配置(用于存放自动过期的临时文件)
|
||||
tempClient *cos.Client
|
||||
tempBucketURL string
|
||||
}
|
||||
|
||||
// newCosClient creates a bare cosFileService with just the SDK client initialised.
|
||||
// Shared by NewCosFileService* constructors and CheckCosConnectivity.
|
||||
func newCosClient(bucketName, region, secretID, secretKey string) (*cosFileService, error) {
|
||||
bucketURL := fmt.Sprintf("https://%s.cos.%s.myqcloud.com/", bucketName, region)
|
||||
u, err := url.Parse(bucketURL)
|
||||
logger.Infof(context.Background(), "newCosClient: bucketURL: %s", bucketURL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse bucketURL: %w", err)
|
||||
}
|
||||
client := cos.NewClient(&cos.BaseURL{BucketURL: u}, &http.Client{
|
||||
Transport: &cos.AuthorizationTransport{
|
||||
SecretID: secretID,
|
||||
SecretKey: secretKey,
|
||||
},
|
||||
})
|
||||
return &cosFileService{client: client, bucketURL: bucketURL}, nil
|
||||
}
|
||||
|
||||
// NewCosFileService creates a new COS file service instance
|
||||
func NewCosFileService(bucketName, region, secretId, secretKey, cosPathPrefix string) (interfaces.FileService, error) {
|
||||
return NewCosFileServiceWithTempBucket(bucketName, region, secretId, secretKey, cosPathPrefix, "", "")
|
||||
@@ -35,49 +53,51 @@ func NewCosFileService(bucketName, region, secretId, secretKey, cosPathPrefix st
|
||||
|
||||
// NewCosFileServiceWithTempBucket creates a new COS file service instance with optional temp bucket
|
||||
func NewCosFileServiceWithTempBucket(bucketName, region, secretId, secretKey, cosPathPrefix, tempBucketName, tempRegion string) (interfaces.FileService, error) {
|
||||
bucketURL := fmt.Sprintf("https://%s.cos.%s.tencentcos.cn/", bucketName, region)
|
||||
u, err := url.Parse(bucketURL)
|
||||
svc, err := newCosClient(bucketName, region, secretId, secretKey)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse bucketURL: %w", err)
|
||||
return nil, err
|
||||
}
|
||||
b := &cos.BaseURL{BucketURL: u}
|
||||
client := cos.NewClient(b, &http.Client{
|
||||
Transport: &cos.AuthorizationTransport{
|
||||
SecretID: secretId,
|
||||
SecretKey: secretKey,
|
||||
},
|
||||
})
|
||||
svc.cosPathPrefix = cosPathPrefix
|
||||
|
||||
svc := &cosFileService{
|
||||
client: client,
|
||||
bucketURL: bucketURL,
|
||||
cosPathPrefix: cosPathPrefix,
|
||||
}
|
||||
|
||||
// 如果配置了临时桶,初始化临时桶客户端
|
||||
if tempBucketName != "" {
|
||||
if tempRegion == "" {
|
||||
tempRegion = region // 默认使用相同的 region
|
||||
tempRegion = region
|
||||
}
|
||||
tempBucketURL := fmt.Sprintf("https://%s.cos.%s.tencentcos.cn/", tempBucketName, tempRegion)
|
||||
tempBucketURL := fmt.Sprintf("https://%s.cos.%s.myqcloud.com/", tempBucketName, tempRegion)
|
||||
tempU, err := url.Parse(tempBucketURL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse temp bucketURL: %w", err)
|
||||
}
|
||||
tempB := &cos.BaseURL{BucketURL: tempU}
|
||||
tempClient := cos.NewClient(tempB, &http.Client{
|
||||
svc.tempClient = cos.NewClient(&cos.BaseURL{BucketURL: tempU}, &http.Client{
|
||||
Transport: &cos.AuthorizationTransport{
|
||||
SecretID: secretId,
|
||||
SecretKey: secretKey,
|
||||
},
|
||||
})
|
||||
svc.tempClient = tempClient
|
||||
svc.tempBucketURL = tempBucketURL
|
||||
}
|
||||
|
||||
return svc, nil
|
||||
}
|
||||
|
||||
// CheckConnectivity verifies COS is reachable by performing a HEAD request on the bucket.
|
||||
func (s *cosFileService) CheckConnectivity(ctx context.Context) error {
|
||||
checkCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
_, err := s.client.Bucket.Head(checkCtx)
|
||||
return err
|
||||
}
|
||||
|
||||
// CheckCosConnectivity tests COS connectivity using the provided credentials.
|
||||
// It creates a temporary service instance internally and delegates to CheckConnectivity.
|
||||
func CheckCosConnectivity(ctx context.Context, bucketName, region, secretID, secretKey string) error {
|
||||
svc, err := newCosClient(bucketName, region, secretID, secretKey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return svc.CheckConnectivity(ctx)
|
||||
}
|
||||
|
||||
// SaveFile saves a file to COS storage
|
||||
// It generates a unique name for the file and organizes it by tenant and knowledge ID
|
||||
func (s *cosFileService) SaveFile(ctx context.Context,
|
||||
|
||||
@@ -14,6 +14,11 @@ import (
|
||||
// used for testing or when file storage is not required
|
||||
type DummyFileService struct{}
|
||||
|
||||
// CheckConnectivity always succeeds for the dummy service.
|
||||
func (s *DummyFileService) CheckConnectivity(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// NewDummyFileService creates a new instance of DummyFileService
|
||||
func NewDummyFileService() interfaces.FileService {
|
||||
return &DummyFileService{}
|
||||
|
||||
@@ -19,6 +19,18 @@ type localFileService struct {
|
||||
baseDir string // Base directory for file storage
|
||||
}
|
||||
|
||||
// CheckConnectivity verifies the local storage directory exists and is accessible.
|
||||
func (s *localFileService) CheckConnectivity(ctx context.Context) error {
|
||||
info, err := os.Stat(s.baseDir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage directory not accessible: %w", err)
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return fmt.Errorf("storage path is not a directory: %s", s.baseDir)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// NewLocalFileService creates a new local file service instance
|
||||
func NewLocalFileService(baseDir string) interfaces.FileService {
|
||||
return &localFileService{
|
||||
|
||||
@@ -22,11 +22,10 @@ type minioFileService struct {
|
||||
bucketName string
|
||||
}
|
||||
|
||||
// NewMinioFileService creates a MinIO file service
|
||||
func NewMinioFileService(endpoint,
|
||||
accessKeyID, secretAccessKey, bucketName string, useSSL bool,
|
||||
) (interfaces.FileService, error) {
|
||||
// Initialize MinIO client
|
||||
// newMinioClient creates a bare minioFileService with just the SDK client initialised.
|
||||
// Shared by NewMinioFileService (which also ensures the bucket exists) and
|
||||
// CheckMinioConnectivity (read-only probe).
|
||||
func newMinioClient(endpoint, accessKeyID, secretAccessKey, bucketName string, useSSL bool) (*minioFileService, error) {
|
||||
client, err := minio.New(endpoint, &minio.Options{
|
||||
Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""),
|
||||
Secure: useSSL,
|
||||
@@ -34,24 +33,60 @@ func NewMinioFileService(endpoint,
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to initialize MinIO client: %w", err)
|
||||
}
|
||||
return &minioFileService{client: client, bucketName: bucketName}, nil
|
||||
}
|
||||
|
||||
// Check if bucket exists, create if not
|
||||
exists, err := client.BucketExists(context.Background(), bucketName)
|
||||
// NewMinioFileService creates a MinIO file service.
|
||||
// It verifies that the bucket exists and creates it if missing.
|
||||
func NewMinioFileService(endpoint,
|
||||
accessKeyID, secretAccessKey, bucketName string, useSSL bool,
|
||||
) (interfaces.FileService, error) {
|
||||
svc, err := newMinioClient(endpoint, accessKeyID, secretAccessKey, bucketName, useSSL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
exists, err := svc.client.BucketExists(context.Background(), bucketName)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to check bucket: %w", err)
|
||||
}
|
||||
|
||||
if !exists {
|
||||
err = client.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{})
|
||||
if err != nil {
|
||||
if err = svc.client.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{}); err != nil {
|
||||
return nil, fmt.Errorf("failed to create bucket: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return &minioFileService{
|
||||
client: client,
|
||||
bucketName: bucketName,
|
||||
}, nil
|
||||
return svc, nil
|
||||
}
|
||||
|
||||
// CheckConnectivity verifies MinIO is reachable and, if a bucket is configured,
|
||||
// that the bucket exists. This is a read-only probe — it never creates a bucket.
|
||||
func (s *minioFileService) CheckConnectivity(ctx context.Context) error {
|
||||
checkCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
if s.bucketName != "" {
|
||||
exists, err := s.client.BucketExists(checkCtx, s.bucketName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !exists {
|
||||
return fmt.Errorf("bucket %q does not exist", s.bucketName)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
_, err := s.client.ListBuckets(checkCtx)
|
||||
return err
|
||||
}
|
||||
|
||||
// CheckMinioConnectivity tests MinIO connectivity using the provided credentials.
|
||||
// It creates a temporary service instance internally and delegates to CheckConnectivity.
|
||||
func CheckMinioConnectivity(ctx context.Context, endpoint, accessKeyID, secretAccessKey, bucketName string, useSSL bool) error {
|
||||
svc, err := newMinioClient(endpoint, accessKeyID, secretAccessKey, bucketName, useSSL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return svc.CheckConnectivity(ctx)
|
||||
}
|
||||
|
||||
// SaveFile saves a file to MinIO
|
||||
|
||||
@@ -0,0 +1,365 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/application/service/retriever"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/models/utils/ollama"
|
||||
"github.com/Tencent/WeKnora/internal/models/vlm"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
"github.com/google/uuid"
|
||||
"github.com/hibiken/asynq"
|
||||
)
|
||||
|
||||
const (
|
||||
vlmOCRPrompt = "请提取这张文档图片中的所有正文内容,用纯 Markdown 格式输出。要求:\n" +
|
||||
"1. 忽略页眉、页脚\n" +
|
||||
"2. 表格使用 Markdown 表格语法\n" +
|
||||
"3. 公式使用 LaTeX 格式(用 $ 或 $$ 包裹)\n" +
|
||||
"4. 按照原文阅读顺序组织\n" +
|
||||
"5. 只输出提取到的文本内容,不要添加任何 HTML 标签\n" +
|
||||
"如果图片中没有可识别的文字内容,请回复:无文字内容。"
|
||||
vlmCaptionPrompt = "简单凝炼的描述图片的主要内容"
|
||||
)
|
||||
|
||||
// ImageMultimodalService handles image:multimodal asynq tasks.
|
||||
// It reads images from local storage, performs OCR and VLM caption,
|
||||
// and creates child chunks for the processed content.
|
||||
type ImageMultimodalService struct {
|
||||
chunkService interfaces.ChunkService
|
||||
modelService interfaces.ModelService
|
||||
kbService interfaces.KnowledgeBaseService
|
||||
knowledgeRepo interfaces.KnowledgeRepository
|
||||
tenantRepo interfaces.TenantRepository
|
||||
retrieveEngine interfaces.RetrieveEngineRegistry
|
||||
ollamaService *ollama.OllamaService
|
||||
}
|
||||
|
||||
func NewImageMultimodalService(
|
||||
chunkService interfaces.ChunkService,
|
||||
modelService interfaces.ModelService,
|
||||
kbService interfaces.KnowledgeBaseService,
|
||||
knowledgeRepo interfaces.KnowledgeRepository,
|
||||
tenantRepo interfaces.TenantRepository,
|
||||
retrieveEngine interfaces.RetrieveEngineRegistry,
|
||||
ollamaService *ollama.OllamaService,
|
||||
) interfaces.TaskHandler {
|
||||
return &ImageMultimodalService{
|
||||
chunkService: chunkService,
|
||||
modelService: modelService,
|
||||
kbService: kbService,
|
||||
knowledgeRepo: knowledgeRepo,
|
||||
tenantRepo: tenantRepo,
|
||||
retrieveEngine: retrieveEngine,
|
||||
ollamaService: ollamaService,
|
||||
}
|
||||
}
|
||||
|
||||
// Handle implements asynq handler for TypeImageMultimodal.
|
||||
func (s *ImageMultimodalService) Handle(ctx context.Context, task *asynq.Task) error {
|
||||
var payload types.ImageMultimodalPayload
|
||||
if err := json.Unmarshal(task.Payload(), &payload); err != nil {
|
||||
return fmt.Errorf("unmarshal image multimodal payload: %w", err)
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "[ImageMultimodal] Processing image: chunk=%s, url=%s, local_path=%s, ocr=%v, caption=%v",
|
||||
payload.ChunkID, payload.ImageURL, payload.ImageLocalPath, payload.EnableOCR, payload.EnableCaption)
|
||||
|
||||
ctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID)
|
||||
|
||||
vlmModel, err := s.resolveVLM(ctx, payload.KnowledgeBaseID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("resolve VLM: %w", err)
|
||||
}
|
||||
|
||||
imgBytes, err := os.ReadFile(payload.ImageLocalPath)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "[ImageMultimodal] Local file %s not available (%v), trying to download from URL", payload.ImageLocalPath, err)
|
||||
imgBytes, err = downloadImageFromURL(payload.ImageURL)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "[ImageMultimodal] Failed to download image from URL %s: %v", payload.ImageURL, err)
|
||||
return fmt.Errorf("read image: local file %s not found and URL download failed: %w", payload.ImageLocalPath, err)
|
||||
}
|
||||
logger.Infof(ctx, "[ImageMultimodal] Image downloaded from URL, len=%d", len(imgBytes))
|
||||
} else {
|
||||
logger.Infof(ctx, "[ImageMultimodal] Image bytes read from %s, len=%d", payload.ImageLocalPath, len(imgBytes))
|
||||
defer os.Remove(payload.ImageLocalPath)
|
||||
}
|
||||
|
||||
imageInfo := types.ImageInfo{
|
||||
URL: payload.ImageURL,
|
||||
OriginalURL: payload.ImageURL,
|
||||
}
|
||||
|
||||
if payload.EnableOCR {
|
||||
ocrText, ocrErr := vlmModel.Predict(ctx, imgBytes, vlmOCRPrompt)
|
||||
if ocrErr != nil {
|
||||
logger.Warnf(ctx, "[ImageMultimodal] OCR failed for %s: %v", payload.ImageURL, ocrErr)
|
||||
} else {
|
||||
ocrText = sanitizeOCRText(ocrText)
|
||||
if ocrText != "" {
|
||||
imageInfo.OCRText = ocrText
|
||||
} else {
|
||||
logger.Warnf(ctx, "[ImageMultimodal] OCR returned empty/invalid content for %s, discarded", payload.ImageURL)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if payload.EnableCaption {
|
||||
caption, capErr := vlmModel.Predict(ctx, imgBytes, vlmCaptionPrompt)
|
||||
if capErr != nil {
|
||||
logger.Warnf(ctx, "[ImageMultimodal] Caption failed for %s: %v", payload.ImageURL, capErr)
|
||||
} else if caption != "" {
|
||||
imageInfo.Caption = caption
|
||||
}
|
||||
}
|
||||
|
||||
// Build child chunks for OCR and caption results
|
||||
imageInfoJSON, _ := json.Marshal([]types.ImageInfo{imageInfo})
|
||||
var newChunks []*types.Chunk
|
||||
|
||||
if imageInfo.OCRText != "" {
|
||||
newChunks = append(newChunks, &types.Chunk{
|
||||
ID: uuid.New().String(),
|
||||
TenantID: payload.TenantID,
|
||||
KnowledgeID: payload.KnowledgeID,
|
||||
KnowledgeBaseID: payload.KnowledgeBaseID,
|
||||
Content: imageInfo.OCRText,
|
||||
ChunkType: types.ChunkTypeImageOCR,
|
||||
ParentChunkID: payload.ChunkID,
|
||||
IsEnabled: true,
|
||||
Flags: types.ChunkFlagRecommended,
|
||||
ImageInfo: string(imageInfoJSON),
|
||||
CreatedAt: time.Now(),
|
||||
UpdatedAt: time.Now(),
|
||||
})
|
||||
}
|
||||
|
||||
if imageInfo.Caption != "" {
|
||||
newChunks = append(newChunks, &types.Chunk{
|
||||
ID: uuid.New().String(),
|
||||
TenantID: payload.TenantID,
|
||||
KnowledgeID: payload.KnowledgeID,
|
||||
KnowledgeBaseID: payload.KnowledgeBaseID,
|
||||
Content: imageInfo.Caption,
|
||||
ChunkType: types.ChunkTypeImageCaption,
|
||||
ParentChunkID: payload.ChunkID,
|
||||
IsEnabled: true,
|
||||
Flags: types.ChunkFlagRecommended,
|
||||
ImageInfo: string(imageInfoJSON),
|
||||
CreatedAt: time.Now(),
|
||||
UpdatedAt: time.Now(),
|
||||
})
|
||||
}
|
||||
|
||||
if len(newChunks) == 0 {
|
||||
// Even if OCR/caption both failed, mark knowledge as completed
|
||||
s.finalizeImageKnowledge(ctx, payload, "")
|
||||
return nil
|
||||
}
|
||||
|
||||
// Persist chunks
|
||||
if err := s.chunkService.CreateChunks(ctx, newChunks); err != nil {
|
||||
return fmt.Errorf("create multimodal chunks: %w", err)
|
||||
}
|
||||
for _, c := range newChunks {
|
||||
logger.Infof(ctx, "[ImageMultimodal] Created %s chunk %s for image %s, len=%d",
|
||||
c.ChunkType, c.ID, payload.ImageURL, len(c.Content))
|
||||
}
|
||||
|
||||
// Index chunks so they can be retrieved
|
||||
s.indexChunks(ctx, payload, newChunks)
|
||||
|
||||
// Update the parent text chunk's ImageInfo (mirrors old docreader behaviour)
|
||||
s.updateParentChunkImageInfo(ctx, payload, imageInfo)
|
||||
|
||||
// For standalone image files, use caption as the knowledge description
|
||||
// and mark the knowledge as completed (it was kept in "processing" until now).
|
||||
s.finalizeImageKnowledge(ctx, payload, imageInfo.Caption)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// finalizeImageKnowledge updates the knowledge after multimodal processing:
|
||||
// - For standalone image files: sets Description from caption and marks ParseStatus as completed
|
||||
// (processChunks kept it in "processing" to wait for multimodal results).
|
||||
// - For images extracted from PDFs: no-op (description comes from summary generation).
|
||||
func (s *ImageMultimodalService) finalizeImageKnowledge(ctx context.Context, payload types.ImageMultimodalPayload, caption string) {
|
||||
knowledge, err := s.knowledgeRepo.GetKnowledgeByIDOnly(ctx, payload.KnowledgeID)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "[ImageMultimodal] Failed to get knowledge %s: %v", payload.KnowledgeID, err)
|
||||
return
|
||||
}
|
||||
if knowledge == nil {
|
||||
return
|
||||
}
|
||||
if !IsImageType(knowledge.FileType) {
|
||||
return
|
||||
}
|
||||
|
||||
if caption != "" {
|
||||
knowledge.Description = caption
|
||||
}
|
||||
knowledge.ParseStatus = types.ParseStatusCompleted
|
||||
knowledge.UpdatedAt = time.Now()
|
||||
if err := s.knowledgeRepo.UpdateKnowledge(ctx, knowledge); err != nil {
|
||||
logger.Warnf(ctx, "[ImageMultimodal] Failed to finalize knowledge: %v", err)
|
||||
} else {
|
||||
logger.Infof(ctx, "[ImageMultimodal] Finalized image knowledge %s (status=completed, description=%d chars)",
|
||||
payload.KnowledgeID, len(knowledge.Description))
|
||||
}
|
||||
}
|
||||
|
||||
// indexChunks indexes the newly created multimodal chunks into the retrieval engine
|
||||
// so they can participate in semantic search.
|
||||
func (s *ImageMultimodalService) indexChunks(ctx context.Context, payload types.ImageMultimodalPayload, chunks []*types.Chunk) {
|
||||
kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID)
|
||||
if err != nil || kb == nil {
|
||||
logger.Warnf(ctx, "[ImageMultimodal] Failed to get KB for indexing: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, kb.EmbeddingModelID)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "[ImageMultimodal] Failed to get embedding model for indexing: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
tenantInfo, err := s.tenantRepo.GetTenantByID(ctx, payload.TenantID)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "[ImageMultimodal] Failed to get tenant for indexing: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
engine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines())
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "[ImageMultimodal] Failed to init retrieve engine: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
indexInfoList := make([]*types.IndexInfo, 0, len(chunks))
|
||||
for _, chunk := range chunks {
|
||||
indexInfoList = append(indexInfoList, &types.IndexInfo{
|
||||
Content: chunk.Content,
|
||||
SourceID: chunk.ID,
|
||||
SourceType: types.ChunkSourceType,
|
||||
ChunkID: chunk.ID,
|
||||
KnowledgeID: chunk.KnowledgeID,
|
||||
KnowledgeBaseID: chunk.KnowledgeBaseID,
|
||||
})
|
||||
}
|
||||
|
||||
if err := engine.BatchIndex(ctx, embeddingModel, indexInfoList); err != nil {
|
||||
logger.Errorf(ctx, "[ImageMultimodal] Failed to index multimodal chunks: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
// Mark chunks as indexed.
|
||||
// Must re-fetch from DB because the in-memory objects lack auto-generated fields
|
||||
// (e.g. seq_id), and GORM Save would overwrite them with zero values.
|
||||
for _, chunk := range chunks {
|
||||
dbChunk, err := s.chunkService.GetChunkByIDOnly(ctx, chunk.ID)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "[ImageMultimodal] Failed to fetch chunk %s for status update: %v", chunk.ID, err)
|
||||
continue
|
||||
}
|
||||
dbChunk.Status = int(types.ChunkStatusIndexed)
|
||||
if err := s.chunkService.UpdateChunk(ctx, dbChunk); err != nil {
|
||||
logger.Warnf(ctx, "[ImageMultimodal] Failed to update chunk %s status to indexed: %v", chunk.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "[ImageMultimodal] Indexed %d multimodal chunks for image %s", len(chunks), payload.ImageURL)
|
||||
}
|
||||
|
||||
// updateParentChunkImageInfo updates the parent text chunk's ImageInfo field,
|
||||
// replicating the behaviour of the old docreader flow where the parent chunk
|
||||
// carried the full image metadata (URL, OCR, caption).
|
||||
func (s *ImageMultimodalService) updateParentChunkImageInfo(ctx context.Context, payload types.ImageMultimodalPayload, imageInfo types.ImageInfo) {
|
||||
if payload.ChunkID == "" {
|
||||
return
|
||||
}
|
||||
|
||||
chunk, err := s.chunkService.GetChunkByIDOnly(ctx, payload.ChunkID)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "[ImageMultimodal] Failed to get parent chunk %s: %v", payload.ChunkID, err)
|
||||
return
|
||||
}
|
||||
|
||||
var existingInfos []types.ImageInfo
|
||||
if chunk.ImageInfo != "" {
|
||||
_ = json.Unmarshal([]byte(chunk.ImageInfo), &existingInfos)
|
||||
}
|
||||
|
||||
found := false
|
||||
for i, info := range existingInfos {
|
||||
if info.URL == imageInfo.URL {
|
||||
existingInfos[i] = imageInfo
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
existingInfos = append(existingInfos, imageInfo)
|
||||
}
|
||||
|
||||
imageInfoJSON, _ := json.Marshal(existingInfos)
|
||||
chunk.ImageInfo = string(imageInfoJSON)
|
||||
chunk.UpdatedAt = time.Now()
|
||||
if err := s.chunkService.UpdateChunk(ctx, chunk); err != nil {
|
||||
logger.Warnf(ctx, "[ImageMultimodal] Failed to update parent chunk %s ImageInfo: %v", chunk.ID, err)
|
||||
} else {
|
||||
logger.Infof(ctx, "[ImageMultimodal] Updated parent chunk %s ImageInfo for image %s", chunk.ID, payload.ImageURL)
|
||||
}
|
||||
}
|
||||
|
||||
// resolveVLM creates a vlm.VLM instance for the given knowledge base,
|
||||
// supporting both new-style (ModelID) and legacy (inline BaseURL) configs.
|
||||
func (s *ImageMultimodalService) resolveVLM(ctx context.Context, kbID string) (vlm.VLM, error) {
|
||||
kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, kbID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get knowledge base %s: %w", kbID, err)
|
||||
}
|
||||
if kb == nil {
|
||||
return nil, fmt.Errorf("knowledge base %s not found", kbID)
|
||||
}
|
||||
|
||||
vlmCfg := kb.VLMConfig
|
||||
if !vlmCfg.IsEnabled() {
|
||||
return nil, fmt.Errorf("VLM is not enabled for knowledge base %s", kbID)
|
||||
}
|
||||
|
||||
// New-style: resolve model through ModelService
|
||||
if vlmCfg.ModelID != "" {
|
||||
return s.modelService.GetVLMModel(ctx, vlmCfg.ModelID)
|
||||
}
|
||||
|
||||
// Legacy: create VLM from inline config
|
||||
return vlm.NewVLMFromLegacyConfig(vlmCfg, s.ollamaService)
|
||||
}
|
||||
|
||||
// downloadImageFromURL downloads image bytes from an HTTP(S) URL.
|
||||
func downloadImageFromURL(imageURL string) ([]byte, error) {
|
||||
if !strings.HasPrefix(imageURL, "http://") && !strings.HasPrefix(imageURL, "https://") {
|
||||
return nil, fmt.Errorf("unsupported URL scheme: %s", imageURL)
|
||||
}
|
||||
client := &http.Client{Timeout: 60 * time.Second}
|
||||
resp, err := client.Get(imageURL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("HTTP GET: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("HTTP %d for %s", resp.StatusCode, imageURL)
|
||||
}
|
||||
return io.ReadAll(resp.Body)
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -32,7 +32,7 @@ type knowledgeBaseService struct {
|
||||
tenantRepo interfaces.TenantRepository
|
||||
fileSvc interfaces.FileService
|
||||
graphEngine interfaces.RetrieveGraphRepository
|
||||
asynqClient *asynq.Client
|
||||
asynqClient interfaces.TaskEnqueuer
|
||||
}
|
||||
|
||||
// NewKnowledgeBaseService creates a new knowledge base service
|
||||
@@ -46,7 +46,7 @@ func NewKnowledgeBaseService(repo interfaces.KnowledgeBaseRepository,
|
||||
tenantRepo interfaces.TenantRepository,
|
||||
fileSvc interfaces.FileService,
|
||||
graphEngine interfaces.RetrieveGraphRepository,
|
||||
asynqClient *asynq.Client,
|
||||
asynqClient interfaces.TaskEnqueuer,
|
||||
) interfaces.KnowledgeBaseService {
|
||||
return &knowledgeBaseService{
|
||||
repo: repo,
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"github.com/Tencent/WeKnora/internal/models/embedding"
|
||||
"github.com/Tencent/WeKnora/internal/models/rerank"
|
||||
"github.com/Tencent/WeKnora/internal/models/utils/ollama"
|
||||
"github.com/Tencent/WeKnora/internal/models/vlm"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
)
|
||||
@@ -405,5 +406,56 @@ func (s *modelService) GetChatModel(ctx context.Context, modelId string) (chat.C
|
||||
return chatModel, nil
|
||||
}
|
||||
|
||||
// GetVLMModel retrieves and initializes a vision language model instance.
|
||||
func (s *modelService) GetVLMModel(ctx context.Context, modelId string) (vlm.VLM, error) {
|
||||
if modelId == "" {
|
||||
return nil, errors.New("model ID cannot be empty")
|
||||
}
|
||||
|
||||
tenantID := ctx.Value(types.TenantIDContextKey).(uint64)
|
||||
|
||||
model, err := s.repo.GetByID(ctx, tenantID, modelId)
|
||||
if err != nil {
|
||||
logger.ErrorWithFields(ctx, err, map[string]interface{}{
|
||||
"model_id": modelId,
|
||||
"tenant_id": tenantID,
|
||||
})
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if model == nil {
|
||||
return nil, ErrModelNotFound
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "Getting VLM model: %s, source: %s", model.Name, model.Source)
|
||||
|
||||
ifType := model.Parameters.InterfaceType
|
||||
if ifType == "" {
|
||||
if model.Source == types.ModelSourceLocal {
|
||||
ifType = "ollama"
|
||||
} else {
|
||||
ifType = "openai"
|
||||
}
|
||||
}
|
||||
|
||||
vlmModel, err := vlm.NewVLM(&vlm.Config{
|
||||
ModelID: model.ID,
|
||||
APIKey: model.Parameters.APIKey,
|
||||
BaseURL: model.Parameters.BaseURL,
|
||||
ModelName: model.Name,
|
||||
Source: model.Source,
|
||||
InterfaceType: ifType,
|
||||
}, s.ollamaService)
|
||||
if err != nil {
|
||||
logger.ErrorWithFields(ctx, err, map[string]interface{}{
|
||||
"model_id": model.ID,
|
||||
"model_name": model.Name,
|
||||
})
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return vlmModel, nil
|
||||
}
|
||||
|
||||
// Note: default model selection logic has been removed; models no longer
|
||||
// maintain a per-type default flag at the service layer.
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
htmltomd "github.com/JohannesKaufmann/html-to-markdown/v2"
|
||||
)
|
||||
|
||||
var (
|
||||
htmlTagPattern = regexp.MustCompile(`<[^>]+>`)
|
||||
codeBlockPattern = regexp.MustCompile("(?s)^\\s*```[a-zA-Z]*\\s*\n(.*?)\n\\s*```\\s*$")
|
||||
htmlDocPattern = regexp.MustCompile(`(?i)^\s*(<\!DOCTYPE|<html|<body|<div|<p[\s>]|<table|<h[1-6][\s>])`)
|
||||
multipleNewlines = regexp.MustCompile(`\n{3,}`)
|
||||
knownEmptyReplies = []string{
|
||||
"无文字内容",
|
||||
"无法识别",
|
||||
"no text",
|
||||
"no content",
|
||||
"empty",
|
||||
"图片中没有文字",
|
||||
"图片中没有可识别的文字",
|
||||
}
|
||||
)
|
||||
|
||||
// sanitizeOCRText cleans up VLM OCR output by stripping HTML wrappers,
|
||||
// converting HTML to markdown, and filtering out useless responses.
|
||||
func sanitizeOCRText(raw string) string {
|
||||
text := strings.TrimSpace(raw)
|
||||
if text == "" {
|
||||
return ""
|
||||
}
|
||||
|
||||
text = stripMarkdownCodeBlock(text)
|
||||
|
||||
// If stripping HTML tags leaves almost no text, the response is useless
|
||||
// (e.g. "<html><body><div class="image"><img/></div></body></html>").
|
||||
plainText := strings.TrimSpace(htmlTagPattern.ReplaceAllString(text, ""))
|
||||
if len(plainText) < 10 && htmlTagPattern.MatchString(text) {
|
||||
return ""
|
||||
}
|
||||
|
||||
if looksLikeHTML(text) {
|
||||
text = ocrHTMLToMarkdown(text)
|
||||
text = strings.TrimSpace(text)
|
||||
if text == "" {
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
if isKnownEmptyReply(text) {
|
||||
return ""
|
||||
}
|
||||
|
||||
text = multipleNewlines.ReplaceAllString(text, "\n\n")
|
||||
return strings.TrimSpace(text)
|
||||
}
|
||||
|
||||
// stripMarkdownCodeBlock removes a markdown code-fence wrapper that some
|
||||
// models add around their output (e.g. ```html\n...\n``` or ```markdown\n...\n```).
|
||||
func stripMarkdownCodeBlock(text string) string {
|
||||
if m := codeBlockPattern.FindStringSubmatch(text); len(m) == 2 {
|
||||
return strings.TrimSpace(m[1])
|
||||
}
|
||||
return text
|
||||
}
|
||||
|
||||
// looksLikeHTML returns true when the text appears to be an HTML document
|
||||
// or contains a significant amount of HTML tags.
|
||||
func looksLikeHTML(text string) bool {
|
||||
if htmlDocPattern.MatchString(text) {
|
||||
return true
|
||||
}
|
||||
tags := htmlTagPattern.FindAllString(text, -1)
|
||||
if len(tags) == 0 {
|
||||
return false
|
||||
}
|
||||
tagChars := 0
|
||||
for _, t := range tags {
|
||||
tagChars += len(t)
|
||||
}
|
||||
return float64(tagChars)/float64(len(text)) > 0.3
|
||||
}
|
||||
|
||||
// ocrHTMLToMarkdown converts HTML content to markdown, falling back to the
|
||||
// original text on failure.
|
||||
func ocrHTMLToMarkdown(content string) string {
|
||||
md, err := htmltomd.ConvertString(content)
|
||||
if err != nil {
|
||||
return content
|
||||
}
|
||||
return md
|
||||
}
|
||||
|
||||
// isKnownEmptyReply checks whether the text matches a known "no content"
|
||||
// reply pattern that VLM models produce when the image has no text.
|
||||
func isKnownEmptyReply(text string) bool {
|
||||
lower := strings.ToLower(strings.TrimSpace(text))
|
||||
for _, phrase := range knownEmptyReplies {
|
||||
if lower == strings.ToLower(phrase) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,213 @@
|
||||
package service
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestSanitizeOCRText(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "empty string",
|
||||
input: "",
|
||||
want: "",
|
||||
},
|
||||
{
|
||||
name: "whitespace only",
|
||||
input: " \n\t ",
|
||||
want: "",
|
||||
},
|
||||
{
|
||||
name: "pure HTML skeleton with no text",
|
||||
input: `<html><body><div class="image"><img/></div></body></html>`,
|
||||
want: "",
|
||||
},
|
||||
{
|
||||
name: "HTML with only whitespace text",
|
||||
input: "<html><body> \n </body></html>",
|
||||
want: "",
|
||||
},
|
||||
{
|
||||
name: "valid markdown passes through",
|
||||
input: "# 标题\n\n这是一段正文,包含一些内容。\n\n| 列1 | 列2 |\n| --- | --- |\n| 数据1 | 数据2 |",
|
||||
want: "# 标题\n\n这是一段正文,包含一些内容。\n\n| 列1 | 列2 |\n| --- | --- |\n| 数据1 | 数据2 |",
|
||||
},
|
||||
{
|
||||
name: "code block wrapper stripped",
|
||||
input: "```markdown\n# 文档标题\n\n正文内容在这里。\n```",
|
||||
want: "# 文档标题\n\n正文内容在这里。",
|
||||
},
|
||||
{
|
||||
name: "html code block wrapper stripped",
|
||||
input: "```html\n<p>这是一段内容</p>\n```",
|
||||
want: "这是一段内容",
|
||||
},
|
||||
{
|
||||
name: "HTML document converted to markdown",
|
||||
input: "<html><body><h1>标题</h1><p>这是一段很长的正文内容,用来测试 HTML 到 Markdown 的转换。</p></body></html>",
|
||||
want: "# 标题\n\n这是一段很长的正文内容,用来测试 HTML 到 Markdown 的转换。",
|
||||
},
|
||||
{
|
||||
name: "known empty reply - Chinese",
|
||||
input: "无文字内容",
|
||||
want: "",
|
||||
},
|
||||
{
|
||||
name: "known empty reply - no text",
|
||||
input: "No text",
|
||||
want: "",
|
||||
},
|
||||
{
|
||||
name: "known empty reply - 图片中没有文字",
|
||||
input: "图片中没有文字",
|
||||
want: "",
|
||||
},
|
||||
{
|
||||
name: "plain text with minimal HTML not converted",
|
||||
input: "这是一段正常文本,价格 <100 元。",
|
||||
want: "这是一段正常文本,价格 <100 元。",
|
||||
},
|
||||
{
|
||||
name: "multiple blank lines collapsed",
|
||||
input: "段落一\n\n\n\n\n段落二",
|
||||
want: "段落一\n\n段落二",
|
||||
},
|
||||
{
|
||||
name: "HTML with substantial text content is converted",
|
||||
input: "<div><h2>报告摘要</h2><p>本季度营收同比增长 15%,净利润达到 2.3 亿元。</p><table><tr><th>指标</th><th>数值</th></tr><tr><td>营收</td><td>10亿</td></tr></table></div>",
|
||||
want: "", // placeholder; will be checked for non-empty
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := sanitizeOCRText(tt.input)
|
||||
|
||||
if tt.name == "HTML with substantial text content is converted" {
|
||||
if got == "" {
|
||||
t.Errorf("sanitizeOCRText() returned empty for substantial HTML content")
|
||||
}
|
||||
if got == tt.input {
|
||||
t.Errorf("sanitizeOCRText() did not convert HTML, got original")
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if got != tt.want {
|
||||
t.Errorf("sanitizeOCRText() = %q, want %q", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestStripMarkdownCodeBlock(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "no code block",
|
||||
input: "just normal text",
|
||||
want: "just normal text",
|
||||
},
|
||||
{
|
||||
name: "markdown code block",
|
||||
input: "```markdown\n# Title\nContent here\n```",
|
||||
want: "# Title\nContent here",
|
||||
},
|
||||
{
|
||||
name: "html code block",
|
||||
input: "```html\n<p>hello</p>\n```",
|
||||
want: "<p>hello</p>",
|
||||
},
|
||||
{
|
||||
name: "plain code block",
|
||||
input: "```\nsome text\n```",
|
||||
want: "some text",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := stripMarkdownCodeBlock(tt.input)
|
||||
if got != tt.want {
|
||||
t.Errorf("stripMarkdownCodeBlock() = %q, want %q", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLooksLikeHTML(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
want bool
|
||||
}{
|
||||
{
|
||||
name: "HTML document",
|
||||
input: "<html><body><p>text</p></body></html>",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "DOCTYPE",
|
||||
input: "<!DOCTYPE html><html><body></body></html>",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "body tag",
|
||||
input: "<body><p>content</p></body>",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "plain markdown",
|
||||
input: "# Title\n\nSome paragraph text",
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "text with minor HTML",
|
||||
input: "This is mostly text with a <b>bold</b> word.",
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "heavy HTML tags",
|
||||
input: "<div><p><span>x</span></p></div><div><p><span>y</span></p></div>",
|
||||
want: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := looksLikeHTML(tt.input)
|
||||
if got != tt.want {
|
||||
t.Errorf("looksLikeHTML() = %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsKnownEmptyReply(t *testing.T) {
|
||||
tests := []struct {
|
||||
input string
|
||||
want bool
|
||||
}{
|
||||
{"无文字内容", true},
|
||||
{"无法识别", true},
|
||||
{"no text", true},
|
||||
{"No Text", true},
|
||||
{"NO CONTENT", true},
|
||||
{"empty", true},
|
||||
{"这是正常内容", false},
|
||||
{"", false},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.input, func(t *testing.T) {
|
||||
got := isKnownEmptyReply(tt.input)
|
||||
if got != tt.want {
|
||||
t.Errorf("isKnownEmptyReply(%q) = %v, want %v", tt.input, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -25,7 +25,7 @@ type knowledgeTagService struct {
|
||||
chunkRepo interfaces.ChunkRepository
|
||||
retrieveEngine interfaces.RetrieveEngineRegistry
|
||||
modelService interfaces.ModelService
|
||||
task *asynq.Client
|
||||
task interfaces.TaskEnqueuer
|
||||
kbShareService interfaces.KBShareService
|
||||
}
|
||||
|
||||
@@ -37,7 +37,7 @@ func NewKnowledgeTagService(
|
||||
chunkRepo interfaces.ChunkRepository,
|
||||
retrieveEngine interfaces.RetrieveEngineRegistry,
|
||||
modelService interfaces.ModelService,
|
||||
task *asynq.Client,
|
||||
task interfaces.TaskEnqueuer,
|
||||
kbShareService interfaces.KBShareService,
|
||||
) (interfaces.KnowledgeTagService, error) {
|
||||
return &knowledgeTagService{
|
||||
|
||||
Reference in New Issue
Block a user