feat: introduce WeKnora Lite edition with lightweight configuration and deployment

- Added a new `.env.lite.example` file for the Lite version, providing a minimal configuration template.
- Updated `.env.example` to remove deprecated variables and include new Docreader settings.
- Enhanced Docker configurations to support the Lite version, including a new Dockerfile for the Docreader service.
- Introduced a Makefile target for building and running the Lite version, along with packaging capabilities.
- Created GitHub workflows for building and releasing Lite binaries, including Homebrew formula support.
- Implemented a new service file for managing the Lite version as a system service.

This update enables a streamlined, single-binary deployment of WeKnora, reducing external dependencies and simplifying setup.
This commit is contained in:
wizardchen
2026-02-28 13:22:07 +08:00
committed by lyingbug
parent 5241dbc39e
commit 397689d2f3
140 changed files with 13365 additions and 5413 deletions
+43 -20
View File
@@ -246,11 +246,11 @@ func (r *chunkRepository) ListChunkByParentID(
return chunks, nil
}
// UpdateChunk updates a chunk using GORM Save, which updates ALL fields.
// Note: This will update all fields including metadata and content_hash.
// UpdateChunk updates a chunk using GORM Save, which updates ALL fields
// except SeqID (auto-increment, must not be overwritten).
// Make sure the chunk object is complete (e.g., fetched from DB) before calling this method.
func (r *chunkRepository) UpdateChunk(ctx context.Context, chunk *types.Chunk) error {
return r.db.WithContext(ctx).Save(chunk).Error
return r.db.WithContext(ctx).Omit("SeqID").Save(chunk).Error
}
// UpdateChunks updates chunks in batch using raw SQL for efficiency.
@@ -332,23 +332,46 @@ func (r *chunkRepository) UpdateChunks(ctx context.Context, chunks []*types.Chun
args = append(args, id)
}
sql := fmt.Sprintf(`
UPDATE chunks SET
content = CASE %s END,
is_enabled = (CASE %s END)::boolean,
tag_id = CASE %s END,
flags = (CASE %s END)::integer,
status = (CASE %s END)::integer,
updated_at = NOW()
WHERE id IN (%s)
`,
strings.Join(contentCases, " "),
strings.Join(isEnabledCases, " "),
strings.Join(tagIDCases, " "),
strings.Join(flagsCases, " "),
strings.Join(statusCases, " "),
strings.Join(inPlaceholders, ","),
)
isPostgres := r.db.Dialector.Name() == "postgres"
var sql string
if isPostgres {
sql = fmt.Sprintf(`
UPDATE chunks SET
content = CASE %s END,
is_enabled = (CASE %s END)::boolean,
tag_id = CASE %s END,
flags = (CASE %s END)::integer,
status = (CASE %s END)::integer,
updated_at = NOW()
WHERE id IN (%s)
`,
strings.Join(contentCases, " "),
strings.Join(isEnabledCases, " "),
strings.Join(tagIDCases, " "),
strings.Join(flagsCases, " "),
strings.Join(statusCases, " "),
strings.Join(inPlaceholders, ","),
)
} else {
sql = fmt.Sprintf(`
UPDATE chunks SET
content = CASE %s END,
is_enabled = CASE %s END,
tag_id = CASE %s END,
flags = CASE %s END,
status = CASE %s END,
updated_at = datetime('now')
WHERE id IN (%s)
`,
strings.Join(contentCases, " "),
strings.Join(isEnabledCases, " "),
strings.Join(tagIDCases, " "),
strings.Join(flagsCases, " "),
strings.Join(statusCases, " "),
strings.Join(inPlaceholders, ","),
)
}
return r.db.WithContext(ctx).Exec(sql, args...).Error
}
@@ -0,0 +1,613 @@
package sqlite
import (
"context"
"fmt"
"strings"
"time"
"unicode"
"github.com/Tencent/WeKnora/internal/common"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
sqlite_vec "github.com/asg017/sqlite-vec-go-bindings/cgo"
"github.com/google/uuid"
"gorm.io/gorm"
"gorm.io/gorm/clause"
)
// sqliteEmbedding stores metadata alongside the vec0 virtual table rows
type sqliteEmbedding struct {
ID uint `gorm:"primarykey;autoIncrement"`
CreatedAt time.Time `gorm:"column:created_at"`
UpdatedAt time.Time `gorm:"column:updated_at"`
SourceID string `gorm:"column:source_id;not null;uniqueIndex:idx_sqlite_emb_source"`
SourceType int `gorm:"column:source_type;not null;uniqueIndex:idx_sqlite_emb_source"`
ChunkID string `gorm:"column:chunk_id;index"`
KnowledgeID string `gorm:"column:knowledge_id;index"`
KnowledgeBaseID string `gorm:"column:knowledge_base_id;index"`
TagID string `gorm:"column:tag_id;index"`
Content string `gorm:"column:content;not null"`
Dimension int `gorm:"column:dimension;not null"`
IsEnabled *bool `gorm:"column:is_enabled;default:true;index"`
}
func (sqliteEmbedding) TableName() string { return "lite_embeddings" }
type sqliteRepository struct {
db *gorm.DB
vecTables map[int]bool // tracks which vec0 tables have been created (keyed by dimension)
}
func NewSQLiteRetrieveEngineRepository(db *gorm.DB) interfaces.RetrieveEngineRepository {
logger.GetLogger(context.Background()).Info("[SQLite] Initializing SQLite retriever engine repository with sqlite-vec")
if err := db.AutoMigrate(&sqliteEmbedding{}); err != nil {
logger.GetLogger(context.Background()).Errorf("[SQLite] Failed to auto-migrate lite_embeddings: %v", err)
}
initFTS5(db)
repo := &sqliteRepository{
db: db,
vecTables: make(map[int]bool),
}
repo.ensureExistingVecTables()
return repo
}
func initFTS5(db *gorm.DB) {
sql := `CREATE VIRTUAL TABLE IF NOT EXISTS lite_embeddings_fts USING fts5(
content, source_id, chunk_id, knowledge_id, knowledge_base_id,
content='lite_embeddings', content_rowid='id',
tokenize='unicode61'
)`
if err := db.Exec(sql).Error; err != nil {
logger.GetLogger(context.Background()).Warnf("[SQLite] Failed to create FTS5 table: %v", err)
}
}
func vecTableName(dim int) string {
return fmt.Sprintf("vec_embeddings_%d", dim)
}
func (r *sqliteRepository) ensureVecTable(dim int) {
if dim <= 0 || r.vecTables[dim] {
return
}
tbl := vecTableName(dim)
createSQL := fmt.Sprintf(
`CREATE VIRTUAL TABLE IF NOT EXISTS %s USING vec0(embedding float[%d] distance_metric=cosine)`,
tbl, dim,
)
if err := r.db.Exec(createSQL).Error; err != nil {
if strings.Contains(err.Error(), "already exists") {
r.vecTables[dim] = true
return
}
logger.GetLogger(context.Background()).Errorf("[SQLite] Failed to create vec0 table for dim %d: %v", dim, err)
return
}
r.vecTables[dim] = true
}
func (r *sqliteRepository) ensureExistingVecTables() {
var dims []int
r.db.Model(&sqliteEmbedding{}).Distinct("dimension").Where("dimension > 0").Pluck("dimension", &dims)
for _, dim := range dims {
r.ensureVecTable(dim)
}
}
func (r *sqliteRepository) EngineType() types.RetrieverEngineType {
return types.SQLiteRetrieverEngineType
}
func (r *sqliteRepository) Support() []types.RetrieverType {
return []types.RetrieverType{types.KeywordsRetrieverType, types.VectorRetrieverType}
}
func (r *sqliteRepository) Save(ctx context.Context, indexInfo *types.IndexInfo, params map[string]any) error {
row := toSQLiteEmbedding(indexInfo)
emb := extractEmbedding(params, indexInfo.SourceID)
if len(emb) > 0 {
row.Dimension = len(emb)
}
if err := r.db.WithContext(ctx).Clauses(clause.OnConflict{DoNothing: true}).Create(row).Error; err != nil {
return err
}
r.syncFTS5Insert(ctx, row)
if len(emb) > 0 && row.ID > 0 {
r.insertVec(ctx, row.ID, row.Dimension, emb)
}
return nil
}
func (r *sqliteRepository) BatchSave(ctx context.Context, indexInfoList []*types.IndexInfo, params map[string]any) error {
if len(indexInfoList) == 0 {
return nil
}
rows := make([]*sqliteEmbedding, len(indexInfoList))
embs := make([][]float32, len(indexInfoList))
for i, info := range indexInfoList {
rows[i] = toSQLiteEmbedding(info)
emb := extractEmbedding(params, info.SourceID)
embs[i] = emb
if len(emb) > 0 {
rows[i].Dimension = len(emb)
}
}
if err := r.db.WithContext(ctx).Clauses(clause.OnConflict{DoNothing: true}).Create(rows).Error; err != nil {
return err
}
for i, row := range rows {
r.syncFTS5Insert(ctx, row)
if len(embs[i]) > 0 && row.ID > 0 {
r.insertVec(ctx, row.ID, row.Dimension, embs[i])
}
}
return nil
}
func (r *sqliteRepository) EstimateStorageSize(_ context.Context, indexInfoList []*types.IndexInfo, _ map[string]any) int64 {
var total int64
for _, info := range indexInfoList {
total += int64(len(info.Content)) + 200
}
return total
}
func (r *sqliteRepository) DeleteByChunkIDList(ctx context.Context, chunkIDList []string, _ int, _ string) error {
var rows []sqliteEmbedding
r.db.WithContext(ctx).Where("chunk_id IN ?", chunkIDList).Find(&rows)
r.deleteRowsAndVecs(ctx, rows)
return r.db.WithContext(ctx).Where("chunk_id IN ?", chunkIDList).Delete(&sqliteEmbedding{}).Error
}
func (r *sqliteRepository) DeleteBySourceIDList(ctx context.Context, sourceIDList []string, _ int, _ string) error {
var rows []sqliteEmbedding
r.db.WithContext(ctx).Where("source_id IN ?", sourceIDList).Find(&rows)
r.deleteRowsAndVecs(ctx, rows)
return r.db.WithContext(ctx).Where("source_id IN ?", sourceIDList).Delete(&sqliteEmbedding{}).Error
}
func (r *sqliteRepository) DeleteByKnowledgeIDList(ctx context.Context, knowledgeIDList []string, _ int, _ string) error {
var rows []sqliteEmbedding
r.db.WithContext(ctx).Where("knowledge_id IN ?", knowledgeIDList).Find(&rows)
r.deleteRowsAndVecs(ctx, rows)
return r.db.WithContext(ctx).Where("knowledge_id IN ?", knowledgeIDList).Delete(&sqliteEmbedding{}).Error
}
func (r *sqliteRepository) CopyIndices(ctx context.Context,
_ string,
sourceToTargetKBIDMap map[string]string,
sourceToTargetChunkIDMap map[string]string,
targetKnowledgeBaseID string,
_ int, _ string,
) error {
for sourceChunkID, targetChunkID := range sourceToTargetChunkIDMap {
var src sqliteEmbedding
if err := r.db.WithContext(ctx).Where("chunk_id = ?", sourceChunkID).First(&src).Error; err != nil {
continue
}
newRow := sqliteEmbedding{
SourceID: uuid.New().String(),
SourceType: src.SourceType,
ChunkID: targetChunkID,
KnowledgeID: sourceToTargetKBIDMap[src.KnowledgeID],
KnowledgeBaseID: targetKnowledgeBaseID,
TagID: src.TagID,
Content: src.Content,
Dimension: src.Dimension,
IsEnabled: src.IsEnabled,
}
if err := r.db.WithContext(ctx).Create(&newRow).Error; err != nil {
logger.GetLogger(ctx).Warnf("[SQLite] CopyIndices: failed to copy chunk %s: %v", sourceChunkID, err)
continue
}
r.syncFTS5Insert(ctx, &newRow)
if src.Dimension > 0 && newRow.ID > 0 {
r.copyVec(ctx, src.ID, newRow.ID, src.Dimension)
}
}
return nil
}
func (r *sqliteRepository) BatchUpdateChunkEnabledStatus(ctx context.Context, chunkStatusMap map[string]bool) error {
for chunkID, enabled := range chunkStatusMap {
r.db.WithContext(ctx).Model(&sqliteEmbedding{}).Where("chunk_id = ?", chunkID).Update("is_enabled", enabled)
}
return nil
}
func (r *sqliteRepository) BatchUpdateChunkTagID(ctx context.Context, chunkTagMap map[string]string) error {
for chunkID, tagID := range chunkTagMap {
r.db.WithContext(ctx).Model(&sqliteEmbedding{}).Where("chunk_id = ?", chunkID).Update("tag_id", tagID)
}
return nil
}
// --- Retrieve ---
func (r *sqliteRepository) Retrieve(ctx context.Context, params types.RetrieveParams) ([]*types.RetrieveResult, error) {
var results []*types.RetrieveResult
if params.RetrieverType == types.KeywordsRetrieverType || params.RetrieverType == "" {
res, err := r.keywordsRetrieve(ctx, params)
if err != nil {
results = append(results, &types.RetrieveResult{
RetrieverEngineType: types.SQLiteRetrieverEngineType,
RetrieverType: types.KeywordsRetrieverType,
Error: err,
})
} else {
results = append(results, res...)
}
}
if params.RetrieverType == types.VectorRetrieverType || params.RetrieverType == "" {
res, err := r.vectorRetrieve(ctx, params)
if err != nil {
results = append(results, &types.RetrieveResult{
RetrieverEngineType: types.SQLiteRetrieverEngineType,
RetrieverType: types.VectorRetrieverType,
Error: err,
})
} else {
results = append(results, res...)
}
}
return results, nil
}
// --- Keywords retrieval via FTS5 ---
func (r *sqliteRepository) keywordsRetrieve(ctx context.Context, params types.RetrieveParams) ([]*types.RetrieveResult, error) {
if params.Query == "" {
return nil, nil
}
ftsQuery := sanitizeFTS5Query(params.Query)
sql := `
SELECT e.id, e.source_id, e.source_type, e.chunk_id,
e.knowledge_id, e.knowledge_base_id, e.tag_id,
e.content,
bm25(lite_embeddings_fts) AS score
FROM lite_embeddings_fts
JOIN lite_embeddings e ON e.id = lite_embeddings_fts.rowid
WHERE lite_embeddings_fts MATCH ?
AND (e.is_enabled IS NULL OR e.is_enabled = 1)
`
args := []interface{}{ftsQuery}
for _, wp := range buildFilterWhere(params) {
sql += " AND " + wp.clause
args = append(args, wp.args...)
}
sql += " ORDER BY score ASC LIMIT ?"
args = append(args, params.TopK)
type ftsResult struct {
ID uint
SourceID string
SourceType int
ChunkID string
KnowledgeID string
KnowledgeBaseID string
TagID string
Content string
Score float64
}
var rows []ftsResult
if err := r.db.WithContext(ctx).Raw(sql, args...).Scan(&rows).Error; err != nil {
return nil, fmt.Errorf("FTS5 query failed: %w", err)
}
logger.GetLogger(ctx).Infof("[SQLite] keywordsRetrieve: query=%q, ftsQuery=%q, matched=%d rows", params.Query, ftsQuery, len(rows))
items := make([]*types.IndexWithScore, len(rows))
for i, row := range rows {
// bm25 越小越相关 → 转成正向分数
score := -row.Score
logger.GetLogger(ctx).Infof("[SQLite] keywordsRetrieve: #%d chunk_id=%s, bm25_raw=%.4f, score=%.4f, content_preview=%.60s",
i+1, row.ChunkID, row.Score, score, row.Content)
items[i] = &types.IndexWithScore{
ID: fmt.Sprintf("%d", row.ID),
SourceID: row.SourceID,
SourceType: types.SourceType(row.SourceType),
ChunkID: row.ChunkID,
KnowledgeID: row.KnowledgeID,
KnowledgeBaseID: row.KnowledgeBaseID,
TagID: row.TagID,
Content: row.Content,
Score: score,
MatchType: types.MatchTypeKeywords,
}
}
return []*types.RetrieveResult{{
Results: items,
RetrieverEngineType: types.SQLiteRetrieverEngineType,
RetrieverType: types.KeywordsRetrieverType,
}}, nil
}
func (r *sqliteRepository) vectorRetrieve(ctx context.Context, params types.RetrieveParams) ([]*types.RetrieveResult, error) {
if len(params.Embedding) == 0 {
return nil, nil
}
dim := len(params.Embedding)
r.ensureVecTable(dim)
queryBlob, err := sqlite_vec.SerializeFloat32(params.Embedding)
if err != nil {
return nil, fmt.Errorf("serialize query vector failed: %w", err)
}
tbl := vecTableName(dim)
// ⚠️ sqlite-vec 要求必须有 k = ?
vecSQL := fmt.Sprintf(`
SELECT v.rowid, v.distance,
e.source_id, e.source_type, e.chunk_id,
e.knowledge_id, e.knowledge_base_id,
e.tag_id, e.content
FROM %s v
JOIN lite_embeddings e ON e.id = v.rowid
WHERE v.embedding MATCH ?
AND k = ?
AND (e.is_enabled IS NULL OR e.is_enabled = 1)
`, tbl)
args := []interface{}{
queryBlob,
params.TopK, // 这里就是 k
}
// 追加过滤条件
for _, wp := range buildFilterWhere(params) {
vecSQL += " AND " + wp.clause
args = append(args, wp.args...)
}
// ⚠️ 这里仍然建议加 ORDER BY,虽然 vec0 已经按距离返回
vecSQL += " ORDER BY v.distance ASC"
type row struct {
Rowid uint
Distance float64
SourceID string
SourceType int
ChunkID string
KnowledgeID string
KnowledgeBaseID string
TagID string
Content string
}
var rows []row
if err := r.db.WithContext(ctx).
Raw(vecSQL, args...).
Scan(&rows).Error; err != nil {
return nil, fmt.Errorf("sqlite-vec query failed: %w", err)
}
logger.GetLogger(ctx).Infof("[SQLite] vectorRetrieve: query_dim=%d, threshold=%.4f, matched=%d rows", dim, params.Threshold, len(rows))
items := make([]*types.IndexWithScore, 0, len(rows))
for i, v := range rows {
// cosine distance = 1 - cosine_similarity
score := 1 - v.Distance
logger.GetLogger(ctx).Infof("[SQLite] vectorRetrieve: #%d chunk_id=%s, distance=%.4f, score=%.4f, content_preview=%.60s",
i+1, v.ChunkID, v.Distance, score, v.Content)
items = append(items, &types.IndexWithScore{
ID: fmt.Sprintf("%d", v.Rowid),
SourceID: v.SourceID,
SourceType: types.SourceType(v.SourceType),
ChunkID: v.ChunkID,
KnowledgeID: v.KnowledgeID,
KnowledgeBaseID: v.KnowledgeBaseID,
TagID: v.TagID,
Content: v.Content,
Score: score,
MatchType: types.MatchTypeEmbedding,
})
}
return []*types.RetrieveResult{{
Results: items,
RetrieverEngineType: types.SQLiteRetrieverEngineType,
RetrieverType: types.VectorRetrieverType,
}}, nil
}
// --- Internal helpers ---
func toSQLiteEmbedding(info *types.IndexInfo) *sqliteEmbedding {
enabled := true
return &sqliteEmbedding{
SourceID: info.SourceID,
SourceType: int(info.SourceType),
ChunkID: info.ChunkID,
KnowledgeID: info.KnowledgeID,
KnowledgeBaseID: info.KnowledgeBaseID,
TagID: info.TagID,
Content: common.CleanInvalidUTF8(info.Content),
Dimension: 0,
IsEnabled: &enabled,
}
}
func extractEmbedding(params map[string]any, sourceID string) []float32 {
if params == nil {
return nil
}
embMap, ok := params["embedding"].(map[string][]float32)
if !ok {
return nil
}
return embMap[sourceID]
}
func (r *sqliteRepository) insertVec(_ context.Context, rowID uint, dim int, emb []float32) {
r.ensureVecTable(dim)
blob, err := sqlite_vec.SerializeFloat32(emb)
if err != nil {
return
}
sql := fmt.Sprintf("INSERT INTO %s(rowid, embedding) VALUES (?, ?)", vecTableName(dim))
r.db.Exec(sql, rowID, blob)
}
func (r *sqliteRepository) deleteRowsAndVecs(_ context.Context, rows []sqliteEmbedding) {
dimIDs := make(map[int][]uint)
for _, row := range rows {
if row.Dimension > 0 {
dimIDs[row.Dimension] = append(dimIDs[row.Dimension], row.ID)
}
}
for dim, ids := range dimIDs {
if !r.vecTables[dim] {
continue
}
tbl := vecTableName(dim)
for _, id := range ids {
r.db.Exec(fmt.Sprintf("DELETE FROM %s WHERE rowid = ?", tbl), id)
}
}
}
func (r *sqliteRepository) copyVec(_ context.Context, srcID, dstID uint, dim int) {
if !r.vecTables[dim] {
return
}
tbl := vecTableName(dim)
r.db.Exec(fmt.Sprintf(
"INSERT INTO %s(rowid, embedding) SELECT ?, embedding FROM %s WHERE rowid = ?",
tbl, tbl,
), dstID, srcID)
}
func (r *sqliteRepository) syncFTS5Insert(_ context.Context, row *sqliteEmbedding) {
if row.ID == 0 {
return
}
sql := `INSERT INTO lite_embeddings_fts(rowid, content, source_id, chunk_id, knowledge_id, knowledge_base_id) VALUES(?, ?, ?, ?, ?, ?)`
r.db.Exec(sql, row.ID, row.Content, row.SourceID, row.ChunkID, row.KnowledgeID, row.KnowledgeBaseID)
}
type whereClause struct {
clause string
args []interface{}
}
func buildFilterWhere(params types.RetrieveParams) []whereClause {
var parts []whereClause
if len(params.KnowledgeBaseIDs) > 0 {
parts = append(parts, whereClause{
clause: "e.knowledge_base_id IN (" + placeholders(len(params.KnowledgeBaseIDs)) + ")",
args: toInterfaceSlice(params.KnowledgeBaseIDs),
})
}
if len(params.KnowledgeIDs) > 0 {
parts = append(parts, whereClause{
clause: "e.knowledge_id IN (" + placeholders(len(params.KnowledgeIDs)) + ")",
args: toInterfaceSlice(params.KnowledgeIDs),
})
}
if len(params.TagIDs) > 0 {
parts = append(parts, whereClause{
clause: "e.tag_id IN (" + placeholders(len(params.TagIDs)) + ")",
args: toInterfaceSlice(params.TagIDs),
})
}
return parts
}
func placeholders(n int) string {
p := make([]string, n)
for i := range p {
p[i] = "?"
}
return strings.Join(p, ",")
}
func toInterfaceSlice(ss []string) []interface{} {
out := make([]interface{}, len(ss))
for i, s := range ss {
out[i] = s
}
return out
}
// sanitizeFTS5Query builds an FTS5 query from user input.
// CJK characters are split into overlapping bigrams (mimicking CJK analyzers used
// by Elasticsearch etc.) and joined with OR for broad partial matching.
// Non-CJK words are kept intact. BM25 ranking naturally boosts documents
// that match more tokens.
func sanitizeFTS5Query(q string) string {
q = strings.TrimSpace(q)
if q == "" {
return q
}
var cjkRunes []rune
var nonCJKWords []string
var buf strings.Builder
flushNonCJK := func() {
if buf.Len() > 0 {
nonCJKWords = append(nonCJKWords, buf.String())
buf.Reset()
}
}
for _, r := range q {
if unicode.Is(unicode.Han, r) {
flushNonCJK()
cjkRunes = append(cjkRunes, r)
} else if unicode.IsSpace(r) || r == '|' {
flushNonCJK()
} else if r == '"' || r == '*' || r == '(' || r == ')' || r == '{' || r == '}' {
// skip FTS5 special characters
} else {
buf.WriteRune(r)
}
}
flushNonCJK()
var parts []string
// CJK bigrams for better matching (e.g. "苹果第四季度" → "苹果" OR "果第" OR "第四" OR ...)
if len(cjkRunes) == 1 {
parts = append(parts, `"`+string(cjkRunes[0])+`"`)
} else {
for i := 0; i < len(cjkRunes)-1; i++ {
bigram := string(cjkRunes[i]) + string(cjkRunes[i+1])
parts = append(parts, `"`+bigram+`"`)
}
}
for _, w := range nonCJKWords {
w = strings.ReplaceAll(w, `"`, `""`)
parts = append(parts, `"`+w+`"`)
}
if len(parts) == 0 {
return `"` + strings.ReplaceAll(q, `"`, `""`) + `"`
}
return strings.Join(parts, " OR ")
}
+1 -1
View File
@@ -114,7 +114,7 @@ func (r *tenantRepository) AdjustStorageUsed(ctx context.Context, tenantID uint6
tenant.StorageUsed += delta
// 保存更新并验证业务规则
if tenant.StorageUsed < 0 {
logger.Error(ctx, "tenant storage used is negative %s: %d", tenant.ID, tenant.StorageUsed)
logger.Errorf(ctx, "tenant storage used is negative %d: %d", tenant.ID, tenant.StorageUsed)
tenant.StorageUsed = 0
}
@@ -183,6 +183,9 @@ func (p *PluginSearch) OnEvent(ctx context.Context,
return
}
if len(res) > 0 {
for _, r := range res {
r.KnowledgeBaseID = t.KnowledgeBaseID
}
pipelineInfo(ctx, "Search", "expansion_hits", map[string]interface{}{
"kb_id": t.KnowledgeBaseID,
"query": q,
@@ -338,6 +341,9 @@ func (p *PluginSearch) searchByTargets(
directResults, skippedIDs := p.tryDirectChunkLoading(ctx, chatManage.TenantID, t.KnowledgeIDs)
if len(directResults) > 0 {
for _, r := range directResults {
r.KnowledgeBaseID = t.KnowledgeBaseID
}
pipelineInfo(ctx, "Search", "direct_load", map[string]interface{}{
"kb_id": t.KnowledgeBaseID,
"loaded_count": len(directResults),
@@ -383,6 +389,9 @@ func (p *PluginSearch) searchByTargets(
})
return
}
for _, r := range res {
r.KnowledgeBaseID = t.KnowledgeBaseID
}
pipelineInfo(ctx, "Search", "kb_result", map[string]interface{}{
"kb_id": t.KnowledgeBaseID,
"target_type": t.Type,
@@ -222,5 +222,6 @@ func chunk2SearchResult(chunk *types.Chunk, knowledge *types.Knowledge) *types.S
KnowledgeFilename: knowledge.FileName,
KnowledgeSource: knowledge.Source,
ChunkMetadata: chunk.Metadata,
KnowledgeBaseID: knowledge.KnowledgeBaseID,
}
}
+2 -2
View File
@@ -79,7 +79,7 @@ const (
// NewChunkExtractTask creates a new chunk extract task
func NewChunkExtractTask(
ctx context.Context,
client *asynq.Client,
client interfaces.TaskEnqueuer,
tenantID uint64,
chunkID string,
modelID string,
@@ -109,7 +109,7 @@ func NewChunkExtractTask(
// NewTableExtractTask creates a new table extract task
func NewDataTableSummaryTask(
ctx context.Context,
client *asynq.Client,
client interfaces.TaskEnqueuer,
tenantID uint64,
knowledgeID string,
summaryModel string,
+43 -23
View File
@@ -12,6 +12,7 @@ import (
"strings"
"time"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/Tencent/WeKnora/internal/utils"
"github.com/google/uuid"
@@ -23,11 +24,28 @@ type cosFileService struct {
client *cos.Client
bucketURL string
cosPathPrefix string
// 临时桶配置(用于存放自动过期的临时文件)
tempClient *cos.Client
tempBucketURL string
}
// newCosClient creates a bare cosFileService with just the SDK client initialised.
// Shared by NewCosFileService* constructors and CheckCosConnectivity.
func newCosClient(bucketName, region, secretID, secretKey string) (*cosFileService, error) {
bucketURL := fmt.Sprintf("https://%s.cos.%s.myqcloud.com/", bucketName, region)
u, err := url.Parse(bucketURL)
logger.Infof(context.Background(), "newCosClient: bucketURL: %s", bucketURL)
if err != nil {
return nil, fmt.Errorf("failed to parse bucketURL: %w", err)
}
client := cos.NewClient(&cos.BaseURL{BucketURL: u}, &http.Client{
Transport: &cos.AuthorizationTransport{
SecretID: secretID,
SecretKey: secretKey,
},
})
return &cosFileService{client: client, bucketURL: bucketURL}, nil
}
// NewCosFileService creates a new COS file service instance
func NewCosFileService(bucketName, region, secretId, secretKey, cosPathPrefix string) (interfaces.FileService, error) {
return NewCosFileServiceWithTempBucket(bucketName, region, secretId, secretKey, cosPathPrefix, "", "")
@@ -35,49 +53,51 @@ func NewCosFileService(bucketName, region, secretId, secretKey, cosPathPrefix st
// NewCosFileServiceWithTempBucket creates a new COS file service instance with optional temp bucket
func NewCosFileServiceWithTempBucket(bucketName, region, secretId, secretKey, cosPathPrefix, tempBucketName, tempRegion string) (interfaces.FileService, error) {
bucketURL := fmt.Sprintf("https://%s.cos.%s.tencentcos.cn/", bucketName, region)
u, err := url.Parse(bucketURL)
svc, err := newCosClient(bucketName, region, secretId, secretKey)
if err != nil {
return nil, fmt.Errorf("failed to parse bucketURL: %w", err)
return nil, err
}
b := &cos.BaseURL{BucketURL: u}
client := cos.NewClient(b, &http.Client{
Transport: &cos.AuthorizationTransport{
SecretID: secretId,
SecretKey: secretKey,
},
})
svc.cosPathPrefix = cosPathPrefix
svc := &cosFileService{
client: client,
bucketURL: bucketURL,
cosPathPrefix: cosPathPrefix,
}
// 如果配置了临时桶,初始化临时桶客户端
if tempBucketName != "" {
if tempRegion == "" {
tempRegion = region // 默认使用相同的 region
tempRegion = region
}
tempBucketURL := fmt.Sprintf("https://%s.cos.%s.tencentcos.cn/", tempBucketName, tempRegion)
tempBucketURL := fmt.Sprintf("https://%s.cos.%s.myqcloud.com/", tempBucketName, tempRegion)
tempU, err := url.Parse(tempBucketURL)
if err != nil {
return nil, fmt.Errorf("failed to parse temp bucketURL: %w", err)
}
tempB := &cos.BaseURL{BucketURL: tempU}
tempClient := cos.NewClient(tempB, &http.Client{
svc.tempClient = cos.NewClient(&cos.BaseURL{BucketURL: tempU}, &http.Client{
Transport: &cos.AuthorizationTransport{
SecretID: secretId,
SecretKey: secretKey,
},
})
svc.tempClient = tempClient
svc.tempBucketURL = tempBucketURL
}
return svc, nil
}
// CheckConnectivity verifies COS is reachable by performing a HEAD request on the bucket.
func (s *cosFileService) CheckConnectivity(ctx context.Context) error {
checkCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
_, err := s.client.Bucket.Head(checkCtx)
return err
}
// CheckCosConnectivity tests COS connectivity using the provided credentials.
// It creates a temporary service instance internally and delegates to CheckConnectivity.
func CheckCosConnectivity(ctx context.Context, bucketName, region, secretID, secretKey string) error {
svc, err := newCosClient(bucketName, region, secretID, secretKey)
if err != nil {
return err
}
return svc.CheckConnectivity(ctx)
}
// SaveFile saves a file to COS storage
// It generates a unique name for the file and organizes it by tenant and knowledge ID
func (s *cosFileService) SaveFile(ctx context.Context,
@@ -14,6 +14,11 @@ import (
// used for testing or when file storage is not required
type DummyFileService struct{}
// CheckConnectivity always succeeds for the dummy service.
func (s *DummyFileService) CheckConnectivity(ctx context.Context) error {
return nil
}
// NewDummyFileService creates a new instance of DummyFileService
func NewDummyFileService() interfaces.FileService {
return &DummyFileService{}
@@ -19,6 +19,18 @@ type localFileService struct {
baseDir string // Base directory for file storage
}
// CheckConnectivity verifies the local storage directory exists and is accessible.
func (s *localFileService) CheckConnectivity(ctx context.Context) error {
info, err := os.Stat(s.baseDir)
if err != nil {
return fmt.Errorf("storage directory not accessible: %w", err)
}
if !info.IsDir() {
return fmt.Errorf("storage path is not a directory: %s", s.baseDir)
}
return nil
}
// NewLocalFileService creates a new local file service instance
func NewLocalFileService(baseDir string) interfaces.FileService {
return &localFileService{
+49 -14
View File
@@ -22,11 +22,10 @@ type minioFileService struct {
bucketName string
}
// NewMinioFileService creates a MinIO file service
func NewMinioFileService(endpoint,
accessKeyID, secretAccessKey, bucketName string, useSSL bool,
) (interfaces.FileService, error) {
// Initialize MinIO client
// newMinioClient creates a bare minioFileService with just the SDK client initialised.
// Shared by NewMinioFileService (which also ensures the bucket exists) and
// CheckMinioConnectivity (read-only probe).
func newMinioClient(endpoint, accessKeyID, secretAccessKey, bucketName string, useSSL bool) (*minioFileService, error) {
client, err := minio.New(endpoint, &minio.Options{
Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""),
Secure: useSSL,
@@ -34,24 +33,60 @@ func NewMinioFileService(endpoint,
if err != nil {
return nil, fmt.Errorf("failed to initialize MinIO client: %w", err)
}
return &minioFileService{client: client, bucketName: bucketName}, nil
}
// Check if bucket exists, create if not
exists, err := client.BucketExists(context.Background(), bucketName)
// NewMinioFileService creates a MinIO file service.
// It verifies that the bucket exists and creates it if missing.
func NewMinioFileService(endpoint,
accessKeyID, secretAccessKey, bucketName string, useSSL bool,
) (interfaces.FileService, error) {
svc, err := newMinioClient(endpoint, accessKeyID, secretAccessKey, bucketName, useSSL)
if err != nil {
return nil, err
}
exists, err := svc.client.BucketExists(context.Background(), bucketName)
if err != nil {
return nil, fmt.Errorf("failed to check bucket: %w", err)
}
if !exists {
err = client.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{})
if err != nil {
if err = svc.client.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{}); err != nil {
return nil, fmt.Errorf("failed to create bucket: %w", err)
}
}
return &minioFileService{
client: client,
bucketName: bucketName,
}, nil
return svc, nil
}
// CheckConnectivity verifies MinIO is reachable and, if a bucket is configured,
// that the bucket exists. This is a read-only probe — it never creates a bucket.
func (s *minioFileService) CheckConnectivity(ctx context.Context) error {
checkCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
if s.bucketName != "" {
exists, err := s.client.BucketExists(checkCtx, s.bucketName)
if err != nil {
return err
}
if !exists {
return fmt.Errorf("bucket %q does not exist", s.bucketName)
}
return nil
}
_, err := s.client.ListBuckets(checkCtx)
return err
}
// CheckMinioConnectivity tests MinIO connectivity using the provided credentials.
// It creates a temporary service instance internally and delegates to CheckConnectivity.
func CheckMinioConnectivity(ctx context.Context, endpoint, accessKeyID, secretAccessKey, bucketName string, useSSL bool) error {
svc, err := newMinioClient(endpoint, accessKeyID, secretAccessKey, bucketName, useSSL)
if err != nil {
return err
}
return svc.CheckConnectivity(ctx)
}
// SaveFile saves a file to MinIO
@@ -0,0 +1,365 @@
package service
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"os"
"strings"
"time"
"github.com/Tencent/WeKnora/internal/application/service/retriever"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/models/utils/ollama"
"github.com/Tencent/WeKnora/internal/models/vlm"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/google/uuid"
"github.com/hibiken/asynq"
)
const (
vlmOCRPrompt = "请提取这张文档图片中的所有正文内容,用纯 Markdown 格式输出。要求:\n" +
"1. 忽略页眉、页脚\n" +
"2. 表格使用 Markdown 表格语法\n" +
"3. 公式使用 LaTeX 格式(用 $ 或 $$ 包裹)\n" +
"4. 按照原文阅读顺序组织\n" +
"5. 只输出提取到的文本内容,不要添加任何 HTML 标签\n" +
"如果图片中没有可识别的文字内容,请回复:无文字内容。"
vlmCaptionPrompt = "简单凝炼的描述图片的主要内容"
)
// ImageMultimodalService handles image:multimodal asynq tasks.
// It reads images from local storage, performs OCR and VLM caption,
// and creates child chunks for the processed content.
type ImageMultimodalService struct {
chunkService interfaces.ChunkService
modelService interfaces.ModelService
kbService interfaces.KnowledgeBaseService
knowledgeRepo interfaces.KnowledgeRepository
tenantRepo interfaces.TenantRepository
retrieveEngine interfaces.RetrieveEngineRegistry
ollamaService *ollama.OllamaService
}
func NewImageMultimodalService(
chunkService interfaces.ChunkService,
modelService interfaces.ModelService,
kbService interfaces.KnowledgeBaseService,
knowledgeRepo interfaces.KnowledgeRepository,
tenantRepo interfaces.TenantRepository,
retrieveEngine interfaces.RetrieveEngineRegistry,
ollamaService *ollama.OllamaService,
) interfaces.TaskHandler {
return &ImageMultimodalService{
chunkService: chunkService,
modelService: modelService,
kbService: kbService,
knowledgeRepo: knowledgeRepo,
tenantRepo: tenantRepo,
retrieveEngine: retrieveEngine,
ollamaService: ollamaService,
}
}
// Handle implements asynq handler for TypeImageMultimodal.
func (s *ImageMultimodalService) Handle(ctx context.Context, task *asynq.Task) error {
var payload types.ImageMultimodalPayload
if err := json.Unmarshal(task.Payload(), &payload); err != nil {
return fmt.Errorf("unmarshal image multimodal payload: %w", err)
}
logger.Infof(ctx, "[ImageMultimodal] Processing image: chunk=%s, url=%s, local_path=%s, ocr=%v, caption=%v",
payload.ChunkID, payload.ImageURL, payload.ImageLocalPath, payload.EnableOCR, payload.EnableCaption)
ctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID)
vlmModel, err := s.resolveVLM(ctx, payload.KnowledgeBaseID)
if err != nil {
return fmt.Errorf("resolve VLM: %w", err)
}
imgBytes, err := os.ReadFile(payload.ImageLocalPath)
if err != nil {
logger.Warnf(ctx, "[ImageMultimodal] Local file %s not available (%v), trying to download from URL", payload.ImageLocalPath, err)
imgBytes, err = downloadImageFromURL(payload.ImageURL)
if err != nil {
logger.Errorf(ctx, "[ImageMultimodal] Failed to download image from URL %s: %v", payload.ImageURL, err)
return fmt.Errorf("read image: local file %s not found and URL download failed: %w", payload.ImageLocalPath, err)
}
logger.Infof(ctx, "[ImageMultimodal] Image downloaded from URL, len=%d", len(imgBytes))
} else {
logger.Infof(ctx, "[ImageMultimodal] Image bytes read from %s, len=%d", payload.ImageLocalPath, len(imgBytes))
defer os.Remove(payload.ImageLocalPath)
}
imageInfo := types.ImageInfo{
URL: payload.ImageURL,
OriginalURL: payload.ImageURL,
}
if payload.EnableOCR {
ocrText, ocrErr := vlmModel.Predict(ctx, imgBytes, vlmOCRPrompt)
if ocrErr != nil {
logger.Warnf(ctx, "[ImageMultimodal] OCR failed for %s: %v", payload.ImageURL, ocrErr)
} else {
ocrText = sanitizeOCRText(ocrText)
if ocrText != "" {
imageInfo.OCRText = ocrText
} else {
logger.Warnf(ctx, "[ImageMultimodal] OCR returned empty/invalid content for %s, discarded", payload.ImageURL)
}
}
}
if payload.EnableCaption {
caption, capErr := vlmModel.Predict(ctx, imgBytes, vlmCaptionPrompt)
if capErr != nil {
logger.Warnf(ctx, "[ImageMultimodal] Caption failed for %s: %v", payload.ImageURL, capErr)
} else if caption != "" {
imageInfo.Caption = caption
}
}
// Build child chunks for OCR and caption results
imageInfoJSON, _ := json.Marshal([]types.ImageInfo{imageInfo})
var newChunks []*types.Chunk
if imageInfo.OCRText != "" {
newChunks = append(newChunks, &types.Chunk{
ID: uuid.New().String(),
TenantID: payload.TenantID,
KnowledgeID: payload.KnowledgeID,
KnowledgeBaseID: payload.KnowledgeBaseID,
Content: imageInfo.OCRText,
ChunkType: types.ChunkTypeImageOCR,
ParentChunkID: payload.ChunkID,
IsEnabled: true,
Flags: types.ChunkFlagRecommended,
ImageInfo: string(imageInfoJSON),
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
})
}
if imageInfo.Caption != "" {
newChunks = append(newChunks, &types.Chunk{
ID: uuid.New().String(),
TenantID: payload.TenantID,
KnowledgeID: payload.KnowledgeID,
KnowledgeBaseID: payload.KnowledgeBaseID,
Content: imageInfo.Caption,
ChunkType: types.ChunkTypeImageCaption,
ParentChunkID: payload.ChunkID,
IsEnabled: true,
Flags: types.ChunkFlagRecommended,
ImageInfo: string(imageInfoJSON),
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
})
}
if len(newChunks) == 0 {
// Even if OCR/caption both failed, mark knowledge as completed
s.finalizeImageKnowledge(ctx, payload, "")
return nil
}
// Persist chunks
if err := s.chunkService.CreateChunks(ctx, newChunks); err != nil {
return fmt.Errorf("create multimodal chunks: %w", err)
}
for _, c := range newChunks {
logger.Infof(ctx, "[ImageMultimodal] Created %s chunk %s for image %s, len=%d",
c.ChunkType, c.ID, payload.ImageURL, len(c.Content))
}
// Index chunks so they can be retrieved
s.indexChunks(ctx, payload, newChunks)
// Update the parent text chunk's ImageInfo (mirrors old docreader behaviour)
s.updateParentChunkImageInfo(ctx, payload, imageInfo)
// For standalone image files, use caption as the knowledge description
// and mark the knowledge as completed (it was kept in "processing" until now).
s.finalizeImageKnowledge(ctx, payload, imageInfo.Caption)
return nil
}
// finalizeImageKnowledge updates the knowledge after multimodal processing:
// - For standalone image files: sets Description from caption and marks ParseStatus as completed
// (processChunks kept it in "processing" to wait for multimodal results).
// - For images extracted from PDFs: no-op (description comes from summary generation).
func (s *ImageMultimodalService) finalizeImageKnowledge(ctx context.Context, payload types.ImageMultimodalPayload, caption string) {
knowledge, err := s.knowledgeRepo.GetKnowledgeByIDOnly(ctx, payload.KnowledgeID)
if err != nil {
logger.Warnf(ctx, "[ImageMultimodal] Failed to get knowledge %s: %v", payload.KnowledgeID, err)
return
}
if knowledge == nil {
return
}
if !IsImageType(knowledge.FileType) {
return
}
if caption != "" {
knowledge.Description = caption
}
knowledge.ParseStatus = types.ParseStatusCompleted
knowledge.UpdatedAt = time.Now()
if err := s.knowledgeRepo.UpdateKnowledge(ctx, knowledge); err != nil {
logger.Warnf(ctx, "[ImageMultimodal] Failed to finalize knowledge: %v", err)
} else {
logger.Infof(ctx, "[ImageMultimodal] Finalized image knowledge %s (status=completed, description=%d chars)",
payload.KnowledgeID, len(knowledge.Description))
}
}
// indexChunks indexes the newly created multimodal chunks into the retrieval engine
// so they can participate in semantic search.
func (s *ImageMultimodalService) indexChunks(ctx context.Context, payload types.ImageMultimodalPayload, chunks []*types.Chunk) {
kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID)
if err != nil || kb == nil {
logger.Warnf(ctx, "[ImageMultimodal] Failed to get KB for indexing: %v", err)
return
}
embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, kb.EmbeddingModelID)
if err != nil {
logger.Warnf(ctx, "[ImageMultimodal] Failed to get embedding model for indexing: %v", err)
return
}
tenantInfo, err := s.tenantRepo.GetTenantByID(ctx, payload.TenantID)
if err != nil {
logger.Warnf(ctx, "[ImageMultimodal] Failed to get tenant for indexing: %v", err)
return
}
engine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines())
if err != nil {
logger.Warnf(ctx, "[ImageMultimodal] Failed to init retrieve engine: %v", err)
return
}
indexInfoList := make([]*types.IndexInfo, 0, len(chunks))
for _, chunk := range chunks {
indexInfoList = append(indexInfoList, &types.IndexInfo{
Content: chunk.Content,
SourceID: chunk.ID,
SourceType: types.ChunkSourceType,
ChunkID: chunk.ID,
KnowledgeID: chunk.KnowledgeID,
KnowledgeBaseID: chunk.KnowledgeBaseID,
})
}
if err := engine.BatchIndex(ctx, embeddingModel, indexInfoList); err != nil {
logger.Errorf(ctx, "[ImageMultimodal] Failed to index multimodal chunks: %v", err)
return
}
// Mark chunks as indexed.
// Must re-fetch from DB because the in-memory objects lack auto-generated fields
// (e.g. seq_id), and GORM Save would overwrite them with zero values.
for _, chunk := range chunks {
dbChunk, err := s.chunkService.GetChunkByIDOnly(ctx, chunk.ID)
if err != nil {
logger.Warnf(ctx, "[ImageMultimodal] Failed to fetch chunk %s for status update: %v", chunk.ID, err)
continue
}
dbChunk.Status = int(types.ChunkStatusIndexed)
if err := s.chunkService.UpdateChunk(ctx, dbChunk); err != nil {
logger.Warnf(ctx, "[ImageMultimodal] Failed to update chunk %s status to indexed: %v", chunk.ID, err)
}
}
logger.Infof(ctx, "[ImageMultimodal] Indexed %d multimodal chunks for image %s", len(chunks), payload.ImageURL)
}
// updateParentChunkImageInfo updates the parent text chunk's ImageInfo field,
// replicating the behaviour of the old docreader flow where the parent chunk
// carried the full image metadata (URL, OCR, caption).
func (s *ImageMultimodalService) updateParentChunkImageInfo(ctx context.Context, payload types.ImageMultimodalPayload, imageInfo types.ImageInfo) {
if payload.ChunkID == "" {
return
}
chunk, err := s.chunkService.GetChunkByIDOnly(ctx, payload.ChunkID)
if err != nil {
logger.Warnf(ctx, "[ImageMultimodal] Failed to get parent chunk %s: %v", payload.ChunkID, err)
return
}
var existingInfos []types.ImageInfo
if chunk.ImageInfo != "" {
_ = json.Unmarshal([]byte(chunk.ImageInfo), &existingInfos)
}
found := false
for i, info := range existingInfos {
if info.URL == imageInfo.URL {
existingInfos[i] = imageInfo
found = true
break
}
}
if !found {
existingInfos = append(existingInfos, imageInfo)
}
imageInfoJSON, _ := json.Marshal(existingInfos)
chunk.ImageInfo = string(imageInfoJSON)
chunk.UpdatedAt = time.Now()
if err := s.chunkService.UpdateChunk(ctx, chunk); err != nil {
logger.Warnf(ctx, "[ImageMultimodal] Failed to update parent chunk %s ImageInfo: %v", chunk.ID, err)
} else {
logger.Infof(ctx, "[ImageMultimodal] Updated parent chunk %s ImageInfo for image %s", chunk.ID, payload.ImageURL)
}
}
// resolveVLM creates a vlm.VLM instance for the given knowledge base,
// supporting both new-style (ModelID) and legacy (inline BaseURL) configs.
func (s *ImageMultimodalService) resolveVLM(ctx context.Context, kbID string) (vlm.VLM, error) {
kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, kbID)
if err != nil {
return nil, fmt.Errorf("get knowledge base %s: %w", kbID, err)
}
if kb == nil {
return nil, fmt.Errorf("knowledge base %s not found", kbID)
}
vlmCfg := kb.VLMConfig
if !vlmCfg.IsEnabled() {
return nil, fmt.Errorf("VLM is not enabled for knowledge base %s", kbID)
}
// New-style: resolve model through ModelService
if vlmCfg.ModelID != "" {
return s.modelService.GetVLMModel(ctx, vlmCfg.ModelID)
}
// Legacy: create VLM from inline config
return vlm.NewVLMFromLegacyConfig(vlmCfg, s.ollamaService)
}
// downloadImageFromURL downloads image bytes from an HTTP(S) URL.
func downloadImageFromURL(imageURL string) ([]byte, error) {
if !strings.HasPrefix(imageURL, "http://") && !strings.HasPrefix(imageURL, "https://") {
return nil, fmt.Errorf("unsupported URL scheme: %s", imageURL)
}
client := &http.Client{Timeout: 60 * time.Second}
resp, err := client.Get(imageURL)
if err != nil {
return nil, fmt.Errorf("HTTP GET: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("HTTP %d for %s", resp.StatusCode, imageURL)
}
return io.ReadAll(resp.Body)
}
File diff suppressed because it is too large Load Diff
@@ -32,7 +32,7 @@ type knowledgeBaseService struct {
tenantRepo interfaces.TenantRepository
fileSvc interfaces.FileService
graphEngine interfaces.RetrieveGraphRepository
asynqClient *asynq.Client
asynqClient interfaces.TaskEnqueuer
}
// NewKnowledgeBaseService creates a new knowledge base service
@@ -46,7 +46,7 @@ func NewKnowledgeBaseService(repo interfaces.KnowledgeBaseRepository,
tenantRepo interfaces.TenantRepository,
fileSvc interfaces.FileService,
graphEngine interfaces.RetrieveGraphRepository,
asynqClient *asynq.Client,
asynqClient interfaces.TaskEnqueuer,
) interfaces.KnowledgeBaseService {
return &knowledgeBaseService{
repo: repo,
+52
View File
@@ -9,6 +9,7 @@ import (
"github.com/Tencent/WeKnora/internal/models/embedding"
"github.com/Tencent/WeKnora/internal/models/rerank"
"github.com/Tencent/WeKnora/internal/models/utils/ollama"
"github.com/Tencent/WeKnora/internal/models/vlm"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
)
@@ -405,5 +406,56 @@ func (s *modelService) GetChatModel(ctx context.Context, modelId string) (chat.C
return chatModel, nil
}
// GetVLMModel retrieves and initializes a vision language model instance.
func (s *modelService) GetVLMModel(ctx context.Context, modelId string) (vlm.VLM, error) {
if modelId == "" {
return nil, errors.New("model ID cannot be empty")
}
tenantID := ctx.Value(types.TenantIDContextKey).(uint64)
model, err := s.repo.GetByID(ctx, tenantID, modelId)
if err != nil {
logger.ErrorWithFields(ctx, err, map[string]interface{}{
"model_id": modelId,
"tenant_id": tenantID,
})
return nil, err
}
if model == nil {
return nil, ErrModelNotFound
}
logger.Infof(ctx, "Getting VLM model: %s, source: %s", model.Name, model.Source)
ifType := model.Parameters.InterfaceType
if ifType == "" {
if model.Source == types.ModelSourceLocal {
ifType = "ollama"
} else {
ifType = "openai"
}
}
vlmModel, err := vlm.NewVLM(&vlm.Config{
ModelID: model.ID,
APIKey: model.Parameters.APIKey,
BaseURL: model.Parameters.BaseURL,
ModelName: model.Name,
Source: model.Source,
InterfaceType: ifType,
}, s.ollamaService)
if err != nil {
logger.ErrorWithFields(ctx, err, map[string]interface{}{
"model_id": model.ID,
"model_name": model.Name,
})
return nil, err
}
return vlmModel, nil
}
// Note: default model selection logic has been removed; models no longer
// maintain a per-type default flag at the service layer.
@@ -0,0 +1,105 @@
package service
import (
"regexp"
"strings"
htmltomd "github.com/JohannesKaufmann/html-to-markdown/v2"
)
var (
htmlTagPattern = regexp.MustCompile(`<[^>]+>`)
codeBlockPattern = regexp.MustCompile("(?s)^\\s*```[a-zA-Z]*\\s*\n(.*?)\n\\s*```\\s*$")
htmlDocPattern = regexp.MustCompile(`(?i)^\s*(<\!DOCTYPE|<html|<body|<div|<p[\s>]|<table|<h[1-6][\s>])`)
multipleNewlines = regexp.MustCompile(`\n{3,}`)
knownEmptyReplies = []string{
"无文字内容",
"无法识别",
"no text",
"no content",
"empty",
"图片中没有文字",
"图片中没有可识别的文字",
}
)
// sanitizeOCRText cleans up VLM OCR output by stripping HTML wrappers,
// converting HTML to markdown, and filtering out useless responses.
func sanitizeOCRText(raw string) string {
text := strings.TrimSpace(raw)
if text == "" {
return ""
}
text = stripMarkdownCodeBlock(text)
// If stripping HTML tags leaves almost no text, the response is useless
// (e.g. "<html><body><div class="image"><img/></div></body></html>").
plainText := strings.TrimSpace(htmlTagPattern.ReplaceAllString(text, ""))
if len(plainText) < 10 && htmlTagPattern.MatchString(text) {
return ""
}
if looksLikeHTML(text) {
text = ocrHTMLToMarkdown(text)
text = strings.TrimSpace(text)
if text == "" {
return ""
}
}
if isKnownEmptyReply(text) {
return ""
}
text = multipleNewlines.ReplaceAllString(text, "\n\n")
return strings.TrimSpace(text)
}
// stripMarkdownCodeBlock removes a markdown code-fence wrapper that some
// models add around their output (e.g. ```html\n...\n``` or ```markdown\n...\n```).
func stripMarkdownCodeBlock(text string) string {
if m := codeBlockPattern.FindStringSubmatch(text); len(m) == 2 {
return strings.TrimSpace(m[1])
}
return text
}
// looksLikeHTML returns true when the text appears to be an HTML document
// or contains a significant amount of HTML tags.
func looksLikeHTML(text string) bool {
if htmlDocPattern.MatchString(text) {
return true
}
tags := htmlTagPattern.FindAllString(text, -1)
if len(tags) == 0 {
return false
}
tagChars := 0
for _, t := range tags {
tagChars += len(t)
}
return float64(tagChars)/float64(len(text)) > 0.3
}
// ocrHTMLToMarkdown converts HTML content to markdown, falling back to the
// original text on failure.
func ocrHTMLToMarkdown(content string) string {
md, err := htmltomd.ConvertString(content)
if err != nil {
return content
}
return md
}
// isKnownEmptyReply checks whether the text matches a known "no content"
// reply pattern that VLM models produce when the image has no text.
func isKnownEmptyReply(text string) bool {
lower := strings.ToLower(strings.TrimSpace(text))
for _, phrase := range knownEmptyReplies {
if lower == strings.ToLower(phrase) {
return true
}
}
return false
}
@@ -0,0 +1,213 @@
package service
import "testing"
func TestSanitizeOCRText(t *testing.T) {
tests := []struct {
name string
input string
want string
}{
{
name: "empty string",
input: "",
want: "",
},
{
name: "whitespace only",
input: " \n\t ",
want: "",
},
{
name: "pure HTML skeleton with no text",
input: `<html><body><div class="image"><img/></div></body></html>`,
want: "",
},
{
name: "HTML with only whitespace text",
input: "<html><body> \n </body></html>",
want: "",
},
{
name: "valid markdown passes through",
input: "# 标题\n\n这是一段正文,包含一些内容。\n\n| 列1 | 列2 |\n| --- | --- |\n| 数据1 | 数据2 |",
want: "# 标题\n\n这是一段正文,包含一些内容。\n\n| 列1 | 列2 |\n| --- | --- |\n| 数据1 | 数据2 |",
},
{
name: "code block wrapper stripped",
input: "```markdown\n# 文档标题\n\n正文内容在这里。\n```",
want: "# 文档标题\n\n正文内容在这里。",
},
{
name: "html code block wrapper stripped",
input: "```html\n<p>这是一段内容</p>\n```",
want: "这是一段内容",
},
{
name: "HTML document converted to markdown",
input: "<html><body><h1>标题</h1><p>这是一段很长的正文内容,用来测试 HTML 到 Markdown 的转换。</p></body></html>",
want: "# 标题\n\n这是一段很长的正文内容,用来测试 HTML 到 Markdown 的转换。",
},
{
name: "known empty reply - Chinese",
input: "无文字内容",
want: "",
},
{
name: "known empty reply - no text",
input: "No text",
want: "",
},
{
name: "known empty reply - 图片中没有文字",
input: "图片中没有文字",
want: "",
},
{
name: "plain text with minimal HTML not converted",
input: "这是一段正常文本,价格 <100 元。",
want: "这是一段正常文本,价格 <100 元。",
},
{
name: "multiple blank lines collapsed",
input: "段落一\n\n\n\n\n段落二",
want: "段落一\n\n段落二",
},
{
name: "HTML with substantial text content is converted",
input: "<div><h2>报告摘要</h2><p>本季度营收同比增长 15%,净利润达到 2.3 亿元。</p><table><tr><th>指标</th><th>数值</th></tr><tr><td>营收</td><td>10亿</td></tr></table></div>",
want: "", // placeholder; will be checked for non-empty
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := sanitizeOCRText(tt.input)
if tt.name == "HTML with substantial text content is converted" {
if got == "" {
t.Errorf("sanitizeOCRText() returned empty for substantial HTML content")
}
if got == tt.input {
t.Errorf("sanitizeOCRText() did not convert HTML, got original")
}
return
}
if got != tt.want {
t.Errorf("sanitizeOCRText() = %q, want %q", got, tt.want)
}
})
}
}
func TestStripMarkdownCodeBlock(t *testing.T) {
tests := []struct {
name string
input string
want string
}{
{
name: "no code block",
input: "just normal text",
want: "just normal text",
},
{
name: "markdown code block",
input: "```markdown\n# Title\nContent here\n```",
want: "# Title\nContent here",
},
{
name: "html code block",
input: "```html\n<p>hello</p>\n```",
want: "<p>hello</p>",
},
{
name: "plain code block",
input: "```\nsome text\n```",
want: "some text",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := stripMarkdownCodeBlock(tt.input)
if got != tt.want {
t.Errorf("stripMarkdownCodeBlock() = %q, want %q", got, tt.want)
}
})
}
}
func TestLooksLikeHTML(t *testing.T) {
tests := []struct {
name string
input string
want bool
}{
{
name: "HTML document",
input: "<html><body><p>text</p></body></html>",
want: true,
},
{
name: "DOCTYPE",
input: "<!DOCTYPE html><html><body></body></html>",
want: true,
},
{
name: "body tag",
input: "<body><p>content</p></body>",
want: true,
},
{
name: "plain markdown",
input: "# Title\n\nSome paragraph text",
want: false,
},
{
name: "text with minor HTML",
input: "This is mostly text with a <b>bold</b> word.",
want: false,
},
{
name: "heavy HTML tags",
input: "<div><p><span>x</span></p></div><div><p><span>y</span></p></div>",
want: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := looksLikeHTML(tt.input)
if got != tt.want {
t.Errorf("looksLikeHTML() = %v, want %v", got, tt.want)
}
})
}
}
func TestIsKnownEmptyReply(t *testing.T) {
tests := []struct {
input string
want bool
}{
{"无文字内容", true},
{"无法识别", true},
{"no text", true},
{"No Text", true},
{"NO CONTENT", true},
{"empty", true},
{"这是正常内容", false},
{"", false},
}
for _, tt := range tests {
t.Run(tt.input, func(t *testing.T) {
got := isKnownEmptyReply(tt.input)
if got != tt.want {
t.Errorf("isKnownEmptyReply(%q) = %v, want %v", tt.input, got, tt.want)
}
})
}
}
+2 -2
View File
@@ -25,7 +25,7 @@ type knowledgeTagService struct {
chunkRepo interfaces.ChunkRepository
retrieveEngine interfaces.RetrieveEngineRegistry
modelService interfaces.ModelService
task *asynq.Client
task interfaces.TaskEnqueuer
kbShareService interfaces.KBShareService
}
@@ -37,7 +37,7 @@ func NewKnowledgeTagService(
chunkRepo interfaces.ChunkRepository,
retrieveEngine interfaces.RetrieveEngineRegistry,
modelService interfaces.ModelService,
task *asynq.Client,
task interfaces.TaskEnqueuer,
kbShareService interfaces.KBShareService,
) (interfaces.KnowledgeTagService, error) {
return &knowledgeTagService{