44d6175559
Lets users stop an in-flight document parse to free up LLM / worker
resources without losing the chunks and index already written. The
core insight is that the previous parse_status=completed flipped as
soon as primary chunks landed, while the most expensive subtasks
(graph extract = N LLM calls per chunk, plus summary, question
generation) were still running in the background — so "completed"
wasn't actually terminal from a resource standpoint.
State machine
pending -> processing -> finalizing -> completed
|
+-> cancelled (any of the three
in-flight states)
+-> failed
+-> deleting
`finalizing` is the new post-process fan-out window. parse_status
only promotes to `completed` once pending_subtasks_count (a new
column tracking summary + question + per-chunk graph extract)
drains to zero via atomic FinalizeSubtask. Wiki ingest is
intentionally excluded from the counter — it's a KB-scoped
debounced batch and would otherwise pin parse_status in
`finalizing` for the wiki batch window.
Backend
- New ParseStatusFinalizing + pending_subtasks_count column with
migration 000056.
- knowledgeRepository.SetFinalizing transitions processing -> finalizing
conditionally so a racing cancel cannot be clobbered.
- knowledgeRepository.FinalizeSubtask atomically decrements the
counter and self-promotes the row to completed when it hits zero.
- KnowledgePostProcess restructured to compute expected subtask
count up front, flip to finalizing (or completed when no
enrichment is enabled), and only then fan out subtasks. Subtask
handlers (summary, question, graph extract) defer-decrement on
terminal exit using the existing isFinalAsynqAttempt convention.
- New POST /api/v1/knowledge/{id}/cancel-parse handler accepting
pending / processing / finalizing. Marks the row cancelled,
zeroes the counter, best-effort dequeues asynq tasks via a new
TaskInspector abstraction (asynq-mode walks pending/scheduled/
retry queues; Lite-mode noop), and scrubs wiki ingest pending op.
- SpanTracker.AbortAttempt flat-sweeps every still-running span
for the attempt via a new repo.CancelAllOpenSpans helper so the
trace viewer's striped bars all flip to cancelled, even leaf
generations whose parent stage already EndSpan'd (multimodal
fan-out pattern). knowledge_post_process closes its postSpan
via SkipSpan on the cancel/deleting entry guard so a worker
that opens a span AFTER the cancel sweep doesn't leak it.
- Housekeeping and resetPendingTasks sweep finalizing rows
identically to processing so a crash/restart can't strand them.
- DeleteKnowledge/DeleteKnowledgeList proactively dequeue
downstream tasks via the same TaskInspector path.
- ChunkExtractService gets a cancel entry guard so the most
expensive enrichment (graph extract) bails immediately when the
parent knowledge is aborted.
Frontend
- New cancelKnowledgeParse API client + "Stop parsing" entry in
both list view and card view more menus, gated on
pending/processing/finalizing.
- Polling predicate refactored to a shared isParseInFlight helper
that recognises `finalizing` (previously the doc list silently
stopped polling once parse_status flipped from processing).
- Knowledge processing timeline: isPolling includes finalizing,
new isHardTerminal short-circuits LIVE for cancelled/failed/
completed so stranded child spans cannot pin LIVE on.
- DocumentListView.computeStatus distinguishes finalizing
("增强中") from completed and shows the previous "生成摘要中"
copy when summary_status is still pending under finalizing.
Added cancelled badge as well.
- i18n: statusFinalizing / statusCancelled / cancelParse* keys
across zh-CN, en-US, ko-KR, ru-RU.
Docs / SDK
- docs/api/knowledge.md: documents the new finalizing state,
cancel-parse semantics, and which statuses accept cancel.
- client (Go SDK): CancelKnowledgeParse with docstring listing
the cancellable statuses.
707 lines
22 KiB
Go
707 lines
22 KiB
Go
package repository
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/Tencent/WeKnora/internal/types"
|
|
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
|
"gorm.io/gorm"
|
|
)
|
|
|
|
var ErrKnowledgeNotFound = errors.New("knowledge not found")
|
|
|
|
// escapeLikeKeyword escapes SQL LIKE wildcards (%, _) in a keyword
|
|
// so they are treated as literal characters.
|
|
func escapeLikeKeyword(keyword string) string {
|
|
keyword = strings.ReplaceAll(keyword, `\`, `\\`)
|
|
keyword = strings.ReplaceAll(keyword, "%", `\%`)
|
|
keyword = strings.ReplaceAll(keyword, "_", `\_`)
|
|
return keyword
|
|
}
|
|
|
|
// omitFieldsOnUpdate defines fields to omit when updating knowledge
|
|
var omitFieldsOnUpdate = []string{"DeletedAt"}
|
|
|
|
// knowledgeRepository implements knowledge base and knowledge repository interface
|
|
type knowledgeRepository struct {
|
|
db *gorm.DB
|
|
}
|
|
|
|
// NewKnowledgeRepository creates a new knowledge repository
|
|
func NewKnowledgeRepository(db *gorm.DB) interfaces.KnowledgeRepository {
|
|
return &knowledgeRepository{db: db}
|
|
}
|
|
|
|
// CreateKnowledge creates knowledge
|
|
func (r *knowledgeRepository) CreateKnowledge(ctx context.Context, knowledge *types.Knowledge) error {
|
|
err := r.db.WithContext(ctx).Create(knowledge).Error
|
|
return err
|
|
}
|
|
|
|
// GetKnowledgeByID gets knowledge
|
|
func (r *knowledgeRepository) GetKnowledgeByID(
|
|
ctx context.Context,
|
|
tenantID uint64,
|
|
id string,
|
|
) (*types.Knowledge, error) {
|
|
var knowledge types.Knowledge
|
|
if err := r.db.WithContext(ctx).Where("tenant_id = ? AND id = ?", tenantID, id).First(&knowledge).Error; err != nil {
|
|
if errors.Is(err, gorm.ErrRecordNotFound) {
|
|
return nil, ErrKnowledgeNotFound
|
|
}
|
|
return nil, err
|
|
}
|
|
return &knowledge, nil
|
|
}
|
|
|
|
// GetKnowledgeByIDOnly returns knowledge by ID without tenant filter (for permission resolution).
|
|
func (r *knowledgeRepository) GetKnowledgeByIDOnly(ctx context.Context, id string) (*types.Knowledge, error) {
|
|
var knowledge types.Knowledge
|
|
if err := r.db.WithContext(ctx).Where("id = ?", id).First(&knowledge).Error; err != nil {
|
|
if errors.Is(err, gorm.ErrRecordNotFound) {
|
|
return nil, ErrKnowledgeNotFound
|
|
}
|
|
return nil, err
|
|
}
|
|
return &knowledge, nil
|
|
}
|
|
|
|
// ListKnowledgeByKnowledgeBaseID lists all knowledge in a knowledge base
|
|
func (r *knowledgeRepository) ListKnowledgeByKnowledgeBaseID(
|
|
ctx context.Context, tenantID uint64, kbID string,
|
|
) ([]*types.Knowledge, error) {
|
|
var knowledges []*types.Knowledge
|
|
if err := r.db.WithContext(ctx).Where("tenant_id = ? AND knowledge_base_id = ?", tenantID, kbID).
|
|
Order("created_at DESC").Find(&knowledges).Error; err != nil {
|
|
return nil, err
|
|
}
|
|
return knowledges, nil
|
|
}
|
|
|
|
// applyKnowledgeListFilter applies the optional filter dimensions of
|
|
// KnowledgeListFilter to a GORM query. Tenant / knowledge base scoping must be
|
|
// applied by the caller before invoking this helper.
|
|
func applyKnowledgeListFilter(query *gorm.DB, filter types.KnowledgeListFilter) *gorm.DB {
|
|
if filter.TagID != "" {
|
|
query = query.Where("tag_id = ?", filter.TagID)
|
|
}
|
|
if filter.Keyword != "" {
|
|
escaped := escapeLikeKeyword(filter.Keyword)
|
|
query = query.Where("(file_name LIKE ? OR title LIKE ?)", "%"+escaped+"%", "%"+escaped+"%")
|
|
}
|
|
// FileType and Source share the same special-case routing onto `type` for
|
|
// the "manual" / "url" values, so callers can pick either control.
|
|
applyTypeOrFileType := func(q *gorm.DB, val string) *gorm.DB {
|
|
switch val {
|
|
case "":
|
|
return q
|
|
case "manual", "url":
|
|
return q.Where("type = ?", val)
|
|
default:
|
|
return q.Where("file_type = ?", val)
|
|
}
|
|
}
|
|
query = applyTypeOrFileType(query, filter.FileType)
|
|
if filter.Source != "" {
|
|
switch filter.Source {
|
|
case "manual", "url":
|
|
query = query.Where("type = ?", filter.Source)
|
|
default:
|
|
query = query.Where("channel = ?", filter.Source)
|
|
}
|
|
}
|
|
if filter.ParseStatus != "" {
|
|
query = query.Where("parse_status = ?", filter.ParseStatus)
|
|
}
|
|
if !filter.UpdatedFrom.IsZero() {
|
|
query = query.Where("updated_at >= ?", filter.UpdatedFrom)
|
|
}
|
|
if !filter.UpdatedTo.IsZero() {
|
|
query = query.Where("updated_at <= ?", filter.UpdatedTo)
|
|
}
|
|
return query
|
|
}
|
|
|
|
// ListPagedKnowledgeByKnowledgeBaseID lists all knowledge in a knowledge base with pagination
|
|
func (r *knowledgeRepository) ListPagedKnowledgeByKnowledgeBaseID(
|
|
ctx context.Context,
|
|
tenantID uint64,
|
|
kbID string,
|
|
page *types.Pagination,
|
|
filter types.KnowledgeListFilter,
|
|
) ([]*types.Knowledge, int64, error) {
|
|
var knowledges []*types.Knowledge
|
|
var total int64
|
|
|
|
scope := func(q *gorm.DB) *gorm.DB {
|
|
return applyKnowledgeListFilter(
|
|
q.Where("tenant_id = ? AND knowledge_base_id = ?", tenantID, kbID),
|
|
filter,
|
|
)
|
|
}
|
|
|
|
if err := scope(r.db.WithContext(ctx).Model(&types.Knowledge{})).Count(&total).Error; err != nil {
|
|
return nil, 0, err
|
|
}
|
|
|
|
if err := scope(r.db.WithContext(ctx)).
|
|
Order("created_at DESC").
|
|
Offset(page.Offset()).
|
|
Limit(page.Limit()).
|
|
Find(&knowledges).Error; err != nil {
|
|
return nil, 0, err
|
|
}
|
|
|
|
return knowledges, total, nil
|
|
}
|
|
|
|
// UpdateKnowledge updates knowledge
|
|
func (r *knowledgeRepository) UpdateKnowledge(ctx context.Context, knowledge *types.Knowledge) error {
|
|
err := r.db.WithContext(ctx).Omit(omitFieldsOnUpdate...).Save(knowledge).Error
|
|
return err
|
|
}
|
|
|
|
// UpdateKnowledgeBatch updates knowledge items in batch
|
|
func (r *knowledgeRepository) UpdateKnowledgeBatch(ctx context.Context, knowledgeList []*types.Knowledge) error {
|
|
if len(knowledgeList) == 0 {
|
|
return nil
|
|
}
|
|
return r.db.Debug().WithContext(ctx).Omit(omitFieldsOnUpdate...).Save(knowledgeList).Error
|
|
}
|
|
|
|
// DeleteKnowledge deletes knowledge
|
|
func (r *knowledgeRepository) DeleteKnowledge(ctx context.Context, tenantID uint64, id string) error {
|
|
return r.db.WithContext(ctx).Where("tenant_id = ? AND id = ?", tenantID, id).Delete(&types.Knowledge{}).Error
|
|
}
|
|
|
|
// DeleteKnowledge deletes knowledge
|
|
func (r *knowledgeRepository) DeleteKnowledgeList(ctx context.Context, tenantID uint64, ids []string) error {
|
|
return r.db.WithContext(ctx).Where("tenant_id = ? AND id in ?", tenantID, ids).Delete(&types.Knowledge{}).Error
|
|
}
|
|
|
|
// GetKnowledgeBatch gets knowledge in batch
|
|
func (r *knowledgeRepository) GetKnowledgeBatch(
|
|
ctx context.Context, tenantID uint64, ids []string,
|
|
) ([]*types.Knowledge, error) {
|
|
var knowledge []*types.Knowledge
|
|
if err := r.db.WithContext(ctx).Debug().
|
|
Where("tenant_id = ? AND id IN ?", tenantID, ids).
|
|
Find(&knowledge).Error; err != nil {
|
|
return nil, err
|
|
}
|
|
return knowledge, nil
|
|
}
|
|
|
|
// CheckKnowledgeExists checks if knowledge already exists
|
|
func (r *knowledgeRepository) CheckKnowledgeExists(
|
|
ctx context.Context,
|
|
tenantID uint64,
|
|
kbID string,
|
|
params *types.KnowledgeCheckParams,
|
|
) (bool, *types.Knowledge, error) {
|
|
query := r.db.WithContext(ctx).Model(&types.Knowledge{}).
|
|
Where("tenant_id = ? AND knowledge_base_id = ? AND parse_status <> ?", tenantID, kbID, "failed")
|
|
|
|
switch params.Type {
|
|
case "file":
|
|
// If file hash exists, prioritize exact match using hash
|
|
if params.FileHash != "" {
|
|
var knowledge types.Knowledge
|
|
err := query.Where("file_hash = ?", params.FileHash).First(&knowledge).Error
|
|
if err != nil {
|
|
if errors.Is(err, gorm.ErrRecordNotFound) {
|
|
return false, nil, nil
|
|
}
|
|
return false, nil, err
|
|
}
|
|
return true, &knowledge, nil
|
|
}
|
|
|
|
// If no hash or hash doesn't match, use filename and size
|
|
if params.FileName != "" && params.FileSize > 0 {
|
|
var knowledge types.Knowledge
|
|
err := query.Where(
|
|
"file_name = ? AND file_size = ?",
|
|
params.FileName, params.FileSize,
|
|
).First(&knowledge).Error
|
|
if err != nil {
|
|
if errors.Is(err, gorm.ErrRecordNotFound) {
|
|
return false, nil, nil
|
|
}
|
|
return false, nil, err
|
|
}
|
|
return true, &knowledge, nil
|
|
}
|
|
case "url":
|
|
// If file hash exists, prioritize exact match using hash
|
|
if params.FileHash != "" {
|
|
var knowledge types.Knowledge
|
|
err := query.Where("type = 'url' AND file_hash = ?", params.FileHash).First(&knowledge).Error
|
|
if err == nil && knowledge.ID != "" {
|
|
return true, &knowledge, nil
|
|
}
|
|
if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
|
|
return false, nil, err
|
|
}
|
|
}
|
|
|
|
if params.URL != "" {
|
|
var knowledge types.Knowledge
|
|
err := query.Where("type = 'url' AND source = ?", params.URL).First(&knowledge).Error
|
|
if err == nil && knowledge.ID != "" {
|
|
return true, &knowledge, nil
|
|
}
|
|
if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
|
|
return false, nil, err
|
|
}
|
|
}
|
|
return false, nil, nil
|
|
}
|
|
|
|
// No valid parameters, default to not existing
|
|
return false, nil, nil
|
|
}
|
|
|
|
func (r *knowledgeRepository) AminusB(
|
|
ctx context.Context,
|
|
Atenant uint64, A string,
|
|
Btenant uint64, B string,
|
|
) ([]string, error) {
|
|
knowledgeIDs := []string{}
|
|
subQuery := r.db.Model(&types.Knowledge{}).
|
|
Where("tenant_id = ? AND knowledge_base_id = ?", Btenant, B).Select("file_hash")
|
|
err := r.db.Model(&types.Knowledge{}).
|
|
Where("tenant_id = ? AND knowledge_base_id = ?", Atenant, A).
|
|
Where("file_hash NOT IN (?)", subQuery).
|
|
Pluck("id", &knowledgeIDs).
|
|
Error
|
|
if errors.Is(err, gorm.ErrRecordNotFound) {
|
|
return knowledgeIDs, nil
|
|
}
|
|
return knowledgeIDs, err
|
|
}
|
|
|
|
func (r *knowledgeRepository) UpdateKnowledgeColumn(
|
|
ctx context.Context,
|
|
id string,
|
|
column string,
|
|
value interface{},
|
|
) error {
|
|
err := r.db.WithContext(ctx).Model(&types.Knowledge{}).Where("id = ?", id).Update(column, value).Error
|
|
return err
|
|
}
|
|
|
|
// UpdateKnowledgeColumns writes multiple columns in a single UPDATE so callers
|
|
// that flip related fields together (parse_status + error_message after
|
|
// dead-letter, for example) cannot leave the row half-updated when the second
|
|
// write fails.
|
|
func (r *knowledgeRepository) UpdateKnowledgeColumns(
|
|
ctx context.Context,
|
|
id string,
|
|
values map[string]interface{},
|
|
) error {
|
|
if len(values) == 0 {
|
|
return nil
|
|
}
|
|
return r.db.WithContext(ctx).Model(&types.Knowledge{}).Where("id = ?", id).Updates(values).Error
|
|
}
|
|
|
|
// FinalizeSubtask atomically decrements pending_subtasks_count and, when
|
|
// the counter reaches zero while parse_status is still 'finalizing',
|
|
// flips the row to 'completed' in the same statement so concurrent
|
|
// subtask completions can't race the promotion.
|
|
//
|
|
// Returns (newCount, promoted, error). promoted is true iff this caller
|
|
// was the one whose UPDATE flipped 'finalizing'→'completed'.
|
|
//
|
|
// The implementation is two statements (atomic decrement, then a guarded
|
|
// promote UPDATE) because GORM does not expose a portable RETURNING
|
|
// across PostgreSQL and SQLite. The promote UPDATE's WHERE clause
|
|
// (parse_status='finalizing' AND pending_subtasks_count=0) makes it
|
|
// safe to run from any number of concurrent callers — at most one wins.
|
|
func (r *knowledgeRepository) FinalizeSubtask(
|
|
ctx context.Context, id string,
|
|
) (int, bool, error) {
|
|
now := time.Now()
|
|
// 1) Atomic decrement, clamped at zero. The `pending_subtasks_count > 0`
|
|
// guard is purely a safety net for accounting bugs — under normal
|
|
// operation each subtask handler decrements at most once per task,
|
|
// so the counter cannot go negative.
|
|
res := r.db.WithContext(ctx).Model(&types.Knowledge{}).
|
|
Where("id = ? AND pending_subtasks_count > 0", id).
|
|
Updates(map[string]interface{}{
|
|
"pending_subtasks_count": gorm.Expr("pending_subtasks_count - 1"),
|
|
"updated_at": now,
|
|
})
|
|
if res.Error != nil {
|
|
return 0, false, res.Error
|
|
}
|
|
|
|
// 2) Re-read to discover the new count and current parse_status.
|
|
// Reading after the UPDATE gives us a value that is at worst one
|
|
// decrement stale relative to other callers — the promote step
|
|
// below is guarded by the same condition at SQL level, so the
|
|
// stale read only causes us to attempt a no-op promote.
|
|
var snap struct {
|
|
PendingSubtasksCount int `gorm:"column:pending_subtasks_count"`
|
|
ParseStatus string `gorm:"column:parse_status"`
|
|
}
|
|
if err := r.db.WithContext(ctx).Model(&types.Knowledge{}).
|
|
Select("pending_subtasks_count", "parse_status").
|
|
Where("id = ?", id).Take(&snap).Error; err != nil {
|
|
return 0, false, err
|
|
}
|
|
|
|
if snap.PendingSubtasksCount != 0 || snap.ParseStatus != types.ParseStatusFinalizing {
|
|
return snap.PendingSubtasksCount, false, nil
|
|
}
|
|
|
|
// 3) Guarded promote. The WHERE clause ensures only ONE caller wins
|
|
// when multiple subtasks decrement to zero in the same instant,
|
|
// and that cancel/delete cannot be clobbered by a late promote.
|
|
promoteRes := r.db.WithContext(ctx).Model(&types.Knowledge{}).
|
|
Where("id = ? AND parse_status = ? AND pending_subtasks_count = 0",
|
|
id, types.ParseStatusFinalizing).
|
|
Updates(map[string]interface{}{
|
|
"parse_status": types.ParseStatusCompleted,
|
|
"processed_at": now,
|
|
"updated_at": now,
|
|
})
|
|
if promoteRes.Error != nil {
|
|
return snap.PendingSubtasksCount, false, promoteRes.Error
|
|
}
|
|
return snap.PendingSubtasksCount, promoteRes.RowsAffected > 0, nil
|
|
}
|
|
|
|
// SetFinalizing atomically transitions a row from 'processing' to
|
|
// 'finalizing' and seeds pending_subtasks_count. Used by
|
|
// KnowledgePostProcess.Handle as the single durable handoff between
|
|
// the synchronous parse stage and the asynchronous enrichment fan-out.
|
|
//
|
|
// The transition is conditional on parse_status='processing' so a row
|
|
// that the user cancelled / deleted between ProcessDocument finishing
|
|
// and post-process starting will NOT get hijacked into finalizing.
|
|
// Returns whether the transition happened.
|
|
func (r *knowledgeRepository) SetFinalizing(
|
|
ctx context.Context, id string, expectedSubtasks int,
|
|
) (bool, error) {
|
|
if expectedSubtasks < 0 {
|
|
expectedSubtasks = 0
|
|
}
|
|
now := time.Now()
|
|
res := r.db.WithContext(ctx).Model(&types.Knowledge{}).
|
|
Where("id = ? AND parse_status = ?", id, types.ParseStatusProcessing).
|
|
Updates(map[string]interface{}{
|
|
"parse_status": types.ParseStatusFinalizing,
|
|
"pending_subtasks_count": expectedSubtasks,
|
|
"updated_at": now,
|
|
})
|
|
if res.Error != nil {
|
|
return false, res.Error
|
|
}
|
|
return res.RowsAffected > 0, nil
|
|
}
|
|
|
|
// CountKnowledgeByKnowledgeBaseID counts the number of knowledge items in a knowledge base
|
|
func (r *knowledgeRepository) CountKnowledgeByKnowledgeBaseID(
|
|
ctx context.Context,
|
|
tenantID uint64,
|
|
kbID string,
|
|
) (int64, error) {
|
|
var count int64
|
|
err := r.db.WithContext(ctx).Model(&types.Knowledge{}).
|
|
Where("tenant_id = ? AND knowledge_base_id = ?", tenantID, kbID).
|
|
Count(&count).Error
|
|
return count, err
|
|
}
|
|
|
|
// CountKnowledgeByStatus counts the number of knowledge items with the specified parse status
|
|
func (r *knowledgeRepository) CountKnowledgeByStatus(
|
|
ctx context.Context,
|
|
tenantID uint64,
|
|
kbID string,
|
|
parseStatuses []string,
|
|
) (int64, error) {
|
|
if len(parseStatuses) == 0 {
|
|
return 0, nil
|
|
}
|
|
|
|
var count int64
|
|
query := r.db.WithContext(ctx).Model(&types.Knowledge{}).
|
|
Where("tenant_id = ? AND knowledge_base_id = ?", tenantID, kbID).
|
|
Where("parse_status IN ?", parseStatuses)
|
|
|
|
if err := query.Count(&count).Error; err != nil {
|
|
return 0, err
|
|
}
|
|
|
|
return count, nil
|
|
}
|
|
|
|
// SearchKnowledge searches knowledge items by keyword across the tenant
|
|
// If keyword is empty, returns recent files
|
|
// Only returns documents from document-type knowledge bases (excludes FAQ)
|
|
// Returns (results, hasMore, error)
|
|
// FindByMetadataKey finds a knowledge item by a key-value pair in the metadata JSON column.
|
|
// Uses Postgres jsonb operator: metadata->>'key' = 'value'.
|
|
func (r *knowledgeRepository) FindByMetadataKey(
|
|
ctx context.Context,
|
|
tenantID uint64,
|
|
kbID string,
|
|
key string,
|
|
value string,
|
|
) (*types.Knowledge, error) {
|
|
var knowledge types.Knowledge
|
|
err := r.db.WithContext(ctx).
|
|
Where("tenant_id = ? AND knowledge_base_id = ? AND deleted_at IS NULL", tenantID, kbID).
|
|
Where("metadata->>? = ?", key, value).
|
|
First(&knowledge).Error
|
|
if err != nil {
|
|
if errors.Is(err, gorm.ErrRecordNotFound) {
|
|
return nil, nil
|
|
}
|
|
return nil, err
|
|
}
|
|
return &knowledge, nil
|
|
}
|
|
|
|
func (r *knowledgeRepository) SearchKnowledge(
|
|
ctx context.Context,
|
|
tenantID uint64,
|
|
keyword string,
|
|
offset, limit int,
|
|
fileTypes []string,
|
|
) ([]*types.Knowledge, bool, error) {
|
|
// Use raw query to properly map knowledge_base_name
|
|
type KnowledgeWithKBName struct {
|
|
types.Knowledge
|
|
KnowledgeBaseName string `gorm:"column:knowledge_base_name"`
|
|
}
|
|
|
|
var results []KnowledgeWithKBName
|
|
query := r.db.WithContext(ctx).
|
|
Table("knowledges").
|
|
Select("knowledges.*, knowledge_bases.name as knowledge_base_name").
|
|
Joins("JOIN knowledge_bases ON knowledge_bases.id = knowledges.knowledge_base_id").
|
|
Where("knowledges.tenant_id = ?", tenantID).
|
|
Where("knowledge_bases.type = ?", types.KnowledgeBaseTypeDocument).
|
|
Where("knowledges.deleted_at IS NULL")
|
|
|
|
// If keyword is provided, filter by file_name or title
|
|
if keyword != "" {
|
|
escaped := escapeLikeKeyword(keyword)
|
|
query = query.Where("(knowledges.file_name LIKE ? OR knowledges.title LIKE ?)", "%"+escaped+"%", "%"+escaped+"%")
|
|
}
|
|
|
|
// If fileTypes is provided, filter by file extension or type
|
|
if len(fileTypes) > 0 {
|
|
seen := make(map[string]bool)
|
|
var uniquePatterns []string
|
|
includeURL := false
|
|
for _, ft := range fileTypes {
|
|
ft = strings.ToLower(strings.TrimPrefix(ft, "."))
|
|
if ft == "url" || ft == "html" {
|
|
includeURL = true
|
|
continue
|
|
}
|
|
pattern := "%." + ft
|
|
if !seen[pattern] {
|
|
seen[pattern] = true
|
|
uniquePatterns = append(uniquePatterns, pattern)
|
|
}
|
|
// Handle common aliases
|
|
var aliases []string
|
|
switch ft {
|
|
case "xlsx":
|
|
aliases = []string{"%.xls"}
|
|
case "xls":
|
|
aliases = []string{"%.xlsx"}
|
|
case "docx":
|
|
aliases = []string{"%.doc"}
|
|
case "doc":
|
|
aliases = []string{"%.docx"}
|
|
case "jpg":
|
|
aliases = []string{"%.jpeg", "%.png"}
|
|
case "jpeg":
|
|
aliases = []string{"%.jpg", "%.png"}
|
|
case "png":
|
|
aliases = []string{"%.jpg", "%.jpeg"}
|
|
}
|
|
for _, alias := range aliases {
|
|
if !seen[alias] {
|
|
seen[alias] = true
|
|
uniquePatterns = append(uniquePatterns, alias)
|
|
}
|
|
}
|
|
}
|
|
var orConditions []string
|
|
var args []interface{}
|
|
for _, p := range uniquePatterns {
|
|
orConditions = append(orConditions, "LOWER(knowledges.file_name) LIKE ?")
|
|
args = append(args, p)
|
|
}
|
|
if includeURL {
|
|
orConditions = append(orConditions, "knowledges.type = ?")
|
|
args = append(args, "url")
|
|
}
|
|
if len(orConditions) > 0 {
|
|
query = query.Where("("+strings.Join(orConditions, " OR ")+")", args...)
|
|
}
|
|
}
|
|
|
|
// Fetch limit+1 to check if there are more results
|
|
err := query.Order("knowledges.created_at DESC").
|
|
Offset(offset).
|
|
Limit(limit + 1).
|
|
Scan(&results).Error
|
|
if err != nil {
|
|
return nil, false, err
|
|
}
|
|
|
|
// Check if there are more results
|
|
hasMore := len(results) > limit
|
|
if hasMore {
|
|
results = results[:limit]
|
|
}
|
|
|
|
// Convert to []*types.Knowledge
|
|
knowledges := make([]*types.Knowledge, len(results))
|
|
for i, r := range results {
|
|
k := r.Knowledge
|
|
k.KnowledgeBaseName = r.KnowledgeBaseName
|
|
knowledges[i] = &k
|
|
}
|
|
return knowledges, hasMore, nil
|
|
}
|
|
|
|
// SearchKnowledgeInScopes searches knowledge items by keyword within the given (tenant_id, kb_id) scopes (e.g. own + shared KBs).
|
|
func (r *knowledgeRepository) SearchKnowledgeInScopes(
|
|
ctx context.Context,
|
|
scopes []types.KnowledgeSearchScope,
|
|
keyword string,
|
|
offset, limit int,
|
|
fileTypes []string,
|
|
) ([]*types.Knowledge, bool, error) {
|
|
if len(scopes) == 0 {
|
|
return nil, false, nil
|
|
}
|
|
|
|
type KnowledgeWithKBName struct {
|
|
types.Knowledge
|
|
KnowledgeBaseName string `gorm:"column:knowledge_base_name"`
|
|
}
|
|
|
|
placeholders := make([]string, len(scopes))
|
|
args := make([]interface{}, 0, len(scopes)*2)
|
|
for i, s := range scopes {
|
|
placeholders[i] = "(?,?)"
|
|
args = append(args, s.TenantID, s.KBID)
|
|
}
|
|
scopeCondition := "(knowledges.tenant_id, knowledges.knowledge_base_id) IN (" + strings.Join(placeholders, ",") + ")"
|
|
|
|
query := r.db.WithContext(ctx).
|
|
Table("knowledges").
|
|
Select("knowledges.*, knowledge_bases.name as knowledge_base_name").
|
|
Joins("JOIN knowledge_bases ON knowledge_bases.id = knowledges.knowledge_base_id AND knowledge_bases.tenant_id = knowledges.tenant_id").
|
|
Where(scopeCondition, args...).
|
|
Where("knowledge_bases.type = ?", types.KnowledgeBaseTypeDocument).
|
|
Where("knowledges.deleted_at IS NULL")
|
|
|
|
if keyword != "" {
|
|
escaped := escapeLikeKeyword(keyword)
|
|
query = query.Where("(knowledges.file_name LIKE ? OR knowledges.title LIKE ?)", "%"+escaped+"%", "%"+escaped+"%")
|
|
}
|
|
|
|
if len(fileTypes) > 0 {
|
|
seen := make(map[string]bool)
|
|
var uniquePatterns []string
|
|
includeURL := false
|
|
for _, ft := range fileTypes {
|
|
ft = strings.ToLower(strings.TrimPrefix(ft, "."))
|
|
if ft == "url" || ft == "html" {
|
|
includeURL = true
|
|
continue
|
|
}
|
|
pattern := "%." + ft
|
|
if !seen[pattern] {
|
|
seen[pattern] = true
|
|
uniquePatterns = append(uniquePatterns, pattern)
|
|
}
|
|
var aliases []string
|
|
switch ft {
|
|
case "xlsx":
|
|
aliases = []string{"%.xls"}
|
|
case "xls":
|
|
aliases = []string{"%.xlsx"}
|
|
case "docx":
|
|
aliases = []string{"%.doc"}
|
|
case "doc":
|
|
aliases = []string{"%.docx"}
|
|
case "jpg":
|
|
aliases = []string{"%.jpeg", "%.png"}
|
|
case "jpeg":
|
|
aliases = []string{"%.jpg", "%.png"}
|
|
case "png":
|
|
aliases = []string{"%.jpg", "%.jpeg"}
|
|
}
|
|
for _, alias := range aliases {
|
|
if !seen[alias] {
|
|
seen[alias] = true
|
|
uniquePatterns = append(uniquePatterns, alias)
|
|
}
|
|
}
|
|
}
|
|
var orConditions []string
|
|
var ftArgs []interface{}
|
|
for _, p := range uniquePatterns {
|
|
orConditions = append(orConditions, "LOWER(knowledges.file_name) LIKE ?")
|
|
ftArgs = append(ftArgs, p)
|
|
}
|
|
if includeURL {
|
|
orConditions = append(orConditions, "knowledges.type = ?")
|
|
ftArgs = append(ftArgs, "url")
|
|
}
|
|
if len(orConditions) > 0 {
|
|
query = query.Where("("+strings.Join(orConditions, " OR ")+")", ftArgs...)
|
|
}
|
|
}
|
|
|
|
var results []KnowledgeWithKBName
|
|
err := query.Order("knowledges.created_at DESC").
|
|
Offset(offset).
|
|
Limit(limit + 1).
|
|
Scan(&results).Error
|
|
if err != nil {
|
|
return nil, false, err
|
|
}
|
|
|
|
hasMore := len(results) > limit
|
|
if hasMore {
|
|
results = results[:limit]
|
|
}
|
|
|
|
knowledges := make([]*types.Knowledge, len(results))
|
|
for i, r := range results {
|
|
k := r.Knowledge
|
|
k.KnowledgeBaseName = r.KnowledgeBaseName
|
|
knowledges[i] = &k
|
|
}
|
|
return knowledges, hasMore, nil
|
|
}
|
|
|
|
// ListIDsByTagID returns all knowledge IDs that have the specified tag ID
|
|
func (r *knowledgeRepository) ListIDsByTagID(
|
|
ctx context.Context,
|
|
tenantID uint64,
|
|
kbID, tagID string,
|
|
) ([]string, error) {
|
|
var ids []string
|
|
err := r.db.WithContext(ctx).Model(&types.Knowledge{}).
|
|
Where("tenant_id = ? AND knowledge_base_id = ? AND tag_id = ?", tenantID, kbID, tagID).
|
|
Pluck("id", &ids).Error
|
|
return ids, err
|
|
}
|