feat: Add async knowledge re-parse API and examples

This commit is contained in:
begoniezhao
2026-02-10 12:15:21 +08:00
parent 10b6c98078
commit bce968833e
7 changed files with 299 additions and 11 deletions
+37
View File
@@ -282,6 +282,43 @@ if err != nil {
}
```
### 示例:重新解析知识
```go
// 重新解析知识(删除现有内容并重新解析)
// 适用场景:
// 1. 原始解析失败,需要重试
// 2. 更新了解析配置(如分块策略、多模态设置等),需要重新解析
// 3. 知识内容已更新,需要刷新解析结果
knowledge, err := apiClient.ReparseKnowledge(context.Background(), knowledgeID)
if err != nil {
// 处理错误
}
// 知识将进入 "pending" 状态,异步重新解析
fmt.Printf("Knowledge ID: %s\n", knowledge.ID)
fmt.Printf("Parse Status: %s\n", knowledge.ParseStatus) // "pending"
fmt.Printf("Enable Status: %s\n", knowledge.EnableStatus) // "disabled"
// 可以轮询检查解析状态
for {
time.Sleep(5 * time.Second)
knowledge, err := apiClient.GetKnowledge(context.Background(), knowledgeID)
if err != nil {
// 处理错误
}
if knowledge.ParseStatus == "completed" {
fmt.Println("Knowledge re-parsing completed!")
break
} else if knowledge.ParseStatus == "failed" {
fmt.Printf("Knowledge re-parsing failed: %s\n", knowledge.ErrorMessage)
break
}
}
```
### 示例:获取会话消息
```go
+39
View File
@@ -356,6 +356,45 @@ func (c *Client) UpdateKnowledge(ctx context.Context, knowledge *Knowledge) erro
return parseResponse(resp, &response)
}
// ReparseKnowledge triggers re-parsing of a knowledge entry
// This method deletes existing document content and re-parses the knowledge asynchronously.
// It's useful when you want to refresh the knowledge content with updated parsing configurations
// or when the original parsing failed and you want to retry.
//
// Parameters:
// - ctx: Context for the request
// - knowledgeID: The ID of the knowledge entry to reparse
//
// Returns:
// - *Knowledge: The updated knowledge entry with status set to "pending"
// - error: Error information if the request fails
//
// Example:
//
// knowledge, err := client.ReparseKnowledge(ctx, "knowledge-id-123")
// if err != nil {
// log.Fatalf("Failed to reparse knowledge: %v", err)
// }
// fmt.Printf("Knowledge reparse task submitted, status: %s\n", knowledge.ParseStatus)
func (c *Client) ReparseKnowledge(ctx context.Context, knowledgeID string) (*Knowledge, error) {
if knowledgeID == "" {
return nil, fmt.Errorf("knowledge ID cannot be empty")
}
path := fmt.Sprintf("/api/v1/knowledge/%s/reparse", knowledgeID)
resp, err := c.doRequest(ctx, http.MethodPost, path, nil, nil)
if err != nil {
return nil, err
}
var response KnowledgeResponse
if err := parseResponse(resp, &response); err != nil {
return nil, err
}
return &response.Data, nil
}
// UpdateChunk updates a chunk's information
// Updates information for a specific chunk under a knowledge document
// Parameters:
+2 -1
View File
@@ -27,6 +27,8 @@ RUN go install -tags 'postgres' github.com/golang-migrate/migrate/v4/cmd/migrate
# Copy go mod and sum files
COPY go.mod go.sum ./
RUN --mount=type=cache,target=/go/pkg/mod go mod download
COPY cmd/download cmd/download
RUN go run cmd/download/duckdb/duckdb.go
COPY . .
# Get version and commit info for build injection
@@ -42,7 +44,6 @@ ENV BUILD_TIME=${BUILD_TIME_ARG}
ENV GO_VERSION=${GO_VERSION_ARG}
# Build the application with version info
RUN --mount=type=cache,target=/go/pkg/mod make download_spatial
RUN --mount=type=cache,target=/go/pkg/mod make build-prod
RUN --mount=type=cache,target=/go/pkg/mod cp -r /go/pkg/mod/github.com/yanyiwu/ /app/yanyiwu/
+156 -3
View File
@@ -2199,6 +2199,158 @@ func (s *knowledgeService) UpdateManualKnowledge(ctx context.Context,
return existing, nil
}
// ReparseKnowledge deletes existing document content and re-parses the knowledge asynchronously.
// This method reuses the logic from UpdateManualKnowledge for resource cleanup and async parsing.
func (s *knowledgeService) ReparseKnowledge(ctx context.Context, knowledgeID string) (*types.Knowledge, error) {
logger.Info(ctx, "Start re-parsing knowledge")
tenantID := ctx.Value(types.TenantIDContextKey).(uint64)
existing, err := s.repo.GetKnowledgeByID(ctx, tenantID, knowledgeID)
if err != nil {
logger.Errorf(ctx, "Failed to load knowledge: %v", err)
return nil, err
}
// Get knowledge base configuration
kb, err := s.kbService.GetKnowledgeBaseByID(ctx, existing.KnowledgeBaseID)
if err != nil {
logger.Errorf(ctx, "Failed to get knowledge base for reparse: %v", err)
return nil, err
}
// Step 1: Clean up existing resources (chunks, embeddings, graph data)
logger.Infof(ctx, "Cleaning up existing resources for knowledge: %s", knowledgeID)
if err := s.cleanupKnowledgeResources(ctx, existing); err != nil {
logger.ErrorWithFields(ctx, err, map[string]interface{}{
"knowledge_id": knowledgeID,
})
return nil, err
}
// Step 2: Update knowledge status and metadata
existing.ParseStatus = "pending"
existing.EnableStatus = "disabled"
existing.Description = ""
existing.ProcessedAt = nil
existing.EmbeddingModelID = kb.EmbeddingModelID
if err := s.repo.UpdateKnowledge(ctx, existing); err != nil {
logger.Errorf(ctx, "Failed to update knowledge status before reparse: %v", err)
return nil, err
}
// Step 3: Trigger async re-parsing based on knowledge type
logger.Infof(ctx, "Knowledge status updated, scheduling async reparse, ID: %s, Type: %s", existing.ID, existing.Type)
// For manual knowledge, extract content from metadata and trigger manual processing
if existing.IsManual() {
meta, err := existing.ManualMetadata()
if err != nil || meta == nil {
logger.Errorf(ctx, "Failed to get manual metadata for reparse: %v", err)
return nil, werrors.NewBadRequestError("无法获取手工知识内容")
}
s.triggerManualProcessing(ctx, kb, existing, meta.Content, false)
return existing, nil
}
// For file-based knowledge, enqueue document processing task
if existing.FilePath != "" {
tenantID := ctx.Value(types.TenantIDContextKey).(uint64)
// Determine multimodal setting
enableMultimodel := kb.IsMultimodalEnabled()
// Check question generation config
enableQuestionGeneration := false
questionCount := 3 // default
if kb.QuestionGenerationConfig != nil && kb.QuestionGenerationConfig.Enabled {
enableQuestionGeneration = true
if kb.QuestionGenerationConfig.QuestionCount > 0 {
questionCount = kb.QuestionGenerationConfig.QuestionCount
}
}
taskPayload := types.DocumentProcessPayload{
TenantID: tenantID,
KnowledgeID: existing.ID,
KnowledgeBaseID: existing.KnowledgeBaseID,
FilePath: existing.FilePath,
FileName: existing.FileName,
FileType: getFileType(existing.FileName),
EnableMultimodel: enableMultimodel,
EnableQuestionGeneration: enableQuestionGeneration,
QuestionCount: questionCount,
}
payloadBytes, err := json.Marshal(taskPayload)
if err != nil {
logger.Errorf(ctx, "Failed to marshal reparse task payload: %v", err)
return existing, nil
}
task := asynq.NewTask(types.TypeDocumentProcess, payloadBytes, asynq.Queue("default"))
info, err := s.task.Enqueue(task)
if err != nil {
logger.Errorf(ctx, "Failed to enqueue reparse task: %v", err)
return existing, nil
}
logger.Infof(ctx, "Enqueued reparse task: id=%s queue=%s knowledge_id=%s", info.ID, info.Queue, existing.ID)
// For data tables (csv, xlsx, xls), also enqueue summary task
if slices.Contains([]string{"csv", "xlsx", "xls"}, getFileType(existing.FileName)) {
NewDataTableSummaryTask(ctx, s.task, tenantID, existing.ID, kb.SummaryModelID, kb.EmbeddingModelID)
}
return existing, nil
}
// For URL-based knowledge, enqueue URL processing task
if existing.Type == "url" && existing.Source != "" {
tenantID := ctx.Value(types.TenantIDContextKey).(uint64)
enableMultimodel := kb.IsMultimodalEnabled()
// Check question generation config
enableQuestionGeneration := false
questionCount := 3
if kb.QuestionGenerationConfig != nil && kb.QuestionGenerationConfig.Enabled {
enableQuestionGeneration = true
if kb.QuestionGenerationConfig.QuestionCount > 0 {
questionCount = kb.QuestionGenerationConfig.QuestionCount
}
}
taskPayload := types.DocumentProcessPayload{
TenantID: tenantID,
KnowledgeID: existing.ID,
KnowledgeBaseID: existing.KnowledgeBaseID,
URL: existing.Source,
EnableMultimodel: enableMultimodel,
EnableQuestionGeneration: enableQuestionGeneration,
QuestionCount: questionCount,
}
payloadBytes, err := json.Marshal(taskPayload)
if err != nil {
logger.Errorf(ctx, "Failed to marshal URL reparse task payload: %v", err)
return existing, nil
}
task := asynq.NewTask(types.TypeDocumentProcess, payloadBytes, asynq.Queue("default"))
info, err := s.task.Enqueue(task)
if err != nil {
logger.Errorf(ctx, "Failed to enqueue URL reparse task: %v", err)
return existing, nil
}
logger.Infof(ctx, "Enqueued URL reparse task: id=%s queue=%s knowledge_id=%s", info.ID, info.Queue, existing.ID)
return existing, nil
}
logger.Warnf(ctx, "Knowledge %s has no parseable content (no file, URL, or manual content)", knowledgeID)
return existing, nil
}
// isValidFileType checks if a file type is supported
func isValidFileType(filename string) bool {
switch strings.ToLower(getFileType(filename)) {
@@ -3079,8 +3231,8 @@ func csvEscape(s string) string {
// saveFAQImportResultToDatabase 保存FAQ导入结果统计到数据库
func (s *knowledgeService) saveFAQImportResultToDatabase(ctx context.Context,
payload *types.FAQImportPayload, progress *types.FAQImportProgress, originalTotalEntries int) error {
payload *types.FAQImportPayload, progress *types.FAQImportProgress, originalTotalEntries int,
) error {
// 获取FAQ知识库实例
tenantID := ctx.Value(types.TenantIDContextKey).(uint64)
knowledge, err := s.repo.GetKnowledgeByID(ctx, tenantID, payload.KnowledgeID)
@@ -6864,7 +7016,8 @@ func (s *knowledgeService) ProcessFAQImport(ctx context.Context, t *asynq.Task)
// finalizeFAQValidation 完成 FAQ 验证/导入任务,生成失败条目 CSV(如果有)
func (s *knowledgeService) finalizeFAQValidation(ctx context.Context, payload *types.FAQImportPayload,
progress *types.FAQImportProgress, originalTotalEntries int) error {
progress *types.FAQImportProgress, originalTotalEntries int,
) error {
// 清理对象存储中的 entries 文件(如果有)
if payload.EntriesURL != "" {
if err := s.fileSvc.DeleteFile(ctx, payload.EntriesURL); err != nil {
+61 -7
View File
@@ -9,10 +9,11 @@ import (
"strconv"
"strings"
goerrors "errors"
"github.com/Tencent/WeKnora/internal/application/service"
"github.com/Tencent/WeKnora/internal/errors"
"github.com/Tencent/WeKnora/internal/logger"
goerrors "errors"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
secutils "github.com/Tencent/WeKnora/internal/utils"
@@ -21,9 +22,9 @@ import (
// KnowledgeHandler processes HTTP requests related to knowledge resources
type KnowledgeHandler struct {
kgService interfaces.KnowledgeService
kbService interfaces.KnowledgeBaseService
kbShareService interfaces.KBShareService
kgService interfaces.KnowledgeService
kbService interfaces.KnowledgeBaseService
kbShareService interfaces.KBShareService
agentShareService interfaces.AgentShareService
}
@@ -35,9 +36,9 @@ func NewKnowledgeHandler(
agentShareService interfaces.AgentShareService,
) *KnowledgeHandler {
return &KnowledgeHandler{
kgService: kgService,
kbService: kbService,
kbShareService: kbShareService,
kgService: kgService,
kbService: kbService,
kbShareService: kbShareService,
agentShareService: agentShareService,
}
}
@@ -938,6 +939,59 @@ func (h *KnowledgeHandler) UpdateManualKnowledge(c *gin.Context) {
})
}
// ReparseKnowledge godoc
// @Summary 重新解析知识
// @Description 删除知识中现有的文档内容并重新解析,使用异步任务方式处理
// @Tags 知识管理
// @Accept json
// @Produce json
// @Param id path string true "知识ID"
// @Success 200 {object} map[string]interface{} "重新解析任务已提交"
// @Failure 400 {object} errors.AppError "请求参数错误"
// @Failure 403 {object} errors.AppError "权限不足"
// @Security Bearer
// @Security ApiKeyAuth
// @Router /knowledge/{id}/reparse [post]
func (h *KnowledgeHandler) ReparseKnowledge(c *gin.Context) {
ctx := c.Request.Context()
logger.Info(ctx, "Start re-parsing knowledge")
id := secutils.SanitizeForLog(c.Param("id"))
if id == "" {
logger.Error(ctx, "Knowledge ID is empty")
c.Error(errors.NewBadRequestError("Knowledge ID cannot be empty"))
return
}
// Validate KB access with editor permission (reparse requires write access)
_, effCtx, err := h.resolveKnowledgeAndValidateKBAccess(c, id, types.OrgRoleEditor)
if err != nil {
c.Error(err)
return
}
// Call service to reparse knowledge
knowledge, err := h.kgService.ReparseKnowledge(effCtx, id)
if err != nil {
if appErr, ok := errors.IsAppError(err); ok {
c.Error(appErr)
return
}
logger.ErrorWithFields(ctx, err, map[string]interface{}{
"knowledge_id": id,
})
c.Error(errors.NewInternalServerError(err.Error()))
return
}
logger.Infof(ctx, "Knowledge reparse task submitted successfully, knowledge ID: %s", id)
c.JSON(http.StatusOK, gin.H{
"success": true,
"message": "Knowledge reparse task submitted",
"data": knowledge,
})
}
type knowledgeTagBatchRequest struct {
Updates map[string]*string `json:"updates" binding:"required,min=1"`
KBID string `json:"kb_id"` // Optional: scope to this KB (validates editor access and uses effective tenant for shared KB)
+2
View File
@@ -169,6 +169,8 @@ func RegisterKnowledgeRoutes(r *gin.RouterGroup, handler *handler.KnowledgeHandl
k.PUT("/:id", handler.UpdateKnowledge)
// 更新手工 Markdown 知识
k.PUT("/manual/:id", handler.UpdateManualKnowledge)
// 重新解析知识
k.POST("/:id/reparse", handler.ReparseKnowledge)
// 获取知识文件
k.GET("/:id/download", handler.DownloadKnowledgeFile)
// 更新图像分块信息
+2
View File
@@ -78,6 +78,8 @@ type KnowledgeService interface {
knowledgeID string,
payload *types.ManualKnowledgePayload,
) (*types.Knowledge, error)
// ReparseKnowledge deletes existing document content and re-parses the knowledge asynchronously.
ReparseKnowledge(ctx context.Context, knowledgeID string) (*types.Knowledge, error)
// CloneKnowledgeBase clones knowledge to another knowledge base.
CloneKnowledgeBase(ctx context.Context, srcID, dstID string) error
// UpdateImageInfo updates image information for a knowledge chunk.