feat: Add async knowledge re-parse API and examples
This commit is contained in:
@@ -282,6 +282,43 @@ if err != nil {
|
||||
}
|
||||
```
|
||||
|
||||
### 示例:重新解析知识
|
||||
|
||||
```go
|
||||
// 重新解析知识(删除现有内容并重新解析)
|
||||
// 适用场景:
|
||||
// 1. 原始解析失败,需要重试
|
||||
// 2. 更新了解析配置(如分块策略、多模态设置等),需要重新解析
|
||||
// 3. 知识内容已更新,需要刷新解析结果
|
||||
|
||||
knowledge, err := apiClient.ReparseKnowledge(context.Background(), knowledgeID)
|
||||
if err != nil {
|
||||
// 处理错误
|
||||
}
|
||||
|
||||
// 知识将进入 "pending" 状态,异步重新解析
|
||||
fmt.Printf("Knowledge ID: %s\n", knowledge.ID)
|
||||
fmt.Printf("Parse Status: %s\n", knowledge.ParseStatus) // "pending"
|
||||
fmt.Printf("Enable Status: %s\n", knowledge.EnableStatus) // "disabled"
|
||||
|
||||
// 可以轮询检查解析状态
|
||||
for {
|
||||
time.Sleep(5 * time.Second)
|
||||
knowledge, err := apiClient.GetKnowledge(context.Background(), knowledgeID)
|
||||
if err != nil {
|
||||
// 处理错误
|
||||
}
|
||||
|
||||
if knowledge.ParseStatus == "completed" {
|
||||
fmt.Println("Knowledge re-parsing completed!")
|
||||
break
|
||||
} else if knowledge.ParseStatus == "failed" {
|
||||
fmt.Printf("Knowledge re-parsing failed: %s\n", knowledge.ErrorMessage)
|
||||
break
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 示例:获取会话消息
|
||||
|
||||
```go
|
||||
|
||||
@@ -356,6 +356,45 @@ func (c *Client) UpdateKnowledge(ctx context.Context, knowledge *Knowledge) erro
|
||||
return parseResponse(resp, &response)
|
||||
}
|
||||
|
||||
// ReparseKnowledge triggers re-parsing of a knowledge entry
|
||||
// This method deletes existing document content and re-parses the knowledge asynchronously.
|
||||
// It's useful when you want to refresh the knowledge content with updated parsing configurations
|
||||
// or when the original parsing failed and you want to retry.
|
||||
//
|
||||
// Parameters:
|
||||
// - ctx: Context for the request
|
||||
// - knowledgeID: The ID of the knowledge entry to reparse
|
||||
//
|
||||
// Returns:
|
||||
// - *Knowledge: The updated knowledge entry with status set to "pending"
|
||||
// - error: Error information if the request fails
|
||||
//
|
||||
// Example:
|
||||
//
|
||||
// knowledge, err := client.ReparseKnowledge(ctx, "knowledge-id-123")
|
||||
// if err != nil {
|
||||
// log.Fatalf("Failed to reparse knowledge: %v", err)
|
||||
// }
|
||||
// fmt.Printf("Knowledge reparse task submitted, status: %s\n", knowledge.ParseStatus)
|
||||
func (c *Client) ReparseKnowledge(ctx context.Context, knowledgeID string) (*Knowledge, error) {
|
||||
if knowledgeID == "" {
|
||||
return nil, fmt.Errorf("knowledge ID cannot be empty")
|
||||
}
|
||||
|
||||
path := fmt.Sprintf("/api/v1/knowledge/%s/reparse", knowledgeID)
|
||||
resp, err := c.doRequest(ctx, http.MethodPost, path, nil, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var response KnowledgeResponse
|
||||
if err := parseResponse(resp, &response); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &response.Data, nil
|
||||
}
|
||||
|
||||
// UpdateChunk updates a chunk's information
|
||||
// Updates information for a specific chunk under a knowledge document
|
||||
// Parameters:
|
||||
|
||||
@@ -27,6 +27,8 @@ RUN go install -tags 'postgres' github.com/golang-migrate/migrate/v4/cmd/migrate
|
||||
# Copy go mod and sum files
|
||||
COPY go.mod go.sum ./
|
||||
RUN --mount=type=cache,target=/go/pkg/mod go mod download
|
||||
COPY cmd/download cmd/download
|
||||
RUN go run cmd/download/duckdb/duckdb.go
|
||||
COPY . .
|
||||
|
||||
# Get version and commit info for build injection
|
||||
@@ -42,7 +44,6 @@ ENV BUILD_TIME=${BUILD_TIME_ARG}
|
||||
ENV GO_VERSION=${GO_VERSION_ARG}
|
||||
|
||||
# Build the application with version info
|
||||
RUN --mount=type=cache,target=/go/pkg/mod make download_spatial
|
||||
RUN --mount=type=cache,target=/go/pkg/mod make build-prod
|
||||
RUN --mount=type=cache,target=/go/pkg/mod cp -r /go/pkg/mod/github.com/yanyiwu/ /app/yanyiwu/
|
||||
|
||||
|
||||
@@ -2199,6 +2199,158 @@ func (s *knowledgeService) UpdateManualKnowledge(ctx context.Context,
|
||||
return existing, nil
|
||||
}
|
||||
|
||||
// ReparseKnowledge deletes existing document content and re-parses the knowledge asynchronously.
|
||||
// This method reuses the logic from UpdateManualKnowledge for resource cleanup and async parsing.
|
||||
func (s *knowledgeService) ReparseKnowledge(ctx context.Context, knowledgeID string) (*types.Knowledge, error) {
|
||||
logger.Info(ctx, "Start re-parsing knowledge")
|
||||
|
||||
tenantID := ctx.Value(types.TenantIDContextKey).(uint64)
|
||||
existing, err := s.repo.GetKnowledgeByID(ctx, tenantID, knowledgeID)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to load knowledge: %v", err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Get knowledge base configuration
|
||||
kb, err := s.kbService.GetKnowledgeBaseByID(ctx, existing.KnowledgeBaseID)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to get knowledge base for reparse: %v", err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Step 1: Clean up existing resources (chunks, embeddings, graph data)
|
||||
logger.Infof(ctx, "Cleaning up existing resources for knowledge: %s", knowledgeID)
|
||||
if err := s.cleanupKnowledgeResources(ctx, existing); err != nil {
|
||||
logger.ErrorWithFields(ctx, err, map[string]interface{}{
|
||||
"knowledge_id": knowledgeID,
|
||||
})
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Step 2: Update knowledge status and metadata
|
||||
existing.ParseStatus = "pending"
|
||||
existing.EnableStatus = "disabled"
|
||||
existing.Description = ""
|
||||
existing.ProcessedAt = nil
|
||||
existing.EmbeddingModelID = kb.EmbeddingModelID
|
||||
|
||||
if err := s.repo.UpdateKnowledge(ctx, existing); err != nil {
|
||||
logger.Errorf(ctx, "Failed to update knowledge status before reparse: %v", err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Step 3: Trigger async re-parsing based on knowledge type
|
||||
logger.Infof(ctx, "Knowledge status updated, scheduling async reparse, ID: %s, Type: %s", existing.ID, existing.Type)
|
||||
|
||||
// For manual knowledge, extract content from metadata and trigger manual processing
|
||||
if existing.IsManual() {
|
||||
meta, err := existing.ManualMetadata()
|
||||
if err != nil || meta == nil {
|
||||
logger.Errorf(ctx, "Failed to get manual metadata for reparse: %v", err)
|
||||
return nil, werrors.NewBadRequestError("无法获取手工知识内容")
|
||||
}
|
||||
s.triggerManualProcessing(ctx, kb, existing, meta.Content, false)
|
||||
return existing, nil
|
||||
}
|
||||
|
||||
// For file-based knowledge, enqueue document processing task
|
||||
if existing.FilePath != "" {
|
||||
tenantID := ctx.Value(types.TenantIDContextKey).(uint64)
|
||||
|
||||
// Determine multimodal setting
|
||||
enableMultimodel := kb.IsMultimodalEnabled()
|
||||
|
||||
// Check question generation config
|
||||
enableQuestionGeneration := false
|
||||
questionCount := 3 // default
|
||||
if kb.QuestionGenerationConfig != nil && kb.QuestionGenerationConfig.Enabled {
|
||||
enableQuestionGeneration = true
|
||||
if kb.QuestionGenerationConfig.QuestionCount > 0 {
|
||||
questionCount = kb.QuestionGenerationConfig.QuestionCount
|
||||
}
|
||||
}
|
||||
|
||||
taskPayload := types.DocumentProcessPayload{
|
||||
TenantID: tenantID,
|
||||
KnowledgeID: existing.ID,
|
||||
KnowledgeBaseID: existing.KnowledgeBaseID,
|
||||
FilePath: existing.FilePath,
|
||||
FileName: existing.FileName,
|
||||
FileType: getFileType(existing.FileName),
|
||||
EnableMultimodel: enableMultimodel,
|
||||
EnableQuestionGeneration: enableQuestionGeneration,
|
||||
QuestionCount: questionCount,
|
||||
}
|
||||
|
||||
payloadBytes, err := json.Marshal(taskPayload)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to marshal reparse task payload: %v", err)
|
||||
return existing, nil
|
||||
}
|
||||
|
||||
task := asynq.NewTask(types.TypeDocumentProcess, payloadBytes, asynq.Queue("default"))
|
||||
info, err := s.task.Enqueue(task)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to enqueue reparse task: %v", err)
|
||||
return existing, nil
|
||||
}
|
||||
logger.Infof(ctx, "Enqueued reparse task: id=%s queue=%s knowledge_id=%s", info.ID, info.Queue, existing.ID)
|
||||
|
||||
// For data tables (csv, xlsx, xls), also enqueue summary task
|
||||
if slices.Contains([]string{"csv", "xlsx", "xls"}, getFileType(existing.FileName)) {
|
||||
NewDataTableSummaryTask(ctx, s.task, tenantID, existing.ID, kb.SummaryModelID, kb.EmbeddingModelID)
|
||||
}
|
||||
|
||||
return existing, nil
|
||||
}
|
||||
|
||||
// For URL-based knowledge, enqueue URL processing task
|
||||
if existing.Type == "url" && existing.Source != "" {
|
||||
tenantID := ctx.Value(types.TenantIDContextKey).(uint64)
|
||||
|
||||
enableMultimodel := kb.IsMultimodalEnabled()
|
||||
|
||||
// Check question generation config
|
||||
enableQuestionGeneration := false
|
||||
questionCount := 3
|
||||
if kb.QuestionGenerationConfig != nil && kb.QuestionGenerationConfig.Enabled {
|
||||
enableQuestionGeneration = true
|
||||
if kb.QuestionGenerationConfig.QuestionCount > 0 {
|
||||
questionCount = kb.QuestionGenerationConfig.QuestionCount
|
||||
}
|
||||
}
|
||||
|
||||
taskPayload := types.DocumentProcessPayload{
|
||||
TenantID: tenantID,
|
||||
KnowledgeID: existing.ID,
|
||||
KnowledgeBaseID: existing.KnowledgeBaseID,
|
||||
URL: existing.Source,
|
||||
EnableMultimodel: enableMultimodel,
|
||||
EnableQuestionGeneration: enableQuestionGeneration,
|
||||
QuestionCount: questionCount,
|
||||
}
|
||||
|
||||
payloadBytes, err := json.Marshal(taskPayload)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to marshal URL reparse task payload: %v", err)
|
||||
return existing, nil
|
||||
}
|
||||
|
||||
task := asynq.NewTask(types.TypeDocumentProcess, payloadBytes, asynq.Queue("default"))
|
||||
info, err := s.task.Enqueue(task)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to enqueue URL reparse task: %v", err)
|
||||
return existing, nil
|
||||
}
|
||||
logger.Infof(ctx, "Enqueued URL reparse task: id=%s queue=%s knowledge_id=%s", info.ID, info.Queue, existing.ID)
|
||||
|
||||
return existing, nil
|
||||
}
|
||||
|
||||
logger.Warnf(ctx, "Knowledge %s has no parseable content (no file, URL, or manual content)", knowledgeID)
|
||||
return existing, nil
|
||||
}
|
||||
|
||||
// isValidFileType checks if a file type is supported
|
||||
func isValidFileType(filename string) bool {
|
||||
switch strings.ToLower(getFileType(filename)) {
|
||||
@@ -3079,8 +3231,8 @@ func csvEscape(s string) string {
|
||||
|
||||
// saveFAQImportResultToDatabase 保存FAQ导入结果统计到数据库
|
||||
func (s *knowledgeService) saveFAQImportResultToDatabase(ctx context.Context,
|
||||
payload *types.FAQImportPayload, progress *types.FAQImportProgress, originalTotalEntries int) error {
|
||||
|
||||
payload *types.FAQImportPayload, progress *types.FAQImportProgress, originalTotalEntries int,
|
||||
) error {
|
||||
// 获取FAQ知识库实例
|
||||
tenantID := ctx.Value(types.TenantIDContextKey).(uint64)
|
||||
knowledge, err := s.repo.GetKnowledgeByID(ctx, tenantID, payload.KnowledgeID)
|
||||
@@ -6864,7 +7016,8 @@ func (s *knowledgeService) ProcessFAQImport(ctx context.Context, t *asynq.Task)
|
||||
|
||||
// finalizeFAQValidation 完成 FAQ 验证/导入任务,生成失败条目 CSV(如果有)
|
||||
func (s *knowledgeService) finalizeFAQValidation(ctx context.Context, payload *types.FAQImportPayload,
|
||||
progress *types.FAQImportProgress, originalTotalEntries int) error {
|
||||
progress *types.FAQImportProgress, originalTotalEntries int,
|
||||
) error {
|
||||
// 清理对象存储中的 entries 文件(如果有)
|
||||
if payload.EntriesURL != "" {
|
||||
if err := s.fileSvc.DeleteFile(ctx, payload.EntriesURL); err != nil {
|
||||
|
||||
@@ -9,10 +9,11 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
goerrors "errors"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/application/service"
|
||||
"github.com/Tencent/WeKnora/internal/errors"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
goerrors "errors"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
||||
@@ -21,9 +22,9 @@ import (
|
||||
|
||||
// KnowledgeHandler processes HTTP requests related to knowledge resources
|
||||
type KnowledgeHandler struct {
|
||||
kgService interfaces.KnowledgeService
|
||||
kbService interfaces.KnowledgeBaseService
|
||||
kbShareService interfaces.KBShareService
|
||||
kgService interfaces.KnowledgeService
|
||||
kbService interfaces.KnowledgeBaseService
|
||||
kbShareService interfaces.KBShareService
|
||||
agentShareService interfaces.AgentShareService
|
||||
}
|
||||
|
||||
@@ -35,9 +36,9 @@ func NewKnowledgeHandler(
|
||||
agentShareService interfaces.AgentShareService,
|
||||
) *KnowledgeHandler {
|
||||
return &KnowledgeHandler{
|
||||
kgService: kgService,
|
||||
kbService: kbService,
|
||||
kbShareService: kbShareService,
|
||||
kgService: kgService,
|
||||
kbService: kbService,
|
||||
kbShareService: kbShareService,
|
||||
agentShareService: agentShareService,
|
||||
}
|
||||
}
|
||||
@@ -938,6 +939,59 @@ func (h *KnowledgeHandler) UpdateManualKnowledge(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
// ReparseKnowledge godoc
|
||||
// @Summary 重新解析知识
|
||||
// @Description 删除知识中现有的文档内容并重新解析,使用异步任务方式处理
|
||||
// @Tags 知识管理
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Param id path string true "知识ID"
|
||||
// @Success 200 {object} map[string]interface{} "重新解析任务已提交"
|
||||
// @Failure 400 {object} errors.AppError "请求参数错误"
|
||||
// @Failure 403 {object} errors.AppError "权限不足"
|
||||
// @Security Bearer
|
||||
// @Security ApiKeyAuth
|
||||
// @Router /knowledge/{id}/reparse [post]
|
||||
func (h *KnowledgeHandler) ReparseKnowledge(c *gin.Context) {
|
||||
ctx := c.Request.Context()
|
||||
logger.Info(ctx, "Start re-parsing knowledge")
|
||||
|
||||
id := secutils.SanitizeForLog(c.Param("id"))
|
||||
if id == "" {
|
||||
logger.Error(ctx, "Knowledge ID is empty")
|
||||
c.Error(errors.NewBadRequestError("Knowledge ID cannot be empty"))
|
||||
return
|
||||
}
|
||||
|
||||
// Validate KB access with editor permission (reparse requires write access)
|
||||
_, effCtx, err := h.resolveKnowledgeAndValidateKBAccess(c, id, types.OrgRoleEditor)
|
||||
if err != nil {
|
||||
c.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
// Call service to reparse knowledge
|
||||
knowledge, err := h.kgService.ReparseKnowledge(effCtx, id)
|
||||
if err != nil {
|
||||
if appErr, ok := errors.IsAppError(err); ok {
|
||||
c.Error(appErr)
|
||||
return
|
||||
}
|
||||
logger.ErrorWithFields(ctx, err, map[string]interface{}{
|
||||
"knowledge_id": id,
|
||||
})
|
||||
c.Error(errors.NewInternalServerError(err.Error()))
|
||||
return
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "Knowledge reparse task submitted successfully, knowledge ID: %s", id)
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"success": true,
|
||||
"message": "Knowledge reparse task submitted",
|
||||
"data": knowledge,
|
||||
})
|
||||
}
|
||||
|
||||
type knowledgeTagBatchRequest struct {
|
||||
Updates map[string]*string `json:"updates" binding:"required,min=1"`
|
||||
KBID string `json:"kb_id"` // Optional: scope to this KB (validates editor access and uses effective tenant for shared KB)
|
||||
|
||||
@@ -169,6 +169,8 @@ func RegisterKnowledgeRoutes(r *gin.RouterGroup, handler *handler.KnowledgeHandl
|
||||
k.PUT("/:id", handler.UpdateKnowledge)
|
||||
// 更新手工 Markdown 知识
|
||||
k.PUT("/manual/:id", handler.UpdateManualKnowledge)
|
||||
// 重新解析知识
|
||||
k.POST("/:id/reparse", handler.ReparseKnowledge)
|
||||
// 获取知识文件
|
||||
k.GET("/:id/download", handler.DownloadKnowledgeFile)
|
||||
// 更新图像分块信息
|
||||
|
||||
@@ -78,6 +78,8 @@ type KnowledgeService interface {
|
||||
knowledgeID string,
|
||||
payload *types.ManualKnowledgePayload,
|
||||
) (*types.Knowledge, error)
|
||||
// ReparseKnowledge deletes existing document content and re-parses the knowledge asynchronously.
|
||||
ReparseKnowledge(ctx context.Context, knowledgeID string) (*types.Knowledge, error)
|
||||
// CloneKnowledgeBase clones knowledge to another knowledge base.
|
||||
CloneKnowledgeBase(ctx context.Context, srcID, dstID string) error
|
||||
// UpdateImageInfo updates image information for a knowledge chunk.
|
||||
|
||||
Reference in New Issue
Block a user