diff --git a/client/README.md b/client/README.md index 1426d98e..e4119aa7 100644 --- a/client/README.md +++ b/client/README.md @@ -282,6 +282,43 @@ if err != nil { } ``` +### 示例:重新解析知识 + +```go +// 重新解析知识(删除现有内容并重新解析) +// 适用场景: +// 1. 原始解析失败,需要重试 +// 2. 更新了解析配置(如分块策略、多模态设置等),需要重新解析 +// 3. 知识内容已更新,需要刷新解析结果 + +knowledge, err := apiClient.ReparseKnowledge(context.Background(), knowledgeID) +if err != nil { + // 处理错误 +} + +// 知识将进入 "pending" 状态,异步重新解析 +fmt.Printf("Knowledge ID: %s\n", knowledge.ID) +fmt.Printf("Parse Status: %s\n", knowledge.ParseStatus) // "pending" +fmt.Printf("Enable Status: %s\n", knowledge.EnableStatus) // "disabled" + +// 可以轮询检查解析状态 +for { + time.Sleep(5 * time.Second) + knowledge, err := apiClient.GetKnowledge(context.Background(), knowledgeID) + if err != nil { + // 处理错误 + } + + if knowledge.ParseStatus == "completed" { + fmt.Println("Knowledge re-parsing completed!") + break + } else if knowledge.ParseStatus == "failed" { + fmt.Printf("Knowledge re-parsing failed: %s\n", knowledge.ErrorMessage) + break + } +} +``` + ### 示例:获取会话消息 ```go diff --git a/client/knowledge.go b/client/knowledge.go index 4828736b..466f12d5 100644 --- a/client/knowledge.go +++ b/client/knowledge.go @@ -356,6 +356,45 @@ func (c *Client) UpdateKnowledge(ctx context.Context, knowledge *Knowledge) erro return parseResponse(resp, &response) } +// ReparseKnowledge triggers re-parsing of a knowledge entry +// This method deletes existing document content and re-parses the knowledge asynchronously. +// It's useful when you want to refresh the knowledge content with updated parsing configurations +// or when the original parsing failed and you want to retry. +// +// Parameters: +// - ctx: Context for the request +// - knowledgeID: The ID of the knowledge entry to reparse +// +// Returns: +// - *Knowledge: The updated knowledge entry with status set to "pending" +// - error: Error information if the request fails +// +// Example: +// +// knowledge, err := client.ReparseKnowledge(ctx, "knowledge-id-123") +// if err != nil { +// log.Fatalf("Failed to reparse knowledge: %v", err) +// } +// fmt.Printf("Knowledge reparse task submitted, status: %s\n", knowledge.ParseStatus) +func (c *Client) ReparseKnowledge(ctx context.Context, knowledgeID string) (*Knowledge, error) { + if knowledgeID == "" { + return nil, fmt.Errorf("knowledge ID cannot be empty") + } + + path := fmt.Sprintf("/api/v1/knowledge/%s/reparse", knowledgeID) + resp, err := c.doRequest(ctx, http.MethodPost, path, nil, nil) + if err != nil { + return nil, err + } + + var response KnowledgeResponse + if err := parseResponse(resp, &response); err != nil { + return nil, err + } + + return &response.Data, nil +} + // UpdateChunk updates a chunk's information // Updates information for a specific chunk under a knowledge document // Parameters: diff --git a/docker/Dockerfile.app b/docker/Dockerfile.app index 1cb219ea..4ff37a81 100644 --- a/docker/Dockerfile.app +++ b/docker/Dockerfile.app @@ -27,6 +27,8 @@ RUN go install -tags 'postgres' github.com/golang-migrate/migrate/v4/cmd/migrate # Copy go mod and sum files COPY go.mod go.sum ./ RUN --mount=type=cache,target=/go/pkg/mod go mod download +COPY cmd/download cmd/download +RUN go run cmd/download/duckdb/duckdb.go COPY . . # Get version and commit info for build injection @@ -42,7 +44,6 @@ ENV BUILD_TIME=${BUILD_TIME_ARG} ENV GO_VERSION=${GO_VERSION_ARG} # Build the application with version info -RUN --mount=type=cache,target=/go/pkg/mod make download_spatial RUN --mount=type=cache,target=/go/pkg/mod make build-prod RUN --mount=type=cache,target=/go/pkg/mod cp -r /go/pkg/mod/github.com/yanyiwu/ /app/yanyiwu/ diff --git a/internal/application/service/knowledge.go b/internal/application/service/knowledge.go index c6bb4421..19ae275e 100644 --- a/internal/application/service/knowledge.go +++ b/internal/application/service/knowledge.go @@ -2199,6 +2199,158 @@ func (s *knowledgeService) UpdateManualKnowledge(ctx context.Context, return existing, nil } +// ReparseKnowledge deletes existing document content and re-parses the knowledge asynchronously. +// This method reuses the logic from UpdateManualKnowledge for resource cleanup and async parsing. +func (s *knowledgeService) ReparseKnowledge(ctx context.Context, knowledgeID string) (*types.Knowledge, error) { + logger.Info(ctx, "Start re-parsing knowledge") + + tenantID := ctx.Value(types.TenantIDContextKey).(uint64) + existing, err := s.repo.GetKnowledgeByID(ctx, tenantID, knowledgeID) + if err != nil { + logger.Errorf(ctx, "Failed to load knowledge: %v", err) + return nil, err + } + + // Get knowledge base configuration + kb, err := s.kbService.GetKnowledgeBaseByID(ctx, existing.KnowledgeBaseID) + if err != nil { + logger.Errorf(ctx, "Failed to get knowledge base for reparse: %v", err) + return nil, err + } + + // Step 1: Clean up existing resources (chunks, embeddings, graph data) + logger.Infof(ctx, "Cleaning up existing resources for knowledge: %s", knowledgeID) + if err := s.cleanupKnowledgeResources(ctx, existing); err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "knowledge_id": knowledgeID, + }) + return nil, err + } + + // Step 2: Update knowledge status and metadata + existing.ParseStatus = "pending" + existing.EnableStatus = "disabled" + existing.Description = "" + existing.ProcessedAt = nil + existing.EmbeddingModelID = kb.EmbeddingModelID + + if err := s.repo.UpdateKnowledge(ctx, existing); err != nil { + logger.Errorf(ctx, "Failed to update knowledge status before reparse: %v", err) + return nil, err + } + + // Step 3: Trigger async re-parsing based on knowledge type + logger.Infof(ctx, "Knowledge status updated, scheduling async reparse, ID: %s, Type: %s", existing.ID, existing.Type) + + // For manual knowledge, extract content from metadata and trigger manual processing + if existing.IsManual() { + meta, err := existing.ManualMetadata() + if err != nil || meta == nil { + logger.Errorf(ctx, "Failed to get manual metadata for reparse: %v", err) + return nil, werrors.NewBadRequestError("无法获取手工知识内容") + } + s.triggerManualProcessing(ctx, kb, existing, meta.Content, false) + return existing, nil + } + + // For file-based knowledge, enqueue document processing task + if existing.FilePath != "" { + tenantID := ctx.Value(types.TenantIDContextKey).(uint64) + + // Determine multimodal setting + enableMultimodel := kb.IsMultimodalEnabled() + + // Check question generation config + enableQuestionGeneration := false + questionCount := 3 // default + if kb.QuestionGenerationConfig != nil && kb.QuestionGenerationConfig.Enabled { + enableQuestionGeneration = true + if kb.QuestionGenerationConfig.QuestionCount > 0 { + questionCount = kb.QuestionGenerationConfig.QuestionCount + } + } + + taskPayload := types.DocumentProcessPayload{ + TenantID: tenantID, + KnowledgeID: existing.ID, + KnowledgeBaseID: existing.KnowledgeBaseID, + FilePath: existing.FilePath, + FileName: existing.FileName, + FileType: getFileType(existing.FileName), + EnableMultimodel: enableMultimodel, + EnableQuestionGeneration: enableQuestionGeneration, + QuestionCount: questionCount, + } + + payloadBytes, err := json.Marshal(taskPayload) + if err != nil { + logger.Errorf(ctx, "Failed to marshal reparse task payload: %v", err) + return existing, nil + } + + task := asynq.NewTask(types.TypeDocumentProcess, payloadBytes, asynq.Queue("default")) + info, err := s.task.Enqueue(task) + if err != nil { + logger.Errorf(ctx, "Failed to enqueue reparse task: %v", err) + return existing, nil + } + logger.Infof(ctx, "Enqueued reparse task: id=%s queue=%s knowledge_id=%s", info.ID, info.Queue, existing.ID) + + // For data tables (csv, xlsx, xls), also enqueue summary task + if slices.Contains([]string{"csv", "xlsx", "xls"}, getFileType(existing.FileName)) { + NewDataTableSummaryTask(ctx, s.task, tenantID, existing.ID, kb.SummaryModelID, kb.EmbeddingModelID) + } + + return existing, nil + } + + // For URL-based knowledge, enqueue URL processing task + if existing.Type == "url" && existing.Source != "" { + tenantID := ctx.Value(types.TenantIDContextKey).(uint64) + + enableMultimodel := kb.IsMultimodalEnabled() + + // Check question generation config + enableQuestionGeneration := false + questionCount := 3 + if kb.QuestionGenerationConfig != nil && kb.QuestionGenerationConfig.Enabled { + enableQuestionGeneration = true + if kb.QuestionGenerationConfig.QuestionCount > 0 { + questionCount = kb.QuestionGenerationConfig.QuestionCount + } + } + + taskPayload := types.DocumentProcessPayload{ + TenantID: tenantID, + KnowledgeID: existing.ID, + KnowledgeBaseID: existing.KnowledgeBaseID, + URL: existing.Source, + EnableMultimodel: enableMultimodel, + EnableQuestionGeneration: enableQuestionGeneration, + QuestionCount: questionCount, + } + + payloadBytes, err := json.Marshal(taskPayload) + if err != nil { + logger.Errorf(ctx, "Failed to marshal URL reparse task payload: %v", err) + return existing, nil + } + + task := asynq.NewTask(types.TypeDocumentProcess, payloadBytes, asynq.Queue("default")) + info, err := s.task.Enqueue(task) + if err != nil { + logger.Errorf(ctx, "Failed to enqueue URL reparse task: %v", err) + return existing, nil + } + logger.Infof(ctx, "Enqueued URL reparse task: id=%s queue=%s knowledge_id=%s", info.ID, info.Queue, existing.ID) + + return existing, nil + } + + logger.Warnf(ctx, "Knowledge %s has no parseable content (no file, URL, or manual content)", knowledgeID) + return existing, nil +} + // isValidFileType checks if a file type is supported func isValidFileType(filename string) bool { switch strings.ToLower(getFileType(filename)) { @@ -3079,8 +3231,8 @@ func csvEscape(s string) string { // saveFAQImportResultToDatabase 保存FAQ导入结果统计到数据库 func (s *knowledgeService) saveFAQImportResultToDatabase(ctx context.Context, - payload *types.FAQImportPayload, progress *types.FAQImportProgress, originalTotalEntries int) error { - + payload *types.FAQImportPayload, progress *types.FAQImportProgress, originalTotalEntries int, +) error { // 获取FAQ知识库实例 tenantID := ctx.Value(types.TenantIDContextKey).(uint64) knowledge, err := s.repo.GetKnowledgeByID(ctx, tenantID, payload.KnowledgeID) @@ -6864,7 +7016,8 @@ func (s *knowledgeService) ProcessFAQImport(ctx context.Context, t *asynq.Task) // finalizeFAQValidation 完成 FAQ 验证/导入任务,生成失败条目 CSV(如果有) func (s *knowledgeService) finalizeFAQValidation(ctx context.Context, payload *types.FAQImportPayload, - progress *types.FAQImportProgress, originalTotalEntries int) error { + progress *types.FAQImportProgress, originalTotalEntries int, +) error { // 清理对象存储中的 entries 文件(如果有) if payload.EntriesURL != "" { if err := s.fileSvc.DeleteFile(ctx, payload.EntriesURL); err != nil { diff --git a/internal/handler/knowledge.go b/internal/handler/knowledge.go index 9e4d1c9e..91b305ed 100644 --- a/internal/handler/knowledge.go +++ b/internal/handler/knowledge.go @@ -9,10 +9,11 @@ import ( "strconv" "strings" + goerrors "errors" + "github.com/Tencent/WeKnora/internal/application/service" "github.com/Tencent/WeKnora/internal/errors" "github.com/Tencent/WeKnora/internal/logger" - goerrors "errors" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" secutils "github.com/Tencent/WeKnora/internal/utils" @@ -21,9 +22,9 @@ import ( // KnowledgeHandler processes HTTP requests related to knowledge resources type KnowledgeHandler struct { - kgService interfaces.KnowledgeService - kbService interfaces.KnowledgeBaseService - kbShareService interfaces.KBShareService + kgService interfaces.KnowledgeService + kbService interfaces.KnowledgeBaseService + kbShareService interfaces.KBShareService agentShareService interfaces.AgentShareService } @@ -35,9 +36,9 @@ func NewKnowledgeHandler( agentShareService interfaces.AgentShareService, ) *KnowledgeHandler { return &KnowledgeHandler{ - kgService: kgService, - kbService: kbService, - kbShareService: kbShareService, + kgService: kgService, + kbService: kbService, + kbShareService: kbShareService, agentShareService: agentShareService, } } @@ -938,6 +939,59 @@ func (h *KnowledgeHandler) UpdateManualKnowledge(c *gin.Context) { }) } +// ReparseKnowledge godoc +// @Summary 重新解析知识 +// @Description 删除知识中现有的文档内容并重新解析,使用异步任务方式处理 +// @Tags 知识管理 +// @Accept json +// @Produce json +// @Param id path string true "知识ID" +// @Success 200 {object} map[string]interface{} "重新解析任务已提交" +// @Failure 400 {object} errors.AppError "请求参数错误" +// @Failure 403 {object} errors.AppError "权限不足" +// @Security Bearer +// @Security ApiKeyAuth +// @Router /knowledge/{id}/reparse [post] +func (h *KnowledgeHandler) ReparseKnowledge(c *gin.Context) { + ctx := c.Request.Context() + logger.Info(ctx, "Start re-parsing knowledge") + + id := secutils.SanitizeForLog(c.Param("id")) + if id == "" { + logger.Error(ctx, "Knowledge ID is empty") + c.Error(errors.NewBadRequestError("Knowledge ID cannot be empty")) + return + } + + // Validate KB access with editor permission (reparse requires write access) + _, effCtx, err := h.resolveKnowledgeAndValidateKBAccess(c, id, types.OrgRoleEditor) + if err != nil { + c.Error(err) + return + } + + // Call service to reparse knowledge + knowledge, err := h.kgService.ReparseKnowledge(effCtx, id) + if err != nil { + if appErr, ok := errors.IsAppError(err); ok { + c.Error(appErr) + return + } + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "knowledge_id": id, + }) + c.Error(errors.NewInternalServerError(err.Error())) + return + } + + logger.Infof(ctx, "Knowledge reparse task submitted successfully, knowledge ID: %s", id) + c.JSON(http.StatusOK, gin.H{ + "success": true, + "message": "Knowledge reparse task submitted", + "data": knowledge, + }) +} + type knowledgeTagBatchRequest struct { Updates map[string]*string `json:"updates" binding:"required,min=1"` KBID string `json:"kb_id"` // Optional: scope to this KB (validates editor access and uses effective tenant for shared KB) diff --git a/internal/router/router.go b/internal/router/router.go index fd5acd62..8d533635 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -169,6 +169,8 @@ func RegisterKnowledgeRoutes(r *gin.RouterGroup, handler *handler.KnowledgeHandl k.PUT("/:id", handler.UpdateKnowledge) // 更新手工 Markdown 知识 k.PUT("/manual/:id", handler.UpdateManualKnowledge) + // 重新解析知识 + k.POST("/:id/reparse", handler.ReparseKnowledge) // 获取知识文件 k.GET("/:id/download", handler.DownloadKnowledgeFile) // 更新图像分块信息 diff --git a/internal/types/interfaces/knowledge.go b/internal/types/interfaces/knowledge.go index 3b4809c9..18d64c92 100644 --- a/internal/types/interfaces/knowledge.go +++ b/internal/types/interfaces/knowledge.go @@ -78,6 +78,8 @@ type KnowledgeService interface { knowledgeID string, payload *types.ManualKnowledgePayload, ) (*types.Knowledge, error) + // ReparseKnowledge deletes existing document content and re-parses the knowledge asynchronously. + ReparseKnowledge(ctx context.Context, knowledgeID string) (*types.Knowledge, error) // CloneKnowledgeBase clones knowledge to another knowledge base. CloneKnowledgeBase(ctx context.Context, srcID, dstID string) error // UpdateImageInfo updates image information for a knowledge chunk.