feat(weknoracloud): implement credential management and status checks

- Introduced a new WeKnoraCloudSettings component for managing APPID and APPSECRET.
- Added functionality to save WeKnoraCloud credentials without automatically creating models.
- Implemented credential status checks across various components to provide user feedback on configuration status.
- Updated API endpoints to handle credential saving and status retrieval.
- Enhanced UI elements to guide users in configuring WeKnoraCloud settings.

This update improves the integration of WeKnoraCloud by allowing users to manage their credentials more effectively and receive real-time feedback on their configuration status.
This commit is contained in:
wizardchen
2026-04-14 17:13:16 +08:00
committed by lyingbug
parent 0ce9dc056a
commit 0eff63b308
26 changed files with 808 additions and 541 deletions
@@ -3,7 +3,6 @@ package docparser
import (
"strings"
"github.com/Tencent/WeKnora/internal/models/provider"
"github.com/Tencent/WeKnora/internal/types"
)
@@ -87,13 +86,13 @@ type weKnoraCloudEngine struct{}
func (e *weKnoraCloudEngine) Name() string { return WeKnoraCloudEngineName }
func (e *weKnoraCloudEngine) Description() string { return "WeKnoraCloud document reader" }
func (e *weKnoraCloudEngine) FileTypes(_ bool) []string {
return weKnoraCloudSupportedFileTypes()
return []string{"docx", "doc", "pdf", "md", "markdown", "xlsx", "xls", "pptx", "ppt"}
}
func (e *weKnoraCloudEngine) CheckAvailable(docreaderConnected bool, overrides map[string]string) (bool, string) {
if docreaderConnected && isWeKnoraCloudDocReaderAddr(overrides["docreader_addr"]) {
if overrides["docreader_app_id"] != "" {
return true, ""
}
return false, "WeKnoraCloud document reader is not initialized"
return false, "WeKnora Cloud credentials not configured. Go to Settings → WeKnora Cloud to set up."
}
// ---------------------------------------------------------------------------
@@ -192,7 +191,3 @@ func ListAllEngines(docreaderConnected bool, overrides map[string]string, remote
return result
}
func isWeKnoraCloudDocReaderAddr(addr string) bool {
return strings.TrimSuffix(strings.TrimSpace(addr), "/") == strings.TrimRight(provider.WeKnoraCloudBaseURL, "/")+"/api/v1/doc/reader"
}
@@ -3,21 +3,19 @@ package docparser
import (
"context"
"fmt"
"log"
"os"
"strconv"
"sync"
"time"
"github.com/Tencent/WeKnora/docreader/proto"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
"google.golang.org/grpc/resolver"
)
var logger = log.New(os.Stdout, "[DocParser] ", log.LstdFlags|log.Lmicroseconds)
func getMaxMessageSize() int {
if sizeStr := os.Getenv("MAX_FILE_SIZE_MB"); sizeStr != "" {
if size, err := strconv.Atoi(sizeStr); err == nil && size > 0 {
@@ -46,7 +44,6 @@ func NewGRPCDocumentReader(addr string) (*GRPCDocumentReader, error) {
}
func (p *GRPCDocumentReader) connect(addr string) error {
logger.Printf("INFO: Connecting to docreader at %s", addr)
maxMsgSize := getMaxMessageSize()
opts := []grpc.DialOption{
@@ -64,7 +61,7 @@ func (p *GRPCDocumentReader) connect(addr string) error {
if err != nil {
return fmt.Errorf("failed to connect to docreader: %w", err)
}
logger.Printf("INFO: Connected to docreader in %v", time.Since(start))
logger.Infof(context.Background(), "Connected to docreader in %v", time.Since(start))
p.conn = conn
p.client = proto.NewDocReaderClient(conn)
+15 -13
View File
@@ -6,6 +6,8 @@ import (
"sort"
"strings"
"time"
"github.com/Tencent/WeKnora/internal/logger"
)
// stringOr returns val (trimmed) if non-empty, otherwise fallback.
@@ -56,7 +58,7 @@ func logResponseStructure(label string, obj interface{}, prefix string) {
keys = append(keys, k)
}
sort.Strings(keys)
logger.Printf("DEBUG: [%s] %s = {object with %d keys: %s}", label, prefix, len(v), strings.Join(keys, ", "))
logger.Infof(context.Background(), "[%s] %s = {object with %d keys: %s}", label, prefix, len(v), strings.Join(keys, ", "))
for _, key := range keys {
val := v[key]
path := prefix + "." + key
@@ -64,36 +66,36 @@ func logResponseStructure(label string, obj interface{}, prefix string) {
case map[string]interface{}:
logResponseStructure(label, inner, path)
case []interface{}:
logger.Printf("DEBUG: [%s] %s = [array with %d items]", label, path, len(inner))
logger.Infof(context.Background(), "[%s] %s = [array with %d items]", label, path, len(inner))
if len(inner) > 0 {
logger.Printf("DEBUG: [%s] %s[0] type=%T", label, path, inner[0])
logger.Infof(context.Background(), "[%s] %s[0] type=%T", label, path, inner[0])
if len(inner) <= 3 {
for i, item := range inner {
logResponseStructure(label, item, fmt.Sprintf("%s[%d]", path, i))
}
} else {
logResponseStructure(label, inner[0], path+"[0]")
logger.Printf("DEBUG: [%s] ... and %d more items in %s", label, len(inner)-1, path)
logger.Infof(context.Background(), "[%s] ... and %d more items in %s", label, len(inner)-1, path)
}
}
case string:
if len(inner) > 200 {
logger.Printf("DEBUG: [%s] %s = string(%d chars): %.200s...", label, path, len(inner), inner)
logger.Infof(context.Background(), "[%s] %s = string(%d chars): %.200s...", label, path, len(inner), inner)
} else {
logger.Printf("DEBUG: [%s] %s = %q", label, path, inner)
logger.Infof(context.Background(), "[%s] %s = %q", label, path, inner)
}
case float64:
logger.Printf("DEBUG: [%s] %s = %v (number)", label, path, inner)
logger.Infof(context.Background(), "[%s] %s = %v (number)", label, path, inner)
case bool:
logger.Printf("DEBUG: [%s] %s = %v (bool)", label, path, inner)
logger.Infof(context.Background(), "[%s] %s = %v (bool)", label, path, inner)
case nil:
logger.Printf("DEBUG: [%s] %s = null", label, path)
logger.Infof(context.Background(), "[%s] %s = null", label, path)
default:
logger.Printf("DEBUG: [%s] %s = %v (%T)", label, path, val, val)
logger.Infof(context.Background(), "[%s] %s = %v (%T)", label, path, val, val)
}
}
case []interface{}:
logger.Printf("DEBUG: [%s] %s = [array with %d items]", label, prefix, len(v))
logger.Infof(context.Background(), "[%s] %s = [array with %d items]", label, prefix, len(v))
if len(v) > 0 {
if len(v) <= 3 {
for i, item := range v {
@@ -101,10 +103,10 @@ func logResponseStructure(label string, obj interface{}, prefix string) {
}
} else {
logResponseStructure(label, v[0], prefix+"[0]")
logger.Printf("DEBUG: [%s] ... and %d more items in %s", label, len(v)-1, prefix)
logger.Infof(context.Background(), "[%s] ... and %d more items in %s", label, len(v)-1, prefix)
}
}
default:
logger.Printf("DEBUG: [%s] %s = %v (%T)", label, prefix, v, v)
logger.Infof(context.Background(), "[%s] %s = %v (%T)", label, prefix, v, v)
}
}
@@ -12,6 +12,7 @@ import (
"sync"
"time"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
)
@@ -73,7 +74,7 @@ func NewHTTPDocumentReader(baseURL string) (*HTTPDocumentReader, error) {
},
}
if p.baseURL != "" {
logger.Printf("INFO: HTTP docreader base URL: %s", p.baseURL)
logger.Infof(context.Background(), "INFO: HTTP docreader base URL: %s", p.baseURL)
}
return p, nil
}
@@ -88,7 +89,7 @@ func (p *HTTPDocumentReader) Reconnect(addr string) error {
p.mu.Lock()
defer p.mu.Unlock()
p.baseURL = strings.TrimSuffix(addr, "/")
logger.Printf("INFO: HTTP docreader base URL set to %s", p.baseURL)
logger.Infof(context.Background(), "INFO: HTTP docreader base URL set to %s", p.baseURL)
return nil
}
@@ -15,6 +15,7 @@ import (
"strings"
"time"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/utils"
"github.com/google/uuid"
@@ -61,7 +62,7 @@ func (c *MinerUCloudReader) Read(ctx context.Context, req *types.ReadRequest) (*
return &types.ReadResult{Error: "no file content provided"}, nil
}
logger.Printf("INFO: [MinerUCloud] Parsing file=%s size=%d via %s", req.FileName, len(content), c.baseURL)
logger.Infof(context.Background(), "[MinerUCloud] Parsing file=%s size=%d via %s", req.FileName, len(content), c.baseURL)
ext := filepath.Ext(req.FileName)
if ext == "" && req.FileType != "" {
@@ -158,7 +159,7 @@ func (c *MinerUCloudReader) applyUploadURLs(ctx context.Context, fileName, ext s
return "", "", fmt.Errorf("API returned no file_urls")
}
logger.Printf("INFO: [MinerUCloud] batch apply ok: batch_id=%s, urls=%d", result.Data.BatchID, len(result.Data.FileURLs))
logger.Infof(context.Background(), "[MinerUCloud] batch apply ok: batch_id=%s, urls=%d", result.Data.BatchID, len(result.Data.FileURLs))
return result.Data.BatchID, result.Data.FileURLs[0], nil
}
@@ -178,7 +179,7 @@ func (c *MinerUCloudReader) uploadFile(ctx context.Context, uploadURL string, co
if resp.StatusCode >= 300 {
return fmt.Errorf("PUT upload status %d", resp.StatusCode)
}
logger.Printf("INFO: [MinerUCloud] file uploaded, status=%d", resp.StatusCode)
logger.Infof(context.Background(), "[MinerUCloud] file uploaded, status=%d", resp.StatusCode)
return nil
}
@@ -218,14 +219,14 @@ func (c *MinerUCloudReader) pollBatchResult(ctx context.Context, batchID string)
items, err := c.fetchBatchStatus(ctx, batchID, headers)
if err != nil {
logger.Printf("WARN: [MinerUCloud] poll #%d failed: %v", pollCount, err)
logger.Errorf(context.Background(), "[MinerUCloud] poll #%d failed: %v", pollCount, err)
sleepCtx(ctx, defaultPollInterval)
continue
}
if len(items) == 0 {
if pollCount <= 3 || pollCount%10 == 0 {
logger.Printf("INFO: [MinerUCloud] poll #%d: extract_result empty, retrying", pollCount)
logger.Infof(context.Background(), "[MinerUCloud] poll #%d: extract_result empty, retrying", pollCount)
}
sleepCtx(ctx, defaultPollInterval)
continue
@@ -235,7 +236,7 @@ func (c *MinerUCloudReader) pollBatchResult(ctx context.Context, batchID string)
state := strings.ToLower(item.State)
if pollCount == 1 || pollCount%10 == 0 || state == "done" || state == "failed" {
logger.Printf("INFO: [MinerUCloud] poll #%d: file=%s state=%s pages=%d/%d",
logger.Infof(context.Background(), "[MinerUCloud] poll #%d: file=%s state=%s pages=%d/%d",
pollCount, item.FileName, state, item.Progress.ExtractedPages, item.Progress.TotalPages)
}
@@ -290,9 +291,9 @@ func (c *MinerUCloudReader) fetchBatchStatus(ctx context.Context, batchID string
// Dump the raw extract_result JSON for debugging
rawExtract := string(pollResp.Data.ExtractResult)
if len(rawExtract) > 4000 {
logger.Printf("DEBUG: [MinerUCloud] Raw extract_result (truncated to 4000 chars): %s ...", rawExtract[:4000])
logger.Infof(context.Background(), "[MinerUCloud] Raw extract_result (truncated to 4000 chars): %s ...", rawExtract[:4000])
} else {
logger.Printf("DEBUG: [MinerUCloud] Raw extract_result: %s", rawExtract)
logger.Infof(context.Background(), "[MinerUCloud] Raw extract_result: %s", rawExtract)
}
// Pretty-print the structure to reveal all available fields
@@ -323,7 +324,7 @@ func (c *MinerUCloudReader) fetchBatchStatus(ctx context.Context, batchID string
func (c *MinerUCloudReader) extractDoneResult(_ context.Context, item *extractResultItem) (string, []types.ImageRef, error) {
text := firstNonEmpty(item.Markdown, item.Content, item.Text)
if text != "" {
logger.Printf("INFO: [MinerUCloud] parsed (inline), length=%d", len(text))
logger.Infof(context.Background(), "[MinerUCloud] parsed (inline), length=%d", len(text))
return text, nil, nil
}
@@ -336,7 +337,7 @@ func (c *MinerUCloudReader) extractDoneResult(_ context.Context, item *extractRe
return "", nil, fmt.Errorf("extract zip: %w", err)
}
logger.Printf("INFO: [MinerUCloud] parsed (zip), markdown=%d chars, images=%d", len(md), len(imageRefs))
logger.Infof(context.Background(), "[MinerUCloud] parsed (zip), markdown=%d chars, images=%d", len(md), len(imageRefs))
return md, imageRefs, nil
}
@@ -410,13 +411,13 @@ func downloadAndExtractZip(zipURL string) (string, []types.ImageRef, error) {
resolved := resolveInZip(imgPath, mdDir, entries)
if resolved == nil {
logger.Printf("WARN: [MinerUCloud] image not found in zip: %s", imgPath)
logger.Errorf(context.Background(), "[MinerUCloud] image not found in zip: %s", imgPath)
continue
}
imgData, err := readZipEntryBytes(resolved)
if err != nil {
logger.Printf("WARN: [MinerUCloud] failed to read zip image %s: %v", imgPath, err)
logger.Errorf(context.Background(), "[MinerUCloud] failed to read zip image %s: %v", imgPath, err)
continue
}
@@ -16,6 +16,7 @@ import (
"time"
htmltomd "github.com/JohannesKaufmann/html-to-markdown/v2"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
)
@@ -56,7 +57,7 @@ func (c *MinerUReader) Read(ctx context.Context, req *types.ReadRequest) (*types
return &types.ReadResult{Error: "no file content provided"}, nil
}
logger.Printf("INFO: [MinerU] Parsing file=%s size=%d via %s", req.FileName, len(content), c.endpoint)
logger.Infof(context.Background(), "[MinerU] Parsing file=%s size=%d via %s", req.FileName, len(content), c.endpoint)
mdContent, imagesB64, err := c.callFileParse(ctx, content)
if err != nil {
@@ -71,7 +72,7 @@ func (c *MinerUReader) Read(ctx context.Context, req *types.ReadRequest) (*types
mdContent, imageRefs = ensureOriginalImageRef(req, mdContent, imageRefs)
logger.Printf("INFO: [MinerU] Parsed successfully, markdown=%d chars, images=%d", len(mdContent), len(imageRefs))
logger.Infof(context.Background(), "[MinerU] Parsed successfully, markdown=%d chars, images=%d", len(mdContent), len(imageRefs))
return &types.ReadResult{
MarkdownContent: mdContent,
@@ -158,9 +159,9 @@ func (c *MinerUReader) callFileParse(ctx context.Context, content []byte) (strin
// Dump raw response for debugging (truncate if too large)
rawStr := string(respBody)
if len(rawStr) > 4000 {
logger.Printf("DEBUG: [MinerU] Raw response (truncated to 4000 chars): %s ...", rawStr[:4000])
logger.Infof(context.Background(), "[MinerU] Raw response (truncated to 4000 chars): %s ...", rawStr[:4000])
} else {
logger.Printf("DEBUG: [MinerU] Raw response: %s", rawStr)
logger.Infof(context.Background(), "[MinerU] Raw response: %s", rawStr)
}
// Also pretty-print the top-level structure (without large base64 blobs)
@@ -179,15 +180,15 @@ func (c *MinerUReader) callFileParse(ctx context.Context, content []byte) (strin
// - some variants: results.files.*
// Prefer document when available, then fallback to files.
if result.Results.Document.MDContent != "" || len(result.Results.Document.Images) > 0 {
logger.Printf("DEBUG: [MinerU] Using response path: results.document")
logger.Infof(context.Background(), "[MinerU] Using response path: results.document")
return result.Results.Document.MDContent, result.Results.Document.Images, nil
}
if result.Results.Files.MDContent != "" || len(result.Results.Files.Images) > 0 {
logger.Printf("DEBUG: [MinerU] Using response path: results.files")
logger.Infof(context.Background(), "[MinerU] Using response path: results.files")
return result.Results.Files.MDContent, result.Results.Files.Images, nil
}
logger.Printf("WARN: [MinerU] Response has no markdown/images under results.document or results.files")
logger.Errorf(context.Background(), "[MinerU] Response has no markdown/images under results.document or results.files")
return "", nil, nil
}
@@ -209,7 +210,7 @@ func (c *MinerUReader) processImages(mdContent string, imagesB64 map[string]stri
ext = m[1]
decoded, err := base64.StdEncoding.DecodeString(m[2])
if err != nil {
logger.Printf("WARN: [MinerU] Failed to decode base64 image %s: %v", ipath, err)
logger.Errorf(context.Background(), "[MinerU] Failed to decode base64 image %s: %v", ipath, err)
continue
}
imgBytes = decoded
@@ -217,7 +218,7 @@ func (c *MinerUReader) processImages(mdContent string, imagesB64 map[string]stri
// raw base64 without data URI prefix
decoded, err := base64.StdEncoding.DecodeString(b64Str)
if err != nil {
logger.Printf("WARN: [MinerU] Failed to decode raw base64 image %s: %v", ipath, err)
logger.Errorf(context.Background(), "[MinerU] Failed to decode raw base64 image %s: %v", ipath, err)
continue
}
imgBytes = decoded
@@ -271,7 +272,7 @@ func PingMinerU(endpoint string) (bool, string) {
func htmlToMarkdown(content string) string {
md, err := htmltomd.ConvertString(content)
if err != nil {
logger.Printf("WARN: [MinerU] html-to-markdown conversion failed, using raw content: %v", err)
logger.Errorf(context.Background(), "[MinerU] html-to-markdown conversion failed, using raw content: %v", err)
return content
}
return md
@@ -8,18 +8,22 @@ import (
"fmt"
"io"
"net/http"
"sort"
"strings"
"time"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/models/utils"
"github.com/Tencent/WeKnora/internal/types"
"github.com/google/uuid"
)
const (
weKnoraCloudReaderBaseURL = "https://weknora.weixin.qq.com/api/v1/doc"
)
// WeKnoraCloudSignedDocumentReader implements the docreader HTTP protocol with WeKnoraCloud signing.
type WeKnoraCloudSignedDocumentReader struct {
baseURL string
appID string
apiKey string
client *http.Client
@@ -28,11 +32,7 @@ type WeKnoraCloudSignedDocumentReader struct {
pollTimeout time.Duration
}
func NewWeKnoraCloudSignedDocumentReader(baseURL, appID, apiKey string) (*WeKnoraCloudSignedDocumentReader, error) {
baseURL = strings.TrimSuffix(strings.TrimSpace(baseURL), "/")
if baseURL == "" {
return nil, fmt.Errorf("docreader base URL is required")
}
func NewWeKnoraCloudSignedDocumentReader(appID, apiKey string) (*WeKnoraCloudSignedDocumentReader, error) {
if appID == "" {
return nil, fmt.Errorf("WeKnoraCloud appID is required")
}
@@ -40,7 +40,6 @@ func NewWeKnoraCloudSignedDocumentReader(baseURL, appID, apiKey string) (*WeKnor
return nil, fmt.Errorf("WeKnoraCloud apiKey is required")
}
return &WeKnoraCloudSignedDocumentReader{
baseURL: baseURL,
appID: appID,
apiKey: apiKey,
initialPollInterval: 500 * time.Millisecond,
@@ -58,28 +57,23 @@ func NewWeKnoraCloudSignedDocumentReader(baseURL, appID, apiKey string) (*WeKnor
}
func (p *WeKnoraCloudSignedDocumentReader) Reconnect(addr string) error {
p.baseURL = strings.TrimSuffix(strings.TrimSpace(addr), "/")
return nil
}
func (p *WeKnoraCloudSignedDocumentReader) IsConnected() bool { return p.baseURL != "" }
func (p *WeKnoraCloudSignedDocumentReader) IsConnected() bool { return true }
func (p *WeKnoraCloudSignedDocumentReader) ListEngines(ctx context.Context, overrides map[string]string) ([]types.ParserEngineInfo, error) {
if !p.IsConnected() {
return nil, errNotConnected
}
return []types.ParserEngineInfo{{
Name: WeKnoraCloudEngineName,
Description: "WeKnoraCloud signed docreader",
FileTypes: weKnoraCloudSupportedFileTypes(),
FileTypes: []string{"docx", "doc", "pdf", "md", "markdown", "xlsx", "xls", "pptx", "ppt"},
Available: true,
}}, nil
}
func (p *WeKnoraCloudSignedDocumentReader) Read(ctx context.Context, req *types.ReadRequest) (*types.ReadResult, error) {
if !p.IsConnected() {
return nil, errNotConnected
}
logger.Infof(ctx, "[WeKnoraCloud] read start file=%q type=%q engine=%q hasURL=%v contentLen=%d requestID=%q",
req.FileName, req.FileType, req.ParserEngine, strings.TrimSpace(req.URL) != "", len(req.FileContent), req.RequestID)
body := httpReadRequest{
FileName: req.FileName,
FileType: req.FileType,
@@ -96,28 +90,35 @@ func (p *WeKnoraCloudSignedDocumentReader) Read(ctx context.Context, req *types.
}
jsonBody, err := json.Marshal(body)
if err != nil {
logger.Errorf(context.Background(), "[WeKnoraCloud] marshal read request: %v", err)
return nil, fmt.Errorf("http marshal read request: %w", err)
}
httpReq, err := p.newSignedRequest(ctx, http.MethodPost, p.baseURL, jsonBody)
httpReq, err := p.newSignedRequest(ctx, http.MethodPost, weKnoraCloudReaderBaseURL+"/reader", jsonBody)
if err != nil {
logger.Errorf(context.Background(), "[WeKnoraCloud] signed read request: %v", err)
return nil, err
}
resp, err := p.client.Do(httpReq)
if err != nil {
logger.Errorf(context.Background(), "[WeKnoraCloud] http read request failed: %v", err)
return nil, fmt.Errorf("http read failed: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusAccepted && resp.StatusCode != http.StatusOK {
bodyBytes, _ := io.ReadAll(resp.Body)
logger.Errorf(context.Background(), "[WeKnoraCloud] http read unexpected status %d: %s", resp.StatusCode, string(bodyBytes))
return nil, fmt.Errorf("http read status %d: %s", resp.StatusCode, string(bodyBytes))
}
var submit weKnoraCloudAsyncSubmitResponse
if err := json.NewDecoder(resp.Body).Decode(&submit); err != nil {
logger.Errorf(context.Background(), "[WeKnoraCloud] decode read submit response: %v", err)
return nil, fmt.Errorf("http decode read submit response: %w", err)
}
if strings.TrimSpace(submit.TaskID) == "" {
logger.Errorf(context.Background(), "[WeKnoraCloud] submit response missing task_id (status=%q message=%q)", submit.Status, submit.Message)
return nil, fmt.Errorf("weknoracloud docreader submit response missing task_id")
}
logger.Infof(ctx, "[WeKnoraCloud] task submitted task_id=%s file=%q type=%q", submit.TaskID, req.FileName, req.FileType)
return p.pollTaskResult(ctx, submit.TaskID)
}
@@ -139,25 +140,6 @@ type weKnoraCloudAsyncTaskResponse struct {
UpdatedAt int64 `json:"updated_at"`
}
func weKnoraCloudSupportedFileTypes() []string {
seen := map[string]struct{}{}
var fileTypes []string
for _, engine := range localEngines {
if engine.Name() == WeKnoraCloudEngineName {
continue
}
for _, fileType := range engine.FileTypes(true) {
if _, ok := seen[fileType]; ok {
continue
}
seen[fileType] = struct{}{}
fileTypes = append(fileTypes, fileType)
}
}
sort.Strings(fileTypes)
return fileTypes
}
func (p *WeKnoraCloudSignedDocumentReader) pollTaskResult(ctx context.Context, taskID string) (*types.ReadResult, error) {
pollCtx := ctx
if _, ok := ctx.Deadline(); !ok && p.pollTimeout > 0 {
@@ -165,15 +147,17 @@ func (p *WeKnoraCloudSignedDocumentReader) pollTaskResult(ctx context.Context, t
pollCtx, cancel = context.WithTimeout(ctx, p.pollTimeout)
defer cancel()
}
statusURL := strings.TrimSuffix(p.baseURL, "/reader") + "/" + taskID
statusURL := weKnoraCloudReaderBaseURL + "/" + taskID
currentInterval := p.initialPollInterval
for {
httpReq, err := p.newSignedRequest(pollCtx, http.MethodGet, statusURL, nil)
if err != nil {
logger.Errorf(context.Background(), "[WeKnoraCloud] poll signed request task_id=%s: %v", taskID, err)
return nil, err
}
resp, err := p.client.Do(httpReq)
if err != nil {
logger.Errorf(context.Background(), "[WeKnoraCloud] http poll task_id=%s failed: %v", taskID, err)
return nil, fmt.Errorf("http poll task failed: %w", err)
}
var taskResp weKnoraCloudAsyncTaskResponse
@@ -182,10 +166,12 @@ func (p *WeKnoraCloudSignedDocumentReader) pollTaskResult(ctx context.Context, t
if resp.StatusCode != http.StatusOK {
bodyBytes, _ := io.ReadAll(resp.Body)
err = fmt.Errorf("http poll task status %d: %s", resp.StatusCode, string(bodyBytes))
logger.Errorf(context.Background(), "[WeKnoraCloud] poll task_id=%s status %d: %s", taskID, resp.StatusCode, string(bodyBytes))
return
}
if decodeErr := json.NewDecoder(resp.Body).Decode(&taskResp); decodeErr != nil {
err = fmt.Errorf("http decode task response: %w", decodeErr)
logger.Errorf(context.Background(), "[WeKnoraCloud] poll task_id=%s decode response: %v", taskID, decodeErr)
}
}()
if err != nil {
@@ -194,27 +180,40 @@ func (p *WeKnoraCloudSignedDocumentReader) pollTaskResult(ctx context.Context, t
switch taskResp.Status {
case "completed":
if taskResp.Result == nil {
logger.Infof(ctx, "[WeKnoraCloud] task_id=%s completed with no result payload", taskID)
return &types.ReadResult{}, nil
}
return fromHTTPReadResponse(taskResp.Result), nil
res := fromHTTPReadResponse(taskResp.Result)
if res.Error != "" {
logger.Errorf(ctx, "[WeKnoraCloud] task_id=%s completed with result.error: %s", taskID, res.Error)
} else {
logger.Debugf(ctx, "[WeKnoraCloud] task_id=%s completed ok markdownLen=%d", taskID, len(res.MarkdownContent))
}
return res, nil
case "failed":
if taskResp.Error != "" {
logger.Errorf(context.Background(), "[WeKnoraCloud] task_id=%s failed: %s", taskID, taskResp.Error)
return nil, fmt.Errorf("weknoracloud docreader task failed: %s", taskResp.Error)
}
logger.Errorf(context.Background(), "[WeKnoraCloud] task_id=%s failed: %s", taskID, taskResp.Message)
return nil, fmt.Errorf("weknoracloud docreader task failed: %s", taskResp.Message)
case "cancelled":
if taskResp.Error != "" {
logger.Errorf(context.Background(), "[WeKnoraCloud] task_id=%s cancelled: %s", taskID, taskResp.Error)
return nil, fmt.Errorf("weknoracloud docreader task cancelled: %s", taskResp.Error)
}
logger.Errorf(context.Background(), "[WeKnoraCloud] task_id=%s cancelled", taskID)
return nil, fmt.Errorf("weknoracloud docreader task cancelled")
}
if err := pollCtx.Err(); err != nil {
logger.Errorf(ctx, "[WeKnoraCloud] poll task_id=%s aborted before sleep: %v", taskID, err)
return nil, err
}
// Exponential backoff: multiply by 1.5 each time, cap at maxPollInterval
select {
case <-pollCtx.Done():
logger.Errorf(ctx, "[WeKnoraCloud] poll task_id=%s stopped: %v", taskID, pollCtx.Err())
return nil, pollCtx.Err()
case <-time.After(currentInterval):
// Update interval for next iteration
@@ -234,6 +233,7 @@ func (p *WeKnoraCloudSignedDocumentReader) newSignedRequest(ctx context.Context,
}
httpReq, err := http.NewRequestWithContext(ctx, method, url, bytes.NewReader(body))
if err != nil {
logger.Errorf(context.Background(), "[WeKnoraCloud] http new request %s %s: %v", method, url, err)
return nil, fmt.Errorf("http new request: %w", err)
}
httpReq.Header.Set("Content-Type", "application/json")