feat(weknoracloud): implement credential management and status checks
- Introduced a new WeKnoraCloudSettings component for managing APPID and APPSECRET. - Added functionality to save WeKnoraCloud credentials without automatically creating models. - Implemented credential status checks across various components to provide user feedback on configuration status. - Updated API endpoints to handle credential saving and status retrieval. - Enhanced UI elements to guide users in configuring WeKnoraCloud settings. This update improves the integration of WeKnoraCloud by allowing users to manage their credentials more effectively and receive real-time feedback on their configuration status.
This commit is contained in:
@@ -3,7 +3,6 @@ package docparser
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/models/provider"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
@@ -87,13 +86,13 @@ type weKnoraCloudEngine struct{}
|
||||
func (e *weKnoraCloudEngine) Name() string { return WeKnoraCloudEngineName }
|
||||
func (e *weKnoraCloudEngine) Description() string { return "WeKnoraCloud document reader" }
|
||||
func (e *weKnoraCloudEngine) FileTypes(_ bool) []string {
|
||||
return weKnoraCloudSupportedFileTypes()
|
||||
return []string{"docx", "doc", "pdf", "md", "markdown", "xlsx", "xls", "pptx", "ppt"}
|
||||
}
|
||||
func (e *weKnoraCloudEngine) CheckAvailable(docreaderConnected bool, overrides map[string]string) (bool, string) {
|
||||
if docreaderConnected && isWeKnoraCloudDocReaderAddr(overrides["docreader_addr"]) {
|
||||
if overrides["docreader_app_id"] != "" {
|
||||
return true, ""
|
||||
}
|
||||
return false, "WeKnoraCloud document reader is not initialized"
|
||||
return false, "WeKnora Cloud credentials not configured. Go to Settings → WeKnora Cloud to set up."
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -192,7 +191,3 @@ func ListAllEngines(docreaderConnected bool, overrides map[string]string, remote
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
func isWeKnoraCloudDocReaderAddr(addr string) bool {
|
||||
return strings.TrimSuffix(strings.TrimSpace(addr), "/") == strings.TrimRight(provider.WeKnoraCloudBaseURL, "/")+"/api/v1/doc/reader"
|
||||
}
|
||||
|
||||
@@ -3,21 +3,19 @@ package docparser
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/docreader/proto"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
"google.golang.org/grpc/resolver"
|
||||
)
|
||||
|
||||
var logger = log.New(os.Stdout, "[DocParser] ", log.LstdFlags|log.Lmicroseconds)
|
||||
|
||||
func getMaxMessageSize() int {
|
||||
if sizeStr := os.Getenv("MAX_FILE_SIZE_MB"); sizeStr != "" {
|
||||
if size, err := strconv.Atoi(sizeStr); err == nil && size > 0 {
|
||||
@@ -46,7 +44,6 @@ func NewGRPCDocumentReader(addr string) (*GRPCDocumentReader, error) {
|
||||
}
|
||||
|
||||
func (p *GRPCDocumentReader) connect(addr string) error {
|
||||
logger.Printf("INFO: Connecting to docreader at %s", addr)
|
||||
|
||||
maxMsgSize := getMaxMessageSize()
|
||||
opts := []grpc.DialOption{
|
||||
@@ -64,7 +61,7 @@ func (p *GRPCDocumentReader) connect(addr string) error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to connect to docreader: %w", err)
|
||||
}
|
||||
logger.Printf("INFO: Connected to docreader in %v", time.Since(start))
|
||||
logger.Infof(context.Background(), "Connected to docreader in %v", time.Since(start))
|
||||
|
||||
p.conn = conn
|
||||
p.client = proto.NewDocReaderClient(conn)
|
||||
|
||||
@@ -6,6 +6,8 @@ import (
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
)
|
||||
|
||||
// stringOr returns val (trimmed) if non-empty, otherwise fallback.
|
||||
@@ -56,7 +58,7 @@ func logResponseStructure(label string, obj interface{}, prefix string) {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
logger.Printf("DEBUG: [%s] %s = {object with %d keys: %s}", label, prefix, len(v), strings.Join(keys, ", "))
|
||||
logger.Infof(context.Background(), "[%s] %s = {object with %d keys: %s}", label, prefix, len(v), strings.Join(keys, ", "))
|
||||
for _, key := range keys {
|
||||
val := v[key]
|
||||
path := prefix + "." + key
|
||||
@@ -64,36 +66,36 @@ func logResponseStructure(label string, obj interface{}, prefix string) {
|
||||
case map[string]interface{}:
|
||||
logResponseStructure(label, inner, path)
|
||||
case []interface{}:
|
||||
logger.Printf("DEBUG: [%s] %s = [array with %d items]", label, path, len(inner))
|
||||
logger.Infof(context.Background(), "[%s] %s = [array with %d items]", label, path, len(inner))
|
||||
if len(inner) > 0 {
|
||||
logger.Printf("DEBUG: [%s] %s[0] type=%T", label, path, inner[0])
|
||||
logger.Infof(context.Background(), "[%s] %s[0] type=%T", label, path, inner[0])
|
||||
if len(inner) <= 3 {
|
||||
for i, item := range inner {
|
||||
logResponseStructure(label, item, fmt.Sprintf("%s[%d]", path, i))
|
||||
}
|
||||
} else {
|
||||
logResponseStructure(label, inner[0], path+"[0]")
|
||||
logger.Printf("DEBUG: [%s] ... and %d more items in %s", label, len(inner)-1, path)
|
||||
logger.Infof(context.Background(), "[%s] ... and %d more items in %s", label, len(inner)-1, path)
|
||||
}
|
||||
}
|
||||
case string:
|
||||
if len(inner) > 200 {
|
||||
logger.Printf("DEBUG: [%s] %s = string(%d chars): %.200s...", label, path, len(inner), inner)
|
||||
logger.Infof(context.Background(), "[%s] %s = string(%d chars): %.200s...", label, path, len(inner), inner)
|
||||
} else {
|
||||
logger.Printf("DEBUG: [%s] %s = %q", label, path, inner)
|
||||
logger.Infof(context.Background(), "[%s] %s = %q", label, path, inner)
|
||||
}
|
||||
case float64:
|
||||
logger.Printf("DEBUG: [%s] %s = %v (number)", label, path, inner)
|
||||
logger.Infof(context.Background(), "[%s] %s = %v (number)", label, path, inner)
|
||||
case bool:
|
||||
logger.Printf("DEBUG: [%s] %s = %v (bool)", label, path, inner)
|
||||
logger.Infof(context.Background(), "[%s] %s = %v (bool)", label, path, inner)
|
||||
case nil:
|
||||
logger.Printf("DEBUG: [%s] %s = null", label, path)
|
||||
logger.Infof(context.Background(), "[%s] %s = null", label, path)
|
||||
default:
|
||||
logger.Printf("DEBUG: [%s] %s = %v (%T)", label, path, val, val)
|
||||
logger.Infof(context.Background(), "[%s] %s = %v (%T)", label, path, val, val)
|
||||
}
|
||||
}
|
||||
case []interface{}:
|
||||
logger.Printf("DEBUG: [%s] %s = [array with %d items]", label, prefix, len(v))
|
||||
logger.Infof(context.Background(), "[%s] %s = [array with %d items]", label, prefix, len(v))
|
||||
if len(v) > 0 {
|
||||
if len(v) <= 3 {
|
||||
for i, item := range v {
|
||||
@@ -101,10 +103,10 @@ func logResponseStructure(label string, obj interface{}, prefix string) {
|
||||
}
|
||||
} else {
|
||||
logResponseStructure(label, v[0], prefix+"[0]")
|
||||
logger.Printf("DEBUG: [%s] ... and %d more items in %s", label, len(v)-1, prefix)
|
||||
logger.Infof(context.Background(), "[%s] ... and %d more items in %s", label, len(v)-1, prefix)
|
||||
}
|
||||
}
|
||||
default:
|
||||
logger.Printf("DEBUG: [%s] %s = %v (%T)", label, prefix, v, v)
|
||||
logger.Infof(context.Background(), "[%s] %s = %v (%T)", label, prefix, v, v)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
@@ -73,7 +74,7 @@ func NewHTTPDocumentReader(baseURL string) (*HTTPDocumentReader, error) {
|
||||
},
|
||||
}
|
||||
if p.baseURL != "" {
|
||||
logger.Printf("INFO: HTTP docreader base URL: %s", p.baseURL)
|
||||
logger.Infof(context.Background(), "INFO: HTTP docreader base URL: %s", p.baseURL)
|
||||
}
|
||||
return p, nil
|
||||
}
|
||||
@@ -88,7 +89,7 @@ func (p *HTTPDocumentReader) Reconnect(addr string) error {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
p.baseURL = strings.TrimSuffix(addr, "/")
|
||||
logger.Printf("INFO: HTTP docreader base URL set to %s", p.baseURL)
|
||||
logger.Infof(context.Background(), "INFO: HTTP docreader base URL set to %s", p.baseURL)
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/utils"
|
||||
"github.com/google/uuid"
|
||||
@@ -61,7 +62,7 @@ func (c *MinerUCloudReader) Read(ctx context.Context, req *types.ReadRequest) (*
|
||||
return &types.ReadResult{Error: "no file content provided"}, nil
|
||||
}
|
||||
|
||||
logger.Printf("INFO: [MinerUCloud] Parsing file=%s size=%d via %s", req.FileName, len(content), c.baseURL)
|
||||
logger.Infof(context.Background(), "[MinerUCloud] Parsing file=%s size=%d via %s", req.FileName, len(content), c.baseURL)
|
||||
|
||||
ext := filepath.Ext(req.FileName)
|
||||
if ext == "" && req.FileType != "" {
|
||||
@@ -158,7 +159,7 @@ func (c *MinerUCloudReader) applyUploadURLs(ctx context.Context, fileName, ext s
|
||||
return "", "", fmt.Errorf("API returned no file_urls")
|
||||
}
|
||||
|
||||
logger.Printf("INFO: [MinerUCloud] batch apply ok: batch_id=%s, urls=%d", result.Data.BatchID, len(result.Data.FileURLs))
|
||||
logger.Infof(context.Background(), "[MinerUCloud] batch apply ok: batch_id=%s, urls=%d", result.Data.BatchID, len(result.Data.FileURLs))
|
||||
return result.Data.BatchID, result.Data.FileURLs[0], nil
|
||||
}
|
||||
|
||||
@@ -178,7 +179,7 @@ func (c *MinerUCloudReader) uploadFile(ctx context.Context, uploadURL string, co
|
||||
if resp.StatusCode >= 300 {
|
||||
return fmt.Errorf("PUT upload status %d", resp.StatusCode)
|
||||
}
|
||||
logger.Printf("INFO: [MinerUCloud] file uploaded, status=%d", resp.StatusCode)
|
||||
logger.Infof(context.Background(), "[MinerUCloud] file uploaded, status=%d", resp.StatusCode)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -218,14 +219,14 @@ func (c *MinerUCloudReader) pollBatchResult(ctx context.Context, batchID string)
|
||||
|
||||
items, err := c.fetchBatchStatus(ctx, batchID, headers)
|
||||
if err != nil {
|
||||
logger.Printf("WARN: [MinerUCloud] poll #%d failed: %v", pollCount, err)
|
||||
logger.Errorf(context.Background(), "[MinerUCloud] poll #%d failed: %v", pollCount, err)
|
||||
sleepCtx(ctx, defaultPollInterval)
|
||||
continue
|
||||
}
|
||||
|
||||
if len(items) == 0 {
|
||||
if pollCount <= 3 || pollCount%10 == 0 {
|
||||
logger.Printf("INFO: [MinerUCloud] poll #%d: extract_result empty, retrying", pollCount)
|
||||
logger.Infof(context.Background(), "[MinerUCloud] poll #%d: extract_result empty, retrying", pollCount)
|
||||
}
|
||||
sleepCtx(ctx, defaultPollInterval)
|
||||
continue
|
||||
@@ -235,7 +236,7 @@ func (c *MinerUCloudReader) pollBatchResult(ctx context.Context, batchID string)
|
||||
state := strings.ToLower(item.State)
|
||||
|
||||
if pollCount == 1 || pollCount%10 == 0 || state == "done" || state == "failed" {
|
||||
logger.Printf("INFO: [MinerUCloud] poll #%d: file=%s state=%s pages=%d/%d",
|
||||
logger.Infof(context.Background(), "[MinerUCloud] poll #%d: file=%s state=%s pages=%d/%d",
|
||||
pollCount, item.FileName, state, item.Progress.ExtractedPages, item.Progress.TotalPages)
|
||||
}
|
||||
|
||||
@@ -290,9 +291,9 @@ func (c *MinerUCloudReader) fetchBatchStatus(ctx context.Context, batchID string
|
||||
// Dump the raw extract_result JSON for debugging
|
||||
rawExtract := string(pollResp.Data.ExtractResult)
|
||||
if len(rawExtract) > 4000 {
|
||||
logger.Printf("DEBUG: [MinerUCloud] Raw extract_result (truncated to 4000 chars): %s ...", rawExtract[:4000])
|
||||
logger.Infof(context.Background(), "[MinerUCloud] Raw extract_result (truncated to 4000 chars): %s ...", rawExtract[:4000])
|
||||
} else {
|
||||
logger.Printf("DEBUG: [MinerUCloud] Raw extract_result: %s", rawExtract)
|
||||
logger.Infof(context.Background(), "[MinerUCloud] Raw extract_result: %s", rawExtract)
|
||||
}
|
||||
|
||||
// Pretty-print the structure to reveal all available fields
|
||||
@@ -323,7 +324,7 @@ func (c *MinerUCloudReader) fetchBatchStatus(ctx context.Context, batchID string
|
||||
func (c *MinerUCloudReader) extractDoneResult(_ context.Context, item *extractResultItem) (string, []types.ImageRef, error) {
|
||||
text := firstNonEmpty(item.Markdown, item.Content, item.Text)
|
||||
if text != "" {
|
||||
logger.Printf("INFO: [MinerUCloud] parsed (inline), length=%d", len(text))
|
||||
logger.Infof(context.Background(), "[MinerUCloud] parsed (inline), length=%d", len(text))
|
||||
return text, nil, nil
|
||||
}
|
||||
|
||||
@@ -336,7 +337,7 @@ func (c *MinerUCloudReader) extractDoneResult(_ context.Context, item *extractRe
|
||||
return "", nil, fmt.Errorf("extract zip: %w", err)
|
||||
}
|
||||
|
||||
logger.Printf("INFO: [MinerUCloud] parsed (zip), markdown=%d chars, images=%d", len(md), len(imageRefs))
|
||||
logger.Infof(context.Background(), "[MinerUCloud] parsed (zip), markdown=%d chars, images=%d", len(md), len(imageRefs))
|
||||
return md, imageRefs, nil
|
||||
}
|
||||
|
||||
@@ -410,13 +411,13 @@ func downloadAndExtractZip(zipURL string) (string, []types.ImageRef, error) {
|
||||
|
||||
resolved := resolveInZip(imgPath, mdDir, entries)
|
||||
if resolved == nil {
|
||||
logger.Printf("WARN: [MinerUCloud] image not found in zip: %s", imgPath)
|
||||
logger.Errorf(context.Background(), "[MinerUCloud] image not found in zip: %s", imgPath)
|
||||
continue
|
||||
}
|
||||
|
||||
imgData, err := readZipEntryBytes(resolved)
|
||||
if err != nil {
|
||||
logger.Printf("WARN: [MinerUCloud] failed to read zip image %s: %v", imgPath, err)
|
||||
logger.Errorf(context.Background(), "[MinerUCloud] failed to read zip image %s: %v", imgPath, err)
|
||||
continue
|
||||
}
|
||||
|
||||
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
"time"
|
||||
|
||||
htmltomd "github.com/JohannesKaufmann/html-to-markdown/v2"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
@@ -56,7 +57,7 @@ func (c *MinerUReader) Read(ctx context.Context, req *types.ReadRequest) (*types
|
||||
return &types.ReadResult{Error: "no file content provided"}, nil
|
||||
}
|
||||
|
||||
logger.Printf("INFO: [MinerU] Parsing file=%s size=%d via %s", req.FileName, len(content), c.endpoint)
|
||||
logger.Infof(context.Background(), "[MinerU] Parsing file=%s size=%d via %s", req.FileName, len(content), c.endpoint)
|
||||
|
||||
mdContent, imagesB64, err := c.callFileParse(ctx, content)
|
||||
if err != nil {
|
||||
@@ -71,7 +72,7 @@ func (c *MinerUReader) Read(ctx context.Context, req *types.ReadRequest) (*types
|
||||
|
||||
mdContent, imageRefs = ensureOriginalImageRef(req, mdContent, imageRefs)
|
||||
|
||||
logger.Printf("INFO: [MinerU] Parsed successfully, markdown=%d chars, images=%d", len(mdContent), len(imageRefs))
|
||||
logger.Infof(context.Background(), "[MinerU] Parsed successfully, markdown=%d chars, images=%d", len(mdContent), len(imageRefs))
|
||||
|
||||
return &types.ReadResult{
|
||||
MarkdownContent: mdContent,
|
||||
@@ -158,9 +159,9 @@ func (c *MinerUReader) callFileParse(ctx context.Context, content []byte) (strin
|
||||
// Dump raw response for debugging (truncate if too large)
|
||||
rawStr := string(respBody)
|
||||
if len(rawStr) > 4000 {
|
||||
logger.Printf("DEBUG: [MinerU] Raw response (truncated to 4000 chars): %s ...", rawStr[:4000])
|
||||
logger.Infof(context.Background(), "[MinerU] Raw response (truncated to 4000 chars): %s ...", rawStr[:4000])
|
||||
} else {
|
||||
logger.Printf("DEBUG: [MinerU] Raw response: %s", rawStr)
|
||||
logger.Infof(context.Background(), "[MinerU] Raw response: %s", rawStr)
|
||||
}
|
||||
|
||||
// Also pretty-print the top-level structure (without large base64 blobs)
|
||||
@@ -179,15 +180,15 @@ func (c *MinerUReader) callFileParse(ctx context.Context, content []byte) (strin
|
||||
// - some variants: results.files.*
|
||||
// Prefer document when available, then fallback to files.
|
||||
if result.Results.Document.MDContent != "" || len(result.Results.Document.Images) > 0 {
|
||||
logger.Printf("DEBUG: [MinerU] Using response path: results.document")
|
||||
logger.Infof(context.Background(), "[MinerU] Using response path: results.document")
|
||||
return result.Results.Document.MDContent, result.Results.Document.Images, nil
|
||||
}
|
||||
if result.Results.Files.MDContent != "" || len(result.Results.Files.Images) > 0 {
|
||||
logger.Printf("DEBUG: [MinerU] Using response path: results.files")
|
||||
logger.Infof(context.Background(), "[MinerU] Using response path: results.files")
|
||||
return result.Results.Files.MDContent, result.Results.Files.Images, nil
|
||||
}
|
||||
|
||||
logger.Printf("WARN: [MinerU] Response has no markdown/images under results.document or results.files")
|
||||
logger.Errorf(context.Background(), "[MinerU] Response has no markdown/images under results.document or results.files")
|
||||
return "", nil, nil
|
||||
}
|
||||
|
||||
@@ -209,7 +210,7 @@ func (c *MinerUReader) processImages(mdContent string, imagesB64 map[string]stri
|
||||
ext = m[1]
|
||||
decoded, err := base64.StdEncoding.DecodeString(m[2])
|
||||
if err != nil {
|
||||
logger.Printf("WARN: [MinerU] Failed to decode base64 image %s: %v", ipath, err)
|
||||
logger.Errorf(context.Background(), "[MinerU] Failed to decode base64 image %s: %v", ipath, err)
|
||||
continue
|
||||
}
|
||||
imgBytes = decoded
|
||||
@@ -217,7 +218,7 @@ func (c *MinerUReader) processImages(mdContent string, imagesB64 map[string]stri
|
||||
// raw base64 without data URI prefix
|
||||
decoded, err := base64.StdEncoding.DecodeString(b64Str)
|
||||
if err != nil {
|
||||
logger.Printf("WARN: [MinerU] Failed to decode raw base64 image %s: %v", ipath, err)
|
||||
logger.Errorf(context.Background(), "[MinerU] Failed to decode raw base64 image %s: %v", ipath, err)
|
||||
continue
|
||||
}
|
||||
imgBytes = decoded
|
||||
@@ -271,7 +272,7 @@ func PingMinerU(endpoint string) (bool, string) {
|
||||
func htmlToMarkdown(content string) string {
|
||||
md, err := htmltomd.ConvertString(content)
|
||||
if err != nil {
|
||||
logger.Printf("WARN: [MinerU] html-to-markdown conversion failed, using raw content: %v", err)
|
||||
logger.Errorf(context.Background(), "[MinerU] html-to-markdown conversion failed, using raw content: %v", err)
|
||||
return content
|
||||
}
|
||||
return md
|
||||
|
||||
@@ -8,18 +8,22 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/models/utils"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
const (
|
||||
weKnoraCloudReaderBaseURL = "https://weknora.weixin.qq.com/api/v1/doc"
|
||||
)
|
||||
|
||||
// WeKnoraCloudSignedDocumentReader implements the docreader HTTP protocol with WeKnoraCloud signing.
|
||||
type WeKnoraCloudSignedDocumentReader struct {
|
||||
baseURL string
|
||||
appID string
|
||||
apiKey string
|
||||
client *http.Client
|
||||
@@ -28,11 +32,7 @@ type WeKnoraCloudSignedDocumentReader struct {
|
||||
pollTimeout time.Duration
|
||||
}
|
||||
|
||||
func NewWeKnoraCloudSignedDocumentReader(baseURL, appID, apiKey string) (*WeKnoraCloudSignedDocumentReader, error) {
|
||||
baseURL = strings.TrimSuffix(strings.TrimSpace(baseURL), "/")
|
||||
if baseURL == "" {
|
||||
return nil, fmt.Errorf("docreader base URL is required")
|
||||
}
|
||||
func NewWeKnoraCloudSignedDocumentReader(appID, apiKey string) (*WeKnoraCloudSignedDocumentReader, error) {
|
||||
if appID == "" {
|
||||
return nil, fmt.Errorf("WeKnoraCloud appID is required")
|
||||
}
|
||||
@@ -40,7 +40,6 @@ func NewWeKnoraCloudSignedDocumentReader(baseURL, appID, apiKey string) (*WeKnor
|
||||
return nil, fmt.Errorf("WeKnoraCloud apiKey is required")
|
||||
}
|
||||
return &WeKnoraCloudSignedDocumentReader{
|
||||
baseURL: baseURL,
|
||||
appID: appID,
|
||||
apiKey: apiKey,
|
||||
initialPollInterval: 500 * time.Millisecond,
|
||||
@@ -58,28 +57,23 @@ func NewWeKnoraCloudSignedDocumentReader(baseURL, appID, apiKey string) (*WeKnor
|
||||
}
|
||||
|
||||
func (p *WeKnoraCloudSignedDocumentReader) Reconnect(addr string) error {
|
||||
p.baseURL = strings.TrimSuffix(strings.TrimSpace(addr), "/")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *WeKnoraCloudSignedDocumentReader) IsConnected() bool { return p.baseURL != "" }
|
||||
func (p *WeKnoraCloudSignedDocumentReader) IsConnected() bool { return true }
|
||||
|
||||
func (p *WeKnoraCloudSignedDocumentReader) ListEngines(ctx context.Context, overrides map[string]string) ([]types.ParserEngineInfo, error) {
|
||||
if !p.IsConnected() {
|
||||
return nil, errNotConnected
|
||||
}
|
||||
return []types.ParserEngineInfo{{
|
||||
Name: WeKnoraCloudEngineName,
|
||||
Description: "WeKnoraCloud signed docreader",
|
||||
FileTypes: weKnoraCloudSupportedFileTypes(),
|
||||
FileTypes: []string{"docx", "doc", "pdf", "md", "markdown", "xlsx", "xls", "pptx", "ppt"},
|
||||
Available: true,
|
||||
}}, nil
|
||||
}
|
||||
|
||||
func (p *WeKnoraCloudSignedDocumentReader) Read(ctx context.Context, req *types.ReadRequest) (*types.ReadResult, error) {
|
||||
if !p.IsConnected() {
|
||||
return nil, errNotConnected
|
||||
}
|
||||
logger.Infof(ctx, "[WeKnoraCloud] read start file=%q type=%q engine=%q hasURL=%v contentLen=%d requestID=%q",
|
||||
req.FileName, req.FileType, req.ParserEngine, strings.TrimSpace(req.URL) != "", len(req.FileContent), req.RequestID)
|
||||
body := httpReadRequest{
|
||||
FileName: req.FileName,
|
||||
FileType: req.FileType,
|
||||
@@ -96,28 +90,35 @@ func (p *WeKnoraCloudSignedDocumentReader) Read(ctx context.Context, req *types.
|
||||
}
|
||||
jsonBody, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
logger.Errorf(context.Background(), "[WeKnoraCloud] marshal read request: %v", err)
|
||||
return nil, fmt.Errorf("http marshal read request: %w", err)
|
||||
}
|
||||
httpReq, err := p.newSignedRequest(ctx, http.MethodPost, p.baseURL, jsonBody)
|
||||
httpReq, err := p.newSignedRequest(ctx, http.MethodPost, weKnoraCloudReaderBaseURL+"/reader", jsonBody)
|
||||
if err != nil {
|
||||
logger.Errorf(context.Background(), "[WeKnoraCloud] signed read request: %v", err)
|
||||
return nil, err
|
||||
}
|
||||
resp, err := p.client.Do(httpReq)
|
||||
if err != nil {
|
||||
logger.Errorf(context.Background(), "[WeKnoraCloud] http read request failed: %v", err)
|
||||
return nil, fmt.Errorf("http read failed: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusAccepted && resp.StatusCode != http.StatusOK {
|
||||
bodyBytes, _ := io.ReadAll(resp.Body)
|
||||
logger.Errorf(context.Background(), "[WeKnoraCloud] http read unexpected status %d: %s", resp.StatusCode, string(bodyBytes))
|
||||
return nil, fmt.Errorf("http read status %d: %s", resp.StatusCode, string(bodyBytes))
|
||||
}
|
||||
var submit weKnoraCloudAsyncSubmitResponse
|
||||
if err := json.NewDecoder(resp.Body).Decode(&submit); err != nil {
|
||||
logger.Errorf(context.Background(), "[WeKnoraCloud] decode read submit response: %v", err)
|
||||
return nil, fmt.Errorf("http decode read submit response: %w", err)
|
||||
}
|
||||
if strings.TrimSpace(submit.TaskID) == "" {
|
||||
logger.Errorf(context.Background(), "[WeKnoraCloud] submit response missing task_id (status=%q message=%q)", submit.Status, submit.Message)
|
||||
return nil, fmt.Errorf("weknoracloud docreader submit response missing task_id")
|
||||
}
|
||||
logger.Infof(ctx, "[WeKnoraCloud] task submitted task_id=%s file=%q type=%q", submit.TaskID, req.FileName, req.FileType)
|
||||
return p.pollTaskResult(ctx, submit.TaskID)
|
||||
}
|
||||
|
||||
@@ -139,25 +140,6 @@ type weKnoraCloudAsyncTaskResponse struct {
|
||||
UpdatedAt int64 `json:"updated_at"`
|
||||
}
|
||||
|
||||
func weKnoraCloudSupportedFileTypes() []string {
|
||||
seen := map[string]struct{}{}
|
||||
var fileTypes []string
|
||||
for _, engine := range localEngines {
|
||||
if engine.Name() == WeKnoraCloudEngineName {
|
||||
continue
|
||||
}
|
||||
for _, fileType := range engine.FileTypes(true) {
|
||||
if _, ok := seen[fileType]; ok {
|
||||
continue
|
||||
}
|
||||
seen[fileType] = struct{}{}
|
||||
fileTypes = append(fileTypes, fileType)
|
||||
}
|
||||
}
|
||||
sort.Strings(fileTypes)
|
||||
return fileTypes
|
||||
}
|
||||
|
||||
func (p *WeKnoraCloudSignedDocumentReader) pollTaskResult(ctx context.Context, taskID string) (*types.ReadResult, error) {
|
||||
pollCtx := ctx
|
||||
if _, ok := ctx.Deadline(); !ok && p.pollTimeout > 0 {
|
||||
@@ -165,15 +147,17 @@ func (p *WeKnoraCloudSignedDocumentReader) pollTaskResult(ctx context.Context, t
|
||||
pollCtx, cancel = context.WithTimeout(ctx, p.pollTimeout)
|
||||
defer cancel()
|
||||
}
|
||||
statusURL := strings.TrimSuffix(p.baseURL, "/reader") + "/" + taskID
|
||||
statusURL := weKnoraCloudReaderBaseURL + "/" + taskID
|
||||
currentInterval := p.initialPollInterval
|
||||
for {
|
||||
httpReq, err := p.newSignedRequest(pollCtx, http.MethodGet, statusURL, nil)
|
||||
if err != nil {
|
||||
logger.Errorf(context.Background(), "[WeKnoraCloud] poll signed request task_id=%s: %v", taskID, err)
|
||||
return nil, err
|
||||
}
|
||||
resp, err := p.client.Do(httpReq)
|
||||
if err != nil {
|
||||
logger.Errorf(context.Background(), "[WeKnoraCloud] http poll task_id=%s failed: %v", taskID, err)
|
||||
return nil, fmt.Errorf("http poll task failed: %w", err)
|
||||
}
|
||||
var taskResp weKnoraCloudAsyncTaskResponse
|
||||
@@ -182,10 +166,12 @@ func (p *WeKnoraCloudSignedDocumentReader) pollTaskResult(ctx context.Context, t
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
bodyBytes, _ := io.ReadAll(resp.Body)
|
||||
err = fmt.Errorf("http poll task status %d: %s", resp.StatusCode, string(bodyBytes))
|
||||
logger.Errorf(context.Background(), "[WeKnoraCloud] poll task_id=%s status %d: %s", taskID, resp.StatusCode, string(bodyBytes))
|
||||
return
|
||||
}
|
||||
if decodeErr := json.NewDecoder(resp.Body).Decode(&taskResp); decodeErr != nil {
|
||||
err = fmt.Errorf("http decode task response: %w", decodeErr)
|
||||
logger.Errorf(context.Background(), "[WeKnoraCloud] poll task_id=%s decode response: %v", taskID, decodeErr)
|
||||
}
|
||||
}()
|
||||
if err != nil {
|
||||
@@ -194,27 +180,40 @@ func (p *WeKnoraCloudSignedDocumentReader) pollTaskResult(ctx context.Context, t
|
||||
switch taskResp.Status {
|
||||
case "completed":
|
||||
if taskResp.Result == nil {
|
||||
logger.Infof(ctx, "[WeKnoraCloud] task_id=%s completed with no result payload", taskID)
|
||||
return &types.ReadResult{}, nil
|
||||
}
|
||||
return fromHTTPReadResponse(taskResp.Result), nil
|
||||
res := fromHTTPReadResponse(taskResp.Result)
|
||||
if res.Error != "" {
|
||||
logger.Errorf(ctx, "[WeKnoraCloud] task_id=%s completed with result.error: %s", taskID, res.Error)
|
||||
} else {
|
||||
logger.Debugf(ctx, "[WeKnoraCloud] task_id=%s completed ok markdownLen=%d", taskID, len(res.MarkdownContent))
|
||||
}
|
||||
return res, nil
|
||||
case "failed":
|
||||
if taskResp.Error != "" {
|
||||
logger.Errorf(context.Background(), "[WeKnoraCloud] task_id=%s failed: %s", taskID, taskResp.Error)
|
||||
return nil, fmt.Errorf("weknoracloud docreader task failed: %s", taskResp.Error)
|
||||
}
|
||||
logger.Errorf(context.Background(), "[WeKnoraCloud] task_id=%s failed: %s", taskID, taskResp.Message)
|
||||
return nil, fmt.Errorf("weknoracloud docreader task failed: %s", taskResp.Message)
|
||||
case "cancelled":
|
||||
if taskResp.Error != "" {
|
||||
logger.Errorf(context.Background(), "[WeKnoraCloud] task_id=%s cancelled: %s", taskID, taskResp.Error)
|
||||
return nil, fmt.Errorf("weknoracloud docreader task cancelled: %s", taskResp.Error)
|
||||
}
|
||||
logger.Errorf(context.Background(), "[WeKnoraCloud] task_id=%s cancelled", taskID)
|
||||
return nil, fmt.Errorf("weknoracloud docreader task cancelled")
|
||||
}
|
||||
if err := pollCtx.Err(); err != nil {
|
||||
logger.Errorf(ctx, "[WeKnoraCloud] poll task_id=%s aborted before sleep: %v", taskID, err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Exponential backoff: multiply by 1.5 each time, cap at maxPollInterval
|
||||
select {
|
||||
case <-pollCtx.Done():
|
||||
logger.Errorf(ctx, "[WeKnoraCloud] poll task_id=%s stopped: %v", taskID, pollCtx.Err())
|
||||
return nil, pollCtx.Err()
|
||||
case <-time.After(currentInterval):
|
||||
// Update interval for next iteration
|
||||
@@ -234,6 +233,7 @@ func (p *WeKnoraCloudSignedDocumentReader) newSignedRequest(ctx context.Context,
|
||||
}
|
||||
httpReq, err := http.NewRequestWithContext(ctx, method, url, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
logger.Errorf(context.Background(), "[WeKnoraCloud] http new request %s %s: %v", method, url, err)
|
||||
return nil, fmt.Errorf("http new request: %w", err)
|
||||
}
|
||||
httpReq.Header.Set("Content-Type", "application/json")
|
||||
|
||||
Reference in New Issue
Block a user