feat: integrate ChatBot provider and docreader
This commit is contained in:
@@ -3,6 +3,7 @@ package docparser
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/models/provider"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
@@ -27,6 +28,7 @@ func RegisterEngine(e EngineRegistration) {
|
||||
func init() {
|
||||
RegisterEngine(&builtinEngine{})
|
||||
RegisterEngine(&simpleEngine{})
|
||||
RegisterEngine(&weKnoraCloudEngine{})
|
||||
RegisterEngine(&mineruEngine{})
|
||||
RegisterEngine(&mineruCloudEngine{})
|
||||
}
|
||||
@@ -54,6 +56,9 @@ func (e *builtinEngine) CheckAvailable(docreaderConnected bool, _ map[string]str
|
||||
// SimpleEngineName is the engine name for Go-native simple format handling.
|
||||
const SimpleEngineName = "simple"
|
||||
|
||||
// WeKnoraCloudEngineName is the engine name for WeKnoraCloud-backed document parsing.
|
||||
const WeKnoraCloudEngineName = "weknoracloud"
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// simple — Go handles md/txt/csv natively, no external service needed.
|
||||
// Distinct from docreader's "builtin" which uses Python libraries for
|
||||
@@ -73,6 +78,24 @@ func (e *simpleEngine) CheckAvailable(_ bool, _ map[string]string) (bool, string
|
||||
return true, ""
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// weknoracloud — Tenant-scoped WeKnoraCloud docreader with signed requests.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type weKnoraCloudEngine struct{}
|
||||
|
||||
func (e *weKnoraCloudEngine) Name() string { return WeKnoraCloudEngineName }
|
||||
func (e *weKnoraCloudEngine) Description() string { return "WeKnoraCloud document reader" }
|
||||
func (e *weKnoraCloudEngine) FileTypes(_ bool) []string {
|
||||
return weKnoraCloudSupportedFileTypes()
|
||||
}
|
||||
func (e *weKnoraCloudEngine) CheckAvailable(docreaderConnected bool, overrides map[string]string) (bool, string) {
|
||||
if docreaderConnected && isWeKnoraCloudDocReaderAddr(overrides["docreader_addr"]) {
|
||||
return true, ""
|
||||
}
|
||||
return false, "WeKnoraCloud document reader is not initialized"
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// mineru — Go-native, calls self-hosted MinerU API directly
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -169,3 +192,7 @@ func ListAllEngines(docreaderConnected bool, overrides map[string]string, remote
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
func isWeKnoraCloudDocReaderAddr(addr string) bool {
|
||||
return strings.TrimSuffix(strings.TrimSpace(addr), "/") == strings.TrimRight(provider.WeKnoraCloudBaseURL, "/")+"/api/v1/doc/reader"
|
||||
}
|
||||
|
||||
@@ -0,0 +1,245 @@
|
||||
package docparser
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/models/utils"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
// WeKnoraCloudSignedDocumentReader implements the docreader HTTP protocol with WeKnoraCloud signing.
|
||||
type WeKnoraCloudSignedDocumentReader struct {
|
||||
baseURL string
|
||||
appID string
|
||||
apiKey string
|
||||
client *http.Client
|
||||
initialPollInterval time.Duration
|
||||
maxPollInterval time.Duration
|
||||
pollTimeout time.Duration
|
||||
}
|
||||
|
||||
func NewWeKnoraCloudSignedDocumentReader(baseURL, appID, apiKey string) (*WeKnoraCloudSignedDocumentReader, error) {
|
||||
baseURL = strings.TrimSuffix(strings.TrimSpace(baseURL), "/")
|
||||
if baseURL == "" {
|
||||
return nil, fmt.Errorf("docreader base URL is required")
|
||||
}
|
||||
if appID == "" {
|
||||
return nil, fmt.Errorf("WeKnoraCloud appID is required")
|
||||
}
|
||||
if apiKey == "" {
|
||||
return nil, fmt.Errorf("WeKnoraCloud apiKey is required")
|
||||
}
|
||||
return &WeKnoraCloudSignedDocumentReader{
|
||||
baseURL: baseURL,
|
||||
appID: appID,
|
||||
apiKey: apiKey,
|
||||
initialPollInterval: 500 * time.Millisecond,
|
||||
maxPollInterval: 10 * time.Second,
|
||||
pollTimeout: 20 * time.Minute,
|
||||
client: &http.Client{
|
||||
Timeout: 500 * time.Minute,
|
||||
Transport: &http.Transport{
|
||||
MaxIdleConns: 10,
|
||||
IdleConnTimeout: 90 * time.Second,
|
||||
MaxIdleConnsPerHost: 5,
|
||||
},
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (p *WeKnoraCloudSignedDocumentReader) Reconnect(addr string) error {
|
||||
p.baseURL = strings.TrimSuffix(strings.TrimSpace(addr), "/")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *WeKnoraCloudSignedDocumentReader) IsConnected() bool { return p.baseURL != "" }
|
||||
|
||||
func (p *WeKnoraCloudSignedDocumentReader) ListEngines(ctx context.Context, overrides map[string]string) ([]types.ParserEngineInfo, error) {
|
||||
if !p.IsConnected() {
|
||||
return nil, errNotConnected
|
||||
}
|
||||
return []types.ParserEngineInfo{{
|
||||
Name: WeKnoraCloudEngineName,
|
||||
Description: "WeKnoraCloud signed docreader",
|
||||
FileTypes: weKnoraCloudSupportedFileTypes(),
|
||||
Available: true,
|
||||
}}, nil
|
||||
}
|
||||
|
||||
func (p *WeKnoraCloudSignedDocumentReader) Read(ctx context.Context, req *types.ReadRequest) (*types.ReadResult, error) {
|
||||
if !p.IsConnected() {
|
||||
return nil, errNotConnected
|
||||
}
|
||||
body := httpReadRequest{
|
||||
FileName: req.FileName,
|
||||
FileType: req.FileType,
|
||||
URL: req.URL,
|
||||
Title: req.Title,
|
||||
RequestID: req.RequestID,
|
||||
Config: &httpReadConfig{
|
||||
ParserEngine: req.ParserEngine,
|
||||
ParserEngineOverrides: req.ParserEngineOverrides,
|
||||
},
|
||||
}
|
||||
if len(req.FileContent) > 0 {
|
||||
body.FileContent = base64.StdEncoding.EncodeToString(req.FileContent)
|
||||
}
|
||||
jsonBody, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("http marshal read request: %w", err)
|
||||
}
|
||||
httpReq, err := p.newSignedRequest(ctx, http.MethodPost, p.baseURL, jsonBody)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resp, err := p.client.Do(httpReq)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("http read failed: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusAccepted && resp.StatusCode != http.StatusOK {
|
||||
bodyBytes, _ := io.ReadAll(resp.Body)
|
||||
return nil, fmt.Errorf("http read status %d: %s", resp.StatusCode, string(bodyBytes))
|
||||
}
|
||||
var submit weKnoraCloudAsyncSubmitResponse
|
||||
if err := json.NewDecoder(resp.Body).Decode(&submit); err != nil {
|
||||
return nil, fmt.Errorf("http decode read submit response: %w", err)
|
||||
}
|
||||
if strings.TrimSpace(submit.TaskID) == "" {
|
||||
return nil, fmt.Errorf("weknoracloud docreader submit response missing task_id")
|
||||
}
|
||||
return p.pollTaskResult(ctx, submit.TaskID)
|
||||
}
|
||||
|
||||
type weKnoraCloudAsyncSubmitResponse struct {
|
||||
TaskID string `json:"task_id"`
|
||||
Status string `json:"status"`
|
||||
Message string `json:"message"`
|
||||
CreatedAt int64 `json:"created_at"`
|
||||
}
|
||||
|
||||
type weKnoraCloudAsyncTaskResponse struct {
|
||||
TaskID string `json:"task_id"`
|
||||
Status string `json:"status"`
|
||||
Message string `json:"message"`
|
||||
Progress float64 `json:"progress"`
|
||||
Result *httpReadResponse `json:"result"`
|
||||
Error string `json:"error"`
|
||||
CreatedAt int64 `json:"created_at"`
|
||||
UpdatedAt int64 `json:"updated_at"`
|
||||
}
|
||||
|
||||
func weKnoraCloudSupportedFileTypes() []string {
|
||||
seen := map[string]struct{}{}
|
||||
var fileTypes []string
|
||||
for _, engine := range localEngines {
|
||||
if engine.Name() == WeKnoraCloudEngineName {
|
||||
continue
|
||||
}
|
||||
for _, fileType := range engine.FileTypes(true) {
|
||||
if _, ok := seen[fileType]; ok {
|
||||
continue
|
||||
}
|
||||
seen[fileType] = struct{}{}
|
||||
fileTypes = append(fileTypes, fileType)
|
||||
}
|
||||
}
|
||||
sort.Strings(fileTypes)
|
||||
return fileTypes
|
||||
}
|
||||
|
||||
func (p *WeKnoraCloudSignedDocumentReader) pollTaskResult(ctx context.Context, taskID string) (*types.ReadResult, error) {
|
||||
pollCtx := ctx
|
||||
if _, ok := ctx.Deadline(); !ok && p.pollTimeout > 0 {
|
||||
var cancel context.CancelFunc
|
||||
pollCtx, cancel = context.WithTimeout(ctx, p.pollTimeout)
|
||||
defer cancel()
|
||||
}
|
||||
statusURL := strings.TrimSuffix(p.baseURL, "/reader") + "/" + taskID
|
||||
currentInterval := p.initialPollInterval
|
||||
for {
|
||||
httpReq, err := p.newSignedRequest(pollCtx, http.MethodGet, statusURL, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resp, err := p.client.Do(httpReq)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("http poll task failed: %w", err)
|
||||
}
|
||||
var taskResp weKnoraCloudAsyncTaskResponse
|
||||
func() {
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
bodyBytes, _ := io.ReadAll(resp.Body)
|
||||
err = fmt.Errorf("http poll task status %d: %s", resp.StatusCode, string(bodyBytes))
|
||||
return
|
||||
}
|
||||
if decodeErr := json.NewDecoder(resp.Body).Decode(&taskResp); decodeErr != nil {
|
||||
err = fmt.Errorf("http decode task response: %w", decodeErr)
|
||||
}
|
||||
}()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
switch taskResp.Status {
|
||||
case "completed":
|
||||
if taskResp.Result == nil {
|
||||
return &types.ReadResult{}, nil
|
||||
}
|
||||
return fromHTTPReadResponse(taskResp.Result), nil
|
||||
case "failed":
|
||||
if taskResp.Error != "" {
|
||||
return nil, fmt.Errorf("weknoracloud docreader task failed: %s", taskResp.Error)
|
||||
}
|
||||
return nil, fmt.Errorf("weknoracloud docreader task failed: %s", taskResp.Message)
|
||||
case "cancelled":
|
||||
if taskResp.Error != "" {
|
||||
return nil, fmt.Errorf("weknoracloud docreader task cancelled: %s", taskResp.Error)
|
||||
}
|
||||
return nil, fmt.Errorf("weknoracloud docreader task cancelled")
|
||||
}
|
||||
if err := pollCtx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Exponential backoff: multiply by 1.5 each time, cap at maxPollInterval
|
||||
select {
|
||||
case <-pollCtx.Done():
|
||||
return nil, pollCtx.Err()
|
||||
case <-time.After(currentInterval):
|
||||
// Update interval for next iteration
|
||||
nextInterval := time.Duration(float64(currentInterval) * 1.5)
|
||||
if nextInterval > p.maxPollInterval {
|
||||
nextInterval = p.maxPollInterval
|
||||
}
|
||||
currentInterval = nextInterval
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (p *WeKnoraCloudSignedDocumentReader) newSignedRequest(ctx context.Context, method, url string, body []byte) (*http.Request, error) {
|
||||
requestID := uuid.New().String()
|
||||
if len(body) == 0 {
|
||||
body = []byte("{}")
|
||||
}
|
||||
httpReq, err := http.NewRequestWithContext(ctx, method, url, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("http new request: %w", err)
|
||||
}
|
||||
httpReq.Header.Set("Content-Type", "application/json")
|
||||
httpReq.ContentLength = int64(len(body))
|
||||
for k, v := range utils.Sign(p.appID, p.apiKey, requestID, string(body)) {
|
||||
httpReq.Header.Set(k, v)
|
||||
}
|
||||
return httpReq, nil
|
||||
}
|
||||
Reference in New Issue
Block a user