feat: enhance document loading and state management in doc-content component
- Introduced a loading state (`loadingChunks`) to manage document chunk loading more effectively. - Updated the scroll handling logic to prevent multiple requests while loading chunks. - Adjusted the pagination logic to ensure correct page increments based on the total document count. This update improves the user experience by ensuring smoother document loading and better state management during scrolling interactions.
This commit is contained in:
@@ -0,0 +1,110 @@
|
||||
package docparser
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// stringOr returns val (trimmed) if non-empty, otherwise fallback.
|
||||
func stringOr(val, fallback string) string {
|
||||
val = strings.TrimSpace(val)
|
||||
if val == "" {
|
||||
return fallback
|
||||
}
|
||||
return val
|
||||
}
|
||||
|
||||
// parseBoolOr parses a truthy string ("true","1","yes"), returning fallback on empty.
|
||||
func parseBoolOr(val string, fallback bool) bool {
|
||||
val = strings.ToLower(strings.TrimSpace(val))
|
||||
if val == "" {
|
||||
return fallback
|
||||
}
|
||||
return val == "true" || val == "1" || val == "yes"
|
||||
}
|
||||
|
||||
// firstNonEmpty returns the first non-empty string, or "" if all are empty.
|
||||
func firstNonEmpty(vals ...string) string {
|
||||
for _, v := range vals {
|
||||
if v != "" {
|
||||
return v
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// sleepCtx sleeps for d but returns early if ctx is cancelled.
|
||||
func sleepCtx(ctx context.Context, d time.Duration) {
|
||||
t := time.NewTimer(d)
|
||||
defer t.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-t.C:
|
||||
}
|
||||
}
|
||||
|
||||
// logResponseStructure recursively logs the structure of an API response,
|
||||
// truncating large string values. label identifies the subsystem (e.g. "MinerU").
|
||||
func logResponseStructure(label string, obj interface{}, prefix string) {
|
||||
switch v := obj.(type) {
|
||||
case map[string]interface{}:
|
||||
keys := make([]string, 0, len(v))
|
||||
for k := range v {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
logger.Printf("DEBUG: [%s] %s = {object with %d keys: %s}", label, prefix, len(v), strings.Join(keys, ", "))
|
||||
for _, key := range keys {
|
||||
val := v[key]
|
||||
path := prefix + "." + key
|
||||
switch inner := val.(type) {
|
||||
case map[string]interface{}:
|
||||
logResponseStructure(label, inner, path)
|
||||
case []interface{}:
|
||||
logger.Printf("DEBUG: [%s] %s = [array with %d items]", label, path, len(inner))
|
||||
if len(inner) > 0 {
|
||||
logger.Printf("DEBUG: [%s] %s[0] type=%T", label, path, inner[0])
|
||||
if len(inner) <= 3 {
|
||||
for i, item := range inner {
|
||||
logResponseStructure(label, item, fmt.Sprintf("%s[%d]", path, i))
|
||||
}
|
||||
} else {
|
||||
logResponseStructure(label, inner[0], path+"[0]")
|
||||
logger.Printf("DEBUG: [%s] ... and %d more items in %s", label, len(inner)-1, path)
|
||||
}
|
||||
}
|
||||
case string:
|
||||
if len(inner) > 200 {
|
||||
logger.Printf("DEBUG: [%s] %s = string(%d chars): %.200s...", label, path, len(inner), inner)
|
||||
} else {
|
||||
logger.Printf("DEBUG: [%s] %s = %q", label, path, inner)
|
||||
}
|
||||
case float64:
|
||||
logger.Printf("DEBUG: [%s] %s = %v (number)", label, path, inner)
|
||||
case bool:
|
||||
logger.Printf("DEBUG: [%s] %s = %v (bool)", label, path, inner)
|
||||
case nil:
|
||||
logger.Printf("DEBUG: [%s] %s = null", label, path)
|
||||
default:
|
||||
logger.Printf("DEBUG: [%s] %s = %v (%T)", label, path, val, val)
|
||||
}
|
||||
}
|
||||
case []interface{}:
|
||||
logger.Printf("DEBUG: [%s] %s = [array with %d items]", label, prefix, len(v))
|
||||
if len(v) > 0 {
|
||||
if len(v) <= 3 {
|
||||
for i, item := range v {
|
||||
logResponseStructure(label, item, fmt.Sprintf("%s[%d]", prefix, i))
|
||||
}
|
||||
} else {
|
||||
logResponseStructure(label, v[0], prefix+"[0]")
|
||||
logger.Printf("DEBUG: [%s] ... and %d more items in %s", label, len(v)-1, prefix)
|
||||
}
|
||||
}
|
||||
default:
|
||||
logger.Printf("DEBUG: [%s] %s = %v (%T)", label, prefix, v, v)
|
||||
}
|
||||
}
|
||||
@@ -2,20 +2,16 @@ package docparser
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/utils"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
var httpImageClient = &http.Client{Timeout: 60 * time.Second}
|
||||
|
||||
// StoredImage describes an image that has been saved to local storage.
|
||||
type StoredImage struct {
|
||||
OriginalRef string // reference in the original markdown
|
||||
@@ -179,19 +175,7 @@ func (r *ImageResolver) readImageBytes(
|
||||
}
|
||||
|
||||
func downloadImage(url string) ([]byte, error) {
|
||||
resp, err := httpImageClient.Get(url)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("HTTP GET: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("HTTP %d for %s", resp.StatusCode, url)
|
||||
}
|
||||
data, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read body: %w", err)
|
||||
}
|
||||
return data, nil
|
||||
return utils.DownloadBytes(url)
|
||||
}
|
||||
|
||||
func extFromMime(mime string) string {
|
||||
|
||||
@@ -297,7 +297,7 @@ func (c *MinerUCloudReader) fetchBatchStatus(ctx context.Context, batchID string
|
||||
// Pretty-print the structure to reveal all available fields
|
||||
var rawObj interface{}
|
||||
if err := json.Unmarshal(pollResp.Data.ExtractResult, &rawObj); err == nil {
|
||||
logCloudResponseStructure(rawObj, "extract_result")
|
||||
logResponseStructure("MinerUCloud", rawObj, "extract_result")
|
||||
}
|
||||
|
||||
// The extract_result can be either a single object or an array
|
||||
@@ -472,68 +472,7 @@ func readZipEntryBytes(f *zip.File) ([]byte, error) {
|
||||
return io.ReadAll(rc)
|
||||
}
|
||||
|
||||
// logCloudResponseStructure recursively logs the structure of the Cloud API response,
|
||||
// truncating large string values to reveal all available fields (e.g. caption, bbox, etc.).
|
||||
func logCloudResponseStructure(obj interface{}, prefix string) {
|
||||
switch v := obj.(type) {
|
||||
case map[string]interface{}:
|
||||
keys := make([]string, 0, len(v))
|
||||
for k := range v {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
logger.Printf("DEBUG: [MinerUCloud] %s = {object with %d keys: %s}", prefix, len(v), strings.Join(keys, ", "))
|
||||
for _, key := range keys {
|
||||
val := v[key]
|
||||
path := prefix + "." + key
|
||||
switch inner := val.(type) {
|
||||
case map[string]interface{}:
|
||||
logCloudResponseStructure(inner, path)
|
||||
case []interface{}:
|
||||
logger.Printf("DEBUG: [MinerUCloud] %s = [array with %d items]", path, len(inner))
|
||||
if len(inner) > 0 {
|
||||
logger.Printf("DEBUG: [MinerUCloud] %s[0] type=%T", path, inner[0])
|
||||
if len(inner) <= 3 {
|
||||
for i, item := range inner {
|
||||
logCloudResponseStructure(item, fmt.Sprintf("%s[%d]", path, i))
|
||||
}
|
||||
} else {
|
||||
logCloudResponseStructure(inner[0], path+"[0]")
|
||||
logger.Printf("DEBUG: [MinerUCloud] ... and %d more items in %s", len(inner)-1, path)
|
||||
}
|
||||
}
|
||||
case string:
|
||||
if len(inner) > 200 {
|
||||
logger.Printf("DEBUG: [MinerUCloud] %s = string(%d chars): %.200s...", path, len(inner), inner)
|
||||
} else {
|
||||
logger.Printf("DEBUG: [MinerUCloud] %s = %q", path, inner)
|
||||
}
|
||||
case float64:
|
||||
logger.Printf("DEBUG: [MinerUCloud] %s = %v (number)", path, inner)
|
||||
case bool:
|
||||
logger.Printf("DEBUG: [MinerUCloud] %s = %v (bool)", path, inner)
|
||||
case nil:
|
||||
logger.Printf("DEBUG: [MinerUCloud] %s = null", path)
|
||||
default:
|
||||
logger.Printf("DEBUG: [MinerUCloud] %s = %v (%T)", path, val, val)
|
||||
}
|
||||
}
|
||||
case []interface{}:
|
||||
logger.Printf("DEBUG: [MinerUCloud] %s = [array with %d items]", prefix, len(v))
|
||||
if len(v) > 0 {
|
||||
if len(v) <= 3 {
|
||||
for i, item := range v {
|
||||
logCloudResponseStructure(item, fmt.Sprintf("%s[%d]", prefix, i))
|
||||
}
|
||||
} else {
|
||||
logCloudResponseStructure(v[0], prefix+"[0]")
|
||||
logger.Printf("DEBUG: [MinerUCloud] ... and %d more items in %s", len(v)-1, prefix)
|
||||
}
|
||||
}
|
||||
default:
|
||||
logger.Printf("DEBUG: [MinerUCloud] %s = %v (%T)", prefix, v, v)
|
||||
}
|
||||
}
|
||||
// logCloudResponseStructure is now unified in helpers.go as logResponseStructure("MinerUCloud", ...)
|
||||
|
||||
// PingMinerUCloud checks if the MinerU Cloud API is reachable with the given API key.
|
||||
func PingMinerUCloud(apiKey, baseURL string) (bool, string) {
|
||||
@@ -568,22 +507,4 @@ func PingMinerUCloud(apiKey, baseURL string) (bool, string) {
|
||||
return true, ""
|
||||
}
|
||||
|
||||
// --- helpers ---
|
||||
|
||||
func firstNonEmpty(vals ...string) string {
|
||||
for _, v := range vals {
|
||||
if v != "" {
|
||||
return v
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func sleepCtx(ctx context.Context, d time.Duration) {
|
||||
t := time.NewTimer(d)
|
||||
defer t.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-t.C:
|
||||
}
|
||||
}
|
||||
// firstNonEmpty, sleepCtx are defined in helpers.go
|
||||
|
||||
@@ -25,12 +25,12 @@ var b64DataURIPattern = regexp.MustCompile(`^data:image/(\w+);base64,(.+)$`)
|
||||
|
||||
// MinerUReader calls a self-hosted MinerU API to read/convert documents.
|
||||
type MinerUReader struct {
|
||||
endpoint string
|
||||
backend string // "pipeline", "vlm-*", "hybrid-*"
|
||||
formulaEnable bool
|
||||
tableEnable bool
|
||||
ocrEnable bool
|
||||
language string
|
||||
endpoint string
|
||||
backend string // "pipeline", "vlm-*", "hybrid-*"
|
||||
formulaEnable bool
|
||||
tableEnable bool
|
||||
ocrEnable bool
|
||||
language string
|
||||
}
|
||||
|
||||
// NewMinerUReader creates a reader from ParserEngineOverrides.
|
||||
@@ -95,18 +95,18 @@ func (c *MinerUReader) callFileParse(ctx context.Context, content []byte) (strin
|
||||
|
||||
// Form fields
|
||||
fields := map[string]string{
|
||||
"return_md": "true",
|
||||
"return_images": "true",
|
||||
"table_enable": fmt.Sprintf("%v", c.tableEnable),
|
||||
"formula_enable": fmt.Sprintf("%v", c.formulaEnable),
|
||||
"parse_method": "ocr",
|
||||
"start_page_id": "0",
|
||||
"end_page_id": "99999",
|
||||
"backend": c.backend,
|
||||
"response_format_zip": "false",
|
||||
"return_middle_json": "false",
|
||||
"return_model_output": "false",
|
||||
"return_content_list": "true",
|
||||
"return_md": "true",
|
||||
"return_images": "true",
|
||||
"table_enable": fmt.Sprintf("%v", c.tableEnable),
|
||||
"formula_enable": fmt.Sprintf("%v", c.formulaEnable),
|
||||
"parse_method": "ocr",
|
||||
"start_page_id": "0",
|
||||
"end_page_id": "99999",
|
||||
"backend": c.backend,
|
||||
"response_format_zip": "false",
|
||||
"return_middle_json": "false",
|
||||
"return_model_output": "false",
|
||||
"return_content_list": "true",
|
||||
}
|
||||
if !c.ocrEnable {
|
||||
fields["parse_method"] = "txt"
|
||||
@@ -162,7 +162,7 @@ func (c *MinerUReader) callFileParse(ctx context.Context, content []byte) (strin
|
||||
// Also pretty-print the top-level structure (without large base64 blobs)
|
||||
var rawMap map[string]interface{}
|
||||
if err := json.Unmarshal(respBody, &rawMap); err == nil {
|
||||
c.logResponseStructure(rawMap, "")
|
||||
c.logMinerUResponseStructure(rawMap, "")
|
||||
}
|
||||
|
||||
var result mineruFileParseResponse
|
||||
@@ -225,60 +225,9 @@ func (c *MinerUReader) processImages(mdContent string, imagesB64 map[string]stri
|
||||
return refs, mdContent
|
||||
}
|
||||
|
||||
// logResponseStructure recursively logs the structure of the API response,
|
||||
// truncating large string values to reveal all available fields.
|
||||
func (c *MinerUReader) logResponseStructure(obj interface{}, prefix string) {
|
||||
switch v := obj.(type) {
|
||||
case map[string]interface{}:
|
||||
for key, val := range v {
|
||||
path := prefix + "." + key
|
||||
if path[0] == '.' {
|
||||
path = path[1:]
|
||||
}
|
||||
switch inner := val.(type) {
|
||||
case map[string]interface{}:
|
||||
logger.Printf("DEBUG: [MinerU] Response field: %s = {object with %d keys: %s}", path, len(inner), mapKeys(inner))
|
||||
c.logResponseStructure(inner, path)
|
||||
case []interface{}:
|
||||
logger.Printf("DEBUG: [MinerU] Response field: %s = [array with %d items]", path, len(inner))
|
||||
if len(inner) > 0 {
|
||||
logger.Printf("DEBUG: [MinerU] Response field: %s[0] type=%T", path, inner[0])
|
||||
if len(inner) <= 3 {
|
||||
for i, item := range inner {
|
||||
c.logResponseStructure(item, fmt.Sprintf("%s[%d]", path, i))
|
||||
}
|
||||
} else {
|
||||
c.logResponseStructure(inner[0], path+"[0]")
|
||||
logger.Printf("DEBUG: [MinerU] ... and %d more items in %s", len(inner)-1, path)
|
||||
}
|
||||
}
|
||||
case string:
|
||||
if len(inner) > 200 {
|
||||
logger.Printf("DEBUG: [MinerU] Response field: %s = string(%d chars): %.200s...", path, len(inner), inner)
|
||||
} else {
|
||||
logger.Printf("DEBUG: [MinerU] Response field: %s = %q", path, inner)
|
||||
}
|
||||
case float64:
|
||||
logger.Printf("DEBUG: [MinerU] Response field: %s = %v (number)", path, inner)
|
||||
case bool:
|
||||
logger.Printf("DEBUG: [MinerU] Response field: %s = %v (bool)", path, inner)
|
||||
case nil:
|
||||
logger.Printf("DEBUG: [MinerU] Response field: %s = null", path)
|
||||
default:
|
||||
logger.Printf("DEBUG: [MinerU] Response field: %s = %v (%T)", path, inner, inner)
|
||||
}
|
||||
}
|
||||
default:
|
||||
logger.Printf("DEBUG: [MinerU] Value at %s: %v (%T)", prefix, v, v)
|
||||
}
|
||||
}
|
||||
|
||||
func mapKeys(m map[string]interface{}) string {
|
||||
keys := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
return strings.Join(keys, ", ")
|
||||
// logMinerUResponseStructure logs the structure of the MinerU API response.
|
||||
func (c *MinerUReader) logMinerUResponseStructure(obj interface{}, prefix string) {
|
||||
logResponseStructure("MinerU", obj, prefix)
|
||||
}
|
||||
|
||||
// PingMinerU checks if the self-hosted MinerU service is reachable.
|
||||
@@ -309,21 +258,3 @@ func htmlToMarkdown(content string) string {
|
||||
}
|
||||
return md
|
||||
}
|
||||
|
||||
// --- helpers ---
|
||||
|
||||
func stringOr(val, fallback string) string {
|
||||
val = strings.TrimSpace(val)
|
||||
if val == "" {
|
||||
return fallback
|
||||
}
|
||||
return val
|
||||
}
|
||||
|
||||
func parseBoolOr(val string, fallback bool) bool {
|
||||
val = strings.ToLower(strings.TrimSpace(val))
|
||||
if val == "" {
|
||||
return fallback
|
||||
}
|
||||
return val == "true" || val == "1" || val == "yes"
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user