fix: harden document pipeline concurrency and diagnostics

- docreader: serialize all pdfium access behind a global lock. pypdfium2
  is process-global and not thread-safe, so concurrent PDF uploads could
  corrupt its shared state and deadlock the whole process (requests stuck
  in "Parsing document" forever). Non-PDF parsers still run concurrently.
- logger: strip ANSI color escapes from file logs so on-disk logs stay
  plain text while stdout keeps terminal colors.
- storage: log baseDir/resolvedPath on file-open failure and when the
  multimodal service resolves a file service, so a LOCAL_STORAGE_BASE_DIR
  mismatch between writer and reader is immediately visible.
- docker: set TZ (default Asia/Shanghai) for the docreader container.
This commit is contained in:
wizardchen
2026-07-09 13:24:38 +08:00
committed by lyingbug
parent d287c70358
commit b6d3420a5d
7 changed files with 63 additions and 3 deletions
+1
View File
@@ -247,6 +247,7 @@ services:
volumes:
- docreader-tmp-dev:/tmp/docreader
environment:
- TZ=${TZ:-Asia/Shanghai}
- DOCREADER_IMAGE_OUTPUT_DIR=/tmp/docreader
- DOCREADER_PDF_FORCE_SCANNED=${DOCREADER_PDF_FORCE_SCANNED:-false}
- MAX_FILE_SIZE_MB=${MAX_FILE_SIZE_MB:-}
+1
View File
@@ -239,6 +239,7 @@ services:
volumes:
- docreader-tmp:/tmp/docreader
environment:
- TZ=${TZ:-Asia/Shanghai}
- DOCREADER_IMAGE_OUTPUT_DIR=/tmp/docreader
- DOCREADER_PDF_FORCE_SCANNED=${DOCREADER_PDF_FORCE_SCANNED:-false}
- MAX_FILE_SIZE_MB=${MAX_FILE_SIZE_MB:-}
+25 -1
View File
@@ -22,6 +22,7 @@ import logging
import os
import re
import statistics
import threading
from docreader.config import CONFIG
from docreader.models.document import Document
@@ -30,6 +31,17 @@ from docreader.parser.concurrency import parser_worker_limit
logger = logging.getLogger(__name__)
# pdfium (the C library behind pypdfium2) is process-global and NOT
# thread-safe: two gRPC worker threads parsing PDFs at the same time corrupt
# its shared state and can deadlock the whole process indefinitely (observed
# as requests stuck in "Parsing document with PDFParser" forever, taking down
# all subsequent uploads too). Every pdfium touch — text extraction, page
# rendering, image extraction — must be serialized behind this single global
# lock. Concurrent PDF uploads are then processed one at a time instead of
# hanging. Non-PDF parsers (docx, xlsx, ...) are unaffected and still run
# concurrently across the gRPC thread pool.
_PDFIUM_LOCK = threading.Lock()
def _env_float(name: str, default: float) -> float:
v = os.environ.get(name)
@@ -1313,7 +1325,11 @@ class PDFScannedParser(BaseParser):
)
try:
with parser_worker_limit("pdf_render", CONFIG.pdf_render_max_workers):
# pdfium lock first, then the render worker limit — same order as
# PDFParser._route so the two never deadlock against each other.
with _PDFIUM_LOCK, parser_worker_limit(
"pdf_render", CONFIG.pdf_render_max_workers
):
pdf = pdfium.PdfDocument(content)
try:
page_count = len(pdf)
@@ -1413,6 +1429,14 @@ class PDFParser(BaseParser):
).parse_into_text(content)
def _route(self, content: bytes) -> Document:
# Serialize all pdfium work: see _PDFIUM_LOCK. Holding it for the whole
# route (both the text pass and the render pass) is what prevents the
# concurrent-upload deadlock; the trailing markdown assembly is cheap
# pure-Python so keeping it inside the lock costs nothing meaningful.
with _PDFIUM_LOCK:
return self._route_locked(content)
def _route_locked(self, content: bytes) -> Document:
import pypdfium2 as pdfium
import pypdfium2.raw as pdfium_r
+4 -1
View File
@@ -120,7 +120,10 @@ func (s *localFileService) GetFile(ctx context.Context, filePath string) (io.Rea
file, err := os.Open(resolved)
if err != nil {
logger.Errorf(ctx, "Failed to open file: %v", err)
// baseDir/resolved are logged so a storage base-dir mismatch (e.g.
// writer and reader started with different LOCAL_STORAGE_BASE_DIR)
// is immediately visible instead of just "no such file or directory".
logger.Errorf(ctx, "Failed to open file: baseDir=%s resolvedPath=%s err=%v", s.baseDir, resolved, err)
return nil, fmt.Errorf("failed to open file: %w", err)
}
@@ -510,6 +510,8 @@ func (s *ImageMultimodalService) resolveFileServiceForPayload(ctx context.Contex
}
baseDir := strings.TrimSpace(os.Getenv("LOCAL_STORAGE_BASE_DIR"))
logger.Infof(ctx, "[ImageMultimodal] resolving file service: tenant=%d provider=%q LOCAL_STORAGE_BASE_DIR=%q imageURL=%s",
payload.TenantID, provider, baseDir, payload.ImageURL)
fileSvc, _, svcErr := filesvc.NewFileServiceFromStorageConfig(provider, tenant.StorageEngineConfig, baseDir)
if svcErr != nil {
logger.Warnf(ctx, "[ImageMultimodal] resolve file service failed (falling back to default): tenant=%d provider=%s err=%v",
+14 -1
View File
@@ -7,6 +7,7 @@ import (
"os"
"path"
"path/filepath"
"regexp"
"runtime"
"sort"
"strings"
@@ -23,8 +24,20 @@ var appLogger = logrus.New()
var (
loggerMu sync.Mutex
activeLogFile io.WriteCloser
ansiEscapeRE = regexp.MustCompile(`\x1b\[[0-9;]*[a-zA-Z]`)
)
// ansiStripWriter removes ANSI color/style sequences so file logs stay plain text
// while stdout can still render colors in a terminal.
type ansiStripWriter struct {
w io.Writer
}
func (s *ansiStripWriter) Write(p []byte) (int, error) {
_, err := s.w.Write(ansiEscapeRE.ReplaceAll(p, nil))
return len(p), err
}
// LogLevel 日志级别类型
type LogLevel string
@@ -241,7 +254,7 @@ func ConfigureFromEnv() {
fmt.Fprintf(os.Stderr, "logger: failed to open log file %s: %v\n", logPath, err)
} else {
activeLogFile = file
writer = io.MultiWriter(os.Stdout, file)
writer = io.MultiWriter(os.Stdout, &ansiStripWriter{w: file})
}
}
+16
View File
@@ -19,6 +19,22 @@ func newEntry(level logrus.Level, msg string, data logrus.Fields) *logrus.Entry
return e
}
func TestAnsiStripWriter(t *testing.T) {
var buf strings.Builder
w := &ansiStripWriter{w: &buf}
in := []byte("\x1b[32mINFO\x1b[0m hello \x1b[31mERROR\x1b[0m")
n, err := w.Write(in)
if err != nil {
t.Fatalf("Write error: %v", err)
}
if n != len(in) {
t.Fatalf("Write n = %d, want %d", n, len(in))
}
if got := buf.String(); got != "INFO hello ERROR" {
t.Fatalf("stripped output = %q, want %q", got, "INFO hello ERROR")
}
}
func TestFormat_DefaultModeUnchanged(t *testing.T) {
f := &CustomFormatter{} // no template, no color
entry := newEntry(logrus.InfoLevel, "hello", logrus.Fields{