fix: harden document pipeline concurrency and diagnostics
- docreader: serialize all pdfium access behind a global lock. pypdfium2 is process-global and not thread-safe, so concurrent PDF uploads could corrupt its shared state and deadlock the whole process (requests stuck in "Parsing document" forever). Non-PDF parsers still run concurrently. - logger: strip ANSI color escapes from file logs so on-disk logs stay plain text while stdout keeps terminal colors. - storage: log baseDir/resolvedPath on file-open failure and when the multimodal service resolves a file service, so a LOCAL_STORAGE_BASE_DIR mismatch between writer and reader is immediately visible. - docker: set TZ (default Asia/Shanghai) for the docreader container.
This commit is contained in:
@@ -247,6 +247,7 @@ services:
|
||||
volumes:
|
||||
- docreader-tmp-dev:/tmp/docreader
|
||||
environment:
|
||||
- TZ=${TZ:-Asia/Shanghai}
|
||||
- DOCREADER_IMAGE_OUTPUT_DIR=/tmp/docreader
|
||||
- DOCREADER_PDF_FORCE_SCANNED=${DOCREADER_PDF_FORCE_SCANNED:-false}
|
||||
- MAX_FILE_SIZE_MB=${MAX_FILE_SIZE_MB:-}
|
||||
|
||||
@@ -239,6 +239,7 @@ services:
|
||||
volumes:
|
||||
- docreader-tmp:/tmp/docreader
|
||||
environment:
|
||||
- TZ=${TZ:-Asia/Shanghai}
|
||||
- DOCREADER_IMAGE_OUTPUT_DIR=/tmp/docreader
|
||||
- DOCREADER_PDF_FORCE_SCANNED=${DOCREADER_PDF_FORCE_SCANNED:-false}
|
||||
- MAX_FILE_SIZE_MB=${MAX_FILE_SIZE_MB:-}
|
||||
|
||||
@@ -22,6 +22,7 @@ import logging
|
||||
import os
|
||||
import re
|
||||
import statistics
|
||||
import threading
|
||||
|
||||
from docreader.config import CONFIG
|
||||
from docreader.models.document import Document
|
||||
@@ -30,6 +31,17 @@ from docreader.parser.concurrency import parser_worker_limit
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# pdfium (the C library behind pypdfium2) is process-global and NOT
|
||||
# thread-safe: two gRPC worker threads parsing PDFs at the same time corrupt
|
||||
# its shared state and can deadlock the whole process indefinitely (observed
|
||||
# as requests stuck in "Parsing document with PDFParser" forever, taking down
|
||||
# all subsequent uploads too). Every pdfium touch — text extraction, page
|
||||
# rendering, image extraction — must be serialized behind this single global
|
||||
# lock. Concurrent PDF uploads are then processed one at a time instead of
|
||||
# hanging. Non-PDF parsers (docx, xlsx, ...) are unaffected and still run
|
||||
# concurrently across the gRPC thread pool.
|
||||
_PDFIUM_LOCK = threading.Lock()
|
||||
|
||||
|
||||
def _env_float(name: str, default: float) -> float:
|
||||
v = os.environ.get(name)
|
||||
@@ -1313,7 +1325,11 @@ class PDFScannedParser(BaseParser):
|
||||
)
|
||||
|
||||
try:
|
||||
with parser_worker_limit("pdf_render", CONFIG.pdf_render_max_workers):
|
||||
# pdfium lock first, then the render worker limit — same order as
|
||||
# PDFParser._route so the two never deadlock against each other.
|
||||
with _PDFIUM_LOCK, parser_worker_limit(
|
||||
"pdf_render", CONFIG.pdf_render_max_workers
|
||||
):
|
||||
pdf = pdfium.PdfDocument(content)
|
||||
try:
|
||||
page_count = len(pdf)
|
||||
@@ -1413,6 +1429,14 @@ class PDFParser(BaseParser):
|
||||
).parse_into_text(content)
|
||||
|
||||
def _route(self, content: bytes) -> Document:
|
||||
# Serialize all pdfium work: see _PDFIUM_LOCK. Holding it for the whole
|
||||
# route (both the text pass and the render pass) is what prevents the
|
||||
# concurrent-upload deadlock; the trailing markdown assembly is cheap
|
||||
# pure-Python so keeping it inside the lock costs nothing meaningful.
|
||||
with _PDFIUM_LOCK:
|
||||
return self._route_locked(content)
|
||||
|
||||
def _route_locked(self, content: bytes) -> Document:
|
||||
import pypdfium2 as pdfium
|
||||
import pypdfium2.raw as pdfium_r
|
||||
|
||||
|
||||
@@ -120,7 +120,10 @@ func (s *localFileService) GetFile(ctx context.Context, filePath string) (io.Rea
|
||||
|
||||
file, err := os.Open(resolved)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to open file: %v", err)
|
||||
// baseDir/resolved are logged so a storage base-dir mismatch (e.g.
|
||||
// writer and reader started with different LOCAL_STORAGE_BASE_DIR)
|
||||
// is immediately visible instead of just "no such file or directory".
|
||||
logger.Errorf(ctx, "Failed to open file: baseDir=%s resolvedPath=%s err=%v", s.baseDir, resolved, err)
|
||||
return nil, fmt.Errorf("failed to open file: %w", err)
|
||||
}
|
||||
|
||||
|
||||
@@ -510,6 +510,8 @@ func (s *ImageMultimodalService) resolveFileServiceForPayload(ctx context.Contex
|
||||
}
|
||||
|
||||
baseDir := strings.TrimSpace(os.Getenv("LOCAL_STORAGE_BASE_DIR"))
|
||||
logger.Infof(ctx, "[ImageMultimodal] resolving file service: tenant=%d provider=%q LOCAL_STORAGE_BASE_DIR=%q imageURL=%s",
|
||||
payload.TenantID, provider, baseDir, payload.ImageURL)
|
||||
fileSvc, _, svcErr := filesvc.NewFileServiceFromStorageConfig(provider, tenant.StorageEngineConfig, baseDir)
|
||||
if svcErr != nil {
|
||||
logger.Warnf(ctx, "[ImageMultimodal] resolve file service failed (falling back to default): tenant=%d provider=%s err=%v",
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"runtime"
|
||||
"sort"
|
||||
"strings"
|
||||
@@ -23,8 +24,20 @@ var appLogger = logrus.New()
|
||||
var (
|
||||
loggerMu sync.Mutex
|
||||
activeLogFile io.WriteCloser
|
||||
ansiEscapeRE = regexp.MustCompile(`\x1b\[[0-9;]*[a-zA-Z]`)
|
||||
)
|
||||
|
||||
// ansiStripWriter removes ANSI color/style sequences so file logs stay plain text
|
||||
// while stdout can still render colors in a terminal.
|
||||
type ansiStripWriter struct {
|
||||
w io.Writer
|
||||
}
|
||||
|
||||
func (s *ansiStripWriter) Write(p []byte) (int, error) {
|
||||
_, err := s.w.Write(ansiEscapeRE.ReplaceAll(p, nil))
|
||||
return len(p), err
|
||||
}
|
||||
|
||||
// LogLevel 日志级别类型
|
||||
type LogLevel string
|
||||
|
||||
@@ -241,7 +254,7 @@ func ConfigureFromEnv() {
|
||||
fmt.Fprintf(os.Stderr, "logger: failed to open log file %s: %v\n", logPath, err)
|
||||
} else {
|
||||
activeLogFile = file
|
||||
writer = io.MultiWriter(os.Stdout, file)
|
||||
writer = io.MultiWriter(os.Stdout, &ansiStripWriter{w: file})
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -19,6 +19,22 @@ func newEntry(level logrus.Level, msg string, data logrus.Fields) *logrus.Entry
|
||||
return e
|
||||
}
|
||||
|
||||
func TestAnsiStripWriter(t *testing.T) {
|
||||
var buf strings.Builder
|
||||
w := &ansiStripWriter{w: &buf}
|
||||
in := []byte("\x1b[32mINFO\x1b[0m hello \x1b[31mERROR\x1b[0m")
|
||||
n, err := w.Write(in)
|
||||
if err != nil {
|
||||
t.Fatalf("Write error: %v", err)
|
||||
}
|
||||
if n != len(in) {
|
||||
t.Fatalf("Write n = %d, want %d", n, len(in))
|
||||
}
|
||||
if got := buf.String(); got != "INFO hello ERROR" {
|
||||
t.Fatalf("stripped output = %q, want %q", got, "INFO hello ERROR")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFormat_DefaultModeUnchanged(t *testing.T) {
|
||||
f := &CustomFormatter{} // no template, no color
|
||||
entry := newEntry(logrus.InfoLevel, "hello", logrus.Fields{
|
||||
|
||||
Reference in New Issue
Block a user