From b6d3420a5d74e1b838426a4fd2ed978e3e6931c0 Mon Sep 17 00:00:00 2001 From: wizardchen Date: Thu, 9 Jul 2026 13:24:38 +0800 Subject: [PATCH] fix: harden document pipeline concurrency and diagnostics - docreader: serialize all pdfium access behind a global lock. pypdfium2 is process-global and not thread-safe, so concurrent PDF uploads could corrupt its shared state and deadlock the whole process (requests stuck in "Parsing document" forever). Non-PDF parsers still run concurrently. - logger: strip ANSI color escapes from file logs so on-disk logs stay plain text while stdout keeps terminal colors. - storage: log baseDir/resolvedPath on file-open failure and when the multimodal service resolves a file service, so a LOCAL_STORAGE_BASE_DIR mismatch between writer and reader is immediately visible. - docker: set TZ (default Asia/Shanghai) for the docreader container. --- docker-compose.dev.yml | 1 + docker-compose.yml | 1 + docreader/parser/pdf_parser.py | 26 ++++++++++++++++++- internal/application/service/file/local.go | 5 +++- .../application/service/image_multimodal.go | 2 ++ internal/logger/logger.go | 15 ++++++++++- internal/logger/logger_test.go | 16 ++++++++++++ 7 files changed, 63 insertions(+), 3 deletions(-) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index d0f90dd6..1c743923 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -247,6 +247,7 @@ services: volumes: - docreader-tmp-dev:/tmp/docreader environment: + - TZ=${TZ:-Asia/Shanghai} - DOCREADER_IMAGE_OUTPUT_DIR=/tmp/docreader - DOCREADER_PDF_FORCE_SCANNED=${DOCREADER_PDF_FORCE_SCANNED:-false} - MAX_FILE_SIZE_MB=${MAX_FILE_SIZE_MB:-} diff --git a/docker-compose.yml b/docker-compose.yml index ce77c40f..698f5843 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -239,6 +239,7 @@ services: volumes: - docreader-tmp:/tmp/docreader environment: + - TZ=${TZ:-Asia/Shanghai} - DOCREADER_IMAGE_OUTPUT_DIR=/tmp/docreader - DOCREADER_PDF_FORCE_SCANNED=${DOCREADER_PDF_FORCE_SCANNED:-false} - MAX_FILE_SIZE_MB=${MAX_FILE_SIZE_MB:-} diff --git a/docreader/parser/pdf_parser.py b/docreader/parser/pdf_parser.py index 35ea202f..d94657f8 100644 --- a/docreader/parser/pdf_parser.py +++ b/docreader/parser/pdf_parser.py @@ -22,6 +22,7 @@ import logging import os import re import statistics +import threading from docreader.config import CONFIG from docreader.models.document import Document @@ -30,6 +31,17 @@ from docreader.parser.concurrency import parser_worker_limit logger = logging.getLogger(__name__) +# pdfium (the C library behind pypdfium2) is process-global and NOT +# thread-safe: two gRPC worker threads parsing PDFs at the same time corrupt +# its shared state and can deadlock the whole process indefinitely (observed +# as requests stuck in "Parsing document with PDFParser" forever, taking down +# all subsequent uploads too). Every pdfium touch — text extraction, page +# rendering, image extraction — must be serialized behind this single global +# lock. Concurrent PDF uploads are then processed one at a time instead of +# hanging. Non-PDF parsers (docx, xlsx, ...) are unaffected and still run +# concurrently across the gRPC thread pool. +_PDFIUM_LOCK = threading.Lock() + def _env_float(name: str, default: float) -> float: v = os.environ.get(name) @@ -1313,7 +1325,11 @@ class PDFScannedParser(BaseParser): ) try: - with parser_worker_limit("pdf_render", CONFIG.pdf_render_max_workers): + # pdfium lock first, then the render worker limit — same order as + # PDFParser._route so the two never deadlock against each other. + with _PDFIUM_LOCK, parser_worker_limit( + "pdf_render", CONFIG.pdf_render_max_workers + ): pdf = pdfium.PdfDocument(content) try: page_count = len(pdf) @@ -1413,6 +1429,14 @@ class PDFParser(BaseParser): ).parse_into_text(content) def _route(self, content: bytes) -> Document: + # Serialize all pdfium work: see _PDFIUM_LOCK. Holding it for the whole + # route (both the text pass and the render pass) is what prevents the + # concurrent-upload deadlock; the trailing markdown assembly is cheap + # pure-Python so keeping it inside the lock costs nothing meaningful. + with _PDFIUM_LOCK: + return self._route_locked(content) + + def _route_locked(self, content: bytes) -> Document: import pypdfium2 as pdfium import pypdfium2.raw as pdfium_r diff --git a/internal/application/service/file/local.go b/internal/application/service/file/local.go index d7092796..80f6fa80 100644 --- a/internal/application/service/file/local.go +++ b/internal/application/service/file/local.go @@ -120,7 +120,10 @@ func (s *localFileService) GetFile(ctx context.Context, filePath string) (io.Rea file, err := os.Open(resolved) if err != nil { - logger.Errorf(ctx, "Failed to open file: %v", err) + // baseDir/resolved are logged so a storage base-dir mismatch (e.g. + // writer and reader started with different LOCAL_STORAGE_BASE_DIR) + // is immediately visible instead of just "no such file or directory". + logger.Errorf(ctx, "Failed to open file: baseDir=%s resolvedPath=%s err=%v", s.baseDir, resolved, err) return nil, fmt.Errorf("failed to open file: %w", err) } diff --git a/internal/application/service/image_multimodal.go b/internal/application/service/image_multimodal.go index e6e52849..f0a49a66 100644 --- a/internal/application/service/image_multimodal.go +++ b/internal/application/service/image_multimodal.go @@ -510,6 +510,8 @@ func (s *ImageMultimodalService) resolveFileServiceForPayload(ctx context.Contex } baseDir := strings.TrimSpace(os.Getenv("LOCAL_STORAGE_BASE_DIR")) + logger.Infof(ctx, "[ImageMultimodal] resolving file service: tenant=%d provider=%q LOCAL_STORAGE_BASE_DIR=%q imageURL=%s", + payload.TenantID, provider, baseDir, payload.ImageURL) fileSvc, _, svcErr := filesvc.NewFileServiceFromStorageConfig(provider, tenant.StorageEngineConfig, baseDir) if svcErr != nil { logger.Warnf(ctx, "[ImageMultimodal] resolve file service failed (falling back to default): tenant=%d provider=%s err=%v", diff --git a/internal/logger/logger.go b/internal/logger/logger.go index 302a4761..636728b6 100644 --- a/internal/logger/logger.go +++ b/internal/logger/logger.go @@ -7,6 +7,7 @@ import ( "os" "path" "path/filepath" + "regexp" "runtime" "sort" "strings" @@ -23,8 +24,20 @@ var appLogger = logrus.New() var ( loggerMu sync.Mutex activeLogFile io.WriteCloser + ansiEscapeRE = regexp.MustCompile(`\x1b\[[0-9;]*[a-zA-Z]`) ) +// ansiStripWriter removes ANSI color/style sequences so file logs stay plain text +// while stdout can still render colors in a terminal. +type ansiStripWriter struct { + w io.Writer +} + +func (s *ansiStripWriter) Write(p []byte) (int, error) { + _, err := s.w.Write(ansiEscapeRE.ReplaceAll(p, nil)) + return len(p), err +} + // LogLevel 日志级别类型 type LogLevel string @@ -241,7 +254,7 @@ func ConfigureFromEnv() { fmt.Fprintf(os.Stderr, "logger: failed to open log file %s: %v\n", logPath, err) } else { activeLogFile = file - writer = io.MultiWriter(os.Stdout, file) + writer = io.MultiWriter(os.Stdout, &ansiStripWriter{w: file}) } } diff --git a/internal/logger/logger_test.go b/internal/logger/logger_test.go index e2fa92e6..841b2bb9 100644 --- a/internal/logger/logger_test.go +++ b/internal/logger/logger_test.go @@ -19,6 +19,22 @@ func newEntry(level logrus.Level, msg string, data logrus.Fields) *logrus.Entry return e } +func TestAnsiStripWriter(t *testing.T) { + var buf strings.Builder + w := &ansiStripWriter{w: &buf} + in := []byte("\x1b[32mINFO\x1b[0m hello \x1b[31mERROR\x1b[0m") + n, err := w.Write(in) + if err != nil { + t.Fatalf("Write error: %v", err) + } + if n != len(in) { + t.Fatalf("Write n = %d, want %d", n, len(in)) + } + if got := buf.String(); got != "INFO hello ERROR" { + t.Fatalf("stripped output = %q, want %q", got, "INFO hello ERROR") + } +} + func TestFormat_DefaultModeUnchanged(t *testing.T) { f := &CustomFormatter{} // no template, no color entry := newEntry(logrus.InfoLevel, "hello", logrus.Fields{