From 7cfae7e0d3729495c36301dab31455cf09a1aab9 Mon Sep 17 00:00:00 2001 From: wizardchen Date: Wed, 10 Sep 2025 16:05:37 +0800 Subject: [PATCH] fix: pre fetch ocr models in docker container --- Makefile | 25 ++++++++-- docker/Dockerfile.docreader | 51 +++++++++------------ services/docreader/src/download_deps.py | 45 +++++++++++++++++- services/docreader/src/parser/ocr_engine.py | 6 +-- 4 files changed, 90 insertions(+), 37 deletions(-) diff --git a/Makefile b/Makefile index 1b040d10..74d1e85d 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help build run test clean docker-build docker-build-docreader docker-build-frontend docker-build-all docker-run migrate-up migrate-down docker-restart docker-stop start-all stop-all start-ollama stop-ollama build-images build-images-app build-images-docreader build-images-frontend clean-images check-env list-containers pull-images +.PHONY: help build run test clean docker-build-app docker-build-docreader docker-build-frontend docker-build-all docker-run migrate-up migrate-down docker-restart docker-stop start-all stop-all start-ollama stop-ollama build-images build-images-app build-images-docreader build-images-frontend clean-images check-env list-containers pull-images show-platform # Show help help: @@ -45,6 +45,7 @@ help: @echo " check-env 检查环境配置" @echo " list-containers 列出运行中的容器" @echo " pull-images 拉取最新镜像" + @echo " show-platform 显示当前构建平台" # Go related variables BINARY_NAME=WeKnora @@ -54,6 +55,17 @@ MAIN_PATH=./cmd/server DOCKER_IMAGE=wechatopenai/weknora-app DOCKER_TAG=latest +# Platform detection +ifeq ($(shell uname -m),x86_64) + PLATFORM=linux/amd64 +else ifeq ($(shell uname -m),aarch64) + PLATFORM=linux/arm64 +else ifeq ($(shell uname -m),arm64) + PLATFORM=linux/arm64 +else + PLATFORM=linux/amd64 +endif + # Build the application build: go build -o $(BINARY_NAME) $(MAIN_PATH) @@ -73,15 +85,15 @@ clean: # Build Docker image docker-build-app: - docker build -f docker/Dockerfile.app -t $(DOCKER_IMAGE):$(DOCKER_TAG) . + docker build --platform $(PLATFORM) -f docker/Dockerfile.app -t $(DOCKER_IMAGE):$(DOCKER_TAG) . # Build docreader Docker image docker-build-docreader: - docker build -f docker/Dockerfile.docreader -t wechatopenai/weknora-docreader:latest . + docker build --platform $(PLATFORM) -f docker/Dockerfile.docreader -t wechatopenai/weknora-docreader:latest . # Build frontend Docker image docker-build-frontend: - docker build -f frontend/Dockerfile -t wechatopenai/weknora-ui:latest frontend/ + docker build --platform $(PLATFORM) -f frontend/Dockerfile -t wechatopenai/weknora-ui:latest frontend/ # Build all Docker images docker-build-all: docker-build-app docker-build-docreader docker-build-frontend @@ -182,4 +194,9 @@ list-containers: pull-images: ./scripts/start_all.sh --pull +# Show current platform +show-platform: + @echo "当前系统架构: $(shell uname -m)" + @echo "Docker构建平台: $(PLATFORM)" + diff --git a/docker/Dockerfile.docreader b/docker/Dockerfile.docreader index d7bae0c9..d512749c 100644 --- a/docker/Dockerfile.docreader +++ b/docker/Dockerfile.docreader @@ -50,22 +50,33 @@ COPY services/docreader/requirements.txt . # 安装依赖 RUN pip cache purge && pip install --no-cache-dir -r requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple -# 预下载 PP-OCRv5 模型 -RUN mkdir -p /root/.paddlex/official_models && \ - wget https://paddle-model-ecology.bj.bcebos.com/paddlex/official_inference_model/paddle3.0.0/PP-OCRv5_server_det_infer.tar \ - -O /root/.paddlex/official_models/PP-OCRv5_server_det_infer.tar && \ - wget https://paddle-model-ecology.bj.bcebos.com/paddlex/official_inference_model/paddle3.0.0/PP-OCRv5_server_rec_infer.tar \ - -O /root/.paddlex/official_models/PP-OCRv5_server_rec_infer.tar && \ - tar -xf /root/.paddlex/official_models/PP-OCRv5_server_det_infer.tar -C /root/.paddlex/official_models/ && \ - tar -xf /root/.paddlex/official_models/PP-OCRv5_server_rec_infer.tar -C /root/.paddlex/official_models/ && \ - rm -rf /root/.paddlex/official_models/PP-OCRv5_server_det_infer.tar /root/.paddlex/official_models/PP-OCRv5_server_rec_infer.tar +# 预下载 PP-OCRv4 模型 +RUN mkdir -p /root/.paddleocr/whl/det/ch && \ + mkdir -p /root/.paddleocr/whl/rec/ch && \ + mkdir -p /root/.paddleocr/whl/cls/ch && \ + # 下载检测模型 + wget https://paddleocr.bj.bcebos.com/PP-OCRv4/chinese/ch_PP-OCRv4_det_infer.tar \ + -O /root/.paddleocr/whl/det/ch/ch_PP-OCRv4_det_infer.tar && \ + tar -xf /root/.paddleocr/whl/det/ch/ch_PP-OCRv4_det_infer.tar -C /root/.paddleocr/whl/det/ch/ && \ + # 下载识别模型 + wget https://paddleocr.bj.bcebos.com/PP-OCRv4/chinese/ch_PP-OCRv4_rec_infer.tar \ + -O /root/.paddleocr/whl/rec/ch/ch_PP-OCRv4_rec_infer.tar && \ + tar -xf /root/.paddleocr/whl/rec/ch/ch_PP-OCRv4_rec_infer.tar -C /root/.paddleocr/whl/rec/ch/ && \ + # 下载文本方向分类模型(用于判断文本是否需要旋转) + wget https://paddleocr.bj.bcebos.com/dygraph_v2.0/ch/ch_ppocr_mobile_v2.0_cls_infer.tar \ + -O /root/.paddleocr/whl/cls/ch_ppocr_mobile_v2.0_cls_infer.tar && \ + tar -xf /root/.paddleocr/whl/cls/ch_ppocr_mobile_v2.0_cls_infer.tar -C /root/.paddleocr/whl/cls/ && \ + # 清理压缩包 + rm -f /root/.paddleocr/whl/det/ch/ch_PP-OCRv4_det_infer.tar && \ + rm -f /root/.paddleocr/whl/rec/ch/ch_PP-OCRv4_rec_infer.tar && \ + rm -f /root/.paddleocr/whl/cls/ch_ppocr_mobile_v2.0_cls_infer.tar # 复制源代码和生成脚本 COPY services/docreader/src/ /app/src/ COPY services/docreader/scripts/ /app/scripts/ # 确保模型目录存在 -RUN ls -la /root/.paddlex/official_models +RUN ls -la /root/.paddleocr/whl/ # 生成 protobuf 代码 RUN chmod +x /app/scripts/generate_proto.sh && bash /app/scripts/generate_proto.sh @@ -104,29 +115,11 @@ RUN apt-get update && apt-get install -y \ libsm6 \ libreoffice \ && rm -rf /var/lib/apt/lists/* - -# # 下载并安装 LibreOffice(区分架构) -# RUN mkdir -p /tmp/libreoffice && cd /tmp/libreoffice && \ -# if [ "$(uname -m)" = "x86_64" ]; then \ -# wget https://mirrors.tuna.tsinghua.edu.cn/libreoffice/libreoffice/stable/25.2.5/deb/x86_64/LibreOffice_25.2.5_Linux_x86-64_deb.tar.gz && \ -# tar -xzf LibreOffice_25.2.5_Linux_x86-64_deb.tar.gz && \ -# cd LibreOffice_25.2.5.2_Linux_x86-64_deb/DEBS && dpkg -i *.deb; \ -# elif [ "$(uname -m)" = "aarch64" ] || [ "$(uname -m)" = "arm64" ]; then \ -# wget https://mirrors.aliyun.com/libreoffice/testing/25.8.0/deb/aarch64/LibreOffice_25.8.0.3_Linux_aarch64_deb.tar.gz && \ -# tar -xzf LibreOffice_25.8.0.3_Linux_aarch64_deb.tar.gz && \ -# cd LibreOffice_25.8.0.3_Linux_aarch64_deb/DEBS && dpkg -i *.deb; \ -# else \ -# echo "Unsupported architecture: $(uname -m)" && exit 1; \ -# fi && \ -# cd / && rm -rf /tmp/libreoffice - -# 设置 LibreOffice 环境变量 -# RUN echo 'export LIBREOFFICE_PATH=/opt/libreoffice25.2/program/soffice' >> /etc/environment; # 从构建阶段复制已安装的依赖和生成的代码 COPY --from=builder /usr/local/lib/python3.10/site-packages /usr/local/lib/python3.10/site-packages COPY --from=builder /usr/local/bin /usr/local/bin -COPY --from=builder /root/.paddlex/official_models /root/.paddlex/official_models +COPY --from=builder /root/.paddleocr /root/.paddleocr COPY --from=builder /app/src /app/src # 安装 Playwright 浏览器 diff --git a/services/docreader/src/download_deps.py b/services/docreader/src/download_deps.py index 1eba539f..d24fbfa4 100644 --- a/services/docreader/src/download_deps.py +++ b/services/docreader/src/download_deps.py @@ -24,4 +24,47 @@ logger = logging.getLogger(__name__) def init_ocr_model(): - PaddleOCR() + """Initialize PaddleOCR model to pre-download and cache models""" + try: + logger.info("Initializing PaddleOCR model for pre-download...") + + # 使用与代码中相同的配置 + ocr_config = { + "use_gpu": False, + "text_det_limit_type": "max", + "text_det_limit_side_len": 960, + "use_doc_orientation_classify": True, # 启用文档方向分类 + "use_doc_unwarping": False, + "use_textline_orientation": True, # 启用文本行方向检测 + "text_recognition_model_name": "PP-OCRv4_server_rec", + "text_detection_model_name": "PP-OCRv4_server_det", + "text_det_thresh": 0.3, + "text_det_box_thresh": 0.6, + "text_det_unclip_ratio": 1.5, + "text_rec_score_thresh": 0.0, + "ocr_version": "PP-OCRv4", + "lang": "ch", + "show_log": False, + "use_dilation": True, + "det_db_score_mode": "slow", + } + + # 初始化PaddleOCR,这会触发模型下载和缓存 + ocr = PaddleOCR(**ocr_config) + logger.info("PaddleOCR model initialization completed successfully") + + # 测试OCR功能以确保模型正常工作 + import numpy as np + from PIL import Image + + # 创建一个简单的测试图像 + test_image = np.ones((100, 300, 3), dtype=np.uint8) * 255 + test_pil = Image.fromarray(test_image) + + # 执行一次OCR测试 + result = ocr.ocr(np.array(test_pil), cls=False) + logger.info("PaddleOCR test completed successfully") + + except Exception as e: + logger.error(f"Failed to initialize PaddleOCR model: {str(e)}") + raise diff --git a/services/docreader/src/parser/ocr_engine.py b/services/docreader/src/parser/ocr_engine.py index f46ce90a..8a634cbb 100644 --- a/services/docreader/src/parser/ocr_engine.py +++ b/services/docreader/src/parser/ocr_engine.py @@ -41,14 +41,14 @@ class PaddleOCRBackend(OCRBackend): paddle.set_device('cpu') from paddleocr import PaddleOCR - # Simplified OCR configuration + # OCR configuration with text orientation classification enabled ocr_config = { "use_gpu": False, "text_det_limit_type": "max", "text_det_limit_side_len": 960, - "use_doc_orientation_classify": False, + "use_doc_orientation_classify": True, # 启用文档方向分类 "use_doc_unwarping": False, - "use_textline_orientation": False, + "use_textline_orientation": True, # 启用文本行方向检测 "text_recognition_model_name": "PP-OCRv4_server_rec", "text_detection_model_name": "PP-OCRv4_server_det", "text_det_thresh": 0.3,