fix: pre fetch ocr models in docker container

This commit is contained in:
wizardchen
2025-09-10 16:05:37 +08:00
committed by lyingbug
parent 19d2493afc
commit 7cfae7e0d3
4 changed files with 90 additions and 37 deletions
+21 -4
View File
@@ -1,4 +1,4 @@
.PHONY: help build run test clean docker-build docker-build-docreader docker-build-frontend docker-build-all docker-run migrate-up migrate-down docker-restart docker-stop start-all stop-all start-ollama stop-ollama build-images build-images-app build-images-docreader build-images-frontend clean-images check-env list-containers pull-images
.PHONY: help build run test clean docker-build-app docker-build-docreader docker-build-frontend docker-build-all docker-run migrate-up migrate-down docker-restart docker-stop start-all stop-all start-ollama stop-ollama build-images build-images-app build-images-docreader build-images-frontend clean-images check-env list-containers pull-images show-platform
# Show help
help:
@@ -45,6 +45,7 @@ help:
@echo " check-env 检查环境配置"
@echo " list-containers 列出运行中的容器"
@echo " pull-images 拉取最新镜像"
@echo " show-platform 显示当前构建平台"
# Go related variables
BINARY_NAME=WeKnora
@@ -54,6 +55,17 @@ MAIN_PATH=./cmd/server
DOCKER_IMAGE=wechatopenai/weknora-app
DOCKER_TAG=latest
# Platform detection
ifeq ($(shell uname -m),x86_64)
PLATFORM=linux/amd64
else ifeq ($(shell uname -m),aarch64)
PLATFORM=linux/arm64
else ifeq ($(shell uname -m),arm64)
PLATFORM=linux/arm64
else
PLATFORM=linux/amd64
endif
# Build the application
build:
go build -o $(BINARY_NAME) $(MAIN_PATH)
@@ -73,15 +85,15 @@ clean:
# Build Docker image
docker-build-app:
docker build -f docker/Dockerfile.app -t $(DOCKER_IMAGE):$(DOCKER_TAG) .
docker build --platform $(PLATFORM) -f docker/Dockerfile.app -t $(DOCKER_IMAGE):$(DOCKER_TAG) .
# Build docreader Docker image
docker-build-docreader:
docker build -f docker/Dockerfile.docreader -t wechatopenai/weknora-docreader:latest .
docker build --platform $(PLATFORM) -f docker/Dockerfile.docreader -t wechatopenai/weknora-docreader:latest .
# Build frontend Docker image
docker-build-frontend:
docker build -f frontend/Dockerfile -t wechatopenai/weknora-ui:latest frontend/
docker build --platform $(PLATFORM) -f frontend/Dockerfile -t wechatopenai/weknora-ui:latest frontend/
# Build all Docker images
docker-build-all: docker-build-app docker-build-docreader docker-build-frontend
@@ -182,4 +194,9 @@ list-containers:
pull-images:
./scripts/start_all.sh --pull
# Show current platform
show-platform:
@echo "当前系统架构: $(shell uname -m)"
@echo "Docker构建平台: $(PLATFORM)"
+22 -29
View File
@@ -50,22 +50,33 @@ COPY services/docreader/requirements.txt .
# 安装依赖
RUN pip cache purge && pip install --no-cache-dir -r requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple
# 预下载 PP-OCRv5 模型
RUN mkdir -p /root/.paddlex/official_models && \
wget https://paddle-model-ecology.bj.bcebos.com/paddlex/official_inference_model/paddle3.0.0/PP-OCRv5_server_det_infer.tar \
-O /root/.paddlex/official_models/PP-OCRv5_server_det_infer.tar && \
wget https://paddle-model-ecology.bj.bcebos.com/paddlex/official_inference_model/paddle3.0.0/PP-OCRv5_server_rec_infer.tar \
-O /root/.paddlex/official_models/PP-OCRv5_server_rec_infer.tar && \
tar -xf /root/.paddlex/official_models/PP-OCRv5_server_det_infer.tar -C /root/.paddlex/official_models/ && \
tar -xf /root/.paddlex/official_models/PP-OCRv5_server_rec_infer.tar -C /root/.paddlex/official_models/ && \
rm -rf /root/.paddlex/official_models/PP-OCRv5_server_det_infer.tar /root/.paddlex/official_models/PP-OCRv5_server_rec_infer.tar
# 预下载 PP-OCRv4 模型
RUN mkdir -p /root/.paddleocr/whl/det/ch && \
mkdir -p /root/.paddleocr/whl/rec/ch && \
mkdir -p /root/.paddleocr/whl/cls/ch && \
# 下载检测模型
wget https://paddleocr.bj.bcebos.com/PP-OCRv4/chinese/ch_PP-OCRv4_det_infer.tar \
-O /root/.paddleocr/whl/det/ch/ch_PP-OCRv4_det_infer.tar && \
tar -xf /root/.paddleocr/whl/det/ch/ch_PP-OCRv4_det_infer.tar -C /root/.paddleocr/whl/det/ch/ && \
# 下载识别模型
wget https://paddleocr.bj.bcebos.com/PP-OCRv4/chinese/ch_PP-OCRv4_rec_infer.tar \
-O /root/.paddleocr/whl/rec/ch/ch_PP-OCRv4_rec_infer.tar && \
tar -xf /root/.paddleocr/whl/rec/ch/ch_PP-OCRv4_rec_infer.tar -C /root/.paddleocr/whl/rec/ch/ && \
# 下载文本方向分类模型(用于判断文本是否需要旋转)
wget https://paddleocr.bj.bcebos.com/dygraph_v2.0/ch/ch_ppocr_mobile_v2.0_cls_infer.tar \
-O /root/.paddleocr/whl/cls/ch_ppocr_mobile_v2.0_cls_infer.tar && \
tar -xf /root/.paddleocr/whl/cls/ch_ppocr_mobile_v2.0_cls_infer.tar -C /root/.paddleocr/whl/cls/ && \
# 清理压缩包
rm -f /root/.paddleocr/whl/det/ch/ch_PP-OCRv4_det_infer.tar && \
rm -f /root/.paddleocr/whl/rec/ch/ch_PP-OCRv4_rec_infer.tar && \
rm -f /root/.paddleocr/whl/cls/ch_ppocr_mobile_v2.0_cls_infer.tar
# 复制源代码和生成脚本
COPY services/docreader/src/ /app/src/
COPY services/docreader/scripts/ /app/scripts/
# 确保模型目录存在
RUN ls -la /root/.paddlex/official_models
RUN ls -la /root/.paddleocr/whl/
# 生成 protobuf 代码
RUN chmod +x /app/scripts/generate_proto.sh && bash /app/scripts/generate_proto.sh
@@ -104,29 +115,11 @@ RUN apt-get update && apt-get install -y \
libsm6 \
libreoffice \
&& rm -rf /var/lib/apt/lists/*
# # 下载并安装 LibreOffice(区分架构)
# RUN mkdir -p /tmp/libreoffice && cd /tmp/libreoffice && \
# if [ "$(uname -m)" = "x86_64" ]; then \
# wget https://mirrors.tuna.tsinghua.edu.cn/libreoffice/libreoffice/stable/25.2.5/deb/x86_64/LibreOffice_25.2.5_Linux_x86-64_deb.tar.gz && \
# tar -xzf LibreOffice_25.2.5_Linux_x86-64_deb.tar.gz && \
# cd LibreOffice_25.2.5.2_Linux_x86-64_deb/DEBS && dpkg -i *.deb; \
# elif [ "$(uname -m)" = "aarch64" ] || [ "$(uname -m)" = "arm64" ]; then \
# wget https://mirrors.aliyun.com/libreoffice/testing/25.8.0/deb/aarch64/LibreOffice_25.8.0.3_Linux_aarch64_deb.tar.gz && \
# tar -xzf LibreOffice_25.8.0.3_Linux_aarch64_deb.tar.gz && \
# cd LibreOffice_25.8.0.3_Linux_aarch64_deb/DEBS && dpkg -i *.deb; \
# else \
# echo "Unsupported architecture: $(uname -m)" && exit 1; \
# fi && \
# cd / && rm -rf /tmp/libreoffice
# 设置 LibreOffice 环境变量
# RUN echo 'export LIBREOFFICE_PATH=/opt/libreoffice25.2/program/soffice' >> /etc/environment;
# 从构建阶段复制已安装的依赖和生成的代码
COPY --from=builder /usr/local/lib/python3.10/site-packages /usr/local/lib/python3.10/site-packages
COPY --from=builder /usr/local/bin /usr/local/bin
COPY --from=builder /root/.paddlex/official_models /root/.paddlex/official_models
COPY --from=builder /root/.paddleocr /root/.paddleocr
COPY --from=builder /app/src /app/src
# 安装 Playwright 浏览器
+44 -1
View File
@@ -24,4 +24,47 @@ logger = logging.getLogger(__name__)
def init_ocr_model():
PaddleOCR()
"""Initialize PaddleOCR model to pre-download and cache models"""
try:
logger.info("Initializing PaddleOCR model for pre-download...")
# 使用与代码中相同的配置
ocr_config = {
"use_gpu": False,
"text_det_limit_type": "max",
"text_det_limit_side_len": 960,
"use_doc_orientation_classify": True, # 启用文档方向分类
"use_doc_unwarping": False,
"use_textline_orientation": True, # 启用文本行方向检测
"text_recognition_model_name": "PP-OCRv4_server_rec",
"text_detection_model_name": "PP-OCRv4_server_det",
"text_det_thresh": 0.3,
"text_det_box_thresh": 0.6,
"text_det_unclip_ratio": 1.5,
"text_rec_score_thresh": 0.0,
"ocr_version": "PP-OCRv4",
"lang": "ch",
"show_log": False,
"use_dilation": True,
"det_db_score_mode": "slow",
}
# 初始化PaddleOCR,这会触发模型下载和缓存
ocr = PaddleOCR(**ocr_config)
logger.info("PaddleOCR model initialization completed successfully")
# 测试OCR功能以确保模型正常工作
import numpy as np
from PIL import Image
# 创建一个简单的测试图像
test_image = np.ones((100, 300, 3), dtype=np.uint8) * 255
test_pil = Image.fromarray(test_image)
# 执行一次OCR测试
result = ocr.ocr(np.array(test_pil), cls=False)
logger.info("PaddleOCR test completed successfully")
except Exception as e:
logger.error(f"Failed to initialize PaddleOCR model: {str(e)}")
raise
+3 -3
View File
@@ -41,14 +41,14 @@ class PaddleOCRBackend(OCRBackend):
paddle.set_device('cpu')
from paddleocr import PaddleOCR
# Simplified OCR configuration
# OCR configuration with text orientation classification enabled
ocr_config = {
"use_gpu": False,
"text_det_limit_type": "max",
"text_det_limit_side_len": 960,
"use_doc_orientation_classify": False,
"use_doc_orientation_classify": True, # 启用文档方向分类
"use_doc_unwarping": False,
"use_textline_orientation": False,
"use_textline_orientation": True, # 启用文本行方向检测
"text_recognition_model_name": "PP-OCRv4_server_rec",
"text_detection_model_name": "PP-OCRv4_server_det",
"text_det_thresh": 0.3,