fix: pre fetch ocr models in docker container
This commit is contained in:
@@ -1,4 +1,4 @@
|
||||
.PHONY: help build run test clean docker-build docker-build-docreader docker-build-frontend docker-build-all docker-run migrate-up migrate-down docker-restart docker-stop start-all stop-all start-ollama stop-ollama build-images build-images-app build-images-docreader build-images-frontend clean-images check-env list-containers pull-images
|
||||
.PHONY: help build run test clean docker-build-app docker-build-docreader docker-build-frontend docker-build-all docker-run migrate-up migrate-down docker-restart docker-stop start-all stop-all start-ollama stop-ollama build-images build-images-app build-images-docreader build-images-frontend clean-images check-env list-containers pull-images show-platform
|
||||
|
||||
# Show help
|
||||
help:
|
||||
@@ -45,6 +45,7 @@ help:
|
||||
@echo " check-env 检查环境配置"
|
||||
@echo " list-containers 列出运行中的容器"
|
||||
@echo " pull-images 拉取最新镜像"
|
||||
@echo " show-platform 显示当前构建平台"
|
||||
|
||||
# Go related variables
|
||||
BINARY_NAME=WeKnora
|
||||
@@ -54,6 +55,17 @@ MAIN_PATH=./cmd/server
|
||||
DOCKER_IMAGE=wechatopenai/weknora-app
|
||||
DOCKER_TAG=latest
|
||||
|
||||
# Platform detection
|
||||
ifeq ($(shell uname -m),x86_64)
|
||||
PLATFORM=linux/amd64
|
||||
else ifeq ($(shell uname -m),aarch64)
|
||||
PLATFORM=linux/arm64
|
||||
else ifeq ($(shell uname -m),arm64)
|
||||
PLATFORM=linux/arm64
|
||||
else
|
||||
PLATFORM=linux/amd64
|
||||
endif
|
||||
|
||||
# Build the application
|
||||
build:
|
||||
go build -o $(BINARY_NAME) $(MAIN_PATH)
|
||||
@@ -73,15 +85,15 @@ clean:
|
||||
|
||||
# Build Docker image
|
||||
docker-build-app:
|
||||
docker build -f docker/Dockerfile.app -t $(DOCKER_IMAGE):$(DOCKER_TAG) .
|
||||
docker build --platform $(PLATFORM) -f docker/Dockerfile.app -t $(DOCKER_IMAGE):$(DOCKER_TAG) .
|
||||
|
||||
# Build docreader Docker image
|
||||
docker-build-docreader:
|
||||
docker build -f docker/Dockerfile.docreader -t wechatopenai/weknora-docreader:latest .
|
||||
docker build --platform $(PLATFORM) -f docker/Dockerfile.docreader -t wechatopenai/weknora-docreader:latest .
|
||||
|
||||
# Build frontend Docker image
|
||||
docker-build-frontend:
|
||||
docker build -f frontend/Dockerfile -t wechatopenai/weknora-ui:latest frontend/
|
||||
docker build --platform $(PLATFORM) -f frontend/Dockerfile -t wechatopenai/weknora-ui:latest frontend/
|
||||
|
||||
# Build all Docker images
|
||||
docker-build-all: docker-build-app docker-build-docreader docker-build-frontend
|
||||
@@ -182,4 +194,9 @@ list-containers:
|
||||
pull-images:
|
||||
./scripts/start_all.sh --pull
|
||||
|
||||
# Show current platform
|
||||
show-platform:
|
||||
@echo "当前系统架构: $(shell uname -m)"
|
||||
@echo "Docker构建平台: $(PLATFORM)"
|
||||
|
||||
|
||||
|
||||
+22
-29
@@ -50,22 +50,33 @@ COPY services/docreader/requirements.txt .
|
||||
# 安装依赖
|
||||
RUN pip cache purge && pip install --no-cache-dir -r requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple
|
||||
|
||||
# 预下载 PP-OCRv5 模型
|
||||
RUN mkdir -p /root/.paddlex/official_models && \
|
||||
wget https://paddle-model-ecology.bj.bcebos.com/paddlex/official_inference_model/paddle3.0.0/PP-OCRv5_server_det_infer.tar \
|
||||
-O /root/.paddlex/official_models/PP-OCRv5_server_det_infer.tar && \
|
||||
wget https://paddle-model-ecology.bj.bcebos.com/paddlex/official_inference_model/paddle3.0.0/PP-OCRv5_server_rec_infer.tar \
|
||||
-O /root/.paddlex/official_models/PP-OCRv5_server_rec_infer.tar && \
|
||||
tar -xf /root/.paddlex/official_models/PP-OCRv5_server_det_infer.tar -C /root/.paddlex/official_models/ && \
|
||||
tar -xf /root/.paddlex/official_models/PP-OCRv5_server_rec_infer.tar -C /root/.paddlex/official_models/ && \
|
||||
rm -rf /root/.paddlex/official_models/PP-OCRv5_server_det_infer.tar /root/.paddlex/official_models/PP-OCRv5_server_rec_infer.tar
|
||||
# 预下载 PP-OCRv4 模型
|
||||
RUN mkdir -p /root/.paddleocr/whl/det/ch && \
|
||||
mkdir -p /root/.paddleocr/whl/rec/ch && \
|
||||
mkdir -p /root/.paddleocr/whl/cls/ch && \
|
||||
# 下载检测模型
|
||||
wget https://paddleocr.bj.bcebos.com/PP-OCRv4/chinese/ch_PP-OCRv4_det_infer.tar \
|
||||
-O /root/.paddleocr/whl/det/ch/ch_PP-OCRv4_det_infer.tar && \
|
||||
tar -xf /root/.paddleocr/whl/det/ch/ch_PP-OCRv4_det_infer.tar -C /root/.paddleocr/whl/det/ch/ && \
|
||||
# 下载识别模型
|
||||
wget https://paddleocr.bj.bcebos.com/PP-OCRv4/chinese/ch_PP-OCRv4_rec_infer.tar \
|
||||
-O /root/.paddleocr/whl/rec/ch/ch_PP-OCRv4_rec_infer.tar && \
|
||||
tar -xf /root/.paddleocr/whl/rec/ch/ch_PP-OCRv4_rec_infer.tar -C /root/.paddleocr/whl/rec/ch/ && \
|
||||
# 下载文本方向分类模型(用于判断文本是否需要旋转)
|
||||
wget https://paddleocr.bj.bcebos.com/dygraph_v2.0/ch/ch_ppocr_mobile_v2.0_cls_infer.tar \
|
||||
-O /root/.paddleocr/whl/cls/ch_ppocr_mobile_v2.0_cls_infer.tar && \
|
||||
tar -xf /root/.paddleocr/whl/cls/ch_ppocr_mobile_v2.0_cls_infer.tar -C /root/.paddleocr/whl/cls/ && \
|
||||
# 清理压缩包
|
||||
rm -f /root/.paddleocr/whl/det/ch/ch_PP-OCRv4_det_infer.tar && \
|
||||
rm -f /root/.paddleocr/whl/rec/ch/ch_PP-OCRv4_rec_infer.tar && \
|
||||
rm -f /root/.paddleocr/whl/cls/ch_ppocr_mobile_v2.0_cls_infer.tar
|
||||
|
||||
# 复制源代码和生成脚本
|
||||
COPY services/docreader/src/ /app/src/
|
||||
COPY services/docreader/scripts/ /app/scripts/
|
||||
|
||||
# 确保模型目录存在
|
||||
RUN ls -la /root/.paddlex/official_models
|
||||
RUN ls -la /root/.paddleocr/whl/
|
||||
|
||||
# 生成 protobuf 代码
|
||||
RUN chmod +x /app/scripts/generate_proto.sh && bash /app/scripts/generate_proto.sh
|
||||
@@ -104,29 +115,11 @@ RUN apt-get update && apt-get install -y \
|
||||
libsm6 \
|
||||
libreoffice \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# # 下载并安装 LibreOffice(区分架构)
|
||||
# RUN mkdir -p /tmp/libreoffice && cd /tmp/libreoffice && \
|
||||
# if [ "$(uname -m)" = "x86_64" ]; then \
|
||||
# wget https://mirrors.tuna.tsinghua.edu.cn/libreoffice/libreoffice/stable/25.2.5/deb/x86_64/LibreOffice_25.2.5_Linux_x86-64_deb.tar.gz && \
|
||||
# tar -xzf LibreOffice_25.2.5_Linux_x86-64_deb.tar.gz && \
|
||||
# cd LibreOffice_25.2.5.2_Linux_x86-64_deb/DEBS && dpkg -i *.deb; \
|
||||
# elif [ "$(uname -m)" = "aarch64" ] || [ "$(uname -m)" = "arm64" ]; then \
|
||||
# wget https://mirrors.aliyun.com/libreoffice/testing/25.8.0/deb/aarch64/LibreOffice_25.8.0.3_Linux_aarch64_deb.tar.gz && \
|
||||
# tar -xzf LibreOffice_25.8.0.3_Linux_aarch64_deb.tar.gz && \
|
||||
# cd LibreOffice_25.8.0.3_Linux_aarch64_deb/DEBS && dpkg -i *.deb; \
|
||||
# else \
|
||||
# echo "Unsupported architecture: $(uname -m)" && exit 1; \
|
||||
# fi && \
|
||||
# cd / && rm -rf /tmp/libreoffice
|
||||
|
||||
# 设置 LibreOffice 环境变量
|
||||
# RUN echo 'export LIBREOFFICE_PATH=/opt/libreoffice25.2/program/soffice' >> /etc/environment;
|
||||
|
||||
# 从构建阶段复制已安装的依赖和生成的代码
|
||||
COPY --from=builder /usr/local/lib/python3.10/site-packages /usr/local/lib/python3.10/site-packages
|
||||
COPY --from=builder /usr/local/bin /usr/local/bin
|
||||
COPY --from=builder /root/.paddlex/official_models /root/.paddlex/official_models
|
||||
COPY --from=builder /root/.paddleocr /root/.paddleocr
|
||||
COPY --from=builder /app/src /app/src
|
||||
|
||||
# 安装 Playwright 浏览器
|
||||
|
||||
@@ -24,4 +24,47 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def init_ocr_model():
|
||||
PaddleOCR()
|
||||
"""Initialize PaddleOCR model to pre-download and cache models"""
|
||||
try:
|
||||
logger.info("Initializing PaddleOCR model for pre-download...")
|
||||
|
||||
# 使用与代码中相同的配置
|
||||
ocr_config = {
|
||||
"use_gpu": False,
|
||||
"text_det_limit_type": "max",
|
||||
"text_det_limit_side_len": 960,
|
||||
"use_doc_orientation_classify": True, # 启用文档方向分类
|
||||
"use_doc_unwarping": False,
|
||||
"use_textline_orientation": True, # 启用文本行方向检测
|
||||
"text_recognition_model_name": "PP-OCRv4_server_rec",
|
||||
"text_detection_model_name": "PP-OCRv4_server_det",
|
||||
"text_det_thresh": 0.3,
|
||||
"text_det_box_thresh": 0.6,
|
||||
"text_det_unclip_ratio": 1.5,
|
||||
"text_rec_score_thresh": 0.0,
|
||||
"ocr_version": "PP-OCRv4",
|
||||
"lang": "ch",
|
||||
"show_log": False,
|
||||
"use_dilation": True,
|
||||
"det_db_score_mode": "slow",
|
||||
}
|
||||
|
||||
# 初始化PaddleOCR,这会触发模型下载和缓存
|
||||
ocr = PaddleOCR(**ocr_config)
|
||||
logger.info("PaddleOCR model initialization completed successfully")
|
||||
|
||||
# 测试OCR功能以确保模型正常工作
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
|
||||
# 创建一个简单的测试图像
|
||||
test_image = np.ones((100, 300, 3), dtype=np.uint8) * 255
|
||||
test_pil = Image.fromarray(test_image)
|
||||
|
||||
# 执行一次OCR测试
|
||||
result = ocr.ocr(np.array(test_pil), cls=False)
|
||||
logger.info("PaddleOCR test completed successfully")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to initialize PaddleOCR model: {str(e)}")
|
||||
raise
|
||||
|
||||
@@ -41,14 +41,14 @@ class PaddleOCRBackend(OCRBackend):
|
||||
paddle.set_device('cpu')
|
||||
|
||||
from paddleocr import PaddleOCR
|
||||
# Simplified OCR configuration
|
||||
# OCR configuration with text orientation classification enabled
|
||||
ocr_config = {
|
||||
"use_gpu": False,
|
||||
"text_det_limit_type": "max",
|
||||
"text_det_limit_side_len": 960,
|
||||
"use_doc_orientation_classify": False,
|
||||
"use_doc_orientation_classify": True, # 启用文档方向分类
|
||||
"use_doc_unwarping": False,
|
||||
"use_textline_orientation": False,
|
||||
"use_textline_orientation": True, # 启用文本行方向检测
|
||||
"text_recognition_model_name": "PP-OCRv4_server_rec",
|
||||
"text_detection_model_name": "PP-OCRv4_server_det",
|
||||
"text_det_thresh": 0.3,
|
||||
|
||||
Reference in New Issue
Block a user