Weknora/Enterprise-Knowledge-Base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(docreader): support forcing PDF scanned-page OCR parsing

Browse Source
This commit is contained in:
杨聪聪
2026-06-25 16:48:57 +08:00
committed by lyingbug
parent 2c44a41f2a
commit 917feceb59
14 changed files with 210 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env.example
+3
View File
@@ -588,6 +588,9 @@ DOCREADER_TRANSPORT=grpc
# DOCREADER_PDF_SANITIZE_TEXT=true
# DOCREADER_PDF_STRIP_CHART_DEBRIS=true
# DOCREADER_PDF_RENDER_VECTOR_FIGURES=true
# 强制 PDF 全页按扫描件解析。适合文本层质量很差、但页面可 OCR 的 PDF。
# 开启后所有 PDF 页都会渲染成图片再走 OCR/VLM，速度和模型调用成本会上升。
# DOCREADER_PDF_FORCE_SCANNED=false
# OpenDataLoader PDF（知识库 parser_engine_rules 指定 engine: opendataloader）
# 需 Java 11+；docreader 镜像已包含 openjdk-17-jre-headless。
docker-compose.dev.yml
+1
View File
@@ -248,6 +248,7 @@ services:
- docreader-tmp-dev:/tmp/docreader
environment:
- DOCREADER_IMAGE_OUTPUT_DIR=/tmp/docreader
- DOCREADER_PDF_FORCE_SCANNED=${DOCREADER_PDF_FORCE_SCANNED:-false}
- MAX_FILE_SIZE_MB=${MAX_FILE_SIZE_MB:-}
- DOCREADER_ODL_MAX_WORKERS=${DOCREADER_ODL_MAX_WORKERS:-1}
- DOCREADER_ODL_HYBRID=${DOCREADER_ODL_HYBRID:-off}
docker-compose.yml
+1
View File
@@ -240,6 +240,7 @@ services:
- docreader-tmp:/tmp/docreader
environment:
- DOCREADER_IMAGE_OUTPUT_DIR=/tmp/docreader
- DOCREADER_PDF_FORCE_SCANNED=${DOCREADER_PDF_FORCE_SCANNED:-false}
- MAX_FILE_SIZE_MB=${MAX_FILE_SIZE_MB:-}
- DOCREADER_ODL_MAX_WORKERS=${DOCREADER_ODL_MAX_WORKERS:-1}
- DOCREADER_ODL_HYBRID=${DOCREADER_ODL_HYBRID:-off}
docreader/parser/pdf_parser.py
+43
View File
@@ -115,6 +115,13 @@ MIN_CHART_REGION_AREA_RATIO = _env_float("DOCREADER_PDF_MIN_CHART_REGION_AREA",
MAX_CHART_REGION_AREA_RATIO = _env_float("DOCREADER_PDF_MAX_CHART_REGION_AREA", 0.42)
MAX_FIGURE_HEIGHT_RATIO = _env_float("DOCREADER_PDF_MAX_FIGURE_HEIGHT_RATIO", 0.38)
# --- Force scanned mode ------------------------------------------------------
# When True, ALL PDF pages are rendered as images and routed through OCR/VLM,
# bypassing the automatic text/scanned page classification. Useful for PDFs
# with a low-quality or misleading text layer (web-print, scanned, image-heavy).
# Can be overridden per-upload via parser_engine_overrides.pdf_force_scanned.
FORCE_SCANNED_PDF = _env_bool("DOCREADER_PDF_FORCE_SCANNED", False)
# pdfium / Adobe text layers often emit U+FFFE for missing hyphenation or ligatures.
_PDF_ARTIFACT_RE = re.compile(r"[\u00ad\u200b-\u200f\ufeff\ufffe\uffff]")
_PDF_ARTIFACT_JOIN_RE = re.compile(r"(\w)[\u00ad\ufffe](\w)")
@@ -1354,9 +1361,45 @@ class PDFParser(BaseParser):
Hybrid documents interleave both in reading order. On any unexpected error
the parser falls back to rendering all pages as images (safe last resort).
Force-scanned mode (``pdf_force_scanned=true`` override or
``DOCREADER_PDF_FORCE_SCANNED=true`` env) skips classification and
renders every page as an image.
"""
def __init__(self, file_name: str = "", file_type=None, **kwargs):
# Capture per-upload override before BaseParser consumes kwargs.
raw = kwargs.pop("pdf_force_scanned", None)
super().__init__(file_name=file_name, file_type=file_type, **kwargs)
# Priority: per-upload override > global env > default (False).
if raw is not None:
self._force_scanned = str(raw).strip().lower() in {
"1", "true", "yes", "y", "on",
}
else:
self._force_scanned = FORCE_SCANNED_PDF
def parse_into_text(self, content: bytes) -> Document:
# Force-scanned short-circuit: render every page as an image.
if self._force_scanned:
logger.info(
"PDFParser: force scanned mode enabled for %s",
self.file_name,
)
doc = PDFScannedParser(
file_name=self.file_name, file_type=self.file_type
).parse_into_text(content)
# Align metadata fields with automatic scanned route
page_count = doc.metadata.get("page_count", 0)
doc.metadata.update({
"scanned_page_count": page_count,
"text_page_count": 0,
"embedded_image_count": 0,
"vector_figure_count": 0,
})
return doc
try:
return self._route(content)
except Exception:
docreader/tests/test_pdf_router.py
+67
View File
@@ -433,5 +433,72 @@ class PDFRouterIntegrationTest(unittest.TestCase):
)
class ForceScannedTest(unittest.TestCase):
"""Tests for the force-scanned PDF parsing mode."""
def test_default_text_pdf_is_not_force_scanned(self):
"""Without any override, a text PDF should NOT use scanned mode."""
parser = PDFParser(file_name="normal.pdf", file_type="pdf")
self.assertFalse(parser._force_scanned)
def test_force_scanned_via_kwarg_true(self):
"""Per-upload override pdf_force_scanned=true enables scanned mode."""
parser = PDFParser(
file_name="test.pdf", file_type="pdf", pdf_force_scanned="true"
)
self.assertTrue(parser._force_scanned)
def test_force_scanned_via_kwarg_false(self):
"""Per-upload override pdf_force_scanned=false disables scanned mode."""
parser = PDFParser(
file_name="test.pdf", file_type="pdf", pdf_force_scanned="false"
)
self.assertFalse(parser._force_scanned)
def test_force_scanned_via_kwarg_overrides_env(self, monkeypatch=None):
"""Per-upload override takes priority over global env var."""
import docreader.parser.pdf_parser as pdf_mod
old = pdf_mod.FORCE_SCANNED_PDF
try:
pdf_mod.FORCE_SCANNED_PDF = True
# Explicit false override should win over env=true
parser = PDFParser(
file_name="test.pdf", file_type="pdf", pdf_force_scanned="false"
)
self.assertFalse(parser._force_scanned)
finally:
pdf_mod.FORCE_SCANNED_PDF = old
def test_force_scanned_via_env(self):
"""Global env variable enables force scanned when no override."""
import docreader.parser.pdf_parser as pdf_mod
old = pdf_mod.FORCE_SCANNED_PDF
try:
pdf_mod.FORCE_SCANNED_PDF = True
parser = PDFParser(file_name="test.pdf", file_type="pdf")
self.assertTrue(parser._force_scanned)
finally:
pdf_mod.FORCE_SCANNED_PDF = old
def test_force_scanned_output_is_scanned_pdf(self):
"""Force-scanned mode produces scanned_pdf metadata and image refs."""
pdf_bytes = _make_image_only_pdf(2)
parser = PDFParser(
file_name="forced.pdf", file_type="pdf", pdf_force_scanned="true"
)
doc = parser.parse_into_text(pdf_bytes)
self.assertEqual(doc.metadata.get("image_source_type"), "scanned_pdf")
self.assertEqual(doc.metadata.get("page_count"), 2)
self.assertEqual(doc.metadata.get("scanned_page_count"), 2)
self.assertEqual(doc.metadata.get("text_page_count"), 0)
self.assertEqual(doc.metadata.get("embedded_image_count"), 0)
self.assertEqual(doc.metadata.get("vector_figure_count"), 0)
self.assertIn("![forced_page_1.jpg]", doc.content)
self.assertIn("![forced_page_2.jpg]", doc.content)
self.assertEqual(len(doc.images), 2)
if __name__ == "__main__":
unittest.main()
frontend/src/i18n/locales/en-US.ts
+4
View File
@@ -701,6 +701,10 @@ export default {
asrRequiredForAudio: 'Off (batch includes audio)',
vlmModelSelectRequired: 'Multimodal is enabled. Please select a VLM model.',
asrModelSelectRequired: 'Speech recognition is enabled. Please select an ASR model.',
pdfForceScanned: {
label: 'Force scanned PDF parsing',
description: 'Useful for web-print, scanned, or image-heavy PDFs. Every page will be rendered as an image and processed via OCR/VLM. May increase processing time and model costs.'
},
continueAdd: 'Add more',
addMoreFiles: 'Add more files',
addMoreFolder: 'Add more from folder',
frontend/src/i18n/locales/zh-CN.ts
+4
View File
@@ -704,6 +704,10 @@ export default {
asrRequiredForAudio: "未开启（本批次含音频）",
vlmModelSelectRequired: "已启用多模态，请选择 VLM 模型",
asrModelSelectRequired: "已启用语音识别，请选择 ASR 模型",
pdfForceScanned: {
label: "按扫描件解析 PDF",
description: "适用于网页打印、扫描件、图片型 PDF。开启后会逐页 OCR，解析更完整但耗时和模型调用更多。"
},
continueAdd: "继续添加",
addMoreFiles: "继续添加文件",
addMoreFolder: "继续添加文件夹",
frontend/src/types/knowledgeProcess.ts
+1
View File
@@ -62,4 +62,5 @@ export interface KnowledgeProcessOverrides {
question_generation_config?: QuestionGenerationConfigOverride
graph_enabled?: boolean
extract_config?: ExtractConfigOverride
parser_engine_overrides?: Record<string, string>
}
frontend/src/views/knowledge/components/UploadConfirmDialog.vue
+32 -1
View File
@@ -124,6 +124,17 @@
:parser-engine-rules="uiState.chunkingConfig.parserEngineRules"
@update:parser-engine-rules="handleParserEngineRulesUpdate"
/>
<div v-if="hasPdf" class="kb-embedded-settings" style="margin-top: 16px;">
<div class="setting-row setting-row--toggle">
<div class="setting-info">
<label>{{ t('uploadConfirm.pdfForceScanned.label') }}</label>
<p class="desc">{{ t('uploadConfirm.pdfForceScanned.description') }}</p>
</div>
<div class="setting-control">
<t-switch v-model="uiState.pdfForceScanned" size="medium" />
</div>
</div>
</div>
</div>
<div v-show="activeSection === 'chunking'" class="section">
<KBChunkingSettings
@@ -290,6 +301,7 @@ interface UploadUIState {
relations: Array<{ node1: string; node2: string; type: string }>
}
graphEnabled: boolean
pdfForceScanned: boolean
}
const props = withDefaults(defineProps<{
@@ -437,6 +449,10 @@ const batchFileExts = computed(() => {
return [...set]
})
const hasPdf = computed(() => {
return batchFileExts.value.includes('pdf')
})
function resolveEngineForExt(ext: string): string {
const rules = uiState.value.chunkingConfig.parserEngineRules
if (rules?.length) {
@@ -624,6 +640,7 @@ function createDefaultUIState(): UploadUIState {
relations: [],
},
graphEnabled: false,
pdfForceScanned: false,
}
}
@@ -670,6 +687,7 @@ function initFromKbInfo(kb: any) {
relations: kb.extract_config?.relations || [],
},
graphEnabled: kb.indexing_strategy?.graph_enabled ?? false,
pdfForceScanned: false,
}
}
@@ -677,7 +695,7 @@ function buildProcessOverrides(): KnowledgeProcessOverrides {
const state = uiState.value
const chunking = state.chunkingConfig
return {
const overrides: KnowledgeProcessOverrides = {
parser_engine_rules: chunking.parserEngineRules,
chunking_config: {
chunk_size: chunking.chunkSize,
@@ -713,6 +731,14 @@ function buildProcessOverrides(): KnowledgeProcessOverrides {
relations: state.nodeExtractConfig.relations,
},
}
if (state.pdfForceScanned) {
overrides.parser_engine_overrides = {
pdf_force_scanned: 'true',
}
}
return overrides
}
function applyOverridesToState(o?: KnowledgeProcessOverrides | null) {
@@ -756,6 +782,11 @@ function applyOverridesToState(o?: KnowledgeProcessOverrides | null) {
if (ec.relations) s.nodeExtractConfig.relations = ec.relations
}
if (o.graph_enabled != null) s.graphEnabled = o.graph_enabled
if (o.parser_engine_overrides && o.parser_engine_overrides.pdf_force_scanned === 'true') {
s.pdfForceScanned = true
} else {
s.pdfForceScanned = false
}
}
async function loadModels() {
internal/application/service/knowledge_process.go
+8 -3
View File
@@ -3099,7 +3099,12 @@ func (s *knowledgeService) convert(
s.beginStage(ctx, knowledge.ID, types.StageDocReader, docInput)
isURL := payload.URL != ""
fileType := payload.FileType
overrides := s.getParserEngineOverridesFromContext(ctx)
tenantOverrides := s.getParserEngineOverridesFromContext(ctx)
var uploadOverrides map[string]string
if processOverrides, err := knowledge.ProcessOverrides(); err == nil && processOverrides != nil {
uploadOverrides = processOverrides.ParserEngineOverrides
}
mergedOverrides := MergeParserEngineOverrides(tenantOverrides, uploadOverrides)
if isURL {
if err := secutils.ValidateURLForSSRF(payload.URL); err != nil {
@@ -3122,7 +3127,7 @@ func (s *knowledgeService) convert(
logger.Infof(ctx, "[convert] kb=%s fileType=%s isURL=%v engine=%q rules=%+v",
kb.ID, fileType, isURL, parserEngine, eff.ChunkingConfig.ParserEngineRules)
var reader interfaces.DocReader = s.resolveDocReader(ctx, parserEngine, fileType, isURL, overrides)
var reader interfaces.DocReader = s.resolveDocReader(ctx, parserEngine, fileType, isURL, mergedOverrides)
if reader == nil {
logger.Errorf(ctx, "[convert] no doc reader for kb=%s knowledge=%s fileType=%s engine=%q isURL=%v",
kb.ID, knowledge.ID, fileType, parserEngine, isURL)
@@ -3140,7 +3145,7 @@ func (s *knowledgeService) convert(
Title: knowledge.Title,
ParserEngine: parserEngine,
RequestID: payload.RequestId,
ParserEngineOverrides: overrides,
ParserEngineOverrides: mergedOverrides,
}
if !isURL {
internal/application/service/knowledge_process_config.go
+12
View File
@@ -246,3 +246,15 @@ func validateImageMultimodalConfig(ctx context.Context, kb *types.KnowledgeBase)
return nil
}
// MergeParserEngineOverrides merges upload overrides on top of tenant overrides safely.
func MergeParserEngineOverrides(tenantOverrides map[string]string, uploadOverrides map[string]string) map[string]string {
merged := make(map[string]string)
for k, v := range tenantOverrides {
merged[k] = v
}
for k, v := range uploadOverrides {
merged[k] = v
}
return merged
}
internal/application/service/knowledge_process_config_test.go
+26
View File
@@ -278,3 +278,29 @@ func TestValidateProcessOverrides_COSIncompleteForImage(t *testing.T) {
err := ValidateProcessOverrides(ctx, kb, &types.KnowledgeProcessOverrides{}, []string{"png"})
require.Error(t, err)
}
func TestMergeParserEngineOverrides(t *testing.T) {
t.Parallel()
// 1. Both nil/empty
merged := MergeParserEngineOverrides(nil, nil)
require.Empty(t, merged)
// 2. Tenant only
merged = MergeParserEngineOverrides(map[string]string{"k1": "v1"}, nil)
require.Equal(t, map[string]string{"k1": "v1"}, merged)
// 3. Upload only
merged = MergeParserEngineOverrides(nil, map[string]string{"k2": "v2"})
require.Equal(t, map[string]string{"k2": "v2"}, merged)
// 4. Overlap priority (upload override should take priority over tenant config)
tenant := map[string]string{"k1": "tenant_val", "k2": "v2"}
upload := map[string]string{"k1": "upload_val", "k3": "v3"}
merged = MergeParserEngineOverrides(tenant, upload)
require.Equal(t, map[string]string{
"k1": "upload_val",
"k2": "v2",
"k3": "v3",
}, merged)
}
internal/types/knowledge_process.go
+4
View File
@@ -10,6 +10,10 @@ type KnowledgeProcessOverrides struct {
QuestionGenerationConfig *QuestionGenerationConfig `json:"question_generation_config,omitempty"`
GraphEnabled *bool `json:"graph_enabled,omitempty"`
ExtractConfig *ExtractConfig `json:"extract_config,omitempty"`
// ParserEngineOverrides passes key-value configuration to docreader parsers
// (e.g. pdf_force_scanned=true). Merged with tenant-level overrides in the
// parse pipeline; per-upload values take priority on conflict.
ParserEngineOverrides map[string]string `json:"parser_engine_overrides,omitempty"`
}
// EffectiveProcessConfig is the merged view used by the parse pipeline.
internal/types/knowledge_process_test.go
+2
View File
@@ -15,6 +15,7 @@ func TestKnowledgeProcessOverridesRoundtrip(t *testing.T) {
overrides := &KnowledgeProcessOverrides{
EnableMultimodel: boolPtr(true),
ChunkingConfig: &ChunkingConfig{ChunkSize: 1024},
ParserEngineOverrides: map[string]string{"pdf_force_scanned": "true"},
}
require.NoError(t, k.SetProcessOverrides(overrides))
got, err := k.ProcessOverrides()
@@ -22,6 +23,7 @@ func TestKnowledgeProcessOverridesRoundtrip(t *testing.T) {
require.NotNil(t, got)
require.True(t, *got.EnableMultimodel)
require.Equal(t, 1024, got.ChunkingConfig.ChunkSize)
require.Equal(t, "true", got.ParserEngineOverrides["pdf_force_scanned"])
}
func TestSetProcessOverridesPreservesOtherMetadata(t *testing.T) {