diff --git a/.env.example b/.env.example index b49f589e..a17e78f4 100644 --- a/.env.example +++ b/.env.example @@ -588,6 +588,9 @@ DOCREADER_TRANSPORT=grpc # DOCREADER_PDF_SANITIZE_TEXT=true # DOCREADER_PDF_STRIP_CHART_DEBRIS=true # DOCREADER_PDF_RENDER_VECTOR_FIGURES=true +# 强制 PDF 全页按扫描件解析。适合文本层质量很差、但页面可 OCR 的 PDF。 +# 开启后所有 PDF 页都会渲染成图片再走 OCR/VLM，速度和模型调用成本会上升。 +# DOCREADER_PDF_FORCE_SCANNED=false # OpenDataLoader PDF（知识库 parser_engine_rules 指定 engine: opendataloader） # 需 Java 11+；docreader 镜像已包含 openjdk-17-jre-headless。 diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index fa2dbb0d..d0f90dd6 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -248,6 +248,7 @@ services: - docreader-tmp-dev:/tmp/docreader environment: - DOCREADER_IMAGE_OUTPUT_DIR=/tmp/docreader + - DOCREADER_PDF_FORCE_SCANNED=${DOCREADER_PDF_FORCE_SCANNED:-false} - MAX_FILE_SIZE_MB=${MAX_FILE_SIZE_MB:-} - DOCREADER_ODL_MAX_WORKERS=${DOCREADER_ODL_MAX_WORKERS:-1} - DOCREADER_ODL_HYBRID=${DOCREADER_ODL_HYBRID:-off} diff --git a/docker-compose.yml b/docker-compose.yml index 725241c5..fec9196e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -240,6 +240,7 @@ services: - docreader-tmp:/tmp/docreader environment: - DOCREADER_IMAGE_OUTPUT_DIR=/tmp/docreader + - DOCREADER_PDF_FORCE_SCANNED=${DOCREADER_PDF_FORCE_SCANNED:-false} - MAX_FILE_SIZE_MB=${MAX_FILE_SIZE_MB:-} - DOCREADER_ODL_MAX_WORKERS=${DOCREADER_ODL_MAX_WORKERS:-1} - DOCREADER_ODL_HYBRID=${DOCREADER_ODL_HYBRID:-off} diff --git a/docreader/parser/pdf_parser.py b/docreader/parser/pdf_parser.py index dde14e11..35ea202f 100644 --- a/docreader/parser/pdf_parser.py +++ b/docreader/parser/pdf_parser.py @@ -115,6 +115,13 @@ MIN_CHART_REGION_AREA_RATIO = _env_float("DOCREADER_PDF_MIN_CHART_REGION_AREA", MAX_CHART_REGION_AREA_RATIO = _env_float("DOCREADER_PDF_MAX_CHART_REGION_AREA", 0.42) MAX_FIGURE_HEIGHT_RATIO = _env_float("DOCREADER_PDF_MAX_FIGURE_HEIGHT_RATIO", 0.38) +# --- Force scanned mode ------------------------------------------------------ +# When True, ALL PDF pages are rendered as images and routed through OCR/VLM, +# bypassing the automatic text/scanned page classification. Useful for PDFs +# with a low-quality or misleading text layer (web-print, scanned, image-heavy). +# Can be overridden per-upload via parser_engine_overrides.pdf_force_scanned. +FORCE_SCANNED_PDF = _env_bool("DOCREADER_PDF_FORCE_SCANNED", False) + # pdfium / Adobe text layers often emit U+FFFE for missing hyphenation or ligatures. _PDF_ARTIFACT_RE = re.compile(r"[\u00ad\u200b-\u200f\ufeff\ufffe\uffff]") _PDF_ARTIFACT_JOIN_RE = re.compile(r"(\w)[\u00ad\ufffe](\w)") @@ -1354,9 +1361,45 @@ class PDFParser(BaseParser): Hybrid documents interleave both in reading order. On any unexpected error the parser falls back to rendering all pages as images (safe last resort). + + Force-scanned mode (``pdf_force_scanned=true`` override or + ``DOCREADER_PDF_FORCE_SCANNED=true`` env) skips classification and + renders every page as an image. """ + def __init__(self, file_name: str = "", file_type=None, **kwargs): + # Capture per-upload override before BaseParser consumes kwargs. + raw = kwargs.pop("pdf_force_scanned", None) + super().__init__(file_name=file_name, file_type=file_type, **kwargs) + # Priority: per-upload override > global env > default (False). + if raw is not None: + self._force_scanned = str(raw).strip().lower() in { + "1", "true", "yes", "y", "on", + } + else: + self._force_scanned = FORCE_SCANNED_PDF + def parse_into_text(self, content: bytes) -> Document: + # Force-scanned short-circuit: render every page as an image. + if self._force_scanned: + logger.info( + "PDFParser: force scanned mode enabled for %s", + self.file_name, + ) + doc = PDFScannedParser( + file_name=self.file_name, file_type=self.file_type + ).parse_into_text(content) + + # Align metadata fields with automatic scanned route + page_count = doc.metadata.get("page_count", 0) + doc.metadata.update({ + "scanned_page_count": page_count, + "text_page_count": 0, + "embedded_image_count": 0, + "vector_figure_count": 0, + }) + return doc + try: return self._route(content) except Exception: diff --git a/docreader/tests/test_pdf_router.py b/docreader/tests/test_pdf_router.py index ae29c875..dc283eaf 100644 --- a/docreader/tests/test_pdf_router.py +++ b/docreader/tests/test_pdf_router.py @@ -433,5 +433,72 @@ class PDFRouterIntegrationTest(unittest.TestCase): ) +class ForceScannedTest(unittest.TestCase): + """Tests for the force-scanned PDF parsing mode.""" + + def test_default_text_pdf_is_not_force_scanned(self): + """Without any override, a text PDF should NOT use scanned mode.""" + parser = PDFParser(file_name="normal.pdf", file_type="pdf") + self.assertFalse(parser._force_scanned) + + def test_force_scanned_via_kwarg_true(self): + """Per-upload override pdf_force_scanned=true enables scanned mode.""" + parser = PDFParser( + file_name="test.pdf", file_type="pdf", pdf_force_scanned="true" + ) + self.assertTrue(parser._force_scanned) + + def test_force_scanned_via_kwarg_false(self): + """Per-upload override pdf_force_scanned=false disables scanned mode.""" + parser = PDFParser( + file_name="test.pdf", file_type="pdf", pdf_force_scanned="false" + ) + self.assertFalse(parser._force_scanned) + + def test_force_scanned_via_kwarg_overrides_env(self, monkeypatch=None): + """Per-upload override takes priority over global env var.""" + import docreader.parser.pdf_parser as pdf_mod + + old = pdf_mod.FORCE_SCANNED_PDF + try: + pdf_mod.FORCE_SCANNED_PDF = True + # Explicit false override should win over env=true + parser = PDFParser( + file_name="test.pdf", file_type="pdf", pdf_force_scanned="false" + ) + self.assertFalse(parser._force_scanned) + finally: + pdf_mod.FORCE_SCANNED_PDF = old + + def test_force_scanned_via_env(self): + """Global env variable enables force scanned when no override.""" + import docreader.parser.pdf_parser as pdf_mod + + old = pdf_mod.FORCE_SCANNED_PDF + try: + pdf_mod.FORCE_SCANNED_PDF = True + parser = PDFParser(file_name="test.pdf", file_type="pdf") + self.assertTrue(parser._force_scanned) + finally: + pdf_mod.FORCE_SCANNED_PDF = old + + def test_force_scanned_output_is_scanned_pdf(self): + """Force-scanned mode produces scanned_pdf metadata and image refs.""" + pdf_bytes = _make_image_only_pdf(2) + parser = PDFParser( + file_name="forced.pdf", file_type="pdf", pdf_force_scanned="true" + ) + doc = parser.parse_into_text(pdf_bytes) + self.assertEqual(doc.metadata.get("image_source_type"), "scanned_pdf") + self.assertEqual(doc.metadata.get("page_count"), 2) + self.assertEqual(doc.metadata.get("scanned_page_count"), 2) + self.assertEqual(doc.metadata.get("text_page_count"), 0) + self.assertEqual(doc.metadata.get("embedded_image_count"), 0) + self.assertEqual(doc.metadata.get("vector_figure_count"), 0) + self.assertIn("![forced_page_1.jpg]", doc.content) + self.assertIn("![forced_page_2.jpg]", doc.content) + self.assertEqual(len(doc.images), 2) + + if __name__ == "__main__": unittest.main() diff --git a/frontend/src/i18n/locales/en-US.ts b/frontend/src/i18n/locales/en-US.ts index fdefb20d..d8b4355c 100755 --- a/frontend/src/i18n/locales/en-US.ts +++ b/frontend/src/i18n/locales/en-US.ts @@ -701,6 +701,10 @@ export default { asrRequiredForAudio: 'Off (batch includes audio)', vlmModelSelectRequired: 'Multimodal is enabled. Please select a VLM model.', asrModelSelectRequired: 'Speech recognition is enabled. Please select an ASR model.', + pdfForceScanned: { + label: 'Force scanned PDF parsing', + description: 'Useful for web-print, scanned, or image-heavy PDFs. Every page will be rendered as an image and processed via OCR/VLM. May increase processing time and model costs.' + }, continueAdd: 'Add more', addMoreFiles: 'Add more files', addMoreFolder: 'Add more from folder', diff --git a/frontend/src/i18n/locales/zh-CN.ts b/frontend/src/i18n/locales/zh-CN.ts index 2d7fe3d8..fe3f78bd 100755 --- a/frontend/src/i18n/locales/zh-CN.ts +++ b/frontend/src/i18n/locales/zh-CN.ts @@ -704,6 +704,10 @@ export default { asrRequiredForAudio: "未开启（本批次含音频）", vlmModelSelectRequired: "已启用多模态，请选择 VLM 模型", asrModelSelectRequired: "已启用语音识别，请选择 ASR 模型", + pdfForceScanned: { + label: "按扫描件解析 PDF", + description: "适用于网页打印、扫描件、图片型 PDF。开启后会逐页 OCR，解析更完整但耗时和模型调用更多。" + }, continueAdd: "继续添加", addMoreFiles: "继续添加文件", addMoreFolder: "继续添加文件夹", diff --git a/frontend/src/types/knowledgeProcess.ts b/frontend/src/types/knowledgeProcess.ts index d1c7af16..86613536 100644 --- a/frontend/src/types/knowledgeProcess.ts +++ b/frontend/src/types/knowledgeProcess.ts @@ -62,4 +62,5 @@ export interface KnowledgeProcessOverrides { question_generation_config?: QuestionGenerationConfigOverride graph_enabled?: boolean extract_config?: ExtractConfigOverride + parser_engine_overrides?: Record } diff --git a/frontend/src/views/knowledge/components/UploadConfirmDialog.vue b/frontend/src/views/knowledge/components/UploadConfirmDialog.vue index a0fa4093..39d5e1a4 100644 --- a/frontend/src/views/knowledge/components/UploadConfirmDialog.vue +++ b/frontend/src/views/knowledge/components/UploadConfirmDialog.vue @@ -124,6 +124,17 @@ :parser-engine-rules="uiState.chunkingConfig.parserEngineRules" @update:parser-engine-rules="handleParserEngineRulesUpdate" /> +
+
+
+ +

{{ t('uploadConfirm.pdfForceScanned.description') }}

+
+
+ +
+
+
} graphEnabled: boolean + pdfForceScanned: boolean } const props = withDefaults(defineProps<{ @@ -437,6 +449,10 @@ const batchFileExts = computed(() => { return [...set] }) +const hasPdf = computed(() => { + return batchFileExts.value.includes('pdf') +}) + function resolveEngineForExt(ext: string): string { const rules = uiState.value.chunkingConfig.parserEngineRules if (rules?.length) { @@ -624,6 +640,7 @@ function createDefaultUIState(): UploadUIState { relations: [], }, graphEnabled: false, + pdfForceScanned: false, } } @@ -670,6 +687,7 @@ function initFromKbInfo(kb: any) { relations: kb.extract_config?.relations || [], }, graphEnabled: kb.indexing_strategy?.graph_enabled ?? false, + pdfForceScanned: false, } } @@ -677,7 +695,7 @@ function buildProcessOverrides(): KnowledgeProcessOverrides { const state = uiState.value const chunking = state.chunkingConfig - return { + const overrides: KnowledgeProcessOverrides = { parser_engine_rules: chunking.parserEngineRules, chunking_config: { chunk_size: chunking.chunkSize, @@ -713,6 +731,14 @@ function buildProcessOverrides(): KnowledgeProcessOverrides { relations: state.nodeExtractConfig.relations, }, } + + if (state.pdfForceScanned) { + overrides.parser_engine_overrides = { + pdf_force_scanned: 'true', + } + } + + return overrides } function applyOverridesToState(o?: KnowledgeProcessOverrides | null) { @@ -756,6 +782,11 @@ function applyOverridesToState(o?: KnowledgeProcessOverrides | null) { if (ec.relations) s.nodeExtractConfig.relations = ec.relations } if (o.graph_enabled != null) s.graphEnabled = o.graph_enabled + if (o.parser_engine_overrides && o.parser_engine_overrides.pdf_force_scanned === 'true') { + s.pdfForceScanned = true + } else { + s.pdfForceScanned = false + } } async function loadModels() { diff --git a/internal/application/service/knowledge_process.go b/internal/application/service/knowledge_process.go index 7fd5e39e..d7a99005 100644 --- a/internal/application/service/knowledge_process.go +++ b/internal/application/service/knowledge_process.go @@ -3099,7 +3099,12 @@ func (s *knowledgeService) convert( s.beginStage(ctx, knowledge.ID, types.StageDocReader, docInput) isURL := payload.URL != "" fileType := payload.FileType - overrides := s.getParserEngineOverridesFromContext(ctx) + tenantOverrides := s.getParserEngineOverridesFromContext(ctx) + var uploadOverrides map[string]string + if processOverrides, err := knowledge.ProcessOverrides(); err == nil && processOverrides != nil { + uploadOverrides = processOverrides.ParserEngineOverrides + } + mergedOverrides := MergeParserEngineOverrides(tenantOverrides, uploadOverrides) if isURL { if err := secutils.ValidateURLForSSRF(payload.URL); err != nil { @@ -3122,7 +3127,7 @@ func (s *knowledgeService) convert( logger.Infof(ctx, "[convert] kb=%s fileType=%s isURL=%v engine=%q rules=%+v", kb.ID, fileType, isURL, parserEngine, eff.ChunkingConfig.ParserEngineRules) - var reader interfaces.DocReader = s.resolveDocReader(ctx, parserEngine, fileType, isURL, overrides) + var reader interfaces.DocReader = s.resolveDocReader(ctx, parserEngine, fileType, isURL, mergedOverrides) if reader == nil { logger.Errorf(ctx, "[convert] no doc reader for kb=%s knowledge=%s fileType=%s engine=%q isURL=%v", kb.ID, knowledge.ID, fileType, parserEngine, isURL) @@ -3140,7 +3145,7 @@ func (s *knowledgeService) convert( Title: knowledge.Title, ParserEngine: parserEngine, RequestID: payload.RequestId, - ParserEngineOverrides: overrides, + ParserEngineOverrides: mergedOverrides, } if !isURL { diff --git a/internal/application/service/knowledge_process_config.go b/internal/application/service/knowledge_process_config.go index 17a4bcd4..c1b6bb94 100644 --- a/internal/application/service/knowledge_process_config.go +++ b/internal/application/service/knowledge_process_config.go @@ -246,3 +246,15 @@ func validateImageMultimodalConfig(ctx context.Context, kb *types.KnowledgeBase) return nil } + +// MergeParserEngineOverrides merges upload overrides on top of tenant overrides safely. +func MergeParserEngineOverrides(tenantOverrides map[string]string, uploadOverrides map[string]string) map[string]string { + merged := make(map[string]string) + for k, v := range tenantOverrides { + merged[k] = v + } + for k, v := range uploadOverrides { + merged[k] = v + } + return merged +} diff --git a/internal/application/service/knowledge_process_config_test.go b/internal/application/service/knowledge_process_config_test.go index 3ca42e82..41a10b27 100644 --- a/internal/application/service/knowledge_process_config_test.go +++ b/internal/application/service/knowledge_process_config_test.go @@ -278,3 +278,29 @@ func TestValidateProcessOverrides_COSIncompleteForImage(t *testing.T) { err := ValidateProcessOverrides(ctx, kb, &types.KnowledgeProcessOverrides{}, []string{"png"}) require.Error(t, err) } + +func TestMergeParserEngineOverrides(t *testing.T) { + t.Parallel() + + // 1. Both nil/empty + merged := MergeParserEngineOverrides(nil, nil) + require.Empty(t, merged) + + // 2. Tenant only + merged = MergeParserEngineOverrides(map[string]string{"k1": "v1"}, nil) + require.Equal(t, map[string]string{"k1": "v1"}, merged) + + // 3. Upload only + merged = MergeParserEngineOverrides(nil, map[string]string{"k2": "v2"}) + require.Equal(t, map[string]string{"k2": "v2"}, merged) + + // 4. Overlap priority (upload override should take priority over tenant config) + tenant := map[string]string{"k1": "tenant_val", "k2": "v2"} + upload := map[string]string{"k1": "upload_val", "k3": "v3"} + merged = MergeParserEngineOverrides(tenant, upload) + require.Equal(t, map[string]string{ + "k1": "upload_val", + "k2": "v2", + "k3": "v3", + }, merged) +} diff --git a/internal/types/knowledge_process.go b/internal/types/knowledge_process.go index 1364d0bd..48c695b4 100644 --- a/internal/types/knowledge_process.go +++ b/internal/types/knowledge_process.go @@ -10,6 +10,10 @@ type KnowledgeProcessOverrides struct { QuestionGenerationConfig *QuestionGenerationConfig `json:"question_generation_config,omitempty"` GraphEnabled *bool `json:"graph_enabled,omitempty"` ExtractConfig *ExtractConfig `json:"extract_config,omitempty"` + // ParserEngineOverrides passes key-value configuration to docreader parsers + // (e.g. pdf_force_scanned=true). Merged with tenant-level overrides in the + // parse pipeline; per-upload values take priority on conflict. + ParserEngineOverrides map[string]string `json:"parser_engine_overrides,omitempty"` } // EffectiveProcessConfig is the merged view used by the parse pipeline. diff --git a/internal/types/knowledge_process_test.go b/internal/types/knowledge_process_test.go index 825f35d1..336f3a05 100644 --- a/internal/types/knowledge_process_test.go +++ b/internal/types/knowledge_process_test.go @@ -13,8 +13,9 @@ func boolPtr(v bool) *bool { func TestKnowledgeProcessOverridesRoundtrip(t *testing.T) { k := &Knowledge{} overrides := &KnowledgeProcessOverrides{ - EnableMultimodel: boolPtr(true), - ChunkingConfig: &ChunkingConfig{ChunkSize: 1024}, + EnableMultimodel: boolPtr(true), + ChunkingConfig: &ChunkingConfig{ChunkSize: 1024}, + ParserEngineOverrides: map[string]string{"pdf_force_scanned": "true"}, } require.NoError(t, k.SetProcessOverrides(overrides)) got, err := k.ProcessOverrides() @@ -22,6 +23,7 @@ func TestKnowledgeProcessOverridesRoundtrip(t *testing.T) { require.NotNil(t, got) require.True(t, *got.EnableMultimodel) require.Equal(t, 1024, got.ChunkingConfig.ChunkSize) + require.Equal(t, "true", got.ParserEngineOverrides["pdf_force_scanned"]) } func TestSetProcessOverridesPreservesOtherMetadata(t *testing.T) {