feat(asset): 清理被截断的结尾图片引用,修复整单 502
仓库首次入库(此前仅存在于本地工作区与测试服构建产物中,未上传)。 改动(src/knowledge_adapter/asset_content_processor.py): - 新增 _TRUNCATED_TRAILING_MARKDOWN_RESOURCE_IMAGE 规则,清除 chunk 末尾被截断的  - 回归测试 tests/test_asset_content_processor.py:: test_truncated_trailing_markdown_resource_image_is_removed 验证证据: - 测试服 120.76.217.51,镜像 knowledge-adapter-service:reviewed-test-aeab4b5fca2d (release aeab4b5fca2d8661):KB d015d05c-25c8-4d63-b2ab-e8590d3c2dfb 5 个查询全部 http=200、records=5、残留 resource:// = 0;KB b9dacd62 无退化;容器日志无 error/exception - 本地:pytest 141 项全通过、ruff 全通过 - 该文件与 258c31e0fe07fd08 源码的差异经逐文件 md5 比对,为唯一变更文件
This commit is contained in:
@@ -0,0 +1,166 @@
|
||||
import json
|
||||
import re
|
||||
from typing import Protocol
|
||||
|
||||
from knowledge_adapter.asset_repository import (
|
||||
AssetMapping,
|
||||
AssetMappingConflictError,
|
||||
AssetRepositoryError,
|
||||
)
|
||||
from knowledge_adapter.models import WeKnoraChunk
|
||||
from knowledge_adapter.resource_locator import InvalidResourceReference, parse_resource_reference
|
||||
from knowledge_adapter.weknora_resource_resolver import (
|
||||
ResolvedAsset,
|
||||
ResourceResolutionError,
|
||||
)
|
||||
|
||||
_MARKDOWN_RESOURCE_IMAGE = re.compile(
|
||||
r"(?P<prefix>!\[[^\]\n]*\]\()"
|
||||
r"(?P<source_ref>resource://[A-Za-z0-9_-]{22})"
|
||||
r"(?P<suffix>\))"
|
||||
)
|
||||
_HTML_RESOURCE_IMAGE = re.compile(
|
||||
r"(?P<prefix><img\b[^>]*?\bsrc\s*=\s*(?P<quote>['\"]))"
|
||||
r"(?P<source_ref>resource://[A-Za-z0-9_-]{22})"
|
||||
r"(?P<suffix>(?P=quote)[^>]*>)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_TRUNCATED_TRAILING_MARKDOWN_RESOURCE_IMAGE = re.compile(
|
||||
r"!\[[^\]\n]*\]\(resource://[A-Za-z0-9_-]{0,22}\Z"
|
||||
)
|
||||
_FORBIDDEN_CONTENT = re.compile(
|
||||
r"(?:"
|
||||
r"local://"
|
||||
r"|(?<![A-Za-z0-9])/(?:home|srv|var|etc|root|tmp)/"
|
||||
r"|https?://(?:localhost|127(?:\.\d{1,3}){3}|10(?:\.\d{1,3}){3}|"
|
||||
r"192\.168(?:\.\d{1,3}){2}|172\.(?:1[6-9]|2\d|3[01])(?:\.\d{1,3}){2})"
|
||||
r"(?=[:/])"
|
||||
r"|https?://[^\s)\]]*\.aliyuncs\.com(?=[:/]|$)"
|
||||
r")",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
class ContentProcessingError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
class MappingRepository(Protocol):
|
||||
async def get_active_by_source_ref(self, source_ref: str) -> AssetMapping | None: ...
|
||||
|
||||
async def get_or_create(
|
||||
self,
|
||||
*,
|
||||
source_ref: str,
|
||||
object_key: str,
|
||||
content_type: str | None,
|
||||
) -> AssetMapping: ...
|
||||
|
||||
|
||||
class ResourceResolver(Protocol):
|
||||
async def resolve(self, source_ref: str) -> ResolvedAsset: ...
|
||||
|
||||
|
||||
class AssetContentProcessor:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
repository: MappingRepository,
|
||||
resolver: ResourceResolver,
|
||||
public_base_url: str,
|
||||
) -> None:
|
||||
self._repository = repository
|
||||
self._resolver = resolver
|
||||
self._public_base_url = public_base_url.rstrip("/")
|
||||
|
||||
async def _public_url(
|
||||
self,
|
||||
source_ref: str,
|
||||
cache: dict[str, str],
|
||||
) -> str:
|
||||
cached = cache.get(source_ref)
|
||||
if cached is not None:
|
||||
return cached
|
||||
try:
|
||||
mapping = await self._repository.get_active_by_source_ref(source_ref)
|
||||
if mapping is None:
|
||||
resolved = await self._resolver.resolve(source_ref)
|
||||
mapping = await self._repository.get_or_create(
|
||||
source_ref=source_ref,
|
||||
object_key=resolved.object_key,
|
||||
content_type=resolved.content_type,
|
||||
)
|
||||
except (
|
||||
AssetMappingConflictError,
|
||||
AssetRepositoryError,
|
||||
ResourceResolutionError,
|
||||
) as error:
|
||||
raise ContentProcessingError("content processing failed") from error
|
||||
public_url = f"{self._public_base_url}/i/{mapping.asset_id}"
|
||||
cache[source_ref] = public_url
|
||||
return public_url
|
||||
|
||||
async def process(self, chunk: WeKnoraChunk) -> str:
|
||||
parts: list[str] = []
|
||||
public_urls: dict[str, str] = {}
|
||||
cursor = 0
|
||||
for match in _MARKDOWN_RESOURCE_IMAGE.finditer(chunk.content):
|
||||
source_ref = match.group("source_ref")
|
||||
public_url = await self._public_url(source_ref, public_urls)
|
||||
parts.extend(
|
||||
(
|
||||
chunk.content[cursor : match.start()],
|
||||
match.group("prefix"),
|
||||
public_url,
|
||||
match.group("suffix"),
|
||||
)
|
||||
)
|
||||
cursor = match.end()
|
||||
parts.append(chunk.content[cursor:])
|
||||
processed = "".join(parts)
|
||||
|
||||
parts = []
|
||||
cursor = 0
|
||||
for match in _HTML_RESOURCE_IMAGE.finditer(processed):
|
||||
source_ref = match.group("source_ref")
|
||||
public_url = await self._public_url(source_ref, public_urls)
|
||||
parts.extend(
|
||||
(
|
||||
processed[cursor : match.start()],
|
||||
match.group("prefix"),
|
||||
public_url,
|
||||
match.group("suffix"),
|
||||
)
|
||||
)
|
||||
cursor = match.end()
|
||||
parts.append(processed[cursor:])
|
||||
processed = "".join(parts)
|
||||
processed = _TRUNCATED_TRAILING_MARKDOWN_RESOURCE_IMAGE.sub("", processed)
|
||||
|
||||
seen_refs = set(public_urls)
|
||||
try:
|
||||
image_entries = json.loads(chunk.image_info) if chunk.image_info else []
|
||||
except json.JSONDecodeError as error:
|
||||
raise ContentProcessingError("content processing failed") from error
|
||||
if not isinstance(image_entries, list):
|
||||
raise ContentProcessingError("content processing failed")
|
||||
for entry in image_entries:
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
for field in ("url", "original_url"):
|
||||
candidate = entry.get(field)
|
||||
if not isinstance(candidate, str):
|
||||
continue
|
||||
try:
|
||||
source_ref = f"resource://{parse_resource_reference(candidate)}"
|
||||
except InvalidResourceReference:
|
||||
continue
|
||||
if source_ref in seen_refs:
|
||||
continue
|
||||
seen_refs.add(source_ref)
|
||||
public_url = await self._public_url(source_ref, public_urls)
|
||||
separator = "\n\n" if processed else ""
|
||||
processed += f"{separator}"
|
||||
if "resource://" in processed or _FORBIDDEN_CONTENT.search(processed):
|
||||
raise ContentProcessingError("content processing failed")
|
||||
return processed
|
||||
Reference in New Issue
Block a user