feat(asset): 清理被截断的结尾图片引用,修复整单 502

仓库首次入库(此前仅存在于本地工作区与测试服构建产物中,未上传)。

改动(src/knowledge_adapter/asset_content_processor.py):
- 新增 _TRUNCATED_TRAILING_MARKDOWN_RESOURCE_IMAGE 规则,清除 chunk 末尾被截断的
  ![](resource://xxxx 残片(WeKnora 分块在图片引用中间截断时产生,旧逻辑无法匹配完整引用,
  残留 resource:// 触发 ContentProcessingError,导致整单 502)
- 回归测试 tests/test_asset_content_processor.py::
  test_truncated_trailing_markdown_resource_image_is_removed

验证证据:
- 测试服 120.76.217.51,镜像 knowledge-adapter-service:reviewed-test-aeab4b5fca2d
  (release aeab4b5fca2d8661):KB d015d05c-25c8-4d63-b2ab-e8590d3c2dfb 5 个查询全部
  http=200、records=5、残留 resource:// = 0;KB b9dacd62 无退化;容器日志无 error/exception
- 本地:pytest 141 项全通过、ruff 全通过
- 该文件与 258c31e0fe07fd08 源码的差异经逐文件 md5 比对,为唯一变更文件
This commit is contained in:
姚章浩
2026-09-16 10:01:27 +08:00
commit 693cd952cd
64 changed files with 7869 additions and 0 deletions
@@ -0,0 +1,166 @@
import json
import re
from typing import Protocol
from knowledge_adapter.asset_repository import (
AssetMapping,
AssetMappingConflictError,
AssetRepositoryError,
)
from knowledge_adapter.models import WeKnoraChunk
from knowledge_adapter.resource_locator import InvalidResourceReference, parse_resource_reference
from knowledge_adapter.weknora_resource_resolver import (
ResolvedAsset,
ResourceResolutionError,
)
_MARKDOWN_RESOURCE_IMAGE = re.compile(
r"(?P<prefix>!\[[^\]\n]*\]\()"
r"(?P<source_ref>resource://[A-Za-z0-9_-]{22})"
r"(?P<suffix>\))"
)
_HTML_RESOURCE_IMAGE = re.compile(
r"(?P<prefix><img\b[^>]*?\bsrc\s*=\s*(?P<quote>['\"]))"
r"(?P<source_ref>resource://[A-Za-z0-9_-]{22})"
r"(?P<suffix>(?P=quote)[^>]*>)",
re.IGNORECASE,
)
_TRUNCATED_TRAILING_MARKDOWN_RESOURCE_IMAGE = re.compile(
r"!\[[^\]\n]*\]\(resource://[A-Za-z0-9_-]{0,22}\Z"
)
_FORBIDDEN_CONTENT = re.compile(
r"(?:"
r"local://"
r"|(?<![A-Za-z0-9])/(?:home|srv|var|etc|root|tmp)/"
r"|https?://(?:localhost|127(?:\.\d{1,3}){3}|10(?:\.\d{1,3}){3}|"
r"192\.168(?:\.\d{1,3}){2}|172\.(?:1[6-9]|2\d|3[01])(?:\.\d{1,3}){2})"
r"(?=[:/])"
r"|https?://[^\s)\]]*\.aliyuncs\.com(?=[:/]|$)"
r")",
re.IGNORECASE,
)
class ContentProcessingError(RuntimeError):
pass
class MappingRepository(Protocol):
async def get_active_by_source_ref(self, source_ref: str) -> AssetMapping | None: ...
async def get_or_create(
self,
*,
source_ref: str,
object_key: str,
content_type: str | None,
) -> AssetMapping: ...
class ResourceResolver(Protocol):
async def resolve(self, source_ref: str) -> ResolvedAsset: ...
class AssetContentProcessor:
def __init__(
self,
*,
repository: MappingRepository,
resolver: ResourceResolver,
public_base_url: str,
) -> None:
self._repository = repository
self._resolver = resolver
self._public_base_url = public_base_url.rstrip("/")
async def _public_url(
self,
source_ref: str,
cache: dict[str, str],
) -> str:
cached = cache.get(source_ref)
if cached is not None:
return cached
try:
mapping = await self._repository.get_active_by_source_ref(source_ref)
if mapping is None:
resolved = await self._resolver.resolve(source_ref)
mapping = await self._repository.get_or_create(
source_ref=source_ref,
object_key=resolved.object_key,
content_type=resolved.content_type,
)
except (
AssetMappingConflictError,
AssetRepositoryError,
ResourceResolutionError,
) as error:
raise ContentProcessingError("content processing failed") from error
public_url = f"{self._public_base_url}/i/{mapping.asset_id}"
cache[source_ref] = public_url
return public_url
async def process(self, chunk: WeKnoraChunk) -> str:
parts: list[str] = []
public_urls: dict[str, str] = {}
cursor = 0
for match in _MARKDOWN_RESOURCE_IMAGE.finditer(chunk.content):
source_ref = match.group("source_ref")
public_url = await self._public_url(source_ref, public_urls)
parts.extend(
(
chunk.content[cursor : match.start()],
match.group("prefix"),
public_url,
match.group("suffix"),
)
)
cursor = match.end()
parts.append(chunk.content[cursor:])
processed = "".join(parts)
parts = []
cursor = 0
for match in _HTML_RESOURCE_IMAGE.finditer(processed):
source_ref = match.group("source_ref")
public_url = await self._public_url(source_ref, public_urls)
parts.extend(
(
processed[cursor : match.start()],
match.group("prefix"),
public_url,
match.group("suffix"),
)
)
cursor = match.end()
parts.append(processed[cursor:])
processed = "".join(parts)
processed = _TRUNCATED_TRAILING_MARKDOWN_RESOURCE_IMAGE.sub("", processed)
seen_refs = set(public_urls)
try:
image_entries = json.loads(chunk.image_info) if chunk.image_info else []
except json.JSONDecodeError as error:
raise ContentProcessingError("content processing failed") from error
if not isinstance(image_entries, list):
raise ContentProcessingError("content processing failed")
for entry in image_entries:
if not isinstance(entry, dict):
continue
for field in ("url", "original_url"):
candidate = entry.get(field)
if not isinstance(candidate, str):
continue
try:
source_ref = f"resource://{parse_resource_reference(candidate)}"
except InvalidResourceReference:
continue
if source_ref in seen_refs:
continue
seen_refs.add(source_ref)
public_url = await self._public_url(source_ref, public_urls)
separator = "\n\n" if processed else ""
processed += f"{separator}![知识图片]({public_url})"
if "resource://" in processed or _FORBIDDEN_CONTENT.search(processed):
raise ContentProcessingError("content processing failed")
return processed