693cd952cd
仓库首次入库(此前仅存在于本地工作区与测试服构建产物中,未上传)。 改动(src/knowledge_adapter/asset_content_processor.py): - 新增 _TRUNCATED_TRAILING_MARKDOWN_RESOURCE_IMAGE 规则,清除 chunk 末尾被截断的  - 回归测试 tests/test_asset_content_processor.py:: test_truncated_trailing_markdown_resource_image_is_removed 验证证据: - 测试服 120.76.217.51,镜像 knowledge-adapter-service:reviewed-test-aeab4b5fca2d (release aeab4b5fca2d8661):KB d015d05c-25c8-4d63-b2ab-e8590d3c2dfb 5 个查询全部 http=200、records=5、残留 resource:// = 0;KB b9dacd62 无退化;容器日志无 error/exception - 本地:pytest 141 项全通过、ruff 全通过 - 该文件与 258c31e0fe07fd08 源码的差异经逐文件 md5 比对,为唯一变更文件
167 lines
5.6 KiB
Python
167 lines
5.6 KiB
Python
import json
|
|
import re
|
|
from typing import Protocol
|
|
|
|
from knowledge_adapter.asset_repository import (
|
|
AssetMapping,
|
|
AssetMappingConflictError,
|
|
AssetRepositoryError,
|
|
)
|
|
from knowledge_adapter.models import WeKnoraChunk
|
|
from knowledge_adapter.resource_locator import InvalidResourceReference, parse_resource_reference
|
|
from knowledge_adapter.weknora_resource_resolver import (
|
|
ResolvedAsset,
|
|
ResourceResolutionError,
|
|
)
|
|
|
|
_MARKDOWN_RESOURCE_IMAGE = re.compile(
|
|
r"(?P<prefix>!\[[^\]\n]*\]\()"
|
|
r"(?P<source_ref>resource://[A-Za-z0-9_-]{22})"
|
|
r"(?P<suffix>\))"
|
|
)
|
|
_HTML_RESOURCE_IMAGE = re.compile(
|
|
r"(?P<prefix><img\b[^>]*?\bsrc\s*=\s*(?P<quote>['\"]))"
|
|
r"(?P<source_ref>resource://[A-Za-z0-9_-]{22})"
|
|
r"(?P<suffix>(?P=quote)[^>]*>)",
|
|
re.IGNORECASE,
|
|
)
|
|
_TRUNCATED_TRAILING_MARKDOWN_RESOURCE_IMAGE = re.compile(
|
|
r"!\[[^\]\n]*\]\(resource://[A-Za-z0-9_-]{0,22}\Z"
|
|
)
|
|
_FORBIDDEN_CONTENT = re.compile(
|
|
r"(?:"
|
|
r"local://"
|
|
r"|(?<![A-Za-z0-9])/(?:home|srv|var|etc|root|tmp)/"
|
|
r"|https?://(?:localhost|127(?:\.\d{1,3}){3}|10(?:\.\d{1,3}){3}|"
|
|
r"192\.168(?:\.\d{1,3}){2}|172\.(?:1[6-9]|2\d|3[01])(?:\.\d{1,3}){2})"
|
|
r"(?=[:/])"
|
|
r"|https?://[^\s)\]]*\.aliyuncs\.com(?=[:/]|$)"
|
|
r")",
|
|
re.IGNORECASE,
|
|
)
|
|
|
|
|
|
class ContentProcessingError(RuntimeError):
|
|
pass
|
|
|
|
|
|
class MappingRepository(Protocol):
|
|
async def get_active_by_source_ref(self, source_ref: str) -> AssetMapping | None: ...
|
|
|
|
async def get_or_create(
|
|
self,
|
|
*,
|
|
source_ref: str,
|
|
object_key: str,
|
|
content_type: str | None,
|
|
) -> AssetMapping: ...
|
|
|
|
|
|
class ResourceResolver(Protocol):
|
|
async def resolve(self, source_ref: str) -> ResolvedAsset: ...
|
|
|
|
|
|
class AssetContentProcessor:
|
|
def __init__(
|
|
self,
|
|
*,
|
|
repository: MappingRepository,
|
|
resolver: ResourceResolver,
|
|
public_base_url: str,
|
|
) -> None:
|
|
self._repository = repository
|
|
self._resolver = resolver
|
|
self._public_base_url = public_base_url.rstrip("/")
|
|
|
|
async def _public_url(
|
|
self,
|
|
source_ref: str,
|
|
cache: dict[str, str],
|
|
) -> str:
|
|
cached = cache.get(source_ref)
|
|
if cached is not None:
|
|
return cached
|
|
try:
|
|
mapping = await self._repository.get_active_by_source_ref(source_ref)
|
|
if mapping is None:
|
|
resolved = await self._resolver.resolve(source_ref)
|
|
mapping = await self._repository.get_or_create(
|
|
source_ref=source_ref,
|
|
object_key=resolved.object_key,
|
|
content_type=resolved.content_type,
|
|
)
|
|
except (
|
|
AssetMappingConflictError,
|
|
AssetRepositoryError,
|
|
ResourceResolutionError,
|
|
) as error:
|
|
raise ContentProcessingError("content processing failed") from error
|
|
public_url = f"{self._public_base_url}/i/{mapping.asset_id}"
|
|
cache[source_ref] = public_url
|
|
return public_url
|
|
|
|
async def process(self, chunk: WeKnoraChunk) -> str:
|
|
parts: list[str] = []
|
|
public_urls: dict[str, str] = {}
|
|
cursor = 0
|
|
for match in _MARKDOWN_RESOURCE_IMAGE.finditer(chunk.content):
|
|
source_ref = match.group("source_ref")
|
|
public_url = await self._public_url(source_ref, public_urls)
|
|
parts.extend(
|
|
(
|
|
chunk.content[cursor : match.start()],
|
|
match.group("prefix"),
|
|
public_url,
|
|
match.group("suffix"),
|
|
)
|
|
)
|
|
cursor = match.end()
|
|
parts.append(chunk.content[cursor:])
|
|
processed = "".join(parts)
|
|
|
|
parts = []
|
|
cursor = 0
|
|
for match in _HTML_RESOURCE_IMAGE.finditer(processed):
|
|
source_ref = match.group("source_ref")
|
|
public_url = await self._public_url(source_ref, public_urls)
|
|
parts.extend(
|
|
(
|
|
processed[cursor : match.start()],
|
|
match.group("prefix"),
|
|
public_url,
|
|
match.group("suffix"),
|
|
)
|
|
)
|
|
cursor = match.end()
|
|
parts.append(processed[cursor:])
|
|
processed = "".join(parts)
|
|
processed = _TRUNCATED_TRAILING_MARKDOWN_RESOURCE_IMAGE.sub("", processed)
|
|
|
|
seen_refs = set(public_urls)
|
|
try:
|
|
image_entries = json.loads(chunk.image_info) if chunk.image_info else []
|
|
except json.JSONDecodeError as error:
|
|
raise ContentProcessingError("content processing failed") from error
|
|
if not isinstance(image_entries, list):
|
|
raise ContentProcessingError("content processing failed")
|
|
for entry in image_entries:
|
|
if not isinstance(entry, dict):
|
|
continue
|
|
for field in ("url", "original_url"):
|
|
candidate = entry.get(field)
|
|
if not isinstance(candidate, str):
|
|
continue
|
|
try:
|
|
source_ref = f"resource://{parse_resource_reference(candidate)}"
|
|
except InvalidResourceReference:
|
|
continue
|
|
if source_ref in seen_refs:
|
|
continue
|
|
seen_refs.add(source_ref)
|
|
public_url = await self._public_url(source_ref, public_urls)
|
|
separator = "\n\n" if processed else ""
|
|
processed += f"{separator}"
|
|
if "resource://" in processed or _FORBIDDEN_CONTENT.search(processed):
|
|
raise ContentProcessingError("content processing failed")
|
|
return processed
|