import json import re from typing import Protocol from knowledge_adapter.asset_repository import ( AssetMapping, AssetMappingConflictError, AssetRepositoryError, ) from knowledge_adapter.models import WeKnoraChunk from knowledge_adapter.resource_locator import InvalidResourceReference, parse_resource_reference from knowledge_adapter.weknora_resource_resolver import ( ResolvedAsset, ResourceResolutionError, ) _MARKDOWN_RESOURCE_IMAGE = re.compile( r"(?P!\[[^\]\n]*\]\()" r"(?Presource://[A-Za-z0-9_-]{22})" r"(?P\))" ) _HTML_RESOURCE_IMAGE = re.compile( r"(?P]*?\bsrc\s*=\s*(?P['\"]))" r"(?Presource://[A-Za-z0-9_-]{22})" r"(?P(?P=quote)[^>]*>)", re.IGNORECASE, ) _TRUNCATED_TRAILING_MARKDOWN_RESOURCE_IMAGE = re.compile( r"!\[[^\]\n]*\]\(resource://[A-Za-z0-9_-]{0,22}\Z" ) _FORBIDDEN_CONTENT = re.compile( r"(?:" r"local://" r"|(? AssetMapping | None: ... async def get_or_create( self, *, source_ref: str, object_key: str, content_type: str | None, ) -> AssetMapping: ... class ResourceResolver(Protocol): async def resolve(self, source_ref: str) -> ResolvedAsset: ... class AssetContentProcessor: def __init__( self, *, repository: MappingRepository, resolver: ResourceResolver, public_base_url: str, ) -> None: self._repository = repository self._resolver = resolver self._public_base_url = public_base_url.rstrip("/") async def _public_url( self, source_ref: str, cache: dict[str, str], ) -> str: cached = cache.get(source_ref) if cached is not None: return cached try: mapping = await self._repository.get_active_by_source_ref(source_ref) if mapping is None: resolved = await self._resolver.resolve(source_ref) mapping = await self._repository.get_or_create( source_ref=source_ref, object_key=resolved.object_key, content_type=resolved.content_type, ) except ( AssetMappingConflictError, AssetRepositoryError, ResourceResolutionError, ) as error: raise ContentProcessingError("content processing failed") from error public_url = f"{self._public_base_url}/i/{mapping.asset_id}" cache[source_ref] = public_url return public_url async def process(self, chunk: WeKnoraChunk) -> str: parts: list[str] = [] public_urls: dict[str, str] = {} cursor = 0 for match in _MARKDOWN_RESOURCE_IMAGE.finditer(chunk.content): source_ref = match.group("source_ref") public_url = await self._public_url(source_ref, public_urls) parts.extend( ( chunk.content[cursor : match.start()], match.group("prefix"), public_url, match.group("suffix"), ) ) cursor = match.end() parts.append(chunk.content[cursor:]) processed = "".join(parts) parts = [] cursor = 0 for match in _HTML_RESOURCE_IMAGE.finditer(processed): source_ref = match.group("source_ref") public_url = await self._public_url(source_ref, public_urls) parts.extend( ( processed[cursor : match.start()], match.group("prefix"), public_url, match.group("suffix"), ) ) cursor = match.end() parts.append(processed[cursor:]) processed = "".join(parts) processed = _TRUNCATED_TRAILING_MARKDOWN_RESOURCE_IMAGE.sub("", processed) seen_refs = set(public_urls) try: image_entries = json.loads(chunk.image_info) if chunk.image_info else [] except json.JSONDecodeError as error: raise ContentProcessingError("content processing failed") from error if not isinstance(image_entries, list): raise ContentProcessingError("content processing failed") for entry in image_entries: if not isinstance(entry, dict): continue for field in ("url", "original_url"): candidate = entry.get(field) if not isinstance(candidate, str): continue try: source_ref = f"resource://{parse_resource_reference(candidate)}" except InvalidResourceReference: continue if source_ref in seen_refs: continue seen_refs.add(source_ref) public_url = await self._public_url(source_ref, public_urls) separator = "\n\n" if processed else "" processed += f"{separator}![知识图片]({public_url})" if "resource://" in processed or _FORBIDDEN_CONTENT.search(processed): raise ContentProcessingError("content processing failed") return processed