姚章浩
693cd952cd
仓库首次入库（此前仅存在于本地工作区与测试服构建产物中，未上传）。 改动（src/knowledge_adapter/asset_content_processor.py）： - 新增 _TRUNCATED_TRAILING_MARKDOWN_RESOURCE_IMAGE 规则，清除 chunk 末尾被截断的 ![](resource://xxxx 残片（WeKnora 分块在图片引用中间截断时产生，旧逻辑无法匹配完整引用， 残留 resource:// 触发 ContentProcessingError，导致整单 502） - 回归测试 tests/test_asset_content_processor.py:: test_truncated_trailing_markdown_resource_image_is_removed 验证证据： - 测试服 120.76.217.51，镜像 knowledge-adapter-service:reviewed-test-aeab4b5fca2d （release aeab4b5fca2d8661）：KB d015d05c-25c8-4d63-b2ab-e8590d3c2dfb 5 个查询全部 http=200、records=5、残留 resource:// = 0；KB b9dacd62 无退化；容器日志无 error/exception - 本地：pytest 141 项全通过、ruff 全通过 - 该文件与 258c31e0fe07fd08 源码的差异经逐文件 md5 比对，为唯一变更文件
303 lines
9.2 KiB
Python
303 lines
9.2 KiB
Python
from uuid import UUID
|
|
|
|
import pytest
|
|
|
|
from knowledge_adapter.content_budget import TRUNCATION_MARKER
|
|
from knowledge_adapter.models import RetrievalRequest, WeKnoraChunk
|
|
from knowledge_adapter.service import InvalidRequestError, RetrievalService
|
|
|
|
ALLOWED_KB_ID = UUID("3904b40e-278b-4273-a6fa-db6cbc975587")
|
|
DENIED_KB_ID = UUID("7fa6e8d2-05d8-4e74-b98c-d8591b680db9")
|
|
|
|
|
|
class RecordingSearchClient:
|
|
def __init__(self, results: list[dict[str, object]] | None = None) -> None:
|
|
self.calls: list[tuple[str, UUID]] = []
|
|
self.results = results or []
|
|
|
|
async def search(self, *, query: str, knowledge_base_id: UUID) -> list[dict[str, object]]:
|
|
self.calls.append((query, knowledge_base_id))
|
|
return self.results
|
|
|
|
|
|
class ExpandingContentProcessor:
|
|
async def process(self, chunk: WeKnoraChunk) -> str:
|
|
return chunk.content * 100
|
|
|
|
|
|
def request_for(
|
|
knowledge_id: UUID = ALLOWED_KB_ID,
|
|
*,
|
|
top_k: int = 3,
|
|
score_threshold: float = 0.5,
|
|
) -> RetrievalRequest:
|
|
return RetrievalRequest.model_validate(
|
|
{
|
|
"knowledge_id": str(knowledge_id),
|
|
"query": "test query",
|
|
"retrieval_setting": {
|
|
"top_k": top_k,
|
|
"score_threshold": score_threshold,
|
|
},
|
|
}
|
|
)
|
|
|
|
|
|
async def test_disallowed_kb_is_rejected_without_calling_upstream() -> None:
|
|
client = RecordingSearchClient()
|
|
service = RetrievalService(
|
|
search_client=client,
|
|
allowed_kb_ids=frozenset({ALLOWED_KB_ID}),
|
|
)
|
|
|
|
with pytest.raises(InvalidRequestError):
|
|
await service.retrieve(request_for(DENIED_KB_ID))
|
|
|
|
assert client.calls == []
|
|
|
|
|
|
async def test_allowed_kb_calls_upstream_once_with_normalized_values() -> None:
|
|
client = RecordingSearchClient()
|
|
service = RetrievalService(
|
|
search_client=client,
|
|
allowed_kb_ids=frozenset({ALLOWED_KB_ID}),
|
|
)
|
|
|
|
response = await service.retrieve(request_for())
|
|
|
|
assert response == {"records": []}
|
|
assert client.calls == [("test query", ALLOWED_KB_ID)]
|
|
|
|
|
|
async def test_matching_chunk_is_mapped_to_dify_record() -> None:
|
|
chunk_id = "c7f61f89-19ce-45b1-ac60-dbeed5e2a491"
|
|
knowledge_id = "6f77a258-27ff-4278-a667-8fe242ff8e10"
|
|
client = RecordingSearchClient(
|
|
[
|
|
{
|
|
"id": chunk_id,
|
|
"content": "safe chunk content",
|
|
"score": 0.82,
|
|
"knowledge_base_id": str(ALLOWED_KB_ID),
|
|
"knowledge_id": knowledge_id,
|
|
"knowledge_title": "Operation Manual",
|
|
"knowledge_filename": "manual.pdf",
|
|
"chunk_index": 4,
|
|
"chunk_type": "text",
|
|
"image_info": "",
|
|
}
|
|
]
|
|
)
|
|
service = RetrievalService(
|
|
search_client=client,
|
|
allowed_kb_ids=frozenset({ALLOWED_KB_ID}),
|
|
)
|
|
|
|
response = await service.retrieve(request_for())
|
|
|
|
assert response == {
|
|
"records": [
|
|
{
|
|
"content": "safe chunk content",
|
|
"score": 0.82,
|
|
"title": "Operation Manual",
|
|
"metadata": {
|
|
"knowledge_base_id": str(ALLOWED_KB_ID),
|
|
"knowledge_id": knowledge_id,
|
|
"chunk_id": chunk_id,
|
|
"chunk_index": 4,
|
|
"chunk_type": "text",
|
|
},
|
|
}
|
|
]
|
|
}
|
|
|
|
|
|
async def test_cross_kb_chunk_is_dropped_after_upstream_returns() -> None:
|
|
client = RecordingSearchClient(
|
|
[
|
|
{
|
|
"id": "c7f61f89-19ce-45b1-ac60-dbeed5e2a491",
|
|
"content": "must not escape",
|
|
"score": 0.99,
|
|
"knowledge_base_id": str(DENIED_KB_ID),
|
|
"knowledge_id": "6f77a258-27ff-4278-a667-8fe242ff8e10",
|
|
"knowledge_title": "Other KB",
|
|
"knowledge_filename": "other.pdf",
|
|
"chunk_index": 0,
|
|
"chunk_type": "text",
|
|
"image_info": "",
|
|
}
|
|
]
|
|
)
|
|
service = RetrievalService(
|
|
search_client=client,
|
|
allowed_kb_ids=frozenset({ALLOWED_KB_ID, DENIED_KB_ID}),
|
|
)
|
|
|
|
response = await service.retrieve(request_for(ALLOWED_KB_ID))
|
|
|
|
assert response == {"records": []}
|
|
|
|
|
|
async def test_score_threshold_filters_without_resorting() -> None:
|
|
base = {
|
|
"knowledge_base_id": str(ALLOWED_KB_ID),
|
|
"knowledge_id": "6f77a258-27ff-4278-a667-8fe242ff8e10",
|
|
"knowledge_title": "Manual",
|
|
"knowledge_filename": "manual.pdf",
|
|
"chunk_type": "text",
|
|
"image_info": "",
|
|
}
|
|
client = RecordingSearchClient(
|
|
[
|
|
{
|
|
**base,
|
|
"id": "c7f61f89-19ce-45b1-ac60-dbeed5e2a491",
|
|
"content": "below",
|
|
"score": 0.49,
|
|
"chunk_index": 0,
|
|
},
|
|
{
|
|
**base,
|
|
"id": "09ab721f-7d05-45de-ae0f-dac303276f6d",
|
|
"content": "at threshold",
|
|
"score": 0.5,
|
|
"chunk_index": 1,
|
|
},
|
|
]
|
|
)
|
|
service = RetrievalService(
|
|
search_client=client,
|
|
allowed_kb_ids=frozenset({ALLOWED_KB_ID}),
|
|
)
|
|
|
|
response = await service.retrieve(request_for())
|
|
|
|
assert [(record["content"], record["score"]) for record in response["records"]] == [
|
|
("at threshold", 0.5)
|
|
]
|
|
|
|
|
|
async def test_requested_top_k_is_clamped_to_service_maximum() -> None:
|
|
raw_chunks = [
|
|
{
|
|
"id": str(UUID(int=index + 1)),
|
|
"content": f"chunk-{index}",
|
|
"score": 0.9,
|
|
"knowledge_base_id": str(ALLOWED_KB_ID),
|
|
"knowledge_id": "6f77a258-27ff-4278-a667-8fe242ff8e10",
|
|
"knowledge_title": "Manual",
|
|
"knowledge_filename": "manual.pdf",
|
|
"chunk_index": index,
|
|
"chunk_type": "text",
|
|
"image_info": "",
|
|
}
|
|
for index in range(6)
|
|
]
|
|
service = RetrievalService(
|
|
search_client=RecordingSearchClient(raw_chunks),
|
|
allowed_kb_ids=frozenset({ALLOWED_KB_ID}),
|
|
)
|
|
|
|
response = await service.retrieve(request_for(top_k=100000))
|
|
|
|
assert [record["content"] for record in response["records"]] == [
|
|
"chunk-0",
|
|
"chunk-1",
|
|
"chunk-2",
|
|
"chunk-3",
|
|
"chunk-4",
|
|
]
|
|
|
|
|
|
async def test_single_chunk_content_is_hard_limited() -> None:
|
|
client = RecordingSearchClient(
|
|
[
|
|
{
|
|
"id": "c7f61f89-19ce-45b1-ac60-dbeed5e2a491",
|
|
"content": "文" * 5000,
|
|
"score": 0.9,
|
|
"knowledge_base_id": str(ALLOWED_KB_ID),
|
|
"knowledge_id": "6f77a258-27ff-4278-a667-8fe242ff8e10",
|
|
"knowledge_title": "Manual",
|
|
"knowledge_filename": "manual.pdf",
|
|
"chunk_index": 0,
|
|
"chunk_type": "text",
|
|
"image_info": "",
|
|
}
|
|
]
|
|
)
|
|
service = RetrievalService(
|
|
search_client=client,
|
|
allowed_kb_ids=frozenset({ALLOWED_KB_ID}),
|
|
)
|
|
|
|
response = await service.retrieve(request_for())
|
|
content = response["records"][0]["content"]
|
|
|
|
assert isinstance(content, str)
|
|
assert len(content) <= 4000
|
|
assert content.endswith(TRUNCATION_MARKER)
|
|
|
|
|
|
async def test_total_returned_content_stops_at_hard_budget() -> None:
|
|
raw_chunks = [
|
|
{
|
|
"id": str(UUID(int=index + 100)),
|
|
"content": str(index) * 4000,
|
|
"score": 0.9,
|
|
"knowledge_base_id": str(ALLOWED_KB_ID),
|
|
"knowledge_id": "6f77a258-27ff-4278-a667-8fe242ff8e10",
|
|
"knowledge_title": "Manual",
|
|
"knowledge_filename": "manual.pdf",
|
|
"chunk_index": index,
|
|
"chunk_type": "text",
|
|
"image_info": "",
|
|
}
|
|
for index in range(4)
|
|
]
|
|
service = RetrievalService(
|
|
search_client=RecordingSearchClient(raw_chunks),
|
|
allowed_kb_ids=frozenset({ALLOWED_KB_ID}),
|
|
)
|
|
|
|
response = await service.retrieve(request_for(top_k=5))
|
|
returned_contents = [record["content"] for record in response["records"]]
|
|
|
|
assert len(returned_contents) == 3
|
|
assert sum(len(content) for content in returned_contents if isinstance(content, str)) <= 12000
|
|
|
|
|
|
async def test_length_budget_is_applied_after_content_processing() -> None:
|
|
client = RecordingSearchClient(
|
|
[
|
|
{
|
|
"id": "c7f61f89-19ce-45b1-ac60-dbeed5e2a491",
|
|
"content": "x",
|
|
"score": 0.9,
|
|
"knowledge_base_id": str(ALLOWED_KB_ID),
|
|
"knowledge_id": "6f77a258-27ff-4278-a667-8fe242ff8e10",
|
|
"knowledge_title": "Manual",
|
|
"knowledge_filename": "manual.pdf",
|
|
"chunk_index": 0,
|
|
"chunk_type": "text",
|
|
"image_info": "",
|
|
}
|
|
]
|
|
)
|
|
limit = len(TRUNCATION_MARKER) + 5
|
|
service = RetrievalService(
|
|
search_client=client,
|
|
allowed_kb_ids=frozenset({ALLOWED_KB_ID}),
|
|
content_processor=ExpandingContentProcessor(),
|
|
max_chunk_chars=limit,
|
|
)
|
|
|
|
response = await service.retrieve(request_for())
|
|
content = response["records"][0]["content"]
|
|
|
|
assert isinstance(content, str)
|
|
assert content == f"xxxxx{TRUNCATION_MARKER}"
|
|
assert len(content) == limit
|