feat(asset): 清理被截断的结尾图片引用,修复整单 502
仓库首次入库(此前仅存在于本地工作区与测试服构建产物中,未上传)。 改动(src/knowledge_adapter/asset_content_processor.py): - 新增 _TRUNCATED_TRAILING_MARKDOWN_RESOURCE_IMAGE 规则,清除 chunk 末尾被截断的  - 回归测试 tests/test_asset_content_processor.py:: test_truncated_trailing_markdown_resource_image_is_removed 验证证据: - 测试服 120.76.217.51,镜像 knowledge-adapter-service:reviewed-test-aeab4b5fca2d (release aeab4b5fca2d8661):KB d015d05c-25c8-4d63-b2ab-e8590d3c2dfb 5 个查询全部 http=200、records=5、残留 resource:// = 0;KB b9dacd62 无退化;容器日志无 error/exception - 本地:pytest 141 项全通过、ruff 全通过 - 该文件与 258c31e0fe07fd08 源码的差异经逐文件 md5 比对,为唯一变更文件
This commit is contained in:
@@ -0,0 +1,258 @@
|
||||
import json
|
||||
import logging
|
||||
from hashlib import sha256
|
||||
from uuid import UUID
|
||||
|
||||
import httpx
|
||||
from _pytest.logging import LogCaptureFixture
|
||||
from fastapi import HTTPException
|
||||
from pydantic import SecretStr
|
||||
|
||||
from knowledge_adapter.app import create_app
|
||||
from knowledge_adapter.auth import DifyKeyAuthenticator
|
||||
from knowledge_adapter.service import RetrievalService
|
||||
from knowledge_adapter.weknora_client import WeKnoraClient
|
||||
|
||||
|
||||
async def test_response_contains_generated_request_id() -> None:
|
||||
transport = httpx.ASGITransport(app=create_app())
|
||||
|
||||
async with httpx.AsyncClient(transport=transport, base_url="http://test") as client:
|
||||
response = await client.get("/healthz")
|
||||
|
||||
request_id = response.headers["x-request-id"]
|
||||
parsed = UUID(request_id)
|
||||
assert parsed.version == 4
|
||||
assert str(parsed) == request_id
|
||||
assert response.json() == {"status": "ok"}
|
||||
|
||||
|
||||
async def test_response_preserves_safe_inbound_request_id() -> None:
|
||||
transport = httpx.ASGITransport(app=create_app())
|
||||
inbound_request_id = "0123456789abcdef0123456789abcdef"
|
||||
|
||||
async with httpx.AsyncClient(transport=transport, base_url="http://test") as client:
|
||||
response = await client.get(
|
||||
"/healthz",
|
||||
headers={"X-Request-ID": inbound_request_id},
|
||||
)
|
||||
|
||||
assert response.headers["x-request-id"] == inbound_request_id
|
||||
|
||||
|
||||
async def test_response_replaces_unsafe_inbound_request_id() -> None:
|
||||
transport = httpx.ASGITransport(app=create_app())
|
||||
unsafe_request_id = "authorization bearer secret value"
|
||||
|
||||
async with httpx.AsyncClient(transport=transport, base_url="http://test") as client:
|
||||
response = await client.get(
|
||||
"/healthz",
|
||||
headers={"X-Request-ID": unsafe_request_id},
|
||||
)
|
||||
|
||||
request_id = response.headers["x-request-id"]
|
||||
assert UUID(request_id).version == 4
|
||||
assert unsafe_request_id not in request_id
|
||||
|
||||
|
||||
async def test_request_log_is_structured_and_uses_route_template(
|
||||
caplog: LogCaptureFixture,
|
||||
) -> None:
|
||||
request_id = "request-123"
|
||||
transport = httpx.ASGITransport(app=create_app())
|
||||
|
||||
with caplog.at_level(logging.INFO, logger="knowledge_adapter.http"):
|
||||
async with httpx.AsyncClient(transport=transport, base_url="http://test") as client:
|
||||
response = await client.get(
|
||||
"/healthz?api_key=must-not-appear",
|
||||
headers={"X-Request-ID": request_id},
|
||||
)
|
||||
|
||||
records = [
|
||||
record
|
||||
for record in caplog.records
|
||||
if record.name == "knowledge_adapter.http"
|
||||
]
|
||||
assert len(records) == 1
|
||||
event = json.loads(records[0].getMessage())
|
||||
assert set(event) == {
|
||||
"duration_ms",
|
||||
"event",
|
||||
"method",
|
||||
"request_id",
|
||||
"route",
|
||||
"status_code",
|
||||
}
|
||||
assert event["event"] == "http_request"
|
||||
assert event["request_id"] == request_id
|
||||
assert event["method"] == "GET"
|
||||
assert event["route"] == "/healthz"
|
||||
assert event["status_code"] == response.status_code == 200
|
||||
assert isinstance(event["duration_ms"], int)
|
||||
assert event["duration_ms"] >= 0
|
||||
assert "must-not-appear" not in records[0].getMessage()
|
||||
|
||||
|
||||
class ExplodingRetrievalService:
|
||||
async def retrieve(self, _request: object) -> dict[str, object]:
|
||||
raise RuntimeError(
|
||||
"Authorization: Bearer server-secret "
|
||||
"resource://AbCdEfGhIjKlMnOpQrSt_1 "
|
||||
"object_key=weknora/10008/private.png "
|
||||
"https://private-assets.oss-cn-shenzhen.aliyuncs.com/private.png?"
|
||||
"x-oss-signature=server-secret"
|
||||
)
|
||||
|
||||
|
||||
async def test_unhandled_exception_is_safe_traceable_and_redacted(
|
||||
caplog: LogCaptureFixture,
|
||||
) -> None:
|
||||
dify_key = "client-secret"
|
||||
request_id = "request-error-123"
|
||||
authenticator = DifyKeyAuthenticator(
|
||||
current_hash=sha256(dify_key.encode()).hexdigest()
|
||||
)
|
||||
app = create_app(
|
||||
authenticator=authenticator,
|
||||
retrieval_service=ExplodingRetrievalService(), # type: ignore[arg-type]
|
||||
)
|
||||
transport = httpx.ASGITransport(app=app, raise_app_exceptions=False)
|
||||
|
||||
with caplog.at_level(logging.INFO, logger="knowledge_adapter.http"):
|
||||
async with httpx.AsyncClient(transport=transport, base_url="http://test") as client:
|
||||
response = await client.post(
|
||||
"/retrieval",
|
||||
headers={
|
||||
"Authorization": f"Bearer {dify_key}",
|
||||
"X-Request-ID": request_id,
|
||||
},
|
||||
json={
|
||||
"knowledge_id": "3904b40e-278b-4273-a6fa-db6cbc975587",
|
||||
"query": "sensitive customer knowledge",
|
||||
"retrieval_setting": {"top_k": 1, "score_threshold": 0},
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 500
|
||||
assert response.headers["x-request-id"] == request_id
|
||||
assert response.json() == {"detail": "internal server error"}
|
||||
adapter_log = "\n".join(
|
||||
record.getMessage()
|
||||
for record in caplog.records
|
||||
if record.name.startswith("knowledge_adapter")
|
||||
)
|
||||
event = json.loads(adapter_log)
|
||||
assert event["route"] == "/retrieval"
|
||||
assert event["status_code"] == 500
|
||||
assert event["error_code"] == "internal_error"
|
||||
for forbidden in (
|
||||
dify_key,
|
||||
"server-secret",
|
||||
"sensitive customer knowledge",
|
||||
"resource://",
|
||||
"object_key",
|
||||
"private.png",
|
||||
"x-oss-signature",
|
||||
):
|
||||
assert forbidden not in response.text
|
||||
assert forbidden not in adapter_log
|
||||
|
||||
|
||||
async def test_unknown_route_uses_safe_uniform_404_without_logging_raw_path(
|
||||
caplog: LogCaptureFixture,
|
||||
) -> None:
|
||||
request_id = "request-not-found"
|
||||
raw_path_secret = "unknown-secret-path"
|
||||
transport = httpx.ASGITransport(app=create_app())
|
||||
|
||||
with caplog.at_level(logging.INFO, logger="knowledge_adapter.http"):
|
||||
async with httpx.AsyncClient(transport=transport, base_url="http://test") as client:
|
||||
response = await client.get(
|
||||
f"/{raw_path_secret}?authorization=server-secret",
|
||||
headers={"X-Request-ID": request_id},
|
||||
)
|
||||
|
||||
assert response.status_code == 404
|
||||
assert response.headers["x-request-id"] == request_id
|
||||
assert response.json() == {"detail": "not found"}
|
||||
adapter_log = "\n".join(
|
||||
record.getMessage()
|
||||
for record in caplog.records
|
||||
if record.name == "knowledge_adapter.http"
|
||||
)
|
||||
event = json.loads(adapter_log)
|
||||
assert event["route"] == "<unmatched>"
|
||||
assert raw_path_secret not in adapter_log
|
||||
assert "server-secret" not in adapter_log
|
||||
|
||||
|
||||
async def test_unknown_http_exception_detail_and_headers_are_redacted() -> None:
|
||||
app = create_app()
|
||||
|
||||
@app.get("/test-only-http-error")
|
||||
async def test_only_http_error() -> None:
|
||||
raise HTTPException(
|
||||
status_code=418,
|
||||
detail="Authorization Bearer server-secret resource://private",
|
||||
headers={"X-Internal-Secret": "server-secret"},
|
||||
)
|
||||
|
||||
transport = httpx.ASGITransport(app=app)
|
||||
async with httpx.AsyncClient(transport=transport, base_url="http://test") as client:
|
||||
response = await client.get("/test-only-http-error")
|
||||
|
||||
assert response.status_code == 418
|
||||
assert response.json() == {"detail": "request failed"}
|
||||
assert "x-internal-secret" not in response.headers
|
||||
assert "server-secret" not in response.text
|
||||
assert "resource://" not in response.text
|
||||
|
||||
|
||||
async def test_safe_request_id_is_forwarded_to_weknora() -> None:
|
||||
dify_key = "client-secret"
|
||||
request_id = "request-upstream-123"
|
||||
knowledge_base_id = UUID("3904b40e-278b-4273-a6fa-db6cbc975587")
|
||||
upstream_headers: list[httpx.Headers] = []
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
upstream_headers.append(request.headers)
|
||||
return httpx.Response(200, json={"success": True, "data": []})
|
||||
|
||||
weknora_client = WeKnoraClient(
|
||||
base_url="http://weknora.internal:8080",
|
||||
api_key=SecretStr("weknora-secret"),
|
||||
timeout_seconds=15,
|
||||
transport=httpx.MockTransport(handler),
|
||||
)
|
||||
service = RetrievalService(
|
||||
search_client=weknora_client,
|
||||
allowed_kb_ids=frozenset({knowledge_base_id}),
|
||||
)
|
||||
authenticator = DifyKeyAuthenticator(
|
||||
current_hash=sha256(dify_key.encode()).hexdigest()
|
||||
)
|
||||
app = create_app(authenticator=authenticator, retrieval_service=service)
|
||||
transport = httpx.ASGITransport(app=app)
|
||||
|
||||
try:
|
||||
async with httpx.AsyncClient(transport=transport, base_url="http://test") as client:
|
||||
response = await client.post(
|
||||
"/retrieval",
|
||||
headers={
|
||||
"Authorization": f"Bearer {dify_key}",
|
||||
"X-Request-ID": request_id,
|
||||
},
|
||||
json={
|
||||
"knowledge_id": str(knowledge_base_id),
|
||||
"query": "query",
|
||||
"retrieval_setting": {"top_k": 1, "score_threshold": 0},
|
||||
},
|
||||
)
|
||||
finally:
|
||||
await weknora_client.aclose()
|
||||
|
||||
assert response.status_code == 200
|
||||
assert len(upstream_headers) == 1
|
||||
assert upstream_headers[0]["x-request-id"] == request_id
|
||||
assert upstream_headers[0]["x-api-key"] == "weknora-secret"
|
||||
assert "authorization" not in upstream_headers[0]
|
||||
Reference in New Issue
Block a user