feat(asset): 清理被截断的结尾图片引用,修复整单 502
仓库首次入库(此前仅存在于本地工作区与测试服构建产物中,未上传)。 改动(src/knowledge_adapter/asset_content_processor.py): - 新增 _TRUNCATED_TRAILING_MARKDOWN_RESOURCE_IMAGE 规则,清除 chunk 末尾被截断的  - 回归测试 tests/test_asset_content_processor.py:: test_truncated_trailing_markdown_resource_image_is_removed 验证证据: - 测试服 120.76.217.51,镜像 knowledge-adapter-service:reviewed-test-aeab4b5fca2d (release aeab4b5fca2d8661):KB d015d05c-25c8-4d63-b2ab-e8590d3c2dfb 5 个查询全部 http=200、records=5、残留 resource:// = 0;KB b9dacd62 无退化;容器日志无 error/exception - 本地:pytest 141 项全通过、ruff 全通过 - 该文件与 258c31e0fe07fd08 源码的差异经逐文件 md5 比对,为唯一变更文件
This commit is contained in:
@@ -0,0 +1,26 @@
|
||||
# Non-secret runtime configuration. Copy outside Git before use.
|
||||
ADAPTER_IMAGE_TAG=reviewed-test
|
||||
DIFY_KEY_HASH_CURRENT=<64-char-sha256>
|
||||
DIFY_KEY_HASH_NEXT=
|
||||
WEKNORA_BASE_URL=http://app:8080
|
||||
WEKNORA_NETWORK_NAME=weknora071full-network
|
||||
WEKNORA_ALLOWED_KB_IDS=<uuid>[,<uuid>]
|
||||
WEKNORA_TENANT_ID=<positive-integer>
|
||||
WEKNORA_OSS_STORAGE_BACKEND_ID=<backend-uuid>
|
||||
OSS_REGION=cn-shenzhen
|
||||
OSS_BUCKET=<private-bucket-name>
|
||||
OSS_CREDENTIAL_MODE=static_ak
|
||||
OSS_RAM_ROLE_NAME=
|
||||
ASSET_PUBLIC_BASE_URL=https://assets.example.com
|
||||
MAX_QUERY_CHARS=2000
|
||||
MAX_TOP_K=5
|
||||
MAX_CHUNK_CHARS=4000
|
||||
MAX_TOTAL_CHARS=12000
|
||||
WEKNORA_TIMEOUT_SECONDS=15
|
||||
|
||||
# Host paths only. Files must be owned by UID 10001 and mode 0400.
|
||||
WEKNORA_API_KEY_SECRET_FILE=/opt/knowledge-adapter/secrets/weknora_api_key
|
||||
ASSET_DATABASE_URL_SECRET_FILE=/opt/knowledge-adapter/secrets/asset_database_url
|
||||
WEKNORA_READONLY_DATABASE_URL_SECRET_FILE=/opt/knowledge-adapter/secrets/weknora_readonly_database_url
|
||||
OSS_ACCESS_KEY_ID_HOST_FILE=/opt/knowledge-adapter/secrets/oss_access_key_id
|
||||
OSS_ACCESS_KEY_SECRET_HOST_FILE=/opt/knowledge-adapter/secrets/oss_access_key_secret
|
||||
@@ -0,0 +1,87 @@
|
||||
name: knowledge-adapter
|
||||
|
||||
services:
|
||||
adapter:
|
||||
image: "knowledge-adapter-service:${ADAPTER_IMAGE_TAG:-local}"
|
||||
build:
|
||||
context: ..
|
||||
dockerfile: Dockerfile
|
||||
user: "10001:10001"
|
||||
init: true
|
||||
restart: unless-stopped
|
||||
read_only: true
|
||||
cap_drop:
|
||||
- ALL
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
pids_limit: 256
|
||||
mem_limit: 1g
|
||||
cpus: 2.0
|
||||
tmpfs:
|
||||
- /tmp:rw,noexec,nosuid,nodev,size=64m
|
||||
ports:
|
||||
- "127.0.0.1:18080:8080"
|
||||
environment:
|
||||
DIFY_KEY_HASH_CURRENT: "${DIFY_KEY_HASH_CURRENT:?set DIFY_KEY_HASH_CURRENT}"
|
||||
DIFY_KEY_HASH_NEXT: "${DIFY_KEY_HASH_NEXT:-}"
|
||||
WEKNORA_BASE_URL: "${WEKNORA_BASE_URL:?set WEKNORA_BASE_URL}"
|
||||
WEKNORA_API_KEY_FILE: /run/secrets/weknora_api_key
|
||||
WEKNORA_ALLOWED_KB_IDS: "${WEKNORA_ALLOWED_KB_IDS:?set WEKNORA_ALLOWED_KB_IDS}"
|
||||
WEKNORA_TENANT_ID: "${WEKNORA_TENANT_ID:?set WEKNORA_TENANT_ID}"
|
||||
WEKNORA_OSS_STORAGE_BACKEND_ID: "${WEKNORA_OSS_STORAGE_BACKEND_ID:?set WEKNORA_OSS_STORAGE_BACKEND_ID}"
|
||||
OSS_REGION: "${OSS_REGION:?set OSS_REGION}"
|
||||
OSS_BUCKET: "${OSS_BUCKET:?set OSS_BUCKET}"
|
||||
OSS_CREDENTIAL_MODE: "${OSS_CREDENTIAL_MODE:-static_ak}"
|
||||
OSS_RAM_ROLE_NAME: "${OSS_RAM_ROLE_NAME:-}"
|
||||
OSS_ACCESS_KEY_ID_FILE: /run/secrets/oss_access_key_id
|
||||
OSS_ACCESS_KEY_SECRET_FILE: /run/secrets/oss_access_key_secret
|
||||
ASSET_PUBLIC_BASE_URL: "${ASSET_PUBLIC_BASE_URL:?set ASSET_PUBLIC_BASE_URL}"
|
||||
MAX_QUERY_CHARS: "${MAX_QUERY_CHARS:-2000}"
|
||||
MAX_TOP_K: "${MAX_TOP_K:-5}"
|
||||
MAX_CHUNK_CHARS: "${MAX_CHUNK_CHARS:-4000}"
|
||||
MAX_TOTAL_CHARS: "${MAX_TOTAL_CHARS:-12000}"
|
||||
WEKNORA_TIMEOUT_SECONDS: "${WEKNORA_TIMEOUT_SECONDS:-15}"
|
||||
ASSET_DATABASE_URL_FILE: /run/secrets/asset_database_url
|
||||
WEKNORA_READONLY_DATABASE_URL_FILE: /run/secrets/weknora_readonly_database_url
|
||||
volumes:
|
||||
- type: bind
|
||||
source: "${WEKNORA_API_KEY_SECRET_FILE:?set WEKNORA_API_KEY_SECRET_FILE}"
|
||||
target: /run/secrets/weknora_api_key
|
||||
read_only: true
|
||||
- type: bind
|
||||
source: "${ASSET_DATABASE_URL_SECRET_FILE:?set ASSET_DATABASE_URL_SECRET_FILE}"
|
||||
target: /run/secrets/asset_database_url
|
||||
read_only: true
|
||||
- type: bind
|
||||
source: "${WEKNORA_READONLY_DATABASE_URL_SECRET_FILE:?set WEKNORA_READONLY_DATABASE_URL_SECRET_FILE}"
|
||||
target: /run/secrets/weknora_readonly_database_url
|
||||
read_only: true
|
||||
- type: bind
|
||||
source: "${OSS_ACCESS_KEY_ID_HOST_FILE:?set OSS_ACCESS_KEY_ID_HOST_FILE}"
|
||||
target: /run/secrets/oss_access_key_id
|
||||
read_only: true
|
||||
- type: bind
|
||||
source: "${OSS_ACCESS_KEY_SECRET_HOST_FILE:?set OSS_ACCESS_KEY_SECRET_HOST_FILE}"
|
||||
target: /run/secrets/oss_access_key_secret
|
||||
read_only: true
|
||||
healthcheck:
|
||||
test:
|
||||
- CMD
|
||||
- python
|
||||
- -c
|
||||
- "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/healthz', timeout=2).read()"
|
||||
interval: 10s
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
start_period: 10s
|
||||
stop_grace_period: 20s
|
||||
networks:
|
||||
- adapter
|
||||
- weknora_upstream
|
||||
|
||||
networks:
|
||||
adapter:
|
||||
driver: bridge
|
||||
weknora_upstream:
|
||||
external: true
|
||||
name: "${WEKNORA_NETWORK_NAME:?set WEKNORA_NETWORK_NAME}"
|
||||
@@ -0,0 +1,87 @@
|
||||
map $http_x_request_id $adapter_request_id {
|
||||
"~^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$" $http_x_request_id;
|
||||
default $request_id;
|
||||
}
|
||||
|
||||
limit_req_zone $binary_remote_addr zone=adapter_retrieval:10m rate=10r/s;
|
||||
limit_req_zone $binary_remote_addr zone=adapter_assets:10m rate=30r/s;
|
||||
|
||||
server {
|
||||
listen 80;
|
||||
server_name assets.example.com;
|
||||
access_log off;
|
||||
return 404;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 443 ssl;
|
||||
http2 on;
|
||||
server_name assets.example.com;
|
||||
|
||||
ssl_certificate /etc/letsencrypt/live/assets.example.com/fullchain.pem;
|
||||
ssl_certificate_key /etc/letsencrypt/live/assets.example.com/privkey.pem;
|
||||
ssl_protocols TLSv1.2 TLSv1.3;
|
||||
ssl_session_cache shared:adapter_tls:10m;
|
||||
ssl_session_timeout 1d;
|
||||
|
||||
server_tokens off;
|
||||
access_log off;
|
||||
error_log /var/log/nginx/knowledge-adapter-error.log error;
|
||||
client_max_body_size 64k;
|
||||
|
||||
add_header X-Content-Type-Options nosniff always;
|
||||
add_header Referrer-Policy no-referrer always;
|
||||
|
||||
location = /retrieval {
|
||||
limit_except POST {
|
||||
deny all;
|
||||
}
|
||||
limit_req zone=adapter_retrieval burst=20 nodelay;
|
||||
proxy_pass http://127.0.0.1:18080;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header Connection "";
|
||||
proxy_set_header X-Request-ID $adapter_request_id;
|
||||
proxy_set_header X-Forwarded-Proto https;
|
||||
proxy_connect_timeout 3s;
|
||||
proxy_send_timeout 20s;
|
||||
proxy_read_timeout 20s;
|
||||
proxy_intercept_errors off;
|
||||
proxy_hide_header Server;
|
||||
proxy_hide_header X-Powered-By;
|
||||
}
|
||||
|
||||
location ~ ^/i/[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$ {
|
||||
limit_except GET {
|
||||
deny all;
|
||||
}
|
||||
limit_req zone=adapter_assets burst=60 nodelay;
|
||||
proxy_pass http://127.0.0.1:18080;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header Connection "";
|
||||
proxy_set_header X-Request-ID $adapter_request_id;
|
||||
proxy_set_header X-Forwarded-Proto https;
|
||||
proxy_connect_timeout 3s;
|
||||
proxy_send_timeout 10s;
|
||||
proxy_read_timeout 10s;
|
||||
proxy_intercept_errors off;
|
||||
proxy_hide_header Server;
|
||||
proxy_hide_header X-Powered-By;
|
||||
}
|
||||
|
||||
location = /healthz {
|
||||
allow 127.0.0.1;
|
||||
allow ::1;
|
||||
deny all;
|
||||
proxy_pass http://127.0.0.1:18080;
|
||||
proxy_set_header X-Request-ID $adapter_request_id;
|
||||
proxy_connect_timeout 2s;
|
||||
proxy_read_timeout 2s;
|
||||
proxy_intercept_errors off;
|
||||
}
|
||||
|
||||
location / {
|
||||
return 404;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,149 @@
|
||||
#!/usr/bin/env python3
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import sys
|
||||
import tempfile
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
|
||||
ENV_PATH = pathlib.Path("/opt/knowledge-adapter/adapter.env")
|
||||
NEW_BASE = "https://weknora-api-test.dobot.cn"
|
||||
API_HOST = "weknora-api-test.dobot.cn"
|
||||
UI_HOST = "weknora-test.dobot.cn"
|
||||
DIFY_KEY_PATH = pathlib.Path("/opt/knowledge-adapter/secrets/dify_test_key")
|
||||
|
||||
|
||||
def parse_env(path: pathlib.Path) -> dict[str, str]:
|
||||
result: dict[str, str] = {}
|
||||
for raw in path.read_text().splitlines():
|
||||
line = raw.strip()
|
||||
if line and not line.startswith("#") and "=" in line:
|
||||
key, value = line.split("=", 1)
|
||||
result[key] = value
|
||||
return result
|
||||
|
||||
|
||||
def update_env() -> None:
|
||||
st = ENV_PATH.stat()
|
||||
lines = ENV_PATH.read_text().splitlines(keepends=True)
|
||||
found = 0
|
||||
output: list[str] = []
|
||||
for line in lines:
|
||||
if line.startswith("ASSET_PUBLIC_BASE_URL="):
|
||||
output.append(f"ASSET_PUBLIC_BASE_URL={NEW_BASE}\n")
|
||||
found += 1
|
||||
else:
|
||||
output.append(line)
|
||||
if found != 1:
|
||||
raise RuntimeError(f"expected exactly one ASSET_PUBLIC_BASE_URL, got {found}")
|
||||
fd, tmp_name = tempfile.mkstemp(prefix=".adapter.env.", dir=str(ENV_PATH.parent))
|
||||
try:
|
||||
with os.fdopen(fd, "w") as handle:
|
||||
handle.writelines(output)
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
os.chmod(tmp_name, st.st_mode & 0o777)
|
||||
os.chown(tmp_name, st.st_uid, st.st_gid)
|
||||
os.replace(tmp_name, ENV_PATH)
|
||||
finally:
|
||||
if os.path.exists(tmp_name):
|
||||
os.unlink(tmp_name)
|
||||
if parse_env(ENV_PATH).get("ASSET_PUBLIC_BASE_URL") != NEW_BASE:
|
||||
raise RuntimeError("ASSET_PUBLIC_BASE_URL readback mismatch")
|
||||
print("adapter_env_update=ok")
|
||||
|
||||
|
||||
def request(method: str, path: str, host: str, *, data: bytes | None = None, headers: dict[str, str] | None = None) -> tuple[int, dict[str, str], bytes]:
|
||||
merged = {"Host": host, "Accept": "application/json"}
|
||||
if headers:
|
||||
merged.update(headers)
|
||||
req = urllib.request.Request(
|
||||
f"http://127.0.0.1{path}", data=data, method=method, headers=merged
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=25) as response:
|
||||
return response.status, dict(response.headers.items()), response.read(2_000_000)
|
||||
except urllib.error.HTTPError as exc:
|
||||
return exc.code, dict(exc.headers.items()), exc.read(2_000_000)
|
||||
|
||||
|
||||
def accept() -> None:
|
||||
env = parse_env(ENV_PATH)
|
||||
allowed = [x.strip() for x in env["WEKNORA_ALLOWED_KB_IDS"].split(",") if x.strip()]
|
||||
if len(allowed) != 1:
|
||||
raise RuntimeError("acceptance expects exactly one allowlisted KB")
|
||||
raw_key = DIFY_KEY_PATH.read_text().strip()
|
||||
if not raw_key:
|
||||
raise RuntimeError("Dify key file is empty")
|
||||
configured_hash = env["DIFY_KEY_HASH_CURRENT"].lower()
|
||||
if hashlib.sha256(raw_key.encode()).hexdigest() != configured_hash:
|
||||
raise RuntimeError("Dify key file does not match configured hash")
|
||||
|
||||
status, _, _ = request("GET", "/", UI_HOST)
|
||||
if status != 200:
|
||||
raise RuntimeError(f"UI Host route expected 200, got {status}")
|
||||
print("ui_host_route=200")
|
||||
|
||||
status, _, _ = request("GET", "/", API_HOST)
|
||||
if status != 404:
|
||||
raise RuntimeError(f"API root expected 404, got {status}")
|
||||
print("api_root=404")
|
||||
|
||||
status, _, body = request(
|
||||
"POST", "/retrieval", API_HOST, data=b"{}", headers={"Content-Type": "application/json"}
|
||||
)
|
||||
if status != 401 or json.loads(body) != {"detail": "invalid API key"}:
|
||||
raise RuntimeError("missing-auth boundary mismatch")
|
||||
print("retrieval_missing_key=401_adapter")
|
||||
|
||||
payload = json.dumps(
|
||||
{
|
||||
"knowledge_id": allowed[0],
|
||||
"query": "越疆机器人技术资料",
|
||||
"retrieval_setting": {"top_k": 3, "score_threshold": 0.0},
|
||||
}
|
||||
).encode()
|
||||
status, _, body = request(
|
||||
"POST",
|
||||
"/retrieval",
|
||||
API_HOST,
|
||||
data=payload,
|
||||
headers={"Content-Type": "application/json", "Authorization": f"Bearer {raw_key}"},
|
||||
)
|
||||
if status != 200:
|
||||
raise RuntimeError(f"allowlisted retrieval expected 200, got {status}")
|
||||
result = json.loads(body)
|
||||
records = result.get("records")
|
||||
if not isinstance(records, list):
|
||||
raise RuntimeError("retrieval response has no records list")
|
||||
print(f"retrieval_allowed=200 records={len(records)}")
|
||||
|
||||
denied_payload = json.dumps(
|
||||
{
|
||||
"knowledge_id": str(uuid.UUID("00000000-0000-4000-8000-000000000001")),
|
||||
"query": "boundary probe",
|
||||
"retrieval_setting": {"top_k": 1, "score_threshold": 0.0},
|
||||
}
|
||||
).encode()
|
||||
status, _, body = request(
|
||||
"POST",
|
||||
"/retrieval",
|
||||
API_HOST,
|
||||
data=denied_payload,
|
||||
headers={"Content-Type": "application/json", "Authorization": f"Bearer {raw_key}"},
|
||||
)
|
||||
if status != 400 or json.loads(body) != {"detail": "invalid request"}:
|
||||
raise RuntimeError("denied-KB boundary mismatch")
|
||||
print("retrieval_denied_kb=400")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) != 2 or sys.argv[1] not in {"update-env", "accept"}:
|
||||
raise SystemExit("usage: helper.py update-env|accept")
|
||||
if sys.argv[1] == "update-env":
|
||||
update_env()
|
||||
else:
|
||||
accept()
|
||||
@@ -0,0 +1,21 @@
|
||||
import asyncio
|
||||
import os
|
||||
import pathlib
|
||||
|
||||
import asyncpg
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
dsn_path = pathlib.Path(os.environ["ASSET_DATABASE_URL_FILE"])
|
||||
connection = await asyncpg.connect(dsn_path.read_text().strip())
|
||||
try:
|
||||
row = await connection.fetchrow(
|
||||
"SELECT asset_id FROM asset_mapping "
|
||||
"WHERE status = 'active' ORDER BY created_at LIMIT 1"
|
||||
)
|
||||
print(row["asset_id"] if row else "")
|
||||
finally:
|
||||
await connection.close()
|
||||
|
||||
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,118 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
|
||||
COMPOSE_FILE=/opt/knowledge-adapter/releases/a842917c69f9ae09/knowledge-adapter-service/deploy/docker-compose.adapter.yml
|
||||
ENV_FILE=/opt/knowledge-adapter/adapter.env
|
||||
PROJECT=knowledge-adapter
|
||||
SERVICE=adapter
|
||||
NGINX_SOURCE=/tmp/weknora-test-origin.conf
|
||||
HELPER_SOURCE=/tmp/nginx_adapter_helper.py
|
||||
APT_SOURCE=/tmp/ubuntu-public-temporary.list
|
||||
NGINX_AVAILABLE=/etc/nginx/sites-available/weknora-test-origin.conf
|
||||
NGINX_ENABLED=/etc/nginx/sites-enabled/weknora-test-origin.conf
|
||||
BACKUP_DIR=/opt/knowledge-adapter/backups/nginx-adapter-$(date -u +%Y%m%dT%H%M%SZ)
|
||||
POLICY_CREATED=0
|
||||
ENV_BACKED_UP=0
|
||||
ENV_CHANGED=0
|
||||
ADAPTER_RECREATED=0
|
||||
|
||||
cleanup_policy() {
|
||||
if [[ "$POLICY_CREATED" == 1 ]]; then
|
||||
rm -f /usr/sbin/policy-rc.d
|
||||
fi
|
||||
}
|
||||
|
||||
rollback_on_error() {
|
||||
rc=$?
|
||||
trap - ERR
|
||||
cleanup_policy
|
||||
printf 'rollout_error=stage_failed rc=%s\n' "$rc" >&2
|
||||
if [[ "$ENV_CHANGED" == 1 && "$ENV_BACKED_UP" == 1 ]]; then
|
||||
install -o root -g root -m 0600 "$BACKUP_DIR/adapter.env" "$ENV_FILE"
|
||||
if [[ "$ADAPTER_RECREATED" == 1 ]]; then
|
||||
docker compose -p "$PROJECT" --env-file "$ENV_FILE" -f "$COMPOSE_FILE" \
|
||||
up -d --no-deps --force-recreate --no-build "$SERVICE" || true
|
||||
fi
|
||||
printf 'adapter_env_rollback=attempted\n' >&2
|
||||
fi
|
||||
exit "$rc"
|
||||
}
|
||||
trap rollback_on_error ERR
|
||||
trap cleanup_policy EXIT
|
||||
|
||||
[[ $(id -u) == 0 ]]
|
||||
[[ -f "$COMPOSE_FILE" ]]
|
||||
[[ -f "$ENV_FILE" ]]
|
||||
[[ -f "$NGINX_SOURCE" ]]
|
||||
[[ -f "$HELPER_SOURCE" ]]
|
||||
[[ -f "$APT_SOURCE" ]]
|
||||
[[ $(stat -c '%a' "$ENV_FILE") == 600 ]]
|
||||
|
||||
printf '[1/7] baseline and protected backup\n'
|
||||
mkdir -p -m 0700 "$BACKUP_DIR"
|
||||
install -o root -g root -m 0600 "$ENV_FILE" "$BACKUP_DIR/adapter.env"
|
||||
ENV_BACKED_UP=1
|
||||
docker inspect -f '{{.Image}}' knowledge-adapter-adapter-1 > "$BACKUP_DIR/adapter-image-id.txt"
|
||||
chmod 0600 "$BACKUP_DIR/adapter-image-id.txt"
|
||||
printf 'backup_dir=%s\n' "$BACKUP_DIR"
|
||||
|
||||
printf '[2/7] install nginx without starting default site\n'
|
||||
if ! command -v nginx >/dev/null 2>&1; then
|
||||
if [[ ! -e /usr/sbin/policy-rc.d ]]; then
|
||||
printf '#!/bin/sh\nexit 101\n' > /usr/sbin/policy-rc.d
|
||||
chmod 0755 /usr/sbin/policy-rc.d
|
||||
POLICY_CREATED=1
|
||||
fi
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
apt-get \
|
||||
-o Dir::Etc::sourcelist="$APT_SOURCE" \
|
||||
-o Dir::Etc::sourceparts="-" \
|
||||
-o APT::Get::List-Cleanup="0" \
|
||||
update
|
||||
apt-get \
|
||||
-o Dir::Etc::sourcelist="$APT_SOURCE" \
|
||||
-o Dir::Etc::sourceparts="-" \
|
||||
install -y nginx
|
||||
cleanup_policy
|
||||
POLICY_CREATED=0
|
||||
fi
|
||||
nginx -v
|
||||
|
||||
printf '[3/7] install test-only Host routing and validate syntax\n'
|
||||
install -o root -g root -m 0644 "$NGINX_SOURCE" "$NGINX_AVAILABLE"
|
||||
ln -sfn "$NGINX_AVAILABLE" "$NGINX_ENABLED"
|
||||
rm -f /etc/nginx/sites-enabled/default
|
||||
nginx -t
|
||||
|
||||
printf '[4/7] atomically update Adapter public base and render Compose\n'
|
||||
python3 "$HELPER_SOURCE" update-env
|
||||
ENV_CHANGED=1
|
||||
docker compose -p "$PROJECT" --env-file "$ENV_FILE" -f "$COMPOSE_FILE" config --quiet
|
||||
|
||||
printf '[5/7] recreate Adapter only, using existing image\n'
|
||||
docker compose -p "$PROJECT" --env-file "$ENV_FILE" -f "$COMPOSE_FILE" \
|
||||
up -d --no-deps --force-recreate --no-build "$SERVICE"
|
||||
ADAPTER_RECREATED=1
|
||||
for attempt in $(seq 1 30); do
|
||||
state=$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' knowledge-adapter-adapter-1)
|
||||
if [[ "$state" == healthy ]]; then
|
||||
break
|
||||
fi
|
||||
if [[ "$state" == unhealthy || "$attempt" == 30 ]]; then
|
||||
printf 'adapter_health=%s attempt=%s\n' "$state" "$attempt" >&2
|
||||
false
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
printf 'adapter_health=healthy\n'
|
||||
|
||||
printf '[6/7] enable origin Nginx\n'
|
||||
systemctl enable --now nginx
|
||||
systemctl is-active --quiet nginx
|
||||
nginx -t
|
||||
printf 'nginx=active\n'
|
||||
|
||||
printf '[7/7] real Host-route and retrieval acceptance\n'
|
||||
python3 "$HELPER_SOURCE" accept
|
||||
ss -ltn '( sport = :80 or sport = :18080 )'
|
||||
printf 'rollout=success\n'
|
||||
@@ -0,0 +1,4 @@
|
||||
deb https://mirrors.aliyun.com/ubuntu noble main restricted universe multiverse
|
||||
deb https://mirrors.aliyun.com/ubuntu noble-updates main restricted universe multiverse
|
||||
deb https://mirrors.aliyun.com/ubuntu noble-backports main restricted universe multiverse
|
||||
deb https://mirrors.aliyun.com/ubuntu noble-security main restricted universe multiverse
|
||||
@@ -0,0 +1,61 @@
|
||||
#!/usr/bin/env python3
|
||||
import hashlib
|
||||
import json
|
||||
import subprocess
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
query_script = open("/tmp/query_active_asset_id.py", "rb").read()
|
||||
result = subprocess.run(
|
||||
["docker", "exec", "-i", "knowledge-adapter-adapter-1", "python", "-"],
|
||||
input=query_script,
|
||||
capture_output=True,
|
||||
check=True,
|
||||
)
|
||||
asset_id = result.stdout.decode().strip()
|
||||
if not asset_id:
|
||||
raise SystemExit("no active asset mapping")
|
||||
path = "/i/" + asset_id
|
||||
|
||||
|
||||
class NoRedirect(urllib.request.HTTPRedirectHandler):
|
||||
def redirect_request(self, *args: object, **kwargs: object) -> None:
|
||||
return None
|
||||
|
||||
|
||||
def request(follow: bool) -> tuple[int, dict[str, str], bytes]:
|
||||
req = urllib.request.Request(
|
||||
"http://127.0.0.1" + path,
|
||||
headers={"Host": "weknora-api-test.dobot.cn"},
|
||||
)
|
||||
opener = (
|
||||
urllib.request.build_opener()
|
||||
if follow
|
||||
else urllib.request.build_opener(NoRedirect())
|
||||
)
|
||||
try:
|
||||
with opener.open(req, timeout=30) as response:
|
||||
return response.status, dict(response.headers.items()), response.read(20_000_000)
|
||||
except urllib.error.HTTPError as exc:
|
||||
return exc.code, dict(exc.headers.items()), exc.read(1000)
|
||||
|
||||
|
||||
status, headers, _ = request(False)
|
||||
location = headers.get("Location", "")
|
||||
report: dict[str, object] = {
|
||||
"active_mapping_found": True,
|
||||
"redirect_status": status,
|
||||
"cache_control": headers.get("Cache-Control"),
|
||||
"referrer_policy": headers.get("Referrer-Policy"),
|
||||
"location_https": location.startswith("https://"),
|
||||
"location_not_logged": True,
|
||||
}
|
||||
if status != 302 or not location.startswith("https://"):
|
||||
raise SystemExit(json.dumps(report))
|
||||
status, _, body = request(True)
|
||||
report["follow"] = {
|
||||
"status": status,
|
||||
"bytes": len(body),
|
||||
"sha256_prefix": hashlib.sha256(body).hexdigest()[:12],
|
||||
}
|
||||
print(json.dumps(report))
|
||||
@@ -0,0 +1,118 @@
|
||||
map $http_x_request_id $adapter_request_id {
|
||||
"~^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$" $http_x_request_id;
|
||||
default $request_id;
|
||||
}
|
||||
|
||||
map $http_upgrade $connection_upgrade {
|
||||
default upgrade;
|
||||
"" close;
|
||||
}
|
||||
|
||||
limit_req_zone $binary_remote_addr zone=adapter_retrieval:10m rate=10r/s;
|
||||
limit_req_zone $binary_remote_addr zone=adapter_assets:10m rate=30r/s;
|
||||
|
||||
server {
|
||||
listen 80 default_server;
|
||||
listen [::]:80 default_server;
|
||||
server_name _;
|
||||
access_log off;
|
||||
return 404;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 80;
|
||||
listen [::]:80;
|
||||
server_name weknora-test.dobot.cn;
|
||||
|
||||
server_tokens off;
|
||||
client_max_body_size 64m;
|
||||
|
||||
add_header X-Content-Type-Options nosniff always;
|
||||
add_header Referrer-Policy no-referrer always;
|
||||
|
||||
location / {
|
||||
proxy_pass http://127.0.0.1:8088;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_set_header Upgrade $http_upgrade;
|
||||
proxy_set_header Connection $connection_upgrade;
|
||||
proxy_connect_timeout 3s;
|
||||
proxy_send_timeout 300s;
|
||||
proxy_read_timeout 300s;
|
||||
proxy_buffering off;
|
||||
proxy_hide_header Server;
|
||||
proxy_hide_header X-Powered-By;
|
||||
}
|
||||
}
|
||||
|
||||
server {
|
||||
listen 80;
|
||||
listen [::]:80;
|
||||
server_name weknora-api-test.dobot.cn;
|
||||
|
||||
server_tokens off;
|
||||
access_log off;
|
||||
error_log /var/log/nginx/weknora-api-test-error.log error;
|
||||
client_max_body_size 64k;
|
||||
|
||||
add_header X-Content-Type-Options nosniff always;
|
||||
add_header Referrer-Policy no-referrer always;
|
||||
|
||||
location = /retrieval {
|
||||
limit_except POST {
|
||||
deny all;
|
||||
}
|
||||
limit_req zone=adapter_retrieval burst=20 nodelay;
|
||||
proxy_pass http://127.0.0.1:18080;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header Connection "";
|
||||
proxy_set_header Authorization $http_authorization;
|
||||
proxy_set_header Content-Type $content_type;
|
||||
proxy_set_header X-Request-ID $adapter_request_id;
|
||||
proxy_set_header X-Forwarded-Proto https;
|
||||
proxy_connect_timeout 3s;
|
||||
proxy_send_timeout 20s;
|
||||
proxy_read_timeout 20s;
|
||||
proxy_intercept_errors off;
|
||||
proxy_hide_header Server;
|
||||
proxy_hide_header X-Powered-By;
|
||||
}
|
||||
|
||||
location ~ "^/i/[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$" {
|
||||
limit_except GET {
|
||||
deny all;
|
||||
}
|
||||
limit_req zone=adapter_assets burst=60 nodelay;
|
||||
proxy_pass http://127.0.0.1:18080;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header Connection "";
|
||||
proxy_set_header X-Request-ID $adapter_request_id;
|
||||
proxy_set_header X-Forwarded-Proto https;
|
||||
proxy_connect_timeout 3s;
|
||||
proxy_send_timeout 10s;
|
||||
proxy_read_timeout 10s;
|
||||
proxy_intercept_errors off;
|
||||
proxy_hide_header Server;
|
||||
proxy_hide_header X-Powered-By;
|
||||
}
|
||||
|
||||
location = /healthz {
|
||||
allow 127.0.0.1;
|
||||
allow ::1;
|
||||
deny all;
|
||||
proxy_pass http://127.0.0.1:18080;
|
||||
proxy_set_header X-Request-ID $adapter_request_id;
|
||||
proxy_connect_timeout 2s;
|
||||
proxy_read_timeout 2s;
|
||||
proxy_intercept_errors off;
|
||||
}
|
||||
|
||||
location / {
|
||||
return 404;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"Version": "1",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"oss:GetObject"
|
||||
],
|
||||
"Resource": [
|
||||
"acs:oss:*:*:${OSS_BUCKET}/weknora/${WEKNORA_TENANT_ID}/*"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
\set ON_ERROR_STOP on
|
||||
|
||||
-- Run with: psql ... --set=adapter_role=knowledge_adapter_weknora_ro -f this-file
|
||||
-- The named role must be a dedicated NOINHERIT login with no other memberships.
|
||||
BEGIN;
|
||||
|
||||
REVOKE ALL PRIVILEGES ON ALL TABLES IN SCHEMA public FROM :"adapter_role";
|
||||
REVOKE ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public FROM :"adapter_role";
|
||||
REVOKE CREATE ON SCHEMA public FROM :"adapter_role";
|
||||
|
||||
GRANT USAGE ON SCHEMA public TO :"adapter_role";
|
||||
GRANT SELECT (handle, tenant_id, storage_backend_id, provider, physical_path, mime_type, state, deleted_at)
|
||||
ON TABLE public.resources TO :"adapter_role";
|
||||
|
||||
COMMIT;
|
||||
Reference in New Issue
Block a user