feat: 知识治理平台(含已验证发布/回滚契约)
全栈知识治理平台首次入库(P1-P10 + 发布链路模拟验证闭环): - 治理核心:源目录/变更集/人工审核/发布批次/发布执行(fail-closed 闸口) - 测试通道:WeKnora 测试 KB 上传/轮询/验收;poll 完成时写入 parser_generation/index_generation,修复 manual/seed 路径发布闸口永久拦截 - 发布适配器 v071(本地 source=测试角色 / 云端 target=生产角色 模拟验证通过): * ES 导出/导入 mapping 自适应(.keyword 子字段、可选 tenant_id) * OSS locator 支持嵌套 scheme;chunks.seq_id 由目标序列分配 * bundle 归档同身份重试原子替换;verifier 按 bundle chunk_type 计数 * 回滚 job payload 支持 prior_execution_id(strict 解码) - 三通道发布+回滚演练实测:publish 4/4 verified,检索命中, rollback 4/4 rolled_back,目标端 PG is_enabled=false、ES is_enabled=false - 前端 A·Data-console 风格全站统一,120 用例通过 - 门禁:go vet / go test 全包通过,gofmt 干净
This commit is contained in:
@@ -0,0 +1,11 @@
|
||||
.git
|
||||
.gitignore
|
||||
.env
|
||||
.env.*
|
||||
!.env.example
|
||||
bin
|
||||
coverage.out
|
||||
secrets
|
||||
web/dist
|
||||
web/node_modules
|
||||
web/tsconfig.tsbuildinfo
|
||||
@@ -0,0 +1,25 @@
|
||||
APP_ADDR=:8080
|
||||
DATABASE_URL=postgres://governance@postgres:5432/governance?sslmode=disable
|
||||
PGPASSFILE=/run/secrets/pgpass
|
||||
WORKER_ID=worker-1
|
||||
WORKER_POLL_INTERVAL=1s
|
||||
WORKER_LEASE_DURATION=30s
|
||||
WORKER_RETRY_DELAY=5s
|
||||
RELEASE_WORKER_ENABLED=false
|
||||
PRODUCTION_RELEASE_REQUESTS_ENABLED=false
|
||||
PP_SCAN_ENABLED=false
|
||||
PP_SCAN_HOST_DATA_ROOT=/home/zheromiku/dobot-portal-sync/data
|
||||
PP_SCAN_CONTAINER_DATA_ROOT=/home/zheromiku/dobot-portal-sync/data
|
||||
TEST_WEKNORA_ENABLED=false
|
||||
TEST_WEKNORA_ALLOW_DOCKER_HTTP=false
|
||||
TEST_WEKNORA_BASE_URL=
|
||||
MINERU_BASE_URL=
|
||||
TEST_WEKNORA_KB_ID=
|
||||
TEST_WEKNORA_API_KEY_FILE=/run/secrets/test_weknora_api_key
|
||||
GOVERNANCE_SPOOL_ROOT=/var/lib/governance/spool
|
||||
TEST_WEKNORA_HTTP_TIMEOUT=60s
|
||||
TEST_WEKNORA_POLL_INITIAL=10s
|
||||
TEST_WEKNORA_POLL_MAX=60s
|
||||
TEST_WEKNORA_STALLED_AFTER=30m
|
||||
TEST_WEKNORA_MAX_UPLOAD_BYTES=52428800
|
||||
TEST_ACCEPTANCE_POLICY_VERSION=structural-v1
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
# Go
|
||||
/bin/
|
||||
/coverage.out
|
||||
|
||||
# Web
|
||||
/web/node_modules/
|
||||
/web/dist/
|
||||
/web/tsconfig.tsbuildinfo
|
||||
|
||||
# Local runtime
|
||||
.env
|
||||
.env.*
|
||||
!.env.example
|
||||
/tmp/
|
||||
/data/
|
||||
/secrets/*
|
||||
!/secrets/*.example
|
||||
|
||||
# Editors and OS
|
||||
.idea/
|
||||
.vscode/
|
||||
.DS_Store
|
||||
|
||||
# Local artifacts (binaries, staging scripts, backups)
|
||||
/release-diag
|
||||
/worker
|
||||
/apply_s5_patch.py
|
||||
/tmp_*
|
||||
.style-backup-*/
|
||||
.hermes/
|
||||
+23
@@ -0,0 +1,23 @@
|
||||
FROM golang:1.26-alpine AS build
|
||||
WORKDIR /src
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
COPY cmd ./cmd
|
||||
COPY internal ./internal
|
||||
RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -o /out/api ./cmd/api \
|
||||
&& CGO_ENABLED=0 GOOS=linux go build -trimpath -o /out/worker ./cmd/worker \
|
||||
&& CGO_ENABLED=0 GOOS=linux go build -trimpath -o /out/release-worker ./cmd/release-worker \
|
||||
&& CGO_ENABLED=0 GOOS=linux go build -trimpath -o /out/test-publication-seed ./cmd/test-publication-seed \
|
||||
&& CGO_ENABLED=0 GOOS=linux go build -trimpath -o /out/test-weknora-e2e-report ./cmd/test-weknora-e2e-report \
|
||||
&& CGO_ENABLED=0 GOOS=linux go build -trimpath -o /out/test-weknora-probe ./cmd/test-weknora-probe
|
||||
|
||||
FROM alpine:3.22
|
||||
ENV PYTHONDONTWRITEBYTECODE=1
|
||||
RUN apk add --no-cache ca-certificates python3 postgresql-client \
|
||||
&& addgroup -g 65532 governance \
|
||||
&& adduser -D -H -u 65532 -G governance governance
|
||||
WORKDIR /app
|
||||
COPY --from=build /out/ /app/
|
||||
COPY scripts/source_catalog_import.py scripts/live_catalog_refresh.py /app/scripts/
|
||||
USER 65532:65532
|
||||
CMD ["/app/api"]
|
||||
@@ -0,0 +1,11 @@
|
||||
FROM node:22-alpine AS build
|
||||
WORKDIR /src
|
||||
COPY web/package.json web/package-lock.json ./
|
||||
RUN npm ci --no-audit --no-fund
|
||||
COPY web/ ./
|
||||
RUN npm run build
|
||||
|
||||
FROM nginxinc/nginx-unprivileged:1.29-alpine
|
||||
COPY deploy/nginx.conf /etc/nginx/conf.d/default.conf
|
||||
COPY --from=build /src/dist /usr/share/nginx/html
|
||||
EXPOSE 8080
|
||||
@@ -0,0 +1,23 @@
|
||||
GO ?= go
|
||||
|
||||
.PHONY: test vet build web-test web-typecheck web-build verify
|
||||
|
||||
test:
|
||||
$(GO) test ./...
|
||||
|
||||
vet:
|
||||
$(GO) vet ./...
|
||||
|
||||
build:
|
||||
$(GO) build ./cmd/...
|
||||
|
||||
web-test:
|
||||
npm --prefix web test
|
||||
|
||||
web-typecheck:
|
||||
npm --prefix web run typecheck
|
||||
|
||||
web-build:
|
||||
npm --prefix web run build
|
||||
|
||||
verify: test vet build web-test web-typecheck web-build
|
||||
@@ -1,2 +1,207 @@
|
||||
# knowledge-governance
|
||||
# 知识同步治理平台
|
||||
|
||||
独立的知识治理控制平面。当前包含P1持久任务队列、P2统一来源与PP历史接管、P3本地/测试WeKnora编排验收、P4来源只读扫描、P5知识工程师审核与封批门禁、默认关闭的P6三通道生产发布与回滚Worker,以及P7持久快照ChangeSet、真实运行监控和每50项一批的发布进度。
|
||||
|
||||
## 安全边界
|
||||
|
||||
- 本工程不修改 WeKnora、PP、AList、钉钉或 DobotAI 源码。
|
||||
- PP接管与核验只读取指定的本地账本、manifest和测试WeKnora;不请求PP写接口,不修改来源文件。
|
||||
- 普通Worker领取演示任务、`test.upload/test.poll/test.validate`和显式启用的`source.scan.pp`;只有普通Worker可挂载测试WeKnora Key和只读PP快照。
|
||||
- Release Worker默认`RELEASE_WORKER_ENABLED=false`,禁用时不读取release config/生产secret,不连接生产数据面或治理数据库。
|
||||
- P5封批只生成治理侧`sealed`记录,不创建生产发布Job;5条真实候选仍为待人工审核。
|
||||
- P6生产请求另受`PRODUCTION_RELEASE_REQUESTS_ENABLED=false`保护;开启时还必须有服务端固定actor和0600 Bearer token文件。当前未授权、未配置、未发布生产。
|
||||
- 凭据文件位于`secrets/`且被Git忽略;仓库只保留示例文件。
|
||||
|
||||
## 目录
|
||||
|
||||
```text
|
||||
cmd/api HTTP API
|
||||
cmd/worker 普通任务Worker
|
||||
cmd/release-worker 默认关闭的发布Worker
|
||||
cmd/pp-ledger-import PP历史账本一次性导入工具
|
||||
internal/jobs 任务模型、租约和Worker循环
|
||||
internal/governance 统一来源治理领域模型
|
||||
internal/platform PostgreSQL仓库与Migration
|
||||
internal/releasebundle v0.7.1发布bundle契约与摘要
|
||||
internal/releaseadapter v0.7.1 PostgreSQL/ES/S3 Sidecar
|
||||
internal/httpapi HTTP路由
|
||||
web React TypeScript状态页
|
||||
deploy Nginx配置
|
||||
docs 验收计划
|
||||
```
|
||||
|
||||
## P2.1来源数据底座
|
||||
|
||||
Migration 2创建五张空表:
|
||||
|
||||
```text
|
||||
source_connections 来源连接配置,只保存secret引用名
|
||||
source_items 来源内稳定对象身份与变化状态
|
||||
content_objects 按完整SHA-256唯一的内容对象
|
||||
source_item_versions 来源对象版本与内容对象关联
|
||||
test_publications 测试知识库导入、处理、检索状态
|
||||
```
|
||||
|
||||
`source_change`、`ingest_state`、`processing_state`和`retrieval_state`相互独立。只有同一来源连接内的`stable_external_id`唯一;只有完整小写SHA-256相同才能共享内容对象。
|
||||
|
||||
P2.1不会自动创建来源连接、读取PP账本或上传文档。完整验收步骤见`docs/P2_SOURCE_FOUNDATION_ACCEPTANCE.md`。
|
||||
|
||||
## P2.2 PP历史账本接管
|
||||
|
||||
一次性工具将PP append-only账本折叠为当前终态:
|
||||
|
||||
```text
|
||||
1093 条历史事件
|
||||
→ 559 个来源对象和版本
|
||||
→ 530 个SHA内容对象
|
||||
→ 530 条历史测试发布
|
||||
→ 29 个content_alias共享canonical内容和Knowledge ID
|
||||
```
|
||||
|
||||
75条旧`completed`记录缺少SHA时,只允许从locator hash匹配的本地manifest恢复,并重新校验文件大小和实际SHA-256。单文件继续遵守旧连接器50MiB上限。连接器默认关闭,历史发布的`retrieval_state`保持`unpublished`,等待未来对目标WeKnora逐条只读核验。
|
||||
|
||||
工具只执行幂等INSERT,不执行UPDATE或DELETE。完整验收步骤见`docs/P2_PP_LEDGER_TAKEOVER_ACCEPTANCE.md`。
|
||||
|
||||
## P2.3 PP完整清单与知识资产页
|
||||
|
||||
P2.3将manifest中未进入旧账本的54个超限来源纳入治理视图:
|
||||
|
||||
```text
|
||||
559 个历史终态 + 54 个over_limit阻断项 = 613 个来源对象
|
||||
530 个内容对象
|
||||
613 个来源版本
|
||||
530 条历史测试发布
|
||||
历史已同步内容计划重新上传数 = 0
|
||||
```
|
||||
|
||||
超限版本没有虚假ContentObject或TestPublication,保持`content_state=over_limit`和阻断状态。`GET /api/v1/source-items`提供只读分页清单,支持`state=completed|content_alias|over_limit`、`limit`和`offset`;响应不包含来源URL、URI哈希、raw记录或Knowledge ID。Web“知识资产”模块读取该真实API。完整验收见`docs/P2_PP_INVENTORY_ACCEPTANCE.md`。
|
||||
|
||||
## P3 WeKnora测试编排
|
||||
|
||||
普通Worker从只读Spool流式上传到测试WeKnora,持久保存远端身份,按单调`poll_sequence`轮询,并在完成后执行Knowledge/Chunk验收。API和Web只返回安全投影;人工重试只允许失败或停滞且仍有已验证制品的Publication。530条PP历史Knowledge已在测试服务器完成在线只读核验,没有重新上传。
|
||||
|
||||
本地、测试服务器、失败恢复、Worker重启和浏览器验收见`docs/P3_LOCAL_AND_CROSS_ENV_ACCEPTANCE.md`。
|
||||
|
||||
## P4.1 来源连接只读工作台
|
||||
|
||||
`GET /api/v1/source-connections`返回来源ID、类型、名称、启用状态、是否配置调度、健康枚举、对象数和更新时间。查询与响应均不包含`base_url`、`secret_ref`或原始健康错误正文。Web“来源管理”模块展示真实PostgreSQL数据,保留左侧导航和顶部多标签。
|
||||
|
||||
P4.1完整验收见`docs/P41_SOURCE_CONNECTIONS_ACCEPTANCE.md`。
|
||||
|
||||
## P4.2 PP只读扫描与变化入账
|
||||
|
||||
普通Worker在`PP_SCAN_ENABLED=true`且三个安全路径完整时注册`source.scan.pp` Handler。Compose仅给普通Worker只读挂载PP数据目录;API、Web和Release Worker均不挂载。Scanner拒绝越界、软链接、非普通文件和不完整manifest;54个历史`over_limit`项允许其制品未下载,但仍保持阻断。
|
||||
|
||||
Migration 5新增`source_scans`和`source_scan_changes`。触发Scan与创建Job原子执行,每个连接最多一个活动Scan;完成时聚合、变化与来源健康原子写入;失败只保存固定错误码。安全API支持触发、扫描历史和连接作用域详情,Web展示真实聚合与变化记录,不返回来源ID/revision/SHA/路径/URL或凭据。
|
||||
|
||||
真实PP快照只读扫描结果:
|
||||
|
||||
```text
|
||||
observed=613
|
||||
unchanged=559
|
||||
over_limit=54
|
||||
added=0
|
||||
updated=0
|
||||
missing=0
|
||||
planned_uploads=0
|
||||
```
|
||||
|
||||
过期`running`租约已由常驻Worker真实重领并以`attempt=2`完成;并发第二次触发返回`409`。完整迁移、备份、恢复、API和Firefox验收见`docs/P42_SOURCE_SCAN_ACCEPTANCE.md`。
|
||||
|
||||
## P5 知识工程师审核与封批
|
||||
|
||||
Migration 6新增追加式`publication_reviews`、`release_batches`和`release_batch_items`。审核决定绑定最新Acceptance run与Publication时间戳;批准仅允许完整结构门禁通过的候选,`retrieval_hit=manual_required`保持人工判断。创建draft和sealed封批均原子验证整个项目集合,任一项目失效则整批拒绝。
|
||||
|
||||
“发布管理”模块展示真实审核候选、固定门禁、决定可见性、草稿批次和封批记录,并明确显示“生产门禁保持关闭”“Release Worker保持关闭”“尚未发布”。当前535条Publication中,530条无逐条Acceptance的PP legacy不自动进入人工队列;5条manual合成验收项显示为ready/pending,持久库没有代替知识工程师创建Review或Batch。
|
||||
|
||||
P5 Store与HTTP写链路已在disposable PostgreSQL和临时API中验证`pending→approved→draft→sealed`,完成后临时数据库与临时密钥文件均清理;持久环境只应用空Migration 6。完整验收见`docs/P5_REVIEW_AND_SEAL_ACCEPTANCE.md`。
|
||||
|
||||
## P6 生产发布Worker与回滚
|
||||
|
||||
Migration 7新增`release_executions`与`release_item_executions`。只有受保护的生产授权请求才能把P5 sealed batch原子转换为`release.publish` Job;rollback只允许针对完整成功的publish execution。终态失败可在同一Bearer门禁下重排,仅重置failed items并重新验证sealed snapshot。API和Release Worker使用两个独立默认关闭的门禁,UI只展示执行审计,不提供一键生产授权按钮。
|
||||
|
||||
Release Worker按Knowledge Base精确路由,固定WeKnora `v0.7.1 / migration 74 / elasticsearch_v8 / 1024维`契约;同一release package协调复制以下内容且不触发生产解析/Embedding:
|
||||
|
||||
```text
|
||||
PostgreSQL: knowledges/chunks/knowledge_tags/relations/resources/bindings
|
||||
对象存储: referenced assets,staging→SHA校验→相同relative key
|
||||
Elasticsearch: chunk text+vectors,先upsert新文档再清理同Knowledge旧_id
|
||||
```
|
||||
|
||||
每个release revision写入0700目录中的0600 bundle并重验整包SHA。失败恢复旧bundle;首次发布无旧revision时禁用Knowledge/Chunk/ES,不删除行、文档或对象。生产WeKnora官方API最终验证Knowledge、Chunk数和knowledge-scoped检索。
|
||||
|
||||
本地真实验收使用完整v0.7.1 75个migration的source/target PostgreSQL、真实MinIO、状态化ES协议服务,已通过`publish v1 → publish v2 → rollback v1`以及PG disable、S3 Open/Put/Copy、Worker受限配置装配。模板见`config/release.v071.example.json`,完整边界与结果见`docs/P6_RELEASE_WORKER_ACCEPTANCE.md`。
|
||||
|
||||
当前生产状态保持:
|
||||
|
||||
```text
|
||||
PRODUCTION_RELEASE_REQUESTS_ENABLED=false
|
||||
RELEASE_WORKER_ENABLED=false
|
||||
release executions/items/jobs=0/0/0
|
||||
```
|
||||
|
||||
## 本地验证
|
||||
|
||||
要求Go 1.26、Node.js 22和npm。
|
||||
|
||||
```bash
|
||||
make test
|
||||
make vet
|
||||
make build
|
||||
make web-test
|
||||
make web-typecheck
|
||||
make web-build
|
||||
```
|
||||
|
||||
## Docker Compose(由操作者执行)
|
||||
|
||||
先创建仅限本机的密钥文件:
|
||||
|
||||
```bash
|
||||
cp secrets/postgres_password.example secrets/postgres_password
|
||||
cp secrets/pgpass.example secrets/pgpass
|
||||
# 将两个文件中的CHANGE_ME_LOCAL_ONLY替换为同一随机本地密码
|
||||
chmod 600 secrets/postgres_password secrets/pgpass
|
||||
docker compose up --build -d
|
||||
```
|
||||
|
||||
Compose默认让三个Go进程以`1000:1000`运行,以读取宿主机Owner为1000且权限为600的密钥文件。若操作者UID/GID不同,启动前设置`APP_UID`和`APP_GID`为密钥文件Owner。
|
||||
|
||||
访问:
|
||||
|
||||
- 状态页:`http://127.0.0.1:8088/`
|
||||
- 同源健康检查:`http://127.0.0.1:8088/health`
|
||||
|
||||
停止时保留数据库:
|
||||
|
||||
```bash
|
||||
docker compose down
|
||||
```
|
||||
|
||||
不要执行`docker compose down -v`,除非明确决定删除本地数据库数据。
|
||||
|
||||
## P1 API
|
||||
|
||||
创建普通演示任务:
|
||||
|
||||
```bash
|
||||
curl -fsS -X POST http://127.0.0.1:8088/api/v1/jobs \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{"type":"demo","payload":{},"idempotency_key":"demo:001"}'
|
||||
```
|
||||
|
||||
创建可用于租约恢复验收的30秒任务:
|
||||
|
||||
```bash
|
||||
curl -fsS -X POST http://127.0.0.1:8088/api/v1/jobs \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{"type":"demo.sleep","payload":{"sleep_ms":30000},"idempotency_key":"demo:sleep:001"}'
|
||||
```
|
||||
|
||||
查看最近任务:
|
||||
|
||||
```bash
|
||||
curl -fsS 'http://127.0.0.1:8088/api/v1/jobs?limit=50'
|
||||
```
|
||||
|
||||
完整人工步骤见`docs/P1_ACCEPTANCE_TEST_PLAN.md`。
|
||||
|
||||
+3145
File diff suppressed because it is too large
Load Diff
+112
@@ -0,0 +1,112 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"knowledge-governance/internal/appconfig"
|
||||
"knowledge-governance/internal/httpapi"
|
||||
"knowledge-governance/internal/monitoring"
|
||||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||||
"knowledge-governance/internal/platform/weknora"
|
||||
)
|
||||
|
||||
func main() {
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
if err := run(ctx, os.Getenv); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func run(ctx context.Context, getenv func(string) string) error {
|
||||
config, err := appconfig.Load(getenv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pool, err := pgxpool.New(ctx, config.DatabaseURL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer pool.Close()
|
||||
if err := pool.Ping(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := postgresstore.ApplyMigrations(ctx, pool); err != nil {
|
||||
return err
|
||||
}
|
||||
releaseAuthorizationToken := ""
|
||||
if config.ProductionReleaseRequestsEnabled {
|
||||
releaseAuthorizationToken, err = weknora.ReadAPIKeyFile(config.ProductionReleaseAuthTokenFile)
|
||||
if err != nil {
|
||||
return errors.New("read production release authorization token")
|
||||
}
|
||||
}
|
||||
// P10: with a mounted API key the probe verifies a real knowledge-base read
|
||||
// (readiness); without one it degrades to the liveness-only HTTP check.
|
||||
var weknoraReadiness func(context.Context) error
|
||||
if strings.TrimSpace(config.TestWeKnoraBaseURL) != "" {
|
||||
if apiKey, keyErr := weknora.ReadAPIKeyFile(config.TestWeKnoraAPIKeyFile); keyErr == nil {
|
||||
probeTimeout := config.TestWeKnoraHTTPTimeout
|
||||
if probeTimeout > 10*time.Second || probeTimeout <= 0 {
|
||||
probeTimeout = 10 * time.Second
|
||||
}
|
||||
if probeClient, clientErr := weknora.NewClient(weknora.ClientConfig{BaseURL: config.TestWeKnoraBaseURL, APIKey: apiKey, Timeout: probeTimeout, AllowDockerHTTP: config.TestWeKnoraAllowDockerHTTP}); clientErr == nil {
|
||||
routingKB := config.InternalRoutingKnowledgeBaseID
|
||||
weknoraReadiness = func(probeCtx context.Context) error {
|
||||
_, _, err := probeClient.ListKnowledge(probeCtx, routingKB, 1, 1)
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
monitorProvider, err := monitoring.NewRealProvider(monitoring.RealProviderConfig{DB: pool, WeKnoraBaseURL: config.TestWeKnoraBaseURL, MinerUBaseURL: config.MinerUBaseURL, WeKnoraReadiness: weknoraReadiness, WorkerStaleAfter: config.WorkerLivenessStaleAfter})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
handler, err := httpapi.NewHandler(httpapi.Config{
|
||||
Store: postgresstore.NewStore(pool),
|
||||
Monitor: monitorProvider,
|
||||
InternalRoutingKnowledgeBaseID: config.InternalRoutingKnowledgeBaseID,
|
||||
ExternalRoutingKnowledgeBaseID: config.ExternalRoutingKnowledgeBaseID,
|
||||
ProductionReleaseRequestsEnabled: config.ProductionReleaseRequestsEnabled,
|
||||
ProductionBackupRestoreVerified: config.ProductionBackupRestoreVerified,
|
||||
ProductionReleaseAuthorizerRef: config.ProductionReleaseAuthorizerRef,
|
||||
ProductionReleaseAuthorizationToken: releaseAuthorizationToken,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
server := &http.Server{
|
||||
Addr: config.AppAddr,
|
||||
Handler: handler,
|
||||
ReadHeaderTimeout: 5 * time.Second,
|
||||
ReadTimeout: 15 * time.Second,
|
||||
WriteTimeout: 30 * time.Second,
|
||||
IdleTimeout: 60 * time.Second,
|
||||
}
|
||||
|
||||
errCh := make(chan error, 1)
|
||||
go func() { errCh <- server.ListenAndServe() }()
|
||||
log.Printf("governance API listening on %s", config.AppAddr)
|
||||
select {
|
||||
case err := <-errCh:
|
||||
if errors.Is(err, http.ErrServerClosed) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
case <-ctx.Done():
|
||||
shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
return server.Shutdown(shutdownCtx)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestRunRequiresDatabaseURL(t *testing.T) {
|
||||
err := run(context.Background(), func(string) string { return "" })
|
||||
if err == nil || !strings.Contains(err.Error(), "DATABASE_URL") {
|
||||
t.Fatalf("run() error = %v, want DATABASE_URL error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPIPropagatesProductionReleaseRequestGate(t *testing.T) {
|
||||
source, err := os.ReadFile("main.go")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
normalized := strings.Join(strings.Fields(string(source)), " ")
|
||||
for _, required := range []string{
|
||||
"if config.ProductionReleaseRequestsEnabled {",
|
||||
"weknora.ReadAPIKeyFile(config.ProductionReleaseAuthTokenFile)",
|
||||
"ProductionReleaseRequestsEnabled: config.ProductionReleaseRequestsEnabled",
|
||||
"ProductionBackupRestoreVerified: config.ProductionBackupRestoreVerified",
|
||||
"ProductionReleaseAuthorizerRef: config.ProductionReleaseAuthorizerRef",
|
||||
"ProductionReleaseAuthorizationToken: releaseAuthorizationToken",
|
||||
} {
|
||||
if !strings.Contains(normalized, required) {
|
||||
t.Fatalf("API production authorization wiring missing %q", required)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestComposeDefaultsProductionBackupRestoreGateToFalse(t *testing.T) {
|
||||
source, err := os.ReadFile("../../compose.yaml")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(string(source), `PRODUCTION_BACKUP_RESTORE_VERIFIED: ${PRODUCTION_BACKUP_RESTORE_VERIFIED:-false}`) {
|
||||
t.Fatal("compose must pass the production backup restore gate with a false default")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
ppconnector "knowledge-governance/internal/connectors/pp"
|
||||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if err := run(context.Background(), os.Args[1:], os.Getenv, os.Stdout); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func run(ctx context.Context, args []string, getenv func(string) string, stdout io.Writer) error {
|
||||
flags := flag.NewFlagSet("pp-ledger-import", flag.ContinueOnError)
|
||||
flags.SetOutput(io.Discard)
|
||||
ledgerPath := flags.String("ledger", "", "path to the legacy PP state JSONL")
|
||||
manifestPath := flags.String("manifest", "", "path to the validated download manifest JSON")
|
||||
dryRun := flags.Bool("dry-run", false, "validate and print aggregate counts without database access")
|
||||
if err := flags.Parse(args); err != nil {
|
||||
return fmt.Errorf("parse arguments: %w", err)
|
||||
}
|
||||
if *ledgerPath == "" || *manifestPath == "" {
|
||||
return fmt.Errorf("-ledger and -manifest are required")
|
||||
}
|
||||
|
||||
ledger, err := os.Open(*ledgerPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open PP ledger: %w", err)
|
||||
}
|
||||
snapshot, foldErr := ppconnector.FoldLedger(ledger)
|
||||
closeErr := ledger.Close()
|
||||
if foldErr != nil {
|
||||
return foldErr
|
||||
}
|
||||
if closeErr != nil {
|
||||
return fmt.Errorf("close PP ledger: %w", closeErr)
|
||||
}
|
||||
|
||||
manifest, err := os.Open(*manifestPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open PP download manifest: %w", err)
|
||||
}
|
||||
recovered, recovery, recoveryErr := ppconnector.RecoverMissingSHA256(snapshot, manifest)
|
||||
closeErr = manifest.Close()
|
||||
if recoveryErr != nil {
|
||||
return recoveryErr
|
||||
}
|
||||
if closeErr != nil {
|
||||
return fmt.Errorf("close PP download manifest: %w", closeErr)
|
||||
}
|
||||
overLimitManifest, err := os.Open(*manifestPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("reopen PP download manifest: %w", err)
|
||||
}
|
||||
complete, overLimit, overLimitErr := ppconnector.MergeOverLimitManifest(recovered, overLimitManifest)
|
||||
closeErr = overLimitManifest.Close()
|
||||
if overLimitErr != nil {
|
||||
return overLimitErr
|
||||
}
|
||||
if closeErr != nil {
|
||||
return fmt.Errorf("close PP download manifest: %w", closeErr)
|
||||
}
|
||||
if err := ppconnector.ValidateSnapshot(complete); err != nil {
|
||||
return fmt.Errorf("validate PP snapshot: %w", err)
|
||||
}
|
||||
plan, err := ppconnector.BuildImportPlan(complete)
|
||||
if err != nil {
|
||||
return fmt.Errorf("build PP import plan: %w", err)
|
||||
}
|
||||
|
||||
completed := 0
|
||||
aliases := 0
|
||||
for _, item := range complete.Items {
|
||||
switch item.Status {
|
||||
case ppconnector.StatusCompleted:
|
||||
completed++
|
||||
case ppconnector.StatusContentAlias:
|
||||
aliases++
|
||||
}
|
||||
}
|
||||
_, _ = fmt.Fprintf(stdout,
|
||||
"events=%d\nsource_items=%d\ncompleted=%d\ncontent_aliases=%d\nover_limit=%d\nrecovered_sha256=%d\nverified_bytes=%d\ncontent_objects=%d\nversions=%d\npublications=%d\n",
|
||||
complete.EventCount,
|
||||
len(plan.SourceItems),
|
||||
completed,
|
||||
aliases,
|
||||
overLimit.Added,
|
||||
recovery.Recovered,
|
||||
recovery.VerifiedBytes,
|
||||
len(plan.ContentObjects),
|
||||
len(plan.Versions),
|
||||
len(plan.Publications),
|
||||
)
|
||||
if *dryRun {
|
||||
_, _ = fmt.Fprintln(stdout, "dry_run=true")
|
||||
return nil
|
||||
}
|
||||
|
||||
databaseURL := strings.TrimSpace(getenv("DATABASE_URL"))
|
||||
if databaseURL == "" {
|
||||
return fmt.Errorf("DATABASE_URL is required unless -dry-run is used")
|
||||
}
|
||||
pool, err := pgxpool.New(ctx, databaseURL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open governance database: %w", err)
|
||||
}
|
||||
defer pool.Close()
|
||||
if err := pool.Ping(ctx); err != nil {
|
||||
return fmt.Errorf("ping governance database: %w", err)
|
||||
}
|
||||
if err := postgresstore.ApplyMigrations(ctx, pool); err != nil {
|
||||
return err
|
||||
}
|
||||
report, err := postgresstore.ImportPPLegacy(ctx, pool, plan)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, _ = fmt.Fprintf(stdout,
|
||||
"inserted_connections=%d\ninserted_source_items=%d\ninserted_content_objects=%d\ninserted_versions=%d\ninserted_publications=%d\n",
|
||||
report.Connections,
|
||||
report.SourceItems,
|
||||
report.ContentObjects,
|
||||
report.Versions,
|
||||
report.Publications,
|
||||
)
|
||||
current, err := postgresstore.LoadPPCurrentItems(ctx, pool, plan.Connection.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
reconciliation, err := ppconnector.PlanReconciliation(complete, current)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, _ = fmt.Fprintf(stdout,
|
||||
"reconcile_unchanged=%d\nreconcile_adopted=%d\nreconcile_over_limit=%d\nplanned_uploads=%d\n",
|
||||
reconciliation.Unchanged,
|
||||
reconciliation.Adopted,
|
||||
reconciliation.OverLimit,
|
||||
reconciliation.UploadRequired,
|
||||
)
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestRunDryRunPrintsOnlyAggregatePlan(t *testing.T) {
|
||||
directory := t.TempDir()
|
||||
ledgerPath := filepath.Join(directory, "state.jsonl")
|
||||
manifestPath := filepath.Join(directory, "manifest.json")
|
||||
sourceHash := strings.Repeat("a", 64)
|
||||
contentHash := strings.Repeat("c", 64)
|
||||
ledger := `{"kb_id":"kb-private","source_url_hash":"` + sourceHash + `","source_ids":["pp-private"],"bytes":10,"sha256":"` + contentHash + `","status":"completed","knowledge_id":"knowledge-private"}`
|
||||
if err := os.WriteFile(ledgerPath, []byte(ledger), 0o600); err != nil {
|
||||
t.Fatalf("write ledger fixture: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(manifestPath, []byte(`{"results":[]}`), 0o600); err != nil {
|
||||
t.Fatalf("write manifest fixture: %v", err)
|
||||
}
|
||||
var stdout bytes.Buffer
|
||||
|
||||
err := run(context.Background(), []string{"-ledger", ledgerPath, "-manifest", manifestPath, "-dry-run"}, func(string) string { return "" }, &stdout)
|
||||
if err != nil {
|
||||
t.Fatalf("run() error = %v", err)
|
||||
}
|
||||
output := stdout.String()
|
||||
for _, required := range []string{"events=1", "source_items=1", "content_objects=1", "publications=1", "dry_run=true"} {
|
||||
if !strings.Contains(output, required) {
|
||||
t.Fatalf("run() output missing %q: %q", required, output)
|
||||
}
|
||||
}
|
||||
for _, forbidden := range []string{"kb-private", sourceHash, "pp-private", "knowledge-private", ledgerPath, manifestPath} {
|
||||
if strings.Contains(output, forbidden) {
|
||||
t.Fatalf("run() output disclosed %q", forbidden)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunDryRunIncludesOverLimitManifestItems(t *testing.T) {
|
||||
directory := t.TempDir()
|
||||
ledgerPath := filepath.Join(directory, "state.jsonl")
|
||||
manifestPath := filepath.Join(directory, "manifest.json")
|
||||
sha := strings.Repeat("c", 64)
|
||||
ledger := `{"kb_id":"kb-1","source_url_hash":"` + strings.Repeat("a", 64) + `","source_ids":["pp-1"],"bytes":10,"sha256":"` + sha + `","status":"completed","knowledge_id":"knowledge-1"}`
|
||||
manifest := `{"results":[{"url":"https://source.invalid/large","status":"over_limit","bytes":52428801,"source_ids":["pp-2"]}]}`
|
||||
if err := os.WriteFile(ledgerPath, []byte(ledger), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(manifestPath, []byte(manifest), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var output bytes.Buffer
|
||||
err := run(context.Background(), []string{"-ledger", ledgerPath, "-manifest", manifestPath, "-dry-run"}, func(string) string { return "" }, &output)
|
||||
if err != nil {
|
||||
t.Fatalf("run() error = %v", err)
|
||||
}
|
||||
text := output.String()
|
||||
if !strings.Contains(text, "source_items=2\n") || !strings.Contains(text, "over_limit=1\n") {
|
||||
t.Fatalf("dry-run output = %q", text)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunImportPrintsZeroUploadReconciliation(t *testing.T) {
|
||||
databaseURL := os.Getenv("TEST_DATABASE_URL")
|
||||
if databaseURL == "" {
|
||||
t.Skip("TEST_DATABASE_URL is not set")
|
||||
}
|
||||
directory := t.TempDir()
|
||||
ledgerPath := filepath.Join(directory, "state.jsonl")
|
||||
manifestPath := filepath.Join(directory, "manifest.json")
|
||||
sha := strings.Repeat("f", 64)
|
||||
ledger := `{"kb_id":"kb-cli-reconcile","source_url_hash":"` + strings.Repeat("e", 64) + `","source_ids":["pp-4"],"bytes":10,"sha256":"` + sha + `","status":"completed","knowledge_id":"knowledge-4"}`
|
||||
if err := os.WriteFile(ledgerPath, []byte(ledger), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(manifestPath, []byte(`{"results":[]}`), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var output bytes.Buffer
|
||||
err := run(context.Background(), []string{"-ledger", ledgerPath, "-manifest", manifestPath}, func(key string) string {
|
||||
if key == "DATABASE_URL" {
|
||||
return databaseURL
|
||||
}
|
||||
return ""
|
||||
}, &output)
|
||||
if err != nil {
|
||||
t.Fatalf("run() error = %v", err)
|
||||
}
|
||||
if !strings.Contains(output.String(), "planned_uploads=0\n") {
|
||||
t.Fatalf("run() output = %q", output.String())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,313 @@
|
||||
// Command release-diag replays the read-only steps of the release export path
|
||||
// (source PG snapshot, source ES snapshot, source object read) and additionally
|
||||
// lists candidate OSS key prefixes, to pinpoint a failing export step with its
|
||||
// real underlying error. Credentials are read from files only and never printed.
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/url"
|
||||
"os"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
awsconfig "github.com/aws/aws-sdk-go-v2/config"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"knowledge-governance/internal/platform/weknora"
|
||||
"knowledge-governance/internal/releaseadapter/v071"
|
||||
)
|
||||
|
||||
type objectCfg struct {
|
||||
Provider string `json:"provider"`
|
||||
StorageBackendID string `json:"storage_backend_id"`
|
||||
Endpoint string `json:"endpoint"`
|
||||
Region string `json:"region"`
|
||||
Bucket string `json:"bucket"`
|
||||
AccessKeyFile string `json:"access_key_file"`
|
||||
SecretKeyFile string `json:"secret_key_file"`
|
||||
}
|
||||
|
||||
type endpointCfg struct {
|
||||
DatabaseURLFile string `json:"database_url_file"`
|
||||
Elasticsearch struct {
|
||||
BaseURL string `json:"base_url"`
|
||||
Index string `json:"index"`
|
||||
} `json:"elasticsearch"`
|
||||
Object objectCfg `json:"object"`
|
||||
WeKnora struct {
|
||||
BaseURL string `json:"base_url"`
|
||||
APIKeyFile string `json:"api_key_file"`
|
||||
} `json:"weknora"`
|
||||
}
|
||||
|
||||
type config struct {
|
||||
Routes []struct {
|
||||
TenantID uint64 `json:"tenant_id"`
|
||||
KnowledgeBaseID string `json:"knowledge_base_id"`
|
||||
EmbeddingModel string `json:"embedding_model"`
|
||||
Source endpointCfg `json:"source"`
|
||||
Target endpointCfg `json:"target"`
|
||||
} `json:"routes"`
|
||||
}
|
||||
|
||||
func secret(path string) string {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
fmt.Println("FATAL read secret:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
return string(data)
|
||||
}
|
||||
|
||||
func main() {
|
||||
configPath := flag.String("config", "", "release config json")
|
||||
knowledgeID := flag.String("knowledge", "", "knowledge id to diagnose")
|
||||
flag.Parse()
|
||||
raw, err := os.ReadFile(*configPath)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
var cfg config
|
||||
if err := json.Unmarshal(raw, &cfg); err != nil || len(cfg.Routes) != 1 {
|
||||
panic(err)
|
||||
}
|
||||
route := cfg.Routes[0]
|
||||
fmt.Println("route:", route.KnowledgeBaseID, "tenant:", route.TenantID)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
|
||||
pool, err := pgxpool.New(ctx, secret(route.Source.DatabaseURLFile))
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer pool.Close()
|
||||
if err := pool.Ping(ctx); err != nil {
|
||||
fmt.Println("STEP source PG connect: FAIL", err)
|
||||
return
|
||||
}
|
||||
fmt.Println("STEP source PG connect: ok")
|
||||
|
||||
postgres, err := v071.NewPostgresExporter(pool)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
snapshot, err := postgres.Export(ctx, v071.PostgresIdentity{TenantID: route.TenantID, KnowledgeBaseID: route.KnowledgeBaseID, KnowledgeID: *knowledgeID})
|
||||
if err != nil {
|
||||
fmt.Println("STEP postgres export: FAIL", err)
|
||||
return
|
||||
}
|
||||
names := make([]string, 0, len(snapshot.Tables))
|
||||
for _, table := range snapshot.Tables {
|
||||
names = append(names, fmt.Sprintf("%s:%d", table.Name, table.RowCount))
|
||||
}
|
||||
fmt.Println("STEP postgres export: ok tables", names)
|
||||
|
||||
search, err := v071.NewSearchExporter(v071.SearchExporterConfig{BaseURL: route.Source.Elasticsearch.BaseURL, Index: route.Source.Elasticsearch.Index, PageSize: 500})
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
searchSnapshot, err := search.Export(ctx, v071.SearchIdentity{TenantID: route.TenantID, KnowledgeBaseID: route.KnowledgeBaseID, KnowledgeID: *knowledgeID})
|
||||
if err != nil {
|
||||
fmt.Println("STEP elasticsearch export: FAIL", err)
|
||||
return
|
||||
}
|
||||
fmt.Printf("STEP elasticsearch export: ok documents=%d\n", len(searchSnapshot.Documents))
|
||||
|
||||
store, err := newStore(ctx, route.Source.Object)
|
||||
if err != nil {
|
||||
fmt.Println("STEP object store build: FAIL", err)
|
||||
return
|
||||
}
|
||||
exporter, err := v071.NewObjectExporter(store, v071.ObjectExporterLimits{})
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
objectSnapshot, err := exporter.Export(ctx, snapshot)
|
||||
if err != nil {
|
||||
fmt.Println("STEP object export: FAIL", err)
|
||||
} else {
|
||||
fmt.Printf("STEP object export: ok objects=%d bytes=%d\n", len(objectSnapshot.Objects), objectSnapshot.TotalBytes)
|
||||
}
|
||||
|
||||
client, err := newRawClient(ctx, route.Source.Object)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
for _, prefix := range []string{"weknora/", "oss://", ""} {
|
||||
listed, err := client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{Bucket: aws.String(route.Source.Object.Bucket), Prefix: aws.String(prefix), MaxKeys: aws.Int32(3)})
|
||||
if err != nil {
|
||||
fmt.Printf("STEP list prefix %q: FAIL %v\n", prefix, err)
|
||||
continue
|
||||
}
|
||||
fmt.Printf("STEP list prefix %q: %d objects\n", prefix, len(listed.Contents))
|
||||
for _, object := range listed.Contents {
|
||||
fmt.Println(" ", aws.ToString(object.Key))
|
||||
}
|
||||
}
|
||||
|
||||
knowledge := snapshot.Table("knowledges")
|
||||
for _, row := range knowledge.Rows {
|
||||
var typed struct {
|
||||
Source string `json:"source"`
|
||||
}
|
||||
_ = json.Unmarshal(row, &typed)
|
||||
fmt.Println("knowledge.source:", typed.Source)
|
||||
}
|
||||
for _, raw := range snapshot.Table("resources").Rows {
|
||||
var typed struct {
|
||||
PhysicalPath string `json:"physical_path"`
|
||||
}
|
||||
_ = json.Unmarshal(raw, &typed)
|
||||
fmt.Println("resource physical_path:", typed.PhysicalPath)
|
||||
}
|
||||
|
||||
fmt.Println("STEP target object stage probe:")
|
||||
targetStore, err := newStore(ctx, route.Target.Object)
|
||||
if err != nil {
|
||||
fmt.Println(" target store build FAIL", err)
|
||||
return
|
||||
}
|
||||
probePath := ".governance-release-diag/probe.txt"
|
||||
payload := []byte("release-sim-probe")
|
||||
if err := targetStore.Put(ctx, route.Target.Object.Provider, probePath, bytes.NewReader(payload), int64(len(payload))); err != nil {
|
||||
fmt.Println(" target Put FAIL:", err)
|
||||
} else {
|
||||
size, digest, err := targetStore.Stat(ctx, route.Target.Object.Provider, probePath)
|
||||
fmt.Println(" target Put ok; Stat:", size, digest[:8], "err:", err)
|
||||
}
|
||||
|
||||
fmt.Println("STEP real ObjectImporter Stage/Promote with exported snapshot:")
|
||||
importer, err := v071.NewObjectImporter(targetStore, v071.TargetObjectConfig{StorageBackendID: route.Target.Object.StorageBackendID, Provider: route.Target.Object.Provider})
|
||||
if err != nil {
|
||||
fmt.Println(" importer build FAIL", err)
|
||||
return
|
||||
}
|
||||
stage, err := importer.Stage(ctx, "diag-stage", objectSnapshot)
|
||||
if err != nil {
|
||||
fmt.Println(" Stage FAIL:", err)
|
||||
return
|
||||
}
|
||||
fmt.Println(" Stage ok objects:", len(stage.Objects))
|
||||
if err := importer.Promote(ctx, stage); err != nil {
|
||||
fmt.Println(" Promote FAIL:", err)
|
||||
return
|
||||
}
|
||||
fmt.Println(" Promote ok")
|
||||
|
||||
fmt.Println("STEP target Postgres replace:")
|
||||
targetPool, err := pgxpool.New(ctx, secret(route.Target.DatabaseURLFile))
|
||||
if err != nil {
|
||||
fmt.Println(" target pool FAIL", err)
|
||||
return
|
||||
}
|
||||
defer targetPool.Close()
|
||||
if err := targetPool.Ping(ctx); err != nil {
|
||||
fmt.Println(" target PG connect FAIL", err)
|
||||
return
|
||||
}
|
||||
pgImporter, err := v071.NewPostgresImporter(targetPool, v071.PostgresImporterConfig{TargetStorageBackendID: route.Target.Object.StorageBackendID, TargetProvider: route.Target.Object.Provider})
|
||||
if err != nil {
|
||||
fmt.Println(" importer config FAIL", err)
|
||||
return
|
||||
}
|
||||
identity := v071.PostgresIdentity{TenantID: route.TenantID, KnowledgeBaseID: route.KnowledgeBaseID, KnowledgeID: *knowledgeID}
|
||||
if err := pgImporter.Replace(ctx, identity, snapshot); err != nil {
|
||||
fmt.Println(" Replace FAIL:", err)
|
||||
return
|
||||
}
|
||||
fmt.Println(" Replace ok")
|
||||
|
||||
fmt.Println("STEP target ES replace:")
|
||||
targetSearch, err := v071.NewSearchImporter(v071.SearchExporterConfig{BaseURL: route.Target.Elasticsearch.BaseURL, Index: route.Target.Elasticsearch.Index, PageSize: 500})
|
||||
if err != nil {
|
||||
fmt.Println(" importer build FAIL", err)
|
||||
return
|
||||
}
|
||||
if _, err := targetSearch.Replace(ctx, v071.SearchIdentity{TenantID: route.TenantID, KnowledgeBaseID: route.KnowledgeBaseID, KnowledgeID: *knowledgeID}, searchSnapshot); err != nil {
|
||||
fmt.Println(" ES Replace FAIL:", err)
|
||||
return
|
||||
}
|
||||
fmt.Println(" ES Replace ok")
|
||||
|
||||
fmt.Println("STEP target WeKnora verification replay:")
|
||||
wknoraClient, err := weknora.NewClient(weknora.ClientConfig{BaseURL: route.Target.WeKnora.BaseURL, APIKey: secret(route.Target.WeKnora.APIKeyFile), Timeout: 30 * time.Second, AllowDockerHTTP: true})
|
||||
if err != nil {
|
||||
fmt.Println(" client build FAIL", err)
|
||||
return
|
||||
}
|
||||
knowledgeRemote, err := wknoraClient.GetKnowledge(ctx, *knowledgeID)
|
||||
fmt.Println(" GetKnowledge err:", err, "id:", knowledgeRemote.ID, "kb:", knowledgeRemote.KnowledgeBaseID, "parse:", knowledgeRemote.ParseStatus)
|
||||
chunksRemote, total, err := wknoraClient.ListChunks(ctx, *knowledgeID, 1, 5, nil)
|
||||
fmt.Println(" ListChunks err:", err, "total:", total, "page:", len(chunksRemote))
|
||||
probe := ""
|
||||
for _, document := range searchSnapshot.Documents {
|
||||
var fields struct {
|
||||
Content string `json:"content"`
|
||||
}
|
||||
_ = json.Unmarshal(document.Source, &fields)
|
||||
if utf8.RuneCountInString(fields.Content) >= 12 {
|
||||
probe = string([]rune(fields.Content)[:min(48, utf8.RuneCountInString(fields.Content))])
|
||||
break
|
||||
}
|
||||
}
|
||||
results, err := wknoraClient.SearchKnowledge(ctx, weknora.SearchRequest{Query: probe, KnowledgeIDs: []string{*knowledgeID}})
|
||||
fmt.Println(" SearchKnowledge err:", err, "probe_len:", len(probe), "results:", len(results))
|
||||
for index, result := range results {
|
||||
if index >= 3 {
|
||||
break
|
||||
}
|
||||
fmt.Printf(" hit knowledge=%s score=%.3f type=%s\n", result.KnowledgeID, result.Score, result.ChunkType)
|
||||
}
|
||||
if len(results) == 0 && probe != "" {
|
||||
results, err = wknoraClient.SearchKnowledge(ctx, weknora.SearchRequest{Query: "CRA 620mm", KnowledgeIDs: []string{*knowledgeID}})
|
||||
fmt.Println(" short-probe SearchKnowledge err:", err, "results:", len(results))
|
||||
}
|
||||
}
|
||||
|
||||
func min(a, b int) int {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func newStore(ctx context.Context, object objectCfg) (*v071.S3ObjectStore, error) {
|
||||
parsed, err := url.Parse(object.Endpoint)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return v071.NewS3ObjectStore(v071.S3ObjectStoreConfig{
|
||||
Provider: object.Provider, Endpoint: object.Endpoint, Region: object.Region,
|
||||
AccessKeyID: secret(object.AccessKeyFile), SecretAccessKey: secret(object.SecretKeyFile),
|
||||
Bucket: object.Bucket, Secure: parsed.Scheme == "https",
|
||||
})
|
||||
}
|
||||
|
||||
func newRawClient(ctx context.Context, object objectCfg) (*s3.Client, error) {
|
||||
parsed, err := url.Parse(object.Endpoint)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
awsCfg, err := awsconfig.LoadDefaultConfig(ctx, awsconfig.WithRegion(object.Region), awsconfig.WithCredentialsProvider(
|
||||
credentials.NewStaticCredentialsProvider(secret(object.AccessKeyFile), secret(object.SecretKeyFile), "")))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s3.NewFromConfig(awsCfg, func(options *s3.Options) {
|
||||
options.BaseEndpoint = aws.String(object.Endpoint)
|
||||
options.UsePathStyle = false
|
||||
_ = parsed
|
||||
}), nil
|
||||
}
|
||||
|
||||
var _ = io.Discard
|
||||
@@ -0,0 +1,155 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
const maxReleaseConfigBytes = 1024 * 1024
|
||||
|
||||
type releaseRuntimeConfig struct {
|
||||
ArchiveRoot string `json:"archive_root"`
|
||||
Routes []releaseRouteConfig `json:"routes"`
|
||||
}
|
||||
|
||||
type releaseRouteConfig struct {
|
||||
TenantID uint64 `json:"tenant_id"`
|
||||
KnowledgeBaseID string `json:"knowledge_base_id"`
|
||||
EmbeddingModel string `json:"embedding_model"`
|
||||
Source releaseEndpointConfig `json:"source"`
|
||||
Target releaseEndpointConfig `json:"target"`
|
||||
}
|
||||
|
||||
type releaseEndpointConfig struct {
|
||||
DatabaseURLFile string `json:"database_url_file"`
|
||||
Elasticsearch releaseElasticsearchConfig `json:"elasticsearch"`
|
||||
Object releaseObjectConfig `json:"object"`
|
||||
WeKnora releaseWeKnoraConfig `json:"weknora"`
|
||||
}
|
||||
|
||||
type releaseElasticsearchConfig struct {
|
||||
BaseURL string `json:"base_url"`
|
||||
Index string `json:"index"`
|
||||
TokenFile string `json:"token_file"`
|
||||
}
|
||||
|
||||
type releaseObjectConfig struct {
|
||||
Provider string `json:"provider"`
|
||||
Endpoint string `json:"endpoint"`
|
||||
Region string `json:"region"`
|
||||
Bucket string `json:"bucket"`
|
||||
StorageBackendID string `json:"storage_backend_id"`
|
||||
AccessKeyFile string `json:"access_key_file"`
|
||||
SecretKeyFile string `json:"secret_key_file"`
|
||||
}
|
||||
|
||||
type releaseWeKnoraConfig struct {
|
||||
BaseURL string `json:"base_url"`
|
||||
APIKeyFile string `json:"api_key_file"`
|
||||
AllowDockerHTTP bool `json:"allow_docker_http"`
|
||||
}
|
||||
|
||||
func loadReleaseRuntimeConfig(path string) (releaseRuntimeConfig, error) {
|
||||
if !filepath.IsAbs(path) {
|
||||
return releaseRuntimeConfig{}, errors.New("release config path must be absolute")
|
||||
}
|
||||
fd, err := syscall.Open(path, syscall.O_RDONLY|syscall.O_NOFOLLOW, 0)
|
||||
if err != nil {
|
||||
return releaseRuntimeConfig{}, errors.New("open release config")
|
||||
}
|
||||
file := os.NewFile(uintptr(fd), "release-config")
|
||||
if file == nil {
|
||||
syscall.Close(fd)
|
||||
return releaseRuntimeConfig{}, errors.New("open release config")
|
||||
}
|
||||
defer file.Close()
|
||||
info, err := file.Stat()
|
||||
if err != nil || !info.Mode().IsRegular() || info.Mode().Perm() != 0o600 || info.Size() <= 0 || info.Size() > maxReleaseConfigBytes {
|
||||
return releaseRuntimeConfig{}, errors.New("release config must be a non-empty regular 0600 file")
|
||||
}
|
||||
payload, err := io.ReadAll(io.LimitReader(file, maxReleaseConfigBytes+1))
|
||||
if err != nil || len(payload) > maxReleaseConfigBytes {
|
||||
return releaseRuntimeConfig{}, errors.New("read release config")
|
||||
}
|
||||
decoder := json.NewDecoder(bytes.NewReader(payload))
|
||||
decoder.DisallowUnknownFields()
|
||||
var config releaseRuntimeConfig
|
||||
if err := decoder.Decode(&config); err != nil {
|
||||
return releaseRuntimeConfig{}, errors.New("decode release config")
|
||||
}
|
||||
var trailing any
|
||||
if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
|
||||
return releaseRuntimeConfig{}, errors.New("release config contains trailing data")
|
||||
}
|
||||
if err := validateReleaseRuntimeConfig(config); err != nil {
|
||||
return releaseRuntimeConfig{}, err
|
||||
}
|
||||
return config, nil
|
||||
}
|
||||
|
||||
func validateReleaseRuntimeConfig(config releaseRuntimeConfig) error {
|
||||
if !filepath.IsAbs(config.ArchiveRoot) || len(config.Routes) == 0 {
|
||||
return errors.New("release archive root and routes are required")
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, route := range config.Routes {
|
||||
if route.TenantID == 0 || !validConfigIdentifier(route.KnowledgeBaseID) || !validConfigIdentifier(route.EmbeddingModel) || seen[route.KnowledgeBaseID] {
|
||||
return errors.New("invalid or duplicate release route")
|
||||
}
|
||||
seen[route.KnowledgeBaseID] = true
|
||||
if !validReleaseEndpoint(route.Source, false) || !validReleaseEndpoint(route.Target, true) {
|
||||
return errors.New("invalid release route endpoint")
|
||||
}
|
||||
if route.Source.DatabaseURLFile == route.Target.DatabaseURLFile ||
|
||||
route.Source.Elasticsearch.BaseURL == route.Target.Elasticsearch.BaseURL ||
|
||||
(route.Source.Object.Endpoint == route.Target.Object.Endpoint && route.Source.Object.Bucket == route.Target.Object.Bucket) {
|
||||
return errors.New("release source and target must be distinct")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validReleaseEndpoint(endpoint releaseEndpointConfig, target bool) bool {
|
||||
if !absoluteFile(endpoint.DatabaseURLFile) || !validHTTPURL(endpoint.Elasticsearch.BaseURL) || !validConfigIdentifier(endpoint.Elasticsearch.Index) ||
|
||||
(endpoint.Elasticsearch.TokenFile != "" && !absoluteFile(endpoint.Elasticsearch.TokenFile)) ||
|
||||
!validObjectEndpoint(endpoint.Object) {
|
||||
return false
|
||||
}
|
||||
if target {
|
||||
return validConfigIdentifier(endpoint.Object.StorageBackendID) && validHTTPURL(endpoint.WeKnora.BaseURL) && absoluteFile(endpoint.WeKnora.APIKeyFile)
|
||||
}
|
||||
return endpoint.WeKnora.BaseURL == "" && endpoint.WeKnora.APIKeyFile == ""
|
||||
}
|
||||
|
||||
func validObjectEndpoint(object releaseObjectConfig) bool {
|
||||
return (object.Provider == "minio" || object.Provider == "s3" || object.Provider == "oss") && validHTTPURL(object.Endpoint) &&
|
||||
validConfigIdentifier(object.Bucket) && absoluteFile(object.AccessKeyFile) && absoluteFile(object.SecretKeyFile)
|
||||
}
|
||||
|
||||
func absoluteFile(path string) bool { return filepath.IsAbs(path) && filepath.Clean(path) == path }
|
||||
|
||||
func validHTTPURL(value string) bool {
|
||||
parsed, err := url.Parse(value)
|
||||
return err == nil && (parsed.Scheme == "http" || parsed.Scheme == "https") && parsed.Host != "" && parsed.User == nil && parsed.RawQuery == "" && parsed.Fragment == ""
|
||||
}
|
||||
|
||||
func validConfigIdentifier(value string) bool {
|
||||
value = strings.TrimSpace(value)
|
||||
if value == "" || len(value) > 128 {
|
||||
return false
|
||||
}
|
||||
for _, char := range value {
|
||||
if (char >= 'a' && char <= 'z') || (char >= 'A' && char <= 'Z') || (char >= '0' && char <= '9') || char == '-' || char == '_' || char == '.' {
|
||||
continue
|
||||
}
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestLoadReleaseRuntimeConfigValidatesRestrictedMultiKBConfig(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
path := filepath.Join(root, "release.json")
|
||||
payload := validReleaseConfigJSON(root)
|
||||
if err := os.WriteFile(path, []byte(payload), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
config, err := loadReleaseRuntimeConfig(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(config.Routes) != 1 || config.Routes[0].KnowledgeBaseID != "kb-safe" || config.Routes[0].Target.Object.StorageBackendID != "target-backend" {
|
||||
t.Fatalf("config=%#v", config)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadReleaseRuntimeConfigRejectsUnsafeFilesAndUnknownFields(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
payload := validReleaseConfigJSON(root)
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
setup func(string) error
|
||||
}{
|
||||
{name: "overbroad", setup: func(path string) error { return os.WriteFile(path, []byte(payload), 0o644) }},
|
||||
{name: "unknown field", setup: func(path string) error {
|
||||
return os.WriteFile(path, []byte(strings.Replace(payload, `"archive_root":`, `"unknown":true,"archive_root":`, 1)), 0o600)
|
||||
}},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
path := filepath.Join(root, strings.ReplaceAll(test.name, " ", "-")+".json")
|
||||
if err := test.setup(path); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := loadReleaseRuntimeConfig(path); err == nil {
|
||||
t.Fatal("unsafe config accepted")
|
||||
}
|
||||
})
|
||||
}
|
||||
real := filepath.Join(root, "real.json")
|
||||
_ = os.WriteFile(real, []byte(payload), 0o600)
|
||||
link := filepath.Join(root, "link.json")
|
||||
_ = os.Symlink(real, link)
|
||||
if _, err := loadReleaseRuntimeConfig(link); err == nil {
|
||||
t.Fatal("symlink accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func validReleaseConfigJSON(root string) string {
|
||||
return `{
|
||||
"archive_root":"` + filepath.ToSlash(filepath.Join(root, "archive")) + `",
|
||||
"routes":[{
|
||||
"tenant_id":10000,"knowledge_base_id":"kb-safe","embedding_model":"bge-m3",
|
||||
"source":{"database_url_file":"/run/secrets/source-db","elasticsearch":{"base_url":"http://source-es:9200","index":"chunks","token_file":"/run/secrets/source-es"},"object":{"provider":"minio","endpoint":"http://source-minio:9000","region":"us-east-1","bucket":"source","access_key_file":"/run/secrets/source-ak","secret_key_file":"/run/secrets/source-sk"}},
|
||||
"target":{"database_url_file":"/run/secrets/target-db","elasticsearch":{"base_url":"https://target-es.example","index":"chunks","token_file":"/run/secrets/target-es"},"object":{"provider":"oss","endpoint":"https://oss.example","region":"cn-shenzhen","bucket":"target","storage_backend_id":"target-backend","access_key_file":"/run/secrets/target-ak","secret_key_file":"/run/secrets/target-sk"},"weknora":{"base_url":"https://weknora.example","api_key_file":"/run/secrets/target-weknora"}}
|
||||
}]}`
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"knowledge-governance/internal/appconfig"
|
||||
"knowledge-governance/internal/jobs"
|
||||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||||
releaseworkflow "knowledge-governance/internal/workflows/release"
|
||||
)
|
||||
|
||||
func main() {
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
if err := run(ctx, os.Getenv); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func run(ctx context.Context, getenv func(string) string) error {
|
||||
config, err := appconfig.Load(getenv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !config.ReleaseWorkerEnabled {
|
||||
log.Print("release worker disabled; no production credentials are loaded")
|
||||
<-ctx.Done()
|
||||
return nil
|
||||
}
|
||||
runtimeConfig, err := loadReleaseRuntimeConfig(config.ReleaseConfigFile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
releaseRuntime, err := buildReleaseRuntime(ctx, runtimeConfig)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer releaseRuntime.Close()
|
||||
|
||||
pool, err := pgxpool.New(ctx, config.DatabaseURL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer pool.Close()
|
||||
if err := pool.Ping(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := postgresstore.ApplyMigrations(ctx, pool); err != nil {
|
||||
return err
|
||||
}
|
||||
store := postgresstore.NewStore(pool)
|
||||
releaseHandler, err := releaseworkflow.NewHandler(store, releaseRuntime.publisher, time.Now)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
worker, err := jobs.NewWorker(jobs.WorkerConfig{
|
||||
Queue: store,
|
||||
WorkerID: config.WorkerID,
|
||||
LeaseDuration: config.LeaseDuration,
|
||||
RetryDelay: config.RetryDelay,
|
||||
Handlers: map[string]jobs.Handler{
|
||||
"release.publish": releaseHandler,
|
||||
"release.rollback": releaseHandler,
|
||||
},
|
||||
OnError: func(err error) { log.Printf("release worker error: %v", err) },
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log.Printf("release worker %s started with %d release route(s)", config.WorkerID, len(runtimeConfig.Routes))
|
||||
return worker.Run(ctx, config.PollInterval)
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestRunReturnsWithoutDatabaseWhenReleaseWorkerDisabled(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
values := map[string]string{
|
||||
"DATABASE_URL": "postgres://must-not-be-contacted.invalid/governance",
|
||||
"RELEASE_WORKER_ENABLED": "false",
|
||||
}
|
||||
if err := run(ctx, func(key string) string { return values[key] }); err != nil {
|
||||
t.Fatalf("run() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnabledReleaseWorkerLoadsRestrictedConfigBeforeDatabaseConnections(t *testing.T) {
|
||||
values := map[string]string{"DATABASE_URL": "not-a-database-url", "RELEASE_WORKER_ENABLED": "true", "RELEASE_CONFIG_FILE": "/does/not/exist/release.json"}
|
||||
err := run(context.Background(), func(key string) string { return values[key] })
|
||||
if err == nil || !strings.Contains(err.Error(), "release config") || strings.Contains(err.Error(), "database") {
|
||||
t.Fatalf("error=%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReleaseWorkerRegistersOnlyPublishAndRollbackHandlers(t *testing.T) {
|
||||
source, err := os.ReadFile("main.go")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
text := string(source)
|
||||
for _, required := range []string{`"release.publish"`, `"release.rollback"`} {
|
||||
if !strings.Contains(text, required) {
|
||||
t.Fatalf("missing handler %s", required)
|
||||
}
|
||||
}
|
||||
if strings.Contains(text, `"release.probe"`) {
|
||||
t.Fatal("probe-only release handler remains")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,222 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/url"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"knowledge-governance/internal/platform/weknora"
|
||||
v071 "knowledge-governance/internal/releaseadapter/v071"
|
||||
releaseworkflow "knowledge-governance/internal/workflows/release"
|
||||
)
|
||||
|
||||
type releaseRuntime struct {
|
||||
publisher *releaseworkflow.RoutingPublisher
|
||||
pools []*pgxpool.Pool
|
||||
}
|
||||
|
||||
func (runtime *releaseRuntime) Close() {
|
||||
for index := len(runtime.pools) - 1; index >= 0; index-- {
|
||||
runtime.pools[index].Close()
|
||||
}
|
||||
}
|
||||
|
||||
func buildReleaseRuntime(ctx context.Context, config releaseRuntimeConfig) (*releaseRuntime, error) {
|
||||
runtime := &releaseRuntime{}
|
||||
publishers := make(map[string]releaseworkflow.Publisher, len(config.Routes))
|
||||
for _, route := range config.Routes {
|
||||
publisher, pools, err := buildReleaseRoute(ctx, config.ArchiveRoot, route)
|
||||
if err != nil {
|
||||
runtime.Close()
|
||||
return nil, err
|
||||
}
|
||||
runtime.pools = append(runtime.pools, pools...)
|
||||
publishers[route.KnowledgeBaseID] = publisher
|
||||
}
|
||||
router, err := releaseworkflow.NewRoutingPublisher(publishers)
|
||||
if err != nil {
|
||||
runtime.Close()
|
||||
return nil, errors.New("create release publisher routes")
|
||||
}
|
||||
runtime.publisher = router
|
||||
return runtime, nil
|
||||
}
|
||||
|
||||
func buildReleaseRoute(ctx context.Context, archiveRoot string, route releaseRouteConfig) (releaseworkflow.Publisher, []*pgxpool.Pool, error) {
|
||||
sourceDatabaseURL, err := readRestrictedValue(route.Source.DatabaseURLFile)
|
||||
if err != nil {
|
||||
return nil, nil, errors.New("read source database credential")
|
||||
}
|
||||
targetDatabaseURL, err := readRestrictedValue(route.Target.DatabaseURLFile)
|
||||
if err != nil {
|
||||
return nil, nil, errors.New("read target database credential")
|
||||
}
|
||||
if err := validateDistinctDatabaseURLs(sourceDatabaseURL, targetDatabaseURL); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
sourcePool, err := openReleasePool(ctx, sourceDatabaseURL)
|
||||
if err != nil {
|
||||
return nil, nil, errors.New("connect release source database")
|
||||
}
|
||||
pools := []*pgxpool.Pool{sourcePool}
|
||||
targetPool, err := openReleasePool(ctx, targetDatabaseURL)
|
||||
if err != nil {
|
||||
sourcePool.Close()
|
||||
return nil, nil, errors.New("connect release target database")
|
||||
}
|
||||
pools = append(pools, targetPool)
|
||||
cleanup := func(err error) (releaseworkflow.Publisher, []*pgxpool.Pool, error) {
|
||||
for index := len(pools) - 1; index >= 0; index-- {
|
||||
pools[index].Close()
|
||||
}
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
sourceSearchToken, err := readOptionalRestrictedValue(route.Source.Elasticsearch.TokenFile)
|
||||
if err != nil {
|
||||
return cleanup(errors.New("read source Elasticsearch credential"))
|
||||
}
|
||||
targetSearchToken, err := readOptionalRestrictedValue(route.Target.Elasticsearch.TokenFile)
|
||||
if err != nil {
|
||||
return cleanup(errors.New("read target Elasticsearch credential"))
|
||||
}
|
||||
sourceSearch, err := v071.NewSearchExporter(v071.SearchExporterConfig{BaseURL: route.Source.Elasticsearch.BaseURL, Index: route.Source.Elasticsearch.Index, AuthorizationToken: sourceSearchToken, PageSize: 500})
|
||||
if err != nil {
|
||||
return cleanup(errors.New("create source Elasticsearch exporter"))
|
||||
}
|
||||
targetSearchConfig := v071.SearchExporterConfig{BaseURL: route.Target.Elasticsearch.BaseURL, Index: route.Target.Elasticsearch.Index, AuthorizationToken: targetSearchToken, PageSize: 500}
|
||||
targetSearchInspector, err := v071.NewSearchExporter(targetSearchConfig)
|
||||
if err != nil {
|
||||
return cleanup(errors.New("create target Elasticsearch inspector"))
|
||||
}
|
||||
targetSearch, err := v071.NewSearchImporter(targetSearchConfig)
|
||||
if err != nil {
|
||||
return cleanup(errors.New("create target Elasticsearch importer"))
|
||||
}
|
||||
|
||||
sourceObject, err := buildS3ObjectStore(route.Source.Object)
|
||||
if err != nil {
|
||||
return cleanup(errors.New("create source object reader"))
|
||||
}
|
||||
targetObject, err := buildS3ObjectStore(route.Target.Object)
|
||||
if err != nil {
|
||||
return cleanup(errors.New("create target object store"))
|
||||
}
|
||||
objectExporter, err := v071.NewObjectExporter(sourceObject, v071.ObjectExporterLimits{})
|
||||
if err != nil {
|
||||
return cleanup(errors.New("create source object exporter"))
|
||||
}
|
||||
objectImporter, err := v071.NewObjectImporter(targetObject, v071.TargetObjectConfig{StorageBackendID: route.Target.Object.StorageBackendID, Provider: route.Target.Object.Provider})
|
||||
if err != nil {
|
||||
return cleanup(errors.New("create target object importer"))
|
||||
}
|
||||
|
||||
sourceContract, err := v071.NewTargetContractInspector(v071.TargetContractInspectorConfig{DB: sourcePool, Mapping: sourceSearch, KnowledgeBaseID: route.KnowledgeBaseID, TenantID: route.TenantID, IndexName: route.Source.Elasticsearch.Index})
|
||||
if err != nil {
|
||||
return cleanup(errors.New("create source release contract inspector"))
|
||||
}
|
||||
targetContract, err := v071.NewTargetContractInspector(v071.TargetContractInspectorConfig{DB: targetPool, Mapping: targetSearchInspector, KnowledgeBaseID: route.KnowledgeBaseID, TenantID: route.TenantID, IndexName: route.Target.Elasticsearch.Index})
|
||||
if err != nil {
|
||||
return cleanup(errors.New("create target release contract inspector"))
|
||||
}
|
||||
postgresExporter, err := v071.NewPostgresExporter(sourcePool)
|
||||
if err != nil {
|
||||
return cleanup(errors.New("create source PostgreSQL exporter"))
|
||||
}
|
||||
postgresImporter, err := v071.NewPostgresImporter(targetPool, v071.PostgresImporterConfig{TargetStorageBackendID: route.Target.Object.StorageBackendID, TargetProvider: route.Target.Object.Provider})
|
||||
if err != nil {
|
||||
return cleanup(errors.New("create target PostgreSQL importer"))
|
||||
}
|
||||
documentExporter, err := v071.NewDocumentExporter(v071.DocumentExporterConfig{Contract: sourceContract, Postgres: postgresExporter, Search: sourceSearch, Objects: objectExporter, TenantID: route.TenantID, EmbeddingModel: route.EmbeddingModel, Now: time.Now})
|
||||
if err != nil {
|
||||
return cleanup(errors.New("create v0.7.1 document exporter"))
|
||||
}
|
||||
|
||||
archive, err := v071.NewBundleArchive(filepath.Join(archiveRoot, route.KnowledgeBaseID), 1024*1024*1024)
|
||||
if err != nil {
|
||||
return cleanup(errors.New("open release bundle archive"))
|
||||
}
|
||||
apiKey, err := readRestrictedValue(route.Target.WeKnora.APIKeyFile)
|
||||
if err != nil {
|
||||
return cleanup(errors.New("read target WeKnora credential"))
|
||||
}
|
||||
weknoraClient, err := weknora.NewClient(weknora.ClientConfig{BaseURL: route.Target.WeKnora.BaseURL, APIKey: apiKey, Timeout: 60 * time.Second, AllowDockerHTTP: route.Target.WeKnora.AllowDockerHTTP})
|
||||
if err != nil {
|
||||
return cleanup(errors.New("create target WeKnora verification client"))
|
||||
}
|
||||
verifier, err := v071.NewWeKnoraVerifier(weknoraClient)
|
||||
if err != nil {
|
||||
return cleanup(errors.New("create target WeKnora verifier"))
|
||||
}
|
||||
documentTarget, err := v071.NewDocumentTarget(v071.DocumentTargetConfig{Archive: archive, Contract: targetContract, Postgres: postgresImporter, Search: targetSearch, Objects: objectImporter, Verifier: verifier})
|
||||
if err != nil {
|
||||
return cleanup(errors.New("create v0.7.1 document target"))
|
||||
}
|
||||
publisher, err := releaseworkflow.NewSidecarPublisher(documentExporter, documentTarget)
|
||||
if err != nil {
|
||||
return cleanup(errors.New("create v0.7.1 release publisher"))
|
||||
}
|
||||
return publisher, pools, nil
|
||||
}
|
||||
|
||||
func buildS3ObjectStore(config releaseObjectConfig) (*v071.S3ObjectStore, error) {
|
||||
accessKey, err := readRestrictedValue(config.AccessKeyFile)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
secretKey, err := readRestrictedValue(config.SecretKeyFile)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
parsed, err := url.Parse(config.Endpoint)
|
||||
if err != nil {
|
||||
return nil, errors.New("invalid object endpoint")
|
||||
}
|
||||
return v071.NewS3ObjectStore(v071.S3ObjectStoreConfig{Provider: config.Provider, Endpoint: config.Endpoint, Region: config.Region, AccessKeyID: accessKey, SecretAccessKey: secretKey, Bucket: config.Bucket, Secure: parsed.Scheme == "https", ForcePathStyle: config.Provider == "minio"})
|
||||
}
|
||||
|
||||
func readRestrictedValue(path string) (string, error) {
|
||||
return weknora.ReadAPIKeyFile(path)
|
||||
}
|
||||
|
||||
func readOptionalRestrictedValue(path string) (string, error) {
|
||||
if path == "" {
|
||||
return "", nil
|
||||
}
|
||||
return readRestrictedValue(path)
|
||||
}
|
||||
|
||||
func openReleasePool(ctx context.Context, databaseURL string) (*pgxpool.Pool, error) {
|
||||
pool, err := pgxpool.New(ctx, databaseURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := pool.Ping(ctx); err != nil {
|
||||
pool.Close()
|
||||
return nil, err
|
||||
}
|
||||
return pool, nil
|
||||
}
|
||||
|
||||
func validateDistinctDatabaseURLs(sourceURL, targetURL string) error {
|
||||
source, err := pgxpool.ParseConfig(sourceURL)
|
||||
if err != nil {
|
||||
return errors.New("invalid source database credential")
|
||||
}
|
||||
target, err := pgxpool.ParseConfig(targetURL)
|
||||
if err != nil {
|
||||
return errors.New("invalid target database credential")
|
||||
}
|
||||
sourceIdentity := strings.ToLower(source.ConnConfig.Host) + "|" + strconv.FormatUint(uint64(source.ConnConfig.Port), 10) + "|" + source.ConnConfig.Database
|
||||
targetIdentity := strings.ToLower(target.ConnConfig.Host) + "|" + strconv.FormatUint(uint64(target.ConnConfig.Port), 10) + "|" + target.ConnConfig.Database
|
||||
if sourceIdentity == targetIdentity {
|
||||
return errors.New("release source and target database must be distinct")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestReleaseRuntimeRejectsSameSourceAndTargetDatabaseValues(t *testing.T) {
|
||||
if err := validateDistinctDatabaseURLs("postgres://user@same/db?sslmode=require", "postgres://user@same/db?sslmode=require"); err == nil {
|
||||
t.Fatal("same database URL accepted")
|
||||
}
|
||||
if err := validateDistinctDatabaseURLs("postgres://user@source/db?sslmode=require", "postgres://user@target/db?sslmode=require"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildReleaseRuntimeWithRestrictedFilesIntegration(t *testing.T) {
|
||||
sourceDatabaseURL := os.Getenv("TEST_WEKNORA_V071_DATABASE_URL")
|
||||
targetDatabaseURL := os.Getenv("TEST_WEKNORA_V071_TARGET_DATABASE_URL")
|
||||
minioEndpoint := os.Getenv("TEST_MINIO_ENDPOINT")
|
||||
if sourceDatabaseURL == "" || targetDatabaseURL == "" || minioEndpoint == "" {
|
||||
t.Skip("release runtime integration environment is not set")
|
||||
}
|
||||
esSource := httptest.NewServer(http.NotFoundHandler())
|
||||
defer esSource.Close()
|
||||
esTarget := httptest.NewServer(http.NotFoundHandler())
|
||||
defer esTarget.Close()
|
||||
weknoraTarget := httptest.NewServer(http.NotFoundHandler())
|
||||
defer weknoraTarget.Close()
|
||||
root := t.TempDir()
|
||||
archiveRoot := filepath.Join(root, "archive")
|
||||
archiveRoute := filepath.Join(archiveRoot, "kb-release-safe")
|
||||
if err := os.MkdirAll(archiveRoute, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
secret := func(name, value string) string {
|
||||
path := filepath.Join(root, name)
|
||||
if err := os.WriteFile(path, []byte(value+"\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
route := releaseRouteConfig{
|
||||
TenantID: 10000, KnowledgeBaseID: "kb-release-safe", EmbeddingModel: "bge-m3",
|
||||
Source: releaseEndpointConfig{DatabaseURLFile: secret("source-db", sourceDatabaseURL), Elasticsearch: releaseElasticsearchConfig{BaseURL: esSource.URL, Index: "chunks"}, Object: releaseObjectConfig{Provider: "minio", Endpoint: minioEndpoint, Region: "us-east-1", Bucket: "release-source", AccessKeyFile: secret("source-ak", "release-test"), SecretKeyFile: secret("source-sk", "release-test-secret")}},
|
||||
Target: releaseEndpointConfig{DatabaseURLFile: secret("target-db", targetDatabaseURL), Elasticsearch: releaseElasticsearchConfig{BaseURL: esTarget.URL, Index: "chunks"}, Object: releaseObjectConfig{Provider: "minio", Endpoint: minioEndpoint, Region: "us-east-1", Bucket: "release-target", StorageBackendID: "target-backend", AccessKeyFile: secret("target-ak", "release-test"), SecretKeyFile: secret("target-sk", "release-test-secret")}, WeKnora: releaseWeKnoraConfig{BaseURL: weknoraTarget.URL, APIKeyFile: secret("target-weknora", "test-api-key"), AllowDockerHTTP: true}},
|
||||
}
|
||||
runtime, err := buildReleaseRuntime(context.Background(), releaseRuntimeConfig{ArchiveRoot: archiveRoot, Routes: []releaseRouteConfig{route}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runtime.Close()
|
||||
if runtime.publisher == nil || len(runtime.pools) != 2 {
|
||||
t.Fatalf("runtime=%#v", runtime)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||||
"knowledge-governance/internal/platform/spool"
|
||||
)
|
||||
|
||||
type manualSeeder interface {
|
||||
SeedManualPublication(context.Context, postgresstore.ManualPublicationSeedRequest) (postgresstore.ManualPublicationSeedResult, error)
|
||||
}
|
||||
|
||||
type seedConfig struct {
|
||||
SpoolRoot string
|
||||
SpoolRef string
|
||||
Format string
|
||||
Title string
|
||||
TestKBID string
|
||||
MaxBytes int64
|
||||
Now time.Time
|
||||
}
|
||||
|
||||
func main() {
|
||||
if err := run(context.Background(), os.Args[1:], os.Getenv, os.Stdout); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func run(ctx context.Context, args []string, getenv func(string) string, output io.Writer) error {
|
||||
flags := flag.NewFlagSet("test-publication-seed", flag.ContinueOnError)
|
||||
flags.SetOutput(io.Discard)
|
||||
spoolRef := flags.String("spool-ref", "", "opaque relative spool reference")
|
||||
format := flags.String("format", "md", "validated artifact format")
|
||||
title := flags.String("title", "", "safe display title")
|
||||
if err := flags.Parse(args); err != nil {
|
||||
return errors.New("invalid seed arguments")
|
||||
}
|
||||
databaseURL := getenv("DATABASE_URL")
|
||||
spoolRoot := getenv("GOVERNANCE_SPOOL_ROOT")
|
||||
if spoolRoot == "" {
|
||||
spoolRoot = "/var/lib/governance/spool"
|
||||
}
|
||||
testKBID := getenv("TEST_WEKNORA_KB_ID")
|
||||
maxBytes := int64(50 * 1024 * 1024)
|
||||
if raw := getenv("TEST_WEKNORA_MAX_UPLOAD_BYTES"); raw != "" {
|
||||
parsed, err := strconv.ParseInt(raw, 10, 64)
|
||||
if err != nil || parsed <= 0 || parsed > 50*1024*1024 {
|
||||
return errors.New("invalid TEST_WEKNORA_MAX_UPLOAD_BYTES")
|
||||
}
|
||||
maxBytes = parsed
|
||||
}
|
||||
if databaseURL == "" || testKBID == "" || *spoolRef == "" || *title == "" {
|
||||
return errors.New("DATABASE_URL, TEST_WEKNORA_KB_ID, --spool-ref and --title are required")
|
||||
}
|
||||
pool, err := pgxpool.New(ctx, databaseURL)
|
||||
if err != nil {
|
||||
return errors.New("open governance database")
|
||||
}
|
||||
defer pool.Close()
|
||||
if err := pool.Ping(ctx); err != nil {
|
||||
return errors.New("connect governance database")
|
||||
}
|
||||
if err := postgresstore.ApplyMigrations(ctx, pool); err != nil {
|
||||
return errors.New("apply governance migrations")
|
||||
}
|
||||
result, err := seedFromSpool(ctx, postgresstore.NewStore(pool), seedConfig{
|
||||
SpoolRoot: spoolRoot,
|
||||
SpoolRef: *spoolRef,
|
||||
Format: *format,
|
||||
Title: *title,
|
||||
TestKBID: testKBID,
|
||||
MaxBytes: maxBytes,
|
||||
Now: time.Now().UTC(),
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return json.NewEncoder(output).Encode(result)
|
||||
}
|
||||
|
||||
func seedFromSpool(ctx context.Context, store manualSeeder, config seedConfig) (postgresstore.ManualPublicationSeedResult, error) {
|
||||
if store == nil {
|
||||
return postgresstore.ManualPublicationSeedResult{}, errors.New("manual publication store is required")
|
||||
}
|
||||
expectation, err := spool.InspectVerified(config.SpoolRoot, config.SpoolRef, config.Format, config.MaxBytes)
|
||||
if err != nil {
|
||||
return postgresstore.ManualPublicationSeedResult{}, errors.New("inspect manual spool artifact")
|
||||
}
|
||||
mime, ok := mimeForFormat(config.Format)
|
||||
if !ok {
|
||||
return postgresstore.ManualPublicationSeedResult{}, errors.New("unsupported manual publication format")
|
||||
}
|
||||
result, err := store.SeedManualPublication(ctx, postgresstore.ManualPublicationSeedRequest{
|
||||
TestKBID: config.TestKBID,
|
||||
Title: config.Title,
|
||||
SHA256: expectation.SHA256,
|
||||
SizeBytes: expectation.SizeBytes,
|
||||
MIME: mime,
|
||||
ValidatedFormat: expectation.Format,
|
||||
LocalSpoolRef: expectation.RelativeRef,
|
||||
Now: config.Now,
|
||||
})
|
||||
if err != nil {
|
||||
return postgresstore.ManualPublicationSeedResult{}, fmt.Errorf("seed manual publication: %w", err)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func mimeForFormat(format string) (string, bool) {
|
||||
mimes := map[string]string{
|
||||
"md": "text/markdown", "txt": "text/plain", "csv": "text/csv", "json": "application/json",
|
||||
"pdf": "application/pdf", "docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
||||
"xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
"pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation",
|
||||
"doc": "application/msword", "xls": "application/vnd.ms-excel", "ppt": "application/vnd.ms-powerpoint",
|
||||
}
|
||||
value, ok := mimes[format]
|
||||
return value, ok
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||||
)
|
||||
|
||||
func TestSeedFromSpoolPassesVerifiedArtifactProjection(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
if err := os.Mkdir(filepath.Join(root, "manual"), 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body := []byte("# Governance fixture\n\nThe unique marker is MIKU-P3-5927.\n")
|
||||
ref := filepath.Join("manual", "fixture.md")
|
||||
if err := os.WriteFile(filepath.Join(root, ref), body, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store := &manualSeederStub{result: postgresstore.ManualPublicationSeedResult{PublicationID: "tp-1", JobID: "job-1"}}
|
||||
now := time.Date(2026, 9, 5, 6, 0, 0, 0, time.UTC)
|
||||
|
||||
result, err := seedFromSpool(context.Background(), store, seedConfig{
|
||||
SpoolRoot: root,
|
||||
SpoolRef: ref,
|
||||
Format: "md",
|
||||
Title: "Governance fixture",
|
||||
TestKBID: "kb-1",
|
||||
MaxBytes: 1024,
|
||||
Now: now,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("seedFromSpool() error = %v", err)
|
||||
}
|
||||
digest := sha256.Sum256(body)
|
||||
if result.PublicationID != "tp-1" || store.request.SHA256 != hex.EncodeToString(digest[:]) || store.request.SizeBytes != int64(len(body)) {
|
||||
t.Fatalf("result/request = %#v %#v", result, store.request)
|
||||
}
|
||||
if store.request.LocalSpoolRef != ref || store.request.ValidatedFormat != "md" || store.request.MIME != "text/markdown" || store.request.Now != now {
|
||||
t.Fatalf("request = %#v", store.request)
|
||||
}
|
||||
}
|
||||
|
||||
type manualSeederStub struct {
|
||||
request postgresstore.ManualPublicationSeedRequest
|
||||
result postgresstore.ManualPublicationSeedResult
|
||||
err error
|
||||
}
|
||||
|
||||
func (store *manualSeederStub) SeedManualPublication(_ context.Context, request postgresstore.ManualPublicationSeedRequest) (postgresstore.ManualPublicationSeedResult, error) {
|
||||
store.request = request
|
||||
return store.result, store.err
|
||||
}
|
||||
@@ -0,0 +1,174 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||||
"knowledge-governance/internal/platform/weknora"
|
||||
)
|
||||
|
||||
type reportStore interface {
|
||||
LoadPublicationForValidation(context.Context, string) (postgresstore.PublicationForValidation, error)
|
||||
ListAcceptanceResults(context.Context, string) ([]governance.AcceptanceRecord, error)
|
||||
}
|
||||
|
||||
type reportClient interface {
|
||||
GetKnowledge(context.Context, string) (weknora.Knowledge, error)
|
||||
ListChunks(context.Context, string, int, int, []string) ([]weknora.Chunk, int64, error)
|
||||
SearchKnowledge(context.Context, weknora.SearchRequest) ([]weknora.SearchResult, error)
|
||||
}
|
||||
|
||||
type e2eReport struct {
|
||||
PublicationID string `json:"publication_id"`
|
||||
RemoteStatus string `json:"remote_status"`
|
||||
ChunkTotal int64 `json:"chunk_total"`
|
||||
ChunkItemsRead int64 `json:"chunk_items_read"`
|
||||
AllChunksMatch bool `json:"all_chunks_match"`
|
||||
SearchResultCount int `json:"search_result_count"`
|
||||
TargetSearchHits int `json:"target_search_hits"`
|
||||
MarkerFound bool `json:"marker_found"`
|
||||
MaxScore float64 `json:"max_score"`
|
||||
StructuralPassed bool `json:"structural_passed"`
|
||||
SemanticState string `json:"semantic_state"`
|
||||
}
|
||||
|
||||
func main() {
|
||||
if err := run(context.Background(), os.Args[1:], os.Getenv, os.Stdout); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func run(ctx context.Context, args []string, getenv func(string) string, output io.Writer) error {
|
||||
flags := flag.NewFlagSet("test-weknora-e2e-report", flag.ContinueOnError)
|
||||
flags.SetOutput(io.Discard)
|
||||
publicationID := flags.String("publication-id", "", "governance publication ID")
|
||||
query := flags.String("query", "", "synthetic scoped retrieval query")
|
||||
expectedMarker := flags.String("expected-marker", "", "synthetic marker expected in a scoped result")
|
||||
if err := flags.Parse(args); err != nil {
|
||||
return errors.New("invalid report arguments")
|
||||
}
|
||||
if *publicationID == "" || *query == "" || *expectedMarker == "" {
|
||||
return errors.New("--publication-id, --query and --expected-marker are required")
|
||||
}
|
||||
databaseURL := getenv("DATABASE_URL")
|
||||
baseURL := getenv("TEST_WEKNORA_BASE_URL")
|
||||
keyFile := getenv("TEST_WEKNORA_API_KEY_FILE")
|
||||
if databaseURL == "" || baseURL == "" || keyFile == "" {
|
||||
return errors.New("database and test WeKnora configuration are required")
|
||||
}
|
||||
allowDockerHTTP := false
|
||||
if raw := getenv("TEST_WEKNORA_ALLOW_DOCKER_HTTP"); raw != "" {
|
||||
parsed, err := strconv.ParseBool(raw)
|
||||
if err != nil {
|
||||
return errors.New("invalid Docker HTTP setting")
|
||||
}
|
||||
allowDockerHTTP = parsed
|
||||
}
|
||||
apiKey, err := weknora.ReadAPIKeyFile(keyFile)
|
||||
if err != nil {
|
||||
return errors.New("read test WeKnora API key")
|
||||
}
|
||||
client, err := weknora.NewClient(weknora.ClientConfig{
|
||||
BaseURL: baseURL,
|
||||
APIKey: apiKey,
|
||||
Timeout: 60 * time.Second,
|
||||
AllowDockerHTTP: allowDockerHTTP,
|
||||
})
|
||||
if err != nil {
|
||||
return errors.New("create test WeKnora client")
|
||||
}
|
||||
pool, err := pgxpool.New(ctx, databaseURL)
|
||||
if err != nil {
|
||||
return errors.New("open governance database")
|
||||
}
|
||||
defer pool.Close()
|
||||
if err := pool.Ping(ctx); err != nil {
|
||||
return errors.New("connect governance database")
|
||||
}
|
||||
report, err := verifyPublication(ctx, postgresstore.NewStore(pool), client, *publicationID, *query, *expectedMarker)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return json.NewEncoder(output).Encode(report)
|
||||
}
|
||||
|
||||
func verifyPublication(ctx context.Context, store reportStore, client reportClient, publicationID, query, expectedMarker string) (e2eReport, error) {
|
||||
publication, err := store.LoadPublicationForValidation(ctx, publicationID)
|
||||
if err != nil {
|
||||
return e2eReport{}, errors.New("load completed publication")
|
||||
}
|
||||
knowledge, err := client.GetKnowledge(ctx, publication.KnowledgeID)
|
||||
if err != nil || knowledge.ID != publication.KnowledgeID || knowledge.KnowledgeBaseID != publication.TestKBID {
|
||||
return e2eReport{}, errors.New("verify remote knowledge identity")
|
||||
}
|
||||
report := e2eReport{
|
||||
PublicationID: publication.ID,
|
||||
RemoteStatus: knowledge.ParseStatus,
|
||||
AllChunksMatch: true,
|
||||
}
|
||||
for page := 1; page <= 100; page++ {
|
||||
chunks, total, err := client.ListChunks(ctx, publication.KnowledgeID, page, 100, []string{"text", "image_ocr"})
|
||||
if err != nil {
|
||||
return e2eReport{}, errors.New("read remote chunks")
|
||||
}
|
||||
report.ChunkTotal = total
|
||||
for _, chunk := range chunks {
|
||||
report.ChunkItemsRead++
|
||||
if chunk.KnowledgeID != publication.KnowledgeID {
|
||||
report.AllChunksMatch = false
|
||||
}
|
||||
}
|
||||
if report.ChunkItemsRead >= total || len(chunks) == 0 {
|
||||
break
|
||||
}
|
||||
if page == 100 {
|
||||
return e2eReport{}, errors.New("remote chunk pagination limit exceeded")
|
||||
}
|
||||
}
|
||||
results, err := client.SearchKnowledge(ctx, weknora.SearchRequest{Query: query, KnowledgeIDs: []string{publication.KnowledgeID}})
|
||||
if err != nil {
|
||||
return e2eReport{}, errors.New("run scoped retrieval probe")
|
||||
}
|
||||
report.SearchResultCount = len(results)
|
||||
for _, result := range results {
|
||||
if result.KnowledgeID == publication.KnowledgeID {
|
||||
report.TargetSearchHits++
|
||||
if strings.Contains(result.Content, expectedMarker) {
|
||||
report.MarkerFound = true
|
||||
}
|
||||
}
|
||||
if result.Score > report.MaxScore {
|
||||
report.MaxScore = result.Score
|
||||
}
|
||||
}
|
||||
acceptance, err := store.ListAcceptanceResults(ctx, publication.ID)
|
||||
if err != nil {
|
||||
return e2eReport{}, errors.New("read governance acceptance results")
|
||||
}
|
||||
structural := map[string]bool{"knowledge_exists": false, "chunks_present": false}
|
||||
for _, record := range acceptance {
|
||||
if _, ok := structural[record.CheckType]; ok && record.State == string(governance.AcceptancePassed) {
|
||||
structural[record.CheckType] = true
|
||||
}
|
||||
if record.CheckType == "retrieval_hit" {
|
||||
report.SemanticState = record.State
|
||||
}
|
||||
}
|
||||
report.StructuralPassed = structural["knowledge_exists"] && structural["chunks_present"]
|
||||
if report.RemoteStatus != "completed" || report.ChunkTotal <= 0 || report.ChunkItemsRead != report.ChunkTotal || !report.AllChunksMatch || report.TargetSearchHits == 0 || !report.MarkerFound {
|
||||
return e2eReport{}, errors.New("end-to-end verification failed")
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||||
"knowledge-governance/internal/platform/weknora"
|
||||
)
|
||||
|
||||
func TestVerifyPublicationReturnsSanitizedEvidence(t *testing.T) {
|
||||
store := &reportStoreStub{
|
||||
publication: postgresstore.PublicationForValidation{ID: "tp-1", TestKBID: "kb-1", KnowledgeID: "knowledge-secret-id"},
|
||||
acceptance: []governance.AcceptanceRecord{
|
||||
{CheckType: "knowledge_exists", State: string(governance.AcceptancePassed)},
|
||||
{CheckType: "chunks_present", State: string(governance.AcceptancePassed)},
|
||||
{CheckType: "retrieval_hit", State: string(governance.AcceptanceManualRequired)},
|
||||
},
|
||||
}
|
||||
client := &reportClientStub{
|
||||
knowledge: weknora.Knowledge{ID: "knowledge-secret-id", KnowledgeBaseID: "kb-1", ParseStatus: "completed"},
|
||||
chunks: []weknora.Chunk{
|
||||
{ID: "chunk-1", KnowledgeID: "knowledge-secret-id", ChunkType: "text"},
|
||||
{ID: "chunk-2", KnowledgeID: "knowledge-secret-id", ChunkType: "text"},
|
||||
},
|
||||
search: []weknora.SearchResult{{KnowledgeID: "knowledge-secret-id", Content: "answer contains MIKU-GOV-P3-5927", Score: 0.02}},
|
||||
}
|
||||
|
||||
report, err := verifyPublication(context.Background(), store, client, "tp-1", "unique marker", "MIKU-GOV-P3-5927")
|
||||
if err != nil {
|
||||
t.Fatalf("verifyPublication() error = %v", err)
|
||||
}
|
||||
if report.PublicationID != "tp-1" || report.RemoteStatus != "completed" || report.ChunkTotal != 2 || !report.AllChunksMatch || report.SearchResultCount != 1 || report.TargetSearchHits != 1 || !report.MarkerFound || !report.StructuralPassed || report.SemanticState != "manual_required" {
|
||||
t.Fatalf("report = %#v", report)
|
||||
}
|
||||
encoded, err := json.Marshal(report)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, forbidden := range []string{"knowledge-secret-id", "chunk-1", "unique marker", "MIKU-GOV-P3-5927"} {
|
||||
if string(encoded) == forbidden || contains(string(encoded), forbidden) {
|
||||
t.Fatalf("report leaked %q: %s", forbidden, encoded)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func contains(value, fragment string) bool {
|
||||
for index := 0; index+len(fragment) <= len(value); index++ {
|
||||
if value[index:index+len(fragment)] == fragment {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
type reportStoreStub struct {
|
||||
publication postgresstore.PublicationForValidation
|
||||
acceptance []governance.AcceptanceRecord
|
||||
}
|
||||
|
||||
func (store *reportStoreStub) LoadPublicationForValidation(context.Context, string) (postgresstore.PublicationForValidation, error) {
|
||||
return store.publication, nil
|
||||
}
|
||||
|
||||
func (store *reportStoreStub) ListAcceptanceResults(context.Context, string) ([]governance.AcceptanceRecord, error) {
|
||||
return store.acceptance, nil
|
||||
}
|
||||
|
||||
type reportClientStub struct {
|
||||
knowledge weknora.Knowledge
|
||||
chunks []weknora.Chunk
|
||||
search []weknora.SearchResult
|
||||
}
|
||||
|
||||
func (client *reportClientStub) GetKnowledge(context.Context, string) (weknora.Knowledge, error) {
|
||||
return client.knowledge, nil
|
||||
}
|
||||
|
||||
func (client *reportClientStub) ListChunks(context.Context, string, int, int, []string) ([]weknora.Chunk, int64, error) {
|
||||
return client.chunks, int64(len(client.chunks)), nil
|
||||
}
|
||||
|
||||
func (client *reportClientStub) SearchKnowledge(context.Context, weknora.SearchRequest) ([]weknora.SearchResult, error) {
|
||||
return client.search, nil
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||||
"knowledge-governance/internal/platform/weknora"
|
||||
"knowledge-governance/internal/workflows/testweknora"
|
||||
)
|
||||
|
||||
type probeConfig struct {
|
||||
DatabaseURL string
|
||||
Concurrency int
|
||||
Online bool
|
||||
BaseURL string
|
||||
APIKeyFile string
|
||||
ChunkSampleEvery int
|
||||
}
|
||||
|
||||
func main() {
|
||||
if err := run(context.Background(), os.Args[1:], os.Getenv, os.Stdout); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func run(ctx context.Context, args []string, getenv func(string) string, stdout io.Writer) error {
|
||||
config, err := parseProbeConfig(args, getenv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
pool, err := pgxpool.New(ctx, config.DatabaseURL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open database: %w", err)
|
||||
}
|
||||
defer pool.Close()
|
||||
if err := pool.Ping(ctx); err != nil {
|
||||
return fmt.Errorf("ping database: %w", err)
|
||||
}
|
||||
store := postgresstore.NewStore(pool)
|
||||
legacy, err := store.ListLegacyVerificationCandidates(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
candidates := make([]testweknora.LegacyCandidate, 0, len(legacy))
|
||||
for _, candidate := range legacy {
|
||||
candidates = append(candidates, testweknora.LegacyCandidate{
|
||||
ID: candidate.ID, TestKBID: candidate.TestKBID, KnowledgeID: candidate.KnowledgeID,
|
||||
})
|
||||
}
|
||||
if !config.Online {
|
||||
_, _ = io.WriteString(stdout, testweknora.FormatDryRunReport(candidates, config.Concurrency))
|
||||
return nil
|
||||
}
|
||||
apiKey, err := weknora.ReadAPIKeyFile(config.APIKeyFile)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read test WeKnora API key: %w", err)
|
||||
}
|
||||
client, err := weknora.NewClient(weknora.ClientConfig{
|
||||
BaseURL: config.BaseURL,
|
||||
APIKey: apiKey,
|
||||
Timeout: 30 * time.Second,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("create test WeKnora client: %w", err)
|
||||
}
|
||||
report, err := testweknora.VerifyLegacyConcurrent(ctx, client, candidates, config.ChunkSampleEvery, config.Concurrency)
|
||||
if err != nil {
|
||||
return fmt.Errorf("verify historical test publications: %w", err)
|
||||
}
|
||||
_, _ = io.WriteString(stdout, testweknora.FormatLegacyVerificationReport(report))
|
||||
return nil
|
||||
}
|
||||
|
||||
func parseProbeConfig(args []string, getenv func(string) string) (probeConfig, error) {
|
||||
flags := flag.NewFlagSet("test-weknora-probe", flag.ContinueOnError)
|
||||
flags.SetOutput(io.Discard)
|
||||
databaseURL := flags.String("database-url", "", "PostgreSQL connection URL (or DATABASE_URL)")
|
||||
concurrency := flags.Int("concurrency", 4, "planned read-only verify concurrency for the online phase")
|
||||
online := flags.Bool("online", false, "run authenticated read-only verification")
|
||||
baseURL := flags.String("base-url", "", "HTTPS WeKnora base URL")
|
||||
apiKeyFile := flags.String("api-key-file", "", "mode-0600 WeKnora API key file")
|
||||
chunkSampleEvery := flags.Int("chunk-sample-every", 25, "check one completed Knowledge chunk count every N items; 0 disables")
|
||||
if err := flags.Parse(args); err != nil {
|
||||
return probeConfig{}, fmt.Errorf("parse arguments: %w", err)
|
||||
}
|
||||
if *databaseURL == "" {
|
||||
*databaseURL = getenv("DATABASE_URL")
|
||||
}
|
||||
if *databaseURL == "" {
|
||||
return probeConfig{}, fmt.Errorf("-database-url or DATABASE_URL is required")
|
||||
}
|
||||
if *concurrency < 1 || *concurrency > 16 || *chunkSampleEvery < 0 {
|
||||
return probeConfig{}, fmt.Errorf("concurrency must be 1..16 and chunk-sample-every must be non-negative")
|
||||
}
|
||||
if *baseURL == "" {
|
||||
*baseURL = getenv("TEST_WEKNORA_BASE_URL")
|
||||
}
|
||||
if *apiKeyFile == "" {
|
||||
*apiKeyFile = getenv("TEST_WEKNORA_API_KEY_FILE")
|
||||
}
|
||||
if *online {
|
||||
parsed, err := url.Parse(strings.TrimSpace(*baseURL))
|
||||
if err != nil || parsed.Scheme != "https" || parsed.Host == "" || parsed.User != nil {
|
||||
return probeConfig{}, fmt.Errorf("online verification requires an HTTPS base-url")
|
||||
}
|
||||
if !filepath.IsAbs(*apiKeyFile) {
|
||||
return probeConfig{}, fmt.Errorf("online verification requires an absolute api-key-file")
|
||||
}
|
||||
}
|
||||
return probeConfig{
|
||||
DatabaseURL: *databaseURL,
|
||||
Concurrency: *concurrency,
|
||||
Online: *online,
|
||||
BaseURL: strings.TrimRight(strings.TrimSpace(*baseURL), "/"),
|
||||
APIKeyFile: *apiKeyFile,
|
||||
ChunkSampleEvery: *chunkSampleEvery,
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestRunRequiresDatabaseURL(t *testing.T) {
|
||||
err := run(context.Background(), nil, func(string) string { return "" }, io.Discard)
|
||||
if err == nil || !strings.Contains(err.Error(), "database-url") {
|
||||
t.Fatalf("run() error = %v, want database-url requirement", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseProbeConfigSupportsExplicitOnlineMode(t *testing.T) {
|
||||
config, err := parseProbeConfig([]string{
|
||||
"--online",
|
||||
"--base-url", "https://weknora-test.example",
|
||||
"--api-key-file", "/run/secrets/test_weknora_api_key",
|
||||
"--concurrency", "4",
|
||||
"--chunk-sample-every", "25",
|
||||
}, func(key string) string {
|
||||
if key == "DATABASE_URL" {
|
||||
return "postgres://governance@example/governance"
|
||||
}
|
||||
return ""
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("parseProbeConfig() error = %v", err)
|
||||
}
|
||||
if !config.Online || config.DatabaseURL == "" || config.BaseURL != "https://weknora-test.example" ||
|
||||
config.APIKeyFile != "/run/secrets/test_weknora_api_key" || config.Concurrency != 4 || config.ChunkSampleEvery != 25 {
|
||||
t.Fatalf("config = %#v", config)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,156 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"knowledge-governance/internal/appconfig"
|
||||
"knowledge-governance/internal/jobs"
|
||||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||||
"knowledge-governance/internal/platform/weknora"
|
||||
catalogrefresh "knowledge-governance/internal/workflows/catalogrefresh"
|
||||
"knowledge-governance/internal/workflows/sourcescan"
|
||||
"knowledge-governance/internal/workflows/testweknora"
|
||||
)
|
||||
|
||||
func main() {
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
if err := run(ctx, os.Getenv); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func run(ctx context.Context, getenv func(string) string) error {
|
||||
config, err := appconfig.Load(getenv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pool, err := pgxpool.New(ctx, config.DatabaseURL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer pool.Close()
|
||||
if err := pool.Ping(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := postgresstore.ApplyMigrations(ctx, pool); err != nil {
|
||||
return err
|
||||
}
|
||||
store := postgresstore.NewStore(pool)
|
||||
handlers, err := workerHandlers(store, config)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
worker, err := jobs.NewWorker(jobs.WorkerConfig{
|
||||
Queue: store,
|
||||
WorkerID: config.WorkerID,
|
||||
LeaseDuration: config.LeaseDuration,
|
||||
RetryDelay: config.RetryDelay,
|
||||
Handlers: handlers,
|
||||
OnError: func(err error) { log.Printf("worker error: %v", err) },
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
go runSourceRefreshScheduler(ctx, store, config.SourceRefreshSchedulerInterval, func(err error) {
|
||||
log.Printf("worker %s source refresh scheduler error: %v", config.WorkerID, err)
|
||||
})
|
||||
log.Printf("worker %s started (source refresh scheduler every %s)", config.WorkerID, config.SourceRefreshSchedulerInterval)
|
||||
return worker.Run(ctx, config.PollInterval)
|
||||
}
|
||||
|
||||
func workerHandlers(store *postgresstore.Store, config appconfig.Config) (map[string]jobs.Handler, error) {
|
||||
handlers := map[string]jobs.Handler{
|
||||
"demo": func(_ context.Context, job jobs.Job) error {
|
||||
log.Printf("completed demo job id=%s", job.ID)
|
||||
return nil
|
||||
},
|
||||
"demo.sleep": jobs.DemoSleepHandler,
|
||||
}
|
||||
catalogRunner, err := catalogrefresh.NewCommandRunner("/usr/bin/python3", "/app/scripts/live_catalog_refresh.py")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create catalog refresh runner: %w", err)
|
||||
}
|
||||
catalogHandler, err := catalogrefresh.NewHandler(catalogrefresh.Config{Store: store, Runner: catalogRunner})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create catalog refresh handler: %w", err)
|
||||
}
|
||||
handlers["source.catalog.refresh"] = catalogHandler.Handle
|
||||
if config.PPScanEnabled {
|
||||
scanner, err := sourcescan.NewFilePPScanner(sourcescan.FilePPScannerConfig{
|
||||
AllowedRoot: config.PPScanAllowedRoot,
|
||||
LedgerPath: config.PPScanLedgerPath,
|
||||
ManifestPath: config.PPScanManifestPath,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create PP source scanner: %w", err)
|
||||
}
|
||||
handler, err := sourcescan.NewPPHandler(sourcescan.PPHandlerConfig{Store: store, Scanner: scanner})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create PP source scan handler: %w", err)
|
||||
}
|
||||
handlers["source.scan.pp"] = handler.Handle
|
||||
}
|
||||
if !config.TestWeKnoraEnabled {
|
||||
return handlers, nil
|
||||
}
|
||||
apiKey, err := weknora.ReadAPIKeyFile(config.TestWeKnoraAPIKeyFile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read test WeKnora API key: %w", err)
|
||||
}
|
||||
client, err := weknora.NewClient(weknora.ClientConfig{
|
||||
BaseURL: config.TestWeKnoraBaseURL,
|
||||
APIKey: apiKey,
|
||||
Timeout: config.TestWeKnoraHTTPTimeout,
|
||||
AllowDockerHTTP: config.TestWeKnoraAllowDockerHTTP,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create test WeKnora client: %w", err)
|
||||
}
|
||||
upload, err := testweknora.NewUploadHandler(testweknora.UploadHandlerConfig{
|
||||
Store: store,
|
||||
Client: client,
|
||||
SpoolRoot: config.GovernanceSpoolRoot,
|
||||
MaxUploadBytes: config.TestWeKnoraMaxUploadBytes,
|
||||
FirstPollDelay: config.TestWeKnoraPollInitial,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create test upload handler: %w", err)
|
||||
}
|
||||
poll, err := testweknora.NewPollHandler(testweknora.PollHandlerConfig{
|
||||
Store: store,
|
||||
Client: client,
|
||||
PollDelay: config.TestWeKnoraPollInitial,
|
||||
StallAfter: config.TestWeKnoraStalledAfter,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create test poll handler: %w", err)
|
||||
}
|
||||
validate, err := testweknora.NewValidateHandler(testweknora.ValidateHandlerConfig{
|
||||
Store: store,
|
||||
Client: client,
|
||||
PolicyVersion: config.TestAcceptancePolicyVersion,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create test validate handler: %w", err)
|
||||
}
|
||||
remove, err := testweknora.NewRemoveHandler(testweknora.RemoveHandlerConfig{
|
||||
Store: store,
|
||||
Client: client,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create test removal handler: %w", err)
|
||||
}
|
||||
handlers["test.upload"] = upload.Handle
|
||||
handlers["test.poll"] = poll.Handle
|
||||
handlers["test.validate"] = validate.Handle
|
||||
handlers["test.remove"] = remove.Handle
|
||||
return handlers, nil
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"knowledge-governance/internal/appconfig"
|
||||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||||
)
|
||||
|
||||
func TestRunRequiresDatabaseURL(t *testing.T) {
|
||||
err := run(context.Background(), func(string) string { return "" })
|
||||
if err == nil || !strings.Contains(err.Error(), "DATABASE_URL") {
|
||||
t.Fatalf("run() error = %v, want DATABASE_URL error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerHandlersExcludeReleaseAndTestJobsByDefault(t *testing.T) {
|
||||
handlers, err := workerHandlers(nil, appconfig.Config{})
|
||||
if err != nil {
|
||||
t.Fatalf("workerHandlers() error = %v", err)
|
||||
}
|
||||
if handlers["demo"] == nil || handlers["demo.sleep"] == nil {
|
||||
t.Fatalf("worker handlers = %#v, want demo and demo.sleep", handlers)
|
||||
}
|
||||
if handlers["release.probe"] != nil {
|
||||
t.Fatal("ordinary worker must not register release.probe")
|
||||
}
|
||||
if handlers["test.upload"] != nil || handlers["test.poll"] != nil || handlers["test.validate"] != nil || handlers["test.remove"] != nil {
|
||||
t.Fatal("ordinary worker must not register test WeKnora jobs when disabled")
|
||||
}
|
||||
if handlers["source.scan.pp"] != nil {
|
||||
t.Fatal("ordinary worker must not register PP scan jobs when disabled")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerHandlersRegisterPPScanWhenExplicitlyEnabled(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
config := appconfig.Config{
|
||||
PPScanEnabled: true,
|
||||
PPScanAllowedRoot: root,
|
||||
PPScanLedgerPath: filepath.Join(root, "state.jsonl"),
|
||||
PPScanManifestPath: filepath.Join(root, "manifest.json"),
|
||||
}
|
||||
handlers, err := workerHandlers(postgresstore.NewStore(nil), config)
|
||||
if err != nil {
|
||||
t.Fatalf("workerHandlers() error = %v", err)
|
||||
}
|
||||
if handlers["source.scan.pp"] == nil {
|
||||
t.Fatal("source.scan.pp handler is not registered")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerHandlersRequireAPIKeyWhenTestWeKnoraEnabled(t *testing.T) {
|
||||
config := appconfig.Config{TestWeKnoraEnabled: true, TestWeKnoraAPIKeyFile: "/nonexistent/api-key"}
|
||||
_, err := workerHandlers(nil, config)
|
||||
if err == nil || !strings.Contains(err.Error(), "read test WeKnora API key") {
|
||||
t.Fatalf("workerHandlers() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerHandlersRegisterTestJobsForExplicitDockerHTTP(t *testing.T) {
|
||||
keyFile := filepath.Join(t.TempDir(), "api-key")
|
||||
if err := os.WriteFile(keyFile, []byte("local-test-key\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
config := appconfig.Config{
|
||||
TestWeKnoraEnabled: true,
|
||||
TestWeKnoraAllowDockerHTTP: true,
|
||||
TestWeKnoraBaseURL: "http://app:8080",
|
||||
TestWeKnoraAPIKeyFile: keyFile,
|
||||
GovernanceSpoolRoot: t.TempDir(),
|
||||
TestWeKnoraHTTPTimeout: time.Second,
|
||||
TestWeKnoraPollInitial: time.Second,
|
||||
TestWeKnoraStalledAfter: time.Minute,
|
||||
TestWeKnoraMaxUploadBytes: 1024,
|
||||
TestAcceptancePolicyVersion: "structural-v1",
|
||||
}
|
||||
handlers, err := workerHandlers(postgresstore.NewStore(nil), config)
|
||||
if err != nil {
|
||||
t.Fatalf("workerHandlers() error = %v", err)
|
||||
}
|
||||
for _, jobType := range []string{"test.upload", "test.poll", "test.validate", "test.remove"} {
|
||||
if handlers[jobType] == nil {
|
||||
t.Fatalf("handler %q is not registered", jobType)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log"
|
||||
"time"
|
||||
)
|
||||
|
||||
// dueSourceClaimer is the storage slice the scheduler needs; *postgresstore.Store
|
||||
// satisfies it and tests can fake it without a database.
|
||||
type dueSourceClaimer interface {
|
||||
ClaimDueSourceRefreshes(ctx context.Context, now time.Time) (int, int, error)
|
||||
}
|
||||
|
||||
// runSourceRefreshTick performs one dispatcher pass and logs only when work
|
||||
// moved, so an idle system does not spam the worker log.
|
||||
func runSourceRefreshTick(ctx context.Context, claimer dueSourceClaimer, now time.Time) error {
|
||||
claimed, enqueued, err := claimer.ClaimDueSourceRefreshes(ctx, now)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if claimed > 0 || enqueued > 0 {
|
||||
log.Printf("source refresh scheduler claimed=%d enqueued=%d", claimed, enqueued)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// runSourceRefreshScheduler ticks until ctx is cancelled. The FIRST tick is
|
||||
// deliberately delayed by one interval: startup already races ApplyMigrations
|
||||
// and handler registration, and a due backlog enqueue adds nothing on the
|
||||
// second the process comes up (the worker will drain whatever is queued anyway
|
||||
// once sources were scheduled before a restart).
|
||||
func runSourceRefreshScheduler(ctx context.Context, claimer dueSourceClaimer, interval time.Duration, onError func(error)) {
|
||||
if claimer == nil || interval <= 0 {
|
||||
return
|
||||
}
|
||||
if onError == nil {
|
||||
onError = func(err error) { log.Printf("source refresh scheduler error: %v", err) }
|
||||
}
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case tick := <-ticker.C:
|
||||
if err := runSourceRefreshTick(ctx, claimer, tick.UTC()); err != nil && !errors.Is(err, context.Canceled) {
|
||||
onError(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type fakeClaimer struct {
|
||||
mu sync.Mutex
|
||||
calls []time.Time
|
||||
claimed [2]int
|
||||
err error
|
||||
}
|
||||
|
||||
func (f *fakeClaimer) ClaimDueSourceRefreshes(_ context.Context, now time.Time) (int, int, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.calls = append(f.calls, now)
|
||||
if f.err != nil {
|
||||
return 0, 0, f.err
|
||||
}
|
||||
return f.claimed[0], f.claimed[1], nil
|
||||
}
|
||||
|
||||
func TestRunSourceRefreshTickSurfacesErrors(t *testing.T) {
|
||||
claim := &fakeClaimer{claimed: [2]int{3, 2}}
|
||||
if err := runSourceRefreshTick(context.Background(), claim, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(claim.calls) != 1 {
|
||||
t.Fatalf("calls=%d want 1", len(claim.calls))
|
||||
}
|
||||
sentinel := errors.New("boom")
|
||||
failing := &fakeClaimer{err: sentinel}
|
||||
if err := runSourceRefreshTick(context.Background(), failing, time.Now()); !errors.Is(err, sentinel) {
|
||||
t.Fatalf("error=%v want sentinel", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSourceRefreshSchedulerTicksAndStops(t *testing.T) {
|
||||
claim := &fakeClaimer{}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
runSourceRefreshScheduler(ctx, claim, 10*time.Millisecond, nil)
|
||||
}()
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
claim.mu.Lock()
|
||||
count := len(claim.calls)
|
||||
claim.mu.Unlock()
|
||||
if count >= 2 {
|
||||
break
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
cancel()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("scheduler did not stop on context cancel")
|
||||
}
|
||||
claim.mu.Lock()
|
||||
defer claim.mu.Unlock()
|
||||
if len(claim.calls) < 2 {
|
||||
t.Fatalf("calls=%d want at least 2", len(claim.calls))
|
||||
}
|
||||
for _, stamp := range claim.calls {
|
||||
if stamp.Location() != time.UTC {
|
||||
t.Fatalf("tick stamp %v not UTC", stamp)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSourceRefreshSchedulerSurvivesTickErrors(t *testing.T) {
|
||||
claim := &fakeClaimer{err: errors.New("database is offline")}
|
||||
var errorsSeen int
|
||||
var mu sync.Mutex
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
runSourceRefreshScheduler(ctx, claim, 10*time.Millisecond, func(error) {
|
||||
mu.Lock()
|
||||
errorsSeen++
|
||||
mu.Unlock()
|
||||
})
|
||||
}()
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
mu.Lock()
|
||||
seen := errorsSeen
|
||||
mu.Unlock()
|
||||
if seen >= 2 {
|
||||
break
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
cancel()
|
||||
<-done
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if errorsSeen < 2 {
|
||||
t.Fatalf("errorsSeen=%d want >=2 (loop must not die on failures)", errorsSeen)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSourceRefreshSchedulerIgnoresNilClaims(t *testing.T) {
|
||||
// guards against misconfiguration killing the process
|
||||
runSourceRefreshScheduler(context.Background(), nil, time.Second, nil)
|
||||
runSourceRefreshScheduler(context.Background(), &fakeClaimer{}, 0, nil)
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
# 本地模拟生产发布专用 override(target=测试服务器 weknora-test-v071)
|
||||
# 用法:docker compose -f compose.yaml -f compose.release-sim.yaml up -d
|
||||
# 真实生产严禁使用本文件(真实生产 = 未来独立授权 + 独立配置)
|
||||
services:
|
||||
api:
|
||||
environment:
|
||||
PRODUCTION_RELEASE_REQUESTS_ENABLED: "true"
|
||||
PRODUCTION_BACKUP_RESTORE_VERIFIED: "true"
|
||||
PRODUCTION_RELEASE_AUTHORIZER_REF: yaozhanghao
|
||||
PRODUCTION_RELEASE_AUTH_TOKEN_FILE: /run/release-secrets/production_release_auth_token
|
||||
volumes:
|
||||
- ./secrets/release-sim:/run/release-secrets:ro
|
||||
|
||||
release-worker:
|
||||
environment:
|
||||
RELEASE_WORKER_ENABLED: "true"
|
||||
RELEASE_CONFIG_FILE: /run/release-sim/release.sim.json
|
||||
RELEASE_DEBUG_ITEM_ERRORS: "true"
|
||||
PRODUCTION_RELEASE_AUTHORIZER_REF: yaozhanghao
|
||||
extra_hosts:
|
||||
- "host.docker.internal:host-gateway"
|
||||
- "prod-pg:host-gateway"
|
||||
- "prod-es:host-gateway"
|
||||
- "prod-weknora:host-gateway"
|
||||
networks:
|
||||
- default
|
||||
- weknora-local
|
||||
volumes:
|
||||
- ./secrets/release-sim:/run/release-secrets:ro
|
||||
- ./config/release.sim.json:/run/release-sim/release.sim.json:ro
|
||||
- release-bundles:/var/lib/knowledge-governance/release-bundles
|
||||
read_only: false
|
||||
|
||||
volumes:
|
||||
release-bundles:
|
||||
name: knowledge-governance-release-bundles
|
||||
+164
@@ -0,0 +1,164 @@
|
||||
name: knowledge-governance-p1
|
||||
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:17-alpine
|
||||
environment:
|
||||
POSTGRES_DB: governance
|
||||
POSTGRES_USER: governance
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
|
||||
secrets:
|
||||
- postgres_password
|
||||
volumes:
|
||||
- governance-pgdata:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U governance -d governance"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
restart: unless-stopped
|
||||
|
||||
api:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
command: ["/app/api"]
|
||||
user: "${APP_UID:-1000}:${APP_GID:-1000}"
|
||||
environment:
|
||||
<<: &app-environment
|
||||
DATABASE_URL: postgres://governance@postgres:5432/governance?sslmode=disable
|
||||
PGPASSFILE: /run/secrets/pgpass
|
||||
APP_ADDR: :8080
|
||||
WORKER_POLL_INTERVAL: 1s
|
||||
WORKER_LEASE_DURATION: 30s
|
||||
WORKER_RETRY_DELAY: 5s
|
||||
TEST_WEKNORA_BASE_URL: "${TEST_WEKNORA_BASE_URL:-}"
|
||||
MINERU_BASE_URL: "${MINERU_BASE_URL:-}"
|
||||
TEST_INTERNAL_WEKNORA_KB_ID: "${TEST_INTERNAL_WEKNORA_KB_ID:-1b52c140-2e18-4676-afc5-4c7d8d190a8c}"
|
||||
TEST_EXTERNAL_WEKNORA_KB_ID: "${TEST_EXTERNAL_WEKNORA_KB_ID:-fd7db6aa-763a-46f1-84d2-ecc7c952a5ea}"
|
||||
# P10 readiness probe reads the routing KB with the same scoped key.
|
||||
TEST_WEKNORA_API_KEY_FILE: /run/secrets/test_weknora_api_key
|
||||
TEST_WEKNORA_ALLOW_DOCKER_HTTP: "${TEST_WEKNORA_ALLOW_DOCKER_HTTP:-false}"
|
||||
WORKER_LIVENESS_STALE_AFTER: "${WORKER_LIVENESS_STALE_AFTER:-2m}"
|
||||
PRODUCTION_RELEASE_REQUESTS_ENABLED: ${PRODUCTION_RELEASE_REQUESTS_ENABLED:-false}
|
||||
PRODUCTION_BACKUP_RESTORE_VERIFIED: ${PRODUCTION_BACKUP_RESTORE_VERIFIED:-false}
|
||||
secrets:
|
||||
- pgpass
|
||||
- test_weknora_api_key
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
# P10: liveness only — a third-party outage must never restart the API.
|
||||
test: ["CMD", "wget", "-qO-", "http://127.0.0.1:8080/livez"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
read_only: true
|
||||
tmpfs:
|
||||
- /tmp
|
||||
networks:
|
||||
- default
|
||||
- weknora-local
|
||||
restart: unless-stopped
|
||||
|
||||
worker:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
command: ["/app/worker"]
|
||||
user: "${APP_UID:-1000}:${APP_GID:-1000}"
|
||||
environment:
|
||||
<<: *app-environment
|
||||
WORKER_ID: worker-1
|
||||
PP_SCAN_ENABLED: "${PP_SCAN_ENABLED:-false}"
|
||||
PP_SCAN_ALLOWED_ROOT: "${PP_SCAN_CONTAINER_DATA_ROOT:-/home/zheromiku/dobot-portal-sync/data}"
|
||||
PP_SCAN_LEDGER_PATH: "${PP_SCAN_CONTAINER_DATA_ROOT:-/home/zheromiku/dobot-portal-sync/data}/public-weknora-sync/state.jsonl"
|
||||
PP_SCAN_MANIFEST_PATH: "${PP_SCAN_CONTAINER_DATA_ROOT:-/home/zheromiku/dobot-portal-sync/data}/public-documents/manifest.json"
|
||||
TEST_WEKNORA_ENABLED: "${TEST_WEKNORA_ENABLED:-false}"
|
||||
TEST_WEKNORA_ALLOW_DOCKER_HTTP: "${TEST_WEKNORA_ALLOW_DOCKER_HTTP:-false}"
|
||||
TEST_WEKNORA_KB_ID: "${TEST_WEKNORA_KB_ID:-}"
|
||||
TEST_WEKNORA_API_KEY_FILE: /run/secrets/test_weknora_api_key
|
||||
GOVERNANCE_SPOOL_ROOT: /var/lib/governance/spool
|
||||
TEST_WEKNORA_HTTP_TIMEOUT: "${TEST_WEKNORA_HTTP_TIMEOUT:-60s}"
|
||||
TEST_WEKNORA_POLL_INITIAL: "${TEST_WEKNORA_POLL_INITIAL:-10s}"
|
||||
TEST_WEKNORA_POLL_MAX: "${TEST_WEKNORA_POLL_MAX:-60s}"
|
||||
TEST_WEKNORA_STALLED_AFTER: "${TEST_WEKNORA_STALLED_AFTER:-30m}"
|
||||
TEST_WEKNORA_MAX_UPLOAD_BYTES: "${TEST_WEKNORA_MAX_UPLOAD_BYTES:-52428800}"
|
||||
TEST_ACCEPTANCE_POLICY_VERSION: "${TEST_ACCEPTANCE_POLICY_VERSION:-structural-v1}"
|
||||
secrets:
|
||||
- pgpass
|
||||
- test_weknora_api_key
|
||||
volumes:
|
||||
- ./spool:/var/lib/governance/spool:ro
|
||||
- "${PP_SCAN_HOST_DATA_ROOT:-/home/zheromiku/dobot-portal-sync/data}:${PP_SCAN_CONTAINER_DATA_ROOT:-/home/zheromiku/dobot-portal-sync/data}:ro"
|
||||
- /home/zheromiku/dingtalk_kb_download:/sources/dingtalk:ro
|
||||
- /home/zheromiku/tmp/alist:/sources/alist:ro
|
||||
- /home/zheromiku/dobot-portal-sync:/sources/pp:ro
|
||||
- ./secrets/dingtalk_capture.json:/run/source-secrets/dingtalk_capture.json:ro
|
||||
- ./secrets/alist_connections.xlsx:/run/source-secrets/alist_connections.xlsx:ro
|
||||
- ./secrets/alist_api_token:/run/source-secrets/alist_api_token:ro
|
||||
- ./secrets/pp_credentials.json:/run/source-secrets/pp_credentials.json:ro
|
||||
networks:
|
||||
- default
|
||||
- weknora-local
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
read_only: true
|
||||
tmpfs:
|
||||
- /tmp
|
||||
restart: unless-stopped
|
||||
|
||||
release-worker:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
command: ["/app/release-worker"]
|
||||
user: "${APP_UID:-1000}:${APP_GID:-1000}"
|
||||
environment:
|
||||
<<: *app-environment
|
||||
WORKER_ID: release-worker-1
|
||||
RELEASE_WORKER_ENABLED: "false"
|
||||
RELEASE_CONFIG_FILE: ""
|
||||
secrets:
|
||||
- pgpass
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
read_only: true
|
||||
tmpfs:
|
||||
- /tmp
|
||||
restart: unless-stopped
|
||||
|
||||
web:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile.web
|
||||
ports:
|
||||
- "8088:8080"
|
||||
depends_on:
|
||||
api:
|
||||
condition: service_healthy
|
||||
read_only: true
|
||||
tmpfs:
|
||||
- /tmp
|
||||
- /var/cache/nginx
|
||||
- /var/run
|
||||
restart: unless-stopped
|
||||
|
||||
secrets:
|
||||
postgres_password:
|
||||
file: ./secrets/postgres_password
|
||||
pgpass:
|
||||
file: ./secrets/pgpass
|
||||
test_weknora_api_key:
|
||||
file: ./secrets/test_weknora_api_key
|
||||
|
||||
volumes:
|
||||
governance-pgdata:
|
||||
|
||||
networks:
|
||||
weknora-local:
|
||||
external: true
|
||||
name: weknora071full-network
|
||||
@@ -0,0 +1,46 @@
|
||||
{
|
||||
"archive_root": "/var/lib/knowledge-governance/release-bundles",
|
||||
"routes": [
|
||||
{
|
||||
"tenant_id": 10000,
|
||||
"knowledge_base_id": "c12ff4d5-dd87-423b-ab0f-af745b9d54a7",
|
||||
"embedding_model": "text-embedding-v4",
|
||||
"source": {
|
||||
"database_url_file": "/run/release-secrets/source_database_url",
|
||||
"elasticsearch": {
|
||||
"base_url": "http://elasticsearch:9200",
|
||||
"index": "weknora_full_chunks"
|
||||
},
|
||||
"object": {
|
||||
"provider": "oss",
|
||||
"endpoint": "https://oss-cn-shenzhen.aliyuncs.com",
|
||||
"region": "cn-shenzhen",
|
||||
"bucket": "yzh-weknora-test",
|
||||
"access_key_file": "/run/release-secrets/source_object_access_key",
|
||||
"secret_key_file": "/run/release-secrets/source_object_secret_key"
|
||||
}
|
||||
},
|
||||
"target": {
|
||||
"database_url_file": "/run/release-secrets/target_database_url",
|
||||
"elasticsearch": {
|
||||
"base_url": "http://prod-es:29200",
|
||||
"index": "weknora_test_chunks"
|
||||
},
|
||||
"object": {
|
||||
"provider": "oss",
|
||||
"endpoint": "https://oss-cn-shenzhen.aliyuncs.com",
|
||||
"region": "cn-shenzhen",
|
||||
"bucket": "dobot-weknora-test",
|
||||
"storage_backend_id": "6129cfd1-a26c-41dd-88cb-e6a800daa3bb",
|
||||
"access_key_file": "/run/release-secrets/target_object_access_key",
|
||||
"secret_key_file": "/run/release-secrets/target_object_secret_key"
|
||||
},
|
||||
"weknora": {
|
||||
"base_url": "http://prod-weknora:28080",
|
||||
"api_key_file": "/run/release-secrets/target_weknora_api_key",
|
||||
"allow_docker_http": true
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
{
|
||||
"archive_root": "/var/lib/knowledge-governance/release-bundles",
|
||||
"routes": [
|
||||
{
|
||||
"tenant_id": 10000,
|
||||
"knowledge_base_id": "replace-with-identical-test-and-production-kb-id",
|
||||
"embedding_model": "bge-m3",
|
||||
"source": {
|
||||
"database_url_file": "/run/release-secrets/source_database_url",
|
||||
"elasticsearch": {
|
||||
"base_url": "http://source-elasticsearch:9200",
|
||||
"index": "chunks",
|
||||
"token_file": "/run/release-secrets/source_elasticsearch_token"
|
||||
},
|
||||
"object": {
|
||||
"provider": "minio",
|
||||
"endpoint": "http://source-object-storage:9000",
|
||||
"region": "us-east-1",
|
||||
"bucket": "source-weknora",
|
||||
"access_key_file": "/run/release-secrets/source_object_access_key",
|
||||
"secret_key_file": "/run/release-secrets/source_object_secret_key"
|
||||
}
|
||||
},
|
||||
"target": {
|
||||
"database_url_file": "/run/release-secrets/target_database_url",
|
||||
"elasticsearch": {
|
||||
"base_url": "https://target-elasticsearch.example",
|
||||
"index": "chunks",
|
||||
"token_file": "/run/release-secrets/target_elasticsearch_token"
|
||||
},
|
||||
"object": {
|
||||
"provider": "oss",
|
||||
"endpoint": "https://target-object-storage.example",
|
||||
"region": "cn-shenzhen",
|
||||
"bucket": "target-weknora",
|
||||
"storage_backend_id": "replace-with-production-storage-backend-id",
|
||||
"access_key_file": "/run/release-secrets/target_object_access_key",
|
||||
"secret_key_file": "/run/release-secrets/target_object_secret_key"
|
||||
},
|
||||
"weknora": {
|
||||
"base_url": "https://production-weknora.example",
|
||||
"api_key_file": "/run/release-secrets/target_weknora_api_key",
|
||||
"allow_docker_http": false
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
server {
|
||||
listen 8080;
|
||||
server_name _;
|
||||
|
||||
root /usr/share/nginx/html;
|
||||
index index.html;
|
||||
|
||||
location /api/ {
|
||||
proxy_pass http://api:8080;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
}
|
||||
|
||||
location = /health {
|
||||
proxy_pass http://api:8080/health;
|
||||
proxy_set_header Host $host;
|
||||
}
|
||||
|
||||
location / {
|
||||
try_files $uri $uri/ /index.html;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
# 工程实施行动清单:第12项至结束验收
|
||||
|
||||
## 范围
|
||||
|
||||
本轮从“来源与目录”的第8项(全局第12项:来源快照差异 ChangeSet)继续到清单结束。来源与目录现有浏览和标注界面保持不变;解析监控、发布管理、系统设置继续完善。生产发布请求与 Release Worker 保持关闭,本验收不代表生产上线。
|
||||
|
||||
## 第12项:来源快照 ChangeSet
|
||||
|
||||
- Migration 15 新增 `source_catalog_change_sets` 与 `source_catalog_change_items`。
|
||||
- 每次快照激活前持久计算 `add / update / move_scope / delete / no_change`。
|
||||
- Migration 16 为现有7个来源回填最新快照差异。
|
||||
- 提供只读安全 API:`GET /api/v1/source-catalog/change-sets/latest`。
|
||||
- 解析监控页展示各来源最新持久 ChangeSet 和变更文件预览,不返回 locator hash、snapshot hash 或凭据。
|
||||
- 真实读回:国内工业新增80、更新3、删除18、未变化4859。
|
||||
|
||||
## 第13–22项:解析链路与监控
|
||||
|
||||
已验证:
|
||||
|
||||
- Spool 文件上传前计算并校验完整 SHA-256、字节数和格式;流式上传后再次校验。
|
||||
- `source_items` 与 `source_item_versions` 分离文件身份和版本;`content_objects.sha256` 全局唯一。
|
||||
- 标注只创建计划;`processing_confirmations` 人工确认后才创建 `test.upload` Job。
|
||||
- `target_scope` 明确对内/对外;禁止导入进入 `test.remove` 并读回确认。
|
||||
- Publication、Job、轮询序列和人工重试均有幂等键或唯一约束。
|
||||
- Worker 支持租约、心跳、退避重试、最大尝试次数和失败终态;页面支持单个/批量重试。
|
||||
- WeKnora 上传、轮询、最终 completed、Knowledge/Chunk 验收已经接通;当前测试编排开关为启用状态。
|
||||
- 解析监控已切换为真实 Provider:主机 CPU、内存、磁盘和可见 GPU;PostgreSQL、任务队列与 WeKnora 健康探测;未配置的 MinerU、OSS、Redis/消息队列明确显示未配置/未部署,不再模拟正常。
|
||||
|
||||
## 第23–28项:发布管理
|
||||
|
||||
已验证:
|
||||
|
||||
- `GET /api/v1/release-plan` 根据来源最终策略、测试终态和最近生产状态生成 add/update/delete/blocked/unchanged。
|
||||
- 发布前 bundle 保存完整 manifest、三通道摘要和 SHA-256;生产恢复验证未通过时授权门禁阻断。
|
||||
- Migration 17 新增 `batch_size/current_batch/total_batches`;发布默认每50项一批并持久显示批次进度。
|
||||
- 一项永久失败后,当前项之后和后续批次停止;未执行项记录 `release_stopped_after_failure`。
|
||||
- 已成功项才进入自动回滚范围;回滚结果区分 restored/rolled_back/failed。
|
||||
- PostgreSQL、Elasticsearch、对象存储完成后均读回验证;不以 HTTP 200 代替一致性验收。
|
||||
- 发布计划、审核、封批、执行历史、逐项结果和固定错误码均可从页面查看。
|
||||
- 修复发布计划安全投影省略空 `publication_id/blocker` 导致前端拒绝整个计划的问题。
|
||||
|
||||
## 第29项:系统设置
|
||||
|
||||
- 来源列表从真实来源配置动态生成,不写死具体来源名称。
|
||||
- Test/Production WeKnora、OSS/备份、刷新与告警参数分区显示。
|
||||
- AList 显示直连 API Token;密钥值不回传、不写任务载荷、不进入页面。
|
||||
- 页面仅允许修改低风险刷新/告警参数;端点、凭据、生产开关继续由管理员受限配置管理。
|
||||
- 系统设置使用真实服务健康结果,不再混用解析监控模拟数据。
|
||||
|
||||
## 第30项:本地故障演练
|
||||
|
||||
通过的自动化场景:
|
||||
|
||||
- 目录标注重复确认不会重复创建上传任务。
|
||||
- SHA/大小/格式不一致时上传前失败;流式过程中文件改变时失败。
|
||||
- 上传完成后进入轮询;WeKnora completed 后创建验收任务,不继续无效轮询。
|
||||
- 禁止导入调用测试删除并读回确认。
|
||||
- Worker 失败按固定延迟重试,达到最大次数进入失败终态。
|
||||
- 105项发布计划分成3批(50/50/5)。
|
||||
- 永久失败立即停止后续项目和批次,并只为已验证项目创建一次自动回滚。
|
||||
- Migration 17→16→17 往返和 Migration 11 历史范围回填均在独立临时数据库通过。
|
||||
- 解析监控、发布管理、系统设置在 1440/1024/768 三种宽度无页面级横向溢出。
|
||||
|
||||
## 当前真实运行状态
|
||||
|
||||
- schema migration:17。
|
||||
- Monitor:`mode=real`、`simulated=false`。
|
||||
- PostgreSQL、任务队列、WeKnora API:真实健康。
|
||||
- Test Publications:537条,均处于 completed。
|
||||
- Release Plan:新增4、阻断614。
|
||||
- Production release executions:0。
|
||||
- `PRODUCTION_RELEASE_REQUESTS_ENABLED=false`。
|
||||
- `RELEASE_WORKER_ENABLED=false`,进程确认未加载生产凭据。
|
||||
|
||||
## 验证命令结果
|
||||
|
||||
- Go:`go test ./... && go vet ./... && go build ./cmd/...` 通过。
|
||||
- Web:22个测试文件、90项测试通过;TypeScript检查和生产构建通过。
|
||||
- PostgreSQL:ChangeSet、标注/确认幂等、发布授权/自动回滚、Migration往返均在一次性临时数据库通过。
|
||||
- Browser:三个剩余页面真实内容加载成功,响应式检查通过。
|
||||
|
||||
## 仍需生产负责人单独确认的外部验收
|
||||
|
||||
生产发布和真实生产回滚没有执行,也不应由本轮开发授权隐含开启。上线前仍需生产负责人提供独立授权,并完成:生产三通道备份恢复验证、代表性小批发布、故障注入和真实回滚读回。当前页面与后端会在该门禁未满足时持续阻断生产操作。
|
||||
@@ -0,0 +1,49 @@
|
||||
# 文档工作台第一版:已交付与未完成边界
|
||||
|
||||
## 打开方式
|
||||
|
||||
本地治理平台:http://100.92.49.78:8088/ (Tailscale 内访问)
|
||||
|
||||
首页“来源与目录”:选择来源/分类,在文件表中搜索、筛选对内/对外和处理状态;点击文件名称打开详情。“解析监控”每15秒刷新当前记录状态;“待处理文件”只列超限、失败、来源异常。“发布管理”保留审核和生产门禁,流程说明默认折叠。“来源连接”显示连接和本机快照检查,历史审计默认折叠。
|
||||
|
||||
文件表区域内滚动,表头固定;不是无限拉长整页。顶部标签保持单例、可关闭、刷新恢复,首页固定。
|
||||
|
||||
## 真实数据与来源边界
|
||||
|
||||
- 治理清单618份:PP历史文档613份,人工隔离验收文档5份。
|
||||
- 618份均有名称;547份能匹配现存PP分类,PP其余66份标记分类待补齐。
|
||||
- 现有快照只证明分类,不证明完整原始目录/父子关系。页面明确标注“PP分类 · 不代表完整原始目录”。
|
||||
- 对外613、对内5,均展示现有记录,不自动改变可见性;解析完成不代表生产已上线。
|
||||
- 已解析535;超限54,列入待处理。其他状态保持后端原值映射,不凑百分比。
|
||||
- 钉钉/AList目录尚未接通,不展示虚构目录;多个来源以连接ID而非来源类型隔离筛选。
|
||||
- PP“检查快照变化”只检查已保存的本机快照,不是实时抓取PP网站。
|
||||
|
||||
## 本轮改动范围
|
||||
|
||||
- SourceAsset API补充安全展示字段(文件名、相对分类/目录、连接、可见性依据、摄取状态、失败码、处理记录和受控重试能力)。拒绝展示绝对主机路径、URL和越界目录。
|
||||
- `scripts/import_display_metadata.py`:读取既有只读manifest和inventory;按已有来源身份匹配,只补空白title/normalized_path。默认事务回滚预览,`--apply`才提交。没有源文件写入、下载、上传、版本/范围/发布变更。重复dry-run更新0行。
|
||||
- 前端完整分页加载、去重及总数校验后建立来源分类清单,不能将第一页冒充全部数据。加载失败不能显示成功的零数量。
|
||||
- 复用后端已存在的失败重试门禁;未对真实文件触发重试。
|
||||
- 发布批次不再要求审核完整候选页,只允许选择已批准且就绪项。生产发布仍独立关闭。
|
||||
|
||||
## 实测证据
|
||||
|
||||
- Web:45/45 tests;TypeScript、Vite build通过。
|
||||
- Go:`GOPROXY=off go test ./...`通过;相关包vet通过。依赖外部配置的集成测试默认仍跳过,不能把全量命令扩称所有远端E2E通过。
|
||||
- 独立PostgreSQL测试库`governance_document_ui_test`:本轮SourceAsset真实查询测试通过;测试库保留,未删除数据。
|
||||
- `scripts/accept_document_workbench.py`:真实Firefox搜索、分类筛选、范围筛选、文件详情、自动刷新、标签恢复、默认折叠说明、列表区域滚动通过;1440/1024/768宽度无页面横向溢出。脚本不执行审核/上传/重试/生产写入。
|
||||
- 结果:`tmp/document-ui-acceptance.json`;截图:`tmp/DOCUMENT_WORKBENCH_FIREFOX.png`;完整清单核对:`tmp/document-ui-catalog.json`。
|
||||
- 本机Web HTTP 200,Tailscale入口HTTP 200;API healthy,API/Web运行镜像匹配、零重启。
|
||||
- 生产请求开关false,Release Worker开关false、配置路径空。
|
||||
- 数据保持source_items/content_objects/versions/test_publications = 618/535/618/535;活动任务0。
|
||||
- 仅重建API/Web,不重建PostgreSQL及两个Worker、不修改WeKnora、不连接生产,不提交或push。
|
||||
- 回退镜像:`knowledge-governance-api:before-document-ui`、`knowledge-governance-web:before-document-ui`。补齐元数据前已生成0600原生数据库备份并检查dump目录。
|
||||
|
||||
## 尚未完成(不得称为全部功能完成)
|
||||
|
||||
1. 来源/目录级对内外规则编辑、继承与单独覆盖、批量设置、规则变更审计和下游权限生效验证。当前范围只可查看/筛选,不是权限管理器。
|
||||
2. 钉钉及AList真实目录接入;PP完整原始层级仍需可靠来源元数据。
|
||||
3. 原文预览、内容差异、逐文件审核核对入口。
|
||||
4. 页面化配置检查计划、目标知识库及通知方式;目前仍由管理员配置。
|
||||
|
||||
以上未完成项是独立实施和验收边界,本轮不以换标题或隐藏技术细节冒充完成。
|
||||
@@ -0,0 +1,47 @@
|
||||
# 文档工作台 V2 验收
|
||||
|
||||
入口:http://100.92.49.78:8088/(Tailscale 内)
|
||||
|
||||
## 已实现
|
||||
|
||||
- 日常首页默认只显示613份PP业务文档;5份隔离验收文档默认隐藏,可手动显示。
|
||||
- 来源/分类 → 文件清单为主操作路径;顶部显示可点击的来源、分类/目录、文件上下文。
|
||||
- 来源与可靠分类可设置对内/对外;规则作用于当前目录及子目录。分类待补齐不允许伪装成独立目录规则。
|
||||
- 单文件可单独覆盖目录规则;清除覆盖后回退至最近目录规则,再回退至来源原始标记。
|
||||
- 每次设置或清除都要求操作人和判断原因,并与当前规则在同一数据库事务中记录审计。
|
||||
- 范围规则只更新治理判断,不修改PP/钉钉/AList源文件,也不自动发布生产。
|
||||
- 解析监控按来源显示:已就绪、独立解析、内容复用、处理中、需处理、等待;不显示虚构百分比。
|
||||
- 文件详情显示取得文件、上传测试库、解析与索引、测试检索四段真实状态,明确原文与版本差异尚未接通。
|
||||
|
||||
## 数据口径
|
||||
|
||||
- 全部记录:618;默认业务清单:613;隐藏验收文档:5。
|
||||
- PP:独立解析530、内容复用29、需处理54、处理中0、等待0;已就绪559/613。
|
||||
- 当前原始范围:对外613、对内5。
|
||||
- Migration 8仅新增`directory_scope_rules`、`source_item_scope_overrides`、`scope_change_events`。
|
||||
|
||||
## 自动化与运行验收
|
||||
|
||||
- Web:12个测试文件、54项测试全部通过;TypeScript和Vite构建通过。
|
||||
- Go:`GOPROXY=off go test ./...`全部通过;范围规则相关包vet通过。
|
||||
- 独立数据库:文件覆盖 → 目录继承 → 来源默认的真实PostgreSQL链路通过。
|
||||
- 真实Firefox:目录规则设置、5份文件生效、单文件覆盖、清除覆盖恢复继承、清除目录恢复来源标记、顶部上下文、处理链、默认隐藏测试文档、1440/1024/768无页面横向溢出均通过。
|
||||
- 三轮浏览器写回验收共留下12条审计事件(目录/文件各3次set和3次clear);当前有效目录规则0、文件覆盖0,没有遗留验收设置。
|
||||
- 最终数据保持:source_items/content_objects/versions/test_publications = 618/535/618/535;活动任务0。
|
||||
- API、PostgreSQL健康;所有服务运行、零重启且镜像匹配;本机和Tailscale入口HTTP 200。
|
||||
- 生产请求开关false;Release Worker开关false、配置路径空。
|
||||
|
||||
证据:
|
||||
|
||||
- `tmp/document-workbench-v2-acceptance.json`
|
||||
- `tmp/DOCUMENT_WORKBENCH_V2_FIREFOX.png`
|
||||
- `tmp/DOCUMENT_MONITOR_V2_FIREFOX.png`
|
||||
- `tmp/backups/pre-scope-rules-20260908T170120.dump`(PostgreSQL custom dump,290185字节,0600,目录可读)
|
||||
- 回退镜像:`knowledge-governance-api:before-scope-v2`、`knowledge-governance-web:before-scope-v2`
|
||||
|
||||
## 仍未伪装成已完成的能力
|
||||
|
||||
- PP目前只有分类快照,不是完整原始目录树;钉钉/AList目录仍未接通。
|
||||
- 原文预览和版本差异仍未接通。
|
||||
- 本轮范围规则会影响后续人工审核时看到的治理范围,但不会绕过审核或自动改变既有生产知识;生产生效仍需独立发布和验证。
|
||||
- 未提交、未push,未连接或写入生产WeKnora。
|
||||
@@ -0,0 +1,293 @@
|
||||
# 知识治理平台领域对象契约
|
||||
|
||||
## 目的
|
||||
|
||||
本文件是知识治理平台当前实现的唯一领域对象映射。它将产品术语映射到现有Go模型、PostgreSQL表和外部边界;后续模块不得另造一套身份、时间或状态语义。
|
||||
|
||||
本文件不把“代码存在”“容器健康”写成已完成能力。尚未存在的对象或关联明确列在 [Current implementation gaps](#current-implementation-gaps)。
|
||||
|
||||
## 统一主链
|
||||
|
||||
```text
|
||||
SourceConnection → SourceItem → SourceItemVersion → ContentObject
|
||||
SourceConnection → SourceCatalogSnapshot → SourceCatalogNode
|
||||
SourceCatalogChangeSet → SourceCatalogChangeItem
|
||||
SourceItemVersion → TestPublication → AcceptanceResult
|
||||
ReleaseBatch → ReleaseExecution → ReleaseItemExecution
|
||||
```
|
||||
|
||||
完整目标主链将在后续工作包补齐为:
|
||||
|
||||
```text
|
||||
SourceConnection
|
||||
→ SourceItem / SourceItemVersion / ContentObject
|
||||
→ SourceCatalogSnapshot
|
||||
→ SourceCatalogChangeSet
|
||||
→ ProcessingExecution / Job
|
||||
→ TestPublication / AcceptanceResult
|
||||
→ ReleaseChangeSet
|
||||
→ ProductionSnapshot
|
||||
→ ReleaseBatch / ReleaseExecution
|
||||
```
|
||||
|
||||
## 全局身份与时间规则
|
||||
|
||||
| 规则 | 契约 |
|
||||
| --- | --- |
|
||||
| ID | 所有主对象使用服务端生成的不透明ID;页面和Job Payload只能传递ID,不能传递受保护来源URL、Token或Cookie。 |
|
||||
| 时间 | `created_at`表示治理对象创建;`updated_at`表示本对象最后治理状态变化;来源侧时间必须单独使用`updated_at_source`或`source_updated_at`。 |
|
||||
| 内容Hash | 只有`content_objects.sha256`是文件内容SHA-256。`canonical_uri_hash`、`source_locator_hash`和`snapshot_sha256`绝不能作为内容Hash展示或用于内容去重。 |
|
||||
| 删除 | 来源不可用不等于来源删除。只有权威来源删除事件或经人工确认的删除ChangeSet才可进入目标移除流程。 |
|
||||
| 目标写入 | 普通治理层不直接写WeKnora数据库;测试写入通过官方API/受控适配器;生产发布由独立Release Worker和门禁处理。 |
|
||||
|
||||
## Source
|
||||
|
||||
**产品含义:**一个独立配置、独立健康状态、独立刷新策略的资料来源。
|
||||
|
||||
**当前实现:**
|
||||
|
||||
| 项目 | 当前映射 |
|
||||
| --- | --- |
|
||||
| Go模型 | `governance.SourceConnection` |
|
||||
| 数据表 | `source_connections` |
|
||||
| 主键 | `source_connections.id` |
|
||||
| 类型 | `source_type`:当前为`pp`、`alist`、`dingtalk`;`manual`是Go层支持的扩展类型。 |
|
||||
| 时间 | `created_at`、`updated_at` |
|
||||
| 可变配置 | `enabled`、`schedule`、`last_health` |
|
||||
| 敏感字段 | `base_url`、`secret_ref`;不得返回前端或日志。 |
|
||||
|
||||
**不可变/关系约束:**
|
||||
|
||||
```text
|
||||
一个SourceConnection可以有多个SourceItem。
|
||||
一个SourceConnection可以有多个SourceCatalogSnapshot。
|
||||
SourceItem、Snapshot、ChangeSet不得跨SourceConnection关联。
|
||||
```
|
||||
|
||||
## SourceFile
|
||||
|
||||
**产品含义:**来源中的一份可治理资料,以及该资料的每个来源版本和内容对象。
|
||||
|
||||
SourceFile不是一个单表对象,当前由三个对象组成:
|
||||
|
||||
| 逻辑层 | Go模型 | 数据表 | 稳定身份 |
|
||||
| --- | --- | --- | --- |
|
||||
| 文件身份 | `governance.SourceItem` | `source_items` | `(source_connection_id, stable_external_id)` |
|
||||
| 文件版本 | `governance.SourceItemVersion` | `source_item_versions` | `(source_item_id, revision)` |
|
||||
| 内容对象 | `governance.ContentObject` | `content_objects` | 全局`sha256` |
|
||||
|
||||
**关键字段:**
|
||||
|
||||
```text
|
||||
SourceItem:
|
||||
- source_connection_id
|
||||
- stable_external_id
|
||||
- canonical_uri_hash
|
||||
- normalized_path
|
||||
- revision
|
||||
- updated_at_source
|
||||
- source_change
|
||||
|
||||
SourceItemVersion:
|
||||
- source_item_id
|
||||
- revision
|
||||
- content_object_id(内容尚未下载或验证前允许为空)
|
||||
- content_state
|
||||
- source_updated_at
|
||||
|
||||
ContentObject:
|
||||
- sha256
|
||||
- size_bytes
|
||||
- mime
|
||||
- validated_format
|
||||
- local_spool_ref
|
||||
- content_state
|
||||
```
|
||||
|
||||
**关系规则:**
|
||||
|
||||
```text
|
||||
路径变化不必然产生新SourceItem。
|
||||
内容Hash变化产生新的SourceItemVersion或新的内容关联。
|
||||
同一内容Hash可以被多个SourceItemVersion复用,但每个版本仍保留自己的来源身份和来源时间。
|
||||
TestPublication必须引用一个SourceItemVersion,不能直接引用路径或ContentObject。
|
||||
```
|
||||
|
||||
## Snapshot
|
||||
|
||||
**产品含义:**一次来源目录扫描获得的不可变目录事实;它记录当时看到的文件/目录,而不是文件正文副本。
|
||||
|
||||
**当前实现:**
|
||||
|
||||
| 项目 | 当前映射 |
|
||||
| --- | --- |
|
||||
| 快照表 | `source_catalog_snapshots` |
|
||||
| 节点表 | `source_catalog_nodes` |
|
||||
| 当前指针 | `source_catalog_heads` |
|
||||
| 快照主键 | `source_catalog_snapshots.id` |
|
||||
| 来源约束 | `(source_connection_id, snapshot_sha256)`唯一 |
|
||||
| 时间 | `observed_at`、`created_at`;Head使用`activated_at` |
|
||||
| 目录节点 | `node_type=folder` |
|
||||
| 文件节点 | `node_type=file` |
|
||||
|
||||
**Snapshot规则:**
|
||||
|
||||
```text
|
||||
SourceConnection → SourceCatalogSnapshot → SourceCatalogNode
|
||||
Head只指向一个当前快照;切换Head不得修改旧快照。
|
||||
节点的source_locator_hash是来源定位摘要,不是内容Hash。
|
||||
节点可选关联source_item_id;未关联节点仍可作为目录事实展示。
|
||||
```
|
||||
|
||||
**操作记录补充:**`source_scans`和`source_scan_changes`是旧扫描运行记录,不能替代当前不可变目录快照。
|
||||
|
||||
## ChangeSet
|
||||
|
||||
**产品含义:**两个不可变来源Snapshot之间的可审查差异计划。
|
||||
|
||||
**当前实现:**
|
||||
|
||||
| 项目 | 当前映射 |
|
||||
| --- | --- |
|
||||
| 主表 | `source_catalog_change_sets` |
|
||||
| 条目表 | `source_catalog_change_items` |
|
||||
| Go投影 | `governance.CatalogChangeSet`、`CatalogChangeItem` |
|
||||
| 主键 | `source_catalog_change_sets.id` |
|
||||
| 快照关系 | `from_snapshot_id`、`to_snapshot_id` |
|
||||
| 动作 | `add`、`update`、`move_scope`、`delete`、`no_change` |
|
||||
| 时间 | `created_at`、`confirmed_at`、`applied_at` |
|
||||
| 当前状态字段 | `planned`、`confirmed`、`applied`、`cancelled` |
|
||||
|
||||
**ChangeSet规则:**
|
||||
|
||||
```text
|
||||
一个ChangeSet只能属于一个SourceConnection。
|
||||
from_snapshot_id和to_snapshot_id必须属于同一SourceConnection。
|
||||
to_snapshot_id当前唯一,因此一份激活快照最多对应一份目录ChangeSet。
|
||||
ChangeItem当前使用(change_set_id, source_locator_hash)唯一。
|
||||
ChangeSet只表达比较结果;它本身不等于已经上传、解析或删除。
|
||||
```
|
||||
|
||||
**后续目标:**S8/P3/P5会补充SourceItemVersion、内容Hash、旧新范围、确认执行和Job关联。
|
||||
|
||||
## Job
|
||||
|
||||
**产品含义:**可租约、可重试、可恢复的异步工作项;Job只保存安全业务ID和进度,不保存敏感连接数据。
|
||||
|
||||
**当前实现:**
|
||||
|
||||
| 项目 | 当前映射 |
|
||||
| --- | --- |
|
||||
| Go模型 | `jobs.Job`(`internal/jobs`) |
|
||||
| 数据表 | `jobs` |
|
||||
| 主键 | `jobs.id` |
|
||||
| 幂等 | `idempotency_key`唯一 |
|
||||
| 调度 | `priority`、`available_at` |
|
||||
| 租约 | `lease_owner`、`lease_until` |
|
||||
| 尝试 | `attempt`、`max_attempts` |
|
||||
| 时间 | `created_at`、`updated_at`、`started_at`、`completed_at` |
|
||||
|
||||
**当前Job类型示例:**
|
||||
|
||||
```text
|
||||
source.catalog.refresh
|
||||
test.upload
|
||||
test.poll
|
||||
test.validate
|
||||
test.remove
|
||||
release.publish
|
||||
release.rollback
|
||||
```
|
||||
|
||||
**Job规则:**
|
||||
|
||||
```text
|
||||
Job完成只表示本地处理器完成;对于WeKnora解析,必须继续由test.poll确认远端终态。
|
||||
Job payload不允许保存Token、Cookie、签名URL、原始文件内容或生产凭据。
|
||||
ReleaseExecution通过job_id外键绑定发布Job。
|
||||
```
|
||||
|
||||
## Release
|
||||
|
||||
**产品含义:**把经过测试验收的知识,以可追溯、可验证、可回滚方式发布到生产。
|
||||
|
||||
当前Release由批次、执行和执行项组成:
|
||||
|
||||
| 逻辑层 | Go模型 | 数据表 | 主键 |
|
||||
| --- | --- | --- | --- |
|
||||
| 发布批次 | `governance.ReleaseBatch` | `release_batches` | `id` |
|
||||
| 批次条目 | `ReleaseBatchItemView`投影 | `release_batch_items` | `(release_batch_id, publication_id)` |
|
||||
| 发布执行 | `governance.ReleaseExecution` | `release_executions` | `id` |
|
||||
| 执行条目 | `ReleaseItemExecutionView`投影 | `release_item_executions` | `id` |
|
||||
|
||||
**Release关系:**
|
||||
|
||||
```text
|
||||
ReleaseBatch → ReleaseExecution → ReleaseItemExecution
|
||||
ReleaseExecution → Job(通过job_id)
|
||||
ReleaseItemExecution → TestPublication(通过publication_id)
|
||||
Rollback ReleaseExecution → prior publish ReleaseExecution
|
||||
```
|
||||
|
||||
**Release规则:**
|
||||
|
||||
```text
|
||||
ReleaseBatch sealed后内容不可变。
|
||||
一个批次最多有一个publish execution。
|
||||
一个publish execution最多有一个rollback execution。
|
||||
ReleaseItemExecution记录生产前后revision和安全错误码。
|
||||
只有独立生产门禁、备份恢复验证和Release Worker均满足时才可以请求生产执行。
|
||||
```
|
||||
|
||||
## TestPublication和验收(辅助对象)
|
||||
|
||||
TestPublication不是原始SourceFile,也不是生产Release;它表示某个SourceItemVersion在指定测试知识库中的处理实例。
|
||||
|
||||
```text
|
||||
SourceItemVersion → TestPublication → AcceptanceResult
|
||||
```
|
||||
|
||||
| 项目 | 当前映射 |
|
||||
| --- | --- |
|
||||
| 测试发布 | `governance.Publication` / API投影`TestPublication` / 表`test_publications` |
|
||||
| 验收结果 | `governance.AcceptanceResult` / 表`acceptance_results` |
|
||||
| 身份 | `(source_item_version_id, test_kb_id)`唯一 |
|
||||
| 处理维度 | ingest_state、processing_state、retrieval_state |
|
||||
| 版本维度 | parser_generation、index_generation |
|
||||
|
||||
## 当前实现缺口
|
||||
|
||||
### Current implementation gaps
|
||||
|
||||
以下缺口是已知事实,不能在任何页面、报告或测试中标为“已完成”:
|
||||
|
||||
1. `SourceCatalogNode`尚不能为所有来源绑定`SourceItem`和`SourceItemVersion`。
|
||||
2. 目录Snapshot节点尚未持久保存文件内容SHA-256;`snapshot_sha256`不是文件内容Hash。
|
||||
3. `SourceCatalogChangeItem`尚未保存旧新内容Hash、旧新范围和SourceItemVersion关联。
|
||||
4. ChangeSet确认尚未直接创建Processing Job;当前Processing Confirmation仅绑定TestPublication。
|
||||
5. Job尚未通过统一外键绑定ChangeSet、文件版本、目标和动作;该工作属于P3/P5。
|
||||
6. `ReleaseChangeSet`和`ProductionSnapshot`尚未实现;当前Release Plan是实时计算,不能冒充不可变发布ChangeSet。
|
||||
7. 当前ReleaseExecution没有来源Snapshot、来源ChangeSet、发布ChangeSet和生产Snapshot的完整外键链;该工作属于R1、R2、R6。
|
||||
8. Source配置、SourceFile身份和内容Hash的运行时证据仍需后续S2、P1、P2补齐。
|
||||
|
||||
## C1验收边界
|
||||
|
||||
C1只完成以下内容:
|
||||
|
||||
```text
|
||||
冻结当前对象术语和真实映射
|
||||
明确身份、时间和关系规则
|
||||
标出当前不存在的对象和关联
|
||||
用测试防止领域契约文档丢失关键对象
|
||||
```
|
||||
|
||||
C1不完成:
|
||||
|
||||
```text
|
||||
统一状态机(C2)
|
||||
OpenAPI(C3)
|
||||
写操作幂等(C4)
|
||||
内容Hash闭环(P1)
|
||||
ChangeSet执行(P3)
|
||||
发布ChangeSet或Production Snapshot(R1/R2)
|
||||
```
|
||||
@@ -0,0 +1,114 @@
|
||||
# P1 独立验收计划
|
||||
|
||||
## 1. 范围
|
||||
|
||||
验证独立治理平台基础骨架:PostgreSQL持久队列、API、普通Worker、默认关闭的Release Worker、React状态页,以及Worker异常中断后的租约恢复。
|
||||
|
||||
明确不验收:PP/AList/钉钉Connector、测试WeKnora导入、生产WeKnora发布、PG/ES/OSS跨环境复制。
|
||||
|
||||
## 2. 前置条件
|
||||
|
||||
- 在`/home/zheromiku/knowledge-governance`执行。
|
||||
- Go 1.26、Node.js 22、npm和Docker Compose可用。
|
||||
- 不使用生产凭据,不连接测试或生产环境。
|
||||
- `secrets/postgres_password`和`secrets/pgpass`使用同一个随机本地密码,权限为600。
|
||||
|
||||
## 3. 自动化检查
|
||||
|
||||
```bash
|
||||
make test
|
||||
make vet
|
||||
make build
|
||||
make web-test
|
||||
make web-typecheck
|
||||
make web-build
|
||||
```
|
||||
|
||||
通过标准:全部退出码为0。
|
||||
|
||||
## 4. 启动与健康检查
|
||||
|
||||
由操作者执行:
|
||||
|
||||
```bash
|
||||
docker compose up --build -d
|
||||
docker compose ps
|
||||
curl -fsS http://127.0.0.1:8088/health
|
||||
```
|
||||
|
||||
通过标准:PostgreSQL和API健康;Web可访问;健康响应为`{"status":"ok"}`。
|
||||
|
||||
## 5. 幂等入队
|
||||
|
||||
连续两次发送完全相同的请求:
|
||||
|
||||
```bash
|
||||
for n in 1 2; do
|
||||
curl -fsS -X POST http://127.0.0.1:8088/api/v1/jobs \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{"type":"demo","payload":{"round":1},"idempotency_key":"acceptance:idempotent:001"}'
|
||||
done
|
||||
```
|
||||
|
||||
通过标准:两次响应中的任务ID相同;列表中该幂等键只有一个任务;任务最终为`completed`。
|
||||
|
||||
## 6. Worker租约恢复
|
||||
|
||||
1. 创建60秒本地任务:
|
||||
|
||||
```bash
|
||||
curl -fsS -X POST http://127.0.0.1:8088/api/v1/jobs \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{"type":"demo.sleep","payload":{"sleep_ms":60000},"idempotency_key":"acceptance:lease:001"}'
|
||||
```
|
||||
|
||||
2. 在状态页或列表API确认任务进入`running`且`attempt=1`。
|
||||
3. 强制终止普通Worker:
|
||||
|
||||
```bash
|
||||
docker compose kill -s KILL worker
|
||||
```
|
||||
|
||||
4. 重新启动Worker:
|
||||
|
||||
```bash
|
||||
docker compose up -d worker
|
||||
```
|
||||
|
||||
5. 等待原租约到期(最多35秒)并持续查询:
|
||||
|
||||
```bash
|
||||
curl -fsS 'http://127.0.0.1:8088/api/v1/jobs?limit=50'
|
||||
```
|
||||
|
||||
通过标准:同一个任务被重新领取,`attempt=2`;继续等待后进入`completed`。不得产生第二条同幂等键任务。
|
||||
|
||||
## 7. Release Worker默认关闭
|
||||
|
||||
```bash
|
||||
docker compose logs --no-color release-worker
|
||||
```
|
||||
|
||||
通过标准:日志包含`release worker disabled`;没有`release worker started in probe-only mode`;没有任何测试或生产外部连接配置。
|
||||
|
||||
## 8. 状态页
|
||||
|
||||
打开`http://127.0.0.1:8088/`:
|
||||
|
||||
- 页面标题为“知识同步治理平台”。
|
||||
- 显示任务总数、运行中、待重试、失败、已完成。
|
||||
- 显示最近50条任务及尝试次数。
|
||||
- 页面明确显示“Release Worker 默认关闭”。
|
||||
- 桌面端无页面级横向溢出;窄屏仅任务表自身可横向滚动。
|
||||
|
||||
## 9. 停止与保留数据
|
||||
|
||||
```bash
|
||||
docker compose down
|
||||
```
|
||||
|
||||
通过标准:容器停止,`governance-pgdata`卷保留。未经单独确认不得执行`docker compose down -v`。
|
||||
|
||||
## 10. 验收记录
|
||||
|
||||
记录每一步的时间、退出码、任务ID和最终状态。任何步骤失败时停止扩展,不连接WeKnora或来源系统,以失败证据定位P1本身。
|
||||
@@ -0,0 +1,84 @@
|
||||
# P2.3 PP完整来源清单验收
|
||||
|
||||
## 交付范围
|
||||
|
||||
- 将旧manifest中54个超过50 MiB、未进入旧账本终态的来源建模为阻断对象;
|
||||
- 形成613个PP来源对象的统一治理快照;
|
||||
- reconciliation以内容SHA revision为依据,同URL内容变化必须计划上传;
|
||||
- 历史已同步快照与治理数据库一致时,计划上传数必须为0;
|
||||
- 提供只读分页/状态筛选API和真实知识资产Web页面。
|
||||
|
||||
不在本阶段执行:来源网络访问、文件下载或上传、测试/生产WeKnora连接、生产发布、AList写操作、钉钉授权、commit或push。
|
||||
|
||||
## 数据契约
|
||||
|
||||
```text
|
||||
source_items=613
|
||||
completed=530
|
||||
content_alias=29
|
||||
over_limit=54
|
||||
content_objects=530
|
||||
source_item_versions=613
|
||||
test_publications=530
|
||||
planned_uploads=0
|
||||
```
|
||||
|
||||
54个over_limit版本必须同时满足:
|
||||
|
||||
```text
|
||||
content_state=over_limit
|
||||
content_object_id IS NULL
|
||||
download_error=legacy_50_mib_limit
|
||||
关联TestPublication数量=0
|
||||
```
|
||||
|
||||
## Reconciliation契约
|
||||
|
||||
- stable source key相同且SHA revision相同:unchanged;
|
||||
- stable source key不存在:adopted;
|
||||
- stable source key相同但SHA revision变化:upload_required;
|
||||
- over_limit:始终计为阻断,不计划上传;
|
||||
- 完整真实快照:unchanged=559、over_limit=54、upload_required=0。
|
||||
|
||||
## 只读API
|
||||
|
||||
```text
|
||||
GET /api/v1/source-items?state=&limit=50&offset=0
|
||||
GET /api/v1/source-items?state=over_limit&limit=100&offset=0
|
||||
```
|
||||
|
||||
`state`只接受`completed`、`content_alias`、`over_limit`或空值;`limit`范围1..100;`offset`范围0..1000000。响应只投影安全治理字段,不返回stable external ID、URI hash、raw_record_summary、Knowledge ID、来源URL、签名URL或凭据。
|
||||
|
||||
## UI验收
|
||||
|
||||
“知识资产”页必须读取真实API,并显示:
|
||||
|
||||
- 来源对象613;
|
||||
- 历史已同步530;
|
||||
- 内容别名29;
|
||||
- 超过上限54;
|
||||
- 状态筛选和服务端分页;
|
||||
- over_limit项显示“超过旧同步上限”和“阻断”。
|
||||
|
||||
桌面宽度1440px必须无页面级横向溢出。多标签工作台、左侧导航和刷新恢复行为保持不变。
|
||||
|
||||
## 数据库升级门禁
|
||||
|
||||
1. 在唯一临时数据库导入真实613项两次;
|
||||
2. 首次新增`1/613/530/613/530`;
|
||||
3. 第二次新增全0;
|
||||
4. 验证54个阻断版本约束;
|
||||
5. 当前库升级前创建非空、mode 600且通过`pg_restore --list`的自定义格式备份;
|
||||
6. 当前库只增量新增54个SourceItem和54个Version;
|
||||
7. 不重启或删除PostgreSQL卷。
|
||||
|
||||
## 自动门禁
|
||||
|
||||
```bash
|
||||
/home/zheromiku/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.26.0.linux-amd64/bin/go test ./... -count=1
|
||||
/home/zheromiku/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.26.0.linux-amd64/bin/go vet ./...
|
||||
/home/zheromiku/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.26.0.linux-amd64/bin/go build ./cmd/...
|
||||
npm --prefix web test -- --run
|
||||
npm --prefix web run typecheck
|
||||
npm --prefix web run build
|
||||
```
|
||||
@@ -0,0 +1,99 @@
|
||||
# P2.2 PP历史账本离线接管验收
|
||||
|
||||
## 范围
|
||||
|
||||
本阶段将旧PP同步器的append-only本地账本接入治理数据库:
|
||||
|
||||
- 不请求PP、AList、钉钉或WeKnora网络API;
|
||||
- 不修改旧同步项目及其账本、manifest或下载文件;
|
||||
- 不下载、上传、移动或删除源文件;
|
||||
- 不启用新PP定时Connector;
|
||||
- 不将离线历史记录宣称为当前WeKnora已验证状态。
|
||||
|
||||
## 折叠与恢复契约
|
||||
|
||||
1. 按旧脚本的`source_url_hash`稳定键折叠;
|
||||
2. 只接受`completed`和带Knowledge ID的`content_alias`终态;
|
||||
3. 单次账本只能包含一个KB;
|
||||
4. 所有终态字段必须完整,source locator和SHA必须是64位小写十六进制;
|
||||
5. 任何仅有中间状态、没有终态的source key都会阻断整批导入;
|
||||
6. 缺SHA终态只能从匹配locator hash的manifest恢复;
|
||||
7. 恢复前验证账本大小、manifest大小、实际文件大小和实际SHA;
|
||||
8. 超过旧连接器50MiB单文件限制时阻断;
|
||||
9. alias必须指向同一快照内的completed canonical,并且SHA、Knowledge ID一致;
|
||||
10. 输出只含聚合数字,不输出KB ID、source ID、Knowledge ID、URL或本地文件路径。
|
||||
|
||||
## 预期真实账本口径
|
||||
|
||||
```text
|
||||
events=1093
|
||||
source_items=559
|
||||
completed=530
|
||||
content_aliases=29
|
||||
recovered_sha256=75
|
||||
verified_bytes=871524116
|
||||
content_objects=530
|
||||
versions=559
|
||||
publications=530
|
||||
```
|
||||
|
||||
`recovered_sha256=75`必须建立在75个文件逐字节重算全部匹配的前提上。
|
||||
|
||||
## 数据库映射
|
||||
|
||||
```text
|
||||
1 source_connection enabled=false,secret_ref/base_url为空
|
||||
559 source_items external + pp_public_tag + unchanged
|
||||
530 content_objects verified,完整SHA唯一,不保存local_spool_ref
|
||||
559 versions raw摘要只保留legacy_status、source_ids和alias canonical hash
|
||||
530 publications uploaded + completed + unpublished
|
||||
```
|
||||
|
||||
29个alias只建立自己的SourceItem和Version,共享canonical ContentObject,不重复创建TestPublication。
|
||||
|
||||
## 幂等和事务
|
||||
|
||||
- 整批导入在单个PostgreSQL事务内完成;
|
||||
- 只允许`INSERT ... ON CONFLICT DO NOTHING`;
|
||||
- 禁止UPDATE和DELETE;
|
||||
- 第一次真实导入期望新增`1/559/530/559/530`;
|
||||
- 第二次相同导入期望新增`0/0/0/0/0`;
|
||||
- 任一步失败必须回滚整批。
|
||||
|
||||
## 自动验证
|
||||
|
||||
在项目根目录运行:
|
||||
|
||||
```bash
|
||||
/home/zheromiku/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.26.0.linux-amd64/bin/go test ./... -count=1
|
||||
/home/zheromiku/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.26.0.linux-amd64/bin/go vet ./...
|
||||
/home/zheromiku/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.26.0.linux-amd64/bin/go build ./cmd/...
|
||||
```
|
||||
|
||||
真实数据库集成测试必须在唯一临时数据库中设置`TEST_DATABASE_URL`后运行;普通无环境测试显示SKIP不等于集成测试通过。
|
||||
|
||||
## 本地库读回验收
|
||||
|
||||
必须验证:
|
||||
|
||||
```text
|
||||
schema_migrations=1,2
|
||||
jobs=2
|
||||
source_connections=1
|
||||
source_items=559
|
||||
content_objects=530
|
||||
source_item_versions=559
|
||||
test_publications=530
|
||||
content_alias versions=29
|
||||
unique publication knowledge IDs=530
|
||||
forbidden raw fields=0
|
||||
```
|
||||
|
||||
530条历史Publication的`retrieval_state`必须是`unpublished`。只有未来通过目标WeKnora只读API确认对象存在、状态和检索行为后,才能改为`enabled`。
|
||||
|
||||
## 备份与运行边界
|
||||
|
||||
- 导入前创建PostgreSQL原生自定义格式备份;
|
||||
- 备份必须非空、权限600并通过`pg_restore --list`;
|
||||
- 导入只使用短期容器,不重启Compose五个长期服务;
|
||||
- 不执行`docker compose down -v`、卷删除、commit、push或外部部署。
|
||||
@@ -0,0 +1,86 @@
|
||||
# P2.1 来源数据底座验收
|
||||
|
||||
## 范围
|
||||
|
||||
本阶段只验证统一来源领域模型和PostgreSQL Migration 2:
|
||||
|
||||
- 来源连接、来源对象、内容对象、来源版本、测试发布五层身份分离;
|
||||
- 来源内`stable_external_id`唯一;
|
||||
- 全局内容对象仅按完整64位小写十六进制SHA-256去重;
|
||||
- 来源变化、内容获取、测试导入、处理和检索状态分开保存;
|
||||
- 新表初始为空,不读取或导入PP、AList、钉钉数据;
|
||||
- 不连接测试或生产WeKnora。
|
||||
|
||||
## 静态与单元验证
|
||||
|
||||
在项目根目录执行:
|
||||
|
||||
```bash
|
||||
/home/zheromiku/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.26.0.linux-amd64/bin/go test ./... -count=1
|
||||
/home/zheromiku/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.26.0.linux-amd64/bin/go vet ./...
|
||||
/home/zheromiku/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.26.0.linux-amd64/bin/go build ./cmd/...
|
||||
```
|
||||
|
||||
在`web/`执行:
|
||||
|
||||
```bash
|
||||
npm test -- --run
|
||||
npm run typecheck
|
||||
npm run build
|
||||
```
|
||||
|
||||
全部命令必须返回exit code 0。
|
||||
|
||||
## 一次性数据库验收
|
||||
|
||||
在唯一命名的临时数据库中依次验证:
|
||||
|
||||
1. 执行Migration 1和Migration 2;
|
||||
2. 再执行一次Migration 2,确认`IF NOT EXISTS`允许安全重跑;
|
||||
3. 插入一条完整的来源→版本→内容→测试发布链;
|
||||
4. 同一来源内重复`stable_external_id`必须被拒绝;
|
||||
5. 不同来源可使用相同`stable_external_id`;
|
||||
6. 重复SHA-256必须被拒绝;
|
||||
7. 非64位、小写十六进制之外的SHA-256必须被拒绝;
|
||||
8. 未声明状态值必须被拒绝;
|
||||
9. 执行Migration 2 down,五张P2表必须按外键依赖反序删除;
|
||||
10. P1的`jobs`表必须保留;
|
||||
11. 删除临时数据库。
|
||||
|
||||
禁止在当前持久本地数据库中执行down验收。
|
||||
|
||||
## 当前本地数据库只读验收
|
||||
|
||||
```bash
|
||||
docker compose exec -T postgres psql -U governance -d governance -At -c \
|
||||
"SELECT string_agg(version::text, ',' ORDER BY version) FROM schema_migrations;"
|
||||
```
|
||||
|
||||
期望:
|
||||
|
||||
```text
|
||||
1,2
|
||||
```
|
||||
|
||||
```bash
|
||||
docker compose exec -T postgres psql -U governance -d governance -At -c \
|
||||
"SELECT COUNT(*) FROM pg_tables WHERE schemaname='public' AND tablename IN ('source_connections','source_items','content_objects','source_item_versions','test_publications');"
|
||||
```
|
||||
|
||||
期望:`5`。
|
||||
|
||||
```bash
|
||||
docker compose exec -T postgres psql -U governance -d governance -At -c \
|
||||
"SELECT (SELECT COUNT(*) FROM source_connections),(SELECT COUNT(*) FROM source_items),(SELECT COUNT(*) FROM content_objects),(SELECT COUNT(*) FROM source_item_versions),(SELECT COUNT(*) FROM test_publications);"
|
||||
```
|
||||
|
||||
在正式导入来源前期望:`0|0|0|0|0`。
|
||||
|
||||
## 数据保留与运行状态
|
||||
|
||||
- Migration前必须创建非空PostgreSQL原生备份,并通过`pg_restore --list`校验;
|
||||
- 只替换API容器,PostgreSQL、普通Worker、Release Worker和Web容器ID不得变化;
|
||||
- Migration后原有`jobs`数量和代表任务状态必须保持;
|
||||
- `/health`必须返回`{"status":"ok"}`;
|
||||
- API启动日志不得包含Migration错误;
|
||||
- 不执行`docker compose down -v`、卷删除、提交、推送或外部部署。
|
||||
@@ -0,0 +1,164 @@
|
||||
# P3 本地 WeKnora 与测试服务器互测验收
|
||||
|
||||
## 范围
|
||||
|
||||
本阶段验证知识治理平台从安全 Spool 制品到 WeKnora 的真实编排链路:
|
||||
|
||||
- 本地 WeKnora 上传、轮询、Chunk 与限定检索;
|
||||
- 上传失败终态、人工重试和持久 Worker 恢复;
|
||||
- 测试服务器 HTTPS 互测;
|
||||
- 530 条 PP 历史 Publication 在线只读核验;
|
||||
- API/UI 安全投影及运行资源;
|
||||
- 不启用生产 Release Worker,不执行生产发布。
|
||||
|
||||
## 本地真实 E2E
|
||||
|
||||
使用 4 份明确无敏感内容的 Markdown 合成制品。每份制品均执行:
|
||||
|
||||
```text
|
||||
spool inspection
|
||||
→ atomic manual source/content/version/publication/job seed
|
||||
→ test.upload
|
||||
→ test.poll
|
||||
→ test.validate
|
||||
→ scoped retrieval probe
|
||||
```
|
||||
|
||||
验收结果:
|
||||
|
||||
- 4/4 Publication `completed`;
|
||||
- 每份文档 Chunk 总数为 1,读取 1/1,Knowledge 归属一致;
|
||||
- 限定 Knowledge 检索均返回目标结果;
|
||||
- 合成 marker 均在内存中验证命中,报告只保存布尔值;
|
||||
- `knowledge_exists=passed`;
|
||||
- `chunks_present=passed`;
|
||||
- 检索阈值未批准,`retrieval_hit=manual_required`。
|
||||
|
||||
## 技术失败与人工重试
|
||||
|
||||
真实故障注入为暂时移走 Spool 制品,不删除文件:
|
||||
|
||||
1. Worker 将 Publication 写为 `ingest_state=failed`、`processing_state=failed`;
|
||||
2. 只保存固定错误码 `upload_artifact_invalid`;
|
||||
3. API 安全投影返回 `can_retry=true`;
|
||||
4. 恢复制品后调用正式 Retry API,返回 `retry_queued`;
|
||||
5. 同一 Publication 最终完成,`retry_count=1`;
|
||||
6. 原失败 Job 与重试 Job均为 `completed/attempt=1`。
|
||||
|
||||
同时兼容历史不一致状态 `ingest=failed, processing=pending`,允许通过正式 Retry API 自愈;普通 pending 或 completed Publication 仍不可重试。
|
||||
|
||||
## 持久 Worker 恢复
|
||||
|
||||
在 Job 已持久化后让 Worker 多次启动失败,再恢复正确配置:
|
||||
|
||||
- Job 未丢失;
|
||||
- Worker 恢复后自动消费;
|
||||
- upload、poll、validate 均为 `completed/attempt=1`;
|
||||
- 未重复上传;
|
||||
- 恢复后的 Worker 为 `running/restarts=0`。
|
||||
|
||||
## 测试服务器互测
|
||||
|
||||
测试服务器使用 HTTPS `weknora-test.dobot.cn`,并创建隔离 KB `governance-cross-env-test`。未写入既有对内、对外或历史 E2E KB。
|
||||
|
||||
互测结果:
|
||||
|
||||
- 合成 Markdown 上传成功;
|
||||
- remote status `completed`;
|
||||
- Chunk 1/1,归属一致;
|
||||
- 限定检索结果 2,目标结果 2;
|
||||
- `marker_found=true`;
|
||||
- 最高检索分 `0.7180294855048308`;
|
||||
- 结构验收通过;
|
||||
- 语义验收保持 `manual_required`。
|
||||
|
||||
## MinerU 阻塞与恢复
|
||||
|
||||
首次互测时,测试 WeKnora 的 default Asynq 队列为:
|
||||
|
||||
```text
|
||||
active=10
|
||||
pending=208
|
||||
archived=186
|
||||
```
|
||||
|
||||
测试服务器硬件空闲,但云 MinerU 健康请求超时。GPU 主机日志确认根因为:
|
||||
|
||||
```text
|
||||
OSError: [Errno 24] Too many open files
|
||||
```
|
||||
|
||||
systemd hard limit 为 524288,但进程 soft limit 只有 1024。仅在 MinerU `processing_tasks=0`、`queued_tasks=0` 后执行:
|
||||
|
||||
- 备份 `/etc/systemd/system/mineru.service`;
|
||||
- 在 `[Service]` 设置 `LimitNOFILE=65536`;
|
||||
- daemon-reload 并重启 MinerU;
|
||||
- 验证 soft/hard 均为 65536;
|
||||
- 验证 port 8000 listener PID 与 systemd MainPID 一致;
|
||||
- 验证 MinerU v3.4.2 health 为 healthy;修复瞬间回读并发为 8;
|
||||
- 验证 A10 已加载模型。
|
||||
|
||||
恢复后测试 WeKnora 的 pending/active 队列归零,隔离互测文档完成。历史 archived 数为 403;本阶段没有删除、重提或改写这些历史失败任务。
|
||||
|
||||
最终状态复核时,MinerU unit 已由另一个独立操作改为并发 2 并显式重启;本阶段没有覆盖该并发变更。`LimitNOFILESoft=65536` 继续生效,health 保持 healthy。
|
||||
|
||||
## 530 条历史 Publication 在线只读核验
|
||||
|
||||
候选查询严格限定为具有 Publication 的 PP 来源,排除所有 manual/future 来源。Dry-run 精确得到 530 条。
|
||||
|
||||
使用 API Key 文件、HTTPS、并发 4,每 25 个 completed 抽查 1 个 Chunk:
|
||||
|
||||
```text
|
||||
total=530
|
||||
existing=530
|
||||
missing=0
|
||||
wrong_kb=0
|
||||
pending=0
|
||||
processing=0
|
||||
finalizing=0
|
||||
completed=530
|
||||
failed=0
|
||||
chunk_checked=21
|
||||
chunk_empty=0
|
||||
http_errors=0
|
||||
```
|
||||
|
||||
核验只调用 GET;未重新上传 530 条历史 Knowledge。
|
||||
|
||||
## API、UI 与密钥边界
|
||||
|
||||
- API/Web 只展示安全 Publication 投影;
|
||||
- 页面未出现 Knowledge ID、KB ID、SHA、Spool 路径、API Key或对象存储地址;
|
||||
- 测试 WeKnora Key 只以普通 0600 文件提供给普通 Worker;
|
||||
- API、Web、Release Worker 不挂载测试 Key;
|
||||
- Spool 在 Worker 中只读;
|
||||
- `RELEASE_WORKER_ENABLED=false`;
|
||||
- Firefox 1440×900 验收通过左导航、顶部多标签、任务页、统一来源清单、手工验收标签及无横向溢出。
|
||||
|
||||
截图:`tmp/P3_LOCAL_E2E_FIREFOX.png`
|
||||
|
||||
## 自动化门禁
|
||||
|
||||
```text
|
||||
Go tests: 16 packages passed
|
||||
Go vet: passed
|
||||
Go race (testweknora): passed
|
||||
Web tests: 24/24 passed
|
||||
TypeScript typecheck: passed
|
||||
Vite production build: passed
|
||||
PostgreSQL isolated integration tests: passed
|
||||
Compose config: passed
|
||||
```
|
||||
|
||||
## 数据与发布边界
|
||||
|
||||
- Migration 1–4 已应用;
|
||||
- 最终治理数据为 618 个 Source Item、535 个 Content、618 个 Version、535 个 Publication 和 69 个 Job;
|
||||
- 535/535 Publication completed,69/69 Job completed;
|
||||
- 5 个 manual E2E 对象对应 15 条验收记录;
|
||||
- 54 个 `over_limit` 项继续阻断;
|
||||
- 不删除 WeKnora Knowledge、源文件、PostgreSQL 数据或命名卷;
|
||||
- 不执行 `docker compose down -v`;
|
||||
- 未连接生产 WeKnora;
|
||||
- 未启用生产发布;
|
||||
- 未 commit、未 push。
|
||||
@@ -0,0 +1,100 @@
|
||||
# P4.1 来源连接只读工作台验收
|
||||
|
||||
## 范围
|
||||
|
||||
本切片将“来源管理”从占位页接入真实 PostgreSQL,只提供安全只读视图:
|
||||
|
||||
- 列出已登记来源连接;
|
||||
- 展示启用状态、是否配置调度、健康枚举、对象数和更新时间;
|
||||
- 汇总总数、启用、停用、健康、降级、异常和未知;
|
||||
- 不展示或读取到 API 投影中的连接地址、secret引用和原始健康错误正文;
|
||||
- 不创建、修改、启用或删除来源连接;
|
||||
- 不调用 PP、AList 或钉钉写接口。
|
||||
|
||||
## API
|
||||
|
||||
```text
|
||||
GET /api/v1/source-connections
|
||||
```
|
||||
|
||||
安全字段:
|
||||
|
||||
```text
|
||||
id
|
||||
source_type
|
||||
name
|
||||
enabled
|
||||
schedule_configured
|
||||
health_status
|
||||
item_count
|
||||
updated_at
|
||||
```
|
||||
|
||||
明确省略:
|
||||
|
||||
```text
|
||||
base_url
|
||||
secret_ref
|
||||
schedule原始JSON
|
||||
last_health原始JSON及错误正文
|
||||
API Key、Token、Cookie
|
||||
```
|
||||
|
||||
## 当前真实数据
|
||||
|
||||
```text
|
||||
connections=2
|
||||
enabled=0
|
||||
disabled=2
|
||||
healthy=0
|
||||
degraded=0
|
||||
unhealthy=0
|
||||
unknown=2
|
||||
```
|
||||
|
||||
来源对象数:
|
||||
|
||||
```text
|
||||
manual=5
|
||||
pp=613
|
||||
```
|
||||
|
||||
## 自动化验证
|
||||
|
||||
```text
|
||||
Store安全投影单元测试:passed
|
||||
HTTP API安全投影测试:passed
|
||||
前端严格解析与额外字段剥离:passed
|
||||
一次性PostgreSQL集成测试:passed (0.121s)
|
||||
Go全量测试:passed
|
||||
Go vet:passed
|
||||
Web测试:26/26 passed
|
||||
TypeScript typecheck:passed
|
||||
Vite production build:passed
|
||||
Compose config:passed
|
||||
```
|
||||
|
||||
一次性 PostgreSQL 测试实际插入包含敏感 `base_url`、`secret_ref` 和健康 detail 的连接,再确认序列化投影不包含这些内容。临时数据库在测试后删除,持久治理库没有写入测试记录。
|
||||
|
||||
## 浏览器验收
|
||||
|
||||
Firefox headless 1440×900:
|
||||
|
||||
- 左侧导航“来源管理”高亮;
|
||||
- 顶部保留“工作台/来源管理”多标签;
|
||||
- 两条真实连接记录可见;
|
||||
- 状态、调度、健康、对象数易读;
|
||||
- 来源占位文案已移除;
|
||||
- 页面敏感字段泄漏为 0;
|
||||
- 紧凑表格修复后“更新时间”标题及两行时间完整可见;
|
||||
- 无横向裁切、遮挡或乱码。
|
||||
|
||||
截图:`tmp/P41_SOURCE_CONNECTIONS_FIREFOX.png`
|
||||
|
||||
## 运行边界
|
||||
|
||||
- 只重建 API 与 Web;
|
||||
- PostgreSQL、普通 Worker、Release Worker 未重建;
|
||||
- Release Worker 继续保持关闭;
|
||||
- 未连接生产;
|
||||
- 未 commit、未 push。
|
||||
@@ -0,0 +1,250 @@
|
||||
# P4.2 来源只读扫描与变化入账验收
|
||||
|
||||
## 范围
|
||||
|
||||
本切片将 PP 本地历史快照接入普通 Worker 的持久扫描链路:
|
||||
|
||||
```text
|
||||
来源管理手工触发
|
||||
→ 原子创建 source_scan + source.scan.pp Job
|
||||
→ 普通 Worker 只读 ledger/manifest/制品元数据
|
||||
→ 与 613 条治理 Source Item 对账
|
||||
→ 原子保存聚合、变化记录与连接健康
|
||||
→ API/Web 展示扫描历史和变化结果
|
||||
```
|
||||
|
||||
本切片不上传、移动、重命名或删除 PP 文件,不调用 PP 网络写接口,不删除治理对象或 WeKnora Knowledge。`missing` 只记录为 `unavailable`,54 个 `over_limit` 项继续阻断,生产 Release Worker 保持关闭。
|
||||
|
||||
## 安全配置与装配
|
||||
|
||||
- `PP_SCAN_ENABLED` 默认 `false`;仅显式启用且 allowed root、ledger、manifest 三个路径完整时注册 `source.scan.pp` Handler。
|
||||
- allowed root 必须是绝对规范路径;ledger 和 manifest 必须位于其内。
|
||||
- Scanner 拒绝越界、软链接、非普通文件、缺少 `local_path`、打开期间文件身份变化和超过字节上限的元数据文件。
|
||||
- `over_limit` 记录允许其根目录内的制品不存在,因为它从未被下载;若制品存在仍必须通过普通文件和非软链接校验。
|
||||
- Compose 只给普通 Worker 挂载 PP 数据目录,挂载模式为只读;API、Web、Release Worker均没有该挂载。
|
||||
- 实际容器探针:ledger/manifest 可读,向 manifest 追加写入被只读文件系统拒绝。
|
||||
- 本地测试 WeKnora Key 仍只挂载给普通 Worker;本切片不调用 WeKnora 上传或删除 API。
|
||||
|
||||
## Migration 5
|
||||
|
||||
Migration 5 新增:
|
||||
|
||||
```text
|
||||
source_scans
|
||||
source_scan_changes
|
||||
idx_source_scans_one_active
|
||||
idx_source_scans_connection_created
|
||||
idx_source_scan_changes_pending
|
||||
```
|
||||
|
||||
关键约束:
|
||||
|
||||
- 每个来源连接最多一个 `pending/running` Scan;
|
||||
- Scan 状态限定为 `pending/running/completed/failed`;
|
||||
- Change 类型限定为 `added/updated/unavailable`;
|
||||
- `snapshot_sha256` 只在内部审计,不进入 API;
|
||||
- down migration 先删除依赖的 changes 表,再删除 scans 表。
|
||||
|
||||
一次性 PostgreSQL 执行了:
|
||||
|
||||
```text
|
||||
up to 5 → down to 4 → up to 5
|
||||
result: passed
|
||||
```
|
||||
|
||||
持久库迁移前逻辑备份:
|
||||
|
||||
```text
|
||||
tmp/backups/governance-pre-migration5-20260906T011223+0800.dump
|
||||
bytes=242987
|
||||
mode=0600
|
||||
pg_restore catalog=valid
|
||||
```
|
||||
|
||||
持久库应用后:
|
||||
|
||||
```text
|
||||
schema_migrations=1,2,3,4,5
|
||||
source_items=618
|
||||
content_objects=535
|
||||
source_item_versions=618
|
||||
test_publications=535
|
||||
```
|
||||
|
||||
旧业务数据计数未因 Migration 5 改变。
|
||||
|
||||
## API
|
||||
|
||||
```text
|
||||
POST /api/v1/source-connections/{id}/scan
|
||||
GET /api/v1/source-connections/{id}/scans?limit=20&offset=0
|
||||
GET /api/v1/source-connections/{id}/scans/{scan_id}
|
||||
```
|
||||
|
||||
触发语义:
|
||||
|
||||
- 成功返回 `202`;
|
||||
- 同一连接已有活动扫描返回 `409`;
|
||||
- 不存在或不可扫描的连接返回 `404`;
|
||||
- 创建 Scan 和 Job 在同一条 PostgreSQL data-modifying CTE 中完成。
|
||||
|
||||
安全只读投影允许:
|
||||
|
||||
```text
|
||||
Scan: id/state/聚合计数/固定错误码/创建完成时间
|
||||
Change: id/source_item_id(可空)/change_type/observed_size_bytes(可空)/created_at
|
||||
```
|
||||
|
||||
明确省略:
|
||||
|
||||
```text
|
||||
snapshot_sha256
|
||||
stable_external_id
|
||||
previous_revision
|
||||
observed_revision
|
||||
allowed root、ledger/manifest/local_path
|
||||
来源 URL、正文、base_url、secret_ref、Token/API Key/Cookie
|
||||
```
|
||||
|
||||
跨连接读取 Scan 详情返回 `404`。前端再次使用 allowlist 重建对象,不透传服务端额外字段。
|
||||
|
||||
## 真实 PP 只读扫描
|
||||
|
||||
真实输入只读基线:
|
||||
|
||||
```text
|
||||
ledger events=1093
|
||||
manifest/source items=613
|
||||
existing artifacts=559
|
||||
historical over_limit without downloaded artifact=54
|
||||
```
|
||||
|
||||
第一次真实扫描按安全规则失败:
|
||||
|
||||
```text
|
||||
state=failed
|
||||
error_code=pp_snapshot_invalid
|
||||
```
|
||||
|
||||
根因是初版 Scanner 把 54 个历史 `over_limit` 项未下载的制品也要求为已存在普通文件。修复后规则为:`over_limit` 缺失制品是合法阻断状态,但路径仍必须声明且位于 allowed root 内;其他状态缺失仍 fail-closed。
|
||||
|
||||
修复后的多次真实扫描均一致:
|
||||
|
||||
```text
|
||||
observed=613
|
||||
unchanged=559
|
||||
added=0
|
||||
updated=0
|
||||
over_limit=54
|
||||
missing=0
|
||||
planned_uploads=0
|
||||
changes=0
|
||||
```
|
||||
|
||||
PP ledger、manifest 和来源制品未被修改;没有生成上传计划。
|
||||
|
||||
## 调度、并发与恢复
|
||||
|
||||
### 单活动扫描
|
||||
|
||||
Worker 停止时连续触发同一连接:
|
||||
|
||||
```text
|
||||
first=202
|
||||
second=409
|
||||
active_scans=1
|
||||
```
|
||||
|
||||
恢复 Worker 后唯一活动扫描完成。
|
||||
|
||||
### pending 恢复
|
||||
|
||||
```text
|
||||
Worker stopped
|
||||
→ POST creates pending Scan/Job
|
||||
→ Worker started
|
||||
→ Scan completed, Job completed
|
||||
```
|
||||
|
||||
### 过期 running 租约重领
|
||||
|
||||
真实持久队列构造“旧 Worker 已领取但完成前崩溃”的状态:
|
||||
|
||||
```text
|
||||
before: scan=running job=running attempt=1 lease_expired=true
|
||||
restart persistent Worker
|
||||
after: scan=completed job=completed attempt=2
|
||||
result: observed=613 unchanged=559 over_limit=54
|
||||
```
|
||||
|
||||
Handler 对 `running` Scan 可恢复;若 Scan 已经 `completed` 而 Job 尚未完成,则重领为幂等 no-op,不重复扫描或写 Change。
|
||||
|
||||
最终状态:
|
||||
|
||||
```text
|
||||
source_scans=6
|
||||
completed=5
|
||||
failed_fixed_code=1
|
||||
active_scans=0
|
||||
active_source_scan_jobs=0
|
||||
```
|
||||
|
||||
## Web 与 Firefox
|
||||
|
||||
“来源管理”保留左侧导航和顶部标签,并新增:
|
||||
|
||||
- 来源连接的最近扫描聚合;
|
||||
- “立即扫描”按钮及活动态禁用;
|
||||
- 最近 20 条扫描历史;
|
||||
- 单次扫描变化记录安全详情;
|
||||
- 零变化时明确显示“本次扫描未发现新增、更新或不可用变化”。
|
||||
|
||||
Firefox headless 1440×1200 验收:
|
||||
|
||||
```text
|
||||
history_rows=6
|
||||
completed=5
|
||||
failed=1
|
||||
detail_selection=passed
|
||||
empty_changes_message=visible
|
||||
safe_projection_leaks=0
|
||||
root_overflow=false
|
||||
table_overflow=false
|
||||
```
|
||||
|
||||
截图:`tmp/P42_SOURCE_SCAN_HISTORY_FIREFOX.png`
|
||||
|
||||
视觉复核确认:导航、顶部标签、5 条完成/1 条失败、观察数、上限阻断和变化面板均清晰;无横向裁切、遮挡或乱码。
|
||||
|
||||
## 自动化验证
|
||||
|
||||
```text
|
||||
PP配置安全默认和路径校验:passed
|
||||
Worker Handler装配测试:passed
|
||||
File Scanner边界与over_limit回归:passed
|
||||
Store扫描触发/完成/失败/恢复/安全读取:passed
|
||||
HTTP触发/历史/详情/404/409安全投影:passed
|
||||
前端allowlist解析、请求和页面渲染:passed
|
||||
一次性PostgreSQL真实扫描与变化读取:passed
|
||||
Migration 5 up/down/up:passed
|
||||
Web tests:31/31 passed
|
||||
TypeScript typecheck:passed
|
||||
Vite production build:passed
|
||||
Compose config:passed
|
||||
Compose全镜像build:passed
|
||||
Go全仓库test:passed
|
||||
Go vet:passed
|
||||
Go cmd build:passed
|
||||
gofmt差异:0
|
||||
Firefox真实页面:passed
|
||||
```
|
||||
|
||||
## 运行边界
|
||||
|
||||
- 本地治理平台与本地测试 WeKnora 范围内完成;
|
||||
- PP 来源只读,未调用来源写/删 API;
|
||||
- 未删除 PostgreSQL 或 WeKnora 卷;
|
||||
- 未连接或修改生产环境;
|
||||
- `RELEASE_WORKER_ENABLED=false`;
|
||||
- 54 个 `over_limit` 项继续阻断;
|
||||
- 未 commit、未 push。
|
||||
@@ -0,0 +1,320 @@
|
||||
# P5 知识工程师审核、封批与发布前门禁验收
|
||||
|
||||
## 范围
|
||||
|
||||
P5 在测试发布完成后增加人工控制平面:
|
||||
|
||||
```text
|
||||
当前 Test Publication + 最新 Acceptance run
|
||||
→ 固定门禁计算
|
||||
→ 知识工程师批准或驳回
|
||||
→ 已批准项目创建 draft 批次
|
||||
→ 封批时再次验证 Publication / Review / Acceptance
|
||||
→ sealed(尚未发布)
|
||||
```
|
||||
|
||||
P5 不实现生产复制,不创建 `release.*` Job,不启用 Release Worker。
|
||||
|
||||
## 数据模型
|
||||
|
||||
Migration 6 新增:
|
||||
|
||||
```text
|
||||
publication_reviews
|
||||
release_batches
|
||||
release_batch_items
|
||||
```
|
||||
|
||||
设计约束:
|
||||
|
||||
- `publication_reviews` 是追加式决定,不覆盖历史决定;
|
||||
- 决定只允许 `approved` / `rejected`;
|
||||
- 可见性只允许 `internal` / `external`;
|
||||
- 使用固定 `decision_code`,不保存自由文本或源正文;
|
||||
- 批准绑定当时最新 `acceptance_run_id` 和 Publication `updated_at`;
|
||||
- 完全相同的决定通过唯一键幂等;
|
||||
- `release_batch_items` 锁定 Publication、Review、Source Item Version、Acceptance run、可见性和 Publication 时间戳;
|
||||
- 同一 Publication 只能进入一个批次;
|
||||
- 批次状态只允许 `draft` / `sealed` / `cancelled`;
|
||||
- 策略版本固定为 `release-gate-v1`;
|
||||
- Migration down 顺序为 item → batch → review;
|
||||
- 没有历史自动回填,没有把 530 条 PP legacy 项标为“已人工审核”。
|
||||
|
||||
Migration round-trip 在 disposable PostgreSQL 验证:
|
||||
|
||||
```text
|
||||
up 6 → down 5 → up 6
|
||||
```
|
||||
|
||||
回滚 6 时,P4 的 `source_scans` / `source_scan_changes` 保持存在。
|
||||
|
||||
## 审核候选门禁
|
||||
|
||||
审核队列只纳入至少有一个 Acceptance run 的 Publication。当前持久数据:
|
||||
|
||||
```text
|
||||
Test Publications 535
|
||||
PP legacy(无逐条 Acceptance) 530
|
||||
manual 合成验收项 5
|
||||
审核候选 5
|
||||
```
|
||||
|
||||
固定 gate code:
|
||||
|
||||
```text
|
||||
ready
|
||||
publication_not_ready
|
||||
source_not_current
|
||||
acceptance_failed
|
||||
acceptance_incomplete
|
||||
```
|
||||
|
||||
批准要求:
|
||||
|
||||
```text
|
||||
ingest_state = uploaded
|
||||
processing_state = completed
|
||||
retrieval_state = unpublished
|
||||
当前 Source Item revision = Publication 使用的 Source Item Version revision
|
||||
source_change 不得为 unavailable / deleted
|
||||
knowledge_exists = passed
|
||||
chunks_present = passed
|
||||
retrieval_hit = passed 或 manual_required
|
||||
```
|
||||
|
||||
`retrieval_hit=manual_required` 没有被自动改成 passed;知识工程师点击批准时明确承担人工判断。
|
||||
|
||||
驳回固定 code:
|
||||
|
||||
```text
|
||||
needs_correction
|
||||
classification_changed
|
||||
acceptance_rejected
|
||||
```
|
||||
|
||||
批准固定 code:
|
||||
|
||||
```text
|
||||
approved_for_release
|
||||
```
|
||||
|
||||
## 封批门禁
|
||||
|
||||
创建 draft 时必须:
|
||||
|
||||
- 请求至少 1 项、最多 100 项;
|
||||
- Publication ID 不重复;
|
||||
- 每项均有当前有效的最新批准;
|
||||
- 批准仍绑定当前 Acceptance run 和 Publication `updated_at`;
|
||||
- 每项仍满足完整 release gate;
|
||||
- 任一项目不合格则整条 SQL 不创建批次或部分 item;
|
||||
- 唯一冲突导致整条语句回滚。
|
||||
|
||||
封批时再次验证:
|
||||
|
||||
- batch 当前为 `draft`;
|
||||
- batch 至少有 1 项;
|
||||
- 每个锁定的 Review、Acceptance run、Publication timestamp、可见性仍匹配;
|
||||
- 当前 Publication 和 Source revision 仍满足门禁;
|
||||
- 所有 item 都有效后才原子变为 `sealed`;
|
||||
- 已 sealed 请求幂等返回现有 sealed 记录;
|
||||
- 不创建 Job。
|
||||
|
||||
P6 在真正发布前仍需再次验证封批快照;P5 的 sealed 只表示“人工封批完成”,不表示生产已发布。
|
||||
|
||||
## API
|
||||
|
||||
```text
|
||||
GET /api/v1/review-candidates
|
||||
POST /api/v1/review-candidates/{publication_id}/decision
|
||||
|
||||
GET /api/v1/release-batches
|
||||
POST /api/v1/release-batches
|
||||
GET /api/v1/release-batches/{batch_id}
|
||||
POST /api/v1/release-batches/{batch_id}/seal
|
||||
```
|
||||
|
||||
请求体使用 strict JSON decoder:
|
||||
|
||||
- 最大 64 KiB;
|
||||
- 未知字段拒绝;
|
||||
- 尾随 JSON 拒绝;
|
||||
- 非法输入为 400;
|
||||
- 不满足审核/创建条件为 422;
|
||||
- 封批快照失效为 409;
|
||||
- 不回显 Store 原始错误。
|
||||
|
||||
安全响应不包含:
|
||||
|
||||
```text
|
||||
knowledge_id
|
||||
test_kb_id
|
||||
acceptance_run_id
|
||||
publication_updated_at
|
||||
stable_external_id
|
||||
canonical_uri_hash
|
||||
local_spool_ref
|
||||
raw_record_summary
|
||||
base_url
|
||||
secret_ref
|
||||
API Key / Token
|
||||
```
|
||||
|
||||
审核 API 的 `reviewer_ref` 当前是本地治理工作台的调用方提供标识,不等同于企业 SSO 强身份。P6 生产启用前仍需将控制面置于经认证/授权的管理员入口;当前 Release Worker关闭,因此该限制不会形成生产发布通路。
|
||||
|
||||
## Web 工作台
|
||||
|
||||
“发布管理”由占位页升级为真实 P5 工作台:
|
||||
|
||||
- 左侧导航 + 顶部可关闭标签保持不变;
|
||||
- 展示审核候选、门禁就绪、待审核和已选数量;
|
||||
- 每条显示对象、分块、检索验收;
|
||||
- `manual_required` 显示为“需人工确认”;
|
||||
- 人工选择 `internal` / `external`;
|
||||
- 工程师标识为空时批准、驳回和封批按钮禁用;
|
||||
- 只有已批准且当前 gate ready 的项目可勾选入批;
|
||||
- 创建 draft 后仍需单独点击“封批”;
|
||||
- 页面固定显示“生产门禁保持关闭”“Release Worker保持关闭”“尚未发布”;
|
||||
- 不在浏览器中录入或显示 API Key、数据库密码或 WeKnora凭据。
|
||||
|
||||
## 测试与真实证据
|
||||
|
||||
### 自动化测试
|
||||
|
||||
```text
|
||||
Go全仓库test passed
|
||||
Go vet passed
|
||||
Go cmd build passed
|
||||
Web tests 35/35 passed
|
||||
TypeScript typecheck passed
|
||||
Vite production build passed
|
||||
Compose config passed
|
||||
Migration 6 contract passed
|
||||
Migration 6 up/down/up passed(disposable PostgreSQL)
|
||||
```
|
||||
|
||||
### Disposable PostgreSQL Store E2E
|
||||
|
||||
验证:
|
||||
|
||||
```text
|
||||
ready → approved → draft → sealed
|
||||
无批准 → create batch 拒绝
|
||||
批准后 Publication updated_at 改变 → seal 拒绝
|
||||
sealed batch list/detail → 读回 1 item
|
||||
P5 操作新增 Jobs → 0
|
||||
```
|
||||
|
||||
### Disposable API HTTP E2E
|
||||
|
||||
使用独立数据库、临时 API 端口和无敏感合成记录:
|
||||
|
||||
```text
|
||||
pending → POST approved
|
||||
approved → POST draft
|
||||
draft → POST sealed
|
||||
GET detail items=1
|
||||
safe projection leaks=0
|
||||
Jobs=0
|
||||
```
|
||||
|
||||
该测试完成后:
|
||||
|
||||
```text
|
||||
临时 API stopped
|
||||
临时数据库 deleted
|
||||
临时 pgpass deleted
|
||||
持久库人工决定 0
|
||||
持久库批次 0
|
||||
持久库批次项目 0
|
||||
```
|
||||
|
||||
### 持久本地环境
|
||||
|
||||
应用 Migration 6 前备份:
|
||||
|
||||
```text
|
||||
tmp/backups/governance-pre-migration6-20260906T030213+0800.dump
|
||||
bytes=252653
|
||||
mode=0600
|
||||
pg_restore catalog=valid
|
||||
```
|
||||
|
||||
应用后:
|
||||
|
||||
```text
|
||||
schema_migrations 1,2,3,4,5,6
|
||||
publication_reviews 0
|
||||
release_batches 0
|
||||
release_batch_items 0
|
||||
source_items 618
|
||||
content_objects 535
|
||||
source_item_versions 618
|
||||
test_publications 535
|
||||
acceptance_results 15
|
||||
```
|
||||
|
||||
真实审核队列:
|
||||
|
||||
```text
|
||||
candidates 5
|
||||
pending 5
|
||||
gate ready 5
|
||||
retrieval manual_required 5
|
||||
```
|
||||
|
||||
未替知识工程师批准任何持久候选。
|
||||
|
||||
### Firefox
|
||||
|
||||
脚本:
|
||||
|
||||
```text
|
||||
tmp/p5_review_gate_browser_accept.py
|
||||
```
|
||||
|
||||
截图:
|
||||
|
||||
```text
|
||||
tmp/P5_REVIEW_GATE_TOP_FIREFOX.png
|
||||
tmp/P5_REVIEW_GATE_FIREFOX.png
|
||||
```
|
||||
|
||||
验收:
|
||||
|
||||
```text
|
||||
review rows 5
|
||||
ready / pending 5 / 5
|
||||
actor required true
|
||||
批准/驳回填写actor后可用 true
|
||||
mutation clicks 0
|
||||
batch rows 0
|
||||
创建批次按钮 disabled
|
||||
safe projection leaks 0
|
||||
root horizontal overflow false
|
||||
table horizontal overflow false
|
||||
```
|
||||
|
||||
截图视觉检查未发现横向裁切、遮挡、乱码或阻断问题。
|
||||
|
||||
## 已发现并修复的问题
|
||||
|
||||
1. PostgreSQL 审核参数同时参与比较与 VARCHAR 插入时类型推断冲突;为参数加显式 cast。
|
||||
2. data-modifying CTE 同语句从基表读取新 batch 看不到新 snapshot;改为直接从 `created_batch RETURNING` 返回。
|
||||
3. 前端 Page parser 使用 object spread 意外保留 `rawItems`,可能泄漏服务端额外字段;改为显式逐字段构造 Page。
|
||||
4. disposable API 的 pgpass 原文件只匹配持久数据库名;使用单次 0600 临时 pgpass 精确匹配 disposable DB,测试后删除,没有改动原密钥文件。
|
||||
|
||||
## 最终边界
|
||||
|
||||
```text
|
||||
业务人工审核决定 0
|
||||
业务封批 0
|
||||
生产发布 未执行
|
||||
生产 WeKnora 未连接
|
||||
Release Worker false
|
||||
release Jobs 0
|
||||
PP 源文件 未修改
|
||||
卷 保留
|
||||
commit / push 未执行
|
||||
凭据输出 0
|
||||
```
|
||||
@@ -0,0 +1,495 @@
|
||||
# P6 生产发布 Worker 与回滚验收
|
||||
|
||||
## 结论
|
||||
|
||||
P6 的发布控制面、v0.7.1 三通道 Sidecar、Release Worker、归档、补偿和回滚代码已经完成,并通过本地真实依赖 E2E。
|
||||
|
||||
生产发布仍未授权、未配置、未启动、未执行:
|
||||
|
||||
```text
|
||||
PRODUCTION_RELEASE_REQUESTS_ENABLED=false
|
||||
RELEASE_WORKER_ENABLED=false
|
||||
persistent release_executions=0
|
||||
persistent release_item_executions=0
|
||||
production release jobs=0
|
||||
```
|
||||
|
||||
因此本文的“通过”表示实现和隔离验收通过,不表示生产已发布。
|
||||
|
||||
## 范围
|
||||
|
||||
```text
|
||||
P5 sealed batch
|
||||
→ 受保护的生产授权请求
|
||||
→ ReleaseExecution + release.publish Job 原子入账
|
||||
→ Release Worker 按 KB 路由
|
||||
→ 从测试 WeKnora 导出经过验收的同一 Knowledge
|
||||
→ PostgreSQL / Elasticsearch / 对象存储三通道发布
|
||||
→ 生产 WeKnora 官方 API 验收
|
||||
→ 成功记录 production revision
|
||||
```
|
||||
|
||||
回滚:
|
||||
|
||||
```text
|
||||
已完成的 publish execution
|
||||
→ 受保护的 rollback 授权
|
||||
→ release.rollback Job
|
||||
→ 载入旧 0600 release bundle
|
||||
→ 三通道恢复
|
||||
→ 官方 API 验收
|
||||
```
|
||||
|
||||
首次发布没有旧 revision 时,失败补偿和回滚采用禁用而不是删除:
|
||||
|
||||
```text
|
||||
PostgreSQL Knowledge enable_status=disabled
|
||||
PostgreSQL Chunks is_enabled=false
|
||||
Elasticsearch 文档保留但 is_enabled=false
|
||||
对象保留
|
||||
官方 knowledge-scoped Search 必须零命中
|
||||
```
|
||||
|
||||
## 人工授权与双门禁
|
||||
|
||||
发布请求门禁:
|
||||
|
||||
```text
|
||||
PRODUCTION_RELEASE_REQUESTS_ENABLED=false
|
||||
```
|
||||
|
||||
执行门禁:
|
||||
|
||||
```text
|
||||
RELEASE_WORKER_ENABLED=false
|
||||
```
|
||||
|
||||
两个门禁默认都关闭且职责不同:
|
||||
|
||||
- 请求门禁关闭时,授权 HTTP API 返回 403,Store 不会被调用。
|
||||
- 执行门禁关闭时,Release Worker 不加载 release config、不读取生产 secret、不连接生产数据面。
|
||||
- P5 的 `sealed` 仍不创建 Job。
|
||||
- 只有受保护的生产授权请求才会原子创建 ReleaseExecution 和 Job。
|
||||
|
||||
授权 HTTP API 不接受调用方提交的审核人身份。审核人来自服务端固定配置:
|
||||
|
||||
```text
|
||||
PRODUCTION_RELEASE_AUTHORIZER_REF
|
||||
```
|
||||
|
||||
门禁开启时还必须提供普通、非软链接、权限 0600 的 Bearer token 文件:
|
||||
|
||||
```text
|
||||
PRODUCTION_RELEASE_AUTH_TOKEN_FILE
|
||||
```
|
||||
|
||||
缺少固定 actor 或 token 文件时 API 拒绝启动。UI 不录入、不读取、不显示 token,并且没有“一键生产发布”按钮。
|
||||
|
||||
固定确认语句:
|
||||
|
||||
```text
|
||||
publish-to-production
|
||||
rollback-production-release
|
||||
retry-production-release
|
||||
```
|
||||
|
||||
## API
|
||||
|
||||
```text
|
||||
POST /api/v1/release-batches/{batch_id}/authorize-production
|
||||
POST /api/v1/release-executions/{execution_id}/authorize-rollback
|
||||
POST /api/v1/release-executions/{execution_id}/retry
|
||||
GET /api/v1/release-capabilities
|
||||
GET /api/v1/release-executions
|
||||
GET /api/v1/release-executions/{execution_id}
|
||||
```
|
||||
|
||||
授权响应和只读审计响应不返回:
|
||||
|
||||
- authorization token
|
||||
- authorization_code
|
||||
- bundle SHA
|
||||
- WeKnora API key
|
||||
- Elasticsearch / PostgreSQL / 对象存储凭据或 URL
|
||||
- 测试 KB 内部连接信息
|
||||
|
||||
审计响应显示:
|
||||
|
||||
- publish / rollback
|
||||
- pending / running / completed / failed
|
||||
- 固定授权人
|
||||
- 总项数、成功数、失败数
|
||||
- 每项固定错误码
|
||||
- production revision / previous production revision
|
||||
- 开始和完成时间
|
||||
|
||||
## Migration 7
|
||||
|
||||
新增:
|
||||
|
||||
```text
|
||||
release_executions
|
||||
release_item_executions
|
||||
```
|
||||
|
||||
约束:
|
||||
|
||||
- operation 只允许 `publish` / `rollback`。
|
||||
- target_environment 固定为 `production`。
|
||||
- state 使用固定枚举。
|
||||
- publish execution 每个 sealed batch 唯一。
|
||||
- rollback execution 每个 prior publish execution 唯一。
|
||||
- Job 外键、逐项 Publication 外键和计数约束保留审计关联。
|
||||
- Migration 7 不回填历史 release execution,也不自动创建 Job。
|
||||
|
||||
往返验收:
|
||||
|
||||
```text
|
||||
up 7 → down 6 → up 7
|
||||
```
|
||||
|
||||
Migration 7 应用前备份:
|
||||
|
||||
```text
|
||||
tmp/backups/governance-pre-migration7-20260906T133742+0800.dump
|
||||
bytes=263001
|
||||
mode=0600
|
||||
pg_restore catalog=valid
|
||||
```
|
||||
|
||||
## Release Worker 状态机
|
||||
|
||||
批次 Job 在 Worker 中按每批最多50项展开,并持久记录当前批次/总批次;批内逐项执行:
|
||||
|
||||
```text
|
||||
pending execution
|
||||
→ running
|
||||
→ 为 batch items 幂等建立 release_item_executions
|
||||
→ 每个 item 执行 publish 或 rollback
|
||||
→ verified / rolled_back / failed
|
||||
→ execution completed 或 failed
|
||||
```
|
||||
|
||||
行为:
|
||||
|
||||
- 一项永久失败会立即停止本项之后的同批项目与全部后续批次;未执行项记录固定`release_stopped_after_failure`,随后仅对已验证成功项安排自动回滚。
|
||||
- 可重试故障在重试次数耗尽前保留重试语义,不把尚未终态的项目误标为永久失败。
|
||||
- 可重试失败在 Job 还有 attempt 时返回固定 `release_retryable`。
|
||||
- 最后一次 attempt 仍失败时,逐项写固定错误码并收口,不留下永远 running 的 Execution。
|
||||
- 外部响应正文、对象 key、路径、凭据和错误正文不写入治理库。
|
||||
- Worker 重启后跳过已经 verified / rolled_back 的项。
|
||||
- 终态failed execution可由同一Bearer门禁和固定`retry-production-release`确认重排;只重置failed items,verified / rolled_back项保持完成,Job从completed/failed回pending并从attempt 0重新开始。
|
||||
- publish重排前重新验证P5 sealed snapshot;过期审核或Publication变化时返回冲突。
|
||||
- 未知 KB 不会落入默认适配器,而是 `release_contract_mismatch`。
|
||||
|
||||
## v0.7.1 release bundle 契约
|
||||
|
||||
固定:
|
||||
|
||||
```text
|
||||
format weknora-release-bundle/v1
|
||||
WeKnora v0.7.1
|
||||
migration 74
|
||||
retriever elasticsearch_v8
|
||||
embedding dimension 1024
|
||||
```
|
||||
|
||||
每个 bundle 绑定:
|
||||
|
||||
- ReleaseExecution ID
|
||||
- Publication ID
|
||||
- SourceItemVersion ID
|
||||
- source revision
|
||||
- parse revision(parser generation + index generation)
|
||||
- release revision(封批时锁定的 review ID)
|
||||
- tenant / KB / Knowledge ID
|
||||
- embedding model identity
|
||||
- Elasticsearch mapping SHA-256
|
||||
- 各 PostgreSQL 表行数和 canonical SHA-256
|
||||
- ES 文档数和 canonical SHA-256
|
||||
- 资产数量、总字节和 canonical SHA-256
|
||||
- 整包 SHA-256
|
||||
|
||||
任何 revision、模型、维度、KB、tenant、mapping 或 bundle digest 不一致都在目标写入前 fail-closed。
|
||||
|
||||
## PostgreSQL 通道
|
||||
|
||||
只允许以下 v0.7.1 内容表:
|
||||
|
||||
```text
|
||||
knowledges
|
||||
chunks
|
||||
knowledge_tags
|
||||
knowledge_tag_relations
|
||||
resources
|
||||
resource_bindings
|
||||
```
|
||||
|
||||
没有读取或复制:
|
||||
|
||||
- users / sessions / messages
|
||||
- tenants / memberships / permissions
|
||||
- API keys / model provider credentials
|
||||
- vector store connection credentials
|
||||
- Jobs / audit logs / chat history
|
||||
|
||||
导出范围同时绑定:
|
||||
|
||||
```text
|
||||
tenant_id + knowledge_base_id + knowledge_id
|
||||
```
|
||||
|
||||
目标导入:
|
||||
|
||||
- 必须是 migration 74 且 dirty=false。
|
||||
- 使用 per-Knowledge PostgreSQL advisory transaction lock。
|
||||
- 单 Knowledge 单事务替换。
|
||||
- ID 在其他 tenant / KB 已存在时拒绝。
|
||||
- Tag 定义和 relation 一起复制;共享 Tag 不删除。
|
||||
- Resource stable handle 保留。
|
||||
- production storage_backend_id、provider、storage URI 和 location_hash 按目标配置重写。
|
||||
- Chunk seq_id 冲突时整个事务失败,不静默改 ID。
|
||||
- 不调用 WeKnora 解析和 Embedding。
|
||||
|
||||
## Elasticsearch 通道
|
||||
|
||||
源导出:
|
||||
|
||||
- tenant / KB / Knowledge 三重 term filter。
|
||||
- 原样保存 `_id`、chunk文本、向量和其他 `_source` 字段。
|
||||
- mapping 只计算 `mappings` canonical SHA,不把索引名混入指纹。
|
||||
- 导出后按 `_id` canonical排序。
|
||||
- 单 Knowledge 超过 10000 个 ES 文档时 fail-closed,不截断。
|
||||
|
||||
目标替换:
|
||||
|
||||
1. 先 bulk upsert 新文档,`refresh=true`。
|
||||
2. 再只删除同 tenant / KB / Knowledge 下、不属于新 release 的旧 `_id`。
|
||||
3. 读回数量和 canonical SHA。
|
||||
4. 失败时自动恢复 prior SearchSnapshot。
|
||||
|
||||
检索禁用使用 knowledge-scoped `_update_by_query` 将 `is_enabled=false`,不删除 ES 文档,并读回确认文档仍在且全部禁用。
|
||||
|
||||
## 对象存储通道
|
||||
|
||||
支持 S3-compatible MinIO / S3 / OSS:
|
||||
|
||||
- secret 仅从 0600 文件读取。
|
||||
- 不在 URL 中携带 token。
|
||||
- 支持 v0.7.1 `storage://<backend-id>/<provider-path>`。
|
||||
- 拒绝 traversal、query、fragment、控制字符和越界 locator。
|
||||
- 导出时核对 resource size / content SHA。
|
||||
- staging:`.governance-release-stage/<release_revision>/<relative-key>`。
|
||||
- staging 写入后做完整流式 SHA-256。
|
||||
- Copy 到 final 相同 relative key 后再次完整流式 SHA-256。
|
||||
- ObjectStore 接口没有 Delete 方法。
|
||||
- 回滚重传旧 bundle 中的资产;不删除新资产。
|
||||
|
||||
## Release bundle 归档
|
||||
|
||||
每个 KB 使用独立目录:
|
||||
|
||||
```text
|
||||
<archive_root>/<knowledge_base_id>/
|
||||
```
|
||||
|
||||
要求:
|
||||
|
||||
- archive root 和 KB 子目录是真实 0700 目录。
|
||||
- 不允许目录或文件软链接。
|
||||
- bundle 文件为普通 0600 文件。
|
||||
- 原子写、文件 fsync、目录 fsync。
|
||||
- 相同 release revision + 相同 SHA 可幂等重写。
|
||||
- 相同 revision + 不同 SHA 为碰撞并拒绝。
|
||||
- 加载时重验 manifest、PG、ES、资产和整包 SHA。
|
||||
- 默认每 bundle 最大 1 GiB。
|
||||
|
||||
Bundle payload 可能包含对象 locator 和资产内容,因此不能进入 Web/API/普通日志;只有摘要进入治理库。
|
||||
|
||||
## Release Worker 配置
|
||||
|
||||
模板:
|
||||
|
||||
```text
|
||||
config/release.v071.example.json
|
||||
```
|
||||
|
||||
Runtime 配置文件必须:
|
||||
|
||||
- 使用绝对路径;
|
||||
- 是普通文件;
|
||||
- 不是软链接;
|
||||
- 权限为 0600;
|
||||
- JSON 不允许未知字段或尾随数据;
|
||||
- 每个 KB 只允许一个 route;
|
||||
- source 与 target 的数据库值、ES endpoint、对象 endpoint+bucket 必须不同;
|
||||
- 所有数据库URL、ES token、对象AK/SK、WeKnora API key 只通过各自 0600 文件引用。
|
||||
|
||||
模板不能直接启用生产。启用前必须根据真实数据面生成独立受限配置和 Compose override,并由人工授权。
|
||||
|
||||
## 本地真实 E2E
|
||||
|
||||
### 精确 schema
|
||||
|
||||
使用:
|
||||
|
||||
```text
|
||||
ParadeDB PostgreSQL 17
|
||||
WeKnora v0.7.1 migrations 000000..000074
|
||||
source DB + target DB
|
||||
```
|
||||
|
||||
结果:
|
||||
|
||||
```text
|
||||
source migration=74
|
||||
目标 migration=74
|
||||
合成 source:
|
||||
knowledges=1
|
||||
chunks=1
|
||||
knowledge_tags=1
|
||||
knowledge_tag_relations=1
|
||||
resources=2
|
||||
resource_bindings=1
|
||||
```
|
||||
|
||||
通过:
|
||||
|
||||
```text
|
||||
TestPostgresExporterV071Integration
|
||||
TestPostgresImporterV071ReplacesAndRestoresDocument
|
||||
```
|
||||
|
||||
覆盖:
|
||||
|
||||
```text
|
||||
old → new → restore old
|
||||
disable keeps 1 Knowledge + 1 Chunk
|
||||
Knowledge=disabled
|
||||
chunk enabled count=0
|
||||
```
|
||||
|
||||
### 对象存储
|
||||
|
||||
使用本机已有的 disposable MinIO,无卷、loopback随机端口、合成凭据和合成bucket。
|
||||
|
||||
通过:
|
||||
|
||||
```text
|
||||
TestS3ObjectStoreIntegration
|
||||
```
|
||||
|
||||
覆盖:
|
||||
|
||||
```text
|
||||
Open storage:// source
|
||||
Put staging
|
||||
streaming SHA Stat
|
||||
Copy final
|
||||
streaming SHA Stat
|
||||
```
|
||||
|
||||
### 三通道发布与回滚
|
||||
|
||||
通过:
|
||||
|
||||
```text
|
||||
TestThreeLaneReleaseAndRollbackV071Integration
|
||||
```
|
||||
|
||||
链路:
|
||||
|
||||
```text
|
||||
publish review-release-1
|
||||
→ publish review-release-2
|
||||
→ rollback review-release-1
|
||||
→ target PG chunks=1
|
||||
→ target ES has es-release
|
||||
→ target two assets non-empty
|
||||
```
|
||||
|
||||
说明:
|
||||
|
||||
- PostgreSQL 和 MinIO 为真实 disposable 服务。
|
||||
- Elasticsearch 为状态化 HTTP contract server,真实执行 mapping/search/bulk/delete-by-query/update-by-query 协议和 digest 回读。
|
||||
- WeKnora verifier 使用现有官方 client 接口 contract stub,真实执行 GetKnowledge/ListChunks/SearchKnowledge 的判断逻辑;没有连接生产 WeKnora。
|
||||
- 所有 disposable 容器最终均已停止并由 `--rm` 删除。
|
||||
|
||||
### Worker装配
|
||||
|
||||
通过:
|
||||
|
||||
```text
|
||||
TestBuildReleaseRuntimeWithRestrictedFilesIntegration
|
||||
```
|
||||
|
||||
验证:
|
||||
|
||||
```text
|
||||
两个真实 v0.7.1 DB连接
|
||||
source/target secret文件0600
|
||||
独立ES endpoint
|
||||
独立对象bucket
|
||||
独立官方WeKnora endpoint
|
||||
KB archive 0700
|
||||
RoutingPublisher构建
|
||||
关闭时两个DB pool均释放
|
||||
```
|
||||
|
||||
## Web验收
|
||||
|
||||
Firefox Selenium只读验收:
|
||||
|
||||
```text
|
||||
review_candidates=5
|
||||
release_batches=0
|
||||
release_executions=0
|
||||
production_request_gate=false
|
||||
unauthorized_release_controls=0
|
||||
```
|
||||
|
||||
截图:
|
||||
|
||||
```text
|
||||
tmp/P6_RELEASE_AUDIT_FIREFOX.png
|
||||
```
|
||||
|
||||
可见:
|
||||
|
||||
- 左侧发布管理导航和顶部标签。
|
||||
- P5审核候选与封批记录。
|
||||
- P6生产执行审计。
|
||||
- 生产请求门禁关闭。
|
||||
- Release Worker保持关闭。
|
||||
- 当前无生产执行记录。
|
||||
- 页面没有生产授权按钮。
|
||||
|
||||
## 当前持久治理库
|
||||
|
||||
```text
|
||||
schema migrations=1,2,3,4,5,6,7
|
||||
publication_reviews=0
|
||||
release_batches=0
|
||||
release_batch_items=0
|
||||
release_executions=0
|
||||
release_item_executions=0
|
||||
release publish/rollback jobs=0
|
||||
```
|
||||
|
||||
现有 5 条候选仍为知识工程师业务待审,未由测试代替人工批准。
|
||||
|
||||
## 生产前仍需人工完成
|
||||
|
||||
以下不属于本地自动验收,也未执行:
|
||||
|
||||
1. 明确批准生产发布窗口。
|
||||
2. 对测试和生产各选一份同ID真实Knowledge,做只读PG/ES/OSS依赖盘点。
|
||||
3. 确认两端均是WeKnora v0.7.1 migration 74、1024维、同模型和同mapping。
|
||||
4. 为source创建只读数据面凭据。
|
||||
5. 为target创建仅允许约定KB内容表、ES索引和对象bucket的最小写权限凭据。
|
||||
6. 创建0600 release config和secret文件,并让Release Worker独占挂载。
|
||||
7. 先做1份无敏感canary。
|
||||
8. 官方API读取、Chunks、检索、资产和审计全部通过后再扩批。
|
||||
9. 保留Release Worker关闭状态,直到上述材料由人工核对。
|
||||
|
||||
不得因为本文完成而自动执行生产发布、创建生产Job、连接生产数据面、删除任何WeKnora Knowledge或对象。
|
||||
@@ -0,0 +1,13 @@
|
||||
# 发布准备模拟版验收
|
||||
|
||||
入口:http://100.92.49.78:8088/ → 发布管理
|
||||
|
||||
页面将旧8步流程收敛为:选择文档→检查数据闭包→锁定清单→生产负责人授权。
|
||||
|
||||
当前候选及数量均明确标为本地模拟。规则引擎已验证:切片/向量/文件/Wiki/图谱均可独立验证时推荐文档级;Wiki或图谱共享时强制整知识库且不能跨KB;解析失败、切片缺失、向量数不一致、文件缺失、Wiki/图谱缺失时阻断。每行显示闭包完整、共享需整库或闭包不完整。
|
||||
|
||||
生产备份到OSS尚未配置,生产授权关闭,Release Worker关闭;只能生成浏览器内模拟清单,不创建治理批次、Job或生产任务。真实适配器目前仅支持Knowledge/Chunk/标签/资源/对象和Elasticsearch文档;Wiki实际导出、Neo4j图谱、整知识库导出和OSS生产备份尚未完成,不能将模拟闭包称为真实可发布。
|
||||
|
||||
验证:发布规划纯规则测试3项、页面测试2项通过;真实Firefox完成文档级选择、模拟清单、共享图谱升级整知识库、清空和失败项禁选;1440/1024/768无页面级横向溢出;外部写入0;生产能力开关false。
|
||||
|
||||
证据:`tmp/release-planner-simulation-acceptance.json`、`tmp/RELEASE_PLANNER_SIMULATION_FIREFOX.png`。
|
||||
@@ -0,0 +1,47 @@
|
||||
# 真实多层来源目录验收
|
||||
|
||||
入口:http://100.92.49.78:8088/(Tailscale内)
|
||||
|
||||
## 交付范围
|
||||
|
||||
- 首页选择来源后默认只显示第一层文件夹;根目录文件收起并显示数量,可主动展开。
|
||||
- 点击文件夹只请求当前目录的直接子项,不把全量目录树塞进浏览器。
|
||||
- 顶部工作标签显示真实来源/目录路径,可逐级返回。
|
||||
- 文件夹优先于文件,显示递归文件数、大小、治理范围、已导入/已解析、处理中和失败数。
|
||||
- 当前为本地只读快照,页面明确标注,不冒充实时来源连接。
|
||||
- 来源文件名中的控制字符转换为可见符号,目录身份不变;不读取或投影宿主机`local_path`。
|
||||
|
||||
## 真实来源
|
||||
|
||||
- 钉钉“公司级技术支持知识库”:4548个节点、1079个实际文件夹、3468份文件;第一层8个目录,另有5份根目录文件默认收起。
|
||||
- 钉钉“技术诊断部知识沉淀”:2173个节点、404个实际文件夹、1768份文件。
|
||||
- AList“国内工业”:6569个节点、1688个实际文件夹、4880份文件;第一层严格为01公司资料、02产品资料、03行业洞察、04销售赋能、05流程制度、06通用模板。
|
||||
- AList同时导入海外工业、国内教育、海外教育三个完整只读快照。
|
||||
- PP资料下载中心:以真实`potalTags.tagType`构造8个第一层入口;613份已治理公开文档去重计数。597份进入至少一个标签目录,16份没有这8类标签,作为根目录文件默认收起。一个文件可出现在多个标签目录,但总数只计算一次。
|
||||
|
||||
总计:7个来源、38986个目录节点、9880个含来源根节点的文件夹节点、28796份跨来源文件。
|
||||
|
||||
## 数据与安全
|
||||
|
||||
Migration 9新增不可变目录快照、节点和当前快照指针。导入流程为:构建安全快照→完整校验→单事务插入→切换当前指针。没有删除旧快照,没有修改来源,没有调用钉钉/PP/AList写入、上传、移动、重命名或删除接口。
|
||||
|
||||
目录API仅投影治理连接ID、节点ID、名称、类型和聚合状态;不返回来源外部ID、来源定位哈希、宿主机路径或快照指纹。
|
||||
|
||||
## 验证
|
||||
|
||||
- Python目录构建测试:7/7通过;完整真实快照在独立PostgreSQL库导入,数据库38986节点与构建结果一致。
|
||||
- Go全项目测试、相关vet通过;目录API独立PostgreSQL测试验证直接子项、文件夹优先、面包屑、状态聚合及未知节点404。
|
||||
- Web:14个测试文件、58项测试通过;TypeScript、Vite构建通过。
|
||||
- Firefox真实验收:公司级知识库3层导航、PP三层导航、AList两层导航;首层目录数8/8/6;根目录文件5/16默认收起;DOM与API直接子项完全一致;1440/1024/768无页面级横向溢出;写请求0。
|
||||
- 视觉复核:顶部路径无重复、范围标签不拆字、递归文件数口径明确、单页不显示无效分页。
|
||||
- Migration=9;原source_items/content_objects/versions/test_publications保持618/535/618/535;活动任务0;范围规则0;文件覆盖0。
|
||||
- API、数据库健康;所有服务运行、零重启且运行镜像匹配;本机和Tailscale入口HTTP 200。
|
||||
- 生产发布请求仍关闭,Release Worker仍关闭。
|
||||
|
||||
证据:
|
||||
- `tmp/source-catalog-acceptance.json`
|
||||
- `tmp/SOURCE_CATALOG_FIREFOX.png`
|
||||
- `tmp/backups/pre-source-catalog-20260908T223638.dump`(299195字节,0600,pg_restore目录检查通过)
|
||||
- 回退镜像:`knowledge-governance-api:before-source-catalog`、`knowledge-governance-web:before-source-catalog`
|
||||
|
||||
未包含:来源实时登录/刷新、钉钉/AList文件到治理`source_items`的全量映射、原文预览。这些属于后续已确认的“来源链接”和解析/导入阶段,不能以目录快照冒充。
|
||||
@@ -0,0 +1,44 @@
|
||||
# 来源目录实时刷新与分流验收
|
||||
|
||||
## 页面功能
|
||||
|
||||
- 进入子目录后显示“返回上一级”;顶部来源/目录标签仍可逐级返回。
|
||||
- 右下角显示当前来源最近一次快照检查/激活时间。
|
||||
- 每个文件显示不选择/对内/对外;有安全本地制品的文件可单条标注并立即进入对应本地WeKnora解析队列。
|
||||
- 当前页支持勾选、全选可入队文件、批量标注对内/对外。零选择按钮显式灰色禁用。
|
||||
- 目录元数据存在但没有可验证文件制品时,分流控件禁用并显示“只有目录元数据,尚无可上传文件”;不会假装入队。
|
||||
- 分流结果显示标注数、新入队、已在队列及暂不可入队数量。
|
||||
|
||||
## 本地WeKnora目标
|
||||
|
||||
新增并读回验证两个隔离本地知识库:
|
||||
- `governance-internal-local`:对内解析队列。
|
||||
- `governance-external-local`:对外解析队列。
|
||||
|
||||
API只返回两个范围是否启用,不把目标知识库ID交给浏览器。文件路由在单个PostgreSQL事务中写人工范围覆盖和审计、创建目标Publication及`test.upload` Job。重复标注同一文件/目标返回already_queued,不重复上传。
|
||||
|
||||
从PP公开Manifest选择最小的626字节TXT,按Manifest字节数和SHA-256复核后复制到治理Spool,并补齐一个content_object的受控spool引用。该文件分别通过对外和对内队列真实上传到本地WeKnora,两个Knowledge均达到`parse_status=completed`。测试对象保留,未调用删除API。
|
||||
|
||||
## 实时快照
|
||||
|
||||
点击“实时刷新目录”创建持久`source.catalog.refresh` Job。普通Worker运行只读扫描器,Job payload只保存来源治理ID、类型、名称、阶段、进度和计数,不保存URL、Token、Cookie或捕获请求。网页轮询Job,可以切换页面;完成后自动读取当前快照。
|
||||
|
||||
阶段:等待后台刷新→验证登录→扫描目录→校验父子关系和数量→写不可变快照/切换当前指针→完成。失败不会切换当前快照。
|
||||
|
||||
真实验收:
|
||||
- 钉钉公司级技术支持知识库:浏览器捕获会话有效;扫描1080个文件夹;新快照4524节点、1079目录、3444文件;原快照3468文件,证明读取的是实时来源变化;新快照原子激活,旧快照保留。
|
||||
- PP:当前凭据文件可读取且网络可达,但登录返回无Token;网页显示“PP登录失败,请在来源链接更新凭据;旧快照仍然保留”。
|
||||
- AList直连:保留各来源原地址,仅向`POST /api/fs/list`添加`x-api-token`;拒绝所有重定向,不依赖钉钉SSO Cookie。2026-09-10 实测四个来源均完成只读全量快照并原子激活:国内工业1718目录/4942文件、海外工业1486目录/4463文件、国内教育3566目录/9067文件、海外教育1742目录/5015文件。
|
||||
|
||||
Worker只读挂载源脚本和来源目录;DingTalk/PP/AList凭据副本位于治理`secrets/`、权限0600,仅普通Worker挂载,API/Web/Release Worker不挂载。AList直连令牌从`secrets/alist_api_token`读取,不进入来源地址表、任务载荷、数据库或日志。
|
||||
|
||||
## 验证证据
|
||||
|
||||
- Python测试12项、Web 77项、Go全项目测试和相关vet通过。
|
||||
- 独立PostgreSQL验证:文件标注、人工范围审计、目标Publication、Job入队、重复幂等和未映射阻断。
|
||||
- 浏览器:返回、快照时间、实时任务完成卡、单条选择、当前页批量、幂等结果、灰态、1440/1024/768无页面级横向溢出。
|
||||
- `tmp/catalog-routing-refresh-acceptance.json`
|
||||
- `tmp/CATALOG_ROUTING_REFRESH_FIREFOX.png`
|
||||
- `tmp/backups/pre-live-refresh-routing-20260909T170939.dump`(3607159字节、0600、pg_restore目录检查通过)
|
||||
|
||||
生产发布请求和Release Worker保持关闭;未连接或写入生产WeKnora、生产数据库或OSS。
|
||||
@@ -0,0 +1,28 @@
|
||||
# 来源链接与系统设置模拟版验收
|
||||
|
||||
入口:http://100.92.49.78:8088/
|
||||
|
||||
## 来源链接
|
||||
|
||||
使用真实7个来源名称、类型、文件/目录快照数量。钉钉展示专用浏览器扫码登录流程,PP展示公开访问,AList展示只读Token;AList站点地址与起始目录分离。模拟验证包含网络、认证、根目录、第一层目录、只读边界五项。
|
||||
|
||||
当前本地模式不连接登录服务、不保存或接收真实Key;Key输入禁用。页面明确不下载文件内容,只读取目录和元数据,并禁止上传、移动、重命名、删除。视觉复核确认模拟状态不冒充实时连接。
|
||||
|
||||
## 系统设置
|
||||
|
||||
六区:当前环境、WeKnora环境、解析器与模型、发布目标、OSS备份、监控与安全。文档工程师仅能在浏览器内调整模拟刷新/停滞参数;管理员端点、凭据、映射、OSS和生产开关全部锁定。测试/生产WeKnora分开;生产写入、生产发布请求均关闭;未接入的Provider明确标待配置或未接入。
|
||||
|
||||
## 验证
|
||||
|
||||
- 当前全Web:20个测试文件、72项测试通过;TypeScript和Vite构建通过。
|
||||
- 来源链接Firefox:7个来源、钉钉/PP/AList三类流程、Key禁用、五项模拟验证、1440/1024/768无页面级横向溢出;外部写入0。
|
||||
- 系统设置Firefox:六区、四项安全模拟设置、管理员按钮禁用、密码输入0、生产开关false、1440/1024/768无页面级横向溢出;外部写入0。
|
||||
- 视觉复核:两页均为文档/内容治理页面,不是开发控制台;无严重遮挡或横向溢出。
|
||||
|
||||
证据:
|
||||
- `tmp/source-link-simulation-acceptance.json`
|
||||
- `tmp/SOURCE_LINK_SIMULATION_FIREFOX.png`
|
||||
- `tmp/system-settings-simulation-acceptance.json`
|
||||
- `tmp/SYSTEM_SETTINGS_SIMULATION_FIREFOX.png`
|
||||
|
||||
真实模式待部署测试服务器后实施:管理员认证、HTTPS、CSRF、Secret单向替换、钉钉扫码会话、AList真实只读Token验证、PP实时目录、WeKnora/模型/MinerU真实Provider、生产目标及OSS备份恢复验证。当前模拟页不能作为真实连接或生产可用证据。
|
||||
@@ -0,0 +1,38 @@
|
||||
# 来源目录批量标注口径(S6 冻结)
|
||||
|
||||
本文件冻结“来源与目录”页批量标注(处理策略)的产品口径。后续任何页面、API 或测试改动与本文件冲突时,必须先修改本文件并经主人确认,不能悄悄改变行为。
|
||||
|
||||
## 冻结规则
|
||||
|
||||
```text
|
||||
1. 文件和目录均可勾选。
|
||||
2. 批量选择仅提供“全选当前页”;不提供“递归全选当前目录”。
|
||||
3. 目录被单独勾选并标注后,策略递归作用于全部后代文件。
|
||||
4. 顶部批量策略按钮仅提供三个:对内、对外、禁止导入。
|
||||
5. 历史“对内 + 对外”(both)数据只读展示为“对内 + 对外”,
|
||||
顶部不提供该批量入口(P3/后续如要开放须先修订本文件)。
|
||||
6. “未选择/未设置”仅在状态列显示,不生成任何专属批量按钮;
|
||||
批量按钮只在存在勾选节点时可用。
|
||||
```
|
||||
|
||||
## 语义边界
|
||||
|
||||
- “标注”只写治理策略(`directory_scope_rules` / `source_item_scope_overrides`)与审计事件,
|
||||
不代表已上传、解析或发布。
|
||||
- 目录标注写入目录级规则并递归生效;文件级覆盖(override)优先于最近的目录规则,
|
||||
目录规则优先于来源默认可见性。
|
||||
- “禁止导入”表示从测试知识库移除(retrieval 进入 disabling),
|
||||
不删除来源文件,也不操作生产知识库。
|
||||
- 每次用户动作使用一个新的 `Idempotency-Key`;网络重试复用原键(见 OpenAPI C4 契约)。
|
||||
|
||||
## 验收口径
|
||||
|
||||
1. UI:`catalog-policy-actions` 区域内恰好 3 个策略按钮(对内/对外/禁止导入);
|
||||
页面不出现“递归全选当前目录”入口;不出现可选择的“对内 + 对外”批量控件。
|
||||
2. Store 集成:标注一级目录后,其二级目录子项与深层文件的 effective visibility
|
||||
均递归变为该策略;进入子目录、返回父目录、刷新读回结果一致。
|
||||
3. 历史 both 行在文件表中仍显示“对内 + 对外”,只读。
|
||||
|
||||
对应测试:
|
||||
- `web/src/SourceCatalogBrowser.test.tsx`
|
||||
- `internal/platform/postgres/s6_scope_policy_integration_test.go`
|
||||
@@ -0,0 +1,289 @@
|
||||
# 知识治理平台状态机与术语契约
|
||||
|
||||
## 目的
|
||||
|
||||
本文件冻结治理平台的状态维度、存储枚举、API枚举、页面中文和终态含义。一个对象不得用单一`status`混合来源变化、人工审核、上传、解析、检索和发布状态。
|
||||
|
||||
原则:
|
||||
|
||||
```text
|
||||
数据库/API使用稳定英文枚举。
|
||||
页面使用中文业务文案。
|
||||
completed是当前统一的“成功完成”内部终态;页面可显示“成功”或“已完成”。
|
||||
unknown、空字符串、未配置不能伪装成healthy或completed。
|
||||
未来状态先写在“保留值”,未对应迁移和实现前不得持久化。
|
||||
```
|
||||
|
||||
## State dimensions
|
||||
|
||||
| 维度 | 负责回答的问题 | 当前主要对象 |
|
||||
| --- | --- | --- |
|
||||
| Scope policy | 资料希望进入哪里? | 目录规则、文件覆盖规则 |
|
||||
| Source change | 来源事实发生了什么? | SourceItem、CatalogChangeItem |
|
||||
| Review | 人工是否批准? | PublicationReview |
|
||||
| Content | 原始内容是否取得和验证? | ContentObject、SourceItemVersion |
|
||||
| Job | 后台执行器是否仍在处理? | jobs |
|
||||
| Ingest | 是否已提交到测试目标? | TestPublication |
|
||||
| Processing | WeKnora解析/索引是否完成? | TestPublication |
|
||||
| Retrieval | 是否允许被检索? | TestPublication/目标系统 |
|
||||
| Acceptance | 测试结果是否通过? | AcceptanceResult |
|
||||
| Release | 生产发布批次/执行进行到哪里? | ReleaseBatch、ReleaseExecution |
|
||||
| Settings | 设置版本能否使用? | UISettingsRevision |
|
||||
|
||||
### Job
|
||||
|
||||
Current storage/API values:
|
||||
|
||||
```text
|
||||
pending | running | retry | completed | failed | cancelled
|
||||
```
|
||||
|
||||
| 值 | 页面中文 | 含义 | 是否终态 |
|
||||
| --- | --- | --- | --- |
|
||||
| `pending` | 等待执行 | 已创建,等待Worker领取 | 否 |
|
||||
| `running` | 执行中 | 已被Worker持有有效租约 | 否 |
|
||||
| `retry` | 等待重试 | 上次失败但仍可按退避重新执行 | 否 |
|
||||
| `completed` | 已完成 | 本地Job处理器完成;不自动等于目标系统最终成功 | 是 |
|
||||
| `failed` | 执行失败 | 达到最大尝试次数或遇到不可重试错误 | 是 |
|
||||
| `cancelled` | 已取消 | 被明确取消,不得再被Worker领取 | 是 |
|
||||
|
||||
规则:
|
||||
|
||||
```text
|
||||
Worker只领取pending、retry或过期running任务。
|
||||
cancelled任务不可以被重新领取。
|
||||
completed仅代表该Job完成;test.upload完成后仍必须由test.poll确认Processing终态。
|
||||
```
|
||||
|
||||
### Scope policy
|
||||
|
||||
Current user-facing desired-policy vocabulary:
|
||||
|
||||
```text
|
||||
unset | internal | external | blocked
|
||||
```
|
||||
|
||||
| 值 | 页面中文 | 当前持久化解释 |
|
||||
| --- | --- | --- |
|
||||
| `unset` | 未设置 | 没有显式目录/文件覆盖规则;当前有效范围由来源默认或历史数据决定。它不是可写的`Visibility`数据库值。 |
|
||||
| `internal` | 对内 | 期望进入对内测试知识库。 |
|
||||
| `external` | 对外 | 期望进入对外测试知识库,后续仍需人工审核。 |
|
||||
| `blocked` | 禁止导入 | 期望不再导入;不得因此自动删除生产知识。 |
|
||||
|
||||
historical compatibility value: both
|
||||
|
||||
`both`是历史兼容持久化值,页面只读显示“对内 + 对外”。目录页顶部不提供新增`both`操作。`mixed`和`unknown`是目录汇总显示值,不是文件期望策略。
|
||||
|
||||
### Source change
|
||||
|
||||
| 存储/API值 | 页面中文 | 含义 |
|
||||
| --- | --- | --- |
|
||||
| `unchanged` | 无变化 | 与基线相比无来源差异。 |
|
||||
| `added` | 新增 | 来源新发现。 |
|
||||
| `updated` | 已更新 | 现有来源身份出现新修订或新内容证据。 |
|
||||
| `deleted` | 已删除 | 仅限权威来源明确删除。 |
|
||||
| `unavailable` | 来源不可用 | 权限、接口或扫描失败;绝不是删除信号。 |
|
||||
|
||||
Catalog ChangeSet使用不同的动作值:`add`、`update`、`move_scope`、`delete`、`no_change`。它是比较计划动作,不直接等同于`SourceItem.source_change`。
|
||||
|
||||
### Review
|
||||
|
||||
```text
|
||||
pending | approved | rejected | not_required
|
||||
```
|
||||
|
||||
| 值 | 页面中文 | 含义 |
|
||||
| --- | --- | --- |
|
||||
| `pending` | 待审核 | 必须等待文档工程师判断。 |
|
||||
| `approved` | 已批准 | 可以进入后续封批条件判断,不代表生产已发布。 |
|
||||
| `rejected` | 已驳回 | 不得自动重新入批。 |
|
||||
| `not_required` | 无需审核 | 仅适用于明确规则允许的情形。 |
|
||||
|
||||
### Content
|
||||
|
||||
```text
|
||||
unknown | downloading | verified | invalid | over_limit | failed
|
||||
```
|
||||
|
||||
| 值 | 页面中文 | 含义 |
|
||||
| --- | --- | --- |
|
||||
| `unknown` | 内容待获取 | 尚未取得可验证字节。 |
|
||||
| `downloading` | 正在获取 | 正在受控下载/导出。 |
|
||||
| `verified` | 内容已校验 | 大小、格式、内容SHA-256已验证。 |
|
||||
| `invalid` | 内容无效 | 格式或文件魔数不符。 |
|
||||
| `over_limit` | 超出限制 | 内容存在但超出当前导入策略。 |
|
||||
| `failed` | 获取失败 | 内容获取或验证失败。 |
|
||||
|
||||
### Ingest
|
||||
|
||||
```text
|
||||
pending | uploading | uploaded | failed
|
||||
```
|
||||
|
||||
| 值 | 页面中文 | 含义 |
|
||||
| --- | --- | --- |
|
||||
| `pending` | 等待上传 | 尚未提交目标。 |
|
||||
| `uploading` | 上传中 | 已开始向测试目标提交。 |
|
||||
| `uploaded` | 已上传 | 目标已接收对象;不代表解析完成。 |
|
||||
| `failed` | 上传失败 | 上传步骤失败。 |
|
||||
|
||||
### Processing
|
||||
|
||||
```text
|
||||
pending | processing | finalizing | completed | failed | stalled
|
||||
```
|
||||
|
||||
| 值 | 页面中文 | 含义 |
|
||||
| --- | --- | --- |
|
||||
| `pending` | 等待解析 | 目标尚未开始处理。 |
|
||||
| `processing` | 解析中 | 正在解析、切片或向量化。 |
|
||||
| `finalizing` | 索引与增强中 | 基础解析可能完成,索引/Wiki/图谱仍未完成。 |
|
||||
| `completed` | 解析完成 | 远端最终处理完成;仍需独立Acceptance。 |
|
||||
| `failed` | 解析失败 | 远端报告失败。 |
|
||||
| `stalled` | 处理停滞 | 超过配置阈值未见进展。 |
|
||||
|
||||
### Retrieval
|
||||
|
||||
```text
|
||||
unpublished | enabled | disabling | disabled | enabling | failed
|
||||
```
|
||||
|
||||
| 值 | 页面中文 | 含义 |
|
||||
| --- | --- | --- |
|
||||
| `unpublished` | 未发布 | 尚未允许检索。 |
|
||||
| `enabled` | 已启用检索 | 已读回确认允许检索。 |
|
||||
| `disabling` | 正在停用检索 | 请求已发出,尚未确认。 |
|
||||
| `disabled` | 已停止检索 | 已读回确认停止检索,可恢复。 |
|
||||
| `enabling` | 正在恢复检索 | 请求已发出,尚未确认。 |
|
||||
| `failed` | 检索状态异常 | 停用/恢复失败或读回失败。 |
|
||||
|
||||
### Acceptance
|
||||
|
||||
```text
|
||||
pending | passed | failed | manual_required
|
||||
```
|
||||
|
||||
| 值 | 页面中文 | 含义 |
|
||||
| --- | --- | --- |
|
||||
| `pending` | 等待验收 | 尚未检查。 |
|
||||
| `passed` | 验收通过 | 指定检查通过。 |
|
||||
| `failed` | 验收失败 | 指定检查失败。 |
|
||||
| `manual_required` | 需人工确认 | 自动检查不足以得出结论。 |
|
||||
|
||||
### Release
|
||||
|
||||
当前批次状态:
|
||||
|
||||
```text
|
||||
draft | sealed | cancelled
|
||||
```
|
||||
|
||||
当前执行状态:
|
||||
|
||||
```text
|
||||
pending | running | completed | failed
|
||||
```
|
||||
|
||||
当前执行条目状态:
|
||||
|
||||
```text
|
||||
pending | staged | activated | verified | failed | rolled_back
|
||||
```
|
||||
|
||||
规则:
|
||||
|
||||
```text
|
||||
sealed只表示发布清单冻结,不表示已发布。
|
||||
ReleaseExecution.completed必须建立在逐项验证成功之上。
|
||||
ReleaseExecution.failed不自动表示整体生产已恢复;需按R4回滚策略判断。
|
||||
```
|
||||
|
||||
### Settings
|
||||
|
||||
```text
|
||||
validated | active | retired | failed
|
||||
```
|
||||
|
||||
| 值 | 页面中文 | 含义 |
|
||||
| --- | --- | --- |
|
||||
| `validated` | 已验证待启用 | 配置校验通过,尚未成为运行版本。 |
|
||||
| `active` | 当前活动版本 | 运行时读取的唯一设置版本。 |
|
||||
| `retired` | 已退役 | 被后续活动版本替代,保留审计。 |
|
||||
| `failed` | 验证失败 | 配置未通过,不得启用。 |
|
||||
|
||||
## Transition rules
|
||||
|
||||
### Job transitions
|
||||
|
||||
```text
|
||||
pending → running
|
||||
running → completed | retry | failed | cancelled
|
||||
retry → running | cancelled
|
||||
pending → cancelled
|
||||
```
|
||||
|
||||
`completed`、`failed`、`cancelled`均为终态。后续“重试”必须创建或重置受控Attempt,不能把已完成任务静默改回pending。
|
||||
|
||||
### Scope policy transitions
|
||||
|
||||
```text
|
||||
unset → internal | external | blocked
|
||||
internal | external | blocked → internal | external | blocked
|
||||
both只能保留和读取,不由当前目录页新增。
|
||||
```
|
||||
|
||||
Scope改变只更新期望状态;S7完成前不得把它视为立即上传或删除命令。
|
||||
|
||||
### Processing transitions
|
||||
|
||||
```text
|
||||
pending → processing → finalizing → completed
|
||||
pending | processing | finalizing → failed | stalled
|
||||
stalled → processing(仅由确认重试后的远端进展恢复)
|
||||
```
|
||||
|
||||
### Release transitions
|
||||
|
||||
当前实现只允许:
|
||||
|
||||
```text
|
||||
pending → running → completed | failed
|
||||
```
|
||||
|
||||
## Reserved future values
|
||||
|
||||
以下值已在产品术语中保留,但当前数据库、API和UI不得写入:
|
||||
|
||||
```text
|
||||
rolling_back
|
||||
rolled_back
|
||||
rollback_failed
|
||||
snapshotting
|
||||
verifying
|
||||
applying
|
||||
```
|
||||
|
||||
`rolling_back` must not be persisted before R4。
|
||||
|
||||
R1、R2、R4完成后,Release状态机会扩展这些值,并新增Production Snapshot与Release ChangeSet关联。
|
||||
|
||||
## C2验收边界
|
||||
|
||||
C2完成:
|
||||
|
||||
```text
|
||||
分维度状态术语冻结
|
||||
completed与页面“成功”含义统一
|
||||
Job cancelled枚举、数据库约束和终态语义补齐
|
||||
历史both策略的兼容边界明确
|
||||
未来发布状态不提前持久化
|
||||
```
|
||||
|
||||
C2不完成:
|
||||
|
||||
```text
|
||||
取消Job的用户界面/API(后续按业务需求另立工作包)
|
||||
SourceFile内容Hash、ChangeSet确认、生产快照和三级回滚
|
||||
Scope策略的实际执行链(S7/P3)
|
||||
Release rolling_back实际状态(R4)
|
||||
```
|
||||
@@ -0,0 +1,29 @@
|
||||
# 本地模拟WeKnora监控验收
|
||||
|
||||
入口:http://100.92.49.78:8088/ → 解析监控
|
||||
|
||||
当前模式严格标记为`simulation`。页面黄色提示、资源卡和文件名均显示模拟属性,不代表测试服务器真实状态。
|
||||
|
||||
## 模拟内容
|
||||
|
||||
- CPU 42%;内存18.6/32.0GB。
|
||||
- WeKnora API、Embedding模型、LLM模型、MinerU连通卡;均为模拟探测及模拟延迟。
|
||||
- 公司级技术支持知识库:已导入5750;completed 4530、pending 450、finalizing 714、processing 50、failed 6,状态合计5750。
|
||||
- 对内已解析/已导入3300/4200;对外1230/1550。
|
||||
- 五种状态各有一份明确以`[模拟]`开头的文件样例,可按状态筛选。
|
||||
- finalizing明确表示基础解析/切片/向量通常完成,Wiki/摘要/图谱仍在生成,不能算全部完成。
|
||||
|
||||
## 验证
|
||||
|
||||
- Go simulation provider与HTTP安全投影测试通过;响应不包含URL、Key、Authorization或Secret。
|
||||
- Web监控客户端和页面测试通过;当前全Web为16个测试文件、62项测试通过;TypeScript和Vite构建通过。
|
||||
- Firefox验证:模拟标识、资源卡、状态合计、对内外计数、finalizing与failed筛选、文件名和错误码均通过;写请求0;1440/1024/768无页面级横向溢出。
|
||||
- 视觉复核无严重遮挡或横向溢出。
|
||||
|
||||
证据:
|
||||
- `tmp/weknora-monitor-simulation-acceptance.json`
|
||||
- `tmp/WEKNORA_MONITOR_SIMULATION_FIREFOX.png`
|
||||
|
||||
## 后续真实模式边界
|
||||
|
||||
治理平台部署到测试服务器后,用同一MonitorProvider接口替换模拟Provider:本机资源读取真实CPU/内存,WeKnora使用认证API汇总Knowledge parse_status,模型使用有时间戳的低频实际探测,MinerU同时检查服务自身和WeKnora到MinerU的路径。切换前模拟数据不能称为真实运行状态。
|
||||
@@ -0,0 +1,159 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="zh-CN">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>解析监控 · 方向A 紧凑表格版 [预览稿·模拟数据]</title>
|
||||
<style>
|
||||
:root{--line:#e5e7eb;--ink:#111827;--sub:#6b7280;--bg:#f7f8fa;--fail:#b91c1c;--ok:#0b6e4f;--warn:#b45309;--run:#1d4ed8}
|
||||
*{box-sizing:border-box}
|
||||
body{margin:0;background:var(--bg);color:var(--ink);font:14px/1.5 "PingFang SC","Microsoft YaHei",sans-serif}
|
||||
header{display:flex;align-items:center;gap:14px;padding:12px 20px;background:#fff;border-bottom:1px solid var(--line)}
|
||||
header h1{font-size:17px;margin:0}
|
||||
.meta{color:var(--sub);font-size:12px}
|
||||
button{font:inherit;padding:5px 12px;border:1px solid var(--line);border-radius:6px;background:#fff;cursor:pointer}
|
||||
button.primary{background:#111827;color:#fff;border-color:#111827}
|
||||
button.danger{color:var(--fail)}
|
||||
.preview-tag{margin-left:auto;background:#fef3c7;color:#92400e;border:1px solid #fcd34d;font-size:12px;padding:2px 10px;border-radius:999px}
|
||||
main{max-width:1400px;margin:0 auto;padding:14px 20px;display:flex;flex-direction:column;gap:14px}
|
||||
section{background:#fff;border:1px solid var(--line);border-radius:8px}
|
||||
.shead{display:flex;align-items:center;gap:10px;padding:10px 14px;border-bottom:1px solid var(--line)}
|
||||
.shead h2{font-size:14px;margin:0}
|
||||
.shead .spacer{flex:1}
|
||||
.counts{display:flex;gap:6px;flex-wrap:wrap}
|
||||
.count{font-size:12px;padding:2px 9px;border-radius:999px;border:1px solid var(--line);color:var(--sub)}
|
||||
.count b{color:var(--ink)}
|
||||
.count.wait b{color:var(--warn)}.count.run b{color:var(--run)}.count.ok b{color:var(--ok)}.count.fail b{color:var(--fail)}
|
||||
table{width:100%;border-collapse:collapse;font-size:13px}
|
||||
th{color:var(--sub);font-weight:500;text-align:left;font-size:12px;padding:7px 14px;border-bottom:1px solid var(--line);white-space:nowrap}
|
||||
td{padding:8px 14px;border-bottom:1px solid #f1f3f6;vertical-align:middle}
|
||||
tr:last-child td{border-bottom:none}
|
||||
tr.fail-row{background:#fef8f8}
|
||||
tr:hover td{background:#fafbfc}
|
||||
.badge{font-size:12px;padding:1px 8px;border-radius:4px;border:1px solid var(--line);white-space:nowrap}
|
||||
.badge.add{color:var(--ok)}.badge.update,.badge.move{color:var(--run)}.badge.move_scope{color:var(--warn)}.badge.delete{color:var(--fail)}
|
||||
.badge.in{background:#eef6ff;border-color:#bfdbfe}.badge.out{background:#f0fdf4;border-color:#bbf7d0}
|
||||
.s{font-size:12px;white-space:nowrap}.s.failed{color:var(--fail)}.s.completed{color:var(--ok)}.s.processing{color:var(--run)}.s.pending{color:var(--sub)}.s.stalled{color:var(--warn)}
|
||||
code{font:12px ui-monospace,Consolas,monospace;color:var(--sub)}
|
||||
.tiny{font-size:12px;color:var(--sub)}
|
||||
input[type=text]{font:inherit;padding:5px 8px;border:1px solid var(--line);border-radius:6px;width:120px}
|
||||
.grid2{display:grid;grid-template-columns:1fr 1fr;gap:8px 20px;font-size:13px}
|
||||
footer{display:flex;gap:0;overflow-x:auto;background:#fff;border:1px solid var(--line);border-radius:8px;font-size:12px}
|
||||
footer div{padding:8px 16px;border-right:1px solid var(--line);white-space:nowrap}
|
||||
footer div:last-child{border-right:none}
|
||||
footer .bad{color:var(--fail)}footer .good{color:var(--ok)}
|
||||
#drawer{position:fixed;top:0;right:-420px;width:400px;height:100%;background:#fff;border-left:1px solid var(--line);box-shadow:-4px 0 18px rgba(0,0,0,.06);transition:right .18s;padding:16px;z-index:9;overflow:auto}
|
||||
#drawer.open{right:0}
|
||||
#drawer h3{margin:0 0 4px;font-size:15px;word-break:break-all}
|
||||
#drawer .close{float:right;border:none;background:none;font-size:18px;cursor:pointer;color:var(--sub)}
|
||||
.timeline{list-style:none;margin:14px 0;padding:0;border-left:2px solid var(--line)}
|
||||
.timeline li{position:relative;padding:0 0 16px 18px;font-size:13px}
|
||||
.timeline li::before{content:'';position:absolute;left:-6px;top:4px;width:10px;height:10px;border-radius:50%;background:#fff;border:2px solid var(--sub)}
|
||||
.timeline li.done::before{border-color:var(--ok);background:var(--ok)}
|
||||
.timeline li.bad::before{border-color:var(--fail);background:var(--fail)}
|
||||
.timeline li time{display:block;font-size:12px;color:var(--sub)}
|
||||
.err{background:#fef2f2;border:1px solid #fecaca;color:var(--fail);font-size:12px;padding:8px 10px;border-radius:6px;margin:8px 0}
|
||||
.dl{display:grid;grid-template-columns:110px 1fr;gap:6px;font-size:13px;margin-top:12px}
|
||||
.dl dt{color:var(--sub)}.dl dd{margin:0;word-break:break-all}
|
||||
@media(max-width:1024px){.grid2{grid-template-columns:1fr}}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<header>
|
||||
<h1>解析监控</h1>
|
||||
<span class="meta">快照 2026-09-12 08:00 · 下次自动 08:05</span>
|
||||
<span class="meta">活跃 Worker <b>2</b> · 等待 <b style="color:var(--warn)">3</b> · 运行 <b style="color:var(--run)">4</b> · 失败 <b style="color:var(--fail)">2</b></span>
|
||||
<button class="spacer" style="margin-left:auto">刷新</button>
|
||||
<span class="preview-tag">预览稿 · 模拟数据 · 方向A</span>
|
||||
</header>
|
||||
<main>
|
||||
|
||||
<section>
|
||||
<div class="shead"><h2>快照差异 → 解析队列 → 死信 → 资源(单页主线,表格为主)</h2>
|
||||
<div class="counts spacer">
|
||||
<span class="count wait">待确认 <b>3</b></span><span class="count run">处理中 <b>4</b></span>
|
||||
<span class="count ok">成功 <b>4530</b></span><span class="count fail">失败 <b>2</b></span>
|
||||
</div>
|
||||
<label class="tiny">筛选
|
||||
<select><option>对内+对外</option><option>对内</option><option>对外</option></select>
|
||||
</label>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<div class="shead"><h2>1 · 快照差异(待确认置顶)</h2><span class="tiny">国内工业 · from 09-08 → to 09-12</span><span class="spacer"></span><span class="tiny">操作人</span><input type="text" value="姚工"></div>
|
||||
<table>
|
||||
<thead><tr><th style="width:34px"><input type="checkbox" checked></th><th>变更</th><th>文件</th><th>路径(旧 → 新)</th><th>目标</th><th>证据</th><th style="width:110px">操作</th></tr></thead>
|
||||
<tbody>
|
||||
<tr><td><input type="checkbox" checked></td><td><span class="badge add">新增</span></td><td><a href="#" onclick="openDrawer('CR20B用户手册V2.pdf');return false">CR20B用户手册V2.pdf</a></td><td class="tiny">— → 01|产品资料/CR20B/</td><td><span class="badge in">对内</span></td><td class="tiny">内容Hash 8e3f…(来源提供)</td><td><button class="primary" style="font-size:12px">确认处理</button></td></tr>
|
||||
<tr><td><input type="checkbox" checked></td><td><span class="badge update">更新</span></td><td><a href="#" onclick="openDrawer('DobotStudio固件说明.docx');return false">DobotStudio固件说明.docx</a></td><td class="tiny">同一文件,版本 r12 → r13</td><td><span class="badge in">对内</span><span class="badge out">对外</span></td><td class="tiny">内容Hash变化(真更新)</td><td><button class="primary" style="font-size:12px">确认处理</button></td></tr>
|
||||
<tr><td><input type="checkbox" checked></td><td><span class="badge move_scope">范围变化</span></td><td><a href="#" onclick="openDrawer('越疆售后FAQ.xlsx');return false">越疆售后FAQ.xlsx</a></td><td class="tiny">对内 → 对内+对外</td><td><span class="badge out">对外</span></td><td class="tiny">Hash未变,仅标注</td><td><button class="primary" style="font-size:12px">确认处理</button></td></tr>
|
||||
<tr><td>—</td><td><span class="badge move">移动</span></td><td>安装指南.pdf</td><td class="tiny">01|产品资料/ → 快速入门/安装指南.pdf</td><td><span class="badge in">对内</span></td><td class="tiny">同名同Hash → 自动改名,不重解析</td><td class="tiny">已自动处理</td></tr>
|
||||
<tr class="fail-row"><td>—</td><td><span class="badge delete">删除</span></td><td>旧手册.pdf</td><td class="tiny">02|已下架资料/旧手册.pdf →</td><td>—</td><td class="tiny">来源缺失</td><td class="tiny">等待移除确认</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<div class="shead"><h2>2 · 解析队列(失败/停滞置顶)</h2><span class="tiny">共 4536 · 1小时:成功 88.9% · p95 30分钟</span><span class="spacer"></span><input type="text" value="姚工"><button class="danger">批量重试(2)</button></div>
|
||||
<table>
|
||||
<thead><tr><th style="width:34px"></th><th>状态</th><th>阶段链</th><th>文件</th><th>目标</th><th>错误码</th><th>已停留</th><th>重试</th></tr></thead>
|
||||
<tbody>
|
||||
<tr class="fail-row"><td><input type="checkbox" checked></td><td class="s failed">失败</td><td class="tiny">✓✓✓✗···· <b style="color:var(--fail)">文档解析</b></td><td><a href="#" onclick="openDrawer('CR20A.pdf');return false">CR20A.pdf</a></td><td><span class="badge out">对外</span></td><td><code>validation_failed</code></td><td>—</td><td>1</td></tr>
|
||||
<tr class="fail-row"><td><input type="checkbox" checked></td><td class="s failed">失败</td><td class="tiny">✓✓✓✓✓✗· <b style="color:var(--fail)">索引增强</b></td><td><a href="#" onclick="openDrawer('MG400碰撞检测.mp4笔记.pdf');return false">MG400碰撞检测笔记.pdf</a></td><td><span class="badge in">对内</span></td><td><code>remote_permanent</code></td><td>—</td><td>5/5 耗尽 · 死信</td></tr>
|
||||
<tr><td>—</td><td class="s processing">解析中</td><td class="tiny">✓✓✓✗···· 文档解析</td><td>XVision标定手册.pdf</td><td><span class="badge in">对内</span></td><td>—</td><td>6分钟</td><td>0</td></tr>
|
||||
<tr><td>—</td><td class="s pending">等待</td><td class="tiny">等待接单</td><td>NeoSs常见问题.docx</td><td><span class="badge out">对外</span></td><td>—</td><td>2分钟</td><td>0</td></tr>
|
||||
<tr><td>—</td><td class="s completed">成功</td><td class="tiny">✓✓✓✓✓✓✓ 完成</td><td>CR5说明书V3.pdf</td><td><span class="badge in">对内</span></td><td>—</td><td>完成</td><td>2</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<div class="shead"><h2>3 · 失败队列 DLQ(2条开放死信)</h2><span class="tiny">仅投影;原始任务与审计不可变。重试=新尝试</span></div>
|
||||
<table>
|
||||
<thead><tr><th>错误码</th><th>类型</th><th>文件</th><th>尝试</th><th>进入死信</th><th></th></tr></thead>
|
||||
<tbody>
|
||||
<tr><td><code>remote_permanent</code></td><td>上传解析</td><td>MG400碰撞检测笔记.pdf</td><td>5/5</td><td class="tiny">09-11 21:04</td><td><button>重试</button></td></tr>
|
||||
<tr><td><code>artifact_invalid</code></td><td>上传解析</td><td>测试固件包.zip.bin</td><td>1/1</td><td class="tiny">09-12 02:11</td><td class="tiny">文件校验永久失败,需来源修复</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</section>
|
||||
|
||||
<footer>
|
||||
<div>10s <b class="good">100%</b></div>
|
||||
<div>1小时 · 成功 <b class="good">88.9%</b> 失败 11.1%</div>
|
||||
<div>24小时 · 成功 <b class="good">96.4%</b></div>
|
||||
<div>耗时 p50 30秒 · p95 30分钟 · 均值 6.9分钟</div>
|
||||
<div>CPU <b>42%</b> 内存 18.6/32GB</div>
|
||||
<div>WeKnora <b class="good">可达</b></div>
|
||||
<div>MinerU <b class="bad">容量已满(429)</b></div>
|
||||
<div>PostgreSQL <b class="good">可接单</b></div>
|
||||
</footer>
|
||||
</main>
|
||||
|
||||
<div id="drawer">
|
||||
<button class="close" onclick="document.getElementById('drawer').classList.remove('open')">✕</button>
|
||||
<h3 id="drawer-title">CR20A.pdf</h3>
|
||||
<p class="tiny">对内+对外 · kb-external · 版本 r7 · 发布 tp-8841</p>
|
||||
<div class="err">validation_failed:解析服务返回 422,切片数量=0。建议:确认 MinerU 接单容量后批量重试。</div>
|
||||
<ul class="timeline">
|
||||
<li class="done">下载完成<time>08:01:02 · 3.2MB</time></li>
|
||||
<li class="done">内容Hash校验 8e3f…d41a<time>08:01:04</time></li>
|
||||
<li class="done">上传 WeKnora<time>08:01:07 · 成功</time></li>
|
||||
<li class="bad">文档解析<time>08:03:40 · 失败(validation_failed)</time></li>
|
||||
<li>索引与增强(未执行)</li>
|
||||
</ul>
|
||||
<dl class="dl">
|
||||
<dt>内容Hash</dt><dd>8e3f19…d41a(来源提供)</dd>
|
||||
<dt>文件身份</dt><dd>来源稳定ID ext-9931 · 置信度 official_id</dd>
|
||||
<dt>最近尝试</dt><dd>job-3f9a… · 第2次 · 164秒</dd>
|
||||
<dt>死信</dt><dd>否(仍可重试)</dd>
|
||||
</dl>
|
||||
<p style="margin-top:16px"><button class="danger" style="width:100%">重试该文件</button></p>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
function openDrawer(name){document.getElementById('drawer-title').textContent=name;document.getElementById('drawer').classList.add('open')}
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,148 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="zh-CN">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>解析监控 · 方向B 任务收件箱版 [预览稿·模拟数据]</title>
|
||||
<style>
|
||||
:root{--line:#e5e7eb;--ink:#111827;--sub:#6b7280;--bg:#f7f8fa;--fail:#b91c1c;--ok:#0b6e4f;--warn:#b45309;--run:#1d4ed8}
|
||||
*{box-sizing:border-box}
|
||||
body{margin:0;background:var(--bg);color:var(--ink);font:14px/1.5 "PingFang SC","Microsoft YaHei",sans-serif}
|
||||
header{display:flex;align-items:center;gap:14px;padding:12px 20px;background:#fff;border-bottom:1px solid var(--line)}
|
||||
header h1{font-size:17px;margin:0}
|
||||
.meta{color:var(--sub);font-size:12px}
|
||||
button{font:inherit;padding:5px 12px;border:1px solid var(--line);border-radius:6px;background:#fff;cursor:pointer}
|
||||
button.primary{background:#111827;color:#fff;border-color:#111827}
|
||||
button.danger{color:var(--fail)}
|
||||
.preview-tag{margin-left:auto;background:#fef3c7;color:#92400e;border:1px solid #fcd34d;font-size:12px;padding:2px 10px;border-radius:999px}
|
||||
main{max-width:1400px;margin:0 auto;padding:14px 20px;display:grid;grid-template-columns:250px 1fr;gap:14px;align-items:start}
|
||||
nav.inbox{background:#fff;border:1px solid var(--line);border-radius:8px;position:sticky;top:14px}
|
||||
nav.inbox h2{font-size:13px;margin:0;padding:10px 14px;border-bottom:1px solid var(--line);color:var(--sub)}
|
||||
nav.inbox a{display:flex;justify-content:space-between;align-items:center;padding:9px 14px;border-bottom:1px solid #f1f3f6;text-decoration:none;color:var(--ink);font-size:13px}
|
||||
nav.inbox a.active{background:#eef2ff;border-left:3px solid #111827;padding-left:11px;font-weight:600}
|
||||
nav.inbox .n{font-size:12px;padding:0 8px;border-radius:999px;background:#f3f4f6;color:var(--sub)}
|
||||
nav.inbox a.blocked .n{background:#fee2e2;color:var(--fail)}
|
||||
nav.inbox a.okall .n{background:#dcfce7;color:var(--ok)}
|
||||
.work{display:flex;flex-direction:column;gap:14px}
|
||||
section{background:#fff;border:1px solid var(--line);border-radius:8px}
|
||||
.shead{display:flex;align-items:center;gap:10px;padding:10px 14px;border-bottom:1px solid var(--line)}
|
||||
.shead h3{font-size:14px;margin:0}
|
||||
.shead .spacer{flex:1}
|
||||
.task{display:grid;grid-template-columns:28px 1fr auto;gap:12px;padding:12px 14px;border-bottom:1px solid #f1f3f6;align-items:start}
|
||||
.task:last-child{border-bottom:none}
|
||||
.task input{margin-top:4px}
|
||||
.task h4{margin:0;font-size:14px}
|
||||
.task .why{font-size:12px;color:var(--sub);margin-top:2px}
|
||||
.task .chain{margin-top:8px;display:flex;gap:4px;align-items:center;font-size:11px;color:var(--sub)}
|
||||
.step{padding:2px 8px;border-radius:4px;border:1px solid var(--line);background:#fafbfc}
|
||||
.step.done{background:#f0fdf4;border-color:#bbf7d0;color:var(--ok)}
|
||||
.step.bad{background:#fef2f2;border-color:#fecaca;color:var(--fail);font-weight:600}
|
||||
.arrow{color:#d1d5db}
|
||||
.act{display:flex;flex-direction:column;gap:6px;align-items:flex-end}
|
||||
.badge{font-size:12px;padding:1px 8px;border-radius:4px;border:1px solid var(--line);white-space:nowrap}
|
||||
.badge.add{color:var(--ok)}.badge.update{color:var(--run)}.badge.move_scope{color:var(--warn)}.badge.delete{color:var(--fail)}
|
||||
.badge.in{background:#eef6ff;border-color:#bfdbfe}.badge.out{background:#f0fdf4;border-color:#bbf7d0}
|
||||
.errline{margin-top:6px;font-size:12px;color:var(--fail)}
|
||||
code{font:12px ui-monospace,Consolas,monospace;color:var(--sub)}
|
||||
.tiny{font-size:12px;color:var(--sub)}
|
||||
input[type=text]{font:inherit;padding:5px 8px;border:1px solid var(--line);border-radius:6px;width:110px}
|
||||
footer{display:flex;overflow-x:auto;background:#fff;border:1px solid var(--line);border-radius:8px;font-size:12px}
|
||||
footer div{padding:8px 16px;border-right:1px solid var(--line);white-space:nowrap}
|
||||
footer div:last-child{border-right:none}
|
||||
footer .bad{color:var(--fail)}footer .good{color:var(--ok)}
|
||||
.empty{padding:26px;text-align:center;color:var(--sub);font-size:13px}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<header>
|
||||
<h1>解析监控 · 收件箱</h1>
|
||||
<span class="meta">快照 2026-09-12 08:00</span>
|
||||
<span class="meta">活跃 Worker <b>2</b> · 队列 等待3/运行4/失败2</span>
|
||||
<button style="margin-left:auto">刷新</button>
|
||||
<span class="preview-tag">预览稿 · 模拟数据 · 方向B</span>
|
||||
</header>
|
||||
<main>
|
||||
<nav class="inbox">
|
||||
<h2>今天需要我做什么(3)</h2>
|
||||
<a class="active blocked" href="#">需人工处理<span class="n">2</span></a>
|
||||
<a href="#">等待我确认的差异<span class="n">1</span></a>
|
||||
<a href="#">处理中<span class="n">4</span></a>
|
||||
<a class="okall" href="#">今日完成<span class="n">38</span></a>
|
||||
<a href="#">死信(不可自动重试)<span class="n">1</span></a>
|
||||
</nav>
|
||||
|
||||
<div class="work">
|
||||
<section>
|
||||
<div class="shead"><h3>1 个任务需要你出手</h3><span class="spacer"></span><span class="tiny">操作人</span><input type="text" value="姚工"><button class="danger">批量重试所选</button></div>
|
||||
|
||||
<div class="task">
|
||||
<input type="checkbox" checked>
|
||||
<div>
|
||||
<h4>CR20A.pdf <span class="badge out">对外</span> <span class="badge update">更新</span></h4>
|
||||
<div class="why">为什么失败:文档解析返回 422(切片数量=0)· 错误码 <code>validation_failed</code> · 第2次尝试 · 停留164秒。同类文件今天已成功 12 个,可能是该文件本身问题。</div>
|
||||
<div class="chain"><span class="step done">下载 ✓</span><span class="arrow">→</span><span class="step done">Hash ✓</span><span class="arrow">→</span><span class="step done">OSS ✓</span><span class="arrow">→</span><span class="step bad">文档解析 ✗</span><span class="arrow">→</span><span class="step">上传</span><span class="arrow">→</span><span class="step">索引</span><span class="arrow">→</span><span class="step">完成</span></div>
|
||||
<div class="errline">建议:先在 WeKnora 手动解析该文件确认格式,再决定是否重试。</div>
|
||||
</div>
|
||||
<div class="act"><button>详情/时间线</button><button class="primary">重试</button></div>
|
||||
</div>
|
||||
|
||||
<div class="task">
|
||||
<input type="checkbox" checked>
|
||||
<div>
|
||||
<h4>MG400碰撞检测笔记.pdf <span class="badge in">对内</span> <span class="badge add">新增</span></h4>
|
||||
<div class="why">为什么失败:索引增强被远端永久拒绝 <code>remote_permanent</code>(5/5 次尝试耗尽,已进入死信)。原因多为 MinerU 容量持续 429。</div>
|
||||
<div class="chain"><span class="step done">下载 ✓</span><span class="arrow">→</span><span class="step done">上传 ✓</span><span class="arrow">→</span><span class="step done">解析 ✓</span><span class="arrow">→</span><span class="step bad">索引 ✗ 429×5</span></div>
|
||||
<div class="errline">资源状态:MinerU 当前容量已满 —— 恢复接单后此任务可一键重新入队。</div>
|
||||
</div>
|
||||
<div class="act"><button>详情/时间线</button><button class="primary">重新入队</button></div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<div class="shead"><h3>等待我确认的差异(1)</h3><span class="spacer"></span><span class="tiny">国内工业 09-08 → 09-12 · 共 5 项变化,3 项自动可处理</span></div>
|
||||
<div class="task">
|
||||
<input type="checkbox" checked>
|
||||
<div>
|
||||
<h4>越疆售后FAQ.xlsx <span class="badge in">对内</span>→<span class="badge out">对外</span> <span class="badge move_scope">范围变化</span></h4>
|
||||
<div class="why">内容Hash未变,仅标注范围升级;确认后进入发布流程。历史审核:09-02 姚工 标记对内。</div>
|
||||
</div>
|
||||
<div class="act"><button>预览文件</button><button class="primary">确认</button><button>驳回</button></div>
|
||||
</div>
|
||||
<div class="task">
|
||||
<span></span>
|
||||
<div>
|
||||
<h4>安装指南.pdf <span class="badge move">移动(改名/换目录)</span></h4>
|
||||
<div class="why">同名同Hash证据充分 → 自动更新路径,不重解析。无需操作。</div>
|
||||
</div>
|
||||
<div class="act"><span class="tiny">已自动处理</span></div>
|
||||
</div>
|
||||
<div class="task">
|
||||
<span></span>
|
||||
<div>
|
||||
<h4>旧手册.pdf <span class="badge delete">删除</span></h4>
|
||||
<div class="why">来源侧消失;将随本批差异确认后从测试知识库移除(不删来源文件)。</div>
|
||||
</div>
|
||||
<div class="act"><span class="tiny">随批次确认</span></div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<div class="shead"><h3>处理中(4)</h3><span class="spacer"></span><span class="tiny">无需操作,仅供查看</span></div>
|
||||
<div class="task"><span></span><div><h4>XVision标定手册.pdf</h4><div class="chain"><span class="step done">下载 ✓</span><span class="arrow">→</span><span class="step done">上传 ✓</span><span class="arrow">→</span><span class="step bad" style="color:var(--run);border-color:#bfdbfe;background:#eff6ff">文档解析中 6分钟</span></div></div><div class="act"><button>详情</button></div></div>
|
||||
<div class="empty">其余 3 个任务正常排队/处理中…</div>
|
||||
</section>
|
||||
|
||||
<footer>
|
||||
<div>10s <b class="good">100%</b></div>
|
||||
<div>1小时·成功 <b class="good">88.9%</b></div>
|
||||
<div>24小时·成功 <b class="good">96.4%</b></div>
|
||||
<div>p50 30秒 · p95 30分钟</div>
|
||||
<div>CPU 42% · 内存 18.6/32GB</div>
|
||||
<div>WeKnora <b class="good">可达</b></div>
|
||||
<div>MinerU <b class="bad">429 容量满</b></div>
|
||||
<div>PostgreSQL <b class="good">可接单</b></div>
|
||||
</footer>
|
||||
</div>
|
||||
</main>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,165 @@
|
||||
<!doctype html>
|
||||
<html lang="zh-CN">
|
||||
<head>
|
||||
<meta charset="UTF-8"><meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>方向A · Data-console(解析监控)</title>
|
||||
<link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=IBM+Plex+Mono:wght@400;500;600&family=Inter:wght@400;500;600;700&family=Noto+Sans+SC:wght@400;500;700&display=swap">
|
||||
<style>
|
||||
*{box-sizing:border-box;margin:0;padding:0}
|
||||
body{font-family:Inter,"Noto Sans SC","PingFang SC",system-ui,sans-serif;background:#fafbfc;color:#0f1720;font-size:13px;line-height:1.5;-webkit-font-smoothing:antialiased}
|
||||
.mono{font-family:"IBM Plex Mono","SF Mono",Menlo,monospace;font-variant-numeric:tabular-nums}
|
||||
.app{display:grid;grid-template-columns:212px 1fr;min-height:100vh}
|
||||
.side{background:#0e1c2b;color:#9fb6c6;padding:18px 0;display:flex;flex-direction:column;border-right:1px solid #0a141f}
|
||||
.side .brand{padding:0 18px 16px;border-bottom:1px solid #1a2f44;display:flex;gap:10px;align-items:center}
|
||||
.side .brand .dot{width:26px;height:26px;background:#2dd4bf;color:#04211d;display:grid;place-items:center;font-weight:700;font-size:13px;border-radius:4px}
|
||||
.side .brand strong{color:#e8f2f8;font-size:13px;font-weight:600;letter-spacing:.02em}
|
||||
.side .brand span{display:block;font-size:11px;color:#5b7789;margin-top:1px}
|
||||
.side nav{padding:14px 8px;display:flex;flex-direction:column;gap:1px}
|
||||
.side nav a{color:#8aa2b3;text-decoration:none;padding:7px 10px;font-size:12.5px;border-radius:4px;display:flex;justify-content:space-between;align-items:center}
|
||||
.side nav a:hover{background:#152a3d;color:#d9e8f1}
|
||||
.side nav a.on{background:#16324a;color:#fff;border-left:2px solid #2dd4bf;padding-left:8px}
|
||||
.side nav a em{font-style:normal;font-family:"IBM Plex Mono",monospace;font-size:11px;color:#4d6a7d}
|
||||
.side nav a.on em{color:#7dd8c8}
|
||||
.side .foot{margin-top:auto;padding:14px 18px;border-top:1px solid #1a2f44;font-size:11px;color:#46647a}
|
||||
.side .foot .live{color:#34d399}
|
||||
|
||||
.main{display:flex;flex-direction:column;min-width:0}
|
||||
.top{height:44px;border-bottom:1px solid #e3e8ec;background:#fff;display:flex;align-items:center;gap:14px;padding:0 20px;font-size:12px;color:#57687a;position:sticky;top:0;z-index:5}
|
||||
.top .crumb{color:#9aa8b4}
|
||||
.top .crumb b{color:#0f1720;font-weight:600}
|
||||
.top .meta{margin-left:auto;display:flex;gap:18px;font-family:"IBM Plex Mono",monospace;font-size:11px}
|
||||
.top .meta span b{color:#0f1720;font-weight:500}
|
||||
.top .meta .warn{color:#b45309}
|
||||
.top .meta .ok{color:#047857}
|
||||
|
||||
.page{padding:22px 24px 40px;max-width:1240px;width:100%}
|
||||
.page h1{font-size:20px;font-weight:600;letter-spacing:-.01em;margin-bottom:4px}
|
||||
.page .sub{color:#57687a;font-size:12.5px;margin-bottom:20px;max-width:640px}
|
||||
|
||||
.tiles{display:grid;grid-template-columns:repeat(5,1fr);gap:1px;background:#e3e8ec;border:1px solid #e3e8ec;margin-bottom:22px}
|
||||
.tile{background:#fff;padding:12px 14px}
|
||||
.tile .lbl{font-size:11px;color:#57687a;margin-bottom:6px;letter-spacing:.02em}
|
||||
.tile .num{font-family:"IBM Plex Mono",monospace;font-size:24px;font-weight:600;line-height:1;color:#0f1720}
|
||||
.tile .delta{font-size:11px;margin-top:5px;font-family:"IBM Plex Mono",monospace}
|
||||
.tile .delta.up{color:#047857}.tile .delta.dn{color:#b91c1c}.tile .delta.flat{color:#9aa8b4}
|
||||
.tile.a-fail{border-top:2px solid #b91c1c}.tile.a-wait{border-top:2px solid #d97706}.tile.a-run{border-top:2px solid #2563eb}.tile.a-done{border-top:2px solid #059669}.tile.a-dead{border-top:2px solid #6b7280}
|
||||
|
||||
.sec-head{display:flex;align-items:baseline;gap:12px;margin:0 0 10px;padding-bottom:8px;border-bottom:1px solid #e3e8ec}
|
||||
.sec-head h2{font-size:13px;font-weight:600;letter-spacing:.02em}
|
||||
.sec-head .hint{font-size:11.5px;color:#8a98a4}
|
||||
.sec-head .act{margin-left:auto;display:flex;gap:8px}
|
||||
.btn{border:1px solid #d3dce3;background:#fff;color:#0f1720;padding:5px 11px;font-size:12px;border-radius:4px;cursor:pointer;font-weight:500}
|
||||
.btn:hover{border-color:#94a3b8;background:#f8fafc}
|
||||
.btn.pri{background:#0e1c2b;border-color:#0e1c2b;color:#fff}
|
||||
.btn.pri:hover{background:#1a2f44}
|
||||
|
||||
.task{border:1px solid #e3e8ec;background:#fff;margin-bottom:1px}
|
||||
.task:first-of-type{border-radius:6px 6px 0 0}
|
||||
.task:last-of-type{border-radius:0 0 6px 6px}
|
||||
.task .row{display:grid;grid-template-columns:20px minmax(0,1fr) auto;gap:12px;padding:12px 14px;align-items:start}
|
||||
.task .chk{margin-top:3px}
|
||||
.task h4{font-size:13px;font-weight:600;display:flex;gap:8px;align-items:center;flex-wrap:wrap}
|
||||
.scope{font-family:"IBM Plex Mono",monospace;font-size:10.5px;padding:1px 6px;border-radius:3px;border:1px solid #e3e8ec;color:#57687a;background:#f8fafc}
|
||||
.scope.in{color:#1d4ed8;border-color:#c7d7f2;background:#eff4fd}
|
||||
.scope.out{color:#047857;border-color:#bfe3d4;background:#eefaf4}
|
||||
.chg{font-size:10.5px;padding:1px 6px;border-radius:3px;font-weight:600}
|
||||
.chg.add{color:#047857;background:#d7f2e4}
|
||||
.chg.upd{color:#1d4ed8;background:#dbe7fd}
|
||||
.why{font-size:11.5px;color:#8a1c1c;margin-top:5px;font-family:"IBM Plex Mono",monospace}
|
||||
.why code{background:#fdecec;padding:0 5px;border-radius:3px;color:#b91c1c}
|
||||
.chain{display:flex;gap:3px;margin-top:8px;flex-wrap:wrap;font-family:"IBM Plex Mono",monospace;font-size:10.5px}
|
||||
.chain span{padding:2px 8px;border:1px solid #e3e8ec;background:#f8fafc;color:#8a98a4;border-radius:3px}
|
||||
.chain span.done{background:#eefaf4;border-color:#bfe3d4;color:#047857}
|
||||
.chain span.bad{background:#fdecec;border-color:#f2baba;color:#b91c1c;font-weight:600}
|
||||
.chain span.act{background:#eff4fd;border-color:#c7d7f2;color:#1d4ed8}
|
||||
.advice{font-size:11.5px;color:#57687a;margin-top:6px}
|
||||
.task .ops{display:flex;flex-direction:column;gap:6px;align-items:flex-end}
|
||||
.task-id{font-family:"IBM Plex Mono",monospace;font-size:10.5px;color:#9aa8b4}
|
||||
|
||||
.dlq{margin-top:22px;border:1px solid #e3e8ec;background:#fff;border-radius:6px;overflow:hidden}
|
||||
.dlq ul{list-style:none}
|
||||
.dlq li{display:grid;grid-template-columns:minmax(0,1fr) auto;gap:14px;padding:10px 14px;border-top:1px solid #f0f3f5;font-size:12px;align-items:center}
|
||||
.dlq li:first-child{border-top:0}
|
||||
.dlq .code{font-family:"IBM Plex Mono",monospace;font-size:11px;color:#b91c1c}
|
||||
.dlq .path{color:#8a98a4;font-size:11.5px;margin-top:2px}
|
||||
|
||||
.strip{margin-top:26px;border:1px solid #e3e8ec;background:#fff;display:flex;flex-wrap:wrap;font-family:"IBM Plex Mono",monospace;font-size:11px;border-radius:6px;overflow:hidden}
|
||||
.strip div{padding:9px 16px;border-right:1px solid #f0f3f5;color:#57687a}
|
||||
.strip div b{color:#0f1720;font-weight:500}
|
||||
.strip .bad{color:#b91c1c}.strip .good{color:#047857}
|
||||
.strip div:last-child{border-right:0}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="app">
|
||||
<aside class="side">
|
||||
<div class="brand"><div class="dot">治</div><div><strong>知识治理</strong><span>governance console</span></div></div>
|
||||
<nav>
|
||||
<a href="#">来源与目录 <em>8</em></a>
|
||||
<a href="#" class="on">解析监控 <em>3</em></a>
|
||||
<a href="#">发布管理 <em>2</em></a>
|
||||
<a href="#">系统设置</a>
|
||||
</nav>
|
||||
<div class="foot"><span class="live">●</span> worker-01 心跳 4s 前<br>快照 2026-09-12 10:41</div>
|
||||
</aside>
|
||||
<div class="main">
|
||||
<div class="top">
|
||||
<div class="crumb">解析监控 / <b>需人工处理</b></div>
|
||||
<div class="meta"><span>1h成功 <b class="warn">88.9%</b></span><span>24h成功 <b class="ok">96.4%</b></span><span>Worker <b>2</b></span><span>队列 <b>待 3 / 跑 4</b></span></div>
|
||||
</div>
|
||||
<div class="page">
|
||||
<h1>需人工处理</h1>
|
||||
<p class="sub">失败原因、处理链定位与建议直接放在卡片上。勾选后可批量重试,死信重新入队会保留原任务与全部审计行。</p>
|
||||
|
||||
<div class="tiles">
|
||||
<div class="tile a-fail"><div class="lbl">失败</div><div class="num">2</div><div class="delta dn">+2 · 今日</div></div>
|
||||
<div class="tile a-wait"><div class="lbl">停滞</div><div class="num">1</div><div class="delta dn">45分钟</div></div>
|
||||
<div class="tile a-run"><div class="lbl">处理中</div><div class="num">4</div><div class="delta flat">—</div></div>
|
||||
<div class="tile a-done"><div class="lbl">已完成</div><div class="num">48</div><div class="delta up">+6 · 今日</div></div>
|
||||
<div class="tile a-dead"><div class="lbl">死信</div><div class="num">1</div><div class="delta flat">可重入队</div></div>
|
||||
</div>
|
||||
|
||||
<div class="sec-head"><h2>失败任务</h2><span class="hint">3 条 · 按失败阶段排序</span><span class="act"><input placeholder="操作人" style="border:1px solid #d3dce3;border-radius:4px;padding:4px 8px;font-size:12px;width:110px"><button class="btn">仅看中英对照</button><button class="btn pri">批量重试所选 (2)</button></span></div>
|
||||
|
||||
<div class="task">
|
||||
<div class="row">
|
||||
<div class="chk"><input type="checkbox" checked></div>
|
||||
<div>
|
||||
<h4>CR20A 电气规格说明书.pdf <span class="scope out">对外</span> <span class="chg upd">更新</span></h4>
|
||||
<div class="why">错误码 <code>remote_parse_failed</code> · 第 2 次尝试 · 已停留 71.1小时 · 失败于 文档解析</div>
|
||||
<div class="chain"><span class="done">下载 ✓</span><span class="done">Hash ✓</span><span class="done">OSS ✓</span><span class="bad">文档解析 ✗</span><span>WeKnora 上传</span><span>索引</span><span>完成</span></div>
|
||||
<div class="advice">建议先在 WeKnora 手动解析该文件确认格式,再决定是否重试。 <span class="task-id">cs-1 · tp-failed</span></div>
|
||||
</div>
|
||||
<div class="ops"><button class="btn pri">重试</button></div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="task">
|
||||
<div class="row">
|
||||
<div class="chk"><input type="checkbox" checked></div>
|
||||
<div>
|
||||
<h4>伺服驱动器调试手册.docx <span class="scope in">对内</span> <span class="chg add">新增</span></h4>
|
||||
<div class="why">错误码 <code>mineru_timeout</code> · 第 1 次尝试 · 已停留 45分钟 · 失败于 索引 / 增强</div>
|
||||
<div class="chain"><span class="done">下载 ✓</span><span class="done">Hash ✓</span><span class="done">OSS ✓</span><span class="done">文档解析 ✓</span><span class="done">WeKnora 上传 ✓</span><span class="bad">索引 ✗</span><span>完成</span></div>
|
||||
<div class="advice">多为 MinerU 容量或超时问题,确认服务恢复接单后重试。 <span class="task-id">cs-3 · tp-044</span></div>
|
||||
</div>
|
||||
<div class="ops"><button class="btn pri">重试</button></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="sec-head" style="margin-top:26px"><h2>死信(尝试耗尽)</h2><span class="hint">1 条 · 不可自动重试</span></div>
|
||||
<div class="dlq"><ul>
|
||||
<li><div><b>对外知识库 · test.upload</b> <span class="code">release_manifest_mismatch</span><div class="path">cs-2 · CR1A 固件变更记录.pdf · 尝试 5/5 · 2026-09-10 14:22</div></div><button class="btn">重新入队</button></li>
|
||||
</ul></div>
|
||||
|
||||
<div class="strip">
|
||||
<div>MinerU <b class="good">可接单</b></div>
|
||||
<div>WeKnora 测试 <b class="good">可接单</b></div>
|
||||
<div>PostgreSQL <b class="good">12ms</b></div>
|
||||
<div>队列 <b>3 待 / 4 跑 / 0 重试</b></div>
|
||||
<div>CPU 42% · 内存 18.6/32.0GB</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,135 @@
|
||||
<!doctype html>
|
||||
<html lang="zh-CN">
|
||||
<head>
|
||||
<meta charset="UTF-8"><meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>方向B · Editorial ledger(解析监控)</title>
|
||||
<link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=IBM+Plex+Mono:wght@400;500&family=Noto+Serif+SC:wght@500;700&family=Noto+Sans+SC:wght@400;500&display=swap">
|
||||
<style>
|
||||
*{box-sizing:border-box;margin:0;padding:0}
|
||||
body{font-family:"Noto Sans SC","PingFang SC",system-ui,sans-serif;background:#f7f4ec;color:#1e1a14;font-size:13px;line-height:1.6;-webkit-font-smoothing:antialiased}
|
||||
.serif{font-family:"Noto Serif SC",serif}
|
||||
.mono{font-family:"IBM Plex Mono",monospace}
|
||||
.app{display:grid;grid-template-columns:220px 1fr;min-height:100vh}
|
||||
.side{background:#231f18;color:#b8ac98;padding:24px 0;display:flex;flex-direction:column}
|
||||
.side .brand{padding:0 20px 18px;border-bottom:1px solid #3d352a}
|
||||
.side .brand h1{font-family:"Noto Serif SC",serif;color:#f0e9da;font-size:17px;font-weight:700;letter-spacing:.06em}
|
||||
.side .brand span{display:block;font-size:10px;color:#7a6f5c;margin-top:4px;letter-spacing:.18em}
|
||||
.side nav{padding:18px 0;display:flex;flex-direction:column}
|
||||
.side nav a{color:#a3967f;text-decoration:none;padding:9px 20px;font-size:13px;border-left:3px solid transparent;display:flex;justify-content:space-between}
|
||||
.side nav a:hover{color:#e8e0cd;background:#2b261d}
|
||||
.side nav a.on{color:#f7f2e5;border-left-color:#c2410c;background:#2b261d;font-weight:500}
|
||||
.side nav a em{font-style:normal;font-family:"IBM Plex Mono",monospace;font-size:11px}
|
||||
.side .foot{margin-top:auto;padding:18px 20px;border-top:1px solid #3d352a;font-size:11px;color:#7a6f5c;line-height:1.7}
|
||||
.side .foot b{color:#d9a05b;font-weight:500}
|
||||
|
||||
.main{min-width:0}
|
||||
.page{max-width:880px;padding:38px 48px 60px}
|
||||
.doc-rule{height:3px;background:#1e1a14;margin-bottom:14px}
|
||||
.doc-rule.thin{height:1px;background:#b8ac98}
|
||||
header h1{font-family:"Noto Serif SC",serif;font-size:26px;font-weight:700;letter-spacing:.02em;line-height:1.2}
|
||||
header .standfirst{font-size:13.5px;color:#5c5342;margin-top:8px;max-width:560px}
|
||||
header .dateline{font-family:"IBM Plex Mono",monospace;font-size:11px;color:#8a7f68;margin-top:12px;padding-top:10px;border-top:1px solid #d5cbb4;display:flex;gap:18px}
|
||||
header .dateline b{color:#1e1a14;font-weight:500}
|
||||
|
||||
.ledger{display:grid;grid-template-columns:repeat(5,1fr);margin:26px 0 30px;border-top:1px solid #b8ac98;border-bottom:1px solid #b8ac98}
|
||||
.cell{padding:14px 4px;text-align:center;border-right:1px solid #ddd3ba}
|
||||
.cell:last-child{border-right:0}
|
||||
.cell .num{font-family:"IBM Plex Mono",monospace;font-size:27px;font-weight:500;line-height:1}
|
||||
.cell .lbl{font-size:11px;color:#6e6450;margin-top:6px;letter-spacing:.14em}
|
||||
.cell.f .num{color:#a1201e}.cell.w .num{color:#b45309}.cell.r .num{color:#1d4ed8}.cell.d .num{color:#166534}.cell.x .num{color:#6e6450}
|
||||
|
||||
h2.sec{font-family:"Noto Serif SC",serif;font-size:15px;font-weight:700;margin:26px 0 12px;display:flex;align-items:baseline;gap:12px}
|
||||
h2.sec .n{font-family:"IBM Plex Mono",monospace;font-size:11px;color:#a1201e;font-weight:400}
|
||||
h2.sec::after{content:"";flex:1;height:1px;background:#d5cbb4}
|
||||
|
||||
.entry{display:grid;grid-template-columns:44px minmax(0,1fr) auto;gap:18px;padding:14px 0 16px;border-bottom:1px solid #ddd3ba}
|
||||
.entry .idx{font-family:"Noto Serif SC",serif;font-size:20px;color:#a8a091;font-weight:700}
|
||||
.entry h3{font-family:"Noto Serif SC",serif;font-size:15px;font-weight:700;letter-spacing:.01em}
|
||||
.entry h3 .tag{font-family:"IBM Plex Mono",monospace;font-size:10px;color:#6e6450;border:1px solid #d5cbb4;padding:1px 7px;border-radius:0;vertical-align:2px;margin-left:8px;font-weight:400;letter-spacing:.04em}
|
||||
.entry .cause{font-size:12.5px;color:#5c5342;margin-top:6px;max-width:600px}
|
||||
.entry .cause code{font-family:"IBM Plex Mono",monospace;font-size:11px;background:#ece6d5;padding:1px 6px;color:#a1201e}
|
||||
.entry .chain{margin-top:10px;font-family:"IBM Plex Mono",monospace;font-size:10.5px;color:#8a7f68;display:flex;gap:10px;flex-wrap:wrap}
|
||||
.entry .chain .ok{color:#166534}.entry .chain .no{color:#a1201e;font-weight:600}
|
||||
.entry .chain i{font-style:normal;color:#c6bda8}
|
||||
.entry .act button{font-family:"Noto Sans SC",sans-serif;font-size:12px;border:1px solid #1e1a14;background:transparent;color:#1e1a14;padding:6px 14px;cursor:pointer;letter-spacing:.06em}
|
||||
.entry .act button:hover{background:#1e1a14;color:#f7f4ec}
|
||||
.entry .meta{font-family:"IBM Plex Mono",monospace;font-size:10.5px;color:#a8a091;margin-top:8px}
|
||||
|
||||
.deadline{margin-top:8px;border-left:3px solid #a1201e;padding:10px 16px;background:#f1ecdf}
|
||||
.deadline h4{font-family:"Noto Serif SC",serif;font-size:13px;margin-bottom:4px}
|
||||
.deadline p{font-size:12px;color:#5c5342}
|
||||
|
||||
footer{margin-top:36px;padding-top:14px;border-top:1px solid #b8ac98;font-family:"IBM Plex Mono",monospace;font-size:11px;color:#8a7f68;display:flex;flex-wrap:wrap;gap:20px}
|
||||
footer b{color:#1e1a14;font-weight:500}
|
||||
footer .ok{color:#166534}footer .bad{color:#a1201e}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="app">
|
||||
<aside class="side">
|
||||
<div class="brand"><h1>知识治理台账</h1><span>GOVERNANCE LEDGER</span></div>
|
||||
<nav>
|
||||
<a href="#">来源与目录 <em>08</em></a>
|
||||
<a href="#" class="on">解析监控 <em>03</em></a>
|
||||
<a href="#">发布管理 <em>02</em></a>
|
||||
<a href="#">系统设置</a>
|
||||
</nav>
|
||||
<div class="foot">当前快照<br><b>2026-09-12 10:41</b><br>操作员 · 文档工程师</div>
|
||||
</aside>
|
||||
<div class="main">
|
||||
<div class="page">
|
||||
<div class="doc-rule"></div>
|
||||
<header>
|
||||
<h1>解析监控 · 需人工处理</h1>
|
||||
<p class="standfirst">失败任务以条目登记:缘由、链路断点、处置建议依次列出。勾选后由操作员署名批量重试,死信须逐条复核后方可重新入队。</p>
|
||||
<div class="dateline"><span>账期 <b>2026-09-12</b></span><span>1 小时成功 <b>88.9%</b></span><span>24 小时成功 <b>96.4%</b></span><span>Worker <b>2</b></span></div>
|
||||
</header>
|
||||
|
||||
<div class="ledger">
|
||||
<div class="cell f"><div class="num">2</div><div class="lbl">失败</div></div>
|
||||
<div class="cell w"><div class="num">1</div><div class="lbl">停滞</div></div>
|
||||
<div class="cell r"><div class="num">4</div><div class="lbl">处理中</div></div>
|
||||
<div class="cell d"><div class="num">48</div><div class="lbl">已完成</div></div>
|
||||
<div class="cell x"><div class="num">1</div><div class="lbl">死信</div></div>
|
||||
</div>
|
||||
|
||||
<h2 class="sec">失败条目 <span class="n">贰条 · 按断点阶段登记</span></h2>
|
||||
|
||||
<div class="entry">
|
||||
<div class="idx">一</div>
|
||||
<div>
|
||||
<h3>CR20A 电气规格说明书.pdf<span class="tag">对外</span><span class="tag">更新</span></h3>
|
||||
<div class="cause">错误码 <code>remote_parse_failed</code>,第二次尝试,已停留七十一小时。建议先在 WeKnora 手动解析该文件确认格式,再决定是否重试。</div>
|
||||
<div class="chain"><span class="ok">下载 ✓</span><i>·</i><span class="ok">Hash ✓</span><i>·</i><span class="ok">OSS ✓</span><i>·</i><span class="no">文档解析 ✗</span><i>·</i><span>WeKnora 上传</span><i>·</i><span>索引</span><i>·</i><span>完成</span></div>
|
||||
</div>
|
||||
<div class="act"><button>重 试</button><div class="meta">cs-1 · tp-failed</div></div>
|
||||
</div>
|
||||
|
||||
<div class="entry">
|
||||
<div class="idx">二</div>
|
||||
<div>
|
||||
<h3>伺服驱动器调试手册.docx<span class="tag">对内</span><span class="tag">新增</span></h3>
|
||||
<div class="cause">错误码 <code>mineru_timeout</code>,第一次尝试,已停留四十五分钟。多为 MinerU 容量或超时问题,确认服务恢复接单后重试。</div>
|
||||
<div class="chain"><span class="ok">下载 ✓</span><i>·</i><span class="ok">Hash ✓</span><i>·</i><span class="ok">OSS ✓</span><i>·</i><span class="ok">文档解析 ✓</span><i>·</i><span class="ok">WeKnora 上传 ✓</span><i>·</i><span class="no">索引 ✗</span><i>·</i><span>完成</span></div>
|
||||
</div>
|
||||
<div class="act"><button>重 试</button><div class="meta">cs-3 · tp-044</div></div>
|
||||
</div>
|
||||
|
||||
<h2 class="sec">死信登记 <span class="n">壹条 · 尝试耗尽</span></h2>
|
||||
<div class="deadline">
|
||||
<h4>对外知识库 · test.upload · <span class="mono">release_manifest_mismatch</span></h4>
|
||||
<p>发布结果与冻结清单不一致,已降级为失败。原任务与审计行保留,重新入队将创建全新尝试。来源 cs-2 · CR1A 固件变更记录.pdf · 尝试 5/5 · 2026-09-10 14:22</p>
|
||||
</div>
|
||||
|
||||
<footer>
|
||||
<span>MinerU <b class="ok">可接单</b></span>
|
||||
<span>WeKnora 测试 <b class="ok">可接单</b></span>
|
||||
<span>PostgreSQL <b>12ms</b></span>
|
||||
<span>队列 <b>3 待 / 4 跑</b></span>
|
||||
<span>CPU 42% · 内存 18.6/32.0GB</span>
|
||||
</footer>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,132 @@
|
||||
<!doctype html>
|
||||
<html lang="zh-CN">
|
||||
<head>
|
||||
<meta charset="UTF-8"><meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>方向C · 现行轻打磨(解析监控)</title>
|
||||
<link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700&family=Noto+Sans+SC:wght@400;500;700&display=swap">
|
||||
<style>
|
||||
*{box-sizing:border-box;margin:0;padding:0}
|
||||
body{font-family:Inter,"Noto Sans SC","PingFang SC",system-ui,sans-serif;background:#f4f7fa;color:#16324f;font-size:13.5px;line-height:1.55;-webkit-font-smoothing:antialiased}
|
||||
.mono{font-family:"SF Mono",Menlo,monospace;font-variant-numeric:tabular-nums}
|
||||
.app{display:grid;grid-template-columns:224px 1fr;min-height:100vh}
|
||||
.side{background:#12365a;color:#b8cfe0;padding:20px 0;display:flex;flex-direction:column}
|
||||
.side .brand{padding:0 18px 16px;border-bottom:1px solid #1e4a75;display:flex;gap:10px;align-items:center}
|
||||
.side .brand .dot{width:30px;height:30px;background:#2dd4bf;color:#04211d;display:grid;place-items:center;font-weight:700;border-radius:6px}
|
||||
.side .brand strong{color:#fff;font-size:14px}
|
||||
.side .brand span{display:block;font-size:11px;color:#7a9ab5;margin-top:2px}
|
||||
.side nav{padding:14px 10px;display:flex;flex-direction:column;gap:2px}
|
||||
.side nav a{color:#a8c3d8;text-decoration:none;padding:8px 12px;font-size:13px;border-radius:6px;display:flex;justify-content:space-between;align-items:center}
|
||||
.side nav a:hover{background:#1a4370;color:#e3eef5}
|
||||
.side nav a.on{background:#1e4a75;color:#fff}
|
||||
.side nav a em{font-style:normal;font-size:11px;background:#0f2c49;padding:1px 7px;border-radius:9px;color:#7aa3c2}
|
||||
.side nav a.on em{background:#2dd4bf;color:#04211d}
|
||||
.side .foot{margin-top:auto;padding:14px 18px;border-top:1px solid #1e4a75;font-size:11px;color:#7a9ab5}
|
||||
.side .foot .live{color:#34d399}
|
||||
|
||||
.main{min-width:0}
|
||||
.page{max-width:1180px;padding:26px 30px 48px}
|
||||
.page h1{font-size:22px;font-weight:700;margin-bottom:5px;letter-spacing:-.01em}
|
||||
.page .sub{color:#60758b;font-size:13px;margin-bottom:22px}
|
||||
|
||||
.tiles{display:grid;grid-template-columns:repeat(5,1fr);gap:12px;margin-bottom:24px}
|
||||
.tile{background:#fff;border:1px solid #dde6ee;border-radius:10px;padding:14px 16px;box-shadow:0 1px 3px rgba(15,45,75,.05)}
|
||||
.tile .lbl{font-size:12px;color:#60758b;margin-bottom:8px}
|
||||
.tile .num{font-size:26px;font-weight:700;color:#0d2a45;font-variant-numeric:tabular-nums}
|
||||
.tile.a-fail{border-top:3px solid #e75252}.tile.a-wait{border-top:3px solid #f3a712}.tile.a-run{border-top:3px solid #1677ff}.tile.a-done{border-top:3px solid #2aa876}.tile.a-dead{border-top:3px solid #94a3b8}
|
||||
|
||||
.panel{background:#fff;border:1px solid #dde6ee;border-radius:10px;overflow:hidden;box-shadow:0 1px 3px rgba(15,45,75,.05)}
|
||||
.panel .head{display:flex;align-items:center;gap:12px;padding:14px 18px;border-bottom:1px solid #e8eef4}
|
||||
.panel .head h2{font-size:14px;font-weight:700}
|
||||
.panel .head .hint{font-size:12px;color:#8aa0b2}
|
||||
.panel .head .act{margin-left:auto;display:flex;gap:8px}
|
||||
.btn{border:1px solid #d3dde5;background:#fff;color:#16324f;padding:6px 13px;font-size:12.5px;border-radius:7px;cursor:pointer;font-weight:500}
|
||||
.btn:hover{background:#f3f7fb}
|
||||
.btn.pri{background:#1677ff;border-color:#1677ff;color:#fff;font-weight:600}
|
||||
.btn.pri:hover{background:#0d63d9}
|
||||
|
||||
.task{padding:14px 18px;border-bottom:1px solid #eef2f6;display:grid;grid-template-columns:22px minmax(0,1fr) auto;gap:14px;align-items:start}
|
||||
.task:last-child{border-bottom:0}
|
||||
.task h4{font-size:13.5px;font-weight:600;display:flex;gap:8px;align-items:center;flex-wrap:wrap}
|
||||
.scope{font-size:11px;padding:2px 8px;border-radius:10px;font-weight:600}
|
||||
.scope.in{background:#e6efff;color:#1d4ed8}
|
||||
.scope.out{background:#e2f6ee;color:#0b6e4f}
|
||||
.chg{font-size:11px;padding:2px 8px;border-radius:10px;font-weight:600}
|
||||
.chg.add{background:#d9f2e4;color:#0b6e4f}.chg.upd{background:#dde9fd;color:#1d4ed8}
|
||||
.why{font-size:12.5px;color:#a82323;margin-top:5px}
|
||||
.why code{background:#fdecec;padding:1px 7px;border-radius:5px;font-size:11.5px}
|
||||
.chain{display:flex;gap:5px;margin-top:9px;flex-wrap:wrap;font-size:11.5px;color:#8aa0b2}
|
||||
.chain span{padding:3px 10px;border-radius:12px;background:#f3f6f9;border:1px solid #e3eaf0}
|
||||
.chain span.done{background:#e2f6ee;color:#0b6e4f;border-color:#bde3d3}
|
||||
.chain span.bad{background:#fdecec;color:#b91c1c;border-color:#f3c0c0;font-weight:600}
|
||||
.advice{font-size:12px;color:#60758b;margin-top:7px}
|
||||
.task .ops{display:flex;flex-direction:column;gap:6px;align-items:flex-end}
|
||||
.task-id{font-family:"SF Mono",Menlo,monospace;font-size:11px;color:#a4b4c2}
|
||||
|
||||
.strip{margin-top:24px;background:#fff;border:1px solid #dde6ee;border-radius:10px;display:flex;flex-wrap:wrap;font-size:12px;color:#60758b;overflow:hidden}
|
||||
.strip div{padding:10px 18px;border-right:1px solid #eef2f6}
|
||||
.strip div b{color:#0d2a45;font-weight:600}
|
||||
.strip .bad{color:#b91c1c}.strip .good{color:#0b6e4f}
|
||||
.strip div:last-child{border-right:0}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="app">
|
||||
<aside class="side">
|
||||
<div class="brand"><div class="dot">治</div><div><strong>知识治理</strong><span>governance console</span></div></div>
|
||||
<nav>
|
||||
<a href="#">来源与目录 <em>8</em></a>
|
||||
<a href="#" class="on">解析监控 <em>3</em></a>
|
||||
<a href="#">发布管理 <em>2</em></a>
|
||||
<a href="#">系统设置</a>
|
||||
</nav>
|
||||
<div class="foot"><span class="live">●</span> worker-01 心跳 4s 前<br>快照 2026-09-12 10:41</div>
|
||||
</aside>
|
||||
<div class="main">
|
||||
<div class="page">
|
||||
<h1>需人工处理</h1>
|
||||
<p class="sub">失败原因、处理链定位与建议直接放在卡片上。勾选后可批量重试,死信重新入队会保留原任务与全部审计行。</p>
|
||||
|
||||
<div class="tiles">
|
||||
<div class="tile a-fail"><div class="lbl">失败</div><div class="num">2</div></div>
|
||||
<div class="tile a-wait"><div class="lbl">停滞</div><div class="num">1</div></div>
|
||||
<div class="tile a-run"><div class="lbl">处理中</div><div class="num">4</div></div>
|
||||
<div class="tile a-done"><div class="lbl">已完成</div><div class="num">48</div></div>
|
||||
<div class="tile a-dead"><div class="lbl">死信</div><div class="num">1</div></div>
|
||||
</div>
|
||||
|
||||
<div class="panel">
|
||||
<div class="head"><h2>失败任务</h2><span class="hint">3 条 · 按失败阶段排序</span><span class="act"><input placeholder="操作人" style="border:1px solid #d3dde5;border-radius:7px;padding:5px 10px;font-size:12.5px;width:110px"><button class="btn">仅看中英对照</button><button class="btn pri">批量重试所选 (2)</button></span></div>
|
||||
<div class="task">
|
||||
<div style="margin-top:3px"><input type="checkbox" checked></div>
|
||||
<div>
|
||||
<h4>CR20A 电气规格说明书.pdf <span class="scope out">对外</span> <span class="chg upd">更新</span></h4>
|
||||
<div class="why">错误码 <code>remote_parse_failed</code> · 第 2 次尝试 · 已停留 71.1小时 · 失败于 文档解析</div>
|
||||
<div class="chain"><span class="done">下载 ✓</span><span class="done">Hash ✓</span><span class="done">OSS ✓</span><span class="bad">文档解析 ✗</span><span>WeKnora 上传</span><span>索引</span><span>完成</span></div>
|
||||
<div class="advice">建议先在 WeKnora 手动解析该文件确认格式,再决定是否重试。 <span class="task-id">cs-1 · tp-failed</span></div>
|
||||
</div>
|
||||
<div class="ops"><button class="btn pri">重试</button></div>
|
||||
</div>
|
||||
<div class="task">
|
||||
<div style="margin-top:3px"><input type="checkbox" checked></div>
|
||||
<div>
|
||||
<h4>伺服驱动器调试手册.docx <span class="scope in">对内</span> <span class="chg add">新增</span></h4>
|
||||
<div class="why">错误码 <code>mineru_timeout</code> · 第 1 次尝试 · 已停留 45分钟 · 失败于 索引 / 增强</div>
|
||||
<div class="chain"><span class="done">下载 ✓</span><span class="done">Hash ✓</span><span class="done">OSS ✓</span><span class="done">文档解析 ✓</span><span class="done">WeKnora 上传 ✓</span><span class="bad">索引 ✗</span><span>完成</span></div>
|
||||
<div class="advice">多为 MinerU 容量或超时问题,确认服务恢复接单后重试。 <span class="task-id">cs-3 · tp-044</span></div>
|
||||
</div>
|
||||
<div class="ops"><button class="btn pri">重试</button></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="strip" style="margin-top:22px">
|
||||
<div>MinerU <b class="good">可接单</b></div>
|
||||
<div>WeKnora 测试 <b class="good">可接单</b></div>
|
||||
<div>PostgreSQL <b>12ms</b></div>
|
||||
<div>队列 <b>3 待 / 4 跑 / 0 重试</b></div>
|
||||
<div>CPU 42% · 内存 18.6/32.0GB</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,33 @@
|
||||
module knowledge-governance
|
||||
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
github.com/aws/aws-sdk-go-v2 v1.47.0
|
||||
github.com/aws/aws-sdk-go-v2/config v1.33.4
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.20.4
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0
|
||||
github.com/jackc/pgx/v5 v5.9.2
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.10.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.38.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.50.0 // indirect
|
||||
github.com/aws/smithy-go v1.28.1 // indirect
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||
github.com/jackc/puddle/v2 v2.2.2 // indirect
|
||||
golang.org/x/sync v0.17.0 // indirect
|
||||
golang.org/x/text v0.29.0 // indirect
|
||||
)
|
||||
@@ -0,0 +1,78 @@
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc=
|
||||
github.com/aws/aws-sdk-go-v2 v1.47.0 h1:0jsHallhJCeaU0Ko48c/3FK1ctOQ7NpzggxriJOQ8MQ=
|
||||
github.com/aws/aws-sdk-go-v2 v1.47.0/go.mod h1:bttEH6JqnUL8LepvDVfdrds/fZ5bCIxzpe3abyUrhDU=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10/go.mod h1:qqY157uZoqm5OXq/amuaBJyC9hgBCBQnsaWnPe905GY=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.33.4 h1:FzvkXKSzwqHni4U7nDigHg4jjtqMpVUuHgmZfSoJVQ0=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.33.4/go.mod h1:VZqGZnZsCWVfK/iGPptJIyNIX3XEX6iQU2Rel4sLrr8=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.16 h1:r3RJBuU7X9ibt8RHbMjWE6y60QbKBiII6wSrXnapxSU=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.16/go.mod h1:6cx7zqDENJDbBIIWX6P8s0h6hqHC8Avbjh9Dseo27ug=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.20.4 h1:hTvrJJseKbvw32kmiE0G+u/9ZqpqscjDrTigHIXP2qs=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.20.4/go.mod h1:gWp9O1ZBWwpcIrgV+mVHk4gZUurAEDkgypu/OXOlIaw=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.0 h1:AM4hHjww+PSFtt6E+UrBrPlZkWsePCLEt9AjkfQX+yM=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.0/go.mod h1:3x/yXezeQjpOvBb4jEMxrS8SXvpdvJ5abv6l5c1gWM8=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 h1:GpT/TrnBYuE5gan2cZbTtvP+JlHsutdmlV2YfEyNde0=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23/go.mod h1:xYWD6BS9ywC5bS3sz9Xh04whO/hzK2plt2Zkyrp4JuA=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.3 h1:Hp/VgjP0BysR3OgLlR057Vz2LcbbVnoWeJ+3qWiS/fY=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.3/go.mod h1:nwGV5qw7F1IZPgxCvA/ph8N2TAuz+BkRG/bXn808qMA=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 h1:bpd8vxhlQi2r1hiueOw02f/duEPTMK59Q4QMAoTTtTo=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23/go.mod h1:15DfR2nw+CRHIk0tqNyifu3G1YdAOy68RftkhMDDwYk=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.3 h1:MUaM4f+kj1ZIBPZfUS8cxP1GKXXZtHJjAthy93AN7SM=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.3/go.mod h1:6YmVmEVRI5ZZzRjCSsb9SryKH0hAlMRdgA7kG9aDvBU=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 h1:OQqn11BtaYv1WLUowvcA30MpzIu8Ti4pcLPIIyoKZrA=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24/go.mod h1:X5ZJyfwVrWA96GzPmUCWFQaEARPR7gCrpq2E92PJwAE=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.3 h1:fuSCw4Z2qfRCztMPO3GXJNSiEp6Wee+WOLwrHHUMy9c=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.3/go.mod h1:6SxcHheD1pPR5+kWm1wGvjlL/YqUsh267sAfEmN4K7A=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 h1:FLudkZLt5ci0ozzgkVo8BJGwvqNaZbTWb3UcucAateA=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9/go.mod h1:w7wZ/s9qK7c8g4al+UyoF1Sp/Z45UwMGcqIzLWVQHWk=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19 h1:bAdDl/HkGCcGPoe25ToSHEw23VIxt6CT5fLcg111BKg=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19/go.mod h1:KaUzbLxv4CeSxh6ZCl9B4m7CuFenS8kUEaDs+f/DQr4=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 h1:ieLCO1JxUWuxTZ1cRd0GAaeX7O6cIxnwk7tc1LsQhC4=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15/go.mod h1:e3IzZvQ3kAWNykvE0Tr0RDZCMFInMvhku3qNpcIQXhM=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 h1:pbrxO/kuIwgEsOPLkaHu0O+m4fNgLU8B3vxQ+72jTPw=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23/go.mod h1:/CMNUqoj46HpS3MNRDEDIwcgEnrtZlKRaHNaHxIFpNA=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.3 h1:bON1rJf67TSTDCKg816AAIE4xSTtoo9tl0XRkO72R+I=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.3/go.mod h1:c5BBpjJcQXpfeq9iASyVKA3T6vX6B6LEXY4mL/gklDY=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 h1:03xatSQO4+AM1lTAbnRg5OK528EUg744nW7F73U8DKw=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23/go.mod h1:M8l3mwgx5ToK7wot2sBBce/ojzgnPzZXUV445gTSyE8=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0 h1:etqBTKY581iwLL/H/S2sVgk3C9lAsTJFeXWFDsDcWOU=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0/go.mod h1:L2dcoOgS2VSgbPLvpak2NyUPsO1TBN7M45Z4H7DlRc4=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.10.0 h1:ZD5qFpWcaOKdTuhBi431pIDkCgrMkMlMT6jlpSPoIRI=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.10.0/go.mod h1:8Nuuf+tR346PjJ3MvZPh9pekbLiLQFWJhzMXfwy7alA=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.38.0 h1:JGeeBcMlhg1xtOXYpeCaTQBZObtXMPQCUqBcmr65NRA=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.38.0/go.mod h1:XwteswG9EOMRFm73UT0t+MbTwyLxMrEXkU6e+v92Lzo=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.0 h1:obhahQXDEdVEv8y5bTKXR30LVaxYe1kyYM0L7l2Iq+k=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.0/go.mod h1:6twZZ/aXHNy1vXUO8koUbp++MYzMASkOgEBdkbJYmO0=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.50.0 h1:khXV3+K5D3f4e8xtplaRdSFn1bEg3gj5EBHQvbCOZbQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.50.0/go.mod h1:/8JRcdTt//hG0Q4BTmGbuOplT7ABe+5rdtqUHqXvYIM=
|
||||
github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI=
|
||||
github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
|
||||
github.com/aws/smithy-go v1.28.1 h1:R/nXH00c8qcfCzQVELtRw+eLQWtzv+VAIEFJ1/xxXlQ=
|
||||
github.com/aws/smithy-go v1.28.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
|
||||
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
|
||||
github.com/jackc/pgx/v5 v5.9.2 h1:3ZhOzMWnR4yJ+RW1XImIPsD1aNSz4T4fyP7zlQb56hw=
|
||||
github.com/jackc/pgx/v5 v5.9.2/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
|
||||
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
|
||||
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug=
|
||||
golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
|
||||
golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk=
|
||||
golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
@@ -0,0 +1,247 @@
|
||||
package appconfig
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
DatabaseURL string
|
||||
AppAddr string
|
||||
WorkerID string
|
||||
PollInterval time.Duration
|
||||
LeaseDuration time.Duration
|
||||
RetryDelay time.Duration
|
||||
SourceRefreshSchedulerInterval time.Duration
|
||||
ReleaseWorkerEnabled bool
|
||||
ReleaseConfigFile string
|
||||
ProductionReleaseRequestsEnabled bool
|
||||
ProductionBackupRestoreVerified bool
|
||||
ProductionReleaseAuthorizerRef string
|
||||
ProductionReleaseAuthTokenFile string
|
||||
PPScanEnabled bool
|
||||
PPScanAllowedRoot string
|
||||
PPScanLedgerPath string
|
||||
PPScanManifestPath string
|
||||
TestWeKnoraEnabled bool
|
||||
TestWeKnoraAllowDockerHTTP bool
|
||||
TestWeKnoraBaseURL string
|
||||
MinerUBaseURL string
|
||||
TestWeKnoraKBID string
|
||||
InternalRoutingKnowledgeBaseID string
|
||||
ExternalRoutingKnowledgeBaseID string
|
||||
TestWeKnoraAPIKeyFile string
|
||||
GovernanceSpoolRoot string
|
||||
WorkerLivenessStaleAfter time.Duration
|
||||
TestWeKnoraHTTPTimeout time.Duration
|
||||
TestWeKnoraPollInitial time.Duration
|
||||
TestWeKnoraPollMax time.Duration
|
||||
TestWeKnoraStalledAfter time.Duration
|
||||
TestWeKnoraMaxUploadBytes int64
|
||||
TestAcceptancePolicyVersion string
|
||||
}
|
||||
|
||||
func Load(getenv func(string) string) (Config, error) {
|
||||
config := Config{
|
||||
DatabaseURL: getenv("DATABASE_URL"),
|
||||
AppAddr: valueOrDefault(getenv("APP_ADDR"), ":8080"),
|
||||
WorkerID: valueOrDefault(getenv("WORKER_ID"), "worker-1"),
|
||||
PollInterval: time.Second,
|
||||
LeaseDuration: 30 * time.Second,
|
||||
RetryDelay: 5 * time.Second,
|
||||
SourceRefreshSchedulerInterval: 30 * time.Second,
|
||||
ReleaseConfigFile: strings.TrimSpace(getenv("RELEASE_CONFIG_FILE")),
|
||||
ProductionReleaseAuthorizerRef: strings.TrimSpace(getenv("PRODUCTION_RELEASE_AUTHORIZER_REF")),
|
||||
ProductionReleaseAuthTokenFile: strings.TrimSpace(getenv("PRODUCTION_RELEASE_AUTH_TOKEN_FILE")),
|
||||
PPScanAllowedRoot: strings.TrimSpace(getenv("PP_SCAN_ALLOWED_ROOT")),
|
||||
PPScanLedgerPath: strings.TrimSpace(getenv("PP_SCAN_LEDGER_PATH")),
|
||||
PPScanManifestPath: strings.TrimSpace(getenv("PP_SCAN_MANIFEST_PATH")),
|
||||
TestWeKnoraBaseURL: strings.TrimSpace(getenv("TEST_WEKNORA_BASE_URL")),
|
||||
MinerUBaseURL: strings.TrimSpace(getenv("MINERU_BASE_URL")),
|
||||
TestWeKnoraKBID: strings.TrimSpace(getenv("TEST_WEKNORA_KB_ID")),
|
||||
InternalRoutingKnowledgeBaseID: strings.TrimSpace(getenv("TEST_INTERNAL_WEKNORA_KB_ID")),
|
||||
ExternalRoutingKnowledgeBaseID: strings.TrimSpace(getenv("TEST_EXTERNAL_WEKNORA_KB_ID")),
|
||||
TestWeKnoraAPIKeyFile: valueOrDefault(getenv("TEST_WEKNORA_API_KEY_FILE"), "/run/secrets/test_weknora_api_key"),
|
||||
GovernanceSpoolRoot: valueOrDefault(getenv("GOVERNANCE_SPOOL_ROOT"), "/var/lib/governance/spool"),
|
||||
WorkerLivenessStaleAfter: 2 * time.Minute,
|
||||
TestWeKnoraHTTPTimeout: 60 * time.Second,
|
||||
TestWeKnoraPollInitial: 10 * time.Second,
|
||||
TestWeKnoraPollMax: time.Minute,
|
||||
TestWeKnoraStalledAfter: 30 * time.Minute,
|
||||
TestWeKnoraMaxUploadBytes: 50 * 1024 * 1024,
|
||||
TestAcceptancePolicyVersion: valueOrDefault(getenv("TEST_ACCEPTANCE_POLICY_VERSION"), "structural-v1"),
|
||||
}
|
||||
if config.DatabaseURL == "" {
|
||||
return Config{}, errors.New("DATABASE_URL is required")
|
||||
}
|
||||
var err error
|
||||
if config.PollInterval, err = durationOrDefault(getenv("WORKER_POLL_INTERVAL"), config.PollInterval); err != nil {
|
||||
return Config{}, fmt.Errorf("WORKER_POLL_INTERVAL: %w", err)
|
||||
}
|
||||
if config.LeaseDuration, err = durationOrDefault(getenv("WORKER_LEASE_DURATION"), config.LeaseDuration); err != nil {
|
||||
return Config{}, fmt.Errorf("WORKER_LEASE_DURATION: %w", err)
|
||||
}
|
||||
if config.RetryDelay, err = durationOrDefault(getenv("WORKER_RETRY_DELAY"), config.RetryDelay); err != nil {
|
||||
return Config{}, fmt.Errorf("WORKER_RETRY_DELAY: %w", err)
|
||||
}
|
||||
if config.SourceRefreshSchedulerInterval, err = durationOrDefault(getenv("SOURCE_REFRESH_SCHEDULER_INTERVAL"), config.SourceRefreshSchedulerInterval); err != nil {
|
||||
return Config{}, fmt.Errorf("SOURCE_REFRESH_SCHEDULER_INTERVAL: %w", err)
|
||||
}
|
||||
if config.SourceRefreshSchedulerInterval < 5*time.Second {
|
||||
return Config{}, errors.New("SOURCE_REFRESH_SCHEDULER_INTERVAL must be at least 5s")
|
||||
}
|
||||
if raw := getenv("RELEASE_WORKER_ENABLED"); raw != "" {
|
||||
config.ReleaseWorkerEnabled, err = strconv.ParseBool(raw)
|
||||
if err != nil {
|
||||
return Config{}, fmt.Errorf("RELEASE_WORKER_ENABLED: %w", err)
|
||||
}
|
||||
}
|
||||
if config.ReleaseWorkerEnabled && (!filepath.IsAbs(config.ReleaseConfigFile) || filepath.Clean(config.ReleaseConfigFile) != config.ReleaseConfigFile) {
|
||||
return Config{}, errors.New("RELEASE_CONFIG_FILE must be an absolute path when release worker is enabled")
|
||||
}
|
||||
if raw := getenv("PRODUCTION_RELEASE_REQUESTS_ENABLED"); raw != "" {
|
||||
config.ProductionReleaseRequestsEnabled, err = strconv.ParseBool(raw)
|
||||
if err != nil {
|
||||
return Config{}, fmt.Errorf("PRODUCTION_RELEASE_REQUESTS_ENABLED: %w", err)
|
||||
}
|
||||
}
|
||||
if raw := getenv("PRODUCTION_BACKUP_RESTORE_VERIFIED"); raw != "" {
|
||||
config.ProductionBackupRestoreVerified, err = strconv.ParseBool(raw)
|
||||
if err != nil {
|
||||
return Config{}, fmt.Errorf("PRODUCTION_BACKUP_RESTORE_VERIFIED: %w", err)
|
||||
}
|
||||
}
|
||||
if raw := getenv("PP_SCAN_ENABLED"); raw != "" {
|
||||
config.PPScanEnabled, err = strconv.ParseBool(raw)
|
||||
if err != nil {
|
||||
return Config{}, fmt.Errorf("PP_SCAN_ENABLED: %w", err)
|
||||
}
|
||||
}
|
||||
if raw := getenv("TEST_WEKNORA_ENABLED"); raw != "" {
|
||||
config.TestWeKnoraEnabled, err = strconv.ParseBool(raw)
|
||||
if err != nil {
|
||||
return Config{}, fmt.Errorf("TEST_WEKNORA_ENABLED: %w", err)
|
||||
}
|
||||
}
|
||||
if raw := getenv("TEST_WEKNORA_ALLOW_DOCKER_HTTP"); raw != "" {
|
||||
config.TestWeKnoraAllowDockerHTTP, err = strconv.ParseBool(raw)
|
||||
if err != nil {
|
||||
return Config{}, fmt.Errorf("TEST_WEKNORA_ALLOW_DOCKER_HTTP: %w", err)
|
||||
}
|
||||
}
|
||||
if config.WorkerLivenessStaleAfter, err = durationOrDefault(getenv("WORKER_LIVENESS_STALE_AFTER"), config.WorkerLivenessStaleAfter); err != nil {
|
||||
return Config{}, fmt.Errorf("WORKER_LIVENESS_STALE_AFTER: %w", err)
|
||||
}
|
||||
if config.TestWeKnoraHTTPTimeout, err = durationOrDefault(getenv("TEST_WEKNORA_HTTP_TIMEOUT"), config.TestWeKnoraHTTPTimeout); err != nil {
|
||||
return Config{}, fmt.Errorf("TEST_WEKNORA_HTTP_TIMEOUT: %w", err)
|
||||
}
|
||||
if config.TestWeKnoraPollInitial, err = durationOrDefault(getenv("TEST_WEKNORA_POLL_INITIAL"), config.TestWeKnoraPollInitial); err != nil {
|
||||
return Config{}, fmt.Errorf("TEST_WEKNORA_POLL_INITIAL: %w", err)
|
||||
}
|
||||
if config.TestWeKnoraPollMax, err = durationOrDefault(getenv("TEST_WEKNORA_POLL_MAX"), config.TestWeKnoraPollMax); err != nil {
|
||||
return Config{}, fmt.Errorf("TEST_WEKNORA_POLL_MAX: %w", err)
|
||||
}
|
||||
if config.TestWeKnoraStalledAfter, err = durationOrDefault(getenv("TEST_WEKNORA_STALLED_AFTER"), config.TestWeKnoraStalledAfter); err != nil {
|
||||
return Config{}, fmt.Errorf("TEST_WEKNORA_STALLED_AFTER: %w", err)
|
||||
}
|
||||
if config.TestWeKnoraMaxUploadBytes, err = positiveInt64OrDefault(getenv("TEST_WEKNORA_MAX_UPLOAD_BYTES"), config.TestWeKnoraMaxUploadBytes); err != nil {
|
||||
return Config{}, fmt.Errorf("TEST_WEKNORA_MAX_UPLOAD_BYTES: %w", err)
|
||||
}
|
||||
if config.TestWeKnoraMaxUploadBytes > 50*1024*1024 {
|
||||
return Config{}, errors.New("TEST_WEKNORA_MAX_UPLOAD_BYTES must not exceed 50 MiB")
|
||||
}
|
||||
if config.TestWeKnoraPollMax < config.TestWeKnoraPollInitial {
|
||||
return Config{}, errors.New("TEST_WEKNORA_POLL_MAX must not be less than TEST_WEKNORA_POLL_INITIAL")
|
||||
}
|
||||
if config.TestWeKnoraStalledAfter <= config.TestWeKnoraPollMax {
|
||||
return Config{}, errors.New("TEST_WEKNORA_STALLED_AFTER must be greater than TEST_WEKNORA_POLL_MAX")
|
||||
}
|
||||
config.TestAcceptancePolicyVersion = strings.TrimSpace(config.TestAcceptancePolicyVersion)
|
||||
if config.TestAcceptancePolicyVersion == "" {
|
||||
return Config{}, errors.New("TEST_ACCEPTANCE_POLICY_VERSION must not be empty")
|
||||
}
|
||||
if config.ProductionReleaseRequestsEnabled {
|
||||
if config.ProductionReleaseAuthorizerRef == "" || len(config.ProductionReleaseAuthorizerRef) > 128 || strings.ContainsAny(config.ProductionReleaseAuthorizerRef, "\x00\r\n ") {
|
||||
return Config{}, errors.New("PRODUCTION_RELEASE_AUTHORIZER_REF is required and must be safe when production release requests are enabled")
|
||||
}
|
||||
if config.ProductionReleaseAuthTokenFile == "" || !filepath.IsAbs(config.ProductionReleaseAuthTokenFile) {
|
||||
return Config{}, errors.New("PRODUCTION_RELEASE_AUTH_TOKEN_FILE must be an absolute path when production release requests are enabled")
|
||||
}
|
||||
}
|
||||
if config.PPScanEnabled {
|
||||
for key, value := range map[string]string{
|
||||
"PP_SCAN_ALLOWED_ROOT": config.PPScanAllowedRoot,
|
||||
"PP_SCAN_LEDGER_PATH": config.PPScanLedgerPath,
|
||||
"PP_SCAN_MANIFEST_PATH": config.PPScanManifestPath,
|
||||
} {
|
||||
if value == "" {
|
||||
return Config{}, fmt.Errorf("%s is required when PP scan is enabled", key)
|
||||
}
|
||||
if !filepath.IsAbs(value) {
|
||||
return Config{}, fmt.Errorf("%s must be absolute", key)
|
||||
}
|
||||
}
|
||||
}
|
||||
if config.TestWeKnoraEnabled {
|
||||
if config.TestWeKnoraBaseURL == "" {
|
||||
return Config{}, errors.New("TEST_WEKNORA_BASE_URL is required when test WeKnora is enabled")
|
||||
}
|
||||
if config.TestWeKnoraKBID == "" {
|
||||
return Config{}, errors.New("TEST_WEKNORA_KB_ID is required when test WeKnora is enabled")
|
||||
}
|
||||
if strings.TrimSpace(config.TestWeKnoraAPIKeyFile) == "" {
|
||||
return Config{}, errors.New("TEST_WEKNORA_API_KEY_FILE is required when test WeKnora is enabled")
|
||||
}
|
||||
if strings.TrimSpace(config.GovernanceSpoolRoot) == "" {
|
||||
return Config{}, errors.New("GOVERNANCE_SPOOL_ROOT is required when test WeKnora is enabled")
|
||||
}
|
||||
}
|
||||
if (config.InternalRoutingKnowledgeBaseID == "") != (config.ExternalRoutingKnowledgeBaseID == "") {
|
||||
return Config{}, errors.New("both internal and external routing knowledge bases must be configured together")
|
||||
}
|
||||
for _, knowledgeBaseID := range []string{config.InternalRoutingKnowledgeBaseID, config.ExternalRoutingKnowledgeBaseID} {
|
||||
if len(knowledgeBaseID) > 128 || strings.ContainsAny(knowledgeBaseID, "\x00\r\n /?#\\") {
|
||||
return Config{}, errors.New("routing knowledge base ID is invalid")
|
||||
}
|
||||
}
|
||||
return config, nil
|
||||
}
|
||||
|
||||
func valueOrDefault(value, fallback string) string {
|
||||
if value == "" {
|
||||
return fallback
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func durationOrDefault(value string, fallback time.Duration) (time.Duration, error) {
|
||||
if value == "" {
|
||||
return fallback, nil
|
||||
}
|
||||
parsed, err := time.ParseDuration(value)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if parsed <= 0 {
|
||||
return 0, errors.New("must be positive")
|
||||
}
|
||||
return parsed, nil
|
||||
}
|
||||
|
||||
func positiveInt64OrDefault(value string, fallback int64) (int64, error) {
|
||||
if value == "" {
|
||||
return fallback, nil
|
||||
}
|
||||
parsed, err := strconv.ParseInt(value, 10, 64)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if parsed <= 0 {
|
||||
return 0, errors.New("must be positive")
|
||||
}
|
||||
return parsed, nil
|
||||
}
|
||||
@@ -0,0 +1,243 @@
|
||||
package appconfig
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestLoadUsesSafeDefaults(t *testing.T) {
|
||||
values := map[string]string{"DATABASE_URL": "postgres://example"}
|
||||
config, err := Load(func(key string) string { return values[key] })
|
||||
if err != nil {
|
||||
t.Fatalf("Load() error = %v", err)
|
||||
}
|
||||
if config.AppAddr != ":8080" || config.WorkerID != "worker-1" {
|
||||
t.Fatalf("address/worker defaults = %#v", config)
|
||||
}
|
||||
if config.PollInterval != time.Second || config.LeaseDuration != 30*time.Second || config.RetryDelay != 5*time.Second {
|
||||
t.Fatalf("duration defaults = %#v", config)
|
||||
}
|
||||
if config.ReleaseWorkerEnabled {
|
||||
t.Fatal("release worker must be disabled by default")
|
||||
}
|
||||
if config.ReleaseConfigFile != "" {
|
||||
t.Fatalf("release config default = %q", config.ReleaseConfigFile)
|
||||
}
|
||||
if config.ProductionReleaseRequestsEnabled {
|
||||
t.Fatal("production release requests must be disabled by default")
|
||||
}
|
||||
if config.TestWeKnoraEnabled {
|
||||
t.Fatal("test WeKnora must be disabled by default")
|
||||
}
|
||||
if config.PPScanEnabled {
|
||||
t.Fatal("PP source scan must be disabled by default")
|
||||
}
|
||||
if config.PPScanAllowedRoot != "" || config.PPScanLedgerPath != "" || config.PPScanManifestPath != "" {
|
||||
t.Fatalf("PP scan paths must be empty by default: %#v", config)
|
||||
}
|
||||
if config.TestWeKnoraAllowDockerHTTP {
|
||||
t.Fatal("test WeKnora Docker HTTP must be disabled by default")
|
||||
}
|
||||
if config.TestWeKnoraAPIKeyFile != "/run/secrets/test_weknora_api_key" {
|
||||
t.Fatalf("test API key file = %q", config.TestWeKnoraAPIKeyFile)
|
||||
}
|
||||
if config.GovernanceSpoolRoot != "/var/lib/governance/spool" {
|
||||
t.Fatalf("spool root = %q", config.GovernanceSpoolRoot)
|
||||
}
|
||||
if config.TestWeKnoraHTTPTimeout != 60*time.Second || config.TestWeKnoraPollInitial != 10*time.Second || config.TestWeKnoraPollMax != time.Minute || config.TestWeKnoraStalledAfter != 30*time.Minute {
|
||||
t.Fatalf("test WeKnora timing defaults = %#v", config)
|
||||
}
|
||||
if config.TestWeKnoraMaxUploadBytes != 50*1024*1024 || config.TestAcceptancePolicyVersion != "structural-v1" {
|
||||
t.Fatalf("test WeKnora safety defaults = %#v", config)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadRequiresDatabaseURL(t *testing.T) {
|
||||
if _, err := Load(func(string) string { return "" }); err == nil {
|
||||
t.Fatal("Load() error = nil, want missing DATABASE_URL error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadParsesProductionReleaseRequestGate(t *testing.T) {
|
||||
values := map[string]string{
|
||||
"DATABASE_URL": "postgres://example",
|
||||
"PRODUCTION_RELEASE_REQUESTS_ENABLED": "true",
|
||||
"PRODUCTION_BACKUP_RESTORE_VERIFIED": "true",
|
||||
"PRODUCTION_RELEASE_AUTHORIZER_REF": "release-owner",
|
||||
"PRODUCTION_RELEASE_AUTH_TOKEN_FILE": "/run/secrets/production_release_auth_token",
|
||||
}
|
||||
config, err := Load(func(key string) string { return values[key] })
|
||||
if err != nil || !config.ProductionReleaseRequestsEnabled || !config.ProductionBackupRestoreVerified || config.ProductionReleaseAuthorizerRef != "release-owner" || config.ProductionReleaseAuthTokenFile != "/run/secrets/production_release_auth_token" {
|
||||
t.Fatalf("config=%#v error=%v", config, err)
|
||||
}
|
||||
values["PRODUCTION_RELEASE_REQUESTS_ENABLED"] = "not-a-bool"
|
||||
if _, err := Load(func(key string) string { return values[key] }); err == nil {
|
||||
t.Fatal("invalid production release request gate must be rejected")
|
||||
}
|
||||
values["PRODUCTION_RELEASE_REQUESTS_ENABLED"] = "true"
|
||||
values["PRODUCTION_BACKUP_RESTORE_VERIFIED"] = "not-a-bool"
|
||||
if _, err := Load(func(key string) string { return values[key] }); err == nil {
|
||||
t.Fatal("invalid production backup restore gate must be rejected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadRequiresProductionReleaseAuthorizationConfigWhenGateEnabled(t *testing.T) {
|
||||
base := map[string]string{
|
||||
"DATABASE_URL": "postgres://example",
|
||||
"PRODUCTION_RELEASE_REQUESTS_ENABLED": "true",
|
||||
"PRODUCTION_RELEASE_AUTHORIZER_REF": "release-owner",
|
||||
"PRODUCTION_RELEASE_AUTH_TOKEN_FILE": "/run/secrets/production_release_auth_token",
|
||||
}
|
||||
for _, missing := range []string{"PRODUCTION_RELEASE_AUTHORIZER_REF", "PRODUCTION_RELEASE_AUTH_TOKEN_FILE"} {
|
||||
values := make(map[string]string, len(base))
|
||||
for key, value := range base {
|
||||
values[key] = value
|
||||
}
|
||||
delete(values, missing)
|
||||
if _, err := Load(func(key string) string { return values[key] }); err == nil {
|
||||
t.Fatalf("missing %s accepted", missing)
|
||||
}
|
||||
}
|
||||
base["PRODUCTION_RELEASE_AUTH_TOKEN_FILE"] = "relative/token"
|
||||
if _, err := Load(func(key string) string { return base[key] }); err == nil {
|
||||
t.Fatal("relative production release token file accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadRequiresReleaseConfigFileWhenReleaseWorkerEnabled(t *testing.T) {
|
||||
if _, err := Load(func(key string) string {
|
||||
values := map[string]string{"DATABASE_URL": "postgres://example", "RELEASE_WORKER_ENABLED": "true"}
|
||||
return values[key]
|
||||
}); err == nil {
|
||||
t.Fatal("enabled release worker without config file accepted")
|
||||
}
|
||||
config, err := Load(func(key string) string {
|
||||
values := map[string]string{"DATABASE_URL": "postgres://example", "RELEASE_WORKER_ENABLED": "true", "RELEASE_CONFIG_FILE": "/run/secrets/release-config"}
|
||||
return values[key]
|
||||
})
|
||||
if err != nil || config.ReleaseConfigFile != "/run/secrets/release-config" {
|
||||
t.Fatalf("config=%#v err=%v", config, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadRequiresAllPPScanPathsWhenEnabled(t *testing.T) {
|
||||
base := map[string]string{
|
||||
"DATABASE_URL": "postgres://example",
|
||||
"PP_SCAN_ENABLED": "true",
|
||||
"PP_SCAN_ALLOWED_ROOT": "/source",
|
||||
"PP_SCAN_LEDGER_PATH": "/source/state.jsonl",
|
||||
"PP_SCAN_MANIFEST_PATH": "/source/manifest.json",
|
||||
}
|
||||
for _, missing := range []string{"PP_SCAN_ALLOWED_ROOT", "PP_SCAN_LEDGER_PATH", "PP_SCAN_MANIFEST_PATH"} {
|
||||
values := make(map[string]string, len(base))
|
||||
for key, value := range base {
|
||||
values[key] = value
|
||||
}
|
||||
delete(values, missing)
|
||||
if _, err := Load(func(key string) string { return values[key] }); err == nil {
|
||||
t.Fatalf("Load() error = nil, want missing %s rejected", missing)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadRequiresTestWeKnoraEndpointAndKBWhenEnabled(t *testing.T) {
|
||||
base := map[string]string{
|
||||
"DATABASE_URL": "postgres://example",
|
||||
"TEST_WEKNORA_ENABLED": "true",
|
||||
}
|
||||
for _, key := range []string{"TEST_WEKNORA_BASE_URL", "TEST_WEKNORA_KB_ID"} {
|
||||
values := make(map[string]string, len(base)+1)
|
||||
for name, value := range base {
|
||||
values[name] = value
|
||||
}
|
||||
if key == "TEST_WEKNORA_BASE_URL" {
|
||||
values["TEST_WEKNORA_KB_ID"] = "kb-test"
|
||||
} else {
|
||||
values["TEST_WEKNORA_BASE_URL"] = "https://weknora.test.example"
|
||||
}
|
||||
if _, err := Load(func(name string) string { return values[name] }); err == nil {
|
||||
t.Fatalf("Load() error = nil, want missing %s error", key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadParsesTestWeKnoraSettings(t *testing.T) {
|
||||
values := map[string]string{
|
||||
"DATABASE_URL": "postgres://example",
|
||||
"TEST_WEKNORA_ENABLED": "true",
|
||||
"TEST_WEKNORA_ALLOW_DOCKER_HTTP": "true",
|
||||
"TEST_WEKNORA_BASE_URL": "https://weknora.test.example",
|
||||
"TEST_WEKNORA_KB_ID": "kb-test",
|
||||
"TEST_WEKNORA_API_KEY_FILE": "/run/secrets/custom_key",
|
||||
"GOVERNANCE_SPOOL_ROOT": "/srv/spool",
|
||||
"TEST_WEKNORA_HTTP_TIMEOUT": "45s",
|
||||
"TEST_WEKNORA_POLL_INITIAL": "5s",
|
||||
"TEST_WEKNORA_POLL_MAX": "20s",
|
||||
"TEST_WEKNORA_STALLED_AFTER": "15m",
|
||||
"TEST_WEKNORA_MAX_UPLOAD_BYTES": "1048576",
|
||||
"TEST_ACCEPTANCE_POLICY_VERSION": "structural-v2",
|
||||
}
|
||||
config, err := Load(func(name string) string { return values[name] })
|
||||
if err != nil {
|
||||
t.Fatalf("Load() error = %v", err)
|
||||
}
|
||||
if !config.TestWeKnoraEnabled || !config.TestWeKnoraAllowDockerHTTP || config.TestWeKnoraBaseURL != values["TEST_WEKNORA_BASE_URL"] || config.TestWeKnoraKBID != "kb-test" {
|
||||
t.Fatalf("test WeKnora identity = %#v", config)
|
||||
}
|
||||
if config.TestWeKnoraAPIKeyFile != "/run/secrets/custom_key" || config.GovernanceSpoolRoot != "/srv/spool" {
|
||||
t.Fatalf("test WeKnora file settings = %#v", config)
|
||||
}
|
||||
if config.TestWeKnoraHTTPTimeout != 45*time.Second || config.TestWeKnoraPollInitial != 5*time.Second || config.TestWeKnoraPollMax != 20*time.Second || config.TestWeKnoraStalledAfter != 15*time.Minute {
|
||||
t.Fatalf("test WeKnora timing settings = %#v", config)
|
||||
}
|
||||
if config.TestWeKnoraMaxUploadBytes != 1048576 || config.TestAcceptancePolicyVersion != "structural-v2" {
|
||||
t.Fatalf("test WeKnora safety settings = %#v", config)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadRejectsUnsafeTestWeKnoraSettings(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
key string
|
||||
value string
|
||||
}{
|
||||
{name: "poll max before initial", key: "TEST_WEKNORA_POLL_MAX", value: "5s"},
|
||||
{name: "stalled before poll max", key: "TEST_WEKNORA_STALLED_AFTER", value: "30s"},
|
||||
{name: "upload limit above 50 MiB", key: "TEST_WEKNORA_MAX_UPLOAD_BYTES", value: "52428801"},
|
||||
{name: "empty policy", key: "TEST_ACCEPTANCE_POLICY_VERSION", value: " "},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
values := map[string]string{
|
||||
"DATABASE_URL": "postgres://example",
|
||||
"TEST_WEKNORA_ENABLED": "true",
|
||||
"TEST_WEKNORA_BASE_URL": "https://weknora.test.example",
|
||||
"TEST_WEKNORA_KB_ID": "kb-test",
|
||||
"TEST_WEKNORA_POLL_INITIAL": "10s",
|
||||
"TEST_WEKNORA_POLL_MAX": "60s",
|
||||
"TEST_WEKNORA_STALLED_AFTER": "30m",
|
||||
"TEST_ACCEPTANCE_POLICY_VERSION": "structural-v1",
|
||||
}
|
||||
values[tt.key] = tt.value
|
||||
if _, err := Load(func(name string) string { return values[name] }); err == nil {
|
||||
t.Fatalf("Load() error = nil, want unsafe %s rejected", tt.key)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadRequiresBothCatalogRoutingKnowledgeBases(t *testing.T) {
|
||||
values := map[string]string{
|
||||
"DATABASE_URL": "postgres://example",
|
||||
"TEST_INTERNAL_WEKNORA_KB_ID": "kb-internal",
|
||||
"TEST_EXTERNAL_WEKNORA_KB_ID": "kb-external",
|
||||
}
|
||||
config, err := Load(func(key string) string { return values[key] })
|
||||
if err != nil || config.InternalRoutingKnowledgeBaseID != "kb-internal" || config.ExternalRoutingKnowledgeBaseID != "kb-external" {
|
||||
t.Fatalf("config=%#v err=%v", config, err)
|
||||
}
|
||||
delete(values, "TEST_EXTERNAL_WEKNORA_KB_ID")
|
||||
if _, err = Load(func(key string) string { return values[key] }); err == nil {
|
||||
t.Fatal("one-sided routing target must fail closed")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,141 @@
|
||||
package pp
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"sort"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
)
|
||||
|
||||
type LegacyStatus string
|
||||
|
||||
const (
|
||||
StatusCompleted LegacyStatus = "completed"
|
||||
StatusContentAlias LegacyStatus = "content_alias"
|
||||
StatusOverLimit LegacyStatus = "over_limit"
|
||||
)
|
||||
|
||||
type Item struct {
|
||||
KBID string `json:"kb_id"`
|
||||
SourceURLHash string `json:"source_url_hash"`
|
||||
SourceIDs []string `json:"source_ids"`
|
||||
Bytes int64 `json:"bytes"`
|
||||
SHA256 string `json:"sha256"`
|
||||
Status LegacyStatus `json:"status"`
|
||||
KnowledgeID string `json:"knowledge_id"`
|
||||
CanonicalSourceURLHash string `json:"canonical_source_url_hash"`
|
||||
}
|
||||
|
||||
type Snapshot struct {
|
||||
KBID string
|
||||
EventCount int
|
||||
UnresolvedCount int
|
||||
Items []Item
|
||||
}
|
||||
|
||||
func FoldLedger(reader io.Reader) (Snapshot, error) {
|
||||
scanner := bufio.NewScanner(reader)
|
||||
scanner.Buffer(make([]byte, 64*1024), 1024*1024)
|
||||
terminal := make(map[string]Item)
|
||||
seen := make(map[string]struct{})
|
||||
result := Snapshot{}
|
||||
for scanner.Scan() {
|
||||
result.EventCount++
|
||||
var item Item
|
||||
if err := json.Unmarshal(scanner.Bytes(), &item); err != nil {
|
||||
return Snapshot{}, fmt.Errorf("decode ledger line %d: %w", result.EventCount, err)
|
||||
}
|
||||
if result.KBID == "" {
|
||||
result.KBID = item.KBID
|
||||
} else if item.KBID != result.KBID {
|
||||
return Snapshot{}, fmt.Errorf("ledger contains multiple knowledge bases at line %d", result.EventCount)
|
||||
}
|
||||
if item.SourceURLHash != "" {
|
||||
seen[item.SourceURLHash] = struct{}{}
|
||||
}
|
||||
if item.Status == StatusCompleted || (item.Status == StatusContentAlias && item.KnowledgeID != "") {
|
||||
if err := validateTerminalItem(item); err != nil {
|
||||
return Snapshot{}, fmt.Errorf("invalid terminal ledger record at line %d: %w", result.EventCount, err)
|
||||
}
|
||||
terminal[item.SourceURLHash] = item
|
||||
}
|
||||
}
|
||||
if err := scanner.Err(); err != nil {
|
||||
return Snapshot{}, fmt.Errorf("read ledger: %w", err)
|
||||
}
|
||||
result.Items = make([]Item, 0, len(terminal))
|
||||
for _, item := range terminal {
|
||||
result.Items = append(result.Items, item)
|
||||
}
|
||||
result.UnresolvedCount = len(seen) - len(terminal)
|
||||
sort.Slice(result.Items, func(i, j int) bool {
|
||||
return result.Items[i].SourceURLHash < result.Items[j].SourceURLHash
|
||||
})
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func validateTerminalItem(item Item) error {
|
||||
if item.KBID == "" {
|
||||
return fmt.Errorf("kb_id is required")
|
||||
}
|
||||
if !governance.ValidSHA256(item.SourceURLHash) {
|
||||
return fmt.Errorf("source_url_hash must be a canonical hash")
|
||||
}
|
||||
if len(item.SourceIDs) == 0 {
|
||||
return fmt.Errorf("source_ids are required")
|
||||
}
|
||||
for _, sourceID := range item.SourceIDs {
|
||||
if sourceID == "" {
|
||||
return fmt.Errorf("source_ids cannot contain an empty value")
|
||||
}
|
||||
}
|
||||
if item.Bytes < 0 {
|
||||
return fmt.Errorf("bytes cannot be negative")
|
||||
}
|
||||
if item.SHA256 != "" && !governance.ValidSHA256(item.SHA256) {
|
||||
return fmt.Errorf("sha256 must be empty or a canonical hash")
|
||||
}
|
||||
if item.KnowledgeID == "" {
|
||||
return fmt.Errorf("knowledge_id is required")
|
||||
}
|
||||
if item.Status == StatusContentAlias && !governance.ValidSHA256(item.CanonicalSourceURLHash) {
|
||||
return fmt.Errorf("canonical_source_url_hash must be a canonical hash")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func ValidateSnapshot(snapshot Snapshot) error {
|
||||
if snapshot.UnresolvedCount != 0 {
|
||||
return fmt.Errorf("snapshot contains %d unresolved source keys", snapshot.UnresolvedCount)
|
||||
}
|
||||
bySourceHash := make(map[string]Item, len(snapshot.Items))
|
||||
for _, item := range snapshot.Items {
|
||||
if item.Status == StatusOverLimit {
|
||||
if item.Bytes <= LegacyMaxBytes {
|
||||
return fmt.Errorf("over-limit item does not exceed the legacy size limit")
|
||||
}
|
||||
bySourceHash[item.SourceURLHash] = item
|
||||
continue
|
||||
}
|
||||
if !governance.ValidSHA256(item.SHA256) {
|
||||
return fmt.Errorf("snapshot item is missing a verified SHA-256")
|
||||
}
|
||||
bySourceHash[item.SourceURLHash] = item
|
||||
}
|
||||
for _, item := range snapshot.Items {
|
||||
if item.Status != StatusContentAlias {
|
||||
continue
|
||||
}
|
||||
canonical, ok := bySourceHash[item.CanonicalSourceURLHash]
|
||||
if !ok || canonical.Status != StatusCompleted {
|
||||
return fmt.Errorf("alias does not reference a canonical completed item")
|
||||
}
|
||||
if item.SHA256 != canonical.SHA256 || item.KnowledgeID != canonical.KnowledgeID {
|
||||
return fmt.Errorf("alias metadata does not match canonical completed item")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,224 @@
|
||||
package pp
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestFoldLedgerKeepsLatestTerminalRecordPerSourceURLHash(t *testing.T) {
|
||||
sourceA := strings.Repeat("a", 64)
|
||||
sourceB := strings.Repeat("b", 64)
|
||||
sha := strings.Repeat("c", 64)
|
||||
ledger := strings.Join([]string{
|
||||
`{"kb_id":"kb-1","source_url_hash":"` + sourceA + `","source_ids":["pp-1"],"bytes":10,"sha256":"` + sha + `","status":"uploaded","knowledge_id":"knowledge-old"}`,
|
||||
`{"kb_id":"kb-1","source_url_hash":"` + sourceA + `","source_ids":["pp-1"],"bytes":10,"sha256":"` + sha + `","status":"completed","knowledge_id":"knowledge-1"}`,
|
||||
`{"kb_id":"kb-1","source_url_hash":"` + sourceA + `","source_ids":["pp-1"],"bytes":10,"sha256":"` + sha + `","status":"parse_timeout","knowledge_id":"knowledge-1"}`,
|
||||
`{"kb_id":"kb-1","source_url_hash":"` + sourceB + `","source_ids":["pp-2"],"bytes":10,"sha256":"` + sha + `","status":"content_alias","knowledge_id":"knowledge-1","canonical_source_url_hash":"` + sourceA + `"}`,
|
||||
}, "\n")
|
||||
|
||||
snapshot, err := FoldLedger(strings.NewReader(ledger))
|
||||
if err != nil {
|
||||
t.Fatalf("FoldLedger() error = %v", err)
|
||||
}
|
||||
if snapshot.EventCount != 4 || snapshot.KBID != "kb-1" || len(snapshot.Items) != 2 {
|
||||
t.Fatalf("FoldLedger() snapshot = %#v", snapshot)
|
||||
}
|
||||
if snapshot.Items[0].SourceURLHash != sourceA || snapshot.Items[0].Status != StatusCompleted || snapshot.Items[0].KnowledgeID != "knowledge-1" {
|
||||
t.Fatalf("completed item = %#v", snapshot.Items[0])
|
||||
}
|
||||
if snapshot.Items[1].SourceURLHash != sourceB || snapshot.Items[1].Status != StatusContentAlias || snapshot.Items[1].CanonicalSourceURLHash != sourceA {
|
||||
t.Fatalf("alias item = %#v", snapshot.Items[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestFoldLedgerReportsUnresolvedSourceKeys(t *testing.T) {
|
||||
source := strings.Repeat("a", 64)
|
||||
sha := strings.Repeat("c", 64)
|
||||
ledger := `{"kb_id":"kb-1","source_url_hash":"` + source + `","source_ids":["pp-1"],"bytes":10,"sha256":"` + sha + `","status":"uploaded","knowledge_id":"knowledge-1"}`
|
||||
|
||||
snapshot, err := FoldLedger(strings.NewReader(ledger))
|
||||
if err != nil {
|
||||
t.Fatalf("FoldLedger() error = %v", err)
|
||||
}
|
||||
if snapshot.UnresolvedCount != 1 {
|
||||
t.Fatalf("UnresolvedCount = %d, want 1", snapshot.UnresolvedCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateSnapshotRejectsUnresolvedSourceKeys(t *testing.T) {
|
||||
err := ValidateSnapshot(Snapshot{KBID: "kb-1", UnresolvedCount: 1})
|
||||
if err == nil || !strings.Contains(err.Error(), "unresolved") {
|
||||
t.Fatalf("ValidateSnapshot() error = %v, want unresolved source error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFoldLedgerRejectsMixedKnowledgeBases(t *testing.T) {
|
||||
sourceA := strings.Repeat("a", 64)
|
||||
sourceB := strings.Repeat("b", 64)
|
||||
sha := strings.Repeat("c", 64)
|
||||
ledger := strings.Join([]string{
|
||||
`{"kb_id":"kb-1","source_url_hash":"` + sourceA + `","source_ids":["pp-1"],"bytes":10,"sha256":"` + sha + `","status":"completed","knowledge_id":"knowledge-1"}`,
|
||||
`{"kb_id":"kb-2","source_url_hash":"` + sourceB + `","source_ids":["pp-2"],"bytes":10,"sha256":"` + sha + `","status":"completed","knowledge_id":"knowledge-2"}`,
|
||||
}, "\n")
|
||||
|
||||
_, err := FoldLedger(strings.NewReader(ledger))
|
||||
if err == nil || !strings.Contains(err.Error(), "multiple knowledge bases") {
|
||||
t.Fatalf("FoldLedger() error = %v, want mixed knowledge base error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFoldLedgerRejectsInvalidTerminalSourceHash(t *testing.T) {
|
||||
sha := strings.Repeat("c", 64)
|
||||
ledger := `{"kb_id":"kb-1","source_url_hash":"not-a-hash","source_ids":["pp-1"],"bytes":10,"sha256":"` + sha + `","status":"completed","knowledge_id":"knowledge-1"}`
|
||||
|
||||
_, err := FoldLedger(strings.NewReader(ledger))
|
||||
if err == nil || !strings.Contains(err.Error(), "source_url_hash") {
|
||||
t.Fatalf("FoldLedger() error = %v, want source hash validation error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFoldLedgerRejectsIncompleteTerminalRecord(t *testing.T) {
|
||||
source := strings.Repeat("a", 64)
|
||||
sha := strings.Repeat("c", 64)
|
||||
tests := []struct {
|
||||
name string
|
||||
ledger string
|
||||
}{
|
||||
{"missing knowledge base", `{"source_url_hash":"` + source + `","source_ids":["pp-1"],"bytes":10,"sha256":"` + sha + `","status":"completed","knowledge_id":"knowledge-1"}`},
|
||||
{"missing source IDs", `{"kb_id":"kb-1","source_url_hash":"` + source + `","source_ids":[],"bytes":10,"sha256":"` + sha + `","status":"completed","knowledge_id":"knowledge-1"}`},
|
||||
{"negative bytes", `{"kb_id":"kb-1","source_url_hash":"` + source + `","source_ids":["pp-1"],"bytes":-1,"sha256":"` + sha + `","status":"completed","knowledge_id":"knowledge-1"}`},
|
||||
{"invalid optional SHA", `{"kb_id":"kb-1","source_url_hash":"` + source + `","source_ids":["pp-1"],"bytes":10,"sha256":"bad","status":"completed","knowledge_id":"knowledge-1"}`},
|
||||
{"missing knowledge ID", `{"kb_id":"kb-1","source_url_hash":"` + source + `","source_ids":["pp-1"],"bytes":10,"sha256":"` + sha + `","status":"completed"}`},
|
||||
{"missing alias canonical hash", `{"kb_id":"kb-1","source_url_hash":"` + source + `","source_ids":["pp-1"],"bytes":10,"sha256":"` + sha + `","status":"content_alias","knowledge_id":"knowledge-1"}`},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if _, err := FoldLedger(strings.NewReader(tt.ledger)); err == nil {
|
||||
t.Fatal("FoldLedger() error = nil, want terminal record validation error")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecoverMissingSHA256RequiresVerifiedManifestArtifact(t *testing.T) {
|
||||
content := []byte("verified PP artifact")
|
||||
contentDigest := sha256.Sum256(content)
|
||||
contentHash := hex.EncodeToString(contentDigest[:])
|
||||
url := "https://source.invalid/document"
|
||||
urlDigest := sha256.Sum256([]byte(url))
|
||||
sourceHash := hex.EncodeToString(urlDigest[:])
|
||||
path := filepath.Join(t.TempDir(), "document.bin")
|
||||
if err := os.WriteFile(path, content, 0o600); err != nil {
|
||||
t.Fatalf("write fixture: %v", err)
|
||||
}
|
||||
manifest := `{"results":[{"url":"` + url + `","local_path":"` + path + `","bytes":20,"sha256":"` + contentHash + `"}]}`
|
||||
snapshot := Snapshot{KBID: "kb-1", Items: []Item{{
|
||||
KBID: "kb-1",
|
||||
SourceURLHash: sourceHash,
|
||||
SourceIDs: []string{"pp-1"},
|
||||
Bytes: 20,
|
||||
Status: StatusCompleted,
|
||||
KnowledgeID: "knowledge-1",
|
||||
}}}
|
||||
|
||||
recovered, report, err := RecoverMissingSHA256(snapshot, strings.NewReader(manifest))
|
||||
if err != nil {
|
||||
t.Fatalf("RecoverMissingSHA256() error = %v", err)
|
||||
}
|
||||
if report.Recovered != 1 || report.VerifiedBytes != int64(len(content)) {
|
||||
t.Fatalf("RecoverMissingSHA256() report = %#v", report)
|
||||
}
|
||||
if recovered.Items[0].SHA256 != contentHash {
|
||||
t.Fatalf("recovered SHA256 = %q, want %q", recovered.Items[0].SHA256, contentHash)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecoverMissingSHA256RejectsArtifactOverLegacyLimit(t *testing.T) {
|
||||
url := "https://source.invalid/oversized"
|
||||
urlDigest := sha256.Sum256([]byte(url))
|
||||
sourceHash := hex.EncodeToString(urlDigest[:])
|
||||
contentHash := strings.Repeat("c", 64)
|
||||
oversized := int64(50*1024*1024 + 1)
|
||||
manifest := `{"results":[{"url":"` + url + `","local_path":"/not-opened","bytes":` + "52428801" + `,"sha256":"` + contentHash + `"}]}`
|
||||
snapshot := Snapshot{KBID: "kb-1", Items: []Item{{
|
||||
KBID: "kb-1",
|
||||
SourceURLHash: sourceHash,
|
||||
SourceIDs: []string{"pp-1"},
|
||||
Bytes: oversized,
|
||||
Status: StatusCompleted,
|
||||
KnowledgeID: "knowledge-1",
|
||||
}}}
|
||||
|
||||
_, _, err := RecoverMissingSHA256(snapshot, strings.NewReader(manifest))
|
||||
if err == nil || !strings.Contains(err.Error(), "50 MiB") {
|
||||
t.Fatalf("RecoverMissingSHA256() error = %v, want legacy size limit error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateSnapshotRejectsAliasWithoutCanonicalCompletedItem(t *testing.T) {
|
||||
sourceAlias := strings.Repeat("b", 64)
|
||||
missingCanonical := strings.Repeat("a", 64)
|
||||
sha := strings.Repeat("c", 64)
|
||||
snapshot := Snapshot{KBID: "kb-1", Items: []Item{{
|
||||
KBID: "kb-1",
|
||||
SourceURLHash: sourceAlias,
|
||||
SourceIDs: []string{"pp-2"},
|
||||
Bytes: 10,
|
||||
SHA256: sha,
|
||||
Status: StatusContentAlias,
|
||||
KnowledgeID: "knowledge-1",
|
||||
CanonicalSourceURLHash: missingCanonical,
|
||||
}}}
|
||||
|
||||
err := ValidateSnapshot(snapshot)
|
||||
if err == nil || !strings.Contains(err.Error(), "canonical completed item") {
|
||||
t.Fatalf("ValidateSnapshot() error = %v, want missing canonical error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateSnapshotRejectsAliasMetadataMismatch(t *testing.T) {
|
||||
sourceCanonical := strings.Repeat("a", 64)
|
||||
sourceAlias := strings.Repeat("b", 64)
|
||||
sha := strings.Repeat("c", 64)
|
||||
canonical := Item{KBID: "kb-1", SourceURLHash: sourceCanonical, SourceIDs: []string{"pp-1"}, Bytes: 10, SHA256: sha, Status: StatusCompleted, KnowledgeID: "knowledge-1"}
|
||||
alias := Item{KBID: "kb-1", SourceURLHash: sourceAlias, SourceIDs: []string{"pp-2"}, Bytes: 10, SHA256: sha, Status: StatusContentAlias, KnowledgeID: "knowledge-1", CanonicalSourceURLHash: sourceCanonical}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
mutate func(*Item)
|
||||
}{
|
||||
{"SHA mismatch", func(item *Item) { item.SHA256 = strings.Repeat("d", 64) }},
|
||||
{"knowledge mismatch", func(item *Item) { item.KnowledgeID = "knowledge-2" }},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
candidate := alias
|
||||
tt.mutate(&candidate)
|
||||
err := ValidateSnapshot(Snapshot{KBID: "kb-1", Items: []Item{canonical, candidate}})
|
||||
if err == nil || !strings.Contains(err.Error(), "alias metadata") {
|
||||
t.Fatalf("ValidateSnapshot() error = %v, want alias metadata error", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateSnapshotRequiresRecoveredSHA256(t *testing.T) {
|
||||
source := strings.Repeat("a", 64)
|
||||
snapshot := Snapshot{KBID: "kb-1", Items: []Item{{
|
||||
KBID: "kb-1",
|
||||
SourceURLHash: source,
|
||||
SourceIDs: []string{"pp-1"},
|
||||
Bytes: 10,
|
||||
Status: StatusCompleted,
|
||||
KnowledgeID: "knowledge-1",
|
||||
}}}
|
||||
|
||||
err := ValidateSnapshot(snapshot)
|
||||
if err == nil || !strings.Contains(err.Error(), "verified SHA-256") {
|
||||
t.Fatalf("ValidateSnapshot() error = %v, want missing SHA error", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
package pp
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
)
|
||||
|
||||
type RecoveryReport struct {
|
||||
Recovered int
|
||||
VerifiedBytes int64
|
||||
}
|
||||
|
||||
const LegacyMaxBytes int64 = 50 * 1024 * 1024
|
||||
|
||||
type manifestDocument struct {
|
||||
Results []manifestItem `json:"results"`
|
||||
}
|
||||
|
||||
type manifestItem struct {
|
||||
URL string `json:"url"`
|
||||
LocalPath string `json:"local_path"`
|
||||
Bytes int64 `json:"bytes"`
|
||||
SHA256 string `json:"sha256"`
|
||||
Status string `json:"status"`
|
||||
SourceIDs []string `json:"source_ids"`
|
||||
}
|
||||
|
||||
func RecoverMissingSHA256(snapshot Snapshot, reader io.Reader) (Snapshot, RecoveryReport, error) {
|
||||
var manifest manifestDocument
|
||||
if err := json.NewDecoder(reader).Decode(&manifest); err != nil {
|
||||
return Snapshot{}, RecoveryReport{}, fmt.Errorf("decode download manifest: %w", err)
|
||||
}
|
||||
bySourceHash := make(map[string]manifestItem, len(manifest.Results))
|
||||
for _, item := range manifest.Results {
|
||||
digest := sha256.Sum256([]byte(item.URL))
|
||||
bySourceHash[hex.EncodeToString(digest[:])] = item
|
||||
}
|
||||
|
||||
result := snapshot
|
||||
result.Items = append([]Item(nil), snapshot.Items...)
|
||||
report := RecoveryReport{}
|
||||
for index := range result.Items {
|
||||
item := &result.Items[index]
|
||||
if item.SHA256 != "" {
|
||||
continue
|
||||
}
|
||||
artifact, ok := bySourceHash[item.SourceURLHash]
|
||||
if !ok {
|
||||
return Snapshot{}, RecoveryReport{}, fmt.Errorf("missing manifest artifact for source hash %s", item.SourceURLHash)
|
||||
}
|
||||
if !governance.ValidSHA256(artifact.SHA256) {
|
||||
return Snapshot{}, RecoveryReport{}, fmt.Errorf("manifest artifact has invalid SHA-256 for source hash %s", item.SourceURLHash)
|
||||
}
|
||||
if artifact.Bytes != item.Bytes {
|
||||
return Snapshot{}, RecoveryReport{}, fmt.Errorf("manifest size differs from ledger for source hash %s", item.SourceURLHash)
|
||||
}
|
||||
if artifact.Bytes > LegacyMaxBytes {
|
||||
return Snapshot{}, RecoveryReport{}, fmt.Errorf("manifest artifact exceeds the legacy 50 MiB limit for source hash %s", item.SourceURLHash)
|
||||
}
|
||||
file, err := os.Open(artifact.LocalPath)
|
||||
if err != nil {
|
||||
return Snapshot{}, RecoveryReport{}, fmt.Errorf("open manifest artifact for source hash %s: %w", item.SourceURLHash, err)
|
||||
}
|
||||
digest := sha256.New()
|
||||
bytesRead, copyErr := io.Copy(digest, file)
|
||||
closeErr := file.Close()
|
||||
if copyErr != nil {
|
||||
return Snapshot{}, RecoveryReport{}, fmt.Errorf("hash manifest artifact for source hash %s: %w", item.SourceURLHash, copyErr)
|
||||
}
|
||||
if closeErr != nil {
|
||||
return Snapshot{}, RecoveryReport{}, fmt.Errorf("close manifest artifact for source hash %s: %w", item.SourceURLHash, closeErr)
|
||||
}
|
||||
if bytesRead != artifact.Bytes {
|
||||
return Snapshot{}, RecoveryReport{}, fmt.Errorf("manifest artifact size mismatch for source hash %s", item.SourceURLHash)
|
||||
}
|
||||
actual := hex.EncodeToString(digest.Sum(nil))
|
||||
if actual != artifact.SHA256 {
|
||||
return Snapshot{}, RecoveryReport{}, fmt.Errorf("manifest artifact SHA-256 mismatch for source hash %s", item.SourceURLHash)
|
||||
}
|
||||
item.SHA256 = actual
|
||||
report.Recovered++
|
||||
report.VerifiedBytes += bytesRead
|
||||
}
|
||||
return result, report, nil
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
package pp
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"sort"
|
||||
)
|
||||
|
||||
type OverLimitReport struct {
|
||||
Added int
|
||||
}
|
||||
|
||||
func MergeOverLimitManifest(snapshot Snapshot, reader io.Reader) (Snapshot, OverLimitReport, error) {
|
||||
var manifest manifestDocument
|
||||
if err := json.NewDecoder(reader).Decode(&manifest); err != nil {
|
||||
return Snapshot{}, OverLimitReport{}, fmt.Errorf("decode download manifest: %w", err)
|
||||
}
|
||||
|
||||
result := snapshot
|
||||
result.Items = append([]Item(nil), snapshot.Items...)
|
||||
seen := make(map[string]struct{}, len(result.Items))
|
||||
for _, item := range result.Items {
|
||||
seen[item.SourceURLHash] = struct{}{}
|
||||
}
|
||||
report := OverLimitReport{}
|
||||
for _, artifact := range manifest.Results {
|
||||
if artifact.Status != string(StatusOverLimit) {
|
||||
continue
|
||||
}
|
||||
if artifact.URL == "" || len(artifact.SourceIDs) == 0 || artifact.Bytes <= LegacyMaxBytes {
|
||||
return Snapshot{}, OverLimitReport{}, fmt.Errorf("invalid over-limit manifest record")
|
||||
}
|
||||
digest := sha256.Sum256([]byte(artifact.URL))
|
||||
sourceHash := hex.EncodeToString(digest[:])
|
||||
if _, exists := seen[sourceHash]; exists {
|
||||
continue
|
||||
}
|
||||
result.Items = append(result.Items, Item{
|
||||
KBID: snapshot.KBID,
|
||||
SourceURLHash: sourceHash,
|
||||
SourceIDs: append([]string(nil), artifact.SourceIDs...),
|
||||
Bytes: artifact.Bytes,
|
||||
Status: StatusOverLimit,
|
||||
})
|
||||
seen[sourceHash] = struct{}{}
|
||||
report.Added++
|
||||
}
|
||||
sort.Slice(result.Items, func(i, j int) bool {
|
||||
return result.Items[i].SourceURLHash < result.Items[j].SourceURLHash
|
||||
})
|
||||
return result, report, nil
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
package pp
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestMergeOverLimitManifestAddsBlockedSourceItem(t *testing.T) {
|
||||
canonicalSource := strings.Repeat("a", 64)
|
||||
contentHash := strings.Repeat("c", 64)
|
||||
url := "https://source.invalid/large-document"
|
||||
urlDigest := sha256.Sum256([]byte(url))
|
||||
overLimitSource := hex.EncodeToString(urlDigest[:])
|
||||
snapshot := Snapshot{KBID: "kb-1", Items: []Item{{
|
||||
KBID: "kb-1",
|
||||
SourceURLHash: canonicalSource,
|
||||
SourceIDs: []string{"pp-1"},
|
||||
Bytes: 20,
|
||||
SHA256: contentHash,
|
||||
Status: StatusCompleted,
|
||||
KnowledgeID: "knowledge-1",
|
||||
}}}
|
||||
manifest := `{"results":[` +
|
||||
`{"url":"https://source.invalid/existing","status":"existing","bytes":20,"source_ids":["pp-1"]},` +
|
||||
`{"url":"` + url + `","local_path":"/not-opened","status":"over_limit","bytes":52428801,"source_ids":["pp-2"]}` +
|
||||
`]}`
|
||||
|
||||
merged, report, err := MergeOverLimitManifest(snapshot, strings.NewReader(manifest))
|
||||
if err != nil {
|
||||
t.Fatalf("MergeOverLimitManifest() error = %v", err)
|
||||
}
|
||||
if report.Added != 1 {
|
||||
t.Fatalf("Added = %d, want 1", report.Added)
|
||||
}
|
||||
if len(merged.Items) != 2 {
|
||||
t.Fatalf("items = %d, want 2", len(merged.Items))
|
||||
}
|
||||
var blocked Item
|
||||
for _, item := range merged.Items {
|
||||
if item.SourceURLHash == overLimitSource {
|
||||
blocked = item
|
||||
}
|
||||
}
|
||||
if blocked.Status != StatusOverLimit || blocked.Bytes != 52428801 || blocked.SHA256 != "" {
|
||||
t.Fatalf("blocked item = %#v", blocked)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateSnapshotAcceptsOverLimitItemWithoutContentHash(t *testing.T) {
|
||||
snapshot := Snapshot{KBID: "kb-1", Items: []Item{{
|
||||
KBID: "kb-1",
|
||||
SourceURLHash: strings.Repeat("a", 64),
|
||||
SourceIDs: []string{"pp-1"},
|
||||
Bytes: LegacyMaxBytes + 1,
|
||||
Status: StatusOverLimit,
|
||||
}}}
|
||||
|
||||
if err := ValidateSnapshot(snapshot); err != nil {
|
||||
t.Fatalf("ValidateSnapshot() error = %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
package pp
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
)
|
||||
|
||||
type ImportPlan struct {
|
||||
Connection governance.SourceConnection
|
||||
SourceItems []governance.SourceItem
|
||||
ContentObjects []governance.ContentObject
|
||||
Versions []governance.SourceItemVersion
|
||||
Publications []governance.Publication
|
||||
}
|
||||
|
||||
func BuildImportPlan(snapshot Snapshot) (ImportPlan, error) {
|
||||
if err := ValidateSnapshot(snapshot); err != nil {
|
||||
return ImportPlan{}, err
|
||||
}
|
||||
connectionID := stableID("sc", snapshot.KBID)
|
||||
plan := ImportPlan{
|
||||
Connection: governance.SourceConnection{
|
||||
ID: connectionID,
|
||||
Type: governance.SourceTypePP,
|
||||
Name: "PP legacy " + connectionID,
|
||||
SecretRef: "",
|
||||
Enabled: false,
|
||||
},
|
||||
SourceItems: make([]governance.SourceItem, 0, len(snapshot.Items)),
|
||||
ContentObjects: make([]governance.ContentObject, 0),
|
||||
Versions: make([]governance.SourceItemVersion, 0, len(snapshot.Items)),
|
||||
Publications: make([]governance.Publication, 0),
|
||||
}
|
||||
contentBySHA := make(map[string]governance.ContentObject)
|
||||
publicationByKnowledgeID := make(map[string]struct{})
|
||||
for _, item := range snapshot.Items {
|
||||
itemID := stableID("si", connectionID, item.SourceURLHash)
|
||||
sizeBytes := item.Bytes
|
||||
revision := item.SHA256
|
||||
if item.Status == StatusOverLimit {
|
||||
revision = metadataRevision(item)
|
||||
}
|
||||
plan.SourceItems = append(plan.SourceItems, governance.SourceItem{
|
||||
ID: itemID,
|
||||
SourceConnectionID: connectionID,
|
||||
StableExternalID: item.SourceURLHash,
|
||||
CanonicalURIHash: item.SourceURLHash,
|
||||
Title: "",
|
||||
ItemType: governance.ItemTypeDocument,
|
||||
SizeBytes: &sizeBytes,
|
||||
Revision: revision,
|
||||
Visibility: governance.VisibilityExternal,
|
||||
VisibilityOrigin: governance.VisibilityOriginPPPublicTag,
|
||||
SourceChange: governance.SourceChangeUnchanged,
|
||||
})
|
||||
|
||||
rawSummary := map[string]any{
|
||||
"legacy_status": item.Status,
|
||||
"source_ids": item.SourceIDs,
|
||||
}
|
||||
if item.Status == StatusContentAlias {
|
||||
rawSummary["canonical_source_url_hash"] = item.CanonicalSourceURLHash
|
||||
}
|
||||
rawJSON, err := json.Marshal(rawSummary)
|
||||
if err != nil {
|
||||
return ImportPlan{}, fmt.Errorf("marshal legacy record summary: %w", err)
|
||||
}
|
||||
versionID := stableID("sv", itemID, revision)
|
||||
if item.Status == StatusOverLimit {
|
||||
plan.Versions = append(plan.Versions, governance.SourceItemVersion{
|
||||
ID: versionID,
|
||||
SourceItemID: itemID,
|
||||
Revision: revision,
|
||||
ContentState: governance.ContentOverLimit,
|
||||
DownloadError: "legacy_50_mib_limit",
|
||||
RawRecordSummary: rawJSON,
|
||||
})
|
||||
continue
|
||||
}
|
||||
|
||||
content, ok := contentBySHA[item.SHA256]
|
||||
if !ok {
|
||||
content = governance.ContentObject{
|
||||
ID: stableID("co", item.SHA256),
|
||||
SHA256: item.SHA256,
|
||||
SizeBytes: item.Bytes,
|
||||
State: governance.ContentVerified,
|
||||
}
|
||||
contentBySHA[item.SHA256] = content
|
||||
plan.ContentObjects = append(plan.ContentObjects, content)
|
||||
} else if content.SizeBytes != item.Bytes {
|
||||
return ImportPlan{}, fmt.Errorf("same SHA-256 has conflicting byte sizes")
|
||||
}
|
||||
|
||||
contentID := content.ID
|
||||
plan.Versions = append(plan.Versions, governance.SourceItemVersion{
|
||||
ID: versionID,
|
||||
SourceItemID: itemID,
|
||||
Revision: item.SHA256,
|
||||
ContentObjectID: &contentID,
|
||||
ContentState: governance.ContentVerified,
|
||||
RawRecordSummary: rawJSON,
|
||||
})
|
||||
if item.Status != StatusCompleted {
|
||||
continue
|
||||
}
|
||||
if _, exists := publicationByKnowledgeID[item.KnowledgeID]; exists {
|
||||
return ImportPlan{}, fmt.Errorf("completed records share a knowledge ID")
|
||||
}
|
||||
publicationByKnowledgeID[item.KnowledgeID] = struct{}{}
|
||||
plan.Publications = append(plan.Publications, governance.Publication{
|
||||
ID: stableID("tp", snapshot.KBID, item.KnowledgeID),
|
||||
SourceItemVersionID: versionID,
|
||||
TestKBID: snapshot.KBID,
|
||||
KnowledgeID: item.KnowledgeID,
|
||||
IngestState: governance.IngestUploaded,
|
||||
ProcessingState: governance.ProcessingCompleted,
|
||||
RetrievalState: governance.RetrievalUnpublished,
|
||||
})
|
||||
}
|
||||
return plan, nil
|
||||
}
|
||||
|
||||
func stableID(prefix string, parts ...string) string {
|
||||
digest := sha256.New()
|
||||
for _, part := range parts {
|
||||
_, _ = digest.Write([]byte{0})
|
||||
_, _ = digest.Write([]byte(part))
|
||||
}
|
||||
return prefix + "-" + hex.EncodeToString(digest.Sum(nil))[:32]
|
||||
}
|
||||
|
||||
func metadataRevision(item Item) string {
|
||||
sourceIDs := append([]string(nil), item.SourceIDs...)
|
||||
sort.Strings(sourceIDs)
|
||||
parts := []string{item.SourceURLHash, strconv.FormatInt(item.Bytes, 10)}
|
||||
parts = append(parts, sourceIDs...)
|
||||
digest := sha256.New()
|
||||
for _, part := range parts {
|
||||
_, _ = digest.Write([]byte{0})
|
||||
_, _ = digest.Write([]byte(part))
|
||||
}
|
||||
return "metadata-sha256:" + hex.EncodeToString(digest.Sum(nil))
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
package pp
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
)
|
||||
|
||||
func TestBuildImportPlanSharesAliasContentWithoutDuplicatePublication(t *testing.T) {
|
||||
sourceCanonical := strings.Repeat("a", 64)
|
||||
sourceAlias := strings.Repeat("b", 64)
|
||||
contentHash := strings.Repeat("c", 64)
|
||||
snapshot := Snapshot{KBID: "kb-1", Items: []Item{
|
||||
{KBID: "kb-1", SourceURLHash: sourceCanonical, SourceIDs: []string{"pp-1"}, Bytes: 10, SHA256: contentHash, Status: StatusCompleted, KnowledgeID: "knowledge-1"},
|
||||
{KBID: "kb-1", SourceURLHash: sourceAlias, SourceIDs: []string{"pp-2"}, Bytes: 10, SHA256: contentHash, Status: StatusContentAlias, KnowledgeID: "knowledge-1", CanonicalSourceURLHash: sourceCanonical},
|
||||
}}
|
||||
|
||||
plan, err := BuildImportPlan(snapshot)
|
||||
if err != nil {
|
||||
t.Fatalf("BuildImportPlan() error = %v", err)
|
||||
}
|
||||
if len(plan.SourceItems) != 2 || len(plan.Versions) != 2 || len(plan.ContentObjects) != 1 || len(plan.Publications) != 1 {
|
||||
t.Fatalf("BuildImportPlan() counts = items:%d versions:%d content:%d publications:%d", len(plan.SourceItems), len(plan.Versions), len(plan.ContentObjects), len(plan.Publications))
|
||||
}
|
||||
if plan.Connection.Enabled || plan.Connection.SecretRef != "" || plan.Connection.Type != governance.SourceTypePP {
|
||||
t.Fatalf("connection safety defaults = %#v", plan.Connection)
|
||||
}
|
||||
if plan.Publications[0].RetrievalState != governance.RetrievalUnpublished || plan.Publications[0].ProcessingState != governance.ProcessingCompleted {
|
||||
t.Fatalf("publication states = %#v", plan.Publications[0])
|
||||
}
|
||||
if *plan.Versions[0].ContentObjectID != plan.ContentObjects[0].ID || *plan.Versions[1].ContentObjectID != plan.ContentObjects[0].ID {
|
||||
t.Fatal("canonical and alias versions must reference the same content object")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildImportPlanMapsOverLimitWithoutContentOrPublication(t *testing.T) {
|
||||
snapshot := Snapshot{KBID: "kb-1", Items: []Item{{
|
||||
KBID: "kb-1",
|
||||
SourceURLHash: strings.Repeat("a", 64),
|
||||
SourceIDs: []string{"pp-1"},
|
||||
Bytes: LegacyMaxBytes + 1,
|
||||
Status: StatusOverLimit,
|
||||
}}}
|
||||
|
||||
plan, err := BuildImportPlan(snapshot)
|
||||
if err != nil {
|
||||
t.Fatalf("BuildImportPlan() error = %v", err)
|
||||
}
|
||||
if len(plan.SourceItems) != 1 || len(plan.Versions) != 1 || len(plan.ContentObjects) != 0 || len(plan.Publications) != 0 {
|
||||
t.Fatalf("plan counts = source:%d version:%d content:%d publication:%d", len(plan.SourceItems), len(plan.Versions), len(plan.ContentObjects), len(plan.Publications))
|
||||
}
|
||||
if !strings.HasPrefix(plan.SourceItems[0].Revision, "metadata-sha256:") {
|
||||
t.Fatalf("source revision = %q, want metadata revision", plan.SourceItems[0].Revision)
|
||||
}
|
||||
version := plan.Versions[0]
|
||||
if version.ContentObjectID != nil || version.ContentState != governance.ContentOverLimit || version.DownloadError != "legacy_50_mib_limit" {
|
||||
t.Fatalf("over-limit version = %#v", version)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
package pp
|
||||
|
||||
import "fmt"
|
||||
|
||||
type CurrentItem struct {
|
||||
Revision string
|
||||
}
|
||||
|
||||
type ReconciliationSummary struct {
|
||||
Unchanged int
|
||||
Adopted int
|
||||
OverLimit int
|
||||
UploadRequired int
|
||||
}
|
||||
|
||||
func PlanReconciliation(snapshot Snapshot, current map[string]CurrentItem) (ReconciliationSummary, error) {
|
||||
if err := ValidateSnapshot(snapshot); err != nil {
|
||||
return ReconciliationSummary{}, fmt.Errorf("validate reconciliation snapshot: %w", err)
|
||||
}
|
||||
summary := ReconciliationSummary{}
|
||||
for _, item := range snapshot.Items {
|
||||
if item.Status == StatusOverLimit {
|
||||
summary.OverLimit++
|
||||
continue
|
||||
}
|
||||
stored, exists := current[item.SourceURLHash]
|
||||
if !exists {
|
||||
summary.Adopted++
|
||||
continue
|
||||
}
|
||||
if stored.Revision == item.SHA256 {
|
||||
summary.Unchanged++
|
||||
continue
|
||||
}
|
||||
summary.UploadRequired++
|
||||
}
|
||||
return summary, nil
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
package pp
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestPlanReconciliationSchedulesNoUploadForAdoptedSnapshot(t *testing.T) {
|
||||
completedSource := strings.Repeat("a", 64)
|
||||
overLimitSource := strings.Repeat("b", 64)
|
||||
contentHash := strings.Repeat("c", 64)
|
||||
overLimit := Item{
|
||||
KBID: "kb-1",
|
||||
SourceURLHash: overLimitSource,
|
||||
SourceIDs: []string{"pp-2"},
|
||||
Bytes: LegacyMaxBytes + 1,
|
||||
Status: StatusOverLimit,
|
||||
}
|
||||
snapshot := Snapshot{KBID: "kb-1", Items: []Item{
|
||||
{KBID: "kb-1", SourceURLHash: completedSource, SourceIDs: []string{"pp-1"}, Bytes: 10, SHA256: contentHash, Status: StatusCompleted, KnowledgeID: "knowledge-1"},
|
||||
overLimit,
|
||||
}}
|
||||
current := map[string]CurrentItem{
|
||||
completedSource: {Revision: contentHash},
|
||||
overLimitSource: {Revision: metadataRevision(overLimit)},
|
||||
}
|
||||
|
||||
summary, err := PlanReconciliation(snapshot, current)
|
||||
if err != nil {
|
||||
t.Fatalf("PlanReconciliation() error = %v", err)
|
||||
}
|
||||
if summary.UploadRequired != 0 || summary.Unchanged != 1 || summary.OverLimit != 1 {
|
||||
t.Fatalf("summary = %#v", summary)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlanReconciliationSchedulesUploadWhenSameSourceContentChanges(t *testing.T) {
|
||||
source := strings.Repeat("a", 64)
|
||||
oldHash := strings.Repeat("b", 64)
|
||||
newHash := strings.Repeat("c", 64)
|
||||
snapshot := Snapshot{KBID: "kb-1", Items: []Item{{
|
||||
KBID: "kb-1",
|
||||
SourceURLHash: source,
|
||||
SourceIDs: []string{"pp-1"},
|
||||
Bytes: 10,
|
||||
SHA256: newHash,
|
||||
Status: StatusCompleted,
|
||||
KnowledgeID: "knowledge-1",
|
||||
}}}
|
||||
|
||||
summary, err := PlanReconciliation(snapshot, map[string]CurrentItem{source: {Revision: oldHash}})
|
||||
if err != nil {
|
||||
t.Fatalf("PlanReconciliation() error = %v", err)
|
||||
}
|
||||
if summary.UploadRequired != 1 || summary.Unchanged != 0 {
|
||||
t.Fatalf("summary = %#v", summary)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
package governance
|
||||
|
||||
import "time"
|
||||
|
||||
// ConfirmCatalogChangeSetRequest is the P3 human confirmation: explicit items of
|
||||
// one planned ChangeSet are executed into publications and processing jobs.
|
||||
type ConfirmCatalogChangeSetRequest struct {
|
||||
ChangeSetID string
|
||||
ItemSeqs []int64
|
||||
OperatorRef string
|
||||
Reason string
|
||||
InternalTargetKBID string
|
||||
ExternalTargetKBID string
|
||||
Now time.Time
|
||||
}
|
||||
|
||||
type ConfirmCatalogChangeSetResult struct {
|
||||
ChangeSetID string `json:"change_set_id"`
|
||||
ConfirmedItems int `json:"confirmed_items"`
|
||||
UploadsCreated int `json:"uploads_created"`
|
||||
UploadsExisting int `json:"uploads_existing"`
|
||||
RemovalsRequested int `json:"removals_requested"`
|
||||
SkippedNoArtifact int `json:"skipped_no_artifact"`
|
||||
SkippedNoTarget int `json:"skipped_no_target"`
|
||||
NoChangeIgnored int `json:"no_change_ignored"`
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
package governance
|
||||
|
||||
import (
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestDomainModelContractDeclaresCanonicalObjectsAndCurrentMappings(t *testing.T) {
|
||||
content, err := os.ReadFile("../../docs/DOMAIN_MODEL.md")
|
||||
if err != nil {
|
||||
t.Fatalf("read domain model contract: %v", err)
|
||||
}
|
||||
|
||||
text := string(content)
|
||||
for _, required := range []string{
|
||||
"## Source",
|
||||
"## SourceFile",
|
||||
"## Snapshot",
|
||||
"## ChangeSet",
|
||||
"## Job",
|
||||
"## Release",
|
||||
"SourceConnection → SourceItem → SourceItemVersion → ContentObject",
|
||||
"SourceConnection → SourceCatalogSnapshot → SourceCatalogNode",
|
||||
"SourceCatalogChangeSet → SourceCatalogChangeItem",
|
||||
"ReleaseBatch → ReleaseExecution → ReleaseItemExecution",
|
||||
"source_items",
|
||||
"source_item_versions",
|
||||
"content_objects",
|
||||
"source_catalog_snapshots",
|
||||
"source_catalog_change_sets",
|
||||
"jobs",
|
||||
"release_executions",
|
||||
"Current implementation gaps",
|
||||
} {
|
||||
if !strings.Contains(text, required) {
|
||||
t.Fatalf("domain model contract is missing %q", required)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
package governance
|
||||
|
||||
import "time"
|
||||
|
||||
// JobFailure is one terminal-attempt dead-letter projection (P6). It never
|
||||
// replaces the original jobs audit row; it points at it.
|
||||
type JobFailure struct {
|
||||
ID string `json:"id"`
|
||||
JobID string `json:"job_id"`
|
||||
JobType string `json:"job_type"`
|
||||
PublicationID string `json:"publication_id,omitempty"`
|
||||
SourceItemID string `json:"source_item_id,omitempty"`
|
||||
ErrorCode string `json:"error_code"`
|
||||
Attempt int `json:"attempt"`
|
||||
MaxAttempts int `json:"max_attempts_at_failure"`
|
||||
ChangeSetID string `json:"change_set_id,omitempty"`
|
||||
Action string `json:"action,omitempty"`
|
||||
TargetKBID string `json:"target_kb_id,omitempty"`
|
||||
State string `json:"state"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
type JobFailureFilter struct {
|
||||
State string // open | requeued | abandoned | "" (defaults to open)
|
||||
Limit int
|
||||
Offset int
|
||||
}
|
||||
|
||||
type JobFailurePage struct {
|
||||
Items []JobFailure `json:"items"`
|
||||
Total int `json:"total"`
|
||||
Limit int `json:"limit"`
|
||||
Offset int `json:"offset"`
|
||||
}
|
||||
|
||||
const (
|
||||
RetryStarted = "started"
|
||||
RetryAlreadyRunning = "already_running"
|
||||
RetryNotRetryable = "not_retryable"
|
||||
)
|
||||
|
||||
// RetryPublicationsBatchRequest is the server-side batch retry (P6): one
|
||||
// atomic operation covering 1..100 publications with per-item outcomes.
|
||||
type RetryPublicationsBatchRequest struct {
|
||||
PublicationIDs []string `json:"publication_ids"`
|
||||
OperatorRef string `json:"operator_ref"`
|
||||
Reason string `json:"reason"`
|
||||
Now time.Time `json:"-"`
|
||||
}
|
||||
|
||||
type RetryOutcome struct {
|
||||
PublicationID string `json:"publication_id"`
|
||||
Result string `json:"result"`
|
||||
Detail string `json:"detail,omitempty"`
|
||||
}
|
||||
|
||||
type RetryPublicationsBatchResult struct {
|
||||
BatchKey string `json:"batch_key"`
|
||||
Total int `json:"total"`
|
||||
Started int `json:"started"`
|
||||
AlreadyRunning int `json:"already_running"`
|
||||
NotRetryable int `json:"not_retryable"`
|
||||
Outcomes []RetryOutcome `json:"outcomes"`
|
||||
}
|
||||
@@ -0,0 +1,285 @@
|
||||
package governance
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"time"
|
||||
)
|
||||
|
||||
type SourceType string
|
||||
|
||||
const (
|
||||
SourceTypePP SourceType = "pp"
|
||||
SourceTypeAList SourceType = "alist"
|
||||
SourceTypeDingTalk SourceType = "dingtalk"
|
||||
SourceTypeManual SourceType = "manual"
|
||||
)
|
||||
|
||||
func (sourceType SourceType) Valid() bool {
|
||||
switch sourceType {
|
||||
case SourceTypePP, SourceTypeAList, SourceTypeDingTalk, SourceTypeManual:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
type ItemType string
|
||||
|
||||
const (
|
||||
ItemTypeDocument ItemType = "document"
|
||||
ItemTypeFile ItemType = "file"
|
||||
ItemTypeLink ItemType = "link"
|
||||
ItemTypeVideo ItemType = "video"
|
||||
)
|
||||
|
||||
type Visibility string
|
||||
|
||||
const (
|
||||
VisibilityInternal Visibility = "internal"
|
||||
VisibilityExternal Visibility = "external"
|
||||
VisibilityBoth Visibility = "both"
|
||||
VisibilityBlocked Visibility = "blocked"
|
||||
)
|
||||
|
||||
type VisibilityOrigin string
|
||||
|
||||
const (
|
||||
VisibilityOriginPPPublicTag VisibilityOrigin = "pp_public_tag"
|
||||
VisibilityOriginConfiguredScope VisibilityOrigin = "configured_scope"
|
||||
VisibilityOriginManual VisibilityOrigin = "manual"
|
||||
)
|
||||
|
||||
type SourceChange string
|
||||
|
||||
const (
|
||||
SourceChangeUnchanged SourceChange = "unchanged"
|
||||
SourceChangeAdded SourceChange = "added"
|
||||
SourceChangeUpdated SourceChange = "updated"
|
||||
SourceChangeDeleted SourceChange = "deleted"
|
||||
SourceChangeUnavailable SourceChange = "unavailable"
|
||||
)
|
||||
|
||||
func (state SourceChange) Valid() bool {
|
||||
switch state {
|
||||
case SourceChangeUnchanged, SourceChangeAdded, SourceChangeUpdated, SourceChangeDeleted, SourceChangeUnavailable:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
type ReviewState string
|
||||
|
||||
const (
|
||||
ReviewPending ReviewState = "pending"
|
||||
ReviewApproved ReviewState = "approved"
|
||||
ReviewRejected ReviewState = "rejected"
|
||||
ReviewNotRequired ReviewState = "not_required"
|
||||
)
|
||||
|
||||
func (state ReviewState) Valid() bool {
|
||||
switch state {
|
||||
case ReviewPending, ReviewApproved, ReviewRejected, ReviewNotRequired:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
type ContentState string
|
||||
|
||||
const (
|
||||
ContentUnknown ContentState = "unknown"
|
||||
ContentDownloading ContentState = "downloading"
|
||||
ContentVerified ContentState = "verified"
|
||||
ContentInvalid ContentState = "invalid"
|
||||
ContentOverLimit ContentState = "over_limit"
|
||||
ContentFailed ContentState = "failed"
|
||||
)
|
||||
|
||||
func (state ContentState) Valid() bool {
|
||||
switch state {
|
||||
case ContentUnknown, ContentDownloading, ContentVerified, ContentInvalid, ContentOverLimit, ContentFailed:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
type IngestState string
|
||||
|
||||
const (
|
||||
IngestPending IngestState = "pending"
|
||||
IngestUploading IngestState = "uploading"
|
||||
IngestUploaded IngestState = "uploaded"
|
||||
IngestFailed IngestState = "failed"
|
||||
)
|
||||
|
||||
func (state IngestState) Valid() bool {
|
||||
switch state {
|
||||
case IngestPending, IngestUploading, IngestUploaded, IngestFailed:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
type ProcessingState string
|
||||
|
||||
const (
|
||||
ProcessingPending ProcessingState = "pending"
|
||||
ProcessingProcessing ProcessingState = "processing"
|
||||
ProcessingFinalizing ProcessingState = "finalizing"
|
||||
ProcessingCompleted ProcessingState = "completed"
|
||||
ProcessingFailed ProcessingState = "failed"
|
||||
ProcessingStalled ProcessingState = "stalled"
|
||||
)
|
||||
|
||||
func (state ProcessingState) Valid() bool {
|
||||
switch state {
|
||||
case ProcessingPending, ProcessingProcessing, ProcessingFinalizing, ProcessingCompleted, ProcessingFailed, ProcessingStalled:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
type RetrievalState string
|
||||
|
||||
const (
|
||||
RetrievalUnpublished RetrievalState = "unpublished"
|
||||
RetrievalEnabled RetrievalState = "enabled"
|
||||
RetrievalDisabling RetrievalState = "disabling"
|
||||
RetrievalDisabled RetrievalState = "disabled"
|
||||
RetrievalEnabling RetrievalState = "enabling"
|
||||
RetrievalFailed RetrievalState = "failed"
|
||||
)
|
||||
|
||||
func (state RetrievalState) Valid() bool {
|
||||
switch state {
|
||||
case RetrievalUnpublished, RetrievalEnabled, RetrievalDisabling, RetrievalDisabled, RetrievalEnabling, RetrievalFailed:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
type AcceptanceState string
|
||||
|
||||
const (
|
||||
AcceptancePending AcceptanceState = "pending"
|
||||
AcceptancePassed AcceptanceState = "passed"
|
||||
AcceptanceFailed AcceptanceState = "failed"
|
||||
AcceptanceManualRequired AcceptanceState = "manual_required"
|
||||
)
|
||||
|
||||
func (state AcceptanceState) Valid() bool {
|
||||
switch state {
|
||||
case AcceptancePending, AcceptancePassed, AcceptanceFailed, AcceptanceManualRequired:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func ValidSHA256(value string) bool {
|
||||
if len(value) != 64 {
|
||||
return false
|
||||
}
|
||||
for _, character := range value {
|
||||
if !((character >= '0' && character <= '9') || (character >= 'a' && character <= 'f')) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
type SourceConnection struct {
|
||||
ID string
|
||||
Type SourceType
|
||||
Name string
|
||||
BaseURL string
|
||||
SecretRef string
|
||||
Enabled bool
|
||||
Schedule json.RawMessage
|
||||
LastHealth json.RawMessage
|
||||
CreatedAt time.Time
|
||||
UpdatedAt time.Time
|
||||
}
|
||||
|
||||
type SourceItem struct {
|
||||
ID string
|
||||
SourceConnectionID string
|
||||
StableExternalID string
|
||||
CanonicalURIHash string
|
||||
ParentExternalID string
|
||||
NormalizedPath string
|
||||
Title string
|
||||
ItemType ItemType
|
||||
SizeBytes *int64
|
||||
MIME string
|
||||
Revision string
|
||||
UpdatedAtSource *time.Time
|
||||
Visibility Visibility
|
||||
VisibilityOrigin VisibilityOrigin
|
||||
LastSeenScanID string
|
||||
SourceChange SourceChange
|
||||
AbsenceStreak int
|
||||
CreatedAt time.Time
|
||||
UpdatedAt time.Time
|
||||
}
|
||||
|
||||
type ContentObject struct {
|
||||
ID string
|
||||
SHA256 string
|
||||
SizeBytes int64
|
||||
MIME string
|
||||
ValidatedFormat string
|
||||
LocalSpoolRef string
|
||||
State ContentState
|
||||
FirstDiscoveredAt time.Time
|
||||
LastDiscoveredAt time.Time
|
||||
CreatedAt time.Time
|
||||
UpdatedAt time.Time
|
||||
}
|
||||
|
||||
type SourceItemVersion struct {
|
||||
ID string
|
||||
SourceItemID string
|
||||
Revision string
|
||||
ContentObjectID *string
|
||||
ContentState ContentState
|
||||
DownloadError string
|
||||
RawRecordSummary json.RawMessage
|
||||
SourceUpdatedAt *time.Time
|
||||
CreatedAt time.Time
|
||||
UpdatedAt time.Time
|
||||
}
|
||||
|
||||
type Publication struct {
|
||||
ID string
|
||||
SourceItemVersionID string
|
||||
TestKBID string
|
||||
KnowledgeID string
|
||||
IngestState IngestState
|
||||
ProcessingState ProcessingState
|
||||
RetrievalState RetrievalState
|
||||
ParserGeneration string
|
||||
IndexGeneration string
|
||||
LastError string
|
||||
RetryCount int
|
||||
CreatedAt time.Time
|
||||
UpdatedAt time.Time
|
||||
}
|
||||
|
||||
type AcceptanceResult struct {
|
||||
ID string
|
||||
PublicationID string
|
||||
RunID string
|
||||
CheckType string
|
||||
State AcceptanceState
|
||||
PolicyVersion string
|
||||
EvidenceSummary json.RawMessage
|
||||
ObservedAt time.Time
|
||||
CreatedAt time.Time
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
package governance
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestDeclaredWorkflowStatesAreValid(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
valid bool
|
||||
}{
|
||||
{"source type", SourceTypeManual.Valid() && SourceTypePP.Valid() && !SourceType("__invalid__").Valid()},
|
||||
{"source change", SourceChangeAdded.Valid() && !SourceChange("__invalid__").Valid()},
|
||||
{"review", ReviewPending.Valid() && !ReviewState("__invalid__").Valid()},
|
||||
{"content", ContentVerified.Valid() && !ContentState("__invalid__").Valid()},
|
||||
{"ingest", IngestUploaded.Valid() && !IngestState("__invalid__").Valid()},
|
||||
{"processing", ProcessingCompleted.Valid() && !ProcessingState("__invalid__").Valid()},
|
||||
{"retrieval", RetrievalEnabled.Valid() && !RetrievalState("__invalid__").Valid()},
|
||||
{"acceptance", AcceptanceManualRequired.Valid() && AcceptancePassed.Valid() && AcceptanceFailed.Valid() && !AcceptanceState("__invalid__").Valid()},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if !tt.valid {
|
||||
t.Fatal("declared state must be valid and unknown state must be invalid")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidSHA256RequiresCanonicalDigest(t *testing.T) {
|
||||
valid := "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
|
||||
for _, tt := range []struct {
|
||||
name string
|
||||
value string
|
||||
want bool
|
||||
}{
|
||||
{"canonical", valid, true},
|
||||
{"short", valid[:63], false},
|
||||
{"uppercase", "A" + valid[1:], false},
|
||||
{"non hex", "g" + valid[1:], false},
|
||||
} {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := ValidSHA256(tt.value); got != tt.want {
|
||||
t.Fatalf("ValidSHA256() = %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestModelsKeepSourceVersionContentAndPublicationSeparate(t *testing.T) {
|
||||
contentID := "content-1"
|
||||
item := SourceItem{ID: "item-1", StableExternalID: "pp-42", SourceChange: SourceChangeAdded}
|
||||
version := SourceItemVersion{ID: "version-1", SourceItemID: item.ID, ContentObjectID: &contentID, ContentState: ContentVerified}
|
||||
publication := Publication{ID: "publication-1", SourceItemVersionID: version.ID, IngestState: IngestUploaded, ProcessingState: ProcessingCompleted, RetrievalState: RetrievalEnabled}
|
||||
|
||||
if version.SourceItemID != item.ID || *version.ContentObjectID != contentID || publication.SourceItemVersionID != version.ID {
|
||||
t.Fatal("source, version, content, and publication identities must remain independently linked")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
package governance
|
||||
|
||||
import "time"
|
||||
|
||||
type MonitorHost struct {
|
||||
CPUPercent float64 `json:"cpu_percent"`
|
||||
MemoryUsedBytes int64 `json:"memory_used_bytes"`
|
||||
MemoryTotalBytes int64 `json:"memory_total_bytes"`
|
||||
DiskUsedBytes int64 `json:"disk_used_bytes"`
|
||||
DiskTotalBytes int64 `json:"disk_total_bytes"`
|
||||
GPUState string `json:"gpu_state"`
|
||||
GPUDetail string `json:"gpu_detail"`
|
||||
Simulated bool `json:"simulated"`
|
||||
}
|
||||
|
||||
type MonitorService struct {
|
||||
Key string `json:"key"`
|
||||
Name string `json:"name"`
|
||||
State string `json:"state"`
|
||||
Detail string `json:"detail"`
|
||||
LatencyMS int `json:"latency_ms"`
|
||||
ObservedAt time.Time `json:"observed_at"`
|
||||
Simulated bool `json:"simulated"`
|
||||
}
|
||||
|
||||
// ServiceHealth (P10) separates "process reachable" (Liveness) from "can
|
||||
// accept work" (Readiness). not_configured and not_applicable are explicit and
|
||||
// never collapse into unknown.
|
||||
type ServiceHealth struct {
|
||||
Key string `json:"key"`
|
||||
Name string `json:"name"`
|
||||
Liveness string `json:"liveness"`
|
||||
Readiness string `json:"readiness"`
|
||||
Reason string `json:"reason"`
|
||||
LatencyMS int64 `json:"latency_ms"`
|
||||
ObservedAt time.Time `json:"observed_at"`
|
||||
LastSuccessAt *time.Time `json:"last_success_at"`
|
||||
ConsecutiveFailures int `json:"consecutive_failures"`
|
||||
Configured bool `json:"configured"`
|
||||
Applicable bool `json:"applicable"`
|
||||
}
|
||||
|
||||
type KnowledgeBaseProgress struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Visibility string `json:"visibility"`
|
||||
Imported int `json:"imported"`
|
||||
Completed int `json:"completed"`
|
||||
Pending int `json:"pending"`
|
||||
Processing int `json:"processing"`
|
||||
Finalizing int `json:"finalizing"`
|
||||
Failed int `json:"failed"`
|
||||
Cancelled int `json:"cancelled"`
|
||||
InternalImported int `json:"internal_imported"`
|
||||
InternalCompleted int `json:"internal_completed"`
|
||||
ExternalImported int `json:"external_imported"`
|
||||
ExternalCompleted int `json:"external_completed"`
|
||||
}
|
||||
|
||||
// JobWindowMetrics reports a frozen statistical window; insufficient samples
|
||||
// stay null and are never rendered as 0.
|
||||
type JobWindowMetrics struct {
|
||||
WindowSeconds int `json:"window_seconds"`
|
||||
JobsStarted int `json:"jobs_started"`
|
||||
JobsCompleted int `json:"jobs_completed"`
|
||||
JobsFailed int `json:"jobs_failed"`
|
||||
SuccessRate *float64 `json:"success_rate"`
|
||||
FailureRate *float64 `json:"failure_rate"`
|
||||
DurationAvgSeconds *float64 `json:"duration_avg_seconds"`
|
||||
DurationP50Seconds *float64 `json:"duration_p50_seconds"`
|
||||
DurationP95Seconds *float64 `json:"duration_p95_seconds"`
|
||||
}
|
||||
|
||||
type QueueGauges struct {
|
||||
Pending int `json:"pending"`
|
||||
Running int `json:"running"`
|
||||
Retry int `json:"retry"`
|
||||
Failed int `json:"failed"`
|
||||
OldestPendingAgeSeconds *float64 `json:"oldest_pending_age_seconds"`
|
||||
}
|
||||
|
||||
type RunMetrics struct {
|
||||
ObservedAt time.Time `json:"observed_at"`
|
||||
ActiveWorkers int `json:"active_workers"`
|
||||
WorkerSource string `json:"worker_source"`
|
||||
Queue QueueGauges `json:"queue"`
|
||||
ShortWindow JobWindowMetrics `json:"short_window"`
|
||||
LongWindow JobWindowMetrics `json:"long_window"`
|
||||
}
|
||||
|
||||
type MonitorOverview struct {
|
||||
Mode string `json:"mode"`
|
||||
Simulated bool `json:"simulated"`
|
||||
ObservedAt time.Time `json:"observed_at"`
|
||||
Host MonitorHost `json:"host"`
|
||||
Services []MonitorService `json:"services"`
|
||||
KnowledgeBases []KnowledgeBaseProgress `json:"knowledge_bases"`
|
||||
Health []ServiceHealth `json:"health"`
|
||||
}
|
||||
|
||||
type KnowledgeMonitorFile struct {
|
||||
Name string `json:"name"`
|
||||
Directory string `json:"directory"`
|
||||
Visibility string `json:"visibility"`
|
||||
Status string `json:"status"`
|
||||
StageDetail string `json:"stage_detail"`
|
||||
ElapsedSecond int `json:"elapsed_seconds"`
|
||||
ErrorCode string `json:"error_code"`
|
||||
}
|
||||
|
||||
type KnowledgeMonitorFilePage struct {
|
||||
Items []KnowledgeMonitorFile `json:"items"`
|
||||
Total int `json:"total"`
|
||||
Limit int `json:"limit"`
|
||||
Offset int `json:"offset"`
|
||||
SampleOnly bool `json:"sample_only"`
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
package governance
|
||||
|
||||
import "time"
|
||||
|
||||
// ReleaseAuditChain is the R6 read-only evidence projection: one execution can
|
||||
// be traced to its batch, the confirmed release change set, the bound source
|
||||
// snapshots and catalog change sets, and every item's review, acceptance run
|
||||
// and source identity. Only safe identifiers, labels, counts and timestamps
|
||||
// travel here — never bundle paths, credentials, or signed URLs.
|
||||
type ReleaseAuditChain struct {
|
||||
ExecutionID string `json:"execution_id"`
|
||||
Operation string `json:"operation"`
|
||||
State string `json:"state"`
|
||||
FailureLevel string `json:"failure_level"`
|
||||
BatchID string `json:"release_batch_id"`
|
||||
BatchName string `json:"release_batch_name"`
|
||||
CreatedBy string `json:"created_by"`
|
||||
SealedBy string `json:"sealed_by"`
|
||||
AuthorizedBy string `json:"authorized_by"`
|
||||
ReleaseChangeSetID string `json:"release_change_set_id"`
|
||||
ChangeSetName string `json:"release_change_set_name"`
|
||||
ChangeSetNumber int `json:"release_change_set_number"`
|
||||
ChangeSetState string `json:"release_change_set_state"`
|
||||
ChangeSetCreatedBy string `json:"release_change_set_created_by"`
|
||||
ChangeSetConfirmedBy string `json:"release_change_set_confirmed_by"`
|
||||
ChangeSetConfirmedAt *time.Time `json:"release_change_set_confirmed_at"`
|
||||
ManifestSHA256 string `json:"manifest_sha256"`
|
||||
Snapshots []AuditSnapshot `json:"snapshots"`
|
||||
CatalogChangeSets []AuditCatalogSet `json:"catalog_change_sets"`
|
||||
PriorExecutionID string `json:"prior_execution_id"`
|
||||
RollbackExecutionID string `json:"rollback_execution_id"`
|
||||
Items []AuditChainItem `json:"items"`
|
||||
}
|
||||
|
||||
type AuditSnapshot struct {
|
||||
ID string `json:"id"`
|
||||
SourceName string `json:"source_name"`
|
||||
NodeCount int `json:"node_count"`
|
||||
FileCount int `json:"file_count"`
|
||||
ObservedAt time.Time `json:"observed_at"`
|
||||
}
|
||||
|
||||
type AuditCatalogSet struct {
|
||||
ID string `json:"id"`
|
||||
State string `json:"state"`
|
||||
AddCount int `json:"add_count"`
|
||||
UpdateCount int `json:"update_count"`
|
||||
MoveCount int `json:"move_count"`
|
||||
DeleteCount int `json:"delete_count"`
|
||||
}
|
||||
|
||||
type AuditChainItem struct {
|
||||
PublicationID string `json:"publication_id"`
|
||||
Title string `json:"title"`
|
||||
Action string `json:"action"`
|
||||
ItemState string `json:"item_state"`
|
||||
ReviewID string `json:"review_id"`
|
||||
AcceptanceRunID string `json:"acceptance_run_id"`
|
||||
SourceItemID string `json:"source_item_id"`
|
||||
SourceVersionID string `json:"source_item_version_id"`
|
||||
ProductionRevision string `json:"production_revision"`
|
||||
ErrorCode string `json:"error_code"`
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
package governance
|
||||
|
||||
import "time"
|
||||
|
||||
type ReleaseBatchState string
|
||||
|
||||
const (
|
||||
ReleaseBatchDraft ReleaseBatchState = "draft"
|
||||
ReleaseBatchSealed ReleaseBatchState = "sealed"
|
||||
ReleaseBatchCancelled ReleaseBatchState = "cancelled"
|
||||
)
|
||||
|
||||
func (state ReleaseBatchState) Valid() bool {
|
||||
switch state {
|
||||
case ReleaseBatchDraft, ReleaseBatchSealed, ReleaseBatchCancelled:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
type ReleaseBatch struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
State ReleaseBatchState `json:"state"`
|
||||
PolicyVersion string `json:"policy_version"`
|
||||
CreatedBy string `json:"created_by"`
|
||||
SealedBy string `json:"sealed_by"`
|
||||
ItemCount int `json:"item_count"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
SealedAt *time.Time `json:"sealed_at"`
|
||||
ReleaseChangeSetID string `json:"release_change_set_id,omitempty"`
|
||||
}
|
||||
|
||||
type CreateReleaseBatchRequest struct {
|
||||
Name string
|
||||
CreatedBy string
|
||||
PublicationIDs []string
|
||||
DeletePublicationIDs []string
|
||||
// ReleaseChangeSetID (R1): a formal batch must cite a CONFIRMED persistent
|
||||
// release change set; empty keeps legacy ad-hoc batches draft-only.
|
||||
ReleaseChangeSetID string
|
||||
Now time.Time
|
||||
}
|
||||
|
||||
type SealReleaseBatchRequest struct {
|
||||
BatchID string
|
||||
SealedBy string
|
||||
Now time.Time
|
||||
}
|
||||
|
||||
type ReleaseBatchFilter struct {
|
||||
State string
|
||||
Limit int
|
||||
Offset int
|
||||
}
|
||||
|
||||
type ReleaseBatchPage struct {
|
||||
Items []ReleaseBatch `json:"items"`
|
||||
Total int `json:"total"`
|
||||
Limit int `json:"limit"`
|
||||
Offset int `json:"offset"`
|
||||
}
|
||||
|
||||
type ReleaseBatchItemView struct {
|
||||
PublicationID string `json:"publication_id"`
|
||||
Title string `json:"title"`
|
||||
DecidedVisibility Visibility `json:"decided_visibility"`
|
||||
Action string `json:"action"`
|
||||
}
|
||||
|
||||
type ReleaseBatchDetail struct {
|
||||
ReleaseBatch
|
||||
Items []ReleaseBatchItemView `json:"items"`
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
package governance
|
||||
|
||||
import "time"
|
||||
|
||||
// ReleaseChangeSet is the R1 persistent, immutable release plan: generated from
|
||||
// the live plan exactly once, bound to the source snapshots it was computed
|
||||
// against, hash-sealed, and confirmed before batches may cite it.
|
||||
type ReleaseChangeSet struct {
|
||||
ID string `json:"id"`
|
||||
Number int `json:"number"`
|
||||
Name string `json:"name"`
|
||||
State string `json:"state"`
|
||||
SourceSnapshotIDs []string `json:"source_snapshot_ids"`
|
||||
SourceChangeSetIDs []string `json:"source_change_set_ids"`
|
||||
AcceptanceCutoff time.Time `json:"acceptance_cutoff"`
|
||||
ProductionBaselineRef string `json:"production_baseline_ref"`
|
||||
ItemCount int `json:"item_count"`
|
||||
AddCount int `json:"add_count"`
|
||||
UpdateCount int `json:"update_count"`
|
||||
DeleteCount int `json:"delete_count"`
|
||||
BlockedCount int `json:"blocked_count"`
|
||||
ManifestSHA256 string `json:"manifest_sha256"`
|
||||
CreatedBy string `json:"created_by"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
ConfirmedBy string `json:"confirmed_by"`
|
||||
ConfirmedAt *time.Time `json:"confirmed_at"`
|
||||
}
|
||||
|
||||
type GenerateReleaseChangeSetRequest struct {
|
||||
Name string
|
||||
CreatedBy string
|
||||
Now time.Time
|
||||
}
|
||||
|
||||
type ConfirmReleaseChangeSetRequest struct {
|
||||
ChangeSetID string
|
||||
ConfirmedBy string
|
||||
Now time.Time
|
||||
}
|
||||
|
||||
type ReleaseChangeSetItem struct {
|
||||
Seq int `json:"seq"`
|
||||
PublicationID string `json:"publication_id"`
|
||||
Title string `json:"title"`
|
||||
TargetScope string `json:"target_scope"`
|
||||
Action string `json:"action"`
|
||||
TestState string `json:"test_state"`
|
||||
ProductionState string `json:"production_state"`
|
||||
CanRelease bool `json:"can_release"`
|
||||
Blocker string `json:"blocker"`
|
||||
}
|
||||
|
||||
type ReleaseChangeSetDetail struct {
|
||||
ReleaseChangeSet
|
||||
Items []ReleaseChangeSetItem `json:"items"`
|
||||
}
|
||||
|
||||
type ReleaseChangeSetPage struct {
|
||||
Items []ReleaseChangeSet `json:"items"`
|
||||
Total int `json:"total"`
|
||||
Limit int `json:"limit"`
|
||||
Offset int `json:"offset"`
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
package governance
|
||||
|
||||
import "time"
|
||||
|
||||
type ReleaseOperation string
|
||||
|
||||
const (
|
||||
ReleaseOperationPublish ReleaseOperation = "publish"
|
||||
ReleaseOperationRollback ReleaseOperation = "rollback"
|
||||
)
|
||||
|
||||
func (operation ReleaseOperation) Valid() bool {
|
||||
return operation == ReleaseOperationPublish || operation == ReleaseOperationRollback
|
||||
}
|
||||
|
||||
type ReleaseExecutionState string
|
||||
|
||||
const (
|
||||
ReleaseExecutionPending ReleaseExecutionState = "pending"
|
||||
ReleaseExecutionRunning ReleaseExecutionState = "running"
|
||||
// R4: a rollback execution compensates in rolling_back and terminates in
|
||||
// rolled_back (all compensated) or rollback_failed (manual action needed).
|
||||
ReleaseExecutionRollingBack ReleaseExecutionState = "rolling_back"
|
||||
ReleaseExecutionRolledBack ReleaseExecutionState = "rolled_back"
|
||||
ReleaseExecutionRollbackFailed ReleaseExecutionState = "rollback_failed"
|
||||
ReleaseExecutionCompleted ReleaseExecutionState = "completed"
|
||||
ReleaseExecutionFailed ReleaseExecutionState = "failed"
|
||||
)
|
||||
|
||||
func (state ReleaseExecutionState) Valid() bool {
|
||||
switch state {
|
||||
case ReleaseExecutionPending, ReleaseExecutionRunning, ReleaseExecutionRollingBack,
|
||||
ReleaseExecutionRolledBack, ReleaseExecutionRollbackFailed,
|
||||
ReleaseExecutionCompleted, ReleaseExecutionFailed:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
type ReleaseExecution struct {
|
||||
ID string `json:"id"`
|
||||
ReleaseBatchID string `json:"release_batch_id"`
|
||||
Operation ReleaseOperation `json:"operation"`
|
||||
PriorExecutionID string `json:"prior_execution_id"`
|
||||
State ReleaseExecutionState `json:"state"`
|
||||
FailureLevel string `json:"failure_level"`
|
||||
RestoreTrigger string `json:"restore_trigger"`
|
||||
ReleaseChangeSetID string `json:"release_change_set_id"`
|
||||
TargetEnvironment string `json:"target_environment"`
|
||||
AuthorizedBy string `json:"authorized_by"`
|
||||
AuthorizationCode string `json:"authorization_code"`
|
||||
JobID string `json:"job_id"`
|
||||
ManifestSHA256 string `json:"manifest_sha256"`
|
||||
ItemCount int `json:"item_count"`
|
||||
BatchSize int `json:"batch_size"`
|
||||
CurrentBatch int `json:"current_batch"`
|
||||
TotalBatches int `json:"total_batches"`
|
||||
CompletedItemCount int `json:"completed_item_count"`
|
||||
FailedItemCount int `json:"failed_item_count"`
|
||||
ErrorCode string `json:"error_code"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
StartedAt *time.Time `json:"started_at"`
|
||||
CompletedAt *time.Time `json:"completed_at"`
|
||||
}
|
||||
|
||||
type AuthorizeReleaseBatchRequest struct {
|
||||
BatchID string
|
||||
AuthorizedBy string
|
||||
AuthorizationCode string
|
||||
Now time.Time
|
||||
}
|
||||
|
||||
type AuthorizeReleaseRollbackRequest struct {
|
||||
PriorExecutionID string
|
||||
AuthorizedBy string
|
||||
AuthorizationCode string
|
||||
Now time.Time
|
||||
}
|
||||
|
||||
type RetryReleaseExecutionRequest struct {
|
||||
ExecutionID string
|
||||
Now time.Time
|
||||
}
|
||||
|
||||
type ReleaseExecutionFilter struct {
|
||||
Operation string
|
||||
State string
|
||||
Limit int
|
||||
Offset int
|
||||
}
|
||||
|
||||
type ReleaseExecutionPage struct {
|
||||
Items []ReleaseExecution
|
||||
Total int
|
||||
Limit int
|
||||
Offset int
|
||||
}
|
||||
|
||||
type ReleaseItemExecutionView struct {
|
||||
PublicationID string `json:"publication_id"`
|
||||
Title string `json:"title"`
|
||||
Action string `json:"action"`
|
||||
State string `json:"state"`
|
||||
PreviousProductionRevision string `json:"previous_production_revision"`
|
||||
ProductionRevision string `json:"production_revision"`
|
||||
ErrorCode string `json:"error_code"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
CompletedAt *time.Time `json:"completed_at"`
|
||||
}
|
||||
|
||||
type ReleaseExecutionDetail struct {
|
||||
ReleaseExecution
|
||||
Items []ReleaseItemExecutionView `json:"items"`
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
package governance
|
||||
|
||||
type ReleasePlanItem struct {
|
||||
PublicationID string `json:"publication_id"`
|
||||
Title string `json:"title"`
|
||||
TargetScope string `json:"target_scope"`
|
||||
Action string `json:"action"`
|
||||
TestState string `json:"test_state"`
|
||||
ProductionState string `json:"production_state"`
|
||||
CanRelease bool `json:"can_release"`
|
||||
Blocker string `json:"blocker"`
|
||||
}
|
||||
|
||||
type ReleasePlanSummary struct {
|
||||
Add int `json:"add"`
|
||||
Update int `json:"update"`
|
||||
Delete int `json:"delete"`
|
||||
Blocked int `json:"blocked"`
|
||||
Unchanged int `json:"unchanged"`
|
||||
}
|
||||
|
||||
type ReleasePlan struct {
|
||||
Items []ReleasePlanItem `json:"items"`
|
||||
Summary ReleasePlanSummary `json:"summary"`
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
package governance
|
||||
|
||||
import "time"
|
||||
|
||||
type ReviewGateStatus string
|
||||
|
||||
const (
|
||||
ReviewGateReady ReviewGateStatus = "ready"
|
||||
ReviewGateBlocked ReviewGateStatus = "blocked"
|
||||
)
|
||||
|
||||
func (status ReviewGateStatus) Valid() bool {
|
||||
return status == ReviewGateReady || status == ReviewGateBlocked
|
||||
}
|
||||
|
||||
type ReviewCandidate struct {
|
||||
PublicationID string `json:"publication_id"`
|
||||
Title string `json:"title"`
|
||||
SourceType SourceType `json:"source_type"`
|
||||
CurrentVisibility Visibility `json:"current_visibility"`
|
||||
AcceptanceRunID string `json:"-"`
|
||||
KnowledgeExists AcceptanceState `json:"knowledge_exists"`
|
||||
ChunksPresent AcceptanceState `json:"chunks_present"`
|
||||
RetrievalHit AcceptanceState `json:"retrieval_hit"`
|
||||
GateStatus ReviewGateStatus `json:"gate_status"`
|
||||
GateCode string `json:"gate_code"`
|
||||
ReviewState ReviewState `json:"review_state"`
|
||||
DecidedVisibility string `json:"decided_visibility"`
|
||||
DecisionCode string `json:"decision_code"`
|
||||
ReviewerRef string `json:"reviewer_ref"`
|
||||
ReviewedAt *time.Time `json:"reviewed_at"`
|
||||
PublicationUpdated time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
type ReviewCandidateFilter struct {
|
||||
ReviewState string
|
||||
Limit int
|
||||
Offset int
|
||||
}
|
||||
|
||||
type ReviewCandidatePage struct {
|
||||
Items []ReviewCandidate `json:"items"`
|
||||
Total int `json:"total"`
|
||||
Limit int `json:"limit"`
|
||||
Offset int `json:"offset"`
|
||||
}
|
||||
|
||||
type PublicationReview struct {
|
||||
ID string `json:"id"`
|
||||
PublicationID string `json:"publication_id"`
|
||||
Decision ReviewState `json:"decision"`
|
||||
DecidedVisibility Visibility `json:"decided_visibility"`
|
||||
ReviewerRef string `json:"reviewer_ref"`
|
||||
DecisionCode string `json:"decision_code"`
|
||||
AcceptanceRunID string `json:"-"`
|
||||
PublicationUpdated time.Time `json:"-"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
type ReviewPublicationRequest struct {
|
||||
PublicationID string
|
||||
Decision ReviewState
|
||||
DecidedVisibility Visibility
|
||||
ReviewerRef string
|
||||
DecisionCode string
|
||||
Now time.Time
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
package governance
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
type ScopeTargetType string
|
||||
|
||||
const (
|
||||
ScopeTargetDirectory ScopeTargetType = "directory"
|
||||
ScopeTargetItem ScopeTargetType = "item"
|
||||
)
|
||||
|
||||
type ScopeRuleMutation struct {
|
||||
TargetType ScopeTargetType
|
||||
ConnectionID string
|
||||
DirectoryPath string
|
||||
SourceItemID string
|
||||
Visibility Visibility
|
||||
OperatorRef string
|
||||
Reason string
|
||||
Clear bool
|
||||
Now time.Time
|
||||
}
|
||||
|
||||
type ScopeRule struct {
|
||||
TargetType ScopeTargetType `json:"target_type"`
|
||||
ConnectionID string `json:"connection_id,omitempty"`
|
||||
DirectoryPath string `json:"directory_path,omitempty"`
|
||||
SourceItemID string `json:"source_item_id,omitempty"`
|
||||
Visibility Visibility `json:"visibility"`
|
||||
UpdatedBy string `json:"updated_by"`
|
||||
Reason string `json:"reason"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
func (request ScopeRuleMutation) Validate() error {
|
||||
if request.TargetType != ScopeTargetDirectory && request.TargetType != ScopeTargetItem {
|
||||
return errors.New("scope target must be directory or item")
|
||||
}
|
||||
if request.TargetType == ScopeTargetDirectory {
|
||||
if strings.TrimSpace(request.ConnectionID) == "" || request.SourceItemID != "" || !validScopePath(request.DirectoryPath) {
|
||||
return errors.New("directory scope target is invalid")
|
||||
}
|
||||
} else if strings.TrimSpace(request.SourceItemID) == "" || request.ConnectionID != "" || request.DirectoryPath != "" {
|
||||
return errors.New("item scope target is invalid")
|
||||
}
|
||||
if !request.Clear && request.Visibility != VisibilityInternal && request.Visibility != VisibilityExternal {
|
||||
return errors.New("visibility must be internal or external")
|
||||
}
|
||||
if request.Clear && request.Visibility != "" {
|
||||
return errors.New("clear scope must not include visibility")
|
||||
}
|
||||
operator := strings.TrimSpace(request.OperatorRef)
|
||||
reason := strings.TrimSpace(request.Reason)
|
||||
if operator == "" || len(operator) > 128 || reason == "" || len(reason) > 500 || request.Now.IsZero() {
|
||||
return errors.New("operator, reason, and time are required")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validScopePath(value string) bool {
|
||||
if len(value) > 2048 || strings.HasPrefix(value, "/") || strings.Contains(value, "\\") {
|
||||
return false
|
||||
}
|
||||
for _, part := range strings.Split(value, "/") {
|
||||
if part == "." || part == ".." {
|
||||
return false
|
||||
}
|
||||
}
|
||||
for _, value := range value {
|
||||
if unicode.IsControl(value) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
package governance
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestValidateScopeRuleMutationAcceptsDirectoryRootAndItemOverride(t *testing.T) {
|
||||
now := time.Now()
|
||||
for _, request := range []ScopeRuleMutation{
|
||||
{TargetType: ScopeTargetDirectory, ConnectionID: "conn-1", DirectoryPath: "", Visibility: VisibilityInternal, OperatorRef: "engineer-1", Reason: "仅供内部使用", Now: now},
|
||||
{TargetType: ScopeTargetDirectory, ConnectionID: "conn-1", DirectoryPath: "产品/说明书", Visibility: VisibilityExternal, OperatorRef: "engineer-1", Reason: "官网公开资料", Now: now},
|
||||
{TargetType: ScopeTargetItem, SourceItemID: "item-1", Visibility: VisibilityInternal, OperatorRef: "engineer-1", Reason: "文件单独包含内部参数", Now: now},
|
||||
} {
|
||||
if err := request.Validate(); err != nil {
|
||||
t.Errorf("Validate(%#v) error = %v", request, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateScopeRuleMutationRejectsAmbiguousOrUnsafeTargets(t *testing.T) {
|
||||
valid := ScopeRuleMutation{TargetType: ScopeTargetDirectory, ConnectionID: "conn-1", Visibility: VisibilityInternal, OperatorRef: "engineer-1", Reason: "范围调整", Now: time.Now()}
|
||||
cases := []ScopeRuleMutation{
|
||||
{},
|
||||
{TargetType: ScopeTargetDirectory, ConnectionID: "conn-1", SourceItemID: "item-1", Visibility: VisibilityInternal, OperatorRef: "engineer", Reason: "x", Now: time.Now()},
|
||||
{TargetType: ScopeTargetDirectory, ConnectionID: "conn-1", DirectoryPath: "/private", Visibility: VisibilityInternal, OperatorRef: "engineer", Reason: "x", Now: time.Now()},
|
||||
{TargetType: ScopeTargetDirectory, ConnectionID: "conn-1", DirectoryPath: "a/../private", Visibility: VisibilityInternal, OperatorRef: "engineer", Reason: "x", Now: time.Now()},
|
||||
{TargetType: ScopeTargetItem, SourceItemID: "item-1", Visibility: Visibility("public"), OperatorRef: "engineer", Reason: "x", Now: time.Now()},
|
||||
{TargetType: ScopeTargetItem, SourceItemID: "item-1", Visibility: VisibilityInternal, OperatorRef: "", Reason: "x", Now: time.Now()},
|
||||
{TargetType: ScopeTargetItem, SourceItemID: "item-1", Visibility: VisibilityInternal, OperatorRef: "engineer", Reason: "", Now: time.Now()},
|
||||
}
|
||||
for _, request := range cases {
|
||||
if err := request.Validate(); err == nil {
|
||||
t.Errorf("Validate(%#v) error=nil", request)
|
||||
}
|
||||
}
|
||||
valid.Clear = true
|
||||
valid.Visibility = ""
|
||||
if err := valid.Validate(); err != nil {
|
||||
t.Fatalf("clear validation error = %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
package governance
|
||||
|
||||
import "time"
|
||||
|
||||
type SourceAsset struct {
|
||||
CanRetry bool `json:"can_retry"`
|
||||
Title string `json:"title"`
|
||||
DirectoryPath string `json:"directory_path"`
|
||||
ConnectionID string `json:"connection_id"`
|
||||
VisibilityOrigin string `json:"visibility_origin"`
|
||||
ScopeRulePath string `json:"scope_rule_path"`
|
||||
IngestState string `json:"ingest_state"`
|
||||
LastError string `json:"last_error"`
|
||||
PublicationID string `json:"publication_id"`
|
||||
ID string `json:"id"`
|
||||
SourceType SourceType `json:"source_type"`
|
||||
ConnectionName string `json:"connection_name"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
Visibility Visibility `json:"visibility"`
|
||||
SourceChange SourceChange `json:"source_change"`
|
||||
ContentState ContentState `json:"content_state"`
|
||||
LegacyStatus string `json:"legacy_status"`
|
||||
HasPublication bool `json:"has_publication"`
|
||||
ProcessingState string `json:"processing_state"`
|
||||
RetrievalState string `json:"retrieval_state"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
type SourceAssetFilter struct {
|
||||
State string
|
||||
Limit int
|
||||
Offset int
|
||||
}
|
||||
|
||||
type SourceAssetSummary struct {
|
||||
Total int `json:"total"`
|
||||
Completed int `json:"completed"`
|
||||
ContentAlias int `json:"content_alias"`
|
||||
OverLimit int `json:"over_limit"`
|
||||
}
|
||||
|
||||
type SourceAssetPage struct {
|
||||
Items []SourceAsset `json:"items"`
|
||||
Summary SourceAssetSummary `json:"summary"`
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
package governance
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSourceAssetIncludesDocumentNavigationMetadata(t *testing.T) {
|
||||
data, err := json.Marshal(SourceAsset{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, field := range []string{"title", "directory_path", "connection_id", "visibility_origin", "ingest_state", "last_error", "publication_id", "can_retry"} {
|
||||
if !strings.Contains(string(data), `"`+field+`"`) {
|
||||
t.Errorf("missing document field %s", field)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,154 @@
|
||||
package governance
|
||||
|
||||
import "time"
|
||||
|
||||
type CatalogSource struct {
|
||||
ConnectionID string `json:"connection_id"`
|
||||
SourceType SourceType `json:"source_type"`
|
||||
Name string `json:"name"`
|
||||
RootNodeID string `json:"root_node_id"`
|
||||
FolderCount int `json:"folder_count"`
|
||||
FileCount int `json:"file_count"`
|
||||
ObservedAt time.Time `json:"observed_at"`
|
||||
SnapshotOnly bool `json:"snapshot_only"`
|
||||
SnapshotStatus string `json:"snapshot_status"`
|
||||
LastRefreshDurationMS int64 `json:"last_refresh_duration_ms"`
|
||||
}
|
||||
|
||||
type CatalogBreadcrumb struct {
|
||||
NodeID string `json:"node_id"`
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
type CatalogNode struct {
|
||||
NodeID string `json:"node_id"`
|
||||
ParentNodeID string `json:"parent_node_id,omitempty"`
|
||||
NodeType string `json:"node_type"`
|
||||
Name string `json:"name"`
|
||||
DirectChildCount int `json:"direct_child_count"`
|
||||
SubtreeFileCount int `json:"subtree_file_count"`
|
||||
SubtreeBytes int64 `json:"subtree_bytes"`
|
||||
SizeBytes *int64 `json:"size_bytes,omitempty"`
|
||||
ModifiedAt *time.Time `json:"modified_at,omitempty"`
|
||||
ContentSHA256 *string `json:"content_sha256,omitempty"`
|
||||
Visibility string `json:"visibility"`
|
||||
ImportedCount int `json:"imported_count"`
|
||||
ParsedCount int `json:"parsed_count"`
|
||||
ActiveCount int `json:"active_count"`
|
||||
FailedCount int `json:"failed_count"`
|
||||
RoutableCount int `json:"routable_count"`
|
||||
}
|
||||
|
||||
type CatalogNodeFilter struct {
|
||||
ConnectionID string
|
||||
ParentNodeID string
|
||||
Limit int
|
||||
Offset int
|
||||
}
|
||||
|
||||
type CatalogNodePage struct {
|
||||
Current CatalogNode `json:"current"`
|
||||
Breadcrumbs []CatalogBreadcrumb `json:"breadcrumbs"`
|
||||
Items []CatalogNode `json:"items"`
|
||||
Total int `json:"total"`
|
||||
Limit int `json:"limit"`
|
||||
Offset int `json:"offset"`
|
||||
}
|
||||
|
||||
type CatalogSnapshot struct {
|
||||
SnapshotID string `json:"snapshot_id"`
|
||||
ConnectionID string `json:"connection_id"`
|
||||
RootNodeID string `json:"root_node_id"`
|
||||
SourceName string `json:"source_name"`
|
||||
NodeCount int `json:"node_count"`
|
||||
FolderCount int `json:"folder_count"`
|
||||
FileCount int `json:"file_count"`
|
||||
ObservedAt time.Time `json:"observed_at"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
IsHead bool `json:"is_head"`
|
||||
}
|
||||
|
||||
type CatalogSnapshotPage struct {
|
||||
Items []CatalogSnapshot `json:"items"`
|
||||
Total int `json:"total"`
|
||||
Limit int `json:"limit"`
|
||||
Offset int `json:"offset"`
|
||||
}
|
||||
|
||||
type CatalogSnapshotNodeFilter struct {
|
||||
ConnectionID string
|
||||
SnapshotID string
|
||||
ParentNodeID string
|
||||
Limit int
|
||||
Offset int
|
||||
}
|
||||
|
||||
type CatalogRouteRequest struct {
|
||||
ConnectionID string
|
||||
NodeIDs []string
|
||||
Visibility Visibility
|
||||
TargetKnowledgeBaseID string
|
||||
TargetKnowledgeBaseIDs []string
|
||||
OperatorRef string
|
||||
Reason string
|
||||
Now time.Time
|
||||
}
|
||||
|
||||
// CatalogRouteResult describes the desired-state write only. S7 guarantees the
|
||||
// annotation never creates publications or jobs; counts explain what the
|
||||
// subsequent ChangeSet/P3 pipeline will be able to act on.
|
||||
type CatalogRouteResult struct {
|
||||
SelectedFiles int `json:"selected_files"`
|
||||
RoutableFiles int `json:"routable_files"`
|
||||
UnboundFiles int `json:"unbound_files"`
|
||||
HashUnknownFiles int `json:"hash_unknown_files"`
|
||||
UnavailableFiles int `json:"unavailable_files"`
|
||||
}
|
||||
|
||||
type CatalogChangeSetFilter struct {
|
||||
ConnectionID string
|
||||
Action string
|
||||
Limit int
|
||||
Offset int
|
||||
}
|
||||
|
||||
type CatalogChangeSet struct {
|
||||
ID string `json:"id"`
|
||||
ConnectionID string `json:"connection_id"`
|
||||
SourceName string `json:"source_name"`
|
||||
State string `json:"state"`
|
||||
AddCount int `json:"add_count"`
|
||||
UpdateCount int `json:"update_count"`
|
||||
MoveCount int `json:"move_count"`
|
||||
MoveScopeCount int `json:"move_scope_count"`
|
||||
DeleteCount int `json:"delete_count"`
|
||||
NoChangeCount int `json:"no_change_count"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
type CatalogChangeItem struct {
|
||||
Action string `json:"action"`
|
||||
Title string `json:"title"`
|
||||
OldPath string `json:"old_path,omitempty"`
|
||||
NewPath string `json:"new_path,omitempty"`
|
||||
OldSizeBytes *int64 `json:"old_size_bytes,omitempty"`
|
||||
NewSizeBytes *int64 `json:"new_size_bytes,omitempty"`
|
||||
OldModified *time.Time `json:"old_modified_at,omitempty"`
|
||||
NewModified *time.Time `json:"new_modified_at,omitempty"`
|
||||
OldContentSHA256 string `json:"old_content_sha256,omitempty"`
|
||||
NewContentSHA256 string `json:"new_content_sha256,omitempty"`
|
||||
ReasonCode string `json:"reason_code"`
|
||||
Seq int64 `json:"seq"`
|
||||
OldScope string `json:"old_scope,omitempty"`
|
||||
NewScope string `json:"new_scope,omitempty"`
|
||||
OldVersionID string `json:"old_version_id,omitempty"`
|
||||
NewVersionID string `json:"new_version_id,omitempty"`
|
||||
}
|
||||
|
||||
type CatalogChangeSetDetail struct {
|
||||
CatalogChangeSet
|
||||
Items []CatalogChangeItem `json:"items"`
|
||||
Total int `json:"total"`
|
||||
Limit int `json:"limit"`
|
||||
Offset int `json:"offset"`
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
package governance
|
||||
|
||||
import "time"
|
||||
|
||||
type SourceHealthStatus string
|
||||
|
||||
const (
|
||||
SourceHealthUnknown SourceHealthStatus = "unknown"
|
||||
SourceHealthHealthy SourceHealthStatus = "healthy"
|
||||
SourceHealthDegraded SourceHealthStatus = "degraded"
|
||||
SourceHealthUnhealthy SourceHealthStatus = "unhealthy"
|
||||
)
|
||||
|
||||
func (status SourceHealthStatus) Valid() bool {
|
||||
switch status {
|
||||
case SourceHealthUnknown, SourceHealthHealthy, SourceHealthDegraded, SourceHealthUnhealthy:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
type SourceConnectionView struct {
|
||||
ID string `json:"id"`
|
||||
SourceType SourceType `json:"source_type"`
|
||||
Name string `json:"name"`
|
||||
Enabled bool `json:"enabled"`
|
||||
ScheduleConfigured bool `json:"schedule_configured"`
|
||||
HealthStatus SourceHealthStatus `json:"health_status"`
|
||||
ItemCount int `json:"item_count"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
NextRefreshAt *time.Time `json:"next_refresh_at"`
|
||||
LastScan *SourceScanSummary `json:"last_scan"`
|
||||
}
|
||||
|
||||
// SourceConnectionRevision is one validated/active/retired settings revision of
|
||||
// a source connection (V1). The JSON surface uses endpoint/credential_ref: the
|
||||
// credential field is a reference NAME only — real secret values never live in
|
||||
// this table and can never be read back through any endpoint.
|
||||
type SourceConnectionRevision struct {
|
||||
ID string `json:"id"`
|
||||
RevisionNumber int `json:"revision_number"`
|
||||
State string `json:"state"`
|
||||
Name string `json:"name"`
|
||||
BaseURL string `json:"endpoint"`
|
||||
SecretRef string `json:"credential_ref"`
|
||||
Enabled bool `json:"enabled"`
|
||||
RefreshIntervalSeconds string `json:"refresh_interval_seconds"`
|
||||
CreatedBy string `json:"created_by"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
ValidatedAt *time.Time `json:"validated_at"`
|
||||
ActivatedAt *time.Time `json:"activated_at"`
|
||||
RetiredAt *time.Time `json:"retired_at"`
|
||||
}
|
||||
|
||||
type SourceScanSummary struct {
|
||||
ID string `json:"id"`
|
||||
State SourceScanState `json:"state"`
|
||||
ObservedItems int `json:"observed_items"`
|
||||
UnchangedItems int `json:"unchanged_items"`
|
||||
AddedItems int `json:"added_items"`
|
||||
UpdatedItems int `json:"updated_items"`
|
||||
OverLimitItems int `json:"over_limit_items"`
|
||||
MissingItems int `json:"missing_items"`
|
||||
PlannedUploads int `json:"planned_uploads"`
|
||||
ErrorCode string `json:"error_code"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
CompletedAt *time.Time `json:"completed_at"`
|
||||
}
|
||||
|
||||
type SourceConnectionSummary struct {
|
||||
Total int `json:"total"`
|
||||
Enabled int `json:"enabled"`
|
||||
Disabled int `json:"disabled"`
|
||||
Healthy int `json:"healthy"`
|
||||
Degraded int `json:"degraded"`
|
||||
Unhealthy int `json:"unhealthy"`
|
||||
Unknown int `json:"unknown"`
|
||||
}
|
||||
|
||||
type SourceConnectionPage struct {
|
||||
Items []SourceConnectionView `json:"items"`
|
||||
Summary SourceConnectionSummary `json:"summary"`
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
package governance
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// The frontend source-settings parser requires the next_refresh_at KEY to
|
||||
// exist whenever a connection is listed — null is a first-class value ("manual
|
||||
// only"), not something to omit. A `omitempty` here once blanked the entire
|
||||
// settings page in the deployed stack because every source defaulted to
|
||||
// manual-only and the strict parser rejected the key-less payload.
|
||||
func TestSourceConnectionViewAlwaysProjectsNextRefreshAt(t *testing.T) {
|
||||
view := SourceConnectionView{ID: "conn-1", SourceType: SourceTypePP, Name: "PP", Enabled: true, ScheduleConfigured: false, HealthStatus: SourceHealthHealthy, ItemCount: 621, UpdatedAt: time.Now().UTC()}
|
||||
payload, err := json.Marshal([]SourceConnectionView{view})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(string(payload), `"next_refresh_at":null`) {
|
||||
t.Fatalf("manual-only connection must project next_refresh_at:null, got %s", payload)
|
||||
}
|
||||
|
||||
at := time.Now().UTC().Truncate(time.Second)
|
||||
view.NextRefreshAt = &at
|
||||
payload, err = json.Marshal([]SourceConnectionView{view})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(string(payload), `"next_refresh_at":"`) {
|
||||
t.Fatalf("scheduled connection must project a timestamp, got %s", payload)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
package governance
|
||||
|
||||
import "time"
|
||||
|
||||
type SourceScanState string
|
||||
|
||||
const (
|
||||
SourceScanPending SourceScanState = "pending"
|
||||
SourceScanRunning SourceScanState = "running"
|
||||
SourceScanCompleted SourceScanState = "completed"
|
||||
SourceScanFailed SourceScanState = "failed"
|
||||
)
|
||||
|
||||
func (s SourceScanState) Valid() bool {
|
||||
switch s {
|
||||
case SourceScanPending, SourceScanRunning, SourceScanCompleted, SourceScanFailed:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
type SourceScan struct {
|
||||
ID string `json:"id"`
|
||||
SourceConnectionID string `json:"source_connection_id"`
|
||||
SourceType SourceType `json:"source_type"`
|
||||
State SourceScanState `json:"state"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
type SourceScanSummaryView struct {
|
||||
ID string `json:"id"`
|
||||
State SourceScanState `json:"state"`
|
||||
ObservedItems int `json:"observed_items"`
|
||||
UnchangedItems int `json:"unchanged_items"`
|
||||
AddedItems int `json:"added_items"`
|
||||
UpdatedItems int `json:"updated_items"`
|
||||
OverLimitItems int `json:"over_limit_items"`
|
||||
MissingItems int `json:"missing_items"`
|
||||
PlannedUploads int `json:"planned_uploads"`
|
||||
ErrorCode string `json:"error_code"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
CompletedAt *time.Time `json:"completed_at"`
|
||||
}
|
||||
|
||||
type SourceScanPage struct {
|
||||
Items []SourceScanSummaryView
|
||||
Total int
|
||||
Limit int
|
||||
Offset int
|
||||
}
|
||||
|
||||
type SourceScanChangeView struct {
|
||||
ID string `json:"id"`
|
||||
SourceItemID *string `json:"source_item_id"`
|
||||
ChangeType SourceScanChangeType `json:"change_type"`
|
||||
ObservedSizeBytes *int64 `json:"observed_size_bytes"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
type SourceScanDetailView struct {
|
||||
SourceScanSummaryView
|
||||
Changes []SourceScanChangeView `json:"changes"`
|
||||
}
|
||||
|
||||
type SourceScanResult struct {
|
||||
SnapshotSHA256 string `json:"-"`
|
||||
ObservedItems int `json:"observed_items"`
|
||||
UnchangedItems int `json:"unchanged_items"`
|
||||
AddedItems int `json:"added_items"`
|
||||
UpdatedItems int `json:"updated_items"`
|
||||
OverLimitItems int `json:"over_limit_items"`
|
||||
MissingItems int `json:"missing_items"`
|
||||
PlannedUploads int `json:"planned_uploads"`
|
||||
Changes []SourceScanChange `json:"-"`
|
||||
}
|
||||
|
||||
type SourceScanChangeType string
|
||||
|
||||
const (
|
||||
SourceScanAdded SourceScanChangeType = "added"
|
||||
SourceScanUpdated SourceScanChangeType = "updated"
|
||||
SourceScanUnavailable SourceScanChangeType = "unavailable"
|
||||
)
|
||||
|
||||
type SourceScanChange struct {
|
||||
StableExternalID string
|
||||
ChangeType SourceScanChangeType
|
||||
PreviousRevision string
|
||||
ObservedRevision string
|
||||
ObservedSizeBytes *int64
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
package governance
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"time"
|
||||
)
|
||||
|
||||
// V1 per-source refresh schedule. The schedule JSONB lives on the ACTIVE
|
||||
// revision; next_refresh_at on source_connections is ALWAYS derived from it by
|
||||
// the server (clients can never push an arbitrary due time).
|
||||
|
||||
const MinRefreshInterval = 5 * time.Minute
|
||||
|
||||
type SourceSchedule struct {
|
||||
RefreshIntervalSeconds *int `json:"refresh_interval_seconds,omitempty"`
|
||||
}
|
||||
|
||||
// ParseSourceSchedule accepts the stored JSON and normalises it. Missing,
|
||||
// zero, or sub-5-minute values are rejected as "not configured" (interval 0,
|
||||
// no error) rather than silently accepted: a source that could stampede the
|
||||
// portal every few seconds is worse than a manual-only source.
|
||||
func ParseSourceSchedule(raw []byte) (time.Duration, error) {
|
||||
if len(raw) == 0 || string(raw) == "{}" {
|
||||
return 0, nil
|
||||
}
|
||||
var schedule SourceSchedule
|
||||
if err := json.Unmarshal(raw, &schedule); err != nil {
|
||||
return 0, errors.New("invalid source schedule")
|
||||
}
|
||||
if schedule.RefreshIntervalSeconds == nil || *schedule.RefreshIntervalSeconds == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
if *schedule.RefreshIntervalSeconds < 0 {
|
||||
return 0, errors.New("invalid source schedule")
|
||||
}
|
||||
interval := time.Duration(*schedule.RefreshIntervalSeconds) * time.Second
|
||||
if interval < MinRefreshInterval {
|
||||
return 0, errors.New("source refresh interval must be at least five minutes")
|
||||
}
|
||||
return interval, nil
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
package governance
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestParseSourceSchedule(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
raw string
|
||||
want time.Duration
|
||||
wantErr bool
|
||||
}{
|
||||
{"empty object = manual only", "{}", 0, false},
|
||||
{"missing key", `{"other":1}`, 0, false},
|
||||
{"explicit zero disables", `{"refresh_interval_seconds":0}`, 0, false},
|
||||
{"valid hourly", `{"refresh_interval_seconds":3600}`, time.Hour, false},
|
||||
{"valid daily", `{"refresh_interval_seconds":86400}`, 24 * time.Hour, false},
|
||||
{"below floor rejected", `{"refresh_interval_seconds":60}`, 0, true},
|
||||
{"negative rejected", `{"refresh_interval_seconds":-5}`, 0, true},
|
||||
{"malformed rejected", `not json`, 0, true},
|
||||
{"string value rejected", `{"refresh_interval_seconds":"3600"}`, 0, true},
|
||||
}
|
||||
for _, test := range cases {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
got, err := ParseSourceSchedule([]byte(test.raw))
|
||||
if (err != nil) != test.wantErr {
|
||||
t.Fatalf("error=%v wantErr=%v", err, test.wantErr)
|
||||
}
|
||||
if got != test.want {
|
||||
t.Fatalf("interval=%v want %v", got, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSourceScheduleRoundTripsThroughJSON(t *testing.T) {
|
||||
seconds := 7200
|
||||
encoded, err := json.Marshal(SourceSchedule{RefreshIntervalSeconds: &seconds})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
interval, err := ParseSourceSchedule(encoded)
|
||||
if err != nil || interval != 2*time.Hour {
|
||||
t.Fatalf("interval=%v err=%v", interval, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
package governance
|
||||
|
||||
import (
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestStateMachineContractSeparatesDimensionsAndReservedStates(t *testing.T) {
|
||||
content, err := os.ReadFile("../../docs/STATE_MACHINE.md")
|
||||
if err != nil {
|
||||
t.Fatalf("read state machine contract: %v", err)
|
||||
}
|
||||
|
||||
text := string(content)
|
||||
for _, required := range []string{
|
||||
"## State dimensions",
|
||||
"### Job",
|
||||
"pending | running | retry | completed | failed | cancelled",
|
||||
"### Scope policy",
|
||||
"unset | internal | external | blocked",
|
||||
"historical compatibility value: both",
|
||||
"### Source change",
|
||||
"### Review",
|
||||
"### Content",
|
||||
"### Ingest",
|
||||
"### Processing",
|
||||
"### Retrieval",
|
||||
"### Release",
|
||||
"## Reserved future values",
|
||||
"rolling_back",
|
||||
"must not be persisted before R4",
|
||||
} {
|
||||
if !strings.Contains(text, required) {
|
||||
t.Fatalf("state machine contract is missing %q", required)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
package governance
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestPublication is the safe projection of a test publication for the API and web console.
|
||||
// It deliberately omits knowledge_id, test_kb_id, source URL, SHA-256, spool path,
|
||||
// raw summaries and any credential material.
|
||||
type TestPublication struct {
|
||||
ID string `json:"id"`
|
||||
Title string `json:"title"`
|
||||
IngestState string `json:"ingest_state"`
|
||||
ProcessingState string `json:"processing_state"`
|
||||
RetrievalState string `json:"retrieval_state"`
|
||||
TargetScope string `json:"target_scope"`
|
||||
ChangeSetID string `json:"change_set_id"`
|
||||
ChangeType string `json:"change_type"`
|
||||
CanStart bool `json:"can_start"`
|
||||
RetryCount int `json:"retry_count"`
|
||||
LastError string `json:"last_error"`
|
||||
CanRetry bool `json:"can_retry"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
type TestPublicationFilter struct {
|
||||
ProcessingState string
|
||||
// TargetScope filters the test queue by KB routing target (P4).
|
||||
TargetScope string
|
||||
Limit int
|
||||
Offset int
|
||||
}
|
||||
|
||||
type TestPublicationPage struct {
|
||||
Items []TestPublication `json:"items"`
|
||||
Total int `json:"total"`
|
||||
}
|
||||
|
||||
type StartTestPublicationsRequest struct {
|
||||
PublicationIDs []string
|
||||
OperatorRef string
|
||||
Reason string
|
||||
Now time.Time
|
||||
}
|
||||
|
||||
type StartTestPublicationsResult struct {
|
||||
Selected int `json:"selected"`
|
||||
Started int `json:"started"`
|
||||
AlreadyStarted int `json:"already_started"`
|
||||
Unavailable int `json:"unavailable"`
|
||||
}
|
||||
|
||||
// AcceptanceRecord is the safe projection of an acceptance result for the API.
|
||||
// EvidenceSummary is returned as an object but only carries counts and booleans,
|
||||
// never remote bodies, paths or credentials.
|
||||
type AcceptanceRecord struct {
|
||||
CheckType string `json:"check_type"`
|
||||
State string `json:"state"`
|
||||
PolicyVersion string `json:"policy_version"`
|
||||
EvidenceSummary json.RawMessage `json:"evidence_summary"`
|
||||
ObservedAt time.Time `json:"observed_at"`
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
package governance
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type UISettings struct {
|
||||
ID string `json:"id"`
|
||||
State string `json:"state"`
|
||||
DocumentRefreshSeconds int `json:"document_refresh_seconds"`
|
||||
ResourceRefreshSeconds int `json:"resource_refresh_seconds"`
|
||||
StalledAfterSeconds int `json:"stalled_after_seconds"`
|
||||
FinalizingAfterSeconds int `json:"finalizing_after_seconds"`
|
||||
UpdatedBy string `json:"updated_by"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
ActivatedAt *time.Time `json:"activated_at"`
|
||||
RetiredAt *time.Time `json:"retired_at"`
|
||||
ErrorCode string `json:"error_code"`
|
||||
}
|
||||
|
||||
type CreateUISettingsCandidateRequest struct {
|
||||
DocumentRefreshSeconds int
|
||||
ResourceRefreshSeconds int
|
||||
StalledAfterSeconds int
|
||||
FinalizingAfterSeconds int
|
||||
UpdatedBy string
|
||||
Now time.Time
|
||||
}
|
||||
|
||||
func (request CreateUISettingsCandidateRequest) Valid() bool {
|
||||
return oneOfInt(request.DocumentRefreshSeconds, 15, 30, 60) &&
|
||||
oneOfInt(request.ResourceRefreshSeconds, 30, 60) &&
|
||||
oneOfInt(request.StalledAfterSeconds, 1800, 3600) &&
|
||||
oneOfInt(request.FinalizingAfterSeconds, 7200, 14400) &&
|
||||
validSettingsActor(request.UpdatedBy) && !request.Now.IsZero()
|
||||
}
|
||||
|
||||
func oneOfInt(value int, allowed ...int) bool {
|
||||
for _, candidate := range allowed {
|
||||
if value == candidate {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func validSettingsActor(value string) bool {
|
||||
trimmed := strings.TrimSpace(value)
|
||||
return trimmed == value && len(trimmed) >= 2 && len(trimmed) <= 128 && !strings.ContainsAny(trimmed, "\x00\r\n\t")
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestOpenAPIContractMatchesRegisteredRoutesAndOmitsSecrets(t *testing.T) {
|
||||
contract, err := os.ReadFile("../../api/openapi.yaml")
|
||||
if err != nil {
|
||||
t.Fatalf("read OpenAPI contract: %v", err)
|
||||
}
|
||||
|
||||
var document struct {
|
||||
OpenAPI string `json:"openapi"`
|
||||
Info map[string]any `json:"info"`
|
||||
Paths map[string]map[string]any `json:"paths"`
|
||||
Components map[string]map[string]any `json:"components"`
|
||||
}
|
||||
if err := json.Unmarshal(contract, &document); err != nil {
|
||||
t.Fatalf("OpenAPI contract must use JSON-compatible YAML: %v", err)
|
||||
}
|
||||
if document.OpenAPI != "3.1.0" || document.Info["title"] != "Knowledge Governance API" || len(document.Paths) == 0 {
|
||||
t.Fatalf("invalid OpenAPI header: %#v", document)
|
||||
}
|
||||
|
||||
routes, err := os.ReadFile("server.go")
|
||||
if err != nil {
|
||||
t.Fatalf("read registered routes: %v", err)
|
||||
}
|
||||
matcher := regexp.MustCompile(`mux\.HandleFunc\("(GET|POST|PUT|DELETE) ([^"]+)"`)
|
||||
for _, match := range matcher.FindAllStringSubmatch(string(routes), -1) {
|
||||
method, path := strings.ToLower(match[1]), match[2]
|
||||
operation, ok := document.Paths[path][method].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("OpenAPI is missing %s %s", strings.ToUpper(method), path)
|
||||
}
|
||||
if strings.TrimSpace(stringValue(operation["operationId"])) == "" {
|
||||
t.Fatalf("OpenAPI operation %s %s has no operationId", strings.ToUpper(method), path)
|
||||
}
|
||||
responses, ok := operation["responses"].(map[string]any)
|
||||
if !ok || responses["400"] == nil || responses["500"] == nil {
|
||||
t.Fatalf("OpenAPI operation %s %s must document 400 and 500 responses", strings.ToUpper(method), path)
|
||||
}
|
||||
}
|
||||
|
||||
lower := strings.ToLower(string(contract))
|
||||
for _, forbidden := range []string{"secret_ref", "authorization_code", "manifest_sha256", "knowledge_id", "test_kb_id", "base_url", "download_code", "cookie"} {
|
||||
if strings.Contains(lower, forbidden) {
|
||||
t.Fatalf("OpenAPI contract must not expose %q", forbidden)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAPICatalogMutationsRequireIdempotencyKeyHeader(t *testing.T) {
|
||||
contract, err := os.ReadFile("../../api/openapi.yaml")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var document struct {
|
||||
Paths map[string]map[string]any `json:"paths"`
|
||||
Components struct {
|
||||
Parameters map[string]map[string]any `json:"parameters"`
|
||||
} `json:"components"`
|
||||
}
|
||||
if err := json.Unmarshal(contract, &document); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
parameter, ok := document.Components.Parameters["IdempotencyKey"]
|
||||
if !ok || parameter["name"] != "Idempotency-Key" || parameter["in"] != "header" || parameter["required"] != true {
|
||||
t.Fatalf("IdempotencyKey header must be required: %#v", parameter)
|
||||
}
|
||||
schema, ok := parameter["schema"].(map[string]any)
|
||||
if !ok || schema["type"] != "string" || schema["minLength"] != float64(1) || schema["maxLength"] != float64(128) {
|
||||
t.Fatalf("IdempotencyKey header schema is unsafe: %#v", parameter["schema"])
|
||||
}
|
||||
for _, path := range []string{"/api/v1/source-catalog/route", "/api/v1/source-catalog/sources/{id}/refresh"} {
|
||||
operation, ok := document.Paths[path]["post"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("missing POST %s", path)
|
||||
}
|
||||
parameters, ok := operation["parameters"].([]any)
|
||||
if !ok {
|
||||
t.Fatalf("POST %s must declare the idempotency header", path)
|
||||
}
|
||||
found := false
|
||||
for _, value := range parameters {
|
||||
parameterRef, _ := value.(map[string]any)
|
||||
if parameterRef["$ref"] == "#/components/parameters/IdempotencyKey" {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("POST %s must reference IdempotencyKey", path)
|
||||
}
|
||||
responses, _ := operation["responses"].(map[string]any)
|
||||
accepted, _ := responses["202"].(map[string]any)
|
||||
if accepted["$ref"] != "#/components/responses/OperationEnvelope" {
|
||||
t.Fatalf("POST %s must document created/replayed operation response: %#v", path, accepted)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func stringValue(value any) string {
|
||||
result, _ := value.(string)
|
||||
return result
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user