Files
knowledge-governance/internal/monitoring/real.go
T
姚章浩 d0a45d563f fix(monitor): 需下架车道恒为 0(判据用了没人写的状态)+ 隔离会改线上设置的集成测试
界面验收第三遍(禁止导入 → 需下架)暴露两处真问题:

1) 需下架车道死掉:判据 `retrieval_state='enabled'`,而 retrieval_state 在这套状态机里
   描述「是否发布到生产」(unpublished/disabling/disabled/failed),全仓没有任何代码写过
   'enabled',线上 5/5 发布单元都是 unpublished → 车道恒 0,界面「仍在知识库里的文件会
   出现在这里」的承诺不成立。改为与设计注释一致的口径:仍在测试库存活(completed + 有
   knowledge_id + 未 disabling/disabled)+ 来源项 policy_state='blocked'。
   线上对照:旧判据 0 → 新判据 1,命中被禁止导入的 DH-AG95夹爪适配-指令协议.md;
   修后页面显示「需下架(1)· 对内知识库 · 1 份仍在检索中」。新增集成测试覆盖
   存活/未禁止/移除中/已移除/未上传五种情形(真实 Postgres + 独立 schema)。

2) 集成测试改到线上设置(直接踩到并已恢复):ui_settings_integration_test.go 只判断
   TEST_DATABASE_URL 是否存在、无 schema 隔离,而它「创建候选并激活」——跑在共享库上会把
   真实 active 设置挤成 retired。已先备份(backups/ui-settings/…pre-restore.jsonl),再用
   单事务(先 retire 再 activate,uq_ui_setting_active 只允许一条 active)恢复
   uis-d40ea1fc…(15/30/1800/7200 + https://weknora-test.dobot.cn/),并复核线上值。
   根因修复:给该用例加 per-test schema 隔离,复跑后线上设置不再被改动。

同时把两处与事实不符的文案改实话(WeKnoraMonitor.tsx 需下架说明、SourceCatalogBrowser.tsx
禁止导入说明):移除走 test.remove 工作流(删除测试库知识条目,可再入库),入口当前由
变更集删除路径触发,页面尚无独立按钮——条目级列表与下架按钮列为待排期。
2026-10-10 10:55:03 +08:00

385 lines
18 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package monitoring
import (
"context"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"os"
"strconv"
"strings"
"syscall"
"time"
"github.com/jackc/pgx/v5"
"knowledge-governance/internal/governance"
)
type Database interface {
Ping(context.Context) error
Query(context.Context, string, ...any) (pgx.Rows, error)
QueryRow(context.Context, string, ...any) pgx.Row
}
type RealProviderConfig struct {
DB Database
Now func() time.Time
HTTPClient *http.Client
WeKnoraBaseURL string
MinerUBaseURL string
// TestWeKnoraResolver reads the test endpoint an admin saved in the web
// settings, so 「服务配置」is the single place the console probes. The env
// value above stays the bootstrap fallback for containers wired before the
// UI existed.
TestWeKnoraResolver func(ctx context.Context) string
// ProductionWeKnoraBaseURL lets the console show the real state of the
// production WeKnora endpoint. Publishing itself keeps reading its target
// from the release runtime config, never from here.
ProductionWeKnoraBaseURL string
// ProductionWeKnoraResolver reads the endpoint an admin saved in the web
// settings. That value is what the console probes — the web page is the
// place operators configure it — and the env value above stays the
// bootstrap fallback for containers that were wired before the UI existed.
ProductionWeKnoraResolver func(ctx context.Context) string
// WeKnoraReadiness optionally exercises a real read against a routing
// knowledge base (nil = API key not configured; liveness-only).
WeKnoraReadiness func(ctx context.Context) error
// WorkerStaleAfter bounds how long a missing worker heartbeat still counts
// as alive (P10).
WorkerStaleAfter time.Duration
}
type RealProvider struct {
db Database
now func() time.Time
httpClient *http.Client
weknoraBaseURL string
minerUBaseURL string
testWeKnoraResolver func(ctx context.Context) string
productionWeKnoraBaseURL string
productionWeKnoraResolver func(ctx context.Context) string
weknoraReadiness func(ctx context.Context) error
workerStaleAfter time.Duration
health *healthTracker
}
// resolveTestWeKnoraBaseURL prefers the admin-saved web setting and falls back to
// the env bootstrap value, so the page is the place to change the test endpoint.
func (provider *RealProvider) resolveTestWeKnoraBaseURL(ctx context.Context) string {
if provider.testWeKnoraResolver != nil {
if value := strings.TrimSpace(provider.testWeKnoraResolver(ctx)); value != "" {
return value
}
}
return provider.weknoraBaseURL
}
// resolveProductionWeKnoraBaseURL prefers the admin-saved web setting and falls
// back to the env bootstrap value: the page wins, the container env only fills in
// when nobody configured it in the UI.
func (provider *RealProvider) resolveProductionWeKnoraBaseURL(ctx context.Context) string {
if provider.productionWeKnoraResolver != nil {
if value := strings.TrimSpace(provider.productionWeKnoraResolver(ctx)); value != "" {
return value
}
}
return provider.productionWeKnoraBaseURL
}
func NewRealProvider(config RealProviderConfig) (*RealProvider, error) {
if config.DB == nil {
return nil, errors.New("monitor database is required")
}
if config.Now == nil {
config.Now = time.Now
}
if config.HTTPClient == nil {
config.HTTPClient = &http.Client{Timeout: 3 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}
}
stale := config.WorkerStaleAfter
if stale <= 0 {
stale = time.Minute
}
return &RealProvider{db: config.DB, now: config.Now, httpClient: config.HTTPClient, weknoraBaseURL: strings.TrimSpace(config.WeKnoraBaseURL), minerUBaseURL: strings.TrimSpace(config.MinerUBaseURL), testWeKnoraResolver: config.TestWeKnoraResolver, productionWeKnoraBaseURL: strings.TrimSpace(config.ProductionWeKnoraBaseURL), productionWeKnoraResolver: config.ProductionWeKnoraResolver, weknoraReadiness: config.WeKnoraReadiness, workerStaleAfter: stale, health: newHealthTracker()}, nil
}
func (provider *RealProvider) Overview(ctx context.Context) (governance.MonitorOverview, error) {
now := provider.now().UTC()
overview := governance.MonitorOverview{Mode: "real", Simulated: false, ObservedAt: now, Services: []governance.MonitorService{}, KnowledgeBases: []governance.KnowledgeBaseProgress{}}
overview.Host = collectHostMetrics()
overview.Services = append(overview.Services, provider.postgresHealth(ctx, now), provider.queueHealth(ctx, now))
overview.Health = provider.Services(ctx)
overview.Services = append(overview.Services,
provider.httpHealth(ctx, "weknora", "WeKnora API", provider.resolveTestWeKnoraBaseURL(ctx), now),
provider.httpHealth(ctx, "weknora-production", "生产 WeKnora", provider.productionWeKnoraBaseURL, now),
governance.MonitorService{Key: "embedding", Name: "Embedding模型", State: "unknown", Detail: "由WeKnora管理,尚无独立探测端点", ObservedAt: now},
governance.MonitorService{Key: "llm", Name: "LLM模型", State: "unknown", Detail: "由WeKnora管理,尚无独立探测端点", ObservedAt: now},
provider.httpHealth(ctx, "mineru", "MinerU", provider.minerUBaseURL, now),
governance.MonitorService{Key: "redis", Name: "Redis / MQ", State: "unknown", Detail: "未部署;任务由PostgreSQL持久队列承载", ObservedAt: now},
governance.MonitorService{Key: "oss", Name: "OSS", State: "unknown", Detail: "未配置独立健康探测", ObservedAt: now},
)
rows, err := provider.db.Query(ctx, `SELECT p.test_kb_id,
CASE WHEN COUNT(DISTINCT p.target_scope)=1 THEN MIN(p.target_scope) ELSE 'mixed' END,
COUNT(*) FILTER (WHERE p.superseded_at IS NULL)::integer,
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.processing_state='completed')::integer,
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.processing_state='pending')::integer,
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.processing_state='processing')::integer,
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.processing_state='finalizing')::integer,
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.processing_state IN ('failed','stalled'))::integer,
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.target_scope='internal')::integer,
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.target_scope='internal' AND p.processing_state='completed')::integer,
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.target_scope='external')::integer,
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.target_scope='external' AND p.processing_state='completed')::integer,
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.processing_state='planned')::integer,
-- 需下架:在测试库里仍然存活(已完成、有知识 id、尚未标记移除)、但来源项已被判为禁止导入的条目。
-- 曾经这里判 p.retrieval_state='enabled',而 retrieval_state 在这套状态机里描述的是
-- “是否发布到生产”(unpublished/disabling/disabled/failed),全仓没有任何代码写过 'enabled',
-- 所以这条车道恒为 0 —— 2026-10-10 界面验收实测:把已入库的文件改成“禁止导入”后
-- 车道仍显示 0,界面承诺的“仍在知识库里的文件会出现在这里”不成立。
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.knowledge_id<>''
AND p.processing_state='completed'
AND p.retrieval_state NOT IN ('disabling','disabled')
AND item.policy_state='blocked')::integer,
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.processing_state='planned'
AND item.policy_state='blocked')::integer
FROM test_publications p
JOIN source_item_versions version ON version.id=p.source_item_version_id
JOIN source_items item ON item.id=version.source_item_id
GROUP BY p.test_kb_id
ORDER BY p.test_kb_id`)
if err != nil {
return governance.MonitorOverview{}, fmt.Errorf("list monitored knowledge bases: %w", err)
}
defer rows.Close()
for rows.Next() {
var item governance.KnowledgeBaseProgress
if err := rows.Scan(&item.ID, &item.Visibility, &item.Imported, &item.Completed, &item.Pending, &item.Processing, &item.Finalizing, &item.Failed, &item.InternalImported, &item.InternalCompleted, &item.ExternalImported, &item.ExternalCompleted,
&item.Planned, &item.NeedsRemoval, &item.StalePlanned); err != nil {
return governance.MonitorOverview{}, fmt.Errorf("scan monitored knowledge base: %w", err)
}
switch item.Visibility {
case "internal":
item.Name = "对内知识库"
case "external":
item.Name = "对外知识库"
default:
item.Name = "测试知识库"
}
overview.KnowledgeBases = append(overview.KnowledgeBases, item)
}
if err := rows.Err(); err != nil {
return governance.MonitorOverview{}, fmt.Errorf("iterate monitored knowledge bases: %w", err)
}
if err := provider.db.QueryRow(ctx, `SELECT
COUNT(*) FILTER (WHERE fetch_state IN ('pending','fetching'))::integer,
COUNT(*) FILTER (WHERE fetch_state='failed')::integer,
COUNT(*) FILTER (WHERE fetch_state='missing_local')::integer,
COALESCE(MAX((SELECT ingestion_concurrency FROM ui_setting_revisions WHERE state='active' LIMIT 1)),2)::integer
FROM catalog_import_intents`).Scan(&overview.Pipeline.FetchPending, &overview.Pipeline.FetchFailed, &overview.Pipeline.FetchMissing, &overview.Pipeline.Concurrency); err != nil {
return governance.MonitorOverview{}, fmt.Errorf("summarize ingestion pipeline: %w", err)
}
return overview, nil
}
func (provider *RealProvider) ListFiles(ctx context.Context, knowledgeBaseID, status string, limit, offset int) (governance.KnowledgeMonitorFilePage, error) {
if strings.TrimSpace(knowledgeBaseID) == "" || limit < 1 || limit > 100 || offset < 0 || !validMonitorStatus(status) {
return governance.KnowledgeMonitorFilePage{}, errors.New("invalid monitor file query")
}
page := governance.KnowledgeMonitorFilePage{Items: []governance.KnowledgeMonitorFile{}, Limit: limit, Offset: offset, SampleOnly: false}
if err := provider.db.QueryRow(ctx, `SELECT COUNT(*)::integer FROM test_publications WHERE test_kb_id=$1 AND ($2='' OR CASE WHEN processing_state='stalled' THEN 'failed' ELSE processing_state END=$2)`, knowledgeBaseID, status).Scan(&page.Total); err != nil {
return governance.KnowledgeMonitorFilePage{}, fmt.Errorf("count monitored files: %w", err)
}
rows, err := provider.db.Query(ctx, `SELECT item.title,item.normalized_path,publication.target_scope,
CASE WHEN publication.processing_state='stalled' THEN 'failed' ELSE publication.processing_state END,
GREATEST(0,EXTRACT(EPOCH FROM ($5-COALESCE(publication.processing_started_at,publication.updated_at)))::integer),
publication.ingest_state,publication.last_error
FROM test_publications publication
JOIN source_item_versions version ON version.id=publication.source_item_version_id
JOIN source_items item ON item.id=version.source_item_id
WHERE publication.test_kb_id=$1
AND ($2='' OR CASE WHEN publication.processing_state='stalled' THEN 'failed' ELSE publication.processing_state END=$2)
ORDER BY CASE publication.processing_state WHEN 'failed' THEN 0 WHEN 'stalled' THEN 0 WHEN 'processing' THEN 1 WHEN 'finalizing' THEN 2 WHEN 'pending' THEN 3 ELSE 4 END,
publication.updated_at DESC,publication.id
LIMIT $3 OFFSET $4`, knowledgeBaseID, status, limit, offset, provider.now().UTC())
if err != nil {
return governance.KnowledgeMonitorFilePage{}, fmt.Errorf("list monitored files: %w", err)
}
defer rows.Close()
for rows.Next() {
var item governance.KnowledgeMonitorFile
var ingestState string
if err := rows.Scan(&item.Name, &item.Directory, &item.Visibility, &item.Status, &item.ElapsedSecond, &ingestState, &item.ErrorCode); err != nil {
return governance.KnowledgeMonitorFilePage{}, fmt.Errorf("scan monitored file: %w", err)
}
item.StageDetail = monitorStageDetail(ingestState, item.Status, item.ErrorCode)
page.Items = append(page.Items, item)
}
if err := rows.Err(); err != nil {
return governance.KnowledgeMonitorFilePage{}, fmt.Errorf("iterate monitored files: %w", err)
}
return page, nil
}
func (provider *RealProvider) postgresHealth(ctx context.Context, now time.Time) governance.MonitorService {
started := time.Now()
state, detail := "healthy", "可连接并可查询"
if err := provider.db.Ping(ctx); err != nil {
state, detail = "unhealthy", "连接或查询失败"
}
return governance.MonitorService{Key: "postgres", Name: "PostgreSQL", State: state, Detail: detail, LatencyMS: int(time.Since(started).Milliseconds()), ObservedAt: now}
}
func (provider *RealProvider) queueHealth(ctx context.Context, now time.Time) governance.MonitorService {
started := time.Now()
var pending, running, failed int
err := provider.db.QueryRow(ctx, `SELECT COUNT(*) FILTER (WHERE state IN ('pending','retry'))::integer,COUNT(*) FILTER (WHERE state='running')::integer,COUNT(*) FILTER (WHERE state='failed')::integer FROM jobs`).Scan(&pending, &running, &failed)
state, detail := "healthy", fmt.Sprintf("可接单;等待%d,运行%d,失败%d", pending, running, failed)
if err != nil {
state, detail = "unhealthy", "队列不可查询"
}
return governance.MonitorService{Key: "queue", Name: "任务队列", State: state, Detail: detail, LatencyMS: int(time.Since(started).Milliseconds()), ObservedAt: now}
}
func (provider *RealProvider) httpHealth(ctx context.Context, key, name, baseURL string, now time.Time) governance.MonitorService {
if baseURL == "" {
return governance.MonitorService{Key: key, Name: name, State: "unknown", Detail: "未配置探测地址", ObservedAt: now}
}
parsed, err := url.Parse(baseURL)
if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Host == "" {
return governance.MonitorService{Key: key, Name: name, State: "unhealthy", Detail: "探测配置无效", ObservedAt: now}
}
parsed.Path = strings.TrimSuffix(parsed.Path, "/") + "/health"
parsed.RawQuery, parsed.Fragment = "", ""
request, err := http.NewRequestWithContext(ctx, http.MethodGet, parsed.String(), nil)
if err != nil {
return governance.MonitorService{Key: key, Name: name, State: "unhealthy", Detail: "无法创建健康探测", ObservedAt: now}
}
started := time.Now()
response, err := provider.httpClient.Do(request)
latency := int(time.Since(started).Milliseconds())
if err != nil {
return governance.MonitorService{Key: key, Name: name, State: "unhealthy", Detail: "健康端点不可达", LatencyMS: latency, ObservedAt: now}
}
defer response.Body.Close()
_, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 4096))
if response.StatusCode < 200 || response.StatusCode >= 300 {
return governance.MonitorService{Key: key, Name: name, State: "degraded", Detail: fmt.Sprintf("健康端点返回HTTP %d", response.StatusCode), LatencyMS: latency, ObservedAt: now}
}
return governance.MonitorService{Key: key, Name: name, State: "healthy", Detail: "健康端点可用", LatencyMS: latency, ObservedAt: now}
}
func collectHostMetrics() governance.MonitorHost {
host := governance.MonitorHost{GPUState: "unknown", GPUDetail: "容器未暴露GPU遥测", Simulated: false}
first, err := os.ReadFile("/proc/stat")
if err == nil {
idle1, total1, parseErr := parseCPUStat(string(first))
if parseErr == nil {
time.Sleep(100 * time.Millisecond)
if second, readErr := os.ReadFile("/proc/stat"); readErr == nil {
idle2, total2, secondErr := parseCPUStat(string(second))
if secondErr == nil {
host.CPUPercent = cpuPercent(idle1, total1, idle2, total2)
}
}
}
}
if raw, err := os.ReadFile("/proc/meminfo"); err == nil {
host.MemoryUsedBytes, host.MemoryTotalBytes, _ = parseMemoryInfo(string(raw))
}
var stats syscall.Statfs_t
if syscall.Statfs("/", &stats) == nil {
host.DiskTotalBytes = int64(stats.Blocks) * int64(stats.Bsize)
host.DiskUsedBytes = int64(stats.Blocks-stats.Bavail) * int64(stats.Bsize)
}
if entries, err := os.ReadDir("/proc/driver/nvidia/gpus"); err == nil && len(entries) > 0 {
host.GPUState, host.GPUDetail = "healthy", fmt.Sprintf("检测到%d块NVIDIA GPU;未读取利用率", len(entries))
}
return host
}
func parseCPUStat(raw string) (idle, total uint64, err error) {
line := strings.SplitN(raw, "\n", 2)[0]
fields := strings.Fields(line)
if len(fields) < 5 || fields[0] != "cpu" {
return 0, 0, errors.New("invalid cpu stat")
}
values := make([]uint64, 0, len(fields)-1)
for _, field := range fields[1:] {
value, parseErr := strconv.ParseUint(field, 10, 64)
if parseErr != nil {
return 0, 0, errors.New("invalid cpu counter")
}
values = append(values, value)
total += value
}
idle = values[3]
if len(values) > 4 {
idle += values[4]
}
return idle, total, nil
}
func parseMemoryInfo(raw string) (used, total int64, err error) {
values := map[string]int64{}
for _, line := range strings.Split(raw, "\n") {
fields := strings.Fields(line)
if len(fields) < 2 {
continue
}
key := strings.TrimSuffix(fields[0], ":")
if key != "MemTotal" && key != "MemAvailable" {
continue
}
value, parseErr := strconv.ParseInt(fields[1], 10, 64)
if parseErr != nil || value < 0 {
return 0, 0, errors.New("invalid memory counter")
}
values[key] = value * 1024
}
total, totalOK := values["MemTotal"]
available, availableOK := values["MemAvailable"]
if !totalOK || !availableOK || available > total {
return 0, 0, errors.New("incomplete memory info")
}
return total - available, total, nil
}
func cpuPercent(idle1, total1, idle2, total2 uint64) float64 {
if total2 <= total1 || idle2 < idle1 || idle2-idle1 > total2-total1 {
return 0
}
return float64((total2-total1)-(idle2-idle1)) * 100 / float64(total2-total1)
}
func validMonitorStatus(status string) bool {
switch status {
case "", "completed", "pending", "processing", "finalizing", "failed", "cancelled":
return true
default:
return false
}
}
func monitorStageDetail(ingestState, processingState, errorCode string) string {
if errorCode != "" {
return "处理失败,可查看错误码后重试"
}
switch {
case ingestState == "pending":
return "等待确认或上传"
case ingestState == "uploading":
return "正在上传测试知识库"
case processingState == "processing":
return "WeKnora正在解析、切片或向量化"
case processingState == "finalizing":
return "基础解析完成,索引或增强处理中"
case processingState == "completed":
return "解析完成,等待或已完成验收"
default:
return "等待远端状态更新"
}
}