Files
knowledge-governance/internal/monitoring/real.go
T
姚章浩 1c80b07f2c
build / build (push) Successful in 1m59s
fix(ui): 修复F12-F16入口与追溯缺陷
Dockerfile 运行阶段补齐 dws_dingtalk_scan(F12),并新增 runtime-scripts 门禁
做脚本依赖闭包正负例;就绪探测每次解析当前端点/绑定/凭据并区分 404、未配置、
不可达(F13);路由写入来源路径与父目录,历史空值仅按同连接同 item 的意图读时
补齐(F14);目标筛选显示短 ID 与当前绑定/历史并提供快捷筛选(F15);刷新终态
移除后台运行、展示失败时间/旧快照/脱敏原因/作业 ID 与重试(F16)。

验证:make verify 17 项全过(identity 本地按设计 skipped);一次性 schema 的
Monitor 目录/路由集成用例通过;8088 部署与真实 UI 验收仍由父会话执行。
2026-11-05 10:38:26 +08:00

385 lines
18 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package monitoring
import (
"context"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"os"
"strconv"
"strings"
"syscall"
"time"
"github.com/jackc/pgx/v5"
"knowledge-governance/internal/governance"
)
type Database interface {
Ping(context.Context) error
Query(context.Context, string, ...any) (pgx.Rows, error)
QueryRow(context.Context, string, ...any) pgx.Row
}
type RealProviderConfig struct {
DB Database
Now func() time.Time
HTTPClient *http.Client
WeKnoraBaseURL string
MinerUBaseURL string
// TestWeKnoraResolver reads the test endpoint an admin saved in the web
// settings, so 「服务配置」is the single place the console probes. The env
// value above stays the bootstrap fallback for containers wired before the
// UI existed.
TestWeKnoraResolver func(ctx context.Context) string
// ProductionWeKnoraBaseURL lets the console show the real state of the
// production WeKnora endpoint. Publishing itself keeps reading its target
// from the release runtime config, never from here.
ProductionWeKnoraBaseURL string
// ProductionWeKnoraResolver reads the endpoint an admin saved in the web
// settings. That value is what the console probes — the web page is the
// place operators configure it — and the env value above stays the
// bootstrap fallback for containers that were wired before the UI existed.
ProductionWeKnoraResolver func(ctx context.Context) string
// WeKnoraReadiness optionally exercises a real read against the CURRENT
// routing binding and reports the sanitized probe target alongside the error.
WeKnoraReadiness func(ctx context.Context) (target string, err error)
// WorkerStaleAfter bounds how long a missing worker heartbeat still counts
// as alive (P10).
WorkerStaleAfter time.Duration
}
type RealProvider struct {
db Database
now func() time.Time
httpClient *http.Client
weknoraBaseURL string
minerUBaseURL string
testWeKnoraResolver func(ctx context.Context) string
productionWeKnoraBaseURL string
productionWeKnoraResolver func(ctx context.Context) string
weknoraReadiness func(ctx context.Context) (string, error)
workerStaleAfter time.Duration
health *healthTracker
}
// resolveTestWeKnoraBaseURL prefers the admin-saved web setting and falls back to
// the env bootstrap value, so the page is the place to change the test endpoint.
func (provider *RealProvider) resolveTestWeKnoraBaseURL(ctx context.Context) string {
if provider.testWeKnoraResolver != nil {
if value := strings.TrimSpace(provider.testWeKnoraResolver(ctx)); value != "" {
return value
}
}
return provider.weknoraBaseURL
}
// resolveProductionWeKnoraBaseURL prefers the admin-saved web setting and falls
// back to the env bootstrap value: the page wins, the container env only fills in
// when nobody configured it in the UI.
func (provider *RealProvider) resolveProductionWeKnoraBaseURL(ctx context.Context) string {
if provider.productionWeKnoraResolver != nil {
if value := strings.TrimSpace(provider.productionWeKnoraResolver(ctx)); value != "" {
return value
}
}
return provider.productionWeKnoraBaseURL
}
func NewRealProvider(config RealProviderConfig) (*RealProvider, error) {
if config.DB == nil {
return nil, errors.New("monitor database is required")
}
if config.Now == nil {
config.Now = time.Now
}
if config.HTTPClient == nil {
config.HTTPClient = &http.Client{Timeout: 3 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}
}
stale := config.WorkerStaleAfter
if stale <= 0 {
stale = time.Minute
}
return &RealProvider{db: config.DB, now: config.Now, httpClient: config.HTTPClient, weknoraBaseURL: strings.TrimSpace(config.WeKnoraBaseURL), minerUBaseURL: strings.TrimSpace(config.MinerUBaseURL), testWeKnoraResolver: config.TestWeKnoraResolver, productionWeKnoraBaseURL: strings.TrimSpace(config.ProductionWeKnoraBaseURL), productionWeKnoraResolver: config.ProductionWeKnoraResolver, weknoraReadiness: config.WeKnoraReadiness, workerStaleAfter: stale, health: newHealthTracker()}, nil
}
func (provider *RealProvider) Overview(ctx context.Context) (governance.MonitorOverview, error) {
now := provider.now().UTC()
overview := governance.MonitorOverview{Mode: "real", Simulated: false, ObservedAt: now, Services: []governance.MonitorService{}, KnowledgeBases: []governance.KnowledgeBaseProgress{}}
overview.Host = collectHostMetrics()
overview.Services = append(overview.Services, provider.postgresHealth(ctx, now), provider.queueHealth(ctx, now))
overview.Health = provider.Services(ctx)
overview.Services = append(overview.Services,
provider.httpHealth(ctx, "weknora", "WeKnora API", provider.resolveTestWeKnoraBaseURL(ctx), now),
provider.httpHealth(ctx, "weknora-production", "生产 WeKnora", provider.productionWeKnoraBaseURL, now),
governance.MonitorService{Key: "embedding", Name: "Embedding模型", State: "unknown", Detail: "由WeKnora管理,尚无独立探测端点", ObservedAt: now},
governance.MonitorService{Key: "llm", Name: "LLM模型", State: "unknown", Detail: "由WeKnora管理,尚无独立探测端点", ObservedAt: now},
provider.httpHealth(ctx, "mineru", "MinerU", provider.minerUBaseURL, now),
governance.MonitorService{Key: "redis", Name: "Redis / MQ", State: "unknown", Detail: "未部署;任务由PostgreSQL持久队列承载", ObservedAt: now},
governance.MonitorService{Key: "oss", Name: "OSS", State: "unknown", Detail: "未配置独立健康探测", ObservedAt: now},
)
rows, err := provider.db.Query(ctx, `SELECT p.test_kb_id,
CASE WHEN COUNT(DISTINCT p.target_scope)=1 THEN MIN(p.target_scope) ELSE 'mixed' END,
COUNT(*) FILTER (WHERE p.superseded_at IS NULL)::integer,
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.processing_state='completed')::integer,
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.processing_state='pending')::integer,
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.processing_state='processing')::integer,
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.processing_state='finalizing')::integer,
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.processing_state IN ('failed','stalled'))::integer,
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.target_scope='internal')::integer,
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.target_scope='internal' AND p.processing_state='completed')::integer,
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.target_scope='external')::integer,
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.target_scope='external' AND p.processing_state='completed')::integer,
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.processing_state='planned')::integer,
-- 需下架:在测试库里仍然存活(已完成、有知识 id、尚未标记移除)、但来源项已被判为禁止导入的条目。
-- 曾经这里判 p.retrieval_state='enabled',而 retrieval_state 在这套状态机里描述的是
-- “是否发布到生产”(unpublished/disabling/disabled/failed),全仓没有任何代码写过 'enabled',
-- 所以这条车道恒为 0 —— 2026-10-10 界面验收实测:把已入库的文件改成“禁止导入”后
-- 车道仍显示 0,界面承诺的“仍在知识库里的文件会出现在这里”不成立。
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.knowledge_id<>''
AND p.processing_state='completed'
AND p.retrieval_state NOT IN ('disabling','disabled')
AND item.policy_state='blocked')::integer,
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.processing_state='planned'
AND item.policy_state='blocked')::integer
FROM test_publications p
JOIN source_item_versions version ON version.id=p.source_item_version_id
JOIN source_items item ON item.id=version.source_item_id
GROUP BY p.test_kb_id
ORDER BY p.test_kb_id`)
if err != nil {
return governance.MonitorOverview{}, fmt.Errorf("list monitored knowledge bases: %w", err)
}
defer rows.Close()
for rows.Next() {
var item governance.KnowledgeBaseProgress
if err := rows.Scan(&item.ID, &item.Visibility, &item.Imported, &item.Completed, &item.Pending, &item.Processing, &item.Finalizing, &item.Failed, &item.InternalImported, &item.InternalCompleted, &item.ExternalImported, &item.ExternalCompleted,
&item.Planned, &item.NeedsRemoval, &item.StalePlanned); err != nil {
return governance.MonitorOverview{}, fmt.Errorf("scan monitored knowledge base: %w", err)
}
switch item.Visibility {
case "internal":
item.Name = "对内知识库"
case "external":
item.Name = "对外知识库"
default:
item.Name = "测试知识库"
}
overview.KnowledgeBases = append(overview.KnowledgeBases, item)
}
if err := rows.Err(); err != nil {
return governance.MonitorOverview{}, fmt.Errorf("iterate monitored knowledge bases: %w", err)
}
if err := provider.db.QueryRow(ctx, `SELECT
COUNT(*) FILTER (WHERE fetch_state IN ('pending','fetching'))::integer,
COUNT(*) FILTER (WHERE fetch_state='failed')::integer,
COUNT(*) FILTER (WHERE fetch_state='missing_local')::integer,
COALESCE(MAX((SELECT ingestion_concurrency FROM ui_setting_revisions WHERE state='active' LIMIT 1)),2)::integer
FROM catalog_import_intents`).Scan(&overview.Pipeline.FetchPending, &overview.Pipeline.FetchFailed, &overview.Pipeline.FetchMissing, &overview.Pipeline.Concurrency); err != nil {
return governance.MonitorOverview{}, fmt.Errorf("summarize ingestion pipeline: %w", err)
}
return overview, nil
}
func (provider *RealProvider) ListFiles(ctx context.Context, knowledgeBaseID, status string, limit, offset int) (governance.KnowledgeMonitorFilePage, error) {
if strings.TrimSpace(knowledgeBaseID) == "" || limit < 1 || limit > 100 || offset < 0 || !validMonitorStatus(status) {
return governance.KnowledgeMonitorFilePage{}, errors.New("invalid monitor file query")
}
page := governance.KnowledgeMonitorFilePage{Items: []governance.KnowledgeMonitorFile{}, Limit: limit, Offset: offset, SampleOnly: false}
if err := provider.db.QueryRow(ctx, `SELECT COUNT(*)::integer FROM test_publications WHERE test_kb_id=$1 AND ($2='' OR CASE WHEN processing_state='stalled' THEN 'failed' ELSE processing_state END=$2)`, knowledgeBaseID, status).Scan(&page.Total); err != nil {
return governance.KnowledgeMonitorFilePage{}, fmt.Errorf("count monitored files: %w", err)
}
rows, err := provider.db.Query(ctx, `SELECT item.title,item.normalized_path,publication.target_scope,
CASE WHEN publication.processing_state='stalled' THEN 'failed' ELSE publication.processing_state END,
GREATEST(0,EXTRACT(EPOCH FROM ($5-COALESCE(publication.processing_started_at,publication.updated_at)))::integer),
publication.ingest_state,publication.last_error
FROM test_publications publication
JOIN source_item_versions version ON version.id=publication.source_item_version_id
JOIN source_items item ON item.id=version.source_item_id
WHERE publication.test_kb_id=$1
AND ($2='' OR CASE WHEN publication.processing_state='stalled' THEN 'failed' ELSE publication.processing_state END=$2)
ORDER BY CASE publication.processing_state WHEN 'failed' THEN 0 WHEN 'stalled' THEN 0 WHEN 'processing' THEN 1 WHEN 'finalizing' THEN 2 WHEN 'pending' THEN 3 ELSE 4 END,
publication.updated_at DESC,publication.id
LIMIT $3 OFFSET $4`, knowledgeBaseID, status, limit, offset, provider.now().UTC())
if err != nil {
return governance.KnowledgeMonitorFilePage{}, fmt.Errorf("list monitored files: %w", err)
}
defer rows.Close()
for rows.Next() {
var item governance.KnowledgeMonitorFile
var ingestState string
if err := rows.Scan(&item.Name, &item.Directory, &item.Visibility, &item.Status, &item.ElapsedSecond, &ingestState, &item.ErrorCode); err != nil {
return governance.KnowledgeMonitorFilePage{}, fmt.Errorf("scan monitored file: %w", err)
}
item.StageDetail = monitorStageDetail(ingestState, item.Status, item.ErrorCode)
page.Items = append(page.Items, item)
}
if err := rows.Err(); err != nil {
return governance.KnowledgeMonitorFilePage{}, fmt.Errorf("iterate monitored files: %w", err)
}
return page, nil
}
func (provider *RealProvider) postgresHealth(ctx context.Context, now time.Time) governance.MonitorService {
started := time.Now()
state, detail := "healthy", "可连接并可查询"
if err := provider.db.Ping(ctx); err != nil {
state, detail = "unhealthy", "连接或查询失败"
}
return governance.MonitorService{Key: "postgres", Name: "PostgreSQL", State: state, Detail: detail, LatencyMS: int(time.Since(started).Milliseconds()), ObservedAt: now}
}
func (provider *RealProvider) queueHealth(ctx context.Context, now time.Time) governance.MonitorService {
started := time.Now()
var pending, running, failed int
err := provider.db.QueryRow(ctx, `SELECT COUNT(*) FILTER (WHERE state IN ('pending','retry'))::integer,COUNT(*) FILTER (WHERE state='running')::integer,COUNT(*) FILTER (WHERE state='failed')::integer FROM jobs`).Scan(&pending, &running, &failed)
state, detail := "healthy", fmt.Sprintf("可接单;等待%d,运行%d,失败%d", pending, running, failed)
if err != nil {
state, detail = "unhealthy", "队列不可查询"
}
return governance.MonitorService{Key: "queue", Name: "任务队列", State: state, Detail: detail, LatencyMS: int(time.Since(started).Milliseconds()), ObservedAt: now}
}
func (provider *RealProvider) httpHealth(ctx context.Context, key, name, baseURL string, now time.Time) governance.MonitorService {
if baseURL == "" {
return governance.MonitorService{Key: key, Name: name, State: "unknown", Detail: "未配置探测地址", ObservedAt: now}
}
parsed, err := url.Parse(baseURL)
if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Host == "" {
return governance.MonitorService{Key: key, Name: name, State: "unhealthy", Detail: "探测配置无效", ObservedAt: now}
}
parsed.Path = strings.TrimSuffix(parsed.Path, "/") + "/health"
parsed.RawQuery, parsed.Fragment = "", ""
request, err := http.NewRequestWithContext(ctx, http.MethodGet, parsed.String(), nil)
if err != nil {
return governance.MonitorService{Key: key, Name: name, State: "unhealthy", Detail: "无法创建健康探测", ObservedAt: now}
}
started := time.Now()
response, err := provider.httpClient.Do(request)
latency := int(time.Since(started).Milliseconds())
if err != nil {
return governance.MonitorService{Key: key, Name: name, State: "unhealthy", Detail: "健康端点不可达", LatencyMS: latency, ObservedAt: now}
}
defer response.Body.Close()
_, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 4096))
if response.StatusCode < 200 || response.StatusCode >= 300 {
return governance.MonitorService{Key: key, Name: name, State: "degraded", Detail: fmt.Sprintf("健康端点返回HTTP %d", response.StatusCode), LatencyMS: latency, ObservedAt: now}
}
return governance.MonitorService{Key: key, Name: name, State: "healthy", Detail: "健康端点可用", LatencyMS: latency, ObservedAt: now}
}
func collectHostMetrics() governance.MonitorHost {
host := governance.MonitorHost{GPUState: "unknown", GPUDetail: "容器未暴露GPU遥测", Simulated: false}
first, err := os.ReadFile("/proc/stat")
if err == nil {
idle1, total1, parseErr := parseCPUStat(string(first))
if parseErr == nil {
time.Sleep(100 * time.Millisecond)
if second, readErr := os.ReadFile("/proc/stat"); readErr == nil {
idle2, total2, secondErr := parseCPUStat(string(second))
if secondErr == nil {
host.CPUPercent = cpuPercent(idle1, total1, idle2, total2)
}
}
}
}
if raw, err := os.ReadFile("/proc/meminfo"); err == nil {
host.MemoryUsedBytes, host.MemoryTotalBytes, _ = parseMemoryInfo(string(raw))
}
var stats syscall.Statfs_t
if syscall.Statfs("/", &stats) == nil {
host.DiskTotalBytes = int64(stats.Blocks) * int64(stats.Bsize)
host.DiskUsedBytes = int64(stats.Blocks-stats.Bavail) * int64(stats.Bsize)
}
if entries, err := os.ReadDir("/proc/driver/nvidia/gpus"); err == nil && len(entries) > 0 {
host.GPUState, host.GPUDetail = "healthy", fmt.Sprintf("检测到%d块NVIDIA GPU;未读取利用率", len(entries))
}
return host
}
func parseCPUStat(raw string) (idle, total uint64, err error) {
line := strings.SplitN(raw, "\n", 2)[0]
fields := strings.Fields(line)
if len(fields) < 5 || fields[0] != "cpu" {
return 0, 0, errors.New("invalid cpu stat")
}
values := make([]uint64, 0, len(fields)-1)
for _, field := range fields[1:] {
value, parseErr := strconv.ParseUint(field, 10, 64)
if parseErr != nil {
return 0, 0, errors.New("invalid cpu counter")
}
values = append(values, value)
total += value
}
idle = values[3]
if len(values) > 4 {
idle += values[4]
}
return idle, total, nil
}
func parseMemoryInfo(raw string) (used, total int64, err error) {
values := map[string]int64{}
for _, line := range strings.Split(raw, "\n") {
fields := strings.Fields(line)
if len(fields) < 2 {
continue
}
key := strings.TrimSuffix(fields[0], ":")
if key != "MemTotal" && key != "MemAvailable" {
continue
}
value, parseErr := strconv.ParseInt(fields[1], 10, 64)
if parseErr != nil || value < 0 {
return 0, 0, errors.New("invalid memory counter")
}
values[key] = value * 1024
}
total, totalOK := values["MemTotal"]
available, availableOK := values["MemAvailable"]
if !totalOK || !availableOK || available > total {
return 0, 0, errors.New("incomplete memory info")
}
return total - available, total, nil
}
func cpuPercent(idle1, total1, idle2, total2 uint64) float64 {
if total2 <= total1 || idle2 < idle1 || idle2-idle1 > total2-total1 {
return 0
}
return float64((total2-total1)-(idle2-idle1)) * 100 / float64(total2-total1)
}
func validMonitorStatus(status string) bool {
switch status {
case "", "completed", "pending", "processing", "finalizing", "failed", "cancelled":
return true
default:
return false
}
}
func monitorStageDetail(ingestState, processingState, errorCode string) string {
if errorCode != "" {
return "处理失败,可查看错误码后重试"
}
switch {
case ingestState == "pending":
return "等待确认或上传"
case ingestState == "uploading":
return "正在上传测试知识库"
case processingState == "processing":
return "WeKnora正在解析、切片或向量化"
case processingState == "finalizing":
return "基础解析完成,索引或增强处理中"
case processingState == "completed":
return "解析完成,等待或已完成验收"
default:
return "等待远端状态更新"
}
}