Files
knowledge-governance/internal/monitoring/real.go
T
狐友 GXXP & Miku ad02e8bb9c
build / build (push) Successful in 1m43s
feat(settings): 页面录入测试/生产 WeKnora 连接与密钥并收敛可达性判定
连接(地址 + 写-only 密钥)从文件引用搬进治理库:AES-GCM 密文 + 独立 0600 主密钥,
页面与接口只回显尾 4 位;测试连接真实列库并区分 地址不可达/密钥无效/无权限/成功,
换地址或换密钥必须测通才能保存(迁移 000083)。

可达性判定收敛为 weknora.HTTPAllowed,客户端与监控探活共用;新增
WEKNORA_ALLOW_PRIVATE_HTTP 只放行 RFC1918/回环/链路本地。生产绑定只保留下拉,
保存后回报冻结/未冻结条数,已冻结目标不被改写。
2026-11-22 15:18:34 +08:00

390 lines
19 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package monitoring
import (
"context"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"os"
"strconv"
"strings"
"syscall"
"time"
"github.com/jackc/pgx/v5"
"knowledge-governance/internal/governance"
)
type Database interface {
Ping(context.Context) error
Query(context.Context, string, ...any) (pgx.Rows, error)
QueryRow(context.Context, string, ...any) pgx.Row
}
type RealProviderConfig struct {
DB Database
Now func() time.Time
HTTPClient *http.Client
WeKnoraBaseURL string
MinerUBaseURL string
// TestWeKnoraResolver reads the test endpoint an admin saved in the web
// settings, so 「服务配置」is the single place the console probes. The env
// value above stays the bootstrap fallback for containers wired before the
// UI existed.
TestWeKnoraResolver func(ctx context.Context) string
// ProductionWeKnoraBaseURL lets the console show the real state of the
// production WeKnora endpoint. Publishing itself keeps reading its target
// from the release runtime config, never from here.
ProductionWeKnoraBaseURL string
// ProductionWeKnoraResolver reads the endpoint an admin saved in the web
// settings. That value is what the console probes — the web page is the
// place operators configure it — and the env value above stays the
// bootstrap fallback for containers that were wired before the UI existed.
ProductionWeKnoraResolver func(ctx context.Context) string
// WeKnoraReadiness optionally exercises a real read against a routing
// knowledge base (nil = API key not configured; liveness-only).
WeKnoraReadiness func(ctx context.Context) error
// WorkerStaleAfter bounds how long a missing worker heartbeat still counts
// as alive (P10).
WorkerStaleAfter time.Duration
// AllowPrivateHTTP relaxes the plain-http rule for same-LAN addresses. The
// console probes use the very same judgement as the WeKnora client, so a
// card can never say 「可达」 while the client refuses the same URL.
AllowPrivateHTTP bool
}
type RealProvider struct {
db Database
now func() time.Time
httpClient *http.Client
weknoraBaseURL string
minerUBaseURL string
testWeKnoraResolver func(ctx context.Context) string
productionWeKnoraBaseURL string
productionWeKnoraResolver func(ctx context.Context) string
weknoraReadiness func(ctx context.Context) error
workerStaleAfter time.Duration
allowPrivateHTTP bool
health *healthTracker
}
// resolveTestWeKnoraBaseURL prefers the admin-saved web setting and falls back to
// the env bootstrap value, so the page is the place to change the test endpoint.
func (provider *RealProvider) resolveTestWeKnoraBaseURL(ctx context.Context) string {
if provider.testWeKnoraResolver != nil {
if value := strings.TrimSpace(provider.testWeKnoraResolver(ctx)); value != "" {
return value
}
}
return provider.weknoraBaseURL
}
// resolveProductionWeKnoraBaseURL prefers the admin-saved web setting and falls
// back to the env bootstrap value: the page wins, the container env only fills in
// when nobody configured it in the UI.
func (provider *RealProvider) resolveProductionWeKnoraBaseURL(ctx context.Context) string {
if provider.productionWeKnoraResolver != nil {
if value := strings.TrimSpace(provider.productionWeKnoraResolver(ctx)); value != "" {
return value
}
}
return provider.productionWeKnoraBaseURL
}
func NewRealProvider(config RealProviderConfig) (*RealProvider, error) {
if config.DB == nil {
return nil, errors.New("monitor database is required")
}
if config.Now == nil {
config.Now = time.Now
}
if config.HTTPClient == nil {
config.HTTPClient = &http.Client{Timeout: 3 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}
}
stale := config.WorkerStaleAfter
if stale <= 0 {
stale = time.Minute
}
return &RealProvider{db: config.DB, now: config.Now, httpClient: config.HTTPClient, weknoraBaseURL: strings.TrimSpace(config.WeKnoraBaseURL), minerUBaseURL: strings.TrimSpace(config.MinerUBaseURL), testWeKnoraResolver: config.TestWeKnoraResolver, productionWeKnoraBaseURL: strings.TrimSpace(config.ProductionWeKnoraBaseURL), productionWeKnoraResolver: config.ProductionWeKnoraResolver, weknoraReadiness: config.WeKnoraReadiness, workerStaleAfter: stale, allowPrivateHTTP: config.AllowPrivateHTTP, health: newHealthTracker()}, nil
}
func (provider *RealProvider) Overview(ctx context.Context) (governance.MonitorOverview, error) {
now := provider.now().UTC()
overview := governance.MonitorOverview{Mode: "real", Simulated: false, ObservedAt: now, Services: []governance.MonitorService{}, KnowledgeBases: []governance.KnowledgeBaseProgress{}}
overview.Host = collectHostMetrics()
overview.Services = append(overview.Services, provider.postgresHealth(ctx, now), provider.queueHealth(ctx, now))
overview.Health = provider.Services(ctx)
overview.Services = append(overview.Services,
provider.httpHealth(ctx, "weknora", "WeKnora API", provider.resolveTestWeKnoraBaseURL(ctx), now),
provider.httpHealth(ctx, "weknora-production", "生产 WeKnora", provider.productionWeKnoraBaseURL, now),
governance.MonitorService{Key: "embedding", Name: "Embedding模型", State: "unknown", Detail: "由WeKnora管理,尚无独立探测端点", ObservedAt: now},
governance.MonitorService{Key: "llm", Name: "LLM模型", State: "unknown", Detail: "由WeKnora管理,尚无独立探测端点", ObservedAt: now},
provider.httpHealth(ctx, "mineru", "MinerU", provider.minerUBaseURL, now),
governance.MonitorService{Key: "redis", Name: "Redis / MQ", State: "unknown", Detail: "未部署;任务由PostgreSQL持久队列承载", ObservedAt: now},
governance.MonitorService{Key: "oss", Name: "OSS", State: "unknown", Detail: "未配置独立健康探测", ObservedAt: now},
)
rows, err := provider.db.Query(ctx, `SELECT p.test_kb_id,
CASE WHEN COUNT(DISTINCT p.target_scope)=1 THEN MIN(p.target_scope) ELSE 'mixed' END,
COUNT(*) FILTER (WHERE p.superseded_at IS NULL)::integer,
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.processing_state='completed')::integer,
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.processing_state='pending')::integer,
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.processing_state='processing')::integer,
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.processing_state='finalizing')::integer,
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.processing_state IN ('failed','stalled'))::integer,
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.target_scope='internal')::integer,
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.target_scope='internal' AND p.processing_state='completed')::integer,
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.target_scope='external')::integer,
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.target_scope='external' AND p.processing_state='completed')::integer,
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.processing_state='planned')::integer,
-- 需下架:在测试库里仍然存活(已完成、有知识 id、尚未标记移除)、但来源项已被判为禁止导入的条目。
-- 曾经这里判 p.retrieval_state='enabled',而 retrieval_state 在这套状态机里描述的是
-- “是否发布到生产”(unpublished/disabling/disabled/failed),全仓没有任何代码写过 'enabled',
-- 所以这条车道恒为 0 —— 2026-10-10 界面验收实测:把已入库的文件改成“禁止导入”后
-- 车道仍显示 0,界面承诺的“仍在知识库里的文件会出现在这里”不成立。
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.knowledge_id<>''
AND p.processing_state='completed'
AND p.retrieval_state NOT IN ('disabling','disabled')
AND item.policy_state='blocked')::integer,
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.processing_state='planned'
AND item.policy_state='blocked')::integer
FROM test_publications p
JOIN source_item_versions version ON version.id=p.source_item_version_id
JOIN source_items item ON item.id=version.source_item_id
GROUP BY p.test_kb_id
ORDER BY p.test_kb_id`)
if err != nil {
return governance.MonitorOverview{}, fmt.Errorf("list monitored knowledge bases: %w", err)
}
defer rows.Close()
for rows.Next() {
var item governance.KnowledgeBaseProgress
if err := rows.Scan(&item.ID, &item.Visibility, &item.Imported, &item.Completed, &item.Pending, &item.Processing, &item.Finalizing, &item.Failed, &item.InternalImported, &item.InternalCompleted, &item.ExternalImported, &item.ExternalCompleted,
&item.Planned, &item.NeedsRemoval, &item.StalePlanned); err != nil {
return governance.MonitorOverview{}, fmt.Errorf("scan monitored knowledge base: %w", err)
}
switch item.Visibility {
case "internal":
item.Name = "对内知识库"
case "external":
item.Name = "对外知识库"
default:
item.Name = "测试知识库"
}
overview.KnowledgeBases = append(overview.KnowledgeBases, item)
}
if err := rows.Err(); err != nil {
return governance.MonitorOverview{}, fmt.Errorf("iterate monitored knowledge bases: %w", err)
}
if err := provider.db.QueryRow(ctx, `SELECT
COUNT(*) FILTER (WHERE fetch_state IN ('pending','fetching'))::integer,
COUNT(*) FILTER (WHERE fetch_state='failed')::integer,
COUNT(*) FILTER (WHERE fetch_state='missing_local')::integer,
COALESCE(MAX((SELECT ingestion_concurrency FROM ui_setting_revisions WHERE state='active' LIMIT 1)),2)::integer
FROM catalog_import_intents`).Scan(&overview.Pipeline.FetchPending, &overview.Pipeline.FetchFailed, &overview.Pipeline.FetchMissing, &overview.Pipeline.Concurrency); err != nil {
return governance.MonitorOverview{}, fmt.Errorf("summarize ingestion pipeline: %w", err)
}
return overview, nil
}
func (provider *RealProvider) ListFiles(ctx context.Context, knowledgeBaseID, status string, limit, offset int) (governance.KnowledgeMonitorFilePage, error) {
if strings.TrimSpace(knowledgeBaseID) == "" || limit < 1 || limit > 100 || offset < 0 || !validMonitorStatus(status) {
return governance.KnowledgeMonitorFilePage{}, errors.New("invalid monitor file query")
}
page := governance.KnowledgeMonitorFilePage{Items: []governance.KnowledgeMonitorFile{}, Limit: limit, Offset: offset, SampleOnly: false}
if err := provider.db.QueryRow(ctx, `SELECT COUNT(*)::integer FROM test_publications WHERE test_kb_id=$1 AND ($2='' OR CASE WHEN processing_state='stalled' THEN 'failed' ELSE processing_state END=$2)`, knowledgeBaseID, status).Scan(&page.Total); err != nil {
return governance.KnowledgeMonitorFilePage{}, fmt.Errorf("count monitored files: %w", err)
}
rows, err := provider.db.Query(ctx, `SELECT item.title,item.normalized_path,publication.target_scope,
CASE WHEN publication.processing_state='stalled' THEN 'failed' ELSE publication.processing_state END,
GREATEST(0,EXTRACT(EPOCH FROM ($5-COALESCE(publication.processing_started_at,publication.updated_at)))::integer),
publication.ingest_state,publication.last_error
FROM test_publications publication
JOIN source_item_versions version ON version.id=publication.source_item_version_id
JOIN source_items item ON item.id=version.source_item_id
WHERE publication.test_kb_id=$1
AND ($2='' OR CASE WHEN publication.processing_state='stalled' THEN 'failed' ELSE publication.processing_state END=$2)
ORDER BY CASE publication.processing_state WHEN 'failed' THEN 0 WHEN 'stalled' THEN 0 WHEN 'processing' THEN 1 WHEN 'finalizing' THEN 2 WHEN 'pending' THEN 3 ELSE 4 END,
publication.updated_at DESC,publication.id
LIMIT $3 OFFSET $4`, knowledgeBaseID, status, limit, offset, provider.now().UTC())
if err != nil {
return governance.KnowledgeMonitorFilePage{}, fmt.Errorf("list monitored files: %w", err)
}
defer rows.Close()
for rows.Next() {
var item governance.KnowledgeMonitorFile
var ingestState string
if err := rows.Scan(&item.Name, &item.Directory, &item.Visibility, &item.Status, &item.ElapsedSecond, &ingestState, &item.ErrorCode); err != nil {
return governance.KnowledgeMonitorFilePage{}, fmt.Errorf("scan monitored file: %w", err)
}
item.StageDetail = monitorStageDetail(ingestState, item.Status, item.ErrorCode)
page.Items = append(page.Items, item)
}
if err := rows.Err(); err != nil {
return governance.KnowledgeMonitorFilePage{}, fmt.Errorf("iterate monitored files: %w", err)
}
return page, nil
}
func (provider *RealProvider) postgresHealth(ctx context.Context, now time.Time) governance.MonitorService {
started := time.Now()
state, detail := "healthy", "可连接并可查询"
if err := provider.db.Ping(ctx); err != nil {
state, detail = "unhealthy", "连接或查询失败"
}
return governance.MonitorService{Key: "postgres", Name: "PostgreSQL", State: state, Detail: detail, LatencyMS: int(time.Since(started).Milliseconds()), ObservedAt: now}
}
func (provider *RealProvider) queueHealth(ctx context.Context, now time.Time) governance.MonitorService {
started := time.Now()
var pending, running, failed int
err := provider.db.QueryRow(ctx, `SELECT COUNT(*) FILTER (WHERE state IN ('pending','retry'))::integer,COUNT(*) FILTER (WHERE state='running')::integer,COUNT(*) FILTER (WHERE state='failed')::integer FROM jobs`).Scan(&pending, &running, &failed)
state, detail := "healthy", fmt.Sprintf("可接单;等待%d,运行%d,失败%d", pending, running, failed)
if err != nil {
state, detail = "unhealthy", "队列不可查询"
}
return governance.MonitorService{Key: "queue", Name: "任务队列", State: state, Detail: detail, LatencyMS: int(time.Since(started).Milliseconds()), ObservedAt: now}
}
func (provider *RealProvider) httpHealth(ctx context.Context, key, name, baseURL string, now time.Time) governance.MonitorService {
if baseURL == "" {
return governance.MonitorService{Key: key, Name: name, State: "unknown", Detail: "未配置探测地址", ObservedAt: now}
}
parsed, err := url.Parse(baseURL)
if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Host == "" {
return governance.MonitorService{Key: key, Name: name, State: "unhealthy", Detail: "探测配置无效", ObservedAt: now}
}
parsed.Path = strings.TrimSuffix(parsed.Path, "/") + "/health"
parsed.RawQuery, parsed.Fragment = "", ""
request, err := http.NewRequestWithContext(ctx, http.MethodGet, parsed.String(), nil)
if err != nil {
return governance.MonitorService{Key: key, Name: name, State: "unhealthy", Detail: "无法创建健康探测", ObservedAt: now}
}
started := time.Now()
response, err := provider.httpClient.Do(request)
latency := int(time.Since(started).Milliseconds())
if err != nil {
return governance.MonitorService{Key: key, Name: name, State: "unhealthy", Detail: "健康端点不可达", LatencyMS: latency, ObservedAt: now}
}
defer response.Body.Close()
_, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 4096))
if response.StatusCode < 200 || response.StatusCode >= 300 {
return governance.MonitorService{Key: key, Name: name, State: "degraded", Detail: fmt.Sprintf("健康端点返回HTTP %d", response.StatusCode), LatencyMS: latency, ObservedAt: now}
}
return governance.MonitorService{Key: key, Name: name, State: "healthy", Detail: "健康端点可用", LatencyMS: latency, ObservedAt: now}
}
func collectHostMetrics() governance.MonitorHost {
host := governance.MonitorHost{GPUState: "unknown", GPUDetail: "容器未暴露GPU遥测", Simulated: false}
first, err := os.ReadFile("/proc/stat")
if err == nil {
idle1, total1, parseErr := parseCPUStat(string(first))
if parseErr == nil {
time.Sleep(100 * time.Millisecond)
if second, readErr := os.ReadFile("/proc/stat"); readErr == nil {
idle2, total2, secondErr := parseCPUStat(string(second))
if secondErr == nil {
host.CPUPercent = cpuPercent(idle1, total1, idle2, total2)
}
}
}
}
if raw, err := os.ReadFile("/proc/meminfo"); err == nil {
host.MemoryUsedBytes, host.MemoryTotalBytes, _ = parseMemoryInfo(string(raw))
}
var stats syscall.Statfs_t
if syscall.Statfs("/", &stats) == nil {
host.DiskTotalBytes = int64(stats.Blocks) * int64(stats.Bsize)
host.DiskUsedBytes = int64(stats.Blocks-stats.Bavail) * int64(stats.Bsize)
}
if entries, err := os.ReadDir("/proc/driver/nvidia/gpus"); err == nil && len(entries) > 0 {
host.GPUState, host.GPUDetail = "healthy", fmt.Sprintf("检测到%d块NVIDIA GPU;未读取利用率", len(entries))
}
return host
}
func parseCPUStat(raw string) (idle, total uint64, err error) {
line := strings.SplitN(raw, "\n", 2)[0]
fields := strings.Fields(line)
if len(fields) < 5 || fields[0] != "cpu" {
return 0, 0, errors.New("invalid cpu stat")
}
values := make([]uint64, 0, len(fields)-1)
for _, field := range fields[1:] {
value, parseErr := strconv.ParseUint(field, 10, 64)
if parseErr != nil {
return 0, 0, errors.New("invalid cpu counter")
}
values = append(values, value)
total += value
}
idle = values[3]
if len(values) > 4 {
idle += values[4]
}
return idle, total, nil
}
func parseMemoryInfo(raw string) (used, total int64, err error) {
values := map[string]int64{}
for _, line := range strings.Split(raw, "\n") {
fields := strings.Fields(line)
if len(fields) < 2 {
continue
}
key := strings.TrimSuffix(fields[0], ":")
if key != "MemTotal" && key != "MemAvailable" {
continue
}
value, parseErr := strconv.ParseInt(fields[1], 10, 64)
if parseErr != nil || value < 0 {
return 0, 0, errors.New("invalid memory counter")
}
values[key] = value * 1024
}
total, totalOK := values["MemTotal"]
available, availableOK := values["MemAvailable"]
if !totalOK || !availableOK || available > total {
return 0, 0, errors.New("incomplete memory info")
}
return total - available, total, nil
}
func cpuPercent(idle1, total1, idle2, total2 uint64) float64 {
if total2 <= total1 || idle2 < idle1 || idle2-idle1 > total2-total1 {
return 0
}
return float64((total2-total1)-(idle2-idle1)) * 100 / float64(total2-total1)
}
func validMonitorStatus(status string) bool {
switch status {
case "", "completed", "pending", "processing", "finalizing", "failed", "cancelled":
return true
default:
return false
}
}
func monitorStageDetail(ingestState, processingState, errorCode string) string {
if errorCode != "" {
return "处理失败,可查看错误码后重试"
}
switch {
case ingestState == "pending":
return "等待确认或上传"
case ingestState == "uploading":
return "正在上传测试知识库"
case processingState == "processing":
return "WeKnora正在解析、切片或向量化"
case processingState == "finalizing":
return "基础解析完成,索引或增强处理中"
case processingState == "completed":
return "解析完成,等待或已完成验收"
default:
return "等待远端状态更新"
}
}