Files
knowledge-governance/internal/monitoring/health.go
T
zheromiku dcdc8d1ddd feat: 知识治理平台(含已验证发布/回滚契约)
全栈知识治理平台首次入库(P1-P10 + 发布链路模拟验证闭环):

- 治理核心:源目录/变更集/人工审核/发布批次/发布执行(fail-closed 闸口)
- 测试通道:WeKnora 测试 KB 上传/轮询/验收;poll 完成时写入
  parser_generation/index_generation,修复 manual/seed 路径发布闸口永久拦截
- 发布适配器 v071(本地 source=测试角色 / 云端 target=生产角色 模拟验证通过):
  * ES 导出/导入 mapping 自适应(.keyword 子字段、可选 tenant_id)
  * OSS locator 支持嵌套 scheme;chunks.seq_id 由目标序列分配
  * bundle 归档同身份重试原子替换;verifier 按 bundle chunk_type 计数
  * 回滚 job payload 支持 prior_execution_id(strict 解码)
- 三通道发布+回滚演练实测:publish 4/4 verified,检索命中,
  rollback 4/4 rolled_back,目标端 PG is_enabled=false、ES is_enabled=false
- 前端 A·Data-console 风格全站统一,120 用例通过
- 门禁:go vet / go test 全包通过,gofmt 干净
2026-09-30 10:54:06 +08:00

329 lines
12 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package monitoring
import (
"context"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/url"
"sort"
"strings"
"time"
"knowledge-governance/internal/governance"
)
// P10: every configured dependency reports Liveness ("process reachable") and
// Readiness ("can accept work") independently. Missing deployment and missing
// configuration are explicit states and never collapse into "unknown".
const (
LiveUp = "up"
LiveDown = "down"
LiveUnknown = "unknown"
ReadReady = "ready"
ReadBlocked = "blocked"
ReadNotConfigured = "not_configured"
ReadNotApplicable = "not_applicable"
)
// ProbeOutcome is one raw probe attempt.
type ProbeOutcome struct {
Reachability error // liveness result (nil = reachable)
WorkReadiness error // readiness result (nil = can accept work)
Latency time.Duration
ObservedAt time.Time
}
var (
errCapacityFull = errors.New("capacity_full")
errNoPermission = errors.New("no_permission")
)
// classify turns one probe outcome into health states. Explicit not-configured
// and not-applicable results win over generic reachability failures.
func classify(key, name string, configured, applicable bool, outcome ProbeOutcome) governance.ServiceHealth {
health := governance.ServiceHealth{Key: key, Name: name, Configured: configured, Applicable: applicable, LatencyMS: outcome.Latency.Milliseconds(), ObservedAt: outcome.ObservedAt}
switch {
case !applicable:
health.Liveness, health.Reason = LiveUnknown, "未部署:本平台不使用该依赖"
health.Readiness = ReadNotApplicable
return health
case !configured:
health.Liveness, health.Reason = LiveUnknown, "已部署但未配置探测参数"
health.Readiness = ReadNotConfigured
return health
}
if outcome.Reachability != nil {
health.Readiness = ReadBlocked
health.Reason = sanitizeProbeError(outcome.Reachability)
if errors.Is(outcome.Reachability, errCapacityFull) || errors.Is(outcome.Reachability, errNoPermission) {
health.Liveness = LiveUp // reachable, but not workable
} else {
health.Liveness = LiveDown
}
return health
}
health.Liveness = LiveUp
if outcome.WorkReadiness != nil {
health.Readiness, health.Reason = ReadBlocked, sanitizeProbeError(outcome.WorkReadiness)
return health
}
health.Readiness = ReadReady
return health
}
// sanitizeProbeError keeps operators useful text without leaking endpoints,
// credentials, or hosts that live inside URLs.
func sanitizeProbeError(err error) string {
switch {
case errors.Is(err, errCapacityFull):
return "服务可达但容量已满(429),暂时不可接单"
case errors.Is(err, errNoPermission):
return "服务可达但目标知识库无读取权限"
case errors.Is(err, context.DeadlineExceeded), errors.Is(err, context.Canceled):
return "探测超时"
}
var dnsErr *net.DNSError
if errors.As(err, &dnsErr) {
return "地址解析失败(DNS)"
}
var netErr net.Error
if errors.As(err, &netErr) {
return "网络不可达"
}
message := err.Error()
if len(message) > 160 {
return message[:160]
}
return message
}
// healthTracker remembers recent success and consecutive failures per service.
type healthTracker struct {
records map[string]*trackedHealth
}
type trackedHealth struct {
lastSuccess *time.Time
failStreak int
}
func newHealthTracker() *healthTracker {
return &healthTracker{records: map[string]*trackedHealth{}}
}
func (tracker *healthTracker) apply(key string, health governance.ServiceHealth) governance.ServiceHealth {
record := tracker.records[key]
if record == nil {
record = &trackedHealth{}
tracker.records[key] = record
}
switch {
case health.Liveness == LiveUp && health.Readiness == ReadReady:
now := health.ObservedAt
record.lastSuccess = &now
record.failStreak = 0
case health.Liveness == LiveUnknown:
// Config-level states say nothing about runtime; keep the history.
default:
record.failStreak++
}
health.LastSuccessAt = record.lastSuccess
health.ConsecutiveFailures = record.failStreak
return health
}
// httpProbe performs a liveness-only HTTP GET on <base>/health, mapped through
// classify. configured=false when the base URL is absent.
func (provider *RealProvider) httpProbe(ctx context.Context, key, name, baseURL string, now time.Time) governance.ServiceHealth {
baseURL = strings.TrimSpace(baseURL)
if baseURL == "" {
return provider.track(classify(key, name, false, true, ProbeOutcome{ObservedAt: now}))
}
parsed, err := url.Parse(baseURL)
if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Host == "" {
return provider.track(classify(key, name, true, true, ProbeOutcome{Reachability: errors.New("探测配置无效"), ObservedAt: now}))
}
parsed.Path = strings.TrimSuffix(parsed.Path, "/") + "/health"
parsed.RawQuery, parsed.Fragment = "", ""
request, err := http.NewRequestWithContext(ctx, http.MethodGet, parsed.String(), nil)
if err != nil {
return provider.track(classify(key, name, true, true, ProbeOutcome{Reachability: err, ObservedAt: now}))
}
started := time.Now()
response, err := provider.httpClient.Do(request)
outcome := ProbeOutcome{Latency: time.Since(started), ObservedAt: now}
if err != nil {
outcome.Reachability = err
return provider.track(classify(key, name, true, true, outcome))
}
defer response.Body.Close()
_, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 4096))
if response.StatusCode < 200 || response.StatusCode >= 300 {
outcome.Reachability = fmt.Errorf("健康端点返回HTTP %d", response.StatusCode)
return provider.track(classify(key, name, true, true, outcome))
}
return provider.track(classify(key, name, true, true, outcome))
}
// weKnoraProbe checks the WeKnora API: liveness via /health, readiness via an
// optional real read against a routing knowledge base.
func (provider *RealProvider) weKnoraProbe(ctx context.Context, now time.Time) governance.ServiceHealth {
if provider.weknoraBaseURL == "" {
return provider.track(classify("weknora", "WeKnora API", false, true, ProbeOutcome{ObservedAt: now}))
}
outcome := provider.httpProbeOutcome(ctx, provider.weknoraBaseURL, now)
if outcome.Reachability == nil && provider.weknoraReadiness != nil {
outcome.WorkReadiness = provider.weknoraReadiness(ctx)
}
return provider.track(classify("weknora", "WeKnora API", true, true, outcome))
}
func (provider *RealProvider) httpProbeOutcome(ctx context.Context, baseURL string, now time.Time) ProbeOutcome {
parsed, err := url.Parse(baseURL)
outcome := ProbeOutcome{ObservedAt: now}
if err != nil || parsed.Scheme != "http" && parsed.Scheme != "https" || parsed.Host == "" {
outcome.Reachability = errors.New("探测配置无效")
return outcome
}
parsed.Path = strings.TrimSuffix(parsed.Path, "/") + "/health"
parsed.RawQuery, parsed.Fragment = "", ""
request, err := http.NewRequestWithContext(ctx, http.MethodGet, parsed.String(), nil)
if err != nil {
outcome.Reachability = err
return outcome
}
started := time.Now()
response, err := provider.httpClient.Do(request)
outcome.Latency = time.Since(started)
if err != nil {
outcome.Reachability = err
return outcome
}
defer response.Body.Close()
_, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 4096))
switch {
case response.StatusCode == http.StatusTooManyRequests:
// Reachable but at capacity: liveness stays up, readiness blocks.
outcome.Reachability = errCapacityFull
case response.StatusCode == http.StatusForbidden || response.StatusCode == http.StatusUnauthorized:
outcome.Reachability = errNoPermission
case response.StatusCode < 200 || response.StatusCode >= 300:
outcome.Reachability = fmt.Errorf("健康端点返回HTTP %d", response.StatusCode)
}
return outcome
}
// workerProbe reads queue-side worker heartbeats: no fresh heartbeat row means
// the worker loop is not touching the queue (liveness down), while the queue
// itself can still be ready for a future worker.
func (provider *RealProvider) workerProbe(ctx context.Context, now time.Time) []governance.ServiceHealth {
rows, err := provider.db.Query(ctx, `SELECT worker_id, last_seen_at FROM worker_heartbeats WHERE last_seen_at >= $1::timestamptz ORDER BY worker_id`, now.Add(-provider.workerStaleAfter))
if err != nil {
return []governance.ServiceHealth{provider.track(classify("worker", "解析Worker", true, true, ProbeOutcome{Reachability: err, ObservedAt: now}))}
}
defer rows.Close()
type entry struct {
id string
seen time.Time
}
live := []entry{}
for rows.Next() {
var id string
var seen time.Time
if err := rows.Scan(&id, &seen); err != nil {
return []governance.ServiceHealth{provider.track(classify("worker", "解析Worker", true, true, ProbeOutcome{Reachability: err, ObservedAt: now}))}
}
live = append(live, entry{id, seen.UTC()})
}
if len(live) == 0 {
return []governance.ServiceHealth{provider.track(classify("worker", "解析Worker", true, true, ProbeOutcome{Reachability: errors.New("队列无新鲜心跳(worker未运行或全部停跳)"), ObservedAt: now}))}
}
out := make([]governance.ServiceHealth, 0, len(live))
for _, item := range live {
outcome := ProbeOutcome{ObservedAt: item.seen}
if now.Sub(item.seen) > provider.workerStaleAfter {
outcome.Reachability = errors.New("心跳过期")
}
out = append(out, provider.track(classify("worker:"+item.id, "解析Worker "+item.id, true, true, outcome)))
}
return out
}
func (provider *RealProvider) track(health governance.ServiceHealth) governance.ServiceHealth {
return provider.health.apply(health.Key, health)
}
// Services reports the independent Liveness/Readiness matrix for every service
// the deployment actually has (P10). Missing services are explicit states.
func (provider *RealProvider) Services(ctx context.Context) []governance.ServiceHealth {
now := provider.now().UTC()
health := make([]governance.ServiceHealth, 0, 8)
// PostgreSQL: liveness = ping, readiness = queryable.
pgOutcome := ProbeOutcome{ObservedAt: now}
started := time.Now()
if err := provider.db.Ping(ctx); err != nil {
pgOutcome.Reachability = err
} else {
pgOutcome.Latency = time.Since(started)
var ok int
if err := provider.db.QueryRow(ctx, "SELECT 1").Scan(&ok); err != nil {
pgOutcome.WorkReadiness = err
}
}
health = append(health, provider.track(classify("postgres", "PostgreSQL", true, true, pgOutcome)))
// Queue readiness rides on the jobs table.
queueOutcome := ProbeOutcome{ObservedAt: now}
var pending int
if err := provider.db.QueryRow(ctx, "SELECT COUNT(*) FROM jobs WHERE state IN ('pending','retry')").Scan(&pending); err != nil {
queueOutcome.Reachability = err
}
health = append(health, provider.track(classify("queue", "任务队列", true, true, queueOutcome)))
health = append(health, provider.workerProbe(ctx, now)...)
health = append(health, provider.weKnoraProbe(ctx, now))
health = append(health, provider.httpProbe(ctx, "mineru", "MinerU", provider.minerUBaseURL, now))
// Not-applicable dependencies stay explicit (embedding/llm managed by
// WeKnora, redis/mq/oss not deployed).
health = append(health,
provider.track(classify("embedding", "Embedding模型", false, false, ProbeOutcome{ObservedAt: now})),
provider.track(classify("llm", "LLM模型", false, false, ProbeOutcome{ObservedAt: now})),
provider.track(classify("redis", "Redis / MQ", false, false, ProbeOutcome{ObservedAt: now})),
provider.track(classify("oss", "OSS", false, false, ProbeOutcome{ObservedAt: now})),
)
sort.Slice(health, func(i, j int) bool { return health[i].Key < health[j].Key })
return health
}
// Readiness blocks only on services that ARE deployed and configured and
// currently fail readiness; not_configured is surfaced as a warning elsewhere.
func overallReadiness(health []governance.ServiceHealth) (bool, []string) {
blocked := []string{}
for _, item := range health {
if !item.Applicable || !item.Configured {
continue
}
if item.Readiness != ReadReady {
blocked = append(blocked, item.Key)
}
}
return len(blocked) == 0, blocked
}
func (tracker *healthTracker) snapshot() []governance.ServiceHealth {
keys := make([]string, 0, len(tracker.records))
for key := range tracker.records {
keys = append(keys, key)
}
sort.Strings(keys)
records := make([]governance.ServiceHealth, 0, len(keys))
for _, key := range keys {
record := tracker.records[key]
item := governance.ServiceHealth{Key: key, LastSuccessAt: record.lastSuccess, ConsecutiveFailures: record.failStreak}
records = append(records, item)
}
return records
}