1db21fb899
- 生产/测试 WeKnora 地址、租户 ID、知识库 ID 由静态 env 收敛为版本化 UI 设置,保留修订号与操作人审计 - 新增 serviceverify:保存前真调目标端点探测,端点无效才拒绝;网络抖动或无凭据仅告警 - 迁移 000042(生产 WeKnora 端点)/000043(服务目标);补齐设置候选与校验契约用例
346 lines
13 KiB
Go
346 lines
13 KiB
Go
package monitoring
|
||
|
||
import (
|
||
"context"
|
||
"errors"
|
||
"fmt"
|
||
"io"
|
||
"net"
|
||
"net/http"
|
||
"net/url"
|
||
"sort"
|
||
"strings"
|
||
"time"
|
||
|
||
"knowledge-governance/internal/governance"
|
||
)
|
||
|
||
// P10: every configured dependency reports Liveness ("process reachable") and
|
||
// Readiness ("can accept work") independently. Missing deployment and missing
|
||
// configuration are explicit states and never collapse into "unknown".
|
||
|
||
const (
|
||
LiveUp = "up"
|
||
LiveDown = "down"
|
||
LiveUnknown = "unknown"
|
||
ReadReady = "ready"
|
||
ReadBlocked = "blocked"
|
||
ReadNotConfigured = "not_configured"
|
||
ReadNotApplicable = "not_applicable"
|
||
)
|
||
|
||
// ProbeOutcome is one raw probe attempt.
|
||
type ProbeOutcome struct {
|
||
Reachability error // liveness result (nil = reachable)
|
||
WorkReadiness error // readiness result (nil = can accept work)
|
||
Latency time.Duration
|
||
ObservedAt time.Time
|
||
}
|
||
|
||
var (
|
||
errCapacityFull = errors.New("capacity_full")
|
||
errNoPermission = errors.New("no_permission")
|
||
)
|
||
|
||
// classify turns one probe outcome into health states. Explicit not-configured
|
||
// and not-applicable results win over generic reachability failures.
|
||
func classify(key, name string, configured, applicable bool, outcome ProbeOutcome) governance.ServiceHealth {
|
||
health := governance.ServiceHealth{Key: key, Name: name, Configured: configured, Applicable: applicable, LatencyMS: outcome.Latency.Milliseconds(), ObservedAt: outcome.ObservedAt}
|
||
switch {
|
||
case !applicable:
|
||
health.Liveness, health.Reason = LiveUnknown, "未部署:本平台不使用该依赖"
|
||
health.Readiness = ReadNotApplicable
|
||
return health
|
||
case !configured:
|
||
health.Liveness, health.Reason = LiveUnknown, "已部署但未配置探测参数"
|
||
health.Readiness = ReadNotConfigured
|
||
return health
|
||
}
|
||
if outcome.Reachability != nil {
|
||
health.Readiness = ReadBlocked
|
||
health.Reason = sanitizeProbeError(outcome.Reachability)
|
||
if errors.Is(outcome.Reachability, errCapacityFull) || errors.Is(outcome.Reachability, errNoPermission) {
|
||
health.Liveness = LiveUp // reachable, but not workable
|
||
} else {
|
||
health.Liveness = LiveDown
|
||
}
|
||
return health
|
||
}
|
||
health.Liveness = LiveUp
|
||
if outcome.WorkReadiness != nil {
|
||
health.Readiness, health.Reason = ReadBlocked, sanitizeProbeError(outcome.WorkReadiness)
|
||
return health
|
||
}
|
||
health.Readiness = ReadReady
|
||
return health
|
||
}
|
||
|
||
// sanitizeProbeError keeps operators useful text without leaking endpoints,
|
||
// credentials, or hosts that live inside URLs.
|
||
func sanitizeProbeError(err error) string {
|
||
switch {
|
||
case errors.Is(err, errCapacityFull):
|
||
return "服务可达但容量已满(429),暂时不可接单"
|
||
case errors.Is(err, errNoPermission):
|
||
return "服务可达但目标知识库无读取权限"
|
||
case errors.Is(err, context.DeadlineExceeded), errors.Is(err, context.Canceled):
|
||
return "探测超时"
|
||
}
|
||
var dnsErr *net.DNSError
|
||
if errors.As(err, &dnsErr) {
|
||
return "地址解析失败(DNS)"
|
||
}
|
||
var netErr net.Error
|
||
if errors.As(err, &netErr) {
|
||
return "网络不可达"
|
||
}
|
||
message := err.Error()
|
||
if len(message) > 160 {
|
||
return message[:160]
|
||
}
|
||
return message
|
||
}
|
||
|
||
// healthTracker remembers recent success and consecutive failures per service.
|
||
type healthTracker struct {
|
||
records map[string]*trackedHealth
|
||
}
|
||
|
||
type trackedHealth struct {
|
||
lastSuccess *time.Time
|
||
failStreak int
|
||
}
|
||
|
||
func newHealthTracker() *healthTracker {
|
||
return &healthTracker{records: map[string]*trackedHealth{}}
|
||
}
|
||
|
||
func (tracker *healthTracker) apply(key string, health governance.ServiceHealth) governance.ServiceHealth {
|
||
record := tracker.records[key]
|
||
if record == nil {
|
||
record = &trackedHealth{}
|
||
tracker.records[key] = record
|
||
}
|
||
switch {
|
||
case health.Liveness == LiveUp && health.Readiness == ReadReady:
|
||
now := health.ObservedAt
|
||
record.lastSuccess = &now
|
||
record.failStreak = 0
|
||
case health.Liveness == LiveUnknown:
|
||
// Config-level states say nothing about runtime; keep the history.
|
||
default:
|
||
record.failStreak++
|
||
}
|
||
health.LastSuccessAt = record.lastSuccess
|
||
health.ConsecutiveFailures = record.failStreak
|
||
return health
|
||
}
|
||
|
||
// httpProbe performs a liveness-only HTTP GET on <base>/health, mapped through
|
||
// classify. configured=false when the base URL is absent.
|
||
func (provider *RealProvider) httpProbe(ctx context.Context, key, name, baseURL string, now time.Time) governance.ServiceHealth {
|
||
baseURL = strings.TrimSpace(baseURL)
|
||
if baseURL == "" {
|
||
return provider.track(classify(key, name, false, true, ProbeOutcome{ObservedAt: now}))
|
||
}
|
||
parsed, err := url.Parse(baseURL)
|
||
if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Host == "" {
|
||
return provider.track(classify(key, name, true, true, ProbeOutcome{Reachability: errors.New("探测配置无效"), ObservedAt: now}))
|
||
}
|
||
parsed.Path = strings.TrimSuffix(parsed.Path, "/") + "/health"
|
||
parsed.RawQuery, parsed.Fragment = "", ""
|
||
request, err := http.NewRequestWithContext(ctx, http.MethodGet, parsed.String(), nil)
|
||
if err != nil {
|
||
return provider.track(classify(key, name, true, true, ProbeOutcome{Reachability: err, ObservedAt: now}))
|
||
}
|
||
started := time.Now()
|
||
response, err := provider.httpClient.Do(request)
|
||
outcome := ProbeOutcome{Latency: time.Since(started), ObservedAt: now}
|
||
if err != nil {
|
||
outcome.Reachability = err
|
||
return provider.track(classify(key, name, true, true, outcome))
|
||
}
|
||
defer response.Body.Close()
|
||
_, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 4096))
|
||
if response.StatusCode < 200 || response.StatusCode >= 300 {
|
||
outcome.Reachability = fmt.Errorf("健康端点返回HTTP %d", response.StatusCode)
|
||
return provider.track(classify(key, name, true, true, outcome))
|
||
}
|
||
return provider.track(classify(key, name, true, true, outcome))
|
||
}
|
||
|
||
// weKnoraProbe checks the WeKnora API: liveness via /health, readiness via an
|
||
// optional real read against a routing knowledge base.
|
||
func (provider *RealProvider) weKnoraProbe(ctx context.Context, now time.Time) governance.ServiceHealth {
|
||
baseURL := provider.resolveTestWeKnoraBaseURL(ctx)
|
||
if baseURL == "" {
|
||
return provider.track(classify("weknora", "WeKnora API", false, true, ProbeOutcome{ObservedAt: now}))
|
||
}
|
||
outcome := provider.httpProbeOutcome(ctx, baseURL, now)
|
||
if outcome.Reachability == nil && provider.weknoraReadiness != nil {
|
||
outcome.WorkReadiness = provider.weknoraReadiness(ctx)
|
||
}
|
||
return provider.track(classify("weknora", "WeKnora API", true, true, outcome))
|
||
}
|
||
|
||
// productionWeKnoraProbe reports the real state of the production WeKnora
|
||
// endpoint (liveness via /health). It is advisory: publishing stays gated by the
|
||
// production switch and keeps its target in the release runtime config.
|
||
func (provider *RealProvider) productionWeKnoraProbe(ctx context.Context, now time.Time) governance.ServiceHealth {
|
||
baseURL := provider.resolveProductionWeKnoraBaseURL(ctx)
|
||
if baseURL == "" {
|
||
return provider.track(classify("weknora-production", "生产 WeKnora", false, true, ProbeOutcome{ObservedAt: now}))
|
||
}
|
||
return provider.track(classify("weknora-production", "生产 WeKnora", true, true, provider.httpProbeOutcome(ctx, baseURL, now)))
|
||
}
|
||
|
||
func (provider *RealProvider) httpProbeOutcome(ctx context.Context, baseURL string, now time.Time) ProbeOutcome {
|
||
parsed, err := url.Parse(baseURL)
|
||
outcome := ProbeOutcome{ObservedAt: now}
|
||
if err != nil || parsed.Scheme != "http" && parsed.Scheme != "https" || parsed.Host == "" {
|
||
outcome.Reachability = errors.New("探测配置无效")
|
||
return outcome
|
||
}
|
||
parsed.Path = strings.TrimSuffix(parsed.Path, "/") + "/health"
|
||
parsed.RawQuery, parsed.Fragment = "", ""
|
||
request, err := http.NewRequestWithContext(ctx, http.MethodGet, parsed.String(), nil)
|
||
if err != nil {
|
||
outcome.Reachability = err
|
||
return outcome
|
||
}
|
||
started := time.Now()
|
||
response, err := provider.httpClient.Do(request)
|
||
outcome.Latency = time.Since(started)
|
||
if err != nil {
|
||
outcome.Reachability = err
|
||
return outcome
|
||
}
|
||
defer response.Body.Close()
|
||
_, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 4096))
|
||
switch {
|
||
case response.StatusCode == http.StatusTooManyRequests:
|
||
// Reachable but at capacity: liveness stays up, readiness blocks.
|
||
outcome.Reachability = errCapacityFull
|
||
case response.StatusCode == http.StatusForbidden || response.StatusCode == http.StatusUnauthorized:
|
||
outcome.Reachability = errNoPermission
|
||
case response.StatusCode < 200 || response.StatusCode >= 300:
|
||
outcome.Reachability = fmt.Errorf("健康端点返回HTTP %d", response.StatusCode)
|
||
}
|
||
return outcome
|
||
}
|
||
|
||
// workerProbe reads queue-side worker heartbeats: no fresh heartbeat row means
|
||
// the worker loop is not touching the queue (liveness down), while the queue
|
||
// itself can still be ready for a future worker.
|
||
func (provider *RealProvider) workerProbe(ctx context.Context, now time.Time) []governance.ServiceHealth {
|
||
rows, err := provider.db.Query(ctx, `SELECT worker_id, last_seen_at FROM worker_heartbeats WHERE last_seen_at >= $1::timestamptz ORDER BY worker_id`, now.Add(-provider.workerStaleAfter))
|
||
if err != nil {
|
||
return []governance.ServiceHealth{provider.track(classify("worker", "解析Worker", true, true, ProbeOutcome{Reachability: err, ObservedAt: now}))}
|
||
}
|
||
defer rows.Close()
|
||
type entry struct {
|
||
id string
|
||
seen time.Time
|
||
}
|
||
live := []entry{}
|
||
for rows.Next() {
|
||
var id string
|
||
var seen time.Time
|
||
if err := rows.Scan(&id, &seen); err != nil {
|
||
return []governance.ServiceHealth{provider.track(classify("worker", "解析Worker", true, true, ProbeOutcome{Reachability: err, ObservedAt: now}))}
|
||
}
|
||
live = append(live, entry{id, seen.UTC()})
|
||
}
|
||
if len(live) == 0 {
|
||
return []governance.ServiceHealth{provider.track(classify("worker", "解析Worker", true, true, ProbeOutcome{Reachability: errors.New("队列无新鲜心跳(worker未运行或全部停跳)"), ObservedAt: now}))}
|
||
}
|
||
out := make([]governance.ServiceHealth, 0, len(live))
|
||
for _, item := range live {
|
||
outcome := ProbeOutcome{ObservedAt: item.seen}
|
||
if now.Sub(item.seen) > provider.workerStaleAfter {
|
||
outcome.Reachability = errors.New("心跳过期")
|
||
}
|
||
out = append(out, provider.track(classify("worker:"+item.id, "解析Worker "+item.id, true, true, outcome)))
|
||
}
|
||
return out
|
||
}
|
||
|
||
func (provider *RealProvider) track(health governance.ServiceHealth) governance.ServiceHealth {
|
||
return provider.health.apply(health.Key, health)
|
||
}
|
||
|
||
// Services reports the independent Liveness/Readiness matrix for every service
|
||
// the deployment actually has (P10). Missing services are explicit states.
|
||
func (provider *RealProvider) Services(ctx context.Context) []governance.ServiceHealth {
|
||
now := provider.now().UTC()
|
||
health := make([]governance.ServiceHealth, 0, 8)
|
||
// PostgreSQL: liveness = ping, readiness = queryable.
|
||
pgOutcome := ProbeOutcome{ObservedAt: now}
|
||
started := time.Now()
|
||
if err := provider.db.Ping(ctx); err != nil {
|
||
pgOutcome.Reachability = err
|
||
} else {
|
||
pgOutcome.Latency = time.Since(started)
|
||
var ok int
|
||
if err := provider.db.QueryRow(ctx, "SELECT 1").Scan(&ok); err != nil {
|
||
pgOutcome.WorkReadiness = err
|
||
}
|
||
}
|
||
health = append(health, provider.track(classify("postgres", "PostgreSQL", true, true, pgOutcome)))
|
||
// Queue readiness rides on the jobs table.
|
||
queueOutcome := ProbeOutcome{ObservedAt: now}
|
||
var pending int
|
||
if err := provider.db.QueryRow(ctx, "SELECT COUNT(*) FROM jobs WHERE state IN ('pending','retry')").Scan(&pending); err != nil {
|
||
queueOutcome.Reachability = err
|
||
}
|
||
health = append(health, provider.track(classify("queue", "任务队列", true, true, queueOutcome)))
|
||
health = append(health, provider.workerProbe(ctx, now)...)
|
||
health = append(health, provider.weKnoraProbe(ctx, now))
|
||
health = append(health, provider.productionWeKnoraProbe(ctx, now))
|
||
health = append(health, provider.httpProbe(ctx, "mineru", "MinerU", provider.minerUBaseURL, now))
|
||
// Not-applicable dependencies stay explicit (embedding/llm managed by
|
||
// WeKnora, redis/mq/oss not deployed).
|
||
health = append(health,
|
||
provider.track(classify("embedding", "Embedding模型", false, false, ProbeOutcome{ObservedAt: now})),
|
||
provider.track(classify("llm", "LLM模型", false, false, ProbeOutcome{ObservedAt: now})),
|
||
provider.track(classify("redis", "Redis / MQ", false, false, ProbeOutcome{ObservedAt: now})),
|
||
provider.track(classify("oss", "OSS", false, false, ProbeOutcome{ObservedAt: now})),
|
||
)
|
||
sort.Slice(health, func(i, j int) bool { return health[i].Key < health[j].Key })
|
||
return health
|
||
}
|
||
|
||
// Readiness blocks only on services that ARE deployed and configured and
|
||
// currently fail readiness; not_configured is surfaced as a warning elsewhere.
|
||
// advisoryHealthKeys are reported to operators but never block the platform's
|
||
// own readiness: a production endpoint outage must not stop the test pipeline.
|
||
var advisoryHealthKeys = map[string]bool{"weknora-production": true}
|
||
|
||
func overallReadiness(health []governance.ServiceHealth) (bool, []string) {
|
||
blocked := []string{}
|
||
for _, item := range health {
|
||
if !item.Applicable || !item.Configured || advisoryHealthKeys[item.Key] {
|
||
continue
|
||
}
|
||
if item.Readiness != ReadReady {
|
||
blocked = append(blocked, item.Key)
|
||
}
|
||
}
|
||
return len(blocked) == 0, blocked
|
||
}
|
||
|
||
func (tracker *healthTracker) snapshot() []governance.ServiceHealth {
|
||
keys := make([]string, 0, len(tracker.records))
|
||
for key := range tracker.records {
|
||
keys = append(keys, key)
|
||
}
|
||
sort.Strings(keys)
|
||
records := make([]governance.ServiceHealth, 0, len(keys))
|
||
for _, key := range keys {
|
||
record := tracker.records[key]
|
||
item := governance.ServiceHealth{Key: key, LastSuccessAt: record.lastSuccess, ConsecutiveFailures: record.failStreak}
|
||
records = append(records, item)
|
||
}
|
||
return records
|
||
}
|