dcdc8d1ddd
全栈知识治理平台首次入库(P1-P10 + 发布链路模拟验证闭环): - 治理核心:源目录/变更集/人工审核/发布批次/发布执行(fail-closed 闸口) - 测试通道:WeKnora 测试 KB 上传/轮询/验收;poll 完成时写入 parser_generation/index_generation,修复 manual/seed 路径发布闸口永久拦截 - 发布适配器 v071(本地 source=测试角色 / 云端 target=生产角色 模拟验证通过): * ES 导出/导入 mapping 自适应(.keyword 子字段、可选 tenant_id) * OSS locator 支持嵌套 scheme;chunks.seq_id 由目标序列分配 * bundle 归档同身份重试原子替换;verifier 按 bundle chunk_type 计数 * 回滚 job payload 支持 prior_execution_id(strict 解码) - 三通道发布+回滚演练实测:publish 4/4 verified,检索命中, rollback 4/4 rolled_back,目标端 PG is_enabled=false、ES is_enabled=false - 前端 A·Data-console 风格全站统一,120 用例通过 - 门禁:go vet / go test 全包通过,gofmt 干净
329 lines
12 KiB
Go
329 lines
12 KiB
Go
package monitoring
|
||
|
||
import (
|
||
"context"
|
||
"errors"
|
||
"fmt"
|
||
"io"
|
||
"net"
|
||
"net/http"
|
||
"net/url"
|
||
"sort"
|
||
"strings"
|
||
"time"
|
||
|
||
"knowledge-governance/internal/governance"
|
||
)
|
||
|
||
// P10: every configured dependency reports Liveness ("process reachable") and
|
||
// Readiness ("can accept work") independently. Missing deployment and missing
|
||
// configuration are explicit states and never collapse into "unknown".
|
||
|
||
const (
|
||
LiveUp = "up"
|
||
LiveDown = "down"
|
||
LiveUnknown = "unknown"
|
||
ReadReady = "ready"
|
||
ReadBlocked = "blocked"
|
||
ReadNotConfigured = "not_configured"
|
||
ReadNotApplicable = "not_applicable"
|
||
)
|
||
|
||
// ProbeOutcome is one raw probe attempt.
|
||
type ProbeOutcome struct {
|
||
Reachability error // liveness result (nil = reachable)
|
||
WorkReadiness error // readiness result (nil = can accept work)
|
||
Latency time.Duration
|
||
ObservedAt time.Time
|
||
}
|
||
|
||
var (
|
||
errCapacityFull = errors.New("capacity_full")
|
||
errNoPermission = errors.New("no_permission")
|
||
)
|
||
|
||
// classify turns one probe outcome into health states. Explicit not-configured
|
||
// and not-applicable results win over generic reachability failures.
|
||
func classify(key, name string, configured, applicable bool, outcome ProbeOutcome) governance.ServiceHealth {
|
||
health := governance.ServiceHealth{Key: key, Name: name, Configured: configured, Applicable: applicable, LatencyMS: outcome.Latency.Milliseconds(), ObservedAt: outcome.ObservedAt}
|
||
switch {
|
||
case !applicable:
|
||
health.Liveness, health.Reason = LiveUnknown, "未部署:本平台不使用该依赖"
|
||
health.Readiness = ReadNotApplicable
|
||
return health
|
||
case !configured:
|
||
health.Liveness, health.Reason = LiveUnknown, "已部署但未配置探测参数"
|
||
health.Readiness = ReadNotConfigured
|
||
return health
|
||
}
|
||
if outcome.Reachability != nil {
|
||
health.Readiness = ReadBlocked
|
||
health.Reason = sanitizeProbeError(outcome.Reachability)
|
||
if errors.Is(outcome.Reachability, errCapacityFull) || errors.Is(outcome.Reachability, errNoPermission) {
|
||
health.Liveness = LiveUp // reachable, but not workable
|
||
} else {
|
||
health.Liveness = LiveDown
|
||
}
|
||
return health
|
||
}
|
||
health.Liveness = LiveUp
|
||
if outcome.WorkReadiness != nil {
|
||
health.Readiness, health.Reason = ReadBlocked, sanitizeProbeError(outcome.WorkReadiness)
|
||
return health
|
||
}
|
||
health.Readiness = ReadReady
|
||
return health
|
||
}
|
||
|
||
// sanitizeProbeError keeps operators useful text without leaking endpoints,
|
||
// credentials, or hosts that live inside URLs.
|
||
func sanitizeProbeError(err error) string {
|
||
switch {
|
||
case errors.Is(err, errCapacityFull):
|
||
return "服务可达但容量已满(429),暂时不可接单"
|
||
case errors.Is(err, errNoPermission):
|
||
return "服务可达但目标知识库无读取权限"
|
||
case errors.Is(err, context.DeadlineExceeded), errors.Is(err, context.Canceled):
|
||
return "探测超时"
|
||
}
|
||
var dnsErr *net.DNSError
|
||
if errors.As(err, &dnsErr) {
|
||
return "地址解析失败(DNS)"
|
||
}
|
||
var netErr net.Error
|
||
if errors.As(err, &netErr) {
|
||
return "网络不可达"
|
||
}
|
||
message := err.Error()
|
||
if len(message) > 160 {
|
||
return message[:160]
|
||
}
|
||
return message
|
||
}
|
||
|
||
// healthTracker remembers recent success and consecutive failures per service.
|
||
type healthTracker struct {
|
||
records map[string]*trackedHealth
|
||
}
|
||
|
||
type trackedHealth struct {
|
||
lastSuccess *time.Time
|
||
failStreak int
|
||
}
|
||
|
||
func newHealthTracker() *healthTracker {
|
||
return &healthTracker{records: map[string]*trackedHealth{}}
|
||
}
|
||
|
||
func (tracker *healthTracker) apply(key string, health governance.ServiceHealth) governance.ServiceHealth {
|
||
record := tracker.records[key]
|
||
if record == nil {
|
||
record = &trackedHealth{}
|
||
tracker.records[key] = record
|
||
}
|
||
switch {
|
||
case health.Liveness == LiveUp && health.Readiness == ReadReady:
|
||
now := health.ObservedAt
|
||
record.lastSuccess = &now
|
||
record.failStreak = 0
|
||
case health.Liveness == LiveUnknown:
|
||
// Config-level states say nothing about runtime; keep the history.
|
||
default:
|
||
record.failStreak++
|
||
}
|
||
health.LastSuccessAt = record.lastSuccess
|
||
health.ConsecutiveFailures = record.failStreak
|
||
return health
|
||
}
|
||
|
||
// httpProbe performs a liveness-only HTTP GET on <base>/health, mapped through
|
||
// classify. configured=false when the base URL is absent.
|
||
func (provider *RealProvider) httpProbe(ctx context.Context, key, name, baseURL string, now time.Time) governance.ServiceHealth {
|
||
baseURL = strings.TrimSpace(baseURL)
|
||
if baseURL == "" {
|
||
return provider.track(classify(key, name, false, true, ProbeOutcome{ObservedAt: now}))
|
||
}
|
||
parsed, err := url.Parse(baseURL)
|
||
if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Host == "" {
|
||
return provider.track(classify(key, name, true, true, ProbeOutcome{Reachability: errors.New("探测配置无效"), ObservedAt: now}))
|
||
}
|
||
parsed.Path = strings.TrimSuffix(parsed.Path, "/") + "/health"
|
||
parsed.RawQuery, parsed.Fragment = "", ""
|
||
request, err := http.NewRequestWithContext(ctx, http.MethodGet, parsed.String(), nil)
|
||
if err != nil {
|
||
return provider.track(classify(key, name, true, true, ProbeOutcome{Reachability: err, ObservedAt: now}))
|
||
}
|
||
started := time.Now()
|
||
response, err := provider.httpClient.Do(request)
|
||
outcome := ProbeOutcome{Latency: time.Since(started), ObservedAt: now}
|
||
if err != nil {
|
||
outcome.Reachability = err
|
||
return provider.track(classify(key, name, true, true, outcome))
|
||
}
|
||
defer response.Body.Close()
|
||
_, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 4096))
|
||
if response.StatusCode < 200 || response.StatusCode >= 300 {
|
||
outcome.Reachability = fmt.Errorf("健康端点返回HTTP %d", response.StatusCode)
|
||
return provider.track(classify(key, name, true, true, outcome))
|
||
}
|
||
return provider.track(classify(key, name, true, true, outcome))
|
||
}
|
||
|
||
// weKnoraProbe checks the WeKnora API: liveness via /health, readiness via an
|
||
// optional real read against a routing knowledge base.
|
||
func (provider *RealProvider) weKnoraProbe(ctx context.Context, now time.Time) governance.ServiceHealth {
|
||
if provider.weknoraBaseURL == "" {
|
||
return provider.track(classify("weknora", "WeKnora API", false, true, ProbeOutcome{ObservedAt: now}))
|
||
}
|
||
outcome := provider.httpProbeOutcome(ctx, provider.weknoraBaseURL, now)
|
||
if outcome.Reachability == nil && provider.weknoraReadiness != nil {
|
||
outcome.WorkReadiness = provider.weknoraReadiness(ctx)
|
||
}
|
||
return provider.track(classify("weknora", "WeKnora API", true, true, outcome))
|
||
}
|
||
|
||
func (provider *RealProvider) httpProbeOutcome(ctx context.Context, baseURL string, now time.Time) ProbeOutcome {
|
||
parsed, err := url.Parse(baseURL)
|
||
outcome := ProbeOutcome{ObservedAt: now}
|
||
if err != nil || parsed.Scheme != "http" && parsed.Scheme != "https" || parsed.Host == "" {
|
||
outcome.Reachability = errors.New("探测配置无效")
|
||
return outcome
|
||
}
|
||
parsed.Path = strings.TrimSuffix(parsed.Path, "/") + "/health"
|
||
parsed.RawQuery, parsed.Fragment = "", ""
|
||
request, err := http.NewRequestWithContext(ctx, http.MethodGet, parsed.String(), nil)
|
||
if err != nil {
|
||
outcome.Reachability = err
|
||
return outcome
|
||
}
|
||
started := time.Now()
|
||
response, err := provider.httpClient.Do(request)
|
||
outcome.Latency = time.Since(started)
|
||
if err != nil {
|
||
outcome.Reachability = err
|
||
return outcome
|
||
}
|
||
defer response.Body.Close()
|
||
_, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 4096))
|
||
switch {
|
||
case response.StatusCode == http.StatusTooManyRequests:
|
||
// Reachable but at capacity: liveness stays up, readiness blocks.
|
||
outcome.Reachability = errCapacityFull
|
||
case response.StatusCode == http.StatusForbidden || response.StatusCode == http.StatusUnauthorized:
|
||
outcome.Reachability = errNoPermission
|
||
case response.StatusCode < 200 || response.StatusCode >= 300:
|
||
outcome.Reachability = fmt.Errorf("健康端点返回HTTP %d", response.StatusCode)
|
||
}
|
||
return outcome
|
||
}
|
||
|
||
// workerProbe reads queue-side worker heartbeats: no fresh heartbeat row means
|
||
// the worker loop is not touching the queue (liveness down), while the queue
|
||
// itself can still be ready for a future worker.
|
||
func (provider *RealProvider) workerProbe(ctx context.Context, now time.Time) []governance.ServiceHealth {
|
||
rows, err := provider.db.Query(ctx, `SELECT worker_id, last_seen_at FROM worker_heartbeats WHERE last_seen_at >= $1::timestamptz ORDER BY worker_id`, now.Add(-provider.workerStaleAfter))
|
||
if err != nil {
|
||
return []governance.ServiceHealth{provider.track(classify("worker", "解析Worker", true, true, ProbeOutcome{Reachability: err, ObservedAt: now}))}
|
||
}
|
||
defer rows.Close()
|
||
type entry struct {
|
||
id string
|
||
seen time.Time
|
||
}
|
||
live := []entry{}
|
||
for rows.Next() {
|
||
var id string
|
||
var seen time.Time
|
||
if err := rows.Scan(&id, &seen); err != nil {
|
||
return []governance.ServiceHealth{provider.track(classify("worker", "解析Worker", true, true, ProbeOutcome{Reachability: err, ObservedAt: now}))}
|
||
}
|
||
live = append(live, entry{id, seen.UTC()})
|
||
}
|
||
if len(live) == 0 {
|
||
return []governance.ServiceHealth{provider.track(classify("worker", "解析Worker", true, true, ProbeOutcome{Reachability: errors.New("队列无新鲜心跳(worker未运行或全部停跳)"), ObservedAt: now}))}
|
||
}
|
||
out := make([]governance.ServiceHealth, 0, len(live))
|
||
for _, item := range live {
|
||
outcome := ProbeOutcome{ObservedAt: item.seen}
|
||
if now.Sub(item.seen) > provider.workerStaleAfter {
|
||
outcome.Reachability = errors.New("心跳过期")
|
||
}
|
||
out = append(out, provider.track(classify("worker:"+item.id, "解析Worker "+item.id, true, true, outcome)))
|
||
}
|
||
return out
|
||
}
|
||
|
||
func (provider *RealProvider) track(health governance.ServiceHealth) governance.ServiceHealth {
|
||
return provider.health.apply(health.Key, health)
|
||
}
|
||
|
||
// Services reports the independent Liveness/Readiness matrix for every service
|
||
// the deployment actually has (P10). Missing services are explicit states.
|
||
func (provider *RealProvider) Services(ctx context.Context) []governance.ServiceHealth {
|
||
now := provider.now().UTC()
|
||
health := make([]governance.ServiceHealth, 0, 8)
|
||
// PostgreSQL: liveness = ping, readiness = queryable.
|
||
pgOutcome := ProbeOutcome{ObservedAt: now}
|
||
started := time.Now()
|
||
if err := provider.db.Ping(ctx); err != nil {
|
||
pgOutcome.Reachability = err
|
||
} else {
|
||
pgOutcome.Latency = time.Since(started)
|
||
var ok int
|
||
if err := provider.db.QueryRow(ctx, "SELECT 1").Scan(&ok); err != nil {
|
||
pgOutcome.WorkReadiness = err
|
||
}
|
||
}
|
||
health = append(health, provider.track(classify("postgres", "PostgreSQL", true, true, pgOutcome)))
|
||
// Queue readiness rides on the jobs table.
|
||
queueOutcome := ProbeOutcome{ObservedAt: now}
|
||
var pending int
|
||
if err := provider.db.QueryRow(ctx, "SELECT COUNT(*) FROM jobs WHERE state IN ('pending','retry')").Scan(&pending); err != nil {
|
||
queueOutcome.Reachability = err
|
||
}
|
||
health = append(health, provider.track(classify("queue", "任务队列", true, true, queueOutcome)))
|
||
health = append(health, provider.workerProbe(ctx, now)...)
|
||
health = append(health, provider.weKnoraProbe(ctx, now))
|
||
health = append(health, provider.httpProbe(ctx, "mineru", "MinerU", provider.minerUBaseURL, now))
|
||
// Not-applicable dependencies stay explicit (embedding/llm managed by
|
||
// WeKnora, redis/mq/oss not deployed).
|
||
health = append(health,
|
||
provider.track(classify("embedding", "Embedding模型", false, false, ProbeOutcome{ObservedAt: now})),
|
||
provider.track(classify("llm", "LLM模型", false, false, ProbeOutcome{ObservedAt: now})),
|
||
provider.track(classify("redis", "Redis / MQ", false, false, ProbeOutcome{ObservedAt: now})),
|
||
provider.track(classify("oss", "OSS", false, false, ProbeOutcome{ObservedAt: now})),
|
||
)
|
||
sort.Slice(health, func(i, j int) bool { return health[i].Key < health[j].Key })
|
||
return health
|
||
}
|
||
|
||
// Readiness blocks only on services that ARE deployed and configured and
|
||
// currently fail readiness; not_configured is surfaced as a warning elsewhere.
|
||
func overallReadiness(health []governance.ServiceHealth) (bool, []string) {
|
||
blocked := []string{}
|
||
for _, item := range health {
|
||
if !item.Applicable || !item.Configured {
|
||
continue
|
||
}
|
||
if item.Readiness != ReadReady {
|
||
blocked = append(blocked, item.Key)
|
||
}
|
||
}
|
||
return len(blocked) == 0, blocked
|
||
}
|
||
|
||
func (tracker *healthTracker) snapshot() []governance.ServiceHealth {
|
||
keys := make([]string, 0, len(tracker.records))
|
||
for key := range tracker.records {
|
||
keys = append(keys, key)
|
||
}
|
||
sort.Strings(keys)
|
||
records := make([]governance.ServiceHealth, 0, len(keys))
|
||
for _, key := range keys {
|
||
record := tracker.records[key]
|
||
item := governance.ServiceHealth{Key: key, LastSuccessAt: record.lastSuccess, ConsecutiveFailures: record.failStreak}
|
||
records = append(records, item)
|
||
}
|
||
return records
|
||
}
|