feat: 知识治理平台(含已验证发布/回滚契约)

全栈知识治理平台首次入库(P1-P10 + 发布链路模拟验证闭环):

- 治理核心:源目录/变更集/人工审核/发布批次/发布执行(fail-closed 闸口)
- 测试通道:WeKnora 测试 KB 上传/轮询/验收;poll 完成时写入
  parser_generation/index_generation,修复 manual/seed 路径发布闸口永久拦截
- 发布适配器 v071(本地 source=测试角色 / 云端 target=生产角色 模拟验证通过):
  * ES 导出/导入 mapping 自适应(.keyword 子字段、可选 tenant_id)
  * OSS locator 支持嵌套 scheme;chunks.seq_id 由目标序列分配
  * bundle 归档同身份重试原子替换;verifier 按 bundle chunk_type 计数
  * 回滚 job payload 支持 prior_execution_id(strict 解码)
- 三通道发布+回滚演练实测:publish 4/4 verified,检索命中,
  rollback 4/4 rolled_back,目标端 PG is_enabled=false、ES is_enabled=false
- 前端 A·Data-console 风格全站统一,120 用例通过
- 门禁:go vet / go test 全包通过,gofmt 干净
This commit is contained in:
2026-09-13 12:02:58 +08:00
committed by 姚章浩
parent ed48c4ceda
commit dcdc8d1ddd
410 changed files with 56661 additions and 1 deletions
+328
View File
@@ -0,0 +1,328 @@
package monitoring
import (
"context"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/url"
"sort"
"strings"
"time"
"knowledge-governance/internal/governance"
)
// P10: every configured dependency reports Liveness ("process reachable") and
// Readiness ("can accept work") independently. Missing deployment and missing
// configuration are explicit states and never collapse into "unknown".
const (
LiveUp = "up"
LiveDown = "down"
LiveUnknown = "unknown"
ReadReady = "ready"
ReadBlocked = "blocked"
ReadNotConfigured = "not_configured"
ReadNotApplicable = "not_applicable"
)
// ProbeOutcome is one raw probe attempt.
type ProbeOutcome struct {
Reachability error // liveness result (nil = reachable)
WorkReadiness error // readiness result (nil = can accept work)
Latency time.Duration
ObservedAt time.Time
}
var (
errCapacityFull = errors.New("capacity_full")
errNoPermission = errors.New("no_permission")
)
// classify turns one probe outcome into health states. Explicit not-configured
// and not-applicable results win over generic reachability failures.
func classify(key, name string, configured, applicable bool, outcome ProbeOutcome) governance.ServiceHealth {
health := governance.ServiceHealth{Key: key, Name: name, Configured: configured, Applicable: applicable, LatencyMS: outcome.Latency.Milliseconds(), ObservedAt: outcome.ObservedAt}
switch {
case !applicable:
health.Liveness, health.Reason = LiveUnknown, "未部署:本平台不使用该依赖"
health.Readiness = ReadNotApplicable
return health
case !configured:
health.Liveness, health.Reason = LiveUnknown, "已部署但未配置探测参数"
health.Readiness = ReadNotConfigured
return health
}
if outcome.Reachability != nil {
health.Readiness = ReadBlocked
health.Reason = sanitizeProbeError(outcome.Reachability)
if errors.Is(outcome.Reachability, errCapacityFull) || errors.Is(outcome.Reachability, errNoPermission) {
health.Liveness = LiveUp // reachable, but not workable
} else {
health.Liveness = LiveDown
}
return health
}
health.Liveness = LiveUp
if outcome.WorkReadiness != nil {
health.Readiness, health.Reason = ReadBlocked, sanitizeProbeError(outcome.WorkReadiness)
return health
}
health.Readiness = ReadReady
return health
}
// sanitizeProbeError keeps operators useful text without leaking endpoints,
// credentials, or hosts that live inside URLs.
func sanitizeProbeError(err error) string {
switch {
case errors.Is(err, errCapacityFull):
return "服务可达但容量已满(429),暂时不可接单"
case errors.Is(err, errNoPermission):
return "服务可达但目标知识库无读取权限"
case errors.Is(err, context.DeadlineExceeded), errors.Is(err, context.Canceled):
return "探测超时"
}
var dnsErr *net.DNSError
if errors.As(err, &dnsErr) {
return "地址解析失败(DNS)"
}
var netErr net.Error
if errors.As(err, &netErr) {
return "网络不可达"
}
message := err.Error()
if len(message) > 160 {
return message[:160]
}
return message
}
// healthTracker remembers recent success and consecutive failures per service.
type healthTracker struct {
records map[string]*trackedHealth
}
type trackedHealth struct {
lastSuccess *time.Time
failStreak int
}
func newHealthTracker() *healthTracker {
return &healthTracker{records: map[string]*trackedHealth{}}
}
func (tracker *healthTracker) apply(key string, health governance.ServiceHealth) governance.ServiceHealth {
record := tracker.records[key]
if record == nil {
record = &trackedHealth{}
tracker.records[key] = record
}
switch {
case health.Liveness == LiveUp && health.Readiness == ReadReady:
now := health.ObservedAt
record.lastSuccess = &now
record.failStreak = 0
case health.Liveness == LiveUnknown:
// Config-level states say nothing about runtime; keep the history.
default:
record.failStreak++
}
health.LastSuccessAt = record.lastSuccess
health.ConsecutiveFailures = record.failStreak
return health
}
// httpProbe performs a liveness-only HTTP GET on <base>/health, mapped through
// classify. configured=false when the base URL is absent.
func (provider *RealProvider) httpProbe(ctx context.Context, key, name, baseURL string, now time.Time) governance.ServiceHealth {
baseURL = strings.TrimSpace(baseURL)
if baseURL == "" {
return provider.track(classify(key, name, false, true, ProbeOutcome{ObservedAt: now}))
}
parsed, err := url.Parse(baseURL)
if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Host == "" {
return provider.track(classify(key, name, true, true, ProbeOutcome{Reachability: errors.New("探测配置无效"), ObservedAt: now}))
}
parsed.Path = strings.TrimSuffix(parsed.Path, "/") + "/health"
parsed.RawQuery, parsed.Fragment = "", ""
request, err := http.NewRequestWithContext(ctx, http.MethodGet, parsed.String(), nil)
if err != nil {
return provider.track(classify(key, name, true, true, ProbeOutcome{Reachability: err, ObservedAt: now}))
}
started := time.Now()
response, err := provider.httpClient.Do(request)
outcome := ProbeOutcome{Latency: time.Since(started), ObservedAt: now}
if err != nil {
outcome.Reachability = err
return provider.track(classify(key, name, true, true, outcome))
}
defer response.Body.Close()
_, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 4096))
if response.StatusCode < 200 || response.StatusCode >= 300 {
outcome.Reachability = fmt.Errorf("健康端点返回HTTP %d", response.StatusCode)
return provider.track(classify(key, name, true, true, outcome))
}
return provider.track(classify(key, name, true, true, outcome))
}
// weKnoraProbe checks the WeKnora API: liveness via /health, readiness via an
// optional real read against a routing knowledge base.
func (provider *RealProvider) weKnoraProbe(ctx context.Context, now time.Time) governance.ServiceHealth {
if provider.weknoraBaseURL == "" {
return provider.track(classify("weknora", "WeKnora API", false, true, ProbeOutcome{ObservedAt: now}))
}
outcome := provider.httpProbeOutcome(ctx, provider.weknoraBaseURL, now)
if outcome.Reachability == nil && provider.weknoraReadiness != nil {
outcome.WorkReadiness = provider.weknoraReadiness(ctx)
}
return provider.track(classify("weknora", "WeKnora API", true, true, outcome))
}
func (provider *RealProvider) httpProbeOutcome(ctx context.Context, baseURL string, now time.Time) ProbeOutcome {
parsed, err := url.Parse(baseURL)
outcome := ProbeOutcome{ObservedAt: now}
if err != nil || parsed.Scheme != "http" && parsed.Scheme != "https" || parsed.Host == "" {
outcome.Reachability = errors.New("探测配置无效")
return outcome
}
parsed.Path = strings.TrimSuffix(parsed.Path, "/") + "/health"
parsed.RawQuery, parsed.Fragment = "", ""
request, err := http.NewRequestWithContext(ctx, http.MethodGet, parsed.String(), nil)
if err != nil {
outcome.Reachability = err
return outcome
}
started := time.Now()
response, err := provider.httpClient.Do(request)
outcome.Latency = time.Since(started)
if err != nil {
outcome.Reachability = err
return outcome
}
defer response.Body.Close()
_, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 4096))
switch {
case response.StatusCode == http.StatusTooManyRequests:
// Reachable but at capacity: liveness stays up, readiness blocks.
outcome.Reachability = errCapacityFull
case response.StatusCode == http.StatusForbidden || response.StatusCode == http.StatusUnauthorized:
outcome.Reachability = errNoPermission
case response.StatusCode < 200 || response.StatusCode >= 300:
outcome.Reachability = fmt.Errorf("健康端点返回HTTP %d", response.StatusCode)
}
return outcome
}
// workerProbe reads queue-side worker heartbeats: no fresh heartbeat row means
// the worker loop is not touching the queue (liveness down), while the queue
// itself can still be ready for a future worker.
func (provider *RealProvider) workerProbe(ctx context.Context, now time.Time) []governance.ServiceHealth {
rows, err := provider.db.Query(ctx, `SELECT worker_id, last_seen_at FROM worker_heartbeats WHERE last_seen_at >= $1::timestamptz ORDER BY worker_id`, now.Add(-provider.workerStaleAfter))
if err != nil {
return []governance.ServiceHealth{provider.track(classify("worker", "解析Worker", true, true, ProbeOutcome{Reachability: err, ObservedAt: now}))}
}
defer rows.Close()
type entry struct {
id string
seen time.Time
}
live := []entry{}
for rows.Next() {
var id string
var seen time.Time
if err := rows.Scan(&id, &seen); err != nil {
return []governance.ServiceHealth{provider.track(classify("worker", "解析Worker", true, true, ProbeOutcome{Reachability: err, ObservedAt: now}))}
}
live = append(live, entry{id, seen.UTC()})
}
if len(live) == 0 {
return []governance.ServiceHealth{provider.track(classify("worker", "解析Worker", true, true, ProbeOutcome{Reachability: errors.New("队列无新鲜心跳(worker未运行或全部停跳)"), ObservedAt: now}))}
}
out := make([]governance.ServiceHealth, 0, len(live))
for _, item := range live {
outcome := ProbeOutcome{ObservedAt: item.seen}
if now.Sub(item.seen) > provider.workerStaleAfter {
outcome.Reachability = errors.New("心跳过期")
}
out = append(out, provider.track(classify("worker:"+item.id, "解析Worker "+item.id, true, true, outcome)))
}
return out
}
func (provider *RealProvider) track(health governance.ServiceHealth) governance.ServiceHealth {
return provider.health.apply(health.Key, health)
}
// Services reports the independent Liveness/Readiness matrix for every service
// the deployment actually has (P10). Missing services are explicit states.
func (provider *RealProvider) Services(ctx context.Context) []governance.ServiceHealth {
now := provider.now().UTC()
health := make([]governance.ServiceHealth, 0, 8)
// PostgreSQL: liveness = ping, readiness = queryable.
pgOutcome := ProbeOutcome{ObservedAt: now}
started := time.Now()
if err := provider.db.Ping(ctx); err != nil {
pgOutcome.Reachability = err
} else {
pgOutcome.Latency = time.Since(started)
var ok int
if err := provider.db.QueryRow(ctx, "SELECT 1").Scan(&ok); err != nil {
pgOutcome.WorkReadiness = err
}
}
health = append(health, provider.track(classify("postgres", "PostgreSQL", true, true, pgOutcome)))
// Queue readiness rides on the jobs table.
queueOutcome := ProbeOutcome{ObservedAt: now}
var pending int
if err := provider.db.QueryRow(ctx, "SELECT COUNT(*) FROM jobs WHERE state IN ('pending','retry')").Scan(&pending); err != nil {
queueOutcome.Reachability = err
}
health = append(health, provider.track(classify("queue", "任务队列", true, true, queueOutcome)))
health = append(health, provider.workerProbe(ctx, now)...)
health = append(health, provider.weKnoraProbe(ctx, now))
health = append(health, provider.httpProbe(ctx, "mineru", "MinerU", provider.minerUBaseURL, now))
// Not-applicable dependencies stay explicit (embedding/llm managed by
// WeKnora, redis/mq/oss not deployed).
health = append(health,
provider.track(classify("embedding", "Embedding模型", false, false, ProbeOutcome{ObservedAt: now})),
provider.track(classify("llm", "LLM模型", false, false, ProbeOutcome{ObservedAt: now})),
provider.track(classify("redis", "Redis / MQ", false, false, ProbeOutcome{ObservedAt: now})),
provider.track(classify("oss", "OSS", false, false, ProbeOutcome{ObservedAt: now})),
)
sort.Slice(health, func(i, j int) bool { return health[i].Key < health[j].Key })
return health
}
// Readiness blocks only on services that ARE deployed and configured and
// currently fail readiness; not_configured is surfaced as a warning elsewhere.
func overallReadiness(health []governance.ServiceHealth) (bool, []string) {
blocked := []string{}
for _, item := range health {
if !item.Applicable || !item.Configured {
continue
}
if item.Readiness != ReadReady {
blocked = append(blocked, item.Key)
}
}
return len(blocked) == 0, blocked
}
func (tracker *healthTracker) snapshot() []governance.ServiceHealth {
keys := make([]string, 0, len(tracker.records))
for key := range tracker.records {
keys = append(keys, key)
}
sort.Strings(keys)
records := make([]governance.ServiceHealth, 0, len(keys))
for _, key := range keys {
record := tracker.records[key]
item := governance.ServiceHealth{Key: key, LastSuccessAt: record.lastSuccess, ConsecutiveFailures: record.failStreak}
records = append(records, item)
}
return records
}
+84
View File
@@ -0,0 +1,84 @@
package monitoring
import (
"context"
"errors"
"fmt"
"net"
"strings"
"testing"
"time"
)
func testNow() time.Time { return time.Date(2026, 9, 12, 9, 0, 0, 0, time.UTC) }
func TestClassifySeparatesNotApplicableNotConfiguredAndFailures(t *testing.T) {
now := testNow()
notDeployed := classify("redis", "Redis / MQ", false, false, ProbeOutcome{ObservedAt: now})
if notDeployed.Liveness != LiveUnknown || notDeployed.Readiness != ReadNotApplicable {
t.Fatalf("not deployed = %+v", notDeployed)
}
notConfigured := classify("mineru", "MinerU", false, true, ProbeOutcome{ObservedAt: now})
if notConfigured.Liveness != LiveUnknown || notConfigured.Readiness != ReadNotConfigured {
t.Fatalf("not configured = %+v", notConfigured)
}
healthy := classify("postgres", "PostgreSQL", true, true, ProbeOutcome{ObservedAt: now})
if healthy.Liveness != LiveUp || healthy.Readiness != ReadReady {
t.Fatalf("healthy = %+v", healthy)
}
// DNS failure: down + blocked with sanitized reason, never the raw host.
dns := &net.DNSError{Err: "no such host", Name: "weknora.internal.example"}
down := classify("weknora", "WeKnora", true, true, ProbeOutcome{ObservedAt: now, Reachability: dns})
if down.Liveness != LiveDown || down.Reason == "" || strings.Contains(down.Reason, "weknora.internal") {
t.Fatalf("dns down = %+v", down)
}
// Timeout.
timeout := classify("weknora", "WeKnora", true, true, ProbeOutcome{ObservedAt: now, Reachability: context.DeadlineExceeded})
if timeout.Liveness != LiveDown || timeout.Readiness != ReadBlocked {
t.Fatalf("timeout = %+v", timeout)
}
// Reachable but capacity full => live up, readiness blocked, distinct reason.
capacity := classify("mineru", "MinerU", true, true, ProbeOutcome{ObservedAt: now, Reachability: errCapacityFull})
if capacity.Liveness != LiveUp || capacity.Readiness != ReadBlocked || capacity.Reason == "" {
t.Fatalf("capacity = %+v", capacity)
}
// Reachable but permission denied on target KB (readiness error only).
perm := classify("weknora", "WeKnora", true, true, ProbeOutcome{ObservedAt: now, WorkReadiness: errNoPermission})
if perm.Liveness != LiveUp || perm.Readiness != ReadBlocked {
t.Fatalf("permission = %+v", perm)
}
}
func TestHealthTrackerRemembersSuccessAndFailStreak(t *testing.T) {
tracker := newHealthTracker()
now := testNow()
first := tracker.apply("postgres", classify("postgres", "PostgreSQL", true, true, ProbeOutcome{ObservedAt: now}))
if first.LastSuccessAt == nil || !first.LastSuccessAt.Equal(now) || first.ConsecutiveFailures != 0 {
t.Fatalf("first = %+v", first)
}
later := now.Add(time.Minute)
bad := tracker.apply("postgres", classify("postgres", "PostgreSQL", true, true, ProbeOutcome{ObservedAt: later, Reachability: context.DeadlineExceeded}))
if !bad.LastSuccessAt.Equal(now) || bad.ConsecutiveFailures != 1 {
t.Fatalf("after one failure = %+v", bad)
}
bad2 := tracker.apply("postgres", classify("postgres", "PostgreSQL", true, true, ProbeOutcome{ObservedAt: bad.ObservedAt.Add(time.Minute), Reachability: context.DeadlineExceeded}))
if bad2.ConsecutiveFailures != 2 {
t.Fatalf("streak = %d", bad2.ConsecutiveFailures)
}
good := tracker.apply("postgres", classify("postgres", "PostgreSQL", true, true, ProbeOutcome{ObservedAt: bad2.ObservedAt.Add(time.Minute)}))
if good.ConsecutiveFailures != 0 || !good.LastSuccessAt.Equal(bad2.ObservedAt.Add(time.Minute)) {
t.Fatalf("recovered = %+v", good)
}
// Unknown probes (not configured) must not reset or extend a real streak.
neutral := tracker.apply("postgres", classify("postgres", "PostgreSQL", false, true, ProbeOutcome{ObservedAt: good.ObservedAt.Add(time.Minute)}))
if neutral.ConsecutiveFailures != 0 {
t.Fatalf("neutral streak = %d", neutral.ConsecutiveFailures)
}
}
func TestSanitizeProbeErrorTruncatesLongMessages(t *testing.T) {
long := errors.New(fmt.Sprintf("%0200d", 1))
if got := sanitizeProbeError(long); len(got) > 160 {
t.Fatalf("len=%d", len(got))
}
}
+171
View File
@@ -0,0 +1,171 @@
package monitoring
import (
"context"
"fmt"
"math"
"sort"
"time"
"knowledge-governance/internal/governance"
)
type windowJobOutcome struct {
state string
durationSeconds float64 // <0 when duration is unknowable
}
const (
metricsShortWindow = time.Hour
metricsLongWindow = 24 * time.Hour
metricsWorkerLease = 90 * time.Second // workers heartbeat every <= lease; fresher than stale leases counts active
metricsSampleMargin = 2 * time.Second
)
func computeJobWindowMetrics(outcomes []windowJobOutcome, windowSeconds int) governance.JobWindowMetrics {
metrics := governance.JobWindowMetrics{WindowSeconds: windowSeconds}
var durations []float64
for _, outcome := range outcomes {
switch outcome.state {
case "completed":
metrics.JobsCompleted++
case "failed":
metrics.JobsFailed++
}
if outcome.durationSeconds >= 0 {
durations = append(durations, outcome.durationSeconds)
}
}
if finished := metrics.JobsCompleted + metrics.JobsFailed; finished > 0 {
success := math.Round(float64(metrics.JobsCompleted)/float64(finished)*10000) / 100
failure := math.Round(float64(metrics.JobsFailed)/float64(finished)*10000) / 100
metrics.SuccessRate = &success
metrics.FailureRate = &failure
}
if len(durations) > 0 {
sort.Float64s(durations)
var sum float64
for _, value := range durations {
sum += value
}
average := math.Round(sum/float64(len(durations))*10) / 10
p50 := math.Round(percentile(durations, 0.50)*10) / 10
p95 := math.Round(percentile(durations, 0.95)*10) / 10
metrics.DurationAvgSeconds = &average
metrics.DurationP50Seconds = &p50
metrics.DurationP95Seconds = &p95
}
return metrics
}
// percentile uses nearest-rank so tiny samples never interpolate a fake value.
func percentile(sorted []float64, fraction float64) float64 {
rank := int(math.Ceil(fraction * float64(len(sorted))))
if rank < 1 {
rank = 1
}
if rank > len(sorted) {
rank = len(sorted)
}
return sorted[rank-1]
}
func (provider *RealProvider) jobMetrics(ctx context.Context, window time.Duration) (governance.JobWindowMetrics, error) {
windowSeconds := int(window.Seconds())
var metrics governance.JobWindowMetrics
// Pull the per-outcome detail (bounded) for exact-rate and percentile math.
// Duration is only measurable when the whole lifecycle lives inside the
// window; older-created jobs report -1 so percentiles never mix partials.
rows, err := provider.db.Query(ctx, `
SELECT state,
CASE WHEN created_at > $2::timestamptz THEN COALESCE(EXTRACT(EPOCH FROM (updated_at - created_at)), -1) ELSE -1 END
FROM jobs
WHERE updated_at > $2::timestamptz AND updated_at <= $1::timestamptz
AND state IN ('completed','failed')
ORDER BY updated_at DESC
LIMIT 10000`, provider.now(), provider.now().Add(-window))
if err != nil {
return metrics, fmt.Errorf("query job window outcomes: %w", err)
}
defer rows.Close()
var outcomes []windowJobOutcome
for rows.Next() {
var outcome windowJobOutcome
if err := rows.Scan(&outcome.state, &outcome.durationSeconds); err != nil {
return metrics, fmt.Errorf("scan job window outcome: %w", err)
}
outcomes = append(outcomes, outcome)
}
if err := rows.Err(); err != nil {
return metrics, fmt.Errorf("iterate job window outcomes: %w", err)
}
metrics = computeJobWindowMetrics(outcomes, windowSeconds)
// Started inside the window uses created_at (any state).
var started int
if err := provider.db.QueryRow(ctx, `SELECT COUNT(*)::integer FROM jobs WHERE created_at > $2::timestamptz AND created_at <= $1::timestamptz`, provider.now(), provider.now().Add(-window)).Scan(&started); err != nil {
return metrics, fmt.Errorf("count jobs started: %w", err)
}
metrics.JobsStarted = started
return metrics, nil
}
func (provider *RealProvider) queueGauges(ctx context.Context) (governance.QueueGauges, error) {
var gauges governance.QueueGauges
var oldestPendingSeconds *float64
err := provider.db.QueryRow(ctx, `
SELECT COUNT(*) FILTER (WHERE state='pending')::integer,
COUNT(*) FILTER (WHERE state='running')::integer,
COUNT(*) FILTER (WHERE state='retry')::integer,
COUNT(*) FILTER (WHERE state='failed')::integer,
CASE WHEN COUNT(*) FILTER (WHERE state IN ('pending','retry')) = 0 THEN NULL
ELSE GREATEST(EXTRACT(EPOCH FROM ($1::timestamptz - MIN(created_at) FILTER (WHERE state IN ('pending','retry')))), 0) END
FROM jobs`, provider.now()).Scan(&gauges.Pending, &gauges.Running, &gauges.Retry, &gauges.Failed, &oldestPendingSeconds)
if err != nil {
return gauges, fmt.Errorf("scan queue gauges: %w", err)
}
gauges.OldestPendingAgeSeconds = oldestPendingSeconds
return gauges, nil
}
func (provider *RealProvider) activeWorkerCount(ctx context.Context) (int, error) {
// Count distinct fresh leases only; never infer from container counts.
var active int
err := provider.db.QueryRow(ctx, `
SELECT COUNT(DISTINCT lease_owner)::integer
FROM jobs
WHERE state = 'running' AND lease_until > $1::timestamptz AND lease_owner IS NOT NULL`,
provider.now().Add(-metricsWorkerLease)).Scan(&active)
if err != nil {
return 0, fmt.Errorf("count active workers: %w", err)
}
return active, nil
}
// Metrics implements the P9 run-metrics endpoint: frozen 1h/24h windows,
// lease-based active workers, and null-safe samples.
func (provider *RealProvider) Metrics(ctx context.Context) (governance.RunMetrics, error) {
now := provider.now().UTC()
metrics := governance.RunMetrics{ObservedAt: now, WorkerSource: "pg_lease"}
short, err := provider.jobMetrics(ctx, metricsShortWindow)
if err != nil {
return metrics, err
}
longWindow, err := provider.jobMetrics(ctx, metricsLongWindow)
if err != nil {
return metrics, err
}
gauges, err := provider.queueGauges(ctx)
if err != nil {
return metrics, err
}
active, err := provider.activeWorkerCount(ctx)
if err != nil {
return metrics, err
}
metrics.ShortWindow = short
metrics.LongWindow = longWindow
metrics.Queue = gauges
metrics.ActiveWorkers = active
return metrics, nil
}
+71
View File
@@ -0,0 +1,71 @@
package monitoring
import (
"encoding/json"
"testing"
)
func TestComputeJobWindowMetricsRates(t *testing.T) {
// Mixed completed/failed inside the window: rates must be exact.
m := computeJobWindowMetrics([]windowJobOutcome{
{state: "completed", durationSeconds: 10},
{state: "completed", durationSeconds: 20},
{state: "completed", durationSeconds: 30},
{state: "failed", durationSeconds: 40},
}, 60)
if m.WindowSeconds != 60 || m.JobsCompleted != 3 || m.JobsFailed != 1 {
t.Fatalf("counts = %+v", m)
}
if m.SuccessRate == nil || *m.SuccessRate != 75 {
t.Fatalf("success rate = %v, want 75", m.SuccessRate)
}
if m.FailureRate == nil || *m.FailureRate != 25 {
t.Fatalf("failure rate = %v, want 25", m.FailureRate)
}
// Durations: avg 25s, p50 20s (nearest-rank on 4 sorted samples 10/20/30/40).
if m.DurationAvgSeconds == nil || *m.DurationAvgSeconds != 25 {
t.Fatalf("avg = %v, want 25", m.DurationAvgSeconds)
}
if m.DurationP50Seconds == nil || *m.DurationP50Seconds != 20 {
t.Fatalf("p50 = %v, want 20", m.DurationP50Seconds)
}
if m.DurationP95Seconds == nil || *m.DurationP95Seconds != 40 {
t.Fatalf("p95 = %v, want 40", m.DurationP95Seconds)
}
}
func TestComputeJobWindowMetricsNullsWhenNoSamples(t *testing.T) {
// Insufficient data must be null, never a fabricated 0%.
m := computeJobWindowMetrics([]windowJobOutcome{{state: "completed", durationSeconds: -1}}, 3600)
if m.JobsCompleted != 1 {
t.Fatalf("completed = %d", m.JobsCompleted)
}
if m.SuccessRate == nil || *m.SuccessRate != 100 {
t.Fatalf("success rate = %v, want 100 (1/1)", m.SuccessRate)
}
if m.DurationAvgSeconds != nil || m.DurationP50Seconds != nil || m.DurationP95Seconds != nil {
t.Fatalf("durations must be null without terminal-time samples: %+v", m)
}
empty := computeJobWindowMetrics(nil, 3600)
if empty.SuccessRate != nil || empty.FailureRate != nil || empty.JobsStarted != 0 {
t.Fatalf("empty window = %+v, want all-null rates", empty)
}
blob, err := json.Marshal(empty)
if err != nil {
t.Fatal(err)
}
for _, key := range []string{`"success_rate":null`, `"failure_rate":null`, `"duration_avg_seconds":null`} {
if !contains(string(blob), key) {
t.Fatalf("metrics JSON missing %s: %s", key, blob)
}
}
}
func contains(haystack, needle string) bool {
for i := 0; i+len(needle) <= len(haystack); i++ {
if haystack[i:i+len(needle)] == needle {
return true
}
}
return false
}
@@ -0,0 +1,116 @@
package monitoring
import (
"context"
"net/http"
"strings"
"testing"
"time"
"github.com/jackc/pgx/v5/pgxpool"
"knowledge-governance/internal/platform/postgres"
)
// TestRealServicesHealthMatrix freezes docs plan P10: liveness and readiness
// are per-service and independent, worker liveness rides on queue heartbeats,
// and missing configuration is an explicit state — never "unknown".
func TestRealServicesHealthMatrix(t *testing.T) {
databaseURL := isolatedMetricsURL(t)
pool, err := pgxpool.New(context.Background(), databaseURL)
if err != nil {
t.Fatal(err)
}
defer pool.Close()
ctx := context.Background()
if err := postgres.ApplyMigrations(ctx, pool); err != nil {
t.Fatal(err)
}
now := time.Date(2026, time.September, 12, 12, 0, 0, 0, time.UTC)
provider, err := NewRealProvider(RealProviderConfig{
DB: metricsPool{pool},
Now: func() time.Time { return now },
HTTPClient: &http.Client{Timeout: time.Second},
WorkerStaleAfter: 2 * time.Minute,
})
if err != nil {
t.Fatal(err)
}
// Before any worker claims: queue/postgres healthy, worker aggregated down.
health := provider.Services(ctx)
byKey := map[string]string{}
for _, item := range health {
byKey[item.Key+"/l"] = item.Liveness
byKey[item.Key+"/r"] = item.Readiness
}
if byKey["postgres/l"] != LiveUp || byKey["postgres/r"] != ReadReady {
t.Fatalf("postgres health = %v/%v", byKey["postgres/l"], byKey["postgres/r"])
}
if byKey["worker/l"] != LiveDown || byKey["worker/r"] != ReadBlocked {
t.Fatalf("worker health with no heartbeat = %v/%v", byKey["worker/l"], byKey["worker/r"])
}
if byKey["weknora/r"] != ReadNotConfigured || byKey["mineru/r"] != ReadNotConfigured {
t.Fatalf("unconfigured deps must be explicit: %v %v", byKey["weknora/r"], byKey["mineru/r"])
}
for _, key := range []string{"redis", "oss", "embedding", "llm"} {
if byKey[key+"/r"] != ReadNotApplicable || byKey[key+"/l"] != LiveUnknown {
t.Fatalf("not-deployed %s = %v/%v", key, byKey[key+"/l"], byKey[key+"/r"])
}
}
// Readiness must be blocked ONLY by configured+deployed failures: worker.
ok, blocked := overallReadiness(health)
if ok || len(blocked) != 1 || blocked[0] != "worker" {
t.Fatalf("overall readiness ok=%v blocked=%v", ok, blocked)
}
// A worker claim refreshes the heartbeat row and flips liveness independently.
store := postgres.NewStore(pool)
if _, err := store.Claim(ctx, "kg-test-worker", []string{"test.upload"}, now, time.Minute); err != nil {
t.Fatal(err)
}
health = provider.Services(ctx)
found := false
for _, item := range health {
if strings.HasPrefix(item.Key, "worker:") {
found = true
if item.Liveness != LiveUp || item.Readiness != ReadReady {
t.Fatalf("fresh worker health = %v/%v", item.Liveness, item.Readiness)
}
}
if item.Key == "worker" {
t.Fatalf("stale aggregate row must disappear once a worker is live")
}
}
if !found {
t.Fatal("live worker row missing after claim")
}
// Heartbeat older than the stale window: liveness down, and the API-side
// process itself stays independent (liveness of the API is never blocked).
if _, err := pool.Exec(ctx, `UPDATE worker_heartbeats SET last_seen_at = $1`, now.Add(-10*time.Minute)); err != nil {
t.Fatal(err)
}
health = provider.Services(ctx)
staleDown := false
for _, item := range health {
if (item.Key == "worker" || strings.HasPrefix(item.Key, "worker:")) && item.Liveness == LiveDown {
staleDown = true
}
}
if !staleDown {
t.Fatal("stale heartbeat must mark worker down")
}
if ok, _ := overallReadiness(health); ok {
t.Fatal("stale worker must block readiness")
}
// Tracker history: postgres was seen successful and keeps no failures.
for _, item := range health {
if item.Key == "postgres" {
if item.LastSuccessAt == nil || item.ConsecutiveFailures != 0 {
t.Fatalf("postgres tracker history = %v failures=%d", item.LastSuccessAt, item.ConsecutiveFailures)
}
}
}
}
@@ -0,0 +1,190 @@
package monitoring
import (
"context"
"crypto/sha256"
"encoding/hex"
"fmt"
"net/url"
"os"
"testing"
"time"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgconn"
"github.com/jackc/pgx/v5/pgxpool"
"knowledge-governance/internal/platform/postgres"
)
type metricsPool struct{ pool *pgxpool.Pool }
func (d metricsPool) Ping(ctx context.Context) error { return d.pool.Ping(ctx) }
func (d metricsPool) Query(ctx context.Context, sql string, args ...any) (pgx.Rows, error) {
return d.pool.Query(ctx, sql, args...)
}
func (d metricsPool) QueryRow(ctx context.Context, sql string, args ...any) pgx.Row {
return d.pool.QueryRow(ctx, sql, args...)
}
func (d metricsPool) Exec(ctx context.Context, sql string, args ...any) (pgconn.CommandTag, error) {
return d.pool.Exec(ctx, sql, args...)
}
func isolatedMetricsURL(t *testing.T) string {
t.Helper()
databaseURL := os.Getenv("TEST_DATABASE_URL")
if databaseURL == "" {
t.Skip("TEST_DATABASE_URL is not set")
}
parsed, err := url.Parse(databaseURL)
if err != nil {
t.Fatal(err)
}
digest := sha256.Sum256([]byte(t.Name()))
schema := "kgtest_" + hex.EncodeToString(digest[:8])
ctx := context.Background()
admin, err := pgxpool.New(ctx, databaseURL)
if err != nil {
t.Fatal(err)
}
defer admin.Close()
if _, err := admin.Exec(ctx, "CREATE SCHEMA "+schema); err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
cleanup, err := pgxpool.New(context.Background(), databaseURL)
if err == nil {
_, _ = cleanup.Exec(context.Background(), "DROP SCHEMA IF EXISTS "+schema+" CASCADE")
cleanup.Close()
}
})
query := parsed.Query()
query.Set("search_path", schema)
parsed.RawQuery = query.Encode()
return parsed.String()
}
// TestRealMetricsWindowsAndLeases freezes docs plan P9: rates and percentiles
// come from frozen 1h/24h windows, active workers count fresh distinct leases
// only, jobs created outside the window never fake a duration, and empty
// windows return nulls instead of 0.
func TestRealMetricsWindowsAndLeases(t *testing.T) {
ctx := context.Background()
pool, err := pgxpool.New(ctx, isolatedMetricsURL(t))
if err != nil {
t.Fatal(err)
}
defer pool.Close()
if err := postgres.ApplyMigrations(ctx, pool); err != nil {
t.Fatal(err)
}
now := time.Date(2026, time.September, 12, 8, 0, 0, 0, time.UTC)
seed := func(id, state string, createdAt, updatedAt time.Time, owner string, leaseUntil *time.Time, attempt int) {
var lease any
if leaseUntil != nil {
lease = *leaseUntil
}
var ownerArg any
if owner != "" {
ownerArg = owner
}
_, err := pool.Exec(ctx, `INSERT INTO jobs(id,type,payload,state,available_at,lease_owner,lease_until,attempt,max_attempts,idempotency_key,created_at,updated_at)
VALUES($1::varchar,'test.upload','{}',$2,$4,$5,$6,$7,5,'k-'||$1::text,$3,$4)`, id, state, createdAt, updatedAt, ownerArg, lease, attempt)
if err != nil {
t.Fatalf("seed %s: %v", id, err)
}
}
// Completed inside both windows with exact durations.
for i, d := range []time.Duration{10 * time.Second, 20 * time.Second, 30 * time.Second, 15 * time.Minute, 30 * time.Minute} {
updated := now.Add(-time.Duration(6-i) * time.Minute)
seed(fmt.Sprintf("job-c%d", i), "completed", updated.Add(-d), updated, "", nil, 1)
}
// Failed: two inside 1h, one inside 24h only, one outside 24h.
seed("job-f1", "failed", now.Add(-20*time.Minute), now.Add(-19*time.Minute), "w1", nil, 5)
seed("job-f2", "failed", now.Add(-10*time.Minute), now.Add(-9*time.Minute), "w1", nil, 5)
seed("job-f3", "failed", now.Add(-20*time.Hour), now.Add(-19*time.Hour), "w1", nil, 5)
seed("job-f4", "failed", now.Add(-40*time.Hour), now.Add(-39*time.Hour), "w1", nil, 5)
// Completed now but created 30h ago: duration unknowable, must not pollute percentiles.
seed("job-old", "completed", now.Add(-30*time.Hour), now.Add(-10*time.Minute), "", nil, 1)
// Fresh leases: two rows share wA, one wB, stale wC must not count.
seed("job-run-a1", "running", now.Add(-5*time.Minute), now.Add(-time.Minute), "wA", ptrTime(now.Add(time.Minute)), 1)
seed("job-run-a2", "running", now.Add(-4*time.Minute), now.Add(-time.Minute), "wA", ptrTime(now.Add(time.Minute)), 1)
seed("job-run-b", "running", now.Add(-3*time.Minute), now.Add(-time.Minute), "wB", ptrTime(now.Add(time.Minute)), 1)
seed("job-run-stale", "running", now.Add(-9*time.Hour), now.Add(-60*time.Minute), "wC", ptrTime(now.Add(-10*time.Minute)), 1)
// Gauges.
seed("job-pending", "pending", now.Add(-3*time.Hour), now.Add(-3*time.Hour), "", nil, 0)
seed("job-retry", "retry", now.Add(-30*time.Minute), now.Add(-25*time.Minute), "", nil, 1)
provider, err := NewRealProvider(RealProviderConfig{
DB: metricsPool{pool: pool},
Now: func() time.Time { return now },
})
if err != nil {
t.Fatal(err)
}
metrics, err := provider.Metrics(ctx)
if err != nil {
t.Fatalf("metrics: %v", err)
}
if !metrics.ObservedAt.Equal(now) || metrics.WorkerSource != "pg_lease" {
t.Fatalf("observed/source = %s/%s", metrics.ObservedAt, metrics.WorkerSource)
}
if metrics.ActiveWorkers != 2 {
t.Fatalf("active workers = %d, want 2 (fresh distinct owners only)", metrics.ActiveWorkers)
}
if metrics.Queue.Pending != 1 || metrics.Queue.Running != 4 || metrics.Queue.Retry != 1 || metrics.Queue.Failed != 4 {
t.Fatalf("queue gauges = %+v", metrics.Queue)
}
if metrics.Queue.OldestPendingAgeSeconds == nil || int(*metrics.Queue.OldestPendingAgeSeconds) != 3*3600 {
t.Fatalf("oldest pending = %v, want 10800s", metrics.Queue.OldestPendingAgeSeconds)
}
short := metrics.ShortWindow
// completed inside 1h: 5 seeded + job-old + job-c* (all updated within 1h except none) -> 6; failed 2.
if short.WindowSeconds != 3600 || short.JobsCompleted != 6 || short.JobsFailed != 2 {
t.Fatalf("short counts = %+v, want 6/2", short)
}
if short.SuccessRate == nil || *short.SuccessRate != 75 || short.FailureRate == nil || *short.FailureRate != 25 {
t.Fatalf("short rates = %v/%v, want 75/25", short.SuccessRate, short.FailureRate)
}
// Durations measurable inside 1h: 10,20,30,900,1800 (job-old excluded).
if short.DurationAvgSeconds == nil || *short.DurationAvgSeconds != 411.4 {
t.Fatalf("short avg = %v, want 411.4(含2条60s失败样本)", *short.DurationAvgSeconds)
}
if short.DurationP95Seconds == nil || *short.DurationP95Seconds != 1800 {
t.Fatalf("short p95 = %v, want 1800", *short.DurationP95Seconds)
}
long := metrics.LongWindow
if long.JobsFailed != 3 {
t.Fatalf("long failures = %d, want 3 (job-f4 outside window)", long.JobsFailed)
}
// 8 measurable samples (5 completed + 2 failed@60s + job-f3 3600s): p50=60, p95=3600.
if long.DurationP50Seconds == nil || *long.DurationP50Seconds != 60 {
t.Fatalf("long p50 = %v, want 60", *long.DurationP50Seconds)
}
if long.DurationP95Seconds == nil || *long.DurationP95Seconds != 3600 {
t.Fatalf("long p95 = %v, want 3600", *long.DurationP95Seconds)
}
// Empty-window honesty: a fresh schema with no rows yields null rates.
empty, err := NewRealProvider(RealProviderConfig{DB: metricsPool{pool: pool}, Now: func() time.Time { return now.Add(-100 * time.Hour) }})
if err != nil {
t.Fatal(err)
}
idle, err := empty.Metrics(ctx)
if err != nil {
t.Fatalf("idle metrics: %v", err)
}
if idle.ShortWindow.SuccessRate != nil || idle.ShortWindow.DurationAvgSeconds != nil {
t.Fatalf("empty window must be null, got %+v", idle.ShortWindow)
}
if idle.Queue.OldestPendingAgeSeconds == nil {
t.Fatalf("oldest pending must remain observable: %+v", idle.Queue)
}
}
func ptrTime(value time.Time) *time.Time { return &value }
+320
View File
@@ -0,0 +1,320 @@
package monitoring
import (
"context"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"os"
"strconv"
"strings"
"syscall"
"time"
"github.com/jackc/pgx/v5"
"knowledge-governance/internal/governance"
)
type Database interface {
Ping(context.Context) error
Query(context.Context, string, ...any) (pgx.Rows, error)
QueryRow(context.Context, string, ...any) pgx.Row
}
type RealProviderConfig struct {
DB Database
Now func() time.Time
HTTPClient *http.Client
WeKnoraBaseURL string
MinerUBaseURL string
// WeKnoraReadiness optionally exercises a real read against a routing
// knowledge base (nil = API key not configured; liveness-only).
WeKnoraReadiness func(ctx context.Context) error
// WorkerStaleAfter bounds how long a missing worker heartbeat still counts
// as alive (P10).
WorkerStaleAfter time.Duration
}
type RealProvider struct {
db Database
now func() time.Time
httpClient *http.Client
weknoraBaseURL string
minerUBaseURL string
weknoraReadiness func(ctx context.Context) error
workerStaleAfter time.Duration
health *healthTracker
}
func NewRealProvider(config RealProviderConfig) (*RealProvider, error) {
if config.DB == nil {
return nil, errors.New("monitor database is required")
}
if config.Now == nil {
config.Now = time.Now
}
if config.HTTPClient == nil {
config.HTTPClient = &http.Client{Timeout: 3 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}
}
stale := config.WorkerStaleAfter
if stale <= 0 {
stale = time.Minute
}
return &RealProvider{db: config.DB, now: config.Now, httpClient: config.HTTPClient, weknoraBaseURL: strings.TrimSpace(config.WeKnoraBaseURL), minerUBaseURL: strings.TrimSpace(config.MinerUBaseURL), weknoraReadiness: config.WeKnoraReadiness, workerStaleAfter: stale, health: newHealthTracker()}, nil
}
func (provider *RealProvider) Overview(ctx context.Context) (governance.MonitorOverview, error) {
now := provider.now().UTC()
overview := governance.MonitorOverview{Mode: "real", Simulated: false, ObservedAt: now, Services: []governance.MonitorService{}, KnowledgeBases: []governance.KnowledgeBaseProgress{}}
overview.Host = collectHostMetrics()
overview.Services = append(overview.Services, provider.postgresHealth(ctx, now), provider.queueHealth(ctx, now))
overview.Health = provider.Services(ctx)
overview.Services = append(overview.Services,
provider.httpHealth(ctx, "weknora", "WeKnora API", provider.weknoraBaseURL, now),
governance.MonitorService{Key: "embedding", Name: "Embedding模型", State: "unknown", Detail: "由WeKnora管理,尚无独立探测端点", ObservedAt: now},
governance.MonitorService{Key: "llm", Name: "LLM模型", State: "unknown", Detail: "由WeKnora管理,尚无独立探测端点", ObservedAt: now},
provider.httpHealth(ctx, "mineru", "MinerU", provider.minerUBaseURL, now),
governance.MonitorService{Key: "redis", Name: "Redis / MQ", State: "unknown", Detail: "未部署;任务由PostgreSQL持久队列承载", ObservedAt: now},
governance.MonitorService{Key: "oss", Name: "OSS", State: "unknown", Detail: "未配置独立健康探测", ObservedAt: now},
)
rows, err := provider.db.Query(ctx, `SELECT p.test_kb_id,
CASE WHEN COUNT(DISTINCT p.target_scope)=1 THEN MIN(p.target_scope) ELSE 'mixed' END,
COUNT(*)::integer,
COUNT(*) FILTER (WHERE p.processing_state='completed')::integer,
COUNT(*) FILTER (WHERE p.processing_state='pending')::integer,
COUNT(*) FILTER (WHERE p.processing_state='processing')::integer,
COUNT(*) FILTER (WHERE p.processing_state='finalizing')::integer,
COUNT(*) FILTER (WHERE p.processing_state IN ('failed','stalled'))::integer,
COUNT(*) FILTER (WHERE p.target_scope='internal')::integer,
COUNT(*) FILTER (WHERE p.target_scope='internal' AND p.processing_state='completed')::integer,
COUNT(*) FILTER (WHERE p.target_scope='external')::integer,
COUNT(*) FILTER (WHERE p.target_scope='external' AND p.processing_state='completed')::integer
FROM test_publications p
GROUP BY p.test_kb_id
ORDER BY p.test_kb_id`)
if err != nil {
return governance.MonitorOverview{}, fmt.Errorf("list monitored knowledge bases: %w", err)
}
defer rows.Close()
for rows.Next() {
var item governance.KnowledgeBaseProgress
if err := rows.Scan(&item.ID, &item.Visibility, &item.Imported, &item.Completed, &item.Pending, &item.Processing, &item.Finalizing, &item.Failed, &item.InternalImported, &item.InternalCompleted, &item.ExternalImported, &item.ExternalCompleted); err != nil {
return governance.MonitorOverview{}, fmt.Errorf("scan monitored knowledge base: %w", err)
}
switch item.Visibility {
case "internal":
item.Name = "对内知识库"
case "external":
item.Name = "对外知识库"
default:
item.Name = "测试知识库"
}
overview.KnowledgeBases = append(overview.KnowledgeBases, item)
}
if err := rows.Err(); err != nil {
return governance.MonitorOverview{}, fmt.Errorf("iterate monitored knowledge bases: %w", err)
}
return overview, nil
}
func (provider *RealProvider) ListFiles(ctx context.Context, knowledgeBaseID, status string, limit, offset int) (governance.KnowledgeMonitorFilePage, error) {
if strings.TrimSpace(knowledgeBaseID) == "" || limit < 1 || limit > 100 || offset < 0 || !validMonitorStatus(status) {
return governance.KnowledgeMonitorFilePage{}, errors.New("invalid monitor file query")
}
page := governance.KnowledgeMonitorFilePage{Items: []governance.KnowledgeMonitorFile{}, Limit: limit, Offset: offset, SampleOnly: false}
if err := provider.db.QueryRow(ctx, `SELECT COUNT(*)::integer FROM test_publications WHERE test_kb_id=$1 AND ($2='' OR CASE WHEN processing_state='stalled' THEN 'failed' ELSE processing_state END=$2)`, knowledgeBaseID, status).Scan(&page.Total); err != nil {
return governance.KnowledgeMonitorFilePage{}, fmt.Errorf("count monitored files: %w", err)
}
rows, err := provider.db.Query(ctx, `SELECT item.title,item.normalized_path,publication.target_scope,
CASE WHEN publication.processing_state='stalled' THEN 'failed' ELSE publication.processing_state END,
GREATEST(0,EXTRACT(EPOCH FROM ($5-COALESCE(publication.processing_started_at,publication.updated_at)))::integer),
publication.ingest_state,publication.last_error
FROM test_publications publication
JOIN source_item_versions version ON version.id=publication.source_item_version_id
JOIN source_items item ON item.id=version.source_item_id
WHERE publication.test_kb_id=$1
AND ($2='' OR CASE WHEN publication.processing_state='stalled' THEN 'failed' ELSE publication.processing_state END=$2)
ORDER BY CASE publication.processing_state WHEN 'failed' THEN 0 WHEN 'stalled' THEN 0 WHEN 'processing' THEN 1 WHEN 'finalizing' THEN 2 WHEN 'pending' THEN 3 ELSE 4 END,
publication.updated_at DESC,publication.id
LIMIT $3 OFFSET $4`, knowledgeBaseID, status, limit, offset, provider.now().UTC())
if err != nil {
return governance.KnowledgeMonitorFilePage{}, fmt.Errorf("list monitored files: %w", err)
}
defer rows.Close()
for rows.Next() {
var item governance.KnowledgeMonitorFile
var ingestState string
if err := rows.Scan(&item.Name, &item.Directory, &item.Visibility, &item.Status, &item.ElapsedSecond, &ingestState, &item.ErrorCode); err != nil {
return governance.KnowledgeMonitorFilePage{}, fmt.Errorf("scan monitored file: %w", err)
}
item.StageDetail = monitorStageDetail(ingestState, item.Status, item.ErrorCode)
page.Items = append(page.Items, item)
}
if err := rows.Err(); err != nil {
return governance.KnowledgeMonitorFilePage{}, fmt.Errorf("iterate monitored files: %w", err)
}
return page, nil
}
func (provider *RealProvider) postgresHealth(ctx context.Context, now time.Time) governance.MonitorService {
started := time.Now()
state, detail := "healthy", "可连接并可查询"
if err := provider.db.Ping(ctx); err != nil {
state, detail = "unhealthy", "连接或查询失败"
}
return governance.MonitorService{Key: "postgres", Name: "PostgreSQL", State: state, Detail: detail, LatencyMS: int(time.Since(started).Milliseconds()), ObservedAt: now}
}
func (provider *RealProvider) queueHealth(ctx context.Context, now time.Time) governance.MonitorService {
started := time.Now()
var pending, running, failed int
err := provider.db.QueryRow(ctx, `SELECT COUNT(*) FILTER (WHERE state IN ('pending','retry'))::integer,COUNT(*) FILTER (WHERE state='running')::integer,COUNT(*) FILTER (WHERE state='failed')::integer FROM jobs`).Scan(&pending, &running, &failed)
state, detail := "healthy", fmt.Sprintf("可接单;等待%d,运行%d,失败%d", pending, running, failed)
if err != nil {
state, detail = "unhealthy", "队列不可查询"
}
return governance.MonitorService{Key: "queue", Name: "任务队列", State: state, Detail: detail, LatencyMS: int(time.Since(started).Milliseconds()), ObservedAt: now}
}
func (provider *RealProvider) httpHealth(ctx context.Context, key, name, baseURL string, now time.Time) governance.MonitorService {
if baseURL == "" {
return governance.MonitorService{Key: key, Name: name, State: "unknown", Detail: "未配置探测地址", ObservedAt: now}
}
parsed, err := url.Parse(baseURL)
if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Host == "" {
return governance.MonitorService{Key: key, Name: name, State: "unhealthy", Detail: "探测配置无效", ObservedAt: now}
}
parsed.Path = strings.TrimSuffix(parsed.Path, "/") + "/health"
parsed.RawQuery, parsed.Fragment = "", ""
request, err := http.NewRequestWithContext(ctx, http.MethodGet, parsed.String(), nil)
if err != nil {
return governance.MonitorService{Key: key, Name: name, State: "unhealthy", Detail: "无法创建健康探测", ObservedAt: now}
}
started := time.Now()
response, err := provider.httpClient.Do(request)
latency := int(time.Since(started).Milliseconds())
if err != nil {
return governance.MonitorService{Key: key, Name: name, State: "unhealthy", Detail: "健康端点不可达", LatencyMS: latency, ObservedAt: now}
}
defer response.Body.Close()
_, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 4096))
if response.StatusCode < 200 || response.StatusCode >= 300 {
return governance.MonitorService{Key: key, Name: name, State: "degraded", Detail: fmt.Sprintf("健康端点返回HTTP %d", response.StatusCode), LatencyMS: latency, ObservedAt: now}
}
return governance.MonitorService{Key: key, Name: name, State: "healthy", Detail: "健康端点可用", LatencyMS: latency, ObservedAt: now}
}
func collectHostMetrics() governance.MonitorHost {
host := governance.MonitorHost{GPUState: "unknown", GPUDetail: "容器未暴露GPU遥测", Simulated: false}
first, err := os.ReadFile("/proc/stat")
if err == nil {
idle1, total1, parseErr := parseCPUStat(string(first))
if parseErr == nil {
time.Sleep(100 * time.Millisecond)
if second, readErr := os.ReadFile("/proc/stat"); readErr == nil {
idle2, total2, secondErr := parseCPUStat(string(second))
if secondErr == nil {
host.CPUPercent = cpuPercent(idle1, total1, idle2, total2)
}
}
}
}
if raw, err := os.ReadFile("/proc/meminfo"); err == nil {
host.MemoryUsedBytes, host.MemoryTotalBytes, _ = parseMemoryInfo(string(raw))
}
var stats syscall.Statfs_t
if syscall.Statfs("/", &stats) == nil {
host.DiskTotalBytes = int64(stats.Blocks) * int64(stats.Bsize)
host.DiskUsedBytes = int64(stats.Blocks-stats.Bavail) * int64(stats.Bsize)
}
if entries, err := os.ReadDir("/proc/driver/nvidia/gpus"); err == nil && len(entries) > 0 {
host.GPUState, host.GPUDetail = "healthy", fmt.Sprintf("检测到%d块NVIDIA GPU;未读取利用率", len(entries))
}
return host
}
func parseCPUStat(raw string) (idle, total uint64, err error) {
line := strings.SplitN(raw, "\n", 2)[0]
fields := strings.Fields(line)
if len(fields) < 5 || fields[0] != "cpu" {
return 0, 0, errors.New("invalid cpu stat")
}
values := make([]uint64, 0, len(fields)-1)
for _, field := range fields[1:] {
value, parseErr := strconv.ParseUint(field, 10, 64)
if parseErr != nil {
return 0, 0, errors.New("invalid cpu counter")
}
values = append(values, value)
total += value
}
idle = values[3]
if len(values) > 4 {
idle += values[4]
}
return idle, total, nil
}
func parseMemoryInfo(raw string) (used, total int64, err error) {
values := map[string]int64{}
for _, line := range strings.Split(raw, "\n") {
fields := strings.Fields(line)
if len(fields) < 2 {
continue
}
key := strings.TrimSuffix(fields[0], ":")
if key != "MemTotal" && key != "MemAvailable" {
continue
}
value, parseErr := strconv.ParseInt(fields[1], 10, 64)
if parseErr != nil || value < 0 {
return 0, 0, errors.New("invalid memory counter")
}
values[key] = value * 1024
}
total, totalOK := values["MemTotal"]
available, availableOK := values["MemAvailable"]
if !totalOK || !availableOK || available > total {
return 0, 0, errors.New("incomplete memory info")
}
return total - available, total, nil
}
func cpuPercent(idle1, total1, idle2, total2 uint64) float64 {
if total2 <= total1 || idle2 < idle1 || idle2-idle1 > total2-total1 {
return 0
}
return float64((total2-total1)-(idle2-idle1)) * 100 / float64(total2-total1)
}
func validMonitorStatus(status string) bool {
switch status {
case "", "completed", "pending", "processing", "finalizing", "failed", "cancelled":
return true
default:
return false
}
}
func monitorStageDetail(ingestState, processingState, errorCode string) string {
if errorCode != "" {
return "处理失败,可查看错误码后重试"
}
switch {
case ingestState == "pending":
return "等待确认或上传"
case ingestState == "uploading":
return "正在上传测试知识库"
case processingState == "processing":
return "WeKnora正在解析、切片或向量化"
case processingState == "finalizing":
return "基础解析完成,索引或增强处理中"
case processingState == "completed":
return "解析完成,等待或已完成验收"
default:
return "等待远端状态更新"
}
}
+24
View File
@@ -0,0 +1,24 @@
package monitoring
import "testing"
func TestParseCPUStatAndMemoryInfo(t *testing.T) {
idle, total, err := parseCPUStat("cpu 100 20 30 400 50 6 7 8 0 0\n")
if err != nil || idle != 450 || total != 621 {
t.Fatalf("idle=%d total=%d err=%v", idle, total, err)
}
used, memoryTotal, err := parseMemoryInfo("MemTotal: 1000 kB\nMemAvailable: 250 kB\n")
if err != nil || used != 750*1024 || memoryTotal != 1000*1024 {
t.Fatalf("used=%d total=%d err=%v", used, memoryTotal, err)
}
}
func TestCPUPercentUsesTwoMonotonicSamples(t *testing.T) {
value := cpuPercent(450, 621, 500, 721)
if value != 50 {
t.Fatalf("cpu percent=%v, want 50", value)
}
if cpuPercent(500, 721, 450, 621) != 0 {
t.Fatal("non-monotonic CPU counters must fail closed to zero")
}
}
+87
View File
@@ -0,0 +1,87 @@
package monitoring
import (
"context"
"errors"
"time"
"knowledge-governance/internal/governance"
)
type SimulationProvider struct{ now func() time.Time }
func NewSimulationProvider(now func() time.Time) *SimulationProvider {
if now == nil {
now = time.Now
}
return &SimulationProvider{now: now}
}
func (provider *SimulationProvider) Overview(context.Context) (governance.MonitorOverview, error) {
now := provider.now().UTC()
return governance.MonitorOverview{
Mode: "simulation", Simulated: true, ObservedAt: now,
Host: governance.MonitorHost{CPUPercent: 42, MemoryUsedBytes: 19971597926, MemoryTotalBytes: 34359738368, DiskUsedBytes: 128849018880, DiskTotalBytes: 536870912000, GPUState: "unknown", GPUDetail: "未接入真实GPU监控", Simulated: true},
Services: []governance.MonitorService{
{Key: "weknora", Name: "WeKnora API", State: "healthy", Detail: "模拟连接正常", LatencyMS: 86, ObservedAt: now, Simulated: true},
{Key: "embedding", Name: "Embedding模型", State: "healthy", Detail: "模拟探测通过", LatencyMS: 142, ObservedAt: now, Simulated: true},
{Key: "llm", Name: "LLM模型", State: "healthy", Detail: "模拟探测通过", LatencyMS: 318, ObservedAt: now, Simulated: true},
{Key: "mineru", Name: "MinerU", State: "healthy", Detail: "模拟处理中2,队列5", LatencyMS: 24, ObservedAt: now, Simulated: true},
{Key: "postgres", Name: "PostgreSQL", State: "unknown", Detail: "未接入真实监控", ObservedAt: now, Simulated: true},
{Key: "redis", Name: "Redis / MQ", State: "unknown", Detail: "未接入真实监控", ObservedAt: now, Simulated: true},
{Key: "oss", Name: "OSS", State: "unknown", Detail: "未接入真实监控", ObservedAt: now, Simulated: true},
},
KnowledgeBases: []governance.KnowledgeBaseProgress{{ID: "simulation-company-support", Name: "公司级技术支持知识库", Visibility: "mixed", Imported: 5750, Completed: 4530, Pending: 450, Processing: 50, Finalizing: 714, Failed: 6, InternalImported: 4200, InternalCompleted: 3300, ExternalImported: 1550, ExternalCompleted: 1230}},
}, nil
}
var simulationFiles = []governance.KnowledgeMonitorFile{
{Name: "[模拟] CR5用户手册.pdf", Directory: "02|产品资料/CR系列", Visibility: "internal", Status: "completed", StageDetail: "解析、切片和增强任务均完成", ElapsedSecond: 0},
{Name: "[模拟] MG400部署说明.pdf", Directory: "03|机器人部署与使用", Visibility: "internal", Status: "pending", StageDetail: "等待进入解析队列", ElapsedSecond: 420},
{Name: "[模拟] 产品参数总表.xlsx", Directory: "02|产品资料", Visibility: "external", Status: "finalizing", StageDetail: "基础检索可用,Wiki与图谱仍在生成", ElapsedSecond: 2100},
{Name: "[模拟] 二次开发接口说明.docx", Directory: "05|二次开发", Visibility: "internal", Status: "processing", StageDetail: "正在切片和向量化", ElapsedSecond: 480},
{Name: "[模拟] CRA-IP68产品介绍.pdf", Directory: "06|应用案例与解决方案", Visibility: "external", Status: "failed", StageDetail: "MinerU解析失败", ElapsedSecond: 720, ErrorCode: "mineru_parse_failed"},
}
func (provider *SimulationProvider) ListFiles(_ context.Context, knowledgeBaseID, status string, limit, offset int) (governance.KnowledgeMonitorFilePage, error) {
if knowledgeBaseID != "simulation-company-support" || limit < 1 || limit > 100 || offset < 0 {
return governance.KnowledgeMonitorFilePage{}, errors.New("invalid monitor file query")
}
allowed := map[string]bool{"": true, "completed": true, "pending": true, "processing": true, "finalizing": true, "failed": true, "cancelled": true}
if !allowed[status] {
return governance.KnowledgeMonitorFilePage{}, errors.New("invalid monitor status")
}
filtered := make([]governance.KnowledgeMonitorFile, 0)
for _, item := range simulationFiles {
if status == "" || item.Status == status {
filtered = append(filtered, item)
}
}
page := governance.KnowledgeMonitorFilePage{Total: len(filtered), Limit: limit, Offset: offset, SampleOnly: true, Items: []governance.KnowledgeMonitorFile{}}
if offset < len(filtered) {
end := offset + limit
if end > len(filtered) {
end = len(filtered)
}
page.Items = append(page.Items, filtered[offset:end]...)
}
return page, nil
}
// Metrics declares that no real run metrics exist in simulation mode: every
// sample-dependent number stays null instead of pretending to be zero (P9).
func (provider *SimulationProvider) Metrics(context.Context) (governance.RunMetrics, error) {
now := provider.now().UTC()
return governance.RunMetrics{
ObservedAt: now,
WorkerSource: "simulation",
ShortWindow: governance.JobWindowMetrics{WindowSeconds: int((1 * time.Hour).Seconds())},
LongWindow: governance.JobWindowMetrics{WindowSeconds: int((24 * time.Hour).Seconds())},
}, nil
}
// Services declares that a simulation provider owns no real dependencies: every
// readiness judgement belongs to the real provider.
func (provider *SimulationProvider) Services(_ context.Context) []governance.ServiceHealth {
return []governance.ServiceHealth{}
}
+52
View File
@@ -0,0 +1,52 @@
package monitoring
import (
"context"
"testing"
"time"
)
func TestSimulationProviderLabelsEveryValueAndReconcilesKnowledgeCounts(t *testing.T) {
now := time.Date(2026, 9, 8, 12, 0, 0, 0, time.UTC)
provider := NewSimulationProvider(func() time.Time { return now })
overview, err := provider.Overview(context.Background())
if err != nil {
t.Fatal(err)
}
if overview.Mode != "simulation" || !overview.Simulated || overview.ObservedAt != now || len(overview.Services) < 4 || len(overview.KnowledgeBases) != 1 {
t.Fatalf("overview=%#v", overview)
}
if overview.Host.DiskTotalBytes <= 0 || overview.Host.DiskUsedBytes <= 0 || overview.Host.GPUState != "unknown" || overview.Host.GPUDetail != "未接入真实GPU监控" {
t.Fatalf("host resources=%#v", overview.Host)
}
requiredServices := map[string]bool{"weknora": false, "mineru": false, "postgres": false, "redis": false, "oss": false, "embedding": false, "llm": false}
for _, service := range overview.Services {
if _, ok := requiredServices[service.Key]; ok {
requiredServices[service.Key] = true
}
}
for key, present := range requiredServices {
if !present {
t.Errorf("missing service status %q", key)
}
}
kb := overview.KnowledgeBases[0]
if kb.Imported != 5750 || kb.Completed != 4530 || kb.Pending != 450 || kb.Finalizing != 714 || kb.Processing != 50 || kb.Failed != 6 {
t.Fatalf("kb=%#v", kb)
}
if kb.Completed+kb.Pending+kb.Finalizing+kb.Processing+kb.Failed+kb.Cancelled != kb.Imported {
t.Fatalf("status counts do not reconcile: %#v", kb)
}
for _, service := range overview.Services {
if !service.Simulated {
t.Fatalf("unlabelled service=%#v", service)
}
}
files, err := provider.ListFiles(context.Background(), kb.ID, "failed", 20, 0)
if err != nil {
t.Fatal(err)
}
if files.Total != 1 || len(files.Items) != 1 || files.Items[0].Status != "failed" || files.Items[0].Name[:len("[模拟]")] != "[模拟]" || !files.SampleOnly {
t.Fatalf("files=%#v", files)
}
}