feat: 知识治理平台(含已验证发布/回滚契约)
全栈知识治理平台首次入库(P1-P10 + 发布链路模拟验证闭环): - 治理核心:源目录/变更集/人工审核/发布批次/发布执行(fail-closed 闸口) - 测试通道:WeKnora 测试 KB 上传/轮询/验收;poll 完成时写入 parser_generation/index_generation,修复 manual/seed 路径发布闸口永久拦截 - 发布适配器 v071(本地 source=测试角色 / 云端 target=生产角色 模拟验证通过): * ES 导出/导入 mapping 自适应(.keyword 子字段、可选 tenant_id) * OSS locator 支持嵌套 scheme;chunks.seq_id 由目标序列分配 * bundle 归档同身份重试原子替换;verifier 按 bundle chunk_type 计数 * 回滚 job payload 支持 prior_execution_id(strict 解码) - 三通道发布+回滚演练实测:publish 4/4 verified,检索命中, rollback 4/4 rolled_back,目标端 PG is_enabled=false、ES is_enabled=false - 前端 A·Data-console 风格全站统一,120 用例通过 - 门禁:go vet / go test 全包通过,gofmt 干净
This commit is contained in:
@@ -0,0 +1,328 @@
|
||||
package monitoring
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
)
|
||||
|
||||
// P10: every configured dependency reports Liveness ("process reachable") and
|
||||
// Readiness ("can accept work") independently. Missing deployment and missing
|
||||
// configuration are explicit states and never collapse into "unknown".
|
||||
|
||||
const (
|
||||
LiveUp = "up"
|
||||
LiveDown = "down"
|
||||
LiveUnknown = "unknown"
|
||||
ReadReady = "ready"
|
||||
ReadBlocked = "blocked"
|
||||
ReadNotConfigured = "not_configured"
|
||||
ReadNotApplicable = "not_applicable"
|
||||
)
|
||||
|
||||
// ProbeOutcome is one raw probe attempt.
|
||||
type ProbeOutcome struct {
|
||||
Reachability error // liveness result (nil = reachable)
|
||||
WorkReadiness error // readiness result (nil = can accept work)
|
||||
Latency time.Duration
|
||||
ObservedAt time.Time
|
||||
}
|
||||
|
||||
var (
|
||||
errCapacityFull = errors.New("capacity_full")
|
||||
errNoPermission = errors.New("no_permission")
|
||||
)
|
||||
|
||||
// classify turns one probe outcome into health states. Explicit not-configured
|
||||
// and not-applicable results win over generic reachability failures.
|
||||
func classify(key, name string, configured, applicable bool, outcome ProbeOutcome) governance.ServiceHealth {
|
||||
health := governance.ServiceHealth{Key: key, Name: name, Configured: configured, Applicable: applicable, LatencyMS: outcome.Latency.Milliseconds(), ObservedAt: outcome.ObservedAt}
|
||||
switch {
|
||||
case !applicable:
|
||||
health.Liveness, health.Reason = LiveUnknown, "未部署:本平台不使用该依赖"
|
||||
health.Readiness = ReadNotApplicable
|
||||
return health
|
||||
case !configured:
|
||||
health.Liveness, health.Reason = LiveUnknown, "已部署但未配置探测参数"
|
||||
health.Readiness = ReadNotConfigured
|
||||
return health
|
||||
}
|
||||
if outcome.Reachability != nil {
|
||||
health.Readiness = ReadBlocked
|
||||
health.Reason = sanitizeProbeError(outcome.Reachability)
|
||||
if errors.Is(outcome.Reachability, errCapacityFull) || errors.Is(outcome.Reachability, errNoPermission) {
|
||||
health.Liveness = LiveUp // reachable, but not workable
|
||||
} else {
|
||||
health.Liveness = LiveDown
|
||||
}
|
||||
return health
|
||||
}
|
||||
health.Liveness = LiveUp
|
||||
if outcome.WorkReadiness != nil {
|
||||
health.Readiness, health.Reason = ReadBlocked, sanitizeProbeError(outcome.WorkReadiness)
|
||||
return health
|
||||
}
|
||||
health.Readiness = ReadReady
|
||||
return health
|
||||
}
|
||||
|
||||
// sanitizeProbeError keeps operators useful text without leaking endpoints,
|
||||
// credentials, or hosts that live inside URLs.
|
||||
func sanitizeProbeError(err error) string {
|
||||
switch {
|
||||
case errors.Is(err, errCapacityFull):
|
||||
return "服务可达但容量已满(429),暂时不可接单"
|
||||
case errors.Is(err, errNoPermission):
|
||||
return "服务可达但目标知识库无读取权限"
|
||||
case errors.Is(err, context.DeadlineExceeded), errors.Is(err, context.Canceled):
|
||||
return "探测超时"
|
||||
}
|
||||
var dnsErr *net.DNSError
|
||||
if errors.As(err, &dnsErr) {
|
||||
return "地址解析失败(DNS)"
|
||||
}
|
||||
var netErr net.Error
|
||||
if errors.As(err, &netErr) {
|
||||
return "网络不可达"
|
||||
}
|
||||
message := err.Error()
|
||||
if len(message) > 160 {
|
||||
return message[:160]
|
||||
}
|
||||
return message
|
||||
}
|
||||
|
||||
// healthTracker remembers recent success and consecutive failures per service.
|
||||
type healthTracker struct {
|
||||
records map[string]*trackedHealth
|
||||
}
|
||||
|
||||
type trackedHealth struct {
|
||||
lastSuccess *time.Time
|
||||
failStreak int
|
||||
}
|
||||
|
||||
func newHealthTracker() *healthTracker {
|
||||
return &healthTracker{records: map[string]*trackedHealth{}}
|
||||
}
|
||||
|
||||
func (tracker *healthTracker) apply(key string, health governance.ServiceHealth) governance.ServiceHealth {
|
||||
record := tracker.records[key]
|
||||
if record == nil {
|
||||
record = &trackedHealth{}
|
||||
tracker.records[key] = record
|
||||
}
|
||||
switch {
|
||||
case health.Liveness == LiveUp && health.Readiness == ReadReady:
|
||||
now := health.ObservedAt
|
||||
record.lastSuccess = &now
|
||||
record.failStreak = 0
|
||||
case health.Liveness == LiveUnknown:
|
||||
// Config-level states say nothing about runtime; keep the history.
|
||||
default:
|
||||
record.failStreak++
|
||||
}
|
||||
health.LastSuccessAt = record.lastSuccess
|
||||
health.ConsecutiveFailures = record.failStreak
|
||||
return health
|
||||
}
|
||||
|
||||
// httpProbe performs a liveness-only HTTP GET on <base>/health, mapped through
|
||||
// classify. configured=false when the base URL is absent.
|
||||
func (provider *RealProvider) httpProbe(ctx context.Context, key, name, baseURL string, now time.Time) governance.ServiceHealth {
|
||||
baseURL = strings.TrimSpace(baseURL)
|
||||
if baseURL == "" {
|
||||
return provider.track(classify(key, name, false, true, ProbeOutcome{ObservedAt: now}))
|
||||
}
|
||||
parsed, err := url.Parse(baseURL)
|
||||
if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Host == "" {
|
||||
return provider.track(classify(key, name, true, true, ProbeOutcome{Reachability: errors.New("探测配置无效"), ObservedAt: now}))
|
||||
}
|
||||
parsed.Path = strings.TrimSuffix(parsed.Path, "/") + "/health"
|
||||
parsed.RawQuery, parsed.Fragment = "", ""
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodGet, parsed.String(), nil)
|
||||
if err != nil {
|
||||
return provider.track(classify(key, name, true, true, ProbeOutcome{Reachability: err, ObservedAt: now}))
|
||||
}
|
||||
started := time.Now()
|
||||
response, err := provider.httpClient.Do(request)
|
||||
outcome := ProbeOutcome{Latency: time.Since(started), ObservedAt: now}
|
||||
if err != nil {
|
||||
outcome.Reachability = err
|
||||
return provider.track(classify(key, name, true, true, outcome))
|
||||
}
|
||||
defer response.Body.Close()
|
||||
_, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 4096))
|
||||
if response.StatusCode < 200 || response.StatusCode >= 300 {
|
||||
outcome.Reachability = fmt.Errorf("健康端点返回HTTP %d", response.StatusCode)
|
||||
return provider.track(classify(key, name, true, true, outcome))
|
||||
}
|
||||
return provider.track(classify(key, name, true, true, outcome))
|
||||
}
|
||||
|
||||
// weKnoraProbe checks the WeKnora API: liveness via /health, readiness via an
|
||||
// optional real read against a routing knowledge base.
|
||||
func (provider *RealProvider) weKnoraProbe(ctx context.Context, now time.Time) governance.ServiceHealth {
|
||||
if provider.weknoraBaseURL == "" {
|
||||
return provider.track(classify("weknora", "WeKnora API", false, true, ProbeOutcome{ObservedAt: now}))
|
||||
}
|
||||
outcome := provider.httpProbeOutcome(ctx, provider.weknoraBaseURL, now)
|
||||
if outcome.Reachability == nil && provider.weknoraReadiness != nil {
|
||||
outcome.WorkReadiness = provider.weknoraReadiness(ctx)
|
||||
}
|
||||
return provider.track(classify("weknora", "WeKnora API", true, true, outcome))
|
||||
}
|
||||
|
||||
func (provider *RealProvider) httpProbeOutcome(ctx context.Context, baseURL string, now time.Time) ProbeOutcome {
|
||||
parsed, err := url.Parse(baseURL)
|
||||
outcome := ProbeOutcome{ObservedAt: now}
|
||||
if err != nil || parsed.Scheme != "http" && parsed.Scheme != "https" || parsed.Host == "" {
|
||||
outcome.Reachability = errors.New("探测配置无效")
|
||||
return outcome
|
||||
}
|
||||
parsed.Path = strings.TrimSuffix(parsed.Path, "/") + "/health"
|
||||
parsed.RawQuery, parsed.Fragment = "", ""
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodGet, parsed.String(), nil)
|
||||
if err != nil {
|
||||
outcome.Reachability = err
|
||||
return outcome
|
||||
}
|
||||
started := time.Now()
|
||||
response, err := provider.httpClient.Do(request)
|
||||
outcome.Latency = time.Since(started)
|
||||
if err != nil {
|
||||
outcome.Reachability = err
|
||||
return outcome
|
||||
}
|
||||
defer response.Body.Close()
|
||||
_, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 4096))
|
||||
switch {
|
||||
case response.StatusCode == http.StatusTooManyRequests:
|
||||
// Reachable but at capacity: liveness stays up, readiness blocks.
|
||||
outcome.Reachability = errCapacityFull
|
||||
case response.StatusCode == http.StatusForbidden || response.StatusCode == http.StatusUnauthorized:
|
||||
outcome.Reachability = errNoPermission
|
||||
case response.StatusCode < 200 || response.StatusCode >= 300:
|
||||
outcome.Reachability = fmt.Errorf("健康端点返回HTTP %d", response.StatusCode)
|
||||
}
|
||||
return outcome
|
||||
}
|
||||
|
||||
// workerProbe reads queue-side worker heartbeats: no fresh heartbeat row means
|
||||
// the worker loop is not touching the queue (liveness down), while the queue
|
||||
// itself can still be ready for a future worker.
|
||||
func (provider *RealProvider) workerProbe(ctx context.Context, now time.Time) []governance.ServiceHealth {
|
||||
rows, err := provider.db.Query(ctx, `SELECT worker_id, last_seen_at FROM worker_heartbeats WHERE last_seen_at >= $1::timestamptz ORDER BY worker_id`, now.Add(-provider.workerStaleAfter))
|
||||
if err != nil {
|
||||
return []governance.ServiceHealth{provider.track(classify("worker", "解析Worker", true, true, ProbeOutcome{Reachability: err, ObservedAt: now}))}
|
||||
}
|
||||
defer rows.Close()
|
||||
type entry struct {
|
||||
id string
|
||||
seen time.Time
|
||||
}
|
||||
live := []entry{}
|
||||
for rows.Next() {
|
||||
var id string
|
||||
var seen time.Time
|
||||
if err := rows.Scan(&id, &seen); err != nil {
|
||||
return []governance.ServiceHealth{provider.track(classify("worker", "解析Worker", true, true, ProbeOutcome{Reachability: err, ObservedAt: now}))}
|
||||
}
|
||||
live = append(live, entry{id, seen.UTC()})
|
||||
}
|
||||
if len(live) == 0 {
|
||||
return []governance.ServiceHealth{provider.track(classify("worker", "解析Worker", true, true, ProbeOutcome{Reachability: errors.New("队列无新鲜心跳(worker未运行或全部停跳)"), ObservedAt: now}))}
|
||||
}
|
||||
out := make([]governance.ServiceHealth, 0, len(live))
|
||||
for _, item := range live {
|
||||
outcome := ProbeOutcome{ObservedAt: item.seen}
|
||||
if now.Sub(item.seen) > provider.workerStaleAfter {
|
||||
outcome.Reachability = errors.New("心跳过期")
|
||||
}
|
||||
out = append(out, provider.track(classify("worker:"+item.id, "解析Worker "+item.id, true, true, outcome)))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (provider *RealProvider) track(health governance.ServiceHealth) governance.ServiceHealth {
|
||||
return provider.health.apply(health.Key, health)
|
||||
}
|
||||
|
||||
// Services reports the independent Liveness/Readiness matrix for every service
|
||||
// the deployment actually has (P10). Missing services are explicit states.
|
||||
func (provider *RealProvider) Services(ctx context.Context) []governance.ServiceHealth {
|
||||
now := provider.now().UTC()
|
||||
health := make([]governance.ServiceHealth, 0, 8)
|
||||
// PostgreSQL: liveness = ping, readiness = queryable.
|
||||
pgOutcome := ProbeOutcome{ObservedAt: now}
|
||||
started := time.Now()
|
||||
if err := provider.db.Ping(ctx); err != nil {
|
||||
pgOutcome.Reachability = err
|
||||
} else {
|
||||
pgOutcome.Latency = time.Since(started)
|
||||
var ok int
|
||||
if err := provider.db.QueryRow(ctx, "SELECT 1").Scan(&ok); err != nil {
|
||||
pgOutcome.WorkReadiness = err
|
||||
}
|
||||
}
|
||||
health = append(health, provider.track(classify("postgres", "PostgreSQL", true, true, pgOutcome)))
|
||||
// Queue readiness rides on the jobs table.
|
||||
queueOutcome := ProbeOutcome{ObservedAt: now}
|
||||
var pending int
|
||||
if err := provider.db.QueryRow(ctx, "SELECT COUNT(*) FROM jobs WHERE state IN ('pending','retry')").Scan(&pending); err != nil {
|
||||
queueOutcome.Reachability = err
|
||||
}
|
||||
health = append(health, provider.track(classify("queue", "任务队列", true, true, queueOutcome)))
|
||||
health = append(health, provider.workerProbe(ctx, now)...)
|
||||
health = append(health, provider.weKnoraProbe(ctx, now))
|
||||
health = append(health, provider.httpProbe(ctx, "mineru", "MinerU", provider.minerUBaseURL, now))
|
||||
// Not-applicable dependencies stay explicit (embedding/llm managed by
|
||||
// WeKnora, redis/mq/oss not deployed).
|
||||
health = append(health,
|
||||
provider.track(classify("embedding", "Embedding模型", false, false, ProbeOutcome{ObservedAt: now})),
|
||||
provider.track(classify("llm", "LLM模型", false, false, ProbeOutcome{ObservedAt: now})),
|
||||
provider.track(classify("redis", "Redis / MQ", false, false, ProbeOutcome{ObservedAt: now})),
|
||||
provider.track(classify("oss", "OSS", false, false, ProbeOutcome{ObservedAt: now})),
|
||||
)
|
||||
sort.Slice(health, func(i, j int) bool { return health[i].Key < health[j].Key })
|
||||
return health
|
||||
}
|
||||
|
||||
// Readiness blocks only on services that ARE deployed and configured and
|
||||
// currently fail readiness; not_configured is surfaced as a warning elsewhere.
|
||||
func overallReadiness(health []governance.ServiceHealth) (bool, []string) {
|
||||
blocked := []string{}
|
||||
for _, item := range health {
|
||||
if !item.Applicable || !item.Configured {
|
||||
continue
|
||||
}
|
||||
if item.Readiness != ReadReady {
|
||||
blocked = append(blocked, item.Key)
|
||||
}
|
||||
}
|
||||
return len(blocked) == 0, blocked
|
||||
}
|
||||
|
||||
func (tracker *healthTracker) snapshot() []governance.ServiceHealth {
|
||||
keys := make([]string, 0, len(tracker.records))
|
||||
for key := range tracker.records {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
records := make([]governance.ServiceHealth, 0, len(keys))
|
||||
for _, key := range keys {
|
||||
record := tracker.records[key]
|
||||
item := governance.ServiceHealth{Key: key, LastSuccessAt: record.lastSuccess, ConsecutiveFailures: record.failStreak}
|
||||
records = append(records, item)
|
||||
}
|
||||
return records
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
package monitoring
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func testNow() time.Time { return time.Date(2026, 9, 12, 9, 0, 0, 0, time.UTC) }
|
||||
|
||||
func TestClassifySeparatesNotApplicableNotConfiguredAndFailures(t *testing.T) {
|
||||
now := testNow()
|
||||
notDeployed := classify("redis", "Redis / MQ", false, false, ProbeOutcome{ObservedAt: now})
|
||||
if notDeployed.Liveness != LiveUnknown || notDeployed.Readiness != ReadNotApplicable {
|
||||
t.Fatalf("not deployed = %+v", notDeployed)
|
||||
}
|
||||
notConfigured := classify("mineru", "MinerU", false, true, ProbeOutcome{ObservedAt: now})
|
||||
if notConfigured.Liveness != LiveUnknown || notConfigured.Readiness != ReadNotConfigured {
|
||||
t.Fatalf("not configured = %+v", notConfigured)
|
||||
}
|
||||
healthy := classify("postgres", "PostgreSQL", true, true, ProbeOutcome{ObservedAt: now})
|
||||
if healthy.Liveness != LiveUp || healthy.Readiness != ReadReady {
|
||||
t.Fatalf("healthy = %+v", healthy)
|
||||
}
|
||||
// DNS failure: down + blocked with sanitized reason, never the raw host.
|
||||
dns := &net.DNSError{Err: "no such host", Name: "weknora.internal.example"}
|
||||
down := classify("weknora", "WeKnora", true, true, ProbeOutcome{ObservedAt: now, Reachability: dns})
|
||||
if down.Liveness != LiveDown || down.Reason == "" || strings.Contains(down.Reason, "weknora.internal") {
|
||||
t.Fatalf("dns down = %+v", down)
|
||||
}
|
||||
// Timeout.
|
||||
timeout := classify("weknora", "WeKnora", true, true, ProbeOutcome{ObservedAt: now, Reachability: context.DeadlineExceeded})
|
||||
if timeout.Liveness != LiveDown || timeout.Readiness != ReadBlocked {
|
||||
t.Fatalf("timeout = %+v", timeout)
|
||||
}
|
||||
// Reachable but capacity full => live up, readiness blocked, distinct reason.
|
||||
capacity := classify("mineru", "MinerU", true, true, ProbeOutcome{ObservedAt: now, Reachability: errCapacityFull})
|
||||
if capacity.Liveness != LiveUp || capacity.Readiness != ReadBlocked || capacity.Reason == "" {
|
||||
t.Fatalf("capacity = %+v", capacity)
|
||||
}
|
||||
// Reachable but permission denied on target KB (readiness error only).
|
||||
perm := classify("weknora", "WeKnora", true, true, ProbeOutcome{ObservedAt: now, WorkReadiness: errNoPermission})
|
||||
if perm.Liveness != LiveUp || perm.Readiness != ReadBlocked {
|
||||
t.Fatalf("permission = %+v", perm)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHealthTrackerRemembersSuccessAndFailStreak(t *testing.T) {
|
||||
tracker := newHealthTracker()
|
||||
now := testNow()
|
||||
first := tracker.apply("postgres", classify("postgres", "PostgreSQL", true, true, ProbeOutcome{ObservedAt: now}))
|
||||
if first.LastSuccessAt == nil || !first.LastSuccessAt.Equal(now) || first.ConsecutiveFailures != 0 {
|
||||
t.Fatalf("first = %+v", first)
|
||||
}
|
||||
later := now.Add(time.Minute)
|
||||
bad := tracker.apply("postgres", classify("postgres", "PostgreSQL", true, true, ProbeOutcome{ObservedAt: later, Reachability: context.DeadlineExceeded}))
|
||||
if !bad.LastSuccessAt.Equal(now) || bad.ConsecutiveFailures != 1 {
|
||||
t.Fatalf("after one failure = %+v", bad)
|
||||
}
|
||||
bad2 := tracker.apply("postgres", classify("postgres", "PostgreSQL", true, true, ProbeOutcome{ObservedAt: bad.ObservedAt.Add(time.Minute), Reachability: context.DeadlineExceeded}))
|
||||
if bad2.ConsecutiveFailures != 2 {
|
||||
t.Fatalf("streak = %d", bad2.ConsecutiveFailures)
|
||||
}
|
||||
good := tracker.apply("postgres", classify("postgres", "PostgreSQL", true, true, ProbeOutcome{ObservedAt: bad2.ObservedAt.Add(time.Minute)}))
|
||||
if good.ConsecutiveFailures != 0 || !good.LastSuccessAt.Equal(bad2.ObservedAt.Add(time.Minute)) {
|
||||
t.Fatalf("recovered = %+v", good)
|
||||
}
|
||||
// Unknown probes (not configured) must not reset or extend a real streak.
|
||||
neutral := tracker.apply("postgres", classify("postgres", "PostgreSQL", false, true, ProbeOutcome{ObservedAt: good.ObservedAt.Add(time.Minute)}))
|
||||
if neutral.ConsecutiveFailures != 0 {
|
||||
t.Fatalf("neutral streak = %d", neutral.ConsecutiveFailures)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSanitizeProbeErrorTruncatesLongMessages(t *testing.T) {
|
||||
long := errors.New(fmt.Sprintf("%0200d", 1))
|
||||
if got := sanitizeProbeError(long); len(got) > 160 {
|
||||
t.Fatalf("len=%d", len(got))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,171 @@
|
||||
package monitoring
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
)
|
||||
|
||||
type windowJobOutcome struct {
|
||||
state string
|
||||
durationSeconds float64 // <0 when duration is unknowable
|
||||
}
|
||||
|
||||
const (
|
||||
metricsShortWindow = time.Hour
|
||||
metricsLongWindow = 24 * time.Hour
|
||||
metricsWorkerLease = 90 * time.Second // workers heartbeat every <= lease; fresher than stale leases counts active
|
||||
metricsSampleMargin = 2 * time.Second
|
||||
)
|
||||
|
||||
func computeJobWindowMetrics(outcomes []windowJobOutcome, windowSeconds int) governance.JobWindowMetrics {
|
||||
metrics := governance.JobWindowMetrics{WindowSeconds: windowSeconds}
|
||||
var durations []float64
|
||||
for _, outcome := range outcomes {
|
||||
switch outcome.state {
|
||||
case "completed":
|
||||
metrics.JobsCompleted++
|
||||
case "failed":
|
||||
metrics.JobsFailed++
|
||||
}
|
||||
if outcome.durationSeconds >= 0 {
|
||||
durations = append(durations, outcome.durationSeconds)
|
||||
}
|
||||
}
|
||||
if finished := metrics.JobsCompleted + metrics.JobsFailed; finished > 0 {
|
||||
success := math.Round(float64(metrics.JobsCompleted)/float64(finished)*10000) / 100
|
||||
failure := math.Round(float64(metrics.JobsFailed)/float64(finished)*10000) / 100
|
||||
metrics.SuccessRate = &success
|
||||
metrics.FailureRate = &failure
|
||||
}
|
||||
if len(durations) > 0 {
|
||||
sort.Float64s(durations)
|
||||
var sum float64
|
||||
for _, value := range durations {
|
||||
sum += value
|
||||
}
|
||||
average := math.Round(sum/float64(len(durations))*10) / 10
|
||||
p50 := math.Round(percentile(durations, 0.50)*10) / 10
|
||||
p95 := math.Round(percentile(durations, 0.95)*10) / 10
|
||||
metrics.DurationAvgSeconds = &average
|
||||
metrics.DurationP50Seconds = &p50
|
||||
metrics.DurationP95Seconds = &p95
|
||||
}
|
||||
return metrics
|
||||
}
|
||||
|
||||
// percentile uses nearest-rank so tiny samples never interpolate a fake value.
|
||||
func percentile(sorted []float64, fraction float64) float64 {
|
||||
rank := int(math.Ceil(fraction * float64(len(sorted))))
|
||||
if rank < 1 {
|
||||
rank = 1
|
||||
}
|
||||
if rank > len(sorted) {
|
||||
rank = len(sorted)
|
||||
}
|
||||
return sorted[rank-1]
|
||||
}
|
||||
|
||||
func (provider *RealProvider) jobMetrics(ctx context.Context, window time.Duration) (governance.JobWindowMetrics, error) {
|
||||
windowSeconds := int(window.Seconds())
|
||||
var metrics governance.JobWindowMetrics
|
||||
// Pull the per-outcome detail (bounded) for exact-rate and percentile math.
|
||||
// Duration is only measurable when the whole lifecycle lives inside the
|
||||
// window; older-created jobs report -1 so percentiles never mix partials.
|
||||
rows, err := provider.db.Query(ctx, `
|
||||
SELECT state,
|
||||
CASE WHEN created_at > $2::timestamptz THEN COALESCE(EXTRACT(EPOCH FROM (updated_at - created_at)), -1) ELSE -1 END
|
||||
FROM jobs
|
||||
WHERE updated_at > $2::timestamptz AND updated_at <= $1::timestamptz
|
||||
AND state IN ('completed','failed')
|
||||
ORDER BY updated_at DESC
|
||||
LIMIT 10000`, provider.now(), provider.now().Add(-window))
|
||||
if err != nil {
|
||||
return metrics, fmt.Errorf("query job window outcomes: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
var outcomes []windowJobOutcome
|
||||
for rows.Next() {
|
||||
var outcome windowJobOutcome
|
||||
if err := rows.Scan(&outcome.state, &outcome.durationSeconds); err != nil {
|
||||
return metrics, fmt.Errorf("scan job window outcome: %w", err)
|
||||
}
|
||||
outcomes = append(outcomes, outcome)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return metrics, fmt.Errorf("iterate job window outcomes: %w", err)
|
||||
}
|
||||
metrics = computeJobWindowMetrics(outcomes, windowSeconds)
|
||||
|
||||
// Started inside the window uses created_at (any state).
|
||||
var started int
|
||||
if err := provider.db.QueryRow(ctx, `SELECT COUNT(*)::integer FROM jobs WHERE created_at > $2::timestamptz AND created_at <= $1::timestamptz`, provider.now(), provider.now().Add(-window)).Scan(&started); err != nil {
|
||||
return metrics, fmt.Errorf("count jobs started: %w", err)
|
||||
}
|
||||
metrics.JobsStarted = started
|
||||
return metrics, nil
|
||||
}
|
||||
|
||||
func (provider *RealProvider) queueGauges(ctx context.Context) (governance.QueueGauges, error) {
|
||||
var gauges governance.QueueGauges
|
||||
var oldestPendingSeconds *float64
|
||||
err := provider.db.QueryRow(ctx, `
|
||||
SELECT COUNT(*) FILTER (WHERE state='pending')::integer,
|
||||
COUNT(*) FILTER (WHERE state='running')::integer,
|
||||
COUNT(*) FILTER (WHERE state='retry')::integer,
|
||||
COUNT(*) FILTER (WHERE state='failed')::integer,
|
||||
CASE WHEN COUNT(*) FILTER (WHERE state IN ('pending','retry')) = 0 THEN NULL
|
||||
ELSE GREATEST(EXTRACT(EPOCH FROM ($1::timestamptz - MIN(created_at) FILTER (WHERE state IN ('pending','retry')))), 0) END
|
||||
FROM jobs`, provider.now()).Scan(&gauges.Pending, &gauges.Running, &gauges.Retry, &gauges.Failed, &oldestPendingSeconds)
|
||||
if err != nil {
|
||||
return gauges, fmt.Errorf("scan queue gauges: %w", err)
|
||||
}
|
||||
gauges.OldestPendingAgeSeconds = oldestPendingSeconds
|
||||
return gauges, nil
|
||||
}
|
||||
|
||||
func (provider *RealProvider) activeWorkerCount(ctx context.Context) (int, error) {
|
||||
// Count distinct fresh leases only; never infer from container counts.
|
||||
var active int
|
||||
err := provider.db.QueryRow(ctx, `
|
||||
SELECT COUNT(DISTINCT lease_owner)::integer
|
||||
FROM jobs
|
||||
WHERE state = 'running' AND lease_until > $1::timestamptz AND lease_owner IS NOT NULL`,
|
||||
provider.now().Add(-metricsWorkerLease)).Scan(&active)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("count active workers: %w", err)
|
||||
}
|
||||
return active, nil
|
||||
}
|
||||
|
||||
// Metrics implements the P9 run-metrics endpoint: frozen 1h/24h windows,
|
||||
// lease-based active workers, and null-safe samples.
|
||||
func (provider *RealProvider) Metrics(ctx context.Context) (governance.RunMetrics, error) {
|
||||
now := provider.now().UTC()
|
||||
metrics := governance.RunMetrics{ObservedAt: now, WorkerSource: "pg_lease"}
|
||||
short, err := provider.jobMetrics(ctx, metricsShortWindow)
|
||||
if err != nil {
|
||||
return metrics, err
|
||||
}
|
||||
longWindow, err := provider.jobMetrics(ctx, metricsLongWindow)
|
||||
if err != nil {
|
||||
return metrics, err
|
||||
}
|
||||
gauges, err := provider.queueGauges(ctx)
|
||||
if err != nil {
|
||||
return metrics, err
|
||||
}
|
||||
active, err := provider.activeWorkerCount(ctx)
|
||||
if err != nil {
|
||||
return metrics, err
|
||||
}
|
||||
metrics.ShortWindow = short
|
||||
metrics.LongWindow = longWindow
|
||||
metrics.Queue = gauges
|
||||
metrics.ActiveWorkers = active
|
||||
return metrics, nil
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
package monitoring
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestComputeJobWindowMetricsRates(t *testing.T) {
|
||||
// Mixed completed/failed inside the window: rates must be exact.
|
||||
m := computeJobWindowMetrics([]windowJobOutcome{
|
||||
{state: "completed", durationSeconds: 10},
|
||||
{state: "completed", durationSeconds: 20},
|
||||
{state: "completed", durationSeconds: 30},
|
||||
{state: "failed", durationSeconds: 40},
|
||||
}, 60)
|
||||
if m.WindowSeconds != 60 || m.JobsCompleted != 3 || m.JobsFailed != 1 {
|
||||
t.Fatalf("counts = %+v", m)
|
||||
}
|
||||
if m.SuccessRate == nil || *m.SuccessRate != 75 {
|
||||
t.Fatalf("success rate = %v, want 75", m.SuccessRate)
|
||||
}
|
||||
if m.FailureRate == nil || *m.FailureRate != 25 {
|
||||
t.Fatalf("failure rate = %v, want 25", m.FailureRate)
|
||||
}
|
||||
// Durations: avg 25s, p50 20s (nearest-rank on 4 sorted samples 10/20/30/40).
|
||||
if m.DurationAvgSeconds == nil || *m.DurationAvgSeconds != 25 {
|
||||
t.Fatalf("avg = %v, want 25", m.DurationAvgSeconds)
|
||||
}
|
||||
if m.DurationP50Seconds == nil || *m.DurationP50Seconds != 20 {
|
||||
t.Fatalf("p50 = %v, want 20", m.DurationP50Seconds)
|
||||
}
|
||||
if m.DurationP95Seconds == nil || *m.DurationP95Seconds != 40 {
|
||||
t.Fatalf("p95 = %v, want 40", m.DurationP95Seconds)
|
||||
}
|
||||
}
|
||||
|
||||
func TestComputeJobWindowMetricsNullsWhenNoSamples(t *testing.T) {
|
||||
// Insufficient data must be null, never a fabricated 0%.
|
||||
m := computeJobWindowMetrics([]windowJobOutcome{{state: "completed", durationSeconds: -1}}, 3600)
|
||||
if m.JobsCompleted != 1 {
|
||||
t.Fatalf("completed = %d", m.JobsCompleted)
|
||||
}
|
||||
if m.SuccessRate == nil || *m.SuccessRate != 100 {
|
||||
t.Fatalf("success rate = %v, want 100 (1/1)", m.SuccessRate)
|
||||
}
|
||||
if m.DurationAvgSeconds != nil || m.DurationP50Seconds != nil || m.DurationP95Seconds != nil {
|
||||
t.Fatalf("durations must be null without terminal-time samples: %+v", m)
|
||||
}
|
||||
empty := computeJobWindowMetrics(nil, 3600)
|
||||
if empty.SuccessRate != nil || empty.FailureRate != nil || empty.JobsStarted != 0 {
|
||||
t.Fatalf("empty window = %+v, want all-null rates", empty)
|
||||
}
|
||||
blob, err := json.Marshal(empty)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, key := range []string{`"success_rate":null`, `"failure_rate":null`, `"duration_avg_seconds":null`} {
|
||||
if !contains(string(blob), key) {
|
||||
t.Fatalf("metrics JSON missing %s: %s", key, blob)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func contains(haystack, needle string) bool {
|
||||
for i := 0; i+len(needle) <= len(haystack); i++ {
|
||||
if haystack[i:i+len(needle)] == needle {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
package monitoring
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"knowledge-governance/internal/platform/postgres"
|
||||
)
|
||||
|
||||
// TestRealServicesHealthMatrix freezes docs plan P10: liveness and readiness
|
||||
// are per-service and independent, worker liveness rides on queue heartbeats,
|
||||
// and missing configuration is an explicit state — never "unknown".
|
||||
func TestRealServicesHealthMatrix(t *testing.T) {
|
||||
databaseURL := isolatedMetricsURL(t)
|
||||
pool, err := pgxpool.New(context.Background(), databaseURL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer pool.Close()
|
||||
ctx := context.Background()
|
||||
if err := postgres.ApplyMigrations(ctx, pool); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
now := time.Date(2026, time.September, 12, 12, 0, 0, 0, time.UTC)
|
||||
provider, err := NewRealProvider(RealProviderConfig{
|
||||
DB: metricsPool{pool},
|
||||
Now: func() time.Time { return now },
|
||||
HTTPClient: &http.Client{Timeout: time.Second},
|
||||
WorkerStaleAfter: 2 * time.Minute,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Before any worker claims: queue/postgres healthy, worker aggregated down.
|
||||
health := provider.Services(ctx)
|
||||
byKey := map[string]string{}
|
||||
for _, item := range health {
|
||||
byKey[item.Key+"/l"] = item.Liveness
|
||||
byKey[item.Key+"/r"] = item.Readiness
|
||||
}
|
||||
if byKey["postgres/l"] != LiveUp || byKey["postgres/r"] != ReadReady {
|
||||
t.Fatalf("postgres health = %v/%v", byKey["postgres/l"], byKey["postgres/r"])
|
||||
}
|
||||
if byKey["worker/l"] != LiveDown || byKey["worker/r"] != ReadBlocked {
|
||||
t.Fatalf("worker health with no heartbeat = %v/%v", byKey["worker/l"], byKey["worker/r"])
|
||||
}
|
||||
if byKey["weknora/r"] != ReadNotConfigured || byKey["mineru/r"] != ReadNotConfigured {
|
||||
t.Fatalf("unconfigured deps must be explicit: %v %v", byKey["weknora/r"], byKey["mineru/r"])
|
||||
}
|
||||
for _, key := range []string{"redis", "oss", "embedding", "llm"} {
|
||||
if byKey[key+"/r"] != ReadNotApplicable || byKey[key+"/l"] != LiveUnknown {
|
||||
t.Fatalf("not-deployed %s = %v/%v", key, byKey[key+"/l"], byKey[key+"/r"])
|
||||
}
|
||||
}
|
||||
// Readiness must be blocked ONLY by configured+deployed failures: worker.
|
||||
ok, blocked := overallReadiness(health)
|
||||
if ok || len(blocked) != 1 || blocked[0] != "worker" {
|
||||
t.Fatalf("overall readiness ok=%v blocked=%v", ok, blocked)
|
||||
}
|
||||
|
||||
// A worker claim refreshes the heartbeat row and flips liveness independently.
|
||||
store := postgres.NewStore(pool)
|
||||
if _, err := store.Claim(ctx, "kg-test-worker", []string{"test.upload"}, now, time.Minute); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
health = provider.Services(ctx)
|
||||
found := false
|
||||
for _, item := range health {
|
||||
if strings.HasPrefix(item.Key, "worker:") {
|
||||
found = true
|
||||
if item.Liveness != LiveUp || item.Readiness != ReadReady {
|
||||
t.Fatalf("fresh worker health = %v/%v", item.Liveness, item.Readiness)
|
||||
}
|
||||
}
|
||||
if item.Key == "worker" {
|
||||
t.Fatalf("stale aggregate row must disappear once a worker is live")
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("live worker row missing after claim")
|
||||
}
|
||||
|
||||
// Heartbeat older than the stale window: liveness down, and the API-side
|
||||
// process itself stays independent (liveness of the API is never blocked).
|
||||
if _, err := pool.Exec(ctx, `UPDATE worker_heartbeats SET last_seen_at = $1`, now.Add(-10*time.Minute)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
health = provider.Services(ctx)
|
||||
staleDown := false
|
||||
for _, item := range health {
|
||||
if (item.Key == "worker" || strings.HasPrefix(item.Key, "worker:")) && item.Liveness == LiveDown {
|
||||
staleDown = true
|
||||
}
|
||||
}
|
||||
if !staleDown {
|
||||
t.Fatal("stale heartbeat must mark worker down")
|
||||
}
|
||||
if ok, _ := overallReadiness(health); ok {
|
||||
t.Fatal("stale worker must block readiness")
|
||||
}
|
||||
// Tracker history: postgres was seen successful and keeps no failures.
|
||||
for _, item := range health {
|
||||
if item.Key == "postgres" {
|
||||
if item.LastSuccessAt == nil || item.ConsecutiveFailures != 0 {
|
||||
t.Fatalf("postgres tracker history = %v failures=%d", item.LastSuccessAt, item.ConsecutiveFailures)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,190 @@
|
||||
package monitoring
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"knowledge-governance/internal/platform/postgres"
|
||||
)
|
||||
|
||||
type metricsPool struct{ pool *pgxpool.Pool }
|
||||
|
||||
func (d metricsPool) Ping(ctx context.Context) error { return d.pool.Ping(ctx) }
|
||||
|
||||
func (d metricsPool) Query(ctx context.Context, sql string, args ...any) (pgx.Rows, error) {
|
||||
return d.pool.Query(ctx, sql, args...)
|
||||
}
|
||||
func (d metricsPool) QueryRow(ctx context.Context, sql string, args ...any) pgx.Row {
|
||||
return d.pool.QueryRow(ctx, sql, args...)
|
||||
}
|
||||
func (d metricsPool) Exec(ctx context.Context, sql string, args ...any) (pgconn.CommandTag, error) {
|
||||
return d.pool.Exec(ctx, sql, args...)
|
||||
}
|
||||
|
||||
func isolatedMetricsURL(t *testing.T) string {
|
||||
t.Helper()
|
||||
databaseURL := os.Getenv("TEST_DATABASE_URL")
|
||||
if databaseURL == "" {
|
||||
t.Skip("TEST_DATABASE_URL is not set")
|
||||
}
|
||||
parsed, err := url.Parse(databaseURL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
digest := sha256.Sum256([]byte(t.Name()))
|
||||
schema := "kgtest_" + hex.EncodeToString(digest[:8])
|
||||
ctx := context.Background()
|
||||
admin, err := pgxpool.New(ctx, databaseURL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer admin.Close()
|
||||
if _, err := admin.Exec(ctx, "CREATE SCHEMA "+schema); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
cleanup, err := pgxpool.New(context.Background(), databaseURL)
|
||||
if err == nil {
|
||||
_, _ = cleanup.Exec(context.Background(), "DROP SCHEMA IF EXISTS "+schema+" CASCADE")
|
||||
cleanup.Close()
|
||||
}
|
||||
})
|
||||
query := parsed.Query()
|
||||
query.Set("search_path", schema)
|
||||
parsed.RawQuery = query.Encode()
|
||||
return parsed.String()
|
||||
}
|
||||
|
||||
// TestRealMetricsWindowsAndLeases freezes docs plan P9: rates and percentiles
|
||||
// come from frozen 1h/24h windows, active workers count fresh distinct leases
|
||||
// only, jobs created outside the window never fake a duration, and empty
|
||||
// windows return nulls instead of 0.
|
||||
func TestRealMetricsWindowsAndLeases(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
pool, err := pgxpool.New(ctx, isolatedMetricsURL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer pool.Close()
|
||||
if err := postgres.ApplyMigrations(ctx, pool); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
now := time.Date(2026, time.September, 12, 8, 0, 0, 0, time.UTC)
|
||||
seed := func(id, state string, createdAt, updatedAt time.Time, owner string, leaseUntil *time.Time, attempt int) {
|
||||
var lease any
|
||||
if leaseUntil != nil {
|
||||
lease = *leaseUntil
|
||||
}
|
||||
var ownerArg any
|
||||
if owner != "" {
|
||||
ownerArg = owner
|
||||
}
|
||||
_, err := pool.Exec(ctx, `INSERT INTO jobs(id,type,payload,state,available_at,lease_owner,lease_until,attempt,max_attempts,idempotency_key,created_at,updated_at)
|
||||
VALUES($1::varchar,'test.upload','{}',$2,$4,$5,$6,$7,5,'k-'||$1::text,$3,$4)`, id, state, createdAt, updatedAt, ownerArg, lease, attempt)
|
||||
if err != nil {
|
||||
t.Fatalf("seed %s: %v", id, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Completed inside both windows with exact durations.
|
||||
for i, d := range []time.Duration{10 * time.Second, 20 * time.Second, 30 * time.Second, 15 * time.Minute, 30 * time.Minute} {
|
||||
updated := now.Add(-time.Duration(6-i) * time.Minute)
|
||||
seed(fmt.Sprintf("job-c%d", i), "completed", updated.Add(-d), updated, "", nil, 1)
|
||||
}
|
||||
// Failed: two inside 1h, one inside 24h only, one outside 24h.
|
||||
seed("job-f1", "failed", now.Add(-20*time.Minute), now.Add(-19*time.Minute), "w1", nil, 5)
|
||||
seed("job-f2", "failed", now.Add(-10*time.Minute), now.Add(-9*time.Minute), "w1", nil, 5)
|
||||
seed("job-f3", "failed", now.Add(-20*time.Hour), now.Add(-19*time.Hour), "w1", nil, 5)
|
||||
seed("job-f4", "failed", now.Add(-40*time.Hour), now.Add(-39*time.Hour), "w1", nil, 5)
|
||||
// Completed now but created 30h ago: duration unknowable, must not pollute percentiles.
|
||||
seed("job-old", "completed", now.Add(-30*time.Hour), now.Add(-10*time.Minute), "", nil, 1)
|
||||
// Fresh leases: two rows share wA, one wB, stale wC must not count.
|
||||
seed("job-run-a1", "running", now.Add(-5*time.Minute), now.Add(-time.Minute), "wA", ptrTime(now.Add(time.Minute)), 1)
|
||||
seed("job-run-a2", "running", now.Add(-4*time.Minute), now.Add(-time.Minute), "wA", ptrTime(now.Add(time.Minute)), 1)
|
||||
seed("job-run-b", "running", now.Add(-3*time.Minute), now.Add(-time.Minute), "wB", ptrTime(now.Add(time.Minute)), 1)
|
||||
seed("job-run-stale", "running", now.Add(-9*time.Hour), now.Add(-60*time.Minute), "wC", ptrTime(now.Add(-10*time.Minute)), 1)
|
||||
// Gauges.
|
||||
seed("job-pending", "pending", now.Add(-3*time.Hour), now.Add(-3*time.Hour), "", nil, 0)
|
||||
seed("job-retry", "retry", now.Add(-30*time.Minute), now.Add(-25*time.Minute), "", nil, 1)
|
||||
|
||||
provider, err := NewRealProvider(RealProviderConfig{
|
||||
DB: metricsPool{pool: pool},
|
||||
Now: func() time.Time { return now },
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
metrics, err := provider.Metrics(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("metrics: %v", err)
|
||||
}
|
||||
if !metrics.ObservedAt.Equal(now) || metrics.WorkerSource != "pg_lease" {
|
||||
t.Fatalf("observed/source = %s/%s", metrics.ObservedAt, metrics.WorkerSource)
|
||||
}
|
||||
if metrics.ActiveWorkers != 2 {
|
||||
t.Fatalf("active workers = %d, want 2 (fresh distinct owners only)", metrics.ActiveWorkers)
|
||||
}
|
||||
if metrics.Queue.Pending != 1 || metrics.Queue.Running != 4 || metrics.Queue.Retry != 1 || metrics.Queue.Failed != 4 {
|
||||
t.Fatalf("queue gauges = %+v", metrics.Queue)
|
||||
}
|
||||
if metrics.Queue.OldestPendingAgeSeconds == nil || int(*metrics.Queue.OldestPendingAgeSeconds) != 3*3600 {
|
||||
t.Fatalf("oldest pending = %v, want 10800s", metrics.Queue.OldestPendingAgeSeconds)
|
||||
}
|
||||
|
||||
short := metrics.ShortWindow
|
||||
// completed inside 1h: 5 seeded + job-old + job-c* (all updated within 1h except none) -> 6; failed 2.
|
||||
if short.WindowSeconds != 3600 || short.JobsCompleted != 6 || short.JobsFailed != 2 {
|
||||
t.Fatalf("short counts = %+v, want 6/2", short)
|
||||
}
|
||||
if short.SuccessRate == nil || *short.SuccessRate != 75 || short.FailureRate == nil || *short.FailureRate != 25 {
|
||||
t.Fatalf("short rates = %v/%v, want 75/25", short.SuccessRate, short.FailureRate)
|
||||
}
|
||||
// Durations measurable inside 1h: 10,20,30,900,1800 (job-old excluded).
|
||||
if short.DurationAvgSeconds == nil || *short.DurationAvgSeconds != 411.4 {
|
||||
t.Fatalf("short avg = %v, want 411.4(含2条60s失败样本)", *short.DurationAvgSeconds)
|
||||
}
|
||||
if short.DurationP95Seconds == nil || *short.DurationP95Seconds != 1800 {
|
||||
t.Fatalf("short p95 = %v, want 1800", *short.DurationP95Seconds)
|
||||
}
|
||||
|
||||
long := metrics.LongWindow
|
||||
if long.JobsFailed != 3 {
|
||||
t.Fatalf("long failures = %d, want 3 (job-f4 outside window)", long.JobsFailed)
|
||||
}
|
||||
// 8 measurable samples (5 completed + 2 failed@60s + job-f3 3600s): p50=60, p95=3600.
|
||||
if long.DurationP50Seconds == nil || *long.DurationP50Seconds != 60 {
|
||||
t.Fatalf("long p50 = %v, want 60", *long.DurationP50Seconds)
|
||||
}
|
||||
if long.DurationP95Seconds == nil || *long.DurationP95Seconds != 3600 {
|
||||
t.Fatalf("long p95 = %v, want 3600", *long.DurationP95Seconds)
|
||||
}
|
||||
|
||||
// Empty-window honesty: a fresh schema with no rows yields null rates.
|
||||
empty, err := NewRealProvider(RealProviderConfig{DB: metricsPool{pool: pool}, Now: func() time.Time { return now.Add(-100 * time.Hour) }})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
idle, err := empty.Metrics(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("idle metrics: %v", err)
|
||||
}
|
||||
if idle.ShortWindow.SuccessRate != nil || idle.ShortWindow.DurationAvgSeconds != nil {
|
||||
t.Fatalf("empty window must be null, got %+v", idle.ShortWindow)
|
||||
}
|
||||
if idle.Queue.OldestPendingAgeSeconds == nil {
|
||||
t.Fatalf("oldest pending must remain observable: %+v", idle.Queue)
|
||||
}
|
||||
}
|
||||
|
||||
func ptrTime(value time.Time) *time.Time { return &value }
|
||||
@@ -0,0 +1,320 @@
|
||||
package monitoring
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
)
|
||||
|
||||
type Database interface {
|
||||
Ping(context.Context) error
|
||||
Query(context.Context, string, ...any) (pgx.Rows, error)
|
||||
QueryRow(context.Context, string, ...any) pgx.Row
|
||||
}
|
||||
|
||||
type RealProviderConfig struct {
|
||||
DB Database
|
||||
Now func() time.Time
|
||||
HTTPClient *http.Client
|
||||
WeKnoraBaseURL string
|
||||
MinerUBaseURL string
|
||||
// WeKnoraReadiness optionally exercises a real read against a routing
|
||||
// knowledge base (nil = API key not configured; liveness-only).
|
||||
WeKnoraReadiness func(ctx context.Context) error
|
||||
// WorkerStaleAfter bounds how long a missing worker heartbeat still counts
|
||||
// as alive (P10).
|
||||
WorkerStaleAfter time.Duration
|
||||
}
|
||||
|
||||
type RealProvider struct {
|
||||
db Database
|
||||
now func() time.Time
|
||||
httpClient *http.Client
|
||||
weknoraBaseURL string
|
||||
minerUBaseURL string
|
||||
weknoraReadiness func(ctx context.Context) error
|
||||
workerStaleAfter time.Duration
|
||||
health *healthTracker
|
||||
}
|
||||
|
||||
func NewRealProvider(config RealProviderConfig) (*RealProvider, error) {
|
||||
if config.DB == nil {
|
||||
return nil, errors.New("monitor database is required")
|
||||
}
|
||||
if config.Now == nil {
|
||||
config.Now = time.Now
|
||||
}
|
||||
if config.HTTPClient == nil {
|
||||
config.HTTPClient = &http.Client{Timeout: 3 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}
|
||||
}
|
||||
stale := config.WorkerStaleAfter
|
||||
if stale <= 0 {
|
||||
stale = time.Minute
|
||||
}
|
||||
return &RealProvider{db: config.DB, now: config.Now, httpClient: config.HTTPClient, weknoraBaseURL: strings.TrimSpace(config.WeKnoraBaseURL), minerUBaseURL: strings.TrimSpace(config.MinerUBaseURL), weknoraReadiness: config.WeKnoraReadiness, workerStaleAfter: stale, health: newHealthTracker()}, nil
|
||||
}
|
||||
|
||||
func (provider *RealProvider) Overview(ctx context.Context) (governance.MonitorOverview, error) {
|
||||
now := provider.now().UTC()
|
||||
overview := governance.MonitorOverview{Mode: "real", Simulated: false, ObservedAt: now, Services: []governance.MonitorService{}, KnowledgeBases: []governance.KnowledgeBaseProgress{}}
|
||||
overview.Host = collectHostMetrics()
|
||||
overview.Services = append(overview.Services, provider.postgresHealth(ctx, now), provider.queueHealth(ctx, now))
|
||||
overview.Health = provider.Services(ctx)
|
||||
overview.Services = append(overview.Services,
|
||||
provider.httpHealth(ctx, "weknora", "WeKnora API", provider.weknoraBaseURL, now),
|
||||
governance.MonitorService{Key: "embedding", Name: "Embedding模型", State: "unknown", Detail: "由WeKnora管理,尚无独立探测端点", ObservedAt: now},
|
||||
governance.MonitorService{Key: "llm", Name: "LLM模型", State: "unknown", Detail: "由WeKnora管理,尚无独立探测端点", ObservedAt: now},
|
||||
provider.httpHealth(ctx, "mineru", "MinerU", provider.minerUBaseURL, now),
|
||||
governance.MonitorService{Key: "redis", Name: "Redis / MQ", State: "unknown", Detail: "未部署;任务由PostgreSQL持久队列承载", ObservedAt: now},
|
||||
governance.MonitorService{Key: "oss", Name: "OSS", State: "unknown", Detail: "未配置独立健康探测", ObservedAt: now},
|
||||
)
|
||||
rows, err := provider.db.Query(ctx, `SELECT p.test_kb_id,
|
||||
CASE WHEN COUNT(DISTINCT p.target_scope)=1 THEN MIN(p.target_scope) ELSE 'mixed' END,
|
||||
COUNT(*)::integer,
|
||||
COUNT(*) FILTER (WHERE p.processing_state='completed')::integer,
|
||||
COUNT(*) FILTER (WHERE p.processing_state='pending')::integer,
|
||||
COUNT(*) FILTER (WHERE p.processing_state='processing')::integer,
|
||||
COUNT(*) FILTER (WHERE p.processing_state='finalizing')::integer,
|
||||
COUNT(*) FILTER (WHERE p.processing_state IN ('failed','stalled'))::integer,
|
||||
COUNT(*) FILTER (WHERE p.target_scope='internal')::integer,
|
||||
COUNT(*) FILTER (WHERE p.target_scope='internal' AND p.processing_state='completed')::integer,
|
||||
COUNT(*) FILTER (WHERE p.target_scope='external')::integer,
|
||||
COUNT(*) FILTER (WHERE p.target_scope='external' AND p.processing_state='completed')::integer
|
||||
FROM test_publications p
|
||||
GROUP BY p.test_kb_id
|
||||
ORDER BY p.test_kb_id`)
|
||||
if err != nil {
|
||||
return governance.MonitorOverview{}, fmt.Errorf("list monitored knowledge bases: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var item governance.KnowledgeBaseProgress
|
||||
if err := rows.Scan(&item.ID, &item.Visibility, &item.Imported, &item.Completed, &item.Pending, &item.Processing, &item.Finalizing, &item.Failed, &item.InternalImported, &item.InternalCompleted, &item.ExternalImported, &item.ExternalCompleted); err != nil {
|
||||
return governance.MonitorOverview{}, fmt.Errorf("scan monitored knowledge base: %w", err)
|
||||
}
|
||||
switch item.Visibility {
|
||||
case "internal":
|
||||
item.Name = "对内知识库"
|
||||
case "external":
|
||||
item.Name = "对外知识库"
|
||||
default:
|
||||
item.Name = "测试知识库"
|
||||
}
|
||||
overview.KnowledgeBases = append(overview.KnowledgeBases, item)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return governance.MonitorOverview{}, fmt.Errorf("iterate monitored knowledge bases: %w", err)
|
||||
}
|
||||
return overview, nil
|
||||
}
|
||||
|
||||
func (provider *RealProvider) ListFiles(ctx context.Context, knowledgeBaseID, status string, limit, offset int) (governance.KnowledgeMonitorFilePage, error) {
|
||||
if strings.TrimSpace(knowledgeBaseID) == "" || limit < 1 || limit > 100 || offset < 0 || !validMonitorStatus(status) {
|
||||
return governance.KnowledgeMonitorFilePage{}, errors.New("invalid monitor file query")
|
||||
}
|
||||
page := governance.KnowledgeMonitorFilePage{Items: []governance.KnowledgeMonitorFile{}, Limit: limit, Offset: offset, SampleOnly: false}
|
||||
if err := provider.db.QueryRow(ctx, `SELECT COUNT(*)::integer FROM test_publications WHERE test_kb_id=$1 AND ($2='' OR CASE WHEN processing_state='stalled' THEN 'failed' ELSE processing_state END=$2)`, knowledgeBaseID, status).Scan(&page.Total); err != nil {
|
||||
return governance.KnowledgeMonitorFilePage{}, fmt.Errorf("count monitored files: %w", err)
|
||||
}
|
||||
rows, err := provider.db.Query(ctx, `SELECT item.title,item.normalized_path,publication.target_scope,
|
||||
CASE WHEN publication.processing_state='stalled' THEN 'failed' ELSE publication.processing_state END,
|
||||
GREATEST(0,EXTRACT(EPOCH FROM ($5-COALESCE(publication.processing_started_at,publication.updated_at)))::integer),
|
||||
publication.ingest_state,publication.last_error
|
||||
FROM test_publications publication
|
||||
JOIN source_item_versions version ON version.id=publication.source_item_version_id
|
||||
JOIN source_items item ON item.id=version.source_item_id
|
||||
WHERE publication.test_kb_id=$1
|
||||
AND ($2='' OR CASE WHEN publication.processing_state='stalled' THEN 'failed' ELSE publication.processing_state END=$2)
|
||||
ORDER BY CASE publication.processing_state WHEN 'failed' THEN 0 WHEN 'stalled' THEN 0 WHEN 'processing' THEN 1 WHEN 'finalizing' THEN 2 WHEN 'pending' THEN 3 ELSE 4 END,
|
||||
publication.updated_at DESC,publication.id
|
||||
LIMIT $3 OFFSET $4`, knowledgeBaseID, status, limit, offset, provider.now().UTC())
|
||||
if err != nil {
|
||||
return governance.KnowledgeMonitorFilePage{}, fmt.Errorf("list monitored files: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var item governance.KnowledgeMonitorFile
|
||||
var ingestState string
|
||||
if err := rows.Scan(&item.Name, &item.Directory, &item.Visibility, &item.Status, &item.ElapsedSecond, &ingestState, &item.ErrorCode); err != nil {
|
||||
return governance.KnowledgeMonitorFilePage{}, fmt.Errorf("scan monitored file: %w", err)
|
||||
}
|
||||
item.StageDetail = monitorStageDetail(ingestState, item.Status, item.ErrorCode)
|
||||
page.Items = append(page.Items, item)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return governance.KnowledgeMonitorFilePage{}, fmt.Errorf("iterate monitored files: %w", err)
|
||||
}
|
||||
return page, nil
|
||||
}
|
||||
|
||||
func (provider *RealProvider) postgresHealth(ctx context.Context, now time.Time) governance.MonitorService {
|
||||
started := time.Now()
|
||||
state, detail := "healthy", "可连接并可查询"
|
||||
if err := provider.db.Ping(ctx); err != nil {
|
||||
state, detail = "unhealthy", "连接或查询失败"
|
||||
}
|
||||
return governance.MonitorService{Key: "postgres", Name: "PostgreSQL", State: state, Detail: detail, LatencyMS: int(time.Since(started).Milliseconds()), ObservedAt: now}
|
||||
}
|
||||
|
||||
func (provider *RealProvider) queueHealth(ctx context.Context, now time.Time) governance.MonitorService {
|
||||
started := time.Now()
|
||||
var pending, running, failed int
|
||||
err := provider.db.QueryRow(ctx, `SELECT COUNT(*) FILTER (WHERE state IN ('pending','retry'))::integer,COUNT(*) FILTER (WHERE state='running')::integer,COUNT(*) FILTER (WHERE state='failed')::integer FROM jobs`).Scan(&pending, &running, &failed)
|
||||
state, detail := "healthy", fmt.Sprintf("可接单;等待%d,运行%d,失败%d", pending, running, failed)
|
||||
if err != nil {
|
||||
state, detail = "unhealthy", "队列不可查询"
|
||||
}
|
||||
return governance.MonitorService{Key: "queue", Name: "任务队列", State: state, Detail: detail, LatencyMS: int(time.Since(started).Milliseconds()), ObservedAt: now}
|
||||
}
|
||||
|
||||
func (provider *RealProvider) httpHealth(ctx context.Context, key, name, baseURL string, now time.Time) governance.MonitorService {
|
||||
if baseURL == "" {
|
||||
return governance.MonitorService{Key: key, Name: name, State: "unknown", Detail: "未配置探测地址", ObservedAt: now}
|
||||
}
|
||||
parsed, err := url.Parse(baseURL)
|
||||
if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Host == "" {
|
||||
return governance.MonitorService{Key: key, Name: name, State: "unhealthy", Detail: "探测配置无效", ObservedAt: now}
|
||||
}
|
||||
parsed.Path = strings.TrimSuffix(parsed.Path, "/") + "/health"
|
||||
parsed.RawQuery, parsed.Fragment = "", ""
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodGet, parsed.String(), nil)
|
||||
if err != nil {
|
||||
return governance.MonitorService{Key: key, Name: name, State: "unhealthy", Detail: "无法创建健康探测", ObservedAt: now}
|
||||
}
|
||||
started := time.Now()
|
||||
response, err := provider.httpClient.Do(request)
|
||||
latency := int(time.Since(started).Milliseconds())
|
||||
if err != nil {
|
||||
return governance.MonitorService{Key: key, Name: name, State: "unhealthy", Detail: "健康端点不可达", LatencyMS: latency, ObservedAt: now}
|
||||
}
|
||||
defer response.Body.Close()
|
||||
_, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 4096))
|
||||
if response.StatusCode < 200 || response.StatusCode >= 300 {
|
||||
return governance.MonitorService{Key: key, Name: name, State: "degraded", Detail: fmt.Sprintf("健康端点返回HTTP %d", response.StatusCode), LatencyMS: latency, ObservedAt: now}
|
||||
}
|
||||
return governance.MonitorService{Key: key, Name: name, State: "healthy", Detail: "健康端点可用", LatencyMS: latency, ObservedAt: now}
|
||||
}
|
||||
|
||||
func collectHostMetrics() governance.MonitorHost {
|
||||
host := governance.MonitorHost{GPUState: "unknown", GPUDetail: "容器未暴露GPU遥测", Simulated: false}
|
||||
first, err := os.ReadFile("/proc/stat")
|
||||
if err == nil {
|
||||
idle1, total1, parseErr := parseCPUStat(string(first))
|
||||
if parseErr == nil {
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
if second, readErr := os.ReadFile("/proc/stat"); readErr == nil {
|
||||
idle2, total2, secondErr := parseCPUStat(string(second))
|
||||
if secondErr == nil {
|
||||
host.CPUPercent = cpuPercent(idle1, total1, idle2, total2)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if raw, err := os.ReadFile("/proc/meminfo"); err == nil {
|
||||
host.MemoryUsedBytes, host.MemoryTotalBytes, _ = parseMemoryInfo(string(raw))
|
||||
}
|
||||
var stats syscall.Statfs_t
|
||||
if syscall.Statfs("/", &stats) == nil {
|
||||
host.DiskTotalBytes = int64(stats.Blocks) * int64(stats.Bsize)
|
||||
host.DiskUsedBytes = int64(stats.Blocks-stats.Bavail) * int64(stats.Bsize)
|
||||
}
|
||||
if entries, err := os.ReadDir("/proc/driver/nvidia/gpus"); err == nil && len(entries) > 0 {
|
||||
host.GPUState, host.GPUDetail = "healthy", fmt.Sprintf("检测到%d块NVIDIA GPU;未读取利用率", len(entries))
|
||||
}
|
||||
return host
|
||||
}
|
||||
|
||||
func parseCPUStat(raw string) (idle, total uint64, err error) {
|
||||
line := strings.SplitN(raw, "\n", 2)[0]
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) < 5 || fields[0] != "cpu" {
|
||||
return 0, 0, errors.New("invalid cpu stat")
|
||||
}
|
||||
values := make([]uint64, 0, len(fields)-1)
|
||||
for _, field := range fields[1:] {
|
||||
value, parseErr := strconv.ParseUint(field, 10, 64)
|
||||
if parseErr != nil {
|
||||
return 0, 0, errors.New("invalid cpu counter")
|
||||
}
|
||||
values = append(values, value)
|
||||
total += value
|
||||
}
|
||||
idle = values[3]
|
||||
if len(values) > 4 {
|
||||
idle += values[4]
|
||||
}
|
||||
return idle, total, nil
|
||||
}
|
||||
|
||||
func parseMemoryInfo(raw string) (used, total int64, err error) {
|
||||
values := map[string]int64{}
|
||||
for _, line := range strings.Split(raw, "\n") {
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) < 2 {
|
||||
continue
|
||||
}
|
||||
key := strings.TrimSuffix(fields[0], ":")
|
||||
if key != "MemTotal" && key != "MemAvailable" {
|
||||
continue
|
||||
}
|
||||
value, parseErr := strconv.ParseInt(fields[1], 10, 64)
|
||||
if parseErr != nil || value < 0 {
|
||||
return 0, 0, errors.New("invalid memory counter")
|
||||
}
|
||||
values[key] = value * 1024
|
||||
}
|
||||
total, totalOK := values["MemTotal"]
|
||||
available, availableOK := values["MemAvailable"]
|
||||
if !totalOK || !availableOK || available > total {
|
||||
return 0, 0, errors.New("incomplete memory info")
|
||||
}
|
||||
return total - available, total, nil
|
||||
}
|
||||
|
||||
func cpuPercent(idle1, total1, idle2, total2 uint64) float64 {
|
||||
if total2 <= total1 || idle2 < idle1 || idle2-idle1 > total2-total1 {
|
||||
return 0
|
||||
}
|
||||
return float64((total2-total1)-(idle2-idle1)) * 100 / float64(total2-total1)
|
||||
}
|
||||
|
||||
func validMonitorStatus(status string) bool {
|
||||
switch status {
|
||||
case "", "completed", "pending", "processing", "finalizing", "failed", "cancelled":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func monitorStageDetail(ingestState, processingState, errorCode string) string {
|
||||
if errorCode != "" {
|
||||
return "处理失败,可查看错误码后重试"
|
||||
}
|
||||
switch {
|
||||
case ingestState == "pending":
|
||||
return "等待确认或上传"
|
||||
case ingestState == "uploading":
|
||||
return "正在上传测试知识库"
|
||||
case processingState == "processing":
|
||||
return "WeKnora正在解析、切片或向量化"
|
||||
case processingState == "finalizing":
|
||||
return "基础解析完成,索引或增强处理中"
|
||||
case processingState == "completed":
|
||||
return "解析完成,等待或已完成验收"
|
||||
default:
|
||||
return "等待远端状态更新"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
package monitoring
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestParseCPUStatAndMemoryInfo(t *testing.T) {
|
||||
idle, total, err := parseCPUStat("cpu 100 20 30 400 50 6 7 8 0 0\n")
|
||||
if err != nil || idle != 450 || total != 621 {
|
||||
t.Fatalf("idle=%d total=%d err=%v", idle, total, err)
|
||||
}
|
||||
used, memoryTotal, err := parseMemoryInfo("MemTotal: 1000 kB\nMemAvailable: 250 kB\n")
|
||||
if err != nil || used != 750*1024 || memoryTotal != 1000*1024 {
|
||||
t.Fatalf("used=%d total=%d err=%v", used, memoryTotal, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCPUPercentUsesTwoMonotonicSamples(t *testing.T) {
|
||||
value := cpuPercent(450, 621, 500, 721)
|
||||
if value != 50 {
|
||||
t.Fatalf("cpu percent=%v, want 50", value)
|
||||
}
|
||||
if cpuPercent(500, 721, 450, 621) != 0 {
|
||||
t.Fatal("non-monotonic CPU counters must fail closed to zero")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
package monitoring
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
)
|
||||
|
||||
type SimulationProvider struct{ now func() time.Time }
|
||||
|
||||
func NewSimulationProvider(now func() time.Time) *SimulationProvider {
|
||||
if now == nil {
|
||||
now = time.Now
|
||||
}
|
||||
return &SimulationProvider{now: now}
|
||||
}
|
||||
|
||||
func (provider *SimulationProvider) Overview(context.Context) (governance.MonitorOverview, error) {
|
||||
now := provider.now().UTC()
|
||||
return governance.MonitorOverview{
|
||||
Mode: "simulation", Simulated: true, ObservedAt: now,
|
||||
Host: governance.MonitorHost{CPUPercent: 42, MemoryUsedBytes: 19971597926, MemoryTotalBytes: 34359738368, DiskUsedBytes: 128849018880, DiskTotalBytes: 536870912000, GPUState: "unknown", GPUDetail: "未接入真实GPU监控", Simulated: true},
|
||||
Services: []governance.MonitorService{
|
||||
{Key: "weknora", Name: "WeKnora API", State: "healthy", Detail: "模拟连接正常", LatencyMS: 86, ObservedAt: now, Simulated: true},
|
||||
{Key: "embedding", Name: "Embedding模型", State: "healthy", Detail: "模拟探测通过", LatencyMS: 142, ObservedAt: now, Simulated: true},
|
||||
{Key: "llm", Name: "LLM模型", State: "healthy", Detail: "模拟探测通过", LatencyMS: 318, ObservedAt: now, Simulated: true},
|
||||
{Key: "mineru", Name: "MinerU", State: "healthy", Detail: "模拟处理中2,队列5", LatencyMS: 24, ObservedAt: now, Simulated: true},
|
||||
{Key: "postgres", Name: "PostgreSQL", State: "unknown", Detail: "未接入真实监控", ObservedAt: now, Simulated: true},
|
||||
{Key: "redis", Name: "Redis / MQ", State: "unknown", Detail: "未接入真实监控", ObservedAt: now, Simulated: true},
|
||||
{Key: "oss", Name: "OSS", State: "unknown", Detail: "未接入真实监控", ObservedAt: now, Simulated: true},
|
||||
},
|
||||
KnowledgeBases: []governance.KnowledgeBaseProgress{{ID: "simulation-company-support", Name: "公司级技术支持知识库", Visibility: "mixed", Imported: 5750, Completed: 4530, Pending: 450, Processing: 50, Finalizing: 714, Failed: 6, InternalImported: 4200, InternalCompleted: 3300, ExternalImported: 1550, ExternalCompleted: 1230}},
|
||||
}, nil
|
||||
}
|
||||
|
||||
var simulationFiles = []governance.KnowledgeMonitorFile{
|
||||
{Name: "[模拟] CR5用户手册.pdf", Directory: "02|产品资料/CR系列", Visibility: "internal", Status: "completed", StageDetail: "解析、切片和增强任务均完成", ElapsedSecond: 0},
|
||||
{Name: "[模拟] MG400部署说明.pdf", Directory: "03|机器人部署与使用", Visibility: "internal", Status: "pending", StageDetail: "等待进入解析队列", ElapsedSecond: 420},
|
||||
{Name: "[模拟] 产品参数总表.xlsx", Directory: "02|产品资料", Visibility: "external", Status: "finalizing", StageDetail: "基础检索可用,Wiki与图谱仍在生成", ElapsedSecond: 2100},
|
||||
{Name: "[模拟] 二次开发接口说明.docx", Directory: "05|二次开发", Visibility: "internal", Status: "processing", StageDetail: "正在切片和向量化", ElapsedSecond: 480},
|
||||
{Name: "[模拟] CRA-IP68产品介绍.pdf", Directory: "06|应用案例与解决方案", Visibility: "external", Status: "failed", StageDetail: "MinerU解析失败", ElapsedSecond: 720, ErrorCode: "mineru_parse_failed"},
|
||||
}
|
||||
|
||||
func (provider *SimulationProvider) ListFiles(_ context.Context, knowledgeBaseID, status string, limit, offset int) (governance.KnowledgeMonitorFilePage, error) {
|
||||
if knowledgeBaseID != "simulation-company-support" || limit < 1 || limit > 100 || offset < 0 {
|
||||
return governance.KnowledgeMonitorFilePage{}, errors.New("invalid monitor file query")
|
||||
}
|
||||
allowed := map[string]bool{"": true, "completed": true, "pending": true, "processing": true, "finalizing": true, "failed": true, "cancelled": true}
|
||||
if !allowed[status] {
|
||||
return governance.KnowledgeMonitorFilePage{}, errors.New("invalid monitor status")
|
||||
}
|
||||
filtered := make([]governance.KnowledgeMonitorFile, 0)
|
||||
for _, item := range simulationFiles {
|
||||
if status == "" || item.Status == status {
|
||||
filtered = append(filtered, item)
|
||||
}
|
||||
}
|
||||
page := governance.KnowledgeMonitorFilePage{Total: len(filtered), Limit: limit, Offset: offset, SampleOnly: true, Items: []governance.KnowledgeMonitorFile{}}
|
||||
if offset < len(filtered) {
|
||||
end := offset + limit
|
||||
if end > len(filtered) {
|
||||
end = len(filtered)
|
||||
}
|
||||
page.Items = append(page.Items, filtered[offset:end]...)
|
||||
}
|
||||
return page, nil
|
||||
}
|
||||
|
||||
// Metrics declares that no real run metrics exist in simulation mode: every
|
||||
// sample-dependent number stays null instead of pretending to be zero (P9).
|
||||
func (provider *SimulationProvider) Metrics(context.Context) (governance.RunMetrics, error) {
|
||||
now := provider.now().UTC()
|
||||
return governance.RunMetrics{
|
||||
ObservedAt: now,
|
||||
WorkerSource: "simulation",
|
||||
ShortWindow: governance.JobWindowMetrics{WindowSeconds: int((1 * time.Hour).Seconds())},
|
||||
LongWindow: governance.JobWindowMetrics{WindowSeconds: int((24 * time.Hour).Seconds())},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Services declares that a simulation provider owns no real dependencies: every
|
||||
// readiness judgement belongs to the real provider.
|
||||
func (provider *SimulationProvider) Services(_ context.Context) []governance.ServiceHealth {
|
||||
return []governance.ServiceHealth{}
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
package monitoring
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestSimulationProviderLabelsEveryValueAndReconcilesKnowledgeCounts(t *testing.T) {
|
||||
now := time.Date(2026, 9, 8, 12, 0, 0, 0, time.UTC)
|
||||
provider := NewSimulationProvider(func() time.Time { return now })
|
||||
overview, err := provider.Overview(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if overview.Mode != "simulation" || !overview.Simulated || overview.ObservedAt != now || len(overview.Services) < 4 || len(overview.KnowledgeBases) != 1 {
|
||||
t.Fatalf("overview=%#v", overview)
|
||||
}
|
||||
if overview.Host.DiskTotalBytes <= 0 || overview.Host.DiskUsedBytes <= 0 || overview.Host.GPUState != "unknown" || overview.Host.GPUDetail != "未接入真实GPU监控" {
|
||||
t.Fatalf("host resources=%#v", overview.Host)
|
||||
}
|
||||
requiredServices := map[string]bool{"weknora": false, "mineru": false, "postgres": false, "redis": false, "oss": false, "embedding": false, "llm": false}
|
||||
for _, service := range overview.Services {
|
||||
if _, ok := requiredServices[service.Key]; ok {
|
||||
requiredServices[service.Key] = true
|
||||
}
|
||||
}
|
||||
for key, present := range requiredServices {
|
||||
if !present {
|
||||
t.Errorf("missing service status %q", key)
|
||||
}
|
||||
}
|
||||
kb := overview.KnowledgeBases[0]
|
||||
if kb.Imported != 5750 || kb.Completed != 4530 || kb.Pending != 450 || kb.Finalizing != 714 || kb.Processing != 50 || kb.Failed != 6 {
|
||||
t.Fatalf("kb=%#v", kb)
|
||||
}
|
||||
if kb.Completed+kb.Pending+kb.Finalizing+kb.Processing+kb.Failed+kb.Cancelled != kb.Imported {
|
||||
t.Fatalf("status counts do not reconcile: %#v", kb)
|
||||
}
|
||||
for _, service := range overview.Services {
|
||||
if !service.Simulated {
|
||||
t.Fatalf("unlabelled service=%#v", service)
|
||||
}
|
||||
}
|
||||
files, err := provider.ListFiles(context.Background(), kb.ID, "failed", 20, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if files.Total != 1 || len(files.Items) != 1 || files.Items[0].Status != "failed" || files.Items[0].Name[:len("[模拟]")] != "[模拟]" || !files.SampleOnly {
|
||||
t.Fatalf("files=%#v", files)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user