package monitoring import ( "context" "errors" "fmt" "io" "net/http" "net/url" "os" "strconv" "strings" "syscall" "time" "github.com/jackc/pgx/v5" "knowledge-governance/internal/governance" ) type Database interface { Ping(context.Context) error Query(context.Context, string, ...any) (pgx.Rows, error) QueryRow(context.Context, string, ...any) pgx.Row } type RealProviderConfig struct { DB Database Now func() time.Time HTTPClient *http.Client WeKnoraBaseURL string MinerUBaseURL string // TestWeKnoraResolver reads the test endpoint an admin saved in the web // settings, so 「服务配置」is the single place the console probes. The env // value above stays the bootstrap fallback for containers wired before the // UI existed. TestWeKnoraResolver func(ctx context.Context) string // ProductionWeKnoraBaseURL lets the console show the real state of the // production WeKnora endpoint. Publishing itself keeps reading its target // from the release runtime config, never from here. ProductionWeKnoraBaseURL string // ProductionWeKnoraResolver reads the endpoint an admin saved in the web // settings. That value is what the console probes — the web page is the // place operators configure it — and the env value above stays the // bootstrap fallback for containers that were wired before the UI existed. ProductionWeKnoraResolver func(ctx context.Context) string // WeKnoraReadiness optionally exercises a real read against a routing // knowledge base (nil = API key not configured; liveness-only). WeKnoraReadiness func(ctx context.Context) error // WorkerStaleAfter bounds how long a missing worker heartbeat still counts // as alive (P10). WorkerStaleAfter time.Duration } type RealProvider struct { db Database now func() time.Time httpClient *http.Client weknoraBaseURL string minerUBaseURL string testWeKnoraResolver func(ctx context.Context) string productionWeKnoraBaseURL string productionWeKnoraResolver func(ctx context.Context) string weknoraReadiness func(ctx context.Context) error workerStaleAfter time.Duration health *healthTracker } // resolveTestWeKnoraBaseURL prefers the admin-saved web setting and falls back to // the env bootstrap value, so the page is the place to change the test endpoint. func (provider *RealProvider) resolveTestWeKnoraBaseURL(ctx context.Context) string { if provider.testWeKnoraResolver != nil { if value := strings.TrimSpace(provider.testWeKnoraResolver(ctx)); value != "" { return value } } return provider.weknoraBaseURL } // resolveProductionWeKnoraBaseURL prefers the admin-saved web setting and falls // back to the env bootstrap value: the page wins, the container env only fills in // when nobody configured it in the UI. func (provider *RealProvider) resolveProductionWeKnoraBaseURL(ctx context.Context) string { if provider.productionWeKnoraResolver != nil { if value := strings.TrimSpace(provider.productionWeKnoraResolver(ctx)); value != "" { return value } } return provider.productionWeKnoraBaseURL } func NewRealProvider(config RealProviderConfig) (*RealProvider, error) { if config.DB == nil { return nil, errors.New("monitor database is required") } if config.Now == nil { config.Now = time.Now } if config.HTTPClient == nil { config.HTTPClient = &http.Client{Timeout: 3 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }} } stale := config.WorkerStaleAfter if stale <= 0 { stale = time.Minute } return &RealProvider{db: config.DB, now: config.Now, httpClient: config.HTTPClient, weknoraBaseURL: strings.TrimSpace(config.WeKnoraBaseURL), minerUBaseURL: strings.TrimSpace(config.MinerUBaseURL), testWeKnoraResolver: config.TestWeKnoraResolver, productionWeKnoraBaseURL: strings.TrimSpace(config.ProductionWeKnoraBaseURL), productionWeKnoraResolver: config.ProductionWeKnoraResolver, weknoraReadiness: config.WeKnoraReadiness, workerStaleAfter: stale, health: newHealthTracker()}, nil } func (provider *RealProvider) Overview(ctx context.Context) (governance.MonitorOverview, error) { now := provider.now().UTC() overview := governance.MonitorOverview{Mode: "real", Simulated: false, ObservedAt: now, Services: []governance.MonitorService{}, KnowledgeBases: []governance.KnowledgeBaseProgress{}} overview.Host = collectHostMetrics() overview.Services = append(overview.Services, provider.postgresHealth(ctx, now), provider.queueHealth(ctx, now)) overview.Health = provider.Services(ctx) overview.Services = append(overview.Services, provider.httpHealth(ctx, "weknora", "WeKnora API", provider.resolveTestWeKnoraBaseURL(ctx), now), provider.httpHealth(ctx, "weknora-production", "生产 WeKnora", provider.productionWeKnoraBaseURL, now), governance.MonitorService{Key: "embedding", Name: "Embedding模型", State: "unknown", Detail: "由WeKnora管理,尚无独立探测端点", ObservedAt: now}, governance.MonitorService{Key: "llm", Name: "LLM模型", State: "unknown", Detail: "由WeKnora管理,尚无独立探测端点", ObservedAt: now}, provider.httpHealth(ctx, "mineru", "MinerU", provider.minerUBaseURL, now), governance.MonitorService{Key: "redis", Name: "Redis / MQ", State: "unknown", Detail: "未部署;任务由PostgreSQL持久队列承载", ObservedAt: now}, governance.MonitorService{Key: "oss", Name: "OSS", State: "unknown", Detail: "未配置独立健康探测", ObservedAt: now}, ) rows, err := provider.db.Query(ctx, `SELECT p.test_kb_id, CASE WHEN COUNT(DISTINCT p.target_scope)=1 THEN MIN(p.target_scope) ELSE 'mixed' END, COUNT(*) FILTER (WHERE p.superseded_at IS NULL)::integer, COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.processing_state='completed')::integer, COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.processing_state='pending')::integer, COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.processing_state='processing')::integer, COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.processing_state='finalizing')::integer, COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.processing_state IN ('failed','stalled'))::integer, COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.target_scope='internal')::integer, COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.target_scope='internal' AND p.processing_state='completed')::integer, COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.target_scope='external')::integer, COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.target_scope='external' AND p.processing_state='completed')::integer, COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.processing_state='planned')::integer, -- 需下架:在测试库里仍然存活(已完成、有知识 id、尚未标记移除)、但来源项已被判为禁止导入的条目。 -- 曾经这里判 p.retrieval_state='enabled',而 retrieval_state 在这套状态机里描述的是 -- “是否发布到生产”(unpublished/disabling/disabled/failed),全仓没有任何代码写过 'enabled', -- 所以这条车道恒为 0 —— 2026-10-10 界面验收实测:把已入库的文件改成“禁止导入”后 -- 车道仍显示 0,界面承诺的“仍在知识库里的文件会出现在这里”不成立。 COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.knowledge_id<>'' AND p.processing_state='completed' AND p.retrieval_state NOT IN ('disabling','disabled') AND item.policy_state='blocked')::integer, COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.processing_state='planned' AND item.policy_state='blocked')::integer FROM test_publications p JOIN source_item_versions version ON version.id=p.source_item_version_id JOIN source_items item ON item.id=version.source_item_id GROUP BY p.test_kb_id ORDER BY p.test_kb_id`) if err != nil { return governance.MonitorOverview{}, fmt.Errorf("list monitored knowledge bases: %w", err) } defer rows.Close() for rows.Next() { var item governance.KnowledgeBaseProgress if err := rows.Scan(&item.ID, &item.Visibility, &item.Imported, &item.Completed, &item.Pending, &item.Processing, &item.Finalizing, &item.Failed, &item.InternalImported, &item.InternalCompleted, &item.ExternalImported, &item.ExternalCompleted, &item.Planned, &item.NeedsRemoval, &item.StalePlanned); err != nil { return governance.MonitorOverview{}, fmt.Errorf("scan monitored knowledge base: %w", err) } switch item.Visibility { case "internal": item.Name = "对内知识库" case "external": item.Name = "对外知识库" default: item.Name = "测试知识库" } overview.KnowledgeBases = append(overview.KnowledgeBases, item) } if err := rows.Err(); err != nil { return governance.MonitorOverview{}, fmt.Errorf("iterate monitored knowledge bases: %w", err) } if err := provider.db.QueryRow(ctx, `SELECT COUNT(*) FILTER (WHERE fetch_state IN ('pending','fetching'))::integer, COUNT(*) FILTER (WHERE fetch_state='failed')::integer, COUNT(*) FILTER (WHERE fetch_state='missing_local')::integer, COALESCE(MAX((SELECT ingestion_concurrency FROM ui_setting_revisions WHERE state='active' LIMIT 1)),2)::integer FROM catalog_import_intents`).Scan(&overview.Pipeline.FetchPending, &overview.Pipeline.FetchFailed, &overview.Pipeline.FetchMissing, &overview.Pipeline.Concurrency); err != nil { return governance.MonitorOverview{}, fmt.Errorf("summarize ingestion pipeline: %w", err) } return overview, nil } func (provider *RealProvider) ListFiles(ctx context.Context, knowledgeBaseID, status string, limit, offset int) (governance.KnowledgeMonitorFilePage, error) { if strings.TrimSpace(knowledgeBaseID) == "" || limit < 1 || limit > 100 || offset < 0 || !validMonitorStatus(status) { return governance.KnowledgeMonitorFilePage{}, errors.New("invalid monitor file query") } page := governance.KnowledgeMonitorFilePage{Items: []governance.KnowledgeMonitorFile{}, Limit: limit, Offset: offset, SampleOnly: false} if err := provider.db.QueryRow(ctx, `SELECT COUNT(*)::integer FROM test_publications WHERE test_kb_id=$1 AND ($2='' OR CASE WHEN processing_state='stalled' THEN 'failed' ELSE processing_state END=$2)`, knowledgeBaseID, status).Scan(&page.Total); err != nil { return governance.KnowledgeMonitorFilePage{}, fmt.Errorf("count monitored files: %w", err) } rows, err := provider.db.Query(ctx, `SELECT item.title,item.normalized_path,publication.target_scope, CASE WHEN publication.processing_state='stalled' THEN 'failed' ELSE publication.processing_state END, GREATEST(0,EXTRACT(EPOCH FROM ($5-COALESCE(publication.processing_started_at,publication.updated_at)))::integer), publication.ingest_state,publication.last_error FROM test_publications publication JOIN source_item_versions version ON version.id=publication.source_item_version_id JOIN source_items item ON item.id=version.source_item_id WHERE publication.test_kb_id=$1 AND ($2='' OR CASE WHEN publication.processing_state='stalled' THEN 'failed' ELSE publication.processing_state END=$2) ORDER BY CASE publication.processing_state WHEN 'failed' THEN 0 WHEN 'stalled' THEN 0 WHEN 'processing' THEN 1 WHEN 'finalizing' THEN 2 WHEN 'pending' THEN 3 ELSE 4 END, publication.updated_at DESC,publication.id LIMIT $3 OFFSET $4`, knowledgeBaseID, status, limit, offset, provider.now().UTC()) if err != nil { return governance.KnowledgeMonitorFilePage{}, fmt.Errorf("list monitored files: %w", err) } defer rows.Close() for rows.Next() { var item governance.KnowledgeMonitorFile var ingestState string if err := rows.Scan(&item.Name, &item.Directory, &item.Visibility, &item.Status, &item.ElapsedSecond, &ingestState, &item.ErrorCode); err != nil { return governance.KnowledgeMonitorFilePage{}, fmt.Errorf("scan monitored file: %w", err) } item.StageDetail = monitorStageDetail(ingestState, item.Status, item.ErrorCode) page.Items = append(page.Items, item) } if err := rows.Err(); err != nil { return governance.KnowledgeMonitorFilePage{}, fmt.Errorf("iterate monitored files: %w", err) } return page, nil } func (provider *RealProvider) postgresHealth(ctx context.Context, now time.Time) governance.MonitorService { started := time.Now() state, detail := "healthy", "可连接并可查询" if err := provider.db.Ping(ctx); err != nil { state, detail = "unhealthy", "连接或查询失败" } return governance.MonitorService{Key: "postgres", Name: "PostgreSQL", State: state, Detail: detail, LatencyMS: int(time.Since(started).Milliseconds()), ObservedAt: now} } func (provider *RealProvider) queueHealth(ctx context.Context, now time.Time) governance.MonitorService { started := time.Now() var pending, running, failed int err := provider.db.QueryRow(ctx, `SELECT COUNT(*) FILTER (WHERE state IN ('pending','retry'))::integer,COUNT(*) FILTER (WHERE state='running')::integer,COUNT(*) FILTER (WHERE state='failed')::integer FROM jobs`).Scan(&pending, &running, &failed) state, detail := "healthy", fmt.Sprintf("可接单;等待%d,运行%d,失败%d", pending, running, failed) if err != nil { state, detail = "unhealthy", "队列不可查询" } return governance.MonitorService{Key: "queue", Name: "任务队列", State: state, Detail: detail, LatencyMS: int(time.Since(started).Milliseconds()), ObservedAt: now} } func (provider *RealProvider) httpHealth(ctx context.Context, key, name, baseURL string, now time.Time) governance.MonitorService { if baseURL == "" { return governance.MonitorService{Key: key, Name: name, State: "unknown", Detail: "未配置探测地址", ObservedAt: now} } parsed, err := url.Parse(baseURL) if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Host == "" { return governance.MonitorService{Key: key, Name: name, State: "unhealthy", Detail: "探测配置无效", ObservedAt: now} } parsed.Path = strings.TrimSuffix(parsed.Path, "/") + "/health" parsed.RawQuery, parsed.Fragment = "", "" request, err := http.NewRequestWithContext(ctx, http.MethodGet, parsed.String(), nil) if err != nil { return governance.MonitorService{Key: key, Name: name, State: "unhealthy", Detail: "无法创建健康探测", ObservedAt: now} } started := time.Now() response, err := provider.httpClient.Do(request) latency := int(time.Since(started).Milliseconds()) if err != nil { return governance.MonitorService{Key: key, Name: name, State: "unhealthy", Detail: "健康端点不可达", LatencyMS: latency, ObservedAt: now} } defer response.Body.Close() _, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 4096)) if response.StatusCode < 200 || response.StatusCode >= 300 { return governance.MonitorService{Key: key, Name: name, State: "degraded", Detail: fmt.Sprintf("健康端点返回HTTP %d", response.StatusCode), LatencyMS: latency, ObservedAt: now} } return governance.MonitorService{Key: key, Name: name, State: "healthy", Detail: "健康端点可用", LatencyMS: latency, ObservedAt: now} } func collectHostMetrics() governance.MonitorHost { host := governance.MonitorHost{GPUState: "unknown", GPUDetail: "容器未暴露GPU遥测", Simulated: false} first, err := os.ReadFile("/proc/stat") if err == nil { idle1, total1, parseErr := parseCPUStat(string(first)) if parseErr == nil { time.Sleep(100 * time.Millisecond) if second, readErr := os.ReadFile("/proc/stat"); readErr == nil { idle2, total2, secondErr := parseCPUStat(string(second)) if secondErr == nil { host.CPUPercent = cpuPercent(idle1, total1, idle2, total2) } } } } if raw, err := os.ReadFile("/proc/meminfo"); err == nil { host.MemoryUsedBytes, host.MemoryTotalBytes, _ = parseMemoryInfo(string(raw)) } var stats syscall.Statfs_t if syscall.Statfs("/", &stats) == nil { host.DiskTotalBytes = int64(stats.Blocks) * int64(stats.Bsize) host.DiskUsedBytes = int64(stats.Blocks-stats.Bavail) * int64(stats.Bsize) } if entries, err := os.ReadDir("/proc/driver/nvidia/gpus"); err == nil && len(entries) > 0 { host.GPUState, host.GPUDetail = "healthy", fmt.Sprintf("检测到%d块NVIDIA GPU;未读取利用率", len(entries)) } return host } func parseCPUStat(raw string) (idle, total uint64, err error) { line := strings.SplitN(raw, "\n", 2)[0] fields := strings.Fields(line) if len(fields) < 5 || fields[0] != "cpu" { return 0, 0, errors.New("invalid cpu stat") } values := make([]uint64, 0, len(fields)-1) for _, field := range fields[1:] { value, parseErr := strconv.ParseUint(field, 10, 64) if parseErr != nil { return 0, 0, errors.New("invalid cpu counter") } values = append(values, value) total += value } idle = values[3] if len(values) > 4 { idle += values[4] } return idle, total, nil } func parseMemoryInfo(raw string) (used, total int64, err error) { values := map[string]int64{} for _, line := range strings.Split(raw, "\n") { fields := strings.Fields(line) if len(fields) < 2 { continue } key := strings.TrimSuffix(fields[0], ":") if key != "MemTotal" && key != "MemAvailable" { continue } value, parseErr := strconv.ParseInt(fields[1], 10, 64) if parseErr != nil || value < 0 { return 0, 0, errors.New("invalid memory counter") } values[key] = value * 1024 } total, totalOK := values["MemTotal"] available, availableOK := values["MemAvailable"] if !totalOK || !availableOK || available > total { return 0, 0, errors.New("incomplete memory info") } return total - available, total, nil } func cpuPercent(idle1, total1, idle2, total2 uint64) float64 { if total2 <= total1 || idle2 < idle1 || idle2-idle1 > total2-total1 { return 0 } return float64((total2-total1)-(idle2-idle1)) * 100 / float64(total2-total1) } func validMonitorStatus(status string) bool { switch status { case "", "completed", "pending", "processing", "finalizing", "failed", "cancelled": return true default: return false } } func monitorStageDetail(ingestState, processingState, errorCode string) string { if errorCode != "" { return "处理失败,可查看错误码后重试" } switch { case ingestState == "pending": return "等待确认或上传" case ingestState == "uploading": return "正在上传测试知识库" case processingState == "processing": return "WeKnora正在解析、切片或向量化" case processingState == "finalizing": return "基础解析完成,索引或增强处理中" case processingState == "completed": return "解析完成,等待或已完成验收" default: return "等待远端状态更新" } }