knowledge-governance/internal/monitoring/real.go

package monitoring

import (
	"context"
	"errors"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"strings"
	"syscall"
	"time"

	"github.com/jackc/pgx/v5"

	"knowledge-governance/internal/governance"
)

type Database interface {
	Ping(context.Context) error
	Query(context.Context, string, ...any) (pgx.Rows, error)
	QueryRow(context.Context, string, ...any) pgx.Row
}

type RealProviderConfig struct {
	DB             Database
	Now            func() time.Time
	HTTPClient     *http.Client
	WeKnoraBaseURL string
	MinerUBaseURL  string
	// WeKnoraReadiness optionally exercises a real read against a routing
	// knowledge base (nil = API key not configured; liveness-only).
	WeKnoraReadiness func(ctx context.Context) error
	// WorkerStaleAfter bounds how long a missing worker heartbeat still counts
	// as alive (P10).
	WorkerStaleAfter time.Duration
}

type RealProvider struct {
	db               Database
	now              func() time.Time
	httpClient       *http.Client
	weknoraBaseURL   string
	minerUBaseURL    string
	weknoraReadiness func(ctx context.Context) error
	workerStaleAfter time.Duration
	health           *healthTracker
}

func NewRealProvider(config RealProviderConfig) (*RealProvider, error) {
	if config.DB == nil {
		return nil, errors.New("monitor database is required")
	}
	if config.Now == nil {
		config.Now = time.Now
	}
	if config.HTTPClient == nil {
		config.HTTPClient = &http.Client{Timeout: 3 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}
	}
	stale := config.WorkerStaleAfter
	if stale <= 0 {
		stale = time.Minute
	}
	return &RealProvider{db: config.DB, now: config.Now, httpClient: config.HTTPClient, weknoraBaseURL: strings.TrimSpace(config.WeKnoraBaseURL), minerUBaseURL: strings.TrimSpace(config.MinerUBaseURL), weknoraReadiness: config.WeKnoraReadiness, workerStaleAfter: stale, health: newHealthTracker()}, nil
}

func (provider *RealProvider) Overview(ctx context.Context) (governance.MonitorOverview, error) {
	now := provider.now().UTC()
	overview := governance.MonitorOverview{Mode: "real", Simulated: false, ObservedAt: now, Services: []governance.MonitorService{}, KnowledgeBases: []governance.KnowledgeBaseProgress{}}
	overview.Host = collectHostMetrics()
	overview.Services = append(overview.Services, provider.postgresHealth(ctx, now), provider.queueHealth(ctx, now))
	overview.Health = provider.Services(ctx)
	overview.Services = append(overview.Services,
		provider.httpHealth(ctx, "weknora", "WeKnora API", provider.weknoraBaseURL, now),
		governance.MonitorService{Key: "embedding", Name: "Embedding模型", State: "unknown", Detail: "由WeKnora管理，尚无独立探测端点", ObservedAt: now},
		governance.MonitorService{Key: "llm", Name: "LLM模型", State: "unknown", Detail: "由WeKnora管理，尚无独立探测端点", ObservedAt: now},
		provider.httpHealth(ctx, "mineru", "MinerU", provider.minerUBaseURL, now),
		governance.MonitorService{Key: "redis", Name: "Redis / MQ", State: "unknown", Detail: "未部署；任务由PostgreSQL持久队列承载", ObservedAt: now},
		governance.MonitorService{Key: "oss", Name: "OSS", State: "unknown", Detail: "未配置独立健康探测", ObservedAt: now},
	)
	rows, err := provider.db.Query(ctx, `SELECT p.test_kb_id,
       CASE WHEN COUNT(DISTINCT p.target_scope)=1 THEN MIN(p.target_scope) ELSE 'mixed' END,
       COUNT(*)::integer,
       COUNT(*) FILTER (WHERE p.processing_state='completed')::integer,
       COUNT(*) FILTER (WHERE p.processing_state='pending')::integer,
       COUNT(*) FILTER (WHERE p.processing_state='processing')::integer,
       COUNT(*) FILTER (WHERE p.processing_state='finalizing')::integer,
       COUNT(*) FILTER (WHERE p.processing_state IN ('failed','stalled'))::integer,
       COUNT(*) FILTER (WHERE p.target_scope='internal')::integer,
       COUNT(*) FILTER (WHERE p.target_scope='internal' AND p.processing_state='completed')::integer,
       COUNT(*) FILTER (WHERE p.target_scope='external')::integer,
       COUNT(*) FILTER (WHERE p.target_scope='external' AND p.processing_state='completed')::integer
FROM test_publications p
GROUP BY p.test_kb_id
ORDER BY p.test_kb_id`)
	if err != nil {
		return governance.MonitorOverview{}, fmt.Errorf("list monitored knowledge bases: %w", err)
	}
	defer rows.Close()
	for rows.Next() {
		var item governance.KnowledgeBaseProgress
		if err := rows.Scan(&item.ID, &item.Visibility, &item.Imported, &item.Completed, &item.Pending, &item.Processing, &item.Finalizing, &item.Failed, &item.InternalImported, &item.InternalCompleted, &item.ExternalImported, &item.ExternalCompleted); err != nil {
			return governance.MonitorOverview{}, fmt.Errorf("scan monitored knowledge base: %w", err)
		}
		switch item.Visibility {
		case "internal":
			item.Name = "对内知识库"
		case "external":
			item.Name = "对外知识库"
		default:
			item.Name = "测试知识库"
		}
		overview.KnowledgeBases = append(overview.KnowledgeBases, item)
	}
	if err := rows.Err(); err != nil {
		return governance.MonitorOverview{}, fmt.Errorf("iterate monitored knowledge bases: %w", err)
	}
	return overview, nil
}

func (provider *RealProvider) ListFiles(ctx context.Context, knowledgeBaseID, status string, limit, offset int) (governance.KnowledgeMonitorFilePage, error) {
	if strings.TrimSpace(knowledgeBaseID) == "" || limit < 1 || limit > 100 || offset < 0 || !validMonitorStatus(status) {
		return governance.KnowledgeMonitorFilePage{}, errors.New("invalid monitor file query")
	}
	page := governance.KnowledgeMonitorFilePage{Items: []governance.KnowledgeMonitorFile{}, Limit: limit, Offset: offset, SampleOnly: false}
	if err := provider.db.QueryRow(ctx, `SELECT COUNT(*)::integer FROM test_publications WHERE test_kb_id=$1 AND ($2='' OR CASE WHEN processing_state='stalled' THEN 'failed' ELSE processing_state END=$2)`, knowledgeBaseID, status).Scan(&page.Total); err != nil {
		return governance.KnowledgeMonitorFilePage{}, fmt.Errorf("count monitored files: %w", err)
	}
	rows, err := provider.db.Query(ctx, `SELECT item.title,item.normalized_path,publication.target_scope,
       CASE WHEN publication.processing_state='stalled' THEN 'failed' ELSE publication.processing_state END,
       GREATEST(0,EXTRACT(EPOCH FROM ($5-COALESCE(publication.processing_started_at,publication.updated_at)))::integer),
       publication.ingest_state,publication.last_error
FROM test_publications publication
JOIN source_item_versions version ON version.id=publication.source_item_version_id
JOIN source_items item ON item.id=version.source_item_id
WHERE publication.test_kb_id=$1
  AND ($2='' OR CASE WHEN publication.processing_state='stalled' THEN 'failed' ELSE publication.processing_state END=$2)
ORDER BY CASE publication.processing_state WHEN 'failed' THEN 0 WHEN 'stalled' THEN 0 WHEN 'processing' THEN 1 WHEN 'finalizing' THEN 2 WHEN 'pending' THEN 3 ELSE 4 END,
         publication.updated_at DESC,publication.id
LIMIT $3 OFFSET $4`, knowledgeBaseID, status, limit, offset, provider.now().UTC())
	if err != nil {
		return governance.KnowledgeMonitorFilePage{}, fmt.Errorf("list monitored files: %w", err)
	}
	defer rows.Close()
	for rows.Next() {
		var item governance.KnowledgeMonitorFile
		var ingestState string
		if err := rows.Scan(&item.Name, &item.Directory, &item.Visibility, &item.Status, &item.ElapsedSecond, &ingestState, &item.ErrorCode); err != nil {
			return governance.KnowledgeMonitorFilePage{}, fmt.Errorf("scan monitored file: %w", err)
		}
		item.StageDetail = monitorStageDetail(ingestState, item.Status, item.ErrorCode)
		page.Items = append(page.Items, item)
	}
	if err := rows.Err(); err != nil {
		return governance.KnowledgeMonitorFilePage{}, fmt.Errorf("iterate monitored files: %w", err)
	}
	return page, nil
}

func (provider *RealProvider) postgresHealth(ctx context.Context, now time.Time) governance.MonitorService {
	started := time.Now()
	state, detail := "healthy", "可连接并可查询"
	if err := provider.db.Ping(ctx); err != nil {
		state, detail = "unhealthy", "连接或查询失败"
	}
	return governance.MonitorService{Key: "postgres", Name: "PostgreSQL", State: state, Detail: detail, LatencyMS: int(time.Since(started).Milliseconds()), ObservedAt: now}
}

func (provider *RealProvider) queueHealth(ctx context.Context, now time.Time) governance.MonitorService {
	started := time.Now()
	var pending, running, failed int
	err := provider.db.QueryRow(ctx, `SELECT COUNT(*) FILTER (WHERE state IN ('pending','retry'))::integer,COUNT(*) FILTER (WHERE state='running')::integer,COUNT(*) FILTER (WHERE state='failed')::integer FROM jobs`).Scan(&pending, &running, &failed)
	state, detail := "healthy", fmt.Sprintf("可接单；等待%d，运行%d，失败%d", pending, running, failed)
	if err != nil {
		state, detail = "unhealthy", "队列不可查询"
	}
	return governance.MonitorService{Key: "queue", Name: "任务队列", State: state, Detail: detail, LatencyMS: int(time.Since(started).Milliseconds()), ObservedAt: now}
}

func (provider *RealProvider) httpHealth(ctx context.Context, key, name, baseURL string, now time.Time) governance.MonitorService {
	if baseURL == "" {
		return governance.MonitorService{Key: key, Name: name, State: "unknown", Detail: "未配置探测地址", ObservedAt: now}
	}
	parsed, err := url.Parse(baseURL)
	if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Host == "" {
		return governance.MonitorService{Key: key, Name: name, State: "unhealthy", Detail: "探测配置无效", ObservedAt: now}
	}
	parsed.Path = strings.TrimSuffix(parsed.Path, "/") + "/health"
	parsed.RawQuery, parsed.Fragment = "", ""
	request, err := http.NewRequestWithContext(ctx, http.MethodGet, parsed.String(), nil)
	if err != nil {
		return governance.MonitorService{Key: key, Name: name, State: "unhealthy", Detail: "无法创建健康探测", ObservedAt: now}
	}
	started := time.Now()
	response, err := provider.httpClient.Do(request)
	latency := int(time.Since(started).Milliseconds())
	if err != nil {
		return governance.MonitorService{Key: key, Name: name, State: "unhealthy", Detail: "健康端点不可达", LatencyMS: latency, ObservedAt: now}
	}
	defer response.Body.Close()
	_, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 4096))
	if response.StatusCode < 200 || response.StatusCode >= 300 {
		return governance.MonitorService{Key: key, Name: name, State: "degraded", Detail: fmt.Sprintf("健康端点返回HTTP %d", response.StatusCode), LatencyMS: latency, ObservedAt: now}
	}
	return governance.MonitorService{Key: key, Name: name, State: "healthy", Detail: "健康端点可用", LatencyMS: latency, ObservedAt: now}
}

func collectHostMetrics() governance.MonitorHost {
	host := governance.MonitorHost{GPUState: "unknown", GPUDetail: "容器未暴露GPU遥测", Simulated: false}
	first, err := os.ReadFile("/proc/stat")
	if err == nil {
		idle1, total1, parseErr := parseCPUStat(string(first))
		if parseErr == nil {
			time.Sleep(100 * time.Millisecond)
			if second, readErr := os.ReadFile("/proc/stat"); readErr == nil {
				idle2, total2, secondErr := parseCPUStat(string(second))
				if secondErr == nil {
					host.CPUPercent = cpuPercent(idle1, total1, idle2, total2)
				}
			}
		}
	}
	if raw, err := os.ReadFile("/proc/meminfo"); err == nil {
		host.MemoryUsedBytes, host.MemoryTotalBytes, _ = parseMemoryInfo(string(raw))
	}
	var stats syscall.Statfs_t
	if syscall.Statfs("/", &stats) == nil {
		host.DiskTotalBytes = int64(stats.Blocks) * int64(stats.Bsize)
		host.DiskUsedBytes = int64(stats.Blocks-stats.Bavail) * int64(stats.Bsize)
	}
	if entries, err := os.ReadDir("/proc/driver/nvidia/gpus"); err == nil && len(entries) > 0 {
		host.GPUState, host.GPUDetail = "healthy", fmt.Sprintf("检测到%d块NVIDIA GPU；未读取利用率", len(entries))
	}
	return host
}

func parseCPUStat(raw string) (idle, total uint64, err error) {
	line := strings.SplitN(raw, "\n", 2)[0]
	fields := strings.Fields(line)
	if len(fields) < 5 || fields[0] != "cpu" {
		return 0, 0, errors.New("invalid cpu stat")
	}
	values := make([]uint64, 0, len(fields)-1)
	for _, field := range fields[1:] {
		value, parseErr := strconv.ParseUint(field, 10, 64)
		if parseErr != nil {
			return 0, 0, errors.New("invalid cpu counter")
		}
		values = append(values, value)
		total += value
	}
	idle = values[3]
	if len(values) > 4 {
		idle += values[4]
	}
	return idle, total, nil
}

func parseMemoryInfo(raw string) (used, total int64, err error) {
	values := map[string]int64{}
	for _, line := range strings.Split(raw, "\n") {
		fields := strings.Fields(line)
		if len(fields) < 2 {
			continue
		}
		key := strings.TrimSuffix(fields[0], ":")
		if key != "MemTotal" && key != "MemAvailable" {
			continue
		}
		value, parseErr := strconv.ParseInt(fields[1], 10, 64)
		if parseErr != nil || value < 0 {
			return 0, 0, errors.New("invalid memory counter")
		}
		values[key] = value * 1024
	}
	total, totalOK := values["MemTotal"]
	available, availableOK := values["MemAvailable"]
	if !totalOK || !availableOK || available > total {
		return 0, 0, errors.New("incomplete memory info")
	}
	return total - available, total, nil
}

func cpuPercent(idle1, total1, idle2, total2 uint64) float64 {
	if total2 <= total1 || idle2 < idle1 || idle2-idle1 > total2-total1 {
		return 0
	}
	return float64((total2-total1)-(idle2-idle1)) * 100 / float64(total2-total1)
}

func validMonitorStatus(status string) bool {
	switch status {
	case "", "completed", "pending", "processing", "finalizing", "failed", "cancelled":
		return true
	default:
		return false
	}
}

func monitorStageDetail(ingestState, processingState, errorCode string) string {
	if errorCode != "" {
		return "处理失败，可查看错误码后重试"
	}
	switch {
	case ingestState == "pending":
		return "等待确认或上传"
	case ingestState == "uploading":
		return "正在上传测试知识库"
	case processingState == "processing":
		return "WeKnora正在解析、切片或向量化"
	case processingState == "finalizing":
		return "基础解析完成，索引或增强处理中"
	case processingState == "completed":
		return "解析完成，等待或已完成验收"
	default:
		return "等待远端状态更新"
	}
}