1db21fb899
- 生产/测试 WeKnora 地址、租户 ID、知识库 ID 由静态 env 收敛为版本化 UI 设置,保留修订号与操作人审计 - 新增 serviceverify:保存前真调目标端点探测,端点无效才拒绝;网络抖动或无凭据仅告警 - 迁移 000042(生产 WeKnora 端点)/000043(服务目标);补齐设置候选与校验契约用例
378 lines
18 KiB
Go
378 lines
18 KiB
Go
package monitoring
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/url"
|
|
"os"
|
|
"strconv"
|
|
"strings"
|
|
"syscall"
|
|
"time"
|
|
|
|
"github.com/jackc/pgx/v5"
|
|
|
|
"knowledge-governance/internal/governance"
|
|
)
|
|
|
|
type Database interface {
|
|
Ping(context.Context) error
|
|
Query(context.Context, string, ...any) (pgx.Rows, error)
|
|
QueryRow(context.Context, string, ...any) pgx.Row
|
|
}
|
|
|
|
type RealProviderConfig struct {
|
|
DB Database
|
|
Now func() time.Time
|
|
HTTPClient *http.Client
|
|
WeKnoraBaseURL string
|
|
MinerUBaseURL string
|
|
// TestWeKnoraResolver reads the test endpoint an admin saved in the web
|
|
// settings, so 「服务配置」is the single place the console probes. The env
|
|
// value above stays the bootstrap fallback for containers wired before the
|
|
// UI existed.
|
|
TestWeKnoraResolver func(ctx context.Context) string
|
|
// ProductionWeKnoraBaseURL lets the console show the real state of the
|
|
// production WeKnora endpoint. Publishing itself keeps reading its target
|
|
// from the release runtime config, never from here.
|
|
ProductionWeKnoraBaseURL string
|
|
// ProductionWeKnoraResolver reads the endpoint an admin saved in the web
|
|
// settings. That value is what the console probes — the web page is the
|
|
// place operators configure it — and the env value above stays the
|
|
// bootstrap fallback for containers that were wired before the UI existed.
|
|
ProductionWeKnoraResolver func(ctx context.Context) string
|
|
// WeKnoraReadiness optionally exercises a real read against a routing
|
|
// knowledge base (nil = API key not configured; liveness-only).
|
|
WeKnoraReadiness func(ctx context.Context) error
|
|
// WorkerStaleAfter bounds how long a missing worker heartbeat still counts
|
|
// as alive (P10).
|
|
WorkerStaleAfter time.Duration
|
|
}
|
|
|
|
type RealProvider struct {
|
|
db Database
|
|
now func() time.Time
|
|
httpClient *http.Client
|
|
weknoraBaseURL string
|
|
minerUBaseURL string
|
|
testWeKnoraResolver func(ctx context.Context) string
|
|
productionWeKnoraBaseURL string
|
|
productionWeKnoraResolver func(ctx context.Context) string
|
|
weknoraReadiness func(ctx context.Context) error
|
|
workerStaleAfter time.Duration
|
|
health *healthTracker
|
|
}
|
|
|
|
// resolveTestWeKnoraBaseURL prefers the admin-saved web setting and falls back to
|
|
// the env bootstrap value, so the page is the place to change the test endpoint.
|
|
func (provider *RealProvider) resolveTestWeKnoraBaseURL(ctx context.Context) string {
|
|
if provider.testWeKnoraResolver != nil {
|
|
if value := strings.TrimSpace(provider.testWeKnoraResolver(ctx)); value != "" {
|
|
return value
|
|
}
|
|
}
|
|
return provider.weknoraBaseURL
|
|
}
|
|
|
|
// resolveProductionWeKnoraBaseURL prefers the admin-saved web setting and falls
|
|
// back to the env bootstrap value: the page wins, the container env only fills in
|
|
// when nobody configured it in the UI.
|
|
func (provider *RealProvider) resolveProductionWeKnoraBaseURL(ctx context.Context) string {
|
|
if provider.productionWeKnoraResolver != nil {
|
|
if value := strings.TrimSpace(provider.productionWeKnoraResolver(ctx)); value != "" {
|
|
return value
|
|
}
|
|
}
|
|
return provider.productionWeKnoraBaseURL
|
|
}
|
|
|
|
func NewRealProvider(config RealProviderConfig) (*RealProvider, error) {
|
|
if config.DB == nil {
|
|
return nil, errors.New("monitor database is required")
|
|
}
|
|
if config.Now == nil {
|
|
config.Now = time.Now
|
|
}
|
|
if config.HTTPClient == nil {
|
|
config.HTTPClient = &http.Client{Timeout: 3 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}
|
|
}
|
|
stale := config.WorkerStaleAfter
|
|
if stale <= 0 {
|
|
stale = time.Minute
|
|
}
|
|
return &RealProvider{db: config.DB, now: config.Now, httpClient: config.HTTPClient, weknoraBaseURL: strings.TrimSpace(config.WeKnoraBaseURL), minerUBaseURL: strings.TrimSpace(config.MinerUBaseURL), testWeKnoraResolver: config.TestWeKnoraResolver, productionWeKnoraBaseURL: strings.TrimSpace(config.ProductionWeKnoraBaseURL), productionWeKnoraResolver: config.ProductionWeKnoraResolver, weknoraReadiness: config.WeKnoraReadiness, workerStaleAfter: stale, health: newHealthTracker()}, nil
|
|
}
|
|
|
|
func (provider *RealProvider) Overview(ctx context.Context) (governance.MonitorOverview, error) {
|
|
now := provider.now().UTC()
|
|
overview := governance.MonitorOverview{Mode: "real", Simulated: false, ObservedAt: now, Services: []governance.MonitorService{}, KnowledgeBases: []governance.KnowledgeBaseProgress{}}
|
|
overview.Host = collectHostMetrics()
|
|
overview.Services = append(overview.Services, provider.postgresHealth(ctx, now), provider.queueHealth(ctx, now))
|
|
overview.Health = provider.Services(ctx)
|
|
overview.Services = append(overview.Services,
|
|
provider.httpHealth(ctx, "weknora", "WeKnora API", provider.resolveTestWeKnoraBaseURL(ctx), now),
|
|
provider.httpHealth(ctx, "weknora-production", "生产 WeKnora", provider.productionWeKnoraBaseURL, now),
|
|
governance.MonitorService{Key: "embedding", Name: "Embedding模型", State: "unknown", Detail: "由WeKnora管理,尚无独立探测端点", ObservedAt: now},
|
|
governance.MonitorService{Key: "llm", Name: "LLM模型", State: "unknown", Detail: "由WeKnora管理,尚无独立探测端点", ObservedAt: now},
|
|
provider.httpHealth(ctx, "mineru", "MinerU", provider.minerUBaseURL, now),
|
|
governance.MonitorService{Key: "redis", Name: "Redis / MQ", State: "unknown", Detail: "未部署;任务由PostgreSQL持久队列承载", ObservedAt: now},
|
|
governance.MonitorService{Key: "oss", Name: "OSS", State: "unknown", Detail: "未配置独立健康探测", ObservedAt: now},
|
|
)
|
|
rows, err := provider.db.Query(ctx, `SELECT p.test_kb_id,
|
|
CASE WHEN COUNT(DISTINCT p.target_scope)=1 THEN MIN(p.target_scope) ELSE 'mixed' END,
|
|
COUNT(*) FILTER (WHERE p.superseded_at IS NULL)::integer,
|
|
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.processing_state='completed')::integer,
|
|
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.processing_state='pending')::integer,
|
|
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.processing_state='processing')::integer,
|
|
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.processing_state='finalizing')::integer,
|
|
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.processing_state IN ('failed','stalled'))::integer,
|
|
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.target_scope='internal')::integer,
|
|
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.target_scope='internal' AND p.processing_state='completed')::integer,
|
|
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.target_scope='external')::integer,
|
|
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.target_scope='external' AND p.processing_state='completed')::integer,
|
|
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.processing_state='planned')::integer,
|
|
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.retrieval_state='enabled'
|
|
AND item.policy_state='blocked')::integer,
|
|
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.processing_state='planned'
|
|
AND item.policy_state='blocked')::integer
|
|
FROM test_publications p
|
|
JOIN source_item_versions version ON version.id=p.source_item_version_id
|
|
JOIN source_items item ON item.id=version.source_item_id
|
|
GROUP BY p.test_kb_id
|
|
ORDER BY p.test_kb_id`)
|
|
if err != nil {
|
|
return governance.MonitorOverview{}, fmt.Errorf("list monitored knowledge bases: %w", err)
|
|
}
|
|
defer rows.Close()
|
|
for rows.Next() {
|
|
var item governance.KnowledgeBaseProgress
|
|
if err := rows.Scan(&item.ID, &item.Visibility, &item.Imported, &item.Completed, &item.Pending, &item.Processing, &item.Finalizing, &item.Failed, &item.InternalImported, &item.InternalCompleted, &item.ExternalImported, &item.ExternalCompleted,
|
|
&item.Planned, &item.NeedsRemoval, &item.StalePlanned); err != nil {
|
|
return governance.MonitorOverview{}, fmt.Errorf("scan monitored knowledge base: %w", err)
|
|
}
|
|
switch item.Visibility {
|
|
case "internal":
|
|
item.Name = "对内知识库"
|
|
case "external":
|
|
item.Name = "对外知识库"
|
|
default:
|
|
item.Name = "测试知识库"
|
|
}
|
|
overview.KnowledgeBases = append(overview.KnowledgeBases, item)
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
return governance.MonitorOverview{}, fmt.Errorf("iterate monitored knowledge bases: %w", err)
|
|
}
|
|
if err := provider.db.QueryRow(ctx, `SELECT
|
|
COUNT(*) FILTER (WHERE fetch_state IN ('pending','fetching'))::integer,
|
|
COUNT(*) FILTER (WHERE fetch_state='failed')::integer,
|
|
COUNT(*) FILTER (WHERE fetch_state='missing_local')::integer,
|
|
COALESCE(MAX((SELECT ingestion_concurrency FROM ui_setting_revisions WHERE state='active' LIMIT 1)),2)::integer
|
|
FROM catalog_import_intents`).Scan(&overview.Pipeline.FetchPending, &overview.Pipeline.FetchFailed, &overview.Pipeline.FetchMissing, &overview.Pipeline.Concurrency); err != nil {
|
|
return governance.MonitorOverview{}, fmt.Errorf("summarize ingestion pipeline: %w", err)
|
|
}
|
|
return overview, nil
|
|
}
|
|
|
|
func (provider *RealProvider) ListFiles(ctx context.Context, knowledgeBaseID, status string, limit, offset int) (governance.KnowledgeMonitorFilePage, error) {
|
|
if strings.TrimSpace(knowledgeBaseID) == "" || limit < 1 || limit > 100 || offset < 0 || !validMonitorStatus(status) {
|
|
return governance.KnowledgeMonitorFilePage{}, errors.New("invalid monitor file query")
|
|
}
|
|
page := governance.KnowledgeMonitorFilePage{Items: []governance.KnowledgeMonitorFile{}, Limit: limit, Offset: offset, SampleOnly: false}
|
|
if err := provider.db.QueryRow(ctx, `SELECT COUNT(*)::integer FROM test_publications WHERE test_kb_id=$1 AND ($2='' OR CASE WHEN processing_state='stalled' THEN 'failed' ELSE processing_state END=$2)`, knowledgeBaseID, status).Scan(&page.Total); err != nil {
|
|
return governance.KnowledgeMonitorFilePage{}, fmt.Errorf("count monitored files: %w", err)
|
|
}
|
|
rows, err := provider.db.Query(ctx, `SELECT item.title,item.normalized_path,publication.target_scope,
|
|
CASE WHEN publication.processing_state='stalled' THEN 'failed' ELSE publication.processing_state END,
|
|
GREATEST(0,EXTRACT(EPOCH FROM ($5-COALESCE(publication.processing_started_at,publication.updated_at)))::integer),
|
|
publication.ingest_state,publication.last_error
|
|
FROM test_publications publication
|
|
JOIN source_item_versions version ON version.id=publication.source_item_version_id
|
|
JOIN source_items item ON item.id=version.source_item_id
|
|
WHERE publication.test_kb_id=$1
|
|
AND ($2='' OR CASE WHEN publication.processing_state='stalled' THEN 'failed' ELSE publication.processing_state END=$2)
|
|
ORDER BY CASE publication.processing_state WHEN 'failed' THEN 0 WHEN 'stalled' THEN 0 WHEN 'processing' THEN 1 WHEN 'finalizing' THEN 2 WHEN 'pending' THEN 3 ELSE 4 END,
|
|
publication.updated_at DESC,publication.id
|
|
LIMIT $3 OFFSET $4`, knowledgeBaseID, status, limit, offset, provider.now().UTC())
|
|
if err != nil {
|
|
return governance.KnowledgeMonitorFilePage{}, fmt.Errorf("list monitored files: %w", err)
|
|
}
|
|
defer rows.Close()
|
|
for rows.Next() {
|
|
var item governance.KnowledgeMonitorFile
|
|
var ingestState string
|
|
if err := rows.Scan(&item.Name, &item.Directory, &item.Visibility, &item.Status, &item.ElapsedSecond, &ingestState, &item.ErrorCode); err != nil {
|
|
return governance.KnowledgeMonitorFilePage{}, fmt.Errorf("scan monitored file: %w", err)
|
|
}
|
|
item.StageDetail = monitorStageDetail(ingestState, item.Status, item.ErrorCode)
|
|
page.Items = append(page.Items, item)
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
return governance.KnowledgeMonitorFilePage{}, fmt.Errorf("iterate monitored files: %w", err)
|
|
}
|
|
return page, nil
|
|
}
|
|
|
|
func (provider *RealProvider) postgresHealth(ctx context.Context, now time.Time) governance.MonitorService {
|
|
started := time.Now()
|
|
state, detail := "healthy", "可连接并可查询"
|
|
if err := provider.db.Ping(ctx); err != nil {
|
|
state, detail = "unhealthy", "连接或查询失败"
|
|
}
|
|
return governance.MonitorService{Key: "postgres", Name: "PostgreSQL", State: state, Detail: detail, LatencyMS: int(time.Since(started).Milliseconds()), ObservedAt: now}
|
|
}
|
|
|
|
func (provider *RealProvider) queueHealth(ctx context.Context, now time.Time) governance.MonitorService {
|
|
started := time.Now()
|
|
var pending, running, failed int
|
|
err := provider.db.QueryRow(ctx, `SELECT COUNT(*) FILTER (WHERE state IN ('pending','retry'))::integer,COUNT(*) FILTER (WHERE state='running')::integer,COUNT(*) FILTER (WHERE state='failed')::integer FROM jobs`).Scan(&pending, &running, &failed)
|
|
state, detail := "healthy", fmt.Sprintf("可接单;等待%d,运行%d,失败%d", pending, running, failed)
|
|
if err != nil {
|
|
state, detail = "unhealthy", "队列不可查询"
|
|
}
|
|
return governance.MonitorService{Key: "queue", Name: "任务队列", State: state, Detail: detail, LatencyMS: int(time.Since(started).Milliseconds()), ObservedAt: now}
|
|
}
|
|
|
|
func (provider *RealProvider) httpHealth(ctx context.Context, key, name, baseURL string, now time.Time) governance.MonitorService {
|
|
if baseURL == "" {
|
|
return governance.MonitorService{Key: key, Name: name, State: "unknown", Detail: "未配置探测地址", ObservedAt: now}
|
|
}
|
|
parsed, err := url.Parse(baseURL)
|
|
if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Host == "" {
|
|
return governance.MonitorService{Key: key, Name: name, State: "unhealthy", Detail: "探测配置无效", ObservedAt: now}
|
|
}
|
|
parsed.Path = strings.TrimSuffix(parsed.Path, "/") + "/health"
|
|
parsed.RawQuery, parsed.Fragment = "", ""
|
|
request, err := http.NewRequestWithContext(ctx, http.MethodGet, parsed.String(), nil)
|
|
if err != nil {
|
|
return governance.MonitorService{Key: key, Name: name, State: "unhealthy", Detail: "无法创建健康探测", ObservedAt: now}
|
|
}
|
|
started := time.Now()
|
|
response, err := provider.httpClient.Do(request)
|
|
latency := int(time.Since(started).Milliseconds())
|
|
if err != nil {
|
|
return governance.MonitorService{Key: key, Name: name, State: "unhealthy", Detail: "健康端点不可达", LatencyMS: latency, ObservedAt: now}
|
|
}
|
|
defer response.Body.Close()
|
|
_, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 4096))
|
|
if response.StatusCode < 200 || response.StatusCode >= 300 {
|
|
return governance.MonitorService{Key: key, Name: name, State: "degraded", Detail: fmt.Sprintf("健康端点返回HTTP %d", response.StatusCode), LatencyMS: latency, ObservedAt: now}
|
|
}
|
|
return governance.MonitorService{Key: key, Name: name, State: "healthy", Detail: "健康端点可用", LatencyMS: latency, ObservedAt: now}
|
|
}
|
|
|
|
func collectHostMetrics() governance.MonitorHost {
|
|
host := governance.MonitorHost{GPUState: "unknown", GPUDetail: "容器未暴露GPU遥测", Simulated: false}
|
|
first, err := os.ReadFile("/proc/stat")
|
|
if err == nil {
|
|
idle1, total1, parseErr := parseCPUStat(string(first))
|
|
if parseErr == nil {
|
|
time.Sleep(100 * time.Millisecond)
|
|
if second, readErr := os.ReadFile("/proc/stat"); readErr == nil {
|
|
idle2, total2, secondErr := parseCPUStat(string(second))
|
|
if secondErr == nil {
|
|
host.CPUPercent = cpuPercent(idle1, total1, idle2, total2)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if raw, err := os.ReadFile("/proc/meminfo"); err == nil {
|
|
host.MemoryUsedBytes, host.MemoryTotalBytes, _ = parseMemoryInfo(string(raw))
|
|
}
|
|
var stats syscall.Statfs_t
|
|
if syscall.Statfs("/", &stats) == nil {
|
|
host.DiskTotalBytes = int64(stats.Blocks) * int64(stats.Bsize)
|
|
host.DiskUsedBytes = int64(stats.Blocks-stats.Bavail) * int64(stats.Bsize)
|
|
}
|
|
if entries, err := os.ReadDir("/proc/driver/nvidia/gpus"); err == nil && len(entries) > 0 {
|
|
host.GPUState, host.GPUDetail = "healthy", fmt.Sprintf("检测到%d块NVIDIA GPU;未读取利用率", len(entries))
|
|
}
|
|
return host
|
|
}
|
|
|
|
func parseCPUStat(raw string) (idle, total uint64, err error) {
|
|
line := strings.SplitN(raw, "\n", 2)[0]
|
|
fields := strings.Fields(line)
|
|
if len(fields) < 5 || fields[0] != "cpu" {
|
|
return 0, 0, errors.New("invalid cpu stat")
|
|
}
|
|
values := make([]uint64, 0, len(fields)-1)
|
|
for _, field := range fields[1:] {
|
|
value, parseErr := strconv.ParseUint(field, 10, 64)
|
|
if parseErr != nil {
|
|
return 0, 0, errors.New("invalid cpu counter")
|
|
}
|
|
values = append(values, value)
|
|
total += value
|
|
}
|
|
idle = values[3]
|
|
if len(values) > 4 {
|
|
idle += values[4]
|
|
}
|
|
return idle, total, nil
|
|
}
|
|
|
|
func parseMemoryInfo(raw string) (used, total int64, err error) {
|
|
values := map[string]int64{}
|
|
for _, line := range strings.Split(raw, "\n") {
|
|
fields := strings.Fields(line)
|
|
if len(fields) < 2 {
|
|
continue
|
|
}
|
|
key := strings.TrimSuffix(fields[0], ":")
|
|
if key != "MemTotal" && key != "MemAvailable" {
|
|
continue
|
|
}
|
|
value, parseErr := strconv.ParseInt(fields[1], 10, 64)
|
|
if parseErr != nil || value < 0 {
|
|
return 0, 0, errors.New("invalid memory counter")
|
|
}
|
|
values[key] = value * 1024
|
|
}
|
|
total, totalOK := values["MemTotal"]
|
|
available, availableOK := values["MemAvailable"]
|
|
if !totalOK || !availableOK || available > total {
|
|
return 0, 0, errors.New("incomplete memory info")
|
|
}
|
|
return total - available, total, nil
|
|
}
|
|
|
|
func cpuPercent(idle1, total1, idle2, total2 uint64) float64 {
|
|
if total2 <= total1 || idle2 < idle1 || idle2-idle1 > total2-total1 {
|
|
return 0
|
|
}
|
|
return float64((total2-total1)-(idle2-idle1)) * 100 / float64(total2-total1)
|
|
}
|
|
|
|
func validMonitorStatus(status string) bool {
|
|
switch status {
|
|
case "", "completed", "pending", "processing", "finalizing", "failed", "cancelled":
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func monitorStageDetail(ingestState, processingState, errorCode string) string {
|
|
if errorCode != "" {
|
|
return "处理失败,可查看错误码后重试"
|
|
}
|
|
switch {
|
|
case ingestState == "pending":
|
|
return "等待确认或上传"
|
|
case ingestState == "uploading":
|
|
return "正在上传测试知识库"
|
|
case processingState == "processing":
|
|
return "WeKnora正在解析、切片或向量化"
|
|
case processingState == "finalizing":
|
|
return "基础解析完成,索引或增强处理中"
|
|
case processingState == "completed":
|
|
return "解析完成,等待或已完成验收"
|
|
default:
|
|
return "等待远端状态更新"
|
|
}
|
|
}
|