1c80b07f2c
build / build (push) Successful in 1m59s
Dockerfile 运行阶段补齐 dws_dingtalk_scan(F12),并新增 runtime-scripts 门禁 做脚本依赖闭包正负例;就绪探测每次解析当前端点/绑定/凭据并区分 404、未配置、 不可达(F13);路由写入来源路径与父目录,历史空值仅按同连接同 item 的意图读时 补齐(F14);目标筛选显示短 ID 与当前绑定/历史并提供快捷筛选(F15);刷新终态 移除后台运行、展示失败时间/旧快照/脱敏原因/作业 ID 与重试(F16)。 验证:make verify 17 项全过(identity 本地按设计 skipped);一次性 schema 的 Monitor 目录/路由集成用例通过;8088 部署与真实 UI 验收仍由父会话执行。
385 lines
18 KiB
Go
385 lines
18 KiB
Go
package monitoring
|
||
|
||
import (
|
||
"context"
|
||
"errors"
|
||
"fmt"
|
||
"io"
|
||
"net/http"
|
||
"net/url"
|
||
"os"
|
||
"strconv"
|
||
"strings"
|
||
"syscall"
|
||
"time"
|
||
|
||
"github.com/jackc/pgx/v5"
|
||
|
||
"knowledge-governance/internal/governance"
|
||
)
|
||
|
||
type Database interface {
|
||
Ping(context.Context) error
|
||
Query(context.Context, string, ...any) (pgx.Rows, error)
|
||
QueryRow(context.Context, string, ...any) pgx.Row
|
||
}
|
||
|
||
type RealProviderConfig struct {
|
||
DB Database
|
||
Now func() time.Time
|
||
HTTPClient *http.Client
|
||
WeKnoraBaseURL string
|
||
MinerUBaseURL string
|
||
// TestWeKnoraResolver reads the test endpoint an admin saved in the web
|
||
// settings, so 「服务配置」is the single place the console probes. The env
|
||
// value above stays the bootstrap fallback for containers wired before the
|
||
// UI existed.
|
||
TestWeKnoraResolver func(ctx context.Context) string
|
||
// ProductionWeKnoraBaseURL lets the console show the real state of the
|
||
// production WeKnora endpoint. Publishing itself keeps reading its target
|
||
// from the release runtime config, never from here.
|
||
ProductionWeKnoraBaseURL string
|
||
// ProductionWeKnoraResolver reads the endpoint an admin saved in the web
|
||
// settings. That value is what the console probes — the web page is the
|
||
// place operators configure it — and the env value above stays the
|
||
// bootstrap fallback for containers that were wired before the UI existed.
|
||
ProductionWeKnoraResolver func(ctx context.Context) string
|
||
// WeKnoraReadiness optionally exercises a real read against the CURRENT
|
||
// routing binding and reports the sanitized probe target alongside the error.
|
||
WeKnoraReadiness func(ctx context.Context) (target string, err error)
|
||
// WorkerStaleAfter bounds how long a missing worker heartbeat still counts
|
||
// as alive (P10).
|
||
WorkerStaleAfter time.Duration
|
||
}
|
||
|
||
type RealProvider struct {
|
||
db Database
|
||
now func() time.Time
|
||
httpClient *http.Client
|
||
weknoraBaseURL string
|
||
minerUBaseURL string
|
||
testWeKnoraResolver func(ctx context.Context) string
|
||
productionWeKnoraBaseURL string
|
||
productionWeKnoraResolver func(ctx context.Context) string
|
||
weknoraReadiness func(ctx context.Context) (string, error)
|
||
workerStaleAfter time.Duration
|
||
health *healthTracker
|
||
}
|
||
|
||
// resolveTestWeKnoraBaseURL prefers the admin-saved web setting and falls back to
|
||
// the env bootstrap value, so the page is the place to change the test endpoint.
|
||
func (provider *RealProvider) resolveTestWeKnoraBaseURL(ctx context.Context) string {
|
||
if provider.testWeKnoraResolver != nil {
|
||
if value := strings.TrimSpace(provider.testWeKnoraResolver(ctx)); value != "" {
|
||
return value
|
||
}
|
||
}
|
||
return provider.weknoraBaseURL
|
||
}
|
||
|
||
// resolveProductionWeKnoraBaseURL prefers the admin-saved web setting and falls
|
||
// back to the env bootstrap value: the page wins, the container env only fills in
|
||
// when nobody configured it in the UI.
|
||
func (provider *RealProvider) resolveProductionWeKnoraBaseURL(ctx context.Context) string {
|
||
if provider.productionWeKnoraResolver != nil {
|
||
if value := strings.TrimSpace(provider.productionWeKnoraResolver(ctx)); value != "" {
|
||
return value
|
||
}
|
||
}
|
||
return provider.productionWeKnoraBaseURL
|
||
}
|
||
|
||
func NewRealProvider(config RealProviderConfig) (*RealProvider, error) {
|
||
if config.DB == nil {
|
||
return nil, errors.New("monitor database is required")
|
||
}
|
||
if config.Now == nil {
|
||
config.Now = time.Now
|
||
}
|
||
if config.HTTPClient == nil {
|
||
config.HTTPClient = &http.Client{Timeout: 3 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}
|
||
}
|
||
stale := config.WorkerStaleAfter
|
||
if stale <= 0 {
|
||
stale = time.Minute
|
||
}
|
||
return &RealProvider{db: config.DB, now: config.Now, httpClient: config.HTTPClient, weknoraBaseURL: strings.TrimSpace(config.WeKnoraBaseURL), minerUBaseURL: strings.TrimSpace(config.MinerUBaseURL), testWeKnoraResolver: config.TestWeKnoraResolver, productionWeKnoraBaseURL: strings.TrimSpace(config.ProductionWeKnoraBaseURL), productionWeKnoraResolver: config.ProductionWeKnoraResolver, weknoraReadiness: config.WeKnoraReadiness, workerStaleAfter: stale, health: newHealthTracker()}, nil
|
||
}
|
||
|
||
func (provider *RealProvider) Overview(ctx context.Context) (governance.MonitorOverview, error) {
|
||
now := provider.now().UTC()
|
||
overview := governance.MonitorOverview{Mode: "real", Simulated: false, ObservedAt: now, Services: []governance.MonitorService{}, KnowledgeBases: []governance.KnowledgeBaseProgress{}}
|
||
overview.Host = collectHostMetrics()
|
||
overview.Services = append(overview.Services, provider.postgresHealth(ctx, now), provider.queueHealth(ctx, now))
|
||
overview.Health = provider.Services(ctx)
|
||
overview.Services = append(overview.Services,
|
||
provider.httpHealth(ctx, "weknora", "WeKnora API", provider.resolveTestWeKnoraBaseURL(ctx), now),
|
||
provider.httpHealth(ctx, "weknora-production", "生产 WeKnora", provider.productionWeKnoraBaseURL, now),
|
||
governance.MonitorService{Key: "embedding", Name: "Embedding模型", State: "unknown", Detail: "由WeKnora管理,尚无独立探测端点", ObservedAt: now},
|
||
governance.MonitorService{Key: "llm", Name: "LLM模型", State: "unknown", Detail: "由WeKnora管理,尚无独立探测端点", ObservedAt: now},
|
||
provider.httpHealth(ctx, "mineru", "MinerU", provider.minerUBaseURL, now),
|
||
governance.MonitorService{Key: "redis", Name: "Redis / MQ", State: "unknown", Detail: "未部署;任务由PostgreSQL持久队列承载", ObservedAt: now},
|
||
governance.MonitorService{Key: "oss", Name: "OSS", State: "unknown", Detail: "未配置独立健康探测", ObservedAt: now},
|
||
)
|
||
rows, err := provider.db.Query(ctx, `SELECT p.test_kb_id,
|
||
CASE WHEN COUNT(DISTINCT p.target_scope)=1 THEN MIN(p.target_scope) ELSE 'mixed' END,
|
||
COUNT(*) FILTER (WHERE p.superseded_at IS NULL)::integer,
|
||
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.processing_state='completed')::integer,
|
||
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.processing_state='pending')::integer,
|
||
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.processing_state='processing')::integer,
|
||
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.processing_state='finalizing')::integer,
|
||
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.processing_state IN ('failed','stalled'))::integer,
|
||
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.target_scope='internal')::integer,
|
||
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.target_scope='internal' AND p.processing_state='completed')::integer,
|
||
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.target_scope='external')::integer,
|
||
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.target_scope='external' AND p.processing_state='completed')::integer,
|
||
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.processing_state='planned')::integer,
|
||
-- 需下架:在测试库里仍然存活(已完成、有知识 id、尚未标记移除)、但来源项已被判为禁止导入的条目。
|
||
-- 曾经这里判 p.retrieval_state='enabled',而 retrieval_state 在这套状态机里描述的是
|
||
-- “是否发布到生产”(unpublished/disabling/disabled/failed),全仓没有任何代码写过 'enabled',
|
||
-- 所以这条车道恒为 0 —— 2026-10-10 界面验收实测:把已入库的文件改成“禁止导入”后
|
||
-- 车道仍显示 0,界面承诺的“仍在知识库里的文件会出现在这里”不成立。
|
||
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.knowledge_id<>''
|
||
AND p.processing_state='completed'
|
||
AND p.retrieval_state NOT IN ('disabling','disabled')
|
||
AND item.policy_state='blocked')::integer,
|
||
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.processing_state='planned'
|
||
AND item.policy_state='blocked')::integer
|
||
FROM test_publications p
|
||
JOIN source_item_versions version ON version.id=p.source_item_version_id
|
||
JOIN source_items item ON item.id=version.source_item_id
|
||
GROUP BY p.test_kb_id
|
||
ORDER BY p.test_kb_id`)
|
||
if err != nil {
|
||
return governance.MonitorOverview{}, fmt.Errorf("list monitored knowledge bases: %w", err)
|
||
}
|
||
defer rows.Close()
|
||
for rows.Next() {
|
||
var item governance.KnowledgeBaseProgress
|
||
if err := rows.Scan(&item.ID, &item.Visibility, &item.Imported, &item.Completed, &item.Pending, &item.Processing, &item.Finalizing, &item.Failed, &item.InternalImported, &item.InternalCompleted, &item.ExternalImported, &item.ExternalCompleted,
|
||
&item.Planned, &item.NeedsRemoval, &item.StalePlanned); err != nil {
|
||
return governance.MonitorOverview{}, fmt.Errorf("scan monitored knowledge base: %w", err)
|
||
}
|
||
switch item.Visibility {
|
||
case "internal":
|
||
item.Name = "对内知识库"
|
||
case "external":
|
||
item.Name = "对外知识库"
|
||
default:
|
||
item.Name = "测试知识库"
|
||
}
|
||
overview.KnowledgeBases = append(overview.KnowledgeBases, item)
|
||
}
|
||
if err := rows.Err(); err != nil {
|
||
return governance.MonitorOverview{}, fmt.Errorf("iterate monitored knowledge bases: %w", err)
|
||
}
|
||
if err := provider.db.QueryRow(ctx, `SELECT
|
||
COUNT(*) FILTER (WHERE fetch_state IN ('pending','fetching'))::integer,
|
||
COUNT(*) FILTER (WHERE fetch_state='failed')::integer,
|
||
COUNT(*) FILTER (WHERE fetch_state='missing_local')::integer,
|
||
COALESCE(MAX((SELECT ingestion_concurrency FROM ui_setting_revisions WHERE state='active' LIMIT 1)),2)::integer
|
||
FROM catalog_import_intents`).Scan(&overview.Pipeline.FetchPending, &overview.Pipeline.FetchFailed, &overview.Pipeline.FetchMissing, &overview.Pipeline.Concurrency); err != nil {
|
||
return governance.MonitorOverview{}, fmt.Errorf("summarize ingestion pipeline: %w", err)
|
||
}
|
||
return overview, nil
|
||
}
|
||
|
||
func (provider *RealProvider) ListFiles(ctx context.Context, knowledgeBaseID, status string, limit, offset int) (governance.KnowledgeMonitorFilePage, error) {
|
||
if strings.TrimSpace(knowledgeBaseID) == "" || limit < 1 || limit > 100 || offset < 0 || !validMonitorStatus(status) {
|
||
return governance.KnowledgeMonitorFilePage{}, errors.New("invalid monitor file query")
|
||
}
|
||
page := governance.KnowledgeMonitorFilePage{Items: []governance.KnowledgeMonitorFile{}, Limit: limit, Offset: offset, SampleOnly: false}
|
||
if err := provider.db.QueryRow(ctx, `SELECT COUNT(*)::integer FROM test_publications WHERE test_kb_id=$1 AND ($2='' OR CASE WHEN processing_state='stalled' THEN 'failed' ELSE processing_state END=$2)`, knowledgeBaseID, status).Scan(&page.Total); err != nil {
|
||
return governance.KnowledgeMonitorFilePage{}, fmt.Errorf("count monitored files: %w", err)
|
||
}
|
||
rows, err := provider.db.Query(ctx, `SELECT item.title,item.normalized_path,publication.target_scope,
|
||
CASE WHEN publication.processing_state='stalled' THEN 'failed' ELSE publication.processing_state END,
|
||
GREATEST(0,EXTRACT(EPOCH FROM ($5-COALESCE(publication.processing_started_at,publication.updated_at)))::integer),
|
||
publication.ingest_state,publication.last_error
|
||
FROM test_publications publication
|
||
JOIN source_item_versions version ON version.id=publication.source_item_version_id
|
||
JOIN source_items item ON item.id=version.source_item_id
|
||
WHERE publication.test_kb_id=$1
|
||
AND ($2='' OR CASE WHEN publication.processing_state='stalled' THEN 'failed' ELSE publication.processing_state END=$2)
|
||
ORDER BY CASE publication.processing_state WHEN 'failed' THEN 0 WHEN 'stalled' THEN 0 WHEN 'processing' THEN 1 WHEN 'finalizing' THEN 2 WHEN 'pending' THEN 3 ELSE 4 END,
|
||
publication.updated_at DESC,publication.id
|
||
LIMIT $3 OFFSET $4`, knowledgeBaseID, status, limit, offset, provider.now().UTC())
|
||
if err != nil {
|
||
return governance.KnowledgeMonitorFilePage{}, fmt.Errorf("list monitored files: %w", err)
|
||
}
|
||
defer rows.Close()
|
||
for rows.Next() {
|
||
var item governance.KnowledgeMonitorFile
|
||
var ingestState string
|
||
if err := rows.Scan(&item.Name, &item.Directory, &item.Visibility, &item.Status, &item.ElapsedSecond, &ingestState, &item.ErrorCode); err != nil {
|
||
return governance.KnowledgeMonitorFilePage{}, fmt.Errorf("scan monitored file: %w", err)
|
||
}
|
||
item.StageDetail = monitorStageDetail(ingestState, item.Status, item.ErrorCode)
|
||
page.Items = append(page.Items, item)
|
||
}
|
||
if err := rows.Err(); err != nil {
|
||
return governance.KnowledgeMonitorFilePage{}, fmt.Errorf("iterate monitored files: %w", err)
|
||
}
|
||
return page, nil
|
||
}
|
||
|
||
func (provider *RealProvider) postgresHealth(ctx context.Context, now time.Time) governance.MonitorService {
|
||
started := time.Now()
|
||
state, detail := "healthy", "可连接并可查询"
|
||
if err := provider.db.Ping(ctx); err != nil {
|
||
state, detail = "unhealthy", "连接或查询失败"
|
||
}
|
||
return governance.MonitorService{Key: "postgres", Name: "PostgreSQL", State: state, Detail: detail, LatencyMS: int(time.Since(started).Milliseconds()), ObservedAt: now}
|
||
}
|
||
|
||
func (provider *RealProvider) queueHealth(ctx context.Context, now time.Time) governance.MonitorService {
|
||
started := time.Now()
|
||
var pending, running, failed int
|
||
err := provider.db.QueryRow(ctx, `SELECT COUNT(*) FILTER (WHERE state IN ('pending','retry'))::integer,COUNT(*) FILTER (WHERE state='running')::integer,COUNT(*) FILTER (WHERE state='failed')::integer FROM jobs`).Scan(&pending, &running, &failed)
|
||
state, detail := "healthy", fmt.Sprintf("可接单;等待%d,运行%d,失败%d", pending, running, failed)
|
||
if err != nil {
|
||
state, detail = "unhealthy", "队列不可查询"
|
||
}
|
||
return governance.MonitorService{Key: "queue", Name: "任务队列", State: state, Detail: detail, LatencyMS: int(time.Since(started).Milliseconds()), ObservedAt: now}
|
||
}
|
||
|
||
func (provider *RealProvider) httpHealth(ctx context.Context, key, name, baseURL string, now time.Time) governance.MonitorService {
|
||
if baseURL == "" {
|
||
return governance.MonitorService{Key: key, Name: name, State: "unknown", Detail: "未配置探测地址", ObservedAt: now}
|
||
}
|
||
parsed, err := url.Parse(baseURL)
|
||
if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Host == "" {
|
||
return governance.MonitorService{Key: key, Name: name, State: "unhealthy", Detail: "探测配置无效", ObservedAt: now}
|
||
}
|
||
parsed.Path = strings.TrimSuffix(parsed.Path, "/") + "/health"
|
||
parsed.RawQuery, parsed.Fragment = "", ""
|
||
request, err := http.NewRequestWithContext(ctx, http.MethodGet, parsed.String(), nil)
|
||
if err != nil {
|
||
return governance.MonitorService{Key: key, Name: name, State: "unhealthy", Detail: "无法创建健康探测", ObservedAt: now}
|
||
}
|
||
started := time.Now()
|
||
response, err := provider.httpClient.Do(request)
|
||
latency := int(time.Since(started).Milliseconds())
|
||
if err != nil {
|
||
return governance.MonitorService{Key: key, Name: name, State: "unhealthy", Detail: "健康端点不可达", LatencyMS: latency, ObservedAt: now}
|
||
}
|
||
defer response.Body.Close()
|
||
_, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 4096))
|
||
if response.StatusCode < 200 || response.StatusCode >= 300 {
|
||
return governance.MonitorService{Key: key, Name: name, State: "degraded", Detail: fmt.Sprintf("健康端点返回HTTP %d", response.StatusCode), LatencyMS: latency, ObservedAt: now}
|
||
}
|
||
return governance.MonitorService{Key: key, Name: name, State: "healthy", Detail: "健康端点可用", LatencyMS: latency, ObservedAt: now}
|
||
}
|
||
|
||
func collectHostMetrics() governance.MonitorHost {
|
||
host := governance.MonitorHost{GPUState: "unknown", GPUDetail: "容器未暴露GPU遥测", Simulated: false}
|
||
first, err := os.ReadFile("/proc/stat")
|
||
if err == nil {
|
||
idle1, total1, parseErr := parseCPUStat(string(first))
|
||
if parseErr == nil {
|
||
time.Sleep(100 * time.Millisecond)
|
||
if second, readErr := os.ReadFile("/proc/stat"); readErr == nil {
|
||
idle2, total2, secondErr := parseCPUStat(string(second))
|
||
if secondErr == nil {
|
||
host.CPUPercent = cpuPercent(idle1, total1, idle2, total2)
|
||
}
|
||
}
|
||
}
|
||
}
|
||
if raw, err := os.ReadFile("/proc/meminfo"); err == nil {
|
||
host.MemoryUsedBytes, host.MemoryTotalBytes, _ = parseMemoryInfo(string(raw))
|
||
}
|
||
var stats syscall.Statfs_t
|
||
if syscall.Statfs("/", &stats) == nil {
|
||
host.DiskTotalBytes = int64(stats.Blocks) * int64(stats.Bsize)
|
||
host.DiskUsedBytes = int64(stats.Blocks-stats.Bavail) * int64(stats.Bsize)
|
||
}
|
||
if entries, err := os.ReadDir("/proc/driver/nvidia/gpus"); err == nil && len(entries) > 0 {
|
||
host.GPUState, host.GPUDetail = "healthy", fmt.Sprintf("检测到%d块NVIDIA GPU;未读取利用率", len(entries))
|
||
}
|
||
return host
|
||
}
|
||
|
||
func parseCPUStat(raw string) (idle, total uint64, err error) {
|
||
line := strings.SplitN(raw, "\n", 2)[0]
|
||
fields := strings.Fields(line)
|
||
if len(fields) < 5 || fields[0] != "cpu" {
|
||
return 0, 0, errors.New("invalid cpu stat")
|
||
}
|
||
values := make([]uint64, 0, len(fields)-1)
|
||
for _, field := range fields[1:] {
|
||
value, parseErr := strconv.ParseUint(field, 10, 64)
|
||
if parseErr != nil {
|
||
return 0, 0, errors.New("invalid cpu counter")
|
||
}
|
||
values = append(values, value)
|
||
total += value
|
||
}
|
||
idle = values[3]
|
||
if len(values) > 4 {
|
||
idle += values[4]
|
||
}
|
||
return idle, total, nil
|
||
}
|
||
|
||
func parseMemoryInfo(raw string) (used, total int64, err error) {
|
||
values := map[string]int64{}
|
||
for _, line := range strings.Split(raw, "\n") {
|
||
fields := strings.Fields(line)
|
||
if len(fields) < 2 {
|
||
continue
|
||
}
|
||
key := strings.TrimSuffix(fields[0], ":")
|
||
if key != "MemTotal" && key != "MemAvailable" {
|
||
continue
|
||
}
|
||
value, parseErr := strconv.ParseInt(fields[1], 10, 64)
|
||
if parseErr != nil || value < 0 {
|
||
return 0, 0, errors.New("invalid memory counter")
|
||
}
|
||
values[key] = value * 1024
|
||
}
|
||
total, totalOK := values["MemTotal"]
|
||
available, availableOK := values["MemAvailable"]
|
||
if !totalOK || !availableOK || available > total {
|
||
return 0, 0, errors.New("incomplete memory info")
|
||
}
|
||
return total - available, total, nil
|
||
}
|
||
|
||
func cpuPercent(idle1, total1, idle2, total2 uint64) float64 {
|
||
if total2 <= total1 || idle2 < idle1 || idle2-idle1 > total2-total1 {
|
||
return 0
|
||
}
|
||
return float64((total2-total1)-(idle2-idle1)) * 100 / float64(total2-total1)
|
||
}
|
||
|
||
func validMonitorStatus(status string) bool {
|
||
switch status {
|
||
case "", "completed", "pending", "processing", "finalizing", "failed", "cancelled":
|
||
return true
|
||
default:
|
||
return false
|
||
}
|
||
}
|
||
|
||
func monitorStageDetail(ingestState, processingState, errorCode string) string {
|
||
if errorCode != "" {
|
||
return "处理失败,可查看错误码后重试"
|
||
}
|
||
switch {
|
||
case ingestState == "pending":
|
||
return "等待确认或上传"
|
||
case ingestState == "uploading":
|
||
return "正在上传测试知识库"
|
||
case processingState == "processing":
|
||
return "WeKnora正在解析、切片或向量化"
|
||
case processingState == "finalizing":
|
||
return "基础解析完成,索引或增强处理中"
|
||
case processingState == "completed":
|
||
return "解析完成,等待或已完成验收"
|
||
default:
|
||
return "等待远端状态更新"
|
||
}
|
||
}
|