姚章浩
5e1482e349
build / build (push) Successful in 1m40s
根因：生产可恢复性只有 PRODUCTION_BACKUP_RESTORE_VERIFIED 这个手填 env 开关， 页面没有任何入口去真跑一次恢复；CLI（librarydrill）又不是 worker 能复用的实现， 所以"已验证"既证明不了跑过、也证明不了跑的是当前目标。 改动： - 共享实现 internal/libraryverify（维护隔离→捕获→基线校验→canary→恢复→复核→关窗口）， cmd/librarydrill 改为薄封装，CLI 与 worker 只允许一份执行体。 - verify 成为独立业务作业：kind + VerifyStages(capture/verify/canary/restore/finalize)， 与 sync/restore/baseline 共用同一把写互斥；迁移 79 加 kind 约束、重建部分唯一索引、 新建 library_restore_verifications（按 target_id 绑定，换库/索引/桶旧证据自动失效， 同 target+scope+job 幂等）。 - worker：admitLibraryJob + runOwnedLibraryAttempt 之后调用共享实现，阶段实时写 library_job_stages，结束后写真实终态；通过写 passed 证据，失败写 failed+原因（不伪成功）； 中断接管按同一作业 id 收掉维护窗口。 - API：POST/GET /api/v1/library/verifications（管理员；闸口关闭/维护中拒绝；返回完整 整库目标、最近证据、失败原因），OpenAPI 同步。 - 闸口：同步/恢复/续作的证据判断改为读当前目标证据；历史 env 仅在证据缺失时兜底， 证据存在时以证据为准。 - 页面：生产保护与恢复面板加「恢复验证」块（完整 KB/索引/桶、状态、最近证据、失败原因、 管理员按钮、5s 轮询），作业类型补 verify 文案。 测试：make verify 16/16 全绿（go 1387 例 + web 454 例 + 契约/迁移/密钥/依赖锁定）； 新增用例覆盖证据四条件、闸口证据优先、页面无证据/失败/通过三种呈现与管理员可见性。
274 lines
12 KiB
Go
274 lines
12 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"log"
|
|
"net/http"
|
|
"os"
|
|
"os/signal"
|
|
"strings"
|
|
"syscall"
|
|
"time"
|
|
|
|
"github.com/jackc/pgx/v5/pgxpool"
|
|
|
|
"knowledge-governance/internal/appconfig"
|
|
"knowledge-governance/internal/backuphost"
|
|
"knowledge-governance/internal/dingtalkdiscovery"
|
|
"knowledge-governance/internal/httpapi"
|
|
"knowledge-governance/internal/libraryhandoff"
|
|
"knowledge-governance/internal/monitoring"
|
|
postgresstore "knowledge-governance/internal/platform/postgres"
|
|
"knowledge-governance/internal/platform/weknora"
|
|
"knowledge-governance/internal/qaopsai"
|
|
"knowledge-governance/internal/qaopsorg"
|
|
"knowledge-governance/internal/qaopsquality"
|
|
"knowledge-governance/internal/qawemail"
|
|
"knowledge-governance/internal/serviceverify"
|
|
"knowledge-governance/internal/version"
|
|
)
|
|
|
|
// dingtalkSourceLister 是「系统设置 → 数据源配置」的数据源发现实现：只读列举钉钉知识库与
|
|
// 钉盘空间。它跑在 api 进程里，但用的是与 worker 共享的同一份 dws 状态目录
|
|
// （见 compose.yaml 的 /data/dws 与文件型 keychain 挂载）——凭据只有一份。
|
|
type dingtalkSourceLister struct{}
|
|
|
|
func (dingtalkSourceLister) ReadDWSMetadata(ctx context.Context) (dingtalkdiscovery.CredentialStatus, error) {
|
|
return dingtalkdiscovery.ReadCredentialStatus(ctx, dingtalkdiscovery.ExecRunner{})
|
|
}
|
|
|
|
func (dingtalkSourceLister) ListAvailableDingtalkSources(ctx context.Context) (dingtalkdiscovery.Result, error) {
|
|
return dingtalkdiscovery.Discover(ctx, dingtalkdiscovery.ExecRunner{})
|
|
}
|
|
|
|
// 同一份 dws 凭据也用来列在线文档/表格并校验可读。
|
|
func (dingtalkSourceLister) ListDingtalkDocuments(ctx context.Context, query string, limit int) (dingtalkdiscovery.DocResult, error) {
|
|
return dingtalkdiscovery.DiscoverDocs(ctx, dingtalkdiscovery.ExecRunner{}, query, limit)
|
|
}
|
|
|
|
func (dingtalkSourceLister) ValidateDingtalkDocument(ctx context.Context, node string) (dingtalkdiscovery.DocumentNode, error) {
|
|
return dingtalkdiscovery.ValidateDocument(ctx, dingtalkdiscovery.ExecRunner{}, node)
|
|
}
|
|
|
|
func main() {
|
|
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
|
defer stop()
|
|
if len(os.Args) == 2 && os.Args[1] == "--version-record-status" {
|
|
os.Exit(versionRecordStatus(ctx, os.Getenv))
|
|
}
|
|
if err := run(ctx, os.Getenv); err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func run(ctx context.Context, getenv func(string) string) error {
|
|
config, err := appconfig.Load(getenv)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
pool, err := pgxpool.New(ctx, config.DatabaseURL)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer pool.Close()
|
|
if err := pool.Ping(ctx); err != nil {
|
|
return err
|
|
}
|
|
if err := postgresstore.ApplyMigrations(ctx, pool); err != nil {
|
|
return err
|
|
}
|
|
authStore := postgresstore.NewStore(pool)
|
|
if err := bootstrapInitialAdmin(ctx, authStore, time.Now); err != nil {
|
|
return err
|
|
}
|
|
releaseAuthorizationToken := ""
|
|
if config.ProductionReleaseRequestsEnabled {
|
|
releaseAuthorizationToken, err = weknora.ReadAPIKeyFile(config.ProductionReleaseAuthTokenFile)
|
|
if err != nil {
|
|
return errors.New("read production release authorization token")
|
|
}
|
|
}
|
|
// P10: with a mounted API key the probe verifies a real knowledge-base read
|
|
// (readiness); without one it degrades to the liveness-only HTTP check.
|
|
var weknoraReadiness func(context.Context) error
|
|
var knowledgeBaseLister httpapi.KnowledgeBaseLister
|
|
if strings.TrimSpace(config.TestWeKnoraBaseURL) != "" {
|
|
if apiKey, keyErr := weknora.ReadAPIKeyFile(config.TestWeKnoraAPIKeyFile); keyErr == nil {
|
|
probeTimeout := config.TestWeKnoraHTTPTimeout
|
|
if probeTimeout > 10*time.Second || probeTimeout <= 0 {
|
|
probeTimeout = 10 * time.Second
|
|
}
|
|
if probeClient, clientErr := weknora.NewClient(weknora.ClientConfig{BaseURL: config.TestWeKnoraBaseURL, APIKey: apiKey, Timeout: probeTimeout, AllowDockerHTTP: config.TestWeKnoraAllowDockerHTTP}); clientErr == nil {
|
|
knowledgeBaseLister = probeClient
|
|
routingKB := config.InternalRoutingKnowledgeBaseID
|
|
weknoraReadiness = func(probeCtx context.Context) error {
|
|
_, _, err := probeClient.ListKnowledge(probeCtx, routingKB, 1, 1)
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
}
|
|
// Production WeKnora is probed for liveness only: the console must show the
|
|
// real state of that endpoint, while credentials and the publish target stay
|
|
// in the release runtime config (RELEASE_CONFIG_FILE).
|
|
// 生产 WeKnora 端点：配置入口在网页「系统设置 → 服务配置」，管理员保存的值是
|
|
// 这里的真源；容器 env 只作为未在网页配置时的兜底。凭据与发布目标仍留在
|
|
// release 运行时配置（RELEASE_CONFIG_FILE）里，不经网页也不落明文。
|
|
actionStore := postgresstore.NewStore(pool)
|
|
productionEndpointResolver := func(resolveCtx context.Context) string {
|
|
settings, err := actionStore.GetActiveUISettings(resolveCtx)
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
return settings.ProductionWeKnoraBaseURL
|
|
}
|
|
testEndpointResolver := func(resolveCtx context.Context) string {
|
|
settings, err := actionStore.GetActiveUISettings(resolveCtx)
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
return settings.TestWeKnoraBaseURL
|
|
}
|
|
monitorProvider, err := monitoring.NewRealProvider(monitoring.RealProviderConfig{DB: pool, WeKnoraBaseURL: config.TestWeKnoraBaseURL, TestWeKnoraResolver: testEndpointResolver, ProductionWeKnoraBaseURL: config.ProductionWeKnoraBaseURL, ProductionWeKnoraResolver: productionEndpointResolver, MinerUBaseURL: config.MinerUBaseURL, WeKnoraReadiness: weknoraReadiness, WorkerStaleAfter: config.WorkerLivenessStaleAfter})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
serviceTargetVerifier := serviceverify.New(serviceverify.Config{
|
|
TestAPIKeyFile: config.TestWeKnoraAPIKeyFile,
|
|
ProductionAPIKeyFile: config.ProductionWeKnoraAPIKeyFile,
|
|
TestBaseURLFallback: config.TestWeKnoraBaseURL,
|
|
ProductionBaseURLFallback: config.ProductionWeKnoraBaseURL,
|
|
TestAllowDockerHTTP: config.TestWeKnoraAllowDockerHTTP,
|
|
ProductionAllowDockerHTTP: config.ProductionWeKnoraAllowDockerHTTP,
|
|
})
|
|
// 「版本记录」：只读查询面在 API 侧接线；真正的抓取在 worker 侧（未配置来源即未接线）。
|
|
versionHistoryStore, versionHistoryRepo := httpapi.VersionHistoryStore(nil), ""
|
|
if config.VersionHistoryGiteaBaseURL != "" {
|
|
versionHistoryStore, versionHistoryRepo = postgresstore.NewStore(pool), config.VersionHistoryRepo
|
|
info := version.Snapshot()
|
|
if info.IdentityAvailable && info.CommitSHA != nil {
|
|
if _, e := actionStore.EnqueueVersionHistorySync(ctx, config.VersionHistoryRepo, "system:deployment", *info.CommitSHA, time.Now()); e != nil {
|
|
log.Print("version history deployment synchronization could not be queued")
|
|
}
|
|
}
|
|
}
|
|
qaReader, qaSettings := openQAOperations(ctx, getenv, config.DatabaseURL, pool)
|
|
if closer, ok := qaReader.(interface{ Close() }); ok {
|
|
defer closer.Close()
|
|
}
|
|
if qaSettings.Problem != "" {
|
|
log.Print("QA operations source is unavailable; governance remains online")
|
|
}
|
|
qaOrganizationStore := qaopsorg.NewStore(pool, getenv("DOBOTAI_OPS_TENANT_KEY"))
|
|
qaSyncSettings, qaSyncErr := qaopsorg.LoadSyncSettings(getenv)
|
|
if qaSyncErr != nil {
|
|
log.Print("QA directory synchronization is not configured; governance remains online")
|
|
}
|
|
var qaSyncStore httpapi.QAOrgSyncStore
|
|
if getenv("DOBOTAI_OPS_TENANT_KEY") != "" {
|
|
qaSyncStore = qaOrganizationStore
|
|
}
|
|
// 快照搬运是否接线：只用于决定"发布/恢复作业先入哪个队列"。
|
|
// 配置不完整即启动失败（半接线会让作业以"快照不在执行器"失败）。
|
|
_, handoffConfigured, handoffErr := libraryhandoff.FromEnv(getenv)
|
|
if handoffErr != nil {
|
|
return fmt.Errorf("快照搬运接线无效: %w", handoffErr)
|
|
}
|
|
log.Printf("library target evidence identity=%s", config.LibraryTargetIdentity())
|
|
handler, err := httpapi.NewHandler(httpapi.Config{
|
|
LibraryHandoffEnabled: handoffConfigured,
|
|
QAAI: qaopsai.NewFromEnvironment(pool, getenv, config.DatabaseURL),
|
|
QAWeeklyMail: qawemail.NewStore(pool, qaSettings.TenantKey, getenv("QA_AI_MASTER_KEY_FILE"), nil),
|
|
QAAITrustTLSProxy: getenv("QA_AI_TRUST_TLS_PROXY") == "true",
|
|
QAQuality: qaopsquality.NewStore(pool, getenv("DOBOTAI_OPS_TENANT_KEY")),
|
|
QAOrgSync: qaSyncStore,
|
|
QAOrgSyncEnabled: qaSyncErr == nil && qaSyncSettings.Enabled,
|
|
QAOrg: qaOrganizationStore,
|
|
QAOps: qaReader,
|
|
QAOpsTenantKey: qaSettings.TenantKey,
|
|
QAOpsEnabled: qaSettings.Enabled,
|
|
QAOpsProblem: qaSettings.Problem,
|
|
Store: postgresstore.NewStore(pool).WithLibraryTarget(config.LibraryTargetIdentity()),
|
|
Monitor: monitorProvider,
|
|
InternalRoutingKnowledgeBaseID: config.InternalRoutingKnowledgeBaseID,
|
|
ExternalRoutingKnowledgeBaseID: config.ExternalRoutingKnowledgeBaseID,
|
|
RoutingBindings: postgresstore.NewStore(pool),
|
|
KnowledgeBases: knowledgeBaseLister,
|
|
ProductionBindings: postgresstore.NewStore(pool),
|
|
ProductionKnowledgeBases: serviceTargetVerifier,
|
|
ProductionWeKnoraBaseURL: config.ProductionWeKnoraBaseURL,
|
|
ServiceTargetVerifier: serviceTargetVerifier,
|
|
ProductionReleaseRequestsEnabled: config.ProductionReleaseRequestsEnabled,
|
|
TestAutoReviewPolicy: config.TestAutoReviewPolicy,
|
|
ProductionBackupRestoreVerified: config.ProductionBackupRestoreVerified,
|
|
ProductionReleaseAuthorizerRef: config.ProductionReleaseAuthorizerRef,
|
|
ProductionReleaseAuthorizationToken: releaseAuthorizationToken,
|
|
// 数据源发现（系统设置 → 数据源配置）：用与 worker 同一份 dws 状态做只读列举。
|
|
DingtalkSources: dingtalkSourceLister{},
|
|
DingtalkDocuments: dingtalkSourceLister{},
|
|
BackupSchedule: postgresstore.NewStore(pool),
|
|
VersionHistory: versionHistoryStore,
|
|
VersionHistoryRepo: versionHistoryRepo,
|
|
LibraryArchiveRoot: config.LibraryArchiveRoot,
|
|
LibraryTargetInstance: config.LibraryTargetInstance,
|
|
LibraryTargetProject: config.LibraryTargetProject,
|
|
LibraryTargetWriters: config.LibraryTargetWriters,
|
|
LibraryTargetIngressURL: config.LibraryTargetIngressURL,
|
|
LibraryTargetIngressService: config.LibraryTargetIngressService,
|
|
LibraryTargetDockerSocket: config.LibraryTargetDockerSocket,
|
|
LibraryTargetJournalRoot: config.LibraryTargetJournalRoot,
|
|
LibrarySourceInstance: config.SourceInstance(),
|
|
LibrarySourceProject: config.SourceProject(),
|
|
LibrarySourceApp: config.LibrarySourceApp,
|
|
LibrarySourceTenantID: config.LibrarySourceTenantID,
|
|
LibrarySourceInternalKB: config.LibrarySourceInternalKB,
|
|
LibrarySourceExternalKB: config.LibrarySourceExternalKB,
|
|
LibraryAgentSocket: config.LibraryAgentSocket,
|
|
LibraryExecutorStatusMaxAge: config.LibraryExecutorStatusMaxAge,
|
|
LibraryCapacityBudgetBytes: config.LibraryCapacityBudgetBytes,
|
|
LibraryProductionDatabaseURL: config.LibraryProductionDatabaseURL,
|
|
LibraryProductionInternalKB: config.LibraryProductionInternalKB,
|
|
LibraryProductionExternalKB: config.LibraryProductionExternalKB,
|
|
LibraryProductionTenantID: config.LibraryProductionTenantID,
|
|
LibraryProductionIndex: config.LibraryProductionIndex,
|
|
LibraryProductionS3Bucket: config.LibraryProductionS3Bucket,
|
|
LibraryRetentionVersions: config.LibraryRetentionVersions,
|
|
LibraryRetentionDays: config.LibraryRetentionDays,
|
|
BackupHost: backuphost.New(backuphost.Config{
|
|
SSHConfig: config.BackupHostSSHConfig,
|
|
Target: config.BackupHostTarget,
|
|
ScriptDir: config.BackupHostScriptDir,
|
|
Root: config.BackupHostRoot,
|
|
}),
|
|
Auth: &httpapi.AuthConfig{Store: authStore, SessionTTL: 12 * time.Hour},
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
server := &http.Server{
|
|
Addr: config.AppAddr,
|
|
Handler: handler,
|
|
ReadHeaderTimeout: 5 * time.Second,
|
|
ReadTimeout: 15 * time.Second,
|
|
WriteTimeout: 30 * time.Second,
|
|
IdleTimeout: 60 * time.Second,
|
|
}
|
|
|
|
errCh := make(chan error, 1)
|
|
go func() { errCh <- server.ListenAndServe() }()
|
|
log.Printf("governance API listening on %s", config.AppAddr)
|
|
select {
|
|
case err := <-errCh:
|
|
if errors.Is(err, http.ErrServerClosed) {
|
|
return nil
|
|
}
|
|
return err
|
|
case <-ctx.Done():
|
|
shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
return server.Shutdown(shutdownCtx)
|
|
}
|
|
}
|