姚章浩
3f603aa293
需求：整库逻辑备份做定时，但要在「系统设置 → 存储与备份」里设置定时时间与储存天数。 后端 - migration 000045 backup_schedule（单行：enabled/schedule_time/retention_days/retention_months/ last_applied_*），store + 领域校验（HH:MM、1~365 天、0~24 月，非法值 400 且不落库不下发） - internal/backuphost：通过 ssh 在目标主机上读回/下发计划并立即备份（未接线时 Available=false， 页面降级提示；主机侧错误带输出回传）。最近一次只认有 MANIFEST.json 的完整目录，跳 *.incomplete - 接口 GET/PUT /api/v1/backup-schedule、GET …/host、POST …/apply、POST …/run； 写操作经 classifyRole 限管理员；run 为异步 202 + 在途单飞（http 写超时 30s，备份约 35s） - openapi.yaml 补齐 5 条路由（契约测试要求每条都写 400/500） 主机侧 - scripts/install_backup_schedule.sh（幂等）：写 /etc/knowledge-governance/backup-schedule.env 与 /etc/cron.d/kb-logical-backup，参数校验，--show/--disable，最后一行 SCHEDULE_APPLIED 供平台核对 - logical-backup.sh 从该文件读保留策略；失败或被中断把目录改名 *.incomplete，不留'像成功'的半成品 - install --show 只回显计划行（原先把 SHELL=/PATH= 当计划显示） 前端 - web/src/backupSchedule.ts（严格解析：缺计划字段即报错，不把默认值伪装成设置） - BackupScheduleSection：容器 + 纯视图（管理员表单/只读、已一致/待下发、在途轮询） 运行前提（已落实）：运行镜像加 openssh-client，且容器 uid 1000 在 /etc/passwd 有名字 （否则 ssh 报 No user exists for uid 1000）；凭据只读挂载 secrets/backup-host（git 忽略） 实测：主机 cron 每天 03:30 / 保留 7 天 / 每月 3 份；立即备份一次 34 秒 / 742.7 MiB / 失败 0； Go 与前端测试全绿（170 vitest）。
307 lines
14 KiB
Go
307 lines
14 KiB
Go
package appconfig
|
|
|
|
import (
|
|
"errors"
|
|
"fmt"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"knowledge-governance/internal/governance"
|
|
)
|
|
|
|
type Config struct {
|
|
DatabaseURL string
|
|
AppAddr string
|
|
WorkerID string
|
|
PollInterval time.Duration
|
|
LeaseDuration time.Duration
|
|
RetryDelay time.Duration
|
|
SourceRefreshSchedulerInterval time.Duration
|
|
ReleaseWorkerEnabled bool
|
|
ReleaseConfigFile string
|
|
ProductionReleaseRequestsEnabled bool
|
|
ProductionBackupRestoreVerified bool
|
|
ProductionReleaseAuthorizerRef string
|
|
ProductionReleaseAuthTokenFile string
|
|
// Production WeKnora endpoint: the release target itself stays declared in
|
|
// the release runtime config (RELEASE_CONFIG_FILE); these settings only let
|
|
// the API probe that endpoint so the console can show its real state.
|
|
ProductionWeKnoraBaseURL string
|
|
ProductionWeKnoraAPIKeyFile string
|
|
ProductionWeKnoraAllowDockerHTTP bool
|
|
PPScanEnabled bool
|
|
PPScanAllowedRoot string
|
|
PPScanLedgerPath string
|
|
PPScanManifestPath string
|
|
TestWeKnoraEnabled bool
|
|
TestWeKnoraAllowDockerHTTP bool
|
|
TestWeKnoraBaseURL string
|
|
MinerUBaseURL string
|
|
TestWeKnoraKBID string
|
|
InternalRoutingKnowledgeBaseID string
|
|
ExternalRoutingKnowledgeBaseID string
|
|
TestWeKnoraAPIKeyFile string
|
|
GovernanceSpoolRoot string
|
|
IngestionSpoolRoot string
|
|
ArtifactFetchMaxBytes int64
|
|
WorkerLivenessStaleAfter time.Duration
|
|
TestWeKnoraHTTPTimeout time.Duration
|
|
TestWeKnoraPollInitial time.Duration
|
|
TestWeKnoraPollMax time.Duration
|
|
TestWeKnoraStalledAfter time.Duration
|
|
TestWeKnoraMaxUploadBytes int64
|
|
TestAcceptancePolicyVersion string
|
|
// 目标库所在主机（整库逻辑备份真正执行的地方）。凭据是挂载进来的 0600 ssh
|
|
// 配置文件与私钥；三项同时给出才接线，缺任一项即"未接线"（页面降级展示）。
|
|
BackupHostSSHConfig string
|
|
BackupHostTarget string
|
|
BackupHostScriptDir string
|
|
BackupHostRoot string
|
|
}
|
|
|
|
func Load(getenv func(string) string) (Config, error) {
|
|
config := Config{
|
|
DatabaseURL: getenv("DATABASE_URL"),
|
|
AppAddr: valueOrDefault(getenv("APP_ADDR"), ":8080"),
|
|
WorkerID: valueOrDefault(getenv("WORKER_ID"), "worker-1"),
|
|
PollInterval: time.Second,
|
|
LeaseDuration: 30 * time.Second,
|
|
RetryDelay: 5 * time.Second,
|
|
SourceRefreshSchedulerInterval: 30 * time.Second,
|
|
ReleaseConfigFile: strings.TrimSpace(getenv("RELEASE_CONFIG_FILE")),
|
|
ProductionReleaseAuthorizerRef: strings.TrimSpace(getenv("PRODUCTION_RELEASE_AUTHORIZER_REF")),
|
|
ProductionReleaseAuthTokenFile: strings.TrimSpace(getenv("PRODUCTION_RELEASE_AUTH_TOKEN_FILE")),
|
|
ProductionWeKnoraBaseURL: strings.TrimSpace(getenv("PRODUCTION_WEKNORA_BASE_URL")),
|
|
ProductionWeKnoraAPIKeyFile: strings.TrimSpace(getenv("PRODUCTION_WEKNORA_API_KEY_FILE")),
|
|
PPScanAllowedRoot: strings.TrimSpace(getenv("PP_SCAN_ALLOWED_ROOT")),
|
|
PPScanLedgerPath: strings.TrimSpace(getenv("PP_SCAN_LEDGER_PATH")),
|
|
PPScanManifestPath: strings.TrimSpace(getenv("PP_SCAN_MANIFEST_PATH")),
|
|
TestWeKnoraBaseURL: strings.TrimSpace(getenv("TEST_WEKNORA_BASE_URL")),
|
|
MinerUBaseURL: strings.TrimSpace(getenv("MINERU_BASE_URL")),
|
|
TestWeKnoraKBID: strings.TrimSpace(getenv("TEST_WEKNORA_KB_ID")),
|
|
InternalRoutingKnowledgeBaseID: strings.TrimSpace(getenv("TEST_INTERNAL_WEKNORA_KB_ID")),
|
|
ExternalRoutingKnowledgeBaseID: strings.TrimSpace(getenv("TEST_EXTERNAL_WEKNORA_KB_ID")),
|
|
TestWeKnoraAPIKeyFile: valueOrDefault(getenv("TEST_WEKNORA_API_KEY_FILE"), "/run/secrets/test_weknora_api_key"),
|
|
GovernanceSpoolRoot: valueOrDefault(getenv("GOVERNANCE_SPOOL_ROOT"), "/var/lib/governance/spool"),
|
|
IngestionSpoolRoot: strings.TrimSpace(getenv("INGESTION_SPOOL_ROOT")),
|
|
ArtifactFetchMaxBytes: 200 * 1024 * 1024,
|
|
WorkerLivenessStaleAfter: 2 * time.Minute,
|
|
TestWeKnoraHTTPTimeout: 60 * time.Second,
|
|
TestWeKnoraPollInitial: 10 * time.Second,
|
|
TestWeKnoraPollMax: time.Minute,
|
|
TestWeKnoraStalledAfter: 30 * time.Minute,
|
|
TestWeKnoraMaxUploadBytes: 50 * 1024 * 1024,
|
|
TestAcceptancePolicyVersion: valueOrDefault(getenv("TEST_ACCEPTANCE_POLICY_VERSION"), "structural-v1"),
|
|
// 整库逻辑备份的目标主机：三项同时给出才接线（脚本目录/备份根有默认值）。
|
|
BackupHostSSHConfig: strings.TrimSpace(getenv("BACKUP_HOST_SSH_CONFIG")),
|
|
BackupHostTarget: strings.TrimSpace(getenv("BACKUP_HOST_TARGET")),
|
|
BackupHostScriptDir: valueOrDefault(strings.TrimSpace(getenv("BACKUP_HOST_SCRIPT_DIR")), "/opt/weknora-test"),
|
|
BackupHostRoot: valueOrDefault(strings.TrimSpace(getenv("BACKUP_HOST_ROOT")), "/srv/kb-backups"),
|
|
}
|
|
if config.DatabaseURL == "" {
|
|
return Config{}, errors.New("DATABASE_URL is required")
|
|
}
|
|
var err error
|
|
if config.PollInterval, err = durationOrDefault(getenv("WORKER_POLL_INTERVAL"), config.PollInterval); err != nil {
|
|
return Config{}, fmt.Errorf("WORKER_POLL_INTERVAL: %w", err)
|
|
}
|
|
if config.LeaseDuration, err = durationOrDefault(getenv("WORKER_LEASE_DURATION"), config.LeaseDuration); err != nil {
|
|
return Config{}, fmt.Errorf("WORKER_LEASE_DURATION: %w", err)
|
|
}
|
|
if config.RetryDelay, err = durationOrDefault(getenv("WORKER_RETRY_DELAY"), config.RetryDelay); err != nil {
|
|
return Config{}, fmt.Errorf("WORKER_RETRY_DELAY: %w", err)
|
|
}
|
|
if config.SourceRefreshSchedulerInterval, err = durationOrDefault(getenv("SOURCE_REFRESH_SCHEDULER_INTERVAL"), config.SourceRefreshSchedulerInterval); err != nil {
|
|
return Config{}, fmt.Errorf("SOURCE_REFRESH_SCHEDULER_INTERVAL: %w", err)
|
|
}
|
|
if config.SourceRefreshSchedulerInterval < 5*time.Second {
|
|
return Config{}, errors.New("SOURCE_REFRESH_SCHEDULER_INTERVAL must be at least 5s")
|
|
}
|
|
if raw := getenv("RELEASE_WORKER_ENABLED"); raw != "" {
|
|
config.ReleaseWorkerEnabled, err = strconv.ParseBool(raw)
|
|
if err != nil {
|
|
return Config{}, fmt.Errorf("RELEASE_WORKER_ENABLED: %w", err)
|
|
}
|
|
}
|
|
if config.ReleaseWorkerEnabled && (!filepath.IsAbs(config.ReleaseConfigFile) || filepath.Clean(config.ReleaseConfigFile) != config.ReleaseConfigFile) {
|
|
return Config{}, errors.New("RELEASE_CONFIG_FILE must be an absolute path when release worker is enabled")
|
|
}
|
|
if raw := getenv("PRODUCTION_RELEASE_REQUESTS_ENABLED"); raw != "" {
|
|
config.ProductionReleaseRequestsEnabled, err = strconv.ParseBool(raw)
|
|
if err != nil {
|
|
return Config{}, fmt.Errorf("PRODUCTION_RELEASE_REQUESTS_ENABLED: %w", err)
|
|
}
|
|
}
|
|
if raw := getenv("PRODUCTION_BACKUP_RESTORE_VERIFIED"); raw != "" {
|
|
config.ProductionBackupRestoreVerified, err = strconv.ParseBool(raw)
|
|
if err != nil {
|
|
return Config{}, fmt.Errorf("PRODUCTION_BACKUP_RESTORE_VERIFIED: %w", err)
|
|
}
|
|
}
|
|
if raw := getenv("PRODUCTION_WEKNORA_ALLOW_DOCKER_HTTP"); raw != "" {
|
|
config.ProductionWeKnoraAllowDockerHTTP, err = strconv.ParseBool(raw)
|
|
if err != nil {
|
|
return Config{}, fmt.Errorf("PRODUCTION_WEKNORA_ALLOW_DOCKER_HTTP: %w", err)
|
|
}
|
|
}
|
|
if config.ProductionWeKnoraBaseURL != "" {
|
|
if err := validateHTTPBaseURL(config.ProductionWeKnoraBaseURL); err != nil {
|
|
return Config{}, fmt.Errorf("PRODUCTION_WEKNORA_BASE_URL: %w", err)
|
|
}
|
|
}
|
|
if config.ProductionWeKnoraAPIKeyFile != "" {
|
|
if config.ProductionWeKnoraBaseURL == "" {
|
|
return Config{}, errors.New("PRODUCTION_WEKNORA_API_KEY_FILE requires PRODUCTION_WEKNORA_BASE_URL")
|
|
}
|
|
if !filepath.IsAbs(config.ProductionWeKnoraAPIKeyFile) {
|
|
return Config{}, errors.New("PRODUCTION_WEKNORA_API_KEY_FILE must be an absolute path")
|
|
}
|
|
}
|
|
if raw := getenv("PP_SCAN_ENABLED"); raw != "" {
|
|
config.PPScanEnabled, err = strconv.ParseBool(raw)
|
|
if err != nil {
|
|
return Config{}, fmt.Errorf("PP_SCAN_ENABLED: %w", err)
|
|
}
|
|
}
|
|
if raw := getenv("TEST_WEKNORA_ENABLED"); raw != "" {
|
|
config.TestWeKnoraEnabled, err = strconv.ParseBool(raw)
|
|
if err != nil {
|
|
return Config{}, fmt.Errorf("TEST_WEKNORA_ENABLED: %w", err)
|
|
}
|
|
}
|
|
if raw := getenv("TEST_WEKNORA_ALLOW_DOCKER_HTTP"); raw != "" {
|
|
config.TestWeKnoraAllowDockerHTTP, err = strconv.ParseBool(raw)
|
|
if err != nil {
|
|
return Config{}, fmt.Errorf("TEST_WEKNORA_ALLOW_DOCKER_HTTP: %w", err)
|
|
}
|
|
}
|
|
if config.WorkerLivenessStaleAfter, err = durationOrDefault(getenv("WORKER_LIVENESS_STALE_AFTER"), config.WorkerLivenessStaleAfter); err != nil {
|
|
return Config{}, fmt.Errorf("WORKER_LIVENESS_STALE_AFTER: %w", err)
|
|
}
|
|
if config.TestWeKnoraHTTPTimeout, err = durationOrDefault(getenv("TEST_WEKNORA_HTTP_TIMEOUT"), config.TestWeKnoraHTTPTimeout); err != nil {
|
|
return Config{}, fmt.Errorf("TEST_WEKNORA_HTTP_TIMEOUT: %w", err)
|
|
}
|
|
if config.TestWeKnoraPollInitial, err = durationOrDefault(getenv("TEST_WEKNORA_POLL_INITIAL"), config.TestWeKnoraPollInitial); err != nil {
|
|
return Config{}, fmt.Errorf("TEST_WEKNORA_POLL_INITIAL: %w", err)
|
|
}
|
|
if config.TestWeKnoraPollMax, err = durationOrDefault(getenv("TEST_WEKNORA_POLL_MAX"), config.TestWeKnoraPollMax); err != nil {
|
|
return Config{}, fmt.Errorf("TEST_WEKNORA_POLL_MAX: %w", err)
|
|
}
|
|
if config.TestWeKnoraStalledAfter, err = durationOrDefault(getenv("TEST_WEKNORA_STALLED_AFTER"), config.TestWeKnoraStalledAfter); err != nil {
|
|
return Config{}, fmt.Errorf("TEST_WEKNORA_STALLED_AFTER: %w", err)
|
|
}
|
|
if config.TestWeKnoraMaxUploadBytes, err = positiveInt64OrDefault(getenv("TEST_WEKNORA_MAX_UPLOAD_BYTES"), config.TestWeKnoraMaxUploadBytes); err != nil {
|
|
return Config{}, fmt.Errorf("TEST_WEKNORA_MAX_UPLOAD_BYTES: %w", err)
|
|
}
|
|
if config.TestWeKnoraMaxUploadBytes > 50*1024*1024 {
|
|
return Config{}, errors.New("TEST_WEKNORA_MAX_UPLOAD_BYTES must not exceed 50 MiB")
|
|
}
|
|
if config.TestWeKnoraPollMax < config.TestWeKnoraPollInitial {
|
|
return Config{}, errors.New("TEST_WEKNORA_POLL_MAX must not be less than TEST_WEKNORA_POLL_INITIAL")
|
|
}
|
|
if config.TestWeKnoraStalledAfter <= config.TestWeKnoraPollMax {
|
|
return Config{}, errors.New("TEST_WEKNORA_STALLED_AFTER must be greater than TEST_WEKNORA_POLL_MAX")
|
|
}
|
|
config.TestAcceptancePolicyVersion = strings.TrimSpace(config.TestAcceptancePolicyVersion)
|
|
if config.TestAcceptancePolicyVersion == "" {
|
|
return Config{}, errors.New("TEST_ACCEPTANCE_POLICY_VERSION must not be empty")
|
|
}
|
|
if config.ProductionReleaseRequestsEnabled {
|
|
// 目标主机凭据要么都给要么都不给：只给一半会把"未接线"伪装成"已接线"。
|
|
if (config.BackupHostSSHConfig == "") != (config.BackupHostTarget == "") {
|
|
return Config{}, errors.New("BACKUP_HOST_SSH_CONFIG and BACKUP_HOST_TARGET must be set together")
|
|
}
|
|
if config.ProductionReleaseAuthorizerRef == "" || len(config.ProductionReleaseAuthorizerRef) > 128 || strings.ContainsAny(config.ProductionReleaseAuthorizerRef, "\x00\r\n ") {
|
|
return Config{}, errors.New("PRODUCTION_RELEASE_AUTHORIZER_REF is required and must be safe when production release requests are enabled")
|
|
}
|
|
if config.ProductionReleaseAuthTokenFile == "" || !filepath.IsAbs(config.ProductionReleaseAuthTokenFile) {
|
|
return Config{}, errors.New("PRODUCTION_RELEASE_AUTH_TOKEN_FILE must be an absolute path when production release requests are enabled")
|
|
}
|
|
}
|
|
if config.PPScanEnabled {
|
|
for key, value := range map[string]string{
|
|
"PP_SCAN_ALLOWED_ROOT": config.PPScanAllowedRoot,
|
|
"PP_SCAN_LEDGER_PATH": config.PPScanLedgerPath,
|
|
"PP_SCAN_MANIFEST_PATH": config.PPScanManifestPath,
|
|
} {
|
|
if value == "" {
|
|
return Config{}, fmt.Errorf("%s is required when PP scan is enabled", key)
|
|
}
|
|
if !filepath.IsAbs(value) {
|
|
return Config{}, fmt.Errorf("%s must be absolute", key)
|
|
}
|
|
}
|
|
}
|
|
if config.TestWeKnoraEnabled {
|
|
if config.TestWeKnoraBaseURL == "" {
|
|
return Config{}, errors.New("TEST_WEKNORA_BASE_URL is required when test WeKnora is enabled")
|
|
}
|
|
if config.TestWeKnoraKBID == "" {
|
|
return Config{}, errors.New("TEST_WEKNORA_KB_ID is required when test WeKnora is enabled")
|
|
}
|
|
if strings.TrimSpace(config.TestWeKnoraAPIKeyFile) == "" {
|
|
return Config{}, errors.New("TEST_WEKNORA_API_KEY_FILE is required when test WeKnora is enabled")
|
|
}
|
|
if strings.TrimSpace(config.GovernanceSpoolRoot) == "" {
|
|
return Config{}, errors.New("GOVERNANCE_SPOOL_ROOT is required when test WeKnora is enabled")
|
|
}
|
|
}
|
|
if (config.InternalRoutingKnowledgeBaseID == "") != (config.ExternalRoutingKnowledgeBaseID == "") {
|
|
return Config{}, errors.New("both internal and external routing knowledge bases must be configured together")
|
|
}
|
|
for _, knowledgeBaseID := range []string{config.InternalRoutingKnowledgeBaseID, config.ExternalRoutingKnowledgeBaseID} {
|
|
if len(knowledgeBaseID) > 128 || strings.ContainsAny(knowledgeBaseID, "\x00\r\n /?#\\") {
|
|
return Config{}, errors.New("routing knowledge base ID is invalid")
|
|
}
|
|
}
|
|
return config, nil
|
|
}
|
|
|
|
// validateHTTPBaseURL accepts only a bare http(s) origin: no credentials, query
|
|
// or fragment, so a probe address can never smuggle in secrets. The rule itself
|
|
// lives with the domain type because the web-configured endpoint must obey the
|
|
// exact same shape.
|
|
func validateHTTPBaseURL(value string) error {
|
|
if !governance.ValidHTTPOrigin(value) {
|
|
return errors.New("must be an http(s) origin without credentials, query or fragment")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func valueOrDefault(value, fallback string) string {
|
|
if value == "" {
|
|
return fallback
|
|
}
|
|
return value
|
|
}
|
|
|
|
func durationOrDefault(value string, fallback time.Duration) (time.Duration, error) {
|
|
if value == "" {
|
|
return fallback, nil
|
|
}
|
|
parsed, err := time.ParseDuration(value)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
if parsed <= 0 {
|
|
return 0, errors.New("must be positive")
|
|
}
|
|
return parsed, nil
|
|
}
|
|
|
|
func positiveInt64OrDefault(value string, fallback int64) (int64, error) {
|
|
if value == "" {
|
|
return fallback, nil
|
|
}
|
|
parsed, err := strconv.ParseInt(value, 10, 64)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
if parsed <= 0 {
|
|
return 0, errors.New("must be positive")
|
|
}
|
|
return parsed, nil
|
|
}
|