From 0aa9f273e6bd91e3296a6088b79d6544416c6df2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A7=9A=E7=AB=A0=E6=B5=A9?= Date: Mon, 16 Nov 2026 20:53:40 +0800 Subject: [PATCH] =?UTF-8?q?feat(library):=20=E9=87=87=E9=9B=86=E6=8E=A5?= =?UTF-8?q?=E5=85=A5=20RDS=20=E5=9E=8B=200.9.0=20=E6=A0=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 为什么:测试 0.9.0 栈(Compose 项目 weknora-v090)没有源库容器,治理库也不在 容器里,两者都是 RDS。旧实现用 -source-pg/-governance-pg 的容器地址拼 DSN、 用 docker exec psql 读路由绑定,在这类栈上采集根本起不来——不是权限问题,是 地址解析方式错了。 怎么改: - 新增 internal/librarydsn 收敛"这次 DSN 从哪来":容器模式保持原形状 (sslmode=disable,目标不变);-source-pg 为空时改用源应用容器 env 的 DB_* 直连托管实例,TLS 取 LIBRARY_SOURCE_DB_SSLMODE(默认 require,prefer/allow 这类会静默降级的取值直接拒);-governance-pg 为空时从 LIBRARY_GOVERNANCE_DSN_FILE 的 0600 文件读 DSN(复用 dobotaiops.ReadDSNFile, 不另写一份同义实现)。两条路径都不进日志/错误/describe,错误只点名缺哪个变量、 缺哪个文件。 - libraryctl 的路由绑定读取按模式分派,只共用行解析;外部路径用只读会话。 libraryagent 的 identity/health 增报 source_db/governance_db 模式;-describe 只打这两项,不再 dump 配置。 - 新增 scripts/library-capture-v090.sh:weknora-v090 的唯一写入者 app + 逐个 声明的 non-writers,两个 -pg 传空走外部模式,归档根可环境变量覆盖, LIBRARYCTL_BIN 缺失即失败;0.7.1 入口脚本一字未动。 怎么验:internal/librarydsn 三条路径(容器/RDS/文件)各有用例,缺变量、缺文件、 0644 文件、非 PostgreSQL 内容均失败关闭且断言错误不含口令;cmd/libraryctl 侧 钉住文件模式接线与"错误不带地址";v090 脚本进 CI 做 bash -n 加标志/服务声明 断言,并拒绝 0.7.1 容器名出现在新入口。make verify 17 项全绿。 未做:真实采集、生产 executor、Key/KB 绑定。 --- CHANGELOG.md | 24 +++ cmd/libraryagent/main.go | 16 +- cmd/libraryctl/governance_dsn_test.go | 41 +++++ cmd/libraryctl/main.go | 112 ++++++++++--- cmd/libraryctl/v090_capture_script_test.go | 59 +++++++ internal/libraryagent/agent.go | 13 +- internal/librarydsn/dsn.go | 140 +++++++++++++++++ internal/librarydsn/dsn_test.go | 174 +++++++++++++++++++++ scripts/library-capture-v090.sh | 56 +++++++ 9 files changed, 601 insertions(+), 34 deletions(-) create mode 100644 cmd/libraryctl/governance_dsn_test.go create mode 100644 cmd/libraryctl/v090_capture_script_test.go create mode 100644 internal/librarydsn/dsn.go create mode 100644 internal/librarydsn/dsn_test.go create mode 100755 scripts/library-capture-v090.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index bce719e..8d3fcba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,29 @@ # Changelog +## 开发快照 0.1.379(采集接入 RDS 型 0.9.0 栈) + +### Added + +- `libraryctl` / `libraryagent` 的数据库来源可选:`-source-pg` 为空时用 `-source-app` + 容器环境里的 `DB_HOST/DB_PORT/DB_NAME/DB_USER/DB_PASSWORD` 直连托管实例(RDS), + TLS 模式取 `LIBRARY_SOURCE_DB_SSLMODE`(默认 `require`,`disable` 仅在显式测试时允许); + `-governance-pg` 为空时从 `LIBRARY_GOVERNANCE_DSN_FILE` 指向的 0600 文件读治理库 DSN。 + 两个都配置时行为与原来完全一致(容器模式),0.7.1 栈不受影响。 +- 新增 `scripts/library-capture-v090.sh`:Compose 项目 `weknora-v090` 的整库采集入口, + 唯一写入者 `app`,其余服务逐个声明理由;两个数据库走外部模式(RDS),归档/日志根 + 默认 `/var/lib/knowledge-governance/{library,freeze}` 并可用环境变量覆盖; + `LIBRARYCTL_BIN` 未设置或不可执行时明确失败,不做"同目录二进制"回退。 +- `libraryctl status` 增印 `databases.source_db/governance_db` 模式(`container|external|file`), + `libraryagent -describe` 输出同样两项;地址与凭据一律不出现。 + +### Tests + +- DSN 来源三条路径(容器 / RDS / 治理库文件)各有用例;缺变量、缺文件、权限过宽、 + 非 PostgreSQL 内容都要求失败关闭,且断言错误里不含口令;`sslmode=prefer` 这类会静默 + 降级的取值直接拒绝。 +- v090 采集脚本进 CI:`bash -n` 语法检查 + 标志/服务声明断言,并拒绝 0.7.1 的 + 容器名与项目名出现在新入口里。 + ## 开发快照 0.1.378(发布写入支持 WeKnora 0.9.0) ### Added diff --git a/cmd/libraryagent/main.go b/cmd/libraryagent/main.go index 81fe64e..9517dd7 100644 --- a/cmd/libraryagent/main.go +++ b/cmd/libraryagent/main.go @@ -14,6 +14,7 @@ import ( "time" "knowledge-governance/internal/libraryagent" + "knowledge-governance/internal/librarydsn" "knowledge-governance/internal/librarymaintenance" ) @@ -25,10 +26,10 @@ func main() { flag.StringVar(&config.ArchiveRoot, "archive", filepath.Join(home, ".local/share/knowledge-governance/library"), "sealed artifact root") flag.StringVar(&config.JournalRoot, "journal", filepath.Join(home, ".local/share/knowledge-governance/freeze"), "freeze journal root") flag.StringVar(&config.SourceApp, "source-app", "WKNFull-app", "source container describing the stack") - flag.StringVar(&config.SourcePG, "source-pg", "WKNFull-postgres", "source PostgreSQL container") + flag.StringVar(&config.SourcePG, "source-pg", "WKNFull-postgres", "source PostgreSQL container; empty = the source DB lives on a managed instance and is reached with the source app env (DB_HOST/...)") flag.StringVar(&config.SourceES, "source-es", "WKNFull-elasticsearch", "source Elasticsearch container") flag.StringVar(&config.SourceS3Endpoint, "source-s3-endpoint", "", "host-reachable source object-store endpoint (overrides the container-internal one)") - flag.StringVar(&config.GovernancePG, "governance-pg", "knowledge-governance-p1-postgres-1", "governance PostgreSQL container") + flag.StringVar(&config.GovernancePG, "governance-pg", "knowledge-governance-p1-postgres-1", "governance PostgreSQL container; empty = read the DSN from LIBRARY_GOVERNANCE_DSN_FILE") flag.StringVar(&config.Project, "project", "weknora071full", "Compose project owning the writers") flag.StringVar(&config.Writers, "writers", "app", "declared writer services") nonWriters := flag.String("non-writers", defaultNonWriters, "service=reason for every other project service") @@ -41,6 +42,13 @@ func main() { if err != nil { log.Fatal(err) } + if *describe { + // describe 只回答"这两个库从哪来"。地址与凭据一律不出现:这段输出会被贴进 + // 工单与日志,而 RDS 的连接串里带口令。 + log.Printf("source_db=%s governance_db=%s", + librarydsn.SourceMode(config.SourcePG), librarydsn.GovernanceMode(config.GovernancePG)) + return + } // 实例级维护接线(入口闸门 + 后台队列)不在 agent 的 Config 里:它由同一份 // 环境声明提供给子进程。启动时先说清楚有没有——缺了封存会在开窗口前被拒, // 不允许"以为接好了"跑到一半才发现没有排空证据。 @@ -49,10 +57,6 @@ func main() { } else { log.Printf("library agent: 实例级维护目标 %s 入口 %s 闸门 %s", target.Instance, target.IngressURL, target.IngressService) } - if *describe { - log.Printf("library agent config: %+v", agent.Describe()) - return - } ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) defer stop() log.Printf("library agent listening on %s (project %s, writers %s)", config.Socket, config.Project, config.Writers) diff --git a/cmd/libraryctl/governance_dsn_test.go b/cmd/libraryctl/governance_dsn_test.go new file mode 100644 index 0000000..605b70b --- /dev/null +++ b/cmd/libraryctl/governance_dsn_test.go @@ -0,0 +1,41 @@ +package main + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" +) + +// 文件模式没配置:必须失败关闭并点名缺的那个变量——"静默退回容器模式"会让 +// 操作员以为在读 RDS,实际在读本机容器(或者干脆读到不存在的容器)。 +func TestGovernanceFileModeFailsClosedWithoutDSNFile(t *testing.T) { + t.Setenv("LIBRARY_GOVERNANCE_DSN_FILE", "") + _, err := readRoutingBindings(context.Background(), options{governancePG: ""}) + if err == nil { + t.Fatal("没有 DSN 文件却读成功了") + } + if !strings.Contains(err.Error(), "LIBRARY_GOVERNANCE_DSN_FILE") { + t.Fatalf("错误没有点名缺哪一项:%v", err) + } +} + +// 外部 DSN 连不上/写错时,错误必须是固定说法:pgx 的错误里带的是连接串(主机、 +// 用户名,口令它自己打码),直接透传就等于把地址写进日志与工单。 +func TestGovernanceFileModeNeverLeaksPasswordInErrors(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "governance.dsn") + // 故意用非法 sslmode + 不可达端口:无论失败在解析还是连接阶段,都不许泄露。 + if err := os.WriteFile(path, []byte("postgres://gov:LEAK-ME-PLEASE@127.0.0.1:1/gov?sslmode=bogus&connect_timeout=1\n"), 0o600); err != nil { + t.Fatal(err) + } + t.Setenv("LIBRARY_GOVERNANCE_DSN_FILE", path) + _, err := readRoutingBindings(context.Background(), options{governancePG: ""}) + if err == nil { + t.Fatal("坏 DSN 竟然读成功了") + } + if strings.Contains(err.Error(), "LEAK-ME-PLEASE") { + t.Fatalf("错误里漏了口令:%v", err) + } +} diff --git a/cmd/libraryctl/main.go b/cmd/libraryctl/main.go index e6c3f16..0bd918a 100644 --- a/cmd/libraryctl/main.go +++ b/cmd/libraryctl/main.go @@ -33,6 +33,7 @@ import ( "github.com/jackc/pgx/v5/pgxpool" "knowledge-governance/internal/librarydeadletter" + "knowledge-governance/internal/librarydsn" "knowledge-governance/internal/libraryfreeze" "knowledge-governance/internal/libraryhandoff" "knowledge-governance/internal/librarymaintenance" @@ -146,9 +147,13 @@ func usage() { Flags (defaults target the local WKNFull test stack): -source-app WKNFull-app container whose env describes the source - -source-pg WKNFull-postgres source PostgreSQL container (address resolution) + -source-pg WKNFull-postgres source PostgreSQL container; empty = the source DB is a + managed instance: connect with the source app env + (DB_HOST/DB_PORT/DB_NAME/DB_USER/DB_PASSWORD), TLS mode from + LIBRARY_SOURCE_DB_SSLMODE (default require) -source-es WKNFull-elasticsearch - -governance-pg knowledge-governance-p1-postgres-1 routing bindings read here + -governance-pg knowledge-governance-p1-postgres-1 routing bindings read here; empty = read + the DSN from LIBRARY_GOVERNANCE_DSN_FILE (0600 file) -project weknora071full Compose project owning the writer group -writers WKNFull-app declared writer services (comma separated) -non-writers WKNFull-frontend=reason,... every other project service @@ -182,9 +187,9 @@ func parseFlags(command string, args []string) (options, []string, error) { set := flag.NewFlagSet(command, flag.ContinueOnError) set.SetOutput(os.Stderr) set.StringVar(&opts.sourceApp, "source-app", opts.sourceApp, "source container describing the stack") - set.StringVar(&opts.sourcePG, "source-pg", opts.sourcePG, "source PostgreSQL container") + set.StringVar(&opts.sourcePG, "source-pg", opts.sourcePG, "source PostgreSQL container; empty = connect to the source app's external (managed) database") set.StringVar(&opts.sourceES, "source-es", opts.sourceES, "source Elasticsearch container") - set.StringVar(&opts.governancePG, "governance-pg", opts.governancePG, "governance PostgreSQL container") + set.StringVar(&opts.governancePG, "governance-pg", opts.governancePG, "governance PostgreSQL container; empty = read the DSN from LIBRARY_GOVERNANCE_DSN_FILE") set.StringVar(&opts.project, "project", opts.project, "Compose project owning the writers") set.StringVar(&opts.writers, "writers", opts.writers, "declared writer services") set.StringVar(&opts.nonWriters, "non-writers", "", "service=reason for every other project service") @@ -285,15 +290,25 @@ func discoverSource(ctx context.Context, opts options, binding map[string]string if app["DB_DRIVER"] != "postgres" { return config, errors.New("source database driver is not postgres") } - address, err := containerAddress(ctx, opts.sourcePG) - if err != nil { - return config, err + // 两种来源:给了 -source-pg 就按容器地址拼(0.7.1 栈);没给说明库是托管 + // 实例(RDS 型 0.9.0 栈没有源库容器),用源应用容器 env 里的 DB_* 直连。 + if librarydsn.SourceMode(opts.sourcePG) == librarydsn.SourceExternal { + dsn, dsnErr := librarydsn.SourceURL(app, os.Getenv) + if dsnErr != nil { + return config, dsnErr + } + config.DatabaseURL = dsn + } else { + address, addressErr := containerAddress(ctx, opts.sourcePG) + if addressErr != nil { + return config, addressErr + } + dsn, dsnErr := librarydsn.ContainerURL(address, app) + if dsnErr != nil { + return config, dsnErr + } + config.DatabaseURL = dsn } - dsn := strings.Join([]string{ - "postgres://" + urlEscape(app["DB_USER"]) + ":" + urlEscape(app["DB_PASSWORD"]) + - "@" + address + ":" + app["DB_PORT"] + "/" + urlEscape(app["DB_NAME"]) + "?sslmode=disable", - }, "") - config.DatabaseURL = dsn config.AppImageID, err = containerImageID(ctx, opts.sourceApp) if err != nil { return config, err @@ -388,17 +403,12 @@ func parseBool(value string) bool { return false } -func urlEscape(value string) string { - replacer := strings.NewReplacer("%", "%25", ":", "%3A", "@", "%40", "/", "%2F", "?", "%3F", "#", "%23", "&", "%26", " ", "%20") - return replacer.Replace(value) -} - // scopeBindings 返回 scope→知识库 映射。治理 API 用的是 LIBRARY_PRODUCTION_KB_*, // CLI 读的是治理库里的 routing bindings;同一台机器上两份不一致时, // CLI 抓到的会是"配置里那个库"而不是 API 认为的目标——所以环境变量优先, // 读不到才退回 bindings。两边讲同一份身份,验收才可比。 -func scopeBindings(ctx context.Context, container string) (map[string]string, error) { - binding, err := readRoutingBindings(ctx, container) +func scopeBindings(ctx context.Context, opts options) (map[string]string, error) { + binding, err := readRoutingBindings(ctx, opts) if err != nil { return nil, err } @@ -410,12 +420,57 @@ func scopeBindings(ctx context.Context, container string) (map[string]string, er return binding, nil } -func readRoutingBindings(ctx context.Context, container string) (map[string]string, error) { - raw, err := dockerOutput(ctx, "exec", container, "psql", "-U", "governance", "-d", "governance", "-tAc", - "SELECT internal_kb_id||'|'||external_kb_id FROM routing_kb_bindings WHERE id='active'") +// readRoutingBindings 读治理库里生效的 scope 绑定。两种来源: +// - 给了 -governance-pg:docker exec 进治理库容器跑 psql(0.7.1 本机栈); +// - 没给:治理库是托管实例(RDS 型 0.9.0 栈没有 governance 容器),DSN 来自 +// LIBRARY_GOVERNANCE_DSN_FILE 指向的 0600 文件。 +// +// 两条路径只共用行解析,不共用连接方式:容器里没有网络到 RDS 的凭据,宿主机也不该 +// 为了读两列去 docker exec 一个不存在的容器。 +func readRoutingBindings(ctx context.Context, opts options) (map[string]string, error) { + if librarydsn.GovernanceMode(opts.governancePG) == librarydsn.GovernanceFile { + dsn, err := librarydsn.GovernanceURL(os.Getenv) + if err != nil { + return nil, err + } + return readRoutingBindingsFromDSN(ctx, dsn) + } + raw, err := dockerOutput(ctx, "exec", opts.governancePG, "psql", "-U", "governance", "-d", "governance", "-tAc", + routingBindingsQuery) if err != nil { return nil, errors.New("read routing bindings failed") } + return parseRoutingBindings(raw) +} + +const routingBindingsQuery = "SELECT internal_kb_id||'|'||external_kb_id FROM routing_kb_bindings WHERE id='active'" + +// readRoutingBindingsFromDSN 用外部 DSN 读同一份绑定。 +// +// 只读会话是刻意的:这条路径只是读两列,出了问题必须是"读不到",而不是在治理库上 +// 留下一次写入。错误一律收敛成固定说法:pgx 的错误里会带连接串(口令它自己打了码, +// 但主机与用户名照旧出去),而这里连那份地址都不该出现在日志或工单里。 +func readRoutingBindingsFromDSN(ctx context.Context, dsn string) (map[string]string, error) { + parsed, err := pgxpool.ParseConfig(dsn) + if err != nil { + return nil, errors.New("governance database url is invalid") + } + parsed.ConnConfig.RuntimeParams["default_transaction_read_only"] = "on" + parsed.ConnConfig.RuntimeParams["statement_timeout"] = "30000" + parsed.MaxConns = 2 + pool, err := pgxpool.NewWithConfig(ctx, parsed) + if err != nil { + return nil, errors.New("open governance database failed") + } + defer pool.Close() + var raw string + if err = pool.QueryRow(ctx, routingBindingsQuery).Scan(&raw); err != nil { + return nil, errors.New("read routing bindings failed") + } + return parseRoutingBindings(raw) +} + +func parseRoutingBindings(raw string) (map[string]string, error) { internal, external, ok := strings.Cut(strings.TrimSpace(raw), "|") if !ok || internal == "" || external == "" { return nil, errors.New("routing bindings are incomplete") @@ -587,7 +642,12 @@ func openArchive(opts options) (*librarysnapshot.Archive, error) { func runStatus(ctx context.Context, opts options) error { report := map[string]any{} - binding, bindingErr := scopeBindings(ctx, opts.governancePG) + // 只报来源模式:地址与凭据连 describe/status 都不出现,日志自然也不会带出去。 + report["databases"] = map[string]string{ + "source_db": librarydsn.SourceMode(opts.sourcePG), + "governance_db": librarydsn.GovernanceMode(opts.governancePG), + } + binding, bindingErr := scopeBindings(ctx, opts) var config sourceConfig var err error if bindingErr != nil { @@ -680,7 +740,7 @@ func runPreflight(ctx context.Context, opts options) error { if opts.scope != "internal" && opts.scope != "external" { return errors.New("-scope must be internal or external") } - binding, err := scopeBindings(ctx, opts.governancePG) + binding, err := scopeBindings(ctx, opts) if err != nil { return err } @@ -773,7 +833,7 @@ func runCapture(ctx context.Context, opts options) error { return err } defer archive.Close() - binding, err := scopeBindings(ctx, opts.governancePG) + binding, err := scopeBindings(ctx, opts) if err != nil { return err } @@ -1150,7 +1210,7 @@ func runDeadLetters(ctx context.Context, opts options) error { if opts.scope != "internal" && opts.scope != "external" { return errors.New("-scope must be internal or external") } - binding, err := scopeBindings(ctx, opts.governancePG) + binding, err := scopeBindings(ctx, opts) if err != nil { return err } diff --git a/cmd/libraryctl/v090_capture_script_test.go b/cmd/libraryctl/v090_capture_script_test.go new file mode 100644 index 0000000..9bd1eb9 --- /dev/null +++ b/cmd/libraryctl/v090_capture_script_test.go @@ -0,0 +1,59 @@ +package main + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// v090 采集脚本是本轮新增的唯一入口。为什么用 bash -n 而不是"跑一遍":跑一遍要 +// 停写入者、连 RDS——那是真实采集,不是单元测试。语法与声明在这里钉住,能让 +// "标志写错一个字母"在 CI 里就红,而不是等到开维护窗口时才发现。 +func TestV090CaptureScriptIsSyntacticallyValid(t *testing.T) { + bash, err := exec.LookPath("bash") + if err != nil { + t.Fatalf("需要 bash 才能检查脚本语法(CI 与测试机都有):%v", err) + } + path := filepath.Join("..", "..", "scripts", "library-capture-v090.sh") + out, err := exec.Command(bash, "-n", path).CombinedOutput() + if err != nil { + t.Fatalf("bash -n %s 失败:%v\n%s", path, err, out) + } +} + +// 脚本内容断言:外部库模式(两个 -pg 传空)、写者声明、容器名、归档根、 +// LIBRARYCTL_BIN 缺失即失败——这些一旦漂移,真跑时会封错栈或读错库。 +func TestV090CaptureScriptDeclaresExternalDatabases(t *testing.T) { + body, err := os.ReadFile(filepath.Join("..", "..", "scripts", "library-capture-v090.sh")) + if err != nil { + t.Fatalf("读脚本:%v", err) + } + for _, want := range []string{ + "set -euo pipefail", + "LIBRARYCTL_MISSING", + "-project weknora-v090", + "-writers app", + "-source-app weknora-v090-app", + "-source-es weknora-v090-elasticsearch", + `-source-pg ""`, + `-governance-pg ""`, + "frontend=UI only; no knowledge writes", + "docreader=parse RPC service; app persists parse output", + "elasticsearch=search store; not an initiator", + "redis=queue/cache store; not an initiator", + "/var/lib/knowledge-governance/library", + "/var/lib/knowledge-governance/freeze", + } { + if !strings.Contains(string(body), want) { + t.Fatalf("脚本缺少 %q", want) + } + } + // 0.7.1 的服务名绝不能出现在 v090 入口里:那会让封存停错栈。 + for _, forbidden := range []string{"WKNFull", "weknora071full"} { + if strings.Contains(string(body), forbidden) { + t.Fatalf("v090 脚本里出现了 0.7.1 的 %q", forbidden) + } + } +} diff --git a/internal/libraryagent/agent.go b/internal/libraryagent/agent.go index f9a1051..d871a27 100644 --- a/internal/libraryagent/agent.go +++ b/internal/libraryagent/agent.go @@ -26,6 +26,8 @@ import ( "sync" "syscall" "time" + + "knowledge-governance/internal/librarydsn" ) // Config is fixed at start-up by the operator (systemd unit / CLI flags). It is @@ -429,15 +431,22 @@ func (a *Agent) health() response { } // identity 被日志与健康应答共用一份,避免两处描述漂移。 +// +// 数据库只报来源模式(container|external|file),不报地址:agent 的应答会被治理侧 +// 存档,DSN 里带口令,一旦写进身份就再也收不回来了。 func (c Config) identity() map[string]any { identity := map[string]any{ "project": c.Project, "archive_root": c.ArchiveRoot, "journal_root": c.JournalRoot, "source_app": c.SourceApp, - "source_pg": c.SourcePG, "source_es": c.SourceES, - "writers": c.Writers, + // source_pg 仍是容器名(不是地址、不是凭据),保留它以让既有健康页/事后核对 + // 读得到同一份字段;数据库**来源**由下面两个模式字段表达,外部模式下它为空。 + "source_pg": c.SourcePG, + "source_db": librarydsn.SourceMode(c.SourcePG), + "governance_db": librarydsn.GovernanceMode(c.GovernancePG), + "writers": c.Writers, } if c.SourceS3Endpoint != "" { identity["source_s3_endpoint"] = c.SourceS3Endpoint diff --git a/internal/librarydsn/dsn.go b/internal/librarydsn/dsn.go new file mode 100644 index 0000000..31b4dea --- /dev/null +++ b/internal/librarydsn/dsn.go @@ -0,0 +1,140 @@ +// Package librarydsn 决定整库采集链路里两个 DSN 究竟从哪来。 +// +// 为什么要有它:容器模式下 DSN 是"源库容器的地址 + 源应用容器的 env"拼出来的; +// RDS 型 0.9.0 栈没有源库容器(库是托管实例),治理库同样只在外部。旧拼法在那种 +// 栈上不是权限不够,而是地址解析方式错了——库根本没有容器地址可查。把"用哪种 +// 来源"收敛成一处,libraryctl 与 libraryagent 共用同一份判定,日志/describe 只报 +// 模式(container|external|file),永远不报地址与凭据。 +package librarydsn + +import ( + "errors" + "fmt" + "net" + "net/url" + "strings" + + "knowledge-governance/internal/dobotaiops" +) + +const ( + // SourceContainer:-source-pg 给了容器名,地址从容器网络解析(0.7.1 栈)。 + SourceContainer = "container" + // SourceExternal:-source-pg 为空,直连源应用容器 env 里声明的托管库(RDS)。 + SourceExternal = "external" + + // GovernanceContainer:-governance-pg 给了容器名,用 docker exec psql 读。 + GovernanceContainer = "container" + // GovernanceFile:-governance-pg 为空,DSN 来自 0600 文件(LIBRARY_GOVERNANCE_DSN_FILE)。 + GovernanceFile = "file" +) + +// allowedSSLModes 是允许的 sslmode 集合。 +// +// 只收这四个:prefer/allow 会在服务端不支持时**静默降级成明文**,托管实例上那等于 +// "以为加密了其实没有"——这种"看起来配好了"的失败模式比直接报错更难发现。 +var allowedSSLModes = map[string]bool{ + "disable": true, // 仅显式测试:本机回环或明确无 TLS 的隔离库 + "require": true, + "verify-ca": true, + "verify-full": true, +} + +// SourceMode 把 -source-pg 的取值翻译成来源模式。空值 = 外部(RDS)。 +func SourceMode(sourcePG string) string { + if strings.TrimSpace(sourcePG) == "" { + return SourceExternal + } + return SourceContainer +} + +// GovernanceMode 把 -governance-pg 的取值翻译成来源模式。空值 = DSN 文件。 +func GovernanceMode(governancePG string) string { + if strings.TrimSpace(governancePG) == "" { + return GovernanceFile + } + return GovernanceContainer +} + +// ContainerURL 复现容器模式原有的 DSN 形状:地址来自容器网络,sslmode=disable +// (容器私网内没有 TLS 终结)。抽到这里只为一件事:让"模式选择"有一条可测的路径, +// 而形状与旧实现逐字节一致(真跑过的栈不能因为重构换了连接串)。 +func ContainerURL(address string, app map[string]string) (string, error) { + if strings.TrimSpace(address) == "" { + return "", errors.New("source container address is empty") + } + for _, key := range []string{"DB_USER", "DB_PASSWORD", "DB_NAME"} { + if app[key] == "" { + return "", fmt.Errorf("source app environment is missing %s", key) + } + } + target := url.URL{ + Scheme: "postgres", + User: url.UserPassword(app["DB_USER"], app["DB_PASSWORD"]), + Host: net.JoinHostPort(address, defaultPort(app["DB_PORT"])), + Path: "/" + app["DB_NAME"], + RawQuery: url.Values{"sslmode": {"disable"}}.Encode(), + } + return target.String(), nil +} + +// SourceURL 用源应用容器 env 里的 DB_* 直连托管库(RDS)。 +// +// 缺哪个变量就点哪个变量名:把"连接被拒"与"配置缺一项"分开,才不用去猜着改。 +// 错误里只有变量名——DB_PASSWORD 的值绝不能出现在任何一处输出里。 +func SourceURL(app map[string]string, getenv func(string) string) (string, error) { + missing := []string{} + for _, key := range []string{"DB_HOST", "DB_USER", "DB_NAME"} { + if strings.TrimSpace(app[key]) == "" { + missing = append(missing, key) + } + } + // 口令不 TrimSpace:前后空格也是口令的一部分,改掉就成另一个口令了。 + if app["DB_PASSWORD"] == "" { + missing = append(missing, "DB_PASSWORD") + } + if len(missing) > 0 { + return "", fmt.Errorf("source app environment is missing %s", strings.Join(missing, ", ")) + } + sslmode := strings.TrimSpace(getenv("LIBRARY_SOURCE_DB_SSLMODE")) + if sslmode == "" { + sslmode = "require" + } + if !allowedSSLModes[sslmode] { + return "", errors.New("LIBRARY_SOURCE_DB_SSLMODE must be one of disable, require, verify-ca, verify-full") + } + target := url.URL{ + Scheme: "postgres", + User: url.UserPassword(app["DB_USER"], app["DB_PASSWORD"]), + Host: net.JoinHostPort(strings.TrimSpace(app["DB_HOST"]), defaultPort(app["DB_PORT"])), + Path: "/" + strings.TrimSpace(app["DB_NAME"]), + RawQuery: url.Values{"sslmode": {sslmode}}.Encode(), + } + return target.String(), nil +} + +// GovernanceURL 从 0600 文件读治理库 DSN。 +// +// 文件读法复用 dobotaiops.ReadDSNFile:那里已经把"必须是普通文件、必须 0600、 +// 错误只报路径与原因"钉住并测过——再写一份同义实现只会让两条路径的规则慢慢分叉。 +func GovernanceURL(getenv func(string) string) (string, error) { + path := strings.TrimSpace(getenv("LIBRARY_GOVERNANCE_DSN_FILE")) + if path == "" { + return "", errors.New("LIBRARY_GOVERNANCE_DSN_FILE is not set: the governance DSN file is required when -governance-pg is empty") + } + dsn, err := dobotaiops.ReadDSNFile(path) + if err != nil { + // ReadDSNFile 的错误只含路径与原因,不含 DSN/口令。 + return "", fmt.Errorf("governance DSN file unusable: %w", err) + } + return dsn, nil +} + +// defaultPort:托管实例的连接信息里端口常常单独给;缺了就按 PostgreSQL 默认。 +// 端口不是凭据,给默认值不会把"配错了"藏起来。 +func defaultPort(port string) string { + if trimmed := strings.TrimSpace(port); trimmed != "" { + return trimmed + } + return "5432" +} diff --git a/internal/librarydsn/dsn_test.go b/internal/librarydsn/dsn_test.go new file mode 100644 index 0000000..776728a --- /dev/null +++ b/internal/librarydsn/dsn_test.go @@ -0,0 +1,174 @@ +package librarydsn + +import ( + "net/url" + "os" + "path/filepath" + "strings" + "testing" +) + +func envFunc(pairs map[string]string) func(string) string { + return func(key string) string { return pairs[key] } +} + +// 容器模式:旧拼法保持同一目标——同样的用户/口令/地址/库名,sslmode=disable。 +// 真跑过的 0.7.1 栈不能因为这次接入换了连接目标(换 sslmode 或换转义方式都会让 +// "同一个库"连不上或连成别的身份)。断言用解析后的分量,不用字面转义: +// userinfo 里 ':' 保不保留不影响解析结果,钉字面只会把重构误判成回归。 +func TestContainerURLKeepsLegacyShape(t *testing.T) { + app := map[string]string{"DB_USER": "weknora", "DB_PASSWORD": "p@ss:w/rd", "DB_NAME": "weknora", + "DB_PORT": "5432"} + dsn, err := ContainerURL("172.18.0.5", app) + if err != nil { + t.Fatalf("container url: %v", err) + } + parsed, err := url.Parse(dsn) + if err != nil { + t.Fatalf("生成的连接串自己都解析不了:%v", err) + } + if parsed.Scheme != "postgres" || parsed.Host != "172.18.0.5:5432" || parsed.Path != "/weknora" { + t.Fatalf("容器模式连接目标变了:%s", dsn) + } + if parsed.User.Username() != "weknora" { + t.Fatalf("用户名变了:%s", dsn) + } + if password, _ := parsed.User.Password(); password != "p@ss:w/rd" { + t.Fatalf("口令转义后取不回来:%s", dsn) + } + if parsed.Query().Get("sslmode") != "disable" { + t.Fatalf("容器模式不该改 sslmode:%s", dsn) + } + if _, err = ContainerURL("", app); err == nil { + t.Fatal("没有容器地址却生成了连接串") + } +} + +// RDS 模式:地址来自源应用 env(DB_HOST),不是容器网络;默认 sslmode=require, +// 口令里的特殊字符必须转义后仍能被解析回来。 +func TestSourceURLUsesRDSHostAndRequiresTLSByDefault(t *testing.T) { + app := map[string]string{ + "DB_HOST": "pgm-2ze0abcd.pg.rds.aliyuncs.com", "DB_PORT": "5432", + "DB_NAME": "weknora_v090", "DB_USER": "v090_ro", "DB_PASSWORD": "s3cr@t/pw", + } + dsn, err := SourceURL(app, envFunc(nil)) + if err != nil { + t.Fatalf("rds url: %v", err) + } + parsed, err := url.Parse(dsn) + if err != nil { + t.Fatalf("生成的连接串自己都解析不了:%v", err) + } + if parsed.Host != "pgm-2ze0abcd.pg.rds.aliyuncs.com:5432" { + t.Fatalf("host 不是 RDS 地址:%q", parsed.Host) + } + if got := parsed.Query().Get("sslmode"); got != "require" { + t.Fatalf("默认 sslmode 不是 require:%q", got) + } + if password, _ := parsed.User.Password(); password != "s3cr@t/pw" { + t.Fatal("口令转义后取不回来") + } + // 显式 disable 只用于本机隔离测试;它是允许的,但必须由调用方写清楚。 + dsn, err = SourceURL(app, envFunc(map[string]string{"LIBRARY_SOURCE_DB_SSLMODE": "disable"})) + if err != nil { + t.Fatalf("explicit disable: %v", err) + } + if !strings.HasSuffix(dsn, "sslmode=disable") { + t.Fatalf("显式 disable 没被采纳:%s", dsn) + } + // prefer 会静默降级成明文:那不是"配好了 TLS",直接拒。 + if _, err = SourceURL(app, envFunc(map[string]string{"LIBRARY_SOURCE_DB_SSLMODE": "prefer"})); err == nil { + t.Fatal("sslmode=prefer 被接受了") + } +} + +// 缺变量必须失败,且点名缺的那一项;错误里不许出现口令。 +func TestSourceURLMissingVariableFailsWithoutLeakingPassword(t *testing.T) { + app := map[string]string{"DB_HOST": "pgm-x.pg.rds.aliyuncs.com", "DB_NAME": "kb", + "DB_USER": "u", "DB_PASSWORD": "TOP-SECRET-VALUE"} + delete(app, "DB_HOST") + _, err := SourceURL(app, envFunc(nil)) + if err == nil { + t.Fatal("缺 DB_HOST 竟然成功了") + } + if !strings.Contains(err.Error(), "DB_HOST") { + t.Fatalf("错误没有点名缺哪一项:%v", err) + } + if strings.Contains(err.Error(), "TOP-SECRET-VALUE") { + t.Fatalf("错误里漏了口令:%v", err) + } + app["DB_HOST"] = "pgm-x.pg.rds.aliyuncs.com" + delete(app, "DB_PASSWORD") + _, err = SourceURL(app, envFunc(nil)) + if err == nil || !strings.Contains(err.Error(), "DB_PASSWORD") { + t.Fatalf("缺 DB_PASSWORD 没有点名:%v", err) + } + if strings.Contains(err.Error(), "TOP-SECRET-VALUE") { + t.Fatalf("错误里漏了口令:%v", err) + } +} + +// 治理库文件模式:0600 文件里的完整 DSN 原样读出;文件缺失/权限过宽/内容不是 +// PostgreSQL DSN 都必须失败关闭,且错误只提到路径与原因。 +func TestGovernanceURLReadsDSNFileAndFailsClosed(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "governance.dsn") + const password = "GOV-PASSWORD-VALUE" + dsn := "postgres://governance:" + password + "@pgm-gov.pg.rds.aliyuncs.com:5432/knowledge_governance_test?sslmode=require" + if err := os.WriteFile(path, []byte(dsn+"\n"), 0o600); err != nil { + t.Fatal(err) + } + got, err := GovernanceURL(envFunc(map[string]string{"LIBRARY_GOVERNANCE_DSN_FILE": path})) + if err != nil { + t.Fatalf("file mode: %v", err) + } + if got != dsn { + t.Fatalf("读出的 DSN 与文件内容不一致(不想在这里做任何改写)") + } + + // 没声明文件:失败并点名变量。 + if _, err = GovernanceURL(envFunc(nil)); err == nil || !strings.Contains(err.Error(), "LIBRARY_GOVERNANCE_DSN_FILE") { + t.Fatalf("没配置文件时没有点名变量:%v", err) + } + + // 文件不存在:点名路径,不泄漏任何内容。 + missing := filepath.Join(dir, "absent.dsn") + if _, err = GovernanceURL(envFunc(map[string]string{"LIBRARY_GOVERNANCE_DSN_FILE": missing})); err == nil || + !strings.Contains(err.Error(), missing) { + t.Fatalf("文件缺失没有点名路径:%v", err) + } + + // 组/其他可读:拒绝(这条规则由 dobotaiops.ReadDSNFile 判定,这里只钉住接线)。 + wide := filepath.Join(dir, "wide.dsn") + if err = os.WriteFile(wide, []byte(dsn+"\n"), 0o644); err != nil { + t.Fatal(err) + } + if err = os.Chmod(wide, 0o644); err != nil { + t.Fatal(err) + } + if _, err = GovernanceURL(envFunc(map[string]string{"LIBRARY_GOVERNANCE_DSN_FILE": wide})); err == nil { + t.Fatal("0644 的 DSN 文件被接受了") + } + + // 内容不是 PostgreSQL DSN:失败,且不要把内容回显进错误。 + bogus := filepath.Join(dir, "bogus.dsn") + if err = os.WriteFile(bogus, []byte("mysql://u:"+password+"@host/db\n"), 0o600); err != nil { + t.Fatal(err) + } + _, err = GovernanceURL(envFunc(map[string]string{"LIBRARY_GOVERNANCE_DSN_FILE": bogus})) + if err == nil { + t.Fatal("非 PostgreSQL DSN 被接受了") + } + if strings.Contains(err.Error(), password) { + t.Fatalf("错误里漏了口令:%v", err) + } +} + +func TestModesDeriveFromEmptyFlag(t *testing.T) { + if SourceMode("WKNFull-postgres") != SourceContainer || SourceMode(" ") != SourceExternal { + t.Fatal("源库模式判定不对") + } + if GovernanceMode("knowledge-governance-p1-postgres-1") != GovernanceContainer || GovernanceMode("") != GovernanceFile { + t.Fatal("治理库模式判定不对") + } +} diff --git a/scripts/library-capture-v090.sh b/scripts/library-capture-v090.sh new file mode 100755 index 0000000..d32fdf6 --- /dev/null +++ b/scripts/library-capture-v090.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# Whole-library capture entry point for the RDS-backed WeKnora 0.9.0 test stack +# (Compose project weknora-v090). +# +# Why a separate file instead of a parameterised one: this stack has no source +# PostgreSQL container and no governance PostgreSQL container. Both databases are +# managed instances (RDS). -source-pg / -governance-pg are therefore passed empty, +# which is libraryctl's documented signal to resolve them from the source app +# container env (DB_HOST/DB_PORT/DB_NAME/DB_USER/DB_PASSWORD + LIBRARY_SOURCE_DB_SSLMODE) +# and from LIBRARY_GOVERNANCE_DSN_FILE (0600 file). Keeping the 0.7.1 entry point +# untouched means a change here can never alter a stack that already ran. +# +# Every service of the Compose project is declared below: exactly one writer (app) +# and a reason for each sibling which must NOT be stopped. ProjectController +# refuses to run if the project gains a service that is not declared here, so a +# new container cannot be silently left outside the maintenance window. +# +# Credentials are read at call time (container env / 0600 file) and are never +# printed. Nothing here writes to the source stack except SIGTERM/SIGSTART of the +# declared writer. +set -euo pipefail + +# The binary is pinned by the caller on purpose: there is no "whatever is in this +# directory" fallback, so a stale build next to the script cannot be executed by +# accident against a live stack. +BIN="${LIBRARYCTL_BIN:-}" +if [ -z "$BIN" ]; then + echo "LIBRARYCTL_MISSING: set LIBRARYCTL_BIN to the libraryctl binary built for this run" >&2 + exit 1 +fi +if [ ! -x "$BIN" ]; then + echo "LIBRARYCTL_NOT_EXECUTABLE: $BIN is not an executable file" >&2 + exit 1 +fi + +# Archives/journals live outside the home directory on this host (they survive the +# operator account); both roots stay overridable for an isolated rehearsal. +ARCHIVE_ROOT="${LIBRARY_ARCHIVE_ROOT:-/var/lib/knowledge-governance/library}" +JOURNAL_ROOT="${LIBRARY_JOURNAL_ROOT:-/var/lib/knowledge-governance/freeze}" + +# Reasons must not contain commas: the flag is comma-separated. +NON_WRITERS="frontend=UI only; no knowledge writes" +NON_WRITERS+=",docreader=parse RPC service; app persists parse output" +NON_WRITERS+=",elasticsearch=search store; not an initiator" +NON_WRITERS+=",redis=queue/cache store; not an initiator" + +exec "$BIN" "$@" \ + -project weknora-v090 \ + -writers app \ + -source-app weknora-v090-app \ + -source-es weknora-v090-elasticsearch \ + -source-pg "" \ + -governance-pg "" \ + -non-writers "$NON_WRITERS" \ + -archive "$ARCHIVE_ROOT" \ + -journal "$JOURNAL_ROOT" \ No newline at end of file