// Package libraryagent is the least-privilege execution boundary for // whole-library capture on the target host. // // Why it exists: capturing a library has to stop the source writer for a few // seconds and read source credentials. Handing the governance containers the // Docker socket (a full write API) or the source credentials would be a much // larger privilege than that operation needs. Instead this agent runs as the // ordinary operator account on the host, listens on one unix socket which only // that account can connect to, and executes one fixed command with a fixed // writer declaration. A caller can choose the scope and (optionally) a job id — // nothing else. package libraryagent import ( "bufio" "context" "encoding/json" "errors" "log" "net" "os" "os/exec" "path/filepath" "regexp" "strings" "sync" "syscall" "time" ) // Config is fixed at start-up by the operator (systemd unit / CLI flags). It is // never taken from a request. type Config struct { Socket string Binary string ArchiveRoot string JournalRoot string // Writer declaration passed to the binary, exactly as reviewed in // scripts/library-capture.sh. Clients cannot influence it. SourceApp string SourcePG string SourceES string // SourceS3Endpoint 让宿主机用可达地址读源对象存储:容器 env 里的端点常常 // 只是服务名(minio:9000),宿主机解析不到,资源导出会失败。 SourceS3Endpoint string GovernancePG string Project string Writers string NonWriters string MaxDuration time.Duration // StopGrace bounds the graceful stop of one operation: SIGTERM first (the // child aborts the maintenance window and restores writers), then SIGKILL // only after this grace period. It must exceed the child's own restore budget. StopGrace time.Duration // RecoverTimeout bounds the recovery run performed after a failed or killed // operation, and at agent start-up. RecoverTimeout time.Duration // Run executes the binary; replaced in tests. Run func(ctx context.Context, name string, args ...string) ([]byte, error) } type request struct { Action string `json:"action"` Scope string `json:"scope,omitempty"` Job string `json:"job,omitempty"` // Released 只允许精确死信指纹(sha256:<64 hex>)。它不改变固定命令形态: // 还能传的只有"这些确切的行已由治理平台审计解除",CLI 会逐字节复核。 Released []string `json:"released,omitempty"` // Expect 是调用方对执行器身份的期望。capture 必须在操作互斥内、任何源操作 // 之前比对它;缺失即拒绝(fail closed),不靠"事后核对"兜底。 Expect *Expectation `json:"expect,omitempty"` } type response struct { OK bool `json:"ok"` Action string `json:"action"` Scope string `json:"scope,omitempty"` Job string `json:"job,omitempty"` Error string `json:"error,omitempty"` Payload json.RawMessage `json:"payload,omitempty"` } var jobPattern = regexp.MustCompile(`^[A-Za-z0-9_-]{1,64}$`) // 死信解除只能按精确指纹传入,不接受通配、实例或任务类型级解除。 var fingerprintPattern = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`) const maxReleasedDeadLetters = 64 // Agent serves capture requests. Only one operation runs at a time: a library // capture owns the writer group, so overlapping requests would race the window. type Agent struct { config Config mu sync.Mutex // stateMu 只保护下面两个"给人看"的字段:健康应答不能被一次 15 分钟的封存 // 卡住(否则页面会把"正在封存"读成"执行器不可达")。 stateMu sync.RWMutex // degraded is non-empty when a failed operation could not be followed by a // verified recovery. While set, capture requests are refused: leaving the // source writer stopped is worse than refusing work. degraded string busy bool wg sync.WaitGroup } // captureState 是健康应答要暴露的执行器状态:是否在跑、是否降级。 type captureState struct { Busy bool `json:"busy"` Degraded string `json:"degraded"` } func (a *Agent) snapshotState() captureState { a.stateMu.RLock() defer a.stateMu.RUnlock() return captureState{Busy: a.busy, Degraded: a.degraded} } func (a *Agent) setBusy(busy bool) { a.stateMu.Lock() defer a.stateMu.Unlock() a.busy = busy } func (a *Agent) setDegraded(reason string) { a.stateMu.Lock() defer a.stateMu.Unlock() a.degraded = reason } func (a *Agent) clearDegraded() { a.setDegraded("") } func New(config Config) (*Agent, error) { if !filepath.IsAbs(config.Socket) || !filepath.IsAbs(config.ArchiveRoot) || !filepath.IsAbs(config.JournalRoot) { return nil, errors.New("library agent paths must be absolute") } if config.Binary == "" || config.Project == "" || config.Writers == "" || !strings.Contains(config.NonWriters, "=") { return nil, errors.New("library agent needs a binary, project, writers and a non-writer declaration") } if config.MaxDuration <= 0 || config.MaxDuration > 30*time.Minute { return nil, errors.New("library agent duration bound is invalid") } if config.StopGrace == 0 { config.StopGrace = 3 * time.Minute } if config.RecoverTimeout == 0 { config.RecoverTimeout = 3 * time.Minute } if config.StopGrace <= 0 || config.StopGrace > 10*time.Minute || config.RecoverTimeout <= 0 || config.RecoverTimeout > 10*time.Minute { return nil, errors.New("library agent recovery/stop budget is invalid") } return &Agent{config: config}, nil } // Serve listens until ctx is cancelled. The socket is created 0600 inside a // 0700 directory and removed on shutdown. func (a *Agent) Serve(ctx context.Context) error { if err := os.MkdirAll(filepath.Dir(a.config.Socket), 0700); err != nil { return err } if err := os.Chmod(filepath.Dir(a.config.Socket), 0700); err != nil { return err } _ = os.Remove(a.config.Socket) listener, err := net.Listen("unix", a.config.Socket) if err != nil { return err } defer func() { listener.Close() _ = os.Remove(a.config.Socket) }() if err := os.Chmod(a.config.Socket, 0600); err != nil { return err } // Startup responsibility: an agent restart may have inherited a stopped // writer from a previous killed run. Recover before accepting work; if that // fails, stay degraded and refuse captures instead of pretending to be ready. if err := a.recoverWriters(ctx); err != nil { a.setDegraded(err.Error()) log.Printf("library agent: pending freeze recovery failed: %v", err) } go func() { <-ctx.Done() _ = listener.Close() }() for { conn, err := listener.Accept() if err != nil { if ctx.Err() != nil { // Drain in-flight operations: each of them still owes a bounded // recovery before the agent may exit. done := make(chan struct{}) go func() { a.wg.Wait(); close(done) }() select { case <-done: case <-time.After(a.config.StopGrace + a.config.RecoverTimeout): log.Printf("library agent: shutdown deadline exceeded with operations in flight") } return nil } return err } a.wg.Add(1) go func(conn net.Conn) { defer a.wg.Done() a.handle(ctx, conn) }(conn) } } func (a *Agent) handle(ctx context.Context, conn net.Conn) { defer conn.Close() if err := peerAllowed(conn); err != nil { _ = json.NewEncoder(conn).Encode(response{OK: false, Error: err.Error()}) return } _ = conn.SetDeadline(time.Now().Add(a.config.MaxDuration + 30*time.Second)) decoder := json.NewDecoder(bufio.NewReader(conn)) var req request if err := decoder.Decode(&req); err != nil { _ = json.NewEncoder(conn).Encode(response{OK: false, Error: "invalid request"}) return } result := a.execute(ctx, req) _ = json.NewEncoder(conn).Encode(result) } // execute serialises operations and maps a request onto the one fixed command. func (a *Agent) execute(ctx context.Context, req request) response { // health 不进操作锁、不跑子进程:调用方要的是"这个执行器指向谁、能不能干活", // 而一次正在进行的封存不该让这个回答变成"不可达"。 if req.Action == "health" { return a.health() } a.mu.Lock() defer a.mu.Unlock() out := response{Action: req.Action, Scope: req.Scope, Job: req.Job} // Degraded blocks new captures, but recover and status must stay reachable: // otherwise there is no way back to a verified state. if state := a.snapshotState(); state.Degraded != "" && req.Action == "capture" { out.Error = "封存执行边界处于降级状态(上次恢复失败:" + state.Degraded + "),先执行 recover" return out } var args []string switch req.Action { case "status": args = []string{"status"} case "recover": args = []string{"recover"} case "capture": if req.Scope != "internal" && req.Scope != "external" { out.Error = "scope must be internal or external" return out } if req.Job != "" && !jobPattern.MatchString(req.Job) { out.Error = "invalid job id" return out } if req.Expect == nil { out.Error = "capture 请求没有携带期望执行器身份,拒绝执行" return out } // 此时已持有操作互斥(a.mu):从"比对身份"到"真正跑子进程"之间没有别的 // 请求能插进来,也不会被另一个 agent 抢走——这是零副作用的前提。 if err := a.config.matchExpectation(*req.Expect); err != nil { out.Error = err.Error() return out } args = []string{"capture", "-scope", req.Scope} if req.Job != "" { args = append(args, "-job", req.Job) } if len(req.Released) > 0 { if len(req.Released) > maxReleasedDeadLetters { out.Error = "too many released dead-letter fingerprints" return out } for _, fingerprint := range req.Released { if !fingerprintPattern.MatchString(fingerprint) { out.Error = "released dead letters must be sha256:<64 hex> fingerprints" return out } } args = append(args, "-released-dead-letters", strings.Join(req.Released, ",")) } default: out.Error = "unsupported action" return out } args = append(args, a.config.declaration()...) bounded, cancel := context.WithTimeout(ctx, a.config.MaxDuration) defer cancel() if req.Action == "capture" || req.Action == "recover" { a.setBusy(true) defer a.setBusy(false) } payload, err := a.config.run(bounded, a.config.Binary, args...) if err != nil { detail := "operation failed" if message := strings.TrimSpace(string(payload)); message != "" && len(message) <= 2000 { detail = message } // A killed or aborted child may have left the source writer stopped. The // recovery responsibility does not end with the failed request: run the // idempotent recovery now, and refuse further captures if it does not // verify. if req.Action == "capture" || req.Action == "recover" { if recoveryErr := a.recoverWriters(ctx); recoveryErr != nil { a.setDegraded(recoveryErr.Error()) out.Error = detail + ";写入者恢复失败:" + recoveryErr.Error() + "(已阻断后续封存,需人工处理)" return out } detail += ";已执行 recover 并确认写入者恢复" } else { // status 失败不需要恢复;它不改变停机状态。 } out.Error = detail return out } if req.Action == "recover" { a.clearDegraded() } out.OK = true if json.Valid(payload) { out.Payload = json.RawMessage(payload) } else { out.Payload = json.RawMessage(`{}`) } // 封存结果必须带上"这次是谁执行的":调用方据此复核,防止心跳与执行之间 // socket 被换成另一个 agent(真正的封错库事故就是这样发生的)。 if req.Action == "capture" && req.Expect != nil { out.Payload = a.attachIdentity(out.Payload, *req.Expect) } return out } // attachIdentity 把执行器身份并进封存结果(保留 CLI 原有的字段,如 snapshot_id)。 // expectation_fingerprint 是"本执行器实际核对过的那份期望"的指纹:调用方据此确认 // 执行时看到的期望与它发出的一致,而不是被替换后重新生成的。 func (a *Agent) attachIdentity(payload json.RawMessage, expectation Expectation) json.RawMessage { var document map[string]any if err := json.Unmarshal(payload, &document); err != nil { return payload } identity := a.config.identity() identity["expectation_fingerprint"] = expectation.Fingerprint document["agent_identity"] = identity encoded, err := json.Marshal(document) if err != nil { return payload } return json.RawMessage(encoded) } // recoverWriters runs the idempotent recovery command with its own budget, even // when the triggering context is already cancelled (timeout / shutdown). func (a *Agent) recoverWriters(ctx context.Context) error { bounded, cancel := context.WithTimeout(context.WithoutCancel(ctx), a.config.RecoverTimeout) defer cancel() if _, err := a.config.run(bounded, a.config.Binary, append([]string{"recover"}, a.config.declaration()...)...); err != nil { return err } a.clearDegraded() return nil } // declaration pins every path and the writer group so a request cannot redirect // the operation to another stack, archive or journal. func (c Config) declaration() []string { args := []string{ "-source-app", c.SourceApp, "-source-pg", c.SourcePG, "-source-es", c.SourceES, } if c.SourceS3Endpoint != "" { args = append(args, "-source-s3-endpoint", c.SourceS3Endpoint) } return append(args, "-governance-pg", c.GovernancePG, "-project", c.Project, "-writers", c.Writers, "-non-writers", c.NonWriters, "-archive", c.ArchiveRoot, "-journal", c.JournalRoot, ) } func (c Config) run(ctx context.Context, name string, args ...string) ([]byte, error) { if c.Run != nil { return c.Run(ctx, name, args...) } command := exec.CommandContext(ctx, name, args...) command.Env = append(os.Environ(), "LIBRARYCTL_AGENT=1") // Default CommandContext behaviour is SIGKILL, which would skip the child's // window restore and leave the source writer stopped. Ask for a graceful stop // first and only escalate after the grace period. command.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} command.Cancel = func() error { if command.Process == nil { return nil } return command.Process.Signal(syscall.SIGTERM) } command.WaitDelay = c.StopGrace return command.CombinedOutput() } // Describe returns a redacted view for logs and health checks. func (a *Agent) Describe() map[string]any { view := a.config.identity() view["scope_choice"] = "request may select internal or external only" return view } // health 是"这个执行器指向谁、现在能不能干活"的唯一回答。它只暴露容器名、 // 路径与声明,不含任何凭据;调用方据此核对源身份,而不是靠配置猜。 func (a *Agent) health() response { state := a.snapshotState() payload := a.config.identity() payload["busy"] = state.Busy payload["degraded"] = state.Degraded encoded, err := json.Marshal(payload) if err != nil { return response{Action: "health", Error: "cannot encode health"} } return response{OK: true, Action: "health", Payload: json.RawMessage(encoded)} } // identity 被日志与健康应答共用一份,避免两处描述漂移。 func (c Config) identity() map[string]any { identity := map[string]any{ "project": c.Project, "archive_root": c.ArchiveRoot, "journal_root": c.JournalRoot, "source_app": c.SourceApp, "source_pg": c.SourcePG, "source_es": c.SourceES, "writers": c.Writers, } if c.SourceS3Endpoint != "" { identity["source_s3_endpoint"] = c.SourceS3Endpoint } return identity }