姚章浩
6ed06357f9
build / build (push) Successful in 1m27s
原来 worker 先 health 再 capture，两次拨号之间 socket 可能被换成另一个实例的 执行器：错的 agent 已经把写入者停了、快照也出了，事后 postcheck 只能拒绝登记， 副作用无法撤销。 现在 capture 请求自带期望身份（治理侧配置生成：project 必填，source_app / archive_root 配了就绑定）与覆盖三项的稳定指纹；接收方在操作互斥内、执行任何源 操作之前比对，缺字段、指纹与字段不一致、project/source_app 不符都直接拒绝， **错的执行器一次 capture 都不跑**。封存结果回显该指纹，worker 事后复核按它对照， postcheck 只作为证据防线。恢复路径不要求期望（它只把写入者恢复回来）。 负例是确定性的：两个真实 agent 各占一个 socket，把调用方指向错的那个，断言它的 capture 调用为 0 且仍能回答 health；同一份期望打到对的 socket 则只跑一次。
447 lines
15 KiB
Go
447 lines
15 KiB
Go
// Package libraryagent is the least-privilege execution boundary for
|
|
// whole-library capture on the target host.
|
|
//
|
|
// Why it exists: capturing a library has to stop the source writer for a few
|
|
// seconds and read source credentials. Handing the governance containers the
|
|
// Docker socket (a full write API) or the source credentials would be a much
|
|
// larger privilege than that operation needs. Instead this agent runs as the
|
|
// ordinary operator account on the host, listens on one unix socket which only
|
|
// that account can connect to, and executes one fixed command with a fixed
|
|
// writer declaration. A caller can choose the scope and (optionally) a job id —
|
|
// nothing else.
|
|
package libraryagent
|
|
|
|
import (
|
|
"bufio"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"log"
|
|
"net"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"regexp"
|
|
"strings"
|
|
"sync"
|
|
"syscall"
|
|
"time"
|
|
)
|
|
|
|
// Config is fixed at start-up by the operator (systemd unit / CLI flags). It is
|
|
// never taken from a request.
|
|
type Config struct {
|
|
Socket string
|
|
Binary string
|
|
ArchiveRoot string
|
|
JournalRoot string
|
|
// Writer declaration passed to the binary, exactly as reviewed in
|
|
// scripts/library-capture.sh. Clients cannot influence it.
|
|
SourceApp string
|
|
SourcePG string
|
|
SourceES string
|
|
// SourceS3Endpoint 让宿主机用可达地址读源对象存储：容器 env 里的端点常常
|
|
// 只是服务名（minio:9000），宿主机解析不到，资源导出会失败。
|
|
SourceS3Endpoint string
|
|
GovernancePG string
|
|
Project string
|
|
Writers string
|
|
NonWriters string
|
|
MaxDuration time.Duration
|
|
// StopGrace bounds the graceful stop of one operation: SIGTERM first (the
|
|
// child aborts the maintenance window and restores writers), then SIGKILL
|
|
// only after this grace period. It must exceed the child's own restore budget.
|
|
StopGrace time.Duration
|
|
// RecoverTimeout bounds the recovery run performed after a failed or killed
|
|
// operation, and at agent start-up.
|
|
RecoverTimeout time.Duration
|
|
// Run executes the binary; replaced in tests.
|
|
Run func(ctx context.Context, name string, args ...string) ([]byte, error)
|
|
}
|
|
|
|
type request struct {
|
|
Action string `json:"action"`
|
|
Scope string `json:"scope,omitempty"`
|
|
Job string `json:"job,omitempty"`
|
|
// Released 只允许精确死信指纹（sha256:<64 hex>）。它不改变固定命令形态：
|
|
// 还能传的只有"这些确切的行已由治理平台审计解除"，CLI 会逐字节复核。
|
|
Released []string `json:"released,omitempty"`
|
|
// Expect 是调用方对执行器身份的期望。capture 必须在操作互斥内、任何源操作
|
|
// 之前比对它；缺失即拒绝（fail closed），不靠"事后核对"兜底。
|
|
Expect *Expectation `json:"expect,omitempty"`
|
|
}
|
|
|
|
type response struct {
|
|
OK bool `json:"ok"`
|
|
Action string `json:"action"`
|
|
Scope string `json:"scope,omitempty"`
|
|
Job string `json:"job,omitempty"`
|
|
Error string `json:"error,omitempty"`
|
|
Payload json.RawMessage `json:"payload,omitempty"`
|
|
}
|
|
|
|
var jobPattern = regexp.MustCompile(`^[A-Za-z0-9_-]{1,64}$`)
|
|
|
|
// 死信解除只能按精确指纹传入，不接受通配、实例或任务类型级解除。
|
|
var fingerprintPattern = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`)
|
|
|
|
const maxReleasedDeadLetters = 64
|
|
|
|
// Agent serves capture requests. Only one operation runs at a time: a library
|
|
// capture owns the writer group, so overlapping requests would race the window.
|
|
type Agent struct {
|
|
config Config
|
|
mu sync.Mutex
|
|
// stateMu 只保护下面两个"给人看"的字段：健康应答不能被一次 15 分钟的封存
|
|
// 卡住（否则页面会把"正在封存"读成"执行器不可达"）。
|
|
stateMu sync.RWMutex
|
|
// degraded is non-empty when a failed operation could not be followed by a
|
|
// verified recovery. While set, capture requests are refused: leaving the
|
|
// source writer stopped is worse than refusing work.
|
|
degraded string
|
|
busy bool
|
|
wg sync.WaitGroup
|
|
}
|
|
|
|
// captureState 是健康应答要暴露的执行器状态：是否在跑、是否降级。
|
|
type captureState struct {
|
|
Busy bool `json:"busy"`
|
|
Degraded string `json:"degraded"`
|
|
}
|
|
|
|
func (a *Agent) snapshotState() captureState {
|
|
a.stateMu.RLock()
|
|
defer a.stateMu.RUnlock()
|
|
return captureState{Busy: a.busy, Degraded: a.degraded}
|
|
}
|
|
|
|
func (a *Agent) setBusy(busy bool) {
|
|
a.stateMu.Lock()
|
|
defer a.stateMu.Unlock()
|
|
a.busy = busy
|
|
}
|
|
|
|
func (a *Agent) setDegraded(reason string) {
|
|
a.stateMu.Lock()
|
|
defer a.stateMu.Unlock()
|
|
a.degraded = reason
|
|
}
|
|
|
|
func (a *Agent) clearDegraded() {
|
|
a.setDegraded("")
|
|
}
|
|
|
|
func New(config Config) (*Agent, error) {
|
|
if !filepath.IsAbs(config.Socket) || !filepath.IsAbs(config.ArchiveRoot) || !filepath.IsAbs(config.JournalRoot) {
|
|
return nil, errors.New("library agent paths must be absolute")
|
|
}
|
|
if config.Binary == "" || config.Project == "" || config.Writers == "" || !strings.Contains(config.NonWriters, "=") {
|
|
return nil, errors.New("library agent needs a binary, project, writers and a non-writer declaration")
|
|
}
|
|
if config.MaxDuration <= 0 || config.MaxDuration > 30*time.Minute {
|
|
return nil, errors.New("library agent duration bound is invalid")
|
|
}
|
|
if config.StopGrace == 0 {
|
|
config.StopGrace = 3 * time.Minute
|
|
}
|
|
if config.RecoverTimeout == 0 {
|
|
config.RecoverTimeout = 3 * time.Minute
|
|
}
|
|
if config.StopGrace <= 0 || config.StopGrace > 10*time.Minute || config.RecoverTimeout <= 0 || config.RecoverTimeout > 10*time.Minute {
|
|
return nil, errors.New("library agent recovery/stop budget is invalid")
|
|
}
|
|
return &Agent{config: config}, nil
|
|
}
|
|
|
|
// Serve listens until ctx is cancelled. The socket is created 0600 inside a
|
|
// 0700 directory and removed on shutdown.
|
|
func (a *Agent) Serve(ctx context.Context) error {
|
|
if err := os.MkdirAll(filepath.Dir(a.config.Socket), 0700); err != nil {
|
|
return err
|
|
}
|
|
if err := os.Chmod(filepath.Dir(a.config.Socket), 0700); err != nil {
|
|
return err
|
|
}
|
|
_ = os.Remove(a.config.Socket)
|
|
listener, err := net.Listen("unix", a.config.Socket)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer func() {
|
|
listener.Close()
|
|
_ = os.Remove(a.config.Socket)
|
|
}()
|
|
if err := os.Chmod(a.config.Socket, 0600); err != nil {
|
|
return err
|
|
}
|
|
// Startup responsibility: an agent restart may have inherited a stopped
|
|
// writer from a previous killed run. Recover before accepting work; if that
|
|
// fails, stay degraded and refuse captures instead of pretending to be ready.
|
|
if err := a.recoverWriters(ctx); err != nil {
|
|
a.setDegraded(err.Error())
|
|
log.Printf("library agent: pending freeze recovery failed: %v", err)
|
|
}
|
|
go func() {
|
|
<-ctx.Done()
|
|
_ = listener.Close()
|
|
}()
|
|
for {
|
|
conn, err := listener.Accept()
|
|
if err != nil {
|
|
if ctx.Err() != nil {
|
|
// Drain in-flight operations: each of them still owes a bounded
|
|
// recovery before the agent may exit.
|
|
done := make(chan struct{})
|
|
go func() { a.wg.Wait(); close(done) }()
|
|
select {
|
|
case <-done:
|
|
case <-time.After(a.config.StopGrace + a.config.RecoverTimeout):
|
|
log.Printf("library agent: shutdown deadline exceeded with operations in flight")
|
|
}
|
|
return nil
|
|
}
|
|
return err
|
|
}
|
|
a.wg.Add(1)
|
|
go func(conn net.Conn) {
|
|
defer a.wg.Done()
|
|
a.handle(ctx, conn)
|
|
}(conn)
|
|
}
|
|
}
|
|
|
|
func (a *Agent) handle(ctx context.Context, conn net.Conn) {
|
|
defer conn.Close()
|
|
if err := peerAllowed(conn); err != nil {
|
|
_ = json.NewEncoder(conn).Encode(response{OK: false, Error: err.Error()})
|
|
return
|
|
}
|
|
_ = conn.SetDeadline(time.Now().Add(a.config.MaxDuration + 30*time.Second))
|
|
decoder := json.NewDecoder(bufio.NewReader(conn))
|
|
var req request
|
|
if err := decoder.Decode(&req); err != nil {
|
|
_ = json.NewEncoder(conn).Encode(response{OK: false, Error: "invalid request"})
|
|
return
|
|
}
|
|
result := a.execute(ctx, req)
|
|
_ = json.NewEncoder(conn).Encode(result)
|
|
}
|
|
|
|
// execute serialises operations and maps a request onto the one fixed command.
|
|
func (a *Agent) execute(ctx context.Context, req request) response {
|
|
// health 不进操作锁、不跑子进程：调用方要的是"这个执行器指向谁、能不能干活"，
|
|
// 而一次正在进行的封存不该让这个回答变成"不可达"。
|
|
if req.Action == "health" {
|
|
return a.health()
|
|
}
|
|
a.mu.Lock()
|
|
defer a.mu.Unlock()
|
|
out := response{Action: req.Action, Scope: req.Scope, Job: req.Job}
|
|
// Degraded blocks new captures, but recover and status must stay reachable:
|
|
// otherwise there is no way back to a verified state.
|
|
if state := a.snapshotState(); state.Degraded != "" && req.Action == "capture" {
|
|
out.Error = "封存执行边界处于降级状态（上次恢复失败：" + state.Degraded + "），先执行 recover"
|
|
return out
|
|
}
|
|
var args []string
|
|
switch req.Action {
|
|
case "status":
|
|
args = []string{"status"}
|
|
case "recover":
|
|
args = []string{"recover"}
|
|
case "capture":
|
|
if req.Scope != "internal" && req.Scope != "external" {
|
|
out.Error = "scope must be internal or external"
|
|
return out
|
|
}
|
|
if req.Job != "" && !jobPattern.MatchString(req.Job) {
|
|
out.Error = "invalid job id"
|
|
return out
|
|
}
|
|
if req.Expect == nil {
|
|
out.Error = "capture 请求没有携带期望执行器身份，拒绝执行"
|
|
return out
|
|
}
|
|
// 此时已持有操作互斥（a.mu）：从"比对身份"到"真正跑子进程"之间没有别的
|
|
// 请求能插进来，也不会被另一个 agent 抢走——这是零副作用的前提。
|
|
if err := a.config.matchExpectation(*req.Expect); err != nil {
|
|
out.Error = err.Error()
|
|
return out
|
|
}
|
|
args = []string{"capture", "-scope", req.Scope}
|
|
if req.Job != "" {
|
|
args = append(args, "-job", req.Job)
|
|
}
|
|
if len(req.Released) > 0 {
|
|
if len(req.Released) > maxReleasedDeadLetters {
|
|
out.Error = "too many released dead-letter fingerprints"
|
|
return out
|
|
}
|
|
for _, fingerprint := range req.Released {
|
|
if !fingerprintPattern.MatchString(fingerprint) {
|
|
out.Error = "released dead letters must be sha256:<64 hex> fingerprints"
|
|
return out
|
|
}
|
|
}
|
|
args = append(args, "-released-dead-letters", strings.Join(req.Released, ","))
|
|
}
|
|
default:
|
|
out.Error = "unsupported action"
|
|
return out
|
|
}
|
|
args = append(args, a.config.declaration()...)
|
|
bounded, cancel := context.WithTimeout(ctx, a.config.MaxDuration)
|
|
defer cancel()
|
|
if req.Action == "capture" || req.Action == "recover" {
|
|
a.setBusy(true)
|
|
defer a.setBusy(false)
|
|
}
|
|
payload, err := a.config.run(bounded, a.config.Binary, args...)
|
|
if err != nil {
|
|
detail := "operation failed"
|
|
if message := strings.TrimSpace(string(payload)); message != "" && len(message) <= 2000 {
|
|
detail = message
|
|
}
|
|
// A killed or aborted child may have left the source writer stopped. The
|
|
// recovery responsibility does not end with the failed request: run the
|
|
// idempotent recovery now, and refuse further captures if it does not
|
|
// verify.
|
|
if req.Action == "capture" || req.Action == "recover" {
|
|
if recoveryErr := a.recoverWriters(ctx); recoveryErr != nil {
|
|
a.setDegraded(recoveryErr.Error())
|
|
out.Error = detail + "；写入者恢复失败：" + recoveryErr.Error() + "（已阻断后续封存，需人工处理）"
|
|
return out
|
|
}
|
|
detail += "；已执行 recover 并确认写入者恢复"
|
|
} else {
|
|
// status 失败不需要恢复；它不改变停机状态。
|
|
}
|
|
out.Error = detail
|
|
return out
|
|
}
|
|
if req.Action == "recover" {
|
|
a.clearDegraded()
|
|
}
|
|
out.OK = true
|
|
if json.Valid(payload) {
|
|
out.Payload = json.RawMessage(payload)
|
|
} else {
|
|
out.Payload = json.RawMessage(`{}`)
|
|
}
|
|
// 封存结果必须带上"这次是谁执行的"：调用方据此复核，防止心跳与执行之间
|
|
// socket 被换成另一个 agent（真正的封错库事故就是这样发生的）。
|
|
if req.Action == "capture" && req.Expect != nil {
|
|
out.Payload = a.attachIdentity(out.Payload, *req.Expect)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// attachIdentity 把执行器身份并进封存结果（保留 CLI 原有的字段，如 snapshot_id）。
|
|
// expectation_fingerprint 是"本执行器实际核对过的那份期望"的指纹：调用方据此确认
|
|
// 执行时看到的期望与它发出的一致，而不是被替换后重新生成的。
|
|
func (a *Agent) attachIdentity(payload json.RawMessage, expectation Expectation) json.RawMessage {
|
|
var document map[string]any
|
|
if err := json.Unmarshal(payload, &document); err != nil {
|
|
return payload
|
|
}
|
|
identity := a.config.identity()
|
|
identity["expectation_fingerprint"] = expectation.Fingerprint
|
|
document["agent_identity"] = identity
|
|
encoded, err := json.Marshal(document)
|
|
if err != nil {
|
|
return payload
|
|
}
|
|
return json.RawMessage(encoded)
|
|
}
|
|
|
|
// recoverWriters runs the idempotent recovery command with its own budget, even
|
|
// when the triggering context is already cancelled (timeout / shutdown).
|
|
func (a *Agent) recoverWriters(ctx context.Context) error {
|
|
bounded, cancel := context.WithTimeout(context.WithoutCancel(ctx), a.config.RecoverTimeout)
|
|
defer cancel()
|
|
if _, err := a.config.run(bounded, a.config.Binary, append([]string{"recover"}, a.config.declaration()...)...); err != nil {
|
|
return err
|
|
}
|
|
a.clearDegraded()
|
|
return nil
|
|
}
|
|
|
|
// declaration pins every path and the writer group so a request cannot redirect
|
|
// the operation to another stack, archive or journal.
|
|
func (c Config) declaration() []string {
|
|
args := []string{
|
|
"-source-app", c.SourceApp,
|
|
"-source-pg", c.SourcePG,
|
|
"-source-es", c.SourceES,
|
|
}
|
|
if c.SourceS3Endpoint != "" {
|
|
args = append(args, "-source-s3-endpoint", c.SourceS3Endpoint)
|
|
}
|
|
return append(args,
|
|
"-governance-pg", c.GovernancePG,
|
|
"-project", c.Project,
|
|
"-writers", c.Writers,
|
|
"-non-writers", c.NonWriters,
|
|
"-archive", c.ArchiveRoot,
|
|
"-journal", c.JournalRoot,
|
|
)
|
|
}
|
|
|
|
func (c Config) run(ctx context.Context, name string, args ...string) ([]byte, error) {
|
|
if c.Run != nil {
|
|
return c.Run(ctx, name, args...)
|
|
}
|
|
command := exec.CommandContext(ctx, name, args...)
|
|
command.Env = append(os.Environ(), "LIBRARYCTL_AGENT=1")
|
|
// Default CommandContext behaviour is SIGKILL, which would skip the child's
|
|
// window restore and leave the source writer stopped. Ask for a graceful stop
|
|
// first and only escalate after the grace period.
|
|
command.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
|
|
command.Cancel = func() error {
|
|
if command.Process == nil {
|
|
return nil
|
|
}
|
|
return command.Process.Signal(syscall.SIGTERM)
|
|
}
|
|
command.WaitDelay = c.StopGrace
|
|
return command.CombinedOutput()
|
|
}
|
|
|
|
// Describe returns a redacted view for logs and health checks.
|
|
func (a *Agent) Describe() map[string]any {
|
|
view := a.config.identity()
|
|
view["scope_choice"] = "request may select internal or external only"
|
|
return view
|
|
}
|
|
|
|
// health 是"这个执行器指向谁、现在能不能干活"的唯一回答。它只暴露容器名、
|
|
// 路径与声明，不含任何凭据；调用方据此核对源身份，而不是靠配置猜。
|
|
func (a *Agent) health() response {
|
|
state := a.snapshotState()
|
|
payload := a.config.identity()
|
|
payload["busy"] = state.Busy
|
|
payload["degraded"] = state.Degraded
|
|
encoded, err := json.Marshal(payload)
|
|
if err != nil {
|
|
return response{Action: "health", Error: "cannot encode health"}
|
|
}
|
|
return response{OK: true, Action: "health", Payload: json.RawMessage(encoded)}
|
|
}
|
|
|
|
// identity 被日志与健康应答共用一份，避免两处描述漂移。
|
|
func (c Config) identity() map[string]any {
|
|
identity := map[string]any{
|
|
"project": c.Project,
|
|
"archive_root": c.ArchiveRoot,
|
|
"journal_root": c.JournalRoot,
|
|
"source_app": c.SourceApp,
|
|
"source_pg": c.SourcePG,
|
|
"source_es": c.SourceES,
|
|
"writers": c.Writers,
|
|
}
|
|
if c.SourceS3Endpoint != "" {
|
|
identity["source_s3_endpoint"] = c.SourceS3Endpoint
|
|
}
|
|
return identity
|
|
}
|