姚章浩
faaf003707
build / build (push) Failing after 17s
上一提交只给 libraryctl 加了 -source-s3-endpoint，agent 却不知道这个 flag， systemd 起不来（exit 2）。端点属于"这台主机怎么访问这个源的存储"，必须由 agent 自己钉住并随固定声明传给 libraryctl；请求仍然不能改。没配置时不凭空加 flag。 试过：make verify 15 项全绿；新增 agent 声明用例（有端点必传、无端点不多传）。
383 lines
12 KiB
Go
383 lines
12 KiB
Go
package libraryagent
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"net"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func testAgent(t *testing.T) (*Agent, string, func() []string) {
|
|
t.Helper()
|
|
root, err := os.MkdirTemp(os.TempDir(), "kgag-")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { os.RemoveAll(root) })
|
|
if err = os.Chmod(root, 0700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var mu sync.Mutex
|
|
var calls []string
|
|
config := Config{
|
|
Socket: filepath.Join(root, "agent.sock"),
|
|
Binary: "/bin/echo",
|
|
ArchiveRoot: filepath.Join(root, "library"),
|
|
JournalRoot: filepath.Join(root, "freeze"),
|
|
SourceApp: "WKNFull-app",
|
|
SourcePG: "WKNFull-postgres",
|
|
SourceES: "WKNFull-elasticsearch",
|
|
GovernancePG: "knowledge-governance-p1-postgres-1",
|
|
Project: "weknora071full",
|
|
Writers: "app",
|
|
NonWriters: "frontend=UI only; no knowledge writes",
|
|
MaxDuration: time.Minute,
|
|
Run: func(_ context.Context, name string, args ...string) ([]byte, error) {
|
|
mu.Lock()
|
|
calls = append(calls, name+" "+strings.Join(args, " "))
|
|
mu.Unlock()
|
|
return []byte(`{"snapshot_id":"abc","state":"succeeded"}`), nil
|
|
},
|
|
}
|
|
agent, err := New(config)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return agent, config.Socket, func() []string {
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
return append([]string(nil), calls...)
|
|
}
|
|
}
|
|
|
|
func roundTrip(t *testing.T, socket string, payload string) response {
|
|
t.Helper()
|
|
conn, err := net.Dial("unix", socket)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer conn.Close()
|
|
if _, err = conn.Write([]byte(payload + "\n")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var out response
|
|
if err = json.NewDecoder(conn).Decode(&out); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// The socket is the whole security boundary: 0600 in a 0700 directory, and the
|
|
// declaration (archive, journal, project, writers) comes from the agent's own
|
|
// configuration, never from the request.
|
|
func TestAgentSocketIsPrivateAndPinsTheDeclaration(t *testing.T) {
|
|
agent, socket, calls := testAgent(t)
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
go func() { _ = agent.Serve(ctx) }()
|
|
deadline := time.Now().Add(2 * time.Second)
|
|
for {
|
|
if _, err := os.Stat(socket); err == nil {
|
|
break
|
|
}
|
|
if time.Now().After(deadline) {
|
|
t.Fatal("agent socket never appeared")
|
|
}
|
|
time.Sleep(10 * time.Millisecond)
|
|
}
|
|
info, err := os.Stat(socket)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if info.Mode().Perm() != 0600 {
|
|
t.Fatalf("socket mode %v", info.Mode().Perm())
|
|
}
|
|
dir, err := os.Stat(filepath.Dir(socket))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if dir.Mode().Perm() != 0700 {
|
|
t.Fatalf("socket directory mode %v", dir.Mode().Perm())
|
|
}
|
|
out := roundTrip(t, socket, `{"action":"capture","scope":"internal","job":"lib-internal-test"}`)
|
|
if !out.OK || out.Job != "lib-internal-test" {
|
|
t.Fatalf("capture rejected: %+v", out)
|
|
}
|
|
recorded := captureCalls(calls())
|
|
if len(recorded) != 1 {
|
|
t.Fatalf("expected one capture execution, got %v", recorded)
|
|
}
|
|
for _, want := range []string{"capture -scope internal -job lib-internal-test",
|
|
"-archive " + filepath.Join(filepath.Dir(socket), "library"),
|
|
"-journal " + filepath.Join(filepath.Dir(socket), "freeze"),
|
|
"-project weknora071full", "-writers app"} {
|
|
if !strings.Contains(recorded[0], want) {
|
|
t.Fatalf("execution %q missing %q", recorded[0], want)
|
|
}
|
|
}
|
|
if strings.Contains(recorded[0], "..") {
|
|
t.Fatal("unexpected path traversal in declaration")
|
|
}
|
|
}
|
|
|
|
func TestAgentRejectsUnsupportedRequestsWithoutExecuting(t *testing.T) {
|
|
agent, socket, calls := testAgent(t)
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
go func() { _ = agent.Serve(ctx) }()
|
|
deadline := time.Now().Add(2 * time.Second)
|
|
for {
|
|
if _, err := os.Stat(socket); err == nil {
|
|
break
|
|
}
|
|
if time.Now().After(deadline) {
|
|
t.Fatal("agent socket never appeared")
|
|
}
|
|
time.Sleep(10 * time.Millisecond)
|
|
}
|
|
for _, payload := range []string{
|
|
`{"action":"shell","command":"docker ps"}`,
|
|
`{"action":"capture","scope":"production"}`,
|
|
`{"action":"capture","scope":"internal","job":"bad job; rm -rf /"}`,
|
|
`not json`,
|
|
} {
|
|
out := roundTrip(t, socket, payload)
|
|
if out.OK || out.Error == "" {
|
|
t.Fatalf("accepted %q: %+v", payload, out)
|
|
}
|
|
}
|
|
if got := captureCalls(calls()); len(got) != 0 {
|
|
t.Fatalf("rejected requests still executed: %v", got)
|
|
}
|
|
}
|
|
|
|
func captureCalls(calls []string) []string {
|
|
kept := make([]string, 0, len(calls))
|
|
for _, call := range calls {
|
|
if strings.Contains(call, " capture ") {
|
|
kept = append(kept, call)
|
|
}
|
|
}
|
|
return kept
|
|
}
|
|
|
|
func recoverCalls(calls []string) []string {
|
|
kept := make([]string, 0, len(calls))
|
|
for _, call := range calls {
|
|
if strings.Contains(call, " recover ") {
|
|
kept = append(kept, call)
|
|
}
|
|
}
|
|
return kept
|
|
}
|
|
|
|
// failingAgent builds an agent whose capture always fails and whose recovery
|
|
// outcome can be switched, to prove the recovery responsibility after a
|
|
// killed/aborted child.
|
|
func failingAgent(t *testing.T) (*Agent, string, func() []string, func(error)) {
|
|
t.Helper()
|
|
root, err := os.MkdirTemp(os.TempDir(), "kgrf-")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { os.RemoveAll(root) })
|
|
if err = os.Chmod(root, 0700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var mu sync.Mutex
|
|
var calls []string
|
|
var recoverErr error
|
|
agent, err := New(Config{
|
|
Socket: filepath.Join(root, "agent.sock"), Binary: "/bin/echo",
|
|
ArchiveRoot: filepath.Join(root, "library"), JournalRoot: filepath.Join(root, "freeze"),
|
|
SourceApp: "app-container", SourcePG: "pg", SourceES: "es", GovernancePG: "gov",
|
|
Project: "weknora071full", Writers: "app", NonWriters: "frontend=UI only",
|
|
MaxDuration: time.Minute, StopGrace: time.Minute, RecoverTimeout: time.Minute,
|
|
Run: func(_ context.Context, name string, args ...string) ([]byte, error) {
|
|
mu.Lock()
|
|
calls = append(calls, name+" "+strings.Join(args, " "))
|
|
current := recoverErr
|
|
mu.Unlock()
|
|
if len(args) > 0 && args[0] == "recover" {
|
|
if current != nil {
|
|
return []byte("writer restart failed"), current
|
|
}
|
|
return []byte(`{"recovered":true}`), nil
|
|
}
|
|
return []byte("LIBRARYCTL_FAILED: stop outcome uncertain"), errors.New("exit status 1")
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return agent, filepath.Join(root, "agent.sock"), func() []string {
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
return append([]string(nil), calls...)
|
|
}, func(err error) {
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
recoverErr = err
|
|
}
|
|
}
|
|
|
|
// A failed capture must be followed by a recovery attempt, not just an error.
|
|
func TestAgentRecoversWritersAfterFailedCapture(t *testing.T) {
|
|
agent, socket, calls, _ := failingAgent(t)
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
go func() { _ = agent.Serve(ctx) }()
|
|
waitSocket(t, socket)
|
|
out := roundTrip(t, socket, `{"action":"capture","scope":"internal","job":"lib-fail-1"}`)
|
|
if out.OK {
|
|
t.Fatalf("failed capture reported success: %+v", out)
|
|
}
|
|
if !strings.Contains(out.Error, "已执行 recover 并确认写入者恢复") {
|
|
t.Fatalf("error does not state the recovery outcome: %q", out.Error)
|
|
}
|
|
if len(recoverCalls(calls())) == 0 {
|
|
t.Fatal("no recovery ran after the failed capture")
|
|
}
|
|
}
|
|
|
|
// If recovery itself fails, the boundary must refuse further captures instead of
|
|
// leaving the source stopped and merely marking a job failed.
|
|
func TestAgentBlocksCapturesWhenRecoveryFails(t *testing.T) {
|
|
agent, socket, calls, setRecoverErr := failingAgent(t)
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
go func() { _ = agent.Serve(ctx) }()
|
|
waitSocket(t, socket)
|
|
setRecoverErr(errors.New("docker restart failed"))
|
|
out := roundTrip(t, socket, `{"action":"capture","scope":"internal","job":"lib-fail-2"}`)
|
|
if out.OK || !strings.Contains(out.Error, "写入者恢复失败") || !strings.Contains(out.Error, "已阻断后续封存") {
|
|
t.Fatalf("recovery failure not surfaced: %+v", out)
|
|
}
|
|
before := len(captureCalls(calls()))
|
|
blocked := roundTrip(t, socket, `{"action":"capture","scope":"external","job":"lib-fail-3"}`)
|
|
if blocked.OK || !strings.Contains(blocked.Error, "降级") {
|
|
t.Fatalf("degraded agent accepted a capture: %+v", blocked)
|
|
}
|
|
if got := len(captureCalls(calls())); got != before {
|
|
t.Fatalf("blocked capture still executed (%d -> %d)", before, got)
|
|
}
|
|
// A verified recovery clears the degraded state and lets work resume.
|
|
setRecoverErr(nil)
|
|
if recovered := roundTrip(t, socket, `{"action":"recover"}`); !recovered.OK {
|
|
t.Fatalf("explicit recovery failed: %+v", recovered)
|
|
}
|
|
if resumed := roundTrip(t, socket, `{"action":"capture","scope":"internal","job":"lib-fail-4"}`); strings.Contains(resumed.Error, "降级") {
|
|
t.Fatalf("recovery did not clear the degraded state: %+v", resumed)
|
|
}
|
|
}
|
|
|
|
// The child is stopped with SIGTERM and the grace period, not an immediate kill.
|
|
func TestRunStopsChildWithSigterm(t *testing.T) {
|
|
dir := t.TempDir()
|
|
script := filepath.Join(dir, "child.sh")
|
|
// A real child that only exits 0 when it receives SIGTERM.
|
|
child := "#!/bin/sh\ntrap 'exit 0' TERM\nwhile :; do sleep 0.05; done\n"
|
|
if err := os.WriteFile(script, []byte(child), 0o700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
agent, err := New(Config{Socket: "/tmp/irrelevant.sock", Binary: script,
|
|
ArchiveRoot: filepath.Join(dir, "library"), JournalRoot: filepath.Join(dir, "freeze"),
|
|
SourceApp: "app-container", SourcePG: "pg", SourceES: "es", GovernancePG: "gov",
|
|
Project: "weknora071full", Writers: "app", NonWriters: "frontend=UI only",
|
|
MaxDuration: 2 * time.Second, StopGrace: 5 * time.Second, RecoverTimeout: 5 * time.Second})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), 300*time.Millisecond)
|
|
defer cancel()
|
|
started := time.Now()
|
|
_, err = agent.config.run(ctx, script)
|
|
if err == nil {
|
|
t.Fatal("cancelled run reported success")
|
|
}
|
|
if elapsed := time.Since(started); elapsed > 4*time.Second {
|
|
t.Fatalf("child was not stopped within the graceful window: %s", elapsed)
|
|
}
|
|
if ctx.Err() == nil {
|
|
t.Fatal("test did not exercise cancellation")
|
|
}
|
|
}
|
|
|
|
func waitSocket(t *testing.T, socket string) {
|
|
t.Helper()
|
|
deadline := time.Now().Add(2 * time.Second)
|
|
for {
|
|
if _, err := os.Stat(socket); err == nil {
|
|
return
|
|
}
|
|
if time.Now().After(deadline) {
|
|
t.Fatal("agent socket never appeared")
|
|
}
|
|
time.Sleep(10 * time.Millisecond)
|
|
}
|
|
}
|
|
|
|
func TestNewRejectsIncompleteConfiguration(t *testing.T) {
|
|
if _, err := New(Config{}); err == nil {
|
|
t.Fatal("empty configuration accepted")
|
|
}
|
|
if _, err := New(Config{Socket: "relative.sock", ArchiveRoot: "/tmp/a", JournalRoot: "/tmp/j",
|
|
Binary: "/bin/true", Project: "p", Writers: "app", NonWriters: "frontend=ui", MaxDuration: time.Minute}); err == nil {
|
|
t.Fatal("relative socket path accepted")
|
|
}
|
|
if _, err := New(Config{Socket: "/tmp/a.sock", ArchiveRoot: "/tmp/a", JournalRoot: "/tmp/j",
|
|
Binary: "/bin/true", Project: "p", Writers: "app", NonWriters: "frontend", MaxDuration: time.Minute}); err == nil {
|
|
t.Fatal("non-writer declaration without a reason accepted")
|
|
}
|
|
}
|
|
|
|
// 宿主机可达的源对象存储端点必须由 agent 自己钉住：请求不能改，也不能被悄悄丢掉
|
|
// （掉了就会退回"容器里那个解析不到的服务名"，真实封存报 read library resource）。
|
|
func TestAgentPinsTheHostSideObjectEndpoint(t *testing.T) {
|
|
config := Config{
|
|
Socket: "/tmp/agent.sock",
|
|
Binary: "/bin/echo",
|
|
ArchiveRoot: "/tmp/library",
|
|
JournalRoot: "/tmp/freeze",
|
|
SourceApp: "app-container",
|
|
SourcePG: "pg",
|
|
SourceES: "es",
|
|
SourceS3Endpoint: "http://127.0.0.1:19100",
|
|
GovernancePG: "gov",
|
|
Project: "proj",
|
|
Writers: "app",
|
|
NonWriters: "postgres=database store",
|
|
MaxDuration: time.Minute,
|
|
}
|
|
agent, err := New(config)
|
|
if err != nil {
|
|
t.Fatalf("new: %v", err)
|
|
}
|
|
args := agent.config.declaration()
|
|
found := false
|
|
for i := 0; i+1 < len(args); i++ {
|
|
if args[i] == "-source-s3-endpoint" && args[i+1] == "http://127.0.0.1:19100" {
|
|
found = true
|
|
}
|
|
}
|
|
if !found {
|
|
t.Fatalf("声明里没有宿主侧对象存储端点：%v", args)
|
|
}
|
|
config.SourceS3Endpoint = ""
|
|
bare, err := New(config)
|
|
if err != nil {
|
|
t.Fatalf("new without endpoint: %v", err)
|
|
}
|
|
for _, arg := range bare.config.declaration() {
|
|
if arg == "-source-s3-endpoint" {
|
|
t.Fatal("没有端点时不该凭空多一个 flag")
|
|
}
|
|
}
|
|
}
|