139 lines
4.2 KiB
Go
139 lines
4.2 KiB
Go
package libraryfreeze
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
func TestFreezeCancelledCaptureCannotSucceed(t *testing.T) {
|
|
_, c, e := freezeFixture(t)
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
result, err := e.Run(ctx, "cancelled", func(context.Context) (string, error) { cancel(); return strings.Repeat("a", 64), nil })
|
|
if err == nil || result.State == "succeeded" {
|
|
t.Fatalf("cancelled work succeeded: %+v %v", result, err)
|
|
}
|
|
if !c.services[0].Running || !c.services[0].Healthy {
|
|
t.Fatal("cancel prevented recovery")
|
|
}
|
|
}
|
|
|
|
type faultyControl struct {
|
|
*fixtureControl
|
|
mode string
|
|
}
|
|
|
|
func (c *faultyControl) Stop(ctx context.Context, id string) error {
|
|
if c.mode == "stop-noop" {
|
|
return nil
|
|
}
|
|
err := c.fixtureControl.Stop(ctx, id)
|
|
if c.mode == "lost-ack" {
|
|
return errors.New("lost stop acknowledgement")
|
|
}
|
|
return err
|
|
}
|
|
func (c *faultyControl) Start(ctx context.Context, id string) error {
|
|
err := c.fixtureControl.Start(ctx, id)
|
|
if c.mode == "unhealthy" {
|
|
for i := range c.services {
|
|
if c.services[i].ID == id {
|
|
c.services[i].Healthy = false
|
|
}
|
|
}
|
|
}
|
|
return err
|
|
}
|
|
func TestFreezeFailureBoundaries(t *testing.T) {
|
|
for _, mode := range []string{"lost-ack", "stop-noop", "unhealthy", "new-replica", "replacement", "capture-error"} {
|
|
t.Run(mode, func(t *testing.T) {
|
|
_, c, e := freezeFixture(t)
|
|
e.control = &faultyControl{fixtureControl: c, mode: mode}
|
|
called := false
|
|
result, err := e.Run(context.Background(), "fault-job", func(context.Context) (string, error) {
|
|
called = true
|
|
switch mode {
|
|
case "new-replica":
|
|
c.services = append(c.services, Service{ID: "unexpected-app", Running: true, Healthy: true})
|
|
case "replacement":
|
|
c.services[0].ID = "replacement-app"
|
|
case "capture-error":
|
|
return "", errors.New("fixture error")
|
|
}
|
|
return strings.Repeat("a", 64), nil
|
|
})
|
|
if err == nil || result.State == "succeeded" {
|
|
t.Fatalf("fault reported success: %+v %v", result, err)
|
|
}
|
|
if (mode == "lost-ack" || mode == "stop-noop") && called {
|
|
t.Fatal("capture ran without verified stop")
|
|
}
|
|
if mode == "replacement" {
|
|
for _, call := range c.calls {
|
|
if strings.HasPrefix(call, "start:") {
|
|
t.Fatal("replacement was started")
|
|
}
|
|
}
|
|
} else if !c.services[0].Running {
|
|
t.Fatal("original service not restored")
|
|
}
|
|
if mode == "unhealthy" || mode == "replacement" {
|
|
j, err := e.load("fault-job")
|
|
if err != nil || terminal(j.State) {
|
|
t.Fatal("unverified recovery marked terminal")
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
func TestFreezeSourceLockBlocksSeparateCoordinator(t *testing.T) {
|
|
root, c, e := freezeFixture(t)
|
|
other, err := Open(root, "test-source", c)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer other.Close()
|
|
_, err = e.Run(context.Background(), "outer", func(context.Context) (string, error) {
|
|
_, blocked := other.Run(context.Background(), "inner", func(context.Context) (string, error) { t.Fatal("overlapping capture"); return "", nil })
|
|
if blocked == nil {
|
|
t.Fatal("source lock did not block")
|
|
}
|
|
if err := other.Recover(context.Background()); err == nil {
|
|
t.Fatal("recovery raced active capture")
|
|
}
|
|
return strings.Repeat("a", 64), nil
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
func TestFreezeRejectsMalformedRecoveryJournal(t *testing.T) {
|
|
for _, mode := range []string{"unknown-state", "invalid-service", "duplicate-service", "false-ownership"} {
|
|
t.Run(mode, func(t *testing.T) {
|
|
_, c, e := freezeFixture(t)
|
|
j := journal{Version: 1, Source: "test-source", Job: "invalid", Result: Result{State: "stopping"}, Services: []serviceRecord{{Service: Service{ID: "app-a", Running: true}, StopRequested: true}}}
|
|
switch mode {
|
|
case "unknown-state":
|
|
j.State = "invented"
|
|
case "invalid-service":
|
|
j.Services[0].ID = "../../wrong"
|
|
case "duplicate-service":
|
|
j.Services = append(j.Services, j.Services[0])
|
|
case "false-ownership":
|
|
j.Services[0].Running = false
|
|
}
|
|
if err := e.save(j); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := e.Recover(context.Background()); err == nil {
|
|
t.Fatal("malformed journal accepted")
|
|
}
|
|
if len(c.calls) != 0 {
|
|
t.Fatalf("malformed journal caused service writes: %v", c.calls)
|
|
}
|
|
})
|
|
}
|
|
}
|