knowledge-governance/internal/qaopsmirror/failure_integration_test.go

package qaopsmirror

import (
	"context"
	"errors"
	"fmt"
	"knowledge-governance/internal/dobotaiops"
	"knowledge-governance/internal/jobs"
	pg "knowledge-governance/internal/platform/postgres"
	"knowledge-governance/internal/qaopsquality"
	"strings"
	"testing"
	"time"
)

func TestLocalMirrorFailuresUpdatesAndReviewPreservation(t *testing.T) {
	p := mirrorFixture(t)
	s := NewStore(p, "fixture-failures")
	q := pg.NewStore(p)
	ctx := context.Background()
	first := &fixtureSource{rows: []dobotaiops.MirrorRecord{fixtureEntry("one", "old answer")}}
	run := func(source *fixtureSource, key string) error {
		j := claimMirrorJob(t, s, q, key)
		e := s.HandleSync(ctx, *j, func(context.Context) (Source, error) { return source, nil })
		if e == nil {
			if e = q.Complete(ctx, j.ID, j.LeaseOwner, time.Now()); e != nil {
				t.Fatal(e)
			}
		} else {
			if _, cleanupErr := p.Exec(ctx, `UPDATE jobs SET state='failed',lease_owner=NULL,lease_until=NULL WHERE id=$1`, j.ID); cleanupErr != nil {
				t.Fatal(cleanupErr)
			}
		}
		return e
	}
	if e := run(first, "initial-good"); e != nil {
		t.Fatal(e)
	}
	quality := qaopsquality.NewStore(p, s.tenant)
	record, _ := s.GetRecord(ctx, s.tenant, "one")
	views, e := quality.Observe(ctx, []dobotaiops.Record{record})
	if e != nil {
		t.Fatal(e)
	}
	v := views["one"]
	saved, e := quality.Save(ctx, record, qaopsquality.SaveRequest{InputHash: v.InputHash, InputEpoch: v.InputEpoch, BaseRevision: v.Revision, Nonce: strings.Repeat("a", 32), Fields: qaopsquality.Fields{Coverage: "covered", Correctness: "correct"}, Reason: "explicit synthetic review"}, "fixture-admin")
	if e != nil {
		t.Fatal(e)
	}
	unchanged := &fixtureSource{rows: first.rows}
	if e = run(unchanged, "unchanged-check"); e != nil || unchanged.payloadCalls != 0 {
		t.Fatal("unchanged poll transferred long answers", e)
	}
	state, _ := s.GetSyncStatus(ctx)
	if state.Revision != 1 {
		t.Fatal("unchanged poll invalidated query generations")
	}
	current, e := quality.Observe(ctx, []dobotaiops.Record{record})
	if e != nil || current["one"].State != "reviewed" || current["one"].Revision != saved.SavedRevision {
		t.Fatal("unchanged synchronization damaged review", e)
	}
	partial := &fixtureSource{payloadError: errors.New("fixture interrupted payload"), failAfter: 2}
	partial.rows = append(partial.rows, fixtureEntry("one", "new answer"))
	for n := 0; n < 101; n++ {
		partial.rows = append(partial.rows, fixtureEntry(fmt.Sprintf("extra-%03d", n), "additional answer"))
	}
	if e = run(partial, "interrupted-full-copy"); e == nil {
		t.Fatal("partial fetch was called complete")
	}
	old, e := s.GetRecord(ctx, s.tenant, "one")
	state, _ = s.GetSyncStatus(ctx)
	if e != nil || old.Answer != "old answer" || state.Records != 1 || state.Revision != 1 || state.ErrorCode == "" || !state.Ready {
		t.Fatal("partial failure mutated last-good local copy", e)
	}
	current, e = quality.Observe(ctx, []dobotaiops.Record{old})
	if e != nil || current["one"].State != "reviewed" {
		t.Fatal("partial source fetch advanced review input", e)
	}
	changed := &fixtureSource{rows: []dobotaiops.MirrorRecord{fixtureEntry("one", "changed answer")}}
	if e = run(changed, "changed-old-record"); e != nil {
		t.Fatal(e)
	}
	fresh, _ := s.GetRecord(ctx, s.tenant, "one")
	current, e = quality.Observe(ctx, []dobotaiops.Record{fresh})
	if e != nil || current["one"].State != "stale" || current["one"].LatestReview == nil || current["one"].LatestReview.Revision != saved.SavedRevision {
		t.Fatal("answer change lost history or retained an obsolete judgment", e)
	}
	denied := &fixtureSource{err: dobotaiops.ErrNotReadOnly}
	if e = run(denied, "authorization-denied"); e == nil {
		t.Fatal("source authorization failure ignored")
	}
	if e = s.Available(ctx); !errors.Is(e, dobotaiops.ErrNotReadOnly) {
		t.Fatal("revoked source left local reads enabled", e)
	}
	recovered := &fixtureSource{rows: changed.rows}
	if e = run(recovered, "authorization-recovered"); e != nil || s.Available(ctx) != nil {
		t.Fatal("verified recovery failed", e)
	}
}
func TestLocalMirrorRejectsUnadmittedJobsBeforeSource(t *testing.T) {
	p := mirrorFixture(t)
	s := NewStore(p, "fixture-unadmitted")
	q := pg.NewStore(p)
	ctx := context.Background()
	_, e := q.Enqueue(ctx, jobs.EnqueueRequest{ID: "fixture-bypass", Type: s.SyncJobType(), Payload: []byte(`{}`), AvailableAt: time.Now(), MaxAttempts: 1, IdempotencyKey: "fixture-bypass"}, time.Now())
	if e != nil {
		t.Fatal(e)
	}
	j, e := q.Claim(ctx, "fixture-bypass-worker", []string{s.SyncJobType()}, time.Now(), time.Minute)
	if e != nil || j == nil {
		t.Fatal(e)
	}
	calls := 0
	e = s.HandleSync(ctx, *j, func(context.Context) (Source, error) { calls++; return &fixtureSource{}, nil })
	if !errors.Is(e, ErrLease) || calls != 0 {
		t.Fatal("unadmitted job reached source", e, calls)
	}
}