package qaopsmirror import ( "context" "encoding/json" "errors" "github.com/jackc/pgx/v5" "knowledge-governance/internal/dobotaiops" "knowledge-governance/internal/jobs" "knowledge-governance/internal/qaopsquality" "regexp" "sort" "time" ) type Source interface { ReadMirrorManifest(context.Context, string) (dobotaiops.MirrorManifest, error) ReadMirrorRecords(context.Context, string, []string) ([]dobotaiops.MirrorRecord, error) } type SourceFactory func(context.Context) (Source, error) var hashPattern = regexp.MustCompile(`^[a-f0-9]{64}$`) func (s *Store) lease(ctx context.Context, q dobotaiops.RecordQuerier, j jobs.Job, lock bool) (bool, error) { if j.Type != s.SyncJobType() || j.LeaseOwner == "" { return false, ErrLease } sql := `SELECT payload ? 'mirror_result' FROM jobs WHERE id=$1 AND type=$2 AND state='running' AND lease_owner=$3 AND attempt=$4 AND lease_until>clock_timestamp() AND EXISTS(SELECT 1 FROM operation_requests op WHERE op.operation_type=jobs.type AND op.result->>'job_id'=jobs.id)` if lock { sql += " FOR UPDATE" } var done bool if q.QueryRow(ctx, sql, j.ID, s.SyncJobType(), j.LeaseOwner, j.Attempt).Scan(&done) != nil { return false, ErrLease } return done, nil } func (s *Store) progress(ctx context.Context, j jobs.Job, loaded, expected, changed int) error { p, _ := json.Marshal(map[string]int{"loaded": loaded, "expected": expected, "changed": changed}) tag, e := s.pool.Exec(ctx, `UPDATE jobs SET payload=jsonb_set(payload,'{mirror_progress}',$5::jsonb),updated_at=now() WHERE id=$1 AND type=$2 AND lease_owner=$3 AND attempt=$4 AND state='running' AND lease_until>clock_timestamp()`, j.ID, s.SyncJobType(), j.LeaseOwner, j.Attempt, p) if e != nil || tag.RowsAffected() != 1 { return ErrLease } return nil } func (s *Store) recordFailure(j jobs.Job, cause error) { ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) defer cancel() code := "mirror_source_unavailable" blocked := errors.Is(cause, dobotaiops.ErrNotReadOnly) if blocked { code = "mirror_source_authorization_failed" } _, _ = s.pool.Exec(ctx, `UPDATE qa_ops_mirror_state SET last_error=$2,blocked=blocked OR $3,next_sync_at=clock_timestamp()+interval '5 minutes' WHERE tenant_key=$1 AND EXISTS(SELECT 1 FROM jobs WHERE id=$4 AND type=$5 AND state='running' AND lease_owner=$6 AND attempt=$7 AND lease_until>clock_timestamp())`, s.tenant, code, blocked, j.ID, s.SyncJobType(), j.LeaseOwner, j.Attempt) } func (s *Store) HandleSync(ctx context.Context, j jobs.Job, factory SourceFactory) (err error) { if !s.valid() || factory == nil { return ErrSync } done, e := s.lease(ctx, poolQuery{s.pool}, j, false) if e != nil { return e } if done { return nil } defer func() { if err != nil && !errors.Is(err, ErrLease) { s.recordFailure(j, err) if errors.Is(err, dobotaiops.ErrNotReadOnly) { err = jobs.Permanent("mirror_source_authorization_failed", ErrSync) } else { err = ErrSync } } }() source, e := factory(ctx) if e != nil { return e } if closer, ok := source.(interface{ Close() }); ok { defer closer.Close() } known := map[string]string{} rows, e := s.pool.Query(ctx, `SELECT id,fingerprint FROM qa_ops_mirror_records WHERE tenant_key=$1`, s.tenant) if e != nil { return dobotaiops.ErrUnavailable } for rows.Next() { var id, fp string if rows.Scan(&id, &fp) != nil { rows.Close() return dobotaiops.ErrUnavailable } known[id] = fp } e = rows.Err() rows.Close() if e != nil { return dobotaiops.ErrUnavailable } manifest, e := source.ReadMirrorManifest(ctx, s.tenant) if e != nil { return e } if manifest.AsOf.IsZero() || manifest.AsOf.After(time.Now().Add(5*time.Minute)) || len(manifest.Keys) > dobotaiops.MaxMirrorRecords { return ErrSync } // A sudden completely empty scope is not accepted as authority to erase a populated local history. if len(manifest.Keys) == 0 && len(known) > 0 { return ErrSync } keys := map[string]string{} changed := []string{} ids := make([]string, 0, len(manifest.Keys)) for _, k := range manifest.Keys { if !validToken(k.ID, 256) || !hashPattern.MatchString(k.Fingerprint) || keys[k.ID] != "" { return ErrSync } keys[k.ID] = k.Fingerprint ids = append(ids, k.ID) if known[k.ID] != k.Fingerprint { changed = append(changed, k.ID) } } sort.Strings(changed) loaded := len(ids) - len(changed) if e = s.progress(ctx, j, loaded, len(ids), 0); e != nil { return e } collected := make([]dobotaiops.MirrorRecord, 0, len(changed)) totalBytes := 0 for start := 0; start < len(changed); start += 100 { if _, e = s.lease(ctx, poolQuery{s.pool}, j, false); e != nil { return e } batch := changed[start:min(start+100, len(changed))] got, e := source.ReadMirrorRecords(ctx, s.tenant, batch) if e != nil { return e } if len(got) != len(batch) { return ErrSync } expected := map[string]bool{} for _, id := range batch { expected[id] = true } for _, row := range got { if !expected[row.Key.ID] || !hashPattern.MatchString(row.Key.Fingerprint) { return ErrSync } delete(expected, row.Key.ID) record, e := dobotaiops.DecodeMirrorPayload(string(row.Payload)) if e != nil || record.ID != row.Key.ID || !hashPattern.MatchString(record.QualityInputHash) || record.QualityObservedAt.IsZero() || record.AskedAt.IsZero() || len(row.Payload) > 8<<20 { return ErrSync } totalBytes += len(row.Payload) if totalBytes > 256<<20 { return ErrSync } // A row can complete between manifest and payload reads. Keep that newer full // row, never store an old manifest hash against a different current payload. keys[row.Key.ID] = row.Key.Fingerprint if record.QualityObservedAt.After(manifest.AsOf) { manifest.AsOf = record.QualityObservedAt } collected = append(collected, row) } loaded += len(got) if e = s.progress(ctx, j, loaded, len(ids), len(collected)); e != nil { return e } } return s.activate(ctx, j, manifest, ids, known, collected) } func (s *Store) activate(ctx context.Context, j jobs.Job, m dobotaiops.MirrorManifest, ids []string, known map[string]string, rows []dobotaiops.MirrorRecord) error { tx, e := s.pool.Begin(ctx) if e != nil { return dobotaiops.ErrUnavailable } defer func() { _ = tx.Rollback(context.Background()) }() if _, e = tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1,0))`, s.SyncJobType()); e != nil { return dobotaiops.ErrUnavailable } done, e := s.lease(ctx, txQuery{tx}, j, true) if e != nil { return e } if done { return nil } quality := qaopsquality.NewStore(s.pool, s.tenant) for start := 0; start < len(rows); start += 100 { batch := rows[start:min(start+100, len(rows))] records := make([]dobotaiops.Record, 0, len(batch)) for _, row := range batch { rec, e := dobotaiops.DecodeMirrorPayload(string(row.Payload)) if e != nil { return ErrSync } records = append(records, rec) identity, _ := json.Marshal(rec.Identity) if _, e = tx.Exec(ctx, `INSERT INTO qa_ops_mirror_records(tenant_key,id,fingerprint,asked_at,question,status,identity,quality_hash,payload) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9) ON CONFLICT(tenant_key,id) DO UPDATE SET fingerprint=EXCLUDED.fingerprint,asked_at=EXCLUDED.asked_at,question=EXCLUDED.question,status=EXCLUDED.status,identity=EXCLUDED.identity,quality_hash=EXCLUDED.quality_hash,payload=EXCLUDED.payload`, s.tenant, rec.ID, row.Key.Fingerprint, rec.AskedAt, rec.Question, rec.Status, identity, rec.QualityInputHash, row.Payload); e != nil { return dobotaiops.ErrUnavailable } } if e = quality.ObserveRecordsInTx(ctx, tx, records); e != nil { return e } } deleted, e := tx.Exec(ctx, `DELETE FROM qa_ops_mirror_records WHERE tenant_key=$1 AND NOT(id=ANY($2::text[]))`, s.tenant, ids) if e != nil { return dobotaiops.ErrUnavailable } changed := len(rows) > 0 || deleted.RowsAffected() > 0 var revision int64 var wasReady bool if tx.QueryRow(ctx, `SELECT revision,ready FROM qa_ops_mirror_state WHERE tenant_key=$1 FOR UPDATE`, s.tenant).Scan(&revision, &wasReady) != nil { return dobotaiops.ErrUnavailable } if changed || !wasReady { revision++ } safety, _ := json.Marshal(m.Safety) if _, e = tx.Exec(ctx, `UPDATE qa_ops_mirror_state SET ready=true,blocked=false,revision=$2,record_count=$3,source_observed_at=$4,synced_at=clock_timestamp(),next_sync_at=clock_timestamp()+interval '5 minutes',last_error='',source_safety=$5 WHERE tenant_key=$1`, s.tenant, revision, len(ids), m.AsOf, safety); e != nil { return dobotaiops.ErrUnavailable } result, _ := json.Marshal(map[string]any{"records": len(ids), "changed": len(rows), "deleted": deleted.RowsAffected(), "revision": revision}) tag, e := tx.Exec(ctx, `UPDATE jobs SET payload=jsonb_set(payload,'{mirror_result}',$5::jsonb),updated_at=now() WHERE id=$1 AND type=$2 AND lease_owner=$3 AND attempt=$4 AND state='running' AND lease_until>clock_timestamp()`, j.ID, s.SyncJobType(), j.LeaseOwner, j.Attempt, result) if e != nil || tag.RowsAffected() != 1 { return ErrLease } if tx.Commit(ctx) != nil { return dobotaiops.ErrUnavailable } return nil } func (s *Store) GetSyncStatus(ctx context.Context) (Status, error) { if !s.valid() { return Status{}, dobotaiops.ErrUnavailable } state, e := s.readState(ctx, poolQuery{s.pool}) if e != nil { return state, e } if s.pool.QueryRow(ctx, workerReadySQL, s.SyncJobType()).Scan(&state.WorkerReady) != nil { return Status{}, dobotaiops.ErrUnavailable } var j SyncJob var payload []byte var expired bool var attempt, maxAttempts int e = s.pool.QueryRow(ctx, `SELECT id,state,payload,COALESCE(last_error_code,''),COALESCE(state='running' AND lease_until= maxAttempts { j.State = "failed" } j.ErrorCode = "mirror_sync_interrupted" } state.Job = &j return state, nil }