package qaopsorg import ( "context" "crypto/sha256" "encoding/hex" "encoding/json" "errors" "time" "github.com/jackc/pgx/v5" "knowledge-governance/internal/jobs" postgresstore "knowledge-governance/internal/platform/postgres" ) var ErrSyncWorkerUnavailable = errors.New("directory sync worker is not connected") const SyncJobPrefix = "qaops.directory.sync." var ErrSyncLease = errors.New("directory sync lease is no longer owned") type DirectoryRunner interface { Run(context.Context) (Import, error) } type SyncReceipt struct { JobID string `json:"job_id"` Deduplicated bool `json:"deduplicated"` } type SyncResult struct { DirectoryID string `json:"directory_id"` Departments int `json:"departments"` People int `json:"people"` SyncedAt time.Time `json:"synced_at"` } type SyncJob struct { ID string `json:"id"` State string `json:"state"` Attempt int `json:"attempt"` MaxAttempts int `json:"max_attempts"` UpdatedAt time.Time `json:"updated_at"` ErrorCode string `json:"error_code"` Result *SyncResult `json:"result"` } type SyncStatus struct { Configured bool `json:"configured"` WorkerReady bool `json:"worker_ready"` Job *SyncJob `json:"job"` } func (s *Store) SyncJobType() string { h := sha256.Sum256([]byte(s.tenant)) return SyncJobPrefix + hex.EncodeToString(h[:]) } const syncWorkerReadySQL = `SELECT EXISTS(SELECT 1 FROM worker_heartbeats WHERE $1=ANY(string_to_array(job_types,',')) AND last_seen_at>clock_timestamp()-interval '2 minutes') OR EXISTS(SELECT 1 FROM jobs WHERE type=$1 AND state='running' AND lease_until>clock_timestamp())` func (s *Store) EnqueueSync(ctx context.Context, key, actor string) (SyncReceipt, bool, error) { if !s.available() { return SyncReceipt{}, false, ErrUnavailable } if !validID(key) || len(key) > 128 || !validID(actor) { return SyncReceipt{}, false, ErrInvalid } fp := sha256.Sum256([]byte(actor)) now := time.Now() raw, replayed, err := postgresstore.NewStore(s.pool).ExecuteOperation(ctx, s.SyncJobType(), key, hex.EncodeToString(fp[:]), now, func(tx pgx.Tx) (json.RawMessage, error) { if _, e := tx.Exec(ctx, "SELECT pg_advisory_xact_lock(hashtextextended($1,0))", s.SyncJobType()); e != nil { return nil, ErrUnavailable } // A last-attempt process may die without recording failure. A new explicit // action retires that exhausted lease; it never impersonates a live owner. if _, e := tx.Exec(ctx, `UPDATE jobs SET state='failed',last_error='directory worker stopped; previous snapshot retained',last_error_code='directory_sync_failed',lease_owner=NULL,lease_until=NULL,updated_at=now() WHERE type=$1 AND state='running' AND lease_until=max_attempts`, s.SyncJobType()); e != nil { return nil, ErrUnavailable } var active string e := tx.QueryRow(ctx, `SELECT id FROM jobs WHERE type=$1 AND state IN ('pending','running','retry') ORDER BY created_at DESC,id DESC LIMIT 1`, s.SyncJobType()).Scan(&active) if e == nil { return json.Marshal(SyncReceipt{JobID: active, Deduplicated: true}) } if !errors.Is(e, pgx.ErrNoRows) { return nil, ErrUnavailable } var ready bool if tx.QueryRow(ctx, syncWorkerReadySQL, s.SyncJobType()).Scan(&ready) != nil { return nil, ErrUnavailable } if !ready { return nil, ErrSyncWorkerUnavailable } digest := sha256.Sum256([]byte(s.SyncJobType() + ":" + key)) id := "job-" + hex.EncodeToString(digest[:16]) payload, _ := json.Marshal(map[string]string{"triggered_by": actor}) job, e := postgresstore.NewStore(tx).Enqueue(ctx, jobs.EnqueueRequest{ID: id, Type: s.SyncJobType(), Payload: payload, AvailableAt: now, MaxAttempts: 3, IdempotencyKey: s.SyncJobType() + ":" + key}, now) if e != nil || job == nil { return nil, ErrUnavailable } return json.Marshal(SyncReceipt{JobID: job.ID}) }) if errors.Is(err, postgresstore.ErrOperationRequestConflict) { return SyncReceipt{}, false, ErrConflict } if err != nil { return SyncReceipt{}, false, err } var result SyncReceipt if json.Unmarshal(raw, &result) != nil || result.JobID == "" { return SyncReceipt{}, false, ErrUnavailable } return result, replayed, nil } func (s *Store) GetSyncStatus(ctx context.Context) (SyncStatus, error) { result := SyncStatus{Configured: true} if !s.available() { return SyncStatus{}, ErrUnavailable } if s.pool.QueryRow(ctx, syncWorkerReadySQL, s.SyncJobType()).Scan(&result.WorkerReady) != nil { return SyncStatus{}, ErrUnavailable } var job SyncJob var payload []byte var expired bool err := s.pool.QueryRow(ctx, `SELECT id,state,attempt,max_attempts,updated_at,payload,COALESCE(state='running' AND lease_until= job.MaxAttempts { job.State = "failed" } } if job.State == "failed" || job.State == "retry" { job.ErrorCode = "directory_sync_failed" } result.Job = &job return result, nil } func (s *Store) syncLease(ctx context.Context, q orgQuerier, job jobs.Job, lock bool) (*SyncResult, error) { if job.Type != s.SyncJobType() || job.LeaseOwner == "" { return nil, ErrSyncLease } query := `SELECT payload FROM jobs WHERE id=$1 AND type=$2 AND state='running' AND lease_owner=$3 AND attempt=$4 AND lease_until>clock_timestamp() AND EXISTS(SELECT 1 FROM operation_requests op WHERE op.operation_type=jobs.type AND op.result->>'job_id'=jobs.id)` if lock { query += " FOR UPDATE" } var raw []byte if q.QueryRow(ctx, query, job.ID, s.SyncJobType(), job.LeaseOwner, job.Attempt).Scan(&raw) != nil { return nil, ErrSyncLease } var payload struct { Result *SyncResult `json:"result"` } if json.Unmarshal(raw, &payload) != nil { return nil, ErrUnavailable } return payload.Result, nil } func (s *Store) HandleDirectorySync(ctx context.Context, job jobs.Job, runner DirectoryRunner) error { if !s.available() || runner == nil { return ErrDirectorySync } result, err := s.syncLease(ctx, s.pool, job, false) if err != nil { return err } if result != nil { return nil } input, err := runner.Run(ctx) if err != nil { return ErrDirectorySync } if input.Source != "weekly_report_live" { return jobs.Permanent("directory_sync_failed", ErrDirectorySync) } _, err = s.replaceDirectory(ctx, input, &job) if err != nil { if errors.Is(err, ErrSyncLease) { return err } return ErrDirectorySync } return nil }