knowledge-governance/internal/qaopsorg/sync.go

package qaopsorg

import (
	"context"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"errors"
	"time"

	"github.com/jackc/pgx/v5"
	"knowledge-governance/internal/jobs"
	postgresstore "knowledge-governance/internal/platform/postgres"
)

var ErrSyncWorkerUnavailable = errors.New("directory sync worker is not connected")

const SyncJobPrefix = "qaops.directory.sync."

var ErrSyncLease = errors.New("directory sync lease is no longer owned")

type DirectoryRunner interface {
	Run(context.Context) (Import, error)
}
type SyncReceipt struct {
	JobID        string `json:"job_id"`
	Deduplicated bool   `json:"deduplicated"`
}
type SyncResult struct {
	DirectoryID string    `json:"directory_id"`
	Departments int       `json:"departments"`
	People      int       `json:"people"`
	SyncedAt    time.Time `json:"synced_at"`
}
type SyncJob struct {
	ID          string      `json:"id"`
	State       string      `json:"state"`
	Attempt     int         `json:"attempt"`
	MaxAttempts int         `json:"max_attempts"`
	UpdatedAt   time.Time   `json:"updated_at"`
	ErrorCode   string      `json:"error_code"`
	Result      *SyncResult `json:"result"`
}
type SyncStatus struct {
	Configured  bool     `json:"configured"`
	WorkerReady bool     `json:"worker_ready"`
	Job         *SyncJob `json:"job"`
}

func (s *Store) SyncJobType() string {
	h := sha256.Sum256([]byte(s.tenant))
	return SyncJobPrefix + hex.EncodeToString(h[:])
}

const syncWorkerReadySQL = `SELECT EXISTS(SELECT 1 FROM worker_heartbeats WHERE $1=ANY(string_to_array(job_types,',')) AND last_seen_at>clock_timestamp()-interval '2 minutes') OR EXISTS(SELECT 1 FROM jobs WHERE type=$1 AND state='running' AND lease_until>clock_timestamp())`

func (s *Store) EnqueueSync(ctx context.Context, key, actor string) (SyncReceipt, bool, error) {
	if !s.available() {
		return SyncReceipt{}, false, ErrUnavailable
	}
	if !validID(key) || len(key) > 128 || !validID(actor) {
		return SyncReceipt{}, false, ErrInvalid
	}
	fp := sha256.Sum256([]byte(actor))
	now := time.Now()
	raw, replayed, err := postgresstore.NewStore(s.pool).ExecuteOperation(ctx, s.SyncJobType(), key, hex.EncodeToString(fp[:]), now, func(tx pgx.Tx) (json.RawMessage, error) {
		if _, e := tx.Exec(ctx, "SELECT pg_advisory_xact_lock(hashtextextended($1,0))", s.SyncJobType()); e != nil {
			return nil, ErrUnavailable
		}
		// A last-attempt process may die without recording failure. A new explicit
		// action retires that exhausted lease; it never impersonates a live owner.
		if _, e := tx.Exec(ctx, `UPDATE jobs SET state='failed',last_error='directory worker stopped; previous snapshot retained',last_error_code='directory_sync_failed',lease_owner=NULL,lease_until=NULL,updated_at=now() WHERE type=$1 AND state='running' AND lease_until<clock_timestamp() AND attempt>=max_attempts`, s.SyncJobType()); e != nil {
			return nil, ErrUnavailable
		}
		var active string
		e := tx.QueryRow(ctx, `SELECT id FROM jobs WHERE type=$1 AND state IN ('pending','running','retry') ORDER BY created_at DESC,id DESC LIMIT 1`, s.SyncJobType()).Scan(&active)
		if e == nil {
			return json.Marshal(SyncReceipt{JobID: active, Deduplicated: true})
		}
		if !errors.Is(e, pgx.ErrNoRows) {
			return nil, ErrUnavailable
		}
		var ready bool
		if tx.QueryRow(ctx, syncWorkerReadySQL, s.SyncJobType()).Scan(&ready) != nil {
			return nil, ErrUnavailable
		}
		if !ready {
			return nil, ErrSyncWorkerUnavailable
		}
		digest := sha256.Sum256([]byte(s.SyncJobType() + ":" + key))
		id := "job-" + hex.EncodeToString(digest[:16])
		payload, _ := json.Marshal(map[string]string{"triggered_by": actor})
		job, e := postgresstore.NewStore(tx).Enqueue(ctx, jobs.EnqueueRequest{ID: id, Type: s.SyncJobType(), Payload: payload, AvailableAt: now, MaxAttempts: 3, IdempotencyKey: s.SyncJobType() + ":" + key}, now)
		if e != nil || job == nil {
			return nil, ErrUnavailable
		}
		return json.Marshal(SyncReceipt{JobID: job.ID})
	})
	if errors.Is(err, postgresstore.ErrOperationRequestConflict) {
		return SyncReceipt{}, false, ErrConflict
	}
	if err != nil {
		return SyncReceipt{}, false, err
	}
	var result SyncReceipt
	if json.Unmarshal(raw, &result) != nil || result.JobID == "" {
		return SyncReceipt{}, false, ErrUnavailable
	}
	return result, replayed, nil
}
func (s *Store) GetSyncStatus(ctx context.Context) (SyncStatus, error) {
	result := SyncStatus{Configured: true}
	if !s.available() {
		return SyncStatus{}, ErrUnavailable
	}
	if s.pool.QueryRow(ctx, syncWorkerReadySQL, s.SyncJobType()).Scan(&result.WorkerReady) != nil {
		return SyncStatus{}, ErrUnavailable
	}
	var job SyncJob
	var payload []byte
	var expired bool
	err := s.pool.QueryRow(ctx, `SELECT id,state,attempt,max_attempts,updated_at,payload,COALESCE(state='running' AND lease_until<clock_timestamp(),false) FROM jobs WHERE type=$1 ORDER BY created_at DESC,id DESC LIMIT 1`, s.SyncJobType()).Scan(&job.ID, &job.State, &job.Attempt, &job.MaxAttempts, &job.UpdatedAt, &payload, &expired)
	if errors.Is(err, pgx.ErrNoRows) {
		return result, nil
	}
	if err != nil {
		return SyncStatus{}, ErrUnavailable
	}
	var parsed struct {
		Result *SyncResult `json:"result"`
	}
	if json.Unmarshal(payload, &parsed) != nil {
		return SyncStatus{}, ErrUnavailable
	}
	job.Result = parsed.Result
	if expired {
		job.State = "retry"
		if job.Attempt >= job.MaxAttempts {
			job.State = "failed"
		}
	}
	if job.State == "failed" || job.State == "retry" {
		job.ErrorCode = "directory_sync_failed"
	}
	result.Job = &job
	return result, nil
}
func (s *Store) syncLease(ctx context.Context, q orgQuerier, job jobs.Job, lock bool) (*SyncResult, error) {
	if job.Type != s.SyncJobType() || job.LeaseOwner == "" {
		return nil, ErrSyncLease
	}
	query := `SELECT payload FROM jobs WHERE id=$1 AND type=$2 AND state='running' AND lease_owner=$3 AND attempt=$4 AND lease_until>clock_timestamp() AND EXISTS(SELECT 1 FROM operation_requests op WHERE op.operation_type=jobs.type AND op.result->>'job_id'=jobs.id)`
	if lock {
		query += " FOR UPDATE"
	}
	var raw []byte
	if q.QueryRow(ctx, query, job.ID, s.SyncJobType(), job.LeaseOwner, job.Attempt).Scan(&raw) != nil {
		return nil, ErrSyncLease
	}
	var payload struct {
		Result *SyncResult `json:"result"`
	}
	if json.Unmarshal(raw, &payload) != nil {
		return nil, ErrUnavailable
	}
	return payload.Result, nil
}
func (s *Store) HandleDirectorySync(ctx context.Context, job jobs.Job, runner DirectoryRunner) error {
	if !s.available() || runner == nil {
		return ErrDirectorySync
	}
	result, err := s.syncLease(ctx, s.pool, job, false)
	if err != nil {
		return err
	}
	if result != nil {
		return nil
	}
	input, err := runner.Run(ctx)
	if err != nil {
		return ErrDirectorySync
	}
	if input.Source != "weekly_report_live" {
		return jobs.Permanent("directory_sync_failed", ErrDirectorySync)
	}
	_, err = s.replaceDirectory(ctx, input, &job)
	if err != nil {
		if errors.Is(err, ErrSyncLease) {
			return err
		}
		return ErrDirectorySync
	}
	return nil
}