knowledge-governance/internal/qaopsmirror/sync.go

package qaopsmirror

import (
	"context"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"errors"
	"github.com/jackc/pgx/v5"
	"knowledge-governance/internal/dobotaiops"
	"knowledge-governance/internal/jobs"
	pg "knowledge-governance/internal/platform/postgres"
	"strings"
	"time"
	"unicode/utf8"
)

const SyncJobPrefix = "qaops.records.sync."

func (s *Store) EnqueueDue(ctx context.Context) (bool, error) {
	if !s.valid() {
		return false, dobotaiops.ErrUnavailable
	}
	// A final-attempt process can die before recordFailure. Rotate the persisted
	// due time once, otherwise ExecuteOperation would replay its dead receipt forever.
	if _, e := s.pool.Exec(ctx, `WITH latest AS (SELECT state,attempt,max_attempts,lease_until,updated_at FROM jobs WHERE type=$2 ORDER BY created_at DESC,id DESC LIMIT 1) UPDATE qa_ops_mirror_state s SET next_sync_at=clock_timestamp()+interval '5 minutes',last_error='mirror_sync_interrupted' FROM latest j WHERE s.tenant_key=$1 AND s.next_sync_at<=j.updated_at AND(j.state='failed' OR(j.state='running' AND j.attempt>=j.max_attempts AND j.lease_until<clock_timestamp()))`, s.tenant, s.SyncJobType()); e != nil {
		return false, dobotaiops.ErrUnavailable
	}
	var next time.Time
	var due bool
	e := s.pool.QueryRow(ctx, `SELECT next_sync_at,next_sync_at<=clock_timestamp() FROM qa_ops_mirror_state WHERE tenant_key=$1`, s.tenant).Scan(&next, &due)
	if e != nil && !errors.Is(e, pgx.ErrNoRows) {
		return false, dobotaiops.ErrUnavailable
	}
	if e == nil && !due {
		return false, nil
	}
	key := "automatic:initial"
	if !next.IsZero() {
		key = "automatic:" + next.UTC().Format(time.RFC3339Nano)
	}
	receipt, replayed, e := s.EnqueueSync(ctx, key, "system:qa-record-sync")
	return e == nil && !replayed && !receipt.Deduplicated, e
}

var ErrWorkerUnavailable = errors.New("local mirror worker is unavailable")
var ErrInvalid = errors.New("invalid local mirror request")
var ErrConflict = errors.New("local mirror request conflict")
var ErrLease = errors.New("local mirror lease lost")
var ErrSync = errors.New("local mirror source synchronization failed; last committed data retained")

type SyncReceipt struct {
	JobID        string `json:"job_id"`
	Deduplicated bool   `json:"deduplicated"`
}

func (s *Store) SyncJobType() string {
	h := sha256.Sum256([]byte(s.tenant))
	return SyncJobPrefix + hex.EncodeToString(h[:])
}

const workerReadySQL = `SELECT EXISTS(SELECT 1 FROM worker_heartbeats WHERE $1=ANY(string_to_array(job_types,',')) AND last_seen_at>clock_timestamp()-interval '2 minutes') OR EXISTS(SELECT 1 FROM jobs WHERE type=$1 AND state='running' AND lease_until>clock_timestamp())`

func validToken(v string, n int) bool {
	return strings.TrimSpace(v) == v && len(v) > 0 && len(v) <= n && utf8.ValidString(v) && !strings.ContainsRune(v, 0)
}
func (s *Store) EnqueueSync(ctx context.Context, key, actor string) (SyncReceipt, bool, error) {
	if !s.valid() {
		return SyncReceipt{}, false, dobotaiops.ErrUnavailable
	}
	if !validToken(key, 128) || !validToken(actor, 256) {
		return SyncReceipt{}, false, ErrInvalid
	}
	fp := sha256.Sum256([]byte(actor))
	now := time.Now()
	raw, replay, err := pg.NewStore(s.pool).ExecuteOperation(ctx, s.SyncJobType(), key, hex.EncodeToString(fp[:]), now, func(tx pgx.Tx) (json.RawMessage, error) {
		if _, e := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1,0))`, s.SyncJobType()); e != nil {
			return nil, dobotaiops.ErrUnavailable
		}
		if _, e := tx.Exec(ctx, `UPDATE jobs SET state='failed',last_error='local mirror worker interrupted',last_error_code='mirror_sync_interrupted',lease_owner=NULL,lease_until=NULL,updated_at=now() WHERE type=$1 AND state='running' AND lease_until<clock_timestamp() AND attempt>=max_attempts`, s.SyncJobType()); e != nil {
			return nil, dobotaiops.ErrUnavailable
		}
		var active string
		e := tx.QueryRow(ctx, `SELECT id FROM jobs WHERE type=$1 AND state IN ('pending','running','retry') ORDER BY created_at DESC,id DESC LIMIT 1`, s.SyncJobType()).Scan(&active)
		if e == nil {
			return json.Marshal(SyncReceipt{JobID: active, Deduplicated: true})
		}
		if !errors.Is(e, pgx.ErrNoRows) {
			return nil, dobotaiops.ErrUnavailable
		}
		var ready bool
		if tx.QueryRow(ctx, workerReadySQL, s.SyncJobType()).Scan(&ready) != nil {
			return nil, dobotaiops.ErrUnavailable
		}
		if !ready {
			return nil, ErrWorkerUnavailable
		}
		if _, e = tx.Exec(ctx, `INSERT INTO qa_ops_mirror_state(tenant_key) VALUES($1) ON CONFLICT DO NOTHING`, s.tenant); e != nil {
			return nil, dobotaiops.ErrUnavailable
		}
		h := sha256.Sum256([]byte(s.SyncJobType() + ":" + key))
		payload, _ := json.Marshal(map[string]string{"triggered_by": actor})
		job, e := pg.NewStore(tx).Enqueue(ctx, jobs.EnqueueRequest{ID: "job-" + hex.EncodeToString(h[:16]), Type: s.SyncJobType(), Payload: payload, AvailableAt: now, MaxAttempts: 3, IdempotencyKey: s.SyncJobType() + ":" + key}, now)
		if e != nil || job == nil {
			return nil, dobotaiops.ErrUnavailable
		}
		return json.Marshal(SyncReceipt{JobID: job.ID})
	})
	if errors.Is(err, pg.ErrOperationRequestConflict) {
		return SyncReceipt{}, false, ErrConflict
	}
	if err != nil {
		return SyncReceipt{}, false, err
	}
	var receipt SyncReceipt
	if json.Unmarshal(raw, &receipt) != nil || receipt.JobID == "" {
		return receipt, false, dobotaiops.ErrUnavailable
	}
	return receipt, replay, nil
}