姚章浩
3e7fcce0b1
build / build (push) Successful in 1m11s
新增迁移50与每5分钟/手动持久同步，API只读本地；同步失败保留、权限阻断、租约与质量失效原子激活。 保留全量稳定分页/固定列，已存质量批量读取，元数据不反复解压长答案；详情安全Markdown与原文。 已运行整仓 make verify、源与副本隔离PG -race、真实只读源及隔离浏览器。 实际同步2525条；固定窗口640条本地中位1.46秒，对照直接源33.10秒。 停worker、缺源凭据、重启API后仍可查询；未更新8088，未修改生产原问答。
284 lines
10 KiB
Go
284 lines
10 KiB
Go
package qaopsmirror
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"github.com/jackc/pgx/v5"
|
|
"knowledge-governance/internal/dobotaiops"
|
|
"knowledge-governance/internal/jobs"
|
|
"knowledge-governance/internal/qaopsquality"
|
|
"regexp"
|
|
"sort"
|
|
"time"
|
|
)
|
|
|
|
type Source interface {
|
|
ReadMirrorManifest(context.Context, string) (dobotaiops.MirrorManifest, error)
|
|
ReadMirrorRecords(context.Context, string, []string) ([]dobotaiops.MirrorRecord, error)
|
|
}
|
|
type SourceFactory func(context.Context) (Source, error)
|
|
|
|
var hashPattern = regexp.MustCompile(`^[a-f0-9]{64}$`)
|
|
|
|
func (s *Store) lease(ctx context.Context, q dobotaiops.RecordQuerier, j jobs.Job, lock bool) (bool, error) {
|
|
if j.Type != s.SyncJobType() || j.LeaseOwner == "" {
|
|
return false, ErrLease
|
|
}
|
|
sql := `SELECT payload ? 'mirror_result' FROM jobs WHERE id=$1 AND type=$2 AND state='running' AND lease_owner=$3 AND attempt=$4 AND lease_until>clock_timestamp() AND EXISTS(SELECT 1 FROM operation_requests op WHERE op.operation_type=jobs.type AND op.result->>'job_id'=jobs.id)`
|
|
if lock {
|
|
sql += " FOR UPDATE"
|
|
}
|
|
var done bool
|
|
if q.QueryRow(ctx, sql, j.ID, s.SyncJobType(), j.LeaseOwner, j.Attempt).Scan(&done) != nil {
|
|
return false, ErrLease
|
|
}
|
|
return done, nil
|
|
}
|
|
func (s *Store) progress(ctx context.Context, j jobs.Job, loaded, expected, changed int) error {
|
|
p, _ := json.Marshal(map[string]int{"loaded": loaded, "expected": expected, "changed": changed})
|
|
tag, e := s.pool.Exec(ctx, `UPDATE jobs SET payload=jsonb_set(payload,'{mirror_progress}',$5::jsonb),updated_at=now() WHERE id=$1 AND type=$2 AND lease_owner=$3 AND attempt=$4 AND state='running' AND lease_until>clock_timestamp()`, j.ID, s.SyncJobType(), j.LeaseOwner, j.Attempt, p)
|
|
if e != nil || tag.RowsAffected() != 1 {
|
|
return ErrLease
|
|
}
|
|
return nil
|
|
}
|
|
func (s *Store) recordFailure(j jobs.Job, cause error) {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
|
defer cancel()
|
|
code := "mirror_source_unavailable"
|
|
blocked := errors.Is(cause, dobotaiops.ErrNotReadOnly)
|
|
if blocked {
|
|
code = "mirror_source_authorization_failed"
|
|
}
|
|
_, _ = s.pool.Exec(ctx, `UPDATE qa_ops_mirror_state SET last_error=$2,blocked=blocked OR $3,next_sync_at=clock_timestamp()+interval '5 minutes' WHERE tenant_key=$1 AND EXISTS(SELECT 1 FROM jobs WHERE id=$4 AND type=$5 AND state='running' AND lease_owner=$6 AND attempt=$7 AND lease_until>clock_timestamp())`, s.tenant, code, blocked, j.ID, s.SyncJobType(), j.LeaseOwner, j.Attempt)
|
|
}
|
|
func (s *Store) HandleSync(ctx context.Context, j jobs.Job, factory SourceFactory) (err error) {
|
|
if !s.valid() || factory == nil {
|
|
return ErrSync
|
|
}
|
|
done, e := s.lease(ctx, poolQuery{s.pool}, j, false)
|
|
if e != nil {
|
|
return e
|
|
}
|
|
if done {
|
|
return nil
|
|
}
|
|
defer func() {
|
|
if err != nil && !errors.Is(err, ErrLease) {
|
|
s.recordFailure(j, err)
|
|
if errors.Is(err, dobotaiops.ErrNotReadOnly) {
|
|
err = jobs.Permanent("mirror_source_authorization_failed", ErrSync)
|
|
} else {
|
|
err = ErrSync
|
|
}
|
|
}
|
|
}()
|
|
source, e := factory(ctx)
|
|
if e != nil {
|
|
return e
|
|
}
|
|
if closer, ok := source.(interface{ Close() }); ok {
|
|
defer closer.Close()
|
|
}
|
|
known := map[string]string{}
|
|
rows, e := s.pool.Query(ctx, `SELECT id,fingerprint FROM qa_ops_mirror_records WHERE tenant_key=$1`, s.tenant)
|
|
if e != nil {
|
|
return dobotaiops.ErrUnavailable
|
|
}
|
|
for rows.Next() {
|
|
var id, fp string
|
|
if rows.Scan(&id, &fp) != nil {
|
|
rows.Close()
|
|
return dobotaiops.ErrUnavailable
|
|
}
|
|
known[id] = fp
|
|
}
|
|
e = rows.Err()
|
|
rows.Close()
|
|
if e != nil {
|
|
return dobotaiops.ErrUnavailable
|
|
}
|
|
manifest, e := source.ReadMirrorManifest(ctx, s.tenant)
|
|
if e != nil {
|
|
return e
|
|
}
|
|
if manifest.AsOf.IsZero() || manifest.AsOf.After(time.Now().Add(5*time.Minute)) || len(manifest.Keys) > dobotaiops.MaxMirrorRecords {
|
|
return ErrSync
|
|
}
|
|
// A sudden completely empty scope is not accepted as authority to erase a populated local history.
|
|
if len(manifest.Keys) == 0 && len(known) > 0 {
|
|
return ErrSync
|
|
}
|
|
keys := map[string]string{}
|
|
changed := []string{}
|
|
ids := make([]string, 0, len(manifest.Keys))
|
|
for _, k := range manifest.Keys {
|
|
if !validToken(k.ID, 256) || !hashPattern.MatchString(k.Fingerprint) || keys[k.ID] != "" {
|
|
return ErrSync
|
|
}
|
|
keys[k.ID] = k.Fingerprint
|
|
ids = append(ids, k.ID)
|
|
if known[k.ID] != k.Fingerprint {
|
|
changed = append(changed, k.ID)
|
|
}
|
|
}
|
|
sort.Strings(changed)
|
|
loaded := len(ids) - len(changed)
|
|
if e = s.progress(ctx, j, loaded, len(ids), 0); e != nil {
|
|
return e
|
|
}
|
|
collected := make([]dobotaiops.MirrorRecord, 0, len(changed))
|
|
totalBytes := 0
|
|
for start := 0; start < len(changed); start += 100 {
|
|
if _, e = s.lease(ctx, poolQuery{s.pool}, j, false); e != nil {
|
|
return e
|
|
}
|
|
batch := changed[start:min(start+100, len(changed))]
|
|
got, e := source.ReadMirrorRecords(ctx, s.tenant, batch)
|
|
if e != nil {
|
|
return e
|
|
}
|
|
if len(got) != len(batch) {
|
|
return ErrSync
|
|
}
|
|
expected := map[string]bool{}
|
|
for _, id := range batch {
|
|
expected[id] = true
|
|
}
|
|
for _, row := range got {
|
|
if !expected[row.Key.ID] || !hashPattern.MatchString(row.Key.Fingerprint) {
|
|
return ErrSync
|
|
}
|
|
delete(expected, row.Key.ID)
|
|
record, e := dobotaiops.DecodeMirrorPayload(string(row.Payload))
|
|
if e != nil || record.ID != row.Key.ID || !hashPattern.MatchString(record.QualityInputHash) || record.QualityObservedAt.IsZero() || record.AskedAt.IsZero() || len(row.Payload) > 8<<20 {
|
|
return ErrSync
|
|
}
|
|
totalBytes += len(row.Payload)
|
|
if totalBytes > 256<<20 {
|
|
return ErrSync
|
|
}
|
|
// A row can complete between manifest and payload reads. Keep that newer full
|
|
// row, never store an old manifest hash against a different current payload.
|
|
keys[row.Key.ID] = row.Key.Fingerprint
|
|
if record.QualityObservedAt.After(manifest.AsOf) {
|
|
manifest.AsOf = record.QualityObservedAt
|
|
}
|
|
collected = append(collected, row)
|
|
}
|
|
loaded += len(got)
|
|
if e = s.progress(ctx, j, loaded, len(ids), len(collected)); e != nil {
|
|
return e
|
|
}
|
|
}
|
|
return s.activate(ctx, j, manifest, ids, known, collected)
|
|
}
|
|
func (s *Store) activate(ctx context.Context, j jobs.Job, m dobotaiops.MirrorManifest, ids []string, known map[string]string, rows []dobotaiops.MirrorRecord) error {
|
|
tx, e := s.pool.Begin(ctx)
|
|
if e != nil {
|
|
return dobotaiops.ErrUnavailable
|
|
}
|
|
defer func() { _ = tx.Rollback(context.Background()) }()
|
|
if _, e = tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1,0))`, s.SyncJobType()); e != nil {
|
|
return dobotaiops.ErrUnavailable
|
|
}
|
|
done, e := s.lease(ctx, txQuery{tx}, j, true)
|
|
if e != nil {
|
|
return e
|
|
}
|
|
if done {
|
|
return nil
|
|
}
|
|
quality := qaopsquality.NewStore(s.pool, s.tenant)
|
|
for start := 0; start < len(rows); start += 100 {
|
|
batch := rows[start:min(start+100, len(rows))]
|
|
records := make([]dobotaiops.Record, 0, len(batch))
|
|
for _, row := range batch {
|
|
rec, e := dobotaiops.DecodeMirrorPayload(string(row.Payload))
|
|
if e != nil {
|
|
return ErrSync
|
|
}
|
|
records = append(records, rec)
|
|
identity, _ := json.Marshal(rec.Identity)
|
|
if _, e = tx.Exec(ctx, `INSERT INTO qa_ops_mirror_records(tenant_key,id,fingerprint,asked_at,question,status,identity,quality_hash,payload) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9) ON CONFLICT(tenant_key,id) DO UPDATE SET fingerprint=EXCLUDED.fingerprint,asked_at=EXCLUDED.asked_at,question=EXCLUDED.question,status=EXCLUDED.status,identity=EXCLUDED.identity,quality_hash=EXCLUDED.quality_hash,payload=EXCLUDED.payload`, s.tenant, rec.ID, row.Key.Fingerprint, rec.AskedAt, rec.Question, rec.Status, identity, rec.QualityInputHash, row.Payload); e != nil {
|
|
return dobotaiops.ErrUnavailable
|
|
}
|
|
}
|
|
if e = quality.ObserveRecordsInTx(ctx, tx, records); e != nil {
|
|
return e
|
|
}
|
|
}
|
|
deleted, e := tx.Exec(ctx, `DELETE FROM qa_ops_mirror_records WHERE tenant_key=$1 AND NOT(id=ANY($2::text[]))`, s.tenant, ids)
|
|
if e != nil {
|
|
return dobotaiops.ErrUnavailable
|
|
}
|
|
changed := len(rows) > 0 || deleted.RowsAffected() > 0
|
|
var revision int64
|
|
var wasReady bool
|
|
if tx.QueryRow(ctx, `SELECT revision,ready FROM qa_ops_mirror_state WHERE tenant_key=$1 FOR UPDATE`, s.tenant).Scan(&revision, &wasReady) != nil {
|
|
return dobotaiops.ErrUnavailable
|
|
}
|
|
if changed || !wasReady {
|
|
revision++
|
|
}
|
|
safety, _ := json.Marshal(m.Safety)
|
|
if _, e = tx.Exec(ctx, `UPDATE qa_ops_mirror_state SET ready=true,blocked=false,revision=$2,record_count=$3,source_observed_at=$4,synced_at=clock_timestamp(),next_sync_at=clock_timestamp()+interval '5 minutes',last_error='',source_safety=$5 WHERE tenant_key=$1`, s.tenant, revision, len(ids), m.AsOf, safety); e != nil {
|
|
return dobotaiops.ErrUnavailable
|
|
}
|
|
result, _ := json.Marshal(map[string]any{"records": len(ids), "changed": len(rows), "deleted": deleted.RowsAffected(), "revision": revision})
|
|
tag, e := tx.Exec(ctx, `UPDATE jobs SET payload=jsonb_set(payload,'{mirror_result}',$5::jsonb),updated_at=now() WHERE id=$1 AND type=$2 AND lease_owner=$3 AND attempt=$4 AND state='running' AND lease_until>clock_timestamp()`, j.ID, s.SyncJobType(), j.LeaseOwner, j.Attempt, result)
|
|
if e != nil || tag.RowsAffected() != 1 {
|
|
return ErrLease
|
|
}
|
|
if tx.Commit(ctx) != nil {
|
|
return dobotaiops.ErrUnavailable
|
|
}
|
|
return nil
|
|
}
|
|
func (s *Store) GetSyncStatus(ctx context.Context) (Status, error) {
|
|
if !s.valid() {
|
|
return Status{}, dobotaiops.ErrUnavailable
|
|
}
|
|
state, e := s.readState(ctx, poolQuery{s.pool})
|
|
if e != nil {
|
|
return state, e
|
|
}
|
|
if s.pool.QueryRow(ctx, workerReadySQL, s.SyncJobType()).Scan(&state.WorkerReady) != nil {
|
|
return Status{}, dobotaiops.ErrUnavailable
|
|
}
|
|
var j SyncJob
|
|
var payload []byte
|
|
var expired bool
|
|
var attempt, maxAttempts int
|
|
e = s.pool.QueryRow(ctx, `SELECT id,state,payload,COALESCE(last_error_code,''),COALESCE(state='running' AND lease_until<clock_timestamp(),false),attempt,max_attempts FROM jobs WHERE type=$1 ORDER BY created_at DESC,id DESC LIMIT 1`, s.SyncJobType()).Scan(&j.ID, &j.State, &payload, &j.ErrorCode, &expired, &attempt, &maxAttempts)
|
|
if errors.Is(e, pgx.ErrNoRows) {
|
|
return state, nil
|
|
}
|
|
if e != nil {
|
|
return Status{}, dobotaiops.ErrUnavailable
|
|
}
|
|
var data struct {
|
|
Progress struct {
|
|
Loaded int `json:"loaded"`
|
|
Expected int `json:"expected"`
|
|
Changed int `json:"changed"`
|
|
} `json:"mirror_progress"`
|
|
}
|
|
if json.Unmarshal(payload, &data) != nil {
|
|
return Status{}, dobotaiops.ErrUnavailable
|
|
}
|
|
j.Loaded = data.Progress.Loaded
|
|
j.Expected = data.Progress.Expected
|
|
j.Changed = data.Progress.Changed
|
|
if expired {
|
|
j.State = "retry"
|
|
if attempt >= maxAttempts {
|
|
j.State = "failed"
|
|
}
|
|
j.ErrorCode = "mirror_sync_interrupted"
|
|
}
|
|
state.Job = &j
|
|
return state, nil
|
|
}
|