姚章浩
3e7fcce0b1
build / build (push) Successful in 1m11s
新增迁移50与每5分钟/手动持久同步，API只读本地；同步失败保留、权限阻断、租约与质量失效原子激活。 保留全量稳定分页/固定列，已存质量批量读取，元数据不反复解压长答案；详情安全Markdown与原文。 已运行整仓 make verify、源与副本隔离PG -race、真实只读源及隔离浏览器。 实际同步2525条；固定窗口640条本地中位1.46秒，对照直接源33.10秒。 停worker、缺源凭据、重启API后仍可查询；未更新8088，未修改生产原问答。
351 lines
13 KiB
Go
351 lines
13 KiB
Go
package qaopsquality
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"github.com/jackc/pgx/v5"
|
|
"knowledge-governance/internal/dobotaiops"
|
|
"sort"
|
|
"time"
|
|
)
|
|
|
|
func (s *Store) Columns(ctx context.Context, keys []dobotaiops.QualityKey) ([]string, error) {
|
|
if !s.available() {
|
|
return nil, ErrUnavailable
|
|
}
|
|
if len(keys) > 100000 {
|
|
return nil, ErrInvalid
|
|
}
|
|
for _, key := range keys {
|
|
if !validID(key.ID) || !hashPattern.MatchString(key.InputHash) {
|
|
return nil, ErrInvalid
|
|
}
|
|
}
|
|
raw, err := json.Marshal(keys)
|
|
if err != nil {
|
|
return nil, ErrInvalid
|
|
}
|
|
if len(keys) == 0 {
|
|
return []string{}, nil
|
|
}
|
|
var coverage, correctness, product, problem, root bool
|
|
err = s.pool.QueryRow(ctx, `SELECT COALESCE(bool_or(r.fields->>'coverage'<>''),false),COALESCE(bool_or(r.fields->>'correctness'<>''),false),COALESCE(bool_or(r.fields->>'product_category'<>''),false),COALESCE(bool_or(r.fields->>'problem_type'<>''),false),COALESCE(bool_or(r.fields->>'root_cause'<>''),false) FROM jsonb_to_recordset($2::jsonb) k(id text,input_hash text) JOIN qa_ops_quality_records h ON h.tenant_key=$1 AND h.record_id=k.id AND h.input_hash=k.input_hash JOIN qa_ops_quality_reviews r ON r.tenant_key=h.tenant_key AND r.record_id=h.record_id AND r.revision=h.review_revision AND r.input_epoch=h.input_epoch`, s.tenant, raw).Scan(&coverage, &correctness, &product, &problem, &root)
|
|
if err != nil {
|
|
return nil, ErrUnavailable
|
|
}
|
|
out := []string{}
|
|
for _, c := range []struct {
|
|
key string
|
|
yes bool
|
|
}{{"coverage", coverage}, {"is_correct", correctness}, {"product_category", product}, {"problem_type", problem}, {"root_cause", root}} {
|
|
if c.yes {
|
|
out = append(out, c.key)
|
|
}
|
|
}
|
|
return out, nil
|
|
}
|
|
func (s *Store) Snapshot(ctx context.Context, id string, epoch int64) (Snapshot, error) {
|
|
if !s.available() {
|
|
return Snapshot{}, ErrUnavailable
|
|
}
|
|
if !validID(id) || epoch < 1 {
|
|
return Snapshot{}, ErrInvalid
|
|
}
|
|
var value Snapshot
|
|
value.InputEpoch = epoch
|
|
err := s.pool.QueryRow(ctx, `SELECT input_hash,payload,created_at FROM qa_ops_quality_inputs WHERE tenant_key=$1 AND record_id=$2 AND input_epoch=$3`, s.tenant, id, epoch).Scan(&value.InputHash, &value.Payload, &value.CreatedAt)
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return Snapshot{}, ErrNotFound
|
|
}
|
|
if err != nil {
|
|
return Snapshot{}, ErrUnavailable
|
|
}
|
|
if len(value.Payload) == 0 {
|
|
return Snapshot{}, ErrSnapshotMissing
|
|
}
|
|
return value, nil
|
|
}
|
|
func (s *Store) available() bool { return s != nil && s.pool != nil && validID(s.tenant) }
|
|
func (s *Store) observeLocked(ctx context.Context, tx pgx.Tx, e Evidence) error {
|
|
id := e.Record.ID
|
|
if _, err := tx.Exec(ctx, `INSERT INTO qa_ops_quality_records(tenant_key,record_id) VALUES($1,$2) ON CONFLICT DO NOTHING`, s.tenant, id); err != nil {
|
|
return ErrUnavailable
|
|
}
|
|
var previous string
|
|
var epoch int64
|
|
var observedAt *time.Time
|
|
if tx.QueryRow(ctx, `SELECT input_hash,input_epoch,observed_at FROM qa_ops_quality_records WHERE tenant_key=$1 AND record_id=$2 FOR UPDATE`, s.tenant, id).Scan(&previous, &epoch, &observedAt) != nil {
|
|
return ErrUnavailable
|
|
}
|
|
// Source transaction time orders concurrent page/detail/scope observations.
|
|
// A late different input is rejected, never projected against a newer answer.
|
|
if observedAt != nil && !e.Record.QualityObservedAt.IsZero() && e.Record.QualityObservedAt.Before(*observedAt) && previous != e.Hash {
|
|
return ErrConflict
|
|
}
|
|
if !e.Record.QualityObservedAt.IsZero() {
|
|
if _, err := tx.Exec(ctx, `UPDATE qa_ops_quality_records SET observed_at=$3 WHERE tenant_key=$1 AND record_id=$2 AND(observed_at IS NULL OR observed_at<$3)`, s.tenant, id, e.Record.QualityObservedAt); err != nil {
|
|
return ErrUnavailable
|
|
}
|
|
}
|
|
var payload any
|
|
if len(e.Payload) > 0 {
|
|
payload = e.Payload
|
|
}
|
|
if previous != e.Hash {
|
|
epoch++
|
|
if _, err := tx.Exec(ctx, `INSERT INTO qa_ops_quality_inputs(tenant_key,record_id,input_epoch,input_hash,payload) VALUES($1,$2,$3,$4,$5)`, s.tenant, id, epoch, e.Hash, payload); err != nil {
|
|
return ErrUnavailable
|
|
}
|
|
tag, err := tx.Exec(ctx, `UPDATE qa_ops_quality_records SET input_hash=$3,input_epoch=$4 WHERE tenant_key=$1 AND record_id=$2`, s.tenant, id, e.Hash, epoch)
|
|
if err != nil || tag.RowsAffected() != 1 {
|
|
return ErrUnavailable
|
|
}
|
|
}
|
|
if payload != nil {
|
|
if _, err := tx.Exec(ctx, `UPDATE qa_ops_quality_inputs SET payload=$4 WHERE tenant_key=$1 AND record_id=$2 AND input_epoch=$3 AND payload IS NULL`, s.tenant, id, epoch, payload); err != nil {
|
|
return ErrUnavailable
|
|
}
|
|
}
|
|
automatic, _ := json.Marshal(e.Result)
|
|
source, _ := json.Marshal(e.Record.SourceCoverage)
|
|
if _, err := tx.Exec(ctx, `INSERT INTO qa_ops_quality_checks(tenant_key,record_id,input_epoch,rule_version,result,source) VALUES($1,$2,$3,$4,$5,$6) ON CONFLICT(tenant_key,record_id,input_epoch,rule_version) DO NOTHING`, s.tenant, id, epoch, dobotaiops.CoverageRuleVersion, automatic, source); err != nil {
|
|
return ErrUnavailable
|
|
}
|
|
return nil
|
|
}
|
|
|
|
const checkObject = `jsonb_build_object('id',c.id,'input_epoch',c.input_epoch,'input_hash',i.input_hash,'result',c.result,'source',c.source,'created_at',c.created_at,'input_available',i.payload IS NOT NULL)`
|
|
const reviewObject = `jsonb_build_object('revision',r.revision,'input_epoch',r.input_epoch,'input_hash',r.input_hash,'fields',r.fields,'reason',r.reason,'reviewer',r.reviewer,'created_at',r.created_at)`
|
|
|
|
func (s *Store) view(ctx context.Context, tx pgx.Tx, id string) (View, error) {
|
|
var hash, checkRaw, reviewRaw string
|
|
var epoch, revision int64
|
|
err := tx.QueryRow(ctx, `SELECT h.input_hash,h.input_epoch,h.review_revision,`+checkObject+`::text,COALESCE((SELECT `+reviewObject+` FROM qa_ops_quality_reviews r WHERE r.tenant_key=h.tenant_key AND r.record_id=h.record_id AND r.revision=h.review_revision),'null'::jsonb)::text FROM qa_ops_quality_records h JOIN qa_ops_quality_inputs i ON i.tenant_key=h.tenant_key AND i.record_id=h.record_id AND i.input_epoch=h.input_epoch JOIN qa_ops_quality_checks c ON c.tenant_key=h.tenant_key AND c.record_id=h.record_id AND c.input_epoch=h.input_epoch AND c.rule_version=$3 WHERE h.tenant_key=$1 AND h.record_id=$2`, s.tenant, id, dobotaiops.CoverageRuleVersion).Scan(&hash, &epoch, &revision, &checkRaw, &reviewRaw)
|
|
var automatic Check
|
|
var review *Review
|
|
if err != nil || json.Unmarshal([]byte(checkRaw), &automatic) != nil || json.Unmarshal([]byte(reviewRaw), &review) != nil {
|
|
return View{}, ErrUnavailable
|
|
}
|
|
return Project(hash, epoch, revision, automatic, review), nil
|
|
}
|
|
|
|
// ObserveRecordsInTx makes mirror activation and review invalidation one commit.
|
|
// Only the caller owns commit/rollback; no remote I/O occurs while these locks are held.
|
|
func (s *Store) ObserveRecordsInTx(ctx context.Context, tx pgx.Tx, records []dobotaiops.Record) error {
|
|
if !s.available() || tx == nil || len(records) > 100 {
|
|
return ErrInvalid
|
|
}
|
|
items := make([]Evidence, 0, len(records))
|
|
size := 0
|
|
for _, r := range records {
|
|
e, err := EvidenceFor(r)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
size += len(e.Payload)
|
|
if size > 32<<20 {
|
|
return ErrInvalid
|
|
}
|
|
items = append(items, e)
|
|
}
|
|
sort.Slice(items, func(i, j int) bool { return items[i].Record.ID < items[j].Record.ID })
|
|
for _, e := range items {
|
|
if err := s.observeLocked(ctx, tx, e); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
func (s *Store) Observe(ctx context.Context, records []dobotaiops.Record) (map[string]View, error) {
|
|
if len(records) > 5000 {
|
|
return nil, ErrInvalid
|
|
}
|
|
items := make([]Evidence, 0, len(records))
|
|
for _, record := range records {
|
|
e, err := EvidenceFor(record)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
items = append(items, e)
|
|
}
|
|
return s.observeEvidence(ctx, items)
|
|
}
|
|
func (s *Store) ObserveKeys(ctx context.Context, keys []dobotaiops.QualityKey) (map[string]View, error) {
|
|
if len(keys) > 5000 {
|
|
return nil, ErrInvalid
|
|
}
|
|
items := make([]Evidence, 0, len(keys))
|
|
for _, key := range keys {
|
|
if !validID(key.ID) || !hashPattern.MatchString(key.InputHash) {
|
|
return nil, ErrInvalid
|
|
}
|
|
items = append(items, Evidence{Hash: key.InputHash, Record: dobotaiops.Record{ID: key.ID, SourceCoverage: key.Source, QualityObservedAt: key.ObservedAt}, Result: dobotaiops.CheckMarkerCoverage(key.Marker)})
|
|
}
|
|
return s.observeEvidence(ctx, items)
|
|
}
|
|
func (s *Store) observeEvidence(ctx context.Context, items []Evidence) (map[string]View, error) {
|
|
if !s.available() {
|
|
return nil, ErrUnavailable
|
|
}
|
|
evidence := map[string]Evidence{}
|
|
ids := []string{}
|
|
bytes := 0
|
|
for _, e := range items {
|
|
id := e.Record.ID
|
|
if old, ok := evidence[id]; ok {
|
|
if old.Hash != e.Hash || old.Result != e.Result {
|
|
return nil, ErrInvalid
|
|
}
|
|
continue
|
|
}
|
|
evidence[id] = e
|
|
ids = append(ids, id)
|
|
bytes += len(e.Payload)
|
|
if bytes > 32<<20 {
|
|
return nil, ErrInvalid
|
|
}
|
|
}
|
|
sort.Strings(ids)
|
|
result := map[string]View{}
|
|
if len(ids) == 0 {
|
|
return result, nil
|
|
}
|
|
tx, err := s.pool.Begin(ctx)
|
|
if err != nil {
|
|
return nil, ErrUnavailable
|
|
}
|
|
defer tx.Rollback(context.Background())
|
|
for _, id := range ids {
|
|
if err = s.observeLocked(ctx, tx, evidence[id]); err != nil {
|
|
return nil, err
|
|
}
|
|
v, err := s.view(ctx, tx, id)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
result[id] = v
|
|
}
|
|
if tx.Commit(ctx) != nil {
|
|
return nil, ErrUnavailable
|
|
}
|
|
return result, nil
|
|
}
|
|
func (s *Store) Save(ctx context.Context, record dobotaiops.Record, request SaveRequest, actor string) (Receipt, error) {
|
|
if !s.available() {
|
|
return Receipt{}, ErrUnavailable
|
|
}
|
|
if Validate(request) != nil || !validID(actor) {
|
|
return Receipt{}, ErrInvalid
|
|
}
|
|
e, err := EvidenceFor(record)
|
|
if err != nil {
|
|
return Receipt{}, err
|
|
}
|
|
payload, _ := json.Marshal(request)
|
|
sum := sha256.Sum256(payload)
|
|
requestHash := hex.EncodeToString(sum[:])
|
|
tx, err := s.pool.Begin(ctx)
|
|
if err != nil {
|
|
return Receipt{}, ErrUnavailable
|
|
}
|
|
defer tx.Rollback(context.Background())
|
|
if err = s.observeLocked(ctx, tx, e); err != nil {
|
|
return Receipt{}, err
|
|
}
|
|
var existingHash, existingActor string
|
|
var savedRevision int64
|
|
err = tx.QueryRow(ctx, `SELECT request_hash,reviewer,revision FROM qa_ops_quality_reviews WHERE tenant_key=$1 AND record_id=$2 AND nonce=$3`, s.tenant, record.ID, request.Nonce).Scan(&existingHash, &existingActor, &savedRevision)
|
|
replay := err == nil
|
|
if err != nil && !errors.Is(err, pgx.ErrNoRows) {
|
|
return Receipt{}, ErrUnavailable
|
|
}
|
|
current, err := s.view(ctx, tx, record.ID)
|
|
if err != nil {
|
|
return Receipt{}, err
|
|
}
|
|
if replay {
|
|
if existingHash != requestHash || existingActor != actor {
|
|
return Receipt{}, ErrConflict
|
|
}
|
|
} else {
|
|
if current.InputHash != request.InputHash || current.InputEpoch != request.InputEpoch || current.Revision != request.BaseRevision {
|
|
if tx.Commit(ctx) != nil {
|
|
return Receipt{}, ErrUnavailable
|
|
}
|
|
return Receipt{}, ErrConflict
|
|
}
|
|
savedRevision = current.Revision + 1
|
|
fields, _ := json.Marshal(request.Fields)
|
|
if _, err = tx.Exec(ctx, `INSERT INTO qa_ops_quality_reviews(tenant_key,record_id,revision,input_epoch,input_hash,fields,reason,reviewer,nonce,request_hash) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)`, s.tenant, record.ID, savedRevision, current.InputEpoch, current.InputHash, fields, request.Reason, actor, request.Nonce, requestHash); err != nil {
|
|
return Receipt{}, ErrUnavailable
|
|
}
|
|
tag, err := tx.Exec(ctx, `UPDATE qa_ops_quality_records SET review_revision=$3 WHERE tenant_key=$1 AND record_id=$2`, s.tenant, record.ID, savedRevision)
|
|
if err != nil || tag.RowsAffected() != 1 {
|
|
return Receipt{}, ErrUnavailable
|
|
}
|
|
current, err = s.view(ctx, tx, record.ID)
|
|
if err != nil {
|
|
return Receipt{}, err
|
|
}
|
|
}
|
|
if tx.Commit(ctx) != nil {
|
|
return Receipt{}, ErrUnavailable
|
|
}
|
|
return Receipt{View: current, SavedRevision: savedRevision, Replayed: replay}, nil
|
|
}
|
|
func (s *Store) History(ctx context.Context, id string, reviewBefore, checkBefore int64) (History, error) {
|
|
result := History{Reviews: []Review{}, Checks: []Check{}}
|
|
if !s.available() {
|
|
return result, ErrUnavailable
|
|
}
|
|
if !validID(id) || reviewBefore < 0 || checkBefore < 0 {
|
|
return result, ErrInvalid
|
|
}
|
|
rows, err := s.pool.Query(ctx, `SELECT `+reviewObject+`::text FROM qa_ops_quality_reviews r WHERE tenant_key=$1 AND record_id=$2 AND ($3::bigint=0 OR revision<$3) ORDER BY revision DESC LIMIT 21`, s.tenant, id, reviewBefore)
|
|
if err != nil {
|
|
return result, ErrUnavailable
|
|
}
|
|
for rows.Next() {
|
|
var raw string
|
|
var value Review
|
|
if rows.Scan(&raw) != nil || json.Unmarshal([]byte(raw), &value) != nil {
|
|
rows.Close()
|
|
return result, ErrUnavailable
|
|
}
|
|
result.Reviews = append(result.Reviews, value)
|
|
}
|
|
err = rows.Err()
|
|
rows.Close()
|
|
if err != nil {
|
|
return result, ErrUnavailable
|
|
}
|
|
if len(result.Reviews) > 20 {
|
|
result.Reviews = result.Reviews[:20]
|
|
result.NextReview = result.Reviews[19].Revision
|
|
}
|
|
rows, err = s.pool.Query(ctx, `SELECT `+checkObject+`::text FROM qa_ops_quality_checks c JOIN qa_ops_quality_inputs i USING(tenant_key,record_id,input_epoch) WHERE c.tenant_key=$1 AND c.record_id=$2 AND ($3::bigint=0 OR c.id<$3) ORDER BY c.id DESC LIMIT 21`, s.tenant, id, checkBefore)
|
|
if err != nil {
|
|
return result, ErrUnavailable
|
|
}
|
|
defer rows.Close()
|
|
for rows.Next() {
|
|
var raw string
|
|
var value Check
|
|
if rows.Scan(&raw) != nil || json.Unmarshal([]byte(raw), &value) != nil {
|
|
return result, ErrUnavailable
|
|
}
|
|
result.Checks = append(result.Checks, value)
|
|
}
|
|
if rows.Err() != nil {
|
|
return result, ErrUnavailable
|
|
}
|
|
if len(result.Checks) > 20 {
|
|
result.Checks = result.Checks[:20]
|
|
result.NextCheck = result.Checks[19].ID
|
|
}
|
|
return result, nil
|
|
}
|