package qaopsai import ( "context" "crypto/rand" "crypto/sha256" "encoding/hex" "encoding/json" "fmt" "github.com/jackc/pgx/v5" "knowledge-governance/internal/dobotaiops" "knowledge-governance/internal/qaopsfields" "knowledge-governance/internal/qaopsquality" "strings" "time" "unicode" "unicode/utf8" ) type Scope struct { Kind string `json:"kind"` IDs []string `json:"ids"` Start string `json:"start"` End string `json:"end"` Keyword string `json:"keyword"` } type Preview struct { ID string `json:"id"` Kind string `json:"kind"` ScopeTotal int `json:"scope_total"` Eligible int `json:"eligible"` Skipped int `json:"skipped"` ConfigRevision int64 `json:"config_revision"` ExpiresAt time.Time `json:"expires_at"` } type Batch struct { ID string `json:"id"` Kind string `json:"kind"` State string `json:"state"` Total int `json:"total"` Succeeded int `json:"succeeded"` Failed int `json:"failed"` Skipped int `json:"skipped"` Running int `json:"running"` Queued int `json:"queued"` Retry int `json:"retry"` Paused int `json:"paused"` CreatedAt time.Time `json:"created_at"` } type recordMeta struct { RootHandled bool ID, Hash, Status, Question string Revision int64 Fields map[string]string InlineFields map[string]string Coverage dobotaiops.CoverageCheck View qaopsquality.View } func newID(prefix string) (string, error) { raw := make([]byte, 16) if _, err := rand.Read(raw); err != nil { return "", ErrUnavailable } return prefix + hex.EncodeToString(raw), nil } func questionHash(question string) string { hash := sha256.Sum256([]byte(question)) return hex.EncodeToString(hash[:]) } func workInputHash(kind string, m recordMeta) string { if kind == "translation" { return questionHash(m.Question) } return m.Hash } func promptVersion(kind string) string { if kind == "translation" { return TranslationPromptVersion } return EvaluationPromptVersion } func (s *Store) readMetas(ctx context.Context, tx pgx.Tx, ids []string, scope *Scope) ([]recordMeta, error) { query := `SELECT m.id,m.quality_hash,m.row_revision,m.status,m.question,m.effective_fields,m.coverage_result,EXISTS(SELECT 1 FROM qa_ops_ai_outputs o JOIN qa_ops_ai_work w ON w.tenant_key=o.tenant_key AND w.id=o.work_id WHERE o.tenant_key=m.tenant_key AND o.record_id=m.id AND o.kind='evaluation' AND o.input_hash=m.quality_hash AND w.state='succeeded' AND o.payload->'fields'?'root_cause' AND o.payload->'fields'->'root_cause'->>'value'=''),m.inline_fields FROM qa_ops_master_read_model m WHERE tenant_key=$1` args := []any{s.tenant} if scope != nil && len(scope.IDs) == 0 { window, err := dobotaiops.NewWindow(s.tenant, scope.Start, scope.End, 90*24*time.Hour, time.Now()) if err != nil || !utf8.ValidString(scope.Keyword) || utf8.RuneCountInString(scope.Keyword) > 200 || strings.ContainsRune(scope.Keyword, 0) { return nil, ErrInvalid } query += ` AND asked_at>=$2 AND asked_at<$3 AND ($4='' OR position(lower($4) IN lower(question||E'\n'||chinese_question||E'\n'||COALESCE(ai_fields->>'question_chinese','')))>0)` args = append(args, window.Start, window.End, scope.Keyword) } else { query += ` AND id=ANY($2::text[])` args = append(args, ids) } query += ` ORDER BY id LIMIT 100001` rows, err := tx.Query(ctx, query, args...) if err != nil { return nil, ErrUnavailable } metas := []recordMeta{} keys := []dobotaiops.QualityKey{} for rows.Next() { var m recordMeta var fields, coverage, edits []byte if rows.Scan(&m.ID, &m.Hash, &m.Revision, &m.Status, &m.Question, &fields, &coverage, &m.RootHandled, &edits) != nil || json.Unmarshal(fields, &m.Fields) != nil || json.Unmarshal(coverage, &m.Coverage) != nil || json.Unmarshal(edits, &m.InlineFields) != nil { rows.Close() return nil, ErrUnavailable } metas = append(metas, m) keys = append(keys, dobotaiops.QualityKey{ID: m.ID, InputHash: m.Hash}) } err = rows.Err() rows.Close() if err != nil { return nil, ErrUnavailable } if len(metas) > 100000 { return nil, ErrInvalid } views := map[string]qaopsquality.View{} quality := qaopsquality.NewStore(s.pool, s.tenant+"::master") for begin := 0; begin < len(keys); begin += 5000 { end := begin + 5000 if end > len(keys) { end = len(keys) } part, e := quality.ReadViewsInTx(ctx, tx, keys[begin:end]) if e != nil { return nil, ErrUnavailable } for id, view := range part { views[id] = view } } for i := range metas { v, ok := views[metas[i].ID] if !ok { return nil, ErrUnavailable } metas[i].View = qaopsquality.ProjectMasterWithEdits(v, metas[i].Fields, metas[i].Coverage, metas[i].InlineFields) } return metas, nil } func fieldValue(m recordMeta, field string) string { f := m.View.Effective switch field { case "is_correct": if value := map[string]string{"correct": "是", "incorrect": "否", "pending": "待确认"}[f.Correctness]; value != "" { return value } case "product_category": if f.ProductCategory != "" { return f.ProductCategory } case "problem_type": if f.ProblemType != "" { return f.ProblemType } case "root_cause": if f.Correctness == "correct" { return "" } if f.RootCause != "" { return f.RootCause } } return m.Fields[field] } func needs(m recordMeta, kind string) []string { if kind == "translation" { if strings.TrimSpace(m.Fields["question_chinese"]) != "" || strings.TrimSpace(m.Question) == "" || qaopsfields.DetectLanguage(m.Question) == "zh" { return nil } for _, r := range qaopsfields.NaturalText(m.Question) { if unicode.IsLetter(r) { return []string{"question_chinese"} } } return nil } if (m.Status != "succeeded" && m.Status != "historical_import") || m.Coverage.State != "checked" || strings.TrimSpace(m.Question) == "" { return nil } if m.View.Effective.Coverage == "covered" && fieldValue(m, "is_correct") == "待确认" { return []string{"is_correct"} } fields := []string{} for _, field := range []string{"is_correct", "product_category", "problem_type", "root_cause"} { if field == "root_cause" && (m.View.Effective.Correctness == "correct" || m.RootHandled) { continue } if strings.TrimSpace(fieldValue(m, field)) == "" { fields = append(fields, field) } } return fields } func (s *Store) Preview(ctx context.Context, scope Scope, actor string) (Preview, error) { if !s.valid() || actor == "" || (scope.Kind != "evaluation" && scope.Kind != "translation") || len(scope.IDs) > 100000 { return Preview{}, ErrInvalid } seen := map[string]bool{} for _, id := range scope.IDs { if strings.TrimSpace(id) == "" || len(id) > 256 || strings.ContainsRune(id, 0) || !utf8.ValidString(id) || seen[id] { return Preview{}, ErrInvalid } seen[id] = true } tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.RepeatableRead}) if err != nil { return Preview{}, ErrUnavailable } defer tx.Rollback(context.Background()) cfg, err := s.readSettings(ctx, tx) if err != nil { return Preview{}, err } if cfg.Verification != "verified" || !cfg.KeyConfigured || !cfg.StorageReady { return Preview{}, ErrUnavailable } metas, err := s.readMetas(ctx, tx, scope.IDs, &scope) if err != nil { return Preview{}, err } if len(scope.IDs) > 0 && len(metas) != len(scope.IDs) { return Preview{}, ErrInvalid } eligible := []string{} for _, m := range metas { if len(needs(m, scope.Kind)) > 0 { eligible = append(eligible, m.ID) } } id, err := newID("qaip-") if err != nil { return Preview{}, err } ids, _ := json.Marshal(eligible) rawScope, _ := json.Marshal(scope) expires := time.Now().UTC().Add(10 * time.Minute) _, err = tx.Exec(ctx, `INSERT INTO qa_ops_ai_previews(tenant_key,id,kind,record_ids,scope,scope_total,config_revision,actor,expires_at)VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9)`, s.tenant, id, scope.Kind, ids, rawScope, len(metas), cfg.Revision, actor, expires) if err != nil { return Preview{}, ErrUnavailable } if tx.Commit(ctx) != nil { return Preview{}, ErrUnavailable } return Preview{ID: id, Kind: scope.Kind, ScopeTotal: len(metas), Eligible: len(eligible), Skipped: len(metas) - len(eligible), ConfigRevision: cfg.Revision, ExpiresAt: expires}, nil } func (s *Store) enqueueMeta(ctx context.Context, tx pgx.Tx, m recordMeta, kind, mode string, revision int64) (string, error) { id, err := newID("qaiw-") if err != nil { return "", err } err = tx.QueryRow(ctx, `INSERT INTO qa_ops_ai_work(tenant_key,id,record_id,kind,trigger_mode,input_hash,master_revision,review_revision,config_revision,prompt_version,dictionary_version)VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11) ON CONFLICT(tenant_key,record_id,kind,input_hash,master_revision,review_revision,config_revision,prompt_version,dictionary_version) DO UPDATE SET trigger_mode=CASE WHEN EXCLUDED.trigger_mode='manual' THEN 'manual' ELSE qa_ops_ai_work.trigger_mode END,state=CASE WHEN EXCLUDED.trigger_mode='manual' AND qa_ops_ai_work.state='paused' THEN 'queued' ELSE qa_ops_ai_work.state END RETURNING id`, s.tenant, id, m.ID, kind, mode, workInputHash(kind, m), m.Revision, m.View.Revision, revision, promptVersion(kind), DictionaryVersion).Scan(&id) if err != nil { return "", ErrUnavailable } return id, nil } func (s *Store) Start(ctx context.Context, previewID, nonce, actor string) (Batch, error) { if !s.valid() || actor == "" || len(nonce) < 16 || len(nonce) > 128 || strings.ContainsRune(nonce, 0) { return Batch{}, ErrInvalid } tx, err := s.pool.Begin(ctx) if err != nil { return Batch{}, ErrUnavailable } defer tx.Rollback(context.Background()) if _, err = tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1,56))`, s.tenant+":"+actor+":"+nonce); err != nil { return Batch{}, ErrUnavailable } var priorID, priorPreview string err = tx.QueryRow(ctx, `SELECT id,preview_id FROM qa_ops_ai_batches WHERE tenant_key=$1 AND actor=$2 AND nonce=$3`, s.tenant, actor, nonce).Scan(&priorID, &priorPreview) if err == nil { if priorPreview != previewID { return Batch{}, ErrConflict } _ = tx.Rollback(ctx) return s.GetBatch(ctx, priorID) } if err != pgx.ErrNoRows { return Batch{}, ErrUnavailable } var ids []string var rawIDs []byte var kind, owner string var revision int64 var expires time.Time err = tx.QueryRow(ctx, `SELECT kind,record_ids,config_revision,actor,expires_at FROM qa_ops_ai_previews WHERE tenant_key=$1 AND id=$2 FOR UPDATE`, s.tenant, previewID).Scan(&kind, &rawIDs, &revision, &owner, &expires) if err != nil || json.Unmarshal(rawIDs, &ids) != nil || owner != actor || !time.Now().Before(expires) { return Batch{}, ErrConflict } cfg, err := s.readSettings(ctx, tx) if err != nil { return Batch{}, err } if cfg.Revision != revision || cfg.Verification != "verified" || !cfg.KeyConfigured || !cfg.StorageReady { return Batch{}, ErrConflict } metas, err := s.readMetas(ctx, tx, ids, nil) if err != nil { return Batch{}, err } if len(metas) != len(ids) { return Batch{}, ErrConflict } id, err := newID("qaib-") if err != nil { return Batch{}, err } _, err = tx.Exec(ctx, `INSERT INTO qa_ops_ai_batches(tenant_key,id,kind,preview_id,nonce,actor)VALUES($1,$2,$3,$4,$5,$6)`, s.tenant, id, kind, previewID, nonce, actor) if err != nil { return Batch{}, ErrUnavailable } for _, m := range metas { workID, e := s.enqueueMeta(ctx, tx, m, kind, "manual", revision) if e != nil { return Batch{}, e } if _, e = tx.Exec(ctx, `INSERT INTO qa_ops_ai_batch_items(tenant_key,batch_id,work_id)VALUES($1,$2,$3)`, s.tenant, id, workID); e != nil { return Batch{}, ErrUnavailable } } if tx.Commit(ctx) != nil { return Batch{}, ErrUnavailable } return s.GetBatch(ctx, id) } const batchSQL = `SELECT b.id,b.kind,b.state,b.created_at,count(w.id),count(*) FILTER(WHERE w.state='succeeded'),count(*) FILTER(WHERE w.state='failed'),count(*) FILTER(WHERE w.state='skipped'),count(*) FILTER(WHERE w.state='running'),count(*) FILTER(WHERE w.state='queued'),count(*) FILTER(WHERE w.state='retry'),count(*) FILTER(WHERE w.state='paused') FROM qa_ops_ai_batches b LEFT JOIN qa_ops_ai_batch_items i ON i.tenant_key=b.tenant_key AND i.batch_id=b.id LEFT JOIN qa_ops_ai_work w ON w.tenant_key=i.tenant_key AND w.id=i.work_id WHERE b.tenant_key=$1 AND b.id=$2 GROUP BY b.id,b.kind,b.state,b.created_at` func (s *Store) GetBatch(ctx context.Context, id string) (Batch, error) { if !s.valid() { return Batch{}, ErrUnavailable } var out Batch err := s.pool.QueryRow(ctx, batchSQL, s.tenant, id).Scan(&out.ID, &out.Kind, &out.State, &out.CreatedAt, &out.Total, &out.Succeeded, &out.Failed, &out.Skipped, &out.Running, &out.Queued, &out.Retry, &out.Paused) if err != nil { return out, ErrUnavailable } if out.Total != out.Succeeded+out.Failed+out.Skipped+out.Running+out.Queued+out.Retry+out.Paused { return out, fmt.Errorf("%w: inconsistent work counters", ErrUnavailable) } return out, nil } func (s *Store) ProcessOne(ctx context.Context) (bool, error) { work, err := s.claim(ctx) if err != nil || work == nil { return false, err } return true, s.execute(ctx, *work) }