Files
knowledge-governance/internal/dobotaiops/records.go
T
姚章浩 00c4c13283
build / build (push) Successful in 1m19s
fix(qa): 首批即显并统一五字段后端筛选
列表正文改为按需读取;保留完整导出、版本隔离和有效标签筛选。
验证:前端全量、Go全量及隔离PG全mirror竞态通过;覆盖首批回调、续载失败与筛选分页。
2026-10-25 12:45:31 +08:00

336 lines
13 KiB
Go

package dobotaiops
import (
"context"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"strings"
"time"
"unicode/utf8"
"github.com/jackc/pgx/v5"
)
const CoverageRuleVersion = "governance-answer-substring-v1"
const noPublishedMarker = "暂无已发布"
// The master follows the explicitly approved v2 policy; legacy/source checks stay immutable.
func CheckMasterCoverage(answer string, available bool) CoverageCheck {
result := CoverageCheck{State: "pending", RuleVersion: "master-answer-substring-v2"}
if !available || strings.TrimSpace(answer) == "" {
return result
}
result.State = "checked"
result.Verdict = "covered"
if strings.Contains(answer, noPublishedMarker) {
result.Verdict = "uncovered"
result.MatchedText = noPublishedMarker
}
return result
}
var ErrRecordNotFound = errors.New("qa record not found")
type CoverageCheck struct {
Verdict string `json:"verdict,omitempty"`
State string `json:"state"`
RuleVersion string `json:"rule_version"`
MatchedText string `json:"matched_text,omitempty"`
}
// Absence of this exact substring is not evidence that knowledge covers a question.
func CheckAnswerCoverage(answer string) CoverageCheck {
return CheckMarkerCoverage(strings.Contains(answer, noPublishedMarker))
}
func CheckMarkerCoverage(matched bool) CoverageCheck {
result := CoverageCheck{State: "pending", RuleVersion: CoverageRuleVersion}
if matched {
result.State = "checked"
result.Verdict = "uncovered"
result.MatchedText = noPublishedMarker
}
return result
}
type SourceCoverage struct {
Verdict string `json:"verdict"`
ReasonCode string `json:"reason_code"`
EvaluatorVersion string `json:"evaluator_version"`
}
type RecordFeedback struct {
AskerValue string `json:"asker_value"`
Up int64 `json:"up"`
Down int64 `json:"down"`
}
type Record struct {
ContentProjection string `json:"content_projection,omitempty"`
FieldEditRevision int64 `json:"field_edit_revision,omitempty"`
BackgroundTasks json.RawMessage `json:"background_tasks,omitempty"`
RecordOrigin string `json:"record_origin,omitempty"`
MasterFields map[string]string `json:"master_fields,omitempty"`
QualityObservedAt time.Time `json:"-"`
QualityInputHash string `json:"-"`
Quality json.RawMessage `json:"quality,omitempty"`
Identity ActorIdentity `json:"-"`
Organization *RecordOrganization `json:"organization,omitempty"`
ID string `json:"id"`
Status string `json:"status"`
Question string `json:"question"`
Answer string `json:"answer"`
AskedAt time.Time `json:"asked_at"`
TimeSource string `json:"time_source"`
Asker string `json:"asker"`
TriggerPerson string `json:"trigger_person"`
Department string `json:"department"`
Region string `json:"region"`
References json.RawMessage `json:"references"`
Media json.RawMessage `json:"media"`
SourceCoverage SourceCoverage `json:"source_coverage"`
Feedback RecordFeedback `json:"feedback"`
Coverage CoverageCheck `json:"coverage"`
}
type RecordQuery struct {
Summary bool
Filters map[string]string
MatchIDs []string
MatchVersion string
Window Window
Keyword string
Cursor string
Limit int
}
type QualityKey struct {
ObservedAt time.Time `json:"-"`
Marker bool `json:"marker"`
Source SourceCoverage `json:"source"`
ID string `json:"id"`
InputHash string `json:"input_hash"`
}
type RecordPage struct {
LocalRevision int64 `json:"local_revision,omitempty"`
ScopeQuality []QualityKey `json:"-"`
ScopeActors []ActorIdentity `json:"-"`
Organization *OrganizationVersion `json:"organization,omitempty"`
Items []Record `json:"items"`
Columns []string `json:"columns"`
Total int64 `json:"total"`
HasMore bool `json:"has_more"`
NextCursor string `json:"next_cursor"`
AsOf time.Time `json:"as_of"`
}
type recordCursor struct {
AskedAt time.Time `json:"at"`
ID string `json:"id"`
Scope string `json:"scope"`
}
// A single inbound-message join controls names/time. Feedback is preaggregated;
// joining each feedback row directly would multiply the business event count.
const actorIdentitySQL = `jsonb_build_object('actor_id',COALESCE(a.id,''),'staff_id',COALESCE(a.external_actor_id,''),'union_id',COALESCE(m.actor_union_id_snapshot,''))`
const qualityHashSQL = `encode(sha256(convert_to(jsonb_build_array('quality-input-v1',t.id,t.status,t.question,t.answer,t.references_json,COALESCE(m.media_json,'[]'::jsonb))::text,'UTF8')),'hex')`
const recordsBaseSQL = `
WITH records AS (
SELECT t.id,COALESCE(m.created_at,t.created_at) AS asked_at,t.question,t.status,` + actorIdentitySQL + ` AS identity,` + qualityHashSQL + ` AS quality_hash,
jsonb_build_object(
'_identity',` + actorIdentitySQL + `, '_quality_hash',` + qualityHashSQL + `,'_quality_observed_at',transaction_timestamp(),
'id',t.id,'status',t.status,'question',t.question,'answer',t.answer,
'asked_at',COALESCE(m.created_at,t.created_at),
'time_source',CASE WHEN m.id IS NULL THEN 'turn.created_at' ELSE 'request_message.created_at' END,
'asker',COALESCE(NULLIF(m.actor_name_snapshot,''),a.display_name,''),
'trigger_person',COALESCE(m.trigger_person_snapshot,''),
'department',COALESCE(m.department_snapshot,''),'region',COALESCE(m.region_snapshot,''),
'references',t.references_json,'media',COALESCE(m.media_json,'[]'::jsonb),
'source_coverage',jsonb_build_object('verdict',COALESCE(c.verdict,''),'reason_code',COALESCE(c.reason_code,''),'evaluator_version',COALESCE(c.evaluator_version,'')),
'feedback',jsonb_build_object('asker_value',COALESCE(f.asker_value,''),'up',f.up_count,'down',f.down_count)
) AS payload
FROM public.turns t
LEFT JOIN public.messages m ON m.id=t.request_message_id AND m.tenant_key=t.tenant_key AND m.channel=t.channel AND m.direction='inbound'
LEFT JOIN public.channel_actors a ON a.id=m.actor_id AND a.tenant_key=t.tenant_key AND a.channel=t.channel
LEFT JOIN LATERAL (
SELECT verdict,reason_code,evaluator_version FROM public.coverage_decisions c
WHERE c.turn_id=t.id AND c.tenant_key=t.tenant_key AND c.channel=t.channel
ORDER BY c.created_at DESC,c.evaluator_version DESC LIMIT 1
) c ON true
LEFT JOIN LATERAL (
SELECT count(*) FILTER(WHERE value='up') AS up_count,count(*) FILTER(WHERE value='down') AS down_count,
max(value) FILTER(WHERE actor_id=m.actor_id) AS asker_value
FROM public.answer_feedback f
WHERE f.turn_id=t.id AND f.answer_message_id=t.answer_message_id AND f.tenant_key=t.tenant_key AND f.channel=t.channel
) f ON true
WHERE t.tenant_key=$1 AND t.channel='dingtalk'
)`
const recordFilterSQL = ` asked_at >= $2 AND asked_at < $3 AND ($4='' OR question ILIKE $4 ESCAPE '\')`
// Availability is over the entire filtered result, not just the current page.
const recordColumnsSQL = `SELECT count(*),jsonb_build_object(
'answer',COALESCE(bool_or(payload->>'answer'<>''),false),
'coverage',COALESCE(bool_or(strpos(payload->>'answer',$5)>0),false),
'asked_at',count(*)>0,
'asker',COALESCE(bool_or(payload->>'asker'<>''),false),
'question',COALESCE(bool_or(question<>''),false),
'trigger_person',COALESCE(bool_or(payload->>'trigger_person'<>''),false),
'department',COALESCE(bool_or(payload->>'department'<>''),false),
'region',COALESCE(bool_or(payload->>'region'<>''),false),
'feedback',COALESCE(bool_or(payload->'feedback'->>'asker_value'<>'' OR
(payload->'feedback'->>'up')::bigint>0 OR (payload->'feedback'->>'down')::bigint>0),false)
)::text,COALESCE(jsonb_agg(DISTINCT identity),'[]'::jsonb)::text,COALESCE(jsonb_agg(jsonb_build_object('id',id,'input_hash',quality_hash,'marker',strpos(payload->>'answer',$5)>0,'source',payload->'source_coverage')),'[]'::jsonb)::text FROM records WHERE `
var recordColumnOrder = []string{"question_chinese", "answer", "coverage", "is_correct", "product_category", "problem_type", "root_cause", "asked_at", "asker", "question", "language", "trigger_person", "department", "region", "feedback"}
func recordKeywordPattern(keyword string) (string, error) {
if !utf8.ValidString(keyword) || utf8.RuneCountInString(keyword) > 200 || strings.ContainsRune(keyword, 0) {
return "", ErrInvalidQuery
}
if keyword == "" {
return "", nil
}
return "%" + strings.NewReplacer(`\`, `\\`, `%`, `\%`, `_`, `\_`).Replace(keyword) + "%", nil
}
func (c *RecordsRepository) ListRecords(ctx context.Context, query RecordQuery) (RecordPage, error) {
if err := c.validateQueryWindow(query.Window); err != nil {
return RecordPage{}, err
}
if query.Limit < 1 || query.Limit > 100 || len(query.Cursor) > 2048 {
return RecordPage{}, fmt.Errorf("%w: invalid record filters", ErrInvalidQuery)
}
scopeBytes, _ := json.Marshal([]string{query.Window.TenantKey, query.Window.Start.UTC().Format(time.RFC3339Nano), query.Window.End.UTC().Format(time.RFC3339Nano), query.Keyword})
if len(query.Filters) > 0 {
if query.MatchIDs == nil {
return RecordPage{}, ErrInvalidQuery
}
extra, _ := json.Marshal(struct {
Filters map[string]string
Version string
}{query.Filters, query.MatchVersion})
scopeBytes = append(scopeBytes, extra...)
}
scopeSum := sha256.Sum256(scopeBytes)
scope := hex.EncodeToString(scopeSum[:])
var cursor recordCursor
var before any
if query.Cursor != "" {
decoded, err := base64.RawURLEncoding.DecodeString(query.Cursor)
if err != nil || json.Unmarshal(decoded, &cursor) != nil || cursor.Scope != scope || cursor.ID == "" || len(cursor.ID) > 256 || cursor.AskedAt.Before(query.Window.Start) || !cursor.AskedAt.Before(query.Window.End) {
return RecordPage{}, fmt.Errorf("%w: invalid or mismatched cursor", ErrInvalidQuery)
}
before = cursor.AskedAt
}
pattern, patternErr := recordKeywordPattern(query.Keyword)
if patternErr != nil {
return RecordPage{}, patternErr
}
args := []any{query.Window.TenantKey, query.Window.Start, query.Window.End, pattern}
page := RecordPage{Items: make([]Record, 0), Columns: make([]string, 0)}
err := c.transact(ctx, func(ctx context.Context, q querier) error {
if err := c.readAsOf(ctx, q, &page.AsOf); err != nil {
return ErrUnavailable
}
var rawColumns, rawActors, rawQuality string
summarySQL := c.baseSQL + recordColumnsSQL
if c.summarySQL != "" {
summarySQL = c.summarySQL
}
summaryFilter := recordFilterSQL
summaryArgs := append(append([]any{}, args...), noPublishedMarker)
if query.MatchIDs != nil {
summaryFilter += " AND id=ANY($6::text[])"
summaryArgs = append(summaryArgs, query.MatchIDs)
}
if err := q.QueryRow(ctx, summarySQL+summaryFilter, summaryArgs...).Scan(&page.Total, &rawColumns, &rawActors, &rawQuality); err != nil {
return fmt.Errorf("%w: record count failed", ErrUnavailable)
}
if json.Unmarshal([]byte(rawQuality), &page.ScopeQuality) != nil {
return ErrUnavailable
}
for i := range page.ScopeQuality {
page.ScopeQuality[i].ObservedAt = page.AsOf
}
var available map[string]bool
if json.Unmarshal([]byte(rawActors), &page.ScopeActors) != nil {
return ErrUnavailable
}
if err := json.Unmarshal([]byte(rawColumns), &available); err != nil {
return ErrUnavailable
}
for _, column := range recordColumnOrder {
if available[column] {
page.Columns = append(page.Columns, column)
}
}
listFilter := recordFilterSQL
listArgs := append(append([]any{}, args...), before, cursor.ID, query.Limit+1)
if query.MatchIDs != nil {
listFilter += " AND id=ANY($8::text[])"
listArgs = append(listArgs, query.MatchIDs)
}
rows, err := q.Query(ctx, c.baseSQL+` SELECT payload::text FROM records WHERE `+listFilter+`
AND ($5::timestamptz IS NULL OR (asked_at,id)<($5,$6)) ORDER BY asked_at DESC,id DESC LIMIT $7`, listArgs...)
if err != nil {
return fmt.Errorf("%w: record list failed", ErrUnavailable)
}
defer rows.Close()
for rows.Next() {
var raw string
if err := rows.Scan(&raw); err != nil {
return ErrUnavailable
}
item, err := decodeRecord(raw)
if err != nil {
return ErrUnavailable
}
page.Items = append(page.Items, item)
}
return rows.Err()
})
if err != nil {
return RecordPage{}, err
}
if len(page.Items) > query.Limit {
page.HasMore = true
page.Items = page.Items[:query.Limit]
last := page.Items[len(page.Items)-1]
raw, err := json.Marshal(recordCursor{AskedAt: last.AskedAt, ID: last.ID, Scope: scope})
if err != nil {
return RecordPage{}, ErrUnavailable
}
page.NextCursor = base64.RawURLEncoding.EncodeToString(raw)
}
return page, nil
}
func (c *RecordsRepository) GetRecord(ctx context.Context, tenantKey, id string) (Record, error) {
if strings.TrimSpace(tenantKey) == "" || (c.tenant != "" && tenantKey != c.tenant) || id == "" || len(id) > 256 || strings.ContainsRune(id, 0) {
return Record{}, ErrInvalidQuery
}
var item Record
err := c.transact(ctx, func(ctx context.Context, q querier) error {
var raw string
err := q.QueryRow(ctx, c.baseSQL+` SELECT payload::text FROM records WHERE id=$2`, tenantKey, id).Scan(&raw)
if errors.Is(err, pgx.ErrNoRows) {
return ErrRecordNotFound
}
if err != nil {
return ErrUnavailable
}
item, err = decodeRecord(raw)
if err != nil {
return ErrUnavailable
}
return nil
})
return item, err
}