Files
knowledge-governance/internal/qaopsmirror/dashboard_store.go
T
姚章浩 00c4c13283
build / build (push) Successful in 1m19s
fix(qa): 首批即显并统一五字段后端筛选
列表正文改为按需读取;保留完整导出、版本隔离和有效标签筛选。
验证:前端全量、Go全量及隔离PG全mirror竞态通过;覆盖首批回调、续载失败与筛选分页。
2026-10-25 12:45:31 +08:00

331 lines
12 KiB
Go

package qaopsmirror
import (
"context"
"encoding/json"
"github.com/jackc/pgx/v5"
"knowledge-governance/internal/dobotaiops"
"knowledge-governance/internal/qaopsorg"
"knowledge-governance/internal/qaopsquality"
"sort"
"strings"
"time"
"unicode/utf8"
)
type dashboardRecord struct {
id, hash string
at time.Time
identity dobotaiops.ActorIdentity
fields map[string]string
edits map[string]string
coverage dobotaiops.CoverageCheck
feedback dobotaiops.RecordFeedback
}
func dashboardRate(n, d int) qaopsquality.Rate {
r := qaopsquality.Rate{Numerator: n, Denominator: d}
if d > 0 {
p := 100 * float64(n) / float64(d)
r.Percent = &p
}
return r
}
func dashboardCategories(counts map[string]int) []DashboardCount {
out := []DashboardCount{}
for label, count := range counts {
out = append(out, DashboardCount{Label: label, Count: count})
}
sort.Slice(out, func(i, j int) bool {
if out[i].Count == out[j].Count {
return out[i].Label < out[j].Label
}
return out[i].Count > out[j].Count
})
return out
}
func dashboardCategory(value string) string {
if value = strings.TrimSpace(value); value != "" {
return value
}
return "未分类"
}
// ReadDashboard never fetches answer payloads or opens a source connection. All
// scopes, quality inputs and organizational attribution use ONE local snapshot.
func (s *Store) ReadDashboard(ctx context.Context, q dobotaiops.RecordQuery) (Dashboard, error) {
return s.readDashboard(ctx, q, false)
}
func (s *Store) readDashboard(ctx context.Context, q dobotaiops.RecordQuery, master bool) (Dashboard, error) {
out := Dashboard{Start: q.Window.Start, End: q.Window.End, Keyword: q.Keyword, IssueTypes: []DashboardCount{}, Products: []DashboardCount{}, Daily: []DashboardDaily{}, Regions: []DashboardRegion{}, FrozenWeeks: []DashboardFrozen{}}
if !s.valid() {
return out, dobotaiops.ErrUnavailable
}
if q.Window.TenantKey != s.tenant || !utf8.ValidString(q.Keyword) || utf8.RuneCountInString(q.Keyword) > 200 || strings.ContainsRune(q.Keyword, 0) {
return out, dobotaiops.ErrInvalidQuery
}
if _, e := dobotaiops.NewWindow(s.tenant, q.Window.Start.Format(time.RFC3339Nano), q.Window.End.Format(time.RFC3339Nano), 90*24*time.Hour, time.Now()); e != nil {
return out, e
}
ctx, cancel := context.WithTimeout(ctx, 20*time.Second)
defer cancel()
if !dobotaiops.ValidQualityFilters(q.Filters) {
return out, dobotaiops.ErrInvalidQuery
}
if len(q.Filters) > 0 {
if !master {
return out, dobotaiops.ErrInvalidQuery
}
var err error
q, err = s.resolveMasterFilters(ctx, q)
if err != nil {
return out, err
}
out.Filters = q.Filters
}
tx, e := s.pool.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.RepeatableRead, AccessMode: pgx.ReadOnly})
if e != nil {
return out, dobotaiops.ErrUnavailable
}
defer func() { _ = tx.Rollback(context.Background()) }()
var state Status
if master {
state.Ready = true
var observed time.Time
e = tx.QueryRow(ctx, `SELECT COALESCE((SELECT revision FROM qa_ops_master_state WHERE tenant_key=$1),0),COALESCE((SELECT updated_at FROM qa_ops_master_state WHERE tenant_key=$1),transaction_timestamp())`, s.tenant).Scan(&state.Revision, &observed)
state.SourceObservedAt = &observed
} else {
state, e = s.readState(ctx, txQuery{tx})
}
if e != nil {
return out, e
}
if state.Blocked {
return out, dobotaiops.ErrNotReadOnly
}
if !state.Ready || state.SourceObservedAt == nil {
return out, dobotaiops.ErrUnavailable
}
out.AsOf = *state.SourceObservedAt
out.LocalRevision = state.Revision
attribution, e := qaopsorg.NewStore(s.pool, s.tenant).GetAttributionInTx(ctx, tx)
if e != nil {
return out, e
}
meta := attribution.Metadata
out.Organization = dobotaiops.OrganizationVersion{State: meta.State, DirectoryID: meta.DirectoryID, DirectorySource: meta.DirectorySource, SyncedAt: meta.DirectorySyncedAt, RevisionID: meta.RevisionID, Basis: "current_directory", RegionNames: meta.RegionLabels()}
out.MonthStart, out.WeekStart = dashboardPeriods(q.Window.End)
var first *time.Time
totalSQL := `SELECT count(*),LEAST((SELECT min(asked_at) FROM qa_ops_mirror_records WHERE tenant_key=$1 AND asked_at<$2),(SELECT first_data_at FROM qa_ops_weekly_quality_tracking WHERE tenant_key=$1),(SELECT min(week_start) FROM qa_ops_weekly_quality_snapshots WHERE tenant_key=$1 AND week_start<$2)) FROM qa_ops_mirror_records WHERE tenant_key=$1 AND asked_at<$2 AND ($3='' OR position(lower($3) IN lower(question))>0)`
if master {
totalSQL = strings.ReplaceAll(totalSQL, "qa_ops_mirror_records", "qa_ops_master_read_model")
totalSQL = strings.ReplaceAll(totalSQL, "lower(question)", "lower(question||E'\\n'||chinese_question||E'\\n'||COALESCE(ai_fields->>'question_chinese',''))")
}
totalArgs := []any{s.tenant, q.Window.End, q.Keyword}
if q.MatchIDs != nil {
totalSQL += " AND id=ANY($4::text[])"
totalArgs = append(totalArgs, q.MatchIDs)
}
if e = tx.QueryRow(ctx, totalSQL, totalArgs...).Scan(&out.TotalIssues, &first); e != nil {
return out, dobotaiops.ErrUnavailable
}
lower := q.Window.Start
if out.MonthStart.Before(lower) {
lower = out.MonthStart
}
if out.WeekStart.Before(lower) {
lower = out.WeekStart
}
rowsSQL := `SELECT id,asked_at,identity,quality_hash FROM qa_ops_mirror_records WHERE tenant_key=$1 AND asked_at>=$2 AND asked_at<$3 AND ($4='' OR position(lower($4) IN lower(question))>0) ORDER BY id LIMIT 100001`
if master {
rowsSQL = strings.ReplaceAll(rowsSQL, "qa_ops_mirror_records", "qa_ops_master_read_model")
rowsSQL = strings.ReplaceAll(rowsSQL, "SELECT id,asked_at,identity,quality_hash FROM", "SELECT id,asked_at,identity,quality_hash,effective_fields,coverage_result,inline_fields,current_feedback FROM")
rowsSQL = strings.ReplaceAll(rowsSQL, "lower(question)", "lower(question||E'\\n'||chinese_question||E'\\n'||COALESCE(ai_fields->>'question_chinese',''))")
}
rowsArgs := []any{s.tenant, lower, q.Window.End, q.Keyword}
if q.MatchIDs != nil {
rowsSQL = strings.Replace(rowsSQL, " ORDER BY id LIMIT", " AND id=ANY($5::text[]) ORDER BY id LIMIT", 1)
rowsArgs = append(rowsArgs, q.MatchIDs)
}
rows, e := tx.Query(ctx, rowsSQL, rowsArgs...)
if e != nil {
return out, dobotaiops.ErrUnavailable
}
records := []dashboardRecord{}
keys := []dobotaiops.QualityKey{}
for rows.Next() {
var r dashboardRecord
var identity []byte
var scanErr error
if master {
var fields, coverage, edits, feedback []byte
scanErr = rows.Scan(&r.id, &r.at, &identity, &r.hash, &fields, &coverage, &edits, &feedback)
if scanErr == nil {
scanErr = json.Unmarshal(fields, &r.fields)
if scanErr == nil {
scanErr = json.Unmarshal(edits, &r.edits)
}
}
if scanErr == nil {
scanErr = json.Unmarshal(coverage, &r.coverage)
}
if scanErr == nil {
scanErr = json.Unmarshal(feedback, &r.feedback)
}
} else {
scanErr = rows.Scan(&r.id, &r.at, &identity, &r.hash)
}
if scanErr != nil || json.Unmarshal(identity, &r.identity) != nil {
rows.Close()
return out, dobotaiops.ErrUnavailable
}
records = append(records, r)
if len(records) > dobotaiops.MaxMirrorRecords {
rows.Close()
return out, dobotaiops.ErrInvalidQuery
}
if !r.at.Before(q.Window.Start) {
keys = append(keys, dobotaiops.QualityKey{ID: r.id, InputHash: r.hash})
}
}
e = rows.Err()
rows.Close()
if e != nil {
return out, dobotaiops.ErrUnavailable
}
views := map[string]qaopsquality.View{}
qualityTenant := s.tenant
if master {
qualityTenant += "::master"
}
quality := qaopsquality.NewStore(s.pool, qualityTenant)
for begin := 0; begin < len(keys); begin += 5000 {
end := begin + 5000
if end > len(keys) {
end = len(keys)
}
batch, err := quality.ReadViewsInTx(ctx, tx, keys[begin:end])
if err != nil {
return out, err
}
for id, v := range batch {
views[id] = v
}
}
if master {
for _, r := range records {
if v, ok := views[r.id]; ok {
views[r.id] = qaopsquality.ProjectMasterWithEdits(v, r.fields, r.coverage, r.edits, r.feedback)
}
}
}
out.Quality = qaopsquality.Summarize(views)
products, types, daily := map[string]int{}, map[string]int{}, map[string]int{}
for day := out.WeekStart; day.Before(q.Window.End); day = day.AddDate(0, 0, 1) {
daily[day.Format("2006-01-02")] = 0
}
regionIndex := map[string]int{}
users := map[string]map[string]bool{}
for _, region := range meta.RegionDefinitions() {
r := DashboardRegion{ID: region.ID, Name: region.Name}
if meta.State == "ready" {
n := meta.View.Regions[region.ID]
r.RosterPeople = &n
r.Users = new(int)
}
regionIndex[region.Name] = len(out.Regions)
users[region.Name] = map[string]bool{}
out.Regions = append(out.Regions, r)
}
for _, r := range records {
if !r.at.Before(out.WeekStart) {
out.NewThisWeek++
daily[r.at.In(dashboardLocation).Format("2006-01-02")]++
}
if !r.at.Before(q.Window.Start) {
v, ok := views[r.id]
if !ok {
return out, qaopsquality.ErrUnavailable
}
products[dashboardCategory(v.Effective.ProductCategory)]++
types[dashboardCategory(v.Effective.ProblemType)]++
}
if !r.at.Before(out.MonthStart) {
person := attribution.Resolve(r.identity.StaffID, r.identity.UnionID, r.fields["asker"], r.fields["trigger_person"])
if index, ok := regionIndex[person.Region]; ok && person.PersonID != "" {
out.Regions[index].Questions++
users[person.Region][person.PersonID] = true
}
}
}
out.Products = dashboardCategories(products)
out.IssueTypes = dashboardCategories(types)
for day := out.WeekStart; day.Before(q.Window.End); day = day.AddDate(0, 0, 1) {
date := day.Format("2006-01-02")
out.Daily = append(out.Daily, DashboardDaily{Date: date, Count: daily[date]})
}
for i := range out.Regions {
r := &out.Regions[i]
if r.Users != nil {
*r.Users = len(users[r.Name])
if *r.Users > *r.RosterPeople {
return out, dobotaiops.ErrUnavailable
}
r.UsageCoverage = dashboardRate(*r.Users, *r.RosterPeople)
if *r.RosterPeople > 0 {
frequency := float64(r.Questions) / float64(*r.RosterPeople)
r.Frequency = &frequency
}
}
}
if first != nil && len(q.Filters) == 0 {
snapshots, err := s.readFrozenDashboard(ctx, tx, q.Keyword, q.Window.End)
if err != nil {
return out, err
}
out.FrozenWeeks = dashboardFrozenSeries(*first, q.Window.End, snapshots)
if len(out.FrozenWeeks) > 0 && out.FrozenWeeks[len(out.FrozenWeeks)-1].WeekEnd.Before(q.Window.End) {
return out, dobotaiops.ErrInvalidQuery
}
}
if master {
timeline, calculated, err := s.readMasterQualityTimeline(ctx, tx, q.Keyword, q.Window.End)
if err != nil {
return out, err
}
if len(q.Filters) > 0 {
filtered := make([]dashboardTimedQuality, 0)
for _, row := range timeline {
if matchesMasterFilters(row.view, q.Filters) {
filtered = append(filtered, row)
}
}
timeline = filtered
}
out.CumulativeWeeks = dashboardCumulativeFromTimeline(timeline, q.Window.End, calculated)
month, cutoff := dashboardQualityMonth(out.WeekStart, q.Window.End, calculated)
out.QualityPeriod = &DashboardQualityPeriod{Month: month.Format("2006-01"), Start: month, CutoffAt: cutoff, CalculatedAt: calculated, Source: dashboardMonthlySource, PolicyVersion: dashboardMonthlyPolicy}
rates := dashboardStrictQuality(timeline, month, cutoff)
out.MonthlyQuality = &rates
out.CalendarMonths = dashboardCalendarMonths(timeline, cutoff, calculated)
} else {
out.CumulativeWeeks = []DashboardCumulative{}
out.CalendarMonths = []DashboardMonth{}
}
if len(q.Filters) == 0 {
out.ReportedWeeks, e = s.readReportedDashboard(ctx, tx, q.Keyword, q.Window.End)
} else {
out.ReportedWeeks = []DashboardReported{}
}
if e != nil {
return out, e
}
if e = s.masterFiltersStillCurrent(ctx, q); e != nil {
return out, e
}
if tx.Commit(ctx) != nil {
return out, dobotaiops.ErrUnavailable
}
return out, nil
}