feat(qa-ops): 持久本地问答副本与安全答案预览
build / build (push) Successful in 1m11s

新增迁移50与每5分钟/手动持久同步,API只读本地;同步失败保留、权限阻断、租约与质量失效原子激活。
保留全量稳定分页/固定列,已存质量批量读取,元数据不反复解压长答案;详情安全Markdown与原文。

已运行整仓 make verify、源与副本隔离PG -race、真实只读源及隔离浏览器。
实际同步2525条;固定窗口640条本地中位1.46秒,对照直接源33.10秒。
停worker、缺源凭据、重启API后仍可查询;未更新8088,未修改生产原问答。
This commit is contained in:
姚章浩
2026-10-18 19:10:47 +08:00
parent 74ba737b1e
commit 3e7fcce0b1
49 changed files with 4038 additions and 59 deletions
@@ -0,0 +1,45 @@
package httpapi
import (
"context"
"knowledge-governance/internal/dobotaiops"
"knowledge-governance/internal/qaopsquality"
"net/http"
"testing"
"time"
)
type mirroredQualitySource struct {
qualitySource
reads int
}
func (s *mirroredQualitySource) ReadStoredQuality(_ context.Context, keys []dobotaiops.QualityKey) (map[string]qaopsquality.View, error) {
s.reads++
out := map[string]qaopsquality.View{}
for _, key := range keys {
out[key.ID] = qaopsquality.Project(key.InputHash, 1, 0, qaopsquality.Check{Result: dobotaiops.CheckMarkerCoverage(true)}, nil)
}
return out, nil
}
func TestMirroredQualityQueriesDoNotRepeatObservationWrites(t *testing.T) {
source := &mirroredQualitySource{qualitySource: qualitySource{record: dobotaiops.Record{ID: "fixture-turn", Question: "fixture", Answer: "暂无已发布"}}}
e, _ := qaopsquality.EvidenceFor(source.record)
source.record.QualityInputHash = e.Hash
legacy := &qualityFake{fail: true}
auth := newFakeAuthStore()
auth.sessions["quality-admin"] = "usr-boss"
h, e2 := NewHandler(Config{Store: &fakeJobStore{}, Auth: &AuthConfig{Store: auth}, QAOps: source, QAOpsTenantKey: "fixed", QAQuality: legacy, Now: func() time.Time { return time.Date(2026, 10, 17, 0, 0, 0, 0, time.UTC) }})
if e2 != nil {
t.Fatal(e2)
}
for _, q := range []struct{ method, path, body string }{{"GET", "/api/v1/qa-ops/records/fixture-turn", ""}, {"GET", "/api/v1/qa-ops/records?start=2026-10-14T00:00:00Z&end=2026-10-15T00:00:00Z", ""}, {"POST", "/api/v1/qa-ops/quality/query", `{"start":"2026-10-14T00:00:00Z","end":"2026-10-15T00:00:00Z","keyword":""}`}} {
w := qualityRequest(h, q.method, q.path, "quality-admin", q.body)
if w.Code != http.StatusOK {
t.Fatal("local mirror read went back through observation writes", q.path, w.Code)
}
}
if source.reads != 3 || legacy.calls != 0 {
t.Fatal("batched stored projection not used", source.reads, legacy.calls)
}
}
+79
View File
@@ -0,0 +1,79 @@
package httpapi
import (
"context"
"encoding/json"
"errors"
"io"
"knowledge-governance/internal/qaopsmirror"
"net/http"
"strings"
)
type qaMirrorSyncStore interface {
GetSyncStatus(context.Context) (qaopsmirror.Status, error)
EnqueueSync(context.Context, string, string) (qaopsmirror.SyncReceipt, bool, error)
}
func qaRecordsSync(w http.ResponseWriter, r *http.Request, c Config) {
if !authorizeQAOps(w, r) {
return
}
if r.URL.RawQuery != "" {
writeJSON(w, 400, map[string]string{"error": "同步入口不接受查询参数"})
return
}
store, ok := c.QAOps.(qaMirrorSyncStore)
if !ok || !c.QAOpsEnabled {
writeJSON(w, 503, map[string]string{"error": "本地问答同步未接线,未创建任务"})
return
}
if r.Method == http.MethodGet {
state, e := store.GetSyncStatus(r.Context())
if e != nil {
qaOpsError(w, e)
return
}
writeJSON(w, 200, map[string]any{"data": state})
return
}
if !authorizeQAIntent(w, r, "qa-records-sync") {
return
}
key := r.Header.Get("Idempotency-Key")
if key == "" || len(key) > 128 || strings.TrimSpace(key) != key {
writeJSON(w, 400, map[string]string{"error": "同步请求需要有效 Idempotency-Key,重试须复用原标识"})
return
}
var body *struct{}
d := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1024))
d.DisallowUnknownFields()
if d.Decode(&body) != nil || body == nil || d.Decode(&struct{}{}) != io.EOF {
writeJSON(w, 400, map[string]string{"error": "同步请求体必须是空对象"})
return
}
identity, _ := sessionFrom(r.Context())
receipt, replayed, e := store.EnqueueSync(r.Context(), key, identity.Username)
if e != nil {
status := 503
message := "同步未成功入队,已有本地数据保持不变"
if errors.Is(e, qaopsmirror.ErrInvalid) {
status = 400
message = "同步请求标识无效"
}
if errors.Is(e, qaopsmirror.ErrConflict) {
status = 409
message = "同步标识已用于另一操作,请重新发起"
}
if errors.Is(e, qaopsmirror.ErrWorkerUnavailable) {
message = "问答同步执行器未在线,未创建任务"
}
writeJSON(w, status, map[string]string{"error": message})
return
}
operation := "created"
if replayed {
operation = "replayed"
}
writeJSON(w, 202, map[string]any{"data": receipt, "operation": operation})
}
+86
View File
@@ -0,0 +1,86 @@
package httpapi
import (
"context"
"encoding/json"
"knowledge-governance/internal/qaopsmirror"
"net/http"
"net/http/httptest"
"strings"
"testing"
)
type mirrorSyncSpy struct {
qaOpsSpy
syncCalls int
key, actor string
}
func (s *mirrorSyncSpy) GetSyncStatus(context.Context) (qaopsmirror.Status, error) {
return qaopsmirror.Status{Mode: "local", IntervalSeconds: 300, WorkerReady: true}, nil
}
func (s *mirrorSyncSpy) EnqueueSync(_ context.Context, key, actor string) (qaopsmirror.SyncReceipt, bool, error) {
s.syncCalls++
s.key = key
s.actor = actor
return qaopsmirror.SyncReceipt{JobID: "fixture-mirror-job"}, false, nil
}
func TestQAMirrorSyncRequiresSessionIntentAndIdempotency(t *testing.T) {
spy := &mirrorSyncSpy{}
auth := newFakeAuthStore()
auth.sessions["mirror-admin"] = "usr-boss"
auth.sessions["mirror-viewer"] = "usr-eye"
h, e := NewHandler(Config{Store: &fakeJobStore{}, QAOps: spy, QAOpsEnabled: true, QAOpsTenantKey: "fixture-tenant", Auth: &AuthConfig{Store: auth}})
if e != nil {
t.Fatal(e)
}
for _, tc := range []struct {
method, token, key, intent, origin, body string
status int
}{
{"GET", "", "", "", "", "", 401}, {"GET", "mirror-viewer", "", "", "", "", 403}, {"GET", "mirror-admin", "", "", "", "", 200},
{"POST", "mirror-admin", "op-one", "", "", "{}", 403}, {"POST", "mirror-admin", "op-one", "qa-records-sync", "https://foreign.invalid", "{}", 403},
{"POST", "mirror-admin", "", "qa-records-sync", "", "{}", 400}, {"POST", "mirror-admin", "op-one", "qa-records-sync", "", "null", 400},
{"POST", "mirror-admin", "op-one", "qa-records-sync", "", `{"tenant":"forged"}`, 400}, {"POST", "mirror-admin", "op-one", "qa-records-sync", "", "{}", 202},
} {
req := httptest.NewRequest(tc.method, "/api/v1/qa-ops/sync", strings.NewReader(tc.body))
if tc.token != "" {
req.AddCookie(&http.Cookie{Name: sessionCookieName, Value: tc.token})
}
req.Header.Set("Idempotency-Key", tc.key)
req.Header.Set("X-Governance-Action", tc.intent)
req.Header.Set("Origin", tc.origin)
w := httptest.NewRecorder()
h.ServeHTTP(w, req)
if w.Code != tc.status {
t.Fatalf("%s %s got %d want %d: %s", tc.method, tc.body, w.Code, tc.status, w.Body.String())
}
}
if spy.syncCalls != 1 || spy.key != "op-one" || spy.actor == "" {
t.Fatal("invalid sync reached store or operator was not session-stamped")
}
req := httptest.NewRequest("GET", "/api/v1/qa-ops/status", nil)
req.AddCookie(&http.Cookie{Name: sessionCookieName, Value: "mirror-admin"})
w := httptest.NewRecorder()
h.ServeHTTP(w, req)
var body struct {
Data struct {
Cache *qaopsmirror.Status `json:"cache"`
}
}
if json.Unmarshal(w.Body.Bytes(), &body) != nil || body.Data.Cache == nil || body.Data.Cache.Mode != "local" {
t.Fatal("local serving mode missing from status")
}
}
func TestQAMirrorSyncNamespaceCannotBypassEntry(t *testing.T) {
auth := newFakeAuthStore()
auth.sessions["mirror-admin"] = "usr-boss"
h, _ := NewHandler(Config{Store: &fakeJobStore{}, Auth: &AuthConfig{Store: auth}})
req := httptest.NewRequest("POST", "/api/v1/jobs", strings.NewReader(`{"type":"qaops.records.sync.fixture","payload":{},"idempotency_key":"bypass"}`))
req.AddCookie(&http.Cookie{Name: sessionCookieName, Value: "mirror-admin"})
w := httptest.NewRecorder()
h.ServeHTTP(w, req)
if w.Code != 400 {
t.Fatal("generic jobs endpoint bypassed mirror sync admission", w.Code)
}
}
+11 -2
View File
@@ -61,7 +61,16 @@ func qaOpsStatus(w http.ResponseWriter, r *http.Request, config Config) {
warnings = append(warnings, "database_transport_unencrypted")
}
}
writeJSON(w, http.StatusOK, map[string]any{"data": map[string]any{"state": state, "code": code, "warnings": warnings, "read_only": true, "source_version": "2.0", "channel": "dingtalk"}})
data := map[string]any{"state": state, "code": code, "warnings": warnings, "read_only": true, "source_version": "2.0", "channel": "dingtalk"}
if mirror, ok := config.QAOps.(qaMirrorSyncStore); ok {
cache, err := mirror.GetSyncStatus(r.Context())
if err != nil {
qaOpsError(w, err)
return
}
data["cache"] = cache
}
writeJSON(w, http.StatusOK, map[string]any{"data": data})
}
func listQAOpsRecords(w http.ResponseWriter, r *http.Request, config Config) {
@@ -144,7 +153,7 @@ func getQAOpsRecord(w http.ResponseWriter, r *http.Request, config Config) {
func qaOpsError(w http.ResponseWriter, err error) {
status := http.StatusServiceUnavailable
code := "source_unavailable"
message := "问答数据源不可用,请检查独立只读连接;原有知识治理功能不受影响"
message := "问答记录当前不可用;本地模式请检查同步状态,原有知识治理功能不受影响"
if errors.Is(err, errQAOrganization) {
code = "organization_unavailable"
message = "通讯录或区域配置读取失败,未将不可用数据伪装成空列,请重试"
+23 -3
View File
@@ -22,6 +22,21 @@ type QAQualityStore interface {
Snapshot(context.Context, string, int64) (qaopsquality.Snapshot, error)
}
type storedQAQualityReader interface {
ReadStoredQuality(context.Context, []dobotaiops.QualityKey) (map[string]qaopsquality.View, error)
}
func qaQualityViews(ctx context.Context, c Config, records []dobotaiops.Record) (map[string]qaopsquality.View, error) {
if reader, ok := c.QAOps.(storedQAQualityReader); ok {
keys := make([]dobotaiops.QualityKey, 0, len(records))
for _, r := range records {
keys = append(keys, dobotaiops.QualityKey{ID: r.ID, InputHash: r.QualityInputHash})
}
return reader.ReadStoredQuality(ctx, keys)
}
return c.QAQuality.Observe(ctx, records)
}
type qaQualityReader interface {
ReadQualityScope(context.Context, dobotaiops.RecordQuery) (dobotaiops.QualityScope, error)
}
@@ -44,7 +59,7 @@ func enrichQAQualityRecord(ctx context.Context, c Config, item *dobotaiops.Recor
if c.QAQuality == nil {
return nil
}
views, err := c.QAQuality.Observe(ctx, []dobotaiops.Record{*item})
views, err := qaQualityViews(ctx, c, []dobotaiops.Record{*item})
if err != nil {
return err
}
@@ -59,7 +74,7 @@ func enrichQAQualityPage(ctx context.Context, c Config, page *dobotaiops.RecordP
if c.QAQuality == nil {
return nil
}
views, err := c.QAQuality.Observe(ctx, page.Items)
views, err := qaQualityViews(ctx, c, page.Items)
if err != nil {
return err
}
@@ -197,7 +212,12 @@ func qaQualityQuery(w http.ResponseWriter, r *http.Request, c Config) {
qaOpsError(w, err)
return
}
views, err := c.QAQuality.ObserveKeys(r.Context(), scope.Keys)
var views map[string]qaopsquality.View
if stored, ok := c.QAOps.(storedQAQualityReader); ok {
views, err = stored.ReadStoredQuality(r.Context(), scope.Keys)
} else {
views, err = c.QAQuality.ObserveKeys(r.Context(), scope.Keys)
}
if err != nil || len(views) != len(scope.Keys) {
if err == nil {
err = qaopsquality.ErrUnavailable
+4 -1
View File
@@ -19,6 +19,7 @@ import (
"knowledge-governance/internal/jobs"
"knowledge-governance/internal/monitoring"
postgresstore "knowledge-governance/internal/platform/postgres"
"knowledge-governance/internal/qaopsmirror"
"knowledge-governance/internal/qaopsorg"
)
@@ -188,6 +189,8 @@ func NewHandler(config Config) (http.Handler, error) {
mux.HandleFunc("GET /api/v1/qa-ops/organization", func(w http.ResponseWriter, r *http.Request) { qaOrganization(w, r, config) })
mux.HandleFunc("POST /api/v1/qa-ops/organization/preview", func(w http.ResponseWriter, r *http.Request) { qaOrganization(w, r, config) })
mux.HandleFunc("PUT /api/v1/qa-ops/organization/regions", func(w http.ResponseWriter, r *http.Request) { qaOrganization(w, r, config) })
mux.HandleFunc("GET /api/v1/qa-ops/sync", func(w http.ResponseWriter, r *http.Request) { qaRecordsSync(w, r, config) })
mux.HandleFunc("POST /api/v1/qa-ops/sync", func(w http.ResponseWriter, r *http.Request) { qaRecordsSync(w, r, config) })
mux.HandleFunc("GET /api/v1/qa-ops/status", func(w http.ResponseWriter, r *http.Request) {
qaOpsStatus(w, r, config)
})
@@ -569,7 +572,7 @@ func createJob(w http.ResponseWriter, r *http.Request, config Config) {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid JSON request"})
return
}
if strings.HasPrefix(input.Type, qaopsorg.SyncJobPrefix) {
if strings.HasPrefix(input.Type, qaopsorg.SyncJobPrefix) || strings.HasPrefix(input.Type, qaopsmirror.SyncJobPrefix) {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "通讯录同步任务只能从问答运营的管理员入口创建"})
return
}