新增迁移50与每5分钟/手动持久同步,API只读本地;同步失败保留、权限阻断、租约与质量失效原子激活。 保留全量稳定分页/固定列,已存质量批量读取,元数据不反复解压长答案;详情安全Markdown与原文。 已运行整仓 make verify、源与副本隔离PG -race、真实只读源及隔离浏览器。 实际同步2525条;固定窗口640条本地中位1.46秒,对照直接源33.10秒。 停worker、缺源凭据、重启API后仍可查询;未更新8088,未修改生产原问答。
This commit is contained in:
@@ -0,0 +1,45 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"knowledge-governance/internal/dobotaiops"
|
||||
"knowledge-governance/internal/qaopsquality"
|
||||
"net/http"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type mirroredQualitySource struct {
|
||||
qualitySource
|
||||
reads int
|
||||
}
|
||||
|
||||
func (s *mirroredQualitySource) ReadStoredQuality(_ context.Context, keys []dobotaiops.QualityKey) (map[string]qaopsquality.View, error) {
|
||||
s.reads++
|
||||
out := map[string]qaopsquality.View{}
|
||||
for _, key := range keys {
|
||||
out[key.ID] = qaopsquality.Project(key.InputHash, 1, 0, qaopsquality.Check{Result: dobotaiops.CheckMarkerCoverage(true)}, nil)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
func TestMirroredQualityQueriesDoNotRepeatObservationWrites(t *testing.T) {
|
||||
source := &mirroredQualitySource{qualitySource: qualitySource{record: dobotaiops.Record{ID: "fixture-turn", Question: "fixture", Answer: "暂无已发布"}}}
|
||||
e, _ := qaopsquality.EvidenceFor(source.record)
|
||||
source.record.QualityInputHash = e.Hash
|
||||
legacy := &qualityFake{fail: true}
|
||||
auth := newFakeAuthStore()
|
||||
auth.sessions["quality-admin"] = "usr-boss"
|
||||
h, e2 := NewHandler(Config{Store: &fakeJobStore{}, Auth: &AuthConfig{Store: auth}, QAOps: source, QAOpsTenantKey: "fixed", QAQuality: legacy, Now: func() time.Time { return time.Date(2026, 10, 17, 0, 0, 0, 0, time.UTC) }})
|
||||
if e2 != nil {
|
||||
t.Fatal(e2)
|
||||
}
|
||||
for _, q := range []struct{ method, path, body string }{{"GET", "/api/v1/qa-ops/records/fixture-turn", ""}, {"GET", "/api/v1/qa-ops/records?start=2026-10-14T00:00:00Z&end=2026-10-15T00:00:00Z", ""}, {"POST", "/api/v1/qa-ops/quality/query", `{"start":"2026-10-14T00:00:00Z","end":"2026-10-15T00:00:00Z","keyword":""}`}} {
|
||||
w := qualityRequest(h, q.method, q.path, "quality-admin", q.body)
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatal("local mirror read went back through observation writes", q.path, w.Code)
|
||||
}
|
||||
}
|
||||
if source.reads != 3 || legacy.calls != 0 {
|
||||
t.Fatal("batched stored projection not used", source.reads, legacy.calls)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"knowledge-governance/internal/qaopsmirror"
|
||||
"net/http"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type qaMirrorSyncStore interface {
|
||||
GetSyncStatus(context.Context) (qaopsmirror.Status, error)
|
||||
EnqueueSync(context.Context, string, string) (qaopsmirror.SyncReceipt, bool, error)
|
||||
}
|
||||
|
||||
func qaRecordsSync(w http.ResponseWriter, r *http.Request, c Config) {
|
||||
if !authorizeQAOps(w, r) {
|
||||
return
|
||||
}
|
||||
if r.URL.RawQuery != "" {
|
||||
writeJSON(w, 400, map[string]string{"error": "同步入口不接受查询参数"})
|
||||
return
|
||||
}
|
||||
store, ok := c.QAOps.(qaMirrorSyncStore)
|
||||
if !ok || !c.QAOpsEnabled {
|
||||
writeJSON(w, 503, map[string]string{"error": "本地问答同步未接线,未创建任务"})
|
||||
return
|
||||
}
|
||||
if r.Method == http.MethodGet {
|
||||
state, e := store.GetSyncStatus(r.Context())
|
||||
if e != nil {
|
||||
qaOpsError(w, e)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, map[string]any{"data": state})
|
||||
return
|
||||
}
|
||||
if !authorizeQAIntent(w, r, "qa-records-sync") {
|
||||
return
|
||||
}
|
||||
key := r.Header.Get("Idempotency-Key")
|
||||
if key == "" || len(key) > 128 || strings.TrimSpace(key) != key {
|
||||
writeJSON(w, 400, map[string]string{"error": "同步请求需要有效 Idempotency-Key,重试须复用原标识"})
|
||||
return
|
||||
}
|
||||
var body *struct{}
|
||||
d := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1024))
|
||||
d.DisallowUnknownFields()
|
||||
if d.Decode(&body) != nil || body == nil || d.Decode(&struct{}{}) != io.EOF {
|
||||
writeJSON(w, 400, map[string]string{"error": "同步请求体必须是空对象"})
|
||||
return
|
||||
}
|
||||
identity, _ := sessionFrom(r.Context())
|
||||
receipt, replayed, e := store.EnqueueSync(r.Context(), key, identity.Username)
|
||||
if e != nil {
|
||||
status := 503
|
||||
message := "同步未成功入队,已有本地数据保持不变"
|
||||
if errors.Is(e, qaopsmirror.ErrInvalid) {
|
||||
status = 400
|
||||
message = "同步请求标识无效"
|
||||
}
|
||||
if errors.Is(e, qaopsmirror.ErrConflict) {
|
||||
status = 409
|
||||
message = "同步标识已用于另一操作,请重新发起"
|
||||
}
|
||||
if errors.Is(e, qaopsmirror.ErrWorkerUnavailable) {
|
||||
message = "问答同步执行器未在线,未创建任务"
|
||||
}
|
||||
writeJSON(w, status, map[string]string{"error": message})
|
||||
return
|
||||
}
|
||||
operation := "created"
|
||||
if replayed {
|
||||
operation = "replayed"
|
||||
}
|
||||
writeJSON(w, 202, map[string]any{"data": receipt, "operation": operation})
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"knowledge-governance/internal/qaopsmirror"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type mirrorSyncSpy struct {
|
||||
qaOpsSpy
|
||||
syncCalls int
|
||||
key, actor string
|
||||
}
|
||||
|
||||
func (s *mirrorSyncSpy) GetSyncStatus(context.Context) (qaopsmirror.Status, error) {
|
||||
return qaopsmirror.Status{Mode: "local", IntervalSeconds: 300, WorkerReady: true}, nil
|
||||
}
|
||||
func (s *mirrorSyncSpy) EnqueueSync(_ context.Context, key, actor string) (qaopsmirror.SyncReceipt, bool, error) {
|
||||
s.syncCalls++
|
||||
s.key = key
|
||||
s.actor = actor
|
||||
return qaopsmirror.SyncReceipt{JobID: "fixture-mirror-job"}, false, nil
|
||||
}
|
||||
func TestQAMirrorSyncRequiresSessionIntentAndIdempotency(t *testing.T) {
|
||||
spy := &mirrorSyncSpy{}
|
||||
auth := newFakeAuthStore()
|
||||
auth.sessions["mirror-admin"] = "usr-boss"
|
||||
auth.sessions["mirror-viewer"] = "usr-eye"
|
||||
h, e := NewHandler(Config{Store: &fakeJobStore{}, QAOps: spy, QAOpsEnabled: true, QAOpsTenantKey: "fixture-tenant", Auth: &AuthConfig{Store: auth}})
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
for _, tc := range []struct {
|
||||
method, token, key, intent, origin, body string
|
||||
status int
|
||||
}{
|
||||
{"GET", "", "", "", "", "", 401}, {"GET", "mirror-viewer", "", "", "", "", 403}, {"GET", "mirror-admin", "", "", "", "", 200},
|
||||
{"POST", "mirror-admin", "op-one", "", "", "{}", 403}, {"POST", "mirror-admin", "op-one", "qa-records-sync", "https://foreign.invalid", "{}", 403},
|
||||
{"POST", "mirror-admin", "", "qa-records-sync", "", "{}", 400}, {"POST", "mirror-admin", "op-one", "qa-records-sync", "", "null", 400},
|
||||
{"POST", "mirror-admin", "op-one", "qa-records-sync", "", `{"tenant":"forged"}`, 400}, {"POST", "mirror-admin", "op-one", "qa-records-sync", "", "{}", 202},
|
||||
} {
|
||||
req := httptest.NewRequest(tc.method, "/api/v1/qa-ops/sync", strings.NewReader(tc.body))
|
||||
if tc.token != "" {
|
||||
req.AddCookie(&http.Cookie{Name: sessionCookieName, Value: tc.token})
|
||||
}
|
||||
req.Header.Set("Idempotency-Key", tc.key)
|
||||
req.Header.Set("X-Governance-Action", tc.intent)
|
||||
req.Header.Set("Origin", tc.origin)
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, req)
|
||||
if w.Code != tc.status {
|
||||
t.Fatalf("%s %s got %d want %d: %s", tc.method, tc.body, w.Code, tc.status, w.Body.String())
|
||||
}
|
||||
}
|
||||
if spy.syncCalls != 1 || spy.key != "op-one" || spy.actor == "" {
|
||||
t.Fatal("invalid sync reached store or operator was not session-stamped")
|
||||
}
|
||||
req := httptest.NewRequest("GET", "/api/v1/qa-ops/status", nil)
|
||||
req.AddCookie(&http.Cookie{Name: sessionCookieName, Value: "mirror-admin"})
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, req)
|
||||
var body struct {
|
||||
Data struct {
|
||||
Cache *qaopsmirror.Status `json:"cache"`
|
||||
}
|
||||
}
|
||||
if json.Unmarshal(w.Body.Bytes(), &body) != nil || body.Data.Cache == nil || body.Data.Cache.Mode != "local" {
|
||||
t.Fatal("local serving mode missing from status")
|
||||
}
|
||||
}
|
||||
func TestQAMirrorSyncNamespaceCannotBypassEntry(t *testing.T) {
|
||||
auth := newFakeAuthStore()
|
||||
auth.sessions["mirror-admin"] = "usr-boss"
|
||||
h, _ := NewHandler(Config{Store: &fakeJobStore{}, Auth: &AuthConfig{Store: auth}})
|
||||
req := httptest.NewRequest("POST", "/api/v1/jobs", strings.NewReader(`{"type":"qaops.records.sync.fixture","payload":{},"idempotency_key":"bypass"}`))
|
||||
req.AddCookie(&http.Cookie{Name: sessionCookieName, Value: "mirror-admin"})
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, req)
|
||||
if w.Code != 400 {
|
||||
t.Fatal("generic jobs endpoint bypassed mirror sync admission", w.Code)
|
||||
}
|
||||
}
|
||||
@@ -61,7 +61,16 @@ func qaOpsStatus(w http.ResponseWriter, r *http.Request, config Config) {
|
||||
warnings = append(warnings, "database_transport_unencrypted")
|
||||
}
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{"data": map[string]any{"state": state, "code": code, "warnings": warnings, "read_only": true, "source_version": "2.0", "channel": "dingtalk"}})
|
||||
data := map[string]any{"state": state, "code": code, "warnings": warnings, "read_only": true, "source_version": "2.0", "channel": "dingtalk"}
|
||||
if mirror, ok := config.QAOps.(qaMirrorSyncStore); ok {
|
||||
cache, err := mirror.GetSyncStatus(r.Context())
|
||||
if err != nil {
|
||||
qaOpsError(w, err)
|
||||
return
|
||||
}
|
||||
data["cache"] = cache
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{"data": data})
|
||||
}
|
||||
|
||||
func listQAOpsRecords(w http.ResponseWriter, r *http.Request, config Config) {
|
||||
@@ -144,7 +153,7 @@ func getQAOpsRecord(w http.ResponseWriter, r *http.Request, config Config) {
|
||||
func qaOpsError(w http.ResponseWriter, err error) {
|
||||
status := http.StatusServiceUnavailable
|
||||
code := "source_unavailable"
|
||||
message := "问答数据源不可用,请检查独立只读连接;原有知识治理功能不受影响"
|
||||
message := "问答记录当前不可用;本地模式请检查同步状态,原有知识治理功能不受影响"
|
||||
if errors.Is(err, errQAOrganization) {
|
||||
code = "organization_unavailable"
|
||||
message = "通讯录或区域配置读取失败,未将不可用数据伪装成空列,请重试"
|
||||
|
||||
@@ -22,6 +22,21 @@ type QAQualityStore interface {
|
||||
Snapshot(context.Context, string, int64) (qaopsquality.Snapshot, error)
|
||||
}
|
||||
|
||||
type storedQAQualityReader interface {
|
||||
ReadStoredQuality(context.Context, []dobotaiops.QualityKey) (map[string]qaopsquality.View, error)
|
||||
}
|
||||
|
||||
func qaQualityViews(ctx context.Context, c Config, records []dobotaiops.Record) (map[string]qaopsquality.View, error) {
|
||||
if reader, ok := c.QAOps.(storedQAQualityReader); ok {
|
||||
keys := make([]dobotaiops.QualityKey, 0, len(records))
|
||||
for _, r := range records {
|
||||
keys = append(keys, dobotaiops.QualityKey{ID: r.ID, InputHash: r.QualityInputHash})
|
||||
}
|
||||
return reader.ReadStoredQuality(ctx, keys)
|
||||
}
|
||||
return c.QAQuality.Observe(ctx, records)
|
||||
}
|
||||
|
||||
type qaQualityReader interface {
|
||||
ReadQualityScope(context.Context, dobotaiops.RecordQuery) (dobotaiops.QualityScope, error)
|
||||
}
|
||||
@@ -44,7 +59,7 @@ func enrichQAQualityRecord(ctx context.Context, c Config, item *dobotaiops.Recor
|
||||
if c.QAQuality == nil {
|
||||
return nil
|
||||
}
|
||||
views, err := c.QAQuality.Observe(ctx, []dobotaiops.Record{*item})
|
||||
views, err := qaQualityViews(ctx, c, []dobotaiops.Record{*item})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -59,7 +74,7 @@ func enrichQAQualityPage(ctx context.Context, c Config, page *dobotaiops.RecordP
|
||||
if c.QAQuality == nil {
|
||||
return nil
|
||||
}
|
||||
views, err := c.QAQuality.Observe(ctx, page.Items)
|
||||
views, err := qaQualityViews(ctx, c, page.Items)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -197,7 +212,12 @@ func qaQualityQuery(w http.ResponseWriter, r *http.Request, c Config) {
|
||||
qaOpsError(w, err)
|
||||
return
|
||||
}
|
||||
views, err := c.QAQuality.ObserveKeys(r.Context(), scope.Keys)
|
||||
var views map[string]qaopsquality.View
|
||||
if stored, ok := c.QAOps.(storedQAQualityReader); ok {
|
||||
views, err = stored.ReadStoredQuality(r.Context(), scope.Keys)
|
||||
} else {
|
||||
views, err = c.QAQuality.ObserveKeys(r.Context(), scope.Keys)
|
||||
}
|
||||
if err != nil || len(views) != len(scope.Keys) {
|
||||
if err == nil {
|
||||
err = qaopsquality.ErrUnavailable
|
||||
|
||||
@@ -19,6 +19,7 @@ import (
|
||||
"knowledge-governance/internal/jobs"
|
||||
"knowledge-governance/internal/monitoring"
|
||||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||||
"knowledge-governance/internal/qaopsmirror"
|
||||
"knowledge-governance/internal/qaopsorg"
|
||||
)
|
||||
|
||||
@@ -188,6 +189,8 @@ func NewHandler(config Config) (http.Handler, error) {
|
||||
mux.HandleFunc("GET /api/v1/qa-ops/organization", func(w http.ResponseWriter, r *http.Request) { qaOrganization(w, r, config) })
|
||||
mux.HandleFunc("POST /api/v1/qa-ops/organization/preview", func(w http.ResponseWriter, r *http.Request) { qaOrganization(w, r, config) })
|
||||
mux.HandleFunc("PUT /api/v1/qa-ops/organization/regions", func(w http.ResponseWriter, r *http.Request) { qaOrganization(w, r, config) })
|
||||
mux.HandleFunc("GET /api/v1/qa-ops/sync", func(w http.ResponseWriter, r *http.Request) { qaRecordsSync(w, r, config) })
|
||||
mux.HandleFunc("POST /api/v1/qa-ops/sync", func(w http.ResponseWriter, r *http.Request) { qaRecordsSync(w, r, config) })
|
||||
mux.HandleFunc("GET /api/v1/qa-ops/status", func(w http.ResponseWriter, r *http.Request) {
|
||||
qaOpsStatus(w, r, config)
|
||||
})
|
||||
@@ -569,7 +572,7 @@ func createJob(w http.ResponseWriter, r *http.Request, config Config) {
|
||||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid JSON request"})
|
||||
return
|
||||
}
|
||||
if strings.HasPrefix(input.Type, qaopsorg.SyncJobPrefix) {
|
||||
if strings.HasPrefix(input.Type, qaopsorg.SyncJobPrefix) || strings.HasPrefix(input.Type, qaopsmirror.SyncJobPrefix) {
|
||||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "通讯录同步任务只能从问答运营的管理员入口创建"})
|
||||
return
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user