新增迁移50与每5分钟/手动持久同步,API只读本地;同步失败保留、权限阻断、租约与质量失效原子激活。 保留全量稳定分页/固定列,已存质量批量读取,元数据不反复解压长答案;详情安全Markdown与原文。 已运行整仓 make verify、源与副本隔离PG -race、真实只读源及隔离浏览器。 实际同步2525条;固定窗口640条本地中位1.46秒,对照直接源33.10秒。 停worker、缺源凭据、重启API后仍可查询;未更新8088,未修改生产原问答。
This commit is contained in:
+1
-1
@@ -137,7 +137,7 @@ func run(ctx context.Context, getenv func(string) string) error {
|
||||
}
|
||||
}
|
||||
}
|
||||
qaReader, qaSettings := openQAOperations(ctx, getenv, config.DatabaseURL)
|
||||
qaReader, qaSettings := openQAOperations(ctx, getenv, config.DatabaseURL, pool)
|
||||
if closer, ok := qaReader.(interface{ Close() }); ok {
|
||||
defer closer.Close()
|
||||
}
|
||||
|
||||
+12
-7
@@ -2,20 +2,25 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"knowledge-governance/internal/dobotaiops"
|
||||
"knowledge-governance/internal/httpapi"
|
||||
"knowledge-governance/internal/qaopsmirror"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// A failed optional source must not replace or take down the main database.
|
||||
func openQAOperations(ctx context.Context, getenv func(string) string, governanceDSN string) (httpapi.QAOpsReader, dobotaiops.Settings) {
|
||||
// API serving never constructs or probes a production-source client. The worker
|
||||
// alone synchronizes the explicitly authorized local copy; this is not a fallback.
|
||||
func openQAOperations(_ context.Context, getenv func(string) string, governanceDSN string, pool *pgxpool.Pool) (httpapi.QAOpsReader, dobotaiops.Settings) {
|
||||
settings := dobotaiops.LoadSettings(getenv, governanceDSN)
|
||||
if !settings.Enabled || settings.Problem != "" {
|
||||
if !settings.Enabled {
|
||||
return nil, settings
|
||||
}
|
||||
client, err := dobotaiops.Build(ctx, settings.Connection)
|
||||
if err != nil {
|
||||
settings.Problem = "source_connection_or_permissions_failed"
|
||||
if settings.TenantKey == "" || len(settings.TenantKey) > 128 || strings.ContainsRune(settings.TenantKey, 0) {
|
||||
settings.Problem = "local_mirror_tenant_invalid"
|
||||
return nil, settings
|
||||
}
|
||||
return client, settings
|
||||
// Missing/offline source credentials stop synchronization, not last-good local reads.
|
||||
settings.Problem = ""
|
||||
return qaopsmirror.NewStore(pool, settings.TenantKey), settings
|
||||
}
|
||||
|
||||
@@ -6,13 +6,13 @@ import (
|
||||
)
|
||||
|
||||
func TestQAOpsConnectionFailureIsContainedOutsideMainDatabase(t *testing.T) {
|
||||
reader, settings := openQAOperations(context.Background(), func(string) string { return "" }, "postgres://main@host/governance")
|
||||
reader, settings := openQAOperations(context.Background(), func(string) string { return "" }, "postgres://main@host/governance", nil)
|
||||
if reader != nil || settings.Enabled || settings.Problem != "" {
|
||||
t.Fatal("QA operations must be disabled by default")
|
||||
}
|
||||
env := map[string]string{"DOBOTAI_OPS_READ_ENABLED": "true", "DOBOTAI_OPS_TENANT_KEY": "approved-tenant", "DOBOTAI_OPS_READ_DSN_FILE": "/nonexistent-qaops-fixture-secret"}
|
||||
reader, settings = openQAOperations(context.Background(), func(k string) string { return env[k] }, "postgres://main@host/governance")
|
||||
if reader != nil || !settings.Enabled || settings.Problem == "" {
|
||||
t.Fatal("failed external connection must be a contained module error")
|
||||
reader, settings = openQAOperations(context.Background(), func(k string) string { return env[k] }, "postgres://main@host/governance", nil)
|
||||
if reader == nil || !settings.Enabled || settings.Problem != "" {
|
||||
t.Fatal("API serving still constructs a source connection instead of a local reader")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -56,6 +56,10 @@ func run(ctx context.Context, getenv func(string) string) error {
|
||||
if syncErr := configureDirectorySync(pool, getenv, handlers); syncErr != nil {
|
||||
log.Print("QA directory synchronization unavailable; handler was not registered")
|
||||
}
|
||||
mirrorStore, mirrorErr := configureRecordsSync(pool, getenv, config.DatabaseURL, handlers)
|
||||
if mirrorErr != nil {
|
||||
log.Print("QA local mirror configuration invalid; handler was not registered")
|
||||
}
|
||||
worker, err := jobs.NewWorker(jobs.WorkerConfig{
|
||||
Queue: store,
|
||||
WorkerID: config.WorkerID,
|
||||
@@ -67,6 +71,9 @@ func run(ctx context.Context, getenv func(string) string) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if mirrorStore != nil {
|
||||
go runRecordsSyncScheduler(ctx, mirrorStore, func(error) { log.Print("QA local mirror scheduler unavailable; last committed data retained") })
|
||||
}
|
||||
go runSourceRefreshScheduler(ctx, store, config.SourceRefreshSchedulerInterval, func(err error) {
|
||||
log.Printf("worker %s source refresh scheduler error: %v", config.WorkerID, err)
|
||||
})
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"knowledge-governance/internal/dobotaiops"
|
||||
"knowledge-governance/internal/jobs"
|
||||
"knowledge-governance/internal/qaopsmirror"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
func configureRecordsSync(pool *pgxpool.Pool, getenv func(string) string, governanceDSN string, handlers map[string]jobs.Handler) (*qaopsmirror.Store, error) {
|
||||
settings := dobotaiops.LoadSettings(getenv, governanceDSN)
|
||||
if !settings.Enabled {
|
||||
return nil, nil
|
||||
}
|
||||
if settings.TenantKey == "" || len(settings.TenantKey) > 128 || strings.ContainsRune(settings.TenantKey, 0) {
|
||||
return nil, qaopsmirror.ErrInvalid
|
||||
}
|
||||
store := qaopsmirror.NewStore(pool, settings.TenantKey)
|
||||
handlers[store.SyncJobType()] = func(ctx context.Context, job jobs.Job) error {
|
||||
return store.HandleSync(ctx, job, func(ctx context.Context) (qaopsmirror.Source, error) {
|
||||
current := dobotaiops.LoadSettings(getenv, governanceDSN)
|
||||
if !current.Enabled || current.Problem != "" || current.TenantKey != settings.TenantKey {
|
||||
return nil, dobotaiops.ErrUnavailable
|
||||
}
|
||||
// One source connection, shared by every manifest and bounded payload read in this job.
|
||||
current.Connection.MaxConns = 1
|
||||
current.Connection.MinConns = 0
|
||||
return dobotaiops.Build(ctx, current.Connection)
|
||||
})
|
||||
}
|
||||
return store, nil
|
||||
}
|
||||
|
||||
// The next due time and automatic operation key live in PostgreSQL, so a worker
|
||||
// restart does not reset the five-minute cadence or duplicate an active import.
|
||||
func runRecordsSyncScheduler(ctx context.Context, store *qaopsmirror.Store, onError func(error)) {
|
||||
tick := time.NewTicker(15 * time.Second)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
if _, e := store.EnqueueDue(ctx); e != nil && ctx.Err() == nil && onError != nil {
|
||||
onError(e)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-tick.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"knowledge-governance/internal/jobs"
|
||||
"knowledge-governance/internal/qaopsmirror"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestLocalMirrorWorkerRegistrationNeverConnectsSource(t *testing.T) {
|
||||
values := map[string]string{}
|
||||
get := func(k string) string { return values[k] }
|
||||
handlers := map[string]jobs.Handler{}
|
||||
if store, e := configureRecordsSync(nil, get, "postgres://fixture@localhost/local", handlers); e != nil || store != nil || len(handlers) != 0 {
|
||||
t.Fatal("disabled mirror registered")
|
||||
}
|
||||
values["DOBOTAI_OPS_READ_ENABLED"] = "true"
|
||||
values["DOBOTAI_OPS_TENANT_KEY"] = "fixture-worker-tenant"
|
||||
values["DOBOTAI_OPS_READ_DSN_FILE"] = "/nonexistent-fixture-source-secret"
|
||||
store, e := configureRecordsSync(nil, get, "postgres://fixture@localhost/local", handlers)
|
||||
if e != nil || store == nil || handlers[qaopsmirror.NewStore(nil, "fixture-worker-tenant").SyncJobType()] == nil {
|
||||
t.Fatal("local mirror handler missing; registration must not connect to source", e)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user