diff --git a/.env.example b/.env.example index 555e86a..b44901c 100644 --- a/.env.example +++ b/.env.example @@ -10,6 +10,17 @@ DOBOTAI_OPS_QUERY_TIMEOUT=4s DOBOTAI_OPS_MAX_CONNS=2 DOBOTAI_OPS_MIN_CONNS=0 +# API and worker share this opt-in binding; no browser-selected paths/credentials. +# Runtime root is the read-only existing weekly_report directory (SDK/config and +# private secret file). Python must be compatible and already provide PyYAML. +# Container deployment requires approved runtime/script mounts; not auto-installed. +DOBOTAI_OPS_DIRECTORY_SYNC_ENABLED=false +DOBOTAI_OPS_DIRECTORY_SYNC_PYTHON= +DOBOTAI_OPS_DIRECTORY_SYNC_SCRIPT= +DOBOTAI_OPS_DIRECTORY_SYNC_RUNTIME_ROOT= +DOBOTAI_OPS_DIRECTORY_SYNC_SPOOL_ROOT= +DOBOTAI_OPS_DIRECTORY_SYNC_TIMEOUT=20m + APP_ADDR=:8080 DATABASE_URL=postgres://governance@postgres:5432/governance?sslmode=disable PGPASSFILE=/run/secrets/pgpass diff --git a/CHANGELOG.md b/CHANGELOG.md index b066623..f4eb152 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,9 +4,10 @@ ### Added -- 新增治理侧通讯录实时桥接,复用既有周报读取能力,向源端读取真实缺失祖先以补齐部门树;不改旧周报脚本、配置或输出。私有 JSON 与受控子进程执行器拒绝不完整数据、模拟同步、公开文件和异常输出,超时清理临时人员文件;网页触发的后台同步队列及问答组织关联尚未接入。 +- 「问答运营 → 区域使用」支持管理员从网页发起持久化通讯录同步,重复请求重放、进行中操作合并;按服务器固定范围注册后台执行器,未接线或未检测到心跳时不创建新任务。界面显示真实任务状态、不编造百分比;失败保留旧快照与区域规则,同步完成不覆盖未保存草稿。旧执行器丢失租约后不能激活结果,重启接管和已提交快照的续跑使用原任务。 +- 新增治理侧通讯录实时桥接,复用既有周报读取能力,向源端读取真实缺失祖先以补齐部门树;不改旧周报脚本、配置或输出。私有 JSON 与受控子进程执行器拒绝不完整数据、模拟同步、公开文件和异常输出,超时清理临时人员文件;问答组织关联尚未接入。同步默认关闭,容器启用仍需批准的既有周报 SDK/配置及兼容 Python 运行时挂载。 - 区域归属表支持长部门路径换行,避免继承全局不换行规则后将生效区域与归属依据列挤出桌面面板。 -- **「问答运营 → 区域使用」区域归属配置切片**:按稳定部门 ID 搜索和勾选完整路径,预览去重归属人数,保存版本后重新读取;父部门包含子部门、子部门显式配置优先,多部门跨区域人员单独待核对。预览不写入,过期版本和不完整部门树被阻断。当前不是实时通讯录同步或使用/绩效统计验收,翻译及 AI 评估模型接入按用户要求暂缓。 +- **「问答运营 → 区域使用」区域归属配置切片**:按稳定部门 ID 搜索和勾选完整路径,预览去重归属人数,保存版本后重新读取;父部门包含子部门、子部门显式配置优先,多部门跨区域人员单独待核对。预览不写入,过期版本和不完整部门树被阻断。当前不是使用/绩效统计验收,翻译及 AI 评估模型接入按用户要求暂缓。 - 新增治理侧组织快照与区域修订迁移、受确认保护的后台导入命令;不修改 DobotAI 原表或钉钉组织结构。 - **「问答运营 → 问答记录」首个开发切片**:在现有导航中查看 DobotAI 2.0 钉钉问答,支持半开时间范围、原问题关键词、稳定游标分页和完整详情;按整个筛选结果隐藏空列,反馈聚合不放大问答条数。外语翻译、AI 评估、人员区域映射尚未接入,不标为第一阶段完成。 - 治理侧重新检查答案:包含 `暂无已发布` 即未覆盖,引用和源端 covered 不能抵消;未命中仍待评估,源端规则单独保留作对照。独立只读源默认关闭、租户后端固定、管理员会话鉴权,源故障不回退治理主库;安全告警明确保留 TEMP/TLS 尚未整改状态。 diff --git a/api/openapi.yaml b/api/openapi.yaml index 66cfbdc..b858e90 100644 --- a/api/openapi.yaml +++ b/api/openapi.yaml @@ -44,6 +44,10 @@ } ], "paths": { + "/api/v1/qa-ops/organization/sync": { + "get": {"tags":["QA operations"],"operationId":"getQAOrganizationSync","summary":"管理员读取通讯录同步执行器状态及最近任务的安全汇总,不返回人员名单或运行配置","responses":{"200":{"$ref":"#/components/responses/Envelope"},"400":{"$ref":"#/components/responses/BadRequest"},"401":{"description":"需要登录"},"403":{"description":"仅管理员"},"500":{"$ref":"#/components/responses/InternalError"},"503":{"description":"状态存储不可用,不伪装为空记录"}}}, + "post": {"tags":["QA operations"],"operationId":"startQAOrganizationSync","summary":"管理员同源发起持久化通讯录同步;重复请求重放同一任务,进行中任务合并;失败保留现有快照","parameters":[{"$ref":"#/components/parameters/IdempotencyKey"},{"name":"X-Governance-Action","in":"header","required":true,"schema":{"const":"qa-organization"}}],"requestBody":{"required":true,"content":{"application/json":{"schema":{"type":"object","additionalProperties":false}}}},"responses":{"202":{"$ref":"#/components/responses/OperationEnvelope"},"400":{"$ref":"#/components/responses/BadRequest"},"401":{"description":"需要登录"},"403":{"description":"仅管理员同源操作"},"409":{"description":"操作标识与原请求冲突"},"500":{"$ref":"#/components/responses/InternalError"},"503":{"description":"同步未接线、执行器离线或入队失败;未创建新任务"}}} + }, "/api/v1/qa-ops/organization": { "get": { "tags": ["QA operations"], diff --git a/cmd/api/main.go b/cmd/api/main.go index 032855f..36417e5 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -133,8 +133,19 @@ func run(ctx context.Context, getenv func(string) string) error { if qaSettings.Problem != "" { log.Print("QA operations source is unavailable; governance remains online") } + qaOrganizationStore := qaopsorg.NewStore(pool, getenv("DOBOTAI_OPS_TENANT_KEY")) + qaSyncSettings, qaSyncErr := qaopsorg.LoadSyncSettings(getenv) + if qaSyncErr != nil { + log.Print("QA directory synchronization is not configured; governance remains online") + } + var qaSyncStore httpapi.QAOrgSyncStore + if getenv("DOBOTAI_OPS_TENANT_KEY") != "" { + qaSyncStore = qaOrganizationStore + } handler, err := httpapi.NewHandler(httpapi.Config{ - QAOrg: qaopsorg.NewStore(pool, getenv("DOBOTAI_OPS_TENANT_KEY")), + QAOrgSync: qaSyncStore, + QAOrgSyncEnabled: qaSyncErr == nil && qaSyncSettings.Enabled, + QAOrg: qaOrganizationStore, QAOps: qaReader, QAOpsTenantKey: qaSettings.TenantKey, QAOpsEnabled: qaSettings.Enabled, diff --git a/cmd/worker/main.go b/cmd/worker/main.go index 055bfa4..99e2e4e 100644 --- a/cmd/worker/main.go +++ b/cmd/worker/main.go @@ -53,6 +53,9 @@ func run(ctx context.Context, getenv func(string) string) error { if err != nil { return err } + if syncErr := configureDirectorySync(pool, getenv, handlers); syncErr != nil { + log.Print("QA directory synchronization unavailable; handler was not registered") + } worker, err := jobs.NewWorker(jobs.WorkerConfig{ Queue: store, WorkerID: config.WorkerID, diff --git a/cmd/worker/qa_org.go b/cmd/worker/qa_org.go new file mode 100644 index 0000000..c0c3d03 --- /dev/null +++ b/cmd/worker/qa_org.go @@ -0,0 +1,26 @@ +package main + +import ( + "context" + "github.com/jackc/pgx/v5/pgxpool" + "knowledge-governance/internal/jobs" + "knowledge-governance/internal/qaopsorg" +) + +func configureDirectorySync(pool *pgxpool.Pool, getenv func(string) string, handlers map[string]jobs.Handler) error { + settings, err := qaopsorg.LoadSyncSettings(getenv) + if err != nil { + return err + } + if !settings.Enabled { + return nil + } + if err = settings.CheckWorkerRuntime(); err != nil { + return err + } + store := qaopsorg.NewStore(pool, settings.Tenant) + handlers[store.SyncJobType()] = func(ctx context.Context, job jobs.Job) error { + return store.HandleDirectorySync(ctx, job, settings.Runner) + } + return nil +} diff --git a/cmd/worker/qa_org_test.go b/cmd/worker/qa_org_test.go new file mode 100644 index 0000000..4fa629b --- /dev/null +++ b/cmd/worker/qa_org_test.go @@ -0,0 +1,39 @@ +package main + +import ( + "knowledge-governance/internal/jobs" + "knowledge-governance/internal/qaopsorg" + "os" + "path/filepath" + "testing" +) + +func TestDirectorySyncWorkerRegistrationIsOptInAndRuntimeChecked(t *testing.T) { + handlers := map[string]jobs.Handler{} + values := map[string]string{} + get := func(k string) string { return values[k] } + if err := configureDirectorySync(nil, get, handlers); err != nil || len(handlers) != 0 { + t.Fatal("disabled sync registered") + } + values["DOBOTAI_OPS_DIRECTORY_SYNC_ENABLED"] = "true" + if err := configureDirectorySync(nil, get, handlers); err == nil { + t.Fatal("incomplete binding registered") + } + root := t.TempDir() + script := filepath.Join(root, "fixture.py") + if err := os.WriteFile(script, []byte("# fixture; not executed by registration"), 0600); err != nil { + t.Fatal(err) + } + values["DOBOTAI_OPS_TENANT_KEY"] = "fixture-tenant" + values["DOBOTAI_OPS_DIRECTORY_SYNC_PYTHON"] = "/bin/sh" + values["DOBOTAI_OPS_DIRECTORY_SYNC_SCRIPT"] = script + values["DOBOTAI_OPS_DIRECTORY_SYNC_RUNTIME_ROOT"] = root + values["DOBOTAI_OPS_DIRECTORY_SYNC_SPOOL_ROOT"] = root + if err := configureDirectorySync(nil, get, handlers); err != nil || handlers[qaopsorg.NewStore(nil, "fixture-tenant").SyncJobType()] == nil { + t.Fatal("configured worker was not wired") + } + values["DOBOTAI_OPS_DIRECTORY_SYNC_SCRIPT"] = filepath.Join(root, "missing.py") + if err := configureDirectorySync(nil, get, map[string]jobs.Handler{}); err == nil { + t.Fatal("missing runtime advertised as ready") + } +} diff --git a/internal/httpapi/openapi_contract_test.go b/internal/httpapi/openapi_contract_test.go index aa923d7..f61bff7 100644 --- a/internal/httpapi/openapi_contract_test.go +++ b/internal/httpapi/openapi_contract_test.go @@ -77,7 +77,7 @@ func TestOpenAPICatalogMutationsRequireIdempotencyKeyHeader(t *testing.T) { if !ok || schema["type"] != "string" || schema["minLength"] != float64(1) || schema["maxLength"] != float64(128) { t.Fatalf("IdempotencyKey header schema is unsafe: %#v", parameter["schema"]) } - for _, path := range []string{"/api/v1/source-catalog/route", "/api/v1/source-catalog/sources/{id}/refresh"} { + for _, path := range []string{"/api/v1/qa-ops/organization/sync", "/api/v1/source-catalog/route", "/api/v1/source-catalog/sources/{id}/refresh"} { operation, ok := document.Paths[path]["post"].(map[string]any) if !ok { t.Fatalf("missing POST %s", path) diff --git a/internal/httpapi/qa_org.go b/internal/httpapi/qa_org.go index aed634c..f1ff249 100644 --- a/internal/httpapi/qa_org.go +++ b/internal/httpapi/qa_org.go @@ -7,7 +7,6 @@ import ( "io" "knowledge-governance/internal/qaopsorg" "net/http" - "net/url" ) type QAOrgStore interface { @@ -33,20 +32,9 @@ func qaOrganization(w http.ResponseWriter, r *http.Request, config Config) { if r.Method == http.MethodGet { result, err = config.QAOrg.GetView(r.Context()) } else { - if r.Header.Get("X-Governance-Action") != "qa-organization" || r.Header.Get("Sec-Fetch-Site") == "cross-site" { - writeJSON(w, 403, map[string]string{"error": "区域配置需要同源操作意图"}) + if !authorizeQAOrganizationWrite(w, r) { return } - // A same-origin browser request can pass a reverse proxy that rewrites - // Host. Fetch Metadata is browser-controlled; keep the explicit action - // header and reject cross-site requests without trusting forwarded hosts. - if raw := r.Header.Get("Origin"); raw != "" && r.Header.Get("Sec-Fetch-Site") != "same-origin" { - origin, parseErr := url.Parse(raw) - if parseErr != nil || (origin.Scheme != "http" && origin.Scheme != "https") || origin.Host != r.Host { - writeJSON(w, 403, map[string]string{"error": "拒绝跨站配置请求"}) - return - } - } var request qaopsorg.SaveRequest decoder := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<20)) decoder.DisallowUnknownFields() diff --git a/internal/httpapi/qa_org_sync.go b/internal/httpapi/qa_org_sync.go new file mode 100644 index 0000000..9ffad8e --- /dev/null +++ b/internal/httpapi/qa_org_sync.go @@ -0,0 +1,92 @@ +package httpapi + +import ( + "context" + "encoding/json" + "errors" + "io" + "knowledge-governance/internal/qaopsorg" + "net/http" + "net/url" + "strings" +) + +type QAOrgSyncStore interface { + GetSyncStatus(context.Context) (qaopsorg.SyncStatus, error) + EnqueueSync(context.Context, string, string) (qaopsorg.SyncReceipt, bool, error) +} + +func authorizeQAOrganizationWrite(w http.ResponseWriter, r *http.Request) bool { + if r.Header.Get("X-Governance-Action") != "qa-organization" || r.Header.Get("Sec-Fetch-Site") == "cross-site" { + writeJSON(w, 403, map[string]string{"error": "区域配置需要同源操作意图"}) + return false + } + // Fetch Metadata survives same-origin proxies that rewrite Host; untrusted + // forwarded headers must not grant cross-origin write authority. + if raw := r.Header.Get("Origin"); raw != "" && r.Header.Get("Sec-Fetch-Site") != "same-origin" { + origin, err := url.Parse(raw) + if err != nil || (origin.Scheme != "http" && origin.Scheme != "https") || origin.Host != r.Host { + writeJSON(w, 403, map[string]string{"error": "拒绝跨站配置请求"}) + return false + } + } + return true +} +func qaOrganizationSync(w http.ResponseWriter, r *http.Request, c Config) { + if !authorizeQAOps(w, r) { + return + } + if r.URL.RawQuery != "" { + writeOrgError(w, qaopsorg.ErrInvalid) + return + } + w.Header().Set("Cache-Control", "no-store") + if r.Method == http.MethodGet { + if c.QAOrgSync == nil { + writeJSON(w, 200, map[string]any{"data": qaopsorg.SyncStatus{}}) + return + } + result, err := c.QAOrgSync.GetSyncStatus(r.Context()) + if err != nil { + writeOrgError(w, err) + return + } + result.Configured = c.QAOrgSyncEnabled + writeJSON(w, 200, map[string]any{"data": result}) + return + } + if !authorizeQAOrganizationWrite(w, r) { + return + } + key := r.Header.Get("Idempotency-Key") + if key == "" || len(key) > 128 || strings.TrimSpace(key) != key { + writeJSON(w, 400, map[string]string{"error": "同步请求需要有效 Idempotency-Key,重试须复用原标识"}) + return + } + var body *struct{} + decoder := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1024)) + decoder.DisallowUnknownFields() + if decoder.Decode(&body) != nil || body == nil || decoder.Decode(&struct{}{}) != io.EOF { + writeOrgError(w, qaopsorg.ErrInvalid) + return + } + if !c.QAOrgSyncEnabled || c.QAOrgSync == nil { + writeJSON(w, 503, map[string]string{"error": "通讯录后台同步未接线,未创建任务"}) + return + } + identity, _ := sessionFrom(r.Context()) + receipt, replayed, err := c.QAOrgSync.EnqueueSync(r.Context(), key, identity.Username) + if err != nil { + if errors.Is(err, qaopsorg.ErrSyncWorkerUnavailable) { + writeJSON(w, 503, map[string]string{"error": "通讯录执行器未在线,未创建任务;请检查后台同步配置"}) + return + } + writeOrgError(w, err) + return + } + operation := "created" + if replayed { + operation = "replayed" + } + writeJSON(w, 202, map[string]any{"data": receipt, "operation": operation}) +} diff --git a/internal/httpapi/qa_org_sync_security_test.go b/internal/httpapi/qa_org_sync_security_test.go new file mode 100644 index 0000000..d3e40e8 --- /dev/null +++ b/internal/httpapi/qa_org_sync_security_test.go @@ -0,0 +1,20 @@ +package httpapi + +import ( + "net/http/httptest" + "strings" + "testing" +) + +func TestGenericJobCreationCannotBypassDirectorySyncAdminPath(t *testing.T) { + h, err := NewHandler(Config{Store: &fakeJobStore{}}) + if err != nil { + t.Fatal(err) + } + req := httptest.NewRequest("POST", "/api/v1/jobs", strings.NewReader(`{"type":"qaops.directory.sync.fixture","payload":{"triggered_by":"forged"},"idempotency_key":"bypass"}`)) + res := httptest.NewRecorder() + h.ServeHTTP(res, req) + if res.Code != 400 { + t.Fatalf("reserved sync job bypassed its dedicated admission path: %d", res.Code) + } +} diff --git a/internal/httpapi/qa_org_sync_test.go b/internal/httpapi/qa_org_sync_test.go new file mode 100644 index 0000000..fb790ba --- /dev/null +++ b/internal/httpapi/qa_org_sync_test.go @@ -0,0 +1,85 @@ +package httpapi + +import ( + "context" + "encoding/json" + "io" + "knowledge-governance/internal/qaopsorg" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +type orgSyncSpy struct { + calls int + key, actor string + replay bool + err error +} + +func (s *orgSyncSpy) GetSyncStatus(context.Context) (qaopsorg.SyncStatus, error) { + s.calls++ + return qaopsorg.SyncStatus{Configured: true, WorkerReady: true}, s.err +} +func (s *orgSyncSpy) EnqueueSync(_ context.Context, key, actor string) (qaopsorg.SyncReceipt, bool, error) { + s.calls++ + s.key = key + s.actor = actor + return qaopsorg.SyncReceipt{JobID: "fixture-job"}, s.replay, s.err +} +func TestQAOrganizationSyncRequiresAdminIntentAndDurableOperation(t *testing.T) { + spy := &orgSyncSpy{} + auth := newFakeAuthStore() + auth.sessions["sync-admin"] = "usr-boss" + auth.sessions["sync-viewer"] = "usr-eye" + handler, err := NewHandler(Config{Store: &fakeJobStore{}, QAOrgSync: spy, QAOrgSyncEnabled: true, Auth: &AuthConfig{Store: auth}}) + if err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + method, token, key, action, origin, body string + status int + }{ + {"GET", "", "", "", "", "", 401}, {"GET", "sync-viewer", "", "", "", "", 403}, {"GET", "sync-admin", "", "", "", "", 200}, + {"POST", "sync-admin", "op-one", "", "", "{}", 403}, {"POST", "sync-admin", "op-one", "qa-organization", "https://evil.invalid", "{}", 403}, + {"POST", "sync-admin", "", "qa-organization", "", "{}", 400}, {"POST", "sync-admin", " op-one ", "qa-organization", "", "{}", 400}, + {"POST", "sync-admin", "op-one", "qa-organization", "", `null`, 400}, + {"POST", "sync-admin", "op-one", "qa-organization", "", `{"tenant":"forged"}`, 400}, {"POST", "sync-admin", "op-one", "qa-organization", "", "{} {}", 400}, + {"POST", "sync-admin", "op-one", "qa-organization", "", "{}", 202}, + } { + req := httptest.NewRequest(tc.method, "/api/v1/qa-ops/organization/sync", strings.NewReader(tc.body)) + if tc.token != "" { + req.AddCookie(&http.Cookie{Name: sessionCookieName, Value: tc.token}) + } + req.Header.Set("Idempotency-Key", tc.key) + req.Header.Set("X-Governance-Action", tc.action) + req.Header.Set("Origin", tc.origin) + res := httptest.NewRecorder() + handler.ServeHTTP(res, req) + if res.Code != tc.status { + t.Fatalf("%+v got %d: %s", tc, res.Code, res.Body.String()) + } + } + if spy.calls != 2 || spy.key != "op-one" || spy.actor == "" { + t.Fatalf("invalid request reached store: %+v", spy) + } + spy.replay = true + req := httptest.NewRequest("POST", "/api/v1/qa-ops/organization/sync", strings.NewReader("{}")) + req.AddCookie(&http.Cookie{Name: sessionCookieName, Value: "sync-admin"}) + req.Header.Set("Idempotency-Key", "op-one") + req.Header.Set("X-Governance-Action", "qa-organization") + res := httptest.NewRecorder() + handler.ServeHTTP(res, req) + var body map[string]any + if json.Unmarshal(res.Body.Bytes(), &body) != nil || body["operation"] != "replayed" { + t.Fatal("operation replay not exposed") + } + disabled, _ := NewHandler(Config{Store: &fakeJobStore{}, QAOrgSync: spy, Auth: &AuthConfig{Store: auth}}) + req.Body = io.NopCloser(strings.NewReader("{}")) + res = httptest.NewRecorder() + disabled.ServeHTTP(res, req) + if res.Code != 503 { + t.Fatal("disabled sync enqueued") + } +} diff --git a/internal/httpapi/server.go b/internal/httpapi/server.go index df5c2d9..ba009af 100644 --- a/internal/httpapi/server.go +++ b/internal/httpapi/server.go @@ -19,6 +19,7 @@ import ( "knowledge-governance/internal/jobs" "knowledge-governance/internal/monitoring" postgresstore "knowledge-governance/internal/platform/postgres" + "knowledge-governance/internal/qaopsorg" ) type JobStore interface { @@ -101,6 +102,8 @@ type Store interface { } type Config struct { + QAOrgSync QAOrgSyncStore + QAOrgSyncEnabled bool QAOrg QAOrgStore QAOps QAOpsReader QAOpsTenantKey string @@ -172,6 +175,8 @@ func NewHandler(config Config) (http.Handler, error) { config.NewID = newID } mux := http.NewServeMux() + mux.HandleFunc("GET /api/v1/qa-ops/organization/sync", func(w http.ResponseWriter, r *http.Request) { qaOrganizationSync(w, r, config) }) + mux.HandleFunc("POST /api/v1/qa-ops/organization/sync", func(w http.ResponseWriter, r *http.Request) { qaOrganizationSync(w, r, config) }) mux.HandleFunc("GET /api/v1/qa-ops/organization", func(w http.ResponseWriter, r *http.Request) { qaOrganization(w, r, config) }) mux.HandleFunc("POST /api/v1/qa-ops/organization/preview", func(w http.ResponseWriter, r *http.Request) { qaOrganization(w, r, config) }) mux.HandleFunc("PUT /api/v1/qa-ops/organization/regions", func(w http.ResponseWriter, r *http.Request) { qaOrganization(w, r, config) }) @@ -556,6 +561,10 @@ func createJob(w http.ResponseWriter, r *http.Request, config Config) { writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid JSON request"}) return } + if strings.HasPrefix(input.Type, qaopsorg.SyncJobPrefix) { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "通讯录同步任务只能从问答运营的管理员入口创建"}) + return + } if strings.TrimSpace(input.Type) == "" || len(input.Type) > 100 { writeJSON(w, http.StatusBadRequest, map[string]string{"error": "type is required and must not exceed 100 characters"}) return diff --git a/internal/platform/postgres/operation_requests.go b/internal/platform/postgres/operation_requests.go index abff998..2562ffe 100644 --- a/internal/platform/postgres/operation_requests.go +++ b/internal/platform/postgres/operation_requests.go @@ -20,6 +20,12 @@ var ErrOperationRequestConflict = errors.New("operation request conflicts with a type operationExecutor func(pgx.Tx) (json.RawMessage, error) +// ExecuteOperation lets bounded modules share the durable operation transaction +// instead of caching responses after their business writes have committed. +func (s *Store) ExecuteOperation(ctx context.Context, operationType, operationID, fingerprint string, now time.Time, execute func(pgx.Tx) (json.RawMessage, error)) (json.RawMessage, bool, error) { + return s.executeOperation(ctx, operationType, operationID, fingerprint, now, execute) +} + func (s *Store) RouteCatalogFilesWithOperation(ctx context.Context, request governance.CatalogRouteRequest, operationID string) (governance.CatalogRouteResult, bool, error) { if err := validateCatalogRouteRequest(request); err != nil { return governance.CatalogRouteResult{}, false, err diff --git a/internal/qaopsorg/store.go b/internal/qaopsorg/store.go index c07cb95..fe0ebb2 100644 --- a/internal/qaopsorg/store.go +++ b/internal/qaopsorg/store.go @@ -7,6 +7,7 @@ import ( "encoding/hex" "encoding/json" "errors" + "knowledge-governance/internal/jobs" "sort" "time" @@ -176,6 +177,9 @@ func (s *Store) SaveRules(ctx context.Context, request SaveRequest, actor string return next, nil } func (s *Store) ReplaceDirectory(ctx context.Context, input Import) (State, error) { + return s.replaceDirectory(ctx, input, nil) +} +func (s *Store) replaceDirectory(ctx context.Context, input Import, lease *jobs.Job) (State, error) { if !s.available() { return State{}, ErrUnavailable } @@ -197,6 +201,11 @@ func (s *Store) ReplaceDirectory(ctx context.Context, input Import) (State, erro return State{}, ErrUnavailable } defer func() { _ = tx.Rollback(context.Background()) }() + if lease != nil { + if _, err = s.syncLease(ctx, tx, *lease, true); err != nil { + return State{}, err + } + } if _, err = tx.Exec(ctx, `INSERT INTO qa_ops_directory_snapshots(tenant_key,id,source,synced_at,payload) VALUES($1,$2,$3,$4,$5) ON CONFLICT(tenant_key,id) DO NOTHING`, s.tenant, id, input.Source, input.SyncedAt, payload); err != nil { return State{}, ErrUnavailable } @@ -207,6 +216,16 @@ func (s *Store) ReplaceDirectory(ctx context.Context, input Import) (State, erro if tag.RowsAffected() != 1 { return State{}, ErrConflict } + if lease != nil { + result, _ := json.Marshal(SyncResult{DirectoryID: id, Departments: len(input.Directory.Departments), People: len(input.Directory.People), SyncedAt: input.SyncedAt}) + tag, err = tx.Exec(ctx, `UPDATE jobs SET payload=jsonb_set(payload,'{result}',$3::jsonb),updated_at=now() WHERE id=$1 AND lease_owner=$2 AND state='running' AND lease_until>clock_timestamp() AND attempt=$4`, lease.ID, lease.LeaseOwner, result, lease.Attempt) + if err != nil { + return State{}, ErrUnavailable + } + if tag.RowsAffected() != 1 { + return State{}, ErrSyncLease + } + } if err = tx.Commit(ctx); err != nil { return State{}, ErrUnavailable } diff --git a/internal/qaopsorg/sync.go b/internal/qaopsorg/sync.go new file mode 100644 index 0000000..6d0f246 --- /dev/null +++ b/internal/qaopsorg/sync.go @@ -0,0 +1,194 @@ +package qaopsorg + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "time" + + "github.com/jackc/pgx/v5" + "knowledge-governance/internal/jobs" + postgresstore "knowledge-governance/internal/platform/postgres" +) + +var ErrSyncWorkerUnavailable = errors.New("directory sync worker is not connected") + +const SyncJobPrefix = "qaops.directory.sync." + +var ErrSyncLease = errors.New("directory sync lease is no longer owned") + +type DirectoryRunner interface { + Run(context.Context) (Import, error) +} +type SyncReceipt struct { + JobID string `json:"job_id"` + Deduplicated bool `json:"deduplicated"` +} +type SyncResult struct { + DirectoryID string `json:"directory_id"` + Departments int `json:"departments"` + People int `json:"people"` + SyncedAt time.Time `json:"synced_at"` +} +type SyncJob struct { + ID string `json:"id"` + State string `json:"state"` + Attempt int `json:"attempt"` + MaxAttempts int `json:"max_attempts"` + UpdatedAt time.Time `json:"updated_at"` + ErrorCode string `json:"error_code"` + Result *SyncResult `json:"result"` +} +type SyncStatus struct { + Configured bool `json:"configured"` + WorkerReady bool `json:"worker_ready"` + Job *SyncJob `json:"job"` +} + +func (s *Store) SyncJobType() string { + h := sha256.Sum256([]byte(s.tenant)) + return SyncJobPrefix + hex.EncodeToString(h[:]) +} + +const syncWorkerReadySQL = `SELECT EXISTS(SELECT 1 FROM worker_heartbeats WHERE $1=ANY(string_to_array(job_types,',')) AND last_seen_at>clock_timestamp()-interval '2 minutes') OR EXISTS(SELECT 1 FROM jobs WHERE type=$1 AND state='running' AND lease_until>clock_timestamp())` + +func (s *Store) EnqueueSync(ctx context.Context, key, actor string) (SyncReceipt, bool, error) { + if !s.available() { + return SyncReceipt{}, false, ErrUnavailable + } + if !validID(key) || len(key) > 128 || !validID(actor) { + return SyncReceipt{}, false, ErrInvalid + } + fp := sha256.Sum256([]byte(actor)) + now := time.Now() + raw, replayed, err := postgresstore.NewStore(s.pool).ExecuteOperation(ctx, s.SyncJobType(), key, hex.EncodeToString(fp[:]), now, func(tx pgx.Tx) (json.RawMessage, error) { + if _, e := tx.Exec(ctx, "SELECT pg_advisory_xact_lock(hashtextextended($1,0))", s.SyncJobType()); e != nil { + return nil, ErrUnavailable + } + // A last-attempt process may die without recording failure. A new explicit + // action retires that exhausted lease; it never impersonates a live owner. + if _, e := tx.Exec(ctx, `UPDATE jobs SET state='failed',last_error='directory worker stopped; previous snapshot retained',last_error_code='directory_sync_failed',lease_owner=NULL,lease_until=NULL,updated_at=now() WHERE type=$1 AND state='running' AND lease_until=max_attempts`, s.SyncJobType()); e != nil { + return nil, ErrUnavailable + } + var active string + e := tx.QueryRow(ctx, `SELECT id FROM jobs WHERE type=$1 AND state IN ('pending','running','retry') ORDER BY created_at DESC,id DESC LIMIT 1`, s.SyncJobType()).Scan(&active) + if e == nil { + return json.Marshal(SyncReceipt{JobID: active, Deduplicated: true}) + } + if !errors.Is(e, pgx.ErrNoRows) { + return nil, ErrUnavailable + } + var ready bool + if tx.QueryRow(ctx, syncWorkerReadySQL, s.SyncJobType()).Scan(&ready) != nil { + return nil, ErrUnavailable + } + if !ready { + return nil, ErrSyncWorkerUnavailable + } + digest := sha256.Sum256([]byte(s.SyncJobType() + ":" + key)) + id := "job-" + hex.EncodeToString(digest[:16]) + payload, _ := json.Marshal(map[string]string{"triggered_by": actor}) + job, e := postgresstore.NewStore(tx).Enqueue(ctx, jobs.EnqueueRequest{ID: id, Type: s.SyncJobType(), Payload: payload, AvailableAt: now, MaxAttempts: 3, IdempotencyKey: s.SyncJobType() + ":" + key}, now) + if e != nil || job == nil { + return nil, ErrUnavailable + } + return json.Marshal(SyncReceipt{JobID: job.ID}) + }) + if errors.Is(err, postgresstore.ErrOperationRequestConflict) { + return SyncReceipt{}, false, ErrConflict + } + if err != nil { + return SyncReceipt{}, false, err + } + var result SyncReceipt + if json.Unmarshal(raw, &result) != nil || result.JobID == "" { + return SyncReceipt{}, false, ErrUnavailable + } + return result, replayed, nil +} +func (s *Store) GetSyncStatus(ctx context.Context) (SyncStatus, error) { + result := SyncStatus{Configured: true} + if !s.available() { + return SyncStatus{}, ErrUnavailable + } + if s.pool.QueryRow(ctx, syncWorkerReadySQL, s.SyncJobType()).Scan(&result.WorkerReady) != nil { + return SyncStatus{}, ErrUnavailable + } + var job SyncJob + var payload []byte + var expired bool + err := s.pool.QueryRow(ctx, `SELECT id,state,attempt,max_attempts,updated_at,payload,COALESCE(state='running' AND lease_until= job.MaxAttempts { + job.State = "failed" + } + } + if job.State == "failed" || job.State == "retry" { + job.ErrorCode = "directory_sync_failed" + } + result.Job = &job + return result, nil +} +func (s *Store) syncLease(ctx context.Context, q orgQuerier, job jobs.Job, lock bool) (*SyncResult, error) { + if job.Type != s.SyncJobType() || job.LeaseOwner == "" { + return nil, ErrSyncLease + } + query := `SELECT payload FROM jobs WHERE id=$1 AND type=$2 AND state='running' AND lease_owner=$3 AND attempt=$4 AND lease_until>clock_timestamp() AND EXISTS(SELECT 1 FROM operation_requests op WHERE op.operation_type=jobs.type AND op.result->>'job_id'=jobs.id)` + if lock { + query += " FOR UPDATE" + } + var raw []byte + if q.QueryRow(ctx, query, job.ID, s.SyncJobType(), job.LeaseOwner, job.Attempt).Scan(&raw) != nil { + return nil, ErrSyncLease + } + var payload struct { + Result *SyncResult `json:"result"` + } + if json.Unmarshal(raw, &payload) != nil { + return nil, ErrUnavailable + } + return payload.Result, nil +} +func (s *Store) HandleDirectorySync(ctx context.Context, job jobs.Job, runner DirectoryRunner) error { + if !s.available() || runner == nil { + return ErrDirectorySync + } + result, err := s.syncLease(ctx, s.pool, job, false) + if err != nil { + return err + } + if result != nil { + return nil + } + input, err := runner.Run(ctx) + if err != nil { + return ErrDirectorySync + } + if input.Source != "weekly_report_live" { + return jobs.Permanent("directory_sync_failed", ErrDirectorySync) + } + _, err = s.replaceDirectory(ctx, input, &job) + if err != nil { + if errors.Is(err, ErrSyncLease) { + return err + } + return ErrDirectorySync + } + return nil +} diff --git a/internal/qaopsorg/sync_integration_test.go b/internal/qaopsorg/sync_integration_test.go new file mode 100644 index 0000000..85d432e --- /dev/null +++ b/internal/qaopsorg/sync_integration_test.go @@ -0,0 +1,244 @@ +package qaopsorg_test + +import ( + "context" + "encoding/json" + "errors" + "net/url" + "os" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + "knowledge-governance/internal/jobs" + postgresstore "knowledge-governance/internal/platform/postgres" + "knowledge-governance/internal/qaopsorg" +) + +type syncRunner func(context.Context) (qaopsorg.Import, error) + +func (f syncRunner) Run(ctx context.Context) (qaopsorg.Import, error) { return f(ctx) } +func TestDirectorySyncQueueDurableReplayAndLeaseFencing(t *testing.T) { + dsn := os.Getenv("KG_QAOPS_ORG_TEST_DSN") + if dsn == "" { + t.Skip("explicit isolated organization database is not configured") + } + u, err := url.Parse(dsn) + if err != nil || u.Hostname() != "127.0.0.1" || !strings.HasPrefix(u.Path, "/kg_qaops_org_test_") || os.Getenv("KG_QAOPS_ORG_TEST_ISOLATED") != "1" { + t.Fatal("refusing non-isolated organization database") + } + ctx := context.Background() + pool, err := pgxpool.New(ctx, dsn) + if err != nil { + t.Fatal("isolated connection failed") + } + defer pool.Close() + if err = postgresstore.ApplyMigrations(ctx, pool); err != nil { + t.Fatal(err) + } + store := qaopsorg.NewStore(pool, "sync-fixture-tenant") + queue := postgresstore.NewStore(pool) + initial := qaopsorg.Import{Source: "weekly_report_live", Complete: true, SyncedAt: time.Now().Add(-time.Hour), Directory: qaopsorg.Directory{Departments: []qaopsorg.Department{{ID: "root", Name: "同步夹具"}}, People: []qaopsorg.Person{{ID: "fixture-person", DepartmentIDs: []string{"root"}}}}} + old, err := store.ReplaceDirectory(ctx, initial) + if err != nil { + t.Fatal(err) + } + saved, err := store.SaveRules(ctx, qaopsorg.SaveRequest{DirectoryID: old.DirectoryID, Rules: []qaopsorg.Rule{{DepartmentID: "root", Region: "APAC"}}}, "fixture-admin") + if err != nil { + t.Fatal(err) + } + if _, _, err = store.EnqueueSync(ctx, "op-no-worker", "fixture-admin"); !errors.Is(err, qaopsorg.ErrSyncWorkerUnavailable) { + t.Fatalf("disconnected worker accepted: %v", err) + } + if _, err = queue.Claim(ctx, "fixture-sync-worker", []string{store.SyncJobType()}, time.Now(), time.Minute); err != nil { + t.Fatal(err) + } + var first qaopsorg.SyncReceipt + var created atomic.Int32 + var wg sync.WaitGroup + failures := make(chan error, 12) + ids := make(chan string, 12) + for range 12 { + wg.Add(1) + go func() { + defer wg.Done() + r, replay, e := qaopsorg.NewStore(pool, "sync-fixture-tenant").EnqueueSync(ctx, "op-same-action", "fixture-admin") + if e != nil { + failures <- e + return + } + if !replay { + created.Add(1) + } + ids <- r.JobID + }() + } + wg.Wait() + close(failures) + for e := range failures { + t.Fatal(e) + } + close(ids) + for id := range ids { + if first.JobID == "" { + first.JobID = id + } + if id != first.JobID { + t.Fatal("duplicate effect") + } + } + if created.Load() != 1 { + t.Fatal("operation replay was not durable") + } + reused, _, err := store.EnqueueSync(ctx, "op-another-click", "fixture-admin") + if err != nil || !reused.Deduplicated || reused.JobID != first.JobID { + t.Fatal("in-flight clicks did not coalesce") + } + if _, _, err = store.EnqueueSync(ctx, "op-same-action", "another-admin"); !errors.Is(err, qaopsorg.ErrConflict) { + t.Fatal("actor fingerprint conflict ignored") + } + other, _ := qaopsorg.NewStore(pool, "other-sync-tenant").GetSyncStatus(ctx) + if other.Job != nil { + t.Fatal("job leaked across tenants") + } + var n int + if err = pool.QueryRow(ctx, "SELECT count(*) FROM jobs WHERE type=$1", store.SyncJobType()).Scan(&n); err != nil || n != 1 { + t.Fatal("job count must remain one") + } + job, err := queue.Claim(ctx, "worker-before-crash", []string{store.SyncJobType()}, time.Now(), time.Minute) + if err != nil || job == nil { + t.Fatal("job was not durable/claimable") + } + // A different process can reclaim expired work; the old owner must not activate data. + if _, err = pool.Exec(ctx, "UPDATE jobs SET lease_until=now()-interval '1 second' WHERE id=$1", job.ID); err != nil { + t.Fatal(err) + } + reclaimed, err := queue.Claim(ctx, "worker-after-restart", []string{store.SyncJobType()}, time.Now(), time.Minute) + if err != nil || reclaimed == nil || reclaimed.Attempt != 2 { + t.Fatal("expired lease did not resume") + } + newer := initial + newer.SyncedAt = time.Now() + newer.Directory.People = append(append([]qaopsorg.Person{}, initial.Directory.People...), qaopsorg.Person{ID: "fixture-person-2", DepartmentIDs: []string{"root"}}) + var fetched atomic.Int32 + run := syncRunner(func(context.Context) (qaopsorg.Import, error) { fetched.Add(1); return newer, nil }) + if err = store.HandleDirectorySync(ctx, *job, run); err == nil { + t.Fatal("stale owner activated a snapshot") + } + still, _ := store.GetView(ctx) + if still.DirectoryID != old.DirectoryID { + t.Fatal("stale lease changed active data") + } + if err = store.HandleDirectorySync(ctx, *reclaimed, run); err != nil { + t.Fatal(err) + } + count := fetched.Load() + // Crash after committing the snapshot, before queue.Complete: do not fetch twice. + if err = store.HandleDirectorySync(ctx, *reclaimed, run); err != nil || fetched.Load() != count { + t.Fatal("committed source was fetched again") + } + if err = queue.Complete(ctx, reclaimed.ID, reclaimed.LeaseOwner, time.Now()); err != nil { + t.Fatal(err) + } + current, _ := store.GetView(ctx) + if current.View.TotalPeople != 2 || current.RevisionID != saved.RevisionID || current.View.Regions["APAC"] != 2 { + t.Fatal("refresh lost regional rules or failed to activate") + } + status, err := store.GetSyncStatus(ctx) + if err != nil || status.Job == nil || status.Job.State != "completed" || status.Job.Result == nil || status.Job.Result.People != 2 { + t.Fatal("safe status missing durable result") + } + raw, _ := json.Marshal(status) + if strings.Contains(string(raw), "fixture-person") || strings.Contains(string(raw), "lease_owner") { + t.Fatal("personnel/lease details leaked in status") + } + // Failure must preserve the previous complete directory and never store raw exceptions. + failed, _, err := store.EnqueueSync(ctx, "op-fails", "fixture-admin") + if err != nil { + t.Fatal(err) + } + bad, err := queue.Claim(ctx, "worker-failure", []string{store.SyncJobType()}, time.Now(), time.Minute) + if err != nil || bad == nil || bad.ID != failed.JobID { + t.Fatal("failure job was not enqueued") + } + failure := store.HandleDirectorySync(ctx, *bad, syncRunner(func(context.Context) (qaopsorg.Import, error) { + return qaopsorg.Import{}, errors.New("sensitive-fixture-must-not-escape") + })) + if failure == nil || strings.Contains(failure.Error(), "sensitive-fixture") { + t.Fatal("unsafe source exception") + } + if err = queue.Fail(ctx, bad.ID, bad.LeaseOwner, failure.Error(), "directory_sync_failed", time.Now(), time.Now().Add(time.Minute)); err != nil { + t.Fatal(err) + } + kept, _ := store.GetView(ctx) + if kept.DirectoryID != current.DirectoryID || kept.RevisionID != saved.RevisionID { + t.Fatal("failed refresh replaced valid state") + } + late := qaopsorg.NewStore(pool, "late-fence-tenant") + lateOld, err := late.ReplaceDirectory(ctx, initial) + if err != nil { + t.Fatal(err) + } + if _, err = queue.Claim(ctx, "late-worker", []string{late.SyncJobType()}, time.Now(), time.Minute); err != nil { + t.Fatal(err) + } + if _, _, err = late.EnqueueSync(ctx, "op-late-fence", "fixture-admin"); err != nil { + t.Fatal(err) + } + lateJob, err := queue.Claim(ctx, "late-worker", []string{late.SyncJobType()}, time.Now(), time.Minute) + if err != nil || lateJob == nil { + t.Fatal("late-fence job unavailable") + } + lateErr := late.HandleDirectorySync(ctx, *lateJob, syncRunner(func(context.Context) (qaopsorg.Import, error) { + _, failure := pool.Exec(ctx, "UPDATE jobs SET lease_owner='replacement-worker',attempt=attempt+1,lease_until=now()+interval '1 minute' WHERE id=$1", lateJob.ID) + return newer, failure + })) + if lateErr == nil { + t.Fatal("lease lost DURING source fetch was not fenced at commit") + } + lateKept, _ := late.GetView(ctx) + if lateKept.DirectoryID != lateOld.DirectoryID { + t.Fatal("late stale worker activated data") + } + if err = pool.QueryRow(ctx, "SELECT count(*) FROM qa_ops_directory_snapshots WHERE tenant_key=$1", "late-fence-tenant").Scan(&n); err != nil || n != 1 { + t.Fatal("rejected stale snapshot left durable rows") + } + if _, err = pool.Exec(ctx, "UPDATE jobs SET lease_until=now()-interval '1 second' WHERE id=$1", lateJob.ID); err != nil { + t.Fatal(err) + } + interrupted, err := late.GetSyncStatus(ctx) + if err != nil || interrupted.Job == nil || interrupted.Job.State != "retry" { + t.Fatal("interrupted lease must wait for takeover, not claim active source processing") + } + if _, err = pool.Exec(ctx, "UPDATE jobs SET attempt=max_attempts,lease_until=now()-interval '1 second' WHERE id=$1", lateJob.ID); err != nil { + t.Fatal(err) + } + exhausted, err := late.GetSyncStatus(ctx) + if err != nil || exhausted.Job == nil || exhausted.Job.State != "failed" { + t.Fatal("exhausted crashed worker reported running forever") + } + fresh, _, err := late.EnqueueSync(ctx, "op-after-exhausted", "fixture-admin") + if err != nil || fresh.JobID == lateJob.ID { + t.Fatal("explicit retry could not replace an exhausted crashed job") + } + ghost := qaopsorg.NewStore(pool, "unadmitted-tenant") + if _, err = ghost.ReplaceDirectory(ctx, initial); err != nil { + t.Fatal(err) + } + _, err = queue.Enqueue(ctx, jobs.EnqueueRequest{ID: "fixture-unadmitted-job", Type: ghost.SyncJobType(), Payload: json.RawMessage(`{}`), AvailableAt: time.Now(), MaxAttempts: 1, IdempotencyKey: "fixture-unadmitted-job"}, time.Now()) + if err != nil { + t.Fatal(err) + } + ghostJob, err := queue.Claim(ctx, "ghost-worker", []string{ghost.SyncJobType()}, time.Now(), time.Minute) + if err != nil || ghostJob == nil { + t.Fatal("forged fixture unavailable") + } + var ghostCalls int + ghostErr := ghost.HandleDirectorySync(ctx, *ghostJob, syncRunner(func(context.Context) (qaopsorg.Import, error) { ghostCalls++; return newer, nil })) + if ghostErr == nil || ghostCalls != 0 { + t.Fatal("unadmitted generic task consumed the directory source") + } +} diff --git a/internal/qaopsorg/sync_settings.go b/internal/qaopsorg/sync_settings.go new file mode 100644 index 0000000..c3c02c6 --- /dev/null +++ b/internal/qaopsorg/sync_settings.go @@ -0,0 +1,59 @@ +package qaopsorg + +import ( + "errors" + "os" + "path/filepath" + "time" +) + +type SyncSettings struct { + Enabled bool + Tenant string + Runner ExecDirectoryRunner +} + +func LoadSyncSettings(getenv func(string) string) (SyncSettings, error) { + var s SyncSettings + enabled := getenv("DOBOTAI_OPS_DIRECTORY_SYNC_ENABLED") + if enabled == "" || enabled == "false" { + return s, nil + } + if enabled != "true" { + return s, errors.New("invalid directory sync enable flag") + } + s.Tenant = getenv("DOBOTAI_OPS_TENANT_KEY") + s.Runner = ExecDirectoryRunner{Python: getenv("DOBOTAI_OPS_DIRECTORY_SYNC_PYTHON"), Script: getenv("DOBOTAI_OPS_DIRECTORY_SYNC_SCRIPT"), RuntimeRoot: getenv("DOBOTAI_OPS_DIRECTORY_SYNC_RUNTIME_ROOT"), SpoolRoot: getenv("DOBOTAI_OPS_DIRECTORY_SYNC_SPOOL_ROOT"), Timeout: 20 * time.Minute} + if value := getenv("DOBOTAI_OPS_DIRECTORY_SYNC_TIMEOUT"); value != "" { + d, err := time.ParseDuration(value) + if err != nil { + return SyncSettings{}, errors.New("invalid directory sync timeout") + } + s.Runner.Timeout = d + } + if !validID(s.Tenant) || s.Runner.Timeout < time.Second || s.Runner.Timeout > 30*time.Minute { + return SyncSettings{}, errors.New("directory sync binding is incomplete") + } + for _, path := range []string{s.Runner.Python, s.Runner.Script, s.Runner.RuntimeRoot, s.Runner.SpoolRoot} { + if !filepath.IsAbs(path) || filepath.Clean(path) != path { + return SyncSettings{}, errors.New("directory sync requires server-owned absolute paths") + } + } + s.Enabled = true + return s, nil +} +func (s SyncSettings) CheckWorkerRuntime() error { + for _, path := range []string{s.Runner.Python, s.Runner.Script} { + info, err := os.Stat(path) + if err != nil || !info.Mode().IsRegular() { + return ErrSyncWorkerUnavailable + } + } + for _, path := range []string{s.Runner.RuntimeRoot, s.Runner.SpoolRoot} { + info, err := os.Stat(path) + if err != nil || !info.IsDir() { + return ErrSyncWorkerUnavailable + } + } + return nil +} diff --git a/internal/qaopsorg/sync_settings_test.go b/internal/qaopsorg/sync_settings_test.go new file mode 100644 index 0000000..322b02f --- /dev/null +++ b/internal/qaopsorg/sync_settings_test.go @@ -0,0 +1,37 @@ +package qaopsorg + +import ( + "testing" + "time" +) + +func TestDirectorySyncSettingsAreExplicitAndServerOwned(t *testing.T) { + values := map[string]string{} + get := func(k string) string { return values[k] } + off, err := LoadSyncSettings(get) + if err != nil || off.Enabled { + t.Fatal("sync must default off") + } + values["DOBOTAI_OPS_DIRECTORY_SYNC_ENABLED"] = "true" + if _, err = LoadSyncSettings(get); err == nil { + t.Fatal("unbound runtime accepted") + } + values["DOBOTAI_OPS_TENANT_KEY"] = "fixture-tenant" + values["DOBOTAI_OPS_DIRECTORY_SYNC_PYTHON"] = "/usr/bin/python3" + values["DOBOTAI_OPS_DIRECTORY_SYNC_SCRIPT"] = "/app/scripts/qa_ops_directory_sync.py" + values["DOBOTAI_OPS_DIRECTORY_SYNC_RUNTIME_ROOT"] = "/data/weekly-report" + values["DOBOTAI_OPS_DIRECTORY_SYNC_SPOOL_ROOT"] = "/data/private-spool" + ready, err := LoadSyncSettings(get) + if err != nil || !ready.Enabled || ready.Runner.Timeout != 20*time.Minute { + t.Fatal("valid runtime not configured") + } + values["DOBOTAI_OPS_DIRECTORY_SYNC_TIMEOUT"] = "31m" + if _, err = LoadSyncSettings(get); err == nil { + t.Fatal("unbounded source timeout") + } + values["DOBOTAI_OPS_DIRECTORY_SYNC_TIMEOUT"] = "10m" + values["DOBOTAI_OPS_DIRECTORY_SYNC_SCRIPT"] = "relative/script.py" + if _, err = LoadSyncSettings(get); err == nil { + t.Fatal("browser-like relative script accepted") + } +} diff --git a/scripts/test-scope.json b/scripts/test-scope.json index f5d61ca..7105950 100644 --- a/scripts/test-scope.json +++ b/scripts/test-scope.json @@ -9,6 +9,11 @@ "⚠️ 文件缺失或解析不了 ⇒ 门禁红:范围声明不出来时,不许默认放行。" ], "optional": [ + { + "package": "knowledge-governance/internal/qaopsorg", + "test": "TestDirectorySyncQueueDurableReplayAndLeaseFencing", + "reason": "仅显式一次性隔离PG库验证持久化入队、并发重放、进程接管与租约栅栏;不调用真实钉钉源、不连接共享或生产库;默认CI无库则明确跳过" + }, { "package": "knowledge-governance/internal/qaopsorg", "test": "TestOrganizationStoreRoundTripAndOptimisticConflict", diff --git a/web/src/QARegions.test.tsx b/web/src/QARegions.test.tsx index af92399..2923234 100644 --- a/web/src/QARegions.test.tsx +++ b/web/src/QARegions.test.tsx @@ -51,6 +51,30 @@ const props = { onRemoveMissing: vi.fn(), } describe('region mapping workbench', () => { + it('shows real synchronization stages without inventing progress and protects dirty drafts', () => { + const syncStatus = { + configured: true, + worker_ready: true, + job: { + id: 'j', + state: 'running' as const, + attempt: 1, + max_attempts: 3, + updated_at: '2026-10-17T08:00:00Z', + error_code: '', + result: null, + }, + } + const html = renderToStaticMarkup() + expect(html).toContain('正在读取并校验通讯录') + expect(html).not.toMatch(/\d+%/) + expect(html).toMatch(/disabled=""[^>]*>同步通讯录/) + const dirty = renderToStaticMarkup( + + ) + expect(dirty).toContain('未保存的区域修改') + expect(dirty).toMatch(/disabled=""[^>]*>同步通讯录/) + }) it('lets long directory paths wrap instead of inheriting the global nowrap table rule', () => { const css = readFileSync(new URL('./qaRegions.css', import.meta.url), 'utf8') const bodyRule = css.match(/\.qa-region-table td\s*\{([^}]+)\}/)?.[1] ?? '' diff --git a/web/src/QARegions.tsx b/web/src/QARegions.tsx index 25cb37e..07e94c3 100644 --- a/web/src/QARegions.tsx +++ b/web/src/QARegions.tsx @@ -2,7 +2,14 @@ import { useEffect, useRef, useState } from 'react' import { changeRegionRule, fetchOrganization, previewOrganization, regionNames, saveOrganization } from './qaOrg' import type { Organization, RegionRule } from './qaOrg' import './qaRegions.css' +import { completedSyncAction, directorySyncActive, fetchDirectorySync, startDirectorySync } from './qaOrgSync' +import type { DirectorySyncStatus } from './qaOrgSync' +import { newOperationID } from './operationKey' interface Props { + syncStatus?: DirectorySyncStatus | null + syncError?: string + syncSubmitting?: boolean + onSync?: () => void state: Organization | null preview: Organization | null rules: RegionRule[] @@ -32,7 +39,45 @@ export function RegionMappingView(p: Props) { + +
+ {!p.syncStatus + ? '正在读取后台同步状态…' + : !p.syncStatus.configured + ? '后台同步未接线' + : p.syncStatus.job + ? { + pending: '等待后台执行器', + running: '正在读取并校验通讯录', + retry: '上次同步失败,等待后台重试;保留原快照', + completed: '同步已完成,完整快照已落库', + failed: '同步失败,保留原快照;可重新同步', + cancelled: '同步已取消,保留原快照', + }[p.syncStatus.job.state] + : p.syncStatus.worker_ready + ? '后台执行器在线,尚未发起同步' + : '后台执行器未在线,不会创建无人处理的任务'} + {p.dirty ? ';存在未保存的区域修改,请先保存或放弃后同步' : ''} +
+ {p.syncError ? ( +

+ {p.syncError} +

+ ) : null} {p.state?.directory_synced_at ? (

{p.state.directory_source === 'weekly_report_snapshot' ? '历史快照' : '全量同步快照'} · 通讯录同步时间: @@ -152,6 +197,14 @@ function signature(rules: RegionRule[]) { return JSON.stringify([...rules].sort((a, b) => a.department_id.localeCompare(b.department_id))) } export function QARegions() { + const [syncStatus, setSyncStatus] = useState(null), + [syncError, setSyncError] = useState(''), + [syncSubmitting, setSyncSubmitting] = useState(false) + const syncOperation = useRef(''), + syncFlight = useRef(false), + syncAction = useRef(null), + lastCompleted = useRef(''), + dirtyRef = useRef(false) const [state, setState] = useState(null), [preview, setPreview] = useState(null), [rules, setRules] = useState([]) @@ -176,7 +229,10 @@ export function QARegions() { setPreview(null) try { const value = await fetchOrganization(controller.signal) - if (!controller.signal.aborted) accept(value) + if (!controller.signal.aborted) { + accept(value) + return true + } } catch (e) { if (!controller.signal.aborted) setError(e instanceof Error ? e.message : '读取失败') } finally { @@ -187,6 +243,72 @@ export function QARegions() { void load() return () => request.current?.abort() }, []) + const dirty = Boolean(state && signature(rules) !== signature(state.rules)) + dirtyRef.current = dirty + const loadRef = useRef(load) + loadRef.current = load + useEffect(() => { + const controller = new AbortController() + let timer: ReturnType + const poll = async () => { + let delay = 10000 + try { + const value = await fetchDirectorySync(controller.signal) + if (controller.signal.aborted) return + setSyncStatus(value) + setSyncError('') + delay = directorySyncActive(value) ? 1500 : 10000 + const job = value.job + if (job?.state === 'completed' && job.id !== lastCompleted.current) { + const action = completedSyncAction(value, lastCompleted.current, dirtyRef.current) + lastCompleted.current = job.id + if (action === 'preserve') setNotice('通讯录已同步,当前未保存草稿仍保留;请核对后重新读取') + else if (action === 'reload' && (await loadRef.current())) setNotice('通讯录同步完成并已重新读取') + } else if (job && lastCompleted.current === '') lastCompleted.current = 'pending:' + job.id + } catch (e) { + if (!controller.signal.aborted) + setSyncError(e instanceof Error ? e.message : '同步状态读取失败,未将现有快照清空') + } finally { + if (!controller.signal.aborted) timer = setTimeout(() => void poll(), delay) + } + } + void poll() + return () => { + controller.abort() + clearTimeout(timer) + syncAction.current?.abort() + } + }, []) + const synchronize = async () => { + if ( + busy || + dirty || + syncFlight.current || + !syncStatus?.configured || + !syncStatus.worker_ready || + directorySyncActive(syncStatus) + ) + return + syncFlight.current = true + setSyncSubmitting(true) + setSyncError('') + const controller = new AbortController() + syncAction.current = controller + try { + if (!syncOperation.current) syncOperation.current = newOperationID() + const receipt = await startDirectorySync(syncOperation.current, controller.signal) + syncOperation.current = '' + if (controller.signal.aborted) return + lastCompleted.current = 'submitted:' + receipt.job_id + setSyncStatus(await fetchDirectorySync(controller.signal)) + } catch (e) { + if (!controller.signal.aborted) + setSyncError(e instanceof Error ? e.message : '同步提交结果未知;重试会复用原操作标识') + } finally { + syncFlight.current = false + if (!controller.signal.aborted) setSyncSubmitting(false) + } + } const edit = (next: RegionRule[]) => { setRules(next) setPreview(null) @@ -225,6 +347,10 @@ export function QARegions() { } return ( void synchronize()} state={state} preview={preview} rules={rules} @@ -233,7 +359,7 @@ export function QARegions() { busy={busy} error={error} notice={notice} - dirty={Boolean(state && signature(rules) !== signature(state.rules))} + dirty={dirty} onRegion={setRegion} onSearch={setSearch} onToggle={(id, checked) => edit(changeRegionRule(rules, id, region, checked))} diff --git a/web/src/qaOrg.ts b/web/src/qaOrg.ts index aa53d23..1c780c6 100644 --- a/web/src/qaOrg.ts +++ b/web/src/qaOrg.ts @@ -30,20 +30,20 @@ export interface Organization { missing_departments: string[] } } -function invalid(): never { +export function invalid(): never { throw new Error('通讯录或区域配置响应格式无效') } -function obj(value: unknown): Record { +export function obj(value: unknown): Record { if (!value || typeof value !== 'object' || Array.isArray(value)) return invalid() return value as Record } -function str(value: unknown): string { +export function str(value: unknown): string { return typeof value === 'string' ? value : invalid() } -function count(value: unknown): number { +export function count(value: unknown): number { return typeof value === 'number' && Number.isSafeInteger(value) && value >= 0 ? value : invalid() } -function instant(value: unknown): string | null { +export function instant(value: unknown): string | null { return value === null ? null : typeof value === 'string' && Number.isFinite(Date.parse(value)) ? value : invalid() } export function parseOrganization(value: unknown): Organization { diff --git a/web/src/qaOrgSync.test.ts b/web/src/qaOrgSync.test.ts new file mode 100644 index 0000000..a5a2b72 --- /dev/null +++ b/web/src/qaOrgSync.test.ts @@ -0,0 +1,72 @@ +import { describe, expect, it, vi, afterEach } from 'vitest' +import { parseDirectorySync, startDirectorySync, fetchDirectorySync, completedSyncAction } from './qaOrgSync' +afterEach(() => vi.unstubAllGlobals()) +const idle = { configured: true, worker_ready: true, job: null } +describe('directory synchronization contract', () => { + it('reloads a completed job even on first observation, without replacing an unsaved draft', () => { + const status = parseDirectorySync({ + ...idle, + job: { + id: 'new-job', + state: 'completed', + attempt: 1, + max_attempts: 3, + updated_at: '2026-10-17T08:00:00Z', + error_code: '', + result: { directory_id: 'd', departments: 1, people: 1, synced_at: '2026-10-17T08:00:00Z' }, + }, + }) + expect(completedSyncAction(status, '', false)).toBe('reload') + expect(completedSyncAction(status, '', true)).toBe('preserve') + expect(completedSyncAction(status, 'new-job', false)).toBe('none') + expect(completedSyncAction({ ...status, job: { ...status.job!, state: 'running' } }, '', false)).toBe('none') + }) + it('keeps unknown/failed state distinct from empty data', () => { + expect(parseDirectorySync(idle).job).toBeNull() + expect(() => parseDirectorySync({ ...idle, worker_ready: 'yes' })).toThrow() + expect(() => parseDirectorySync({ ...idle, job: { id: 'x', state: 'mystery' } })).toThrow() + const result = { directory_id: 'd', departments: 327, people: 1534, synced_at: '2026-10-17T07:00:00Z' } + const job = { + id: 'j', + state: 'completed', + attempt: 1, + max_attempts: 3, + updated_at: '2026-10-17T07:01:00Z', + error_code: '', + result, + } + expect(parseDirectorySync({ ...idle, job }).job?.result?.people).toBe(1534) + expect(() => parseDirectorySync({ ...idle, job: { ...job, result: { ...result, people: -1 } } })).toThrow() + }) + it('sends a stable idempotency header and no tenant, credential or executable path', async () => { + const fetcher = vi.fn().mockResolvedValue({ + ok: true, + json: async () => ({ data: { job_id: 'j', deduplicated: false }, operation: 'replayed' }), + }) + vi.stubGlobal('fetch', fetcher) + expect((await startDirectorySync('same-action')).operation).toBe('replayed') + expect(fetcher).toHaveBeenCalledWith( + '/api/v1/qa-ops/organization/sync', + expect.objectContaining({ + method: 'POST', + credentials: 'same-origin', + body: '{}', + headers: expect.objectContaining({ + 'Idempotency-Key': 'same-action', + 'X-Governance-Action': 'qa-organization', + }), + }) + ) + }) + it('reads live job state and surfaces server failure instead of a successful empty result', async () => { + vi.stubGlobal( + 'fetch', + vi + .fn() + .mockResolvedValueOnce({ ok: true, json: async () => ({ data: idle }) }) + .mockResolvedValueOnce({ ok: false, status: 503, json: async () => ({ error: '执行器未在线' }) }) + ) + expect((await fetchDirectorySync()).worker_ready).toBe(true) + await expect(startDirectorySync('same-action')).rejects.toThrow('执行器未在线') + }) +}) diff --git a/web/src/qaOrgSync.ts b/web/src/qaOrgSync.ts new file mode 100644 index 0000000..fbed936 --- /dev/null +++ b/web/src/qaOrgSync.ts @@ -0,0 +1,104 @@ +export interface DirectorySyncResult { + directory_id: string + departments: number + people: number + synced_at: string +} +export interface DirectorySyncJob { + id: string + state: 'pending' | 'running' | 'retry' | 'completed' | 'failed' | 'cancelled' + attempt: number + max_attempts: number + updated_at: string + error_code: string + result: DirectorySyncResult | null +} +export interface DirectorySyncStatus { + configured: boolean + worker_ready: boolean + job: DirectorySyncJob | null +} +export function completedSyncAction( + s: DirectorySyncStatus, + last: string, + dirty: boolean +): 'none' | 'preserve' | 'reload' { + if (s.job?.state !== 'completed' || s.job.id === last) return 'none' + return dirty ? 'preserve' : 'reload' +} +export const directorySyncActive = (s: DirectorySyncStatus | null) => + Boolean(s?.job && ['pending', 'running', 'retry'].includes(s.job.state)) +import { invalid as bad, obj as object, str as text, count, instant as nullableInstant } from './qaOrg' +function instant(v: unknown): string { + return nullableInstant(v) ?? bad() +} +export function parseDirectorySync(raw: unknown): DirectorySyncStatus { + const s = object(raw) + if (typeof s.configured !== 'boolean' || typeof s.worker_ready !== 'boolean') return bad() + let job: DirectorySyncJob | null = null + if (s.job !== null) { + const j = object(s.job) + if (!['pending', 'running', 'retry', 'completed', 'failed', 'cancelled'].includes(text(j.state))) return bad() + let result: DirectorySyncResult | null = null + if (j.result !== null) { + const r = object(j.result) + result = { + directory_id: text(r.directory_id), + departments: count(r.departments), + people: count(r.people), + synced_at: instant(r.synced_at), + } + if (!result.directory_id || !result.departments || !result.people) return bad() + } + job = { + id: text(j.id), + state: j.state as DirectorySyncJob['state'], + attempt: count(j.attempt), + max_attempts: count(j.max_attempts), + updated_at: instant(j.updated_at), + error_code: text(j.error_code), + result, + } + if (!job.id || !job.max_attempts || job.attempt > job.max_attempts || (job.state === 'completed' && !result)) + return bad() + } + return { configured: s.configured, worker_ready: s.worker_ready, job } +} +async function request(method: string, operationID?: string, signal?: AbortSignal) { + const response = await fetch('/api/v1/qa-ops/organization/sync', { + method, + credentials: 'same-origin', + cache: 'no-store', + signal, + headers: + method === 'POST' + ? { + 'Content-Type': 'application/json', + 'X-Governance-Action': 'qa-organization', + 'Idempotency-Key': operationID ?? '', + } + : undefined, + body: method === 'POST' ? '{}' : undefined, + }) + const raw: unknown = await response.json().catch(() => null) + if (!response.ok) { + const error = raw && typeof raw === 'object' ? (raw as Record).error : undefined + throw new Error(typeof error === 'string' ? error : `通讯录同步请求失败(HTTP ${response.status})`) + } + return object(raw) +} +export async function fetchDirectorySync(signal?: AbortSignal) { + return parseDirectorySync((await request('GET', undefined, signal)).data) +} +export async function startDirectorySync(operationID: string, signal?: AbortSignal) { + if (!operationID || operationID.trim() !== operationID || operationID.length > 128) return bad() + const envelope = await request('POST', operationID, signal), + data = object(envelope.data) + if ( + (envelope.operation !== 'created' && envelope.operation !== 'replayed') || + typeof data.deduplicated !== 'boolean' || + !text(data.job_id) + ) + return bad() + return { job_id: text(data.job_id), deduplicated: data.deduplicated, operation: envelope.operation } +}