feat(governance): policy-driven ingestion from catalog annotations

Annotation in 来源与目录 is now the only import decision surface:
- migration 000033: catalog_import_intents (durable, rescan-proof per-file
  intent with locator snapshot + fetch lifecycle), source_items.policy_state,
  planned publication state, ui_setting_revisions.ingestion_concurrency
- marking fans a folder decision out to per-file intents, materializes
  versioned items, reuses already-verified local bytes (compare, don't pull),
  and plans one publication per routed KB; blocked retires planned rows
  while live publications stay untouched (需下架 shows in the KB diff)
- artifact.fetch worker lane (scripts/artifact_fetch.py): DingTalk export
  tree/live box API, AList /api/fs/get direct download with SSO-redirect
  fail-closed, PP ledger via locator hash; content-level dedupe on sha256
- 开始处理 promotes planned->pending inside the claim-time concurrency budget
  from system settings; claims enforce it in SQL
- monitor overview rebuilt as knowledge-base diff: planned/needs_removal per
  KB + fetch pipeline summary; frontend lanes 待入库/需下架 replace the
  deprecated snapshot change-set confirmation
- spool: images (png/jpg/gif/bmp/webp) accepted, ppt/pptx rejected in favour
  of PDF twins; DingTalk PPT resolves its converted PDF pair
This commit is contained in:
姚章浩
2026-09-13 22:17:12 +08:00
parent f6cb90cd70
commit 63f3b2c503
38 changed files with 1597 additions and 307 deletions
+7 -7
View File
@@ -348,7 +348,7 @@ func TestHandlerRejectsInvalidCatalogParentBeforeStore(t *testing.T) {
func TestHandlerRoutesSelectedCatalogFilesToServerBoundScopeKB(t *testing.T) {
now := time.Date(2026, 9, 9, 1, 0, 0, 0, time.UTC)
store := &fakeJobStore{catalogRouteResult: governance.CatalogRouteResult{SelectedFiles: 2, RoutableFiles: 2}}
store := &fakeJobStore{catalogRouteResult: governance.CatalogRouteResult{SelectedFiles: 2, PlannedPublications: 2}}
handler, err := NewHandler(Config{Store: store, Now: func() time.Time { return now }, InternalRoutingKnowledgeBaseID: "kb-internal", ExternalRoutingKnowledgeBaseID: "kb-external"})
if err != nil {
t.Fatal(err)
@@ -360,7 +360,7 @@ func TestHandlerRoutesSelectedCatalogFilesToServerBoundScopeKB(t *testing.T) {
if response.Code != http.StatusAccepted || store.catalogRouteRequest.TargetKnowledgeBaseID != "kb-internal" || store.catalogRouteRequest.Now != now || store.catalogRouteRequest.Visibility != governance.VisibilityInternal {
t.Fatalf("status=%d request=%#v body=%s", response.Code, store.catalogRouteRequest, response.Body.String())
}
if strings.Contains(response.Body.String(), "kb-internal") || !strings.Contains(response.Body.String(), `"routable_files":2`) {
if strings.Contains(response.Body.String(), "kb-internal") || !strings.Contains(response.Body.String(), `"planned_publications":2`) {
t.Fatalf("unsafe route response=%s", response.Body.String())
}
capabilities := httptest.NewRecorder()
@@ -372,7 +372,7 @@ func TestHandlerRoutesSelectedCatalogFilesToServerBoundScopeKB(t *testing.T) {
func TestHandlerRoutesBothScopesInOneAtomicRequest(t *testing.T) {
now := time.Date(2026, 9, 9, 1, 30, 0, 0, time.UTC)
store := &fakeJobStore{catalogRouteResult: governance.CatalogRouteResult{SelectedFiles: 1, RoutableFiles: 1}}
store := &fakeJobStore{catalogRouteResult: governance.CatalogRouteResult{SelectedFiles: 1, PlannedPublications: 1}}
handler, err := NewHandler(Config{Store: store, Now: func() time.Time { return now }, InternalRoutingKnowledgeBaseID: "kb-internal", ExternalRoutingKnowledgeBaseID: "kb-external"})
if err != nil {
t.Fatal(err)
@@ -393,7 +393,7 @@ func TestHandlerRoutesBothScopesInOneAtomicRequest(t *testing.T) {
}
func TestHandlerAcceptsBlockedPolicyWithoutAConfiguredTarget(t *testing.T) {
store := &fakeJobStore{catalogRouteResult: governance.CatalogRouteResult{SelectedFiles: 1, RoutableFiles: 1}}
store := &fakeJobStore{catalogRouteResult: governance.CatalogRouteResult{SelectedFiles: 1, BlockedItems: 1}}
handler, err := NewHandler(Config{Store: store})
if err != nil {
t.Fatal(err)
@@ -405,7 +405,7 @@ func TestHandlerAcceptsBlockedPolicyWithoutAConfiguredTarget(t *testing.T) {
if response.Code != http.StatusAccepted || store.catalogRouteRequest.Visibility != governance.VisibilityBlocked || len(store.catalogRouteRequest.TargetKnowledgeBaseIDs) != 0 {
t.Fatalf("status=%d request=%#v body=%s", response.Code, store.catalogRouteRequest, response.Body.String())
}
if !strings.Contains(response.Body.String(), `"routable_files":1`) {
if !strings.Contains(response.Body.String(), `"blocked_items":1`) {
t.Fatalf("blocked route response=%s", response.Body.String())
}
}
@@ -952,12 +952,12 @@ func TestHandlerVersionsAndActivatesUISettings(t *testing.T) {
t.Fatalf("get status=%d body=%s", get.Code, get.Body.String())
}
noIntent := httptest.NewRecorder()
handler.ServeHTTP(noIntent, httptest.NewRequest(http.MethodPost, "/api/v1/ui-settings/candidates", bytes.NewBufferString(`{"document_refresh_seconds":30,"resource_refresh_seconds":60,"stalled_after_seconds":3600,"finalizing_after_seconds":14400}`)))
handler.ServeHTTP(noIntent, httptest.NewRequest(http.MethodPost, "/api/v1/ui-settings/candidates", bytes.NewBufferString(`{"document_refresh_seconds":30,"resource_refresh_seconds":60,"stalled_after_seconds":3600,"finalizing_after_seconds":14400,"ingestion_concurrency":2}`)))
if noIntent.Code != http.StatusForbidden {
t.Fatalf("missing intent status=%d body=%s", noIntent.Code, noIntent.Body.String())
}
create := httptest.NewRecorder()
createRequest := httptest.NewRequest(http.MethodPost, "/api/v1/ui-settings/candidates", bytes.NewBufferString(`{"document_refresh_seconds":30,"resource_refresh_seconds":60,"stalled_after_seconds":3600,"finalizing_after_seconds":14400}`))
createRequest := httptest.NewRequest(http.MethodPost, "/api/v1/ui-settings/candidates", bytes.NewBufferString(`{"document_refresh_seconds":30,"resource_refresh_seconds":60,"stalled_after_seconds":3600,"finalizing_after_seconds":14400,"ingestion_concurrency":2}`))
createRequest.Header.Set("X-Governance-Action", "ui-settings")
createRequest.Header.Set("Idempotency-Key", "op-uis-create")
handler.ServeHTTP(create, createRequest)