feat(governance): policy-driven ingestion from catalog annotations
Annotation in 来源与目录 is now the only import decision surface: - migration 000033: catalog_import_intents (durable, rescan-proof per-file intent with locator snapshot + fetch lifecycle), source_items.policy_state, planned publication state, ui_setting_revisions.ingestion_concurrency - marking fans a folder decision out to per-file intents, materializes versioned items, reuses already-verified local bytes (compare, don't pull), and plans one publication per routed KB; blocked retires planned rows while live publications stay untouched (需下架 shows in the KB diff) - artifact.fetch worker lane (scripts/artifact_fetch.py): DingTalk export tree/live box API, AList /api/fs/get direct download with SSO-redirect fail-closed, PP ledger via locator hash; content-level dedupe on sha256 - 开始处理 promotes planned->pending inside the claim-time concurrency budget from system settings; claims enforce it in SQL - monitor overview rebuilt as knowledge-base diff: planned/needs_removal per KB + fetch pipeline summary; frontend lanes 待入库/需下架 replace the deprecated snapshot change-set confirmation - spool: images (png/jpg/gif/bmp/webp) accepted, ppt/pptx rejected in favour of PDF twins; DingTalk PPT resolves its converted PDF pair
This commit is contained in:
@@ -348,7 +348,7 @@ func TestHandlerRejectsInvalidCatalogParentBeforeStore(t *testing.T) {
|
||||
|
||||
func TestHandlerRoutesSelectedCatalogFilesToServerBoundScopeKB(t *testing.T) {
|
||||
now := time.Date(2026, 9, 9, 1, 0, 0, 0, time.UTC)
|
||||
store := &fakeJobStore{catalogRouteResult: governance.CatalogRouteResult{SelectedFiles: 2, RoutableFiles: 2}}
|
||||
store := &fakeJobStore{catalogRouteResult: governance.CatalogRouteResult{SelectedFiles: 2, PlannedPublications: 2}}
|
||||
handler, err := NewHandler(Config{Store: store, Now: func() time.Time { return now }, InternalRoutingKnowledgeBaseID: "kb-internal", ExternalRoutingKnowledgeBaseID: "kb-external"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -360,7 +360,7 @@ func TestHandlerRoutesSelectedCatalogFilesToServerBoundScopeKB(t *testing.T) {
|
||||
if response.Code != http.StatusAccepted || store.catalogRouteRequest.TargetKnowledgeBaseID != "kb-internal" || store.catalogRouteRequest.Now != now || store.catalogRouteRequest.Visibility != governance.VisibilityInternal {
|
||||
t.Fatalf("status=%d request=%#v body=%s", response.Code, store.catalogRouteRequest, response.Body.String())
|
||||
}
|
||||
if strings.Contains(response.Body.String(), "kb-internal") || !strings.Contains(response.Body.String(), `"routable_files":2`) {
|
||||
if strings.Contains(response.Body.String(), "kb-internal") || !strings.Contains(response.Body.String(), `"planned_publications":2`) {
|
||||
t.Fatalf("unsafe route response=%s", response.Body.String())
|
||||
}
|
||||
capabilities := httptest.NewRecorder()
|
||||
@@ -372,7 +372,7 @@ func TestHandlerRoutesSelectedCatalogFilesToServerBoundScopeKB(t *testing.T) {
|
||||
|
||||
func TestHandlerRoutesBothScopesInOneAtomicRequest(t *testing.T) {
|
||||
now := time.Date(2026, 9, 9, 1, 30, 0, 0, time.UTC)
|
||||
store := &fakeJobStore{catalogRouteResult: governance.CatalogRouteResult{SelectedFiles: 1, RoutableFiles: 1}}
|
||||
store := &fakeJobStore{catalogRouteResult: governance.CatalogRouteResult{SelectedFiles: 1, PlannedPublications: 1}}
|
||||
handler, err := NewHandler(Config{Store: store, Now: func() time.Time { return now }, InternalRoutingKnowledgeBaseID: "kb-internal", ExternalRoutingKnowledgeBaseID: "kb-external"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -393,7 +393,7 @@ func TestHandlerRoutesBothScopesInOneAtomicRequest(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestHandlerAcceptsBlockedPolicyWithoutAConfiguredTarget(t *testing.T) {
|
||||
store := &fakeJobStore{catalogRouteResult: governance.CatalogRouteResult{SelectedFiles: 1, RoutableFiles: 1}}
|
||||
store := &fakeJobStore{catalogRouteResult: governance.CatalogRouteResult{SelectedFiles: 1, BlockedItems: 1}}
|
||||
handler, err := NewHandler(Config{Store: store})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -405,7 +405,7 @@ func TestHandlerAcceptsBlockedPolicyWithoutAConfiguredTarget(t *testing.T) {
|
||||
if response.Code != http.StatusAccepted || store.catalogRouteRequest.Visibility != governance.VisibilityBlocked || len(store.catalogRouteRequest.TargetKnowledgeBaseIDs) != 0 {
|
||||
t.Fatalf("status=%d request=%#v body=%s", response.Code, store.catalogRouteRequest, response.Body.String())
|
||||
}
|
||||
if !strings.Contains(response.Body.String(), `"routable_files":1`) {
|
||||
if !strings.Contains(response.Body.String(), `"blocked_items":1`) {
|
||||
t.Fatalf("blocked route response=%s", response.Body.String())
|
||||
}
|
||||
}
|
||||
@@ -952,12 +952,12 @@ func TestHandlerVersionsAndActivatesUISettings(t *testing.T) {
|
||||
t.Fatalf("get status=%d body=%s", get.Code, get.Body.String())
|
||||
}
|
||||
noIntent := httptest.NewRecorder()
|
||||
handler.ServeHTTP(noIntent, httptest.NewRequest(http.MethodPost, "/api/v1/ui-settings/candidates", bytes.NewBufferString(`{"document_refresh_seconds":30,"resource_refresh_seconds":60,"stalled_after_seconds":3600,"finalizing_after_seconds":14400}`)))
|
||||
handler.ServeHTTP(noIntent, httptest.NewRequest(http.MethodPost, "/api/v1/ui-settings/candidates", bytes.NewBufferString(`{"document_refresh_seconds":30,"resource_refresh_seconds":60,"stalled_after_seconds":3600,"finalizing_after_seconds":14400,"ingestion_concurrency":2}`)))
|
||||
if noIntent.Code != http.StatusForbidden {
|
||||
t.Fatalf("missing intent status=%d body=%s", noIntent.Code, noIntent.Body.String())
|
||||
}
|
||||
create := httptest.NewRecorder()
|
||||
createRequest := httptest.NewRequest(http.MethodPost, "/api/v1/ui-settings/candidates", bytes.NewBufferString(`{"document_refresh_seconds":30,"resource_refresh_seconds":60,"stalled_after_seconds":3600,"finalizing_after_seconds":14400}`))
|
||||
createRequest := httptest.NewRequest(http.MethodPost, "/api/v1/ui-settings/candidates", bytes.NewBufferString(`{"document_refresh_seconds":30,"resource_refresh_seconds":60,"stalled_after_seconds":3600,"finalizing_after_seconds":14400,"ingestion_concurrency":2}`))
|
||||
createRequest.Header.Set("X-Governance-Action", "ui-settings")
|
||||
createRequest.Header.Set("Idempotency-Key", "op-uis-create")
|
||||
handler.ServeHTTP(create, createRequest)
|
||||
|
||||
Reference in New Issue
Block a user