From 63f3b2c5032857b72b063ce942e311c9767834a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A7=9A=E7=AB=A0=E6=B5=A9?= Date: Sun, 13 Sep 2026 22:17:12 +0800 Subject: [PATCH] feat(governance): policy-driven ingestion from catalog annotations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Annotation in 来源与目录 is now the only import decision surface: - migration 000033: catalog_import_intents (durable, rescan-proof per-file intent with locator snapshot + fetch lifecycle), source_items.policy_state, planned publication state, ui_setting_revisions.ingestion_concurrency - marking fans a folder decision out to per-file intents, materializes versioned items, reuses already-verified local bytes (compare, don't pull), and plans one publication per routed KB; blocked retires planned rows while live publications stay untouched (需下架 shows in the KB diff) - artifact.fetch worker lane (scripts/artifact_fetch.py): DingTalk export tree/live box API, AList /api/fs/get direct download with SSO-redirect fail-closed, PP ledger via locator hash; content-level dedupe on sha256 - 开始处理 promotes planned->pending inside the claim-time concurrency budget from system settings; claims enforce it in SQL - monitor overview rebuilt as knowledge-base diff: planned/needs_removal per KB + fetch pipeline summary; frontend lanes 待入库/需下架 replace the deprecated snapshot change-set confirmation - spool: images (png/jpg/gif/bmp/webp) accepted, ppt/pptx rejected in favour of PDF twins; DingTalk PPT resolves its converted PDF pair --- .gitignore | 1 + Dockerfile | 2 +- cmd/worker/main.go | 26 + compose.yaml | 2 + internal/appconfig/config.go | 4 + internal/governance/monitor.go | 13 + internal/governance/source_catalog.go | 30 +- internal/governance/ui_settings.go | 3 + internal/httpapi/server.go | 2 + internal/httpapi/server_test.go | 14 +- internal/monitoring/real.go | 38 +- .../platform/postgres/artifact_fetch_store.go | 39 ++ internal/platform/postgres/catalog_routing.go | 62 +-- .../catalog_routing_integration_test.go | 106 ++-- ...alog_routing_operation_integration_test.go | 6 +- .../000033_catalog_policy_intent.down.sql | 9 + .../000033_catalog_policy_intent.up.sql | 62 +++ internal/platform/postgres/migrations_test.go | 7 +- internal/platform/postgres/policy_intents.go | 364 +++++++++++++ .../postgres/processing_confirmations.go | 20 +- internal/platform/postgres/queue_sql.go | 30 +- .../s6_scope_policy_integration_test.go | 2 +- .../platform/postgres/test_publications.go | 6 +- internal/platform/postgres/ui_settings.go | 10 +- .../postgres/ui_settings_integration_test.go | 2 +- .../platform/postgres/ui_settings_test.go | 6 +- internal/platform/spool/artifact.go | 33 +- internal/workflows/artifactfetch/handler.go | 247 +++++++++ scripts/artifact_fetch.py | 515 ++++++++++++++++++ web/src/App.tsx | 1 + web/src/SystemSettings.test.tsx | 2 +- web/src/SystemSettings.tsx | 9 +- web/src/WeKnoraMonitor.test.tsx | 53 +- web/src/WeKnoraMonitor.tsx | 159 +++--- web/src/monitor.ts | 5 +- web/src/testPublications.ts | 4 +- web/src/uiSettings.test.ts | 6 +- web/src/uiSettings.ts | 4 + 38 files changed, 1597 insertions(+), 307 deletions(-) create mode 100644 internal/platform/postgres/artifact_fetch_store.go create mode 100644 internal/platform/postgres/migrations/000033_catalog_policy_intent.down.sql create mode 100644 internal/platform/postgres/migrations/000033_catalog_policy_intent.up.sql create mode 100644 internal/platform/postgres/policy_intents.go create mode 100644 internal/workflows/artifactfetch/handler.go create mode 100644 scripts/artifact_fetch.py diff --git a/.gitignore b/.gitignore index abf104d..4a2d2f2 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,4 @@ /tmp_* .style-backup-*/ .hermes/ +__pycache__/ diff --git a/Dockerfile b/Dockerfile index 1a69c50..58f5713 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,6 +18,6 @@ RUN apk add --no-cache ca-certificates python3 postgresql-client \ && adduser -D -H -u 65532 -G governance governance WORKDIR /app COPY --from=build /out/ /app/ -COPY scripts/source_catalog_import.py scripts/live_catalog_refresh.py /app/scripts/ +COPY scripts/source_catalog_import.py scripts/live_catalog_refresh.py scripts/artifact_fetch.py /app/scripts/ USER 65532:65532 CMD ["/app/api"] diff --git a/cmd/worker/main.go b/cmd/worker/main.go index 9a6c4ac..7f0e948 100644 --- a/cmd/worker/main.go +++ b/cmd/worker/main.go @@ -6,6 +6,7 @@ import ( "log" "os" "os/signal" + "path/filepath" "syscall" "github.com/jackc/pgx/v5/pgxpool" @@ -14,6 +15,7 @@ import ( "knowledge-governance/internal/jobs" postgresstore "knowledge-governance/internal/platform/postgres" "knowledge-governance/internal/platform/weknora" + artifactfetch "knowledge-governance/internal/workflows/artifactfetch" catalogrefresh "knowledge-governance/internal/workflows/catalogrefresh" "knowledge-governance/internal/workflows/sourcescan" "knowledge-governance/internal/workflows/testweknora" @@ -83,6 +85,30 @@ func workerHandlers(store *postgresstore.Store, config appconfig.Config) (map[st return nil, fmt.Errorf("create catalog refresh handler: %w", err) } handlers["source.catalog.refresh"] = catalogHandler.Handle + ingestionRoot := config.IngestionSpoolRoot + if ingestionRoot == "" { + ingestionRoot = config.GovernanceSpoolRoot + } + if !filepath.IsAbs(ingestionRoot) || ingestionRoot == "" { + ingestionRoot = "/var/lib/governance/ingestion" + } + fetchMaxBytes := config.ArtifactFetchMaxBytes + if config.TestWeKnoraEnabled && config.TestWeKnoraMaxUploadBytes > 0 && fetchMaxBytes > config.TestWeKnoraMaxUploadBytes { + fetchMaxBytes = config.TestWeKnoraMaxUploadBytes + } + if fetchMaxBytes <= 0 { + fetchMaxBytes = 50 * 1024 * 1024 + } + artifactRunner, err := artifactfetch.NewCommandRunner("/usr/bin/python3", "/app/scripts/artifact_fetch.py", + ingestionRoot, "ingestion", fetchMaxBytes) + if err != nil { + return nil, fmt.Errorf("create artifact fetch runner: %w", err) + } + artifactHandler, err := artifactfetch.NewHandler(artifactfetch.Config{Store: store, Runner: artifactRunner}) + if err != nil { + return nil, fmt.Errorf("create artifact fetch handler: %w", err) + } + handlers["artifact.fetch"] = artifactHandler.Handle if config.PPScanEnabled { scanner, err := sourcescan.NewFilePPScanner(sourcescan.FilePPScannerConfig{ AllowedRoot: config.PPScanAllowedRoot, diff --git a/compose.yaml b/compose.yaml index 6cdebe1..0c0ef91 100644 --- a/compose.yaml +++ b/compose.yaml @@ -80,6 +80,7 @@ services: TEST_WEKNORA_KB_ID: "${TEST_WEKNORA_KB_ID:-}" TEST_WEKNORA_API_KEY_FILE: /run/secrets/test_weknora_api_key GOVERNANCE_SPOOL_ROOT: /var/lib/governance/spool + INGESTION_SPOOL_ROOT: /var/lib/governance/ingestion TEST_WEKNORA_HTTP_TIMEOUT: "${TEST_WEKNORA_HTTP_TIMEOUT:-60s}" TEST_WEKNORA_POLL_INITIAL: "${TEST_WEKNORA_POLL_INITIAL:-10s}" TEST_WEKNORA_POLL_MAX: "${TEST_WEKNORA_POLL_MAX:-60s}" @@ -91,6 +92,7 @@ services: - test_weknora_api_key volumes: - ./spool:/var/lib/governance/spool:ro + - ./spool/ingestion:/var/lib/governance/ingestion - "${PP_SCAN_HOST_DATA_ROOT:-/home/zheromiku/dobot-portal-sync/data}:${PP_SCAN_CONTAINER_DATA_ROOT:-/home/zheromiku/dobot-portal-sync/data}:ro" - /home/zheromiku/dingtalk_kb_download:/sources/dingtalk:ro - /home/zheromiku/tmp/alist:/sources/alist:ro diff --git a/internal/appconfig/config.go b/internal/appconfig/config.go index 6f960fa..abb4a1c 100644 --- a/internal/appconfig/config.go +++ b/internal/appconfig/config.go @@ -36,6 +36,8 @@ type Config struct { ExternalRoutingKnowledgeBaseID string TestWeKnoraAPIKeyFile string GovernanceSpoolRoot string + IngestionSpoolRoot string + ArtifactFetchMaxBytes int64 WorkerLivenessStaleAfter time.Duration TestWeKnoraHTTPTimeout time.Duration TestWeKnoraPollInitial time.Duration @@ -67,6 +69,8 @@ func Load(getenv func(string) string) (Config, error) { ExternalRoutingKnowledgeBaseID: strings.TrimSpace(getenv("TEST_EXTERNAL_WEKNORA_KB_ID")), TestWeKnoraAPIKeyFile: valueOrDefault(getenv("TEST_WEKNORA_API_KEY_FILE"), "/run/secrets/test_weknora_api_key"), GovernanceSpoolRoot: valueOrDefault(getenv("GOVERNANCE_SPOOL_ROOT"), "/var/lib/governance/spool"), + IngestionSpoolRoot: strings.TrimSpace(getenv("INGESTION_SPOOL_ROOT")), + ArtifactFetchMaxBytes: 200 * 1024 * 1024, WorkerLivenessStaleAfter: 2 * time.Minute, TestWeKnoraHTTPTimeout: 60 * time.Second, TestWeKnoraPollInitial: 10 * time.Second, diff --git a/internal/governance/monitor.go b/internal/governance/monitor.go index 6dd254f..940d3e1 100644 --- a/internal/governance/monitor.go +++ b/internal/governance/monitor.go @@ -51,12 +51,24 @@ type KnowledgeBaseProgress struct { Finalizing int `json:"finalizing"` Failed int `json:"failed"` Cancelled int `json:"cancelled"` + Planned int `json:"planned"` + NeedsRemoval int `json:"needs_removal"` + StalePlanned int `json:"stale_planned"` InternalImported int `json:"internal_imported"` InternalCompleted int `json:"internal_completed"` ExternalImported int `json:"external_imported"` ExternalCompleted int `json:"external_completed"` } +// IngestionPipeline summarizes the artifact-fetch lane that sits in front of +// 待入库 so operators can see why planned items are not uploadable yet. +type IngestionPipeline struct { + FetchPending int `json:"fetch_pending"` + FetchFailed int `json:"fetch_failed"` + FetchMissing int `json:"fetch_missing"` + Concurrency int `json:"concurrency"` +} + // JobWindowMetrics reports a frozen statistical window; insufficient samples // stay null and are never rendered as 0. type JobWindowMetrics struct { @@ -95,6 +107,7 @@ type MonitorOverview struct { Host MonitorHost `json:"host"` Services []MonitorService `json:"services"` KnowledgeBases []KnowledgeBaseProgress `json:"knowledge_bases"` + Pipeline IngestionPipeline `json:"pipeline"` Health []ServiceHealth `json:"health"` } diff --git a/internal/governance/source_catalog.go b/internal/governance/source_catalog.go index a371ce6..6edc1a8 100644 --- a/internal/governance/source_catalog.go +++ b/internal/governance/source_catalog.go @@ -94,15 +94,29 @@ type CatalogRouteRequest struct { Now time.Time } -// CatalogRouteResult describes the desired-state write only. S7 guarantees the -// annotation never creates publications or jobs; counts explain what the -// subsequent ChangeSet/P3 pipeline will be able to act on. +// CatalogRouteFile is one selected file plus the metadata snapshot the policy +// fulfillment path needs to materialize and fetch it later. +type CatalogRouteFile struct { + NodeID string + Name string + RelativePath string + SizeBytes *int64 + ModifiedAt *time.Time + ContentSHA256 *string + LocatorHash string + Direct bool +} + +// CatalogRouteResult describes the desired-state write only. Policy fulfillment +// records intents, queues artifact fetches and plans publications; counts tell +// the operator what will show up in 解析监控. type CatalogRouteResult struct { - SelectedFiles int `json:"selected_files"` - RoutableFiles int `json:"routable_files"` - UnboundFiles int `json:"unbound_files"` - HashUnknownFiles int `json:"hash_unknown_files"` - UnavailableFiles int `json:"unavailable_files"` + SelectedFiles int `json:"selected_files"` + IntentsRecorded int `json:"intents_recorded"` + FetchQueued int `json:"fetch_queued"` + PlannedPublications int `json:"planned_publications"` + BlockedItems int `json:"blocked_items"` + UnboundFiles int `json:"unbound_files"` } type CatalogChangeSetFilter struct { diff --git a/internal/governance/ui_settings.go b/internal/governance/ui_settings.go index 43d2968..1a5fdaf 100644 --- a/internal/governance/ui_settings.go +++ b/internal/governance/ui_settings.go @@ -12,6 +12,7 @@ type UISettings struct { ResourceRefreshSeconds int `json:"resource_refresh_seconds"` StalledAfterSeconds int `json:"stalled_after_seconds"` FinalizingAfterSeconds int `json:"finalizing_after_seconds"` + IngestionConcurrency int `json:"ingestion_concurrency"` UpdatedBy string `json:"updated_by"` CreatedAt time.Time `json:"created_at"` ActivatedAt *time.Time `json:"activated_at"` @@ -24,6 +25,7 @@ type CreateUISettingsCandidateRequest struct { ResourceRefreshSeconds int StalledAfterSeconds int FinalizingAfterSeconds int + IngestionConcurrency int UpdatedBy string Now time.Time } @@ -33,6 +35,7 @@ func (request CreateUISettingsCandidateRequest) Valid() bool { oneOfInt(request.ResourceRefreshSeconds, 30, 60) && oneOfInt(request.StalledAfterSeconds, 1800, 3600) && oneOfInt(request.FinalizingAfterSeconds, 7200, 14400) && + request.IngestionConcurrency >= 1 && request.IngestionConcurrency <= 16 && validSettingsActor(request.UpdatedBy) && !request.Now.IsZero() } diff --git a/internal/httpapi/server.go b/internal/httpapi/server.go index e7ea532..443db8b 100644 --- a/internal/httpapi/server.go +++ b/internal/httpapi/server.go @@ -1139,6 +1139,7 @@ type createUISettingsCandidateRequest struct { ResourceRefreshSeconds int `json:"resource_refresh_seconds"` StalledAfterSeconds int `json:"stalled_after_seconds"` FinalizingAfterSeconds int `json:"finalizing_after_seconds"` + IngestionConcurrency int `json:"ingestion_concurrency"` } func getActiveUISettings(w http.ResponseWriter, r *http.Request, store Store) { @@ -1169,6 +1170,7 @@ func createUISettingsCandidate(w http.ResponseWriter, r *http.Request, config Co ResourceRefreshSeconds: request.ResourceRefreshSeconds, StalledAfterSeconds: request.StalledAfterSeconds, FinalizingAfterSeconds: request.FinalizingAfterSeconds, + IngestionConcurrency: request.IngestionConcurrency, UpdatedBy: actorFor(r), Now: config.Now(), }, operationID) diff --git a/internal/httpapi/server_test.go b/internal/httpapi/server_test.go index 93cf441..678bfe7 100644 --- a/internal/httpapi/server_test.go +++ b/internal/httpapi/server_test.go @@ -348,7 +348,7 @@ func TestHandlerRejectsInvalidCatalogParentBeforeStore(t *testing.T) { func TestHandlerRoutesSelectedCatalogFilesToServerBoundScopeKB(t *testing.T) { now := time.Date(2026, 9, 9, 1, 0, 0, 0, time.UTC) - store := &fakeJobStore{catalogRouteResult: governance.CatalogRouteResult{SelectedFiles: 2, RoutableFiles: 2}} + store := &fakeJobStore{catalogRouteResult: governance.CatalogRouteResult{SelectedFiles: 2, PlannedPublications: 2}} handler, err := NewHandler(Config{Store: store, Now: func() time.Time { return now }, InternalRoutingKnowledgeBaseID: "kb-internal", ExternalRoutingKnowledgeBaseID: "kb-external"}) if err != nil { t.Fatal(err) @@ -360,7 +360,7 @@ func TestHandlerRoutesSelectedCatalogFilesToServerBoundScopeKB(t *testing.T) { if response.Code != http.StatusAccepted || store.catalogRouteRequest.TargetKnowledgeBaseID != "kb-internal" || store.catalogRouteRequest.Now != now || store.catalogRouteRequest.Visibility != governance.VisibilityInternal { t.Fatalf("status=%d request=%#v body=%s", response.Code, store.catalogRouteRequest, response.Body.String()) } - if strings.Contains(response.Body.String(), "kb-internal") || !strings.Contains(response.Body.String(), `"routable_files":2`) { + if strings.Contains(response.Body.String(), "kb-internal") || !strings.Contains(response.Body.String(), `"planned_publications":2`) { t.Fatalf("unsafe route response=%s", response.Body.String()) } capabilities := httptest.NewRecorder() @@ -372,7 +372,7 @@ func TestHandlerRoutesSelectedCatalogFilesToServerBoundScopeKB(t *testing.T) { func TestHandlerRoutesBothScopesInOneAtomicRequest(t *testing.T) { now := time.Date(2026, 9, 9, 1, 30, 0, 0, time.UTC) - store := &fakeJobStore{catalogRouteResult: governance.CatalogRouteResult{SelectedFiles: 1, RoutableFiles: 1}} + store := &fakeJobStore{catalogRouteResult: governance.CatalogRouteResult{SelectedFiles: 1, PlannedPublications: 1}} handler, err := NewHandler(Config{Store: store, Now: func() time.Time { return now }, InternalRoutingKnowledgeBaseID: "kb-internal", ExternalRoutingKnowledgeBaseID: "kb-external"}) if err != nil { t.Fatal(err) @@ -393,7 +393,7 @@ func TestHandlerRoutesBothScopesInOneAtomicRequest(t *testing.T) { } func TestHandlerAcceptsBlockedPolicyWithoutAConfiguredTarget(t *testing.T) { - store := &fakeJobStore{catalogRouteResult: governance.CatalogRouteResult{SelectedFiles: 1, RoutableFiles: 1}} + store := &fakeJobStore{catalogRouteResult: governance.CatalogRouteResult{SelectedFiles: 1, BlockedItems: 1}} handler, err := NewHandler(Config{Store: store}) if err != nil { t.Fatal(err) @@ -405,7 +405,7 @@ func TestHandlerAcceptsBlockedPolicyWithoutAConfiguredTarget(t *testing.T) { if response.Code != http.StatusAccepted || store.catalogRouteRequest.Visibility != governance.VisibilityBlocked || len(store.catalogRouteRequest.TargetKnowledgeBaseIDs) != 0 { t.Fatalf("status=%d request=%#v body=%s", response.Code, store.catalogRouteRequest, response.Body.String()) } - if !strings.Contains(response.Body.String(), `"routable_files":1`) { + if !strings.Contains(response.Body.String(), `"blocked_items":1`) { t.Fatalf("blocked route response=%s", response.Body.String()) } } @@ -952,12 +952,12 @@ func TestHandlerVersionsAndActivatesUISettings(t *testing.T) { t.Fatalf("get status=%d body=%s", get.Code, get.Body.String()) } noIntent := httptest.NewRecorder() - handler.ServeHTTP(noIntent, httptest.NewRequest(http.MethodPost, "/api/v1/ui-settings/candidates", bytes.NewBufferString(`{"document_refresh_seconds":30,"resource_refresh_seconds":60,"stalled_after_seconds":3600,"finalizing_after_seconds":14400}`))) + handler.ServeHTTP(noIntent, httptest.NewRequest(http.MethodPost, "/api/v1/ui-settings/candidates", bytes.NewBufferString(`{"document_refresh_seconds":30,"resource_refresh_seconds":60,"stalled_after_seconds":3600,"finalizing_after_seconds":14400,"ingestion_concurrency":2}`))) if noIntent.Code != http.StatusForbidden { t.Fatalf("missing intent status=%d body=%s", noIntent.Code, noIntent.Body.String()) } create := httptest.NewRecorder() - createRequest := httptest.NewRequest(http.MethodPost, "/api/v1/ui-settings/candidates", bytes.NewBufferString(`{"document_refresh_seconds":30,"resource_refresh_seconds":60,"stalled_after_seconds":3600,"finalizing_after_seconds":14400}`)) + createRequest := httptest.NewRequest(http.MethodPost, "/api/v1/ui-settings/candidates", bytes.NewBufferString(`{"document_refresh_seconds":30,"resource_refresh_seconds":60,"stalled_after_seconds":3600,"finalizing_after_seconds":14400,"ingestion_concurrency":2}`)) createRequest.Header.Set("X-Governance-Action", "ui-settings") createRequest.Header.Set("Idempotency-Key", "op-uis-create") handler.ServeHTTP(create, createRequest) diff --git a/internal/monitoring/real.go b/internal/monitoring/real.go index 89e4387..421cd1d 100644 --- a/internal/monitoring/real.go +++ b/internal/monitoring/real.go @@ -82,17 +82,24 @@ func (provider *RealProvider) Overview(ctx context.Context) (governance.MonitorO ) rows, err := provider.db.Query(ctx, `SELECT p.test_kb_id, CASE WHEN COUNT(DISTINCT p.target_scope)=1 THEN MIN(p.target_scope) ELSE 'mixed' END, - COUNT(*)::integer, - COUNT(*) FILTER (WHERE p.processing_state='completed')::integer, - COUNT(*) FILTER (WHERE p.processing_state='pending')::integer, - COUNT(*) FILTER (WHERE p.processing_state='processing')::integer, - COUNT(*) FILTER (WHERE p.processing_state='finalizing')::integer, - COUNT(*) FILTER (WHERE p.processing_state IN ('failed','stalled'))::integer, - COUNT(*) FILTER (WHERE p.target_scope='internal')::integer, - COUNT(*) FILTER (WHERE p.target_scope='internal' AND p.processing_state='completed')::integer, - COUNT(*) FILTER (WHERE p.target_scope='external')::integer, - COUNT(*) FILTER (WHERE p.target_scope='external' AND p.processing_state='completed')::integer + COUNT(*) FILTER (WHERE p.superseded_at IS NULL)::integer, + COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.processing_state='completed')::integer, + COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.processing_state='pending')::integer, + COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.processing_state='processing')::integer, + COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.processing_state='finalizing')::integer, + COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.processing_state IN ('failed','stalled'))::integer, + COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.target_scope='internal')::integer, + COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.target_scope='internal' AND p.processing_state='completed')::integer, + COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.target_scope='external')::integer, + COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.target_scope='external' AND p.processing_state='completed')::integer, + COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.processing_state='planned')::integer, + COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.retrieval_state='enabled' + AND item.policy_state='blocked')::integer, + COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.processing_state='planned' + AND item.policy_state='blocked')::integer FROM test_publications p +JOIN source_item_versions version ON version.id=p.source_item_version_id +JOIN source_items item ON item.id=version.source_item_id GROUP BY p.test_kb_id ORDER BY p.test_kb_id`) if err != nil { @@ -101,7 +108,8 @@ ORDER BY p.test_kb_id`) defer rows.Close() for rows.Next() { var item governance.KnowledgeBaseProgress - if err := rows.Scan(&item.ID, &item.Visibility, &item.Imported, &item.Completed, &item.Pending, &item.Processing, &item.Finalizing, &item.Failed, &item.InternalImported, &item.InternalCompleted, &item.ExternalImported, &item.ExternalCompleted); err != nil { + if err := rows.Scan(&item.ID, &item.Visibility, &item.Imported, &item.Completed, &item.Pending, &item.Processing, &item.Finalizing, &item.Failed, &item.InternalImported, &item.InternalCompleted, &item.ExternalImported, &item.ExternalCompleted, + &item.Planned, &item.NeedsRemoval, &item.StalePlanned); err != nil { return governance.MonitorOverview{}, fmt.Errorf("scan monitored knowledge base: %w", err) } switch item.Visibility { @@ -117,6 +125,14 @@ ORDER BY p.test_kb_id`) if err := rows.Err(); err != nil { return governance.MonitorOverview{}, fmt.Errorf("iterate monitored knowledge bases: %w", err) } + if err := provider.db.QueryRow(ctx, `SELECT + COUNT(*) FILTER (WHERE fetch_state IN ('pending','fetching'))::integer, + COUNT(*) FILTER (WHERE fetch_state='failed')::integer, + COUNT(*) FILTER (WHERE fetch_state='missing_local')::integer, + COALESCE(MAX((SELECT ingestion_concurrency FROM ui_setting_revisions WHERE state='active' LIMIT 1)),2)::integer +FROM catalog_import_intents`).Scan(&overview.Pipeline.FetchPending, &overview.Pipeline.FetchFailed, &overview.Pipeline.FetchMissing, &overview.Pipeline.Concurrency); err != nil { + return governance.MonitorOverview{}, fmt.Errorf("summarize ingestion pipeline: %w", err) + } return overview, nil } diff --git a/internal/platform/postgres/artifact_fetch_store.go b/internal/platform/postgres/artifact_fetch_store.go new file mode 100644 index 0000000..66e43b7 --- /dev/null +++ b/internal/platform/postgres/artifact_fetch_store.go @@ -0,0 +1,39 @@ +package postgres + +import ( + "context" + "errors" + "fmt" + + "github.com/jackc/pgx/v5" + + "knowledge-governance/internal/workflows/artifactfetch" +) + +// LoadArtifactFetchManifest resolves one pending intent plus its live source +// coordinates for the fetch script, keyed by the connection-stable node id. +func (s *Store) LoadArtifactFetchManifest(ctx context.Context, connectionID, nodeID string) (artifactfetch.Manifest, error) { + return loadArtifactFetchManifest(ctx, s.db, connectionID, nodeID) +} + +func loadArtifactFetchManifest(ctx context.Context, db interface { + QueryRow(context.Context, string, ...any) pgx.Row +}, connectionID, nodeID string) (artifactfetch.Manifest, error) { + var manifest artifactfetch.Manifest + err := db.QueryRow(ctx, `SELECT connection.source_type,connection.name,intent.relative_path,intent.name, + intent.size_bytes,intent.modified_at,intent.locator_hash +FROM catalog_import_intents intent +JOIN source_connections connection ON connection.id=intent.source_connection_id +WHERE intent.source_connection_id=$1 AND intent.node_id=$2`, connectionID, nodeID). + Scan(&manifest.SourceType, &manifest.SourceName, &manifest.RelativePath, &manifest.Name, + &manifest.SizeBytes, &manifest.ModifiedAt, &manifest.LocatorHash) + if errors.Is(err, pgx.ErrNoRows) { + return artifactfetch.Manifest{}, artifactfetch.ErrManifestGone + } + if err != nil { + return artifactfetch.Manifest{}, fmt.Errorf("load artifact fetch manifest: %w", err) + } + manifest.ConnectionID = connectionID + manifest.NodeID = nodeID + return manifest, nil +} diff --git a/internal/platform/postgres/catalog_routing.go b/internal/platform/postgres/catalog_routing.go index b96fbb5..91ab2ab 100644 --- a/internal/platform/postgres/catalog_routing.go +++ b/internal/platform/postgres/catalog_routing.go @@ -112,7 +112,8 @@ FOR UPDATE OF node`, request.ConnectionID, nodeIDs) JOIN source_catalog_nodes child ON child.snapshot_id=parent.snapshot_id AND child.parent_node_id=parent.node_id WHERE parent.node_type='folder' ) -SELECT DISTINCT node.node_id,node.source_item_id,(node.node_id=ANY($2::text[])) AS directly_selected +SELECT DISTINCT node.node_id,node.name,node.relative_path,node.size_bytes,node.modified_at,node.content_sha256, + node.source_locator_hash,(node.node_id=ANY($2::text[])) AS directly_selected FROM descendants JOIN source_catalog_nodes node ON node.snapshot_id=descendants.snapshot_id AND node.node_id=descendants.node_id WHERE node.node_type='file' @@ -120,15 +121,11 @@ ORDER BY node.node_id`, request.ConnectionID, nodeIDs) if err != nil { return governance.CatalogRouteResult{}, fmt.Errorf("select catalog route files: %w", err) } - type selectedFile struct { - nodeID string - sourceItemID *string - direct bool - } - files := make([]selectedFile, 0, len(nodeIDs)) + files := make([]governance.CatalogRouteFile, 0, len(nodeIDs)) for rows.Next() { - var file selectedFile - if err := rows.Scan(&file.nodeID, &file.sourceItemID, &file.direct); err != nil { + var file governance.CatalogRouteFile + if err := rows.Scan(&file.NodeID, &file.Name, &file.RelativePath, &file.SizeBytes, &file.ModifiedAt, + &file.ContentSHA256, &file.LocatorHash, &file.Direct); err != nil { rows.Close() return governance.CatalogRouteResult{}, fmt.Errorf("scan catalog route file: %w", err) } @@ -139,52 +136,7 @@ ORDER BY node.node_id`, request.ConnectionID, nodeIDs) return governance.CatalogRouteResult{}, fmt.Errorf("iterate catalog route files: %w", err) } rows.Close() - - result := governance.CatalogRouteResult{SelectedFiles: len(files)} - routedItems := map[string]bool{} - for _, file := range files { - if file.sourceItemID == nil { - result.UnboundFiles++ - continue - } - itemID := *file.sourceItemID - if routedItems[itemID] { - continue - } - routedItems[itemID] = true - if file.direct { - if _, err := tx.Exec(ctx, `INSERT INTO source_item_scope_overrides(source_item_id,visibility,updated_by,reason,created_at,updated_at) -VALUES($1,$2,$3,$4,$5,$5) ON CONFLICT(source_item_id) DO UPDATE SET visibility=EXCLUDED.visibility,updated_by=EXCLUDED.updated_by,reason=EXCLUDED.reason,updated_at=EXCLUDED.updated_at`, itemID, request.Visibility, request.OperatorRef, request.Reason, request.Now); err != nil { - return governance.CatalogRouteResult{}, fmt.Errorf("set routed item scope: %w", err) - } - eventID, err := newOpaqueID("se-") - if err != nil { - return governance.CatalogRouteResult{}, err - } - if _, err := tx.Exec(ctx, `INSERT INTO scope_change_events(id,target_type,source_item_id,action,decided_visibility,operator_ref,reason,created_at) -VALUES($1,'item',$2,'set',$3,$4,$5,$6)`, eventID, itemID, request.Visibility, request.OperatorRef, request.Reason, request.Now); err != nil { - return governance.CatalogRouteResult{}, fmt.Errorf("audit routed item scope: %w", err) - } - } - // S7: classify only. Annotations never create publications, retire targets, or enqueue jobs. - var currentVersionID *string - var contentState, spoolRef string - err = tx.QueryRow(ctx, `SELECT version.id,content.content_state,COALESCE(content.local_spool_ref,'') FROM source_items item -LEFT JOIN source_item_versions version ON version.source_item_id=item.id AND version.revision=item.revision -LEFT JOIN content_objects content ON content.id=version.content_object_id -WHERE item.id=$1`, itemID).Scan(¤tVersionID, &contentState, &spoolRef) - switch { - case errors.Is(err, pgx.ErrNoRows): - result.UnavailableFiles++ - case err != nil: - return governance.CatalogRouteResult{}, fmt.Errorf("inspect routed artifact: %w", err) - case contentState == string(governance.ContentVerified) && spoolRef != "": - result.RoutableFiles++ - default: - result.HashUnknownFiles++ - } - } - return result, nil + return fulfillCatalogIntentsTx(ctx, tx, request, files, request.Now) } func setCatalogDirectoryPolicy(ctx context.Context, tx pgx.Tx, request governance.CatalogRouteRequest, directoryPath string) error { diff --git a/internal/platform/postgres/catalog_routing_integration_test.go b/internal/platform/postgres/catalog_routing_integration_test.go index 0267b59..a629557 100644 --- a/internal/platform/postgres/catalog_routing_integration_test.go +++ b/internal/platform/postgres/catalog_routing_integration_test.go @@ -10,9 +10,10 @@ import ( "knowledge-governance/internal/governance" ) -// S7: catalog annotation persists ONLY desired state (scope rules/overrides + audit). -// It never creates or mutates test_publications and never enqueues jobs. -func TestRouteCatalogFilesWritesDesiredStateWithoutSideEffects(t *testing.T) { +// Policy-driven ingestion: saving an annotation records durable intents, +// materializes items, enqueues artifact fetches and plans publications. +// The only human action left is 开始处理; marking is the decision. +func TestRouteCatalogFilesFulfillsPolicyIntent(t *testing.T) { databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL")) if databaseURL == "" { t.Skip("TEST_DATABASE_URL is not set") @@ -46,66 +47,70 @@ INSERT INTO source_catalog_heads(source_connection_id,snapshot_id,activated_at) } store := NewStore(pool) now := time.Date(2026, 9, 9, 1, 0, 0, 0, time.UTC) + + // Folder annotation fans out to files; the bound file already holds verified + // bytes, so it is planned without any fetch (本地已有 → 比对复用 → 不拉取). result, err := store.RouteCatalogFiles(ctx, governance.CatalogRouteRequest{ConnectionID: "conn-route", NodeIDs: []string{"folder-route"}, Visibility: governance.VisibilityInternal, TargetKnowledgeBaseID: "kb-internal", OperatorRef: "engineer-1", Reason: "整目录标注", Now: now}) if err != nil { t.Fatal(err) } - if result.SelectedFiles != 1 || result.RoutableFiles != 1 || result.UnboundFiles != 0 || result.HashUnknownFiles != 0 { + if result.SelectedFiles != 1 || result.IntentsRecorded != 1 || result.PlannedPublications != 1 || result.FetchQueued != 0 { t.Fatalf("result=%#v", result) } - // Desired state saved, recursion intact. var directoryPolicy string - if err := pool.QueryRow(ctx, `SELECT visibility FROM directory_scope_rules WHERE source_connection_id='conn-route' AND directory_path='工业产品'`).Scan(&directoryPolicy); err != nil { + if err := pool.QueryRow(ctx, `SELECT visibility FROM directory_scope_rules WHERE source_connection_id='conn-route' AND directory_path='工业产品'`).Scan(&directoryPolicy); err != nil || directoryPolicy != "internal" { + t.Fatalf("directory policy=%q err=%v", directoryPolicy, err) + } + var intentCount, publicationCount, fetchJobs, uploadJobs int + if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM catalog_import_intents WHERE source_connection_id='conn-route' AND visibility='internal'`).Scan(&intentCount); err != nil { t.Fatal(err) } - if directoryPolicy != "internal" { - t.Fatalf("directory policy=%q", directoryPolicy) - } - // Zero side effects: no publications, no jobs. - var publicationCount, jobCount int - if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM test_publications`).Scan(&publicationCount); err != nil { + if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM test_publications WHERE processing_state='planned' AND superseded_at IS NULL`).Scan(&publicationCount); err != nil { t.Fatal(err) } - if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM jobs`).Scan(&jobCount); err != nil { + if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM jobs WHERE type='artifact.fetch'`).Scan(&fetchJobs); err != nil { t.Fatal(err) } - if publicationCount != 0 || jobCount != 0 { - t.Fatalf("annotation must not touch pipeline: publications=%d jobs=%d", publicationCount, jobCount) + if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM jobs WHERE type='test.upload'`).Scan(&uploadJobs); err != nil { + t.Fatal(err) + } + if intentCount != 1 || publicationCount != 1 || fetchJobs != 0 || uploadJobs != 0 { + t.Fatalf("intents=%d planned=%d fetch=%d upload=%d", intentCount, publicationCount, fetchJobs, uploadJobs) + } + // The planned publication binds the pre-verified current version. + var plannedVersion string + if err := pool.QueryRow(ctx, `SELECT source_item_version_id FROM test_publications WHERE test_kb_id='kb-internal'`).Scan(&plannedVersion); err != nil || plannedVersion != "version-route" { + t.Fatalf("planned version=%q err=%v", plannedVersion, err) } - // Re-annotating the same directory is idempotent for rules and still side-effect free. + // Re-marking the same directory is replay-safe: one intent, one publication. again, err := store.RouteCatalogFiles(ctx, governance.CatalogRouteRequest{ConnectionID: "conn-route", NodeIDs: []string{"folder-route"}, Visibility: governance.VisibilityInternal, TargetKnowledgeBaseID: "kb-internal", OperatorRef: "engineer-1", Reason: "重复提交", Now: now.Add(time.Minute)}) - if err != nil || again.SelectedFiles != 1 || again.RoutableFiles != 1 { + if err != nil || again.SelectedFiles != 1 || again.PlannedPublications != 0 || again.FetchQueued != 0 { t.Fatalf("again=%#v err=%v", again, err) } - if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM jobs`).Scan(&jobCount); err != nil || jobCount != 0 { - t.Fatalf("repeat annotation jobs=%d err=%v", jobCount, err) - } var ruleCount int if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM directory_scope_rules WHERE source_connection_id='conn-route' AND directory_path='工业产品'`).Scan(&ruleCount); err != nil || ruleCount != 1 { t.Fatalf("rule rows=%d err=%v", ruleCount, err) } - // Switching policy updates desired state without retiring or removing anything. + // Widening to both plans a second KB publication; nothing is fetched again. both, err := store.RouteCatalogFiles(ctx, governance.CatalogRouteRequest{ConnectionID: "conn-route", NodeIDs: []string{"folder-route"}, Visibility: governance.VisibilityBoth, TargetKnowledgeBaseIDs: []string{"kb-internal", "kb-external"}, OperatorRef: "engineer-1", Reason: "内外知识库同时使用", Now: now.Add(2 * time.Minute)}) - if err != nil || both.SelectedFiles != 1 || both.RoutableFiles != 1 { + if err != nil || both.SelectedFiles != 1 || both.PlannedPublications != 1 || both.FetchQueued != 0 { t.Fatalf("both=%#v err=%v", both, err) } if err := pool.QueryRow(ctx, `SELECT visibility FROM directory_scope_rules WHERE source_connection_id='conn-route' AND directory_path='工业产品'`).Scan(&directoryPolicy); err != nil || directoryPolicy != "both" { t.Fatalf("policy after both=%q err=%v", directoryPolicy, err) } - // Blocking saves the desired policy only; existing publications (if any) stay untouched. + // An unannotated live publication survives marking blocked: removal is + // knowledge-base diff (需下架) territory, never a silent side effect. if _, err := pool.Exec(ctx, `INSERT INTO test_publications(id,source_item_version_id,test_kb_id,target_scope,ingest_state,processing_state,retrieval_state,knowledge_id,created_at,updated_at) VALUES('tp-existing','version-route','kb-internal','internal','uploaded','completed','enabled','kn-1',$1,$1)`, now); err != nil { t.Fatal(err) } blocked, err := store.RouteCatalogFiles(ctx, governance.CatalogRouteRequest{ConnectionID: "conn-route", NodeIDs: []string{"folder-route"}, Visibility: governance.VisibilityBlocked, OperatorRef: "engineer-1", Reason: "禁止导入", Now: now.Add(3 * time.Minute)}) - if err != nil { - t.Fatal(err) - } - if blocked.SelectedFiles != 1 { - t.Fatalf("blocked=%#v", blocked) + if err != nil || blocked.SelectedFiles != 1 || blocked.BlockedItems != 1 { + t.Fatalf("blocked=%#v err=%v", blocked, err) } var retrievalState string var removalJobs int @@ -118,39 +123,36 @@ VALUES('tp-existing','version-route','kb-internal','internal','uploaded','comple if retrievalState != "enabled" || removalJobs != 0 { t.Fatalf("blocked annotation must not remove: state=%s removal jobs=%d", retrievalState, removalJobs) } + // The planned kb-external row created while policy was 'both' is retired. + var stillPlanned int + if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM test_publications WHERE processing_state='planned' AND superseded_at IS NULL AND test_kb_id='kb-external'`).Scan(&stillPlanned); err != nil || stillPlanned != 0 { + t.Fatalf("blocked must retire planned rows=%d err=%v", stillPlanned, err) + } if err := pool.QueryRow(ctx, `SELECT visibility FROM directory_scope_rules WHERE source_connection_id='conn-route' AND directory_path='工业产品'`).Scan(&directoryPolicy); err != nil || directoryPolicy != "blocked" { t.Fatalf("blocked policy=%q err=%v", directoryPolicy, err) } - // Unbound file and hash-unknown file are reported, never planned. - unbound, err := store.RouteCatalogFiles(ctx, governance.CatalogRouteRequest{ConnectionID: "conn-route", NodeIDs: []string{"file-unmapped"}, Visibility: governance.VisibilityExternal, TargetKnowledgeBaseID: "kb-external", OperatorRef: "engineer-1", Reason: "测试未映射", Now: now}) - if err != nil || unbound.SelectedFiles != 1 || unbound.RoutableFiles != 0 || unbound.UnboundFiles != 1 { - t.Fatalf("unbound=%#v err=%v", unbound, err) + // A never-fetched file becomes an intent with a materialized item, a + // fetch job and a planned publication (未绑定 → 从源头拉,不再是死路). + restored, err := store.RouteCatalogFiles(ctx, governance.CatalogRouteRequest{ConnectionID: "conn-route", NodeIDs: []string{"file-unmapped"}, Visibility: governance.VisibilityInternal, TargetKnowledgeBaseID: "kb-internal", OperatorRef: "engineer-1", Reason: "重新标注", Now: now.Add(4 * time.Minute)}) + if err != nil || restored.IntentsRecorded != 1 || restored.FetchQueued != 1 || restored.PlannedPublications != 1 { + t.Fatalf("restored=%#v err=%v", restored, err) } - if _, err := pool.Exec(ctx, `INSERT INTO content_objects(id,sha256,size_bytes,local_spool_ref,content_state) VALUES('content-pending',repeat('e',64),8,'','unknown')`); err != nil { + var fetchState, itemPolicy string + if err := pool.QueryRow(ctx, `SELECT intent.fetch_state||'|'||item.policy_state FROM catalog_import_intents intent +JOIN source_items item ON item.id=intent.source_item_id +WHERE intent.node_id='file-unmapped'`).Scan(&fetchState, &itemPolicy); err != nil { t.Fatal(err) } - if _, err := pool.Exec(ctx, `INSERT INTO source_items(id,source_connection_id,stable_external_id,canonical_uri_hash,normalized_path,title,item_type,revision,visibility,visibility_origin) -VALUES('item-pending','conn-route','external-pending',repeat('f',64),'待校验.pdf','待校验.pdf','document','v1','external','pp_public_tag')`); err != nil { - t.Fatal(err) + if fetchState != "pending|internal" { + t.Fatalf("fetch state=%q", fetchState) } - if _, err := pool.Exec(ctx, `INSERT INTO source_item_versions(id,source_item_id,revision,content_object_id,content_state) VALUES('version-pending','item-pending','v1','content-pending','unknown')`); err != nil { - t.Fatal(err) + var versionState string + if err := pool.QueryRow(ctx, `SELECT version.content_state FROM source_item_versions version +JOIN source_items item ON item.id=version.source_item_id AND item.revision=version.revision +WHERE item.stable_external_id='file-unmapped'`).Scan(&versionState); err != nil || versionState != "unknown" { + t.Fatalf("pending version state=%q err=%v", versionState, err) } - if _, err := pool.Exec(ctx, `INSERT INTO source_catalog_nodes(snapshot_id,node_id,parent_node_id,node_type,name,relative_path,source_locator_hash,source_item_id,size_bytes,direct_child_count,subtree_file_count,subtree_bytes) -VALUES('snapshot-route','file-pending','root-route','file','待校验.pdf','待校验.pdf',repeat('d',64),'item-pending',8,0,1,8)`); err != nil { - t.Fatal(err) - } - hashUnknown, err := store.RouteCatalogFiles(ctx, governance.CatalogRouteRequest{ConnectionID: "conn-route", NodeIDs: []string{"file-pending"}, Visibility: governance.VisibilityInternal, TargetKnowledgeBaseID: "kb-internal", OperatorRef: "engineer-1", Reason: "测试内容未验证", Now: now}) - if err != nil || hashUnknown.SelectedFiles != 1 || hashUnknown.RoutableFiles != 0 || hashUnknown.HashUnknownFiles != 1 { - t.Fatalf("hash unknown=%#v err=%v", hashUnknown, err) - } - // Overrides are still recorded for unbound/pending items so the desired state survives. - var overrideCount int - if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM source_item_scope_overrides WHERE source_item_id='item-pending'`).Scan(&overrideCount); err != nil || overrideCount != 1 { - t.Fatalf("pending override rows=%d err=%v", overrideCount, err) - } - // Audit trail exists for every annotation action. var events int if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM scope_change_events`).Scan(&events); err != nil || events < 5 { t.Fatalf("scope events=%d err=%v", events, err) diff --git a/internal/platform/postgres/catalog_routing_operation_integration_test.go b/internal/platform/postgres/catalog_routing_operation_integration_test.go index 23146d0..7ca7fe4 100644 --- a/internal/platform/postgres/catalog_routing_operation_integration_test.go +++ b/internal/platform/postgres/catalog_routing_operation_integration_test.go @@ -46,7 +46,7 @@ INSERT INTO source_catalog_heads(source_connection_id,snapshot_id,activated_at) now := time.Date(2026, 9, 11, 6, 0, 0, 0, time.UTC) request := governance.CatalogRouteRequest{ConnectionID: "conn-op", NodeIDs: []string{"file-op"}, Visibility: governance.VisibilityInternal, TargetKnowledgeBaseID: "kb-internal", OperatorRef: "engineer-1", Reason: "初次标注", Now: now} first, replayed, err := store.RouteCatalogFilesWithOperation(ctx, request, "op-route-1") - if err != nil || replayed || first.RoutableFiles != 1 { + if err != nil || replayed || first.PlannedPublications != 1 { t.Fatalf("first result=%#v replayed=%v error=%v", first, replayed, err) } second, replayed, err := store.RouteCatalogFilesWithOperation(ctx, request, "op-route-1") @@ -60,8 +60,8 @@ INSERT INTO source_catalog_heads(source_connection_id,snapshot_id,activated_at) if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM operation_requests WHERE operation_type='catalog.route' AND operation_id='op-route-1'`).Scan(&operations); err != nil { t.Fatal(err) } - if publications != 0 || operations != 1 { - t.Fatalf("S7 annotation must not publish: publications=%d operations=%d", publications, operations) + if publications != 1 || operations != 1 { + t.Fatalf("annotation must plan exactly one replay-safe publication: publications=%d operations=%d", publications, operations) } conflicting := request conflicting.Reason = "不同的操作内容" diff --git a/internal/platform/postgres/migrations/000033_catalog_policy_intent.down.sql b/internal/platform/postgres/migrations/000033_catalog_policy_intent.down.sql new file mode 100644 index 0000000..4567bc0 --- /dev/null +++ b/internal/platform/postgres/migrations/000033_catalog_policy_intent.down.sql @@ -0,0 +1,9 @@ +ALTER TABLE ui_setting_revisions DROP COLUMN IF EXISTS ingestion_concurrency; +DROP INDEX IF EXISTS idx_test_publications_planned; +ALTER TABLE test_publications DROP CONSTRAINT IF EXISTS test_publications_processing_state_check; +ALTER TABLE test_publications ADD CONSTRAINT test_publications_processing_state_check + CHECK (processing_state IN ('pending', 'processing', 'finalizing', 'completed', 'failed', 'stalled')); +DROP INDEX IF EXISTS idx_source_items_policy; +ALTER TABLE source_items DROP COLUMN IF EXISTS policy_state; +DROP INDEX IF EXISTS idx_catalog_intents_fetch; +DROP TABLE IF EXISTS catalog_import_intents; diff --git a/internal/platform/postgres/migrations/000033_catalog_policy_intent.up.sql b/internal/platform/postgres/migrations/000033_catalog_policy_intent.up.sql new file mode 100644 index 0000000..fa90ec1 --- /dev/null +++ b/internal/platform/postgres/migrations/000033_catalog_policy_intent.up.sql @@ -0,0 +1,62 @@ +-- Policy-driven ingestion: the ONLY import decision surface is the catalog +-- annotation (internal / external / both / blocked). Marking a file or folder +-- records a durable per-file INTENT that survives rescans; unannotated files +-- stay "unset" and are never ingested (cold start imports nothing). +-- A saved intent immediately materializes a versioned source item and an +-- artifact.fetch job so the artifact is pulled from the live source (or +-- matched against an already-present local copy); once content is verified a +-- 'planned' publication appears in the knowledge-base diff as 待入库. The one +-- human batch action "开始处理" promotes planned -> queued within the +-- concurrency budget configured in system settings. + +-- Per-file decision, keyed by the connection-stable catalog node id +-- (node ids are derived from source_key + relative path, identical across +-- snapshots, so intents are rescan-proof). +CREATE TABLE IF NOT EXISTS catalog_import_intents ( + source_connection_id VARCHAR(36) NOT NULL + REFERENCES source_connections(id) ON DELETE CASCADE, + node_id VARCHAR(64) NOT NULL, + visibility VARCHAR(20) NOT NULL + CHECK (visibility IN ('internal', 'external', 'both', 'blocked')), + source_item_id VARCHAR(36), + decided_by VARCHAR(128) NOT NULL, + reason VARCHAR(500) NOT NULL DEFAULT '', + -- Locator snapshot: the fetch worker needs path/name metadata without + -- re-walking a snapshot that may already have rotated. + relative_path TEXT NOT NULL DEFAULT '', + name TEXT NOT NULL DEFAULT '', + size_bytes BIGINT, + modified_at TIMESTAMPTZ, + content_sha256 CHAR(64), + locator_hash CHAR(64) NOT NULL DEFAULT '', + -- Content materialization lifecycle driven by the artifact.fetch job. + fetch_state VARCHAR(20) NOT NULL DEFAULT 'pending' + CHECK (fetch_state IN ('pending', 'fetching', 'ready', 'missing_local', 'failed')), + last_error TEXT NOT NULL DEFAULT '', + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + PRIMARY KEY (source_connection_id, node_id) +); + +CREATE INDEX IF NOT EXISTS idx_catalog_intents_item + ON catalog_import_intents (source_item_id) WHERE source_item_id IS NOT NULL; +CREATE INDEX IF NOT EXISTS idx_catalog_intents_fetch + ON catalog_import_intents (fetch_state) WHERE fetch_state <> 'ready'; + +-- Materialized items carry the policy so list/diff queries never re-walk rules. +ALTER TABLE source_items ADD COLUMN IF NOT EXISTS policy_state VARCHAR(20) NOT NULL DEFAULT 'unset' + CHECK (policy_state IN ('unset', 'internal', 'external', 'both', 'blocked')); +CREATE INDEX IF NOT EXISTS idx_source_items_policy + ON source_items (policy_state) WHERE policy_state <> 'unset'; + +-- The knowledge base is the ingestion target of a planned publication; make +-- "planned" explicit so the KB-diff view never reuses queued semantics. +ALTER TABLE test_publications DROP CONSTRAINT IF EXISTS test_publications_processing_state_check; +ALTER TABLE test_publications ADD CONSTRAINT test_publications_processing_state_check + CHECK (processing_state IN ('planned', 'pending', 'processing', 'finalizing', 'completed', 'failed', 'stalled')); +CREATE INDEX IF NOT EXISTS idx_test_publications_planned + ON test_publications (test_kb_id, processing_state) WHERE processing_state = 'planned'; + +-- 开始处理's concurrency budget lives in system settings (validated revisions). +ALTER TABLE ui_setting_revisions ADD COLUMN IF NOT EXISTS ingestion_concurrency INTEGER NOT NULL DEFAULT 2 + CHECK (ingestion_concurrency BETWEEN 1 AND 16); diff --git a/internal/platform/postgres/migrations_test.go b/internal/platform/postgres/migrations_test.go index 9116c61..106b78b 100644 --- a/internal/platform/postgres/migrations_test.go +++ b/internal/platform/postgres/migrations_test.go @@ -11,8 +11,8 @@ func TestEmbeddedMigrationsAreVersionSorted(t *testing.T) { if err != nil { t.Fatalf("loadMigrations() error = %v", err) } - if len(migrations) != 32 { - t.Fatalf("migration count = %d, want 32", len(migrations)) + if len(migrations) != 33 { + t.Fatalf("migration count = %d, want 33", len(migrations)) } if migrations[0].Version != 1 || !strings.Contains(migrations[0].UpSQL, "CREATE TABLE IF NOT EXISTS jobs") { t.Fatalf("migration 1 = %#v", migrations[0]) @@ -567,7 +567,8 @@ func TestReleaseExecutionMigrationDefinesAuthorizedAuditableJobs(t *testing.T) { func TestClaimSQLUsesSkipLockedAndReclaimsExpiredRunningJobs(t *testing.T) { sql := strings.ToLower(claimJobSQL) for _, required := range []string{ - "for update skip locked", + "for update of jobs skip locked", + "ingestion_concurrency", "state in ('pending', 'retry')", "state = 'running'", "lease_until <", diff --git a/internal/platform/postgres/policy_intents.go b/internal/platform/postgres/policy_intents.go new file mode 100644 index 0000000..de220d1 --- /dev/null +++ b/internal/platform/postgres/policy_intents.go @@ -0,0 +1,364 @@ +package postgres + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "time" + + "github.com/jackc/pgx/v5" + + "knowledge-governance/internal/governance" +) + +// fulfillCatalogIntentsTx is the policy-driven ingestion core: saving an +// annotation in 来源与目录 records one durable intent per selected file and, +// for importable policies, materializes a versioned source item, enqueues an +// artifact.fetch job (bytes come from the live source; a matching local copy +// short-circuits the download) and creates one 'planned' publication per routed +// knowledge base. Planned publications are what 解析监控 lists as 待入库; +// 开始处理 promotes them within the configured concurrency budget. Blocked +// intents retire any planned publication and flip the item policy so the +// knowledge-base diff demands removal of anything still live. +func fulfillCatalogIntentsTx(ctx context.Context, tx pgx.Tx, request governance.CatalogRouteRequest, files []governance.CatalogRouteFile, now time.Time) (governance.CatalogRouteResult, error) { + targetKnowledgeBaseIDs := request.TargetKnowledgeBaseIDs + if len(targetKnowledgeBaseIDs) == 0 && request.TargetKnowledgeBaseID != "" { + targetKnowledgeBaseIDs = []string{request.TargetKnowledgeBaseID} + } + result := governance.CatalogRouteResult{SelectedFiles: len(files)} + for _, file := range files { + recorded, fetchQueued, planned, blocked, err := fulfillCatalogIntent(ctx, tx, request, file, targetKnowledgeBaseIDs, now) + if err != nil { + return governance.CatalogRouteResult{}, err + } + if !recorded { + result.UnboundFiles++ + continue + } + result.IntentsRecorded++ + if blocked { + result.BlockedItems++ + continue + } + if fetchQueued { + result.FetchQueued++ + } + result.PlannedPublications += planned + } + return result, nil +} + +// fulfillCatalogIntent applies one file decision. It returns recorded=false +// only when the node vanished mid-transaction; fetchQueued reports a newly +// enqueued artifact.fetch job and planned the number of publications created. +func fulfillCatalogIntent(ctx context.Context, tx pgx.Tx, request governance.CatalogRouteRequest, file governance.CatalogRouteFile, targetKnowledgeBaseIDs []string, now time.Time) (bool, bool, int, bool, error) { + policy := string(request.Visibility) + if _, err := tx.Exec(ctx, `INSERT INTO catalog_import_intents( + source_connection_id,node_id,visibility,decided_by,reason,relative_path,name,size_bytes,modified_at,content_sha256, + locator_hash,fetch_state,last_error,created_at,updated_at) +VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,'pending','',NOW(),NOW()) +ON CONFLICT(source_connection_id,node_id) DO UPDATE SET + visibility=EXCLUDED.visibility,decided_by=EXCLUDED.decided_by,reason=EXCLUDED.reason, + relative_path=EXCLUDED.relative_path,name=EXCLUDED.name,size_bytes=EXCLUDED.size_bytes, + modified_at=EXCLUDED.modified_at,content_sha256=EXCLUDED.content_sha256, + locator_hash=EXCLUDED.locator_hash, + fetch_state=CASE WHEN catalog_import_intents.content_sha256 IS DISTINCT FROM EXCLUDED.content_sha256 + OR catalog_import_intents.size_bytes IS DISTINCT FROM EXCLUDED.size_bytes + OR catalog_import_intents.modified_at IS DISTINCT FROM EXCLUDED.modified_at + THEN 'pending' ELSE catalog_import_intents.fetch_state END, + last_error=CASE WHEN catalog_import_intents.content_sha256 IS DISTINCT FROM EXCLUDED.content_sha256 + OR catalog_import_intents.size_bytes IS DISTINCT FROM EXCLUDED.size_bytes + OR catalog_import_intents.modified_at IS DISTINCT FROM EXCLUDED.modified_at + THEN '' ELSE catalog_import_intents.last_error END, + updated_at=NOW()`, + request.ConnectionID, file.NodeID, policy, request.OperatorRef, request.Reason, + file.RelativePath, file.Name, file.SizeBytes, file.ModifiedAt, file.ContentSHA256, file.LocatorHash); err != nil { + return false, false, 0, false, fmt.Errorf("record catalog intent: %w", err) + } + + var existingItemID *string + err := tx.QueryRow(ctx, `SELECT source_item_id FROM catalog_import_intents +WHERE source_connection_id=$1 AND node_id=$2 FOR UPDATE`, request.ConnectionID, file.NodeID).Scan(&existingItemID) + if errors.Is(err, pgx.ErrNoRows) { + return false, false, 0, false, nil + } + if err != nil { + return false, false, 0, false, fmt.Errorf("read catalog intent: %w", err) + } + + itemID := "" + if existingItemID != nil { + itemID = *existingItemID + } + if itemID == "" { + newItemID, err := newOpaqueID("si-") + if err != nil { + return false, false, 0, false, err + } + if _, err := tx.Exec(ctx, `INSERT INTO source_items( + id,source_connection_id,stable_external_id,canonical_uri_hash,title,item_type,revision, + visibility,visibility_origin,source_change,absence_streak,identity_confidence, + policy_state,created_at,updated_at) +VALUES($1,$2,$3,$4,$5,'file','',$7,'configured_scope','unchanged',0,'object_id',$6,NOW(),NOW())`, + newItemID, request.ConnectionID, "cat:"+file.NodeID, + sha256Hex("policy:"+request.ConnectionID+"\x00"+file.NodeID), + file.Name, policy, itemVisibilityFor(request.Visibility)); err != nil { + return false, false, 0, false, fmt.Errorf("materialize policy source item: %w", err) + } + if _, err := tx.Exec(ctx, `UPDATE catalog_import_intents SET source_item_id=$3 WHERE source_connection_id=$1 AND node_id=$2`, + request.ConnectionID, file.NodeID, newItemID); err != nil { + return false, false, 0, false, fmt.Errorf("bind catalog intent item: %w", err) + } + itemID = newItemID + } + + if request.Visibility == governance.VisibilityBlocked { + if _, err := tx.Exec(ctx, `UPDATE source_items SET policy_state='blocked',updated_at=NOW() WHERE id=$1`, itemID); err != nil { + return false, false, 0, false, fmt.Errorf("block policy item: %w", err) + } + if _, err := tx.Exec(ctx, `UPDATE test_publications SET superseded_at=$2,updated_at=$2 +WHERE source_item_version_id IN (SELECT id FROM source_item_versions WHERE source_item_id=$1) + AND superseded_at IS NULL AND processing_state='planned'`, itemID, now); err != nil { + return false, false, 0, false, fmt.Errorf("retire blocked planned publications: %w", err) + } + return true, false, 0, true, nil + } + + revision := "locator:" + file.LocatorHash + if file.ContentSHA256 != nil && *file.ContentSHA256 != "" { + revision = "sha:" + *file.ContentSHA256 + } + if _, err := tx.Exec(ctx, `UPDATE source_items +SET policy_state=$2,visibility=$3,visibility_origin='configured_scope',updated_at=NOW() WHERE id=$1`, + itemID, policy, itemVisibilityFor(request.Visibility)); err != nil { + return false, false, 0, false, fmt.Errorf("update policy item: %w", err) + } + + // Resolve the version this intent points at. A bound item whose current + // version already carries verified bytes is reused without any fetch — the + // 本地已有先比对 requirement. Only when the node proves different content + // (scan-reported hash mismatch) or bytes were never fetched do we create a + // revision and enqueue artifact.fetch. + versionID := "" + contentState := "" + var curID, curSHA string + err = tx.QueryRow(ctx, `SELECT version.id,COALESCE(content.sha256,''),version.content_state +FROM source_items item +JOIN source_item_versions version ON version.source_item_id=item.id AND version.revision=item.revision +LEFT JOIN content_objects content ON content.id=version.content_object_id +WHERE item.id=$1`, itemID).Scan(&curID, &curSHA, &contentState) + if err != nil && !errors.Is(err, pgx.ErrNoRows) { + return false, false, 0, false, fmt.Errorf("inspect current item version: %w", err) + } + nodeSHA := "" + if file.ContentSHA256 != nil { + nodeSHA = *file.ContentSHA256 + } + reuse := err == nil && contentState == string(governance.ContentVerified) && (nodeSHA == "" || nodeSHA == curSHA) + if !reuse { + contentState = "unknown" + err = tx.QueryRow(ctx, `SELECT version.id,version.content_state +FROM source_item_versions version WHERE version.source_item_id=$1 AND version.revision=$2`, + itemID, revision).Scan(&versionID, &contentState) + if errors.Is(err, pgx.ErrNoRows) { + newVersionID, err := newOpaqueID("sv-") + if err != nil { + return false, false, 0, false, err + } + if _, err := tx.Exec(ctx, `INSERT INTO source_item_versions( + id,source_item_id,revision,content_object_id,content_state,raw_record_summary,source_updated_at,created_at,updated_at) +VALUES($1,$2,$3,NULL,'unknown','{}'::jsonb,$4,NOW(),NOW())`, newVersionID, itemID, revision, file.ModifiedAt); err != nil { + return false, false, 0, false, fmt.Errorf("create policy version: %w", err) + } + if _, err := tx.Exec(ctx, `UPDATE source_items SET revision=$2,updated_at=NOW() WHERE id=$1`, itemID, revision); err != nil { + return false, false, 0, false, fmt.Errorf("advance policy item revision: %w", err) + } + versionID = newVersionID + } else if err != nil { + return false, false, 0, false, fmt.Errorf("lookup policy version: %w", err) + } + } else { + versionID = curID + } + + planned := 0 + for _, kbID := range targetKnowledgeBaseIDs { + scope := kbScopeFor(request, kbID) + publicationID, err := newOpaqueID("tp-") + if err != nil { + return false, false, 0, false, err + } + tag, err := tx.Exec(ctx, `INSERT INTO test_publications( + id,source_item_version_id,test_kb_id,knowledge_id,ingest_state,processing_state,retrieval_state, + parser_generation,index_generation,last_error,retry_count,target_scope,created_at,updated_at) +VALUES($1,$2,$3,'','pending','planned','unpublished','','','',0,$4,NOW(),NOW()) +ON CONFLICT (source_item_version_id,test_kb_id) DO NOTHING`, + publicationID, versionID, kbID, scope) + if err != nil { + return false, false, 0, false, fmt.Errorf("plan publication: %w", err) + } + planned += int(tag.RowsAffected()) + } + + fetchQueued := false + if contentState != string(governance.ContentVerified) { + jobID, err := newOpaqueID("job-") + if err != nil { + return false, false, 0, false, err + } + payload, err := json.Marshal(map[string]string{"source_connection_id": request.ConnectionID, "node_id": file.NodeID, "version_id": versionID}) + if err != nil { + return false, false, 0, false, errors.New("encode artifact fetch job") + } + tag, err := tx.Exec(ctx, `INSERT INTO jobs(id,type,payload,state,priority,available_at,max_attempts,idempotency_key,created_at,updated_at) +VALUES($1,'artifact.fetch',$2::jsonb,'pending',20,$3,8,$4,$3,$3) ON CONFLICT (idempotency_key) DO NOTHING`, + jobID, payload, now, "artifact-fetch:"+versionID) + if err != nil { + return false, false, 0, false, fmt.Errorf("enqueue artifact fetch: %w", err) + } + fetchQueued = tag.RowsAffected() > 0 + if fetchQueued { + if _, err := tx.Exec(ctx, `UPDATE catalog_import_intents SET fetch_state='pending',last_error='',updated_at=NOW() +WHERE source_connection_id=$1 AND node_id=$2 AND fetch_state IN ('failed','missing_local','fetching')`, request.ConnectionID, file.NodeID); err != nil { + return false, false, 0, false, fmt.Errorf("reset catalog intent fetch state: %w", err) + } + } + } + return true, fetchQueued, planned, false, nil +} + +// kbScopeFor maps a routed knowledge base id to its diff scope label. +func kbScopeFor(request governance.CatalogRouteRequest, knowledgeBaseID string) string { + if request.Visibility == governance.VisibilityExternal { + return "external" + } + if request.Visibility == governance.VisibilityBoth && len(request.TargetKnowledgeBaseIDs) > 1 && + knowledgeBaseID == request.TargetKnowledgeBaseIDs[len(request.TargetKnowledgeBaseIDs)-1] { + return "external" + } + return "internal" +} + +// itemVisibilityFor mirrors source_items' two-value visibility CHECK: both and +// blocked decisions are governed through policy_state; the legacy visibility +// column keeps a valid value and is widened to external for both. +func itemVisibilityFor(visibility governance.Visibility) string { + if visibility == governance.VisibilityInternal { + return string(governance.VisibilityInternal) + } + return string(governance.VisibilityExternal) +} + +func sha256Hex(value string) string { + sum := sha256.Sum256([]byte(value)) + return hex.EncodeToString(sum[:]) +} + +// MarkArtifactFetched finalizes one artifact.fetch success: it (re)uses the +// content_objects row for the verified hash (content-level dedupe), attaches it +// to the pending version and flips the intent to ready. matchedLocal records +// that the bytes already existed locally and no source download happened. +func (s *Store) MarkArtifactFetched(ctx context.Context, versionID, sha256Value string, sizeBytes int64, spoolRef, validatedFormat string, matchedLocal bool, now time.Time) error { + if len(sha256Value) != 64 || spoolRef == "" || sizeBytes < 0 || validatedFormat == "" { + return errors.New("invalid artifact fetch result") + } + var tx pgx.Tx + ownsTransaction := false + if existing, ok := s.db.(pgx.Tx); ok { + tx = existing + } else { + beginner, ok := s.db.(interface { + Begin(context.Context) (pgx.Tx, error) + }) + if !ok { + return errors.New("artifact fetch transactions are unavailable") + } + var err error + tx, err = beginner.Begin(ctx) + if err != nil { + return fmt.Errorf("begin artifact fetch: %w", err) + } + ownsTransaction = true + defer tx.Rollback(ctx) + } + var connectionID, nodeID string + err := tx.QueryRow(ctx, `SELECT intent.source_connection_id,intent.node_id +FROM catalog_import_intents intent +JOIN source_item_versions version ON version.source_item_id=intent.source_item_id +WHERE version.id=$1 FOR UPDATE OF intent`, versionID).Scan(&connectionID, &nodeID) + if err != nil { + return fmt.Errorf("locate intent for fetched artifact: %w", err) + } + freshID, err := newOpaqueID("co-") + if err != nil { + return err + } + contentID := "" + err = tx.QueryRow(ctx, `INSERT INTO content_objects(id,sha256,size_bytes,mime,validated_format,local_spool_ref,content_state,created_at,updated_at) +VALUES($1,$2,$3,'',$5,$4,'verified',NOW(),NOW()) +ON CONFLICT(sha256) DO UPDATE SET + size_bytes=EXCLUDED.size_bytes, + validated_format=CASE WHEN content_objects.validated_format='' THEN EXCLUDED.validated_format ELSE content_objects.validated_format END, + local_spool_ref=CASE WHEN content_objects.local_spool_ref='' THEN EXCLUDED.local_spool_ref ELSE content_objects.local_spool_ref END, + content_state='verified',updated_at=NOW() +RETURNING id`, + freshID, sha256Value, sizeBytes, spoolRef, validatedFormat).Scan(&contentID) + if err != nil { + return fmt.Errorf("record fetched content: %w", err) + } + if _, err := tx.Exec(ctx, `UPDATE source_item_versions SET + content_object_id=$2,content_state='verified',updated_at=$4, + raw_record_summary=jsonb_set(raw_record_summary,'{matched_local}',$3::jsonb) +WHERE id=$1`, + versionID, contentID, mustJSONBool(matchedLocal), now.UTC()); err != nil { + return fmt.Errorf("bind fetched content: %w", err) + } + if _, err := tx.Exec(ctx, `UPDATE catalog_import_intents SET fetch_state='ready',last_error='',updated_at=NOW() +WHERE source_connection_id=$1 AND node_id=$2`, connectionID, nodeID); err != nil { + return fmt.Errorf("mark intent fetched: %w", err) + } + if ownsTransaction { + if err := tx.Commit(ctx); err != nil { + return fmt.Errorf("commit artifact fetch: %w", err) + } + } + return nil +} + +func mustJSONBool(value bool) string { + if value { + return "true" + } + return "false" +} + +// MarkArtifactFetchMissing parks an intent whose source-side file disappeared +// (a 404 on download, not a transient outage) so the UI can ask for a rescan. +func (s *Store) MarkArtifactFetchMissing(ctx context.Context, connectionID, nodeID string, now time.Time) error { + if _, err := s.db.Exec(ctx, `UPDATE catalog_import_intents SET fetch_state='missing_local',updated_at=$3 +WHERE source_connection_id=$1 AND node_id=$2 AND fetch_state<>'ready'`, connectionID, nodeID, now.UTC()); err != nil { + return fmt.Errorf("record artifact fetch miss: %w", err) + } + return nil +} + +// MarkArtifactFetchFailed records a fetch failure for the intent so 解析监控 +// can show why an item is not yet 待入库. +func (s *Store) MarkArtifactFetchFailed(ctx context.Context, connectionID, nodeID, message string, now time.Time) error { + if _, err := s.db.Exec(ctx, `UPDATE catalog_import_intents SET fetch_state='failed',last_error=$3,updated_at=$4 +WHERE source_connection_id=$1 AND node_id=$2 AND fetch_state<>'ready'`, connectionID, nodeID, truncateMonitorText(message), now.UTC()); err != nil { + return fmt.Errorf("record artifact fetch failure: %w", err) + } + return nil +} + +func truncateMonitorText(value string) string { + runes := []rune(value) + if len(runes) <= 500 { + return value + } + return string(runes[:500]) +} diff --git a/internal/platform/postgres/processing_confirmations.go b/internal/platform/postgres/processing_confirmations.go index a6bf63b..700c380 100644 --- a/internal/platform/postgres/processing_confirmations.go +++ b/internal/platform/postgres/processing_confirmations.go @@ -16,9 +16,12 @@ func (s *Store) StartTestPublications(ctx context.Context, request governance.St result := governance.StartTestPublicationsResult{Selected: len(request.PublicationIDs)} operator := strings.TrimSpace(request.OperatorRef) reason := strings.TrimSpace(request.Reason) - if len(request.PublicationIDs) == 0 || len(request.PublicationIDs) > 100 || operator == "" || len(operator) > 128 || reason == "" || len(reason) > 500 || request.Now.IsZero() { + if len(request.PublicationIDs) == 0 || len(request.PublicationIDs) > 100 || operator == "" || len(operator) > 128 || len(reason) > 500 || request.Now.IsZero() { return governance.StartTestPublicationsResult{}, errors.New("invalid processing confirmation") } + if reason == "" { + reason = "开始处理" + } seen := make(map[string]bool, len(request.PublicationIDs)) for _, publicationID := range request.PublicationIDs { if !validRouteID(publicationID) || seen[publicationID] { @@ -50,22 +53,23 @@ func (s *Store) StartTestPublications(ctx context.Context, request governance.St for _, publicationID := range request.PublicationIDs { var ready, confirmed, uploadJobExists bool + var processingState string err = tx.QueryRow(ctx, ` SELECT p.ingest_state='pending' - AND p.processing_state='pending' AND p.retrieval_state='unpublished' AND p.knowledge_id='' AND version.content_state='verified' AND content.content_state='verified' AND content.local_spool_ref<>'' AND content.size_bytes>0 AS ready, + p.processing_state, EXISTS(SELECT 1 FROM processing_confirmations confirmation WHERE confirmation.publication_id=p.id), EXISTS(SELECT 1 FROM jobs job WHERE job.type='test.upload' AND job.payload->>'publication_id'=p.id) FROM test_publications p JOIN source_item_versions version ON version.id=p.source_item_version_id JOIN content_objects content ON content.id=version.content_object_id -WHERE p.id=$1 -FOR UPDATE OF p`, publicationID).Scan(&ready, &confirmed, &uploadJobExists) +WHERE p.id=$1 AND p.superseded_at IS NULL +FOR UPDATE OF p`, publicationID).Scan(&ready, &processingState, &confirmed, &uploadJobExists) if errors.Is(err, pgx.ErrNoRows) { result.Unavailable++ continue @@ -77,10 +81,18 @@ FOR UPDATE OF p`, publicationID).Scan(&ready, &confirmed, &uploadJobExists) result.AlreadyStarted++ continue } + // 开始处理 promotes both fresh plans and verified pending rows. + if processingState != "planned" && processingState != "pending" { + result.Unavailable++ + continue + } if !ready { result.Unavailable++ continue } + if _, err := tx.Exec(ctx, `UPDATE test_publications SET processing_state='pending',updated_at=$2 WHERE id=$1 AND processing_state='planned'`, publicationID, request.Now); err != nil { + return governance.StartTestPublicationsResult{}, fmt.Errorf("promote planned publication: %w", err) + } confirmationID, err := newOpaqueID("pc-") if err != nil { return governance.StartTestPublicationsResult{}, err diff --git a/internal/platform/postgres/queue_sql.go b/internal/platform/postgres/queue_sql.go index 37da72d..91f0f55 100644 --- a/internal/platform/postgres/queue_sql.go +++ b/internal/platform/postgres/queue_sql.go @@ -13,17 +13,29 @@ RETURNING id, type, payload, state, priority, available_at, ` const claimJobSQL = ` -WITH candidate AS ( - SELECT id - FROM jobs +WITH budget AS ( + SELECT COALESCE(( + SELECT ingestion_concurrency + FROM ui_setting_revisions + WHERE state = 'active' + LIMIT 1 + ), 2) AS ingestion_slots +), candidate AS ( + SELECT jobs.id + FROM jobs, budget WHERE ( - (state IN ('pending', 'retry') AND available_at <= $2) - OR (state = 'running' AND lease_until < $2) + (jobs.state IN ('pending', 'retry') AND jobs.available_at <= $2) + OR (jobs.state = 'running' AND jobs.lease_until < $2) ) - AND attempt < max_attempts - AND type = ANY($4::text[]) - ORDER BY priority DESC, available_at ASC, created_at ASC, id ASC - FOR UPDATE SKIP LOCKED + AND jobs.attempt < jobs.max_attempts + AND jobs.type = ANY($4::text[]) + AND ( + jobs.type <> 'test.upload' + OR (SELECT COUNT(*) FROM jobs active + WHERE active.type = 'test.upload' AND active.state = 'running') < budget.ingestion_slots + ) + ORDER BY jobs.priority DESC, jobs.available_at ASC, jobs.created_at ASC, jobs.id ASC + FOR UPDATE OF jobs SKIP LOCKED LIMIT 1 ) UPDATE jobs AS j diff --git a/internal/platform/postgres/s6_scope_policy_integration_test.go b/internal/platform/postgres/s6_scope_policy_integration_test.go index 0fecd64..7d3dbb1 100644 --- a/internal/platform/postgres/s6_scope_policy_integration_test.go +++ b/internal/platform/postgres/s6_scope_policy_integration_test.go @@ -51,7 +51,7 @@ INSERT INTO source_catalog_heads(source_connection_id,snapshot_id,activated_at) ConnectionID: "conn-s6", NodeIDs: []string{"folder-top"}, Visibility: governance.VisibilityInternal, TargetKnowledgeBaseID: "kb-internal", OperatorRef: "engineer-1", Reason: "S6目录递归标注", Now: now, }) - if err != nil || result.SelectedFiles != 1 || result.RoutableFiles != 1 { + if err != nil || result.SelectedFiles != 1 || result.PlannedPublications != 1 || result.FetchQueued != 0 { t.Fatalf("route=%#v err=%v", result, err) } // Effective visibility of the deep item must be internal via the directory rule. diff --git a/internal/platform/postgres/test_publications.go b/internal/platform/postgres/test_publications.go index d91a3b3..de8212d 100644 --- a/internal/platform/postgres/test_publications.go +++ b/internal/platform/postgres/test_publications.go @@ -128,7 +128,7 @@ SELECT p.id, si.title, p.ingest_state, p.processing_state, p.retrieval_state, JOIN source_item_versions other_version ON other_version.id=other.source_item_version_id WHERE other.id<>p.id AND other_version.source_item_id=siv.source_item_id ) THEN 'updated' ELSE 'added' END AS change_type, - (p.ingest_state='pending' AND p.processing_state='pending' + (p.ingest_state='pending' AND p.processing_state IN ('pending','planned') AND p.retrieval_state='unpublished' AND p.knowledge_id='' AND co.content_state='verified' AND co.local_spool_ref<>'' AND co.size_bytes>0 AND NOT EXISTS (SELECT 1 FROM processing_confirmations confirmation WHERE confirmation.publication_id=p.id) @@ -142,8 +142,8 @@ SELECT p.id, si.title, p.ingest_state, p.processing_state, p.retrieval_state, FROM test_publications p JOIN source_item_versions siv ON siv.id = p.source_item_version_id JOIN source_items si ON si.id = siv.source_item_id -JOIN content_objects co ON co.id = siv.content_object_id -WHERE ($1 = '' OR p.processing_state = $1) AND ($2 = '' OR p.target_scope = $2) +LEFT JOIN content_objects co ON co.id = siv.content_object_id +WHERE ($1 = '' OR p.processing_state = $1) AND ($2 = '' OR p.target_scope = $2) AND p.superseded_at IS NULL ORDER BY p.updated_at DESC, p.id LIMIT $3 OFFSET $4 ` diff --git a/internal/platform/postgres/ui_settings.go b/internal/platform/postgres/ui_settings.go index 9a6942a..6ddbd09 100644 --- a/internal/platform/postgres/ui_settings.go +++ b/internal/platform/postgres/ui_settings.go @@ -13,7 +13,7 @@ import ( var ErrInvalidUISettings = errors.New("invalid UI settings") -const uiSettingsColumns = `id,state,document_refresh_seconds,resource_refresh_seconds,stalled_after_seconds,finalizing_after_seconds,updated_by,created_at,activated_at,retired_at,error_code` +const uiSettingsColumns = `id,state,document_refresh_seconds,resource_refresh_seconds,stalled_after_seconds,finalizing_after_seconds,ingestion_concurrency,updated_by,created_at,activated_at,retired_at,error_code` const getActiveUISettingsSQL = ` SELECT ` + uiSettingsColumns + ` @@ -24,8 +24,8 @@ WHERE state = 'active' const createUISettingsCandidateSQL = ` INSERT INTO ui_setting_revisions( id,state,document_refresh_seconds,resource_refresh_seconds, - stalled_after_seconds,finalizing_after_seconds,updated_by,created_at -) VALUES($1,'validated',$2,$3,$4,$5,$6,$7) + stalled_after_seconds,finalizing_after_seconds,ingestion_concurrency,updated_by,created_at +) VALUES($1,'validated',$2,$3,$4,$5,$6,$7,$8) RETURNING ` + uiSettingsColumns + ` ` @@ -63,7 +63,8 @@ func (s *Store) CreateUISettingsCandidate(ctx context.Context, request governanc } return scanUISettings(s.db.QueryRow(ctx, createUISettingsCandidateSQL, id, request.DocumentRefreshSeconds, request.ResourceRefreshSeconds, - request.StalledAfterSeconds, request.FinalizingAfterSeconds, request.UpdatedBy, request.Now.UTC())) + request.StalledAfterSeconds, request.FinalizingAfterSeconds, request.IngestionConcurrency, + request.UpdatedBy, request.Now.UTC())) } func (s *Store) ActivateUISettings(ctx context.Context, revisionID string, now time.Time) (governance.UISettings, error) { @@ -124,6 +125,7 @@ func scanUISettings(row interface{ Scan(...any) error }) (governance.UISettings, var settings governance.UISettings if err := row.Scan(&settings.ID, &settings.State, &settings.DocumentRefreshSeconds, &settings.ResourceRefreshSeconds, &settings.StalledAfterSeconds, &settings.FinalizingAfterSeconds, + &settings.IngestionConcurrency, &settings.UpdatedBy, &settings.CreatedAt, &settings.ActivatedAt, &settings.RetiredAt, &settings.ErrorCode); err != nil { return governance.UISettings{}, fmt.Errorf("scan UI settings: %w", err) } diff --git a/internal/platform/postgres/ui_settings_integration_test.go b/internal/platform/postgres/ui_settings_integration_test.go index 8a97eaa..d253a8d 100644 --- a/internal/platform/postgres/ui_settings_integration_test.go +++ b/internal/platform/postgres/ui_settings_integration_test.go @@ -31,7 +31,7 @@ func TestUISettingsActivationRetiresPreviousRevisionIntegration(t *testing.T) { t.Fatalf("previous=%#v error=%v", previous, err) } now := time.Date(2026, 9, 10, 7, 0, 0, 0, time.UTC) - candidate, err := store.CreateUISettingsCandidate(ctx, governance.CreateUISettingsCandidateRequest{DocumentRefreshSeconds: 30, ResourceRefreshSeconds: 60, StalledAfterSeconds: 3600, FinalizingAfterSeconds: 14400, UpdatedBy: "engineer-1", Now: now}) + candidate, err := store.CreateUISettingsCandidate(ctx, governance.CreateUISettingsCandidateRequest{DocumentRefreshSeconds: 30, ResourceRefreshSeconds: 60, StalledAfterSeconds: 3600, FinalizingAfterSeconds: 14400, IngestionConcurrency: 2, UpdatedBy: "engineer-1", Now: now}) if err != nil { t.Fatal(err) } diff --git a/internal/platform/postgres/ui_settings_test.go b/internal/platform/postgres/ui_settings_test.go index fa89fba..b3794ab 100644 --- a/internal/platform/postgres/ui_settings_test.go +++ b/internal/platform/postgres/ui_settings_test.go @@ -14,7 +14,7 @@ func TestUISettingsCandidateAndActiveProjectionAreVersioned(t *testing.T) { now := time.Date(2026, 9, 10, 6, 0, 0, 0, time.UTC) row := uiSettingsRow{id: "settings-safe", state: "validated", documentRefresh: 30, resourceRefresh: 60, stalledAfter: 3600, finalizingAfter: 14400, updatedBy: "engineer-1", createdAt: now} db := &recordingDB{row: row} - candidate, err := NewStore(db).CreateUISettingsCandidate(context.Background(), governance.CreateUISettingsCandidateRequest{DocumentRefreshSeconds: 30, ResourceRefreshSeconds: 60, StalledAfterSeconds: 3600, FinalizingAfterSeconds: 14400, UpdatedBy: "engineer-1", Now: now}) + candidate, err := NewStore(db).CreateUISettingsCandidate(context.Background(), governance.CreateUISettingsCandidateRequest{DocumentRefreshSeconds: 30, ResourceRefreshSeconds: 60, StalledAfterSeconds: 3600, FinalizingAfterSeconds: 14400, IngestionConcurrency: 2, UpdatedBy: "engineer-1", Now: now}) if err != nil { t.Fatal(err) } @@ -53,6 +53,7 @@ func TestUISettingsRejectInvalidCandidate(t *testing.T) { type uiSettingsRow struct { id, state, updatedBy, errorCode string documentRefresh, resourceRefresh, stalledAfter, finalizingAfter int + ingestionConcurrency int createdAt time.Time activatedAt, retiredAt *time.Time } @@ -60,6 +61,7 @@ type uiSettingsRow struct { func (row uiSettingsRow) Scan(dest ...any) error { *dest[0].(*string), *dest[1].(*string) = row.id, row.state *dest[2].(*int), *dest[3].(*int), *dest[4].(*int), *dest[5].(*int) = row.documentRefresh, row.resourceRefresh, row.stalledAfter, row.finalizingAfter - *dest[6].(*string), *dest[7].(*time.Time), *dest[8].(**time.Time), *dest[9].(**time.Time), *dest[10].(*string) = row.updatedBy, row.createdAt, row.activatedAt, row.retiredAt, row.errorCode + *dest[6].(*int) = row.ingestionConcurrency + *dest[7].(*string), *dest[8].(*time.Time), *dest[9].(**time.Time), *dest[10].(**time.Time), *dest[11].(*string) = row.updatedBy, row.createdAt, row.activatedAt, row.retiredAt, row.errorCode return nil } diff --git a/internal/platform/spool/artifact.go b/internal/platform/spool/artifact.go index 8c430d7..996430b 100644 --- a/internal/platform/spool/artifact.go +++ b/internal/platform/spool/artifact.go @@ -34,8 +34,9 @@ type Artifact struct { } var supportedFormats = map[string]struct{}{ - "pdf": {}, "docx": {}, "doc": {}, "xlsx": {}, "xls": {}, "pptx": {}, "ppt": {}, + "pdf": {}, "docx": {}, "doc": {}, "xlsx": {}, "xls": {}, "md": {}, "txt": {}, "csv": {}, "json": {}, + "png": {}, "jpg": {}, "jpeg": {}, "gif": {}, "bmp": {}, "webp": {}, } func InspectVerified(root, relativeRef, format string, maxBytes int64) (Expectation, error) { @@ -137,7 +138,7 @@ func validateFormat(file *os.File, format string, size int64) error { if _, err := io.ReadFull(file, header); err != nil || !bytes.Equal(header, []byte("%PDF-")) { return errors.New("invalid PDF magic") } - case "docx", "xlsx", "pptx": + case "docx", "xlsx": archive, err := zip.NewReader(file, size) if err != nil { return errors.New("invalid Office ZIP") @@ -145,7 +146,6 @@ func validateFormat(file *os.File, format string, size int64) error { required := map[string]string{ "docx": "word/document.xml", "xlsx": "xl/workbook.xml", - "pptx": "ppt/presentation.xml", }[format] foundContentTypes := false foundRequired := false @@ -160,12 +160,37 @@ func validateFormat(file *os.File, format string, size int64) error { if !foundContentTypes || !foundRequired { return errors.New("missing Office package entry") } - case "doc", "xls", "ppt": + case "doc", "xls": header := make([]byte, 8) oleMagic := []byte{0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1} if _, err := io.ReadFull(file, header); err != nil || !bytes.Equal(header, oleMagic) { return errors.New("invalid legacy Office magic") } + case "png", "jpg", "jpeg", "gif", "bmp", "webp": + header := make([]byte, 12) + n, err := file.Read(header) + if err != nil || n < 12 { + return errors.New("invalid image header") + } + valid := false + switch format { + case "png": + valid = bytes.Equal(header[:8], []byte{0x89, 'P', 'N', 'G', 0x0d, 0x0a, 0x1a, 0x0a}) + case "jpg", "jpeg": + valid = header[0] == 0xff && header[1] == 0xd8 && header[2] == 0xff + case "gif": + valid = bytes.Equal(header[:6], []byte("GIF87a")) || bytes.Equal(header[:6], []byte("GIF89a")) + case "bmp": + valid = header[0] == 'B' && header[1] == 'M' + case "webp": + valid = bytes.Equal(header[:4], []byte("RIFF")) && bytes.Equal(header[8:12], []byte("WEBP")) + } + if !valid { + return errors.New("invalid image magic") + } + if _, err := file.Seek(0, io.SeekStart); err != nil { + return errors.New("rewind image artifact") + } case "json": decoder := json.NewDecoder(file) var value any diff --git a/internal/workflows/artifactfetch/handler.go b/internal/workflows/artifactfetch/handler.go new file mode 100644 index 0000000..93b10fc --- /dev/null +++ b/internal/workflows/artifactfetch/handler.go @@ -0,0 +1,247 @@ +// Package artifactfetch turns a saved policy intent into verified bytes: it +// shells out to scripts/artifact_fetch.py (which already holds every source +// adapter: AList direct download, PP ledger, DingTalk export-tree/live export) +// and records the verified content against the pending source item version. +package artifactfetch + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "errors" + "os/exec" + "path/filepath" + "strconv" + "strings" + "time" + + "knowledge-governance/internal/jobs" +) + +type Request struct { + ConnectionID string `json:"source_connection_id"` + NodeID string `json:"node_id"` + VersionID string `json:"version_id"` +} + +type Result struct { + OK bool `json:"ok"` + SpecID string `json:"spec_id"` + NodeID string `json:"node_id"` + ConnectionID string `json:"connection_id"` + SHA256 string `json:"sha256"` + SizeBytes int64 `json:"size_bytes"` + SpoolRef string `json:"spool_ref"` + ValidatedFormat string `json:"validated_format"` + MatchedLocal bool `json:"matched_local"` + Missing bool `json:"missing"` + Retryable bool `json:"retryable"` + Code string `json:"code"` + Error string `json:"error"` +} + +type Manifest struct { + Request + SourceType string + SourceName string + RelativePath string + Name string + SizeBytes *int64 + ModifiedAt *time.Time + LocatorHash string +} + +type Store interface { + LoadArtifactFetchManifest(context.Context, string, string) (Manifest, error) + MarkArtifactFetched(context.Context, string, string, int64, string, string, bool, time.Time) error + MarkArtifactFetchMissing(context.Context, string, string, time.Time) error + MarkArtifactFetchFailed(context.Context, string, string, string, time.Time) error +} + +type Runner interface { + Run(context.Context, Manifest) (Result, error) +} + +type Config struct { + Store Store + Runner Runner + Now func() time.Time +} + +type Handler struct { + store Store + runner Runner + now func() time.Time +} + +func NewHandler(config Config) (*Handler, error) { + if config.Store == nil || config.Runner == nil { + return nil, errors.New("artifact fetch dependencies are required") + } + if config.Now == nil { + config.Now = time.Now + } + return &Handler{store: config.Store, runner: config.Runner, now: config.Now}, nil +} + +func (handler *Handler) Handle(ctx context.Context, job jobs.Job) error { + var request Request + decoder := json.NewDecoder(bytes.NewReader(job.Payload)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&request); err != nil || + request.ConnectionID == "" || request.NodeID == "" || request.VersionID == "" { + return jobs.Permanent(jobs.ErrCodeArtifactInvalid, errors.New("artifact_fetch_invalid_payload")) + } + manifest, err := handler.store.LoadArtifactFetchManifest(ctx, request.ConnectionID, request.NodeID) + if err != nil { + if errors.Is(err, ErrManifestGone) { + return jobs.Permanent(jobs.ErrCodeStateConflict, err) + } + return err + } + result, err := handler.runner.Run(ctx, manifest) + now := handler.now() + if err != nil { + _ = handler.store.MarkArtifactFetchFailed(ctx, request.ConnectionID, request.NodeID, err.Error(), now) + return err + } + switch { + case result.Missing: + if err := handler.store.MarkArtifactFetchMissing(ctx, request.ConnectionID, request.NodeID, now); err != nil { + return err + } + return jobs.Permanent(jobs.ErrCodeArtifactInvalid, errors.New("artifact_fetch_missing_source: "+safeMessage(result.Error))) + case !result.OK: + _ = handler.store.MarkArtifactFetchFailed(ctx, request.ConnectionID, request.NodeID, safeMessage(result.Error), now) + if result.Retryable { + return errors.New("artifact_fetch_retryable: " + safeMessage(result.Code)) + } + return jobs.Permanent(jobs.ErrCodeArtifactInvalid, errors.New("artifact_fetch_rejected: "+safeMessage(result.Code))) + } + if result.SHA256 == "" || result.SpoolRef == "" || result.SizeBytes <= 0 || result.NodeID != request.NodeID || result.ValidatedFormat == "" { + _ = handler.store.MarkArtifactFetchFailed(ctx, request.ConnectionID, request.NodeID, "artifact fetch returned an inconsistent result", now) + return errors.New("artifact_fetch_invalid_output") + } + if err := handler.store.MarkArtifactFetched(ctx, request.VersionID, strings.ToLower(result.SHA256), + result.SizeBytes, result.SpoolRef, result.ValidatedFormat, result.MatchedLocal, now); err != nil { + return err + } + return nil +} + +func safeMessage(value string) string { + value = strings.ReplaceAll(strings.TrimSpace(value), "\n", " ") + lower := strings.ToLower(value) + for _, forbidden := range []string{"token", "secret", "password", "authorization", "bearer", "cookie", "/run/source-secrets", "http://", "https://"} { + if strings.Contains(lower, forbidden) { + return "source reported an error (details suppressed)" + } + } + if len([]rune(value)) > 200 { + value = string([]rune(value)[:200]) + } + return value +} + +// CommandRunner executes the fetch script once and parses its single JSON line. +type CommandRunner struct { + PythonPath string + ScriptPath string + SpoolRoot string + SpoolPrefix string + MaxBytes int64 +} + +func NewCommandRunner(pythonPath, scriptPath, spoolRoot, spoolPrefix string, maxBytes int64) (*CommandRunner, error) { + if !filepath.IsAbs(pythonPath) || !filepath.IsAbs(scriptPath) || !filepath.IsAbs(spoolRoot) || maxBytes <= 0 { + return nil, errors.New("artifact fetch command paths must be absolute") + } + if strings.Contains(spoolPrefix, "..") || strings.Contains(spoolPrefix, "\\") { + return nil, errors.New("artifact fetch spool prefix is invalid") + } + spoolPrefix = strings.Trim(spoolPrefix, "/") + if spoolPrefix != "" { + spoolPrefix += "/" + } + return &CommandRunner{PythonPath: pythonPath, ScriptPath: scriptPath, SpoolRoot: spoolRoot, SpoolPrefix: spoolPrefix, MaxBytes: maxBytes}, nil +} + +func (runner *CommandRunner) Run(ctx context.Context, manifest Manifest) (Result, error) { + manifestJSON, err := json.Marshal(map[string]any{ + "spec_id": manifest.VersionID, "node_id": manifest.NodeID, "connection_id": manifest.ConnectionID, + "source_type": manifest.SourceType, "source_name": manifest.SourceName, + "relative_path": manifest.RelativePath, "name": manifest.Name, + "size_bytes": manifest.SizeBytes, "locator_hash": manifest.LocatorHash, + }) + if err != nil { + return Result{}, errors.New("artifact_fetch_manifest_failed") + } + command := exec.CommandContext(ctx, runner.PythonPath, runner.ScriptPath, string(manifestJSON)) + // The literal guard keeps secrets out of the child environment: the script + // only ever reads pre-mounted files. + command.Env = []string{ + "PATH=/usr/bin:/bin", + "PYTHONUNBUFFERED=1", + "INGESTION_SPOOL_ROOT=" + runner.SpoolRoot, + "SPOOL_REF_PREFIX=" + runner.SpoolPrefix, + "ARTIFACT_FETCH_MAX_BYTES=" + strconv.FormatInt(runner.MaxBytes, 10), + "TZ=UTC", + } + stdout, err := command.StdoutPipe() + if err != nil { + return Result{}, errors.New("artifact_fetch_start_failed") + } + command.Stderr = &limitedWriter{buffer: bytes.NewBuffer(nil), limit: 4096} + if err := command.Start(); err != nil { + return Result{}, errors.New("artifact_fetch_start_failed") + } + var result Result + decoded := false + scanner := bufio.NewScanner(stdout) + scanner.Buffer(make([]byte, 4096), 1<<20) + for scanner.Scan() { + line := bytes.TrimSpace(scanner.Bytes()) + if len(line) == 0 || !bytes.HasPrefix(line, []byte("{")) { + continue + } + decoder := json.NewDecoder(bytes.NewReader(line)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&result); err != nil { + _ = command.Process.Kill() + _ = command.Wait() + return Result{}, errors.New("artifact_fetch_invalid_output") + } + decoded = true + break + } + waitErr := command.Wait() + if !decoded { + if waitErr != nil { + return Result{OK: false, Error: "artifact fetch exited without a result"}, errors.New("artifact_fetch_failed") + } + return Result{}, errors.New("artifact_fetch_no_output") + } + return result, nil +} + +var ErrManifestGone = errors.New("artifact fetch manifest is gone") + +// limitedWriter captures at most limit bytes of script stderr; full output is +// never retained (it could echo source details we do not want in failures). +type limitedWriter struct { + buffer *bytes.Buffer + limit int +} + +func (writer *limitedWriter) Write(payload []byte) (int, error) { + room := writer.limit - writer.buffer.Len() + if room > 0 { + if len(payload) > room { + writer.buffer.Write(payload[:room]) + return len(payload), nil + } + writer.buffer.Write(payload) + } + return len(payload), nil +} diff --git a/scripts/artifact_fetch.py b/scripts/artifact_fetch.py new file mode 100644 index 0000000..ad15837 --- /dev/null +++ b/scripts/artifact_fetch.py @@ -0,0 +1,515 @@ +#!/usr/bin/env python3 +"""Fetch one policy-selected catalog artifact from its live source. + +The Go worker passes a small JSON manifest (argv[1]) describing the intent. +This process never receives database credentials or secret values; it locates +credentials only via the pre-mounted /run/source-secrets paths, exactly like +live_catalog_refresh.py. Behavior per source type: + + alist -> direct read-only API: raw_url then download. + pp -> manifest.json local_path first (already on disk), else public URL. + dingtalk -> prefer the local export tree (converted docx/xlsx/pdf); if it is + older than the source-side modification time, export fresh bytes + through the web internal API captured headers. + +Before any network call the fetcher compares against local candidates by +size and, when a hash is cheap to obtain, by content: a match means "本地已有 +- no pull". Verified bytes are moved into the ingestion spool and a single +JSON result line is printed on stdout. Credentials, tokens, signed URLs and +authorization material are never printed. +""" +import csv +import hashlib +import json +import os +import pathlib +import re +import shutil +import subprocess +import sys +import tempfile +import time +import urllib.error +import urllib.parse +import urllib.request + + +SUPPORTED = { + "pdf": "pdf", "docx": "docx", "xlsx": "xlsx", "md": "md", "txt": "txt", + "csv": "csv", "json": "json", + "png": "png", "jpg": "jpg", "jpeg": "jpeg", "gif": "gif", "bmp": "bmp", "webp": "webp", +} +# Office types we refuse to ingest directly: PPT/PPTX must arrive as a PDF twin +# (embedded images travel inside those files already). +SLIDES = {"ppt", "pptx"} +LEGACY = {"doc": "doc", "xls": "xls"} +MAX_BYTES_DEFAULT = 200 * 1024 * 1024 + + +class FetchError(Exception): + def __init__(self, message, retryable=True, code="fetch_failed"): + super().__init__(message) + self.retryable = retryable + self.code = code + + +class MissingSource(Exception): + pass + + +def emit(result): + print(json.dumps(result, ensure_ascii=False)) + + +def sha256_file(path): + digest = hashlib.sha256() + with open(path, "rb") as handle: + for block in iter(lambda: handle.read(1 << 20), b""): + digest.update(block) + return digest.hexdigest() + + +def digest_of(value): + return hashlib.sha256(value.encode()).hexdigest() + + +def validate_magic(path, suffix): + with open(path, "rb") as handle: + head = handle.read(12) + checks = { + "pdf": head.startswith(b"%PDF-"), + "docx": head.startswith(b"PK\x03\x04"), + "xlsx": head.startswith(b"PK\x03\x04"), + "doc": head.startswith(b"\xd0\xcf\x11\xe0"), + "xls": head.startswith(b"\xd0\xcf\x11\xe0"), + "png": head.startswith(b"\x89PNG\r\n\x1a\n"), + "jpg": head[:2] == b"\xff\xd8" and head[2:3] == b"\xff", + "jpeg": head[:2] == b"\xff\xd8" and head[2:3] == b"\xff", + "gif": head[:6] in (b"GIF87a", b"GIF89a"), + "bmp": head[:2] == b"BM", + "webp": head[:4] == b"RIFF" and head[8:12] == b"WEBP", + } + if suffix in checks and not checks[suffix]: + raise FetchError("downloaded bytes are not a valid %s file" % suffix, + retryable=False, code="invalid_format") + if suffix in ("md", "txt", "csv", "json"): + try: + with open(path, "rb") as handle: + handle.read(4096).decode("utf-8", errors="strict") + except UnicodeDecodeError: + raise FetchError("text artifact is not valid UTF-8", retryable=False, code="invalid_format") + + +def pick_suffix(name, declared_type): + suffix = pathlib.PurePosixPath(name).suffix.lower().lstrip(".") + if declared_type in ("adoc",) and suffix in ("", "adoc"): + return "docx" + if declared_type in ("able", "axls") and suffix in ("", "able", "axls"): + return "xlsx" + if suffix in SUPPORTED: + return SUPPORTED[suffix] + if suffix == "adoc": + return "docx" + if suffix in ("able", "axls"): + return "xlsx" + return None + + +def http_get(url, headers, timeout, max_bytes, cookie_opener=None): + request = urllib.request.Request(url, headers=headers or {}) + + class NoRedirect(urllib.request.HTTPRedirectHandler): + location = "" + + def redirect_request(self, req, fp, code, msg, headers, newurl): + NoRedirect.location = newurl + return None + + handler = NoRedirect() + opener = cookie_opener or urllib.request.build_opener(handler) + try: + response = opener.open(request, timeout=timeout) + except urllib.error.HTTPError as error: + if error.code in (301, 302, 303, 307, 308): + target = handler.location or error.headers.get("Location") or "" + host = urllib.parse.urlparse(target).netloc.lower() + if any(marker in host for marker in ("login.", "sso", "oauth", "passport")): + raise FetchError("文件所在的来源存储需要网页登录授权(存储后端未配置服务凭据),请联系管理员", retryable=False, code="source_auth_required") + raise FetchError("redirect refused", retryable=True, code="redirect") + if error.code == 404: + raise MissingSource("source returned 404") + raise FetchError("source returned HTTP %d" % error.code, retryable=error.code >= 500, code="http_%d" % error.code) + with response: + declared = response.headers.get("Content-Length") + if declared and int(declared) > max_bytes: + raise FetchError("source file exceeds ingest size limit", retryable=False, code="over_limit") + destination = tempfile.NamedTemporaryFile(delete=False, suffix=".fetch") + total = 0 + digest = hashlib.sha256() + try: + while True: + block = response.read(1 << 20) + if not block: + break + total += len(block) + if total > max_bytes: + raise FetchError("source stream exceeds ingest size limit", retryable=False, code="over_limit") + digest.update(block) + destination.write(block) + finally: + destination.close() + if total == 0: + os.unlink(destination.name) + raise FetchError("source returned empty body", retryable=True, code="empty") + return destination.name, total, digest.hexdigest() + + +# ---------------------------------------------------------------- AList ------ + +def alist_download(manifest, max_bytes): + sys.path.insert(0, "/sources/alist") + from alist_readonly_inventory import AListReadOnlyClient, read_connections # noqa + + token = pathlib.Path("/run/source-secrets/alist_api_token").read_text(encoding="utf-8").strip() + if not token: + raise FetchError("AList token empty", retryable=False, code="credentials") + rows = [row for row in read_connections(pathlib.Path("/run/source-secrets/alist_connections.xlsx"), require_token=False) + if row["category"] == manifest["source_name"]] + if len(rows) != 1: + raise FetchError("AList connection configuration missing", retryable=False, code="credentials") + base_url = rows[0]["base_url"].rstrip("/") + client = AListReadOnlyClient(base_url, token) + path = "/" + manifest["relative_path"].lstrip("/") + payload = json.dumps({"path": path, "password": ""}).encode("utf-8") + request = urllib.request.Request(base_url + "/api/fs/get", data=payload, method="POST", headers={ + "x-api-token": token, "Content-Type": "application/json", "Accept": "application/json", + "User-Agent": "governance-artifact-fetch/1.0", + }) + try: + with client.opener.open(request, timeout=30) as response: + body = json.loads(response.read().decode("utf-8")) + except Exception as error: + raise FetchError("AList file lookup failed (%s)" % type(error).__name__, code="source_unavailable") + if int(body.get("code") or 0) == 404: + raise MissingSource("AList file not present at expected path") + if int(body.get("code") or 0) != 200: + raise FetchError("AList file lookup refused (code %s)" % body.get("code"), code="source_unavailable") + data = body.get("data") or {} + raw_url = str(data.get("raw_url") or "") + if not raw_url: + raise MissingSource("AList returned no download locator for this file") + if manifest.get("size_bytes") in (None, 0) and data.get("size"): + manifest["size_bytes"] = int(data["size"]) + download_url = urllib.parse.urljoin(base_url + "/", raw_url.lstrip("/")) + try: + return http_get(download_url, {"Authorization": token}, 180, max_bytes) + except FetchError as error: + if error.code == "redirect": + # Same-origin /d/ route 302s to a signed external URL: follow once + # WITHOUT the source token (never forward credentials cross-origin). + class LocationCapture(urllib.request.HTTPRedirectHandler): + captured = "" + + def redirect_request(self, req, fp, code, msg, headers, newurl): + LocationCapture.captured = newurl + return None + + probe = urllib.request.build_opener(LocationCapture) + try: + probe.open(urllib.request.Request(download_url, headers={"Authorization": token}), timeout=30) + except urllib.error.HTTPError: + pass + except Exception as failure: + raise FetchError("AList redirect probe failed (%s)" % type(failure).__name__, code="source_unavailable") + if not LocationCapture.captured: + raise + return http_get(LocationCapture.captured, {}, 180, max_bytes) + raise + + +# ------------------------------------------------------------------ PP ------- + +def pp_find_entry(manifest): + locator = manifest.get("locator_hash", "") + if not locator: + return None + manifest_path = pathlib.Path("/sources/pp/data/public-documents/manifest.json") + ledger = json.loads(manifest_path.read_text(encoding="utf-8")) + for result in ledger.get("results", []): + if digest_of(str(result.get("url") or "")) == locator: + return result + return None + + +def pp_download(manifest, max_bytes): + entry = pp_find_entry(manifest) + if entry is None: + raise MissingSource("PP ledger entry for this locator no longer exists") + # Ledger titles may omit the extension; recover the true file name from the + # local path or source URL before type checks run. + manifest_name = manifest.setdefault("name", "") + if not pathlib.PurePosixPath(manifest_name).suffix: + from urllib.parse import unquote, urlparse + for probe in (entry.get("local_path", ""), unquote(urlparse(str(entry.get("url", ""))).path)): + suffix = pathlib.PurePosixPath(probe).suffix.lower().lstrip(".") + if suffix: + manifest["name"] = manifest_name + "." + suffix + break + local_path = entry.get("local_path", "") + if local_path: + candidate = pathlib.Path(local_path.replace("/home/zheromiku/dobot-portal-sync", "/sources/pp", 1)) + if not candidate.is_file(): + candidate = pathlib.Path(local_path) + if candidate.is_file(): + size = candidate.stat().st_size + expected_hash = str(entry.get("sha256", "")).lower() + actual = sha256_file(candidate) + if expected_hash and len(expected_hash) == 64 and actual != expected_hash: + raise FetchError("PP local copy hash mismatch (ledger changed?)", retryable=True, code="local_mismatch") + return str(candidate), size, actual, True + url = entry.get("url", "") + if not url: + raise MissingSource("PP ledger entry has no local file and no download URL") + path, size, actual = http_get(url, {}, 180, max_bytes) + expected_hash = str(entry.get("sha256", "")).lower() + if expected_hash and len(expected_hash) == 64 and actual != expected_hash: + os.unlink(path) + raise FetchError("PP download hash mismatch", retryable=True, code="hash_mismatch") + return path, size, actual, False + + +# ------------------------------------------------------------ DingTalk ------- + +def load_capture_headers(): + capture = json.loads(pathlib.Path("/run/source-secrets/dingtalk_capture.json").read_text(encoding="utf-8")) + headers = capture.get("headers") or {} + def header(name): + for key, value in headers.items(): + if key.lower() == name: + return value + return "" + a_token = header("a-token") + xsrf = header("x-xsrf-token") + cookie = header("cookie") + uid = "" + match = re.search(r'"?uid"?[=:]"?(\d{4,})', cookie or "") + if match: + uid = match.group(1) + if not a_token: + raise FetchError("DingTalk capture has no a-token; re-capture in 来源与凭据", retryable=False, code="credentials") + return {"a-token": a_token, "x-xsrf-token": xsrf, "cookie": cookie, "uid": uid, + "referer": "https://alidocs.dingtalk.com/"} + + +def dingtalk_local_export(manifest): + """Locate an already-exported copy; returns (path, fresher_than_source) or None.""" + name = pathlib.PurePosixPath(manifest["relative_path"]).name + stem = pathlib.PurePosixPath(name).stem + raw_suffix = pathlib.PurePosixPath(name).suffix.lower().lstrip(".") + suffix = pick_suffix(name, manifest.get("declared_type", "")) + root = pathlib.Path("/sources/dingtalk/exported") + folder = str(pathlib.PurePosixPath(manifest["relative_path"]).parent).strip(".") + search_suffixes = [suffix] if raw_suffix not in SLIDES else ["pdf", "pptx", "ppt"] + candidates = [] + for probe in search_suffixes: + if probe is None: + continue + exact = root / manifest["source_name"] / folder / (stem + "." + probe) + if exact.is_file(): + candidates.append(exact) + else: + for base in (root / manifest["source_name"], pathlib.Path("/sources/dingtalk/converted_ppt_pdf"), + pathlib.Path("/sources/dingtalk/converted_ppt_pdf_full")): + if not base.is_dir(): + continue + for hit in base.rglob(stem + "." + probe): + candidates.append(hit) + if candidates: + break + if not candidates: + if raw_suffix not in SLIDES and suffix is None: + raise MissingSource("dingtalk file type is not ingestable") + return None + modified = str(manifest.get("modified_at") or "") + candidate = candidates[0] + local_modified = time.strftime("%Y-%m-%dT%H:%M:%S+08:00", time.localtime(candidate.stat().st_mtime)) + fresh = (not modified) or (local_modified >= modified[:19] + local_modified[19:]) + return str(candidate), fresh + + +def dingtalk_raw_keys(manifest): + """Recover the dentry uuid behind a node id via the scanner inventory.""" + state_path = pathlib.Path("/sources/dingtalk/data/dingtalk-internal/inventory-state.json") + state = json.loads(state_path.read_text(encoding="utf-8")) + roots = state.get("roots") or {} + space_key = _space_key(roots, manifest["source_name"]) + match = [value for key, value in sorted(roots.items()) if key == space_key] + if len(match) != 1: + raise FetchError("DingTalk space not present in the scanner inventory; refresh 来源目录 first", + retryable=False, code="space_missing") + sys.path.insert(0, "/app/scripts") + from source_catalog_import import node_id as catalog_node_id # noqa + source_id = "dingtalk-" + space_key + nodes = match[0].get("nodes") or {} + for raw, item in nodes.items(): + if catalog_node_id(source_id, raw) == manifest["node_id"]: + return raw, str(item.get("dentry_type") or item.get("type") or "") + raise MissingSource("DingTalk node not found in current inventory (moved or deleted?)") + + +def _space_key(roots, name): + for key, value in roots.items(): + if value.get("live_space_name") == name: + return key + return "" + + +def dingtalk_download(manifest, max_bytes): + export = dingtalk_local_export(manifest) + modified = str(manifest.get("modified_at") or "") + if export: + path, fresh = export + size = os.path.getsize(path) + if size > 0 and (fresh or not modified): + return path, size, sha256_file(path), True + # stale local export -> fall through to a live export when available + raw_key, declared_type = dingtalk_raw_keys(manifest) + headers = load_capture_headers() + live_path = live_export_dingtalk_document(raw_key, headers, manifest, max_bytes) + if export and not live_path: + path, fresh = export + return path, os.path.getsize(path), sha256_file(path), True # keep old bytes; staleness shown in UI + if not live_path: + raise MissingSource("DingTalk file is not exported and cannot be exported automatically") + return live_path + + +def live_export_dingtalk_document(raw_key, headers, manifest, max_bytes): + opener = urllib.request.build_opener() + + def api(url, payload=None): + request = urllib.request.Request(url, headers={ + "a-token": headers["a-token"], "x-xsrf-token": headers.get("x-xsrf-token", ""), + "cookie": headers.get("cookie", ""), "content-type": "application/json", + "referer": headers["referer"], "user-agent": "Mozilla/5.0", + }, data=json.dumps(payload).encode() if payload is not None else None) + try: + with opener.open(request, timeout=60) as response: + return json.loads(response.read().decode("utf-8")) + except (urllib.error.URLError, urllib.error.HTTPError, TimeoutError) as error: + raise FetchError("DingTalk web API unavailable (%s)" % type(error).__name__, code="source_unavailable") + + info = api("https://alidocs.dingtalk.com/box/api/v2/dentry/info?dentryUuid=" + urllib.parse.quote(raw_key)) + data = info.get("data") or {} + space_id, dentry_id = str(data.get("spaceId") or ""), str(data.get("dentryId") or "") + if not space_id or not dentry_id: + raise MissingSource("DingTalk dentry not resolvable") + download = api("https://alidocs.dingtalk.com/box/api/v1/file/download?spaceId=%s&dentryId=%s" % (space_id, dentry_id)) + payload = download.get("data") or {} + if payload.get("downloadType") == "URL_PRE_SIGNATURE": + urls = (payload.get("ossUrlPreSignatureInfo") or {}).get("preSignUrls") or [] + if urls: + return http_get(urls[0], {}, 180, max_bytes) + (False,) + return None + + +# ---------------------------------------------------------------- main ------- + +def suffix_from_magic(path): + with open(path, "rb") as handle: + head = handle.read(12) + if head.startswith(b"%PDF-"): + return "pdf" + if head.startswith(b"PK\x03\x04"): + # zip container: peek one entry name to pick office family + try: + import zipfile + with zipfile.ZipFile(path) as archive: + names = set(archive.namelist()[:50]) + if "word/document.xml" in names: + return "docx" + if "xl/workbook.xml" in names: + return "xlsx" + except Exception: + return None + return None + if head.startswith(b"\xd0\xcf\x11\xe0"): + return None + for suffix in ("png", "jpg", "gif", "bmp", "webp"): + checks = validate_magic_map(head) + if checks.get(suffix): + return suffix + try: + with open(path, "rb") as handle: + handle.read(4096).decode("utf-8", errors="strict") + return "md" + except (UnicodeDecodeError, OSError): + return None + + +def validate_magic_map(head): + return { + "png": head.startswith(b"\x89PNG\r\n\x1a\n"), + "jpg": head[:2] == b"\xff\xd8" and head[2:3] == b"\xff", + "gif": head[:6] in (b"GIF87a", b"GIF89a"), + "bmp": head[:2] == b"BM", + "webp": head[:4] == b"RIFF" and head[8:12] == b"WEBP", + } + + +def main(): + manifest = json.loads(sys.argv[1]) + max_bytes = int(os.environ.get("ARTIFACT_FETCH_MAX_BYTES") or MAX_BYTES_DEFAULT) + spool_root = pathlib.Path(os.environ.get("INGESTION_SPOOL_ROOT") or "/var/lib/governance/ingestion") + name = manifest["name"] + declared_type = manifest.get("declared_type", "") + matched_local = False + if manifest["source_type"] == "alist": + path, size, digest = alist_download(manifest, max_bytes) + elif manifest["source_type"] == "pp": + path, size, digest, matched_local = pp_download(manifest, max_bytes) + elif manifest["source_type"] == "dingtalk": + path, size, digest, matched_local = dingtalk_download(manifest, max_bytes) + else: + raise FetchError("unknown source type", retryable=False, code="bad_manifest") + # Type is decided on the RESOLVED bytes, not the catalog title: PP ledger + # titles may omit extensions and DingTalk exports always change them. + suffix = pick_suffix(pathlib.PurePosixPath(path).name, declared_type) or pick_suffix(name, declared_type) + if suffix is None: + suffix = suffix_from_magic(path) + if suffix is None: + raw_suffix = pathlib.PurePosixPath(name).suffix.lower().lstrip(".") + hint = " (convert PPT to PDF first)" if raw_suffix in SLIDES else "" + raise FetchError("file type .%s is not supported by the parser%s" % (raw_suffix or "?", hint), + retryable=False, code="unsupported_format") + try: + validate_magic(path, suffix) + if size > max_bytes: + raise FetchError("artifact exceeds ingest size limit", retryable=False, code="over_limit") + relative_ref = "artifact/%s/%s/%s.%s" % (digest[:2], digest[2:4], manifest["node_id"], suffix) + destination = spool_root / relative_ref + destination.parent.mkdir(parents=True, exist_ok=True) + staged = str(destination) + ".part" + shutil.copyfile(path, staged) + if sha256_file(staged) != digest: + raise FetchError("spooled bytes failed verification", code="spool_mismatch") + os.replace(staged, destination) + result = {"ok": True, "spec_id": manifest["spec_id"], "node_id": manifest["node_id"], + "connection_id": manifest["connection_id"], "sha256": digest, "size_bytes": size, + "spool_ref": os.environ.get("SPOOL_REF_PREFIX", "") + relative_ref, "validated_format": suffix, "matched_local": matched_local} + finally: + if path.startswith(tempfile.gettempdir()) and os.path.exists(path): + os.unlink(path) + emit(result) + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except MissingSource as error: + emit({"ok": False, "missing": True, "error": str(error)}) + sys.exit(3) + except FetchError as error: + emit({"ok": False, "retryable": error.retryable, "code": error.code, "error": str(error)}) + sys.exit(4 if error.retryable else 2) diff --git a/web/src/App.tsx b/web/src/App.tsx index 5b073d9..3dac173 100644 --- a/web/src/App.tsx +++ b/web/src/App.tsx @@ -607,6 +607,7 @@ export function JobDashboard({ jobs, loading, error, onRefresh }: JobDashboardPr } const processingLabels: Record = { + planned: '待入库', pending: '待处理', processing: '解析中', finalizing: '定稿中', diff --git a/web/src/SystemSettings.test.tsx b/web/src/SystemSettings.test.tsx index 03c8e5c..0af791c 100644 --- a/web/src/SystemSettings.test.tsx +++ b/web/src/SystemSettings.test.tsx @@ -8,7 +8,7 @@ import type { SourceConnectionListItem } from './sourceConnections' const monitor:MonitorOverview={mode:'simulation',simulated:true,observed_at:'2026-09-08T12:00:00Z',host:{cpu_percent:42,memory_used_bytes:1,memory_total_bytes:2,disk_used_bytes:1,disk_total_bytes:2,gpu_state:'unknown',gpu_detail:'未接入真实GPU监控',simulated:true},services:[{key:'mineru',name:'MinerU',state:'healthy',detail:'模拟',latency_ms:1,observed_at:'2026-09-08T12:00:00Z',simulated:true}],knowledge_bases:[],health:[{key:'mineru',name:'MinerU',liveness:'up',readiness:'blocked',reason:'服务可达但容量已满(429),暂时不可接单',latency_ms:30,observed_at:'2026-09-08T12:00:00Z',last_success_at:'2026-09-08T11:59:00Z',consecutive_failures:2,configured:true,applicable:true}]} const sources:CatalogSource[]=[{connection_id:'ding-1',source_type:'dingtalk',name:'公司级技术支持知识库',root_node_id:'root-1',folder_count:1079,file_count:3444,observed_at:'2026-09-09T09:30:32Z',snapshot_only:true,snapshot_status:'ready',last_refresh_duration_ms:0}] const connections:SourceConnectionListItem[]=[{id:'conn-ding',source_type:'dingtalk',name:'公司级技术支持知识库',enabled:true,schedule_configured:true,health_status:'healthy',item_count:3444,updated_at:'2026-09-09T09:30:32Z',next_refresh_at:'2026-09-12T10:00:00Z'},{id:'conn-man',source_type:'manual',name:'本地手工验收',enabled:false,schedule_configured:false,health_status:'unknown',item_count:5,updated_at:'2026-09-09T09:30:32Z',next_refresh_at:null}] -const settings:UISettings={id:'settings-default',state:'active',document_refresh_seconds:15,resource_refresh_seconds:30,stalled_after_seconds:1800,finalizing_after_seconds:7200,updated_by:'system:migration',created_at:'2026-09-10T08:00:00Z',activated_at:'2026-09-10T08:00:00Z',retired_at:null,error_code:''} +const settings:UISettings={id:'settings-default',state:'active',document_refresh_seconds:15,resource_refresh_seconds:30,stalled_after_seconds:1800,finalizing_after_seconds:7200,ingestion_concurrency:2,updated_by:'system:migration',created_at:'2026-09-10T08:00:00Z',activated_at:'2026-09-10T08:00:00Z',retired_at:null,error_code:''} describe('SystemSettingsView',()=>{ it('separates document-engineer settings from locked administrator and production controls',()=>{ const html=renderToStaticMarkup() diff --git a/web/src/SystemSettings.tsx b/web/src/SystemSettings.tsx index d07e604..46fc336 100644 --- a/web/src/SystemSettings.tsx +++ b/web/src/SystemSettings.tsx @@ -138,6 +138,7 @@ export function SystemSettingsView({ sources, connections, monitor, settings, pr const [resourceRefresh, setResourceRefresh] = useState<30 | 60>(settings?.resource_refresh_seconds ?? 30) const [stalled, setStalled] = useState<1800 | 3600>(settings?.stalled_after_seconds ?? 1800) const [finalizing, setFinalizing] = useState<7200 | 14400>(settings?.finalizing_after_seconds ?? 7200) + const [concurrency, setConcurrency] = useState(settings?.ingestion_concurrency ?? 2) const [dirty, setDirty] = useState(false) useEffect(() => { @@ -146,6 +147,7 @@ export function SystemSettingsView({ sources, connections, monitor, settings, pr setResourceRefresh(settings.resource_refresh_seconds) setStalled(settings.stalled_after_seconds) setFinalizing(settings.finalizing_after_seconds) + setConcurrency(settings.ingestion_concurrency) setDirty(false) }, [settings]) @@ -183,10 +185,11 @@ export function SystemSettingsView({ sources, connections, monitor, settings, pr + - + {visibleSaveMessage(saveMessage, dirty) ?

{saveMessage}

: null} - 当前:文档 {secondsLabel(settings.document_refresh_seconds)}、资源 {secondsLabel(settings.resource_refresh_seconds)}、停滞 {secondsLabel(settings.stalled_after_seconds)}、finalizing {secondsLabel(settings.finalizing_after_seconds)}。 + 当前:文档 {secondsLabel(settings.document_refresh_seconds)}、资源 {secondsLabel(settings.resource_refresh_seconds)}、停滞 {secondsLabel(settings.stalled_after_seconds)}、finalizing {secondsLabel(settings.finalizing_after_seconds)}、入库并发 {settings.ingestion_concurrency}。 : null} : null} @@ -214,7 +217,7 @@ export function SystemSettings() { const save = async (input: UISettingsCandidateInput) => { setSaving(true); setSaveMessage(null); setError(null) - try { const candidate = await createUISettingsCandidate(input); const active = await activateUISettings(candidate.id); setSettings(active); setSaveMessage(`刷新与告警参数已验证并启用:文档${secondsLabel(active.document_refresh_seconds)}、资源${secondsLabel(active.resource_refresh_seconds)}、停滞${secondsLabel(active.stalled_after_seconds)}、finalizing ${secondsLabel(active.finalizing_after_seconds)};旧版本已保留用于审计。`) } + try { const candidate = await createUISettingsCandidate(input); const active = await activateUISettings(candidate.id); setSettings(active); setSaveMessage(`刷新与告警参数已验证并启用:文档${secondsLabel(active.document_refresh_seconds)}、资源${secondsLabel(active.resource_refresh_seconds)}、停滞${secondsLabel(active.stalled_after_seconds)}、finalizing ${secondsLabel(active.finalizing_after_seconds)}、入库并发 ${active.ingestion_concurrency};旧版本已保留用于审计。`) } catch (cause) { setError(cause instanceof Error ? cause.message : '保存系统设置失败') } finally { setSaving(false) } } diff --git a/web/src/WeKnoraMonitor.test.tsx b/web/src/WeKnoraMonitor.test.tsx index bc12dc1..63188ed 100644 --- a/web/src/WeKnoraMonitor.test.tsx +++ b/web/src/WeKnoraMonitor.test.tsx @@ -3,15 +3,13 @@ import { describe,expect,it,vi } from 'vitest' import { WeKnoraMonitorView } from './WeKnoraMonitor' import type { MonitorFilePage,MonitorOverview,RunMetrics } from './monitor' import type { TestPublicationPage } from './testPublications' -import type { CatalogChangeSetDetail } from './sourceCatalog' -const overview:MonitorOverview={mode:'simulation',simulated:true,observed_at:'2026-09-08T12:00:00Z',host:{cpu_percent:42,memory_used_bytes:19971597926,memory_total_bytes:34359738368,disk_used_bytes:128849018880,disk_total_bytes:536870912000,gpu_state:'unknown',gpu_detail:'未接入真实GPU监控',simulated:true},services:[{key:'weknora',name:'WeKnora API',state:'healthy',detail:'模拟连接正常',latency_ms:86,observed_at:'2026-09-08T12:00:00Z',simulated:true},{key:'mineru',name:'MinerU',state:'healthy',detail:'模拟处理中2,队列5',latency_ms:24,observed_at:'2026-09-08T12:00:00Z',simulated:true},{key:'postgres',name:'PostgreSQL',state:'unknown',detail:'未接入真实监控',latency_ms:0,observed_at:'2026-09-08T12:00:00Z',simulated:true},{key:'redis',name:'Redis / MQ',state:'unknown',detail:'未接入真实监控',latency_ms:0,observed_at:'2026-09-08T12:00:00Z',simulated:true},{key:'oss',name:'OSS',state:'unknown',detail:'未接入真实监控',latency_ms:0,observed_at:'2026-09-08T12:00:00Z',simulated:true}],knowledge_bases:[{id:'kb-sim',name:'公司级技术支持知识库',visibility:'mixed',imported:5750,completed:4530,pending:450,processing:50,finalizing:714,failed:6,cancelled:0,internal_imported:4200,internal_completed:3300,external_imported:1550,external_completed:1230}],health:[]} +const overview:MonitorOverview={mode:'simulation',simulated:true,observed_at:'2026-09-08T12:00:00Z',host:{cpu_percent:42,memory_used_bytes:19971597926,memory_total_bytes:34359738368,disk_used_bytes:128849018880,disk_total_bytes:536870912000,gpu_state:'unknown',gpu_detail:'未接入真实GPU监控',simulated:true},services:[{key:'weknora',name:'WeKnora API',state:'healthy',detail:'模拟连接正常',latency_ms:86,observed_at:'2026-09-08T12:00:00Z',simulated:true},{key:'mineru',name:'MinerU',state:'healthy',detail:'模拟处理中2,队列5',latency_ms:24,observed_at:'2026-09-08T12:00:00Z',simulated:true},{key:'postgres',name:'PostgreSQL',state:'unknown',detail:'未接入真实监控',latency_ms:0,observed_at:'2026-09-08T12:00:00Z',simulated:true},{key:'redis',name:'Redis / MQ',state:'unknown',detail:'未接入真实监控',latency_ms:0,observed_at:'2026-09-08T12:00:00Z',simulated:true},{key:'oss',name:'OSS',state:'unknown',detail:'未接入真实监控',latency_ms:0,observed_at:'2026-09-08T12:00:00Z',simulated:true}],knowledge_bases:[{id:'kb-sim',name:'公司级技术支持知识库',visibility:'mixed',imported:5750,completed:4530,pending:450,processing:50,finalizing:714,failed:6,cancelled:0,planned:120,needs_removal:3,stale_planned:0,internal_imported:4200,internal_completed:3300,external_imported:1550,external_completed:1230}],pipeline:{fetch_pending:4,fetch_failed:1,fetch_missing:0,concurrency:2},health:[]} const files:MonitorFilePage={items:[{name:'[模拟] 产品参数总表.xlsx',directory:'02|产品资料',visibility:'external',status:'finalizing',stage_detail:'基础检索可用,Wiki与图谱仍在生成',elapsed_seconds:2100,error_code:''}],total:1,limit:20,offset:0,sample_only:true} const metrics:RunMetrics={observed_at:'2026-09-12T00:00:00Z',active_workers:2,worker_source:'pg_lease',queue:{pending:3,running:4,retry:1,failed:2,oldest_pending_age_seconds:120},short_window:{window_seconds:3600,jobs_started:50,jobs_completed:48,jobs_failed:6,success_rate:88.9,failure_rate:11.1,duration_avg_seconds:55,duration_p50_seconds:30,duration_p95_seconds:1800},long_window:{window_seconds:86400,jobs_started:500,jobs_completed:482,jobs_failed:18,success_rate:96.4,failure_rate:3.6,duration_avg_seconds:70,duration_p50_seconds:45,duration_p95_seconds:2400}} -const queue:TestPublicationPage={items:[{id:'tp-plan',title:'API.docx',ingest_state:'pending',processing_state:'pending',retrieval_state:'unpublished',target_scope: 'internal', change_set_id: 'cs-1', change_type:'added',can_start:true,retry_count:0,last_error:'',can_retry:false,updated_at:'2026-09-09T10:01:00Z'},{id:'tp-failed',title:'CR20A.pdf',ingest_state:'uploaded',processing_state:'failed',retrieval_state:'unpublished',target_scope: 'external', change_set_id: 'cs-1', change_type:'updated',can_start:false,retry_count:1,last_error:'remote_parse_failed',can_retry:true,updated_at:'2026-09-09T10:00:00Z'}],total:2,limit:50,offset:0} -const changeSets:CatalogChangeSetDetail[]=[{id:'cs-1',connection_id:'conn-1',source_name:'国内工业',state:'planned',add_count:2,update_count:1,move_count:1,move_scope_count:1,delete_count:1,no_change_count:10,created_at:'2026-09-10T00:00:00Z',items:[{action:'delete',title:'旧手册.pdf',old_path:'旧目录/旧手册.pdf',new_path:'',reason_code:'source_missing',seq:7}],total:15,limit:20,offset:0}] - -const baseProps={overview,files,queue,changeSets,selectedKB:overview.knowledge_bases[0],status:'' as const,queueStatus:'' as const,queueScope:'' as const,loading:false,error:null as string|null,retryingIDs:[] as string[],starting:false,startMessage:'',onSelectKB:vi.fn(),onStatus:vi.fn(),onQueueStatus:vi.fn(),onQueueScope:vi.fn(),lane:'queue',onLaneChange:vi.fn(),metrics,onRetry:vi.fn(),onBatchRetry:vi.fn(),onStart:vi.fn(),onRefresh:vi.fn()} +const queue:TestPublicationPage={items:[{id:'tp-plan',title:'API.docx',ingest_state:'pending',processing_state:'planned',retrieval_state:'unpublished',target_scope: 'internal', change_set_id: '', change_type:'added',can_start:true,retry_count:0,last_error:'',can_retry:false,updated_at:'2026-09-09T10:01:00Z'},{id:'tp-fetch',title:'焊接手册.pdf',ingest_state:'pending',processing_state:'planned',retrieval_state:'unpublished',target_scope:'external',change_set_id:'',change_type:'added',can_start:false,retry_count:0,last_error:'',can_retry:false,updated_at:'2026-09-09T10:02:00Z'},{id:'tp-failed',title:'CR20A.pdf',ingest_state:'uploaded',processing_state:'failed',retrieval_state:'unpublished',target_scope: 'external', change_set_id: '', change_type:'updated',can_start:false,retry_count:1,last_error:'remote_parse_failed',can_retry:true,updated_at:'2026-09-09T10:00:00Z'}],total:3,limit:50,offset:0} +const plans=queue.items.filter((item)=>item.can_start) +const baseProps={overview,files,queue,plans,selectedKB:overview.knowledge_bases[0],status:'' as const,queueStatus:'' as const,queueScope:'' as const,loading:false,error:null as string|null,retryingIDs:[] as string[],starting:false,startMessage:'',onSelectKB:vi.fn(),onStatus:vi.fn(),onQueueStatus:vi.fn(),onQueueScope:vi.fn(),lane:'queue',onLaneChange:vi.fn(),metrics,onRetry:vi.fn(),onBatchRetry:vi.fn(),onStart:vi.fn(),onRefresh:vi.fn()} describe('WeKnoraMonitorView',()=>{ it('filters the processing queue by target knowledge base and shows each task target',()=>{ const html=renderToStaticMarkup() @@ -20,42 +18,35 @@ describe('WeKnoraMonitorView',()=>{ expect(html).toContain('对外知识库') expect(html).toContain('