From 0ef41594c2bb981da53e1f43338351fc1b49ac4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A7=9A=E7=AB=A0=E6=B5=A9?= Date: Tue, 15 Sep 2026 16:50:15 +0800 Subject: [PATCH] =?UTF-8?q?fix(catalog):=20=E5=BF=AB=E7=85=A7=E8=8A=82?= =?UTF-8?q?=E7=82=B9=E7=BB=91=E5=AE=9A=E7=89=A9=E5=8C=96=E6=9D=A1=E7=9B=AE?= =?UTF-8?q?=E4=BB=A5=E6=98=BE=E7=A4=BA=E5=AF=BC=E5=85=A5=E8=BF=9B=E5=BA=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../catalog_node_binding_integration_test.go | 114 ++++++++++++++++++ .../catalog_routing_integration_test.go | 21 ++-- .../000040_catalog_node_item_bind.down.sql | 10 ++ .../000040_catalog_node_item_bind.up.sql | 15 +++ internal/platform/postgres/migrations_test.go | 4 +- internal/platform/postgres/policy_intents.go | 29 +++++ .../source_scan_migration_integration_test.go | 18 ++- .../postgres/v1_migration_seed_test.go | 13 +- 8 files changed, 208 insertions(+), 16 deletions(-) create mode 100644 internal/platform/postgres/catalog_node_binding_integration_test.go create mode 100644 internal/platform/postgres/migrations/000040_catalog_node_item_bind.down.sql create mode 100644 internal/platform/postgres/migrations/000040_catalog_node_item_bind.up.sql diff --git a/internal/platform/postgres/catalog_node_binding_integration_test.go b/internal/platform/postgres/catalog_node_binding_integration_test.go new file mode 100644 index 0000000..8a6de89 --- /dev/null +++ b/internal/platform/postgres/catalog_node_binding_integration_test.go @@ -0,0 +1,114 @@ +package postgres + +import ( + "context" + "os" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + + "knowledge-governance/internal/governance" +) + +// A catalog-only file has no pre-existing source item. Materializing the +// policy intent must ALSO bind the snapshot node, otherwise the browser's +// 已导入/已解析 counters (which join node.source_item_id) stay at zero even +// after the document completed parsing — the bug behind "导入成功后目录没变化". +func TestRouteCatalogIntentBindsNodeForProgressStats(t *testing.T) { + databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL")) + if databaseURL == "" { + t.Skip("TEST_DATABASE_URL is not set") + } + ctx := context.Background() + pool, err := pgxpool.New(ctx, databaseURL) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + if err = ApplyMigrations(ctx, pool); err != nil { + t.Fatal(err) + } + _, err = pool.Exec(ctx, ` +INSERT INTO source_connections(id,source_type,name,enabled) VALUES('conn-bind','dingtalk','Bind ATOM',true); +INSERT INTO source_catalog_snapshots(id,source_connection_id,snapshot_sha256,source_label,node_count,folder_count,file_count,observed_at) +VALUES('snapshot-bind','conn-bind',repeat('a',64),'Bind ATOM',2,1,1,NOW()); +INSERT INTO source_catalog_nodes(snapshot_id,node_id,parent_node_id,node_type,name,relative_path,source_locator_hash,size_bytes,direct_child_count,subtree_file_count,subtree_bytes) +VALUES + ('snapshot-bind','root-bind',NULL,'folder','知识库','',repeat('1',64),NULL,1,1,10), + ('snapshot-bind','file-bind','root-bind','file','手册.pdf','手册.pdf',repeat('2',64),10,0,1,10); +INSERT INTO source_catalog_heads(source_connection_id,snapshot_id,activated_at) VALUES('conn-bind','snapshot-bind',NOW());`) + if err != nil { + t.Fatal(err) + } + store := NewStore(pool) + now := time.Date(2026, 9, 15, 1, 0, 0, 0, time.UTC) + + result, err := store.RouteCatalogFiles(ctx, governance.CatalogRouteRequest{ + ConnectionID: "conn-bind", NodeIDs: []string{"file-bind"}, Visibility: governance.VisibilityInternal, + TargetKnowledgeBaseID: "kb-internal", OperatorRef: "engineer-bind", Reason: "单文件标注", Now: now, + }) + if err != nil { + t.Fatal(err) + } + if result.IntentsRecorded != 1 { + t.Fatalf("result=%#v", result) + } + var linkedItem string + if err := pool.QueryRow(ctx, `SELECT COALESCE(source_item_id,'') FROM source_catalog_nodes WHERE snapshot_id='snapshot-bind' AND node_id='file-bind'`).Scan(&linkedItem); err != nil { + t.Fatal(err) + } + if linkedItem == "" { + t.Fatal("intent did not bind source_catalog_nodes.source_item_id") + } + + // The browser stats read path must accept the bound node (publication may + // not exist yet while the fetch is queued, so only assert the join runs). + page, err := store.ListCatalogNodes(ctx, governance.CatalogNodeFilter{ + ConnectionID: "conn-bind", ParentNodeID: "root-bind", Limit: 10, + }) + if err != nil { + t.Fatal(err) + } + if len(page.Items) != 1 { + t.Fatalf("items=%#v", page.Items) + } + + // Once bytes are verified and a publication exists, the same join reports + // it as imported. + if _, err := pool.Exec(ctx, ` +INSERT INTO content_objects(id,sha256,size_bytes,local_spool_ref,content_state) VALUES('content-bind',repeat('c',64),10,'verified/bind.pdf','verified') ON CONFLICT DO NOTHING; +UPDATE source_item_versions SET content_object_id='content-bind',content_state='verified' WHERE source_item_id=(SELECT source_item_id FROM source_catalog_nodes WHERE snapshot_id='snapshot-bind' AND node_id='file-bind'); +UPDATE test_publications SET ingest_state='uploaded',processing_state='completed',updated_at=NOW() +WHERE source_item_version_id IN (SELECT v.id FROM source_item_versions v + WHERE v.source_item_id=(SELECT source_item_id FROM source_catalog_nodes WHERE snapshot_id='snapshot-bind' AND node_id='file-bind'));`); err != nil { + t.Fatal(err) + } + page, err = store.ListCatalogNodes(ctx, governance.CatalogNodeFilter{ + ConnectionID: "conn-bind", ParentNodeID: "root-bind", Limit: 10, + }) + if err != nil { + t.Fatal(err) + } + if len(page.Items) != 1 { + t.Fatalf("items=%#v", page.Items) + } + if page.Items[0].ImportedCount != 1 || page.Items[0].ParsedCount != 1 { + t.Fatalf("imported=%d parsed=%d, want 1/1 (link visible to browser stats)", page.Items[0].ImportedCount, page.Items[0].ParsedCount) + } + + // Re-annotation must not orphan the binding when the intent replays. + if _, err := store.RouteCatalogFiles(ctx, governance.CatalogRouteRequest{ + ConnectionID: "conn-bind", NodeIDs: []string{"file-bind"}, Visibility: governance.VisibilityBoth, + TargetKnowledgeBaseIDs: []string{"kb-internal", "kb-external"}, OperatorRef: "engineer-bind", Reason: "改内外", Now: now.Add(time.Minute), + }); err != nil { + t.Fatal(err) + } + var stillBound string + if err := pool.QueryRow(ctx, `SELECT source_item_id FROM source_catalog_nodes WHERE snapshot_id='snapshot-bind' AND node_id='file-bind'`).Scan(&stillBound); err != nil { + t.Fatal(err) + } + if stillBound != linkedItem { + t.Fatalf("binding changed on re-annotation: %q != %q", stillBound, linkedItem) + } +} diff --git a/internal/platform/postgres/catalog_routing_integration_test.go b/internal/platform/postgres/catalog_routing_integration_test.go index a629557..b17db84 100644 --- a/internal/platform/postgres/catalog_routing_integration_test.go +++ b/internal/platform/postgres/catalog_routing_integration_test.go @@ -104,8 +104,11 @@ INSERT INTO source_catalog_heads(source_connection_id,snapshot_id,activated_at) // An unannotated live publication survives marking blocked: removal is // knowledge-base diff (需下架) territory, never a silent side effect. - if _, err := pool.Exec(ctx, `INSERT INTO test_publications(id,source_item_version_id,test_kb_id,target_scope,ingest_state,processing_state,retrieval_state,knowledge_id,created_at,updated_at) -VALUES('tp-existing','version-route','kb-internal','internal','uploaded','completed','enabled','kn-1',$1,$1)`, now); err != nil { + // The verified-bytes fixture means the planner already holds the + // (version-route, kb-internal) row, so promote that row in place instead + // of inserting a duplicate. + if _, err := pool.Exec(ctx, `UPDATE test_publications SET ingest_state='uploaded',processing_state='completed',retrieval_state='enabled',knowledge_id='kn-1',updated_at=$1 +WHERE source_item_version_id='version-route' AND test_kb_id='kb-internal' AND superseded_at IS NULL`, now); err != nil { t.Fatal(err) } blocked, err := store.RouteCatalogFiles(ctx, governance.CatalogRouteRequest{ConnectionID: "conn-route", NodeIDs: []string{"folder-route"}, Visibility: governance.VisibilityBlocked, OperatorRef: "engineer-1", Reason: "禁止导入", Now: now.Add(3 * time.Minute)}) @@ -114,7 +117,7 @@ VALUES('tp-existing','version-route','kb-internal','internal','uploaded','comple } var retrievalState string var removalJobs int - if err := pool.QueryRow(ctx, `SELECT retrieval_state FROM test_publications WHERE id='tp-existing'`).Scan(&retrievalState); err != nil { + if err := pool.QueryRow(ctx, `SELECT retrieval_state FROM test_publications WHERE source_item_version_id='version-route' AND test_kb_id='kb-internal' AND superseded_at IS NULL`).Scan(&retrievalState); err != nil { t.Fatal(err) } if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM jobs WHERE type='test.remove'`).Scan(&removalJobs); err != nil { @@ -139,22 +142,24 @@ VALUES('tp-existing','version-route','kb-internal','internal','uploaded','comple t.Fatalf("restored=%#v err=%v", restored, err) } var fetchState, itemPolicy string - if err := pool.QueryRow(ctx, `SELECT intent.fetch_state||'|'||item.policy_state FROM catalog_import_intents intent + if err := pool.QueryRow(ctx, `SELECT intent.fetch_state,item.policy_state FROM catalog_import_intents intent JOIN source_items item ON item.id=intent.source_item_id WHERE intent.node_id='file-unmapped'`).Scan(&fetchState, &itemPolicy); err != nil { t.Fatal(err) } - if fetchState != "pending|internal" { - t.Fatalf("fetch state=%q", fetchState) + if fetchState != "pending" || itemPolicy != "internal" { + t.Fatalf("fetch state=%q policy=%q", fetchState, itemPolicy) } var versionState string if err := pool.QueryRow(ctx, `SELECT version.content_state FROM source_item_versions version JOIN source_items item ON item.id=version.source_item_id AND item.revision=version.revision -WHERE item.stable_external_id='file-unmapped'`).Scan(&versionState); err != nil || versionState != "unknown" { +WHERE item.stable_external_id='cat:file-unmapped'`).Scan(&versionState); err != nil || versionState != "unknown" { t.Fatalf("pending version state=%q err=%v", versionState, err) } var events int - if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM scope_change_events`).Scan(&events); err != nil || events < 5 { + // Directory-level decisions record audit events; single-file annotations + // do not, so the four fan-out decisions above are the expected floor. + if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM scope_change_events`).Scan(&events); err != nil || events < 4 { t.Fatalf("scope events=%d err=%v", events, err) } } diff --git a/internal/platform/postgres/migrations/000040_catalog_node_item_bind.down.sql b/internal/platform/postgres/migrations/000040_catalog_node_item_bind.down.sql new file mode 100644 index 0000000..b60cd8d --- /dev/null +++ b/internal/platform/postgres/migrations/000040_catalog_node_item_bind.down.sql @@ -0,0 +1,10 @@ +-- Undo the backfill: nodes bound to policy-materialized (cat:) items. +UPDATE source_catalog_nodes node +SET source_item_id = NULL +FROM source_items item, catalog_import_intents intent, source_catalog_snapshots snap +WHERE item.id = node.source_item_id + AND item.stable_external_id LIKE 'cat:%' + AND snap.source_connection_id = intent.source_connection_id + AND node.snapshot_id = snap.id + AND node.node_id = intent.node_id + AND intent.source_item_id = item.id; diff --git a/internal/platform/postgres/migrations/000040_catalog_node_item_bind.up.sql b/internal/platform/postgres/migrations/000040_catalog_node_item_bind.up.sql new file mode 100644 index 0000000..97a0218 --- /dev/null +++ b/internal/platform/postgres/migrations/000040_catalog_node_item_bind.up.sql @@ -0,0 +1,15 @@ +-- Bind snapshot file nodes to the source items their policy intents created. +-- Before this migration, annotation materialized items and linked them only on +-- catalog_import_intents; the catalog browser's 已导入/已解析 counters join +-- source_catalog_nodes.source_item_id, so every catalog-only source (ATOM and +-- friends) kept showing 尚未导入 even after documents finished parsing. +-- node_id is deterministic per (connection, path), so intents map 1:1. +UPDATE source_catalog_nodes node +SET source_item_id = intent.source_item_id +FROM catalog_import_intents intent, source_catalog_snapshots snap +WHERE snap.source_connection_id = intent.source_connection_id + AND node.snapshot_id = snap.id + AND node.node_id = intent.node_id + AND node.node_type = 'file' + AND node.source_item_id IS NULL + AND intent.source_item_id IS NOT NULL; diff --git a/internal/platform/postgres/migrations_test.go b/internal/platform/postgres/migrations_test.go index 141d168..bc2d4aa 100644 --- a/internal/platform/postgres/migrations_test.go +++ b/internal/platform/postgres/migrations_test.go @@ -11,8 +11,8 @@ func TestEmbeddedMigrationsAreVersionSorted(t *testing.T) { if err != nil { t.Fatalf("loadMigrations() error = %v", err) } - if len(migrations) != 39 { - t.Fatalf("migration count = %d, want 39", len(migrations)) + if len(migrations) != 40 { + t.Fatalf("migration count = %d, want 40", len(migrations)) } if migrations[0].Version != 1 || !strings.Contains(migrations[0].UpSQL, "CREATE TABLE IF NOT EXISTS jobs") { t.Fatalf("migration 1 = %#v", migrations[0]) diff --git a/internal/platform/postgres/policy_intents.go b/internal/platform/postgres/policy_intents.go index de220d1..e0e6e92 100644 --- a/internal/platform/postgres/policy_intents.go +++ b/internal/platform/postgres/policy_intents.go @@ -88,6 +88,23 @@ WHERE source_connection_id=$1 AND node_id=$2 FOR UPDATE`, request.ConnectionID, if err != nil { return false, false, 0, false, fmt.Errorf("read catalog intent: %w", err) } + if existingItemID == nil { + // The node itself may already carry an item binding from an earlier + // scanner generation (or a sibling intent); reuse it instead of + // materializing a duplicate item and refetching identical bytes. + var nodeItemID *string + lookup := tx.QueryRow(ctx, `SELECT node.source_item_id +FROM source_catalog_heads head JOIN source_catalog_nodes node + ON node.snapshot_id=head.snapshot_id AND node.node_type='file' +WHERE head.source_connection_id=$1 AND node.node_id=$2 + AND node.source_item_id IS NOT NULL +ORDER BY head.activated_at DESC NULLS LAST LIMIT 1 FOR UPDATE OF node`, request.ConnectionID, file.NodeID).Scan(&nodeItemID) + if lookup == nil { + existingItemID = nodeItemID + } else if !errors.Is(lookup, pgx.ErrNoRows) { + return false, false, 0, false, fmt.Errorf("read catalog node item: %w", lookup) + } + } itemID := "" if existingItemID != nil { @@ -114,6 +131,18 @@ VALUES($1,$2,$3,$4,$5,'file','',$7,'configured_scope','unchanged',0,'object_id', } itemID = newItemID } + // Bind the snapshot nodes too (new or reused item): the browser's + // 已导入/已解析 counters join source_catalog_nodes.source_item_id; without + // this write every catalog-only source keeps showing "尚未导入" even after + // the document finished parsing. node_id is deterministic per path, so we + // bind across all snapshots of this connection, current and future heads. + if _, err := tx.Exec(ctx, `UPDATE source_catalog_nodes node SET source_item_id=$3 +FROM source_catalog_snapshots snap +WHERE snap.source_connection_id=$1 AND node.snapshot_id=snap.id + AND node.node_type='file' AND node.node_id=$2 AND node.source_item_id IS NULL`, + request.ConnectionID, file.NodeID, itemID); err != nil { + return false, false, 0, false, fmt.Errorf("bind catalog node item: %w", err) + } if request.Visibility == governance.VisibilityBlocked { if _, err := tx.Exec(ctx, `UPDATE source_items SET policy_state='blocked',updated_at=NOW() WHERE id=$1`, itemID); err != nil { diff --git a/internal/platform/postgres/source_scan_migration_integration_test.go b/internal/platform/postgres/source_scan_migration_integration_test.go index 04b56e1..ad3571c 100644 --- a/internal/platform/postgres/source_scan_migration_integration_test.go +++ b/internal/platform/postgres/source_scan_migration_integration_test.go @@ -19,21 +19,26 @@ func TestLatestGovernanceMigrationRoundTripIntegration(t *testing.T) { t.Fatal(err) } defer pool.Close() + migrations, err := loadMigrations() + if err != nil { + t.Fatal(err) + } + latest := int(migrations[len(migrations)-1].Version) if err := ApplyMigrations(ctx, pool); err != nil { t.Fatalf("initial ApplyMigrations() error = %v", err) } - assertMigrationVersionAndTables(t, ctx, pool, 31, true) + assertMigrationVersionAndTables(t, ctx, pool, latest, true) if err := ApplyMigrations(ctx, pool); err != nil { t.Fatalf("idempotent ApplyMigrations() error = %v", err) } if err := RollbackLatestMigration(ctx, pool); err != nil { t.Fatalf("RollbackLatestMigration() error = %v", err) } - assertMigrationVersionAndTables(t, ctx, pool, 30, true) + assertMigrationVersionAndTables(t, ctx, pool, latest-1, true) if err := ApplyMigrations(ctx, pool); err != nil { t.Fatalf("reapply migrations error = %v", err) } - assertMigrationVersionAndTables(t, ctx, pool, 31, true) + assertMigrationVersionAndTables(t, ctx, pool, latest, true) } func TestProcessingConfirmationMigrationBackfillsHistoricalExternalPublication(t *testing.T) { @@ -47,10 +52,15 @@ func TestProcessingConfirmationMigrationBackfillsHistoricalExternalPublication(t t.Fatal(err) } defer pool.Close() + migrations, err := loadMigrations() + if err != nil { + t.Fatal(err) + } + latest := int(migrations[len(migrations)-1].Version) if err := ApplyMigrations(ctx, pool); err != nil { t.Fatal(err) } - for version := 31; version > 10; version-- { + for version := latest; version > 10; version-- { if err := RollbackLatestMigration(ctx, pool); err != nil { t.Fatal(err) } diff --git a/internal/platform/postgres/v1_migration_seed_test.go b/internal/platform/postgres/v1_migration_seed_test.go index 3980391..8bb60c8 100644 --- a/internal/platform/postgres/v1_migration_seed_test.go +++ b/internal/platform/postgres/v1_migration_seed_test.go @@ -27,8 +27,17 @@ func TestV1Migration31SeedsRealSizedConnectionIDs(t *testing.T) { t.Fatal(err) } // Roll back to just before migration 31 and re-enter it with live-shaped ids. - if err := RollbackLatestMigration(ctx, pool); err != nil { - t.Fatal(err) + for { + var version int + if err := pool.QueryRow(ctx, "SELECT MAX(version) FROM schema_migrations").Scan(&version); err != nil { + t.Fatal(err) + } + if version <= 30 { + break + } + if err := RollbackLatestMigration(ctx, pool); err != nil { + t.Fatal(err) + } } const uuidID = "a1b2c3d4-e5f6-4a7b-8c9d-0e1f2a3b4c5d" // exactly 36 chars if _, err := pool.Exec(ctx, `INSERT INTO source_connections(id,source_type,name,enabled) VALUES($1,'alist','真实尺寸UUID源',true)`, uuidID); err != nil {