fix(catalog): 快照节点绑定物化条目以显示导入进度

This commit is contained in:
姚章浩
2026-09-15 16:50:15 +08:00
parent 40d0c511d6
commit 0ef41594c2
8 changed files with 208 additions and 16 deletions
@@ -0,0 +1,114 @@
package postgres
import (
"context"
"os"
"testing"
"time"
"github.com/jackc/pgx/v5/pgxpool"
"knowledge-governance/internal/governance"
)
// A catalog-only file has no pre-existing source item. Materializing the
// policy intent must ALSO bind the snapshot node, otherwise the browser's
// 已导入/已解析 counters (which join node.source_item_id) stay at zero even
// after the document completed parsing — the bug behind "导入成功后目录没变化".
func TestRouteCatalogIntentBindsNodeForProgressStats(t *testing.T) {
databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL"))
if databaseURL == "" {
t.Skip("TEST_DATABASE_URL is not set")
}
ctx := context.Background()
pool, err := pgxpool.New(ctx, databaseURL)
if err != nil {
t.Fatal(err)
}
defer pool.Close()
if err = ApplyMigrations(ctx, pool); err != nil {
t.Fatal(err)
}
_, err = pool.Exec(ctx, `
INSERT INTO source_connections(id,source_type,name,enabled) VALUES('conn-bind','dingtalk','Bind ATOM',true);
INSERT INTO source_catalog_snapshots(id,source_connection_id,snapshot_sha256,source_label,node_count,folder_count,file_count,observed_at)
VALUES('snapshot-bind','conn-bind',repeat('a',64),'Bind ATOM',2,1,1,NOW());
INSERT INTO source_catalog_nodes(snapshot_id,node_id,parent_node_id,node_type,name,relative_path,source_locator_hash,size_bytes,direct_child_count,subtree_file_count,subtree_bytes)
VALUES
('snapshot-bind','root-bind',NULL,'folder','知识库','',repeat('1',64),NULL,1,1,10),
('snapshot-bind','file-bind','root-bind','file','手册.pdf','手册.pdf',repeat('2',64),10,0,1,10);
INSERT INTO source_catalog_heads(source_connection_id,snapshot_id,activated_at) VALUES('conn-bind','snapshot-bind',NOW());`)
if err != nil {
t.Fatal(err)
}
store := NewStore(pool)
now := time.Date(2026, 9, 15, 1, 0, 0, 0, time.UTC)
result, err := store.RouteCatalogFiles(ctx, governance.CatalogRouteRequest{
ConnectionID: "conn-bind", NodeIDs: []string{"file-bind"}, Visibility: governance.VisibilityInternal,
TargetKnowledgeBaseID: "kb-internal", OperatorRef: "engineer-bind", Reason: "单文件标注", Now: now,
})
if err != nil {
t.Fatal(err)
}
if result.IntentsRecorded != 1 {
t.Fatalf("result=%#v", result)
}
var linkedItem string
if err := pool.QueryRow(ctx, `SELECT COALESCE(source_item_id,'') FROM source_catalog_nodes WHERE snapshot_id='snapshot-bind' AND node_id='file-bind'`).Scan(&linkedItem); err != nil {
t.Fatal(err)
}
if linkedItem == "" {
t.Fatal("intent did not bind source_catalog_nodes.source_item_id")
}
// The browser stats read path must accept the bound node (publication may
// not exist yet while the fetch is queued, so only assert the join runs).
page, err := store.ListCatalogNodes(ctx, governance.CatalogNodeFilter{
ConnectionID: "conn-bind", ParentNodeID: "root-bind", Limit: 10,
})
if err != nil {
t.Fatal(err)
}
if len(page.Items) != 1 {
t.Fatalf("items=%#v", page.Items)
}
// Once bytes are verified and a publication exists, the same join reports
// it as imported.
if _, err := pool.Exec(ctx, `
INSERT INTO content_objects(id,sha256,size_bytes,local_spool_ref,content_state) VALUES('content-bind',repeat('c',64),10,'verified/bind.pdf','verified') ON CONFLICT DO NOTHING;
UPDATE source_item_versions SET content_object_id='content-bind',content_state='verified' WHERE source_item_id=(SELECT source_item_id FROM source_catalog_nodes WHERE snapshot_id='snapshot-bind' AND node_id='file-bind');
UPDATE test_publications SET ingest_state='uploaded',processing_state='completed',updated_at=NOW()
WHERE source_item_version_id IN (SELECT v.id FROM source_item_versions v
WHERE v.source_item_id=(SELECT source_item_id FROM source_catalog_nodes WHERE snapshot_id='snapshot-bind' AND node_id='file-bind'));`); err != nil {
t.Fatal(err)
}
page, err = store.ListCatalogNodes(ctx, governance.CatalogNodeFilter{
ConnectionID: "conn-bind", ParentNodeID: "root-bind", Limit: 10,
})
if err != nil {
t.Fatal(err)
}
if len(page.Items) != 1 {
t.Fatalf("items=%#v", page.Items)
}
if page.Items[0].ImportedCount != 1 || page.Items[0].ParsedCount != 1 {
t.Fatalf("imported=%d parsed=%d, want 1/1 (link visible to browser stats)", page.Items[0].ImportedCount, page.Items[0].ParsedCount)
}
// Re-annotation must not orphan the binding when the intent replays.
if _, err := store.RouteCatalogFiles(ctx, governance.CatalogRouteRequest{
ConnectionID: "conn-bind", NodeIDs: []string{"file-bind"}, Visibility: governance.VisibilityBoth,
TargetKnowledgeBaseIDs: []string{"kb-internal", "kb-external"}, OperatorRef: "engineer-bind", Reason: "改内外", Now: now.Add(time.Minute),
}); err != nil {
t.Fatal(err)
}
var stillBound string
if err := pool.QueryRow(ctx, `SELECT source_item_id FROM source_catalog_nodes WHERE snapshot_id='snapshot-bind' AND node_id='file-bind'`).Scan(&stillBound); err != nil {
t.Fatal(err)
}
if stillBound != linkedItem {
t.Fatalf("binding changed on re-annotation: %q != %q", stillBound, linkedItem)
}
}
@@ -104,8 +104,11 @@ INSERT INTO source_catalog_heads(source_connection_id,snapshot_id,activated_at)
// An unannotated live publication survives marking blocked: removal is
// knowledge-base diff (需下架) territory, never a silent side effect.
if _, err := pool.Exec(ctx, `INSERT INTO test_publications(id,source_item_version_id,test_kb_id,target_scope,ingest_state,processing_state,retrieval_state,knowledge_id,created_at,updated_at)
VALUES('tp-existing','version-route','kb-internal','internal','uploaded','completed','enabled','kn-1',$1,$1)`, now); err != nil {
// The verified-bytes fixture means the planner already holds the
// (version-route, kb-internal) row, so promote that row in place instead
// of inserting a duplicate.
if _, err := pool.Exec(ctx, `UPDATE test_publications SET ingest_state='uploaded',processing_state='completed',retrieval_state='enabled',knowledge_id='kn-1',updated_at=$1
WHERE source_item_version_id='version-route' AND test_kb_id='kb-internal' AND superseded_at IS NULL`, now); err != nil {
t.Fatal(err)
}
blocked, err := store.RouteCatalogFiles(ctx, governance.CatalogRouteRequest{ConnectionID: "conn-route", NodeIDs: []string{"folder-route"}, Visibility: governance.VisibilityBlocked, OperatorRef: "engineer-1", Reason: "禁止导入", Now: now.Add(3 * time.Minute)})
@@ -114,7 +117,7 @@ VALUES('tp-existing','version-route','kb-internal','internal','uploaded','comple
}
var retrievalState string
var removalJobs int
if err := pool.QueryRow(ctx, `SELECT retrieval_state FROM test_publications WHERE id='tp-existing'`).Scan(&retrievalState); err != nil {
if err := pool.QueryRow(ctx, `SELECT retrieval_state FROM test_publications WHERE source_item_version_id='version-route' AND test_kb_id='kb-internal' AND superseded_at IS NULL`).Scan(&retrievalState); err != nil {
t.Fatal(err)
}
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM jobs WHERE type='test.remove'`).Scan(&removalJobs); err != nil {
@@ -139,22 +142,24 @@ VALUES('tp-existing','version-route','kb-internal','internal','uploaded','comple
t.Fatalf("restored=%#v err=%v", restored, err)
}
var fetchState, itemPolicy string
if err := pool.QueryRow(ctx, `SELECT intent.fetch_state||'|'||item.policy_state FROM catalog_import_intents intent
if err := pool.QueryRow(ctx, `SELECT intent.fetch_state,item.policy_state FROM catalog_import_intents intent
JOIN source_items item ON item.id=intent.source_item_id
WHERE intent.node_id='file-unmapped'`).Scan(&fetchState, &itemPolicy); err != nil {
t.Fatal(err)
}
if fetchState != "pending|internal" {
t.Fatalf("fetch state=%q", fetchState)
if fetchState != "pending" || itemPolicy != "internal" {
t.Fatalf("fetch state=%q policy=%q", fetchState, itemPolicy)
}
var versionState string
if err := pool.QueryRow(ctx, `SELECT version.content_state FROM source_item_versions version
JOIN source_items item ON item.id=version.source_item_id AND item.revision=version.revision
WHERE item.stable_external_id='file-unmapped'`).Scan(&versionState); err != nil || versionState != "unknown" {
WHERE item.stable_external_id='cat:file-unmapped'`).Scan(&versionState); err != nil || versionState != "unknown" {
t.Fatalf("pending version state=%q err=%v", versionState, err)
}
var events int
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM scope_change_events`).Scan(&events); err != nil || events < 5 {
// Directory-level decisions record audit events; single-file annotations
// do not, so the four fan-out decisions above are the expected floor.
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM scope_change_events`).Scan(&events); err != nil || events < 4 {
t.Fatalf("scope events=%d err=%v", events, err)
}
}
@@ -0,0 +1,10 @@
-- Undo the backfill: nodes bound to policy-materialized (cat:) items.
UPDATE source_catalog_nodes node
SET source_item_id = NULL
FROM source_items item, catalog_import_intents intent, source_catalog_snapshots snap
WHERE item.id = node.source_item_id
AND item.stable_external_id LIKE 'cat:%'
AND snap.source_connection_id = intent.source_connection_id
AND node.snapshot_id = snap.id
AND node.node_id = intent.node_id
AND intent.source_item_id = item.id;
@@ -0,0 +1,15 @@
-- Bind snapshot file nodes to the source items their policy intents created.
-- Before this migration, annotation materialized items and linked them only on
-- catalog_import_intents; the catalog browser's 已导入/已解析 counters join
-- source_catalog_nodes.source_item_id, so every catalog-only source (ATOM and
-- friends) kept showing 尚未导入 even after documents finished parsing.
-- node_id is deterministic per (connection, path), so intents map 1:1.
UPDATE source_catalog_nodes node
SET source_item_id = intent.source_item_id
FROM catalog_import_intents intent, source_catalog_snapshots snap
WHERE snap.source_connection_id = intent.source_connection_id
AND node.snapshot_id = snap.id
AND node.node_id = intent.node_id
AND node.node_type = 'file'
AND node.source_item_id IS NULL
AND intent.source_item_id IS NOT NULL;
@@ -11,8 +11,8 @@ func TestEmbeddedMigrationsAreVersionSorted(t *testing.T) {
if err != nil {
t.Fatalf("loadMigrations() error = %v", err)
}
if len(migrations) != 39 {
t.Fatalf("migration count = %d, want 39", len(migrations))
if len(migrations) != 40 {
t.Fatalf("migration count = %d, want 40", len(migrations))
}
if migrations[0].Version != 1 || !strings.Contains(migrations[0].UpSQL, "CREATE TABLE IF NOT EXISTS jobs") {
t.Fatalf("migration 1 = %#v", migrations[0])
@@ -88,6 +88,23 @@ WHERE source_connection_id=$1 AND node_id=$2 FOR UPDATE`, request.ConnectionID,
if err != nil {
return false, false, 0, false, fmt.Errorf("read catalog intent: %w", err)
}
if existingItemID == nil {
// The node itself may already carry an item binding from an earlier
// scanner generation (or a sibling intent); reuse it instead of
// materializing a duplicate item and refetching identical bytes.
var nodeItemID *string
lookup := tx.QueryRow(ctx, `SELECT node.source_item_id
FROM source_catalog_heads head JOIN source_catalog_nodes node
ON node.snapshot_id=head.snapshot_id AND node.node_type='file'
WHERE head.source_connection_id=$1 AND node.node_id=$2
AND node.source_item_id IS NOT NULL
ORDER BY head.activated_at DESC NULLS LAST LIMIT 1 FOR UPDATE OF node`, request.ConnectionID, file.NodeID).Scan(&nodeItemID)
if lookup == nil {
existingItemID = nodeItemID
} else if !errors.Is(lookup, pgx.ErrNoRows) {
return false, false, 0, false, fmt.Errorf("read catalog node item: %w", lookup)
}
}
itemID := ""
if existingItemID != nil {
@@ -114,6 +131,18 @@ VALUES($1,$2,$3,$4,$5,'file','',$7,'configured_scope','unchanged',0,'object_id',
}
itemID = newItemID
}
// Bind the snapshot nodes too (new or reused item): the browser's
// 已导入/已解析 counters join source_catalog_nodes.source_item_id; without
// this write every catalog-only source keeps showing "尚未导入" even after
// the document finished parsing. node_id is deterministic per path, so we
// bind across all snapshots of this connection, current and future heads.
if _, err := tx.Exec(ctx, `UPDATE source_catalog_nodes node SET source_item_id=$3
FROM source_catalog_snapshots snap
WHERE snap.source_connection_id=$1 AND node.snapshot_id=snap.id
AND node.node_type='file' AND node.node_id=$2 AND node.source_item_id IS NULL`,
request.ConnectionID, file.NodeID, itemID); err != nil {
return false, false, 0, false, fmt.Errorf("bind catalog node item: %w", err)
}
if request.Visibility == governance.VisibilityBlocked {
if _, err := tx.Exec(ctx, `UPDATE source_items SET policy_state='blocked',updated_at=NOW() WHERE id=$1`, itemID); err != nil {
@@ -19,21 +19,26 @@ func TestLatestGovernanceMigrationRoundTripIntegration(t *testing.T) {
t.Fatal(err)
}
defer pool.Close()
migrations, err := loadMigrations()
if err != nil {
t.Fatal(err)
}
latest := int(migrations[len(migrations)-1].Version)
if err := ApplyMigrations(ctx, pool); err != nil {
t.Fatalf("initial ApplyMigrations() error = %v", err)
}
assertMigrationVersionAndTables(t, ctx, pool, 31, true)
assertMigrationVersionAndTables(t, ctx, pool, latest, true)
if err := ApplyMigrations(ctx, pool); err != nil {
t.Fatalf("idempotent ApplyMigrations() error = %v", err)
}
if err := RollbackLatestMigration(ctx, pool); err != nil {
t.Fatalf("RollbackLatestMigration() error = %v", err)
}
assertMigrationVersionAndTables(t, ctx, pool, 30, true)
assertMigrationVersionAndTables(t, ctx, pool, latest-1, true)
if err := ApplyMigrations(ctx, pool); err != nil {
t.Fatalf("reapply migrations error = %v", err)
}
assertMigrationVersionAndTables(t, ctx, pool, 31, true)
assertMigrationVersionAndTables(t, ctx, pool, latest, true)
}
func TestProcessingConfirmationMigrationBackfillsHistoricalExternalPublication(t *testing.T) {
@@ -47,10 +52,15 @@ func TestProcessingConfirmationMigrationBackfillsHistoricalExternalPublication(t
t.Fatal(err)
}
defer pool.Close()
migrations, err := loadMigrations()
if err != nil {
t.Fatal(err)
}
latest := int(migrations[len(migrations)-1].Version)
if err := ApplyMigrations(ctx, pool); err != nil {
t.Fatal(err)
}
for version := 31; version > 10; version-- {
for version := latest; version > 10; version-- {
if err := RollbackLatestMigration(ctx, pool); err != nil {
t.Fatal(err)
}
@@ -27,8 +27,17 @@ func TestV1Migration31SeedsRealSizedConnectionIDs(t *testing.T) {
t.Fatal(err)
}
// Roll back to just before migration 31 and re-enter it with live-shaped ids.
if err := RollbackLatestMigration(ctx, pool); err != nil {
t.Fatal(err)
for {
var version int
if err := pool.QueryRow(ctx, "SELECT MAX(version) FROM schema_migrations").Scan(&version); err != nil {
t.Fatal(err)
}
if version <= 30 {
break
}
if err := RollbackLatestMigration(ctx, pool); err != nil {
t.Fatal(err)
}
}
const uuidID = "a1b2c3d4-e5f6-4a7b-8c9d-0e1f2a3b4c5d" // exactly 36 chars
if _, err := pool.Exec(ctx, `INSERT INTO source_connections(id,source_type,name,enabled) VALUES($1,'alist','真实尺寸UUID源',true)`, uuidID); err != nil {