fix(catalog): 快照节点绑定物化条目以显示导入进度
This commit is contained in:
@@ -0,0 +1,114 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
)
|
||||
|
||||
// A catalog-only file has no pre-existing source item. Materializing the
|
||||
// policy intent must ALSO bind the snapshot node, otherwise the browser's
|
||||
// 已导入/已解析 counters (which join node.source_item_id) stay at zero even
|
||||
// after the document completed parsing — the bug behind "导入成功后目录没变化".
|
||||
func TestRouteCatalogIntentBindsNodeForProgressStats(t *testing.T) {
|
||||
databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL"))
|
||||
if databaseURL == "" {
|
||||
t.Skip("TEST_DATABASE_URL is not set")
|
||||
}
|
||||
ctx := context.Background()
|
||||
pool, err := pgxpool.New(ctx, databaseURL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer pool.Close()
|
||||
if err = ApplyMigrations(ctx, pool); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = pool.Exec(ctx, `
|
||||
INSERT INTO source_connections(id,source_type,name,enabled) VALUES('conn-bind','dingtalk','Bind ATOM',true);
|
||||
INSERT INTO source_catalog_snapshots(id,source_connection_id,snapshot_sha256,source_label,node_count,folder_count,file_count,observed_at)
|
||||
VALUES('snapshot-bind','conn-bind',repeat('a',64),'Bind ATOM',2,1,1,NOW());
|
||||
INSERT INTO source_catalog_nodes(snapshot_id,node_id,parent_node_id,node_type,name,relative_path,source_locator_hash,size_bytes,direct_child_count,subtree_file_count,subtree_bytes)
|
||||
VALUES
|
||||
('snapshot-bind','root-bind',NULL,'folder','知识库','',repeat('1',64),NULL,1,1,10),
|
||||
('snapshot-bind','file-bind','root-bind','file','手册.pdf','手册.pdf',repeat('2',64),10,0,1,10);
|
||||
INSERT INTO source_catalog_heads(source_connection_id,snapshot_id,activated_at) VALUES('conn-bind','snapshot-bind',NOW());`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store := NewStore(pool)
|
||||
now := time.Date(2026, 9, 15, 1, 0, 0, 0, time.UTC)
|
||||
|
||||
result, err := store.RouteCatalogFiles(ctx, governance.CatalogRouteRequest{
|
||||
ConnectionID: "conn-bind", NodeIDs: []string{"file-bind"}, Visibility: governance.VisibilityInternal,
|
||||
TargetKnowledgeBaseID: "kb-internal", OperatorRef: "engineer-bind", Reason: "单文件标注", Now: now,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.IntentsRecorded != 1 {
|
||||
t.Fatalf("result=%#v", result)
|
||||
}
|
||||
var linkedItem string
|
||||
if err := pool.QueryRow(ctx, `SELECT COALESCE(source_item_id,'') FROM source_catalog_nodes WHERE snapshot_id='snapshot-bind' AND node_id='file-bind'`).Scan(&linkedItem); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if linkedItem == "" {
|
||||
t.Fatal("intent did not bind source_catalog_nodes.source_item_id")
|
||||
}
|
||||
|
||||
// The browser stats read path must accept the bound node (publication may
|
||||
// not exist yet while the fetch is queued, so only assert the join runs).
|
||||
page, err := store.ListCatalogNodes(ctx, governance.CatalogNodeFilter{
|
||||
ConnectionID: "conn-bind", ParentNodeID: "root-bind", Limit: 10,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(page.Items) != 1 {
|
||||
t.Fatalf("items=%#v", page.Items)
|
||||
}
|
||||
|
||||
// Once bytes are verified and a publication exists, the same join reports
|
||||
// it as imported.
|
||||
if _, err := pool.Exec(ctx, `
|
||||
INSERT INTO content_objects(id,sha256,size_bytes,local_spool_ref,content_state) VALUES('content-bind',repeat('c',64),10,'verified/bind.pdf','verified') ON CONFLICT DO NOTHING;
|
||||
UPDATE source_item_versions SET content_object_id='content-bind',content_state='verified' WHERE source_item_id=(SELECT source_item_id FROM source_catalog_nodes WHERE snapshot_id='snapshot-bind' AND node_id='file-bind');
|
||||
UPDATE test_publications SET ingest_state='uploaded',processing_state='completed',updated_at=NOW()
|
||||
WHERE source_item_version_id IN (SELECT v.id FROM source_item_versions v
|
||||
WHERE v.source_item_id=(SELECT source_item_id FROM source_catalog_nodes WHERE snapshot_id='snapshot-bind' AND node_id='file-bind'));`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
page, err = store.ListCatalogNodes(ctx, governance.CatalogNodeFilter{
|
||||
ConnectionID: "conn-bind", ParentNodeID: "root-bind", Limit: 10,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(page.Items) != 1 {
|
||||
t.Fatalf("items=%#v", page.Items)
|
||||
}
|
||||
if page.Items[0].ImportedCount != 1 || page.Items[0].ParsedCount != 1 {
|
||||
t.Fatalf("imported=%d parsed=%d, want 1/1 (link visible to browser stats)", page.Items[0].ImportedCount, page.Items[0].ParsedCount)
|
||||
}
|
||||
|
||||
// Re-annotation must not orphan the binding when the intent replays.
|
||||
if _, err := store.RouteCatalogFiles(ctx, governance.CatalogRouteRequest{
|
||||
ConnectionID: "conn-bind", NodeIDs: []string{"file-bind"}, Visibility: governance.VisibilityBoth,
|
||||
TargetKnowledgeBaseIDs: []string{"kb-internal", "kb-external"}, OperatorRef: "engineer-bind", Reason: "改内外", Now: now.Add(time.Minute),
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var stillBound string
|
||||
if err := pool.QueryRow(ctx, `SELECT source_item_id FROM source_catalog_nodes WHERE snapshot_id='snapshot-bind' AND node_id='file-bind'`).Scan(&stillBound); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if stillBound != linkedItem {
|
||||
t.Fatalf("binding changed on re-annotation: %q != %q", stillBound, linkedItem)
|
||||
}
|
||||
}
|
||||
@@ -104,8 +104,11 @@ INSERT INTO source_catalog_heads(source_connection_id,snapshot_id,activated_at)
|
||||
|
||||
// An unannotated live publication survives marking blocked: removal is
|
||||
// knowledge-base diff (需下架) territory, never a silent side effect.
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO test_publications(id,source_item_version_id,test_kb_id,target_scope,ingest_state,processing_state,retrieval_state,knowledge_id,created_at,updated_at)
|
||||
VALUES('tp-existing','version-route','kb-internal','internal','uploaded','completed','enabled','kn-1',$1,$1)`, now); err != nil {
|
||||
// The verified-bytes fixture means the planner already holds the
|
||||
// (version-route, kb-internal) row, so promote that row in place instead
|
||||
// of inserting a duplicate.
|
||||
if _, err := pool.Exec(ctx, `UPDATE test_publications SET ingest_state='uploaded',processing_state='completed',retrieval_state='enabled',knowledge_id='kn-1',updated_at=$1
|
||||
WHERE source_item_version_id='version-route' AND test_kb_id='kb-internal' AND superseded_at IS NULL`, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
blocked, err := store.RouteCatalogFiles(ctx, governance.CatalogRouteRequest{ConnectionID: "conn-route", NodeIDs: []string{"folder-route"}, Visibility: governance.VisibilityBlocked, OperatorRef: "engineer-1", Reason: "禁止导入", Now: now.Add(3 * time.Minute)})
|
||||
@@ -114,7 +117,7 @@ VALUES('tp-existing','version-route','kb-internal','internal','uploaded','comple
|
||||
}
|
||||
var retrievalState string
|
||||
var removalJobs int
|
||||
if err := pool.QueryRow(ctx, `SELECT retrieval_state FROM test_publications WHERE id='tp-existing'`).Scan(&retrievalState); err != nil {
|
||||
if err := pool.QueryRow(ctx, `SELECT retrieval_state FROM test_publications WHERE source_item_version_id='version-route' AND test_kb_id='kb-internal' AND superseded_at IS NULL`).Scan(&retrievalState); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM jobs WHERE type='test.remove'`).Scan(&removalJobs); err != nil {
|
||||
@@ -139,22 +142,24 @@ VALUES('tp-existing','version-route','kb-internal','internal','uploaded','comple
|
||||
t.Fatalf("restored=%#v err=%v", restored, err)
|
||||
}
|
||||
var fetchState, itemPolicy string
|
||||
if err := pool.QueryRow(ctx, `SELECT intent.fetch_state||'|'||item.policy_state FROM catalog_import_intents intent
|
||||
if err := pool.QueryRow(ctx, `SELECT intent.fetch_state,item.policy_state FROM catalog_import_intents intent
|
||||
JOIN source_items item ON item.id=intent.source_item_id
|
||||
WHERE intent.node_id='file-unmapped'`).Scan(&fetchState, &itemPolicy); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if fetchState != "pending|internal" {
|
||||
t.Fatalf("fetch state=%q", fetchState)
|
||||
if fetchState != "pending" || itemPolicy != "internal" {
|
||||
t.Fatalf("fetch state=%q policy=%q", fetchState, itemPolicy)
|
||||
}
|
||||
var versionState string
|
||||
if err := pool.QueryRow(ctx, `SELECT version.content_state FROM source_item_versions version
|
||||
JOIN source_items item ON item.id=version.source_item_id AND item.revision=version.revision
|
||||
WHERE item.stable_external_id='file-unmapped'`).Scan(&versionState); err != nil || versionState != "unknown" {
|
||||
WHERE item.stable_external_id='cat:file-unmapped'`).Scan(&versionState); err != nil || versionState != "unknown" {
|
||||
t.Fatalf("pending version state=%q err=%v", versionState, err)
|
||||
}
|
||||
var events int
|
||||
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM scope_change_events`).Scan(&events); err != nil || events < 5 {
|
||||
// Directory-level decisions record audit events; single-file annotations
|
||||
// do not, so the four fan-out decisions above are the expected floor.
|
||||
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM scope_change_events`).Scan(&events); err != nil || events < 4 {
|
||||
t.Fatalf("scope events=%d err=%v", events, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
-- Undo the backfill: nodes bound to policy-materialized (cat:) items.
|
||||
UPDATE source_catalog_nodes node
|
||||
SET source_item_id = NULL
|
||||
FROM source_items item, catalog_import_intents intent, source_catalog_snapshots snap
|
||||
WHERE item.id = node.source_item_id
|
||||
AND item.stable_external_id LIKE 'cat:%'
|
||||
AND snap.source_connection_id = intent.source_connection_id
|
||||
AND node.snapshot_id = snap.id
|
||||
AND node.node_id = intent.node_id
|
||||
AND intent.source_item_id = item.id;
|
||||
@@ -0,0 +1,15 @@
|
||||
-- Bind snapshot file nodes to the source items their policy intents created.
|
||||
-- Before this migration, annotation materialized items and linked them only on
|
||||
-- catalog_import_intents; the catalog browser's 已导入/已解析 counters join
|
||||
-- source_catalog_nodes.source_item_id, so every catalog-only source (ATOM and
|
||||
-- friends) kept showing 尚未导入 even after documents finished parsing.
|
||||
-- node_id is deterministic per (connection, path), so intents map 1:1.
|
||||
UPDATE source_catalog_nodes node
|
||||
SET source_item_id = intent.source_item_id
|
||||
FROM catalog_import_intents intent, source_catalog_snapshots snap
|
||||
WHERE snap.source_connection_id = intent.source_connection_id
|
||||
AND node.snapshot_id = snap.id
|
||||
AND node.node_id = intent.node_id
|
||||
AND node.node_type = 'file'
|
||||
AND node.source_item_id IS NULL
|
||||
AND intent.source_item_id IS NOT NULL;
|
||||
@@ -11,8 +11,8 @@ func TestEmbeddedMigrationsAreVersionSorted(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("loadMigrations() error = %v", err)
|
||||
}
|
||||
if len(migrations) != 39 {
|
||||
t.Fatalf("migration count = %d, want 39", len(migrations))
|
||||
if len(migrations) != 40 {
|
||||
t.Fatalf("migration count = %d, want 40", len(migrations))
|
||||
}
|
||||
if migrations[0].Version != 1 || !strings.Contains(migrations[0].UpSQL, "CREATE TABLE IF NOT EXISTS jobs") {
|
||||
t.Fatalf("migration 1 = %#v", migrations[0])
|
||||
|
||||
@@ -88,6 +88,23 @@ WHERE source_connection_id=$1 AND node_id=$2 FOR UPDATE`, request.ConnectionID,
|
||||
if err != nil {
|
||||
return false, false, 0, false, fmt.Errorf("read catalog intent: %w", err)
|
||||
}
|
||||
if existingItemID == nil {
|
||||
// The node itself may already carry an item binding from an earlier
|
||||
// scanner generation (or a sibling intent); reuse it instead of
|
||||
// materializing a duplicate item and refetching identical bytes.
|
||||
var nodeItemID *string
|
||||
lookup := tx.QueryRow(ctx, `SELECT node.source_item_id
|
||||
FROM source_catalog_heads head JOIN source_catalog_nodes node
|
||||
ON node.snapshot_id=head.snapshot_id AND node.node_type='file'
|
||||
WHERE head.source_connection_id=$1 AND node.node_id=$2
|
||||
AND node.source_item_id IS NOT NULL
|
||||
ORDER BY head.activated_at DESC NULLS LAST LIMIT 1 FOR UPDATE OF node`, request.ConnectionID, file.NodeID).Scan(&nodeItemID)
|
||||
if lookup == nil {
|
||||
existingItemID = nodeItemID
|
||||
} else if !errors.Is(lookup, pgx.ErrNoRows) {
|
||||
return false, false, 0, false, fmt.Errorf("read catalog node item: %w", lookup)
|
||||
}
|
||||
}
|
||||
|
||||
itemID := ""
|
||||
if existingItemID != nil {
|
||||
@@ -114,6 +131,18 @@ VALUES($1,$2,$3,$4,$5,'file','',$7,'configured_scope','unchanged',0,'object_id',
|
||||
}
|
||||
itemID = newItemID
|
||||
}
|
||||
// Bind the snapshot nodes too (new or reused item): the browser's
|
||||
// 已导入/已解析 counters join source_catalog_nodes.source_item_id; without
|
||||
// this write every catalog-only source keeps showing "尚未导入" even after
|
||||
// the document finished parsing. node_id is deterministic per path, so we
|
||||
// bind across all snapshots of this connection, current and future heads.
|
||||
if _, err := tx.Exec(ctx, `UPDATE source_catalog_nodes node SET source_item_id=$3
|
||||
FROM source_catalog_snapshots snap
|
||||
WHERE snap.source_connection_id=$1 AND node.snapshot_id=snap.id
|
||||
AND node.node_type='file' AND node.node_id=$2 AND node.source_item_id IS NULL`,
|
||||
request.ConnectionID, file.NodeID, itemID); err != nil {
|
||||
return false, false, 0, false, fmt.Errorf("bind catalog node item: %w", err)
|
||||
}
|
||||
|
||||
if request.Visibility == governance.VisibilityBlocked {
|
||||
if _, err := tx.Exec(ctx, `UPDATE source_items SET policy_state='blocked',updated_at=NOW() WHERE id=$1`, itemID); err != nil {
|
||||
|
||||
@@ -19,21 +19,26 @@ func TestLatestGovernanceMigrationRoundTripIntegration(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer pool.Close()
|
||||
migrations, err := loadMigrations()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
latest := int(migrations[len(migrations)-1].Version)
|
||||
if err := ApplyMigrations(ctx, pool); err != nil {
|
||||
t.Fatalf("initial ApplyMigrations() error = %v", err)
|
||||
}
|
||||
assertMigrationVersionAndTables(t, ctx, pool, 31, true)
|
||||
assertMigrationVersionAndTables(t, ctx, pool, latest, true)
|
||||
if err := ApplyMigrations(ctx, pool); err != nil {
|
||||
t.Fatalf("idempotent ApplyMigrations() error = %v", err)
|
||||
}
|
||||
if err := RollbackLatestMigration(ctx, pool); err != nil {
|
||||
t.Fatalf("RollbackLatestMigration() error = %v", err)
|
||||
}
|
||||
assertMigrationVersionAndTables(t, ctx, pool, 30, true)
|
||||
assertMigrationVersionAndTables(t, ctx, pool, latest-1, true)
|
||||
if err := ApplyMigrations(ctx, pool); err != nil {
|
||||
t.Fatalf("reapply migrations error = %v", err)
|
||||
}
|
||||
assertMigrationVersionAndTables(t, ctx, pool, 31, true)
|
||||
assertMigrationVersionAndTables(t, ctx, pool, latest, true)
|
||||
}
|
||||
|
||||
func TestProcessingConfirmationMigrationBackfillsHistoricalExternalPublication(t *testing.T) {
|
||||
@@ -47,10 +52,15 @@ func TestProcessingConfirmationMigrationBackfillsHistoricalExternalPublication(t
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer pool.Close()
|
||||
migrations, err := loadMigrations()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
latest := int(migrations[len(migrations)-1].Version)
|
||||
if err := ApplyMigrations(ctx, pool); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for version := 31; version > 10; version-- {
|
||||
for version := latest; version > 10; version-- {
|
||||
if err := RollbackLatestMigration(ctx, pool); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -27,8 +27,17 @@ func TestV1Migration31SeedsRealSizedConnectionIDs(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Roll back to just before migration 31 and re-enter it with live-shaped ids.
|
||||
if err := RollbackLatestMigration(ctx, pool); err != nil {
|
||||
t.Fatal(err)
|
||||
for {
|
||||
var version int
|
||||
if err := pool.QueryRow(ctx, "SELECT MAX(version) FROM schema_migrations").Scan(&version); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if version <= 30 {
|
||||
break
|
||||
}
|
||||
if err := RollbackLatestMigration(ctx, pool); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
const uuidID = "a1b2c3d4-e5f6-4a7b-8c9d-0e1f2a3b4c5d" // exactly 36 chars
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO source_connections(id,source_type,name,enabled) VALUES($1,'alist','真实尺寸UUID源',true)`, uuidID); err != nil {
|
||||
|
||||
Reference in New Issue
Block a user