feat(governance): policy-driven ingestion from catalog annotations
Annotation in 来源与目录 is now the only import decision surface: - migration 000033: catalog_import_intents (durable, rescan-proof per-file intent with locator snapshot + fetch lifecycle), source_items.policy_state, planned publication state, ui_setting_revisions.ingestion_concurrency - marking fans a folder decision out to per-file intents, materializes versioned items, reuses already-verified local bytes (compare, don't pull), and plans one publication per routed KB; blocked retires planned rows while live publications stay untouched (需下架 shows in the KB diff) - artifact.fetch worker lane (scripts/artifact_fetch.py): DingTalk export tree/live box API, AList /api/fs/get direct download with SSO-redirect fail-closed, PP ledger via locator hash; content-level dedupe on sha256 - 开始处理 promotes planned->pending inside the claim-time concurrency budget from system settings; claims enforce it in SQL - monitor overview rebuilt as knowledge-base diff: planned/needs_removal per KB + fetch pipeline summary; frontend lanes 待入库/需下架 replace the deprecated snapshot change-set confirmation - spool: images (png/jpg/gif/bmp/webp) accepted, ppt/pptx rejected in favour of PDF twins; DingTalk PPT resolves its converted PDF pair
This commit is contained in:
@@ -0,0 +1,39 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"knowledge-governance/internal/workflows/artifactfetch"
|
||||
)
|
||||
|
||||
// LoadArtifactFetchManifest resolves one pending intent plus its live source
|
||||
// coordinates for the fetch script, keyed by the connection-stable node id.
|
||||
func (s *Store) LoadArtifactFetchManifest(ctx context.Context, connectionID, nodeID string) (artifactfetch.Manifest, error) {
|
||||
return loadArtifactFetchManifest(ctx, s.db, connectionID, nodeID)
|
||||
}
|
||||
|
||||
func loadArtifactFetchManifest(ctx context.Context, db interface {
|
||||
QueryRow(context.Context, string, ...any) pgx.Row
|
||||
}, connectionID, nodeID string) (artifactfetch.Manifest, error) {
|
||||
var manifest artifactfetch.Manifest
|
||||
err := db.QueryRow(ctx, `SELECT connection.source_type,connection.name,intent.relative_path,intent.name,
|
||||
intent.size_bytes,intent.modified_at,intent.locator_hash
|
||||
FROM catalog_import_intents intent
|
||||
JOIN source_connections connection ON connection.id=intent.source_connection_id
|
||||
WHERE intent.source_connection_id=$1 AND intent.node_id=$2`, connectionID, nodeID).
|
||||
Scan(&manifest.SourceType, &manifest.SourceName, &manifest.RelativePath, &manifest.Name,
|
||||
&manifest.SizeBytes, &manifest.ModifiedAt, &manifest.LocatorHash)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return artifactfetch.Manifest{}, artifactfetch.ErrManifestGone
|
||||
}
|
||||
if err != nil {
|
||||
return artifactfetch.Manifest{}, fmt.Errorf("load artifact fetch manifest: %w", err)
|
||||
}
|
||||
manifest.ConnectionID = connectionID
|
||||
manifest.NodeID = nodeID
|
||||
return manifest, nil
|
||||
}
|
||||
@@ -112,7 +112,8 @@ FOR UPDATE OF node`, request.ConnectionID, nodeIDs)
|
||||
JOIN source_catalog_nodes child ON child.snapshot_id=parent.snapshot_id AND child.parent_node_id=parent.node_id
|
||||
WHERE parent.node_type='folder'
|
||||
)
|
||||
SELECT DISTINCT node.node_id,node.source_item_id,(node.node_id=ANY($2::text[])) AS directly_selected
|
||||
SELECT DISTINCT node.node_id,node.name,node.relative_path,node.size_bytes,node.modified_at,node.content_sha256,
|
||||
node.source_locator_hash,(node.node_id=ANY($2::text[])) AS directly_selected
|
||||
FROM descendants
|
||||
JOIN source_catalog_nodes node ON node.snapshot_id=descendants.snapshot_id AND node.node_id=descendants.node_id
|
||||
WHERE node.node_type='file'
|
||||
@@ -120,15 +121,11 @@ ORDER BY node.node_id`, request.ConnectionID, nodeIDs)
|
||||
if err != nil {
|
||||
return governance.CatalogRouteResult{}, fmt.Errorf("select catalog route files: %w", err)
|
||||
}
|
||||
type selectedFile struct {
|
||||
nodeID string
|
||||
sourceItemID *string
|
||||
direct bool
|
||||
}
|
||||
files := make([]selectedFile, 0, len(nodeIDs))
|
||||
files := make([]governance.CatalogRouteFile, 0, len(nodeIDs))
|
||||
for rows.Next() {
|
||||
var file selectedFile
|
||||
if err := rows.Scan(&file.nodeID, &file.sourceItemID, &file.direct); err != nil {
|
||||
var file governance.CatalogRouteFile
|
||||
if err := rows.Scan(&file.NodeID, &file.Name, &file.RelativePath, &file.SizeBytes, &file.ModifiedAt,
|
||||
&file.ContentSHA256, &file.LocatorHash, &file.Direct); err != nil {
|
||||
rows.Close()
|
||||
return governance.CatalogRouteResult{}, fmt.Errorf("scan catalog route file: %w", err)
|
||||
}
|
||||
@@ -139,52 +136,7 @@ ORDER BY node.node_id`, request.ConnectionID, nodeIDs)
|
||||
return governance.CatalogRouteResult{}, fmt.Errorf("iterate catalog route files: %w", err)
|
||||
}
|
||||
rows.Close()
|
||||
|
||||
result := governance.CatalogRouteResult{SelectedFiles: len(files)}
|
||||
routedItems := map[string]bool{}
|
||||
for _, file := range files {
|
||||
if file.sourceItemID == nil {
|
||||
result.UnboundFiles++
|
||||
continue
|
||||
}
|
||||
itemID := *file.sourceItemID
|
||||
if routedItems[itemID] {
|
||||
continue
|
||||
}
|
||||
routedItems[itemID] = true
|
||||
if file.direct {
|
||||
if _, err := tx.Exec(ctx, `INSERT INTO source_item_scope_overrides(source_item_id,visibility,updated_by,reason,created_at,updated_at)
|
||||
VALUES($1,$2,$3,$4,$5,$5) ON CONFLICT(source_item_id) DO UPDATE SET visibility=EXCLUDED.visibility,updated_by=EXCLUDED.updated_by,reason=EXCLUDED.reason,updated_at=EXCLUDED.updated_at`, itemID, request.Visibility, request.OperatorRef, request.Reason, request.Now); err != nil {
|
||||
return governance.CatalogRouteResult{}, fmt.Errorf("set routed item scope: %w", err)
|
||||
}
|
||||
eventID, err := newOpaqueID("se-")
|
||||
if err != nil {
|
||||
return governance.CatalogRouteResult{}, err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `INSERT INTO scope_change_events(id,target_type,source_item_id,action,decided_visibility,operator_ref,reason,created_at)
|
||||
VALUES($1,'item',$2,'set',$3,$4,$5,$6)`, eventID, itemID, request.Visibility, request.OperatorRef, request.Reason, request.Now); err != nil {
|
||||
return governance.CatalogRouteResult{}, fmt.Errorf("audit routed item scope: %w", err)
|
||||
}
|
||||
}
|
||||
// S7: classify only. Annotations never create publications, retire targets, or enqueue jobs.
|
||||
var currentVersionID *string
|
||||
var contentState, spoolRef string
|
||||
err = tx.QueryRow(ctx, `SELECT version.id,content.content_state,COALESCE(content.local_spool_ref,'') FROM source_items item
|
||||
LEFT JOIN source_item_versions version ON version.source_item_id=item.id AND version.revision=item.revision
|
||||
LEFT JOIN content_objects content ON content.id=version.content_object_id
|
||||
WHERE item.id=$1`, itemID).Scan(¤tVersionID, &contentState, &spoolRef)
|
||||
switch {
|
||||
case errors.Is(err, pgx.ErrNoRows):
|
||||
result.UnavailableFiles++
|
||||
case err != nil:
|
||||
return governance.CatalogRouteResult{}, fmt.Errorf("inspect routed artifact: %w", err)
|
||||
case contentState == string(governance.ContentVerified) && spoolRef != "":
|
||||
result.RoutableFiles++
|
||||
default:
|
||||
result.HashUnknownFiles++
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
return fulfillCatalogIntentsTx(ctx, tx, request, files, request.Now)
|
||||
}
|
||||
|
||||
func setCatalogDirectoryPolicy(ctx context.Context, tx pgx.Tx, request governance.CatalogRouteRequest, directoryPath string) error {
|
||||
|
||||
@@ -10,9 +10,10 @@ import (
|
||||
"knowledge-governance/internal/governance"
|
||||
)
|
||||
|
||||
// S7: catalog annotation persists ONLY desired state (scope rules/overrides + audit).
|
||||
// It never creates or mutates test_publications and never enqueues jobs.
|
||||
func TestRouteCatalogFilesWritesDesiredStateWithoutSideEffects(t *testing.T) {
|
||||
// Policy-driven ingestion: saving an annotation records durable intents,
|
||||
// materializes items, enqueues artifact fetches and plans publications.
|
||||
// The only human action left is 开始处理; marking is the decision.
|
||||
func TestRouteCatalogFilesFulfillsPolicyIntent(t *testing.T) {
|
||||
databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL"))
|
||||
if databaseURL == "" {
|
||||
t.Skip("TEST_DATABASE_URL is not set")
|
||||
@@ -46,66 +47,70 @@ INSERT INTO source_catalog_heads(source_connection_id,snapshot_id,activated_at)
|
||||
}
|
||||
store := NewStore(pool)
|
||||
now := time.Date(2026, 9, 9, 1, 0, 0, 0, time.UTC)
|
||||
|
||||
// Folder annotation fans out to files; the bound file already holds verified
|
||||
// bytes, so it is planned without any fetch (本地已有 → 比对复用 → 不拉取).
|
||||
result, err := store.RouteCatalogFiles(ctx, governance.CatalogRouteRequest{ConnectionID: "conn-route", NodeIDs: []string{"folder-route"}, Visibility: governance.VisibilityInternal, TargetKnowledgeBaseID: "kb-internal", OperatorRef: "engineer-1", Reason: "整目录标注", Now: now})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.SelectedFiles != 1 || result.RoutableFiles != 1 || result.UnboundFiles != 0 || result.HashUnknownFiles != 0 {
|
||||
if result.SelectedFiles != 1 || result.IntentsRecorded != 1 || result.PlannedPublications != 1 || result.FetchQueued != 0 {
|
||||
t.Fatalf("result=%#v", result)
|
||||
}
|
||||
// Desired state saved, recursion intact.
|
||||
var directoryPolicy string
|
||||
if err := pool.QueryRow(ctx, `SELECT visibility FROM directory_scope_rules WHERE source_connection_id='conn-route' AND directory_path='工业产品'`).Scan(&directoryPolicy); err != nil {
|
||||
if err := pool.QueryRow(ctx, `SELECT visibility FROM directory_scope_rules WHERE source_connection_id='conn-route' AND directory_path='工业产品'`).Scan(&directoryPolicy); err != nil || directoryPolicy != "internal" {
|
||||
t.Fatalf("directory policy=%q err=%v", directoryPolicy, err)
|
||||
}
|
||||
var intentCount, publicationCount, fetchJobs, uploadJobs int
|
||||
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM catalog_import_intents WHERE source_connection_id='conn-route' AND visibility='internal'`).Scan(&intentCount); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if directoryPolicy != "internal" {
|
||||
t.Fatalf("directory policy=%q", directoryPolicy)
|
||||
}
|
||||
// Zero side effects: no publications, no jobs.
|
||||
var publicationCount, jobCount int
|
||||
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM test_publications`).Scan(&publicationCount); err != nil {
|
||||
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM test_publications WHERE processing_state='planned' AND superseded_at IS NULL`).Scan(&publicationCount); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM jobs`).Scan(&jobCount); err != nil {
|
||||
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM jobs WHERE type='artifact.fetch'`).Scan(&fetchJobs); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if publicationCount != 0 || jobCount != 0 {
|
||||
t.Fatalf("annotation must not touch pipeline: publications=%d jobs=%d", publicationCount, jobCount)
|
||||
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM jobs WHERE type='test.upload'`).Scan(&uploadJobs); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if intentCount != 1 || publicationCount != 1 || fetchJobs != 0 || uploadJobs != 0 {
|
||||
t.Fatalf("intents=%d planned=%d fetch=%d upload=%d", intentCount, publicationCount, fetchJobs, uploadJobs)
|
||||
}
|
||||
// The planned publication binds the pre-verified current version.
|
||||
var plannedVersion string
|
||||
if err := pool.QueryRow(ctx, `SELECT source_item_version_id FROM test_publications WHERE test_kb_id='kb-internal'`).Scan(&plannedVersion); err != nil || plannedVersion != "version-route" {
|
||||
t.Fatalf("planned version=%q err=%v", plannedVersion, err)
|
||||
}
|
||||
|
||||
// Re-annotating the same directory is idempotent for rules and still side-effect free.
|
||||
// Re-marking the same directory is replay-safe: one intent, one publication.
|
||||
again, err := store.RouteCatalogFiles(ctx, governance.CatalogRouteRequest{ConnectionID: "conn-route", NodeIDs: []string{"folder-route"}, Visibility: governance.VisibilityInternal, TargetKnowledgeBaseID: "kb-internal", OperatorRef: "engineer-1", Reason: "重复提交", Now: now.Add(time.Minute)})
|
||||
if err != nil || again.SelectedFiles != 1 || again.RoutableFiles != 1 {
|
||||
if err != nil || again.SelectedFiles != 1 || again.PlannedPublications != 0 || again.FetchQueued != 0 {
|
||||
t.Fatalf("again=%#v err=%v", again, err)
|
||||
}
|
||||
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM jobs`).Scan(&jobCount); err != nil || jobCount != 0 {
|
||||
t.Fatalf("repeat annotation jobs=%d err=%v", jobCount, err)
|
||||
}
|
||||
var ruleCount int
|
||||
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM directory_scope_rules WHERE source_connection_id='conn-route' AND directory_path='工业产品'`).Scan(&ruleCount); err != nil || ruleCount != 1 {
|
||||
t.Fatalf("rule rows=%d err=%v", ruleCount, err)
|
||||
}
|
||||
|
||||
// Switching policy updates desired state without retiring or removing anything.
|
||||
// Widening to both plans a second KB publication; nothing is fetched again.
|
||||
both, err := store.RouteCatalogFiles(ctx, governance.CatalogRouteRequest{ConnectionID: "conn-route", NodeIDs: []string{"folder-route"}, Visibility: governance.VisibilityBoth, TargetKnowledgeBaseIDs: []string{"kb-internal", "kb-external"}, OperatorRef: "engineer-1", Reason: "内外知识库同时使用", Now: now.Add(2 * time.Minute)})
|
||||
if err != nil || both.SelectedFiles != 1 || both.RoutableFiles != 1 {
|
||||
if err != nil || both.SelectedFiles != 1 || both.PlannedPublications != 1 || both.FetchQueued != 0 {
|
||||
t.Fatalf("both=%#v err=%v", both, err)
|
||||
}
|
||||
if err := pool.QueryRow(ctx, `SELECT visibility FROM directory_scope_rules WHERE source_connection_id='conn-route' AND directory_path='工业产品'`).Scan(&directoryPolicy); err != nil || directoryPolicy != "both" {
|
||||
t.Fatalf("policy after both=%q err=%v", directoryPolicy, err)
|
||||
}
|
||||
|
||||
// Blocking saves the desired policy only; existing publications (if any) stay untouched.
|
||||
// An unannotated live publication survives marking blocked: removal is
|
||||
// knowledge-base diff (需下架) territory, never a silent side effect.
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO test_publications(id,source_item_version_id,test_kb_id,target_scope,ingest_state,processing_state,retrieval_state,knowledge_id,created_at,updated_at)
|
||||
VALUES('tp-existing','version-route','kb-internal','internal','uploaded','completed','enabled','kn-1',$1,$1)`, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
blocked, err := store.RouteCatalogFiles(ctx, governance.CatalogRouteRequest{ConnectionID: "conn-route", NodeIDs: []string{"folder-route"}, Visibility: governance.VisibilityBlocked, OperatorRef: "engineer-1", Reason: "禁止导入", Now: now.Add(3 * time.Minute)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if blocked.SelectedFiles != 1 {
|
||||
t.Fatalf("blocked=%#v", blocked)
|
||||
if err != nil || blocked.SelectedFiles != 1 || blocked.BlockedItems != 1 {
|
||||
t.Fatalf("blocked=%#v err=%v", blocked, err)
|
||||
}
|
||||
var retrievalState string
|
||||
var removalJobs int
|
||||
@@ -118,39 +123,36 @@ VALUES('tp-existing','version-route','kb-internal','internal','uploaded','comple
|
||||
if retrievalState != "enabled" || removalJobs != 0 {
|
||||
t.Fatalf("blocked annotation must not remove: state=%s removal jobs=%d", retrievalState, removalJobs)
|
||||
}
|
||||
// The planned kb-external row created while policy was 'both' is retired.
|
||||
var stillPlanned int
|
||||
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM test_publications WHERE processing_state='planned' AND superseded_at IS NULL AND test_kb_id='kb-external'`).Scan(&stillPlanned); err != nil || stillPlanned != 0 {
|
||||
t.Fatalf("blocked must retire planned rows=%d err=%v", stillPlanned, err)
|
||||
}
|
||||
if err := pool.QueryRow(ctx, `SELECT visibility FROM directory_scope_rules WHERE source_connection_id='conn-route' AND directory_path='工业产品'`).Scan(&directoryPolicy); err != nil || directoryPolicy != "blocked" {
|
||||
t.Fatalf("blocked policy=%q err=%v", directoryPolicy, err)
|
||||
}
|
||||
|
||||
// Unbound file and hash-unknown file are reported, never planned.
|
||||
unbound, err := store.RouteCatalogFiles(ctx, governance.CatalogRouteRequest{ConnectionID: "conn-route", NodeIDs: []string{"file-unmapped"}, Visibility: governance.VisibilityExternal, TargetKnowledgeBaseID: "kb-external", OperatorRef: "engineer-1", Reason: "测试未映射", Now: now})
|
||||
if err != nil || unbound.SelectedFiles != 1 || unbound.RoutableFiles != 0 || unbound.UnboundFiles != 1 {
|
||||
t.Fatalf("unbound=%#v err=%v", unbound, err)
|
||||
// A never-fetched file becomes an intent with a materialized item, a
|
||||
// fetch job and a planned publication (未绑定 → 从源头拉,不再是死路).
|
||||
restored, err := store.RouteCatalogFiles(ctx, governance.CatalogRouteRequest{ConnectionID: "conn-route", NodeIDs: []string{"file-unmapped"}, Visibility: governance.VisibilityInternal, TargetKnowledgeBaseID: "kb-internal", OperatorRef: "engineer-1", Reason: "重新标注", Now: now.Add(4 * time.Minute)})
|
||||
if err != nil || restored.IntentsRecorded != 1 || restored.FetchQueued != 1 || restored.PlannedPublications != 1 {
|
||||
t.Fatalf("restored=%#v err=%v", restored, err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO content_objects(id,sha256,size_bytes,local_spool_ref,content_state) VALUES('content-pending',repeat('e',64),8,'','unknown')`); err != nil {
|
||||
var fetchState, itemPolicy string
|
||||
if err := pool.QueryRow(ctx, `SELECT intent.fetch_state||'|'||item.policy_state FROM catalog_import_intents intent
|
||||
JOIN source_items item ON item.id=intent.source_item_id
|
||||
WHERE intent.node_id='file-unmapped'`).Scan(&fetchState, &itemPolicy); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO source_items(id,source_connection_id,stable_external_id,canonical_uri_hash,normalized_path,title,item_type,revision,visibility,visibility_origin)
|
||||
VALUES('item-pending','conn-route','external-pending',repeat('f',64),'待校验.pdf','待校验.pdf','document','v1','external','pp_public_tag')`); err != nil {
|
||||
t.Fatal(err)
|
||||
if fetchState != "pending|internal" {
|
||||
t.Fatalf("fetch state=%q", fetchState)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO source_item_versions(id,source_item_id,revision,content_object_id,content_state) VALUES('version-pending','item-pending','v1','content-pending','unknown')`); err != nil {
|
||||
t.Fatal(err)
|
||||
var versionState string
|
||||
if err := pool.QueryRow(ctx, `SELECT version.content_state FROM source_item_versions version
|
||||
JOIN source_items item ON item.id=version.source_item_id AND item.revision=version.revision
|
||||
WHERE item.stable_external_id='file-unmapped'`).Scan(&versionState); err != nil || versionState != "unknown" {
|
||||
t.Fatalf("pending version state=%q err=%v", versionState, err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO source_catalog_nodes(snapshot_id,node_id,parent_node_id,node_type,name,relative_path,source_locator_hash,source_item_id,size_bytes,direct_child_count,subtree_file_count,subtree_bytes)
|
||||
VALUES('snapshot-route','file-pending','root-route','file','待校验.pdf','待校验.pdf',repeat('d',64),'item-pending',8,0,1,8)`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hashUnknown, err := store.RouteCatalogFiles(ctx, governance.CatalogRouteRequest{ConnectionID: "conn-route", NodeIDs: []string{"file-pending"}, Visibility: governance.VisibilityInternal, TargetKnowledgeBaseID: "kb-internal", OperatorRef: "engineer-1", Reason: "测试内容未验证", Now: now})
|
||||
if err != nil || hashUnknown.SelectedFiles != 1 || hashUnknown.RoutableFiles != 0 || hashUnknown.HashUnknownFiles != 1 {
|
||||
t.Fatalf("hash unknown=%#v err=%v", hashUnknown, err)
|
||||
}
|
||||
// Overrides are still recorded for unbound/pending items so the desired state survives.
|
||||
var overrideCount int
|
||||
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM source_item_scope_overrides WHERE source_item_id='item-pending'`).Scan(&overrideCount); err != nil || overrideCount != 1 {
|
||||
t.Fatalf("pending override rows=%d err=%v", overrideCount, err)
|
||||
}
|
||||
// Audit trail exists for every annotation action.
|
||||
var events int
|
||||
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM scope_change_events`).Scan(&events); err != nil || events < 5 {
|
||||
t.Fatalf("scope events=%d err=%v", events, err)
|
||||
|
||||
@@ -46,7 +46,7 @@ INSERT INTO source_catalog_heads(source_connection_id,snapshot_id,activated_at)
|
||||
now := time.Date(2026, 9, 11, 6, 0, 0, 0, time.UTC)
|
||||
request := governance.CatalogRouteRequest{ConnectionID: "conn-op", NodeIDs: []string{"file-op"}, Visibility: governance.VisibilityInternal, TargetKnowledgeBaseID: "kb-internal", OperatorRef: "engineer-1", Reason: "初次标注", Now: now}
|
||||
first, replayed, err := store.RouteCatalogFilesWithOperation(ctx, request, "op-route-1")
|
||||
if err != nil || replayed || first.RoutableFiles != 1 {
|
||||
if err != nil || replayed || first.PlannedPublications != 1 {
|
||||
t.Fatalf("first result=%#v replayed=%v error=%v", first, replayed, err)
|
||||
}
|
||||
second, replayed, err := store.RouteCatalogFilesWithOperation(ctx, request, "op-route-1")
|
||||
@@ -60,8 +60,8 @@ INSERT INTO source_catalog_heads(source_connection_id,snapshot_id,activated_at)
|
||||
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM operation_requests WHERE operation_type='catalog.route' AND operation_id='op-route-1'`).Scan(&operations); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if publications != 0 || operations != 1 {
|
||||
t.Fatalf("S7 annotation must not publish: publications=%d operations=%d", publications, operations)
|
||||
if publications != 1 || operations != 1 {
|
||||
t.Fatalf("annotation must plan exactly one replay-safe publication: publications=%d operations=%d", publications, operations)
|
||||
}
|
||||
conflicting := request
|
||||
conflicting.Reason = "不同的操作内容"
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
ALTER TABLE ui_setting_revisions DROP COLUMN IF EXISTS ingestion_concurrency;
|
||||
DROP INDEX IF EXISTS idx_test_publications_planned;
|
||||
ALTER TABLE test_publications DROP CONSTRAINT IF EXISTS test_publications_processing_state_check;
|
||||
ALTER TABLE test_publications ADD CONSTRAINT test_publications_processing_state_check
|
||||
CHECK (processing_state IN ('pending', 'processing', 'finalizing', 'completed', 'failed', 'stalled'));
|
||||
DROP INDEX IF EXISTS idx_source_items_policy;
|
||||
ALTER TABLE source_items DROP COLUMN IF EXISTS policy_state;
|
||||
DROP INDEX IF EXISTS idx_catalog_intents_fetch;
|
||||
DROP TABLE IF EXISTS catalog_import_intents;
|
||||
@@ -0,0 +1,62 @@
|
||||
-- Policy-driven ingestion: the ONLY import decision surface is the catalog
|
||||
-- annotation (internal / external / both / blocked). Marking a file or folder
|
||||
-- records a durable per-file INTENT that survives rescans; unannotated files
|
||||
-- stay "unset" and are never ingested (cold start imports nothing).
|
||||
-- A saved intent immediately materializes a versioned source item and an
|
||||
-- artifact.fetch job so the artifact is pulled from the live source (or
|
||||
-- matched against an already-present local copy); once content is verified a
|
||||
-- 'planned' publication appears in the knowledge-base diff as 待入库. The one
|
||||
-- human batch action "开始处理" promotes planned -> queued within the
|
||||
-- concurrency budget configured in system settings.
|
||||
|
||||
-- Per-file decision, keyed by the connection-stable catalog node id
|
||||
-- (node ids are derived from source_key + relative path, identical across
|
||||
-- snapshots, so intents are rescan-proof).
|
||||
CREATE TABLE IF NOT EXISTS catalog_import_intents (
|
||||
source_connection_id VARCHAR(36) NOT NULL
|
||||
REFERENCES source_connections(id) ON DELETE CASCADE,
|
||||
node_id VARCHAR(64) NOT NULL,
|
||||
visibility VARCHAR(20) NOT NULL
|
||||
CHECK (visibility IN ('internal', 'external', 'both', 'blocked')),
|
||||
source_item_id VARCHAR(36),
|
||||
decided_by VARCHAR(128) NOT NULL,
|
||||
reason VARCHAR(500) NOT NULL DEFAULT '',
|
||||
-- Locator snapshot: the fetch worker needs path/name metadata without
|
||||
-- re-walking a snapshot that may already have rotated.
|
||||
relative_path TEXT NOT NULL DEFAULT '',
|
||||
name TEXT NOT NULL DEFAULT '',
|
||||
size_bytes BIGINT,
|
||||
modified_at TIMESTAMPTZ,
|
||||
content_sha256 CHAR(64),
|
||||
locator_hash CHAR(64) NOT NULL DEFAULT '',
|
||||
-- Content materialization lifecycle driven by the artifact.fetch job.
|
||||
fetch_state VARCHAR(20) NOT NULL DEFAULT 'pending'
|
||||
CHECK (fetch_state IN ('pending', 'fetching', 'ready', 'missing_local', 'failed')),
|
||||
last_error TEXT NOT NULL DEFAULT '',
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
PRIMARY KEY (source_connection_id, node_id)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_catalog_intents_item
|
||||
ON catalog_import_intents (source_item_id) WHERE source_item_id IS NOT NULL;
|
||||
CREATE INDEX IF NOT EXISTS idx_catalog_intents_fetch
|
||||
ON catalog_import_intents (fetch_state) WHERE fetch_state <> 'ready';
|
||||
|
||||
-- Materialized items carry the policy so list/diff queries never re-walk rules.
|
||||
ALTER TABLE source_items ADD COLUMN IF NOT EXISTS policy_state VARCHAR(20) NOT NULL DEFAULT 'unset'
|
||||
CHECK (policy_state IN ('unset', 'internal', 'external', 'both', 'blocked'));
|
||||
CREATE INDEX IF NOT EXISTS idx_source_items_policy
|
||||
ON source_items (policy_state) WHERE policy_state <> 'unset';
|
||||
|
||||
-- The knowledge base is the ingestion target of a planned publication; make
|
||||
-- "planned" explicit so the KB-diff view never reuses queued semantics.
|
||||
ALTER TABLE test_publications DROP CONSTRAINT IF EXISTS test_publications_processing_state_check;
|
||||
ALTER TABLE test_publications ADD CONSTRAINT test_publications_processing_state_check
|
||||
CHECK (processing_state IN ('planned', 'pending', 'processing', 'finalizing', 'completed', 'failed', 'stalled'));
|
||||
CREATE INDEX IF NOT EXISTS idx_test_publications_planned
|
||||
ON test_publications (test_kb_id, processing_state) WHERE processing_state = 'planned';
|
||||
|
||||
-- 开始处理's concurrency budget lives in system settings (validated revisions).
|
||||
ALTER TABLE ui_setting_revisions ADD COLUMN IF NOT EXISTS ingestion_concurrency INTEGER NOT NULL DEFAULT 2
|
||||
CHECK (ingestion_concurrency BETWEEN 1 AND 16);
|
||||
@@ -11,8 +11,8 @@ func TestEmbeddedMigrationsAreVersionSorted(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("loadMigrations() error = %v", err)
|
||||
}
|
||||
if len(migrations) != 32 {
|
||||
t.Fatalf("migration count = %d, want 32", len(migrations))
|
||||
if len(migrations) != 33 {
|
||||
t.Fatalf("migration count = %d, want 33", len(migrations))
|
||||
}
|
||||
if migrations[0].Version != 1 || !strings.Contains(migrations[0].UpSQL, "CREATE TABLE IF NOT EXISTS jobs") {
|
||||
t.Fatalf("migration 1 = %#v", migrations[0])
|
||||
@@ -567,7 +567,8 @@ func TestReleaseExecutionMigrationDefinesAuthorizedAuditableJobs(t *testing.T) {
|
||||
func TestClaimSQLUsesSkipLockedAndReclaimsExpiredRunningJobs(t *testing.T) {
|
||||
sql := strings.ToLower(claimJobSQL)
|
||||
for _, required := range []string{
|
||||
"for update skip locked",
|
||||
"for update of jobs skip locked",
|
||||
"ingestion_concurrency",
|
||||
"state in ('pending', 'retry')",
|
||||
"state = 'running'",
|
||||
"lease_until <",
|
||||
|
||||
@@ -0,0 +1,364 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
)
|
||||
|
||||
// fulfillCatalogIntentsTx is the policy-driven ingestion core: saving an
|
||||
// annotation in 来源与目录 records one durable intent per selected file and,
|
||||
// for importable policies, materializes a versioned source item, enqueues an
|
||||
// artifact.fetch job (bytes come from the live source; a matching local copy
|
||||
// short-circuits the download) and creates one 'planned' publication per routed
|
||||
// knowledge base. Planned publications are what 解析监控 lists as 待入库;
|
||||
// 开始处理 promotes them within the configured concurrency budget. Blocked
|
||||
// intents retire any planned publication and flip the item policy so the
|
||||
// knowledge-base diff demands removal of anything still live.
|
||||
func fulfillCatalogIntentsTx(ctx context.Context, tx pgx.Tx, request governance.CatalogRouteRequest, files []governance.CatalogRouteFile, now time.Time) (governance.CatalogRouteResult, error) {
|
||||
targetKnowledgeBaseIDs := request.TargetKnowledgeBaseIDs
|
||||
if len(targetKnowledgeBaseIDs) == 0 && request.TargetKnowledgeBaseID != "" {
|
||||
targetKnowledgeBaseIDs = []string{request.TargetKnowledgeBaseID}
|
||||
}
|
||||
result := governance.CatalogRouteResult{SelectedFiles: len(files)}
|
||||
for _, file := range files {
|
||||
recorded, fetchQueued, planned, blocked, err := fulfillCatalogIntent(ctx, tx, request, file, targetKnowledgeBaseIDs, now)
|
||||
if err != nil {
|
||||
return governance.CatalogRouteResult{}, err
|
||||
}
|
||||
if !recorded {
|
||||
result.UnboundFiles++
|
||||
continue
|
||||
}
|
||||
result.IntentsRecorded++
|
||||
if blocked {
|
||||
result.BlockedItems++
|
||||
continue
|
||||
}
|
||||
if fetchQueued {
|
||||
result.FetchQueued++
|
||||
}
|
||||
result.PlannedPublications += planned
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// fulfillCatalogIntent applies one file decision. It returns recorded=false
|
||||
// only when the node vanished mid-transaction; fetchQueued reports a newly
|
||||
// enqueued artifact.fetch job and planned the number of publications created.
|
||||
func fulfillCatalogIntent(ctx context.Context, tx pgx.Tx, request governance.CatalogRouteRequest, file governance.CatalogRouteFile, targetKnowledgeBaseIDs []string, now time.Time) (bool, bool, int, bool, error) {
|
||||
policy := string(request.Visibility)
|
||||
if _, err := tx.Exec(ctx, `INSERT INTO catalog_import_intents(
|
||||
source_connection_id,node_id,visibility,decided_by,reason,relative_path,name,size_bytes,modified_at,content_sha256,
|
||||
locator_hash,fetch_state,last_error,created_at,updated_at)
|
||||
VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,'pending','',NOW(),NOW())
|
||||
ON CONFLICT(source_connection_id,node_id) DO UPDATE SET
|
||||
visibility=EXCLUDED.visibility,decided_by=EXCLUDED.decided_by,reason=EXCLUDED.reason,
|
||||
relative_path=EXCLUDED.relative_path,name=EXCLUDED.name,size_bytes=EXCLUDED.size_bytes,
|
||||
modified_at=EXCLUDED.modified_at,content_sha256=EXCLUDED.content_sha256,
|
||||
locator_hash=EXCLUDED.locator_hash,
|
||||
fetch_state=CASE WHEN catalog_import_intents.content_sha256 IS DISTINCT FROM EXCLUDED.content_sha256
|
||||
OR catalog_import_intents.size_bytes IS DISTINCT FROM EXCLUDED.size_bytes
|
||||
OR catalog_import_intents.modified_at IS DISTINCT FROM EXCLUDED.modified_at
|
||||
THEN 'pending' ELSE catalog_import_intents.fetch_state END,
|
||||
last_error=CASE WHEN catalog_import_intents.content_sha256 IS DISTINCT FROM EXCLUDED.content_sha256
|
||||
OR catalog_import_intents.size_bytes IS DISTINCT FROM EXCLUDED.size_bytes
|
||||
OR catalog_import_intents.modified_at IS DISTINCT FROM EXCLUDED.modified_at
|
||||
THEN '' ELSE catalog_import_intents.last_error END,
|
||||
updated_at=NOW()`,
|
||||
request.ConnectionID, file.NodeID, policy, request.OperatorRef, request.Reason,
|
||||
file.RelativePath, file.Name, file.SizeBytes, file.ModifiedAt, file.ContentSHA256, file.LocatorHash); err != nil {
|
||||
return false, false, 0, false, fmt.Errorf("record catalog intent: %w", err)
|
||||
}
|
||||
|
||||
var existingItemID *string
|
||||
err := tx.QueryRow(ctx, `SELECT source_item_id FROM catalog_import_intents
|
||||
WHERE source_connection_id=$1 AND node_id=$2 FOR UPDATE`, request.ConnectionID, file.NodeID).Scan(&existingItemID)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return false, false, 0, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, false, 0, false, fmt.Errorf("read catalog intent: %w", err)
|
||||
}
|
||||
|
||||
itemID := ""
|
||||
if existingItemID != nil {
|
||||
itemID = *existingItemID
|
||||
}
|
||||
if itemID == "" {
|
||||
newItemID, err := newOpaqueID("si-")
|
||||
if err != nil {
|
||||
return false, false, 0, false, err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `INSERT INTO source_items(
|
||||
id,source_connection_id,stable_external_id,canonical_uri_hash,title,item_type,revision,
|
||||
visibility,visibility_origin,source_change,absence_streak,identity_confidence,
|
||||
policy_state,created_at,updated_at)
|
||||
VALUES($1,$2,$3,$4,$5,'file','',$7,'configured_scope','unchanged',0,'object_id',$6,NOW(),NOW())`,
|
||||
newItemID, request.ConnectionID, "cat:"+file.NodeID,
|
||||
sha256Hex("policy:"+request.ConnectionID+"\x00"+file.NodeID),
|
||||
file.Name, policy, itemVisibilityFor(request.Visibility)); err != nil {
|
||||
return false, false, 0, false, fmt.Errorf("materialize policy source item: %w", err)
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `UPDATE catalog_import_intents SET source_item_id=$3 WHERE source_connection_id=$1 AND node_id=$2`,
|
||||
request.ConnectionID, file.NodeID, newItemID); err != nil {
|
||||
return false, false, 0, false, fmt.Errorf("bind catalog intent item: %w", err)
|
||||
}
|
||||
itemID = newItemID
|
||||
}
|
||||
|
||||
if request.Visibility == governance.VisibilityBlocked {
|
||||
if _, err := tx.Exec(ctx, `UPDATE source_items SET policy_state='blocked',updated_at=NOW() WHERE id=$1`, itemID); err != nil {
|
||||
return false, false, 0, false, fmt.Errorf("block policy item: %w", err)
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `UPDATE test_publications SET superseded_at=$2,updated_at=$2
|
||||
WHERE source_item_version_id IN (SELECT id FROM source_item_versions WHERE source_item_id=$1)
|
||||
AND superseded_at IS NULL AND processing_state='planned'`, itemID, now); err != nil {
|
||||
return false, false, 0, false, fmt.Errorf("retire blocked planned publications: %w", err)
|
||||
}
|
||||
return true, false, 0, true, nil
|
||||
}
|
||||
|
||||
revision := "locator:" + file.LocatorHash
|
||||
if file.ContentSHA256 != nil && *file.ContentSHA256 != "" {
|
||||
revision = "sha:" + *file.ContentSHA256
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `UPDATE source_items
|
||||
SET policy_state=$2,visibility=$3,visibility_origin='configured_scope',updated_at=NOW() WHERE id=$1`,
|
||||
itemID, policy, itemVisibilityFor(request.Visibility)); err != nil {
|
||||
return false, false, 0, false, fmt.Errorf("update policy item: %w", err)
|
||||
}
|
||||
|
||||
// Resolve the version this intent points at. A bound item whose current
|
||||
// version already carries verified bytes is reused without any fetch — the
|
||||
// 本地已有先比对 requirement. Only when the node proves different content
|
||||
// (scan-reported hash mismatch) or bytes were never fetched do we create a
|
||||
// revision and enqueue artifact.fetch.
|
||||
versionID := ""
|
||||
contentState := ""
|
||||
var curID, curSHA string
|
||||
err = tx.QueryRow(ctx, `SELECT version.id,COALESCE(content.sha256,''),version.content_state
|
||||
FROM source_items item
|
||||
JOIN source_item_versions version ON version.source_item_id=item.id AND version.revision=item.revision
|
||||
LEFT JOIN content_objects content ON content.id=version.content_object_id
|
||||
WHERE item.id=$1`, itemID).Scan(&curID, &curSHA, &contentState)
|
||||
if err != nil && !errors.Is(err, pgx.ErrNoRows) {
|
||||
return false, false, 0, false, fmt.Errorf("inspect current item version: %w", err)
|
||||
}
|
||||
nodeSHA := ""
|
||||
if file.ContentSHA256 != nil {
|
||||
nodeSHA = *file.ContentSHA256
|
||||
}
|
||||
reuse := err == nil && contentState == string(governance.ContentVerified) && (nodeSHA == "" || nodeSHA == curSHA)
|
||||
if !reuse {
|
||||
contentState = "unknown"
|
||||
err = tx.QueryRow(ctx, `SELECT version.id,version.content_state
|
||||
FROM source_item_versions version WHERE version.source_item_id=$1 AND version.revision=$2`,
|
||||
itemID, revision).Scan(&versionID, &contentState)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
newVersionID, err := newOpaqueID("sv-")
|
||||
if err != nil {
|
||||
return false, false, 0, false, err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `INSERT INTO source_item_versions(
|
||||
id,source_item_id,revision,content_object_id,content_state,raw_record_summary,source_updated_at,created_at,updated_at)
|
||||
VALUES($1,$2,$3,NULL,'unknown','{}'::jsonb,$4,NOW(),NOW())`, newVersionID, itemID, revision, file.ModifiedAt); err != nil {
|
||||
return false, false, 0, false, fmt.Errorf("create policy version: %w", err)
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `UPDATE source_items SET revision=$2,updated_at=NOW() WHERE id=$1`, itemID, revision); err != nil {
|
||||
return false, false, 0, false, fmt.Errorf("advance policy item revision: %w", err)
|
||||
}
|
||||
versionID = newVersionID
|
||||
} else if err != nil {
|
||||
return false, false, 0, false, fmt.Errorf("lookup policy version: %w", err)
|
||||
}
|
||||
} else {
|
||||
versionID = curID
|
||||
}
|
||||
|
||||
planned := 0
|
||||
for _, kbID := range targetKnowledgeBaseIDs {
|
||||
scope := kbScopeFor(request, kbID)
|
||||
publicationID, err := newOpaqueID("tp-")
|
||||
if err != nil {
|
||||
return false, false, 0, false, err
|
||||
}
|
||||
tag, err := tx.Exec(ctx, `INSERT INTO test_publications(
|
||||
id,source_item_version_id,test_kb_id,knowledge_id,ingest_state,processing_state,retrieval_state,
|
||||
parser_generation,index_generation,last_error,retry_count,target_scope,created_at,updated_at)
|
||||
VALUES($1,$2,$3,'','pending','planned','unpublished','','','',0,$4,NOW(),NOW())
|
||||
ON CONFLICT (source_item_version_id,test_kb_id) DO NOTHING`,
|
||||
publicationID, versionID, kbID, scope)
|
||||
if err != nil {
|
||||
return false, false, 0, false, fmt.Errorf("plan publication: %w", err)
|
||||
}
|
||||
planned += int(tag.RowsAffected())
|
||||
}
|
||||
|
||||
fetchQueued := false
|
||||
if contentState != string(governance.ContentVerified) {
|
||||
jobID, err := newOpaqueID("job-")
|
||||
if err != nil {
|
||||
return false, false, 0, false, err
|
||||
}
|
||||
payload, err := json.Marshal(map[string]string{"source_connection_id": request.ConnectionID, "node_id": file.NodeID, "version_id": versionID})
|
||||
if err != nil {
|
||||
return false, false, 0, false, errors.New("encode artifact fetch job")
|
||||
}
|
||||
tag, err := tx.Exec(ctx, `INSERT INTO jobs(id,type,payload,state,priority,available_at,max_attempts,idempotency_key,created_at,updated_at)
|
||||
VALUES($1,'artifact.fetch',$2::jsonb,'pending',20,$3,8,$4,$3,$3) ON CONFLICT (idempotency_key) DO NOTHING`,
|
||||
jobID, payload, now, "artifact-fetch:"+versionID)
|
||||
if err != nil {
|
||||
return false, false, 0, false, fmt.Errorf("enqueue artifact fetch: %w", err)
|
||||
}
|
||||
fetchQueued = tag.RowsAffected() > 0
|
||||
if fetchQueued {
|
||||
if _, err := tx.Exec(ctx, `UPDATE catalog_import_intents SET fetch_state='pending',last_error='',updated_at=NOW()
|
||||
WHERE source_connection_id=$1 AND node_id=$2 AND fetch_state IN ('failed','missing_local','fetching')`, request.ConnectionID, file.NodeID); err != nil {
|
||||
return false, false, 0, false, fmt.Errorf("reset catalog intent fetch state: %w", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return true, fetchQueued, planned, false, nil
|
||||
}
|
||||
|
||||
// kbScopeFor maps a routed knowledge base id to its diff scope label.
|
||||
func kbScopeFor(request governance.CatalogRouteRequest, knowledgeBaseID string) string {
|
||||
if request.Visibility == governance.VisibilityExternal {
|
||||
return "external"
|
||||
}
|
||||
if request.Visibility == governance.VisibilityBoth && len(request.TargetKnowledgeBaseIDs) > 1 &&
|
||||
knowledgeBaseID == request.TargetKnowledgeBaseIDs[len(request.TargetKnowledgeBaseIDs)-1] {
|
||||
return "external"
|
||||
}
|
||||
return "internal"
|
||||
}
|
||||
|
||||
// itemVisibilityFor mirrors source_items' two-value visibility CHECK: both and
|
||||
// blocked decisions are governed through policy_state; the legacy visibility
|
||||
// column keeps a valid value and is widened to external for both.
|
||||
func itemVisibilityFor(visibility governance.Visibility) string {
|
||||
if visibility == governance.VisibilityInternal {
|
||||
return string(governance.VisibilityInternal)
|
||||
}
|
||||
return string(governance.VisibilityExternal)
|
||||
}
|
||||
|
||||
func sha256Hex(value string) string {
|
||||
sum := sha256.Sum256([]byte(value))
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// MarkArtifactFetched finalizes one artifact.fetch success: it (re)uses the
|
||||
// content_objects row for the verified hash (content-level dedupe), attaches it
|
||||
// to the pending version and flips the intent to ready. matchedLocal records
|
||||
// that the bytes already existed locally and no source download happened.
|
||||
func (s *Store) MarkArtifactFetched(ctx context.Context, versionID, sha256Value string, sizeBytes int64, spoolRef, validatedFormat string, matchedLocal bool, now time.Time) error {
|
||||
if len(sha256Value) != 64 || spoolRef == "" || sizeBytes < 0 || validatedFormat == "" {
|
||||
return errors.New("invalid artifact fetch result")
|
||||
}
|
||||
var tx pgx.Tx
|
||||
ownsTransaction := false
|
||||
if existing, ok := s.db.(pgx.Tx); ok {
|
||||
tx = existing
|
||||
} else {
|
||||
beginner, ok := s.db.(interface {
|
||||
Begin(context.Context) (pgx.Tx, error)
|
||||
})
|
||||
if !ok {
|
||||
return errors.New("artifact fetch transactions are unavailable")
|
||||
}
|
||||
var err error
|
||||
tx, err = beginner.Begin(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("begin artifact fetch: %w", err)
|
||||
}
|
||||
ownsTransaction = true
|
||||
defer tx.Rollback(ctx)
|
||||
}
|
||||
var connectionID, nodeID string
|
||||
err := tx.QueryRow(ctx, `SELECT intent.source_connection_id,intent.node_id
|
||||
FROM catalog_import_intents intent
|
||||
JOIN source_item_versions version ON version.source_item_id=intent.source_item_id
|
||||
WHERE version.id=$1 FOR UPDATE OF intent`, versionID).Scan(&connectionID, &nodeID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("locate intent for fetched artifact: %w", err)
|
||||
}
|
||||
freshID, err := newOpaqueID("co-")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
contentID := ""
|
||||
err = tx.QueryRow(ctx, `INSERT INTO content_objects(id,sha256,size_bytes,mime,validated_format,local_spool_ref,content_state,created_at,updated_at)
|
||||
VALUES($1,$2,$3,'',$5,$4,'verified',NOW(),NOW())
|
||||
ON CONFLICT(sha256) DO UPDATE SET
|
||||
size_bytes=EXCLUDED.size_bytes,
|
||||
validated_format=CASE WHEN content_objects.validated_format='' THEN EXCLUDED.validated_format ELSE content_objects.validated_format END,
|
||||
local_spool_ref=CASE WHEN content_objects.local_spool_ref='' THEN EXCLUDED.local_spool_ref ELSE content_objects.local_spool_ref END,
|
||||
content_state='verified',updated_at=NOW()
|
||||
RETURNING id`,
|
||||
freshID, sha256Value, sizeBytes, spoolRef, validatedFormat).Scan(&contentID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("record fetched content: %w", err)
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `UPDATE source_item_versions SET
|
||||
content_object_id=$2,content_state='verified',updated_at=$4,
|
||||
raw_record_summary=jsonb_set(raw_record_summary,'{matched_local}',$3::jsonb)
|
||||
WHERE id=$1`,
|
||||
versionID, contentID, mustJSONBool(matchedLocal), now.UTC()); err != nil {
|
||||
return fmt.Errorf("bind fetched content: %w", err)
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `UPDATE catalog_import_intents SET fetch_state='ready',last_error='',updated_at=NOW()
|
||||
WHERE source_connection_id=$1 AND node_id=$2`, connectionID, nodeID); err != nil {
|
||||
return fmt.Errorf("mark intent fetched: %w", err)
|
||||
}
|
||||
if ownsTransaction {
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return fmt.Errorf("commit artifact fetch: %w", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func mustJSONBool(value bool) string {
|
||||
if value {
|
||||
return "true"
|
||||
}
|
||||
return "false"
|
||||
}
|
||||
|
||||
// MarkArtifactFetchMissing parks an intent whose source-side file disappeared
|
||||
// (a 404 on download, not a transient outage) so the UI can ask for a rescan.
|
||||
func (s *Store) MarkArtifactFetchMissing(ctx context.Context, connectionID, nodeID string, now time.Time) error {
|
||||
if _, err := s.db.Exec(ctx, `UPDATE catalog_import_intents SET fetch_state='missing_local',updated_at=$3
|
||||
WHERE source_connection_id=$1 AND node_id=$2 AND fetch_state<>'ready'`, connectionID, nodeID, now.UTC()); err != nil {
|
||||
return fmt.Errorf("record artifact fetch miss: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// MarkArtifactFetchFailed records a fetch failure for the intent so 解析监控
|
||||
// can show why an item is not yet 待入库.
|
||||
func (s *Store) MarkArtifactFetchFailed(ctx context.Context, connectionID, nodeID, message string, now time.Time) error {
|
||||
if _, err := s.db.Exec(ctx, `UPDATE catalog_import_intents SET fetch_state='failed',last_error=$3,updated_at=$4
|
||||
WHERE source_connection_id=$1 AND node_id=$2 AND fetch_state<>'ready'`, connectionID, nodeID, truncateMonitorText(message), now.UTC()); err != nil {
|
||||
return fmt.Errorf("record artifact fetch failure: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func truncateMonitorText(value string) string {
|
||||
runes := []rune(value)
|
||||
if len(runes) <= 500 {
|
||||
return value
|
||||
}
|
||||
return string(runes[:500])
|
||||
}
|
||||
@@ -16,9 +16,12 @@ func (s *Store) StartTestPublications(ctx context.Context, request governance.St
|
||||
result := governance.StartTestPublicationsResult{Selected: len(request.PublicationIDs)}
|
||||
operator := strings.TrimSpace(request.OperatorRef)
|
||||
reason := strings.TrimSpace(request.Reason)
|
||||
if len(request.PublicationIDs) == 0 || len(request.PublicationIDs) > 100 || operator == "" || len(operator) > 128 || reason == "" || len(reason) > 500 || request.Now.IsZero() {
|
||||
if len(request.PublicationIDs) == 0 || len(request.PublicationIDs) > 100 || operator == "" || len(operator) > 128 || len(reason) > 500 || request.Now.IsZero() {
|
||||
return governance.StartTestPublicationsResult{}, errors.New("invalid processing confirmation")
|
||||
}
|
||||
if reason == "" {
|
||||
reason = "开始处理"
|
||||
}
|
||||
seen := make(map[string]bool, len(request.PublicationIDs))
|
||||
for _, publicationID := range request.PublicationIDs {
|
||||
if !validRouteID(publicationID) || seen[publicationID] {
|
||||
@@ -50,22 +53,23 @@ func (s *Store) StartTestPublications(ctx context.Context, request governance.St
|
||||
|
||||
for _, publicationID := range request.PublicationIDs {
|
||||
var ready, confirmed, uploadJobExists bool
|
||||
var processingState string
|
||||
err = tx.QueryRow(ctx, `
|
||||
SELECT p.ingest_state='pending'
|
||||
AND p.processing_state='pending'
|
||||
AND p.retrieval_state='unpublished'
|
||||
AND p.knowledge_id=''
|
||||
AND version.content_state='verified'
|
||||
AND content.content_state='verified'
|
||||
AND content.local_spool_ref<>''
|
||||
AND content.size_bytes>0 AS ready,
|
||||
p.processing_state,
|
||||
EXISTS(SELECT 1 FROM processing_confirmations confirmation WHERE confirmation.publication_id=p.id),
|
||||
EXISTS(SELECT 1 FROM jobs job WHERE job.type='test.upload' AND job.payload->>'publication_id'=p.id)
|
||||
FROM test_publications p
|
||||
JOIN source_item_versions version ON version.id=p.source_item_version_id
|
||||
JOIN content_objects content ON content.id=version.content_object_id
|
||||
WHERE p.id=$1
|
||||
FOR UPDATE OF p`, publicationID).Scan(&ready, &confirmed, &uploadJobExists)
|
||||
WHERE p.id=$1 AND p.superseded_at IS NULL
|
||||
FOR UPDATE OF p`, publicationID).Scan(&ready, &processingState, &confirmed, &uploadJobExists)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
result.Unavailable++
|
||||
continue
|
||||
@@ -77,10 +81,18 @@ FOR UPDATE OF p`, publicationID).Scan(&ready, &confirmed, &uploadJobExists)
|
||||
result.AlreadyStarted++
|
||||
continue
|
||||
}
|
||||
// 开始处理 promotes both fresh plans and verified pending rows.
|
||||
if processingState != "planned" && processingState != "pending" {
|
||||
result.Unavailable++
|
||||
continue
|
||||
}
|
||||
if !ready {
|
||||
result.Unavailable++
|
||||
continue
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `UPDATE test_publications SET processing_state='pending',updated_at=$2 WHERE id=$1 AND processing_state='planned'`, publicationID, request.Now); err != nil {
|
||||
return governance.StartTestPublicationsResult{}, fmt.Errorf("promote planned publication: %w", err)
|
||||
}
|
||||
confirmationID, err := newOpaqueID("pc-")
|
||||
if err != nil {
|
||||
return governance.StartTestPublicationsResult{}, err
|
||||
|
||||
@@ -13,17 +13,29 @@ RETURNING id, type, payload, state, priority, available_at,
|
||||
`
|
||||
|
||||
const claimJobSQL = `
|
||||
WITH candidate AS (
|
||||
SELECT id
|
||||
FROM jobs
|
||||
WITH budget AS (
|
||||
SELECT COALESCE((
|
||||
SELECT ingestion_concurrency
|
||||
FROM ui_setting_revisions
|
||||
WHERE state = 'active'
|
||||
LIMIT 1
|
||||
), 2) AS ingestion_slots
|
||||
), candidate AS (
|
||||
SELECT jobs.id
|
||||
FROM jobs, budget
|
||||
WHERE (
|
||||
(state IN ('pending', 'retry') AND available_at <= $2)
|
||||
OR (state = 'running' AND lease_until < $2)
|
||||
(jobs.state IN ('pending', 'retry') AND jobs.available_at <= $2)
|
||||
OR (jobs.state = 'running' AND jobs.lease_until < $2)
|
||||
)
|
||||
AND attempt < max_attempts
|
||||
AND type = ANY($4::text[])
|
||||
ORDER BY priority DESC, available_at ASC, created_at ASC, id ASC
|
||||
FOR UPDATE SKIP LOCKED
|
||||
AND jobs.attempt < jobs.max_attempts
|
||||
AND jobs.type = ANY($4::text[])
|
||||
AND (
|
||||
jobs.type <> 'test.upload'
|
||||
OR (SELECT COUNT(*) FROM jobs active
|
||||
WHERE active.type = 'test.upload' AND active.state = 'running') < budget.ingestion_slots
|
||||
)
|
||||
ORDER BY jobs.priority DESC, jobs.available_at ASC, jobs.created_at ASC, jobs.id ASC
|
||||
FOR UPDATE OF jobs SKIP LOCKED
|
||||
LIMIT 1
|
||||
)
|
||||
UPDATE jobs AS j
|
||||
|
||||
@@ -51,7 +51,7 @@ INSERT INTO source_catalog_heads(source_connection_id,snapshot_id,activated_at)
|
||||
ConnectionID: "conn-s6", NodeIDs: []string{"folder-top"}, Visibility: governance.VisibilityInternal,
|
||||
TargetKnowledgeBaseID: "kb-internal", OperatorRef: "engineer-1", Reason: "S6目录递归标注", Now: now,
|
||||
})
|
||||
if err != nil || result.SelectedFiles != 1 || result.RoutableFiles != 1 {
|
||||
if err != nil || result.SelectedFiles != 1 || result.PlannedPublications != 1 || result.FetchQueued != 0 {
|
||||
t.Fatalf("route=%#v err=%v", result, err)
|
||||
}
|
||||
// Effective visibility of the deep item must be internal via the directory rule.
|
||||
|
||||
@@ -128,7 +128,7 @@ SELECT p.id, si.title, p.ingest_state, p.processing_state, p.retrieval_state,
|
||||
JOIN source_item_versions other_version ON other_version.id=other.source_item_version_id
|
||||
WHERE other.id<>p.id AND other_version.source_item_id=siv.source_item_id
|
||||
) THEN 'updated' ELSE 'added' END AS change_type,
|
||||
(p.ingest_state='pending' AND p.processing_state='pending'
|
||||
(p.ingest_state='pending' AND p.processing_state IN ('pending','planned')
|
||||
AND p.retrieval_state='unpublished' AND p.knowledge_id=''
|
||||
AND co.content_state='verified' AND co.local_spool_ref<>'' AND co.size_bytes>0
|
||||
AND NOT EXISTS (SELECT 1 FROM processing_confirmations confirmation WHERE confirmation.publication_id=p.id)
|
||||
@@ -142,8 +142,8 @@ SELECT p.id, si.title, p.ingest_state, p.processing_state, p.retrieval_state,
|
||||
FROM test_publications p
|
||||
JOIN source_item_versions siv ON siv.id = p.source_item_version_id
|
||||
JOIN source_items si ON si.id = siv.source_item_id
|
||||
JOIN content_objects co ON co.id = siv.content_object_id
|
||||
WHERE ($1 = '' OR p.processing_state = $1) AND ($2 = '' OR p.target_scope = $2)
|
||||
LEFT JOIN content_objects co ON co.id = siv.content_object_id
|
||||
WHERE ($1 = '' OR p.processing_state = $1) AND ($2 = '' OR p.target_scope = $2) AND p.superseded_at IS NULL
|
||||
ORDER BY p.updated_at DESC, p.id
|
||||
LIMIT $3 OFFSET $4
|
||||
`
|
||||
|
||||
@@ -13,7 +13,7 @@ import (
|
||||
|
||||
var ErrInvalidUISettings = errors.New("invalid UI settings")
|
||||
|
||||
const uiSettingsColumns = `id,state,document_refresh_seconds,resource_refresh_seconds,stalled_after_seconds,finalizing_after_seconds,updated_by,created_at,activated_at,retired_at,error_code`
|
||||
const uiSettingsColumns = `id,state,document_refresh_seconds,resource_refresh_seconds,stalled_after_seconds,finalizing_after_seconds,ingestion_concurrency,updated_by,created_at,activated_at,retired_at,error_code`
|
||||
|
||||
const getActiveUISettingsSQL = `
|
||||
SELECT ` + uiSettingsColumns + `
|
||||
@@ -24,8 +24,8 @@ WHERE state = 'active'
|
||||
const createUISettingsCandidateSQL = `
|
||||
INSERT INTO ui_setting_revisions(
|
||||
id,state,document_refresh_seconds,resource_refresh_seconds,
|
||||
stalled_after_seconds,finalizing_after_seconds,updated_by,created_at
|
||||
) VALUES($1,'validated',$2,$3,$4,$5,$6,$7)
|
||||
stalled_after_seconds,finalizing_after_seconds,ingestion_concurrency,updated_by,created_at
|
||||
) VALUES($1,'validated',$2,$3,$4,$5,$6,$7,$8)
|
||||
RETURNING ` + uiSettingsColumns + `
|
||||
`
|
||||
|
||||
@@ -63,7 +63,8 @@ func (s *Store) CreateUISettingsCandidate(ctx context.Context, request governanc
|
||||
}
|
||||
return scanUISettings(s.db.QueryRow(ctx, createUISettingsCandidateSQL,
|
||||
id, request.DocumentRefreshSeconds, request.ResourceRefreshSeconds,
|
||||
request.StalledAfterSeconds, request.FinalizingAfterSeconds, request.UpdatedBy, request.Now.UTC()))
|
||||
request.StalledAfterSeconds, request.FinalizingAfterSeconds, request.IngestionConcurrency,
|
||||
request.UpdatedBy, request.Now.UTC()))
|
||||
}
|
||||
|
||||
func (s *Store) ActivateUISettings(ctx context.Context, revisionID string, now time.Time) (governance.UISettings, error) {
|
||||
@@ -124,6 +125,7 @@ func scanUISettings(row interface{ Scan(...any) error }) (governance.UISettings,
|
||||
var settings governance.UISettings
|
||||
if err := row.Scan(&settings.ID, &settings.State, &settings.DocumentRefreshSeconds,
|
||||
&settings.ResourceRefreshSeconds, &settings.StalledAfterSeconds, &settings.FinalizingAfterSeconds,
|
||||
&settings.IngestionConcurrency,
|
||||
&settings.UpdatedBy, &settings.CreatedAt, &settings.ActivatedAt, &settings.RetiredAt, &settings.ErrorCode); err != nil {
|
||||
return governance.UISettings{}, fmt.Errorf("scan UI settings: %w", err)
|
||||
}
|
||||
|
||||
@@ -31,7 +31,7 @@ func TestUISettingsActivationRetiresPreviousRevisionIntegration(t *testing.T) {
|
||||
t.Fatalf("previous=%#v error=%v", previous, err)
|
||||
}
|
||||
now := time.Date(2026, 9, 10, 7, 0, 0, 0, time.UTC)
|
||||
candidate, err := store.CreateUISettingsCandidate(ctx, governance.CreateUISettingsCandidateRequest{DocumentRefreshSeconds: 30, ResourceRefreshSeconds: 60, StalledAfterSeconds: 3600, FinalizingAfterSeconds: 14400, UpdatedBy: "engineer-1", Now: now})
|
||||
candidate, err := store.CreateUISettingsCandidate(ctx, governance.CreateUISettingsCandidateRequest{DocumentRefreshSeconds: 30, ResourceRefreshSeconds: 60, StalledAfterSeconds: 3600, FinalizingAfterSeconds: 14400, IngestionConcurrency: 2, UpdatedBy: "engineer-1", Now: now})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -14,7 +14,7 @@ func TestUISettingsCandidateAndActiveProjectionAreVersioned(t *testing.T) {
|
||||
now := time.Date(2026, 9, 10, 6, 0, 0, 0, time.UTC)
|
||||
row := uiSettingsRow{id: "settings-safe", state: "validated", documentRefresh: 30, resourceRefresh: 60, stalledAfter: 3600, finalizingAfter: 14400, updatedBy: "engineer-1", createdAt: now}
|
||||
db := &recordingDB{row: row}
|
||||
candidate, err := NewStore(db).CreateUISettingsCandidate(context.Background(), governance.CreateUISettingsCandidateRequest{DocumentRefreshSeconds: 30, ResourceRefreshSeconds: 60, StalledAfterSeconds: 3600, FinalizingAfterSeconds: 14400, UpdatedBy: "engineer-1", Now: now})
|
||||
candidate, err := NewStore(db).CreateUISettingsCandidate(context.Background(), governance.CreateUISettingsCandidateRequest{DocumentRefreshSeconds: 30, ResourceRefreshSeconds: 60, StalledAfterSeconds: 3600, FinalizingAfterSeconds: 14400, IngestionConcurrency: 2, UpdatedBy: "engineer-1", Now: now})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -53,6 +53,7 @@ func TestUISettingsRejectInvalidCandidate(t *testing.T) {
|
||||
type uiSettingsRow struct {
|
||||
id, state, updatedBy, errorCode string
|
||||
documentRefresh, resourceRefresh, stalledAfter, finalizingAfter int
|
||||
ingestionConcurrency int
|
||||
createdAt time.Time
|
||||
activatedAt, retiredAt *time.Time
|
||||
}
|
||||
@@ -60,6 +61,7 @@ type uiSettingsRow struct {
|
||||
func (row uiSettingsRow) Scan(dest ...any) error {
|
||||
*dest[0].(*string), *dest[1].(*string) = row.id, row.state
|
||||
*dest[2].(*int), *dest[3].(*int), *dest[4].(*int), *dest[5].(*int) = row.documentRefresh, row.resourceRefresh, row.stalledAfter, row.finalizingAfter
|
||||
*dest[6].(*string), *dest[7].(*time.Time), *dest[8].(**time.Time), *dest[9].(**time.Time), *dest[10].(*string) = row.updatedBy, row.createdAt, row.activatedAt, row.retiredAt, row.errorCode
|
||||
*dest[6].(*int) = row.ingestionConcurrency
|
||||
*dest[7].(*string), *dest[8].(*time.Time), *dest[9].(**time.Time), *dest[10].(**time.Time), *dest[11].(*string) = row.updatedBy, row.createdAt, row.activatedAt, row.retiredAt, row.errorCode
|
||||
return nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user