feat(governance): policy-driven ingestion from catalog annotations

Annotation in 来源与目录 is now the only import decision surface:
- migration 000033: catalog_import_intents (durable, rescan-proof per-file
  intent with locator snapshot + fetch lifecycle), source_items.policy_state,
  planned publication state, ui_setting_revisions.ingestion_concurrency
- marking fans a folder decision out to per-file intents, materializes
  versioned items, reuses already-verified local bytes (compare, don't pull),
  and plans one publication per routed KB; blocked retires planned rows
  while live publications stay untouched (需下架 shows in the KB diff)
- artifact.fetch worker lane (scripts/artifact_fetch.py): DingTalk export
  tree/live box API, AList /api/fs/get direct download with SSO-redirect
  fail-closed, PP ledger via locator hash; content-level dedupe on sha256
- 开始处理 promotes planned->pending inside the claim-time concurrency budget
  from system settings; claims enforce it in SQL
- monitor overview rebuilt as knowledge-base diff: planned/needs_removal per
  KB + fetch pipeline summary; frontend lanes 待入库/需下架 replace the
  deprecated snapshot change-set confirmation
- spool: images (png/jpg/gif/bmp/webp) accepted, ppt/pptx rejected in favour
  of PDF twins; DingTalk PPT resolves its converted PDF pair
This commit is contained in:
姚章浩
2026-09-13 22:17:12 +08:00
parent f6cb90cd70
commit 63f3b2c503
38 changed files with 1597 additions and 307 deletions
@@ -0,0 +1,39 @@
package postgres
import (
"context"
"errors"
"fmt"
"github.com/jackc/pgx/v5"
"knowledge-governance/internal/workflows/artifactfetch"
)
// LoadArtifactFetchManifest resolves one pending intent plus its live source
// coordinates for the fetch script, keyed by the connection-stable node id.
func (s *Store) LoadArtifactFetchManifest(ctx context.Context, connectionID, nodeID string) (artifactfetch.Manifest, error) {
return loadArtifactFetchManifest(ctx, s.db, connectionID, nodeID)
}
func loadArtifactFetchManifest(ctx context.Context, db interface {
QueryRow(context.Context, string, ...any) pgx.Row
}, connectionID, nodeID string) (artifactfetch.Manifest, error) {
var manifest artifactfetch.Manifest
err := db.QueryRow(ctx, `SELECT connection.source_type,connection.name,intent.relative_path,intent.name,
intent.size_bytes,intent.modified_at,intent.locator_hash
FROM catalog_import_intents intent
JOIN source_connections connection ON connection.id=intent.source_connection_id
WHERE intent.source_connection_id=$1 AND intent.node_id=$2`, connectionID, nodeID).
Scan(&manifest.SourceType, &manifest.SourceName, &manifest.RelativePath, &manifest.Name,
&manifest.SizeBytes, &manifest.ModifiedAt, &manifest.LocatorHash)
if errors.Is(err, pgx.ErrNoRows) {
return artifactfetch.Manifest{}, artifactfetch.ErrManifestGone
}
if err != nil {
return artifactfetch.Manifest{}, fmt.Errorf("load artifact fetch manifest: %w", err)
}
manifest.ConnectionID = connectionID
manifest.NodeID = nodeID
return manifest, nil
}
+7 -55
View File
@@ -112,7 +112,8 @@ FOR UPDATE OF node`, request.ConnectionID, nodeIDs)
JOIN source_catalog_nodes child ON child.snapshot_id=parent.snapshot_id AND child.parent_node_id=parent.node_id
WHERE parent.node_type='folder'
)
SELECT DISTINCT node.node_id,node.source_item_id,(node.node_id=ANY($2::text[])) AS directly_selected
SELECT DISTINCT node.node_id,node.name,node.relative_path,node.size_bytes,node.modified_at,node.content_sha256,
node.source_locator_hash,(node.node_id=ANY($2::text[])) AS directly_selected
FROM descendants
JOIN source_catalog_nodes node ON node.snapshot_id=descendants.snapshot_id AND node.node_id=descendants.node_id
WHERE node.node_type='file'
@@ -120,15 +121,11 @@ ORDER BY node.node_id`, request.ConnectionID, nodeIDs)
if err != nil {
return governance.CatalogRouteResult{}, fmt.Errorf("select catalog route files: %w", err)
}
type selectedFile struct {
nodeID string
sourceItemID *string
direct bool
}
files := make([]selectedFile, 0, len(nodeIDs))
files := make([]governance.CatalogRouteFile, 0, len(nodeIDs))
for rows.Next() {
var file selectedFile
if err := rows.Scan(&file.nodeID, &file.sourceItemID, &file.direct); err != nil {
var file governance.CatalogRouteFile
if err := rows.Scan(&file.NodeID, &file.Name, &file.RelativePath, &file.SizeBytes, &file.ModifiedAt,
&file.ContentSHA256, &file.LocatorHash, &file.Direct); err != nil {
rows.Close()
return governance.CatalogRouteResult{}, fmt.Errorf("scan catalog route file: %w", err)
}
@@ -139,52 +136,7 @@ ORDER BY node.node_id`, request.ConnectionID, nodeIDs)
return governance.CatalogRouteResult{}, fmt.Errorf("iterate catalog route files: %w", err)
}
rows.Close()
result := governance.CatalogRouteResult{SelectedFiles: len(files)}
routedItems := map[string]bool{}
for _, file := range files {
if file.sourceItemID == nil {
result.UnboundFiles++
continue
}
itemID := *file.sourceItemID
if routedItems[itemID] {
continue
}
routedItems[itemID] = true
if file.direct {
if _, err := tx.Exec(ctx, `INSERT INTO source_item_scope_overrides(source_item_id,visibility,updated_by,reason,created_at,updated_at)
VALUES($1,$2,$3,$4,$5,$5) ON CONFLICT(source_item_id) DO UPDATE SET visibility=EXCLUDED.visibility,updated_by=EXCLUDED.updated_by,reason=EXCLUDED.reason,updated_at=EXCLUDED.updated_at`, itemID, request.Visibility, request.OperatorRef, request.Reason, request.Now); err != nil {
return governance.CatalogRouteResult{}, fmt.Errorf("set routed item scope: %w", err)
}
eventID, err := newOpaqueID("se-")
if err != nil {
return governance.CatalogRouteResult{}, err
}
if _, err := tx.Exec(ctx, `INSERT INTO scope_change_events(id,target_type,source_item_id,action,decided_visibility,operator_ref,reason,created_at)
VALUES($1,'item',$2,'set',$3,$4,$5,$6)`, eventID, itemID, request.Visibility, request.OperatorRef, request.Reason, request.Now); err != nil {
return governance.CatalogRouteResult{}, fmt.Errorf("audit routed item scope: %w", err)
}
}
// S7: classify only. Annotations never create publications, retire targets, or enqueue jobs.
var currentVersionID *string
var contentState, spoolRef string
err = tx.QueryRow(ctx, `SELECT version.id,content.content_state,COALESCE(content.local_spool_ref,'') FROM source_items item
LEFT JOIN source_item_versions version ON version.source_item_id=item.id AND version.revision=item.revision
LEFT JOIN content_objects content ON content.id=version.content_object_id
WHERE item.id=$1`, itemID).Scan(&currentVersionID, &contentState, &spoolRef)
switch {
case errors.Is(err, pgx.ErrNoRows):
result.UnavailableFiles++
case err != nil:
return governance.CatalogRouteResult{}, fmt.Errorf("inspect routed artifact: %w", err)
case contentState == string(governance.ContentVerified) && spoolRef != "":
result.RoutableFiles++
default:
result.HashUnknownFiles++
}
}
return result, nil
return fulfillCatalogIntentsTx(ctx, tx, request, files, request.Now)
}
func setCatalogDirectoryPolicy(ctx context.Context, tx pgx.Tx, request governance.CatalogRouteRequest, directoryPath string) error {
@@ -10,9 +10,10 @@ import (
"knowledge-governance/internal/governance"
)
// S7: catalog annotation persists ONLY desired state (scope rules/overrides + audit).
// It never creates or mutates test_publications and never enqueues jobs.
func TestRouteCatalogFilesWritesDesiredStateWithoutSideEffects(t *testing.T) {
// Policy-driven ingestion: saving an annotation records durable intents,
// materializes items, enqueues artifact fetches and plans publications.
// The only human action left is 开始处理; marking is the decision.
func TestRouteCatalogFilesFulfillsPolicyIntent(t *testing.T) {
databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL"))
if databaseURL == "" {
t.Skip("TEST_DATABASE_URL is not set")
@@ -46,66 +47,70 @@ INSERT INTO source_catalog_heads(source_connection_id,snapshot_id,activated_at)
}
store := NewStore(pool)
now := time.Date(2026, 9, 9, 1, 0, 0, 0, time.UTC)
// Folder annotation fans out to files; the bound file already holds verified
// bytes, so it is planned without any fetch (本地已有 → 比对复用 → 不拉取).
result, err := store.RouteCatalogFiles(ctx, governance.CatalogRouteRequest{ConnectionID: "conn-route", NodeIDs: []string{"folder-route"}, Visibility: governance.VisibilityInternal, TargetKnowledgeBaseID: "kb-internal", OperatorRef: "engineer-1", Reason: "整目录标注", Now: now})
if err != nil {
t.Fatal(err)
}
if result.SelectedFiles != 1 || result.RoutableFiles != 1 || result.UnboundFiles != 0 || result.HashUnknownFiles != 0 {
if result.SelectedFiles != 1 || result.IntentsRecorded != 1 || result.PlannedPublications != 1 || result.FetchQueued != 0 {
t.Fatalf("result=%#v", result)
}
// Desired state saved, recursion intact.
var directoryPolicy string
if err := pool.QueryRow(ctx, `SELECT visibility FROM directory_scope_rules WHERE source_connection_id='conn-route' AND directory_path='工业产品'`).Scan(&directoryPolicy); err != nil {
if err := pool.QueryRow(ctx, `SELECT visibility FROM directory_scope_rules WHERE source_connection_id='conn-route' AND directory_path='工业产品'`).Scan(&directoryPolicy); err != nil || directoryPolicy != "internal" {
t.Fatalf("directory policy=%q err=%v", directoryPolicy, err)
}
var intentCount, publicationCount, fetchJobs, uploadJobs int
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM catalog_import_intents WHERE source_connection_id='conn-route' AND visibility='internal'`).Scan(&intentCount); err != nil {
t.Fatal(err)
}
if directoryPolicy != "internal" {
t.Fatalf("directory policy=%q", directoryPolicy)
}
// Zero side effects: no publications, no jobs.
var publicationCount, jobCount int
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM test_publications`).Scan(&publicationCount); err != nil {
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM test_publications WHERE processing_state='planned' AND superseded_at IS NULL`).Scan(&publicationCount); err != nil {
t.Fatal(err)
}
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM jobs`).Scan(&jobCount); err != nil {
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM jobs WHERE type='artifact.fetch'`).Scan(&fetchJobs); err != nil {
t.Fatal(err)
}
if publicationCount != 0 || jobCount != 0 {
t.Fatalf("annotation must not touch pipeline: publications=%d jobs=%d", publicationCount, jobCount)
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM jobs WHERE type='test.upload'`).Scan(&uploadJobs); err != nil {
t.Fatal(err)
}
if intentCount != 1 || publicationCount != 1 || fetchJobs != 0 || uploadJobs != 0 {
t.Fatalf("intents=%d planned=%d fetch=%d upload=%d", intentCount, publicationCount, fetchJobs, uploadJobs)
}
// The planned publication binds the pre-verified current version.
var plannedVersion string
if err := pool.QueryRow(ctx, `SELECT source_item_version_id FROM test_publications WHERE test_kb_id='kb-internal'`).Scan(&plannedVersion); err != nil || plannedVersion != "version-route" {
t.Fatalf("planned version=%q err=%v", plannedVersion, err)
}
// Re-annotating the same directory is idempotent for rules and still side-effect free.
// Re-marking the same directory is replay-safe: one intent, one publication.
again, err := store.RouteCatalogFiles(ctx, governance.CatalogRouteRequest{ConnectionID: "conn-route", NodeIDs: []string{"folder-route"}, Visibility: governance.VisibilityInternal, TargetKnowledgeBaseID: "kb-internal", OperatorRef: "engineer-1", Reason: "重复提交", Now: now.Add(time.Minute)})
if err != nil || again.SelectedFiles != 1 || again.RoutableFiles != 1 {
if err != nil || again.SelectedFiles != 1 || again.PlannedPublications != 0 || again.FetchQueued != 0 {
t.Fatalf("again=%#v err=%v", again, err)
}
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM jobs`).Scan(&jobCount); err != nil || jobCount != 0 {
t.Fatalf("repeat annotation jobs=%d err=%v", jobCount, err)
}
var ruleCount int
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM directory_scope_rules WHERE source_connection_id='conn-route' AND directory_path='工业产品'`).Scan(&ruleCount); err != nil || ruleCount != 1 {
t.Fatalf("rule rows=%d err=%v", ruleCount, err)
}
// Switching policy updates desired state without retiring or removing anything.
// Widening to both plans a second KB publication; nothing is fetched again.
both, err := store.RouteCatalogFiles(ctx, governance.CatalogRouteRequest{ConnectionID: "conn-route", NodeIDs: []string{"folder-route"}, Visibility: governance.VisibilityBoth, TargetKnowledgeBaseIDs: []string{"kb-internal", "kb-external"}, OperatorRef: "engineer-1", Reason: "内外知识库同时使用", Now: now.Add(2 * time.Minute)})
if err != nil || both.SelectedFiles != 1 || both.RoutableFiles != 1 {
if err != nil || both.SelectedFiles != 1 || both.PlannedPublications != 1 || both.FetchQueued != 0 {
t.Fatalf("both=%#v err=%v", both, err)
}
if err := pool.QueryRow(ctx, `SELECT visibility FROM directory_scope_rules WHERE source_connection_id='conn-route' AND directory_path='工业产品'`).Scan(&directoryPolicy); err != nil || directoryPolicy != "both" {
t.Fatalf("policy after both=%q err=%v", directoryPolicy, err)
}
// Blocking saves the desired policy only; existing publications (if any) stay untouched.
// An unannotated live publication survives marking blocked: removal is
// knowledge-base diff (需下架) territory, never a silent side effect.
if _, err := pool.Exec(ctx, `INSERT INTO test_publications(id,source_item_version_id,test_kb_id,target_scope,ingest_state,processing_state,retrieval_state,knowledge_id,created_at,updated_at)
VALUES('tp-existing','version-route','kb-internal','internal','uploaded','completed','enabled','kn-1',$1,$1)`, now); err != nil {
t.Fatal(err)
}
blocked, err := store.RouteCatalogFiles(ctx, governance.CatalogRouteRequest{ConnectionID: "conn-route", NodeIDs: []string{"folder-route"}, Visibility: governance.VisibilityBlocked, OperatorRef: "engineer-1", Reason: "禁止导入", Now: now.Add(3 * time.Minute)})
if err != nil {
t.Fatal(err)
}
if blocked.SelectedFiles != 1 {
t.Fatalf("blocked=%#v", blocked)
if err != nil || blocked.SelectedFiles != 1 || blocked.BlockedItems != 1 {
t.Fatalf("blocked=%#v err=%v", blocked, err)
}
var retrievalState string
var removalJobs int
@@ -118,39 +123,36 @@ VALUES('tp-existing','version-route','kb-internal','internal','uploaded','comple
if retrievalState != "enabled" || removalJobs != 0 {
t.Fatalf("blocked annotation must not remove: state=%s removal jobs=%d", retrievalState, removalJobs)
}
// The planned kb-external row created while policy was 'both' is retired.
var stillPlanned int
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM test_publications WHERE processing_state='planned' AND superseded_at IS NULL AND test_kb_id='kb-external'`).Scan(&stillPlanned); err != nil || stillPlanned != 0 {
t.Fatalf("blocked must retire planned rows=%d err=%v", stillPlanned, err)
}
if err := pool.QueryRow(ctx, `SELECT visibility FROM directory_scope_rules WHERE source_connection_id='conn-route' AND directory_path='工业产品'`).Scan(&directoryPolicy); err != nil || directoryPolicy != "blocked" {
t.Fatalf("blocked policy=%q err=%v", directoryPolicy, err)
}
// Unbound file and hash-unknown file are reported, never planned.
unbound, err := store.RouteCatalogFiles(ctx, governance.CatalogRouteRequest{ConnectionID: "conn-route", NodeIDs: []string{"file-unmapped"}, Visibility: governance.VisibilityExternal, TargetKnowledgeBaseID: "kb-external", OperatorRef: "engineer-1", Reason: "测试未映射", Now: now})
if err != nil || unbound.SelectedFiles != 1 || unbound.RoutableFiles != 0 || unbound.UnboundFiles != 1 {
t.Fatalf("unbound=%#v err=%v", unbound, err)
// A never-fetched file becomes an intent with a materialized item, a
// fetch job and a planned publication (未绑定 → 从源头拉,不再是死路).
restored, err := store.RouteCatalogFiles(ctx, governance.CatalogRouteRequest{ConnectionID: "conn-route", NodeIDs: []string{"file-unmapped"}, Visibility: governance.VisibilityInternal, TargetKnowledgeBaseID: "kb-internal", OperatorRef: "engineer-1", Reason: "重新标注", Now: now.Add(4 * time.Minute)})
if err != nil || restored.IntentsRecorded != 1 || restored.FetchQueued != 1 || restored.PlannedPublications != 1 {
t.Fatalf("restored=%#v err=%v", restored, err)
}
if _, err := pool.Exec(ctx, `INSERT INTO content_objects(id,sha256,size_bytes,local_spool_ref,content_state) VALUES('content-pending',repeat('e',64),8,'','unknown')`); err != nil {
var fetchState, itemPolicy string
if err := pool.QueryRow(ctx, `SELECT intent.fetch_state||'|'||item.policy_state FROM catalog_import_intents intent
JOIN source_items item ON item.id=intent.source_item_id
WHERE intent.node_id='file-unmapped'`).Scan(&fetchState, &itemPolicy); err != nil {
t.Fatal(err)
}
if _, err := pool.Exec(ctx, `INSERT INTO source_items(id,source_connection_id,stable_external_id,canonical_uri_hash,normalized_path,title,item_type,revision,visibility,visibility_origin)
VALUES('item-pending','conn-route','external-pending',repeat('f',64),'待校验.pdf','待校验.pdf','document','v1','external','pp_public_tag')`); err != nil {
t.Fatal(err)
if fetchState != "pending|internal" {
t.Fatalf("fetch state=%q", fetchState)
}
if _, err := pool.Exec(ctx, `INSERT INTO source_item_versions(id,source_item_id,revision,content_object_id,content_state) VALUES('version-pending','item-pending','v1','content-pending','unknown')`); err != nil {
t.Fatal(err)
var versionState string
if err := pool.QueryRow(ctx, `SELECT version.content_state FROM source_item_versions version
JOIN source_items item ON item.id=version.source_item_id AND item.revision=version.revision
WHERE item.stable_external_id='file-unmapped'`).Scan(&versionState); err != nil || versionState != "unknown" {
t.Fatalf("pending version state=%q err=%v", versionState, err)
}
if _, err := pool.Exec(ctx, `INSERT INTO source_catalog_nodes(snapshot_id,node_id,parent_node_id,node_type,name,relative_path,source_locator_hash,source_item_id,size_bytes,direct_child_count,subtree_file_count,subtree_bytes)
VALUES('snapshot-route','file-pending','root-route','file','待校验.pdf','待校验.pdf',repeat('d',64),'item-pending',8,0,1,8)`); err != nil {
t.Fatal(err)
}
hashUnknown, err := store.RouteCatalogFiles(ctx, governance.CatalogRouteRequest{ConnectionID: "conn-route", NodeIDs: []string{"file-pending"}, Visibility: governance.VisibilityInternal, TargetKnowledgeBaseID: "kb-internal", OperatorRef: "engineer-1", Reason: "测试内容未验证", Now: now})
if err != nil || hashUnknown.SelectedFiles != 1 || hashUnknown.RoutableFiles != 0 || hashUnknown.HashUnknownFiles != 1 {
t.Fatalf("hash unknown=%#v err=%v", hashUnknown, err)
}
// Overrides are still recorded for unbound/pending items so the desired state survives.
var overrideCount int
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM source_item_scope_overrides WHERE source_item_id='item-pending'`).Scan(&overrideCount); err != nil || overrideCount != 1 {
t.Fatalf("pending override rows=%d err=%v", overrideCount, err)
}
// Audit trail exists for every annotation action.
var events int
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM scope_change_events`).Scan(&events); err != nil || events < 5 {
t.Fatalf("scope events=%d err=%v", events, err)
@@ -46,7 +46,7 @@ INSERT INTO source_catalog_heads(source_connection_id,snapshot_id,activated_at)
now := time.Date(2026, 9, 11, 6, 0, 0, 0, time.UTC)
request := governance.CatalogRouteRequest{ConnectionID: "conn-op", NodeIDs: []string{"file-op"}, Visibility: governance.VisibilityInternal, TargetKnowledgeBaseID: "kb-internal", OperatorRef: "engineer-1", Reason: "初次标注", Now: now}
first, replayed, err := store.RouteCatalogFilesWithOperation(ctx, request, "op-route-1")
if err != nil || replayed || first.RoutableFiles != 1 {
if err != nil || replayed || first.PlannedPublications != 1 {
t.Fatalf("first result=%#v replayed=%v error=%v", first, replayed, err)
}
second, replayed, err := store.RouteCatalogFilesWithOperation(ctx, request, "op-route-1")
@@ -60,8 +60,8 @@ INSERT INTO source_catalog_heads(source_connection_id,snapshot_id,activated_at)
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM operation_requests WHERE operation_type='catalog.route' AND operation_id='op-route-1'`).Scan(&operations); err != nil {
t.Fatal(err)
}
if publications != 0 || operations != 1 {
t.Fatalf("S7 annotation must not publish: publications=%d operations=%d", publications, operations)
if publications != 1 || operations != 1 {
t.Fatalf("annotation must plan exactly one replay-safe publication: publications=%d operations=%d", publications, operations)
}
conflicting := request
conflicting.Reason = "不同的操作内容"
@@ -0,0 +1,9 @@
ALTER TABLE ui_setting_revisions DROP COLUMN IF EXISTS ingestion_concurrency;
DROP INDEX IF EXISTS idx_test_publications_planned;
ALTER TABLE test_publications DROP CONSTRAINT IF EXISTS test_publications_processing_state_check;
ALTER TABLE test_publications ADD CONSTRAINT test_publications_processing_state_check
CHECK (processing_state IN ('pending', 'processing', 'finalizing', 'completed', 'failed', 'stalled'));
DROP INDEX IF EXISTS idx_source_items_policy;
ALTER TABLE source_items DROP COLUMN IF EXISTS policy_state;
DROP INDEX IF EXISTS idx_catalog_intents_fetch;
DROP TABLE IF EXISTS catalog_import_intents;
@@ -0,0 +1,62 @@
-- Policy-driven ingestion: the ONLY import decision surface is the catalog
-- annotation (internal / external / both / blocked). Marking a file or folder
-- records a durable per-file INTENT that survives rescans; unannotated files
-- stay "unset" and are never ingested (cold start imports nothing).
-- A saved intent immediately materializes a versioned source item and an
-- artifact.fetch job so the artifact is pulled from the live source (or
-- matched against an already-present local copy); once content is verified a
-- 'planned' publication appears in the knowledge-base diff as 待入库. The one
-- human batch action "开始处理" promotes planned -> queued within the
-- concurrency budget configured in system settings.
-- Per-file decision, keyed by the connection-stable catalog node id
-- (node ids are derived from source_key + relative path, identical across
-- snapshots, so intents are rescan-proof).
CREATE TABLE IF NOT EXISTS catalog_import_intents (
source_connection_id VARCHAR(36) NOT NULL
REFERENCES source_connections(id) ON DELETE CASCADE,
node_id VARCHAR(64) NOT NULL,
visibility VARCHAR(20) NOT NULL
CHECK (visibility IN ('internal', 'external', 'both', 'blocked')),
source_item_id VARCHAR(36),
decided_by VARCHAR(128) NOT NULL,
reason VARCHAR(500) NOT NULL DEFAULT '',
-- Locator snapshot: the fetch worker needs path/name metadata without
-- re-walking a snapshot that may already have rotated.
relative_path TEXT NOT NULL DEFAULT '',
name TEXT NOT NULL DEFAULT '',
size_bytes BIGINT,
modified_at TIMESTAMPTZ,
content_sha256 CHAR(64),
locator_hash CHAR(64) NOT NULL DEFAULT '',
-- Content materialization lifecycle driven by the artifact.fetch job.
fetch_state VARCHAR(20) NOT NULL DEFAULT 'pending'
CHECK (fetch_state IN ('pending', 'fetching', 'ready', 'missing_local', 'failed')),
last_error TEXT NOT NULL DEFAULT '',
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
PRIMARY KEY (source_connection_id, node_id)
);
CREATE INDEX IF NOT EXISTS idx_catalog_intents_item
ON catalog_import_intents (source_item_id) WHERE source_item_id IS NOT NULL;
CREATE INDEX IF NOT EXISTS idx_catalog_intents_fetch
ON catalog_import_intents (fetch_state) WHERE fetch_state <> 'ready';
-- Materialized items carry the policy so list/diff queries never re-walk rules.
ALTER TABLE source_items ADD COLUMN IF NOT EXISTS policy_state VARCHAR(20) NOT NULL DEFAULT 'unset'
CHECK (policy_state IN ('unset', 'internal', 'external', 'both', 'blocked'));
CREATE INDEX IF NOT EXISTS idx_source_items_policy
ON source_items (policy_state) WHERE policy_state <> 'unset';
-- The knowledge base is the ingestion target of a planned publication; make
-- "planned" explicit so the KB-diff view never reuses queued semantics.
ALTER TABLE test_publications DROP CONSTRAINT IF EXISTS test_publications_processing_state_check;
ALTER TABLE test_publications ADD CONSTRAINT test_publications_processing_state_check
CHECK (processing_state IN ('planned', 'pending', 'processing', 'finalizing', 'completed', 'failed', 'stalled'));
CREATE INDEX IF NOT EXISTS idx_test_publications_planned
ON test_publications (test_kb_id, processing_state) WHERE processing_state = 'planned';
-- 开始处理's concurrency budget lives in system settings (validated revisions).
ALTER TABLE ui_setting_revisions ADD COLUMN IF NOT EXISTS ingestion_concurrency INTEGER NOT NULL DEFAULT 2
CHECK (ingestion_concurrency BETWEEN 1 AND 16);
@@ -11,8 +11,8 @@ func TestEmbeddedMigrationsAreVersionSorted(t *testing.T) {
if err != nil {
t.Fatalf("loadMigrations() error = %v", err)
}
if len(migrations) != 32 {
t.Fatalf("migration count = %d, want 32", len(migrations))
if len(migrations) != 33 {
t.Fatalf("migration count = %d, want 33", len(migrations))
}
if migrations[0].Version != 1 || !strings.Contains(migrations[0].UpSQL, "CREATE TABLE IF NOT EXISTS jobs") {
t.Fatalf("migration 1 = %#v", migrations[0])
@@ -567,7 +567,8 @@ func TestReleaseExecutionMigrationDefinesAuthorizedAuditableJobs(t *testing.T) {
func TestClaimSQLUsesSkipLockedAndReclaimsExpiredRunningJobs(t *testing.T) {
sql := strings.ToLower(claimJobSQL)
for _, required := range []string{
"for update skip locked",
"for update of jobs skip locked",
"ingestion_concurrency",
"state in ('pending', 'retry')",
"state = 'running'",
"lease_until <",
@@ -0,0 +1,364 @@
package postgres
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"time"
"github.com/jackc/pgx/v5"
"knowledge-governance/internal/governance"
)
// fulfillCatalogIntentsTx is the policy-driven ingestion core: saving an
// annotation in 来源与目录 records one durable intent per selected file and,
// for importable policies, materializes a versioned source item, enqueues an
// artifact.fetch job (bytes come from the live source; a matching local copy
// short-circuits the download) and creates one 'planned' publication per routed
// knowledge base. Planned publications are what 解析监控 lists as 待入库;
// 开始处理 promotes them within the configured concurrency budget. Blocked
// intents retire any planned publication and flip the item policy so the
// knowledge-base diff demands removal of anything still live.
func fulfillCatalogIntentsTx(ctx context.Context, tx pgx.Tx, request governance.CatalogRouteRequest, files []governance.CatalogRouteFile, now time.Time) (governance.CatalogRouteResult, error) {
targetKnowledgeBaseIDs := request.TargetKnowledgeBaseIDs
if len(targetKnowledgeBaseIDs) == 0 && request.TargetKnowledgeBaseID != "" {
targetKnowledgeBaseIDs = []string{request.TargetKnowledgeBaseID}
}
result := governance.CatalogRouteResult{SelectedFiles: len(files)}
for _, file := range files {
recorded, fetchQueued, planned, blocked, err := fulfillCatalogIntent(ctx, tx, request, file, targetKnowledgeBaseIDs, now)
if err != nil {
return governance.CatalogRouteResult{}, err
}
if !recorded {
result.UnboundFiles++
continue
}
result.IntentsRecorded++
if blocked {
result.BlockedItems++
continue
}
if fetchQueued {
result.FetchQueued++
}
result.PlannedPublications += planned
}
return result, nil
}
// fulfillCatalogIntent applies one file decision. It returns recorded=false
// only when the node vanished mid-transaction; fetchQueued reports a newly
// enqueued artifact.fetch job and planned the number of publications created.
func fulfillCatalogIntent(ctx context.Context, tx pgx.Tx, request governance.CatalogRouteRequest, file governance.CatalogRouteFile, targetKnowledgeBaseIDs []string, now time.Time) (bool, bool, int, bool, error) {
policy := string(request.Visibility)
if _, err := tx.Exec(ctx, `INSERT INTO catalog_import_intents(
source_connection_id,node_id,visibility,decided_by,reason,relative_path,name,size_bytes,modified_at,content_sha256,
locator_hash,fetch_state,last_error,created_at,updated_at)
VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,'pending','',NOW(),NOW())
ON CONFLICT(source_connection_id,node_id) DO UPDATE SET
visibility=EXCLUDED.visibility,decided_by=EXCLUDED.decided_by,reason=EXCLUDED.reason,
relative_path=EXCLUDED.relative_path,name=EXCLUDED.name,size_bytes=EXCLUDED.size_bytes,
modified_at=EXCLUDED.modified_at,content_sha256=EXCLUDED.content_sha256,
locator_hash=EXCLUDED.locator_hash,
fetch_state=CASE WHEN catalog_import_intents.content_sha256 IS DISTINCT FROM EXCLUDED.content_sha256
OR catalog_import_intents.size_bytes IS DISTINCT FROM EXCLUDED.size_bytes
OR catalog_import_intents.modified_at IS DISTINCT FROM EXCLUDED.modified_at
THEN 'pending' ELSE catalog_import_intents.fetch_state END,
last_error=CASE WHEN catalog_import_intents.content_sha256 IS DISTINCT FROM EXCLUDED.content_sha256
OR catalog_import_intents.size_bytes IS DISTINCT FROM EXCLUDED.size_bytes
OR catalog_import_intents.modified_at IS DISTINCT FROM EXCLUDED.modified_at
THEN '' ELSE catalog_import_intents.last_error END,
updated_at=NOW()`,
request.ConnectionID, file.NodeID, policy, request.OperatorRef, request.Reason,
file.RelativePath, file.Name, file.SizeBytes, file.ModifiedAt, file.ContentSHA256, file.LocatorHash); err != nil {
return false, false, 0, false, fmt.Errorf("record catalog intent: %w", err)
}
var existingItemID *string
err := tx.QueryRow(ctx, `SELECT source_item_id FROM catalog_import_intents
WHERE source_connection_id=$1 AND node_id=$2 FOR UPDATE`, request.ConnectionID, file.NodeID).Scan(&existingItemID)
if errors.Is(err, pgx.ErrNoRows) {
return false, false, 0, false, nil
}
if err != nil {
return false, false, 0, false, fmt.Errorf("read catalog intent: %w", err)
}
itemID := ""
if existingItemID != nil {
itemID = *existingItemID
}
if itemID == "" {
newItemID, err := newOpaqueID("si-")
if err != nil {
return false, false, 0, false, err
}
if _, err := tx.Exec(ctx, `INSERT INTO source_items(
id,source_connection_id,stable_external_id,canonical_uri_hash,title,item_type,revision,
visibility,visibility_origin,source_change,absence_streak,identity_confidence,
policy_state,created_at,updated_at)
VALUES($1,$2,$3,$4,$5,'file','',$7,'configured_scope','unchanged',0,'object_id',$6,NOW(),NOW())`,
newItemID, request.ConnectionID, "cat:"+file.NodeID,
sha256Hex("policy:"+request.ConnectionID+"\x00"+file.NodeID),
file.Name, policy, itemVisibilityFor(request.Visibility)); err != nil {
return false, false, 0, false, fmt.Errorf("materialize policy source item: %w", err)
}
if _, err := tx.Exec(ctx, `UPDATE catalog_import_intents SET source_item_id=$3 WHERE source_connection_id=$1 AND node_id=$2`,
request.ConnectionID, file.NodeID, newItemID); err != nil {
return false, false, 0, false, fmt.Errorf("bind catalog intent item: %w", err)
}
itemID = newItemID
}
if request.Visibility == governance.VisibilityBlocked {
if _, err := tx.Exec(ctx, `UPDATE source_items SET policy_state='blocked',updated_at=NOW() WHERE id=$1`, itemID); err != nil {
return false, false, 0, false, fmt.Errorf("block policy item: %w", err)
}
if _, err := tx.Exec(ctx, `UPDATE test_publications SET superseded_at=$2,updated_at=$2
WHERE source_item_version_id IN (SELECT id FROM source_item_versions WHERE source_item_id=$1)
AND superseded_at IS NULL AND processing_state='planned'`, itemID, now); err != nil {
return false, false, 0, false, fmt.Errorf("retire blocked planned publications: %w", err)
}
return true, false, 0, true, nil
}
revision := "locator:" + file.LocatorHash
if file.ContentSHA256 != nil && *file.ContentSHA256 != "" {
revision = "sha:" + *file.ContentSHA256
}
if _, err := tx.Exec(ctx, `UPDATE source_items
SET policy_state=$2,visibility=$3,visibility_origin='configured_scope',updated_at=NOW() WHERE id=$1`,
itemID, policy, itemVisibilityFor(request.Visibility)); err != nil {
return false, false, 0, false, fmt.Errorf("update policy item: %w", err)
}
// Resolve the version this intent points at. A bound item whose current
// version already carries verified bytes is reused without any fetch — the
// 本地已有先比对 requirement. Only when the node proves different content
// (scan-reported hash mismatch) or bytes were never fetched do we create a
// revision and enqueue artifact.fetch.
versionID := ""
contentState := ""
var curID, curSHA string
err = tx.QueryRow(ctx, `SELECT version.id,COALESCE(content.sha256,''),version.content_state
FROM source_items item
JOIN source_item_versions version ON version.source_item_id=item.id AND version.revision=item.revision
LEFT JOIN content_objects content ON content.id=version.content_object_id
WHERE item.id=$1`, itemID).Scan(&curID, &curSHA, &contentState)
if err != nil && !errors.Is(err, pgx.ErrNoRows) {
return false, false, 0, false, fmt.Errorf("inspect current item version: %w", err)
}
nodeSHA := ""
if file.ContentSHA256 != nil {
nodeSHA = *file.ContentSHA256
}
reuse := err == nil && contentState == string(governance.ContentVerified) && (nodeSHA == "" || nodeSHA == curSHA)
if !reuse {
contentState = "unknown"
err = tx.QueryRow(ctx, `SELECT version.id,version.content_state
FROM source_item_versions version WHERE version.source_item_id=$1 AND version.revision=$2`,
itemID, revision).Scan(&versionID, &contentState)
if errors.Is(err, pgx.ErrNoRows) {
newVersionID, err := newOpaqueID("sv-")
if err != nil {
return false, false, 0, false, err
}
if _, err := tx.Exec(ctx, `INSERT INTO source_item_versions(
id,source_item_id,revision,content_object_id,content_state,raw_record_summary,source_updated_at,created_at,updated_at)
VALUES($1,$2,$3,NULL,'unknown','{}'::jsonb,$4,NOW(),NOW())`, newVersionID, itemID, revision, file.ModifiedAt); err != nil {
return false, false, 0, false, fmt.Errorf("create policy version: %w", err)
}
if _, err := tx.Exec(ctx, `UPDATE source_items SET revision=$2,updated_at=NOW() WHERE id=$1`, itemID, revision); err != nil {
return false, false, 0, false, fmt.Errorf("advance policy item revision: %w", err)
}
versionID = newVersionID
} else if err != nil {
return false, false, 0, false, fmt.Errorf("lookup policy version: %w", err)
}
} else {
versionID = curID
}
planned := 0
for _, kbID := range targetKnowledgeBaseIDs {
scope := kbScopeFor(request, kbID)
publicationID, err := newOpaqueID("tp-")
if err != nil {
return false, false, 0, false, err
}
tag, err := tx.Exec(ctx, `INSERT INTO test_publications(
id,source_item_version_id,test_kb_id,knowledge_id,ingest_state,processing_state,retrieval_state,
parser_generation,index_generation,last_error,retry_count,target_scope,created_at,updated_at)
VALUES($1,$2,$3,'','pending','planned','unpublished','','','',0,$4,NOW(),NOW())
ON CONFLICT (source_item_version_id,test_kb_id) DO NOTHING`,
publicationID, versionID, kbID, scope)
if err != nil {
return false, false, 0, false, fmt.Errorf("plan publication: %w", err)
}
planned += int(tag.RowsAffected())
}
fetchQueued := false
if contentState != string(governance.ContentVerified) {
jobID, err := newOpaqueID("job-")
if err != nil {
return false, false, 0, false, err
}
payload, err := json.Marshal(map[string]string{"source_connection_id": request.ConnectionID, "node_id": file.NodeID, "version_id": versionID})
if err != nil {
return false, false, 0, false, errors.New("encode artifact fetch job")
}
tag, err := tx.Exec(ctx, `INSERT INTO jobs(id,type,payload,state,priority,available_at,max_attempts,idempotency_key,created_at,updated_at)
VALUES($1,'artifact.fetch',$2::jsonb,'pending',20,$3,8,$4,$3,$3) ON CONFLICT (idempotency_key) DO NOTHING`,
jobID, payload, now, "artifact-fetch:"+versionID)
if err != nil {
return false, false, 0, false, fmt.Errorf("enqueue artifact fetch: %w", err)
}
fetchQueued = tag.RowsAffected() > 0
if fetchQueued {
if _, err := tx.Exec(ctx, `UPDATE catalog_import_intents SET fetch_state='pending',last_error='',updated_at=NOW()
WHERE source_connection_id=$1 AND node_id=$2 AND fetch_state IN ('failed','missing_local','fetching')`, request.ConnectionID, file.NodeID); err != nil {
return false, false, 0, false, fmt.Errorf("reset catalog intent fetch state: %w", err)
}
}
}
return true, fetchQueued, planned, false, nil
}
// kbScopeFor maps a routed knowledge base id to its diff scope label.
func kbScopeFor(request governance.CatalogRouteRequest, knowledgeBaseID string) string {
if request.Visibility == governance.VisibilityExternal {
return "external"
}
if request.Visibility == governance.VisibilityBoth && len(request.TargetKnowledgeBaseIDs) > 1 &&
knowledgeBaseID == request.TargetKnowledgeBaseIDs[len(request.TargetKnowledgeBaseIDs)-1] {
return "external"
}
return "internal"
}
// itemVisibilityFor mirrors source_items' two-value visibility CHECK: both and
// blocked decisions are governed through policy_state; the legacy visibility
// column keeps a valid value and is widened to external for both.
func itemVisibilityFor(visibility governance.Visibility) string {
if visibility == governance.VisibilityInternal {
return string(governance.VisibilityInternal)
}
return string(governance.VisibilityExternal)
}
func sha256Hex(value string) string {
sum := sha256.Sum256([]byte(value))
return hex.EncodeToString(sum[:])
}
// MarkArtifactFetched finalizes one artifact.fetch success: it (re)uses the
// content_objects row for the verified hash (content-level dedupe), attaches it
// to the pending version and flips the intent to ready. matchedLocal records
// that the bytes already existed locally and no source download happened.
func (s *Store) MarkArtifactFetched(ctx context.Context, versionID, sha256Value string, sizeBytes int64, spoolRef, validatedFormat string, matchedLocal bool, now time.Time) error {
if len(sha256Value) != 64 || spoolRef == "" || sizeBytes < 0 || validatedFormat == "" {
return errors.New("invalid artifact fetch result")
}
var tx pgx.Tx
ownsTransaction := false
if existing, ok := s.db.(pgx.Tx); ok {
tx = existing
} else {
beginner, ok := s.db.(interface {
Begin(context.Context) (pgx.Tx, error)
})
if !ok {
return errors.New("artifact fetch transactions are unavailable")
}
var err error
tx, err = beginner.Begin(ctx)
if err != nil {
return fmt.Errorf("begin artifact fetch: %w", err)
}
ownsTransaction = true
defer tx.Rollback(ctx)
}
var connectionID, nodeID string
err := tx.QueryRow(ctx, `SELECT intent.source_connection_id,intent.node_id
FROM catalog_import_intents intent
JOIN source_item_versions version ON version.source_item_id=intent.source_item_id
WHERE version.id=$1 FOR UPDATE OF intent`, versionID).Scan(&connectionID, &nodeID)
if err != nil {
return fmt.Errorf("locate intent for fetched artifact: %w", err)
}
freshID, err := newOpaqueID("co-")
if err != nil {
return err
}
contentID := ""
err = tx.QueryRow(ctx, `INSERT INTO content_objects(id,sha256,size_bytes,mime,validated_format,local_spool_ref,content_state,created_at,updated_at)
VALUES($1,$2,$3,'',$5,$4,'verified',NOW(),NOW())
ON CONFLICT(sha256) DO UPDATE SET
size_bytes=EXCLUDED.size_bytes,
validated_format=CASE WHEN content_objects.validated_format='' THEN EXCLUDED.validated_format ELSE content_objects.validated_format END,
local_spool_ref=CASE WHEN content_objects.local_spool_ref='' THEN EXCLUDED.local_spool_ref ELSE content_objects.local_spool_ref END,
content_state='verified',updated_at=NOW()
RETURNING id`,
freshID, sha256Value, sizeBytes, spoolRef, validatedFormat).Scan(&contentID)
if err != nil {
return fmt.Errorf("record fetched content: %w", err)
}
if _, err := tx.Exec(ctx, `UPDATE source_item_versions SET
content_object_id=$2,content_state='verified',updated_at=$4,
raw_record_summary=jsonb_set(raw_record_summary,'{matched_local}',$3::jsonb)
WHERE id=$1`,
versionID, contentID, mustJSONBool(matchedLocal), now.UTC()); err != nil {
return fmt.Errorf("bind fetched content: %w", err)
}
if _, err := tx.Exec(ctx, `UPDATE catalog_import_intents SET fetch_state='ready',last_error='',updated_at=NOW()
WHERE source_connection_id=$1 AND node_id=$2`, connectionID, nodeID); err != nil {
return fmt.Errorf("mark intent fetched: %w", err)
}
if ownsTransaction {
if err := tx.Commit(ctx); err != nil {
return fmt.Errorf("commit artifact fetch: %w", err)
}
}
return nil
}
func mustJSONBool(value bool) string {
if value {
return "true"
}
return "false"
}
// MarkArtifactFetchMissing parks an intent whose source-side file disappeared
// (a 404 on download, not a transient outage) so the UI can ask for a rescan.
func (s *Store) MarkArtifactFetchMissing(ctx context.Context, connectionID, nodeID string, now time.Time) error {
if _, err := s.db.Exec(ctx, `UPDATE catalog_import_intents SET fetch_state='missing_local',updated_at=$3
WHERE source_connection_id=$1 AND node_id=$2 AND fetch_state<>'ready'`, connectionID, nodeID, now.UTC()); err != nil {
return fmt.Errorf("record artifact fetch miss: %w", err)
}
return nil
}
// MarkArtifactFetchFailed records a fetch failure for the intent so 解析监控
// can show why an item is not yet 待入库.
func (s *Store) MarkArtifactFetchFailed(ctx context.Context, connectionID, nodeID, message string, now time.Time) error {
if _, err := s.db.Exec(ctx, `UPDATE catalog_import_intents SET fetch_state='failed',last_error=$3,updated_at=$4
WHERE source_connection_id=$1 AND node_id=$2 AND fetch_state<>'ready'`, connectionID, nodeID, truncateMonitorText(message), now.UTC()); err != nil {
return fmt.Errorf("record artifact fetch failure: %w", err)
}
return nil
}
func truncateMonitorText(value string) string {
runes := []rune(value)
if len(runes) <= 500 {
return value
}
return string(runes[:500])
}
@@ -16,9 +16,12 @@ func (s *Store) StartTestPublications(ctx context.Context, request governance.St
result := governance.StartTestPublicationsResult{Selected: len(request.PublicationIDs)}
operator := strings.TrimSpace(request.OperatorRef)
reason := strings.TrimSpace(request.Reason)
if len(request.PublicationIDs) == 0 || len(request.PublicationIDs) > 100 || operator == "" || len(operator) > 128 || reason == "" || len(reason) > 500 || request.Now.IsZero() {
if len(request.PublicationIDs) == 0 || len(request.PublicationIDs) > 100 || operator == "" || len(operator) > 128 || len(reason) > 500 || request.Now.IsZero() {
return governance.StartTestPublicationsResult{}, errors.New("invalid processing confirmation")
}
if reason == "" {
reason = "开始处理"
}
seen := make(map[string]bool, len(request.PublicationIDs))
for _, publicationID := range request.PublicationIDs {
if !validRouteID(publicationID) || seen[publicationID] {
@@ -50,22 +53,23 @@ func (s *Store) StartTestPublications(ctx context.Context, request governance.St
for _, publicationID := range request.PublicationIDs {
var ready, confirmed, uploadJobExists bool
var processingState string
err = tx.QueryRow(ctx, `
SELECT p.ingest_state='pending'
AND p.processing_state='pending'
AND p.retrieval_state='unpublished'
AND p.knowledge_id=''
AND version.content_state='verified'
AND content.content_state='verified'
AND content.local_spool_ref<>''
AND content.size_bytes>0 AS ready,
p.processing_state,
EXISTS(SELECT 1 FROM processing_confirmations confirmation WHERE confirmation.publication_id=p.id),
EXISTS(SELECT 1 FROM jobs job WHERE job.type='test.upload' AND job.payload->>'publication_id'=p.id)
FROM test_publications p
JOIN source_item_versions version ON version.id=p.source_item_version_id
JOIN content_objects content ON content.id=version.content_object_id
WHERE p.id=$1
FOR UPDATE OF p`, publicationID).Scan(&ready, &confirmed, &uploadJobExists)
WHERE p.id=$1 AND p.superseded_at IS NULL
FOR UPDATE OF p`, publicationID).Scan(&ready, &processingState, &confirmed, &uploadJobExists)
if errors.Is(err, pgx.ErrNoRows) {
result.Unavailable++
continue
@@ -77,10 +81,18 @@ FOR UPDATE OF p`, publicationID).Scan(&ready, &confirmed, &uploadJobExists)
result.AlreadyStarted++
continue
}
// 开始处理 promotes both fresh plans and verified pending rows.
if processingState != "planned" && processingState != "pending" {
result.Unavailable++
continue
}
if !ready {
result.Unavailable++
continue
}
if _, err := tx.Exec(ctx, `UPDATE test_publications SET processing_state='pending',updated_at=$2 WHERE id=$1 AND processing_state='planned'`, publicationID, request.Now); err != nil {
return governance.StartTestPublicationsResult{}, fmt.Errorf("promote planned publication: %w", err)
}
confirmationID, err := newOpaqueID("pc-")
if err != nil {
return governance.StartTestPublicationsResult{}, err
+21 -9
View File
@@ -13,17 +13,29 @@ RETURNING id, type, payload, state, priority, available_at,
`
const claimJobSQL = `
WITH candidate AS (
SELECT id
FROM jobs
WITH budget AS (
SELECT COALESCE((
SELECT ingestion_concurrency
FROM ui_setting_revisions
WHERE state = 'active'
LIMIT 1
), 2) AS ingestion_slots
), candidate AS (
SELECT jobs.id
FROM jobs, budget
WHERE (
(state IN ('pending', 'retry') AND available_at <= $2)
OR (state = 'running' AND lease_until < $2)
(jobs.state IN ('pending', 'retry') AND jobs.available_at <= $2)
OR (jobs.state = 'running' AND jobs.lease_until < $2)
)
AND attempt < max_attempts
AND type = ANY($4::text[])
ORDER BY priority DESC, available_at ASC, created_at ASC, id ASC
FOR UPDATE SKIP LOCKED
AND jobs.attempt < jobs.max_attempts
AND jobs.type = ANY($4::text[])
AND (
jobs.type <> 'test.upload'
OR (SELECT COUNT(*) FROM jobs active
WHERE active.type = 'test.upload' AND active.state = 'running') < budget.ingestion_slots
)
ORDER BY jobs.priority DESC, jobs.available_at ASC, jobs.created_at ASC, jobs.id ASC
FOR UPDATE OF jobs SKIP LOCKED
LIMIT 1
)
UPDATE jobs AS j
@@ -51,7 +51,7 @@ INSERT INTO source_catalog_heads(source_connection_id,snapshot_id,activated_at)
ConnectionID: "conn-s6", NodeIDs: []string{"folder-top"}, Visibility: governance.VisibilityInternal,
TargetKnowledgeBaseID: "kb-internal", OperatorRef: "engineer-1", Reason: "S6目录递归标注", Now: now,
})
if err != nil || result.SelectedFiles != 1 || result.RoutableFiles != 1 {
if err != nil || result.SelectedFiles != 1 || result.PlannedPublications != 1 || result.FetchQueued != 0 {
t.Fatalf("route=%#v err=%v", result, err)
}
// Effective visibility of the deep item must be internal via the directory rule.
@@ -128,7 +128,7 @@ SELECT p.id, si.title, p.ingest_state, p.processing_state, p.retrieval_state,
JOIN source_item_versions other_version ON other_version.id=other.source_item_version_id
WHERE other.id<>p.id AND other_version.source_item_id=siv.source_item_id
) THEN 'updated' ELSE 'added' END AS change_type,
(p.ingest_state='pending' AND p.processing_state='pending'
(p.ingest_state='pending' AND p.processing_state IN ('pending','planned')
AND p.retrieval_state='unpublished' AND p.knowledge_id=''
AND co.content_state='verified' AND co.local_spool_ref<>'' AND co.size_bytes>0
AND NOT EXISTS (SELECT 1 FROM processing_confirmations confirmation WHERE confirmation.publication_id=p.id)
@@ -142,8 +142,8 @@ SELECT p.id, si.title, p.ingest_state, p.processing_state, p.retrieval_state,
FROM test_publications p
JOIN source_item_versions siv ON siv.id = p.source_item_version_id
JOIN source_items si ON si.id = siv.source_item_id
JOIN content_objects co ON co.id = siv.content_object_id
WHERE ($1 = '' OR p.processing_state = $1) AND ($2 = '' OR p.target_scope = $2)
LEFT JOIN content_objects co ON co.id = siv.content_object_id
WHERE ($1 = '' OR p.processing_state = $1) AND ($2 = '' OR p.target_scope = $2) AND p.superseded_at IS NULL
ORDER BY p.updated_at DESC, p.id
LIMIT $3 OFFSET $4
`
+6 -4
View File
@@ -13,7 +13,7 @@ import (
var ErrInvalidUISettings = errors.New("invalid UI settings")
const uiSettingsColumns = `id,state,document_refresh_seconds,resource_refresh_seconds,stalled_after_seconds,finalizing_after_seconds,updated_by,created_at,activated_at,retired_at,error_code`
const uiSettingsColumns = `id,state,document_refresh_seconds,resource_refresh_seconds,stalled_after_seconds,finalizing_after_seconds,ingestion_concurrency,updated_by,created_at,activated_at,retired_at,error_code`
const getActiveUISettingsSQL = `
SELECT ` + uiSettingsColumns + `
@@ -24,8 +24,8 @@ WHERE state = 'active'
const createUISettingsCandidateSQL = `
INSERT INTO ui_setting_revisions(
id,state,document_refresh_seconds,resource_refresh_seconds,
stalled_after_seconds,finalizing_after_seconds,updated_by,created_at
) VALUES($1,'validated',$2,$3,$4,$5,$6,$7)
stalled_after_seconds,finalizing_after_seconds,ingestion_concurrency,updated_by,created_at
) VALUES($1,'validated',$2,$3,$4,$5,$6,$7,$8)
RETURNING ` + uiSettingsColumns + `
`
@@ -63,7 +63,8 @@ func (s *Store) CreateUISettingsCandidate(ctx context.Context, request governanc
}
return scanUISettings(s.db.QueryRow(ctx, createUISettingsCandidateSQL,
id, request.DocumentRefreshSeconds, request.ResourceRefreshSeconds,
request.StalledAfterSeconds, request.FinalizingAfterSeconds, request.UpdatedBy, request.Now.UTC()))
request.StalledAfterSeconds, request.FinalizingAfterSeconds, request.IngestionConcurrency,
request.UpdatedBy, request.Now.UTC()))
}
func (s *Store) ActivateUISettings(ctx context.Context, revisionID string, now time.Time) (governance.UISettings, error) {
@@ -124,6 +125,7 @@ func scanUISettings(row interface{ Scan(...any) error }) (governance.UISettings,
var settings governance.UISettings
if err := row.Scan(&settings.ID, &settings.State, &settings.DocumentRefreshSeconds,
&settings.ResourceRefreshSeconds, &settings.StalledAfterSeconds, &settings.FinalizingAfterSeconds,
&settings.IngestionConcurrency,
&settings.UpdatedBy, &settings.CreatedAt, &settings.ActivatedAt, &settings.RetiredAt, &settings.ErrorCode); err != nil {
return governance.UISettings{}, fmt.Errorf("scan UI settings: %w", err)
}
@@ -31,7 +31,7 @@ func TestUISettingsActivationRetiresPreviousRevisionIntegration(t *testing.T) {
t.Fatalf("previous=%#v error=%v", previous, err)
}
now := time.Date(2026, 9, 10, 7, 0, 0, 0, time.UTC)
candidate, err := store.CreateUISettingsCandidate(ctx, governance.CreateUISettingsCandidateRequest{DocumentRefreshSeconds: 30, ResourceRefreshSeconds: 60, StalledAfterSeconds: 3600, FinalizingAfterSeconds: 14400, UpdatedBy: "engineer-1", Now: now})
candidate, err := store.CreateUISettingsCandidate(ctx, governance.CreateUISettingsCandidateRequest{DocumentRefreshSeconds: 30, ResourceRefreshSeconds: 60, StalledAfterSeconds: 3600, FinalizingAfterSeconds: 14400, IngestionConcurrency: 2, UpdatedBy: "engineer-1", Now: now})
if err != nil {
t.Fatal(err)
}
@@ -14,7 +14,7 @@ func TestUISettingsCandidateAndActiveProjectionAreVersioned(t *testing.T) {
now := time.Date(2026, 9, 10, 6, 0, 0, 0, time.UTC)
row := uiSettingsRow{id: "settings-safe", state: "validated", documentRefresh: 30, resourceRefresh: 60, stalledAfter: 3600, finalizingAfter: 14400, updatedBy: "engineer-1", createdAt: now}
db := &recordingDB{row: row}
candidate, err := NewStore(db).CreateUISettingsCandidate(context.Background(), governance.CreateUISettingsCandidateRequest{DocumentRefreshSeconds: 30, ResourceRefreshSeconds: 60, StalledAfterSeconds: 3600, FinalizingAfterSeconds: 14400, UpdatedBy: "engineer-1", Now: now})
candidate, err := NewStore(db).CreateUISettingsCandidate(context.Background(), governance.CreateUISettingsCandidateRequest{DocumentRefreshSeconds: 30, ResourceRefreshSeconds: 60, StalledAfterSeconds: 3600, FinalizingAfterSeconds: 14400, IngestionConcurrency: 2, UpdatedBy: "engineer-1", Now: now})
if err != nil {
t.Fatal(err)
}
@@ -53,6 +53,7 @@ func TestUISettingsRejectInvalidCandidate(t *testing.T) {
type uiSettingsRow struct {
id, state, updatedBy, errorCode string
documentRefresh, resourceRefresh, stalledAfter, finalizingAfter int
ingestionConcurrency int
createdAt time.Time
activatedAt, retiredAt *time.Time
}
@@ -60,6 +61,7 @@ type uiSettingsRow struct {
func (row uiSettingsRow) Scan(dest ...any) error {
*dest[0].(*string), *dest[1].(*string) = row.id, row.state
*dest[2].(*int), *dest[3].(*int), *dest[4].(*int), *dest[5].(*int) = row.documentRefresh, row.resourceRefresh, row.stalledAfter, row.finalizingAfter
*dest[6].(*string), *dest[7].(*time.Time), *dest[8].(**time.Time), *dest[9].(**time.Time), *dest[10].(*string) = row.updatedBy, row.createdAt, row.activatedAt, row.retiredAt, row.errorCode
*dest[6].(*int) = row.ingestionConcurrency
*dest[7].(*string), *dest[8].(*time.Time), *dest[9].(**time.Time), *dest[10].(**time.Time), *dest[11].(*string) = row.updatedBy, row.createdAt, row.activatedAt, row.retiredAt, row.errorCode
return nil
}