feat(governance): policy-driven ingestion from catalog annotations

Annotation in 来源与目录 is now the only import decision surface:
- migration 000033: catalog_import_intents (durable, rescan-proof per-file
  intent with locator snapshot + fetch lifecycle), source_items.policy_state,
  planned publication state, ui_setting_revisions.ingestion_concurrency
- marking fans a folder decision out to per-file intents, materializes
  versioned items, reuses already-verified local bytes (compare, don't pull),
  and plans one publication per routed KB; blocked retires planned rows
  while live publications stay untouched (需下架 shows in the KB diff)
- artifact.fetch worker lane (scripts/artifact_fetch.py): DingTalk export
  tree/live box API, AList /api/fs/get direct download with SSO-redirect
  fail-closed, PP ledger via locator hash; content-level dedupe on sha256
- 开始处理 promotes planned->pending inside the claim-time concurrency budget
  from system settings; claims enforce it in SQL
- monitor overview rebuilt as knowledge-base diff: planned/needs_removal per
  KB + fetch pipeline summary; frontend lanes 待入库/需下架 replace the
  deprecated snapshot change-set confirmation
- spool: images (png/jpg/gif/bmp/webp) accepted, ppt/pptx rejected in favour
  of PDF twins; DingTalk PPT resolves its converted PDF pair
This commit is contained in:
姚章浩
2026-09-13 22:17:12 +08:00
parent f6cb90cd70
commit 63f3b2c503
38 changed files with 1597 additions and 307 deletions
+1
View File
@@ -28,3 +28,4 @@
/tmp_*
.style-backup-*/
.hermes/
__pycache__/
+1 -1
View File
@@ -18,6 +18,6 @@ RUN apk add --no-cache ca-certificates python3 postgresql-client \
&& adduser -D -H -u 65532 -G governance governance
WORKDIR /app
COPY --from=build /out/ /app/
COPY scripts/source_catalog_import.py scripts/live_catalog_refresh.py /app/scripts/
COPY scripts/source_catalog_import.py scripts/live_catalog_refresh.py scripts/artifact_fetch.py /app/scripts/
USER 65532:65532
CMD ["/app/api"]
+26
View File
@@ -6,6 +6,7 @@ import (
"log"
"os"
"os/signal"
"path/filepath"
"syscall"
"github.com/jackc/pgx/v5/pgxpool"
@@ -14,6 +15,7 @@ import (
"knowledge-governance/internal/jobs"
postgresstore "knowledge-governance/internal/platform/postgres"
"knowledge-governance/internal/platform/weknora"
artifactfetch "knowledge-governance/internal/workflows/artifactfetch"
catalogrefresh "knowledge-governance/internal/workflows/catalogrefresh"
"knowledge-governance/internal/workflows/sourcescan"
"knowledge-governance/internal/workflows/testweknora"
@@ -83,6 +85,30 @@ func workerHandlers(store *postgresstore.Store, config appconfig.Config) (map[st
return nil, fmt.Errorf("create catalog refresh handler: %w", err)
}
handlers["source.catalog.refresh"] = catalogHandler.Handle
ingestionRoot := config.IngestionSpoolRoot
if ingestionRoot == "" {
ingestionRoot = config.GovernanceSpoolRoot
}
if !filepath.IsAbs(ingestionRoot) || ingestionRoot == "" {
ingestionRoot = "/var/lib/governance/ingestion"
}
fetchMaxBytes := config.ArtifactFetchMaxBytes
if config.TestWeKnoraEnabled && config.TestWeKnoraMaxUploadBytes > 0 && fetchMaxBytes > config.TestWeKnoraMaxUploadBytes {
fetchMaxBytes = config.TestWeKnoraMaxUploadBytes
}
if fetchMaxBytes <= 0 {
fetchMaxBytes = 50 * 1024 * 1024
}
artifactRunner, err := artifactfetch.NewCommandRunner("/usr/bin/python3", "/app/scripts/artifact_fetch.py",
ingestionRoot, "ingestion", fetchMaxBytes)
if err != nil {
return nil, fmt.Errorf("create artifact fetch runner: %w", err)
}
artifactHandler, err := artifactfetch.NewHandler(artifactfetch.Config{Store: store, Runner: artifactRunner})
if err != nil {
return nil, fmt.Errorf("create artifact fetch handler: %w", err)
}
handlers["artifact.fetch"] = artifactHandler.Handle
if config.PPScanEnabled {
scanner, err := sourcescan.NewFilePPScanner(sourcescan.FilePPScannerConfig{
AllowedRoot: config.PPScanAllowedRoot,
+2
View File
@@ -80,6 +80,7 @@ services:
TEST_WEKNORA_KB_ID: "${TEST_WEKNORA_KB_ID:-}"
TEST_WEKNORA_API_KEY_FILE: /run/secrets/test_weknora_api_key
GOVERNANCE_SPOOL_ROOT: /var/lib/governance/spool
INGESTION_SPOOL_ROOT: /var/lib/governance/ingestion
TEST_WEKNORA_HTTP_TIMEOUT: "${TEST_WEKNORA_HTTP_TIMEOUT:-60s}"
TEST_WEKNORA_POLL_INITIAL: "${TEST_WEKNORA_POLL_INITIAL:-10s}"
TEST_WEKNORA_POLL_MAX: "${TEST_WEKNORA_POLL_MAX:-60s}"
@@ -91,6 +92,7 @@ services:
- test_weknora_api_key
volumes:
- ./spool:/var/lib/governance/spool:ro
- ./spool/ingestion:/var/lib/governance/ingestion
- "${PP_SCAN_HOST_DATA_ROOT:-/home/zheromiku/dobot-portal-sync/data}:${PP_SCAN_CONTAINER_DATA_ROOT:-/home/zheromiku/dobot-portal-sync/data}:ro"
- /home/zheromiku/dingtalk_kb_download:/sources/dingtalk:ro
- /home/zheromiku/tmp/alist:/sources/alist:ro
+4
View File
@@ -36,6 +36,8 @@ type Config struct {
ExternalRoutingKnowledgeBaseID string
TestWeKnoraAPIKeyFile string
GovernanceSpoolRoot string
IngestionSpoolRoot string
ArtifactFetchMaxBytes int64
WorkerLivenessStaleAfter time.Duration
TestWeKnoraHTTPTimeout time.Duration
TestWeKnoraPollInitial time.Duration
@@ -67,6 +69,8 @@ func Load(getenv func(string) string) (Config, error) {
ExternalRoutingKnowledgeBaseID: strings.TrimSpace(getenv("TEST_EXTERNAL_WEKNORA_KB_ID")),
TestWeKnoraAPIKeyFile: valueOrDefault(getenv("TEST_WEKNORA_API_KEY_FILE"), "/run/secrets/test_weknora_api_key"),
GovernanceSpoolRoot: valueOrDefault(getenv("GOVERNANCE_SPOOL_ROOT"), "/var/lib/governance/spool"),
IngestionSpoolRoot: strings.TrimSpace(getenv("INGESTION_SPOOL_ROOT")),
ArtifactFetchMaxBytes: 200 * 1024 * 1024,
WorkerLivenessStaleAfter: 2 * time.Minute,
TestWeKnoraHTTPTimeout: 60 * time.Second,
TestWeKnoraPollInitial: 10 * time.Second,
+13
View File
@@ -51,12 +51,24 @@ type KnowledgeBaseProgress struct {
Finalizing int `json:"finalizing"`
Failed int `json:"failed"`
Cancelled int `json:"cancelled"`
Planned int `json:"planned"`
NeedsRemoval int `json:"needs_removal"`
StalePlanned int `json:"stale_planned"`
InternalImported int `json:"internal_imported"`
InternalCompleted int `json:"internal_completed"`
ExternalImported int `json:"external_imported"`
ExternalCompleted int `json:"external_completed"`
}
// IngestionPipeline summarizes the artifact-fetch lane that sits in front of
// 待入库 so operators can see why planned items are not uploadable yet.
type IngestionPipeline struct {
FetchPending int `json:"fetch_pending"`
FetchFailed int `json:"fetch_failed"`
FetchMissing int `json:"fetch_missing"`
Concurrency int `json:"concurrency"`
}
// JobWindowMetrics reports a frozen statistical window; insufficient samples
// stay null and are never rendered as 0.
type JobWindowMetrics struct {
@@ -95,6 +107,7 @@ type MonitorOverview struct {
Host MonitorHost `json:"host"`
Services []MonitorService `json:"services"`
KnowledgeBases []KnowledgeBaseProgress `json:"knowledge_bases"`
Pipeline IngestionPipeline `json:"pipeline"`
Health []ServiceHealth `json:"health"`
}
+22 -8
View File
@@ -94,15 +94,29 @@ type CatalogRouteRequest struct {
Now time.Time
}
// CatalogRouteResult describes the desired-state write only. S7 guarantees the
// annotation never creates publications or jobs; counts explain what the
// subsequent ChangeSet/P3 pipeline will be able to act on.
// CatalogRouteFile is one selected file plus the metadata snapshot the policy
// fulfillment path needs to materialize and fetch it later.
type CatalogRouteFile struct {
NodeID string
Name string
RelativePath string
SizeBytes *int64
ModifiedAt *time.Time
ContentSHA256 *string
LocatorHash string
Direct bool
}
// CatalogRouteResult describes the desired-state write only. Policy fulfillment
// records intents, queues artifact fetches and plans publications; counts tell
// the operator what will show up in 解析监控.
type CatalogRouteResult struct {
SelectedFiles int `json:"selected_files"`
RoutableFiles int `json:"routable_files"`
UnboundFiles int `json:"unbound_files"`
HashUnknownFiles int `json:"hash_unknown_files"`
UnavailableFiles int `json:"unavailable_files"`
SelectedFiles int `json:"selected_files"`
IntentsRecorded int `json:"intents_recorded"`
FetchQueued int `json:"fetch_queued"`
PlannedPublications int `json:"planned_publications"`
BlockedItems int `json:"blocked_items"`
UnboundFiles int `json:"unbound_files"`
}
type CatalogChangeSetFilter struct {
+3
View File
@@ -12,6 +12,7 @@ type UISettings struct {
ResourceRefreshSeconds int `json:"resource_refresh_seconds"`
StalledAfterSeconds int `json:"stalled_after_seconds"`
FinalizingAfterSeconds int `json:"finalizing_after_seconds"`
IngestionConcurrency int `json:"ingestion_concurrency"`
UpdatedBy string `json:"updated_by"`
CreatedAt time.Time `json:"created_at"`
ActivatedAt *time.Time `json:"activated_at"`
@@ -24,6 +25,7 @@ type CreateUISettingsCandidateRequest struct {
ResourceRefreshSeconds int
StalledAfterSeconds int
FinalizingAfterSeconds int
IngestionConcurrency int
UpdatedBy string
Now time.Time
}
@@ -33,6 +35,7 @@ func (request CreateUISettingsCandidateRequest) Valid() bool {
oneOfInt(request.ResourceRefreshSeconds, 30, 60) &&
oneOfInt(request.StalledAfterSeconds, 1800, 3600) &&
oneOfInt(request.FinalizingAfterSeconds, 7200, 14400) &&
request.IngestionConcurrency >= 1 && request.IngestionConcurrency <= 16 &&
validSettingsActor(request.UpdatedBy) && !request.Now.IsZero()
}
+2
View File
@@ -1139,6 +1139,7 @@ type createUISettingsCandidateRequest struct {
ResourceRefreshSeconds int `json:"resource_refresh_seconds"`
StalledAfterSeconds int `json:"stalled_after_seconds"`
FinalizingAfterSeconds int `json:"finalizing_after_seconds"`
IngestionConcurrency int `json:"ingestion_concurrency"`
}
func getActiveUISettings(w http.ResponseWriter, r *http.Request, store Store) {
@@ -1169,6 +1170,7 @@ func createUISettingsCandidate(w http.ResponseWriter, r *http.Request, config Co
ResourceRefreshSeconds: request.ResourceRefreshSeconds,
StalledAfterSeconds: request.StalledAfterSeconds,
FinalizingAfterSeconds: request.FinalizingAfterSeconds,
IngestionConcurrency: request.IngestionConcurrency,
UpdatedBy: actorFor(r),
Now: config.Now(),
}, operationID)
+7 -7
View File
@@ -348,7 +348,7 @@ func TestHandlerRejectsInvalidCatalogParentBeforeStore(t *testing.T) {
func TestHandlerRoutesSelectedCatalogFilesToServerBoundScopeKB(t *testing.T) {
now := time.Date(2026, 9, 9, 1, 0, 0, 0, time.UTC)
store := &fakeJobStore{catalogRouteResult: governance.CatalogRouteResult{SelectedFiles: 2, RoutableFiles: 2}}
store := &fakeJobStore{catalogRouteResult: governance.CatalogRouteResult{SelectedFiles: 2, PlannedPublications: 2}}
handler, err := NewHandler(Config{Store: store, Now: func() time.Time { return now }, InternalRoutingKnowledgeBaseID: "kb-internal", ExternalRoutingKnowledgeBaseID: "kb-external"})
if err != nil {
t.Fatal(err)
@@ -360,7 +360,7 @@ func TestHandlerRoutesSelectedCatalogFilesToServerBoundScopeKB(t *testing.T) {
if response.Code != http.StatusAccepted || store.catalogRouteRequest.TargetKnowledgeBaseID != "kb-internal" || store.catalogRouteRequest.Now != now || store.catalogRouteRequest.Visibility != governance.VisibilityInternal {
t.Fatalf("status=%d request=%#v body=%s", response.Code, store.catalogRouteRequest, response.Body.String())
}
if strings.Contains(response.Body.String(), "kb-internal") || !strings.Contains(response.Body.String(), `"routable_files":2`) {
if strings.Contains(response.Body.String(), "kb-internal") || !strings.Contains(response.Body.String(), `"planned_publications":2`) {
t.Fatalf("unsafe route response=%s", response.Body.String())
}
capabilities := httptest.NewRecorder()
@@ -372,7 +372,7 @@ func TestHandlerRoutesSelectedCatalogFilesToServerBoundScopeKB(t *testing.T) {
func TestHandlerRoutesBothScopesInOneAtomicRequest(t *testing.T) {
now := time.Date(2026, 9, 9, 1, 30, 0, 0, time.UTC)
store := &fakeJobStore{catalogRouteResult: governance.CatalogRouteResult{SelectedFiles: 1, RoutableFiles: 1}}
store := &fakeJobStore{catalogRouteResult: governance.CatalogRouteResult{SelectedFiles: 1, PlannedPublications: 1}}
handler, err := NewHandler(Config{Store: store, Now: func() time.Time { return now }, InternalRoutingKnowledgeBaseID: "kb-internal", ExternalRoutingKnowledgeBaseID: "kb-external"})
if err != nil {
t.Fatal(err)
@@ -393,7 +393,7 @@ func TestHandlerRoutesBothScopesInOneAtomicRequest(t *testing.T) {
}
func TestHandlerAcceptsBlockedPolicyWithoutAConfiguredTarget(t *testing.T) {
store := &fakeJobStore{catalogRouteResult: governance.CatalogRouteResult{SelectedFiles: 1, RoutableFiles: 1}}
store := &fakeJobStore{catalogRouteResult: governance.CatalogRouteResult{SelectedFiles: 1, BlockedItems: 1}}
handler, err := NewHandler(Config{Store: store})
if err != nil {
t.Fatal(err)
@@ -405,7 +405,7 @@ func TestHandlerAcceptsBlockedPolicyWithoutAConfiguredTarget(t *testing.T) {
if response.Code != http.StatusAccepted || store.catalogRouteRequest.Visibility != governance.VisibilityBlocked || len(store.catalogRouteRequest.TargetKnowledgeBaseIDs) != 0 {
t.Fatalf("status=%d request=%#v body=%s", response.Code, store.catalogRouteRequest, response.Body.String())
}
if !strings.Contains(response.Body.String(), `"routable_files":1`) {
if !strings.Contains(response.Body.String(), `"blocked_items":1`) {
t.Fatalf("blocked route response=%s", response.Body.String())
}
}
@@ -952,12 +952,12 @@ func TestHandlerVersionsAndActivatesUISettings(t *testing.T) {
t.Fatalf("get status=%d body=%s", get.Code, get.Body.String())
}
noIntent := httptest.NewRecorder()
handler.ServeHTTP(noIntent, httptest.NewRequest(http.MethodPost, "/api/v1/ui-settings/candidates", bytes.NewBufferString(`{"document_refresh_seconds":30,"resource_refresh_seconds":60,"stalled_after_seconds":3600,"finalizing_after_seconds":14400}`)))
handler.ServeHTTP(noIntent, httptest.NewRequest(http.MethodPost, "/api/v1/ui-settings/candidates", bytes.NewBufferString(`{"document_refresh_seconds":30,"resource_refresh_seconds":60,"stalled_after_seconds":3600,"finalizing_after_seconds":14400,"ingestion_concurrency":2}`)))
if noIntent.Code != http.StatusForbidden {
t.Fatalf("missing intent status=%d body=%s", noIntent.Code, noIntent.Body.String())
}
create := httptest.NewRecorder()
createRequest := httptest.NewRequest(http.MethodPost, "/api/v1/ui-settings/candidates", bytes.NewBufferString(`{"document_refresh_seconds":30,"resource_refresh_seconds":60,"stalled_after_seconds":3600,"finalizing_after_seconds":14400}`))
createRequest := httptest.NewRequest(http.MethodPost, "/api/v1/ui-settings/candidates", bytes.NewBufferString(`{"document_refresh_seconds":30,"resource_refresh_seconds":60,"stalled_after_seconds":3600,"finalizing_after_seconds":14400,"ingestion_concurrency":2}`))
createRequest.Header.Set("X-Governance-Action", "ui-settings")
createRequest.Header.Set("Idempotency-Key", "op-uis-create")
handler.ServeHTTP(create, createRequest)
+27 -11
View File
@@ -82,17 +82,24 @@ func (provider *RealProvider) Overview(ctx context.Context) (governance.MonitorO
)
rows, err := provider.db.Query(ctx, `SELECT p.test_kb_id,
CASE WHEN COUNT(DISTINCT p.target_scope)=1 THEN MIN(p.target_scope) ELSE 'mixed' END,
COUNT(*)::integer,
COUNT(*) FILTER (WHERE p.processing_state='completed')::integer,
COUNT(*) FILTER (WHERE p.processing_state='pending')::integer,
COUNT(*) FILTER (WHERE p.processing_state='processing')::integer,
COUNT(*) FILTER (WHERE p.processing_state='finalizing')::integer,
COUNT(*) FILTER (WHERE p.processing_state IN ('failed','stalled'))::integer,
COUNT(*) FILTER (WHERE p.target_scope='internal')::integer,
COUNT(*) FILTER (WHERE p.target_scope='internal' AND p.processing_state='completed')::integer,
COUNT(*) FILTER (WHERE p.target_scope='external')::integer,
COUNT(*) FILTER (WHERE p.target_scope='external' AND p.processing_state='completed')::integer
COUNT(*) FILTER (WHERE p.superseded_at IS NULL)::integer,
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.processing_state='completed')::integer,
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.processing_state='pending')::integer,
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.processing_state='processing')::integer,
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.processing_state='finalizing')::integer,
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.processing_state IN ('failed','stalled'))::integer,
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.target_scope='internal')::integer,
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.target_scope='internal' AND p.processing_state='completed')::integer,
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.target_scope='external')::integer,
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.target_scope='external' AND p.processing_state='completed')::integer,
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.processing_state='planned')::integer,
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.retrieval_state='enabled'
AND item.policy_state='blocked')::integer,
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.processing_state='planned'
AND item.policy_state='blocked')::integer
FROM test_publications p
JOIN source_item_versions version ON version.id=p.source_item_version_id
JOIN source_items item ON item.id=version.source_item_id
GROUP BY p.test_kb_id
ORDER BY p.test_kb_id`)
if err != nil {
@@ -101,7 +108,8 @@ ORDER BY p.test_kb_id`)
defer rows.Close()
for rows.Next() {
var item governance.KnowledgeBaseProgress
if err := rows.Scan(&item.ID, &item.Visibility, &item.Imported, &item.Completed, &item.Pending, &item.Processing, &item.Finalizing, &item.Failed, &item.InternalImported, &item.InternalCompleted, &item.ExternalImported, &item.ExternalCompleted); err != nil {
if err := rows.Scan(&item.ID, &item.Visibility, &item.Imported, &item.Completed, &item.Pending, &item.Processing, &item.Finalizing, &item.Failed, &item.InternalImported, &item.InternalCompleted, &item.ExternalImported, &item.ExternalCompleted,
&item.Planned, &item.NeedsRemoval, &item.StalePlanned); err != nil {
return governance.MonitorOverview{}, fmt.Errorf("scan monitored knowledge base: %w", err)
}
switch item.Visibility {
@@ -117,6 +125,14 @@ ORDER BY p.test_kb_id`)
if err := rows.Err(); err != nil {
return governance.MonitorOverview{}, fmt.Errorf("iterate monitored knowledge bases: %w", err)
}
if err := provider.db.QueryRow(ctx, `SELECT
COUNT(*) FILTER (WHERE fetch_state IN ('pending','fetching'))::integer,
COUNT(*) FILTER (WHERE fetch_state='failed')::integer,
COUNT(*) FILTER (WHERE fetch_state='missing_local')::integer,
COALESCE(MAX((SELECT ingestion_concurrency FROM ui_setting_revisions WHERE state='active' LIMIT 1)),2)::integer
FROM catalog_import_intents`).Scan(&overview.Pipeline.FetchPending, &overview.Pipeline.FetchFailed, &overview.Pipeline.FetchMissing, &overview.Pipeline.Concurrency); err != nil {
return governance.MonitorOverview{}, fmt.Errorf("summarize ingestion pipeline: %w", err)
}
return overview, nil
}
@@ -0,0 +1,39 @@
package postgres
import (
"context"
"errors"
"fmt"
"github.com/jackc/pgx/v5"
"knowledge-governance/internal/workflows/artifactfetch"
)
// LoadArtifactFetchManifest resolves one pending intent plus its live source
// coordinates for the fetch script, keyed by the connection-stable node id.
func (s *Store) LoadArtifactFetchManifest(ctx context.Context, connectionID, nodeID string) (artifactfetch.Manifest, error) {
return loadArtifactFetchManifest(ctx, s.db, connectionID, nodeID)
}
func loadArtifactFetchManifest(ctx context.Context, db interface {
QueryRow(context.Context, string, ...any) pgx.Row
}, connectionID, nodeID string) (artifactfetch.Manifest, error) {
var manifest artifactfetch.Manifest
err := db.QueryRow(ctx, `SELECT connection.source_type,connection.name,intent.relative_path,intent.name,
intent.size_bytes,intent.modified_at,intent.locator_hash
FROM catalog_import_intents intent
JOIN source_connections connection ON connection.id=intent.source_connection_id
WHERE intent.source_connection_id=$1 AND intent.node_id=$2`, connectionID, nodeID).
Scan(&manifest.SourceType, &manifest.SourceName, &manifest.RelativePath, &manifest.Name,
&manifest.SizeBytes, &manifest.ModifiedAt, &manifest.LocatorHash)
if errors.Is(err, pgx.ErrNoRows) {
return artifactfetch.Manifest{}, artifactfetch.ErrManifestGone
}
if err != nil {
return artifactfetch.Manifest{}, fmt.Errorf("load artifact fetch manifest: %w", err)
}
manifest.ConnectionID = connectionID
manifest.NodeID = nodeID
return manifest, nil
}
+7 -55
View File
@@ -112,7 +112,8 @@ FOR UPDATE OF node`, request.ConnectionID, nodeIDs)
JOIN source_catalog_nodes child ON child.snapshot_id=parent.snapshot_id AND child.parent_node_id=parent.node_id
WHERE parent.node_type='folder'
)
SELECT DISTINCT node.node_id,node.source_item_id,(node.node_id=ANY($2::text[])) AS directly_selected
SELECT DISTINCT node.node_id,node.name,node.relative_path,node.size_bytes,node.modified_at,node.content_sha256,
node.source_locator_hash,(node.node_id=ANY($2::text[])) AS directly_selected
FROM descendants
JOIN source_catalog_nodes node ON node.snapshot_id=descendants.snapshot_id AND node.node_id=descendants.node_id
WHERE node.node_type='file'
@@ -120,15 +121,11 @@ ORDER BY node.node_id`, request.ConnectionID, nodeIDs)
if err != nil {
return governance.CatalogRouteResult{}, fmt.Errorf("select catalog route files: %w", err)
}
type selectedFile struct {
nodeID string
sourceItemID *string
direct bool
}
files := make([]selectedFile, 0, len(nodeIDs))
files := make([]governance.CatalogRouteFile, 0, len(nodeIDs))
for rows.Next() {
var file selectedFile
if err := rows.Scan(&file.nodeID, &file.sourceItemID, &file.direct); err != nil {
var file governance.CatalogRouteFile
if err := rows.Scan(&file.NodeID, &file.Name, &file.RelativePath, &file.SizeBytes, &file.ModifiedAt,
&file.ContentSHA256, &file.LocatorHash, &file.Direct); err != nil {
rows.Close()
return governance.CatalogRouteResult{}, fmt.Errorf("scan catalog route file: %w", err)
}
@@ -139,52 +136,7 @@ ORDER BY node.node_id`, request.ConnectionID, nodeIDs)
return governance.CatalogRouteResult{}, fmt.Errorf("iterate catalog route files: %w", err)
}
rows.Close()
result := governance.CatalogRouteResult{SelectedFiles: len(files)}
routedItems := map[string]bool{}
for _, file := range files {
if file.sourceItemID == nil {
result.UnboundFiles++
continue
}
itemID := *file.sourceItemID
if routedItems[itemID] {
continue
}
routedItems[itemID] = true
if file.direct {
if _, err := tx.Exec(ctx, `INSERT INTO source_item_scope_overrides(source_item_id,visibility,updated_by,reason,created_at,updated_at)
VALUES($1,$2,$3,$4,$5,$5) ON CONFLICT(source_item_id) DO UPDATE SET visibility=EXCLUDED.visibility,updated_by=EXCLUDED.updated_by,reason=EXCLUDED.reason,updated_at=EXCLUDED.updated_at`, itemID, request.Visibility, request.OperatorRef, request.Reason, request.Now); err != nil {
return governance.CatalogRouteResult{}, fmt.Errorf("set routed item scope: %w", err)
}
eventID, err := newOpaqueID("se-")
if err != nil {
return governance.CatalogRouteResult{}, err
}
if _, err := tx.Exec(ctx, `INSERT INTO scope_change_events(id,target_type,source_item_id,action,decided_visibility,operator_ref,reason,created_at)
VALUES($1,'item',$2,'set',$3,$4,$5,$6)`, eventID, itemID, request.Visibility, request.OperatorRef, request.Reason, request.Now); err != nil {
return governance.CatalogRouteResult{}, fmt.Errorf("audit routed item scope: %w", err)
}
}
// S7: classify only. Annotations never create publications, retire targets, or enqueue jobs.
var currentVersionID *string
var contentState, spoolRef string
err = tx.QueryRow(ctx, `SELECT version.id,content.content_state,COALESCE(content.local_spool_ref,'') FROM source_items item
LEFT JOIN source_item_versions version ON version.source_item_id=item.id AND version.revision=item.revision
LEFT JOIN content_objects content ON content.id=version.content_object_id
WHERE item.id=$1`, itemID).Scan(&currentVersionID, &contentState, &spoolRef)
switch {
case errors.Is(err, pgx.ErrNoRows):
result.UnavailableFiles++
case err != nil:
return governance.CatalogRouteResult{}, fmt.Errorf("inspect routed artifact: %w", err)
case contentState == string(governance.ContentVerified) && spoolRef != "":
result.RoutableFiles++
default:
result.HashUnknownFiles++
}
}
return result, nil
return fulfillCatalogIntentsTx(ctx, tx, request, files, request.Now)
}
func setCatalogDirectoryPolicy(ctx context.Context, tx pgx.Tx, request governance.CatalogRouteRequest, directoryPath string) error {
@@ -10,9 +10,10 @@ import (
"knowledge-governance/internal/governance"
)
// S7: catalog annotation persists ONLY desired state (scope rules/overrides + audit).
// It never creates or mutates test_publications and never enqueues jobs.
func TestRouteCatalogFilesWritesDesiredStateWithoutSideEffects(t *testing.T) {
// Policy-driven ingestion: saving an annotation records durable intents,
// materializes items, enqueues artifact fetches and plans publications.
// The only human action left is 开始处理; marking is the decision.
func TestRouteCatalogFilesFulfillsPolicyIntent(t *testing.T) {
databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL"))
if databaseURL == "" {
t.Skip("TEST_DATABASE_URL is not set")
@@ -46,66 +47,70 @@ INSERT INTO source_catalog_heads(source_connection_id,snapshot_id,activated_at)
}
store := NewStore(pool)
now := time.Date(2026, 9, 9, 1, 0, 0, 0, time.UTC)
// Folder annotation fans out to files; the bound file already holds verified
// bytes, so it is planned without any fetch (本地已有 → 比对复用 → 不拉取).
result, err := store.RouteCatalogFiles(ctx, governance.CatalogRouteRequest{ConnectionID: "conn-route", NodeIDs: []string{"folder-route"}, Visibility: governance.VisibilityInternal, TargetKnowledgeBaseID: "kb-internal", OperatorRef: "engineer-1", Reason: "整目录标注", Now: now})
if err != nil {
t.Fatal(err)
}
if result.SelectedFiles != 1 || result.RoutableFiles != 1 || result.UnboundFiles != 0 || result.HashUnknownFiles != 0 {
if result.SelectedFiles != 1 || result.IntentsRecorded != 1 || result.PlannedPublications != 1 || result.FetchQueued != 0 {
t.Fatalf("result=%#v", result)
}
// Desired state saved, recursion intact.
var directoryPolicy string
if err := pool.QueryRow(ctx, `SELECT visibility FROM directory_scope_rules WHERE source_connection_id='conn-route' AND directory_path='工业产品'`).Scan(&directoryPolicy); err != nil {
if err := pool.QueryRow(ctx, `SELECT visibility FROM directory_scope_rules WHERE source_connection_id='conn-route' AND directory_path='工业产品'`).Scan(&directoryPolicy); err != nil || directoryPolicy != "internal" {
t.Fatalf("directory policy=%q err=%v", directoryPolicy, err)
}
var intentCount, publicationCount, fetchJobs, uploadJobs int
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM catalog_import_intents WHERE source_connection_id='conn-route' AND visibility='internal'`).Scan(&intentCount); err != nil {
t.Fatal(err)
}
if directoryPolicy != "internal" {
t.Fatalf("directory policy=%q", directoryPolicy)
}
// Zero side effects: no publications, no jobs.
var publicationCount, jobCount int
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM test_publications`).Scan(&publicationCount); err != nil {
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM test_publications WHERE processing_state='planned' AND superseded_at IS NULL`).Scan(&publicationCount); err != nil {
t.Fatal(err)
}
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM jobs`).Scan(&jobCount); err != nil {
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM jobs WHERE type='artifact.fetch'`).Scan(&fetchJobs); err != nil {
t.Fatal(err)
}
if publicationCount != 0 || jobCount != 0 {
t.Fatalf("annotation must not touch pipeline: publications=%d jobs=%d", publicationCount, jobCount)
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM jobs WHERE type='test.upload'`).Scan(&uploadJobs); err != nil {
t.Fatal(err)
}
if intentCount != 1 || publicationCount != 1 || fetchJobs != 0 || uploadJobs != 0 {
t.Fatalf("intents=%d planned=%d fetch=%d upload=%d", intentCount, publicationCount, fetchJobs, uploadJobs)
}
// The planned publication binds the pre-verified current version.
var plannedVersion string
if err := pool.QueryRow(ctx, `SELECT source_item_version_id FROM test_publications WHERE test_kb_id='kb-internal'`).Scan(&plannedVersion); err != nil || plannedVersion != "version-route" {
t.Fatalf("planned version=%q err=%v", plannedVersion, err)
}
// Re-annotating the same directory is idempotent for rules and still side-effect free.
// Re-marking the same directory is replay-safe: one intent, one publication.
again, err := store.RouteCatalogFiles(ctx, governance.CatalogRouteRequest{ConnectionID: "conn-route", NodeIDs: []string{"folder-route"}, Visibility: governance.VisibilityInternal, TargetKnowledgeBaseID: "kb-internal", OperatorRef: "engineer-1", Reason: "重复提交", Now: now.Add(time.Minute)})
if err != nil || again.SelectedFiles != 1 || again.RoutableFiles != 1 {
if err != nil || again.SelectedFiles != 1 || again.PlannedPublications != 0 || again.FetchQueued != 0 {
t.Fatalf("again=%#v err=%v", again, err)
}
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM jobs`).Scan(&jobCount); err != nil || jobCount != 0 {
t.Fatalf("repeat annotation jobs=%d err=%v", jobCount, err)
}
var ruleCount int
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM directory_scope_rules WHERE source_connection_id='conn-route' AND directory_path='工业产品'`).Scan(&ruleCount); err != nil || ruleCount != 1 {
t.Fatalf("rule rows=%d err=%v", ruleCount, err)
}
// Switching policy updates desired state without retiring or removing anything.
// Widening to both plans a second KB publication; nothing is fetched again.
both, err := store.RouteCatalogFiles(ctx, governance.CatalogRouteRequest{ConnectionID: "conn-route", NodeIDs: []string{"folder-route"}, Visibility: governance.VisibilityBoth, TargetKnowledgeBaseIDs: []string{"kb-internal", "kb-external"}, OperatorRef: "engineer-1", Reason: "内外知识库同时使用", Now: now.Add(2 * time.Minute)})
if err != nil || both.SelectedFiles != 1 || both.RoutableFiles != 1 {
if err != nil || both.SelectedFiles != 1 || both.PlannedPublications != 1 || both.FetchQueued != 0 {
t.Fatalf("both=%#v err=%v", both, err)
}
if err := pool.QueryRow(ctx, `SELECT visibility FROM directory_scope_rules WHERE source_connection_id='conn-route' AND directory_path='工业产品'`).Scan(&directoryPolicy); err != nil || directoryPolicy != "both" {
t.Fatalf("policy after both=%q err=%v", directoryPolicy, err)
}
// Blocking saves the desired policy only; existing publications (if any) stay untouched.
// An unannotated live publication survives marking blocked: removal is
// knowledge-base diff (需下架) territory, never a silent side effect.
if _, err := pool.Exec(ctx, `INSERT INTO test_publications(id,source_item_version_id,test_kb_id,target_scope,ingest_state,processing_state,retrieval_state,knowledge_id,created_at,updated_at)
VALUES('tp-existing','version-route','kb-internal','internal','uploaded','completed','enabled','kn-1',$1,$1)`, now); err != nil {
t.Fatal(err)
}
blocked, err := store.RouteCatalogFiles(ctx, governance.CatalogRouteRequest{ConnectionID: "conn-route", NodeIDs: []string{"folder-route"}, Visibility: governance.VisibilityBlocked, OperatorRef: "engineer-1", Reason: "禁止导入", Now: now.Add(3 * time.Minute)})
if err != nil {
t.Fatal(err)
}
if blocked.SelectedFiles != 1 {
t.Fatalf("blocked=%#v", blocked)
if err != nil || blocked.SelectedFiles != 1 || blocked.BlockedItems != 1 {
t.Fatalf("blocked=%#v err=%v", blocked, err)
}
var retrievalState string
var removalJobs int
@@ -118,39 +123,36 @@ VALUES('tp-existing','version-route','kb-internal','internal','uploaded','comple
if retrievalState != "enabled" || removalJobs != 0 {
t.Fatalf("blocked annotation must not remove: state=%s removal jobs=%d", retrievalState, removalJobs)
}
// The planned kb-external row created while policy was 'both' is retired.
var stillPlanned int
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM test_publications WHERE processing_state='planned' AND superseded_at IS NULL AND test_kb_id='kb-external'`).Scan(&stillPlanned); err != nil || stillPlanned != 0 {
t.Fatalf("blocked must retire planned rows=%d err=%v", stillPlanned, err)
}
if err := pool.QueryRow(ctx, `SELECT visibility FROM directory_scope_rules WHERE source_connection_id='conn-route' AND directory_path='工业产品'`).Scan(&directoryPolicy); err != nil || directoryPolicy != "blocked" {
t.Fatalf("blocked policy=%q err=%v", directoryPolicy, err)
}
// Unbound file and hash-unknown file are reported, never planned.
unbound, err := store.RouteCatalogFiles(ctx, governance.CatalogRouteRequest{ConnectionID: "conn-route", NodeIDs: []string{"file-unmapped"}, Visibility: governance.VisibilityExternal, TargetKnowledgeBaseID: "kb-external", OperatorRef: "engineer-1", Reason: "测试未映射", Now: now})
if err != nil || unbound.SelectedFiles != 1 || unbound.RoutableFiles != 0 || unbound.UnboundFiles != 1 {
t.Fatalf("unbound=%#v err=%v", unbound, err)
// A never-fetched file becomes an intent with a materialized item, a
// fetch job and a planned publication (未绑定 → 从源头拉,不再是死路).
restored, err := store.RouteCatalogFiles(ctx, governance.CatalogRouteRequest{ConnectionID: "conn-route", NodeIDs: []string{"file-unmapped"}, Visibility: governance.VisibilityInternal, TargetKnowledgeBaseID: "kb-internal", OperatorRef: "engineer-1", Reason: "重新标注", Now: now.Add(4 * time.Minute)})
if err != nil || restored.IntentsRecorded != 1 || restored.FetchQueued != 1 || restored.PlannedPublications != 1 {
t.Fatalf("restored=%#v err=%v", restored, err)
}
if _, err := pool.Exec(ctx, `INSERT INTO content_objects(id,sha256,size_bytes,local_spool_ref,content_state) VALUES('content-pending',repeat('e',64),8,'','unknown')`); err != nil {
var fetchState, itemPolicy string
if err := pool.QueryRow(ctx, `SELECT intent.fetch_state||'|'||item.policy_state FROM catalog_import_intents intent
JOIN source_items item ON item.id=intent.source_item_id
WHERE intent.node_id='file-unmapped'`).Scan(&fetchState, &itemPolicy); err != nil {
t.Fatal(err)
}
if _, err := pool.Exec(ctx, `INSERT INTO source_items(id,source_connection_id,stable_external_id,canonical_uri_hash,normalized_path,title,item_type,revision,visibility,visibility_origin)
VALUES('item-pending','conn-route','external-pending',repeat('f',64),'待校验.pdf','待校验.pdf','document','v1','external','pp_public_tag')`); err != nil {
t.Fatal(err)
if fetchState != "pending|internal" {
t.Fatalf("fetch state=%q", fetchState)
}
if _, err := pool.Exec(ctx, `INSERT INTO source_item_versions(id,source_item_id,revision,content_object_id,content_state) VALUES('version-pending','item-pending','v1','content-pending','unknown')`); err != nil {
t.Fatal(err)
var versionState string
if err := pool.QueryRow(ctx, `SELECT version.content_state FROM source_item_versions version
JOIN source_items item ON item.id=version.source_item_id AND item.revision=version.revision
WHERE item.stable_external_id='file-unmapped'`).Scan(&versionState); err != nil || versionState != "unknown" {
t.Fatalf("pending version state=%q err=%v", versionState, err)
}
if _, err := pool.Exec(ctx, `INSERT INTO source_catalog_nodes(snapshot_id,node_id,parent_node_id,node_type,name,relative_path,source_locator_hash,source_item_id,size_bytes,direct_child_count,subtree_file_count,subtree_bytes)
VALUES('snapshot-route','file-pending','root-route','file','待校验.pdf','待校验.pdf',repeat('d',64),'item-pending',8,0,1,8)`); err != nil {
t.Fatal(err)
}
hashUnknown, err := store.RouteCatalogFiles(ctx, governance.CatalogRouteRequest{ConnectionID: "conn-route", NodeIDs: []string{"file-pending"}, Visibility: governance.VisibilityInternal, TargetKnowledgeBaseID: "kb-internal", OperatorRef: "engineer-1", Reason: "测试内容未验证", Now: now})
if err != nil || hashUnknown.SelectedFiles != 1 || hashUnknown.RoutableFiles != 0 || hashUnknown.HashUnknownFiles != 1 {
t.Fatalf("hash unknown=%#v err=%v", hashUnknown, err)
}
// Overrides are still recorded for unbound/pending items so the desired state survives.
var overrideCount int
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM source_item_scope_overrides WHERE source_item_id='item-pending'`).Scan(&overrideCount); err != nil || overrideCount != 1 {
t.Fatalf("pending override rows=%d err=%v", overrideCount, err)
}
// Audit trail exists for every annotation action.
var events int
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM scope_change_events`).Scan(&events); err != nil || events < 5 {
t.Fatalf("scope events=%d err=%v", events, err)
@@ -46,7 +46,7 @@ INSERT INTO source_catalog_heads(source_connection_id,snapshot_id,activated_at)
now := time.Date(2026, 9, 11, 6, 0, 0, 0, time.UTC)
request := governance.CatalogRouteRequest{ConnectionID: "conn-op", NodeIDs: []string{"file-op"}, Visibility: governance.VisibilityInternal, TargetKnowledgeBaseID: "kb-internal", OperatorRef: "engineer-1", Reason: "初次标注", Now: now}
first, replayed, err := store.RouteCatalogFilesWithOperation(ctx, request, "op-route-1")
if err != nil || replayed || first.RoutableFiles != 1 {
if err != nil || replayed || first.PlannedPublications != 1 {
t.Fatalf("first result=%#v replayed=%v error=%v", first, replayed, err)
}
second, replayed, err := store.RouteCatalogFilesWithOperation(ctx, request, "op-route-1")
@@ -60,8 +60,8 @@ INSERT INTO source_catalog_heads(source_connection_id,snapshot_id,activated_at)
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM operation_requests WHERE operation_type='catalog.route' AND operation_id='op-route-1'`).Scan(&operations); err != nil {
t.Fatal(err)
}
if publications != 0 || operations != 1 {
t.Fatalf("S7 annotation must not publish: publications=%d operations=%d", publications, operations)
if publications != 1 || operations != 1 {
t.Fatalf("annotation must plan exactly one replay-safe publication: publications=%d operations=%d", publications, operations)
}
conflicting := request
conflicting.Reason = "不同的操作内容"
@@ -0,0 +1,9 @@
ALTER TABLE ui_setting_revisions DROP COLUMN IF EXISTS ingestion_concurrency;
DROP INDEX IF EXISTS idx_test_publications_planned;
ALTER TABLE test_publications DROP CONSTRAINT IF EXISTS test_publications_processing_state_check;
ALTER TABLE test_publications ADD CONSTRAINT test_publications_processing_state_check
CHECK (processing_state IN ('pending', 'processing', 'finalizing', 'completed', 'failed', 'stalled'));
DROP INDEX IF EXISTS idx_source_items_policy;
ALTER TABLE source_items DROP COLUMN IF EXISTS policy_state;
DROP INDEX IF EXISTS idx_catalog_intents_fetch;
DROP TABLE IF EXISTS catalog_import_intents;
@@ -0,0 +1,62 @@
-- Policy-driven ingestion: the ONLY import decision surface is the catalog
-- annotation (internal / external / both / blocked). Marking a file or folder
-- records a durable per-file INTENT that survives rescans; unannotated files
-- stay "unset" and are never ingested (cold start imports nothing).
-- A saved intent immediately materializes a versioned source item and an
-- artifact.fetch job so the artifact is pulled from the live source (or
-- matched against an already-present local copy); once content is verified a
-- 'planned' publication appears in the knowledge-base diff as 待入库. The one
-- human batch action "开始处理" promotes planned -> queued within the
-- concurrency budget configured in system settings.
-- Per-file decision, keyed by the connection-stable catalog node id
-- (node ids are derived from source_key + relative path, identical across
-- snapshots, so intents are rescan-proof).
CREATE TABLE IF NOT EXISTS catalog_import_intents (
source_connection_id VARCHAR(36) NOT NULL
REFERENCES source_connections(id) ON DELETE CASCADE,
node_id VARCHAR(64) NOT NULL,
visibility VARCHAR(20) NOT NULL
CHECK (visibility IN ('internal', 'external', 'both', 'blocked')),
source_item_id VARCHAR(36),
decided_by VARCHAR(128) NOT NULL,
reason VARCHAR(500) NOT NULL DEFAULT '',
-- Locator snapshot: the fetch worker needs path/name metadata without
-- re-walking a snapshot that may already have rotated.
relative_path TEXT NOT NULL DEFAULT '',
name TEXT NOT NULL DEFAULT '',
size_bytes BIGINT,
modified_at TIMESTAMPTZ,
content_sha256 CHAR(64),
locator_hash CHAR(64) NOT NULL DEFAULT '',
-- Content materialization lifecycle driven by the artifact.fetch job.
fetch_state VARCHAR(20) NOT NULL DEFAULT 'pending'
CHECK (fetch_state IN ('pending', 'fetching', 'ready', 'missing_local', 'failed')),
last_error TEXT NOT NULL DEFAULT '',
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
PRIMARY KEY (source_connection_id, node_id)
);
CREATE INDEX IF NOT EXISTS idx_catalog_intents_item
ON catalog_import_intents (source_item_id) WHERE source_item_id IS NOT NULL;
CREATE INDEX IF NOT EXISTS idx_catalog_intents_fetch
ON catalog_import_intents (fetch_state) WHERE fetch_state <> 'ready';
-- Materialized items carry the policy so list/diff queries never re-walk rules.
ALTER TABLE source_items ADD COLUMN IF NOT EXISTS policy_state VARCHAR(20) NOT NULL DEFAULT 'unset'
CHECK (policy_state IN ('unset', 'internal', 'external', 'both', 'blocked'));
CREATE INDEX IF NOT EXISTS idx_source_items_policy
ON source_items (policy_state) WHERE policy_state <> 'unset';
-- The knowledge base is the ingestion target of a planned publication; make
-- "planned" explicit so the KB-diff view never reuses queued semantics.
ALTER TABLE test_publications DROP CONSTRAINT IF EXISTS test_publications_processing_state_check;
ALTER TABLE test_publications ADD CONSTRAINT test_publications_processing_state_check
CHECK (processing_state IN ('planned', 'pending', 'processing', 'finalizing', 'completed', 'failed', 'stalled'));
CREATE INDEX IF NOT EXISTS idx_test_publications_planned
ON test_publications (test_kb_id, processing_state) WHERE processing_state = 'planned';
-- 开始处理's concurrency budget lives in system settings (validated revisions).
ALTER TABLE ui_setting_revisions ADD COLUMN IF NOT EXISTS ingestion_concurrency INTEGER NOT NULL DEFAULT 2
CHECK (ingestion_concurrency BETWEEN 1 AND 16);
@@ -11,8 +11,8 @@ func TestEmbeddedMigrationsAreVersionSorted(t *testing.T) {
if err != nil {
t.Fatalf("loadMigrations() error = %v", err)
}
if len(migrations) != 32 {
t.Fatalf("migration count = %d, want 32", len(migrations))
if len(migrations) != 33 {
t.Fatalf("migration count = %d, want 33", len(migrations))
}
if migrations[0].Version != 1 || !strings.Contains(migrations[0].UpSQL, "CREATE TABLE IF NOT EXISTS jobs") {
t.Fatalf("migration 1 = %#v", migrations[0])
@@ -567,7 +567,8 @@ func TestReleaseExecutionMigrationDefinesAuthorizedAuditableJobs(t *testing.T) {
func TestClaimSQLUsesSkipLockedAndReclaimsExpiredRunningJobs(t *testing.T) {
sql := strings.ToLower(claimJobSQL)
for _, required := range []string{
"for update skip locked",
"for update of jobs skip locked",
"ingestion_concurrency",
"state in ('pending', 'retry')",
"state = 'running'",
"lease_until <",
@@ -0,0 +1,364 @@
package postgres
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"time"
"github.com/jackc/pgx/v5"
"knowledge-governance/internal/governance"
)
// fulfillCatalogIntentsTx is the policy-driven ingestion core: saving an
// annotation in 来源与目录 records one durable intent per selected file and,
// for importable policies, materializes a versioned source item, enqueues an
// artifact.fetch job (bytes come from the live source; a matching local copy
// short-circuits the download) and creates one 'planned' publication per routed
// knowledge base. Planned publications are what 解析监控 lists as 待入库;
// 开始处理 promotes them within the configured concurrency budget. Blocked
// intents retire any planned publication and flip the item policy so the
// knowledge-base diff demands removal of anything still live.
func fulfillCatalogIntentsTx(ctx context.Context, tx pgx.Tx, request governance.CatalogRouteRequest, files []governance.CatalogRouteFile, now time.Time) (governance.CatalogRouteResult, error) {
targetKnowledgeBaseIDs := request.TargetKnowledgeBaseIDs
if len(targetKnowledgeBaseIDs) == 0 && request.TargetKnowledgeBaseID != "" {
targetKnowledgeBaseIDs = []string{request.TargetKnowledgeBaseID}
}
result := governance.CatalogRouteResult{SelectedFiles: len(files)}
for _, file := range files {
recorded, fetchQueued, planned, blocked, err := fulfillCatalogIntent(ctx, tx, request, file, targetKnowledgeBaseIDs, now)
if err != nil {
return governance.CatalogRouteResult{}, err
}
if !recorded {
result.UnboundFiles++
continue
}
result.IntentsRecorded++
if blocked {
result.BlockedItems++
continue
}
if fetchQueued {
result.FetchQueued++
}
result.PlannedPublications += planned
}
return result, nil
}
// fulfillCatalogIntent applies one file decision. It returns recorded=false
// only when the node vanished mid-transaction; fetchQueued reports a newly
// enqueued artifact.fetch job and planned the number of publications created.
func fulfillCatalogIntent(ctx context.Context, tx pgx.Tx, request governance.CatalogRouteRequest, file governance.CatalogRouteFile, targetKnowledgeBaseIDs []string, now time.Time) (bool, bool, int, bool, error) {
policy := string(request.Visibility)
if _, err := tx.Exec(ctx, `INSERT INTO catalog_import_intents(
source_connection_id,node_id,visibility,decided_by,reason,relative_path,name,size_bytes,modified_at,content_sha256,
locator_hash,fetch_state,last_error,created_at,updated_at)
VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,'pending','',NOW(),NOW())
ON CONFLICT(source_connection_id,node_id) DO UPDATE SET
visibility=EXCLUDED.visibility,decided_by=EXCLUDED.decided_by,reason=EXCLUDED.reason,
relative_path=EXCLUDED.relative_path,name=EXCLUDED.name,size_bytes=EXCLUDED.size_bytes,
modified_at=EXCLUDED.modified_at,content_sha256=EXCLUDED.content_sha256,
locator_hash=EXCLUDED.locator_hash,
fetch_state=CASE WHEN catalog_import_intents.content_sha256 IS DISTINCT FROM EXCLUDED.content_sha256
OR catalog_import_intents.size_bytes IS DISTINCT FROM EXCLUDED.size_bytes
OR catalog_import_intents.modified_at IS DISTINCT FROM EXCLUDED.modified_at
THEN 'pending' ELSE catalog_import_intents.fetch_state END,
last_error=CASE WHEN catalog_import_intents.content_sha256 IS DISTINCT FROM EXCLUDED.content_sha256
OR catalog_import_intents.size_bytes IS DISTINCT FROM EXCLUDED.size_bytes
OR catalog_import_intents.modified_at IS DISTINCT FROM EXCLUDED.modified_at
THEN '' ELSE catalog_import_intents.last_error END,
updated_at=NOW()`,
request.ConnectionID, file.NodeID, policy, request.OperatorRef, request.Reason,
file.RelativePath, file.Name, file.SizeBytes, file.ModifiedAt, file.ContentSHA256, file.LocatorHash); err != nil {
return false, false, 0, false, fmt.Errorf("record catalog intent: %w", err)
}
var existingItemID *string
err := tx.QueryRow(ctx, `SELECT source_item_id FROM catalog_import_intents
WHERE source_connection_id=$1 AND node_id=$2 FOR UPDATE`, request.ConnectionID, file.NodeID).Scan(&existingItemID)
if errors.Is(err, pgx.ErrNoRows) {
return false, false, 0, false, nil
}
if err != nil {
return false, false, 0, false, fmt.Errorf("read catalog intent: %w", err)
}
itemID := ""
if existingItemID != nil {
itemID = *existingItemID
}
if itemID == "" {
newItemID, err := newOpaqueID("si-")
if err != nil {
return false, false, 0, false, err
}
if _, err := tx.Exec(ctx, `INSERT INTO source_items(
id,source_connection_id,stable_external_id,canonical_uri_hash,title,item_type,revision,
visibility,visibility_origin,source_change,absence_streak,identity_confidence,
policy_state,created_at,updated_at)
VALUES($1,$2,$3,$4,$5,'file','',$7,'configured_scope','unchanged',0,'object_id',$6,NOW(),NOW())`,
newItemID, request.ConnectionID, "cat:"+file.NodeID,
sha256Hex("policy:"+request.ConnectionID+"\x00"+file.NodeID),
file.Name, policy, itemVisibilityFor(request.Visibility)); err != nil {
return false, false, 0, false, fmt.Errorf("materialize policy source item: %w", err)
}
if _, err := tx.Exec(ctx, `UPDATE catalog_import_intents SET source_item_id=$3 WHERE source_connection_id=$1 AND node_id=$2`,
request.ConnectionID, file.NodeID, newItemID); err != nil {
return false, false, 0, false, fmt.Errorf("bind catalog intent item: %w", err)
}
itemID = newItemID
}
if request.Visibility == governance.VisibilityBlocked {
if _, err := tx.Exec(ctx, `UPDATE source_items SET policy_state='blocked',updated_at=NOW() WHERE id=$1`, itemID); err != nil {
return false, false, 0, false, fmt.Errorf("block policy item: %w", err)
}
if _, err := tx.Exec(ctx, `UPDATE test_publications SET superseded_at=$2,updated_at=$2
WHERE source_item_version_id IN (SELECT id FROM source_item_versions WHERE source_item_id=$1)
AND superseded_at IS NULL AND processing_state='planned'`, itemID, now); err != nil {
return false, false, 0, false, fmt.Errorf("retire blocked planned publications: %w", err)
}
return true, false, 0, true, nil
}
revision := "locator:" + file.LocatorHash
if file.ContentSHA256 != nil && *file.ContentSHA256 != "" {
revision = "sha:" + *file.ContentSHA256
}
if _, err := tx.Exec(ctx, `UPDATE source_items
SET policy_state=$2,visibility=$3,visibility_origin='configured_scope',updated_at=NOW() WHERE id=$1`,
itemID, policy, itemVisibilityFor(request.Visibility)); err != nil {
return false, false, 0, false, fmt.Errorf("update policy item: %w", err)
}
// Resolve the version this intent points at. A bound item whose current
// version already carries verified bytes is reused without any fetch — the
// 本地已有先比对 requirement. Only when the node proves different content
// (scan-reported hash mismatch) or bytes were never fetched do we create a
// revision and enqueue artifact.fetch.
versionID := ""
contentState := ""
var curID, curSHA string
err = tx.QueryRow(ctx, `SELECT version.id,COALESCE(content.sha256,''),version.content_state
FROM source_items item
JOIN source_item_versions version ON version.source_item_id=item.id AND version.revision=item.revision
LEFT JOIN content_objects content ON content.id=version.content_object_id
WHERE item.id=$1`, itemID).Scan(&curID, &curSHA, &contentState)
if err != nil && !errors.Is(err, pgx.ErrNoRows) {
return false, false, 0, false, fmt.Errorf("inspect current item version: %w", err)
}
nodeSHA := ""
if file.ContentSHA256 != nil {
nodeSHA = *file.ContentSHA256
}
reuse := err == nil && contentState == string(governance.ContentVerified) && (nodeSHA == "" || nodeSHA == curSHA)
if !reuse {
contentState = "unknown"
err = tx.QueryRow(ctx, `SELECT version.id,version.content_state
FROM source_item_versions version WHERE version.source_item_id=$1 AND version.revision=$2`,
itemID, revision).Scan(&versionID, &contentState)
if errors.Is(err, pgx.ErrNoRows) {
newVersionID, err := newOpaqueID("sv-")
if err != nil {
return false, false, 0, false, err
}
if _, err := tx.Exec(ctx, `INSERT INTO source_item_versions(
id,source_item_id,revision,content_object_id,content_state,raw_record_summary,source_updated_at,created_at,updated_at)
VALUES($1,$2,$3,NULL,'unknown','{}'::jsonb,$4,NOW(),NOW())`, newVersionID, itemID, revision, file.ModifiedAt); err != nil {
return false, false, 0, false, fmt.Errorf("create policy version: %w", err)
}
if _, err := tx.Exec(ctx, `UPDATE source_items SET revision=$2,updated_at=NOW() WHERE id=$1`, itemID, revision); err != nil {
return false, false, 0, false, fmt.Errorf("advance policy item revision: %w", err)
}
versionID = newVersionID
} else if err != nil {
return false, false, 0, false, fmt.Errorf("lookup policy version: %w", err)
}
} else {
versionID = curID
}
planned := 0
for _, kbID := range targetKnowledgeBaseIDs {
scope := kbScopeFor(request, kbID)
publicationID, err := newOpaqueID("tp-")
if err != nil {
return false, false, 0, false, err
}
tag, err := tx.Exec(ctx, `INSERT INTO test_publications(
id,source_item_version_id,test_kb_id,knowledge_id,ingest_state,processing_state,retrieval_state,
parser_generation,index_generation,last_error,retry_count,target_scope,created_at,updated_at)
VALUES($1,$2,$3,'','pending','planned','unpublished','','','',0,$4,NOW(),NOW())
ON CONFLICT (source_item_version_id,test_kb_id) DO NOTHING`,
publicationID, versionID, kbID, scope)
if err != nil {
return false, false, 0, false, fmt.Errorf("plan publication: %w", err)
}
planned += int(tag.RowsAffected())
}
fetchQueued := false
if contentState != string(governance.ContentVerified) {
jobID, err := newOpaqueID("job-")
if err != nil {
return false, false, 0, false, err
}
payload, err := json.Marshal(map[string]string{"source_connection_id": request.ConnectionID, "node_id": file.NodeID, "version_id": versionID})
if err != nil {
return false, false, 0, false, errors.New("encode artifact fetch job")
}
tag, err := tx.Exec(ctx, `INSERT INTO jobs(id,type,payload,state,priority,available_at,max_attempts,idempotency_key,created_at,updated_at)
VALUES($1,'artifact.fetch',$2::jsonb,'pending',20,$3,8,$4,$3,$3) ON CONFLICT (idempotency_key) DO NOTHING`,
jobID, payload, now, "artifact-fetch:"+versionID)
if err != nil {
return false, false, 0, false, fmt.Errorf("enqueue artifact fetch: %w", err)
}
fetchQueued = tag.RowsAffected() > 0
if fetchQueued {
if _, err := tx.Exec(ctx, `UPDATE catalog_import_intents SET fetch_state='pending',last_error='',updated_at=NOW()
WHERE source_connection_id=$1 AND node_id=$2 AND fetch_state IN ('failed','missing_local','fetching')`, request.ConnectionID, file.NodeID); err != nil {
return false, false, 0, false, fmt.Errorf("reset catalog intent fetch state: %w", err)
}
}
}
return true, fetchQueued, planned, false, nil
}
// kbScopeFor maps a routed knowledge base id to its diff scope label.
func kbScopeFor(request governance.CatalogRouteRequest, knowledgeBaseID string) string {
if request.Visibility == governance.VisibilityExternal {
return "external"
}
if request.Visibility == governance.VisibilityBoth && len(request.TargetKnowledgeBaseIDs) > 1 &&
knowledgeBaseID == request.TargetKnowledgeBaseIDs[len(request.TargetKnowledgeBaseIDs)-1] {
return "external"
}
return "internal"
}
// itemVisibilityFor mirrors source_items' two-value visibility CHECK: both and
// blocked decisions are governed through policy_state; the legacy visibility
// column keeps a valid value and is widened to external for both.
func itemVisibilityFor(visibility governance.Visibility) string {
if visibility == governance.VisibilityInternal {
return string(governance.VisibilityInternal)
}
return string(governance.VisibilityExternal)
}
func sha256Hex(value string) string {
sum := sha256.Sum256([]byte(value))
return hex.EncodeToString(sum[:])
}
// MarkArtifactFetched finalizes one artifact.fetch success: it (re)uses the
// content_objects row for the verified hash (content-level dedupe), attaches it
// to the pending version and flips the intent to ready. matchedLocal records
// that the bytes already existed locally and no source download happened.
func (s *Store) MarkArtifactFetched(ctx context.Context, versionID, sha256Value string, sizeBytes int64, spoolRef, validatedFormat string, matchedLocal bool, now time.Time) error {
if len(sha256Value) != 64 || spoolRef == "" || sizeBytes < 0 || validatedFormat == "" {
return errors.New("invalid artifact fetch result")
}
var tx pgx.Tx
ownsTransaction := false
if existing, ok := s.db.(pgx.Tx); ok {
tx = existing
} else {
beginner, ok := s.db.(interface {
Begin(context.Context) (pgx.Tx, error)
})
if !ok {
return errors.New("artifact fetch transactions are unavailable")
}
var err error
tx, err = beginner.Begin(ctx)
if err != nil {
return fmt.Errorf("begin artifact fetch: %w", err)
}
ownsTransaction = true
defer tx.Rollback(ctx)
}
var connectionID, nodeID string
err := tx.QueryRow(ctx, `SELECT intent.source_connection_id,intent.node_id
FROM catalog_import_intents intent
JOIN source_item_versions version ON version.source_item_id=intent.source_item_id
WHERE version.id=$1 FOR UPDATE OF intent`, versionID).Scan(&connectionID, &nodeID)
if err != nil {
return fmt.Errorf("locate intent for fetched artifact: %w", err)
}
freshID, err := newOpaqueID("co-")
if err != nil {
return err
}
contentID := ""
err = tx.QueryRow(ctx, `INSERT INTO content_objects(id,sha256,size_bytes,mime,validated_format,local_spool_ref,content_state,created_at,updated_at)
VALUES($1,$2,$3,'',$5,$4,'verified',NOW(),NOW())
ON CONFLICT(sha256) DO UPDATE SET
size_bytes=EXCLUDED.size_bytes,
validated_format=CASE WHEN content_objects.validated_format='' THEN EXCLUDED.validated_format ELSE content_objects.validated_format END,
local_spool_ref=CASE WHEN content_objects.local_spool_ref='' THEN EXCLUDED.local_spool_ref ELSE content_objects.local_spool_ref END,
content_state='verified',updated_at=NOW()
RETURNING id`,
freshID, sha256Value, sizeBytes, spoolRef, validatedFormat).Scan(&contentID)
if err != nil {
return fmt.Errorf("record fetched content: %w", err)
}
if _, err := tx.Exec(ctx, `UPDATE source_item_versions SET
content_object_id=$2,content_state='verified',updated_at=$4,
raw_record_summary=jsonb_set(raw_record_summary,'{matched_local}',$3::jsonb)
WHERE id=$1`,
versionID, contentID, mustJSONBool(matchedLocal), now.UTC()); err != nil {
return fmt.Errorf("bind fetched content: %w", err)
}
if _, err := tx.Exec(ctx, `UPDATE catalog_import_intents SET fetch_state='ready',last_error='',updated_at=NOW()
WHERE source_connection_id=$1 AND node_id=$2`, connectionID, nodeID); err != nil {
return fmt.Errorf("mark intent fetched: %w", err)
}
if ownsTransaction {
if err := tx.Commit(ctx); err != nil {
return fmt.Errorf("commit artifact fetch: %w", err)
}
}
return nil
}
func mustJSONBool(value bool) string {
if value {
return "true"
}
return "false"
}
// MarkArtifactFetchMissing parks an intent whose source-side file disappeared
// (a 404 on download, not a transient outage) so the UI can ask for a rescan.
func (s *Store) MarkArtifactFetchMissing(ctx context.Context, connectionID, nodeID string, now time.Time) error {
if _, err := s.db.Exec(ctx, `UPDATE catalog_import_intents SET fetch_state='missing_local',updated_at=$3
WHERE source_connection_id=$1 AND node_id=$2 AND fetch_state<>'ready'`, connectionID, nodeID, now.UTC()); err != nil {
return fmt.Errorf("record artifact fetch miss: %w", err)
}
return nil
}
// MarkArtifactFetchFailed records a fetch failure for the intent so 解析监控
// can show why an item is not yet 待入库.
func (s *Store) MarkArtifactFetchFailed(ctx context.Context, connectionID, nodeID, message string, now time.Time) error {
if _, err := s.db.Exec(ctx, `UPDATE catalog_import_intents SET fetch_state='failed',last_error=$3,updated_at=$4
WHERE source_connection_id=$1 AND node_id=$2 AND fetch_state<>'ready'`, connectionID, nodeID, truncateMonitorText(message), now.UTC()); err != nil {
return fmt.Errorf("record artifact fetch failure: %w", err)
}
return nil
}
func truncateMonitorText(value string) string {
runes := []rune(value)
if len(runes) <= 500 {
return value
}
return string(runes[:500])
}
@@ -16,9 +16,12 @@ func (s *Store) StartTestPublications(ctx context.Context, request governance.St
result := governance.StartTestPublicationsResult{Selected: len(request.PublicationIDs)}
operator := strings.TrimSpace(request.OperatorRef)
reason := strings.TrimSpace(request.Reason)
if len(request.PublicationIDs) == 0 || len(request.PublicationIDs) > 100 || operator == "" || len(operator) > 128 || reason == "" || len(reason) > 500 || request.Now.IsZero() {
if len(request.PublicationIDs) == 0 || len(request.PublicationIDs) > 100 || operator == "" || len(operator) > 128 || len(reason) > 500 || request.Now.IsZero() {
return governance.StartTestPublicationsResult{}, errors.New("invalid processing confirmation")
}
if reason == "" {
reason = "开始处理"
}
seen := make(map[string]bool, len(request.PublicationIDs))
for _, publicationID := range request.PublicationIDs {
if !validRouteID(publicationID) || seen[publicationID] {
@@ -50,22 +53,23 @@ func (s *Store) StartTestPublications(ctx context.Context, request governance.St
for _, publicationID := range request.PublicationIDs {
var ready, confirmed, uploadJobExists bool
var processingState string
err = tx.QueryRow(ctx, `
SELECT p.ingest_state='pending'
AND p.processing_state='pending'
AND p.retrieval_state='unpublished'
AND p.knowledge_id=''
AND version.content_state='verified'
AND content.content_state='verified'
AND content.local_spool_ref<>''
AND content.size_bytes>0 AS ready,
p.processing_state,
EXISTS(SELECT 1 FROM processing_confirmations confirmation WHERE confirmation.publication_id=p.id),
EXISTS(SELECT 1 FROM jobs job WHERE job.type='test.upload' AND job.payload->>'publication_id'=p.id)
FROM test_publications p
JOIN source_item_versions version ON version.id=p.source_item_version_id
JOIN content_objects content ON content.id=version.content_object_id
WHERE p.id=$1
FOR UPDATE OF p`, publicationID).Scan(&ready, &confirmed, &uploadJobExists)
WHERE p.id=$1 AND p.superseded_at IS NULL
FOR UPDATE OF p`, publicationID).Scan(&ready, &processingState, &confirmed, &uploadJobExists)
if errors.Is(err, pgx.ErrNoRows) {
result.Unavailable++
continue
@@ -77,10 +81,18 @@ FOR UPDATE OF p`, publicationID).Scan(&ready, &confirmed, &uploadJobExists)
result.AlreadyStarted++
continue
}
// 开始处理 promotes both fresh plans and verified pending rows.
if processingState != "planned" && processingState != "pending" {
result.Unavailable++
continue
}
if !ready {
result.Unavailable++
continue
}
if _, err := tx.Exec(ctx, `UPDATE test_publications SET processing_state='pending',updated_at=$2 WHERE id=$1 AND processing_state='planned'`, publicationID, request.Now); err != nil {
return governance.StartTestPublicationsResult{}, fmt.Errorf("promote planned publication: %w", err)
}
confirmationID, err := newOpaqueID("pc-")
if err != nil {
return governance.StartTestPublicationsResult{}, err
+21 -9
View File
@@ -13,17 +13,29 @@ RETURNING id, type, payload, state, priority, available_at,
`
const claimJobSQL = `
WITH candidate AS (
SELECT id
FROM jobs
WITH budget AS (
SELECT COALESCE((
SELECT ingestion_concurrency
FROM ui_setting_revisions
WHERE state = 'active'
LIMIT 1
), 2) AS ingestion_slots
), candidate AS (
SELECT jobs.id
FROM jobs, budget
WHERE (
(state IN ('pending', 'retry') AND available_at <= $2)
OR (state = 'running' AND lease_until < $2)
(jobs.state IN ('pending', 'retry') AND jobs.available_at <= $2)
OR (jobs.state = 'running' AND jobs.lease_until < $2)
)
AND attempt < max_attempts
AND type = ANY($4::text[])
ORDER BY priority DESC, available_at ASC, created_at ASC, id ASC
FOR UPDATE SKIP LOCKED
AND jobs.attempt < jobs.max_attempts
AND jobs.type = ANY($4::text[])
AND (
jobs.type <> 'test.upload'
OR (SELECT COUNT(*) FROM jobs active
WHERE active.type = 'test.upload' AND active.state = 'running') < budget.ingestion_slots
)
ORDER BY jobs.priority DESC, jobs.available_at ASC, jobs.created_at ASC, jobs.id ASC
FOR UPDATE OF jobs SKIP LOCKED
LIMIT 1
)
UPDATE jobs AS j
@@ -51,7 +51,7 @@ INSERT INTO source_catalog_heads(source_connection_id,snapshot_id,activated_at)
ConnectionID: "conn-s6", NodeIDs: []string{"folder-top"}, Visibility: governance.VisibilityInternal,
TargetKnowledgeBaseID: "kb-internal", OperatorRef: "engineer-1", Reason: "S6目录递归标注", Now: now,
})
if err != nil || result.SelectedFiles != 1 || result.RoutableFiles != 1 {
if err != nil || result.SelectedFiles != 1 || result.PlannedPublications != 1 || result.FetchQueued != 0 {
t.Fatalf("route=%#v err=%v", result, err)
}
// Effective visibility of the deep item must be internal via the directory rule.
@@ -128,7 +128,7 @@ SELECT p.id, si.title, p.ingest_state, p.processing_state, p.retrieval_state,
JOIN source_item_versions other_version ON other_version.id=other.source_item_version_id
WHERE other.id<>p.id AND other_version.source_item_id=siv.source_item_id
) THEN 'updated' ELSE 'added' END AS change_type,
(p.ingest_state='pending' AND p.processing_state='pending'
(p.ingest_state='pending' AND p.processing_state IN ('pending','planned')
AND p.retrieval_state='unpublished' AND p.knowledge_id=''
AND co.content_state='verified' AND co.local_spool_ref<>'' AND co.size_bytes>0
AND NOT EXISTS (SELECT 1 FROM processing_confirmations confirmation WHERE confirmation.publication_id=p.id)
@@ -142,8 +142,8 @@ SELECT p.id, si.title, p.ingest_state, p.processing_state, p.retrieval_state,
FROM test_publications p
JOIN source_item_versions siv ON siv.id = p.source_item_version_id
JOIN source_items si ON si.id = siv.source_item_id
JOIN content_objects co ON co.id = siv.content_object_id
WHERE ($1 = '' OR p.processing_state = $1) AND ($2 = '' OR p.target_scope = $2)
LEFT JOIN content_objects co ON co.id = siv.content_object_id
WHERE ($1 = '' OR p.processing_state = $1) AND ($2 = '' OR p.target_scope = $2) AND p.superseded_at IS NULL
ORDER BY p.updated_at DESC, p.id
LIMIT $3 OFFSET $4
`
+6 -4
View File
@@ -13,7 +13,7 @@ import (
var ErrInvalidUISettings = errors.New("invalid UI settings")
const uiSettingsColumns = `id,state,document_refresh_seconds,resource_refresh_seconds,stalled_after_seconds,finalizing_after_seconds,updated_by,created_at,activated_at,retired_at,error_code`
const uiSettingsColumns = `id,state,document_refresh_seconds,resource_refresh_seconds,stalled_after_seconds,finalizing_after_seconds,ingestion_concurrency,updated_by,created_at,activated_at,retired_at,error_code`
const getActiveUISettingsSQL = `
SELECT ` + uiSettingsColumns + `
@@ -24,8 +24,8 @@ WHERE state = 'active'
const createUISettingsCandidateSQL = `
INSERT INTO ui_setting_revisions(
id,state,document_refresh_seconds,resource_refresh_seconds,
stalled_after_seconds,finalizing_after_seconds,updated_by,created_at
) VALUES($1,'validated',$2,$3,$4,$5,$6,$7)
stalled_after_seconds,finalizing_after_seconds,ingestion_concurrency,updated_by,created_at
) VALUES($1,'validated',$2,$3,$4,$5,$6,$7,$8)
RETURNING ` + uiSettingsColumns + `
`
@@ -63,7 +63,8 @@ func (s *Store) CreateUISettingsCandidate(ctx context.Context, request governanc
}
return scanUISettings(s.db.QueryRow(ctx, createUISettingsCandidateSQL,
id, request.DocumentRefreshSeconds, request.ResourceRefreshSeconds,
request.StalledAfterSeconds, request.FinalizingAfterSeconds, request.UpdatedBy, request.Now.UTC()))
request.StalledAfterSeconds, request.FinalizingAfterSeconds, request.IngestionConcurrency,
request.UpdatedBy, request.Now.UTC()))
}
func (s *Store) ActivateUISettings(ctx context.Context, revisionID string, now time.Time) (governance.UISettings, error) {
@@ -124,6 +125,7 @@ func scanUISettings(row interface{ Scan(...any) error }) (governance.UISettings,
var settings governance.UISettings
if err := row.Scan(&settings.ID, &settings.State, &settings.DocumentRefreshSeconds,
&settings.ResourceRefreshSeconds, &settings.StalledAfterSeconds, &settings.FinalizingAfterSeconds,
&settings.IngestionConcurrency,
&settings.UpdatedBy, &settings.CreatedAt, &settings.ActivatedAt, &settings.RetiredAt, &settings.ErrorCode); err != nil {
return governance.UISettings{}, fmt.Errorf("scan UI settings: %w", err)
}
@@ -31,7 +31,7 @@ func TestUISettingsActivationRetiresPreviousRevisionIntegration(t *testing.T) {
t.Fatalf("previous=%#v error=%v", previous, err)
}
now := time.Date(2026, 9, 10, 7, 0, 0, 0, time.UTC)
candidate, err := store.CreateUISettingsCandidate(ctx, governance.CreateUISettingsCandidateRequest{DocumentRefreshSeconds: 30, ResourceRefreshSeconds: 60, StalledAfterSeconds: 3600, FinalizingAfterSeconds: 14400, UpdatedBy: "engineer-1", Now: now})
candidate, err := store.CreateUISettingsCandidate(ctx, governance.CreateUISettingsCandidateRequest{DocumentRefreshSeconds: 30, ResourceRefreshSeconds: 60, StalledAfterSeconds: 3600, FinalizingAfterSeconds: 14400, IngestionConcurrency: 2, UpdatedBy: "engineer-1", Now: now})
if err != nil {
t.Fatal(err)
}
@@ -14,7 +14,7 @@ func TestUISettingsCandidateAndActiveProjectionAreVersioned(t *testing.T) {
now := time.Date(2026, 9, 10, 6, 0, 0, 0, time.UTC)
row := uiSettingsRow{id: "settings-safe", state: "validated", documentRefresh: 30, resourceRefresh: 60, stalledAfter: 3600, finalizingAfter: 14400, updatedBy: "engineer-1", createdAt: now}
db := &recordingDB{row: row}
candidate, err := NewStore(db).CreateUISettingsCandidate(context.Background(), governance.CreateUISettingsCandidateRequest{DocumentRefreshSeconds: 30, ResourceRefreshSeconds: 60, StalledAfterSeconds: 3600, FinalizingAfterSeconds: 14400, UpdatedBy: "engineer-1", Now: now})
candidate, err := NewStore(db).CreateUISettingsCandidate(context.Background(), governance.CreateUISettingsCandidateRequest{DocumentRefreshSeconds: 30, ResourceRefreshSeconds: 60, StalledAfterSeconds: 3600, FinalizingAfterSeconds: 14400, IngestionConcurrency: 2, UpdatedBy: "engineer-1", Now: now})
if err != nil {
t.Fatal(err)
}
@@ -53,6 +53,7 @@ func TestUISettingsRejectInvalidCandidate(t *testing.T) {
type uiSettingsRow struct {
id, state, updatedBy, errorCode string
documentRefresh, resourceRefresh, stalledAfter, finalizingAfter int
ingestionConcurrency int
createdAt time.Time
activatedAt, retiredAt *time.Time
}
@@ -60,6 +61,7 @@ type uiSettingsRow struct {
func (row uiSettingsRow) Scan(dest ...any) error {
*dest[0].(*string), *dest[1].(*string) = row.id, row.state
*dest[2].(*int), *dest[3].(*int), *dest[4].(*int), *dest[5].(*int) = row.documentRefresh, row.resourceRefresh, row.stalledAfter, row.finalizingAfter
*dest[6].(*string), *dest[7].(*time.Time), *dest[8].(**time.Time), *dest[9].(**time.Time), *dest[10].(*string) = row.updatedBy, row.createdAt, row.activatedAt, row.retiredAt, row.errorCode
*dest[6].(*int) = row.ingestionConcurrency
*dest[7].(*string), *dest[8].(*time.Time), *dest[9].(**time.Time), *dest[10].(**time.Time), *dest[11].(*string) = row.updatedBy, row.createdAt, row.activatedAt, row.retiredAt, row.errorCode
return nil
}
+29 -4
View File
@@ -34,8 +34,9 @@ type Artifact struct {
}
var supportedFormats = map[string]struct{}{
"pdf": {}, "docx": {}, "doc": {}, "xlsx": {}, "xls": {}, "pptx": {}, "ppt": {},
"pdf": {}, "docx": {}, "doc": {}, "xlsx": {}, "xls": {},
"md": {}, "txt": {}, "csv": {}, "json": {},
"png": {}, "jpg": {}, "jpeg": {}, "gif": {}, "bmp": {}, "webp": {},
}
func InspectVerified(root, relativeRef, format string, maxBytes int64) (Expectation, error) {
@@ -137,7 +138,7 @@ func validateFormat(file *os.File, format string, size int64) error {
if _, err := io.ReadFull(file, header); err != nil || !bytes.Equal(header, []byte("%PDF-")) {
return errors.New("invalid PDF magic")
}
case "docx", "xlsx", "pptx":
case "docx", "xlsx":
archive, err := zip.NewReader(file, size)
if err != nil {
return errors.New("invalid Office ZIP")
@@ -145,7 +146,6 @@ func validateFormat(file *os.File, format string, size int64) error {
required := map[string]string{
"docx": "word/document.xml",
"xlsx": "xl/workbook.xml",
"pptx": "ppt/presentation.xml",
}[format]
foundContentTypes := false
foundRequired := false
@@ -160,12 +160,37 @@ func validateFormat(file *os.File, format string, size int64) error {
if !foundContentTypes || !foundRequired {
return errors.New("missing Office package entry")
}
case "doc", "xls", "ppt":
case "doc", "xls":
header := make([]byte, 8)
oleMagic := []byte{0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1}
if _, err := io.ReadFull(file, header); err != nil || !bytes.Equal(header, oleMagic) {
return errors.New("invalid legacy Office magic")
}
case "png", "jpg", "jpeg", "gif", "bmp", "webp":
header := make([]byte, 12)
n, err := file.Read(header)
if err != nil || n < 12 {
return errors.New("invalid image header")
}
valid := false
switch format {
case "png":
valid = bytes.Equal(header[:8], []byte{0x89, 'P', 'N', 'G', 0x0d, 0x0a, 0x1a, 0x0a})
case "jpg", "jpeg":
valid = header[0] == 0xff && header[1] == 0xd8 && header[2] == 0xff
case "gif":
valid = bytes.Equal(header[:6], []byte("GIF87a")) || bytes.Equal(header[:6], []byte("GIF89a"))
case "bmp":
valid = header[0] == 'B' && header[1] == 'M'
case "webp":
valid = bytes.Equal(header[:4], []byte("RIFF")) && bytes.Equal(header[8:12], []byte("WEBP"))
}
if !valid {
return errors.New("invalid image magic")
}
if _, err := file.Seek(0, io.SeekStart); err != nil {
return errors.New("rewind image artifact")
}
case "json":
decoder := json.NewDecoder(file)
var value any
+247
View File
@@ -0,0 +1,247 @@
// Package artifactfetch turns a saved policy intent into verified bytes: it
// shells out to scripts/artifact_fetch.py (which already holds every source
// adapter: AList direct download, PP ledger, DingTalk export-tree/live export)
// and records the verified content against the pending source item version.
package artifactfetch
import (
"bufio"
"bytes"
"context"
"encoding/json"
"errors"
"os/exec"
"path/filepath"
"strconv"
"strings"
"time"
"knowledge-governance/internal/jobs"
)
type Request struct {
ConnectionID string `json:"source_connection_id"`
NodeID string `json:"node_id"`
VersionID string `json:"version_id"`
}
type Result struct {
OK bool `json:"ok"`
SpecID string `json:"spec_id"`
NodeID string `json:"node_id"`
ConnectionID string `json:"connection_id"`
SHA256 string `json:"sha256"`
SizeBytes int64 `json:"size_bytes"`
SpoolRef string `json:"spool_ref"`
ValidatedFormat string `json:"validated_format"`
MatchedLocal bool `json:"matched_local"`
Missing bool `json:"missing"`
Retryable bool `json:"retryable"`
Code string `json:"code"`
Error string `json:"error"`
}
type Manifest struct {
Request
SourceType string
SourceName string
RelativePath string
Name string
SizeBytes *int64
ModifiedAt *time.Time
LocatorHash string
}
type Store interface {
LoadArtifactFetchManifest(context.Context, string, string) (Manifest, error)
MarkArtifactFetched(context.Context, string, string, int64, string, string, bool, time.Time) error
MarkArtifactFetchMissing(context.Context, string, string, time.Time) error
MarkArtifactFetchFailed(context.Context, string, string, string, time.Time) error
}
type Runner interface {
Run(context.Context, Manifest) (Result, error)
}
type Config struct {
Store Store
Runner Runner
Now func() time.Time
}
type Handler struct {
store Store
runner Runner
now func() time.Time
}
func NewHandler(config Config) (*Handler, error) {
if config.Store == nil || config.Runner == nil {
return nil, errors.New("artifact fetch dependencies are required")
}
if config.Now == nil {
config.Now = time.Now
}
return &Handler{store: config.Store, runner: config.Runner, now: config.Now}, nil
}
func (handler *Handler) Handle(ctx context.Context, job jobs.Job) error {
var request Request
decoder := json.NewDecoder(bytes.NewReader(job.Payload))
decoder.DisallowUnknownFields()
if err := decoder.Decode(&request); err != nil ||
request.ConnectionID == "" || request.NodeID == "" || request.VersionID == "" {
return jobs.Permanent(jobs.ErrCodeArtifactInvalid, errors.New("artifact_fetch_invalid_payload"))
}
manifest, err := handler.store.LoadArtifactFetchManifest(ctx, request.ConnectionID, request.NodeID)
if err != nil {
if errors.Is(err, ErrManifestGone) {
return jobs.Permanent(jobs.ErrCodeStateConflict, err)
}
return err
}
result, err := handler.runner.Run(ctx, manifest)
now := handler.now()
if err != nil {
_ = handler.store.MarkArtifactFetchFailed(ctx, request.ConnectionID, request.NodeID, err.Error(), now)
return err
}
switch {
case result.Missing:
if err := handler.store.MarkArtifactFetchMissing(ctx, request.ConnectionID, request.NodeID, now); err != nil {
return err
}
return jobs.Permanent(jobs.ErrCodeArtifactInvalid, errors.New("artifact_fetch_missing_source: "+safeMessage(result.Error)))
case !result.OK:
_ = handler.store.MarkArtifactFetchFailed(ctx, request.ConnectionID, request.NodeID, safeMessage(result.Error), now)
if result.Retryable {
return errors.New("artifact_fetch_retryable: " + safeMessage(result.Code))
}
return jobs.Permanent(jobs.ErrCodeArtifactInvalid, errors.New("artifact_fetch_rejected: "+safeMessage(result.Code)))
}
if result.SHA256 == "" || result.SpoolRef == "" || result.SizeBytes <= 0 || result.NodeID != request.NodeID || result.ValidatedFormat == "" {
_ = handler.store.MarkArtifactFetchFailed(ctx, request.ConnectionID, request.NodeID, "artifact fetch returned an inconsistent result", now)
return errors.New("artifact_fetch_invalid_output")
}
if err := handler.store.MarkArtifactFetched(ctx, request.VersionID, strings.ToLower(result.SHA256),
result.SizeBytes, result.SpoolRef, result.ValidatedFormat, result.MatchedLocal, now); err != nil {
return err
}
return nil
}
func safeMessage(value string) string {
value = strings.ReplaceAll(strings.TrimSpace(value), "\n", " ")
lower := strings.ToLower(value)
for _, forbidden := range []string{"token", "secret", "password", "authorization", "bearer", "cookie", "/run/source-secrets", "http://", "https://"} {
if strings.Contains(lower, forbidden) {
return "source reported an error (details suppressed)"
}
}
if len([]rune(value)) > 200 {
value = string([]rune(value)[:200])
}
return value
}
// CommandRunner executes the fetch script once and parses its single JSON line.
type CommandRunner struct {
PythonPath string
ScriptPath string
SpoolRoot string
SpoolPrefix string
MaxBytes int64
}
func NewCommandRunner(pythonPath, scriptPath, spoolRoot, spoolPrefix string, maxBytes int64) (*CommandRunner, error) {
if !filepath.IsAbs(pythonPath) || !filepath.IsAbs(scriptPath) || !filepath.IsAbs(spoolRoot) || maxBytes <= 0 {
return nil, errors.New("artifact fetch command paths must be absolute")
}
if strings.Contains(spoolPrefix, "..") || strings.Contains(spoolPrefix, "\\") {
return nil, errors.New("artifact fetch spool prefix is invalid")
}
spoolPrefix = strings.Trim(spoolPrefix, "/")
if spoolPrefix != "" {
spoolPrefix += "/"
}
return &CommandRunner{PythonPath: pythonPath, ScriptPath: scriptPath, SpoolRoot: spoolRoot, SpoolPrefix: spoolPrefix, MaxBytes: maxBytes}, nil
}
func (runner *CommandRunner) Run(ctx context.Context, manifest Manifest) (Result, error) {
manifestJSON, err := json.Marshal(map[string]any{
"spec_id": manifest.VersionID, "node_id": manifest.NodeID, "connection_id": manifest.ConnectionID,
"source_type": manifest.SourceType, "source_name": manifest.SourceName,
"relative_path": manifest.RelativePath, "name": manifest.Name,
"size_bytes": manifest.SizeBytes, "locator_hash": manifest.LocatorHash,
})
if err != nil {
return Result{}, errors.New("artifact_fetch_manifest_failed")
}
command := exec.CommandContext(ctx, runner.PythonPath, runner.ScriptPath, string(manifestJSON))
// The literal guard keeps secrets out of the child environment: the script
// only ever reads pre-mounted files.
command.Env = []string{
"PATH=/usr/bin:/bin",
"PYTHONUNBUFFERED=1",
"INGESTION_SPOOL_ROOT=" + runner.SpoolRoot,
"SPOOL_REF_PREFIX=" + runner.SpoolPrefix,
"ARTIFACT_FETCH_MAX_BYTES=" + strconv.FormatInt(runner.MaxBytes, 10),
"TZ=UTC",
}
stdout, err := command.StdoutPipe()
if err != nil {
return Result{}, errors.New("artifact_fetch_start_failed")
}
command.Stderr = &limitedWriter{buffer: bytes.NewBuffer(nil), limit: 4096}
if err := command.Start(); err != nil {
return Result{}, errors.New("artifact_fetch_start_failed")
}
var result Result
decoded := false
scanner := bufio.NewScanner(stdout)
scanner.Buffer(make([]byte, 4096), 1<<20)
for scanner.Scan() {
line := bytes.TrimSpace(scanner.Bytes())
if len(line) == 0 || !bytes.HasPrefix(line, []byte("{")) {
continue
}
decoder := json.NewDecoder(bytes.NewReader(line))
decoder.DisallowUnknownFields()
if err := decoder.Decode(&result); err != nil {
_ = command.Process.Kill()
_ = command.Wait()
return Result{}, errors.New("artifact_fetch_invalid_output")
}
decoded = true
break
}
waitErr := command.Wait()
if !decoded {
if waitErr != nil {
return Result{OK: false, Error: "artifact fetch exited without a result"}, errors.New("artifact_fetch_failed")
}
return Result{}, errors.New("artifact_fetch_no_output")
}
return result, nil
}
var ErrManifestGone = errors.New("artifact fetch manifest is gone")
// limitedWriter captures at most limit bytes of script stderr; full output is
// never retained (it could echo source details we do not want in failures).
type limitedWriter struct {
buffer *bytes.Buffer
limit int
}
func (writer *limitedWriter) Write(payload []byte) (int, error) {
room := writer.limit - writer.buffer.Len()
if room > 0 {
if len(payload) > room {
writer.buffer.Write(payload[:room])
return len(payload), nil
}
writer.buffer.Write(payload)
}
return len(payload), nil
}
+515
View File
@@ -0,0 +1,515 @@
#!/usr/bin/env python3
"""Fetch one policy-selected catalog artifact from its live source.
The Go worker passes a small JSON manifest (argv[1]) describing the intent.
This process never receives database credentials or secret values; it locates
credentials only via the pre-mounted /run/source-secrets paths, exactly like
live_catalog_refresh.py. Behavior per source type:
alist -> direct read-only API: raw_url then download.
pp -> manifest.json local_path first (already on disk), else public URL.
dingtalk -> prefer the local export tree (converted docx/xlsx/pdf); if it is
older than the source-side modification time, export fresh bytes
through the web internal API captured headers.
Before any network call the fetcher compares against local candidates by
size and, when a hash is cheap to obtain, by content: a match means "本地已有
- no pull". Verified bytes are moved into the ingestion spool and a single
JSON result line is printed on stdout. Credentials, tokens, signed URLs and
authorization material are never printed.
"""
import csv
import hashlib
import json
import os
import pathlib
import re
import shutil
import subprocess
import sys
import tempfile
import time
import urllib.error
import urllib.parse
import urllib.request
SUPPORTED = {
"pdf": "pdf", "docx": "docx", "xlsx": "xlsx", "md": "md", "txt": "txt",
"csv": "csv", "json": "json",
"png": "png", "jpg": "jpg", "jpeg": "jpeg", "gif": "gif", "bmp": "bmp", "webp": "webp",
}
# Office types we refuse to ingest directly: PPT/PPTX must arrive as a PDF twin
# (embedded images travel inside those files already).
SLIDES = {"ppt", "pptx"}
LEGACY = {"doc": "doc", "xls": "xls"}
MAX_BYTES_DEFAULT = 200 * 1024 * 1024
class FetchError(Exception):
def __init__(self, message, retryable=True, code="fetch_failed"):
super().__init__(message)
self.retryable = retryable
self.code = code
class MissingSource(Exception):
pass
def emit(result):
print(json.dumps(result, ensure_ascii=False))
def sha256_file(path):
digest = hashlib.sha256()
with open(path, "rb") as handle:
for block in iter(lambda: handle.read(1 << 20), b""):
digest.update(block)
return digest.hexdigest()
def digest_of(value):
return hashlib.sha256(value.encode()).hexdigest()
def validate_magic(path, suffix):
with open(path, "rb") as handle:
head = handle.read(12)
checks = {
"pdf": head.startswith(b"%PDF-"),
"docx": head.startswith(b"PK\x03\x04"),
"xlsx": head.startswith(b"PK\x03\x04"),
"doc": head.startswith(b"\xd0\xcf\x11\xe0"),
"xls": head.startswith(b"\xd0\xcf\x11\xe0"),
"png": head.startswith(b"\x89PNG\r\n\x1a\n"),
"jpg": head[:2] == b"\xff\xd8" and head[2:3] == b"\xff",
"jpeg": head[:2] == b"\xff\xd8" and head[2:3] == b"\xff",
"gif": head[:6] in (b"GIF87a", b"GIF89a"),
"bmp": head[:2] == b"BM",
"webp": head[:4] == b"RIFF" and head[8:12] == b"WEBP",
}
if suffix in checks and not checks[suffix]:
raise FetchError("downloaded bytes are not a valid %s file" % suffix,
retryable=False, code="invalid_format")
if suffix in ("md", "txt", "csv", "json"):
try:
with open(path, "rb") as handle:
handle.read(4096).decode("utf-8", errors="strict")
except UnicodeDecodeError:
raise FetchError("text artifact is not valid UTF-8", retryable=False, code="invalid_format")
def pick_suffix(name, declared_type):
suffix = pathlib.PurePosixPath(name).suffix.lower().lstrip(".")
if declared_type in ("adoc",) and suffix in ("", "adoc"):
return "docx"
if declared_type in ("able", "axls") and suffix in ("", "able", "axls"):
return "xlsx"
if suffix in SUPPORTED:
return SUPPORTED[suffix]
if suffix == "adoc":
return "docx"
if suffix in ("able", "axls"):
return "xlsx"
return None
def http_get(url, headers, timeout, max_bytes, cookie_opener=None):
request = urllib.request.Request(url, headers=headers or {})
class NoRedirect(urllib.request.HTTPRedirectHandler):
location = ""
def redirect_request(self, req, fp, code, msg, headers, newurl):
NoRedirect.location = newurl
return None
handler = NoRedirect()
opener = cookie_opener or urllib.request.build_opener(handler)
try:
response = opener.open(request, timeout=timeout)
except urllib.error.HTTPError as error:
if error.code in (301, 302, 303, 307, 308):
target = handler.location or error.headers.get("Location") or ""
host = urllib.parse.urlparse(target).netloc.lower()
if any(marker in host for marker in ("login.", "sso", "oauth", "passport")):
raise FetchError("文件所在的来源存储需要网页登录授权(存储后端未配置服务凭据),请联系管理员", retryable=False, code="source_auth_required")
raise FetchError("redirect refused", retryable=True, code="redirect")
if error.code == 404:
raise MissingSource("source returned 404")
raise FetchError("source returned HTTP %d" % error.code, retryable=error.code >= 500, code="http_%d" % error.code)
with response:
declared = response.headers.get("Content-Length")
if declared and int(declared) > max_bytes:
raise FetchError("source file exceeds ingest size limit", retryable=False, code="over_limit")
destination = tempfile.NamedTemporaryFile(delete=False, suffix=".fetch")
total = 0
digest = hashlib.sha256()
try:
while True:
block = response.read(1 << 20)
if not block:
break
total += len(block)
if total > max_bytes:
raise FetchError("source stream exceeds ingest size limit", retryable=False, code="over_limit")
digest.update(block)
destination.write(block)
finally:
destination.close()
if total == 0:
os.unlink(destination.name)
raise FetchError("source returned empty body", retryable=True, code="empty")
return destination.name, total, digest.hexdigest()
# ---------------------------------------------------------------- AList ------
def alist_download(manifest, max_bytes):
sys.path.insert(0, "/sources/alist")
from alist_readonly_inventory import AListReadOnlyClient, read_connections # noqa
token = pathlib.Path("/run/source-secrets/alist_api_token").read_text(encoding="utf-8").strip()
if not token:
raise FetchError("AList token empty", retryable=False, code="credentials")
rows = [row for row in read_connections(pathlib.Path("/run/source-secrets/alist_connections.xlsx"), require_token=False)
if row["category"] == manifest["source_name"]]
if len(rows) != 1:
raise FetchError("AList connection configuration missing", retryable=False, code="credentials")
base_url = rows[0]["base_url"].rstrip("/")
client = AListReadOnlyClient(base_url, token)
path = "/" + manifest["relative_path"].lstrip("/")
payload = json.dumps({"path": path, "password": ""}).encode("utf-8")
request = urllib.request.Request(base_url + "/api/fs/get", data=payload, method="POST", headers={
"x-api-token": token, "Content-Type": "application/json", "Accept": "application/json",
"User-Agent": "governance-artifact-fetch/1.0",
})
try:
with client.opener.open(request, timeout=30) as response:
body = json.loads(response.read().decode("utf-8"))
except Exception as error:
raise FetchError("AList file lookup failed (%s)" % type(error).__name__, code="source_unavailable")
if int(body.get("code") or 0) == 404:
raise MissingSource("AList file not present at expected path")
if int(body.get("code") or 0) != 200:
raise FetchError("AList file lookup refused (code %s)" % body.get("code"), code="source_unavailable")
data = body.get("data") or {}
raw_url = str(data.get("raw_url") or "")
if not raw_url:
raise MissingSource("AList returned no download locator for this file")
if manifest.get("size_bytes") in (None, 0) and data.get("size"):
manifest["size_bytes"] = int(data["size"])
download_url = urllib.parse.urljoin(base_url + "/", raw_url.lstrip("/"))
try:
return http_get(download_url, {"Authorization": token}, 180, max_bytes)
except FetchError as error:
if error.code == "redirect":
# Same-origin /d/ route 302s to a signed external URL: follow once
# WITHOUT the source token (never forward credentials cross-origin).
class LocationCapture(urllib.request.HTTPRedirectHandler):
captured = ""
def redirect_request(self, req, fp, code, msg, headers, newurl):
LocationCapture.captured = newurl
return None
probe = urllib.request.build_opener(LocationCapture)
try:
probe.open(urllib.request.Request(download_url, headers={"Authorization": token}), timeout=30)
except urllib.error.HTTPError:
pass
except Exception as failure:
raise FetchError("AList redirect probe failed (%s)" % type(failure).__name__, code="source_unavailable")
if not LocationCapture.captured:
raise
return http_get(LocationCapture.captured, {}, 180, max_bytes)
raise
# ------------------------------------------------------------------ PP -------
def pp_find_entry(manifest):
locator = manifest.get("locator_hash", "")
if not locator:
return None
manifest_path = pathlib.Path("/sources/pp/data/public-documents/manifest.json")
ledger = json.loads(manifest_path.read_text(encoding="utf-8"))
for result in ledger.get("results", []):
if digest_of(str(result.get("url") or "")) == locator:
return result
return None
def pp_download(manifest, max_bytes):
entry = pp_find_entry(manifest)
if entry is None:
raise MissingSource("PP ledger entry for this locator no longer exists")
# Ledger titles may omit the extension; recover the true file name from the
# local path or source URL before type checks run.
manifest_name = manifest.setdefault("name", "")
if not pathlib.PurePosixPath(manifest_name).suffix:
from urllib.parse import unquote, urlparse
for probe in (entry.get("local_path", ""), unquote(urlparse(str(entry.get("url", ""))).path)):
suffix = pathlib.PurePosixPath(probe).suffix.lower().lstrip(".")
if suffix:
manifest["name"] = manifest_name + "." + suffix
break
local_path = entry.get("local_path", "")
if local_path:
candidate = pathlib.Path(local_path.replace("/home/zheromiku/dobot-portal-sync", "/sources/pp", 1))
if not candidate.is_file():
candidate = pathlib.Path(local_path)
if candidate.is_file():
size = candidate.stat().st_size
expected_hash = str(entry.get("sha256", "")).lower()
actual = sha256_file(candidate)
if expected_hash and len(expected_hash) == 64 and actual != expected_hash:
raise FetchError("PP local copy hash mismatch (ledger changed?)", retryable=True, code="local_mismatch")
return str(candidate), size, actual, True
url = entry.get("url", "")
if not url:
raise MissingSource("PP ledger entry has no local file and no download URL")
path, size, actual = http_get(url, {}, 180, max_bytes)
expected_hash = str(entry.get("sha256", "")).lower()
if expected_hash and len(expected_hash) == 64 and actual != expected_hash:
os.unlink(path)
raise FetchError("PP download hash mismatch", retryable=True, code="hash_mismatch")
return path, size, actual, False
# ------------------------------------------------------------ DingTalk -------
def load_capture_headers():
capture = json.loads(pathlib.Path("/run/source-secrets/dingtalk_capture.json").read_text(encoding="utf-8"))
headers = capture.get("headers") or {}
def header(name):
for key, value in headers.items():
if key.lower() == name:
return value
return ""
a_token = header("a-token")
xsrf = header("x-xsrf-token")
cookie = header("cookie")
uid = ""
match = re.search(r'"?uid"?[=:]"?(\d{4,})', cookie or "")
if match:
uid = match.group(1)
if not a_token:
raise FetchError("DingTalk capture has no a-token; re-capture in 来源与凭据", retryable=False, code="credentials")
return {"a-token": a_token, "x-xsrf-token": xsrf, "cookie": cookie, "uid": uid,
"referer": "https://alidocs.dingtalk.com/"}
def dingtalk_local_export(manifest):
"""Locate an already-exported copy; returns (path, fresher_than_source) or None."""
name = pathlib.PurePosixPath(manifest["relative_path"]).name
stem = pathlib.PurePosixPath(name).stem
raw_suffix = pathlib.PurePosixPath(name).suffix.lower().lstrip(".")
suffix = pick_suffix(name, manifest.get("declared_type", ""))
root = pathlib.Path("/sources/dingtalk/exported")
folder = str(pathlib.PurePosixPath(manifest["relative_path"]).parent).strip(".")
search_suffixes = [suffix] if raw_suffix not in SLIDES else ["pdf", "pptx", "ppt"]
candidates = []
for probe in search_suffixes:
if probe is None:
continue
exact = root / manifest["source_name"] / folder / (stem + "." + probe)
if exact.is_file():
candidates.append(exact)
else:
for base in (root / manifest["source_name"], pathlib.Path("/sources/dingtalk/converted_ppt_pdf"),
pathlib.Path("/sources/dingtalk/converted_ppt_pdf_full")):
if not base.is_dir():
continue
for hit in base.rglob(stem + "." + probe):
candidates.append(hit)
if candidates:
break
if not candidates:
if raw_suffix not in SLIDES and suffix is None:
raise MissingSource("dingtalk file type is not ingestable")
return None
modified = str(manifest.get("modified_at") or "")
candidate = candidates[0]
local_modified = time.strftime("%Y-%m-%dT%H:%M:%S+08:00", time.localtime(candidate.stat().st_mtime))
fresh = (not modified) or (local_modified >= modified[:19] + local_modified[19:])
return str(candidate), fresh
def dingtalk_raw_keys(manifest):
"""Recover the dentry uuid behind a node id via the scanner inventory."""
state_path = pathlib.Path("/sources/dingtalk/data/dingtalk-internal/inventory-state.json")
state = json.loads(state_path.read_text(encoding="utf-8"))
roots = state.get("roots") or {}
space_key = _space_key(roots, manifest["source_name"])
match = [value for key, value in sorted(roots.items()) if key == space_key]
if len(match) != 1:
raise FetchError("DingTalk space not present in the scanner inventory; refresh 来源目录 first",
retryable=False, code="space_missing")
sys.path.insert(0, "/app/scripts")
from source_catalog_import import node_id as catalog_node_id # noqa
source_id = "dingtalk-" + space_key
nodes = match[0].get("nodes") or {}
for raw, item in nodes.items():
if catalog_node_id(source_id, raw) == manifest["node_id"]:
return raw, str(item.get("dentry_type") or item.get("type") or "")
raise MissingSource("DingTalk node not found in current inventory (moved or deleted?)")
def _space_key(roots, name):
for key, value in roots.items():
if value.get("live_space_name") == name:
return key
return ""
def dingtalk_download(manifest, max_bytes):
export = dingtalk_local_export(manifest)
modified = str(manifest.get("modified_at") or "")
if export:
path, fresh = export
size = os.path.getsize(path)
if size > 0 and (fresh or not modified):
return path, size, sha256_file(path), True
# stale local export -> fall through to a live export when available
raw_key, declared_type = dingtalk_raw_keys(manifest)
headers = load_capture_headers()
live_path = live_export_dingtalk_document(raw_key, headers, manifest, max_bytes)
if export and not live_path:
path, fresh = export
return path, os.path.getsize(path), sha256_file(path), True # keep old bytes; staleness shown in UI
if not live_path:
raise MissingSource("DingTalk file is not exported and cannot be exported automatically")
return live_path
def live_export_dingtalk_document(raw_key, headers, manifest, max_bytes):
opener = urllib.request.build_opener()
def api(url, payload=None):
request = urllib.request.Request(url, headers={
"a-token": headers["a-token"], "x-xsrf-token": headers.get("x-xsrf-token", ""),
"cookie": headers.get("cookie", ""), "content-type": "application/json",
"referer": headers["referer"], "user-agent": "Mozilla/5.0",
}, data=json.dumps(payload).encode() if payload is not None else None)
try:
with opener.open(request, timeout=60) as response:
return json.loads(response.read().decode("utf-8"))
except (urllib.error.URLError, urllib.error.HTTPError, TimeoutError) as error:
raise FetchError("DingTalk web API unavailable (%s)" % type(error).__name__, code="source_unavailable")
info = api("https://alidocs.dingtalk.com/box/api/v2/dentry/info?dentryUuid=" + urllib.parse.quote(raw_key))
data = info.get("data") or {}
space_id, dentry_id = str(data.get("spaceId") or ""), str(data.get("dentryId") or "")
if not space_id or not dentry_id:
raise MissingSource("DingTalk dentry not resolvable")
download = api("https://alidocs.dingtalk.com/box/api/v1/file/download?spaceId=%s&dentryId=%s" % (space_id, dentry_id))
payload = download.get("data") or {}
if payload.get("downloadType") == "URL_PRE_SIGNATURE":
urls = (payload.get("ossUrlPreSignatureInfo") or {}).get("preSignUrls") or []
if urls:
return http_get(urls[0], {}, 180, max_bytes) + (False,)
return None
# ---------------------------------------------------------------- main -------
def suffix_from_magic(path):
with open(path, "rb") as handle:
head = handle.read(12)
if head.startswith(b"%PDF-"):
return "pdf"
if head.startswith(b"PK\x03\x04"):
# zip container: peek one entry name to pick office family
try:
import zipfile
with zipfile.ZipFile(path) as archive:
names = set(archive.namelist()[:50])
if "word/document.xml" in names:
return "docx"
if "xl/workbook.xml" in names:
return "xlsx"
except Exception:
return None
return None
if head.startswith(b"\xd0\xcf\x11\xe0"):
return None
for suffix in ("png", "jpg", "gif", "bmp", "webp"):
checks = validate_magic_map(head)
if checks.get(suffix):
return suffix
try:
with open(path, "rb") as handle:
handle.read(4096).decode("utf-8", errors="strict")
return "md"
except (UnicodeDecodeError, OSError):
return None
def validate_magic_map(head):
return {
"png": head.startswith(b"\x89PNG\r\n\x1a\n"),
"jpg": head[:2] == b"\xff\xd8" and head[2:3] == b"\xff",
"gif": head[:6] in (b"GIF87a", b"GIF89a"),
"bmp": head[:2] == b"BM",
"webp": head[:4] == b"RIFF" and head[8:12] == b"WEBP",
}
def main():
manifest = json.loads(sys.argv[1])
max_bytes = int(os.environ.get("ARTIFACT_FETCH_MAX_BYTES") or MAX_BYTES_DEFAULT)
spool_root = pathlib.Path(os.environ.get("INGESTION_SPOOL_ROOT") or "/var/lib/governance/ingestion")
name = manifest["name"]
declared_type = manifest.get("declared_type", "")
matched_local = False
if manifest["source_type"] == "alist":
path, size, digest = alist_download(manifest, max_bytes)
elif manifest["source_type"] == "pp":
path, size, digest, matched_local = pp_download(manifest, max_bytes)
elif manifest["source_type"] == "dingtalk":
path, size, digest, matched_local = dingtalk_download(manifest, max_bytes)
else:
raise FetchError("unknown source type", retryable=False, code="bad_manifest")
# Type is decided on the RESOLVED bytes, not the catalog title: PP ledger
# titles may omit extensions and DingTalk exports always change them.
suffix = pick_suffix(pathlib.PurePosixPath(path).name, declared_type) or pick_suffix(name, declared_type)
if suffix is None:
suffix = suffix_from_magic(path)
if suffix is None:
raw_suffix = pathlib.PurePosixPath(name).suffix.lower().lstrip(".")
hint = " (convert PPT to PDF first)" if raw_suffix in SLIDES else ""
raise FetchError("file type .%s is not supported by the parser%s" % (raw_suffix or "?", hint),
retryable=False, code="unsupported_format")
try:
validate_magic(path, suffix)
if size > max_bytes:
raise FetchError("artifact exceeds ingest size limit", retryable=False, code="over_limit")
relative_ref = "artifact/%s/%s/%s.%s" % (digest[:2], digest[2:4], manifest["node_id"], suffix)
destination = spool_root / relative_ref
destination.parent.mkdir(parents=True, exist_ok=True)
staged = str(destination) + ".part"
shutil.copyfile(path, staged)
if sha256_file(staged) != digest:
raise FetchError("spooled bytes failed verification", code="spool_mismatch")
os.replace(staged, destination)
result = {"ok": True, "spec_id": manifest["spec_id"], "node_id": manifest["node_id"],
"connection_id": manifest["connection_id"], "sha256": digest, "size_bytes": size,
"spool_ref": os.environ.get("SPOOL_REF_PREFIX", "") + relative_ref, "validated_format": suffix, "matched_local": matched_local}
finally:
if path.startswith(tempfile.gettempdir()) and os.path.exists(path):
os.unlink(path)
emit(result)
return 0
if __name__ == "__main__":
try:
sys.exit(main())
except MissingSource as error:
emit({"ok": False, "missing": True, "error": str(error)})
sys.exit(3)
except FetchError as error:
emit({"ok": False, "retryable": error.retryable, "code": error.code, "error": str(error)})
sys.exit(4 if error.retryable else 2)
+1
View File
@@ -607,6 +607,7 @@ export function JobDashboard({ jobs, loading, error, onRefresh }: JobDashboardPr
}
const processingLabels: Record<ProcessingState, string> = {
planned: '待入库',
pending: '待处理',
processing: '解析中',
finalizing: '定稿中',
+1 -1
View File
@@ -8,7 +8,7 @@ import type { SourceConnectionListItem } from './sourceConnections'
const monitor:MonitorOverview={mode:'simulation',simulated:true,observed_at:'2026-09-08T12:00:00Z',host:{cpu_percent:42,memory_used_bytes:1,memory_total_bytes:2,disk_used_bytes:1,disk_total_bytes:2,gpu_state:'unknown',gpu_detail:'未接入真实GPU监控',simulated:true},services:[{key:'mineru',name:'MinerU',state:'healthy',detail:'模拟',latency_ms:1,observed_at:'2026-09-08T12:00:00Z',simulated:true}],knowledge_bases:[],health:[{key:'mineru',name:'MinerU',liveness:'up',readiness:'blocked',reason:'服务可达但容量已满(429),暂时不可接单',latency_ms:30,observed_at:'2026-09-08T12:00:00Z',last_success_at:'2026-09-08T11:59:00Z',consecutive_failures:2,configured:true,applicable:true}]}
const sources:CatalogSource[]=[{connection_id:'ding-1',source_type:'dingtalk',name:'公司级技术支持知识库',root_node_id:'root-1',folder_count:1079,file_count:3444,observed_at:'2026-09-09T09:30:32Z',snapshot_only:true,snapshot_status:'ready',last_refresh_duration_ms:0}]
const connections:SourceConnectionListItem[]=[{id:'conn-ding',source_type:'dingtalk',name:'公司级技术支持知识库',enabled:true,schedule_configured:true,health_status:'healthy',item_count:3444,updated_at:'2026-09-09T09:30:32Z',next_refresh_at:'2026-09-12T10:00:00Z'},{id:'conn-man',source_type:'manual',name:'本地手工验收',enabled:false,schedule_configured:false,health_status:'unknown',item_count:5,updated_at:'2026-09-09T09:30:32Z',next_refresh_at:null}]
const settings:UISettings={id:'settings-default',state:'active',document_refresh_seconds:15,resource_refresh_seconds:30,stalled_after_seconds:1800,finalizing_after_seconds:7200,updated_by:'system:migration',created_at:'2026-09-10T08:00:00Z',activated_at:'2026-09-10T08:00:00Z',retired_at:null,error_code:''}
const settings:UISettings={id:'settings-default',state:'active',document_refresh_seconds:15,resource_refresh_seconds:30,stalled_after_seconds:1800,finalizing_after_seconds:7200,ingestion_concurrency:2,updated_by:'system:migration',created_at:'2026-09-10T08:00:00Z',activated_at:'2026-09-10T08:00:00Z',retired_at:null,error_code:''}
describe('SystemSettingsView',()=>{
it('separates document-engineer settings from locked administrator and production controls',()=>{
const html=renderToStaticMarkup(<SystemSettingsView sources={sources} connections={connections} monitor={monitor} settings={settings} productionEnabled={false} routingCapabilities={{internal_enabled:true,external_enabled:true}} loading={false} saving={false} error={null} saveMessage="刷新与告警参数已验证并启用:文档15秒、资源30秒、停滞30分钟、finalizing 2小时" onSave={vi.fn()} onSourcesChanged={vi.fn()} onRefresh={vi.fn()} />)
+6 -3
View File
@@ -138,6 +138,7 @@ export function SystemSettingsView({ sources, connections, monitor, settings, pr
const [resourceRefresh, setResourceRefresh] = useState<30 | 60>(settings?.resource_refresh_seconds ?? 30)
const [stalled, setStalled] = useState<1800 | 3600>(settings?.stalled_after_seconds ?? 1800)
const [finalizing, setFinalizing] = useState<7200 | 14400>(settings?.finalizing_after_seconds ?? 7200)
const [concurrency, setConcurrency] = useState<number>(settings?.ingestion_concurrency ?? 2)
const [dirty, setDirty] = useState(false)
useEffect(() => {
@@ -146,6 +147,7 @@ export function SystemSettingsView({ sources, connections, monitor, settings, pr
setResourceRefresh(settings.resource_refresh_seconds)
setStalled(settings.stalled_after_seconds)
setFinalizing(settings.finalizing_after_seconds)
setConcurrency(settings.ingestion_concurrency)
setDirty(false)
}, [settings])
@@ -183,10 +185,11 @@ export function SystemSettingsView({ sources, connections, monitor, settings, pr
<label>资源刷新间隔<select value={resourceRefresh} onChange={(event) => { setResourceRefresh(Number(event.target.value) as 30 | 60); setDirty(true) }}><option value={30}>30秒</option><option value={60}>1分钟</option></select></label>
<label>解析停滞判定<select value={stalled} onChange={(event) => { setStalled(Number(event.target.value) as 1800 | 3600); setDirty(true) }}><option value={1800}>30分钟</option><option value={3600}>1小时</option></select></label>
<label>finalizing 告警<select value={finalizing} onChange={(event) => { setFinalizing(Number(event.target.value) as 7200 | 14400); setDirty(true) }}><option value={7200}>2小时</option><option value={14400}>4小时</option></select></label>
<label>入库并发上限<select value={concurrency} onChange={(event) => { setConcurrency(Number(event.target.value)); setDirty(true) }}><option value={1}>1(谨慎,MinerU 并发 1)</option><option value={2}>2</option><option value={3}>3</option><option value={4}>4</option><option value={6}>6</option><option value={8}>8</option></select></label>
</div>
<button onClick={() => { setDirty(false); onSave({ document_refresh_seconds: documentRefresh, resource_refresh_seconds: resourceRefresh, stalled_after_seconds: stalled, finalizing_after_seconds: finalizing }) }} disabled={!canSave}>{saving ? '正在验证并应用…' : '保存并应用刷新与告警参数'}</button>
<button onClick={() => { setDirty(false); onSave({ document_refresh_seconds: documentRefresh, resource_refresh_seconds: resourceRefresh, stalled_after_seconds: stalled, finalizing_after_seconds: finalizing, ingestion_concurrency: concurrency }) }} disabled={!canSave}>{saving ? '正在验证并应用…' : '保存并应用刷新与告警参数'}</button>
{visibleSaveMessage(saveMessage, dirty) ? <p className="success-message" role="status">{saveMessage}</p> : null}
<small>当前:文档 {secondsLabel(settings.document_refresh_seconds)}、资源 {secondsLabel(settings.resource_refresh_seconds)}、停滞 {secondsLabel(settings.stalled_after_seconds)}、finalizing {secondsLabel(settings.finalizing_after_seconds)}。</small>
<small>当前:文档 {secondsLabel(settings.document_refresh_seconds)}、资源 {secondsLabel(settings.resource_refresh_seconds)}、停滞 {secondsLabel(settings.stalled_after_seconds)}、finalizing {secondsLabel(settings.finalizing_after_seconds)}、入库并发 {settings.ingestion_concurrency}。</small>
</section> : null}
</div>
</div> : null}
@@ -214,7 +217,7 @@ export function SystemSettings() {
const save = async (input: UISettingsCandidateInput) => {
setSaving(true); setSaveMessage(null); setError(null)
try { const candidate = await createUISettingsCandidate(input); const active = await activateUISettings(candidate.id); setSettings(active); setSaveMessage(`刷新与告警参数已验证并启用:文档${secondsLabel(active.document_refresh_seconds)}、资源${secondsLabel(active.resource_refresh_seconds)}、停滞${secondsLabel(active.stalled_after_seconds)}、finalizing ${secondsLabel(active.finalizing_after_seconds)};旧版本已保留用于审计。`) }
try { const candidate = await createUISettingsCandidate(input); const active = await activateUISettings(candidate.id); setSettings(active); setSaveMessage(`刷新与告警参数已验证并启用:文档${secondsLabel(active.document_refresh_seconds)}、资源${secondsLabel(active.resource_refresh_seconds)}、停滞${secondsLabel(active.stalled_after_seconds)}、finalizing ${secondsLabel(active.finalizing_after_seconds)}、入库并发 ${active.ingestion_concurrency};旧版本已保留用于审计。`) }
catch (cause) { setError(cause instanceof Error ? cause.message : '保存系统设置失败') }
finally { setSaving(false) }
}
+22 -31
View File
@@ -3,15 +3,13 @@ import { describe,expect,it,vi } from 'vitest'
import { WeKnoraMonitorView } from './WeKnoraMonitor'
import type { MonitorFilePage,MonitorOverview,RunMetrics } from './monitor'
import type { TestPublicationPage } from './testPublications'
import type { CatalogChangeSetDetail } from './sourceCatalog'
const overview:MonitorOverview={mode:'simulation',simulated:true,observed_at:'2026-09-08T12:00:00Z',host:{cpu_percent:42,memory_used_bytes:19971597926,memory_total_bytes:34359738368,disk_used_bytes:128849018880,disk_total_bytes:536870912000,gpu_state:'unknown',gpu_detail:'未接入真实GPU监控',simulated:true},services:[{key:'weknora',name:'WeKnora API',state:'healthy',detail:'模拟连接正常',latency_ms:86,observed_at:'2026-09-08T12:00:00Z',simulated:true},{key:'mineru',name:'MinerU',state:'healthy',detail:'模拟处理中2,队列5',latency_ms:24,observed_at:'2026-09-08T12:00:00Z',simulated:true},{key:'postgres',name:'PostgreSQL',state:'unknown',detail:'未接入真实监控',latency_ms:0,observed_at:'2026-09-08T12:00:00Z',simulated:true},{key:'redis',name:'Redis / MQ',state:'unknown',detail:'未接入真实监控',latency_ms:0,observed_at:'2026-09-08T12:00:00Z',simulated:true},{key:'oss',name:'OSS',state:'unknown',detail:'未接入真实监控',latency_ms:0,observed_at:'2026-09-08T12:00:00Z',simulated:true}],knowledge_bases:[{id:'kb-sim',name:'公司级技术支持知识库',visibility:'mixed',imported:5750,completed:4530,pending:450,processing:50,finalizing:714,failed:6,cancelled:0,internal_imported:4200,internal_completed:3300,external_imported:1550,external_completed:1230}],health:[]}
const overview:MonitorOverview={mode:'simulation',simulated:true,observed_at:'2026-09-08T12:00:00Z',host:{cpu_percent:42,memory_used_bytes:19971597926,memory_total_bytes:34359738368,disk_used_bytes:128849018880,disk_total_bytes:536870912000,gpu_state:'unknown',gpu_detail:'未接入真实GPU监控',simulated:true},services:[{key:'weknora',name:'WeKnora API',state:'healthy',detail:'模拟连接正常',latency_ms:86,observed_at:'2026-09-08T12:00:00Z',simulated:true},{key:'mineru',name:'MinerU',state:'healthy',detail:'模拟处理中2,队列5',latency_ms:24,observed_at:'2026-09-08T12:00:00Z',simulated:true},{key:'postgres',name:'PostgreSQL',state:'unknown',detail:'未接入真实监控',latency_ms:0,observed_at:'2026-09-08T12:00:00Z',simulated:true},{key:'redis',name:'Redis / MQ',state:'unknown',detail:'未接入真实监控',latency_ms:0,observed_at:'2026-09-08T12:00:00Z',simulated:true},{key:'oss',name:'OSS',state:'unknown',detail:'未接入真实监控',latency_ms:0,observed_at:'2026-09-08T12:00:00Z',simulated:true}],knowledge_bases:[{id:'kb-sim',name:'公司级技术支持知识库',visibility:'mixed',imported:5750,completed:4530,pending:450,processing:50,finalizing:714,failed:6,cancelled:0,planned:120,needs_removal:3,stale_planned:0,internal_imported:4200,internal_completed:3300,external_imported:1550,external_completed:1230}],pipeline:{fetch_pending:4,fetch_failed:1,fetch_missing:0,concurrency:2},health:[]}
const files:MonitorFilePage={items:[{name:'[模拟] 产品参数总表.xlsx',directory:'02|产品资料',visibility:'external',status:'finalizing',stage_detail:'基础检索可用,Wiki与图谱仍在生成',elapsed_seconds:2100,error_code:''}],total:1,limit:20,offset:0,sample_only:true}
const metrics:RunMetrics={observed_at:'2026-09-12T00:00:00Z',active_workers:2,worker_source:'pg_lease',queue:{pending:3,running:4,retry:1,failed:2,oldest_pending_age_seconds:120},short_window:{window_seconds:3600,jobs_started:50,jobs_completed:48,jobs_failed:6,success_rate:88.9,failure_rate:11.1,duration_avg_seconds:55,duration_p50_seconds:30,duration_p95_seconds:1800},long_window:{window_seconds:86400,jobs_started:500,jobs_completed:482,jobs_failed:18,success_rate:96.4,failure_rate:3.6,duration_avg_seconds:70,duration_p50_seconds:45,duration_p95_seconds:2400}}
const queue:TestPublicationPage={items:[{id:'tp-plan',title:'API.docx',ingest_state:'pending',processing_state:'pending',retrieval_state:'unpublished',target_scope: 'internal', change_set_id: 'cs-1', change_type:'added',can_start:true,retry_count:0,last_error:'',can_retry:false,updated_at:'2026-09-09T10:01:00Z'},{id:'tp-failed',title:'CR20A.pdf',ingest_state:'uploaded',processing_state:'failed',retrieval_state:'unpublished',target_scope: 'external', change_set_id: 'cs-1', change_type:'updated',can_start:false,retry_count:1,last_error:'remote_parse_failed',can_retry:true,updated_at:'2026-09-09T10:00:00Z'}],total:2,limit:50,offset:0}
const changeSets:CatalogChangeSetDetail[]=[{id:'cs-1',connection_id:'conn-1',source_name:'国内工业',state:'planned',add_count:2,update_count:1,move_count:1,move_scope_count:1,delete_count:1,no_change_count:10,created_at:'2026-09-10T00:00:00Z',items:[{action:'delete',title:'旧手册.pdf',old_path:'旧目录/旧手册.pdf',new_path:'',reason_code:'source_missing',seq:7}],total:15,limit:20,offset:0}]
const baseProps={overview,files,queue,changeSets,selectedKB:overview.knowledge_bases[0],status:'' as const,queueStatus:'' as const,queueScope:'' as const,loading:false,error:null as string|null,retryingIDs:[] as string[],starting:false,startMessage:'',onSelectKB:vi.fn(),onStatus:vi.fn(),onQueueStatus:vi.fn(),onQueueScope:vi.fn(),lane:'queue',onLaneChange:vi.fn(),metrics,onRetry:vi.fn(),onBatchRetry:vi.fn(),onStart:vi.fn(),onRefresh:vi.fn()}
const queue:TestPublicationPage={items:[{id:'tp-plan',title:'API.docx',ingest_state:'pending',processing_state:'planned',retrieval_state:'unpublished',target_scope: 'internal', change_set_id: '', change_type:'added',can_start:true,retry_count:0,last_error:'',can_retry:false,updated_at:'2026-09-09T10:01:00Z'},{id:'tp-fetch',title:'焊接手册.pdf',ingest_state:'pending',processing_state:'planned',retrieval_state:'unpublished',target_scope:'external',change_set_id:'',change_type:'added',can_start:false,retry_count:0,last_error:'',can_retry:false,updated_at:'2026-09-09T10:02:00Z'},{id:'tp-failed',title:'CR20A.pdf',ingest_state:'uploaded',processing_state:'failed',retrieval_state:'unpublished',target_scope: 'external', change_set_id: '', change_type:'updated',can_start:false,retry_count:1,last_error:'remote_parse_failed',can_retry:true,updated_at:'2026-09-09T10:00:00Z'}],total:3,limit:50,offset:0}
const plans=queue.items.filter((item)=>item.can_start)
const baseProps={overview,files,queue,plans,selectedKB:overview.knowledge_bases[0],status:'' as const,queueStatus:'' as const,queueScope:'' as const,loading:false,error:null as string|null,retryingIDs:[] as string[],starting:false,startMessage:'',onSelectKB:vi.fn(),onStatus:vi.fn(),onQueueStatus:vi.fn(),onQueueScope:vi.fn(),lane:'queue',onLaneChange:vi.fn(),metrics,onRetry:vi.fn(),onBatchRetry:vi.fn(),onStart:vi.fn(),onRefresh:vi.fn()}
describe('WeKnoraMonitorView',()=>{
it('filters the processing queue by target knowledge base and shows each task target',()=>{
const html=renderToStaticMarkup(<WeKnoraMonitorView {...baseProps} queueScope="internal" onQueueScope={vi.fn()} />)
@@ -20,42 +18,35 @@ describe('WeKnoraMonitorView',()=>{
expect(html).toContain('对外知识库</option>')
expect(html).toContain('<option value="internal" selected="">对内知识库')
expect(html).toContain('目标知识库')
// the one internal task row shows its target (select option matches are not counted)
expect(html).toContain('scope-badge internal">对内')
expect(html).toContain('scope-badge external">对外')
})
it('offers explicit change-set confirmation with selection and result message (actor comes from session)',()=>{
const html=renderToStaticMarkup(<WeKnoraMonitorView {...baseProps} lane='changes' changeSets={changeSets} confirmingSetID='cs-1' confirmMessage='已提交:确认 2 项;新建上传任务 2 个' onConfirmChangeSet={vi.fn()} />)
expect(html).toContain('预览并选择变更文件')
expect(html).toContain('aria-label="选择旧手册.pdf"')
expect(html).toContain('确认处理(0)')
expect(html).not.toContain('操作人<input')
expect(html).toContain('已提交:确认 2 项;新建上传任务 2 个')
it('treats the catalog annotation as the only decision: 待入库 lists plans with one 开始处理 button and no reason form',()=>{
const waiting=renderToStaticMarkup(<WeKnoraMonitorView {...baseProps} lane="waiting" onStart={vi.fn()} />)
expect(waiting).toContain('待入库(2)')
expect(waiting).toContain('开始处理(1)')
expect(waiting).toContain('并发上限(当前 2)')
expect(waiting).not.toContain('确认原因')
expect(waiting).not.toContain('操作人<input')
expect(waiting).toContain('正在从源头取回文件')
})
it('hides confirmation controls after the change set leaves planned state',()=>{
const confirmed=[{...changeSets[0],state:'confirmed' as const}]
const html=renderToStaticMarkup(<WeKnoraMonitorView {...baseProps} lane='changes' changeSets={confirmed} onConfirmChangeSet={vi.fn()} />)
expect(html).toContain('已确认')
expect(html).not.toContain('确认处理')
expect(html).toContain('aria-label="选择旧手册.pdf" disabled')
it('shows knowledge-base removal demand from blocked-but-live items',()=>{
const removal=renderToStaticMarkup(<WeKnoraMonitorView {...baseProps} lane="removal" />)
expect(removal).toContain('需下架(3)')
expect(removal).toContain('3 份仍在检索中')
})
it('shows a plainly labelled simulation with per-lane inbox controls',()=>{
const changes=renderToStaticMarkup(<WeKnoraMonitorView {...baseProps} lane='changes' />)
for(const text of ['本地模拟数据','持久快照差异','国内工业','新增 2','更新 1','移动 1','删除 1','未变化 10','旧手册.pdf','确认原因','开始处理(1)','刷新来源 ≠ 马上导入知识库','不代表测试服务器真实数据'])expect(changes).toContain(text)
expect(changes).not.toContain('操作人<input')
it('shows inbox lanes for the knowledge-base diff view',()=>{
const attention=renderToStaticMarkup(<WeKnoraMonitorView {...baseProps} lane='attention' />)
for(const text of ['需要你出手','CR20A.pdf','tp-failed'.slice(0,0)+'remote_parse_failed','失败阶段 文档解析','批量重试所选(1)','死信(尝试耗尽,不可自动重试)','下载 ✓','文档解析 ✗','建议先在 WeKnora 手动解析该文件确认格式'])expect(attention).toContain(text)
for(const text of ['需要你出手','CR20A.pdf','remote_parse_failed','失败阶段 文档解析','批量重试所选(1)','死信(尝试耗尽,不可自动重试)'])expect(attention).toContain(text)
const inflight=renderToStaticMarkup(<WeKnoraMonitorView {...baseProps} lane='inflight' />)
for(const text of ['处理中','知识库解析进度','已导入 5750','completed 4530','pending 450','finalizing 714','对内 3300 / 4200','对外 1230 / 1550','[模拟] 产品参数总表.xlsx','基础检索可用,Wiki与图谱仍在生成','下载 → Hash → OSS → 文档解析 → WeKnora 上传 → 索引 → 完成'])expect(inflight).toContain(text)
for(const text of ['处理中','知识库解析进度','已导入 5750','待入库 120','对内 3300 / 4200','对外 1230 / 1550','源头取件 → Hash'])expect(inflight).toContain(text)
const queueView=renderToStaticMarkup(<WeKnoraMonitorView {...baseProps} lane='queue' />)
expect(queueView).toContain('全部任务(2)')
expect(queueView).toContain('确认并解析')
expect(queueView).toContain('全部任务(3)')
expect(queueView).toContain('重试')
expect(queueView).toContain('等待人工确认')
expect(queueView).toContain('1小时·成功')
expect(queueView).toContain('88.9%')
const nav=renderToStaticMarkup(<WeKnoraMonitorView {...baseProps} lane='pending' />)
for(const text of ['今天需要我做什么','需人工处理','等待我确认的差异','排队等待','已完成','全部任务'])expect(nav).toContain(text)
const nav=renderToStaticMarkup(<WeKnoraMonitorView {...baseProps} lane='waiting' />)
for(const text of ['待入库','需人工处理','需下架','处理中','已完成','全部任务'])expect(nav).toContain(text)
})
it('does not show simulated success totals after a load error without data',()=>{
const html=renderToStaticMarkup(<WeKnoraMonitorView overview={null} files={null} selectedKB={null} status="" queueStatus="" queueScope="" loading={false} error="加载失败" onSelectKB={vi.fn()} onStatus={vi.fn()} onQueueStatus={vi.fn()} onQueueScope={vi.fn()} onRefresh={vi.fn()} />)
+62 -97
View File
@@ -6,8 +6,6 @@ import type { KnowledgeProgress, MonitorFilePage, MonitorOverview, MonitorStatus
import { fetchJobFailures, fetchTestPublications, retryPublication, retryPublicationsBatch, startTestPublications } from './testPublications'
import type { JobFailure, ProcessingState, TestPublication, TestPublicationPage } from './testPublications'
import { fetchActiveUISettings, settingsRefreshDelays } from './uiSettings'
import { confirmCatalogChangeSet, fetchCatalogSources, fetchLatestCatalogChangeSet } from './sourceCatalog'
import type { CatalogChangeSetDetail } from './sourceCatalog'
import { WorkspaceContextTabsSetter } from './workspaceContext'
import './weknoraMonitor.css'
@@ -15,15 +13,15 @@ const emptyQueue: TestPublicationPage = { items: [], total: 0, limit: 50, offset
const gb = (value: number) => `${(value / 1073741824).toFixed(1)}`
const scope = (value: string) => value === 'internal' ? '对内' : value === 'external' ? '对外' : '混合'
const duration = (seconds: number) => seconds <= 0 ? '完成' : seconds < 60 ? `${seconds}秒` : seconds < 3600 ? `${Math.floor(seconds / 60)}分钟` : `${(seconds / 3600).toFixed(1)}小时`
const serviceState = (state: string) => state === 'healthy' ? '正常' : state === 'degraded' ? '异常' : state === 'unknown' ? '未接入' : '不可用'
const statusLabels: Record<MonitorStatus, string> = { completed: 'completed', pending: 'pending', processing: 'processing', finalizing: 'finalizing', failed: 'failed', cancelled: 'cancelled' }
const processingLabels: Record<ProcessingState, string> = { pending: '等待解析', processing: '解析中', finalizing: '索引与增强', completed: '成功', failed: '失败', stalled: '停滞' }
const processingLabels: Record<ProcessingState, string> = { planned: '待入库', pending: '等待解析', processing: '解析中', finalizing: '索引与增强', completed: '成功', failed: '失败', stalled: '停滞' }
const secondsSince = (iso: string) => Math.max(0, Math.round((Date.now() - new Date(iso).getTime()) / 1000))
const adviceFor = (stage: string, code: string) => {
if (stage === '文档解析') return '建议先在 WeKnora 手动解析该文件确认格式,再决定是否重试。'
if (stage === '索引 / 增强') return '多为 MinerU 容量或超时问题,确认服务恢复接单后重试。'
if (stage === 'WeKnora 上传') return '上传失败,确认 WeKnora 可达后重试;重复提交不会创建第二份知识。'
if (stage === '源头取件') return '文件尚未从源头取回:钉钉来源需可导出的登录会话,AList/PP 需直连可用;恢复后系统会自动重试。'
if (stage === 'Hash / 文件校验' || stage === 'OSS') return '文件或暂存校验问题,重新入队会重新拉取来源文件。'
if (stage === '测试知识移除') return '移除失败,请先在测试知识库确认该文档当前状态。'
if (stage === '治理策略') return '按治理策略阻止,请先在来源目录修正对内/对外标注。'
@@ -31,25 +29,26 @@ const adviceFor = (stage: string, code: string) => {
return '确认错误码后可直接重试;尝试次数耗尽将进入死信队列。'
}
const pipelineStages = ['下载', 'Hash', 'OSS', '文档解析', 'WeKnora 上传', '索引', '完成']
const pipelineStages = ['源头取件', 'Hash', 'OSS', '文档解析', 'WeKnora 上传', '索引', '完成']
const stageIndex = (label: string) => label === 'Hash / 文件校验' ? 1 : label === 'OSS' ? 2 : label === '文档解析' ? 3 : label === 'WeKnora 上传' ? 4 : label === '索引 / 增强' || label === '测试知识移除' || label === '治理策略' || label === '处理链路' ? 5 : -1
function chainSteps(item: TestPublication) {
const broken = item.processing_state === 'failed' || item.processing_state === 'stalled'
const stopAt = broken ? stageIndex(failureStage(item)) : -1
const activeAt = item.processing_state === 'completed' ? 7 : item.processing_state === 'finalizing' ? 5 : item.processing_state === 'processing' ? 3 : 0
const activeAt = item.processing_state === 'completed' ? 7 : item.processing_state === 'finalizing' ? 5 : item.processing_state === 'processing' ? 3 : item.processing_state === 'planned' && !item.can_start ? 0 : item.processing_state === 'pending' ? 1 : 0
return pipelineStages.map((stage, index) => {
if (stopAt >= 0 && index === stopAt) return { stage, state: 'bad' as const }
if (stopAt >= 0) return { stage, state: index < stopAt ? 'done' as const : 'todo' as const }
if (index < activeAt) return { stage, state: 'done' as const }
if (index === activeAt && item.processing_state !== 'pending' && item.processing_state !== 'completed') return { stage, state: 'active' as const }
if (item.processing_state === 'pending' && index === 0) return { stage, state: 'active' as const }
if (index === activeAt && item.processing_state !== 'pending' && item.processing_state !== 'completed' && item.processing_state !== 'planned') return { stage, state: 'active' as const }
if ((item.processing_state === 'pending' || (item.processing_state === 'planned' && !item.can_start)) && index === 0) return { stage, state: item.can_start || item.processing_state === 'pending' ? 'done' as const : 'active' as const }
return { stage, state: 'todo' as const }
})
}
function failureStage(item: TestPublication) {
const code = item.last_error
if (code.includes('artifact') || code.includes('hash')) return 'Hash / 文件校验'
if (code.includes('artifact') || code.includes('fetch')) return '源头取件'
if (code.includes('hash')) return 'Hash / 文件校验'
if (code.includes('oss')) return 'OSS'
if (code.includes('upload')) return 'WeKnora 上传'
if (code.includes('parse')) return '文档解析'
@@ -64,10 +63,6 @@ interface ViewProps {
files: MonitorFilePage | null
queue?: TestPublicationPage
plans?: TestPublication[]
changeSets?: CatalogChangeSetDetail[]
confirmingSetID?: string | null
confirmMessage?: string
onConfirmChangeSet?: (changeSetID: string, itemSeqs: number[], reason: string) => void
selectedKB: KnowledgeProgress | null
status: MonitorStatus | ''
queueStatus?: ProcessingState | ''
@@ -88,33 +83,20 @@ interface ViewProps {
onLaneChange?: (lane: string) => void
onRetry?: (id: string) => void
onBatchRetry?: (ids: string[]) => void
onStart?: (ids: string[], reason: string) => void
onStart?: (ids: string[]) => void
onRefresh: () => void
}
function ChangeSetCard({changeSet,confirming,message,onConfirm}:{changeSet:CatalogChangeSetDetail;confirming:boolean;message:string;onConfirm?:(changeSetID:string,itemSeqs:number[],reason:string)=>void}){
const [open,setOpen]=useState(false)
const [selected,setSelected]=useState<number[]>([])
const actionable=changeSet.items.filter(item=>item.action!=='no_change')
const confirmable=changeSet.state==='planned'&&actionable.length>0&&Boolean(onConfirm)
const toggle=(seq:number)=>setSelected(current=>current.includes(seq)?current.filter(value=>value!==seq):[...current,seq])
return <article><div><strong>{changeSet.source_name}</strong><small>{new Date(changeSet.created_at).toLocaleString('zh-CN',{hour12:false})}</small>{changeSet.state!=='planned'?<span className="scope-badge internal">已确认</span>:null}</div><div className="persisted-change-counts"><span>新增 {changeSet.add_count}</span><span>更新 {changeSet.update_count}</span><span>移动 {changeSet.move_count}</span><span>范围变化 {changeSet.move_scope_count}</span><span>删除 {changeSet.delete_count}</span><span>未变化 {changeSet.no_change_count}</span></div>{actionable.length?<details open={open} onToggle={event=>setOpen((event.target as HTMLDetailsElement).open)}><summary>预览并选择变更文件</summary><div className="changeset-selection">{actionable.map(item=><label key={item.seq} className="changeset-choice"><input type="checkbox" aria-label={`选择${item.title}`} disabled={!confirmable} checked={selected.includes(item.seq)} onChange={()=>toggle(item.seq)} /><b>{item.action==='add'?'新增':item.action==='update'?'更新':item.action==='move'?'移动(改名)':item.action==='move_scope'?'范围变化':'删除'}</b><span>{item.title}</span><small>{item.new_path||item.old_path}</small>{item.action==='delete'&&item.reason_code==='source_missing'?<small>来源缺失</small>:null}</label>)}</div>{confirmable?<div className="changeset-confirm"><button disabled={confirming||selected.length===0} onClick={()=>onConfirm?.(changeSet.id,[...selected].sort((a,b)=>a-b),'人工确认快照差异执行')}>确认处理({selected.length})</button></div>:null}{message?<p role="status">{message}</p>:null}</details>:null}</article>
}
export function WeKnoraMonitorView({
overview,
files,
queue = emptyQueue,
plans = queue.items,
changeSets = [],
confirmingSetID = null,
confirmMessage = '',
onConfirmChangeSet,
selectedKB,
status,
queueStatus = '',
queueScope = '',
lane = 'attention',
lane = 'waiting',
onLaneChange = () => undefined,
loading,
error,
@@ -133,8 +115,9 @@ export function WeKnoraMonitorView({
onStart = () => undefined,
onRefresh,
}: ViewProps) {
const [startReason, setStartReason] = useState('已核对来源快照与测试知识库差异')
const [selectedRetryIDs, setSelectedRetryIDs] = useState<string[]>([])
const [selectedStartIDs, setSelectedStartIDs] = useState<string[]>([])
const [selectedRetryIDs, setSelectedRetryIDs] = useState<string[]>([])
const toggleStartSelection = (id: string) => setSelectedStartIDs((current) => current.includes(id) ? current.filter((value) => value !== id) : [...current, id])
const toggleRetrySelection = (id: string) => setSelectedRetryIDs((current) => current.includes(id) ? current.filter((value) => value !== id) : [...current, id])
const stages: Array<[MonitorStatus, number]> = selectedKB ? [
['completed', selectedKB.completed], ['pending', selectedKB.pending], ['finalizing', selectedKB.finalizing],
@@ -146,65 +129,73 @@ export function WeKnoraMonitorView({
}), [queue.items])
const retryableIDs = queue.items.filter((item) => item.can_retry).map((item) => item.id)
const batchTargets = selectedRetryIDs.length ? retryableIDs.filter((id) => selectedRetryIDs.includes(id)) : retryableIDs
const pendingPlans = plans.filter((item) => item.can_start)
const addedPlans = pendingPlans.filter((item) => item.change_type === 'added').length
const updatedPlans = pendingPlans.filter((item) => item.change_type === 'updated').length
const startablePlans = plans.filter((item) => item.can_start)
const startTargets = selectedStartIDs.length ? startablePlans.filter((item) => selectedStartIDs.includes(item.id)).map((item) => item.id) : startablePlans.map((item) => item.id)
const attentionItems = queue.items.filter((item) => item.processing_state === 'failed' || item.processing_state === 'stalled')
const inflightItems = queue.items.filter((item) => item.processing_state === 'processing' || item.processing_state === 'finalizing' || (item.processing_state === 'pending' && !item.can_start))
const doneTodayItems = queue.items.filter((item) => item.processing_state === 'completed')
const waitingItems = queue.items.filter((item) => item.processing_state === 'pending' && !item.can_start)
const laneCounts: Record<string, number> = { attention: attentionItems.length + failures.length, changes: changeSets.filter((set) => set.state === 'planned').length, inflight: inflightItems.length, pending: waitingItems.length, done: doneTodayItems.length, queue: queue.total }
const queueFiltered = waitingItems
const queueFilteredAll = orderedQueue
const doneItems = queue.items.filter((item) => item.processing_state === 'completed')
const plannedItems = queue.items.filter((item) => item.processing_state === 'planned')
const fetchingPlanned = plannedItems.filter((item) => !item.can_start)
const laneCounts: Record<string, number> = {
waiting: startablePlans.length + fetchingPlanned.length,
attention: attentionItems.length + failures.length,
removal: (overview?.knowledge_bases || []).reduce((total, kb) => total + (kb.needs_removal || 0), 0),
inflight: inflightItems.length,
done: doneItems.length,
queue: queue.total,
}
const laneTitles: Record<string, string> = { waiting: '待入库', attention: '需人工处理', removal: '需下架', inflight: '处理中', done: '已完成', queue: '全部任务' }
const actionTask = (item: TestPublication) => <article className="inbox-task" key={item.id}>
<div className="inbox-task-mark">{item.can_retry ? <input type="checkbox" aria-label={`选择重试${item.title}`} checked={selectedRetryIDs.includes(item.id)} onChange={() => toggleRetrySelection(item.id)} /> : null}</div>
<div className="inbox-task-mark">{item.can_start ? <input type="checkbox" aria-label={`选择入库${item.title}`} checked={selectedStartIDs.length === 0 || selectedStartIDs.includes(item.id)} onChange={() => toggleStartSelection(item.id)} /> : item.can_retry ? <input type="checkbox" aria-label={`选择重试${item.title}`} checked={selectedRetryIDs.includes(item.id)} onChange={() => toggleRetrySelection(item.id)} /> : null}</div>
<div>
<h4>{item.title} <span className={`scope-badge ${item.target_scope}`}>{scope(item.target_scope)}</span> {item.change_type === 'added' ? <span className="task-badge add">新增</span> : item.change_type === 'updated' ? <span className="task-badge update">更新</span> : null}</h4>
<div className="why">{item.processing_state === 'failed' || item.processing_state === 'stalled' ? <>{item.processing_state === 'stalled' ? '任务停滞:' : '为什么失败:'}错误码 <code>{item.last_error || '—'}</code> · 第{item.retry_count + 1}次尝试 · 已停留{duration(secondsSince(item.updated_at))} · 失败阶段 {failureStage(item)}</> : <>状态 <code>{item.can_start ? '等待人工确认' : processingLabels[item.processing_state]}</code>{item.last_error ? <> · 最近错误 <code>{item.last_error}</code></> : null} · 已停留{duration(secondsSince(item.updated_at))}</>}</div>
<div className="why">{item.processing_state === 'failed' || item.processing_state === 'stalled' ? <>{item.processing_state === 'stalled' ? '任务停滞:' : '为什么失败:'}错误码 <code>{item.last_error || '—'}</code> · 第{item.retry_count + 1}次尝试 · 已停留{duration(secondsSince(item.updated_at))} · 失败阶段 {failureStage(item)}</> : <>状态 <code>{processingLabels[item.processing_state]}</code>{item.last_error ? <> · 最近错误 <code>{item.last_error}</code></> : null} · 已停留{duration(secondsSince(item.updated_at))}</>}</div>
<div className="task-chain" aria-label={`处理链${item.title}`}>{chainSteps(item).map((step) => <span key={step.stage} className={`chain-step ${step.state}`}>{step.stage}{step.state === 'done' ? ' ✓' : step.state === 'bad' ? ' ✗' : step.state === 'active' ? ` 中 ${duration(secondsSince(item.updated_at))}` : ''}</span>)}</div>
<div className={item.processing_state === 'failed' || item.processing_state === 'stalled' ? 'errline' : 'tiny muted-line'}>{item.processing_state === 'failed' || item.processing_state === 'stalled' ? adviceFor(failureStage(item), item.last_error) : item.can_start ? '确认后将创建解析任务' : '无需操作,等待自动处理'}{item.change_set_id ? <> · 来源变更集 <code>{item.change_set_id}</code></> : null} · <code>{item.id}</code></div>
<div className={item.processing_state === 'failed' || item.processing_state === 'stalled' ? 'errline' : 'tiny muted-line'}>{item.processing_state === 'failed' || item.processing_state === 'stalled' ? adviceFor(failureStage(item), item.last_error) : item.processing_state === 'planned' && !item.can_start ? '正在从源头取回文件,完成后自动进入可处理状态' : item.can_start ? '文件已就绪,等待开始处理' : '无需操作,等待自动处理'} · <code>{item.id}</code></div>
</div>
<div className="inbox-task-actions">{item.can_start ? <button className="primary" disabled={starting || !startReason.trim()} onClick={() => onStart([item.id], startReason.trim())}>确认并解析</button> : null}{item.can_retry ? <button className="primary" disabled={retryingIDs.includes(item.id)} onClick={() => onRetry(item.id)}>{retryingIDs.includes(item.id) ? '重试中…' : '重试'}</button> : null}</div>
<div className="inbox-task-actions">{item.can_retry ? <button className="primary" disabled={retryingIDs.includes(item.id)} onClick={() => onRetry(item.id)}>{retryingIDs.includes(item.id) ? '重试中…' : '重试'}</button> : null}</div>
</article>
const emptyLane = (text: string) => <div className="empty-state">{text}</div>
return <main className="page-shell weknora-monitor inbox-layout">
<header className="hero compact"><div><h1>解析监控</h1><p className="subtitle">今天需要我做什么,一目了然;处理链进度、失败原因与建议直接摆在任务卡片上。</p></div><span className="monitor-meta">{overview ? `快照 ${new Date(overview.observed_at).toLocaleString('zh-CN', { hour12: false })}` : ''} · 队列 等待{queue.total}条</span><button onClick={onRefresh} disabled={loading}>{loading ? '刷新中…' : '刷新'}</button></header>
<header className="hero compact"><div><h1>解析监控</h1><p className="subtitle">站在知识库视角看差异:来源与目录里标好的对内/对外文件应不应该在这里,还差哪一步,一目了然。</p></div><span className="monitor-meta">{overview ? `快照 ${new Date(overview.observed_at).toLocaleString('zh-CN', { hour12: false })}` : ''} · 队列 等待{queue.total}条{overview?.pipeline ? ` · 取件中 ${overview.pipeline.fetch_pending}${overview.pipeline.fetch_failed ? ` · 取件失败 ${overview.pipeline.fetch_failed}` : ''}` : ''}</span><button onClick={onRefresh} disabled={loading}>{loading ? '刷新中…' : '刷新'}</button></header>
{error ? <div className="error-banner" role="alert">{error}</div> : null}
{overview?.simulated ? <div className="simulation-banner"><strong>本地模拟数据</strong><span>资源和知识库汇总用于确认页面流程,不代表测试服务器真实数据;任务卡片来自真实治理 API。</span></div> : null}
<div className="inbox-grid">
<nav className="inbox-nav" aria-label="解析监控收件箱">
<h2>今天需要我做什么</h2>
{[['attention', '需人工处理', 'blocked'], ['changes', '等待我确认的差异', ''], ['inflight', '处理中', ''], ['pending', '排队等待', ''], ['done', '已完成', 'okall']].map(([value, label, tone]) => <button key={value} className={`${lane === value ? 'active' : ''} ${laneCounts[value] ? tone : ''}`} aria-label={`切换到${label}`} onClick={() => onLaneChange(value)}><span>{label}</span><em className="lane-count">{laneCounts[value]}</em></button>)}
<h2>知识库差异</h2>
{[['waiting', '待入库', 'blocked'], ['attention', '需人工处理', ''], ['removal', '需下架', ''], ['inflight', '处理中', ''], ['done', '已完成', 'okall']].map(([value, label, tone]) => <button key={value} className={`${lane === value ? 'active' : ''} ${laneCounts[value] ? tone : ''}`} aria-label={`切换到${label}`} onClick={() => onLaneChange(value)}><span>{label}</span><em className="lane-count">{laneCounts[value]}</em></button>)}
<button className={`${lane === 'queue' ? 'active' : ''}`} aria-label="切换到全部任务" onClick={() => onLaneChange('queue')}><span>全部任务</span><em className="lane-count">{queue.total}</em></button>
</nav>
<div className="inbox-work">
{lane === 'waiting' ? <section className="inbox-section" aria-label="待入库">
<div className="section-head"><h2>待入库({startablePlans.length + fetchingPlanned.length})</h2><span className="spacer" /><button className="primary" disabled={starting || !startTargets.length} onClick={() => onStart(startTargets)}>{starting ? '正在生成任务…' : `开始处理(${startTargets.length})`}</button></div>
<p className="tiny">来源目录中标记对内/对外的文件会自动出现在这里;“开始处理”按系统设置的并发上限(当前 {overview?.pipeline?.concurrency ?? 2})创建解析任务,无需再确认。</p>
{startMessage ? <p className="difference-message" role="status">{startMessage}</p> : null}
{plannedItems.length ? plannedItems.map(actionTask) : emptyLane('还没有标记入库的文件。去“来源与目录”把文件或目录标为对内/对外即可。')}
{fetchingPlanned.length ? <p className="tiny">其中 {fetchingPlanned.length} 项仍在从源头取件,取回后自动可处理。</p> : null}
</section> : null}
{lane === 'attention' ? <section className="inbox-section" aria-label="需人工处理">
<div className="section-head"><h2>需要你出手({attentionItems.length})</h2><span className="spacer" /><button className="danger" disabled={!batchTargets.length || retryingIDs.length > 0} onClick={() => onBatchRetry(batchTargets)}>{retryingIDs.length ? '重试中…' : `批量重试所选(${batchTargets.length})`}</button></div>
{batchMessage ? <p className="queue-batch-message" role="status">{batchMessage}</p> : null}
{attentionItems.length ? attentionItems.map(actionTask) : emptyLane('没有失败或停滞的任务。')}
<div className="section-head sub"><h3>死信(尝试耗尽,不可自动重试)</h3><span className="tiny">原始任务与审计行保留不动;重新入队会创建新尝试</span></div>
{failures.length ? <ul className="dlq-list">{failures.map((failure) => <li key={failure.id}><span className="monitor-status failed">{failure.error_code}</span><b>{failure.job_type === 'test.upload' ? '上传解析' : failure.job_type}</b>{failure.publication_id ? <code>{failure.publication_id}</code> : null}<small>{failure.attempt}/{failure.max_attempts_at_failure} 次 · {new Date(failure.created_at).toLocaleString('zh-CN', { hour12: false })}</small>{failure.publication_id ? <button disabled={retryingIDs.includes(failure.publication_id)} onClick={() => onBatchRetry([failure.publication_id as string])}>重新入队</button> : null}</li>)}</ul> : emptyLane('当前没有等待处理的死信记录。')}
{failures.length ? <ul className="dlq-list">{failures.map((failure) => <li key={failure.id}><span className="monitor-status failed">{failure.error_code}</span><b>{failure.job_type === 'test.upload' ? '上传解析' : failure.job_type === 'artifact.fetch' ? '源头取件' : failure.job_type}</b>{failure.publication_id ? <code>{failure.publication_id}</code> : null}<small>{failure.attempt}/{failure.max_attempts_at_failure} 次 · {new Date(failure.created_at).toLocaleString('zh-CN', { hour12: false })}</small>{failure.publication_id ? <button disabled={retryingIDs.includes(failure.publication_id)} onClick={() => onBatchRetry([failure.publication_id as string])}>重新入队</button> : null}</li>)}</ul> : emptyLane('当前没有等待处理的死信记录。')}
</section> : null}
{lane === 'changes' ? <section className="inbox-section" aria-label="等待确认的差异">
<div className="section-head"><h2>持久快照差异 · 等待确认({changeSets.filter((set) => set.state === 'planned').length} 个来源)</h2><span className="tiny">刷新来源 ≠ 马上导入知识库;勾选变更项并确认后才创建解析任务</span></div>
{changeSets.length ? <div className="persisted-change-grid">{changeSets.map(changeSet => <ChangeSetCard key={changeSet.id} changeSet={changeSet} confirming={confirmingSetID === changeSet.id} message={confirmingSetID === changeSet.id ? confirmMessage : ''} onConfirm={onConfirmChangeSet} />)}</div> : emptyLane('当前没有持久快照差异。')}
<div className="difference-confirmation">
<label>确认原因<input value={startReason} onChange={(event) => setStartReason(event.target.value)} /></label>
<button disabled={starting || !pendingPlans.length || !startReason.trim()} onClick={() => onStart(pendingPlans.map((item) => item.id), startReason.trim())}>{starting ? '正在生成任务…' : `开始处理(${pendingPlans.length})`}</button>
</div>
<p className="tiny">待确认 {pendingPlans.length} · 新增 {addedPlans} · 更新 {updatedPlans}</p>
{startMessage ? <p className="difference-message" role="status">{startMessage}</p> : null}
{lane === 'removal' ? <section className="inbox-section" aria-label="需下架">
<div className="section-head"><h2>需下架({laneCounts.removal})</h2><span className="tiny">来源目录里被改为“禁止导入”或撤销标注、但仍在知识库中可检索的文件</span></div>
{(overview?.knowledge_bases || []).some((kb) => kb.needs_removal > 0) ? <ul className="dlq-list">{(overview?.knowledge_bases || []).filter((kb) => kb.needs_removal > 0).map((kb) => <li key={kb.id}><span className="monitor-status failed">需下架</span><b>{kb.name}</b><small>{kb.needs_removal} 份仍在检索中</small></li>)}</ul> : emptyLane('没有标注为禁止但仍留在知识库里的文件。')}
<p className="tiny">下架在“知识清单”里操作(停用检索,可恢复);这里只负责把差异暴露出来。</p>
</section> : null}
{lane === 'inflight' ? <section className="inbox-section" aria-label="处理中">
<div className="section-head"><h2>处理中({inflightItems.length})</h2><span className="tiny">无需操作,仅供查看</span></div>
{inflightItems.length ? inflightItems.map(actionTask) : emptyLane('当前没有正在处理或索引增强的任务。')}
{overview ? <>
<div className="pipeline-strip" aria-label="解析处理链">下载 → Hash → OSS → 文档解析 → WeKnora 上传 → 索引 → 完成</div>
<div className="pipeline-strip" aria-label="解析处理链">源头取件 → Hash → OSS → 文档解析 → WeKnora 上传 → 索引 → 完成</div>
<div className="monitor-kb-layout" id="monitor-kb-progress">
<aside className="monitor-kb-list"><h2>知识库解析进度</h2>{overview.knowledge_bases.map((kb) => <button key={kb.id} className={selectedKB?.id === kb.id ? 'active' : ''} onClick={() => onSelectKB(kb)}><span><b>{kb.name}</b><small>{scope(kb.visibility)} · 已导入 {kb.imported}</small></span><strong>{kb.completed}完成</strong></button>)}</aside>
<aside className="monitor-kb-list"><h2>知识库解析进度</h2>{overview.knowledge_bases.map((kb) => <button key={kb.id} className={selectedKB?.id === kb.id ? 'active' : ''} onClick={() => onSelectKB(kb)}><span><b>{kb.name}</b><small>{scope(kb.visibility)} · 已导入 {kb.imported}{kb.planned ? ` · 待入库 ${kb.planned}` : ''}</small></span><strong>{kb.completed}完成</strong></button>)}</aside>
<div className="monitor-kb-detail">{selectedKB ? <>
<div className="monitor-kb-heading"><div><h2>{selectedKB.name}</h2><span>{scope(selectedKB.visibility)} · 已导入 {selectedKB.imported}</span></div><div><span>对内 {selectedKB.internal_completed} / {selectedKB.internal_imported}</span><span>对外 {selectedKB.external_completed} / {selectedKB.external_imported}</span></div></div>
<div className="monitor-kb-heading"><div><h2>{selectedKB.name}</h2><span>{scope(selectedKB.visibility)} · 已导入 {selectedKB.imported}{selectedKB.planned ? ` · 待入库 ${selectedKB.planned}` : ''}</span></div><div><span>对内 {selectedKB.internal_completed} / {selectedKB.internal_imported}</span><span>对外 {selectedKB.external_completed} / {selectedKB.external_imported}</span></div></div>
<div className="monitor-stage-tabs"><button className={status === '' ? 'active' : ''} onClick={() => onStatus('')}>全部 {selectedKB.imported}</button>{stages.filter(([, count]) => count > 0).map(([value, count]) => <button className={status === value ? 'active' : ''} key={value} onClick={() => onStatus(value)}>{statusLabels[value]} {count}</button>)}</div>
<p className="finalizing-note"><b>finalizing:</b>基础解析、切片和向量通常已完成,Wiki、摘要或图谱仍在生成;不能算全部完成。</p>
{files?.sample_only ? <p className="sample-note">当前只展示本地模拟文件样例;真实模式会列出所选状态下的实际文件。</p> : null}
@@ -213,17 +204,13 @@ export function WeKnoraMonitorView({
</div>
</> : null}
</section> : null}
{lane === 'pending' ? <section className="inbox-section" aria-label="排队等待">
<div className="section-head"><h2>排队等待({queueFiltered.length})</h2><span className="spacer" /><label>目标知识库<select aria-label="队列目标知识库" value={queueScope} onChange={(event) => onQueueScope(event.target.value as 'internal' | 'external' | '')}><option value="">内/外全部</option><option value="internal">对内知识库</option><option value="external">对外知识库</option></select></label></div>
{queueFiltered.length ? queueFiltered.map(actionTask) : emptyLane('当前筛选没有排队任务。')}
</section> : null}
{lane === 'done' ? <section className="inbox-section" aria-label="已完成">
<div className="section-head"><h2>已完成({doneTodayItems.length})</h2><span className="tiny">最近一页数据中的成功任务</span></div>
{doneTodayItems.length ? doneTodayItems.map(actionTask) : emptyLane('当前页没有已完成任务;用全部任务视图翻页查看。')}
<div className="section-head"><h2>已完成({doneItems.length})</h2><span className="tiny">最近一页数据中的成功任务</span></div>
{doneItems.length ? doneItems.map(actionTask) : emptyLane('当前页没有已完成任务;用全部任务视图翻页查看。')}
</section> : null}
{lane === 'queue' ? <section className="inbox-section" aria-label="全部任务">
<div className="section-head"><h2>全部任务({queue.total})</h2><span className="spacer" /><label>目标知识库<select aria-label="队列目标知识库" value={queueScope} onChange={(event) => onQueueScope(event.target.value as 'internal' | 'external' | '')}><option value="">内/外全部</option><option value="internal">对内知识库</option><option value="external">对外知识库</option></select><select aria-label="队列状态" value={queueStatus} onChange={(event) => onQueueStatus(event.target.value as ProcessingState | '')}><option value="">全部状态</option><option value="pending">等待解析</option><option value="processing">解析中</option><option value="finalizing">索引与增强</option><option value="completed">成功</option><option value="failed">失败</option><option value="stalled">停滞</option></select></label></div>
{queueFilteredAll.length ? queueFilteredAll.map(actionTask) : !loading ? emptyLane('当前筛选没有解析任务。') : null}
<div className="section-head"><h2>全部任务({queue.total})</h2><span className="spacer" /><label>目标知识库<select aria-label="队列目标知识库" value={queueScope} onChange={(event) => onQueueScope(event.target.value as 'internal' | 'external' | '')}><option value="">内/外全部</option><option value="internal">对内知识库</option><option value="external">对外知识库</option></select><select aria-label="队列状态" value={queueStatus} onChange={(event) => onQueueStatus(event.target.value as ProcessingState | '')}><option value="">全部状态</option><option value="planned">待入库</option><option value="pending">等待解析</option><option value="processing">解析中</option><option value="finalizing">索引与增强</option><option value="completed">成功</option><option value="failed">失败</option><option value="stalled">停滞</option></select></label></div>
{orderedQueue.length ? orderedQueue.map(actionTask) : !loading ? emptyLane('当前筛选没有解析任务。') : null}
</section> : null}
{metrics ? <footer className="monitor-strip" aria-label="真实运行指标"><div>1小时·成功 <b className={metrics.short_window.failure_rate !== null && metrics.short_window.failure_rate > 5 ? 'bad' : 'good'}>{metrics.short_window.success_rate === null ? '数据不足' : `${metrics.short_window.success_rate}%`}</b></div><div>24小时·成功 <b>{metrics.long_window.success_rate === null ? '数据不足' : `${metrics.long_window.success_rate}%`}</b></div><div>p50 {metrics.short_window.duration_p50_seconds === null ? '—' : duration(Math.round(metrics.short_window.duration_p50_seconds))} · p95 {metrics.short_window.duration_p95_seconds === null ? '—' : duration(Math.round(metrics.short_window.duration_p95_seconds))}</div><div>Worker {metrics.active_workers} · 等待 {metrics.queue.pending} · 运行 {metrics.queue.running}{metrics.queue.failed ? <span className="bad"> · 失败 {metrics.queue.failed}</span> : null}</div>{overview?.health.map((service) => <div key={service.key}>{service.name} <b className={service.readiness === 'ready' ? 'good' : service.readiness === 'blocked' || service.readiness === 'down' ? 'bad' : ''}>{service.readiness === 'ready' ? '可接单' : service.readiness === 'degraded' ? '降级' : service.readiness === 'not_configured' ? '未配置' : service.readiness === 'down' ? '不可达' : '阻断'}</b></div>)}{overview ? <div>CPU {overview.host.cpu_percent.toFixed(0)}% · 内存 {gb(overview.host.memory_used_bytes)}/{gb(overview.host.memory_total_bytes)}GB</div> : null}</footer> : null}
</div>
@@ -237,14 +224,10 @@ export function WeKnoraMonitor() {
const [files, setFiles] = useState<MonitorFilePage | null>(null)
const [queue, setQueue] = useState<TestPublicationPage>(emptyQueue)
const [plans, setPlans] = useState<TestPublication[]>([])
const [changeSets, setChangeSets] = useState<CatalogChangeSetDetail[]>([])
const [confirmingSetID, setConfirmingSetID] = useState<string | null>(null)
const [lastConfirmSetID, setLastConfirmSetID] = useState<string | null>(null)
const [confirmMessage, setConfirmMessage] = useState('')
const [status, setStatus] = useState<MonitorStatus | ''>('')
const [queueStatus, setQueueStatus] = useState<ProcessingState | ''>('')
const [queueScope, setQueueScope] = useState<'internal' | 'external' | ''>('')
const [lane, setLane] = useState<'attention' | 'changes' | 'inflight' | 'pending' | 'done' | 'queue'>('attention')
const [lane, setLane] = useState<'waiting' | 'attention' | 'removal' | 'inflight' | 'done' | 'queue'>('waiting')
const [loading, setLoading] = useState(true)
const [error, setError] = useState<string | null>(null)
const [documentRevision, setDocumentRevision] = useState(0)
@@ -283,13 +266,9 @@ export function WeKnoraMonitor() {
const controller = new AbortController(); setLoading(true)
void Promise.all([
fetchTestPublications({ processing_state: queueStatus, target_scope: queueScope, limit: 50, offset: 0 }, controller.signal),
fetchTestPublications({ processing_state: 'pending', limit: 100, offset: 0 }, controller.signal),
fetchCatalogSources(controller.signal).then(async sources => {
const results=await Promise.allSettled(sources.map(source=>fetchLatestCatalogChangeSet(source.connection_id,'',20,0,controller.signal)))
return results.flatMap(result=>result.status==='fulfilled'?[result.value]:[])
}),
]).then(([queuePage, planPage, latestChangeSets]) => {
if (!controller.signal.aborted) { setQueue(queuePage); setPlans(planPage.items.filter((item) => item.can_start)); setChangeSets(latestChangeSets); setError(null) }
fetchTestPublications({ processing_state: 'planned', limit: 100, offset: 0 }, controller.signal),
]).then(([queuePage, planPage]) => {
if (!controller.signal.aborted) { setQueue(queuePage); setPlans(planPage.items); setError(null) }
}).catch((cause) => { if (!controller.signal.aborted) setError(cause instanceof Error ? cause.message : '加载解析队列失败') }).finally(() => { if (!controller.signal.aborted) setLoading(false) })
return () => controller.abort()
}, [queueStatus, queueScope, documentRevision])
@@ -360,12 +339,12 @@ export function WeKnoraMonitor() {
}
}
const startPlans = async (ids: string[], reason: string) => {
const startPlans = async (ids: string[]) => {
setStarting(true)
setStartMessage('')
try {
const result = await startTestPublications(ids, reason)
setStartMessage(`已确认 ${result.selected} 项;新建解析任务 ${result.started} 项;已开始 ${result.already_started} 项;不可处理 ${result.unavailable} 项。`)
const result = await startTestPublications(ids, '开始处理')
setStartMessage(`已提交 ${result.selected} 项;新建解析任务 ${result.started} 项;已在队列 ${result.already_started} 项;不可处理 ${result.unavailable} 项(多为源头取件未完成,稍后自动重试)。`)
setError(null)
setDocumentRevision((value) => value + 1)
} catch (cause) {
@@ -374,19 +353,5 @@ export function WeKnoraMonitor() {
setStarting(false)
}
}
const confirmChangeSet = async (changeSetID: string, itemSeqs: number[], reason: string) => {
if (confirmingSetID) return
setConfirmingSetID(changeSetID); setLastConfirmSetID(changeSetID); setConfirmMessage('')
try {
const {result,operation} = await confirmCatalogChangeSet(changeSetID, itemSeqs, reason)
setConfirmMessage(`已${operation==='replayed'?'沿用上次结果':'提交'}:确认 ${result.confirmed_items} 项;新建上传任务 ${result.uploads_created} 个;已有任务 ${result.uploads_existing} 个;请求移除 ${result.removals_requested} 份;内容未就绪 ${result.skipped_no_artifact} 份;缺少目标知识库 ${result.skipped_no_target} 份;未变化 ${result.no_change_ignored} 项已忽略。`)
setDocumentRevision((value) => value + 1)
} catch (cause) {
setConfirmMessage(cause instanceof Error ? cause.message : '确认快照差异失败')
} finally {
setConfirmingSetID(null)
}
}
return <WeKnoraMonitorView overview={overview} files={files} queue={queue} plans={plans} changeSets={changeSets} confirmingSetID={confirmingSetID ?? lastConfirmSetID} confirmMessage={confirmMessage} onConfirmChangeSet={(id,seqs,reason)=>void confirmChangeSet(id,seqs,reason)} selectedKB={selected} status={status} queueStatus={queueStatus} queueScope={queueScope} loading={loading} error={error} retryingIDs={retryingIDs} starting={starting} startMessage={startMessage} batchMessage={batchMessage} failures={failures} metrics={metrics} onSelectKB={(kb) => { setSelected(kb); setStatus(''); setFiles(null) }} onStatus={(value) => { setStatus(value); setFiles(null) }} onQueueStatus={setQueueStatus} onQueueScope={setQueueScope} lane={lane} onLaneChange={(value) => setLane(value as typeof lane)} onRetry={(id) => void retryOne(id)} onBatchRetry={(ids) => void retryBatch(ids)} onStart={(ids, reason) => void startPlans(ids, reason)} onRefresh={() => { setDocumentRevision((value) => value + 1); setResourceRevision((value) => value + 1) }} />
}
return <WeKnoraMonitorView overview={overview} files={files} queue={queue} plans={plans} selectedKB={selected} status={status} queueStatus={queueStatus} queueScope={queueScope} loading={loading} error={error} retryingIDs={retryingIDs} starting={starting} startMessage={startMessage} batchMessage={batchMessage} failures={failures} metrics={metrics} onSelectKB={(kb) => { setSelected(kb); setStatus(''); setFiles(null) }} onStatus={(value) => { setStatus(value); setFiles(null) }} onQueueStatus={setQueueStatus} onQueueScope={setQueueScope} lane={lane} onLaneChange={(value) => setLane(value as typeof lane)} onRetry={(id) => void retryOne(id)} onBatchRetry={(ids) => void retryBatch(ids)} onStart={(ids) => void startPlans(ids)} onRefresh={() => { setDocumentRevision((value) => value + 1); setResourceRevision((value) => value + 1) }} />
}
+3 -2
View File
@@ -1,9 +1,10 @@
export type MonitorStatus='completed'|'pending'|'processing'|'finalizing'|'failed'|'cancelled'
export interface MonitorHost{cpu_percent:number;memory_used_bytes:number;memory_total_bytes:number;disk_used_bytes:number;disk_total_bytes:number;gpu_state:string;gpu_detail:string;simulated:boolean}
export interface MonitorService{key:string;name:string;state:string;detail:string;latency_ms:number;observed_at:string;simulated:boolean}
export interface KnowledgeProgress{id:string;name:string;visibility:string;imported:number;completed:number;pending:number;processing:number;finalizing:number;failed:number;cancelled:number;internal_imported:number;internal_completed:number;external_imported:number;external_completed:number}
export interface KnowledgeProgress{id:string;name:string;visibility:string;imported:number;completed:number;pending:number;processing:number;finalizing:number;failed:number;cancelled:number;planned:number;needs_removal:number;stale_planned:number;internal_imported:number;internal_completed:number;external_imported:number;external_completed:number}
export interface ServiceHealth{key:string;name:string;liveness:string;readiness:string;reason:string;latency_ms:number;observed_at:string;last_success_at:string|null;consecutive_failures:number;configured:boolean;applicable:boolean}
export interface MonitorOverview{mode:string;simulated:boolean;observed_at:string;host:MonitorHost;services:MonitorService[];knowledge_bases:KnowledgeProgress[];health:ServiceHealth[]}
export interface IngestionPipeline{fetch_pending:number;fetch_failed:number;fetch_missing:number;concurrency:number}
export interface MonitorOverview{mode:string;simulated:boolean;observed_at:string;host:MonitorHost;services:MonitorService[];knowledge_bases:KnowledgeProgress[];pipeline?:IngestionPipeline;health:ServiceHealth[]}
export interface MonitorFile{name:string;directory:string;visibility:string;status:MonitorStatus;stage_detail:string;elapsed_seconds:number;error_code:string}
export interface MonitorFilePage{items:MonitorFile[];total:number;limit:number;offset:number;sample_only:boolean}
const record=(value:unknown):value is Record<string,unknown>=>typeof value==='object'&&value!==null
+2 -2
View File
@@ -1,6 +1,6 @@
import { newOperationID } from './operationKey'
export type IngestState = 'pending' | 'uploading' | 'uploaded' | 'failed'
export type ProcessingState = 'pending' | 'processing' | 'finalizing' | 'completed' | 'failed' | 'stalled'
export type ProcessingState = 'planned' | 'pending' | 'processing' | 'finalizing' | 'completed' | 'failed' | 'stalled'
export interface TestPublication {
id: string
@@ -50,7 +50,7 @@ export interface StartTestPublicationsResult {
const isRecord = (value: unknown): value is Record<string, unknown> => typeof value === 'object' && value !== null
const isCount = (value: unknown): value is number => typeof value === 'number' && Number.isInteger(value) && value >= 0
const ingestStates = new Set<IngestState>(['pending', 'uploading', 'uploaded', 'failed'])
const processingStates = new Set<ProcessingState>(['pending', 'processing', 'finalizing', 'completed', 'failed', 'stalled'])
const processingStates = new Set<ProcessingState>(['planned', 'pending', 'processing', 'finalizing', 'completed', 'failed', 'stalled'])
function parseTestPublication(value: unknown): TestPublication {
if (!isRecord(value) || typeof value.id !== 'string' || typeof value.title !== 'string' ||
+3 -3
View File
@@ -5,7 +5,7 @@ import { activateUISettings, createUISettingsCandidate, fetchActiveUISettings, p
const active = {
id: 'settings-default', state: 'active', document_refresh_seconds: 15,
resource_refresh_seconds: 30, stalled_after_seconds: 1800,
finalizing_after_seconds: 7200, updated_by: 'system:migration',
finalizing_after_seconds: 7200, ingestion_concurrency: 2, updated_by: 'system:migration',
created_at: '2026-09-10T08:00:00Z', activated_at: '2026-09-10T08:00:00Z',
retired_at: null, error_code: '',
}
@@ -29,12 +29,12 @@ describe('UI settings API', () => {
vi.stubGlobal('fetch', fetchMock)
await fetchActiveUISettings()
await createUISettingsCandidate({ document_refresh_seconds: 30, resource_refresh_seconds: 60, stalled_after_seconds: 3600, finalizing_after_seconds: 14400 })
await createUISettingsCandidate({ document_refresh_seconds: 30, resource_refresh_seconds: 60, stalled_after_seconds: 3600, finalizing_after_seconds: 14400, ingestion_concurrency: 3 })
await activateUISettings('uis-candidate')
expect(fetchMock.mock.calls[0][0]).toBe('/api/v1/ui-settings')
expect(fetchMock.mock.calls[1][0]).toBe('/api/v1/ui-settings/candidates')
expect(fetchMock.mock.calls[1][1]).toMatchObject({ method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ document_refresh_seconds: 30, resource_refresh_seconds: 60, stalled_after_seconds: 3600, finalizing_after_seconds: 14400 }) })
expect(fetchMock.mock.calls[1][1]).toMatchObject({ method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ document_refresh_seconds: 30, resource_refresh_seconds: 60, stalled_after_seconds: 3600, finalizing_after_seconds: 14400, ingestion_concurrency: 3 }) })
expect(fetchMock.mock.calls[2][0]).toBe('/api/v1/ui-settings/uis-candidate/activate')
expect(fetchMock.mock.calls[2][1]).toMatchObject({ method: 'POST', headers: { 'Content-Type': 'application/json' }, body: '{}' })
vi.unstubAllGlobals()
+4
View File
@@ -6,6 +6,7 @@ export interface UISettings {
resource_refresh_seconds: 30 | 60
stalled_after_seconds: 1800 | 3600
finalizing_after_seconds: 7200 | 14400
ingestion_concurrency: number
updated_by: string
created_at: string
activated_at: string | null
@@ -18,6 +19,7 @@ export interface UISettingsCandidateInput {
resource_refresh_seconds: UISettings['resource_refresh_seconds']
stalled_after_seconds: UISettings['stalled_after_seconds']
finalizing_after_seconds: UISettings['finalizing_after_seconds']
ingestion_concurrency: number
}
export function settingsRefreshDelays(settings: Pick<UISettings, 'document_refresh_seconds' | 'resource_refresh_seconds'>) {
@@ -40,6 +42,7 @@ export function parseUISettingsResponse(value: unknown): UISettings {
typeof item.resource_refresh_seconds !== 'number' || !resourceRefresh.has(item.resource_refresh_seconds) ||
typeof item.stalled_after_seconds !== 'number' || !stalledAfter.has(item.stalled_after_seconds) ||
typeof item.finalizing_after_seconds !== 'number' || !finalizingAfter.has(item.finalizing_after_seconds) ||
typeof item.ingestion_concurrency !== 'number' || !Number.isInteger(item.ingestion_concurrency) || item.ingestion_concurrency < 1 || item.ingestion_concurrency > 16 ||
typeof item.updated_by !== 'string' || typeof item.created_at !== 'string' ||
(item.activated_at !== null && typeof item.activated_at !== 'string') ||
(item.retired_at !== null && typeof item.retired_at !== 'string') || typeof item.error_code !== 'string') {
@@ -52,6 +55,7 @@ export function parseUISettingsResponse(value: unknown): UISettings {
resource_refresh_seconds: item.resource_refresh_seconds as UISettings['resource_refresh_seconds'],
stalled_after_seconds: item.stalled_after_seconds as UISettings['stalled_after_seconds'],
finalizing_after_seconds: item.finalizing_after_seconds as UISettings['finalizing_after_seconds'],
ingestion_concurrency: item.ingestion_concurrency,
updated_by: item.updated_by,
created_at: item.created_at,
activated_at: item.activated_at as string | null,