feat(governance): policy-driven ingestion from catalog annotations
Annotation in 来源与目录 is now the only import decision surface: - migration 000033: catalog_import_intents (durable, rescan-proof per-file intent with locator snapshot + fetch lifecycle), source_items.policy_state, planned publication state, ui_setting_revisions.ingestion_concurrency - marking fans a folder decision out to per-file intents, materializes versioned items, reuses already-verified local bytes (compare, don't pull), and plans one publication per routed KB; blocked retires planned rows while live publications stay untouched (需下架 shows in the KB diff) - artifact.fetch worker lane (scripts/artifact_fetch.py): DingTalk export tree/live box API, AList /api/fs/get direct download with SSO-redirect fail-closed, PP ledger via locator hash; content-level dedupe on sha256 - 开始处理 promotes planned->pending inside the claim-time concurrency budget from system settings; claims enforce it in SQL - monitor overview rebuilt as knowledge-base diff: planned/needs_removal per KB + fetch pipeline summary; frontend lanes 待入库/需下架 replace the deprecated snapshot change-set confirmation - spool: images (png/jpg/gif/bmp/webp) accepted, ppt/pptx rejected in favour of PDF twins; DingTalk PPT resolves its converted PDF pair
This commit is contained in:
@@ -28,3 +28,4 @@
|
||||
/tmp_*
|
||||
.style-backup-*/
|
||||
.hermes/
|
||||
__pycache__/
|
||||
|
||||
+1
-1
@@ -18,6 +18,6 @@ RUN apk add --no-cache ca-certificates python3 postgresql-client \
|
||||
&& adduser -D -H -u 65532 -G governance governance
|
||||
WORKDIR /app
|
||||
COPY --from=build /out/ /app/
|
||||
COPY scripts/source_catalog_import.py scripts/live_catalog_refresh.py /app/scripts/
|
||||
COPY scripts/source_catalog_import.py scripts/live_catalog_refresh.py scripts/artifact_fetch.py /app/scripts/
|
||||
USER 65532:65532
|
||||
CMD ["/app/api"]
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"log"
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"syscall"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
@@ -14,6 +15,7 @@ import (
|
||||
"knowledge-governance/internal/jobs"
|
||||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||||
"knowledge-governance/internal/platform/weknora"
|
||||
artifactfetch "knowledge-governance/internal/workflows/artifactfetch"
|
||||
catalogrefresh "knowledge-governance/internal/workflows/catalogrefresh"
|
||||
"knowledge-governance/internal/workflows/sourcescan"
|
||||
"knowledge-governance/internal/workflows/testweknora"
|
||||
@@ -83,6 +85,30 @@ func workerHandlers(store *postgresstore.Store, config appconfig.Config) (map[st
|
||||
return nil, fmt.Errorf("create catalog refresh handler: %w", err)
|
||||
}
|
||||
handlers["source.catalog.refresh"] = catalogHandler.Handle
|
||||
ingestionRoot := config.IngestionSpoolRoot
|
||||
if ingestionRoot == "" {
|
||||
ingestionRoot = config.GovernanceSpoolRoot
|
||||
}
|
||||
if !filepath.IsAbs(ingestionRoot) || ingestionRoot == "" {
|
||||
ingestionRoot = "/var/lib/governance/ingestion"
|
||||
}
|
||||
fetchMaxBytes := config.ArtifactFetchMaxBytes
|
||||
if config.TestWeKnoraEnabled && config.TestWeKnoraMaxUploadBytes > 0 && fetchMaxBytes > config.TestWeKnoraMaxUploadBytes {
|
||||
fetchMaxBytes = config.TestWeKnoraMaxUploadBytes
|
||||
}
|
||||
if fetchMaxBytes <= 0 {
|
||||
fetchMaxBytes = 50 * 1024 * 1024
|
||||
}
|
||||
artifactRunner, err := artifactfetch.NewCommandRunner("/usr/bin/python3", "/app/scripts/artifact_fetch.py",
|
||||
ingestionRoot, "ingestion", fetchMaxBytes)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create artifact fetch runner: %w", err)
|
||||
}
|
||||
artifactHandler, err := artifactfetch.NewHandler(artifactfetch.Config{Store: store, Runner: artifactRunner})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create artifact fetch handler: %w", err)
|
||||
}
|
||||
handlers["artifact.fetch"] = artifactHandler.Handle
|
||||
if config.PPScanEnabled {
|
||||
scanner, err := sourcescan.NewFilePPScanner(sourcescan.FilePPScannerConfig{
|
||||
AllowedRoot: config.PPScanAllowedRoot,
|
||||
|
||||
@@ -80,6 +80,7 @@ services:
|
||||
TEST_WEKNORA_KB_ID: "${TEST_WEKNORA_KB_ID:-}"
|
||||
TEST_WEKNORA_API_KEY_FILE: /run/secrets/test_weknora_api_key
|
||||
GOVERNANCE_SPOOL_ROOT: /var/lib/governance/spool
|
||||
INGESTION_SPOOL_ROOT: /var/lib/governance/ingestion
|
||||
TEST_WEKNORA_HTTP_TIMEOUT: "${TEST_WEKNORA_HTTP_TIMEOUT:-60s}"
|
||||
TEST_WEKNORA_POLL_INITIAL: "${TEST_WEKNORA_POLL_INITIAL:-10s}"
|
||||
TEST_WEKNORA_POLL_MAX: "${TEST_WEKNORA_POLL_MAX:-60s}"
|
||||
@@ -91,6 +92,7 @@ services:
|
||||
- test_weknora_api_key
|
||||
volumes:
|
||||
- ./spool:/var/lib/governance/spool:ro
|
||||
- ./spool/ingestion:/var/lib/governance/ingestion
|
||||
- "${PP_SCAN_HOST_DATA_ROOT:-/home/zheromiku/dobot-portal-sync/data}:${PP_SCAN_CONTAINER_DATA_ROOT:-/home/zheromiku/dobot-portal-sync/data}:ro"
|
||||
- /home/zheromiku/dingtalk_kb_download:/sources/dingtalk:ro
|
||||
- /home/zheromiku/tmp/alist:/sources/alist:ro
|
||||
|
||||
@@ -36,6 +36,8 @@ type Config struct {
|
||||
ExternalRoutingKnowledgeBaseID string
|
||||
TestWeKnoraAPIKeyFile string
|
||||
GovernanceSpoolRoot string
|
||||
IngestionSpoolRoot string
|
||||
ArtifactFetchMaxBytes int64
|
||||
WorkerLivenessStaleAfter time.Duration
|
||||
TestWeKnoraHTTPTimeout time.Duration
|
||||
TestWeKnoraPollInitial time.Duration
|
||||
@@ -67,6 +69,8 @@ func Load(getenv func(string) string) (Config, error) {
|
||||
ExternalRoutingKnowledgeBaseID: strings.TrimSpace(getenv("TEST_EXTERNAL_WEKNORA_KB_ID")),
|
||||
TestWeKnoraAPIKeyFile: valueOrDefault(getenv("TEST_WEKNORA_API_KEY_FILE"), "/run/secrets/test_weknora_api_key"),
|
||||
GovernanceSpoolRoot: valueOrDefault(getenv("GOVERNANCE_SPOOL_ROOT"), "/var/lib/governance/spool"),
|
||||
IngestionSpoolRoot: strings.TrimSpace(getenv("INGESTION_SPOOL_ROOT")),
|
||||
ArtifactFetchMaxBytes: 200 * 1024 * 1024,
|
||||
WorkerLivenessStaleAfter: 2 * time.Minute,
|
||||
TestWeKnoraHTTPTimeout: 60 * time.Second,
|
||||
TestWeKnoraPollInitial: 10 * time.Second,
|
||||
|
||||
@@ -51,12 +51,24 @@ type KnowledgeBaseProgress struct {
|
||||
Finalizing int `json:"finalizing"`
|
||||
Failed int `json:"failed"`
|
||||
Cancelled int `json:"cancelled"`
|
||||
Planned int `json:"planned"`
|
||||
NeedsRemoval int `json:"needs_removal"`
|
||||
StalePlanned int `json:"stale_planned"`
|
||||
InternalImported int `json:"internal_imported"`
|
||||
InternalCompleted int `json:"internal_completed"`
|
||||
ExternalImported int `json:"external_imported"`
|
||||
ExternalCompleted int `json:"external_completed"`
|
||||
}
|
||||
|
||||
// IngestionPipeline summarizes the artifact-fetch lane that sits in front of
|
||||
// 待入库 so operators can see why planned items are not uploadable yet.
|
||||
type IngestionPipeline struct {
|
||||
FetchPending int `json:"fetch_pending"`
|
||||
FetchFailed int `json:"fetch_failed"`
|
||||
FetchMissing int `json:"fetch_missing"`
|
||||
Concurrency int `json:"concurrency"`
|
||||
}
|
||||
|
||||
// JobWindowMetrics reports a frozen statistical window; insufficient samples
|
||||
// stay null and are never rendered as 0.
|
||||
type JobWindowMetrics struct {
|
||||
@@ -95,6 +107,7 @@ type MonitorOverview struct {
|
||||
Host MonitorHost `json:"host"`
|
||||
Services []MonitorService `json:"services"`
|
||||
KnowledgeBases []KnowledgeBaseProgress `json:"knowledge_bases"`
|
||||
Pipeline IngestionPipeline `json:"pipeline"`
|
||||
Health []ServiceHealth `json:"health"`
|
||||
}
|
||||
|
||||
|
||||
@@ -94,15 +94,29 @@ type CatalogRouteRequest struct {
|
||||
Now time.Time
|
||||
}
|
||||
|
||||
// CatalogRouteResult describes the desired-state write only. S7 guarantees the
|
||||
// annotation never creates publications or jobs; counts explain what the
|
||||
// subsequent ChangeSet/P3 pipeline will be able to act on.
|
||||
// CatalogRouteFile is one selected file plus the metadata snapshot the policy
|
||||
// fulfillment path needs to materialize and fetch it later.
|
||||
type CatalogRouteFile struct {
|
||||
NodeID string
|
||||
Name string
|
||||
RelativePath string
|
||||
SizeBytes *int64
|
||||
ModifiedAt *time.Time
|
||||
ContentSHA256 *string
|
||||
LocatorHash string
|
||||
Direct bool
|
||||
}
|
||||
|
||||
// CatalogRouteResult describes the desired-state write only. Policy fulfillment
|
||||
// records intents, queues artifact fetches and plans publications; counts tell
|
||||
// the operator what will show up in 解析监控.
|
||||
type CatalogRouteResult struct {
|
||||
SelectedFiles int `json:"selected_files"`
|
||||
RoutableFiles int `json:"routable_files"`
|
||||
UnboundFiles int `json:"unbound_files"`
|
||||
HashUnknownFiles int `json:"hash_unknown_files"`
|
||||
UnavailableFiles int `json:"unavailable_files"`
|
||||
SelectedFiles int `json:"selected_files"`
|
||||
IntentsRecorded int `json:"intents_recorded"`
|
||||
FetchQueued int `json:"fetch_queued"`
|
||||
PlannedPublications int `json:"planned_publications"`
|
||||
BlockedItems int `json:"blocked_items"`
|
||||
UnboundFiles int `json:"unbound_files"`
|
||||
}
|
||||
|
||||
type CatalogChangeSetFilter struct {
|
||||
|
||||
@@ -12,6 +12,7 @@ type UISettings struct {
|
||||
ResourceRefreshSeconds int `json:"resource_refresh_seconds"`
|
||||
StalledAfterSeconds int `json:"stalled_after_seconds"`
|
||||
FinalizingAfterSeconds int `json:"finalizing_after_seconds"`
|
||||
IngestionConcurrency int `json:"ingestion_concurrency"`
|
||||
UpdatedBy string `json:"updated_by"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
ActivatedAt *time.Time `json:"activated_at"`
|
||||
@@ -24,6 +25,7 @@ type CreateUISettingsCandidateRequest struct {
|
||||
ResourceRefreshSeconds int
|
||||
StalledAfterSeconds int
|
||||
FinalizingAfterSeconds int
|
||||
IngestionConcurrency int
|
||||
UpdatedBy string
|
||||
Now time.Time
|
||||
}
|
||||
@@ -33,6 +35,7 @@ func (request CreateUISettingsCandidateRequest) Valid() bool {
|
||||
oneOfInt(request.ResourceRefreshSeconds, 30, 60) &&
|
||||
oneOfInt(request.StalledAfterSeconds, 1800, 3600) &&
|
||||
oneOfInt(request.FinalizingAfterSeconds, 7200, 14400) &&
|
||||
request.IngestionConcurrency >= 1 && request.IngestionConcurrency <= 16 &&
|
||||
validSettingsActor(request.UpdatedBy) && !request.Now.IsZero()
|
||||
}
|
||||
|
||||
|
||||
@@ -1139,6 +1139,7 @@ type createUISettingsCandidateRequest struct {
|
||||
ResourceRefreshSeconds int `json:"resource_refresh_seconds"`
|
||||
StalledAfterSeconds int `json:"stalled_after_seconds"`
|
||||
FinalizingAfterSeconds int `json:"finalizing_after_seconds"`
|
||||
IngestionConcurrency int `json:"ingestion_concurrency"`
|
||||
}
|
||||
|
||||
func getActiveUISettings(w http.ResponseWriter, r *http.Request, store Store) {
|
||||
@@ -1169,6 +1170,7 @@ func createUISettingsCandidate(w http.ResponseWriter, r *http.Request, config Co
|
||||
ResourceRefreshSeconds: request.ResourceRefreshSeconds,
|
||||
StalledAfterSeconds: request.StalledAfterSeconds,
|
||||
FinalizingAfterSeconds: request.FinalizingAfterSeconds,
|
||||
IngestionConcurrency: request.IngestionConcurrency,
|
||||
UpdatedBy: actorFor(r),
|
||||
Now: config.Now(),
|
||||
}, operationID)
|
||||
|
||||
@@ -348,7 +348,7 @@ func TestHandlerRejectsInvalidCatalogParentBeforeStore(t *testing.T) {
|
||||
|
||||
func TestHandlerRoutesSelectedCatalogFilesToServerBoundScopeKB(t *testing.T) {
|
||||
now := time.Date(2026, 9, 9, 1, 0, 0, 0, time.UTC)
|
||||
store := &fakeJobStore{catalogRouteResult: governance.CatalogRouteResult{SelectedFiles: 2, RoutableFiles: 2}}
|
||||
store := &fakeJobStore{catalogRouteResult: governance.CatalogRouteResult{SelectedFiles: 2, PlannedPublications: 2}}
|
||||
handler, err := NewHandler(Config{Store: store, Now: func() time.Time { return now }, InternalRoutingKnowledgeBaseID: "kb-internal", ExternalRoutingKnowledgeBaseID: "kb-external"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -360,7 +360,7 @@ func TestHandlerRoutesSelectedCatalogFilesToServerBoundScopeKB(t *testing.T) {
|
||||
if response.Code != http.StatusAccepted || store.catalogRouteRequest.TargetKnowledgeBaseID != "kb-internal" || store.catalogRouteRequest.Now != now || store.catalogRouteRequest.Visibility != governance.VisibilityInternal {
|
||||
t.Fatalf("status=%d request=%#v body=%s", response.Code, store.catalogRouteRequest, response.Body.String())
|
||||
}
|
||||
if strings.Contains(response.Body.String(), "kb-internal") || !strings.Contains(response.Body.String(), `"routable_files":2`) {
|
||||
if strings.Contains(response.Body.String(), "kb-internal") || !strings.Contains(response.Body.String(), `"planned_publications":2`) {
|
||||
t.Fatalf("unsafe route response=%s", response.Body.String())
|
||||
}
|
||||
capabilities := httptest.NewRecorder()
|
||||
@@ -372,7 +372,7 @@ func TestHandlerRoutesSelectedCatalogFilesToServerBoundScopeKB(t *testing.T) {
|
||||
|
||||
func TestHandlerRoutesBothScopesInOneAtomicRequest(t *testing.T) {
|
||||
now := time.Date(2026, 9, 9, 1, 30, 0, 0, time.UTC)
|
||||
store := &fakeJobStore{catalogRouteResult: governance.CatalogRouteResult{SelectedFiles: 1, RoutableFiles: 1}}
|
||||
store := &fakeJobStore{catalogRouteResult: governance.CatalogRouteResult{SelectedFiles: 1, PlannedPublications: 1}}
|
||||
handler, err := NewHandler(Config{Store: store, Now: func() time.Time { return now }, InternalRoutingKnowledgeBaseID: "kb-internal", ExternalRoutingKnowledgeBaseID: "kb-external"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -393,7 +393,7 @@ func TestHandlerRoutesBothScopesInOneAtomicRequest(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestHandlerAcceptsBlockedPolicyWithoutAConfiguredTarget(t *testing.T) {
|
||||
store := &fakeJobStore{catalogRouteResult: governance.CatalogRouteResult{SelectedFiles: 1, RoutableFiles: 1}}
|
||||
store := &fakeJobStore{catalogRouteResult: governance.CatalogRouteResult{SelectedFiles: 1, BlockedItems: 1}}
|
||||
handler, err := NewHandler(Config{Store: store})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -405,7 +405,7 @@ func TestHandlerAcceptsBlockedPolicyWithoutAConfiguredTarget(t *testing.T) {
|
||||
if response.Code != http.StatusAccepted || store.catalogRouteRequest.Visibility != governance.VisibilityBlocked || len(store.catalogRouteRequest.TargetKnowledgeBaseIDs) != 0 {
|
||||
t.Fatalf("status=%d request=%#v body=%s", response.Code, store.catalogRouteRequest, response.Body.String())
|
||||
}
|
||||
if !strings.Contains(response.Body.String(), `"routable_files":1`) {
|
||||
if !strings.Contains(response.Body.String(), `"blocked_items":1`) {
|
||||
t.Fatalf("blocked route response=%s", response.Body.String())
|
||||
}
|
||||
}
|
||||
@@ -952,12 +952,12 @@ func TestHandlerVersionsAndActivatesUISettings(t *testing.T) {
|
||||
t.Fatalf("get status=%d body=%s", get.Code, get.Body.String())
|
||||
}
|
||||
noIntent := httptest.NewRecorder()
|
||||
handler.ServeHTTP(noIntent, httptest.NewRequest(http.MethodPost, "/api/v1/ui-settings/candidates", bytes.NewBufferString(`{"document_refresh_seconds":30,"resource_refresh_seconds":60,"stalled_after_seconds":3600,"finalizing_after_seconds":14400}`)))
|
||||
handler.ServeHTTP(noIntent, httptest.NewRequest(http.MethodPost, "/api/v1/ui-settings/candidates", bytes.NewBufferString(`{"document_refresh_seconds":30,"resource_refresh_seconds":60,"stalled_after_seconds":3600,"finalizing_after_seconds":14400,"ingestion_concurrency":2}`)))
|
||||
if noIntent.Code != http.StatusForbidden {
|
||||
t.Fatalf("missing intent status=%d body=%s", noIntent.Code, noIntent.Body.String())
|
||||
}
|
||||
create := httptest.NewRecorder()
|
||||
createRequest := httptest.NewRequest(http.MethodPost, "/api/v1/ui-settings/candidates", bytes.NewBufferString(`{"document_refresh_seconds":30,"resource_refresh_seconds":60,"stalled_after_seconds":3600,"finalizing_after_seconds":14400}`))
|
||||
createRequest := httptest.NewRequest(http.MethodPost, "/api/v1/ui-settings/candidates", bytes.NewBufferString(`{"document_refresh_seconds":30,"resource_refresh_seconds":60,"stalled_after_seconds":3600,"finalizing_after_seconds":14400,"ingestion_concurrency":2}`))
|
||||
createRequest.Header.Set("X-Governance-Action", "ui-settings")
|
||||
createRequest.Header.Set("Idempotency-Key", "op-uis-create")
|
||||
handler.ServeHTTP(create, createRequest)
|
||||
|
||||
+27
-11
@@ -82,17 +82,24 @@ func (provider *RealProvider) Overview(ctx context.Context) (governance.MonitorO
|
||||
)
|
||||
rows, err := provider.db.Query(ctx, `SELECT p.test_kb_id,
|
||||
CASE WHEN COUNT(DISTINCT p.target_scope)=1 THEN MIN(p.target_scope) ELSE 'mixed' END,
|
||||
COUNT(*)::integer,
|
||||
COUNT(*) FILTER (WHERE p.processing_state='completed')::integer,
|
||||
COUNT(*) FILTER (WHERE p.processing_state='pending')::integer,
|
||||
COUNT(*) FILTER (WHERE p.processing_state='processing')::integer,
|
||||
COUNT(*) FILTER (WHERE p.processing_state='finalizing')::integer,
|
||||
COUNT(*) FILTER (WHERE p.processing_state IN ('failed','stalled'))::integer,
|
||||
COUNT(*) FILTER (WHERE p.target_scope='internal')::integer,
|
||||
COUNT(*) FILTER (WHERE p.target_scope='internal' AND p.processing_state='completed')::integer,
|
||||
COUNT(*) FILTER (WHERE p.target_scope='external')::integer,
|
||||
COUNT(*) FILTER (WHERE p.target_scope='external' AND p.processing_state='completed')::integer
|
||||
COUNT(*) FILTER (WHERE p.superseded_at IS NULL)::integer,
|
||||
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.processing_state='completed')::integer,
|
||||
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.processing_state='pending')::integer,
|
||||
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.processing_state='processing')::integer,
|
||||
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.processing_state='finalizing')::integer,
|
||||
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.processing_state IN ('failed','stalled'))::integer,
|
||||
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.target_scope='internal')::integer,
|
||||
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.target_scope='internal' AND p.processing_state='completed')::integer,
|
||||
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.target_scope='external')::integer,
|
||||
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.target_scope='external' AND p.processing_state='completed')::integer,
|
||||
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.processing_state='planned')::integer,
|
||||
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.retrieval_state='enabled'
|
||||
AND item.policy_state='blocked')::integer,
|
||||
COUNT(*) FILTER (WHERE p.superseded_at IS NULL AND p.processing_state='planned'
|
||||
AND item.policy_state='blocked')::integer
|
||||
FROM test_publications p
|
||||
JOIN source_item_versions version ON version.id=p.source_item_version_id
|
||||
JOIN source_items item ON item.id=version.source_item_id
|
||||
GROUP BY p.test_kb_id
|
||||
ORDER BY p.test_kb_id`)
|
||||
if err != nil {
|
||||
@@ -101,7 +108,8 @@ ORDER BY p.test_kb_id`)
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var item governance.KnowledgeBaseProgress
|
||||
if err := rows.Scan(&item.ID, &item.Visibility, &item.Imported, &item.Completed, &item.Pending, &item.Processing, &item.Finalizing, &item.Failed, &item.InternalImported, &item.InternalCompleted, &item.ExternalImported, &item.ExternalCompleted); err != nil {
|
||||
if err := rows.Scan(&item.ID, &item.Visibility, &item.Imported, &item.Completed, &item.Pending, &item.Processing, &item.Finalizing, &item.Failed, &item.InternalImported, &item.InternalCompleted, &item.ExternalImported, &item.ExternalCompleted,
|
||||
&item.Planned, &item.NeedsRemoval, &item.StalePlanned); err != nil {
|
||||
return governance.MonitorOverview{}, fmt.Errorf("scan monitored knowledge base: %w", err)
|
||||
}
|
||||
switch item.Visibility {
|
||||
@@ -117,6 +125,14 @@ ORDER BY p.test_kb_id`)
|
||||
if err := rows.Err(); err != nil {
|
||||
return governance.MonitorOverview{}, fmt.Errorf("iterate monitored knowledge bases: %w", err)
|
||||
}
|
||||
if err := provider.db.QueryRow(ctx, `SELECT
|
||||
COUNT(*) FILTER (WHERE fetch_state IN ('pending','fetching'))::integer,
|
||||
COUNT(*) FILTER (WHERE fetch_state='failed')::integer,
|
||||
COUNT(*) FILTER (WHERE fetch_state='missing_local')::integer,
|
||||
COALESCE(MAX((SELECT ingestion_concurrency FROM ui_setting_revisions WHERE state='active' LIMIT 1)),2)::integer
|
||||
FROM catalog_import_intents`).Scan(&overview.Pipeline.FetchPending, &overview.Pipeline.FetchFailed, &overview.Pipeline.FetchMissing, &overview.Pipeline.Concurrency); err != nil {
|
||||
return governance.MonitorOverview{}, fmt.Errorf("summarize ingestion pipeline: %w", err)
|
||||
}
|
||||
return overview, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"knowledge-governance/internal/workflows/artifactfetch"
|
||||
)
|
||||
|
||||
// LoadArtifactFetchManifest resolves one pending intent plus its live source
|
||||
// coordinates for the fetch script, keyed by the connection-stable node id.
|
||||
func (s *Store) LoadArtifactFetchManifest(ctx context.Context, connectionID, nodeID string) (artifactfetch.Manifest, error) {
|
||||
return loadArtifactFetchManifest(ctx, s.db, connectionID, nodeID)
|
||||
}
|
||||
|
||||
func loadArtifactFetchManifest(ctx context.Context, db interface {
|
||||
QueryRow(context.Context, string, ...any) pgx.Row
|
||||
}, connectionID, nodeID string) (artifactfetch.Manifest, error) {
|
||||
var manifest artifactfetch.Manifest
|
||||
err := db.QueryRow(ctx, `SELECT connection.source_type,connection.name,intent.relative_path,intent.name,
|
||||
intent.size_bytes,intent.modified_at,intent.locator_hash
|
||||
FROM catalog_import_intents intent
|
||||
JOIN source_connections connection ON connection.id=intent.source_connection_id
|
||||
WHERE intent.source_connection_id=$1 AND intent.node_id=$2`, connectionID, nodeID).
|
||||
Scan(&manifest.SourceType, &manifest.SourceName, &manifest.RelativePath, &manifest.Name,
|
||||
&manifest.SizeBytes, &manifest.ModifiedAt, &manifest.LocatorHash)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return artifactfetch.Manifest{}, artifactfetch.ErrManifestGone
|
||||
}
|
||||
if err != nil {
|
||||
return artifactfetch.Manifest{}, fmt.Errorf("load artifact fetch manifest: %w", err)
|
||||
}
|
||||
manifest.ConnectionID = connectionID
|
||||
manifest.NodeID = nodeID
|
||||
return manifest, nil
|
||||
}
|
||||
@@ -112,7 +112,8 @@ FOR UPDATE OF node`, request.ConnectionID, nodeIDs)
|
||||
JOIN source_catalog_nodes child ON child.snapshot_id=parent.snapshot_id AND child.parent_node_id=parent.node_id
|
||||
WHERE parent.node_type='folder'
|
||||
)
|
||||
SELECT DISTINCT node.node_id,node.source_item_id,(node.node_id=ANY($2::text[])) AS directly_selected
|
||||
SELECT DISTINCT node.node_id,node.name,node.relative_path,node.size_bytes,node.modified_at,node.content_sha256,
|
||||
node.source_locator_hash,(node.node_id=ANY($2::text[])) AS directly_selected
|
||||
FROM descendants
|
||||
JOIN source_catalog_nodes node ON node.snapshot_id=descendants.snapshot_id AND node.node_id=descendants.node_id
|
||||
WHERE node.node_type='file'
|
||||
@@ -120,15 +121,11 @@ ORDER BY node.node_id`, request.ConnectionID, nodeIDs)
|
||||
if err != nil {
|
||||
return governance.CatalogRouteResult{}, fmt.Errorf("select catalog route files: %w", err)
|
||||
}
|
||||
type selectedFile struct {
|
||||
nodeID string
|
||||
sourceItemID *string
|
||||
direct bool
|
||||
}
|
||||
files := make([]selectedFile, 0, len(nodeIDs))
|
||||
files := make([]governance.CatalogRouteFile, 0, len(nodeIDs))
|
||||
for rows.Next() {
|
||||
var file selectedFile
|
||||
if err := rows.Scan(&file.nodeID, &file.sourceItemID, &file.direct); err != nil {
|
||||
var file governance.CatalogRouteFile
|
||||
if err := rows.Scan(&file.NodeID, &file.Name, &file.RelativePath, &file.SizeBytes, &file.ModifiedAt,
|
||||
&file.ContentSHA256, &file.LocatorHash, &file.Direct); err != nil {
|
||||
rows.Close()
|
||||
return governance.CatalogRouteResult{}, fmt.Errorf("scan catalog route file: %w", err)
|
||||
}
|
||||
@@ -139,52 +136,7 @@ ORDER BY node.node_id`, request.ConnectionID, nodeIDs)
|
||||
return governance.CatalogRouteResult{}, fmt.Errorf("iterate catalog route files: %w", err)
|
||||
}
|
||||
rows.Close()
|
||||
|
||||
result := governance.CatalogRouteResult{SelectedFiles: len(files)}
|
||||
routedItems := map[string]bool{}
|
||||
for _, file := range files {
|
||||
if file.sourceItemID == nil {
|
||||
result.UnboundFiles++
|
||||
continue
|
||||
}
|
||||
itemID := *file.sourceItemID
|
||||
if routedItems[itemID] {
|
||||
continue
|
||||
}
|
||||
routedItems[itemID] = true
|
||||
if file.direct {
|
||||
if _, err := tx.Exec(ctx, `INSERT INTO source_item_scope_overrides(source_item_id,visibility,updated_by,reason,created_at,updated_at)
|
||||
VALUES($1,$2,$3,$4,$5,$5) ON CONFLICT(source_item_id) DO UPDATE SET visibility=EXCLUDED.visibility,updated_by=EXCLUDED.updated_by,reason=EXCLUDED.reason,updated_at=EXCLUDED.updated_at`, itemID, request.Visibility, request.OperatorRef, request.Reason, request.Now); err != nil {
|
||||
return governance.CatalogRouteResult{}, fmt.Errorf("set routed item scope: %w", err)
|
||||
}
|
||||
eventID, err := newOpaqueID("se-")
|
||||
if err != nil {
|
||||
return governance.CatalogRouteResult{}, err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `INSERT INTO scope_change_events(id,target_type,source_item_id,action,decided_visibility,operator_ref,reason,created_at)
|
||||
VALUES($1,'item',$2,'set',$3,$4,$5,$6)`, eventID, itemID, request.Visibility, request.OperatorRef, request.Reason, request.Now); err != nil {
|
||||
return governance.CatalogRouteResult{}, fmt.Errorf("audit routed item scope: %w", err)
|
||||
}
|
||||
}
|
||||
// S7: classify only. Annotations never create publications, retire targets, or enqueue jobs.
|
||||
var currentVersionID *string
|
||||
var contentState, spoolRef string
|
||||
err = tx.QueryRow(ctx, `SELECT version.id,content.content_state,COALESCE(content.local_spool_ref,'') FROM source_items item
|
||||
LEFT JOIN source_item_versions version ON version.source_item_id=item.id AND version.revision=item.revision
|
||||
LEFT JOIN content_objects content ON content.id=version.content_object_id
|
||||
WHERE item.id=$1`, itemID).Scan(¤tVersionID, &contentState, &spoolRef)
|
||||
switch {
|
||||
case errors.Is(err, pgx.ErrNoRows):
|
||||
result.UnavailableFiles++
|
||||
case err != nil:
|
||||
return governance.CatalogRouteResult{}, fmt.Errorf("inspect routed artifact: %w", err)
|
||||
case contentState == string(governance.ContentVerified) && spoolRef != "":
|
||||
result.RoutableFiles++
|
||||
default:
|
||||
result.HashUnknownFiles++
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
return fulfillCatalogIntentsTx(ctx, tx, request, files, request.Now)
|
||||
}
|
||||
|
||||
func setCatalogDirectoryPolicy(ctx context.Context, tx pgx.Tx, request governance.CatalogRouteRequest, directoryPath string) error {
|
||||
|
||||
@@ -10,9 +10,10 @@ import (
|
||||
"knowledge-governance/internal/governance"
|
||||
)
|
||||
|
||||
// S7: catalog annotation persists ONLY desired state (scope rules/overrides + audit).
|
||||
// It never creates or mutates test_publications and never enqueues jobs.
|
||||
func TestRouteCatalogFilesWritesDesiredStateWithoutSideEffects(t *testing.T) {
|
||||
// Policy-driven ingestion: saving an annotation records durable intents,
|
||||
// materializes items, enqueues artifact fetches and plans publications.
|
||||
// The only human action left is 开始处理; marking is the decision.
|
||||
func TestRouteCatalogFilesFulfillsPolicyIntent(t *testing.T) {
|
||||
databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL"))
|
||||
if databaseURL == "" {
|
||||
t.Skip("TEST_DATABASE_URL is not set")
|
||||
@@ -46,66 +47,70 @@ INSERT INTO source_catalog_heads(source_connection_id,snapshot_id,activated_at)
|
||||
}
|
||||
store := NewStore(pool)
|
||||
now := time.Date(2026, 9, 9, 1, 0, 0, 0, time.UTC)
|
||||
|
||||
// Folder annotation fans out to files; the bound file already holds verified
|
||||
// bytes, so it is planned without any fetch (本地已有 → 比对复用 → 不拉取).
|
||||
result, err := store.RouteCatalogFiles(ctx, governance.CatalogRouteRequest{ConnectionID: "conn-route", NodeIDs: []string{"folder-route"}, Visibility: governance.VisibilityInternal, TargetKnowledgeBaseID: "kb-internal", OperatorRef: "engineer-1", Reason: "整目录标注", Now: now})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.SelectedFiles != 1 || result.RoutableFiles != 1 || result.UnboundFiles != 0 || result.HashUnknownFiles != 0 {
|
||||
if result.SelectedFiles != 1 || result.IntentsRecorded != 1 || result.PlannedPublications != 1 || result.FetchQueued != 0 {
|
||||
t.Fatalf("result=%#v", result)
|
||||
}
|
||||
// Desired state saved, recursion intact.
|
||||
var directoryPolicy string
|
||||
if err := pool.QueryRow(ctx, `SELECT visibility FROM directory_scope_rules WHERE source_connection_id='conn-route' AND directory_path='工业产品'`).Scan(&directoryPolicy); err != nil {
|
||||
if err := pool.QueryRow(ctx, `SELECT visibility FROM directory_scope_rules WHERE source_connection_id='conn-route' AND directory_path='工业产品'`).Scan(&directoryPolicy); err != nil || directoryPolicy != "internal" {
|
||||
t.Fatalf("directory policy=%q err=%v", directoryPolicy, err)
|
||||
}
|
||||
var intentCount, publicationCount, fetchJobs, uploadJobs int
|
||||
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM catalog_import_intents WHERE source_connection_id='conn-route' AND visibility='internal'`).Scan(&intentCount); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if directoryPolicy != "internal" {
|
||||
t.Fatalf("directory policy=%q", directoryPolicy)
|
||||
}
|
||||
// Zero side effects: no publications, no jobs.
|
||||
var publicationCount, jobCount int
|
||||
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM test_publications`).Scan(&publicationCount); err != nil {
|
||||
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM test_publications WHERE processing_state='planned' AND superseded_at IS NULL`).Scan(&publicationCount); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM jobs`).Scan(&jobCount); err != nil {
|
||||
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM jobs WHERE type='artifact.fetch'`).Scan(&fetchJobs); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if publicationCount != 0 || jobCount != 0 {
|
||||
t.Fatalf("annotation must not touch pipeline: publications=%d jobs=%d", publicationCount, jobCount)
|
||||
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM jobs WHERE type='test.upload'`).Scan(&uploadJobs); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if intentCount != 1 || publicationCount != 1 || fetchJobs != 0 || uploadJobs != 0 {
|
||||
t.Fatalf("intents=%d planned=%d fetch=%d upload=%d", intentCount, publicationCount, fetchJobs, uploadJobs)
|
||||
}
|
||||
// The planned publication binds the pre-verified current version.
|
||||
var plannedVersion string
|
||||
if err := pool.QueryRow(ctx, `SELECT source_item_version_id FROM test_publications WHERE test_kb_id='kb-internal'`).Scan(&plannedVersion); err != nil || plannedVersion != "version-route" {
|
||||
t.Fatalf("planned version=%q err=%v", plannedVersion, err)
|
||||
}
|
||||
|
||||
// Re-annotating the same directory is idempotent for rules and still side-effect free.
|
||||
// Re-marking the same directory is replay-safe: one intent, one publication.
|
||||
again, err := store.RouteCatalogFiles(ctx, governance.CatalogRouteRequest{ConnectionID: "conn-route", NodeIDs: []string{"folder-route"}, Visibility: governance.VisibilityInternal, TargetKnowledgeBaseID: "kb-internal", OperatorRef: "engineer-1", Reason: "重复提交", Now: now.Add(time.Minute)})
|
||||
if err != nil || again.SelectedFiles != 1 || again.RoutableFiles != 1 {
|
||||
if err != nil || again.SelectedFiles != 1 || again.PlannedPublications != 0 || again.FetchQueued != 0 {
|
||||
t.Fatalf("again=%#v err=%v", again, err)
|
||||
}
|
||||
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM jobs`).Scan(&jobCount); err != nil || jobCount != 0 {
|
||||
t.Fatalf("repeat annotation jobs=%d err=%v", jobCount, err)
|
||||
}
|
||||
var ruleCount int
|
||||
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM directory_scope_rules WHERE source_connection_id='conn-route' AND directory_path='工业产品'`).Scan(&ruleCount); err != nil || ruleCount != 1 {
|
||||
t.Fatalf("rule rows=%d err=%v", ruleCount, err)
|
||||
}
|
||||
|
||||
// Switching policy updates desired state without retiring or removing anything.
|
||||
// Widening to both plans a second KB publication; nothing is fetched again.
|
||||
both, err := store.RouteCatalogFiles(ctx, governance.CatalogRouteRequest{ConnectionID: "conn-route", NodeIDs: []string{"folder-route"}, Visibility: governance.VisibilityBoth, TargetKnowledgeBaseIDs: []string{"kb-internal", "kb-external"}, OperatorRef: "engineer-1", Reason: "内外知识库同时使用", Now: now.Add(2 * time.Minute)})
|
||||
if err != nil || both.SelectedFiles != 1 || both.RoutableFiles != 1 {
|
||||
if err != nil || both.SelectedFiles != 1 || both.PlannedPublications != 1 || both.FetchQueued != 0 {
|
||||
t.Fatalf("both=%#v err=%v", both, err)
|
||||
}
|
||||
if err := pool.QueryRow(ctx, `SELECT visibility FROM directory_scope_rules WHERE source_connection_id='conn-route' AND directory_path='工业产品'`).Scan(&directoryPolicy); err != nil || directoryPolicy != "both" {
|
||||
t.Fatalf("policy after both=%q err=%v", directoryPolicy, err)
|
||||
}
|
||||
|
||||
// Blocking saves the desired policy only; existing publications (if any) stay untouched.
|
||||
// An unannotated live publication survives marking blocked: removal is
|
||||
// knowledge-base diff (需下架) territory, never a silent side effect.
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO test_publications(id,source_item_version_id,test_kb_id,target_scope,ingest_state,processing_state,retrieval_state,knowledge_id,created_at,updated_at)
|
||||
VALUES('tp-existing','version-route','kb-internal','internal','uploaded','completed','enabled','kn-1',$1,$1)`, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
blocked, err := store.RouteCatalogFiles(ctx, governance.CatalogRouteRequest{ConnectionID: "conn-route", NodeIDs: []string{"folder-route"}, Visibility: governance.VisibilityBlocked, OperatorRef: "engineer-1", Reason: "禁止导入", Now: now.Add(3 * time.Minute)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if blocked.SelectedFiles != 1 {
|
||||
t.Fatalf("blocked=%#v", blocked)
|
||||
if err != nil || blocked.SelectedFiles != 1 || blocked.BlockedItems != 1 {
|
||||
t.Fatalf("blocked=%#v err=%v", blocked, err)
|
||||
}
|
||||
var retrievalState string
|
||||
var removalJobs int
|
||||
@@ -118,39 +123,36 @@ VALUES('tp-existing','version-route','kb-internal','internal','uploaded','comple
|
||||
if retrievalState != "enabled" || removalJobs != 0 {
|
||||
t.Fatalf("blocked annotation must not remove: state=%s removal jobs=%d", retrievalState, removalJobs)
|
||||
}
|
||||
// The planned kb-external row created while policy was 'both' is retired.
|
||||
var stillPlanned int
|
||||
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM test_publications WHERE processing_state='planned' AND superseded_at IS NULL AND test_kb_id='kb-external'`).Scan(&stillPlanned); err != nil || stillPlanned != 0 {
|
||||
t.Fatalf("blocked must retire planned rows=%d err=%v", stillPlanned, err)
|
||||
}
|
||||
if err := pool.QueryRow(ctx, `SELECT visibility FROM directory_scope_rules WHERE source_connection_id='conn-route' AND directory_path='工业产品'`).Scan(&directoryPolicy); err != nil || directoryPolicy != "blocked" {
|
||||
t.Fatalf("blocked policy=%q err=%v", directoryPolicy, err)
|
||||
}
|
||||
|
||||
// Unbound file and hash-unknown file are reported, never planned.
|
||||
unbound, err := store.RouteCatalogFiles(ctx, governance.CatalogRouteRequest{ConnectionID: "conn-route", NodeIDs: []string{"file-unmapped"}, Visibility: governance.VisibilityExternal, TargetKnowledgeBaseID: "kb-external", OperatorRef: "engineer-1", Reason: "测试未映射", Now: now})
|
||||
if err != nil || unbound.SelectedFiles != 1 || unbound.RoutableFiles != 0 || unbound.UnboundFiles != 1 {
|
||||
t.Fatalf("unbound=%#v err=%v", unbound, err)
|
||||
// A never-fetched file becomes an intent with a materialized item, a
|
||||
// fetch job and a planned publication (未绑定 → 从源头拉,不再是死路).
|
||||
restored, err := store.RouteCatalogFiles(ctx, governance.CatalogRouteRequest{ConnectionID: "conn-route", NodeIDs: []string{"file-unmapped"}, Visibility: governance.VisibilityInternal, TargetKnowledgeBaseID: "kb-internal", OperatorRef: "engineer-1", Reason: "重新标注", Now: now.Add(4 * time.Minute)})
|
||||
if err != nil || restored.IntentsRecorded != 1 || restored.FetchQueued != 1 || restored.PlannedPublications != 1 {
|
||||
t.Fatalf("restored=%#v err=%v", restored, err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO content_objects(id,sha256,size_bytes,local_spool_ref,content_state) VALUES('content-pending',repeat('e',64),8,'','unknown')`); err != nil {
|
||||
var fetchState, itemPolicy string
|
||||
if err := pool.QueryRow(ctx, `SELECT intent.fetch_state||'|'||item.policy_state FROM catalog_import_intents intent
|
||||
JOIN source_items item ON item.id=intent.source_item_id
|
||||
WHERE intent.node_id='file-unmapped'`).Scan(&fetchState, &itemPolicy); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO source_items(id,source_connection_id,stable_external_id,canonical_uri_hash,normalized_path,title,item_type,revision,visibility,visibility_origin)
|
||||
VALUES('item-pending','conn-route','external-pending',repeat('f',64),'待校验.pdf','待校验.pdf','document','v1','external','pp_public_tag')`); err != nil {
|
||||
t.Fatal(err)
|
||||
if fetchState != "pending|internal" {
|
||||
t.Fatalf("fetch state=%q", fetchState)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO source_item_versions(id,source_item_id,revision,content_object_id,content_state) VALUES('version-pending','item-pending','v1','content-pending','unknown')`); err != nil {
|
||||
t.Fatal(err)
|
||||
var versionState string
|
||||
if err := pool.QueryRow(ctx, `SELECT version.content_state FROM source_item_versions version
|
||||
JOIN source_items item ON item.id=version.source_item_id AND item.revision=version.revision
|
||||
WHERE item.stable_external_id='file-unmapped'`).Scan(&versionState); err != nil || versionState != "unknown" {
|
||||
t.Fatalf("pending version state=%q err=%v", versionState, err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO source_catalog_nodes(snapshot_id,node_id,parent_node_id,node_type,name,relative_path,source_locator_hash,source_item_id,size_bytes,direct_child_count,subtree_file_count,subtree_bytes)
|
||||
VALUES('snapshot-route','file-pending','root-route','file','待校验.pdf','待校验.pdf',repeat('d',64),'item-pending',8,0,1,8)`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hashUnknown, err := store.RouteCatalogFiles(ctx, governance.CatalogRouteRequest{ConnectionID: "conn-route", NodeIDs: []string{"file-pending"}, Visibility: governance.VisibilityInternal, TargetKnowledgeBaseID: "kb-internal", OperatorRef: "engineer-1", Reason: "测试内容未验证", Now: now})
|
||||
if err != nil || hashUnknown.SelectedFiles != 1 || hashUnknown.RoutableFiles != 0 || hashUnknown.HashUnknownFiles != 1 {
|
||||
t.Fatalf("hash unknown=%#v err=%v", hashUnknown, err)
|
||||
}
|
||||
// Overrides are still recorded for unbound/pending items so the desired state survives.
|
||||
var overrideCount int
|
||||
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM source_item_scope_overrides WHERE source_item_id='item-pending'`).Scan(&overrideCount); err != nil || overrideCount != 1 {
|
||||
t.Fatalf("pending override rows=%d err=%v", overrideCount, err)
|
||||
}
|
||||
// Audit trail exists for every annotation action.
|
||||
var events int
|
||||
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM scope_change_events`).Scan(&events); err != nil || events < 5 {
|
||||
t.Fatalf("scope events=%d err=%v", events, err)
|
||||
|
||||
@@ -46,7 +46,7 @@ INSERT INTO source_catalog_heads(source_connection_id,snapshot_id,activated_at)
|
||||
now := time.Date(2026, 9, 11, 6, 0, 0, 0, time.UTC)
|
||||
request := governance.CatalogRouteRequest{ConnectionID: "conn-op", NodeIDs: []string{"file-op"}, Visibility: governance.VisibilityInternal, TargetKnowledgeBaseID: "kb-internal", OperatorRef: "engineer-1", Reason: "初次标注", Now: now}
|
||||
first, replayed, err := store.RouteCatalogFilesWithOperation(ctx, request, "op-route-1")
|
||||
if err != nil || replayed || first.RoutableFiles != 1 {
|
||||
if err != nil || replayed || first.PlannedPublications != 1 {
|
||||
t.Fatalf("first result=%#v replayed=%v error=%v", first, replayed, err)
|
||||
}
|
||||
second, replayed, err := store.RouteCatalogFilesWithOperation(ctx, request, "op-route-1")
|
||||
@@ -60,8 +60,8 @@ INSERT INTO source_catalog_heads(source_connection_id,snapshot_id,activated_at)
|
||||
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM operation_requests WHERE operation_type='catalog.route' AND operation_id='op-route-1'`).Scan(&operations); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if publications != 0 || operations != 1 {
|
||||
t.Fatalf("S7 annotation must not publish: publications=%d operations=%d", publications, operations)
|
||||
if publications != 1 || operations != 1 {
|
||||
t.Fatalf("annotation must plan exactly one replay-safe publication: publications=%d operations=%d", publications, operations)
|
||||
}
|
||||
conflicting := request
|
||||
conflicting.Reason = "不同的操作内容"
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
ALTER TABLE ui_setting_revisions DROP COLUMN IF EXISTS ingestion_concurrency;
|
||||
DROP INDEX IF EXISTS idx_test_publications_planned;
|
||||
ALTER TABLE test_publications DROP CONSTRAINT IF EXISTS test_publications_processing_state_check;
|
||||
ALTER TABLE test_publications ADD CONSTRAINT test_publications_processing_state_check
|
||||
CHECK (processing_state IN ('pending', 'processing', 'finalizing', 'completed', 'failed', 'stalled'));
|
||||
DROP INDEX IF EXISTS idx_source_items_policy;
|
||||
ALTER TABLE source_items DROP COLUMN IF EXISTS policy_state;
|
||||
DROP INDEX IF EXISTS idx_catalog_intents_fetch;
|
||||
DROP TABLE IF EXISTS catalog_import_intents;
|
||||
@@ -0,0 +1,62 @@
|
||||
-- Policy-driven ingestion: the ONLY import decision surface is the catalog
|
||||
-- annotation (internal / external / both / blocked). Marking a file or folder
|
||||
-- records a durable per-file INTENT that survives rescans; unannotated files
|
||||
-- stay "unset" and are never ingested (cold start imports nothing).
|
||||
-- A saved intent immediately materializes a versioned source item and an
|
||||
-- artifact.fetch job so the artifact is pulled from the live source (or
|
||||
-- matched against an already-present local copy); once content is verified a
|
||||
-- 'planned' publication appears in the knowledge-base diff as 待入库. The one
|
||||
-- human batch action "开始处理" promotes planned -> queued within the
|
||||
-- concurrency budget configured in system settings.
|
||||
|
||||
-- Per-file decision, keyed by the connection-stable catalog node id
|
||||
-- (node ids are derived from source_key + relative path, identical across
|
||||
-- snapshots, so intents are rescan-proof).
|
||||
CREATE TABLE IF NOT EXISTS catalog_import_intents (
|
||||
source_connection_id VARCHAR(36) NOT NULL
|
||||
REFERENCES source_connections(id) ON DELETE CASCADE,
|
||||
node_id VARCHAR(64) NOT NULL,
|
||||
visibility VARCHAR(20) NOT NULL
|
||||
CHECK (visibility IN ('internal', 'external', 'both', 'blocked')),
|
||||
source_item_id VARCHAR(36),
|
||||
decided_by VARCHAR(128) NOT NULL,
|
||||
reason VARCHAR(500) NOT NULL DEFAULT '',
|
||||
-- Locator snapshot: the fetch worker needs path/name metadata without
|
||||
-- re-walking a snapshot that may already have rotated.
|
||||
relative_path TEXT NOT NULL DEFAULT '',
|
||||
name TEXT NOT NULL DEFAULT '',
|
||||
size_bytes BIGINT,
|
||||
modified_at TIMESTAMPTZ,
|
||||
content_sha256 CHAR(64),
|
||||
locator_hash CHAR(64) NOT NULL DEFAULT '',
|
||||
-- Content materialization lifecycle driven by the artifact.fetch job.
|
||||
fetch_state VARCHAR(20) NOT NULL DEFAULT 'pending'
|
||||
CHECK (fetch_state IN ('pending', 'fetching', 'ready', 'missing_local', 'failed')),
|
||||
last_error TEXT NOT NULL DEFAULT '',
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
PRIMARY KEY (source_connection_id, node_id)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_catalog_intents_item
|
||||
ON catalog_import_intents (source_item_id) WHERE source_item_id IS NOT NULL;
|
||||
CREATE INDEX IF NOT EXISTS idx_catalog_intents_fetch
|
||||
ON catalog_import_intents (fetch_state) WHERE fetch_state <> 'ready';
|
||||
|
||||
-- Materialized items carry the policy so list/diff queries never re-walk rules.
|
||||
ALTER TABLE source_items ADD COLUMN IF NOT EXISTS policy_state VARCHAR(20) NOT NULL DEFAULT 'unset'
|
||||
CHECK (policy_state IN ('unset', 'internal', 'external', 'both', 'blocked'));
|
||||
CREATE INDEX IF NOT EXISTS idx_source_items_policy
|
||||
ON source_items (policy_state) WHERE policy_state <> 'unset';
|
||||
|
||||
-- The knowledge base is the ingestion target of a planned publication; make
|
||||
-- "planned" explicit so the KB-diff view never reuses queued semantics.
|
||||
ALTER TABLE test_publications DROP CONSTRAINT IF EXISTS test_publications_processing_state_check;
|
||||
ALTER TABLE test_publications ADD CONSTRAINT test_publications_processing_state_check
|
||||
CHECK (processing_state IN ('planned', 'pending', 'processing', 'finalizing', 'completed', 'failed', 'stalled'));
|
||||
CREATE INDEX IF NOT EXISTS idx_test_publications_planned
|
||||
ON test_publications (test_kb_id, processing_state) WHERE processing_state = 'planned';
|
||||
|
||||
-- 开始处理's concurrency budget lives in system settings (validated revisions).
|
||||
ALTER TABLE ui_setting_revisions ADD COLUMN IF NOT EXISTS ingestion_concurrency INTEGER NOT NULL DEFAULT 2
|
||||
CHECK (ingestion_concurrency BETWEEN 1 AND 16);
|
||||
@@ -11,8 +11,8 @@ func TestEmbeddedMigrationsAreVersionSorted(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("loadMigrations() error = %v", err)
|
||||
}
|
||||
if len(migrations) != 32 {
|
||||
t.Fatalf("migration count = %d, want 32", len(migrations))
|
||||
if len(migrations) != 33 {
|
||||
t.Fatalf("migration count = %d, want 33", len(migrations))
|
||||
}
|
||||
if migrations[0].Version != 1 || !strings.Contains(migrations[0].UpSQL, "CREATE TABLE IF NOT EXISTS jobs") {
|
||||
t.Fatalf("migration 1 = %#v", migrations[0])
|
||||
@@ -567,7 +567,8 @@ func TestReleaseExecutionMigrationDefinesAuthorizedAuditableJobs(t *testing.T) {
|
||||
func TestClaimSQLUsesSkipLockedAndReclaimsExpiredRunningJobs(t *testing.T) {
|
||||
sql := strings.ToLower(claimJobSQL)
|
||||
for _, required := range []string{
|
||||
"for update skip locked",
|
||||
"for update of jobs skip locked",
|
||||
"ingestion_concurrency",
|
||||
"state in ('pending', 'retry')",
|
||||
"state = 'running'",
|
||||
"lease_until <",
|
||||
|
||||
@@ -0,0 +1,364 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
)
|
||||
|
||||
// fulfillCatalogIntentsTx is the policy-driven ingestion core: saving an
|
||||
// annotation in 来源与目录 records one durable intent per selected file and,
|
||||
// for importable policies, materializes a versioned source item, enqueues an
|
||||
// artifact.fetch job (bytes come from the live source; a matching local copy
|
||||
// short-circuits the download) and creates one 'planned' publication per routed
|
||||
// knowledge base. Planned publications are what 解析监控 lists as 待入库;
|
||||
// 开始处理 promotes them within the configured concurrency budget. Blocked
|
||||
// intents retire any planned publication and flip the item policy so the
|
||||
// knowledge-base diff demands removal of anything still live.
|
||||
func fulfillCatalogIntentsTx(ctx context.Context, tx pgx.Tx, request governance.CatalogRouteRequest, files []governance.CatalogRouteFile, now time.Time) (governance.CatalogRouteResult, error) {
|
||||
targetKnowledgeBaseIDs := request.TargetKnowledgeBaseIDs
|
||||
if len(targetKnowledgeBaseIDs) == 0 && request.TargetKnowledgeBaseID != "" {
|
||||
targetKnowledgeBaseIDs = []string{request.TargetKnowledgeBaseID}
|
||||
}
|
||||
result := governance.CatalogRouteResult{SelectedFiles: len(files)}
|
||||
for _, file := range files {
|
||||
recorded, fetchQueued, planned, blocked, err := fulfillCatalogIntent(ctx, tx, request, file, targetKnowledgeBaseIDs, now)
|
||||
if err != nil {
|
||||
return governance.CatalogRouteResult{}, err
|
||||
}
|
||||
if !recorded {
|
||||
result.UnboundFiles++
|
||||
continue
|
||||
}
|
||||
result.IntentsRecorded++
|
||||
if blocked {
|
||||
result.BlockedItems++
|
||||
continue
|
||||
}
|
||||
if fetchQueued {
|
||||
result.FetchQueued++
|
||||
}
|
||||
result.PlannedPublications += planned
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// fulfillCatalogIntent applies one file decision. It returns recorded=false
|
||||
// only when the node vanished mid-transaction; fetchQueued reports a newly
|
||||
// enqueued artifact.fetch job and planned the number of publications created.
|
||||
func fulfillCatalogIntent(ctx context.Context, tx pgx.Tx, request governance.CatalogRouteRequest, file governance.CatalogRouteFile, targetKnowledgeBaseIDs []string, now time.Time) (bool, bool, int, bool, error) {
|
||||
policy := string(request.Visibility)
|
||||
if _, err := tx.Exec(ctx, `INSERT INTO catalog_import_intents(
|
||||
source_connection_id,node_id,visibility,decided_by,reason,relative_path,name,size_bytes,modified_at,content_sha256,
|
||||
locator_hash,fetch_state,last_error,created_at,updated_at)
|
||||
VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,'pending','',NOW(),NOW())
|
||||
ON CONFLICT(source_connection_id,node_id) DO UPDATE SET
|
||||
visibility=EXCLUDED.visibility,decided_by=EXCLUDED.decided_by,reason=EXCLUDED.reason,
|
||||
relative_path=EXCLUDED.relative_path,name=EXCLUDED.name,size_bytes=EXCLUDED.size_bytes,
|
||||
modified_at=EXCLUDED.modified_at,content_sha256=EXCLUDED.content_sha256,
|
||||
locator_hash=EXCLUDED.locator_hash,
|
||||
fetch_state=CASE WHEN catalog_import_intents.content_sha256 IS DISTINCT FROM EXCLUDED.content_sha256
|
||||
OR catalog_import_intents.size_bytes IS DISTINCT FROM EXCLUDED.size_bytes
|
||||
OR catalog_import_intents.modified_at IS DISTINCT FROM EXCLUDED.modified_at
|
||||
THEN 'pending' ELSE catalog_import_intents.fetch_state END,
|
||||
last_error=CASE WHEN catalog_import_intents.content_sha256 IS DISTINCT FROM EXCLUDED.content_sha256
|
||||
OR catalog_import_intents.size_bytes IS DISTINCT FROM EXCLUDED.size_bytes
|
||||
OR catalog_import_intents.modified_at IS DISTINCT FROM EXCLUDED.modified_at
|
||||
THEN '' ELSE catalog_import_intents.last_error END,
|
||||
updated_at=NOW()`,
|
||||
request.ConnectionID, file.NodeID, policy, request.OperatorRef, request.Reason,
|
||||
file.RelativePath, file.Name, file.SizeBytes, file.ModifiedAt, file.ContentSHA256, file.LocatorHash); err != nil {
|
||||
return false, false, 0, false, fmt.Errorf("record catalog intent: %w", err)
|
||||
}
|
||||
|
||||
var existingItemID *string
|
||||
err := tx.QueryRow(ctx, `SELECT source_item_id FROM catalog_import_intents
|
||||
WHERE source_connection_id=$1 AND node_id=$2 FOR UPDATE`, request.ConnectionID, file.NodeID).Scan(&existingItemID)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return false, false, 0, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, false, 0, false, fmt.Errorf("read catalog intent: %w", err)
|
||||
}
|
||||
|
||||
itemID := ""
|
||||
if existingItemID != nil {
|
||||
itemID = *existingItemID
|
||||
}
|
||||
if itemID == "" {
|
||||
newItemID, err := newOpaqueID("si-")
|
||||
if err != nil {
|
||||
return false, false, 0, false, err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `INSERT INTO source_items(
|
||||
id,source_connection_id,stable_external_id,canonical_uri_hash,title,item_type,revision,
|
||||
visibility,visibility_origin,source_change,absence_streak,identity_confidence,
|
||||
policy_state,created_at,updated_at)
|
||||
VALUES($1,$2,$3,$4,$5,'file','',$7,'configured_scope','unchanged',0,'object_id',$6,NOW(),NOW())`,
|
||||
newItemID, request.ConnectionID, "cat:"+file.NodeID,
|
||||
sha256Hex("policy:"+request.ConnectionID+"\x00"+file.NodeID),
|
||||
file.Name, policy, itemVisibilityFor(request.Visibility)); err != nil {
|
||||
return false, false, 0, false, fmt.Errorf("materialize policy source item: %w", err)
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `UPDATE catalog_import_intents SET source_item_id=$3 WHERE source_connection_id=$1 AND node_id=$2`,
|
||||
request.ConnectionID, file.NodeID, newItemID); err != nil {
|
||||
return false, false, 0, false, fmt.Errorf("bind catalog intent item: %w", err)
|
||||
}
|
||||
itemID = newItemID
|
||||
}
|
||||
|
||||
if request.Visibility == governance.VisibilityBlocked {
|
||||
if _, err := tx.Exec(ctx, `UPDATE source_items SET policy_state='blocked',updated_at=NOW() WHERE id=$1`, itemID); err != nil {
|
||||
return false, false, 0, false, fmt.Errorf("block policy item: %w", err)
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `UPDATE test_publications SET superseded_at=$2,updated_at=$2
|
||||
WHERE source_item_version_id IN (SELECT id FROM source_item_versions WHERE source_item_id=$1)
|
||||
AND superseded_at IS NULL AND processing_state='planned'`, itemID, now); err != nil {
|
||||
return false, false, 0, false, fmt.Errorf("retire blocked planned publications: %w", err)
|
||||
}
|
||||
return true, false, 0, true, nil
|
||||
}
|
||||
|
||||
revision := "locator:" + file.LocatorHash
|
||||
if file.ContentSHA256 != nil && *file.ContentSHA256 != "" {
|
||||
revision = "sha:" + *file.ContentSHA256
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `UPDATE source_items
|
||||
SET policy_state=$2,visibility=$3,visibility_origin='configured_scope',updated_at=NOW() WHERE id=$1`,
|
||||
itemID, policy, itemVisibilityFor(request.Visibility)); err != nil {
|
||||
return false, false, 0, false, fmt.Errorf("update policy item: %w", err)
|
||||
}
|
||||
|
||||
// Resolve the version this intent points at. A bound item whose current
|
||||
// version already carries verified bytes is reused without any fetch — the
|
||||
// 本地已有先比对 requirement. Only when the node proves different content
|
||||
// (scan-reported hash mismatch) or bytes were never fetched do we create a
|
||||
// revision and enqueue artifact.fetch.
|
||||
versionID := ""
|
||||
contentState := ""
|
||||
var curID, curSHA string
|
||||
err = tx.QueryRow(ctx, `SELECT version.id,COALESCE(content.sha256,''),version.content_state
|
||||
FROM source_items item
|
||||
JOIN source_item_versions version ON version.source_item_id=item.id AND version.revision=item.revision
|
||||
LEFT JOIN content_objects content ON content.id=version.content_object_id
|
||||
WHERE item.id=$1`, itemID).Scan(&curID, &curSHA, &contentState)
|
||||
if err != nil && !errors.Is(err, pgx.ErrNoRows) {
|
||||
return false, false, 0, false, fmt.Errorf("inspect current item version: %w", err)
|
||||
}
|
||||
nodeSHA := ""
|
||||
if file.ContentSHA256 != nil {
|
||||
nodeSHA = *file.ContentSHA256
|
||||
}
|
||||
reuse := err == nil && contentState == string(governance.ContentVerified) && (nodeSHA == "" || nodeSHA == curSHA)
|
||||
if !reuse {
|
||||
contentState = "unknown"
|
||||
err = tx.QueryRow(ctx, `SELECT version.id,version.content_state
|
||||
FROM source_item_versions version WHERE version.source_item_id=$1 AND version.revision=$2`,
|
||||
itemID, revision).Scan(&versionID, &contentState)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
newVersionID, err := newOpaqueID("sv-")
|
||||
if err != nil {
|
||||
return false, false, 0, false, err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `INSERT INTO source_item_versions(
|
||||
id,source_item_id,revision,content_object_id,content_state,raw_record_summary,source_updated_at,created_at,updated_at)
|
||||
VALUES($1,$2,$3,NULL,'unknown','{}'::jsonb,$4,NOW(),NOW())`, newVersionID, itemID, revision, file.ModifiedAt); err != nil {
|
||||
return false, false, 0, false, fmt.Errorf("create policy version: %w", err)
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `UPDATE source_items SET revision=$2,updated_at=NOW() WHERE id=$1`, itemID, revision); err != nil {
|
||||
return false, false, 0, false, fmt.Errorf("advance policy item revision: %w", err)
|
||||
}
|
||||
versionID = newVersionID
|
||||
} else if err != nil {
|
||||
return false, false, 0, false, fmt.Errorf("lookup policy version: %w", err)
|
||||
}
|
||||
} else {
|
||||
versionID = curID
|
||||
}
|
||||
|
||||
planned := 0
|
||||
for _, kbID := range targetKnowledgeBaseIDs {
|
||||
scope := kbScopeFor(request, kbID)
|
||||
publicationID, err := newOpaqueID("tp-")
|
||||
if err != nil {
|
||||
return false, false, 0, false, err
|
||||
}
|
||||
tag, err := tx.Exec(ctx, `INSERT INTO test_publications(
|
||||
id,source_item_version_id,test_kb_id,knowledge_id,ingest_state,processing_state,retrieval_state,
|
||||
parser_generation,index_generation,last_error,retry_count,target_scope,created_at,updated_at)
|
||||
VALUES($1,$2,$3,'','pending','planned','unpublished','','','',0,$4,NOW(),NOW())
|
||||
ON CONFLICT (source_item_version_id,test_kb_id) DO NOTHING`,
|
||||
publicationID, versionID, kbID, scope)
|
||||
if err != nil {
|
||||
return false, false, 0, false, fmt.Errorf("plan publication: %w", err)
|
||||
}
|
||||
planned += int(tag.RowsAffected())
|
||||
}
|
||||
|
||||
fetchQueued := false
|
||||
if contentState != string(governance.ContentVerified) {
|
||||
jobID, err := newOpaqueID("job-")
|
||||
if err != nil {
|
||||
return false, false, 0, false, err
|
||||
}
|
||||
payload, err := json.Marshal(map[string]string{"source_connection_id": request.ConnectionID, "node_id": file.NodeID, "version_id": versionID})
|
||||
if err != nil {
|
||||
return false, false, 0, false, errors.New("encode artifact fetch job")
|
||||
}
|
||||
tag, err := tx.Exec(ctx, `INSERT INTO jobs(id,type,payload,state,priority,available_at,max_attempts,idempotency_key,created_at,updated_at)
|
||||
VALUES($1,'artifact.fetch',$2::jsonb,'pending',20,$3,8,$4,$3,$3) ON CONFLICT (idempotency_key) DO NOTHING`,
|
||||
jobID, payload, now, "artifact-fetch:"+versionID)
|
||||
if err != nil {
|
||||
return false, false, 0, false, fmt.Errorf("enqueue artifact fetch: %w", err)
|
||||
}
|
||||
fetchQueued = tag.RowsAffected() > 0
|
||||
if fetchQueued {
|
||||
if _, err := tx.Exec(ctx, `UPDATE catalog_import_intents SET fetch_state='pending',last_error='',updated_at=NOW()
|
||||
WHERE source_connection_id=$1 AND node_id=$2 AND fetch_state IN ('failed','missing_local','fetching')`, request.ConnectionID, file.NodeID); err != nil {
|
||||
return false, false, 0, false, fmt.Errorf("reset catalog intent fetch state: %w", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return true, fetchQueued, planned, false, nil
|
||||
}
|
||||
|
||||
// kbScopeFor maps a routed knowledge base id to its diff scope label.
|
||||
func kbScopeFor(request governance.CatalogRouteRequest, knowledgeBaseID string) string {
|
||||
if request.Visibility == governance.VisibilityExternal {
|
||||
return "external"
|
||||
}
|
||||
if request.Visibility == governance.VisibilityBoth && len(request.TargetKnowledgeBaseIDs) > 1 &&
|
||||
knowledgeBaseID == request.TargetKnowledgeBaseIDs[len(request.TargetKnowledgeBaseIDs)-1] {
|
||||
return "external"
|
||||
}
|
||||
return "internal"
|
||||
}
|
||||
|
||||
// itemVisibilityFor mirrors source_items' two-value visibility CHECK: both and
|
||||
// blocked decisions are governed through policy_state; the legacy visibility
|
||||
// column keeps a valid value and is widened to external for both.
|
||||
func itemVisibilityFor(visibility governance.Visibility) string {
|
||||
if visibility == governance.VisibilityInternal {
|
||||
return string(governance.VisibilityInternal)
|
||||
}
|
||||
return string(governance.VisibilityExternal)
|
||||
}
|
||||
|
||||
func sha256Hex(value string) string {
|
||||
sum := sha256.Sum256([]byte(value))
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// MarkArtifactFetched finalizes one artifact.fetch success: it (re)uses the
|
||||
// content_objects row for the verified hash (content-level dedupe), attaches it
|
||||
// to the pending version and flips the intent to ready. matchedLocal records
|
||||
// that the bytes already existed locally and no source download happened.
|
||||
func (s *Store) MarkArtifactFetched(ctx context.Context, versionID, sha256Value string, sizeBytes int64, spoolRef, validatedFormat string, matchedLocal bool, now time.Time) error {
|
||||
if len(sha256Value) != 64 || spoolRef == "" || sizeBytes < 0 || validatedFormat == "" {
|
||||
return errors.New("invalid artifact fetch result")
|
||||
}
|
||||
var tx pgx.Tx
|
||||
ownsTransaction := false
|
||||
if existing, ok := s.db.(pgx.Tx); ok {
|
||||
tx = existing
|
||||
} else {
|
||||
beginner, ok := s.db.(interface {
|
||||
Begin(context.Context) (pgx.Tx, error)
|
||||
})
|
||||
if !ok {
|
||||
return errors.New("artifact fetch transactions are unavailable")
|
||||
}
|
||||
var err error
|
||||
tx, err = beginner.Begin(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("begin artifact fetch: %w", err)
|
||||
}
|
||||
ownsTransaction = true
|
||||
defer tx.Rollback(ctx)
|
||||
}
|
||||
var connectionID, nodeID string
|
||||
err := tx.QueryRow(ctx, `SELECT intent.source_connection_id,intent.node_id
|
||||
FROM catalog_import_intents intent
|
||||
JOIN source_item_versions version ON version.source_item_id=intent.source_item_id
|
||||
WHERE version.id=$1 FOR UPDATE OF intent`, versionID).Scan(&connectionID, &nodeID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("locate intent for fetched artifact: %w", err)
|
||||
}
|
||||
freshID, err := newOpaqueID("co-")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
contentID := ""
|
||||
err = tx.QueryRow(ctx, `INSERT INTO content_objects(id,sha256,size_bytes,mime,validated_format,local_spool_ref,content_state,created_at,updated_at)
|
||||
VALUES($1,$2,$3,'',$5,$4,'verified',NOW(),NOW())
|
||||
ON CONFLICT(sha256) DO UPDATE SET
|
||||
size_bytes=EXCLUDED.size_bytes,
|
||||
validated_format=CASE WHEN content_objects.validated_format='' THEN EXCLUDED.validated_format ELSE content_objects.validated_format END,
|
||||
local_spool_ref=CASE WHEN content_objects.local_spool_ref='' THEN EXCLUDED.local_spool_ref ELSE content_objects.local_spool_ref END,
|
||||
content_state='verified',updated_at=NOW()
|
||||
RETURNING id`,
|
||||
freshID, sha256Value, sizeBytes, spoolRef, validatedFormat).Scan(&contentID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("record fetched content: %w", err)
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `UPDATE source_item_versions SET
|
||||
content_object_id=$2,content_state='verified',updated_at=$4,
|
||||
raw_record_summary=jsonb_set(raw_record_summary,'{matched_local}',$3::jsonb)
|
||||
WHERE id=$1`,
|
||||
versionID, contentID, mustJSONBool(matchedLocal), now.UTC()); err != nil {
|
||||
return fmt.Errorf("bind fetched content: %w", err)
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `UPDATE catalog_import_intents SET fetch_state='ready',last_error='',updated_at=NOW()
|
||||
WHERE source_connection_id=$1 AND node_id=$2`, connectionID, nodeID); err != nil {
|
||||
return fmt.Errorf("mark intent fetched: %w", err)
|
||||
}
|
||||
if ownsTransaction {
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return fmt.Errorf("commit artifact fetch: %w", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func mustJSONBool(value bool) string {
|
||||
if value {
|
||||
return "true"
|
||||
}
|
||||
return "false"
|
||||
}
|
||||
|
||||
// MarkArtifactFetchMissing parks an intent whose source-side file disappeared
|
||||
// (a 404 on download, not a transient outage) so the UI can ask for a rescan.
|
||||
func (s *Store) MarkArtifactFetchMissing(ctx context.Context, connectionID, nodeID string, now time.Time) error {
|
||||
if _, err := s.db.Exec(ctx, `UPDATE catalog_import_intents SET fetch_state='missing_local',updated_at=$3
|
||||
WHERE source_connection_id=$1 AND node_id=$2 AND fetch_state<>'ready'`, connectionID, nodeID, now.UTC()); err != nil {
|
||||
return fmt.Errorf("record artifact fetch miss: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// MarkArtifactFetchFailed records a fetch failure for the intent so 解析监控
|
||||
// can show why an item is not yet 待入库.
|
||||
func (s *Store) MarkArtifactFetchFailed(ctx context.Context, connectionID, nodeID, message string, now time.Time) error {
|
||||
if _, err := s.db.Exec(ctx, `UPDATE catalog_import_intents SET fetch_state='failed',last_error=$3,updated_at=$4
|
||||
WHERE source_connection_id=$1 AND node_id=$2 AND fetch_state<>'ready'`, connectionID, nodeID, truncateMonitorText(message), now.UTC()); err != nil {
|
||||
return fmt.Errorf("record artifact fetch failure: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func truncateMonitorText(value string) string {
|
||||
runes := []rune(value)
|
||||
if len(runes) <= 500 {
|
||||
return value
|
||||
}
|
||||
return string(runes[:500])
|
||||
}
|
||||
@@ -16,9 +16,12 @@ func (s *Store) StartTestPublications(ctx context.Context, request governance.St
|
||||
result := governance.StartTestPublicationsResult{Selected: len(request.PublicationIDs)}
|
||||
operator := strings.TrimSpace(request.OperatorRef)
|
||||
reason := strings.TrimSpace(request.Reason)
|
||||
if len(request.PublicationIDs) == 0 || len(request.PublicationIDs) > 100 || operator == "" || len(operator) > 128 || reason == "" || len(reason) > 500 || request.Now.IsZero() {
|
||||
if len(request.PublicationIDs) == 0 || len(request.PublicationIDs) > 100 || operator == "" || len(operator) > 128 || len(reason) > 500 || request.Now.IsZero() {
|
||||
return governance.StartTestPublicationsResult{}, errors.New("invalid processing confirmation")
|
||||
}
|
||||
if reason == "" {
|
||||
reason = "开始处理"
|
||||
}
|
||||
seen := make(map[string]bool, len(request.PublicationIDs))
|
||||
for _, publicationID := range request.PublicationIDs {
|
||||
if !validRouteID(publicationID) || seen[publicationID] {
|
||||
@@ -50,22 +53,23 @@ func (s *Store) StartTestPublications(ctx context.Context, request governance.St
|
||||
|
||||
for _, publicationID := range request.PublicationIDs {
|
||||
var ready, confirmed, uploadJobExists bool
|
||||
var processingState string
|
||||
err = tx.QueryRow(ctx, `
|
||||
SELECT p.ingest_state='pending'
|
||||
AND p.processing_state='pending'
|
||||
AND p.retrieval_state='unpublished'
|
||||
AND p.knowledge_id=''
|
||||
AND version.content_state='verified'
|
||||
AND content.content_state='verified'
|
||||
AND content.local_spool_ref<>''
|
||||
AND content.size_bytes>0 AS ready,
|
||||
p.processing_state,
|
||||
EXISTS(SELECT 1 FROM processing_confirmations confirmation WHERE confirmation.publication_id=p.id),
|
||||
EXISTS(SELECT 1 FROM jobs job WHERE job.type='test.upload' AND job.payload->>'publication_id'=p.id)
|
||||
FROM test_publications p
|
||||
JOIN source_item_versions version ON version.id=p.source_item_version_id
|
||||
JOIN content_objects content ON content.id=version.content_object_id
|
||||
WHERE p.id=$1
|
||||
FOR UPDATE OF p`, publicationID).Scan(&ready, &confirmed, &uploadJobExists)
|
||||
WHERE p.id=$1 AND p.superseded_at IS NULL
|
||||
FOR UPDATE OF p`, publicationID).Scan(&ready, &processingState, &confirmed, &uploadJobExists)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
result.Unavailable++
|
||||
continue
|
||||
@@ -77,10 +81,18 @@ FOR UPDATE OF p`, publicationID).Scan(&ready, &confirmed, &uploadJobExists)
|
||||
result.AlreadyStarted++
|
||||
continue
|
||||
}
|
||||
// 开始处理 promotes both fresh plans and verified pending rows.
|
||||
if processingState != "planned" && processingState != "pending" {
|
||||
result.Unavailable++
|
||||
continue
|
||||
}
|
||||
if !ready {
|
||||
result.Unavailable++
|
||||
continue
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `UPDATE test_publications SET processing_state='pending',updated_at=$2 WHERE id=$1 AND processing_state='planned'`, publicationID, request.Now); err != nil {
|
||||
return governance.StartTestPublicationsResult{}, fmt.Errorf("promote planned publication: %w", err)
|
||||
}
|
||||
confirmationID, err := newOpaqueID("pc-")
|
||||
if err != nil {
|
||||
return governance.StartTestPublicationsResult{}, err
|
||||
|
||||
@@ -13,17 +13,29 @@ RETURNING id, type, payload, state, priority, available_at,
|
||||
`
|
||||
|
||||
const claimJobSQL = `
|
||||
WITH candidate AS (
|
||||
SELECT id
|
||||
FROM jobs
|
||||
WITH budget AS (
|
||||
SELECT COALESCE((
|
||||
SELECT ingestion_concurrency
|
||||
FROM ui_setting_revisions
|
||||
WHERE state = 'active'
|
||||
LIMIT 1
|
||||
), 2) AS ingestion_slots
|
||||
), candidate AS (
|
||||
SELECT jobs.id
|
||||
FROM jobs, budget
|
||||
WHERE (
|
||||
(state IN ('pending', 'retry') AND available_at <= $2)
|
||||
OR (state = 'running' AND lease_until < $2)
|
||||
(jobs.state IN ('pending', 'retry') AND jobs.available_at <= $2)
|
||||
OR (jobs.state = 'running' AND jobs.lease_until < $2)
|
||||
)
|
||||
AND attempt < max_attempts
|
||||
AND type = ANY($4::text[])
|
||||
ORDER BY priority DESC, available_at ASC, created_at ASC, id ASC
|
||||
FOR UPDATE SKIP LOCKED
|
||||
AND jobs.attempt < jobs.max_attempts
|
||||
AND jobs.type = ANY($4::text[])
|
||||
AND (
|
||||
jobs.type <> 'test.upload'
|
||||
OR (SELECT COUNT(*) FROM jobs active
|
||||
WHERE active.type = 'test.upload' AND active.state = 'running') < budget.ingestion_slots
|
||||
)
|
||||
ORDER BY jobs.priority DESC, jobs.available_at ASC, jobs.created_at ASC, jobs.id ASC
|
||||
FOR UPDATE OF jobs SKIP LOCKED
|
||||
LIMIT 1
|
||||
)
|
||||
UPDATE jobs AS j
|
||||
|
||||
@@ -51,7 +51,7 @@ INSERT INTO source_catalog_heads(source_connection_id,snapshot_id,activated_at)
|
||||
ConnectionID: "conn-s6", NodeIDs: []string{"folder-top"}, Visibility: governance.VisibilityInternal,
|
||||
TargetKnowledgeBaseID: "kb-internal", OperatorRef: "engineer-1", Reason: "S6目录递归标注", Now: now,
|
||||
})
|
||||
if err != nil || result.SelectedFiles != 1 || result.RoutableFiles != 1 {
|
||||
if err != nil || result.SelectedFiles != 1 || result.PlannedPublications != 1 || result.FetchQueued != 0 {
|
||||
t.Fatalf("route=%#v err=%v", result, err)
|
||||
}
|
||||
// Effective visibility of the deep item must be internal via the directory rule.
|
||||
|
||||
@@ -128,7 +128,7 @@ SELECT p.id, si.title, p.ingest_state, p.processing_state, p.retrieval_state,
|
||||
JOIN source_item_versions other_version ON other_version.id=other.source_item_version_id
|
||||
WHERE other.id<>p.id AND other_version.source_item_id=siv.source_item_id
|
||||
) THEN 'updated' ELSE 'added' END AS change_type,
|
||||
(p.ingest_state='pending' AND p.processing_state='pending'
|
||||
(p.ingest_state='pending' AND p.processing_state IN ('pending','planned')
|
||||
AND p.retrieval_state='unpublished' AND p.knowledge_id=''
|
||||
AND co.content_state='verified' AND co.local_spool_ref<>'' AND co.size_bytes>0
|
||||
AND NOT EXISTS (SELECT 1 FROM processing_confirmations confirmation WHERE confirmation.publication_id=p.id)
|
||||
@@ -142,8 +142,8 @@ SELECT p.id, si.title, p.ingest_state, p.processing_state, p.retrieval_state,
|
||||
FROM test_publications p
|
||||
JOIN source_item_versions siv ON siv.id = p.source_item_version_id
|
||||
JOIN source_items si ON si.id = siv.source_item_id
|
||||
JOIN content_objects co ON co.id = siv.content_object_id
|
||||
WHERE ($1 = '' OR p.processing_state = $1) AND ($2 = '' OR p.target_scope = $2)
|
||||
LEFT JOIN content_objects co ON co.id = siv.content_object_id
|
||||
WHERE ($1 = '' OR p.processing_state = $1) AND ($2 = '' OR p.target_scope = $2) AND p.superseded_at IS NULL
|
||||
ORDER BY p.updated_at DESC, p.id
|
||||
LIMIT $3 OFFSET $4
|
||||
`
|
||||
|
||||
@@ -13,7 +13,7 @@ import (
|
||||
|
||||
var ErrInvalidUISettings = errors.New("invalid UI settings")
|
||||
|
||||
const uiSettingsColumns = `id,state,document_refresh_seconds,resource_refresh_seconds,stalled_after_seconds,finalizing_after_seconds,updated_by,created_at,activated_at,retired_at,error_code`
|
||||
const uiSettingsColumns = `id,state,document_refresh_seconds,resource_refresh_seconds,stalled_after_seconds,finalizing_after_seconds,ingestion_concurrency,updated_by,created_at,activated_at,retired_at,error_code`
|
||||
|
||||
const getActiveUISettingsSQL = `
|
||||
SELECT ` + uiSettingsColumns + `
|
||||
@@ -24,8 +24,8 @@ WHERE state = 'active'
|
||||
const createUISettingsCandidateSQL = `
|
||||
INSERT INTO ui_setting_revisions(
|
||||
id,state,document_refresh_seconds,resource_refresh_seconds,
|
||||
stalled_after_seconds,finalizing_after_seconds,updated_by,created_at
|
||||
) VALUES($1,'validated',$2,$3,$4,$5,$6,$7)
|
||||
stalled_after_seconds,finalizing_after_seconds,ingestion_concurrency,updated_by,created_at
|
||||
) VALUES($1,'validated',$2,$3,$4,$5,$6,$7,$8)
|
||||
RETURNING ` + uiSettingsColumns + `
|
||||
`
|
||||
|
||||
@@ -63,7 +63,8 @@ func (s *Store) CreateUISettingsCandidate(ctx context.Context, request governanc
|
||||
}
|
||||
return scanUISettings(s.db.QueryRow(ctx, createUISettingsCandidateSQL,
|
||||
id, request.DocumentRefreshSeconds, request.ResourceRefreshSeconds,
|
||||
request.StalledAfterSeconds, request.FinalizingAfterSeconds, request.UpdatedBy, request.Now.UTC()))
|
||||
request.StalledAfterSeconds, request.FinalizingAfterSeconds, request.IngestionConcurrency,
|
||||
request.UpdatedBy, request.Now.UTC()))
|
||||
}
|
||||
|
||||
func (s *Store) ActivateUISettings(ctx context.Context, revisionID string, now time.Time) (governance.UISettings, error) {
|
||||
@@ -124,6 +125,7 @@ func scanUISettings(row interface{ Scan(...any) error }) (governance.UISettings,
|
||||
var settings governance.UISettings
|
||||
if err := row.Scan(&settings.ID, &settings.State, &settings.DocumentRefreshSeconds,
|
||||
&settings.ResourceRefreshSeconds, &settings.StalledAfterSeconds, &settings.FinalizingAfterSeconds,
|
||||
&settings.IngestionConcurrency,
|
||||
&settings.UpdatedBy, &settings.CreatedAt, &settings.ActivatedAt, &settings.RetiredAt, &settings.ErrorCode); err != nil {
|
||||
return governance.UISettings{}, fmt.Errorf("scan UI settings: %w", err)
|
||||
}
|
||||
|
||||
@@ -31,7 +31,7 @@ func TestUISettingsActivationRetiresPreviousRevisionIntegration(t *testing.T) {
|
||||
t.Fatalf("previous=%#v error=%v", previous, err)
|
||||
}
|
||||
now := time.Date(2026, 9, 10, 7, 0, 0, 0, time.UTC)
|
||||
candidate, err := store.CreateUISettingsCandidate(ctx, governance.CreateUISettingsCandidateRequest{DocumentRefreshSeconds: 30, ResourceRefreshSeconds: 60, StalledAfterSeconds: 3600, FinalizingAfterSeconds: 14400, UpdatedBy: "engineer-1", Now: now})
|
||||
candidate, err := store.CreateUISettingsCandidate(ctx, governance.CreateUISettingsCandidateRequest{DocumentRefreshSeconds: 30, ResourceRefreshSeconds: 60, StalledAfterSeconds: 3600, FinalizingAfterSeconds: 14400, IngestionConcurrency: 2, UpdatedBy: "engineer-1", Now: now})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -14,7 +14,7 @@ func TestUISettingsCandidateAndActiveProjectionAreVersioned(t *testing.T) {
|
||||
now := time.Date(2026, 9, 10, 6, 0, 0, 0, time.UTC)
|
||||
row := uiSettingsRow{id: "settings-safe", state: "validated", documentRefresh: 30, resourceRefresh: 60, stalledAfter: 3600, finalizingAfter: 14400, updatedBy: "engineer-1", createdAt: now}
|
||||
db := &recordingDB{row: row}
|
||||
candidate, err := NewStore(db).CreateUISettingsCandidate(context.Background(), governance.CreateUISettingsCandidateRequest{DocumentRefreshSeconds: 30, ResourceRefreshSeconds: 60, StalledAfterSeconds: 3600, FinalizingAfterSeconds: 14400, UpdatedBy: "engineer-1", Now: now})
|
||||
candidate, err := NewStore(db).CreateUISettingsCandidate(context.Background(), governance.CreateUISettingsCandidateRequest{DocumentRefreshSeconds: 30, ResourceRefreshSeconds: 60, StalledAfterSeconds: 3600, FinalizingAfterSeconds: 14400, IngestionConcurrency: 2, UpdatedBy: "engineer-1", Now: now})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -53,6 +53,7 @@ func TestUISettingsRejectInvalidCandidate(t *testing.T) {
|
||||
type uiSettingsRow struct {
|
||||
id, state, updatedBy, errorCode string
|
||||
documentRefresh, resourceRefresh, stalledAfter, finalizingAfter int
|
||||
ingestionConcurrency int
|
||||
createdAt time.Time
|
||||
activatedAt, retiredAt *time.Time
|
||||
}
|
||||
@@ -60,6 +61,7 @@ type uiSettingsRow struct {
|
||||
func (row uiSettingsRow) Scan(dest ...any) error {
|
||||
*dest[0].(*string), *dest[1].(*string) = row.id, row.state
|
||||
*dest[2].(*int), *dest[3].(*int), *dest[4].(*int), *dest[5].(*int) = row.documentRefresh, row.resourceRefresh, row.stalledAfter, row.finalizingAfter
|
||||
*dest[6].(*string), *dest[7].(*time.Time), *dest[8].(**time.Time), *dest[9].(**time.Time), *dest[10].(*string) = row.updatedBy, row.createdAt, row.activatedAt, row.retiredAt, row.errorCode
|
||||
*dest[6].(*int) = row.ingestionConcurrency
|
||||
*dest[7].(*string), *dest[8].(*time.Time), *dest[9].(**time.Time), *dest[10].(**time.Time), *dest[11].(*string) = row.updatedBy, row.createdAt, row.activatedAt, row.retiredAt, row.errorCode
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -34,8 +34,9 @@ type Artifact struct {
|
||||
}
|
||||
|
||||
var supportedFormats = map[string]struct{}{
|
||||
"pdf": {}, "docx": {}, "doc": {}, "xlsx": {}, "xls": {}, "pptx": {}, "ppt": {},
|
||||
"pdf": {}, "docx": {}, "doc": {}, "xlsx": {}, "xls": {},
|
||||
"md": {}, "txt": {}, "csv": {}, "json": {},
|
||||
"png": {}, "jpg": {}, "jpeg": {}, "gif": {}, "bmp": {}, "webp": {},
|
||||
}
|
||||
|
||||
func InspectVerified(root, relativeRef, format string, maxBytes int64) (Expectation, error) {
|
||||
@@ -137,7 +138,7 @@ func validateFormat(file *os.File, format string, size int64) error {
|
||||
if _, err := io.ReadFull(file, header); err != nil || !bytes.Equal(header, []byte("%PDF-")) {
|
||||
return errors.New("invalid PDF magic")
|
||||
}
|
||||
case "docx", "xlsx", "pptx":
|
||||
case "docx", "xlsx":
|
||||
archive, err := zip.NewReader(file, size)
|
||||
if err != nil {
|
||||
return errors.New("invalid Office ZIP")
|
||||
@@ -145,7 +146,6 @@ func validateFormat(file *os.File, format string, size int64) error {
|
||||
required := map[string]string{
|
||||
"docx": "word/document.xml",
|
||||
"xlsx": "xl/workbook.xml",
|
||||
"pptx": "ppt/presentation.xml",
|
||||
}[format]
|
||||
foundContentTypes := false
|
||||
foundRequired := false
|
||||
@@ -160,12 +160,37 @@ func validateFormat(file *os.File, format string, size int64) error {
|
||||
if !foundContentTypes || !foundRequired {
|
||||
return errors.New("missing Office package entry")
|
||||
}
|
||||
case "doc", "xls", "ppt":
|
||||
case "doc", "xls":
|
||||
header := make([]byte, 8)
|
||||
oleMagic := []byte{0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1}
|
||||
if _, err := io.ReadFull(file, header); err != nil || !bytes.Equal(header, oleMagic) {
|
||||
return errors.New("invalid legacy Office magic")
|
||||
}
|
||||
case "png", "jpg", "jpeg", "gif", "bmp", "webp":
|
||||
header := make([]byte, 12)
|
||||
n, err := file.Read(header)
|
||||
if err != nil || n < 12 {
|
||||
return errors.New("invalid image header")
|
||||
}
|
||||
valid := false
|
||||
switch format {
|
||||
case "png":
|
||||
valid = bytes.Equal(header[:8], []byte{0x89, 'P', 'N', 'G', 0x0d, 0x0a, 0x1a, 0x0a})
|
||||
case "jpg", "jpeg":
|
||||
valid = header[0] == 0xff && header[1] == 0xd8 && header[2] == 0xff
|
||||
case "gif":
|
||||
valid = bytes.Equal(header[:6], []byte("GIF87a")) || bytes.Equal(header[:6], []byte("GIF89a"))
|
||||
case "bmp":
|
||||
valid = header[0] == 'B' && header[1] == 'M'
|
||||
case "webp":
|
||||
valid = bytes.Equal(header[:4], []byte("RIFF")) && bytes.Equal(header[8:12], []byte("WEBP"))
|
||||
}
|
||||
if !valid {
|
||||
return errors.New("invalid image magic")
|
||||
}
|
||||
if _, err := file.Seek(0, io.SeekStart); err != nil {
|
||||
return errors.New("rewind image artifact")
|
||||
}
|
||||
case "json":
|
||||
decoder := json.NewDecoder(file)
|
||||
var value any
|
||||
|
||||
@@ -0,0 +1,247 @@
|
||||
// Package artifactfetch turns a saved policy intent into verified bytes: it
|
||||
// shells out to scripts/artifact_fetch.py (which already holds every source
|
||||
// adapter: AList direct download, PP ledger, DingTalk export-tree/live export)
|
||||
// and records the verified content against the pending source item version.
|
||||
package artifactfetch
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"knowledge-governance/internal/jobs"
|
||||
)
|
||||
|
||||
type Request struct {
|
||||
ConnectionID string `json:"source_connection_id"`
|
||||
NodeID string `json:"node_id"`
|
||||
VersionID string `json:"version_id"`
|
||||
}
|
||||
|
||||
type Result struct {
|
||||
OK bool `json:"ok"`
|
||||
SpecID string `json:"spec_id"`
|
||||
NodeID string `json:"node_id"`
|
||||
ConnectionID string `json:"connection_id"`
|
||||
SHA256 string `json:"sha256"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
SpoolRef string `json:"spool_ref"`
|
||||
ValidatedFormat string `json:"validated_format"`
|
||||
MatchedLocal bool `json:"matched_local"`
|
||||
Missing bool `json:"missing"`
|
||||
Retryable bool `json:"retryable"`
|
||||
Code string `json:"code"`
|
||||
Error string `json:"error"`
|
||||
}
|
||||
|
||||
type Manifest struct {
|
||||
Request
|
||||
SourceType string
|
||||
SourceName string
|
||||
RelativePath string
|
||||
Name string
|
||||
SizeBytes *int64
|
||||
ModifiedAt *time.Time
|
||||
LocatorHash string
|
||||
}
|
||||
|
||||
type Store interface {
|
||||
LoadArtifactFetchManifest(context.Context, string, string) (Manifest, error)
|
||||
MarkArtifactFetched(context.Context, string, string, int64, string, string, bool, time.Time) error
|
||||
MarkArtifactFetchMissing(context.Context, string, string, time.Time) error
|
||||
MarkArtifactFetchFailed(context.Context, string, string, string, time.Time) error
|
||||
}
|
||||
|
||||
type Runner interface {
|
||||
Run(context.Context, Manifest) (Result, error)
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
Store Store
|
||||
Runner Runner
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
type Handler struct {
|
||||
store Store
|
||||
runner Runner
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
func NewHandler(config Config) (*Handler, error) {
|
||||
if config.Store == nil || config.Runner == nil {
|
||||
return nil, errors.New("artifact fetch dependencies are required")
|
||||
}
|
||||
if config.Now == nil {
|
||||
config.Now = time.Now
|
||||
}
|
||||
return &Handler{store: config.Store, runner: config.Runner, now: config.Now}, nil
|
||||
}
|
||||
|
||||
func (handler *Handler) Handle(ctx context.Context, job jobs.Job) error {
|
||||
var request Request
|
||||
decoder := json.NewDecoder(bytes.NewReader(job.Payload))
|
||||
decoder.DisallowUnknownFields()
|
||||
if err := decoder.Decode(&request); err != nil ||
|
||||
request.ConnectionID == "" || request.NodeID == "" || request.VersionID == "" {
|
||||
return jobs.Permanent(jobs.ErrCodeArtifactInvalid, errors.New("artifact_fetch_invalid_payload"))
|
||||
}
|
||||
manifest, err := handler.store.LoadArtifactFetchManifest(ctx, request.ConnectionID, request.NodeID)
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrManifestGone) {
|
||||
return jobs.Permanent(jobs.ErrCodeStateConflict, err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
result, err := handler.runner.Run(ctx, manifest)
|
||||
now := handler.now()
|
||||
if err != nil {
|
||||
_ = handler.store.MarkArtifactFetchFailed(ctx, request.ConnectionID, request.NodeID, err.Error(), now)
|
||||
return err
|
||||
}
|
||||
switch {
|
||||
case result.Missing:
|
||||
if err := handler.store.MarkArtifactFetchMissing(ctx, request.ConnectionID, request.NodeID, now); err != nil {
|
||||
return err
|
||||
}
|
||||
return jobs.Permanent(jobs.ErrCodeArtifactInvalid, errors.New("artifact_fetch_missing_source: "+safeMessage(result.Error)))
|
||||
case !result.OK:
|
||||
_ = handler.store.MarkArtifactFetchFailed(ctx, request.ConnectionID, request.NodeID, safeMessage(result.Error), now)
|
||||
if result.Retryable {
|
||||
return errors.New("artifact_fetch_retryable: " + safeMessage(result.Code))
|
||||
}
|
||||
return jobs.Permanent(jobs.ErrCodeArtifactInvalid, errors.New("artifact_fetch_rejected: "+safeMessage(result.Code)))
|
||||
}
|
||||
if result.SHA256 == "" || result.SpoolRef == "" || result.SizeBytes <= 0 || result.NodeID != request.NodeID || result.ValidatedFormat == "" {
|
||||
_ = handler.store.MarkArtifactFetchFailed(ctx, request.ConnectionID, request.NodeID, "artifact fetch returned an inconsistent result", now)
|
||||
return errors.New("artifact_fetch_invalid_output")
|
||||
}
|
||||
if err := handler.store.MarkArtifactFetched(ctx, request.VersionID, strings.ToLower(result.SHA256),
|
||||
result.SizeBytes, result.SpoolRef, result.ValidatedFormat, result.MatchedLocal, now); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func safeMessage(value string) string {
|
||||
value = strings.ReplaceAll(strings.TrimSpace(value), "\n", " ")
|
||||
lower := strings.ToLower(value)
|
||||
for _, forbidden := range []string{"token", "secret", "password", "authorization", "bearer", "cookie", "/run/source-secrets", "http://", "https://"} {
|
||||
if strings.Contains(lower, forbidden) {
|
||||
return "source reported an error (details suppressed)"
|
||||
}
|
||||
}
|
||||
if len([]rune(value)) > 200 {
|
||||
value = string([]rune(value)[:200])
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
// CommandRunner executes the fetch script once and parses its single JSON line.
|
||||
type CommandRunner struct {
|
||||
PythonPath string
|
||||
ScriptPath string
|
||||
SpoolRoot string
|
||||
SpoolPrefix string
|
||||
MaxBytes int64
|
||||
}
|
||||
|
||||
func NewCommandRunner(pythonPath, scriptPath, spoolRoot, spoolPrefix string, maxBytes int64) (*CommandRunner, error) {
|
||||
if !filepath.IsAbs(pythonPath) || !filepath.IsAbs(scriptPath) || !filepath.IsAbs(spoolRoot) || maxBytes <= 0 {
|
||||
return nil, errors.New("artifact fetch command paths must be absolute")
|
||||
}
|
||||
if strings.Contains(spoolPrefix, "..") || strings.Contains(spoolPrefix, "\\") {
|
||||
return nil, errors.New("artifact fetch spool prefix is invalid")
|
||||
}
|
||||
spoolPrefix = strings.Trim(spoolPrefix, "/")
|
||||
if spoolPrefix != "" {
|
||||
spoolPrefix += "/"
|
||||
}
|
||||
return &CommandRunner{PythonPath: pythonPath, ScriptPath: scriptPath, SpoolRoot: spoolRoot, SpoolPrefix: spoolPrefix, MaxBytes: maxBytes}, nil
|
||||
}
|
||||
|
||||
func (runner *CommandRunner) Run(ctx context.Context, manifest Manifest) (Result, error) {
|
||||
manifestJSON, err := json.Marshal(map[string]any{
|
||||
"spec_id": manifest.VersionID, "node_id": manifest.NodeID, "connection_id": manifest.ConnectionID,
|
||||
"source_type": manifest.SourceType, "source_name": manifest.SourceName,
|
||||
"relative_path": manifest.RelativePath, "name": manifest.Name,
|
||||
"size_bytes": manifest.SizeBytes, "locator_hash": manifest.LocatorHash,
|
||||
})
|
||||
if err != nil {
|
||||
return Result{}, errors.New("artifact_fetch_manifest_failed")
|
||||
}
|
||||
command := exec.CommandContext(ctx, runner.PythonPath, runner.ScriptPath, string(manifestJSON))
|
||||
// The literal guard keeps secrets out of the child environment: the script
|
||||
// only ever reads pre-mounted files.
|
||||
command.Env = []string{
|
||||
"PATH=/usr/bin:/bin",
|
||||
"PYTHONUNBUFFERED=1",
|
||||
"INGESTION_SPOOL_ROOT=" + runner.SpoolRoot,
|
||||
"SPOOL_REF_PREFIX=" + runner.SpoolPrefix,
|
||||
"ARTIFACT_FETCH_MAX_BYTES=" + strconv.FormatInt(runner.MaxBytes, 10),
|
||||
"TZ=UTC",
|
||||
}
|
||||
stdout, err := command.StdoutPipe()
|
||||
if err != nil {
|
||||
return Result{}, errors.New("artifact_fetch_start_failed")
|
||||
}
|
||||
command.Stderr = &limitedWriter{buffer: bytes.NewBuffer(nil), limit: 4096}
|
||||
if err := command.Start(); err != nil {
|
||||
return Result{}, errors.New("artifact_fetch_start_failed")
|
||||
}
|
||||
var result Result
|
||||
decoded := false
|
||||
scanner := bufio.NewScanner(stdout)
|
||||
scanner.Buffer(make([]byte, 4096), 1<<20)
|
||||
for scanner.Scan() {
|
||||
line := bytes.TrimSpace(scanner.Bytes())
|
||||
if len(line) == 0 || !bytes.HasPrefix(line, []byte("{")) {
|
||||
continue
|
||||
}
|
||||
decoder := json.NewDecoder(bytes.NewReader(line))
|
||||
decoder.DisallowUnknownFields()
|
||||
if err := decoder.Decode(&result); err != nil {
|
||||
_ = command.Process.Kill()
|
||||
_ = command.Wait()
|
||||
return Result{}, errors.New("artifact_fetch_invalid_output")
|
||||
}
|
||||
decoded = true
|
||||
break
|
||||
}
|
||||
waitErr := command.Wait()
|
||||
if !decoded {
|
||||
if waitErr != nil {
|
||||
return Result{OK: false, Error: "artifact fetch exited without a result"}, errors.New("artifact_fetch_failed")
|
||||
}
|
||||
return Result{}, errors.New("artifact_fetch_no_output")
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
var ErrManifestGone = errors.New("artifact fetch manifest is gone")
|
||||
|
||||
// limitedWriter captures at most limit bytes of script stderr; full output is
|
||||
// never retained (it could echo source details we do not want in failures).
|
||||
type limitedWriter struct {
|
||||
buffer *bytes.Buffer
|
||||
limit int
|
||||
}
|
||||
|
||||
func (writer *limitedWriter) Write(payload []byte) (int, error) {
|
||||
room := writer.limit - writer.buffer.Len()
|
||||
if room > 0 {
|
||||
if len(payload) > room {
|
||||
writer.buffer.Write(payload[:room])
|
||||
return len(payload), nil
|
||||
}
|
||||
writer.buffer.Write(payload)
|
||||
}
|
||||
return len(payload), nil
|
||||
}
|
||||
@@ -0,0 +1,515 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Fetch one policy-selected catalog artifact from its live source.
|
||||
|
||||
The Go worker passes a small JSON manifest (argv[1]) describing the intent.
|
||||
This process never receives database credentials or secret values; it locates
|
||||
credentials only via the pre-mounted /run/source-secrets paths, exactly like
|
||||
live_catalog_refresh.py. Behavior per source type:
|
||||
|
||||
alist -> direct read-only API: raw_url then download.
|
||||
pp -> manifest.json local_path first (already on disk), else public URL.
|
||||
dingtalk -> prefer the local export tree (converted docx/xlsx/pdf); if it is
|
||||
older than the source-side modification time, export fresh bytes
|
||||
through the web internal API captured headers.
|
||||
|
||||
Before any network call the fetcher compares against local candidates by
|
||||
size and, when a hash is cheap to obtain, by content: a match means "本地已有
|
||||
- no pull". Verified bytes are moved into the ingestion spool and a single
|
||||
JSON result line is printed on stdout. Credentials, tokens, signed URLs and
|
||||
authorization material are never printed.
|
||||
"""
|
||||
import csv
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
|
||||
|
||||
SUPPORTED = {
|
||||
"pdf": "pdf", "docx": "docx", "xlsx": "xlsx", "md": "md", "txt": "txt",
|
||||
"csv": "csv", "json": "json",
|
||||
"png": "png", "jpg": "jpg", "jpeg": "jpeg", "gif": "gif", "bmp": "bmp", "webp": "webp",
|
||||
}
|
||||
# Office types we refuse to ingest directly: PPT/PPTX must arrive as a PDF twin
|
||||
# (embedded images travel inside those files already).
|
||||
SLIDES = {"ppt", "pptx"}
|
||||
LEGACY = {"doc": "doc", "xls": "xls"}
|
||||
MAX_BYTES_DEFAULT = 200 * 1024 * 1024
|
||||
|
||||
|
||||
class FetchError(Exception):
|
||||
def __init__(self, message, retryable=True, code="fetch_failed"):
|
||||
super().__init__(message)
|
||||
self.retryable = retryable
|
||||
self.code = code
|
||||
|
||||
|
||||
class MissingSource(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def emit(result):
|
||||
print(json.dumps(result, ensure_ascii=False))
|
||||
|
||||
|
||||
def sha256_file(path):
|
||||
digest = hashlib.sha256()
|
||||
with open(path, "rb") as handle:
|
||||
for block in iter(lambda: handle.read(1 << 20), b""):
|
||||
digest.update(block)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def digest_of(value):
|
||||
return hashlib.sha256(value.encode()).hexdigest()
|
||||
|
||||
|
||||
def validate_magic(path, suffix):
|
||||
with open(path, "rb") as handle:
|
||||
head = handle.read(12)
|
||||
checks = {
|
||||
"pdf": head.startswith(b"%PDF-"),
|
||||
"docx": head.startswith(b"PK\x03\x04"),
|
||||
"xlsx": head.startswith(b"PK\x03\x04"),
|
||||
"doc": head.startswith(b"\xd0\xcf\x11\xe0"),
|
||||
"xls": head.startswith(b"\xd0\xcf\x11\xe0"),
|
||||
"png": head.startswith(b"\x89PNG\r\n\x1a\n"),
|
||||
"jpg": head[:2] == b"\xff\xd8" and head[2:3] == b"\xff",
|
||||
"jpeg": head[:2] == b"\xff\xd8" and head[2:3] == b"\xff",
|
||||
"gif": head[:6] in (b"GIF87a", b"GIF89a"),
|
||||
"bmp": head[:2] == b"BM",
|
||||
"webp": head[:4] == b"RIFF" and head[8:12] == b"WEBP",
|
||||
}
|
||||
if suffix in checks and not checks[suffix]:
|
||||
raise FetchError("downloaded bytes are not a valid %s file" % suffix,
|
||||
retryable=False, code="invalid_format")
|
||||
if suffix in ("md", "txt", "csv", "json"):
|
||||
try:
|
||||
with open(path, "rb") as handle:
|
||||
handle.read(4096).decode("utf-8", errors="strict")
|
||||
except UnicodeDecodeError:
|
||||
raise FetchError("text artifact is not valid UTF-8", retryable=False, code="invalid_format")
|
||||
|
||||
|
||||
def pick_suffix(name, declared_type):
|
||||
suffix = pathlib.PurePosixPath(name).suffix.lower().lstrip(".")
|
||||
if declared_type in ("adoc",) and suffix in ("", "adoc"):
|
||||
return "docx"
|
||||
if declared_type in ("able", "axls") and suffix in ("", "able", "axls"):
|
||||
return "xlsx"
|
||||
if suffix in SUPPORTED:
|
||||
return SUPPORTED[suffix]
|
||||
if suffix == "adoc":
|
||||
return "docx"
|
||||
if suffix in ("able", "axls"):
|
||||
return "xlsx"
|
||||
return None
|
||||
|
||||
|
||||
def http_get(url, headers, timeout, max_bytes, cookie_opener=None):
|
||||
request = urllib.request.Request(url, headers=headers or {})
|
||||
|
||||
class NoRedirect(urllib.request.HTTPRedirectHandler):
|
||||
location = ""
|
||||
|
||||
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
||||
NoRedirect.location = newurl
|
||||
return None
|
||||
|
||||
handler = NoRedirect()
|
||||
opener = cookie_opener or urllib.request.build_opener(handler)
|
||||
try:
|
||||
response = opener.open(request, timeout=timeout)
|
||||
except urllib.error.HTTPError as error:
|
||||
if error.code in (301, 302, 303, 307, 308):
|
||||
target = handler.location or error.headers.get("Location") or ""
|
||||
host = urllib.parse.urlparse(target).netloc.lower()
|
||||
if any(marker in host for marker in ("login.", "sso", "oauth", "passport")):
|
||||
raise FetchError("文件所在的来源存储需要网页登录授权(存储后端未配置服务凭据),请联系管理员", retryable=False, code="source_auth_required")
|
||||
raise FetchError("redirect refused", retryable=True, code="redirect")
|
||||
if error.code == 404:
|
||||
raise MissingSource("source returned 404")
|
||||
raise FetchError("source returned HTTP %d" % error.code, retryable=error.code >= 500, code="http_%d" % error.code)
|
||||
with response:
|
||||
declared = response.headers.get("Content-Length")
|
||||
if declared and int(declared) > max_bytes:
|
||||
raise FetchError("source file exceeds ingest size limit", retryable=False, code="over_limit")
|
||||
destination = tempfile.NamedTemporaryFile(delete=False, suffix=".fetch")
|
||||
total = 0
|
||||
digest = hashlib.sha256()
|
||||
try:
|
||||
while True:
|
||||
block = response.read(1 << 20)
|
||||
if not block:
|
||||
break
|
||||
total += len(block)
|
||||
if total > max_bytes:
|
||||
raise FetchError("source stream exceeds ingest size limit", retryable=False, code="over_limit")
|
||||
digest.update(block)
|
||||
destination.write(block)
|
||||
finally:
|
||||
destination.close()
|
||||
if total == 0:
|
||||
os.unlink(destination.name)
|
||||
raise FetchError("source returned empty body", retryable=True, code="empty")
|
||||
return destination.name, total, digest.hexdigest()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- AList ------
|
||||
|
||||
def alist_download(manifest, max_bytes):
|
||||
sys.path.insert(0, "/sources/alist")
|
||||
from alist_readonly_inventory import AListReadOnlyClient, read_connections # noqa
|
||||
|
||||
token = pathlib.Path("/run/source-secrets/alist_api_token").read_text(encoding="utf-8").strip()
|
||||
if not token:
|
||||
raise FetchError("AList token empty", retryable=False, code="credentials")
|
||||
rows = [row for row in read_connections(pathlib.Path("/run/source-secrets/alist_connections.xlsx"), require_token=False)
|
||||
if row["category"] == manifest["source_name"]]
|
||||
if len(rows) != 1:
|
||||
raise FetchError("AList connection configuration missing", retryable=False, code="credentials")
|
||||
base_url = rows[0]["base_url"].rstrip("/")
|
||||
client = AListReadOnlyClient(base_url, token)
|
||||
path = "/" + manifest["relative_path"].lstrip("/")
|
||||
payload = json.dumps({"path": path, "password": ""}).encode("utf-8")
|
||||
request = urllib.request.Request(base_url + "/api/fs/get", data=payload, method="POST", headers={
|
||||
"x-api-token": token, "Content-Type": "application/json", "Accept": "application/json",
|
||||
"User-Agent": "governance-artifact-fetch/1.0",
|
||||
})
|
||||
try:
|
||||
with client.opener.open(request, timeout=30) as response:
|
||||
body = json.loads(response.read().decode("utf-8"))
|
||||
except Exception as error:
|
||||
raise FetchError("AList file lookup failed (%s)" % type(error).__name__, code="source_unavailable")
|
||||
if int(body.get("code") or 0) == 404:
|
||||
raise MissingSource("AList file not present at expected path")
|
||||
if int(body.get("code") or 0) != 200:
|
||||
raise FetchError("AList file lookup refused (code %s)" % body.get("code"), code="source_unavailable")
|
||||
data = body.get("data") or {}
|
||||
raw_url = str(data.get("raw_url") or "")
|
||||
if not raw_url:
|
||||
raise MissingSource("AList returned no download locator for this file")
|
||||
if manifest.get("size_bytes") in (None, 0) and data.get("size"):
|
||||
manifest["size_bytes"] = int(data["size"])
|
||||
download_url = urllib.parse.urljoin(base_url + "/", raw_url.lstrip("/"))
|
||||
try:
|
||||
return http_get(download_url, {"Authorization": token}, 180, max_bytes)
|
||||
except FetchError as error:
|
||||
if error.code == "redirect":
|
||||
# Same-origin /d/ route 302s to a signed external URL: follow once
|
||||
# WITHOUT the source token (never forward credentials cross-origin).
|
||||
class LocationCapture(urllib.request.HTTPRedirectHandler):
|
||||
captured = ""
|
||||
|
||||
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
||||
LocationCapture.captured = newurl
|
||||
return None
|
||||
|
||||
probe = urllib.request.build_opener(LocationCapture)
|
||||
try:
|
||||
probe.open(urllib.request.Request(download_url, headers={"Authorization": token}), timeout=30)
|
||||
except urllib.error.HTTPError:
|
||||
pass
|
||||
except Exception as failure:
|
||||
raise FetchError("AList redirect probe failed (%s)" % type(failure).__name__, code="source_unavailable")
|
||||
if not LocationCapture.captured:
|
||||
raise
|
||||
return http_get(LocationCapture.captured, {}, 180, max_bytes)
|
||||
raise
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ PP -------
|
||||
|
||||
def pp_find_entry(manifest):
|
||||
locator = manifest.get("locator_hash", "")
|
||||
if not locator:
|
||||
return None
|
||||
manifest_path = pathlib.Path("/sources/pp/data/public-documents/manifest.json")
|
||||
ledger = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
for result in ledger.get("results", []):
|
||||
if digest_of(str(result.get("url") or "")) == locator:
|
||||
return result
|
||||
return None
|
||||
|
||||
|
||||
def pp_download(manifest, max_bytes):
|
||||
entry = pp_find_entry(manifest)
|
||||
if entry is None:
|
||||
raise MissingSource("PP ledger entry for this locator no longer exists")
|
||||
# Ledger titles may omit the extension; recover the true file name from the
|
||||
# local path or source URL before type checks run.
|
||||
manifest_name = manifest.setdefault("name", "")
|
||||
if not pathlib.PurePosixPath(manifest_name).suffix:
|
||||
from urllib.parse import unquote, urlparse
|
||||
for probe in (entry.get("local_path", ""), unquote(urlparse(str(entry.get("url", ""))).path)):
|
||||
suffix = pathlib.PurePosixPath(probe).suffix.lower().lstrip(".")
|
||||
if suffix:
|
||||
manifest["name"] = manifest_name + "." + suffix
|
||||
break
|
||||
local_path = entry.get("local_path", "")
|
||||
if local_path:
|
||||
candidate = pathlib.Path(local_path.replace("/home/zheromiku/dobot-portal-sync", "/sources/pp", 1))
|
||||
if not candidate.is_file():
|
||||
candidate = pathlib.Path(local_path)
|
||||
if candidate.is_file():
|
||||
size = candidate.stat().st_size
|
||||
expected_hash = str(entry.get("sha256", "")).lower()
|
||||
actual = sha256_file(candidate)
|
||||
if expected_hash and len(expected_hash) == 64 and actual != expected_hash:
|
||||
raise FetchError("PP local copy hash mismatch (ledger changed?)", retryable=True, code="local_mismatch")
|
||||
return str(candidate), size, actual, True
|
||||
url = entry.get("url", "")
|
||||
if not url:
|
||||
raise MissingSource("PP ledger entry has no local file and no download URL")
|
||||
path, size, actual = http_get(url, {}, 180, max_bytes)
|
||||
expected_hash = str(entry.get("sha256", "")).lower()
|
||||
if expected_hash and len(expected_hash) == 64 and actual != expected_hash:
|
||||
os.unlink(path)
|
||||
raise FetchError("PP download hash mismatch", retryable=True, code="hash_mismatch")
|
||||
return path, size, actual, False
|
||||
|
||||
|
||||
# ------------------------------------------------------------ DingTalk -------
|
||||
|
||||
def load_capture_headers():
|
||||
capture = json.loads(pathlib.Path("/run/source-secrets/dingtalk_capture.json").read_text(encoding="utf-8"))
|
||||
headers = capture.get("headers") or {}
|
||||
def header(name):
|
||||
for key, value in headers.items():
|
||||
if key.lower() == name:
|
||||
return value
|
||||
return ""
|
||||
a_token = header("a-token")
|
||||
xsrf = header("x-xsrf-token")
|
||||
cookie = header("cookie")
|
||||
uid = ""
|
||||
match = re.search(r'"?uid"?[=:]"?(\d{4,})', cookie or "")
|
||||
if match:
|
||||
uid = match.group(1)
|
||||
if not a_token:
|
||||
raise FetchError("DingTalk capture has no a-token; re-capture in 来源与凭据", retryable=False, code="credentials")
|
||||
return {"a-token": a_token, "x-xsrf-token": xsrf, "cookie": cookie, "uid": uid,
|
||||
"referer": "https://alidocs.dingtalk.com/"}
|
||||
|
||||
|
||||
def dingtalk_local_export(manifest):
|
||||
"""Locate an already-exported copy; returns (path, fresher_than_source) or None."""
|
||||
name = pathlib.PurePosixPath(manifest["relative_path"]).name
|
||||
stem = pathlib.PurePosixPath(name).stem
|
||||
raw_suffix = pathlib.PurePosixPath(name).suffix.lower().lstrip(".")
|
||||
suffix = pick_suffix(name, manifest.get("declared_type", ""))
|
||||
root = pathlib.Path("/sources/dingtalk/exported")
|
||||
folder = str(pathlib.PurePosixPath(manifest["relative_path"]).parent).strip(".")
|
||||
search_suffixes = [suffix] if raw_suffix not in SLIDES else ["pdf", "pptx", "ppt"]
|
||||
candidates = []
|
||||
for probe in search_suffixes:
|
||||
if probe is None:
|
||||
continue
|
||||
exact = root / manifest["source_name"] / folder / (stem + "." + probe)
|
||||
if exact.is_file():
|
||||
candidates.append(exact)
|
||||
else:
|
||||
for base in (root / manifest["source_name"], pathlib.Path("/sources/dingtalk/converted_ppt_pdf"),
|
||||
pathlib.Path("/sources/dingtalk/converted_ppt_pdf_full")):
|
||||
if not base.is_dir():
|
||||
continue
|
||||
for hit in base.rglob(stem + "." + probe):
|
||||
candidates.append(hit)
|
||||
if candidates:
|
||||
break
|
||||
if not candidates:
|
||||
if raw_suffix not in SLIDES and suffix is None:
|
||||
raise MissingSource("dingtalk file type is not ingestable")
|
||||
return None
|
||||
modified = str(manifest.get("modified_at") or "")
|
||||
candidate = candidates[0]
|
||||
local_modified = time.strftime("%Y-%m-%dT%H:%M:%S+08:00", time.localtime(candidate.stat().st_mtime))
|
||||
fresh = (not modified) or (local_modified >= modified[:19] + local_modified[19:])
|
||||
return str(candidate), fresh
|
||||
|
||||
|
||||
def dingtalk_raw_keys(manifest):
|
||||
"""Recover the dentry uuid behind a node id via the scanner inventory."""
|
||||
state_path = pathlib.Path("/sources/dingtalk/data/dingtalk-internal/inventory-state.json")
|
||||
state = json.loads(state_path.read_text(encoding="utf-8"))
|
||||
roots = state.get("roots") or {}
|
||||
space_key = _space_key(roots, manifest["source_name"])
|
||||
match = [value for key, value in sorted(roots.items()) if key == space_key]
|
||||
if len(match) != 1:
|
||||
raise FetchError("DingTalk space not present in the scanner inventory; refresh 来源目录 first",
|
||||
retryable=False, code="space_missing")
|
||||
sys.path.insert(0, "/app/scripts")
|
||||
from source_catalog_import import node_id as catalog_node_id # noqa
|
||||
source_id = "dingtalk-" + space_key
|
||||
nodes = match[0].get("nodes") or {}
|
||||
for raw, item in nodes.items():
|
||||
if catalog_node_id(source_id, raw) == manifest["node_id"]:
|
||||
return raw, str(item.get("dentry_type") or item.get("type") or "")
|
||||
raise MissingSource("DingTalk node not found in current inventory (moved or deleted?)")
|
||||
|
||||
|
||||
def _space_key(roots, name):
|
||||
for key, value in roots.items():
|
||||
if value.get("live_space_name") == name:
|
||||
return key
|
||||
return ""
|
||||
|
||||
|
||||
def dingtalk_download(manifest, max_bytes):
|
||||
export = dingtalk_local_export(manifest)
|
||||
modified = str(manifest.get("modified_at") or "")
|
||||
if export:
|
||||
path, fresh = export
|
||||
size = os.path.getsize(path)
|
||||
if size > 0 and (fresh or not modified):
|
||||
return path, size, sha256_file(path), True
|
||||
# stale local export -> fall through to a live export when available
|
||||
raw_key, declared_type = dingtalk_raw_keys(manifest)
|
||||
headers = load_capture_headers()
|
||||
live_path = live_export_dingtalk_document(raw_key, headers, manifest, max_bytes)
|
||||
if export and not live_path:
|
||||
path, fresh = export
|
||||
return path, os.path.getsize(path), sha256_file(path), True # keep old bytes; staleness shown in UI
|
||||
if not live_path:
|
||||
raise MissingSource("DingTalk file is not exported and cannot be exported automatically")
|
||||
return live_path
|
||||
|
||||
|
||||
def live_export_dingtalk_document(raw_key, headers, manifest, max_bytes):
|
||||
opener = urllib.request.build_opener()
|
||||
|
||||
def api(url, payload=None):
|
||||
request = urllib.request.Request(url, headers={
|
||||
"a-token": headers["a-token"], "x-xsrf-token": headers.get("x-xsrf-token", ""),
|
||||
"cookie": headers.get("cookie", ""), "content-type": "application/json",
|
||||
"referer": headers["referer"], "user-agent": "Mozilla/5.0",
|
||||
}, data=json.dumps(payload).encode() if payload is not None else None)
|
||||
try:
|
||||
with opener.open(request, timeout=60) as response:
|
||||
return json.loads(response.read().decode("utf-8"))
|
||||
except (urllib.error.URLError, urllib.error.HTTPError, TimeoutError) as error:
|
||||
raise FetchError("DingTalk web API unavailable (%s)" % type(error).__name__, code="source_unavailable")
|
||||
|
||||
info = api("https://alidocs.dingtalk.com/box/api/v2/dentry/info?dentryUuid=" + urllib.parse.quote(raw_key))
|
||||
data = info.get("data") or {}
|
||||
space_id, dentry_id = str(data.get("spaceId") or ""), str(data.get("dentryId") or "")
|
||||
if not space_id or not dentry_id:
|
||||
raise MissingSource("DingTalk dentry not resolvable")
|
||||
download = api("https://alidocs.dingtalk.com/box/api/v1/file/download?spaceId=%s&dentryId=%s" % (space_id, dentry_id))
|
||||
payload = download.get("data") or {}
|
||||
if payload.get("downloadType") == "URL_PRE_SIGNATURE":
|
||||
urls = (payload.get("ossUrlPreSignatureInfo") or {}).get("preSignUrls") or []
|
||||
if urls:
|
||||
return http_get(urls[0], {}, 180, max_bytes) + (False,)
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- main -------
|
||||
|
||||
def suffix_from_magic(path):
|
||||
with open(path, "rb") as handle:
|
||||
head = handle.read(12)
|
||||
if head.startswith(b"%PDF-"):
|
||||
return "pdf"
|
||||
if head.startswith(b"PK\x03\x04"):
|
||||
# zip container: peek one entry name to pick office family
|
||||
try:
|
||||
import zipfile
|
||||
with zipfile.ZipFile(path) as archive:
|
||||
names = set(archive.namelist()[:50])
|
||||
if "word/document.xml" in names:
|
||||
return "docx"
|
||||
if "xl/workbook.xml" in names:
|
||||
return "xlsx"
|
||||
except Exception:
|
||||
return None
|
||||
return None
|
||||
if head.startswith(b"\xd0\xcf\x11\xe0"):
|
||||
return None
|
||||
for suffix in ("png", "jpg", "gif", "bmp", "webp"):
|
||||
checks = validate_magic_map(head)
|
||||
if checks.get(suffix):
|
||||
return suffix
|
||||
try:
|
||||
with open(path, "rb") as handle:
|
||||
handle.read(4096).decode("utf-8", errors="strict")
|
||||
return "md"
|
||||
except (UnicodeDecodeError, OSError):
|
||||
return None
|
||||
|
||||
|
||||
def validate_magic_map(head):
|
||||
return {
|
||||
"png": head.startswith(b"\x89PNG\r\n\x1a\n"),
|
||||
"jpg": head[:2] == b"\xff\xd8" and head[2:3] == b"\xff",
|
||||
"gif": head[:6] in (b"GIF87a", b"GIF89a"),
|
||||
"bmp": head[:2] == b"BM",
|
||||
"webp": head[:4] == b"RIFF" and head[8:12] == b"WEBP",
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
manifest = json.loads(sys.argv[1])
|
||||
max_bytes = int(os.environ.get("ARTIFACT_FETCH_MAX_BYTES") or MAX_BYTES_DEFAULT)
|
||||
spool_root = pathlib.Path(os.environ.get("INGESTION_SPOOL_ROOT") or "/var/lib/governance/ingestion")
|
||||
name = manifest["name"]
|
||||
declared_type = manifest.get("declared_type", "")
|
||||
matched_local = False
|
||||
if manifest["source_type"] == "alist":
|
||||
path, size, digest = alist_download(manifest, max_bytes)
|
||||
elif manifest["source_type"] == "pp":
|
||||
path, size, digest, matched_local = pp_download(manifest, max_bytes)
|
||||
elif manifest["source_type"] == "dingtalk":
|
||||
path, size, digest, matched_local = dingtalk_download(manifest, max_bytes)
|
||||
else:
|
||||
raise FetchError("unknown source type", retryable=False, code="bad_manifest")
|
||||
# Type is decided on the RESOLVED bytes, not the catalog title: PP ledger
|
||||
# titles may omit extensions and DingTalk exports always change them.
|
||||
suffix = pick_suffix(pathlib.PurePosixPath(path).name, declared_type) or pick_suffix(name, declared_type)
|
||||
if suffix is None:
|
||||
suffix = suffix_from_magic(path)
|
||||
if suffix is None:
|
||||
raw_suffix = pathlib.PurePosixPath(name).suffix.lower().lstrip(".")
|
||||
hint = " (convert PPT to PDF first)" if raw_suffix in SLIDES else ""
|
||||
raise FetchError("file type .%s is not supported by the parser%s" % (raw_suffix or "?", hint),
|
||||
retryable=False, code="unsupported_format")
|
||||
try:
|
||||
validate_magic(path, suffix)
|
||||
if size > max_bytes:
|
||||
raise FetchError("artifact exceeds ingest size limit", retryable=False, code="over_limit")
|
||||
relative_ref = "artifact/%s/%s/%s.%s" % (digest[:2], digest[2:4], manifest["node_id"], suffix)
|
||||
destination = spool_root / relative_ref
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
staged = str(destination) + ".part"
|
||||
shutil.copyfile(path, staged)
|
||||
if sha256_file(staged) != digest:
|
||||
raise FetchError("spooled bytes failed verification", code="spool_mismatch")
|
||||
os.replace(staged, destination)
|
||||
result = {"ok": True, "spec_id": manifest["spec_id"], "node_id": manifest["node_id"],
|
||||
"connection_id": manifest["connection_id"], "sha256": digest, "size_bytes": size,
|
||||
"spool_ref": os.environ.get("SPOOL_REF_PREFIX", "") + relative_ref, "validated_format": suffix, "matched_local": matched_local}
|
||||
finally:
|
||||
if path.startswith(tempfile.gettempdir()) and os.path.exists(path):
|
||||
os.unlink(path)
|
||||
emit(result)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
sys.exit(main())
|
||||
except MissingSource as error:
|
||||
emit({"ok": False, "missing": True, "error": str(error)})
|
||||
sys.exit(3)
|
||||
except FetchError as error:
|
||||
emit({"ok": False, "retryable": error.retryable, "code": error.code, "error": str(error)})
|
||||
sys.exit(4 if error.retryable else 2)
|
||||
@@ -607,6 +607,7 @@ export function JobDashboard({ jobs, loading, error, onRefresh }: JobDashboardPr
|
||||
}
|
||||
|
||||
const processingLabels: Record<ProcessingState, string> = {
|
||||
planned: '待入库',
|
||||
pending: '待处理',
|
||||
processing: '解析中',
|
||||
finalizing: '定稿中',
|
||||
|
||||
@@ -8,7 +8,7 @@ import type { SourceConnectionListItem } from './sourceConnections'
|
||||
const monitor:MonitorOverview={mode:'simulation',simulated:true,observed_at:'2026-09-08T12:00:00Z',host:{cpu_percent:42,memory_used_bytes:1,memory_total_bytes:2,disk_used_bytes:1,disk_total_bytes:2,gpu_state:'unknown',gpu_detail:'未接入真实GPU监控',simulated:true},services:[{key:'mineru',name:'MinerU',state:'healthy',detail:'模拟',latency_ms:1,observed_at:'2026-09-08T12:00:00Z',simulated:true}],knowledge_bases:[],health:[{key:'mineru',name:'MinerU',liveness:'up',readiness:'blocked',reason:'服务可达但容量已满(429),暂时不可接单',latency_ms:30,observed_at:'2026-09-08T12:00:00Z',last_success_at:'2026-09-08T11:59:00Z',consecutive_failures:2,configured:true,applicable:true}]}
|
||||
const sources:CatalogSource[]=[{connection_id:'ding-1',source_type:'dingtalk',name:'公司级技术支持知识库',root_node_id:'root-1',folder_count:1079,file_count:3444,observed_at:'2026-09-09T09:30:32Z',snapshot_only:true,snapshot_status:'ready',last_refresh_duration_ms:0}]
|
||||
const connections:SourceConnectionListItem[]=[{id:'conn-ding',source_type:'dingtalk',name:'公司级技术支持知识库',enabled:true,schedule_configured:true,health_status:'healthy',item_count:3444,updated_at:'2026-09-09T09:30:32Z',next_refresh_at:'2026-09-12T10:00:00Z'},{id:'conn-man',source_type:'manual',name:'本地手工验收',enabled:false,schedule_configured:false,health_status:'unknown',item_count:5,updated_at:'2026-09-09T09:30:32Z',next_refresh_at:null}]
|
||||
const settings:UISettings={id:'settings-default',state:'active',document_refresh_seconds:15,resource_refresh_seconds:30,stalled_after_seconds:1800,finalizing_after_seconds:7200,updated_by:'system:migration',created_at:'2026-09-10T08:00:00Z',activated_at:'2026-09-10T08:00:00Z',retired_at:null,error_code:''}
|
||||
const settings:UISettings={id:'settings-default',state:'active',document_refresh_seconds:15,resource_refresh_seconds:30,stalled_after_seconds:1800,finalizing_after_seconds:7200,ingestion_concurrency:2,updated_by:'system:migration',created_at:'2026-09-10T08:00:00Z',activated_at:'2026-09-10T08:00:00Z',retired_at:null,error_code:''}
|
||||
describe('SystemSettingsView',()=>{
|
||||
it('separates document-engineer settings from locked administrator and production controls',()=>{
|
||||
const html=renderToStaticMarkup(<SystemSettingsView sources={sources} connections={connections} monitor={monitor} settings={settings} productionEnabled={false} routingCapabilities={{internal_enabled:true,external_enabled:true}} loading={false} saving={false} error={null} saveMessage="刷新与告警参数已验证并启用:文档15秒、资源30秒、停滞30分钟、finalizing 2小时" onSave={vi.fn()} onSourcesChanged={vi.fn()} onRefresh={vi.fn()} />)
|
||||
|
||||
@@ -138,6 +138,7 @@ export function SystemSettingsView({ sources, connections, monitor, settings, pr
|
||||
const [resourceRefresh, setResourceRefresh] = useState<30 | 60>(settings?.resource_refresh_seconds ?? 30)
|
||||
const [stalled, setStalled] = useState<1800 | 3600>(settings?.stalled_after_seconds ?? 1800)
|
||||
const [finalizing, setFinalizing] = useState<7200 | 14400>(settings?.finalizing_after_seconds ?? 7200)
|
||||
const [concurrency, setConcurrency] = useState<number>(settings?.ingestion_concurrency ?? 2)
|
||||
const [dirty, setDirty] = useState(false)
|
||||
|
||||
useEffect(() => {
|
||||
@@ -146,6 +147,7 @@ export function SystemSettingsView({ sources, connections, monitor, settings, pr
|
||||
setResourceRefresh(settings.resource_refresh_seconds)
|
||||
setStalled(settings.stalled_after_seconds)
|
||||
setFinalizing(settings.finalizing_after_seconds)
|
||||
setConcurrency(settings.ingestion_concurrency)
|
||||
setDirty(false)
|
||||
}, [settings])
|
||||
|
||||
@@ -183,10 +185,11 @@ export function SystemSettingsView({ sources, connections, monitor, settings, pr
|
||||
<label>资源刷新间隔<select value={resourceRefresh} onChange={(event) => { setResourceRefresh(Number(event.target.value) as 30 | 60); setDirty(true) }}><option value={30}>30秒</option><option value={60}>1分钟</option></select></label>
|
||||
<label>解析停滞判定<select value={stalled} onChange={(event) => { setStalled(Number(event.target.value) as 1800 | 3600); setDirty(true) }}><option value={1800}>30分钟</option><option value={3600}>1小时</option></select></label>
|
||||
<label>finalizing 告警<select value={finalizing} onChange={(event) => { setFinalizing(Number(event.target.value) as 7200 | 14400); setDirty(true) }}><option value={7200}>2小时</option><option value={14400}>4小时</option></select></label>
|
||||
<label>入库并发上限<select value={concurrency} onChange={(event) => { setConcurrency(Number(event.target.value)); setDirty(true) }}><option value={1}>1(谨慎,MinerU 并发 1)</option><option value={2}>2</option><option value={3}>3</option><option value={4}>4</option><option value={6}>6</option><option value={8}>8</option></select></label>
|
||||
</div>
|
||||
<button onClick={() => { setDirty(false); onSave({ document_refresh_seconds: documentRefresh, resource_refresh_seconds: resourceRefresh, stalled_after_seconds: stalled, finalizing_after_seconds: finalizing }) }} disabled={!canSave}>{saving ? '正在验证并应用…' : '保存并应用刷新与告警参数'}</button>
|
||||
<button onClick={() => { setDirty(false); onSave({ document_refresh_seconds: documentRefresh, resource_refresh_seconds: resourceRefresh, stalled_after_seconds: stalled, finalizing_after_seconds: finalizing, ingestion_concurrency: concurrency }) }} disabled={!canSave}>{saving ? '正在验证并应用…' : '保存并应用刷新与告警参数'}</button>
|
||||
{visibleSaveMessage(saveMessage, dirty) ? <p className="success-message" role="status">{saveMessage}</p> : null}
|
||||
<small>当前:文档 {secondsLabel(settings.document_refresh_seconds)}、资源 {secondsLabel(settings.resource_refresh_seconds)}、停滞 {secondsLabel(settings.stalled_after_seconds)}、finalizing {secondsLabel(settings.finalizing_after_seconds)}。</small>
|
||||
<small>当前:文档 {secondsLabel(settings.document_refresh_seconds)}、资源 {secondsLabel(settings.resource_refresh_seconds)}、停滞 {secondsLabel(settings.stalled_after_seconds)}、finalizing {secondsLabel(settings.finalizing_after_seconds)}、入库并发 {settings.ingestion_concurrency}。</small>
|
||||
</section> : null}
|
||||
</div>
|
||||
</div> : null}
|
||||
@@ -214,7 +217,7 @@ export function SystemSettings() {
|
||||
|
||||
const save = async (input: UISettingsCandidateInput) => {
|
||||
setSaving(true); setSaveMessage(null); setError(null)
|
||||
try { const candidate = await createUISettingsCandidate(input); const active = await activateUISettings(candidate.id); setSettings(active); setSaveMessage(`刷新与告警参数已验证并启用:文档${secondsLabel(active.document_refresh_seconds)}、资源${secondsLabel(active.resource_refresh_seconds)}、停滞${secondsLabel(active.stalled_after_seconds)}、finalizing ${secondsLabel(active.finalizing_after_seconds)};旧版本已保留用于审计。`) }
|
||||
try { const candidate = await createUISettingsCandidate(input); const active = await activateUISettings(candidate.id); setSettings(active); setSaveMessage(`刷新与告警参数已验证并启用:文档${secondsLabel(active.document_refresh_seconds)}、资源${secondsLabel(active.resource_refresh_seconds)}、停滞${secondsLabel(active.stalled_after_seconds)}、finalizing ${secondsLabel(active.finalizing_after_seconds)}、入库并发 ${active.ingestion_concurrency};旧版本已保留用于审计。`) }
|
||||
catch (cause) { setError(cause instanceof Error ? cause.message : '保存系统设置失败') }
|
||||
finally { setSaving(false) }
|
||||
}
|
||||
|
||||
@@ -3,15 +3,13 @@ import { describe,expect,it,vi } from 'vitest'
|
||||
import { WeKnoraMonitorView } from './WeKnoraMonitor'
|
||||
import type { MonitorFilePage,MonitorOverview,RunMetrics } from './monitor'
|
||||
import type { TestPublicationPage } from './testPublications'
|
||||
import type { CatalogChangeSetDetail } from './sourceCatalog'
|
||||
|
||||
const overview:MonitorOverview={mode:'simulation',simulated:true,observed_at:'2026-09-08T12:00:00Z',host:{cpu_percent:42,memory_used_bytes:19971597926,memory_total_bytes:34359738368,disk_used_bytes:128849018880,disk_total_bytes:536870912000,gpu_state:'unknown',gpu_detail:'未接入真实GPU监控',simulated:true},services:[{key:'weknora',name:'WeKnora API',state:'healthy',detail:'模拟连接正常',latency_ms:86,observed_at:'2026-09-08T12:00:00Z',simulated:true},{key:'mineru',name:'MinerU',state:'healthy',detail:'模拟处理中2,队列5',latency_ms:24,observed_at:'2026-09-08T12:00:00Z',simulated:true},{key:'postgres',name:'PostgreSQL',state:'unknown',detail:'未接入真实监控',latency_ms:0,observed_at:'2026-09-08T12:00:00Z',simulated:true},{key:'redis',name:'Redis / MQ',state:'unknown',detail:'未接入真实监控',latency_ms:0,observed_at:'2026-09-08T12:00:00Z',simulated:true},{key:'oss',name:'OSS',state:'unknown',detail:'未接入真实监控',latency_ms:0,observed_at:'2026-09-08T12:00:00Z',simulated:true}],knowledge_bases:[{id:'kb-sim',name:'公司级技术支持知识库',visibility:'mixed',imported:5750,completed:4530,pending:450,processing:50,finalizing:714,failed:6,cancelled:0,internal_imported:4200,internal_completed:3300,external_imported:1550,external_completed:1230}],health:[]}
|
||||
const overview:MonitorOverview={mode:'simulation',simulated:true,observed_at:'2026-09-08T12:00:00Z',host:{cpu_percent:42,memory_used_bytes:19971597926,memory_total_bytes:34359738368,disk_used_bytes:128849018880,disk_total_bytes:536870912000,gpu_state:'unknown',gpu_detail:'未接入真实GPU监控',simulated:true},services:[{key:'weknora',name:'WeKnora API',state:'healthy',detail:'模拟连接正常',latency_ms:86,observed_at:'2026-09-08T12:00:00Z',simulated:true},{key:'mineru',name:'MinerU',state:'healthy',detail:'模拟处理中2,队列5',latency_ms:24,observed_at:'2026-09-08T12:00:00Z',simulated:true},{key:'postgres',name:'PostgreSQL',state:'unknown',detail:'未接入真实监控',latency_ms:0,observed_at:'2026-09-08T12:00:00Z',simulated:true},{key:'redis',name:'Redis / MQ',state:'unknown',detail:'未接入真实监控',latency_ms:0,observed_at:'2026-09-08T12:00:00Z',simulated:true},{key:'oss',name:'OSS',state:'unknown',detail:'未接入真实监控',latency_ms:0,observed_at:'2026-09-08T12:00:00Z',simulated:true}],knowledge_bases:[{id:'kb-sim',name:'公司级技术支持知识库',visibility:'mixed',imported:5750,completed:4530,pending:450,processing:50,finalizing:714,failed:6,cancelled:0,planned:120,needs_removal:3,stale_planned:0,internal_imported:4200,internal_completed:3300,external_imported:1550,external_completed:1230}],pipeline:{fetch_pending:4,fetch_failed:1,fetch_missing:0,concurrency:2},health:[]}
|
||||
const files:MonitorFilePage={items:[{name:'[模拟] 产品参数总表.xlsx',directory:'02|产品资料',visibility:'external',status:'finalizing',stage_detail:'基础检索可用,Wiki与图谱仍在生成',elapsed_seconds:2100,error_code:''}],total:1,limit:20,offset:0,sample_only:true}
|
||||
const metrics:RunMetrics={observed_at:'2026-09-12T00:00:00Z',active_workers:2,worker_source:'pg_lease',queue:{pending:3,running:4,retry:1,failed:2,oldest_pending_age_seconds:120},short_window:{window_seconds:3600,jobs_started:50,jobs_completed:48,jobs_failed:6,success_rate:88.9,failure_rate:11.1,duration_avg_seconds:55,duration_p50_seconds:30,duration_p95_seconds:1800},long_window:{window_seconds:86400,jobs_started:500,jobs_completed:482,jobs_failed:18,success_rate:96.4,failure_rate:3.6,duration_avg_seconds:70,duration_p50_seconds:45,duration_p95_seconds:2400}}
|
||||
const queue:TestPublicationPage={items:[{id:'tp-plan',title:'API.docx',ingest_state:'pending',processing_state:'pending',retrieval_state:'unpublished',target_scope: 'internal', change_set_id: 'cs-1', change_type:'added',can_start:true,retry_count:0,last_error:'',can_retry:false,updated_at:'2026-09-09T10:01:00Z'},{id:'tp-failed',title:'CR20A.pdf',ingest_state:'uploaded',processing_state:'failed',retrieval_state:'unpublished',target_scope: 'external', change_set_id: 'cs-1', change_type:'updated',can_start:false,retry_count:1,last_error:'remote_parse_failed',can_retry:true,updated_at:'2026-09-09T10:00:00Z'}],total:2,limit:50,offset:0}
|
||||
const changeSets:CatalogChangeSetDetail[]=[{id:'cs-1',connection_id:'conn-1',source_name:'国内工业',state:'planned',add_count:2,update_count:1,move_count:1,move_scope_count:1,delete_count:1,no_change_count:10,created_at:'2026-09-10T00:00:00Z',items:[{action:'delete',title:'旧手册.pdf',old_path:'旧目录/旧手册.pdf',new_path:'',reason_code:'source_missing',seq:7}],total:15,limit:20,offset:0}]
|
||||
|
||||
const baseProps={overview,files,queue,changeSets,selectedKB:overview.knowledge_bases[0],status:'' as const,queueStatus:'' as const,queueScope:'' as const,loading:false,error:null as string|null,retryingIDs:[] as string[],starting:false,startMessage:'',onSelectKB:vi.fn(),onStatus:vi.fn(),onQueueStatus:vi.fn(),onQueueScope:vi.fn(),lane:'queue',onLaneChange:vi.fn(),metrics,onRetry:vi.fn(),onBatchRetry:vi.fn(),onStart:vi.fn(),onRefresh:vi.fn()}
|
||||
const queue:TestPublicationPage={items:[{id:'tp-plan',title:'API.docx',ingest_state:'pending',processing_state:'planned',retrieval_state:'unpublished',target_scope: 'internal', change_set_id: '', change_type:'added',can_start:true,retry_count:0,last_error:'',can_retry:false,updated_at:'2026-09-09T10:01:00Z'},{id:'tp-fetch',title:'焊接手册.pdf',ingest_state:'pending',processing_state:'planned',retrieval_state:'unpublished',target_scope:'external',change_set_id:'',change_type:'added',can_start:false,retry_count:0,last_error:'',can_retry:false,updated_at:'2026-09-09T10:02:00Z'},{id:'tp-failed',title:'CR20A.pdf',ingest_state:'uploaded',processing_state:'failed',retrieval_state:'unpublished',target_scope: 'external', change_set_id: '', change_type:'updated',can_start:false,retry_count:1,last_error:'remote_parse_failed',can_retry:true,updated_at:'2026-09-09T10:00:00Z'}],total:3,limit:50,offset:0}
|
||||
const plans=queue.items.filter((item)=>item.can_start)
|
||||
const baseProps={overview,files,queue,plans,selectedKB:overview.knowledge_bases[0],status:'' as const,queueStatus:'' as const,queueScope:'' as const,loading:false,error:null as string|null,retryingIDs:[] as string[],starting:false,startMessage:'',onSelectKB:vi.fn(),onStatus:vi.fn(),onQueueStatus:vi.fn(),onQueueScope:vi.fn(),lane:'queue',onLaneChange:vi.fn(),metrics,onRetry:vi.fn(),onBatchRetry:vi.fn(),onStart:vi.fn(),onRefresh:vi.fn()}
|
||||
describe('WeKnoraMonitorView',()=>{
|
||||
it('filters the processing queue by target knowledge base and shows each task target',()=>{
|
||||
const html=renderToStaticMarkup(<WeKnoraMonitorView {...baseProps} queueScope="internal" onQueueScope={vi.fn()} />)
|
||||
@@ -20,42 +18,35 @@ describe('WeKnoraMonitorView',()=>{
|
||||
expect(html).toContain('对外知识库</option>')
|
||||
expect(html).toContain('<option value="internal" selected="">对内知识库')
|
||||
expect(html).toContain('目标知识库')
|
||||
// the one internal task row shows its target (select option matches are not counted)
|
||||
expect(html).toContain('scope-badge internal">对内')
|
||||
expect(html).toContain('scope-badge external">对外')
|
||||
})
|
||||
it('offers explicit change-set confirmation with selection and result message (actor comes from session)',()=>{
|
||||
const html=renderToStaticMarkup(<WeKnoraMonitorView {...baseProps} lane='changes' changeSets={changeSets} confirmingSetID='cs-1' confirmMessage='已提交:确认 2 项;新建上传任务 2 个' onConfirmChangeSet={vi.fn()} />)
|
||||
expect(html).toContain('预览并选择变更文件')
|
||||
expect(html).toContain('aria-label="选择旧手册.pdf"')
|
||||
expect(html).toContain('确认处理(0)')
|
||||
expect(html).not.toContain('操作人<input')
|
||||
expect(html).toContain('已提交:确认 2 项;新建上传任务 2 个')
|
||||
it('treats the catalog annotation as the only decision: 待入库 lists plans with one 开始处理 button and no reason form',()=>{
|
||||
const waiting=renderToStaticMarkup(<WeKnoraMonitorView {...baseProps} lane="waiting" onStart={vi.fn()} />)
|
||||
expect(waiting).toContain('待入库(2)')
|
||||
expect(waiting).toContain('开始处理(1)')
|
||||
expect(waiting).toContain('并发上限(当前 2)')
|
||||
expect(waiting).not.toContain('确认原因')
|
||||
expect(waiting).not.toContain('操作人<input')
|
||||
expect(waiting).toContain('正在从源头取回文件')
|
||||
})
|
||||
it('hides confirmation controls after the change set leaves planned state',()=>{
|
||||
const confirmed=[{...changeSets[0],state:'confirmed' as const}]
|
||||
const html=renderToStaticMarkup(<WeKnoraMonitorView {...baseProps} lane='changes' changeSets={confirmed} onConfirmChangeSet={vi.fn()} />)
|
||||
expect(html).toContain('已确认')
|
||||
expect(html).not.toContain('确认处理')
|
||||
expect(html).toContain('aria-label="选择旧手册.pdf" disabled')
|
||||
it('shows knowledge-base removal demand from blocked-but-live items',()=>{
|
||||
const removal=renderToStaticMarkup(<WeKnoraMonitorView {...baseProps} lane="removal" />)
|
||||
expect(removal).toContain('需下架(3)')
|
||||
expect(removal).toContain('3 份仍在检索中')
|
||||
})
|
||||
it('shows a plainly labelled simulation with per-lane inbox controls',()=>{
|
||||
const changes=renderToStaticMarkup(<WeKnoraMonitorView {...baseProps} lane='changes' />)
|
||||
for(const text of ['本地模拟数据','持久快照差异','国内工业','新增 2','更新 1','移动 1','删除 1','未变化 10','旧手册.pdf','确认原因','开始处理(1)','刷新来源 ≠ 马上导入知识库','不代表测试服务器真实数据'])expect(changes).toContain(text)
|
||||
expect(changes).not.toContain('操作人<input')
|
||||
it('shows inbox lanes for the knowledge-base diff view',()=>{
|
||||
const attention=renderToStaticMarkup(<WeKnoraMonitorView {...baseProps} lane='attention' />)
|
||||
for(const text of ['需要你出手','CR20A.pdf','tp-failed'.slice(0,0)+'remote_parse_failed','失败阶段 文档解析','批量重试所选(1)','死信(尝试耗尽,不可自动重试)','下载 ✓','文档解析 ✗','建议先在 WeKnora 手动解析该文件确认格式'])expect(attention).toContain(text)
|
||||
for(const text of ['需要你出手','CR20A.pdf','remote_parse_failed','失败阶段 文档解析','批量重试所选(1)','死信(尝试耗尽,不可自动重试)'])expect(attention).toContain(text)
|
||||
const inflight=renderToStaticMarkup(<WeKnoraMonitorView {...baseProps} lane='inflight' />)
|
||||
for(const text of ['处理中','知识库解析进度','已导入 5750','completed 4530','pending 450','finalizing 714','对内 3300 / 4200','对外 1230 / 1550','[模拟] 产品参数总表.xlsx','基础检索可用,Wiki与图谱仍在生成','下载 → Hash → OSS → 文档解析 → WeKnora 上传 → 索引 → 完成'])expect(inflight).toContain(text)
|
||||
for(const text of ['处理中','知识库解析进度','已导入 5750','待入库 120','对内 3300 / 4200','对外 1230 / 1550','源头取件 → Hash'])expect(inflight).toContain(text)
|
||||
const queueView=renderToStaticMarkup(<WeKnoraMonitorView {...baseProps} lane='queue' />)
|
||||
expect(queueView).toContain('全部任务(2)')
|
||||
expect(queueView).toContain('确认并解析')
|
||||
expect(queueView).toContain('全部任务(3)')
|
||||
expect(queueView).toContain('重试')
|
||||
expect(queueView).toContain('等待人工确认')
|
||||
expect(queueView).toContain('1小时·成功')
|
||||
expect(queueView).toContain('88.9%')
|
||||
const nav=renderToStaticMarkup(<WeKnoraMonitorView {...baseProps} lane='pending' />)
|
||||
for(const text of ['今天需要我做什么','需人工处理','等待我确认的差异','排队等待','已完成','全部任务'])expect(nav).toContain(text)
|
||||
const nav=renderToStaticMarkup(<WeKnoraMonitorView {...baseProps} lane='waiting' />)
|
||||
for(const text of ['待入库','需人工处理','需下架','处理中','已完成','全部任务'])expect(nav).toContain(text)
|
||||
})
|
||||
it('does not show simulated success totals after a load error without data',()=>{
|
||||
const html=renderToStaticMarkup(<WeKnoraMonitorView overview={null} files={null} selectedKB={null} status="" queueStatus="" queueScope="" loading={false} error="加载失败" onSelectKB={vi.fn()} onStatus={vi.fn()} onQueueStatus={vi.fn()} onQueueScope={vi.fn()} onRefresh={vi.fn()} />)
|
||||
|
||||
+62
-97
@@ -6,8 +6,6 @@ import type { KnowledgeProgress, MonitorFilePage, MonitorOverview, MonitorStatus
|
||||
import { fetchJobFailures, fetchTestPublications, retryPublication, retryPublicationsBatch, startTestPublications } from './testPublications'
|
||||
import type { JobFailure, ProcessingState, TestPublication, TestPublicationPage } from './testPublications'
|
||||
import { fetchActiveUISettings, settingsRefreshDelays } from './uiSettings'
|
||||
import { confirmCatalogChangeSet, fetchCatalogSources, fetchLatestCatalogChangeSet } from './sourceCatalog'
|
||||
import type { CatalogChangeSetDetail } from './sourceCatalog'
|
||||
import { WorkspaceContextTabsSetter } from './workspaceContext'
|
||||
import './weknoraMonitor.css'
|
||||
|
||||
@@ -15,15 +13,15 @@ const emptyQueue: TestPublicationPage = { items: [], total: 0, limit: 50, offset
|
||||
const gb = (value: number) => `${(value / 1073741824).toFixed(1)}`
|
||||
const scope = (value: string) => value === 'internal' ? '对内' : value === 'external' ? '对外' : '混合'
|
||||
const duration = (seconds: number) => seconds <= 0 ? '完成' : seconds < 60 ? `${seconds}秒` : seconds < 3600 ? `${Math.floor(seconds / 60)}分钟` : `${(seconds / 3600).toFixed(1)}小时`
|
||||
const serviceState = (state: string) => state === 'healthy' ? '正常' : state === 'degraded' ? '异常' : state === 'unknown' ? '未接入' : '不可用'
|
||||
const statusLabels: Record<MonitorStatus, string> = { completed: 'completed', pending: 'pending', processing: 'processing', finalizing: 'finalizing', failed: 'failed', cancelled: 'cancelled' }
|
||||
const processingLabels: Record<ProcessingState, string> = { pending: '等待解析', processing: '解析中', finalizing: '索引与增强', completed: '成功', failed: '失败', stalled: '停滞' }
|
||||
const processingLabels: Record<ProcessingState, string> = { planned: '待入库', pending: '等待解析', processing: '解析中', finalizing: '索引与增强', completed: '成功', failed: '失败', stalled: '停滞' }
|
||||
|
||||
const secondsSince = (iso: string) => Math.max(0, Math.round((Date.now() - new Date(iso).getTime()) / 1000))
|
||||
const adviceFor = (stage: string, code: string) => {
|
||||
if (stage === '文档解析') return '建议先在 WeKnora 手动解析该文件确认格式,再决定是否重试。'
|
||||
if (stage === '索引 / 增强') return '多为 MinerU 容量或超时问题,确认服务恢复接单后重试。'
|
||||
if (stage === 'WeKnora 上传') return '上传失败,确认 WeKnora 可达后重试;重复提交不会创建第二份知识。'
|
||||
if (stage === '源头取件') return '文件尚未从源头取回:钉钉来源需可导出的登录会话,AList/PP 需直连可用;恢复后系统会自动重试。'
|
||||
if (stage === 'Hash / 文件校验' || stage === 'OSS') return '文件或暂存校验问题,重新入队会重新拉取来源文件。'
|
||||
if (stage === '测试知识移除') return '移除失败,请先在测试知识库确认该文档当前状态。'
|
||||
if (stage === '治理策略') return '按治理策略阻止,请先在来源目录修正对内/对外标注。'
|
||||
@@ -31,25 +29,26 @@ const adviceFor = (stage: string, code: string) => {
|
||||
return '确认错误码后可直接重试;尝试次数耗尽将进入死信队列。'
|
||||
}
|
||||
|
||||
const pipelineStages = ['下载', 'Hash', 'OSS', '文档解析', 'WeKnora 上传', '索引', '完成']
|
||||
const pipelineStages = ['源头取件', 'Hash', 'OSS', '文档解析', 'WeKnora 上传', '索引', '完成']
|
||||
const stageIndex = (label: string) => label === 'Hash / 文件校验' ? 1 : label === 'OSS' ? 2 : label === '文档解析' ? 3 : label === 'WeKnora 上传' ? 4 : label === '索引 / 增强' || label === '测试知识移除' || label === '治理策略' || label === '处理链路' ? 5 : -1
|
||||
function chainSteps(item: TestPublication) {
|
||||
const broken = item.processing_state === 'failed' || item.processing_state === 'stalled'
|
||||
const stopAt = broken ? stageIndex(failureStage(item)) : -1
|
||||
const activeAt = item.processing_state === 'completed' ? 7 : item.processing_state === 'finalizing' ? 5 : item.processing_state === 'processing' ? 3 : 0
|
||||
const activeAt = item.processing_state === 'completed' ? 7 : item.processing_state === 'finalizing' ? 5 : item.processing_state === 'processing' ? 3 : item.processing_state === 'planned' && !item.can_start ? 0 : item.processing_state === 'pending' ? 1 : 0
|
||||
return pipelineStages.map((stage, index) => {
|
||||
if (stopAt >= 0 && index === stopAt) return { stage, state: 'bad' as const }
|
||||
if (stopAt >= 0) return { stage, state: index < stopAt ? 'done' as const : 'todo' as const }
|
||||
if (index < activeAt) return { stage, state: 'done' as const }
|
||||
if (index === activeAt && item.processing_state !== 'pending' && item.processing_state !== 'completed') return { stage, state: 'active' as const }
|
||||
if (item.processing_state === 'pending' && index === 0) return { stage, state: 'active' as const }
|
||||
if (index === activeAt && item.processing_state !== 'pending' && item.processing_state !== 'completed' && item.processing_state !== 'planned') return { stage, state: 'active' as const }
|
||||
if ((item.processing_state === 'pending' || (item.processing_state === 'planned' && !item.can_start)) && index === 0) return { stage, state: item.can_start || item.processing_state === 'pending' ? 'done' as const : 'active' as const }
|
||||
return { stage, state: 'todo' as const }
|
||||
})
|
||||
}
|
||||
|
||||
function failureStage(item: TestPublication) {
|
||||
const code = item.last_error
|
||||
if (code.includes('artifact') || code.includes('hash')) return 'Hash / 文件校验'
|
||||
if (code.includes('artifact') || code.includes('fetch')) return '源头取件'
|
||||
if (code.includes('hash')) return 'Hash / 文件校验'
|
||||
if (code.includes('oss')) return 'OSS'
|
||||
if (code.includes('upload')) return 'WeKnora 上传'
|
||||
if (code.includes('parse')) return '文档解析'
|
||||
@@ -64,10 +63,6 @@ interface ViewProps {
|
||||
files: MonitorFilePage | null
|
||||
queue?: TestPublicationPage
|
||||
plans?: TestPublication[]
|
||||
changeSets?: CatalogChangeSetDetail[]
|
||||
confirmingSetID?: string | null
|
||||
confirmMessage?: string
|
||||
onConfirmChangeSet?: (changeSetID: string, itemSeqs: number[], reason: string) => void
|
||||
selectedKB: KnowledgeProgress | null
|
||||
status: MonitorStatus | ''
|
||||
queueStatus?: ProcessingState | ''
|
||||
@@ -88,33 +83,20 @@ interface ViewProps {
|
||||
onLaneChange?: (lane: string) => void
|
||||
onRetry?: (id: string) => void
|
||||
onBatchRetry?: (ids: string[]) => void
|
||||
onStart?: (ids: string[], reason: string) => void
|
||||
onStart?: (ids: string[]) => void
|
||||
onRefresh: () => void
|
||||
}
|
||||
|
||||
function ChangeSetCard({changeSet,confirming,message,onConfirm}:{changeSet:CatalogChangeSetDetail;confirming:boolean;message:string;onConfirm?:(changeSetID:string,itemSeqs:number[],reason:string)=>void}){
|
||||
const [open,setOpen]=useState(false)
|
||||
const [selected,setSelected]=useState<number[]>([])
|
||||
const actionable=changeSet.items.filter(item=>item.action!=='no_change')
|
||||
const confirmable=changeSet.state==='planned'&&actionable.length>0&&Boolean(onConfirm)
|
||||
const toggle=(seq:number)=>setSelected(current=>current.includes(seq)?current.filter(value=>value!==seq):[...current,seq])
|
||||
return <article><div><strong>{changeSet.source_name}</strong><small>{new Date(changeSet.created_at).toLocaleString('zh-CN',{hour12:false})}</small>{changeSet.state!=='planned'?<span className="scope-badge internal">已确认</span>:null}</div><div className="persisted-change-counts"><span>新增 {changeSet.add_count}</span><span>更新 {changeSet.update_count}</span><span>移动 {changeSet.move_count}</span><span>范围变化 {changeSet.move_scope_count}</span><span>删除 {changeSet.delete_count}</span><span>未变化 {changeSet.no_change_count}</span></div>{actionable.length?<details open={open} onToggle={event=>setOpen((event.target as HTMLDetailsElement).open)}><summary>预览并选择变更文件</summary><div className="changeset-selection">{actionable.map(item=><label key={item.seq} className="changeset-choice"><input type="checkbox" aria-label={`选择${item.title}`} disabled={!confirmable} checked={selected.includes(item.seq)} onChange={()=>toggle(item.seq)} /><b>{item.action==='add'?'新增':item.action==='update'?'更新':item.action==='move'?'移动(改名)':item.action==='move_scope'?'范围变化':'删除'}</b><span>{item.title}</span><small>{item.new_path||item.old_path}</small>{item.action==='delete'&&item.reason_code==='source_missing'?<small>来源缺失</small>:null}</label>)}</div>{confirmable?<div className="changeset-confirm"><button disabled={confirming||selected.length===0} onClick={()=>onConfirm?.(changeSet.id,[...selected].sort((a,b)=>a-b),'人工确认快照差异执行')}>确认处理({selected.length})</button></div>:null}{message?<p role="status">{message}</p>:null}</details>:null}</article>
|
||||
}
|
||||
|
||||
export function WeKnoraMonitorView({
|
||||
overview,
|
||||
files,
|
||||
queue = emptyQueue,
|
||||
plans = queue.items,
|
||||
changeSets = [],
|
||||
confirmingSetID = null,
|
||||
confirmMessage = '',
|
||||
onConfirmChangeSet,
|
||||
selectedKB,
|
||||
status,
|
||||
queueStatus = '',
|
||||
queueScope = '',
|
||||
lane = 'attention',
|
||||
lane = 'waiting',
|
||||
onLaneChange = () => undefined,
|
||||
loading,
|
||||
error,
|
||||
@@ -133,8 +115,9 @@ export function WeKnoraMonitorView({
|
||||
onStart = () => undefined,
|
||||
onRefresh,
|
||||
}: ViewProps) {
|
||||
const [startReason, setStartReason] = useState('已核对来源快照与测试知识库差异')
|
||||
const [selectedRetryIDs, setSelectedRetryIDs] = useState<string[]>([])
|
||||
const [selectedStartIDs, setSelectedStartIDs] = useState<string[]>([])
|
||||
const [selectedRetryIDs, setSelectedRetryIDs] = useState<string[]>([])
|
||||
const toggleStartSelection = (id: string) => setSelectedStartIDs((current) => current.includes(id) ? current.filter((value) => value !== id) : [...current, id])
|
||||
const toggleRetrySelection = (id: string) => setSelectedRetryIDs((current) => current.includes(id) ? current.filter((value) => value !== id) : [...current, id])
|
||||
const stages: Array<[MonitorStatus, number]> = selectedKB ? [
|
||||
['completed', selectedKB.completed], ['pending', selectedKB.pending], ['finalizing', selectedKB.finalizing],
|
||||
@@ -146,65 +129,73 @@ export function WeKnoraMonitorView({
|
||||
}), [queue.items])
|
||||
const retryableIDs = queue.items.filter((item) => item.can_retry).map((item) => item.id)
|
||||
const batchTargets = selectedRetryIDs.length ? retryableIDs.filter((id) => selectedRetryIDs.includes(id)) : retryableIDs
|
||||
const pendingPlans = plans.filter((item) => item.can_start)
|
||||
const addedPlans = pendingPlans.filter((item) => item.change_type === 'added').length
|
||||
const updatedPlans = pendingPlans.filter((item) => item.change_type === 'updated').length
|
||||
const startablePlans = plans.filter((item) => item.can_start)
|
||||
const startTargets = selectedStartIDs.length ? startablePlans.filter((item) => selectedStartIDs.includes(item.id)).map((item) => item.id) : startablePlans.map((item) => item.id)
|
||||
const attentionItems = queue.items.filter((item) => item.processing_state === 'failed' || item.processing_state === 'stalled')
|
||||
const inflightItems = queue.items.filter((item) => item.processing_state === 'processing' || item.processing_state === 'finalizing' || (item.processing_state === 'pending' && !item.can_start))
|
||||
const doneTodayItems = queue.items.filter((item) => item.processing_state === 'completed')
|
||||
const waitingItems = queue.items.filter((item) => item.processing_state === 'pending' && !item.can_start)
|
||||
const laneCounts: Record<string, number> = { attention: attentionItems.length + failures.length, changes: changeSets.filter((set) => set.state === 'planned').length, inflight: inflightItems.length, pending: waitingItems.length, done: doneTodayItems.length, queue: queue.total }
|
||||
const queueFiltered = waitingItems
|
||||
const queueFilteredAll = orderedQueue
|
||||
const doneItems = queue.items.filter((item) => item.processing_state === 'completed')
|
||||
const plannedItems = queue.items.filter((item) => item.processing_state === 'planned')
|
||||
const fetchingPlanned = plannedItems.filter((item) => !item.can_start)
|
||||
const laneCounts: Record<string, number> = {
|
||||
waiting: startablePlans.length + fetchingPlanned.length,
|
||||
attention: attentionItems.length + failures.length,
|
||||
removal: (overview?.knowledge_bases || []).reduce((total, kb) => total + (kb.needs_removal || 0), 0),
|
||||
inflight: inflightItems.length,
|
||||
done: doneItems.length,
|
||||
queue: queue.total,
|
||||
}
|
||||
const laneTitles: Record<string, string> = { waiting: '待入库', attention: '需人工处理', removal: '需下架', inflight: '处理中', done: '已完成', queue: '全部任务' }
|
||||
|
||||
const actionTask = (item: TestPublication) => <article className="inbox-task" key={item.id}>
|
||||
<div className="inbox-task-mark">{item.can_retry ? <input type="checkbox" aria-label={`选择重试${item.title}`} checked={selectedRetryIDs.includes(item.id)} onChange={() => toggleRetrySelection(item.id)} /> : null}</div>
|
||||
<div className="inbox-task-mark">{item.can_start ? <input type="checkbox" aria-label={`选择入库${item.title}`} checked={selectedStartIDs.length === 0 || selectedStartIDs.includes(item.id)} onChange={() => toggleStartSelection(item.id)} /> : item.can_retry ? <input type="checkbox" aria-label={`选择重试${item.title}`} checked={selectedRetryIDs.includes(item.id)} onChange={() => toggleRetrySelection(item.id)} /> : null}</div>
|
||||
<div>
|
||||
<h4>{item.title} <span className={`scope-badge ${item.target_scope}`}>{scope(item.target_scope)}</span> {item.change_type === 'added' ? <span className="task-badge add">新增</span> : item.change_type === 'updated' ? <span className="task-badge update">更新</span> : null}</h4>
|
||||
<div className="why">{item.processing_state === 'failed' || item.processing_state === 'stalled' ? <>{item.processing_state === 'stalled' ? '任务停滞:' : '为什么失败:'}错误码 <code>{item.last_error || '—'}</code> · 第{item.retry_count + 1}次尝试 · 已停留{duration(secondsSince(item.updated_at))} · 失败阶段 {failureStage(item)}</> : <>状态 <code>{item.can_start ? '等待人工确认' : processingLabels[item.processing_state]}</code>{item.last_error ? <> · 最近错误 <code>{item.last_error}</code></> : null} · 已停留{duration(secondsSince(item.updated_at))}</>}</div>
|
||||
<div className="why">{item.processing_state === 'failed' || item.processing_state === 'stalled' ? <>{item.processing_state === 'stalled' ? '任务停滞:' : '为什么失败:'}错误码 <code>{item.last_error || '—'}</code> · 第{item.retry_count + 1}次尝试 · 已停留{duration(secondsSince(item.updated_at))} · 失败阶段 {failureStage(item)}</> : <>状态 <code>{processingLabels[item.processing_state]}</code>{item.last_error ? <> · 最近错误 <code>{item.last_error}</code></> : null} · 已停留{duration(secondsSince(item.updated_at))}</>}</div>
|
||||
<div className="task-chain" aria-label={`处理链${item.title}`}>{chainSteps(item).map((step) => <span key={step.stage} className={`chain-step ${step.state}`}>{step.stage}{step.state === 'done' ? ' ✓' : step.state === 'bad' ? ' ✗' : step.state === 'active' ? ` 中 ${duration(secondsSince(item.updated_at))}` : ''}</span>)}</div>
|
||||
<div className={item.processing_state === 'failed' || item.processing_state === 'stalled' ? 'errline' : 'tiny muted-line'}>{item.processing_state === 'failed' || item.processing_state === 'stalled' ? adviceFor(failureStage(item), item.last_error) : item.can_start ? '确认后将创建解析任务' : '无需操作,等待自动处理'}{item.change_set_id ? <> · 来源变更集 <code>{item.change_set_id}</code></> : null} · <code>{item.id}</code></div>
|
||||
<div className={item.processing_state === 'failed' || item.processing_state === 'stalled' ? 'errline' : 'tiny muted-line'}>{item.processing_state === 'failed' || item.processing_state === 'stalled' ? adviceFor(failureStage(item), item.last_error) : item.processing_state === 'planned' && !item.can_start ? '正在从源头取回文件,完成后自动进入可处理状态' : item.can_start ? '文件已就绪,等待开始处理' : '无需操作,等待自动处理'} · <code>{item.id}</code></div>
|
||||
</div>
|
||||
<div className="inbox-task-actions">{item.can_start ? <button className="primary" disabled={starting || !startReason.trim()} onClick={() => onStart([item.id], startReason.trim())}>确认并解析</button> : null}{item.can_retry ? <button className="primary" disabled={retryingIDs.includes(item.id)} onClick={() => onRetry(item.id)}>{retryingIDs.includes(item.id) ? '重试中…' : '重试'}</button> : null}</div>
|
||||
<div className="inbox-task-actions">{item.can_retry ? <button className="primary" disabled={retryingIDs.includes(item.id)} onClick={() => onRetry(item.id)}>{retryingIDs.includes(item.id) ? '重试中…' : '重试'}</button> : null}</div>
|
||||
</article>
|
||||
const emptyLane = (text: string) => <div className="empty-state">{text}</div>
|
||||
return <main className="page-shell weknora-monitor inbox-layout">
|
||||
<header className="hero compact"><div><h1>解析监控</h1><p className="subtitle">今天需要我做什么,一目了然;处理链进度、失败原因与建议直接摆在任务卡片上。</p></div><span className="monitor-meta">{overview ? `快照 ${new Date(overview.observed_at).toLocaleString('zh-CN', { hour12: false })}` : ''} · 队列 等待{queue.total}条</span><button onClick={onRefresh} disabled={loading}>{loading ? '刷新中…' : '刷新'}</button></header>
|
||||
<header className="hero compact"><div><h1>解析监控</h1><p className="subtitle">站在知识库视角看差异:来源与目录里标好的对内/对外文件应不应该在这里,还差哪一步,一目了然。</p></div><span className="monitor-meta">{overview ? `快照 ${new Date(overview.observed_at).toLocaleString('zh-CN', { hour12: false })}` : ''} · 队列 等待{queue.total}条{overview?.pipeline ? ` · 取件中 ${overview.pipeline.fetch_pending}${overview.pipeline.fetch_failed ? ` · 取件失败 ${overview.pipeline.fetch_failed}` : ''}` : ''}</span><button onClick={onRefresh} disabled={loading}>{loading ? '刷新中…' : '刷新'}</button></header>
|
||||
{error ? <div className="error-banner" role="alert">{error}</div> : null}
|
||||
{overview?.simulated ? <div className="simulation-banner"><strong>本地模拟数据</strong><span>资源和知识库汇总用于确认页面流程,不代表测试服务器真实数据;任务卡片来自真实治理 API。</span></div> : null}
|
||||
<div className="inbox-grid">
|
||||
<nav className="inbox-nav" aria-label="解析监控收件箱">
|
||||
<h2>今天需要我做什么</h2>
|
||||
{[['attention', '需人工处理', 'blocked'], ['changes', '等待我确认的差异', ''], ['inflight', '处理中', ''], ['pending', '排队等待', ''], ['done', '已完成', 'okall']].map(([value, label, tone]) => <button key={value} className={`${lane === value ? 'active' : ''} ${laneCounts[value] ? tone : ''}`} aria-label={`切换到${label}`} onClick={() => onLaneChange(value)}><span>{label}</span><em className="lane-count">{laneCounts[value]}</em></button>)}
|
||||
<h2>知识库差异</h2>
|
||||
{[['waiting', '待入库', 'blocked'], ['attention', '需人工处理', ''], ['removal', '需下架', ''], ['inflight', '处理中', ''], ['done', '已完成', 'okall']].map(([value, label, tone]) => <button key={value} className={`${lane === value ? 'active' : ''} ${laneCounts[value] ? tone : ''}`} aria-label={`切换到${label}`} onClick={() => onLaneChange(value)}><span>{label}</span><em className="lane-count">{laneCounts[value]}</em></button>)}
|
||||
<button className={`${lane === 'queue' ? 'active' : ''}`} aria-label="切换到全部任务" onClick={() => onLaneChange('queue')}><span>全部任务</span><em className="lane-count">{queue.total}</em></button>
|
||||
</nav>
|
||||
<div className="inbox-work">
|
||||
{lane === 'waiting' ? <section className="inbox-section" aria-label="待入库">
|
||||
<div className="section-head"><h2>待入库({startablePlans.length + fetchingPlanned.length})</h2><span className="spacer" /><button className="primary" disabled={starting || !startTargets.length} onClick={() => onStart(startTargets)}>{starting ? '正在生成任务…' : `开始处理(${startTargets.length})`}</button></div>
|
||||
<p className="tiny">来源目录中标记对内/对外的文件会自动出现在这里;“开始处理”按系统设置的并发上限(当前 {overview?.pipeline?.concurrency ?? 2})创建解析任务,无需再确认。</p>
|
||||
{startMessage ? <p className="difference-message" role="status">{startMessage}</p> : null}
|
||||
{plannedItems.length ? plannedItems.map(actionTask) : emptyLane('还没有标记入库的文件。去“来源与目录”把文件或目录标为对内/对外即可。')}
|
||||
{fetchingPlanned.length ? <p className="tiny">其中 {fetchingPlanned.length} 项仍在从源头取件,取回后自动可处理。</p> : null}
|
||||
</section> : null}
|
||||
{lane === 'attention' ? <section className="inbox-section" aria-label="需人工处理">
|
||||
<div className="section-head"><h2>需要你出手({attentionItems.length})</h2><span className="spacer" /><button className="danger" disabled={!batchTargets.length || retryingIDs.length > 0} onClick={() => onBatchRetry(batchTargets)}>{retryingIDs.length ? '重试中…' : `批量重试所选(${batchTargets.length})`}</button></div>
|
||||
{batchMessage ? <p className="queue-batch-message" role="status">{batchMessage}</p> : null}
|
||||
{attentionItems.length ? attentionItems.map(actionTask) : emptyLane('没有失败或停滞的任务。')}
|
||||
<div className="section-head sub"><h3>死信(尝试耗尽,不可自动重试)</h3><span className="tiny">原始任务与审计行保留不动;重新入队会创建新尝试</span></div>
|
||||
{failures.length ? <ul className="dlq-list">{failures.map((failure) => <li key={failure.id}><span className="monitor-status failed">{failure.error_code}</span><b>{failure.job_type === 'test.upload' ? '上传解析' : failure.job_type}</b>{failure.publication_id ? <code>{failure.publication_id}</code> : null}<small>{failure.attempt}/{failure.max_attempts_at_failure} 次 · {new Date(failure.created_at).toLocaleString('zh-CN', { hour12: false })}</small>{failure.publication_id ? <button disabled={retryingIDs.includes(failure.publication_id)} onClick={() => onBatchRetry([failure.publication_id as string])}>重新入队</button> : null}</li>)}</ul> : emptyLane('当前没有等待处理的死信记录。')}
|
||||
{failures.length ? <ul className="dlq-list">{failures.map((failure) => <li key={failure.id}><span className="monitor-status failed">{failure.error_code}</span><b>{failure.job_type === 'test.upload' ? '上传解析' : failure.job_type === 'artifact.fetch' ? '源头取件' : failure.job_type}</b>{failure.publication_id ? <code>{failure.publication_id}</code> : null}<small>{failure.attempt}/{failure.max_attempts_at_failure} 次 · {new Date(failure.created_at).toLocaleString('zh-CN', { hour12: false })}</small>{failure.publication_id ? <button disabled={retryingIDs.includes(failure.publication_id)} onClick={() => onBatchRetry([failure.publication_id as string])}>重新入队</button> : null}</li>)}</ul> : emptyLane('当前没有等待处理的死信记录。')}
|
||||
</section> : null}
|
||||
{lane === 'changes' ? <section className="inbox-section" aria-label="等待确认的差异">
|
||||
<div className="section-head"><h2>持久快照差异 · 等待确认({changeSets.filter((set) => set.state === 'planned').length} 个来源)</h2><span className="tiny">刷新来源 ≠ 马上导入知识库;勾选变更项并确认后才创建解析任务</span></div>
|
||||
{changeSets.length ? <div className="persisted-change-grid">{changeSets.map(changeSet => <ChangeSetCard key={changeSet.id} changeSet={changeSet} confirming={confirmingSetID === changeSet.id} message={confirmingSetID === changeSet.id ? confirmMessage : ''} onConfirm={onConfirmChangeSet} />)}</div> : emptyLane('当前没有持久快照差异。')}
|
||||
<div className="difference-confirmation">
|
||||
<label>确认原因<input value={startReason} onChange={(event) => setStartReason(event.target.value)} /></label>
|
||||
<button disabled={starting || !pendingPlans.length || !startReason.trim()} onClick={() => onStart(pendingPlans.map((item) => item.id), startReason.trim())}>{starting ? '正在生成任务…' : `开始处理(${pendingPlans.length})`}</button>
|
||||
</div>
|
||||
<p className="tiny">待确认 {pendingPlans.length} · 新增 {addedPlans} · 更新 {updatedPlans}</p>
|
||||
{startMessage ? <p className="difference-message" role="status">{startMessage}</p> : null}
|
||||
{lane === 'removal' ? <section className="inbox-section" aria-label="需下架">
|
||||
<div className="section-head"><h2>需下架({laneCounts.removal})</h2><span className="tiny">来源目录里被改为“禁止导入”或撤销标注、但仍在知识库中可检索的文件</span></div>
|
||||
{(overview?.knowledge_bases || []).some((kb) => kb.needs_removal > 0) ? <ul className="dlq-list">{(overview?.knowledge_bases || []).filter((kb) => kb.needs_removal > 0).map((kb) => <li key={kb.id}><span className="monitor-status failed">需下架</span><b>{kb.name}</b><small>{kb.needs_removal} 份仍在检索中</small></li>)}</ul> : emptyLane('没有标注为禁止但仍留在知识库里的文件。')}
|
||||
<p className="tiny">下架在“知识清单”里操作(停用检索,可恢复);这里只负责把差异暴露出来。</p>
|
||||
</section> : null}
|
||||
{lane === 'inflight' ? <section className="inbox-section" aria-label="处理中">
|
||||
<div className="section-head"><h2>处理中({inflightItems.length})</h2><span className="tiny">无需操作,仅供查看</span></div>
|
||||
{inflightItems.length ? inflightItems.map(actionTask) : emptyLane('当前没有正在处理或索引增强的任务。')}
|
||||
{overview ? <>
|
||||
<div className="pipeline-strip" aria-label="解析处理链">下载 → Hash → OSS → 文档解析 → WeKnora 上传 → 索引 → 完成</div>
|
||||
<div className="pipeline-strip" aria-label="解析处理链">源头取件 → Hash → OSS → 文档解析 → WeKnora 上传 → 索引 → 完成</div>
|
||||
<div className="monitor-kb-layout" id="monitor-kb-progress">
|
||||
<aside className="monitor-kb-list"><h2>知识库解析进度</h2>{overview.knowledge_bases.map((kb) => <button key={kb.id} className={selectedKB?.id === kb.id ? 'active' : ''} onClick={() => onSelectKB(kb)}><span><b>{kb.name}</b><small>{scope(kb.visibility)} · 已导入 {kb.imported}</small></span><strong>{kb.completed}完成</strong></button>)}</aside>
|
||||
<aside className="monitor-kb-list"><h2>知识库解析进度</h2>{overview.knowledge_bases.map((kb) => <button key={kb.id} className={selectedKB?.id === kb.id ? 'active' : ''} onClick={() => onSelectKB(kb)}><span><b>{kb.name}</b><small>{scope(kb.visibility)} · 已导入 {kb.imported}{kb.planned ? ` · 待入库 ${kb.planned}` : ''}</small></span><strong>{kb.completed}完成</strong></button>)}</aside>
|
||||
<div className="monitor-kb-detail">{selectedKB ? <>
|
||||
<div className="monitor-kb-heading"><div><h2>{selectedKB.name}</h2><span>{scope(selectedKB.visibility)} · 已导入 {selectedKB.imported}</span></div><div><span>对内 {selectedKB.internal_completed} / {selectedKB.internal_imported}</span><span>对外 {selectedKB.external_completed} / {selectedKB.external_imported}</span></div></div>
|
||||
<div className="monitor-kb-heading"><div><h2>{selectedKB.name}</h2><span>{scope(selectedKB.visibility)} · 已导入 {selectedKB.imported}{selectedKB.planned ? ` · 待入库 ${selectedKB.planned}` : ''}</span></div><div><span>对内 {selectedKB.internal_completed} / {selectedKB.internal_imported}</span><span>对外 {selectedKB.external_completed} / {selectedKB.external_imported}</span></div></div>
|
||||
<div className="monitor-stage-tabs"><button className={status === '' ? 'active' : ''} onClick={() => onStatus('')}>全部 {selectedKB.imported}</button>{stages.filter(([, count]) => count > 0).map(([value, count]) => <button className={status === value ? 'active' : ''} key={value} onClick={() => onStatus(value)}>{statusLabels[value]} {count}</button>)}</div>
|
||||
<p className="finalizing-note"><b>finalizing:</b>基础解析、切片和向量通常已完成,Wiki、摘要或图谱仍在生成;不能算全部完成。</p>
|
||||
{files?.sample_only ? <p className="sample-note">当前只展示本地模拟文件样例;真实模式会列出所选状态下的实际文件。</p> : null}
|
||||
@@ -213,17 +204,13 @@ export function WeKnoraMonitorView({
|
||||
</div>
|
||||
</> : null}
|
||||
</section> : null}
|
||||
{lane === 'pending' ? <section className="inbox-section" aria-label="排队等待">
|
||||
<div className="section-head"><h2>排队等待({queueFiltered.length})</h2><span className="spacer" /><label>目标知识库<select aria-label="队列目标知识库" value={queueScope} onChange={(event) => onQueueScope(event.target.value as 'internal' | 'external' | '')}><option value="">内/外全部</option><option value="internal">对内知识库</option><option value="external">对外知识库</option></select></label></div>
|
||||
{queueFiltered.length ? queueFiltered.map(actionTask) : emptyLane('当前筛选没有排队任务。')}
|
||||
</section> : null}
|
||||
{lane === 'done' ? <section className="inbox-section" aria-label="已完成">
|
||||
<div className="section-head"><h2>已完成({doneTodayItems.length})</h2><span className="tiny">最近一页数据中的成功任务</span></div>
|
||||
{doneTodayItems.length ? doneTodayItems.map(actionTask) : emptyLane('当前页没有已完成任务;用全部任务视图翻页查看。')}
|
||||
<div className="section-head"><h2>已完成({doneItems.length})</h2><span className="tiny">最近一页数据中的成功任务</span></div>
|
||||
{doneItems.length ? doneItems.map(actionTask) : emptyLane('当前页没有已完成任务;用全部任务视图翻页查看。')}
|
||||
</section> : null}
|
||||
{lane === 'queue' ? <section className="inbox-section" aria-label="全部任务">
|
||||
<div className="section-head"><h2>全部任务({queue.total})</h2><span className="spacer" /><label>目标知识库<select aria-label="队列目标知识库" value={queueScope} onChange={(event) => onQueueScope(event.target.value as 'internal' | 'external' | '')}><option value="">内/外全部</option><option value="internal">对内知识库</option><option value="external">对外知识库</option></select><select aria-label="队列状态" value={queueStatus} onChange={(event) => onQueueStatus(event.target.value as ProcessingState | '')}><option value="">全部状态</option><option value="pending">等待解析</option><option value="processing">解析中</option><option value="finalizing">索引与增强</option><option value="completed">成功</option><option value="failed">失败</option><option value="stalled">停滞</option></select></label></div>
|
||||
{queueFilteredAll.length ? queueFilteredAll.map(actionTask) : !loading ? emptyLane('当前筛选没有解析任务。') : null}
|
||||
<div className="section-head"><h2>全部任务({queue.total})</h2><span className="spacer" /><label>目标知识库<select aria-label="队列目标知识库" value={queueScope} onChange={(event) => onQueueScope(event.target.value as 'internal' | 'external' | '')}><option value="">内/外全部</option><option value="internal">对内知识库</option><option value="external">对外知识库</option></select><select aria-label="队列状态" value={queueStatus} onChange={(event) => onQueueStatus(event.target.value as ProcessingState | '')}><option value="">全部状态</option><option value="planned">待入库</option><option value="pending">等待解析</option><option value="processing">解析中</option><option value="finalizing">索引与增强</option><option value="completed">成功</option><option value="failed">失败</option><option value="stalled">停滞</option></select></label></div>
|
||||
{orderedQueue.length ? orderedQueue.map(actionTask) : !loading ? emptyLane('当前筛选没有解析任务。') : null}
|
||||
</section> : null}
|
||||
{metrics ? <footer className="monitor-strip" aria-label="真实运行指标"><div>1小时·成功 <b className={metrics.short_window.failure_rate !== null && metrics.short_window.failure_rate > 5 ? 'bad' : 'good'}>{metrics.short_window.success_rate === null ? '数据不足' : `${metrics.short_window.success_rate}%`}</b></div><div>24小时·成功 <b>{metrics.long_window.success_rate === null ? '数据不足' : `${metrics.long_window.success_rate}%`}</b></div><div>p50 {metrics.short_window.duration_p50_seconds === null ? '—' : duration(Math.round(metrics.short_window.duration_p50_seconds))} · p95 {metrics.short_window.duration_p95_seconds === null ? '—' : duration(Math.round(metrics.short_window.duration_p95_seconds))}</div><div>Worker {metrics.active_workers} · 等待 {metrics.queue.pending} · 运行 {metrics.queue.running}{metrics.queue.failed ? <span className="bad"> · 失败 {metrics.queue.failed}</span> : null}</div>{overview?.health.map((service) => <div key={service.key}>{service.name} <b className={service.readiness === 'ready' ? 'good' : service.readiness === 'blocked' || service.readiness === 'down' ? 'bad' : ''}>{service.readiness === 'ready' ? '可接单' : service.readiness === 'degraded' ? '降级' : service.readiness === 'not_configured' ? '未配置' : service.readiness === 'down' ? '不可达' : '阻断'}</b></div>)}{overview ? <div>CPU {overview.host.cpu_percent.toFixed(0)}% · 内存 {gb(overview.host.memory_used_bytes)}/{gb(overview.host.memory_total_bytes)}GB</div> : null}</footer> : null}
|
||||
</div>
|
||||
@@ -237,14 +224,10 @@ export function WeKnoraMonitor() {
|
||||
const [files, setFiles] = useState<MonitorFilePage | null>(null)
|
||||
const [queue, setQueue] = useState<TestPublicationPage>(emptyQueue)
|
||||
const [plans, setPlans] = useState<TestPublication[]>([])
|
||||
const [changeSets, setChangeSets] = useState<CatalogChangeSetDetail[]>([])
|
||||
const [confirmingSetID, setConfirmingSetID] = useState<string | null>(null)
|
||||
const [lastConfirmSetID, setLastConfirmSetID] = useState<string | null>(null)
|
||||
const [confirmMessage, setConfirmMessage] = useState('')
|
||||
const [status, setStatus] = useState<MonitorStatus | ''>('')
|
||||
const [queueStatus, setQueueStatus] = useState<ProcessingState | ''>('')
|
||||
const [queueScope, setQueueScope] = useState<'internal' | 'external' | ''>('')
|
||||
const [lane, setLane] = useState<'attention' | 'changes' | 'inflight' | 'pending' | 'done' | 'queue'>('attention')
|
||||
const [lane, setLane] = useState<'waiting' | 'attention' | 'removal' | 'inflight' | 'done' | 'queue'>('waiting')
|
||||
const [loading, setLoading] = useState(true)
|
||||
const [error, setError] = useState<string | null>(null)
|
||||
const [documentRevision, setDocumentRevision] = useState(0)
|
||||
@@ -283,13 +266,9 @@ export function WeKnoraMonitor() {
|
||||
const controller = new AbortController(); setLoading(true)
|
||||
void Promise.all([
|
||||
fetchTestPublications({ processing_state: queueStatus, target_scope: queueScope, limit: 50, offset: 0 }, controller.signal),
|
||||
fetchTestPublications({ processing_state: 'pending', limit: 100, offset: 0 }, controller.signal),
|
||||
fetchCatalogSources(controller.signal).then(async sources => {
|
||||
const results=await Promise.allSettled(sources.map(source=>fetchLatestCatalogChangeSet(source.connection_id,'',20,0,controller.signal)))
|
||||
return results.flatMap(result=>result.status==='fulfilled'?[result.value]:[])
|
||||
}),
|
||||
]).then(([queuePage, planPage, latestChangeSets]) => {
|
||||
if (!controller.signal.aborted) { setQueue(queuePage); setPlans(planPage.items.filter((item) => item.can_start)); setChangeSets(latestChangeSets); setError(null) }
|
||||
fetchTestPublications({ processing_state: 'planned', limit: 100, offset: 0 }, controller.signal),
|
||||
]).then(([queuePage, planPage]) => {
|
||||
if (!controller.signal.aborted) { setQueue(queuePage); setPlans(planPage.items); setError(null) }
|
||||
}).catch((cause) => { if (!controller.signal.aborted) setError(cause instanceof Error ? cause.message : '加载解析队列失败') }).finally(() => { if (!controller.signal.aborted) setLoading(false) })
|
||||
return () => controller.abort()
|
||||
}, [queueStatus, queueScope, documentRevision])
|
||||
@@ -360,12 +339,12 @@ export function WeKnoraMonitor() {
|
||||
}
|
||||
}
|
||||
|
||||
const startPlans = async (ids: string[], reason: string) => {
|
||||
const startPlans = async (ids: string[]) => {
|
||||
setStarting(true)
|
||||
setStartMessage('')
|
||||
try {
|
||||
const result = await startTestPublications(ids, reason)
|
||||
setStartMessage(`已确认 ${result.selected} 项;新建解析任务 ${result.started} 项;已开始 ${result.already_started} 项;不可处理 ${result.unavailable} 项。`)
|
||||
const result = await startTestPublications(ids, '开始处理')
|
||||
setStartMessage(`已提交 ${result.selected} 项;新建解析任务 ${result.started} 项;已在队列 ${result.already_started} 项;不可处理 ${result.unavailable} 项(多为源头取件未完成,稍后自动重试)。`)
|
||||
setError(null)
|
||||
setDocumentRevision((value) => value + 1)
|
||||
} catch (cause) {
|
||||
@@ -374,19 +353,5 @@ export function WeKnoraMonitor() {
|
||||
setStarting(false)
|
||||
}
|
||||
}
|
||||
|
||||
const confirmChangeSet = async (changeSetID: string, itemSeqs: number[], reason: string) => {
|
||||
if (confirmingSetID) return
|
||||
setConfirmingSetID(changeSetID); setLastConfirmSetID(changeSetID); setConfirmMessage('')
|
||||
try {
|
||||
const {result,operation} = await confirmCatalogChangeSet(changeSetID, itemSeqs, reason)
|
||||
setConfirmMessage(`已${operation==='replayed'?'沿用上次结果':'提交'}:确认 ${result.confirmed_items} 项;新建上传任务 ${result.uploads_created} 个;已有任务 ${result.uploads_existing} 个;请求移除 ${result.removals_requested} 份;内容未就绪 ${result.skipped_no_artifact} 份;缺少目标知识库 ${result.skipped_no_target} 份;未变化 ${result.no_change_ignored} 项已忽略。`)
|
||||
setDocumentRevision((value) => value + 1)
|
||||
} catch (cause) {
|
||||
setConfirmMessage(cause instanceof Error ? cause.message : '确认快照差异失败')
|
||||
} finally {
|
||||
setConfirmingSetID(null)
|
||||
}
|
||||
}
|
||||
return <WeKnoraMonitorView overview={overview} files={files} queue={queue} plans={plans} changeSets={changeSets} confirmingSetID={confirmingSetID ?? lastConfirmSetID} confirmMessage={confirmMessage} onConfirmChangeSet={(id,seqs,reason)=>void confirmChangeSet(id,seqs,reason)} selectedKB={selected} status={status} queueStatus={queueStatus} queueScope={queueScope} loading={loading} error={error} retryingIDs={retryingIDs} starting={starting} startMessage={startMessage} batchMessage={batchMessage} failures={failures} metrics={metrics} onSelectKB={(kb) => { setSelected(kb); setStatus(''); setFiles(null) }} onStatus={(value) => { setStatus(value); setFiles(null) }} onQueueStatus={setQueueStatus} onQueueScope={setQueueScope} lane={lane} onLaneChange={(value) => setLane(value as typeof lane)} onRetry={(id) => void retryOne(id)} onBatchRetry={(ids) => void retryBatch(ids)} onStart={(ids, reason) => void startPlans(ids, reason)} onRefresh={() => { setDocumentRevision((value) => value + 1); setResourceRevision((value) => value + 1) }} />
|
||||
}
|
||||
return <WeKnoraMonitorView overview={overview} files={files} queue={queue} plans={plans} selectedKB={selected} status={status} queueStatus={queueStatus} queueScope={queueScope} loading={loading} error={error} retryingIDs={retryingIDs} starting={starting} startMessage={startMessage} batchMessage={batchMessage} failures={failures} metrics={metrics} onSelectKB={(kb) => { setSelected(kb); setStatus(''); setFiles(null) }} onStatus={(value) => { setStatus(value); setFiles(null) }} onQueueStatus={setQueueStatus} onQueueScope={setQueueScope} lane={lane} onLaneChange={(value) => setLane(value as typeof lane)} onRetry={(id) => void retryOne(id)} onBatchRetry={(ids) => void retryBatch(ids)} onStart={(ids) => void startPlans(ids)} onRefresh={() => { setDocumentRevision((value) => value + 1); setResourceRevision((value) => value + 1) }} />
|
||||
}
|
||||
|
||||
+3
-2
@@ -1,9 +1,10 @@
|
||||
export type MonitorStatus='completed'|'pending'|'processing'|'finalizing'|'failed'|'cancelled'
|
||||
export interface MonitorHost{cpu_percent:number;memory_used_bytes:number;memory_total_bytes:number;disk_used_bytes:number;disk_total_bytes:number;gpu_state:string;gpu_detail:string;simulated:boolean}
|
||||
export interface MonitorService{key:string;name:string;state:string;detail:string;latency_ms:number;observed_at:string;simulated:boolean}
|
||||
export interface KnowledgeProgress{id:string;name:string;visibility:string;imported:number;completed:number;pending:number;processing:number;finalizing:number;failed:number;cancelled:number;internal_imported:number;internal_completed:number;external_imported:number;external_completed:number}
|
||||
export interface KnowledgeProgress{id:string;name:string;visibility:string;imported:number;completed:number;pending:number;processing:number;finalizing:number;failed:number;cancelled:number;planned:number;needs_removal:number;stale_planned:number;internal_imported:number;internal_completed:number;external_imported:number;external_completed:number}
|
||||
export interface ServiceHealth{key:string;name:string;liveness:string;readiness:string;reason:string;latency_ms:number;observed_at:string;last_success_at:string|null;consecutive_failures:number;configured:boolean;applicable:boolean}
|
||||
export interface MonitorOverview{mode:string;simulated:boolean;observed_at:string;host:MonitorHost;services:MonitorService[];knowledge_bases:KnowledgeProgress[];health:ServiceHealth[]}
|
||||
export interface IngestionPipeline{fetch_pending:number;fetch_failed:number;fetch_missing:number;concurrency:number}
|
||||
export interface MonitorOverview{mode:string;simulated:boolean;observed_at:string;host:MonitorHost;services:MonitorService[];knowledge_bases:KnowledgeProgress[];pipeline?:IngestionPipeline;health:ServiceHealth[]}
|
||||
export interface MonitorFile{name:string;directory:string;visibility:string;status:MonitorStatus;stage_detail:string;elapsed_seconds:number;error_code:string}
|
||||
export interface MonitorFilePage{items:MonitorFile[];total:number;limit:number;offset:number;sample_only:boolean}
|
||||
const record=(value:unknown):value is Record<string,unknown>=>typeof value==='object'&&value!==null
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { newOperationID } from './operationKey'
|
||||
export type IngestState = 'pending' | 'uploading' | 'uploaded' | 'failed'
|
||||
export type ProcessingState = 'pending' | 'processing' | 'finalizing' | 'completed' | 'failed' | 'stalled'
|
||||
export type ProcessingState = 'planned' | 'pending' | 'processing' | 'finalizing' | 'completed' | 'failed' | 'stalled'
|
||||
|
||||
export interface TestPublication {
|
||||
id: string
|
||||
@@ -50,7 +50,7 @@ export interface StartTestPublicationsResult {
|
||||
const isRecord = (value: unknown): value is Record<string, unknown> => typeof value === 'object' && value !== null
|
||||
const isCount = (value: unknown): value is number => typeof value === 'number' && Number.isInteger(value) && value >= 0
|
||||
const ingestStates = new Set<IngestState>(['pending', 'uploading', 'uploaded', 'failed'])
|
||||
const processingStates = new Set<ProcessingState>(['pending', 'processing', 'finalizing', 'completed', 'failed', 'stalled'])
|
||||
const processingStates = new Set<ProcessingState>(['planned', 'pending', 'processing', 'finalizing', 'completed', 'failed', 'stalled'])
|
||||
|
||||
function parseTestPublication(value: unknown): TestPublication {
|
||||
if (!isRecord(value) || typeof value.id !== 'string' || typeof value.title !== 'string' ||
|
||||
|
||||
@@ -5,7 +5,7 @@ import { activateUISettings, createUISettingsCandidate, fetchActiveUISettings, p
|
||||
const active = {
|
||||
id: 'settings-default', state: 'active', document_refresh_seconds: 15,
|
||||
resource_refresh_seconds: 30, stalled_after_seconds: 1800,
|
||||
finalizing_after_seconds: 7200, updated_by: 'system:migration',
|
||||
finalizing_after_seconds: 7200, ingestion_concurrency: 2, updated_by: 'system:migration',
|
||||
created_at: '2026-09-10T08:00:00Z', activated_at: '2026-09-10T08:00:00Z',
|
||||
retired_at: null, error_code: '',
|
||||
}
|
||||
@@ -29,12 +29,12 @@ describe('UI settings API', () => {
|
||||
vi.stubGlobal('fetch', fetchMock)
|
||||
|
||||
await fetchActiveUISettings()
|
||||
await createUISettingsCandidate({ document_refresh_seconds: 30, resource_refresh_seconds: 60, stalled_after_seconds: 3600, finalizing_after_seconds: 14400 })
|
||||
await createUISettingsCandidate({ document_refresh_seconds: 30, resource_refresh_seconds: 60, stalled_after_seconds: 3600, finalizing_after_seconds: 14400, ingestion_concurrency: 3 })
|
||||
await activateUISettings('uis-candidate')
|
||||
|
||||
expect(fetchMock.mock.calls[0][0]).toBe('/api/v1/ui-settings')
|
||||
expect(fetchMock.mock.calls[1][0]).toBe('/api/v1/ui-settings/candidates')
|
||||
expect(fetchMock.mock.calls[1][1]).toMatchObject({ method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ document_refresh_seconds: 30, resource_refresh_seconds: 60, stalled_after_seconds: 3600, finalizing_after_seconds: 14400 }) })
|
||||
expect(fetchMock.mock.calls[1][1]).toMatchObject({ method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ document_refresh_seconds: 30, resource_refresh_seconds: 60, stalled_after_seconds: 3600, finalizing_after_seconds: 14400, ingestion_concurrency: 3 }) })
|
||||
expect(fetchMock.mock.calls[2][0]).toBe('/api/v1/ui-settings/uis-candidate/activate')
|
||||
expect(fetchMock.mock.calls[2][1]).toMatchObject({ method: 'POST', headers: { 'Content-Type': 'application/json' }, body: '{}' })
|
||||
vi.unstubAllGlobals()
|
||||
|
||||
@@ -6,6 +6,7 @@ export interface UISettings {
|
||||
resource_refresh_seconds: 30 | 60
|
||||
stalled_after_seconds: 1800 | 3600
|
||||
finalizing_after_seconds: 7200 | 14400
|
||||
ingestion_concurrency: number
|
||||
updated_by: string
|
||||
created_at: string
|
||||
activated_at: string | null
|
||||
@@ -18,6 +19,7 @@ export interface UISettingsCandidateInput {
|
||||
resource_refresh_seconds: UISettings['resource_refresh_seconds']
|
||||
stalled_after_seconds: UISettings['stalled_after_seconds']
|
||||
finalizing_after_seconds: UISettings['finalizing_after_seconds']
|
||||
ingestion_concurrency: number
|
||||
}
|
||||
|
||||
export function settingsRefreshDelays(settings: Pick<UISettings, 'document_refresh_seconds' | 'resource_refresh_seconds'>) {
|
||||
@@ -40,6 +42,7 @@ export function parseUISettingsResponse(value: unknown): UISettings {
|
||||
typeof item.resource_refresh_seconds !== 'number' || !resourceRefresh.has(item.resource_refresh_seconds) ||
|
||||
typeof item.stalled_after_seconds !== 'number' || !stalledAfter.has(item.stalled_after_seconds) ||
|
||||
typeof item.finalizing_after_seconds !== 'number' || !finalizingAfter.has(item.finalizing_after_seconds) ||
|
||||
typeof item.ingestion_concurrency !== 'number' || !Number.isInteger(item.ingestion_concurrency) || item.ingestion_concurrency < 1 || item.ingestion_concurrency > 16 ||
|
||||
typeof item.updated_by !== 'string' || typeof item.created_at !== 'string' ||
|
||||
(item.activated_at !== null && typeof item.activated_at !== 'string') ||
|
||||
(item.retired_at !== null && typeof item.retired_at !== 'string') || typeof item.error_code !== 'string') {
|
||||
@@ -52,6 +55,7 @@ export function parseUISettingsResponse(value: unknown): UISettings {
|
||||
resource_refresh_seconds: item.resource_refresh_seconds as UISettings['resource_refresh_seconds'],
|
||||
stalled_after_seconds: item.stalled_after_seconds as UISettings['stalled_after_seconds'],
|
||||
finalizing_after_seconds: item.finalizing_after_seconds as UISettings['finalizing_after_seconds'],
|
||||
ingestion_concurrency: item.ingestion_concurrency,
|
||||
updated_by: item.updated_by,
|
||||
created_at: item.created_at,
|
||||
activated_at: item.activated_at as string | null,
|
||||
|
||||
Reference in New Issue
Block a user