Files
姚章浩 63f3b2c503 feat(governance): policy-driven ingestion from catalog annotations
Annotation in 来源与目录 is now the only import decision surface:
- migration 000033: catalog_import_intents (durable, rescan-proof per-file
  intent with locator snapshot + fetch lifecycle), source_items.policy_state,
  planned publication state, ui_setting_revisions.ingestion_concurrency
- marking fans a folder decision out to per-file intents, materializes
  versioned items, reuses already-verified local bytes (compare, don't pull),
  and plans one publication per routed KB; blocked retires planned rows
  while live publications stay untouched (需下架 shows in the KB diff)
- artifact.fetch worker lane (scripts/artifact_fetch.py): DingTalk export
  tree/live box API, AList /api/fs/get direct download with SSO-redirect
  fail-closed, PP ledger via locator hash; content-level dedupe on sha256
- 开始处理 promotes planned->pending inside the claim-time concurrency budget
  from system settings; claims enforce it in SQL
- monitor overview rebuilt as knowledge-base diff: planned/needs_removal per
  KB + fetch pipeline summary; frontend lanes 待入库/需下架 replace the
  deprecated snapshot change-set confirmation
- spool: images (png/jpg/gif/bmp/webp) accepted, ppt/pptx rejected in favour
  of PDF twins; DingTalk PPT resolves its converted PDF pair
2026-09-30 10:54:07 +08:00

150 lines
5.9 KiB
Go

package postgres
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"github.com/jackc/pgx/v5"
"knowledge-governance/internal/governance"
)
func (s *Store) StartTestPublications(ctx context.Context, request governance.StartTestPublicationsRequest) (governance.StartTestPublicationsResult, error) {
result := governance.StartTestPublicationsResult{Selected: len(request.PublicationIDs)}
operator := strings.TrimSpace(request.OperatorRef)
reason := strings.TrimSpace(request.Reason)
if len(request.PublicationIDs) == 0 || len(request.PublicationIDs) > 100 || operator == "" || len(operator) > 128 || len(reason) > 500 || request.Now.IsZero() {
return governance.StartTestPublicationsResult{}, errors.New("invalid processing confirmation")
}
if reason == "" {
reason = "开始处理"
}
seen := make(map[string]bool, len(request.PublicationIDs))
for _, publicationID := range request.PublicationIDs {
if !validRouteID(publicationID) || seen[publicationID] {
return governance.StartTestPublicationsResult{}, errors.New("invalid processing publication IDs")
}
seen[publicationID] = true
}
var tx pgx.Tx
var err error
ownsTransaction := false
if existing, ok := s.db.(pgx.Tx); ok {
tx = existing
} else {
beginner, ok := s.db.(interface {
Begin(context.Context) (pgx.Tx, error)
})
if !ok {
return governance.StartTestPublicationsResult{}, errors.New("processing confirmation transactions are unavailable")
}
var err error
tx, err = beginner.Begin(ctx)
if err != nil {
return governance.StartTestPublicationsResult{}, fmt.Errorf("begin processing confirmation: %w", err)
}
ownsTransaction = true
defer tx.Rollback(ctx)
}
for _, publicationID := range request.PublicationIDs {
var ready, confirmed, uploadJobExists bool
var processingState string
err = tx.QueryRow(ctx, `
SELECT p.ingest_state='pending'
AND p.retrieval_state='unpublished'
AND p.knowledge_id=''
AND version.content_state='verified'
AND content.content_state='verified'
AND content.local_spool_ref<>''
AND content.size_bytes>0 AS ready,
p.processing_state,
EXISTS(SELECT 1 FROM processing_confirmations confirmation WHERE confirmation.publication_id=p.id),
EXISTS(SELECT 1 FROM jobs job WHERE job.type='test.upload' AND job.payload->>'publication_id'=p.id)
FROM test_publications p
JOIN source_item_versions version ON version.id=p.source_item_version_id
JOIN content_objects content ON content.id=version.content_object_id
WHERE p.id=$1 AND p.superseded_at IS NULL
FOR UPDATE OF p`, publicationID).Scan(&ready, &processingState, &confirmed, &uploadJobExists)
if errors.Is(err, pgx.ErrNoRows) {
result.Unavailable++
continue
}
if err != nil {
return governance.StartTestPublicationsResult{}, fmt.Errorf("inspect processing plan: %w", err)
}
if confirmed || uploadJobExists {
result.AlreadyStarted++
continue
}
// 开始处理 promotes both fresh plans and verified pending rows.
if processingState != "planned" && processingState != "pending" {
result.Unavailable++
continue
}
if !ready {
result.Unavailable++
continue
}
if _, err := tx.Exec(ctx, `UPDATE test_publications SET processing_state='pending',updated_at=$2 WHERE id=$1 AND processing_state='planned'`, publicationID, request.Now); err != nil {
return governance.StartTestPublicationsResult{}, fmt.Errorf("promote planned publication: %w", err)
}
confirmationID, err := newOpaqueID("pc-")
if err != nil {
return governance.StartTestPublicationsResult{}, err
}
jobID, err := newOpaqueID("job-")
if err != nil {
return governance.StartTestPublicationsResult{}, err
}
payload, err := json.Marshal(map[string]string{"publication_id": publicationID})
if err != nil {
return governance.StartTestPublicationsResult{}, errors.New("encode processing upload job")
}
if _, err := tx.Exec(ctx, `INSERT INTO processing_confirmations(id,publication_id,operator_ref,reason,confirmed_at,created_at) VALUES($1,$2,$3,$4,$5,$5)`, confirmationID, publicationID, operator, reason, request.Now); err != nil {
return governance.StartTestPublicationsResult{}, fmt.Errorf("record processing confirmation: %w", err)
}
if _, err := tx.Exec(ctx, `INSERT INTO jobs(id,type,payload,state,priority,available_at,max_attempts,idempotency_key,created_at,updated_at)
VALUES($1,'test.upload',$2::jsonb,'pending',10,$3,5,$4,$3,$3)`, jobID, payload, request.Now, "test-start:"+publicationID); err != nil {
return governance.StartTestPublicationsResult{}, fmt.Errorf("enqueue confirmed processing: %w", err)
}
result.Started++
}
if ownsTransaction {
if err := tx.Commit(ctx); err != nil {
return governance.StartTestPublicationsResult{}, fmt.Errorf("commit processing confirmation: %w", err)
}
}
return result, nil
}
func (s *Store) StartTestPublicationsWithOperation(ctx context.Context, request governance.StartTestPublicationsRequest, operationID string) (governance.StartTestPublicationsResult, bool, error) {
fingerprint, err := operationFingerprint(struct {
PublicationIDs []string `json:"publication_ids"`
OperatorRef string `json:"operator_ref"`
Reason string `json:"reason"`
}{request.PublicationIDs, request.OperatorRef, request.Reason})
if err != nil {
return governance.StartTestPublicationsResult{}, false, err
}
raw, replayed, err := s.executeOperation(ctx, "test.publications.start", operationID, fingerprint, request.Now, func(tx pgx.Tx) (json.RawMessage, error) {
result, err := NewStore(tx).StartTestPublications(ctx, request)
if err != nil {
return nil, err
}
return json.Marshal(result)
})
if err != nil {
return governance.StartTestPublicationsResult{}, false, err
}
var result governance.StartTestPublicationsResult
if err := json.Unmarshal(raw, &result); err != nil {
return governance.StartTestPublicationsResult{}, false, fmt.Errorf("decode stored processing result: %w", err)
}
return result, replayed, nil
}