63f3b2c503
Annotation in 来源与目录 is now the only import decision surface: - migration 000033: catalog_import_intents (durable, rescan-proof per-file intent with locator snapshot + fetch lifecycle), source_items.policy_state, planned publication state, ui_setting_revisions.ingestion_concurrency - marking fans a folder decision out to per-file intents, materializes versioned items, reuses already-verified local bytes (compare, don't pull), and plans one publication per routed KB; blocked retires planned rows while live publications stay untouched (需下架 shows in the KB diff) - artifact.fetch worker lane (scripts/artifact_fetch.py): DingTalk export tree/live box API, AList /api/fs/get direct download with SSO-redirect fail-closed, PP ledger via locator hash; content-level dedupe on sha256 - 开始处理 promotes planned->pending inside the claim-time concurrency budget from system settings; claims enforce it in SQL - monitor overview rebuilt as knowledge-base diff: planned/needs_removal per KB + fetch pipeline summary; frontend lanes 待入库/需下架 replace the deprecated snapshot change-set confirmation - spool: images (png/jpg/gif/bmp/webp) accepted, ppt/pptx rejected in favour of PDF twins; DingTalk PPT resolves its converted PDF pair
150 lines
5.9 KiB
Go
150 lines
5.9 KiB
Go
package postgres
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"strings"
|
|
|
|
"github.com/jackc/pgx/v5"
|
|
|
|
"knowledge-governance/internal/governance"
|
|
)
|
|
|
|
func (s *Store) StartTestPublications(ctx context.Context, request governance.StartTestPublicationsRequest) (governance.StartTestPublicationsResult, error) {
|
|
result := governance.StartTestPublicationsResult{Selected: len(request.PublicationIDs)}
|
|
operator := strings.TrimSpace(request.OperatorRef)
|
|
reason := strings.TrimSpace(request.Reason)
|
|
if len(request.PublicationIDs) == 0 || len(request.PublicationIDs) > 100 || operator == "" || len(operator) > 128 || len(reason) > 500 || request.Now.IsZero() {
|
|
return governance.StartTestPublicationsResult{}, errors.New("invalid processing confirmation")
|
|
}
|
|
if reason == "" {
|
|
reason = "开始处理"
|
|
}
|
|
seen := make(map[string]bool, len(request.PublicationIDs))
|
|
for _, publicationID := range request.PublicationIDs {
|
|
if !validRouteID(publicationID) || seen[publicationID] {
|
|
return governance.StartTestPublicationsResult{}, errors.New("invalid processing publication IDs")
|
|
}
|
|
seen[publicationID] = true
|
|
}
|
|
|
|
var tx pgx.Tx
|
|
var err error
|
|
ownsTransaction := false
|
|
if existing, ok := s.db.(pgx.Tx); ok {
|
|
tx = existing
|
|
} else {
|
|
beginner, ok := s.db.(interface {
|
|
Begin(context.Context) (pgx.Tx, error)
|
|
})
|
|
if !ok {
|
|
return governance.StartTestPublicationsResult{}, errors.New("processing confirmation transactions are unavailable")
|
|
}
|
|
var err error
|
|
tx, err = beginner.Begin(ctx)
|
|
if err != nil {
|
|
return governance.StartTestPublicationsResult{}, fmt.Errorf("begin processing confirmation: %w", err)
|
|
}
|
|
ownsTransaction = true
|
|
defer tx.Rollback(ctx)
|
|
}
|
|
|
|
for _, publicationID := range request.PublicationIDs {
|
|
var ready, confirmed, uploadJobExists bool
|
|
var processingState string
|
|
err = tx.QueryRow(ctx, `
|
|
SELECT p.ingest_state='pending'
|
|
AND p.retrieval_state='unpublished'
|
|
AND p.knowledge_id=''
|
|
AND version.content_state='verified'
|
|
AND content.content_state='verified'
|
|
AND content.local_spool_ref<>''
|
|
AND content.size_bytes>0 AS ready,
|
|
p.processing_state,
|
|
EXISTS(SELECT 1 FROM processing_confirmations confirmation WHERE confirmation.publication_id=p.id),
|
|
EXISTS(SELECT 1 FROM jobs job WHERE job.type='test.upload' AND job.payload->>'publication_id'=p.id)
|
|
FROM test_publications p
|
|
JOIN source_item_versions version ON version.id=p.source_item_version_id
|
|
JOIN content_objects content ON content.id=version.content_object_id
|
|
WHERE p.id=$1 AND p.superseded_at IS NULL
|
|
FOR UPDATE OF p`, publicationID).Scan(&ready, &processingState, &confirmed, &uploadJobExists)
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
result.Unavailable++
|
|
continue
|
|
}
|
|
if err != nil {
|
|
return governance.StartTestPublicationsResult{}, fmt.Errorf("inspect processing plan: %w", err)
|
|
}
|
|
if confirmed || uploadJobExists {
|
|
result.AlreadyStarted++
|
|
continue
|
|
}
|
|
// 开始处理 promotes both fresh plans and verified pending rows.
|
|
if processingState != "planned" && processingState != "pending" {
|
|
result.Unavailable++
|
|
continue
|
|
}
|
|
if !ready {
|
|
result.Unavailable++
|
|
continue
|
|
}
|
|
if _, err := tx.Exec(ctx, `UPDATE test_publications SET processing_state='pending',updated_at=$2 WHERE id=$1 AND processing_state='planned'`, publicationID, request.Now); err != nil {
|
|
return governance.StartTestPublicationsResult{}, fmt.Errorf("promote planned publication: %w", err)
|
|
}
|
|
confirmationID, err := newOpaqueID("pc-")
|
|
if err != nil {
|
|
return governance.StartTestPublicationsResult{}, err
|
|
}
|
|
jobID, err := newOpaqueID("job-")
|
|
if err != nil {
|
|
return governance.StartTestPublicationsResult{}, err
|
|
}
|
|
payload, err := json.Marshal(map[string]string{"publication_id": publicationID})
|
|
if err != nil {
|
|
return governance.StartTestPublicationsResult{}, errors.New("encode processing upload job")
|
|
}
|
|
if _, err := tx.Exec(ctx, `INSERT INTO processing_confirmations(id,publication_id,operator_ref,reason,confirmed_at,created_at) VALUES($1,$2,$3,$4,$5,$5)`, confirmationID, publicationID, operator, reason, request.Now); err != nil {
|
|
return governance.StartTestPublicationsResult{}, fmt.Errorf("record processing confirmation: %w", err)
|
|
}
|
|
if _, err := tx.Exec(ctx, `INSERT INTO jobs(id,type,payload,state,priority,available_at,max_attempts,idempotency_key,created_at,updated_at)
|
|
VALUES($1,'test.upload',$2::jsonb,'pending',10,$3,5,$4,$3,$3)`, jobID, payload, request.Now, "test-start:"+publicationID); err != nil {
|
|
return governance.StartTestPublicationsResult{}, fmt.Errorf("enqueue confirmed processing: %w", err)
|
|
}
|
|
result.Started++
|
|
}
|
|
if ownsTransaction {
|
|
if err := tx.Commit(ctx); err != nil {
|
|
return governance.StartTestPublicationsResult{}, fmt.Errorf("commit processing confirmation: %w", err)
|
|
}
|
|
}
|
|
return result, nil
|
|
}
|
|
|
|
func (s *Store) StartTestPublicationsWithOperation(ctx context.Context, request governance.StartTestPublicationsRequest, operationID string) (governance.StartTestPublicationsResult, bool, error) {
|
|
fingerprint, err := operationFingerprint(struct {
|
|
PublicationIDs []string `json:"publication_ids"`
|
|
OperatorRef string `json:"operator_ref"`
|
|
Reason string `json:"reason"`
|
|
}{request.PublicationIDs, request.OperatorRef, request.Reason})
|
|
if err != nil {
|
|
return governance.StartTestPublicationsResult{}, false, err
|
|
}
|
|
raw, replayed, err := s.executeOperation(ctx, "test.publications.start", operationID, fingerprint, request.Now, func(tx pgx.Tx) (json.RawMessage, error) {
|
|
result, err := NewStore(tx).StartTestPublications(ctx, request)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return json.Marshal(result)
|
|
})
|
|
if err != nil {
|
|
return governance.StartTestPublicationsResult{}, false, err
|
|
}
|
|
var result governance.StartTestPublicationsResult
|
|
if err := json.Unmarshal(raw, &result); err != nil {
|
|
return governance.StartTestPublicationsResult{}, false, fmt.Errorf("decode stored processing result: %w", err)
|
|
}
|
|
return result, replayed, nil
|
|
}
|