2eff9447e1
build / build (push) Failing after 1m20s
ReconcileLibraryJobs 不再把 completed capture queue 当成不存在,保留待核对 pending 封存作业;failed/cancelled/missing 仍按真实队列状态处置。管理员核对 入口只允许精确的启动误判形态,并要求 completed queue、export succeeded、 唯一 test_sealed 版本、来源身份与完整归档复核。新增 library_job_repairs 不可变审计表,在同一事务内 CAS 保存原 state/stage/error/recovery/finished_at 及本次 actor/原因/证据后修复;真实失败/取消不开放。 验证:真实隔离 PostgreSQL 启动收敛 RED/GREEN、启动→handler 纵向 pending 与 系统误判 failed、审计保留/幂等、负例;真实 Go 全量与 make verify 17 项通过。
247 lines
10 KiB
Go
247 lines
10 KiB
Go
package httpapi
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"net/http"
|
|
"strings"
|
|
"time"
|
|
|
|
"knowledge-governance/internal/governance"
|
|
"knowledge-governance/internal/jobs"
|
|
"knowledge-governance/internal/librarysnapshot"
|
|
)
|
|
|
|
// libraryCaptureReconciler is deliberately narrow: it repairs one business
|
|
// seal job only after the completed capture queue, the successful export stage,
|
|
// the exact test-sealed version and the archived bytes all agree. It never
|
|
// captures, freezes writers, creates snapshots, writes source/target content or
|
|
// changes historical failure evidence.
|
|
type libraryCaptureReconciler interface {
|
|
LibraryJob(ctx context.Context, id string) (governance.LibraryJob, error)
|
|
LibraryCaptureExportEvidence(ctx context.Context, jobID string) (string, string, error)
|
|
LibraryTestSealedVersionForJob(ctx context.Context, scope, jobID string) (governance.LibraryVersion, bool, error)
|
|
EnsureLibraryOperation(ctx context.Context, op governance.LibraryOperation) (governance.LibraryOperation, bool, error)
|
|
ReconcileLibraryCaptureTerminal(ctx context.Context, jobID, scope, queueJobID, snapshotID string, now time.Time) (bool, error)
|
|
ReconcileMisclassifiedLibraryCapture(ctx context.Context, jobID, scope, queueJobID, snapshotID, expectedStage, expectedError, expectedRecovery string, expectedFinished *time.Time, auditID, actor, reason string, evidence map[string]any, now time.Time) (bool, error)
|
|
}
|
|
|
|
const (
|
|
legacyOrphanError = "队列中已无对应作业,业务作业未收到结果"
|
|
legacyOrphanRecovery = "封存作业由 agent 恢复来源;确认来源 healthy 后可重发"
|
|
)
|
|
|
|
type libraryCaptureQueueReader interface {
|
|
QueueJob(ctx context.Context, id string) (jobs.Job, bool, error)
|
|
}
|
|
|
|
func libraryCaptureReconcile(w http.ResponseWriter, r *http.Request, config Config) {
|
|
if r.Method != http.MethodPost {
|
|
writeJSON(w, http.StatusMethodNotAllowed, map[string]string{"error": "method not allowed"})
|
|
return
|
|
}
|
|
store, ok := config.Store.(libraryCaptureReconciler)
|
|
if !ok {
|
|
writeJSON(w, http.StatusServiceUnavailable, map[string]any{"available": false,
|
|
"reason": "封存作业核对存储不可用"})
|
|
return
|
|
}
|
|
queueStore, ok := config.Store.(libraryCaptureQueueReader)
|
|
if !ok {
|
|
writeJSON(w, http.StatusServiceUnavailable, map[string]any{"available": false,
|
|
"reason": "队列精确读取不可用"})
|
|
return
|
|
}
|
|
jobID := strings.TrimSpace(r.PathValue("id"))
|
|
if jobID == "" {
|
|
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "job id required"})
|
|
return
|
|
}
|
|
job, err := store.LibraryJob(r.Context(), jobID)
|
|
if err != nil {
|
|
writeJSON(w, http.StatusNotFound, map[string]any{"available": false, "reason": "作业不存在"})
|
|
return
|
|
}
|
|
if job.Kind != governance.LibraryJobSeal {
|
|
conflict(w, "只有封存作业可以核对封存结果")
|
|
return
|
|
}
|
|
systemMisclassified := job.State == governance.LibraryJobFailed && job.Stage == "" &&
|
|
job.Error == legacyOrphanError && job.Recovery == legacyOrphanRecovery && job.SnapshotID == ""
|
|
if job.State != governance.LibraryJobPending && job.State != governance.LibraryJobRunning &&
|
|
job.State != governance.LibraryJobSucceeded && !systemMisclassified {
|
|
conflict(w, "作业是真实失败/取消终态,不能改写成成功")
|
|
return
|
|
}
|
|
if job.QueueJobID == "" {
|
|
conflict(w, "作业没有关联队列号,不能证明封存完成")
|
|
return
|
|
}
|
|
captureJob := strings.TrimPrefix(job.ID, "libseal-")
|
|
if captureJob == "" || captureJob == job.ID {
|
|
conflict(w, "作业号不是封存作业身份")
|
|
return
|
|
}
|
|
queue, found, err := queueStore.QueueJob(r.Context(), job.QueueJobID)
|
|
if err != nil {
|
|
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot read capture queue"})
|
|
return
|
|
}
|
|
if !found {
|
|
conflict(w, "队列作业不存在,不能证明封存完成")
|
|
return
|
|
}
|
|
if queue.Type != libraryCaptureJobType || queue.State != jobs.StateCompleted {
|
|
conflict(w, "队列作业不是 completed 的 library.capture")
|
|
return
|
|
}
|
|
var payload struct {
|
|
Scope string `json:"scope"`
|
|
CaptureJob string `json:"capture_job"`
|
|
}
|
|
if err := json.Unmarshal(queue.Payload, &payload); err != nil ||
|
|
payload.Scope != job.Scope || payload.CaptureJob != captureJob ||
|
|
queue.IdempotencyKey != "library.capture:"+captureJob {
|
|
conflict(w, "队列作业的 scope/capture_job/幂等键与业务作业不一致")
|
|
return
|
|
}
|
|
stageState, _, err := store.LibraryCaptureExportEvidence(r.Context(), job.ID)
|
|
if err != nil || stageState != "succeeded" {
|
|
conflict(w, "导出阶段没有成功证据,不能修复终态")
|
|
return
|
|
}
|
|
version, found, err := store.LibraryTestSealedVersionForJob(r.Context(), job.Scope, job.ID)
|
|
if err != nil {
|
|
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot read exact test-sealed version"})
|
|
return
|
|
}
|
|
if !found || version.Kind != governance.LibraryVersionTestSealed ||
|
|
version.Scope != job.Scope || version.SourceJobID != job.ID || version.SnapshotID == "" {
|
|
conflict(w, "没有与作业/scope 精确绑定的测试封存版本")
|
|
return
|
|
}
|
|
source := configuredLibrarySource(config, job.Scope)
|
|
sourceID, err := source.ID()
|
|
if err != nil {
|
|
writeJSON(w, http.StatusServiceUnavailable, map[string]any{"available": false,
|
|
"reason": "来源身份配置不完整,无法核对封存来源"})
|
|
return
|
|
}
|
|
evidenceSource, _ := version.Evidence["source_id"].(string)
|
|
evidenceCapture, _ := version.Evidence["capture_job"].(string)
|
|
if evidenceSource != sourceID || evidenceCapture != captureJob {
|
|
conflict(w, "版本来源或 capture_job 与当前作业不一致")
|
|
return
|
|
}
|
|
if config.LibraryArchiveRoot == "" {
|
|
writeJSON(w, http.StatusServiceUnavailable, map[string]any{"available": false,
|
|
"reason": "未接线归档根,无法复核归档完整性"})
|
|
return
|
|
}
|
|
archive, err := librarysnapshot.Open(config.LibraryArchiveRoot, 256*1024*1024)
|
|
if err != nil {
|
|
writeJSON(w, http.StatusServiceUnavailable, map[string]any{"available": false,
|
|
"reason": "归档不可读,不能确认封存完整"})
|
|
return
|
|
}
|
|
defer archive.Close()
|
|
manifest, err := archive.Load(version.SnapshotID)
|
|
if err != nil {
|
|
conflict(w, "归档字节复核失败,不能修复作业")
|
|
return
|
|
}
|
|
if manifest.Scope != source.Scope || manifest.TenantID != source.TenantID ||
|
|
manifest.KnowledgeBaseID != source.KnowledgeBaseID {
|
|
conflict(w, "归档清单来源身份与当前来源不一致")
|
|
return
|
|
}
|
|
if job.SnapshotID != "" && job.SnapshotID != version.SnapshotID {
|
|
conflict(w, "作业已有不同快照,不能覆盖")
|
|
return
|
|
}
|
|
if job.State == governance.LibraryJobSucceeded {
|
|
if job.SnapshotID != version.SnapshotID {
|
|
conflict(w, "作业已成功但快照与精确版本不一致")
|
|
return
|
|
}
|
|
writeJSON(w, http.StatusOK, map[string]any{
|
|
"repaired": false, "already_succeeded": true, "job_id": job.ID,
|
|
"snapshot_id": version.SnapshotID, "queue_job_id": queue.ID,
|
|
"version_id": version.ID,
|
|
})
|
|
return
|
|
}
|
|
actor := ""
|
|
if identity, ok := sessionFrom(r.Context()); ok {
|
|
actor = identity.Username
|
|
}
|
|
now := time.Now().UTC()
|
|
digest, err := governance.LibraryRequestDigest(job.Scope, governance.LibraryJobRecover, map[string]any{
|
|
"kind": "capture_reconcile", "job_id": job.ID, "queue_job_id": queue.ID,
|
|
"snapshot_id": version.SnapshotID, "capture_job": captureJob, "stage": "export",
|
|
})
|
|
if err != nil {
|
|
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot build reconcile audit digest"})
|
|
return
|
|
}
|
|
if systemMisclassified {
|
|
auditID := "librepair-" + digest[:16]
|
|
updated, repairErr := store.ReconcileMisclassifiedLibraryCapture(r.Context(), job.ID, job.Scope,
|
|
job.QueueJobID, version.SnapshotID, job.Stage, legacyOrphanError, legacyOrphanRecovery,
|
|
job.FinishedAt, auditID, actor, "completed capture queue + export succeeded + exact version + archive verified; legacy orphan misclassification", map[string]any{
|
|
"original_state": job.State, "original_error": job.Error, "original_recovery": job.Recovery,
|
|
"original_finished_at": job.FinishedAt, "queue_job_id": queue.ID,
|
|
"snapshot_id": version.SnapshotID, "version_id": version.ID, "archive_verified": true,
|
|
}, now)
|
|
if repairErr != nil {
|
|
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot persist misclassification repair"})
|
|
return
|
|
}
|
|
if !updated {
|
|
conflict(w, "原误判行状态/错误/完成时间已变化,拒绝覆盖")
|
|
return
|
|
}
|
|
writeJSON(w, http.StatusOK, map[string]any{
|
|
"repaired": true, "already_succeeded": false, "misclassification_repaired": true,
|
|
"job_id": job.ID, "snapshot_id": version.SnapshotID, "queue_job_id": queue.ID,
|
|
"stage": "export", "version_id": version.ID, "audit_repair_id": auditID,
|
|
})
|
|
return
|
|
}
|
|
opID := "libop-capture-reconcile-" + digest[:16]
|
|
op, _, err := store.EnsureLibraryOperation(r.Context(), governance.LibraryOperation{
|
|
ID: opID, Scope: job.Scope, Kind: governance.LibraryJobRecover,
|
|
TargetSnapshotID: version.SnapshotID, RequestDigest: digest,
|
|
OperationKey: "capture-reconcile:" + job.ID, Actor: actor, CreatedAt: now,
|
|
})
|
|
if errors.Is(err, governance.ErrLibraryOperationConflict) {
|
|
conflict(w, "同一核对键对应了不同证据,拒绝修复")
|
|
return
|
|
}
|
|
if err != nil {
|
|
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot record reconcile audit"})
|
|
return
|
|
}
|
|
updated, err := store.ReconcileLibraryCaptureTerminal(r.Context(), job.ID, job.Scope, job.QueueJobID, version.SnapshotID, now)
|
|
if err != nil {
|
|
writeJSON(w, http.StatusInternalServerError, map[string]any{
|
|
"error": "cannot persist reconciled job terminal", "audit_operation_id": op.ID,
|
|
})
|
|
return
|
|
}
|
|
if !updated {
|
|
conflict(w, "作业状态/身份在核对期间变化,拒绝覆盖")
|
|
return
|
|
}
|
|
writeJSON(w, http.StatusOK, map[string]any{
|
|
"repaired": true, "already_succeeded": false, "job_id": job.ID,
|
|
"snapshot_id": version.SnapshotID, "queue_job_id": queue.ID,
|
|
"stage": "export", "version_id": version.ID, "audit_operation_id": op.ID,
|
|
})
|
|
}
|
|
|
|
func conflict(w http.ResponseWriter, reason string) {
|
|
writeJSON(w, http.StatusConflict, map[string]any{"available": false, "repaired": false, "reason": reason})
|
|
}
|