feat(library): 整库发布身份、写集合与收尾边界
build / build (push) Successful in 1m45s

执行时真实时钟与持久授权回读(撤销/过期即停并保留预备份)、写集合写前落盘、
租约与 attempt 身份、恢复队列与原发送身份分离、成功收尾必须释放维护后再登记终态。
This commit is contained in:
姚章浩
2026-11-01 11:35:10 +08:00
parent 5afd8c0140
commit 8d838d9cb4
82 changed files with 4809 additions and 551 deletions
+1 -1
View File
@@ -99,7 +99,7 @@ func isAuthEndpoint(method, path string) bool {
// classify maps a request to the minimum role required.
func classifyRole(method, path string) string {
if path == "/api/v1/source-connections/audit" {
if path == "/api/v1/source-connections/audit" || path == "/api/v1/library/source-associations" {
return roleAdmin
}
// The member roster exposes account identities and roles: admin-only for
+2 -23
View File
@@ -9,7 +9,6 @@ import (
"time"
"knowledge-governance/internal/governance"
"knowledge-governance/internal/jobs"
postgresstore "knowledge-governance/internal/platform/postgres"
)
@@ -107,12 +106,12 @@ func libraryBaselineEnqueue(w http.ResponseWriter, r *http.Request, config Confi
return
}
now := time.Now().UTC()
job, reused, err := store.CreateLibraryJob(r.Context(), governance.LibraryJob{
job, reused, err := store.CreateLibraryDispatch(r.Context(), governance.LibraryJob{
ID: fmt.Sprintf("libbase-%s-%d", request.Scope, now.UnixNano()), Scope: request.Scope,
Kind: governance.LibraryJobBaseline,
IdempotencyKey: "library.baseline:" + request.Scope + ":" + now.Format("20060102T150405.000000000"),
RequestDigest: digest, Actor: identity.Username,
}, governance.BaselineStages(), now)
}, governance.BaselineStages(), false, now)
if errors.Is(err, governance.ErrLibraryJobActive) {
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_write_active",
"error": "该目标已有在飞的写作业(同步/恢复/基线互斥),等它结束再发起"})
@@ -131,26 +130,6 @@ func libraryBaselineEnqueue(w http.ResponseWriter, r *http.Request, config Confi
writeJSON(w, http.StatusAccepted, map[string]any{"job": job, "reused": true})
return
}
payload, err := json.Marshal(map[string]string{"library_job_id": job.ID, "scope": request.Scope})
if err != nil {
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot build baseline payload"})
return
}
queueJob, err := config.Store.Enqueue(r.Context(), jobs.EnqueueRequest{
ID: governance.QueueJobIDFor(job.ID), Type: libraryBaselineJobType, Payload: payload,
AvailableAt: now, MaxAttempts: 1, IdempotencyKey: "library.baseline.queue:" + job.ID,
}, now)
if err != nil || queueJob == nil {
// 作业先落库、队列投递失败就把作业取消:不留下"看起来在跑"的空作业,
// 否则它会一直占着该 scope 的写互斥。
_ = store.CancelLibraryJob(r.Context(), job.ID, "队列投递失败,未开始建立基线", now)
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot enqueue baseline job"})
return
}
if attachErr := store.AttachLibraryQueueJob(r.Context(), job.ID, queueJob.ID, now); attachErr != nil {
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot attach baseline job"})
return
}
writeJSON(w, http.StatusAccepted, map[string]any{"job": job, "reused": false,
"target_counts": counts})
}
+18 -44
View File
@@ -2,12 +2,9 @@ package httpapi
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"log"
"net/http"
"strings"
"time"
@@ -360,6 +357,7 @@ type libraryVersionRef struct {
}
type libraryJobStore interface {
CreateLibraryDispatch(context.Context, governance.LibraryJob, []string, bool, time.Time) (governance.LibraryJob, bool, error)
CreateLibraryJob(ctx context.Context, job governance.LibraryJob, stages []string, now time.Time) (governance.LibraryJob, bool, error)
// EnsureLibraryOperation 幂等登记"一次明确意图":同键+同摘要 ⇒ 同一次意图(客户端重放),
// 换键 ⇒ 新意图。作业是意图的执行代次,不能反过来用"历史成功作业"回答新意图(F01)。
@@ -470,8 +468,7 @@ func libraryMaintenanceRecoverHandler(w http.ResponseWriter, r *http.Request, co
}
// 队列作业 id 与幂等键都按窗口固定:重复点击只会得到同一个恢复作业。
// 队列 id 有长度上限(36),窗口作业 id 比它还长,所以按窗口摘要取短 id。
sum := sha256.Sum256([]byte(held.OwnerJobID))
queueID := "lrec-" + hex.EncodeToString(sum[:14])
queueID := governance.LibraryRecoveryQueueID(held.OwnerJobID)
queueJob, err := config.Store.Enqueue(r.Context(), jobs.EnqueueRequest{
ID: queueID, Type: libraryRecoverJobType, Payload: payload,
AvailableAt: now, MaxAttempts: 1, IdempotencyKey: "library.recover:" + held.OwnerJobID,
@@ -727,7 +724,11 @@ func libraryOverviewHandler(w http.ResponseWriter, r *http.Request, config Confi
return
}
if archiveReadable {
overview.TestSealed, overview.TestSealedReason = testSealedOf(versions, byID)
var candidateErr error
overview.TestSealed, candidateErr = currentSourceCandidate(r.Context(), config, scope)
if candidateErr != nil {
overview.TestSealedReason = candidateErr.Error()
}
}
overview.Production = libraryKnownView{State: "unknown", Reason: "还没有生产验证通过的生效版本记录"}
overview.Prebackup = libraryKnownView{State: "unknown", Reason: "这个范围还没有任何生产预备份"}
@@ -873,6 +874,7 @@ func librarySyncEnqueue(w http.ResponseWriter, r *http.Request, config Config) {
// OperationKey 是客户端重放键:同一次意图重发要命中同一个 operation。
// 不传就是一次新意图(同一个快照再发布一次也必须真的执行)。
OperationKey string `json:"operation_key"`
SourceID string `json:"source_id"`
}
if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 4096)).Decode(&request); err != nil {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid request body"})
@@ -939,6 +941,11 @@ func librarySyncEnqueue(w http.ResponseWriter, r *http.Request, config Config) {
return
}
candidate, candidateErr := currentSourceCandidate(r.Context(), config, request.Scope)
if candidateErr != nil || request.SourceID == "" || candidate == nil || candidate.SourceID != request.SourceID || candidate.SnapshotID != request.Target {
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_source_candidate_changed", "error": "来源候选已变化或身份未绑定,请刷新后重试", "detail": errString(candidateErr)})
return
}
// 走到这里说明闸口与证据都齐了才真正建作业;互斥与幂等由数据库索引保证。
digest, err := governance.LibraryRequestDigest(request.Scope, governance.LibraryJobSync,
map[string]any{"target_snapshot_id": request.Target})
@@ -965,11 +972,11 @@ func librarySyncEnqueue(w http.ResponseWriter, r *http.Request, config Config) {
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot register library intent"})
return
}
job, reused, err := store.CreateLibraryJob(r.Context(), governance.LibraryJob{
job, reused, err := store.CreateLibraryDispatch(r.Context(), governance.LibraryJob{
ID: jobID, Scope: request.Scope, Kind: governance.LibraryJobSync,
IdempotencyKey: intent.JobKey, RequestDigest: digest, OperationID: intent.Operation.ID,
TargetSnapshotID: request.Target, Actor: actor,
}, governance.SyncStages(), now)
}, governance.SyncStages(), config.LibraryHandoffEnabled, now)
if errors.Is(err, governance.ErrLibraryJobActive) {
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_sync_active",
"error": "该目标已有在飞的生产写作业(同步与恢复互斥)"})
@@ -986,13 +993,13 @@ func librarySyncEnqueue(w http.ResponseWriter, r *http.Request, config Config) {
// 上一次同意图请求失败/被取消后,幂等键不能再把新请求塞回那个死作业里:
// 在**同一个 operation** 下开一个新的执行作业(重试代次),旧作业保留为审计记录。
if reused && (job.State == governance.LibraryJobFailed || job.State == governance.LibraryJobCancelled) {
job, reused, err = store.CreateLibraryJob(r.Context(), governance.LibraryJob{
job, reused, err = store.CreateLibraryDispatch(r.Context(), governance.LibraryJob{
ID: fmt.Sprintf("libsync-%s-%d", request.Scope, time.Now().UTC().UnixNano()), Scope: request.Scope,
Kind: governance.LibraryJobSync,
IdempotencyKey: intent.JobKey + ":retry:" + now.Format("20060102T150405.000000000"),
RequestDigest: digest, OperationID: intent.Operation.ID,
TargetSnapshotID: request.Target, Actor: actor,
}, governance.SyncStages(), now)
}, governance.SyncStages(), config.LibraryHandoffEnabled, now)
if err != nil || reused {
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_sync_retry_conflict",
"error": "无法为重试创建新作业", "detail": errString(err)})
@@ -1003,27 +1010,6 @@ func librarySyncEnqueue(w http.ResponseWriter, r *http.Request, config Config) {
writeJSON(w, http.StatusAccepted, map[string]any{"job": job, "reused": true})
return
}
// 业务作业先落库,再投递队列:投递失败就把作业标失败,不留下"看起来在跑"的空作业。
queueID, queueType, payload, err := libraryQueueRequest(config.LibraryHandoffEnabled, job.ID,
request.Scope, request.Target, librarySyncJobType)
if err != nil {
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot build sync payload"})
return
}
queueJob, err := config.Store.Enqueue(r.Context(), jobs.EnqueueRequest{
ID: queueID, Type: queueType, Payload: payload, AvailableAt: now,
MaxAttempts: 1, IdempotencyKey: queueType + ".queue:" + job.ID,
}, now)
if err != nil || queueJob == nil {
log.Printf("library sync enqueue failed job=%s: %v", job.ID, err)
_ = store.CancelLibraryJob(r.Context(), job.ID, "队列投递失败,未开始同步", now)
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot enqueue sync job"})
return
}
if attachErr := store.AttachLibraryQueueJob(r.Context(), job.ID, queueJob.ID, now); attachErr != nil {
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot attach sync job"})
return
}
writeJSON(w, http.StatusAccepted, map[string]any{"job": job, "reused": false})
}
@@ -1033,17 +1019,5 @@ func librarySyncEnqueue(w http.ResponseWriter, r *http.Request, config Config) {
// 由它在成功后再入 releaseType;搬运未接线时保持原样。两种情况下 releaseType
// 作业的 ID 都是 QueueJobIDFor(jobID),所以接管/续作的既有路径不受影响。
func libraryQueueRequest(handoff bool, jobID, scope, snapshotID, releaseType string) (string, string, []byte, error) {
payload := map[string]string{"library_job_id": jobID, "scope": scope, "target_snapshot_id": snapshotID}
queueID := governance.QueueJobIDFor(jobID)
queueType := releaseType
if handoff {
payload["next_type"] = releaseType
queueID = governance.LibraryHandoffQueueJobID(jobID)
queueType = libraryHandoffJobType
}
raw, err := json.Marshal(payload)
if err != nil {
return "", "", nil, err
}
return queueID, queueType, raw, nil
return governance.LibraryQueueRequest(handoff, jobID, scope, snapshotID, releaseType)
}
+48
View File
@@ -0,0 +1,48 @@
package httpapi
import (
"context"
"encoding/json"
"io"
"knowledge-governance/internal/governance"
"net/http"
"time"
)
type libraryLegacySourceBinder interface {
BindLegacyLibrarySource(context.Context, string, string, governance.LibrarySourceIdentity, string, time.Time) error
}
func libraryLegacySourceBind(w http.ResponseWriter, r *http.Request, config Config) {
identity, ok := sessionFrom(r.Context())
if !ok || identity.Role != roleAdmin || identity.Username == "" {
writeJSON(w, http.StatusForbidden, map[string]string{"error": "administrator session required"})
return
}
var input struct {
Scope string `json:"scope"`
SnapshotID string `json:"snapshot_id"`
}
decoder := json.NewDecoder(http.MaxBytesReader(w, r.Body, 4096))
decoder.DisallowUnknownFields()
if decoder.Decode(&input) != nil || decoder.Decode(new(any)) != io.EOF || (input.Scope != "internal" && input.Scope != "external") || input.SnapshotID == "" {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "exact snapshot and scope required"})
return
}
source := configuredLibrarySource(config, input.Scope)
sourceID, err := source.ID()
if err != nil {
writeJSON(w, http.StatusServiceUnavailable, map[string]string{"error": err.Error()})
return
}
store, ok := config.Store.(libraryLegacySourceBinder)
if !ok {
writeJSON(w, http.StatusServiceUnavailable, map[string]string{"error": "source association store unavailable"})
return
}
if err = store.BindLegacyLibrarySource(r.Context(), input.SnapshotID, config.LibraryArchiveRoot, source, identity.Username, time.Now().UTC()); err != nil {
writeJSON(w, http.StatusConflict, map[string]string{"error": err.Error()})
return
}
writeJSON(w, http.StatusOK, map[string]any{"snapshot_id": input.SnapshotID, "source_id": sourceID, "source_identity": source, "associated": true, "published": false})
}
+4 -23
View File
@@ -104,12 +104,12 @@ func libraryRestoreEnqueue(w http.ResponseWriter, r *http.Request, config Config
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot register restore intent"})
return
}
job, reused, err := store.CreateLibraryJob(r.Context(), governance.LibraryJob{
job, reused, err := store.CreateLibraryDispatch(r.Context(), governance.LibraryJob{
ID: jobID, Scope: request.Scope, Kind: governance.LibraryJobRestore,
IdempotencyKey: intent.JobKey,
RequestDigest: digest, OperationID: intent.Operation.ID, TargetSnapshotID: request.SnapshotID,
Actor: identity.Username,
}, governance.SyncStages(), now)
}, governance.SyncStages(), config.LibraryHandoffEnabled, now)
if errors.Is(err, governance.ErrLibraryJobActive) {
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_restore_active", "error": "该目标已有在飞作业"})
return
@@ -123,12 +123,12 @@ func libraryRestoreEnqueue(w http.ResponseWriter, r *http.Request, config Config
return
}
if reused && (job.State == governance.LibraryJobFailed || job.State == governance.LibraryJobCancelled) {
job, reused, err = store.CreateLibraryJob(r.Context(), governance.LibraryJob{
job, reused, err = store.CreateLibraryDispatch(r.Context(), governance.LibraryJob{
ID: jobID + "-retry", Scope: request.Scope, Kind: governance.LibraryJobRestore,
IdempotencyKey: intent.JobKey + ":retry:" + strconv.FormatInt(now.UnixNano(), 10),
RequestDigest: digest, OperationID: intent.Operation.ID, TargetSnapshotID: request.SnapshotID,
Actor: identity.Username,
}, governance.SyncStages(), now)
}, governance.SyncStages(), config.LibraryHandoffEnabled, now)
if err != nil || reused {
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_restore_retry_conflict", "error": "无法为重试创建新作业"})
return
@@ -138,25 +138,6 @@ func libraryRestoreEnqueue(w http.ResponseWriter, r *http.Request, config Config
writeJSON(w, http.StatusAccepted, map[string]any{"job": job, "reused": true})
return
}
queueID, queueType, payload, err := libraryQueueRequest(config.LibraryHandoffEnabled, job.ID,
request.Scope, request.SnapshotID, libraryRestoreJobType)
if err != nil {
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot build restore payload"})
return
}
queueJob, err := config.Store.Enqueue(r.Context(), jobs.EnqueueRequest{
ID: queueID, Type: queueType, Payload: payload, AvailableAt: now,
MaxAttempts: 1, IdempotencyKey: queueType + ".queue:" + job.ID,
}, now)
if err != nil || queueJob == nil {
_ = store.CancelLibraryJob(r.Context(), job.ID, "队列投递失败,未开始恢复", now)
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot enqueue restore job"})
return
}
if attachErr := store.AttachLibraryQueueJob(r.Context(), job.ID, queueJob.ID, now); attachErr != nil {
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot attach restore job"})
return
}
writeJSON(w, http.StatusAccepted, map[string]any{"job": job, "reused": false})
}
+16 -13
View File
@@ -8,6 +8,7 @@ import (
"strings"
"time"
"knowledge-governance/internal/governance"
"knowledge-governance/internal/librarysnapshot"
)
@@ -28,19 +29,21 @@ type librarySnapshotPart struct {
}
type librarySnapshotView struct {
SnapshotID string `json:"snapshot_id"`
Scope string `json:"scope"`
KnowledgeBaseID string `json:"knowledge_base_id"`
TenantID uint64 `json:"tenant_id"`
CreatedAt time.Time `json:"created_at"`
ValidationPolicy string `json:"validation_policy"`
Format string `json:"format"`
Compatibility librarysnapshot.Compatibility `json:"compatibility"`
Parts []librarySnapshotPart `json:"parts"`
LogicalBytes int64 `json:"logical_bytes"`
UniqueBytes int64 `json:"unique_bytes"`
SharedBytes int64 `json:"shared_bytes"`
Totals map[string]int64 `json:"totals"`
SourceID string `json:"source_id,omitempty"`
SourceIdentity *governance.LibrarySourceIdentity `json:"source_identity,omitempty"`
SnapshotID string `json:"snapshot_id"`
Scope string `json:"scope"`
KnowledgeBaseID string `json:"knowledge_base_id"`
TenantID uint64 `json:"tenant_id"`
CreatedAt time.Time `json:"created_at"`
ValidationPolicy string `json:"validation_policy"`
Format string `json:"format"`
Compatibility librarysnapshot.Compatibility `json:"compatibility"`
Parts []librarySnapshotPart `json:"parts"`
LogicalBytes int64 `json:"logical_bytes"`
UniqueBytes int64 `json:"unique_bytes"`
SharedBytes int64 `json:"shared_bytes"`
Totals map[string]int64 `json:"totals"`
// Verified 只在这一份版本的每个部件都被重新哈希并核对通过时为 true。
// 列表口径下它恒为 false:枚举只证明清单可读,不是"完整性已验证"。
Verified bool `json:"verified"`
@@ -0,0 +1,58 @@
package httpapi
import (
"context"
"errors"
"knowledge-governance/internal/governance"
"knowledge-governance/internal/librarysnapshot"
)
type librarySourceCandidateStore interface {
LibrarySourceCandidate(context.Context, string, string) (governance.LibraryVersion, bool, error)
}
func configuredLibrarySource(config Config, scope string) governance.LibrarySourceIdentity {
kb := config.LibrarySourceInternalKB
if scope == "external" {
kb = config.LibrarySourceExternalKB
}
return governance.LibrarySourceIdentity{Project: config.LibrarySourceProject, App: config.LibrarySourceApp, Scope: scope, TenantID: config.LibrarySourceTenantID, KnowledgeBaseID: kb}
}
func currentSourceCandidate(ctx context.Context, config Config, scope string) (*librarySnapshotView, error) {
source := configuredLibrarySource(config, scope)
id, err := source.ID()
if err != nil {
return nil, err
}
store, ok := config.Store.(librarySourceCandidateStore)
if !ok {
return nil, errors.New("source candidate exact query unavailable")
}
candidate, found, err := store.LibrarySourceCandidate(ctx, scope, id)
if err != nil {
return nil, err
}
if !found {
return nil, errors.New("当前来源没有已绑定封存;旧快照须以原捕获证据核验关联,不能继承其他来源")
}
if candidate.Kind != governance.LibraryVersionTestSealed || candidate.Scope != scope || candidate.Evidence["source_id"] != id {
return nil, errors.New("candidate source binding mismatch")
}
archive, err := librarysnapshot.Open(config.LibraryArchiveRoot, 256*1024*1024)
if err != nil {
return nil, err
}
defer archive.Close()
manifest, err := archive.Load(candidate.SnapshotID)
if err != nil {
return nil, err
}
if manifest.Scope != scope || manifest.TenantID != source.TenantID || manifest.KnowledgeBaseID != source.KnowledgeBaseID {
return nil, errors.New("candidate manifest differs from configured source")
}
view := librarySnapshotViewOfNoUsage(candidate.SnapshotID, manifest)
view.SourceID = id
view.SourceIdentity = &source
return &view, nil
}
+8 -1
View File
@@ -167,7 +167,11 @@ type Config struct {
LibraryAgentSocket string
// LibrarySourceProject 是执行器应指向的实例名(agent -project,封存源)。
// 与它不一致时按"封错库"处理:不可发起,并说明差在哪。
LibrarySourceProject string
LibrarySourceProject string
LibrarySourceApp string
LibrarySourceTenantID uint64
LibrarySourceInternalKB string
LibrarySourceExternalKB string
// LibraryExecutorStatusMaxAge 是执行器观测的有效期;超过即"无法确认"。
LibraryExecutorStatusMaxAge time.Duration
// LibraryCapacityBudgetBytes 是整库归档容量预算;0 = 未设置(容量门禁不放行)。
@@ -449,6 +453,9 @@ func NewHandler(config Config) (http.Handler, error) {
librarySnapshotDetail(w, r, config)
})
// 页面发起封存:只投递作业(主机侧 agent 执行),并回读作业真实状态。
mux.HandleFunc("POST /api/v1/library/source-associations", func(w http.ResponseWriter, r *http.Request) {
libraryLegacySourceBind(w, r, config)
})
mux.HandleFunc("POST /api/v1/library/captures", func(w http.ResponseWriter, r *http.Request) {
libraryCaptureEnqueue(w, r, config)
})