Files
knowledge-governance/internal/workflows/testweknora/upload.go
T
zheromiku dcdc8d1ddd feat: 知识治理平台(含已验证发布/回滚契约)
全栈知识治理平台首次入库(P1-P10 + 发布链路模拟验证闭环):

- 治理核心:源目录/变更集/人工审核/发布批次/发布执行(fail-closed 闸口)
- 测试通道:WeKnora 测试 KB 上传/轮询/验收;poll 完成时写入
  parser_generation/index_generation,修复 manual/seed 路径发布闸口永久拦截
- 发布适配器 v071(本地 source=测试角色 / 云端 target=生产角色 模拟验证通过):
  * ES 导出/导入 mapping 自适应(.keyword 子字段、可选 tenant_id)
  * OSS locator 支持嵌套 scheme;chunks.seq_id 由目标序列分配
  * bundle 归档同身份重试原子替换;verifier 按 bundle chunk_type 计数
  * 回滚 job payload 支持 prior_execution_id(strict 解码)
- 三通道发布+回滚演练实测:publish 4/4 verified,检索命中,
  rollback 4/4 rolled_back,目标端 PG is_enabled=false、ES is_enabled=false
- 前端 A·Data-console 风格全站统一,120 用例通过
- 门禁:go vet / go test 全包通过,gofmt 干净
2026-09-30 10:54:06 +08:00

242 lines
8.2 KiB
Go

package testweknora
import (
"bytes"
"context"
"encoding/json"
"errors"
"io"
"path/filepath"
"strings"
"time"
"knowledge-governance/internal/governance"
"knowledge-governance/internal/jobs"
postgresstore "knowledge-governance/internal/platform/postgres"
"knowledge-governance/internal/platform/spool"
"knowledge-governance/internal/platform/weknora"
)
var (
ErrInvalidUploadJobPayload = errors.New("test_upload_invalid_payload")
ErrUploadArtifactInvalid = errors.New("test_upload_artifact_invalid")
ErrUploadRemoteFailed = errors.New("test_upload_remote_failed")
ErrUploadRemoteIdentity = errors.New("test_upload_remote_identity_invalid")
ErrUploadStateConflict = errors.New("test_upload_state_conflict")
)
type UploadStore interface {
LoadPublicationForUpload(context.Context, string) (postgresstore.PublicationForUpload, error)
MarkUploading(context.Context, string, time.Time) error
RecordUploadAndSchedulePoll(context.Context, string, string, time.Time, time.Time) (int, error)
RecordUploadFailure(context.Context, string, string, time.Time) error
}
type UploadClient interface {
UploadFile(context.Context, weknora.UploadRequest) (weknora.Knowledge, error)
ListKnowledge(context.Context, string, int, int) ([]weknora.Knowledge, int64, error)
GetKnowledge(context.Context, string) (weknora.Knowledge, error)
}
type UploadHandlerConfig struct {
Store UploadStore
Client UploadClient
SpoolRoot string
MaxUploadBytes int64
FirstPollDelay time.Duration
Now func() time.Time
}
type UploadHandler struct {
store UploadStore
client UploadClient
spoolRoot string
maxUploadBytes int64
firstPollDelay time.Duration
now func() time.Time
}
type uploadPayload struct {
PublicationID string `json:"publication_id"`
TargetScope string `json:"target_scope,omitempty"`
TargetKBID string `json:"target_kb_id,omitempty"`
}
func NewUploadHandler(config UploadHandlerConfig) (*UploadHandler, error) {
if config.Store == nil || config.Client == nil || !filepath.IsAbs(config.SpoolRoot) ||
config.MaxUploadBytes <= 0 || config.FirstPollDelay <= 0 {
return nil, errors.New("invalid test upload handler configuration")
}
if config.Now == nil {
config.Now = time.Now
}
return &UploadHandler{
store: config.Store, client: config.Client, spoolRoot: config.SpoolRoot,
maxUploadBytes: config.MaxUploadBytes, firstPollDelay: config.FirstPollDelay, now: config.Now,
}, nil
}
func (handler *UploadHandler) Handle(ctx context.Context, job jobs.Job) error {
var payload uploadPayload
if err := decodeStrictJSON(job.Payload, &payload); err != nil || payload.PublicationID == "" {
return jobs.Permanent(jobs.ErrCodeHandlerFallback, ErrInvalidUploadJobPayload)
}
candidate, err := handler.store.LoadPublicationForUpload(ctx, payload.PublicationID)
if err != nil {
return ErrUploadStateConflict
}
// P4: the job snapshots its target; a mismatch against the stored publication
// means the plan changed underneath and this delivery must not guess.
if payload.TargetKBID != "" && payload.TargetKBID != candidate.TestKBID {
return ErrUploadStateConflict
}
if payload.TargetScope != "" && payload.TargetScope != string(candidate.TargetScope) {
return ErrUploadStateConflict
}
if candidate.IngestState == governance.IngestUploading {
knowledge, err := handler.recoverAmbiguousUpload(ctx, candidate)
if err != nil {
if errors.Is(err, ErrUploadRemoteIdentity) {
return handler.recordTerminalUploadFailure(ctx, candidate.ID, "upload_identity_ambiguous")
}
return err
}
now := handler.now()
if _, err := handler.store.RecordUploadAndSchedulePoll(ctx, candidate.ID, knowledge.ID, now, now.Add(handler.firstPollDelay)); err != nil {
return ErrUploadStateConflict
}
return nil
}
artifact, err := spool.OpenVerified(handler.spoolRoot, spool.Expectation{
RelativeRef: candidate.LocalSpoolRef,
SHA256: candidate.SHA256,
SizeBytes: candidate.SizeBytes,
Format: candidate.ValidatedFormat,
}, handler.maxUploadBytes)
if err != nil {
return handler.recordTerminalUploadFailure(ctx, candidate.ID, "upload_artifact_invalid")
}
defer artifact.Close()
if err := handler.store.MarkUploading(ctx, candidate.ID, handler.now()); err != nil {
return ErrUploadStateConflict
}
knowledge, uploadErr := handler.client.UploadFile(ctx, weknora.UploadRequest{
KnowledgeBaseID: candidate.TestKBID,
FileName: safeUploadFileName(candidate.Title, candidate.ID, candidate.ValidatedFormat),
Content: artifact,
Metadata: map[string]string{
"content_sha256": candidate.SHA256,
"publication_id": candidate.ID,
},
Channel: "governance",
})
if err := artifact.VerifyComplete(); err != nil {
return jobs.Permanent(jobs.ErrCodeArtifactInvalid, ErrUploadArtifactInvalid)
}
if uploadErr != nil {
if !errors.Is(uploadErr, weknora.ErrConflict) && !weknora.IsRetryable(uploadErr) {
return ErrUploadRemoteFailed
}
knowledge, err = handler.recoverAmbiguousUpload(ctx, candidate)
if err != nil {
if errors.Is(err, ErrUploadRemoteIdentity) {
return handler.recordTerminalUploadFailure(ctx, candidate.ID, "upload_identity_ambiguous")
}
return err
}
}
if knowledge.ID == "" || knowledge.KnowledgeBaseID != candidate.TestKBID {
return ErrUploadRemoteIdentity
}
now := handler.now()
if _, err := handler.store.RecordUploadAndSchedulePoll(ctx, candidate.ID, knowledge.ID, now, now.Add(handler.firstPollDelay)); err != nil {
return ErrUploadStateConflict
}
return nil
}
func (handler *UploadHandler) recoverAmbiguousUpload(ctx context.Context, candidate postgresstore.PublicationForUpload) (weknora.Knowledge, error) {
const pageSize = 100
matches := make([]weknora.Knowledge, 0, 2)
for page := 1; page <= 100; page++ {
items, total, err := handler.client.ListKnowledge(ctx, candidate.TestKBID, page, pageSize)
if err != nil {
return weknora.Knowledge{}, ErrUploadRemoteFailed
}
for _, item := range items {
publicationID, publicationOK := metadataString(item.Metadata, "publication_id")
contentSHA, contentOK := metadataString(item.Metadata, "content_sha256")
if item.KnowledgeBaseID == candidate.TestKBID && publicationOK && contentOK && publicationID == candidate.ID && contentSHA == candidate.SHA256 {
matches = append(matches, item)
if len(matches) > 1 {
return weknora.Knowledge{}, ErrUploadRemoteIdentity
}
}
}
if int64(page*pageSize) >= total {
break
}
if len(items) == 0 {
return weknora.Knowledge{}, ErrUploadRemoteFailed
}
}
if len(matches) != 1 {
return weknora.Knowledge{}, ErrUploadRemoteFailed
}
verified, err := handler.client.GetKnowledge(ctx, matches[0].ID)
if err != nil {
return weknora.Knowledge{}, ErrUploadRemoteFailed
}
publicationID, publicationOK := metadataString(verified.Metadata, "publication_id")
contentSHA, contentOK := metadataString(verified.Metadata, "content_sha256")
if verified.ID != matches[0].ID || verified.KnowledgeBaseID != candidate.TestKBID ||
!publicationOK || !contentOK || publicationID != candidate.ID || contentSHA != candidate.SHA256 {
return weknora.Knowledge{}, ErrUploadRemoteIdentity
}
return verified, nil
}
func metadataString(metadata map[string]any, key string) (string, bool) {
value, ok := metadata[key].(string)
return value, ok
}
func (handler *UploadHandler) recordTerminalUploadFailure(ctx context.Context, publicationID, code string) error {
if err := handler.store.RecordUploadFailure(ctx, publicationID, code, handler.now()); err != nil {
return ErrUploadStateConflict
}
return nil
}
func decodeStrictJSON(data []byte, target any) error {
decoder := json.NewDecoder(bytes.NewReader(data))
decoder.DisallowUnknownFields()
if err := decoder.Decode(target); err != nil {
return err
}
if err := decoder.Decode(&struct{}{}); err != io.EOF {
return errors.New("extra JSON value")
}
return nil
}
func safeUploadFileName(title, fallback, format string) string {
base := strings.TrimSpace(title)
base = strings.Map(func(character rune) rune {
switch character {
case '/', '\\', '\x00', '\r', '\n', '\t':
return '_'
default:
return character
}
}, base)
base = strings.Trim(base, ". ")
if base == "" {
base = fallback
}
if len(base) > 120 {
base = base[:120]
}
return base + "." + format
}