feat: 知识治理平台(含已验证发布/回滚契约)

全栈知识治理平台首次入库(P1-P10 + 发布链路模拟验证闭环):

- 治理核心:源目录/变更集/人工审核/发布批次/发布执行(fail-closed 闸口)
- 测试通道:WeKnora 测试 KB 上传/轮询/验收;poll 完成时写入
  parser_generation/index_generation,修复 manual/seed 路径发布闸口永久拦截
- 发布适配器 v071(本地 source=测试角色 / 云端 target=生产角色 模拟验证通过):
  * ES 导出/导入 mapping 自适应(.keyword 子字段、可选 tenant_id)
  * OSS locator 支持嵌套 scheme;chunks.seq_id 由目标序列分配
  * bundle 归档同身份重试原子替换;verifier 按 bundle chunk_type 计数
  * 回滚 job payload 支持 prior_execution_id(strict 解码)
- 三通道发布+回滚演练实测:publish 4/4 verified,检索命中,
  rollback 4/4 rolled_back,目标端 PG is_enabled=false、ES is_enabled=false
- 前端 A·Data-console 风格全站统一,120 用例通过
- 门禁:go vet / go test 全包通过,gofmt 干净
This commit is contained in:
2026-09-13 12:02:58 +08:00
committed by 姚章浩
parent ed48c4ceda
commit dcdc8d1ddd
410 changed files with 56661 additions and 1 deletions
@@ -0,0 +1,191 @@
package catalogrefresh
import (
"bufio"
"bytes"
"context"
"encoding/json"
"errors"
"io"
"os/exec"
"path/filepath"
"strings"
"time"
"unicode"
"knowledge-governance/internal/jobs"
)
type Request struct {
ConnectionID string `json:"connection_id"`
SourceType string `json:"source_type"`
SourceName string `json:"source_name"`
Stage string `json:"stage"`
Progress int `json:"progress"`
Message string `json:"message"`
Processed int `json:"processed"`
Total int `json:"total"`
}
type Progress struct {
Stage string `json:"stage"`
Progress int `json:"progress"`
Message string `json:"message"`
Processed int `json:"processed"`
Total int `json:"total"`
SourceName string `json:"source_name,omitempty"`
CurrentDirectory string `json:"current_directory,omitempty"`
ScannedDirectories int `json:"scanned_directories,omitempty"`
DiscoveredDirectories int `json:"discovered_directories,omitempty"`
ScannedFiles int `json:"scanned_files,omitempty"`
EstimatedTotalFiles int `json:"estimated_total_files,omitempty"`
TotalIsEstimate bool `json:"total_is_estimate,omitempty"`
}
type Store interface {
UpdateCatalogRefreshProgress(context.Context, string, string, Progress, time.Time) error
}
var progressForbiddenSubstrings = []string{"http://", "https://", "token", "secret", "password", "authorization", "bearer", "cookie", "/run/source-secrets", "api-key", "apikey"}
type Runner interface {
Run(context.Context, Request, func(Progress) error) error
}
type Config struct {
Store Store
Runner Runner
Now func() time.Time
}
type Handler struct {
store Store
runner Runner
now func() time.Time
}
func NewHandler(config Config) (*Handler, error) {
if config.Store == nil || config.Runner == nil {
return nil, errors.New("catalog refresh dependencies are required")
}
if config.Now == nil {
config.Now = time.Now
}
return &Handler{store: config.Store, runner: config.Runner, now: config.Now}, nil
}
func (handler *Handler) Handle(ctx context.Context, job jobs.Job) error {
var request Request
decoder := json.NewDecoder(bytes.NewReader(job.Payload))
decoder.DisallowUnknownFields()
if err := decoder.Decode(&request); err != nil || request.ConnectionID == "" || request.SourceName == "" || (request.SourceType != "pp" && request.SourceType != "dingtalk" && request.SourceType != "alist") || job.LeaseOwner == "" {
return errors.New("catalog_refresh_invalid_payload")
}
var last Progress
return handler.runner.Run(ctx, request, func(progress Progress) error {
progress = clampProgress(last, progress)
if !validProgress(progress) {
return errors.New("catalog_refresh_invalid_progress")
}
last = progress
return handler.store.UpdateCatalogRefreshProgress(ctx, job.ID, job.LeaseOwner, progress, handler.now().UTC())
})
}
func validProgress(progress Progress) bool {
if progress.Progress < 0 || progress.Progress > 100 || progress.Processed < 0 || progress.Total < 0 ||
progress.ScannedDirectories < 0 || progress.DiscoveredDirectories < 0 || progress.ScannedFiles < 0 || progress.EstimatedTotalFiles < 0 ||
strings.TrimSpace(progress.Stage) == "" || len(progress.Stage) > 32 ||
strings.TrimSpace(progress.Message) == "" || len(progress.Message) > 300 ||
len(progress.CurrentDirectory) > 1024 || len(progress.SourceName) > 255 {
return false
}
for _, text := range []string{progress.Stage, progress.Message, progress.CurrentDirectory, progress.SourceName} {
for _, r := range text {
if unicode.IsControl(r) && r != ' ' {
return false
}
}
lowered := strings.ToLower(text)
for _, forbidden := range progressForbiddenSubstrings {
if strings.Contains(lowered, forbidden) {
return false
}
}
}
return true
}
// clampProgress enforces that display progress never regresses when discovered
// totals shift mid-scan (BFS estimates grow), and never clamps an explicit
// failure snapshot.
func clampProgress(previous, next Progress) Progress {
if next.Stage == "failed" {
return next
}
if next.Total < previous.Total && previous.Total >= next.Processed {
next.Total = previous.Total
}
if next.Total > 0 {
estimated := next.Processed * 100 / next.Total
if next.Progress < previous.Progress && previous.Progress <= estimated {
next.Progress = previous.Progress
}
}
if next.ScannedDirectories < previous.ScannedDirectories {
next.ScannedDirectories = previous.ScannedDirectories
}
if next.ScannedFiles < previous.ScannedFiles {
next.ScannedFiles = previous.ScannedFiles
}
if next.DiscoveredDirectories < previous.DiscoveredDirectories {
next.DiscoveredDirectories = previous.DiscoveredDirectories
}
if next.EstimatedTotalFiles < previous.EstimatedTotalFiles {
next.EstimatedTotalFiles = previous.EstimatedTotalFiles
}
return next
}
type CommandRunner struct {
PythonPath string
ScriptPath string
}
func NewCommandRunner(pythonPath, scriptPath string) (*CommandRunner, error) {
if !filepath.IsAbs(pythonPath) || !filepath.IsAbs(scriptPath) {
return nil, errors.New("catalog refresh command paths must be absolute")
}
return &CommandRunner{PythonPath: pythonPath, ScriptPath: scriptPath}, nil
}
func (runner *CommandRunner) Run(ctx context.Context, request Request, report func(Progress) error) error {
command := exec.CommandContext(ctx, runner.PythonPath, runner.ScriptPath, "--connection-id", request.ConnectionID, "--source-type", request.SourceType, "--source-name", request.SourceName)
stdout, err := command.StdoutPipe()
if err != nil {
return errors.New("catalog_refresh_start_failed")
}
command.Stderr = io.Discard
if err = command.Start(); err != nil {
return errors.New("catalog_refresh_start_failed")
}
scanner := bufio.NewScanner(stdout)
scanner.Buffer(make([]byte, 4096), 1<<20)
for scanner.Scan() {
decoder := json.NewDecoder(bytes.NewReader(scanner.Bytes()))
decoder.DisallowUnknownFields()
var progress Progress
if decoder.Decode(&progress) != nil || !validProgress(progress) {
_ = command.Process.Kill()
_ = command.Wait()
return errors.New("catalog_refresh_invalid_output")
}
if err = report(progress); err != nil {
_ = command.Process.Kill()
_ = command.Wait()
return err
}
}
if scanner.Err() != nil {
_ = command.Process.Kill()
_ = command.Wait()
return errors.New("catalog_refresh_output_failed")
}
if err = command.Wait(); err != nil {
return errors.New("catalog_refresh_failed")
}
return nil
}
@@ -0,0 +1,126 @@
package catalogrefresh
import (
"context"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"time"
"knowledge-governance/internal/jobs"
)
type fakeStore struct{ updates []Progress }
func (s *fakeStore) UpdateCatalogRefreshProgress(_ context.Context, _ string, _ string, p Progress, _ time.Time) error {
s.updates = append(s.updates, p)
return nil
}
func TestHandlerClampsMonotonicProgressAndKeepsFailuresVisible(t *testing.T) {
store := &fakeStore{}
handler, err := NewHandler(Config{Store: store, Runner: fakeRunner{}, Now: func() time.Time { return time.Now() }})
if err != nil {
t.Fatal(err)
}
runner := func(ctx context.Context, _ Request, report func(Progress) error) error {
_ = report(Progress{Stage: "scanning", Progress: 40, Message: "扫描中", Processed: 60, Total: 100})
// Discovered more directories than estimated: total must not shrink, percent must not regress.
_ = report(Progress{Stage: "scanning", Progress: 30, Message: "发现更多目录", Processed: 70, Total: 50})
return report(Progress{Stage: "failed", Progress: 0, Message: "来源不可用"})
}
custom := *handler
custom.runner = runnerFunc(runner)
job := jobs.Job{ID: "job-1", LeaseOwner: "worker-1", Payload: []byte(`{"connection_id":"conn-1","source_type":"alist","source_name":"国内工业","stage":"queued","progress":0,"message":"等待后台刷新","processed":0,"total":0}`)}
if err = custom.Handle(context.Background(), job); err != nil {
t.Fatal(err)
}
if len(store.updates) != 3 {
t.Fatalf("updates=%#v", store.updates)
}
if store.updates[1].Progress < 40 || store.updates[1].Total < 70 {
t.Fatalf("regression not clamped: %#v", store.updates[1])
}
if store.updates[2].Stage != "failed" || store.updates[2].Progress != 0 {
t.Fatalf("failure must keep its own state: %#v", store.updates[2])
}
}
func TestValidProgressRejectsControlCharsURLsAndCredentials(t *testing.T) {
base := Progress{Stage: "scanning", Progress: 50, Message: "正在扫描", Processed: 10, Total: 20}
if !validProgress(base) {
t.Fatal("baseline rejected")
}
for name, mutate := range map[string]func(*Progress){
"control": func(p *Progress) { p.CurrentDirectory = "/产品\a资料" },
"url": func(p *Progress) { p.CurrentDirectory = "https://evil.invalid/x" },
"url-message": func(p *Progress) { p.Message = "访问 https://evil.invalid 失败" },
"token": func(p *Progress) { p.Message = "token=abc123 已过期" },
"too-long": func(p *Progress) { p.CurrentDirectory = strings.Repeat("目", 1100) },
"negative": func(p *Progress) { p.ScannedFiles = -1 },
"secret-dir": func(p *Progress) { p.CurrentDirectory = "/run/source-secrets/pp" },
} {
item := base
mutate(&item)
if validProgress(item) {
t.Errorf("%s accepted", name)
}
}
}
func TestCommandRunnerRejectsUnknownProgressFields(t *testing.T) {
python, err := exec.LookPath("python3")
if err != nil {
t.Skip("python3 unavailable")
}
runner, err := NewCommandRunner(python, writeFakeProgressScript(t))
if err != nil {
t.Fatal(err)
}
err = runner.Run(context.Background(), Request{ConnectionID: "c", SourceType: "pp", SourceName: "n"}, func(Progress) error { return nil })
if err == nil || err.Error() != "catalog_refresh_invalid_output" {
t.Fatalf("unknown field must fail: %v", err)
}
}
func writeFakeProgressScript(t *testing.T) string {
t.Helper()
directory := t.TempDir()
path := filepath.Join(directory, "progress.py")
body := "import json\nprint(json.dumps({'stage':'scanning','progress':10,'message':'ok','processed':1,'total':2,'unexpected_field':'leak'}),flush=True)\n"
if err := os.WriteFile(path, []byte(body), 0o644); err != nil {
t.Fatal(err)
}
return path
}
type runnerFunc func(context.Context, Request, func(Progress) error) error
func (f runnerFunc) Run(ctx context.Context, request Request, report func(Progress) error) error {
return f(ctx, request, report)
}
type fakeRunner struct{}
func (fakeRunner) Run(_ context.Context, _ Request, report func(Progress) error) error {
if err := report(Progress{Stage: "scanning", Progress: 50, Message: "正在扫描"}); err != nil {
return err
}
return report(Progress{Stage: "completed", Progress: 100, Message: "完成"})
}
func TestHandlerRunsLiveRefreshAndPersistsProgress(t *testing.T) {
store := &fakeStore{}
handler, err := NewHandler(Config{Store: store, Runner: fakeRunner{}, Now: func() time.Time { return time.Now() }})
if err != nil {
t.Fatal(err)
}
job := jobs.Job{ID: "job-1", LeaseOwner: "worker-1", Payload: []byte(`{"connection_id":"conn-1","source_type":"pp","source_name":"PP资料下载中心","stage":"queued","progress":0,"message":"等待后台刷新","processed":0,"total":0}`)}
if err = handler.Handle(context.Background(), job); err != nil {
t.Fatal(err)
}
if len(store.updates) != 2 || store.updates[0].Stage != "scanning" || store.updates[1].Progress != 100 {
t.Fatalf("updates=%#v", store.updates)
}
}
+299
View File
@@ -0,0 +1,299 @@
package release
import (
"context"
"encoding/json"
"errors"
"io"
"log"
"os"
"strings"
"time"
"knowledge-governance/internal/governance"
"knowledge-governance/internal/jobs"
)
// debugReleaseItemErrors enables fail-closed-safe diagnostics: only the
// generic step-level cause text is logged, never payload or credentials.
var debugReleaseItemErrors = os.Getenv("RELEASE_DEBUG_ITEM_ERRORS") == "true"
const (
ErrorCodeContractMismatch = "release_contract_mismatch"
ErrorCodeBundleInvalid = "release_bundle_invalid"
ErrorCodeTargetUnavailable = "release_target_unavailable"
ErrorCodeTargetWriteFailed = "release_target_write_failed"
ErrorCodeVerificationFailed = "release_verification_failed"
ErrorCodeRollbackArtifactMissing = "release_rollback_artifact_missing"
ErrorCodeRollbackFailed = "release_rollback_failed"
ErrorCodeStoppedAfterFailure = "release_stopped_after_failure"
ErrorCodeInternal = "release_internal_error"
)
var (
ErrInvalidReleaseJob = errors.New("invalid_release_job")
ErrReleaseRetryable = errors.New("release_retryable")
ErrReleaseState = errors.New("release_state_failed")
)
type PlanItem struct {
ReleaseExecutionID string `json:"release_execution_id"`
PublicationID string `json:"publication_id"`
Action string `json:"action"`
SourceItemVersionID string `json:"source_item_version_id"`
SourceRevision string `json:"source_revision"`
ParseRevision string `json:"parse_revision"`
ReleaseRevision string `json:"release_revision"`
TestKnowledgeBaseID string `json:"test_knowledge_base_id"`
KnowledgeID string `json:"knowledge_id"`
DecidedVisibility governance.Visibility `json:"decided_visibility"`
PreviousProductionRevision string `json:"previous_production_revision"`
}
type ItemResult struct {
BundleSHA256 string
PreviousProductionRevision string
ProductionRevision string
}
type StartExecutionRequest struct {
ExecutionID string
JobID string
Operation governance.ReleaseOperation
Now time.Time
}
type CompleteItemRequest struct {
ExecutionID string
PublicationID string
BundleSHA256 string
PreviousProductionRevision string
ProductionRevision string
Now time.Time
}
type FailItemRequest struct {
ExecutionID string
PublicationID string
ErrorCode string
Now time.Time
}
type Store interface {
StartExecution(context.Context, StartExecutionRequest) ([]PlanItem, error)
BeginBatch(context.Context, string, int, int, time.Time) error
CompleteItem(context.Context, CompleteItemRequest) error
FailItem(context.Context, FailItemRequest) error
FinishExecution(context.Context, string, time.Time) error
ScheduleAutomaticRollback(context.Context, string, time.Time) error
}
type Publisher interface {
Publish(context.Context, PlanItem) (ItemResult, error)
Delete(context.Context, PlanItem) (ItemResult, error)
Rollback(context.Context, PlanItem) (ItemResult, error)
}
type ItemError struct {
Code string
Retryable bool
Cause error
}
func (err *ItemError) Error() string {
if err == nil || !ValidErrorCode(err.Code) {
return ErrorCodeInternal
}
return err.Code
}
func (err *ItemError) Unwrap() error {
if err == nil {
return nil
}
return err.Cause
}
type jobPayload struct {
ExecutionID string `json:"execution_id"`
ReleaseBatchID string `json:"release_batch_id"`
Operation string `json:"operation"`
PriorExecutionID string `json:"prior_execution_id"`
}
func NewHandler(store Store, publisher Publisher, now func() time.Time) (jobs.Handler, error) {
if store == nil || publisher == nil || now == nil {
return nil, errors.New("release handler dependencies are required")
}
return func(ctx context.Context, job jobs.Job) error {
payload, operation, err := parseJob(job)
if err != nil {
return err
}
items, err := store.StartExecution(ctx, StartExecutionRequest{
ExecutionID: payload.ExecutionID, JobID: job.ID, Operation: operation, Now: now().UTC(),
})
if err != nil {
return ErrReleaseState
}
retryableFailure := false
permanentFailure := false
failureIndex := -1
const batchSize = 50
totalBatches := (len(items) + batchSize - 1) / batchSize
for start := 0; start < len(items); start += batchSize {
batchNumber := start/batchSize + 1
if err := store.BeginBatch(ctx, payload.ExecutionID, batchNumber, totalBatches, now().UTC()); err != nil {
return ErrReleaseState
}
end := start + batchSize
if end > len(items) {
end = len(items)
}
for index := start; index < end; index++ {
item := items[index]
result, itemErr := publishItem(ctx, publisher, operation, item)
if itemErr != nil {
code, retryable := classifyItemError(itemErr)
if debugReleaseItemErrors {
var itemErrDetail *ItemError
causeText := ""
if errors.As(itemErr, &itemErrDetail) && itemErrDetail != nil && itemErrDetail.Cause != nil {
causeText = itemErrDetail.Cause.Error()
}
log.Printf("release item %s (%s/%s) failed code=%s cause=%s", item.PublicationID, item.Action, item.KnowledgeID, code, causeText)
}
if retryable && job.Attempt < job.MaxAttempts {
retryableFailure = true
continue
}
if err := store.FailItem(ctx, FailItemRequest{ExecutionID: payload.ExecutionID, PublicationID: item.PublicationID, ErrorCode: code, Now: now().UTC()}); err != nil {
return ErrReleaseState
}
permanentFailure = true
failureIndex = index
break
}
if !validResult(operation, result) {
if err := store.FailItem(ctx, FailItemRequest{ExecutionID: payload.ExecutionID, PublicationID: item.PublicationID, ErrorCode: ErrorCodeBundleInvalid, Now: now().UTC()}); err != nil {
return ErrReleaseState
}
permanentFailure = true
failureIndex = index
break
}
if err := store.CompleteItem(ctx, CompleteItemRequest{
ExecutionID: payload.ExecutionID, PublicationID: item.PublicationID,
BundleSHA256: result.BundleSHA256, PreviousProductionRevision: result.PreviousProductionRevision,
ProductionRevision: result.ProductionRevision, Now: now().UTC(),
}); err != nil {
return ErrReleaseState
}
}
if permanentFailure {
break
}
}
if permanentFailure {
for _, item := range items[failureIndex+1:] {
if err := store.FailItem(ctx, FailItemRequest{ExecutionID: payload.ExecutionID, PublicationID: item.PublicationID, ErrorCode: ErrorCodeStoppedAfterFailure, Now: now().UTC()}); err != nil {
return ErrReleaseState
}
}
}
if retryableFailure {
return ErrReleaseRetryable
}
if err := store.FinishExecution(ctx, payload.ExecutionID, now().UTC()); err != nil {
return ErrReleaseState
}
if permanentFailure && operation == governance.ReleaseOperationPublish {
if err := store.ScheduleAutomaticRollback(ctx, payload.ExecutionID, now().UTC()); err != nil {
return ErrReleaseState
}
}
return nil
}, nil
}
func parseJob(job jobs.Job) (jobPayload, governance.ReleaseOperation, error) {
var payload jobPayload
decoder := json.NewDecoder(strings.NewReader(string(job.Payload)))
decoder.DisallowUnknownFields()
if err := decoder.Decode(&payload); err != nil {
return jobPayload{}, "", ErrInvalidReleaseJob
}
var trailing any
if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
return jobPayload{}, "", ErrInvalidReleaseJob
}
operation := governance.ReleaseOperation(payload.Operation)
expectedType := "release." + payload.Operation
if !validID(payload.ExecutionID, "rx-") || !validID(payload.ReleaseBatchID, "rb-") || !operation.Valid() || job.Type != expectedType || job.Attempt < 1 || job.MaxAttempts < job.Attempt {
return jobPayload{}, "", ErrInvalidReleaseJob
}
return payload, operation, nil
}
func publishItem(ctx context.Context, publisher Publisher, operation governance.ReleaseOperation, item PlanItem) (ItemResult, error) {
if operation == governance.ReleaseOperationRollback {
return publisher.Rollback(ctx, item)
}
if item.Action == "delete" {
return publisher.Delete(ctx, item)
}
return publisher.Publish(ctx, item)
}
func classifyItemError(err error) (string, bool) {
var itemErr *ItemError
if errors.As(err, &itemErr) && ValidErrorCode(itemErr.Code) {
return itemErr.Code, itemErr.Retryable
}
return ErrorCodeInternal, true
}
func validResult(operation governance.ReleaseOperation, result ItemResult) bool {
if strings.TrimSpace(result.ProductionRevision) == "" || len(result.ProductionRevision) > 128 {
return false
}
if operation == governance.ReleaseOperationPublish {
return validSHA256(result.BundleSHA256)
}
return true
}
func validSHA256(value string) bool {
if len(value) != 64 {
return false
}
for _, char := range value {
if (char < '0' || char > '9') && (char < 'a' || char > 'f') {
return false
}
}
return true
}
func validID(value, prefix string) bool {
if !strings.HasPrefix(value, prefix) || len(value) > 36 || len(value) <= len(prefix) {
return false
}
for _, char := range value {
if (char < 'a' || char > 'z') && (char < '0' || char > '9') && char != '-' {
return false
}
}
return true
}
func ValidErrorCode(code string) bool {
switch code {
case ErrorCodeContractMismatch, ErrorCodeBundleInvalid, ErrorCodeTargetUnavailable,
ErrorCodeTargetWriteFailed, ErrorCodeVerificationFailed, ErrorCodeRollbackArtifactMissing,
ErrorCodeRollbackFailed, ErrorCodeStoppedAfterFailure, ErrorCodeInternal:
return true
default:
return false
}
}
+209
View File
@@ -0,0 +1,209 @@
package release
import (
"context"
"errors"
"fmt"
"reflect"
"testing"
"time"
"knowledge-governance/internal/governance"
"knowledge-governance/internal/jobs"
)
func TestHandlerStopsAfterPermanentItemFailureAndSchedulesAutomaticRollback(t *testing.T) {
store := &fakeStore{items: []PlanItem{{PublicationID: "tp-1"}, {PublicationID: "tp-2"}, {PublicationID: "tp-3"}}}
publisher := &fakePublisher{publish: map[string]publishResult{
"tp-1": {result: ItemResult{BundleSHA256: repeatHex('a'), ProductionRevision: "prod-1"}},
"tp-2": {err: &ItemError{Code: ErrorCodeContractMismatch, Retryable: false}},
"tp-3": {result: ItemResult{BundleSHA256: repeatHex('b'), ProductionRevision: "prod-3"}},
}}
handler, err := NewHandler(store, publisher, func() time.Time { return time.Date(2026, 9, 6, 7, 0, 0, 0, time.UTC) })
if err != nil {
t.Fatal(err)
}
job := releaseJob("release.publish", 1, 3)
if err := handler(context.Background(), job); err != nil {
t.Fatal(err)
}
if len(store.completed) != 1 || store.completed[0].PublicationID != "tp-1" {
t.Fatalf("completed=%#v", store.completed)
}
if len(store.failed) != 2 || store.failed[0].ErrorCode != ErrorCodeContractMismatch || store.failed[1].PublicationID != "tp-3" || store.failed[1].ErrorCode != ErrorCodeStoppedAfterFailure {
t.Fatalf("failed=%#v", store.failed)
}
if store.finishedExecutionID != "rx-safe" || store.automaticRollbackExecutionID != "rx-safe" || publisher.publishCalls != 2 {
t.Fatalf("finished=%q rollback=%q publishCalls=%d", store.finishedExecutionID, store.automaticRollbackExecutionID, publisher.publishCalls)
}
}
func TestHandlerPersistsProgressAtFiftyItemBatchBoundaries(t *testing.T) {
items := make([]PlanItem, 0, 105)
results := make(map[string]publishResult, 105)
for index := 0; index < 105; index++ {
id := fmt.Sprintf("tp-%03d", index)
items = append(items, PlanItem{PublicationID: id})
results[id] = publishResult{result: ItemResult{BundleSHA256: repeatHex('a'), ProductionRevision: "prod-1"}}
}
store := &fakeStore{items: items}
handler, err := NewHandler(store, &fakePublisher{publish: results}, time.Now)
if err != nil {
t.Fatal(err)
}
if err := handler(context.Background(), releaseJob("release.publish", 1, 3)); err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(store.startedBatches, [][2]int{{1, 3}, {2, 3}, {3, 3}}) {
t.Fatalf("started batches=%v", store.startedBatches)
}
}
func TestHandlerPublishesDeleteActionAsVerifiedAbsentRevision(t *testing.T) {
store := &fakeStore{items: []PlanItem{{PublicationID: "tp-delete", Action: "delete", PreviousProductionRevision: "prod-current"}}}
publisher := &fakePublisher{delete: map[string]publishResult{
"tp-delete": {result: ItemResult{BundleSHA256: repeatHex('d'), PreviousProductionRevision: "prod-current", ProductionRevision: "absent"}},
}}
handler, err := NewHandler(store, publisher, func() time.Time { return time.Date(2026, 9, 10, 4, 0, 0, 0, time.UTC) })
if err != nil {
t.Fatal(err)
}
if err := handler(context.Background(), jobs.Job{ID: "job-delete", Type: "release.publish", Payload: []byte(`{"execution_id":"rx-delete","release_batch_id":"rb-delete","operation":"publish"}`), Attempt: 1, MaxAttempts: 3}); err != nil {
t.Fatal(err)
}
if publisher.deleteCalls != 1 || publisher.publishCalls != 0 || len(store.completed) != 1 || store.completed[0].ProductionRevision != "absent" {
t.Fatalf("deleteCalls=%d publishCalls=%d completed=%#v", publisher.deleteCalls, publisher.publishCalls, store.completed)
}
}
func TestHandlerReturnsOnlySanitizedRetryErrorBeforeLastAttempt(t *testing.T) {
store := &fakeStore{items: []PlanItem{{PublicationID: "tp-1"}}}
publisher := &fakePublisher{publish: map[string]publishResult{
"tp-1": {err: &ItemError{Code: ErrorCodeTargetUnavailable, Retryable: true, Cause: errors.New("raw endpoint and credential text")}},
}}
handler, _ := NewHandler(store, publisher, time.Now)
err := handler(context.Background(), releaseJob("release.publish", 1, 3))
if !errors.Is(err, ErrReleaseRetryable) || err.Error() != ErrReleaseRetryable.Error() {
t.Fatalf("error=%v", err)
}
if len(store.failed) != 0 || store.finishedExecutionID != "" {
t.Fatalf("failed=%#v finished=%q", store.failed, store.finishedExecutionID)
}
}
func TestHandlerFailsRetryableItemsWithFixedCodeOnLastAttempt(t *testing.T) {
store := &fakeStore{items: []PlanItem{{PublicationID: "tp-1"}}}
publisher := &fakePublisher{publish: map[string]publishResult{
"tp-1": {err: &ItemError{Code: ErrorCodeTargetUnavailable, Retryable: true}},
}}
handler, _ := NewHandler(store, publisher, time.Now)
if err := handler(context.Background(), releaseJob("release.publish", 3, 3)); err != nil {
t.Fatal(err)
}
if len(store.failed) != 1 || store.failed[0].ErrorCode != ErrorCodeTargetUnavailable || store.finishedExecutionID != "rx-safe" {
t.Fatalf("failed=%#v finished=%q", store.failed, store.finishedExecutionID)
}
}
func TestHandlerDispatchesRollbackAndRejectsInvalidPayload(t *testing.T) {
store := &fakeStore{items: []PlanItem{{PublicationID: "tp-1"}}}
publisher := &fakePublisher{rollback: map[string]publishResult{"tp-1": {result: ItemResult{ProductionRevision: "prod-old"}}}}
handler, _ := NewHandler(store, publisher, time.Now)
if err := handler(context.Background(), releaseJob("release.rollback", 1, 3)); err != nil {
t.Fatal(err)
}
if publisher.rollbackCalls != 1 || publisher.publishCalls != 0 {
t.Fatalf("publish=%d rollback=%d", publisher.publishCalls, publisher.rollbackCalls)
}
bad := releaseJob("release.publish", 1, 3)
bad.Payload = []byte(`{"execution_id":"../bad"}`)
if err := handler(context.Background(), bad); !errors.Is(err, ErrInvalidReleaseJob) {
t.Fatalf("error=%v", err)
}
// Production rollback payloads carry prior_execution_id; the strict
// decoder must accept it (regression: release jobs failed permanently as
// invalid_release_job the moment rollback executions included the field).
withPrior := releaseJob("release.rollback", 1, 3)
withPrior.Payload = []byte(`{"execution_id":"rx-safe","release_batch_id":"rb-safe","operation":"rollback","prior_execution_id":"rx-prior"}`)
if err := handler(context.Background(), withPrior); err != nil {
t.Fatalf("rollback payload with prior_execution_id rejected: %v", err)
}
}
func releaseJob(jobType string, attempt, maxAttempts int) jobs.Job {
payload := []byte(`{"execution_id":"rx-safe","release_batch_id":"rb-safe","operation":"publish"}`)
if jobType == "release.rollback" {
payload = []byte(`{"execution_id":"rx-safe","release_batch_id":"rb-safe","operation":"rollback"}`)
}
return jobs.Job{ID: "job-safe", Type: jobType, Payload: payload, Attempt: attempt, MaxAttempts: maxAttempts}
}
type fakeStore struct {
items []PlanItem
completed []CompleteItemRequest
failed []FailItemRequest
finishedExecutionID string
automaticRollbackExecutionID string
startedBatches [][2]int
}
func (s *fakeStore) StartExecution(context.Context, StartExecutionRequest) ([]PlanItem, error) {
return s.items, nil
}
func (s *fakeStore) CompleteItem(_ context.Context, request CompleteItemRequest) error {
s.completed = append(s.completed, request)
return nil
}
func (s *fakeStore) FailItem(_ context.Context, request FailItemRequest) error {
s.failed = append(s.failed, request)
return nil
}
func (s *fakeStore) FinishExecution(_ context.Context, executionID string, _ time.Time) error {
s.finishedExecutionID = executionID
return nil
}
func (s *fakeStore) ScheduleAutomaticRollback(_ context.Context, executionID string, _ time.Time) error {
s.automaticRollbackExecutionID = executionID
return nil
}
func (s *fakeStore) BeginBatch(_ context.Context, _ string, batch, total int, _ time.Time) error {
s.startedBatches = append(s.startedBatches, [2]int{batch, total})
return nil
}
type publishResult struct {
result ItemResult
err error
}
type fakePublisher struct {
publish, rollback, delete map[string]publishResult
publishCalls, rollbackCalls, deleteCalls int
}
func (p *fakePublisher) Publish(_ context.Context, item PlanItem) (ItemResult, error) {
p.publishCalls++
value := p.publish[item.PublicationID]
return value.result, value.err
}
func (p *fakePublisher) Rollback(_ context.Context, item PlanItem) (ItemResult, error) {
p.rollbackCalls++
value := p.rollback[item.PublicationID]
return value.result, value.err
}
func (p *fakePublisher) Delete(_ context.Context, item PlanItem) (ItemResult, error) {
p.deleteCalls++
value := p.delete[item.PublicationID]
return value.result, value.err
}
func repeatHex(value byte) string {
result := make([]byte, 64)
for index := range result {
result[index] = value
}
return string(result)
}
var _ = governance.ReleaseOperationPublish
@@ -0,0 +1,65 @@
package release
import (
"context"
"errors"
"strings"
)
type RoutingPublisher struct {
publishers map[string]Publisher
}
func NewRoutingPublisher(publishers map[string]Publisher) (*RoutingPublisher, error) {
if len(publishers) == 0 {
return nil, errors.New("release publisher routes are required")
}
copied := make(map[string]Publisher, len(publishers))
for knowledgeBaseID, publisher := range publishers {
if !validRoutingID(knowledgeBaseID) || publisher == nil {
return nil, errors.New("invalid release publisher route")
}
copied[knowledgeBaseID] = publisher
}
return &RoutingPublisher{publishers: copied}, nil
}
func (router *RoutingPublisher) Publish(ctx context.Context, item PlanItem) (ItemResult, error) {
publisher, ok := router.publishers[item.TestKnowledgeBaseID]
if !ok {
return ItemResult{}, &ItemError{Code: ErrorCodeContractMismatch, Retryable: false}
}
return publisher.Publish(ctx, item)
}
func (router *RoutingPublisher) Rollback(ctx context.Context, item PlanItem) (ItemResult, error) {
publisher, ok := router.publishers[item.TestKnowledgeBaseID]
if !ok {
return ItemResult{}, &ItemError{Code: ErrorCodeContractMismatch, Retryable: false}
}
return publisher.Rollback(ctx, item)
}
func (router *RoutingPublisher) Delete(ctx context.Context, item PlanItem) (ItemResult, error) {
publisher, ok := router.publishers[item.TestKnowledgeBaseID]
if !ok {
return ItemResult{}, &ItemError{Code: ErrorCodeContractMismatch, Retryable: false}
}
return publisher.Delete(ctx, item)
}
func validRoutingID(value string) bool {
value = strings.TrimSpace(value)
if value == "" || len(value) > 128 {
return false
}
for _, char := range value {
if (char >= 'a' && char <= 'z') || (char >= 'A' && char <= 'Z') || (char >= '0' && char <= '9') || char == '-' || char == '_' || char == '.' {
continue
}
return false
}
return true
}
var _ Publisher = (*RoutingPublisher)(nil)
@@ -0,0 +1,52 @@
package release
import (
"context"
"testing"
)
func TestRoutingPublisherSelectsExactKnowledgeBase(t *testing.T) {
first := &routingPublisherStub{result: ItemResult{ProductionRevision: "prod-a"}}
second := &routingPublisherStub{result: ItemResult{ProductionRevision: "prod-b"}}
router, err := NewRoutingPublisher(map[string]Publisher{"kb-a": first, "kb-b": second})
if err != nil {
t.Fatal(err)
}
item := testPlanItem()
item.TestKnowledgeBaseID = "kb-b"
result, err := router.Publish(context.Background(), item)
if err != nil {
t.Fatal(err)
}
if result.ProductionRevision != "prod-b" || first.calls != 0 || second.calls != 1 {
t.Fatalf("result=%#v calls=%d/%d", result, first.calls, second.calls)
}
if _, err := router.Delete(context.Background(), item); err != nil || first.deleteCalls != 0 || second.deleteCalls != 1 {
t.Fatalf("delete error=%v deleteCalls=%d/%d", err, first.deleteCalls, second.deleteCalls)
}
item.TestKnowledgeBaseID = "unknown"
if _, err := router.Publish(context.Background(), item); err == nil || err.Error() != ErrorCodeContractMismatch {
t.Fatalf("error=%v", err)
}
}
type routingPublisherStub struct {
result ItemResult
err error
calls int
deleteCalls int
}
func (s *routingPublisherStub) Publish(context.Context, PlanItem) (ItemResult, error) {
s.calls++
return s.result, s.err
}
func (s *routingPublisherStub) Rollback(context.Context, PlanItem) (ItemResult, error) {
s.calls++
return s.result, s.err
}
func (s *routingPublisherStub) Delete(context.Context, PlanItem) (ItemResult, error) {
s.deleteCalls++
return s.result, s.err
}
@@ -0,0 +1,162 @@
package release
import (
"context"
"errors"
"strings"
"knowledge-governance/internal/releasebundle"
)
type BundleExporter interface {
Export(ctx context.Context, item PlanItem) (ExportedBundle, error)
}
type ExportedBundle interface {
ReleaseManifest() releasebundle.Manifest
}
type PreparedChange struct {
PreviousRevision string
ProductionRevision string
}
type SidecarTarget interface {
Contract(ctx context.Context, item PlanItem) (releasebundle.TargetContract, error)
Prepare(ctx context.Context, item PlanItem, bundle ExportedBundle) (PreparedChange, error)
Activate(ctx context.Context, item PlanItem, change PreparedChange) error
Verify(ctx context.Context, item PlanItem, change PreparedChange) error
Compensate(ctx context.Context, item PlanItem, change PreparedChange) error
Restore(ctx context.Context, item PlanItem, revision string) (string, error)
VerifyRestored(ctx context.Context, item PlanItem, revision string) error
}
type SidecarPublisher struct {
exporter BundleExporter
target SidecarTarget
}
func NewSidecarPublisher(exporter BundleExporter, target SidecarTarget) (*SidecarPublisher, error) {
if exporter == nil || target == nil {
return nil, errors.New("release sidecar dependencies are required")
}
return &SidecarPublisher{exporter: exporter, target: target}, nil
}
func (publisher *SidecarPublisher) Publish(ctx context.Context, item PlanItem) (ItemResult, error) {
bundle, err := publisher.exporter.Export(ctx, item)
if err != nil {
return ItemResult{}, newSidecarError(ErrorCodeTargetUnavailable, true, err)
}
if bundle == nil {
return ItemResult{}, newSidecarError(ErrorCodeBundleInvalid, false, errors.New("empty release bundle"))
}
manifest := bundle.ReleaseManifest()
if manifest.ExecutionID != item.ReleaseExecutionID || manifest.PublicationID != item.PublicationID ||
manifest.SourceItemVersionID != item.SourceItemVersionID || manifest.SourceRevision != item.SourceRevision ||
manifest.ParseRevision != item.ParseRevision || manifest.ReleaseRevision != item.ReleaseRevision ||
manifest.Contract.KnowledgeBaseID != item.TestKnowledgeBaseID || manifest.Contract.KnowledgeID != item.KnowledgeID {
return ItemResult{}, newSidecarError(ErrorCodeContractMismatch, false, errors.New("bundle identity mismatch"))
}
targetContract, err := publisher.target.Contract(ctx, item)
if err != nil {
return ItemResult{}, newSidecarError(ErrorCodeTargetUnavailable, true, err)
}
if err := releasebundle.Validate(manifest, targetContract); err != nil {
return ItemResult{}, newSidecarError(ErrorCodeContractMismatch, false, err)
}
change, err := publisher.target.Prepare(ctx, item, bundle)
if err != nil {
return ItemResult{}, newSidecarError(ErrorCodeTargetWriteFailed, true, err)
}
if !validRevision(change.ProductionRevision) || (change.PreviousRevision != "" && !validRevision(change.PreviousRevision)) {
return ItemResult{}, publisher.compensateError(ctx, item, change, ErrorCodeBundleInvalid, errors.New("invalid prepared revision"))
}
if err := publisher.target.Activate(ctx, item, change); err != nil {
return ItemResult{}, publisher.compensateError(ctx, item, change, ErrorCodeTargetWriteFailed, err)
}
if err := publisher.target.Verify(ctx, item, change); err != nil {
return ItemResult{}, publisher.compensateError(ctx, item, change, ErrorCodeVerificationFailed, err)
}
return ItemResult{
BundleSHA256: manifest.BundleSHA256,
PreviousProductionRevision: change.PreviousRevision,
ProductionRevision: change.ProductionRevision,
}, nil
}
func (publisher *SidecarPublisher) Rollback(ctx context.Context, item PlanItem) (ItemResult, error) {
if !validRevision(item.PreviousProductionRevision) {
return ItemResult{}, newSidecarError(ErrorCodeRollbackArtifactMissing, false, errors.New("missing rollback revision"))
}
restoredRevision, err := publisher.target.Restore(ctx, item, item.PreviousProductionRevision)
if err != nil {
return ItemResult{}, newSidecarError(ErrorCodeTargetWriteFailed, true, err)
}
if restoredRevision != item.PreviousProductionRevision {
return ItemResult{}, newSidecarError(ErrorCodeContractMismatch, false, errors.New("restored revision mismatch"))
}
if err := publisher.target.VerifyRestored(ctx, item, restoredRevision); err != nil {
return ItemResult{}, newSidecarError(ErrorCodeVerificationFailed, false, err)
}
return ItemResult{ProductionRevision: restoredRevision}, nil
}
func (publisher *SidecarPublisher) Delete(ctx context.Context, item PlanItem) (ItemResult, error) {
bundle, err := publisher.exporter.Export(ctx, item)
if err != nil {
return ItemResult{}, newSidecarError(ErrorCodeTargetUnavailable, true, err)
}
if bundle == nil {
return ItemResult{}, newSidecarError(ErrorCodeBundleInvalid, false, errors.New("empty release bundle"))
}
manifest := bundle.ReleaseManifest()
if manifest.ExecutionID != item.ReleaseExecutionID || manifest.PublicationID != item.PublicationID ||
manifest.SourceItemVersionID != item.SourceItemVersionID || manifest.SourceRevision != item.SourceRevision ||
manifest.ParseRevision != item.ParseRevision || manifest.ReleaseRevision != item.ReleaseRevision ||
manifest.Contract.KnowledgeBaseID != item.TestKnowledgeBaseID || manifest.Contract.KnowledgeID != item.KnowledgeID {
return ItemResult{}, newSidecarError(ErrorCodeContractMismatch, false, errors.New("bundle identity mismatch"))
}
targetContract, err := publisher.target.Contract(ctx, item)
if err != nil {
return ItemResult{}, newSidecarError(ErrorCodeTargetUnavailable, true, err)
}
if err := releasebundle.Validate(manifest, targetContract); err != nil {
return ItemResult{}, newSidecarError(ErrorCodeContractMismatch, false, err)
}
change, err := publisher.target.Prepare(ctx, item, bundle)
if err != nil {
return ItemResult{}, newSidecarError(ErrorCodeTargetWriteFailed, true, err)
}
if !validRevision(change.PreviousRevision) || change.ProductionRevision != item.ReleaseRevision {
return ItemResult{}, publisher.compensateError(ctx, item, change, ErrorCodeBundleInvalid, errors.New("invalid delete snapshot revision"))
}
restoredRevision, err := publisher.target.Restore(ctx, item, "absent")
if err != nil {
return ItemResult{}, publisher.compensateError(ctx, item, change, ErrorCodeTargetWriteFailed, err)
}
if restoredRevision != "absent" {
return ItemResult{}, publisher.compensateError(ctx, item, change, ErrorCodeContractMismatch, errors.New("deleted revision mismatch"))
}
if err := publisher.target.VerifyRestored(ctx, item, "absent"); err != nil {
return ItemResult{}, publisher.compensateError(ctx, item, change, ErrorCodeVerificationFailed, err)
}
return ItemResult{BundleSHA256: manifest.BundleSHA256, PreviousProductionRevision: change.PreviousRevision, ProductionRevision: "absent"}, nil
}
func (publisher *SidecarPublisher) compensateError(ctx context.Context, item PlanItem, change PreparedChange, code string, cause error) error {
if err := publisher.target.Compensate(ctx, item, change); err != nil {
return newSidecarError(ErrorCodeRollbackFailed, false, err)
}
return newSidecarError(code, false, cause)
}
func newSidecarError(code string, retryable bool, cause error) error {
return &ItemError{Code: code, Retryable: retryable, Cause: cause}
}
func validRevision(value string) bool {
return value == strings.TrimSpace(value) && value != "" && len(value) <= 128 && !strings.ContainsAny(value, "\x00\r\n\t/?#\\")
}
var _ Publisher = (*SidecarPublisher)(nil)
@@ -0,0 +1,217 @@
package release
import (
"context"
"errors"
"reflect"
"strings"
"testing"
"time"
"knowledge-governance/internal/releasebundle"
)
func TestSidecarPublisherPublishesWithContractAndVerification(t *testing.T) {
calls := []string{}
manifest, err := releasebundle.Seal(testManifest())
if err != nil {
t.Fatal(err)
}
exporter := &fakeExporter{manifest: manifest, calls: &calls}
target := &fakeSidecarTarget{contract: testTargetContract(), calls: &calls, prepared: PreparedChange{PreviousRevision: "prod-v0", ProductionRevision: "prod-v1"}}
publisher, err := NewSidecarPublisher(exporter, target)
if err != nil {
t.Fatal(err)
}
result, err := publisher.Publish(context.Background(), testPlanItem())
if err != nil {
t.Fatal(err)
}
if result.BundleSHA256 != manifest.BundleSHA256 || result.PreviousProductionRevision != "prod-v0" || result.ProductionRevision != "prod-v1" {
t.Fatalf("result=%#v", result)
}
want := []string{"export", "contract", "prepare", "activate", "verify"}
if !reflect.DeepEqual(calls, want) {
t.Fatalf("calls=%v want=%v", calls, want)
}
}
func TestSidecarPublisherRollsBackFailedVerificationWithoutLeakingCause(t *testing.T) {
calls := []string{}
manifest, err := releasebundle.Seal(testManifest())
if err != nil {
t.Fatal(err)
}
target := &fakeSidecarTarget{contract: testTargetContract(), calls: &calls, prepared: PreparedChange{PreviousRevision: "prod-v0", ProductionRevision: "prod-v1"}, verifyErr: errors.New("signed-url-secret-should-not-leak")}
publisher, err := NewSidecarPublisher(&fakeExporter{manifest: manifest, calls: &calls}, target)
if err != nil {
t.Fatal(err)
}
_, err = publisher.Publish(context.Background(), testPlanItem())
var itemErr *ItemError
if !errors.As(err, &itemErr) || itemErr.Code != ErrorCodeVerificationFailed || itemErr.Retryable {
t.Fatalf("error=%#v", err)
}
if strings.Contains(err.Error(), "signed-url") {
t.Fatalf("error leaked cause: %v", err)
}
want := []string{"export", "contract", "prepare", "activate", "verify", "compensate"}
if !reflect.DeepEqual(calls, want) {
t.Fatalf("calls=%v want=%v", calls, want)
}
}
func TestSidecarPublisherRejectsContractDriftBeforeTargetMutation(t *testing.T) {
calls := []string{}
manifest, err := releasebundle.Seal(testManifest())
if err != nil {
t.Fatal(err)
}
contract := testTargetContract()
contract.EmbeddingDimension = 768
publisher, err := NewSidecarPublisher(&fakeExporter{manifest: manifest, calls: &calls}, &fakeSidecarTarget{contract: contract, calls: &calls})
if err != nil {
t.Fatal(err)
}
_, err = publisher.Publish(context.Background(), testPlanItem())
var itemErr *ItemError
if !errors.As(err, &itemErr) || itemErr.Code != ErrorCodeContractMismatch {
t.Fatalf("error=%#v", err)
}
if !reflect.DeepEqual(calls, []string{"export", "contract"}) {
t.Fatalf("calls=%v", calls)
}
}
func TestSidecarPublisherRejectsRevisionMismatchBeforeTargetMutation(t *testing.T) {
calls := []string{}
manifest, err := releasebundle.Seal(testManifest())
if err != nil {
t.Fatal(err)
}
item := testPlanItem()
item.ParseRevision = "other-parser/index"
publisher, err := NewSidecarPublisher(&fakeExporter{manifest: manifest, calls: &calls}, &fakeSidecarTarget{contract: testTargetContract(), calls: &calls})
if err != nil {
t.Fatal(err)
}
_, err = publisher.Publish(context.Background(), item)
var itemErr *ItemError
if !errors.As(err, &itemErr) || itemErr.Code != ErrorCodeContractMismatch {
t.Fatalf("error=%#v", err)
}
if !reflect.DeepEqual(calls, []string{"export"}) {
t.Fatalf("calls=%v", calls)
}
}
func TestSidecarPublisherDeletesOnlyAfterArchivingAndVerifiesAbsent(t *testing.T) {
calls := []string{}
manifest, err := releasebundle.Seal(testManifest())
if err != nil {
t.Fatal(err)
}
target := &fakeSidecarTarget{contract: testTargetContract(), calls: &calls, prepared: PreparedChange{PreviousRevision: "prod-v0", ProductionRevision: "release-v1"}, expectedRestoreRevision: "absent", restoredRevision: "absent"}
publisher, err := NewSidecarPublisher(&fakeExporter{manifest: manifest, calls: &calls}, target)
if err != nil {
t.Fatal(err)
}
result, err := publisher.Delete(context.Background(), testPlanItem())
if err != nil {
t.Fatal(err)
}
if result.BundleSHA256 != manifest.BundleSHA256 || result.PreviousProductionRevision != "prod-v0" || result.ProductionRevision != "absent" {
t.Fatalf("result=%#v", result)
}
if want := []string{"export", "contract", "prepare", "restore", "verify-restored"}; !reflect.DeepEqual(calls, want) {
t.Fatalf("calls=%v want=%v", calls, want)
}
}
func TestSidecarPublisherRestoresArchivedRevision(t *testing.T) {
calls := []string{}
target := &fakeSidecarTarget{calls: &calls, restoredRevision: "prod-v0"}
publisher, err := NewSidecarPublisher(&fakeExporter{}, target)
if err != nil {
t.Fatal(err)
}
result, err := publisher.Rollback(context.Background(), testPlanItem())
if err != nil {
t.Fatal(err)
}
if result.ProductionRevision != "prod-v0" || !reflect.DeepEqual(calls, []string{"restore", "verify-restored"}) {
t.Fatalf("result=%#v calls=%v", result, calls)
}
}
type fakeExporter struct {
manifest releasebundle.Manifest
calls *[]string
err error
}
func (f *fakeExporter) Export(context.Context, PlanItem) (ExportedBundle, error) {
if f.calls != nil {
*f.calls = append(*f.calls, "export")
}
return fakeExportedBundle{manifest: f.manifest}, f.err
}
type fakeExportedBundle struct{ manifest releasebundle.Manifest }
func (bundle fakeExportedBundle) ReleaseManifest() releasebundle.Manifest { return bundle.manifest }
type fakeSidecarTarget struct {
contract releasebundle.TargetContract
calls *[]string
prepared PreparedChange
prepareErr, activateErr, verifyErr, compensateErr, restoreErr, verifyRestoredErr error
restoredRevision string
expectedRestoreRevision string
}
func (f *fakeSidecarTarget) Contract(context.Context, PlanItem) (releasebundle.TargetContract, error) {
*f.calls = append(*f.calls, "contract")
return f.contract, nil
}
func (f *fakeSidecarTarget) Prepare(context.Context, PlanItem, ExportedBundle) (PreparedChange, error) {
*f.calls = append(*f.calls, "prepare")
return f.prepared, f.prepareErr
}
func (f *fakeSidecarTarget) Activate(context.Context, PlanItem, PreparedChange) error {
*f.calls = append(*f.calls, "activate")
return f.activateErr
}
func (f *fakeSidecarTarget) Verify(context.Context, PlanItem, PreparedChange) error {
*f.calls = append(*f.calls, "verify")
return f.verifyErr
}
func (f *fakeSidecarTarget) Compensate(context.Context, PlanItem, PreparedChange) error {
*f.calls = append(*f.calls, "compensate")
return f.compensateErr
}
func (f *fakeSidecarTarget) Restore(_ context.Context, _ PlanItem, revision string) (string, error) {
*f.calls = append(*f.calls, "restore")
expected := f.expectedRestoreRevision
if expected == "" {
expected = "prod-v0"
}
if revision != expected {
return "", errors.New("wrong revision")
}
return f.restoredRevision, f.restoreErr
}
func (f *fakeSidecarTarget) VerifyRestored(context.Context, PlanItem, string) error {
*f.calls = append(*f.calls, "verify-restored")
return f.verifyRestoredErr
}
func testPlanItem() PlanItem {
return PlanItem{ReleaseExecutionID: "rx-safe", PublicationID: "tp-safe", SourceItemVersionID: "sv-safe", SourceRevision: "source-v1", ParseRevision: "parser-v1@index-v1", ReleaseRevision: "release-v1", TestKnowledgeBaseID: "kb-safe", KnowledgeID: "knowledge-safe", DecidedVisibility: "internal", PreviousProductionRevision: "prod-v0"}
}
func testManifest() releasebundle.Manifest {
return releasebundle.Manifest{FormatVersion: releasebundle.FormatVersionV1, ExecutionID: "rx-safe", PublicationID: "tp-safe", SourceItemVersionID: "sv-safe", SourceRevision: "source-v1", ParseRevision: "parser-v1@index-v1", ReleaseRevision: "release-v1", CreatedAt: time.Date(2026, 9, 6, 8, 0, 0, 0, time.UTC), Contract: releasebundle.Contract{WeKnoraVersion: releasebundle.WeKnoraVersionV071, MigrationVersion: releasebundle.WeKnoraMigrationV071, TenantID: 10000, KnowledgeBaseID: "kb-safe", KnowledgeID: "knowledge-safe", RetrieverEngine: "elasticsearch_v8", EmbeddingModel: "bge-m3", EmbeddingDimension: releasebundle.EmbeddingDimensionV071, ElasticsearchMappingSHA256: strings.Repeat("a", 64)}, Tables: []releasebundle.TableDigest{{Name: "knowledges", RowCount: 1, SHA256: strings.Repeat("b", 64)}, {Name: "chunks", RowCount: 1, SHA256: strings.Repeat("c", 64)}}, Search: releasebundle.SearchDigest{DocumentCount: 1, SHA256: strings.Repeat("d", 64)}, Assets: releasebundle.AssetDigest{SHA256: strings.Repeat("e", 64)}}
}
func testTargetContract() releasebundle.TargetContract {
return releasebundle.TargetContract{WeKnoraVersion: releasebundle.WeKnoraVersionV071, MigrationVersion: releasebundle.WeKnoraMigrationV071, TenantID: 10000, KnowledgeBaseID: "kb-safe", RetrieverEngine: "elasticsearch_v8", EmbeddingModel: "bge-m3", EmbeddingDimension: releasebundle.EmbeddingDimensionV071, ElasticsearchMappingSHA256: strings.Repeat("a", 64)}
}
@@ -0,0 +1,194 @@
package sourcescan
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"path/filepath"
ppconnector "knowledge-governance/internal/connectors/pp"
)
const maxPPMetadataBytes int64 = 64 * 1024 * 1024
type FilePPScannerConfig struct {
AllowedRoot string
LedgerPath string
ManifestPath string
}
type FilePPScanner struct {
root string
ledgerPath string
manifestPath string
}
func NewFilePPScanner(config FilePPScannerConfig) (*FilePPScanner, error) {
root, err := filepath.Abs(config.AllowedRoot)
if err != nil || config.AllowedRoot == "" {
return nil, errors.New("PP scan allowed root is required")
}
resolvedRoot, err := filepath.EvalSymlinks(root)
if err != nil || resolvedRoot != root {
return nil, errors.New("PP scan allowed root must be a canonical directory")
}
info, err := os.Lstat(root)
if err != nil || !info.IsDir() || info.Mode()&os.ModeSymlink != 0 {
return nil, errors.New("PP scan allowed root must be a canonical directory")
}
if config.LedgerPath == "" || config.ManifestPath == "" {
return nil, errors.New("PP ledger and manifest paths are required")
}
return &FilePPScanner{root: root, ledgerPath: config.LedgerPath, manifestPath: config.ManifestPath}, nil
}
func (s *FilePPScanner) Scan(ctx context.Context) (ppconnector.Snapshot, string, error) {
if err := ctx.Err(); err != nil {
return ppconnector.Snapshot{}, "", err
}
ledger, err := s.readFile(s.ledgerPath, maxPPMetadataBytes)
if err != nil {
return ppconnector.Snapshot{}, "", fmt.Errorf("read PP ledger: %w", err)
}
manifest, err := s.readFile(s.manifestPath, maxPPMetadataBytes)
if err != nil {
return ppconnector.Snapshot{}, "", fmt.Errorf("read PP manifest: %w", err)
}
if err := s.validateManifestPaths(manifest); err != nil {
return ppconnector.Snapshot{}, "", err
}
snapshot, err := ppconnector.FoldLedger(bytes.NewReader(ledger))
if err != nil {
return ppconnector.Snapshot{}, "", err
}
snapshot, _, err = ppconnector.RecoverMissingSHA256(snapshot, bytes.NewReader(manifest))
if err != nil {
return ppconnector.Snapshot{}, "", err
}
snapshot, _, err = ppconnector.MergeOverLimitManifest(snapshot, bytes.NewReader(manifest))
if err != nil {
return ppconnector.Snapshot{}, "", err
}
if err := ppconnector.ValidateSnapshot(snapshot); err != nil {
return ppconnector.Snapshot{}, "", err
}
digest := sha256.New()
_, _ = digest.Write(ledger)
_, _ = digest.Write([]byte{0})
_, _ = digest.Write(manifest)
return snapshot, hex.EncodeToString(digest.Sum(nil)), nil
}
func (s *FilePPScanner) validateManifestPaths(body []byte) error {
var manifest struct {
Results []struct {
LocalPath string `json:"local_path"`
Status string `json:"status"`
} `json:"results"`
}
if err := json.Unmarshal(body, &manifest); err != nil {
return fmt.Errorf("decode PP manifest paths: %w", err)
}
for _, item := range manifest.Results {
if item.LocalPath == "" {
return errors.New("PP manifest artifact local_path is required")
}
if item.Status == string(ppconnector.StatusOverLimit) {
if err := s.validateDeclaredPath(item.LocalPath); err != nil {
return fmt.Errorf("validate PP over-limit artifact path: %w", err)
}
file, err := s.openRegular(item.LocalPath)
if errors.Is(err, os.ErrNotExist) {
continue
}
if err != nil {
return fmt.Errorf("validate PP over-limit artifact: %w", err)
}
if err := file.Close(); err != nil {
return errors.New("close PP over-limit artifact")
}
continue
}
file, err := s.openRegular(item.LocalPath)
if err != nil {
return fmt.Errorf("validate PP manifest artifact: %w", err)
}
if err := file.Close(); err != nil {
return errors.New("close PP manifest artifact")
}
}
return nil
}
func (s *FilePPScanner) readFile(path string, maxBytes int64) ([]byte, error) {
file, err := s.openRegular(path)
if err != nil {
return nil, err
}
body, readErr := io.ReadAll(io.LimitReader(file, maxBytes+1))
closeErr := file.Close()
if readErr != nil {
return nil, readErr
}
if closeErr != nil {
return nil, closeErr
}
if int64(len(body)) > maxBytes {
return nil, errors.New("PP metadata file exceeds size limit")
}
return body, nil
}
func (s *FilePPScanner) validateDeclaredPath(path string) error {
absolute, err := filepath.Abs(path)
if err != nil {
return errors.New("invalid PP source path")
}
relative, err := filepath.Rel(s.root, absolute)
if err != nil || relative == ".." || filepath.IsAbs(relative) || len(relative) >= 3 && relative[:3] == ".."+string(filepath.Separator) {
return errors.New("PP source path is outside allowed root")
}
parent := filepath.Dir(absolute)
resolvedParent, err := filepath.EvalSymlinks(parent)
if err != nil || resolvedParent != parent {
return errors.New("PP source path parent must not contain symlinks")
}
return nil
}
func (s *FilePPScanner) openRegular(path string) (*os.File, error) {
if err := s.validateDeclaredPath(path); err != nil {
return nil, err
}
absolute, _ := filepath.Abs(path)
resolved, err := filepath.EvalSymlinks(absolute)
if err != nil {
return nil, err
}
if resolved != absolute {
return nil, errors.New("PP source path must not contain symlinks")
}
before, err := os.Lstat(absolute)
if err != nil {
return nil, err
}
if !before.Mode().IsRegular() || before.Mode()&os.ModeSymlink != 0 {
return nil, errors.New("PP source path must be a regular file")
}
file, err := os.Open(absolute)
if err != nil {
return nil, err
}
after, err := file.Stat()
if err != nil || !after.Mode().IsRegular() || !os.SameFile(before, after) {
_ = file.Close()
return nil, errors.New("PP source file changed while opening")
}
return file, nil
}
+127
View File
@@ -0,0 +1,127 @@
package sourcescan
import (
"context"
"encoding/json"
"errors"
"sort"
"time"
ppconnector "knowledge-governance/internal/connectors/pp"
"knowledge-governance/internal/governance"
"knowledge-governance/internal/jobs"
)
type PPScanner interface {
Scan(context.Context) (ppconnector.Snapshot, string, error)
}
type PPScanStore interface {
StartSourceScan(context.Context, string, time.Time) (governance.SourceScan, error)
LoadPPCurrentItems(context.Context, string) (map[string]ppconnector.CurrentItem, error)
CompleteSourceScan(context.Context, string, governance.SourceScanResult, time.Time) error
FailSourceScan(context.Context, string, string, time.Time) error
}
type PPHandlerConfig struct {
Store PPScanStore
Scanner PPScanner
Now func() time.Time
}
type PPHandler struct {
store PPScanStore
scanner PPScanner
now func() time.Time
}
func NewPPHandler(config PPHandlerConfig) (*PPHandler, error) {
if config.Store == nil || config.Scanner == nil {
return nil, errors.New("PP scan store and scanner are required")
}
if config.Now == nil {
config.Now = time.Now
}
return &PPHandler{store: config.Store, scanner: config.Scanner, now: config.Now}, nil
}
func (h *PPHandler) Handle(ctx context.Context, job jobs.Job) error {
var payload struct {
ScanID string `json:"scan_id"`
}
if err := json.Unmarshal(job.Payload, &payload); err != nil || payload.ScanID == "" {
return errors.New("invalid source scan payload")
}
now := h.now()
scan, err := h.store.StartSourceScan(ctx, payload.ScanID, now)
if err != nil {
return err
}
if scan.State == governance.SourceScanCompleted {
return nil
}
if scan.State != governance.SourceScanRunning {
return errors.New("source scan is not running")
}
if scan.SourceType != governance.SourceTypePP {
return h.store.FailSourceScan(ctx, scan.ID, "source_type_unsupported", now)
}
snapshot, snapshotSHA, err := h.scanner.Scan(ctx)
if err != nil || !governance.ValidSHA256(snapshotSHA) {
return h.store.FailSourceScan(ctx, scan.ID, "pp_snapshot_invalid", now)
}
current, err := h.store.LoadPPCurrentItems(ctx, scan.SourceConnectionID)
if err != nil {
return err
}
reconciliation, err := ppconnector.PlanReconciliation(snapshot, current)
if err != nil {
return h.store.FailSourceScan(ctx, scan.ID, "pp_snapshot_invalid", now)
}
observed := make(map[string]struct{}, len(snapshot.Items))
changes := make([]governance.SourceScanChange, 0)
for _, item := range snapshot.Items {
observed[item.SourceURLHash] = struct{}{}
if item.Status == ppconnector.StatusOverLimit {
continue
}
size := item.Bytes
stored, exists := current[item.SourceURLHash]
if !exists {
changes = append(changes, governance.SourceScanChange{
StableExternalID: item.SourceURLHash, ChangeType: governance.SourceScanAdded,
ObservedRevision: item.SHA256, ObservedSizeBytes: &size,
})
} else if stored.Revision != item.SHA256 {
changes = append(changes, governance.SourceScanChange{
StableExternalID: item.SourceURLHash, ChangeType: governance.SourceScanUpdated,
PreviousRevision: stored.Revision, ObservedRevision: item.SHA256, ObservedSizeBytes: &size,
})
}
}
missingIDs := make([]string, 0)
for sourceID := range current {
if _, ok := observed[sourceID]; !ok {
missingIDs = append(missingIDs, sourceID)
}
}
sort.Strings(missingIDs)
for _, sourceID := range missingIDs {
changes = append(changes, governance.SourceScanChange{
StableExternalID: sourceID, ChangeType: governance.SourceScanUnavailable,
PreviousRevision: current[sourceID].Revision,
})
}
result := governance.SourceScanResult{
SnapshotSHA256: snapshotSHA,
ObservedItems: len(snapshot.Items),
UnchangedItems: reconciliation.Unchanged,
AddedItems: reconciliation.Adopted,
UpdatedItems: reconciliation.UploadRequired,
OverLimitItems: reconciliation.OverLimit,
MissingItems: len(missingIDs),
PlannedUploads: reconciliation.UploadRequired,
Changes: changes,
}
return h.store.CompleteSourceScan(ctx, scan.ID, result, now)
}
@@ -0,0 +1,208 @@
package sourcescan
import (
"context"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
"time"
ppconnector "knowledge-governance/internal/connectors/pp"
"knowledge-governance/internal/governance"
"knowledge-governance/internal/jobs"
)
func TestFilePPScannerReadsValidatedSnapshotUnderAllowedRoot(t *testing.T) {
root := t.TempDir()
source := strings.Repeat("a", 64)
sha := strings.Repeat("b", 64)
ledgerPath := filepath.Join(root, "state.jsonl")
manifestPath := filepath.Join(root, "manifest.json")
ledger := `{"kb_id":"kb-1","source_url_hash":"` + source + `","source_ids":["pp-1"],"bytes":10,"sha256":"` + sha + `","status":"completed","knowledge_id":"knowledge-1"}`
if err := os.WriteFile(ledgerPath, []byte(ledger), 0o600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(manifestPath, []byte(`{"results":[]}`), 0o600); err != nil {
t.Fatal(err)
}
scanner, err := NewFilePPScanner(FilePPScannerConfig{AllowedRoot: root, LedgerPath: ledgerPath, ManifestPath: manifestPath})
if err != nil {
t.Fatalf("NewFilePPScanner() error = %v", err)
}
snapshot, fingerprint, err := scanner.Scan(context.Background())
if err != nil {
t.Fatalf("Scan() error = %v", err)
}
if len(snapshot.Items) != 1 || snapshot.Items[0].SourceURLHash != source {
t.Fatalf("snapshot = %#v", snapshot)
}
if !governance.ValidSHA256(fingerprint) {
t.Fatalf("fingerprint = %q", fingerprint)
}
}
func TestFilePPScannerRejectsManifestItemWithoutLocalPath(t *testing.T) {
root := t.TempDir()
source := strings.Repeat("a", 64)
sha := strings.Repeat("b", 64)
ledgerPath := filepath.Join(root, "state.jsonl")
manifestPath := filepath.Join(root, "manifest.json")
ledger := `{"kb_id":"kb-1","source_url_hash":"` + source + `","source_ids":["pp-1"],"bytes":10,"sha256":"` + sha + `","status":"completed","knowledge_id":"knowledge-1"}`
if err := os.WriteFile(ledgerPath, []byte(ledger), 0o600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(manifestPath, []byte(`{"results":[{"local_path":""}]}`), 0o600); err != nil {
t.Fatal(err)
}
scanner, err := NewFilePPScanner(FilePPScannerConfig{AllowedRoot: root, LedgerPath: ledgerPath, ManifestPath: manifestPath})
if err != nil {
t.Fatal(err)
}
if _, _, err := scanner.Scan(context.Background()); err == nil || !strings.Contains(err.Error(), "local_path") {
t.Fatalf("Scan() error = %v, want missing local_path error", err)
}
}
func TestFilePPScannerAllowsMissingArtifactForOverLimitRecord(t *testing.T) {
root := t.TempDir()
source := strings.Repeat("a", 64)
sha := strings.Repeat("b", 64)
ledgerPath := filepath.Join(root, "state.jsonl")
manifestPath := filepath.Join(root, "manifest.json")
ledger := `{"kb_id":"kb-1","source_url_hash":"` + source + `","source_ids":["pp-1"],"bytes":10,"sha256":"` + sha + `","status":"completed","knowledge_id":"knowledge-1"}`
manifest := `{"results":[{"url":"https://source.invalid/large","local_path":"` + filepath.Join(root, "not-downloaded.bin") + `","bytes":52428801,"status":"over_limit","source_ids":["pp-large"]}]}`
if err := os.WriteFile(ledgerPath, []byte(ledger), 0o600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(manifestPath, []byte(manifest), 0o600); err != nil {
t.Fatal(err)
}
scanner, err := NewFilePPScanner(FilePPScannerConfig{AllowedRoot: root, LedgerPath: ledgerPath, ManifestPath: manifestPath})
if err != nil {
t.Fatal(err)
}
snapshot, _, err := scanner.Scan(context.Background())
if err != nil {
t.Fatalf("Scan() error = %v", err)
}
if len(snapshot.Items) != 2 || snapshot.Items[1].Status != ppconnector.StatusOverLimit {
t.Fatalf("snapshot = %#v", snapshot)
}
}
func TestPPHandlerCompletesAggregateReadOnlyReconciliation(t *testing.T) {
now := time.Date(2026, 9, 6, 3, 0, 0, 0, time.UTC)
sameSource := strings.Repeat("a", 64)
updatedSource := strings.Repeat("b", 64)
overLimitSource := strings.Repeat("c", 64)
missingSource := strings.Repeat("d", 64)
addedSource := strings.Repeat("e", 64)
sameSHA := strings.Repeat("1", 64)
newSHA := strings.Repeat("2", 64)
oldSHA := strings.Repeat("3", 64)
addedSHA := strings.Repeat("5", 64)
store := &fakePPScanStore{
scan: governance.SourceScan{ID: "scan-1", SourceConnectionID: "conn-pp", SourceType: governance.SourceTypePP, State: governance.SourceScanPending},
current: map[string]ppconnector.CurrentItem{
sameSource: {Revision: sameSHA},
updatedSource: {Revision: oldSHA},
missingSource: {Revision: strings.Repeat("4", 64)},
},
}
scanner := &fakePPScanner{snapshot: ppconnector.Snapshot{KBID: "kb-1", Items: []ppconnector.Item{
{KBID: "kb-1", SourceURLHash: sameSource, SourceIDs: []string{"pp-1"}, Bytes: 10, SHA256: sameSHA, Status: ppconnector.StatusCompleted, KnowledgeID: "knowledge-1"},
{KBID: "kb-1", SourceURLHash: updatedSource, SourceIDs: []string{"pp-2"}, Bytes: 20, SHA256: newSHA, Status: ppconnector.StatusCompleted, KnowledgeID: "knowledge-2"},
{KBID: "kb-1", SourceURLHash: overLimitSource, SourceIDs: []string{"pp-3"}, Bytes: ppconnector.LegacyMaxBytes + 1, Status: ppconnector.StatusOverLimit},
{KBID: "kb-1", SourceURLHash: addedSource, SourceIDs: []string{"pp-4"}, Bytes: 30, SHA256: addedSHA, Status: ppconnector.StatusCompleted, KnowledgeID: "knowledge-4"},
}}, snapshotSHA: strings.Repeat("f", 64)}
handler, err := NewPPHandler(PPHandlerConfig{Store: store, Scanner: scanner, Now: func() time.Time { return now }})
if err != nil {
t.Fatalf("NewPPHandler() error = %v", err)
}
job := jobs.Job{Type: "source.scan.pp", Payload: []byte(`{"scan_id":"scan-1"}`)}
if err := handler.Handle(context.Background(), job); err != nil {
t.Fatalf("Handle() error = %v", err)
}
want := governance.SourceScanResult{
SnapshotSHA256: strings.Repeat("f", 64), ObservedItems: 4, UnchangedItems: 1, AddedItems: 1,
UpdatedItems: 1, OverLimitItems: 1, MissingItems: 1, PlannedUploads: 1,
Changes: []governance.SourceScanChange{
{StableExternalID: updatedSource, ChangeType: governance.SourceScanUpdated, PreviousRevision: oldSHA, ObservedRevision: newSHA, ObservedSizeBytes: int64Pointer(20)},
{StableExternalID: addedSource, ChangeType: governance.SourceScanAdded, ObservedRevision: addedSHA, ObservedSizeBytes: int64Pointer(30)},
{StableExternalID: missingSource, ChangeType: governance.SourceScanUnavailable, PreviousRevision: strings.Repeat("4", 64)},
},
}
if store.completedScanID != "scan-1" || !reflect.DeepEqual(store.completed, want) {
t.Fatalf("completed scan = %q %#v, want %#v", store.completedScanID, store.completed, want)
}
if store.completedAt != now || store.failedCode != "" {
t.Fatalf("completedAt=%v failedCode=%q", store.completedAt, store.failedCode)
}
}
func TestPPHandlerNoOpsWhenReclaimedScanAlreadyCompleted(t *testing.T) {
store := &fakePPScanStore{scan: governance.SourceScan{
ID: "scan-completed", SourceConnectionID: "conn-pp", SourceType: governance.SourceTypePP, State: governance.SourceScanCompleted,
}}
scanner := &fakePPScanner{}
handler, err := NewPPHandler(PPHandlerConfig{Store: store, Scanner: scanner})
if err != nil {
t.Fatal(err)
}
if err := handler.Handle(context.Background(), jobs.Job{Payload: []byte(`{"scan_id":"scan-completed"}`)}); err != nil {
t.Fatalf("Handle() error = %v", err)
}
if scanner.calls != 0 || store.completedScanID != "" || store.failedCode != "" {
t.Fatalf("scanner calls=%d completed=%q failed=%q", scanner.calls, store.completedScanID, store.failedCode)
}
}
func int64Pointer(value int64) *int64 { return &value }
type fakePPScanner struct {
snapshot ppconnector.Snapshot
snapshotSHA string
err error
calls int
}
func (s *fakePPScanner) Scan(context.Context) (ppconnector.Snapshot, string, error) {
s.calls++
return s.snapshot, s.snapshotSHA, s.err
}
type fakePPScanStore struct {
scan governance.SourceScan
current map[string]ppconnector.CurrentItem
completedScanID string
completed governance.SourceScanResult
completedAt time.Time
failedCode string
}
func (s *fakePPScanStore) StartSourceScan(context.Context, string, time.Time) (governance.SourceScan, error) {
if s.scan.State == governance.SourceScanPending {
s.scan.State = governance.SourceScanRunning
}
return s.scan, nil
}
func (s *fakePPScanStore) LoadPPCurrentItems(context.Context, string) (map[string]ppconnector.CurrentItem, error) {
return s.current, nil
}
func (s *fakePPScanStore) CompleteSourceScan(_ context.Context, scanID string, result governance.SourceScanResult, now time.Time) error {
s.completedScanID, s.completed, s.completedAt = scanID, result, now
return nil
}
func (s *fakePPScanStore) FailSourceScan(_ context.Context, _ string, code string, _ time.Time) error {
s.failedCode = code
return nil
}
@@ -0,0 +1,196 @@
package testweknora
import (
"context"
"errors"
"fmt"
"sort"
"strings"
"sync"
"sync/atomic"
"knowledge-governance/internal/platform/weknora"
)
type LegacyCandidate struct {
ID string
KnowledgeID string
TestKBID string
}
type LegacyVerifyClient interface {
GetKnowledge(context.Context, string) (weknora.Knowledge, error)
ListChunks(context.Context, string, int, int, []string) ([]weknora.Chunk, int64, error)
}
type LegacyVerificationReport struct {
Total int
Existing int
Missing int
WrongKB int
Pending int
Processing int
Finalizing int
Completed int
Failed int
ChunkChecked int
ChunkEmpty int
HTTPErrors int
}
func VerifyLegacy(ctx context.Context, client LegacyVerifyClient, candidates []LegacyCandidate, chunkSampleEvery int) (LegacyVerificationReport, error) {
report := LegacyVerificationReport{Total: len(candidates)}
completedSeen := 0
for _, candidate := range candidates {
knowledge, err := client.GetKnowledge(ctx, candidate.KnowledgeID)
if err != nil {
var httpErr *weknora.HTTPError
if errors.As(err, &httpErr) && httpErr.StatusCode == 404 {
report.Missing++
} else {
report.HTTPErrors++
}
continue
}
if knowledge.KnowledgeBaseID != candidate.TestKBID {
report.WrongKB++
continue
}
report.Existing++
switch knowledge.ParseStatus {
case "pending":
report.Pending++
case "processing":
report.Processing++
case "finalizing":
report.Finalizing++
case "completed":
report.Completed++
completedSeen++
if chunkSampleEvery > 0 && completedSeen%chunkSampleEvery == 0 {
if err := verifyLegacyChunk(ctx, client, candidate.KnowledgeID, &report); err != nil {
report.HTTPErrors++
}
}
case "failed":
report.Failed++
}
}
return report, nil
}
func VerifyLegacyConcurrent(ctx context.Context, client LegacyVerifyClient, candidates []LegacyCandidate, chunkSampleEvery, concurrency int) (LegacyVerificationReport, error) {
if client == nil || concurrency < 1 || concurrency > 32 || chunkSampleEvery < 0 {
return LegacyVerificationReport{}, errors.New("invalid legacy verification configuration")
}
jobs := make(chan LegacyCandidate)
results := make(chan LegacyVerificationReport)
var workers sync.WaitGroup
var completedSeen atomic.Int64
workerCount := min(concurrency, len(candidates))
workers.Add(workerCount)
for range workerCount {
go func() {
defer workers.Done()
for candidate := range jobs {
partial, _ := VerifyLegacy(ctx, client, []LegacyCandidate{candidate}, 0)
if partial.Completed == 1 {
completed := completedSeen.Add(1)
if chunkSampleEvery > 0 && completed%int64(chunkSampleEvery) == 0 {
if err := verifyLegacyChunk(ctx, client, candidate.KnowledgeID, &partial); err != nil {
partial.HTTPErrors++
}
}
}
results <- partial
}
}()
}
go func() {
defer close(jobs)
for _, candidate := range candidates {
select {
case jobs <- candidate:
case <-ctx.Done():
return
}
}
}()
go func() {
workers.Wait()
close(results)
}()
report := LegacyVerificationReport{}
for partial := range results {
mergeLegacyVerificationReport(&report, partial)
}
if err := ctx.Err(); err != nil {
return report, err
}
return report, nil
}
func mergeLegacyVerificationReport(report *LegacyVerificationReport, partial LegacyVerificationReport) {
report.Total += partial.Total
report.Existing += partial.Existing
report.Missing += partial.Missing
report.WrongKB += partial.WrongKB
report.Pending += partial.Pending
report.Processing += partial.Processing
report.Finalizing += partial.Finalizing
report.Completed += partial.Completed
report.Failed += partial.Failed
report.ChunkChecked += partial.ChunkChecked
report.ChunkEmpty += partial.ChunkEmpty
report.HTTPErrors += partial.HTTPErrors
}
func verifyLegacyChunk(ctx context.Context, client LegacyVerifyClient, knowledgeID string, report *LegacyVerificationReport) error {
_, total, err := client.ListChunks(ctx, knowledgeID, 1, 1, nil)
if err != nil {
return err
}
report.ChunkChecked++
if total == 0 {
report.ChunkEmpty++
}
return nil
}
func FormatDryRunReport(candidates []LegacyCandidate, concurrency int) string {
kbCounts := make(map[string]int)
for _, candidate := range candidates {
kbCounts[candidate.TestKBID]++
}
kbs := make([]string, 0, len(kbCounts))
for kb := range kbCounts {
kbs = append(kbs, kb)
}
sort.Strings(kbs)
var builder strings.Builder
fmt.Fprintf(&builder, "total=%d\n", len(candidates))
fmt.Fprintf(&builder, "concurrency=%d\n", concurrency)
for _, kb := range kbs {
fmt.Fprintf(&builder, "knowledge_base=%s count=%d\n", kb, kbCounts[kb])
}
return builder.String()
}
func FormatLegacyVerificationReport(report LegacyVerificationReport) string {
return fmt.Sprintf(
"total=%d\nexisting=%d\nmissing=%d\nwrong_kb=%d\npending=%d\nprocessing=%d\nfinalizing=%d\ncompleted=%d\nfailed=%d\nchunk_checked=%d\nchunk_empty=%d\nhttp_errors=%d\n",
report.Total,
report.Existing,
report.Missing,
report.WrongKB,
report.Pending,
report.Processing,
report.Finalizing,
report.Completed,
report.Failed,
report.ChunkChecked,
report.ChunkEmpty,
report.HTTPErrors,
)
}
@@ -0,0 +1,192 @@
package testweknora
import (
"context"
"strings"
"sync/atomic"
"testing"
"time"
"knowledge-governance/internal/platform/weknora"
)
func TestVerifyLegacyClassifiesWithoutLeaking(t *testing.T) {
client := &legacyClientStub{getResults: map[string]legacyGetResult{
"knowledge-completed-a": {knowledge: weknora.Knowledge{KnowledgeBaseID: "kb-1", ParseStatus: "completed"}},
"knowledge-completed-b": {knowledge: weknora.Knowledge{KnowledgeBaseID: "kb-1", ParseStatus: "completed"}},
"knowledge-pending": {knowledge: weknora.Knowledge{KnowledgeBaseID: "kb-1", ParseStatus: "pending"}},
"knowledge-processing": {knowledge: weknora.Knowledge{KnowledgeBaseID: "kb-1", ParseStatus: "processing"}},
"knowledge-finalizing": {knowledge: weknora.Knowledge{KnowledgeBaseID: "kb-1", ParseStatus: "finalizing"}},
"knowledge-failed": {knowledge: weknora.Knowledge{KnowledgeBaseID: "kb-1", ParseStatus: "failed"}},
"knowledge-wrong-kb": {knowledge: weknora.Knowledge{KnowledgeBaseID: "kb-other", ParseStatus: "completed"}},
"knowledge-missing": {err: &weknora.HTTPError{StatusCode: 404}},
"knowledge-server-error": {err: &weknora.HTTPError{StatusCode: 500}},
}, chunkTotal: map[string]int64{
"knowledge-completed-a": 2,
"knowledge-completed-b": 0,
}}
candidates := []LegacyCandidate{
{ID: "pub-1", KnowledgeID: "knowledge-completed-a", TestKBID: "kb-1"},
{ID: "pub-2", KnowledgeID: "knowledge-completed-b", TestKBID: "kb-1"},
{ID: "pub-3", KnowledgeID: "knowledge-pending", TestKBID: "kb-1"},
{ID: "pub-4", KnowledgeID: "knowledge-processing", TestKBID: "kb-1"},
{ID: "pub-5", KnowledgeID: "knowledge-finalizing", TestKBID: "kb-1"},
{ID: "pub-6", KnowledgeID: "knowledge-failed", TestKBID: "kb-1"},
{ID: "pub-7", KnowledgeID: "knowledge-wrong-kb", TestKBID: "kb-1"},
{ID: "pub-8", KnowledgeID: "knowledge-missing", TestKBID: "kb-1"},
{ID: "pub-9", KnowledgeID: "knowledge-server-error", TestKBID: "kb-1"},
}
report, err := VerifyLegacy(context.Background(), client, candidates, 1)
if err != nil {
t.Fatalf("VerifyLegacy() error = %v", err)
}
want := LegacyVerificationReport{
Total: 9, Existing: 6, Missing: 1, WrongKB: 1,
Pending: 1, Processing: 1, Finalizing: 1, Completed: 2, Failed: 1,
ChunkChecked: 2, ChunkEmpty: 1, HTTPErrors: 1,
}
if report != want {
t.Fatalf("report = %#v, want %#v", report, want)
}
}
func TestVerifyLegacyChunkSampleEverySkipsUnsampledCompleted(t *testing.T) {
client := &legacyClientStub{getResults: map[string]legacyGetResult{
"knowledge-a": {knowledge: weknora.Knowledge{KnowledgeBaseID: "kb-1", ParseStatus: "completed"}},
"knowledge-b": {knowledge: weknora.Knowledge{KnowledgeBaseID: "kb-1", ParseStatus: "completed"}},
}, chunkTotal: map[string]int64{
"knowledge-b": 5,
}}
candidates := []LegacyCandidate{
{ID: "pub-1", KnowledgeID: "knowledge-a", TestKBID: "kb-1"},
{ID: "pub-2", KnowledgeID: "knowledge-b", TestKBID: "kb-1"},
}
report, err := VerifyLegacy(context.Background(), client, candidates, 2)
if err != nil {
t.Fatal(err)
}
if report.Completed != 2 || report.ChunkChecked != 1 {
t.Fatalf("report = %#v", report)
}
if client.chunkQueries != 1 {
t.Fatalf("chunk queries = %d, want 1", client.chunkQueries)
}
}
func TestVerifyLegacyConcurrentBoundsConcurrencyAndAggregates(t *testing.T) {
client := &concurrentLegacyClient{}
candidates := make([]LegacyCandidate, 12)
for index := range candidates {
candidates[index] = LegacyCandidate{ID: "publication", KnowledgeID: "knowledge", TestKBID: "kb-1"}
}
report, err := VerifyLegacyConcurrent(context.Background(), client, candidates, 0, 3)
if err != nil {
t.Fatalf("VerifyLegacyConcurrent() error = %v", err)
}
if report.Total != 12 || report.Existing != 12 || report.Completed != 12 {
t.Fatalf("report = %#v", report)
}
if maximum := client.maximum.Load(); maximum < 2 || maximum > 3 {
t.Fatalf("maximum concurrency = %d, want 2..3", maximum)
}
}
func TestVerifyLegacyConcurrentReturnsImmediatelyForEmptyInput(t *testing.T) {
report, err := VerifyLegacyConcurrent(context.Background(), &concurrentLegacyClient{}, nil, 10, 4)
if err != nil {
t.Fatalf("VerifyLegacyConcurrent() error = %v", err)
}
if report != (LegacyVerificationReport{}) {
t.Fatalf("report = %#v", report)
}
}
func TestFormatDryRunReportAggregatesWithoutKnowledgeIDs(t *testing.T) {
candidates := []LegacyCandidate{
{ID: "pub-1", KnowledgeID: "knowledge-1", TestKBID: "kb-1"},
{ID: "pub-2", KnowledgeID: "knowledge-2", TestKBID: "kb-1"},
{ID: "pub-3", KnowledgeID: "knowledge-3", TestKBID: "kb-2"},
}
report := FormatDryRunReport(candidates, 4)
if !strings.Contains(report, "total=3") || !strings.Contains(report, "concurrency=4") ||
!strings.Contains(report, "knowledge_base=kb-1 count=2") || !strings.Contains(report, "knowledge_base=kb-2 count=1") {
t.Fatalf("report = %q", report)
}
if strings.Contains(report, "knowledge-1") || strings.Contains(report, "pub-1") {
t.Fatalf("report leaked knowledge/publication ID: %q", report)
}
}
func TestFormatLegacyVerificationReportContainsOnlyAggregateCounts(t *testing.T) {
report := FormatLegacyVerificationReport(LegacyVerificationReport{
Total: 530, Existing: 500, Missing: 20, WrongKB: 10,
Completed: 490, Failed: 10, ChunkChecked: 20, ChunkEmpty: 2, HTTPErrors: 3,
})
for _, expected := range []string{"total=530", "existing=500", "missing=20", "wrong_kb=10", "completed=490", "failed=10", "chunk_checked=20", "chunk_empty=2", "http_errors=3"} {
if !strings.Contains(report, expected) {
t.Fatalf("report missing %q: %q", expected, report)
}
}
for _, forbidden := range []string{"knowledge_id", "knowledge_base", "publication_id"} {
if strings.Contains(report, forbidden) {
t.Fatalf("report leaked identifier field %q: %q", forbidden, report)
}
}
}
type legacyClientStub struct {
getResults map[string]legacyGetResult
chunkTotal map[string]int64
chunkQueries int
}
type legacyGetResult struct {
knowledge weknora.Knowledge
err error
}
type concurrentLegacyClient struct {
current atomic.Int64
maximum atomic.Int64
}
func (client *concurrentLegacyClient) GetKnowledge(context.Context, string) (weknora.Knowledge, error) {
current := client.current.Add(1)
for {
maximum := client.maximum.Load()
if current <= maximum || client.maximum.CompareAndSwap(maximum, current) {
break
}
}
time.Sleep(10 * time.Millisecond)
client.current.Add(-1)
return weknora.Knowledge{KnowledgeBaseID: "kb-1", ParseStatus: "completed"}, nil
}
func (*concurrentLegacyClient) ListChunks(context.Context, string, int, int, []string) ([]weknora.Chunk, int64, error) {
return nil, 1, nil
}
func (client *legacyClientStub) GetKnowledge(_ context.Context, knowledgeID string) (weknora.Knowledge, error) {
result, ok := client.getResults[knowledgeID]
if !ok {
return weknora.Knowledge{}, &weknora.HTTPError{StatusCode: 404}
}
return result.knowledge, result.err
}
func (client *legacyClientStub) ListChunks(_ context.Context, knowledgeID string, _ int, _ int, _ []string) ([]weknora.Chunk, int64, error) {
client.chunkQueries++
total, ok := client.chunkTotal[knowledgeID]
if !ok {
total = 0
}
chunks := make([]weknora.Chunk, 0, total)
for i := int64(0); i < total; i++ {
chunks = append(chunks, weknora.Chunk{ID: "chunk", KnowledgeID: knowledgeID})
}
return chunks, total, nil
}
+141
View File
@@ -0,0 +1,141 @@
package testweknora
import (
"context"
"errors"
"time"
"knowledge-governance/internal/governance"
"knowledge-governance/internal/jobs"
postgresstore "knowledge-governance/internal/platform/postgres"
"knowledge-governance/internal/platform/weknora"
)
var (
ErrInvalidPollJobPayload = errors.New("test_poll_invalid_payload")
ErrPollRemoteFailed = errors.New("test_poll_remote_failed")
ErrPollRemoteIdentity = errors.New("test_poll_remote_identity_invalid")
ErrPollStateConflict = errors.New("test_poll_state_conflict")
)
type PollStore interface {
LoadPublicationForPoll(context.Context, string) (postgresstore.PublicationForPoll, error)
RecordRemoteStatusAndScheduleNextPoll(context.Context, string, int, governance.ProcessingState, *time.Time, time.Time, time.Time) (int, error)
RecordCompletedAndScheduleValidation(context.Context, string, int, *time.Time, time.Time) error
RecordProcessingFailure(context.Context, string, int, string, time.Time) error
MarkStalled(context.Context, string, int, time.Time, time.Time) error
}
type PollClient interface {
GetKnowledge(context.Context, string) (weknora.Knowledge, error)
}
type PollHandlerConfig struct {
Store PollStore
Client PollClient
PollDelay time.Duration
StallAfter time.Duration
Now func() time.Time
}
type PollHandler struct {
store PollStore
client PollClient
pollDelay time.Duration
stallAfter time.Duration
now func() time.Time
}
type pollPayload struct {
PublicationID string `json:"publication_id"`
Sequence int `json:"sequence"`
}
func NewPollHandler(config PollHandlerConfig) (*PollHandler, error) {
if config.Store == nil || config.Client == nil || config.PollDelay <= 0 || config.StallAfter <= 0 {
return nil, errors.New("invalid test poll handler configuration")
}
if config.Now == nil {
config.Now = time.Now
}
return &PollHandler{
store: config.Store, client: config.Client, pollDelay: config.PollDelay,
stallAfter: config.StallAfter, now: config.Now,
}, nil
}
func (handler *PollHandler) Handle(ctx context.Context, job jobs.Job) error {
var payload pollPayload
if err := decodeStrictJSON(job.Payload, &payload); err != nil || payload.PublicationID == "" || payload.Sequence < 1 {
return jobs.Permanent(jobs.ErrCodeHandlerFallback, ErrInvalidPollJobPayload)
}
publication, err := handler.store.LoadPublicationForPoll(ctx, payload.PublicationID)
if err != nil {
return ErrPollStateConflict
}
if publication.KnowledgeID == "" || publication.PollSequence != payload.Sequence {
return ErrPollStateConflict
}
now := handler.now()
knowledge, err := handler.client.GetKnowledge(ctx, publication.KnowledgeID)
if err != nil {
return ErrPollRemoteFailed
}
if knowledge.ID != publication.KnowledgeID || knowledge.KnowledgeBaseID != publication.TestKBID {
return ErrPollRemoteIdentity
}
state := mapParseStatus(knowledge.ParseStatus)
if state == governance.ProcessingCompleted {
if err := handler.store.RecordCompletedAndScheduleValidation(ctx, publication.ID, publication.PollSequence, &knowledge.UpdatedAt, now); err != nil {
return ErrPollStateConflict
}
return nil
}
if state == governance.ProcessingFailed {
if err := handler.store.RecordProcessingFailure(ctx, publication.ID, publication.PollSequence, "remote_parse_failed", now); err != nil {
return ErrPollStateConflict
}
return nil
}
if !state.Valid() {
if err := handler.store.RecordProcessingFailure(ctx, publication.ID, publication.PollSequence, "remote_unknown_status", now); err != nil {
return ErrPollStateConflict
}
return nil
}
cutoff := now.Add(-handler.stallAfter)
lastProgress := publication.LastProgressAt
if lastProgress == nil {
lastProgress = publication.ProcessingStartedAt
}
if lastProgress == nil {
lastProgress = publication.LastObservedAt
}
if lastProgress != nil && !lastProgress.After(cutoff) {
if err := handler.store.MarkStalled(ctx, publication.ID, publication.PollSequence, cutoff, now); err != nil {
return ErrPollStateConflict
}
return nil
}
if _, err := handler.store.RecordRemoteStatusAndScheduleNextPoll(ctx, publication.ID, publication.PollSequence, state, &knowledge.UpdatedAt, now, now.Add(handler.pollDelay)); err != nil {
return ErrPollStateConflict
}
return nil
}
func mapParseStatus(status string) governance.ProcessingState {
switch status {
case "pending":
return governance.ProcessingPending
case "processing":
return governance.ProcessingProcessing
case "finalizing":
return governance.ProcessingFinalizing
case "completed":
return governance.ProcessingCompleted
case "failed":
return governance.ProcessingFailed
default:
return governance.ProcessingState(status)
}
}
+188
View File
@@ -0,0 +1,188 @@
package testweknora
import (
"context"
"errors"
"net/http"
"net/http/httptest"
"testing"
"time"
"knowledge-governance/internal/governance"
"knowledge-governance/internal/jobs"
postgresstore "knowledge-governance/internal/platform/postgres"
"knowledge-governance/internal/platform/weknora"
)
func TestPollHandlerSchedulesNextSequenceForRemoteProgress(t *testing.T) {
remoteUpdated := time.Date(2026, time.September, 4, 3, 39, 0, 0, time.UTC)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet || r.URL.Path != "/api/v1/knowledge/knowledge-1" {
t.Fatalf("request = %s %s", r.Method, r.URL.Path)
}
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"success":true,"data":{"id":"knowledge-1","knowledge_base_id":"kb-1","parse_status":"finalizing","updated_at":"2026-09-04T03:39:00Z"}}`))
}))
defer server.Close()
client, err := weknora.NewClient(weknora.ClientConfig{BaseURL: server.URL, APIKey: "test-api-key", Timeout: time.Second})
if err != nil {
t.Fatal(err)
}
now := time.Date(2026, time.September, 4, 3, 40, 0, 0, time.UTC)
lastProgress := now.Add(-time.Minute)
store := &pollStoreStub{publication: postgresstore.PublicationForPoll{
ID: "publication-1", TestKBID: "kb-1", KnowledgeID: "knowledge-1",
ProcessingState: governance.ProcessingProcessing, PollSequence: 2,
LastRemoteStatus: "processing", LastProgressAt: &lastProgress,
}}
handler, err := NewPollHandler(PollHandlerConfig{
Store: store, Client: client, PollDelay: 15 * time.Second,
StallAfter: 30 * time.Minute, Now: func() time.Time { return now },
})
if err != nil {
t.Fatal(err)
}
if err := handler.Handle(context.Background(), jobs.Job{Payload: []byte(`{"publication_id":"publication-1","sequence":2}`)}); err != nil {
t.Fatalf("Handle() error = %v", err)
}
if store.recordedState != governance.ProcessingFinalizing || store.recordedRemoteUpdated == nil || !store.recordedRemoteUpdated.Equal(remoteUpdated) || !store.nextPollAt.Equal(now.Add(15*time.Second)) {
t.Fatalf("store = %#v", store)
}
}
func TestPollHandlerCompletesAndSchedulesValidationWithoutNextPoll(t *testing.T) {
completedAt := time.Date(2026, time.September, 4, 3, 45, 0, 0, time.UTC)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet || r.URL.Path != "/api/v1/knowledge/knowledge-1" {
t.Fatalf("request = %s %s", r.Method, r.URL.Path)
}
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"success":true,"data":{"id":"knowledge-1","knowledge_base_id":"kb-1","parse_status":"completed","updated_at":"2026-09-04T03:45:00Z"}}`))
}))
defer server.Close()
client, err := weknora.NewClient(weknora.ClientConfig{BaseURL: server.URL, APIKey: "test-api-key", Timeout: time.Second})
if err != nil {
t.Fatal(err)
}
now := time.Date(2026, time.September, 4, 3, 46, 0, 0, time.UTC)
lastProgress := now.Add(-time.Minute)
store := &pollStoreStub{publication: postgresstore.PublicationForPoll{
ID: "publication-1", TestKBID: "kb-1", KnowledgeID: "knowledge-1",
ProcessingState: governance.ProcessingProcessing, PollSequence: 3,
LastRemoteStatus: "processing", LastProgressAt: &lastProgress,
}}
handler, err := NewPollHandler(PollHandlerConfig{
Store: store, Client: client, PollDelay: 15 * time.Second,
StallAfter: 30 * time.Minute, Now: func() time.Time { return now },
})
if err != nil {
t.Fatal(err)
}
if err := handler.Handle(context.Background(), jobs.Job{Payload: []byte(`{"publication_id":"publication-1","sequence":3}`)}); err != nil {
t.Fatalf("Handle() error = %v", err)
}
if !store.completedValidated || store.recordedState != "" || store.recordedRemoteUpdated == nil || !store.recordedRemoteUpdated.Equal(completedAt) {
t.Fatalf("store = %#v", store)
}
}
func TestPollHandlerMarksStalledWhenNoProgressExceedsThreshold(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet || r.URL.Path != "/api/v1/knowledge/knowledge-1" {
t.Fatalf("request = %s %s", r.Method, r.URL.Path)
}
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"success":true,"data":{"id":"knowledge-1","knowledge_base_id":"kb-1","parse_status":"finalizing","updated_at":"2026-09-04T03:10:00Z"}}`))
}))
defer server.Close()
client, err := weknora.NewClient(weknora.ClientConfig{BaseURL: server.URL, APIKey: "test-api-key", Timeout: time.Second})
if err != nil {
t.Fatal(err)
}
now := time.Date(2026, time.September, 4, 3, 50, 0, 0, time.UTC)
stalledSince := now.Add(-45 * time.Minute)
store := &pollStoreStub{publication: postgresstore.PublicationForPoll{
ID: "publication-1", TestKBID: "kb-1", KnowledgeID: "knowledge-1",
ProcessingState: governance.ProcessingProcessing, PollSequence: 4,
LastRemoteStatus: "processing", LastProgressAt: &stalledSince,
}}
handler, err := NewPollHandler(PollHandlerConfig{
Store: store, Client: client, PollDelay: 15 * time.Second,
StallAfter: 30 * time.Minute, Now: func() time.Time { return now },
})
if err != nil {
t.Fatal(err)
}
if err := handler.Handle(context.Background(), jobs.Job{Payload: []byte(`{"publication_id":"publication-1","sequence":4}`)}); err != nil {
t.Fatalf("Handle() error = %v", err)
}
if !store.stalledMarked || store.recordedState != "" || !store.stalledCutoff.Equal(now.Add(-30*time.Minute)) {
t.Fatalf("store = %#v", store)
}
}
func TestPollHandlerRejectsStaleSequenceWithoutPollingRemote(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
t.Fatal("stale sequence must not poll the remote knowledge endpoint")
}))
defer server.Close()
client, err := weknora.NewClient(weknora.ClientConfig{BaseURL: server.URL, APIKey: "test-api-key", Timeout: time.Second})
if err != nil {
t.Fatal(err)
}
now := time.Date(2026, time.September, 4, 4, 0, 0, 0, time.UTC)
store := &pollStoreStub{publication: postgresstore.PublicationForPoll{
ID: "publication-1", TestKBID: "kb-1", KnowledgeID: "knowledge-1",
ProcessingState: governance.ProcessingProcessing, PollSequence: 5,
}}
handler, err := NewPollHandler(PollHandlerConfig{
Store: store, Client: client, PollDelay: 15 * time.Second,
StallAfter: 30 * time.Minute, Now: func() time.Time { return now },
})
if err != nil {
t.Fatal(err)
}
if err := handler.Handle(context.Background(), jobs.Job{Payload: []byte(`{"publication_id":"publication-1","sequence":2}`)}); !errors.Is(err, ErrPollStateConflict) {
t.Fatalf("Handle() error = %v, want state conflict", err)
}
if store.recordedState != "" || store.stalledMarked || store.completedValidated {
t.Fatalf("store mutated for stale sequence: %#v", store)
}
}
type pollStoreStub struct {
publication postgresstore.PublicationForPoll
recordedState governance.ProcessingState
recordedRemoteUpdated *time.Time
nextPollAt time.Time
completedValidated bool
stalledMarked bool
stalledCutoff time.Time
}
func (store *pollStoreStub) LoadPublicationForPoll(context.Context, string) (postgresstore.PublicationForPoll, error) {
return store.publication, nil
}
func (store *pollStoreStub) RecordRemoteStatusAndScheduleNextPoll(_ context.Context, _ string, _ int, state governance.ProcessingState, remoteUpdated *time.Time, _ time.Time, nextAt time.Time) (int, error) {
store.recordedState = state
store.recordedRemoteUpdated = remoteUpdated
store.nextPollAt = nextAt
return store.publication.PollSequence + 1, nil
}
func (store *pollStoreStub) RecordCompletedAndScheduleValidation(_ context.Context, _ string, _ int, remoteUpdated *time.Time, _ time.Time) error {
store.completedValidated = true
store.recordedRemoteUpdated = remoteUpdated
return nil
}
func (store *pollStoreStub) RecordProcessingFailure(context.Context, string, int, string, time.Time) error {
return nil
}
func (store *pollStoreStub) MarkStalled(_ context.Context, _ string, _ int, cutoff, _ time.Time) error {
store.stalledMarked = true
store.stalledCutoff = cutoff
return nil
}
+117
View File
@@ -0,0 +1,117 @@
package testweknora
import (
"context"
"errors"
"net/http"
"time"
"knowledge-governance/internal/jobs"
postgresstore "knowledge-governance/internal/platform/postgres"
"knowledge-governance/internal/platform/weknora"
)
var (
ErrInvalidRemoveJobPayload = errors.New("test_remove_invalid_payload")
ErrRemoveRemoteFailed = errors.New("test_remove_remote_failed")
ErrRemoveRemoteIdentity = errors.New("test_remove_remote_identity_invalid")
ErrRemovePending = errors.New("test_remove_pending")
ErrRemoveStateConflict = errors.New("test_remove_state_conflict")
)
type RemoveStore interface {
LoadPublicationForRemoval(context.Context, string) (postgresstore.PublicationForRemoval, error)
RecordRemovalCompleted(context.Context, string, time.Time) error
RecordRemovalFailure(context.Context, string, string, time.Time) error
}
type RemoveClient interface {
GetKnowledge(context.Context, string) (weknora.Knowledge, error)
DeleteKnowledge(context.Context, string) error
}
type RemoveHandlerConfig struct {
Store RemoveStore
Client RemoveClient
Now func() time.Time
}
type RemoveHandler struct {
store RemoveStore
client RemoveClient
now func() time.Time
}
type removePayload struct {
PublicationID string `json:"publication_id"`
}
func NewRemoveHandler(config RemoveHandlerConfig) (*RemoveHandler, error) {
if config.Store == nil || config.Client == nil {
return nil, errors.New("invalid test removal handler configuration")
}
if config.Now == nil {
config.Now = time.Now
}
return &RemoveHandler{store: config.Store, client: config.Client, now: config.Now}, nil
}
func (handler *RemoveHandler) Handle(ctx context.Context, job jobs.Job) error {
var payload removePayload
if err := decodeStrictJSON(job.Payload, &payload); err != nil || payload.PublicationID == "" {
return jobs.Permanent(jobs.ErrCodeHandlerFallback, ErrInvalidRemoveJobPayload)
}
candidate, err := handler.store.LoadPublicationForRemoval(ctx, payload.PublicationID)
if err != nil {
return ErrRemoveStateConflict
}
knowledge, err := handler.client.GetKnowledge(ctx, candidate.KnowledgeID)
if isNotFound(err) {
return handler.complete(ctx, candidate.ID)
}
if err != nil {
return ErrRemoveRemoteFailed
}
if knowledge.ID != candidate.KnowledgeID || knowledge.KnowledgeBaseID != candidate.TestKBID {
return handler.failClosed(ctx, candidate.ID, "removal_identity_invalid")
}
if err := handler.client.DeleteKnowledge(ctx, candidate.KnowledgeID); err != nil {
if isNotFound(err) {
return handler.complete(ctx, candidate.ID)
}
if !weknora.IsRetryable(err) {
return handler.failClosed(ctx, candidate.ID, "removal_remote_rejected")
}
return ErrRemoveRemoteFailed
}
knowledge, err = handler.client.GetKnowledge(ctx, candidate.KnowledgeID)
if isNotFound(err) {
return handler.complete(ctx, candidate.ID)
}
if err != nil {
return ErrRemoveRemoteFailed
}
if knowledge.ID != candidate.KnowledgeID || knowledge.KnowledgeBaseID != candidate.TestKBID {
return handler.failClosed(ctx, candidate.ID, "removal_identity_invalid")
}
return ErrRemovePending
}
func (handler *RemoveHandler) complete(ctx context.Context, publicationID string) error {
if err := handler.store.RecordRemovalCompleted(ctx, publicationID, handler.now()); err != nil {
return ErrRemoveStateConflict
}
return nil
}
func (handler *RemoveHandler) failClosed(ctx context.Context, publicationID, code string) error {
if err := handler.store.RecordRemovalFailure(ctx, publicationID, code, handler.now()); err != nil {
return ErrRemoveStateConflict
}
return nil
}
func isNotFound(err error) bool {
var httpErr *weknora.HTTPError
return errors.As(err, &httpErr) && httpErr.StatusCode == http.StatusNotFound
}
@@ -0,0 +1,100 @@
package testweknora
import (
"context"
"errors"
"testing"
"time"
"knowledge-governance/internal/governance"
"knowledge-governance/internal/jobs"
postgresstore "knowledge-governance/internal/platform/postgres"
"knowledge-governance/internal/platform/weknora"
)
type removalStoreStub struct {
candidate postgresstore.PublicationForRemoval
completed string
failed string
}
func (store *removalStoreStub) LoadPublicationForRemoval(context.Context, string) (postgresstore.PublicationForRemoval, error) {
return store.candidate, nil
}
func (store *removalStoreStub) RecordRemovalCompleted(_ context.Context, publicationID string, _ time.Time) error {
store.completed = publicationID
return nil
}
func (store *removalStoreStub) RecordRemovalFailure(_ context.Context, publicationID, code string, _ time.Time) error {
store.failed = publicationID + ":" + code
return nil
}
type removalClientStub struct {
knowledge weknora.Knowledge
getCalls int
deleted string
keepAfterDelete bool
}
func (client *removalClientStub) GetKnowledge(context.Context, string) (weknora.Knowledge, error) {
client.getCalls++
if client.getCalls > 1 && !client.keepAfterDelete {
return weknora.Knowledge{}, &weknora.HTTPError{StatusCode: 404}
}
return client.knowledge, nil
}
func (client *removalClientStub) DeleteKnowledge(_ context.Context, knowledgeID string) error {
client.deleted = knowledgeID
return nil
}
func TestRemoveHandlerVerifiesIdentityAndReadBackBeforeCompletion(t *testing.T) {
store := &removalStoreStub{candidate: postgresstore.PublicationForRemoval{
ID: "publication-1", TestKBID: "kb-1", KnowledgeID: "knowledge-1", RetrievalState: governance.RetrievalDisabling,
}}
client := &removalClientStub{knowledge: weknora.Knowledge{ID: "knowledge-1", KnowledgeBaseID: "kb-1"}}
handler, err := NewRemoveHandler(RemoveHandlerConfig{Store: store, Client: client, Now: func() time.Time { return time.Date(2026, 9, 9, 3, 0, 0, 0, time.UTC) }})
if err != nil {
t.Fatal(err)
}
if err := handler.Handle(context.Background(), jobs.Job{Payload: []byte(`{"publication_id":"publication-1"}`)}); err != nil {
t.Fatalf("Handle() error = %v", err)
}
if client.deleted != "knowledge-1" || store.completed != "publication-1" || store.failed != "" {
t.Fatalf("deleted=%q completed=%q failed=%q", client.deleted, store.completed, store.failed)
}
}
func TestRemoveHandlerRejectsMismatchedKnowledgeBaseWithoutDeleting(t *testing.T) {
store := &removalStoreStub{candidate: postgresstore.PublicationForRemoval{
ID: "publication-1", TestKBID: "kb-1", KnowledgeID: "knowledge-1", RetrievalState: governance.RetrievalDisabling,
}}
client := &removalClientStub{knowledge: weknora.Knowledge{ID: "knowledge-1", KnowledgeBaseID: "other-kb"}}
handler, err := NewRemoveHandler(RemoveHandlerConfig{Store: store, Client: client})
if err != nil {
t.Fatal(err)
}
err = handler.Handle(context.Background(), jobs.Job{Payload: []byte(`{"publication_id":"publication-1"}`)})
if err != nil || client.deleted != "" || store.failed != "publication-1:removal_identity_invalid" {
t.Fatalf("error=%v deleted=%q failed=%q", err, client.deleted, store.failed)
}
}
func TestRemoveHandlerRetriesWhileAsynchronousDeleteIsStillVisible(t *testing.T) {
store := &removalStoreStub{candidate: postgresstore.PublicationForRemoval{
ID: "publication-1", TestKBID: "kb-1", KnowledgeID: "knowledge-1", RetrievalState: governance.RetrievalDisabling,
}}
client := &removalClientStub{knowledge: weknora.Knowledge{ID: "knowledge-1", KnowledgeBaseID: "kb-1"}, keepAfterDelete: true}
handler, err := NewRemoveHandler(RemoveHandlerConfig{Store: store, Client: client})
if err != nil {
t.Fatal(err)
}
err = handler.Handle(context.Background(), jobs.Job{Payload: []byte(`{"publication_id":"publication-1"}`)})
if !errors.Is(err, ErrRemovePending) || store.completed != "" {
t.Fatalf("error=%v completed=%q", err, store.completed)
}
}
+241
View File
@@ -0,0 +1,241 @@
package testweknora
import (
"bytes"
"context"
"encoding/json"
"errors"
"io"
"path/filepath"
"strings"
"time"
"knowledge-governance/internal/governance"
"knowledge-governance/internal/jobs"
postgresstore "knowledge-governance/internal/platform/postgres"
"knowledge-governance/internal/platform/spool"
"knowledge-governance/internal/platform/weknora"
)
var (
ErrInvalidUploadJobPayload = errors.New("test_upload_invalid_payload")
ErrUploadArtifactInvalid = errors.New("test_upload_artifact_invalid")
ErrUploadRemoteFailed = errors.New("test_upload_remote_failed")
ErrUploadRemoteIdentity = errors.New("test_upload_remote_identity_invalid")
ErrUploadStateConflict = errors.New("test_upload_state_conflict")
)
type UploadStore interface {
LoadPublicationForUpload(context.Context, string) (postgresstore.PublicationForUpload, error)
MarkUploading(context.Context, string, time.Time) error
RecordUploadAndSchedulePoll(context.Context, string, string, time.Time, time.Time) (int, error)
RecordUploadFailure(context.Context, string, string, time.Time) error
}
type UploadClient interface {
UploadFile(context.Context, weknora.UploadRequest) (weknora.Knowledge, error)
ListKnowledge(context.Context, string, int, int) ([]weknora.Knowledge, int64, error)
GetKnowledge(context.Context, string) (weknora.Knowledge, error)
}
type UploadHandlerConfig struct {
Store UploadStore
Client UploadClient
SpoolRoot string
MaxUploadBytes int64
FirstPollDelay time.Duration
Now func() time.Time
}
type UploadHandler struct {
store UploadStore
client UploadClient
spoolRoot string
maxUploadBytes int64
firstPollDelay time.Duration
now func() time.Time
}
type uploadPayload struct {
PublicationID string `json:"publication_id"`
TargetScope string `json:"target_scope,omitempty"`
TargetKBID string `json:"target_kb_id,omitempty"`
}
func NewUploadHandler(config UploadHandlerConfig) (*UploadHandler, error) {
if config.Store == nil || config.Client == nil || !filepath.IsAbs(config.SpoolRoot) ||
config.MaxUploadBytes <= 0 || config.FirstPollDelay <= 0 {
return nil, errors.New("invalid test upload handler configuration")
}
if config.Now == nil {
config.Now = time.Now
}
return &UploadHandler{
store: config.Store, client: config.Client, spoolRoot: config.SpoolRoot,
maxUploadBytes: config.MaxUploadBytes, firstPollDelay: config.FirstPollDelay, now: config.Now,
}, nil
}
func (handler *UploadHandler) Handle(ctx context.Context, job jobs.Job) error {
var payload uploadPayload
if err := decodeStrictJSON(job.Payload, &payload); err != nil || payload.PublicationID == "" {
return jobs.Permanent(jobs.ErrCodeHandlerFallback, ErrInvalidUploadJobPayload)
}
candidate, err := handler.store.LoadPublicationForUpload(ctx, payload.PublicationID)
if err != nil {
return ErrUploadStateConflict
}
// P4: the job snapshots its target; a mismatch against the stored publication
// means the plan changed underneath and this delivery must not guess.
if payload.TargetKBID != "" && payload.TargetKBID != candidate.TestKBID {
return ErrUploadStateConflict
}
if payload.TargetScope != "" && payload.TargetScope != string(candidate.TargetScope) {
return ErrUploadStateConflict
}
if candidate.IngestState == governance.IngestUploading {
knowledge, err := handler.recoverAmbiguousUpload(ctx, candidate)
if err != nil {
if errors.Is(err, ErrUploadRemoteIdentity) {
return handler.recordTerminalUploadFailure(ctx, candidate.ID, "upload_identity_ambiguous")
}
return err
}
now := handler.now()
if _, err := handler.store.RecordUploadAndSchedulePoll(ctx, candidate.ID, knowledge.ID, now, now.Add(handler.firstPollDelay)); err != nil {
return ErrUploadStateConflict
}
return nil
}
artifact, err := spool.OpenVerified(handler.spoolRoot, spool.Expectation{
RelativeRef: candidate.LocalSpoolRef,
SHA256: candidate.SHA256,
SizeBytes: candidate.SizeBytes,
Format: candidate.ValidatedFormat,
}, handler.maxUploadBytes)
if err != nil {
return handler.recordTerminalUploadFailure(ctx, candidate.ID, "upload_artifact_invalid")
}
defer artifact.Close()
if err := handler.store.MarkUploading(ctx, candidate.ID, handler.now()); err != nil {
return ErrUploadStateConflict
}
knowledge, uploadErr := handler.client.UploadFile(ctx, weknora.UploadRequest{
KnowledgeBaseID: candidate.TestKBID,
FileName: safeUploadFileName(candidate.Title, candidate.ID, candidate.ValidatedFormat),
Content: artifact,
Metadata: map[string]string{
"content_sha256": candidate.SHA256,
"publication_id": candidate.ID,
},
Channel: "governance",
})
if err := artifact.VerifyComplete(); err != nil {
return jobs.Permanent(jobs.ErrCodeArtifactInvalid, ErrUploadArtifactInvalid)
}
if uploadErr != nil {
if !errors.Is(uploadErr, weknora.ErrConflict) && !weknora.IsRetryable(uploadErr) {
return ErrUploadRemoteFailed
}
knowledge, err = handler.recoverAmbiguousUpload(ctx, candidate)
if err != nil {
if errors.Is(err, ErrUploadRemoteIdentity) {
return handler.recordTerminalUploadFailure(ctx, candidate.ID, "upload_identity_ambiguous")
}
return err
}
}
if knowledge.ID == "" || knowledge.KnowledgeBaseID != candidate.TestKBID {
return ErrUploadRemoteIdentity
}
now := handler.now()
if _, err := handler.store.RecordUploadAndSchedulePoll(ctx, candidate.ID, knowledge.ID, now, now.Add(handler.firstPollDelay)); err != nil {
return ErrUploadStateConflict
}
return nil
}
func (handler *UploadHandler) recoverAmbiguousUpload(ctx context.Context, candidate postgresstore.PublicationForUpload) (weknora.Knowledge, error) {
const pageSize = 100
matches := make([]weknora.Knowledge, 0, 2)
for page := 1; page <= 100; page++ {
items, total, err := handler.client.ListKnowledge(ctx, candidate.TestKBID, page, pageSize)
if err != nil {
return weknora.Knowledge{}, ErrUploadRemoteFailed
}
for _, item := range items {
publicationID, publicationOK := metadataString(item.Metadata, "publication_id")
contentSHA, contentOK := metadataString(item.Metadata, "content_sha256")
if item.KnowledgeBaseID == candidate.TestKBID && publicationOK && contentOK && publicationID == candidate.ID && contentSHA == candidate.SHA256 {
matches = append(matches, item)
if len(matches) > 1 {
return weknora.Knowledge{}, ErrUploadRemoteIdentity
}
}
}
if int64(page*pageSize) >= total {
break
}
if len(items) == 0 {
return weknora.Knowledge{}, ErrUploadRemoteFailed
}
}
if len(matches) != 1 {
return weknora.Knowledge{}, ErrUploadRemoteFailed
}
verified, err := handler.client.GetKnowledge(ctx, matches[0].ID)
if err != nil {
return weknora.Knowledge{}, ErrUploadRemoteFailed
}
publicationID, publicationOK := metadataString(verified.Metadata, "publication_id")
contentSHA, contentOK := metadataString(verified.Metadata, "content_sha256")
if verified.ID != matches[0].ID || verified.KnowledgeBaseID != candidate.TestKBID ||
!publicationOK || !contentOK || publicationID != candidate.ID || contentSHA != candidate.SHA256 {
return weknora.Knowledge{}, ErrUploadRemoteIdentity
}
return verified, nil
}
func metadataString(metadata map[string]any, key string) (string, bool) {
value, ok := metadata[key].(string)
return value, ok
}
func (handler *UploadHandler) recordTerminalUploadFailure(ctx context.Context, publicationID, code string) error {
if err := handler.store.RecordUploadFailure(ctx, publicationID, code, handler.now()); err != nil {
return ErrUploadStateConflict
}
return nil
}
func decodeStrictJSON(data []byte, target any) error {
decoder := json.NewDecoder(bytes.NewReader(data))
decoder.DisallowUnknownFields()
if err := decoder.Decode(target); err != nil {
return err
}
if err := decoder.Decode(&struct{}{}); err != io.EOF {
return errors.New("extra JSON value")
}
return nil
}
func safeUploadFileName(title, fallback, format string) string {
base := strings.TrimSpace(title)
base = strings.Map(func(character rune) rune {
switch character {
case '/', '\\', '\x00', '\r', '\n', '\t':
return '_'
default:
return character
}
}, base)
base = strings.Trim(base, ". ")
if base == "" {
base = fallback
}
if len(base) > 120 {
base = base[:120]
}
return base + "." + format
}
@@ -0,0 +1,354 @@
package testweknora
import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"sync/atomic"
"testing"
"time"
"knowledge-governance/internal/governance"
"knowledge-governance/internal/jobs"
postgresstore "knowledge-governance/internal/platform/postgres"
"knowledge-governance/internal/platform/weknora"
)
func TestUploadHandlerStreamsVerifiedArtifactAndSchedulesFirstPoll(t *testing.T) {
root := t.TempDir()
if err := os.Mkdir(filepath.Join(root, "objects"), 0o700); err != nil {
t.Fatal(err)
}
body := []byte("safe governance document\n")
if err := os.WriteFile(filepath.Join(root, "objects", "artifact.md"), body, 0o600); err != nil {
t.Fatal(err)
}
digest := sha256.Sum256(body)
sha := hex.EncodeToString(digest[:])
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost || r.URL.Path != "/api/v1/knowledge-bases/kb-1/knowledge/file" {
t.Fatalf("request = %s %s", r.Method, r.URL.Path)
}
if err := r.ParseMultipartForm(1 << 20); err != nil {
t.Fatal(err)
}
file, _, err := r.FormFile("file")
if err != nil {
t.Fatal(err)
}
defer file.Close()
uploaded, err := io.ReadAll(file)
if err != nil || string(uploaded) != string(body) {
t.Fatalf("uploaded body = %q, %v", uploaded, err)
}
if r.FormValue("metadata") != `{"content_sha256":"`+sha+`","publication_id":"publication-1"}` {
t.Fatalf("metadata = %q", r.FormValue("metadata"))
}
if r.FormValue("channel") != "governance" {
t.Fatalf("channel = %q", r.FormValue("channel"))
}
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"success":true,"data":{"id":"knowledge-1","knowledge_base_id":"kb-1","parse_status":"pending"}}`))
}))
defer server.Close()
client, err := weknora.NewClient(weknora.ClientConfig{BaseURL: server.URL, APIKey: "test-api-key", Timeout: time.Second})
if err != nil {
t.Fatal(err)
}
now := time.Date(2026, time.September, 4, 3, 0, 0, 0, time.UTC)
store := &uploadStoreStub{candidate: postgresstore.PublicationForUpload{
ID: "publication-1", SourceItemVersionID: "version-1", TestKBID: "kb-1",
IngestState: governance.IngestPending, ProcessingState: governance.ProcessingPending,
ContentObjectID: "content-1", SHA256: sha, SizeBytes: int64(len(body)),
ValidatedFormat: "md", LocalSpoolRef: "objects/artifact.md", Title: "Safe title",
}}
handler, err := NewUploadHandler(UploadHandlerConfig{
Store: store, Client: client, SpoolRoot: root, MaxUploadBytes: 50 * 1024 * 1024,
FirstPollDelay: 10 * time.Second, Now: func() time.Time { return now },
})
if err != nil {
t.Fatal(err)
}
job := jobs.Job{ID: "job-1", Type: "test.upload", Payload: []byte(`{"publication_id":"publication-1"}`)}
if err := handler.Handle(context.Background(), job); err != nil {
t.Fatalf("Handle() error = %v", err)
}
if !store.markedUploading || store.recordedKnowledgeID != "knowledge-1" || !store.nextPollAt.Equal(now.Add(10*time.Second)) {
t.Fatalf("store = %#v", store)
}
}
func TestUploadHandlerRejectsJobSnapshotWithMismatchedTarget(t *testing.T) {
root := t.TempDir()
now := time.Date(2026, time.September, 4, 3, 0, 0, 0, time.UTC)
store := &uploadStoreStub{candidate: postgresstore.PublicationForUpload{
ID: "publication-1", SourceItemVersionID: "version-1", TestKBID: "kb-1", TargetScope: governance.VisibilityInternal,
IngestState: governance.IngestPending, ProcessingState: governance.ProcessingPending,
}}
client, err := weknora.NewClient(weknora.ClientConfig{BaseURL: "http://127.0.0.1:1", APIKey: "***", Timeout: time.Second})
if err != nil {
t.Fatal(err)
}
handler, err := NewUploadHandler(UploadHandlerConfig{Store: store, Client: client, SpoolRoot: root, MaxUploadBytes: 1 << 20, FirstPollDelay: time.Second, Now: func() time.Time { return now }})
if err != nil {
t.Fatal(err)
}
job := jobs.Job{ID: "job-1", Type: "test.upload", Payload: []byte(`{"publication_id":"publication-1","target_scope":"external","target_kb_id":"kb-1"}`)}
if err := handler.Handle(context.Background(), job); !errors.Is(err, ErrUploadStateConflict) {
t.Fatalf("scope mismatch error = %v", err)
}
job = jobs.Job{ID: "job-2", Type: "test.upload", Payload: []byte(`{"publication_id":"publication-1","target_scope":"internal","target_kb_id":"kb-other"}`)}
if err := handler.Handle(context.Background(), job); !errors.Is(err, ErrUploadStateConflict) {
t.Fatalf("kb mismatch error = %v", err)
}
if store.markedUploading {
t.Fatal("mismatched job must not mark uploading")
}
}
func TestUploadHandlerRecoversUniqueRemoteKnowledgeAfterResponseLoss(t *testing.T) {
root := t.TempDir()
if err := os.Mkdir(filepath.Join(root, "objects"), 0o700); err != nil {
t.Fatal(err)
}
body := []byte("safe ambiguous upload\n")
if err := os.WriteFile(filepath.Join(root, "objects", "artifact.md"), body, 0o600); err != nil {
t.Fatal(err)
}
digest := sha256.Sum256(body)
sha := hex.EncodeToString(digest[:])
var uploads atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch {
case r.Method == http.MethodPost:
uploads.Add(1)
if err := r.ParseMultipartForm(1 << 20); err != nil {
t.Fatal(err)
}
file, _, err := r.FormFile("file")
if err != nil {
t.Fatal(err)
}
_, _ = io.Copy(io.Discard, file)
_ = file.Close()
connection, _, err := w.(http.Hijacker).Hijack()
if err != nil {
t.Fatal(err)
}
_ = connection.Close()
case r.Method == http.MethodGet && r.URL.Path == "/api/v1/knowledge-bases/kb-1/knowledge":
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"success":true,"data":[{"id":"knowledge-recovered","knowledge_base_id":"kb-1","metadata":{"publication_id":"publication-1","content_sha256":"` + sha + `"}}],"total":1,"page":1,"page_size":100}`))
case r.Method == http.MethodGet && r.URL.Path == "/api/v1/knowledge/knowledge-recovered":
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"success":true,"data":{"id":"knowledge-recovered","knowledge_base_id":"kb-1","metadata":{"publication_id":"publication-1","content_sha256":"` + sha + `"}}}`))
default:
http.NotFound(w, r)
}
}))
defer server.Close()
client, err := weknora.NewClient(weknora.ClientConfig{BaseURL: server.URL, APIKey: "test-api-key", Timeout: time.Second})
if err != nil {
t.Fatal(err)
}
now := time.Date(2026, time.September, 4, 3, 10, 0, 0, time.UTC)
store := &uploadStoreStub{candidate: postgresstore.PublicationForUpload{
ID: "publication-1", TestKBID: "kb-1", IngestState: governance.IngestPending,
ProcessingState: governance.ProcessingPending, SHA256: sha, SizeBytes: int64(len(body)),
ValidatedFormat: "md", LocalSpoolRef: "objects/artifact.md", Title: "Safe title",
}}
handler, err := NewUploadHandler(UploadHandlerConfig{
Store: store, Client: client, SpoolRoot: root, MaxUploadBytes: 50 * 1024 * 1024,
FirstPollDelay: time.Second, Now: func() time.Time { return now },
})
if err != nil {
t.Fatal(err)
}
if err := handler.Handle(context.Background(), jobs.Job{Payload: []byte(`{"publication_id":"publication-1"}`)}); err != nil {
t.Fatalf("Handle() error = %v", err)
}
if uploads.Load() != 1 || store.recordedKnowledgeID != "knowledge-recovered" {
t.Fatalf("uploads=%d knowledge=%q", uploads.Load(), store.recordedKnowledgeID)
}
}
func TestUploadHandlerRecoversUploadingStateBeforeAnotherPost(t *testing.T) {
root := t.TempDir()
if err := os.Mkdir(filepath.Join(root, "objects"), 0o700); err != nil {
t.Fatal(err)
}
body := []byte("safe retry recovery\n")
if err := os.WriteFile(filepath.Join(root, "objects", "artifact.md"), body, 0o600); err != nil {
t.Fatal(err)
}
digest := sha256.Sum256(body)
sha := hex.EncodeToString(digest[:])
var uploads atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch {
case r.Method == http.MethodPost:
uploads.Add(1)
http.Error(w, "must not upload", http.StatusInternalServerError)
case r.Method == http.MethodGet && r.URL.Path == "/api/v1/knowledge-bases/kb-1/knowledge":
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"success":true,"data":[{"id":"knowledge-existing","knowledge_base_id":"kb-1","metadata":{"publication_id":"publication-1","content_sha256":"` + sha + `"}}],"total":1,"page":1,"page_size":100}`))
case r.Method == http.MethodGet && r.URL.Path == "/api/v1/knowledge/knowledge-existing":
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"success":true,"data":{"id":"knowledge-existing","knowledge_base_id":"kb-1","metadata":{"publication_id":"publication-1","content_sha256":"` + sha + `"}}}`))
default:
http.NotFound(w, r)
}
}))
defer server.Close()
client, err := weknora.NewClient(weknora.ClientConfig{BaseURL: server.URL, APIKey: "test-api-key", Timeout: time.Second})
if err != nil {
t.Fatal(err)
}
now := time.Date(2026, time.September, 4, 3, 20, 0, 0, time.UTC)
store := &uploadStoreStub{candidate: postgresstore.PublicationForUpload{
ID: "publication-1", TestKBID: "kb-1", IngestState: governance.IngestUploading,
ProcessingState: governance.ProcessingPending, SHA256: sha, SizeBytes: int64(len(body)),
ValidatedFormat: "md", LocalSpoolRef: "objects/artifact.md", Title: "Safe title",
}}
handler, err := NewUploadHandler(UploadHandlerConfig{
Store: store, Client: client, SpoolRoot: root, MaxUploadBytes: 50 * 1024 * 1024,
FirstPollDelay: time.Second, Now: func() time.Time { return now },
})
if err != nil {
t.Fatal(err)
}
if err := handler.Handle(context.Background(), jobs.Job{Payload: []byte(`{"publication_id":"publication-1"}`)}); err != nil {
t.Fatalf("Handle() error = %v", err)
}
if uploads.Load() != 0 || store.recordedKnowledgeID != "knowledge-existing" {
t.Fatalf("uploads=%d knowledge=%q", uploads.Load(), store.recordedKnowledgeID)
}
}
func TestUploadHandlerFailsClosedWhenAmbiguousRecoveryHasMultipleMatches(t *testing.T) {
root := t.TempDir()
if err := os.Mkdir(filepath.Join(root, "objects"), 0o700); err != nil {
t.Fatal(err)
}
body := []byte("safe ambiguous duplicate\n")
if err := os.WriteFile(filepath.Join(root, "objects", "artifact.md"), body, 0o600); err != nil {
t.Fatal(err)
}
digest := sha256.Sum256(body)
sha := hex.EncodeToString(digest[:])
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch {
case r.Method == http.MethodPost:
_, _ = io.Copy(io.Discard, r.Body)
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusConflict)
_, _ = w.Write([]byte(`{"success":false,"data":{}}`))
case r.Method == http.MethodGet && r.URL.Path == "/api/v1/knowledge-bases/kb-1/knowledge":
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"success":true,"data":[` +
`{"id":"knowledge-a","knowledge_base_id":"kb-1","metadata":{"publication_id":"publication-1","content_sha256":"` + sha + `"}},` +
`{"id":"knowledge-b","knowledge_base_id":"kb-1","metadata":{"publication_id":"publication-1","content_sha256":"` + sha + `"}}` +
`],"total":2,"page":1,"page_size":100}`))
default:
http.NotFound(w, r)
}
}))
defer server.Close()
client, err := weknora.NewClient(weknora.ClientConfig{BaseURL: server.URL, APIKey: "test-api-key", Timeout: time.Second})
if err != nil {
t.Fatal(err)
}
now := time.Date(2026, time.September, 4, 3, 30, 0, 0, time.UTC)
store := &uploadStoreStub{candidate: postgresstore.PublicationForUpload{
ID: "publication-1", TestKBID: "kb-1", IngestState: governance.IngestPending,
ProcessingState: governance.ProcessingPending, SHA256: sha, SizeBytes: int64(len(body)),
ValidatedFormat: "md", LocalSpoolRef: "objects/artifact.md", Title: "Safe title",
}}
handler, err := NewUploadHandler(UploadHandlerConfig{
Store: store, Client: client, SpoolRoot: root, MaxUploadBytes: 50 * 1024 * 1024,
FirstPollDelay: time.Second, Now: func() time.Time { return now },
})
if err != nil {
t.Fatal(err)
}
if err := handler.Handle(context.Background(), jobs.Job{Payload: []byte(`{"publication_id":"publication-1"}`)}); err != nil {
t.Fatalf("Handle() error = %v", err)
}
if store.uploadFailureCode != "upload_identity_ambiguous" || store.recordedKnowledgeID != "" {
t.Fatalf("failure=%q knowledge=%q", store.uploadFailureCode, store.recordedKnowledgeID)
}
}
func TestUploadHandlerRecordsInvalidArtifactForManualRetry(t *testing.T) {
root := t.TempDir()
store := &uploadStoreStub{candidate: postgresstore.PublicationForUpload{
ID: "publication-1", TestKBID: "kb-1", IngestState: governance.IngestPending,
ProcessingState: governance.ProcessingPending, SHA256: strings.Repeat("a", 64), SizeBytes: 128,
ValidatedFormat: "md", LocalSpoolRef: "missing/artifact.md", Title: "Safe title",
}}
handler, err := NewUploadHandler(UploadHandlerConfig{
Store: store, Client: uploadClientStub{}, SpoolRoot: root, MaxUploadBytes: 1024,
FirstPollDelay: time.Second,
})
if err != nil {
t.Fatal(err)
}
if err := handler.Handle(context.Background(), jobs.Job{Payload: []byte(`{"publication_id":"publication-1"}`)}); err != nil {
t.Fatalf("Handle() error = %v, want terminal failure recorded", err)
}
if store.uploadFailureCode != "upload_artifact_invalid" || store.markedUploading {
t.Fatalf("failure=%q markedUploading=%v", store.uploadFailureCode, store.markedUploading)
}
}
type uploadStoreStub struct {
candidate postgresstore.PublicationForUpload
markedUploading bool
recordedKnowledgeID string
nextPollAt time.Time
uploadFailureCode string
}
func (store *uploadStoreStub) LoadPublicationForUpload(context.Context, string) (postgresstore.PublicationForUpload, error) {
return store.candidate, nil
}
func (store *uploadStoreStub) MarkUploading(context.Context, string, time.Time) error {
store.markedUploading = true
return nil
}
func (store *uploadStoreStub) RecordUploadAndSchedulePoll(_ context.Context, _ string, knowledgeID string, _ time.Time, nextAt time.Time) (int, error) {
store.recordedKnowledgeID = knowledgeID
store.nextPollAt = nextAt
return 1, nil
}
func (store *uploadStoreStub) RecordUploadFailure(_ context.Context, _ string, code string, _ time.Time) error {
store.uploadFailureCode = code
return nil
}
type uploadClientStub struct{}
func (uploadClientStub) UploadFile(context.Context, weknora.UploadRequest) (weknora.Knowledge, error) {
return weknora.Knowledge{}, errors.New("unexpected upload")
}
func (uploadClientStub) ListKnowledge(context.Context, string, int, int) ([]weknora.Knowledge, int64, error) {
return nil, 0, errors.New("unexpected list")
}
func (uploadClientStub) GetKnowledge(context.Context, string) (weknora.Knowledge, error) {
return weknora.Knowledge{}, errors.New("unexpected get")
}
+143
View File
@@ -0,0 +1,143 @@
package testweknora
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"time"
"knowledge-governance/internal/governance"
"knowledge-governance/internal/jobs"
postgresstore "knowledge-governance/internal/platform/postgres"
"knowledge-governance/internal/platform/weknora"
)
var (
ErrInvalidValidateJobPayload = errors.New("test_validate_invalid_payload")
ErrValidateStateConflict = errors.New("test_validate_state_conflict")
ErrValidateRecordFailed = errors.New("test_validate_record_failed")
)
type ValidateStore interface {
LoadPublicationForValidation(context.Context, string) (postgresstore.PublicationForValidation, error)
AppendAcceptanceResult(context.Context, governance.AcceptanceResult) error
}
type ValidateClient interface {
GetKnowledge(context.Context, string) (weknora.Knowledge, error)
ListChunks(context.Context, string, int, int, []string) ([]weknora.Chunk, int64, error)
}
type ValidateHandlerConfig struct {
Store ValidateStore
Client ValidateClient
RunID string
PolicyVersion string
Now func() time.Time
}
type ValidateHandler struct {
store ValidateStore
client ValidateClient
runID string
policyVersion string
now func() time.Time
}
type validatePayload struct {
PublicationID string `json:"publication_id"`
}
func NewValidateHandler(config ValidateHandlerConfig) (*ValidateHandler, error) {
if config.Store == nil || config.Client == nil || config.PolicyVersion == "" {
return nil, errors.New("invalid test validate handler configuration")
}
if config.RunID == "" {
config.RunID = "default"
}
if config.Now == nil {
config.Now = time.Now
}
return &ValidateHandler{
store: config.Store, client: config.Client, runID: config.RunID,
policyVersion: config.PolicyVersion, now: config.Now,
}, nil
}
func (handler *ValidateHandler) Handle(ctx context.Context, job jobs.Job) error {
var payload validatePayload
if err := decodeStrictJSON(job.Payload, &payload); err != nil || payload.PublicationID == "" {
return ErrInvalidValidateJobPayload
}
publication, err := handler.store.LoadPublicationForValidation(ctx, payload.PublicationID)
if err != nil {
return ErrValidateStateConflict
}
now := handler.now()
runID := handler.runID
if job.ID != "" {
runID = job.ID
}
knowledge, knowledgeErr := handler.client.GetKnowledge(ctx, publication.KnowledgeID)
knowledgeState := governance.AcceptancePassed
knowledgeEvidence := json.RawMessage(`{"matched":true}`)
if knowledgeErr != nil || knowledge.ID != publication.KnowledgeID || knowledge.KnowledgeBaseID != publication.TestKBID {
knowledgeState = governance.AcceptanceFailed
knowledgeEvidence = json.RawMessage(`{"matched":false}`)
}
if err := handler.append(ctx, publication.ID, runID, "knowledge_exists", knowledgeState, knowledgeEvidence, now); err != nil {
return err
}
_, chunkTotal, chunksErr := handler.client.ListChunks(ctx, publication.KnowledgeID, 1, 1, nil)
chunksState := governance.AcceptancePassed
if chunksErr != nil || chunkTotal <= 0 {
chunksState = governance.AcceptanceFailed
}
if err := handler.append(ctx, publication.ID, runID, "chunks_present", chunksState, mustJSON(map[string]any{"chunk_total": chunkTotal}), now); err != nil {
return err
}
if err := handler.append(ctx, publication.ID, runID, "retrieval_hit", governance.AcceptanceManualRequired, json.RawMessage(`{"reason":"thresholds_unresolved"}`), now); err != nil {
return err
}
return nil
}
func (handler *ValidateHandler) append(ctx context.Context, publicationID, runID, checkType string, state governance.AcceptanceState, evidence json.RawMessage, observedAt time.Time) error {
result := governance.AcceptanceResult{
ID: acceptanceResultID(publicationID, runID, checkType),
PublicationID: publicationID,
RunID: runID,
CheckType: checkType,
State: state,
PolicyVersion: handler.policyVersion,
EvidenceSummary: evidence,
ObservedAt: observedAt,
}
if err := handler.store.AppendAcceptanceResult(ctx, result); err != nil {
return ErrValidateRecordFailed
}
return nil
}
func acceptanceResultID(publicationID, runID, checkType string) string {
digest := sha256.New()
_, _ = digest.Write([]byte(publicationID))
_, _ = digest.Write([]byte{0})
_, _ = digest.Write([]byte(runID))
_, _ = digest.Write([]byte{0})
_, _ = digest.Write([]byte(checkType))
return "acc-" + hex.EncodeToString(digest.Sum(nil))[:32]
}
func mustJSON(value any) json.RawMessage {
data, err := json.Marshal(value)
if err != nil {
return json.RawMessage(`{}`)
}
return data
}
@@ -0,0 +1,118 @@
package testweknora
import (
"context"
"testing"
"time"
"knowledge-governance/internal/governance"
"knowledge-governance/internal/jobs"
postgresstore "knowledge-governance/internal/platform/postgres"
"knowledge-governance/internal/platform/weknora"
)
func TestValidateHandlerAppendsDeterministicChecksAndManualRetrieval(t *testing.T) {
client := &validateClientStub{
knowledge: weknora.Knowledge{ID: "knowledge-1", KnowledgeBaseID: "kb-1"},
chunkTotal: 3,
}
store := &validateStoreStub{publication: postgresstore.PublicationForValidation{
ID: "publication-1", TestKBID: "kb-1", KnowledgeID: "knowledge-1",
}}
now := time.Date(2026, time.September, 4, 4, 10, 0, 0, time.UTC)
handler, err := NewValidateHandler(ValidateHandlerConfig{
Store: store, Client: client, RunID: "run-1", PolicyVersion: "structural-v1",
Now: func() time.Time { return now },
})
if err != nil {
t.Fatal(err)
}
if err := handler.Handle(context.Background(), jobs.Job{Payload: []byte(`{"publication_id":"publication-1"}`)}); err != nil {
t.Fatalf("Handle() error = %v", err)
}
if len(store.appended) != 3 {
t.Fatalf("appended = %d, want 3: %#v", len(store.appended), store.appended)
}
byType := map[string]governance.AcceptanceResult{}
for _, result := range store.appended {
byType[result.CheckType] = result
}
if byType["knowledge_exists"].State != governance.AcceptancePassed {
t.Fatalf("knowledge_exists = %#v", byType["knowledge_exists"])
}
if byType["chunks_present"].State != governance.AcceptancePassed {
t.Fatalf("chunks_present = %#v", byType["chunks_present"])
}
if byType["retrieval_hit"].State != governance.AcceptanceManualRequired {
t.Fatalf("retrieval_hit = %#v", byType["retrieval_hit"])
}
for _, result := range store.appended {
if result.PublicationID != "publication-1" || result.RunID != "run-1" || result.ObservedAt != now {
t.Fatalf("acceptance metadata = %#v", result)
}
}
}
func TestValidateHandlerFailsChecksWithoutAborting(t *testing.T) {
client := &validateClientStub{
knowledge: weknora.Knowledge{ID: "knowledge-mismatch", KnowledgeBaseID: "kb-other"},
chunkTotal: 0,
}
store := &validateStoreStub{publication: postgresstore.PublicationForValidation{
ID: "publication-1", TestKBID: "kb-1", KnowledgeID: "knowledge-1",
}}
handler, err := NewValidateHandler(ValidateHandlerConfig{
Store: store, Client: client, RunID: "run-2", PolicyVersion: "structural-v1",
Now: func() time.Time { return time.Date(2026, time.September, 4, 4, 11, 0, 0, time.UTC) },
})
if err != nil {
t.Fatal(err)
}
if err := handler.Handle(context.Background(), jobs.Job{Payload: []byte(`{"publication_id":"publication-1"}`)}); err != nil {
t.Fatalf("Handle() error = %v", err)
}
if len(store.appended) != 3 {
t.Fatalf("appended = %d, want 3", len(store.appended))
}
byType := map[string]governance.AcceptanceResult{}
for _, result := range store.appended {
byType[result.CheckType] = result
}
if byType["knowledge_exists"].State != governance.AcceptanceFailed || byType["chunks_present"].State != governance.AcceptanceFailed {
t.Fatalf("failed checks = %#v", store.appended)
}
if byType["retrieval_hit"].State != governance.AcceptanceManualRequired {
t.Fatalf("retrieval still manual = %#v", byType["retrieval_hit"])
}
}
type validateStoreStub struct {
publication postgresstore.PublicationForValidation
appended []governance.AcceptanceResult
}
func (store *validateStoreStub) LoadPublicationForValidation(context.Context, string) (postgresstore.PublicationForValidation, error) {
return store.publication, nil
}
func (store *validateStoreStub) AppendAcceptanceResult(_ context.Context, result governance.AcceptanceResult) error {
store.appended = append(store.appended, result)
return nil
}
type validateClientStub struct {
knowledge weknora.Knowledge
chunkTotal int64
}
func (client *validateClientStub) GetKnowledge(context.Context, string) (weknora.Knowledge, error) {
return client.knowledge, nil
}
func (client *validateClientStub) ListChunks(context.Context, string, int, int, []string) ([]weknora.Chunk, int64, error) {
chunks := make([]weknora.Chunk, 0, client.chunkTotal)
for i := int64(0); i < client.chunkTotal; i++ {
chunks = append(chunks, weknora.Chunk{ID: "chunk", KnowledgeID: "knowledge-1"})
}
return chunks, client.chunkTotal, nil
}