feat: 知识治理平台(含已验证发布/回滚契约)
全栈知识治理平台首次入库(P1-P10 + 发布链路模拟验证闭环): - 治理核心:源目录/变更集/人工审核/发布批次/发布执行(fail-closed 闸口) - 测试通道:WeKnora 测试 KB 上传/轮询/验收;poll 完成时写入 parser_generation/index_generation,修复 manual/seed 路径发布闸口永久拦截 - 发布适配器 v071(本地 source=测试角色 / 云端 target=生产角色 模拟验证通过): * ES 导出/导入 mapping 自适应(.keyword 子字段、可选 tenant_id) * OSS locator 支持嵌套 scheme;chunks.seq_id 由目标序列分配 * bundle 归档同身份重试原子替换;verifier 按 bundle chunk_type 计数 * 回滚 job payload 支持 prior_execution_id(strict 解码) - 三通道发布+回滚演练实测:publish 4/4 verified,检索命中, rollback 4/4 rolled_back,目标端 PG is_enabled=false、ES is_enabled=false - 前端 A·Data-console 风格全站统一,120 用例通过 - 门禁:go vet / go test 全包通过,gofmt 干净
This commit is contained in:
@@ -0,0 +1,191 @@
|
||||
package catalogrefresh
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode"
|
||||
|
||||
"knowledge-governance/internal/jobs"
|
||||
)
|
||||
|
||||
type Request struct {
|
||||
ConnectionID string `json:"connection_id"`
|
||||
SourceType string `json:"source_type"`
|
||||
SourceName string `json:"source_name"`
|
||||
Stage string `json:"stage"`
|
||||
Progress int `json:"progress"`
|
||||
Message string `json:"message"`
|
||||
Processed int `json:"processed"`
|
||||
Total int `json:"total"`
|
||||
}
|
||||
type Progress struct {
|
||||
Stage string `json:"stage"`
|
||||
Progress int `json:"progress"`
|
||||
Message string `json:"message"`
|
||||
Processed int `json:"processed"`
|
||||
Total int `json:"total"`
|
||||
SourceName string `json:"source_name,omitempty"`
|
||||
CurrentDirectory string `json:"current_directory,omitempty"`
|
||||
ScannedDirectories int `json:"scanned_directories,omitempty"`
|
||||
DiscoveredDirectories int `json:"discovered_directories,omitempty"`
|
||||
ScannedFiles int `json:"scanned_files,omitempty"`
|
||||
EstimatedTotalFiles int `json:"estimated_total_files,omitempty"`
|
||||
TotalIsEstimate bool `json:"total_is_estimate,omitempty"`
|
||||
}
|
||||
type Store interface {
|
||||
UpdateCatalogRefreshProgress(context.Context, string, string, Progress, time.Time) error
|
||||
}
|
||||
|
||||
var progressForbiddenSubstrings = []string{"http://", "https://", "token", "secret", "password", "authorization", "bearer", "cookie", "/run/source-secrets", "api-key", "apikey"}
|
||||
|
||||
type Runner interface {
|
||||
Run(context.Context, Request, func(Progress) error) error
|
||||
}
|
||||
type Config struct {
|
||||
Store Store
|
||||
Runner Runner
|
||||
Now func() time.Time
|
||||
}
|
||||
type Handler struct {
|
||||
store Store
|
||||
runner Runner
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
func NewHandler(config Config) (*Handler, error) {
|
||||
if config.Store == nil || config.Runner == nil {
|
||||
return nil, errors.New("catalog refresh dependencies are required")
|
||||
}
|
||||
if config.Now == nil {
|
||||
config.Now = time.Now
|
||||
}
|
||||
return &Handler{store: config.Store, runner: config.Runner, now: config.Now}, nil
|
||||
}
|
||||
func (handler *Handler) Handle(ctx context.Context, job jobs.Job) error {
|
||||
var request Request
|
||||
decoder := json.NewDecoder(bytes.NewReader(job.Payload))
|
||||
decoder.DisallowUnknownFields()
|
||||
if err := decoder.Decode(&request); err != nil || request.ConnectionID == "" || request.SourceName == "" || (request.SourceType != "pp" && request.SourceType != "dingtalk" && request.SourceType != "alist") || job.LeaseOwner == "" {
|
||||
return errors.New("catalog_refresh_invalid_payload")
|
||||
}
|
||||
var last Progress
|
||||
return handler.runner.Run(ctx, request, func(progress Progress) error {
|
||||
progress = clampProgress(last, progress)
|
||||
if !validProgress(progress) {
|
||||
return errors.New("catalog_refresh_invalid_progress")
|
||||
}
|
||||
last = progress
|
||||
return handler.store.UpdateCatalogRefreshProgress(ctx, job.ID, job.LeaseOwner, progress, handler.now().UTC())
|
||||
})
|
||||
}
|
||||
func validProgress(progress Progress) bool {
|
||||
if progress.Progress < 0 || progress.Progress > 100 || progress.Processed < 0 || progress.Total < 0 ||
|
||||
progress.ScannedDirectories < 0 || progress.DiscoveredDirectories < 0 || progress.ScannedFiles < 0 || progress.EstimatedTotalFiles < 0 ||
|
||||
strings.TrimSpace(progress.Stage) == "" || len(progress.Stage) > 32 ||
|
||||
strings.TrimSpace(progress.Message) == "" || len(progress.Message) > 300 ||
|
||||
len(progress.CurrentDirectory) > 1024 || len(progress.SourceName) > 255 {
|
||||
return false
|
||||
}
|
||||
for _, text := range []string{progress.Stage, progress.Message, progress.CurrentDirectory, progress.SourceName} {
|
||||
for _, r := range text {
|
||||
if unicode.IsControl(r) && r != ' ' {
|
||||
return false
|
||||
}
|
||||
}
|
||||
lowered := strings.ToLower(text)
|
||||
for _, forbidden := range progressForbiddenSubstrings {
|
||||
if strings.Contains(lowered, forbidden) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// clampProgress enforces that display progress never regresses when discovered
|
||||
// totals shift mid-scan (BFS estimates grow), and never clamps an explicit
|
||||
// failure snapshot.
|
||||
func clampProgress(previous, next Progress) Progress {
|
||||
if next.Stage == "failed" {
|
||||
return next
|
||||
}
|
||||
if next.Total < previous.Total && previous.Total >= next.Processed {
|
||||
next.Total = previous.Total
|
||||
}
|
||||
if next.Total > 0 {
|
||||
estimated := next.Processed * 100 / next.Total
|
||||
if next.Progress < previous.Progress && previous.Progress <= estimated {
|
||||
next.Progress = previous.Progress
|
||||
}
|
||||
}
|
||||
if next.ScannedDirectories < previous.ScannedDirectories {
|
||||
next.ScannedDirectories = previous.ScannedDirectories
|
||||
}
|
||||
if next.ScannedFiles < previous.ScannedFiles {
|
||||
next.ScannedFiles = previous.ScannedFiles
|
||||
}
|
||||
if next.DiscoveredDirectories < previous.DiscoveredDirectories {
|
||||
next.DiscoveredDirectories = previous.DiscoveredDirectories
|
||||
}
|
||||
if next.EstimatedTotalFiles < previous.EstimatedTotalFiles {
|
||||
next.EstimatedTotalFiles = previous.EstimatedTotalFiles
|
||||
}
|
||||
return next
|
||||
}
|
||||
|
||||
type CommandRunner struct {
|
||||
PythonPath string
|
||||
ScriptPath string
|
||||
}
|
||||
|
||||
func NewCommandRunner(pythonPath, scriptPath string) (*CommandRunner, error) {
|
||||
if !filepath.IsAbs(pythonPath) || !filepath.IsAbs(scriptPath) {
|
||||
return nil, errors.New("catalog refresh command paths must be absolute")
|
||||
}
|
||||
return &CommandRunner{PythonPath: pythonPath, ScriptPath: scriptPath}, nil
|
||||
}
|
||||
func (runner *CommandRunner) Run(ctx context.Context, request Request, report func(Progress) error) error {
|
||||
command := exec.CommandContext(ctx, runner.PythonPath, runner.ScriptPath, "--connection-id", request.ConnectionID, "--source-type", request.SourceType, "--source-name", request.SourceName)
|
||||
stdout, err := command.StdoutPipe()
|
||||
if err != nil {
|
||||
return errors.New("catalog_refresh_start_failed")
|
||||
}
|
||||
command.Stderr = io.Discard
|
||||
if err = command.Start(); err != nil {
|
||||
return errors.New("catalog_refresh_start_failed")
|
||||
}
|
||||
scanner := bufio.NewScanner(stdout)
|
||||
scanner.Buffer(make([]byte, 4096), 1<<20)
|
||||
for scanner.Scan() {
|
||||
decoder := json.NewDecoder(bytes.NewReader(scanner.Bytes()))
|
||||
decoder.DisallowUnknownFields()
|
||||
var progress Progress
|
||||
if decoder.Decode(&progress) != nil || !validProgress(progress) {
|
||||
_ = command.Process.Kill()
|
||||
_ = command.Wait()
|
||||
return errors.New("catalog_refresh_invalid_output")
|
||||
}
|
||||
if err = report(progress); err != nil {
|
||||
_ = command.Process.Kill()
|
||||
_ = command.Wait()
|
||||
return err
|
||||
}
|
||||
}
|
||||
if scanner.Err() != nil {
|
||||
_ = command.Process.Kill()
|
||||
_ = command.Wait()
|
||||
return errors.New("catalog_refresh_output_failed")
|
||||
}
|
||||
if err = command.Wait(); err != nil {
|
||||
return errors.New("catalog_refresh_failed")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
package catalogrefresh
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"knowledge-governance/internal/jobs"
|
||||
)
|
||||
|
||||
type fakeStore struct{ updates []Progress }
|
||||
|
||||
func (s *fakeStore) UpdateCatalogRefreshProgress(_ context.Context, _ string, _ string, p Progress, _ time.Time) error {
|
||||
s.updates = append(s.updates, p)
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestHandlerClampsMonotonicProgressAndKeepsFailuresVisible(t *testing.T) {
|
||||
store := &fakeStore{}
|
||||
handler, err := NewHandler(Config{Store: store, Runner: fakeRunner{}, Now: func() time.Time { return time.Now() }})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runner := func(ctx context.Context, _ Request, report func(Progress) error) error {
|
||||
_ = report(Progress{Stage: "scanning", Progress: 40, Message: "扫描中", Processed: 60, Total: 100})
|
||||
// Discovered more directories than estimated: total must not shrink, percent must not regress.
|
||||
_ = report(Progress{Stage: "scanning", Progress: 30, Message: "发现更多目录", Processed: 70, Total: 50})
|
||||
return report(Progress{Stage: "failed", Progress: 0, Message: "来源不可用"})
|
||||
}
|
||||
custom := *handler
|
||||
custom.runner = runnerFunc(runner)
|
||||
job := jobs.Job{ID: "job-1", LeaseOwner: "worker-1", Payload: []byte(`{"connection_id":"conn-1","source_type":"alist","source_name":"国内工业","stage":"queued","progress":0,"message":"等待后台刷新","processed":0,"total":0}`)}
|
||||
if err = custom.Handle(context.Background(), job); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(store.updates) != 3 {
|
||||
t.Fatalf("updates=%#v", store.updates)
|
||||
}
|
||||
if store.updates[1].Progress < 40 || store.updates[1].Total < 70 {
|
||||
t.Fatalf("regression not clamped: %#v", store.updates[1])
|
||||
}
|
||||
if store.updates[2].Stage != "failed" || store.updates[2].Progress != 0 {
|
||||
t.Fatalf("failure must keep its own state: %#v", store.updates[2])
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidProgressRejectsControlCharsURLsAndCredentials(t *testing.T) {
|
||||
base := Progress{Stage: "scanning", Progress: 50, Message: "正在扫描", Processed: 10, Total: 20}
|
||||
if !validProgress(base) {
|
||||
t.Fatal("baseline rejected")
|
||||
}
|
||||
for name, mutate := range map[string]func(*Progress){
|
||||
"control": func(p *Progress) { p.CurrentDirectory = "/产品\a资料" },
|
||||
"url": func(p *Progress) { p.CurrentDirectory = "https://evil.invalid/x" },
|
||||
"url-message": func(p *Progress) { p.Message = "访问 https://evil.invalid 失败" },
|
||||
"token": func(p *Progress) { p.Message = "token=abc123 已过期" },
|
||||
"too-long": func(p *Progress) { p.CurrentDirectory = strings.Repeat("目", 1100) },
|
||||
"negative": func(p *Progress) { p.ScannedFiles = -1 },
|
||||
"secret-dir": func(p *Progress) { p.CurrentDirectory = "/run/source-secrets/pp" },
|
||||
} {
|
||||
item := base
|
||||
mutate(&item)
|
||||
if validProgress(item) {
|
||||
t.Errorf("%s accepted", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCommandRunnerRejectsUnknownProgressFields(t *testing.T) {
|
||||
python, err := exec.LookPath("python3")
|
||||
if err != nil {
|
||||
t.Skip("python3 unavailable")
|
||||
}
|
||||
runner, err := NewCommandRunner(python, writeFakeProgressScript(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = runner.Run(context.Background(), Request{ConnectionID: "c", SourceType: "pp", SourceName: "n"}, func(Progress) error { return nil })
|
||||
if err == nil || err.Error() != "catalog_refresh_invalid_output" {
|
||||
t.Fatalf("unknown field must fail: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func writeFakeProgressScript(t *testing.T) string {
|
||||
t.Helper()
|
||||
directory := t.TempDir()
|
||||
path := filepath.Join(directory, "progress.py")
|
||||
body := "import json\nprint(json.dumps({'stage':'scanning','progress':10,'message':'ok','processed':1,'total':2,'unexpected_field':'leak'}),flush=True)\n"
|
||||
if err := os.WriteFile(path, []byte(body), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
type runnerFunc func(context.Context, Request, func(Progress) error) error
|
||||
|
||||
func (f runnerFunc) Run(ctx context.Context, request Request, report func(Progress) error) error {
|
||||
return f(ctx, request, report)
|
||||
}
|
||||
|
||||
type fakeRunner struct{}
|
||||
|
||||
func (fakeRunner) Run(_ context.Context, _ Request, report func(Progress) error) error {
|
||||
if err := report(Progress{Stage: "scanning", Progress: 50, Message: "正在扫描"}); err != nil {
|
||||
return err
|
||||
}
|
||||
return report(Progress{Stage: "completed", Progress: 100, Message: "完成"})
|
||||
}
|
||||
func TestHandlerRunsLiveRefreshAndPersistsProgress(t *testing.T) {
|
||||
store := &fakeStore{}
|
||||
handler, err := NewHandler(Config{Store: store, Runner: fakeRunner{}, Now: func() time.Time { return time.Now() }})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
job := jobs.Job{ID: "job-1", LeaseOwner: "worker-1", Payload: []byte(`{"connection_id":"conn-1","source_type":"pp","source_name":"PP资料下载中心","stage":"queued","progress":0,"message":"等待后台刷新","processed":0,"total":0}`)}
|
||||
if err = handler.Handle(context.Background(), job); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(store.updates) != 2 || store.updates[0].Stage != "scanning" || store.updates[1].Progress != 100 {
|
||||
t.Fatalf("updates=%#v", store.updates)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,299 @@
|
||||
package release
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
"knowledge-governance/internal/jobs"
|
||||
)
|
||||
|
||||
// debugReleaseItemErrors enables fail-closed-safe diagnostics: only the
|
||||
// generic step-level cause text is logged, never payload or credentials.
|
||||
var debugReleaseItemErrors = os.Getenv("RELEASE_DEBUG_ITEM_ERRORS") == "true"
|
||||
|
||||
const (
|
||||
ErrorCodeContractMismatch = "release_contract_mismatch"
|
||||
ErrorCodeBundleInvalid = "release_bundle_invalid"
|
||||
ErrorCodeTargetUnavailable = "release_target_unavailable"
|
||||
ErrorCodeTargetWriteFailed = "release_target_write_failed"
|
||||
ErrorCodeVerificationFailed = "release_verification_failed"
|
||||
ErrorCodeRollbackArtifactMissing = "release_rollback_artifact_missing"
|
||||
ErrorCodeRollbackFailed = "release_rollback_failed"
|
||||
ErrorCodeStoppedAfterFailure = "release_stopped_after_failure"
|
||||
ErrorCodeInternal = "release_internal_error"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrInvalidReleaseJob = errors.New("invalid_release_job")
|
||||
ErrReleaseRetryable = errors.New("release_retryable")
|
||||
ErrReleaseState = errors.New("release_state_failed")
|
||||
)
|
||||
|
||||
type PlanItem struct {
|
||||
ReleaseExecutionID string `json:"release_execution_id"`
|
||||
PublicationID string `json:"publication_id"`
|
||||
Action string `json:"action"`
|
||||
SourceItemVersionID string `json:"source_item_version_id"`
|
||||
SourceRevision string `json:"source_revision"`
|
||||
ParseRevision string `json:"parse_revision"`
|
||||
ReleaseRevision string `json:"release_revision"`
|
||||
TestKnowledgeBaseID string `json:"test_knowledge_base_id"`
|
||||
KnowledgeID string `json:"knowledge_id"`
|
||||
DecidedVisibility governance.Visibility `json:"decided_visibility"`
|
||||
PreviousProductionRevision string `json:"previous_production_revision"`
|
||||
}
|
||||
|
||||
type ItemResult struct {
|
||||
BundleSHA256 string
|
||||
PreviousProductionRevision string
|
||||
ProductionRevision string
|
||||
}
|
||||
|
||||
type StartExecutionRequest struct {
|
||||
ExecutionID string
|
||||
JobID string
|
||||
Operation governance.ReleaseOperation
|
||||
Now time.Time
|
||||
}
|
||||
|
||||
type CompleteItemRequest struct {
|
||||
ExecutionID string
|
||||
PublicationID string
|
||||
BundleSHA256 string
|
||||
PreviousProductionRevision string
|
||||
ProductionRevision string
|
||||
Now time.Time
|
||||
}
|
||||
|
||||
type FailItemRequest struct {
|
||||
ExecutionID string
|
||||
PublicationID string
|
||||
ErrorCode string
|
||||
Now time.Time
|
||||
}
|
||||
|
||||
type Store interface {
|
||||
StartExecution(context.Context, StartExecutionRequest) ([]PlanItem, error)
|
||||
BeginBatch(context.Context, string, int, int, time.Time) error
|
||||
CompleteItem(context.Context, CompleteItemRequest) error
|
||||
FailItem(context.Context, FailItemRequest) error
|
||||
FinishExecution(context.Context, string, time.Time) error
|
||||
ScheduleAutomaticRollback(context.Context, string, time.Time) error
|
||||
}
|
||||
|
||||
type Publisher interface {
|
||||
Publish(context.Context, PlanItem) (ItemResult, error)
|
||||
Delete(context.Context, PlanItem) (ItemResult, error)
|
||||
Rollback(context.Context, PlanItem) (ItemResult, error)
|
||||
}
|
||||
|
||||
type ItemError struct {
|
||||
Code string
|
||||
Retryable bool
|
||||
Cause error
|
||||
}
|
||||
|
||||
func (err *ItemError) Error() string {
|
||||
if err == nil || !ValidErrorCode(err.Code) {
|
||||
return ErrorCodeInternal
|
||||
}
|
||||
return err.Code
|
||||
}
|
||||
|
||||
func (err *ItemError) Unwrap() error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
return err.Cause
|
||||
}
|
||||
|
||||
type jobPayload struct {
|
||||
ExecutionID string `json:"execution_id"`
|
||||
ReleaseBatchID string `json:"release_batch_id"`
|
||||
Operation string `json:"operation"`
|
||||
PriorExecutionID string `json:"prior_execution_id"`
|
||||
}
|
||||
|
||||
func NewHandler(store Store, publisher Publisher, now func() time.Time) (jobs.Handler, error) {
|
||||
if store == nil || publisher == nil || now == nil {
|
||||
return nil, errors.New("release handler dependencies are required")
|
||||
}
|
||||
return func(ctx context.Context, job jobs.Job) error {
|
||||
payload, operation, err := parseJob(job)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
items, err := store.StartExecution(ctx, StartExecutionRequest{
|
||||
ExecutionID: payload.ExecutionID, JobID: job.ID, Operation: operation, Now: now().UTC(),
|
||||
})
|
||||
if err != nil {
|
||||
return ErrReleaseState
|
||||
}
|
||||
retryableFailure := false
|
||||
permanentFailure := false
|
||||
failureIndex := -1
|
||||
const batchSize = 50
|
||||
totalBatches := (len(items) + batchSize - 1) / batchSize
|
||||
for start := 0; start < len(items); start += batchSize {
|
||||
batchNumber := start/batchSize + 1
|
||||
if err := store.BeginBatch(ctx, payload.ExecutionID, batchNumber, totalBatches, now().UTC()); err != nil {
|
||||
return ErrReleaseState
|
||||
}
|
||||
end := start + batchSize
|
||||
if end > len(items) {
|
||||
end = len(items)
|
||||
}
|
||||
for index := start; index < end; index++ {
|
||||
item := items[index]
|
||||
result, itemErr := publishItem(ctx, publisher, operation, item)
|
||||
if itemErr != nil {
|
||||
code, retryable := classifyItemError(itemErr)
|
||||
if debugReleaseItemErrors {
|
||||
var itemErrDetail *ItemError
|
||||
causeText := ""
|
||||
if errors.As(itemErr, &itemErrDetail) && itemErrDetail != nil && itemErrDetail.Cause != nil {
|
||||
causeText = itemErrDetail.Cause.Error()
|
||||
}
|
||||
log.Printf("release item %s (%s/%s) failed code=%s cause=%s", item.PublicationID, item.Action, item.KnowledgeID, code, causeText)
|
||||
}
|
||||
if retryable && job.Attempt < job.MaxAttempts {
|
||||
retryableFailure = true
|
||||
continue
|
||||
}
|
||||
if err := store.FailItem(ctx, FailItemRequest{ExecutionID: payload.ExecutionID, PublicationID: item.PublicationID, ErrorCode: code, Now: now().UTC()}); err != nil {
|
||||
return ErrReleaseState
|
||||
}
|
||||
permanentFailure = true
|
||||
failureIndex = index
|
||||
break
|
||||
}
|
||||
if !validResult(operation, result) {
|
||||
if err := store.FailItem(ctx, FailItemRequest{ExecutionID: payload.ExecutionID, PublicationID: item.PublicationID, ErrorCode: ErrorCodeBundleInvalid, Now: now().UTC()}); err != nil {
|
||||
return ErrReleaseState
|
||||
}
|
||||
permanentFailure = true
|
||||
failureIndex = index
|
||||
break
|
||||
}
|
||||
if err := store.CompleteItem(ctx, CompleteItemRequest{
|
||||
ExecutionID: payload.ExecutionID, PublicationID: item.PublicationID,
|
||||
BundleSHA256: result.BundleSHA256, PreviousProductionRevision: result.PreviousProductionRevision,
|
||||
ProductionRevision: result.ProductionRevision, Now: now().UTC(),
|
||||
}); err != nil {
|
||||
return ErrReleaseState
|
||||
}
|
||||
}
|
||||
if permanentFailure {
|
||||
break
|
||||
}
|
||||
}
|
||||
if permanentFailure {
|
||||
for _, item := range items[failureIndex+1:] {
|
||||
if err := store.FailItem(ctx, FailItemRequest{ExecutionID: payload.ExecutionID, PublicationID: item.PublicationID, ErrorCode: ErrorCodeStoppedAfterFailure, Now: now().UTC()}); err != nil {
|
||||
return ErrReleaseState
|
||||
}
|
||||
}
|
||||
}
|
||||
if retryableFailure {
|
||||
return ErrReleaseRetryable
|
||||
}
|
||||
if err := store.FinishExecution(ctx, payload.ExecutionID, now().UTC()); err != nil {
|
||||
return ErrReleaseState
|
||||
}
|
||||
if permanentFailure && operation == governance.ReleaseOperationPublish {
|
||||
if err := store.ScheduleAutomaticRollback(ctx, payload.ExecutionID, now().UTC()); err != nil {
|
||||
return ErrReleaseState
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}, nil
|
||||
}
|
||||
|
||||
func parseJob(job jobs.Job) (jobPayload, governance.ReleaseOperation, error) {
|
||||
var payload jobPayload
|
||||
decoder := json.NewDecoder(strings.NewReader(string(job.Payload)))
|
||||
decoder.DisallowUnknownFields()
|
||||
if err := decoder.Decode(&payload); err != nil {
|
||||
return jobPayload{}, "", ErrInvalidReleaseJob
|
||||
}
|
||||
var trailing any
|
||||
if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
|
||||
return jobPayload{}, "", ErrInvalidReleaseJob
|
||||
}
|
||||
operation := governance.ReleaseOperation(payload.Operation)
|
||||
expectedType := "release." + payload.Operation
|
||||
if !validID(payload.ExecutionID, "rx-") || !validID(payload.ReleaseBatchID, "rb-") || !operation.Valid() || job.Type != expectedType || job.Attempt < 1 || job.MaxAttempts < job.Attempt {
|
||||
return jobPayload{}, "", ErrInvalidReleaseJob
|
||||
}
|
||||
return payload, operation, nil
|
||||
}
|
||||
|
||||
func publishItem(ctx context.Context, publisher Publisher, operation governance.ReleaseOperation, item PlanItem) (ItemResult, error) {
|
||||
if operation == governance.ReleaseOperationRollback {
|
||||
return publisher.Rollback(ctx, item)
|
||||
}
|
||||
if item.Action == "delete" {
|
||||
return publisher.Delete(ctx, item)
|
||||
}
|
||||
return publisher.Publish(ctx, item)
|
||||
}
|
||||
|
||||
func classifyItemError(err error) (string, bool) {
|
||||
var itemErr *ItemError
|
||||
if errors.As(err, &itemErr) && ValidErrorCode(itemErr.Code) {
|
||||
return itemErr.Code, itemErr.Retryable
|
||||
}
|
||||
return ErrorCodeInternal, true
|
||||
}
|
||||
|
||||
func validResult(operation governance.ReleaseOperation, result ItemResult) bool {
|
||||
if strings.TrimSpace(result.ProductionRevision) == "" || len(result.ProductionRevision) > 128 {
|
||||
return false
|
||||
}
|
||||
if operation == governance.ReleaseOperationPublish {
|
||||
return validSHA256(result.BundleSHA256)
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func validSHA256(value string) bool {
|
||||
if len(value) != 64 {
|
||||
return false
|
||||
}
|
||||
for _, char := range value {
|
||||
if (char < '0' || char > '9') && (char < 'a' || char > 'f') {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func validID(value, prefix string) bool {
|
||||
if !strings.HasPrefix(value, prefix) || len(value) > 36 || len(value) <= len(prefix) {
|
||||
return false
|
||||
}
|
||||
for _, char := range value {
|
||||
if (char < 'a' || char > 'z') && (char < '0' || char > '9') && char != '-' {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func ValidErrorCode(code string) bool {
|
||||
switch code {
|
||||
case ErrorCodeContractMismatch, ErrorCodeBundleInvalid, ErrorCodeTargetUnavailable,
|
||||
ErrorCodeTargetWriteFailed, ErrorCodeVerificationFailed, ErrorCodeRollbackArtifactMissing,
|
||||
ErrorCodeRollbackFailed, ErrorCodeStoppedAfterFailure, ErrorCodeInternal:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,209 @@
|
||||
package release
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
"knowledge-governance/internal/jobs"
|
||||
)
|
||||
|
||||
func TestHandlerStopsAfterPermanentItemFailureAndSchedulesAutomaticRollback(t *testing.T) {
|
||||
store := &fakeStore{items: []PlanItem{{PublicationID: "tp-1"}, {PublicationID: "tp-2"}, {PublicationID: "tp-3"}}}
|
||||
publisher := &fakePublisher{publish: map[string]publishResult{
|
||||
"tp-1": {result: ItemResult{BundleSHA256: repeatHex('a'), ProductionRevision: "prod-1"}},
|
||||
"tp-2": {err: &ItemError{Code: ErrorCodeContractMismatch, Retryable: false}},
|
||||
"tp-3": {result: ItemResult{BundleSHA256: repeatHex('b'), ProductionRevision: "prod-3"}},
|
||||
}}
|
||||
handler, err := NewHandler(store, publisher, func() time.Time { return time.Date(2026, 9, 6, 7, 0, 0, 0, time.UTC) })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
job := releaseJob("release.publish", 1, 3)
|
||||
if err := handler(context.Background(), job); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(store.completed) != 1 || store.completed[0].PublicationID != "tp-1" {
|
||||
t.Fatalf("completed=%#v", store.completed)
|
||||
}
|
||||
if len(store.failed) != 2 || store.failed[0].ErrorCode != ErrorCodeContractMismatch || store.failed[1].PublicationID != "tp-3" || store.failed[1].ErrorCode != ErrorCodeStoppedAfterFailure {
|
||||
t.Fatalf("failed=%#v", store.failed)
|
||||
}
|
||||
if store.finishedExecutionID != "rx-safe" || store.automaticRollbackExecutionID != "rx-safe" || publisher.publishCalls != 2 {
|
||||
t.Fatalf("finished=%q rollback=%q publishCalls=%d", store.finishedExecutionID, store.automaticRollbackExecutionID, publisher.publishCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandlerPersistsProgressAtFiftyItemBatchBoundaries(t *testing.T) {
|
||||
items := make([]PlanItem, 0, 105)
|
||||
results := make(map[string]publishResult, 105)
|
||||
for index := 0; index < 105; index++ {
|
||||
id := fmt.Sprintf("tp-%03d", index)
|
||||
items = append(items, PlanItem{PublicationID: id})
|
||||
results[id] = publishResult{result: ItemResult{BundleSHA256: repeatHex('a'), ProductionRevision: "prod-1"}}
|
||||
}
|
||||
store := &fakeStore{items: items}
|
||||
handler, err := NewHandler(store, &fakePublisher{publish: results}, time.Now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := handler(context.Background(), releaseJob("release.publish", 1, 3)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(store.startedBatches, [][2]int{{1, 3}, {2, 3}, {3, 3}}) {
|
||||
t.Fatalf("started batches=%v", store.startedBatches)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandlerPublishesDeleteActionAsVerifiedAbsentRevision(t *testing.T) {
|
||||
store := &fakeStore{items: []PlanItem{{PublicationID: "tp-delete", Action: "delete", PreviousProductionRevision: "prod-current"}}}
|
||||
publisher := &fakePublisher{delete: map[string]publishResult{
|
||||
"tp-delete": {result: ItemResult{BundleSHA256: repeatHex('d'), PreviousProductionRevision: "prod-current", ProductionRevision: "absent"}},
|
||||
}}
|
||||
handler, err := NewHandler(store, publisher, func() time.Time { return time.Date(2026, 9, 10, 4, 0, 0, 0, time.UTC) })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := handler(context.Background(), jobs.Job{ID: "job-delete", Type: "release.publish", Payload: []byte(`{"execution_id":"rx-delete","release_batch_id":"rb-delete","operation":"publish"}`), Attempt: 1, MaxAttempts: 3}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if publisher.deleteCalls != 1 || publisher.publishCalls != 0 || len(store.completed) != 1 || store.completed[0].ProductionRevision != "absent" {
|
||||
t.Fatalf("deleteCalls=%d publishCalls=%d completed=%#v", publisher.deleteCalls, publisher.publishCalls, store.completed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandlerReturnsOnlySanitizedRetryErrorBeforeLastAttempt(t *testing.T) {
|
||||
store := &fakeStore{items: []PlanItem{{PublicationID: "tp-1"}}}
|
||||
publisher := &fakePublisher{publish: map[string]publishResult{
|
||||
"tp-1": {err: &ItemError{Code: ErrorCodeTargetUnavailable, Retryable: true, Cause: errors.New("raw endpoint and credential text")}},
|
||||
}}
|
||||
handler, _ := NewHandler(store, publisher, time.Now)
|
||||
err := handler(context.Background(), releaseJob("release.publish", 1, 3))
|
||||
if !errors.Is(err, ErrReleaseRetryable) || err.Error() != ErrReleaseRetryable.Error() {
|
||||
t.Fatalf("error=%v", err)
|
||||
}
|
||||
if len(store.failed) != 0 || store.finishedExecutionID != "" {
|
||||
t.Fatalf("failed=%#v finished=%q", store.failed, store.finishedExecutionID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandlerFailsRetryableItemsWithFixedCodeOnLastAttempt(t *testing.T) {
|
||||
store := &fakeStore{items: []PlanItem{{PublicationID: "tp-1"}}}
|
||||
publisher := &fakePublisher{publish: map[string]publishResult{
|
||||
"tp-1": {err: &ItemError{Code: ErrorCodeTargetUnavailable, Retryable: true}},
|
||||
}}
|
||||
handler, _ := NewHandler(store, publisher, time.Now)
|
||||
if err := handler(context.Background(), releaseJob("release.publish", 3, 3)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(store.failed) != 1 || store.failed[0].ErrorCode != ErrorCodeTargetUnavailable || store.finishedExecutionID != "rx-safe" {
|
||||
t.Fatalf("failed=%#v finished=%q", store.failed, store.finishedExecutionID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandlerDispatchesRollbackAndRejectsInvalidPayload(t *testing.T) {
|
||||
store := &fakeStore{items: []PlanItem{{PublicationID: "tp-1"}}}
|
||||
publisher := &fakePublisher{rollback: map[string]publishResult{"tp-1": {result: ItemResult{ProductionRevision: "prod-old"}}}}
|
||||
handler, _ := NewHandler(store, publisher, time.Now)
|
||||
if err := handler(context.Background(), releaseJob("release.rollback", 1, 3)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if publisher.rollbackCalls != 1 || publisher.publishCalls != 0 {
|
||||
t.Fatalf("publish=%d rollback=%d", publisher.publishCalls, publisher.rollbackCalls)
|
||||
}
|
||||
bad := releaseJob("release.publish", 1, 3)
|
||||
bad.Payload = []byte(`{"execution_id":"../bad"}`)
|
||||
if err := handler(context.Background(), bad); !errors.Is(err, ErrInvalidReleaseJob) {
|
||||
t.Fatalf("error=%v", err)
|
||||
}
|
||||
// Production rollback payloads carry prior_execution_id; the strict
|
||||
// decoder must accept it (regression: release jobs failed permanently as
|
||||
// invalid_release_job the moment rollback executions included the field).
|
||||
withPrior := releaseJob("release.rollback", 1, 3)
|
||||
withPrior.Payload = []byte(`{"execution_id":"rx-safe","release_batch_id":"rb-safe","operation":"rollback","prior_execution_id":"rx-prior"}`)
|
||||
if err := handler(context.Background(), withPrior); err != nil {
|
||||
t.Fatalf("rollback payload with prior_execution_id rejected: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func releaseJob(jobType string, attempt, maxAttempts int) jobs.Job {
|
||||
payload := []byte(`{"execution_id":"rx-safe","release_batch_id":"rb-safe","operation":"publish"}`)
|
||||
if jobType == "release.rollback" {
|
||||
payload = []byte(`{"execution_id":"rx-safe","release_batch_id":"rb-safe","operation":"rollback"}`)
|
||||
}
|
||||
return jobs.Job{ID: "job-safe", Type: jobType, Payload: payload, Attempt: attempt, MaxAttempts: maxAttempts}
|
||||
}
|
||||
|
||||
type fakeStore struct {
|
||||
items []PlanItem
|
||||
completed []CompleteItemRequest
|
||||
failed []FailItemRequest
|
||||
finishedExecutionID string
|
||||
automaticRollbackExecutionID string
|
||||
startedBatches [][2]int
|
||||
}
|
||||
|
||||
func (s *fakeStore) StartExecution(context.Context, StartExecutionRequest) ([]PlanItem, error) {
|
||||
return s.items, nil
|
||||
}
|
||||
func (s *fakeStore) CompleteItem(_ context.Context, request CompleteItemRequest) error {
|
||||
s.completed = append(s.completed, request)
|
||||
return nil
|
||||
}
|
||||
func (s *fakeStore) FailItem(_ context.Context, request FailItemRequest) error {
|
||||
s.failed = append(s.failed, request)
|
||||
return nil
|
||||
}
|
||||
func (s *fakeStore) FinishExecution(_ context.Context, executionID string, _ time.Time) error {
|
||||
s.finishedExecutionID = executionID
|
||||
return nil
|
||||
}
|
||||
func (s *fakeStore) ScheduleAutomaticRollback(_ context.Context, executionID string, _ time.Time) error {
|
||||
s.automaticRollbackExecutionID = executionID
|
||||
return nil
|
||||
}
|
||||
func (s *fakeStore) BeginBatch(_ context.Context, _ string, batch, total int, _ time.Time) error {
|
||||
s.startedBatches = append(s.startedBatches, [2]int{batch, total})
|
||||
return nil
|
||||
}
|
||||
|
||||
type publishResult struct {
|
||||
result ItemResult
|
||||
err error
|
||||
}
|
||||
|
||||
type fakePublisher struct {
|
||||
publish, rollback, delete map[string]publishResult
|
||||
publishCalls, rollbackCalls, deleteCalls int
|
||||
}
|
||||
|
||||
func (p *fakePublisher) Publish(_ context.Context, item PlanItem) (ItemResult, error) {
|
||||
p.publishCalls++
|
||||
value := p.publish[item.PublicationID]
|
||||
return value.result, value.err
|
||||
}
|
||||
func (p *fakePublisher) Rollback(_ context.Context, item PlanItem) (ItemResult, error) {
|
||||
p.rollbackCalls++
|
||||
value := p.rollback[item.PublicationID]
|
||||
return value.result, value.err
|
||||
}
|
||||
|
||||
func (p *fakePublisher) Delete(_ context.Context, item PlanItem) (ItemResult, error) {
|
||||
p.deleteCalls++
|
||||
value := p.delete[item.PublicationID]
|
||||
return value.result, value.err
|
||||
}
|
||||
|
||||
func repeatHex(value byte) string {
|
||||
result := make([]byte, 64)
|
||||
for index := range result {
|
||||
result[index] = value
|
||||
}
|
||||
return string(result)
|
||||
}
|
||||
|
||||
var _ = governance.ReleaseOperationPublish
|
||||
@@ -0,0 +1,65 @@
|
||||
package release
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type RoutingPublisher struct {
|
||||
publishers map[string]Publisher
|
||||
}
|
||||
|
||||
func NewRoutingPublisher(publishers map[string]Publisher) (*RoutingPublisher, error) {
|
||||
if len(publishers) == 0 {
|
||||
return nil, errors.New("release publisher routes are required")
|
||||
}
|
||||
copied := make(map[string]Publisher, len(publishers))
|
||||
for knowledgeBaseID, publisher := range publishers {
|
||||
if !validRoutingID(knowledgeBaseID) || publisher == nil {
|
||||
return nil, errors.New("invalid release publisher route")
|
||||
}
|
||||
copied[knowledgeBaseID] = publisher
|
||||
}
|
||||
return &RoutingPublisher{publishers: copied}, nil
|
||||
}
|
||||
|
||||
func (router *RoutingPublisher) Publish(ctx context.Context, item PlanItem) (ItemResult, error) {
|
||||
publisher, ok := router.publishers[item.TestKnowledgeBaseID]
|
||||
if !ok {
|
||||
return ItemResult{}, &ItemError{Code: ErrorCodeContractMismatch, Retryable: false}
|
||||
}
|
||||
return publisher.Publish(ctx, item)
|
||||
}
|
||||
|
||||
func (router *RoutingPublisher) Rollback(ctx context.Context, item PlanItem) (ItemResult, error) {
|
||||
publisher, ok := router.publishers[item.TestKnowledgeBaseID]
|
||||
if !ok {
|
||||
return ItemResult{}, &ItemError{Code: ErrorCodeContractMismatch, Retryable: false}
|
||||
}
|
||||
return publisher.Rollback(ctx, item)
|
||||
}
|
||||
|
||||
func (router *RoutingPublisher) Delete(ctx context.Context, item PlanItem) (ItemResult, error) {
|
||||
publisher, ok := router.publishers[item.TestKnowledgeBaseID]
|
||||
if !ok {
|
||||
return ItemResult{}, &ItemError{Code: ErrorCodeContractMismatch, Retryable: false}
|
||||
}
|
||||
return publisher.Delete(ctx, item)
|
||||
}
|
||||
|
||||
func validRoutingID(value string) bool {
|
||||
value = strings.TrimSpace(value)
|
||||
if value == "" || len(value) > 128 {
|
||||
return false
|
||||
}
|
||||
for _, char := range value {
|
||||
if (char >= 'a' && char <= 'z') || (char >= 'A' && char <= 'Z') || (char >= '0' && char <= '9') || char == '-' || char == '_' || char == '.' {
|
||||
continue
|
||||
}
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
var _ Publisher = (*RoutingPublisher)(nil)
|
||||
@@ -0,0 +1,52 @@
|
||||
package release
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestRoutingPublisherSelectsExactKnowledgeBase(t *testing.T) {
|
||||
first := &routingPublisherStub{result: ItemResult{ProductionRevision: "prod-a"}}
|
||||
second := &routingPublisherStub{result: ItemResult{ProductionRevision: "prod-b"}}
|
||||
router, err := NewRoutingPublisher(map[string]Publisher{"kb-a": first, "kb-b": second})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
item := testPlanItem()
|
||||
item.TestKnowledgeBaseID = "kb-b"
|
||||
result, err := router.Publish(context.Background(), item)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.ProductionRevision != "prod-b" || first.calls != 0 || second.calls != 1 {
|
||||
t.Fatalf("result=%#v calls=%d/%d", result, first.calls, second.calls)
|
||||
}
|
||||
if _, err := router.Delete(context.Background(), item); err != nil || first.deleteCalls != 0 || second.deleteCalls != 1 {
|
||||
t.Fatalf("delete error=%v deleteCalls=%d/%d", err, first.deleteCalls, second.deleteCalls)
|
||||
}
|
||||
item.TestKnowledgeBaseID = "unknown"
|
||||
if _, err := router.Publish(context.Background(), item); err == nil || err.Error() != ErrorCodeContractMismatch {
|
||||
t.Fatalf("error=%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
type routingPublisherStub struct {
|
||||
result ItemResult
|
||||
err error
|
||||
calls int
|
||||
deleteCalls int
|
||||
}
|
||||
|
||||
func (s *routingPublisherStub) Publish(context.Context, PlanItem) (ItemResult, error) {
|
||||
s.calls++
|
||||
return s.result, s.err
|
||||
}
|
||||
func (s *routingPublisherStub) Rollback(context.Context, PlanItem) (ItemResult, error) {
|
||||
s.calls++
|
||||
return s.result, s.err
|
||||
}
|
||||
|
||||
func (s *routingPublisherStub) Delete(context.Context, PlanItem) (ItemResult, error) {
|
||||
s.deleteCalls++
|
||||
return s.result, s.err
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
package release
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
|
||||
"knowledge-governance/internal/releasebundle"
|
||||
)
|
||||
|
||||
type BundleExporter interface {
|
||||
Export(ctx context.Context, item PlanItem) (ExportedBundle, error)
|
||||
}
|
||||
|
||||
type ExportedBundle interface {
|
||||
ReleaseManifest() releasebundle.Manifest
|
||||
}
|
||||
|
||||
type PreparedChange struct {
|
||||
PreviousRevision string
|
||||
ProductionRevision string
|
||||
}
|
||||
|
||||
type SidecarTarget interface {
|
||||
Contract(ctx context.Context, item PlanItem) (releasebundle.TargetContract, error)
|
||||
Prepare(ctx context.Context, item PlanItem, bundle ExportedBundle) (PreparedChange, error)
|
||||
Activate(ctx context.Context, item PlanItem, change PreparedChange) error
|
||||
Verify(ctx context.Context, item PlanItem, change PreparedChange) error
|
||||
Compensate(ctx context.Context, item PlanItem, change PreparedChange) error
|
||||
Restore(ctx context.Context, item PlanItem, revision string) (string, error)
|
||||
VerifyRestored(ctx context.Context, item PlanItem, revision string) error
|
||||
}
|
||||
|
||||
type SidecarPublisher struct {
|
||||
exporter BundleExporter
|
||||
target SidecarTarget
|
||||
}
|
||||
|
||||
func NewSidecarPublisher(exporter BundleExporter, target SidecarTarget) (*SidecarPublisher, error) {
|
||||
if exporter == nil || target == nil {
|
||||
return nil, errors.New("release sidecar dependencies are required")
|
||||
}
|
||||
return &SidecarPublisher{exporter: exporter, target: target}, nil
|
||||
}
|
||||
|
||||
func (publisher *SidecarPublisher) Publish(ctx context.Context, item PlanItem) (ItemResult, error) {
|
||||
bundle, err := publisher.exporter.Export(ctx, item)
|
||||
if err != nil {
|
||||
return ItemResult{}, newSidecarError(ErrorCodeTargetUnavailable, true, err)
|
||||
}
|
||||
if bundle == nil {
|
||||
return ItemResult{}, newSidecarError(ErrorCodeBundleInvalid, false, errors.New("empty release bundle"))
|
||||
}
|
||||
manifest := bundle.ReleaseManifest()
|
||||
if manifest.ExecutionID != item.ReleaseExecutionID || manifest.PublicationID != item.PublicationID ||
|
||||
manifest.SourceItemVersionID != item.SourceItemVersionID || manifest.SourceRevision != item.SourceRevision ||
|
||||
manifest.ParseRevision != item.ParseRevision || manifest.ReleaseRevision != item.ReleaseRevision ||
|
||||
manifest.Contract.KnowledgeBaseID != item.TestKnowledgeBaseID || manifest.Contract.KnowledgeID != item.KnowledgeID {
|
||||
return ItemResult{}, newSidecarError(ErrorCodeContractMismatch, false, errors.New("bundle identity mismatch"))
|
||||
}
|
||||
targetContract, err := publisher.target.Contract(ctx, item)
|
||||
if err != nil {
|
||||
return ItemResult{}, newSidecarError(ErrorCodeTargetUnavailable, true, err)
|
||||
}
|
||||
if err := releasebundle.Validate(manifest, targetContract); err != nil {
|
||||
return ItemResult{}, newSidecarError(ErrorCodeContractMismatch, false, err)
|
||||
}
|
||||
change, err := publisher.target.Prepare(ctx, item, bundle)
|
||||
if err != nil {
|
||||
return ItemResult{}, newSidecarError(ErrorCodeTargetWriteFailed, true, err)
|
||||
}
|
||||
if !validRevision(change.ProductionRevision) || (change.PreviousRevision != "" && !validRevision(change.PreviousRevision)) {
|
||||
return ItemResult{}, publisher.compensateError(ctx, item, change, ErrorCodeBundleInvalid, errors.New("invalid prepared revision"))
|
||||
}
|
||||
if err := publisher.target.Activate(ctx, item, change); err != nil {
|
||||
return ItemResult{}, publisher.compensateError(ctx, item, change, ErrorCodeTargetWriteFailed, err)
|
||||
}
|
||||
if err := publisher.target.Verify(ctx, item, change); err != nil {
|
||||
return ItemResult{}, publisher.compensateError(ctx, item, change, ErrorCodeVerificationFailed, err)
|
||||
}
|
||||
return ItemResult{
|
||||
BundleSHA256: manifest.BundleSHA256,
|
||||
PreviousProductionRevision: change.PreviousRevision,
|
||||
ProductionRevision: change.ProductionRevision,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (publisher *SidecarPublisher) Rollback(ctx context.Context, item PlanItem) (ItemResult, error) {
|
||||
if !validRevision(item.PreviousProductionRevision) {
|
||||
return ItemResult{}, newSidecarError(ErrorCodeRollbackArtifactMissing, false, errors.New("missing rollback revision"))
|
||||
}
|
||||
restoredRevision, err := publisher.target.Restore(ctx, item, item.PreviousProductionRevision)
|
||||
if err != nil {
|
||||
return ItemResult{}, newSidecarError(ErrorCodeTargetWriteFailed, true, err)
|
||||
}
|
||||
if restoredRevision != item.PreviousProductionRevision {
|
||||
return ItemResult{}, newSidecarError(ErrorCodeContractMismatch, false, errors.New("restored revision mismatch"))
|
||||
}
|
||||
if err := publisher.target.VerifyRestored(ctx, item, restoredRevision); err != nil {
|
||||
return ItemResult{}, newSidecarError(ErrorCodeVerificationFailed, false, err)
|
||||
}
|
||||
return ItemResult{ProductionRevision: restoredRevision}, nil
|
||||
}
|
||||
|
||||
func (publisher *SidecarPublisher) Delete(ctx context.Context, item PlanItem) (ItemResult, error) {
|
||||
bundle, err := publisher.exporter.Export(ctx, item)
|
||||
if err != nil {
|
||||
return ItemResult{}, newSidecarError(ErrorCodeTargetUnavailable, true, err)
|
||||
}
|
||||
if bundle == nil {
|
||||
return ItemResult{}, newSidecarError(ErrorCodeBundleInvalid, false, errors.New("empty release bundle"))
|
||||
}
|
||||
manifest := bundle.ReleaseManifest()
|
||||
if manifest.ExecutionID != item.ReleaseExecutionID || manifest.PublicationID != item.PublicationID ||
|
||||
manifest.SourceItemVersionID != item.SourceItemVersionID || manifest.SourceRevision != item.SourceRevision ||
|
||||
manifest.ParseRevision != item.ParseRevision || manifest.ReleaseRevision != item.ReleaseRevision ||
|
||||
manifest.Contract.KnowledgeBaseID != item.TestKnowledgeBaseID || manifest.Contract.KnowledgeID != item.KnowledgeID {
|
||||
return ItemResult{}, newSidecarError(ErrorCodeContractMismatch, false, errors.New("bundle identity mismatch"))
|
||||
}
|
||||
targetContract, err := publisher.target.Contract(ctx, item)
|
||||
if err != nil {
|
||||
return ItemResult{}, newSidecarError(ErrorCodeTargetUnavailable, true, err)
|
||||
}
|
||||
if err := releasebundle.Validate(manifest, targetContract); err != nil {
|
||||
return ItemResult{}, newSidecarError(ErrorCodeContractMismatch, false, err)
|
||||
}
|
||||
change, err := publisher.target.Prepare(ctx, item, bundle)
|
||||
if err != nil {
|
||||
return ItemResult{}, newSidecarError(ErrorCodeTargetWriteFailed, true, err)
|
||||
}
|
||||
if !validRevision(change.PreviousRevision) || change.ProductionRevision != item.ReleaseRevision {
|
||||
return ItemResult{}, publisher.compensateError(ctx, item, change, ErrorCodeBundleInvalid, errors.New("invalid delete snapshot revision"))
|
||||
}
|
||||
restoredRevision, err := publisher.target.Restore(ctx, item, "absent")
|
||||
if err != nil {
|
||||
return ItemResult{}, publisher.compensateError(ctx, item, change, ErrorCodeTargetWriteFailed, err)
|
||||
}
|
||||
if restoredRevision != "absent" {
|
||||
return ItemResult{}, publisher.compensateError(ctx, item, change, ErrorCodeContractMismatch, errors.New("deleted revision mismatch"))
|
||||
}
|
||||
if err := publisher.target.VerifyRestored(ctx, item, "absent"); err != nil {
|
||||
return ItemResult{}, publisher.compensateError(ctx, item, change, ErrorCodeVerificationFailed, err)
|
||||
}
|
||||
return ItemResult{BundleSHA256: manifest.BundleSHA256, PreviousProductionRevision: change.PreviousRevision, ProductionRevision: "absent"}, nil
|
||||
}
|
||||
|
||||
func (publisher *SidecarPublisher) compensateError(ctx context.Context, item PlanItem, change PreparedChange, code string, cause error) error {
|
||||
if err := publisher.target.Compensate(ctx, item, change); err != nil {
|
||||
return newSidecarError(ErrorCodeRollbackFailed, false, err)
|
||||
}
|
||||
return newSidecarError(code, false, cause)
|
||||
}
|
||||
|
||||
func newSidecarError(code string, retryable bool, cause error) error {
|
||||
return &ItemError{Code: code, Retryable: retryable, Cause: cause}
|
||||
}
|
||||
|
||||
func validRevision(value string) bool {
|
||||
return value == strings.TrimSpace(value) && value != "" && len(value) <= 128 && !strings.ContainsAny(value, "\x00\r\n\t/?#\\")
|
||||
}
|
||||
|
||||
var _ Publisher = (*SidecarPublisher)(nil)
|
||||
@@ -0,0 +1,217 @@
|
||||
package release
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"knowledge-governance/internal/releasebundle"
|
||||
)
|
||||
|
||||
func TestSidecarPublisherPublishesWithContractAndVerification(t *testing.T) {
|
||||
calls := []string{}
|
||||
manifest, err := releasebundle.Seal(testManifest())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
exporter := &fakeExporter{manifest: manifest, calls: &calls}
|
||||
target := &fakeSidecarTarget{contract: testTargetContract(), calls: &calls, prepared: PreparedChange{PreviousRevision: "prod-v0", ProductionRevision: "prod-v1"}}
|
||||
publisher, err := NewSidecarPublisher(exporter, target)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result, err := publisher.Publish(context.Background(), testPlanItem())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.BundleSHA256 != manifest.BundleSHA256 || result.PreviousProductionRevision != "prod-v0" || result.ProductionRevision != "prod-v1" {
|
||||
t.Fatalf("result=%#v", result)
|
||||
}
|
||||
want := []string{"export", "contract", "prepare", "activate", "verify"}
|
||||
if !reflect.DeepEqual(calls, want) {
|
||||
t.Fatalf("calls=%v want=%v", calls, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSidecarPublisherRollsBackFailedVerificationWithoutLeakingCause(t *testing.T) {
|
||||
calls := []string{}
|
||||
manifest, err := releasebundle.Seal(testManifest())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
target := &fakeSidecarTarget{contract: testTargetContract(), calls: &calls, prepared: PreparedChange{PreviousRevision: "prod-v0", ProductionRevision: "prod-v1"}, verifyErr: errors.New("signed-url-secret-should-not-leak")}
|
||||
publisher, err := NewSidecarPublisher(&fakeExporter{manifest: manifest, calls: &calls}, target)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = publisher.Publish(context.Background(), testPlanItem())
|
||||
var itemErr *ItemError
|
||||
if !errors.As(err, &itemErr) || itemErr.Code != ErrorCodeVerificationFailed || itemErr.Retryable {
|
||||
t.Fatalf("error=%#v", err)
|
||||
}
|
||||
if strings.Contains(err.Error(), "signed-url") {
|
||||
t.Fatalf("error leaked cause: %v", err)
|
||||
}
|
||||
want := []string{"export", "contract", "prepare", "activate", "verify", "compensate"}
|
||||
if !reflect.DeepEqual(calls, want) {
|
||||
t.Fatalf("calls=%v want=%v", calls, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSidecarPublisherRejectsContractDriftBeforeTargetMutation(t *testing.T) {
|
||||
calls := []string{}
|
||||
manifest, err := releasebundle.Seal(testManifest())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
contract := testTargetContract()
|
||||
contract.EmbeddingDimension = 768
|
||||
publisher, err := NewSidecarPublisher(&fakeExporter{manifest: manifest, calls: &calls}, &fakeSidecarTarget{contract: contract, calls: &calls})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = publisher.Publish(context.Background(), testPlanItem())
|
||||
var itemErr *ItemError
|
||||
if !errors.As(err, &itemErr) || itemErr.Code != ErrorCodeContractMismatch {
|
||||
t.Fatalf("error=%#v", err)
|
||||
}
|
||||
if !reflect.DeepEqual(calls, []string{"export", "contract"}) {
|
||||
t.Fatalf("calls=%v", calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSidecarPublisherRejectsRevisionMismatchBeforeTargetMutation(t *testing.T) {
|
||||
calls := []string{}
|
||||
manifest, err := releasebundle.Seal(testManifest())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
item := testPlanItem()
|
||||
item.ParseRevision = "other-parser/index"
|
||||
publisher, err := NewSidecarPublisher(&fakeExporter{manifest: manifest, calls: &calls}, &fakeSidecarTarget{contract: testTargetContract(), calls: &calls})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = publisher.Publish(context.Background(), item)
|
||||
var itemErr *ItemError
|
||||
if !errors.As(err, &itemErr) || itemErr.Code != ErrorCodeContractMismatch {
|
||||
t.Fatalf("error=%#v", err)
|
||||
}
|
||||
if !reflect.DeepEqual(calls, []string{"export"}) {
|
||||
t.Fatalf("calls=%v", calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSidecarPublisherDeletesOnlyAfterArchivingAndVerifiesAbsent(t *testing.T) {
|
||||
calls := []string{}
|
||||
manifest, err := releasebundle.Seal(testManifest())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
target := &fakeSidecarTarget{contract: testTargetContract(), calls: &calls, prepared: PreparedChange{PreviousRevision: "prod-v0", ProductionRevision: "release-v1"}, expectedRestoreRevision: "absent", restoredRevision: "absent"}
|
||||
publisher, err := NewSidecarPublisher(&fakeExporter{manifest: manifest, calls: &calls}, target)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result, err := publisher.Delete(context.Background(), testPlanItem())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.BundleSHA256 != manifest.BundleSHA256 || result.PreviousProductionRevision != "prod-v0" || result.ProductionRevision != "absent" {
|
||||
t.Fatalf("result=%#v", result)
|
||||
}
|
||||
if want := []string{"export", "contract", "prepare", "restore", "verify-restored"}; !reflect.DeepEqual(calls, want) {
|
||||
t.Fatalf("calls=%v want=%v", calls, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSidecarPublisherRestoresArchivedRevision(t *testing.T) {
|
||||
calls := []string{}
|
||||
target := &fakeSidecarTarget{calls: &calls, restoredRevision: "prod-v0"}
|
||||
publisher, err := NewSidecarPublisher(&fakeExporter{}, target)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result, err := publisher.Rollback(context.Background(), testPlanItem())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.ProductionRevision != "prod-v0" || !reflect.DeepEqual(calls, []string{"restore", "verify-restored"}) {
|
||||
t.Fatalf("result=%#v calls=%v", result, calls)
|
||||
}
|
||||
}
|
||||
|
||||
type fakeExporter struct {
|
||||
manifest releasebundle.Manifest
|
||||
calls *[]string
|
||||
err error
|
||||
}
|
||||
|
||||
func (f *fakeExporter) Export(context.Context, PlanItem) (ExportedBundle, error) {
|
||||
if f.calls != nil {
|
||||
*f.calls = append(*f.calls, "export")
|
||||
}
|
||||
return fakeExportedBundle{manifest: f.manifest}, f.err
|
||||
}
|
||||
|
||||
type fakeExportedBundle struct{ manifest releasebundle.Manifest }
|
||||
|
||||
func (bundle fakeExportedBundle) ReleaseManifest() releasebundle.Manifest { return bundle.manifest }
|
||||
|
||||
type fakeSidecarTarget struct {
|
||||
contract releasebundle.TargetContract
|
||||
calls *[]string
|
||||
prepared PreparedChange
|
||||
prepareErr, activateErr, verifyErr, compensateErr, restoreErr, verifyRestoredErr error
|
||||
restoredRevision string
|
||||
expectedRestoreRevision string
|
||||
}
|
||||
|
||||
func (f *fakeSidecarTarget) Contract(context.Context, PlanItem) (releasebundle.TargetContract, error) {
|
||||
*f.calls = append(*f.calls, "contract")
|
||||
return f.contract, nil
|
||||
}
|
||||
func (f *fakeSidecarTarget) Prepare(context.Context, PlanItem, ExportedBundle) (PreparedChange, error) {
|
||||
*f.calls = append(*f.calls, "prepare")
|
||||
return f.prepared, f.prepareErr
|
||||
}
|
||||
func (f *fakeSidecarTarget) Activate(context.Context, PlanItem, PreparedChange) error {
|
||||
*f.calls = append(*f.calls, "activate")
|
||||
return f.activateErr
|
||||
}
|
||||
func (f *fakeSidecarTarget) Verify(context.Context, PlanItem, PreparedChange) error {
|
||||
*f.calls = append(*f.calls, "verify")
|
||||
return f.verifyErr
|
||||
}
|
||||
func (f *fakeSidecarTarget) Compensate(context.Context, PlanItem, PreparedChange) error {
|
||||
*f.calls = append(*f.calls, "compensate")
|
||||
return f.compensateErr
|
||||
}
|
||||
func (f *fakeSidecarTarget) Restore(_ context.Context, _ PlanItem, revision string) (string, error) {
|
||||
*f.calls = append(*f.calls, "restore")
|
||||
expected := f.expectedRestoreRevision
|
||||
if expected == "" {
|
||||
expected = "prod-v0"
|
||||
}
|
||||
if revision != expected {
|
||||
return "", errors.New("wrong revision")
|
||||
}
|
||||
return f.restoredRevision, f.restoreErr
|
||||
}
|
||||
func (f *fakeSidecarTarget) VerifyRestored(context.Context, PlanItem, string) error {
|
||||
*f.calls = append(*f.calls, "verify-restored")
|
||||
return f.verifyRestoredErr
|
||||
}
|
||||
|
||||
func testPlanItem() PlanItem {
|
||||
return PlanItem{ReleaseExecutionID: "rx-safe", PublicationID: "tp-safe", SourceItemVersionID: "sv-safe", SourceRevision: "source-v1", ParseRevision: "parser-v1@index-v1", ReleaseRevision: "release-v1", TestKnowledgeBaseID: "kb-safe", KnowledgeID: "knowledge-safe", DecidedVisibility: "internal", PreviousProductionRevision: "prod-v0"}
|
||||
}
|
||||
func testManifest() releasebundle.Manifest {
|
||||
return releasebundle.Manifest{FormatVersion: releasebundle.FormatVersionV1, ExecutionID: "rx-safe", PublicationID: "tp-safe", SourceItemVersionID: "sv-safe", SourceRevision: "source-v1", ParseRevision: "parser-v1@index-v1", ReleaseRevision: "release-v1", CreatedAt: time.Date(2026, 9, 6, 8, 0, 0, 0, time.UTC), Contract: releasebundle.Contract{WeKnoraVersion: releasebundle.WeKnoraVersionV071, MigrationVersion: releasebundle.WeKnoraMigrationV071, TenantID: 10000, KnowledgeBaseID: "kb-safe", KnowledgeID: "knowledge-safe", RetrieverEngine: "elasticsearch_v8", EmbeddingModel: "bge-m3", EmbeddingDimension: releasebundle.EmbeddingDimensionV071, ElasticsearchMappingSHA256: strings.Repeat("a", 64)}, Tables: []releasebundle.TableDigest{{Name: "knowledges", RowCount: 1, SHA256: strings.Repeat("b", 64)}, {Name: "chunks", RowCount: 1, SHA256: strings.Repeat("c", 64)}}, Search: releasebundle.SearchDigest{DocumentCount: 1, SHA256: strings.Repeat("d", 64)}, Assets: releasebundle.AssetDigest{SHA256: strings.Repeat("e", 64)}}
|
||||
}
|
||||
func testTargetContract() releasebundle.TargetContract {
|
||||
return releasebundle.TargetContract{WeKnoraVersion: releasebundle.WeKnoraVersionV071, MigrationVersion: releasebundle.WeKnoraMigrationV071, TenantID: 10000, KnowledgeBaseID: "kb-safe", RetrieverEngine: "elasticsearch_v8", EmbeddingModel: "bge-m3", EmbeddingDimension: releasebundle.EmbeddingDimensionV071, ElasticsearchMappingSHA256: strings.Repeat("a", 64)}
|
||||
}
|
||||
@@ -0,0 +1,194 @@
|
||||
package sourcescan
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
ppconnector "knowledge-governance/internal/connectors/pp"
|
||||
)
|
||||
|
||||
const maxPPMetadataBytes int64 = 64 * 1024 * 1024
|
||||
|
||||
type FilePPScannerConfig struct {
|
||||
AllowedRoot string
|
||||
LedgerPath string
|
||||
ManifestPath string
|
||||
}
|
||||
|
||||
type FilePPScanner struct {
|
||||
root string
|
||||
ledgerPath string
|
||||
manifestPath string
|
||||
}
|
||||
|
||||
func NewFilePPScanner(config FilePPScannerConfig) (*FilePPScanner, error) {
|
||||
root, err := filepath.Abs(config.AllowedRoot)
|
||||
if err != nil || config.AllowedRoot == "" {
|
||||
return nil, errors.New("PP scan allowed root is required")
|
||||
}
|
||||
resolvedRoot, err := filepath.EvalSymlinks(root)
|
||||
if err != nil || resolvedRoot != root {
|
||||
return nil, errors.New("PP scan allowed root must be a canonical directory")
|
||||
}
|
||||
info, err := os.Lstat(root)
|
||||
if err != nil || !info.IsDir() || info.Mode()&os.ModeSymlink != 0 {
|
||||
return nil, errors.New("PP scan allowed root must be a canonical directory")
|
||||
}
|
||||
if config.LedgerPath == "" || config.ManifestPath == "" {
|
||||
return nil, errors.New("PP ledger and manifest paths are required")
|
||||
}
|
||||
return &FilePPScanner{root: root, ledgerPath: config.LedgerPath, manifestPath: config.ManifestPath}, nil
|
||||
}
|
||||
|
||||
func (s *FilePPScanner) Scan(ctx context.Context) (ppconnector.Snapshot, string, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return ppconnector.Snapshot{}, "", err
|
||||
}
|
||||
ledger, err := s.readFile(s.ledgerPath, maxPPMetadataBytes)
|
||||
if err != nil {
|
||||
return ppconnector.Snapshot{}, "", fmt.Errorf("read PP ledger: %w", err)
|
||||
}
|
||||
manifest, err := s.readFile(s.manifestPath, maxPPMetadataBytes)
|
||||
if err != nil {
|
||||
return ppconnector.Snapshot{}, "", fmt.Errorf("read PP manifest: %w", err)
|
||||
}
|
||||
if err := s.validateManifestPaths(manifest); err != nil {
|
||||
return ppconnector.Snapshot{}, "", err
|
||||
}
|
||||
snapshot, err := ppconnector.FoldLedger(bytes.NewReader(ledger))
|
||||
if err != nil {
|
||||
return ppconnector.Snapshot{}, "", err
|
||||
}
|
||||
snapshot, _, err = ppconnector.RecoverMissingSHA256(snapshot, bytes.NewReader(manifest))
|
||||
if err != nil {
|
||||
return ppconnector.Snapshot{}, "", err
|
||||
}
|
||||
snapshot, _, err = ppconnector.MergeOverLimitManifest(snapshot, bytes.NewReader(manifest))
|
||||
if err != nil {
|
||||
return ppconnector.Snapshot{}, "", err
|
||||
}
|
||||
if err := ppconnector.ValidateSnapshot(snapshot); err != nil {
|
||||
return ppconnector.Snapshot{}, "", err
|
||||
}
|
||||
digest := sha256.New()
|
||||
_, _ = digest.Write(ledger)
|
||||
_, _ = digest.Write([]byte{0})
|
||||
_, _ = digest.Write(manifest)
|
||||
return snapshot, hex.EncodeToString(digest.Sum(nil)), nil
|
||||
}
|
||||
|
||||
func (s *FilePPScanner) validateManifestPaths(body []byte) error {
|
||||
var manifest struct {
|
||||
Results []struct {
|
||||
LocalPath string `json:"local_path"`
|
||||
Status string `json:"status"`
|
||||
} `json:"results"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &manifest); err != nil {
|
||||
return fmt.Errorf("decode PP manifest paths: %w", err)
|
||||
}
|
||||
for _, item := range manifest.Results {
|
||||
if item.LocalPath == "" {
|
||||
return errors.New("PP manifest artifact local_path is required")
|
||||
}
|
||||
if item.Status == string(ppconnector.StatusOverLimit) {
|
||||
if err := s.validateDeclaredPath(item.LocalPath); err != nil {
|
||||
return fmt.Errorf("validate PP over-limit artifact path: %w", err)
|
||||
}
|
||||
file, err := s.openRegular(item.LocalPath)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("validate PP over-limit artifact: %w", err)
|
||||
}
|
||||
if err := file.Close(); err != nil {
|
||||
return errors.New("close PP over-limit artifact")
|
||||
}
|
||||
continue
|
||||
}
|
||||
file, err := s.openRegular(item.LocalPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("validate PP manifest artifact: %w", err)
|
||||
}
|
||||
if err := file.Close(); err != nil {
|
||||
return errors.New("close PP manifest artifact")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *FilePPScanner) readFile(path string, maxBytes int64) ([]byte, error) {
|
||||
file, err := s.openRegular(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
body, readErr := io.ReadAll(io.LimitReader(file, maxBytes+1))
|
||||
closeErr := file.Close()
|
||||
if readErr != nil {
|
||||
return nil, readErr
|
||||
}
|
||||
if closeErr != nil {
|
||||
return nil, closeErr
|
||||
}
|
||||
if int64(len(body)) > maxBytes {
|
||||
return nil, errors.New("PP metadata file exceeds size limit")
|
||||
}
|
||||
return body, nil
|
||||
}
|
||||
|
||||
func (s *FilePPScanner) validateDeclaredPath(path string) error {
|
||||
absolute, err := filepath.Abs(path)
|
||||
if err != nil {
|
||||
return errors.New("invalid PP source path")
|
||||
}
|
||||
relative, err := filepath.Rel(s.root, absolute)
|
||||
if err != nil || relative == ".." || filepath.IsAbs(relative) || len(relative) >= 3 && relative[:3] == ".."+string(filepath.Separator) {
|
||||
return errors.New("PP source path is outside allowed root")
|
||||
}
|
||||
parent := filepath.Dir(absolute)
|
||||
resolvedParent, err := filepath.EvalSymlinks(parent)
|
||||
if err != nil || resolvedParent != parent {
|
||||
return errors.New("PP source path parent must not contain symlinks")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *FilePPScanner) openRegular(path string) (*os.File, error) {
|
||||
if err := s.validateDeclaredPath(path); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
absolute, _ := filepath.Abs(path)
|
||||
resolved, err := filepath.EvalSymlinks(absolute)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resolved != absolute {
|
||||
return nil, errors.New("PP source path must not contain symlinks")
|
||||
}
|
||||
before, err := os.Lstat(absolute)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !before.Mode().IsRegular() || before.Mode()&os.ModeSymlink != 0 {
|
||||
return nil, errors.New("PP source path must be a regular file")
|
||||
}
|
||||
file, err := os.Open(absolute)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
after, err := file.Stat()
|
||||
if err != nil || !after.Mode().IsRegular() || !os.SameFile(before, after) {
|
||||
_ = file.Close()
|
||||
return nil, errors.New("PP source file changed while opening")
|
||||
}
|
||||
return file, nil
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
package sourcescan
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
ppconnector "knowledge-governance/internal/connectors/pp"
|
||||
"knowledge-governance/internal/governance"
|
||||
"knowledge-governance/internal/jobs"
|
||||
)
|
||||
|
||||
type PPScanner interface {
|
||||
Scan(context.Context) (ppconnector.Snapshot, string, error)
|
||||
}
|
||||
|
||||
type PPScanStore interface {
|
||||
StartSourceScan(context.Context, string, time.Time) (governance.SourceScan, error)
|
||||
LoadPPCurrentItems(context.Context, string) (map[string]ppconnector.CurrentItem, error)
|
||||
CompleteSourceScan(context.Context, string, governance.SourceScanResult, time.Time) error
|
||||
FailSourceScan(context.Context, string, string, time.Time) error
|
||||
}
|
||||
|
||||
type PPHandlerConfig struct {
|
||||
Store PPScanStore
|
||||
Scanner PPScanner
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
type PPHandler struct {
|
||||
store PPScanStore
|
||||
scanner PPScanner
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
func NewPPHandler(config PPHandlerConfig) (*PPHandler, error) {
|
||||
if config.Store == nil || config.Scanner == nil {
|
||||
return nil, errors.New("PP scan store and scanner are required")
|
||||
}
|
||||
if config.Now == nil {
|
||||
config.Now = time.Now
|
||||
}
|
||||
return &PPHandler{store: config.Store, scanner: config.Scanner, now: config.Now}, nil
|
||||
}
|
||||
|
||||
func (h *PPHandler) Handle(ctx context.Context, job jobs.Job) error {
|
||||
var payload struct {
|
||||
ScanID string `json:"scan_id"`
|
||||
}
|
||||
if err := json.Unmarshal(job.Payload, &payload); err != nil || payload.ScanID == "" {
|
||||
return errors.New("invalid source scan payload")
|
||||
}
|
||||
now := h.now()
|
||||
scan, err := h.store.StartSourceScan(ctx, payload.ScanID, now)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if scan.State == governance.SourceScanCompleted {
|
||||
return nil
|
||||
}
|
||||
if scan.State != governance.SourceScanRunning {
|
||||
return errors.New("source scan is not running")
|
||||
}
|
||||
if scan.SourceType != governance.SourceTypePP {
|
||||
return h.store.FailSourceScan(ctx, scan.ID, "source_type_unsupported", now)
|
||||
}
|
||||
snapshot, snapshotSHA, err := h.scanner.Scan(ctx)
|
||||
if err != nil || !governance.ValidSHA256(snapshotSHA) {
|
||||
return h.store.FailSourceScan(ctx, scan.ID, "pp_snapshot_invalid", now)
|
||||
}
|
||||
current, err := h.store.LoadPPCurrentItems(ctx, scan.SourceConnectionID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
reconciliation, err := ppconnector.PlanReconciliation(snapshot, current)
|
||||
if err != nil {
|
||||
return h.store.FailSourceScan(ctx, scan.ID, "pp_snapshot_invalid", now)
|
||||
}
|
||||
observed := make(map[string]struct{}, len(snapshot.Items))
|
||||
changes := make([]governance.SourceScanChange, 0)
|
||||
for _, item := range snapshot.Items {
|
||||
observed[item.SourceURLHash] = struct{}{}
|
||||
if item.Status == ppconnector.StatusOverLimit {
|
||||
continue
|
||||
}
|
||||
size := item.Bytes
|
||||
stored, exists := current[item.SourceURLHash]
|
||||
if !exists {
|
||||
changes = append(changes, governance.SourceScanChange{
|
||||
StableExternalID: item.SourceURLHash, ChangeType: governance.SourceScanAdded,
|
||||
ObservedRevision: item.SHA256, ObservedSizeBytes: &size,
|
||||
})
|
||||
} else if stored.Revision != item.SHA256 {
|
||||
changes = append(changes, governance.SourceScanChange{
|
||||
StableExternalID: item.SourceURLHash, ChangeType: governance.SourceScanUpdated,
|
||||
PreviousRevision: stored.Revision, ObservedRevision: item.SHA256, ObservedSizeBytes: &size,
|
||||
})
|
||||
}
|
||||
}
|
||||
missingIDs := make([]string, 0)
|
||||
for sourceID := range current {
|
||||
if _, ok := observed[sourceID]; !ok {
|
||||
missingIDs = append(missingIDs, sourceID)
|
||||
}
|
||||
}
|
||||
sort.Strings(missingIDs)
|
||||
for _, sourceID := range missingIDs {
|
||||
changes = append(changes, governance.SourceScanChange{
|
||||
StableExternalID: sourceID, ChangeType: governance.SourceScanUnavailable,
|
||||
PreviousRevision: current[sourceID].Revision,
|
||||
})
|
||||
}
|
||||
result := governance.SourceScanResult{
|
||||
SnapshotSHA256: snapshotSHA,
|
||||
ObservedItems: len(snapshot.Items),
|
||||
UnchangedItems: reconciliation.Unchanged,
|
||||
AddedItems: reconciliation.Adopted,
|
||||
UpdatedItems: reconciliation.UploadRequired,
|
||||
OverLimitItems: reconciliation.OverLimit,
|
||||
MissingItems: len(missingIDs),
|
||||
PlannedUploads: reconciliation.UploadRequired,
|
||||
Changes: changes,
|
||||
}
|
||||
return h.store.CompleteSourceScan(ctx, scan.ID, result, now)
|
||||
}
|
||||
@@ -0,0 +1,208 @@
|
||||
package sourcescan
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
ppconnector "knowledge-governance/internal/connectors/pp"
|
||||
"knowledge-governance/internal/governance"
|
||||
"knowledge-governance/internal/jobs"
|
||||
)
|
||||
|
||||
func TestFilePPScannerReadsValidatedSnapshotUnderAllowedRoot(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
source := strings.Repeat("a", 64)
|
||||
sha := strings.Repeat("b", 64)
|
||||
ledgerPath := filepath.Join(root, "state.jsonl")
|
||||
manifestPath := filepath.Join(root, "manifest.json")
|
||||
ledger := `{"kb_id":"kb-1","source_url_hash":"` + source + `","source_ids":["pp-1"],"bytes":10,"sha256":"` + sha + `","status":"completed","knowledge_id":"knowledge-1"}`
|
||||
if err := os.WriteFile(ledgerPath, []byte(ledger), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(manifestPath, []byte(`{"results":[]}`), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
scanner, err := NewFilePPScanner(FilePPScannerConfig{AllowedRoot: root, LedgerPath: ledgerPath, ManifestPath: manifestPath})
|
||||
if err != nil {
|
||||
t.Fatalf("NewFilePPScanner() error = %v", err)
|
||||
}
|
||||
snapshot, fingerprint, err := scanner.Scan(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("Scan() error = %v", err)
|
||||
}
|
||||
if len(snapshot.Items) != 1 || snapshot.Items[0].SourceURLHash != source {
|
||||
t.Fatalf("snapshot = %#v", snapshot)
|
||||
}
|
||||
if !governance.ValidSHA256(fingerprint) {
|
||||
t.Fatalf("fingerprint = %q", fingerprint)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilePPScannerRejectsManifestItemWithoutLocalPath(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
source := strings.Repeat("a", 64)
|
||||
sha := strings.Repeat("b", 64)
|
||||
ledgerPath := filepath.Join(root, "state.jsonl")
|
||||
manifestPath := filepath.Join(root, "manifest.json")
|
||||
ledger := `{"kb_id":"kb-1","source_url_hash":"` + source + `","source_ids":["pp-1"],"bytes":10,"sha256":"` + sha + `","status":"completed","knowledge_id":"knowledge-1"}`
|
||||
if err := os.WriteFile(ledgerPath, []byte(ledger), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(manifestPath, []byte(`{"results":[{"local_path":""}]}`), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
scanner, err := NewFilePPScanner(FilePPScannerConfig{AllowedRoot: root, LedgerPath: ledgerPath, ManifestPath: manifestPath})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err := scanner.Scan(context.Background()); err == nil || !strings.Contains(err.Error(), "local_path") {
|
||||
t.Fatalf("Scan() error = %v, want missing local_path error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilePPScannerAllowsMissingArtifactForOverLimitRecord(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
source := strings.Repeat("a", 64)
|
||||
sha := strings.Repeat("b", 64)
|
||||
ledgerPath := filepath.Join(root, "state.jsonl")
|
||||
manifestPath := filepath.Join(root, "manifest.json")
|
||||
ledger := `{"kb_id":"kb-1","source_url_hash":"` + source + `","source_ids":["pp-1"],"bytes":10,"sha256":"` + sha + `","status":"completed","knowledge_id":"knowledge-1"}`
|
||||
manifest := `{"results":[{"url":"https://source.invalid/large","local_path":"` + filepath.Join(root, "not-downloaded.bin") + `","bytes":52428801,"status":"over_limit","source_ids":["pp-large"]}]}`
|
||||
if err := os.WriteFile(ledgerPath, []byte(ledger), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(manifestPath, []byte(manifest), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
scanner, err := NewFilePPScanner(FilePPScannerConfig{AllowedRoot: root, LedgerPath: ledgerPath, ManifestPath: manifestPath})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
snapshot, _, err := scanner.Scan(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("Scan() error = %v", err)
|
||||
}
|
||||
if len(snapshot.Items) != 2 || snapshot.Items[1].Status != ppconnector.StatusOverLimit {
|
||||
t.Fatalf("snapshot = %#v", snapshot)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPPHandlerCompletesAggregateReadOnlyReconciliation(t *testing.T) {
|
||||
now := time.Date(2026, 9, 6, 3, 0, 0, 0, time.UTC)
|
||||
sameSource := strings.Repeat("a", 64)
|
||||
updatedSource := strings.Repeat("b", 64)
|
||||
overLimitSource := strings.Repeat("c", 64)
|
||||
missingSource := strings.Repeat("d", 64)
|
||||
addedSource := strings.Repeat("e", 64)
|
||||
sameSHA := strings.Repeat("1", 64)
|
||||
newSHA := strings.Repeat("2", 64)
|
||||
oldSHA := strings.Repeat("3", 64)
|
||||
addedSHA := strings.Repeat("5", 64)
|
||||
|
||||
store := &fakePPScanStore{
|
||||
scan: governance.SourceScan{ID: "scan-1", SourceConnectionID: "conn-pp", SourceType: governance.SourceTypePP, State: governance.SourceScanPending},
|
||||
current: map[string]ppconnector.CurrentItem{
|
||||
sameSource: {Revision: sameSHA},
|
||||
updatedSource: {Revision: oldSHA},
|
||||
missingSource: {Revision: strings.Repeat("4", 64)},
|
||||
},
|
||||
}
|
||||
scanner := &fakePPScanner{snapshot: ppconnector.Snapshot{KBID: "kb-1", Items: []ppconnector.Item{
|
||||
{KBID: "kb-1", SourceURLHash: sameSource, SourceIDs: []string{"pp-1"}, Bytes: 10, SHA256: sameSHA, Status: ppconnector.StatusCompleted, KnowledgeID: "knowledge-1"},
|
||||
{KBID: "kb-1", SourceURLHash: updatedSource, SourceIDs: []string{"pp-2"}, Bytes: 20, SHA256: newSHA, Status: ppconnector.StatusCompleted, KnowledgeID: "knowledge-2"},
|
||||
{KBID: "kb-1", SourceURLHash: overLimitSource, SourceIDs: []string{"pp-3"}, Bytes: ppconnector.LegacyMaxBytes + 1, Status: ppconnector.StatusOverLimit},
|
||||
{KBID: "kb-1", SourceURLHash: addedSource, SourceIDs: []string{"pp-4"}, Bytes: 30, SHA256: addedSHA, Status: ppconnector.StatusCompleted, KnowledgeID: "knowledge-4"},
|
||||
}}, snapshotSHA: strings.Repeat("f", 64)}
|
||||
|
||||
handler, err := NewPPHandler(PPHandlerConfig{Store: store, Scanner: scanner, Now: func() time.Time { return now }})
|
||||
if err != nil {
|
||||
t.Fatalf("NewPPHandler() error = %v", err)
|
||||
}
|
||||
job := jobs.Job{Type: "source.scan.pp", Payload: []byte(`{"scan_id":"scan-1"}`)}
|
||||
if err := handler.Handle(context.Background(), job); err != nil {
|
||||
t.Fatalf("Handle() error = %v", err)
|
||||
}
|
||||
|
||||
want := governance.SourceScanResult{
|
||||
SnapshotSHA256: strings.Repeat("f", 64), ObservedItems: 4, UnchangedItems: 1, AddedItems: 1,
|
||||
UpdatedItems: 1, OverLimitItems: 1, MissingItems: 1, PlannedUploads: 1,
|
||||
Changes: []governance.SourceScanChange{
|
||||
{StableExternalID: updatedSource, ChangeType: governance.SourceScanUpdated, PreviousRevision: oldSHA, ObservedRevision: newSHA, ObservedSizeBytes: int64Pointer(20)},
|
||||
{StableExternalID: addedSource, ChangeType: governance.SourceScanAdded, ObservedRevision: addedSHA, ObservedSizeBytes: int64Pointer(30)},
|
||||
{StableExternalID: missingSource, ChangeType: governance.SourceScanUnavailable, PreviousRevision: strings.Repeat("4", 64)},
|
||||
},
|
||||
}
|
||||
if store.completedScanID != "scan-1" || !reflect.DeepEqual(store.completed, want) {
|
||||
t.Fatalf("completed scan = %q %#v, want %#v", store.completedScanID, store.completed, want)
|
||||
}
|
||||
if store.completedAt != now || store.failedCode != "" {
|
||||
t.Fatalf("completedAt=%v failedCode=%q", store.completedAt, store.failedCode)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPPHandlerNoOpsWhenReclaimedScanAlreadyCompleted(t *testing.T) {
|
||||
store := &fakePPScanStore{scan: governance.SourceScan{
|
||||
ID: "scan-completed", SourceConnectionID: "conn-pp", SourceType: governance.SourceTypePP, State: governance.SourceScanCompleted,
|
||||
}}
|
||||
scanner := &fakePPScanner{}
|
||||
handler, err := NewPPHandler(PPHandlerConfig{Store: store, Scanner: scanner})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := handler.Handle(context.Background(), jobs.Job{Payload: []byte(`{"scan_id":"scan-completed"}`)}); err != nil {
|
||||
t.Fatalf("Handle() error = %v", err)
|
||||
}
|
||||
if scanner.calls != 0 || store.completedScanID != "" || store.failedCode != "" {
|
||||
t.Fatalf("scanner calls=%d completed=%q failed=%q", scanner.calls, store.completedScanID, store.failedCode)
|
||||
}
|
||||
}
|
||||
|
||||
func int64Pointer(value int64) *int64 { return &value }
|
||||
|
||||
type fakePPScanner struct {
|
||||
snapshot ppconnector.Snapshot
|
||||
snapshotSHA string
|
||||
err error
|
||||
calls int
|
||||
}
|
||||
|
||||
func (s *fakePPScanner) Scan(context.Context) (ppconnector.Snapshot, string, error) {
|
||||
s.calls++
|
||||
return s.snapshot, s.snapshotSHA, s.err
|
||||
}
|
||||
|
||||
type fakePPScanStore struct {
|
||||
scan governance.SourceScan
|
||||
current map[string]ppconnector.CurrentItem
|
||||
completedScanID string
|
||||
completed governance.SourceScanResult
|
||||
completedAt time.Time
|
||||
failedCode string
|
||||
}
|
||||
|
||||
func (s *fakePPScanStore) StartSourceScan(context.Context, string, time.Time) (governance.SourceScan, error) {
|
||||
if s.scan.State == governance.SourceScanPending {
|
||||
s.scan.State = governance.SourceScanRunning
|
||||
}
|
||||
return s.scan, nil
|
||||
}
|
||||
|
||||
func (s *fakePPScanStore) LoadPPCurrentItems(context.Context, string) (map[string]ppconnector.CurrentItem, error) {
|
||||
return s.current, nil
|
||||
}
|
||||
|
||||
func (s *fakePPScanStore) CompleteSourceScan(_ context.Context, scanID string, result governance.SourceScanResult, now time.Time) error {
|
||||
s.completedScanID, s.completed, s.completedAt = scanID, result, now
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *fakePPScanStore) FailSourceScan(_ context.Context, _ string, code string, _ time.Time) error {
|
||||
s.failedCode = code
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,196 @@
|
||||
package testweknora
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"knowledge-governance/internal/platform/weknora"
|
||||
)
|
||||
|
||||
type LegacyCandidate struct {
|
||||
ID string
|
||||
KnowledgeID string
|
||||
TestKBID string
|
||||
}
|
||||
|
||||
type LegacyVerifyClient interface {
|
||||
GetKnowledge(context.Context, string) (weknora.Knowledge, error)
|
||||
ListChunks(context.Context, string, int, int, []string) ([]weknora.Chunk, int64, error)
|
||||
}
|
||||
|
||||
type LegacyVerificationReport struct {
|
||||
Total int
|
||||
Existing int
|
||||
Missing int
|
||||
WrongKB int
|
||||
Pending int
|
||||
Processing int
|
||||
Finalizing int
|
||||
Completed int
|
||||
Failed int
|
||||
ChunkChecked int
|
||||
ChunkEmpty int
|
||||
HTTPErrors int
|
||||
}
|
||||
|
||||
func VerifyLegacy(ctx context.Context, client LegacyVerifyClient, candidates []LegacyCandidate, chunkSampleEvery int) (LegacyVerificationReport, error) {
|
||||
report := LegacyVerificationReport{Total: len(candidates)}
|
||||
completedSeen := 0
|
||||
for _, candidate := range candidates {
|
||||
knowledge, err := client.GetKnowledge(ctx, candidate.KnowledgeID)
|
||||
if err != nil {
|
||||
var httpErr *weknora.HTTPError
|
||||
if errors.As(err, &httpErr) && httpErr.StatusCode == 404 {
|
||||
report.Missing++
|
||||
} else {
|
||||
report.HTTPErrors++
|
||||
}
|
||||
continue
|
||||
}
|
||||
if knowledge.KnowledgeBaseID != candidate.TestKBID {
|
||||
report.WrongKB++
|
||||
continue
|
||||
}
|
||||
report.Existing++
|
||||
switch knowledge.ParseStatus {
|
||||
case "pending":
|
||||
report.Pending++
|
||||
case "processing":
|
||||
report.Processing++
|
||||
case "finalizing":
|
||||
report.Finalizing++
|
||||
case "completed":
|
||||
report.Completed++
|
||||
completedSeen++
|
||||
if chunkSampleEvery > 0 && completedSeen%chunkSampleEvery == 0 {
|
||||
if err := verifyLegacyChunk(ctx, client, candidate.KnowledgeID, &report); err != nil {
|
||||
report.HTTPErrors++
|
||||
}
|
||||
}
|
||||
case "failed":
|
||||
report.Failed++
|
||||
}
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func VerifyLegacyConcurrent(ctx context.Context, client LegacyVerifyClient, candidates []LegacyCandidate, chunkSampleEvery, concurrency int) (LegacyVerificationReport, error) {
|
||||
if client == nil || concurrency < 1 || concurrency > 32 || chunkSampleEvery < 0 {
|
||||
return LegacyVerificationReport{}, errors.New("invalid legacy verification configuration")
|
||||
}
|
||||
jobs := make(chan LegacyCandidate)
|
||||
results := make(chan LegacyVerificationReport)
|
||||
var workers sync.WaitGroup
|
||||
var completedSeen atomic.Int64
|
||||
workerCount := min(concurrency, len(candidates))
|
||||
workers.Add(workerCount)
|
||||
for range workerCount {
|
||||
go func() {
|
||||
defer workers.Done()
|
||||
for candidate := range jobs {
|
||||
partial, _ := VerifyLegacy(ctx, client, []LegacyCandidate{candidate}, 0)
|
||||
if partial.Completed == 1 {
|
||||
completed := completedSeen.Add(1)
|
||||
if chunkSampleEvery > 0 && completed%int64(chunkSampleEvery) == 0 {
|
||||
if err := verifyLegacyChunk(ctx, client, candidate.KnowledgeID, &partial); err != nil {
|
||||
partial.HTTPErrors++
|
||||
}
|
||||
}
|
||||
}
|
||||
results <- partial
|
||||
}
|
||||
}()
|
||||
}
|
||||
go func() {
|
||||
defer close(jobs)
|
||||
for _, candidate := range candidates {
|
||||
select {
|
||||
case jobs <- candidate:
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
go func() {
|
||||
workers.Wait()
|
||||
close(results)
|
||||
}()
|
||||
|
||||
report := LegacyVerificationReport{}
|
||||
for partial := range results {
|
||||
mergeLegacyVerificationReport(&report, partial)
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return report, err
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func mergeLegacyVerificationReport(report *LegacyVerificationReport, partial LegacyVerificationReport) {
|
||||
report.Total += partial.Total
|
||||
report.Existing += partial.Existing
|
||||
report.Missing += partial.Missing
|
||||
report.WrongKB += partial.WrongKB
|
||||
report.Pending += partial.Pending
|
||||
report.Processing += partial.Processing
|
||||
report.Finalizing += partial.Finalizing
|
||||
report.Completed += partial.Completed
|
||||
report.Failed += partial.Failed
|
||||
report.ChunkChecked += partial.ChunkChecked
|
||||
report.ChunkEmpty += partial.ChunkEmpty
|
||||
report.HTTPErrors += partial.HTTPErrors
|
||||
}
|
||||
|
||||
func verifyLegacyChunk(ctx context.Context, client LegacyVerifyClient, knowledgeID string, report *LegacyVerificationReport) error {
|
||||
_, total, err := client.ListChunks(ctx, knowledgeID, 1, 1, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
report.ChunkChecked++
|
||||
if total == 0 {
|
||||
report.ChunkEmpty++
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func FormatDryRunReport(candidates []LegacyCandidate, concurrency int) string {
|
||||
kbCounts := make(map[string]int)
|
||||
for _, candidate := range candidates {
|
||||
kbCounts[candidate.TestKBID]++
|
||||
}
|
||||
kbs := make([]string, 0, len(kbCounts))
|
||||
for kb := range kbCounts {
|
||||
kbs = append(kbs, kb)
|
||||
}
|
||||
sort.Strings(kbs)
|
||||
var builder strings.Builder
|
||||
fmt.Fprintf(&builder, "total=%d\n", len(candidates))
|
||||
fmt.Fprintf(&builder, "concurrency=%d\n", concurrency)
|
||||
for _, kb := range kbs {
|
||||
fmt.Fprintf(&builder, "knowledge_base=%s count=%d\n", kb, kbCounts[kb])
|
||||
}
|
||||
return builder.String()
|
||||
}
|
||||
|
||||
func FormatLegacyVerificationReport(report LegacyVerificationReport) string {
|
||||
return fmt.Sprintf(
|
||||
"total=%d\nexisting=%d\nmissing=%d\nwrong_kb=%d\npending=%d\nprocessing=%d\nfinalizing=%d\ncompleted=%d\nfailed=%d\nchunk_checked=%d\nchunk_empty=%d\nhttp_errors=%d\n",
|
||||
report.Total,
|
||||
report.Existing,
|
||||
report.Missing,
|
||||
report.WrongKB,
|
||||
report.Pending,
|
||||
report.Processing,
|
||||
report.Finalizing,
|
||||
report.Completed,
|
||||
report.Failed,
|
||||
report.ChunkChecked,
|
||||
report.ChunkEmpty,
|
||||
report.HTTPErrors,
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,192 @@
|
||||
package testweknora
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"knowledge-governance/internal/platform/weknora"
|
||||
)
|
||||
|
||||
func TestVerifyLegacyClassifiesWithoutLeaking(t *testing.T) {
|
||||
client := &legacyClientStub{getResults: map[string]legacyGetResult{
|
||||
"knowledge-completed-a": {knowledge: weknora.Knowledge{KnowledgeBaseID: "kb-1", ParseStatus: "completed"}},
|
||||
"knowledge-completed-b": {knowledge: weknora.Knowledge{KnowledgeBaseID: "kb-1", ParseStatus: "completed"}},
|
||||
"knowledge-pending": {knowledge: weknora.Knowledge{KnowledgeBaseID: "kb-1", ParseStatus: "pending"}},
|
||||
"knowledge-processing": {knowledge: weknora.Knowledge{KnowledgeBaseID: "kb-1", ParseStatus: "processing"}},
|
||||
"knowledge-finalizing": {knowledge: weknora.Knowledge{KnowledgeBaseID: "kb-1", ParseStatus: "finalizing"}},
|
||||
"knowledge-failed": {knowledge: weknora.Knowledge{KnowledgeBaseID: "kb-1", ParseStatus: "failed"}},
|
||||
"knowledge-wrong-kb": {knowledge: weknora.Knowledge{KnowledgeBaseID: "kb-other", ParseStatus: "completed"}},
|
||||
"knowledge-missing": {err: &weknora.HTTPError{StatusCode: 404}},
|
||||
"knowledge-server-error": {err: &weknora.HTTPError{StatusCode: 500}},
|
||||
}, chunkTotal: map[string]int64{
|
||||
"knowledge-completed-a": 2,
|
||||
"knowledge-completed-b": 0,
|
||||
}}
|
||||
|
||||
candidates := []LegacyCandidate{
|
||||
{ID: "pub-1", KnowledgeID: "knowledge-completed-a", TestKBID: "kb-1"},
|
||||
{ID: "pub-2", KnowledgeID: "knowledge-completed-b", TestKBID: "kb-1"},
|
||||
{ID: "pub-3", KnowledgeID: "knowledge-pending", TestKBID: "kb-1"},
|
||||
{ID: "pub-4", KnowledgeID: "knowledge-processing", TestKBID: "kb-1"},
|
||||
{ID: "pub-5", KnowledgeID: "knowledge-finalizing", TestKBID: "kb-1"},
|
||||
{ID: "pub-6", KnowledgeID: "knowledge-failed", TestKBID: "kb-1"},
|
||||
{ID: "pub-7", KnowledgeID: "knowledge-wrong-kb", TestKBID: "kb-1"},
|
||||
{ID: "pub-8", KnowledgeID: "knowledge-missing", TestKBID: "kb-1"},
|
||||
{ID: "pub-9", KnowledgeID: "knowledge-server-error", TestKBID: "kb-1"},
|
||||
}
|
||||
|
||||
report, err := VerifyLegacy(context.Background(), client, candidates, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("VerifyLegacy() error = %v", err)
|
||||
}
|
||||
want := LegacyVerificationReport{
|
||||
Total: 9, Existing: 6, Missing: 1, WrongKB: 1,
|
||||
Pending: 1, Processing: 1, Finalizing: 1, Completed: 2, Failed: 1,
|
||||
ChunkChecked: 2, ChunkEmpty: 1, HTTPErrors: 1,
|
||||
}
|
||||
if report != want {
|
||||
t.Fatalf("report = %#v, want %#v", report, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerifyLegacyChunkSampleEverySkipsUnsampledCompleted(t *testing.T) {
|
||||
client := &legacyClientStub{getResults: map[string]legacyGetResult{
|
||||
"knowledge-a": {knowledge: weknora.Knowledge{KnowledgeBaseID: "kb-1", ParseStatus: "completed"}},
|
||||
"knowledge-b": {knowledge: weknora.Knowledge{KnowledgeBaseID: "kb-1", ParseStatus: "completed"}},
|
||||
}, chunkTotal: map[string]int64{
|
||||
"knowledge-b": 5,
|
||||
}}
|
||||
candidates := []LegacyCandidate{
|
||||
{ID: "pub-1", KnowledgeID: "knowledge-a", TestKBID: "kb-1"},
|
||||
{ID: "pub-2", KnowledgeID: "knowledge-b", TestKBID: "kb-1"},
|
||||
}
|
||||
report, err := VerifyLegacy(context.Background(), client, candidates, 2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if report.Completed != 2 || report.ChunkChecked != 1 {
|
||||
t.Fatalf("report = %#v", report)
|
||||
}
|
||||
if client.chunkQueries != 1 {
|
||||
t.Fatalf("chunk queries = %d, want 1", client.chunkQueries)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerifyLegacyConcurrentBoundsConcurrencyAndAggregates(t *testing.T) {
|
||||
client := &concurrentLegacyClient{}
|
||||
candidates := make([]LegacyCandidate, 12)
|
||||
for index := range candidates {
|
||||
candidates[index] = LegacyCandidate{ID: "publication", KnowledgeID: "knowledge", TestKBID: "kb-1"}
|
||||
}
|
||||
|
||||
report, err := VerifyLegacyConcurrent(context.Background(), client, candidates, 0, 3)
|
||||
if err != nil {
|
||||
t.Fatalf("VerifyLegacyConcurrent() error = %v", err)
|
||||
}
|
||||
if report.Total != 12 || report.Existing != 12 || report.Completed != 12 {
|
||||
t.Fatalf("report = %#v", report)
|
||||
}
|
||||
if maximum := client.maximum.Load(); maximum < 2 || maximum > 3 {
|
||||
t.Fatalf("maximum concurrency = %d, want 2..3", maximum)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerifyLegacyConcurrentReturnsImmediatelyForEmptyInput(t *testing.T) {
|
||||
report, err := VerifyLegacyConcurrent(context.Background(), &concurrentLegacyClient{}, nil, 10, 4)
|
||||
if err != nil {
|
||||
t.Fatalf("VerifyLegacyConcurrent() error = %v", err)
|
||||
}
|
||||
if report != (LegacyVerificationReport{}) {
|
||||
t.Fatalf("report = %#v", report)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFormatDryRunReportAggregatesWithoutKnowledgeIDs(t *testing.T) {
|
||||
candidates := []LegacyCandidate{
|
||||
{ID: "pub-1", KnowledgeID: "knowledge-1", TestKBID: "kb-1"},
|
||||
{ID: "pub-2", KnowledgeID: "knowledge-2", TestKBID: "kb-1"},
|
||||
{ID: "pub-3", KnowledgeID: "knowledge-3", TestKBID: "kb-2"},
|
||||
}
|
||||
report := FormatDryRunReport(candidates, 4)
|
||||
if !strings.Contains(report, "total=3") || !strings.Contains(report, "concurrency=4") ||
|
||||
!strings.Contains(report, "knowledge_base=kb-1 count=2") || !strings.Contains(report, "knowledge_base=kb-2 count=1") {
|
||||
t.Fatalf("report = %q", report)
|
||||
}
|
||||
if strings.Contains(report, "knowledge-1") || strings.Contains(report, "pub-1") {
|
||||
t.Fatalf("report leaked knowledge/publication ID: %q", report)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFormatLegacyVerificationReportContainsOnlyAggregateCounts(t *testing.T) {
|
||||
report := FormatLegacyVerificationReport(LegacyVerificationReport{
|
||||
Total: 530, Existing: 500, Missing: 20, WrongKB: 10,
|
||||
Completed: 490, Failed: 10, ChunkChecked: 20, ChunkEmpty: 2, HTTPErrors: 3,
|
||||
})
|
||||
for _, expected := range []string{"total=530", "existing=500", "missing=20", "wrong_kb=10", "completed=490", "failed=10", "chunk_checked=20", "chunk_empty=2", "http_errors=3"} {
|
||||
if !strings.Contains(report, expected) {
|
||||
t.Fatalf("report missing %q: %q", expected, report)
|
||||
}
|
||||
}
|
||||
for _, forbidden := range []string{"knowledge_id", "knowledge_base", "publication_id"} {
|
||||
if strings.Contains(report, forbidden) {
|
||||
t.Fatalf("report leaked identifier field %q: %q", forbidden, report)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type legacyClientStub struct {
|
||||
getResults map[string]legacyGetResult
|
||||
chunkTotal map[string]int64
|
||||
chunkQueries int
|
||||
}
|
||||
|
||||
type legacyGetResult struct {
|
||||
knowledge weknora.Knowledge
|
||||
err error
|
||||
}
|
||||
|
||||
type concurrentLegacyClient struct {
|
||||
current atomic.Int64
|
||||
maximum atomic.Int64
|
||||
}
|
||||
|
||||
func (client *concurrentLegacyClient) GetKnowledge(context.Context, string) (weknora.Knowledge, error) {
|
||||
current := client.current.Add(1)
|
||||
for {
|
||||
maximum := client.maximum.Load()
|
||||
if current <= maximum || client.maximum.CompareAndSwap(maximum, current) {
|
||||
break
|
||||
}
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
client.current.Add(-1)
|
||||
return weknora.Knowledge{KnowledgeBaseID: "kb-1", ParseStatus: "completed"}, nil
|
||||
}
|
||||
|
||||
func (*concurrentLegacyClient) ListChunks(context.Context, string, int, int, []string) ([]weknora.Chunk, int64, error) {
|
||||
return nil, 1, nil
|
||||
}
|
||||
|
||||
func (client *legacyClientStub) GetKnowledge(_ context.Context, knowledgeID string) (weknora.Knowledge, error) {
|
||||
result, ok := client.getResults[knowledgeID]
|
||||
if !ok {
|
||||
return weknora.Knowledge{}, &weknora.HTTPError{StatusCode: 404}
|
||||
}
|
||||
return result.knowledge, result.err
|
||||
}
|
||||
|
||||
func (client *legacyClientStub) ListChunks(_ context.Context, knowledgeID string, _ int, _ int, _ []string) ([]weknora.Chunk, int64, error) {
|
||||
client.chunkQueries++
|
||||
total, ok := client.chunkTotal[knowledgeID]
|
||||
if !ok {
|
||||
total = 0
|
||||
}
|
||||
chunks := make([]weknora.Chunk, 0, total)
|
||||
for i := int64(0); i < total; i++ {
|
||||
chunks = append(chunks, weknora.Chunk{ID: "chunk", KnowledgeID: knowledgeID})
|
||||
}
|
||||
return chunks, total, nil
|
||||
}
|
||||
@@ -0,0 +1,141 @@
|
||||
package testweknora
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
"knowledge-governance/internal/jobs"
|
||||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||||
"knowledge-governance/internal/platform/weknora"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrInvalidPollJobPayload = errors.New("test_poll_invalid_payload")
|
||||
ErrPollRemoteFailed = errors.New("test_poll_remote_failed")
|
||||
ErrPollRemoteIdentity = errors.New("test_poll_remote_identity_invalid")
|
||||
ErrPollStateConflict = errors.New("test_poll_state_conflict")
|
||||
)
|
||||
|
||||
type PollStore interface {
|
||||
LoadPublicationForPoll(context.Context, string) (postgresstore.PublicationForPoll, error)
|
||||
RecordRemoteStatusAndScheduleNextPoll(context.Context, string, int, governance.ProcessingState, *time.Time, time.Time, time.Time) (int, error)
|
||||
RecordCompletedAndScheduleValidation(context.Context, string, int, *time.Time, time.Time) error
|
||||
RecordProcessingFailure(context.Context, string, int, string, time.Time) error
|
||||
MarkStalled(context.Context, string, int, time.Time, time.Time) error
|
||||
}
|
||||
|
||||
type PollClient interface {
|
||||
GetKnowledge(context.Context, string) (weknora.Knowledge, error)
|
||||
}
|
||||
|
||||
type PollHandlerConfig struct {
|
||||
Store PollStore
|
||||
Client PollClient
|
||||
PollDelay time.Duration
|
||||
StallAfter time.Duration
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
type PollHandler struct {
|
||||
store PollStore
|
||||
client PollClient
|
||||
pollDelay time.Duration
|
||||
stallAfter time.Duration
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
type pollPayload struct {
|
||||
PublicationID string `json:"publication_id"`
|
||||
Sequence int `json:"sequence"`
|
||||
}
|
||||
|
||||
func NewPollHandler(config PollHandlerConfig) (*PollHandler, error) {
|
||||
if config.Store == nil || config.Client == nil || config.PollDelay <= 0 || config.StallAfter <= 0 {
|
||||
return nil, errors.New("invalid test poll handler configuration")
|
||||
}
|
||||
if config.Now == nil {
|
||||
config.Now = time.Now
|
||||
}
|
||||
return &PollHandler{
|
||||
store: config.Store, client: config.Client, pollDelay: config.PollDelay,
|
||||
stallAfter: config.StallAfter, now: config.Now,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (handler *PollHandler) Handle(ctx context.Context, job jobs.Job) error {
|
||||
var payload pollPayload
|
||||
if err := decodeStrictJSON(job.Payload, &payload); err != nil || payload.PublicationID == "" || payload.Sequence < 1 {
|
||||
return jobs.Permanent(jobs.ErrCodeHandlerFallback, ErrInvalidPollJobPayload)
|
||||
}
|
||||
publication, err := handler.store.LoadPublicationForPoll(ctx, payload.PublicationID)
|
||||
if err != nil {
|
||||
return ErrPollStateConflict
|
||||
}
|
||||
if publication.KnowledgeID == "" || publication.PollSequence != payload.Sequence {
|
||||
return ErrPollStateConflict
|
||||
}
|
||||
now := handler.now()
|
||||
knowledge, err := handler.client.GetKnowledge(ctx, publication.KnowledgeID)
|
||||
if err != nil {
|
||||
return ErrPollRemoteFailed
|
||||
}
|
||||
if knowledge.ID != publication.KnowledgeID || knowledge.KnowledgeBaseID != publication.TestKBID {
|
||||
return ErrPollRemoteIdentity
|
||||
}
|
||||
state := mapParseStatus(knowledge.ParseStatus)
|
||||
if state == governance.ProcessingCompleted {
|
||||
if err := handler.store.RecordCompletedAndScheduleValidation(ctx, publication.ID, publication.PollSequence, &knowledge.UpdatedAt, now); err != nil {
|
||||
return ErrPollStateConflict
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if state == governance.ProcessingFailed {
|
||||
if err := handler.store.RecordProcessingFailure(ctx, publication.ID, publication.PollSequence, "remote_parse_failed", now); err != nil {
|
||||
return ErrPollStateConflict
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if !state.Valid() {
|
||||
if err := handler.store.RecordProcessingFailure(ctx, publication.ID, publication.PollSequence, "remote_unknown_status", now); err != nil {
|
||||
return ErrPollStateConflict
|
||||
}
|
||||
return nil
|
||||
}
|
||||
cutoff := now.Add(-handler.stallAfter)
|
||||
lastProgress := publication.LastProgressAt
|
||||
if lastProgress == nil {
|
||||
lastProgress = publication.ProcessingStartedAt
|
||||
}
|
||||
if lastProgress == nil {
|
||||
lastProgress = publication.LastObservedAt
|
||||
}
|
||||
if lastProgress != nil && !lastProgress.After(cutoff) {
|
||||
if err := handler.store.MarkStalled(ctx, publication.ID, publication.PollSequence, cutoff, now); err != nil {
|
||||
return ErrPollStateConflict
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if _, err := handler.store.RecordRemoteStatusAndScheduleNextPoll(ctx, publication.ID, publication.PollSequence, state, &knowledge.UpdatedAt, now, now.Add(handler.pollDelay)); err != nil {
|
||||
return ErrPollStateConflict
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func mapParseStatus(status string) governance.ProcessingState {
|
||||
switch status {
|
||||
case "pending":
|
||||
return governance.ProcessingPending
|
||||
case "processing":
|
||||
return governance.ProcessingProcessing
|
||||
case "finalizing":
|
||||
return governance.ProcessingFinalizing
|
||||
case "completed":
|
||||
return governance.ProcessingCompleted
|
||||
case "failed":
|
||||
return governance.ProcessingFailed
|
||||
default:
|
||||
return governance.ProcessingState(status)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,188 @@
|
||||
package testweknora
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
"knowledge-governance/internal/jobs"
|
||||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||||
"knowledge-governance/internal/platform/weknora"
|
||||
)
|
||||
|
||||
func TestPollHandlerSchedulesNextSequenceForRemoteProgress(t *testing.T) {
|
||||
remoteUpdated := time.Date(2026, time.September, 4, 3, 39, 0, 0, time.UTC)
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodGet || r.URL.Path != "/api/v1/knowledge/knowledge-1" {
|
||||
t.Fatalf("request = %s %s", r.Method, r.URL.Path)
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"success":true,"data":{"id":"knowledge-1","knowledge_base_id":"kb-1","parse_status":"finalizing","updated_at":"2026-09-04T03:39:00Z"}}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
client, err := weknora.NewClient(weknora.ClientConfig{BaseURL: server.URL, APIKey: "test-api-key", Timeout: time.Second})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Date(2026, time.September, 4, 3, 40, 0, 0, time.UTC)
|
||||
lastProgress := now.Add(-time.Minute)
|
||||
store := &pollStoreStub{publication: postgresstore.PublicationForPoll{
|
||||
ID: "publication-1", TestKBID: "kb-1", KnowledgeID: "knowledge-1",
|
||||
ProcessingState: governance.ProcessingProcessing, PollSequence: 2,
|
||||
LastRemoteStatus: "processing", LastProgressAt: &lastProgress,
|
||||
}}
|
||||
handler, err := NewPollHandler(PollHandlerConfig{
|
||||
Store: store, Client: client, PollDelay: 15 * time.Second,
|
||||
StallAfter: 30 * time.Minute, Now: func() time.Time { return now },
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := handler.Handle(context.Background(), jobs.Job{Payload: []byte(`{"publication_id":"publication-1","sequence":2}`)}); err != nil {
|
||||
t.Fatalf("Handle() error = %v", err)
|
||||
}
|
||||
if store.recordedState != governance.ProcessingFinalizing || store.recordedRemoteUpdated == nil || !store.recordedRemoteUpdated.Equal(remoteUpdated) || !store.nextPollAt.Equal(now.Add(15*time.Second)) {
|
||||
t.Fatalf("store = %#v", store)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPollHandlerCompletesAndSchedulesValidationWithoutNextPoll(t *testing.T) {
|
||||
completedAt := time.Date(2026, time.September, 4, 3, 45, 0, 0, time.UTC)
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodGet || r.URL.Path != "/api/v1/knowledge/knowledge-1" {
|
||||
t.Fatalf("request = %s %s", r.Method, r.URL.Path)
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"success":true,"data":{"id":"knowledge-1","knowledge_base_id":"kb-1","parse_status":"completed","updated_at":"2026-09-04T03:45:00Z"}}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
client, err := weknora.NewClient(weknora.ClientConfig{BaseURL: server.URL, APIKey: "test-api-key", Timeout: time.Second})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Date(2026, time.September, 4, 3, 46, 0, 0, time.UTC)
|
||||
lastProgress := now.Add(-time.Minute)
|
||||
store := &pollStoreStub{publication: postgresstore.PublicationForPoll{
|
||||
ID: "publication-1", TestKBID: "kb-1", KnowledgeID: "knowledge-1",
|
||||
ProcessingState: governance.ProcessingProcessing, PollSequence: 3,
|
||||
LastRemoteStatus: "processing", LastProgressAt: &lastProgress,
|
||||
}}
|
||||
handler, err := NewPollHandler(PollHandlerConfig{
|
||||
Store: store, Client: client, PollDelay: 15 * time.Second,
|
||||
StallAfter: 30 * time.Minute, Now: func() time.Time { return now },
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := handler.Handle(context.Background(), jobs.Job{Payload: []byte(`{"publication_id":"publication-1","sequence":3}`)}); err != nil {
|
||||
t.Fatalf("Handle() error = %v", err)
|
||||
}
|
||||
if !store.completedValidated || store.recordedState != "" || store.recordedRemoteUpdated == nil || !store.recordedRemoteUpdated.Equal(completedAt) {
|
||||
t.Fatalf("store = %#v", store)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPollHandlerMarksStalledWhenNoProgressExceedsThreshold(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodGet || r.URL.Path != "/api/v1/knowledge/knowledge-1" {
|
||||
t.Fatalf("request = %s %s", r.Method, r.URL.Path)
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"success":true,"data":{"id":"knowledge-1","knowledge_base_id":"kb-1","parse_status":"finalizing","updated_at":"2026-09-04T03:10:00Z"}}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
client, err := weknora.NewClient(weknora.ClientConfig{BaseURL: server.URL, APIKey: "test-api-key", Timeout: time.Second})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Date(2026, time.September, 4, 3, 50, 0, 0, time.UTC)
|
||||
stalledSince := now.Add(-45 * time.Minute)
|
||||
store := &pollStoreStub{publication: postgresstore.PublicationForPoll{
|
||||
ID: "publication-1", TestKBID: "kb-1", KnowledgeID: "knowledge-1",
|
||||
ProcessingState: governance.ProcessingProcessing, PollSequence: 4,
|
||||
LastRemoteStatus: "processing", LastProgressAt: &stalledSince,
|
||||
}}
|
||||
handler, err := NewPollHandler(PollHandlerConfig{
|
||||
Store: store, Client: client, PollDelay: 15 * time.Second,
|
||||
StallAfter: 30 * time.Minute, Now: func() time.Time { return now },
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := handler.Handle(context.Background(), jobs.Job{Payload: []byte(`{"publication_id":"publication-1","sequence":4}`)}); err != nil {
|
||||
t.Fatalf("Handle() error = %v", err)
|
||||
}
|
||||
if !store.stalledMarked || store.recordedState != "" || !store.stalledCutoff.Equal(now.Add(-30*time.Minute)) {
|
||||
t.Fatalf("store = %#v", store)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPollHandlerRejectsStaleSequenceWithoutPollingRemote(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
t.Fatal("stale sequence must not poll the remote knowledge endpoint")
|
||||
}))
|
||||
defer server.Close()
|
||||
client, err := weknora.NewClient(weknora.ClientConfig{BaseURL: server.URL, APIKey: "test-api-key", Timeout: time.Second})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Date(2026, time.September, 4, 4, 0, 0, 0, time.UTC)
|
||||
store := &pollStoreStub{publication: postgresstore.PublicationForPoll{
|
||||
ID: "publication-1", TestKBID: "kb-1", KnowledgeID: "knowledge-1",
|
||||
ProcessingState: governance.ProcessingProcessing, PollSequence: 5,
|
||||
}}
|
||||
handler, err := NewPollHandler(PollHandlerConfig{
|
||||
Store: store, Client: client, PollDelay: 15 * time.Second,
|
||||
StallAfter: 30 * time.Minute, Now: func() time.Time { return now },
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := handler.Handle(context.Background(), jobs.Job{Payload: []byte(`{"publication_id":"publication-1","sequence":2}`)}); !errors.Is(err, ErrPollStateConflict) {
|
||||
t.Fatalf("Handle() error = %v, want state conflict", err)
|
||||
}
|
||||
if store.recordedState != "" || store.stalledMarked || store.completedValidated {
|
||||
t.Fatalf("store mutated for stale sequence: %#v", store)
|
||||
}
|
||||
}
|
||||
|
||||
type pollStoreStub struct {
|
||||
publication postgresstore.PublicationForPoll
|
||||
recordedState governance.ProcessingState
|
||||
recordedRemoteUpdated *time.Time
|
||||
nextPollAt time.Time
|
||||
completedValidated bool
|
||||
stalledMarked bool
|
||||
stalledCutoff time.Time
|
||||
}
|
||||
|
||||
func (store *pollStoreStub) LoadPublicationForPoll(context.Context, string) (postgresstore.PublicationForPoll, error) {
|
||||
return store.publication, nil
|
||||
}
|
||||
|
||||
func (store *pollStoreStub) RecordRemoteStatusAndScheduleNextPoll(_ context.Context, _ string, _ int, state governance.ProcessingState, remoteUpdated *time.Time, _ time.Time, nextAt time.Time) (int, error) {
|
||||
store.recordedState = state
|
||||
store.recordedRemoteUpdated = remoteUpdated
|
||||
store.nextPollAt = nextAt
|
||||
return store.publication.PollSequence + 1, nil
|
||||
}
|
||||
|
||||
func (store *pollStoreStub) RecordCompletedAndScheduleValidation(_ context.Context, _ string, _ int, remoteUpdated *time.Time, _ time.Time) error {
|
||||
store.completedValidated = true
|
||||
store.recordedRemoteUpdated = remoteUpdated
|
||||
return nil
|
||||
}
|
||||
|
||||
func (store *pollStoreStub) RecordProcessingFailure(context.Context, string, int, string, time.Time) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (store *pollStoreStub) MarkStalled(_ context.Context, _ string, _ int, cutoff, _ time.Time) error {
|
||||
store.stalledMarked = true
|
||||
store.stalledCutoff = cutoff
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
package testweknora
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"knowledge-governance/internal/jobs"
|
||||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||||
"knowledge-governance/internal/platform/weknora"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrInvalidRemoveJobPayload = errors.New("test_remove_invalid_payload")
|
||||
ErrRemoveRemoteFailed = errors.New("test_remove_remote_failed")
|
||||
ErrRemoveRemoteIdentity = errors.New("test_remove_remote_identity_invalid")
|
||||
ErrRemovePending = errors.New("test_remove_pending")
|
||||
ErrRemoveStateConflict = errors.New("test_remove_state_conflict")
|
||||
)
|
||||
|
||||
type RemoveStore interface {
|
||||
LoadPublicationForRemoval(context.Context, string) (postgresstore.PublicationForRemoval, error)
|
||||
RecordRemovalCompleted(context.Context, string, time.Time) error
|
||||
RecordRemovalFailure(context.Context, string, string, time.Time) error
|
||||
}
|
||||
|
||||
type RemoveClient interface {
|
||||
GetKnowledge(context.Context, string) (weknora.Knowledge, error)
|
||||
DeleteKnowledge(context.Context, string) error
|
||||
}
|
||||
|
||||
type RemoveHandlerConfig struct {
|
||||
Store RemoveStore
|
||||
Client RemoveClient
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
type RemoveHandler struct {
|
||||
store RemoveStore
|
||||
client RemoveClient
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
type removePayload struct {
|
||||
PublicationID string `json:"publication_id"`
|
||||
}
|
||||
|
||||
func NewRemoveHandler(config RemoveHandlerConfig) (*RemoveHandler, error) {
|
||||
if config.Store == nil || config.Client == nil {
|
||||
return nil, errors.New("invalid test removal handler configuration")
|
||||
}
|
||||
if config.Now == nil {
|
||||
config.Now = time.Now
|
||||
}
|
||||
return &RemoveHandler{store: config.Store, client: config.Client, now: config.Now}, nil
|
||||
}
|
||||
|
||||
func (handler *RemoveHandler) Handle(ctx context.Context, job jobs.Job) error {
|
||||
var payload removePayload
|
||||
if err := decodeStrictJSON(job.Payload, &payload); err != nil || payload.PublicationID == "" {
|
||||
return jobs.Permanent(jobs.ErrCodeHandlerFallback, ErrInvalidRemoveJobPayload)
|
||||
}
|
||||
candidate, err := handler.store.LoadPublicationForRemoval(ctx, payload.PublicationID)
|
||||
if err != nil {
|
||||
return ErrRemoveStateConflict
|
||||
}
|
||||
knowledge, err := handler.client.GetKnowledge(ctx, candidate.KnowledgeID)
|
||||
if isNotFound(err) {
|
||||
return handler.complete(ctx, candidate.ID)
|
||||
}
|
||||
if err != nil {
|
||||
return ErrRemoveRemoteFailed
|
||||
}
|
||||
if knowledge.ID != candidate.KnowledgeID || knowledge.KnowledgeBaseID != candidate.TestKBID {
|
||||
return handler.failClosed(ctx, candidate.ID, "removal_identity_invalid")
|
||||
}
|
||||
if err := handler.client.DeleteKnowledge(ctx, candidate.KnowledgeID); err != nil {
|
||||
if isNotFound(err) {
|
||||
return handler.complete(ctx, candidate.ID)
|
||||
}
|
||||
if !weknora.IsRetryable(err) {
|
||||
return handler.failClosed(ctx, candidate.ID, "removal_remote_rejected")
|
||||
}
|
||||
return ErrRemoveRemoteFailed
|
||||
}
|
||||
knowledge, err = handler.client.GetKnowledge(ctx, candidate.KnowledgeID)
|
||||
if isNotFound(err) {
|
||||
return handler.complete(ctx, candidate.ID)
|
||||
}
|
||||
if err != nil {
|
||||
return ErrRemoveRemoteFailed
|
||||
}
|
||||
if knowledge.ID != candidate.KnowledgeID || knowledge.KnowledgeBaseID != candidate.TestKBID {
|
||||
return handler.failClosed(ctx, candidate.ID, "removal_identity_invalid")
|
||||
}
|
||||
return ErrRemovePending
|
||||
}
|
||||
|
||||
func (handler *RemoveHandler) complete(ctx context.Context, publicationID string) error {
|
||||
if err := handler.store.RecordRemovalCompleted(ctx, publicationID, handler.now()); err != nil {
|
||||
return ErrRemoveStateConflict
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (handler *RemoveHandler) failClosed(ctx context.Context, publicationID, code string) error {
|
||||
if err := handler.store.RecordRemovalFailure(ctx, publicationID, code, handler.now()); err != nil {
|
||||
return ErrRemoveStateConflict
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func isNotFound(err error) bool {
|
||||
var httpErr *weknora.HTTPError
|
||||
return errors.As(err, &httpErr) && httpErr.StatusCode == http.StatusNotFound
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
package testweknora
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
"knowledge-governance/internal/jobs"
|
||||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||||
"knowledge-governance/internal/platform/weknora"
|
||||
)
|
||||
|
||||
type removalStoreStub struct {
|
||||
candidate postgresstore.PublicationForRemoval
|
||||
completed string
|
||||
failed string
|
||||
}
|
||||
|
||||
func (store *removalStoreStub) LoadPublicationForRemoval(context.Context, string) (postgresstore.PublicationForRemoval, error) {
|
||||
return store.candidate, nil
|
||||
}
|
||||
|
||||
func (store *removalStoreStub) RecordRemovalCompleted(_ context.Context, publicationID string, _ time.Time) error {
|
||||
store.completed = publicationID
|
||||
return nil
|
||||
}
|
||||
|
||||
func (store *removalStoreStub) RecordRemovalFailure(_ context.Context, publicationID, code string, _ time.Time) error {
|
||||
store.failed = publicationID + ":" + code
|
||||
return nil
|
||||
}
|
||||
|
||||
type removalClientStub struct {
|
||||
knowledge weknora.Knowledge
|
||||
getCalls int
|
||||
deleted string
|
||||
keepAfterDelete bool
|
||||
}
|
||||
|
||||
func (client *removalClientStub) GetKnowledge(context.Context, string) (weknora.Knowledge, error) {
|
||||
client.getCalls++
|
||||
if client.getCalls > 1 && !client.keepAfterDelete {
|
||||
return weknora.Knowledge{}, &weknora.HTTPError{StatusCode: 404}
|
||||
}
|
||||
return client.knowledge, nil
|
||||
}
|
||||
|
||||
func (client *removalClientStub) DeleteKnowledge(_ context.Context, knowledgeID string) error {
|
||||
client.deleted = knowledgeID
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestRemoveHandlerVerifiesIdentityAndReadBackBeforeCompletion(t *testing.T) {
|
||||
store := &removalStoreStub{candidate: postgresstore.PublicationForRemoval{
|
||||
ID: "publication-1", TestKBID: "kb-1", KnowledgeID: "knowledge-1", RetrievalState: governance.RetrievalDisabling,
|
||||
}}
|
||||
client := &removalClientStub{knowledge: weknora.Knowledge{ID: "knowledge-1", KnowledgeBaseID: "kb-1"}}
|
||||
handler, err := NewRemoveHandler(RemoveHandlerConfig{Store: store, Client: client, Now: func() time.Time { return time.Date(2026, 9, 9, 3, 0, 0, 0, time.UTC) }})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := handler.Handle(context.Background(), jobs.Job{Payload: []byte(`{"publication_id":"publication-1"}`)}); err != nil {
|
||||
t.Fatalf("Handle() error = %v", err)
|
||||
}
|
||||
if client.deleted != "knowledge-1" || store.completed != "publication-1" || store.failed != "" {
|
||||
t.Fatalf("deleted=%q completed=%q failed=%q", client.deleted, store.completed, store.failed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoveHandlerRejectsMismatchedKnowledgeBaseWithoutDeleting(t *testing.T) {
|
||||
store := &removalStoreStub{candidate: postgresstore.PublicationForRemoval{
|
||||
ID: "publication-1", TestKBID: "kb-1", KnowledgeID: "knowledge-1", RetrievalState: governance.RetrievalDisabling,
|
||||
}}
|
||||
client := &removalClientStub{knowledge: weknora.Knowledge{ID: "knowledge-1", KnowledgeBaseID: "other-kb"}}
|
||||
handler, err := NewRemoveHandler(RemoveHandlerConfig{Store: store, Client: client})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = handler.Handle(context.Background(), jobs.Job{Payload: []byte(`{"publication_id":"publication-1"}`)})
|
||||
if err != nil || client.deleted != "" || store.failed != "publication-1:removal_identity_invalid" {
|
||||
t.Fatalf("error=%v deleted=%q failed=%q", err, client.deleted, store.failed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoveHandlerRetriesWhileAsynchronousDeleteIsStillVisible(t *testing.T) {
|
||||
store := &removalStoreStub{candidate: postgresstore.PublicationForRemoval{
|
||||
ID: "publication-1", TestKBID: "kb-1", KnowledgeID: "knowledge-1", RetrievalState: governance.RetrievalDisabling,
|
||||
}}
|
||||
client := &removalClientStub{knowledge: weknora.Knowledge{ID: "knowledge-1", KnowledgeBaseID: "kb-1"}, keepAfterDelete: true}
|
||||
handler, err := NewRemoveHandler(RemoveHandlerConfig{Store: store, Client: client})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = handler.Handle(context.Background(), jobs.Job{Payload: []byte(`{"publication_id":"publication-1"}`)})
|
||||
if !errors.Is(err, ErrRemovePending) || store.completed != "" {
|
||||
t.Fatalf("error=%v completed=%q", err, store.completed)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,241 @@
|
||||
package testweknora
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
"knowledge-governance/internal/jobs"
|
||||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||||
"knowledge-governance/internal/platform/spool"
|
||||
"knowledge-governance/internal/platform/weknora"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrInvalidUploadJobPayload = errors.New("test_upload_invalid_payload")
|
||||
ErrUploadArtifactInvalid = errors.New("test_upload_artifact_invalid")
|
||||
ErrUploadRemoteFailed = errors.New("test_upload_remote_failed")
|
||||
ErrUploadRemoteIdentity = errors.New("test_upload_remote_identity_invalid")
|
||||
ErrUploadStateConflict = errors.New("test_upload_state_conflict")
|
||||
)
|
||||
|
||||
type UploadStore interface {
|
||||
LoadPublicationForUpload(context.Context, string) (postgresstore.PublicationForUpload, error)
|
||||
MarkUploading(context.Context, string, time.Time) error
|
||||
RecordUploadAndSchedulePoll(context.Context, string, string, time.Time, time.Time) (int, error)
|
||||
RecordUploadFailure(context.Context, string, string, time.Time) error
|
||||
}
|
||||
|
||||
type UploadClient interface {
|
||||
UploadFile(context.Context, weknora.UploadRequest) (weknora.Knowledge, error)
|
||||
ListKnowledge(context.Context, string, int, int) ([]weknora.Knowledge, int64, error)
|
||||
GetKnowledge(context.Context, string) (weknora.Knowledge, error)
|
||||
}
|
||||
|
||||
type UploadHandlerConfig struct {
|
||||
Store UploadStore
|
||||
Client UploadClient
|
||||
SpoolRoot string
|
||||
MaxUploadBytes int64
|
||||
FirstPollDelay time.Duration
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
type UploadHandler struct {
|
||||
store UploadStore
|
||||
client UploadClient
|
||||
spoolRoot string
|
||||
maxUploadBytes int64
|
||||
firstPollDelay time.Duration
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
type uploadPayload struct {
|
||||
PublicationID string `json:"publication_id"`
|
||||
TargetScope string `json:"target_scope,omitempty"`
|
||||
TargetKBID string `json:"target_kb_id,omitempty"`
|
||||
}
|
||||
|
||||
func NewUploadHandler(config UploadHandlerConfig) (*UploadHandler, error) {
|
||||
if config.Store == nil || config.Client == nil || !filepath.IsAbs(config.SpoolRoot) ||
|
||||
config.MaxUploadBytes <= 0 || config.FirstPollDelay <= 0 {
|
||||
return nil, errors.New("invalid test upload handler configuration")
|
||||
}
|
||||
if config.Now == nil {
|
||||
config.Now = time.Now
|
||||
}
|
||||
return &UploadHandler{
|
||||
store: config.Store, client: config.Client, spoolRoot: config.SpoolRoot,
|
||||
maxUploadBytes: config.MaxUploadBytes, firstPollDelay: config.FirstPollDelay, now: config.Now,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (handler *UploadHandler) Handle(ctx context.Context, job jobs.Job) error {
|
||||
var payload uploadPayload
|
||||
if err := decodeStrictJSON(job.Payload, &payload); err != nil || payload.PublicationID == "" {
|
||||
return jobs.Permanent(jobs.ErrCodeHandlerFallback, ErrInvalidUploadJobPayload)
|
||||
}
|
||||
candidate, err := handler.store.LoadPublicationForUpload(ctx, payload.PublicationID)
|
||||
if err != nil {
|
||||
return ErrUploadStateConflict
|
||||
}
|
||||
// P4: the job snapshots its target; a mismatch against the stored publication
|
||||
// means the plan changed underneath and this delivery must not guess.
|
||||
if payload.TargetKBID != "" && payload.TargetKBID != candidate.TestKBID {
|
||||
return ErrUploadStateConflict
|
||||
}
|
||||
if payload.TargetScope != "" && payload.TargetScope != string(candidate.TargetScope) {
|
||||
return ErrUploadStateConflict
|
||||
}
|
||||
if candidate.IngestState == governance.IngestUploading {
|
||||
knowledge, err := handler.recoverAmbiguousUpload(ctx, candidate)
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrUploadRemoteIdentity) {
|
||||
return handler.recordTerminalUploadFailure(ctx, candidate.ID, "upload_identity_ambiguous")
|
||||
}
|
||||
return err
|
||||
}
|
||||
now := handler.now()
|
||||
if _, err := handler.store.RecordUploadAndSchedulePoll(ctx, candidate.ID, knowledge.ID, now, now.Add(handler.firstPollDelay)); err != nil {
|
||||
return ErrUploadStateConflict
|
||||
}
|
||||
return nil
|
||||
}
|
||||
artifact, err := spool.OpenVerified(handler.spoolRoot, spool.Expectation{
|
||||
RelativeRef: candidate.LocalSpoolRef,
|
||||
SHA256: candidate.SHA256,
|
||||
SizeBytes: candidate.SizeBytes,
|
||||
Format: candidate.ValidatedFormat,
|
||||
}, handler.maxUploadBytes)
|
||||
if err != nil {
|
||||
return handler.recordTerminalUploadFailure(ctx, candidate.ID, "upload_artifact_invalid")
|
||||
}
|
||||
defer artifact.Close()
|
||||
if err := handler.store.MarkUploading(ctx, candidate.ID, handler.now()); err != nil {
|
||||
return ErrUploadStateConflict
|
||||
}
|
||||
knowledge, uploadErr := handler.client.UploadFile(ctx, weknora.UploadRequest{
|
||||
KnowledgeBaseID: candidate.TestKBID,
|
||||
FileName: safeUploadFileName(candidate.Title, candidate.ID, candidate.ValidatedFormat),
|
||||
Content: artifact,
|
||||
Metadata: map[string]string{
|
||||
"content_sha256": candidate.SHA256,
|
||||
"publication_id": candidate.ID,
|
||||
},
|
||||
Channel: "governance",
|
||||
})
|
||||
if err := artifact.VerifyComplete(); err != nil {
|
||||
return jobs.Permanent(jobs.ErrCodeArtifactInvalid, ErrUploadArtifactInvalid)
|
||||
}
|
||||
if uploadErr != nil {
|
||||
if !errors.Is(uploadErr, weknora.ErrConflict) && !weknora.IsRetryable(uploadErr) {
|
||||
return ErrUploadRemoteFailed
|
||||
}
|
||||
knowledge, err = handler.recoverAmbiguousUpload(ctx, candidate)
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrUploadRemoteIdentity) {
|
||||
return handler.recordTerminalUploadFailure(ctx, candidate.ID, "upload_identity_ambiguous")
|
||||
}
|
||||
return err
|
||||
}
|
||||
}
|
||||
if knowledge.ID == "" || knowledge.KnowledgeBaseID != candidate.TestKBID {
|
||||
return ErrUploadRemoteIdentity
|
||||
}
|
||||
now := handler.now()
|
||||
if _, err := handler.store.RecordUploadAndSchedulePoll(ctx, candidate.ID, knowledge.ID, now, now.Add(handler.firstPollDelay)); err != nil {
|
||||
return ErrUploadStateConflict
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (handler *UploadHandler) recoverAmbiguousUpload(ctx context.Context, candidate postgresstore.PublicationForUpload) (weknora.Knowledge, error) {
|
||||
const pageSize = 100
|
||||
matches := make([]weknora.Knowledge, 0, 2)
|
||||
for page := 1; page <= 100; page++ {
|
||||
items, total, err := handler.client.ListKnowledge(ctx, candidate.TestKBID, page, pageSize)
|
||||
if err != nil {
|
||||
return weknora.Knowledge{}, ErrUploadRemoteFailed
|
||||
}
|
||||
for _, item := range items {
|
||||
publicationID, publicationOK := metadataString(item.Metadata, "publication_id")
|
||||
contentSHA, contentOK := metadataString(item.Metadata, "content_sha256")
|
||||
if item.KnowledgeBaseID == candidate.TestKBID && publicationOK && contentOK && publicationID == candidate.ID && contentSHA == candidate.SHA256 {
|
||||
matches = append(matches, item)
|
||||
if len(matches) > 1 {
|
||||
return weknora.Knowledge{}, ErrUploadRemoteIdentity
|
||||
}
|
||||
}
|
||||
}
|
||||
if int64(page*pageSize) >= total {
|
||||
break
|
||||
}
|
||||
if len(items) == 0 {
|
||||
return weknora.Knowledge{}, ErrUploadRemoteFailed
|
||||
}
|
||||
}
|
||||
if len(matches) != 1 {
|
||||
return weknora.Knowledge{}, ErrUploadRemoteFailed
|
||||
}
|
||||
verified, err := handler.client.GetKnowledge(ctx, matches[0].ID)
|
||||
if err != nil {
|
||||
return weknora.Knowledge{}, ErrUploadRemoteFailed
|
||||
}
|
||||
publicationID, publicationOK := metadataString(verified.Metadata, "publication_id")
|
||||
contentSHA, contentOK := metadataString(verified.Metadata, "content_sha256")
|
||||
if verified.ID != matches[0].ID || verified.KnowledgeBaseID != candidate.TestKBID ||
|
||||
!publicationOK || !contentOK || publicationID != candidate.ID || contentSHA != candidate.SHA256 {
|
||||
return weknora.Knowledge{}, ErrUploadRemoteIdentity
|
||||
}
|
||||
return verified, nil
|
||||
}
|
||||
|
||||
func metadataString(metadata map[string]any, key string) (string, bool) {
|
||||
value, ok := metadata[key].(string)
|
||||
return value, ok
|
||||
}
|
||||
|
||||
func (handler *UploadHandler) recordTerminalUploadFailure(ctx context.Context, publicationID, code string) error {
|
||||
if err := handler.store.RecordUploadFailure(ctx, publicationID, code, handler.now()); err != nil {
|
||||
return ErrUploadStateConflict
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func decodeStrictJSON(data []byte, target any) error {
|
||||
decoder := json.NewDecoder(bytes.NewReader(data))
|
||||
decoder.DisallowUnknownFields()
|
||||
if err := decoder.Decode(target); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := decoder.Decode(&struct{}{}); err != io.EOF {
|
||||
return errors.New("extra JSON value")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func safeUploadFileName(title, fallback, format string) string {
|
||||
base := strings.TrimSpace(title)
|
||||
base = strings.Map(func(character rune) rune {
|
||||
switch character {
|
||||
case '/', '\\', '\x00', '\r', '\n', '\t':
|
||||
return '_'
|
||||
default:
|
||||
return character
|
||||
}
|
||||
}, base)
|
||||
base = strings.Trim(base, ". ")
|
||||
if base == "" {
|
||||
base = fallback
|
||||
}
|
||||
if len(base) > 120 {
|
||||
base = base[:120]
|
||||
}
|
||||
return base + "." + format
|
||||
}
|
||||
@@ -0,0 +1,354 @@
|
||||
package testweknora
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
"knowledge-governance/internal/jobs"
|
||||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||||
"knowledge-governance/internal/platform/weknora"
|
||||
)
|
||||
|
||||
func TestUploadHandlerStreamsVerifiedArtifactAndSchedulesFirstPoll(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
if err := os.Mkdir(filepath.Join(root, "objects"), 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body := []byte("safe governance document\n")
|
||||
if err := os.WriteFile(filepath.Join(root, "objects", "artifact.md"), body, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
digest := sha256.Sum256(body)
|
||||
sha := hex.EncodeToString(digest[:])
|
||||
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPost || r.URL.Path != "/api/v1/knowledge-bases/kb-1/knowledge/file" {
|
||||
t.Fatalf("request = %s %s", r.Method, r.URL.Path)
|
||||
}
|
||||
if err := r.ParseMultipartForm(1 << 20); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
file, _, err := r.FormFile("file")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer file.Close()
|
||||
uploaded, err := io.ReadAll(file)
|
||||
if err != nil || string(uploaded) != string(body) {
|
||||
t.Fatalf("uploaded body = %q, %v", uploaded, err)
|
||||
}
|
||||
if r.FormValue("metadata") != `{"content_sha256":"`+sha+`","publication_id":"publication-1"}` {
|
||||
t.Fatalf("metadata = %q", r.FormValue("metadata"))
|
||||
}
|
||||
if r.FormValue("channel") != "governance" {
|
||||
t.Fatalf("channel = %q", r.FormValue("channel"))
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"success":true,"data":{"id":"knowledge-1","knowledge_base_id":"kb-1","parse_status":"pending"}}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
client, err := weknora.NewClient(weknora.ClientConfig{BaseURL: server.URL, APIKey: "test-api-key", Timeout: time.Second})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
now := time.Date(2026, time.September, 4, 3, 0, 0, 0, time.UTC)
|
||||
store := &uploadStoreStub{candidate: postgresstore.PublicationForUpload{
|
||||
ID: "publication-1", SourceItemVersionID: "version-1", TestKBID: "kb-1",
|
||||
IngestState: governance.IngestPending, ProcessingState: governance.ProcessingPending,
|
||||
ContentObjectID: "content-1", SHA256: sha, SizeBytes: int64(len(body)),
|
||||
ValidatedFormat: "md", LocalSpoolRef: "objects/artifact.md", Title: "Safe title",
|
||||
}}
|
||||
handler, err := NewUploadHandler(UploadHandlerConfig{
|
||||
Store: store, Client: client, SpoolRoot: root, MaxUploadBytes: 50 * 1024 * 1024,
|
||||
FirstPollDelay: 10 * time.Second, Now: func() time.Time { return now },
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
job := jobs.Job{ID: "job-1", Type: "test.upload", Payload: []byte(`{"publication_id":"publication-1"}`)}
|
||||
if err := handler.Handle(context.Background(), job); err != nil {
|
||||
t.Fatalf("Handle() error = %v", err)
|
||||
}
|
||||
if !store.markedUploading || store.recordedKnowledgeID != "knowledge-1" || !store.nextPollAt.Equal(now.Add(10*time.Second)) {
|
||||
t.Fatalf("store = %#v", store)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadHandlerRejectsJobSnapshotWithMismatchedTarget(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
now := time.Date(2026, time.September, 4, 3, 0, 0, 0, time.UTC)
|
||||
store := &uploadStoreStub{candidate: postgresstore.PublicationForUpload{
|
||||
ID: "publication-1", SourceItemVersionID: "version-1", TestKBID: "kb-1", TargetScope: governance.VisibilityInternal,
|
||||
IngestState: governance.IngestPending, ProcessingState: governance.ProcessingPending,
|
||||
}}
|
||||
client, err := weknora.NewClient(weknora.ClientConfig{BaseURL: "http://127.0.0.1:1", APIKey: "***", Timeout: time.Second})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
handler, err := NewUploadHandler(UploadHandlerConfig{Store: store, Client: client, SpoolRoot: root, MaxUploadBytes: 1 << 20, FirstPollDelay: time.Second, Now: func() time.Time { return now }})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
job := jobs.Job{ID: "job-1", Type: "test.upload", Payload: []byte(`{"publication_id":"publication-1","target_scope":"external","target_kb_id":"kb-1"}`)}
|
||||
if err := handler.Handle(context.Background(), job); !errors.Is(err, ErrUploadStateConflict) {
|
||||
t.Fatalf("scope mismatch error = %v", err)
|
||||
}
|
||||
job = jobs.Job{ID: "job-2", Type: "test.upload", Payload: []byte(`{"publication_id":"publication-1","target_scope":"internal","target_kb_id":"kb-other"}`)}
|
||||
if err := handler.Handle(context.Background(), job); !errors.Is(err, ErrUploadStateConflict) {
|
||||
t.Fatalf("kb mismatch error = %v", err)
|
||||
}
|
||||
if store.markedUploading {
|
||||
t.Fatal("mismatched job must not mark uploading")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadHandlerRecoversUniqueRemoteKnowledgeAfterResponseLoss(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
if err := os.Mkdir(filepath.Join(root, "objects"), 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body := []byte("safe ambiguous upload\n")
|
||||
if err := os.WriteFile(filepath.Join(root, "objects", "artifact.md"), body, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
digest := sha256.Sum256(body)
|
||||
sha := hex.EncodeToString(digest[:])
|
||||
var uploads atomic.Int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch {
|
||||
case r.Method == http.MethodPost:
|
||||
uploads.Add(1)
|
||||
if err := r.ParseMultipartForm(1 << 20); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
file, _, err := r.FormFile("file")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, _ = io.Copy(io.Discard, file)
|
||||
_ = file.Close()
|
||||
connection, _, err := w.(http.Hijacker).Hijack()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = connection.Close()
|
||||
case r.Method == http.MethodGet && r.URL.Path == "/api/v1/knowledge-bases/kb-1/knowledge":
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"success":true,"data":[{"id":"knowledge-recovered","knowledge_base_id":"kb-1","metadata":{"publication_id":"publication-1","content_sha256":"` + sha + `"}}],"total":1,"page":1,"page_size":100}`))
|
||||
case r.Method == http.MethodGet && r.URL.Path == "/api/v1/knowledge/knowledge-recovered":
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"success":true,"data":{"id":"knowledge-recovered","knowledge_base_id":"kb-1","metadata":{"publication_id":"publication-1","content_sha256":"` + sha + `"}}}`))
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
client, err := weknora.NewClient(weknora.ClientConfig{BaseURL: server.URL, APIKey: "test-api-key", Timeout: time.Second})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Date(2026, time.September, 4, 3, 10, 0, 0, time.UTC)
|
||||
store := &uploadStoreStub{candidate: postgresstore.PublicationForUpload{
|
||||
ID: "publication-1", TestKBID: "kb-1", IngestState: governance.IngestPending,
|
||||
ProcessingState: governance.ProcessingPending, SHA256: sha, SizeBytes: int64(len(body)),
|
||||
ValidatedFormat: "md", LocalSpoolRef: "objects/artifact.md", Title: "Safe title",
|
||||
}}
|
||||
handler, err := NewUploadHandler(UploadHandlerConfig{
|
||||
Store: store, Client: client, SpoolRoot: root, MaxUploadBytes: 50 * 1024 * 1024,
|
||||
FirstPollDelay: time.Second, Now: func() time.Time { return now },
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := handler.Handle(context.Background(), jobs.Job{Payload: []byte(`{"publication_id":"publication-1"}`)}); err != nil {
|
||||
t.Fatalf("Handle() error = %v", err)
|
||||
}
|
||||
if uploads.Load() != 1 || store.recordedKnowledgeID != "knowledge-recovered" {
|
||||
t.Fatalf("uploads=%d knowledge=%q", uploads.Load(), store.recordedKnowledgeID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadHandlerRecoversUploadingStateBeforeAnotherPost(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
if err := os.Mkdir(filepath.Join(root, "objects"), 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body := []byte("safe retry recovery\n")
|
||||
if err := os.WriteFile(filepath.Join(root, "objects", "artifact.md"), body, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
digest := sha256.Sum256(body)
|
||||
sha := hex.EncodeToString(digest[:])
|
||||
var uploads atomic.Int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch {
|
||||
case r.Method == http.MethodPost:
|
||||
uploads.Add(1)
|
||||
http.Error(w, "must not upload", http.StatusInternalServerError)
|
||||
case r.Method == http.MethodGet && r.URL.Path == "/api/v1/knowledge-bases/kb-1/knowledge":
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"success":true,"data":[{"id":"knowledge-existing","knowledge_base_id":"kb-1","metadata":{"publication_id":"publication-1","content_sha256":"` + sha + `"}}],"total":1,"page":1,"page_size":100}`))
|
||||
case r.Method == http.MethodGet && r.URL.Path == "/api/v1/knowledge/knowledge-existing":
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"success":true,"data":{"id":"knowledge-existing","knowledge_base_id":"kb-1","metadata":{"publication_id":"publication-1","content_sha256":"` + sha + `"}}}`))
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
client, err := weknora.NewClient(weknora.ClientConfig{BaseURL: server.URL, APIKey: "test-api-key", Timeout: time.Second})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Date(2026, time.September, 4, 3, 20, 0, 0, time.UTC)
|
||||
store := &uploadStoreStub{candidate: postgresstore.PublicationForUpload{
|
||||
ID: "publication-1", TestKBID: "kb-1", IngestState: governance.IngestUploading,
|
||||
ProcessingState: governance.ProcessingPending, SHA256: sha, SizeBytes: int64(len(body)),
|
||||
ValidatedFormat: "md", LocalSpoolRef: "objects/artifact.md", Title: "Safe title",
|
||||
}}
|
||||
handler, err := NewUploadHandler(UploadHandlerConfig{
|
||||
Store: store, Client: client, SpoolRoot: root, MaxUploadBytes: 50 * 1024 * 1024,
|
||||
FirstPollDelay: time.Second, Now: func() time.Time { return now },
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := handler.Handle(context.Background(), jobs.Job{Payload: []byte(`{"publication_id":"publication-1"}`)}); err != nil {
|
||||
t.Fatalf("Handle() error = %v", err)
|
||||
}
|
||||
if uploads.Load() != 0 || store.recordedKnowledgeID != "knowledge-existing" {
|
||||
t.Fatalf("uploads=%d knowledge=%q", uploads.Load(), store.recordedKnowledgeID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadHandlerFailsClosedWhenAmbiguousRecoveryHasMultipleMatches(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
if err := os.Mkdir(filepath.Join(root, "objects"), 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body := []byte("safe ambiguous duplicate\n")
|
||||
if err := os.WriteFile(filepath.Join(root, "objects", "artifact.md"), body, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
digest := sha256.Sum256(body)
|
||||
sha := hex.EncodeToString(digest[:])
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch {
|
||||
case r.Method == http.MethodPost:
|
||||
_, _ = io.Copy(io.Discard, r.Body)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusConflict)
|
||||
_, _ = w.Write([]byte(`{"success":false,"data":{}}`))
|
||||
case r.Method == http.MethodGet && r.URL.Path == "/api/v1/knowledge-bases/kb-1/knowledge":
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"success":true,"data":[` +
|
||||
`{"id":"knowledge-a","knowledge_base_id":"kb-1","metadata":{"publication_id":"publication-1","content_sha256":"` + sha + `"}},` +
|
||||
`{"id":"knowledge-b","knowledge_base_id":"kb-1","metadata":{"publication_id":"publication-1","content_sha256":"` + sha + `"}}` +
|
||||
`],"total":2,"page":1,"page_size":100}`))
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
client, err := weknora.NewClient(weknora.ClientConfig{BaseURL: server.URL, APIKey: "test-api-key", Timeout: time.Second})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Date(2026, time.September, 4, 3, 30, 0, 0, time.UTC)
|
||||
store := &uploadStoreStub{candidate: postgresstore.PublicationForUpload{
|
||||
ID: "publication-1", TestKBID: "kb-1", IngestState: governance.IngestPending,
|
||||
ProcessingState: governance.ProcessingPending, SHA256: sha, SizeBytes: int64(len(body)),
|
||||
ValidatedFormat: "md", LocalSpoolRef: "objects/artifact.md", Title: "Safe title",
|
||||
}}
|
||||
handler, err := NewUploadHandler(UploadHandlerConfig{
|
||||
Store: store, Client: client, SpoolRoot: root, MaxUploadBytes: 50 * 1024 * 1024,
|
||||
FirstPollDelay: time.Second, Now: func() time.Time { return now },
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := handler.Handle(context.Background(), jobs.Job{Payload: []byte(`{"publication_id":"publication-1"}`)}); err != nil {
|
||||
t.Fatalf("Handle() error = %v", err)
|
||||
}
|
||||
if store.uploadFailureCode != "upload_identity_ambiguous" || store.recordedKnowledgeID != "" {
|
||||
t.Fatalf("failure=%q knowledge=%q", store.uploadFailureCode, store.recordedKnowledgeID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadHandlerRecordsInvalidArtifactForManualRetry(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
store := &uploadStoreStub{candidate: postgresstore.PublicationForUpload{
|
||||
ID: "publication-1", TestKBID: "kb-1", IngestState: governance.IngestPending,
|
||||
ProcessingState: governance.ProcessingPending, SHA256: strings.Repeat("a", 64), SizeBytes: 128,
|
||||
ValidatedFormat: "md", LocalSpoolRef: "missing/artifact.md", Title: "Safe title",
|
||||
}}
|
||||
handler, err := NewUploadHandler(UploadHandlerConfig{
|
||||
Store: store, Client: uploadClientStub{}, SpoolRoot: root, MaxUploadBytes: 1024,
|
||||
FirstPollDelay: time.Second,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := handler.Handle(context.Background(), jobs.Job{Payload: []byte(`{"publication_id":"publication-1"}`)}); err != nil {
|
||||
t.Fatalf("Handle() error = %v, want terminal failure recorded", err)
|
||||
}
|
||||
if store.uploadFailureCode != "upload_artifact_invalid" || store.markedUploading {
|
||||
t.Fatalf("failure=%q markedUploading=%v", store.uploadFailureCode, store.markedUploading)
|
||||
}
|
||||
}
|
||||
|
||||
type uploadStoreStub struct {
|
||||
candidate postgresstore.PublicationForUpload
|
||||
markedUploading bool
|
||||
recordedKnowledgeID string
|
||||
nextPollAt time.Time
|
||||
uploadFailureCode string
|
||||
}
|
||||
|
||||
func (store *uploadStoreStub) LoadPublicationForUpload(context.Context, string) (postgresstore.PublicationForUpload, error) {
|
||||
return store.candidate, nil
|
||||
}
|
||||
|
||||
func (store *uploadStoreStub) MarkUploading(context.Context, string, time.Time) error {
|
||||
store.markedUploading = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func (store *uploadStoreStub) RecordUploadAndSchedulePoll(_ context.Context, _ string, knowledgeID string, _ time.Time, nextAt time.Time) (int, error) {
|
||||
store.recordedKnowledgeID = knowledgeID
|
||||
store.nextPollAt = nextAt
|
||||
return 1, nil
|
||||
}
|
||||
|
||||
func (store *uploadStoreStub) RecordUploadFailure(_ context.Context, _ string, code string, _ time.Time) error {
|
||||
store.uploadFailureCode = code
|
||||
return nil
|
||||
}
|
||||
|
||||
type uploadClientStub struct{}
|
||||
|
||||
func (uploadClientStub) UploadFile(context.Context, weknora.UploadRequest) (weknora.Knowledge, error) {
|
||||
return weknora.Knowledge{}, errors.New("unexpected upload")
|
||||
}
|
||||
|
||||
func (uploadClientStub) ListKnowledge(context.Context, string, int, int) ([]weknora.Knowledge, int64, error) {
|
||||
return nil, 0, errors.New("unexpected list")
|
||||
}
|
||||
|
||||
func (uploadClientStub) GetKnowledge(context.Context, string) (weknora.Knowledge, error) {
|
||||
return weknora.Knowledge{}, errors.New("unexpected get")
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
package testweknora
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
"knowledge-governance/internal/jobs"
|
||||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||||
"knowledge-governance/internal/platform/weknora"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrInvalidValidateJobPayload = errors.New("test_validate_invalid_payload")
|
||||
ErrValidateStateConflict = errors.New("test_validate_state_conflict")
|
||||
ErrValidateRecordFailed = errors.New("test_validate_record_failed")
|
||||
)
|
||||
|
||||
type ValidateStore interface {
|
||||
LoadPublicationForValidation(context.Context, string) (postgresstore.PublicationForValidation, error)
|
||||
AppendAcceptanceResult(context.Context, governance.AcceptanceResult) error
|
||||
}
|
||||
|
||||
type ValidateClient interface {
|
||||
GetKnowledge(context.Context, string) (weknora.Knowledge, error)
|
||||
ListChunks(context.Context, string, int, int, []string) ([]weknora.Chunk, int64, error)
|
||||
}
|
||||
|
||||
type ValidateHandlerConfig struct {
|
||||
Store ValidateStore
|
||||
Client ValidateClient
|
||||
RunID string
|
||||
PolicyVersion string
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
type ValidateHandler struct {
|
||||
store ValidateStore
|
||||
client ValidateClient
|
||||
runID string
|
||||
policyVersion string
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
type validatePayload struct {
|
||||
PublicationID string `json:"publication_id"`
|
||||
}
|
||||
|
||||
func NewValidateHandler(config ValidateHandlerConfig) (*ValidateHandler, error) {
|
||||
if config.Store == nil || config.Client == nil || config.PolicyVersion == "" {
|
||||
return nil, errors.New("invalid test validate handler configuration")
|
||||
}
|
||||
if config.RunID == "" {
|
||||
config.RunID = "default"
|
||||
}
|
||||
if config.Now == nil {
|
||||
config.Now = time.Now
|
||||
}
|
||||
return &ValidateHandler{
|
||||
store: config.Store, client: config.Client, runID: config.RunID,
|
||||
policyVersion: config.PolicyVersion, now: config.Now,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (handler *ValidateHandler) Handle(ctx context.Context, job jobs.Job) error {
|
||||
var payload validatePayload
|
||||
if err := decodeStrictJSON(job.Payload, &payload); err != nil || payload.PublicationID == "" {
|
||||
return ErrInvalidValidateJobPayload
|
||||
}
|
||||
publication, err := handler.store.LoadPublicationForValidation(ctx, payload.PublicationID)
|
||||
if err != nil {
|
||||
return ErrValidateStateConflict
|
||||
}
|
||||
now := handler.now()
|
||||
runID := handler.runID
|
||||
if job.ID != "" {
|
||||
runID = job.ID
|
||||
}
|
||||
|
||||
knowledge, knowledgeErr := handler.client.GetKnowledge(ctx, publication.KnowledgeID)
|
||||
knowledgeState := governance.AcceptancePassed
|
||||
knowledgeEvidence := json.RawMessage(`{"matched":true}`)
|
||||
if knowledgeErr != nil || knowledge.ID != publication.KnowledgeID || knowledge.KnowledgeBaseID != publication.TestKBID {
|
||||
knowledgeState = governance.AcceptanceFailed
|
||||
knowledgeEvidence = json.RawMessage(`{"matched":false}`)
|
||||
}
|
||||
if err := handler.append(ctx, publication.ID, runID, "knowledge_exists", knowledgeState, knowledgeEvidence, now); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, chunkTotal, chunksErr := handler.client.ListChunks(ctx, publication.KnowledgeID, 1, 1, nil)
|
||||
chunksState := governance.AcceptancePassed
|
||||
if chunksErr != nil || chunkTotal <= 0 {
|
||||
chunksState = governance.AcceptanceFailed
|
||||
}
|
||||
if err := handler.append(ctx, publication.ID, runID, "chunks_present", chunksState, mustJSON(map[string]any{"chunk_total": chunkTotal}), now); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := handler.append(ctx, publication.ID, runID, "retrieval_hit", governance.AcceptanceManualRequired, json.RawMessage(`{"reason":"thresholds_unresolved"}`), now); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (handler *ValidateHandler) append(ctx context.Context, publicationID, runID, checkType string, state governance.AcceptanceState, evidence json.RawMessage, observedAt time.Time) error {
|
||||
result := governance.AcceptanceResult{
|
||||
ID: acceptanceResultID(publicationID, runID, checkType),
|
||||
PublicationID: publicationID,
|
||||
RunID: runID,
|
||||
CheckType: checkType,
|
||||
State: state,
|
||||
PolicyVersion: handler.policyVersion,
|
||||
EvidenceSummary: evidence,
|
||||
ObservedAt: observedAt,
|
||||
}
|
||||
if err := handler.store.AppendAcceptanceResult(ctx, result); err != nil {
|
||||
return ErrValidateRecordFailed
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func acceptanceResultID(publicationID, runID, checkType string) string {
|
||||
digest := sha256.New()
|
||||
_, _ = digest.Write([]byte(publicationID))
|
||||
_, _ = digest.Write([]byte{0})
|
||||
_, _ = digest.Write([]byte(runID))
|
||||
_, _ = digest.Write([]byte{0})
|
||||
_, _ = digest.Write([]byte(checkType))
|
||||
return "acc-" + hex.EncodeToString(digest.Sum(nil))[:32]
|
||||
}
|
||||
|
||||
func mustJSON(value any) json.RawMessage {
|
||||
data, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
return json.RawMessage(`{}`)
|
||||
}
|
||||
return data
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
package testweknora
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
"knowledge-governance/internal/jobs"
|
||||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||||
"knowledge-governance/internal/platform/weknora"
|
||||
)
|
||||
|
||||
func TestValidateHandlerAppendsDeterministicChecksAndManualRetrieval(t *testing.T) {
|
||||
client := &validateClientStub{
|
||||
knowledge: weknora.Knowledge{ID: "knowledge-1", KnowledgeBaseID: "kb-1"},
|
||||
chunkTotal: 3,
|
||||
}
|
||||
store := &validateStoreStub{publication: postgresstore.PublicationForValidation{
|
||||
ID: "publication-1", TestKBID: "kb-1", KnowledgeID: "knowledge-1",
|
||||
}}
|
||||
now := time.Date(2026, time.September, 4, 4, 10, 0, 0, time.UTC)
|
||||
handler, err := NewValidateHandler(ValidateHandlerConfig{
|
||||
Store: store, Client: client, RunID: "run-1", PolicyVersion: "structural-v1",
|
||||
Now: func() time.Time { return now },
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := handler.Handle(context.Background(), jobs.Job{Payload: []byte(`{"publication_id":"publication-1"}`)}); err != nil {
|
||||
t.Fatalf("Handle() error = %v", err)
|
||||
}
|
||||
if len(store.appended) != 3 {
|
||||
t.Fatalf("appended = %d, want 3: %#v", len(store.appended), store.appended)
|
||||
}
|
||||
byType := map[string]governance.AcceptanceResult{}
|
||||
for _, result := range store.appended {
|
||||
byType[result.CheckType] = result
|
||||
}
|
||||
if byType["knowledge_exists"].State != governance.AcceptancePassed {
|
||||
t.Fatalf("knowledge_exists = %#v", byType["knowledge_exists"])
|
||||
}
|
||||
if byType["chunks_present"].State != governance.AcceptancePassed {
|
||||
t.Fatalf("chunks_present = %#v", byType["chunks_present"])
|
||||
}
|
||||
if byType["retrieval_hit"].State != governance.AcceptanceManualRequired {
|
||||
t.Fatalf("retrieval_hit = %#v", byType["retrieval_hit"])
|
||||
}
|
||||
for _, result := range store.appended {
|
||||
if result.PublicationID != "publication-1" || result.RunID != "run-1" || result.ObservedAt != now {
|
||||
t.Fatalf("acceptance metadata = %#v", result)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateHandlerFailsChecksWithoutAborting(t *testing.T) {
|
||||
client := &validateClientStub{
|
||||
knowledge: weknora.Knowledge{ID: "knowledge-mismatch", KnowledgeBaseID: "kb-other"},
|
||||
chunkTotal: 0,
|
||||
}
|
||||
store := &validateStoreStub{publication: postgresstore.PublicationForValidation{
|
||||
ID: "publication-1", TestKBID: "kb-1", KnowledgeID: "knowledge-1",
|
||||
}}
|
||||
handler, err := NewValidateHandler(ValidateHandlerConfig{
|
||||
Store: store, Client: client, RunID: "run-2", PolicyVersion: "structural-v1",
|
||||
Now: func() time.Time { return time.Date(2026, time.September, 4, 4, 11, 0, 0, time.UTC) },
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := handler.Handle(context.Background(), jobs.Job{Payload: []byte(`{"publication_id":"publication-1"}`)}); err != nil {
|
||||
t.Fatalf("Handle() error = %v", err)
|
||||
}
|
||||
if len(store.appended) != 3 {
|
||||
t.Fatalf("appended = %d, want 3", len(store.appended))
|
||||
}
|
||||
byType := map[string]governance.AcceptanceResult{}
|
||||
for _, result := range store.appended {
|
||||
byType[result.CheckType] = result
|
||||
}
|
||||
if byType["knowledge_exists"].State != governance.AcceptanceFailed || byType["chunks_present"].State != governance.AcceptanceFailed {
|
||||
t.Fatalf("failed checks = %#v", store.appended)
|
||||
}
|
||||
if byType["retrieval_hit"].State != governance.AcceptanceManualRequired {
|
||||
t.Fatalf("retrieval still manual = %#v", byType["retrieval_hit"])
|
||||
}
|
||||
}
|
||||
|
||||
type validateStoreStub struct {
|
||||
publication postgresstore.PublicationForValidation
|
||||
appended []governance.AcceptanceResult
|
||||
}
|
||||
|
||||
func (store *validateStoreStub) LoadPublicationForValidation(context.Context, string) (postgresstore.PublicationForValidation, error) {
|
||||
return store.publication, nil
|
||||
}
|
||||
|
||||
func (store *validateStoreStub) AppendAcceptanceResult(_ context.Context, result governance.AcceptanceResult) error {
|
||||
store.appended = append(store.appended, result)
|
||||
return nil
|
||||
}
|
||||
|
||||
type validateClientStub struct {
|
||||
knowledge weknora.Knowledge
|
||||
chunkTotal int64
|
||||
}
|
||||
|
||||
func (client *validateClientStub) GetKnowledge(context.Context, string) (weknora.Knowledge, error) {
|
||||
return client.knowledge, nil
|
||||
}
|
||||
|
||||
func (client *validateClientStub) ListChunks(context.Context, string, int, int, []string) ([]weknora.Chunk, int64, error) {
|
||||
chunks := make([]weknora.Chunk, 0, client.chunkTotal)
|
||||
for i := int64(0); i < client.chunkTotal; i++ {
|
||||
chunks = append(chunks, weknora.Chunk{ID: "chunk", KnowledgeID: "knowledge-1"})
|
||||
}
|
||||
return chunks, client.chunkTotal, nil
|
||||
}
|
||||
Reference in New Issue
Block a user