feat: 知识治理平台(含已验证发布/回滚契约)
全栈知识治理平台首次入库(P1-P10 + 发布链路模拟验证闭环): - 治理核心:源目录/变更集/人工审核/发布批次/发布执行(fail-closed 闸口) - 测试通道:WeKnora 测试 KB 上传/轮询/验收;poll 完成时写入 parser_generation/index_generation,修复 manual/seed 路径发布闸口永久拦截 - 发布适配器 v071(本地 source=测试角色 / 云端 target=生产角色 模拟验证通过): * ES 导出/导入 mapping 自适应(.keyword 子字段、可选 tenant_id) * OSS locator 支持嵌套 scheme;chunks.seq_id 由目标序列分配 * bundle 归档同身份重试原子替换;verifier 按 bundle chunk_type 计数 * 回滚 job payload 支持 prior_execution_id(strict 解码) - 三通道发布+回滚演练实测:publish 4/4 verified,检索命中, rollback 4/4 rolled_back,目标端 PG is_enabled=false、ES is_enabled=false - 前端 A·Data-console 风格全站统一,120 用例通过 - 门禁:go vet / go test 全包通过,gofmt 干净
This commit is contained in:
@@ -0,0 +1,196 @@
|
||||
package testweknora
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"knowledge-governance/internal/platform/weknora"
|
||||
)
|
||||
|
||||
type LegacyCandidate struct {
|
||||
ID string
|
||||
KnowledgeID string
|
||||
TestKBID string
|
||||
}
|
||||
|
||||
type LegacyVerifyClient interface {
|
||||
GetKnowledge(context.Context, string) (weknora.Knowledge, error)
|
||||
ListChunks(context.Context, string, int, int, []string) ([]weknora.Chunk, int64, error)
|
||||
}
|
||||
|
||||
type LegacyVerificationReport struct {
|
||||
Total int
|
||||
Existing int
|
||||
Missing int
|
||||
WrongKB int
|
||||
Pending int
|
||||
Processing int
|
||||
Finalizing int
|
||||
Completed int
|
||||
Failed int
|
||||
ChunkChecked int
|
||||
ChunkEmpty int
|
||||
HTTPErrors int
|
||||
}
|
||||
|
||||
func VerifyLegacy(ctx context.Context, client LegacyVerifyClient, candidates []LegacyCandidate, chunkSampleEvery int) (LegacyVerificationReport, error) {
|
||||
report := LegacyVerificationReport{Total: len(candidates)}
|
||||
completedSeen := 0
|
||||
for _, candidate := range candidates {
|
||||
knowledge, err := client.GetKnowledge(ctx, candidate.KnowledgeID)
|
||||
if err != nil {
|
||||
var httpErr *weknora.HTTPError
|
||||
if errors.As(err, &httpErr) && httpErr.StatusCode == 404 {
|
||||
report.Missing++
|
||||
} else {
|
||||
report.HTTPErrors++
|
||||
}
|
||||
continue
|
||||
}
|
||||
if knowledge.KnowledgeBaseID != candidate.TestKBID {
|
||||
report.WrongKB++
|
||||
continue
|
||||
}
|
||||
report.Existing++
|
||||
switch knowledge.ParseStatus {
|
||||
case "pending":
|
||||
report.Pending++
|
||||
case "processing":
|
||||
report.Processing++
|
||||
case "finalizing":
|
||||
report.Finalizing++
|
||||
case "completed":
|
||||
report.Completed++
|
||||
completedSeen++
|
||||
if chunkSampleEvery > 0 && completedSeen%chunkSampleEvery == 0 {
|
||||
if err := verifyLegacyChunk(ctx, client, candidate.KnowledgeID, &report); err != nil {
|
||||
report.HTTPErrors++
|
||||
}
|
||||
}
|
||||
case "failed":
|
||||
report.Failed++
|
||||
}
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func VerifyLegacyConcurrent(ctx context.Context, client LegacyVerifyClient, candidates []LegacyCandidate, chunkSampleEvery, concurrency int) (LegacyVerificationReport, error) {
|
||||
if client == nil || concurrency < 1 || concurrency > 32 || chunkSampleEvery < 0 {
|
||||
return LegacyVerificationReport{}, errors.New("invalid legacy verification configuration")
|
||||
}
|
||||
jobs := make(chan LegacyCandidate)
|
||||
results := make(chan LegacyVerificationReport)
|
||||
var workers sync.WaitGroup
|
||||
var completedSeen atomic.Int64
|
||||
workerCount := min(concurrency, len(candidates))
|
||||
workers.Add(workerCount)
|
||||
for range workerCount {
|
||||
go func() {
|
||||
defer workers.Done()
|
||||
for candidate := range jobs {
|
||||
partial, _ := VerifyLegacy(ctx, client, []LegacyCandidate{candidate}, 0)
|
||||
if partial.Completed == 1 {
|
||||
completed := completedSeen.Add(1)
|
||||
if chunkSampleEvery > 0 && completed%int64(chunkSampleEvery) == 0 {
|
||||
if err := verifyLegacyChunk(ctx, client, candidate.KnowledgeID, &partial); err != nil {
|
||||
partial.HTTPErrors++
|
||||
}
|
||||
}
|
||||
}
|
||||
results <- partial
|
||||
}
|
||||
}()
|
||||
}
|
||||
go func() {
|
||||
defer close(jobs)
|
||||
for _, candidate := range candidates {
|
||||
select {
|
||||
case jobs <- candidate:
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
go func() {
|
||||
workers.Wait()
|
||||
close(results)
|
||||
}()
|
||||
|
||||
report := LegacyVerificationReport{}
|
||||
for partial := range results {
|
||||
mergeLegacyVerificationReport(&report, partial)
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return report, err
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func mergeLegacyVerificationReport(report *LegacyVerificationReport, partial LegacyVerificationReport) {
|
||||
report.Total += partial.Total
|
||||
report.Existing += partial.Existing
|
||||
report.Missing += partial.Missing
|
||||
report.WrongKB += partial.WrongKB
|
||||
report.Pending += partial.Pending
|
||||
report.Processing += partial.Processing
|
||||
report.Finalizing += partial.Finalizing
|
||||
report.Completed += partial.Completed
|
||||
report.Failed += partial.Failed
|
||||
report.ChunkChecked += partial.ChunkChecked
|
||||
report.ChunkEmpty += partial.ChunkEmpty
|
||||
report.HTTPErrors += partial.HTTPErrors
|
||||
}
|
||||
|
||||
func verifyLegacyChunk(ctx context.Context, client LegacyVerifyClient, knowledgeID string, report *LegacyVerificationReport) error {
|
||||
_, total, err := client.ListChunks(ctx, knowledgeID, 1, 1, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
report.ChunkChecked++
|
||||
if total == 0 {
|
||||
report.ChunkEmpty++
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func FormatDryRunReport(candidates []LegacyCandidate, concurrency int) string {
|
||||
kbCounts := make(map[string]int)
|
||||
for _, candidate := range candidates {
|
||||
kbCounts[candidate.TestKBID]++
|
||||
}
|
||||
kbs := make([]string, 0, len(kbCounts))
|
||||
for kb := range kbCounts {
|
||||
kbs = append(kbs, kb)
|
||||
}
|
||||
sort.Strings(kbs)
|
||||
var builder strings.Builder
|
||||
fmt.Fprintf(&builder, "total=%d\n", len(candidates))
|
||||
fmt.Fprintf(&builder, "concurrency=%d\n", concurrency)
|
||||
for _, kb := range kbs {
|
||||
fmt.Fprintf(&builder, "knowledge_base=%s count=%d\n", kb, kbCounts[kb])
|
||||
}
|
||||
return builder.String()
|
||||
}
|
||||
|
||||
func FormatLegacyVerificationReport(report LegacyVerificationReport) string {
|
||||
return fmt.Sprintf(
|
||||
"total=%d\nexisting=%d\nmissing=%d\nwrong_kb=%d\npending=%d\nprocessing=%d\nfinalizing=%d\ncompleted=%d\nfailed=%d\nchunk_checked=%d\nchunk_empty=%d\nhttp_errors=%d\n",
|
||||
report.Total,
|
||||
report.Existing,
|
||||
report.Missing,
|
||||
report.WrongKB,
|
||||
report.Pending,
|
||||
report.Processing,
|
||||
report.Finalizing,
|
||||
report.Completed,
|
||||
report.Failed,
|
||||
report.ChunkChecked,
|
||||
report.ChunkEmpty,
|
||||
report.HTTPErrors,
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,192 @@
|
||||
package testweknora
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"knowledge-governance/internal/platform/weknora"
|
||||
)
|
||||
|
||||
func TestVerifyLegacyClassifiesWithoutLeaking(t *testing.T) {
|
||||
client := &legacyClientStub{getResults: map[string]legacyGetResult{
|
||||
"knowledge-completed-a": {knowledge: weknora.Knowledge{KnowledgeBaseID: "kb-1", ParseStatus: "completed"}},
|
||||
"knowledge-completed-b": {knowledge: weknora.Knowledge{KnowledgeBaseID: "kb-1", ParseStatus: "completed"}},
|
||||
"knowledge-pending": {knowledge: weknora.Knowledge{KnowledgeBaseID: "kb-1", ParseStatus: "pending"}},
|
||||
"knowledge-processing": {knowledge: weknora.Knowledge{KnowledgeBaseID: "kb-1", ParseStatus: "processing"}},
|
||||
"knowledge-finalizing": {knowledge: weknora.Knowledge{KnowledgeBaseID: "kb-1", ParseStatus: "finalizing"}},
|
||||
"knowledge-failed": {knowledge: weknora.Knowledge{KnowledgeBaseID: "kb-1", ParseStatus: "failed"}},
|
||||
"knowledge-wrong-kb": {knowledge: weknora.Knowledge{KnowledgeBaseID: "kb-other", ParseStatus: "completed"}},
|
||||
"knowledge-missing": {err: &weknora.HTTPError{StatusCode: 404}},
|
||||
"knowledge-server-error": {err: &weknora.HTTPError{StatusCode: 500}},
|
||||
}, chunkTotal: map[string]int64{
|
||||
"knowledge-completed-a": 2,
|
||||
"knowledge-completed-b": 0,
|
||||
}}
|
||||
|
||||
candidates := []LegacyCandidate{
|
||||
{ID: "pub-1", KnowledgeID: "knowledge-completed-a", TestKBID: "kb-1"},
|
||||
{ID: "pub-2", KnowledgeID: "knowledge-completed-b", TestKBID: "kb-1"},
|
||||
{ID: "pub-3", KnowledgeID: "knowledge-pending", TestKBID: "kb-1"},
|
||||
{ID: "pub-4", KnowledgeID: "knowledge-processing", TestKBID: "kb-1"},
|
||||
{ID: "pub-5", KnowledgeID: "knowledge-finalizing", TestKBID: "kb-1"},
|
||||
{ID: "pub-6", KnowledgeID: "knowledge-failed", TestKBID: "kb-1"},
|
||||
{ID: "pub-7", KnowledgeID: "knowledge-wrong-kb", TestKBID: "kb-1"},
|
||||
{ID: "pub-8", KnowledgeID: "knowledge-missing", TestKBID: "kb-1"},
|
||||
{ID: "pub-9", KnowledgeID: "knowledge-server-error", TestKBID: "kb-1"},
|
||||
}
|
||||
|
||||
report, err := VerifyLegacy(context.Background(), client, candidates, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("VerifyLegacy() error = %v", err)
|
||||
}
|
||||
want := LegacyVerificationReport{
|
||||
Total: 9, Existing: 6, Missing: 1, WrongKB: 1,
|
||||
Pending: 1, Processing: 1, Finalizing: 1, Completed: 2, Failed: 1,
|
||||
ChunkChecked: 2, ChunkEmpty: 1, HTTPErrors: 1,
|
||||
}
|
||||
if report != want {
|
||||
t.Fatalf("report = %#v, want %#v", report, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerifyLegacyChunkSampleEverySkipsUnsampledCompleted(t *testing.T) {
|
||||
client := &legacyClientStub{getResults: map[string]legacyGetResult{
|
||||
"knowledge-a": {knowledge: weknora.Knowledge{KnowledgeBaseID: "kb-1", ParseStatus: "completed"}},
|
||||
"knowledge-b": {knowledge: weknora.Knowledge{KnowledgeBaseID: "kb-1", ParseStatus: "completed"}},
|
||||
}, chunkTotal: map[string]int64{
|
||||
"knowledge-b": 5,
|
||||
}}
|
||||
candidates := []LegacyCandidate{
|
||||
{ID: "pub-1", KnowledgeID: "knowledge-a", TestKBID: "kb-1"},
|
||||
{ID: "pub-2", KnowledgeID: "knowledge-b", TestKBID: "kb-1"},
|
||||
}
|
||||
report, err := VerifyLegacy(context.Background(), client, candidates, 2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if report.Completed != 2 || report.ChunkChecked != 1 {
|
||||
t.Fatalf("report = %#v", report)
|
||||
}
|
||||
if client.chunkQueries != 1 {
|
||||
t.Fatalf("chunk queries = %d, want 1", client.chunkQueries)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerifyLegacyConcurrentBoundsConcurrencyAndAggregates(t *testing.T) {
|
||||
client := &concurrentLegacyClient{}
|
||||
candidates := make([]LegacyCandidate, 12)
|
||||
for index := range candidates {
|
||||
candidates[index] = LegacyCandidate{ID: "publication", KnowledgeID: "knowledge", TestKBID: "kb-1"}
|
||||
}
|
||||
|
||||
report, err := VerifyLegacyConcurrent(context.Background(), client, candidates, 0, 3)
|
||||
if err != nil {
|
||||
t.Fatalf("VerifyLegacyConcurrent() error = %v", err)
|
||||
}
|
||||
if report.Total != 12 || report.Existing != 12 || report.Completed != 12 {
|
||||
t.Fatalf("report = %#v", report)
|
||||
}
|
||||
if maximum := client.maximum.Load(); maximum < 2 || maximum > 3 {
|
||||
t.Fatalf("maximum concurrency = %d, want 2..3", maximum)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerifyLegacyConcurrentReturnsImmediatelyForEmptyInput(t *testing.T) {
|
||||
report, err := VerifyLegacyConcurrent(context.Background(), &concurrentLegacyClient{}, nil, 10, 4)
|
||||
if err != nil {
|
||||
t.Fatalf("VerifyLegacyConcurrent() error = %v", err)
|
||||
}
|
||||
if report != (LegacyVerificationReport{}) {
|
||||
t.Fatalf("report = %#v", report)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFormatDryRunReportAggregatesWithoutKnowledgeIDs(t *testing.T) {
|
||||
candidates := []LegacyCandidate{
|
||||
{ID: "pub-1", KnowledgeID: "knowledge-1", TestKBID: "kb-1"},
|
||||
{ID: "pub-2", KnowledgeID: "knowledge-2", TestKBID: "kb-1"},
|
||||
{ID: "pub-3", KnowledgeID: "knowledge-3", TestKBID: "kb-2"},
|
||||
}
|
||||
report := FormatDryRunReport(candidates, 4)
|
||||
if !strings.Contains(report, "total=3") || !strings.Contains(report, "concurrency=4") ||
|
||||
!strings.Contains(report, "knowledge_base=kb-1 count=2") || !strings.Contains(report, "knowledge_base=kb-2 count=1") {
|
||||
t.Fatalf("report = %q", report)
|
||||
}
|
||||
if strings.Contains(report, "knowledge-1") || strings.Contains(report, "pub-1") {
|
||||
t.Fatalf("report leaked knowledge/publication ID: %q", report)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFormatLegacyVerificationReportContainsOnlyAggregateCounts(t *testing.T) {
|
||||
report := FormatLegacyVerificationReport(LegacyVerificationReport{
|
||||
Total: 530, Existing: 500, Missing: 20, WrongKB: 10,
|
||||
Completed: 490, Failed: 10, ChunkChecked: 20, ChunkEmpty: 2, HTTPErrors: 3,
|
||||
})
|
||||
for _, expected := range []string{"total=530", "existing=500", "missing=20", "wrong_kb=10", "completed=490", "failed=10", "chunk_checked=20", "chunk_empty=2", "http_errors=3"} {
|
||||
if !strings.Contains(report, expected) {
|
||||
t.Fatalf("report missing %q: %q", expected, report)
|
||||
}
|
||||
}
|
||||
for _, forbidden := range []string{"knowledge_id", "knowledge_base", "publication_id"} {
|
||||
if strings.Contains(report, forbidden) {
|
||||
t.Fatalf("report leaked identifier field %q: %q", forbidden, report)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type legacyClientStub struct {
|
||||
getResults map[string]legacyGetResult
|
||||
chunkTotal map[string]int64
|
||||
chunkQueries int
|
||||
}
|
||||
|
||||
type legacyGetResult struct {
|
||||
knowledge weknora.Knowledge
|
||||
err error
|
||||
}
|
||||
|
||||
type concurrentLegacyClient struct {
|
||||
current atomic.Int64
|
||||
maximum atomic.Int64
|
||||
}
|
||||
|
||||
func (client *concurrentLegacyClient) GetKnowledge(context.Context, string) (weknora.Knowledge, error) {
|
||||
current := client.current.Add(1)
|
||||
for {
|
||||
maximum := client.maximum.Load()
|
||||
if current <= maximum || client.maximum.CompareAndSwap(maximum, current) {
|
||||
break
|
||||
}
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
client.current.Add(-1)
|
||||
return weknora.Knowledge{KnowledgeBaseID: "kb-1", ParseStatus: "completed"}, nil
|
||||
}
|
||||
|
||||
func (*concurrentLegacyClient) ListChunks(context.Context, string, int, int, []string) ([]weknora.Chunk, int64, error) {
|
||||
return nil, 1, nil
|
||||
}
|
||||
|
||||
func (client *legacyClientStub) GetKnowledge(_ context.Context, knowledgeID string) (weknora.Knowledge, error) {
|
||||
result, ok := client.getResults[knowledgeID]
|
||||
if !ok {
|
||||
return weknora.Knowledge{}, &weknora.HTTPError{StatusCode: 404}
|
||||
}
|
||||
return result.knowledge, result.err
|
||||
}
|
||||
|
||||
func (client *legacyClientStub) ListChunks(_ context.Context, knowledgeID string, _ int, _ int, _ []string) ([]weknora.Chunk, int64, error) {
|
||||
client.chunkQueries++
|
||||
total, ok := client.chunkTotal[knowledgeID]
|
||||
if !ok {
|
||||
total = 0
|
||||
}
|
||||
chunks := make([]weknora.Chunk, 0, total)
|
||||
for i := int64(0); i < total; i++ {
|
||||
chunks = append(chunks, weknora.Chunk{ID: "chunk", KnowledgeID: knowledgeID})
|
||||
}
|
||||
return chunks, total, nil
|
||||
}
|
||||
@@ -0,0 +1,141 @@
|
||||
package testweknora
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
"knowledge-governance/internal/jobs"
|
||||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||||
"knowledge-governance/internal/platform/weknora"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrInvalidPollJobPayload = errors.New("test_poll_invalid_payload")
|
||||
ErrPollRemoteFailed = errors.New("test_poll_remote_failed")
|
||||
ErrPollRemoteIdentity = errors.New("test_poll_remote_identity_invalid")
|
||||
ErrPollStateConflict = errors.New("test_poll_state_conflict")
|
||||
)
|
||||
|
||||
type PollStore interface {
|
||||
LoadPublicationForPoll(context.Context, string) (postgresstore.PublicationForPoll, error)
|
||||
RecordRemoteStatusAndScheduleNextPoll(context.Context, string, int, governance.ProcessingState, *time.Time, time.Time, time.Time) (int, error)
|
||||
RecordCompletedAndScheduleValidation(context.Context, string, int, *time.Time, time.Time) error
|
||||
RecordProcessingFailure(context.Context, string, int, string, time.Time) error
|
||||
MarkStalled(context.Context, string, int, time.Time, time.Time) error
|
||||
}
|
||||
|
||||
type PollClient interface {
|
||||
GetKnowledge(context.Context, string) (weknora.Knowledge, error)
|
||||
}
|
||||
|
||||
type PollHandlerConfig struct {
|
||||
Store PollStore
|
||||
Client PollClient
|
||||
PollDelay time.Duration
|
||||
StallAfter time.Duration
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
type PollHandler struct {
|
||||
store PollStore
|
||||
client PollClient
|
||||
pollDelay time.Duration
|
||||
stallAfter time.Duration
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
type pollPayload struct {
|
||||
PublicationID string `json:"publication_id"`
|
||||
Sequence int `json:"sequence"`
|
||||
}
|
||||
|
||||
func NewPollHandler(config PollHandlerConfig) (*PollHandler, error) {
|
||||
if config.Store == nil || config.Client == nil || config.PollDelay <= 0 || config.StallAfter <= 0 {
|
||||
return nil, errors.New("invalid test poll handler configuration")
|
||||
}
|
||||
if config.Now == nil {
|
||||
config.Now = time.Now
|
||||
}
|
||||
return &PollHandler{
|
||||
store: config.Store, client: config.Client, pollDelay: config.PollDelay,
|
||||
stallAfter: config.StallAfter, now: config.Now,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (handler *PollHandler) Handle(ctx context.Context, job jobs.Job) error {
|
||||
var payload pollPayload
|
||||
if err := decodeStrictJSON(job.Payload, &payload); err != nil || payload.PublicationID == "" || payload.Sequence < 1 {
|
||||
return jobs.Permanent(jobs.ErrCodeHandlerFallback, ErrInvalidPollJobPayload)
|
||||
}
|
||||
publication, err := handler.store.LoadPublicationForPoll(ctx, payload.PublicationID)
|
||||
if err != nil {
|
||||
return ErrPollStateConflict
|
||||
}
|
||||
if publication.KnowledgeID == "" || publication.PollSequence != payload.Sequence {
|
||||
return ErrPollStateConflict
|
||||
}
|
||||
now := handler.now()
|
||||
knowledge, err := handler.client.GetKnowledge(ctx, publication.KnowledgeID)
|
||||
if err != nil {
|
||||
return ErrPollRemoteFailed
|
||||
}
|
||||
if knowledge.ID != publication.KnowledgeID || knowledge.KnowledgeBaseID != publication.TestKBID {
|
||||
return ErrPollRemoteIdentity
|
||||
}
|
||||
state := mapParseStatus(knowledge.ParseStatus)
|
||||
if state == governance.ProcessingCompleted {
|
||||
if err := handler.store.RecordCompletedAndScheduleValidation(ctx, publication.ID, publication.PollSequence, &knowledge.UpdatedAt, now); err != nil {
|
||||
return ErrPollStateConflict
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if state == governance.ProcessingFailed {
|
||||
if err := handler.store.RecordProcessingFailure(ctx, publication.ID, publication.PollSequence, "remote_parse_failed", now); err != nil {
|
||||
return ErrPollStateConflict
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if !state.Valid() {
|
||||
if err := handler.store.RecordProcessingFailure(ctx, publication.ID, publication.PollSequence, "remote_unknown_status", now); err != nil {
|
||||
return ErrPollStateConflict
|
||||
}
|
||||
return nil
|
||||
}
|
||||
cutoff := now.Add(-handler.stallAfter)
|
||||
lastProgress := publication.LastProgressAt
|
||||
if lastProgress == nil {
|
||||
lastProgress = publication.ProcessingStartedAt
|
||||
}
|
||||
if lastProgress == nil {
|
||||
lastProgress = publication.LastObservedAt
|
||||
}
|
||||
if lastProgress != nil && !lastProgress.After(cutoff) {
|
||||
if err := handler.store.MarkStalled(ctx, publication.ID, publication.PollSequence, cutoff, now); err != nil {
|
||||
return ErrPollStateConflict
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if _, err := handler.store.RecordRemoteStatusAndScheduleNextPoll(ctx, publication.ID, publication.PollSequence, state, &knowledge.UpdatedAt, now, now.Add(handler.pollDelay)); err != nil {
|
||||
return ErrPollStateConflict
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func mapParseStatus(status string) governance.ProcessingState {
|
||||
switch status {
|
||||
case "pending":
|
||||
return governance.ProcessingPending
|
||||
case "processing":
|
||||
return governance.ProcessingProcessing
|
||||
case "finalizing":
|
||||
return governance.ProcessingFinalizing
|
||||
case "completed":
|
||||
return governance.ProcessingCompleted
|
||||
case "failed":
|
||||
return governance.ProcessingFailed
|
||||
default:
|
||||
return governance.ProcessingState(status)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,188 @@
|
||||
package testweknora
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
"knowledge-governance/internal/jobs"
|
||||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||||
"knowledge-governance/internal/platform/weknora"
|
||||
)
|
||||
|
||||
func TestPollHandlerSchedulesNextSequenceForRemoteProgress(t *testing.T) {
|
||||
remoteUpdated := time.Date(2026, time.September, 4, 3, 39, 0, 0, time.UTC)
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodGet || r.URL.Path != "/api/v1/knowledge/knowledge-1" {
|
||||
t.Fatalf("request = %s %s", r.Method, r.URL.Path)
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"success":true,"data":{"id":"knowledge-1","knowledge_base_id":"kb-1","parse_status":"finalizing","updated_at":"2026-09-04T03:39:00Z"}}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
client, err := weknora.NewClient(weknora.ClientConfig{BaseURL: server.URL, APIKey: "test-api-key", Timeout: time.Second})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Date(2026, time.September, 4, 3, 40, 0, 0, time.UTC)
|
||||
lastProgress := now.Add(-time.Minute)
|
||||
store := &pollStoreStub{publication: postgresstore.PublicationForPoll{
|
||||
ID: "publication-1", TestKBID: "kb-1", KnowledgeID: "knowledge-1",
|
||||
ProcessingState: governance.ProcessingProcessing, PollSequence: 2,
|
||||
LastRemoteStatus: "processing", LastProgressAt: &lastProgress,
|
||||
}}
|
||||
handler, err := NewPollHandler(PollHandlerConfig{
|
||||
Store: store, Client: client, PollDelay: 15 * time.Second,
|
||||
StallAfter: 30 * time.Minute, Now: func() time.Time { return now },
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := handler.Handle(context.Background(), jobs.Job{Payload: []byte(`{"publication_id":"publication-1","sequence":2}`)}); err != nil {
|
||||
t.Fatalf("Handle() error = %v", err)
|
||||
}
|
||||
if store.recordedState != governance.ProcessingFinalizing || store.recordedRemoteUpdated == nil || !store.recordedRemoteUpdated.Equal(remoteUpdated) || !store.nextPollAt.Equal(now.Add(15*time.Second)) {
|
||||
t.Fatalf("store = %#v", store)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPollHandlerCompletesAndSchedulesValidationWithoutNextPoll(t *testing.T) {
|
||||
completedAt := time.Date(2026, time.September, 4, 3, 45, 0, 0, time.UTC)
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodGet || r.URL.Path != "/api/v1/knowledge/knowledge-1" {
|
||||
t.Fatalf("request = %s %s", r.Method, r.URL.Path)
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"success":true,"data":{"id":"knowledge-1","knowledge_base_id":"kb-1","parse_status":"completed","updated_at":"2026-09-04T03:45:00Z"}}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
client, err := weknora.NewClient(weknora.ClientConfig{BaseURL: server.URL, APIKey: "test-api-key", Timeout: time.Second})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Date(2026, time.September, 4, 3, 46, 0, 0, time.UTC)
|
||||
lastProgress := now.Add(-time.Minute)
|
||||
store := &pollStoreStub{publication: postgresstore.PublicationForPoll{
|
||||
ID: "publication-1", TestKBID: "kb-1", KnowledgeID: "knowledge-1",
|
||||
ProcessingState: governance.ProcessingProcessing, PollSequence: 3,
|
||||
LastRemoteStatus: "processing", LastProgressAt: &lastProgress,
|
||||
}}
|
||||
handler, err := NewPollHandler(PollHandlerConfig{
|
||||
Store: store, Client: client, PollDelay: 15 * time.Second,
|
||||
StallAfter: 30 * time.Minute, Now: func() time.Time { return now },
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := handler.Handle(context.Background(), jobs.Job{Payload: []byte(`{"publication_id":"publication-1","sequence":3}`)}); err != nil {
|
||||
t.Fatalf("Handle() error = %v", err)
|
||||
}
|
||||
if !store.completedValidated || store.recordedState != "" || store.recordedRemoteUpdated == nil || !store.recordedRemoteUpdated.Equal(completedAt) {
|
||||
t.Fatalf("store = %#v", store)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPollHandlerMarksStalledWhenNoProgressExceedsThreshold(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodGet || r.URL.Path != "/api/v1/knowledge/knowledge-1" {
|
||||
t.Fatalf("request = %s %s", r.Method, r.URL.Path)
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"success":true,"data":{"id":"knowledge-1","knowledge_base_id":"kb-1","parse_status":"finalizing","updated_at":"2026-09-04T03:10:00Z"}}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
client, err := weknora.NewClient(weknora.ClientConfig{BaseURL: server.URL, APIKey: "test-api-key", Timeout: time.Second})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Date(2026, time.September, 4, 3, 50, 0, 0, time.UTC)
|
||||
stalledSince := now.Add(-45 * time.Minute)
|
||||
store := &pollStoreStub{publication: postgresstore.PublicationForPoll{
|
||||
ID: "publication-1", TestKBID: "kb-1", KnowledgeID: "knowledge-1",
|
||||
ProcessingState: governance.ProcessingProcessing, PollSequence: 4,
|
||||
LastRemoteStatus: "processing", LastProgressAt: &stalledSince,
|
||||
}}
|
||||
handler, err := NewPollHandler(PollHandlerConfig{
|
||||
Store: store, Client: client, PollDelay: 15 * time.Second,
|
||||
StallAfter: 30 * time.Minute, Now: func() time.Time { return now },
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := handler.Handle(context.Background(), jobs.Job{Payload: []byte(`{"publication_id":"publication-1","sequence":4}`)}); err != nil {
|
||||
t.Fatalf("Handle() error = %v", err)
|
||||
}
|
||||
if !store.stalledMarked || store.recordedState != "" || !store.stalledCutoff.Equal(now.Add(-30*time.Minute)) {
|
||||
t.Fatalf("store = %#v", store)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPollHandlerRejectsStaleSequenceWithoutPollingRemote(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
t.Fatal("stale sequence must not poll the remote knowledge endpoint")
|
||||
}))
|
||||
defer server.Close()
|
||||
client, err := weknora.NewClient(weknora.ClientConfig{BaseURL: server.URL, APIKey: "test-api-key", Timeout: time.Second})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Date(2026, time.September, 4, 4, 0, 0, 0, time.UTC)
|
||||
store := &pollStoreStub{publication: postgresstore.PublicationForPoll{
|
||||
ID: "publication-1", TestKBID: "kb-1", KnowledgeID: "knowledge-1",
|
||||
ProcessingState: governance.ProcessingProcessing, PollSequence: 5,
|
||||
}}
|
||||
handler, err := NewPollHandler(PollHandlerConfig{
|
||||
Store: store, Client: client, PollDelay: 15 * time.Second,
|
||||
StallAfter: 30 * time.Minute, Now: func() time.Time { return now },
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := handler.Handle(context.Background(), jobs.Job{Payload: []byte(`{"publication_id":"publication-1","sequence":2}`)}); !errors.Is(err, ErrPollStateConflict) {
|
||||
t.Fatalf("Handle() error = %v, want state conflict", err)
|
||||
}
|
||||
if store.recordedState != "" || store.stalledMarked || store.completedValidated {
|
||||
t.Fatalf("store mutated for stale sequence: %#v", store)
|
||||
}
|
||||
}
|
||||
|
||||
type pollStoreStub struct {
|
||||
publication postgresstore.PublicationForPoll
|
||||
recordedState governance.ProcessingState
|
||||
recordedRemoteUpdated *time.Time
|
||||
nextPollAt time.Time
|
||||
completedValidated bool
|
||||
stalledMarked bool
|
||||
stalledCutoff time.Time
|
||||
}
|
||||
|
||||
func (store *pollStoreStub) LoadPublicationForPoll(context.Context, string) (postgresstore.PublicationForPoll, error) {
|
||||
return store.publication, nil
|
||||
}
|
||||
|
||||
func (store *pollStoreStub) RecordRemoteStatusAndScheduleNextPoll(_ context.Context, _ string, _ int, state governance.ProcessingState, remoteUpdated *time.Time, _ time.Time, nextAt time.Time) (int, error) {
|
||||
store.recordedState = state
|
||||
store.recordedRemoteUpdated = remoteUpdated
|
||||
store.nextPollAt = nextAt
|
||||
return store.publication.PollSequence + 1, nil
|
||||
}
|
||||
|
||||
func (store *pollStoreStub) RecordCompletedAndScheduleValidation(_ context.Context, _ string, _ int, remoteUpdated *time.Time, _ time.Time) error {
|
||||
store.completedValidated = true
|
||||
store.recordedRemoteUpdated = remoteUpdated
|
||||
return nil
|
||||
}
|
||||
|
||||
func (store *pollStoreStub) RecordProcessingFailure(context.Context, string, int, string, time.Time) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (store *pollStoreStub) MarkStalled(_ context.Context, _ string, _ int, cutoff, _ time.Time) error {
|
||||
store.stalledMarked = true
|
||||
store.stalledCutoff = cutoff
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
package testweknora
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"knowledge-governance/internal/jobs"
|
||||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||||
"knowledge-governance/internal/platform/weknora"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrInvalidRemoveJobPayload = errors.New("test_remove_invalid_payload")
|
||||
ErrRemoveRemoteFailed = errors.New("test_remove_remote_failed")
|
||||
ErrRemoveRemoteIdentity = errors.New("test_remove_remote_identity_invalid")
|
||||
ErrRemovePending = errors.New("test_remove_pending")
|
||||
ErrRemoveStateConflict = errors.New("test_remove_state_conflict")
|
||||
)
|
||||
|
||||
type RemoveStore interface {
|
||||
LoadPublicationForRemoval(context.Context, string) (postgresstore.PublicationForRemoval, error)
|
||||
RecordRemovalCompleted(context.Context, string, time.Time) error
|
||||
RecordRemovalFailure(context.Context, string, string, time.Time) error
|
||||
}
|
||||
|
||||
type RemoveClient interface {
|
||||
GetKnowledge(context.Context, string) (weknora.Knowledge, error)
|
||||
DeleteKnowledge(context.Context, string) error
|
||||
}
|
||||
|
||||
type RemoveHandlerConfig struct {
|
||||
Store RemoveStore
|
||||
Client RemoveClient
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
type RemoveHandler struct {
|
||||
store RemoveStore
|
||||
client RemoveClient
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
type removePayload struct {
|
||||
PublicationID string `json:"publication_id"`
|
||||
}
|
||||
|
||||
func NewRemoveHandler(config RemoveHandlerConfig) (*RemoveHandler, error) {
|
||||
if config.Store == nil || config.Client == nil {
|
||||
return nil, errors.New("invalid test removal handler configuration")
|
||||
}
|
||||
if config.Now == nil {
|
||||
config.Now = time.Now
|
||||
}
|
||||
return &RemoveHandler{store: config.Store, client: config.Client, now: config.Now}, nil
|
||||
}
|
||||
|
||||
func (handler *RemoveHandler) Handle(ctx context.Context, job jobs.Job) error {
|
||||
var payload removePayload
|
||||
if err := decodeStrictJSON(job.Payload, &payload); err != nil || payload.PublicationID == "" {
|
||||
return jobs.Permanent(jobs.ErrCodeHandlerFallback, ErrInvalidRemoveJobPayload)
|
||||
}
|
||||
candidate, err := handler.store.LoadPublicationForRemoval(ctx, payload.PublicationID)
|
||||
if err != nil {
|
||||
return ErrRemoveStateConflict
|
||||
}
|
||||
knowledge, err := handler.client.GetKnowledge(ctx, candidate.KnowledgeID)
|
||||
if isNotFound(err) {
|
||||
return handler.complete(ctx, candidate.ID)
|
||||
}
|
||||
if err != nil {
|
||||
return ErrRemoveRemoteFailed
|
||||
}
|
||||
if knowledge.ID != candidate.KnowledgeID || knowledge.KnowledgeBaseID != candidate.TestKBID {
|
||||
return handler.failClosed(ctx, candidate.ID, "removal_identity_invalid")
|
||||
}
|
||||
if err := handler.client.DeleteKnowledge(ctx, candidate.KnowledgeID); err != nil {
|
||||
if isNotFound(err) {
|
||||
return handler.complete(ctx, candidate.ID)
|
||||
}
|
||||
if !weknora.IsRetryable(err) {
|
||||
return handler.failClosed(ctx, candidate.ID, "removal_remote_rejected")
|
||||
}
|
||||
return ErrRemoveRemoteFailed
|
||||
}
|
||||
knowledge, err = handler.client.GetKnowledge(ctx, candidate.KnowledgeID)
|
||||
if isNotFound(err) {
|
||||
return handler.complete(ctx, candidate.ID)
|
||||
}
|
||||
if err != nil {
|
||||
return ErrRemoveRemoteFailed
|
||||
}
|
||||
if knowledge.ID != candidate.KnowledgeID || knowledge.KnowledgeBaseID != candidate.TestKBID {
|
||||
return handler.failClosed(ctx, candidate.ID, "removal_identity_invalid")
|
||||
}
|
||||
return ErrRemovePending
|
||||
}
|
||||
|
||||
func (handler *RemoveHandler) complete(ctx context.Context, publicationID string) error {
|
||||
if err := handler.store.RecordRemovalCompleted(ctx, publicationID, handler.now()); err != nil {
|
||||
return ErrRemoveStateConflict
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (handler *RemoveHandler) failClosed(ctx context.Context, publicationID, code string) error {
|
||||
if err := handler.store.RecordRemovalFailure(ctx, publicationID, code, handler.now()); err != nil {
|
||||
return ErrRemoveStateConflict
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func isNotFound(err error) bool {
|
||||
var httpErr *weknora.HTTPError
|
||||
return errors.As(err, &httpErr) && httpErr.StatusCode == http.StatusNotFound
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
package testweknora
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
"knowledge-governance/internal/jobs"
|
||||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||||
"knowledge-governance/internal/platform/weknora"
|
||||
)
|
||||
|
||||
type removalStoreStub struct {
|
||||
candidate postgresstore.PublicationForRemoval
|
||||
completed string
|
||||
failed string
|
||||
}
|
||||
|
||||
func (store *removalStoreStub) LoadPublicationForRemoval(context.Context, string) (postgresstore.PublicationForRemoval, error) {
|
||||
return store.candidate, nil
|
||||
}
|
||||
|
||||
func (store *removalStoreStub) RecordRemovalCompleted(_ context.Context, publicationID string, _ time.Time) error {
|
||||
store.completed = publicationID
|
||||
return nil
|
||||
}
|
||||
|
||||
func (store *removalStoreStub) RecordRemovalFailure(_ context.Context, publicationID, code string, _ time.Time) error {
|
||||
store.failed = publicationID + ":" + code
|
||||
return nil
|
||||
}
|
||||
|
||||
type removalClientStub struct {
|
||||
knowledge weknora.Knowledge
|
||||
getCalls int
|
||||
deleted string
|
||||
keepAfterDelete bool
|
||||
}
|
||||
|
||||
func (client *removalClientStub) GetKnowledge(context.Context, string) (weknora.Knowledge, error) {
|
||||
client.getCalls++
|
||||
if client.getCalls > 1 && !client.keepAfterDelete {
|
||||
return weknora.Knowledge{}, &weknora.HTTPError{StatusCode: 404}
|
||||
}
|
||||
return client.knowledge, nil
|
||||
}
|
||||
|
||||
func (client *removalClientStub) DeleteKnowledge(_ context.Context, knowledgeID string) error {
|
||||
client.deleted = knowledgeID
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestRemoveHandlerVerifiesIdentityAndReadBackBeforeCompletion(t *testing.T) {
|
||||
store := &removalStoreStub{candidate: postgresstore.PublicationForRemoval{
|
||||
ID: "publication-1", TestKBID: "kb-1", KnowledgeID: "knowledge-1", RetrievalState: governance.RetrievalDisabling,
|
||||
}}
|
||||
client := &removalClientStub{knowledge: weknora.Knowledge{ID: "knowledge-1", KnowledgeBaseID: "kb-1"}}
|
||||
handler, err := NewRemoveHandler(RemoveHandlerConfig{Store: store, Client: client, Now: func() time.Time { return time.Date(2026, 9, 9, 3, 0, 0, 0, time.UTC) }})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := handler.Handle(context.Background(), jobs.Job{Payload: []byte(`{"publication_id":"publication-1"}`)}); err != nil {
|
||||
t.Fatalf("Handle() error = %v", err)
|
||||
}
|
||||
if client.deleted != "knowledge-1" || store.completed != "publication-1" || store.failed != "" {
|
||||
t.Fatalf("deleted=%q completed=%q failed=%q", client.deleted, store.completed, store.failed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoveHandlerRejectsMismatchedKnowledgeBaseWithoutDeleting(t *testing.T) {
|
||||
store := &removalStoreStub{candidate: postgresstore.PublicationForRemoval{
|
||||
ID: "publication-1", TestKBID: "kb-1", KnowledgeID: "knowledge-1", RetrievalState: governance.RetrievalDisabling,
|
||||
}}
|
||||
client := &removalClientStub{knowledge: weknora.Knowledge{ID: "knowledge-1", KnowledgeBaseID: "other-kb"}}
|
||||
handler, err := NewRemoveHandler(RemoveHandlerConfig{Store: store, Client: client})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = handler.Handle(context.Background(), jobs.Job{Payload: []byte(`{"publication_id":"publication-1"}`)})
|
||||
if err != nil || client.deleted != "" || store.failed != "publication-1:removal_identity_invalid" {
|
||||
t.Fatalf("error=%v deleted=%q failed=%q", err, client.deleted, store.failed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoveHandlerRetriesWhileAsynchronousDeleteIsStillVisible(t *testing.T) {
|
||||
store := &removalStoreStub{candidate: postgresstore.PublicationForRemoval{
|
||||
ID: "publication-1", TestKBID: "kb-1", KnowledgeID: "knowledge-1", RetrievalState: governance.RetrievalDisabling,
|
||||
}}
|
||||
client := &removalClientStub{knowledge: weknora.Knowledge{ID: "knowledge-1", KnowledgeBaseID: "kb-1"}, keepAfterDelete: true}
|
||||
handler, err := NewRemoveHandler(RemoveHandlerConfig{Store: store, Client: client})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = handler.Handle(context.Background(), jobs.Job{Payload: []byte(`{"publication_id":"publication-1"}`)})
|
||||
if !errors.Is(err, ErrRemovePending) || store.completed != "" {
|
||||
t.Fatalf("error=%v completed=%q", err, store.completed)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,241 @@
|
||||
package testweknora
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
"knowledge-governance/internal/jobs"
|
||||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||||
"knowledge-governance/internal/platform/spool"
|
||||
"knowledge-governance/internal/platform/weknora"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrInvalidUploadJobPayload = errors.New("test_upload_invalid_payload")
|
||||
ErrUploadArtifactInvalid = errors.New("test_upload_artifact_invalid")
|
||||
ErrUploadRemoteFailed = errors.New("test_upload_remote_failed")
|
||||
ErrUploadRemoteIdentity = errors.New("test_upload_remote_identity_invalid")
|
||||
ErrUploadStateConflict = errors.New("test_upload_state_conflict")
|
||||
)
|
||||
|
||||
type UploadStore interface {
|
||||
LoadPublicationForUpload(context.Context, string) (postgresstore.PublicationForUpload, error)
|
||||
MarkUploading(context.Context, string, time.Time) error
|
||||
RecordUploadAndSchedulePoll(context.Context, string, string, time.Time, time.Time) (int, error)
|
||||
RecordUploadFailure(context.Context, string, string, time.Time) error
|
||||
}
|
||||
|
||||
type UploadClient interface {
|
||||
UploadFile(context.Context, weknora.UploadRequest) (weknora.Knowledge, error)
|
||||
ListKnowledge(context.Context, string, int, int) ([]weknora.Knowledge, int64, error)
|
||||
GetKnowledge(context.Context, string) (weknora.Knowledge, error)
|
||||
}
|
||||
|
||||
type UploadHandlerConfig struct {
|
||||
Store UploadStore
|
||||
Client UploadClient
|
||||
SpoolRoot string
|
||||
MaxUploadBytes int64
|
||||
FirstPollDelay time.Duration
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
type UploadHandler struct {
|
||||
store UploadStore
|
||||
client UploadClient
|
||||
spoolRoot string
|
||||
maxUploadBytes int64
|
||||
firstPollDelay time.Duration
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
type uploadPayload struct {
|
||||
PublicationID string `json:"publication_id"`
|
||||
TargetScope string `json:"target_scope,omitempty"`
|
||||
TargetKBID string `json:"target_kb_id,omitempty"`
|
||||
}
|
||||
|
||||
func NewUploadHandler(config UploadHandlerConfig) (*UploadHandler, error) {
|
||||
if config.Store == nil || config.Client == nil || !filepath.IsAbs(config.SpoolRoot) ||
|
||||
config.MaxUploadBytes <= 0 || config.FirstPollDelay <= 0 {
|
||||
return nil, errors.New("invalid test upload handler configuration")
|
||||
}
|
||||
if config.Now == nil {
|
||||
config.Now = time.Now
|
||||
}
|
||||
return &UploadHandler{
|
||||
store: config.Store, client: config.Client, spoolRoot: config.SpoolRoot,
|
||||
maxUploadBytes: config.MaxUploadBytes, firstPollDelay: config.FirstPollDelay, now: config.Now,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (handler *UploadHandler) Handle(ctx context.Context, job jobs.Job) error {
|
||||
var payload uploadPayload
|
||||
if err := decodeStrictJSON(job.Payload, &payload); err != nil || payload.PublicationID == "" {
|
||||
return jobs.Permanent(jobs.ErrCodeHandlerFallback, ErrInvalidUploadJobPayload)
|
||||
}
|
||||
candidate, err := handler.store.LoadPublicationForUpload(ctx, payload.PublicationID)
|
||||
if err != nil {
|
||||
return ErrUploadStateConflict
|
||||
}
|
||||
// P4: the job snapshots its target; a mismatch against the stored publication
|
||||
// means the plan changed underneath and this delivery must not guess.
|
||||
if payload.TargetKBID != "" && payload.TargetKBID != candidate.TestKBID {
|
||||
return ErrUploadStateConflict
|
||||
}
|
||||
if payload.TargetScope != "" && payload.TargetScope != string(candidate.TargetScope) {
|
||||
return ErrUploadStateConflict
|
||||
}
|
||||
if candidate.IngestState == governance.IngestUploading {
|
||||
knowledge, err := handler.recoverAmbiguousUpload(ctx, candidate)
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrUploadRemoteIdentity) {
|
||||
return handler.recordTerminalUploadFailure(ctx, candidate.ID, "upload_identity_ambiguous")
|
||||
}
|
||||
return err
|
||||
}
|
||||
now := handler.now()
|
||||
if _, err := handler.store.RecordUploadAndSchedulePoll(ctx, candidate.ID, knowledge.ID, now, now.Add(handler.firstPollDelay)); err != nil {
|
||||
return ErrUploadStateConflict
|
||||
}
|
||||
return nil
|
||||
}
|
||||
artifact, err := spool.OpenVerified(handler.spoolRoot, spool.Expectation{
|
||||
RelativeRef: candidate.LocalSpoolRef,
|
||||
SHA256: candidate.SHA256,
|
||||
SizeBytes: candidate.SizeBytes,
|
||||
Format: candidate.ValidatedFormat,
|
||||
}, handler.maxUploadBytes)
|
||||
if err != nil {
|
||||
return handler.recordTerminalUploadFailure(ctx, candidate.ID, "upload_artifact_invalid")
|
||||
}
|
||||
defer artifact.Close()
|
||||
if err := handler.store.MarkUploading(ctx, candidate.ID, handler.now()); err != nil {
|
||||
return ErrUploadStateConflict
|
||||
}
|
||||
knowledge, uploadErr := handler.client.UploadFile(ctx, weknora.UploadRequest{
|
||||
KnowledgeBaseID: candidate.TestKBID,
|
||||
FileName: safeUploadFileName(candidate.Title, candidate.ID, candidate.ValidatedFormat),
|
||||
Content: artifact,
|
||||
Metadata: map[string]string{
|
||||
"content_sha256": candidate.SHA256,
|
||||
"publication_id": candidate.ID,
|
||||
},
|
||||
Channel: "governance",
|
||||
})
|
||||
if err := artifact.VerifyComplete(); err != nil {
|
||||
return jobs.Permanent(jobs.ErrCodeArtifactInvalid, ErrUploadArtifactInvalid)
|
||||
}
|
||||
if uploadErr != nil {
|
||||
if !errors.Is(uploadErr, weknora.ErrConflict) && !weknora.IsRetryable(uploadErr) {
|
||||
return ErrUploadRemoteFailed
|
||||
}
|
||||
knowledge, err = handler.recoverAmbiguousUpload(ctx, candidate)
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrUploadRemoteIdentity) {
|
||||
return handler.recordTerminalUploadFailure(ctx, candidate.ID, "upload_identity_ambiguous")
|
||||
}
|
||||
return err
|
||||
}
|
||||
}
|
||||
if knowledge.ID == "" || knowledge.KnowledgeBaseID != candidate.TestKBID {
|
||||
return ErrUploadRemoteIdentity
|
||||
}
|
||||
now := handler.now()
|
||||
if _, err := handler.store.RecordUploadAndSchedulePoll(ctx, candidate.ID, knowledge.ID, now, now.Add(handler.firstPollDelay)); err != nil {
|
||||
return ErrUploadStateConflict
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (handler *UploadHandler) recoverAmbiguousUpload(ctx context.Context, candidate postgresstore.PublicationForUpload) (weknora.Knowledge, error) {
|
||||
const pageSize = 100
|
||||
matches := make([]weknora.Knowledge, 0, 2)
|
||||
for page := 1; page <= 100; page++ {
|
||||
items, total, err := handler.client.ListKnowledge(ctx, candidate.TestKBID, page, pageSize)
|
||||
if err != nil {
|
||||
return weknora.Knowledge{}, ErrUploadRemoteFailed
|
||||
}
|
||||
for _, item := range items {
|
||||
publicationID, publicationOK := metadataString(item.Metadata, "publication_id")
|
||||
contentSHA, contentOK := metadataString(item.Metadata, "content_sha256")
|
||||
if item.KnowledgeBaseID == candidate.TestKBID && publicationOK && contentOK && publicationID == candidate.ID && contentSHA == candidate.SHA256 {
|
||||
matches = append(matches, item)
|
||||
if len(matches) > 1 {
|
||||
return weknora.Knowledge{}, ErrUploadRemoteIdentity
|
||||
}
|
||||
}
|
||||
}
|
||||
if int64(page*pageSize) >= total {
|
||||
break
|
||||
}
|
||||
if len(items) == 0 {
|
||||
return weknora.Knowledge{}, ErrUploadRemoteFailed
|
||||
}
|
||||
}
|
||||
if len(matches) != 1 {
|
||||
return weknora.Knowledge{}, ErrUploadRemoteFailed
|
||||
}
|
||||
verified, err := handler.client.GetKnowledge(ctx, matches[0].ID)
|
||||
if err != nil {
|
||||
return weknora.Knowledge{}, ErrUploadRemoteFailed
|
||||
}
|
||||
publicationID, publicationOK := metadataString(verified.Metadata, "publication_id")
|
||||
contentSHA, contentOK := metadataString(verified.Metadata, "content_sha256")
|
||||
if verified.ID != matches[0].ID || verified.KnowledgeBaseID != candidate.TestKBID ||
|
||||
!publicationOK || !contentOK || publicationID != candidate.ID || contentSHA != candidate.SHA256 {
|
||||
return weknora.Knowledge{}, ErrUploadRemoteIdentity
|
||||
}
|
||||
return verified, nil
|
||||
}
|
||||
|
||||
func metadataString(metadata map[string]any, key string) (string, bool) {
|
||||
value, ok := metadata[key].(string)
|
||||
return value, ok
|
||||
}
|
||||
|
||||
func (handler *UploadHandler) recordTerminalUploadFailure(ctx context.Context, publicationID, code string) error {
|
||||
if err := handler.store.RecordUploadFailure(ctx, publicationID, code, handler.now()); err != nil {
|
||||
return ErrUploadStateConflict
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func decodeStrictJSON(data []byte, target any) error {
|
||||
decoder := json.NewDecoder(bytes.NewReader(data))
|
||||
decoder.DisallowUnknownFields()
|
||||
if err := decoder.Decode(target); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := decoder.Decode(&struct{}{}); err != io.EOF {
|
||||
return errors.New("extra JSON value")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func safeUploadFileName(title, fallback, format string) string {
|
||||
base := strings.TrimSpace(title)
|
||||
base = strings.Map(func(character rune) rune {
|
||||
switch character {
|
||||
case '/', '\\', '\x00', '\r', '\n', '\t':
|
||||
return '_'
|
||||
default:
|
||||
return character
|
||||
}
|
||||
}, base)
|
||||
base = strings.Trim(base, ". ")
|
||||
if base == "" {
|
||||
base = fallback
|
||||
}
|
||||
if len(base) > 120 {
|
||||
base = base[:120]
|
||||
}
|
||||
return base + "." + format
|
||||
}
|
||||
@@ -0,0 +1,354 @@
|
||||
package testweknora
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
"knowledge-governance/internal/jobs"
|
||||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||||
"knowledge-governance/internal/platform/weknora"
|
||||
)
|
||||
|
||||
func TestUploadHandlerStreamsVerifiedArtifactAndSchedulesFirstPoll(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
if err := os.Mkdir(filepath.Join(root, "objects"), 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body := []byte("safe governance document\n")
|
||||
if err := os.WriteFile(filepath.Join(root, "objects", "artifact.md"), body, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
digest := sha256.Sum256(body)
|
||||
sha := hex.EncodeToString(digest[:])
|
||||
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPost || r.URL.Path != "/api/v1/knowledge-bases/kb-1/knowledge/file" {
|
||||
t.Fatalf("request = %s %s", r.Method, r.URL.Path)
|
||||
}
|
||||
if err := r.ParseMultipartForm(1 << 20); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
file, _, err := r.FormFile("file")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer file.Close()
|
||||
uploaded, err := io.ReadAll(file)
|
||||
if err != nil || string(uploaded) != string(body) {
|
||||
t.Fatalf("uploaded body = %q, %v", uploaded, err)
|
||||
}
|
||||
if r.FormValue("metadata") != `{"content_sha256":"`+sha+`","publication_id":"publication-1"}` {
|
||||
t.Fatalf("metadata = %q", r.FormValue("metadata"))
|
||||
}
|
||||
if r.FormValue("channel") != "governance" {
|
||||
t.Fatalf("channel = %q", r.FormValue("channel"))
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"success":true,"data":{"id":"knowledge-1","knowledge_base_id":"kb-1","parse_status":"pending"}}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
client, err := weknora.NewClient(weknora.ClientConfig{BaseURL: server.URL, APIKey: "test-api-key", Timeout: time.Second})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
now := time.Date(2026, time.September, 4, 3, 0, 0, 0, time.UTC)
|
||||
store := &uploadStoreStub{candidate: postgresstore.PublicationForUpload{
|
||||
ID: "publication-1", SourceItemVersionID: "version-1", TestKBID: "kb-1",
|
||||
IngestState: governance.IngestPending, ProcessingState: governance.ProcessingPending,
|
||||
ContentObjectID: "content-1", SHA256: sha, SizeBytes: int64(len(body)),
|
||||
ValidatedFormat: "md", LocalSpoolRef: "objects/artifact.md", Title: "Safe title",
|
||||
}}
|
||||
handler, err := NewUploadHandler(UploadHandlerConfig{
|
||||
Store: store, Client: client, SpoolRoot: root, MaxUploadBytes: 50 * 1024 * 1024,
|
||||
FirstPollDelay: 10 * time.Second, Now: func() time.Time { return now },
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
job := jobs.Job{ID: "job-1", Type: "test.upload", Payload: []byte(`{"publication_id":"publication-1"}`)}
|
||||
if err := handler.Handle(context.Background(), job); err != nil {
|
||||
t.Fatalf("Handle() error = %v", err)
|
||||
}
|
||||
if !store.markedUploading || store.recordedKnowledgeID != "knowledge-1" || !store.nextPollAt.Equal(now.Add(10*time.Second)) {
|
||||
t.Fatalf("store = %#v", store)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadHandlerRejectsJobSnapshotWithMismatchedTarget(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
now := time.Date(2026, time.September, 4, 3, 0, 0, 0, time.UTC)
|
||||
store := &uploadStoreStub{candidate: postgresstore.PublicationForUpload{
|
||||
ID: "publication-1", SourceItemVersionID: "version-1", TestKBID: "kb-1", TargetScope: governance.VisibilityInternal,
|
||||
IngestState: governance.IngestPending, ProcessingState: governance.ProcessingPending,
|
||||
}}
|
||||
client, err := weknora.NewClient(weknora.ClientConfig{BaseURL: "http://127.0.0.1:1", APIKey: "***", Timeout: time.Second})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
handler, err := NewUploadHandler(UploadHandlerConfig{Store: store, Client: client, SpoolRoot: root, MaxUploadBytes: 1 << 20, FirstPollDelay: time.Second, Now: func() time.Time { return now }})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
job := jobs.Job{ID: "job-1", Type: "test.upload", Payload: []byte(`{"publication_id":"publication-1","target_scope":"external","target_kb_id":"kb-1"}`)}
|
||||
if err := handler.Handle(context.Background(), job); !errors.Is(err, ErrUploadStateConflict) {
|
||||
t.Fatalf("scope mismatch error = %v", err)
|
||||
}
|
||||
job = jobs.Job{ID: "job-2", Type: "test.upload", Payload: []byte(`{"publication_id":"publication-1","target_scope":"internal","target_kb_id":"kb-other"}`)}
|
||||
if err := handler.Handle(context.Background(), job); !errors.Is(err, ErrUploadStateConflict) {
|
||||
t.Fatalf("kb mismatch error = %v", err)
|
||||
}
|
||||
if store.markedUploading {
|
||||
t.Fatal("mismatched job must not mark uploading")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadHandlerRecoversUniqueRemoteKnowledgeAfterResponseLoss(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
if err := os.Mkdir(filepath.Join(root, "objects"), 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body := []byte("safe ambiguous upload\n")
|
||||
if err := os.WriteFile(filepath.Join(root, "objects", "artifact.md"), body, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
digest := sha256.Sum256(body)
|
||||
sha := hex.EncodeToString(digest[:])
|
||||
var uploads atomic.Int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch {
|
||||
case r.Method == http.MethodPost:
|
||||
uploads.Add(1)
|
||||
if err := r.ParseMultipartForm(1 << 20); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
file, _, err := r.FormFile("file")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, _ = io.Copy(io.Discard, file)
|
||||
_ = file.Close()
|
||||
connection, _, err := w.(http.Hijacker).Hijack()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = connection.Close()
|
||||
case r.Method == http.MethodGet && r.URL.Path == "/api/v1/knowledge-bases/kb-1/knowledge":
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"success":true,"data":[{"id":"knowledge-recovered","knowledge_base_id":"kb-1","metadata":{"publication_id":"publication-1","content_sha256":"` + sha + `"}}],"total":1,"page":1,"page_size":100}`))
|
||||
case r.Method == http.MethodGet && r.URL.Path == "/api/v1/knowledge/knowledge-recovered":
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"success":true,"data":{"id":"knowledge-recovered","knowledge_base_id":"kb-1","metadata":{"publication_id":"publication-1","content_sha256":"` + sha + `"}}}`))
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
client, err := weknora.NewClient(weknora.ClientConfig{BaseURL: server.URL, APIKey: "test-api-key", Timeout: time.Second})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Date(2026, time.September, 4, 3, 10, 0, 0, time.UTC)
|
||||
store := &uploadStoreStub{candidate: postgresstore.PublicationForUpload{
|
||||
ID: "publication-1", TestKBID: "kb-1", IngestState: governance.IngestPending,
|
||||
ProcessingState: governance.ProcessingPending, SHA256: sha, SizeBytes: int64(len(body)),
|
||||
ValidatedFormat: "md", LocalSpoolRef: "objects/artifact.md", Title: "Safe title",
|
||||
}}
|
||||
handler, err := NewUploadHandler(UploadHandlerConfig{
|
||||
Store: store, Client: client, SpoolRoot: root, MaxUploadBytes: 50 * 1024 * 1024,
|
||||
FirstPollDelay: time.Second, Now: func() time.Time { return now },
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := handler.Handle(context.Background(), jobs.Job{Payload: []byte(`{"publication_id":"publication-1"}`)}); err != nil {
|
||||
t.Fatalf("Handle() error = %v", err)
|
||||
}
|
||||
if uploads.Load() != 1 || store.recordedKnowledgeID != "knowledge-recovered" {
|
||||
t.Fatalf("uploads=%d knowledge=%q", uploads.Load(), store.recordedKnowledgeID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadHandlerRecoversUploadingStateBeforeAnotherPost(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
if err := os.Mkdir(filepath.Join(root, "objects"), 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body := []byte("safe retry recovery\n")
|
||||
if err := os.WriteFile(filepath.Join(root, "objects", "artifact.md"), body, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
digest := sha256.Sum256(body)
|
||||
sha := hex.EncodeToString(digest[:])
|
||||
var uploads atomic.Int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch {
|
||||
case r.Method == http.MethodPost:
|
||||
uploads.Add(1)
|
||||
http.Error(w, "must not upload", http.StatusInternalServerError)
|
||||
case r.Method == http.MethodGet && r.URL.Path == "/api/v1/knowledge-bases/kb-1/knowledge":
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"success":true,"data":[{"id":"knowledge-existing","knowledge_base_id":"kb-1","metadata":{"publication_id":"publication-1","content_sha256":"` + sha + `"}}],"total":1,"page":1,"page_size":100}`))
|
||||
case r.Method == http.MethodGet && r.URL.Path == "/api/v1/knowledge/knowledge-existing":
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"success":true,"data":{"id":"knowledge-existing","knowledge_base_id":"kb-1","metadata":{"publication_id":"publication-1","content_sha256":"` + sha + `"}}}`))
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
client, err := weknora.NewClient(weknora.ClientConfig{BaseURL: server.URL, APIKey: "test-api-key", Timeout: time.Second})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Date(2026, time.September, 4, 3, 20, 0, 0, time.UTC)
|
||||
store := &uploadStoreStub{candidate: postgresstore.PublicationForUpload{
|
||||
ID: "publication-1", TestKBID: "kb-1", IngestState: governance.IngestUploading,
|
||||
ProcessingState: governance.ProcessingPending, SHA256: sha, SizeBytes: int64(len(body)),
|
||||
ValidatedFormat: "md", LocalSpoolRef: "objects/artifact.md", Title: "Safe title",
|
||||
}}
|
||||
handler, err := NewUploadHandler(UploadHandlerConfig{
|
||||
Store: store, Client: client, SpoolRoot: root, MaxUploadBytes: 50 * 1024 * 1024,
|
||||
FirstPollDelay: time.Second, Now: func() time.Time { return now },
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := handler.Handle(context.Background(), jobs.Job{Payload: []byte(`{"publication_id":"publication-1"}`)}); err != nil {
|
||||
t.Fatalf("Handle() error = %v", err)
|
||||
}
|
||||
if uploads.Load() != 0 || store.recordedKnowledgeID != "knowledge-existing" {
|
||||
t.Fatalf("uploads=%d knowledge=%q", uploads.Load(), store.recordedKnowledgeID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadHandlerFailsClosedWhenAmbiguousRecoveryHasMultipleMatches(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
if err := os.Mkdir(filepath.Join(root, "objects"), 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body := []byte("safe ambiguous duplicate\n")
|
||||
if err := os.WriteFile(filepath.Join(root, "objects", "artifact.md"), body, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
digest := sha256.Sum256(body)
|
||||
sha := hex.EncodeToString(digest[:])
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch {
|
||||
case r.Method == http.MethodPost:
|
||||
_, _ = io.Copy(io.Discard, r.Body)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusConflict)
|
||||
_, _ = w.Write([]byte(`{"success":false,"data":{}}`))
|
||||
case r.Method == http.MethodGet && r.URL.Path == "/api/v1/knowledge-bases/kb-1/knowledge":
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"success":true,"data":[` +
|
||||
`{"id":"knowledge-a","knowledge_base_id":"kb-1","metadata":{"publication_id":"publication-1","content_sha256":"` + sha + `"}},` +
|
||||
`{"id":"knowledge-b","knowledge_base_id":"kb-1","metadata":{"publication_id":"publication-1","content_sha256":"` + sha + `"}}` +
|
||||
`],"total":2,"page":1,"page_size":100}`))
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
client, err := weknora.NewClient(weknora.ClientConfig{BaseURL: server.URL, APIKey: "test-api-key", Timeout: time.Second})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Date(2026, time.September, 4, 3, 30, 0, 0, time.UTC)
|
||||
store := &uploadStoreStub{candidate: postgresstore.PublicationForUpload{
|
||||
ID: "publication-1", TestKBID: "kb-1", IngestState: governance.IngestPending,
|
||||
ProcessingState: governance.ProcessingPending, SHA256: sha, SizeBytes: int64(len(body)),
|
||||
ValidatedFormat: "md", LocalSpoolRef: "objects/artifact.md", Title: "Safe title",
|
||||
}}
|
||||
handler, err := NewUploadHandler(UploadHandlerConfig{
|
||||
Store: store, Client: client, SpoolRoot: root, MaxUploadBytes: 50 * 1024 * 1024,
|
||||
FirstPollDelay: time.Second, Now: func() time.Time { return now },
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := handler.Handle(context.Background(), jobs.Job{Payload: []byte(`{"publication_id":"publication-1"}`)}); err != nil {
|
||||
t.Fatalf("Handle() error = %v", err)
|
||||
}
|
||||
if store.uploadFailureCode != "upload_identity_ambiguous" || store.recordedKnowledgeID != "" {
|
||||
t.Fatalf("failure=%q knowledge=%q", store.uploadFailureCode, store.recordedKnowledgeID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadHandlerRecordsInvalidArtifactForManualRetry(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
store := &uploadStoreStub{candidate: postgresstore.PublicationForUpload{
|
||||
ID: "publication-1", TestKBID: "kb-1", IngestState: governance.IngestPending,
|
||||
ProcessingState: governance.ProcessingPending, SHA256: strings.Repeat("a", 64), SizeBytes: 128,
|
||||
ValidatedFormat: "md", LocalSpoolRef: "missing/artifact.md", Title: "Safe title",
|
||||
}}
|
||||
handler, err := NewUploadHandler(UploadHandlerConfig{
|
||||
Store: store, Client: uploadClientStub{}, SpoolRoot: root, MaxUploadBytes: 1024,
|
||||
FirstPollDelay: time.Second,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := handler.Handle(context.Background(), jobs.Job{Payload: []byte(`{"publication_id":"publication-1"}`)}); err != nil {
|
||||
t.Fatalf("Handle() error = %v, want terminal failure recorded", err)
|
||||
}
|
||||
if store.uploadFailureCode != "upload_artifact_invalid" || store.markedUploading {
|
||||
t.Fatalf("failure=%q markedUploading=%v", store.uploadFailureCode, store.markedUploading)
|
||||
}
|
||||
}
|
||||
|
||||
type uploadStoreStub struct {
|
||||
candidate postgresstore.PublicationForUpload
|
||||
markedUploading bool
|
||||
recordedKnowledgeID string
|
||||
nextPollAt time.Time
|
||||
uploadFailureCode string
|
||||
}
|
||||
|
||||
func (store *uploadStoreStub) LoadPublicationForUpload(context.Context, string) (postgresstore.PublicationForUpload, error) {
|
||||
return store.candidate, nil
|
||||
}
|
||||
|
||||
func (store *uploadStoreStub) MarkUploading(context.Context, string, time.Time) error {
|
||||
store.markedUploading = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func (store *uploadStoreStub) RecordUploadAndSchedulePoll(_ context.Context, _ string, knowledgeID string, _ time.Time, nextAt time.Time) (int, error) {
|
||||
store.recordedKnowledgeID = knowledgeID
|
||||
store.nextPollAt = nextAt
|
||||
return 1, nil
|
||||
}
|
||||
|
||||
func (store *uploadStoreStub) RecordUploadFailure(_ context.Context, _ string, code string, _ time.Time) error {
|
||||
store.uploadFailureCode = code
|
||||
return nil
|
||||
}
|
||||
|
||||
type uploadClientStub struct{}
|
||||
|
||||
func (uploadClientStub) UploadFile(context.Context, weknora.UploadRequest) (weknora.Knowledge, error) {
|
||||
return weknora.Knowledge{}, errors.New("unexpected upload")
|
||||
}
|
||||
|
||||
func (uploadClientStub) ListKnowledge(context.Context, string, int, int) ([]weknora.Knowledge, int64, error) {
|
||||
return nil, 0, errors.New("unexpected list")
|
||||
}
|
||||
|
||||
func (uploadClientStub) GetKnowledge(context.Context, string) (weknora.Knowledge, error) {
|
||||
return weknora.Knowledge{}, errors.New("unexpected get")
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
package testweknora
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
"knowledge-governance/internal/jobs"
|
||||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||||
"knowledge-governance/internal/platform/weknora"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrInvalidValidateJobPayload = errors.New("test_validate_invalid_payload")
|
||||
ErrValidateStateConflict = errors.New("test_validate_state_conflict")
|
||||
ErrValidateRecordFailed = errors.New("test_validate_record_failed")
|
||||
)
|
||||
|
||||
type ValidateStore interface {
|
||||
LoadPublicationForValidation(context.Context, string) (postgresstore.PublicationForValidation, error)
|
||||
AppendAcceptanceResult(context.Context, governance.AcceptanceResult) error
|
||||
}
|
||||
|
||||
type ValidateClient interface {
|
||||
GetKnowledge(context.Context, string) (weknora.Knowledge, error)
|
||||
ListChunks(context.Context, string, int, int, []string) ([]weknora.Chunk, int64, error)
|
||||
}
|
||||
|
||||
type ValidateHandlerConfig struct {
|
||||
Store ValidateStore
|
||||
Client ValidateClient
|
||||
RunID string
|
||||
PolicyVersion string
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
type ValidateHandler struct {
|
||||
store ValidateStore
|
||||
client ValidateClient
|
||||
runID string
|
||||
policyVersion string
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
type validatePayload struct {
|
||||
PublicationID string `json:"publication_id"`
|
||||
}
|
||||
|
||||
func NewValidateHandler(config ValidateHandlerConfig) (*ValidateHandler, error) {
|
||||
if config.Store == nil || config.Client == nil || config.PolicyVersion == "" {
|
||||
return nil, errors.New("invalid test validate handler configuration")
|
||||
}
|
||||
if config.RunID == "" {
|
||||
config.RunID = "default"
|
||||
}
|
||||
if config.Now == nil {
|
||||
config.Now = time.Now
|
||||
}
|
||||
return &ValidateHandler{
|
||||
store: config.Store, client: config.Client, runID: config.RunID,
|
||||
policyVersion: config.PolicyVersion, now: config.Now,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (handler *ValidateHandler) Handle(ctx context.Context, job jobs.Job) error {
|
||||
var payload validatePayload
|
||||
if err := decodeStrictJSON(job.Payload, &payload); err != nil || payload.PublicationID == "" {
|
||||
return ErrInvalidValidateJobPayload
|
||||
}
|
||||
publication, err := handler.store.LoadPublicationForValidation(ctx, payload.PublicationID)
|
||||
if err != nil {
|
||||
return ErrValidateStateConflict
|
||||
}
|
||||
now := handler.now()
|
||||
runID := handler.runID
|
||||
if job.ID != "" {
|
||||
runID = job.ID
|
||||
}
|
||||
|
||||
knowledge, knowledgeErr := handler.client.GetKnowledge(ctx, publication.KnowledgeID)
|
||||
knowledgeState := governance.AcceptancePassed
|
||||
knowledgeEvidence := json.RawMessage(`{"matched":true}`)
|
||||
if knowledgeErr != nil || knowledge.ID != publication.KnowledgeID || knowledge.KnowledgeBaseID != publication.TestKBID {
|
||||
knowledgeState = governance.AcceptanceFailed
|
||||
knowledgeEvidence = json.RawMessage(`{"matched":false}`)
|
||||
}
|
||||
if err := handler.append(ctx, publication.ID, runID, "knowledge_exists", knowledgeState, knowledgeEvidence, now); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, chunkTotal, chunksErr := handler.client.ListChunks(ctx, publication.KnowledgeID, 1, 1, nil)
|
||||
chunksState := governance.AcceptancePassed
|
||||
if chunksErr != nil || chunkTotal <= 0 {
|
||||
chunksState = governance.AcceptanceFailed
|
||||
}
|
||||
if err := handler.append(ctx, publication.ID, runID, "chunks_present", chunksState, mustJSON(map[string]any{"chunk_total": chunkTotal}), now); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := handler.append(ctx, publication.ID, runID, "retrieval_hit", governance.AcceptanceManualRequired, json.RawMessage(`{"reason":"thresholds_unresolved"}`), now); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (handler *ValidateHandler) append(ctx context.Context, publicationID, runID, checkType string, state governance.AcceptanceState, evidence json.RawMessage, observedAt time.Time) error {
|
||||
result := governance.AcceptanceResult{
|
||||
ID: acceptanceResultID(publicationID, runID, checkType),
|
||||
PublicationID: publicationID,
|
||||
RunID: runID,
|
||||
CheckType: checkType,
|
||||
State: state,
|
||||
PolicyVersion: handler.policyVersion,
|
||||
EvidenceSummary: evidence,
|
||||
ObservedAt: observedAt,
|
||||
}
|
||||
if err := handler.store.AppendAcceptanceResult(ctx, result); err != nil {
|
||||
return ErrValidateRecordFailed
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func acceptanceResultID(publicationID, runID, checkType string) string {
|
||||
digest := sha256.New()
|
||||
_, _ = digest.Write([]byte(publicationID))
|
||||
_, _ = digest.Write([]byte{0})
|
||||
_, _ = digest.Write([]byte(runID))
|
||||
_, _ = digest.Write([]byte{0})
|
||||
_, _ = digest.Write([]byte(checkType))
|
||||
return "acc-" + hex.EncodeToString(digest.Sum(nil))[:32]
|
||||
}
|
||||
|
||||
func mustJSON(value any) json.RawMessage {
|
||||
data, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
return json.RawMessage(`{}`)
|
||||
}
|
||||
return data
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
package testweknora
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
"knowledge-governance/internal/jobs"
|
||||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||||
"knowledge-governance/internal/platform/weknora"
|
||||
)
|
||||
|
||||
func TestValidateHandlerAppendsDeterministicChecksAndManualRetrieval(t *testing.T) {
|
||||
client := &validateClientStub{
|
||||
knowledge: weknora.Knowledge{ID: "knowledge-1", KnowledgeBaseID: "kb-1"},
|
||||
chunkTotal: 3,
|
||||
}
|
||||
store := &validateStoreStub{publication: postgresstore.PublicationForValidation{
|
||||
ID: "publication-1", TestKBID: "kb-1", KnowledgeID: "knowledge-1",
|
||||
}}
|
||||
now := time.Date(2026, time.September, 4, 4, 10, 0, 0, time.UTC)
|
||||
handler, err := NewValidateHandler(ValidateHandlerConfig{
|
||||
Store: store, Client: client, RunID: "run-1", PolicyVersion: "structural-v1",
|
||||
Now: func() time.Time { return now },
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := handler.Handle(context.Background(), jobs.Job{Payload: []byte(`{"publication_id":"publication-1"}`)}); err != nil {
|
||||
t.Fatalf("Handle() error = %v", err)
|
||||
}
|
||||
if len(store.appended) != 3 {
|
||||
t.Fatalf("appended = %d, want 3: %#v", len(store.appended), store.appended)
|
||||
}
|
||||
byType := map[string]governance.AcceptanceResult{}
|
||||
for _, result := range store.appended {
|
||||
byType[result.CheckType] = result
|
||||
}
|
||||
if byType["knowledge_exists"].State != governance.AcceptancePassed {
|
||||
t.Fatalf("knowledge_exists = %#v", byType["knowledge_exists"])
|
||||
}
|
||||
if byType["chunks_present"].State != governance.AcceptancePassed {
|
||||
t.Fatalf("chunks_present = %#v", byType["chunks_present"])
|
||||
}
|
||||
if byType["retrieval_hit"].State != governance.AcceptanceManualRequired {
|
||||
t.Fatalf("retrieval_hit = %#v", byType["retrieval_hit"])
|
||||
}
|
||||
for _, result := range store.appended {
|
||||
if result.PublicationID != "publication-1" || result.RunID != "run-1" || result.ObservedAt != now {
|
||||
t.Fatalf("acceptance metadata = %#v", result)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateHandlerFailsChecksWithoutAborting(t *testing.T) {
|
||||
client := &validateClientStub{
|
||||
knowledge: weknora.Knowledge{ID: "knowledge-mismatch", KnowledgeBaseID: "kb-other"},
|
||||
chunkTotal: 0,
|
||||
}
|
||||
store := &validateStoreStub{publication: postgresstore.PublicationForValidation{
|
||||
ID: "publication-1", TestKBID: "kb-1", KnowledgeID: "knowledge-1",
|
||||
}}
|
||||
handler, err := NewValidateHandler(ValidateHandlerConfig{
|
||||
Store: store, Client: client, RunID: "run-2", PolicyVersion: "structural-v1",
|
||||
Now: func() time.Time { return time.Date(2026, time.September, 4, 4, 11, 0, 0, time.UTC) },
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := handler.Handle(context.Background(), jobs.Job{Payload: []byte(`{"publication_id":"publication-1"}`)}); err != nil {
|
||||
t.Fatalf("Handle() error = %v", err)
|
||||
}
|
||||
if len(store.appended) != 3 {
|
||||
t.Fatalf("appended = %d, want 3", len(store.appended))
|
||||
}
|
||||
byType := map[string]governance.AcceptanceResult{}
|
||||
for _, result := range store.appended {
|
||||
byType[result.CheckType] = result
|
||||
}
|
||||
if byType["knowledge_exists"].State != governance.AcceptanceFailed || byType["chunks_present"].State != governance.AcceptanceFailed {
|
||||
t.Fatalf("failed checks = %#v", store.appended)
|
||||
}
|
||||
if byType["retrieval_hit"].State != governance.AcceptanceManualRequired {
|
||||
t.Fatalf("retrieval still manual = %#v", byType["retrieval_hit"])
|
||||
}
|
||||
}
|
||||
|
||||
type validateStoreStub struct {
|
||||
publication postgresstore.PublicationForValidation
|
||||
appended []governance.AcceptanceResult
|
||||
}
|
||||
|
||||
func (store *validateStoreStub) LoadPublicationForValidation(context.Context, string) (postgresstore.PublicationForValidation, error) {
|
||||
return store.publication, nil
|
||||
}
|
||||
|
||||
func (store *validateStoreStub) AppendAcceptanceResult(_ context.Context, result governance.AcceptanceResult) error {
|
||||
store.appended = append(store.appended, result)
|
||||
return nil
|
||||
}
|
||||
|
||||
type validateClientStub struct {
|
||||
knowledge weknora.Knowledge
|
||||
chunkTotal int64
|
||||
}
|
||||
|
||||
func (client *validateClientStub) GetKnowledge(context.Context, string) (weknora.Knowledge, error) {
|
||||
return client.knowledge, nil
|
||||
}
|
||||
|
||||
func (client *validateClientStub) ListChunks(context.Context, string, int, int, []string) ([]weknora.Chunk, int64, error) {
|
||||
chunks := make([]weknora.Chunk, 0, client.chunkTotal)
|
||||
for i := int64(0); i < client.chunkTotal; i++ {
|
||||
chunks = append(chunks, weknora.Chunk{ID: "chunk", KnowledgeID: "knowledge-1"})
|
||||
}
|
||||
return chunks, client.chunkTotal, nil
|
||||
}
|
||||
Reference in New Issue
Block a user