package testweknora import ( "bytes" "context" "encoding/json" "errors" "io" "path/filepath" "strings" "time" "knowledge-governance/internal/governance" "knowledge-governance/internal/jobs" postgresstore "knowledge-governance/internal/platform/postgres" "knowledge-governance/internal/platform/spool" "knowledge-governance/internal/platform/weknora" ) var ( ErrInvalidUploadJobPayload = errors.New("test_upload_invalid_payload") ErrUploadArtifactInvalid = errors.New("test_upload_artifact_invalid") ErrUploadRemoteFailed = errors.New("test_upload_remote_failed") ErrUploadRemoteIdentity = errors.New("test_upload_remote_identity_invalid") ErrUploadStateConflict = errors.New("test_upload_state_conflict") ) type UploadStore interface { LoadPublicationForUpload(context.Context, string) (postgresstore.PublicationForUpload, error) MarkUploading(context.Context, string, time.Time) error RecordUploadAndSchedulePoll(context.Context, string, string, time.Time, time.Time) (int, error) RecordUploadFailure(context.Context, string, string, time.Time) error } type UploadClient interface { UploadFile(context.Context, weknora.UploadRequest) (weknora.Knowledge, error) ListKnowledge(context.Context, string, int, int) ([]weknora.Knowledge, int64, error) GetKnowledge(context.Context, string) (weknora.Knowledge, error) } type UploadHandlerConfig struct { Store UploadStore Client UploadClient SpoolRoot string MaxUploadBytes int64 FirstPollDelay time.Duration Now func() time.Time } type UploadHandler struct { store UploadStore client UploadClient spoolRoot string maxUploadBytes int64 firstPollDelay time.Duration now func() time.Time } type uploadPayload struct { PublicationID string `json:"publication_id"` TargetScope string `json:"target_scope,omitempty"` TargetKBID string `json:"target_kb_id,omitempty"` } func NewUploadHandler(config UploadHandlerConfig) (*UploadHandler, error) { if config.Store == nil || config.Client == nil || !filepath.IsAbs(config.SpoolRoot) || config.MaxUploadBytes <= 0 || config.FirstPollDelay <= 0 { return nil, errors.New("invalid test upload handler configuration") } if config.Now == nil { config.Now = time.Now } return &UploadHandler{ store: config.Store, client: config.Client, spoolRoot: config.SpoolRoot, maxUploadBytes: config.MaxUploadBytes, firstPollDelay: config.FirstPollDelay, now: config.Now, }, nil } func (handler *UploadHandler) Handle(ctx context.Context, job jobs.Job) error { var payload uploadPayload if err := decodeStrictJSON(job.Payload, &payload); err != nil || payload.PublicationID == "" { return jobs.Permanent(jobs.ErrCodeHandlerFallback, ErrInvalidUploadJobPayload) } candidate, err := handler.store.LoadPublicationForUpload(ctx, payload.PublicationID) if err != nil { return ErrUploadStateConflict } // P4: the job snapshots its target; a mismatch against the stored publication // means the plan changed underneath and this delivery must not guess. if payload.TargetKBID != "" && payload.TargetKBID != candidate.TestKBID { return ErrUploadStateConflict } if payload.TargetScope != "" && payload.TargetScope != string(candidate.TargetScope) { return ErrUploadStateConflict } if candidate.IngestState == governance.IngestUploading { knowledge, err := handler.recoverAmbiguousUpload(ctx, candidate) if err != nil { if errors.Is(err, ErrUploadRemoteIdentity) { return handler.recordTerminalUploadFailure(ctx, candidate.ID, "upload_identity_ambiguous") } return err } now := handler.now() if _, err := handler.store.RecordUploadAndSchedulePoll(ctx, candidate.ID, knowledge.ID, now, now.Add(handler.firstPollDelay)); err != nil { return ErrUploadStateConflict } return nil } artifact, err := spool.OpenVerified(handler.spoolRoot, spool.Expectation{ RelativeRef: candidate.LocalSpoolRef, SHA256: candidate.SHA256, SizeBytes: candidate.SizeBytes, Format: candidate.ValidatedFormat, }, handler.maxUploadBytes) if err != nil { return handler.recordTerminalUploadFailure(ctx, candidate.ID, "upload_artifact_invalid") } defer artifact.Close() if err := handler.store.MarkUploading(ctx, candidate.ID, handler.now()); err != nil { return ErrUploadStateConflict } knowledge, uploadErr := handler.client.UploadFile(ctx, weknora.UploadRequest{ KnowledgeBaseID: candidate.TestKBID, FileName: safeUploadFileName(candidate.Title, candidate.ID, candidate.ValidatedFormat), Content: artifact, Metadata: map[string]string{ "content_sha256": candidate.SHA256, "publication_id": candidate.ID, }, Channel: "governance", }) if err := artifact.VerifyComplete(); err != nil { return jobs.Permanent(jobs.ErrCodeArtifactInvalid, ErrUploadArtifactInvalid) } if uploadErr != nil { if errors.Is(uploadErr, weknora.ErrConflict) && knowledge.ID != "" { // WeKnora dedupes identical file bytes across publications: when the // same document was already uploaded under a sibling publication (the // same file living in two source folders), the conflict response names // the existing item. Adopt it instead of dead-lettering forever. if verified, verifyErr := handler.client.GetKnowledge(ctx, knowledge.ID); verifyErr == nil && verified.ID == knowledge.ID && verified.KnowledgeBaseID == candidate.TestKBID && shaMatches(verified, candidate.SHA256) { knowledge = verified } else { return ErrUploadRemoteFailed } } else if !errors.Is(uploadErr, weknora.ErrConflict) && !weknora.IsRetryable(uploadErr) { return ErrUploadRemoteFailed } else { knowledge, err = handler.recoverAmbiguousUpload(ctx, candidate) if err != nil { if errors.Is(err, ErrUploadRemoteIdentity) { return handler.recordTerminalUploadFailure(ctx, candidate.ID, "upload_identity_ambiguous") } return err } } } if knowledge.ID == "" || knowledge.KnowledgeBaseID != candidate.TestKBID { return ErrUploadRemoteIdentity } now := handler.now() if _, err := handler.store.RecordUploadAndSchedulePoll(ctx, candidate.ID, knowledge.ID, now, now.Add(handler.firstPollDelay)); err != nil { return ErrUploadStateConflict } return nil } func (handler *UploadHandler) recoverAmbiguousUpload(ctx context.Context, candidate postgresstore.PublicationForUpload) (weknora.Knowledge, error) { const pageSize = 100 matches := make([]weknora.Knowledge, 0, 2) sameContent := make([]weknora.Knowledge, 0, 2) for page := 1; page <= 100; page++ { items, total, err := handler.client.ListKnowledge(ctx, candidate.TestKBID, page, pageSize) if err != nil { return weknora.Knowledge{}, ErrUploadRemoteFailed } for _, item := range items { publicationID, publicationOK := metadataString(item.Metadata, "publication_id") contentSHA, contentOK := metadataString(item.Metadata, "content_sha256") if item.KnowledgeBaseID != candidate.TestKBID || !contentOK || contentSHA != candidate.SHA256 { continue } if publicationOK && publicationID == candidate.ID { matches = append(matches, item) if len(matches) > 1 { return weknora.Knowledge{}, ErrUploadRemoteIdentity } } else { // Same bytes already delivered under a SIBLING publication (the // same file exists in two source folders). A unique match here is // safe to adopt: WeKnora dedupes by content and would reject our // own upload forever otherwise. sameContent = append(sameContent, item) } } if int64(page*pageSize) >= total { break } if len(items) == 0 { return weknora.Knowledge{}, ErrUploadRemoteFailed } } if len(matches) == 0 && len(sameContent) == 1 { matches = sameContent } if len(matches) != 1 { return weknora.Knowledge{}, ErrUploadRemoteFailed } verified, err := handler.client.GetKnowledge(ctx, matches[0].ID) if err != nil { return weknora.Knowledge{}, ErrUploadRemoteFailed } publicationID, publicationOK := metadataString(verified.Metadata, "publication_id") contentSHA, contentOK := metadataString(verified.Metadata, "content_sha256") // The identity anchor is the content hash: it matches either our own // recovery (same publication_id) or a sibling's adoption (duplicate file // across folders). publication_id is checked only when it is ours. if verified.ID != matches[0].ID || verified.KnowledgeBaseID != candidate.TestKBID || !contentOK || contentSHA != candidate.SHA256 || (publicationOK && publicationID != candidate.ID && len(sameContent) > 1) { return weknora.Knowledge{}, ErrUploadRemoteIdentity } return verified, nil } func metadataString(metadata map[string]any, key string) (string, bool) { value, ok := metadata[key].(string) return value, ok } // shaMatches reports whether a remote item carries our content hash, either in // its metadata (uploads through governance) or as its file hash. func shaMatches(item weknora.Knowledge, sha string) bool { if value, ok := metadataString(item.Metadata, "content_sha256"); ok && value == sha { return true } return sha != "" && strings.EqualFold(item.FileHash, sha) } func (handler *UploadHandler) recordTerminalUploadFailure(ctx context.Context, publicationID, code string) error { if err := handler.store.RecordUploadFailure(ctx, publicationID, code, handler.now()); err != nil { return ErrUploadStateConflict } return nil } func decodeStrictJSON(data []byte, target any) error { decoder := json.NewDecoder(bytes.NewReader(data)) decoder.DisallowUnknownFields() if err := decoder.Decode(target); err != nil { return err } if err := decoder.Decode(&struct{}{}); err != io.EOF { return errors.New("extra JSON value") } return nil } func safeUploadFileName(title, fallback, format string) string { base := strings.TrimSpace(title) base = strings.Map(func(character rune) rune { switch character { case '/', '\\', '\x00', '\r', '\n', '\t': return '_' default: return character } }, base) base = strings.Trim(base, ". ") if base == "" { base = fallback } if len(base) > 120 { base = base[:120] } return base + "." + format }