姚章浩
281 lines
9.9 KiB
Go
281 lines
9.9 KiB
Go
package testweknora
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"io"
|
|
"path/filepath"
|
|
"strings"
|
|
"time"
|
|
|
|
"knowledge-governance/internal/governance"
|
|
"knowledge-governance/internal/jobs"
|
|
postgresstore "knowledge-governance/internal/platform/postgres"
|
|
"knowledge-governance/internal/platform/spool"
|
|
"knowledge-governance/internal/platform/weknora"
|
|
)
|
|
|
|
var (
|
|
ErrInvalidUploadJobPayload = errors.New("test_upload_invalid_payload")
|
|
ErrUploadArtifactInvalid = errors.New("test_upload_artifact_invalid")
|
|
ErrUploadRemoteFailed = errors.New("test_upload_remote_failed")
|
|
ErrUploadRemoteIdentity = errors.New("test_upload_remote_identity_invalid")
|
|
ErrUploadStateConflict = errors.New("test_upload_state_conflict")
|
|
)
|
|
|
|
type UploadStore interface {
|
|
LoadPublicationForUpload(context.Context, string) (postgresstore.PublicationForUpload, error)
|
|
MarkUploading(context.Context, string, time.Time) error
|
|
RecordUploadAndSchedulePoll(context.Context, string, string, time.Time, time.Time) (int, error)
|
|
RecordUploadFailure(context.Context, string, string, time.Time) error
|
|
}
|
|
|
|
type UploadClient interface {
|
|
UploadFile(context.Context, weknora.UploadRequest) (weknora.Knowledge, error)
|
|
ListKnowledge(context.Context, string, int, int) ([]weknora.Knowledge, int64, error)
|
|
GetKnowledge(context.Context, string) (weknora.Knowledge, error)
|
|
}
|
|
|
|
type UploadHandlerConfig struct {
|
|
Store UploadStore
|
|
Client UploadClient
|
|
SpoolRoot string
|
|
MaxUploadBytes int64
|
|
FirstPollDelay time.Duration
|
|
Now func() time.Time
|
|
}
|
|
|
|
type UploadHandler struct {
|
|
store UploadStore
|
|
client UploadClient
|
|
spoolRoot string
|
|
maxUploadBytes int64
|
|
firstPollDelay time.Duration
|
|
now func() time.Time
|
|
}
|
|
|
|
type uploadPayload struct {
|
|
PublicationID string `json:"publication_id"`
|
|
TargetScope string `json:"target_scope,omitempty"`
|
|
TargetKBID string `json:"target_kb_id,omitempty"`
|
|
}
|
|
|
|
func NewUploadHandler(config UploadHandlerConfig) (*UploadHandler, error) {
|
|
if config.Store == nil || config.Client == nil || !filepath.IsAbs(config.SpoolRoot) ||
|
|
config.MaxUploadBytes <= 0 || config.FirstPollDelay <= 0 {
|
|
return nil, errors.New("invalid test upload handler configuration")
|
|
}
|
|
if config.Now == nil {
|
|
config.Now = time.Now
|
|
}
|
|
return &UploadHandler{
|
|
store: config.Store, client: config.Client, spoolRoot: config.SpoolRoot,
|
|
maxUploadBytes: config.MaxUploadBytes, firstPollDelay: config.FirstPollDelay, now: config.Now,
|
|
}, nil
|
|
}
|
|
|
|
func (handler *UploadHandler) Handle(ctx context.Context, job jobs.Job) error {
|
|
var payload uploadPayload
|
|
if err := decodeStrictJSON(job.Payload, &payload); err != nil || payload.PublicationID == "" {
|
|
return jobs.Permanent(jobs.ErrCodeHandlerFallback, ErrInvalidUploadJobPayload)
|
|
}
|
|
candidate, err := handler.store.LoadPublicationForUpload(ctx, payload.PublicationID)
|
|
if err != nil {
|
|
return ErrUploadStateConflict
|
|
}
|
|
// P4: the job snapshots its target; a mismatch against the stored publication
|
|
// means the plan changed underneath and this delivery must not guess.
|
|
if payload.TargetKBID != "" && payload.TargetKBID != candidate.TestKBID {
|
|
return ErrUploadStateConflict
|
|
}
|
|
if payload.TargetScope != "" && payload.TargetScope != string(candidate.TargetScope) {
|
|
return ErrUploadStateConflict
|
|
}
|
|
if candidate.IngestState == governance.IngestUploading {
|
|
knowledge, err := handler.recoverAmbiguousUpload(ctx, candidate)
|
|
if err != nil {
|
|
if errors.Is(err, ErrUploadRemoteIdentity) {
|
|
return handler.recordTerminalUploadFailure(ctx, candidate.ID, "upload_identity_ambiguous")
|
|
}
|
|
return err
|
|
}
|
|
now := handler.now()
|
|
if _, err := handler.store.RecordUploadAndSchedulePoll(ctx, candidate.ID, knowledge.ID, now, now.Add(handler.firstPollDelay)); err != nil {
|
|
return ErrUploadStateConflict
|
|
}
|
|
return nil
|
|
}
|
|
artifact, err := spool.OpenVerified(handler.spoolRoot, spool.Expectation{
|
|
RelativeRef: candidate.LocalSpoolRef,
|
|
SHA256: candidate.SHA256,
|
|
SizeBytes: candidate.SizeBytes,
|
|
Format: candidate.ValidatedFormat,
|
|
}, handler.maxUploadBytes)
|
|
if err != nil {
|
|
return handler.recordTerminalUploadFailure(ctx, candidate.ID, "upload_artifact_invalid")
|
|
}
|
|
defer artifact.Close()
|
|
if err := handler.store.MarkUploading(ctx, candidate.ID, handler.now()); err != nil {
|
|
return ErrUploadStateConflict
|
|
}
|
|
knowledge, uploadErr := handler.client.UploadFile(ctx, weknora.UploadRequest{
|
|
KnowledgeBaseID: candidate.TestKBID,
|
|
FileName: safeUploadFileName(candidate.Title, candidate.ID, candidate.ValidatedFormat),
|
|
Content: artifact,
|
|
Metadata: map[string]string{
|
|
"content_sha256": candidate.SHA256,
|
|
"publication_id": candidate.ID,
|
|
},
|
|
Channel: "governance",
|
|
})
|
|
if err := artifact.VerifyComplete(); err != nil {
|
|
return jobs.Permanent(jobs.ErrCodeArtifactInvalid, ErrUploadArtifactInvalid)
|
|
}
|
|
if uploadErr != nil {
|
|
if errors.Is(uploadErr, weknora.ErrConflict) && knowledge.ID != "" {
|
|
// WeKnora dedupes identical file bytes across publications: when the
|
|
// same document was already uploaded under a sibling publication (the
|
|
// same file living in two source folders), the conflict response names
|
|
// the existing item. Adopt it instead of dead-lettering forever.
|
|
if verified, verifyErr := handler.client.GetKnowledge(ctx, knowledge.ID); verifyErr == nil &&
|
|
verified.ID == knowledge.ID && verified.KnowledgeBaseID == candidate.TestKBID &&
|
|
shaMatches(verified, candidate.SHA256) {
|
|
knowledge = verified
|
|
} else {
|
|
return ErrUploadRemoteFailed
|
|
}
|
|
} else if !errors.Is(uploadErr, weknora.ErrConflict) && !weknora.IsRetryable(uploadErr) {
|
|
return ErrUploadRemoteFailed
|
|
} else {
|
|
knowledge, err = handler.recoverAmbiguousUpload(ctx, candidate)
|
|
if err != nil {
|
|
if errors.Is(err, ErrUploadRemoteIdentity) {
|
|
return handler.recordTerminalUploadFailure(ctx, candidate.ID, "upload_identity_ambiguous")
|
|
}
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
if knowledge.ID == "" || knowledge.KnowledgeBaseID != candidate.TestKBID {
|
|
return ErrUploadRemoteIdentity
|
|
}
|
|
now := handler.now()
|
|
if _, err := handler.store.RecordUploadAndSchedulePoll(ctx, candidate.ID, knowledge.ID, now, now.Add(handler.firstPollDelay)); err != nil {
|
|
return ErrUploadStateConflict
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (handler *UploadHandler) recoverAmbiguousUpload(ctx context.Context, candidate postgresstore.PublicationForUpload) (weknora.Knowledge, error) {
|
|
const pageSize = 100
|
|
matches := make([]weknora.Knowledge, 0, 2)
|
|
sameContent := make([]weknora.Knowledge, 0, 2)
|
|
for page := 1; page <= 100; page++ {
|
|
items, total, err := handler.client.ListKnowledge(ctx, candidate.TestKBID, page, pageSize)
|
|
if err != nil {
|
|
return weknora.Knowledge{}, ErrUploadRemoteFailed
|
|
}
|
|
for _, item := range items {
|
|
publicationID, publicationOK := metadataString(item.Metadata, "publication_id")
|
|
contentSHA, contentOK := metadataString(item.Metadata, "content_sha256")
|
|
if item.KnowledgeBaseID != candidate.TestKBID || !contentOK || contentSHA != candidate.SHA256 {
|
|
continue
|
|
}
|
|
if publicationOK && publicationID == candidate.ID {
|
|
matches = append(matches, item)
|
|
if len(matches) > 1 {
|
|
return weknora.Knowledge{}, ErrUploadRemoteIdentity
|
|
}
|
|
} else {
|
|
// Same bytes already delivered under a SIBLING publication (the
|
|
// same file exists in two source folders). A unique match here is
|
|
// safe to adopt: WeKnora dedupes by content and would reject our
|
|
// own upload forever otherwise.
|
|
sameContent = append(sameContent, item)
|
|
}
|
|
}
|
|
if int64(page*pageSize) >= total {
|
|
break
|
|
}
|
|
if len(items) == 0 {
|
|
return weknora.Knowledge{}, ErrUploadRemoteFailed
|
|
}
|
|
}
|
|
if len(matches) == 0 && len(sameContent) == 1 {
|
|
matches = sameContent
|
|
}
|
|
if len(matches) != 1 {
|
|
return weknora.Knowledge{}, ErrUploadRemoteFailed
|
|
}
|
|
verified, err := handler.client.GetKnowledge(ctx, matches[0].ID)
|
|
if err != nil {
|
|
return weknora.Knowledge{}, ErrUploadRemoteFailed
|
|
}
|
|
publicationID, publicationOK := metadataString(verified.Metadata, "publication_id")
|
|
contentSHA, contentOK := metadataString(verified.Metadata, "content_sha256")
|
|
// The identity anchor is the content hash: it matches either our own
|
|
// recovery (same publication_id) or a sibling's adoption (duplicate file
|
|
// across folders). publication_id is checked only when it is ours.
|
|
if verified.ID != matches[0].ID || verified.KnowledgeBaseID != candidate.TestKBID ||
|
|
!contentOK || contentSHA != candidate.SHA256 ||
|
|
(publicationOK && publicationID != candidate.ID && len(sameContent) > 1) {
|
|
return weknora.Knowledge{}, ErrUploadRemoteIdentity
|
|
}
|
|
return verified, nil
|
|
}
|
|
|
|
func metadataString(metadata map[string]any, key string) (string, bool) {
|
|
value, ok := metadata[key].(string)
|
|
return value, ok
|
|
}
|
|
|
|
// shaMatches reports whether a remote item carries our content hash, either in
|
|
// its metadata (uploads through governance) or as its file hash.
|
|
func shaMatches(item weknora.Knowledge, sha string) bool {
|
|
if value, ok := metadataString(item.Metadata, "content_sha256"); ok && value == sha {
|
|
return true
|
|
}
|
|
return sha != "" && strings.EqualFold(item.FileHash, sha)
|
|
}
|
|
|
|
func (handler *UploadHandler) recordTerminalUploadFailure(ctx context.Context, publicationID, code string) error {
|
|
if err := handler.store.RecordUploadFailure(ctx, publicationID, code, handler.now()); err != nil {
|
|
return ErrUploadStateConflict
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func decodeStrictJSON(data []byte, target any) error {
|
|
decoder := json.NewDecoder(bytes.NewReader(data))
|
|
decoder.DisallowUnknownFields()
|
|
if err := decoder.Decode(target); err != nil {
|
|
return err
|
|
}
|
|
if err := decoder.Decode(&struct{}{}); err != io.EOF {
|
|
return errors.New("extra JSON value")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func safeUploadFileName(title, fallback, format string) string {
|
|
base := strings.TrimSpace(title)
|
|
base = strings.Map(func(character rune) rune {
|
|
switch character {
|
|
case '/', '\\', '\x00', '\r', '\n', '\t':
|
|
return '_'
|
|
default:
|
|
return character
|
|
}
|
|
}, base)
|
|
base = strings.Trim(base, ". ")
|
|
if base == "" {
|
|
base = fallback
|
|
}
|
|
if len(base) > 120 {
|
|
base = base[:120]
|
|
}
|
|
return base + "." + format
|
|
}
|