fix(weknora): 同内容重复上传改为认领而非死信
This commit is contained in:
@@ -171,6 +171,7 @@ func (handler *UploadHandler) Handle(ctx context.Context, job jobs.Job) error {
|
||||
func (handler *UploadHandler) recoverAmbiguousUpload(ctx context.Context, candidate postgresstore.PublicationForUpload) (weknora.Knowledge, error) {
|
||||
const pageSize = 100
|
||||
matches := make([]weknora.Knowledge, 0, 2)
|
||||
sameContent := make([]weknora.Knowledge, 0, 2)
|
||||
for page := 1; page <= 100; page++ {
|
||||
items, total, err := handler.client.ListKnowledge(ctx, candidate.TestKBID, page, pageSize)
|
||||
if err != nil {
|
||||
@@ -179,11 +180,20 @@ func (handler *UploadHandler) recoverAmbiguousUpload(ctx context.Context, candid
|
||||
for _, item := range items {
|
||||
publicationID, publicationOK := metadataString(item.Metadata, "publication_id")
|
||||
contentSHA, contentOK := metadataString(item.Metadata, "content_sha256")
|
||||
if item.KnowledgeBaseID == candidate.TestKBID && publicationOK && contentOK && publicationID == candidate.ID && contentSHA == candidate.SHA256 {
|
||||
if item.KnowledgeBaseID != candidate.TestKBID || !contentOK || contentSHA != candidate.SHA256 {
|
||||
continue
|
||||
}
|
||||
if publicationOK && publicationID == candidate.ID {
|
||||
matches = append(matches, item)
|
||||
if len(matches) > 1 {
|
||||
return weknora.Knowledge{}, ErrUploadRemoteIdentity
|
||||
}
|
||||
} else {
|
||||
// Same bytes already delivered under a SIBLING publication (the
|
||||
// same file exists in two source folders). A unique match here is
|
||||
// safe to adopt: WeKnora dedupes by content and would reject our
|
||||
// own upload forever otherwise.
|
||||
sameContent = append(sameContent, item)
|
||||
}
|
||||
}
|
||||
if int64(page*pageSize) >= total {
|
||||
@@ -193,6 +203,9 @@ func (handler *UploadHandler) recoverAmbiguousUpload(ctx context.Context, candid
|
||||
return weknora.Knowledge{}, ErrUploadRemoteFailed
|
||||
}
|
||||
}
|
||||
if len(matches) == 0 && len(sameContent) == 1 {
|
||||
matches = sameContent
|
||||
}
|
||||
if len(matches) != 1 {
|
||||
return weknora.Knowledge{}, ErrUploadRemoteFailed
|
||||
}
|
||||
@@ -202,8 +215,12 @@ func (handler *UploadHandler) recoverAmbiguousUpload(ctx context.Context, candid
|
||||
}
|
||||
publicationID, publicationOK := metadataString(verified.Metadata, "publication_id")
|
||||
contentSHA, contentOK := metadataString(verified.Metadata, "content_sha256")
|
||||
// The identity anchor is the content hash: it matches either our own
|
||||
// recovery (same publication_id) or a sibling's adoption (duplicate file
|
||||
// across folders). publication_id is checked only when it is ours.
|
||||
if verified.ID != matches[0].ID || verified.KnowledgeBaseID != candidate.TestKBID ||
|
||||
!publicationOK || !contentOK || publicationID != candidate.ID || contentSHA != candidate.SHA256 {
|
||||
!contentOK || contentSHA != candidate.SHA256 ||
|
||||
(publicationOK && publicationID != candidate.ID && len(sameContent) > 1) {
|
||||
return weknora.Knowledge{}, ErrUploadRemoteIdentity
|
||||
}
|
||||
return verified, nil
|
||||
|
||||
@@ -402,3 +402,55 @@ func TestUploadHandlerAdoptsSameSHAItemOnDuplicateConflict(t *testing.T) {
|
||||
t.Fatalf("knowledge=%q", store.recordedKnowledgeID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadHandlerAdoptsUniqueSameSHAItemWhenOwnRecoveryFails(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
if err := os.Mkdir(filepath.Join(root, "objects"), 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body := []byte("uploading-state duplicate\n")
|
||||
if err := os.WriteFile(filepath.Join(root, "objects", "artifact.md"), body, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
digest := sha256.Sum256(body)
|
||||
sha := hex.EncodeToString(digest[:])
|
||||
var uploads atomic.Int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch {
|
||||
case r.Method == http.MethodPost:
|
||||
uploads.Add(1)
|
||||
http.Error(w, "must not upload while recovering", http.StatusInternalServerError)
|
||||
case r.Method == http.MethodGet && r.URL.Path == "/api/v1/knowledge-bases/kb-1/knowledge":
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"success":true,"data":[{"id":"knowledge-sibling","knowledge_base_id":"kb-1","metadata":{"publication_id":"publication-sibling","content_sha256":"` + sha + `"}}],"total":1,"page":1,"page_size":100}`))
|
||||
case r.Method == http.MethodGet && r.URL.Path == "/api/v1/knowledge/knowledge-sibling":
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"success":true,"data":{"id":"knowledge-sibling","knowledge_base_id":"kb-1","metadata":{"publication_id":"publication-sibling","content_sha256":"` + sha + `"}}}`))
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
client, err := weknora.NewClient(weknora.ClientConfig{BaseURL: server.URL, APIKey: "test-key", Timeout: time.Second})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store := &uploadStoreStub{candidate: postgresstore.PublicationForUpload{
|
||||
ID: "publication-1", TestKBID: "kb-1", IngestState: governance.IngestUploading,
|
||||
ProcessingState: governance.ProcessingPending, SHA256: sha, SizeBytes: int64(len(body)),
|
||||
ValidatedFormat: "md", LocalSpoolRef: "objects/artifact.md", Title: "Safe title",
|
||||
}}
|
||||
handler, err := NewUploadHandler(UploadHandlerConfig{
|
||||
Store: store, Client: client, SpoolRoot: root, MaxUploadBytes: 50 * 1024 * 1024,
|
||||
FirstPollDelay: time.Second, Now: func() time.Time { return time.Now() },
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := handler.Handle(context.Background(), jobs.Job{Payload: []byte(`{"publication_id":"publication-1"}`)}); err != nil {
|
||||
t.Fatalf("Handle() error = %v, want sibling adoption", err)
|
||||
}
|
||||
if uploads.Load() != 0 || store.recordedKnowledgeID != "knowledge-sibling" {
|
||||
t.Fatalf("uploads=%d knowledge=%q", uploads.Load(), store.recordedKnowledgeID)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user