fix(weknora): 同内容重复上传改为认领而非死信

This commit is contained in:
姚章浩
2026-09-15 15:02:13 +08:00
parent e8dae5c7f7
commit 40d0c511d6
2 changed files with 71 additions and 2 deletions
+19 -2
View File
@@ -171,6 +171,7 @@ func (handler *UploadHandler) Handle(ctx context.Context, job jobs.Job) error {
func (handler *UploadHandler) recoverAmbiguousUpload(ctx context.Context, candidate postgresstore.PublicationForUpload) (weknora.Knowledge, error) {
const pageSize = 100
matches := make([]weknora.Knowledge, 0, 2)
sameContent := make([]weknora.Knowledge, 0, 2)
for page := 1; page <= 100; page++ {
items, total, err := handler.client.ListKnowledge(ctx, candidate.TestKBID, page, pageSize)
if err != nil {
@@ -179,11 +180,20 @@ func (handler *UploadHandler) recoverAmbiguousUpload(ctx context.Context, candid
for _, item := range items {
publicationID, publicationOK := metadataString(item.Metadata, "publication_id")
contentSHA, contentOK := metadataString(item.Metadata, "content_sha256")
if item.KnowledgeBaseID == candidate.TestKBID && publicationOK && contentOK && publicationID == candidate.ID && contentSHA == candidate.SHA256 {
if item.KnowledgeBaseID != candidate.TestKBID || !contentOK || contentSHA != candidate.SHA256 {
continue
}
if publicationOK && publicationID == candidate.ID {
matches = append(matches, item)
if len(matches) > 1 {
return weknora.Knowledge{}, ErrUploadRemoteIdentity
}
} else {
// Same bytes already delivered under a SIBLING publication (the
// same file exists in two source folders). A unique match here is
// safe to adopt: WeKnora dedupes by content and would reject our
// own upload forever otherwise.
sameContent = append(sameContent, item)
}
}
if int64(page*pageSize) >= total {
@@ -193,6 +203,9 @@ func (handler *UploadHandler) recoverAmbiguousUpload(ctx context.Context, candid
return weknora.Knowledge{}, ErrUploadRemoteFailed
}
}
if len(matches) == 0 && len(sameContent) == 1 {
matches = sameContent
}
if len(matches) != 1 {
return weknora.Knowledge{}, ErrUploadRemoteFailed
}
@@ -202,8 +215,12 @@ func (handler *UploadHandler) recoverAmbiguousUpload(ctx context.Context, candid
}
publicationID, publicationOK := metadataString(verified.Metadata, "publication_id")
contentSHA, contentOK := metadataString(verified.Metadata, "content_sha256")
// The identity anchor is the content hash: it matches either our own
// recovery (same publication_id) or a sibling's adoption (duplicate file
// across folders). publication_id is checked only when it is ours.
if verified.ID != matches[0].ID || verified.KnowledgeBaseID != candidate.TestKBID ||
!publicationOK || !contentOK || publicationID != candidate.ID || contentSHA != candidate.SHA256 {
!contentOK || contentSHA != candidate.SHA256 ||
(publicationOK && publicationID != candidate.ID && len(sameContent) > 1) {
return weknora.Knowledge{}, ErrUploadRemoteIdentity
}
return verified, nil
@@ -402,3 +402,55 @@ func TestUploadHandlerAdoptsSameSHAItemOnDuplicateConflict(t *testing.T) {
t.Fatalf("knowledge=%q", store.recordedKnowledgeID)
}
}
func TestUploadHandlerAdoptsUniqueSameSHAItemWhenOwnRecoveryFails(t *testing.T) {
root := t.TempDir()
if err := os.Mkdir(filepath.Join(root, "objects"), 0o700); err != nil {
t.Fatal(err)
}
body := []byte("uploading-state duplicate\n")
if err := os.WriteFile(filepath.Join(root, "objects", "artifact.md"), body, 0o600); err != nil {
t.Fatal(err)
}
digest := sha256.Sum256(body)
sha := hex.EncodeToString(digest[:])
var uploads atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch {
case r.Method == http.MethodPost:
uploads.Add(1)
http.Error(w, "must not upload while recovering", http.StatusInternalServerError)
case r.Method == http.MethodGet && r.URL.Path == "/api/v1/knowledge-bases/kb-1/knowledge":
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"success":true,"data":[{"id":"knowledge-sibling","knowledge_base_id":"kb-1","metadata":{"publication_id":"publication-sibling","content_sha256":"` + sha + `"}}],"total":1,"page":1,"page_size":100}`))
case r.Method == http.MethodGet && r.URL.Path == "/api/v1/knowledge/knowledge-sibling":
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"success":true,"data":{"id":"knowledge-sibling","knowledge_base_id":"kb-1","metadata":{"publication_id":"publication-sibling","content_sha256":"` + sha + `"}}}`))
default:
http.NotFound(w, r)
}
}))
defer server.Close()
client, err := weknora.NewClient(weknora.ClientConfig{BaseURL: server.URL, APIKey: "test-key", Timeout: time.Second})
if err != nil {
t.Fatal(err)
}
store := &uploadStoreStub{candidate: postgresstore.PublicationForUpload{
ID: "publication-1", TestKBID: "kb-1", IngestState: governance.IngestUploading,
ProcessingState: governance.ProcessingPending, SHA256: sha, SizeBytes: int64(len(body)),
ValidatedFormat: "md", LocalSpoolRef: "objects/artifact.md", Title: "Safe title",
}}
handler, err := NewUploadHandler(UploadHandlerConfig{
Store: store, Client: client, SpoolRoot: root, MaxUploadBytes: 50 * 1024 * 1024,
FirstPollDelay: time.Second, Now: func() time.Time { return time.Now() },
})
if err != nil {
t.Fatal(err)
}
if err := handler.Handle(context.Background(), jobs.Job{Payload: []byte(`{"publication_id":"publication-1"}`)}); err != nil {
t.Fatalf("Handle() error = %v, want sibling adoption", err)
}
if uploads.Load() != 0 || store.recordedKnowledgeID != "knowledge-sibling" {
t.Fatalf("uploads=%d knowledge=%q", uploads.Load(), store.recordedKnowledgeID)
}
}