diff --git a/internal/datasource/connector/feishu/client.go b/internal/datasource/connector/feishu/client.go index 814ff8c5..2884f0f6 100644 --- a/internal/datasource/connector/feishu/client.go +++ b/internal/datasource/connector/feishu/client.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "net/http" + "strings" "sync" "time" @@ -27,6 +28,26 @@ type Client struct { tokenExpAt time.Time } +type wikiNodeListFailure struct { + Node wikiNode + Err error +} + +type partialWikiNodeListError struct { + Failures []wikiNodeListFailure +} + +func (e *partialWikiNodeListError) Error() string { + if e == nil || len(e.Failures) == 0 { + return "partial wiki node listing failed" + } + parts := make([]string, 0, len(e.Failures)) + for _, failure := range e.Failures { + parts = append(parts, failure.Err.Error()) + } + return strings.Join(parts, "; ") +} + // NewClient creates a new Feishu API client. func NewClient(config *Config) *Client { return &Client{ @@ -237,9 +258,10 @@ func (c *Client) ListAllWikiNodesRecursive(ctx context.Context, spaceID string) } var allNodes []wikiNode - var walk func(nodes []wikiNode) error + var failures []wikiNodeListFailure + var walk func(nodes []wikiNode) - walk = func(nodes []wikiNode) error { + walk = func(nodes []wikiNode) { for _, node := range nodes { allNodes = append(allNodes, node) @@ -247,18 +269,23 @@ func (c *Client) ListAllWikiNodesRecursive(ctx context.Context, spaceID string) if node.HasChild { children, err := c.ListWikiNodes(ctx, spaceID, node.NodeToken) if err != nil { - return fmt.Errorf("list children of %s: %w", node.NodeToken, err) - } - if err := walk(children); err != nil { - return err + wrappedErr := fmt.Errorf("list children of %s: %w", node.NodeToken, err) + failures = append(failures, wikiNodeListFailure{ + Node: node, + Err: wrappedErr, + }) + logger.Warnf(ctx, "[Feishu] partial wiki node listing failure: space=%s node=%s err=%v", + spaceID, node.NodeToken, err) + continue } + walk(children) } } - return nil } - if err := walk(topNodes); err != nil { - return nil, err + walk(topNodes) + if len(failures) > 0 { + return allNodes, &partialWikiNodeListError{Failures: failures} } return allNodes, nil diff --git a/internal/datasource/connector/feishu/connector.go b/internal/datasource/connector/feishu/connector.go index d49da72c..f5625786 100644 --- a/internal/datasource/connector/feishu/connector.go +++ b/internal/datasource/connector/feishu/connector.go @@ -3,6 +3,7 @@ package feishu import ( "context" "encoding/json" + "errors" "fmt" "strconv" "strings" @@ -85,7 +86,11 @@ func (c *Connector) FetchAll(ctx context.Context, config *types.DataSourceConfig // List all nodes in this wiki space recursively nodes, err := client.ListAllWikiNodesRecursive(ctx, spaceID) if err != nil { - return nil, fmt.Errorf("list nodes in space %s: %w", spaceID, err) + var partialErr *partialWikiNodeListError + if !errors.As(err, &partialErr) { + return nil, fmt.Errorf("list nodes in space %s: %w", spaceID, err) + } + allItems = appendWikiNodeListFailureItems(allItems, spaceID, partialErr.Failures) } // Fetch content for each document node @@ -146,11 +151,22 @@ func (c *Connector) FetchIncremental(ctx context.Context, config *types.DataSour for _, spaceID := range resourceIDs { // List all nodes in this space nodes, err := client.ListAllWikiNodesRecursive(ctx, spaceID) + var partialErr *partialWikiNodeListError if err != nil { - return nil, nil, fmt.Errorf("list nodes in space %s: %w", spaceID, err) + if !errors.As(err, &partialErr) { + return nil, nil, fmt.Errorf("list nodes in space %s: %w", spaceID, err) + } + changedItems = appendWikiNodeListFailureItems(changedItems, spaceID, partialErr.Failures) } newCursor.SpaceNodeTimes[spaceID] = make(map[string]string) + if partialErr != nil && prevCursor.SpaceNodeTimes != nil { + if prevTimes, ok := prevCursor.SpaceNodeTimes[spaceID]; ok { + for nodeToken, editTime := range prevTimes { + newCursor.SpaceNodeTimes[spaceID][nodeToken] = editTime + } + } + } // Build a set of current node tokens for deletion detection currentNodes := make(map[string]bool) @@ -197,7 +213,7 @@ func (c *Connector) FetchIncremental(ctx context.Context, config *types.DataSour } // Detect deleted nodes - if prevCursor.SpaceNodeTimes != nil { + if partialErr == nil && prevCursor.SpaceNodeTimes != nil { if prevTimes, ok := prevCursor.SpaceNodeTimes[spaceID]; ok { for nodeToken := range prevTimes { if !currentNodes[nodeToken] { @@ -226,6 +242,29 @@ func (c *Connector) FetchIncremental(ctx context.Context, config *types.DataSour return changedItems, nextSyncCursor, nil } +func appendWikiNodeListFailureItems(items []types.FetchedItem, spaceID string, failures []wikiNodeListFailure) []types.FetchedItem { + for _, failure := range failures { + node := failure.Node + title := node.Title + if title == "" { + title = node.NodeToken + } + items = append(items, types.FetchedItem{ + ExternalID: node.NodeToken, + Title: title, + SourceResourceID: spaceID, + Metadata: map[string]string{ + "error": failure.Err.Error(), + "channel": types.ChannelFeishu, + "node_token": node.NodeToken, + "space_id": spaceID, + "failure_stage": "list_children", + }, + }) + } + return items +} + // fetchNodeContent fetches the content of a single wiki node and converts it to FetchedItem. // Dispatches to different retrieval strategies based on obj_type: // - docx/doc → export API → .docx file diff --git a/internal/datasource/connector/feishu/connector_test.go b/internal/datasource/connector/feishu/connector_test.go index 751684bb..89deefdb 100644 --- a/internal/datasource/connector/feishu/connector_test.go +++ b/internal/datasource/connector/feishu/connector_test.go @@ -67,7 +67,9 @@ func fakeFeishu(nodes []wikiNode) (*httptest.Server, *Config) { if r.Method == http.MethodPost { writeJSON(w, exportTaskCreateResponse{ apiResponse: apiResponse{Code: 0}, - Data: struct{ Ticket string `json:"ticket"` }{Ticket: "ticket-123"}, + Data: struct { + Ticket string `json:"ticket"` + }{Ticket: "ticket-123"}, }) return } @@ -153,6 +155,62 @@ func fakeFeishu(nodes []wikiNode) (*httptest.Server, *Config) { return ts, cfg } +func fakeFeishuWithChildFailure(topNodes []wikiNode, failingParentToken string) (*httptest.Server, *Config) { + return fakeFeishuHierarchy(topNodes, nil, failingParentToken) +} + +func fakeFeishuHierarchy(topNodes []wikiNode, childNodes map[string][]wikiNode, failingParentToken string) (*httptest.Server, *Config) { + mux := http.NewServeMux() + + mux.HandleFunc("/open-apis/auth/v3/tenant_access_token/internal", func(w http.ResponseWriter, r *http.Request) { + writeJSON(w, tokenResponse{ + apiResponse: apiResponse{Code: 0}, + TenantAccessToken: "fake-token", + Expire: 7200, + }) + }) + + mux.HandleFunc("/open-apis/wiki/v2/spaces/space1/nodes", func(w http.ResponseWriter, r *http.Request) { + parentToken := r.URL.Query().Get("parent_node_token") + if parentToken != "" && parentToken == failingParentToken { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(`{"code":1663,"msg":"internal error"}`)) + return + } + nodes := topNodes + if parentToken != "" { + nodes = childNodes[parentToken] + } + writeJSON(w, wikiNodeListResponse{ + apiResponse: apiResponse{Code: 0}, + Data: struct { + Items []wikiNode `json:"items"` + HasMore bool `json:"has_more"` + PageToken string `json:"page_token"` + }{ + Items: nodes, + }, + }) + }) + + mux.HandleFunc("/open-apis/drive/v1/files/", func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "/download") { + w.Header().Set("Content-Type", "application/octet-stream") + _, _ = w.Write([]byte("fake-file-content")) + return + } + http.NotFound(w, r) + }) + + ts := httptest.NewServer(mux) + cfg := &Config{ + AppID: "test-app-id", + AppSecret: "test-app-secret", + BaseURL: ts.URL, + } + return ts, cfg +} + func writeJSON(w http.ResponseWriter, v interface{}) { w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(v) @@ -528,6 +586,55 @@ func TestFetchAll_MixedTypes(t *testing.T) { } } +func TestFetchAll_ChildNodeListErrorReturnsPartialItems(t *testing.T) { + nodes := []wikiNode{ + {NodeToken: "nt-parent", ObjToken: "obj-parent", ObjType: "file", Title: "Parent.pdf", NodeEditTime: "100", HasChild: true}, + {NodeToken: "nt-peer", ObjToken: "obj-peer", ObjType: "file", Title: "Peer.pdf", NodeEditTime: "200"}, + } + ts, cfg := fakeFeishuWithChildFailure(nodes, "nt-parent") + defer ts.Close() + + items, err := NewConnector().FetchAll(context.Background(), makeConfig(cfg, []string{"space1"}), []string{"space1"}) + if err != nil { + t.Fatalf("FetchAll must not abort when one child listing fails: %v", err) + } + + if len(items) != 3 { + t.Fatalf("want 3 items (2 fetched + 1 failure placeholder), got %d: %+v", len(items), items) + } + + var parent, peer, placeholder *types.FetchedItem + for i := range items { + switch { + case items[i].ExternalID == "nt-parent" && len(items[i].Content) > 0: + parent = &items[i] + case items[i].ExternalID == "nt-peer" && len(items[i].Content) > 0: + peer = &items[i] + case items[i].Metadata["error"] != "": + placeholder = &items[i] + } + } + + if parent == nil || peer == nil { + t.Fatalf("expected both successfully listed nodes to be fetched, parent=%v peer=%v", parent != nil, peer != nil) + } + if placeholder == nil { + t.Fatal("expected a failure placeholder for the child listing error") + } + if placeholder.ExternalID != "nt-parent" || placeholder.Title != "Parent.pdf" { + t.Errorf("placeholder identity wrong: %+v", placeholder) + } + if placeholder.Metadata["channel"] != types.ChannelFeishu { + t.Errorf("placeholder channel = %q", placeholder.Metadata["channel"]) + } + if placeholder.Metadata["node_token"] != "nt-parent" || placeholder.Metadata["space_id"] != "space1" { + t.Errorf("placeholder missing traceability metadata: %+v", placeholder.Metadata) + } + if !strings.Contains(placeholder.Metadata["error"], "list children of nt-parent") { + t.Errorf("placeholder error = %q", placeholder.Metadata["error"]) + } +} + // ────────────────────────────────────────────────────────────────────── // FetchIncremental tests // ────────────────────────────────────────────────────────────────────── @@ -648,6 +755,86 @@ func TestFetchIncremental_NoResourceIDs(t *testing.T) { } } +func TestFetchIncremental_ChildNodeListErrorReturnsPartialItemsAndCursor(t *testing.T) { + nodes := []wikiNode{ + {NodeToken: "nt-parent", ObjToken: "obj-parent", ObjType: "file", Title: "Parent.pdf", NodeEditTime: "100", HasChild: true}, + {NodeToken: "nt-peer", ObjToken: "obj-peer", ObjType: "file", Title: "Peer.pdf", NodeEditTime: "200"}, + } + ts, cfg := fakeFeishuWithChildFailure(nodes, "nt-parent") + defer ts.Close() + + items, cursor, err := NewConnector().FetchIncremental(context.Background(), makeConfig(cfg, []string{"space1"}), nil) + if err != nil { + t.Fatalf("FetchIncremental must not abort when one child listing fails: %v", err) + } + if cursor == nil { + t.Fatal("expected cursor to be returned after partial sync") + } + if len(items) != 3 { + t.Fatalf("want 3 items (2 fetched + 1 failure placeholder), got %d: %+v", len(items), items) + } + + var placeholder *types.FetchedItem + for i := range items { + if items[i].Metadata["error"] != "" { + placeholder = &items[i] + break + } + } + if placeholder == nil { + t.Fatal("expected a failure placeholder for the child listing error") + } + if placeholder.Metadata["node_token"] != "nt-parent" { + t.Errorf("placeholder node_token = %q", placeholder.Metadata["node_token"]) + } +} + +func TestFetchIncremental_ChildNodeListErrorDoesNotDeletePreviouslySeenChildren(t *testing.T) { + firstNodes := []wikiNode{ + {NodeToken: "nt-parent", ObjToken: "obj-parent", ObjType: "file", Title: "Parent.pdf", NodeEditTime: "100", HasChild: true}, + } + firstChildren := map[string][]wikiNode{ + "nt-parent": { + {NodeToken: "nt-child", ObjToken: "obj-child", ObjType: "file", Title: "Child.pdf", NodeEditTime: "150"}, + }, + } + ts, cfg := fakeFeishuHierarchy(firstNodes, firstChildren, "") + + firstItems, cursor, err := NewConnector().FetchIncremental(context.Background(), makeConfig(cfg, []string{"space1"}), nil) + if err != nil { + t.Fatalf("first sync error: %v", err) + } + if len(firstItems) != 2 { + t.Fatalf("want 2 first-sync items, got %d", len(firstItems)) + } + ts.Close() + + secondNodes := []wikiNode{ + {NodeToken: "nt-parent", ObjToken: "obj-parent", ObjType: "file", Title: "Parent.pdf", NodeEditTime: "100", HasChild: true}, + } + ts2, cfg2 := fakeFeishuHierarchy(secondNodes, nil, "nt-parent") + defer ts2.Close() + + items, nextCursor, err := NewConnector().FetchIncremental(context.Background(), makeConfig(cfg2, []string{"space1"}), cursor) + if err != nil { + t.Fatalf("partial second sync error: %v", err) + } + for _, item := range items { + if item.IsDeleted { + t.Fatalf("partial child listing must not mark previously seen children as deleted: %+v", item) + } + } + + cursorBytes, _ := json.Marshal(nextCursor.ConnectorCursor) + var restored feishuCursor + if err := json.Unmarshal(cursorBytes, &restored); err != nil { + t.Fatalf("restore cursor: %v", err) + } + if restored.SpaceNodeTimes["space1"]["nt-child"] != "150" { + t.Fatalf("partial sync should preserve previous child cursor entry, got %+v", restored.SpaceNodeTimes["space1"]) + } +} + // ────────────────────────────────────────────────────────────────────── // Client tests // ──────────────────────────────────────────────────────────────────────