fix(feishu): tolerate partial wiki node listing failures

This commit is contained in:
wolfkill
2026-05-12 17:27:42 +08:00
committed by lyingbug
parent 8b2a36d759
commit b387f3637a
3 changed files with 266 additions and 13 deletions
+36 -9
View File
@@ -7,6 +7,7 @@ import (
"fmt"
"io"
"net/http"
"strings"
"sync"
"time"
@@ -27,6 +28,26 @@ type Client struct {
tokenExpAt time.Time
}
type wikiNodeListFailure struct {
Node wikiNode
Err error
}
type partialWikiNodeListError struct {
Failures []wikiNodeListFailure
}
func (e *partialWikiNodeListError) Error() string {
if e == nil || len(e.Failures) == 0 {
return "partial wiki node listing failed"
}
parts := make([]string, 0, len(e.Failures))
for _, failure := range e.Failures {
parts = append(parts, failure.Err.Error())
}
return strings.Join(parts, "; ")
}
// NewClient creates a new Feishu API client.
func NewClient(config *Config) *Client {
return &Client{
@@ -237,9 +258,10 @@ func (c *Client) ListAllWikiNodesRecursive(ctx context.Context, spaceID string)
}
var allNodes []wikiNode
var walk func(nodes []wikiNode) error
var failures []wikiNodeListFailure
var walk func(nodes []wikiNode)
walk = func(nodes []wikiNode) error {
walk = func(nodes []wikiNode) {
for _, node := range nodes {
allNodes = append(allNodes, node)
@@ -247,18 +269,23 @@ func (c *Client) ListAllWikiNodesRecursive(ctx context.Context, spaceID string)
if node.HasChild {
children, err := c.ListWikiNodes(ctx, spaceID, node.NodeToken)
if err != nil {
return fmt.Errorf("list children of %s: %w", node.NodeToken, err)
}
if err := walk(children); err != nil {
return err
wrappedErr := fmt.Errorf("list children of %s: %w", node.NodeToken, err)
failures = append(failures, wikiNodeListFailure{
Node: node,
Err: wrappedErr,
})
logger.Warnf(ctx, "[Feishu] partial wiki node listing failure: space=%s node=%s err=%v",
spaceID, node.NodeToken, err)
continue
}
walk(children)
}
}
return nil
}
if err := walk(topNodes); err != nil {
return nil, err
walk(topNodes)
if len(failures) > 0 {
return allNodes, &partialWikiNodeListError{Failures: failures}
}
return allNodes, nil
@@ -3,6 +3,7 @@ package feishu
import (
"context"
"encoding/json"
"errors"
"fmt"
"strconv"
"strings"
@@ -85,7 +86,11 @@ func (c *Connector) FetchAll(ctx context.Context, config *types.DataSourceConfig
// List all nodes in this wiki space recursively
nodes, err := client.ListAllWikiNodesRecursive(ctx, spaceID)
if err != nil {
return nil, fmt.Errorf("list nodes in space %s: %w", spaceID, err)
var partialErr *partialWikiNodeListError
if !errors.As(err, &partialErr) {
return nil, fmt.Errorf("list nodes in space %s: %w", spaceID, err)
}
allItems = appendWikiNodeListFailureItems(allItems, spaceID, partialErr.Failures)
}
// Fetch content for each document node
@@ -146,11 +151,22 @@ func (c *Connector) FetchIncremental(ctx context.Context, config *types.DataSour
for _, spaceID := range resourceIDs {
// List all nodes in this space
nodes, err := client.ListAllWikiNodesRecursive(ctx, spaceID)
var partialErr *partialWikiNodeListError
if err != nil {
return nil, nil, fmt.Errorf("list nodes in space %s: %w", spaceID, err)
if !errors.As(err, &partialErr) {
return nil, nil, fmt.Errorf("list nodes in space %s: %w", spaceID, err)
}
changedItems = appendWikiNodeListFailureItems(changedItems, spaceID, partialErr.Failures)
}
newCursor.SpaceNodeTimes[spaceID] = make(map[string]string)
if partialErr != nil && prevCursor.SpaceNodeTimes != nil {
if prevTimes, ok := prevCursor.SpaceNodeTimes[spaceID]; ok {
for nodeToken, editTime := range prevTimes {
newCursor.SpaceNodeTimes[spaceID][nodeToken] = editTime
}
}
}
// Build a set of current node tokens for deletion detection
currentNodes := make(map[string]bool)
@@ -197,7 +213,7 @@ func (c *Connector) FetchIncremental(ctx context.Context, config *types.DataSour
}
// Detect deleted nodes
if prevCursor.SpaceNodeTimes != nil {
if partialErr == nil && prevCursor.SpaceNodeTimes != nil {
if prevTimes, ok := prevCursor.SpaceNodeTimes[spaceID]; ok {
for nodeToken := range prevTimes {
if !currentNodes[nodeToken] {
@@ -226,6 +242,29 @@ func (c *Connector) FetchIncremental(ctx context.Context, config *types.DataSour
return changedItems, nextSyncCursor, nil
}
func appendWikiNodeListFailureItems(items []types.FetchedItem, spaceID string, failures []wikiNodeListFailure) []types.FetchedItem {
for _, failure := range failures {
node := failure.Node
title := node.Title
if title == "" {
title = node.NodeToken
}
items = append(items, types.FetchedItem{
ExternalID: node.NodeToken,
Title: title,
SourceResourceID: spaceID,
Metadata: map[string]string{
"error": failure.Err.Error(),
"channel": types.ChannelFeishu,
"node_token": node.NodeToken,
"space_id": spaceID,
"failure_stage": "list_children",
},
})
}
return items
}
// fetchNodeContent fetches the content of a single wiki node and converts it to FetchedItem.
// Dispatches to different retrieval strategies based on obj_type:
// - docx/doc → export API → .docx file
@@ -67,7 +67,9 @@ func fakeFeishu(nodes []wikiNode) (*httptest.Server, *Config) {
if r.Method == http.MethodPost {
writeJSON(w, exportTaskCreateResponse{
apiResponse: apiResponse{Code: 0},
Data: struct{ Ticket string `json:"ticket"` }{Ticket: "ticket-123"},
Data: struct {
Ticket string `json:"ticket"`
}{Ticket: "ticket-123"},
})
return
}
@@ -153,6 +155,62 @@ func fakeFeishu(nodes []wikiNode) (*httptest.Server, *Config) {
return ts, cfg
}
func fakeFeishuWithChildFailure(topNodes []wikiNode, failingParentToken string) (*httptest.Server, *Config) {
return fakeFeishuHierarchy(topNodes, nil, failingParentToken)
}
func fakeFeishuHierarchy(topNodes []wikiNode, childNodes map[string][]wikiNode, failingParentToken string) (*httptest.Server, *Config) {
mux := http.NewServeMux()
mux.HandleFunc("/open-apis/auth/v3/tenant_access_token/internal", func(w http.ResponseWriter, r *http.Request) {
writeJSON(w, tokenResponse{
apiResponse: apiResponse{Code: 0},
TenantAccessToken: "fake-token",
Expire: 7200,
})
})
mux.HandleFunc("/open-apis/wiki/v2/spaces/space1/nodes", func(w http.ResponseWriter, r *http.Request) {
parentToken := r.URL.Query().Get("parent_node_token")
if parentToken != "" && parentToken == failingParentToken {
w.WriteHeader(http.StatusInternalServerError)
_, _ = w.Write([]byte(`{"code":1663,"msg":"internal error"}`))
return
}
nodes := topNodes
if parentToken != "" {
nodes = childNodes[parentToken]
}
writeJSON(w, wikiNodeListResponse{
apiResponse: apiResponse{Code: 0},
Data: struct {
Items []wikiNode `json:"items"`
HasMore bool `json:"has_more"`
PageToken string `json:"page_token"`
}{
Items: nodes,
},
})
})
mux.HandleFunc("/open-apis/drive/v1/files/", func(w http.ResponseWriter, r *http.Request) {
if strings.HasSuffix(r.URL.Path, "/download") {
w.Header().Set("Content-Type", "application/octet-stream")
_, _ = w.Write([]byte("fake-file-content"))
return
}
http.NotFound(w, r)
})
ts := httptest.NewServer(mux)
cfg := &Config{
AppID: "test-app-id",
AppSecret: "test-app-secret",
BaseURL: ts.URL,
}
return ts, cfg
}
func writeJSON(w http.ResponseWriter, v interface{}) {
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(v)
@@ -528,6 +586,55 @@ func TestFetchAll_MixedTypes(t *testing.T) {
}
}
func TestFetchAll_ChildNodeListErrorReturnsPartialItems(t *testing.T) {
nodes := []wikiNode{
{NodeToken: "nt-parent", ObjToken: "obj-parent", ObjType: "file", Title: "Parent.pdf", NodeEditTime: "100", HasChild: true},
{NodeToken: "nt-peer", ObjToken: "obj-peer", ObjType: "file", Title: "Peer.pdf", NodeEditTime: "200"},
}
ts, cfg := fakeFeishuWithChildFailure(nodes, "nt-parent")
defer ts.Close()
items, err := NewConnector().FetchAll(context.Background(), makeConfig(cfg, []string{"space1"}), []string{"space1"})
if err != nil {
t.Fatalf("FetchAll must not abort when one child listing fails: %v", err)
}
if len(items) != 3 {
t.Fatalf("want 3 items (2 fetched + 1 failure placeholder), got %d: %+v", len(items), items)
}
var parent, peer, placeholder *types.FetchedItem
for i := range items {
switch {
case items[i].ExternalID == "nt-parent" && len(items[i].Content) > 0:
parent = &items[i]
case items[i].ExternalID == "nt-peer" && len(items[i].Content) > 0:
peer = &items[i]
case items[i].Metadata["error"] != "":
placeholder = &items[i]
}
}
if parent == nil || peer == nil {
t.Fatalf("expected both successfully listed nodes to be fetched, parent=%v peer=%v", parent != nil, peer != nil)
}
if placeholder == nil {
t.Fatal("expected a failure placeholder for the child listing error")
}
if placeholder.ExternalID != "nt-parent" || placeholder.Title != "Parent.pdf" {
t.Errorf("placeholder identity wrong: %+v", placeholder)
}
if placeholder.Metadata["channel"] != types.ChannelFeishu {
t.Errorf("placeholder channel = %q", placeholder.Metadata["channel"])
}
if placeholder.Metadata["node_token"] != "nt-parent" || placeholder.Metadata["space_id"] != "space1" {
t.Errorf("placeholder missing traceability metadata: %+v", placeholder.Metadata)
}
if !strings.Contains(placeholder.Metadata["error"], "list children of nt-parent") {
t.Errorf("placeholder error = %q", placeholder.Metadata["error"])
}
}
// ──────────────────────────────────────────────────────────────────────
// FetchIncremental tests
// ──────────────────────────────────────────────────────────────────────
@@ -648,6 +755,86 @@ func TestFetchIncremental_NoResourceIDs(t *testing.T) {
}
}
func TestFetchIncremental_ChildNodeListErrorReturnsPartialItemsAndCursor(t *testing.T) {
nodes := []wikiNode{
{NodeToken: "nt-parent", ObjToken: "obj-parent", ObjType: "file", Title: "Parent.pdf", NodeEditTime: "100", HasChild: true},
{NodeToken: "nt-peer", ObjToken: "obj-peer", ObjType: "file", Title: "Peer.pdf", NodeEditTime: "200"},
}
ts, cfg := fakeFeishuWithChildFailure(nodes, "nt-parent")
defer ts.Close()
items, cursor, err := NewConnector().FetchIncremental(context.Background(), makeConfig(cfg, []string{"space1"}), nil)
if err != nil {
t.Fatalf("FetchIncremental must not abort when one child listing fails: %v", err)
}
if cursor == nil {
t.Fatal("expected cursor to be returned after partial sync")
}
if len(items) != 3 {
t.Fatalf("want 3 items (2 fetched + 1 failure placeholder), got %d: %+v", len(items), items)
}
var placeholder *types.FetchedItem
for i := range items {
if items[i].Metadata["error"] != "" {
placeholder = &items[i]
break
}
}
if placeholder == nil {
t.Fatal("expected a failure placeholder for the child listing error")
}
if placeholder.Metadata["node_token"] != "nt-parent" {
t.Errorf("placeholder node_token = %q", placeholder.Metadata["node_token"])
}
}
func TestFetchIncremental_ChildNodeListErrorDoesNotDeletePreviouslySeenChildren(t *testing.T) {
firstNodes := []wikiNode{
{NodeToken: "nt-parent", ObjToken: "obj-parent", ObjType: "file", Title: "Parent.pdf", NodeEditTime: "100", HasChild: true},
}
firstChildren := map[string][]wikiNode{
"nt-parent": {
{NodeToken: "nt-child", ObjToken: "obj-child", ObjType: "file", Title: "Child.pdf", NodeEditTime: "150"},
},
}
ts, cfg := fakeFeishuHierarchy(firstNodes, firstChildren, "")
firstItems, cursor, err := NewConnector().FetchIncremental(context.Background(), makeConfig(cfg, []string{"space1"}), nil)
if err != nil {
t.Fatalf("first sync error: %v", err)
}
if len(firstItems) != 2 {
t.Fatalf("want 2 first-sync items, got %d", len(firstItems))
}
ts.Close()
secondNodes := []wikiNode{
{NodeToken: "nt-parent", ObjToken: "obj-parent", ObjType: "file", Title: "Parent.pdf", NodeEditTime: "100", HasChild: true},
}
ts2, cfg2 := fakeFeishuHierarchy(secondNodes, nil, "nt-parent")
defer ts2.Close()
items, nextCursor, err := NewConnector().FetchIncremental(context.Background(), makeConfig(cfg2, []string{"space1"}), cursor)
if err != nil {
t.Fatalf("partial second sync error: %v", err)
}
for _, item := range items {
if item.IsDeleted {
t.Fatalf("partial child listing must not mark previously seen children as deleted: %+v", item)
}
}
cursorBytes, _ := json.Marshal(nextCursor.ConnectorCursor)
var restored feishuCursor
if err := json.Unmarshal(cursorBytes, &restored); err != nil {
t.Fatalf("restore cursor: %v", err)
}
if restored.SpaceNodeTimes["space1"]["nt-child"] != "150" {
t.Fatalf("partial sync should preserve previous child cursor entry, got %+v", restored.SpaceNodeTimes["space1"])
}
}
// ──────────────────────────────────────────────────────────────────────
// Client tests
// ──────────────────────────────────────────────────────────────────────