fix(feishu): tolerate partial wiki node listing failures
This commit is contained in:
@@ -7,6 +7,7 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -27,6 +28,26 @@ type Client struct {
|
||||
tokenExpAt time.Time
|
||||
}
|
||||
|
||||
type wikiNodeListFailure struct {
|
||||
Node wikiNode
|
||||
Err error
|
||||
}
|
||||
|
||||
type partialWikiNodeListError struct {
|
||||
Failures []wikiNodeListFailure
|
||||
}
|
||||
|
||||
func (e *partialWikiNodeListError) Error() string {
|
||||
if e == nil || len(e.Failures) == 0 {
|
||||
return "partial wiki node listing failed"
|
||||
}
|
||||
parts := make([]string, 0, len(e.Failures))
|
||||
for _, failure := range e.Failures {
|
||||
parts = append(parts, failure.Err.Error())
|
||||
}
|
||||
return strings.Join(parts, "; ")
|
||||
}
|
||||
|
||||
// NewClient creates a new Feishu API client.
|
||||
func NewClient(config *Config) *Client {
|
||||
return &Client{
|
||||
@@ -237,9 +258,10 @@ func (c *Client) ListAllWikiNodesRecursive(ctx context.Context, spaceID string)
|
||||
}
|
||||
|
||||
var allNodes []wikiNode
|
||||
var walk func(nodes []wikiNode) error
|
||||
var failures []wikiNodeListFailure
|
||||
var walk func(nodes []wikiNode)
|
||||
|
||||
walk = func(nodes []wikiNode) error {
|
||||
walk = func(nodes []wikiNode) {
|
||||
for _, node := range nodes {
|
||||
allNodes = append(allNodes, node)
|
||||
|
||||
@@ -247,18 +269,23 @@ func (c *Client) ListAllWikiNodesRecursive(ctx context.Context, spaceID string)
|
||||
if node.HasChild {
|
||||
children, err := c.ListWikiNodes(ctx, spaceID, node.NodeToken)
|
||||
if err != nil {
|
||||
return fmt.Errorf("list children of %s: %w", node.NodeToken, err)
|
||||
}
|
||||
if err := walk(children); err != nil {
|
||||
return err
|
||||
wrappedErr := fmt.Errorf("list children of %s: %w", node.NodeToken, err)
|
||||
failures = append(failures, wikiNodeListFailure{
|
||||
Node: node,
|
||||
Err: wrappedErr,
|
||||
})
|
||||
logger.Warnf(ctx, "[Feishu] partial wiki node listing failure: space=%s node=%s err=%v",
|
||||
spaceID, node.NodeToken, err)
|
||||
continue
|
||||
}
|
||||
walk(children)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := walk(topNodes); err != nil {
|
||||
return nil, err
|
||||
walk(topNodes)
|
||||
if len(failures) > 0 {
|
||||
return allNodes, &partialWikiNodeListError{Failures: failures}
|
||||
}
|
||||
|
||||
return allNodes, nil
|
||||
|
||||
@@ -3,6 +3,7 @@ package feishu
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -85,7 +86,11 @@ func (c *Connector) FetchAll(ctx context.Context, config *types.DataSourceConfig
|
||||
// List all nodes in this wiki space recursively
|
||||
nodes, err := client.ListAllWikiNodesRecursive(ctx, spaceID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list nodes in space %s: %w", spaceID, err)
|
||||
var partialErr *partialWikiNodeListError
|
||||
if !errors.As(err, &partialErr) {
|
||||
return nil, fmt.Errorf("list nodes in space %s: %w", spaceID, err)
|
||||
}
|
||||
allItems = appendWikiNodeListFailureItems(allItems, spaceID, partialErr.Failures)
|
||||
}
|
||||
|
||||
// Fetch content for each document node
|
||||
@@ -146,11 +151,22 @@ func (c *Connector) FetchIncremental(ctx context.Context, config *types.DataSour
|
||||
for _, spaceID := range resourceIDs {
|
||||
// List all nodes in this space
|
||||
nodes, err := client.ListAllWikiNodesRecursive(ctx, spaceID)
|
||||
var partialErr *partialWikiNodeListError
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("list nodes in space %s: %w", spaceID, err)
|
||||
if !errors.As(err, &partialErr) {
|
||||
return nil, nil, fmt.Errorf("list nodes in space %s: %w", spaceID, err)
|
||||
}
|
||||
changedItems = appendWikiNodeListFailureItems(changedItems, spaceID, partialErr.Failures)
|
||||
}
|
||||
|
||||
newCursor.SpaceNodeTimes[spaceID] = make(map[string]string)
|
||||
if partialErr != nil && prevCursor.SpaceNodeTimes != nil {
|
||||
if prevTimes, ok := prevCursor.SpaceNodeTimes[spaceID]; ok {
|
||||
for nodeToken, editTime := range prevTimes {
|
||||
newCursor.SpaceNodeTimes[spaceID][nodeToken] = editTime
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Build a set of current node tokens for deletion detection
|
||||
currentNodes := make(map[string]bool)
|
||||
@@ -197,7 +213,7 @@ func (c *Connector) FetchIncremental(ctx context.Context, config *types.DataSour
|
||||
}
|
||||
|
||||
// Detect deleted nodes
|
||||
if prevCursor.SpaceNodeTimes != nil {
|
||||
if partialErr == nil && prevCursor.SpaceNodeTimes != nil {
|
||||
if prevTimes, ok := prevCursor.SpaceNodeTimes[spaceID]; ok {
|
||||
for nodeToken := range prevTimes {
|
||||
if !currentNodes[nodeToken] {
|
||||
@@ -226,6 +242,29 @@ func (c *Connector) FetchIncremental(ctx context.Context, config *types.DataSour
|
||||
return changedItems, nextSyncCursor, nil
|
||||
}
|
||||
|
||||
func appendWikiNodeListFailureItems(items []types.FetchedItem, spaceID string, failures []wikiNodeListFailure) []types.FetchedItem {
|
||||
for _, failure := range failures {
|
||||
node := failure.Node
|
||||
title := node.Title
|
||||
if title == "" {
|
||||
title = node.NodeToken
|
||||
}
|
||||
items = append(items, types.FetchedItem{
|
||||
ExternalID: node.NodeToken,
|
||||
Title: title,
|
||||
SourceResourceID: spaceID,
|
||||
Metadata: map[string]string{
|
||||
"error": failure.Err.Error(),
|
||||
"channel": types.ChannelFeishu,
|
||||
"node_token": node.NodeToken,
|
||||
"space_id": spaceID,
|
||||
"failure_stage": "list_children",
|
||||
},
|
||||
})
|
||||
}
|
||||
return items
|
||||
}
|
||||
|
||||
// fetchNodeContent fetches the content of a single wiki node and converts it to FetchedItem.
|
||||
// Dispatches to different retrieval strategies based on obj_type:
|
||||
// - docx/doc → export API → .docx file
|
||||
|
||||
@@ -67,7 +67,9 @@ func fakeFeishu(nodes []wikiNode) (*httptest.Server, *Config) {
|
||||
if r.Method == http.MethodPost {
|
||||
writeJSON(w, exportTaskCreateResponse{
|
||||
apiResponse: apiResponse{Code: 0},
|
||||
Data: struct{ Ticket string `json:"ticket"` }{Ticket: "ticket-123"},
|
||||
Data: struct {
|
||||
Ticket string `json:"ticket"`
|
||||
}{Ticket: "ticket-123"},
|
||||
})
|
||||
return
|
||||
}
|
||||
@@ -153,6 +155,62 @@ func fakeFeishu(nodes []wikiNode) (*httptest.Server, *Config) {
|
||||
return ts, cfg
|
||||
}
|
||||
|
||||
func fakeFeishuWithChildFailure(topNodes []wikiNode, failingParentToken string) (*httptest.Server, *Config) {
|
||||
return fakeFeishuHierarchy(topNodes, nil, failingParentToken)
|
||||
}
|
||||
|
||||
func fakeFeishuHierarchy(topNodes []wikiNode, childNodes map[string][]wikiNode, failingParentToken string) (*httptest.Server, *Config) {
|
||||
mux := http.NewServeMux()
|
||||
|
||||
mux.HandleFunc("/open-apis/auth/v3/tenant_access_token/internal", func(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, tokenResponse{
|
||||
apiResponse: apiResponse{Code: 0},
|
||||
TenantAccessToken: "fake-token",
|
||||
Expire: 7200,
|
||||
})
|
||||
})
|
||||
|
||||
mux.HandleFunc("/open-apis/wiki/v2/spaces/space1/nodes", func(w http.ResponseWriter, r *http.Request) {
|
||||
parentToken := r.URL.Query().Get("parent_node_token")
|
||||
if parentToken != "" && parentToken == failingParentToken {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
_, _ = w.Write([]byte(`{"code":1663,"msg":"internal error"}`))
|
||||
return
|
||||
}
|
||||
nodes := topNodes
|
||||
if parentToken != "" {
|
||||
nodes = childNodes[parentToken]
|
||||
}
|
||||
writeJSON(w, wikiNodeListResponse{
|
||||
apiResponse: apiResponse{Code: 0},
|
||||
Data: struct {
|
||||
Items []wikiNode `json:"items"`
|
||||
HasMore bool `json:"has_more"`
|
||||
PageToken string `json:"page_token"`
|
||||
}{
|
||||
Items: nodes,
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
mux.HandleFunc("/open-apis/drive/v1/files/", func(w http.ResponseWriter, r *http.Request) {
|
||||
if strings.HasSuffix(r.URL.Path, "/download") {
|
||||
w.Header().Set("Content-Type", "application/octet-stream")
|
||||
_, _ = w.Write([]byte("fake-file-content"))
|
||||
return
|
||||
}
|
||||
http.NotFound(w, r)
|
||||
})
|
||||
|
||||
ts := httptest.NewServer(mux)
|
||||
cfg := &Config{
|
||||
AppID: "test-app-id",
|
||||
AppSecret: "test-app-secret",
|
||||
BaseURL: ts.URL,
|
||||
}
|
||||
return ts, cfg
|
||||
}
|
||||
|
||||
func writeJSON(w http.ResponseWriter, v interface{}) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(v)
|
||||
@@ -528,6 +586,55 @@ func TestFetchAll_MixedTypes(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchAll_ChildNodeListErrorReturnsPartialItems(t *testing.T) {
|
||||
nodes := []wikiNode{
|
||||
{NodeToken: "nt-parent", ObjToken: "obj-parent", ObjType: "file", Title: "Parent.pdf", NodeEditTime: "100", HasChild: true},
|
||||
{NodeToken: "nt-peer", ObjToken: "obj-peer", ObjType: "file", Title: "Peer.pdf", NodeEditTime: "200"},
|
||||
}
|
||||
ts, cfg := fakeFeishuWithChildFailure(nodes, "nt-parent")
|
||||
defer ts.Close()
|
||||
|
||||
items, err := NewConnector().FetchAll(context.Background(), makeConfig(cfg, []string{"space1"}), []string{"space1"})
|
||||
if err != nil {
|
||||
t.Fatalf("FetchAll must not abort when one child listing fails: %v", err)
|
||||
}
|
||||
|
||||
if len(items) != 3 {
|
||||
t.Fatalf("want 3 items (2 fetched + 1 failure placeholder), got %d: %+v", len(items), items)
|
||||
}
|
||||
|
||||
var parent, peer, placeholder *types.FetchedItem
|
||||
for i := range items {
|
||||
switch {
|
||||
case items[i].ExternalID == "nt-parent" && len(items[i].Content) > 0:
|
||||
parent = &items[i]
|
||||
case items[i].ExternalID == "nt-peer" && len(items[i].Content) > 0:
|
||||
peer = &items[i]
|
||||
case items[i].Metadata["error"] != "":
|
||||
placeholder = &items[i]
|
||||
}
|
||||
}
|
||||
|
||||
if parent == nil || peer == nil {
|
||||
t.Fatalf("expected both successfully listed nodes to be fetched, parent=%v peer=%v", parent != nil, peer != nil)
|
||||
}
|
||||
if placeholder == nil {
|
||||
t.Fatal("expected a failure placeholder for the child listing error")
|
||||
}
|
||||
if placeholder.ExternalID != "nt-parent" || placeholder.Title != "Parent.pdf" {
|
||||
t.Errorf("placeholder identity wrong: %+v", placeholder)
|
||||
}
|
||||
if placeholder.Metadata["channel"] != types.ChannelFeishu {
|
||||
t.Errorf("placeholder channel = %q", placeholder.Metadata["channel"])
|
||||
}
|
||||
if placeholder.Metadata["node_token"] != "nt-parent" || placeholder.Metadata["space_id"] != "space1" {
|
||||
t.Errorf("placeholder missing traceability metadata: %+v", placeholder.Metadata)
|
||||
}
|
||||
if !strings.Contains(placeholder.Metadata["error"], "list children of nt-parent") {
|
||||
t.Errorf("placeholder error = %q", placeholder.Metadata["error"])
|
||||
}
|
||||
}
|
||||
|
||||
// ──────────────────────────────────────────────────────────────────────
|
||||
// FetchIncremental tests
|
||||
// ──────────────────────────────────────────────────────────────────────
|
||||
@@ -648,6 +755,86 @@ func TestFetchIncremental_NoResourceIDs(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchIncremental_ChildNodeListErrorReturnsPartialItemsAndCursor(t *testing.T) {
|
||||
nodes := []wikiNode{
|
||||
{NodeToken: "nt-parent", ObjToken: "obj-parent", ObjType: "file", Title: "Parent.pdf", NodeEditTime: "100", HasChild: true},
|
||||
{NodeToken: "nt-peer", ObjToken: "obj-peer", ObjType: "file", Title: "Peer.pdf", NodeEditTime: "200"},
|
||||
}
|
||||
ts, cfg := fakeFeishuWithChildFailure(nodes, "nt-parent")
|
||||
defer ts.Close()
|
||||
|
||||
items, cursor, err := NewConnector().FetchIncremental(context.Background(), makeConfig(cfg, []string{"space1"}), nil)
|
||||
if err != nil {
|
||||
t.Fatalf("FetchIncremental must not abort when one child listing fails: %v", err)
|
||||
}
|
||||
if cursor == nil {
|
||||
t.Fatal("expected cursor to be returned after partial sync")
|
||||
}
|
||||
if len(items) != 3 {
|
||||
t.Fatalf("want 3 items (2 fetched + 1 failure placeholder), got %d: %+v", len(items), items)
|
||||
}
|
||||
|
||||
var placeholder *types.FetchedItem
|
||||
for i := range items {
|
||||
if items[i].Metadata["error"] != "" {
|
||||
placeholder = &items[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
if placeholder == nil {
|
||||
t.Fatal("expected a failure placeholder for the child listing error")
|
||||
}
|
||||
if placeholder.Metadata["node_token"] != "nt-parent" {
|
||||
t.Errorf("placeholder node_token = %q", placeholder.Metadata["node_token"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchIncremental_ChildNodeListErrorDoesNotDeletePreviouslySeenChildren(t *testing.T) {
|
||||
firstNodes := []wikiNode{
|
||||
{NodeToken: "nt-parent", ObjToken: "obj-parent", ObjType: "file", Title: "Parent.pdf", NodeEditTime: "100", HasChild: true},
|
||||
}
|
||||
firstChildren := map[string][]wikiNode{
|
||||
"nt-parent": {
|
||||
{NodeToken: "nt-child", ObjToken: "obj-child", ObjType: "file", Title: "Child.pdf", NodeEditTime: "150"},
|
||||
},
|
||||
}
|
||||
ts, cfg := fakeFeishuHierarchy(firstNodes, firstChildren, "")
|
||||
|
||||
firstItems, cursor, err := NewConnector().FetchIncremental(context.Background(), makeConfig(cfg, []string{"space1"}), nil)
|
||||
if err != nil {
|
||||
t.Fatalf("first sync error: %v", err)
|
||||
}
|
||||
if len(firstItems) != 2 {
|
||||
t.Fatalf("want 2 first-sync items, got %d", len(firstItems))
|
||||
}
|
||||
ts.Close()
|
||||
|
||||
secondNodes := []wikiNode{
|
||||
{NodeToken: "nt-parent", ObjToken: "obj-parent", ObjType: "file", Title: "Parent.pdf", NodeEditTime: "100", HasChild: true},
|
||||
}
|
||||
ts2, cfg2 := fakeFeishuHierarchy(secondNodes, nil, "nt-parent")
|
||||
defer ts2.Close()
|
||||
|
||||
items, nextCursor, err := NewConnector().FetchIncremental(context.Background(), makeConfig(cfg2, []string{"space1"}), cursor)
|
||||
if err != nil {
|
||||
t.Fatalf("partial second sync error: %v", err)
|
||||
}
|
||||
for _, item := range items {
|
||||
if item.IsDeleted {
|
||||
t.Fatalf("partial child listing must not mark previously seen children as deleted: %+v", item)
|
||||
}
|
||||
}
|
||||
|
||||
cursorBytes, _ := json.Marshal(nextCursor.ConnectorCursor)
|
||||
var restored feishuCursor
|
||||
if err := json.Unmarshal(cursorBytes, &restored); err != nil {
|
||||
t.Fatalf("restore cursor: %v", err)
|
||||
}
|
||||
if restored.SpaceNodeTimes["space1"]["nt-child"] != "150" {
|
||||
t.Fatalf("partial sync should preserve previous child cursor entry, got %+v", restored.SpaceNodeTimes["space1"])
|
||||
}
|
||||
}
|
||||
|
||||
// ──────────────────────────────────────────────────────────────────────
|
||||
// Client tests
|
||||
// ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
Reference in New Issue
Block a user