fix(datasource): lazy-load Feishu wiki resources to avoid listing timeout (#1672)

Listing resources for a Feishu data source eagerly walked the entire wiki
tree (one API call per node across every space), which exceeded the request
timeout for large knowledge bases. ListResources now takes a parentID and
returns a single level at a time: empty lists the wiki spaces, a space/node
ID lists only its direct children. The frontend fetches children on demand
when a node is expanded. Notion (full tree in one call) and Yuque (flat)
keep their existing behavior by ignoring non-empty parentID.

Refs: https://github.com/Tencent/WeKnora/issues/1672
This commit is contained in:
wizardchen
2026-06-22 17:41:18 +08:00
committed by lyingbug
parent 74c845a1f4
commit d8f13ada4a
12 changed files with 204 additions and 62 deletions
@@ -43,46 +43,60 @@ func (c *Connector) Validate(ctx context.Context, config *types.DataSourceConfig
return nil
}
// ListResources lists all accessible Feishu Wiki spaces and nodes as selectable resources.
func (c *Connector) ListResources(ctx context.Context, config *types.DataSourceConfig) ([]types.Resource, error) {
// ListResources lists Feishu Wiki resources for selection, loading the tree
// lazily one level at a time to avoid traversing the entire wiki up front.
//
// - parentID == "" → list all accessible wiki spaces.
// - parentID == spaceID → list the top-level nodes of that space.
// - parentID == "spaceID:nodeToken" → list the direct children of that node.
//
// Eagerly recursing the whole tree here used to time out for large wikis
// (Tencent/WeKnora#1672); the recursive walk now happens only at sync time.
func (c *Connector) ListResources(
ctx context.Context, config *types.DataSourceConfig, parentID string,
) ([]types.Resource, error) {
feishuConfig, err := parseFeishuConfig(config)
if err != nil {
return nil, err
}
client := NewClient(feishuConfig)
spaces, err := client.ListWikiSpaces(ctx)
if err != nil {
return nil, fmt.Errorf("list feishu wiki spaces: %w", err)
}
resources := make([]types.Resource, 0, len(spaces))
for _, space := range spaces {
resources = append(resources, types.Resource{
ExternalID: space.SpaceID,
Name: space.Name,
Type: "wiki_space",
Description: space.Description,
URL: fmt.Sprintf("https://feishu.cn/wiki/%s", space.SpaceID),
HasChildren: true,
Metadata: map[string]interface{}{
"visibility": space.Visibility,
"space_id": space.SpaceID,
},
})
nodes, err := client.ListAllWikiNodesRecursive(ctx, space.SpaceID)
if parentID == "" {
spaces, err := client.ListWikiSpaces(ctx)
if err != nil {
var partialErr *partialWikiNodeListError
if !errors.As(err, &partialErr) {
return nil, fmt.Errorf("list feishu wiki nodes in space %s: %w", space.SpaceID, err)
}
return nil, fmt.Errorf("list feishu wiki spaces: %w", err)
}
for _, node := range nodes {
resources = append(resources, wikiNodeToResource(space.SpaceID, node))
resources := make([]types.Resource, 0, len(spaces))
for _, space := range spaces {
resources = append(resources, types.Resource{
ExternalID: space.SpaceID,
Name: space.Name,
Type: "wiki_space",
Description: space.Description,
URL: fmt.Sprintf("https://feishu.cn/wiki/%s", space.SpaceID),
HasChildren: true,
Metadata: map[string]interface{}{
"visibility": space.Visibility,
"space_id": space.SpaceID,
},
})
}
return resources, nil
}
// Lazy load: list only the direct children of the given space / node.
spaceID, nodeToken := parseWikiResourceID(parentID)
nodes, err := client.ListWikiNodes(ctx, spaceID, nodeToken)
if err != nil {
return nil, fmt.Errorf("list feishu wiki nodes under %s: %w", parentID, err)
}
resources := make([]types.Resource, 0, len(nodes))
for _, node := range nodes {
resources = append(resources, wikiNodeToResource(spaceID, node))
}
return resources, nil
}
@@ -446,7 +446,7 @@ func TestConnectorListResources(t *testing.T) {
defer ts.Close()
c := NewConnector()
resources, err := c.ListResources(context.Background(), makeConfig(cfg, nil))
resources, err := c.ListResources(context.Background(), makeConfig(cfg, nil), "")
if err != nil {
t.Fatalf("ListResources() error: %v", err)
}
@@ -463,9 +463,15 @@ func TestConnectorListResources(t *testing.T) {
if resources[0].Type != "wiki_space" {
t.Errorf("Type = %q, want %q", resources[0].Type, "wiki_space")
}
if !resources[0].HasChildren {
t.Errorf("HasChildren = false, want true")
}
}
func TestConnectorListResources_IncludesWikiNodeTree(t *testing.T) {
// TestConnectorListResources_LazyLoadsOneLevel verifies that ListResources loads
// the wiki tree lazily — only the requested level — instead of recursing the whole
// tree up front (Tencent/WeKnora#1672).
func TestConnectorListResources_LazyLoadsOneLevel(t *testing.T) {
topNodes := []wikiNode{
{NodeToken: "nt-root", ObjToken: "obj-root", ObjType: "docx", Title: "Root", HasChild: true, ObjEditTime: "100"},
{NodeToken: "nt-peer", ObjToken: "obj-peer", ObjType: "docx", Title: "Peer", ObjEditTime: "200"},
@@ -478,24 +484,44 @@ func TestConnectorListResources_IncludesWikiNodeTree(t *testing.T) {
ts, cfg := fakeFeishuHierarchy(topNodes, childNodes, "")
defer ts.Close()
resources, err := NewConnector().ListResources(context.Background(), makeConfig(cfg, nil))
c := NewConnector()
// Root listing: only the space, no descendants.
spaces, err := c.ListResources(context.Background(), makeConfig(cfg, nil), "")
if err != nil {
t.Fatalf("ListResources() error: %v", err)
t.Fatalf("ListResources(root) error: %v", err)
}
if len(resources) != 4 {
t.Fatalf("want 4 resources (space + 3 nodes), got %d: %+v", len(resources), resources)
if len(spaces) != 1 || spaces[0].ExternalID != "space1" || !spaces[0].HasChildren {
t.Fatalf("want single space with children, got %+v", spaces)
}
// Expanding the space returns only its top-level nodes.
top, err := c.ListResources(context.Background(), makeConfig(cfg, nil), "space1")
if err != nil {
t.Fatalf("ListResources(space1) error: %v", err)
}
if len(top) != 2 {
t.Fatalf("want 2 top-level nodes, got %d: %+v", len(top), top)
}
byID := make(map[string]types.Resource)
for _, resource := range resources {
byID[resource.ExternalID] = resource
for _, r := range top {
byID[r.ExternalID] = r
}
root := byID["space1:nt-root"]
if root.ParentID != "space1" || !root.HasChildren {
t.Fatalf("root resource wrong: %+v", root)
t.Fatalf("root node resource wrong: %+v", root)
}
child := byID["space1:nt-child"]
if child.ParentID != "space1:nt-root" || child.Name != "Child" {
// Expanding a node returns only its direct children.
children, err := c.ListResources(context.Background(), makeConfig(cfg, nil), "space1:nt-root")
if err != nil {
t.Fatalf("ListResources(space1:nt-root) error: %v", err)
}
if len(children) != 1 {
t.Fatalf("want 1 child node, got %d: %+v", len(children), children)
}
child := children[0]
if child.ExternalID != "space1:nt-child" || child.ParentID != "space1:nt-root" || child.Name != "Child" {
t.Fatalf("child resource wrong: %+v", child)
}
if child.Metadata["space_id"] != "space1" || child.Metadata["node_token"] != "nt-child" {