fix(datasource): lazy-load Feishu wiki resources to avoid listing timeout (#1672)
Listing resources for a Feishu data source eagerly walked the entire wiki tree (one API call per node across every space), which exceeded the request timeout for large knowledge bases. ListResources now takes a parentID and returns a single level at a time: empty lists the wiki spaces, a space/node ID lists only its direct children. The frontend fetches children on demand when a node is expanded. Notion (full tree in one call) and Yuque (flat) keep their existing behavior by ignoring non-empty parentID. Refs: https://github.com/Tencent/WeKnora/issues/1672
This commit is contained in:
@@ -43,46 +43,60 @@ func (c *Connector) Validate(ctx context.Context, config *types.DataSourceConfig
|
||||
return nil
|
||||
}
|
||||
|
||||
// ListResources lists all accessible Feishu Wiki spaces and nodes as selectable resources.
|
||||
func (c *Connector) ListResources(ctx context.Context, config *types.DataSourceConfig) ([]types.Resource, error) {
|
||||
// ListResources lists Feishu Wiki resources for selection, loading the tree
|
||||
// lazily one level at a time to avoid traversing the entire wiki up front.
|
||||
//
|
||||
// - parentID == "" → list all accessible wiki spaces.
|
||||
// - parentID == spaceID → list the top-level nodes of that space.
|
||||
// - parentID == "spaceID:nodeToken" → list the direct children of that node.
|
||||
//
|
||||
// Eagerly recursing the whole tree here used to time out for large wikis
|
||||
// (Tencent/WeKnora#1672); the recursive walk now happens only at sync time.
|
||||
func (c *Connector) ListResources(
|
||||
ctx context.Context, config *types.DataSourceConfig, parentID string,
|
||||
) ([]types.Resource, error) {
|
||||
feishuConfig, err := parseFeishuConfig(config)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
client := NewClient(feishuConfig)
|
||||
spaces, err := client.ListWikiSpaces(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list feishu wiki spaces: %w", err)
|
||||
}
|
||||
|
||||
resources := make([]types.Resource, 0, len(spaces))
|
||||
for _, space := range spaces {
|
||||
resources = append(resources, types.Resource{
|
||||
ExternalID: space.SpaceID,
|
||||
Name: space.Name,
|
||||
Type: "wiki_space",
|
||||
Description: space.Description,
|
||||
URL: fmt.Sprintf("https://feishu.cn/wiki/%s", space.SpaceID),
|
||||
HasChildren: true,
|
||||
Metadata: map[string]interface{}{
|
||||
"visibility": space.Visibility,
|
||||
"space_id": space.SpaceID,
|
||||
},
|
||||
})
|
||||
|
||||
nodes, err := client.ListAllWikiNodesRecursive(ctx, space.SpaceID)
|
||||
if parentID == "" {
|
||||
spaces, err := client.ListWikiSpaces(ctx)
|
||||
if err != nil {
|
||||
var partialErr *partialWikiNodeListError
|
||||
if !errors.As(err, &partialErr) {
|
||||
return nil, fmt.Errorf("list feishu wiki nodes in space %s: %w", space.SpaceID, err)
|
||||
}
|
||||
return nil, fmt.Errorf("list feishu wiki spaces: %w", err)
|
||||
}
|
||||
for _, node := range nodes {
|
||||
resources = append(resources, wikiNodeToResource(space.SpaceID, node))
|
||||
|
||||
resources := make([]types.Resource, 0, len(spaces))
|
||||
for _, space := range spaces {
|
||||
resources = append(resources, types.Resource{
|
||||
ExternalID: space.SpaceID,
|
||||
Name: space.Name,
|
||||
Type: "wiki_space",
|
||||
Description: space.Description,
|
||||
URL: fmt.Sprintf("https://feishu.cn/wiki/%s", space.SpaceID),
|
||||
HasChildren: true,
|
||||
Metadata: map[string]interface{}{
|
||||
"visibility": space.Visibility,
|
||||
"space_id": space.SpaceID,
|
||||
},
|
||||
})
|
||||
}
|
||||
return resources, nil
|
||||
}
|
||||
|
||||
// Lazy load: list only the direct children of the given space / node.
|
||||
spaceID, nodeToken := parseWikiResourceID(parentID)
|
||||
nodes, err := client.ListWikiNodes(ctx, spaceID, nodeToken)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list feishu wiki nodes under %s: %w", parentID, err)
|
||||
}
|
||||
|
||||
resources := make([]types.Resource, 0, len(nodes))
|
||||
for _, node := range nodes {
|
||||
resources = append(resources, wikiNodeToResource(spaceID, node))
|
||||
}
|
||||
return resources, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -446,7 +446,7 @@ func TestConnectorListResources(t *testing.T) {
|
||||
defer ts.Close()
|
||||
|
||||
c := NewConnector()
|
||||
resources, err := c.ListResources(context.Background(), makeConfig(cfg, nil))
|
||||
resources, err := c.ListResources(context.Background(), makeConfig(cfg, nil), "")
|
||||
if err != nil {
|
||||
t.Fatalf("ListResources() error: %v", err)
|
||||
}
|
||||
@@ -463,9 +463,15 @@ func TestConnectorListResources(t *testing.T) {
|
||||
if resources[0].Type != "wiki_space" {
|
||||
t.Errorf("Type = %q, want %q", resources[0].Type, "wiki_space")
|
||||
}
|
||||
if !resources[0].HasChildren {
|
||||
t.Errorf("HasChildren = false, want true")
|
||||
}
|
||||
}
|
||||
|
||||
func TestConnectorListResources_IncludesWikiNodeTree(t *testing.T) {
|
||||
// TestConnectorListResources_LazyLoadsOneLevel verifies that ListResources loads
|
||||
// the wiki tree lazily — only the requested level — instead of recursing the whole
|
||||
// tree up front (Tencent/WeKnora#1672).
|
||||
func TestConnectorListResources_LazyLoadsOneLevel(t *testing.T) {
|
||||
topNodes := []wikiNode{
|
||||
{NodeToken: "nt-root", ObjToken: "obj-root", ObjType: "docx", Title: "Root", HasChild: true, ObjEditTime: "100"},
|
||||
{NodeToken: "nt-peer", ObjToken: "obj-peer", ObjType: "docx", Title: "Peer", ObjEditTime: "200"},
|
||||
@@ -478,24 +484,44 @@ func TestConnectorListResources_IncludesWikiNodeTree(t *testing.T) {
|
||||
ts, cfg := fakeFeishuHierarchy(topNodes, childNodes, "")
|
||||
defer ts.Close()
|
||||
|
||||
resources, err := NewConnector().ListResources(context.Background(), makeConfig(cfg, nil))
|
||||
c := NewConnector()
|
||||
|
||||
// Root listing: only the space, no descendants.
|
||||
spaces, err := c.ListResources(context.Background(), makeConfig(cfg, nil), "")
|
||||
if err != nil {
|
||||
t.Fatalf("ListResources() error: %v", err)
|
||||
t.Fatalf("ListResources(root) error: %v", err)
|
||||
}
|
||||
if len(resources) != 4 {
|
||||
t.Fatalf("want 4 resources (space + 3 nodes), got %d: %+v", len(resources), resources)
|
||||
if len(spaces) != 1 || spaces[0].ExternalID != "space1" || !spaces[0].HasChildren {
|
||||
t.Fatalf("want single space with children, got %+v", spaces)
|
||||
}
|
||||
|
||||
// Expanding the space returns only its top-level nodes.
|
||||
top, err := c.ListResources(context.Background(), makeConfig(cfg, nil), "space1")
|
||||
if err != nil {
|
||||
t.Fatalf("ListResources(space1) error: %v", err)
|
||||
}
|
||||
if len(top) != 2 {
|
||||
t.Fatalf("want 2 top-level nodes, got %d: %+v", len(top), top)
|
||||
}
|
||||
byID := make(map[string]types.Resource)
|
||||
for _, resource := range resources {
|
||||
byID[resource.ExternalID] = resource
|
||||
for _, r := range top {
|
||||
byID[r.ExternalID] = r
|
||||
}
|
||||
root := byID["space1:nt-root"]
|
||||
if root.ParentID != "space1" || !root.HasChildren {
|
||||
t.Fatalf("root resource wrong: %+v", root)
|
||||
t.Fatalf("root node resource wrong: %+v", root)
|
||||
}
|
||||
child := byID["space1:nt-child"]
|
||||
if child.ParentID != "space1:nt-root" || child.Name != "Child" {
|
||||
|
||||
// Expanding a node returns only its direct children.
|
||||
children, err := c.ListResources(context.Background(), makeConfig(cfg, nil), "space1:nt-root")
|
||||
if err != nil {
|
||||
t.Fatalf("ListResources(space1:nt-root) error: %v", err)
|
||||
}
|
||||
if len(children) != 1 {
|
||||
t.Fatalf("want 1 child node, got %d: %+v", len(children), children)
|
||||
}
|
||||
child := children[0]
|
||||
if child.ExternalID != "space1:nt-child" || child.ParentID != "space1:nt-root" || child.Name != "Child" {
|
||||
t.Fatalf("child resource wrong: %+v", child)
|
||||
}
|
||||
if child.Metadata["space_id"] != "space1" || child.Metadata["node_token"] != "nt-child" {
|
||||
|
||||
Reference in New Issue
Block a user