fix(datasource): lazy-load Feishu wiki resources to avoid listing timeout (#1672)

Listing resources for a Feishu data source eagerly walked the entire wiki
tree (one API call per node across every space), which exceeded the request
timeout for large knowledge bases. ListResources now takes a parentID and
returns a single level at a time: empty lists the wiki spaces, a space/node
ID lists only its direct children. The frontend fetches children on demand
when a node is expanded. Notion (full tree in one call) and Yuque (flat)
keep their existing behavior by ignoring non-empty parentID.

Refs: https://github.com/Tencent/WeKnora/issues/1672
This commit is contained in:
wizardchen
2026-06-22 17:41:18 +08:00
committed by lyingbug
parent 74c845a1f4
commit d8f13ada4a
12 changed files with 204 additions and 62 deletions
+6 -2
View File
@@ -95,8 +95,12 @@ export function validateCredentials(type: string, credentials: Record<string, an
return post('/api/v1/datasource/validate-credentials', { type, credentials })
}
export function listResources(id: string) {
return get(`/api/v1/datasource/${id}/resources`, { timeout: 120000 })
// listResources lists selectable resources for a data source. Pass parentId to
// lazily load the direct children of a resource (e.g. expanding a Feishu wiki
// space/node), which avoids traversing the whole tree up front.
export function listResources(id: string, parentId?: string) {
const query = parentId ? `?parent_id=${encodeURIComponent(parentId)}` : ''
return get(`/api/v1/datasource/${id}/resources${query}`, { timeout: 120000 })
}
export function triggerSync(id: string) {
@@ -136,6 +136,11 @@ const resources = ref<Resource[]>([])
const loadingResources = ref(false)
const selectedResourceIds = ref<string[]>([])
const expandedResourceIds = ref(new Set<string>())
// Lazy loading: parents whose children have already been fetched, and parents
// currently being fetched. Used to load hierarchical sources (e.g. Feishu wiki)
// one level at a time instead of traversing the whole tree up front (#1672).
const loadedChildrenIds = ref(new Set<string>())
const loadingChildrenIds = ref(new Set<string>())
// Shared children/parent indexes — used by tree rendering and selection logic
const childrenMap = computed(() => {
@@ -188,9 +193,51 @@ const checkStates = computed(() => {
function toggleExpand(id: string) {
const next = new Set(expandedResourceIds.value)
if (next.has(id)) next.delete(id)
else next.add(id)
if (next.has(id)) {
next.delete(id)
expandedResourceIds.value = next
return
}
next.add(id)
expandedResourceIds.value = next
void ensureChildrenLoaded(id)
}
// ensureChildrenLoaded fetches the direct children of a node on demand. It is a
// no-op when the children are already present (e.g. connectors that return the
// full tree in one call) or have already been fetched.
async function ensureChildrenLoaded(id: string) {
if (!tempDsId.value) return
if (loadedChildrenIds.value.has(id) || loadingChildrenIds.value.has(id)) return
if ((childrenMap.value.get(id) || []).length > 0) {
loadedChildrenIds.value = new Set(loadedChildrenIds.value).add(id)
return
}
loadingChildrenIds.value = new Set(loadingChildrenIds.value).add(id)
try {
const res = await listResources(tempDsId.value, id)
const children: Resource[] = res?.data || res || []
if (children.length > 0) {
const existing = new Set(resources.value.map(r => r.external_id))
const merged = resources.value.slice()
for (const c of children) {
if (!existing.has(c.external_id)) merged.push(c)
}
resources.value = merged
}
loadedChildrenIds.value = new Set(loadedChildrenIds.value).add(id)
} catch (e: any) {
MessagePlugin.error(e?.message || e?.error || t('datasource.resourceLoadFailed'))
// Collapse again so the user can retry the expand.
const next = new Set(expandedResourceIds.value)
next.delete(id)
expandedResourceIds.value = next
} finally {
const s = new Set(loadingChildrenIds.value)
s.delete(id)
loadingChildrenIds.value = s
}
}
const visibleTree = computed(() => {
@@ -310,6 +357,9 @@ watch(visible, async (v) => {
pendingRemoveCredentials.value = false
resources.value = []
selectedResourceIds.value = []
expandedResourceIds.value = new Set()
loadedChildrenIds.value = new Set()
loadingChildrenIds.value = new Set()
if (isEdit.value && props.dataSource) {
// Reset edit/replace toggle every open so an aborted replace doesn't
@@ -424,8 +474,20 @@ async function loadResources() {
const res = await listResources(tempDsId.value)
resources.value = res?.data || res || []
// Any parent that already arrived with children (connectors returning the
// full tree, e.g. Notion) needs no further lazy fetch.
const parentsWithChildren = new Set<string>()
for (const r of resources.value) {
if (r.parent_id) parentsWithChildren.add(r.parent_id)
}
loadedChildrenIds.value = parentsWithChildren
loadingChildrenIds.value = new Set<string>()
// Auto-expand top-level nodes whose children are already loaded; lazy nodes
// (children not yet fetched) stay collapsed until the user expands them.
expandedResourceIds.value = new Set(
resources.value.filter(r => !r.parent_id && r.has_children).map(r => r.external_id),
resources.value
.filter(r => !r.parent_id && r.has_children && parentsWithChildren.has(r.external_id))
.map(r => r.external_id),
)
} catch (e: any) {
MessagePlugin.error(e?.message || e?.error || t('datasource.resourceLoadFailed'))
@@ -665,9 +727,12 @@ function resourceIconName(r: Resource): string {
}
function expandAllNodes() {
expandedResourceIds.value = new Set(
resources.value.filter(r => r.has_children).map(r => r.external_id),
)
const expandable = resources.value.filter(r => r.has_children)
expandedResourceIds.value = new Set(expandable.map(r => r.external_id))
// Lazily load children of any expanded node that hasn't been fetched yet.
for (const r of expandable) {
void ensureChildrenLoaded(r.external_id)
}
}
function collapseAllNodes() {
@@ -1022,7 +1087,9 @@ const drawerConfirmText = computed(() => {
: t('knowledgeStages.expandBranch')"
@click.stop="toggleExpand(r.external_id)"
>
<t-loading v-if="loadingChildrenIds.has(r.external_id)" size="12px" />
<t-icon
v-else
:name="expandedResourceIds.has(r.external_id) ? 'chevron-down' : 'chevron-right'"
size="12px"
/>
@@ -348,8 +348,12 @@ func (s *DataSourceService) ValidateConnection(ctx context.Context, dsID string)
return nil
}
// ListAvailableResources lists resources available for sync in the external system
func (s *DataSourceService) ListAvailableResources(ctx context.Context, dsID string) ([]types.Resource, error) {
// ListAvailableResources lists resources available for sync in the external system.
// parentID enables lazy (on-demand) loading of hierarchical resources: pass "" to
// list the top level, or a resource's ExternalID to list only its direct children.
func (s *DataSourceService) ListAvailableResources(
ctx context.Context, dsID string, parentID string,
) ([]types.Resource, error) {
ds, err := s.GetDataSource(ctx, dsID)
if err != nil {
return nil, err
@@ -368,7 +372,7 @@ func (s *DataSourceService) ListAvailableResources(ctx context.Context, dsID str
}
// List resources
resources, err := connector.ListResources(ctx, config)
resources, err := connector.ListResources(ctx, config, parentID)
if err != nil {
logger.Errorf(ctx, "failed to list resources: %v", err)
return nil, err
+8 -1
View File
@@ -18,7 +18,14 @@ type Connector interface {
// ListResources lists available resources that can be synced (documents, spaces, folders, etc.)
// Returns a list of Resource objects that the user can select for syncing.
ListResources(ctx context.Context, config *types.DataSourceConfig) ([]types.Resource, error)
//
// parentID controls lazy (on-demand) loading of hierarchical resources:
// - parentID == "" → return the top-level resources (e.g. Feishu wiki spaces).
// - parentID != "" → return only the direct children of that resource.
// Connectors whose listing is already flat or returns the full tree in a single
// call may ignore parentID for the root call and return an empty slice for any
// non-empty parentID.
ListResources(ctx context.Context, config *types.DataSourceConfig, parentID string) ([]types.Resource, error)
// FetchAll performs a full sync of the specified resources.
// Returns all items from the given resource IDs.
@@ -43,46 +43,60 @@ func (c *Connector) Validate(ctx context.Context, config *types.DataSourceConfig
return nil
}
// ListResources lists all accessible Feishu Wiki spaces and nodes as selectable resources.
func (c *Connector) ListResources(ctx context.Context, config *types.DataSourceConfig) ([]types.Resource, error) {
// ListResources lists Feishu Wiki resources for selection, loading the tree
// lazily one level at a time to avoid traversing the entire wiki up front.
//
// - parentID == "" → list all accessible wiki spaces.
// - parentID == spaceID → list the top-level nodes of that space.
// - parentID == "spaceID:nodeToken" → list the direct children of that node.
//
// Eagerly recursing the whole tree here used to time out for large wikis
// (Tencent/WeKnora#1672); the recursive walk now happens only at sync time.
func (c *Connector) ListResources(
ctx context.Context, config *types.DataSourceConfig, parentID string,
) ([]types.Resource, error) {
feishuConfig, err := parseFeishuConfig(config)
if err != nil {
return nil, err
}
client := NewClient(feishuConfig)
spaces, err := client.ListWikiSpaces(ctx)
if err != nil {
return nil, fmt.Errorf("list feishu wiki spaces: %w", err)
}
resources := make([]types.Resource, 0, len(spaces))
for _, space := range spaces {
resources = append(resources, types.Resource{
ExternalID: space.SpaceID,
Name: space.Name,
Type: "wiki_space",
Description: space.Description,
URL: fmt.Sprintf("https://feishu.cn/wiki/%s", space.SpaceID),
HasChildren: true,
Metadata: map[string]interface{}{
"visibility": space.Visibility,
"space_id": space.SpaceID,
},
})
nodes, err := client.ListAllWikiNodesRecursive(ctx, space.SpaceID)
if parentID == "" {
spaces, err := client.ListWikiSpaces(ctx)
if err != nil {
var partialErr *partialWikiNodeListError
if !errors.As(err, &partialErr) {
return nil, fmt.Errorf("list feishu wiki nodes in space %s: %w", space.SpaceID, err)
}
return nil, fmt.Errorf("list feishu wiki spaces: %w", err)
}
for _, node := range nodes {
resources = append(resources, wikiNodeToResource(space.SpaceID, node))
resources := make([]types.Resource, 0, len(spaces))
for _, space := range spaces {
resources = append(resources, types.Resource{
ExternalID: space.SpaceID,
Name: space.Name,
Type: "wiki_space",
Description: space.Description,
URL: fmt.Sprintf("https://feishu.cn/wiki/%s", space.SpaceID),
HasChildren: true,
Metadata: map[string]interface{}{
"visibility": space.Visibility,
"space_id": space.SpaceID,
},
})
}
return resources, nil
}
// Lazy load: list only the direct children of the given space / node.
spaceID, nodeToken := parseWikiResourceID(parentID)
nodes, err := client.ListWikiNodes(ctx, spaceID, nodeToken)
if err != nil {
return nil, fmt.Errorf("list feishu wiki nodes under %s: %w", parentID, err)
}
resources := make([]types.Resource, 0, len(nodes))
for _, node := range nodes {
resources = append(resources, wikiNodeToResource(spaceID, node))
}
return resources, nil
}
@@ -446,7 +446,7 @@ func TestConnectorListResources(t *testing.T) {
defer ts.Close()
c := NewConnector()
resources, err := c.ListResources(context.Background(), makeConfig(cfg, nil))
resources, err := c.ListResources(context.Background(), makeConfig(cfg, nil), "")
if err != nil {
t.Fatalf("ListResources() error: %v", err)
}
@@ -463,9 +463,15 @@ func TestConnectorListResources(t *testing.T) {
if resources[0].Type != "wiki_space" {
t.Errorf("Type = %q, want %q", resources[0].Type, "wiki_space")
}
if !resources[0].HasChildren {
t.Errorf("HasChildren = false, want true")
}
}
func TestConnectorListResources_IncludesWikiNodeTree(t *testing.T) {
// TestConnectorListResources_LazyLoadsOneLevel verifies that ListResources loads
// the wiki tree lazily — only the requested level — instead of recursing the whole
// tree up front (Tencent/WeKnora#1672).
func TestConnectorListResources_LazyLoadsOneLevel(t *testing.T) {
topNodes := []wikiNode{
{NodeToken: "nt-root", ObjToken: "obj-root", ObjType: "docx", Title: "Root", HasChild: true, ObjEditTime: "100"},
{NodeToken: "nt-peer", ObjToken: "obj-peer", ObjType: "docx", Title: "Peer", ObjEditTime: "200"},
@@ -478,24 +484,44 @@ func TestConnectorListResources_IncludesWikiNodeTree(t *testing.T) {
ts, cfg := fakeFeishuHierarchy(topNodes, childNodes, "")
defer ts.Close()
resources, err := NewConnector().ListResources(context.Background(), makeConfig(cfg, nil))
c := NewConnector()
// Root listing: only the space, no descendants.
spaces, err := c.ListResources(context.Background(), makeConfig(cfg, nil), "")
if err != nil {
t.Fatalf("ListResources() error: %v", err)
t.Fatalf("ListResources(root) error: %v", err)
}
if len(resources) != 4 {
t.Fatalf("want 4 resources (space + 3 nodes), got %d: %+v", len(resources), resources)
if len(spaces) != 1 || spaces[0].ExternalID != "space1" || !spaces[0].HasChildren {
t.Fatalf("want single space with children, got %+v", spaces)
}
// Expanding the space returns only its top-level nodes.
top, err := c.ListResources(context.Background(), makeConfig(cfg, nil), "space1")
if err != nil {
t.Fatalf("ListResources(space1) error: %v", err)
}
if len(top) != 2 {
t.Fatalf("want 2 top-level nodes, got %d: %+v", len(top), top)
}
byID := make(map[string]types.Resource)
for _, resource := range resources {
byID[resource.ExternalID] = resource
for _, r := range top {
byID[r.ExternalID] = r
}
root := byID["space1:nt-root"]
if root.ParentID != "space1" || !root.HasChildren {
t.Fatalf("root resource wrong: %+v", root)
t.Fatalf("root node resource wrong: %+v", root)
}
child := byID["space1:nt-child"]
if child.ParentID != "space1:nt-root" || child.Name != "Child" {
// Expanding a node returns only its direct children.
children, err := c.ListResources(context.Background(), makeConfig(cfg, nil), "space1:nt-root")
if err != nil {
t.Fatalf("ListResources(space1:nt-root) error: %v", err)
}
if len(children) != 1 {
t.Fatalf("want 1 child node, got %d: %+v", len(children), children)
}
child := children[0]
if child.ExternalID != "space1:nt-child" || child.ParentID != "space1:nt-root" || child.Name != "Child" {
t.Fatalf("child resource wrong: %+v", child)
}
if child.Metadata["space_id"] != "space1" || child.Metadata["node_token"] != "nt-child" {
@@ -47,7 +47,16 @@ func (c *Connector) Validate(ctx context.Context, config *types.DataSourceConfig
// ListResources lists all accessible Notion pages and databases as selectable resources.
// Returns all objects with parent-child relationships populated, allowing the frontend
// to render a tree view. Root objects have empty ParentID.
func (c *Connector) ListResources(ctx context.Context, config *types.DataSourceConfig) ([]types.Resource, error) {
func (c *Connector) ListResources(
ctx context.Context, config *types.DataSourceConfig, parentID string,
) ([]types.Resource, error) {
// Notion returns the full hierarchy (with parent_id populated) in a single
// call, so children are already delivered with the root listing. Lazy-load
// requests for a specific parent therefore have nothing extra to return.
if parentID != "" {
return []types.Resource{}, nil
}
notionCfg, err := parseNotionConfig(config)
if err != nil {
return nil, err
@@ -903,7 +912,7 @@ func computeExcludedSet(visibleIDs []string, parentOf map[string]string, selecte
// and computes the deselected set. Used by FetchAll where no other code path
// already has the page list in hand.
func (c *Connector) excludedSetFromListResources(ctx context.Context, config *types.DataSourceConfig, selectedIDs []string) map[string]bool {
visible, err := c.ListResources(ctx, config)
visible, err := c.ListResources(ctx, config, "")
if err != nil {
logger.Warnf(ctx, "[Notion] failed to list visible resources for exclusion: %v", err)
return map[string]bool{}
@@ -57,7 +57,7 @@ func TestConnectorListResources(t *testing.T) {
defer ts.Close()
c := NewConnector()
resources, err := c.ListResources(context.Background(), makeNotionConfig(cfg, ts.URL, nil))
resources, err := c.ListResources(context.Background(), makeNotionConfig(cfg, ts.URL, nil), "")
if err != nil {
t.Fatalf("ListResources() error: %v", err)
}
@@ -42,7 +42,15 @@ func (c *Connector) Validate(ctx context.Context, config *types.DataSourceConfig
// ListResources returns all repos (personal + team) accessible to the token.
// Serial fetch for v1 (user groups typically <10). TODO(perf): parallelize if slow.
func (c *Connector) ListResources(ctx context.Context, config *types.DataSourceConfig) ([]types.Resource, error) {
func (c *Connector) ListResources(
ctx context.Context, config *types.DataSourceConfig, parentID string,
) ([]types.Resource, error) {
// Yuque resources are a flat list of repositories (no nesting), so a
// lazy-load request for a specific parent has nothing extra to return.
if parentID != "" {
return []types.Resource{}, nil
}
cfg, err := parseYuqueConfig(config)
if err != nil {
return nil, err
@@ -69,7 +69,7 @@ func TestConnector_ListResources_Aggregates(t *testing.T) {
{ID: 30, Slug: "bb", Name: "B Book", Type: "Book", Namespace: "team-b/bb"},
}})
resources, err := NewConnector().ListResources(context.Background(), makeDSConfig(f, nil))
resources, err := NewConnector().ListResources(context.Background(), makeDSConfig(f, nil), "")
if err != nil {
t.Fatalf("ListResources: %v", err)
}
@@ -108,7 +108,7 @@ func TestConnector_ListResources_DedupByID(t *testing.T) {
{ID: 99, Slug: "shared", Type: "Book", Namespace: "alice/shared"},
}})
resources, err := NewConnector().ListResources(context.Background(), makeDSConfig(f, nil))
resources, err := NewConnector().ListResources(context.Background(), makeDSConfig(f, nil), "")
if err != nil {
t.Fatalf("ListResources: %v", err)
}
@@ -457,7 +457,7 @@ func TestConnector_ListResources_ContinuesOnGroupFailure(t *testing.T) {
_, _ = w.Write([]byte(`{"message":"forbidden"}`))
})
resources, err := NewConnector().ListResources(context.Background(), makeDSConfig(f, nil))
resources, err := NewConnector().ListResources(context.Background(), makeDSConfig(f, nil), "")
if err != nil {
t.Fatalf("ListResources should not fail on per-group error, got %v", err)
}
+4 -2
View File
@@ -330,10 +330,11 @@ func (h *DataSourceHandler) ValidateCredentials(c *gin.Context) {
}
// @Summary List available resources in data source
// @Description List resources available for sync in the external system
// @Description List resources available for sync in the external system. Pass parent_id to lazily load the direct children of a resource (used for large hierarchical sources such as Feishu wiki).
// @Tags DataSource
// @Produce json
// @Param id path string true "Data source ID"
// @Param parent_id query string false "Parent resource ExternalID; empty lists the top level"
// @Success 200 {object} []types.Resource
// @Failure 400 {object} map[string]string
// @Router /datasource/{id}/resources [get]
@@ -346,13 +347,14 @@ func (h *DataSourceHandler) ListAvailableResources(c *gin.Context) {
}
id := c.Param("id")
parentID := c.Query("parent_id")
if _, status, msg := h.getOwnedDataSource(ctx, tenantID, id); status != http.StatusOK {
c.JSON(status, gin.H{"error": msg})
return
}
resources, err := h.service.ListAvailableResources(ctx, id)
resources, err := h.service.ListAvailableResources(ctx, id, parentID)
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
+3 -2
View File
@@ -43,8 +43,9 @@ type DataSourceService interface {
// This is used by the frontend "Test Connection" button before creating a data source.
ValidateCredentials(ctx context.Context, connectorType string, credentials map[string]interface{}) error
// ListAvailableResources lists resources available for sync in the external system
ListAvailableResources(ctx context.Context, dsID string) ([]types.Resource, error)
// ListAvailableResources lists resources available for sync in the external system.
// parentID enables lazy loading: "" lists the top level, a resource ExternalID lists its children.
ListAvailableResources(ctx context.Context, dsID string, parentID string) ([]types.Resource, error)
// ManualSync triggers an immediate sync for a data source
ManualSync(ctx context.Context, dsID string) (*types.SyncLog, error)