From d8f13ada4a34b9fe451018af9f791816dad841a9 Mon Sep 17 00:00:00 2001 From: wizardchen Date: Mon, 22 Jun 2026 17:41:18 +0800 Subject: [PATCH] fix(datasource): lazy-load Feishu wiki resources to avoid listing timeout (#1672) Listing resources for a Feishu data source eagerly walked the entire wiki tree (one API call per node across every space), which exceeded the request timeout for large knowledge bases. ListResources now takes a parentID and returns a single level at a time: empty lists the wiki spaces, a space/node ID lists only its direct children. The frontend fetches children on demand when a node is expanded. Notion (full tree in one call) and Yuque (flat) keep their existing behavior by ignoring non-empty parentID. Refs: https://github.com/Tencent/WeKnora/issues/1672 --- frontend/src/api/datasource/index.ts | 8 +- .../settings/DataSourceEditorDialog.vue | 79 +++++++++++++++++-- .../application/service/datasource_service.go | 10 ++- internal/datasource/connector.go | 9 ++- .../datasource/connector/feishu/connector.go | 70 +++++++++------- .../connector/feishu/connector_test.go | 48 ++++++++--- .../datasource/connector/notion/connector.go | 13 ++- .../connector/notion/connector_test.go | 2 +- .../datasource/connector/yuque/connector.go | 10 ++- .../connector/yuque/connector_test.go | 6 +- internal/handler/datasource.go | 6 +- internal/types/interfaces/datasource.go | 5 +- 12 files changed, 204 insertions(+), 62 deletions(-) diff --git a/frontend/src/api/datasource/index.ts b/frontend/src/api/datasource/index.ts index 1858f6b3..ee42b898 100644 --- a/frontend/src/api/datasource/index.ts +++ b/frontend/src/api/datasource/index.ts @@ -95,8 +95,12 @@ export function validateCredentials(type: string, credentials: Record([]) const loadingResources = ref(false) const selectedResourceIds = ref([]) const expandedResourceIds = ref(new Set()) +// Lazy loading: parents whose children have already been fetched, and parents +// currently being fetched. Used to load hierarchical sources (e.g. Feishu wiki) +// one level at a time instead of traversing the whole tree up front (#1672). +const loadedChildrenIds = ref(new Set()) +const loadingChildrenIds = ref(new Set()) // Shared children/parent indexes — used by tree rendering and selection logic const childrenMap = computed(() => { @@ -188,9 +193,51 @@ const checkStates = computed(() => { function toggleExpand(id: string) { const next = new Set(expandedResourceIds.value) - if (next.has(id)) next.delete(id) - else next.add(id) + if (next.has(id)) { + next.delete(id) + expandedResourceIds.value = next + return + } + next.add(id) expandedResourceIds.value = next + void ensureChildrenLoaded(id) +} + +// ensureChildrenLoaded fetches the direct children of a node on demand. It is a +// no-op when the children are already present (e.g. connectors that return the +// full tree in one call) or have already been fetched. +async function ensureChildrenLoaded(id: string) { + if (!tempDsId.value) return + if (loadedChildrenIds.value.has(id) || loadingChildrenIds.value.has(id)) return + if ((childrenMap.value.get(id) || []).length > 0) { + loadedChildrenIds.value = new Set(loadedChildrenIds.value).add(id) + return + } + + loadingChildrenIds.value = new Set(loadingChildrenIds.value).add(id) + try { + const res = await listResources(tempDsId.value, id) + const children: Resource[] = res?.data || res || [] + if (children.length > 0) { + const existing = new Set(resources.value.map(r => r.external_id)) + const merged = resources.value.slice() + for (const c of children) { + if (!existing.has(c.external_id)) merged.push(c) + } + resources.value = merged + } + loadedChildrenIds.value = new Set(loadedChildrenIds.value).add(id) + } catch (e: any) { + MessagePlugin.error(e?.message || e?.error || t('datasource.resourceLoadFailed')) + // Collapse again so the user can retry the expand. + const next = new Set(expandedResourceIds.value) + next.delete(id) + expandedResourceIds.value = next + } finally { + const s = new Set(loadingChildrenIds.value) + s.delete(id) + loadingChildrenIds.value = s + } } const visibleTree = computed(() => { @@ -310,6 +357,9 @@ watch(visible, async (v) => { pendingRemoveCredentials.value = false resources.value = [] selectedResourceIds.value = [] + expandedResourceIds.value = new Set() + loadedChildrenIds.value = new Set() + loadingChildrenIds.value = new Set() if (isEdit.value && props.dataSource) { // Reset edit/replace toggle every open so an aborted replace doesn't @@ -424,8 +474,20 @@ async function loadResources() { const res = await listResources(tempDsId.value) resources.value = res?.data || res || [] + // Any parent that already arrived with children (connectors returning the + // full tree, e.g. Notion) needs no further lazy fetch. + const parentsWithChildren = new Set() + for (const r of resources.value) { + if (r.parent_id) parentsWithChildren.add(r.parent_id) + } + loadedChildrenIds.value = parentsWithChildren + loadingChildrenIds.value = new Set() + // Auto-expand top-level nodes whose children are already loaded; lazy nodes + // (children not yet fetched) stay collapsed until the user expands them. expandedResourceIds.value = new Set( - resources.value.filter(r => !r.parent_id && r.has_children).map(r => r.external_id), + resources.value + .filter(r => !r.parent_id && r.has_children && parentsWithChildren.has(r.external_id)) + .map(r => r.external_id), ) } catch (e: any) { MessagePlugin.error(e?.message || e?.error || t('datasource.resourceLoadFailed')) @@ -665,9 +727,12 @@ function resourceIconName(r: Resource): string { } function expandAllNodes() { - expandedResourceIds.value = new Set( - resources.value.filter(r => r.has_children).map(r => r.external_id), - ) + const expandable = resources.value.filter(r => r.has_children) + expandedResourceIds.value = new Set(expandable.map(r => r.external_id)) + // Lazily load children of any expanded node that hasn't been fetched yet. + for (const r of expandable) { + void ensureChildrenLoaded(r.external_id) + } } function collapseAllNodes() { @@ -1022,7 +1087,9 @@ const drawerConfirmText = computed(() => { : t('knowledgeStages.expandBranch')" @click.stop="toggleExpand(r.external_id)" > + diff --git a/internal/application/service/datasource_service.go b/internal/application/service/datasource_service.go index 4dc82611..2798da1f 100644 --- a/internal/application/service/datasource_service.go +++ b/internal/application/service/datasource_service.go @@ -348,8 +348,12 @@ func (s *DataSourceService) ValidateConnection(ctx context.Context, dsID string) return nil } -// ListAvailableResources lists resources available for sync in the external system -func (s *DataSourceService) ListAvailableResources(ctx context.Context, dsID string) ([]types.Resource, error) { +// ListAvailableResources lists resources available for sync in the external system. +// parentID enables lazy (on-demand) loading of hierarchical resources: pass "" to +// list the top level, or a resource's ExternalID to list only its direct children. +func (s *DataSourceService) ListAvailableResources( + ctx context.Context, dsID string, parentID string, +) ([]types.Resource, error) { ds, err := s.GetDataSource(ctx, dsID) if err != nil { return nil, err @@ -368,7 +372,7 @@ func (s *DataSourceService) ListAvailableResources(ctx context.Context, dsID str } // List resources - resources, err := connector.ListResources(ctx, config) + resources, err := connector.ListResources(ctx, config, parentID) if err != nil { logger.Errorf(ctx, "failed to list resources: %v", err) return nil, err diff --git a/internal/datasource/connector.go b/internal/datasource/connector.go index 96c356a3..e6542980 100644 --- a/internal/datasource/connector.go +++ b/internal/datasource/connector.go @@ -18,7 +18,14 @@ type Connector interface { // ListResources lists available resources that can be synced (documents, spaces, folders, etc.) // Returns a list of Resource objects that the user can select for syncing. - ListResources(ctx context.Context, config *types.DataSourceConfig) ([]types.Resource, error) + // + // parentID controls lazy (on-demand) loading of hierarchical resources: + // - parentID == "" → return the top-level resources (e.g. Feishu wiki spaces). + // - parentID != "" → return only the direct children of that resource. + // Connectors whose listing is already flat or returns the full tree in a single + // call may ignore parentID for the root call and return an empty slice for any + // non-empty parentID. + ListResources(ctx context.Context, config *types.DataSourceConfig, parentID string) ([]types.Resource, error) // FetchAll performs a full sync of the specified resources. // Returns all items from the given resource IDs. diff --git a/internal/datasource/connector/feishu/connector.go b/internal/datasource/connector/feishu/connector.go index 0c9ce672..422f24bc 100644 --- a/internal/datasource/connector/feishu/connector.go +++ b/internal/datasource/connector/feishu/connector.go @@ -43,46 +43,60 @@ func (c *Connector) Validate(ctx context.Context, config *types.DataSourceConfig return nil } -// ListResources lists all accessible Feishu Wiki spaces and nodes as selectable resources. -func (c *Connector) ListResources(ctx context.Context, config *types.DataSourceConfig) ([]types.Resource, error) { +// ListResources lists Feishu Wiki resources for selection, loading the tree +// lazily one level at a time to avoid traversing the entire wiki up front. +// +// - parentID == "" → list all accessible wiki spaces. +// - parentID == spaceID → list the top-level nodes of that space. +// - parentID == "spaceID:nodeToken" → list the direct children of that node. +// +// Eagerly recursing the whole tree here used to time out for large wikis +// (Tencent/WeKnora#1672); the recursive walk now happens only at sync time. +func (c *Connector) ListResources( + ctx context.Context, config *types.DataSourceConfig, parentID string, +) ([]types.Resource, error) { feishuConfig, err := parseFeishuConfig(config) if err != nil { return nil, err } client := NewClient(feishuConfig) - spaces, err := client.ListWikiSpaces(ctx) - if err != nil { - return nil, fmt.Errorf("list feishu wiki spaces: %w", err) - } - resources := make([]types.Resource, 0, len(spaces)) - for _, space := range spaces { - resources = append(resources, types.Resource{ - ExternalID: space.SpaceID, - Name: space.Name, - Type: "wiki_space", - Description: space.Description, - URL: fmt.Sprintf("https://feishu.cn/wiki/%s", space.SpaceID), - HasChildren: true, - Metadata: map[string]interface{}{ - "visibility": space.Visibility, - "space_id": space.SpaceID, - }, - }) - - nodes, err := client.ListAllWikiNodesRecursive(ctx, space.SpaceID) + if parentID == "" { + spaces, err := client.ListWikiSpaces(ctx) if err != nil { - var partialErr *partialWikiNodeListError - if !errors.As(err, &partialErr) { - return nil, fmt.Errorf("list feishu wiki nodes in space %s: %w", space.SpaceID, err) - } + return nil, fmt.Errorf("list feishu wiki spaces: %w", err) } - for _, node := range nodes { - resources = append(resources, wikiNodeToResource(space.SpaceID, node)) + + resources := make([]types.Resource, 0, len(spaces)) + for _, space := range spaces { + resources = append(resources, types.Resource{ + ExternalID: space.SpaceID, + Name: space.Name, + Type: "wiki_space", + Description: space.Description, + URL: fmt.Sprintf("https://feishu.cn/wiki/%s", space.SpaceID), + HasChildren: true, + Metadata: map[string]interface{}{ + "visibility": space.Visibility, + "space_id": space.SpaceID, + }, + }) } + return resources, nil } + // Lazy load: list only the direct children of the given space / node. + spaceID, nodeToken := parseWikiResourceID(parentID) + nodes, err := client.ListWikiNodes(ctx, spaceID, nodeToken) + if err != nil { + return nil, fmt.Errorf("list feishu wiki nodes under %s: %w", parentID, err) + } + + resources := make([]types.Resource, 0, len(nodes)) + for _, node := range nodes { + resources = append(resources, wikiNodeToResource(spaceID, node)) + } return resources, nil } diff --git a/internal/datasource/connector/feishu/connector_test.go b/internal/datasource/connector/feishu/connector_test.go index 1e70bbf7..317eb78e 100644 --- a/internal/datasource/connector/feishu/connector_test.go +++ b/internal/datasource/connector/feishu/connector_test.go @@ -446,7 +446,7 @@ func TestConnectorListResources(t *testing.T) { defer ts.Close() c := NewConnector() - resources, err := c.ListResources(context.Background(), makeConfig(cfg, nil)) + resources, err := c.ListResources(context.Background(), makeConfig(cfg, nil), "") if err != nil { t.Fatalf("ListResources() error: %v", err) } @@ -463,9 +463,15 @@ func TestConnectorListResources(t *testing.T) { if resources[0].Type != "wiki_space" { t.Errorf("Type = %q, want %q", resources[0].Type, "wiki_space") } + if !resources[0].HasChildren { + t.Errorf("HasChildren = false, want true") + } } -func TestConnectorListResources_IncludesWikiNodeTree(t *testing.T) { +// TestConnectorListResources_LazyLoadsOneLevel verifies that ListResources loads +// the wiki tree lazily — only the requested level — instead of recursing the whole +// tree up front (Tencent/WeKnora#1672). +func TestConnectorListResources_LazyLoadsOneLevel(t *testing.T) { topNodes := []wikiNode{ {NodeToken: "nt-root", ObjToken: "obj-root", ObjType: "docx", Title: "Root", HasChild: true, ObjEditTime: "100"}, {NodeToken: "nt-peer", ObjToken: "obj-peer", ObjType: "docx", Title: "Peer", ObjEditTime: "200"}, @@ -478,24 +484,44 @@ func TestConnectorListResources_IncludesWikiNodeTree(t *testing.T) { ts, cfg := fakeFeishuHierarchy(topNodes, childNodes, "") defer ts.Close() - resources, err := NewConnector().ListResources(context.Background(), makeConfig(cfg, nil)) + c := NewConnector() + + // Root listing: only the space, no descendants. + spaces, err := c.ListResources(context.Background(), makeConfig(cfg, nil), "") if err != nil { - t.Fatalf("ListResources() error: %v", err) + t.Fatalf("ListResources(root) error: %v", err) } - if len(resources) != 4 { - t.Fatalf("want 4 resources (space + 3 nodes), got %d: %+v", len(resources), resources) + if len(spaces) != 1 || spaces[0].ExternalID != "space1" || !spaces[0].HasChildren { + t.Fatalf("want single space with children, got %+v", spaces) } + // Expanding the space returns only its top-level nodes. + top, err := c.ListResources(context.Background(), makeConfig(cfg, nil), "space1") + if err != nil { + t.Fatalf("ListResources(space1) error: %v", err) + } + if len(top) != 2 { + t.Fatalf("want 2 top-level nodes, got %d: %+v", len(top), top) + } byID := make(map[string]types.Resource) - for _, resource := range resources { - byID[resource.ExternalID] = resource + for _, r := range top { + byID[r.ExternalID] = r } root := byID["space1:nt-root"] if root.ParentID != "space1" || !root.HasChildren { - t.Fatalf("root resource wrong: %+v", root) + t.Fatalf("root node resource wrong: %+v", root) } - child := byID["space1:nt-child"] - if child.ParentID != "space1:nt-root" || child.Name != "Child" { + + // Expanding a node returns only its direct children. + children, err := c.ListResources(context.Background(), makeConfig(cfg, nil), "space1:nt-root") + if err != nil { + t.Fatalf("ListResources(space1:nt-root) error: %v", err) + } + if len(children) != 1 { + t.Fatalf("want 1 child node, got %d: %+v", len(children), children) + } + child := children[0] + if child.ExternalID != "space1:nt-child" || child.ParentID != "space1:nt-root" || child.Name != "Child" { t.Fatalf("child resource wrong: %+v", child) } if child.Metadata["space_id"] != "space1" || child.Metadata["node_token"] != "nt-child" { diff --git a/internal/datasource/connector/notion/connector.go b/internal/datasource/connector/notion/connector.go index 487b3a8e..59b42781 100644 --- a/internal/datasource/connector/notion/connector.go +++ b/internal/datasource/connector/notion/connector.go @@ -47,7 +47,16 @@ func (c *Connector) Validate(ctx context.Context, config *types.DataSourceConfig // ListResources lists all accessible Notion pages and databases as selectable resources. // Returns all objects with parent-child relationships populated, allowing the frontend // to render a tree view. Root objects have empty ParentID. -func (c *Connector) ListResources(ctx context.Context, config *types.DataSourceConfig) ([]types.Resource, error) { +func (c *Connector) ListResources( + ctx context.Context, config *types.DataSourceConfig, parentID string, +) ([]types.Resource, error) { + // Notion returns the full hierarchy (with parent_id populated) in a single + // call, so children are already delivered with the root listing. Lazy-load + // requests for a specific parent therefore have nothing extra to return. + if parentID != "" { + return []types.Resource{}, nil + } + notionCfg, err := parseNotionConfig(config) if err != nil { return nil, err @@ -903,7 +912,7 @@ func computeExcludedSet(visibleIDs []string, parentOf map[string]string, selecte // and computes the deselected set. Used by FetchAll where no other code path // already has the page list in hand. func (c *Connector) excludedSetFromListResources(ctx context.Context, config *types.DataSourceConfig, selectedIDs []string) map[string]bool { - visible, err := c.ListResources(ctx, config) + visible, err := c.ListResources(ctx, config, "") if err != nil { logger.Warnf(ctx, "[Notion] failed to list visible resources for exclusion: %v", err) return map[string]bool{} diff --git a/internal/datasource/connector/notion/connector_test.go b/internal/datasource/connector/notion/connector_test.go index 5846e085..741b25d4 100644 --- a/internal/datasource/connector/notion/connector_test.go +++ b/internal/datasource/connector/notion/connector_test.go @@ -57,7 +57,7 @@ func TestConnectorListResources(t *testing.T) { defer ts.Close() c := NewConnector() - resources, err := c.ListResources(context.Background(), makeNotionConfig(cfg, ts.URL, nil)) + resources, err := c.ListResources(context.Background(), makeNotionConfig(cfg, ts.URL, nil), "") if err != nil { t.Fatalf("ListResources() error: %v", err) } diff --git a/internal/datasource/connector/yuque/connector.go b/internal/datasource/connector/yuque/connector.go index 04605758..1e4393e0 100644 --- a/internal/datasource/connector/yuque/connector.go +++ b/internal/datasource/connector/yuque/connector.go @@ -42,7 +42,15 @@ func (c *Connector) Validate(ctx context.Context, config *types.DataSourceConfig // ListResources returns all repos (personal + team) accessible to the token. // Serial fetch for v1 (user groups typically <10). TODO(perf): parallelize if slow. -func (c *Connector) ListResources(ctx context.Context, config *types.DataSourceConfig) ([]types.Resource, error) { +func (c *Connector) ListResources( + ctx context.Context, config *types.DataSourceConfig, parentID string, +) ([]types.Resource, error) { + // Yuque resources are a flat list of repositories (no nesting), so a + // lazy-load request for a specific parent has nothing extra to return. + if parentID != "" { + return []types.Resource{}, nil + } + cfg, err := parseYuqueConfig(config) if err != nil { return nil, err diff --git a/internal/datasource/connector/yuque/connector_test.go b/internal/datasource/connector/yuque/connector_test.go index 9db69cbe..e38dcb05 100644 --- a/internal/datasource/connector/yuque/connector_test.go +++ b/internal/datasource/connector/yuque/connector_test.go @@ -69,7 +69,7 @@ func TestConnector_ListResources_Aggregates(t *testing.T) { {ID: 30, Slug: "bb", Name: "B Book", Type: "Book", Namespace: "team-b/bb"}, }}) - resources, err := NewConnector().ListResources(context.Background(), makeDSConfig(f, nil)) + resources, err := NewConnector().ListResources(context.Background(), makeDSConfig(f, nil), "") if err != nil { t.Fatalf("ListResources: %v", err) } @@ -108,7 +108,7 @@ func TestConnector_ListResources_DedupByID(t *testing.T) { {ID: 99, Slug: "shared", Type: "Book", Namespace: "alice/shared"}, }}) - resources, err := NewConnector().ListResources(context.Background(), makeDSConfig(f, nil)) + resources, err := NewConnector().ListResources(context.Background(), makeDSConfig(f, nil), "") if err != nil { t.Fatalf("ListResources: %v", err) } @@ -457,7 +457,7 @@ func TestConnector_ListResources_ContinuesOnGroupFailure(t *testing.T) { _, _ = w.Write([]byte(`{"message":"forbidden"}`)) }) - resources, err := NewConnector().ListResources(context.Background(), makeDSConfig(f, nil)) + resources, err := NewConnector().ListResources(context.Background(), makeDSConfig(f, nil), "") if err != nil { t.Fatalf("ListResources should not fail on per-group error, got %v", err) } diff --git a/internal/handler/datasource.go b/internal/handler/datasource.go index 89d1b72e..be49975f 100644 --- a/internal/handler/datasource.go +++ b/internal/handler/datasource.go @@ -330,10 +330,11 @@ func (h *DataSourceHandler) ValidateCredentials(c *gin.Context) { } // @Summary List available resources in data source -// @Description List resources available for sync in the external system +// @Description List resources available for sync in the external system. Pass parent_id to lazily load the direct children of a resource (used for large hierarchical sources such as Feishu wiki). // @Tags DataSource // @Produce json // @Param id path string true "Data source ID" +// @Param parent_id query string false "Parent resource ExternalID; empty lists the top level" // @Success 200 {object} []types.Resource // @Failure 400 {object} map[string]string // @Router /datasource/{id}/resources [get] @@ -346,13 +347,14 @@ func (h *DataSourceHandler) ListAvailableResources(c *gin.Context) { } id := c.Param("id") + parentID := c.Query("parent_id") if _, status, msg := h.getOwnedDataSource(ctx, tenantID, id); status != http.StatusOK { c.JSON(status, gin.H{"error": msg}) return } - resources, err := h.service.ListAvailableResources(ctx, id) + resources, err := h.service.ListAvailableResources(ctx, id, parentID) if err != nil { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) return diff --git a/internal/types/interfaces/datasource.go b/internal/types/interfaces/datasource.go index f9235a84..c84e86cd 100644 --- a/internal/types/interfaces/datasource.go +++ b/internal/types/interfaces/datasource.go @@ -43,8 +43,9 @@ type DataSourceService interface { // This is used by the frontend "Test Connection" button before creating a data source. ValidateCredentials(ctx context.Context, connectorType string, credentials map[string]interface{}) error - // ListAvailableResources lists resources available for sync in the external system - ListAvailableResources(ctx context.Context, dsID string) ([]types.Resource, error) + // ListAvailableResources lists resources available for sync in the external system. + // parentID enables lazy loading: "" lists the top level, a resource ExternalID lists its children. + ListAvailableResources(ctx context.Context, dsID string, parentID string) ([]types.Resource, error) // ManualSync triggers an immediate sync for a data source ManualSync(ctx context.Context, dsID string) (*types.SyncLog, error)