diff --git a/frontend/src/api/datasource/index.ts b/frontend/src/api/datasource/index.ts index ee42b898..2c4caa8e 100644 --- a/frontend/src/api/datasource/index.ts +++ b/frontend/src/api/datasource/index.ts @@ -103,6 +103,13 @@ export function listResources(id: string, parentId?: string) { return get(`/api/v1/datasource/${id}/resources${query}`, { timeout: 120000 }) } +// resolveResourceAncestors returns the ExternalIDs of every parent that must be +// expanded to reveal the given (possibly deeply nested) selections in a lazily +// loaded picker. Used when editing a data source to restore an existing selection. +export function resolveResourceAncestors(id: string, resourceIds: string[]) { + return post(`/api/v1/datasource/${id}/resource-ancestors`, { resource_ids: resourceIds }, { timeout: 120000 }) +} + export function triggerSync(id: string) { return post(`/api/v1/datasource/${id}/sync`, {}) } diff --git a/frontend/src/views/knowledge/settings/DataSourceEditorDialog.vue b/frontend/src/views/knowledge/settings/DataSourceEditorDialog.vue index 2402d8d4..4c5e0adc 100644 --- a/frontend/src/views/knowledge/settings/DataSourceEditorDialog.vue +++ b/frontend/src/views/knowledge/settings/DataSourceEditorDialog.vue @@ -9,6 +9,7 @@ import { validateConnection, validateCredentials, listResources, + resolveResourceAncestors, deleteDataSource, putDataSourceCredentials, deleteDataSourceCredentials, @@ -141,6 +142,10 @@ const expandedResourceIds = ref(new Set()) // one level at a time instead of traversing the whole tree up front (#1672). const loadedChildrenIds = ref(new Set()) const loadingChildrenIds = ref(new Set()) +// True when the initial listing already returned the whole tree (connectors like +// Notion populate parent_id on the first call). In that case expanding a node +// never needs an extra request. +const treeFullyLoaded = ref(false) // Shared children/parent indexes — used by tree rendering and selection logic const childrenMap = computed(() => { @@ -204,12 +209,12 @@ function toggleExpand(id: string) { } // ensureChildrenLoaded fetches the direct children of a node on demand. It is a -// no-op when the children are already present (e.g. connectors that return the -// full tree in one call) or have already been fetched. +// no-op when the connector already delivered the whole tree in one call (e.g. +// Notion) or when this node's children have already been fetched. async function ensureChildrenLoaded(id: string) { if (!tempDsId.value) return if (loadedChildrenIds.value.has(id) || loadingChildrenIds.value.has(id)) return - if ((childrenMap.value.get(id) || []).length > 0) { + if (treeFullyLoaded.value) { loadedChildrenIds.value = new Set(loadedChildrenIds.value).add(id) return } @@ -360,6 +365,7 @@ watch(visible, async (v) => { expandedResourceIds.value = new Set() loadedChildrenIds.value = new Set() loadingChildrenIds.value = new Set() + treeFullyLoaded.value = false if (isEdit.value && props.dataSource) { // Reset edit/replace toggle every open so an aborted replace doesn't @@ -482,6 +488,9 @@ async function loadResources() { } loadedChildrenIds.value = parentsWithChildren loadingChildrenIds.value = new Set() + // If any resource already has a parent, the connector returned the whole tree + // up front, so per-node lazy fetching is unnecessary. + treeFullyLoaded.value = parentsWithChildren.size > 0 // Auto-expand top-level nodes whose children are already loaded; lazy nodes // (children not yet fetched) stay collapsed until the user expands them. expandedResourceIds.value = new Set( @@ -489,12 +498,39 @@ async function loadResources() { .filter(r => !r.parent_id && r.has_children && parentsWithChildren.has(r.external_id)) .map(r => r.external_id), ) + // When editing a lazily-loaded source, reveal pre-existing selections that + // live below the (not-yet-loaded) tree so they are visible and checked. + if (isEdit.value && !treeFullyLoaded.value) { + const loaded = new Set(resources.value.map(r => r.external_id)) + const hidden = selectedResourceIds.value.filter(id => !loaded.has(id)) + if (hidden.length > 0) void revealExistingSelections(hidden) + } } catch (e: any) { MessagePlugin.error(e?.message || e?.error || t('datasource.resourceLoadFailed')) } loadingResources.value = false } +// revealExistingSelections asks the backend which ancestors must be expanded to +// surface the current (possibly deeply nested) selection, then loads each level +// so the saved selection becomes visible and correctly checked in the tree. +async function revealExistingSelections(hiddenIds: string[]) { + if (!tempDsId.value || hiddenIds.length === 0) return + try { + const res = await resolveResourceAncestors(tempDsId.value, hiddenIds) + const ancestors: string[] = res?.data?.ancestors || res?.ancestors || [] + if (ancestors.length === 0) return + const expanded = new Set(expandedResourceIds.value) + for (const id of ancestors) expanded.add(id) + expandedResourceIds.value = expanded + // Load each ancestor level (children include the next ancestor / the + // selection itself); calls are independent and dedup on merge. + await Promise.all(ancestors.map(id => ensureChildrenLoaded(id))) + } catch (e: any) { + MessagePlugin.error(e?.message || e?.error || t('datasource.resourceLoadFailed')) + } +} + function getDescendantIds(id: string): string[] { const ids: string[] = [] const children = childrenMap.value.get(id) || [] diff --git a/internal/application/service/datasource_service.go b/internal/application/service/datasource_service.go index 2798da1f..0cfe8ce7 100644 --- a/internal/application/service/datasource_service.go +++ b/internal/application/service/datasource_service.go @@ -381,6 +381,39 @@ func (s *DataSourceService) ListAvailableResources( return resources, nil } +// ResolveResourceAncestors resolves the ancestor ExternalIDs needed to reveal the +// given resources in a lazily-loaded picker (see the connector method for details). +func (s *DataSourceService) ResolveResourceAncestors( + ctx context.Context, dsID string, resourceIDs []string, +) ([]string, error) { + if len(resourceIDs) == 0 { + return []string{}, nil + } + + ds, err := s.GetDataSource(ctx, dsID) + if err != nil { + return nil, err + } + + connector, err := s.connectorRegistry.Get(ds.Type) + if err != nil { + return nil, err + } + + config, err := ds.ParseConfig() + if err != nil { + return nil, datasource.ErrInvalidConfig + } + + ancestors, err := connector.ResolveResourceAncestors(ctx, config, resourceIDs) + if err != nil { + logger.Errorf(ctx, "failed to resolve resource ancestors: %v", err) + return nil, err + } + + return ancestors, nil +} + // ManualSync triggers an immediate sync for a data source func (s *DataSourceService) ManualSync(ctx context.Context, dsID string) (*types.SyncLog, error) { ds, err := s.GetDataSource(ctx, dsID) diff --git a/internal/datasource/connector.go b/internal/datasource/connector.go index e6542980..d85595cd 100644 --- a/internal/datasource/connector.go +++ b/internal/datasource/connector.go @@ -27,6 +27,20 @@ type Connector interface { // non-empty parentID. ListResources(ctx context.Context, config *types.DataSourceConfig, parentID string) ([]types.Resource, error) + // ResolveResourceAncestors resolves, for each of the given resource IDs, the + // ExternalIDs of every ancestor whose direct children must be loaded so a + // lazily-loaded picker can reveal a pre-existing (possibly deeply nested) + // selection. The returned set is deduplicated and unordered. + // + // It exists so connectors that load their tree one level at a time (e.g. the + // Feishu wiki) can expose, in O(depth) per selection, the path back to the + // root without re-traversing the whole tree. Connectors that already return + // the full tree (Notion) or a flat list (Yuque) have nothing to reveal and + // return an empty slice. + ResolveResourceAncestors( + ctx context.Context, config *types.DataSourceConfig, resourceIDs []string, + ) ([]string, error) + // FetchAll performs a full sync of the specified resources. // Returns all items from the given resource IDs. FetchAll(ctx context.Context, config *types.DataSourceConfig, resourceIDs []string) ([]types.FetchedItem, error) diff --git a/internal/datasource/connector/feishu/connector.go b/internal/datasource/connector/feishu/connector.go index 422f24bc..7e5817c2 100644 --- a/internal/datasource/connector/feishu/connector.go +++ b/internal/datasource/connector/feishu/connector.go @@ -10,6 +10,7 @@ import ( "time" "unicode/utf8" + "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/types" ) @@ -100,6 +101,61 @@ func (c *Connector) ListResources( return resources, nil } +// ResolveResourceAncestors returns the resource IDs of every parent that has to +// be expanded so the lazily-loaded picker can reveal each given selection. For a +// selected node "spaceID:nodeToken" that is its space plus every intermediate +// node up the tree; the walk uses GetWikiNode (parent_node_token) and is O(depth) +// per selection, so it never re-traverses the whole wiki. +func (c *Connector) ResolveResourceAncestors( + ctx context.Context, config *types.DataSourceConfig, resourceIDs []string, +) ([]string, error) { + feishuConfig, err := parseFeishuConfig(config) + if err != nil { + return nil, err + } + client := NewClient(feishuConfig) + + seen := make(map[string]bool) + ancestors := make([]string, 0) + add := func(id string) { + if id != "" && !seen[id] { + seen[id] = true + ancestors = append(ancestors, id) + } + } + + for _, rid := range resourceIDs { + spaceID, nodeToken := parseWikiResourceID(rid) + if spaceID == "" || nodeToken == "" { + // A space-level selection is already a top-level node in the picker; + // there is nothing above it to reveal. + continue + } + // The space's direct children must be loaded to reveal the top-level node. + add(spaceID) + + // Walk up from the selection to the top, loading each intermediate + // parent so the path down to the selection becomes visible. + current := nodeToken + for current != "" { + node, err := client.GetWikiNode(ctx, spaceID, current) + if err != nil { + // Best-effort: a broken path just stays collapsed, the rest of + // the selections are still revealed. + logger.Warnf(ctx, "[Feishu] resolve ancestors: get node %s:%s: %v", spaceID, current, err) + break + } + if node.ParentNodeID == "" { + break + } + add(makeWikiNodeResourceID(spaceID, node.ParentNodeID)) + current = node.ParentNodeID + } + } + + return ancestors, nil +} + // FetchAll performs a full sync of all documents from the specified wiki spaces. func (c *Connector) FetchAll(ctx context.Context, config *types.DataSourceConfig, resourceIDs []string) ([]types.FetchedItem, error) { feishuConfig, err := parseFeishuConfig(config) diff --git a/internal/datasource/connector/feishu/connector_test.go b/internal/datasource/connector/feishu/connector_test.go index 317eb78e..0768d77c 100644 --- a/internal/datasource/connector/feishu/connector_test.go +++ b/internal/datasource/connector/feishu/connector_test.go @@ -529,6 +529,69 @@ func TestConnectorListResources_LazyLoadsOneLevel(t *testing.T) { } } +// TestConnectorResolveResourceAncestors verifies that the ancestor chain of a +// deeply nested selection is resolved (so an edit-mode picker can reveal it) +// without listing the whole tree. +func TestConnectorResolveResourceAncestors(t *testing.T) { + topNodes := []wikiNode{ + {NodeToken: "nt-root", ObjToken: "obj-root", ObjType: "docx", Title: "Root", HasChild: true}, + } + childNodes := map[string][]wikiNode{ + "nt-root": { + {NodeToken: "nt-child", ObjToken: "obj-child", ObjType: "docx", Title: "Child", HasChild: true}, + }, + "nt-child": { + {NodeToken: "nt-grandchild", ObjToken: "obj-gc", ObjType: "docx", Title: "Grandchild"}, + }, + } + ts, cfg := fakeFeishuHierarchy(topNodes, childNodes, "") + defer ts.Close() + + c := NewConnector() + + // A deeply nested node resolves to its space plus every intermediate parent. + ancestors, err := c.ResolveResourceAncestors( + context.Background(), makeConfig(cfg, nil), []string{"space1:nt-grandchild"}, + ) + if err != nil { + t.Fatalf("ResolveResourceAncestors error: %v", err) + } + got := make(map[string]bool) + for _, a := range ancestors { + got[a] = true + } + for _, want := range []string{"space1", "space1:nt-root", "space1:nt-child"} { + if !got[want] { + t.Fatalf("missing ancestor %q, got %+v", want, ancestors) + } + } + if got["space1:nt-grandchild"] { + t.Fatalf("selection itself must not be returned as its own ancestor: %+v", ancestors) + } + + // A top-level node only needs its space loaded. + ancestors, err = c.ResolveResourceAncestors( + context.Background(), makeConfig(cfg, nil), []string{"space1:nt-root"}, + ) + if err != nil { + t.Fatalf("ResolveResourceAncestors(top-level) error: %v", err) + } + if len(ancestors) != 1 || ancestors[0] != "space1" { + t.Fatalf("want [space1] for a top-level node, got %+v", ancestors) + } + + // A whole-space selection is already top-level: nothing to reveal. + ancestors, err = c.ResolveResourceAncestors( + context.Background(), makeConfig(cfg, nil), []string{"space1"}, + ) + if err != nil { + t.Fatalf("ResolveResourceAncestors(space) error: %v", err) + } + if len(ancestors) != 0 { + t.Fatalf("want no ancestors for a space selection, got %+v", ancestors) + } +} + // ────────────────────────────────────────────────────────────────────── // FetchAll: test all supported doc types // ────────────────────────────────────────────────────────────────────── diff --git a/internal/datasource/connector/feishu/types.go b/internal/datasource/connector/feishu/types.go index 148b83ff..72e1a089 100644 --- a/internal/datasource/connector/feishu/types.go +++ b/internal/datasource/connector/feishu/types.go @@ -121,18 +121,18 @@ type wikiNodeListResponse struct { // wikiNode represents a node (document or folder) in a Feishu Wiki space. type wikiNode struct { - SpaceID string `json:"space_id"` - NodeToken string `json:"node_token"` - ObjToken string `json:"obj_token"` // document token - ObjType string `json:"obj_type"` // "doc", "sheet", "mindnote", "bitable", "file", "docx", "slides" - ParentNodeID string `json:"parent_node_id"` - NodeType string `json:"node_type"` // "origin" or "shortcut" - OriginNodeID string `json:"origin_node_id"` - OriginSpaceID string `json:"origin_space_id"` - HasChild bool `json:"has_child"` - Title string `json:"title"` - Creator string `json:"creator"` - Owner string `json:"owner"` + SpaceID string `json:"space_id"` + NodeToken string `json:"node_token"` + ObjToken string `json:"obj_token"` // document token + ObjType string `json:"obj_type"` // "doc", "sheet", "mindnote", "bitable", "file", "docx", "slides" + ParentNodeID string `json:"parent_node_token"` + NodeType string `json:"node_type"` // "origin" or "shortcut" + OriginNodeID string `json:"origin_node_id"` + OriginSpaceID string `json:"origin_space_id"` + HasChild bool `json:"has_child"` + Title string `json:"title"` + Creator string `json:"creator"` + Owner string `json:"owner"` ObjCreateTime string `json:"obj_create_time"` // document creation time (unix timestamp string) ObjEditTime string `json:"obj_edit_time"` // document last edit time (unix timestamp string) — tracks content changes NodeCreateTime string `json:"node_create_time"` // node creation time (unix timestamp string) diff --git a/internal/datasource/connector/notion/connector.go b/internal/datasource/connector/notion/connector.go index 59b42781..4d91fb27 100644 --- a/internal/datasource/connector/notion/connector.go +++ b/internal/datasource/connector/notion/connector.go @@ -44,6 +44,15 @@ func (c *Connector) Validate(ctx context.Context, config *types.DataSourceConfig return client.Ping(ctx) } +// ResolveResourceAncestors has nothing to do for Notion: ListResources already +// returns the full tree with parent links, so any pre-existing selection is +// already present and revealed by the picker without on-demand loading. +func (c *Connector) ResolveResourceAncestors( + ctx context.Context, config *types.DataSourceConfig, resourceIDs []string, +) ([]string, error) { + return []string{}, nil +} + // ListResources lists all accessible Notion pages and databases as selectable resources. // Returns all objects with parent-child relationships populated, allowing the frontend // to render a tree view. Root objects have empty ParentID. diff --git a/internal/datasource/connector/yuque/connector.go b/internal/datasource/connector/yuque/connector.go index 1e4393e0..d07fce0c 100644 --- a/internal/datasource/connector/yuque/connector.go +++ b/internal/datasource/connector/yuque/connector.go @@ -40,6 +40,14 @@ func (c *Connector) Validate(ctx context.Context, config *types.DataSourceConfig return nil } +// ResolveResourceAncestors has nothing to do for Yuque: repositories are a flat +// list with no nesting, so a selection has no ancestors to reveal. +func (c *Connector) ResolveResourceAncestors( + ctx context.Context, config *types.DataSourceConfig, resourceIDs []string, +) ([]string, error) { + return []string{}, nil +} + // ListResources returns all repos (personal + team) accessible to the token. // Serial fetch for v1 (user groups typically <10). TODO(perf): parallelize if slow. func (c *Connector) ListResources( diff --git a/internal/handler/datasource.go b/internal/handler/datasource.go index be49975f..af6adb22 100644 --- a/internal/handler/datasource.go +++ b/internal/handler/datasource.go @@ -366,6 +366,54 @@ func (h *DataSourceHandler) ListAvailableResources(c *gin.Context) { c.JSON(http.StatusOK, resources) } +// @Summary Resolve resource ancestors +// @Description Resolve the ancestor ExternalIDs that must be expanded to reveal the given (possibly deeply nested) resources in a lazily-loaded picker. Used to restore an existing selection when editing a data source. +// @Tags DataSource +// @Accept json +// @Produce json +// @Param id path string true "Data source ID" +// @Param request body resolveAncestorsRequest true "Resource IDs to resolve" +// @Success 200 {object} map[string][]string +// @Failure 400 {object} map[string]string +// @Router /datasource/{id}/resource-ancestors [post] +func (h *DataSourceHandler) ResolveResourceAncestors(c *gin.Context) { + ctx := c.Request.Context() + tenantID := h.getTenantID(c) + if tenantID == 0 { + c.JSON(http.StatusUnauthorized, gin.H{"error": "unauthorized"}) + return + } + + id := c.Param("id") + + if _, status, msg := h.getOwnedDataSource(ctx, tenantID, id); status != http.StatusOK { + c.JSON(status, gin.H{"error": msg}) + return + } + + var req resolveAncestorsRequest + if err := c.ShouldBindJSON(&req); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + + ancestors, err := h.service.ResolveResourceAncestors(ctx, id, req.ResourceIDs) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + + if ancestors == nil { + ancestors = make([]string, 0) + } + c.JSON(http.StatusOK, gin.H{"ancestors": ancestors}) +} + +// resolveAncestorsRequest is the body for ResolveResourceAncestors. +type resolveAncestorsRequest struct { + ResourceIDs []string `json:"resource_ids"` +} + // ManualSync godoc // @Summary Trigger immediate sync // @Description Trigger an immediate sync for a data source diff --git a/internal/router/router.go b/internal/router/router.go index 25d783ab..1cf9c2f8 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -1716,6 +1716,7 @@ func RegisterDataSourceRoutes( // Connection and resource management — Admin+ ds.POST("/:id/validate", g.Admin(), handler.ValidateConnection) ds.GET("/:id/resources", g.Admin(), handler.ListAvailableResources) + ds.POST("/:id/resource-ancestors", g.Admin(), handler.ResolveResourceAncestors) // Sync management — Admin+ ds.POST("/:id/sync", g.Admin(), handler.ManualSync) diff --git a/internal/types/interfaces/datasource.go b/internal/types/interfaces/datasource.go index c84e86cd..cdbb92a7 100644 --- a/internal/types/interfaces/datasource.go +++ b/internal/types/interfaces/datasource.go @@ -47,6 +47,12 @@ type DataSourceService interface { // parentID enables lazy loading: "" lists the top level, a resource ExternalID lists its children. ListAvailableResources(ctx context.Context, dsID string, parentID string) ([]types.Resource, error) + // ResolveResourceAncestors returns the deduplicated ExternalIDs of every + // ancestor that must be expanded to reveal the given (possibly deeply nested) + // resources in a lazily-loaded picker. Used to restore an existing selection + // when editing a data source. + ResolveResourceAncestors(ctx context.Context, dsID string, resourceIDs []string) ([]string, error) + // ManualSync triggers an immediate sync for a data source ManualSync(ctx context.Context, dsID string) (*types.SyncLog, error)