feat(datasource): reveal pre-existing selections in lazily-loaded picker

When editing a data source whose tree loads one level at a time (Feishu
wiki), a saved deep-node selection was invisible: only top-level spaces
were listed, so the selected node could not be seen and its ancestor
space rendered as unchecked.

- Add Connector.ResolveResourceAncestors: given selected resource IDs,
  return the ancestor IDs that must be expanded to reveal them. Feishu
  walks up via GetWikiNode (O(depth) per selection, no full-tree walk);
  Notion/Yuque return empty.
- Fix wikiNode.ParentNodeID JSON tag: the Feishu API field is
  parent_node_token, not parent_node_id, so GetWikiNode can return the
  parent and the ancestor walk works in production.
- Expose POST /datasource/{id}/resource-ancestors (service + handler +
  route) and a matching frontend API client.
- Edit-mode picker reveals hidden selections by expanding and lazily
  loading each ancestor level, so selections are visible and checked.
- Gate child fetching on treeFullyLoaded instead of a partial-children
  heuristic, avoiding a redundant request for full-tree connectors.
This commit is contained in:
wizardchen
2026-06-22 18:15:27 +08:00
committed by lyingbug
parent d8f13ada4a
commit 5d0d317a36
12 changed files with 296 additions and 15 deletions
+7
View File
@@ -103,6 +103,13 @@ export function listResources(id: string, parentId?: string) {
return get(`/api/v1/datasource/${id}/resources${query}`, { timeout: 120000 })
}
// resolveResourceAncestors returns the ExternalIDs of every parent that must be
// expanded to reveal the given (possibly deeply nested) selections in a lazily
// loaded picker. Used when editing a data source to restore an existing selection.
export function resolveResourceAncestors(id: string, resourceIds: string[]) {
return post(`/api/v1/datasource/${id}/resource-ancestors`, { resource_ids: resourceIds }, { timeout: 120000 })
}
export function triggerSync(id: string) {
return post(`/api/v1/datasource/${id}/sync`, {})
}
@@ -9,6 +9,7 @@ import {
validateConnection,
validateCredentials,
listResources,
resolveResourceAncestors,
deleteDataSource,
putDataSourceCredentials,
deleteDataSourceCredentials,
@@ -141,6 +142,10 @@ const expandedResourceIds = ref(new Set<string>())
// one level at a time instead of traversing the whole tree up front (#1672).
const loadedChildrenIds = ref(new Set<string>())
const loadingChildrenIds = ref(new Set<string>())
// True when the initial listing already returned the whole tree (connectors like
// Notion populate parent_id on the first call). In that case expanding a node
// never needs an extra request.
const treeFullyLoaded = ref(false)
// Shared children/parent indexes — used by tree rendering and selection logic
const childrenMap = computed(() => {
@@ -204,12 +209,12 @@ function toggleExpand(id: string) {
}
// ensureChildrenLoaded fetches the direct children of a node on demand. It is a
// no-op when the children are already present (e.g. connectors that return the
// full tree in one call) or have already been fetched.
// no-op when the connector already delivered the whole tree in one call (e.g.
// Notion) or when this node's children have already been fetched.
async function ensureChildrenLoaded(id: string) {
if (!tempDsId.value) return
if (loadedChildrenIds.value.has(id) || loadingChildrenIds.value.has(id)) return
if ((childrenMap.value.get(id) || []).length > 0) {
if (treeFullyLoaded.value) {
loadedChildrenIds.value = new Set(loadedChildrenIds.value).add(id)
return
}
@@ -360,6 +365,7 @@ watch(visible, async (v) => {
expandedResourceIds.value = new Set()
loadedChildrenIds.value = new Set()
loadingChildrenIds.value = new Set()
treeFullyLoaded.value = false
if (isEdit.value && props.dataSource) {
// Reset edit/replace toggle every open so an aborted replace doesn't
@@ -482,6 +488,9 @@ async function loadResources() {
}
loadedChildrenIds.value = parentsWithChildren
loadingChildrenIds.value = new Set<string>()
// If any resource already has a parent, the connector returned the whole tree
// up front, so per-node lazy fetching is unnecessary.
treeFullyLoaded.value = parentsWithChildren.size > 0
// Auto-expand top-level nodes whose children are already loaded; lazy nodes
// (children not yet fetched) stay collapsed until the user expands them.
expandedResourceIds.value = new Set(
@@ -489,12 +498,39 @@ async function loadResources() {
.filter(r => !r.parent_id && r.has_children && parentsWithChildren.has(r.external_id))
.map(r => r.external_id),
)
// When editing a lazily-loaded source, reveal pre-existing selections that
// live below the (not-yet-loaded) tree so they are visible and checked.
if (isEdit.value && !treeFullyLoaded.value) {
const loaded = new Set(resources.value.map(r => r.external_id))
const hidden = selectedResourceIds.value.filter(id => !loaded.has(id))
if (hidden.length > 0) void revealExistingSelections(hidden)
}
} catch (e: any) {
MessagePlugin.error(e?.message || e?.error || t('datasource.resourceLoadFailed'))
}
loadingResources.value = false
}
// revealExistingSelections asks the backend which ancestors must be expanded to
// surface the current (possibly deeply nested) selection, then loads each level
// so the saved selection becomes visible and correctly checked in the tree.
async function revealExistingSelections(hiddenIds: string[]) {
if (!tempDsId.value || hiddenIds.length === 0) return
try {
const res = await resolveResourceAncestors(tempDsId.value, hiddenIds)
const ancestors: string[] = res?.data?.ancestors || res?.ancestors || []
if (ancestors.length === 0) return
const expanded = new Set(expandedResourceIds.value)
for (const id of ancestors) expanded.add(id)
expandedResourceIds.value = expanded
// Load each ancestor level (children include the next ancestor / the
// selection itself); calls are independent and dedup on merge.
await Promise.all(ancestors.map(id => ensureChildrenLoaded(id)))
} catch (e: any) {
MessagePlugin.error(e?.message || e?.error || t('datasource.resourceLoadFailed'))
}
}
function getDescendantIds(id: string): string[] {
const ids: string[] = []
const children = childrenMap.value.get(id) || []
@@ -381,6 +381,39 @@ func (s *DataSourceService) ListAvailableResources(
return resources, nil
}
// ResolveResourceAncestors resolves the ancestor ExternalIDs needed to reveal the
// given resources in a lazily-loaded picker (see the connector method for details).
func (s *DataSourceService) ResolveResourceAncestors(
ctx context.Context, dsID string, resourceIDs []string,
) ([]string, error) {
if len(resourceIDs) == 0 {
return []string{}, nil
}
ds, err := s.GetDataSource(ctx, dsID)
if err != nil {
return nil, err
}
connector, err := s.connectorRegistry.Get(ds.Type)
if err != nil {
return nil, err
}
config, err := ds.ParseConfig()
if err != nil {
return nil, datasource.ErrInvalidConfig
}
ancestors, err := connector.ResolveResourceAncestors(ctx, config, resourceIDs)
if err != nil {
logger.Errorf(ctx, "failed to resolve resource ancestors: %v", err)
return nil, err
}
return ancestors, nil
}
// ManualSync triggers an immediate sync for a data source
func (s *DataSourceService) ManualSync(ctx context.Context, dsID string) (*types.SyncLog, error) {
ds, err := s.GetDataSource(ctx, dsID)
+14
View File
@@ -27,6 +27,20 @@ type Connector interface {
// non-empty parentID.
ListResources(ctx context.Context, config *types.DataSourceConfig, parentID string) ([]types.Resource, error)
// ResolveResourceAncestors resolves, for each of the given resource IDs, the
// ExternalIDs of every ancestor whose direct children must be loaded so a
// lazily-loaded picker can reveal a pre-existing (possibly deeply nested)
// selection. The returned set is deduplicated and unordered.
//
// It exists so connectors that load their tree one level at a time (e.g. the
// Feishu wiki) can expose, in O(depth) per selection, the path back to the
// root without re-traversing the whole tree. Connectors that already return
// the full tree (Notion) or a flat list (Yuque) have nothing to reveal and
// return an empty slice.
ResolveResourceAncestors(
ctx context.Context, config *types.DataSourceConfig, resourceIDs []string,
) ([]string, error)
// FetchAll performs a full sync of the specified resources.
// Returns all items from the given resource IDs.
FetchAll(ctx context.Context, config *types.DataSourceConfig, resourceIDs []string) ([]types.FetchedItem, error)
@@ -10,6 +10,7 @@ import (
"time"
"unicode/utf8"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
)
@@ -100,6 +101,61 @@ func (c *Connector) ListResources(
return resources, nil
}
// ResolveResourceAncestors returns the resource IDs of every parent that has to
// be expanded so the lazily-loaded picker can reveal each given selection. For a
// selected node "spaceID:nodeToken" that is its space plus every intermediate
// node up the tree; the walk uses GetWikiNode (parent_node_token) and is O(depth)
// per selection, so it never re-traverses the whole wiki.
func (c *Connector) ResolveResourceAncestors(
ctx context.Context, config *types.DataSourceConfig, resourceIDs []string,
) ([]string, error) {
feishuConfig, err := parseFeishuConfig(config)
if err != nil {
return nil, err
}
client := NewClient(feishuConfig)
seen := make(map[string]bool)
ancestors := make([]string, 0)
add := func(id string) {
if id != "" && !seen[id] {
seen[id] = true
ancestors = append(ancestors, id)
}
}
for _, rid := range resourceIDs {
spaceID, nodeToken := parseWikiResourceID(rid)
if spaceID == "" || nodeToken == "" {
// A space-level selection is already a top-level node in the picker;
// there is nothing above it to reveal.
continue
}
// The space's direct children must be loaded to reveal the top-level node.
add(spaceID)
// Walk up from the selection to the top, loading each intermediate
// parent so the path down to the selection becomes visible.
current := nodeToken
for current != "" {
node, err := client.GetWikiNode(ctx, spaceID, current)
if err != nil {
// Best-effort: a broken path just stays collapsed, the rest of
// the selections are still revealed.
logger.Warnf(ctx, "[Feishu] resolve ancestors: get node %s:%s: %v", spaceID, current, err)
break
}
if node.ParentNodeID == "" {
break
}
add(makeWikiNodeResourceID(spaceID, node.ParentNodeID))
current = node.ParentNodeID
}
}
return ancestors, nil
}
// FetchAll performs a full sync of all documents from the specified wiki spaces.
func (c *Connector) FetchAll(ctx context.Context, config *types.DataSourceConfig, resourceIDs []string) ([]types.FetchedItem, error) {
feishuConfig, err := parseFeishuConfig(config)
@@ -529,6 +529,69 @@ func TestConnectorListResources_LazyLoadsOneLevel(t *testing.T) {
}
}
// TestConnectorResolveResourceAncestors verifies that the ancestor chain of a
// deeply nested selection is resolved (so an edit-mode picker can reveal it)
// without listing the whole tree.
func TestConnectorResolveResourceAncestors(t *testing.T) {
topNodes := []wikiNode{
{NodeToken: "nt-root", ObjToken: "obj-root", ObjType: "docx", Title: "Root", HasChild: true},
}
childNodes := map[string][]wikiNode{
"nt-root": {
{NodeToken: "nt-child", ObjToken: "obj-child", ObjType: "docx", Title: "Child", HasChild: true},
},
"nt-child": {
{NodeToken: "nt-grandchild", ObjToken: "obj-gc", ObjType: "docx", Title: "Grandchild"},
},
}
ts, cfg := fakeFeishuHierarchy(topNodes, childNodes, "")
defer ts.Close()
c := NewConnector()
// A deeply nested node resolves to its space plus every intermediate parent.
ancestors, err := c.ResolveResourceAncestors(
context.Background(), makeConfig(cfg, nil), []string{"space1:nt-grandchild"},
)
if err != nil {
t.Fatalf("ResolveResourceAncestors error: %v", err)
}
got := make(map[string]bool)
for _, a := range ancestors {
got[a] = true
}
for _, want := range []string{"space1", "space1:nt-root", "space1:nt-child"} {
if !got[want] {
t.Fatalf("missing ancestor %q, got %+v", want, ancestors)
}
}
if got["space1:nt-grandchild"] {
t.Fatalf("selection itself must not be returned as its own ancestor: %+v", ancestors)
}
// A top-level node only needs its space loaded.
ancestors, err = c.ResolveResourceAncestors(
context.Background(), makeConfig(cfg, nil), []string{"space1:nt-root"},
)
if err != nil {
t.Fatalf("ResolveResourceAncestors(top-level) error: %v", err)
}
if len(ancestors) != 1 || ancestors[0] != "space1" {
t.Fatalf("want [space1] for a top-level node, got %+v", ancestors)
}
// A whole-space selection is already top-level: nothing to reveal.
ancestors, err = c.ResolveResourceAncestors(
context.Background(), makeConfig(cfg, nil), []string{"space1"},
)
if err != nil {
t.Fatalf("ResolveResourceAncestors(space) error: %v", err)
}
if len(ancestors) != 0 {
t.Fatalf("want no ancestors for a space selection, got %+v", ancestors)
}
}
// ──────────────────────────────────────────────────────────────────────
// FetchAll: test all supported doc types
// ──────────────────────────────────────────────────────────────────────
+12 -12
View File
@@ -121,18 +121,18 @@ type wikiNodeListResponse struct {
// wikiNode represents a node (document or folder) in a Feishu Wiki space.
type wikiNode struct {
SpaceID string `json:"space_id"`
NodeToken string `json:"node_token"`
ObjToken string `json:"obj_token"` // document token
ObjType string `json:"obj_type"` // "doc", "sheet", "mindnote", "bitable", "file", "docx", "slides"
ParentNodeID string `json:"parent_node_id"`
NodeType string `json:"node_type"` // "origin" or "shortcut"
OriginNodeID string `json:"origin_node_id"`
OriginSpaceID string `json:"origin_space_id"`
HasChild bool `json:"has_child"`
Title string `json:"title"`
Creator string `json:"creator"`
Owner string `json:"owner"`
SpaceID string `json:"space_id"`
NodeToken string `json:"node_token"`
ObjToken string `json:"obj_token"` // document token
ObjType string `json:"obj_type"` // "doc", "sheet", "mindnote", "bitable", "file", "docx", "slides"
ParentNodeID string `json:"parent_node_token"`
NodeType string `json:"node_type"` // "origin" or "shortcut"
OriginNodeID string `json:"origin_node_id"`
OriginSpaceID string `json:"origin_space_id"`
HasChild bool `json:"has_child"`
Title string `json:"title"`
Creator string `json:"creator"`
Owner string `json:"owner"`
ObjCreateTime string `json:"obj_create_time"` // document creation time (unix timestamp string)
ObjEditTime string `json:"obj_edit_time"` // document last edit time (unix timestamp string) — tracks content changes
NodeCreateTime string `json:"node_create_time"` // node creation time (unix timestamp string)
@@ -44,6 +44,15 @@ func (c *Connector) Validate(ctx context.Context, config *types.DataSourceConfig
return client.Ping(ctx)
}
// ResolveResourceAncestors has nothing to do for Notion: ListResources already
// returns the full tree with parent links, so any pre-existing selection is
// already present and revealed by the picker without on-demand loading.
func (c *Connector) ResolveResourceAncestors(
ctx context.Context, config *types.DataSourceConfig, resourceIDs []string,
) ([]string, error) {
return []string{}, nil
}
// ListResources lists all accessible Notion pages and databases as selectable resources.
// Returns all objects with parent-child relationships populated, allowing the frontend
// to render a tree view. Root objects have empty ParentID.
@@ -40,6 +40,14 @@ func (c *Connector) Validate(ctx context.Context, config *types.DataSourceConfig
return nil
}
// ResolveResourceAncestors has nothing to do for Yuque: repositories are a flat
// list with no nesting, so a selection has no ancestors to reveal.
func (c *Connector) ResolveResourceAncestors(
ctx context.Context, config *types.DataSourceConfig, resourceIDs []string,
) ([]string, error) {
return []string{}, nil
}
// ListResources returns all repos (personal + team) accessible to the token.
// Serial fetch for v1 (user groups typically <10). TODO(perf): parallelize if slow.
func (c *Connector) ListResources(
+48
View File
@@ -366,6 +366,54 @@ func (h *DataSourceHandler) ListAvailableResources(c *gin.Context) {
c.JSON(http.StatusOK, resources)
}
// @Summary Resolve resource ancestors
// @Description Resolve the ancestor ExternalIDs that must be expanded to reveal the given (possibly deeply nested) resources in a lazily-loaded picker. Used to restore an existing selection when editing a data source.
// @Tags DataSource
// @Accept json
// @Produce json
// @Param id path string true "Data source ID"
// @Param request body resolveAncestorsRequest true "Resource IDs to resolve"
// @Success 200 {object} map[string][]string
// @Failure 400 {object} map[string]string
// @Router /datasource/{id}/resource-ancestors [post]
func (h *DataSourceHandler) ResolveResourceAncestors(c *gin.Context) {
ctx := c.Request.Context()
tenantID := h.getTenantID(c)
if tenantID == 0 {
c.JSON(http.StatusUnauthorized, gin.H{"error": "unauthorized"})
return
}
id := c.Param("id")
if _, status, msg := h.getOwnedDataSource(ctx, tenantID, id); status != http.StatusOK {
c.JSON(status, gin.H{"error": msg})
return
}
var req resolveAncestorsRequest
if err := c.ShouldBindJSON(&req); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
ancestors, err := h.service.ResolveResourceAncestors(ctx, id, req.ResourceIDs)
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
if ancestors == nil {
ancestors = make([]string, 0)
}
c.JSON(http.StatusOK, gin.H{"ancestors": ancestors})
}
// resolveAncestorsRequest is the body for ResolveResourceAncestors.
type resolveAncestorsRequest struct {
ResourceIDs []string `json:"resource_ids"`
}
// ManualSync godoc
// @Summary Trigger immediate sync
// @Description Trigger an immediate sync for a data source
+1
View File
@@ -1716,6 +1716,7 @@ func RegisterDataSourceRoutes(
// Connection and resource management — Admin+
ds.POST("/:id/validate", g.Admin(), handler.ValidateConnection)
ds.GET("/:id/resources", g.Admin(), handler.ListAvailableResources)
ds.POST("/:id/resource-ancestors", g.Admin(), handler.ResolveResourceAncestors)
// Sync management — Admin+
ds.POST("/:id/sync", g.Admin(), handler.ManualSync)
+6
View File
@@ -47,6 +47,12 @@ type DataSourceService interface {
// parentID enables lazy loading: "" lists the top level, a resource ExternalID lists its children.
ListAvailableResources(ctx context.Context, dsID string, parentID string) ([]types.Resource, error)
// ResolveResourceAncestors returns the deduplicated ExternalIDs of every
// ancestor that must be expanded to reveal the given (possibly deeply nested)
// resources in a lazily-loaded picker. Used to restore an existing selection
// when editing a data source.
ResolveResourceAncestors(ctx context.Context, dsID string, resourceIDs []string) ([]string, error)
// ManualSync triggers an immediate sync for a data source
ManualSync(ctx context.Context, dsID string) (*types.SyncLog, error)