feat(datasource): reveal pre-existing selections in lazily-loaded picker
When editing a data source whose tree loads one level at a time (Feishu
wiki), a saved deep-node selection was invisible: only top-level spaces
were listed, so the selected node could not be seen and its ancestor
space rendered as unchecked.
- Add Connector.ResolveResourceAncestors: given selected resource IDs,
return the ancestor IDs that must be expanded to reveal them. Feishu
walks up via GetWikiNode (O(depth) per selection, no full-tree walk);
Notion/Yuque return empty.
- Fix wikiNode.ParentNodeID JSON tag: the Feishu API field is
parent_node_token, not parent_node_id, so GetWikiNode can return the
parent and the ancestor walk works in production.
- Expose POST /datasource/{id}/resource-ancestors (service + handler +
route) and a matching frontend API client.
- Edit-mode picker reveals hidden selections by expanding and lazily
loading each ancestor level, so selections are visible and checked.
- Gate child fetching on treeFullyLoaded instead of a partial-children
heuristic, avoiding a redundant request for full-tree connectors.
This commit is contained in:
@@ -103,6 +103,13 @@ export function listResources(id: string, parentId?: string) {
|
||||
return get(`/api/v1/datasource/${id}/resources${query}`, { timeout: 120000 })
|
||||
}
|
||||
|
||||
// resolveResourceAncestors returns the ExternalIDs of every parent that must be
|
||||
// expanded to reveal the given (possibly deeply nested) selections in a lazily
|
||||
// loaded picker. Used when editing a data source to restore an existing selection.
|
||||
export function resolveResourceAncestors(id: string, resourceIds: string[]) {
|
||||
return post(`/api/v1/datasource/${id}/resource-ancestors`, { resource_ids: resourceIds }, { timeout: 120000 })
|
||||
}
|
||||
|
||||
export function triggerSync(id: string) {
|
||||
return post(`/api/v1/datasource/${id}/sync`, {})
|
||||
}
|
||||
|
||||
@@ -9,6 +9,7 @@ import {
|
||||
validateConnection,
|
||||
validateCredentials,
|
||||
listResources,
|
||||
resolveResourceAncestors,
|
||||
deleteDataSource,
|
||||
putDataSourceCredentials,
|
||||
deleteDataSourceCredentials,
|
||||
@@ -141,6 +142,10 @@ const expandedResourceIds = ref(new Set<string>())
|
||||
// one level at a time instead of traversing the whole tree up front (#1672).
|
||||
const loadedChildrenIds = ref(new Set<string>())
|
||||
const loadingChildrenIds = ref(new Set<string>())
|
||||
// True when the initial listing already returned the whole tree (connectors like
|
||||
// Notion populate parent_id on the first call). In that case expanding a node
|
||||
// never needs an extra request.
|
||||
const treeFullyLoaded = ref(false)
|
||||
|
||||
// Shared children/parent indexes — used by tree rendering and selection logic
|
||||
const childrenMap = computed(() => {
|
||||
@@ -204,12 +209,12 @@ function toggleExpand(id: string) {
|
||||
}
|
||||
|
||||
// ensureChildrenLoaded fetches the direct children of a node on demand. It is a
|
||||
// no-op when the children are already present (e.g. connectors that return the
|
||||
// full tree in one call) or have already been fetched.
|
||||
// no-op when the connector already delivered the whole tree in one call (e.g.
|
||||
// Notion) or when this node's children have already been fetched.
|
||||
async function ensureChildrenLoaded(id: string) {
|
||||
if (!tempDsId.value) return
|
||||
if (loadedChildrenIds.value.has(id) || loadingChildrenIds.value.has(id)) return
|
||||
if ((childrenMap.value.get(id) || []).length > 0) {
|
||||
if (treeFullyLoaded.value) {
|
||||
loadedChildrenIds.value = new Set(loadedChildrenIds.value).add(id)
|
||||
return
|
||||
}
|
||||
@@ -360,6 +365,7 @@ watch(visible, async (v) => {
|
||||
expandedResourceIds.value = new Set()
|
||||
loadedChildrenIds.value = new Set()
|
||||
loadingChildrenIds.value = new Set()
|
||||
treeFullyLoaded.value = false
|
||||
|
||||
if (isEdit.value && props.dataSource) {
|
||||
// Reset edit/replace toggle every open so an aborted replace doesn't
|
||||
@@ -482,6 +488,9 @@ async function loadResources() {
|
||||
}
|
||||
loadedChildrenIds.value = parentsWithChildren
|
||||
loadingChildrenIds.value = new Set<string>()
|
||||
// If any resource already has a parent, the connector returned the whole tree
|
||||
// up front, so per-node lazy fetching is unnecessary.
|
||||
treeFullyLoaded.value = parentsWithChildren.size > 0
|
||||
// Auto-expand top-level nodes whose children are already loaded; lazy nodes
|
||||
// (children not yet fetched) stay collapsed until the user expands them.
|
||||
expandedResourceIds.value = new Set(
|
||||
@@ -489,12 +498,39 @@ async function loadResources() {
|
||||
.filter(r => !r.parent_id && r.has_children && parentsWithChildren.has(r.external_id))
|
||||
.map(r => r.external_id),
|
||||
)
|
||||
// When editing a lazily-loaded source, reveal pre-existing selections that
|
||||
// live below the (not-yet-loaded) tree so they are visible and checked.
|
||||
if (isEdit.value && !treeFullyLoaded.value) {
|
||||
const loaded = new Set(resources.value.map(r => r.external_id))
|
||||
const hidden = selectedResourceIds.value.filter(id => !loaded.has(id))
|
||||
if (hidden.length > 0) void revealExistingSelections(hidden)
|
||||
}
|
||||
} catch (e: any) {
|
||||
MessagePlugin.error(e?.message || e?.error || t('datasource.resourceLoadFailed'))
|
||||
}
|
||||
loadingResources.value = false
|
||||
}
|
||||
|
||||
// revealExistingSelections asks the backend which ancestors must be expanded to
|
||||
// surface the current (possibly deeply nested) selection, then loads each level
|
||||
// so the saved selection becomes visible and correctly checked in the tree.
|
||||
async function revealExistingSelections(hiddenIds: string[]) {
|
||||
if (!tempDsId.value || hiddenIds.length === 0) return
|
||||
try {
|
||||
const res = await resolveResourceAncestors(tempDsId.value, hiddenIds)
|
||||
const ancestors: string[] = res?.data?.ancestors || res?.ancestors || []
|
||||
if (ancestors.length === 0) return
|
||||
const expanded = new Set(expandedResourceIds.value)
|
||||
for (const id of ancestors) expanded.add(id)
|
||||
expandedResourceIds.value = expanded
|
||||
// Load each ancestor level (children include the next ancestor / the
|
||||
// selection itself); calls are independent and dedup on merge.
|
||||
await Promise.all(ancestors.map(id => ensureChildrenLoaded(id)))
|
||||
} catch (e: any) {
|
||||
MessagePlugin.error(e?.message || e?.error || t('datasource.resourceLoadFailed'))
|
||||
}
|
||||
}
|
||||
|
||||
function getDescendantIds(id: string): string[] {
|
||||
const ids: string[] = []
|
||||
const children = childrenMap.value.get(id) || []
|
||||
|
||||
@@ -381,6 +381,39 @@ func (s *DataSourceService) ListAvailableResources(
|
||||
return resources, nil
|
||||
}
|
||||
|
||||
// ResolveResourceAncestors resolves the ancestor ExternalIDs needed to reveal the
|
||||
// given resources in a lazily-loaded picker (see the connector method for details).
|
||||
func (s *DataSourceService) ResolveResourceAncestors(
|
||||
ctx context.Context, dsID string, resourceIDs []string,
|
||||
) ([]string, error) {
|
||||
if len(resourceIDs) == 0 {
|
||||
return []string{}, nil
|
||||
}
|
||||
|
||||
ds, err := s.GetDataSource(ctx, dsID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
connector, err := s.connectorRegistry.Get(ds.Type)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
config, err := ds.ParseConfig()
|
||||
if err != nil {
|
||||
return nil, datasource.ErrInvalidConfig
|
||||
}
|
||||
|
||||
ancestors, err := connector.ResolveResourceAncestors(ctx, config, resourceIDs)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "failed to resolve resource ancestors: %v", err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return ancestors, nil
|
||||
}
|
||||
|
||||
// ManualSync triggers an immediate sync for a data source
|
||||
func (s *DataSourceService) ManualSync(ctx context.Context, dsID string) (*types.SyncLog, error) {
|
||||
ds, err := s.GetDataSource(ctx, dsID)
|
||||
|
||||
@@ -27,6 +27,20 @@ type Connector interface {
|
||||
// non-empty parentID.
|
||||
ListResources(ctx context.Context, config *types.DataSourceConfig, parentID string) ([]types.Resource, error)
|
||||
|
||||
// ResolveResourceAncestors resolves, for each of the given resource IDs, the
|
||||
// ExternalIDs of every ancestor whose direct children must be loaded so a
|
||||
// lazily-loaded picker can reveal a pre-existing (possibly deeply nested)
|
||||
// selection. The returned set is deduplicated and unordered.
|
||||
//
|
||||
// It exists so connectors that load their tree one level at a time (e.g. the
|
||||
// Feishu wiki) can expose, in O(depth) per selection, the path back to the
|
||||
// root without re-traversing the whole tree. Connectors that already return
|
||||
// the full tree (Notion) or a flat list (Yuque) have nothing to reveal and
|
||||
// return an empty slice.
|
||||
ResolveResourceAncestors(
|
||||
ctx context.Context, config *types.DataSourceConfig, resourceIDs []string,
|
||||
) ([]string, error)
|
||||
|
||||
// FetchAll performs a full sync of the specified resources.
|
||||
// Returns all items from the given resource IDs.
|
||||
FetchAll(ctx context.Context, config *types.DataSourceConfig, resourceIDs []string) ([]types.FetchedItem, error)
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
@@ -100,6 +101,61 @@ func (c *Connector) ListResources(
|
||||
return resources, nil
|
||||
}
|
||||
|
||||
// ResolveResourceAncestors returns the resource IDs of every parent that has to
|
||||
// be expanded so the lazily-loaded picker can reveal each given selection. For a
|
||||
// selected node "spaceID:nodeToken" that is its space plus every intermediate
|
||||
// node up the tree; the walk uses GetWikiNode (parent_node_token) and is O(depth)
|
||||
// per selection, so it never re-traverses the whole wiki.
|
||||
func (c *Connector) ResolveResourceAncestors(
|
||||
ctx context.Context, config *types.DataSourceConfig, resourceIDs []string,
|
||||
) ([]string, error) {
|
||||
feishuConfig, err := parseFeishuConfig(config)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
client := NewClient(feishuConfig)
|
||||
|
||||
seen := make(map[string]bool)
|
||||
ancestors := make([]string, 0)
|
||||
add := func(id string) {
|
||||
if id != "" && !seen[id] {
|
||||
seen[id] = true
|
||||
ancestors = append(ancestors, id)
|
||||
}
|
||||
}
|
||||
|
||||
for _, rid := range resourceIDs {
|
||||
spaceID, nodeToken := parseWikiResourceID(rid)
|
||||
if spaceID == "" || nodeToken == "" {
|
||||
// A space-level selection is already a top-level node in the picker;
|
||||
// there is nothing above it to reveal.
|
||||
continue
|
||||
}
|
||||
// The space's direct children must be loaded to reveal the top-level node.
|
||||
add(spaceID)
|
||||
|
||||
// Walk up from the selection to the top, loading each intermediate
|
||||
// parent so the path down to the selection becomes visible.
|
||||
current := nodeToken
|
||||
for current != "" {
|
||||
node, err := client.GetWikiNode(ctx, spaceID, current)
|
||||
if err != nil {
|
||||
// Best-effort: a broken path just stays collapsed, the rest of
|
||||
// the selections are still revealed.
|
||||
logger.Warnf(ctx, "[Feishu] resolve ancestors: get node %s:%s: %v", spaceID, current, err)
|
||||
break
|
||||
}
|
||||
if node.ParentNodeID == "" {
|
||||
break
|
||||
}
|
||||
add(makeWikiNodeResourceID(spaceID, node.ParentNodeID))
|
||||
current = node.ParentNodeID
|
||||
}
|
||||
}
|
||||
|
||||
return ancestors, nil
|
||||
}
|
||||
|
||||
// FetchAll performs a full sync of all documents from the specified wiki spaces.
|
||||
func (c *Connector) FetchAll(ctx context.Context, config *types.DataSourceConfig, resourceIDs []string) ([]types.FetchedItem, error) {
|
||||
feishuConfig, err := parseFeishuConfig(config)
|
||||
|
||||
@@ -529,6 +529,69 @@ func TestConnectorListResources_LazyLoadsOneLevel(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestConnectorResolveResourceAncestors verifies that the ancestor chain of a
|
||||
// deeply nested selection is resolved (so an edit-mode picker can reveal it)
|
||||
// without listing the whole tree.
|
||||
func TestConnectorResolveResourceAncestors(t *testing.T) {
|
||||
topNodes := []wikiNode{
|
||||
{NodeToken: "nt-root", ObjToken: "obj-root", ObjType: "docx", Title: "Root", HasChild: true},
|
||||
}
|
||||
childNodes := map[string][]wikiNode{
|
||||
"nt-root": {
|
||||
{NodeToken: "nt-child", ObjToken: "obj-child", ObjType: "docx", Title: "Child", HasChild: true},
|
||||
},
|
||||
"nt-child": {
|
||||
{NodeToken: "nt-grandchild", ObjToken: "obj-gc", ObjType: "docx", Title: "Grandchild"},
|
||||
},
|
||||
}
|
||||
ts, cfg := fakeFeishuHierarchy(topNodes, childNodes, "")
|
||||
defer ts.Close()
|
||||
|
||||
c := NewConnector()
|
||||
|
||||
// A deeply nested node resolves to its space plus every intermediate parent.
|
||||
ancestors, err := c.ResolveResourceAncestors(
|
||||
context.Background(), makeConfig(cfg, nil), []string{"space1:nt-grandchild"},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("ResolveResourceAncestors error: %v", err)
|
||||
}
|
||||
got := make(map[string]bool)
|
||||
for _, a := range ancestors {
|
||||
got[a] = true
|
||||
}
|
||||
for _, want := range []string{"space1", "space1:nt-root", "space1:nt-child"} {
|
||||
if !got[want] {
|
||||
t.Fatalf("missing ancestor %q, got %+v", want, ancestors)
|
||||
}
|
||||
}
|
||||
if got["space1:nt-grandchild"] {
|
||||
t.Fatalf("selection itself must not be returned as its own ancestor: %+v", ancestors)
|
||||
}
|
||||
|
||||
// A top-level node only needs its space loaded.
|
||||
ancestors, err = c.ResolveResourceAncestors(
|
||||
context.Background(), makeConfig(cfg, nil), []string{"space1:nt-root"},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("ResolveResourceAncestors(top-level) error: %v", err)
|
||||
}
|
||||
if len(ancestors) != 1 || ancestors[0] != "space1" {
|
||||
t.Fatalf("want [space1] for a top-level node, got %+v", ancestors)
|
||||
}
|
||||
|
||||
// A whole-space selection is already top-level: nothing to reveal.
|
||||
ancestors, err = c.ResolveResourceAncestors(
|
||||
context.Background(), makeConfig(cfg, nil), []string{"space1"},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("ResolveResourceAncestors(space) error: %v", err)
|
||||
}
|
||||
if len(ancestors) != 0 {
|
||||
t.Fatalf("want no ancestors for a space selection, got %+v", ancestors)
|
||||
}
|
||||
}
|
||||
|
||||
// ──────────────────────────────────────────────────────────────────────
|
||||
// FetchAll: test all supported doc types
|
||||
// ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
@@ -121,18 +121,18 @@ type wikiNodeListResponse struct {
|
||||
|
||||
// wikiNode represents a node (document or folder) in a Feishu Wiki space.
|
||||
type wikiNode struct {
|
||||
SpaceID string `json:"space_id"`
|
||||
NodeToken string `json:"node_token"`
|
||||
ObjToken string `json:"obj_token"` // document token
|
||||
ObjType string `json:"obj_type"` // "doc", "sheet", "mindnote", "bitable", "file", "docx", "slides"
|
||||
ParentNodeID string `json:"parent_node_id"`
|
||||
NodeType string `json:"node_type"` // "origin" or "shortcut"
|
||||
OriginNodeID string `json:"origin_node_id"`
|
||||
OriginSpaceID string `json:"origin_space_id"`
|
||||
HasChild bool `json:"has_child"`
|
||||
Title string `json:"title"`
|
||||
Creator string `json:"creator"`
|
||||
Owner string `json:"owner"`
|
||||
SpaceID string `json:"space_id"`
|
||||
NodeToken string `json:"node_token"`
|
||||
ObjToken string `json:"obj_token"` // document token
|
||||
ObjType string `json:"obj_type"` // "doc", "sheet", "mindnote", "bitable", "file", "docx", "slides"
|
||||
ParentNodeID string `json:"parent_node_token"`
|
||||
NodeType string `json:"node_type"` // "origin" or "shortcut"
|
||||
OriginNodeID string `json:"origin_node_id"`
|
||||
OriginSpaceID string `json:"origin_space_id"`
|
||||
HasChild bool `json:"has_child"`
|
||||
Title string `json:"title"`
|
||||
Creator string `json:"creator"`
|
||||
Owner string `json:"owner"`
|
||||
ObjCreateTime string `json:"obj_create_time"` // document creation time (unix timestamp string)
|
||||
ObjEditTime string `json:"obj_edit_time"` // document last edit time (unix timestamp string) — tracks content changes
|
||||
NodeCreateTime string `json:"node_create_time"` // node creation time (unix timestamp string)
|
||||
|
||||
@@ -44,6 +44,15 @@ func (c *Connector) Validate(ctx context.Context, config *types.DataSourceConfig
|
||||
return client.Ping(ctx)
|
||||
}
|
||||
|
||||
// ResolveResourceAncestors has nothing to do for Notion: ListResources already
|
||||
// returns the full tree with parent links, so any pre-existing selection is
|
||||
// already present and revealed by the picker without on-demand loading.
|
||||
func (c *Connector) ResolveResourceAncestors(
|
||||
ctx context.Context, config *types.DataSourceConfig, resourceIDs []string,
|
||||
) ([]string, error) {
|
||||
return []string{}, nil
|
||||
}
|
||||
|
||||
// ListResources lists all accessible Notion pages and databases as selectable resources.
|
||||
// Returns all objects with parent-child relationships populated, allowing the frontend
|
||||
// to render a tree view. Root objects have empty ParentID.
|
||||
|
||||
@@ -40,6 +40,14 @@ func (c *Connector) Validate(ctx context.Context, config *types.DataSourceConfig
|
||||
return nil
|
||||
}
|
||||
|
||||
// ResolveResourceAncestors has nothing to do for Yuque: repositories are a flat
|
||||
// list with no nesting, so a selection has no ancestors to reveal.
|
||||
func (c *Connector) ResolveResourceAncestors(
|
||||
ctx context.Context, config *types.DataSourceConfig, resourceIDs []string,
|
||||
) ([]string, error) {
|
||||
return []string{}, nil
|
||||
}
|
||||
|
||||
// ListResources returns all repos (personal + team) accessible to the token.
|
||||
// Serial fetch for v1 (user groups typically <10). TODO(perf): parallelize if slow.
|
||||
func (c *Connector) ListResources(
|
||||
|
||||
@@ -366,6 +366,54 @@ func (h *DataSourceHandler) ListAvailableResources(c *gin.Context) {
|
||||
c.JSON(http.StatusOK, resources)
|
||||
}
|
||||
|
||||
// @Summary Resolve resource ancestors
|
||||
// @Description Resolve the ancestor ExternalIDs that must be expanded to reveal the given (possibly deeply nested) resources in a lazily-loaded picker. Used to restore an existing selection when editing a data source.
|
||||
// @Tags DataSource
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Param id path string true "Data source ID"
|
||||
// @Param request body resolveAncestorsRequest true "Resource IDs to resolve"
|
||||
// @Success 200 {object} map[string][]string
|
||||
// @Failure 400 {object} map[string]string
|
||||
// @Router /datasource/{id}/resource-ancestors [post]
|
||||
func (h *DataSourceHandler) ResolveResourceAncestors(c *gin.Context) {
|
||||
ctx := c.Request.Context()
|
||||
tenantID := h.getTenantID(c)
|
||||
if tenantID == 0 {
|
||||
c.JSON(http.StatusUnauthorized, gin.H{"error": "unauthorized"})
|
||||
return
|
||||
}
|
||||
|
||||
id := c.Param("id")
|
||||
|
||||
if _, status, msg := h.getOwnedDataSource(ctx, tenantID, id); status != http.StatusOK {
|
||||
c.JSON(status, gin.H{"error": msg})
|
||||
return
|
||||
}
|
||||
|
||||
var req resolveAncestorsRequest
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
ancestors, err := h.service.ResolveResourceAncestors(ctx, id, req.ResourceIDs)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
if ancestors == nil {
|
||||
ancestors = make([]string, 0)
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"ancestors": ancestors})
|
||||
}
|
||||
|
||||
// resolveAncestorsRequest is the body for ResolveResourceAncestors.
|
||||
type resolveAncestorsRequest struct {
|
||||
ResourceIDs []string `json:"resource_ids"`
|
||||
}
|
||||
|
||||
// ManualSync godoc
|
||||
// @Summary Trigger immediate sync
|
||||
// @Description Trigger an immediate sync for a data source
|
||||
|
||||
@@ -1716,6 +1716,7 @@ func RegisterDataSourceRoutes(
|
||||
// Connection and resource management — Admin+
|
||||
ds.POST("/:id/validate", g.Admin(), handler.ValidateConnection)
|
||||
ds.GET("/:id/resources", g.Admin(), handler.ListAvailableResources)
|
||||
ds.POST("/:id/resource-ancestors", g.Admin(), handler.ResolveResourceAncestors)
|
||||
|
||||
// Sync management — Admin+
|
||||
ds.POST("/:id/sync", g.Admin(), handler.ManualSync)
|
||||
|
||||
@@ -47,6 +47,12 @@ type DataSourceService interface {
|
||||
// parentID enables lazy loading: "" lists the top level, a resource ExternalID lists its children.
|
||||
ListAvailableResources(ctx context.Context, dsID string, parentID string) ([]types.Resource, error)
|
||||
|
||||
// ResolveResourceAncestors returns the deduplicated ExternalIDs of every
|
||||
// ancestor that must be expanded to reveal the given (possibly deeply nested)
|
||||
// resources in a lazily-loaded picker. Used to restore an existing selection
|
||||
// when editing a data source.
|
||||
ResolveResourceAncestors(ctx context.Context, dsID string, resourceIDs []string) ([]string, error)
|
||||
|
||||
// ManualSync triggers an immediate sync for a data source
|
||||
ManualSync(ctx context.Context, dsID string) (*types.SyncLog, error)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user