adabf8e089
- Added a new API for managing data sources, including CRUD operations and connection validation. - Introduced UI components for data source settings, sync logs, and editor dialog, enhancing user experience in managing external data sources. - Updated localization files to support new data source features in multiple languages. - Implemented cron expression humanization for better scheduling visibility. - Enhanced knowledge base editor to include data source management options.
377 lines
11 KiB
Go
377 lines
11 KiB
Go
package feishu
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/Tencent/WeKnora/internal/types"
|
|
)
|
|
|
|
// Connector implements the datasource.Connector interface for Feishu.
|
|
type Connector struct{}
|
|
|
|
// NewConnector creates a new Feishu connector.
|
|
func NewConnector() *Connector {
|
|
return &Connector{}
|
|
}
|
|
|
|
// Type returns the connector type identifier.
|
|
func (c *Connector) Type() string {
|
|
return types.ConnectorTypeFeishu
|
|
}
|
|
|
|
// Validate verifies that the Feishu configuration is valid by testing connectivity.
|
|
func (c *Connector) Validate(ctx context.Context, config *types.DataSourceConfig) error {
|
|
feishuConfig, err := parseFeishuConfig(config)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
client := NewClient(feishuConfig)
|
|
if err := client.Ping(ctx); err != nil {
|
|
return fmt.Errorf("feishu connection failed: %w", err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// ListResources lists all accessible Feishu Wiki spaces as selectable resources.
|
|
func (c *Connector) ListResources(ctx context.Context, config *types.DataSourceConfig) ([]types.Resource, error) {
|
|
feishuConfig, err := parseFeishuConfig(config)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
client := NewClient(feishuConfig)
|
|
spaces, err := client.ListWikiSpaces(ctx)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("list feishu wiki spaces: %w", err)
|
|
}
|
|
|
|
resources := make([]types.Resource, 0, len(spaces))
|
|
for _, space := range spaces {
|
|
resources = append(resources, types.Resource{
|
|
ExternalID: space.SpaceID,
|
|
Name: space.Name,
|
|
Type: "wiki_space",
|
|
Description: space.Description,
|
|
URL: fmt.Sprintf("https://feishu.cn/wiki/%s", space.SpaceID),
|
|
Metadata: map[string]interface{}{
|
|
"visibility": space.Visibility,
|
|
},
|
|
})
|
|
}
|
|
|
|
return resources, nil
|
|
}
|
|
|
|
// FetchAll performs a full sync of all documents from the specified wiki spaces.
|
|
func (c *Connector) FetchAll(ctx context.Context, config *types.DataSourceConfig, resourceIDs []string) ([]types.FetchedItem, error) {
|
|
feishuConfig, err := parseFeishuConfig(config)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
client := NewClient(feishuConfig)
|
|
|
|
var allItems []types.FetchedItem
|
|
|
|
for _, spaceID := range resourceIDs {
|
|
// List all nodes in this wiki space recursively
|
|
nodes, err := client.ListAllWikiNodesRecursive(ctx, spaceID)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("list nodes in space %s: %w", spaceID, err)
|
|
}
|
|
|
|
// Fetch content for each document node
|
|
for _, node := range nodes {
|
|
item, err := c.fetchNodeContent(ctx, client, node, spaceID)
|
|
if err != nil {
|
|
// Log error but continue with other nodes
|
|
allItems = append(allItems, types.FetchedItem{
|
|
ExternalID: node.NodeToken,
|
|
Title: node.Title,
|
|
SourceResourceID: spaceID,
|
|
Metadata: map[string]string{
|
|
"error": err.Error(),
|
|
},
|
|
})
|
|
continue
|
|
}
|
|
if item != nil {
|
|
allItems = append(allItems, *item)
|
|
}
|
|
}
|
|
}
|
|
|
|
return allItems, nil
|
|
}
|
|
|
|
// FetchIncremental performs an incremental sync by comparing node edit times
|
|
// against the previously recorded state.
|
|
func (c *Connector) FetchIncremental(ctx context.Context, config *types.DataSourceConfig, cursor *types.SyncCursor) ([]types.FetchedItem, *types.SyncCursor, error) {
|
|
feishuConfig, err := parseFeishuConfig(config)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
|
|
client := NewClient(feishuConfig)
|
|
|
|
// Parse the previous cursor state
|
|
var prevCursor feishuCursor
|
|
if cursor != nil && cursor.ConnectorCursor != nil {
|
|
cursorBytes, _ := json.Marshal(cursor.ConnectorCursor)
|
|
_ = json.Unmarshal(cursorBytes, &prevCursor)
|
|
}
|
|
|
|
// Build new cursor to track current state
|
|
newCursor := feishuCursor{
|
|
LastSyncTime: time.Now(),
|
|
SpaceNodeTimes: make(map[string]map[string]string),
|
|
}
|
|
|
|
var changedItems []types.FetchedItem
|
|
|
|
// Get resource IDs from config
|
|
resourceIDs := config.ResourceIDs
|
|
if len(resourceIDs) == 0 {
|
|
return nil, nil, fmt.Errorf("no resource IDs (wiki space IDs) configured")
|
|
}
|
|
|
|
for _, spaceID := range resourceIDs {
|
|
// List all nodes in this space
|
|
nodes, err := client.ListAllWikiNodesRecursive(ctx, spaceID)
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("list nodes in space %s: %w", spaceID, err)
|
|
}
|
|
|
|
newCursor.SpaceNodeTimes[spaceID] = make(map[string]string)
|
|
|
|
// Build a set of current node tokens for deletion detection
|
|
currentNodes := make(map[string]bool)
|
|
|
|
for _, node := range nodes {
|
|
currentNodes[node.NodeToken] = true
|
|
// Use ObjEditTime (document content edit time) for change detection,
|
|
// NOT NodeEditTime which only tracks node attribute changes (title, position).
|
|
editTimeStr := node.ObjEditTime
|
|
if editTimeStr == "" {
|
|
editTimeStr = node.NodeEditTime // fallback for nodes that don't have obj_edit_time
|
|
}
|
|
newCursor.SpaceNodeTimes[spaceID][node.NodeToken] = editTimeStr
|
|
|
|
// Check if node has changed since last sync
|
|
if prevCursor.SpaceNodeTimes != nil {
|
|
if prevTimes, ok := prevCursor.SpaceNodeTimes[spaceID]; ok {
|
|
if prevEditTime, exists := prevTimes[node.NodeToken]; exists {
|
|
if prevEditTime == editTimeStr {
|
|
// Node unchanged, skip
|
|
continue
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Node is new or changed — fetch its content
|
|
item, err := c.fetchNodeContent(ctx, client, node, spaceID)
|
|
if err != nil {
|
|
// Record failed items
|
|
changedItems = append(changedItems, types.FetchedItem{
|
|
ExternalID: node.NodeToken,
|
|
Title: node.Title,
|
|
SourceResourceID: spaceID,
|
|
Metadata: map[string]string{
|
|
"error": err.Error(),
|
|
},
|
|
})
|
|
continue
|
|
}
|
|
if item != nil {
|
|
changedItems = append(changedItems, *item)
|
|
}
|
|
}
|
|
|
|
// Detect deleted nodes
|
|
if prevCursor.SpaceNodeTimes != nil {
|
|
if prevTimes, ok := prevCursor.SpaceNodeTimes[spaceID]; ok {
|
|
for nodeToken := range prevTimes {
|
|
if !currentNodes[nodeToken] {
|
|
// Node was deleted
|
|
changedItems = append(changedItems, types.FetchedItem{
|
|
ExternalID: nodeToken,
|
|
IsDeleted: true,
|
|
SourceResourceID: spaceID,
|
|
})
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Build next sync cursor
|
|
nextCursorMap := make(map[string]interface{})
|
|
cursorBytes, _ := json.Marshal(newCursor)
|
|
_ = json.Unmarshal(cursorBytes, &nextCursorMap)
|
|
|
|
nextSyncCursor := &types.SyncCursor{
|
|
LastSyncTime: time.Now(),
|
|
ConnectorCursor: nextCursorMap,
|
|
}
|
|
|
|
return changedItems, nextSyncCursor, nil
|
|
}
|
|
|
|
// fetchNodeContent fetches the content of a single wiki node and converts it to FetchedItem.
|
|
// Dispatches to different retrieval strategies based on obj_type:
|
|
// - docx/doc → export API → .docx file
|
|
// - sheet → export API → .xlsx file
|
|
// - bitable → export API → .xlsx file
|
|
// - file → drive download → original file (PDF/Word/image/etc.)
|
|
// - mindnote → skip (no API)
|
|
// - slides → skip (no API)
|
|
func (c *Connector) fetchNodeContent(ctx context.Context, client *Client, node wikiNode, spaceID string) (*types.FetchedItem, error) {
|
|
if !isSupportedDocType(node.ObjType) {
|
|
return nil, nil
|
|
}
|
|
|
|
editTime := parseFeishuTimestamp(node.NodeEditTime)
|
|
baseMeta := map[string]string{
|
|
"obj_token": node.ObjToken,
|
|
"obj_type": node.ObjType,
|
|
"node_token": node.NodeToken,
|
|
"space_id": spaceID,
|
|
"creator": node.Creator,
|
|
"owner": node.Owner,
|
|
"channel": types.ChannelFeishu,
|
|
}
|
|
|
|
switch node.ObjType {
|
|
case "docx", "doc", "sheet", "bitable":
|
|
// Export as a file via the async export API
|
|
data, fileName, err := client.ExportAndDownload(ctx, node.ObjToken, node.ObjType)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("export %s (%s): %w", node.Title, node.ObjType, err)
|
|
}
|
|
|
|
// Ensure a reasonable file name with correct extension
|
|
ext := exportFileExtToSuffix[objTypeToExportFileExtension[node.ObjType]]
|
|
if fileName == "" {
|
|
fileName = sanitizeFileName(node.Title) + ext
|
|
} else if !strings.HasSuffix(strings.ToLower(fileName), ext) {
|
|
// Feishu often returns the doc title without extension — append it
|
|
fileName = sanitizeFileName(fileName) + ext
|
|
}
|
|
|
|
return &types.FetchedItem{
|
|
ExternalID: node.NodeToken,
|
|
Title: node.Title,
|
|
Content: data,
|
|
ContentType: "application/octet-stream",
|
|
FileName: fileName,
|
|
URL: fmt.Sprintf("https://feishu.cn/wiki/%s", node.NodeToken),
|
|
UpdatedAt: editTime,
|
|
SourceResourceID: spaceID,
|
|
Metadata: baseMeta,
|
|
}, nil
|
|
|
|
case "file":
|
|
// Download the original uploaded file from Drive
|
|
data, err := client.DownloadDriveFile(ctx, node.ObjToken)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("download file %s (%s): %w", node.Title, node.ObjToken, err)
|
|
}
|
|
|
|
// Use the node title as file name; it usually preserves the original extension
|
|
fileName := node.Title
|
|
if fileName == "" {
|
|
fileName = node.ObjToken
|
|
}
|
|
|
|
return &types.FetchedItem{
|
|
ExternalID: node.NodeToken,
|
|
Title: node.Title,
|
|
Content: data,
|
|
ContentType: "application/octet-stream",
|
|
FileName: fileName,
|
|
URL: fmt.Sprintf("https://feishu.cn/wiki/%s", node.NodeToken),
|
|
UpdatedAt: editTime,
|
|
SourceResourceID: spaceID,
|
|
Metadata: baseMeta,
|
|
}, nil
|
|
|
|
default:
|
|
return nil, nil
|
|
}
|
|
}
|
|
|
|
// --- Helper functions ---
|
|
|
|
// parseFeishuConfig extracts and validates Feishu-specific configuration.
|
|
func parseFeishuConfig(config *types.DataSourceConfig) (*Config, error) {
|
|
if config == nil {
|
|
return nil, fmt.Errorf("config is nil")
|
|
}
|
|
|
|
credBytes, err := json.Marshal(config.Credentials)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("marshal credentials: %w", err)
|
|
}
|
|
|
|
var feishuConfig Config
|
|
if err := json.Unmarshal(credBytes, &feishuConfig); err != nil {
|
|
return nil, fmt.Errorf("parse feishu credentials: %w", err)
|
|
}
|
|
|
|
if feishuConfig.AppID == "" || feishuConfig.AppSecret == "" {
|
|
return nil, fmt.Errorf("feishu app_id and app_secret are required")
|
|
}
|
|
|
|
return &feishuConfig, nil
|
|
}
|
|
|
|
// isSupportedDocType checks if a Feishu document type can be synced.
|
|
// mindnote and slides have no content read API and are skipped.
|
|
func isSupportedDocType(objType string) bool {
|
|
switch objType {
|
|
case "docx", "doc", "sheet", "bitable", "file":
|
|
return true
|
|
default:
|
|
// mindnote, slides — no content retrieval API available
|
|
return false
|
|
}
|
|
}
|
|
|
|
// parseFeishuTimestamp parses a Feishu unix timestamp string (seconds) into time.Time.
|
|
func parseFeishuTimestamp(ts string) time.Time {
|
|
if ts == "" {
|
|
return time.Time{}
|
|
}
|
|
sec, err := strconv.ParseInt(ts, 10, 64)
|
|
if err != nil {
|
|
return time.Time{}
|
|
}
|
|
return time.Unix(sec, 0)
|
|
}
|
|
|
|
// sanitizeFileName removes characters that are invalid in filenames.
|
|
func sanitizeFileName(name string) string {
|
|
if name == "" {
|
|
return "untitled"
|
|
}
|
|
// Replace common invalid characters with underscore
|
|
replacer := strings.NewReplacer(
|
|
"/", "_", "\\", "_", ":", "_", "*", "_",
|
|
"?", "_", "\"", "_", "<", "_", ">", "_", "|", "_",
|
|
)
|
|
result := replacer.Replace(name)
|
|
// Limit length
|
|
if len(result) > 200 {
|
|
result = result[:200]
|
|
}
|
|
return result
|
|
}
|