Files
Enterprise-Knowledge-Base/internal/datasource/connector/feishu/connector.go
T
wizardchen adabf8e089 feat(datasource): implement data source management functionality
- Added a new API for managing data sources, including CRUD operations and connection validation.
- Introduced UI components for data source settings, sync logs, and editor dialog, enhancing user experience in managing external data sources.
- Updated localization files to support new data source features in multiple languages.
- Implemented cron expression humanization for better scheduling visibility.
- Enhanced knowledge base editor to include data source management options.
2026-03-30 20:46:07 +08:00

377 lines
11 KiB
Go

package feishu
import (
"context"
"encoding/json"
"fmt"
"strconv"
"strings"
"time"
"github.com/Tencent/WeKnora/internal/types"
)
// Connector implements the datasource.Connector interface for Feishu.
type Connector struct{}
// NewConnector creates a new Feishu connector.
func NewConnector() *Connector {
return &Connector{}
}
// Type returns the connector type identifier.
func (c *Connector) Type() string {
return types.ConnectorTypeFeishu
}
// Validate verifies that the Feishu configuration is valid by testing connectivity.
func (c *Connector) Validate(ctx context.Context, config *types.DataSourceConfig) error {
feishuConfig, err := parseFeishuConfig(config)
if err != nil {
return err
}
client := NewClient(feishuConfig)
if err := client.Ping(ctx); err != nil {
return fmt.Errorf("feishu connection failed: %w", err)
}
return nil
}
// ListResources lists all accessible Feishu Wiki spaces as selectable resources.
func (c *Connector) ListResources(ctx context.Context, config *types.DataSourceConfig) ([]types.Resource, error) {
feishuConfig, err := parseFeishuConfig(config)
if err != nil {
return nil, err
}
client := NewClient(feishuConfig)
spaces, err := client.ListWikiSpaces(ctx)
if err != nil {
return nil, fmt.Errorf("list feishu wiki spaces: %w", err)
}
resources := make([]types.Resource, 0, len(spaces))
for _, space := range spaces {
resources = append(resources, types.Resource{
ExternalID: space.SpaceID,
Name: space.Name,
Type: "wiki_space",
Description: space.Description,
URL: fmt.Sprintf("https://feishu.cn/wiki/%s", space.SpaceID),
Metadata: map[string]interface{}{
"visibility": space.Visibility,
},
})
}
return resources, nil
}
// FetchAll performs a full sync of all documents from the specified wiki spaces.
func (c *Connector) FetchAll(ctx context.Context, config *types.DataSourceConfig, resourceIDs []string) ([]types.FetchedItem, error) {
feishuConfig, err := parseFeishuConfig(config)
if err != nil {
return nil, err
}
client := NewClient(feishuConfig)
var allItems []types.FetchedItem
for _, spaceID := range resourceIDs {
// List all nodes in this wiki space recursively
nodes, err := client.ListAllWikiNodesRecursive(ctx, spaceID)
if err != nil {
return nil, fmt.Errorf("list nodes in space %s: %w", spaceID, err)
}
// Fetch content for each document node
for _, node := range nodes {
item, err := c.fetchNodeContent(ctx, client, node, spaceID)
if err != nil {
// Log error but continue with other nodes
allItems = append(allItems, types.FetchedItem{
ExternalID: node.NodeToken,
Title: node.Title,
SourceResourceID: spaceID,
Metadata: map[string]string{
"error": err.Error(),
},
})
continue
}
if item != nil {
allItems = append(allItems, *item)
}
}
}
return allItems, nil
}
// FetchIncremental performs an incremental sync by comparing node edit times
// against the previously recorded state.
func (c *Connector) FetchIncremental(ctx context.Context, config *types.DataSourceConfig, cursor *types.SyncCursor) ([]types.FetchedItem, *types.SyncCursor, error) {
feishuConfig, err := parseFeishuConfig(config)
if err != nil {
return nil, nil, err
}
client := NewClient(feishuConfig)
// Parse the previous cursor state
var prevCursor feishuCursor
if cursor != nil && cursor.ConnectorCursor != nil {
cursorBytes, _ := json.Marshal(cursor.ConnectorCursor)
_ = json.Unmarshal(cursorBytes, &prevCursor)
}
// Build new cursor to track current state
newCursor := feishuCursor{
LastSyncTime: time.Now(),
SpaceNodeTimes: make(map[string]map[string]string),
}
var changedItems []types.FetchedItem
// Get resource IDs from config
resourceIDs := config.ResourceIDs
if len(resourceIDs) == 0 {
return nil, nil, fmt.Errorf("no resource IDs (wiki space IDs) configured")
}
for _, spaceID := range resourceIDs {
// List all nodes in this space
nodes, err := client.ListAllWikiNodesRecursive(ctx, spaceID)
if err != nil {
return nil, nil, fmt.Errorf("list nodes in space %s: %w", spaceID, err)
}
newCursor.SpaceNodeTimes[spaceID] = make(map[string]string)
// Build a set of current node tokens for deletion detection
currentNodes := make(map[string]bool)
for _, node := range nodes {
currentNodes[node.NodeToken] = true
// Use ObjEditTime (document content edit time) for change detection,
// NOT NodeEditTime which only tracks node attribute changes (title, position).
editTimeStr := node.ObjEditTime
if editTimeStr == "" {
editTimeStr = node.NodeEditTime // fallback for nodes that don't have obj_edit_time
}
newCursor.SpaceNodeTimes[spaceID][node.NodeToken] = editTimeStr
// Check if node has changed since last sync
if prevCursor.SpaceNodeTimes != nil {
if prevTimes, ok := prevCursor.SpaceNodeTimes[spaceID]; ok {
if prevEditTime, exists := prevTimes[node.NodeToken]; exists {
if prevEditTime == editTimeStr {
// Node unchanged, skip
continue
}
}
}
}
// Node is new or changed — fetch its content
item, err := c.fetchNodeContent(ctx, client, node, spaceID)
if err != nil {
// Record failed items
changedItems = append(changedItems, types.FetchedItem{
ExternalID: node.NodeToken,
Title: node.Title,
SourceResourceID: spaceID,
Metadata: map[string]string{
"error": err.Error(),
},
})
continue
}
if item != nil {
changedItems = append(changedItems, *item)
}
}
// Detect deleted nodes
if prevCursor.SpaceNodeTimes != nil {
if prevTimes, ok := prevCursor.SpaceNodeTimes[spaceID]; ok {
for nodeToken := range prevTimes {
if !currentNodes[nodeToken] {
// Node was deleted
changedItems = append(changedItems, types.FetchedItem{
ExternalID: nodeToken,
IsDeleted: true,
SourceResourceID: spaceID,
})
}
}
}
}
}
// Build next sync cursor
nextCursorMap := make(map[string]interface{})
cursorBytes, _ := json.Marshal(newCursor)
_ = json.Unmarshal(cursorBytes, &nextCursorMap)
nextSyncCursor := &types.SyncCursor{
LastSyncTime: time.Now(),
ConnectorCursor: nextCursorMap,
}
return changedItems, nextSyncCursor, nil
}
// fetchNodeContent fetches the content of a single wiki node and converts it to FetchedItem.
// Dispatches to different retrieval strategies based on obj_type:
// - docx/doc → export API → .docx file
// - sheet → export API → .xlsx file
// - bitable → export API → .xlsx file
// - file → drive download → original file (PDF/Word/image/etc.)
// - mindnote → skip (no API)
// - slides → skip (no API)
func (c *Connector) fetchNodeContent(ctx context.Context, client *Client, node wikiNode, spaceID string) (*types.FetchedItem, error) {
if !isSupportedDocType(node.ObjType) {
return nil, nil
}
editTime := parseFeishuTimestamp(node.NodeEditTime)
baseMeta := map[string]string{
"obj_token": node.ObjToken,
"obj_type": node.ObjType,
"node_token": node.NodeToken,
"space_id": spaceID,
"creator": node.Creator,
"owner": node.Owner,
"channel": types.ChannelFeishu,
}
switch node.ObjType {
case "docx", "doc", "sheet", "bitable":
// Export as a file via the async export API
data, fileName, err := client.ExportAndDownload(ctx, node.ObjToken, node.ObjType)
if err != nil {
return nil, fmt.Errorf("export %s (%s): %w", node.Title, node.ObjType, err)
}
// Ensure a reasonable file name with correct extension
ext := exportFileExtToSuffix[objTypeToExportFileExtension[node.ObjType]]
if fileName == "" {
fileName = sanitizeFileName(node.Title) + ext
} else if !strings.HasSuffix(strings.ToLower(fileName), ext) {
// Feishu often returns the doc title without extension — append it
fileName = sanitizeFileName(fileName) + ext
}
return &types.FetchedItem{
ExternalID: node.NodeToken,
Title: node.Title,
Content: data,
ContentType: "application/octet-stream",
FileName: fileName,
URL: fmt.Sprintf("https://feishu.cn/wiki/%s", node.NodeToken),
UpdatedAt: editTime,
SourceResourceID: spaceID,
Metadata: baseMeta,
}, nil
case "file":
// Download the original uploaded file from Drive
data, err := client.DownloadDriveFile(ctx, node.ObjToken)
if err != nil {
return nil, fmt.Errorf("download file %s (%s): %w", node.Title, node.ObjToken, err)
}
// Use the node title as file name; it usually preserves the original extension
fileName := node.Title
if fileName == "" {
fileName = node.ObjToken
}
return &types.FetchedItem{
ExternalID: node.NodeToken,
Title: node.Title,
Content: data,
ContentType: "application/octet-stream",
FileName: fileName,
URL: fmt.Sprintf("https://feishu.cn/wiki/%s", node.NodeToken),
UpdatedAt: editTime,
SourceResourceID: spaceID,
Metadata: baseMeta,
}, nil
default:
return nil, nil
}
}
// --- Helper functions ---
// parseFeishuConfig extracts and validates Feishu-specific configuration.
func parseFeishuConfig(config *types.DataSourceConfig) (*Config, error) {
if config == nil {
return nil, fmt.Errorf("config is nil")
}
credBytes, err := json.Marshal(config.Credentials)
if err != nil {
return nil, fmt.Errorf("marshal credentials: %w", err)
}
var feishuConfig Config
if err := json.Unmarshal(credBytes, &feishuConfig); err != nil {
return nil, fmt.Errorf("parse feishu credentials: %w", err)
}
if feishuConfig.AppID == "" || feishuConfig.AppSecret == "" {
return nil, fmt.Errorf("feishu app_id and app_secret are required")
}
return &feishuConfig, nil
}
// isSupportedDocType checks if a Feishu document type can be synced.
// mindnote and slides have no content read API and are skipped.
func isSupportedDocType(objType string) bool {
switch objType {
case "docx", "doc", "sheet", "bitable", "file":
return true
default:
// mindnote, slides — no content retrieval API available
return false
}
}
// parseFeishuTimestamp parses a Feishu unix timestamp string (seconds) into time.Time.
func parseFeishuTimestamp(ts string) time.Time {
if ts == "" {
return time.Time{}
}
sec, err := strconv.ParseInt(ts, 10, 64)
if err != nil {
return time.Time{}
}
return time.Unix(sec, 0)
}
// sanitizeFileName removes characters that are invalid in filenames.
func sanitizeFileName(name string) string {
if name == "" {
return "untitled"
}
// Replace common invalid characters with underscore
replacer := strings.NewReplacer(
"/", "_", "\\", "_", ":", "_", "*", "_",
"?", "_", "\"", "_", "<", "_", ">", "_", "|", "_",
)
result := replacer.Replace(name)
// Limit length
if len(result) > 200 {
result = result[:200]
}
return result
}