feat(datasource): implement data source management functionality

- Added a new API for managing data sources, including CRUD operations and connection validation.
- Introduced UI components for data source settings, sync logs, and editor dialog, enhancing user experience in managing external data sources.
- Updated localization files to support new data source features in multiple languages.
- Implemented cron expression humanization for better scheduling visibility.
- Enhanced knowledge base editor to include data source management options.
This commit is contained in:
wizardchen
2026-03-27 15:02:40 +08:00
committed by lyingbug
parent ed393c4683
commit adabf8e089
44 changed files with 8159 additions and 42 deletions
@@ -0,0 +1,451 @@
package feishu
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"sync"
"time"
"github.com/Tencent/WeKnora/internal/logger"
)
// Client wraps the Feishu Open Platform API for document/wiki operations.
type Client struct {
baseURL string
appID string
appSecret string
httpClient *http.Client
// Token cache (thread-safe)
tokenMu sync.Mutex
tokenCache string
tokenExpAt time.Time
}
// NewClient creates a new Feishu API client.
func NewClient(config *Config) *Client {
return &Client{
baseURL: config.GetBaseURL(),
appID: config.AppID,
appSecret: config.AppSecret,
httpClient: &http.Client{Timeout: 30 * time.Second},
}
}
// getTenantAccessToken retrieves (or returns cached) tenant access token.
// Feishu tokens expire in 2 hours; we cache with a 5-minute safety margin.
func (c *Client) getTenantAccessToken(ctx context.Context) (string, error) {
c.tokenMu.Lock()
defer c.tokenMu.Unlock()
if c.tokenCache != "" && time.Now().Before(c.tokenExpAt) {
return c.tokenCache, nil
}
payload, _ := json.Marshal(map[string]string{
"app_id": c.appID,
"app_secret": c.appSecret,
})
url := c.baseURL + "/open-apis/auth/v3/tenant_access_token/internal"
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(payload))
if err != nil {
return "", fmt.Errorf("create token request: %w", err)
}
req.Header.Set("Content-Type", "application/json; charset=utf-8")
resp, err := c.httpClient.Do(req)
if err != nil {
return "", fmt.Errorf("request token: %w", err)
}
defer resp.Body.Close()
var result tokenResponse
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
return "", fmt.Errorf("decode token response: %w", err)
}
if result.Code != 0 {
return "", fmt.Errorf("feishu auth error: code=%d msg=%s", result.Code, result.Msg)
}
c.tokenCache = result.TenantAccessToken
ttl := time.Duration(result.Expire) * time.Second
if ttl > 5*time.Minute {
ttl -= 5 * time.Minute
}
c.tokenExpAt = time.Now().Add(ttl)
logger.Infof(ctx, "[Feishu] got tenant_access_token: %s...%s expire=%ds",
result.TenantAccessToken[:8], result.TenantAccessToken[len(result.TenantAccessToken)-4:], result.Expire)
return c.tokenCache, nil
}
// doRequest executes an authenticated API request and decodes the JSON response.
func (c *Client) doRequest(ctx context.Context, method, path string, body interface{}, result interface{}) error {
token, err := c.getTenantAccessToken(ctx)
if err != nil {
return err
}
var bodyReader io.Reader
if body != nil {
bodyBytes, err := json.Marshal(body)
if err != nil {
return fmt.Errorf("marshal request body: %w", err)
}
bodyReader = bytes.NewReader(bodyBytes)
}
url := c.baseURL + path
req, err := http.NewRequestWithContext(ctx, method, url, bodyReader)
if err != nil {
return fmt.Errorf("create request: %w", err)
}
req.Header.Set("Content-Type", "application/json; charset=utf-8")
req.Header.Set("Authorization", "Bearer "+token)
logger.Infof(ctx, "[Feishu] %s %s", method, path)
resp, err := c.httpClient.Do(req)
if err != nil {
return fmt.Errorf("execute request: %w", err)
}
defer resp.Body.Close()
// Read body once for logging + decoding
respBody, err := io.ReadAll(resp.Body)
if err != nil {
return fmt.Errorf("read response body: %w", err)
}
logger.Infof(ctx, "[Feishu] %s %s → status=%d bodyLen=%d body=%s",
method, path, resp.StatusCode, len(respBody), truncate(string(respBody), 1000))
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("feishu api error: status=%d body=%s", resp.StatusCode, string(respBody))
}
if result != nil {
if err := json.Unmarshal(respBody, result); err != nil {
return fmt.Errorf("decode response: %w", err)
}
}
return nil
}
// truncate truncates a string to maxLen and appends "..." if truncated.
func truncate(s string, maxLen int) string {
if len(s) <= maxLen {
return s
}
return s[:maxLen] + "..."
}
// ListWikiSpaces returns all wiki spaces accessible to the app.
func (c *Client) ListWikiSpaces(ctx context.Context) ([]wikiSpace, error) {
var allSpaces []wikiSpace
pageToken := ""
for {
path := "/open-apis/wiki/v2/spaces?page_size=50"
if pageToken != "" {
path += "&page_token=" + pageToken
}
var resp wikiSpaceListResponse
if err := c.doRequest(ctx, http.MethodGet, path, nil, &resp); err != nil {
return nil, fmt.Errorf("list wiki spaces: %w", err)
}
if resp.Code != 0 {
logger.Errorf(ctx, "[Feishu] ListWikiSpaces error: code=%d msg=%s", resp.Code, resp.Msg)
return nil, fmt.Errorf("list wiki spaces error: code=%d msg=%s", resp.Code, resp.Msg)
}
logger.Infof(ctx, "[Feishu] ListWikiSpaces: got %d spaces, has_more=%v", len(resp.Data.Items), resp.Data.HasMore)
for i, s := range resp.Data.Items {
logger.Infof(ctx, "[Feishu] space[%d]: id=%s name=%q visibility=%s", i, s.SpaceID, s.Name, s.Visibility)
}
allSpaces = append(allSpaces, resp.Data.Items...)
if !resp.Data.HasMore || resp.Data.PageToken == "" {
break
}
pageToken = resp.Data.PageToken
}
logger.Infof(ctx, "[Feishu] ListWikiSpaces: total %d spaces", len(allSpaces))
return allSpaces, nil
}
// ListWikiNodes returns all nodes (documents) under a wiki space.
// If parentNodeToken is empty, returns top-level nodes.
func (c *Client) ListWikiNodes(ctx context.Context, spaceID string, parentNodeToken string) ([]wikiNode, error) {
var allNodes []wikiNode
pageToken := ""
for {
path := fmt.Sprintf("/open-apis/wiki/v2/spaces/%s/nodes?page_size=50", spaceID)
if parentNodeToken != "" {
path += "&parent_node_token=" + parentNodeToken
}
if pageToken != "" {
path += "&page_token=" + pageToken
}
var resp wikiNodeListResponse
if err := c.doRequest(ctx, http.MethodGet, path, nil, &resp); err != nil {
return nil, fmt.Errorf("list wiki nodes: %w", err)
}
if resp.Code != 0 {
return nil, fmt.Errorf("list wiki nodes error: code=%d msg=%s", resp.Code, resp.Msg)
}
allNodes = append(allNodes, resp.Data.Items...)
if !resp.Data.HasMore || resp.Data.PageToken == "" {
break
}
pageToken = resp.Data.PageToken
}
return allNodes, nil
}
// ListAllWikiNodesRecursive recursively lists all nodes under a wiki space.
// It walks the tree depth-first to discover all nested documents.
func (c *Client) ListAllWikiNodesRecursive(ctx context.Context, spaceID string) ([]wikiNode, error) {
// Start with top-level nodes
topNodes, err := c.ListWikiNodes(ctx, spaceID, "")
if err != nil {
return nil, err
}
var allNodes []wikiNode
var walk func(nodes []wikiNode) error
walk = func(nodes []wikiNode) error {
for _, node := range nodes {
allNodes = append(allNodes, node)
// Recurse into child nodes if this node has children
if node.HasChild {
children, err := c.ListWikiNodes(ctx, spaceID, node.NodeToken)
if err != nil {
return fmt.Errorf("list children of %s: %w", node.NodeToken, err)
}
if err := walk(children); err != nil {
return err
}
}
}
return nil
}
if err := walk(topNodes); err != nil {
return nil, err
}
return allNodes, nil
}
// GetDocumentRawContent retrieves the raw text content of a Feishu docx document.
// This returns plain text (not rich text / block structure).
// Deprecated: prefer ExportAndDownload which preserves formatting.
func (c *Client) GetDocumentRawContent(ctx context.Context, documentID string) (string, error) {
path := fmt.Sprintf("/open-apis/docx/v1/documents/%s/raw_content", documentID)
var resp docRawContentResponse
if err := c.doRequest(ctx, http.MethodGet, path, nil, &resp); err != nil {
return "", fmt.Errorf("get document raw content: %w", err)
}
if resp.Code != 0 {
return "", fmt.Errorf("get document raw content error: code=%d msg=%s", resp.Code, resp.Msg)
}
return resp.Data.Content, nil
}
// Ping verifies the credentials by attempting to get a tenant access token.
func (c *Client) Ping(ctx context.Context) error {
_, err := c.getTenantAccessToken(ctx)
return err
}
// ──────────────────────────────────────────────────────────────────────
// Export task API: export docx/sheet/bitable to downloadable files
//
// Flow:
// 1. POST /drive/v1/export_tasks → create export task, get ticket
// 2. GET /drive/v1/export_tasks/:ticket → poll until status=0 (success)
// 3. GET /drive/v1/export_tasks/file/:ticket/download → download file bytes
// ──────────────────────────────────────────────────────────────────────
// CreateExportTask creates an async export task for a Feishu document.
// - token: the obj_token of the document (e.g. docx token, sheet token)
// - objType: the Feishu obj_type ("docx", "doc", "sheet", "bitable")
// - fileExtension: desired output format ("docx", "xlsx", "pdf")
func (c *Client) CreateExportTask(ctx context.Context, token, objType, fileExtension string) (string, error) {
body := map[string]string{
"file_extension": fileExtension,
"token": token,
"type": objType,
}
var resp exportTaskCreateResponse
if err := c.doRequest(ctx, http.MethodPost, "/open-apis/drive/v1/export_tasks", body, &resp); err != nil {
return "", fmt.Errorf("create export task: %w", err)
}
if resp.Code != 0 {
return "", fmt.Errorf("create export task error: code=%d msg=%s", resp.Code, resp.Msg)
}
return resp.Data.Ticket, nil
}
// GetExportTaskStatus polls the status of an export task.
// Returns (fileToken, fileName, error). fileToken is non-empty only when the job succeeds.
// The token parameter is the obj_token of the document being exported (required by the API).
func (c *Client) GetExportTaskStatus(ctx context.Context, ticket string, token string) (string, string, error) {
path := fmt.Sprintf("/open-apis/drive/v1/export_tasks/%s?token=%s", ticket, token)
var resp exportTaskStatusResponse
if err := c.doRequest(ctx, http.MethodGet, path, nil, &resp); err != nil {
return "", "", fmt.Errorf("get export task status: %w", err)
}
if resp.Code != 0 {
return "", "", fmt.Errorf("get export task status error: code=%d msg=%s", resp.Code, resp.Msg)
}
r := resp.Data.Result
switch r.JobStatus {
case 0: // success
return r.FileToken, r.FileName, nil
case 1, 2: // initializing, processing
return "", "", nil // not ready yet
default:
return "", "", fmt.Errorf("export task failed: status=%d msg=%s", r.JobStatus, r.JobErrorMsg)
}
}
// DownloadExportFile downloads the exported file by its file_token.
// The file_token is returned by GetExportTaskStatus when the export job completes.
// The file must be downloaded within 10 minutes of export completion.
func (c *Client) DownloadExportFile(ctx context.Context, fileToken string) ([]byte, error) {
path := fmt.Sprintf("/open-apis/drive/v1/export_tasks/file/%s/download", fileToken)
return c.downloadRawBytes(ctx, path)
}
// ExportAndDownload is a high-level helper that creates an export task, polls until
// completion, and downloads the resulting file. Returns (fileBytes, fileName, error).
//
// Timeout: 60 seconds. Poll interval: 2 seconds.
func (c *Client) ExportAndDownload(ctx context.Context, objToken, objType string) ([]byte, string, error) {
// Determine export format
fileExt, ok := objTypeToExportFileExtension[objType]
if !ok {
return nil, "", fmt.Errorf("unsupported obj_type for export: %s", objType)
}
exportType, ok := objTypeToExportType[objType]
if !ok {
return nil, "", fmt.Errorf("unsupported obj_type for export: %s", objType)
}
// Step 1: create export task
ticket, err := c.CreateExportTask(ctx, objToken, exportType, fileExt)
if err != nil {
return nil, "", err
}
// Step 2: poll until ready (max 60s, every 2s)
deadline := time.Now().Add(60 * time.Second)
var fileToken, fileName string
for time.Now().Before(deadline) {
fileToken, fileName, err = c.GetExportTaskStatus(ctx, ticket, objToken)
if err != nil {
return nil, "", err
}
if fileToken != "" {
break // export ready
}
select {
case <-ctx.Done():
return nil, "", ctx.Err()
case <-time.After(2 * time.Second):
}
}
if fileToken == "" {
return nil, "", fmt.Errorf("export task timed out after 60s (ticket=%s)", ticket)
}
// Step 3: download file using file_token (NOT ticket)
data, err := c.DownloadExportFile(ctx, fileToken)
if err != nil {
return nil, "", err
}
// Build a sensible file name
if fileName == "" {
fileName = "export" + exportFileExtToSuffix[fileExt]
}
return data, fileName, nil
}
// ──────────────────────────────────────────────────────────────────────
// Drive file download: for "file" type wiki nodes (uploaded PDF/Word/etc.)
// ──────────────────────────────────────────────────────────────────────
// DownloadDriveFile downloads a file from Feishu Drive by its file token.
// Used for wiki nodes with obj_type="file" (user-uploaded PDF, Word, images, etc.).
func (c *Client) DownloadDriveFile(ctx context.Context, fileToken string) ([]byte, error) {
path := fmt.Sprintf("/open-apis/drive/v1/files/%s/download", fileToken)
return c.downloadRawBytes(ctx, path)
}
// downloadRawBytes performs an authenticated GET and returns the raw response body.
func (c *Client) downloadRawBytes(ctx context.Context, path string) ([]byte, error) {
token, err := c.getTenantAccessToken(ctx)
if err != nil {
return nil, err
}
url := c.baseURL + path
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, fmt.Errorf("create download request: %w", err)
}
req.Header.Set("Authorization", "Bearer "+token)
logger.Infof(ctx, "[Feishu] download GET %s", path)
resp, err := c.httpClient.Do(req)
if err != nil {
return nil, fmt.Errorf("download request: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body)
logger.Errorf(ctx, "[Feishu] download GET %s → status=%d body=%s", path, resp.StatusCode, truncate(string(body), 500))
return nil, fmt.Errorf("download failed: status=%d body=%s", resp.StatusCode, string(body))
}
data, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("read download body: %w", err)
}
logger.Infof(ctx, "[Feishu] download GET %s → OK, %d bytes", path, len(data))
return data, nil
}
@@ -0,0 +1,376 @@
package feishu
import (
"context"
"encoding/json"
"fmt"
"strconv"
"strings"
"time"
"github.com/Tencent/WeKnora/internal/types"
)
// Connector implements the datasource.Connector interface for Feishu.
type Connector struct{}
// NewConnector creates a new Feishu connector.
func NewConnector() *Connector {
return &Connector{}
}
// Type returns the connector type identifier.
func (c *Connector) Type() string {
return types.ConnectorTypeFeishu
}
// Validate verifies that the Feishu configuration is valid by testing connectivity.
func (c *Connector) Validate(ctx context.Context, config *types.DataSourceConfig) error {
feishuConfig, err := parseFeishuConfig(config)
if err != nil {
return err
}
client := NewClient(feishuConfig)
if err := client.Ping(ctx); err != nil {
return fmt.Errorf("feishu connection failed: %w", err)
}
return nil
}
// ListResources lists all accessible Feishu Wiki spaces as selectable resources.
func (c *Connector) ListResources(ctx context.Context, config *types.DataSourceConfig) ([]types.Resource, error) {
feishuConfig, err := parseFeishuConfig(config)
if err != nil {
return nil, err
}
client := NewClient(feishuConfig)
spaces, err := client.ListWikiSpaces(ctx)
if err != nil {
return nil, fmt.Errorf("list feishu wiki spaces: %w", err)
}
resources := make([]types.Resource, 0, len(spaces))
for _, space := range spaces {
resources = append(resources, types.Resource{
ExternalID: space.SpaceID,
Name: space.Name,
Type: "wiki_space",
Description: space.Description,
URL: fmt.Sprintf("https://feishu.cn/wiki/%s", space.SpaceID),
Metadata: map[string]interface{}{
"visibility": space.Visibility,
},
})
}
return resources, nil
}
// FetchAll performs a full sync of all documents from the specified wiki spaces.
func (c *Connector) FetchAll(ctx context.Context, config *types.DataSourceConfig, resourceIDs []string) ([]types.FetchedItem, error) {
feishuConfig, err := parseFeishuConfig(config)
if err != nil {
return nil, err
}
client := NewClient(feishuConfig)
var allItems []types.FetchedItem
for _, spaceID := range resourceIDs {
// List all nodes in this wiki space recursively
nodes, err := client.ListAllWikiNodesRecursive(ctx, spaceID)
if err != nil {
return nil, fmt.Errorf("list nodes in space %s: %w", spaceID, err)
}
// Fetch content for each document node
for _, node := range nodes {
item, err := c.fetchNodeContent(ctx, client, node, spaceID)
if err != nil {
// Log error but continue with other nodes
allItems = append(allItems, types.FetchedItem{
ExternalID: node.NodeToken,
Title: node.Title,
SourceResourceID: spaceID,
Metadata: map[string]string{
"error": err.Error(),
},
})
continue
}
if item != nil {
allItems = append(allItems, *item)
}
}
}
return allItems, nil
}
// FetchIncremental performs an incremental sync by comparing node edit times
// against the previously recorded state.
func (c *Connector) FetchIncremental(ctx context.Context, config *types.DataSourceConfig, cursor *types.SyncCursor) ([]types.FetchedItem, *types.SyncCursor, error) {
feishuConfig, err := parseFeishuConfig(config)
if err != nil {
return nil, nil, err
}
client := NewClient(feishuConfig)
// Parse the previous cursor state
var prevCursor feishuCursor
if cursor != nil && cursor.ConnectorCursor != nil {
cursorBytes, _ := json.Marshal(cursor.ConnectorCursor)
_ = json.Unmarshal(cursorBytes, &prevCursor)
}
// Build new cursor to track current state
newCursor := feishuCursor{
LastSyncTime: time.Now(),
SpaceNodeTimes: make(map[string]map[string]string),
}
var changedItems []types.FetchedItem
// Get resource IDs from config
resourceIDs := config.ResourceIDs
if len(resourceIDs) == 0 {
return nil, nil, fmt.Errorf("no resource IDs (wiki space IDs) configured")
}
for _, spaceID := range resourceIDs {
// List all nodes in this space
nodes, err := client.ListAllWikiNodesRecursive(ctx, spaceID)
if err != nil {
return nil, nil, fmt.Errorf("list nodes in space %s: %w", spaceID, err)
}
newCursor.SpaceNodeTimes[spaceID] = make(map[string]string)
// Build a set of current node tokens for deletion detection
currentNodes := make(map[string]bool)
for _, node := range nodes {
currentNodes[node.NodeToken] = true
// Use ObjEditTime (document content edit time) for change detection,
// NOT NodeEditTime which only tracks node attribute changes (title, position).
editTimeStr := node.ObjEditTime
if editTimeStr == "" {
editTimeStr = node.NodeEditTime // fallback for nodes that don't have obj_edit_time
}
newCursor.SpaceNodeTimes[spaceID][node.NodeToken] = editTimeStr
// Check if node has changed since last sync
if prevCursor.SpaceNodeTimes != nil {
if prevTimes, ok := prevCursor.SpaceNodeTimes[spaceID]; ok {
if prevEditTime, exists := prevTimes[node.NodeToken]; exists {
if prevEditTime == editTimeStr {
// Node unchanged, skip
continue
}
}
}
}
// Node is new or changed — fetch its content
item, err := c.fetchNodeContent(ctx, client, node, spaceID)
if err != nil {
// Record failed items
changedItems = append(changedItems, types.FetchedItem{
ExternalID: node.NodeToken,
Title: node.Title,
SourceResourceID: spaceID,
Metadata: map[string]string{
"error": err.Error(),
},
})
continue
}
if item != nil {
changedItems = append(changedItems, *item)
}
}
// Detect deleted nodes
if prevCursor.SpaceNodeTimes != nil {
if prevTimes, ok := prevCursor.SpaceNodeTimes[spaceID]; ok {
for nodeToken := range prevTimes {
if !currentNodes[nodeToken] {
// Node was deleted
changedItems = append(changedItems, types.FetchedItem{
ExternalID: nodeToken,
IsDeleted: true,
SourceResourceID: spaceID,
})
}
}
}
}
}
// Build next sync cursor
nextCursorMap := make(map[string]interface{})
cursorBytes, _ := json.Marshal(newCursor)
_ = json.Unmarshal(cursorBytes, &nextCursorMap)
nextSyncCursor := &types.SyncCursor{
LastSyncTime: time.Now(),
ConnectorCursor: nextCursorMap,
}
return changedItems, nextSyncCursor, nil
}
// fetchNodeContent fetches the content of a single wiki node and converts it to FetchedItem.
// Dispatches to different retrieval strategies based on obj_type:
// - docx/doc → export API → .docx file
// - sheet → export API → .xlsx file
// - bitable → export API → .xlsx file
// - file → drive download → original file (PDF/Word/image/etc.)
// - mindnote → skip (no API)
// - slides → skip (no API)
func (c *Connector) fetchNodeContent(ctx context.Context, client *Client, node wikiNode, spaceID string) (*types.FetchedItem, error) {
if !isSupportedDocType(node.ObjType) {
return nil, nil
}
editTime := parseFeishuTimestamp(node.NodeEditTime)
baseMeta := map[string]string{
"obj_token": node.ObjToken,
"obj_type": node.ObjType,
"node_token": node.NodeToken,
"space_id": spaceID,
"creator": node.Creator,
"owner": node.Owner,
"channel": types.ChannelFeishu,
}
switch node.ObjType {
case "docx", "doc", "sheet", "bitable":
// Export as a file via the async export API
data, fileName, err := client.ExportAndDownload(ctx, node.ObjToken, node.ObjType)
if err != nil {
return nil, fmt.Errorf("export %s (%s): %w", node.Title, node.ObjType, err)
}
// Ensure a reasonable file name with correct extension
ext := exportFileExtToSuffix[objTypeToExportFileExtension[node.ObjType]]
if fileName == "" {
fileName = sanitizeFileName(node.Title) + ext
} else if !strings.HasSuffix(strings.ToLower(fileName), ext) {
// Feishu often returns the doc title without extension — append it
fileName = sanitizeFileName(fileName) + ext
}
return &types.FetchedItem{
ExternalID: node.NodeToken,
Title: node.Title,
Content: data,
ContentType: "application/octet-stream",
FileName: fileName,
URL: fmt.Sprintf("https://feishu.cn/wiki/%s", node.NodeToken),
UpdatedAt: editTime,
SourceResourceID: spaceID,
Metadata: baseMeta,
}, nil
case "file":
// Download the original uploaded file from Drive
data, err := client.DownloadDriveFile(ctx, node.ObjToken)
if err != nil {
return nil, fmt.Errorf("download file %s (%s): %w", node.Title, node.ObjToken, err)
}
// Use the node title as file name; it usually preserves the original extension
fileName := node.Title
if fileName == "" {
fileName = node.ObjToken
}
return &types.FetchedItem{
ExternalID: node.NodeToken,
Title: node.Title,
Content: data,
ContentType: "application/octet-stream",
FileName: fileName,
URL: fmt.Sprintf("https://feishu.cn/wiki/%s", node.NodeToken),
UpdatedAt: editTime,
SourceResourceID: spaceID,
Metadata: baseMeta,
}, nil
default:
return nil, nil
}
}
// --- Helper functions ---
// parseFeishuConfig extracts and validates Feishu-specific configuration.
func parseFeishuConfig(config *types.DataSourceConfig) (*Config, error) {
if config == nil {
return nil, fmt.Errorf("config is nil")
}
credBytes, err := json.Marshal(config.Credentials)
if err != nil {
return nil, fmt.Errorf("marshal credentials: %w", err)
}
var feishuConfig Config
if err := json.Unmarshal(credBytes, &feishuConfig); err != nil {
return nil, fmt.Errorf("parse feishu credentials: %w", err)
}
if feishuConfig.AppID == "" || feishuConfig.AppSecret == "" {
return nil, fmt.Errorf("feishu app_id and app_secret are required")
}
return &feishuConfig, nil
}
// isSupportedDocType checks if a Feishu document type can be synced.
// mindnote and slides have no content read API and are skipped.
func isSupportedDocType(objType string) bool {
switch objType {
case "docx", "doc", "sheet", "bitable", "file":
return true
default:
// mindnote, slides — no content retrieval API available
return false
}
}
// parseFeishuTimestamp parses a Feishu unix timestamp string (seconds) into time.Time.
func parseFeishuTimestamp(ts string) time.Time {
if ts == "" {
return time.Time{}
}
sec, err := strconv.ParseInt(ts, 10, 64)
if err != nil {
return time.Time{}
}
return time.Unix(sec, 0)
}
// sanitizeFileName removes characters that are invalid in filenames.
func sanitizeFileName(name string) string {
if name == "" {
return "untitled"
}
// Replace common invalid characters with underscore
replacer := strings.NewReplacer(
"/", "_", "\\", "_", ":", "_", "*", "_",
"?", "_", "\"", "_", "<", "_", ">", "_", "|", "_",
)
result := replacer.Replace(name)
// Limit length
if len(result) > 200 {
result = result[:200]
}
return result
}
@@ -0,0 +1,817 @@
package feishu
import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"github.com/Tencent/WeKnora/internal/types"
)
// ──────────────────────────────────────────────────────────────────────
// Fake Feishu API server
// ──────────────────────────────────────────────────────────────────────
// fakeFeishu builds an httptest.Server that emulates the relevant Feishu APIs.
// It returns the server and a Config pointing at it.
func fakeFeishu(nodes []wikiNode) (*httptest.Server, *Config) {
mux := http.NewServeMux()
// --- auth ---
mux.HandleFunc("/open-apis/auth/v3/tenant_access_token/internal", func(w http.ResponseWriter, r *http.Request) {
writeJSON(w, tokenResponse{
apiResponse: apiResponse{Code: 0},
TenantAccessToken: "fake-token",
Expire: 7200,
})
})
// --- wiki spaces ---
mux.HandleFunc("/open-apis/wiki/v2/spaces", func(w http.ResponseWriter, r *http.Request) {
writeJSON(w, wikiSpaceListResponse{
apiResponse: apiResponse{Code: 0},
Data: struct {
Items []wikiSpace `json:"items"`
HasMore bool `json:"has_more"`
PageToken string `json:"page_token"`
}{
Items: []wikiSpace{
{SpaceID: "space1", Name: "Test Space", Description: "desc", Visibility: "public"},
},
},
})
})
// --- wiki nodes (top-level only for simplicity) ---
mux.HandleFunc("/open-apis/wiki/v2/spaces/space1/nodes", func(w http.ResponseWriter, r *http.Request) {
writeJSON(w, wikiNodeListResponse{
apiResponse: apiResponse{Code: 0},
Data: struct {
Items []wikiNode `json:"items"`
HasMore bool `json:"has_more"`
PageToken string `json:"page_token"`
}{
Items: nodes,
},
})
})
// --- export task: create ---
mux.HandleFunc("/open-apis/drive/v1/export_tasks", func(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodPost {
writeJSON(w, exportTaskCreateResponse{
apiResponse: apiResponse{Code: 0},
Data: struct{ Ticket string `json:"ticket"` }{Ticket: "ticket-123"},
})
return
}
// GET /open-apis/drive/v1/export_tasks/ticket-123
writeJSON(w, exportTaskStatusResponse{
apiResponse: apiResponse{Code: 0},
Data: struct {
Result struct {
FileToken string `json:"file_token"`
FileSize int64 `json:"file_size"`
JobStatus int `json:"job_status"`
JobErrorMsg string `json:"job_error_msg"`
FileName string `json:"file_name"`
} `json:"result"`
}{
Result: struct {
FileToken string `json:"file_token"`
FileSize int64 `json:"file_size"`
JobStatus int `json:"job_status"`
JobErrorMsg string `json:"job_error_msg"`
FileName string `json:"file_name"`
}{
FileToken: "ft-abc",
FileSize: 100,
JobStatus: 0, // success
FileName: "exported.docx",
},
},
})
})
// --- export task: status polling (pattern match with ticket) ---
mux.HandleFunc("/open-apis/drive/v1/export_tasks/ticket-123", func(w http.ResponseWriter, r *http.Request) {
writeJSON(w, exportTaskStatusResponse{
apiResponse: apiResponse{Code: 0},
Data: struct {
Result struct {
FileToken string `json:"file_token"`
FileSize int64 `json:"file_size"`
JobStatus int `json:"job_status"`
JobErrorMsg string `json:"job_error_msg"`
FileName string `json:"file_name"`
} `json:"result"`
}{
Result: struct {
FileToken string `json:"file_token"`
FileSize int64 `json:"file_size"`
JobStatus int `json:"job_status"`
JobErrorMsg string `json:"job_error_msg"`
FileName string `json:"file_name"`
}{
FileToken: "ft-abc",
FileSize: 100,
JobStatus: 0,
FileName: "exported.docx",
},
},
})
})
// --- export file download ---
mux.HandleFunc("/open-apis/drive/v1/export_tasks/file/ticket-123/download", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/octet-stream")
w.Write([]byte("fake-docx-content"))
})
// --- drive file download (for "file" type nodes) ---
mux.HandleFunc("/open-apis/drive/v1/files/", func(w http.ResponseWriter, r *http.Request) {
if strings.HasSuffix(r.URL.Path, "/download") {
w.Header().Set("Content-Type", "application/octet-stream")
w.Write([]byte("fake-pdf-binary"))
return
}
http.NotFound(w, r)
})
ts := httptest.NewServer(mux)
cfg := &Config{
AppID: "test-app-id",
AppSecret: "test-app-secret",
BaseURL: ts.URL,
}
return ts, cfg
}
func writeJSON(w http.ResponseWriter, v interface{}) {
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(v)
}
func makeConfig(cfg *Config, resourceIDs []string) *types.DataSourceConfig {
creds := map[string]interface{}{
"app_id": cfg.AppID,
"app_secret": cfg.AppSecret,
"base_url": cfg.BaseURL,
}
return &types.DataSourceConfig{
Type: types.ConnectorTypeFeishu,
Credentials: creds,
ResourceIDs: resourceIDs,
}
}
// ──────────────────────────────────────────────────────────────────────
// Helper function tests
// ──────────────────────────────────────────────────────────────────────
func TestIsSupportedDocType(t *testing.T) {
tests := []struct {
objType string
expected bool
}{
{"docx", true},
{"doc", true},
{"sheet", true},
{"bitable", true},
{"file", true},
{"mindnote", false},
{"slides", false},
{"unknown", false},
{"", false},
}
for _, tt := range tests {
t.Run(tt.objType, func(t *testing.T) {
got := isSupportedDocType(tt.objType)
if got != tt.expected {
t.Errorf("isSupportedDocType(%q) = %v, want %v", tt.objType, got, tt.expected)
}
})
}
}
func TestSanitizeFileName(t *testing.T) {
tests := []struct {
input string
expected string
}{
{"hello", "hello"},
{"", "untitled"},
{"a/b\\c:d*e", "a_b_c_d_e"},
{"normal file.docx", "normal file.docx"},
{strings.Repeat("a", 300), strings.Repeat("a", 200)},
}
for _, tt := range tests {
t.Run(tt.input, func(t *testing.T) {
got := sanitizeFileName(tt.input)
if got != tt.expected {
t.Errorf("sanitizeFileName(%q) = %q, want %q", tt.input, got, tt.expected)
}
})
}
}
func TestParseFeishuTimestamp(t *testing.T) {
ts := parseFeishuTimestamp("1711468800") // 2024-03-27 00:00:00 UTC
if ts.IsZero() {
t.Fatal("expected non-zero time")
}
if ts.Unix() != 1711468800 {
t.Errorf("unexpected unix = %d", ts.Unix())
}
if !parseFeishuTimestamp("").IsZero() {
t.Error("expected zero time for empty string")
}
if !parseFeishuTimestamp("invalid").IsZero() {
t.Error("expected zero time for invalid string")
}
}
func TestParseFeishuConfig(t *testing.T) {
t.Run("valid", func(t *testing.T) {
cfg, err := parseFeishuConfig(&types.DataSourceConfig{
Credentials: map[string]interface{}{
"app_id": "id1",
"app_secret": "sec1",
"base_url": "https://custom.example.com",
},
})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if cfg.AppID != "id1" || cfg.AppSecret != "sec1" || cfg.BaseURL != "https://custom.example.com" {
t.Errorf("unexpected config: %+v", cfg)
}
})
t.Run("nil config", func(t *testing.T) {
_, err := parseFeishuConfig(nil)
if err == nil {
t.Fatal("expected error for nil config")
}
})
t.Run("missing credentials", func(t *testing.T) {
_, err := parseFeishuConfig(&types.DataSourceConfig{
Credentials: map[string]interface{}{
"app_id": "id1",
// missing app_secret
},
})
if err == nil {
t.Fatal("expected error for missing app_secret")
}
})
}
// ──────────────────────────────────────────────────────────────────────
// Connector interface tests
// ──────────────────────────────────────────────────────────────────────
func TestConnectorType(t *testing.T) {
c := NewConnector()
if c.Type() != types.ConnectorTypeFeishu {
t.Errorf("Type() = %q, want %q", c.Type(), types.ConnectorTypeFeishu)
}
}
func TestConnectorValidate(t *testing.T) {
ts, cfg := fakeFeishu(nil)
defer ts.Close()
c := NewConnector()
err := c.Validate(context.Background(), makeConfig(cfg, nil))
if err != nil {
t.Fatalf("Validate() error: %v", err)
}
}
func TestConnectorValidate_BadCredentials(t *testing.T) {
c := NewConnector()
err := c.Validate(context.Background(), &types.DataSourceConfig{
Credentials: map[string]interface{}{
"app_id": "bad",
"app_secret": "bad",
"base_url": "http://127.0.0.1:1", // will fail to connect
},
})
if err == nil {
t.Fatal("expected error for bad credentials")
}
}
func TestConnectorListResources(t *testing.T) {
ts, cfg := fakeFeishu(nil)
defer ts.Close()
c := NewConnector()
resources, err := c.ListResources(context.Background(), makeConfig(cfg, nil))
if err != nil {
t.Fatalf("ListResources() error: %v", err)
}
if len(resources) != 1 {
t.Fatalf("expected 1 resource, got %d", len(resources))
}
if resources[0].ExternalID != "space1" {
t.Errorf("ExternalID = %q, want %q", resources[0].ExternalID, "space1")
}
if resources[0].Name != "Test Space" {
t.Errorf("Name = %q, want %q", resources[0].Name, "Test Space")
}
if resources[0].Type != "wiki_space" {
t.Errorf("Type = %q, want %q", resources[0].Type, "wiki_space")
}
}
// ──────────────────────────────────────────────────────────────────────
// FetchAll: test all supported doc types
// ──────────────────────────────────────────────────────────────────────
func TestFetchAll_DocxNode(t *testing.T) {
nodes := []wikiNode{{
NodeToken: "nt1",
ObjToken: "obj-docx-1",
ObjType: "docx",
Title: "My Document",
NodeEditTime: "1711468800",
}}
ts, cfg := fakeFeishu(nodes)
defer ts.Close()
c := NewConnector()
items, err := c.FetchAll(context.Background(), makeConfig(cfg, []string{"space1"}), []string{"space1"})
if err != nil {
t.Fatalf("FetchAll() error: %v", err)
}
if len(items) != 1 {
t.Fatalf("expected 1 item, got %d", len(items))
}
item := items[0]
if item.ExternalID != "nt1" {
t.Errorf("ExternalID = %q, want %q", item.ExternalID, "nt1")
}
if item.Title != "My Document" {
t.Errorf("Title = %q", item.Title)
}
if string(item.Content) != "fake-docx-content" {
t.Errorf("Content = %q, want %q", string(item.Content), "fake-docx-content")
}
if item.FileName != "exported.docx" {
t.Errorf("FileName = %q, want %q", item.FileName, "exported.docx")
}
if item.Metadata["obj_type"] != "docx" {
t.Errorf("Metadata[obj_type] = %q", item.Metadata["obj_type"])
}
if item.Metadata["channel"] != types.ChannelFeishu {
t.Errorf("Metadata[channel] = %q", item.Metadata["channel"])
}
}
func TestFetchAll_SheetNode(t *testing.T) {
nodes := []wikiNode{{
NodeToken: "nt-sheet",
ObjToken: "obj-sheet-1",
ObjType: "sheet",
Title: "Sales Report",
NodeEditTime: "1711468800",
}}
ts, cfg := fakeFeishu(nodes)
defer ts.Close()
c := NewConnector()
items, err := c.FetchAll(context.Background(), makeConfig(cfg, []string{"space1"}), []string{"space1"})
if err != nil {
t.Fatalf("FetchAll() error: %v", err)
}
if len(items) != 1 {
t.Fatalf("expected 1 item, got %d", len(items))
}
if items[0].Metadata["obj_type"] != "sheet" {
t.Errorf("obj_type = %q, want sheet", items[0].Metadata["obj_type"])
}
}
func TestFetchAll_BitableNode(t *testing.T) {
nodes := []wikiNode{{
NodeToken: "nt-bitable",
ObjToken: "obj-bitable-1",
ObjType: "bitable",
Title: "Project Tracker",
NodeEditTime: "1711468800",
}}
ts, cfg := fakeFeishu(nodes)
defer ts.Close()
c := NewConnector()
items, err := c.FetchAll(context.Background(), makeConfig(cfg, []string{"space1"}), []string{"space1"})
if err != nil {
t.Fatalf("FetchAll() error: %v", err)
}
if len(items) != 1 {
t.Fatalf("expected 1 item, got %d", len(items))
}
if items[0].Metadata["obj_type"] != "bitable" {
t.Errorf("obj_type = %q, want bitable", items[0].Metadata["obj_type"])
}
}
func TestFetchAll_FileNode(t *testing.T) {
nodes := []wikiNode{{
NodeToken: "nt-file",
ObjToken: "obj-file-1",
ObjType: "file",
Title: "manual.pdf",
NodeEditTime: "1711468800",
}}
ts, cfg := fakeFeishu(nodes)
defer ts.Close()
c := NewConnector()
items, err := c.FetchAll(context.Background(), makeConfig(cfg, []string{"space1"}), []string{"space1"})
if err != nil {
t.Fatalf("FetchAll() error: %v", err)
}
if len(items) != 1 {
t.Fatalf("expected 1 item, got %d", len(items))
}
item := items[0]
if string(item.Content) != "fake-pdf-binary" {
t.Errorf("Content = %q, want %q", string(item.Content), "fake-pdf-binary")
}
if item.FileName != "manual.pdf" {
t.Errorf("FileName = %q, want %q", item.FileName, "manual.pdf")
}
if item.Metadata["obj_type"] != "file" {
t.Errorf("obj_type = %q, want file", item.Metadata["obj_type"])
}
}
func TestFetchAll_SkipsMindnoteAndSlides(t *testing.T) {
nodes := []wikiNode{
{NodeToken: "nt-mn", ObjToken: "obj-mn", ObjType: "mindnote", Title: "Brain Map"},
{NodeToken: "nt-sl", ObjToken: "obj-sl", ObjType: "slides", Title: "Presentation"},
}
ts, cfg := fakeFeishu(nodes)
defer ts.Close()
c := NewConnector()
items, err := c.FetchAll(context.Background(), makeConfig(cfg, []string{"space1"}), []string{"space1"})
if err != nil {
t.Fatalf("FetchAll() error: %v", err)
}
// Both should be skipped (nil returned by fetchNodeContent)
if len(items) != 0 {
t.Errorf("expected 0 items (mindnote+slides skipped), got %d", len(items))
}
}
func TestFetchAll_MixedTypes(t *testing.T) {
nodes := []wikiNode{
{NodeToken: "nt1", ObjToken: "obj1", ObjType: "docx", Title: "Doc", NodeEditTime: "1711468800"},
{NodeToken: "nt2", ObjToken: "obj2", ObjType: "sheet", Title: "Sheet", NodeEditTime: "1711468800"},
{NodeToken: "nt3", ObjToken: "obj3", ObjType: "file", Title: "report.pdf", NodeEditTime: "1711468800"},
{NodeToken: "nt4", ObjToken: "obj4", ObjType: "mindnote", Title: "Mind", NodeEditTime: "1711468800"},
{NodeToken: "nt5", ObjToken: "obj5", ObjType: "slides", Title: "Slides", NodeEditTime: "1711468800"},
{NodeToken: "nt6", ObjToken: "obj6", ObjType: "bitable", Title: "Table", NodeEditTime: "1711468800"},
}
ts, cfg := fakeFeishu(nodes)
defer ts.Close()
c := NewConnector()
items, err := c.FetchAll(context.Background(), makeConfig(cfg, []string{"space1"}), []string{"space1"})
if err != nil {
t.Fatalf("FetchAll() error: %v", err)
}
// docx + sheet + file + bitable = 4 items; mindnote + slides = skipped
if len(items) != 4 {
t.Errorf("expected 4 items, got %d", len(items))
for i, it := range items {
t.Logf(" item[%d]: %s (obj_type=%s)", i, it.Title, it.Metadata["obj_type"])
}
}
}
// ──────────────────────────────────────────────────────────────────────
// FetchIncremental tests
// ──────────────────────────────────────────────────────────────────────
func TestFetchIncremental_FirstSync(t *testing.T) {
nodes := []wikiNode{
{NodeToken: "nt1", ObjToken: "obj1", ObjType: "docx", Title: "Doc1", NodeEditTime: "100"},
{NodeToken: "nt2", ObjToken: "obj2", ObjType: "file", Title: "file.pdf", NodeEditTime: "200"},
}
ts, cfg := fakeFeishu(nodes)
defer ts.Close()
c := NewConnector()
dsConfig := makeConfig(cfg, []string{"space1"})
// First sync with no cursor → all items should be fetched
items, cursor, err := c.FetchIncremental(context.Background(), dsConfig, nil)
if err != nil {
t.Fatalf("FetchIncremental() error: %v", err)
}
if len(items) != 2 {
t.Fatalf("expected 2 items on first sync, got %d", len(items))
}
if cursor == nil {
t.Fatal("expected non-nil cursor")
}
if cursor.LastSyncTime.IsZero() {
t.Error("cursor.LastSyncTime should not be zero")
}
}
func TestFetchIncremental_NoChanges(t *testing.T) {
nodes := []wikiNode{
{NodeToken: "nt1", ObjToken: "obj1", ObjType: "docx", Title: "Doc1", NodeEditTime: "100"},
}
ts, cfg := fakeFeishu(nodes)
defer ts.Close()
c := NewConnector()
dsConfig := makeConfig(cfg, []string{"space1"})
// First sync
_, cursor1, err := c.FetchIncremental(context.Background(), dsConfig, nil)
if err != nil {
t.Fatalf("first sync error: %v", err)
}
// Second sync with same edit times → should return 0 changed items
items, _, err := c.FetchIncremental(context.Background(), dsConfig, cursor1)
if err != nil {
t.Fatalf("second sync error: %v", err)
}
if len(items) != 0 {
t.Errorf("expected 0 items (no changes), got %d", len(items))
}
}
func TestFetchIncremental_DetectsDeleted(t *testing.T) {
// First sync: 2 nodes
allNodes := []wikiNode{
{NodeToken: "nt1", ObjToken: "obj1", ObjType: "docx", Title: "Doc1", NodeEditTime: "100"},
{NodeToken: "nt2", ObjToken: "obj2", ObjType: "docx", Title: "Doc2", NodeEditTime: "200"},
}
ts, cfg := fakeFeishu(allNodes)
c := NewConnector()
dsConfig := makeConfig(cfg, []string{"space1"})
_, cursor1, err := c.FetchIncremental(context.Background(), dsConfig, nil)
if err != nil {
t.Fatalf("first sync error: %v", err)
}
ts.Close()
// Second sync: only 1 node remains (nt2 was deleted)
ts2, cfg2 := fakeFeishu([]wikiNode{
{NodeToken: "nt1", ObjToken: "obj1", ObjType: "docx", Title: "Doc1", NodeEditTime: "100"},
})
defer ts2.Close()
dsConfig2 := makeConfig(cfg2, []string{"space1"})
items, _, err := c.FetchIncremental(context.Background(), dsConfig2, cursor1)
if err != nil {
t.Fatalf("second sync error: %v", err)
}
// Should have 1 deleted item for nt2
var deletedCount int
for _, item := range items {
if item.IsDeleted {
deletedCount++
if item.ExternalID != "nt2" {
t.Errorf("expected deleted ExternalID=nt2, got %q", item.ExternalID)
}
}
}
if deletedCount != 1 {
t.Errorf("expected 1 deleted item, got %d", deletedCount)
}
}
func TestFetchIncremental_NoResourceIDs(t *testing.T) {
ts, cfg := fakeFeishu(nil)
defer ts.Close()
c := NewConnector()
dsConfig := makeConfig(cfg, nil) // no resource IDs
dsConfig.ResourceIDs = nil
_, _, err := c.FetchIncremental(context.Background(), dsConfig, nil)
if err == nil {
t.Fatal("expected error for empty resource IDs")
}
if !strings.Contains(err.Error(), "no resource IDs") {
t.Errorf("unexpected error: %v", err)
}
}
// ──────────────────────────────────────────────────────────────────────
// Client tests
// ──────────────────────────────────────────────────────────────────────
func TestClientPing(t *testing.T) {
ts, cfg := fakeFeishu(nil)
defer ts.Close()
client := NewClient(cfg)
if err := client.Ping(context.Background()); err != nil {
t.Fatalf("Ping() error: %v", err)
}
}
func TestClientTokenCaching(t *testing.T) {
callCount := 0
mux := http.NewServeMux()
mux.HandleFunc("/open-apis/auth/v3/tenant_access_token/internal", func(w http.ResponseWriter, r *http.Request) {
callCount++
writeJSON(w, tokenResponse{
apiResponse: apiResponse{Code: 0},
TenantAccessToken: fmt.Sprintf("token-%d", callCount),
Expire: 7200,
})
})
ts := httptest.NewServer(mux)
defer ts.Close()
client := NewClient(&Config{AppID: "a", AppSecret: "b", BaseURL: ts.URL})
// First call: fetches token
t1, _ := client.getTenantAccessToken(context.Background())
// Second call: should use cache
t2, _ := client.getTenantAccessToken(context.Background())
if t1 != t2 {
t.Errorf("expected cached token, got different tokens: %q vs %q", t1, t2)
}
if callCount != 1 {
t.Errorf("expected 1 auth call, got %d", callCount)
}
}
func TestClientExportAndDownload(t *testing.T) {
ts, cfg := fakeFeishu(nil)
defer ts.Close()
client := NewClient(cfg)
data, fileName, err := client.ExportAndDownload(context.Background(), "obj-token-1", "docx")
if err != nil {
t.Fatalf("ExportAndDownload() error: %v", err)
}
if string(data) != "fake-docx-content" {
t.Errorf("data = %q, want %q", string(data), "fake-docx-content")
}
if fileName != "exported.docx" {
t.Errorf("fileName = %q, want %q", fileName, "exported.docx")
}
}
func TestClientExportAndDownload_UnsupportedType(t *testing.T) {
ts, cfg := fakeFeishu(nil)
defer ts.Close()
client := NewClient(cfg)
_, _, err := client.ExportAndDownload(context.Background(), "obj-token-1", "mindnote")
if err == nil {
t.Fatal("expected error for unsupported type")
}
if !strings.Contains(err.Error(), "unsupported") {
t.Errorf("unexpected error: %v", err)
}
}
func TestClientDownloadDriveFile(t *testing.T) {
ts, cfg := fakeFeishu(nil)
defer ts.Close()
client := NewClient(cfg)
data, err := client.DownloadDriveFile(context.Background(), "file-token-abc")
if err != nil {
t.Fatalf("DownloadDriveFile() error: %v", err)
}
if string(data) != "fake-pdf-binary" {
t.Errorf("data = %q, want %q", string(data), "fake-pdf-binary")
}
}
func TestClientListWikiSpaces(t *testing.T) {
ts, cfg := fakeFeishu(nil)
defer ts.Close()
client := NewClient(cfg)
spaces, err := client.ListWikiSpaces(context.Background())
if err != nil {
t.Fatalf("ListWikiSpaces() error: %v", err)
}
if len(spaces) != 1 {
t.Fatalf("expected 1 space, got %d", len(spaces))
}
if spaces[0].SpaceID != "space1" {
t.Errorf("SpaceID = %q", spaces[0].SpaceID)
}
}
// ──────────────────────────────────────────────────────────────────────
// Type mapping tests
// ──────────────────────────────────────────────────────────────────────
func TestObjTypeToExportMappings(t *testing.T) {
// Verify all exportable types have valid mappings
exportable := []string{"docx", "doc", "sheet", "bitable"}
for _, ot := range exportable {
if _, ok := objTypeToExportFileExtension[ot]; !ok {
t.Errorf("objTypeToExportFileExtension missing %q", ot)
}
if _, ok := objTypeToExportType[ot]; !ok {
t.Errorf("objTypeToExportType missing %q", ot)
}
}
// Verify non-exportable types do NOT have mappings
nonExportable := []string{"file", "mindnote", "slides"}
for _, ot := range nonExportable {
if _, ok := objTypeToExportFileExtension[ot]; ok {
t.Errorf("objTypeToExportFileExtension should NOT contain %q", ot)
}
}
}
func TestExportFileExtToSuffix(t *testing.T) {
if exportFileExtToSuffix[ExportTypeDocx] != ".docx" {
t.Errorf("docx suffix = %q", exportFileExtToSuffix[ExportTypeDocx])
}
if exportFileExtToSuffix[ExportTypeXlsx] != ".xlsx" {
t.Errorf("xlsx suffix = %q", exportFileExtToSuffix[ExportTypeXlsx])
}
if exportFileExtToSuffix[ExportTypePDF] != ".pdf" {
t.Errorf("pdf suffix = %q", exportFileExtToSuffix[ExportTypePDF])
}
}
// ──────────────────────────────────────────────────────────────────────
// Feishu cursor serialization
// ──────────────────────────────────────────────────────────────────────
func TestFeishuCursorRoundTrip(t *testing.T) {
original := feishuCursor{
LastSyncTime: time.Date(2026, 3, 26, 12, 0, 0, 0, time.UTC),
SpaceNodeTimes: map[string]map[string]string{
"space1": {
"nt1": "100",
"nt2": "200",
},
},
}
// Serialize to map[string]interface{} (as stored in SyncCursor.ConnectorCursor)
data, err := json.Marshal(original)
if err != nil {
t.Fatalf("marshal error: %v", err)
}
var cursorMap map[string]interface{}
if err := json.Unmarshal(data, &cursorMap); err != nil {
t.Fatalf("unmarshal to map error: %v", err)
}
// Deserialize back
data2, _ := json.Marshal(cursorMap)
var restored feishuCursor
if err := json.Unmarshal(data2, &restored); err != nil {
t.Fatalf("restore error: %v", err)
}
if restored.SpaceNodeTimes["space1"]["nt1"] != "100" {
t.Errorf("restored nt1 = %q, want 100", restored.SpaceNodeTimes["space1"]["nt1"])
}
if restored.SpaceNodeTimes["space1"]["nt2"] != "200" {
t.Errorf("restored nt2 = %q, want 200", restored.SpaceNodeTimes["space1"]["nt2"])
}
}
@@ -0,0 +1,206 @@
// Package feishu implements the Feishu (飞书/Lark) data source connector for WeKnora.
//
// It syncs documents from Feishu Wiki spaces and cloud documents into WeKnora knowledge bases.
//
// Feishu API docs:
// - Wiki spaces: https://open.feishu.cn/document/server-docs/docs/wiki-v2/space/list
// - Wiki nodes: https://open.feishu.cn/document/server-docs/docs/wiki-v2/space-node/list
// - Export tasks: https://open.feishu.cn/document/server-docs/docs/drive-v1/export_task/export-user-guide
// - File download: https://open.feishu.cn/document/server-docs/docs/drive-v1/file/download
// - Auth: https://open.feishu.cn/document/server-docs/authentication-management/access-token/tenant_access_token_internal
package feishu
import "time"
// Config holds Feishu-specific configuration for the data source connector.
// Uses the self-built app (企业自建应用) authentication model.
type Config struct {
// App ID from Feishu developer console
AppID string `json:"app_id"`
// App Secret from Feishu developer console
AppSecret string `json:"app_secret"`
// Base URL for Feishu API (default: https://open.feishu.cn)
// Use https://open.larksuite.com for Lark (international) deployments
BaseURL string `json:"base_url,omitempty"`
}
// DefaultBaseURL is the default Feishu Open Platform API base URL.
const DefaultBaseURL = "https://open.feishu.cn"
// LarkBaseURL is the Lark (international) API base URL.
const LarkBaseURL = "https://open.larksuite.com"
// GetBaseURL returns the effective base URL, defaulting to Feishu if not set.
func (c *Config) GetBaseURL() string {
if c.BaseURL != "" {
return c.BaseURL
}
return DefaultBaseURL
}
// --- Export format constants ---
// Used by the export task API: POST /drive/v1/export_tasks
const (
// ExportTypeDocx exports Feishu documents to .docx format.
ExportTypeDocx = "docx"
// ExportTypeXlsx exports spreadsheets / bitable to .xlsx format.
ExportTypeXlsx = "xlsx"
// ExportTypePDF exports documents to .pdf format (fallback).
ExportTypePDF = "pdf"
)
// objTypeToExportFileExtension maps Feishu obj_type to the best export file_extension.
var objTypeToExportFileExtension = map[string]string{
"docx": ExportTypeDocx,
"doc": ExportTypeDocx,
"sheet": ExportTypeXlsx,
"bitable": ExportTypeXlsx,
}
// objTypeToExportType maps Feishu obj_type to the export API "type" parameter.
// See: https://open.feishu.cn/document/server-docs/docs/drive-v1/export_task/create
var objTypeToExportType = map[string]string{
"docx": "docx",
"doc": "doc",
"sheet": "sheet",
"bitable": "bitable",
}
// exportFileExtToSuffix maps export file_extension to the file suffix for FileName.
var exportFileExtToSuffix = map[string]string{
ExportTypeDocx: ".docx",
ExportTypeXlsx: ".xlsx",
ExportTypePDF: ".pdf",
}
// --- Feishu API response structures ---
// apiResponse is the common Feishu API response wrapper.
type apiResponse struct {
Code int `json:"code"`
Msg string `json:"msg"`
}
// tokenResponse is the response for tenant_access_token API.
type tokenResponse struct {
apiResponse
TenantAccessToken string `json:"tenant_access_token"`
Expire int `json:"expire"` // seconds
}
// wikiSpaceListResponse is the response for GET /open-apis/wiki/v2/spaces.
type wikiSpaceListResponse struct {
apiResponse
Data struct {
Items []wikiSpace `json:"items"`
HasMore bool `json:"has_more"`
PageToken string `json:"page_token"`
} `json:"data"`
}
// wikiSpace represents a Feishu Wiki space.
type wikiSpace struct {
SpaceID string `json:"space_id"`
Name string `json:"name"`
Description string `json:"description"`
Visibility string `json:"visibility"` // "public" or "private"
}
// wikiNodeListResponse is the response for GET /open-apis/wiki/v2/spaces/:space_id/nodes.
type wikiNodeListResponse struct {
apiResponse
Data struct {
Items []wikiNode `json:"items"`
HasMore bool `json:"has_more"`
PageToken string `json:"page_token"`
} `json:"data"`
}
// wikiNode represents a node (document or folder) in a Feishu Wiki space.
type wikiNode struct {
SpaceID string `json:"space_id"`
NodeToken string `json:"node_token"`
ObjToken string `json:"obj_token"` // document token
ObjType string `json:"obj_type"` // "doc", "sheet", "mindnote", "bitable", "file", "docx", "slides"
ParentNodeID string `json:"parent_node_id"`
NodeType string `json:"node_type"` // "origin" or "shortcut"
OriginNodeID string `json:"origin_node_id"`
OriginSpaceID string `json:"origin_space_id"`
HasChild bool `json:"has_child"`
Title string `json:"title"`
Creator string `json:"creator"`
Owner string `json:"owner"`
ObjCreateTime string `json:"obj_create_time"` // document creation time (unix timestamp string)
ObjEditTime string `json:"obj_edit_time"` // document last edit time (unix timestamp string) — tracks content changes
NodeCreateTime string `json:"node_create_time"` // node creation time (unix timestamp string)
NodeEditTime string `json:"node_edit_time"` // node edit time (unix timestamp string) — only tracks node attribute changes
}
// wikiNodeInfoResponse is the response for GET /open-apis/wiki/v2/spaces/get_node.
type wikiNodeInfoResponse struct {
apiResponse
Data struct {
Node wikiNode `json:"node"`
} `json:"data"`
}
// --- Export task API responses ---
// docRawContentResponse is the response for GET /open-apis/docx/v1/documents/:document_id/raw_content.
// Deprecated: prefer export API for full-fidelity document export.
type docRawContentResponse struct {
apiResponse
Data struct {
Content string `json:"content"`
} `json:"data"`
}
// exportTaskCreateResponse is the response for POST /drive/v1/export_tasks.
type exportTaskCreateResponse struct {
apiResponse
Data struct {
Ticket string `json:"ticket"`
} `json:"data"`
}
// exportTaskStatusResponse is the response for GET /drive/v1/export_tasks/{ticket}.
type exportTaskStatusResponse struct {
apiResponse
Data struct {
Result struct {
FileToken string `json:"file_token"`
FileSize int64 `json:"file_size"`
// JobStatus: 0=success, 1=initializing, 2=processing
JobStatus int `json:"job_status"`
JobErrorMsg string `json:"job_error_msg"`
FileName string `json:"file_name"`
} `json:"result"`
} `json:"data"`
}
// --- File download response ---
// driveFileMetaResponse is the response for GET /drive/v1/metas for file type nodes.
type driveFileMetaResponse struct {
apiResponse
Data struct {
Metas []struct {
DocToken string `json:"doc_token"`
DocType string `json:"doc_type"`
Title string `json:"title"`
} `json:"metas"`
} `json:"data"`
}
// feishuCursor stores incremental sync state for Feishu.
type feishuCursor struct {
// LastSyncTime is the timestamp of the last successful sync.
LastSyncTime time.Time `json:"last_sync_time"`
// SpaceNodeTimes maps space_id -> node_token -> last known edit time.
// Used to detect which nodes have changed since last sync.
SpaceNodeTimes map[string]map[string]string `json:"space_node_times,omitempty"`
}