feat(datasource): implement data source management functionality

- Added a new API for managing data sources, including CRUD operations and connection validation.
- Introduced UI components for data source settings, sync logs, and editor dialog, enhancing user experience in managing external data sources.
- Updated localization files to support new data source features in multiple languages.
- Implemented cron expression humanization for better scheduling visibility.
- Enhanced knowledge base editor to include data source management options.
This commit is contained in:
wizardchen
2026-03-27 15:02:40 +08:00
committed by lyingbug
parent ed393c4683
commit adabf8e089
44 changed files with 8159 additions and 42 deletions
@@ -0,0 +1,497 @@
# Adding a New Connector - Step-by-Step Guide
This guide walks you through implementing a new connector (e.g., Feishu, Notion, Confluence) for the data source sync framework.
## Overview
A connector is an adapter that translates between WeKnora's data model and an external platform's API. It handles:
- Connection validation (credentials, connectivity)
- Resource listing (documents, spaces, folders the user can choose from)
- Full sync (fetch all items from selected resources)
- Incremental sync (fetch only changed items since last sync)
## Step 1: Create Connector Package Structure
```bash
mkdir -p internal/datasource/connector/yourtype/
```
Create three files:
- `client.go` - API client wrapper
- `connector.go` - Implements Connector interface
- `types.go` - Platform-specific data structures
## Step 2: Define Platform Types (types.go)
```go
package yourtype
import "time"
// Platform-specific configuration
type Config struct {
BaseURL string `json:"base_url"`
APIToken string `json:"api_token"`
// Or OAuth fields:
AccessToken string `json:"access_token"`
RefreshToken string `json:"refresh_token"`
ExpiresAt time.Time `json:"expires_at"`
}
// Platform-specific resource representation
type YourResource struct {
ID string
Name string
Type string // "document", "folder", "space", etc.
ModifiedAt time.Time
URL string
}
// Platform-specific item representation
type YourItem struct {
ID string
Title string
Content string
ContentHTML string
ModifiedAt time.Time
URL string
CreatedBy string
}
// Platform-specific pagination/cursor
type YourCursor struct {
Offset int `json:"offset,omitempty"`
LastModified time.Time `json:"last_modified,omitempty"`
PageToken string `json:"page_token,omitempty"`
}
```
## Step 3: Implement API Client (client.go)
```go
package yourtype
import (
"context"
"fmt"
"net/http"
"encoding/json"
)
type Client struct {
baseURL string
apiToken string
httpClient *http.Client
}
// NewClient creates a new API client
func NewClient(config *Config) *Client {
return &Client{
baseURL: config.BaseURL,
apiToken: config.APIToken,
httpClient: &http.Client{Timeout: 30 * time.Second},
}
}
// Example methods
func (c *Client) GetResources(ctx context.Context) ([]YourResource, error) {
// Call platform API
// Parse response
// Return resources
}
func (c *Client) GetDocument(ctx context.Context, docID string) (*YourItem, error) {
// Fetch single document
}
func (c *Client) GetDocumentsModifiedSince(ctx context.Context, since time.Time) ([]YourItem, error) {
// Fetch documents modified since timestamp
}
```
## Step 4: Implement Connector Interface (connector.go)
```go
package yourtype
import (
"context"
"fmt"
"github.com/Tencent/WeKnora/internal/types"
)
type YourConnector struct {
client *Client
}
// NewConnector creates a new connector
func NewConnector() *YourConnector {
return &YourConnector{}
}
// Type returns the connector type identifier
func (c *YourConnector) Type() string {
return types.ConnectorTypeYourType // Must match constant in types/datasource.go
}
// Validate verifies that the configuration is valid
func (c *YourConnector) Validate(ctx context.Context, config *types.DataSourceConfig) error {
if config == nil {
return fmt.Errorf("config is nil")
}
// Parse your type-specific config
yourConfig := &Config{}
if err := parseConfig(config, yourConfig); err != nil {
return fmt.Errorf("invalid config: %w", err)
}
// Create client
client := NewClient(yourConfig)
// Test connection
_, err := client.GetResources(ctx)
if err != nil {
return fmt.Errorf("connection failed: %w", err)
}
return nil
}
// ListResources lists available resources (documents, spaces, folders)
func (c *YourConnector) ListResources(ctx context.Context, config *types.DataSourceConfig) ([]types.Resource, error) {
yourConfig := &Config{}
if err := parseConfig(config, yourConfig); err != nil {
return nil, err
}
client := NewClient(yourConfig)
yourResources, err := client.GetResources(ctx)
if err != nil {
return nil, err
}
// Convert to WeKnora Resource format
resources := make([]types.Resource, len(yourResources))
for i, yr := range yourResources {
resources[i] = types.Resource{
ExternalID: yr.ID,
Name: yr.Name,
Type: yr.Type,
URL: yr.URL,
ModifiedAt: yr.ModifiedAt,
}
}
return resources, nil
}
// FetchAll performs a full sync
func (c *YourConnector) FetchAll(ctx context.Context, config *types.DataSourceConfig, resourceIDs []string) ([]types.FetchedItem, error) {
yourConfig := &Config{}
if err := parseConfig(config, yourConfig); err != nil {
return nil, err
}
client := NewClient(yourConfig)
var allItems []types.FetchedItem
// Fetch all documents from specified resources
for _, resourceID := range resourceIDs {
// Get documents from this resource (implementation depends on platform)
yourItems, err := client.GetDocumentsFromResource(ctx, resourceID)
if err != nil {
return nil, fmt.Errorf("failed to fetch resource %s: %w", resourceID, err)
}
// Convert to FetchedItem format
for _, yi := range yourItems {
item := types.FetchedItem{
ExternalID: yi.ID,
Title: yi.Title,
Content: []byte(yi.Content),
ContentType: "text/markdown",
FileName: fmt.Sprintf("%s.md", yi.Title),
URL: yi.URL,
UpdatedAt: yi.ModifiedAt,
SourceResourceID: resourceID,
Metadata: map[string]string{
"created_by": yi.CreatedBy,
"platform": "yourtype",
},
}
allItems = append(allItems, item)
}
}
return allItems, nil
}
// FetchIncremental performs an incremental sync
func (c *YourConnector) FetchIncremental(ctx context.Context, config *types.DataSourceConfig, cursor *types.SyncCursor) ([]types.FetchedItem, *types.SyncCursor, error) {
yourConfig := &Config{}
if err := parseConfig(config, yourConfig); err != nil {
return nil, nil, err
}
client := NewClient(yourConfig)
// Determine start time for incremental fetch
var sinceTime time.Time
if cursor != nil && !cursor.LastSyncTime.IsZero() {
sinceTime = cursor.LastSyncTime
} else {
sinceTime = time.Now().AddDate(0, 0, -7) // Default: last 7 days
}
// Fetch changed items
yourItems, err := client.GetDocumentsModifiedSince(ctx, sinceTime)
if err != nil {
return nil, nil, fmt.Errorf("incremental fetch failed: %w", err)
}
// Convert to FetchedItem format
items := make([]types.FetchedItem, len(yourItems))
for i, yi := range yourItems {
items[i] = types.FetchedItem{
ExternalID: yi.ID,
Title: yi.Title,
Content: []byte(yi.Content),
ContentType: "text/markdown",
FileName: fmt.Sprintf("%s.md", yi.Title),
URL: yi.URL,
UpdatedAt: yi.ModifiedAt,
Metadata: map[string]string{
"created_by": yi.CreatedBy,
"platform": "yourtype",
},
}
}
// Create new cursor for next sync
nextCursor := &types.SyncCursor{
LastSyncTime: time.Now(),
ConnectorCursor: map[string]interface{}{
"last_modified": time.Now(),
},
}
return items, nextCursor, nil
}
// Helper function to parse config
func parseConfig(config *types.DataSourceConfig, target interface{}) error {
data, err := json.Marshal(config.Credentials)
if err != nil {
return err
}
return json.Unmarshal(data, target)
}
```
## Step 5: Register in Container
Edit `internal/container/container.go`:
```go
import (
// ... existing imports
yourconnector "github.com/Tencent/WeKnora/internal/datasource/connector/yourtype"
)
func setupContainer() (*dig.Container, error) {
container := dig.New()
// ... existing registrations ...
// Register your connector
must(container.Provide(func() *datasource.Connector {
var c datasource.Connector = yourconnector.NewConnector()
return &c
}))
// ... rest of setup ...
}
```
Better yet, register through the registry in the connector setup:
```go
// In the service initialization section
connectorRegistry := datasource.NewConnectorRegistry()
connectorRegistry.Register(yourconnector.NewConnector())
connectorRegistry.Register(feishuconnector.NewConnector())
// ... etc
container.Provide(func() *datasource.ConnectorRegistry {
return connectorRegistry
})
```
## Step 6: Add Connector Type Constant
Edit `internal/types/datasource.go`:
```go
const (
// ... existing types ...
ConnectorTypeYourType = "yourtype"
)
```
## Step 7: Add Metadata
Edit `internal/datasource/connector.go`:
```go
var ConnectorMetadataRegistry = map[string]ConnectorMetadata{
// ... existing entries ...
types.ConnectorTypeYourType: {
Type: types.ConnectorTypeYourType,
Name: "Your Platform Name",
Description: "Sync documents from Your Platform",
Priority: X, // Lower number = higher priority in UI
AuthType: "oauth2", // or "api_key", "token", "password"
Capabilities: []string{"incremental", "webhook", "deletion_sync"},
},
}
```
## Step 8: Test Your Connector
```go
// Example test
func TestYourConnectorValidate(t *testing.T) {
connector := NewConnector()
config := &types.DataSourceConfig{
Type: types.ConnectorTypeYourType,
Credentials: map[string]interface{}{
"api_token": "test_token",
},
}
err := connector.Validate(context.Background(), config)
// assert no error
}
func TestYourConnectorFetchAll(t *testing.T) {
connector := NewConnector()
config := &types.DataSourceConfig{
Type: types.ConnectorTypeYourType,
Credentials: map[string]interface{}{
"api_token": "test_token",
},
ResourceIDs: []string{"resource_1"},
}
items, err := connector.FetchAll(context.Background(), config, []string{"resource_1"})
// assert results
}
```
## Checklist
- [ ] Created `internal/datasource/connector/yourtype/` package
- [ ] Implemented `types.go` with platform data structures
- [ ] Implemented `client.go` with API wrapper
- [ ] Implemented `connector.go` with Connector interface
- [ ] Added connector type constant
- [ ] Registered in container
- [ ] Added metadata entry
- [ ] Added unit tests
- [ ] Tested manually with real API
- [ ] Documented any special requirements
## Common Patterns
### OAuth Flow
If using OAuth, store tokens in config:
```go
type Config struct {
AccessToken string
RefreshToken string
ExpiresAt time.Time
}
// Refresh tokens when expired
func (c *Client) ensureValidToken(ctx context.Context) error {
if time.Now().After(c.config.ExpiresAt) {
return c.refreshToken(ctx)
}
return nil
}
```
### Pagination
For platforms with pagination:
```go
func (c *Client) GetDocumentsPage(ctx context.Context, pageToken string) (*Page, error) {
// Returns {Items, NextPageToken}
}
```
### Incremental Sync
For timestamp-based incremental sync:
```go
func (c *Client) GetModifiedSince(ctx context.Context, since time.Time) ([]Item, error) {
// Uses API parameter like &modified_after=2026-03-26T10:00:00Z
}
```
### Deletion Tracking
For platforms that report deletions:
```go
type Item struct {
IsDeleted bool // Set when item is deleted
}
```
## Testing with Real API
1. Set up test credentials
2. Create small test resource (e.g., single document)
3. Test each method:
```go
connector := NewConnector()
config := &types.DataSourceConfig{...}
// Test Validate
err := connector.Validate(ctx, config)
// Test ListResources
resources, err := connector.ListResources(ctx, config)
// Test FetchAll
items, err := connector.FetchAll(ctx, config, []string{resources[0].ExternalID})
// Test FetchIncremental
items, cursor, err := connector.FetchIncremental(ctx, config, nil)
```
---
## Example: Feishu Connector Reference
The Feishu connector would be a good first implementation since:
1. Feishu API is well-documented
2. WeKnora already has `internal/im/feishu/` for reference
3. Popular in China (key market)
4. Has webhook support for real-time sync
Structure:
```
internal/datasource/connector/feishu/
├── client.go (Feishu API client)
├── connector.go (Implements Connector)
└── types.go (Feishu types)
```
Key files to reference:
- `internal/im/feishu/adapter.go` - Feishu API patterns
- `internal/im/feishu/longconn.go` - Connection handling
This will be a good model for other connectors.
+260
View File
@@ -0,0 +1,260 @@
# WeKnora Data Source Sync Framework
## Overview
The data source sync framework enables WeKnora to automatically import and synchronize content from external platforms (Feishu, Notion, Confluence, etc.) into knowledge bases. This is the foundational layer upon which all specific connectors are built.
## Architecture
### Core Components
```
┌─────────────────────────────────────────────────────────────┐
│ External Data Sources │
│ (Feishu, Notion, Confluence, GitHub, Google Drive...) │
└─────────────┬───────────────────────────────────────────────┘
│
▼
┌─────────────────────────────────────────────────────────────┐
│ Connector Registry & Adapters │
│ Each platform (feishu/, notion/, confluence/, etc) │
│ implements the Connector interface │
└─────────────┬───────────────────────────────────────────────┘
│
▼
┌─────────────────────────────────────────────────────────────┐
│ DataSourceService (Business Logic) │
│ ├─ CreateDataSource / UpdateDataSource │
│ ├─ ManualSync / ListAvailableResources │
│ ├─ ValidateConnection / PauseDataSource │
│ └─ ProcessSync (asynq task handler) │
└─────────────┬───────────────────────────────────────────────┘
│
▼
┌─────────────────────────────────────────────────────────────┐
│ HTTP Handler & API Routes (/api/v1/datasource) │
│ REST endpoints for UI/programmatic access │
└─────────────┬───────────────────────────────────────────────┘
│
▼
┌─────────────────────────────────────────────────────────────┐
│ Database (MySQL / PostgreSQL) │
│ ├─ data_sources (configuration & state) │
│ └─ sync_logs (history & metadata) │
└─────────────────────────────────────────────────────────────┘
```
### Task Flow
```
Manual Trigger or Scheduled Job (cron)
↓
Scheduler enqueues asynq Task
↓
ProcessSync (task handler) executes
↓
Connector.Fetch* methods pull data
↓
Diff engine identifies changes (new/update/delete)
↓
KnowledgeService creates/updates Knowledge entries
↓
Documents parsed → chunks → vectors → indexed
```
## File Structure
```
internal/datasource/
├── connector.go # Connector interface & registry
├── errors.go # Error definitions
└── README.md # This file
internal/types/
├── datasource.go # Data models (DataSource, SyncLog, etc)
└── interfaces/datasource.go # Service interfaces
internal/application/
├── repository/datasource_repo.go # Database access layer
└── service/datasource_service.go # Business logic
internal/handler/
└── datasource.go # HTTP request handlers
internal/router/
├── router.go # Route registration
├── task.go # Asynq task registration
└── sync_task.go # Lite mode task registration
migrations/versioned/
└── 000028_datasource_tables.up/down.sql # Database schema
```
## Data Models
### DataSource
Represents a configured external data source:
- **ID**: Unique identifier
- **Type**: Connector type (feishu, notion, confluence, etc)
- **Config**: Encrypted credentials and configuration (JSON)
- **SyncSchedule**: Cron expression (e.g., "0 */6 * * *")
- **SyncMode**: "incremental" or "full"
- **Status**: active | paused | error
- **ConflictStrategy**: overwrite | skip
- **SyncDeletions**: Whether to sync deletions from source
- **LastSyncAt**: Timestamp of last successful sync
- **LastSyncCursor**: State for incremental sync
- **LastSyncResult**: Summary of last sync
### SyncLog
Tracks execution of each sync operation:
- **Status**: running | success | partial | failed | canceled
- **StartedAt/FinishedAt**: Execution timestamps
- **Items***: Counters (total, created, updated, deleted, skipped, failed)
- **ErrorMessage**: Error details if failed
- **Result**: Detailed sync result (JSON)
### Connector Interface
All external data sources implement this interface:
```go
type Connector interface {
Type() string
Validate(ctx, config) error
ListResources(ctx, config) ([]Resource, error)
FetchAll(ctx, config, resourceIDs) ([]FetchedItem, error)
FetchIncremental(ctx, config, cursor) ([]FetchedItem, *SyncCursor, error)
}
```
## API Endpoints
### Data Source Management
```
POST /api/v1/datasource # Create
GET /api/v1/datasource # List (by kb_id)
GET /api/v1/datasource/:id # Get
PUT /api/v1/datasource/:id # Update
DELETE /api/v1/datasource/:id # Delete
```
### Operations
```
POST /api/v1/datasource/:id/validate # Test connection
GET /api/v1/datasource/:id/resources # List available resources
POST /api/v1/datasource/:id/sync # Trigger manual sync
POST /api/v1/datasource/:id/pause # Pause
POST /api/v1/datasource/:id/resume # Resume
```
### Logs
```
GET /api/v1/datasource/:id/logs # List sync logs
GET /api/v1/datasource/logs/:log_id # Get specific log
```
### Metadata
```
GET /api/v1/datasource/types # Available connectors
```
## Implementation Guide
### Adding a New Connector
1. **Create the connector package** in `internal/datasource/connector/<type>/`
```go
// connector/<type>/client.go - API client
// connector/<type>/connector.go - Implements Connector interface
// connector/<type>/types.go - Type-specific models
```
2. **Implement the Connector interface**
```go
type YourConnector struct {
// fields
}
func (c *YourConnector) Type() string {
return types.ConnectorTypeYourType
}
func (c *YourConnector) Validate(ctx, config) error {
// Test connectivity and credentials
}
func (c *YourConnector) ListResources(ctx, config) ([]types.Resource, error) {
// List available resources (documents, spaces, etc)
}
func (c *YourConnector) FetchAll(ctx, config, resourceIDs) ([]types.FetchedItem, error) {
// Full sync - fetch all items from specified resources
}
func (c *YourConnector) FetchIncremental(ctx, config, cursor) ([]types.FetchedItem, *types.SyncCursor, error) {
// Incremental sync - fetch only changed items since cursor
}
```
3. **Register the connector** in the container (in container.go)
```go
container.Provide(func() datasource.Connector {
return yourconnector.NewConnector()
})
```
4. **Add metadata** in `connector.go`
```go
types.ConnectorTypeYourType: {
Type: types.ConnectorTypeYourType,
Name: "Your Platform",
AuthType: "oauth2",
// ...
}
```
## Database Schema
### data_sources table
- Stores data source configurations
- Indexes: tenant_id, knowledge_base_id, type, status
- Encrypted config field (AES-256-GCM)
### sync_logs table
- Stores sync operation history
- Indexes: data_source_id, tenant_id, status, started_at
- Foreign key to data_sources with cascade delete
## Key Design Decisions
1. **Adapter Pattern**: Each connector is a separate implementation, easy to add new ones
2. **Async Task Queue**: Syncs are non-blocking, using asynq (or sync mode in Lite)
3. **Incremental Sync**: Supports cursor-based incremental syncs to minimize API calls
4. **Encryption**: API keys/tokens stored encrypted with AES-256-GCM
5. **Multi-tenant**: All operations are tenant-isolated
6. **Channel Tracking**: Knowledge.Channel field tracks data source type for origin tracking
## Future Enhancements
1. **Webhook Support**: Real-time push from platforms that support webhooks
2. **Conflict Resolution**: Advanced merge/conflict strategies for overlapping content
3. **Rate Limiting**: Per-connector rate limiting and backoff strategies
4. **Scheduling**: Full cron scheduler with time zone support
5. **Monitoring**: Metrics, alerting, and sync health dashboards
6. **Filtering**: User-defined filters for selective syncing (by title, date, tags, etc)
7. **Transformation**: Content transformation pipelines (e.g., extract tables, summarize)
8. **Deduplication**: Smart deduplication across multiple sources
## Testing
The framework is designed to be testable:
- Mock connectors can be created for testing
- No external dependencies required for core logic
- Database operations isolated and mockable
## Error Handling
All operations return detailed error information:
- Connection failures are captured and status updated
- Partial failures are logged (some items succeed, some fail)
- Error messages stored in DataSource for user troubleshooting
+221
View File
@@ -0,0 +1,221 @@
package datasource
import (
"context"
"github.com/Tencent/WeKnora/internal/types"
)
// Connector is the interface that all external data source connectors must implement.
// Each connector (Feishu, Notion, Confluence, etc.) provides an implementation of this interface.
type Connector interface {
// Type returns the connector type identifier (e.g., "feishu", "notion")
Type() string
// Validate verifies that the provided configuration is valid by testing connectivity
// and checking credentials. Returns error if validation fails.
Validate(ctx context.Context, config *types.DataSourceConfig) error
// ListResources lists available resources that can be synced (documents, spaces, folders, etc.)
// Returns a list of Resource objects that the user can select for syncing.
ListResources(ctx context.Context, config *types.DataSourceConfig) ([]types.Resource, error)
// FetchAll performs a full sync of the specified resources.
// Returns all items from the given resource IDs.
FetchAll(ctx context.Context, config *types.DataSourceConfig, resourceIDs []string) ([]types.FetchedItem, error)
// FetchIncremental performs an incremental sync based on the provided cursor.
// Returns items that have changed since the last sync, a new cursor for the next sync,
// and an error if the operation fails.
FetchIncremental(ctx context.Context, config *types.DataSourceConfig, cursor *types.SyncCursor) ([]types.FetchedItem, *types.SyncCursor, error)
}
// ConnectorRegistry manages the registration and lookup of available connectors
type ConnectorRegistry struct {
connectors map[string]Connector
}
// NewConnectorRegistry creates a new connector registry
func NewConnectorRegistry() *ConnectorRegistry {
return &ConnectorRegistry{
connectors: make(map[string]Connector),
}
}
// Register registers a connector with the registry
func (r *ConnectorRegistry) Register(connector Connector) error {
if connector == nil {
return ErrConnectorNil
}
if connector.Type() == "" {
return ErrConnectorTypeEmpty
}
r.connectors[connector.Type()] = connector
return nil
}
// Get retrieves a connector by type
func (r *ConnectorRegistry) Get(connectorType string) (Connector, error) {
connector, exists := r.connectors[connectorType]
if !exists {
return nil, ErrConnectorNotFound
}
return connector, nil
}
// List returns all registered connector types
func (r *ConnectorRegistry) List() []string {
types := make([]string, 0, len(r.connectors))
for t := range r.connectors {
types = append(types, t)
}
return types
}
// ConnectorMetadata provides metadata about available connectors
type ConnectorMetadata struct {
Type string `json:"type"`
Name string `json:"name"`
Description string `json:"description"`
Icon string `json:"icon,omitempty"`
Priority int `json:"priority"` // Priority order for UI display (lower = higher priority)
AuthType string `json:"auth_type"` // "oauth2", "api_key", "token", etc.
Capabilities []string `json:"capabilities"` // "incremental", "webhook", "deletion_sync", etc.
}
// GetConnectorMetadata returns metadata for all available connectors
// This is used by the frontend to display connector options
var ConnectorMetadataRegistry = map[string]ConnectorMetadata{
types.ConnectorTypeFeishu: {
Type: types.ConnectorTypeFeishu,
Name: "Feishu (飞书)",
Description: "Sync documents, wikis, and content from Feishu",
Priority: 0,
AuthType: "oauth2",
Capabilities: []string{"incremental", "webhook", "deletion_sync"},
},
types.ConnectorTypeNotion: {
Type: types.ConnectorTypeNotion,
Name: "Notion",
Description: "Sync pages and databases from Notion",
Priority: 1,
AuthType: "api_key",
Capabilities: []string{"incremental"},
},
types.ConnectorTypeConfluence: {
Type: types.ConnectorTypeConfluence,
Name: "Confluence",
Description: "Sync spaces and pages from Atlassian Confluence",
Priority: 2,
AuthType: "api_key",
Capabilities: []string{"incremental"},
},
types.ConnectorTypeYuque: {
Type: types.ConnectorTypeYuque,
Name: "Yuque (语雀)",
Description: "Sync knowledge bases and documents from Yuque",
Priority: 3,
AuthType: "api_key",
Capabilities: []string{"incremental"},
},
types.ConnectorTypeGitHub: {
Type: types.ConnectorTypeGitHub,
Name: "GitHub",
Description: "Sync repositories, wikis, and issues from GitHub",
Priority: 4,
AuthType: "oauth2",
Capabilities: []string{"incremental"},
},
types.ConnectorTypeGoogleDrive: {
Type: types.ConnectorTypeGoogleDrive,
Name: "Google Drive",
Description: "Sync documents and files from Google Drive",
Priority: 5,
AuthType: "oauth2",
Capabilities: []string{"incremental"},
},
types.ConnectorTypeOneDrive: {
Type: types.ConnectorTypeOneDrive,
Name: "OneDrive / SharePoint",
Description: "Sync documents and files from Microsoft OneDrive",
Priority: 6,
AuthType: "oauth2",
Capabilities: []string{"incremental"},
},
types.ConnectorTypeDingTalk: {
Type: types.ConnectorTypeDingTalk,
Name: "DingTalk (钉钉)",
Description: "Sync documents and content from DingTalk",
Priority: 7,
AuthType: "api_key",
Capabilities: []string{"incremental"},
},
types.ConnectorTypeWecomDoc: {
Type: types.ConnectorTypeWecomDoc,
Name: "WeCom Docs (企微文档)",
Description: "Sync documents and content from WeCom",
Priority: 8,
AuthType: "api_key",
Capabilities: []string{"incremental"},
},
types.ConnectorTypeWebCrawler: {
Type: types.ConnectorTypeWebCrawler,
Name: "Web Crawler (Sitemap)",
Description: "Crawl websites via Sitemap.xml",
Priority: 9,
AuthType: "none",
Capabilities: []string{},
},
types.ConnectorTypeSlack: {
Type: types.ConnectorTypeSlack,
Name: "Slack",
Description: "Sync channel messages and files from Slack",
Priority: 10,
AuthType: "oauth2",
Capabilities: []string{"incremental"},
},
types.ConnectorTypeIMAP: {
Type: types.ConnectorTypeIMAP,
Name: "Email (IMAP)",
Description: "Sync email content from IMAP servers",
Priority: 11,
AuthType: "password",
Capabilities: []string{},
},
types.ConnectorTypeRSS: {
Type: types.ConnectorTypeRSS,
Name: "RSS / Atom Feed",
Description: "Sync articles from RSS/Atom feeds",
Priority: 12,
AuthType: "none",
Capabilities: []string{},
},
}
// GetConnectorMetadata returns metadata for a specific connector type
func GetConnectorMetadata(connectorType string) (ConnectorMetadata, bool) {
meta, ok := ConnectorMetadataRegistry[connectorType]
return meta, ok
}
// ListAvailableConnectors returns all available connector metadata
// sorted by priority
func ListAvailableConnectors() []ConnectorMetadata {
metadata := make([]ConnectorMetadata, 0, len(ConnectorMetadataRegistry))
for _, meta := range ConnectorMetadataRegistry {
metadata = append(metadata, meta)
}
// Sort by priority (insertion sort for simplicity)
for i := 1; i < len(metadata); i++ {
key := metadata[i]
j := i - 1
for j >= 0 && metadata[j].Priority > key.Priority {
metadata[j+1] = metadata[j]
j--
}
metadata[j+1] = key
}
return metadata
}
@@ -0,0 +1,451 @@
package feishu
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"sync"
"time"
"github.com/Tencent/WeKnora/internal/logger"
)
// Client wraps the Feishu Open Platform API for document/wiki operations.
type Client struct {
baseURL string
appID string
appSecret string
httpClient *http.Client
// Token cache (thread-safe)
tokenMu sync.Mutex
tokenCache string
tokenExpAt time.Time
}
// NewClient creates a new Feishu API client.
func NewClient(config *Config) *Client {
return &Client{
baseURL: config.GetBaseURL(),
appID: config.AppID,
appSecret: config.AppSecret,
httpClient: &http.Client{Timeout: 30 * time.Second},
}
}
// getTenantAccessToken retrieves (or returns cached) tenant access token.
// Feishu tokens expire in 2 hours; we cache with a 5-minute safety margin.
func (c *Client) getTenantAccessToken(ctx context.Context) (string, error) {
c.tokenMu.Lock()
defer c.tokenMu.Unlock()
if c.tokenCache != "" && time.Now().Before(c.tokenExpAt) {
return c.tokenCache, nil
}
payload, _ := json.Marshal(map[string]string{
"app_id": c.appID,
"app_secret": c.appSecret,
})
url := c.baseURL + "/open-apis/auth/v3/tenant_access_token/internal"
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(payload))
if err != nil {
return "", fmt.Errorf("create token request: %w", err)
}
req.Header.Set("Content-Type", "application/json; charset=utf-8")
resp, err := c.httpClient.Do(req)
if err != nil {
return "", fmt.Errorf("request token: %w", err)
}
defer resp.Body.Close()
var result tokenResponse
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
return "", fmt.Errorf("decode token response: %w", err)
}
if result.Code != 0 {
return "", fmt.Errorf("feishu auth error: code=%d msg=%s", result.Code, result.Msg)
}
c.tokenCache = result.TenantAccessToken
ttl := time.Duration(result.Expire) * time.Second
if ttl > 5*time.Minute {
ttl -= 5 * time.Minute
}
c.tokenExpAt = time.Now().Add(ttl)
logger.Infof(ctx, "[Feishu] got tenant_access_token: %s...%s expire=%ds",
result.TenantAccessToken[:8], result.TenantAccessToken[len(result.TenantAccessToken)-4:], result.Expire)
return c.tokenCache, nil
}
// doRequest executes an authenticated API request and decodes the JSON response.
func (c *Client) doRequest(ctx context.Context, method, path string, body interface{}, result interface{}) error {
token, err := c.getTenantAccessToken(ctx)
if err != nil {
return err
}
var bodyReader io.Reader
if body != nil {
bodyBytes, err := json.Marshal(body)
if err != nil {
return fmt.Errorf("marshal request body: %w", err)
}
bodyReader = bytes.NewReader(bodyBytes)
}
url := c.baseURL + path
req, err := http.NewRequestWithContext(ctx, method, url, bodyReader)
if err != nil {
return fmt.Errorf("create request: %w", err)
}
req.Header.Set("Content-Type", "application/json; charset=utf-8")
req.Header.Set("Authorization", "Bearer "+token)
logger.Infof(ctx, "[Feishu] %s %s", method, path)
resp, err := c.httpClient.Do(req)
if err != nil {
return fmt.Errorf("execute request: %w", err)
}
defer resp.Body.Close()
// Read body once for logging + decoding
respBody, err := io.ReadAll(resp.Body)
if err != nil {
return fmt.Errorf("read response body: %w", err)
}
logger.Infof(ctx, "[Feishu] %s %s → status=%d bodyLen=%d body=%s",
method, path, resp.StatusCode, len(respBody), truncate(string(respBody), 1000))
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("feishu api error: status=%d body=%s", resp.StatusCode, string(respBody))
}
if result != nil {
if err := json.Unmarshal(respBody, result); err != nil {
return fmt.Errorf("decode response: %w", err)
}
}
return nil
}
// truncate truncates a string to maxLen and appends "..." if truncated.
func truncate(s string, maxLen int) string {
if len(s) <= maxLen {
return s
}
return s[:maxLen] + "..."
}
// ListWikiSpaces returns all wiki spaces accessible to the app.
func (c *Client) ListWikiSpaces(ctx context.Context) ([]wikiSpace, error) {
var allSpaces []wikiSpace
pageToken := ""
for {
path := "/open-apis/wiki/v2/spaces?page_size=50"
if pageToken != "" {
path += "&page_token=" + pageToken
}
var resp wikiSpaceListResponse
if err := c.doRequest(ctx, http.MethodGet, path, nil, &resp); err != nil {
return nil, fmt.Errorf("list wiki spaces: %w", err)
}
if resp.Code != 0 {
logger.Errorf(ctx, "[Feishu] ListWikiSpaces error: code=%d msg=%s", resp.Code, resp.Msg)
return nil, fmt.Errorf("list wiki spaces error: code=%d msg=%s", resp.Code, resp.Msg)
}
logger.Infof(ctx, "[Feishu] ListWikiSpaces: got %d spaces, has_more=%v", len(resp.Data.Items), resp.Data.HasMore)
for i, s := range resp.Data.Items {
logger.Infof(ctx, "[Feishu] space[%d]: id=%s name=%q visibility=%s", i, s.SpaceID, s.Name, s.Visibility)
}
allSpaces = append(allSpaces, resp.Data.Items...)
if !resp.Data.HasMore || resp.Data.PageToken == "" {
break
}
pageToken = resp.Data.PageToken
}
logger.Infof(ctx, "[Feishu] ListWikiSpaces: total %d spaces", len(allSpaces))
return allSpaces, nil
}
// ListWikiNodes returns all nodes (documents) under a wiki space.
// If parentNodeToken is empty, returns top-level nodes.
func (c *Client) ListWikiNodes(ctx context.Context, spaceID string, parentNodeToken string) ([]wikiNode, error) {
var allNodes []wikiNode
pageToken := ""
for {
path := fmt.Sprintf("/open-apis/wiki/v2/spaces/%s/nodes?page_size=50", spaceID)
if parentNodeToken != "" {
path += "&parent_node_token=" + parentNodeToken
}
if pageToken != "" {
path += "&page_token=" + pageToken
}
var resp wikiNodeListResponse
if err := c.doRequest(ctx, http.MethodGet, path, nil, &resp); err != nil {
return nil, fmt.Errorf("list wiki nodes: %w", err)
}
if resp.Code != 0 {
return nil, fmt.Errorf("list wiki nodes error: code=%d msg=%s", resp.Code, resp.Msg)
}
allNodes = append(allNodes, resp.Data.Items...)
if !resp.Data.HasMore || resp.Data.PageToken == "" {
break
}
pageToken = resp.Data.PageToken
}
return allNodes, nil
}
// ListAllWikiNodesRecursive recursively lists all nodes under a wiki space.
// It walks the tree depth-first to discover all nested documents.
func (c *Client) ListAllWikiNodesRecursive(ctx context.Context, spaceID string) ([]wikiNode, error) {
// Start with top-level nodes
topNodes, err := c.ListWikiNodes(ctx, spaceID, "")
if err != nil {
return nil, err
}
var allNodes []wikiNode
var walk func(nodes []wikiNode) error
walk = func(nodes []wikiNode) error {
for _, node := range nodes {
allNodes = append(allNodes, node)
// Recurse into child nodes if this node has children
if node.HasChild {
children, err := c.ListWikiNodes(ctx, spaceID, node.NodeToken)
if err != nil {
return fmt.Errorf("list children of %s: %w", node.NodeToken, err)
}
if err := walk(children); err != nil {
return err
}
}
}
return nil
}
if err := walk(topNodes); err != nil {
return nil, err
}
return allNodes, nil
}
// GetDocumentRawContent retrieves the raw text content of a Feishu docx document.
// This returns plain text (not rich text / block structure).
// Deprecated: prefer ExportAndDownload which preserves formatting.
func (c *Client) GetDocumentRawContent(ctx context.Context, documentID string) (string, error) {
path := fmt.Sprintf("/open-apis/docx/v1/documents/%s/raw_content", documentID)
var resp docRawContentResponse
if err := c.doRequest(ctx, http.MethodGet, path, nil, &resp); err != nil {
return "", fmt.Errorf("get document raw content: %w", err)
}
if resp.Code != 0 {
return "", fmt.Errorf("get document raw content error: code=%d msg=%s", resp.Code, resp.Msg)
}
return resp.Data.Content, nil
}
// Ping verifies the credentials by attempting to get a tenant access token.
func (c *Client) Ping(ctx context.Context) error {
_, err := c.getTenantAccessToken(ctx)
return err
}
// ──────────────────────────────────────────────────────────────────────
// Export task API: export docx/sheet/bitable to downloadable files
//
// Flow:
// 1. POST /drive/v1/export_tasks → create export task, get ticket
// 2. GET /drive/v1/export_tasks/:ticket → poll until status=0 (success)
// 3. GET /drive/v1/export_tasks/file/:ticket/download → download file bytes
// ──────────────────────────────────────────────────────────────────────
// CreateExportTask creates an async export task for a Feishu document.
// - token: the obj_token of the document (e.g. docx token, sheet token)
// - objType: the Feishu obj_type ("docx", "doc", "sheet", "bitable")
// - fileExtension: desired output format ("docx", "xlsx", "pdf")
func (c *Client) CreateExportTask(ctx context.Context, token, objType, fileExtension string) (string, error) {
body := map[string]string{
"file_extension": fileExtension,
"token": token,
"type": objType,
}
var resp exportTaskCreateResponse
if err := c.doRequest(ctx, http.MethodPost, "/open-apis/drive/v1/export_tasks", body, &resp); err != nil {
return "", fmt.Errorf("create export task: %w", err)
}
if resp.Code != 0 {
return "", fmt.Errorf("create export task error: code=%d msg=%s", resp.Code, resp.Msg)
}
return resp.Data.Ticket, nil
}
// GetExportTaskStatus polls the status of an export task.
// Returns (fileToken, fileName, error). fileToken is non-empty only when the job succeeds.
// The token parameter is the obj_token of the document being exported (required by the API).
func (c *Client) GetExportTaskStatus(ctx context.Context, ticket string, token string) (string, string, error) {
path := fmt.Sprintf("/open-apis/drive/v1/export_tasks/%s?token=%s", ticket, token)
var resp exportTaskStatusResponse
if err := c.doRequest(ctx, http.MethodGet, path, nil, &resp); err != nil {
return "", "", fmt.Errorf("get export task status: %w", err)
}
if resp.Code != 0 {
return "", "", fmt.Errorf("get export task status error: code=%d msg=%s", resp.Code, resp.Msg)
}
r := resp.Data.Result
switch r.JobStatus {
case 0: // success
return r.FileToken, r.FileName, nil
case 1, 2: // initializing, processing
return "", "", nil // not ready yet
default:
return "", "", fmt.Errorf("export task failed: status=%d msg=%s", r.JobStatus, r.JobErrorMsg)
}
}
// DownloadExportFile downloads the exported file by its file_token.
// The file_token is returned by GetExportTaskStatus when the export job completes.
// The file must be downloaded within 10 minutes of export completion.
func (c *Client) DownloadExportFile(ctx context.Context, fileToken string) ([]byte, error) {
path := fmt.Sprintf("/open-apis/drive/v1/export_tasks/file/%s/download", fileToken)
return c.downloadRawBytes(ctx, path)
}
// ExportAndDownload is a high-level helper that creates an export task, polls until
// completion, and downloads the resulting file. Returns (fileBytes, fileName, error).
//
// Timeout: 60 seconds. Poll interval: 2 seconds.
func (c *Client) ExportAndDownload(ctx context.Context, objToken, objType string) ([]byte, string, error) {
// Determine export format
fileExt, ok := objTypeToExportFileExtension[objType]
if !ok {
return nil, "", fmt.Errorf("unsupported obj_type for export: %s", objType)
}
exportType, ok := objTypeToExportType[objType]
if !ok {
return nil, "", fmt.Errorf("unsupported obj_type for export: %s", objType)
}
// Step 1: create export task
ticket, err := c.CreateExportTask(ctx, objToken, exportType, fileExt)
if err != nil {
return nil, "", err
}
// Step 2: poll until ready (max 60s, every 2s)
deadline := time.Now().Add(60 * time.Second)
var fileToken, fileName string
for time.Now().Before(deadline) {
fileToken, fileName, err = c.GetExportTaskStatus(ctx, ticket, objToken)
if err != nil {
return nil, "", err
}
if fileToken != "" {
break // export ready
}
select {
case <-ctx.Done():
return nil, "", ctx.Err()
case <-time.After(2 * time.Second):
}
}
if fileToken == "" {
return nil, "", fmt.Errorf("export task timed out after 60s (ticket=%s)", ticket)
}
// Step 3: download file using file_token (NOT ticket)
data, err := c.DownloadExportFile(ctx, fileToken)
if err != nil {
return nil, "", err
}
// Build a sensible file name
if fileName == "" {
fileName = "export" + exportFileExtToSuffix[fileExt]
}
return data, fileName, nil
}
// ──────────────────────────────────────────────────────────────────────
// Drive file download: for "file" type wiki nodes (uploaded PDF/Word/etc.)
// ──────────────────────────────────────────────────────────────────────
// DownloadDriveFile downloads a file from Feishu Drive by its file token.
// Used for wiki nodes with obj_type="file" (user-uploaded PDF, Word, images, etc.).
func (c *Client) DownloadDriveFile(ctx context.Context, fileToken string) ([]byte, error) {
path := fmt.Sprintf("/open-apis/drive/v1/files/%s/download", fileToken)
return c.downloadRawBytes(ctx, path)
}
// downloadRawBytes performs an authenticated GET and returns the raw response body.
func (c *Client) downloadRawBytes(ctx context.Context, path string) ([]byte, error) {
token, err := c.getTenantAccessToken(ctx)
if err != nil {
return nil, err
}
url := c.baseURL + path
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, fmt.Errorf("create download request: %w", err)
}
req.Header.Set("Authorization", "Bearer "+token)
logger.Infof(ctx, "[Feishu] download GET %s", path)
resp, err := c.httpClient.Do(req)
if err != nil {
return nil, fmt.Errorf("download request: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body)
logger.Errorf(ctx, "[Feishu] download GET %s → status=%d body=%s", path, resp.StatusCode, truncate(string(body), 500))
return nil, fmt.Errorf("download failed: status=%d body=%s", resp.StatusCode, string(body))
}
data, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("read download body: %w", err)
}
logger.Infof(ctx, "[Feishu] download GET %s → OK, %d bytes", path, len(data))
return data, nil
}
@@ -0,0 +1,376 @@
package feishu
import (
"context"
"encoding/json"
"fmt"
"strconv"
"strings"
"time"
"github.com/Tencent/WeKnora/internal/types"
)
// Connector implements the datasource.Connector interface for Feishu.
type Connector struct{}
// NewConnector creates a new Feishu connector.
func NewConnector() *Connector {
return &Connector{}
}
// Type returns the connector type identifier.
func (c *Connector) Type() string {
return types.ConnectorTypeFeishu
}
// Validate verifies that the Feishu configuration is valid by testing connectivity.
func (c *Connector) Validate(ctx context.Context, config *types.DataSourceConfig) error {
feishuConfig, err := parseFeishuConfig(config)
if err != nil {
return err
}
client := NewClient(feishuConfig)
if err := client.Ping(ctx); err != nil {
return fmt.Errorf("feishu connection failed: %w", err)
}
return nil
}
// ListResources lists all accessible Feishu Wiki spaces as selectable resources.
func (c *Connector) ListResources(ctx context.Context, config *types.DataSourceConfig) ([]types.Resource, error) {
feishuConfig, err := parseFeishuConfig(config)
if err != nil {
return nil, err
}
client := NewClient(feishuConfig)
spaces, err := client.ListWikiSpaces(ctx)
if err != nil {
return nil, fmt.Errorf("list feishu wiki spaces: %w", err)
}
resources := make([]types.Resource, 0, len(spaces))
for _, space := range spaces {
resources = append(resources, types.Resource{
ExternalID: space.SpaceID,
Name: space.Name,
Type: "wiki_space",
Description: space.Description,
URL: fmt.Sprintf("https://feishu.cn/wiki/%s", space.SpaceID),
Metadata: map[string]interface{}{
"visibility": space.Visibility,
},
})
}
return resources, nil
}
// FetchAll performs a full sync of all documents from the specified wiki spaces.
func (c *Connector) FetchAll(ctx context.Context, config *types.DataSourceConfig, resourceIDs []string) ([]types.FetchedItem, error) {
feishuConfig, err := parseFeishuConfig(config)
if err != nil {
return nil, err
}
client := NewClient(feishuConfig)
var allItems []types.FetchedItem
for _, spaceID := range resourceIDs {
// List all nodes in this wiki space recursively
nodes, err := client.ListAllWikiNodesRecursive(ctx, spaceID)
if err != nil {
return nil, fmt.Errorf("list nodes in space %s: %w", spaceID, err)
}
// Fetch content for each document node
for _, node := range nodes {
item, err := c.fetchNodeContent(ctx, client, node, spaceID)
if err != nil {
// Log error but continue with other nodes
allItems = append(allItems, types.FetchedItem{
ExternalID: node.NodeToken,
Title: node.Title,
SourceResourceID: spaceID,
Metadata: map[string]string{
"error": err.Error(),
},
})
continue
}
if item != nil {
allItems = append(allItems, *item)
}
}
}
return allItems, nil
}
// FetchIncremental performs an incremental sync by comparing node edit times
// against the previously recorded state.
func (c *Connector) FetchIncremental(ctx context.Context, config *types.DataSourceConfig, cursor *types.SyncCursor) ([]types.FetchedItem, *types.SyncCursor, error) {
feishuConfig, err := parseFeishuConfig(config)
if err != nil {
return nil, nil, err
}
client := NewClient(feishuConfig)
// Parse the previous cursor state
var prevCursor feishuCursor
if cursor != nil && cursor.ConnectorCursor != nil {
cursorBytes, _ := json.Marshal(cursor.ConnectorCursor)
_ = json.Unmarshal(cursorBytes, &prevCursor)
}
// Build new cursor to track current state
newCursor := feishuCursor{
LastSyncTime: time.Now(),
SpaceNodeTimes: make(map[string]map[string]string),
}
var changedItems []types.FetchedItem
// Get resource IDs from config
resourceIDs := config.ResourceIDs
if len(resourceIDs) == 0 {
return nil, nil, fmt.Errorf("no resource IDs (wiki space IDs) configured")
}
for _, spaceID := range resourceIDs {
// List all nodes in this space
nodes, err := client.ListAllWikiNodesRecursive(ctx, spaceID)
if err != nil {
return nil, nil, fmt.Errorf("list nodes in space %s: %w", spaceID, err)
}
newCursor.SpaceNodeTimes[spaceID] = make(map[string]string)
// Build a set of current node tokens for deletion detection
currentNodes := make(map[string]bool)
for _, node := range nodes {
currentNodes[node.NodeToken] = true
// Use ObjEditTime (document content edit time) for change detection,
// NOT NodeEditTime which only tracks node attribute changes (title, position).
editTimeStr := node.ObjEditTime
if editTimeStr == "" {
editTimeStr = node.NodeEditTime // fallback for nodes that don't have obj_edit_time
}
newCursor.SpaceNodeTimes[spaceID][node.NodeToken] = editTimeStr
// Check if node has changed since last sync
if prevCursor.SpaceNodeTimes != nil {
if prevTimes, ok := prevCursor.SpaceNodeTimes[spaceID]; ok {
if prevEditTime, exists := prevTimes[node.NodeToken]; exists {
if prevEditTime == editTimeStr {
// Node unchanged, skip
continue
}
}
}
}
// Node is new or changed — fetch its content
item, err := c.fetchNodeContent(ctx, client, node, spaceID)
if err != nil {
// Record failed items
changedItems = append(changedItems, types.FetchedItem{
ExternalID: node.NodeToken,
Title: node.Title,
SourceResourceID: spaceID,
Metadata: map[string]string{
"error": err.Error(),
},
})
continue
}
if item != nil {
changedItems = append(changedItems, *item)
}
}
// Detect deleted nodes
if prevCursor.SpaceNodeTimes != nil {
if prevTimes, ok := prevCursor.SpaceNodeTimes[spaceID]; ok {
for nodeToken := range prevTimes {
if !currentNodes[nodeToken] {
// Node was deleted
changedItems = append(changedItems, types.FetchedItem{
ExternalID: nodeToken,
IsDeleted: true,
SourceResourceID: spaceID,
})
}
}
}
}
}
// Build next sync cursor
nextCursorMap := make(map[string]interface{})
cursorBytes, _ := json.Marshal(newCursor)
_ = json.Unmarshal(cursorBytes, &nextCursorMap)
nextSyncCursor := &types.SyncCursor{
LastSyncTime: time.Now(),
ConnectorCursor: nextCursorMap,
}
return changedItems, nextSyncCursor, nil
}
// fetchNodeContent fetches the content of a single wiki node and converts it to FetchedItem.
// Dispatches to different retrieval strategies based on obj_type:
// - docx/doc → export API → .docx file
// - sheet → export API → .xlsx file
// - bitable → export API → .xlsx file
// - file → drive download → original file (PDF/Word/image/etc.)
// - mindnote → skip (no API)
// - slides → skip (no API)
func (c *Connector) fetchNodeContent(ctx context.Context, client *Client, node wikiNode, spaceID string) (*types.FetchedItem, error) {
if !isSupportedDocType(node.ObjType) {
return nil, nil
}
editTime := parseFeishuTimestamp(node.NodeEditTime)
baseMeta := map[string]string{
"obj_token": node.ObjToken,
"obj_type": node.ObjType,
"node_token": node.NodeToken,
"space_id": spaceID,
"creator": node.Creator,
"owner": node.Owner,
"channel": types.ChannelFeishu,
}
switch node.ObjType {
case "docx", "doc", "sheet", "bitable":
// Export as a file via the async export API
data, fileName, err := client.ExportAndDownload(ctx, node.ObjToken, node.ObjType)
if err != nil {
return nil, fmt.Errorf("export %s (%s): %w", node.Title, node.ObjType, err)
}
// Ensure a reasonable file name with correct extension
ext := exportFileExtToSuffix[objTypeToExportFileExtension[node.ObjType]]
if fileName == "" {
fileName = sanitizeFileName(node.Title) + ext
} else if !strings.HasSuffix(strings.ToLower(fileName), ext) {
// Feishu often returns the doc title without extension — append it
fileName = sanitizeFileName(fileName) + ext
}
return &types.FetchedItem{
ExternalID: node.NodeToken,
Title: node.Title,
Content: data,
ContentType: "application/octet-stream",
FileName: fileName,
URL: fmt.Sprintf("https://feishu.cn/wiki/%s", node.NodeToken),
UpdatedAt: editTime,
SourceResourceID: spaceID,
Metadata: baseMeta,
}, nil
case "file":
// Download the original uploaded file from Drive
data, err := client.DownloadDriveFile(ctx, node.ObjToken)
if err != nil {
return nil, fmt.Errorf("download file %s (%s): %w", node.Title, node.ObjToken, err)
}
// Use the node title as file name; it usually preserves the original extension
fileName := node.Title
if fileName == "" {
fileName = node.ObjToken
}
return &types.FetchedItem{
ExternalID: node.NodeToken,
Title: node.Title,
Content: data,
ContentType: "application/octet-stream",
FileName: fileName,
URL: fmt.Sprintf("https://feishu.cn/wiki/%s", node.NodeToken),
UpdatedAt: editTime,
SourceResourceID: spaceID,
Metadata: baseMeta,
}, nil
default:
return nil, nil
}
}
// --- Helper functions ---
// parseFeishuConfig extracts and validates Feishu-specific configuration.
func parseFeishuConfig(config *types.DataSourceConfig) (*Config, error) {
if config == nil {
return nil, fmt.Errorf("config is nil")
}
credBytes, err := json.Marshal(config.Credentials)
if err != nil {
return nil, fmt.Errorf("marshal credentials: %w", err)
}
var feishuConfig Config
if err := json.Unmarshal(credBytes, &feishuConfig); err != nil {
return nil, fmt.Errorf("parse feishu credentials: %w", err)
}
if feishuConfig.AppID == "" || feishuConfig.AppSecret == "" {
return nil, fmt.Errorf("feishu app_id and app_secret are required")
}
return &feishuConfig, nil
}
// isSupportedDocType checks if a Feishu document type can be synced.
// mindnote and slides have no content read API and are skipped.
func isSupportedDocType(objType string) bool {
switch objType {
case "docx", "doc", "sheet", "bitable", "file":
return true
default:
// mindnote, slides — no content retrieval API available
return false
}
}
// parseFeishuTimestamp parses a Feishu unix timestamp string (seconds) into time.Time.
func parseFeishuTimestamp(ts string) time.Time {
if ts == "" {
return time.Time{}
}
sec, err := strconv.ParseInt(ts, 10, 64)
if err != nil {
return time.Time{}
}
return time.Unix(sec, 0)
}
// sanitizeFileName removes characters that are invalid in filenames.
func sanitizeFileName(name string) string {
if name == "" {
return "untitled"
}
// Replace common invalid characters with underscore
replacer := strings.NewReplacer(
"/", "_", "\\", "_", ":", "_", "*", "_",
"?", "_", "\"", "_", "<", "_", ">", "_", "|", "_",
)
result := replacer.Replace(name)
// Limit length
if len(result) > 200 {
result = result[:200]
}
return result
}
@@ -0,0 +1,817 @@
package feishu
import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"github.com/Tencent/WeKnora/internal/types"
)
// ──────────────────────────────────────────────────────────────────────
// Fake Feishu API server
// ──────────────────────────────────────────────────────────────────────
// fakeFeishu builds an httptest.Server that emulates the relevant Feishu APIs.
// It returns the server and a Config pointing at it.
func fakeFeishu(nodes []wikiNode) (*httptest.Server, *Config) {
mux := http.NewServeMux()
// --- auth ---
mux.HandleFunc("/open-apis/auth/v3/tenant_access_token/internal", func(w http.ResponseWriter, r *http.Request) {
writeJSON(w, tokenResponse{
apiResponse: apiResponse{Code: 0},
TenantAccessToken: "fake-token",
Expire: 7200,
})
})
// --- wiki spaces ---
mux.HandleFunc("/open-apis/wiki/v2/spaces", func(w http.ResponseWriter, r *http.Request) {
writeJSON(w, wikiSpaceListResponse{
apiResponse: apiResponse{Code: 0},
Data: struct {
Items []wikiSpace `json:"items"`
HasMore bool `json:"has_more"`
PageToken string `json:"page_token"`
}{
Items: []wikiSpace{
{SpaceID: "space1", Name: "Test Space", Description: "desc", Visibility: "public"},
},
},
})
})
// --- wiki nodes (top-level only for simplicity) ---
mux.HandleFunc("/open-apis/wiki/v2/spaces/space1/nodes", func(w http.ResponseWriter, r *http.Request) {
writeJSON(w, wikiNodeListResponse{
apiResponse: apiResponse{Code: 0},
Data: struct {
Items []wikiNode `json:"items"`
HasMore bool `json:"has_more"`
PageToken string `json:"page_token"`
}{
Items: nodes,
},
})
})
// --- export task: create ---
mux.HandleFunc("/open-apis/drive/v1/export_tasks", func(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodPost {
writeJSON(w, exportTaskCreateResponse{
apiResponse: apiResponse{Code: 0},
Data: struct{ Ticket string `json:"ticket"` }{Ticket: "ticket-123"},
})
return
}
// GET /open-apis/drive/v1/export_tasks/ticket-123
writeJSON(w, exportTaskStatusResponse{
apiResponse: apiResponse{Code: 0},
Data: struct {
Result struct {
FileToken string `json:"file_token"`
FileSize int64 `json:"file_size"`
JobStatus int `json:"job_status"`
JobErrorMsg string `json:"job_error_msg"`
FileName string `json:"file_name"`
} `json:"result"`
}{
Result: struct {
FileToken string `json:"file_token"`
FileSize int64 `json:"file_size"`
JobStatus int `json:"job_status"`
JobErrorMsg string `json:"job_error_msg"`
FileName string `json:"file_name"`
}{
FileToken: "ft-abc",
FileSize: 100,
JobStatus: 0, // success
FileName: "exported.docx",
},
},
})
})
// --- export task: status polling (pattern match with ticket) ---
mux.HandleFunc("/open-apis/drive/v1/export_tasks/ticket-123", func(w http.ResponseWriter, r *http.Request) {
writeJSON(w, exportTaskStatusResponse{
apiResponse: apiResponse{Code: 0},
Data: struct {
Result struct {
FileToken string `json:"file_token"`
FileSize int64 `json:"file_size"`
JobStatus int `json:"job_status"`
JobErrorMsg string `json:"job_error_msg"`
FileName string `json:"file_name"`
} `json:"result"`
}{
Result: struct {
FileToken string `json:"file_token"`
FileSize int64 `json:"file_size"`
JobStatus int `json:"job_status"`
JobErrorMsg string `json:"job_error_msg"`
FileName string `json:"file_name"`
}{
FileToken: "ft-abc",
FileSize: 100,
JobStatus: 0,
FileName: "exported.docx",
},
},
})
})
// --- export file download ---
mux.HandleFunc("/open-apis/drive/v1/export_tasks/file/ticket-123/download", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/octet-stream")
w.Write([]byte("fake-docx-content"))
})
// --- drive file download (for "file" type nodes) ---
mux.HandleFunc("/open-apis/drive/v1/files/", func(w http.ResponseWriter, r *http.Request) {
if strings.HasSuffix(r.URL.Path, "/download") {
w.Header().Set("Content-Type", "application/octet-stream")
w.Write([]byte("fake-pdf-binary"))
return
}
http.NotFound(w, r)
})
ts := httptest.NewServer(mux)
cfg := &Config{
AppID: "test-app-id",
AppSecret: "test-app-secret",
BaseURL: ts.URL,
}
return ts, cfg
}
func writeJSON(w http.ResponseWriter, v interface{}) {
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(v)
}
func makeConfig(cfg *Config, resourceIDs []string) *types.DataSourceConfig {
creds := map[string]interface{}{
"app_id": cfg.AppID,
"app_secret": cfg.AppSecret,
"base_url": cfg.BaseURL,
}
return &types.DataSourceConfig{
Type: types.ConnectorTypeFeishu,
Credentials: creds,
ResourceIDs: resourceIDs,
}
}
// ──────────────────────────────────────────────────────────────────────
// Helper function tests
// ──────────────────────────────────────────────────────────────────────
func TestIsSupportedDocType(t *testing.T) {
tests := []struct {
objType string
expected bool
}{
{"docx", true},
{"doc", true},
{"sheet", true},
{"bitable", true},
{"file", true},
{"mindnote", false},
{"slides", false},
{"unknown", false},
{"", false},
}
for _, tt := range tests {
t.Run(tt.objType, func(t *testing.T) {
got := isSupportedDocType(tt.objType)
if got != tt.expected {
t.Errorf("isSupportedDocType(%q) = %v, want %v", tt.objType, got, tt.expected)
}
})
}
}
func TestSanitizeFileName(t *testing.T) {
tests := []struct {
input string
expected string
}{
{"hello", "hello"},
{"", "untitled"},
{"a/b\\c:d*e", "a_b_c_d_e"},
{"normal file.docx", "normal file.docx"},
{strings.Repeat("a", 300), strings.Repeat("a", 200)},
}
for _, tt := range tests {
t.Run(tt.input, func(t *testing.T) {
got := sanitizeFileName(tt.input)
if got != tt.expected {
t.Errorf("sanitizeFileName(%q) = %q, want %q", tt.input, got, tt.expected)
}
})
}
}
func TestParseFeishuTimestamp(t *testing.T) {
ts := parseFeishuTimestamp("1711468800") // 2024-03-27 00:00:00 UTC
if ts.IsZero() {
t.Fatal("expected non-zero time")
}
if ts.Unix() != 1711468800 {
t.Errorf("unexpected unix = %d", ts.Unix())
}
if !parseFeishuTimestamp("").IsZero() {
t.Error("expected zero time for empty string")
}
if !parseFeishuTimestamp("invalid").IsZero() {
t.Error("expected zero time for invalid string")
}
}
func TestParseFeishuConfig(t *testing.T) {
t.Run("valid", func(t *testing.T) {
cfg, err := parseFeishuConfig(&types.DataSourceConfig{
Credentials: map[string]interface{}{
"app_id": "id1",
"app_secret": "sec1",
"base_url": "https://custom.example.com",
},
})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if cfg.AppID != "id1" || cfg.AppSecret != "sec1" || cfg.BaseURL != "https://custom.example.com" {
t.Errorf("unexpected config: %+v", cfg)
}
})
t.Run("nil config", func(t *testing.T) {
_, err := parseFeishuConfig(nil)
if err == nil {
t.Fatal("expected error for nil config")
}
})
t.Run("missing credentials", func(t *testing.T) {
_, err := parseFeishuConfig(&types.DataSourceConfig{
Credentials: map[string]interface{}{
"app_id": "id1",
// missing app_secret
},
})
if err == nil {
t.Fatal("expected error for missing app_secret")
}
})
}
// ──────────────────────────────────────────────────────────────────────
// Connector interface tests
// ──────────────────────────────────────────────────────────────────────
func TestConnectorType(t *testing.T) {
c := NewConnector()
if c.Type() != types.ConnectorTypeFeishu {
t.Errorf("Type() = %q, want %q", c.Type(), types.ConnectorTypeFeishu)
}
}
func TestConnectorValidate(t *testing.T) {
ts, cfg := fakeFeishu(nil)
defer ts.Close()
c := NewConnector()
err := c.Validate(context.Background(), makeConfig(cfg, nil))
if err != nil {
t.Fatalf("Validate() error: %v", err)
}
}
func TestConnectorValidate_BadCredentials(t *testing.T) {
c := NewConnector()
err := c.Validate(context.Background(), &types.DataSourceConfig{
Credentials: map[string]interface{}{
"app_id": "bad",
"app_secret": "bad",
"base_url": "http://127.0.0.1:1", // will fail to connect
},
})
if err == nil {
t.Fatal("expected error for bad credentials")
}
}
func TestConnectorListResources(t *testing.T) {
ts, cfg := fakeFeishu(nil)
defer ts.Close()
c := NewConnector()
resources, err := c.ListResources(context.Background(), makeConfig(cfg, nil))
if err != nil {
t.Fatalf("ListResources() error: %v", err)
}
if len(resources) != 1 {
t.Fatalf("expected 1 resource, got %d", len(resources))
}
if resources[0].ExternalID != "space1" {
t.Errorf("ExternalID = %q, want %q", resources[0].ExternalID, "space1")
}
if resources[0].Name != "Test Space" {
t.Errorf("Name = %q, want %q", resources[0].Name, "Test Space")
}
if resources[0].Type != "wiki_space" {
t.Errorf("Type = %q, want %q", resources[0].Type, "wiki_space")
}
}
// ──────────────────────────────────────────────────────────────────────
// FetchAll: test all supported doc types
// ──────────────────────────────────────────────────────────────────────
func TestFetchAll_DocxNode(t *testing.T) {
nodes := []wikiNode{{
NodeToken: "nt1",
ObjToken: "obj-docx-1",
ObjType: "docx",
Title: "My Document",
NodeEditTime: "1711468800",
}}
ts, cfg := fakeFeishu(nodes)
defer ts.Close()
c := NewConnector()
items, err := c.FetchAll(context.Background(), makeConfig(cfg, []string{"space1"}), []string{"space1"})
if err != nil {
t.Fatalf("FetchAll() error: %v", err)
}
if len(items) != 1 {
t.Fatalf("expected 1 item, got %d", len(items))
}
item := items[0]
if item.ExternalID != "nt1" {
t.Errorf("ExternalID = %q, want %q", item.ExternalID, "nt1")
}
if item.Title != "My Document" {
t.Errorf("Title = %q", item.Title)
}
if string(item.Content) != "fake-docx-content" {
t.Errorf("Content = %q, want %q", string(item.Content), "fake-docx-content")
}
if item.FileName != "exported.docx" {
t.Errorf("FileName = %q, want %q", item.FileName, "exported.docx")
}
if item.Metadata["obj_type"] != "docx" {
t.Errorf("Metadata[obj_type] = %q", item.Metadata["obj_type"])
}
if item.Metadata["channel"] != types.ChannelFeishu {
t.Errorf("Metadata[channel] = %q", item.Metadata["channel"])
}
}
func TestFetchAll_SheetNode(t *testing.T) {
nodes := []wikiNode{{
NodeToken: "nt-sheet",
ObjToken: "obj-sheet-1",
ObjType: "sheet",
Title: "Sales Report",
NodeEditTime: "1711468800",
}}
ts, cfg := fakeFeishu(nodes)
defer ts.Close()
c := NewConnector()
items, err := c.FetchAll(context.Background(), makeConfig(cfg, []string{"space1"}), []string{"space1"})
if err != nil {
t.Fatalf("FetchAll() error: %v", err)
}
if len(items) != 1 {
t.Fatalf("expected 1 item, got %d", len(items))
}
if items[0].Metadata["obj_type"] != "sheet" {
t.Errorf("obj_type = %q, want sheet", items[0].Metadata["obj_type"])
}
}
func TestFetchAll_BitableNode(t *testing.T) {
nodes := []wikiNode{{
NodeToken: "nt-bitable",
ObjToken: "obj-bitable-1",
ObjType: "bitable",
Title: "Project Tracker",
NodeEditTime: "1711468800",
}}
ts, cfg := fakeFeishu(nodes)
defer ts.Close()
c := NewConnector()
items, err := c.FetchAll(context.Background(), makeConfig(cfg, []string{"space1"}), []string{"space1"})
if err != nil {
t.Fatalf("FetchAll() error: %v", err)
}
if len(items) != 1 {
t.Fatalf("expected 1 item, got %d", len(items))
}
if items[0].Metadata["obj_type"] != "bitable" {
t.Errorf("obj_type = %q, want bitable", items[0].Metadata["obj_type"])
}
}
func TestFetchAll_FileNode(t *testing.T) {
nodes := []wikiNode{{
NodeToken: "nt-file",
ObjToken: "obj-file-1",
ObjType: "file",
Title: "manual.pdf",
NodeEditTime: "1711468800",
}}
ts, cfg := fakeFeishu(nodes)
defer ts.Close()
c := NewConnector()
items, err := c.FetchAll(context.Background(), makeConfig(cfg, []string{"space1"}), []string{"space1"})
if err != nil {
t.Fatalf("FetchAll() error: %v", err)
}
if len(items) != 1 {
t.Fatalf("expected 1 item, got %d", len(items))
}
item := items[0]
if string(item.Content) != "fake-pdf-binary" {
t.Errorf("Content = %q, want %q", string(item.Content), "fake-pdf-binary")
}
if item.FileName != "manual.pdf" {
t.Errorf("FileName = %q, want %q", item.FileName, "manual.pdf")
}
if item.Metadata["obj_type"] != "file" {
t.Errorf("obj_type = %q, want file", item.Metadata["obj_type"])
}
}
func TestFetchAll_SkipsMindnoteAndSlides(t *testing.T) {
nodes := []wikiNode{
{NodeToken: "nt-mn", ObjToken: "obj-mn", ObjType: "mindnote", Title: "Brain Map"},
{NodeToken: "nt-sl", ObjToken: "obj-sl", ObjType: "slides", Title: "Presentation"},
}
ts, cfg := fakeFeishu(nodes)
defer ts.Close()
c := NewConnector()
items, err := c.FetchAll(context.Background(), makeConfig(cfg, []string{"space1"}), []string{"space1"})
if err != nil {
t.Fatalf("FetchAll() error: %v", err)
}
// Both should be skipped (nil returned by fetchNodeContent)
if len(items) != 0 {
t.Errorf("expected 0 items (mindnote+slides skipped), got %d", len(items))
}
}
func TestFetchAll_MixedTypes(t *testing.T) {
nodes := []wikiNode{
{NodeToken: "nt1", ObjToken: "obj1", ObjType: "docx", Title: "Doc", NodeEditTime: "1711468800"},
{NodeToken: "nt2", ObjToken: "obj2", ObjType: "sheet", Title: "Sheet", NodeEditTime: "1711468800"},
{NodeToken: "nt3", ObjToken: "obj3", ObjType: "file", Title: "report.pdf", NodeEditTime: "1711468800"},
{NodeToken: "nt4", ObjToken: "obj4", ObjType: "mindnote", Title: "Mind", NodeEditTime: "1711468800"},
{NodeToken: "nt5", ObjToken: "obj5", ObjType: "slides", Title: "Slides", NodeEditTime: "1711468800"},
{NodeToken: "nt6", ObjToken: "obj6", ObjType: "bitable", Title: "Table", NodeEditTime: "1711468800"},
}
ts, cfg := fakeFeishu(nodes)
defer ts.Close()
c := NewConnector()
items, err := c.FetchAll(context.Background(), makeConfig(cfg, []string{"space1"}), []string{"space1"})
if err != nil {
t.Fatalf("FetchAll() error: %v", err)
}
// docx + sheet + file + bitable = 4 items; mindnote + slides = skipped
if len(items) != 4 {
t.Errorf("expected 4 items, got %d", len(items))
for i, it := range items {
t.Logf(" item[%d]: %s (obj_type=%s)", i, it.Title, it.Metadata["obj_type"])
}
}
}
// ──────────────────────────────────────────────────────────────────────
// FetchIncremental tests
// ──────────────────────────────────────────────────────────────────────
func TestFetchIncremental_FirstSync(t *testing.T) {
nodes := []wikiNode{
{NodeToken: "nt1", ObjToken: "obj1", ObjType: "docx", Title: "Doc1", NodeEditTime: "100"},
{NodeToken: "nt2", ObjToken: "obj2", ObjType: "file", Title: "file.pdf", NodeEditTime: "200"},
}
ts, cfg := fakeFeishu(nodes)
defer ts.Close()
c := NewConnector()
dsConfig := makeConfig(cfg, []string{"space1"})
// First sync with no cursor → all items should be fetched
items, cursor, err := c.FetchIncremental(context.Background(), dsConfig, nil)
if err != nil {
t.Fatalf("FetchIncremental() error: %v", err)
}
if len(items) != 2 {
t.Fatalf("expected 2 items on first sync, got %d", len(items))
}
if cursor == nil {
t.Fatal("expected non-nil cursor")
}
if cursor.LastSyncTime.IsZero() {
t.Error("cursor.LastSyncTime should not be zero")
}
}
func TestFetchIncremental_NoChanges(t *testing.T) {
nodes := []wikiNode{
{NodeToken: "nt1", ObjToken: "obj1", ObjType: "docx", Title: "Doc1", NodeEditTime: "100"},
}
ts, cfg := fakeFeishu(nodes)
defer ts.Close()
c := NewConnector()
dsConfig := makeConfig(cfg, []string{"space1"})
// First sync
_, cursor1, err := c.FetchIncremental(context.Background(), dsConfig, nil)
if err != nil {
t.Fatalf("first sync error: %v", err)
}
// Second sync with same edit times → should return 0 changed items
items, _, err := c.FetchIncremental(context.Background(), dsConfig, cursor1)
if err != nil {
t.Fatalf("second sync error: %v", err)
}
if len(items) != 0 {
t.Errorf("expected 0 items (no changes), got %d", len(items))
}
}
func TestFetchIncremental_DetectsDeleted(t *testing.T) {
// First sync: 2 nodes
allNodes := []wikiNode{
{NodeToken: "nt1", ObjToken: "obj1", ObjType: "docx", Title: "Doc1", NodeEditTime: "100"},
{NodeToken: "nt2", ObjToken: "obj2", ObjType: "docx", Title: "Doc2", NodeEditTime: "200"},
}
ts, cfg := fakeFeishu(allNodes)
c := NewConnector()
dsConfig := makeConfig(cfg, []string{"space1"})
_, cursor1, err := c.FetchIncremental(context.Background(), dsConfig, nil)
if err != nil {
t.Fatalf("first sync error: %v", err)
}
ts.Close()
// Second sync: only 1 node remains (nt2 was deleted)
ts2, cfg2 := fakeFeishu([]wikiNode{
{NodeToken: "nt1", ObjToken: "obj1", ObjType: "docx", Title: "Doc1", NodeEditTime: "100"},
})
defer ts2.Close()
dsConfig2 := makeConfig(cfg2, []string{"space1"})
items, _, err := c.FetchIncremental(context.Background(), dsConfig2, cursor1)
if err != nil {
t.Fatalf("second sync error: %v", err)
}
// Should have 1 deleted item for nt2
var deletedCount int
for _, item := range items {
if item.IsDeleted {
deletedCount++
if item.ExternalID != "nt2" {
t.Errorf("expected deleted ExternalID=nt2, got %q", item.ExternalID)
}
}
}
if deletedCount != 1 {
t.Errorf("expected 1 deleted item, got %d", deletedCount)
}
}
func TestFetchIncremental_NoResourceIDs(t *testing.T) {
ts, cfg := fakeFeishu(nil)
defer ts.Close()
c := NewConnector()
dsConfig := makeConfig(cfg, nil) // no resource IDs
dsConfig.ResourceIDs = nil
_, _, err := c.FetchIncremental(context.Background(), dsConfig, nil)
if err == nil {
t.Fatal("expected error for empty resource IDs")
}
if !strings.Contains(err.Error(), "no resource IDs") {
t.Errorf("unexpected error: %v", err)
}
}
// ──────────────────────────────────────────────────────────────────────
// Client tests
// ──────────────────────────────────────────────────────────────────────
func TestClientPing(t *testing.T) {
ts, cfg := fakeFeishu(nil)
defer ts.Close()
client := NewClient(cfg)
if err := client.Ping(context.Background()); err != nil {
t.Fatalf("Ping() error: %v", err)
}
}
func TestClientTokenCaching(t *testing.T) {
callCount := 0
mux := http.NewServeMux()
mux.HandleFunc("/open-apis/auth/v3/tenant_access_token/internal", func(w http.ResponseWriter, r *http.Request) {
callCount++
writeJSON(w, tokenResponse{
apiResponse: apiResponse{Code: 0},
TenantAccessToken: fmt.Sprintf("token-%d", callCount),
Expire: 7200,
})
})
ts := httptest.NewServer(mux)
defer ts.Close()
client := NewClient(&Config{AppID: "a", AppSecret: "b", BaseURL: ts.URL})
// First call: fetches token
t1, _ := client.getTenantAccessToken(context.Background())
// Second call: should use cache
t2, _ := client.getTenantAccessToken(context.Background())
if t1 != t2 {
t.Errorf("expected cached token, got different tokens: %q vs %q", t1, t2)
}
if callCount != 1 {
t.Errorf("expected 1 auth call, got %d", callCount)
}
}
func TestClientExportAndDownload(t *testing.T) {
ts, cfg := fakeFeishu(nil)
defer ts.Close()
client := NewClient(cfg)
data, fileName, err := client.ExportAndDownload(context.Background(), "obj-token-1", "docx")
if err != nil {
t.Fatalf("ExportAndDownload() error: %v", err)
}
if string(data) != "fake-docx-content" {
t.Errorf("data = %q, want %q", string(data), "fake-docx-content")
}
if fileName != "exported.docx" {
t.Errorf("fileName = %q, want %q", fileName, "exported.docx")
}
}
func TestClientExportAndDownload_UnsupportedType(t *testing.T) {
ts, cfg := fakeFeishu(nil)
defer ts.Close()
client := NewClient(cfg)
_, _, err := client.ExportAndDownload(context.Background(), "obj-token-1", "mindnote")
if err == nil {
t.Fatal("expected error for unsupported type")
}
if !strings.Contains(err.Error(), "unsupported") {
t.Errorf("unexpected error: %v", err)
}
}
func TestClientDownloadDriveFile(t *testing.T) {
ts, cfg := fakeFeishu(nil)
defer ts.Close()
client := NewClient(cfg)
data, err := client.DownloadDriveFile(context.Background(), "file-token-abc")
if err != nil {
t.Fatalf("DownloadDriveFile() error: %v", err)
}
if string(data) != "fake-pdf-binary" {
t.Errorf("data = %q, want %q", string(data), "fake-pdf-binary")
}
}
func TestClientListWikiSpaces(t *testing.T) {
ts, cfg := fakeFeishu(nil)
defer ts.Close()
client := NewClient(cfg)
spaces, err := client.ListWikiSpaces(context.Background())
if err != nil {
t.Fatalf("ListWikiSpaces() error: %v", err)
}
if len(spaces) != 1 {
t.Fatalf("expected 1 space, got %d", len(spaces))
}
if spaces[0].SpaceID != "space1" {
t.Errorf("SpaceID = %q", spaces[0].SpaceID)
}
}
// ──────────────────────────────────────────────────────────────────────
// Type mapping tests
// ──────────────────────────────────────────────────────────────────────
func TestObjTypeToExportMappings(t *testing.T) {
// Verify all exportable types have valid mappings
exportable := []string{"docx", "doc", "sheet", "bitable"}
for _, ot := range exportable {
if _, ok := objTypeToExportFileExtension[ot]; !ok {
t.Errorf("objTypeToExportFileExtension missing %q", ot)
}
if _, ok := objTypeToExportType[ot]; !ok {
t.Errorf("objTypeToExportType missing %q", ot)
}
}
// Verify non-exportable types do NOT have mappings
nonExportable := []string{"file", "mindnote", "slides"}
for _, ot := range nonExportable {
if _, ok := objTypeToExportFileExtension[ot]; ok {
t.Errorf("objTypeToExportFileExtension should NOT contain %q", ot)
}
}
}
func TestExportFileExtToSuffix(t *testing.T) {
if exportFileExtToSuffix[ExportTypeDocx] != ".docx" {
t.Errorf("docx suffix = %q", exportFileExtToSuffix[ExportTypeDocx])
}
if exportFileExtToSuffix[ExportTypeXlsx] != ".xlsx" {
t.Errorf("xlsx suffix = %q", exportFileExtToSuffix[ExportTypeXlsx])
}
if exportFileExtToSuffix[ExportTypePDF] != ".pdf" {
t.Errorf("pdf suffix = %q", exportFileExtToSuffix[ExportTypePDF])
}
}
// ──────────────────────────────────────────────────────────────────────
// Feishu cursor serialization
// ──────────────────────────────────────────────────────────────────────
func TestFeishuCursorRoundTrip(t *testing.T) {
original := feishuCursor{
LastSyncTime: time.Date(2026, 3, 26, 12, 0, 0, 0, time.UTC),
SpaceNodeTimes: map[string]map[string]string{
"space1": {
"nt1": "100",
"nt2": "200",
},
},
}
// Serialize to map[string]interface{} (as stored in SyncCursor.ConnectorCursor)
data, err := json.Marshal(original)
if err != nil {
t.Fatalf("marshal error: %v", err)
}
var cursorMap map[string]interface{}
if err := json.Unmarshal(data, &cursorMap); err != nil {
t.Fatalf("unmarshal to map error: %v", err)
}
// Deserialize back
data2, _ := json.Marshal(cursorMap)
var restored feishuCursor
if err := json.Unmarshal(data2, &restored); err != nil {
t.Fatalf("restore error: %v", err)
}
if restored.SpaceNodeTimes["space1"]["nt1"] != "100" {
t.Errorf("restored nt1 = %q, want 100", restored.SpaceNodeTimes["space1"]["nt1"])
}
if restored.SpaceNodeTimes["space1"]["nt2"] != "200" {
t.Errorf("restored nt2 = %q, want 200", restored.SpaceNodeTimes["space1"]["nt2"])
}
}
@@ -0,0 +1,206 @@
// Package feishu implements the Feishu (飞书/Lark) data source connector for WeKnora.
//
// It syncs documents from Feishu Wiki spaces and cloud documents into WeKnora knowledge bases.
//
// Feishu API docs:
// - Wiki spaces: https://open.feishu.cn/document/server-docs/docs/wiki-v2/space/list
// - Wiki nodes: https://open.feishu.cn/document/server-docs/docs/wiki-v2/space-node/list
// - Export tasks: https://open.feishu.cn/document/server-docs/docs/drive-v1/export_task/export-user-guide
// - File download: https://open.feishu.cn/document/server-docs/docs/drive-v1/file/download
// - Auth: https://open.feishu.cn/document/server-docs/authentication-management/access-token/tenant_access_token_internal
package feishu
import "time"
// Config holds Feishu-specific configuration for the data source connector.
// Uses the self-built app (企业自建应用) authentication model.
type Config struct {
// App ID from Feishu developer console
AppID string `json:"app_id"`
// App Secret from Feishu developer console
AppSecret string `json:"app_secret"`
// Base URL for Feishu API (default: https://open.feishu.cn)
// Use https://open.larksuite.com for Lark (international) deployments
BaseURL string `json:"base_url,omitempty"`
}
// DefaultBaseURL is the default Feishu Open Platform API base URL.
const DefaultBaseURL = "https://open.feishu.cn"
// LarkBaseURL is the Lark (international) API base URL.
const LarkBaseURL = "https://open.larksuite.com"
// GetBaseURL returns the effective base URL, defaulting to Feishu if not set.
func (c *Config) GetBaseURL() string {
if c.BaseURL != "" {
return c.BaseURL
}
return DefaultBaseURL
}
// --- Export format constants ---
// Used by the export task API: POST /drive/v1/export_tasks
const (
// ExportTypeDocx exports Feishu documents to .docx format.
ExportTypeDocx = "docx"
// ExportTypeXlsx exports spreadsheets / bitable to .xlsx format.
ExportTypeXlsx = "xlsx"
// ExportTypePDF exports documents to .pdf format (fallback).
ExportTypePDF = "pdf"
)
// objTypeToExportFileExtension maps Feishu obj_type to the best export file_extension.
var objTypeToExportFileExtension = map[string]string{
"docx": ExportTypeDocx,
"doc": ExportTypeDocx,
"sheet": ExportTypeXlsx,
"bitable": ExportTypeXlsx,
}
// objTypeToExportType maps Feishu obj_type to the export API "type" parameter.
// See: https://open.feishu.cn/document/server-docs/docs/drive-v1/export_task/create
var objTypeToExportType = map[string]string{
"docx": "docx",
"doc": "doc",
"sheet": "sheet",
"bitable": "bitable",
}
// exportFileExtToSuffix maps export file_extension to the file suffix for FileName.
var exportFileExtToSuffix = map[string]string{
ExportTypeDocx: ".docx",
ExportTypeXlsx: ".xlsx",
ExportTypePDF: ".pdf",
}
// --- Feishu API response structures ---
// apiResponse is the common Feishu API response wrapper.
type apiResponse struct {
Code int `json:"code"`
Msg string `json:"msg"`
}
// tokenResponse is the response for tenant_access_token API.
type tokenResponse struct {
apiResponse
TenantAccessToken string `json:"tenant_access_token"`
Expire int `json:"expire"` // seconds
}
// wikiSpaceListResponse is the response for GET /open-apis/wiki/v2/spaces.
type wikiSpaceListResponse struct {
apiResponse
Data struct {
Items []wikiSpace `json:"items"`
HasMore bool `json:"has_more"`
PageToken string `json:"page_token"`
} `json:"data"`
}
// wikiSpace represents a Feishu Wiki space.
type wikiSpace struct {
SpaceID string `json:"space_id"`
Name string `json:"name"`
Description string `json:"description"`
Visibility string `json:"visibility"` // "public" or "private"
}
// wikiNodeListResponse is the response for GET /open-apis/wiki/v2/spaces/:space_id/nodes.
type wikiNodeListResponse struct {
apiResponse
Data struct {
Items []wikiNode `json:"items"`
HasMore bool `json:"has_more"`
PageToken string `json:"page_token"`
} `json:"data"`
}
// wikiNode represents a node (document or folder) in a Feishu Wiki space.
type wikiNode struct {
SpaceID string `json:"space_id"`
NodeToken string `json:"node_token"`
ObjToken string `json:"obj_token"` // document token
ObjType string `json:"obj_type"` // "doc", "sheet", "mindnote", "bitable", "file", "docx", "slides"
ParentNodeID string `json:"parent_node_id"`
NodeType string `json:"node_type"` // "origin" or "shortcut"
OriginNodeID string `json:"origin_node_id"`
OriginSpaceID string `json:"origin_space_id"`
HasChild bool `json:"has_child"`
Title string `json:"title"`
Creator string `json:"creator"`
Owner string `json:"owner"`
ObjCreateTime string `json:"obj_create_time"` // document creation time (unix timestamp string)
ObjEditTime string `json:"obj_edit_time"` // document last edit time (unix timestamp string) — tracks content changes
NodeCreateTime string `json:"node_create_time"` // node creation time (unix timestamp string)
NodeEditTime string `json:"node_edit_time"` // node edit time (unix timestamp string) — only tracks node attribute changes
}
// wikiNodeInfoResponse is the response for GET /open-apis/wiki/v2/spaces/get_node.
type wikiNodeInfoResponse struct {
apiResponse
Data struct {
Node wikiNode `json:"node"`
} `json:"data"`
}
// --- Export task API responses ---
// docRawContentResponse is the response for GET /open-apis/docx/v1/documents/:document_id/raw_content.
// Deprecated: prefer export API for full-fidelity document export.
type docRawContentResponse struct {
apiResponse
Data struct {
Content string `json:"content"`
} `json:"data"`
}
// exportTaskCreateResponse is the response for POST /drive/v1/export_tasks.
type exportTaskCreateResponse struct {
apiResponse
Data struct {
Ticket string `json:"ticket"`
} `json:"data"`
}
// exportTaskStatusResponse is the response for GET /drive/v1/export_tasks/{ticket}.
type exportTaskStatusResponse struct {
apiResponse
Data struct {
Result struct {
FileToken string `json:"file_token"`
FileSize int64 `json:"file_size"`
// JobStatus: 0=success, 1=initializing, 2=processing
JobStatus int `json:"job_status"`
JobErrorMsg string `json:"job_error_msg"`
FileName string `json:"file_name"`
} `json:"result"`
} `json:"data"`
}
// --- File download response ---
// driveFileMetaResponse is the response for GET /drive/v1/metas for file type nodes.
type driveFileMetaResponse struct {
apiResponse
Data struct {
Metas []struct {
DocToken string `json:"doc_token"`
DocType string `json:"doc_type"`
Title string `json:"title"`
} `json:"metas"`
} `json:"data"`
}
// feishuCursor stores incremental sync state for Feishu.
type feishuCursor struct {
// LastSyncTime is the timestamp of the last successful sync.
LastSyncTime time.Time `json:"last_sync_time"`
// SpaceNodeTimes maps space_id -> node_token -> last known edit time.
// Used to detect which nodes have changed since last sync.
SpaceNodeTimes map[string]map[string]string `json:"space_node_times,omitempty"`
}
+32
View File
@@ -0,0 +1,32 @@
package datasource
import "errors"
// Error definitions for datasource operations
var (
// Connector errors
ErrConnectorNil = errors.New("connector is nil")
ErrConnectorTypeEmpty = errors.New("connector type is empty")
ErrConnectorNotFound = errors.New("connector type not found in registry")
// DataSource errors
ErrDataSourceNotFound = errors.New("data source not found")
ErrDataSourceInvalid = errors.New("data source configuration is invalid")
ErrDataSourceNotActive = errors.New("data source is not active")
// Configuration errors
ErrInvalidConfig = errors.New("invalid configuration")
ErrInvalidCredentials = errors.New("invalid credentials")
// Sync errors
ErrSyncFailed = errors.New("sync operation failed")
ErrSyncCanceled = errors.New("sync operation was canceled")
ErrFetchFailed = errors.New("failed to fetch items from source")
ErrResourceNotFound = errors.New("resource not found in source system")
// Knowledge base errors
ErrKnowledgeBaseNotFound = errors.New("knowledge base not found")
// Sync log errors
ErrSyncLogNotFound = errors.New("sync log not found")
)
+208
View File
@@ -0,0 +1,208 @@
package datasource
import (
"context"
"encoding/json"
"fmt"
"sync"
"time"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/hibiken/asynq"
"github.com/robfig/cron/v3"
)
// Scheduler manages cron-based periodic sync for data sources.
//
// robfig/cron fires at absolute wall-clock times (e.g. "0 0 * * * *" always fires
// at the top of every hour regardless of when the process started). So multiple
// instances will fire at the same moment. Dedup is handled by two layers:
//
// 1. HasRunningSync — if a previous sync is still running, skip (prevent overlap).
// 2. asynq.TaskID — deterministic ID per (dataSourceID, minute). Redis ensures
// only one task with a given ID is enqueued. Losers get ErrTaskIDConflict.
type Scheduler struct {
cron *cron.Cron
dsRepo interfaces.DataSourceRepository
syncLogRepo interfaces.SyncLogRepository
taskEnqueuer interfaces.TaskEnqueuer
mu sync.Mutex
entries map[string]cron.EntryID // dataSourceID → cron entry ID
}
// NewScheduler creates a new Scheduler.
func NewScheduler(
dsRepo interfaces.DataSourceRepository,
syncLogRepo interfaces.SyncLogRepository,
taskEnqueuer interfaces.TaskEnqueuer,
) *Scheduler {
return &Scheduler{
cron: cron.New(cron.WithSeconds(), cron.WithChain(
cron.Recover(cron.DefaultLogger),
)),
dsRepo: dsRepo,
syncLogRepo: syncLogRepo,
taskEnqueuer: taskEnqueuer,
entries: make(map[string]cron.EntryID),
}
}
// Start loads all active data sources from the database and registers their
// cron schedules. Then starts the cron runner in the background.
func (s *Scheduler) Start(ctx context.Context) error {
dataSources, err := s.dsRepo.FindActive(ctx)
if err != nil {
return fmt.Errorf("load active data sources: %w", err)
}
for _, ds := range dataSources {
if ds.SyncSchedule == "" {
continue
}
if err := s.addEntry(ds); err != nil {
logger.Warnf(ctx, "[Scheduler] failed to register cron for ds=%s schedule=%q: %v",
ds.ID, ds.SyncSchedule, err)
}
}
s.cron.Start()
logger.Infof(ctx, "[Scheduler] started with %d cron entries", len(s.entries))
return nil
}
// Stop gracefully stops the cron runner and waits for running jobs to finish.
func (s *Scheduler) Stop() {
ctx := s.cron.Stop()
<-ctx.Done()
}
// AddOrUpdate registers (or re-registers) a cron entry for the given data source.
func (s *Scheduler) AddOrUpdate(ds *types.DataSource) error {
s.mu.Lock()
defer s.mu.Unlock()
if entryID, ok := s.entries[ds.ID]; ok {
s.cron.Remove(entryID)
delete(s.entries, ds.ID)
}
if ds.Status != types.DataSourceStatusActive || ds.SyncSchedule == "" {
return nil
}
return s.addEntryLocked(ds)
}
// Remove removes the cron entry for a data source.
func (s *Scheduler) Remove(dataSourceID string) {
s.mu.Lock()
defer s.mu.Unlock()
if entryID, ok := s.entries[dataSourceID]; ok {
s.cron.Remove(entryID)
delete(s.entries, dataSourceID)
}
}
func (s *Scheduler) addEntry(ds *types.DataSource) error {
s.mu.Lock()
defer s.mu.Unlock()
return s.addEntryLocked(ds)
}
func (s *Scheduler) addEntryLocked(ds *types.DataSource) error {
dsID := ds.ID
tenantID := ds.TenantID
entryID, err := s.cron.AddFunc(ds.SyncSchedule, func() {
s.triggerSync(dsID, tenantID)
})
if err != nil {
return fmt.Errorf("invalid cron expression %q: %w", ds.SyncSchedule, err)
}
s.entries[dsID] = entryID
return nil
}
// triggerSync is called by the cron runner on each tick.
//
// Layer 1 — DB: if a previous sync is still running, skip. This prevents
// overlap when a sync takes longer than the cron interval.
//
// Layer 2 — Redis: deterministic asynq.TaskID = "dssync:<dsID>:<minute>".
// Since robfig/cron fires at absolute wall-clock times, all instances trigger
// at the same minute. The first Enqueue wins; others get ErrTaskIDConflict.
func (s *Scheduler) triggerSync(dataSourceID string, tenantID uint64) {
ctx := context.Background()
ds, err := s.dsRepo.FindByID(ctx, dataSourceID)
if err != nil || ds == nil || ds.Status != types.DataSourceStatusActive {
logger.Infof(ctx, "[Scheduler] skipping sync for ds=%s (not active or not found)", dataSourceID)
return
}
// Layer 1: prevent overlap with a still-running sync
if running, _ := s.syncLogRepo.HasRunningSync(ctx, dataSourceID); running {
logger.Infof(ctx, "[Scheduler] skipping sync for ds=%s (previous sync still running)", dataSourceID)
return
}
syncLog := &types.SyncLog{
DataSourceID: dataSourceID,
TenantID: tenantID,
Status: types.SyncLogStatusRunning,
StartedAt: time.Now(),
}
if err := s.syncLogRepo.Create(ctx, syncLog); err != nil {
logger.Errorf(ctx, "[Scheduler] failed to create sync log for ds=%s: %v", dataSourceID, err)
return
}
payload := &types.DataSourceSyncPayload{
DataSourceID: dataSourceID,
TenantID: tenantID,
SyncLogID: syncLog.ID,
ForceFull: false,
}
payloadJSON, _ := json.Marshal(payload)
task := asynq.NewTask(types.TypeDataSourceSync, payloadJSON)
// Layer 2: deterministic TaskID — all instances in the same minute produce the same ID
taskID := fmt.Sprintf("dssync:%s:%s", dataSourceID, time.Now().UTC().Truncate(time.Minute).Format("200601021504"))
_, err = s.taskEnqueuer.Enqueue(task,
asynq.Queue("low"),
asynq.TaskID(taskID),
)
if err != nil {
if err == asynq.ErrTaskIDConflict {
logger.Infof(ctx, "[Scheduler] sync already enqueued by another instance for ds=%s", dataSourceID)
syncLog.Status = types.SyncLogStatusCanceled
now := time.Now()
syncLog.FinishedAt = &now
syncLog.ErrorMessage = "deduplicated: another instance enqueued first"
_ = s.syncLogRepo.Update(ctx, syncLog)
return
}
logger.Errorf(ctx, "[Scheduler] failed to enqueue sync task for ds=%s: %v", dataSourceID, err)
syncLog.Status = types.SyncLogStatusFailed
now := time.Now()
syncLog.FinishedAt = &now
syncLog.ErrorMessage = fmt.Sprintf("enqueue failed: %v", err)
_ = s.syncLogRepo.Update(ctx, syncLog)
return
}
logger.Infof(ctx, "[Scheduler] sync task enqueued for ds=%s syncLog=%s", dataSourceID, syncLog.ID)
}
// EntryCount returns the number of active cron entries (for testing/monitoring).
func (s *Scheduler) EntryCount() int {
s.mu.Lock()
defer s.mu.Unlock()
return len(s.entries)
}
+356
View File
@@ -0,0 +1,356 @@
package datasource
import (
"context"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/Tencent/WeKnora/internal/types"
"github.com/hibiken/asynq"
)
// ──────────────────────────────────────────────────────────────────────
// Fake implementations for testing
// ──────────────────────────────────────────────────────────────────────
// fakeDataSourceRepo is an in-memory DataSourceRepository.
type fakeDataSourceRepo struct {
mu sync.Mutex
dataSources map[string]*types.DataSource
}
func newFakeDataSourceRepo() *fakeDataSourceRepo {
return &fakeDataSourceRepo{dataSources: make(map[string]*types.DataSource)}
}
func (r *fakeDataSourceRepo) Create(_ context.Context, ds *types.DataSource) error {
r.mu.Lock()
defer r.mu.Unlock()
r.dataSources[ds.ID] = ds
return nil
}
func (r *fakeDataSourceRepo) FindByID(_ context.Context, id string) (*types.DataSource, error) {
r.mu.Lock()
defer r.mu.Unlock()
ds, ok := r.dataSources[id]
if !ok {
return nil, ErrDataSourceNotFound
}
return ds, nil
}
func (r *fakeDataSourceRepo) FindByKnowledgeBase(_ context.Context, kbID string) ([]*types.DataSource, error) {
return nil, nil
}
func (r *fakeDataSourceRepo) Update(_ context.Context, ds *types.DataSource) error {
r.mu.Lock()
defer r.mu.Unlock()
r.dataSources[ds.ID] = ds
return nil
}
func (r *fakeDataSourceRepo) Delete(_ context.Context, id string) error {
r.mu.Lock()
defer r.mu.Unlock()
delete(r.dataSources, id)
return nil
}
func (r *fakeDataSourceRepo) FindActive(_ context.Context) ([]*types.DataSource, error) {
r.mu.Lock()
defer r.mu.Unlock()
var result []*types.DataSource
for _, ds := range r.dataSources {
if ds.Status == types.DataSourceStatusActive && ds.SyncSchedule != "" {
result = append(result, ds)
}
}
return result, nil
}
// fakeSyncLogRepo is an in-memory SyncLogRepository.
type fakeSyncLogRepo struct {
mu sync.Mutex
logs map[string]*types.SyncLog
}
func newFakeSyncLogRepo() *fakeSyncLogRepo {
return &fakeSyncLogRepo{logs: make(map[string]*types.SyncLog)}
}
func (r *fakeSyncLogRepo) Create(_ context.Context, log *types.SyncLog) error {
r.mu.Lock()
defer r.mu.Unlock()
if log.ID == "" {
log.ID = "log-" + time.Now().Format("150405.000")
}
r.logs[log.ID] = log
return nil
}
func (r *fakeSyncLogRepo) FindByID(_ context.Context, id string) (*types.SyncLog, error) {
r.mu.Lock()
defer r.mu.Unlock()
l, ok := r.logs[id]
if !ok {
return nil, ErrSyncLogNotFound
}
return l, nil
}
func (r *fakeSyncLogRepo) FindByDataSource(_ context.Context, dsID string, limit, offset int) ([]*types.SyncLog, error) {
return nil, nil
}
func (r *fakeSyncLogRepo) FindLatest(_ context.Context, dsID string) (*types.SyncLog, error) {
return nil, nil
}
func (r *fakeSyncLogRepo) Update(_ context.Context, log *types.SyncLog) error {
r.mu.Lock()
defer r.mu.Unlock()
r.logs[log.ID] = log
return nil
}
func (r *fakeSyncLogRepo) CleanupOldLogs(_ context.Context, retentionDays int) error {
return nil
}
func (r *fakeSyncLogRepo) HasRunningSync(_ context.Context, dsID string) (bool, error) {
r.mu.Lock()
defer r.mu.Unlock()
for _, log := range r.logs {
if log.DataSourceID == dsID && log.Status == types.SyncLogStatusRunning {
return true, nil
}
}
return false, nil
}
// fakeTaskEnqueuer counts how many tasks are enqueued.
type fakeTaskEnqueuer struct {
count atomic.Int64
}
func (e *fakeTaskEnqueuer) Enqueue(task *asynq.Task, opts ...asynq.Option) (*asynq.TaskInfo, error) {
e.count.Add(1)
return &asynq.TaskInfo{ID: "task-fake"}, nil
}
// ──────────────────────────────────────────────────────────────────────
// Tests
// ──────────────────────────────────────────────────────────────────────
func TestScheduler_StartWithActiveDataSources(t *testing.T) {
repo := newFakeDataSourceRepo()
_ = repo.Create(context.Background(), &types.DataSource{
ID: "ds-1",
TenantID: 1,
Status: types.DataSourceStatusActive,
SyncSchedule: "*/2 * * * * *", // every 2 seconds (6-field cron with seconds)
})
_ = repo.Create(context.Background(), &types.DataSource{
ID: "ds-2",
TenantID: 1,
Status: types.DataSourceStatusPaused, // should NOT be scheduled
SyncSchedule: "*/2 * * * * *",
})
enqueuer := &fakeTaskEnqueuer{}
scheduler := NewScheduler(repo, newFakeSyncLogRepo(), enqueuer)
if err := scheduler.Start(context.Background()); err != nil {
t.Fatalf("Start() error: %v", err)
}
defer scheduler.Stop()
// Only ds-1 should be registered (ds-2 is paused, not returned by FindActive)
if scheduler.EntryCount() != 1 {
t.Errorf("EntryCount() = %d, want 1", scheduler.EntryCount())
}
}
func TestScheduler_CronFires(t *testing.T) {
repo := newFakeDataSourceRepo()
_ = repo.Create(context.Background(), &types.DataSource{
ID: "ds-fire",
TenantID: 1,
Status: types.DataSourceStatusActive,
SyncSchedule: "* * * * * *", // every second
})
enqueuer := &fakeTaskEnqueuer{}
scheduler := NewScheduler(repo, newFakeSyncLogRepo(), enqueuer)
if err := scheduler.Start(context.Background()); err != nil {
t.Fatalf("Start() error: %v", err)
}
// Wait for at least one tick
time.Sleep(2500 * time.Millisecond)
scheduler.Stop()
if enqueuer.count.Load() == 0 {
t.Error("expected at least 1 enqueue, got 0")
}
}
func TestScheduler_AddOrUpdate(t *testing.T) {
enqueuer := &fakeTaskEnqueuer{}
scheduler := NewScheduler(newFakeDataSourceRepo(), newFakeSyncLogRepo(), enqueuer)
scheduler.cron.Start()
defer scheduler.Stop()
ds := &types.DataSource{
ID: "ds-new",
TenantID: 1,
Status: types.DataSourceStatusActive,
SyncSchedule: "0 0 * * * *", // every hour
}
// Add
if err := scheduler.AddOrUpdate(ds); err != nil {
t.Fatalf("AddOrUpdate() error: %v", err)
}
if scheduler.EntryCount() != 1 {
t.Errorf("EntryCount() = %d, want 1", scheduler.EntryCount())
}
// Update schedule
ds.SyncSchedule = "0 30 * * * *" // every half hour
if err := scheduler.AddOrUpdate(ds); err != nil {
t.Fatalf("AddOrUpdate() (update) error: %v", err)
}
if scheduler.EntryCount() != 1 {
t.Errorf("after update: EntryCount() = %d, want 1 (should replace, not add)", scheduler.EntryCount())
}
}
func TestScheduler_AddOrUpdate_PausedIsNoop(t *testing.T) {
enqueuer := &fakeTaskEnqueuer{}
scheduler := NewScheduler(newFakeDataSourceRepo(), newFakeSyncLogRepo(), enqueuer)
scheduler.cron.Start()
defer scheduler.Stop()
ds := &types.DataSource{
ID: "ds-paused",
TenantID: 1,
Status: types.DataSourceStatusPaused,
SyncSchedule: "0 0 * * * *",
}
if err := scheduler.AddOrUpdate(ds); err != nil {
t.Fatalf("AddOrUpdate() error: %v", err)
}
if scheduler.EntryCount() != 0 {
t.Errorf("paused ds should not be scheduled, EntryCount() = %d", scheduler.EntryCount())
}
}
func TestScheduler_AddOrUpdate_EmptyScheduleIsNoop(t *testing.T) {
enqueuer := &fakeTaskEnqueuer{}
scheduler := NewScheduler(newFakeDataSourceRepo(), newFakeSyncLogRepo(), enqueuer)
scheduler.cron.Start()
defer scheduler.Stop()
ds := &types.DataSource{
ID: "ds-no-sched",
TenantID: 1,
Status: types.DataSourceStatusActive,
SyncSchedule: "",
}
if err := scheduler.AddOrUpdate(ds); err != nil {
t.Fatalf("AddOrUpdate() error: %v", err)
}
if scheduler.EntryCount() != 0 {
t.Errorf("empty schedule should not be registered, EntryCount() = %d", scheduler.EntryCount())
}
}
func TestScheduler_Remove(t *testing.T) {
enqueuer := &fakeTaskEnqueuer{}
scheduler := NewScheduler(newFakeDataSourceRepo(), newFakeSyncLogRepo(), enqueuer)
scheduler.cron.Start()
defer scheduler.Stop()
ds := &types.DataSource{
ID: "ds-rm",
TenantID: 1,
Status: types.DataSourceStatusActive,
SyncSchedule: "0 0 * * * *",
}
_ = scheduler.AddOrUpdate(ds)
if scheduler.EntryCount() != 1 {
t.Fatalf("pre-remove: EntryCount() = %d, want 1", scheduler.EntryCount())
}
scheduler.Remove("ds-rm")
if scheduler.EntryCount() != 0 {
t.Errorf("post-remove: EntryCount() = %d, want 0", scheduler.EntryCount())
}
// Remove non-existent is safe
scheduler.Remove("does-not-exist")
}
func TestScheduler_InvalidCron(t *testing.T) {
enqueuer := &fakeTaskEnqueuer{}
scheduler := NewScheduler(newFakeDataSourceRepo(), newFakeSyncLogRepo(), enqueuer)
scheduler.cron.Start()
defer scheduler.Stop()
ds := &types.DataSource{
ID: "ds-bad",
TenantID: 1,
Status: types.DataSourceStatusActive,
SyncSchedule: "not a cron",
}
err := scheduler.AddOrUpdate(ds)
if err == nil {
t.Fatal("expected error for invalid cron expression")
}
if scheduler.EntryCount() != 0 {
t.Errorf("invalid cron should not be registered, EntryCount() = %d", scheduler.EntryCount())
}
}
func TestScheduler_TriggerSync_InactiveSkipped(t *testing.T) {
repo := newFakeDataSourceRepo()
// Create a data source that is paused
_ = repo.Create(context.Background(), &types.DataSource{
ID: "ds-inactive",
TenantID: 1,
Status: types.DataSourceStatusPaused,
})
enqueuer := &fakeTaskEnqueuer{}
scheduler := NewScheduler(repo, newFakeSyncLogRepo(), enqueuer)
// Directly call triggerSync — it should skip because ds is not active
scheduler.triggerSync("ds-inactive", 1)
if enqueuer.count.Load() != 0 {
t.Error("should not enqueue for inactive data source")
}
}
func TestScheduler_TriggerSync_NotFound(t *testing.T) {
repo := newFakeDataSourceRepo()
enqueuer := &fakeTaskEnqueuer{}
scheduler := NewScheduler(repo, newFakeSyncLogRepo(), enqueuer)
// Should not panic, just skip
scheduler.triggerSync("nonexistent", 1)
if enqueuer.count.Load() != 0 {
t.Error("should not enqueue for non-existent data source")
}
}