feat(datasource): implement data source management functionality
- Added a new API for managing data sources, including CRUD operations and connection validation. - Introduced UI components for data source settings, sync logs, and editor dialog, enhancing user experience in managing external data sources. - Updated localization files to support new data source features in multiple languages. - Implemented cron expression humanization for better scheduling visibility. - Enhanced knowledge base editor to include data source management options.
This commit is contained in:
@@ -0,0 +1,497 @@
|
||||
# Adding a New Connector - Step-by-Step Guide
|
||||
|
||||
This guide walks you through implementing a new connector (e.g., Feishu, Notion, Confluence) for the data source sync framework.
|
||||
|
||||
## Overview
|
||||
|
||||
A connector is an adapter that translates between WeKnora's data model and an external platform's API. It handles:
|
||||
- Connection validation (credentials, connectivity)
|
||||
- Resource listing (documents, spaces, folders the user can choose from)
|
||||
- Full sync (fetch all items from selected resources)
|
||||
- Incremental sync (fetch only changed items since last sync)
|
||||
|
||||
## Step 1: Create Connector Package Structure
|
||||
|
||||
```bash
|
||||
mkdir -p internal/datasource/connector/yourtype/
|
||||
```
|
||||
|
||||
Create three files:
|
||||
- `client.go` - API client wrapper
|
||||
- `connector.go` - Implements Connector interface
|
||||
- `types.go` - Platform-specific data structures
|
||||
|
||||
## Step 2: Define Platform Types (types.go)
|
||||
|
||||
```go
|
||||
package yourtype
|
||||
|
||||
import "time"
|
||||
|
||||
// Platform-specific configuration
|
||||
type Config struct {
|
||||
BaseURL string `json:"base_url"`
|
||||
APIToken string `json:"api_token"`
|
||||
|
||||
// Or OAuth fields:
|
||||
AccessToken string `json:"access_token"`
|
||||
RefreshToken string `json:"refresh_token"`
|
||||
ExpiresAt time.Time `json:"expires_at"`
|
||||
}
|
||||
|
||||
// Platform-specific resource representation
|
||||
type YourResource struct {
|
||||
ID string
|
||||
Name string
|
||||
Type string // "document", "folder", "space", etc.
|
||||
ModifiedAt time.Time
|
||||
URL string
|
||||
}
|
||||
|
||||
// Platform-specific item representation
|
||||
type YourItem struct {
|
||||
ID string
|
||||
Title string
|
||||
Content string
|
||||
ContentHTML string
|
||||
ModifiedAt time.Time
|
||||
URL string
|
||||
CreatedBy string
|
||||
}
|
||||
|
||||
// Platform-specific pagination/cursor
|
||||
type YourCursor struct {
|
||||
Offset int `json:"offset,omitempty"`
|
||||
LastModified time.Time `json:"last_modified,omitempty"`
|
||||
PageToken string `json:"page_token,omitempty"`
|
||||
}
|
||||
```
|
||||
|
||||
## Step 3: Implement API Client (client.go)
|
||||
|
||||
```go
|
||||
package yourtype
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"encoding/json"
|
||||
)
|
||||
|
||||
type Client struct {
|
||||
baseURL string
|
||||
apiToken string
|
||||
httpClient *http.Client
|
||||
}
|
||||
|
||||
// NewClient creates a new API client
|
||||
func NewClient(config *Config) *Client {
|
||||
return &Client{
|
||||
baseURL: config.BaseURL,
|
||||
apiToken: config.APIToken,
|
||||
httpClient: &http.Client{Timeout: 30 * time.Second},
|
||||
}
|
||||
}
|
||||
|
||||
// Example methods
|
||||
func (c *Client) GetResources(ctx context.Context) ([]YourResource, error) {
|
||||
// Call platform API
|
||||
// Parse response
|
||||
// Return resources
|
||||
}
|
||||
|
||||
func (c *Client) GetDocument(ctx context.Context, docID string) (*YourItem, error) {
|
||||
// Fetch single document
|
||||
}
|
||||
|
||||
func (c *Client) GetDocumentsModifiedSince(ctx context.Context, since time.Time) ([]YourItem, error) {
|
||||
// Fetch documents modified since timestamp
|
||||
}
|
||||
```
|
||||
|
||||
## Step 4: Implement Connector Interface (connector.go)
|
||||
|
||||
```go
|
||||
package yourtype
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
type YourConnector struct {
|
||||
client *Client
|
||||
}
|
||||
|
||||
// NewConnector creates a new connector
|
||||
func NewConnector() *YourConnector {
|
||||
return &YourConnector{}
|
||||
}
|
||||
|
||||
// Type returns the connector type identifier
|
||||
func (c *YourConnector) Type() string {
|
||||
return types.ConnectorTypeYourType // Must match constant in types/datasource.go
|
||||
}
|
||||
|
||||
// Validate verifies that the configuration is valid
|
||||
func (c *YourConnector) Validate(ctx context.Context, config *types.DataSourceConfig) error {
|
||||
if config == nil {
|
||||
return fmt.Errorf("config is nil")
|
||||
}
|
||||
|
||||
// Parse your type-specific config
|
||||
yourConfig := &Config{}
|
||||
if err := parseConfig(config, yourConfig); err != nil {
|
||||
return fmt.Errorf("invalid config: %w", err)
|
||||
}
|
||||
|
||||
// Create client
|
||||
client := NewClient(yourConfig)
|
||||
|
||||
// Test connection
|
||||
_, err := client.GetResources(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("connection failed: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// ListResources lists available resources (documents, spaces, folders)
|
||||
func (c *YourConnector) ListResources(ctx context.Context, config *types.DataSourceConfig) ([]types.Resource, error) {
|
||||
yourConfig := &Config{}
|
||||
if err := parseConfig(config, yourConfig); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
client := NewClient(yourConfig)
|
||||
yourResources, err := client.GetResources(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Convert to WeKnora Resource format
|
||||
resources := make([]types.Resource, len(yourResources))
|
||||
for i, yr := range yourResources {
|
||||
resources[i] = types.Resource{
|
||||
ExternalID: yr.ID,
|
||||
Name: yr.Name,
|
||||
Type: yr.Type,
|
||||
URL: yr.URL,
|
||||
ModifiedAt: yr.ModifiedAt,
|
||||
}
|
||||
}
|
||||
|
||||
return resources, nil
|
||||
}
|
||||
|
||||
// FetchAll performs a full sync
|
||||
func (c *YourConnector) FetchAll(ctx context.Context, config *types.DataSourceConfig, resourceIDs []string) ([]types.FetchedItem, error) {
|
||||
yourConfig := &Config{}
|
||||
if err := parseConfig(config, yourConfig); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
client := NewClient(yourConfig)
|
||||
|
||||
var allItems []types.FetchedItem
|
||||
|
||||
// Fetch all documents from specified resources
|
||||
for _, resourceID := range resourceIDs {
|
||||
// Get documents from this resource (implementation depends on platform)
|
||||
yourItems, err := client.GetDocumentsFromResource(ctx, resourceID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to fetch resource %s: %w", resourceID, err)
|
||||
}
|
||||
|
||||
// Convert to FetchedItem format
|
||||
for _, yi := range yourItems {
|
||||
item := types.FetchedItem{
|
||||
ExternalID: yi.ID,
|
||||
Title: yi.Title,
|
||||
Content: []byte(yi.Content),
|
||||
ContentType: "text/markdown",
|
||||
FileName: fmt.Sprintf("%s.md", yi.Title),
|
||||
URL: yi.URL,
|
||||
UpdatedAt: yi.ModifiedAt,
|
||||
SourceResourceID: resourceID,
|
||||
Metadata: map[string]string{
|
||||
"created_by": yi.CreatedBy,
|
||||
"platform": "yourtype",
|
||||
},
|
||||
}
|
||||
allItems = append(allItems, item)
|
||||
}
|
||||
}
|
||||
|
||||
return allItems, nil
|
||||
}
|
||||
|
||||
// FetchIncremental performs an incremental sync
|
||||
func (c *YourConnector) FetchIncremental(ctx context.Context, config *types.DataSourceConfig, cursor *types.SyncCursor) ([]types.FetchedItem, *types.SyncCursor, error) {
|
||||
yourConfig := &Config{}
|
||||
if err := parseConfig(config, yourConfig); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
client := NewClient(yourConfig)
|
||||
|
||||
// Determine start time for incremental fetch
|
||||
var sinceTime time.Time
|
||||
if cursor != nil && !cursor.LastSyncTime.IsZero() {
|
||||
sinceTime = cursor.LastSyncTime
|
||||
} else {
|
||||
sinceTime = time.Now().AddDate(0, 0, -7) // Default: last 7 days
|
||||
}
|
||||
|
||||
// Fetch changed items
|
||||
yourItems, err := client.GetDocumentsModifiedSince(ctx, sinceTime)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("incremental fetch failed: %w", err)
|
||||
}
|
||||
|
||||
// Convert to FetchedItem format
|
||||
items := make([]types.FetchedItem, len(yourItems))
|
||||
for i, yi := range yourItems {
|
||||
items[i] = types.FetchedItem{
|
||||
ExternalID: yi.ID,
|
||||
Title: yi.Title,
|
||||
Content: []byte(yi.Content),
|
||||
ContentType: "text/markdown",
|
||||
FileName: fmt.Sprintf("%s.md", yi.Title),
|
||||
URL: yi.URL,
|
||||
UpdatedAt: yi.ModifiedAt,
|
||||
Metadata: map[string]string{
|
||||
"created_by": yi.CreatedBy,
|
||||
"platform": "yourtype",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// Create new cursor for next sync
|
||||
nextCursor := &types.SyncCursor{
|
||||
LastSyncTime: time.Now(),
|
||||
ConnectorCursor: map[string]interface{}{
|
||||
"last_modified": time.Now(),
|
||||
},
|
||||
}
|
||||
|
||||
return items, nextCursor, nil
|
||||
}
|
||||
|
||||
// Helper function to parse config
|
||||
func parseConfig(config *types.DataSourceConfig, target interface{}) error {
|
||||
data, err := json.Marshal(config.Credentials)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return json.Unmarshal(data, target)
|
||||
}
|
||||
```
|
||||
|
||||
## Step 5: Register in Container
|
||||
|
||||
Edit `internal/container/container.go`:
|
||||
|
||||
```go
|
||||
import (
|
||||
// ... existing imports
|
||||
yourconnector "github.com/Tencent/WeKnora/internal/datasource/connector/yourtype"
|
||||
)
|
||||
|
||||
func setupContainer() (*dig.Container, error) {
|
||||
container := dig.New()
|
||||
|
||||
// ... existing registrations ...
|
||||
|
||||
// Register your connector
|
||||
must(container.Provide(func() *datasource.Connector {
|
||||
var c datasource.Connector = yourconnector.NewConnector()
|
||||
return &c
|
||||
}))
|
||||
|
||||
// ... rest of setup ...
|
||||
}
|
||||
```
|
||||
|
||||
Better yet, register through the registry in the connector setup:
|
||||
|
||||
```go
|
||||
// In the service initialization section
|
||||
connectorRegistry := datasource.NewConnectorRegistry()
|
||||
connectorRegistry.Register(yourconnector.NewConnector())
|
||||
connectorRegistry.Register(feishuconnector.NewConnector())
|
||||
// ... etc
|
||||
|
||||
container.Provide(func() *datasource.ConnectorRegistry {
|
||||
return connectorRegistry
|
||||
})
|
||||
```
|
||||
|
||||
## Step 6: Add Connector Type Constant
|
||||
|
||||
Edit `internal/types/datasource.go`:
|
||||
|
||||
```go
|
||||
const (
|
||||
// ... existing types ...
|
||||
ConnectorTypeYourType = "yourtype"
|
||||
)
|
||||
```
|
||||
|
||||
## Step 7: Add Metadata
|
||||
|
||||
Edit `internal/datasource/connector.go`:
|
||||
|
||||
```go
|
||||
var ConnectorMetadataRegistry = map[string]ConnectorMetadata{
|
||||
// ... existing entries ...
|
||||
types.ConnectorTypeYourType: {
|
||||
Type: types.ConnectorTypeYourType,
|
||||
Name: "Your Platform Name",
|
||||
Description: "Sync documents from Your Platform",
|
||||
Priority: X, // Lower number = higher priority in UI
|
||||
AuthType: "oauth2", // or "api_key", "token", "password"
|
||||
Capabilities: []string{"incremental", "webhook", "deletion_sync"},
|
||||
},
|
||||
}
|
||||
```
|
||||
|
||||
## Step 8: Test Your Connector
|
||||
|
||||
```go
|
||||
// Example test
|
||||
func TestYourConnectorValidate(t *testing.T) {
|
||||
connector := NewConnector()
|
||||
|
||||
config := &types.DataSourceConfig{
|
||||
Type: types.ConnectorTypeYourType,
|
||||
Credentials: map[string]interface{}{
|
||||
"api_token": "test_token",
|
||||
},
|
||||
}
|
||||
|
||||
err := connector.Validate(context.Background(), config)
|
||||
// assert no error
|
||||
}
|
||||
|
||||
func TestYourConnectorFetchAll(t *testing.T) {
|
||||
connector := NewConnector()
|
||||
config := &types.DataSourceConfig{
|
||||
Type: types.ConnectorTypeYourType,
|
||||
Credentials: map[string]interface{}{
|
||||
"api_token": "test_token",
|
||||
},
|
||||
ResourceIDs: []string{"resource_1"},
|
||||
}
|
||||
|
||||
items, err := connector.FetchAll(context.Background(), config, []string{"resource_1"})
|
||||
// assert results
|
||||
}
|
||||
```
|
||||
|
||||
## Checklist
|
||||
|
||||
- [ ] Created `internal/datasource/connector/yourtype/` package
|
||||
- [ ] Implemented `types.go` with platform data structures
|
||||
- [ ] Implemented `client.go` with API wrapper
|
||||
- [ ] Implemented `connector.go` with Connector interface
|
||||
- [ ] Added connector type constant
|
||||
- [ ] Registered in container
|
||||
- [ ] Added metadata entry
|
||||
- [ ] Added unit tests
|
||||
- [ ] Tested manually with real API
|
||||
- [ ] Documented any special requirements
|
||||
|
||||
## Common Patterns
|
||||
|
||||
### OAuth Flow
|
||||
If using OAuth, store tokens in config:
|
||||
```go
|
||||
type Config struct {
|
||||
AccessToken string
|
||||
RefreshToken string
|
||||
ExpiresAt time.Time
|
||||
}
|
||||
|
||||
// Refresh tokens when expired
|
||||
func (c *Client) ensureValidToken(ctx context.Context) error {
|
||||
if time.Now().After(c.config.ExpiresAt) {
|
||||
return c.refreshToken(ctx)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
```
|
||||
|
||||
### Pagination
|
||||
For platforms with pagination:
|
||||
```go
|
||||
func (c *Client) GetDocumentsPage(ctx context.Context, pageToken string) (*Page, error) {
|
||||
// Returns {Items, NextPageToken}
|
||||
}
|
||||
```
|
||||
|
||||
### Incremental Sync
|
||||
For timestamp-based incremental sync:
|
||||
```go
|
||||
func (c *Client) GetModifiedSince(ctx context.Context, since time.Time) ([]Item, error) {
|
||||
// Uses API parameter like &modified_after=2026-03-26T10:00:00Z
|
||||
}
|
||||
```
|
||||
|
||||
### Deletion Tracking
|
||||
For platforms that report deletions:
|
||||
```go
|
||||
type Item struct {
|
||||
IsDeleted bool // Set when item is deleted
|
||||
}
|
||||
```
|
||||
|
||||
## Testing with Real API
|
||||
|
||||
1. Set up test credentials
|
||||
2. Create small test resource (e.g., single document)
|
||||
3. Test each method:
|
||||
```go
|
||||
connector := NewConnector()
|
||||
config := &types.DataSourceConfig{...}
|
||||
|
||||
// Test Validate
|
||||
err := connector.Validate(ctx, config)
|
||||
|
||||
// Test ListResources
|
||||
resources, err := connector.ListResources(ctx, config)
|
||||
|
||||
// Test FetchAll
|
||||
items, err := connector.FetchAll(ctx, config, []string{resources[0].ExternalID})
|
||||
|
||||
// Test FetchIncremental
|
||||
items, cursor, err := connector.FetchIncremental(ctx, config, nil)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Example: Feishu Connector Reference
|
||||
|
||||
The Feishu connector would be a good first implementation since:
|
||||
1. Feishu API is well-documented
|
||||
2. WeKnora already has `internal/im/feishu/` for reference
|
||||
3. Popular in China (key market)
|
||||
4. Has webhook support for real-time sync
|
||||
|
||||
Structure:
|
||||
```
|
||||
internal/datasource/connector/feishu/
|
||||
├── client.go (Feishu API client)
|
||||
├── connector.go (Implements Connector)
|
||||
└── types.go (Feishu types)
|
||||
```
|
||||
|
||||
Key files to reference:
|
||||
- `internal/im/feishu/adapter.go` - Feishu API patterns
|
||||
- `internal/im/feishu/longconn.go` - Connection handling
|
||||
|
||||
This will be a good model for other connectors.
|
||||
@@ -0,0 +1,260 @@
|
||||
# WeKnora Data Source Sync Framework
|
||||
|
||||
## Overview
|
||||
|
||||
The data source sync framework enables WeKnora to automatically import and synchronize content from external platforms (Feishu, Notion, Confluence, etc.) into knowledge bases. This is the foundational layer upon which all specific connectors are built.
|
||||
|
||||
## Architecture
|
||||
|
||||
### Core Components
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────┐
|
||||
│ External Data Sources │
|
||||
│ (Feishu, Notion, Confluence, GitHub, Google Drive...) │
|
||||
└─────────────┬───────────────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────┐
|
||||
│ Connector Registry & Adapters │
|
||||
│ Each platform (feishu/, notion/, confluence/, etc) │
|
||||
│ implements the Connector interface │
|
||||
└─────────────┬───────────────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────┐
|
||||
│ DataSourceService (Business Logic) │
|
||||
│ ├─ CreateDataSource / UpdateDataSource │
|
||||
│ ├─ ManualSync / ListAvailableResources │
|
||||
│ ├─ ValidateConnection / PauseDataSource │
|
||||
│ └─ ProcessSync (asynq task handler) │
|
||||
└─────────────┬───────────────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────┐
|
||||
│ HTTP Handler & API Routes (/api/v1/datasource) │
|
||||
│ REST endpoints for UI/programmatic access │
|
||||
└─────────────┬───────────────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────┐
|
||||
│ Database (MySQL / PostgreSQL) │
|
||||
│ ├─ data_sources (configuration & state) │
|
||||
│ └─ sync_logs (history & metadata) │
|
||||
└─────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
### Task Flow
|
||||
|
||||
```
|
||||
Manual Trigger or Scheduled Job (cron)
|
||||
↓
|
||||
Scheduler enqueues asynq Task
|
||||
↓
|
||||
ProcessSync (task handler) executes
|
||||
↓
|
||||
Connector.Fetch* methods pull data
|
||||
↓
|
||||
Diff engine identifies changes (new/update/delete)
|
||||
↓
|
||||
KnowledgeService creates/updates Knowledge entries
|
||||
↓
|
||||
Documents parsed → chunks → vectors → indexed
|
||||
```
|
||||
|
||||
## File Structure
|
||||
|
||||
```
|
||||
internal/datasource/
|
||||
├── connector.go # Connector interface & registry
|
||||
├── errors.go # Error definitions
|
||||
└── README.md # This file
|
||||
|
||||
internal/types/
|
||||
├── datasource.go # Data models (DataSource, SyncLog, etc)
|
||||
└── interfaces/datasource.go # Service interfaces
|
||||
|
||||
internal/application/
|
||||
├── repository/datasource_repo.go # Database access layer
|
||||
└── service/datasource_service.go # Business logic
|
||||
|
||||
internal/handler/
|
||||
└── datasource.go # HTTP request handlers
|
||||
|
||||
internal/router/
|
||||
├── router.go # Route registration
|
||||
├── task.go # Asynq task registration
|
||||
└── sync_task.go # Lite mode task registration
|
||||
|
||||
migrations/versioned/
|
||||
└── 000028_datasource_tables.up/down.sql # Database schema
|
||||
```
|
||||
|
||||
## Data Models
|
||||
|
||||
### DataSource
|
||||
Represents a configured external data source:
|
||||
- **ID**: Unique identifier
|
||||
- **Type**: Connector type (feishu, notion, confluence, etc)
|
||||
- **Config**: Encrypted credentials and configuration (JSON)
|
||||
- **SyncSchedule**: Cron expression (e.g., "0 */6 * * *")
|
||||
- **SyncMode**: "incremental" or "full"
|
||||
- **Status**: active | paused | error
|
||||
- **ConflictStrategy**: overwrite | skip
|
||||
- **SyncDeletions**: Whether to sync deletions from source
|
||||
- **LastSyncAt**: Timestamp of last successful sync
|
||||
- **LastSyncCursor**: State for incremental sync
|
||||
- **LastSyncResult**: Summary of last sync
|
||||
|
||||
### SyncLog
|
||||
Tracks execution of each sync operation:
|
||||
- **Status**: running | success | partial | failed | canceled
|
||||
- **StartedAt/FinishedAt**: Execution timestamps
|
||||
- **Items***: Counters (total, created, updated, deleted, skipped, failed)
|
||||
- **ErrorMessage**: Error details if failed
|
||||
- **Result**: Detailed sync result (JSON)
|
||||
|
||||
### Connector Interface
|
||||
All external data sources implement this interface:
|
||||
|
||||
```go
|
||||
type Connector interface {
|
||||
Type() string
|
||||
Validate(ctx, config) error
|
||||
ListResources(ctx, config) ([]Resource, error)
|
||||
FetchAll(ctx, config, resourceIDs) ([]FetchedItem, error)
|
||||
FetchIncremental(ctx, config, cursor) ([]FetchedItem, *SyncCursor, error)
|
||||
}
|
||||
```
|
||||
|
||||
## API Endpoints
|
||||
|
||||
### Data Source Management
|
||||
```
|
||||
POST /api/v1/datasource # Create
|
||||
GET /api/v1/datasource # List (by kb_id)
|
||||
GET /api/v1/datasource/:id # Get
|
||||
PUT /api/v1/datasource/:id # Update
|
||||
DELETE /api/v1/datasource/:id # Delete
|
||||
```
|
||||
|
||||
### Operations
|
||||
```
|
||||
POST /api/v1/datasource/:id/validate # Test connection
|
||||
GET /api/v1/datasource/:id/resources # List available resources
|
||||
POST /api/v1/datasource/:id/sync # Trigger manual sync
|
||||
POST /api/v1/datasource/:id/pause # Pause
|
||||
POST /api/v1/datasource/:id/resume # Resume
|
||||
```
|
||||
|
||||
### Logs
|
||||
```
|
||||
GET /api/v1/datasource/:id/logs # List sync logs
|
||||
GET /api/v1/datasource/logs/:log_id # Get specific log
|
||||
```
|
||||
|
||||
### Metadata
|
||||
```
|
||||
GET /api/v1/datasource/types # Available connectors
|
||||
```
|
||||
|
||||
## Implementation Guide
|
||||
|
||||
### Adding a New Connector
|
||||
|
||||
1. **Create the connector package** in `internal/datasource/connector/<type>/`
|
||||
```go
|
||||
// connector/<type>/client.go - API client
|
||||
// connector/<type>/connector.go - Implements Connector interface
|
||||
// connector/<type>/types.go - Type-specific models
|
||||
```
|
||||
|
||||
2. **Implement the Connector interface**
|
||||
```go
|
||||
type YourConnector struct {
|
||||
// fields
|
||||
}
|
||||
|
||||
func (c *YourConnector) Type() string {
|
||||
return types.ConnectorTypeYourType
|
||||
}
|
||||
|
||||
func (c *YourConnector) Validate(ctx, config) error {
|
||||
// Test connectivity and credentials
|
||||
}
|
||||
|
||||
func (c *YourConnector) ListResources(ctx, config) ([]types.Resource, error) {
|
||||
// List available resources (documents, spaces, etc)
|
||||
}
|
||||
|
||||
func (c *YourConnector) FetchAll(ctx, config, resourceIDs) ([]types.FetchedItem, error) {
|
||||
// Full sync - fetch all items from specified resources
|
||||
}
|
||||
|
||||
func (c *YourConnector) FetchIncremental(ctx, config, cursor) ([]types.FetchedItem, *types.SyncCursor, error) {
|
||||
// Incremental sync - fetch only changed items since cursor
|
||||
}
|
||||
```
|
||||
|
||||
3. **Register the connector** in the container (in container.go)
|
||||
```go
|
||||
container.Provide(func() datasource.Connector {
|
||||
return yourconnector.NewConnector()
|
||||
})
|
||||
```
|
||||
|
||||
4. **Add metadata** in `connector.go`
|
||||
```go
|
||||
types.ConnectorTypeYourType: {
|
||||
Type: types.ConnectorTypeYourType,
|
||||
Name: "Your Platform",
|
||||
AuthType: "oauth2",
|
||||
// ...
|
||||
}
|
||||
```
|
||||
|
||||
## Database Schema
|
||||
|
||||
### data_sources table
|
||||
- Stores data source configurations
|
||||
- Indexes: tenant_id, knowledge_base_id, type, status
|
||||
- Encrypted config field (AES-256-GCM)
|
||||
|
||||
### sync_logs table
|
||||
- Stores sync operation history
|
||||
- Indexes: data_source_id, tenant_id, status, started_at
|
||||
- Foreign key to data_sources with cascade delete
|
||||
|
||||
## Key Design Decisions
|
||||
|
||||
1. **Adapter Pattern**: Each connector is a separate implementation, easy to add new ones
|
||||
2. **Async Task Queue**: Syncs are non-blocking, using asynq (or sync mode in Lite)
|
||||
3. **Incremental Sync**: Supports cursor-based incremental syncs to minimize API calls
|
||||
4. **Encryption**: API keys/tokens stored encrypted with AES-256-GCM
|
||||
5. **Multi-tenant**: All operations are tenant-isolated
|
||||
6. **Channel Tracking**: Knowledge.Channel field tracks data source type for origin tracking
|
||||
|
||||
## Future Enhancements
|
||||
|
||||
1. **Webhook Support**: Real-time push from platforms that support webhooks
|
||||
2. **Conflict Resolution**: Advanced merge/conflict strategies for overlapping content
|
||||
3. **Rate Limiting**: Per-connector rate limiting and backoff strategies
|
||||
4. **Scheduling**: Full cron scheduler with time zone support
|
||||
5. **Monitoring**: Metrics, alerting, and sync health dashboards
|
||||
6. **Filtering**: User-defined filters for selective syncing (by title, date, tags, etc)
|
||||
7. **Transformation**: Content transformation pipelines (e.g., extract tables, summarize)
|
||||
8. **Deduplication**: Smart deduplication across multiple sources
|
||||
|
||||
## Testing
|
||||
|
||||
The framework is designed to be testable:
|
||||
- Mock connectors can be created for testing
|
||||
- No external dependencies required for core logic
|
||||
- Database operations isolated and mockable
|
||||
|
||||
## Error Handling
|
||||
|
||||
All operations return detailed error information:
|
||||
- Connection failures are captured and status updated
|
||||
- Partial failures are logged (some items succeed, some fail)
|
||||
- Error messages stored in DataSource for user troubleshooting
|
||||
@@ -0,0 +1,221 @@
|
||||
package datasource
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
// Connector is the interface that all external data source connectors must implement.
|
||||
// Each connector (Feishu, Notion, Confluence, etc.) provides an implementation of this interface.
|
||||
type Connector interface {
|
||||
// Type returns the connector type identifier (e.g., "feishu", "notion")
|
||||
Type() string
|
||||
|
||||
// Validate verifies that the provided configuration is valid by testing connectivity
|
||||
// and checking credentials. Returns error if validation fails.
|
||||
Validate(ctx context.Context, config *types.DataSourceConfig) error
|
||||
|
||||
// ListResources lists available resources that can be synced (documents, spaces, folders, etc.)
|
||||
// Returns a list of Resource objects that the user can select for syncing.
|
||||
ListResources(ctx context.Context, config *types.DataSourceConfig) ([]types.Resource, error)
|
||||
|
||||
// FetchAll performs a full sync of the specified resources.
|
||||
// Returns all items from the given resource IDs.
|
||||
FetchAll(ctx context.Context, config *types.DataSourceConfig, resourceIDs []string) ([]types.FetchedItem, error)
|
||||
|
||||
// FetchIncremental performs an incremental sync based on the provided cursor.
|
||||
// Returns items that have changed since the last sync, a new cursor for the next sync,
|
||||
// and an error if the operation fails.
|
||||
FetchIncremental(ctx context.Context, config *types.DataSourceConfig, cursor *types.SyncCursor) ([]types.FetchedItem, *types.SyncCursor, error)
|
||||
}
|
||||
|
||||
// ConnectorRegistry manages the registration and lookup of available connectors
|
||||
type ConnectorRegistry struct {
|
||||
connectors map[string]Connector
|
||||
}
|
||||
|
||||
// NewConnectorRegistry creates a new connector registry
|
||||
func NewConnectorRegistry() *ConnectorRegistry {
|
||||
return &ConnectorRegistry{
|
||||
connectors: make(map[string]Connector),
|
||||
}
|
||||
}
|
||||
|
||||
// Register registers a connector with the registry
|
||||
func (r *ConnectorRegistry) Register(connector Connector) error {
|
||||
if connector == nil {
|
||||
return ErrConnectorNil
|
||||
}
|
||||
if connector.Type() == "" {
|
||||
return ErrConnectorTypeEmpty
|
||||
}
|
||||
r.connectors[connector.Type()] = connector
|
||||
return nil
|
||||
}
|
||||
|
||||
// Get retrieves a connector by type
|
||||
func (r *ConnectorRegistry) Get(connectorType string) (Connector, error) {
|
||||
connector, exists := r.connectors[connectorType]
|
||||
if !exists {
|
||||
return nil, ErrConnectorNotFound
|
||||
}
|
||||
return connector, nil
|
||||
}
|
||||
|
||||
// List returns all registered connector types
|
||||
func (r *ConnectorRegistry) List() []string {
|
||||
types := make([]string, 0, len(r.connectors))
|
||||
for t := range r.connectors {
|
||||
types = append(types, t)
|
||||
}
|
||||
return types
|
||||
}
|
||||
|
||||
// ConnectorMetadata provides metadata about available connectors
|
||||
type ConnectorMetadata struct {
|
||||
Type string `json:"type"`
|
||||
Name string `json:"name"`
|
||||
Description string `json:"description"`
|
||||
Icon string `json:"icon,omitempty"`
|
||||
Priority int `json:"priority"` // Priority order for UI display (lower = higher priority)
|
||||
AuthType string `json:"auth_type"` // "oauth2", "api_key", "token", etc.
|
||||
Capabilities []string `json:"capabilities"` // "incremental", "webhook", "deletion_sync", etc.
|
||||
}
|
||||
|
||||
// GetConnectorMetadata returns metadata for all available connectors
|
||||
// This is used by the frontend to display connector options
|
||||
var ConnectorMetadataRegistry = map[string]ConnectorMetadata{
|
||||
types.ConnectorTypeFeishu: {
|
||||
Type: types.ConnectorTypeFeishu,
|
||||
Name: "Feishu (飞书)",
|
||||
Description: "Sync documents, wikis, and content from Feishu",
|
||||
Priority: 0,
|
||||
AuthType: "oauth2",
|
||||
Capabilities: []string{"incremental", "webhook", "deletion_sync"},
|
||||
},
|
||||
types.ConnectorTypeNotion: {
|
||||
Type: types.ConnectorTypeNotion,
|
||||
Name: "Notion",
|
||||
Description: "Sync pages and databases from Notion",
|
||||
Priority: 1,
|
||||
AuthType: "api_key",
|
||||
Capabilities: []string{"incremental"},
|
||||
},
|
||||
types.ConnectorTypeConfluence: {
|
||||
Type: types.ConnectorTypeConfluence,
|
||||
Name: "Confluence",
|
||||
Description: "Sync spaces and pages from Atlassian Confluence",
|
||||
Priority: 2,
|
||||
AuthType: "api_key",
|
||||
Capabilities: []string{"incremental"},
|
||||
},
|
||||
types.ConnectorTypeYuque: {
|
||||
Type: types.ConnectorTypeYuque,
|
||||
Name: "Yuque (语雀)",
|
||||
Description: "Sync knowledge bases and documents from Yuque",
|
||||
Priority: 3,
|
||||
AuthType: "api_key",
|
||||
Capabilities: []string{"incremental"},
|
||||
},
|
||||
types.ConnectorTypeGitHub: {
|
||||
Type: types.ConnectorTypeGitHub,
|
||||
Name: "GitHub",
|
||||
Description: "Sync repositories, wikis, and issues from GitHub",
|
||||
Priority: 4,
|
||||
AuthType: "oauth2",
|
||||
Capabilities: []string{"incremental"},
|
||||
},
|
||||
types.ConnectorTypeGoogleDrive: {
|
||||
Type: types.ConnectorTypeGoogleDrive,
|
||||
Name: "Google Drive",
|
||||
Description: "Sync documents and files from Google Drive",
|
||||
Priority: 5,
|
||||
AuthType: "oauth2",
|
||||
Capabilities: []string{"incremental"},
|
||||
},
|
||||
types.ConnectorTypeOneDrive: {
|
||||
Type: types.ConnectorTypeOneDrive,
|
||||
Name: "OneDrive / SharePoint",
|
||||
Description: "Sync documents and files from Microsoft OneDrive",
|
||||
Priority: 6,
|
||||
AuthType: "oauth2",
|
||||
Capabilities: []string{"incremental"},
|
||||
},
|
||||
types.ConnectorTypeDingTalk: {
|
||||
Type: types.ConnectorTypeDingTalk,
|
||||
Name: "DingTalk (钉钉)",
|
||||
Description: "Sync documents and content from DingTalk",
|
||||
Priority: 7,
|
||||
AuthType: "api_key",
|
||||
Capabilities: []string{"incremental"},
|
||||
},
|
||||
types.ConnectorTypeWecomDoc: {
|
||||
Type: types.ConnectorTypeWecomDoc,
|
||||
Name: "WeCom Docs (企微文档)",
|
||||
Description: "Sync documents and content from WeCom",
|
||||
Priority: 8,
|
||||
AuthType: "api_key",
|
||||
Capabilities: []string{"incremental"},
|
||||
},
|
||||
types.ConnectorTypeWebCrawler: {
|
||||
Type: types.ConnectorTypeWebCrawler,
|
||||
Name: "Web Crawler (Sitemap)",
|
||||
Description: "Crawl websites via Sitemap.xml",
|
||||
Priority: 9,
|
||||
AuthType: "none",
|
||||
Capabilities: []string{},
|
||||
},
|
||||
types.ConnectorTypeSlack: {
|
||||
Type: types.ConnectorTypeSlack,
|
||||
Name: "Slack",
|
||||
Description: "Sync channel messages and files from Slack",
|
||||
Priority: 10,
|
||||
AuthType: "oauth2",
|
||||
Capabilities: []string{"incremental"},
|
||||
},
|
||||
types.ConnectorTypeIMAP: {
|
||||
Type: types.ConnectorTypeIMAP,
|
||||
Name: "Email (IMAP)",
|
||||
Description: "Sync email content from IMAP servers",
|
||||
Priority: 11,
|
||||
AuthType: "password",
|
||||
Capabilities: []string{},
|
||||
},
|
||||
types.ConnectorTypeRSS: {
|
||||
Type: types.ConnectorTypeRSS,
|
||||
Name: "RSS / Atom Feed",
|
||||
Description: "Sync articles from RSS/Atom feeds",
|
||||
Priority: 12,
|
||||
AuthType: "none",
|
||||
Capabilities: []string{},
|
||||
},
|
||||
}
|
||||
|
||||
// GetConnectorMetadata returns metadata for a specific connector type
|
||||
func GetConnectorMetadata(connectorType string) (ConnectorMetadata, bool) {
|
||||
meta, ok := ConnectorMetadataRegistry[connectorType]
|
||||
return meta, ok
|
||||
}
|
||||
|
||||
// ListAvailableConnectors returns all available connector metadata
|
||||
// sorted by priority
|
||||
func ListAvailableConnectors() []ConnectorMetadata {
|
||||
metadata := make([]ConnectorMetadata, 0, len(ConnectorMetadataRegistry))
|
||||
for _, meta := range ConnectorMetadataRegistry {
|
||||
metadata = append(metadata, meta)
|
||||
}
|
||||
|
||||
// Sort by priority (insertion sort for simplicity)
|
||||
for i := 1; i < len(metadata); i++ {
|
||||
key := metadata[i]
|
||||
j := i - 1
|
||||
for j >= 0 && metadata[j].Priority > key.Priority {
|
||||
metadata[j+1] = metadata[j]
|
||||
j--
|
||||
}
|
||||
metadata[j+1] = key
|
||||
}
|
||||
|
||||
return metadata
|
||||
}
|
||||
@@ -0,0 +1,451 @@
|
||||
package feishu
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
)
|
||||
|
||||
// Client wraps the Feishu Open Platform API for document/wiki operations.
|
||||
type Client struct {
|
||||
baseURL string
|
||||
appID string
|
||||
appSecret string
|
||||
|
||||
httpClient *http.Client
|
||||
|
||||
// Token cache (thread-safe)
|
||||
tokenMu sync.Mutex
|
||||
tokenCache string
|
||||
tokenExpAt time.Time
|
||||
}
|
||||
|
||||
// NewClient creates a new Feishu API client.
|
||||
func NewClient(config *Config) *Client {
|
||||
return &Client{
|
||||
baseURL: config.GetBaseURL(),
|
||||
appID: config.AppID,
|
||||
appSecret: config.AppSecret,
|
||||
httpClient: &http.Client{Timeout: 30 * time.Second},
|
||||
}
|
||||
}
|
||||
|
||||
// getTenantAccessToken retrieves (or returns cached) tenant access token.
|
||||
// Feishu tokens expire in 2 hours; we cache with a 5-minute safety margin.
|
||||
func (c *Client) getTenantAccessToken(ctx context.Context) (string, error) {
|
||||
c.tokenMu.Lock()
|
||||
defer c.tokenMu.Unlock()
|
||||
|
||||
if c.tokenCache != "" && time.Now().Before(c.tokenExpAt) {
|
||||
return c.tokenCache, nil
|
||||
}
|
||||
|
||||
payload, _ := json.Marshal(map[string]string{
|
||||
"app_id": c.appID,
|
||||
"app_secret": c.appSecret,
|
||||
})
|
||||
|
||||
url := c.baseURL + "/open-apis/auth/v3/tenant_access_token/internal"
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(payload))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("create token request: %w", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json; charset=utf-8")
|
||||
|
||||
resp, err := c.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("request token: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
var result tokenResponse
|
||||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||
return "", fmt.Errorf("decode token response: %w", err)
|
||||
}
|
||||
if result.Code != 0 {
|
||||
return "", fmt.Errorf("feishu auth error: code=%d msg=%s", result.Code, result.Msg)
|
||||
}
|
||||
|
||||
c.tokenCache = result.TenantAccessToken
|
||||
ttl := time.Duration(result.Expire) * time.Second
|
||||
if ttl > 5*time.Minute {
|
||||
ttl -= 5 * time.Minute
|
||||
}
|
||||
c.tokenExpAt = time.Now().Add(ttl)
|
||||
|
||||
logger.Infof(ctx, "[Feishu] got tenant_access_token: %s...%s expire=%ds",
|
||||
result.TenantAccessToken[:8], result.TenantAccessToken[len(result.TenantAccessToken)-4:], result.Expire)
|
||||
|
||||
return c.tokenCache, nil
|
||||
}
|
||||
|
||||
// doRequest executes an authenticated API request and decodes the JSON response.
|
||||
func (c *Client) doRequest(ctx context.Context, method, path string, body interface{}, result interface{}) error {
|
||||
token, err := c.getTenantAccessToken(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var bodyReader io.Reader
|
||||
if body != nil {
|
||||
bodyBytes, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal request body: %w", err)
|
||||
}
|
||||
bodyReader = bytes.NewReader(bodyBytes)
|
||||
}
|
||||
|
||||
url := c.baseURL + path
|
||||
req, err := http.NewRequestWithContext(ctx, method, url, bodyReader)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create request: %w", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json; charset=utf-8")
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
|
||||
logger.Infof(ctx, "[Feishu] %s %s", method, path)
|
||||
|
||||
resp, err := c.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("execute request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
// Read body once for logging + decoding
|
||||
respBody, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read response body: %w", err)
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "[Feishu] %s %s → status=%d bodyLen=%d body=%s",
|
||||
method, path, resp.StatusCode, len(respBody), truncate(string(respBody), 1000))
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("feishu api error: status=%d body=%s", resp.StatusCode, string(respBody))
|
||||
}
|
||||
|
||||
if result != nil {
|
||||
if err := json.Unmarshal(respBody, result); err != nil {
|
||||
return fmt.Errorf("decode response: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// truncate truncates a string to maxLen and appends "..." if truncated.
|
||||
func truncate(s string, maxLen int) string {
|
||||
if len(s) <= maxLen {
|
||||
return s
|
||||
}
|
||||
return s[:maxLen] + "..."
|
||||
}
|
||||
|
||||
// ListWikiSpaces returns all wiki spaces accessible to the app.
|
||||
func (c *Client) ListWikiSpaces(ctx context.Context) ([]wikiSpace, error) {
|
||||
var allSpaces []wikiSpace
|
||||
pageToken := ""
|
||||
|
||||
for {
|
||||
path := "/open-apis/wiki/v2/spaces?page_size=50"
|
||||
if pageToken != "" {
|
||||
path += "&page_token=" + pageToken
|
||||
}
|
||||
|
||||
var resp wikiSpaceListResponse
|
||||
if err := c.doRequest(ctx, http.MethodGet, path, nil, &resp); err != nil {
|
||||
return nil, fmt.Errorf("list wiki spaces: %w", err)
|
||||
}
|
||||
if resp.Code != 0 {
|
||||
logger.Errorf(ctx, "[Feishu] ListWikiSpaces error: code=%d msg=%s", resp.Code, resp.Msg)
|
||||
return nil, fmt.Errorf("list wiki spaces error: code=%d msg=%s", resp.Code, resp.Msg)
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "[Feishu] ListWikiSpaces: got %d spaces, has_more=%v", len(resp.Data.Items), resp.Data.HasMore)
|
||||
for i, s := range resp.Data.Items {
|
||||
logger.Infof(ctx, "[Feishu] space[%d]: id=%s name=%q visibility=%s", i, s.SpaceID, s.Name, s.Visibility)
|
||||
}
|
||||
|
||||
allSpaces = append(allSpaces, resp.Data.Items...)
|
||||
|
||||
if !resp.Data.HasMore || resp.Data.PageToken == "" {
|
||||
break
|
||||
}
|
||||
pageToken = resp.Data.PageToken
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "[Feishu] ListWikiSpaces: total %d spaces", len(allSpaces))
|
||||
return allSpaces, nil
|
||||
}
|
||||
|
||||
// ListWikiNodes returns all nodes (documents) under a wiki space.
|
||||
// If parentNodeToken is empty, returns top-level nodes.
|
||||
func (c *Client) ListWikiNodes(ctx context.Context, spaceID string, parentNodeToken string) ([]wikiNode, error) {
|
||||
var allNodes []wikiNode
|
||||
pageToken := ""
|
||||
|
||||
for {
|
||||
path := fmt.Sprintf("/open-apis/wiki/v2/spaces/%s/nodes?page_size=50", spaceID)
|
||||
if parentNodeToken != "" {
|
||||
path += "&parent_node_token=" + parentNodeToken
|
||||
}
|
||||
if pageToken != "" {
|
||||
path += "&page_token=" + pageToken
|
||||
}
|
||||
|
||||
var resp wikiNodeListResponse
|
||||
if err := c.doRequest(ctx, http.MethodGet, path, nil, &resp); err != nil {
|
||||
return nil, fmt.Errorf("list wiki nodes: %w", err)
|
||||
}
|
||||
if resp.Code != 0 {
|
||||
return nil, fmt.Errorf("list wiki nodes error: code=%d msg=%s", resp.Code, resp.Msg)
|
||||
}
|
||||
|
||||
allNodes = append(allNodes, resp.Data.Items...)
|
||||
|
||||
if !resp.Data.HasMore || resp.Data.PageToken == "" {
|
||||
break
|
||||
}
|
||||
pageToken = resp.Data.PageToken
|
||||
}
|
||||
|
||||
return allNodes, nil
|
||||
}
|
||||
|
||||
// ListAllWikiNodesRecursive recursively lists all nodes under a wiki space.
|
||||
// It walks the tree depth-first to discover all nested documents.
|
||||
func (c *Client) ListAllWikiNodesRecursive(ctx context.Context, spaceID string) ([]wikiNode, error) {
|
||||
// Start with top-level nodes
|
||||
topNodes, err := c.ListWikiNodes(ctx, spaceID, "")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var allNodes []wikiNode
|
||||
var walk func(nodes []wikiNode) error
|
||||
|
||||
walk = func(nodes []wikiNode) error {
|
||||
for _, node := range nodes {
|
||||
allNodes = append(allNodes, node)
|
||||
|
||||
// Recurse into child nodes if this node has children
|
||||
if node.HasChild {
|
||||
children, err := c.ListWikiNodes(ctx, spaceID, node.NodeToken)
|
||||
if err != nil {
|
||||
return fmt.Errorf("list children of %s: %w", node.NodeToken, err)
|
||||
}
|
||||
if err := walk(children); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := walk(topNodes); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return allNodes, nil
|
||||
}
|
||||
|
||||
// GetDocumentRawContent retrieves the raw text content of a Feishu docx document.
|
||||
// This returns plain text (not rich text / block structure).
|
||||
// Deprecated: prefer ExportAndDownload which preserves formatting.
|
||||
func (c *Client) GetDocumentRawContent(ctx context.Context, documentID string) (string, error) {
|
||||
path := fmt.Sprintf("/open-apis/docx/v1/documents/%s/raw_content", documentID)
|
||||
|
||||
var resp docRawContentResponse
|
||||
if err := c.doRequest(ctx, http.MethodGet, path, nil, &resp); err != nil {
|
||||
return "", fmt.Errorf("get document raw content: %w", err)
|
||||
}
|
||||
if resp.Code != 0 {
|
||||
return "", fmt.Errorf("get document raw content error: code=%d msg=%s", resp.Code, resp.Msg)
|
||||
}
|
||||
|
||||
return resp.Data.Content, nil
|
||||
}
|
||||
|
||||
// Ping verifies the credentials by attempting to get a tenant access token.
|
||||
func (c *Client) Ping(ctx context.Context) error {
|
||||
_, err := c.getTenantAccessToken(ctx)
|
||||
return err
|
||||
}
|
||||
|
||||
// ──────────────────────────────────────────────────────────────────────
|
||||
// Export task API: export docx/sheet/bitable to downloadable files
|
||||
//
|
||||
// Flow:
|
||||
// 1. POST /drive/v1/export_tasks → create export task, get ticket
|
||||
// 2. GET /drive/v1/export_tasks/:ticket → poll until status=0 (success)
|
||||
// 3. GET /drive/v1/export_tasks/file/:ticket/download → download file bytes
|
||||
// ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
// CreateExportTask creates an async export task for a Feishu document.
|
||||
// - token: the obj_token of the document (e.g. docx token, sheet token)
|
||||
// - objType: the Feishu obj_type ("docx", "doc", "sheet", "bitable")
|
||||
// - fileExtension: desired output format ("docx", "xlsx", "pdf")
|
||||
func (c *Client) CreateExportTask(ctx context.Context, token, objType, fileExtension string) (string, error) {
|
||||
body := map[string]string{
|
||||
"file_extension": fileExtension,
|
||||
"token": token,
|
||||
"type": objType,
|
||||
}
|
||||
|
||||
var resp exportTaskCreateResponse
|
||||
if err := c.doRequest(ctx, http.MethodPost, "/open-apis/drive/v1/export_tasks", body, &resp); err != nil {
|
||||
return "", fmt.Errorf("create export task: %w", err)
|
||||
}
|
||||
if resp.Code != 0 {
|
||||
return "", fmt.Errorf("create export task error: code=%d msg=%s", resp.Code, resp.Msg)
|
||||
}
|
||||
|
||||
return resp.Data.Ticket, nil
|
||||
}
|
||||
|
||||
// GetExportTaskStatus polls the status of an export task.
|
||||
// Returns (fileToken, fileName, error). fileToken is non-empty only when the job succeeds.
|
||||
// The token parameter is the obj_token of the document being exported (required by the API).
|
||||
func (c *Client) GetExportTaskStatus(ctx context.Context, ticket string, token string) (string, string, error) {
|
||||
path := fmt.Sprintf("/open-apis/drive/v1/export_tasks/%s?token=%s", ticket, token)
|
||||
|
||||
var resp exportTaskStatusResponse
|
||||
if err := c.doRequest(ctx, http.MethodGet, path, nil, &resp); err != nil {
|
||||
return "", "", fmt.Errorf("get export task status: %w", err)
|
||||
}
|
||||
if resp.Code != 0 {
|
||||
return "", "", fmt.Errorf("get export task status error: code=%d msg=%s", resp.Code, resp.Msg)
|
||||
}
|
||||
|
||||
r := resp.Data.Result
|
||||
switch r.JobStatus {
|
||||
case 0: // success
|
||||
return r.FileToken, r.FileName, nil
|
||||
case 1, 2: // initializing, processing
|
||||
return "", "", nil // not ready yet
|
||||
default:
|
||||
return "", "", fmt.Errorf("export task failed: status=%d msg=%s", r.JobStatus, r.JobErrorMsg)
|
||||
}
|
||||
}
|
||||
|
||||
// DownloadExportFile downloads the exported file by its file_token.
|
||||
// The file_token is returned by GetExportTaskStatus when the export job completes.
|
||||
// The file must be downloaded within 10 minutes of export completion.
|
||||
func (c *Client) DownloadExportFile(ctx context.Context, fileToken string) ([]byte, error) {
|
||||
path := fmt.Sprintf("/open-apis/drive/v1/export_tasks/file/%s/download", fileToken)
|
||||
return c.downloadRawBytes(ctx, path)
|
||||
}
|
||||
|
||||
// ExportAndDownload is a high-level helper that creates an export task, polls until
|
||||
// completion, and downloads the resulting file. Returns (fileBytes, fileName, error).
|
||||
//
|
||||
// Timeout: 60 seconds. Poll interval: 2 seconds.
|
||||
func (c *Client) ExportAndDownload(ctx context.Context, objToken, objType string) ([]byte, string, error) {
|
||||
// Determine export format
|
||||
fileExt, ok := objTypeToExportFileExtension[objType]
|
||||
if !ok {
|
||||
return nil, "", fmt.Errorf("unsupported obj_type for export: %s", objType)
|
||||
}
|
||||
|
||||
exportType, ok := objTypeToExportType[objType]
|
||||
if !ok {
|
||||
return nil, "", fmt.Errorf("unsupported obj_type for export: %s", objType)
|
||||
}
|
||||
|
||||
// Step 1: create export task
|
||||
ticket, err := c.CreateExportTask(ctx, objToken, exportType, fileExt)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
|
||||
// Step 2: poll until ready (max 60s, every 2s)
|
||||
deadline := time.Now().Add(60 * time.Second)
|
||||
var fileToken, fileName string
|
||||
|
||||
for time.Now().Before(deadline) {
|
||||
fileToken, fileName, err = c.GetExportTaskStatus(ctx, ticket, objToken)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
if fileToken != "" {
|
||||
break // export ready
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, "", ctx.Err()
|
||||
case <-time.After(2 * time.Second):
|
||||
}
|
||||
}
|
||||
|
||||
if fileToken == "" {
|
||||
return nil, "", fmt.Errorf("export task timed out after 60s (ticket=%s)", ticket)
|
||||
}
|
||||
|
||||
// Step 3: download file using file_token (NOT ticket)
|
||||
data, err := c.DownloadExportFile(ctx, fileToken)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
|
||||
// Build a sensible file name
|
||||
if fileName == "" {
|
||||
fileName = "export" + exportFileExtToSuffix[fileExt]
|
||||
}
|
||||
|
||||
return data, fileName, nil
|
||||
}
|
||||
|
||||
// ──────────────────────────────────────────────────────────────────────
|
||||
// Drive file download: for "file" type wiki nodes (uploaded PDF/Word/etc.)
|
||||
// ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
// DownloadDriveFile downloads a file from Feishu Drive by its file token.
|
||||
// Used for wiki nodes with obj_type="file" (user-uploaded PDF, Word, images, etc.).
|
||||
func (c *Client) DownloadDriveFile(ctx context.Context, fileToken string) ([]byte, error) {
|
||||
path := fmt.Sprintf("/open-apis/drive/v1/files/%s/download", fileToken)
|
||||
return c.downloadRawBytes(ctx, path)
|
||||
}
|
||||
|
||||
// downloadRawBytes performs an authenticated GET and returns the raw response body.
|
||||
func (c *Client) downloadRawBytes(ctx context.Context, path string) ([]byte, error) {
|
||||
token, err := c.getTenantAccessToken(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
url := c.baseURL + path
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create download request: %w", err)
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
|
||||
logger.Infof(ctx, "[Feishu] download GET %s", path)
|
||||
|
||||
resp, err := c.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("download request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
logger.Errorf(ctx, "[Feishu] download GET %s → status=%d body=%s", path, resp.StatusCode, truncate(string(body), 500))
|
||||
return nil, fmt.Errorf("download failed: status=%d body=%s", resp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
data, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read download body: %w", err)
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "[Feishu] download GET %s → OK, %d bytes", path, len(data))
|
||||
return data, nil
|
||||
}
|
||||
@@ -0,0 +1,376 @@
|
||||
package feishu
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
// Connector implements the datasource.Connector interface for Feishu.
|
||||
type Connector struct{}
|
||||
|
||||
// NewConnector creates a new Feishu connector.
|
||||
func NewConnector() *Connector {
|
||||
return &Connector{}
|
||||
}
|
||||
|
||||
// Type returns the connector type identifier.
|
||||
func (c *Connector) Type() string {
|
||||
return types.ConnectorTypeFeishu
|
||||
}
|
||||
|
||||
// Validate verifies that the Feishu configuration is valid by testing connectivity.
|
||||
func (c *Connector) Validate(ctx context.Context, config *types.DataSourceConfig) error {
|
||||
feishuConfig, err := parseFeishuConfig(config)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
client := NewClient(feishuConfig)
|
||||
if err := client.Ping(ctx); err != nil {
|
||||
return fmt.Errorf("feishu connection failed: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// ListResources lists all accessible Feishu Wiki spaces as selectable resources.
|
||||
func (c *Connector) ListResources(ctx context.Context, config *types.DataSourceConfig) ([]types.Resource, error) {
|
||||
feishuConfig, err := parseFeishuConfig(config)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
client := NewClient(feishuConfig)
|
||||
spaces, err := client.ListWikiSpaces(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list feishu wiki spaces: %w", err)
|
||||
}
|
||||
|
||||
resources := make([]types.Resource, 0, len(spaces))
|
||||
for _, space := range spaces {
|
||||
resources = append(resources, types.Resource{
|
||||
ExternalID: space.SpaceID,
|
||||
Name: space.Name,
|
||||
Type: "wiki_space",
|
||||
Description: space.Description,
|
||||
URL: fmt.Sprintf("https://feishu.cn/wiki/%s", space.SpaceID),
|
||||
Metadata: map[string]interface{}{
|
||||
"visibility": space.Visibility,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
return resources, nil
|
||||
}
|
||||
|
||||
// FetchAll performs a full sync of all documents from the specified wiki spaces.
|
||||
func (c *Connector) FetchAll(ctx context.Context, config *types.DataSourceConfig, resourceIDs []string) ([]types.FetchedItem, error) {
|
||||
feishuConfig, err := parseFeishuConfig(config)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
client := NewClient(feishuConfig)
|
||||
|
||||
var allItems []types.FetchedItem
|
||||
|
||||
for _, spaceID := range resourceIDs {
|
||||
// List all nodes in this wiki space recursively
|
||||
nodes, err := client.ListAllWikiNodesRecursive(ctx, spaceID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list nodes in space %s: %w", spaceID, err)
|
||||
}
|
||||
|
||||
// Fetch content for each document node
|
||||
for _, node := range nodes {
|
||||
item, err := c.fetchNodeContent(ctx, client, node, spaceID)
|
||||
if err != nil {
|
||||
// Log error but continue with other nodes
|
||||
allItems = append(allItems, types.FetchedItem{
|
||||
ExternalID: node.NodeToken,
|
||||
Title: node.Title,
|
||||
SourceResourceID: spaceID,
|
||||
Metadata: map[string]string{
|
||||
"error": err.Error(),
|
||||
},
|
||||
})
|
||||
continue
|
||||
}
|
||||
if item != nil {
|
||||
allItems = append(allItems, *item)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return allItems, nil
|
||||
}
|
||||
|
||||
// FetchIncremental performs an incremental sync by comparing node edit times
|
||||
// against the previously recorded state.
|
||||
func (c *Connector) FetchIncremental(ctx context.Context, config *types.DataSourceConfig, cursor *types.SyncCursor) ([]types.FetchedItem, *types.SyncCursor, error) {
|
||||
feishuConfig, err := parseFeishuConfig(config)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
client := NewClient(feishuConfig)
|
||||
|
||||
// Parse the previous cursor state
|
||||
var prevCursor feishuCursor
|
||||
if cursor != nil && cursor.ConnectorCursor != nil {
|
||||
cursorBytes, _ := json.Marshal(cursor.ConnectorCursor)
|
||||
_ = json.Unmarshal(cursorBytes, &prevCursor)
|
||||
}
|
||||
|
||||
// Build new cursor to track current state
|
||||
newCursor := feishuCursor{
|
||||
LastSyncTime: time.Now(),
|
||||
SpaceNodeTimes: make(map[string]map[string]string),
|
||||
}
|
||||
|
||||
var changedItems []types.FetchedItem
|
||||
|
||||
// Get resource IDs from config
|
||||
resourceIDs := config.ResourceIDs
|
||||
if len(resourceIDs) == 0 {
|
||||
return nil, nil, fmt.Errorf("no resource IDs (wiki space IDs) configured")
|
||||
}
|
||||
|
||||
for _, spaceID := range resourceIDs {
|
||||
// List all nodes in this space
|
||||
nodes, err := client.ListAllWikiNodesRecursive(ctx, spaceID)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("list nodes in space %s: %w", spaceID, err)
|
||||
}
|
||||
|
||||
newCursor.SpaceNodeTimes[spaceID] = make(map[string]string)
|
||||
|
||||
// Build a set of current node tokens for deletion detection
|
||||
currentNodes := make(map[string]bool)
|
||||
|
||||
for _, node := range nodes {
|
||||
currentNodes[node.NodeToken] = true
|
||||
// Use ObjEditTime (document content edit time) for change detection,
|
||||
// NOT NodeEditTime which only tracks node attribute changes (title, position).
|
||||
editTimeStr := node.ObjEditTime
|
||||
if editTimeStr == "" {
|
||||
editTimeStr = node.NodeEditTime // fallback for nodes that don't have obj_edit_time
|
||||
}
|
||||
newCursor.SpaceNodeTimes[spaceID][node.NodeToken] = editTimeStr
|
||||
|
||||
// Check if node has changed since last sync
|
||||
if prevCursor.SpaceNodeTimes != nil {
|
||||
if prevTimes, ok := prevCursor.SpaceNodeTimes[spaceID]; ok {
|
||||
if prevEditTime, exists := prevTimes[node.NodeToken]; exists {
|
||||
if prevEditTime == editTimeStr {
|
||||
// Node unchanged, skip
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Node is new or changed — fetch its content
|
||||
item, err := c.fetchNodeContent(ctx, client, node, spaceID)
|
||||
if err != nil {
|
||||
// Record failed items
|
||||
changedItems = append(changedItems, types.FetchedItem{
|
||||
ExternalID: node.NodeToken,
|
||||
Title: node.Title,
|
||||
SourceResourceID: spaceID,
|
||||
Metadata: map[string]string{
|
||||
"error": err.Error(),
|
||||
},
|
||||
})
|
||||
continue
|
||||
}
|
||||
if item != nil {
|
||||
changedItems = append(changedItems, *item)
|
||||
}
|
||||
}
|
||||
|
||||
// Detect deleted nodes
|
||||
if prevCursor.SpaceNodeTimes != nil {
|
||||
if prevTimes, ok := prevCursor.SpaceNodeTimes[spaceID]; ok {
|
||||
for nodeToken := range prevTimes {
|
||||
if !currentNodes[nodeToken] {
|
||||
// Node was deleted
|
||||
changedItems = append(changedItems, types.FetchedItem{
|
||||
ExternalID: nodeToken,
|
||||
IsDeleted: true,
|
||||
SourceResourceID: spaceID,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Build next sync cursor
|
||||
nextCursorMap := make(map[string]interface{})
|
||||
cursorBytes, _ := json.Marshal(newCursor)
|
||||
_ = json.Unmarshal(cursorBytes, &nextCursorMap)
|
||||
|
||||
nextSyncCursor := &types.SyncCursor{
|
||||
LastSyncTime: time.Now(),
|
||||
ConnectorCursor: nextCursorMap,
|
||||
}
|
||||
|
||||
return changedItems, nextSyncCursor, nil
|
||||
}
|
||||
|
||||
// fetchNodeContent fetches the content of a single wiki node and converts it to FetchedItem.
|
||||
// Dispatches to different retrieval strategies based on obj_type:
|
||||
// - docx/doc → export API → .docx file
|
||||
// - sheet → export API → .xlsx file
|
||||
// - bitable → export API → .xlsx file
|
||||
// - file → drive download → original file (PDF/Word/image/etc.)
|
||||
// - mindnote → skip (no API)
|
||||
// - slides → skip (no API)
|
||||
func (c *Connector) fetchNodeContent(ctx context.Context, client *Client, node wikiNode, spaceID string) (*types.FetchedItem, error) {
|
||||
if !isSupportedDocType(node.ObjType) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
editTime := parseFeishuTimestamp(node.NodeEditTime)
|
||||
baseMeta := map[string]string{
|
||||
"obj_token": node.ObjToken,
|
||||
"obj_type": node.ObjType,
|
||||
"node_token": node.NodeToken,
|
||||
"space_id": spaceID,
|
||||
"creator": node.Creator,
|
||||
"owner": node.Owner,
|
||||
"channel": types.ChannelFeishu,
|
||||
}
|
||||
|
||||
switch node.ObjType {
|
||||
case "docx", "doc", "sheet", "bitable":
|
||||
// Export as a file via the async export API
|
||||
data, fileName, err := client.ExportAndDownload(ctx, node.ObjToken, node.ObjType)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("export %s (%s): %w", node.Title, node.ObjType, err)
|
||||
}
|
||||
|
||||
// Ensure a reasonable file name with correct extension
|
||||
ext := exportFileExtToSuffix[objTypeToExportFileExtension[node.ObjType]]
|
||||
if fileName == "" {
|
||||
fileName = sanitizeFileName(node.Title) + ext
|
||||
} else if !strings.HasSuffix(strings.ToLower(fileName), ext) {
|
||||
// Feishu often returns the doc title without extension — append it
|
||||
fileName = sanitizeFileName(fileName) + ext
|
||||
}
|
||||
|
||||
return &types.FetchedItem{
|
||||
ExternalID: node.NodeToken,
|
||||
Title: node.Title,
|
||||
Content: data,
|
||||
ContentType: "application/octet-stream",
|
||||
FileName: fileName,
|
||||
URL: fmt.Sprintf("https://feishu.cn/wiki/%s", node.NodeToken),
|
||||
UpdatedAt: editTime,
|
||||
SourceResourceID: spaceID,
|
||||
Metadata: baseMeta,
|
||||
}, nil
|
||||
|
||||
case "file":
|
||||
// Download the original uploaded file from Drive
|
||||
data, err := client.DownloadDriveFile(ctx, node.ObjToken)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("download file %s (%s): %w", node.Title, node.ObjToken, err)
|
||||
}
|
||||
|
||||
// Use the node title as file name; it usually preserves the original extension
|
||||
fileName := node.Title
|
||||
if fileName == "" {
|
||||
fileName = node.ObjToken
|
||||
}
|
||||
|
||||
return &types.FetchedItem{
|
||||
ExternalID: node.NodeToken,
|
||||
Title: node.Title,
|
||||
Content: data,
|
||||
ContentType: "application/octet-stream",
|
||||
FileName: fileName,
|
||||
URL: fmt.Sprintf("https://feishu.cn/wiki/%s", node.NodeToken),
|
||||
UpdatedAt: editTime,
|
||||
SourceResourceID: spaceID,
|
||||
Metadata: baseMeta,
|
||||
}, nil
|
||||
|
||||
default:
|
||||
return nil, nil
|
||||
}
|
||||
}
|
||||
|
||||
// --- Helper functions ---
|
||||
|
||||
// parseFeishuConfig extracts and validates Feishu-specific configuration.
|
||||
func parseFeishuConfig(config *types.DataSourceConfig) (*Config, error) {
|
||||
if config == nil {
|
||||
return nil, fmt.Errorf("config is nil")
|
||||
}
|
||||
|
||||
credBytes, err := json.Marshal(config.Credentials)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal credentials: %w", err)
|
||||
}
|
||||
|
||||
var feishuConfig Config
|
||||
if err := json.Unmarshal(credBytes, &feishuConfig); err != nil {
|
||||
return nil, fmt.Errorf("parse feishu credentials: %w", err)
|
||||
}
|
||||
|
||||
if feishuConfig.AppID == "" || feishuConfig.AppSecret == "" {
|
||||
return nil, fmt.Errorf("feishu app_id and app_secret are required")
|
||||
}
|
||||
|
||||
return &feishuConfig, nil
|
||||
}
|
||||
|
||||
// isSupportedDocType checks if a Feishu document type can be synced.
|
||||
// mindnote and slides have no content read API and are skipped.
|
||||
func isSupportedDocType(objType string) bool {
|
||||
switch objType {
|
||||
case "docx", "doc", "sheet", "bitable", "file":
|
||||
return true
|
||||
default:
|
||||
// mindnote, slides — no content retrieval API available
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// parseFeishuTimestamp parses a Feishu unix timestamp string (seconds) into time.Time.
|
||||
func parseFeishuTimestamp(ts string) time.Time {
|
||||
if ts == "" {
|
||||
return time.Time{}
|
||||
}
|
||||
sec, err := strconv.ParseInt(ts, 10, 64)
|
||||
if err != nil {
|
||||
return time.Time{}
|
||||
}
|
||||
return time.Unix(sec, 0)
|
||||
}
|
||||
|
||||
// sanitizeFileName removes characters that are invalid in filenames.
|
||||
func sanitizeFileName(name string) string {
|
||||
if name == "" {
|
||||
return "untitled"
|
||||
}
|
||||
// Replace common invalid characters with underscore
|
||||
replacer := strings.NewReplacer(
|
||||
"/", "_", "\\", "_", ":", "_", "*", "_",
|
||||
"?", "_", "\"", "_", "<", "_", ">", "_", "|", "_",
|
||||
)
|
||||
result := replacer.Replace(name)
|
||||
// Limit length
|
||||
if len(result) > 200 {
|
||||
result = result[:200]
|
||||
}
|
||||
return result
|
||||
}
|
||||
@@ -0,0 +1,817 @@
|
||||
package feishu
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
// ──────────────────────────────────────────────────────────────────────
|
||||
// Fake Feishu API server
|
||||
// ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
// fakeFeishu builds an httptest.Server that emulates the relevant Feishu APIs.
|
||||
// It returns the server and a Config pointing at it.
|
||||
func fakeFeishu(nodes []wikiNode) (*httptest.Server, *Config) {
|
||||
mux := http.NewServeMux()
|
||||
|
||||
// --- auth ---
|
||||
mux.HandleFunc("/open-apis/auth/v3/tenant_access_token/internal", func(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, tokenResponse{
|
||||
apiResponse: apiResponse{Code: 0},
|
||||
TenantAccessToken: "fake-token",
|
||||
Expire: 7200,
|
||||
})
|
||||
})
|
||||
|
||||
// --- wiki spaces ---
|
||||
mux.HandleFunc("/open-apis/wiki/v2/spaces", func(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, wikiSpaceListResponse{
|
||||
apiResponse: apiResponse{Code: 0},
|
||||
Data: struct {
|
||||
Items []wikiSpace `json:"items"`
|
||||
HasMore bool `json:"has_more"`
|
||||
PageToken string `json:"page_token"`
|
||||
}{
|
||||
Items: []wikiSpace{
|
||||
{SpaceID: "space1", Name: "Test Space", Description: "desc", Visibility: "public"},
|
||||
},
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
// --- wiki nodes (top-level only for simplicity) ---
|
||||
mux.HandleFunc("/open-apis/wiki/v2/spaces/space1/nodes", func(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, wikiNodeListResponse{
|
||||
apiResponse: apiResponse{Code: 0},
|
||||
Data: struct {
|
||||
Items []wikiNode `json:"items"`
|
||||
HasMore bool `json:"has_more"`
|
||||
PageToken string `json:"page_token"`
|
||||
}{
|
||||
Items: nodes,
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
// --- export task: create ---
|
||||
mux.HandleFunc("/open-apis/drive/v1/export_tasks", func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method == http.MethodPost {
|
||||
writeJSON(w, exportTaskCreateResponse{
|
||||
apiResponse: apiResponse{Code: 0},
|
||||
Data: struct{ Ticket string `json:"ticket"` }{Ticket: "ticket-123"},
|
||||
})
|
||||
return
|
||||
}
|
||||
// GET /open-apis/drive/v1/export_tasks/ticket-123
|
||||
writeJSON(w, exportTaskStatusResponse{
|
||||
apiResponse: apiResponse{Code: 0},
|
||||
Data: struct {
|
||||
Result struct {
|
||||
FileToken string `json:"file_token"`
|
||||
FileSize int64 `json:"file_size"`
|
||||
JobStatus int `json:"job_status"`
|
||||
JobErrorMsg string `json:"job_error_msg"`
|
||||
FileName string `json:"file_name"`
|
||||
} `json:"result"`
|
||||
}{
|
||||
Result: struct {
|
||||
FileToken string `json:"file_token"`
|
||||
FileSize int64 `json:"file_size"`
|
||||
JobStatus int `json:"job_status"`
|
||||
JobErrorMsg string `json:"job_error_msg"`
|
||||
FileName string `json:"file_name"`
|
||||
}{
|
||||
FileToken: "ft-abc",
|
||||
FileSize: 100,
|
||||
JobStatus: 0, // success
|
||||
FileName: "exported.docx",
|
||||
},
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
// --- export task: status polling (pattern match with ticket) ---
|
||||
mux.HandleFunc("/open-apis/drive/v1/export_tasks/ticket-123", func(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, exportTaskStatusResponse{
|
||||
apiResponse: apiResponse{Code: 0},
|
||||
Data: struct {
|
||||
Result struct {
|
||||
FileToken string `json:"file_token"`
|
||||
FileSize int64 `json:"file_size"`
|
||||
JobStatus int `json:"job_status"`
|
||||
JobErrorMsg string `json:"job_error_msg"`
|
||||
FileName string `json:"file_name"`
|
||||
} `json:"result"`
|
||||
}{
|
||||
Result: struct {
|
||||
FileToken string `json:"file_token"`
|
||||
FileSize int64 `json:"file_size"`
|
||||
JobStatus int `json:"job_status"`
|
||||
JobErrorMsg string `json:"job_error_msg"`
|
||||
FileName string `json:"file_name"`
|
||||
}{
|
||||
FileToken: "ft-abc",
|
||||
FileSize: 100,
|
||||
JobStatus: 0,
|
||||
FileName: "exported.docx",
|
||||
},
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
// --- export file download ---
|
||||
mux.HandleFunc("/open-apis/drive/v1/export_tasks/file/ticket-123/download", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/octet-stream")
|
||||
w.Write([]byte("fake-docx-content"))
|
||||
})
|
||||
|
||||
// --- drive file download (for "file" type nodes) ---
|
||||
mux.HandleFunc("/open-apis/drive/v1/files/", func(w http.ResponseWriter, r *http.Request) {
|
||||
if strings.HasSuffix(r.URL.Path, "/download") {
|
||||
w.Header().Set("Content-Type", "application/octet-stream")
|
||||
w.Write([]byte("fake-pdf-binary"))
|
||||
return
|
||||
}
|
||||
http.NotFound(w, r)
|
||||
})
|
||||
|
||||
ts := httptest.NewServer(mux)
|
||||
cfg := &Config{
|
||||
AppID: "test-app-id",
|
||||
AppSecret: "test-app-secret",
|
||||
BaseURL: ts.URL,
|
||||
}
|
||||
return ts, cfg
|
||||
}
|
||||
|
||||
func writeJSON(w http.ResponseWriter, v interface{}) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(v)
|
||||
}
|
||||
|
||||
func makeConfig(cfg *Config, resourceIDs []string) *types.DataSourceConfig {
|
||||
creds := map[string]interface{}{
|
||||
"app_id": cfg.AppID,
|
||||
"app_secret": cfg.AppSecret,
|
||||
"base_url": cfg.BaseURL,
|
||||
}
|
||||
return &types.DataSourceConfig{
|
||||
Type: types.ConnectorTypeFeishu,
|
||||
Credentials: creds,
|
||||
ResourceIDs: resourceIDs,
|
||||
}
|
||||
}
|
||||
|
||||
// ──────────────────────────────────────────────────────────────────────
|
||||
// Helper function tests
|
||||
// ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
func TestIsSupportedDocType(t *testing.T) {
|
||||
tests := []struct {
|
||||
objType string
|
||||
expected bool
|
||||
}{
|
||||
{"docx", true},
|
||||
{"doc", true},
|
||||
{"sheet", true},
|
||||
{"bitable", true},
|
||||
{"file", true},
|
||||
{"mindnote", false},
|
||||
{"slides", false},
|
||||
{"unknown", false},
|
||||
{"", false},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.objType, func(t *testing.T) {
|
||||
got := isSupportedDocType(tt.objType)
|
||||
if got != tt.expected {
|
||||
t.Errorf("isSupportedDocType(%q) = %v, want %v", tt.objType, got, tt.expected)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSanitizeFileName(t *testing.T) {
|
||||
tests := []struct {
|
||||
input string
|
||||
expected string
|
||||
}{
|
||||
{"hello", "hello"},
|
||||
{"", "untitled"},
|
||||
{"a/b\\c:d*e", "a_b_c_d_e"},
|
||||
{"normal file.docx", "normal file.docx"},
|
||||
{strings.Repeat("a", 300), strings.Repeat("a", 200)},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.input, func(t *testing.T) {
|
||||
got := sanitizeFileName(tt.input)
|
||||
if got != tt.expected {
|
||||
t.Errorf("sanitizeFileName(%q) = %q, want %q", tt.input, got, tt.expected)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFeishuTimestamp(t *testing.T) {
|
||||
ts := parseFeishuTimestamp("1711468800") // 2024-03-27 00:00:00 UTC
|
||||
if ts.IsZero() {
|
||||
t.Fatal("expected non-zero time")
|
||||
}
|
||||
if ts.Unix() != 1711468800 {
|
||||
t.Errorf("unexpected unix = %d", ts.Unix())
|
||||
}
|
||||
|
||||
if !parseFeishuTimestamp("").IsZero() {
|
||||
t.Error("expected zero time for empty string")
|
||||
}
|
||||
if !parseFeishuTimestamp("invalid").IsZero() {
|
||||
t.Error("expected zero time for invalid string")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFeishuConfig(t *testing.T) {
|
||||
t.Run("valid", func(t *testing.T) {
|
||||
cfg, err := parseFeishuConfig(&types.DataSourceConfig{
|
||||
Credentials: map[string]interface{}{
|
||||
"app_id": "id1",
|
||||
"app_secret": "sec1",
|
||||
"base_url": "https://custom.example.com",
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if cfg.AppID != "id1" || cfg.AppSecret != "sec1" || cfg.BaseURL != "https://custom.example.com" {
|
||||
t.Errorf("unexpected config: %+v", cfg)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("nil config", func(t *testing.T) {
|
||||
_, err := parseFeishuConfig(nil)
|
||||
if err == nil {
|
||||
t.Fatal("expected error for nil config")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("missing credentials", func(t *testing.T) {
|
||||
_, err := parseFeishuConfig(&types.DataSourceConfig{
|
||||
Credentials: map[string]interface{}{
|
||||
"app_id": "id1",
|
||||
// missing app_secret
|
||||
},
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected error for missing app_secret")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// ──────────────────────────────────────────────────────────────────────
|
||||
// Connector interface tests
|
||||
// ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
func TestConnectorType(t *testing.T) {
|
||||
c := NewConnector()
|
||||
if c.Type() != types.ConnectorTypeFeishu {
|
||||
t.Errorf("Type() = %q, want %q", c.Type(), types.ConnectorTypeFeishu)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConnectorValidate(t *testing.T) {
|
||||
ts, cfg := fakeFeishu(nil)
|
||||
defer ts.Close()
|
||||
|
||||
c := NewConnector()
|
||||
err := c.Validate(context.Background(), makeConfig(cfg, nil))
|
||||
if err != nil {
|
||||
t.Fatalf("Validate() error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConnectorValidate_BadCredentials(t *testing.T) {
|
||||
c := NewConnector()
|
||||
err := c.Validate(context.Background(), &types.DataSourceConfig{
|
||||
Credentials: map[string]interface{}{
|
||||
"app_id": "bad",
|
||||
"app_secret": "bad",
|
||||
"base_url": "http://127.0.0.1:1", // will fail to connect
|
||||
},
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected error for bad credentials")
|
||||
}
|
||||
}
|
||||
|
||||
func TestConnectorListResources(t *testing.T) {
|
||||
ts, cfg := fakeFeishu(nil)
|
||||
defer ts.Close()
|
||||
|
||||
c := NewConnector()
|
||||
resources, err := c.ListResources(context.Background(), makeConfig(cfg, nil))
|
||||
if err != nil {
|
||||
t.Fatalf("ListResources() error: %v", err)
|
||||
}
|
||||
|
||||
if len(resources) != 1 {
|
||||
t.Fatalf("expected 1 resource, got %d", len(resources))
|
||||
}
|
||||
if resources[0].ExternalID != "space1" {
|
||||
t.Errorf("ExternalID = %q, want %q", resources[0].ExternalID, "space1")
|
||||
}
|
||||
if resources[0].Name != "Test Space" {
|
||||
t.Errorf("Name = %q, want %q", resources[0].Name, "Test Space")
|
||||
}
|
||||
if resources[0].Type != "wiki_space" {
|
||||
t.Errorf("Type = %q, want %q", resources[0].Type, "wiki_space")
|
||||
}
|
||||
}
|
||||
|
||||
// ──────────────────────────────────────────────────────────────────────
|
||||
// FetchAll: test all supported doc types
|
||||
// ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
func TestFetchAll_DocxNode(t *testing.T) {
|
||||
nodes := []wikiNode{{
|
||||
NodeToken: "nt1",
|
||||
ObjToken: "obj-docx-1",
|
||||
ObjType: "docx",
|
||||
Title: "My Document",
|
||||
NodeEditTime: "1711468800",
|
||||
}}
|
||||
ts, cfg := fakeFeishu(nodes)
|
||||
defer ts.Close()
|
||||
|
||||
c := NewConnector()
|
||||
items, err := c.FetchAll(context.Background(), makeConfig(cfg, []string{"space1"}), []string{"space1"})
|
||||
if err != nil {
|
||||
t.Fatalf("FetchAll() error: %v", err)
|
||||
}
|
||||
|
||||
if len(items) != 1 {
|
||||
t.Fatalf("expected 1 item, got %d", len(items))
|
||||
}
|
||||
|
||||
item := items[0]
|
||||
if item.ExternalID != "nt1" {
|
||||
t.Errorf("ExternalID = %q, want %q", item.ExternalID, "nt1")
|
||||
}
|
||||
if item.Title != "My Document" {
|
||||
t.Errorf("Title = %q", item.Title)
|
||||
}
|
||||
if string(item.Content) != "fake-docx-content" {
|
||||
t.Errorf("Content = %q, want %q", string(item.Content), "fake-docx-content")
|
||||
}
|
||||
if item.FileName != "exported.docx" {
|
||||
t.Errorf("FileName = %q, want %q", item.FileName, "exported.docx")
|
||||
}
|
||||
if item.Metadata["obj_type"] != "docx" {
|
||||
t.Errorf("Metadata[obj_type] = %q", item.Metadata["obj_type"])
|
||||
}
|
||||
if item.Metadata["channel"] != types.ChannelFeishu {
|
||||
t.Errorf("Metadata[channel] = %q", item.Metadata["channel"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchAll_SheetNode(t *testing.T) {
|
||||
nodes := []wikiNode{{
|
||||
NodeToken: "nt-sheet",
|
||||
ObjToken: "obj-sheet-1",
|
||||
ObjType: "sheet",
|
||||
Title: "Sales Report",
|
||||
NodeEditTime: "1711468800",
|
||||
}}
|
||||
ts, cfg := fakeFeishu(nodes)
|
||||
defer ts.Close()
|
||||
|
||||
c := NewConnector()
|
||||
items, err := c.FetchAll(context.Background(), makeConfig(cfg, []string{"space1"}), []string{"space1"})
|
||||
if err != nil {
|
||||
t.Fatalf("FetchAll() error: %v", err)
|
||||
}
|
||||
|
||||
if len(items) != 1 {
|
||||
t.Fatalf("expected 1 item, got %d", len(items))
|
||||
}
|
||||
if items[0].Metadata["obj_type"] != "sheet" {
|
||||
t.Errorf("obj_type = %q, want sheet", items[0].Metadata["obj_type"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchAll_BitableNode(t *testing.T) {
|
||||
nodes := []wikiNode{{
|
||||
NodeToken: "nt-bitable",
|
||||
ObjToken: "obj-bitable-1",
|
||||
ObjType: "bitable",
|
||||
Title: "Project Tracker",
|
||||
NodeEditTime: "1711468800",
|
||||
}}
|
||||
ts, cfg := fakeFeishu(nodes)
|
||||
defer ts.Close()
|
||||
|
||||
c := NewConnector()
|
||||
items, err := c.FetchAll(context.Background(), makeConfig(cfg, []string{"space1"}), []string{"space1"})
|
||||
if err != nil {
|
||||
t.Fatalf("FetchAll() error: %v", err)
|
||||
}
|
||||
if len(items) != 1 {
|
||||
t.Fatalf("expected 1 item, got %d", len(items))
|
||||
}
|
||||
if items[0].Metadata["obj_type"] != "bitable" {
|
||||
t.Errorf("obj_type = %q, want bitable", items[0].Metadata["obj_type"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchAll_FileNode(t *testing.T) {
|
||||
nodes := []wikiNode{{
|
||||
NodeToken: "nt-file",
|
||||
ObjToken: "obj-file-1",
|
||||
ObjType: "file",
|
||||
Title: "manual.pdf",
|
||||
NodeEditTime: "1711468800",
|
||||
}}
|
||||
ts, cfg := fakeFeishu(nodes)
|
||||
defer ts.Close()
|
||||
|
||||
c := NewConnector()
|
||||
items, err := c.FetchAll(context.Background(), makeConfig(cfg, []string{"space1"}), []string{"space1"})
|
||||
if err != nil {
|
||||
t.Fatalf("FetchAll() error: %v", err)
|
||||
}
|
||||
|
||||
if len(items) != 1 {
|
||||
t.Fatalf("expected 1 item, got %d", len(items))
|
||||
}
|
||||
|
||||
item := items[0]
|
||||
if string(item.Content) != "fake-pdf-binary" {
|
||||
t.Errorf("Content = %q, want %q", string(item.Content), "fake-pdf-binary")
|
||||
}
|
||||
if item.FileName != "manual.pdf" {
|
||||
t.Errorf("FileName = %q, want %q", item.FileName, "manual.pdf")
|
||||
}
|
||||
if item.Metadata["obj_type"] != "file" {
|
||||
t.Errorf("obj_type = %q, want file", item.Metadata["obj_type"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchAll_SkipsMindnoteAndSlides(t *testing.T) {
|
||||
nodes := []wikiNode{
|
||||
{NodeToken: "nt-mn", ObjToken: "obj-mn", ObjType: "mindnote", Title: "Brain Map"},
|
||||
{NodeToken: "nt-sl", ObjToken: "obj-sl", ObjType: "slides", Title: "Presentation"},
|
||||
}
|
||||
ts, cfg := fakeFeishu(nodes)
|
||||
defer ts.Close()
|
||||
|
||||
c := NewConnector()
|
||||
items, err := c.FetchAll(context.Background(), makeConfig(cfg, []string{"space1"}), []string{"space1"})
|
||||
if err != nil {
|
||||
t.Fatalf("FetchAll() error: %v", err)
|
||||
}
|
||||
|
||||
// Both should be skipped (nil returned by fetchNodeContent)
|
||||
if len(items) != 0 {
|
||||
t.Errorf("expected 0 items (mindnote+slides skipped), got %d", len(items))
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchAll_MixedTypes(t *testing.T) {
|
||||
nodes := []wikiNode{
|
||||
{NodeToken: "nt1", ObjToken: "obj1", ObjType: "docx", Title: "Doc", NodeEditTime: "1711468800"},
|
||||
{NodeToken: "nt2", ObjToken: "obj2", ObjType: "sheet", Title: "Sheet", NodeEditTime: "1711468800"},
|
||||
{NodeToken: "nt3", ObjToken: "obj3", ObjType: "file", Title: "report.pdf", NodeEditTime: "1711468800"},
|
||||
{NodeToken: "nt4", ObjToken: "obj4", ObjType: "mindnote", Title: "Mind", NodeEditTime: "1711468800"},
|
||||
{NodeToken: "nt5", ObjToken: "obj5", ObjType: "slides", Title: "Slides", NodeEditTime: "1711468800"},
|
||||
{NodeToken: "nt6", ObjToken: "obj6", ObjType: "bitable", Title: "Table", NodeEditTime: "1711468800"},
|
||||
}
|
||||
ts, cfg := fakeFeishu(nodes)
|
||||
defer ts.Close()
|
||||
|
||||
c := NewConnector()
|
||||
items, err := c.FetchAll(context.Background(), makeConfig(cfg, []string{"space1"}), []string{"space1"})
|
||||
if err != nil {
|
||||
t.Fatalf("FetchAll() error: %v", err)
|
||||
}
|
||||
|
||||
// docx + sheet + file + bitable = 4 items; mindnote + slides = skipped
|
||||
if len(items) != 4 {
|
||||
t.Errorf("expected 4 items, got %d", len(items))
|
||||
for i, it := range items {
|
||||
t.Logf(" item[%d]: %s (obj_type=%s)", i, it.Title, it.Metadata["obj_type"])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ──────────────────────────────────────────────────────────────────────
|
||||
// FetchIncremental tests
|
||||
// ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
func TestFetchIncremental_FirstSync(t *testing.T) {
|
||||
nodes := []wikiNode{
|
||||
{NodeToken: "nt1", ObjToken: "obj1", ObjType: "docx", Title: "Doc1", NodeEditTime: "100"},
|
||||
{NodeToken: "nt2", ObjToken: "obj2", ObjType: "file", Title: "file.pdf", NodeEditTime: "200"},
|
||||
}
|
||||
ts, cfg := fakeFeishu(nodes)
|
||||
defer ts.Close()
|
||||
|
||||
c := NewConnector()
|
||||
dsConfig := makeConfig(cfg, []string{"space1"})
|
||||
|
||||
// First sync with no cursor → all items should be fetched
|
||||
items, cursor, err := c.FetchIncremental(context.Background(), dsConfig, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("FetchIncremental() error: %v", err)
|
||||
}
|
||||
|
||||
if len(items) != 2 {
|
||||
t.Fatalf("expected 2 items on first sync, got %d", len(items))
|
||||
}
|
||||
if cursor == nil {
|
||||
t.Fatal("expected non-nil cursor")
|
||||
}
|
||||
if cursor.LastSyncTime.IsZero() {
|
||||
t.Error("cursor.LastSyncTime should not be zero")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchIncremental_NoChanges(t *testing.T) {
|
||||
nodes := []wikiNode{
|
||||
{NodeToken: "nt1", ObjToken: "obj1", ObjType: "docx", Title: "Doc1", NodeEditTime: "100"},
|
||||
}
|
||||
ts, cfg := fakeFeishu(nodes)
|
||||
defer ts.Close()
|
||||
|
||||
c := NewConnector()
|
||||
dsConfig := makeConfig(cfg, []string{"space1"})
|
||||
|
||||
// First sync
|
||||
_, cursor1, err := c.FetchIncremental(context.Background(), dsConfig, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("first sync error: %v", err)
|
||||
}
|
||||
|
||||
// Second sync with same edit times → should return 0 changed items
|
||||
items, _, err := c.FetchIncremental(context.Background(), dsConfig, cursor1)
|
||||
if err != nil {
|
||||
t.Fatalf("second sync error: %v", err)
|
||||
}
|
||||
|
||||
if len(items) != 0 {
|
||||
t.Errorf("expected 0 items (no changes), got %d", len(items))
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchIncremental_DetectsDeleted(t *testing.T) {
|
||||
// First sync: 2 nodes
|
||||
allNodes := []wikiNode{
|
||||
{NodeToken: "nt1", ObjToken: "obj1", ObjType: "docx", Title: "Doc1", NodeEditTime: "100"},
|
||||
{NodeToken: "nt2", ObjToken: "obj2", ObjType: "docx", Title: "Doc2", NodeEditTime: "200"},
|
||||
}
|
||||
ts, cfg := fakeFeishu(allNodes)
|
||||
|
||||
c := NewConnector()
|
||||
dsConfig := makeConfig(cfg, []string{"space1"})
|
||||
|
||||
_, cursor1, err := c.FetchIncremental(context.Background(), dsConfig, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("first sync error: %v", err)
|
||||
}
|
||||
ts.Close()
|
||||
|
||||
// Second sync: only 1 node remains (nt2 was deleted)
|
||||
ts2, cfg2 := fakeFeishu([]wikiNode{
|
||||
{NodeToken: "nt1", ObjToken: "obj1", ObjType: "docx", Title: "Doc1", NodeEditTime: "100"},
|
||||
})
|
||||
defer ts2.Close()
|
||||
dsConfig2 := makeConfig(cfg2, []string{"space1"})
|
||||
|
||||
items, _, err := c.FetchIncremental(context.Background(), dsConfig2, cursor1)
|
||||
if err != nil {
|
||||
t.Fatalf("second sync error: %v", err)
|
||||
}
|
||||
|
||||
// Should have 1 deleted item for nt2
|
||||
var deletedCount int
|
||||
for _, item := range items {
|
||||
if item.IsDeleted {
|
||||
deletedCount++
|
||||
if item.ExternalID != "nt2" {
|
||||
t.Errorf("expected deleted ExternalID=nt2, got %q", item.ExternalID)
|
||||
}
|
||||
}
|
||||
}
|
||||
if deletedCount != 1 {
|
||||
t.Errorf("expected 1 deleted item, got %d", deletedCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchIncremental_NoResourceIDs(t *testing.T) {
|
||||
ts, cfg := fakeFeishu(nil)
|
||||
defer ts.Close()
|
||||
|
||||
c := NewConnector()
|
||||
dsConfig := makeConfig(cfg, nil) // no resource IDs
|
||||
dsConfig.ResourceIDs = nil
|
||||
|
||||
_, _, err := c.FetchIncremental(context.Background(), dsConfig, nil)
|
||||
if err == nil {
|
||||
t.Fatal("expected error for empty resource IDs")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "no resource IDs") {
|
||||
t.Errorf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// ──────────────────────────────────────────────────────────────────────
|
||||
// Client tests
|
||||
// ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
func TestClientPing(t *testing.T) {
|
||||
ts, cfg := fakeFeishu(nil)
|
||||
defer ts.Close()
|
||||
|
||||
client := NewClient(cfg)
|
||||
if err := client.Ping(context.Background()); err != nil {
|
||||
t.Fatalf("Ping() error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClientTokenCaching(t *testing.T) {
|
||||
callCount := 0
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/open-apis/auth/v3/tenant_access_token/internal", func(w http.ResponseWriter, r *http.Request) {
|
||||
callCount++
|
||||
writeJSON(w, tokenResponse{
|
||||
apiResponse: apiResponse{Code: 0},
|
||||
TenantAccessToken: fmt.Sprintf("token-%d", callCount),
|
||||
Expire: 7200,
|
||||
})
|
||||
})
|
||||
ts := httptest.NewServer(mux)
|
||||
defer ts.Close()
|
||||
|
||||
client := NewClient(&Config{AppID: "a", AppSecret: "b", BaseURL: ts.URL})
|
||||
|
||||
// First call: fetches token
|
||||
t1, _ := client.getTenantAccessToken(context.Background())
|
||||
// Second call: should use cache
|
||||
t2, _ := client.getTenantAccessToken(context.Background())
|
||||
|
||||
if t1 != t2 {
|
||||
t.Errorf("expected cached token, got different tokens: %q vs %q", t1, t2)
|
||||
}
|
||||
if callCount != 1 {
|
||||
t.Errorf("expected 1 auth call, got %d", callCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClientExportAndDownload(t *testing.T) {
|
||||
ts, cfg := fakeFeishu(nil)
|
||||
defer ts.Close()
|
||||
|
||||
client := NewClient(cfg)
|
||||
data, fileName, err := client.ExportAndDownload(context.Background(), "obj-token-1", "docx")
|
||||
if err != nil {
|
||||
t.Fatalf("ExportAndDownload() error: %v", err)
|
||||
}
|
||||
|
||||
if string(data) != "fake-docx-content" {
|
||||
t.Errorf("data = %q, want %q", string(data), "fake-docx-content")
|
||||
}
|
||||
if fileName != "exported.docx" {
|
||||
t.Errorf("fileName = %q, want %q", fileName, "exported.docx")
|
||||
}
|
||||
}
|
||||
|
||||
func TestClientExportAndDownload_UnsupportedType(t *testing.T) {
|
||||
ts, cfg := fakeFeishu(nil)
|
||||
defer ts.Close()
|
||||
|
||||
client := NewClient(cfg)
|
||||
_, _, err := client.ExportAndDownload(context.Background(), "obj-token-1", "mindnote")
|
||||
if err == nil {
|
||||
t.Fatal("expected error for unsupported type")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "unsupported") {
|
||||
t.Errorf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClientDownloadDriveFile(t *testing.T) {
|
||||
ts, cfg := fakeFeishu(nil)
|
||||
defer ts.Close()
|
||||
|
||||
client := NewClient(cfg)
|
||||
data, err := client.DownloadDriveFile(context.Background(), "file-token-abc")
|
||||
if err != nil {
|
||||
t.Fatalf("DownloadDriveFile() error: %v", err)
|
||||
}
|
||||
|
||||
if string(data) != "fake-pdf-binary" {
|
||||
t.Errorf("data = %q, want %q", string(data), "fake-pdf-binary")
|
||||
}
|
||||
}
|
||||
|
||||
func TestClientListWikiSpaces(t *testing.T) {
|
||||
ts, cfg := fakeFeishu(nil)
|
||||
defer ts.Close()
|
||||
|
||||
client := NewClient(cfg)
|
||||
spaces, err := client.ListWikiSpaces(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("ListWikiSpaces() error: %v", err)
|
||||
}
|
||||
if len(spaces) != 1 {
|
||||
t.Fatalf("expected 1 space, got %d", len(spaces))
|
||||
}
|
||||
if spaces[0].SpaceID != "space1" {
|
||||
t.Errorf("SpaceID = %q", spaces[0].SpaceID)
|
||||
}
|
||||
}
|
||||
|
||||
// ──────────────────────────────────────────────────────────────────────
|
||||
// Type mapping tests
|
||||
// ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
func TestObjTypeToExportMappings(t *testing.T) {
|
||||
// Verify all exportable types have valid mappings
|
||||
exportable := []string{"docx", "doc", "sheet", "bitable"}
|
||||
for _, ot := range exportable {
|
||||
if _, ok := objTypeToExportFileExtension[ot]; !ok {
|
||||
t.Errorf("objTypeToExportFileExtension missing %q", ot)
|
||||
}
|
||||
if _, ok := objTypeToExportType[ot]; !ok {
|
||||
t.Errorf("objTypeToExportType missing %q", ot)
|
||||
}
|
||||
}
|
||||
|
||||
// Verify non-exportable types do NOT have mappings
|
||||
nonExportable := []string{"file", "mindnote", "slides"}
|
||||
for _, ot := range nonExportable {
|
||||
if _, ok := objTypeToExportFileExtension[ot]; ok {
|
||||
t.Errorf("objTypeToExportFileExtension should NOT contain %q", ot)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestExportFileExtToSuffix(t *testing.T) {
|
||||
if exportFileExtToSuffix[ExportTypeDocx] != ".docx" {
|
||||
t.Errorf("docx suffix = %q", exportFileExtToSuffix[ExportTypeDocx])
|
||||
}
|
||||
if exportFileExtToSuffix[ExportTypeXlsx] != ".xlsx" {
|
||||
t.Errorf("xlsx suffix = %q", exportFileExtToSuffix[ExportTypeXlsx])
|
||||
}
|
||||
if exportFileExtToSuffix[ExportTypePDF] != ".pdf" {
|
||||
t.Errorf("pdf suffix = %q", exportFileExtToSuffix[ExportTypePDF])
|
||||
}
|
||||
}
|
||||
|
||||
// ──────────────────────────────────────────────────────────────────────
|
||||
// Feishu cursor serialization
|
||||
// ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
func TestFeishuCursorRoundTrip(t *testing.T) {
|
||||
original := feishuCursor{
|
||||
LastSyncTime: time.Date(2026, 3, 26, 12, 0, 0, 0, time.UTC),
|
||||
SpaceNodeTimes: map[string]map[string]string{
|
||||
"space1": {
|
||||
"nt1": "100",
|
||||
"nt2": "200",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
// Serialize to map[string]interface{} (as stored in SyncCursor.ConnectorCursor)
|
||||
data, err := json.Marshal(original)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal error: %v", err)
|
||||
}
|
||||
|
||||
var cursorMap map[string]interface{}
|
||||
if err := json.Unmarshal(data, &cursorMap); err != nil {
|
||||
t.Fatalf("unmarshal to map error: %v", err)
|
||||
}
|
||||
|
||||
// Deserialize back
|
||||
data2, _ := json.Marshal(cursorMap)
|
||||
var restored feishuCursor
|
||||
if err := json.Unmarshal(data2, &restored); err != nil {
|
||||
t.Fatalf("restore error: %v", err)
|
||||
}
|
||||
|
||||
if restored.SpaceNodeTimes["space1"]["nt1"] != "100" {
|
||||
t.Errorf("restored nt1 = %q, want 100", restored.SpaceNodeTimes["space1"]["nt1"])
|
||||
}
|
||||
if restored.SpaceNodeTimes["space1"]["nt2"] != "200" {
|
||||
t.Errorf("restored nt2 = %q, want 200", restored.SpaceNodeTimes["space1"]["nt2"])
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,206 @@
|
||||
// Package feishu implements the Feishu (飞书/Lark) data source connector for WeKnora.
|
||||
//
|
||||
// It syncs documents from Feishu Wiki spaces and cloud documents into WeKnora knowledge bases.
|
||||
//
|
||||
// Feishu API docs:
|
||||
// - Wiki spaces: https://open.feishu.cn/document/server-docs/docs/wiki-v2/space/list
|
||||
// - Wiki nodes: https://open.feishu.cn/document/server-docs/docs/wiki-v2/space-node/list
|
||||
// - Export tasks: https://open.feishu.cn/document/server-docs/docs/drive-v1/export_task/export-user-guide
|
||||
// - File download: https://open.feishu.cn/document/server-docs/docs/drive-v1/file/download
|
||||
// - Auth: https://open.feishu.cn/document/server-docs/authentication-management/access-token/tenant_access_token_internal
|
||||
package feishu
|
||||
|
||||
import "time"
|
||||
|
||||
// Config holds Feishu-specific configuration for the data source connector.
|
||||
// Uses the self-built app (企业自建应用) authentication model.
|
||||
type Config struct {
|
||||
// App ID from Feishu developer console
|
||||
AppID string `json:"app_id"`
|
||||
|
||||
// App Secret from Feishu developer console
|
||||
AppSecret string `json:"app_secret"`
|
||||
|
||||
// Base URL for Feishu API (default: https://open.feishu.cn)
|
||||
// Use https://open.larksuite.com for Lark (international) deployments
|
||||
BaseURL string `json:"base_url,omitempty"`
|
||||
}
|
||||
|
||||
// DefaultBaseURL is the default Feishu Open Platform API base URL.
|
||||
const DefaultBaseURL = "https://open.feishu.cn"
|
||||
|
||||
// LarkBaseURL is the Lark (international) API base URL.
|
||||
const LarkBaseURL = "https://open.larksuite.com"
|
||||
|
||||
// GetBaseURL returns the effective base URL, defaulting to Feishu if not set.
|
||||
func (c *Config) GetBaseURL() string {
|
||||
if c.BaseURL != "" {
|
||||
return c.BaseURL
|
||||
}
|
||||
return DefaultBaseURL
|
||||
}
|
||||
|
||||
// --- Export format constants ---
|
||||
// Used by the export task API: POST /drive/v1/export_tasks
|
||||
|
||||
const (
|
||||
// ExportTypeDocx exports Feishu documents to .docx format.
|
||||
ExportTypeDocx = "docx"
|
||||
// ExportTypeXlsx exports spreadsheets / bitable to .xlsx format.
|
||||
ExportTypeXlsx = "xlsx"
|
||||
// ExportTypePDF exports documents to .pdf format (fallback).
|
||||
ExportTypePDF = "pdf"
|
||||
)
|
||||
|
||||
// objTypeToExportFileExtension maps Feishu obj_type to the best export file_extension.
|
||||
var objTypeToExportFileExtension = map[string]string{
|
||||
"docx": ExportTypeDocx,
|
||||
"doc": ExportTypeDocx,
|
||||
"sheet": ExportTypeXlsx,
|
||||
"bitable": ExportTypeXlsx,
|
||||
}
|
||||
|
||||
// objTypeToExportType maps Feishu obj_type to the export API "type" parameter.
|
||||
// See: https://open.feishu.cn/document/server-docs/docs/drive-v1/export_task/create
|
||||
var objTypeToExportType = map[string]string{
|
||||
"docx": "docx",
|
||||
"doc": "doc",
|
||||
"sheet": "sheet",
|
||||
"bitable": "bitable",
|
||||
}
|
||||
|
||||
// exportFileExtToSuffix maps export file_extension to the file suffix for FileName.
|
||||
var exportFileExtToSuffix = map[string]string{
|
||||
ExportTypeDocx: ".docx",
|
||||
ExportTypeXlsx: ".xlsx",
|
||||
ExportTypePDF: ".pdf",
|
||||
}
|
||||
|
||||
// --- Feishu API response structures ---
|
||||
|
||||
// apiResponse is the common Feishu API response wrapper.
|
||||
type apiResponse struct {
|
||||
Code int `json:"code"`
|
||||
Msg string `json:"msg"`
|
||||
}
|
||||
|
||||
// tokenResponse is the response for tenant_access_token API.
|
||||
type tokenResponse struct {
|
||||
apiResponse
|
||||
TenantAccessToken string `json:"tenant_access_token"`
|
||||
Expire int `json:"expire"` // seconds
|
||||
}
|
||||
|
||||
// wikiSpaceListResponse is the response for GET /open-apis/wiki/v2/spaces.
|
||||
type wikiSpaceListResponse struct {
|
||||
apiResponse
|
||||
Data struct {
|
||||
Items []wikiSpace `json:"items"`
|
||||
HasMore bool `json:"has_more"`
|
||||
PageToken string `json:"page_token"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
// wikiSpace represents a Feishu Wiki space.
|
||||
type wikiSpace struct {
|
||||
SpaceID string `json:"space_id"`
|
||||
Name string `json:"name"`
|
||||
Description string `json:"description"`
|
||||
Visibility string `json:"visibility"` // "public" or "private"
|
||||
}
|
||||
|
||||
// wikiNodeListResponse is the response for GET /open-apis/wiki/v2/spaces/:space_id/nodes.
|
||||
type wikiNodeListResponse struct {
|
||||
apiResponse
|
||||
Data struct {
|
||||
Items []wikiNode `json:"items"`
|
||||
HasMore bool `json:"has_more"`
|
||||
PageToken string `json:"page_token"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
// wikiNode represents a node (document or folder) in a Feishu Wiki space.
|
||||
type wikiNode struct {
|
||||
SpaceID string `json:"space_id"`
|
||||
NodeToken string `json:"node_token"`
|
||||
ObjToken string `json:"obj_token"` // document token
|
||||
ObjType string `json:"obj_type"` // "doc", "sheet", "mindnote", "bitable", "file", "docx", "slides"
|
||||
ParentNodeID string `json:"parent_node_id"`
|
||||
NodeType string `json:"node_type"` // "origin" or "shortcut"
|
||||
OriginNodeID string `json:"origin_node_id"`
|
||||
OriginSpaceID string `json:"origin_space_id"`
|
||||
HasChild bool `json:"has_child"`
|
||||
Title string `json:"title"`
|
||||
Creator string `json:"creator"`
|
||||
Owner string `json:"owner"`
|
||||
ObjCreateTime string `json:"obj_create_time"` // document creation time (unix timestamp string)
|
||||
ObjEditTime string `json:"obj_edit_time"` // document last edit time (unix timestamp string) — tracks content changes
|
||||
NodeCreateTime string `json:"node_create_time"` // node creation time (unix timestamp string)
|
||||
NodeEditTime string `json:"node_edit_time"` // node edit time (unix timestamp string) — only tracks node attribute changes
|
||||
}
|
||||
|
||||
// wikiNodeInfoResponse is the response for GET /open-apis/wiki/v2/spaces/get_node.
|
||||
type wikiNodeInfoResponse struct {
|
||||
apiResponse
|
||||
Data struct {
|
||||
Node wikiNode `json:"node"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
// --- Export task API responses ---
|
||||
|
||||
// docRawContentResponse is the response for GET /open-apis/docx/v1/documents/:document_id/raw_content.
|
||||
// Deprecated: prefer export API for full-fidelity document export.
|
||||
type docRawContentResponse struct {
|
||||
apiResponse
|
||||
Data struct {
|
||||
Content string `json:"content"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
// exportTaskCreateResponse is the response for POST /drive/v1/export_tasks.
|
||||
type exportTaskCreateResponse struct {
|
||||
apiResponse
|
||||
Data struct {
|
||||
Ticket string `json:"ticket"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
// exportTaskStatusResponse is the response for GET /drive/v1/export_tasks/{ticket}.
|
||||
type exportTaskStatusResponse struct {
|
||||
apiResponse
|
||||
Data struct {
|
||||
Result struct {
|
||||
FileToken string `json:"file_token"`
|
||||
FileSize int64 `json:"file_size"`
|
||||
// JobStatus: 0=success, 1=initializing, 2=processing
|
||||
JobStatus int `json:"job_status"`
|
||||
JobErrorMsg string `json:"job_error_msg"`
|
||||
FileName string `json:"file_name"`
|
||||
} `json:"result"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
// --- File download response ---
|
||||
|
||||
// driveFileMetaResponse is the response for GET /drive/v1/metas for file type nodes.
|
||||
type driveFileMetaResponse struct {
|
||||
apiResponse
|
||||
Data struct {
|
||||
Metas []struct {
|
||||
DocToken string `json:"doc_token"`
|
||||
DocType string `json:"doc_type"`
|
||||
Title string `json:"title"`
|
||||
} `json:"metas"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
// feishuCursor stores incremental sync state for Feishu.
|
||||
type feishuCursor struct {
|
||||
// LastSyncTime is the timestamp of the last successful sync.
|
||||
LastSyncTime time.Time `json:"last_sync_time"`
|
||||
|
||||
// SpaceNodeTimes maps space_id -> node_token -> last known edit time.
|
||||
// Used to detect which nodes have changed since last sync.
|
||||
SpaceNodeTimes map[string]map[string]string `json:"space_node_times,omitempty"`
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
package datasource
|
||||
|
||||
import "errors"
|
||||
|
||||
// Error definitions for datasource operations
|
||||
var (
|
||||
// Connector errors
|
||||
ErrConnectorNil = errors.New("connector is nil")
|
||||
ErrConnectorTypeEmpty = errors.New("connector type is empty")
|
||||
ErrConnectorNotFound = errors.New("connector type not found in registry")
|
||||
|
||||
// DataSource errors
|
||||
ErrDataSourceNotFound = errors.New("data source not found")
|
||||
ErrDataSourceInvalid = errors.New("data source configuration is invalid")
|
||||
ErrDataSourceNotActive = errors.New("data source is not active")
|
||||
|
||||
// Configuration errors
|
||||
ErrInvalidConfig = errors.New("invalid configuration")
|
||||
ErrInvalidCredentials = errors.New("invalid credentials")
|
||||
|
||||
// Sync errors
|
||||
ErrSyncFailed = errors.New("sync operation failed")
|
||||
ErrSyncCanceled = errors.New("sync operation was canceled")
|
||||
ErrFetchFailed = errors.New("failed to fetch items from source")
|
||||
ErrResourceNotFound = errors.New("resource not found in source system")
|
||||
|
||||
// Knowledge base errors
|
||||
ErrKnowledgeBaseNotFound = errors.New("knowledge base not found")
|
||||
|
||||
// Sync log errors
|
||||
ErrSyncLogNotFound = errors.New("sync log not found")
|
||||
)
|
||||
@@ -0,0 +1,208 @@
|
||||
package datasource
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
"github.com/hibiken/asynq"
|
||||
"github.com/robfig/cron/v3"
|
||||
)
|
||||
|
||||
// Scheduler manages cron-based periodic sync for data sources.
|
||||
//
|
||||
// robfig/cron fires at absolute wall-clock times (e.g. "0 0 * * * *" always fires
|
||||
// at the top of every hour regardless of when the process started). So multiple
|
||||
// instances will fire at the same moment. Dedup is handled by two layers:
|
||||
//
|
||||
// 1. HasRunningSync — if a previous sync is still running, skip (prevent overlap).
|
||||
// 2. asynq.TaskID — deterministic ID per (dataSourceID, minute). Redis ensures
|
||||
// only one task with a given ID is enqueued. Losers get ErrTaskIDConflict.
|
||||
type Scheduler struct {
|
||||
cron *cron.Cron
|
||||
dsRepo interfaces.DataSourceRepository
|
||||
syncLogRepo interfaces.SyncLogRepository
|
||||
taskEnqueuer interfaces.TaskEnqueuer
|
||||
|
||||
mu sync.Mutex
|
||||
entries map[string]cron.EntryID // dataSourceID → cron entry ID
|
||||
}
|
||||
|
||||
// NewScheduler creates a new Scheduler.
|
||||
func NewScheduler(
|
||||
dsRepo interfaces.DataSourceRepository,
|
||||
syncLogRepo interfaces.SyncLogRepository,
|
||||
taskEnqueuer interfaces.TaskEnqueuer,
|
||||
) *Scheduler {
|
||||
return &Scheduler{
|
||||
cron: cron.New(cron.WithSeconds(), cron.WithChain(
|
||||
cron.Recover(cron.DefaultLogger),
|
||||
)),
|
||||
dsRepo: dsRepo,
|
||||
syncLogRepo: syncLogRepo,
|
||||
taskEnqueuer: taskEnqueuer,
|
||||
entries: make(map[string]cron.EntryID),
|
||||
}
|
||||
}
|
||||
|
||||
// Start loads all active data sources from the database and registers their
|
||||
// cron schedules. Then starts the cron runner in the background.
|
||||
func (s *Scheduler) Start(ctx context.Context) error {
|
||||
dataSources, err := s.dsRepo.FindActive(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("load active data sources: %w", err)
|
||||
}
|
||||
|
||||
for _, ds := range dataSources {
|
||||
if ds.SyncSchedule == "" {
|
||||
continue
|
||||
}
|
||||
if err := s.addEntry(ds); err != nil {
|
||||
logger.Warnf(ctx, "[Scheduler] failed to register cron for ds=%s schedule=%q: %v",
|
||||
ds.ID, ds.SyncSchedule, err)
|
||||
}
|
||||
}
|
||||
|
||||
s.cron.Start()
|
||||
logger.Infof(ctx, "[Scheduler] started with %d cron entries", len(s.entries))
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stop gracefully stops the cron runner and waits for running jobs to finish.
|
||||
func (s *Scheduler) Stop() {
|
||||
ctx := s.cron.Stop()
|
||||
<-ctx.Done()
|
||||
}
|
||||
|
||||
// AddOrUpdate registers (or re-registers) a cron entry for the given data source.
|
||||
func (s *Scheduler) AddOrUpdate(ds *types.DataSource) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
if entryID, ok := s.entries[ds.ID]; ok {
|
||||
s.cron.Remove(entryID)
|
||||
delete(s.entries, ds.ID)
|
||||
}
|
||||
|
||||
if ds.Status != types.DataSourceStatusActive || ds.SyncSchedule == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
return s.addEntryLocked(ds)
|
||||
}
|
||||
|
||||
// Remove removes the cron entry for a data source.
|
||||
func (s *Scheduler) Remove(dataSourceID string) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
if entryID, ok := s.entries[dataSourceID]; ok {
|
||||
s.cron.Remove(entryID)
|
||||
delete(s.entries, dataSourceID)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Scheduler) addEntry(ds *types.DataSource) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.addEntryLocked(ds)
|
||||
}
|
||||
|
||||
func (s *Scheduler) addEntryLocked(ds *types.DataSource) error {
|
||||
dsID := ds.ID
|
||||
tenantID := ds.TenantID
|
||||
|
||||
entryID, err := s.cron.AddFunc(ds.SyncSchedule, func() {
|
||||
s.triggerSync(dsID, tenantID)
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid cron expression %q: %w", ds.SyncSchedule, err)
|
||||
}
|
||||
|
||||
s.entries[dsID] = entryID
|
||||
return nil
|
||||
}
|
||||
|
||||
// triggerSync is called by the cron runner on each tick.
|
||||
//
|
||||
// Layer 1 — DB: if a previous sync is still running, skip. This prevents
|
||||
// overlap when a sync takes longer than the cron interval.
|
||||
//
|
||||
// Layer 2 — Redis: deterministic asynq.TaskID = "dssync:<dsID>:<minute>".
|
||||
// Since robfig/cron fires at absolute wall-clock times, all instances trigger
|
||||
// at the same minute. The first Enqueue wins; others get ErrTaskIDConflict.
|
||||
func (s *Scheduler) triggerSync(dataSourceID string, tenantID uint64) {
|
||||
ctx := context.Background()
|
||||
|
||||
ds, err := s.dsRepo.FindByID(ctx, dataSourceID)
|
||||
if err != nil || ds == nil || ds.Status != types.DataSourceStatusActive {
|
||||
logger.Infof(ctx, "[Scheduler] skipping sync for ds=%s (not active or not found)", dataSourceID)
|
||||
return
|
||||
}
|
||||
|
||||
// Layer 1: prevent overlap with a still-running sync
|
||||
if running, _ := s.syncLogRepo.HasRunningSync(ctx, dataSourceID); running {
|
||||
logger.Infof(ctx, "[Scheduler] skipping sync for ds=%s (previous sync still running)", dataSourceID)
|
||||
return
|
||||
}
|
||||
|
||||
syncLog := &types.SyncLog{
|
||||
DataSourceID: dataSourceID,
|
||||
TenantID: tenantID,
|
||||
Status: types.SyncLogStatusRunning,
|
||||
StartedAt: time.Now(),
|
||||
}
|
||||
if err := s.syncLogRepo.Create(ctx, syncLog); err != nil {
|
||||
logger.Errorf(ctx, "[Scheduler] failed to create sync log for ds=%s: %v", dataSourceID, err)
|
||||
return
|
||||
}
|
||||
|
||||
payload := &types.DataSourceSyncPayload{
|
||||
DataSourceID: dataSourceID,
|
||||
TenantID: tenantID,
|
||||
SyncLogID: syncLog.ID,
|
||||
ForceFull: false,
|
||||
}
|
||||
payloadJSON, _ := json.Marshal(payload)
|
||||
task := asynq.NewTask(types.TypeDataSourceSync, payloadJSON)
|
||||
|
||||
// Layer 2: deterministic TaskID — all instances in the same minute produce the same ID
|
||||
taskID := fmt.Sprintf("dssync:%s:%s", dataSourceID, time.Now().UTC().Truncate(time.Minute).Format("200601021504"))
|
||||
|
||||
_, err = s.taskEnqueuer.Enqueue(task,
|
||||
asynq.Queue("low"),
|
||||
asynq.TaskID(taskID),
|
||||
)
|
||||
if err != nil {
|
||||
if err == asynq.ErrTaskIDConflict {
|
||||
logger.Infof(ctx, "[Scheduler] sync already enqueued by another instance for ds=%s", dataSourceID)
|
||||
syncLog.Status = types.SyncLogStatusCanceled
|
||||
now := time.Now()
|
||||
syncLog.FinishedAt = &now
|
||||
syncLog.ErrorMessage = "deduplicated: another instance enqueued first"
|
||||
_ = s.syncLogRepo.Update(ctx, syncLog)
|
||||
return
|
||||
}
|
||||
logger.Errorf(ctx, "[Scheduler] failed to enqueue sync task for ds=%s: %v", dataSourceID, err)
|
||||
syncLog.Status = types.SyncLogStatusFailed
|
||||
now := time.Now()
|
||||
syncLog.FinishedAt = &now
|
||||
syncLog.ErrorMessage = fmt.Sprintf("enqueue failed: %v", err)
|
||||
_ = s.syncLogRepo.Update(ctx, syncLog)
|
||||
return
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "[Scheduler] sync task enqueued for ds=%s syncLog=%s", dataSourceID, syncLog.ID)
|
||||
}
|
||||
|
||||
// EntryCount returns the number of active cron entries (for testing/monitoring).
|
||||
func (s *Scheduler) EntryCount() int {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return len(s.entries)
|
||||
}
|
||||
@@ -0,0 +1,356 @@
|
||||
package datasource
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/hibiken/asynq"
|
||||
)
|
||||
|
||||
// ──────────────────────────────────────────────────────────────────────
|
||||
// Fake implementations for testing
|
||||
// ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
// fakeDataSourceRepo is an in-memory DataSourceRepository.
|
||||
type fakeDataSourceRepo struct {
|
||||
mu sync.Mutex
|
||||
dataSources map[string]*types.DataSource
|
||||
}
|
||||
|
||||
func newFakeDataSourceRepo() *fakeDataSourceRepo {
|
||||
return &fakeDataSourceRepo{dataSources: make(map[string]*types.DataSource)}
|
||||
}
|
||||
|
||||
func (r *fakeDataSourceRepo) Create(_ context.Context, ds *types.DataSource) error {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.dataSources[ds.ID] = ds
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *fakeDataSourceRepo) FindByID(_ context.Context, id string) (*types.DataSource, error) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
ds, ok := r.dataSources[id]
|
||||
if !ok {
|
||||
return nil, ErrDataSourceNotFound
|
||||
}
|
||||
return ds, nil
|
||||
}
|
||||
|
||||
func (r *fakeDataSourceRepo) FindByKnowledgeBase(_ context.Context, kbID string) ([]*types.DataSource, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (r *fakeDataSourceRepo) Update(_ context.Context, ds *types.DataSource) error {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.dataSources[ds.ID] = ds
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *fakeDataSourceRepo) Delete(_ context.Context, id string) error {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
delete(r.dataSources, id)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *fakeDataSourceRepo) FindActive(_ context.Context) ([]*types.DataSource, error) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
var result []*types.DataSource
|
||||
for _, ds := range r.dataSources {
|
||||
if ds.Status == types.DataSourceStatusActive && ds.SyncSchedule != "" {
|
||||
result = append(result, ds)
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// fakeSyncLogRepo is an in-memory SyncLogRepository.
|
||||
type fakeSyncLogRepo struct {
|
||||
mu sync.Mutex
|
||||
logs map[string]*types.SyncLog
|
||||
}
|
||||
|
||||
func newFakeSyncLogRepo() *fakeSyncLogRepo {
|
||||
return &fakeSyncLogRepo{logs: make(map[string]*types.SyncLog)}
|
||||
}
|
||||
|
||||
func (r *fakeSyncLogRepo) Create(_ context.Context, log *types.SyncLog) error {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if log.ID == "" {
|
||||
log.ID = "log-" + time.Now().Format("150405.000")
|
||||
}
|
||||
r.logs[log.ID] = log
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *fakeSyncLogRepo) FindByID(_ context.Context, id string) (*types.SyncLog, error) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
l, ok := r.logs[id]
|
||||
if !ok {
|
||||
return nil, ErrSyncLogNotFound
|
||||
}
|
||||
return l, nil
|
||||
}
|
||||
|
||||
func (r *fakeSyncLogRepo) FindByDataSource(_ context.Context, dsID string, limit, offset int) ([]*types.SyncLog, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (r *fakeSyncLogRepo) FindLatest(_ context.Context, dsID string) (*types.SyncLog, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (r *fakeSyncLogRepo) Update(_ context.Context, log *types.SyncLog) error {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.logs[log.ID] = log
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *fakeSyncLogRepo) CleanupOldLogs(_ context.Context, retentionDays int) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *fakeSyncLogRepo) HasRunningSync(_ context.Context, dsID string) (bool, error) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
for _, log := range r.logs {
|
||||
if log.DataSourceID == dsID && log.Status == types.SyncLogStatusRunning {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// fakeTaskEnqueuer counts how many tasks are enqueued.
|
||||
type fakeTaskEnqueuer struct {
|
||||
count atomic.Int64
|
||||
}
|
||||
|
||||
func (e *fakeTaskEnqueuer) Enqueue(task *asynq.Task, opts ...asynq.Option) (*asynq.TaskInfo, error) {
|
||||
e.count.Add(1)
|
||||
return &asynq.TaskInfo{ID: "task-fake"}, nil
|
||||
}
|
||||
|
||||
// ──────────────────────────────────────────────────────────────────────
|
||||
// Tests
|
||||
// ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
func TestScheduler_StartWithActiveDataSources(t *testing.T) {
|
||||
repo := newFakeDataSourceRepo()
|
||||
_ = repo.Create(context.Background(), &types.DataSource{
|
||||
ID: "ds-1",
|
||||
TenantID: 1,
|
||||
Status: types.DataSourceStatusActive,
|
||||
SyncSchedule: "*/2 * * * * *", // every 2 seconds (6-field cron with seconds)
|
||||
})
|
||||
_ = repo.Create(context.Background(), &types.DataSource{
|
||||
ID: "ds-2",
|
||||
TenantID: 1,
|
||||
Status: types.DataSourceStatusPaused, // should NOT be scheduled
|
||||
SyncSchedule: "*/2 * * * * *",
|
||||
})
|
||||
|
||||
enqueuer := &fakeTaskEnqueuer{}
|
||||
scheduler := NewScheduler(repo, newFakeSyncLogRepo(), enqueuer)
|
||||
|
||||
if err := scheduler.Start(context.Background()); err != nil {
|
||||
t.Fatalf("Start() error: %v", err)
|
||||
}
|
||||
defer scheduler.Stop()
|
||||
|
||||
// Only ds-1 should be registered (ds-2 is paused, not returned by FindActive)
|
||||
if scheduler.EntryCount() != 1 {
|
||||
t.Errorf("EntryCount() = %d, want 1", scheduler.EntryCount())
|
||||
}
|
||||
}
|
||||
|
||||
func TestScheduler_CronFires(t *testing.T) {
|
||||
repo := newFakeDataSourceRepo()
|
||||
_ = repo.Create(context.Background(), &types.DataSource{
|
||||
ID: "ds-fire",
|
||||
TenantID: 1,
|
||||
Status: types.DataSourceStatusActive,
|
||||
SyncSchedule: "* * * * * *", // every second
|
||||
})
|
||||
|
||||
enqueuer := &fakeTaskEnqueuer{}
|
||||
scheduler := NewScheduler(repo, newFakeSyncLogRepo(), enqueuer)
|
||||
|
||||
if err := scheduler.Start(context.Background()); err != nil {
|
||||
t.Fatalf("Start() error: %v", err)
|
||||
}
|
||||
|
||||
// Wait for at least one tick
|
||||
time.Sleep(2500 * time.Millisecond)
|
||||
scheduler.Stop()
|
||||
|
||||
if enqueuer.count.Load() == 0 {
|
||||
t.Error("expected at least 1 enqueue, got 0")
|
||||
}
|
||||
}
|
||||
|
||||
func TestScheduler_AddOrUpdate(t *testing.T) {
|
||||
enqueuer := &fakeTaskEnqueuer{}
|
||||
scheduler := NewScheduler(newFakeDataSourceRepo(), newFakeSyncLogRepo(), enqueuer)
|
||||
scheduler.cron.Start()
|
||||
defer scheduler.Stop()
|
||||
|
||||
ds := &types.DataSource{
|
||||
ID: "ds-new",
|
||||
TenantID: 1,
|
||||
Status: types.DataSourceStatusActive,
|
||||
SyncSchedule: "0 0 * * * *", // every hour
|
||||
}
|
||||
|
||||
// Add
|
||||
if err := scheduler.AddOrUpdate(ds); err != nil {
|
||||
t.Fatalf("AddOrUpdate() error: %v", err)
|
||||
}
|
||||
if scheduler.EntryCount() != 1 {
|
||||
t.Errorf("EntryCount() = %d, want 1", scheduler.EntryCount())
|
||||
}
|
||||
|
||||
// Update schedule
|
||||
ds.SyncSchedule = "0 30 * * * *" // every half hour
|
||||
if err := scheduler.AddOrUpdate(ds); err != nil {
|
||||
t.Fatalf("AddOrUpdate() (update) error: %v", err)
|
||||
}
|
||||
if scheduler.EntryCount() != 1 {
|
||||
t.Errorf("after update: EntryCount() = %d, want 1 (should replace, not add)", scheduler.EntryCount())
|
||||
}
|
||||
}
|
||||
|
||||
func TestScheduler_AddOrUpdate_PausedIsNoop(t *testing.T) {
|
||||
enqueuer := &fakeTaskEnqueuer{}
|
||||
scheduler := NewScheduler(newFakeDataSourceRepo(), newFakeSyncLogRepo(), enqueuer)
|
||||
scheduler.cron.Start()
|
||||
defer scheduler.Stop()
|
||||
|
||||
ds := &types.DataSource{
|
||||
ID: "ds-paused",
|
||||
TenantID: 1,
|
||||
Status: types.DataSourceStatusPaused,
|
||||
SyncSchedule: "0 0 * * * *",
|
||||
}
|
||||
|
||||
if err := scheduler.AddOrUpdate(ds); err != nil {
|
||||
t.Fatalf("AddOrUpdate() error: %v", err)
|
||||
}
|
||||
if scheduler.EntryCount() != 0 {
|
||||
t.Errorf("paused ds should not be scheduled, EntryCount() = %d", scheduler.EntryCount())
|
||||
}
|
||||
}
|
||||
|
||||
func TestScheduler_AddOrUpdate_EmptyScheduleIsNoop(t *testing.T) {
|
||||
enqueuer := &fakeTaskEnqueuer{}
|
||||
scheduler := NewScheduler(newFakeDataSourceRepo(), newFakeSyncLogRepo(), enqueuer)
|
||||
scheduler.cron.Start()
|
||||
defer scheduler.Stop()
|
||||
|
||||
ds := &types.DataSource{
|
||||
ID: "ds-no-sched",
|
||||
TenantID: 1,
|
||||
Status: types.DataSourceStatusActive,
|
||||
SyncSchedule: "",
|
||||
}
|
||||
|
||||
if err := scheduler.AddOrUpdate(ds); err != nil {
|
||||
t.Fatalf("AddOrUpdate() error: %v", err)
|
||||
}
|
||||
if scheduler.EntryCount() != 0 {
|
||||
t.Errorf("empty schedule should not be registered, EntryCount() = %d", scheduler.EntryCount())
|
||||
}
|
||||
}
|
||||
|
||||
func TestScheduler_Remove(t *testing.T) {
|
||||
enqueuer := &fakeTaskEnqueuer{}
|
||||
scheduler := NewScheduler(newFakeDataSourceRepo(), newFakeSyncLogRepo(), enqueuer)
|
||||
scheduler.cron.Start()
|
||||
defer scheduler.Stop()
|
||||
|
||||
ds := &types.DataSource{
|
||||
ID: "ds-rm",
|
||||
TenantID: 1,
|
||||
Status: types.DataSourceStatusActive,
|
||||
SyncSchedule: "0 0 * * * *",
|
||||
}
|
||||
|
||||
_ = scheduler.AddOrUpdate(ds)
|
||||
if scheduler.EntryCount() != 1 {
|
||||
t.Fatalf("pre-remove: EntryCount() = %d, want 1", scheduler.EntryCount())
|
||||
}
|
||||
|
||||
scheduler.Remove("ds-rm")
|
||||
if scheduler.EntryCount() != 0 {
|
||||
t.Errorf("post-remove: EntryCount() = %d, want 0", scheduler.EntryCount())
|
||||
}
|
||||
|
||||
// Remove non-existent is safe
|
||||
scheduler.Remove("does-not-exist")
|
||||
}
|
||||
|
||||
func TestScheduler_InvalidCron(t *testing.T) {
|
||||
enqueuer := &fakeTaskEnqueuer{}
|
||||
scheduler := NewScheduler(newFakeDataSourceRepo(), newFakeSyncLogRepo(), enqueuer)
|
||||
scheduler.cron.Start()
|
||||
defer scheduler.Stop()
|
||||
|
||||
ds := &types.DataSource{
|
||||
ID: "ds-bad",
|
||||
TenantID: 1,
|
||||
Status: types.DataSourceStatusActive,
|
||||
SyncSchedule: "not a cron",
|
||||
}
|
||||
|
||||
err := scheduler.AddOrUpdate(ds)
|
||||
if err == nil {
|
||||
t.Fatal("expected error for invalid cron expression")
|
||||
}
|
||||
if scheduler.EntryCount() != 0 {
|
||||
t.Errorf("invalid cron should not be registered, EntryCount() = %d", scheduler.EntryCount())
|
||||
}
|
||||
}
|
||||
|
||||
func TestScheduler_TriggerSync_InactiveSkipped(t *testing.T) {
|
||||
repo := newFakeDataSourceRepo()
|
||||
// Create a data source that is paused
|
||||
_ = repo.Create(context.Background(), &types.DataSource{
|
||||
ID: "ds-inactive",
|
||||
TenantID: 1,
|
||||
Status: types.DataSourceStatusPaused,
|
||||
})
|
||||
|
||||
enqueuer := &fakeTaskEnqueuer{}
|
||||
scheduler := NewScheduler(repo, newFakeSyncLogRepo(), enqueuer)
|
||||
|
||||
// Directly call triggerSync — it should skip because ds is not active
|
||||
scheduler.triggerSync("ds-inactive", 1)
|
||||
|
||||
if enqueuer.count.Load() != 0 {
|
||||
t.Error("should not enqueue for inactive data source")
|
||||
}
|
||||
}
|
||||
|
||||
func TestScheduler_TriggerSync_NotFound(t *testing.T) {
|
||||
repo := newFakeDataSourceRepo()
|
||||
enqueuer := &fakeTaskEnqueuer{}
|
||||
scheduler := NewScheduler(repo, newFakeSyncLogRepo(), enqueuer)
|
||||
|
||||
// Should not panic, just skip
|
||||
scheduler.triggerSync("nonexistent", 1)
|
||||
|
||||
if enqueuer.count.Load() != 0 {
|
||||
t.Error("should not enqueue for non-existent data source")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user