Files
Enterprise-Knowledge-Base/internal/application/service/datasource_service.go
T
nullkey c695125dce fix(feishu): resilient large-wiki sync — retry, streaming resume, failure visibility
Large Feishu wikis stopped syncing after the first batch (Tencent/WeKnora#2136:
5000+ docs, only ~150 ingested; #1262: a child-list failure aborted the run).
Three compounding causes, fixed in layers:

1. Rate-limit / transient failures had no retry. Feishu's wiki/drive/export
   APIs are aggressively rate limited; a thousand-doc sync issues tens of
   thousands of calls, and a single 429/5xx burst failed whole swathes of docs.
   Add exponential backoff honouring Retry-After to doRequest (JSON) and
   downloadRawBytes (files): 429 up to 3x, 5xx once, transport errors back off.

2. The batch path fetched the entire tree into memory before ingesting, so a
   sync that hit the 2h task timeout lost all progress and restarted from
   scratch, and a transient per-document export failure advanced the cursor and
   skipped that doc permanently. Add an optional StreamingConnector interface,
   implemented for Feishu: FetchStream interleaves fetch -> ingest -> checkpoint,
   bounding memory to one item and persisting a per-node edit-time cursor at
   page boundaries. The service dispatches to it via type assertion; connectors
   without it fall back to FetchAll / FetchIncremental unchanged (non-breaking).
   A resumed/incremental run skips nodes already recorded at their current edit
   time (converges), a failed fetch does not advance the cursor (retried next
   run instead of dropped), and partial listing failures emit error items and
   carry prior edit times forward instead of aborting the sync.

3. Failures and skips were invisible. Unsupported node types (mindnote/slides/…)
   were skipped with no item, error or log ("13 docs synced only 3"). FetchAll
   now tallies discovered/fetched/failed/skipped per resource with a per-type
   skip breakdown and a progress line every 100 nodes. The backend already
   persisted per-item errors in SyncLog.result.errors but the frontend type
   omitted the field; declare `result` on SyncLog and render up to 50 failed
   documents in the sync-log drawer (#1262). i18n for zh-CN/en-US/ko-KR/ru-RU.
   result.errors is a jsonb column shipped in every list response, so it is
   capped at 100 samples server-side (recordSyncError); the exact failure total
   comes from the bounded items_failed counter, which the drawer uses for its
   count and "N more" line so a sync that fails thousands of docs stays bounded.
   Each sample is a structured, localisable SyncItemError instead of a raw
   "title: ...api error: status=500 body={json}" dump that repeated the title
   and leaked status/body/log_id. The connector (feishuFailure) classifies the
   error into a stable i18n code (+ the feishu code as a param); the service
   builds {title, code, params, message}; the drawer localises via vue-i18n to
   the viewer's language (zh/en/ko/ru), falling back to message. A custom
   UnmarshalJSON keeps old string-form logs readable. The banner is localised
   from items_failed too. Raw errors stay in the server logs. Follows the
   two-tier norm of Airbyte/Fivetran/Onyx: human, actionable, localised
   summaries in the UI; raw detail in logs. Transient errors say "will retry"
   (the cursor is retained); auth/permission errors point at the fix instead.

Tests: client_retry_test (backoff), tally_test (accounting), connector_stream /
datasource_stream (streaming dispatch + checkpoint), connector_convergence
(timeout + transient-failure resume converges to full coverage; proven to catch
#2136 via bug injection), datasource_result_cap (error-sample cap + structured
localisable SyncItemError, no raw body), connector_error_reason (error→i18n-code
classification), sync_item_error (legacy-string + structured back-compat) — all
in the normal suite. connector_realapi / connector_seed are an opt-in real-Feishu
harness gated by the feishu_integration build tag AND env vars, never run in CI.
2026-07-21 16:14:22 +08:00

1212 lines
39 KiB
Go

package service
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"mime/multipart"
"net/textproto"
"reflect"
"strings"
"time"
"github.com/Tencent/WeKnora/internal/datasource"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/tracing/langfuse"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
secutils "github.com/Tencent/WeKnora/internal/utils"
"github.com/hibiken/asynq"
)
// DataSourceService implements the DataSourceService interface
type DataSourceService struct {
dsRepo interfaces.DataSourceRepository
syncLogRepo interfaces.SyncLogRepository
knowledgeService interfaces.KnowledgeService
kbService interfaces.KnowledgeBaseService
taskEnqueuer interfaces.TaskEnqueuer
connectorRegistry *datasource.ConnectorRegistry
scheduler *datasource.Scheduler
tenantRepo interfaces.TenantRepository
tagService interfaces.KnowledgeTagService
}
// NewDataSourceService creates a new data source service
func NewDataSourceService(
dsRepo interfaces.DataSourceRepository,
syncLogRepo interfaces.SyncLogRepository,
knowledgeService interfaces.KnowledgeService,
kbService interfaces.KnowledgeBaseService,
taskEnqueuer interfaces.TaskEnqueuer,
connectorRegistry *datasource.ConnectorRegistry,
scheduler *datasource.Scheduler,
tenantRepo interfaces.TenantRepository,
tagService interfaces.KnowledgeTagService,
) interfaces.DataSourceService {
return &DataSourceService{
dsRepo: dsRepo,
syncLogRepo: syncLogRepo,
knowledgeService: knowledgeService,
kbService: kbService,
taskEnqueuer: taskEnqueuer,
connectorRegistry: connectorRegistry,
scheduler: scheduler,
tenantRepo: tenantRepo,
tagService: tagService,
}
}
// CreateDataSource creates a new data source configuration
func (s *DataSourceService) CreateDataSource(ctx context.Context, ds *types.DataSource) (*types.DataSource, error) {
if ds == nil {
return nil, datasource.ErrDataSourceInvalid
}
// Validate knowledge base exists
kb, err := s.kbService.GetKnowledgeBaseByID(ctx, ds.KnowledgeBaseID)
if err != nil || kb == nil {
return nil, datasource.ErrKnowledgeBaseNotFound
}
if kb.TenantID != ds.TenantID {
return nil, datasource.ErrKnowledgeBaseNotFound
}
// Validate connector type
_, err = s.connectorRegistry.Get(ds.Type)
if err != nil {
return nil, err
}
// Validate configuration
if cfg, err := ds.ParseConfig(); err == nil && cfg != nil {
cfg.StripNonSecretCredentials(ds.Type)
if blob, err := cfg.ToJSON(); err == nil {
ds.Config = blob
}
}
if err := s.validateDataSourceConfig(ctx, ds); err != nil {
return nil, err
}
// Create in database
if err := s.dsRepo.Create(ctx, ds); err != nil {
logger.Errorf(ctx, "failed to create data source: %v", err)
return nil, err
}
// Register cron schedule if configured
if ds.SyncSchedule != "" && ds.Status == types.DataSourceStatusActive {
if err := s.scheduler.AddOrUpdate(ds); err != nil {
logger.Warnf(ctx, "failed to register cron for ds=%s: %v", ds.ID, err)
}
}
logger.Infof(ctx, "data source created: id=%s type=%s kb=%s", ds.ID, ds.Type, ds.KnowledgeBaseID)
return ds, nil
}
// GetDataSource retrieves a data source by ID
func (s *DataSourceService) GetDataSource(ctx context.Context, id string) (*types.DataSource, error) {
ds, err := s.dsRepo.FindByID(ctx, id)
if err != nil {
return nil, err
}
return ds, nil
}
// ListDataSources lists all data sources for a knowledge base
func (s *DataSourceService) ListDataSources(ctx context.Context, kbID string) ([]*types.DataSource, error) {
dataSources, err := s.dsRepo.FindByKnowledgeBase(ctx, kbID)
if err != nil {
logger.Errorf(ctx, "failed to list data sources: %v", err)
return nil, err
}
// Attach latest sync log to each data source
for _, ds := range dataSources {
log, _ := s.syncLogRepo.FindLatest(ctx, ds.ID)
if log != nil {
ds.LatestSyncLog = log
}
}
return dataSources, nil
}
// UpdateDataSource updates an existing data source
func (s *DataSourceService) UpdateDataSource(ctx context.Context, ds *types.DataSource) (*types.DataSource, error) {
if ds == nil || ds.ID == "" {
return nil, datasource.ErrDataSourceInvalid
}
// Verify data source exists
existing, err := s.dsRepo.FindByID(ctx, ds.ID)
if err != nil {
return nil, err
}
if ds.KnowledgeBaseID == "" {
ds.KnowledgeBaseID = existing.KnowledgeBaseID
}
if ds.KnowledgeBaseID != existing.KnowledgeBaseID {
return nil, fmt.Errorf("changing knowledge base is not allowed")
}
if ds.TenantID == 0 {
ds.TenantID = existing.TenantID
}
if ds.TenantID != existing.TenantID {
return nil, datasource.ErrDataSourceInvalid
}
// Credentials NEVER flow through this endpoint — they live behind the
// /credentials subresource. Force-preserve the stored credentials map
// regardless of what the body says. Log a warning if a stale caller
// passes one so we can spot them and migrate later. Non-credential
// fields of Config (Type / ResourceIDs / Settings) flow through.
var mergedCfg, existingParsedCfg *types.DataSourceConfig
if len(ds.Config) > 0 {
incomingCfg, parseIncErr := ds.ParseConfig()
existingCfg, parseExErr := existing.ParseConfig()
if parseIncErr == nil && parseExErr == nil && incomingCfg != nil {
if incomingCfg.HasCredentials() {
logger.Warnf(ctx,
"deprecated: credentials in PUT /datasource/%s body are ignored; use PUT /credentials instead",
secutils.SanitizeForLog(ds.ID))
}
merged := *incomingCfg
if existingCfg != nil {
merged.Credentials = existingCfg.Credentials
} else {
merged.Credentials = nil
}
merged.StripNonSecretCredentials(ds.Type)
if blob, err := merged.ToJSON(); err == nil {
ds.Config = blob
}
mergedCfg = &merged
existingParsedCfg = existingCfg
}
}
// Validate new configuration if non-credential fields changed. Skip
// when there are no stored credentials yet (validators would fail with
// no token to call the live API) and when the parsed config is
// structurally identical.
configActuallyChanged := true
if mergedCfg != nil && existingParsedCfg != nil {
configActuallyChanged = !reflect.DeepEqual(*mergedCfg, *existingParsedCfg)
}
hasCreds := mergedCfg != nil && mergedCfg.HasConfiguredCredentials(ds.Type)
if hasCreds && (ds.Type != existing.Type || configActuallyChanged) {
if err := s.validateDataSourceConfig(ctx, ds); err != nil {
return nil, err
}
}
if err := s.dsRepo.Update(ctx, ds); err != nil {
logger.Errorf(ctx, "failed to update data source: %v", err)
return nil, err
}
// Update cron schedule
if err := s.scheduler.AddOrUpdate(ds); err != nil {
logger.Warnf(ctx, "failed to update cron for ds=%s: %v", ds.ID, err)
}
logger.Infof(ctx, "data source updated: id=%s", ds.ID)
return ds, nil
}
// UpdateDataSourceCredentials replaces the connector credential map. This is
// a single atomic write; the previous credential set is discarded entirely
// (callers cannot patch individual keys because half-configured connector
// auth is meaningless). After persisting, the live connection is validated
// so the caller learns immediately if the new credentials are wrong.
func (s *DataSourceService) UpdateDataSourceCredentials(
ctx context.Context, id string, credentials map[string]interface{},
) (*types.DataSource, error) {
if id == "" {
return nil, datasource.ErrDataSourceInvalid
}
existing, err := s.dsRepo.FindByID(ctx, id)
if err != nil {
return nil, err
}
parsed, err := existing.ParseConfig()
if err != nil {
return nil, err
}
if parsed == nil {
parsed = &types.DataSourceConfig{Type: existing.Type}
}
parsed.Credentials = credentials
parsed.StripNonSecretCredentials(existing.Type)
blob, err := parsed.ToJSON()
if err != nil {
return nil, err
}
existing.Config = blob
// Run live validation now that the credentials are in place — surfaces
// "wrong token" feedback immediately to the user instead of waiting for
// the next scheduled sync.
if err := s.validateDataSourceConfig(ctx, existing); err != nil {
return nil, err
}
if err := s.dsRepo.Update(ctx, existing); err != nil {
return nil, err
}
logger.Infof(ctx, "DataSource credentials updated: id=%s", secutils.SanitizeForLog(id))
return existing, nil
}
// ClearDataSourceCredentials wipes the connector credential map without
// touching any other config field. Idempotent.
func (s *DataSourceService) ClearDataSourceCredentials(ctx context.Context, id string) error {
if id == "" {
return datasource.ErrDataSourceInvalid
}
existing, err := s.dsRepo.FindByID(ctx, id)
if err != nil {
return err
}
parsed, err := existing.ParseConfig()
if err != nil {
return err
}
if parsed == nil {
return nil
}
parsed.StripNonSecretCredentials(existing.Type)
if !parsed.HasConfiguredCredentials(existing.Type) {
blob, err := parsed.ToJSON()
if err != nil {
return err
}
existing.Config = blob
return s.dsRepo.Update(ctx, existing)
}
parsed.Credentials = nil
blob, err := parsed.ToJSON()
if err != nil {
return err
}
existing.Config = blob
if err := s.dsRepo.Update(ctx, existing); err != nil {
return err
}
logger.Infof(ctx, "DataSource credentials cleared by user: id=%s", secutils.SanitizeForLog(id))
return nil
}
// DeleteDataSource deletes a data source (soft delete)
func (s *DataSourceService) DeleteDataSource(ctx context.Context, id string) error {
// Verify data source exists
_, err := s.dsRepo.FindByID(ctx, id)
if err != nil {
return err
}
if err := s.dsRepo.Delete(ctx, id); err != nil {
logger.Errorf(ctx, "failed to delete data source: %v", err)
return err
}
// Remove cron schedule
s.scheduler.Remove(id)
// Cancel any pending/running sync logs so queued asynq tasks won't retry
if err := s.syncLogRepo.CancelPendingByDataSource(ctx, id); err != nil {
logger.Warnf(ctx, "failed to cancel pending sync logs for ds=%s: %v", id, err)
}
logger.Infof(ctx, "data source deleted: id=%s", id)
return nil
}
// ValidateConnection tests the connection to an external data source
func (s *DataSourceService) ValidateConnection(ctx context.Context, dsID string) error {
ds, err := s.GetDataSource(ctx, dsID)
if err != nil {
return err
}
// Get connector
connector, err := s.connectorRegistry.Get(ds.Type)
if err != nil {
return err
}
// Parse configuration
config, err := ds.ParseConfig()
if err != nil {
return datasource.ErrInvalidConfig
}
// Validate connection
if err := connector.Validate(ctx, config); err != nil {
// Update data source with error
ds.Status = types.DataSourceStatusError
ds.ErrorMessage = err.Error()
_ = s.dsRepo.Update(ctx, ds)
return err
}
// Clear error if it was previously in error state
if ds.Status == types.DataSourceStatusError {
ds.Status = types.DataSourceStatusActive
ds.ErrorMessage = ""
_ = s.dsRepo.Update(ctx, ds)
}
return nil
}
// ListAvailableResources lists resources available for sync in the external system.
// parentID enables lazy (on-demand) loading of hierarchical resources: pass "" to
// list the top level, or a resource's ExternalID to list only its direct children.
func (s *DataSourceService) ListAvailableResources(
ctx context.Context, dsID string, parentID string,
) ([]types.Resource, error) {
ds, err := s.GetDataSource(ctx, dsID)
if err != nil {
return nil, err
}
// Get connector
connector, err := s.connectorRegistry.Get(ds.Type)
if err != nil {
return nil, err
}
// Parse configuration
config, err := ds.ParseConfig()
if err != nil {
return nil, datasource.ErrInvalidConfig
}
// List resources
resources, err := connector.ListResources(ctx, config, parentID)
if err != nil {
logger.Errorf(ctx, "failed to list resources: %v", err)
return nil, err
}
return resources, nil
}
// ResolveResourceAncestors resolves the ancestor ExternalIDs needed to reveal the
// given resources in a lazily-loaded picker (see the connector method for details).
func (s *DataSourceService) ResolveResourceAncestors(
ctx context.Context, dsID string, resourceIDs []string,
) ([]string, error) {
if len(resourceIDs) == 0 {
return []string{}, nil
}
ds, err := s.GetDataSource(ctx, dsID)
if err != nil {
return nil, err
}
connector, err := s.connectorRegistry.Get(ds.Type)
if err != nil {
return nil, err
}
config, err := ds.ParseConfig()
if err != nil {
return nil, datasource.ErrInvalidConfig
}
ancestors, err := connector.ResolveResourceAncestors(ctx, config, resourceIDs)
if err != nil {
logger.Errorf(ctx, "failed to resolve resource ancestors: %v", err)
return nil, err
}
return ancestors, nil
}
// ManualSync triggers an immediate sync for a data source
func (s *DataSourceService) ManualSync(ctx context.Context, dsID string) (*types.SyncLog, error) {
ds, err := s.GetDataSource(ctx, dsID)
if err != nil {
return nil, err
}
if ds.Status != types.DataSourceStatusActive &&
ds.Status != types.DataSourceStatusError &&
ds.Status != types.DataSourceStatusPaused {
return nil, datasource.ErrDataSourceNotActive
}
// Create sync log
syncLog := &types.SyncLog{
DataSourceID: dsID,
TenantID: ds.TenantID,
Status: types.SyncLogStatusRunning,
StartedAt: time.Now().UTC(),
}
if err := s.syncLogRepo.Create(ctx, syncLog); err != nil {
logger.Errorf(ctx, "failed to create sync log: %v", err)
return nil, err
}
// Enqueue sync task
payload := &types.DataSourceSyncPayload{
DataSourceID: dsID,
TenantID: ds.TenantID,
SyncLogID: syncLog.ID,
ForceFull: false,
}
langfuse.InjectTracing(ctx, payload)
payloadJSON, _ := json.Marshal(payload)
task := asynq.NewTask(types.TypeDataSourceSync, payloadJSON,
asynq.Queue(types.QueueSync), asynq.MaxRetry(5), asynq.Timeout(2*time.Hour))
_, err = s.taskEnqueuer.Enqueue(task)
if err != nil {
logger.Errorf(ctx, "failed to enqueue sync task: %v", err)
syncLog.Status = types.SyncLogStatusFailed
syncLog.FinishedAt = timePtr(time.Now().UTC())
syncLog.ErrorMessage = err.Error()
_ = s.syncLogRepo.Update(ctx, syncLog)
if ds.Status != types.DataSourceStatusPaused {
ds.Status = types.DataSourceStatusError
}
ds.ErrorMessage = fmt.Sprintf("Failed to enqueue sync: %v", err)
_ = s.dsRepo.Update(ctx, ds)
return nil, err
}
logger.Infof(ctx, "sync task enqueued: ds=%s syncLog=%s", dsID, syncLog.ID)
return syncLog, nil
}
// PauseDataSource pauses a data source's scheduled syncs
func (s *DataSourceService) PauseDataSource(ctx context.Context, id string) error {
ds, err := s.GetDataSource(ctx, id)
if err != nil {
return err
}
ds.Status = types.DataSourceStatusPaused
if err := s.dsRepo.Update(ctx, ds); err != nil {
logger.Errorf(ctx, "failed to pause data source: %v", err)
return err
}
// Remove cron schedule
s.scheduler.Remove(id)
logger.Infof(ctx, "data source paused: id=%s", id)
return nil
}
// ResumeDataSource resumes a paused data source
func (s *DataSourceService) ResumeDataSource(ctx context.Context, id string) error {
ds, err := s.GetDataSource(ctx, id)
if err != nil {
return err
}
ds.Status = types.DataSourceStatusActive
if err := s.dsRepo.Update(ctx, ds); err != nil {
logger.Errorf(ctx, "failed to resume data source: %v", err)
return err
}
// Re-register cron schedule
if err := s.scheduler.AddOrUpdate(ds); err != nil {
logger.Warnf(ctx, "failed to re-register cron for ds=%s: %v", ds.ID, err)
}
logger.Infof(ctx, "data source resumed: id=%s", id)
return nil
}
// GetSyncLogs retrieves sync history for a data source
func (s *DataSourceService) GetSyncLogs(ctx context.Context, dsID string, limit int, offset int) ([]*types.SyncLog, error) {
logs, err := s.syncLogRepo.FindByDataSource(ctx, dsID, limit, offset)
if err != nil {
logger.Errorf(ctx, "failed to get sync logs: %v", err)
return nil, err
}
return logs, nil
}
// GetSyncLog retrieves a specific sync log entry
func (s *DataSourceService) GetSyncLog(ctx context.Context, syncLogID string) (*types.SyncLog, error) {
log, err := s.syncLogRepo.FindByID(ctx, syncLogID)
if err != nil {
return nil, err
}
return log, nil
}
// ProcessSync handles the actual sync operation (called by asynq task)
func (s *DataSourceService) ProcessSync(ctx context.Context, task *asynq.Task) error {
var payload types.DataSourceSyncPayload
if err := json.Unmarshal(task.Payload(), &payload); err != nil {
logger.Errorf(ctx, "failed to unmarshal sync payload: %v", err)
return err
}
logger.Infof(ctx, "processing data source sync: ds=%s syncLog=%s", payload.DataSourceID, payload.SyncLogID)
// Get data source
ds, err := s.GetDataSource(ctx, payload.DataSourceID)
if err != nil {
logger.Warnf(ctx, "data source not found (likely deleted), cancelling sync: ds=%s err=%v", payload.DataSourceID, err)
if syncLog, slErr := s.syncLogRepo.FindByID(ctx, payload.SyncLogID); slErr == nil && syncLog != nil {
syncLog.Status = types.SyncLogStatusCanceled
syncLog.FinishedAt = timePtr(time.Now().UTC())
syncLog.ErrorMessage = "data source has been deleted"
_ = s.syncLogRepo.Update(ctx, syncLog)
}
return nil
}
// Get sync log
syncLog, err := s.syncLogRepo.FindByID(ctx, payload.SyncLogID)
if err != nil {
logger.Errorf(ctx, "failed to get sync log: %v", err)
return nil
}
if _, err := s.kbService.GetKnowledgeBaseByID(ctx, ds.KnowledgeBaseID); err != nil {
logger.Warnf(ctx, "knowledge base not found (likely deleted), cancelling sync: kb=%s ds=%s err=%v",
ds.KnowledgeBaseID, payload.DataSourceID, err)
syncLog.Status = types.SyncLogStatusCanceled
syncLog.FinishedAt = timePtr(time.Now().UTC())
syncLog.ErrorMessage = "knowledge base has been deleted"
_ = s.syncLogRepo.Update(ctx, syncLog)
return nil
}
wasPaused := ds.Status == types.DataSourceStatusPaused
// Get connector
connector, err := s.connectorRegistry.Get(ds.Type)
if err != nil {
logger.Errorf(ctx, "connector not found: type=%s", ds.Type)
syncLog.Status = types.SyncLogStatusFailed
syncLog.FinishedAt = timePtr(time.Now().UTC())
syncLog.ErrorMessage = fmt.Sprintf("Connector not found: %s", ds.Type)
_ = s.syncLogRepo.Update(ctx, syncLog)
if !wasPaused {
ds.Status = types.DataSourceStatusError
}
ds.ErrorMessage = syncLog.ErrorMessage
_ = s.dsRepo.Update(ctx, ds)
return err
}
// Parse configuration
config, err := ds.ParseConfig()
if err != nil {
logger.Errorf(ctx, "failed to parse config: %v", err)
syncLog.Status = types.SyncLogStatusFailed
syncLog.FinishedAt = timePtr(time.Now().UTC())
syncLog.ErrorMessage = fmt.Sprintf("Invalid configuration: %v", err)
_ = s.syncLogRepo.Update(ctx, syncLog)
if !wasPaused {
ds.Status = types.DataSourceStatusError
}
ds.ErrorMessage = syncLog.ErrorMessage
_ = s.dsRepo.Update(ctx, ds)
return err
}
// Streaming path: connectors that support it interleave fetch→ingest→
// checkpoint so a large sync bounds memory and resumes after a timeout
// instead of restarting (Tencent/WeKnora#2136). Others fall back below.
if sc, ok := connector.(datasource.StreamingConnector); ok {
return s.processSyncStreaming(ctx, sc, ds, syncLog, config, payload, wasPaused)
}
// Fetch items based on sync mode
var items []types.FetchedItem
var nextCursor *types.SyncCursor
var fetchErr error
if payload.ForceFull || ds.SyncMode == types.SyncModeFull {
// Full sync
items, fetchErr = connector.FetchAll(ctx, config, config.ResourceIDs)
logger.Infof(ctx, "full sync fetched %d items", len(items))
} else {
// Incremental sync
cursor, _ := ds.ParseSyncCursor()
items, nextCursor, fetchErr = connector.FetchIncremental(ctx, config, cursor)
logger.Infof(ctx, "incremental sync fetched %d items", len(items))
}
var fetchWarnings []string
var partialFetch *datasource.PartialFetchError
if errors.As(fetchErr, &partialFetch) {
fetchWarnings = partialFetch.Details
fetchErr = nil
}
if fetchErr != nil {
// Persist connector cursor even when fetch failed so transient outages
// (e.g. RSS feed downtime) do not force a full re-ingest on recovery.
if nextCursor != nil {
if cursorJSON, cerr := nextCursor.ToJSON(); cerr == nil {
ds.LastSyncCursor = cursorJSON
if uerr := s.dsRepo.UpdateSyncState(ctx, ds); uerr != nil {
logger.Warnf(ctx, "failed to persist sync cursor after fetch error: %v", uerr)
}
}
}
logger.Errorf(ctx, "fetch operation failed: %v", fetchErr)
syncLog.Status = types.SyncLogStatusFailed
syncLog.FinishedAt = timePtr(time.Now().UTC())
syncLog.ErrorMessage = fmt.Sprintf("Fetch failed: %v", fetchErr)
_ = s.syncLogRepo.Update(ctx, syncLog)
if !wasPaused {
ds.Status = types.DataSourceStatusError
}
ds.ErrorMessage = syncLog.ErrorMessage
_ = s.dsRepo.Update(ctx, ds)
return fetchErr
}
// Process fetched items and write to knowledge base
var result = &types.SyncResult{
Total: len(items),
}
// Set tenant context so KnowledgeService can resolve tenant info correctly
ctx = context.WithValue(ctx, types.TenantIDContextKey, ds.TenantID)
tenant, err := s.tenantRepo.GetTenantByID(ctx, ds.TenantID)
if err != nil {
logger.Errorf(ctx, "failed to get tenant info: %v", err)
syncLog.Status = types.SyncLogStatusFailed
syncLog.FinishedAt = timePtr(time.Now().UTC())
syncLog.ErrorMessage = fmt.Sprintf("Failed to get tenant info: %v", err)
_ = s.syncLogRepo.Update(ctx, syncLog)
if !wasPaused {
ds.Status = types.DataSourceStatusError
}
ds.ErrorMessage = syncLog.ErrorMessage
_ = s.dsRepo.Update(ctx, ds)
return err
}
ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenant)
// Auto-tag: find or create a tag for this data source so synced items are easily identifiable
autoTagIDs := s.resolveAutoTagIDs(ctx, ds)
for _, item := range items {
item := item
s.applyFetchedItem(ctx, ds, &item, autoTagIDs, result)
}
resultJSON, _ := result.ToJSON()
if err := allFetchedItemsFailedError(result); err != nil {
logger.Errorf(ctx, "data source sync failed while processing fetched items: %v", err)
s.updateSyncRunResult(ctx, ds, syncLog, result, resultJSON, types.SyncLogStatusFailed, err.Error(), wasPaused)
return err
}
// Update cursor for next incremental sync
if nextCursor != nil {
cursorJSON, _ := nextCursor.ToJSON()
ds.LastSyncCursor = cursorJSON
}
ds.LastSyncAt = timePtr(time.Now().UTC())
syncStatus := types.SyncLogStatusSuccess
syncErrorMessage := ""
if len(fetchWarnings) > 0 {
syncStatus = types.SyncLogStatusPartial
syncErrorMessage = fmt.Sprintf("Some feeds failed: %s", strings.Join(fetchWarnings, "; "))
for _, w := range fetchWarnings {
result.Errors = append(result.Errors, types.SyncItemError{Message: w})
}
resultJSON, _ = result.ToJSON()
}
s.updateSyncRunResult(ctx, ds, syncLog, result, resultJSON, syncStatus, syncErrorMessage, wasPaused)
logger.Infof(ctx, "data source sync completed: ds=%s created=%d updated=%d deleted=%d",
payload.DataSourceID, syncLog.ItemsCreated, syncLog.ItemsUpdated, syncLog.ItemsDeleted)
return nil
}
// resolveAutoTagIDs finds or creates the per-data-source tag applied to every
// synced item so results are identifiable in the KB. A tag failure is
// non-fatal: the sync proceeds untagged.
func (s *DataSourceService) resolveAutoTagIDs(ctx context.Context, ds *types.DataSource) []string {
autoTagIDs := []string{}
if autoTag, tagErr := s.tagService.FindOrCreateTagByName(ctx, ds.KnowledgeBaseID, ds.Name); tagErr != nil {
logger.Warnf(ctx, "failed to find/create auto-tag %q: %v (proceeding without tag)", ds.Name, tagErr)
} else if autoTag != nil {
autoTagIDs = append(autoTagIDs, autoTag.ID)
logger.Infof(ctx, "using auto-tag %q (id=%s) for data source sync", ds.Name, autoTag.ID)
}
return autoTagIDs
}
// maxSyncResultErrors bounds the per-item error sample retained in
// SyncResult.Errors. That slice is persisted as jsonb and returned in every
// sync-log list response, so an unbounded list on a sync that fails thousands of
// documents means multi-MB DB rows and payloads. The accurate failure count
// lives in SyncResult.Failed (a bounded int); this list only keeps a sample for
// display (Tencent/WeKnora#2136 / #1262).
const maxSyncResultErrors = 100
// recordSyncError appends an error sample to result.Errors, capped at
// maxSyncResultErrors. Callers still increment result.Failed for the exact count.
func recordSyncError(result *types.SyncResult, item types.SyncItemError) {
if len(result.Errors) < maxSyncResultErrors {
result.Errors = append(result.Errors, item)
}
}
// fetchFailureSyncError maps a connector error item into a structured, user-
// facing sample. Connectors that classify their errors (Feishu) provide a stable
// i18n code + params via metadata so the frontend localises it to the viewer's
// language; the raw status/body/log_id never leaves the server logs. Connectors
// without codes keep the raw text as a Message fallback. Best practice per
// Airbyte/Fivetran/Onyx: humanised, actionable, localised UI; raw detail in logs.
func fetchFailureSyncError(item *types.FetchedItem, rawMsg string) types.SyncItemError {
e := types.SyncItemError{Title: item.Title}
if code := item.Metadata["error_reason_code"]; code != "" {
e.Code = code
if v := item.Metadata["error_reason_code_value"]; v != "" {
e.Params = map[string]string{"code": v}
}
e.Message = item.Metadata["error_reason"] // fallback if the client lacks the key
} else {
e.Message = rawMsg
}
return e
}
// applyFetchedItem writes a single fetched item into the knowledge base and
// updates result counters. It is the shared core of the batch loop and the
// streaming handler so item classification (deleted / empty / ingest outcome)
// stays identical across both fetch paths.
func (s *DataSourceService) applyFetchedItem(
ctx context.Context, ds *types.DataSource, item *types.FetchedItem,
tagIDs []string, result *types.SyncResult,
) {
if item.IsDeleted {
if ds.SyncDeletions {
// Count only — actual KB deletion is intentionally not performed.
// Users manage knowledge removal explicitly via the KB UI to avoid
// accidental data loss from connector misdetection or reconfiguration.
result.Deleted++
}
return
}
if len(item.Content) == 0 && item.URL == "" {
// Check if this is an error item from the connector (failed to fetch content)
if errMsg, hasErr := item.Metadata["error"]; hasErr {
logger.Warnf(ctx, "item %q (external_id=%s) fetch failed: %s", item.Title, item.ExternalID, errMsg)
result.Failed++
recordSyncError(result, fetchFailureSyncError(item, errMsg))
} else {
logger.Infof(ctx, "skipping item %q (external_id=%s): no content or URL", item.Title, item.ExternalID)
result.Skipped++
}
return
}
isUpdate, err := s.ingestItem(ctx, ds, item, tagIDs)
if err != nil {
// Duplicate file/URL is not a failure — count as skipped
var dupErr *types.DuplicateKnowledgeError
if errors.As(err, &dupErr) {
logger.Infof(ctx, "item %q (external_id=%s) already exists, skipping", item.Title, item.ExternalID)
result.Skipped++
} else {
logger.Warnf(ctx, "failed to ingest item %q (external_id=%s): %v", item.Title, item.ExternalID, err)
result.Failed++
recordSyncError(result, types.SyncItemError{
Title: item.Title,
Code: "ingest_failed",
Message: "Ingest failed; see server logs",
})
}
} else if isUpdate {
result.Updated++
} else {
result.Created++
}
}
// streamStartCursor decides which cursor a streaming fetch should resume from.
// A user-triggered full sync on its first attempt drops the cursor so every
// item is re-fetched; a retried full sync (attempt > 0) and every incremental
// sync resume from the last persisted checkpoint so a timed-out run converges
// instead of restarting from scratch.
func streamStartCursor(ds *types.DataSource, forceFull bool, attempt int) (*types.SyncCursor, error) {
if forceFull && attempt == 0 {
return nil, nil
}
return ds.ParseSyncCursor()
}
// streamSyncHandler adapts a streaming fetch to the knowledge-base ingest path.
// Emit ingests each item as it arrives (bounding memory) and Checkpoint persists
// the connector cursor plus live progress counts at page boundaries.
type streamSyncHandler struct {
svc *DataSourceService
ds *types.DataSource
tagIDs []string
result *types.SyncResult
syncLog *types.SyncLog
}
// Emit ingests one streamed item. A canceled context aborts the stream so the
// connector stops fetching; per-item ingest failures are recorded in result and
// do NOT abort (matching the batch loop, which never fails the whole sync for
// one bad document).
func (h *streamSyncHandler) Emit(ctx context.Context, item types.FetchedItem) error {
if err := ctx.Err(); err != nil {
return err
}
h.result.Total++
h.svc.applyFetchedItem(ctx, h.ds, &item, h.tagIDs, h.result)
return nil
}
// Checkpoint persists the connector cursor onto the data source and mirrors the
// running counts into the sync log so progress survives a crash and the UI can
// reflect a long sync mid-flight instead of jumping from 0 to done.
func (h *streamSyncHandler) Checkpoint(ctx context.Context, cursor *types.SyncCursor) error {
if cursor == nil {
return nil
}
cursorJSON, err := cursor.ToJSON()
if err != nil {
return err
}
h.ds.LastSyncCursor = cursorJSON
if err := h.svc.dsRepo.UpdateSyncState(ctx, h.ds); err != nil {
return err
}
// Best-effort live progress; a failure here must not abort the sync.
h.syncLog.ItemsTotal = h.result.Total
h.syncLog.ItemsCreated = h.result.Created
h.syncLog.ItemsUpdated = h.result.Updated
h.syncLog.ItemsDeleted = h.result.Deleted
h.syncLog.ItemsSkipped = h.result.Skipped
h.syncLog.ItemsFailed = h.result.Failed
if err := h.svc.syncLogRepo.UpdateResult(ctx, h.syncLog); err != nil {
logger.Warnf(ctx, "failed to persist sync log progress at checkpoint: %v", err)
}
return nil
}
// processSyncStreaming runs a sync through a StreamingConnector, ingesting each
// item as it arrives and checkpointing progress so the run is memory-bounded and
// resumable after a timeout.
func (s *DataSourceService) processSyncStreaming(
ctx context.Context, sc datasource.StreamingConnector,
ds *types.DataSource, syncLog *types.SyncLog,
config *types.DataSourceConfig, payload types.DataSourceSyncPayload, wasPaused bool,
) error {
// Tenant + auto-tag setup must precede fetching because the stream ingests
// each item on the fly.
ctx = context.WithValue(ctx, types.TenantIDContextKey, ds.TenantID)
tenant, err := s.tenantRepo.GetTenantByID(ctx, ds.TenantID)
if err != nil {
logger.Errorf(ctx, "failed to get tenant info: %v", err)
s.updateSyncRunResult(ctx, ds, syncLog, &types.SyncResult{}, nil,
types.SyncLogStatusFailed, fmt.Sprintf("Failed to get tenant info: %v", err), wasPaused)
return err
}
ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenant)
autoTagIDs := s.resolveAutoTagIDs(ctx, ds)
forceFull := payload.ForceFull || ds.SyncMode == types.SyncModeFull
attempt, _ := asynq.GetRetryCount(ctx)
startCursor, err := streamStartCursor(ds, forceFull, attempt)
if err != nil {
logger.Errorf(ctx, "failed to parse sync cursor: %v", err)
s.updateSyncRunResult(ctx, ds, syncLog, &types.SyncResult{}, nil,
types.SyncLogStatusFailed, fmt.Sprintf("Invalid cursor: %v", err), wasPaused)
return err
}
result := &types.SyncResult{}
handler := &streamSyncHandler{svc: s, ds: ds, tagIDs: autoTagIDs, result: result, syncLog: syncLog}
nextCursor, fetchErr := sc.FetchStream(ctx, config, startCursor, handler)
if fetchErr != nil {
// Progress so far is already checkpointed onto ds.LastSyncCursor; leave
// it in place so the Asynq retry resumes from there. Persist counts.
logger.Errorf(ctx, "streaming fetch failed: %v", fetchErr)
resultJSON, _ := result.ToJSON()
s.updateSyncRunResult(ctx, ds, syncLog, result, resultJSON,
types.SyncLogStatusFailed, fmt.Sprintf("Fetch failed: %v", fetchErr), wasPaused)
return fetchErr
}
resultJSON, _ := result.ToJSON()
if err := allFetchedItemsFailedError(result); err != nil {
logger.Errorf(ctx, "streaming sync failed while processing fetched items: %v", err)
s.updateSyncRunResult(ctx, ds, syncLog, result, resultJSON, types.SyncLogStatusFailed, err.Error(), wasPaused)
return err
}
// Persist the final cursor for the next incremental sync.
if nextCursor != nil {
if cursorJSON, cerr := nextCursor.ToJSON(); cerr == nil {
ds.LastSyncCursor = cursorJSON
}
}
ds.LastSyncAt = timePtr(time.Now().UTC())
// Surface per-document failures as a partial sync (not silent success), so
// the sync-log drawer's failure detail explains which docs didn't make it —
// the visibility gap behind "status normal but not everything syncs"
// (Tencent/WeKnora#2136). Failed nodes were not advanced in the cursor, so
// the next run retries them.
status := types.SyncLogStatusSuccess
errMsg := ""
if result.Failed > 0 {
status = types.SyncLogStatusPartial
errMsg = fmt.Sprintf("%d document(s) failed to sync", result.Failed)
}
s.updateSyncRunResult(ctx, ds, syncLog, result, resultJSON, status, errMsg, wasPaused)
logger.Infof(ctx, "streaming sync completed: ds=%s created=%d updated=%d deleted=%d skipped=%d failed=%d",
payload.DataSourceID, result.Created, result.Updated, result.Deleted, result.Skipped, result.Failed)
return nil
}
func (s *DataSourceService) updateSyncRunResult(
ctx context.Context,
ds *types.DataSource,
syncLog *types.SyncLog,
result *types.SyncResult,
resultJSON types.JSON,
status string,
errorMessage string,
wasPaused bool,
) {
syncLog.ItemsTotal = result.Total
syncLog.ItemsCreated = result.Created
syncLog.ItemsUpdated = result.Updated
syncLog.ItemsDeleted = result.Deleted
syncLog.ItemsSkipped = result.Skipped
syncLog.ItemsFailed = result.Failed
syncLog.Status = status
syncLog.FinishedAt = timePtr(time.Now().UTC())
syncLog.ErrorMessage = errorMessage
syncLog.Result = resultJSON
if err := s.syncLogRepo.UpdateResult(ctx, syncLog); err != nil {
logger.Errorf(ctx, "failed to update sync log: %v", err)
}
if status == types.SyncLogStatusFailed {
if !wasPaused {
ds.Status = types.DataSourceStatusError
}
} else if wasPaused {
ds.Status = types.DataSourceStatusPaused
} else {
ds.Status = types.DataSourceStatusActive
}
ds.ErrorMessage = errorMessage
ds.LastSyncResult = resultJSON
if err := s.dsRepo.UpdateSyncState(ctx, ds); err != nil {
logger.Errorf(ctx, "failed to update data source: %v", err)
}
}
func allFetchedItemsFailedError(result *types.SyncResult) error {
if result == nil || result.Total == 0 {
return nil
}
if result.Failed != result.Total || result.Created != 0 || result.Updated != 0 ||
result.Deleted != 0 || result.Skipped != 0 {
return nil
}
detail := ""
if len(result.Errors) > 0 {
detail = result.Errors[0].Display()
const maxDetailLen = 500
if len(detail) > maxDetailLen {
detail = detail[:maxDetailLen] + "..."
}
}
if detail == "" {
return fmt.Errorf("all fetched items failed during sync (%d/%d)", result.Failed, result.Total)
}
return fmt.Errorf("all fetched items failed during sync (%d/%d): %s", result.Failed, result.Total, detail)
}
// ValidateCredentials tests connectivity using raw credentials without persisting anything.
func (s *DataSourceService) ValidateCredentials(ctx context.Context, connectorType string, credentials map[string]interface{}) error {
connector, err := s.connectorRegistry.Get(connectorType)
if err != nil {
return err
}
config := &types.DataSourceConfig{
Type: connectorType,
Credentials: credentials,
}
if err := connector.Validate(ctx, config); err != nil {
return err
}
return nil
}
// Helper functions
func (s *DataSourceService) validateDataSourceConfig(ctx context.Context, ds *types.DataSource) error {
connector, err := s.connectorRegistry.Get(ds.Type)
if err != nil {
return err
}
config, err := ds.ParseConfig()
if err != nil {
return datasource.ErrInvalidConfig
}
return connector.Validate(ctx, config)
}
// ingestItem writes a single FetchedItem into the knowledge base.
// If a knowledge item with the same external_id already exists, it is deleted first (update = delete + re-create).
//
// Routing logic:
// - Has Content bytes → CreateKnowledgeFromFile (走完整的文档解析 pipeline)
// - Has URL only → CreateKnowledgeFromURL (让 WeKnora 下载并解析)
//
// Returns (isUpdate, error) — isUpdate is true when an existing item was replaced.
func (s *DataSourceService) ingestItem(ctx context.Context, ds *types.DataSource, item *types.FetchedItem, tagIDs []string) (bool, error) {
channel := ds.Type // e.g. "feishu", "notion"
metadata := map[string]string{
"external_id": item.ExternalID,
"source_resource_id": item.SourceResourceID,
"datasource_id": ds.ID,
}
for k, v := range item.Metadata {
metadata[k] = v
}
// Check if a knowledge item with this external_id already exists → delete it first (update)
isUpdate := false
if item.ExternalID != "" {
repo := s.knowledgeService.GetRepository()
existing, err := repo.FindByMetadataKey(ctx, ds.TenantID, ds.KnowledgeBaseID, "external_id", item.ExternalID)
if err != nil {
logger.Warnf(ctx, "failed to check existing knowledge for external_id=%s: %v", item.ExternalID, err)
// Non-fatal: proceed with creation (may produce duplicate)
} else if existing != nil {
logger.Infof(ctx, "found existing knowledge %s for external_id=%s, deleting for update", existing.ID, item.ExternalID)
if err := s.knowledgeService.DeleteKnowledge(ctx, existing.ID); err != nil {
logger.Warnf(ctx, "failed to delete existing knowledge %s: %v", existing.ID, err)
} else {
isUpdate = true
}
}
}
// Case 1: content already fetched → build a FileHeader from bytes and call CreateKnowledgeFromFile
if len(item.Content) > 0 {
fh, err := bytesToFileHeader(item.Content, item.FileName)
if err != nil {
return isUpdate, fmt.Errorf("build file header: %w", err)
}
_, err = s.knowledgeService.CreateKnowledgeFromFile(
ctx,
ds.KnowledgeBaseID,
fh,
metadata,
nil, // use KB default for multimodal
item.FileName, // customFileName — must include extension for file-type validation
tagIDs, // auto-tag from data source
channel,
nil,
)
return isUpdate, err
}
// Case 2: only a remote URL — let WeKnora handle downloading and parsing
if item.URL != "" {
_, err := s.knowledgeService.CreateKnowledgeFromURL(
ctx,
ds.KnowledgeBaseID,
item.URL,
item.FileName,
"", // auto-detect file type
nil, // use KB default for multimodal
item.Title,
tagIDs, // auto-tag from data source
channel,
nil,
)
return isUpdate, err
}
return isUpdate, fmt.Errorf("item has neither content nor URL")
}
// bytesToFileHeader wraps a []byte into a *multipart.FileHeader so it can be
// consumed by KnowledgeService.CreateKnowledgeFromFile.
func bytesToFileHeader(data []byte, filename string) (*multipart.FileHeader, error) {
var buf bytes.Buffer
writer := multipart.NewWriter(&buf)
// Create a form file part
partHeader := make(textproto.MIMEHeader)
partHeader.Set("Content-Disposition", fmt.Sprintf(`form-data; name="file"; filename="%s"`, filename))
partHeader.Set("Content-Type", "application/octet-stream")
part, err := writer.CreatePart(partHeader)
if err != nil {
return nil, fmt.Errorf("create multipart part: %w", err)
}
if _, err := part.Write(data); err != nil {
return nil, fmt.Errorf("write data to part: %w", err)
}
if err := writer.Close(); err != nil {
return nil, fmt.Errorf("close multipart writer: %w", err)
}
// Parse the multipart data to get a FileHeader
reader := multipart.NewReader(&buf, writer.Boundary())
form, err := reader.ReadForm(int64(len(data)) + 1024)
if err != nil {
return nil, fmt.Errorf("read multipart form: %w", err)
}
files := form.File["file"]
if len(files) == 0 {
return nil, fmt.Errorf("no file in multipart form")
}
return files[0], nil
}
func timePtr(t time.Time) *time.Time {
utc := t.UTC()
return &utc
}