c695125dce
Large Feishu wikis stopped syncing after the first batch (Tencent/WeKnora#2136: 5000+ docs, only ~150 ingested; #1262: a child-list failure aborted the run). Three compounding causes, fixed in layers: 1. Rate-limit / transient failures had no retry. Feishu's wiki/drive/export APIs are aggressively rate limited; a thousand-doc sync issues tens of thousands of calls, and a single 429/5xx burst failed whole swathes of docs. Add exponential backoff honouring Retry-After to doRequest (JSON) and downloadRawBytes (files): 429 up to 3x, 5xx once, transport errors back off. 2. The batch path fetched the entire tree into memory before ingesting, so a sync that hit the 2h task timeout lost all progress and restarted from scratch, and a transient per-document export failure advanced the cursor and skipped that doc permanently. Add an optional StreamingConnector interface, implemented for Feishu: FetchStream interleaves fetch -> ingest -> checkpoint, bounding memory to one item and persisting a per-node edit-time cursor at page boundaries. The service dispatches to it via type assertion; connectors without it fall back to FetchAll / FetchIncremental unchanged (non-breaking). A resumed/incremental run skips nodes already recorded at their current edit time (converges), a failed fetch does not advance the cursor (retried next run instead of dropped), and partial listing failures emit error items and carry prior edit times forward instead of aborting the sync. 3. Failures and skips were invisible. Unsupported node types (mindnote/slides/…) were skipped with no item, error or log ("13 docs synced only 3"). FetchAll now tallies discovered/fetched/failed/skipped per resource with a per-type skip breakdown and a progress line every 100 nodes. The backend already persisted per-item errors in SyncLog.result.errors but the frontend type omitted the field; declare `result` on SyncLog and render up to 50 failed documents in the sync-log drawer (#1262). i18n for zh-CN/en-US/ko-KR/ru-RU. result.errors is a jsonb column shipped in every list response, so it is capped at 100 samples server-side (recordSyncError); the exact failure total comes from the bounded items_failed counter, which the drawer uses for its count and "N more" line so a sync that fails thousands of docs stays bounded. Each sample is a structured, localisable SyncItemError instead of a raw "title: ...api error: status=500 body={json}" dump that repeated the title and leaked status/body/log_id. The connector (feishuFailure) classifies the error into a stable i18n code (+ the feishu code as a param); the service builds {title, code, params, message}; the drawer localises via vue-i18n to the viewer's language (zh/en/ko/ru), falling back to message. A custom UnmarshalJSON keeps old string-form logs readable. The banner is localised from items_failed too. Raw errors stay in the server logs. Follows the two-tier norm of Airbyte/Fivetran/Onyx: human, actionable, localised summaries in the UI; raw detail in logs. Transient errors say "will retry" (the cursor is retained); auth/permission errors point at the fix instead. Tests: client_retry_test (backoff), tally_test (accounting), connector_stream / datasource_stream (streaming dispatch + checkpoint), connector_convergence (timeout + transient-failure resume converges to full coverage; proven to catch #2136 via bug injection), datasource_result_cap (error-sample cap + structured localisable SyncItemError, no raw body), connector_error_reason (error→i18n-code classification), sync_item_error (legacy-string + structured back-compat) — all in the normal suite. connector_realapi / connector_seed are an opt-in real-Feishu harness gated by the feishu_integration build tag AND env vars, never run in CI.
1212 lines
39 KiB
Go
1212 lines
39 KiB
Go
package service
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"mime/multipart"
|
|
"net/textproto"
|
|
"reflect"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/Tencent/WeKnora/internal/datasource"
|
|
"github.com/Tencent/WeKnora/internal/logger"
|
|
"github.com/Tencent/WeKnora/internal/tracing/langfuse"
|
|
"github.com/Tencent/WeKnora/internal/types"
|
|
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
|
secutils "github.com/Tencent/WeKnora/internal/utils"
|
|
"github.com/hibiken/asynq"
|
|
)
|
|
|
|
// DataSourceService implements the DataSourceService interface
|
|
type DataSourceService struct {
|
|
dsRepo interfaces.DataSourceRepository
|
|
syncLogRepo interfaces.SyncLogRepository
|
|
knowledgeService interfaces.KnowledgeService
|
|
kbService interfaces.KnowledgeBaseService
|
|
taskEnqueuer interfaces.TaskEnqueuer
|
|
connectorRegistry *datasource.ConnectorRegistry
|
|
scheduler *datasource.Scheduler
|
|
tenantRepo interfaces.TenantRepository
|
|
tagService interfaces.KnowledgeTagService
|
|
}
|
|
|
|
// NewDataSourceService creates a new data source service
|
|
func NewDataSourceService(
|
|
dsRepo interfaces.DataSourceRepository,
|
|
syncLogRepo interfaces.SyncLogRepository,
|
|
knowledgeService interfaces.KnowledgeService,
|
|
kbService interfaces.KnowledgeBaseService,
|
|
taskEnqueuer interfaces.TaskEnqueuer,
|
|
connectorRegistry *datasource.ConnectorRegistry,
|
|
scheduler *datasource.Scheduler,
|
|
tenantRepo interfaces.TenantRepository,
|
|
tagService interfaces.KnowledgeTagService,
|
|
) interfaces.DataSourceService {
|
|
return &DataSourceService{
|
|
dsRepo: dsRepo,
|
|
syncLogRepo: syncLogRepo,
|
|
knowledgeService: knowledgeService,
|
|
kbService: kbService,
|
|
taskEnqueuer: taskEnqueuer,
|
|
connectorRegistry: connectorRegistry,
|
|
scheduler: scheduler,
|
|
tenantRepo: tenantRepo,
|
|
tagService: tagService,
|
|
}
|
|
}
|
|
|
|
// CreateDataSource creates a new data source configuration
|
|
func (s *DataSourceService) CreateDataSource(ctx context.Context, ds *types.DataSource) (*types.DataSource, error) {
|
|
if ds == nil {
|
|
return nil, datasource.ErrDataSourceInvalid
|
|
}
|
|
|
|
// Validate knowledge base exists
|
|
kb, err := s.kbService.GetKnowledgeBaseByID(ctx, ds.KnowledgeBaseID)
|
|
if err != nil || kb == nil {
|
|
return nil, datasource.ErrKnowledgeBaseNotFound
|
|
}
|
|
if kb.TenantID != ds.TenantID {
|
|
return nil, datasource.ErrKnowledgeBaseNotFound
|
|
}
|
|
|
|
// Validate connector type
|
|
_, err = s.connectorRegistry.Get(ds.Type)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Validate configuration
|
|
if cfg, err := ds.ParseConfig(); err == nil && cfg != nil {
|
|
cfg.StripNonSecretCredentials(ds.Type)
|
|
if blob, err := cfg.ToJSON(); err == nil {
|
|
ds.Config = blob
|
|
}
|
|
}
|
|
if err := s.validateDataSourceConfig(ctx, ds); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Create in database
|
|
if err := s.dsRepo.Create(ctx, ds); err != nil {
|
|
logger.Errorf(ctx, "failed to create data source: %v", err)
|
|
return nil, err
|
|
}
|
|
|
|
// Register cron schedule if configured
|
|
if ds.SyncSchedule != "" && ds.Status == types.DataSourceStatusActive {
|
|
if err := s.scheduler.AddOrUpdate(ds); err != nil {
|
|
logger.Warnf(ctx, "failed to register cron for ds=%s: %v", ds.ID, err)
|
|
}
|
|
}
|
|
|
|
logger.Infof(ctx, "data source created: id=%s type=%s kb=%s", ds.ID, ds.Type, ds.KnowledgeBaseID)
|
|
return ds, nil
|
|
}
|
|
|
|
// GetDataSource retrieves a data source by ID
|
|
func (s *DataSourceService) GetDataSource(ctx context.Context, id string) (*types.DataSource, error) {
|
|
ds, err := s.dsRepo.FindByID(ctx, id)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return ds, nil
|
|
}
|
|
|
|
// ListDataSources lists all data sources for a knowledge base
|
|
func (s *DataSourceService) ListDataSources(ctx context.Context, kbID string) ([]*types.DataSource, error) {
|
|
dataSources, err := s.dsRepo.FindByKnowledgeBase(ctx, kbID)
|
|
if err != nil {
|
|
logger.Errorf(ctx, "failed to list data sources: %v", err)
|
|
return nil, err
|
|
}
|
|
|
|
// Attach latest sync log to each data source
|
|
for _, ds := range dataSources {
|
|
log, _ := s.syncLogRepo.FindLatest(ctx, ds.ID)
|
|
if log != nil {
|
|
ds.LatestSyncLog = log
|
|
}
|
|
}
|
|
|
|
return dataSources, nil
|
|
}
|
|
|
|
// UpdateDataSource updates an existing data source
|
|
func (s *DataSourceService) UpdateDataSource(ctx context.Context, ds *types.DataSource) (*types.DataSource, error) {
|
|
if ds == nil || ds.ID == "" {
|
|
return nil, datasource.ErrDataSourceInvalid
|
|
}
|
|
|
|
// Verify data source exists
|
|
existing, err := s.dsRepo.FindByID(ctx, ds.ID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if ds.KnowledgeBaseID == "" {
|
|
ds.KnowledgeBaseID = existing.KnowledgeBaseID
|
|
}
|
|
if ds.KnowledgeBaseID != existing.KnowledgeBaseID {
|
|
return nil, fmt.Errorf("changing knowledge base is not allowed")
|
|
}
|
|
|
|
if ds.TenantID == 0 {
|
|
ds.TenantID = existing.TenantID
|
|
}
|
|
if ds.TenantID != existing.TenantID {
|
|
return nil, datasource.ErrDataSourceInvalid
|
|
}
|
|
|
|
// Credentials NEVER flow through this endpoint — they live behind the
|
|
// /credentials subresource. Force-preserve the stored credentials map
|
|
// regardless of what the body says. Log a warning if a stale caller
|
|
// passes one so we can spot them and migrate later. Non-credential
|
|
// fields of Config (Type / ResourceIDs / Settings) flow through.
|
|
var mergedCfg, existingParsedCfg *types.DataSourceConfig
|
|
if len(ds.Config) > 0 {
|
|
incomingCfg, parseIncErr := ds.ParseConfig()
|
|
existingCfg, parseExErr := existing.ParseConfig()
|
|
if parseIncErr == nil && parseExErr == nil && incomingCfg != nil {
|
|
if incomingCfg.HasCredentials() {
|
|
logger.Warnf(ctx,
|
|
"deprecated: credentials in PUT /datasource/%s body are ignored; use PUT /credentials instead",
|
|
secutils.SanitizeForLog(ds.ID))
|
|
}
|
|
merged := *incomingCfg
|
|
if existingCfg != nil {
|
|
merged.Credentials = existingCfg.Credentials
|
|
} else {
|
|
merged.Credentials = nil
|
|
}
|
|
merged.StripNonSecretCredentials(ds.Type)
|
|
if blob, err := merged.ToJSON(); err == nil {
|
|
ds.Config = blob
|
|
}
|
|
mergedCfg = &merged
|
|
existingParsedCfg = existingCfg
|
|
}
|
|
}
|
|
|
|
// Validate new configuration if non-credential fields changed. Skip
|
|
// when there are no stored credentials yet (validators would fail with
|
|
// no token to call the live API) and when the parsed config is
|
|
// structurally identical.
|
|
configActuallyChanged := true
|
|
if mergedCfg != nil && existingParsedCfg != nil {
|
|
configActuallyChanged = !reflect.DeepEqual(*mergedCfg, *existingParsedCfg)
|
|
}
|
|
hasCreds := mergedCfg != nil && mergedCfg.HasConfiguredCredentials(ds.Type)
|
|
if hasCreds && (ds.Type != existing.Type || configActuallyChanged) {
|
|
if err := s.validateDataSourceConfig(ctx, ds); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
|
|
if err := s.dsRepo.Update(ctx, ds); err != nil {
|
|
logger.Errorf(ctx, "failed to update data source: %v", err)
|
|
return nil, err
|
|
}
|
|
|
|
// Update cron schedule
|
|
if err := s.scheduler.AddOrUpdate(ds); err != nil {
|
|
logger.Warnf(ctx, "failed to update cron for ds=%s: %v", ds.ID, err)
|
|
}
|
|
|
|
logger.Infof(ctx, "data source updated: id=%s", ds.ID)
|
|
return ds, nil
|
|
}
|
|
|
|
// UpdateDataSourceCredentials replaces the connector credential map. This is
|
|
// a single atomic write; the previous credential set is discarded entirely
|
|
// (callers cannot patch individual keys because half-configured connector
|
|
// auth is meaningless). After persisting, the live connection is validated
|
|
// so the caller learns immediately if the new credentials are wrong.
|
|
func (s *DataSourceService) UpdateDataSourceCredentials(
|
|
ctx context.Context, id string, credentials map[string]interface{},
|
|
) (*types.DataSource, error) {
|
|
if id == "" {
|
|
return nil, datasource.ErrDataSourceInvalid
|
|
}
|
|
existing, err := s.dsRepo.FindByID(ctx, id)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
parsed, err := existing.ParseConfig()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if parsed == nil {
|
|
parsed = &types.DataSourceConfig{Type: existing.Type}
|
|
}
|
|
parsed.Credentials = credentials
|
|
parsed.StripNonSecretCredentials(existing.Type)
|
|
blob, err := parsed.ToJSON()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
existing.Config = blob
|
|
|
|
// Run live validation now that the credentials are in place — surfaces
|
|
// "wrong token" feedback immediately to the user instead of waiting for
|
|
// the next scheduled sync.
|
|
if err := s.validateDataSourceConfig(ctx, existing); err != nil {
|
|
return nil, err
|
|
}
|
|
if err := s.dsRepo.Update(ctx, existing); err != nil {
|
|
return nil, err
|
|
}
|
|
logger.Infof(ctx, "DataSource credentials updated: id=%s", secutils.SanitizeForLog(id))
|
|
return existing, nil
|
|
}
|
|
|
|
// ClearDataSourceCredentials wipes the connector credential map without
|
|
// touching any other config field. Idempotent.
|
|
func (s *DataSourceService) ClearDataSourceCredentials(ctx context.Context, id string) error {
|
|
if id == "" {
|
|
return datasource.ErrDataSourceInvalid
|
|
}
|
|
existing, err := s.dsRepo.FindByID(ctx, id)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
parsed, err := existing.ParseConfig()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if parsed == nil {
|
|
return nil
|
|
}
|
|
parsed.StripNonSecretCredentials(existing.Type)
|
|
if !parsed.HasConfiguredCredentials(existing.Type) {
|
|
blob, err := parsed.ToJSON()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
existing.Config = blob
|
|
return s.dsRepo.Update(ctx, existing)
|
|
}
|
|
parsed.Credentials = nil
|
|
blob, err := parsed.ToJSON()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
existing.Config = blob
|
|
if err := s.dsRepo.Update(ctx, existing); err != nil {
|
|
return err
|
|
}
|
|
logger.Infof(ctx, "DataSource credentials cleared by user: id=%s", secutils.SanitizeForLog(id))
|
|
return nil
|
|
}
|
|
|
|
// DeleteDataSource deletes a data source (soft delete)
|
|
func (s *DataSourceService) DeleteDataSource(ctx context.Context, id string) error {
|
|
// Verify data source exists
|
|
_, err := s.dsRepo.FindByID(ctx, id)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := s.dsRepo.Delete(ctx, id); err != nil {
|
|
logger.Errorf(ctx, "failed to delete data source: %v", err)
|
|
return err
|
|
}
|
|
|
|
// Remove cron schedule
|
|
s.scheduler.Remove(id)
|
|
|
|
// Cancel any pending/running sync logs so queued asynq tasks won't retry
|
|
if err := s.syncLogRepo.CancelPendingByDataSource(ctx, id); err != nil {
|
|
logger.Warnf(ctx, "failed to cancel pending sync logs for ds=%s: %v", id, err)
|
|
}
|
|
|
|
logger.Infof(ctx, "data source deleted: id=%s", id)
|
|
return nil
|
|
}
|
|
|
|
// ValidateConnection tests the connection to an external data source
|
|
func (s *DataSourceService) ValidateConnection(ctx context.Context, dsID string) error {
|
|
ds, err := s.GetDataSource(ctx, dsID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Get connector
|
|
connector, err := s.connectorRegistry.Get(ds.Type)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Parse configuration
|
|
config, err := ds.ParseConfig()
|
|
if err != nil {
|
|
return datasource.ErrInvalidConfig
|
|
}
|
|
|
|
// Validate connection
|
|
if err := connector.Validate(ctx, config); err != nil {
|
|
// Update data source with error
|
|
ds.Status = types.DataSourceStatusError
|
|
ds.ErrorMessage = err.Error()
|
|
_ = s.dsRepo.Update(ctx, ds)
|
|
return err
|
|
}
|
|
|
|
// Clear error if it was previously in error state
|
|
if ds.Status == types.DataSourceStatusError {
|
|
ds.Status = types.DataSourceStatusActive
|
|
ds.ErrorMessage = ""
|
|
_ = s.dsRepo.Update(ctx, ds)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// ListAvailableResources lists resources available for sync in the external system.
|
|
// parentID enables lazy (on-demand) loading of hierarchical resources: pass "" to
|
|
// list the top level, or a resource's ExternalID to list only its direct children.
|
|
func (s *DataSourceService) ListAvailableResources(
|
|
ctx context.Context, dsID string, parentID string,
|
|
) ([]types.Resource, error) {
|
|
ds, err := s.GetDataSource(ctx, dsID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Get connector
|
|
connector, err := s.connectorRegistry.Get(ds.Type)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Parse configuration
|
|
config, err := ds.ParseConfig()
|
|
if err != nil {
|
|
return nil, datasource.ErrInvalidConfig
|
|
}
|
|
|
|
// List resources
|
|
resources, err := connector.ListResources(ctx, config, parentID)
|
|
if err != nil {
|
|
logger.Errorf(ctx, "failed to list resources: %v", err)
|
|
return nil, err
|
|
}
|
|
|
|
return resources, nil
|
|
}
|
|
|
|
// ResolveResourceAncestors resolves the ancestor ExternalIDs needed to reveal the
|
|
// given resources in a lazily-loaded picker (see the connector method for details).
|
|
func (s *DataSourceService) ResolveResourceAncestors(
|
|
ctx context.Context, dsID string, resourceIDs []string,
|
|
) ([]string, error) {
|
|
if len(resourceIDs) == 0 {
|
|
return []string{}, nil
|
|
}
|
|
|
|
ds, err := s.GetDataSource(ctx, dsID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
connector, err := s.connectorRegistry.Get(ds.Type)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
config, err := ds.ParseConfig()
|
|
if err != nil {
|
|
return nil, datasource.ErrInvalidConfig
|
|
}
|
|
|
|
ancestors, err := connector.ResolveResourceAncestors(ctx, config, resourceIDs)
|
|
if err != nil {
|
|
logger.Errorf(ctx, "failed to resolve resource ancestors: %v", err)
|
|
return nil, err
|
|
}
|
|
|
|
return ancestors, nil
|
|
}
|
|
|
|
// ManualSync triggers an immediate sync for a data source
|
|
func (s *DataSourceService) ManualSync(ctx context.Context, dsID string) (*types.SyncLog, error) {
|
|
ds, err := s.GetDataSource(ctx, dsID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if ds.Status != types.DataSourceStatusActive &&
|
|
ds.Status != types.DataSourceStatusError &&
|
|
ds.Status != types.DataSourceStatusPaused {
|
|
return nil, datasource.ErrDataSourceNotActive
|
|
}
|
|
|
|
// Create sync log
|
|
syncLog := &types.SyncLog{
|
|
DataSourceID: dsID,
|
|
TenantID: ds.TenantID,
|
|
Status: types.SyncLogStatusRunning,
|
|
StartedAt: time.Now().UTC(),
|
|
}
|
|
|
|
if err := s.syncLogRepo.Create(ctx, syncLog); err != nil {
|
|
logger.Errorf(ctx, "failed to create sync log: %v", err)
|
|
return nil, err
|
|
}
|
|
|
|
// Enqueue sync task
|
|
payload := &types.DataSourceSyncPayload{
|
|
DataSourceID: dsID,
|
|
TenantID: ds.TenantID,
|
|
SyncLogID: syncLog.ID,
|
|
ForceFull: false,
|
|
}
|
|
langfuse.InjectTracing(ctx, payload)
|
|
|
|
payloadJSON, _ := json.Marshal(payload)
|
|
task := asynq.NewTask(types.TypeDataSourceSync, payloadJSON,
|
|
asynq.Queue(types.QueueSync), asynq.MaxRetry(5), asynq.Timeout(2*time.Hour))
|
|
|
|
_, err = s.taskEnqueuer.Enqueue(task)
|
|
if err != nil {
|
|
logger.Errorf(ctx, "failed to enqueue sync task: %v", err)
|
|
syncLog.Status = types.SyncLogStatusFailed
|
|
syncLog.FinishedAt = timePtr(time.Now().UTC())
|
|
syncLog.ErrorMessage = err.Error()
|
|
_ = s.syncLogRepo.Update(ctx, syncLog)
|
|
if ds.Status != types.DataSourceStatusPaused {
|
|
ds.Status = types.DataSourceStatusError
|
|
}
|
|
ds.ErrorMessage = fmt.Sprintf("Failed to enqueue sync: %v", err)
|
|
_ = s.dsRepo.Update(ctx, ds)
|
|
return nil, err
|
|
}
|
|
|
|
logger.Infof(ctx, "sync task enqueued: ds=%s syncLog=%s", dsID, syncLog.ID)
|
|
return syncLog, nil
|
|
}
|
|
|
|
// PauseDataSource pauses a data source's scheduled syncs
|
|
func (s *DataSourceService) PauseDataSource(ctx context.Context, id string) error {
|
|
ds, err := s.GetDataSource(ctx, id)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
ds.Status = types.DataSourceStatusPaused
|
|
if err := s.dsRepo.Update(ctx, ds); err != nil {
|
|
logger.Errorf(ctx, "failed to pause data source: %v", err)
|
|
return err
|
|
}
|
|
|
|
// Remove cron schedule
|
|
s.scheduler.Remove(id)
|
|
|
|
logger.Infof(ctx, "data source paused: id=%s", id)
|
|
return nil
|
|
}
|
|
|
|
// ResumeDataSource resumes a paused data source
|
|
func (s *DataSourceService) ResumeDataSource(ctx context.Context, id string) error {
|
|
ds, err := s.GetDataSource(ctx, id)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
ds.Status = types.DataSourceStatusActive
|
|
if err := s.dsRepo.Update(ctx, ds); err != nil {
|
|
logger.Errorf(ctx, "failed to resume data source: %v", err)
|
|
return err
|
|
}
|
|
|
|
// Re-register cron schedule
|
|
if err := s.scheduler.AddOrUpdate(ds); err != nil {
|
|
logger.Warnf(ctx, "failed to re-register cron for ds=%s: %v", ds.ID, err)
|
|
}
|
|
|
|
logger.Infof(ctx, "data source resumed: id=%s", id)
|
|
return nil
|
|
}
|
|
|
|
// GetSyncLogs retrieves sync history for a data source
|
|
func (s *DataSourceService) GetSyncLogs(ctx context.Context, dsID string, limit int, offset int) ([]*types.SyncLog, error) {
|
|
logs, err := s.syncLogRepo.FindByDataSource(ctx, dsID, limit, offset)
|
|
if err != nil {
|
|
logger.Errorf(ctx, "failed to get sync logs: %v", err)
|
|
return nil, err
|
|
}
|
|
return logs, nil
|
|
}
|
|
|
|
// GetSyncLog retrieves a specific sync log entry
|
|
func (s *DataSourceService) GetSyncLog(ctx context.Context, syncLogID string) (*types.SyncLog, error) {
|
|
log, err := s.syncLogRepo.FindByID(ctx, syncLogID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return log, nil
|
|
}
|
|
|
|
// ProcessSync handles the actual sync operation (called by asynq task)
|
|
func (s *DataSourceService) ProcessSync(ctx context.Context, task *asynq.Task) error {
|
|
var payload types.DataSourceSyncPayload
|
|
if err := json.Unmarshal(task.Payload(), &payload); err != nil {
|
|
logger.Errorf(ctx, "failed to unmarshal sync payload: %v", err)
|
|
return err
|
|
}
|
|
|
|
logger.Infof(ctx, "processing data source sync: ds=%s syncLog=%s", payload.DataSourceID, payload.SyncLogID)
|
|
|
|
// Get data source
|
|
ds, err := s.GetDataSource(ctx, payload.DataSourceID)
|
|
if err != nil {
|
|
logger.Warnf(ctx, "data source not found (likely deleted), cancelling sync: ds=%s err=%v", payload.DataSourceID, err)
|
|
if syncLog, slErr := s.syncLogRepo.FindByID(ctx, payload.SyncLogID); slErr == nil && syncLog != nil {
|
|
syncLog.Status = types.SyncLogStatusCanceled
|
|
syncLog.FinishedAt = timePtr(time.Now().UTC())
|
|
syncLog.ErrorMessage = "data source has been deleted"
|
|
_ = s.syncLogRepo.Update(ctx, syncLog)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Get sync log
|
|
syncLog, err := s.syncLogRepo.FindByID(ctx, payload.SyncLogID)
|
|
if err != nil {
|
|
logger.Errorf(ctx, "failed to get sync log: %v", err)
|
|
return nil
|
|
}
|
|
|
|
if _, err := s.kbService.GetKnowledgeBaseByID(ctx, ds.KnowledgeBaseID); err != nil {
|
|
logger.Warnf(ctx, "knowledge base not found (likely deleted), cancelling sync: kb=%s ds=%s err=%v",
|
|
ds.KnowledgeBaseID, payload.DataSourceID, err)
|
|
syncLog.Status = types.SyncLogStatusCanceled
|
|
syncLog.FinishedAt = timePtr(time.Now().UTC())
|
|
syncLog.ErrorMessage = "knowledge base has been deleted"
|
|
_ = s.syncLogRepo.Update(ctx, syncLog)
|
|
return nil
|
|
}
|
|
|
|
wasPaused := ds.Status == types.DataSourceStatusPaused
|
|
|
|
// Get connector
|
|
connector, err := s.connectorRegistry.Get(ds.Type)
|
|
if err != nil {
|
|
logger.Errorf(ctx, "connector not found: type=%s", ds.Type)
|
|
syncLog.Status = types.SyncLogStatusFailed
|
|
syncLog.FinishedAt = timePtr(time.Now().UTC())
|
|
syncLog.ErrorMessage = fmt.Sprintf("Connector not found: %s", ds.Type)
|
|
_ = s.syncLogRepo.Update(ctx, syncLog)
|
|
if !wasPaused {
|
|
ds.Status = types.DataSourceStatusError
|
|
}
|
|
ds.ErrorMessage = syncLog.ErrorMessage
|
|
_ = s.dsRepo.Update(ctx, ds)
|
|
return err
|
|
}
|
|
|
|
// Parse configuration
|
|
config, err := ds.ParseConfig()
|
|
if err != nil {
|
|
logger.Errorf(ctx, "failed to parse config: %v", err)
|
|
syncLog.Status = types.SyncLogStatusFailed
|
|
syncLog.FinishedAt = timePtr(time.Now().UTC())
|
|
syncLog.ErrorMessage = fmt.Sprintf("Invalid configuration: %v", err)
|
|
_ = s.syncLogRepo.Update(ctx, syncLog)
|
|
if !wasPaused {
|
|
ds.Status = types.DataSourceStatusError
|
|
}
|
|
ds.ErrorMessage = syncLog.ErrorMessage
|
|
_ = s.dsRepo.Update(ctx, ds)
|
|
return err
|
|
}
|
|
|
|
// Streaming path: connectors that support it interleave fetch→ingest→
|
|
// checkpoint so a large sync bounds memory and resumes after a timeout
|
|
// instead of restarting (Tencent/WeKnora#2136). Others fall back below.
|
|
if sc, ok := connector.(datasource.StreamingConnector); ok {
|
|
return s.processSyncStreaming(ctx, sc, ds, syncLog, config, payload, wasPaused)
|
|
}
|
|
|
|
// Fetch items based on sync mode
|
|
var items []types.FetchedItem
|
|
var nextCursor *types.SyncCursor
|
|
var fetchErr error
|
|
|
|
if payload.ForceFull || ds.SyncMode == types.SyncModeFull {
|
|
// Full sync
|
|
items, fetchErr = connector.FetchAll(ctx, config, config.ResourceIDs)
|
|
logger.Infof(ctx, "full sync fetched %d items", len(items))
|
|
} else {
|
|
// Incremental sync
|
|
cursor, _ := ds.ParseSyncCursor()
|
|
items, nextCursor, fetchErr = connector.FetchIncremental(ctx, config, cursor)
|
|
logger.Infof(ctx, "incremental sync fetched %d items", len(items))
|
|
}
|
|
|
|
var fetchWarnings []string
|
|
var partialFetch *datasource.PartialFetchError
|
|
if errors.As(fetchErr, &partialFetch) {
|
|
fetchWarnings = partialFetch.Details
|
|
fetchErr = nil
|
|
}
|
|
|
|
if fetchErr != nil {
|
|
// Persist connector cursor even when fetch failed so transient outages
|
|
// (e.g. RSS feed downtime) do not force a full re-ingest on recovery.
|
|
if nextCursor != nil {
|
|
if cursorJSON, cerr := nextCursor.ToJSON(); cerr == nil {
|
|
ds.LastSyncCursor = cursorJSON
|
|
if uerr := s.dsRepo.UpdateSyncState(ctx, ds); uerr != nil {
|
|
logger.Warnf(ctx, "failed to persist sync cursor after fetch error: %v", uerr)
|
|
}
|
|
}
|
|
}
|
|
logger.Errorf(ctx, "fetch operation failed: %v", fetchErr)
|
|
syncLog.Status = types.SyncLogStatusFailed
|
|
syncLog.FinishedAt = timePtr(time.Now().UTC())
|
|
syncLog.ErrorMessage = fmt.Sprintf("Fetch failed: %v", fetchErr)
|
|
_ = s.syncLogRepo.Update(ctx, syncLog)
|
|
if !wasPaused {
|
|
ds.Status = types.DataSourceStatusError
|
|
}
|
|
ds.ErrorMessage = syncLog.ErrorMessage
|
|
_ = s.dsRepo.Update(ctx, ds)
|
|
return fetchErr
|
|
}
|
|
|
|
// Process fetched items and write to knowledge base
|
|
var result = &types.SyncResult{
|
|
Total: len(items),
|
|
}
|
|
|
|
// Set tenant context so KnowledgeService can resolve tenant info correctly
|
|
ctx = context.WithValue(ctx, types.TenantIDContextKey, ds.TenantID)
|
|
|
|
tenant, err := s.tenantRepo.GetTenantByID(ctx, ds.TenantID)
|
|
if err != nil {
|
|
logger.Errorf(ctx, "failed to get tenant info: %v", err)
|
|
syncLog.Status = types.SyncLogStatusFailed
|
|
syncLog.FinishedAt = timePtr(time.Now().UTC())
|
|
syncLog.ErrorMessage = fmt.Sprintf("Failed to get tenant info: %v", err)
|
|
_ = s.syncLogRepo.Update(ctx, syncLog)
|
|
if !wasPaused {
|
|
ds.Status = types.DataSourceStatusError
|
|
}
|
|
ds.ErrorMessage = syncLog.ErrorMessage
|
|
_ = s.dsRepo.Update(ctx, ds)
|
|
return err
|
|
}
|
|
ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenant)
|
|
|
|
// Auto-tag: find or create a tag for this data source so synced items are easily identifiable
|
|
autoTagIDs := s.resolveAutoTagIDs(ctx, ds)
|
|
|
|
for _, item := range items {
|
|
item := item
|
|
s.applyFetchedItem(ctx, ds, &item, autoTagIDs, result)
|
|
}
|
|
|
|
resultJSON, _ := result.ToJSON()
|
|
if err := allFetchedItemsFailedError(result); err != nil {
|
|
logger.Errorf(ctx, "data source sync failed while processing fetched items: %v", err)
|
|
s.updateSyncRunResult(ctx, ds, syncLog, result, resultJSON, types.SyncLogStatusFailed, err.Error(), wasPaused)
|
|
return err
|
|
}
|
|
|
|
// Update cursor for next incremental sync
|
|
if nextCursor != nil {
|
|
cursorJSON, _ := nextCursor.ToJSON()
|
|
ds.LastSyncCursor = cursorJSON
|
|
}
|
|
|
|
ds.LastSyncAt = timePtr(time.Now().UTC())
|
|
syncStatus := types.SyncLogStatusSuccess
|
|
syncErrorMessage := ""
|
|
if len(fetchWarnings) > 0 {
|
|
syncStatus = types.SyncLogStatusPartial
|
|
syncErrorMessage = fmt.Sprintf("Some feeds failed: %s", strings.Join(fetchWarnings, "; "))
|
|
for _, w := range fetchWarnings {
|
|
result.Errors = append(result.Errors, types.SyncItemError{Message: w})
|
|
}
|
|
resultJSON, _ = result.ToJSON()
|
|
}
|
|
s.updateSyncRunResult(ctx, ds, syncLog, result, resultJSON, syncStatus, syncErrorMessage, wasPaused)
|
|
|
|
logger.Infof(ctx, "data source sync completed: ds=%s created=%d updated=%d deleted=%d",
|
|
payload.DataSourceID, syncLog.ItemsCreated, syncLog.ItemsUpdated, syncLog.ItemsDeleted)
|
|
|
|
return nil
|
|
}
|
|
|
|
// resolveAutoTagIDs finds or creates the per-data-source tag applied to every
|
|
// synced item so results are identifiable in the KB. A tag failure is
|
|
// non-fatal: the sync proceeds untagged.
|
|
func (s *DataSourceService) resolveAutoTagIDs(ctx context.Context, ds *types.DataSource) []string {
|
|
autoTagIDs := []string{}
|
|
if autoTag, tagErr := s.tagService.FindOrCreateTagByName(ctx, ds.KnowledgeBaseID, ds.Name); tagErr != nil {
|
|
logger.Warnf(ctx, "failed to find/create auto-tag %q: %v (proceeding without tag)", ds.Name, tagErr)
|
|
} else if autoTag != nil {
|
|
autoTagIDs = append(autoTagIDs, autoTag.ID)
|
|
logger.Infof(ctx, "using auto-tag %q (id=%s) for data source sync", ds.Name, autoTag.ID)
|
|
}
|
|
return autoTagIDs
|
|
}
|
|
|
|
// maxSyncResultErrors bounds the per-item error sample retained in
|
|
// SyncResult.Errors. That slice is persisted as jsonb and returned in every
|
|
// sync-log list response, so an unbounded list on a sync that fails thousands of
|
|
// documents means multi-MB DB rows and payloads. The accurate failure count
|
|
// lives in SyncResult.Failed (a bounded int); this list only keeps a sample for
|
|
// display (Tencent/WeKnora#2136 / #1262).
|
|
const maxSyncResultErrors = 100
|
|
|
|
// recordSyncError appends an error sample to result.Errors, capped at
|
|
// maxSyncResultErrors. Callers still increment result.Failed for the exact count.
|
|
func recordSyncError(result *types.SyncResult, item types.SyncItemError) {
|
|
if len(result.Errors) < maxSyncResultErrors {
|
|
result.Errors = append(result.Errors, item)
|
|
}
|
|
}
|
|
|
|
// fetchFailureSyncError maps a connector error item into a structured, user-
|
|
// facing sample. Connectors that classify their errors (Feishu) provide a stable
|
|
// i18n code + params via metadata so the frontend localises it to the viewer's
|
|
// language; the raw status/body/log_id never leaves the server logs. Connectors
|
|
// without codes keep the raw text as a Message fallback. Best practice per
|
|
// Airbyte/Fivetran/Onyx: humanised, actionable, localised UI; raw detail in logs.
|
|
func fetchFailureSyncError(item *types.FetchedItem, rawMsg string) types.SyncItemError {
|
|
e := types.SyncItemError{Title: item.Title}
|
|
if code := item.Metadata["error_reason_code"]; code != "" {
|
|
e.Code = code
|
|
if v := item.Metadata["error_reason_code_value"]; v != "" {
|
|
e.Params = map[string]string{"code": v}
|
|
}
|
|
e.Message = item.Metadata["error_reason"] // fallback if the client lacks the key
|
|
} else {
|
|
e.Message = rawMsg
|
|
}
|
|
return e
|
|
}
|
|
|
|
// applyFetchedItem writes a single fetched item into the knowledge base and
|
|
// updates result counters. It is the shared core of the batch loop and the
|
|
// streaming handler so item classification (deleted / empty / ingest outcome)
|
|
// stays identical across both fetch paths.
|
|
func (s *DataSourceService) applyFetchedItem(
|
|
ctx context.Context, ds *types.DataSource, item *types.FetchedItem,
|
|
tagIDs []string, result *types.SyncResult,
|
|
) {
|
|
if item.IsDeleted {
|
|
if ds.SyncDeletions {
|
|
// Count only — actual KB deletion is intentionally not performed.
|
|
// Users manage knowledge removal explicitly via the KB UI to avoid
|
|
// accidental data loss from connector misdetection or reconfiguration.
|
|
result.Deleted++
|
|
}
|
|
return
|
|
}
|
|
|
|
if len(item.Content) == 0 && item.URL == "" {
|
|
// Check if this is an error item from the connector (failed to fetch content)
|
|
if errMsg, hasErr := item.Metadata["error"]; hasErr {
|
|
logger.Warnf(ctx, "item %q (external_id=%s) fetch failed: %s", item.Title, item.ExternalID, errMsg)
|
|
result.Failed++
|
|
recordSyncError(result, fetchFailureSyncError(item, errMsg))
|
|
} else {
|
|
logger.Infof(ctx, "skipping item %q (external_id=%s): no content or URL", item.Title, item.ExternalID)
|
|
result.Skipped++
|
|
}
|
|
return
|
|
}
|
|
|
|
isUpdate, err := s.ingestItem(ctx, ds, item, tagIDs)
|
|
if err != nil {
|
|
// Duplicate file/URL is not a failure — count as skipped
|
|
var dupErr *types.DuplicateKnowledgeError
|
|
if errors.As(err, &dupErr) {
|
|
logger.Infof(ctx, "item %q (external_id=%s) already exists, skipping", item.Title, item.ExternalID)
|
|
result.Skipped++
|
|
} else {
|
|
logger.Warnf(ctx, "failed to ingest item %q (external_id=%s): %v", item.Title, item.ExternalID, err)
|
|
result.Failed++
|
|
recordSyncError(result, types.SyncItemError{
|
|
Title: item.Title,
|
|
Code: "ingest_failed",
|
|
Message: "Ingest failed; see server logs",
|
|
})
|
|
}
|
|
} else if isUpdate {
|
|
result.Updated++
|
|
} else {
|
|
result.Created++
|
|
}
|
|
}
|
|
|
|
// streamStartCursor decides which cursor a streaming fetch should resume from.
|
|
// A user-triggered full sync on its first attempt drops the cursor so every
|
|
// item is re-fetched; a retried full sync (attempt > 0) and every incremental
|
|
// sync resume from the last persisted checkpoint so a timed-out run converges
|
|
// instead of restarting from scratch.
|
|
func streamStartCursor(ds *types.DataSource, forceFull bool, attempt int) (*types.SyncCursor, error) {
|
|
if forceFull && attempt == 0 {
|
|
return nil, nil
|
|
}
|
|
return ds.ParseSyncCursor()
|
|
}
|
|
|
|
// streamSyncHandler adapts a streaming fetch to the knowledge-base ingest path.
|
|
// Emit ingests each item as it arrives (bounding memory) and Checkpoint persists
|
|
// the connector cursor plus live progress counts at page boundaries.
|
|
type streamSyncHandler struct {
|
|
svc *DataSourceService
|
|
ds *types.DataSource
|
|
tagIDs []string
|
|
result *types.SyncResult
|
|
syncLog *types.SyncLog
|
|
}
|
|
|
|
// Emit ingests one streamed item. A canceled context aborts the stream so the
|
|
// connector stops fetching; per-item ingest failures are recorded in result and
|
|
// do NOT abort (matching the batch loop, which never fails the whole sync for
|
|
// one bad document).
|
|
func (h *streamSyncHandler) Emit(ctx context.Context, item types.FetchedItem) error {
|
|
if err := ctx.Err(); err != nil {
|
|
return err
|
|
}
|
|
h.result.Total++
|
|
h.svc.applyFetchedItem(ctx, h.ds, &item, h.tagIDs, h.result)
|
|
return nil
|
|
}
|
|
|
|
// Checkpoint persists the connector cursor onto the data source and mirrors the
|
|
// running counts into the sync log so progress survives a crash and the UI can
|
|
// reflect a long sync mid-flight instead of jumping from 0 to done.
|
|
func (h *streamSyncHandler) Checkpoint(ctx context.Context, cursor *types.SyncCursor) error {
|
|
if cursor == nil {
|
|
return nil
|
|
}
|
|
cursorJSON, err := cursor.ToJSON()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
h.ds.LastSyncCursor = cursorJSON
|
|
if err := h.svc.dsRepo.UpdateSyncState(ctx, h.ds); err != nil {
|
|
return err
|
|
}
|
|
|
|
// Best-effort live progress; a failure here must not abort the sync.
|
|
h.syncLog.ItemsTotal = h.result.Total
|
|
h.syncLog.ItemsCreated = h.result.Created
|
|
h.syncLog.ItemsUpdated = h.result.Updated
|
|
h.syncLog.ItemsDeleted = h.result.Deleted
|
|
h.syncLog.ItemsSkipped = h.result.Skipped
|
|
h.syncLog.ItemsFailed = h.result.Failed
|
|
if err := h.svc.syncLogRepo.UpdateResult(ctx, h.syncLog); err != nil {
|
|
logger.Warnf(ctx, "failed to persist sync log progress at checkpoint: %v", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// processSyncStreaming runs a sync through a StreamingConnector, ingesting each
|
|
// item as it arrives and checkpointing progress so the run is memory-bounded and
|
|
// resumable after a timeout.
|
|
func (s *DataSourceService) processSyncStreaming(
|
|
ctx context.Context, sc datasource.StreamingConnector,
|
|
ds *types.DataSource, syncLog *types.SyncLog,
|
|
config *types.DataSourceConfig, payload types.DataSourceSyncPayload, wasPaused bool,
|
|
) error {
|
|
// Tenant + auto-tag setup must precede fetching because the stream ingests
|
|
// each item on the fly.
|
|
ctx = context.WithValue(ctx, types.TenantIDContextKey, ds.TenantID)
|
|
tenant, err := s.tenantRepo.GetTenantByID(ctx, ds.TenantID)
|
|
if err != nil {
|
|
logger.Errorf(ctx, "failed to get tenant info: %v", err)
|
|
s.updateSyncRunResult(ctx, ds, syncLog, &types.SyncResult{}, nil,
|
|
types.SyncLogStatusFailed, fmt.Sprintf("Failed to get tenant info: %v", err), wasPaused)
|
|
return err
|
|
}
|
|
ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenant)
|
|
|
|
autoTagIDs := s.resolveAutoTagIDs(ctx, ds)
|
|
|
|
forceFull := payload.ForceFull || ds.SyncMode == types.SyncModeFull
|
|
attempt, _ := asynq.GetRetryCount(ctx)
|
|
startCursor, err := streamStartCursor(ds, forceFull, attempt)
|
|
if err != nil {
|
|
logger.Errorf(ctx, "failed to parse sync cursor: %v", err)
|
|
s.updateSyncRunResult(ctx, ds, syncLog, &types.SyncResult{}, nil,
|
|
types.SyncLogStatusFailed, fmt.Sprintf("Invalid cursor: %v", err), wasPaused)
|
|
return err
|
|
}
|
|
|
|
result := &types.SyncResult{}
|
|
handler := &streamSyncHandler{svc: s, ds: ds, tagIDs: autoTagIDs, result: result, syncLog: syncLog}
|
|
|
|
nextCursor, fetchErr := sc.FetchStream(ctx, config, startCursor, handler)
|
|
if fetchErr != nil {
|
|
// Progress so far is already checkpointed onto ds.LastSyncCursor; leave
|
|
// it in place so the Asynq retry resumes from there. Persist counts.
|
|
logger.Errorf(ctx, "streaming fetch failed: %v", fetchErr)
|
|
resultJSON, _ := result.ToJSON()
|
|
s.updateSyncRunResult(ctx, ds, syncLog, result, resultJSON,
|
|
types.SyncLogStatusFailed, fmt.Sprintf("Fetch failed: %v", fetchErr), wasPaused)
|
|
return fetchErr
|
|
}
|
|
|
|
resultJSON, _ := result.ToJSON()
|
|
if err := allFetchedItemsFailedError(result); err != nil {
|
|
logger.Errorf(ctx, "streaming sync failed while processing fetched items: %v", err)
|
|
s.updateSyncRunResult(ctx, ds, syncLog, result, resultJSON, types.SyncLogStatusFailed, err.Error(), wasPaused)
|
|
return err
|
|
}
|
|
|
|
// Persist the final cursor for the next incremental sync.
|
|
if nextCursor != nil {
|
|
if cursorJSON, cerr := nextCursor.ToJSON(); cerr == nil {
|
|
ds.LastSyncCursor = cursorJSON
|
|
}
|
|
}
|
|
ds.LastSyncAt = timePtr(time.Now().UTC())
|
|
|
|
// Surface per-document failures as a partial sync (not silent success), so
|
|
// the sync-log drawer's failure detail explains which docs didn't make it —
|
|
// the visibility gap behind "status normal but not everything syncs"
|
|
// (Tencent/WeKnora#2136). Failed nodes were not advanced in the cursor, so
|
|
// the next run retries them.
|
|
status := types.SyncLogStatusSuccess
|
|
errMsg := ""
|
|
if result.Failed > 0 {
|
|
status = types.SyncLogStatusPartial
|
|
errMsg = fmt.Sprintf("%d document(s) failed to sync", result.Failed)
|
|
}
|
|
s.updateSyncRunResult(ctx, ds, syncLog, result, resultJSON, status, errMsg, wasPaused)
|
|
logger.Infof(ctx, "streaming sync completed: ds=%s created=%d updated=%d deleted=%d skipped=%d failed=%d",
|
|
payload.DataSourceID, result.Created, result.Updated, result.Deleted, result.Skipped, result.Failed)
|
|
return nil
|
|
}
|
|
|
|
func (s *DataSourceService) updateSyncRunResult(
|
|
ctx context.Context,
|
|
ds *types.DataSource,
|
|
syncLog *types.SyncLog,
|
|
result *types.SyncResult,
|
|
resultJSON types.JSON,
|
|
status string,
|
|
errorMessage string,
|
|
wasPaused bool,
|
|
) {
|
|
syncLog.ItemsTotal = result.Total
|
|
syncLog.ItemsCreated = result.Created
|
|
syncLog.ItemsUpdated = result.Updated
|
|
syncLog.ItemsDeleted = result.Deleted
|
|
syncLog.ItemsSkipped = result.Skipped
|
|
syncLog.ItemsFailed = result.Failed
|
|
syncLog.Status = status
|
|
syncLog.FinishedAt = timePtr(time.Now().UTC())
|
|
syncLog.ErrorMessage = errorMessage
|
|
syncLog.Result = resultJSON
|
|
if err := s.syncLogRepo.UpdateResult(ctx, syncLog); err != nil {
|
|
logger.Errorf(ctx, "failed to update sync log: %v", err)
|
|
}
|
|
|
|
if status == types.SyncLogStatusFailed {
|
|
if !wasPaused {
|
|
ds.Status = types.DataSourceStatusError
|
|
}
|
|
} else if wasPaused {
|
|
ds.Status = types.DataSourceStatusPaused
|
|
} else {
|
|
ds.Status = types.DataSourceStatusActive
|
|
}
|
|
ds.ErrorMessage = errorMessage
|
|
ds.LastSyncResult = resultJSON
|
|
if err := s.dsRepo.UpdateSyncState(ctx, ds); err != nil {
|
|
logger.Errorf(ctx, "failed to update data source: %v", err)
|
|
}
|
|
}
|
|
|
|
func allFetchedItemsFailedError(result *types.SyncResult) error {
|
|
if result == nil || result.Total == 0 {
|
|
return nil
|
|
}
|
|
if result.Failed != result.Total || result.Created != 0 || result.Updated != 0 ||
|
|
result.Deleted != 0 || result.Skipped != 0 {
|
|
return nil
|
|
}
|
|
|
|
detail := ""
|
|
if len(result.Errors) > 0 {
|
|
detail = result.Errors[0].Display()
|
|
const maxDetailLen = 500
|
|
if len(detail) > maxDetailLen {
|
|
detail = detail[:maxDetailLen] + "..."
|
|
}
|
|
}
|
|
if detail == "" {
|
|
return fmt.Errorf("all fetched items failed during sync (%d/%d)", result.Failed, result.Total)
|
|
}
|
|
return fmt.Errorf("all fetched items failed during sync (%d/%d): %s", result.Failed, result.Total, detail)
|
|
}
|
|
|
|
// ValidateCredentials tests connectivity using raw credentials without persisting anything.
|
|
func (s *DataSourceService) ValidateCredentials(ctx context.Context, connectorType string, credentials map[string]interface{}) error {
|
|
connector, err := s.connectorRegistry.Get(connectorType)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
config := &types.DataSourceConfig{
|
|
Type: connectorType,
|
|
Credentials: credentials,
|
|
}
|
|
|
|
if err := connector.Validate(ctx, config); err != nil {
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// Helper functions
|
|
|
|
func (s *DataSourceService) validateDataSourceConfig(ctx context.Context, ds *types.DataSource) error {
|
|
connector, err := s.connectorRegistry.Get(ds.Type)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
config, err := ds.ParseConfig()
|
|
if err != nil {
|
|
return datasource.ErrInvalidConfig
|
|
}
|
|
|
|
return connector.Validate(ctx, config)
|
|
}
|
|
|
|
// ingestItem writes a single FetchedItem into the knowledge base.
|
|
// If a knowledge item with the same external_id already exists, it is deleted first (update = delete + re-create).
|
|
//
|
|
// Routing logic:
|
|
// - Has Content bytes → CreateKnowledgeFromFile (走完整的文档解析 pipeline)
|
|
// - Has URL only → CreateKnowledgeFromURL (让 WeKnora 下载并解析)
|
|
//
|
|
// Returns (isUpdate, error) — isUpdate is true when an existing item was replaced.
|
|
func (s *DataSourceService) ingestItem(ctx context.Context, ds *types.DataSource, item *types.FetchedItem, tagIDs []string) (bool, error) {
|
|
channel := ds.Type // e.g. "feishu", "notion"
|
|
|
|
metadata := map[string]string{
|
|
"external_id": item.ExternalID,
|
|
"source_resource_id": item.SourceResourceID,
|
|
"datasource_id": ds.ID,
|
|
}
|
|
for k, v := range item.Metadata {
|
|
metadata[k] = v
|
|
}
|
|
|
|
// Check if a knowledge item with this external_id already exists → delete it first (update)
|
|
isUpdate := false
|
|
if item.ExternalID != "" {
|
|
repo := s.knowledgeService.GetRepository()
|
|
existing, err := repo.FindByMetadataKey(ctx, ds.TenantID, ds.KnowledgeBaseID, "external_id", item.ExternalID)
|
|
if err != nil {
|
|
logger.Warnf(ctx, "failed to check existing knowledge for external_id=%s: %v", item.ExternalID, err)
|
|
// Non-fatal: proceed with creation (may produce duplicate)
|
|
} else if existing != nil {
|
|
logger.Infof(ctx, "found existing knowledge %s for external_id=%s, deleting for update", existing.ID, item.ExternalID)
|
|
if err := s.knowledgeService.DeleteKnowledge(ctx, existing.ID); err != nil {
|
|
logger.Warnf(ctx, "failed to delete existing knowledge %s: %v", existing.ID, err)
|
|
} else {
|
|
isUpdate = true
|
|
}
|
|
}
|
|
}
|
|
|
|
// Case 1: content already fetched → build a FileHeader from bytes and call CreateKnowledgeFromFile
|
|
if len(item.Content) > 0 {
|
|
fh, err := bytesToFileHeader(item.Content, item.FileName)
|
|
if err != nil {
|
|
return isUpdate, fmt.Errorf("build file header: %w", err)
|
|
}
|
|
_, err = s.knowledgeService.CreateKnowledgeFromFile(
|
|
ctx,
|
|
ds.KnowledgeBaseID,
|
|
fh,
|
|
metadata,
|
|
nil, // use KB default for multimodal
|
|
item.FileName, // customFileName — must include extension for file-type validation
|
|
tagIDs, // auto-tag from data source
|
|
channel,
|
|
nil,
|
|
)
|
|
return isUpdate, err
|
|
}
|
|
|
|
// Case 2: only a remote URL — let WeKnora handle downloading and parsing
|
|
if item.URL != "" {
|
|
_, err := s.knowledgeService.CreateKnowledgeFromURL(
|
|
ctx,
|
|
ds.KnowledgeBaseID,
|
|
item.URL,
|
|
item.FileName,
|
|
"", // auto-detect file type
|
|
nil, // use KB default for multimodal
|
|
item.Title,
|
|
tagIDs, // auto-tag from data source
|
|
channel,
|
|
nil,
|
|
)
|
|
return isUpdate, err
|
|
}
|
|
|
|
return isUpdate, fmt.Errorf("item has neither content nor URL")
|
|
}
|
|
|
|
// bytesToFileHeader wraps a []byte into a *multipart.FileHeader so it can be
|
|
// consumed by KnowledgeService.CreateKnowledgeFromFile.
|
|
func bytesToFileHeader(data []byte, filename string) (*multipart.FileHeader, error) {
|
|
var buf bytes.Buffer
|
|
writer := multipart.NewWriter(&buf)
|
|
|
|
// Create a form file part
|
|
partHeader := make(textproto.MIMEHeader)
|
|
partHeader.Set("Content-Disposition", fmt.Sprintf(`form-data; name="file"; filename="%s"`, filename))
|
|
partHeader.Set("Content-Type", "application/octet-stream")
|
|
|
|
part, err := writer.CreatePart(partHeader)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("create multipart part: %w", err)
|
|
}
|
|
|
|
if _, err := part.Write(data); err != nil {
|
|
return nil, fmt.Errorf("write data to part: %w", err)
|
|
}
|
|
|
|
if err := writer.Close(); err != nil {
|
|
return nil, fmt.Errorf("close multipart writer: %w", err)
|
|
}
|
|
|
|
// Parse the multipart data to get a FileHeader
|
|
reader := multipart.NewReader(&buf, writer.Boundary())
|
|
form, err := reader.ReadForm(int64(len(data)) + 1024)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("read multipart form: %w", err)
|
|
}
|
|
|
|
files := form.File["file"]
|
|
if len(files) == 0 {
|
|
return nil, fmt.Errorf("no file in multipart form")
|
|
}
|
|
|
|
return files[0], nil
|
|
}
|
|
|
|
func timePtr(t time.Time) *time.Time {
|
|
utc := t.UTC()
|
|
return &utc
|
|
}
|